[clangd] Implement configs to stop clangd produce a certain semantic tokens

This patch introduces the following configurations to .clangd:

```
SemanticTokens:
    DisabledKinds: [ ... ]
    DisabledModifiers: [ ... ]
```

Based on the config, clangd would stop producing a certain type of semantic tokens from the source file.

Fixes https://github.com/clangd/clangd/discussions/1598

Reviewed By: nridge

Differential Revision: https://reviews.llvm.org/D148489

clang-tools-extra/clangd/Config.h

@@ -150,6 +150,13 @@ struct Config {
     // Limit the length of type names in inlay hints. (0 means no limit)
     uint32_t TypeNameLimit = 32;
   } InlayHints;
+
+  struct {
+    /// Controls highlighting kinds that are disabled.
+    std::vector<std::string> DisabledKinds;
+    /// Controls highlighting modifiers that are disabled.
+    std::vector<std::string> DisabledModifiers;
+  } SemanticTokens;
 };
 
 } // namespace clangd

clang-tools-extra/clangd/ConfigCompile.cpp

@@ -196,6 +196,7 @@ struct FragmentCompiler {
     compile(std::move(F.Completion));
     compile(std::move(F.Hover));
     compile(std::move(F.InlayHints));
+    compile(std::move(F.SemanticTokens));
     compile(std::move(F.Style));
   }
 
@@ -618,6 +619,37 @@ struct FragmentCompiler {
           });
   }
 
+  void compile(Fragment::SemanticTokensBlock &&F) {
+    if (!F.DisabledKinds.empty()) {
+      std::vector<std::string> DisabledKinds;
+      for (auto &Kind : F.DisabledKinds)
+        DisabledKinds.push_back(std::move(*Kind));
+
+      Out.Apply.push_back(
+          [DisabledKinds(std::move(DisabledKinds))](const Params &, Config &C) {
+            for (auto &Kind : DisabledKinds) {
+              auto It = llvm::find(C.SemanticTokens.DisabledKinds, Kind);
+              if (It == C.SemanticTokens.DisabledKinds.end())
+                C.SemanticTokens.DisabledKinds.push_back(std::move(Kind));
+            }
+          });
+    }
+    if (!F.DisabledModifiers.empty()) {
+      std::vector<std::string> DisabledModifiers;
+      for (auto &Kind : F.DisabledModifiers)
+        DisabledModifiers.push_back(std::move(*Kind));
+
+      Out.Apply.push_back([DisabledModifiers(std::move(DisabledModifiers))](
+                              const Params &, Config &C) {
+        for (auto &Kind : DisabledModifiers) {
+          auto It = llvm::find(C.SemanticTokens.DisabledModifiers, Kind);
+          if (It == C.SemanticTokens.DisabledModifiers.end())
+            C.SemanticTokens.DisabledModifiers.push_back(std::move(Kind));
+        }
+      });
+    }
+  }
+
   constexpr static llvm::SourceMgr::DiagKind Error = llvm::SourceMgr::DK_Error;
   constexpr static llvm::SourceMgr::DiagKind Warning =
       llvm::SourceMgr::DK_Warning;

clang-tools-extra/clangd/ConfigFragment.h

@@ -235,7 +235,6 @@ struct Fragment {
     /// - std::nullopt
     std::optional<Located<std::string>> UnusedIncludes;
 
-
     /// Enable emitting diagnostics using stale preambles.
     std::optional<Located<bool>> AllowStalePreamble;
 
@@ -326,6 +325,15 @@ struct Fragment {
     std::optional<Located<uint32_t>> TypeNameLimit;
   };
   InlayHintsBlock InlayHints;
+
+  /// Configures semantic tokens that are produced by clangd.
+  struct SemanticTokensBlock {
+    /// Disables clangd to produce semantic tokens for the given kinds.
+    std::vector<Located<std::string>> DisabledKinds;
+    /// Disables clangd to assign semantic tokens with the given modifiers.
+    std::vector<Located<std::string>> DisabledModifiers;
+  };
+  SemanticTokensBlock SemanticTokens;
 };
 
 } // namespace config

clang-tools-extra/clangd/ConfigYAML.cpp

@@ -68,6 +68,7 @@ public:
     Dict.handle("Completion", [&](Node &N) { parse(F.Completion, N); });
     Dict.handle("Hover", [&](Node &N) { parse(F.Hover, N); });
     Dict.handle("InlayHints", [&](Node &N) { parse(F.InlayHints, N); });
+    Dict.handle("SemanticTokens", [&](Node &N) { parse(F.SemanticTokens, N); });
     Dict.parse(N);
     return !(N.failed() || HadError);
   }
@@ -261,6 +262,19 @@ private:
     Dict.parse(N);
   }
 
+  void parse(Fragment::SemanticTokensBlock &F, Node &N) {
+    DictParser Dict("SemanticTokens", this);
+    Dict.handle("DisabledKinds", [&](Node &N) {
+      if (auto Values = scalarValues(N))
+        F.DisabledKinds = std::move(*Values);
+    });
+    Dict.handle("DisabledModifiers", [&](Node &N) {
+      if (auto Values = scalarValues(N))
+        F.DisabledModifiers = std::move(*Values);
+    });
+    Dict.parse(N);
+  }
+
   // Helper for parsing mapping nodes (dictionaries).
   // We don't use YamlIO as we want to control over unknown keys.
   class DictParser {

clang-tools-extra/clangd/SemanticHighlighting.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "SemanticHighlighting.h"
+#include "Config.h"
 #include "FindTarget.h"
 #include "HeuristicResolver.h"
 #include "ParsedAST.h"
@@ -354,13 +355,57 @@ resolveConflict(ArrayRef<HighlightingToken> Tokens) {
   return Winner;
 }
 
+/// Filter to remove particular kinds of highlighting tokens and modifiers from
+/// the output.
+class HighlightingFilter {
+public:
+  HighlightingFilter() {
+    for (auto &Active : ActiveKindLookup)
+      Active = true;
+
+    ActiveModifiersMask = ~0;
+  }
+
+  void disableKind(HighlightingKind Kind) {
+    ActiveKindLookup[static_cast<size_t>(Kind)] = false;
+  }
+
+  void disableModifier(HighlightingModifier Modifier) {
+    ActiveModifiersMask &= ~(1 << static_cast<uint32_t>(Modifier));
+  }
+
+  bool isHighlightKindActive(HighlightingKind Kind) const {
+    return ActiveKindLookup[static_cast<size_t>(Kind)];
+  }
+
+  uint32_t maskModifiers(uint32_t Modifiers) const {
+    return Modifiers & ActiveModifiersMask;
+  }
+
+  static HighlightingFilter fromCurrentConfig() {
+    const Config &C = Config::current();
+    HighlightingFilter Filter;
+    for (const auto &Kind : C.SemanticTokens.DisabledKinds)
+      if (auto K = highlightingKindFromString(Kind))
+        Filter.disableKind(*K);
+    for (const auto &Modifier : C.SemanticTokens.DisabledModifiers)
+      if (auto M = highlightingModifierFromString(Modifier))
+        Filter.disableModifier(*M);
+
+    return Filter;
+  }
+
+private:
+  bool ActiveKindLookup[static_cast<size_t>(HighlightingKind::LastKind) + 1];
+  uint32_t ActiveModifiersMask;
+};
+
 /// Consumes source locations and maps them to text ranges for highlightings.
 class HighlightingsBuilder {
 public:
-  HighlightingsBuilder(const ParsedAST &AST, bool IncludeInactiveRegionTokens)
+  HighlightingsBuilder(const ParsedAST &AST, const HighlightingFilter &Filter)
       : TB(AST.getTokens()), SourceMgr(AST.getSourceManager()),
-        LangOpts(AST.getLangOpts()),
-        IncludeInactiveRegionTokens(IncludeInactiveRegionTokens) {}
+        LangOpts(AST.getLangOpts()), Filter(Filter) {}
 
   HighlightingToken &addToken(SourceLocation Loc, HighlightingKind Kind) {
     auto Range = getRangeForSourceLocation(Loc);
@@ -412,6 +457,9 @@ public:
   }
 
   HighlightingToken &addToken(Range R, HighlightingKind Kind) {
+    if (!Filter.isHighlightKindActive(Kind))
+      return InvalidHighlightingToken;
+
     HighlightingToken HT;
     HT.R = std::move(R);
     HT.Kind = Kind;
@@ -452,6 +500,7 @@ public:
           }
         }
 
+        Resolved->Modifiers = Filter.maskModifiers(Resolved->Modifiers);
         NonConflicting.push_back(*Resolved);
       }
       // TokRef[Conflicting.size()] is the next token with a different range (or
@@ -459,7 +508,7 @@ public:
       TokRef = TokRef.drop_front(Conflicting.size());
     }
 
-    if (!IncludeInactiveRegionTokens)
+    if (!Filter.isHighlightKindActive(HighlightingKind::InactiveCode))
       return NonConflicting;
 
     const auto &SM = AST.getSourceManager();
@@ -535,7 +584,7 @@ private:
   const syntax::TokenBuffer &TB;
   const SourceManager &SourceMgr;
   const LangOptions &LangOpts;
-  bool IncludeInactiveRegionTokens;
+  HighlightingFilter Filter;
   std::vector<HighlightingToken> Tokens;
   std::map<Range, llvm::SmallVector<HighlightingModifier, 1>> ExtraModifiers;
   const HeuristicResolver *Resolver = nullptr;
@@ -1104,8 +1153,11 @@ private:
 std::vector<HighlightingToken>
 getSemanticHighlightings(ParsedAST &AST, bool IncludeInactiveRegionTokens) {
   auto &C = AST.getASTContext();
+  HighlightingFilter Filter = HighlightingFilter::fromCurrentConfig();
+  if (!IncludeInactiveRegionTokens)
+    Filter.disableKind(HighlightingKind::InactiveCode);
   // Add highlightings for AST nodes.
-  HighlightingsBuilder Builder(AST, IncludeInactiveRegionTokens);
+  HighlightingsBuilder Builder(AST, Filter);
   // Highlight 'decltype' and 'auto' as their underlying types.
   CollectExtraHighlightings(Builder).TraverseAST(C);
   // Highlight all decls and references coming from the AST.
@@ -1224,6 +1276,38 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
   }
   llvm_unreachable("invalid HighlightingKind");
 }
+std::optional<HighlightingKind>
+highlightingKindFromString(llvm::StringRef Name) {
+  static llvm::StringMap<HighlightingKind> Lookup = {
+      {"Variable", HighlightingKind::Variable},
+      {"LocalVariable", HighlightingKind::LocalVariable},
+      {"Parameter", HighlightingKind::Parameter},
+      {"Function", HighlightingKind::Function},
+      {"Method", HighlightingKind::Method},
+      {"StaticMethod", HighlightingKind::StaticMethod},
+      {"Field", HighlightingKind::Field},
+      {"StaticField", HighlightingKind::StaticField},
+      {"Class", HighlightingKind::Class},
+      {"Interface", HighlightingKind::Interface},
+      {"Enum", HighlightingKind::Enum},
+      {"EnumConstant", HighlightingKind::EnumConstant},
+      {"Typedef", HighlightingKind::Typedef},
+      {"Type", HighlightingKind::Type},
+      {"Unknown", HighlightingKind::Unknown},
+      {"Namespace", HighlightingKind::Namespace},
+      {"TemplateParameter", HighlightingKind::TemplateParameter},
+      {"Concept", HighlightingKind::Concept},
+      {"Primitive", HighlightingKind::Primitive},
+      {"Macro", HighlightingKind::Macro},
+      {"Modifier", HighlightingKind::Modifier},
+      {"Operator", HighlightingKind::Operator},
+      {"Bracket", HighlightingKind::Bracket},
+      {"InactiveCode", HighlightingKind::InactiveCode},
+  };
+
+  auto It = Lookup.find(Name);
+  return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
+}
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
   switch (K) {
   case HighlightingModifier::Declaration:
@@ -1236,6 +1320,33 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K) {
     return OS << toSemanticTokenModifier(K);
   }
 }
+std::optional<HighlightingModifier>
+highlightingModifierFromString(llvm::StringRef Name) {
+  static llvm::StringMap<HighlightingModifier> Lookup = {
+      {"Declaration", HighlightingModifier::Declaration},
+      {"Definition", HighlightingModifier::Definition},
+      {"Deprecated", HighlightingModifier::Deprecated},
+      {"Deduced", HighlightingModifier::Deduced},
+      {"Readonly", HighlightingModifier::Readonly},
+      {"Static", HighlightingModifier::Static},
+      {"Abstract", HighlightingModifier::Abstract},
+      {"Virtual", HighlightingModifier::Virtual},
+      {"DependentName", HighlightingModifier::DependentName},
+      {"DefaultLibrary", HighlightingModifier::DefaultLibrary},
+      {"UsedAsMutableReference", HighlightingModifier::UsedAsMutableReference},
+      {"UsedAsMutablePointer", HighlightingModifier::UsedAsMutablePointer},
+      {"ConstructorOrDestructor",
+       HighlightingModifier::ConstructorOrDestructor},
+      {"UserDefined", HighlightingModifier::UserDefined},
+      {"FunctionScope", HighlightingModifier::FunctionScope},
+      {"ClassScope", HighlightingModifier::ClassScope},
+      {"FileScope", HighlightingModifier::FileScope},
+      {"GlobalScope", HighlightingModifier::GlobalScope},
+  };
+
+  auto It = Lookup.find(Name);
+  return It != Lookup.end() ? std::make_optional(It->getValue()) : std::nullopt;
+}
 
 bool operator==(const HighlightingToken &L, const HighlightingToken &R) {
   return std::tie(L.R, L.Kind, L.Modifiers) ==

clang-tools-extra/clangd/SemanticHighlighting.h

@@ -61,6 +61,8 @@ enum class HighlightingKind {
 };
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K);
+std::optional<HighlightingKind>
+highlightingKindFromString(llvm::StringRef Name);
 
 enum class HighlightingModifier {
   Declaration,
@@ -88,6 +90,8 @@ enum class HighlightingModifier {
 static_assert(static_cast<unsigned>(HighlightingModifier::LastModifier) < 32,
               "Increase width of modifiers bitfield!");
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingModifier K);
+std::optional<HighlightingModifier>
+highlightingModifierFromString(llvm::StringRef Name);
 
 // Contains all information needed for the highlighting a token.
 struct HighlightingToken {

clang-tools-extra/clangd/unittests/ConfigYAMLTests.cpp

@@ -246,6 +246,23 @@ InlayHints:
   EXPECT_EQ(Results[0].InlayHints.DeducedTypes, std::nullopt);
 }
 
+TEST(ParseYAML, SemanticTokens) {
+  CapturedDiags Diags;
+  Annotations YAML(R"yaml(
+SemanticTokens:
+  DisabledKinds: [ Operator, InactiveCode]
+  DisabledModifiers: Readonly
+  )yaml");
+  auto Results =
+      Fragment::parseYAML(YAML.code(), "config.yaml", Diags.callback());
+  ASSERT_THAT(Diags.Diagnostics, IsEmpty());
+  ASSERT_EQ(Results.size(), 1u);
+  EXPECT_THAT(Results[0].SemanticTokens.DisabledKinds,
+              ElementsAre(val("Operator"), val("InactiveCode")));
+  EXPECT_THAT(Results[0].SemanticTokens.DisabledModifiers,
+              ElementsAre(val("Readonly")));
+}
+
 TEST(ParseYAML, IncludesIgnoreHeader) {
   CapturedDiags Diags;
   Annotations YAML(R"yaml(

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

@@ -7,6 +7,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "Annotations.h"
+#include "Config.h"
 #include "Protocol.h"
 #include "SemanticHighlighting.h"
 #include "SourceCode.h"
@@ -1260,6 +1261,17 @@ o]] [[bar]])cpp";
   EXPECT_EQ(Toks[3].deltaStart, 2u);
   EXPECT_EQ(Toks[3].length, 3u);
 }
+
+TEST(SemanticHighlighting, WithHighlightingFilter) {
+  llvm::StringRef AnnotatedCode = R"cpp(
+int *$Variable[[x]] = new int;
+)cpp";
+  Config Cfg;
+  Cfg.SemanticTokens.DisabledKinds = {"Operator"};
+  Cfg.SemanticTokens.DisabledModifiers = {"Declaration", "Definition"};
+  WithContextValue WithCfg(Config::Key, std::move(Cfg));
+  checkHighlightings(AnnotatedCode, {}, ~ScopeModifierMask);
+}
 } // namespace
 } // namespace clangd
 } // namespace clang