[clangd] Inactive regions support as an extension to semantic highlighting

Differential Revision: https://reviews.llvm.org/D67536

clang-tools-extra/clangd/CollectMacros.h

@@ -30,6 +30,8 @@ struct MainFileMacros {
   // reference to an undefined macro. Store them separately, e.g. for semantic
   // highlighting.
   std::vector<Range> UnknownMacros;
+  // Ranges skipped by the preprocessor due to being inactive.
+  std::vector<Range> SkippedRanges;
 };
 
 /// Collects macro references (e.g. definitions, expansions) in the main file.
@@ -78,6 +80,14 @@ public:
     add(MacroName, MD.getMacroInfo());
   }
 
+  void SourceRangeSkipped(SourceRange R, SourceLocation EndifLoc) override {
+    if (!InMainFile)
+      return;
+    Position Begin = sourceLocToPosition(SM, R.getBegin());
+    Position End = sourceLocToPosition(SM, R.getEnd());
+    Out.SkippedRanges.push_back(Range{Begin, End});
+  }
+
 private:
   void add(const Token &MacroNameTok, const MacroInfo *MI) {
     if (!InMainFile)

clang-tools-extra/clangd/Protocol.cpp

@@ -1063,7 +1063,8 @@ bool operator==(const SemanticHighlightingInformation &Lhs,
 
 llvm::json::Value toJSON(const SemanticHighlightingInformation &Highlighting) {
   return llvm::json::Object{{"line", Highlighting.Line},
-                            {"tokens", Highlighting.Tokens}};
+                            {"tokens", Highlighting.Tokens},
+                            {"isInactive", Highlighting.IsInactive}};
 }
 
 llvm::json::Value toJSON(const SemanticHighlightingParams &Highlighting) {

clang-tools-extra/clangd/Protocol.h

@@ -1209,6 +1209,11 @@ struct SemanticHighlightingInformation {
   int Line = 0;
   /// The base64 encoded string of highlighting tokens.
   std::string Tokens;
+  /// Is the line in an inactive preprocessor branch?
+  /// This is a clangd extension.
+  /// An inactive line can still contain highlighting tokens as well;
+  /// clients should combine line style and token style if possible.
+  bool IsInactive = false;
 };
 bool operator==(const SemanticHighlightingInformation &Lhs,
                 const SemanticHighlightingInformation &Rhs);

clang-tools-extra/clangd/SemanticHighlighting.cpp

@@ -25,6 +25,7 @@
 #include "clang/Basic/SourceManager.h"
 #include "llvm/ADT/None.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/STLExtras.h"
 #include "llvm/Support/Casting.h"
 #include <algorithm>
 
@@ -160,7 +161,7 @@ public:
     Tokens.push_back(HighlightingToken{Kind, *Range});
   }
 
-  std::vector<HighlightingToken> collect() && {
+  std::vector<HighlightingToken> collect(ParsedAST &AST) && {
     // Initializer lists can give duplicates of tokens, therefore all tokens
     // must be deduplicated.
     llvm::sort(Tokens);
@@ -187,6 +188,22 @@ public:
       // the end of the Tokens).
       TokRef = TokRef.drop_front(Conflicting.size());
     }
+    // Add tokens indicating lines skipped by the preprocessor.
+    for (const Range &R : AST.getMacros().SkippedRanges) {
+      // Create one token for each line in the skipped range, so it works
+      // with line-based diffing.
+      assert(R.start.line <= R.end.line);
+      for (int Line = R.start.line; Line < R.end.line; ++Line) {
+        // Don't bother computing the offset for the end of the line, just use
+        // zero. The client will treat this highlighting kind specially, and
+        // highlight the entire line visually (i.e. not just to where the text
+        // on the line ends, but to the end of the screen).
+        NonConflicting.push_back({HighlightingKind::InactiveCode,
+                                  {Position{Line, 0}, Position{Line, 0}}});
+      }
+    }
+    // Re-sort the tokens because that's what the diffing expects.
+    llvm::sort(NonConflicting);
     return NonConflicting;
   }
 
@@ -319,7 +336,7 @@ std::vector<HighlightingToken> getSemanticHighlightings(ParsedAST &AST) {
   for (const auto &M : AST.getMacros().UnknownMacros)
     Builder.addToken({HighlightingKind::Macro, M});
 
-  return std::move(Builder).collect();
+  return std::move(Builder).collect(AST);
 }
 
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
@@ -360,6 +377,8 @@ llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K) {
     return OS << "Primitive";
   case HighlightingKind::Macro:
     return OS << "Macro";
+  case HighlightingKind::InactiveCode:
+    return OS << "InactiveCode";
   }
   llvm_unreachable("invalid HighlightingKind");
 }
@@ -404,8 +423,19 @@ diffHighlightings(ArrayRef<HighlightingToken> New,
        LineNumber = NextLineNumber()) {
     NewLine = takeLine(New, NewLine.end(), LineNumber);
     OldLine = takeLine(Old, OldLine.end(), LineNumber);
-    if (NewLine != OldLine)
-      DiffedLines.push_back({LineNumber, NewLine});
+    if (NewLine != OldLine) {
+      DiffedLines.push_back({LineNumber, NewLine, /*IsInactive=*/false});
+
+      // Turn a HighlightingKind::InactiveCode token into the IsInactive flag.
+      auto &AddedLine = DiffedLines.back();
+      llvm::erase_if(AddedLine.Tokens, [&](const HighlightingToken &T) {
+        if (T.Kind == HighlightingKind::InactiveCode) {
+          AddedLine.IsInactive = true;
+          return true;
+        }
+        return false;
+      });
+    }
   }
 
   return DiffedLines;
@@ -444,7 +474,7 @@ toSemanticHighlightingInformation(llvm::ArrayRef<LineHighlightings> Tokens) {
       write16be(static_cast<int>(Token.Kind), OS);
     }
 
-    Lines.push_back({Line.Line, encodeBase64(LineByteTokens)});
+    Lines.push_back({Line.Line, encodeBase64(LineByteTokens), Line.IsInactive});
   }
 
   return Lines;
@@ -489,6 +519,8 @@ llvm::StringRef toTextMateScope(HighlightingKind Kind) {
     return "storage.type.primitive.cpp";
   case HighlightingKind::Macro:
     return "entity.name.function.preprocessor.cpp";
+  case HighlightingKind::InactiveCode:
+    return "meta.disabled";
   }
   llvm_unreachable("unhandled HighlightingKind");
 }

clang-tools-extra/clangd/SemanticHighlighting.h

@@ -44,7 +44,11 @@ enum class HighlightingKind {
   Primitive,
   Macro,
 
-  LastKind = Macro
+  // This one is different from the other kinds as it's a line style
+  // rather than a token style.
+  InactiveCode,
+
+  LastKind = InactiveCode
 };
 llvm::raw_ostream &operator<<(llvm::raw_ostream &OS, HighlightingKind K);
 
@@ -61,6 +65,7 @@ bool operator<(const HighlightingToken &L, const HighlightingToken &R);
 struct LineHighlightings {
   int Line;
   std::vector<HighlightingToken> Tokens;
+  bool IsInactive;
 };
 
 bool operator==(const LineHighlightings &L, const LineHighlightings &R);

clang-tools-extra/clangd/test/semantic-highlighting.test

@@ -57,6 +57,9 @@
 # CHECK-NEXT:          ],
 # CHECK-NEXT:          [
 # CHECK-NEXT:            "entity.name.function.preprocessor.cpp"
+# CHECK-NEXT:          ],
+# CHECK-NEXT:          [
+# CHECK-NEXT:            "meta.disabled"
 # CHECK-NEXT:          ]
 # CHECK-NEXT:        ]
 # CHECK-NEXT:      },
@@ -66,6 +69,7 @@
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
+# CHECK-NEXT:        "isInactive": false,
 # CHECK-NEXT:        "line": 0,
 # CHECK-NEXT:        "tokens": "AAAABAABAAA="
 # CHECK-NEXT:      }
@@ -81,10 +85,12 @@
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
+# CHECK-NEXT:        "isInactive": false,
 # CHECK-NEXT:        "line": 0,
 # CHECK-NEXT:        "tokens": "AAAABAABAAA="
 # CHECK-NEXT:      }
 # CHECK-NEXT:      {
+# CHECK-NEXT:        "isInactive": false,
 # CHECK-NEXT:        "line": 1,
 # CHECK-NEXT:        "tokens": "AAAABAABAAA="
 # CHECK-NEXT:      }
@@ -100,6 +106,7 @@
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
+# CHECK-NEXT:        "isInactive": false,
 # CHECK-NEXT:        "line": 1,
 # CHECK-NEXT:        "tokens": "AAAABAABAAA="
 # CHECK-NEXT:      }
@@ -115,6 +122,7 @@
 # CHECK-NEXT:  "params": {
 # CHECK-NEXT:    "lines": [
 # CHECK-NEXT:      {
+# CHECK-NEXT:        "isInactive": false,
 # CHECK-NEXT:        "line": 1,
 # CHECK-NEXT:        "tokens": ""
 # CHECK-NEXT:      }

clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp

@@ -140,7 +140,7 @@ void checkDiffedHighlights(llvm::StringRef OldCode, llvm::StringRef NewCode) {
   }
   for (auto &LineTokens : ExpectedLines)
     ExpectedLinePairHighlighting.push_back(
-        {LineTokens.first, LineTokens.second});
+        {LineTokens.first, LineTokens.second, /*IsInactive = */ false});
 
   std::vector<LineHighlightings> ActualDiffed =
       diffHighlightings(NewTokens, OldTokens);
@@ -493,11 +493,11 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
 
       #define $Macro[[test]]
       #undef $Macro[[test]]
-      #ifdef $Macro[[test]]
-      #endif
+$InactiveCode[[]]      #ifdef $Macro[[test]]
+$InactiveCode[[]]      #endif
 
-      #if defined($Macro[[test]])
-      #endif
+$InactiveCode[[]]      #if defined($Macro[[test]])
+$InactiveCode[[]]      #endif
     )cpp",
       R"cpp(
       struct $Class[[S]] {
@@ -598,6 +598,33 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
         $Class[[Foo]]<$TemplateParameter[[TT]], $TemplateParameter[[TTs]]...>
           *$Field[[t]];
       }
+    )cpp",
+      // Inactive code highlighting
+      R"cpp(
+      // Code in the preamble.
+      // Inactive lines get an empty InactiveCode token at the beginning.
+$InactiveCode[[]]      #ifdef $Macro[[test]]
+$InactiveCode[[]]      #endif
+
+      // A declaration to cause the preamble to end.
+      int $Variable[[EndPreamble]];
+
+      // Code after the preamble.
+      // Code inside inactive blocks does not get regular highlightings
+      // because it's not part of the AST.
+$InactiveCode[[]]      #ifdef $Macro[[test]]
+$InactiveCode[[]]      int Inactive2;
+$InactiveCode[[]]      #endif
+
+      #ifndef $Macro[[test]]
+      int $Variable[[Active1]];
+      #endif
+
+$InactiveCode[[]]      #ifdef $Macro[[test]]
+$InactiveCode[[]]      int Inactive3;
+$InactiveCode[[]]      #else
+      int $Variable[[Active2]];
+      #endif
     )cpp"};
   for (const auto &TestCase : TestCases) {
     checkHighlightings(TestCase);
@@ -665,10 +692,12 @@ TEST(SemanticHighlighting, toSemanticHighlightingInformation) {
        {{HighlightingKind::Variable,
          Range{CreatePosition(3, 8), CreatePosition(3, 12)}},
         {HighlightingKind::Function,
-         Range{CreatePosition(3, 4), CreatePosition(3, 7)}}}},
+         Range{CreatePosition(3, 4), CreatePosition(3, 7)}}},
+       /* IsInactive = */ false},
       {1,
        {{HighlightingKind::Variable,
-         Range{CreatePosition(1, 1), CreatePosition(1, 5)}}}}};
+         Range{CreatePosition(1, 1), CreatePosition(1, 5)}}},
+       /* IsInactive = */ true}};
   std::vector<SemanticHighlightingInformation> ActualResults =
       toSemanticHighlightingInformation(Tokens);
   std::vector<SemanticHighlightingInformation> ExpectedResults = {