[clangd] Added highlighting for tokens that are macro arguments.

Summary:
Adds semantic highlighting for tokens that are a macro argument.
Example:
```
D_V(SomeVar);
```
The "SomeVar" inside the macro is highlighted as a variable now.

Tokens that are in a macro body expansion are ignored in this patch for three reasons.
* The spelling loc is inside the macro "definition" meaning it would highlight inside the macro definition (could probably easily be fixed by using getExpansionLoc instead of getSpellingLoc?)
* If wanting to highlight the macro definition this could create duplicate tokens. And if the tokens are of different types there would be conflicts (tokens in the same range but with different types). Say a macro defines some name and both a variable declaration and a function use this, there would be two tokens in the macro definition but one with Kind "Variable" and the other with Kind "Function".
* Thirdly, macro body expansions could come from a file that is not the main file (easily fixed, just check that the Loc is in the main file and not even a problem if we wanted to highlight the actual macro "invocation")

Reviewers: hokein, sammccall, ilya-biryukov

Subscribers: MaskRay, jkorous, arphaman, kadircet, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D64741

llvm-svn: 369275

1 files changed, 63 insertions, 4 deletions

diff --git a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
index d2d874c4b4a..903f719132a 100644
--- a/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
+++ b/clang-tools-extra/clangd/unittests/SemanticHighlightingTests.cpp
@@ -16,10 +16,6 @@
 
 namespace clang {
 namespace clangd {
-  void PrintTo(const HighlightingToken &T, ::std::ostream *OS) {
-  *OS << "(" << T.R.start.line << ", " << T.R.start.character << ") -> (" << T.R.end.line << ", " << T.R.end.character << "): " << (int)T.Kind;
-}
-
 namespace {
 
 std::vector<HighlightingToken>
@@ -380,6 +376,56 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
         $Class[[G]]<$Class[[F]], &$Class[[F]]::$Method[[f]]> $Variable[[GG]];
         $Variable[[GG]].$Method[[foo]](&$Variable[[FF]]);
         $Class[[A]]<$Function[[foo]]> $Variable[[AA]];
+    )cpp",
+    // Tokens that share a source range but have conflicting Kinds are not
+    // highlighted.
+    R"cpp(
+      #define DEF_MULTIPLE(X) namespace X { class X { int X; }; }
+      #define DEF_CLASS(T) class T {};
+      DEF_MULTIPLE(XYZ);
+      DEF_MULTIPLE(XYZW);
+      DEF_CLASS($Class[[A]])
+      #define MACRO_CONCAT(X, V, T) T foo##X = V
+      #define DEF_VAR(X, V) int X = V
+      #define DEF_VAR_T(T, X, V) T X = V
+      #define DEF_VAR_REV(V, X) DEF_VAR(X, V)
+      #define CPY(X) X
+      #define DEF_VAR_TYPE(X, Y) X Y
+      #define SOME_NAME variable
+      #define SOME_NAME_SET variable2 = 123
+      #define INC_VAR(X) X += 2
+      $Primitive[[void]] $Function[[foo]]() {
+        DEF_VAR($Variable[[X]],  123);
+        DEF_VAR_REV(908, $Variable[[XY]]);
+        $Primitive[[int]] CPY( $Variable[[XX]] );
+        DEF_VAR_TYPE($Class[[A]], $Variable[[AA]]);
+        $Primitive[[double]] SOME_NAME;
+        $Primitive[[int]] SOME_NAME_SET;
+        $Variable[[variable]] = 20.1;
+        MACRO_CONCAT(var, 2, $Primitive[[float]]);
+        DEF_VAR_T($Class[[A]], CPY(CPY($Variable[[Nested]])),
+              CPY($Class[[A]]()));
+        INC_VAR($Variable[[variable]]);
+      }
+      $Primitive[[void]] SOME_NAME();
+      DEF_VAR($Variable[[XYZ]], 567);
+      DEF_VAR_REV(756, $Variable[[AB]]);
+
+      #define CALL_FN(F) F();
+      #define DEF_FN(F) void F ()
+      DEF_FN($Function[[g]]) {
+        CALL_FN($Function[[foo]]);
+      }
+    )cpp",
+    R"cpp(
+      #define fail(expr) expr
+      #define assert(COND) if (!(COND)) { fail("assertion failed" #COND); }
+      $Primitive[[int]] $Variable[[x]];
+      $Primitive[[int]] $Variable[[y]];
+      $Primitive[[int]] $Function[[f]]();
+      $Primitive[[void]] $Function[[foo]]() {
+        assert($Variable[[x]] != $Variable[[y]]);
+        assert($Variable[[x]] != $Function[[f]]());
       }
     )cpp"};
   for (const auto &TestCase : TestCases) {
@@ -396,6 +442,19 @@ TEST(SemanticHighlighting, GetsCorrectTokens) {
     int someMethod();
     void otherMethod();
   )cpp"}});
+
+  // A separate test for macros in headers.
+  checkHighlightings(R"cpp(
+    #include "imp.h"
+    DEFINE_Y
+    DXYZ_Y(A);
+  )cpp",
+                     {{"imp.h", R"cpp(
+    #define DXYZ(X) class X {};
+    #define DXYZ_Y(Y) DXYZ(x##Y)
+    #define DEFINE(X) int X;
+    #define DEFINE_Y DEFINE(Y)
+  )cpp"}});
 }
 
 TEST(SemanticHighlighting, GeneratesHighlightsWhenFileChange) {