[Syntax] Merge overlapping top-level macros in TokenBuffer

Summary:
Our previous definition of "top-level" was too informal, and didn't
allow for overlapping macros that each directly produce expanded tokens.
See D77507 for previous discussion.

Fixes http://bugs.llvm.org/show_bug.cgi?id=45428

Reviewers: kadircet, vabridgers

Subscribers: cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D77615

(cherry picked from commit d66afd6dde542dc373f87e07fe764c071fe20d76)

1 files changed, 21 insertions, 4 deletions

diff --git a/clang/unittests/Tooling/Syntax/TokensTest.cpp b/clang/unittests/Tooling/Syntax/TokensTest.cpp
index b2ad3859104..f1e2dda963c 100644
--- a/clang/unittests/Tooling/Syntax/TokensTest.cpp
+++ b/clang/unittests/Tooling/Syntax/TokensTest.cpp
@@ -470,11 +470,28 @@ file './input.cpp'
   mappings:
     ['#'_0, 'int'_7) => ['int'_0, 'int'_0)
     ['FOO'_10, '<eof>'_11) => ['10'_3, '<eof>'_7)
-)"}};
+)"},
+      {R"cpp(
+         #define NUM 42
+         #define ID(a) a
+         #define M 1 + ID
+         M(NUM)
+       )cpp",
+       R"(expanded tokens:
+  1 + 42
+file './input.cpp'
+  spelled tokens:
+    # define NUM 42 # define ID ( a ) a # define M 1 + ID M ( NUM )
+  mappings:
+    ['#'_0, 'M'_17) => ['1'_0, '1'_0)
+    ['M'_17, '<eof>'_21) => ['1'_0, '<eof>'_3)
+)"},
+  };
 
-  for (auto &Test : TestCases)
-    EXPECT_EQ(Test.second, collectAndDump(Test.first))
-        << collectAndDump(Test.first);
+  for (auto &Test : TestCases) {
+    std::string Dump = collectAndDump(Test.first);
+    EXPECT_EQ(Test.second, Dump) << Dump;
+  }
 }
 
 TEST_F(TokenCollectorTest, SpecialTokens) {