2 files changed, 41 insertions, 0 deletions

diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index 45bdce32062..f5e8cdc25d3 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -35,6 +35,29 @@ void Preprocessor::CommitBacktrackedTokens() {
   BacktrackPositions.pop_back();
 }
 
+Preprocessor::CachedTokensRange Preprocessor::LastCachedTokenRange() {
+  assert(isBacktrackEnabled());
+  auto PrevCachedLexPos = BacktrackPositions.back();
+  return CachedTokensRange{PrevCachedLexPos, CachedLexPos};
+}
+
+void Preprocessor::EraseCachedTokens(CachedTokensRange TokenRange) {
+  assert(TokenRange.Begin <= TokenRange.End);
+  if (CachedLexPos == TokenRange.Begin && TokenRange.Begin != TokenRange.End) {
+    // We have backtracked to the start of the token range as we want to consume
+    // them again. Erase the tokens only after consuming then.
+    assert(!CachedTokenRangeToErase);
+    CachedTokenRangeToErase = TokenRange;
+    return;
+  }
+  // The cached tokens were committed, so they should be erased now.
+  assert(TokenRange.End == CachedLexPos);
+  CachedTokens.erase(CachedTokens.begin() + TokenRange.Begin,
+                     CachedTokens.begin() + TokenRange.End);
+  CachedLexPos = TokenRange.Begin;
+  ExitCachingLexMode();
+}
+
 // Make Preprocessor re-lex the tokens that were lexed since
 // EnableBacktrackAtThisPos() was previously called.
 void Preprocessor::Backtrack() {
@@ -51,6 +74,13 @@ void Preprocessor::CachingLex(Token &Result) {
 
   if (CachedLexPos < CachedTokens.size()) {
     Result = CachedTokens[CachedLexPos++];
+    // Erase the some of the cached tokens after they are consumed when
+    // asked to do so.
+    if (CachedTokenRangeToErase &&
+        CachedTokenRangeToErase->End == CachedLexPos) {
+      EraseCachedTokens(*CachedTokenRangeToErase);
+      CachedTokenRangeToErase = None;
+    }
     return;
   }
 
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index 100da514144..f81eaa31e9e 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -160,12 +160,23 @@ public:
 
   ~LexingFor_PragmaRAII() {
     if (InMacroArgPreExpansion) {
+      // When committing/backtracking the cached pragma tokens in a macro
+      // argument pre-expansion we want to ensure that either the tokens which
+      // have been committed will be removed from the cache or that the tokens
+      // over which we just backtracked won't remain in the cache after they're
+      // consumed and that the caching will stop after consuming them.
+      // Otherwise the caching will interfere with the way macro expansion
+      // works, because we will continue to cache tokens after consuming the
+      // backtracked tokens, which shouldn't happen when we're dealing with
+      // macro argument pre-expansion.
+      auto CachedTokenRange = PP.LastCachedTokenRange();
       if (Failed) {
         PP.CommitBacktrackedTokens();
       } else {
         PP.Backtrack();
         OutTok = PragmaTok;
       }
+      PP.EraseCachedTokens(CachedTokenRange);
     }
   }