[Lex] Allow to consume tokens while preprocessing

Summary:
By adding a hook to consume all tokens produced by the preprocessor.
The intention of this change is to make it possible to consume the
expanded tokens without re-runnig the preprocessor with minimal changes
to the preprocessor and minimal performance penalty when preprocessing
without recording the tokens.

The added hook is very low-level and reconstructing the expanded token
stream requires more work in the client code, the actual algorithm to
collect the tokens using this hook can be found in the follow-up change.

Reviewers: rsmith

Reviewed By: rsmith

Subscribers: eraman, nemanjai, kbarton, jsji, riccibruno, cfe-commits

Tags: #clang

Differential Revision: https://reviews.llvm.org/D59885

llvm-svn: 361007

1 files changed, 8 insertions, 5 deletions

diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index 5f547d9f1cc..cfbb547fae3 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -154,10 +154,11 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
 /// must be freed.
 ///
 void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
-                                    bool DisableMacroExpansion,
-                                    bool OwnsTokens) {
+                                    bool DisableMacroExpansion, bool OwnsTokens,
+                                    bool IsReinject) {
   if (CurLexerKind == CLK_CachingLexer) {
     if (CachedLexPos < CachedTokens.size()) {
+      assert(IsReinject && "new tokens in the middle of cached stream");
       // We're entering tokens into the middle of our cached token stream. We
       // can't represent that, so just insert the tokens into the buffer.
       CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
@@ -170,7 +171,8 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
     // New tokens are at the end of the cached token sequnece; insert the
     // token stream underneath the caching lexer.
     ExitCachingLexMode();
-    EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+    EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
+                     IsReinject);
     EnterCachingLexMode();
     return;
   }
@@ -179,10 +181,11 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
   std::unique_ptr<TokenLexer> TokLexer;
   if (NumCachedTokenLexers == 0) {
     TokLexer = llvm::make_unique<TokenLexer>(
-        Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
+        Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this);
   } else {
     TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
-    TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+    TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
+                   IsReinject);
   }
 
   // Save our current state.