we do in fact have to cache the EOF token returned by the preprocessor.

In the case of backtracking, the cached token lexer will be the only 
lexer on the stack, without this the token stack will be empty and EOF
won't be returned.

This fixes PR7072.

llvm-svn: 108124

1 files changed, 3 insertions, 6 deletions

diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index 6aeb6fa3a2f..20e3b3dd9d8 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -60,13 +60,10 @@ void Preprocessor::CachingLex(Token &Result) {
     return;
   }
 
-  // We should cache the lexed token.
-
+  // Cache the lexed token.
   EnterCachingLexMode();
-  if (Result.isNot(tok::eof)) {
-    CachedTokens.push_back(Result);
-    ++CachedLexPos;
-  }
+  CachedTokens.push_back(Result);
+  ++CachedLexPos;
 }
 
 void Preprocessor::EnterCachingLexMode() {