Allow the preprocessor to cache the lexed tokens, so that we can do efficient lookahead and backtracking.

1) New public methods added:
  -EnableBacktrackAtThisPos
  -DisableBacktrack
  -Backtrack
  -isBacktrackEnabled

2) LookAhead() implementation is replaced with a more efficient one.
3) LookNext() is removed.

llvm-svn: 54611

1 files changed, 63 insertions, 0 deletions

diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
new file mode 100644
index 00000000000..794e9c4e765
--- /dev/null
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -0,0 +1,63 @@
+//===--- PPCaching.cpp - Handle caching lexed tokens ----------------------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements pieces of the Preprocessor interface that manage the
+// caching of lexed tokens.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Lex/Preprocessor.h"
+using namespace clang;
+
+void Preprocessor::CachingLex(Token &Result) {
+  if (CachedLexPos < CachedTokens.size()) {
+    Result = CachedTokens[CachedLexPos++];
+    return;
+  }
+
+  ExitCachingLexMode();
+  Lex(Result);
+
+  if (!CacheTokens) {
+    // All cached tokens were consumed.
+    CachedTokens.clear();
+    CachedLexPos = 0;
+    return;
+  }
+
+  // We should cache the lexed token.
+
+  EnterCachingLexMode();
+  if (Result.isNot(tok::eof)) {
+    CachedTokens.push_back(Result);
+    ++CachedLexPos;
+  }
+}
+
+void Preprocessor::EnterCachingLexMode() {
+  if (InCachingLexMode())
+    return;
+
+  IncludeMacroStack.push_back(IncludeStackInfo(CurLexer, CurDirLookup,
+                                               CurTokenLexer));
+  CurLexer = 0;
+  CurTokenLexer = 0;
+}
+
+
+const Token &Preprocessor::PeekAhead(unsigned N) {
+  assert(CachedLexPos + N > CachedTokens.size() && "Confused caching.");
+  ExitCachingLexMode();
+  for (unsigned C = CachedLexPos + N - CachedTokens.size(); C > 0; --C) {
+    CachedTokens.push_back(Token());
+    Lex(CachedTokens.back());
+  }
+  EnterCachingLexMode();
+  return CachedTokens.back();
+}