Make end-of-file handling much less recursive. This reduces the worst case

stack depth sampled by shark from ~34 to ~17 frames when preprocessing <iostream>.

llvm-svn: 38726

5 files changed, 37 insertions, 21 deletions

diff --git a/clang/Lex/Lexer.cpp b/clang/Lex/Lexer.cpp
index 69c3a878dd1..7f50f350e50 100644
--- a/clang/Lex/Lexer.cpp
+++ b/clang/Lex/Lexer.cpp
@@ -889,7 +889,9 @@ std::string Lexer::ReadToEndOfLine() {
 
 /// LexEndOfFile - CurPtr points to the end of this file.  Handle this
 /// condition, reporting diagnostics and handling other edge cases as required.
-void Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) {
+/// This returns true if Result contains a token, false if PP.Lex should be
+/// called again.
+bool Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) {
   // If we hit the end of the file while parsing a preprocessor directive,
   // end the preprocessor directive first.  The next token returned will
   // then be the end of file.
@@ -899,7 +901,7 @@ void Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) {
     Result.SetKind(tok::eom);
     // Update the location of token as well as BufferPtr.
     FormTokenWithChars(Result, CurPtr);
-    return;
+    return true;  // Have a token.
   }        
 
   // If we aren't in raw mode, issue diagnostics. If we are in raw mode, let the
@@ -919,7 +921,7 @@ void Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) {
   }
   
   BufferPtr = CurPtr;
-  PP.HandleEndOfFile(Result);
+  return PP.HandleEndOfFile(Result);
 }
 
 /// isNextPPTokenLParen - Return 1 if the next unexpanded token lexed from
@@ -985,8 +987,14 @@ LexNextToken:
   switch (Char) {
   case 0:  // Null.
     // Found end of file?
-    if (CurPtr-1 == BufferEnd)
-      return LexEndOfFile(Result, CurPtr-1);  // Retreat back into the file.
+    if (CurPtr-1 == BufferEnd) {
+      // Read the PP instance variable into an automatic variable, because
+      // LexEndOfFile will often delete 'this'.
+      Preprocessor &PPCache = PP;
+      if (LexEndOfFile(Result, CurPtr-1))  // Retreat back into the file.
+        return;   // Got a token to return.
+      return PPCache.Lex(Result);
+    }
     
     Diag(CurPtr-1, diag::null_in_file);
     Result.SetFlag(LexerToken::LeadingSpace);
diff --git a/clang/Lex/MacroExpander.cpp b/clang/Lex/MacroExpander.cpp
index 50552b94c17..c09c2d9600f 100644
--- a/clang/Lex/MacroExpander.cpp
+++ b/clang/Lex/MacroExpander.cpp
@@ -326,9 +326,6 @@ void MacroExpander::ExpandFunctionArguments() {
         continue;
       }
       
-      // Okay, we have a token that is either the LHS or RHS of a paste (##)
-      // argument.
-      
       // FIXME: Handle comma swallowing GNU extension.
       
       // FIXME: handle pasted args.  Handle 'placemarker' stuff.
@@ -355,8 +352,14 @@ void MacroExpander::Lex(LexerToken &Tok) {
     if (Macro) Macro->EnableMacro();
 
     // Pop this context off the preprocessors lexer stack and get the next
-    // token.
-    return PP.HandleEndOfMacro(Tok);
+    // token.  This will delete "this" so remember the PP instance var.
+    Preprocessor &PPCache = PP;
+    if (PP.HandleEndOfMacro(Tok))
+      return;
+
+    // HandleEndOfMacro may not return a token.  If it doesn't, lex whatever is
+    // next.
+    return PPCache.Lex(Tok);
   }
   
   // Get the next token to return.
diff --git a/clang/Lex/Preprocessor.cpp b/clang/Lex/Preprocessor.cpp
index fab5c69b6d5..932fae74f8f 100644
--- a/clang/Lex/Preprocessor.cpp
+++ b/clang/Lex/Preprocessor.cpp
@@ -1005,7 +1005,7 @@ void Preprocessor::HandleIdentifier(LexerToken &Identifier) {
 /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
 /// the current file.  This either returns the EOF token or pops a level off
 /// the include stack and keeps going.
-void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
+bool Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
   assert(!CurMacroExpander &&
          "Ending a file when currently in a macro!");
   
@@ -1019,7 +1019,7 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
     CurLexer->BufferPtr = CurLexer->BufferEnd;
     CurLexer->FormTokenWithChars(Result, CurLexer->BufferEnd);
     Result.SetKind(tok::eof);
-    return;
+    return true;
   }
   
   // See if this file had a controlling macro.
@@ -1051,8 +1051,9 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
       FileChangeHandler(CurLexer->getSourceLocation(CurLexer->BufferPtr),
                         ExitFile, FileType);
     }
-    
-    return Lex(Result);
+
+    // Client should lex another token.
+    return false;
   }
   
   Result.StartToken();
@@ -1069,11 +1070,13 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
   // have not been used.
   if (Diags.getDiagnosticLevel(diag::pp_macro_not_used) != Diagnostic::Ignored)
     Identifiers.VisitIdentifiers(UnusedIdentifierReporter(*this));
+  
+  return true;
 }
 
 /// HandleEndOfMacro - This callback is invoked when the lexer hits the end of
 /// the current macro expansion or token stream expansion.
-void Preprocessor::HandleEndOfMacro(LexerToken &Result) {
+bool Preprocessor::HandleEndOfMacro(LexerToken &Result) {
   assert(CurMacroExpander && !CurLexer &&
          "Ending a macro when currently in a #include file!");
 
diff --git a/clang/include/clang/Lex/Lexer.h b/clang/include/clang/Lex/Lexer.h
index 60a1923bf15..b7e8c53fa54 100644
--- a/clang/include/clang/Lex/Lexer.h
+++ b/clang/include/clang/Lex/Lexer.h
@@ -336,7 +336,7 @@ private:
   void LexStringLiteral      (LexerToken &Result, const char *CurPtr);
   void LexAngledStringLiteral(LexerToken &Result, const char *CurPtr);
   void LexCharConstant       (LexerToken &Result, const char *CurPtr);
-  void LexEndOfFile          (LexerToken &Result, const char *CurPtr);
+  bool LexEndOfFile          (LexerToken &Result, const char *CurPtr);
   
   void SkipWhitespace        (LexerToken &Result, const char *CurPtr);
   void SkipBCPLComment       (LexerToken &Result, const char *CurPtr);
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 3a99e4029ff..b0a893a3f46 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -417,13 +417,15 @@ public:
 
   
   /// HandleEndOfFile - This callback is invoked when the lexer hits the end of
-  /// the current file.  This either returns the EOF token or pops a level off
-  /// the include stack and keeps going.
-  void HandleEndOfFile(LexerToken &Result, bool isEndOfMacro = false);
+  /// the current file.  This either returns the EOF token and returns true, or
+  /// pops a level off the include stack and returns false, at which point the
+  /// client should call lex again.
+  bool HandleEndOfFile(LexerToken &Result, bool isEndOfMacro = false);
   
   /// HandleEndOfMacro - This callback is invoked when the lexer hits the end of
-  /// the current macro line.
-  void HandleEndOfMacro(LexerToken &Result);
+  /// the current macro line.  It returns true if Result is filled in with a
+  /// token, or false if Lex should be called again.
+  bool HandleEndOfMacro(LexerToken &Result);
   
   /// HandleDirective - This callback is invoked when the lexer sees a # token
   /// at the start of a line.  This consumes the directive, modifies the