Parse: Don't parse after the eof has been consumed

ParseCXXNonStaticMemberInitializer stashes away all the tokens for the
initializer and an additional EOF token to denote where the initializer
ends.  However, it is possible for ParseLexedMemberInitializer to get
its hands on the "real" EOF token; since the two tokens are
indistinguishable, we end up consuming the EOF and descend into madness.

Instead, make it possible to tell which EOF token we are looking at.

This fixes PR21872.

llvm-svn: 224505

3 files changed, 22 insertions, 1 deletions

diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index a048757ff35..a4b4e125575 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -23,6 +23,7 @@
 
 namespace clang {
 
+class Decl;
 class IdentifierInfo;
 
 /// Token - This structure provides full information about a lexed token.
@@ -58,6 +59,8 @@ class Token {
   ///    may be dirty (have trigraphs / escaped newlines).
   ///  Annotations (resolved type names, C++ scopes, etc): isAnnotation().
   ///    This is a pointer to sema-specific data for the annotation token.
+  ///  Eof:
+  //     This is a pointer to a Decl.
   ///  Other:
   ///    This is null.
   void *PtrData;
@@ -164,12 +167,23 @@ public:
     assert(!isAnnotation() &&
            "getIdentifierInfo() on an annotation token!");
     if (isLiteral()) return nullptr;
+    if (is(tok::eof)) return nullptr;
     return (IdentifierInfo*) PtrData;
   }
   void setIdentifierInfo(IdentifierInfo *II) {
     PtrData = (void*) II;
   }
 
+  const Decl *getDecl() const {
+    assert(is(tok::eof));
+    return reinterpret_cast<const Decl *>(PtrData);
+  }
+  void setDecl(const Decl *D) {
+    assert(is(tok::eof));
+    assert(!PtrData);
+    PtrData = const_cast<Decl *>(D);
+  }
+
   /// getRawIdentifier - For a raw identifier token (i.e., an identifier
   /// lexed in raw mode), returns a reference to the text substring in the
   /// buffer if known.
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index e1a5b8e9945..96d35bc249c 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -218,6 +218,7 @@ void Parser::ParseCXXNonStaticMemberInitializer(Decl *VarD) {
   Eof.startToken();
   Eof.setKind(tok::eof);
   Eof.setLocation(Tok.getLocation());
+  Eof.setDecl(VarD);
   Toks.push_back(Eof);
 }
 
@@ -622,7 +623,9 @@ void Parser::ParseLexedMemberInitializer(LateParsedMemberInitializer &MI) {
     while (Tok.isNot(tok::eof))
       ConsumeAnyToken();
   }
-  ConsumeAnyToken();
+  // Make sure this is *our* artificial EOF token.
+  if (Tok.getDecl() == MI.Field)
+    ConsumeAnyToken();
 }
 
 /// ConsumeAndStoreUntil - Consume and store the token at the passed token
diff --git a/clang/test/Parser/PR21872.cpp b/clang/test/Parser/PR21872.cpp
new file mode 100644
index 00000000000..ae0a13d9c1e
--- /dev/null
+++ b/clang/test/Parser/PR21872.cpp
@@ -0,0 +1,4 @@
+// RUN: not %clang_cc1 -fsyntax-only %s
+template <typename T> struct S {
+    int k = ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((
+int f;