[Parser] Update CachedTokens while parsing ObjectiveC template argument list

Consider the following ObjC++ snippet:

--
@protocol PA;
@protocol PB;

@class NSArray<ObjectType>;
typedef int some_t;

id<PA> FA(NSArray<id<PB>> *h, some_t group);
--

This would hit an assertion in the parser after generating an annotation token
while trying to update the token cache:

Assertion failed: (CachedTokens[CachedLexPos-1].getLastLoc() == Tok.getAnnotationEndLoc() && "The annotation should be until the most recent cached token")
...
7 clang::Preprocessor::AnnotatePreviousCachedTokens(clang::Token const&) + 494
8 clang::Parser::TryAnnotateTypeOrScopeTokenAfterScopeSpec(bool, bool, clang::CXXScopeSpec&, bool) + 1163
9 clang::Parser::TryAnnotateTypeOrScopeToken(bool, bool) + 361
10 clang::Parser::isCXXDeclarationSpecifier(clang::Parser::TPResult, bool*) + 598
...

The cached preprocessor token in this case is:

greatergreater '>>' Loc=<testcase.mm:7:24>

while the annotation ("NSArray<id<PB>>") ends at "testcase.mm:7:25", hence the
assertion.

Properly update the CachedTokens during template parsing to contain
two greater tokens instead of a greatergreater.

Differential Revision: http://reviews.llvm.org/D15173

rdar://problem/23494277

llvm-svn: 259311

2 files changed, 38 insertions, 0 deletions

diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index bd48ae64ab4..4742aae5c12 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -116,3 +116,29 @@ void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
     }
   }
 }
+
+bool Preprocessor::IsPreviousCachedToken(const Token &Tok) const {
+  // There's currently no cached token...
+  if (!CachedLexPos)
+    return false;
+
+  const Token LastCachedTok = CachedTokens[CachedLexPos - 1];
+  if (LastCachedTok.getKind() != Tok.getKind())
+    return false;
+
+  int RelOffset = 0;
+  if ((!getSourceManager().isInSameSLocAddrSpace(
+          Tok.getLocation(), getLastCachedTokenLocation(), &RelOffset)) ||
+      RelOffset)
+    return false;
+
+  return true;
+}
+
+void Preprocessor::ReplacePreviousCachedToken(ArrayRef<Token> NewToks) {
+  assert(CachedLexPos != 0 && "Expected to have some cached tokens");
+  CachedTokens.insert(CachedTokens.begin() + CachedLexPos - 1, NewToks.begin(),
+                      NewToks.end());
+  CachedTokens.erase(CachedTokens.begin() + CachedLexPos - 1 + NewToks.size());
+  CachedLexPos += NewToks.size() - 1;
+}
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index fffc40ee4e1..9e68c4a15d3 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -831,6 +831,7 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
   }
 
   // Strip the initial '>' from the token.
+  Token PrevTok = Tok;
   if (RemainingToken == tok::equal && Next.is(tok::equal) &&
       areTokensAdjacent(Tok, Next)) {
     // Join two adjacent '=' tokens into one, for cases like:
@@ -847,6 +848,17 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
                                                  PP.getSourceManager(),
                                                  getLangOpts()));
 
+  // The advance from '>>' to '>' in a ObjectiveC template argument list needs
+  // to be properly reflected in the token cache to allow correct interaction
+  // between annotation and backtracking.
+  if (ObjCGenericList && PrevTok.getKind() == tok::greatergreater &&
+      RemainingToken == tok::greater && PP.IsPreviousCachedToken(PrevTok)) {
+    PrevTok.setKind(RemainingToken);
+    PrevTok.setLength(1);
+    Token NewToks[] = {PrevTok, Tok};
+    PP.ReplacePreviousCachedToken(NewToks);
+  }
+
   if (!ConsumeLastToken) {
     // Since we're not supposed to consume the '>' token, we need to push
     // this token and revert the current token back to the '>'.