summaryrefslogtreecommitdiffstats
path: root/clang/lib
diff options
context:
space:
mode:
Diffstat (limited to 'clang/lib')
-rw-r--r--clang/lib/Frontend/PrintPreprocessedOutput.cpp3
-rw-r--r--clang/lib/Lex/MacroArgs.cpp2
-rw-r--r--clang/lib/Lex/PPCaching.cpp4
-rw-r--r--clang/lib/Lex/PPDirectives.cpp4
-rw-r--r--clang/lib/Lex/PPLexerChange.cpp13
-rw-r--r--clang/lib/Lex/PPMacroExpansion.cpp2
-rw-r--r--clang/lib/Lex/Pragma.cpp13
-rw-r--r--clang/lib/Lex/Preprocessor.cpp13
-rw-r--r--clang/lib/Lex/TokenLexer.cpp8
-rw-r--r--clang/lib/Parse/ParseCXXInlineMethods.cpp11
-rw-r--r--clang/lib/Parse/ParseDecl.cpp6
-rw-r--r--clang/lib/Parse/ParseDeclCXX.cpp12
-rw-r--r--clang/lib/Parse/ParseExpr.cpp6
-rw-r--r--clang/lib/Parse/ParseExprCXX.cpp13
-rw-r--r--clang/lib/Parse/ParseObjc.cpp4
-rw-r--r--clang/lib/Parse/ParseOpenMP.cpp7
-rw-r--r--clang/lib/Parse/ParsePragma.cpp60
-rw-r--r--clang/lib/Parse/ParseStmtAsm.cpp3
-rw-r--r--clang/lib/Parse/ParseTemplate.cpp4
-rw-r--r--clang/lib/Parse/Parser.cpp4
-rw-r--r--clang/lib/Rewrite/HTMLRewrite.cpp2
21 files changed, 113 insertions, 81 deletions
diff --git a/clang/lib/Frontend/PrintPreprocessedOutput.cpp b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
index 584f94db133..720ed51c5f6 100644
--- a/clang/lib/Frontend/PrintPreprocessedOutput.cpp
+++ b/clang/lib/Frontend/PrintPreprocessedOutput.cpp
@@ -678,7 +678,8 @@ struct UnknownPragmaHandler : public PragmaHandler {
auto Toks = llvm::make_unique<Token[]>(1);
Toks[0] = PragmaTok;
PP.EnterTokenStream(std::move(Toks), /*NumToks=*/1,
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false,
+ /*IsReinject=*/false);
PP.Lex(PragmaTok);
}
Token PrevToken;
diff --git a/clang/lib/Lex/MacroArgs.cpp b/clang/lib/Lex/MacroArgs.cpp
index 06e3add1548..5aa4679fad4 100644
--- a/clang/lib/Lex/MacroArgs.cpp
+++ b/clang/lib/Lex/MacroArgs.cpp
@@ -181,7 +181,7 @@ const std::vector<Token> &MacroArgs::getPreExpArgument(unsigned Arg,
// list. With this installed, we lex expanded tokens until we hit the EOF
// token at the end of the unexp list.
PP.EnterTokenStream(AT, NumToks, false /*disable expand*/,
- false /*owns tokens*/);
+ false /*owns tokens*/, false /*is reinject*/);
// Lex all of the macro-expanded tokens into Result.
do {
diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index 363a185d3d2..31548d246d5 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -45,7 +45,7 @@ void Preprocessor::Backtrack() {
recomputeCurLexerKind();
}
-void Preprocessor::CachingLex(Token &Result, bool &IsNewToken) {
+void Preprocessor::CachingLex(Token &Result) {
if (!InCachingLexMode())
return;
@@ -55,7 +55,7 @@ void Preprocessor::CachingLex(Token &Result, bool &IsNewToken) {
if (CachedLexPos < CachedTokens.size()) {
Result = CachedTokens[CachedLexPos++];
- IsNewToken = false;
+ Result.setFlag(Token::IsReinjected);
return;
}
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 193388650d2..90bba7e09bd 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1035,7 +1035,7 @@ void Preprocessor::HandleDirective(Token &Result) {
// Enter this token stream so that we re-lex the tokens. Make sure to
// enable macro expansion, in case the token after the # is an identifier
// that is expanded.
- EnterTokenStream(std::move(Toks), 2, false);
+ EnterTokenStream(std::move(Toks), 2, false, /*IsReinject*/false);
return;
}
@@ -1518,7 +1518,7 @@ void Preprocessor::EnterAnnotationToken(SourceRange Range,
Tok[0].setLocation(Range.getBegin());
Tok[0].setAnnotationEndLoc(Range.getEnd());
Tok[0].setAnnotationValue(AnnotationVal);
- EnterTokenStream(std::move(Tok), 1, true);
+ EnterTokenStream(std::move(Tok), 1, true, /*IsReinject*/ false);
}
/// Produce a diagnostic informing the user that a #include or similar
diff --git a/clang/lib/Lex/PPLexerChange.cpp b/clang/lib/Lex/PPLexerChange.cpp
index 5f547d9f1cc..cfbb547fae3 100644
--- a/clang/lib/Lex/PPLexerChange.cpp
+++ b/clang/lib/Lex/PPLexerChange.cpp
@@ -154,10 +154,11 @@ void Preprocessor::EnterMacro(Token &Tok, SourceLocation ILEnd,
/// must be freed.
///
void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
- bool DisableMacroExpansion,
- bool OwnsTokens) {
+ bool DisableMacroExpansion, bool OwnsTokens,
+ bool IsReinject) {
if (CurLexerKind == CLK_CachingLexer) {
if (CachedLexPos < CachedTokens.size()) {
+ assert(IsReinject && "new tokens in the middle of cached stream");
// We're entering tokens into the middle of our cached token stream. We
// can't represent that, so just insert the tokens into the buffer.
CachedTokens.insert(CachedTokens.begin() + CachedLexPos,
@@ -170,7 +171,8 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
// New tokens are at the end of the cached token sequnece; insert the
// token stream underneath the caching lexer.
ExitCachingLexMode();
- EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+ EnterTokenStream(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
+ IsReinject);
EnterCachingLexMode();
return;
}
@@ -179,10 +181,11 @@ void Preprocessor::EnterTokenStream(const Token *Toks, unsigned NumToks,
std::unique_ptr<TokenLexer> TokLexer;
if (NumCachedTokenLexers == 0) {
TokLexer = llvm::make_unique<TokenLexer>(
- Toks, NumToks, DisableMacroExpansion, OwnsTokens, *this);
+ Toks, NumToks, DisableMacroExpansion, OwnsTokens, IsReinject, *this);
} else {
TokLexer = std::move(TokenLexerCache[--NumCachedTokenLexers]);
- TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens);
+ TokLexer->Init(Toks, NumToks, DisableMacroExpansion, OwnsTokens,
+ IsReinject);
}
// Save our current state.
diff --git a/clang/lib/Lex/PPMacroExpansion.cpp b/clang/lib/Lex/PPMacroExpansion.cpp
index d576f331d1b..4576f1a47e1 100644
--- a/clang/lib/Lex/PPMacroExpansion.cpp
+++ b/clang/lib/Lex/PPMacroExpansion.cpp
@@ -806,7 +806,7 @@ MacroArgs *Preprocessor::ReadMacroCallArgumentList(Token &MacroName,
// Do not lose the EOF/EOD.
auto Toks = llvm::make_unique<Token[]>(1);
Toks[0] = Tok;
- EnterTokenStream(std::move(Toks), 1, true);
+ EnterTokenStream(std::move(Toks), 1, true, /*IsReinject*/ false);
break;
} else if (Tok.is(tok::r_paren)) {
// If we found the ) token, the macro arg list is done.
diff --git a/clang/lib/Lex/Pragma.cpp b/clang/lib/Lex/Pragma.cpp
index 8aff163d798..2404a47c1dc 100644
--- a/clang/lib/Lex/Pragma.cpp
+++ b/clang/lib/Lex/Pragma.cpp
@@ -189,7 +189,8 @@ void Preprocessor::Handle_Pragma(Token &Tok) {
std::copy(Tokens.begin() + 1, Tokens.end(), Toks.get());
Toks[Tokens.size() - 1] = Tok;
Self.EnterTokenStream(std::move(Toks), Tokens.size(),
- /*DisableMacroExpansion*/ true);
+ /*DisableMacroExpansion*/ true,
+ /*IsReinject*/ true);
// ... and return the _Pragma token unchanged.
Tok = *Tokens.begin();
@@ -366,7 +367,8 @@ void Preprocessor::HandleMicrosoft__pragma(Token &Tok) {
std::copy(PragmaToks.begin(), PragmaToks.end(), TokArray);
// Push the tokens onto the stack.
- EnterTokenStream(TokArray, PragmaToks.size(), true, true);
+ EnterTokenStream(TokArray, PragmaToks.size(), true, true,
+ /*IsReinject*/ false);
// With everything set up, lex this as a #pragma directive.
HandlePragmaDirective(PragmaLoc, PIK___pragma);
@@ -1028,7 +1030,7 @@ struct PragmaDebugHandler : public PragmaHandler {
Crasher.startToken();
Crasher.setKind(tok::annot_pragma_parser_crash);
Crasher.setAnnotationRange(SourceRange(Tok.getLocation()));
- PP.EnterToken(Crasher);
+ PP.EnterToken(Crasher, /*IsReinject*/false);
} else if (II->isStr("dump")) {
Token Identifier;
PP.LexUnexpandedToken(Identifier);
@@ -1040,7 +1042,7 @@ struct PragmaDebugHandler : public PragmaHandler {
SourceRange(Tok.getLocation(), Identifier.getLocation()));
DumpAnnot.setAnnotationValue(DumpII);
PP.DiscardUntilEndOfDirective();
- PP.EnterToken(DumpAnnot);
+ PP.EnterToken(DumpAnnot, /*IsReinject*/false);
} else {
PP.Diag(Identifier, diag::warn_pragma_debug_missing_argument)
<< II->getName();
@@ -1123,7 +1125,8 @@ struct PragmaDebugHandler : public PragmaHandler {
Toks[0].setKind(tok::annot_pragma_captured);
Toks[0].setLocation(NameLoc);
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// Disable MSVC warning about runtime stack overflow.
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index e16c8ac1f34..dd619389556 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -635,8 +635,7 @@ void Preprocessor::SkipTokensWhileUsingPCH() {
CurTokenLexer->Lex(Tok);
break;
case CLK_CachingLexer:
- bool IsNewToken;
- CachingLex(Tok, IsNewToken);
+ CachingLex(Tok);
break;
case CLK_LexAfterModuleImport:
LexAfterModuleImport(Tok);
@@ -881,7 +880,6 @@ void Preprocessor::Lex(Token &Result) {
// We loop here until a lex function returns a token; this avoids recursion.
bool ReturnedToken;
- bool IsNewToken = true;
do {
switch (CurLexerKind) {
case CLK_Lexer:
@@ -891,7 +889,7 @@ void Preprocessor::Lex(Token &Result) {
ReturnedToken = CurTokenLexer->Lex(Result);
break;
case CLK_CachingLexer:
- CachingLex(Result, IsNewToken);
+ CachingLex(Result);
ReturnedToken = true;
break;
case CLK_LexAfterModuleImport:
@@ -911,7 +909,8 @@ void Preprocessor::Lex(Token &Result) {
// Update ImportSeqState to track our position within a C++20 import-seq
// if this token is being produced as a result of phase 4 of translation.
- if (getLangOpts().CPlusPlusModules && LexLevel == 1 && IsNewToken) {
+ if (getLangOpts().CPlusPlusModules && LexLevel == 1 &&
+ !Result.getFlag(Token::IsReinjected)) {
switch (Result.getKind()) {
case tok::l_paren: case tok::l_square: case tok::l_brace:
ImportSeqState.handleOpenBracket();
@@ -952,6 +951,8 @@ void Preprocessor::Lex(Token &Result) {
LastTokenWasAt = Result.is(tok::at);
--LexLevel;
+ if (OnToken && LexLevel == 0 && !Result.getFlag(Token::IsReinjected))
+ OnToken(Result);
}
/// Lex a header-name token (including one formed from header-name-tokens if
@@ -1131,7 +1132,7 @@ bool Preprocessor::LexAfterModuleImport(Token &Result) {
auto ToksCopy = llvm::make_unique<Token[]>(Toks.size());
std::copy(Toks.begin(), Toks.end(), ToksCopy.get());
EnterTokenStream(std::move(ToksCopy), Toks.size(),
- /*DisableMacroExpansion*/ true);
+ /*DisableMacroExpansion*/ true, /*IsReinject*/ false);
};
// Check for a header-name.
diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
index 9d132a545c8..a7957e82e49 100644
--- a/clang/lib/Lex/TokenLexer.cpp
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -53,6 +53,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
Tokens = &*Macro->tokens_begin();
OwnsTokens = false;
DisableMacroExpansion = false;
+ IsReinject = false;
NumTokens = Macro->tokens_end()-Macro->tokens_begin();
MacroExpansionStart = SourceLocation();
@@ -91,7 +92,9 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
/// Create a TokenLexer for the specified token stream. This does not
/// take ownership of the specified token vector.
void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
- bool disableMacroExpansion, bool ownsTokens) {
+ bool disableMacroExpansion, bool ownsTokens,
+ bool isReinject) {
+ assert(!isReinject || disableMacroExpansion);
// If the client is reusing a TokenLexer, make sure to free any memory
// associated with it.
destroy();
@@ -101,6 +104,7 @@ void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
Tokens = TokArray;
OwnsTokens = ownsTokens;
DisableMacroExpansion = disableMacroExpansion;
+ IsReinject = isReinject;
NumTokens = NumToks;
CurTokenIdx = 0;
ExpandLocStart = ExpandLocEnd = SourceLocation();
@@ -647,6 +651,8 @@ bool TokenLexer::Lex(Token &Tok) {
// Get the next token to return.
Tok = Tokens[CurTokenIdx++];
+ if (IsReinject)
+ Tok.setFlag(Token::IsReinjected);
bool TokenIsFromPaste = false;
diff --git a/clang/lib/Parse/ParseCXXInlineMethods.cpp b/clang/lib/Parse/ParseCXXInlineMethods.cpp
index 5598058932e..f2e552a3ce1 100644
--- a/clang/lib/Parse/ParseCXXInlineMethods.cpp
+++ b/clang/lib/Parse/ParseCXXInlineMethods.cpp
@@ -323,7 +323,7 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
// Parse the default argument from its saved token stream.
Toks->push_back(Tok); // So that the current token doesn't get lost
- PP.EnterTokenStream(*Toks, true);
+ PP.EnterTokenStream(*Toks, true, /*IsReinject*/ true);
// Consume the previously-pushed token.
ConsumeAnyToken();
@@ -396,7 +396,7 @@ void Parser::ParseLexedMethodDeclaration(LateParsedMethodDeclaration &LM) {
// Parse the default argument from its saved token stream.
Toks->push_back(Tok); // So that the current token doesn't get lost
- PP.EnterTokenStream(*Toks, true);
+ PP.EnterTokenStream(*Toks, true, /*IsReinject*/true);
// Consume the previously-pushed token.
ConsumeAnyToken();
@@ -503,7 +503,7 @@ void Parser::ParseLexedMethodDef(LexedMethod &LM) {
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LM.Toks.push_back(Tok);
- PP.EnterTokenStream(LM.Toks, true);
+ PP.EnterTokenStream(LM.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -617,7 +617,7 @@ void Parser::ParseLexedMemberInitializer(LateParsedMemberInitializer &MI) {
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
MI.Toks.push_back(Tok);
- PP.EnterTokenStream(MI.Toks, true);
+ PP.EnterTokenStream(MI.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -989,7 +989,8 @@ public:
auto Buffer = llvm::make_unique<Token[]>(Toks.size());
std::copy(Toks.begin() + 1, Toks.end(), Buffer.get());
Buffer[Toks.size() - 1] = Self.Tok;
- Self.PP.EnterTokenStream(std::move(Buffer), Toks.size(), true);
+ Self.PP.EnterTokenStream(std::move(Buffer), Toks.size(), true,
+ /*IsReinject*/ true);
Self.Tok = Toks.front();
}
diff --git a/clang/lib/Parse/ParseDecl.cpp b/clang/lib/Parse/ParseDecl.cpp
index fe597ed2dfb..edadc907081 100644
--- a/clang/lib/Parse/ParseDecl.cpp
+++ b/clang/lib/Parse/ParseDecl.cpp
@@ -1470,7 +1470,7 @@ void Parser::ParseLexedAttribute(LateParsedAttribute &LA,
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LA.Toks.push_back(Tok);
- PP.EnterTokenStream(LA.Toks, true);
+ PP.EnterTokenStream(LA.Toks, true, /*IsReinject=*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -4424,7 +4424,7 @@ void Parser::ParseEnumSpecifier(SourceLocation StartLoc, DeclSpec &DS,
if (Tok.isNot(tok::semi)) {
// A semicolon was missing after this declaration. Diagnose and recover.
ExpectAndConsume(tok::semi, diag::err_expected_after, "enum");
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::semi);
}
} else {
@@ -4702,7 +4702,7 @@ void Parser::ParseEnumBody(SourceLocation StartLoc, Decl *EnumDecl) {
// Push this token back into the preprocessor and change our current token
// to ';' so that the rest of the code recovers as though there were an
// ';' after the definition.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::semi);
}
}
diff --git a/clang/lib/Parse/ParseDeclCXX.cpp b/clang/lib/Parse/ParseDeclCXX.cpp
index e8848841635..6f44b844828 100644
--- a/clang/lib/Parse/ParseDeclCXX.cpp
+++ b/clang/lib/Parse/ParseDeclCXX.cpp
@@ -1040,7 +1040,7 @@ void Parser::AnnotateExistingDecltypeSpecifier(const DeclSpec& DS,
if (PP.isBacktrackEnabled())
PP.RevertCachedTokens(1);
else
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::annot_decltype);
setExprAnnotation(Tok,
@@ -1736,7 +1736,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
// A semicolon was missing after this declaration. Diagnose and recover.
ExpectAndConsume(tok::semi, diag::err_expected_after,
DeclSpec::getSpecifierName(TagType, PPol));
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::semi);
}
} else
@@ -2013,7 +2013,7 @@ void Parser::ParseClassSpecifier(tok::TokenKind TagTokKind,
// Push this token back into the preprocessor and change our current token
// to ';' so that the rest of the code recovers as though there were an
// ';' after the definition.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::semi);
}
}
@@ -3280,7 +3280,7 @@ void Parser::ParseCXXMemberSpecification(SourceLocation RecordLoc,
if (SuggestFixIt) {
LBraceDiag << FixItHint::CreateInsertion(BraceLoc, " {");
// Try recovering from missing { after base-clause.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok.setKind(tok::l_brace);
} else {
if (TagDecl)
@@ -3376,12 +3376,12 @@ void Parser::DiagnoseUnexpectedNamespace(NamedDecl *D) {
diag::note_missing_end_of_definition_before) << D;
// Push '};' onto the token stream to recover.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
Tok.startToken();
Tok.setLocation(PP.getLocForEndOfToken(PrevTokLocation));
Tok.setKind(tok::semi);
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
Tok.setKind(tok::r_brace);
}
diff --git a/clang/lib/Parse/ParseExpr.cpp b/clang/lib/Parse/ParseExpr.cpp
index af034f9fc27..dbea7741183 100644
--- a/clang/lib/Parse/ParseExpr.cpp
+++ b/clang/lib/Parse/ParseExpr.cpp
@@ -303,7 +303,7 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
// We can't do this before consuming the comma, because
// isNotExpressionStart() looks at the token stream.
if (OpToken.is(tok::comma) && isNotExpressionStart()) {
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok = OpToken;
return LHS;
}
@@ -313,7 +313,7 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
if (isFoldOperator(NextTokPrec) && Tok.is(tok::ellipsis)) {
// FIXME: We can't check this via lookahead before we consume the token
// because that tickles a lexer bug.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok = OpToken;
return LHS;
}
@@ -326,7 +326,7 @@ Parser::ParseRHSOfBinaryExpression(ExprResult LHS, prec::Level MinPrec) {
if (getLangOpts().ObjC && getLangOpts().CPlusPlus &&
Tok.isOneOf(tok::colon, tok::r_square) &&
OpToken.getIdentifierInfo() != nullptr) {
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
Tok = OpToken;
return LHS;
}
diff --git a/clang/lib/Parse/ParseExprCXX.cpp b/clang/lib/Parse/ParseExprCXX.cpp
index c6f457fb149..8abd6969c0f 100644
--- a/clang/lib/Parse/ParseExprCXX.cpp
+++ b/clang/lib/Parse/ParseExprCXX.cpp
@@ -69,9 +69,9 @@ static void FixDigraph(Parser &P, Preprocessor &PP, Token &DigraphToken,
DigraphToken.setLength(1);
// Push new tokens back to token stream.
- PP.EnterToken(ColonToken);
+ PP.EnterToken(ColonToken, /*IsReinject*/ true);
if (!AtDigraph)
- PP.EnterToken(DigraphToken);
+ PP.EnterToken(DigraphToken, /*IsReinject*/ true);
}
// Check for '<::' which should be '< ::' instead of '[:' when following
@@ -434,7 +434,7 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
Token ColonColon;
PP.Lex(ColonColon);
ColonColon.setKind(tok::colon);
- PP.EnterToken(ColonColon);
+ PP.EnterToken(ColonColon, /*IsReinject*/ true);
break;
}
}
@@ -460,8 +460,8 @@ bool Parser::ParseOptionalCXXScopeSpecifier(CXXScopeSpec &SS,
// mistyped '::' instead of ':'.
if (CorrectionFlagPtr && IsCorrectedToColon) {
ColonColon.setKind(tok::colon);
- PP.EnterToken(Tok);
- PP.EnterToken(ColonColon);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
+ PP.EnterToken(ColonColon, /*IsReinject*/ true);
Tok = Identifier;
break;
}
@@ -3300,7 +3300,8 @@ Parser::ParseCXXAmbiguousParenExpression(ParenParseOption &ExprType,
Toks.push_back(Tok);
// Re-enter the stored parenthesized tokens into the token stream, so we may
// parse them now.
- PP.EnterTokenStream(Toks, true /*DisableMacroExpansion*/);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion*/ true,
+ /*IsReinject*/ true);
// Drop the current token and bring the first cached one. It's the same token
// as when we entered this function.
ConsumeAnyToken();
diff --git a/clang/lib/Parse/ParseObjc.cpp b/clang/lib/Parse/ParseObjc.cpp
index 274ea879e63..c2e73d5e82c 100644
--- a/clang/lib/Parse/ParseObjc.cpp
+++ b/clang/lib/Parse/ParseObjc.cpp
@@ -1943,7 +1943,7 @@ void Parser::ParseObjCClassInstanceVariables(Decl *interfaceDecl,
Tok.setLocation(Tok.getLocation().getLocWithOffset(-1));
Tok.setKind(tok::at);
Tok.setLength(1);
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/true);
HelperActionsForIvarDeclarations(interfaceDecl, atLoc,
T, AllIvarDecls, true);
return;
@@ -3654,7 +3654,7 @@ void Parser::ParseLexedObjCMethodDefs(LexedMethod &LM, bool parseMethod) {
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LM.Toks.push_back(Tok);
- PP.EnterTokenStream(LM.Toks, true);
+ PP.EnterTokenStream(LM.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
diff --git a/clang/lib/Parse/ParseOpenMP.cpp b/clang/lib/Parse/ParseOpenMP.cpp
index 34798ea39c7..52a68f6d693 100644
--- a/clang/lib/Parse/ParseOpenMP.cpp
+++ b/clang/lib/Parse/ParseOpenMP.cpp
@@ -747,8 +747,9 @@ static bool parseDeclareSimdClauses(
Parser::DeclGroupPtrTy
Parser::ParseOMPDeclareSimdClauses(Parser::DeclGroupPtrTy Ptr,
CachedTokens &Toks, SourceLocation Loc) {
- PP.EnterToken(Tok);
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject*/ true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
@@ -1319,7 +1320,7 @@ Parser::ParseOpenMPDeclarativeOrExecutableDirective(ParsedStmtContext StmtCtx) {
FlushHasClause = true;
// Push copy of the current token back to stream to properly parse
// pseudo-clause OMPFlushClause.
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject*/ true);
}
LLVM_FALLTHROUGH;
case OMPD_taskyield:
diff --git a/clang/lib/Parse/ParsePragma.cpp b/clang/lib/Parse/ParsePragma.cpp
index a0458fdbb73..e0f67d1e6fc 100644
--- a/clang/lib/Parse/ParsePragma.cpp
+++ b/clang/lib/Parse/ParsePragma.cpp
@@ -117,7 +117,8 @@ struct PragmaSTDC_FENV_ACCESSHandler : public PragmaHandler {
Toks[0].setAnnotationEndLoc(Tok.getLocation());
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(OOS)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
};
@@ -739,7 +740,8 @@ void Parser::HandlePragmaMSPragma() {
// Grab the tokens out of the annotation and enter them into the stream.
auto TheTokens =
(std::pair<std::unique_ptr<Token[]>, size_t> *)Tok.getAnnotationValue();
- PP.EnterTokenStream(std::move(TheTokens->first), TheTokens->second, true);
+ PP.EnterTokenStream(std::move(TheTokens->first), TheTokens->second, true,
+ /*IsReinject=*/true);
SourceLocation PragmaLocation = ConsumeAnnotationToken();
assert(Tok.isAnyIdentifier());
StringRef PragmaName = Tok.getIdentifierInfo()->getName();
@@ -1111,7 +1113,8 @@ bool Parser::HandlePragmaLoopHint(LoopHint &Hint) {
Hint.StateLoc = IdentifierLoc::create(Actions.Context, StateLoc, StateInfo);
} else {
// Enter constant expression including eof terminator into token stream.
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/false,
+ /*IsReinject=*/false);
ConsumeAnnotationToken();
ExprResult R = ParseConstantExpression();
@@ -1414,7 +1417,8 @@ void Parser::HandlePragmaAttribute() {
return;
}
- PP.EnterTokenStream(Info->Tokens, /*DisableMacroExpansion=*/false);
+ PP.EnterTokenStream(Info->Tokens, /*DisableMacroExpansion=*/false,
+ /*IsReinject=*/false);
ConsumeAnnotationToken();
ParsedAttributes &Attrs = Info->Attributes;
@@ -1623,8 +1627,9 @@ void PragmaGCCVisibilityHandler::HandlePragma(Preprocessor &PP,
Toks[0].setLocation(VisLoc);
Toks[0].setAnnotationEndLoc(EndLoc);
Toks[0].setAnnotationValue(
- const_cast<void*>(static_cast<const void*>(VisType)));
- PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true);
+ const_cast<void *>(static_cast<const void *>(VisType)));
+ PP.EnterTokenStream(std::move(Toks), 1, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma pack(...) comes in the following delicious flavors:
@@ -1737,7 +1742,8 @@ void PragmaPackHandler::HandlePragma(Preprocessor &PP,
Toks[0].setLocation(PackLoc);
Toks[0].setAnnotationEndLoc(RParenLoc);
Toks[0].setAnnotationValue(static_cast<void*>(Info));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma ms_struct on
@@ -1780,7 +1786,8 @@ void PragmaMSStructHandler::HandlePragma(Preprocessor &PP,
Toks[0].setAnnotationEndLoc(EndLoc);
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(Kind)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma clang section bss="abc" data="" rodata="def" text=""
@@ -1893,7 +1900,8 @@ static void ParseAlignPragma(Preprocessor &PP, Token &FirstTok,
Toks[0].setAnnotationEndLoc(EndLoc);
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(Kind)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
void PragmaAlignHandler::HandlePragma(Preprocessor &PP,
@@ -1985,7 +1993,8 @@ void PragmaUnusedHandler::HandlePragma(Preprocessor &PP,
pragmaUnusedTok.setLocation(UnusedLoc);
idTok = Identifiers[i];
}
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
// #pragma weak identifier
@@ -2034,7 +2043,8 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP,
pragmaUnusedTok.setAnnotationEndLoc(AliasName.getLocation());
Toks[1] = WeakName;
Toks[2] = AliasName;
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
} else {
MutableArrayRef<Token> Toks(
PP.getPreprocessorAllocator().Allocate<Token>(2), 2);
@@ -2044,7 +2054,8 @@ void PragmaWeakHandler::HandlePragma(Preprocessor &PP,
pragmaUnusedTok.setLocation(WeakLoc);
pragmaUnusedTok.setAnnotationEndLoc(WeakLoc);
Toks[1] = WeakName;
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
}
@@ -2089,7 +2100,8 @@ void PragmaRedefineExtnameHandler::HandlePragma(Preprocessor &PP,
pragmaRedefTok.setAnnotationEndLoc(AliasName.getLocation());
Toks[1] = RedefName;
Toks[2] = AliasName;
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
@@ -2109,7 +2121,8 @@ PragmaFPContractHandler::HandlePragma(Preprocessor &PP,
Toks[0].setAnnotationEndLoc(Tok.getLocation());
Toks[0].setAnnotationValue(reinterpret_cast<void*>(
static_cast<uintptr_t>(OOS)));
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
}
void
@@ -2171,7 +2184,8 @@ PragmaOpenCLExtensionHandler::HandlePragma(Preprocessor &PP,
Toks[0].setLocation(NameLoc);
Toks[0].setAnnotationValue(static_cast<void*>(Info));
Toks[0].setAnnotationEndLoc(StateLoc);
- PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true);
+ PP.EnterTokenStream(Toks, /*DisableMacroExpansion=*/true,
+ /*IsReinject=*/false);
if (PP.getPPCallbacks())
PP.getPPCallbacks()->PragmaOpenCLExtension(NameLoc, Ext,
@@ -2230,7 +2244,7 @@ PragmaOpenMPHandler::HandlePragma(Preprocessor &PP,
auto Toks = llvm::make_unique<Token[]>(Pragma.size());
std::copy(Pragma.begin(), Pragma.end(), Toks.get());
PP.EnterTokenStream(std::move(Toks), Pragma.size(),
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
/// Handle '#pragma pointers_to_members'
@@ -2328,7 +2342,7 @@ void PragmaMSPointersToMembers::HandlePragma(Preprocessor &PP,
AnnotTok.setAnnotationEndLoc(EndLoc);
AnnotTok.setAnnotationValue(
reinterpret_cast<void *>(static_cast<uintptr_t>(RepresentationMethod)));
- PP.EnterToken(AnnotTok);
+ PP.EnterToken(AnnotTok, /*IsReinject=*/true);
}
/// Handle '#pragma vtordisp'
@@ -2421,7 +2435,7 @@ void PragmaMSVtorDisp::HandlePragma(Preprocessor &PP,
AnnotTok.setAnnotationEndLoc(EndLoc);
AnnotTok.setAnnotationValue(reinterpret_cast<void *>(
static_cast<uintptr_t>((Action << 16) | (Value & 0xFFFF))));
- PP.EnterToken(AnnotTok);
+ PP.EnterToken(AnnotTok, /*IsReinject=*/false);
}
/// Handle all MS pragmas. Simply forwards the tokens after inserting
@@ -2452,7 +2466,7 @@ void PragmaMSPragma::HandlePragma(Preprocessor &PP,
std::pair<std::unique_ptr<Token[]>, size_t>(std::move(TokenArray),
TokenVector.size());
AnnotTok.setAnnotationValue(Value);
- PP.EnterToken(AnnotTok);
+ PP.EnterToken(AnnotTok, /*IsReinject*/ false);
}
/// Handle the Microsoft \#pragma detect_mismatch extension.
@@ -2736,7 +2750,7 @@ void PragmaFPHandler::HandlePragma(Preprocessor &PP,
std::copy(TokenList.begin(), TokenList.end(), TokenArray.get());
PP.EnterTokenStream(std::move(TokenArray), TokenList.size(),
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
void Parser::HandlePragmaFP() {
@@ -2919,7 +2933,7 @@ void PragmaLoopHintHandler::HandlePragma(Preprocessor &PP,
std::copy(TokenList.begin(), TokenList.end(), TokenArray.get());
PP.EnterTokenStream(std::move(TokenArray), TokenList.size(),
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
/// Handle the loop unroll optimization pragmas.
@@ -2994,7 +3008,7 @@ void PragmaUnrollHintHandler::HandlePragma(Preprocessor &PP,
TokenArray[0].setAnnotationEndLoc(PragmaName.getLocation());
TokenArray[0].setAnnotationValue(static_cast<void *>(Info));
PP.EnterTokenStream(std::move(TokenArray), 1,
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
/// Handle the Microsoft \#pragma intrinsic extension.
@@ -3266,5 +3280,5 @@ void PragmaAttributeHandler::HandlePragma(Preprocessor &PP,
TokenArray[0].setAnnotationEndLoc(FirstToken.getLocation());
TokenArray[0].setAnnotationValue(static_cast<void *>(Info));
PP.EnterTokenStream(std::move(TokenArray), 1,
- /*DisableMacroExpansion=*/false);
+ /*DisableMacroExpansion=*/false, /*IsReinject=*/false);
}
diff --git a/clang/lib/Parse/ParseStmtAsm.cpp b/clang/lib/Parse/ParseStmtAsm.cpp
index 3c4bc07321d..c63808a472b 100644
--- a/clang/lib/Parse/ParseStmtAsm.cpp
+++ b/clang/lib/Parse/ParseStmtAsm.cpp
@@ -213,7 +213,8 @@ ExprResult Parser::ParseMSAsmIdentifier(llvm::SmallVectorImpl<Token> &LineToks,
// Also copy the current token over.
LineToks.push_back(Tok);
- PP.EnterTokenStream(LineToks, /*DisableMacroExpansions*/ true);
+ PP.EnterTokenStream(LineToks, /*DisableMacroExpansions*/ true,
+ /*IsReinject*/ true);
// Clear the current token and advance to the first token in LineToks.
ConsumeAnyToken();
diff --git a/clang/lib/Parse/ParseTemplate.cpp b/clang/lib/Parse/ParseTemplate.cpp
index b53e6ea09bb..78e0c04f53e 100644
--- a/clang/lib/Parse/ParseTemplate.cpp
+++ b/clang/lib/Parse/ParseTemplate.cpp
@@ -919,7 +919,7 @@ bool Parser::ParseGreaterThanInTemplateList(SourceLocation &RAngleLoc,
PrevTokLocation = RAngleLoc;
} else {
PrevTokLocation = TokBeforeGreaterLoc;
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok = Greater;
}
@@ -1402,7 +1402,7 @@ void Parser::ParseLateTemplatedFuncDef(LateParsedTemplate &LPT) {
// Append the current token at the end of the new token stream so that it
// doesn't get lost.
LPT.Toks.push_back(Tok);
- PP.EnterTokenStream(LPT.Toks, true);
+ PP.EnterTokenStream(LPT.Toks, true, /*IsReinject*/true);
// Consume the previously pushed token.
ConsumeAnyToken(/*ConsumeCodeCompletionTok=*/true);
diff --git a/clang/lib/Parse/Parser.cpp b/clang/lib/Parse/Parser.cpp
index 0c8e203caf4..9124f155866 100644
--- a/clang/lib/Parse/Parser.cpp
+++ b/clang/lib/Parse/Parser.cpp
@@ -1545,7 +1545,7 @@ void Parser::AnnotateScopeToken(CXXScopeSpec &SS, bool IsNewAnnotation) {
if (PP.isBacktrackEnabled())
PP.RevertCachedTokens(1);
else
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok.setKind(tok::annot_cxxscope);
Tok.setAnnotationValue(Actions.SaveNestedNameSpecifierAnnotation(SS));
Tok.setAnnotationRange(SS.getRange());
@@ -1764,7 +1764,7 @@ bool Parser::TryAnnotateTypeOrScopeToken() {
Token TypedefToken;
PP.Lex(TypedefToken);
bool Result = TryAnnotateTypeOrScopeToken();
- PP.EnterToken(Tok);
+ PP.EnterToken(Tok, /*IsReinject=*/true);
Tok = TypedefToken;
if (!Result)
Diag(Tok.getLocation(), diag::warn_expected_qualified_after_typename);
diff --git a/clang/lib/Rewrite/HTMLRewrite.cpp b/clang/lib/Rewrite/HTMLRewrite.cpp
index 783c020f618..688dd7f7875 100644
--- a/clang/lib/Rewrite/HTMLRewrite.cpp
+++ b/clang/lib/Rewrite/HTMLRewrite.cpp
@@ -572,7 +572,7 @@ void html::HighlightMacros(Rewriter &R, FileID FID, const Preprocessor& PP) {
// Enter the tokens we just lexed. This will cause them to be macro expanded
// but won't enter sub-files (because we removed #'s).
- TmpPP.EnterTokenStream(TokenStream, false);
+ TmpPP.EnterTokenStream(TokenStream, false, /*IsReinject=*/false);
TokenConcatenation ConcatInfo(TmpPP);
OpenPOWER on IntegriCloud