summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--clang/Lex/MacroExpander.cpp110
-rw-r--r--clang/Lex/Preprocessor.cpp42
-rw-r--r--clang/include/clang/Basic/SourceManager.h13
-rw-r--r--clang/include/clang/Lex/MacroExpander.h30
-rw-r--r--clang/include/clang/Lex/Preprocessor.h11
5 files changed, 166 insertions, 40 deletions
diff --git a/clang/Lex/MacroExpander.cpp b/clang/Lex/MacroExpander.cpp
index 58e76a5259c..50552b94c17 100644
--- a/clang/Lex/MacroExpander.cpp
+++ b/clang/Lex/MacroExpander.cpp
@@ -70,6 +70,40 @@ bool MacroArgs::ArgNeedsPreexpansion(unsigned ArgNo) const {
return false;
}
+/// getPreExpArgument - Return the pre-expanded form of the specified
+/// argument.
+const std::vector<LexerToken> &
+MacroArgs::getPreExpArgument(unsigned Arg, Preprocessor &PP) {
+ assert(Arg < UnexpArgTokens.size() && "Invalid argument number!");
+
+ // If we have already computed this, return it.
+ if (PreExpArgTokens.empty())
+ PreExpArgTokens.resize(UnexpArgTokens.size());
+
+ std::vector<LexerToken> &Result = PreExpArgTokens[Arg];
+ if (!Result.empty()) return Result;
+
+ // Otherwise, we have to pre-expand this argument, populating Result. To do
+ // this, we set up a fake MacroExpander to lex from the unexpanded argument
+ // list. With this installed, we lex expanded tokens until we hit the EOF
+ // token at the end of the unexp list.
+ PP.EnterTokenStream(UnexpArgTokens[Arg]);
+
+ // Lex all of the macro-expanded tokens into Result.
+ do {
+ Result.push_back(LexerToken());
+ PP.Lex(Result.back());
+ } while (Result.back().getKind() != tok::eof);
+
+ // Pop the token stream off the top of the stack. We know that the internal
+ // pointer inside of it is to the "end" of the token stream, but the stack
+ // will not otherwise be popped until the next token is lexed. The problem is
+ // that the token may be lexed sometime after the vector of tokens itself is
+ // destroyed, which would be badness.
+ PP.RemoveTopOfLexerStack();
+ return Result;
+}
+
/// StringifyArgument - Implement C99 6.10.3.2p2, converting a sequence of
/// tokens into the literal string token that should be produced by the C #
@@ -83,9 +117,9 @@ static LexerToken StringifyArgument(const std::vector<LexerToken> &Toks,
// Stringify all the tokens.
std::string Result = "\"";
+ // FIXME: Optimize this loop to not use std::strings.
for (unsigned i = 0, e = Toks.size()-1 /*no eof*/; i != e; ++i) {
const LexerToken &Tok = Toks[i];
- // FIXME: Optimize this.
if (i != 0 && Tok.hasLeadingSpace())
Result += ' ';
@@ -163,25 +197,49 @@ const LexerToken &MacroArgs::getStringifiedArgument(unsigned ArgNo,
// MacroExpander Implementation
//===----------------------------------------------------------------------===//
+/// Create a macro expander for the specified macro with the specified actual
+/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
MacroExpander::MacroExpander(LexerToken &Tok, MacroArgs *Actuals,
Preprocessor &pp)
- : Macro(*Tok.getIdentifierInfo()->getMacroInfo()),
+ : Macro(Tok.getIdentifierInfo()->getMacroInfo()),
ActualArgs(Actuals), PP(pp), CurToken(0),
InstantiateLoc(Tok.getLocation()),
AtStartOfLine(Tok.isAtStartOfLine()),
HasLeadingSpace(Tok.hasLeadingSpace()) {
- MacroTokens = &Macro.getReplacementTokens();
+ MacroTokens = &Macro->getReplacementTokens();
// If this is a function-like macro, expand the arguments and change
// MacroTokens to point to the expanded tokens.
- if (Macro.isFunctionLike() && Macro.getNumArgs())
+ if (Macro->isFunctionLike() && Macro->getNumArgs())
ExpandFunctionArguments();
+
+ // Mark the macro as currently disabled, so that it is not recursively
+ // expanded. The macro must be disabled only after argument pre-expansion of
+ // function-like macro arguments occurs.
+ Macro->DisableMacro();
}
+/// Create a macro expander for the specified token stream. This does not
+/// take ownership of the specified token vector.
+MacroExpander::MacroExpander(const std::vector<LexerToken> &TokStream,
+ Preprocessor &pp)
+ : Macro(0), ActualArgs(0), PP(pp), MacroTokens(&TokStream), CurToken(0),
+ InstantiateLoc(SourceLocation()), AtStartOfLine(false),
+ HasLeadingSpace(false) {
+
+ // Set HasLeadingSpace/AtStartOfLine so that the first token will be
+ // returned unmodified.
+ if (!TokStream.empty()) {
+ AtStartOfLine = TokStream[0].isAtStartOfLine();
+ HasLeadingSpace = TokStream[0].hasLeadingSpace();
+ }
+}
+
+
MacroExpander::~MacroExpander() {
// If this was a function-like macro that actually uses its arguments, delete
// the expanded tokens.
- if (MacroTokens != &Macro.getReplacementTokens())
+ if (Macro && MacroTokens != &Macro->getReplacementTokens())
delete MacroTokens;
// MacroExpander owns its formal arguments.
@@ -205,7 +263,7 @@ void MacroExpander::ExpandFunctionArguments() {
// when the #define was parsed.
const LexerToken &CurTok = (*MacroTokens)[i];
if (CurTok.getKind() == tok::hash || CurTok.getKind() == tok::hashat) {
- int ArgNo = Macro.getArgumentNum((*MacroTokens)[i+1].getIdentifierInfo());
+ int ArgNo =Macro->getArgumentNum((*MacroTokens)[i+1].getIdentifierInfo());
assert(ArgNo != -1 && "Token following # is not an argument?");
if (CurTok.getKind() == tok::hash) // Stringify
@@ -227,7 +285,7 @@ void MacroExpander::ExpandFunctionArguments() {
// Otherwise, if this is not an argument token, just add the token to the
// output buffer.
IdentifierInfo *II = CurTok.getIdentifierInfo();
- int ArgNo = II ? Macro.getArgumentNum(II) : -1;
+ int ArgNo = II ? Macro->getArgumentNum(II) : -1;
if (ArgNo == -1) {
ResultToks.push_back(CurTok);
continue;
@@ -251,14 +309,10 @@ void MacroExpander::ExpandFunctionArguments() {
const std::vector<LexerToken> *ArgToks;
// Only preexpand the argument if it could possibly need it. This
// avoids some work in common cases.
- if (ActualArgs->ArgNeedsPreexpansion(ArgNo)) {
- // FIXME: WRONG
- ArgToks = &ActualArgs->getUnexpArgument(ArgNo);
- } else {
- // If we don't need to pre-expand the argument, just substitute in the
- // unexpanded tokens.
+ if (ActualArgs->ArgNeedsPreexpansion(ArgNo))
+ ArgToks = &ActualArgs->getPreExpArgument(ArgNo, PP);
+ else
ArgToks = &ActualArgs->getUnexpArgument(ArgNo);
- }
unsigned FirstTok = ResultToks.size();
ResultToks.insert(ResultToks.end(), ArgToks->begin(), ArgToks->end()-1);
@@ -272,7 +326,12 @@ void MacroExpander::ExpandFunctionArguments() {
continue;
}
- // FIXME: handle pasted args.
+ // Okay, we have a token that is either the LHS or RHS of a paste (##)
+ // argument.
+
+ // FIXME: Handle comma swallowing GNU extension.
+
+ // FIXME: handle pasted args. Handle 'placemarker' stuff.
ResultToks.push_back(CurTok);
}
}
@@ -290,8 +349,15 @@ void MacroExpander::ExpandFunctionArguments() {
///
void MacroExpander::Lex(LexerToken &Tok) {
// Lexing off the end of the macro, pop this macro off the expansion stack.
- if (isAtEnd())
+ if (isAtEnd()) {
+ // If this is a macro (not a token stream), mark the macro enabled now
+ // that it is no longer being expanded.
+ if (Macro) Macro->EnableMacro();
+
+ // Pop this context off the preprocessors lexer stack and get the next
+ // token.
return PP.HandleEndOfMacro(Tok);
+ }
// Get the next token to return.
Tok = (*MacroTokens)[CurToken++];
@@ -301,9 +367,15 @@ void MacroExpander::Lex(LexerToken &Tok) {
// diagnostics for the expanded token should appear as if they came from
// InstantiationLoc. Pull this information together into a new SourceLocation
// that captures all of this.
- Tok.SetLocation(PP.getSourceManager().getInstantiationLoc(Tok.getLocation(),
- InstantiateLoc));
-
+ if (InstantiateLoc.isValid()) { // Don't do this for token streams.
+ SourceManager &SrcMgr = PP.getSourceManager();
+ // The token could have come from a prior macro expansion. In that case,
+ // ignore the macro expand part to get to the physloc. This happens for
+ // stuff like: #define A(X) X A(A(X)) A(1)
+ SourceLocation PhysLoc = SrcMgr.getPhysicalLoc(Tok.getLocation());
+ Tok.SetLocation(SrcMgr.getInstantiationLoc(PhysLoc, InstantiateLoc));
+ }
+
// If this is the first token, set the lexical properties of the token to
// match the lexical properties of the macro identifier.
if (CurToken == 1) {
diff --git a/clang/Lex/Preprocessor.cpp b/clang/Lex/Preprocessor.cpp
index 4b3989cf5cc..fab5c69b6d5 100644
--- a/clang/Lex/Preprocessor.cpp
+++ b/clang/Lex/Preprocessor.cpp
@@ -450,12 +450,38 @@ void Preprocessor::EnterMacro(LexerToken &Tok, MacroArgs *Args) {
CurLexer = 0;
CurDirLookup = 0;
- // Mark the macro as currently disabled, so that it is not recursively
- // expanded.
- MI.DisableMacro();
CurMacroExpander = new MacroExpander(Tok, Args, *this);
}
+/// EnterTokenStream - Add a "macro" context to the top of the include stack,
+/// which will cause the lexer to start returning the specified tokens. Note
+/// that these tokens will be re-macro-expanded when/if expansion is enabled.
+/// This method assumes that the specified stream of tokens has a permanent
+/// owner somewhere, so they do not need to be copied.
+void Preprocessor::EnterTokenStream(const std::vector<LexerToken> &Stream) {
+ // Save our current state.
+ IncludeMacroStack.push_back(IncludeStackInfo(CurLexer, CurDirLookup,
+ CurMacroExpander));
+ CurLexer = 0;
+ CurDirLookup = 0;
+
+ // Create a macro expander to expand from the specified token stream.
+ CurMacroExpander = new MacroExpander(Stream, *this);
+}
+
+/// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
+/// lexer stack. This should only be used in situations where the current
+/// state of the top-of-stack lexer is known.
+void Preprocessor::RemoveTopOfLexerStack() {
+ assert(!IncludeMacroStack.empty() && "Ran out of stack entries to load");
+ delete CurLexer;
+ delete CurMacroExpander;
+ CurLexer = IncludeMacroStack.back().TheLexer;
+ CurDirLookup = IncludeMacroStack.back().TheDirLookup;
+ CurMacroExpander = IncludeMacroStack.back().TheMacroExpander;
+ IncludeMacroStack.pop_back();
+}
+
//===----------------------------------------------------------------------===//
// Macro Expansion Handling.
//===----------------------------------------------------------------------===//
@@ -1011,11 +1037,7 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
// lexing the #includer file.
if (!IncludeMacroStack.empty()) {
// We're done with the #included file.
- delete CurLexer;
- CurLexer = IncludeMacroStack.back().TheLexer;
- CurDirLookup = IncludeMacroStack.back().TheDirLookup;
- CurMacroExpander = IncludeMacroStack.back().TheMacroExpander;
- IncludeMacroStack.pop_back();
+ RemoveTopOfLexerStack();
// Notify the client, if desired, that we are in a new source file.
if (FileChangeHandler && !isEndOfMacro && CurLexer) {
@@ -1050,13 +1072,11 @@ void Preprocessor::HandleEndOfFile(LexerToken &Result, bool isEndOfMacro) {
}
/// HandleEndOfMacro - This callback is invoked when the lexer hits the end of
-/// the current macro expansion.
+/// the current macro expansion or token stream expansion.
void Preprocessor::HandleEndOfMacro(LexerToken &Result) {
assert(CurMacroExpander && !CurLexer &&
"Ending a macro when currently in a #include file!");
- // Mark macro not ignored now that it is no longer being expanded.
- CurMacroExpander->getMacro().EnableMacro();
delete CurMacroExpander;
// Handle this like a #include file being popped off the stack.
diff --git a/clang/include/clang/Basic/SourceManager.h b/clang/include/clang/Basic/SourceManager.h
index 1628851cdd6..5343cf7dabe 100644
--- a/clang/include/clang/Basic/SourceManager.h
+++ b/clang/include/clang/Basic/SourceManager.h
@@ -262,6 +262,19 @@ public:
return Loc;
}
+ /// getPhysicalLoc - Given a SourceLocation object, return the physical
+ /// location referenced by the ID.
+ SourceLocation getPhysicalLoc(SourceLocation Loc) const {
+ if (Loc.getFileID() == 0) return Loc;
+
+ // For Macros, the physical loc is specified by the MacroTokenFileID.
+ const SrcMgr::FileIDInfo *FIDInfo = getFIDInfo(Loc.getFileID());
+ if (FIDInfo->IDType == SrcMgr::FileIDInfo::MacroExpansion)
+ return SourceLocation(FIDInfo->u.MacroTokenFileID,
+ Loc.getRawFilePos());
+ return Loc;
+ }
+
/// PrintStats - Print statistics to stderr.
///
void PrintStats() const;
diff --git a/clang/include/clang/Lex/MacroExpander.h b/clang/include/clang/Lex/MacroExpander.h
index 4ff4c34369c..1439a049c34 100644
--- a/clang/include/clang/Lex/MacroExpander.h
+++ b/clang/include/clang/Lex/MacroExpander.h
@@ -30,9 +30,10 @@ class MacroArgs {
/// an 'EOF' marker at the end of each argument.
std::vector<std::vector<LexerToken> > UnexpArgTokens;
- /// ExpArgTokens - Pre-expanded tokens for arguments that need them. Empty if
- /// not yet computed. This includes the EOF marker at the end of the stream.
- std::vector<std::vector<LexerToken> > ExpArgTokens;
+ /// PreExpArgTokens - Pre-expanded tokens for arguments that need them. Empty
+ /// if not yet computed. This includes the EOF marker at the end of the
+ /// stream.
+ std::vector<std::vector<LexerToken> > PreExpArgTokens;
/// StringifiedArgs - This contains arguments in 'stringified' form. If the
/// stringified form of an argument has not yet been computed, this is empty.
@@ -57,6 +58,11 @@ public:
return UnexpArgTokens[Arg];
}
+ /// getPreExpArgument - Return the pre-expanded form of the specified
+ /// argument.
+ const std::vector<LexerToken> &
+ getPreExpArgument(unsigned Arg, Preprocessor &PP);
+
/// getStringifiedArgument - Compute, cache, and return the specified argument
/// that has been 'stringified' as required by the # operator.
const LexerToken &getStringifiedArgument(unsigned ArgNo, Preprocessor &PP);
@@ -68,12 +74,13 @@ public:
/// MacroExpander - This implements a lexer that returns token from a macro body
-/// instead of lexing from a character buffer.
+/// or token stream instead of lexing from a character buffer.
///
class MacroExpander {
- /// Macro - The macro we are expanding from.
+ /// Macro - The macro we are expanding from. This is null if expanding a
+ /// token stream.
///
- MacroInfo &Macro;
+ MacroInfo *Macro;
/// ActualArgs - The actual arguments specified for a function-like macro, or
/// null. The MacroExpander owns the pointed-to object.
@@ -84,7 +91,8 @@ class MacroExpander {
Preprocessor &PP;
/// MacroTokens - This is the pointer to the list of tokens that the macro is
- /// defined to, with arguments expanded for function-like macros.
+ /// defined to, with arguments expanded for function-like macros. If this is
+ /// a token stream, this are the tokens we are returning.
const std::vector<LexerToken> *MacroTokens;
/// CurToken - This is the next token that Lex will return.
@@ -102,9 +110,13 @@ class MacroExpander {
MacroExpander(const MacroExpander&); // DO NOT IMPLEMENT
void operator=(const MacroExpander&); // DO NOT IMPLEMENT
public:
- /// Create a macro expander of the specified macro with the specified actual
+ /// Create a macro expander for the specified macro with the specified actual
/// arguments. Note that this ctor takes ownership of the ActualArgs pointer.
MacroExpander(LexerToken &Tok, MacroArgs *ActualArgs, Preprocessor &PP);
+
+ /// Create a macro expander for the specified token stream. This does not
+ /// take ownership of the specified token vector.
+ MacroExpander(const std::vector<LexerToken> &TokStream, Preprocessor &PP);
~MacroExpander();
/// isNextTokenLParen - If the next token lexed will pop this macro off the
@@ -112,8 +124,6 @@ public:
/// 1, otherwise return 0.
unsigned isNextTokenLParen() const;
- MacroInfo &getMacro() const { return Macro; }
-
/// Lex - Lex and return a token from this macro stream.
void Lex(LexerToken &Tok);
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 96d10cfbb5f..3a99e4029ff 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -325,6 +325,17 @@ public:
/// tokens input to a function-like macro.
void EnterMacro(LexerToken &Identifier, MacroArgs *Args);
+ /// EnterTokenStream - Add a "macro" context to the top of the include stack,
+ /// which will cause the lexer to start returning the specified tokens. Note
+ /// that these tokens will be re-macro-expanded when/if expansion is enabled.
+ /// This method assumes that the specified stream of tokens has a permanent
+ /// owner somewhere, so they do not need to be copied.
+ void EnterTokenStream(const std::vector<LexerToken> &Stream);
+
+ /// RemoveTopOfLexerStack - Pop the current lexer/macro exp off the top of the
+ /// lexer stack. This should only be used in situations where the current
+ /// state of the top-of-stack lexer is known.
+ void RemoveTopOfLexerStack();
/// Lex - To lex a token from the preprocessor, just pull a token from the
/// current lexer or macro object.
OpenPOWER on IntegriCloud