summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--clang/include/clang/Basic/TokenKinds.def8
-rw-r--r--clang/include/clang/Lex/Preprocessor.h23
-rw-r--r--clang/include/clang/Lex/Token.h69
-rw-r--r--clang/lib/Lex/PPCaching.cpp24
4 files changed, 117 insertions, 7 deletions
diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index a557d5b9b95..706c5fb85f1 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -35,6 +35,9 @@
#ifndef OBJC2_AT_KEYWORD
#define OBJC2_AT_KEYWORD(X)
#endif
+#ifndef ANNOTATION
+#define ANNOTATION(X) TOK(annot_ ## X)
+#endif
//===----------------------------------------------------------------------===//
// Preprocessor keywords.
@@ -365,6 +368,11 @@ OBJC2_AT_KEYWORD(dynamic)
// TODO: What to do about context-sensitive keywords like:
// bycopy/byref/in/inout/oneway/out?
+ANNOTATION(cxxscope) // annotation for a C++ scope spec, e.g. "::foo::bar::"
+ANNOTATION(qualtypename) // annotation for a C typedef name, or a C++ (possibly
+ // qualified) typename, e.g. "foo::MyClass"
+
+#undef ANNOTATION
#undef OBJC2_AT_KEYWORD
#undef OBJC1_AT_KEYWORD
#undef CXX_KEYWORD_OPERATOR
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 4c8fa67e943..c0caee660eb 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -352,6 +352,28 @@ public:
else
return PeekAhead(N+1);
}
+
+ /// EnterToken - Enters a token in the token stream to be lexed next. If
+ /// BackTrack() is called afterwards, the token will remain at the insertion
+ /// point.
+ void EnterToken(const Token &Tok) {
+ EnterCachingLexMode();
+ CachedTokens.insert(CachedTokens.begin()+CachedLexPos, Tok);
+ }
+
+ /// AnnotateCachedTokens - We notify the Preprocessor that if it is caching
+ /// tokens (because backtrack is enabled) it should replace the most recent
+ /// cached tokens with the given annotation token. This function has no effect
+ /// if backtracking is not enabled.
+ ///
+ /// Note that the use of this function is just for optimization; so that the
+ /// cached tokens doesn't get re-parsed and re-resolved after a backtrack is
+ /// invoked.
+ void AnnotateCachedTokens(const Token &Tok) {
+ assert(Tok.isAnnotationToken() && "Expected annotation token");
+ if (CachedLexPos != 0 && InCachingLexMode())
+ AnnotatePreviousCachedTokens(Tok);
+ }
/// Diag - Forwarding function for diagnostics. This emits a diagnostic at
/// the specified Token's location, translating the token's start
@@ -559,6 +581,7 @@ private:
RemoveTopOfLexerStack();
}
const Token &PeekAhead(unsigned N);
+ void AnnotatePreviousCachedTokens(const Token &Tok);
//===--------------------------------------------------------------------===//
/// Handle*Directive - implement the various preprocessor directives. These
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index 45d8f2b5d46..1dd6d7bef38 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -25,14 +25,31 @@ class IdentifierInfo;
/// It is not intended to be space efficient, it is intended to return as much
/// information as possible about each returned token. This is expected to be
/// compressed into a smaller form if memory footprint is important.
+///
+/// The parser can create a special "annotation token" representing a stream of
+/// tokens that were parsed and semantically resolved, e.g.: "foo::MyClass<int>"
+/// can be represented by a single typename annotation token that carries
+/// information about the SourceRange of the tokens and the type object.
class Token {
- /// The location and length of the token text itself.
+ /// The location of the token.
SourceLocation Loc;
- unsigned Length;
+
+ union {
+ /// The end of the SourceRange of an annotation token.
+ unsigned AnnotEndLocID;
+
+ /// The length of the token text itself.
+ unsigned Length;
+ };
- /// IdentifierInfo - If this was an identifier, this points to the uniqued
- /// information about this identifier.
- IdentifierInfo *IdentInfo;
+ union {
+ /// IdentifierInfo - If this was an identifier, this points to the uniqued
+ /// information about this identifier.
+ IdentifierInfo *IdentInfo;
+
+ /// AnnotVal - Information specific to an annotation token.
+ void *AnnotVal;
+ };
/// Kind - The actual flavor of token this is.
///
@@ -60,13 +77,39 @@ public:
bool is(tok::TokenKind K) const { return Kind == (unsigned) K; }
bool isNot(tok::TokenKind K) const { return Kind != (unsigned) K; }
+ bool isAnnotationToken() const {
+ return is(tok::annot_qualtypename) || is(tok::annot_cxxscope);
+ }
+
/// getLocation - Return a source location identifier for the specified
/// offset in the current file.
SourceLocation getLocation() const { return Loc; }
- unsigned getLength() const { return Length; }
+ unsigned getLength() const {
+ assert(!isAnnotationToken() && "Used Length on annotation token");
+ return Length;
+ }
void setLocation(SourceLocation L) { Loc = L; }
void setLength(unsigned Len) { Length = Len; }
+
+ SourceLocation getAnnotationEndLoc() const {
+ assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token");
+ return SourceLocation::getFromRawEncoding(AnnotEndLocID);
+ }
+ void setAnnotationEndLoc(SourceLocation L) {
+ assert(isAnnotationToken() && "Used AnnotEndLocID on non-annotation token");
+ AnnotEndLocID = L.getRawEncoding();
+ }
+
+ /// getAnnotationRange - SourceRange of the group of tokens that this
+ /// annotation token represents.
+ SourceRange getAnnotationRange() const {
+ return SourceRange(getLocation(), getAnnotationEndLoc());
+ }
+ void setAnnotationRange(SourceRange R) {
+ setLocation(R.getBegin());
+ setAnnotationEndLoc(R.getEnd());
+ }
const char *getName() const {
return tok::getTokenName( (tok::TokenKind) Kind);
@@ -80,10 +123,22 @@ public:
Loc = SourceLocation();
}
- IdentifierInfo *getIdentifierInfo() const { return IdentInfo; }
+ IdentifierInfo *getIdentifierInfo() const {
+ assert(!isAnnotationToken() && "Used IdentInfo on annotation token");
+ return IdentInfo;
+ }
void setIdentifierInfo(IdentifierInfo *II) {
IdentInfo = II;
}
+
+ void *getAnnotationValue() const {
+ assert(isAnnotationToken() && "Used AnnotVal on non-annotation token");
+ return AnnotVal;
+ }
+ void setAnnotationValue(void *val) {
+ assert(isAnnotationToken() && "Used AnnotVal on non-annotation token");
+ AnnotVal = val;
+ }
/// setFlag - Set the specified flag.
void setFlag(TokenFlags Flag) {
diff --git a/clang/lib/Lex/PPCaching.cpp b/clang/lib/Lex/PPCaching.cpp
index 822b207cdcb..1af79b4ba37 100644
--- a/clang/lib/Lex/PPCaching.cpp
+++ b/clang/lib/Lex/PPCaching.cpp
@@ -93,3 +93,27 @@ const Token &Preprocessor::PeekAhead(unsigned N) {
EnterCachingLexMode();
return CachedTokens.back();
}
+
+void Preprocessor::AnnotatePreviousCachedTokens(const Token &Tok) {
+ assert(Tok.isAnnotationToken() && "Expected annotation token");
+ assert(CachedLexPos != 0 && "Expected to have some cached tokens");
+ assert(CachedTokens[CachedLexPos-1].getLocation() == Tok.getAnnotationEndLoc()
+ && "The annotation should be until the most recent cached token");
+
+ // Start from the end of the cached tokens list and look for the token
+ // that is the beginning of the annotation token.
+ for (CachedTokensTy::size_type i = CachedLexPos; i != 0; --i) {
+ CachedTokensTy::iterator AnnotBegin = CachedTokens.begin() + i-1;
+ if (AnnotBegin->getLocation() == Tok.getLocation()) {
+ assert((BacktrackPositions.empty() || BacktrackPositions.back() < i) &&
+ "The backtrack pos points inside the annotated tokens!");
+ // Replace the cached tokens with the single annotation token.
+ CachedTokens.erase(AnnotBegin + 1, CachedTokens.begin() + CachedLexPos);
+ *AnnotBegin = Tok;
+ CachedLexPos = i;
+ return;
+ }
+ }
+
+ assert(0&&"Didn't find the first token represented by the annotation token!");
+}
OpenPOWER on IntegriCloud