Introduced raw_identifier token kind.

llvm-svn: 122394

3 files changed, 27 insertions, 5 deletions

diff --git a/clang/include/clang/Basic/TokenKinds.def b/clang/include/clang/Basic/TokenKinds.def
index abdd5b4b7a7..c61930e67ac 100644
--- a/clang/include/clang/Basic/TokenKinds.def
+++ b/clang/include/clang/Basic/TokenKinds.def
@@ -103,6 +103,7 @@ TOK(comment)             // Comment (only in -E -C[C] mode)
 
 // C99 6.4.2: Identifiers.
 TOK(identifier)          // abcde123
+TOK(raw_identifier)      // Used only in raw lexing mode.
 
 // C99 6.4.4.1: Integer Constants
 // C99 6.4.4.2: Floating Constants
diff --git a/clang/include/clang/Lex/Preprocessor.h b/clang/include/clang/Lex/Preprocessor.h
index 1ced6a58004..ca4b9fddabb 100644
--- a/clang/include/clang/Lex/Preprocessor.h
+++ b/clang/include/clang/Lex/Preprocessor.h
@@ -751,10 +751,10 @@ public:
   // Preprocessor callback methods.  These are invoked by a lexer as various
   // directives and events are found.
 
-  /// LookUpIdentifierInfo - Given a tok::identifier token, look up the
-  /// identifier information for the token and install it into the token.
-  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier,
-                                       const char *BufPtr = 0) const;
+  /// LookUpIdentifierInfo - Given a tok::raw_identifier token, look up the
+  /// identifier information for the token and install it into the token,
+  /// updating the token kind accordingly.
+  IdentifierInfo *LookUpIdentifierInfo(Token &Identifier) const;
 
   /// HandleIdentifier - This callback is invoked when the lexer reads an
   /// identifier and has filled in the tokens IdentifierInfo member.  This
diff --git a/clang/include/clang/Lex/Token.h b/clang/include/clang/Lex/Token.h
index 2a190839069..ef4f02cb575 100644
--- a/clang/include/clang/Lex/Token.h
+++ b/clang/include/clang/Lex/Token.h
@@ -88,6 +88,12 @@ public:
   bool is(tok::TokenKind K) const { return Kind == (unsigned) K; }
   bool isNot(tok::TokenKind K) const { return Kind != (unsigned) K; }
 
+  /// isAnyIdentifier - Return true if this is a raw identifier (when lexing
+  /// in raw mode) or a non-keyword identifier (when lexing in non-raw mode).
+  bool isAnyIdentifier() const {
+    return is(tok::identifier) || is(tok::raw_identifier);
+  }
+
   /// isLiteral - Return true if this is a "literal", like a numeric
   /// constant, string, etc.
   bool isLiteral() const {
@@ -154,7 +160,10 @@ public:
   }
 
   IdentifierInfo *getIdentifierInfo() const {
-    assert(!isAnnotation() && "Used IdentInfo on annotation token!");
+    assert(isNot(tok::raw_identifier) &&
+           "getIdentifierInfo() on a tok::raw_identifier token!");
+    assert(!isAnnotation() &&
+           "getIdentifierInfo() on an annotation token!");
     if (isLiteral()) return 0;
     return (IdentifierInfo*) PtrData;
   }
@@ -162,6 +171,18 @@ public:
     PtrData = (void*) II;
   }
 
+  /// getRawIdentifierData - For a raw identifier token (i.e., an identifier
+  /// lexed in raw mode), returns a pointer to the start of it in the text
+  /// buffer if known, null otherwise.
+  const char *getRawIdentifierData() const {
+    assert(is(tok::raw_identifier));
+    return reinterpret_cast<const char*>(PtrData);
+  }
+  void setRawIdentifierData(const char *Ptr) {
+    assert(is(tok::raw_identifier));
+    PtrData = const_cast<char*>(Ptr);
+  }
+
   /// getLiteralData - For a literal token (numeric constant, string, etc), this
   /// returns a pointer to the start of it in the text buffer if known, null
   /// otherwise.