diff options
Diffstat (limited to 'clang/Lex/Lexer.cpp')
| -rw-r--r-- | clang/Lex/Lexer.cpp | 1473 |
1 files changed, 1473 insertions, 0 deletions
diff --git a/clang/Lex/Lexer.cpp b/clang/Lex/Lexer.cpp new file mode 100644 index 00000000000..df7539bc022 --- /dev/null +++ b/clang/Lex/Lexer.cpp @@ -0,0 +1,1473 @@ +//===--- Lexer.cpp - C Language Family Lexer ------------------------------===// +// +// The LLVM Compiler Infrastructure +// +// This file was developed by Chris Lattner and is distributed under +// the University of Illinois Open Source License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file implements the Lexer and LexerToken interfaces. +// +//===----------------------------------------------------------------------===// +// +// TODO: GCC Diagnostics emitted by the lexer: +// PEDWARN: (form feed|vertical tab) in preprocessing directive +// +// Universal characters, unicode, char mapping: +// WARNING: `%.*s' is not in NFKC +// WARNING: `%.*s' is not in NFC +// +// Other: +// ERROR : attempt to use poisoned \"%s\" +// +// TODO: Options to support: +// -fexec-charset,-fwide-exec-charset +// +//===----------------------------------------------------------------------===// + +#include "clang/Lex/Lexer.h" +#include "clang/Lex/Preprocessor.h" +#include "clang/Basic/Diagnostic.h" +#include "clang/Basic/SourceBuffer.h" +#include "clang/Basic/SourceLocation.h" +#include "llvm/Config/alloca.h" +#include <cassert> +#include <cctype> +#include <iostream> +using namespace llvm; +using namespace clang; + +static void InitCharacterInfo(); + +Lexer::Lexer(const SourceBuffer *File, unsigned fileid, Preprocessor &pp) + : BufferPtr(File->getBufferStart()), BufferStart(BufferPtr), + BufferEnd(File->getBufferEnd()), InputFile(File), CurFileID(fileid), PP(pp), + Features(PP.getLangOptions()) { + InitCharacterInfo(); + + assert(BufferEnd[0] == 0 && + "We assume that the input buffer has a null character at the end" + " to simplify lexing!"); + + // Start of the file is a start of line. + IsAtStartOfLine = true; + + // We are not after parsing a #. + ParsingPreprocessorDirective = false; + + // We are not after parsing #include. + ParsingFilename = false; +} + +//===----------------------------------------------------------------------===// +// LexerToken implementation. +//===----------------------------------------------------------------------===// + +/// getSourceLocation - Return a source location identifier for the specified +/// offset in the current file. +SourceLocation LexerToken::getSourceLocation() const { + if (TheLexer) + return TheLexer->getSourceLocation(Start); + return SourceLocation(); +} + + +/// dump - Print the token to stderr, used for debugging. +/// +void LexerToken::dump(bool DumpFlags) const { + std::cerr << clang::tok::getTokenName(Kind) << " '"; + + if (needsCleaning()) { + if (getLexer()) + std::cerr << getLexer()->getSpelling(*this); + else { + // FIXME: expansion from macros clears location info. Testcase: + // #define TWELVE 1\ <whitespace only> + // 2 + // TWELVE + std::cerr << "*unspelled*" << std::string(Start, End); + } + } else + std::cerr << std::string(Start, End); + std::cerr << "'"; + + if (DumpFlags) { + std::cerr << "\t"; + if (isAtStartOfLine()) + std::cerr << " [StartOfLine]"; + if (hasLeadingSpace()) + std::cerr << " [LeadingSpace]"; + if (needsCleaning()) + std::cerr << " [Spelling='" << std::string(Start, End) << "']"; + } +} + +//===----------------------------------------------------------------------===// +// Character information. +//===----------------------------------------------------------------------===// + +static unsigned char CharInfo[256]; + +enum { + CHAR_HORZ_WS = 0x01, // ' ', '\t', '\f', '\v'. Note, no '\0' + CHAR_VERT_WS = 0x02, // '\r', '\n' + CHAR_LETTER = 0x04, // a-z,A-Z + CHAR_NUMBER = 0x08, // 0-9 + CHAR_UNDER = 0x10, // _ + CHAR_PERIOD = 0x20 // . +}; + +static void InitCharacterInfo() { + static bool isInited = false; + if (isInited) return; + isInited = true; + + // Intiialize the CharInfo table. + // TODO: statically initialize this. + CharInfo[(int)' '] = CharInfo[(int)'\t'] = + CharInfo[(int)'\f'] = CharInfo[(int)'\v'] = CHAR_HORZ_WS; + CharInfo[(int)'\n'] = CharInfo[(int)'\r'] = CHAR_VERT_WS; + + CharInfo[(int)'_'] = CHAR_UNDER; + for (unsigned i = 'a'; i <= 'z'; ++i) + CharInfo[i] = CharInfo[i+'A'-'a'] = CHAR_LETTER; + for (unsigned i = '0'; i <= '9'; ++i) + CharInfo[i] = CHAR_NUMBER; +} + +/// isIdentifierBody - Return true if this is the body character of an +/// identifier, which is [a-zA-Z0-9_]. +static inline bool isIdentifierBody(unsigned char c) { + return CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER); +} + +/// isHorizontalWhitespace - Return true if this character is horizontal +/// whitespace: ' ', '\t', '\f', '\v'. Note that this returns false for '\0'. +static inline bool isHorizontalWhitespace(unsigned char c) { + return CharInfo[c] & CHAR_HORZ_WS; +} + +/// isWhitespace - Return true if this character is horizontal or vertical +/// whitespace: ' ', '\t', '\f', '\v', '\n', '\r'. Note that this returns false +/// for '\0'. +static inline bool isWhitespace(unsigned char c) { + return CharInfo[c] & (CHAR_HORZ_WS|CHAR_VERT_WS); +} + +/// isNumberBody - Return true if this is the body character of an +/// preprocessing number, which is [a-zA-Z0-9_.]. +static inline bool isNumberBody(unsigned char c) { + return CharInfo[c] & (CHAR_LETTER|CHAR_NUMBER|CHAR_UNDER|CHAR_PERIOD); +} + +//===----------------------------------------------------------------------===// +// Diagnostics forwarding code. +//===----------------------------------------------------------------------===// + +/// getSourceLocation - Return a source location identifier for the specified +/// offset in the current file. +SourceLocation Lexer::getSourceLocation(const char *Loc) const { + assert(Loc >= InputFile->getBufferStart() && Loc <= InputFile->getBufferEnd() + && "Location out of range for this buffer!"); + return SourceLocation(CurFileID, Loc-InputFile->getBufferStart()); +} + + +/// Diag - Forwarding function for diagnostics. This translate a source +/// position in the current buffer into a SourceLocation object for rendering. +bool Lexer::Diag(const char *Loc, unsigned DiagID, + const std::string &Msg) const { + return PP.Diag(getSourceLocation(Loc), DiagID, Msg); +} + +//===----------------------------------------------------------------------===// +// Trigraph and Escaped Newline Handling Code. +//===----------------------------------------------------------------------===// + +/// GetTrigraphCharForLetter - Given a character that occurs after a ?? pair, +/// return the decoded trigraph letter it corresponds to, or '\0' if nothing. +static char GetTrigraphCharForLetter(char Letter) { + switch (Letter) { + default: return 0; + case '=': return '#'; + case ')': return ']'; + case '(': return '['; + case '!': return '|'; + case '\'': return '^'; + case '>': return '}'; + case '/': return '\\'; + case '<': return '{'; + case '-': return '~'; + } +} + +/// DecodeTrigraphChar - If the specified character is a legal trigraph when +/// prefixed with ??, emit a trigraph warning. If trigraphs are enabled, +/// return the result character. Finally, emit a warning about trigraph use +/// whether trigraphs are enabled or not. +static char DecodeTrigraphChar(const char *CP, Lexer *L) { + char Res = GetTrigraphCharForLetter(*CP); + if (Res && L) { + if (!L->getFeatures().Trigraphs) { + L->Diag(CP-2, diag::trigraph_ignored); + return 0; + } else { + L->Diag(CP-2, diag::trigraph_converted, std::string()+Res); + } + } + return Res; +} + +/// getCharAndSizeSlow - Peek a single 'character' from the specified buffer, +/// get its size, and return it. This is tricky in several cases: +/// 1. If currently at the start of a trigraph, we warn about the trigraph, +/// then either return the trigraph (skipping 3 chars) or the '?', +/// depending on whether trigraphs are enabled or not. +/// 2. If this is an escaped newline (potentially with whitespace between +/// the backslash and newline), implicitly skip the newline and return +/// the char after it. +/// 3. If this is a UCN, return it. FIXME: for C++? +/// +/// This handles the slow/uncommon case of the getCharAndSize method. Here we +/// know that we can accumulate into Size, and that we have already incremented +/// Ptr by Size bytes. +/// +/// When this method is updated, getCharAndSizeSlowNoWarn (below) should be +/// updated to match. +/// +char Lexer::getCharAndSizeSlow(const char *Ptr, unsigned &Size, + LexerToken *Tok) { + // If we have a slash, look for an escaped newline. + if (Ptr[0] == '\\') { + ++Size; + ++Ptr; +Slash: + // Common case, backslash-char where the char is not whitespace. + if (!isWhitespace(Ptr[0])) return '\\'; + + // See if we have optional whitespace characters followed by a newline. + { + unsigned SizeTmp = 0; + do { + ++SizeTmp; + if (Ptr[SizeTmp-1] == '\n' || Ptr[SizeTmp-1] == '\r') { + // Remember that this token needs to be cleaned. + if (Tok) Tok->SetFlag(LexerToken::NeedsCleaning); + + // Warn if there was whitespace between the backslash and newline. + if (SizeTmp != 1 && Tok) + Diag(Ptr, diag::backslash_newline_space); + + // If this is a \r\n or \n\r, skip the newlines. + if ((Ptr[SizeTmp] == '\r' || Ptr[SizeTmp] == '\n') && + Ptr[SizeTmp-1] != Ptr[SizeTmp]) + ++SizeTmp; + + // Found backslash<whitespace><newline>. Parse the char after it. + Size += SizeTmp; + Ptr += SizeTmp; + // Use slow version to accumulate a correct size field. + return getCharAndSizeSlow(Ptr, Size, Tok); + } + } while (isWhitespace(Ptr[SizeTmp])); + } + + // Otherwise, this is not an escaped newline, just return the slash. + return '\\'; + } + + // If this is a trigraph, process it. + if (Ptr[0] == '?' && Ptr[1] == '?') { + // If this is actually a legal trigraph (not something like "??x"), emit + // a trigraph warning. If so, and if trigraphs are enabled, return it. + if (char C = DecodeTrigraphChar(Ptr+2, Tok ? this : 0)) { + // Remember that this token needs to be cleaned. + if (Tok) Tok->SetFlag(LexerToken::NeedsCleaning); + + Ptr += 3; + Size += 3; + if (C == '\\') goto Slash; + return C; + } + } + + // If this is neither, return a single character. + ++Size; + return *Ptr; +} + +/// getCharAndSizeSlowNoWarn - Handle the slow/uncommon case of the +/// getCharAndSizeNoWarn method. Here we know that we can accumulate into Size, +/// and that we have already incremented Ptr by Size bytes. +/// +/// When this method is updated, getCharAndSizeSlow (above) should be updated to +/// match. +static char getCharAndSizeSlowNoWarn(const char *Ptr, unsigned &Size, + const LangOptions &Features) { + // If we have a slash, look for an escaped newline. + if (Ptr[0] == '\\') { + ++Size; + ++Ptr; +Slash: + // Common case, backslash-char where the char is not whitespace. + if (!isWhitespace(Ptr[0])) return '\\'; + + // See if we have optional whitespace characters followed by a newline. + { + unsigned SizeTmp = 0; + do { + ++SizeTmp; + if (Ptr[SizeTmp-1] == '\n' || Ptr[SizeTmp-1] == '\r') { + + // If this is a \r\n or \n\r, skip the newlines. + if ((Ptr[SizeTmp] == '\r' || Ptr[SizeTmp] == '\n') && + Ptr[SizeTmp-1] != Ptr[SizeTmp]) + ++SizeTmp; + + // Found backslash<whitespace><newline>. Parse the char after it. + Size += SizeTmp; + Ptr += SizeTmp; + + // Use slow version to accumulate a correct size field. + return getCharAndSizeSlowNoWarn(Ptr, Size, Features); + } + } while (isWhitespace(Ptr[SizeTmp])); + } + + // Otherwise, this is not an escaped newline, just return the slash. + return '\\'; + } + + // If this is a trigraph, process it. + if (Features.Trigraphs && Ptr[0] == '?' && Ptr[1] == '?') { + // If this is actually a legal trigraph (not something like "??x"), return + // it. + if (char C = GetTrigraphCharForLetter(Ptr[2])) { + Ptr += 3; + Size += 3; + if (C == '\\') goto Slash; + return C; + } + } + + // If this is neither, return a single character. + ++Size; + return *Ptr; +} + +/// getCharAndSizeNoWarn - Like the getCharAndSize method, but does not ever +/// emit a warning. +static inline char getCharAndSizeNoWarn(const char *Ptr, unsigned &Size, + const LangOptions &Features) { + // If this is not a trigraph and not a UCN or escaped newline, return + // quickly. + if (Ptr[0] != '?' && Ptr[0] != '\\') { + Size = 1; + return *Ptr; + } + + Size = 0; + return getCharAndSizeSlowNoWarn(Ptr, Size, Features); +} + + +/// getSpelling() - Return the 'spelling' of this token. The spelling of a +/// token are the characters used to represent the token in the source file +/// after trigraph expansion and escaped-newline folding. In particular, this +/// wants to get the true, uncanonicalized, spelling of things like digraphs +/// UCNs, etc. +std::string Lexer::getSpelling(const LexerToken &Tok, + const LangOptions &Features) { + assert(Tok.getStart() <= Tok.getEnd() && "Token character range is bogus!"); + + // If this token contains nothing interesting, return it directly. + if (!Tok.needsCleaning()) + return std::string(Tok.getStart(), Tok.getEnd()); + + // Otherwise, hard case, relex the characters into the string. + std::string Result; + Result.reserve(Tok.getEnd()-Tok.getStart()); + + for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) { + unsigned CharSize; + Result.push_back(getCharAndSizeNoWarn(Ptr, CharSize, Features)); + Ptr += CharSize; + } + assert(Result.size() != unsigned(Tok.getEnd()-Tok.getStart()) && + "NeedsCleaning flag set on something that didn't need cleaning!"); + return Result; +} + +/// getSpelling - This method is used to get the spelling of a token into a +/// preallocated buffer, instead of as an std::string. The caller is required +/// to allocate enough space for the token, which is guaranteed to be at most +/// Tok.End-Tok.Start bytes long. The actual length of the token is returned. +unsigned Lexer::getSpelling(const LexerToken &Tok, char *Buffer, + const LangOptions &Features) { + assert(Tok.getStart() <= Tok.getEnd() && "Token character range is bogus!"); + + // If this token contains nothing interesting, return it directly. + if (!Tok.needsCleaning()) { + unsigned Size = Tok.getEnd()-Tok.getStart(); + memcpy(Buffer, Tok.getStart(), Size); + return Size; + } + // Otherwise, hard case, relex the characters into the string. + std::string Result; + Result.reserve(Tok.getEnd()-Tok.getStart()); + + char *OutBuf = Buffer; + for (const char *Ptr = Tok.getStart(), *End = Tok.getEnd(); Ptr != End; ) { + unsigned CharSize; + *OutBuf++ = getCharAndSizeNoWarn(Ptr, CharSize, Features); + Ptr += CharSize; + } + assert(OutBuf-Buffer != Tok.getEnd()-Tok.getStart() && + "NeedsCleaning flag set on something that didn't need cleaning!"); + + return OutBuf-Buffer; +} + + +//===----------------------------------------------------------------------===// +// Helper methods for lexing. +//===----------------------------------------------------------------------===// + +bool Lexer::LexIdentifier(LexerToken &Result, const char *CurPtr) { + // Match [_A-Za-z0-9]*, we have already matched [_A-Za-z$] + unsigned Size; + unsigned char C = *CurPtr++; + while (isIdentifierBody(C)) { + C = *CurPtr++; + } + --CurPtr; // Back up over the skipped character. + + // Fast path, no $,\,? in identifier found. '\' might be an escaped newline + // or UCN, and ? might be a trigraph for '\', an escaped newline or UCN. + // FIXME: universal chars. + if (C != '\\' && C != '?' && (C != '$' || !Features.DollarIdents)) { +FinishIdentifier: + Result.SetEnd(BufferPtr = CurPtr); + Result.SetKind(tok::identifier); + + // Look up this token, see if it is a macro, or if it is a language keyword. + const char *SpelledTokStart, *SpelledTokEnd; + if (!Result.needsCleaning()) { + // No cleaning needed, just use the characters from the lexed buffer. + SpelledTokStart = Result.getStart(); + SpelledTokEnd = Result.getEnd(); + } else { + // Cleaning needed, alloca a buffer, clean into it, then use the buffer. + char *TmpBuf = (char*)alloca(Result.getEnd()-Result.getStart()); + unsigned Size = getSpelling(Result, TmpBuf); + SpelledTokStart = TmpBuf; + SpelledTokEnd = TmpBuf+Size; + } + + Result.SetIdentifierInfo(PP.getIdentifierInfo(SpelledTokStart, + SpelledTokEnd)); + return PP.HandleIdentifier(Result); + } + + // Otherwise, $,\,? in identifier found. Enter slower path. + + C = getCharAndSize(CurPtr, Size); + while (1) { + if (C == '$') { + // If we hit a $ and they are not supported in identifiers, we are done. + if (!Features.DollarIdents) goto FinishIdentifier; + + // Otherwise, emit a diagnostic and continue. + if (Diag(CurPtr, diag::ext_dollar_in_identifier)) + return true; + CurPtr = ConsumeChar(CurPtr, Size, Result); + C = getCharAndSize(CurPtr, Size); + continue; + } else if (!isIdentifierBody(C)) { // FIXME: universal chars. + // Found end of identifier. + goto FinishIdentifier; + } + + // Otherwise, this character is good, consume it. + CurPtr = ConsumeChar(CurPtr, Size, Result); + + C = getCharAndSize(CurPtr, Size); + while (isIdentifierBody(C)) { // FIXME: universal chars. + CurPtr = ConsumeChar(CurPtr, Size, Result); + C = getCharAndSize(CurPtr, Size); + } + } +} + + +/// LexNumericConstant - Lex the remainer of a integer or floating point +/// constant. From[-1] is the first character lexed. Return the end of the +/// constant. +bool Lexer::LexNumericConstant(LexerToken &Result, const char *CurPtr) { + unsigned Size; + char C = getCharAndSize(CurPtr, Size); + char PrevCh = 0; + while (isNumberBody(C)) { // FIXME: universal chars? + CurPtr = ConsumeChar(CurPtr, Size, Result); + PrevCh = C; + C = getCharAndSize(CurPtr, Size); + } + + // If we fell out, check for a sign, due to 1e+12. If we have one, continue. + if ((C == '-' || C == '+') && (PrevCh == 'E' || PrevCh == 'e')) + return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); + + // If we have a hex FP constant, continue. + if (Features.HexFloats && + (C == '-' || C == '+') && (PrevCh == 'P' || PrevCh == 'p')) + return LexNumericConstant(Result, ConsumeChar(CurPtr, Size, Result)); + + Result.SetKind(tok::numeric_constant); + + // Update the end of token position as well as the BufferPtr instance var. + Result.SetEnd(BufferPtr = CurPtr); + return false; +} + +/// LexStringLiteral - Lex the remainder of a string literal, after having lexed +/// either " or L". +bool Lexer::LexStringLiteral(LexerToken &Result, const char *CurPtr) { + const char *NulCharacter = 0; // Does this string contain the \0 character? + + char C = getAndAdvanceChar(CurPtr, Result); + while (C != '"') { + // Skip escaped characters. + if (C == '\\') { + // Skip the escaped character. + C = getAndAdvanceChar(CurPtr, Result); + } else if (C == '\n' || C == '\r' || // Newline. + (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. + if (Diag(Result.getStart(), diag::err_unterminated_string)) + return true; + BufferPtr = CurPtr-1; + return LexTokenInternal(Result); + } else if (C == 0) { + NulCharacter = CurPtr-1; + } + C = getAndAdvanceChar(CurPtr, Result); + } + + if (NulCharacter && Diag(NulCharacter, diag::null_in_string)) + return true; + + Result.SetKind(tok::string_literal); + + // Update the end of token position as well as the BufferPtr instance var. + Result.SetEnd(BufferPtr = CurPtr); + return false; +} + +/// LexAngledStringLiteral - Lex the remainder of an angled string literal, +/// after having lexed the '<' character. This is used for #include filenames. +bool Lexer::LexAngledStringLiteral(LexerToken &Result, const char *CurPtr) { + const char *NulCharacter = 0; // Does this string contain the \0 character? + + char C = getAndAdvanceChar(CurPtr, Result); + while (C != '>') { + // Skip escaped characters. + if (C == '\\') { + // Skip the escaped character. + C = getAndAdvanceChar(CurPtr, Result); + } else if (C == '\n' || C == '\r' || // Newline. + (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. + if (Diag(Result.getStart(), diag::err_unterminated_string)) + return true; + BufferPtr = CurPtr-1; + return LexTokenInternal(Result); + } else if (C == 0) { + NulCharacter = CurPtr-1; + } + C = getAndAdvanceChar(CurPtr, Result); + } + + if (NulCharacter && Diag(NulCharacter, diag::null_in_string)) + return true; + + Result.SetKind(tok::angle_string_literal); + + // Update the end of token position as well as the BufferPtr instance var. + Result.SetEnd(BufferPtr = CurPtr); + return false; +} + + +/// LexCharConstant - Lex the remainder of a character constant, after having +/// lexed either ' or L'. +bool Lexer::LexCharConstant(LexerToken &Result, const char *CurPtr) { + const char *NulCharacter = 0; // Does this character contain the \0 character? + + // Handle the common case of 'x' and '\y' efficiently. + char C = getAndAdvanceChar(CurPtr, Result); + if (C == '\'') { + if (Diag(Result.getStart(), diag::err_empty_character)) + return true; + BufferPtr = CurPtr; + return LexTokenInternal(Result); + } else if (C == '\\') { + // Skip the escaped character. + // FIXME: UCN's. + C = getAndAdvanceChar(CurPtr, Result); + } + + if (C && C != '\n' && C != '\r' && CurPtr[0] == '\'') { + ++CurPtr; + } else { + // Fall back on generic code for embedded nulls, newlines, wide chars. + do { + // Skip escaped characters. + if (C == '\\') { + // Skip the escaped character. + C = getAndAdvanceChar(CurPtr, Result); + } else if (C == '\n' || C == '\r' || // Newline. + (C == 0 && CurPtr-1 == BufferEnd)) { // End of file. + if (Diag(Result.getStart(), diag::err_unterminated_char)) + return true; + BufferPtr = CurPtr-1; + return LexTokenInternal(Result); + } else if (C == 0) { + NulCharacter = CurPtr-1; + } + C = getAndAdvanceChar(CurPtr, Result); + } while (C != '\''); + } + + if (NulCharacter && Diag(NulCharacter, diag::null_in_char)) + return true; + + Result.SetKind(tok::char_constant); + + // Update the end of token position as well as the BufferPtr instance var. + Result.SetEnd(BufferPtr = CurPtr); + return false; +} + +/// SkipWhitespace - Efficiently skip over a series of whitespace characters. +/// Update BufferPtr to point to the next non-whitespace character and return. +bool Lexer::SkipWhitespace(LexerToken &Result, const char *CurPtr) { + // Whitespace - Skip it, then return the token after the whitespace. + unsigned char Char = *CurPtr; // Skip consequtive spaces efficiently. + while (1) { + // Skip horizontal whitespace very aggressively. + while (isHorizontalWhitespace(Char)) + Char = *++CurPtr; + + // Otherwise if we something other than whitespace, we're done. + if (Char != '\n' && Char != '\r') + break; + + if (ParsingPreprocessorDirective) { + // End of preprocessor directive line, let LexTokenInternal handle this. + BufferPtr = CurPtr; + return false; + } + + // ok, but handle newline. + // The returned token is at the start of the line. + Result.SetFlag(LexerToken::StartOfLine); + // No leading whitespace seen so far. + Result.ClearFlag(LexerToken::LeadingSpace); + Char = *++CurPtr; + } + + // If this isn't immediately after a newline, there is leading space. + char PrevChar = CurPtr[-1]; + if (PrevChar != '\n' && PrevChar != '\r') + Result.SetFlag(LexerToken::LeadingSpace); + + // If the next token is obviously a // or /* */ comment, skip it efficiently + // too (without going through the big switch stmt). + if (Char == '/' && CurPtr[1] == '/') { + Result.SetStart(CurPtr); + return SkipBCPLComment(Result, CurPtr+1); + } + if (Char == '/' && CurPtr[1] == '*') { + Result.SetStart(CurPtr); + return SkipBlockComment(Result, CurPtr+2); + } + BufferPtr = CurPtr; + return false; +} + +// SkipBCPLComment - We have just read the // characters from input. Skip until +// we find the newline character thats terminate the comment. Then update +/// BufferPtr and return. +bool Lexer::SkipBCPLComment(LexerToken &Result, const char *CurPtr) { + // If BCPL comments aren't explicitly enabled for this language, emit an + // extension warning. + if (!Features.BCPLComment) { + if (Diag(Result.getStart(), diag::ext_bcpl_comment)) + return true; + + // Mark them enabled so we only emit one warning for this translation + // unit. + Features.BCPLComment = true; + } + + // Scan over the body of the comment. The common case, when scanning, is that + // the comment contains normal ascii characters with nothing interesting in + // them. As such, optimize for this case with the inner loop. + char C; + do { + C = *CurPtr; + // FIXME: just scan for a \n or \r character. If we find a \n character, + // scan backwards, checking to see if it's an escaped newline, like we do + // for block comments. + + // Skip over characters in the fast loop. + while (C != 0 && // Potentially EOF. + C != '\\' && // Potentially escaped newline. + C != '?' && // Potentially trigraph. + C != '\n' && C != '\r') // Newline or DOS-style newline. + C = *++CurPtr; + + // If this is a newline, we're done. + if (C == '\n' || C == '\r') + break; // Found the newline? Break out! + + // Otherwise, this is a hard case. Fall back on getAndAdvanceChar to + // properly decode the character. + const char *OldPtr = CurPtr; + C = getAndAdvanceChar(CurPtr, Result); + + // If we read multiple characters, and one of those characters was a \r or + // \n, then we had an escaped newline within the comment. Emit diagnostic. + if (CurPtr != OldPtr+1) { + for (; OldPtr != CurPtr; ++OldPtr) + if (OldPtr[0] == '\n' || OldPtr[0] == '\r') { + if (Diag(OldPtr-1, diag::ext_multi_line_bcpl_comment)) + return true; + } + } + + if (CurPtr == BufferEnd+1) goto FoundEOF; + } while (C != '\n' && C != '\r'); + + // Found and did not consume a newline. + + // If we are inside a preprocessor directive and we see the end of line, + // return immediately, so that the lexer can return this as an EOM token. + if (ParsingPreprocessorDirective) { + BufferPtr = CurPtr; + return false; + } + + // Otherwise, eat the \n character. We don't care if this is a \n\r or + // \r\n sequence. + ++CurPtr; + + // The next returned token is at the start of the line. + Result.SetFlag(LexerToken::StartOfLine); + // No leading whitespace seen so far. + Result.ClearFlag(LexerToken::LeadingSpace); + + // It is common for the tokens immediately after a // comment to be + // whitespace (indentation for the next line). Instead of going through the + // big switch, handle it efficiently now. + if (isWhitespace(*CurPtr)) { + Result.SetFlag(LexerToken::LeadingSpace); + return SkipWhitespace(Result, CurPtr+1); + } + + BufferPtr = CurPtr; + return false; + +FoundEOF: // If we ran off the end of the buffer, return EOF. + BufferPtr = CurPtr-1; + return false; +} + +/// isEndOfEscapedNewLine - Return true if the specified newline character +/// (either \n or \r) is part of an escaped newline sequence. Issue a +/// diagnostic if so. We know that the is inside of a block comment. +bool Lexer::isBlockCommentEndOfEscapedNewLine(const char *CurPtr, + char &PrevChar) { + assert(CurPtr[0] == '\n' || CurPtr[0] == '\r'); + PrevChar = 0; + + // Back up off the newline. + --CurPtr; + + // If this is a two-character newline sequence, skip the other character. + if (CurPtr[0] == '\n' || CurPtr[0] == '\r') { + // \n\n or \r\r -> not escaped newline. + if (CurPtr[0] == CurPtr[1]) + return false; + // \n\r or \r\n -> skip the newline. + --CurPtr; + } + + // If we have horizontal whitespace, skip over it. We allow whitespace + // between the slash and newline. + bool HasSpace = false; + while (isHorizontalWhitespace(*CurPtr) || *CurPtr == 0) { + --CurPtr; + HasSpace = true; + } + + // If we have a slash, we know this is an escaped newline. + if (*CurPtr == '\\') { + PrevChar = CurPtr[-1]; + if (PrevChar != '*') return false; + } else { + // It isn't a slash, is it the ?? / trigraph? + if (*CurPtr != '/' || CurPtr[-1] != '?' || CurPtr[-2] != '?') + return false; + // This is the trigraph. Emit a stern warning! + if ((PrevChar = CurPtr[-3]) != '*') return false; + CurPtr -= 2; + + // If no trigraphs are enabled, warn that we ignored this trigraph and + // ignore this * character. + if (!Features.Trigraphs) { + PrevChar = 0; + return Diag(CurPtr, diag::trigraph_ignored_block_comment); + } else { + if (Diag(CurPtr, diag::trigraph_ends_block_comment)) + return true; + } + } + + // Warn about having an escaped newline between the */ characters. + if (Diag(CurPtr, diag::escaped_newline_block_comment_end)) + return true; + + // If there was space between the backslash and newline, warn about it. + if (HasSpace && + Diag(CurPtr, diag::backslash_newline_space)) + return true; + + return false; +} + +/// SkipBlockComment - We have just read the /* characters from input. Read +/// until we find the */ characters that terminate the comment. Note that we +/// don't bother decoding trigraphs or escaped newlines in block comments, +/// because they cannot cause the comment to end. The only thing that can +/// happen is the comment could end with an escaped newline between the */ end +/// of comment. +bool Lexer::SkipBlockComment(LexerToken &Result, const char *CurPtr) { + // Scan one character past where we should, looking for a '/' character. Once + // we find it, check to see if it was preceeded by a *. This common + // optimization helps people who like to put a lot of * characters in their + // comments. + unsigned char C = *CurPtr++; + if (C == 0 && CurPtr == BufferEnd+1) { + if (Diag(Result.getStart(), diag::err_unterminated_block_comment)) + return true; + BufferPtr = CurPtr-1; + return false; + } + + while (1) { + // Skip over all non-interesting characters. + // TODO: Vectorize this. Note: memchr on Darwin is slower than this loop. + while (C != '/' && C != '\0') + C = *CurPtr++; + + if (C == '/') { + char T; + if (CurPtr[-2] == '*') // We found the final */. We're done! + break; + + if ((CurPtr[-2] == '\n' || CurPtr[-2] == '\r')) { + char Prev; + if (isBlockCommentEndOfEscapedNewLine(CurPtr-2, Prev)) + return true; + if (Prev == '*') { + // We found the final */, though it had an escaped newline between the + // * and /. We're done! + break; + } + } + if (CurPtr[0] == '*' && CurPtr[1] != '/') { + // If this is a /* inside of the comment, emit a warning. Don't do this + // if this is a /*/, which will end the comment. This misses cases with + // embedded escaped newlines, but oh well. + if (Diag(CurPtr-1, diag::nested_block_comment)) + return true; + } + } else if (C == 0 && CurPtr == BufferEnd+1) { + if (Diag(Result.getStart(), diag::err_unterminated_block_comment)) + return true; + // Note: the user probably forgot a */. We could continue immediately + // after the /*, but this would involve lexing a lot of what really is the + // comment, which surely would confuse the parser. + BufferPtr = CurPtr-1; + return false; + } + C = *CurPtr++; + } + + // It is common for the tokens immediately after a /**/ comment to be + // whitespace. Instead of going through the big switch, handle it + // efficiently now. + if (isHorizontalWhitespace(*CurPtr)) { + Result.SetFlag(LexerToken::LeadingSpace); + return SkipWhitespace(Result, CurPtr+1); + } + + // Otherwise, just return so that the next character will be lexed as a token. + BufferPtr = CurPtr; + Result.SetFlag(LexerToken::LeadingSpace); + return false; +} + +//===----------------------------------------------------------------------===// +// Primary Lexing Entry Points +//===----------------------------------------------------------------------===// + +/// LexIncludeFilename - After the preprocessor has parsed a #include, lex and +/// (potentially) macro expand the filename. +bool Lexer::LexIncludeFilename(LexerToken &Result) { + assert(ParsingPreprocessorDirective && + ParsingFilename == false && + "Must be in a preprocessing directive!"); + + // We are now parsing a filename! + ParsingFilename = true; + + // There should be exactly two tokens here if everything is good: first the + // filename, then the EOM. + if (Lex(Result)) return true; + + // We should have gotten the filename now. + ParsingFilename = false; + + // No filename? + if (Result.getKind() == tok::eom) + return Diag(Result.getStart(), diag::err_pp_expects_filename); + + // Verify that there is nothing after the filename, other than EOM. + LexerToken EndTok; + if (Lex(EndTok)) return true; + + if (EndTok.getKind() != tok::eom) { + if (Diag(Result.getStart(), diag::err_pp_expects_filename)) + return true; + + // Lex until the end of the preprocessor directive line. + while (EndTok.getKind() != tok::eom) { + if (Lex(EndTok)) return true; + } + + Result.SetKind(tok::eom); + } + + // We're done now. + return false; +} + +/// ReadToEndOfLine - Read the rest of the current preprocessor line as an +/// uninterpreted string. This switches the lexer out of directive mode. +std::string Lexer::ReadToEndOfLine() { + assert(ParsingPreprocessorDirective && ParsingFilename == false && + "Must be in a preprocessing directive!"); + std::string Result; + LexerToken Tmp; + + // CurPtr - Cache BufferPtr in an automatic variable. + const char *CurPtr = BufferPtr; + Tmp.SetStart(CurPtr); + + while (1) { + char Char = getAndAdvanceChar(CurPtr, Tmp); + switch (Char) { + default: + Result += Char; + break; + case 0: // Null. + // Found end of file? + if (CurPtr-1 != BufferEnd) { + // Nope, normal character, continue. + Result += Char; + break; + } + // FALL THROUGH. + case '\r': + case '\n': + // Okay, we found the end of the line. First, back up past the \0, \r, \n. + assert(CurPtr[-1] == Char && "Trigraphs for newline?"); + BufferPtr = CurPtr-1; + + // Next, lex the character, which should handle the EOM transition. + bool Err = Lex(Tmp); + assert(Tmp.getKind() == tok::eom && "Unexpected token!"); + assert(!Err && "Shouldn't have error exiting macro!"); + + // Finally, we're done, return the string we found. + return Result; + } + } +} + +/// LexEndOfFile - CurPtr points to the end of this file. Handle this +/// condition, reporting diagnostics and handling other edge cases as required. +bool Lexer::LexEndOfFile(LexerToken &Result, const char *CurPtr) { + // If we hit the end of the file while parsing a preprocessor directive, + // end the preprocessor directive first. The next token returned will + // then be the end of file. + if (ParsingPreprocessorDirective) { + // Done parsing the "line". + ParsingPreprocessorDirective = false; + Result.SetKind(tok::eom); + // Update the end of token position as well as the BufferPtr instance var. + Result.SetEnd(BufferPtr = CurPtr); + return false; + } + + // If we are in a #if directive, emit an error. + while (!ConditionalStack.empty()) { + if (Diag(ConditionalStack.back().IfLoc, + diag::err_pp_unterminated_conditional)) + return true; + ConditionalStack.pop_back(); + } + + // If the file was empty or didn't end in a newline, issue a pedwarn. + if (CurPtr[-1] != '\n' && CurPtr[-1] != '\r' && + Diag(BufferEnd, diag::ext_no_newline_eof)) + return true; + + BufferPtr = CurPtr; + return PP.HandleEndOfFile(Result); +} + + +/// LexTokenInternal - This implements a simple C family lexer. It is an +/// extremely performance critical piece of code. This assumes that the buffer +/// has a null character at the end of the file. Return true if an error +/// occurred and compilation should terminate, false if normal. This returns a +/// preprocessing token, not a normal token, as such, it is an internal +/// interface. It assumes that the Flags of result have been cleared before +/// calling this. +bool Lexer::LexTokenInternal(LexerToken &Result) { +LexNextToken: + // New token, can't need cleaning yet. + Result.ClearFlag(LexerToken::NeedsCleaning); + + // CurPtr - Cache BufferPtr in an automatic variable. + const char *CurPtr = BufferPtr; + Result.SetStart(CurPtr); + + unsigned SizeTmp, SizeTmp2; // Temporaries for use in cases below. + + // Read a character, advancing over it. + char Char = getAndAdvanceChar(CurPtr, Result); + switch (Char) { + case 0: // Null. + // Found end of file? + if (CurPtr-1 == BufferEnd) + return LexEndOfFile(Result, CurPtr-1); // Retreat back into the file. + + if (Diag(CurPtr-1, diag::null_in_file)) + return true; + Result.SetFlag(LexerToken::LeadingSpace); + if (SkipWhitespace(Result, CurPtr)) return true; + goto LexNextToken; // GCC isn't tail call eliminating. + case '\n': + case '\r': + // If we are inside a preprocessor directive and we see the end of line, + // we know we are done with the directive, so return an EOM token. + if (ParsingPreprocessorDirective) { + // Done parsing the "line". + ParsingPreprocessorDirective = false; + + // Since we consumed a newline, we are back at the start of a line. + IsAtStartOfLine = true; + + Result.SetKind(tok::eom); + break; + } + // The returned token is at the start of the line. + Result.SetFlag(LexerToken::StartOfLine); + // No leading whitespace seen so far. + Result.ClearFlag(LexerToken::LeadingSpace); + if (SkipWhitespace(Result, CurPtr)) return true; + goto LexNextToken; // GCC isn't tail call eliminating. + case ' ': + case '\t': + case '\f': + case '\v': + Result.SetFlag(LexerToken::LeadingSpace); + if (SkipWhitespace(Result, CurPtr)) return true; + goto LexNextToken; // GCC isn't tail call eliminating. + + case 'L': + Char = getCharAndSize(CurPtr, SizeTmp); + + // Wide string literal. + if (Char == '"') + return LexStringLiteral(Result, ConsumeChar(CurPtr, SizeTmp, Result)); + + // Wide character constant. + if (Char == '\'') + return LexCharConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); + // FALL THROUGH, treating L like the start of an identifier. + + // C99 6.4.2: Identifiers. + case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': + case 'H': case 'I': case 'J': case 'K': /*'L'*/case 'M': case 'N': + case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': + case 'V': case 'W': case 'X': case 'Y': case 'Z': + case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': + case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': + case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': + case 'v': case 'w': case 'x': case 'y': case 'z': + case '_': + return LexIdentifier(Result, CurPtr); + + // C99 6.4.4.1: Integer Constants. + // C99 6.4.4.2: Floating Constants. + case '0': case '1': case '2': case '3': case '4': + case '5': case '6': case '7': case '8': case '9': + return LexNumericConstant(Result, CurPtr); + + // C99 6.4.4: Character Constants. + case '\'': + return LexCharConstant(Result, CurPtr); + + // C99 6.4.5: String Literals. + case '"': + return LexStringLiteral(Result, CurPtr); + + // C99 6.4.6: Punctuators. + case '?': + Result.SetKind(tok::question); + break; + case '[': + Result.SetKind(tok::l_square); + break; + case ']': + Result.SetKind(tok::r_square); + break; + case '(': + Result.SetKind(tok::l_paren); + break; + case ')': + Result.SetKind(tok::r_paren); + break; + case '{': + Result.SetKind(tok::l_brace); + break; + case '}': + Result.SetKind(tok::r_brace); + break; + case '.': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char >= '0' && Char <= '9') { + return LexNumericConstant(Result, ConsumeChar(CurPtr, SizeTmp, Result)); + } else if (Features.CPlusPlus && Char == '*') { + Result.SetKind(tok::periodstar); + CurPtr += SizeTmp; + } else if (Char == '.' && + getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '.') { + Result.SetKind(tok::ellipsis); + CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), + SizeTmp2, Result); + } else { + Result.SetKind(tok::period); + } + break; + case '&': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '&') { + Result.SetKind(tok::ampamp); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Char == '=') { + Result.SetKind(tok::ampequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::amp); + } + break; + case '*': + if (getCharAndSize(CurPtr, SizeTmp) == '=') { + Result.SetKind(tok::starequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::star); + } + break; + case '+': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '+') { + Result.SetKind(tok::plusplus); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Char == '=') { + Result.SetKind(tok::plusequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::plus); + } + break; + case '-': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '-') { + Result.SetKind(tok::minusminus); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Char == '>' && Features.CPlusPlus && + getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '*') { + Result.SetKind(tok::arrowstar); // C++ ->* + CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), + SizeTmp2, Result); + } else if (Char == '>') { + Result.SetKind(tok::arrow); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Char == '=') { + Result.SetKind(tok::minusequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::minus); + } + break; + case '~': + Result.SetKind(tok::tilde); + break; + case '!': + if (getCharAndSize(CurPtr, SizeTmp) == '=') { + Result.SetKind(tok::exclaimequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::exclaim); + } + break; + case '/': + // 6.4.9: Comments + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '/') { // BCPL comment. + Result.SetFlag(LexerToken::LeadingSpace); + if (SkipBCPLComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) + return true; + goto LexNextToken; // GCC isn't tail call eliminating. + } else if (Char == '*') { // /**/ comment. + Result.SetFlag(LexerToken::LeadingSpace); + if (SkipBlockComment(Result, ConsumeChar(CurPtr, SizeTmp, Result))) + return true; + goto LexNextToken; // GCC isn't tail call eliminating. + } else if (Char == '=') { + Result.SetKind(tok::slashequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::slash); + } + break; + case '%': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '=') { + Result.SetKind(tok::percentequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Features.Digraphs && Char == '>') { + Result.SetKind(tok::r_brace); // '%>' -> '}' + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Features.Digraphs && Char == ':') { + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + if (getCharAndSize(CurPtr, SizeTmp) == '%' && + getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == ':') { + Result.SetKind(tok::hashhash); // '%:%:' -> '##' + CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), + SizeTmp2, Result); + } else { + Result.SetKind(tok::hash); // '%:' -> '#' + + // We parsed a # character. If this occurs at the start of the line, + // it's actually the start of a preprocessing directive. Callback to + // the preprocessor to handle it. + // FIXME: -fpreprocessed mode?? + if (Result.isAtStartOfLine() && !PP.isSkipping()) { + BufferPtr = CurPtr; + if (PP.HandleDirective(Result)) return true; + + // As an optimization, if the preprocessor didn't switch lexers, tail + // recurse. + if (PP.isCurrentLexer(this)) { + // Start a new token. If this is a #include or something, the PP may + // want us starting at the beginning of the line again. If so, set + // the StartOfLine flag. + if (IsAtStartOfLine) { + Result.SetFlag(LexerToken::StartOfLine); + IsAtStartOfLine = false; + } + goto LexNextToken; // GCC isn't tail call eliminating. + } + + return PP.Lex(Result); + } + } + } else { + Result.SetKind(tok::percent); + } + break; + case '<': + Char = getCharAndSize(CurPtr, SizeTmp); + if (ParsingFilename) { + return LexAngledStringLiteral(Result, CurPtr+SizeTmp); + } else if (Char == '<' && + getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '=') { + Result.SetKind(tok::lesslessequal); + CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), + SizeTmp2, Result); + } else if (Char == '<') { + Result.SetKind(tok::lessless); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Char == '=') { + Result.SetKind(tok::lessequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Features.Digraphs && Char == ':') { + Result.SetKind(tok::l_square); // '<:' -> '[' + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Features.Digraphs && Char == '>') { + Result.SetKind(tok::l_brace); // '<%' -> '{' + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Features.CPPMinMax && Char == '?') { // <? + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + if (Diag(Result.getStart(), diag::min_max_deprecated)) + return true; + + if (getCharAndSize(CurPtr, SizeTmp) == '=') { // <?= + Result.SetKind(tok::lessquestionequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::lessquestion); + } + } else { + Result.SetKind(tok::less); + } + break; + case '>': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '=') { + Result.SetKind(tok::greaterequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Char == '>' && + getCharAndSize(CurPtr+SizeTmp, SizeTmp2) == '=') { + Result.SetKind(tok::greatergreaterequal); + CurPtr = ConsumeChar(ConsumeChar(CurPtr, SizeTmp, Result), + SizeTmp2, Result); + } else if (Char == '>') { + Result.SetKind(tok::greatergreater); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Features.CPPMinMax && Char == '?') { + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + if (Diag(Result.getStart(), diag::min_max_deprecated)) + return true; + + if (getCharAndSize(CurPtr, SizeTmp) == '=') { + Result.SetKind(tok::greaterquestionequal); // >?= + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::greaterquestion); // >? + } + } else { + Result.SetKind(tok::greater); + } + break; + case '^': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '=') { + Result.SetKind(tok::caretequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::caret); + } + break; + case '|': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '=') { + Result.SetKind(tok::pipeequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Char == '|') { + Result.SetKind(tok::pipepipe); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::pipe); + } + break; + case ':': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Features.Digraphs && Char == '>') { + Result.SetKind(tok::r_square); // ':>' -> ']' + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else if (Features.CPlusPlus && Char == ':') { + Result.SetKind(tok::coloncolon); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::colon); + } + break; + case ';': + Result.SetKind(tok::semi); + break; + case '=': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '=') { + Result.SetKind(tok::equalequal); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::equal); + } + break; + case ',': + Result.SetKind(tok::comma); + break; + case '#': + Char = getCharAndSize(CurPtr, SizeTmp); + if (Char == '#') { + Result.SetKind(tok::hashhash); + CurPtr = ConsumeChar(CurPtr, SizeTmp, Result); + } else { + Result.SetKind(tok::hash); + // We parsed a # character. If this occurs at the start of the line, + // it's actually the start of a preprocessing directive. Callback to + // the preprocessor to handle it. + // FIXME: not in preprocessed mode?? + if (Result.isAtStartOfLine() && !PP.isSkipping()) { + BufferPtr = CurPtr; + if (PP.HandleDirective(Result)) return true; + + // As an optimization, if the preprocessor didn't switch lexers, tail + // recurse. + if (PP.isCurrentLexer(this)) { + // Start a new token. If this is a #include or something, the PP may + // want us starting at the beginning of the line again. If so, set + // the StartOfLine flag. + if (IsAtStartOfLine) { + Result.SetFlag(LexerToken::StartOfLine); + IsAtStartOfLine = false; + } + goto LexNextToken; // GCC isn't tail call eliminating. + } + return PP.Lex(Result); + } + } + break; + + case '\\': + // FIXME: handle UCN's. + // FALL THROUGH. + default: + // Objective C support. + if (CurPtr[-1] == '@' && Features.ObjC1) { + Result.SetKind(tok::at); + break; + } else if (CurPtr[-1] == '$' && Features.DollarIdents) {// $ in identifiers. + if (Diag(CurPtr-1, diag::ext_dollar_in_identifier)) + return true; + return LexIdentifier(Result, CurPtr); + } + + if (!PP.isSkipping() && Diag(CurPtr-1, diag::err_stray_character)) + return true; + BufferPtr = CurPtr; + goto LexNextToken; // GCC isn't tail call eliminating. + } + + // Update the end of token position as well as the BufferPtr instance var. + Result.SetEnd(BufferPtr = CurPtr); + return false; +} |
