Factor out repeated code parsing and concatenating header-names from

tokens.

We now actually form an angled_string_literal token for a header name by
concatenation rather than just working out what its contents would be.
This substantially simplifies downstream processing and is necessary for
C++20 header unit imports.

llvm-svn: 356433

1 files changed, 74 insertions, 0 deletions

diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 7b4983dcfe4..ebe3797717c 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -895,6 +895,80 @@ void Preprocessor::Lex(Token &Result) {
   LastTokenWasAt = Result.is(tok::at);
 }
 
+/// Lex a header-name token (including one formed from header-name-tokens if
+/// \p AllowConcatenation is \c true).
+///
+/// \param FilenameTok Filled in with the next token. On success, this will
+///        be either an angle_header_name or a string_literal token. On
+///        failure, it will be whatever other token was found instead.
+/// \param AllowConcatenation If \c true, allow a < token, followed by other
+///        tokens and finally a > token, to form a single header-name token.
+/// \return \c true if we reached EOD or EOF while looking for a > token in
+///         a concatenated header name and diagnosed it. \c false otherwise.
+bool Preprocessor::LexHeaderName(Token &FilenameTok, bool AllowConcatenation) {
+  // Lex using header-name tokenization rules if tokens are being lexed from
+  // a file. Just grab a token normally if we're in a macro expansion.
+  if (CurPPLexer)
+    CurPPLexer->LexIncludeFilename(FilenameTok);
+  else
+    Lex(FilenameTok);
+
+  // This could be a <foo/bar.h> file coming from a macro expansion.  In this
+  // case, glue the tokens together into an angle_string_literal token.
+  if (FilenameTok.is(tok::less) && AllowConcatenation) {
+    SmallString<128> FilenameBuffer;
+    SourceLocation Start = FilenameTok.getLocation();
+    SourceLocation End;
+    FilenameBuffer.push_back('<');
+
+    // Consume tokens until we find a '>'.
+    while (FilenameTok.isNot(tok::greater)) {
+      Lex(FilenameTok);
+      if (FilenameTok.isOneOf(tok::eod, tok::eof)) {
+        Diag(FilenameTok.getLocation(), diag::err_expected) << tok::greater;
+        Diag(Start, diag::note_matching) << tok::less;
+        return true;
+      }
+
+      End = FilenameTok.getLocation();
+
+      // FIXME: Provide code completion for #includes.
+      if (FilenameTok.is(tok::code_completion)) {
+        setCodeCompletionReached();
+        Lex(FilenameTok);
+        continue;
+      }
+
+      // Append the spelling of this token to the buffer. If there was a space
+      // before it, add it now.
+      if (FilenameTok.hasLeadingSpace())
+        FilenameBuffer.push_back(' ');
+
+      // Get the spelling of the token, directly into FilenameBuffer if
+      // possible.
+      size_t PreAppendSize = FilenameBuffer.size();
+      FilenameBuffer.resize(PreAppendSize + FilenameTok.getLength());
+
+      const char *BufPtr = &FilenameBuffer[PreAppendSize];
+      unsigned ActualLen = getSpelling(FilenameTok, BufPtr);
+
+      // If the token was spelled somewhere else, copy it into FilenameBuffer.
+      if (BufPtr != &FilenameBuffer[PreAppendSize])
+        memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
+
+      // Resize FilenameBuffer to the correct size.
+      if (FilenameTok.getLength() != ActualLen)
+        FilenameBuffer.resize(PreAppendSize + ActualLen);
+    }
+
+    FilenameTok.startToken();
+    FilenameTok.setKind(tok::angle_string_literal);
+    CreateString(FilenameBuffer, FilenameTok, Start, End);
+  }
+
+  return false;
+}
+
 /// Lex a token following the 'import' contextual keyword.
 ///
 void Preprocessor::LexAfterModuleImport(Token &Result) {