Factor out repeated code parsing and concatenating header-names from

tokens.

We now actually form an angled_string_literal token for a header name by
concatenation rather than just working out what its contents would be.
This substantially simplifies downstream processing and is necessary for
C++20 header unit imports.

llvm-svn: 356433

1 files changed, 24 insertions, 106 deletions

diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 62796293b4d..40eb3af2573 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -1480,67 +1480,6 @@ bool Preprocessor::GetIncludeFilenameSpelling(SourceLocation Loc,
   return isAngled;
 }
 
-// Handle cases where the \#include name is expanded from a macro
-// as multiple tokens, which need to be glued together.
-//
-// This occurs for code like:
-// \code
-//    \#define FOO <a/b.h>
-//    \#include FOO
-// \endcode
-// because in this case, "<a/b.h>" is returned as 7 tokens, not one.
-//
-// This code concatenates and consumes tokens up to the '>' token.  It returns
-// false if the > was found, otherwise it returns true if it finds and consumes
-// the EOD marker.
-bool Preprocessor::ConcatenateIncludeName(SmallString<128> &FilenameBuffer,
-                                          SourceLocation &End) {
-  Token CurTok;
-
-  Lex(CurTok);
-  while (CurTok.isNot(tok::eod)) {
-    End = CurTok.getLocation();
-
-    // FIXME: Provide code completion for #includes.
-    if (CurTok.is(tok::code_completion)) {
-      setCodeCompletionReached();
-      Lex(CurTok);
-      continue;
-    }
-
-    // Append the spelling of this token to the buffer. If there was a space
-    // before it, add it now.
-    if (CurTok.hasLeadingSpace())
-      FilenameBuffer.push_back(' ');
-
-    // Get the spelling of the token, directly into FilenameBuffer if possible.
-    size_t PreAppendSize = FilenameBuffer.size();
-    FilenameBuffer.resize(PreAppendSize+CurTok.getLength());
-
-    const char *BufPtr = &FilenameBuffer[PreAppendSize];
-    unsigned ActualLen = getSpelling(CurTok, BufPtr);
-
-    // If the token was spelled somewhere else, copy it into FilenameBuffer.
-    if (BufPtr != &FilenameBuffer[PreAppendSize])
-      memcpy(&FilenameBuffer[PreAppendSize], BufPtr, ActualLen);
-
-    // Resize FilenameBuffer to the correct size.
-    if (CurTok.getLength() != ActualLen)
-      FilenameBuffer.resize(PreAppendSize+ActualLen);
-
-    // If we found the '>' marker, return success.
-    if (CurTok.is(tok::greater))
-      return false;
-
-    Lex(CurTok);
-  }
-
-  // If we hit the eod marker, emit an error and return true so that the caller
-  // knows the EOD has been read.
-  Diag(CurTok.getLocation(), diag::err_pp_expects_filename);
-  return true;
-}
-
 /// Push a token onto the token stream containing an annotation.
 void Preprocessor::EnterAnnotationToken(SourceRange Range,
                                         tok::TokenKind Kind,
@@ -1671,43 +1610,23 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
                                           const FileEntry *LookupFromFile,
                                           bool isImport) {
   Token FilenameTok;
-  CurPPLexer->LexIncludeFilename(FilenameTok);
-
-  // Reserve a buffer to get the spelling.
-  SmallString<128> FilenameBuffer;
-  StringRef Filename;
-  SourceLocation End;
-  SourceLocation CharEnd; // the end of this directive, in characters
-
-  switch (FilenameTok.getKind()) {
-  case tok::eod:
-    // If the token kind is EOD, the error has already been diagnosed.
+  if (LexHeaderName(FilenameTok))
     return;
 
-  case tok::angle_string_literal:
-  case tok::string_literal:
-    Filename = getSpelling(FilenameTok, FilenameBuffer);
-    End = FilenameTok.getLocation();
-    CharEnd = End.getLocWithOffset(FilenameTok.getLength());
-    break;
-
-  case tok::less:
-    // This could be a <foo/bar.h> file coming from a macro expansion.  In this
-    // case, glue the tokens together into FilenameBuffer and interpret those.
-    FilenameBuffer.push_back('<');
-    if (ConcatenateIncludeName(FilenameBuffer, End))
-      return;   // Found <eod> but no ">"?  Diagnostic already emitted.
-    Filename = FilenameBuffer;
-    CharEnd = End.getLocWithOffset(1);
-    break;
-  default:
+  if (!FilenameTok.isOneOf(tok::angle_string_literal, tok::string_literal)) {
     Diag(FilenameTok.getLocation(), diag::err_pp_expects_filename);
-    DiscardUntilEndOfDirective();
+    if (FilenameTok.isNot(tok::eod))
+      DiscardUntilEndOfDirective();
     return;
   }
 
+  SmallString<128> FilenameBuffer;
+  StringRef Filename = getSpelling(FilenameTok, FilenameBuffer);
+  SourceLocation CharEnd = FilenameTok.getEndLoc();
+
   CharSourceRange FilenameRange
     = CharSourceRange::getCharRange(FilenameTok.getLocation(), CharEnd);
+  SourceRange DirectiveRange(HashLoc, FilenameTok.getLocation());
   StringRef OriginalFilename = Filename;
   bool isAngled =
     GetIncludeFilenameSpelling(FilenameTok.getLocation(), Filename);
@@ -1808,10 +1727,11 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
             Callbacks ? &RelativePath : nullptr, &SuggestedModule, &IsMapped,
             /*IsFrameworkFound=*/nullptr);
         if (File) {
-          SourceRange Range(FilenameTok.getLocation(), CharEnd);
-          Diag(FilenameTok, diag::err_pp_file_not_found_angled_include_not_fatal) <<
-            Filename <<
-            FixItHint::CreateReplacement(Range, "\"" + Filename.str() + "\"");
+          Diag(FilenameTok,
+               diag::err_pp_file_not_found_angled_include_not_fatal)
+              << Filename
+              << FixItHint::CreateReplacement(FilenameRange,
+                                              "\"" + Filename.str() + "\"");
         }
       }
 
@@ -1845,12 +1765,12 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
             Callbacks ? &RelativePath : nullptr, &SuggestedModule, &IsMapped,
             /*IsFrameworkFound=*/nullptr);
         if (File) {
-          SourceRange Range(FilenameTok.getLocation(), CharEnd);
-          auto Hint = isAngled
-                          ? FixItHint::CreateReplacement(
-                                Range, "<" + TypoCorrectionName.str() + ">")
-                          : FixItHint::CreateReplacement(
-                                Range, "\"" + TypoCorrectionName.str() + "\"");
+          auto Hint =
+              isAngled
+                  ? FixItHint::CreateReplacement(
+                        FilenameRange, "<" + TypoCorrectionName.str() + ">")
+                  : FixItHint::CreateReplacement(
+                        FilenameRange, "\"" + TypoCorrectionName.str() + "\"");
           Diag(FilenameTok, diag::err_pp_file_not_found_typo_not_fatal)
               << OriginalFilename << TypoCorrectionName << Hint;
           // We found the file, so set the Filename to the name after typo
@@ -2035,9 +1955,8 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
       // For other system headers, we don't. They can be controlled separately.
       auto DiagId = (FileCharacter == SrcMgr::C_User || warnByDefaultOnWrongCase(Name)) ?
           diag::pp_nonportable_path : diag::pp_nonportable_system_path;
-      SourceRange Range(FilenameTok.getLocation(), CharEnd);
       Diag(FilenameTok, DiagId) << Path <<
-        FixItHint::CreateReplacement(Range, Path);
+        FixItHint::CreateReplacement(FilenameRange, Path);
     }
   }
 
@@ -2058,8 +1977,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
 
       if (IncludeTok.getIdentifierInfo()->getPPKeywordID() !=
           tok::pp___include_macros)
-        EnterAnnotationToken(SourceRange(HashLoc, End),
-                             tok::annot_module_include, M);
+        EnterAnnotationToken(DirectiveRange, tok::annot_module_include, M);
     }
     return;
   }
@@ -2072,7 +1990,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
   }
 
   // Look up the file, create a File ID for it.
-  SourceLocation IncludePos = End;
+  SourceLocation IncludePos = FilenameTok.getLocation();
   // If the filename string was the result of macro expansions, set the include
   // position on the file where it will be included and after the expansions.
   if (IncludePos.isMacroID())
@@ -2114,7 +2032,7 @@ void Preprocessor::HandleIncludeDirective(SourceLocation HashLoc,
     // submodule.
     // FIXME: There's no point doing this if we're handling a #__include_macros
     // directive.
-    EnterAnnotationToken(SourceRange(HashLoc, End), tok::annot_module_begin, M);
+    EnterAnnotationToken(DirectiveRange, tok::annot_module_begin, M);
   }
 }