[c++20] Implement tweaked __VA_OPT__ rules from P1042R1:

 * __VA_OPT__ is expanded if the *expanded* __VA_ARGS__ is non-empty,
   not if the original argument contained no tokens.
 * Placemarkers at the start and end of __VA_OPT__ are retained just
   long enough to paste them with adjacent ## operators. We never paste
   "across" a discarded placemarker.

llvm-svn: 359964

1 files changed, 47 insertions, 5 deletions

diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
index b0d5286a4f2..9d132a545c8 100644
--- a/clang/lib/Lex/TokenLexer.cpp
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -243,8 +243,7 @@ void TokenLexer::ExpandFunctionArguments() {
   // we install the newly expanded sequence as the new 'Tokens' list.
   bool MadeChange = false;
 
-  const bool CalledWithVariadicArguments =
-      ActualArgs->invokedWithVariadicArgument(Macro);
+  Optional<bool> CalledWithVariadicArguments;
 
   VAOptExpansionContext VCtx(PP);
 
@@ -291,7 +290,12 @@ void TokenLexer::ExpandFunctionArguments() {
       // this token. Note sawClosingParen() returns true only if the r_paren matches
       // the closing r_paren of the __VA_OPT__.
       if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
-        if (!CalledWithVariadicArguments) {
+        // Lazily expand __VA_ARGS__ when we see the first __VA_OPT__.
+        if (!CalledWithVariadicArguments.hasValue()) {
+          CalledWithVariadicArguments =
+              ActualArgs->invokedWithVariadicArgument(Macro, PP);
+        }
+        if (!*CalledWithVariadicArguments) {
           // Skip this token.
           continue;
         }
@@ -314,8 +318,8 @@ void TokenLexer::ExpandFunctionArguments() {
           stringifyVAOPTContents(ResultToks, VCtx,
                                  /*ClosingParenLoc*/ Tokens[I].getLocation());
 
-        } else if (/*No tokens within VAOPT*/ !(
-            ResultToks.size() - VCtx.getNumberOfTokensPriorToVAOpt())) {
+        } else if (/*No tokens within VAOPT*/
+                   ResultToks.size() == VCtx.getNumberOfTokensPriorToVAOpt()) {
           // Treat VAOPT as a placemarker token.  Eat either the '##' before the
           // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
           // hashhash was not a placemarker) or the '##'
@@ -326,6 +330,26 @@ void TokenLexer::ExpandFunctionArguments() {
           } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
             ++I; // Skip the following hashhash.
           }
+        } else {
+          // If there's a ## before the __VA_OPT__, we might have discovered
+          // that the __VA_OPT__ begins with a placeholder. We delay action on
+          // that to now to avoid messing up our stashed count of tokens before
+          // __VA_OPT__.
+          if (VCtx.beginsWithPlaceholder()) {
+            assert(VCtx.getNumberOfTokensPriorToVAOpt() > 0 &&
+                   ResultToks.size() >= VCtx.getNumberOfTokensPriorToVAOpt() &&
+                   ResultToks[VCtx.getNumberOfTokensPriorToVAOpt() - 1].is(
+                       tok::hashhash) &&
+                   "no token paste before __VA_OPT__");
+            ResultToks.erase(ResultToks.begin() +
+                             VCtx.getNumberOfTokensPriorToVAOpt() - 1);
+          }
+          // If the expansion of __VA_OPT__ ends with a placeholder, eat any
+          // following '##' token.
+          if (VCtx.endsWithPlaceholder() && I + 1 != E &&
+              Tokens[I + 1].is(tok::hashhash)) {
+            ++I;
+          }
         }
         VCtx.reset();
         // We processed __VA_OPT__'s closing paren (and the exit out of
@@ -386,6 +410,7 @@ void TokenLexer::ExpandFunctionArguments() {
       !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
     bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
     bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
+    bool RParenAfter = I+1 != E && Tokens[I+1].is(tok::r_paren);
 
     assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
            "unexpected ## in ResultToks");
@@ -470,6 +495,18 @@ void TokenLexer::ExpandFunctionArguments() {
                                              NextTokGetsSpace);
         ResultToks[FirstResult].setFlagValue(Token::StartOfLine, false);
         NextTokGetsSpace = false;
+      } else {
+        // We're creating a placeholder token. Usually this doesn't matter,
+        // but it can affect paste behavior when at the start or end of a
+        // __VA_OPT__.
+        if (NonEmptyPasteBefore) {
+          // We're imagining a placeholder token is inserted here. If this is
+          // the first token in a __VA_OPT__ after a ##, delete the ##.
+          assert(VCtx.isInVAOpt() && "should only happen inside a __VA_OPT__");
+          VCtx.hasPlaceholderAfterHashhashAtStart();
+        }
+        if (RParenAfter)
+          VCtx.hasPlaceholderBeforeRParen();
       }
       continue;
     }
@@ -534,6 +571,9 @@ void TokenLexer::ExpandFunctionArguments() {
       continue;
     }
 
+    if (RParenAfter)
+      VCtx.hasPlaceholderBeforeRParen();
+
     // If this is on the RHS of a paste operator, we've already copied the
     // paste operator to the ResultToks list, unless the LHS was empty too.
     // Remove it.
@@ -547,6 +587,8 @@ void TokenLexer::ExpandFunctionArguments() {
       if (!VCtx.isInVAOpt() ||
           ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
         ResultToks.pop_back();
+      else
+        VCtx.hasPlaceholderAfterHashhashAtStart();
     }
 
     // If this is the __VA_ARGS__ token, and if the argument wasn't provided,