[c++2a] Implement P0306 __VA_OPT__ (Comma omission and comma deletion)

This patch implements an extension to the preprocessor:

__VA_OPT__(contents) --> which expands into its contents if variadic arguments are supplied to the parent macro, or behaves as an empty token if none.

  - Currently this feature is only enabled for C++2a (this could be enabled, with some careful tweaks, for other dialects with the appropriate extension or compatibility warnings)

  - The patch was reviewed here: https://reviews.llvm.org/D35782 and asides from the above (and moving some of the definition and expansion recognition logic into the corresponding state machines), I believe I incorporated all of Richard's suggestions.

A few technicalities (most of which were clarified through private correspondence between rsmith, hubert and thomas) are worth mentioning.  Given:

    #define F(a,...) a #__VA_OPT__(a ## a)  a ## __VA_OPT__(__VA_ARGS__)

    - The call F(,) Does not supply any tokens for the variadic arguments and hence VA_OPT behaves as a placeholder.
    - When expanding VA_OPT (for e.g. F(,1) token pasting occurs eagerly within its contents if the contents need to be stringified.
    - A hash or a hashhash prior to VA_OPT does not inhibit expansion of arguments if they are the first token within VA_OPT.
    - When a variadic argument is supplied, argument substitution occurs within the contents as does stringification - and these resulting tokens are inserted back into the macro expansions token stream just prior to the entire stream being rescanned and concatenated.

See wg21.link/P0306 for further details on the feature.


Acknowledgment: This patch would have been poorer if not for Richard Smith's usual thoughtful analysis and feedback.
llvm-svn: 315840

4 files changed, 251 insertions, 21 deletions

diff --git a/clang/lib/Lex/MacroArgs.cpp b/clang/lib/Lex/MacroArgs.cpp
index d2525c0cd6c..5c0f0623c3e 100644
--- a/clang/lib/Lex/MacroArgs.cpp
+++ b/clang/lib/Lex/MacroArgs.cpp
@@ -135,6 +135,16 @@ const Token *MacroArgs::getUnexpArgument(unsigned Arg) const {
   return Result;
 }
 
+// This function assumes that the variadic arguments are the tokens
+// corresponding to the last parameter (ellipsis) - and since tokens are
+// separated by the 'eof' token, if that is the only token corresponding to that
+// last parameter, we know no variadic arguments were supplied.
+bool MacroArgs::invokedWithVariadicArgument(const MacroInfo *const MI) const {
+  if (!MI->isVariadic())
+    return false;
+  const int VariadicArgIndex = getNumMacroArguments() - 1;
+  return getUnexpArgument(VariadicArgIndex)->isNot(tok::eof);
+}
 
 /// ArgNeedsPreexpansion - If we can prove that the argument won't be affected
 /// by pre-expansion, return false.  Otherwise, conservatively return true.
diff --git a/clang/lib/Lex/PPDirectives.cpp b/clang/lib/Lex/PPDirectives.cpp
index 318931663c1..f9a97505bf4 100644
--- a/clang/lib/Lex/PPDirectives.cpp
+++ b/clang/lib/Lex/PPDirectives.cpp
@@ -2368,12 +2368,50 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody(
     // Otherwise, read the body of a function-like macro.  While we are at it,
     // check C99 6.10.3.2p1: ensure that # operators are followed by macro
     // parameters in function-like macro expansions.
+
+    VAOptDefinitionContext VAOCtx(*this);
+
     while (Tok.isNot(tok::eod)) {
       LastTok = Tok;
 
       if (!Tok.isOneOf(tok::hash, tok::hashat, tok::hashhash)) {
         MI->AddTokenToBody(Tok);
 
+        if (VAOCtx.isVAOptToken(Tok)) {
+          // If we're already within a VAOPT, emit an error.
+          if (VAOCtx.isInVAOpt()) {
+            Diag(Tok, diag::err_pp_vaopt_nested_use);
+            return nullptr;
+          }
+          // Ensure VAOPT is followed by a '(' .
+          LexUnexpandedToken(Tok);
+          if (Tok.isNot(tok::l_paren)) {
+            Diag(Tok, diag::err_pp_missing_lparen_in_vaopt_use);
+            return nullptr;
+          }
+          MI->AddTokenToBody(Tok);
+          VAOCtx.sawVAOptFollowedByOpeningParens(Tok.getLocation());
+          LexUnexpandedToken(Tok);
+          if (Tok.is(tok::hashhash)) {
+            Diag(Tok, diag::err_vaopt_paste_at_start);
+            return nullptr;
+          }
+          continue;
+        } else if (VAOCtx.isInVAOpt()) {
+          if (Tok.is(tok::r_paren)) {
+            if (VAOCtx.sawClosingParen()) {
+              const unsigned NumTokens = MI->getNumTokens();
+              assert(NumTokens >= 3 && "Must have seen at least __VA_OPT__( "
+                                       "and a subsequent tok::r_paren");
+              if (MI->getReplacementToken(NumTokens - 2).is(tok::hashhash)) {
+                Diag(Tok, diag::err_vaopt_paste_at_end);
+                return nullptr;
+              }
+            }
+          } else if (Tok.is(tok::l_paren)) {
+            VAOCtx.sawOpeningParen(Tok.getLocation());
+          }
+        }
         // Get the next token of the macro.
         LexUnexpandedToken(Tok);
         continue;
@@ -2414,12 +2452,14 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody(
         continue;
       }
 
+      // Our Token is a stringization operator.
       // Get the next token of the macro.
       LexUnexpandedToken(Tok);
 
-      // Check for a valid macro arg identifier.
-      if (Tok.getIdentifierInfo() == nullptr ||
-          MI->getParameterNum(Tok.getIdentifierInfo()) == -1) {
+      // Check for a valid macro arg identifier or __VA_OPT__.
+      if (!VAOCtx.isVAOptToken(Tok) &&
+          (Tok.getIdentifierInfo() == nullptr ||
+           MI->getParameterNum(Tok.getIdentifierInfo()) == -1)) {
 
         // If this is assembler-with-cpp mode, we accept random gibberish after
         // the '#' because '#' is often a comment character.  However, change
@@ -2438,11 +2478,24 @@ MacroInfo *Preprocessor::ReadOptionalMacroParameterListAndBody(
 
       // Things look ok, add the '#' and param name tokens to the macro.
       MI->AddTokenToBody(LastTok);
-      MI->AddTokenToBody(Tok);
-      LastTok = Tok;
 
-      // Get the next token of the macro.
-      LexUnexpandedToken(Tok);
+      // If the token following '#' is VAOPT, let the next iteration handle it
+      // and check it for correctness, otherwise add the token and prime the
+      // loop with the next one.
+      if (!VAOCtx.isVAOptToken(Tok)) {
+        MI->AddTokenToBody(Tok);
+        LastTok = Tok;
+
+        // Get the next token of the macro.
+        LexUnexpandedToken(Tok);
+      }
+    }
+    if (VAOCtx.isInVAOpt()) {
+      assert(Tok.is(tok::eod) && "Must be at End Of preprocessing Directive");
+      Diag(Tok, diag::err_pp_expected_after)
+        << LastTok.getKind() << tok::r_paren;
+      Diag(VAOCtx.getUnmatchedOpeningParenLoc(), diag::note_matching) << tok::l_paren;
+      return nullptr;
     }
   }
   MI->setDefinitionEndLoc(LastTok.getLocation());
diff --git a/clang/lib/Lex/Preprocessor.cpp b/clang/lib/Lex/Preprocessor.cpp
index 1f9a469bc59..65df6a57f1d 100644
--- a/clang/lib/Lex/Preprocessor.cpp
+++ b/clang/lib/Lex/Preprocessor.cpp
@@ -121,12 +121,18 @@ Preprocessor::Preprocessor(std::shared_ptr<PreprocessorOptions> PPOpts,
 
   // We haven't read anything from the external source.
   ReadMacrosFromExternalSource = false;
-  
-  // "Poison" __VA_ARGS__, which can only appear in the expansion of a macro.
-  // This gets unpoisoned where it is allowed.
+
+  // "Poison" __VA_ARGS__, __VA_OPT__ which can only appear in the expansion of
+  // a macro. They get unpoisoned where it is allowed.
   (Ident__VA_ARGS__ = getIdentifierInfo("__VA_ARGS__"))->setIsPoisoned();
   SetPoisonReason(Ident__VA_ARGS__,diag::ext_pp_bad_vaargs_use);
-  
+  if (getLangOpts().CPlusPlus2a) {
+    (Ident__VA_OPT__ = getIdentifierInfo("__VA_OPT__"))->setIsPoisoned();
+    SetPoisonReason(Ident__VA_OPT__,diag::ext_pp_bad_vaopt_use);
+  } else {
+    Ident__VA_OPT__ = nullptr;
+  }
+
   // Initialize the pragma handlers.
   RegisterBuiltinPragmas();
   
@@ -667,13 +673,15 @@ bool Preprocessor::HandleIdentifier(Token &Identifier) {
   // unpoisoned it if we're defining a C99 macro.
   if (II.isOutOfDate()) {
     bool CurrentIsPoisoned = false;
-    if (&II == Ident__VA_ARGS__)
-      CurrentIsPoisoned = Ident__VA_ARGS__->isPoisoned();
+    const bool IsSpecialVariadicMacro =
+        &II == Ident__VA_ARGS__ || &II == Ident__VA_OPT__;
+    if (IsSpecialVariadicMacro)
+      CurrentIsPoisoned = II.isPoisoned();
 
     updateOutOfDateIdentifier(II);
     Identifier.setKind(II.getTokenID());
 
-    if (&II == Ident__VA_ARGS__)
+    if (IsSpecialVariadicMacro)
       II.setIsPoisoned(CurrentIsPoisoned);
   }
   
diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
index 6ec128efc15..c0883dd48e0 100644
--- a/clang/lib/Lex/TokenLexer.cpp
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -17,6 +17,7 @@
 #include "clang/Lex/MacroArgs.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/VariadicMacroSupport.h"
 #include "llvm/ADT/SmallString.h"
 
 using namespace clang;
@@ -168,6 +169,65 @@ bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(
   return true;
 }
 
+void TokenLexer::stringifyVAOPTContents(
+    SmallVectorImpl<Token> &ResultToks, const VAOptExpansionContext &VCtx,
+    const SourceLocation VAOPTClosingParenLoc) {
+  const int NumToksPriorToVAOpt = VCtx.getNumberOfTokensPriorToVAOpt();
+  const unsigned int NumVAOptTokens = ResultToks.size() - NumToksPriorToVAOpt;
+  Token *const VAOPTTokens =
+      NumVAOptTokens ? &ResultToks[NumToksPriorToVAOpt] : nullptr;
+
+  SmallVector<Token, 64> ConcatenatedVAOPTResultToks;
+  // FIXME: Should we keep track within VCtx that we did or didnot
+  // encounter pasting - and only then perform this loop.
+
+  // Perform token pasting (concatenation) prior to stringization.
+  for (unsigned int CurTokenIdx = 0; CurTokenIdx != NumVAOptTokens;
+       ++CurTokenIdx) {
+    const unsigned int PrevTokenIdx = CurTokenIdx;
+
+    if (VAOPTTokens[CurTokenIdx].is(tok::hashhash)) {
+      assert(CurTokenIdx != 0 &&
+             "Can not have __VAOPT__ contents begin with a ##");
+      Token &LHS = VAOPTTokens[CurTokenIdx - 1];
+      pasteTokens(LHS, llvm::makeArrayRef(VAOPTTokens, NumVAOptTokens),
+                  CurTokenIdx);
+      // CurTokenIdx is either the same as NumTokens or one past the
+      // last token concatenated.
+      // PrevTokenIdx is the index of the hashhash
+      const unsigned NumTokensPastedTogether = CurTokenIdx - PrevTokenIdx + 1;
+      // Replace the token prior to the first ## in this iteration.
+      ConcatenatedVAOPTResultToks.back() = LHS;
+      if (CurTokenIdx == NumVAOptTokens)
+        break;
+    }
+    ConcatenatedVAOPTResultToks.push_back(VAOPTTokens[CurTokenIdx]);
+  }
+
+  ConcatenatedVAOPTResultToks.push_back(VCtx.getEOFTok());
+  // Get the SourceLocation that represents the start location within
+  // the macro definition that marks where this string is substituted
+  // into: i.e. the __VA_OPT__ and the ')' within the spelling of the
+  // macro definition, and use it to indicate that the stringified token
+  // was generated from that location.
+  const SourceLocation ExpansionLocStartWithinMacro =
+      getExpansionLocForMacroDefLoc(VCtx.getVAOptLoc());
+  const SourceLocation ExpansionLocEndWithinMacro =
+      getExpansionLocForMacroDefLoc(VAOPTClosingParenLoc);
+
+  Token StringifiedVAOPT = MacroArgs::StringifyArgument(
+      &ConcatenatedVAOPTResultToks[0], PP, VCtx.hasCharifyBefore() /*Charify*/,
+      ExpansionLocStartWithinMacro, ExpansionLocEndWithinMacro);
+
+  if (VCtx.getLeadingSpaceForStringifiedToken())
+    StringifiedVAOPT.setFlag(Token::LeadingSpace);
+
+  StringifiedVAOPT.setFlag(Token::StringifiedInMacro);
+  // Resize (shrink) the token stream to just capture this stringified token.
+  ResultToks.resize(NumToksPriorToVAOpt + 1);
+  ResultToks.back() = StringifiedVAOPT;
+}
+
 /// Expand the arguments of a function-like macro so that we can quickly
 /// return preexpanded tokens from Tokens.
 void TokenLexer::ExpandFunctionArguments() {
@@ -178,10 +238,13 @@ void TokenLexer::ExpandFunctionArguments() {
   // we install the newly expanded sequence as the new 'Tokens' list.
   bool MadeChange = false;
 
+  const bool CalledWithVariadicArguments =
+      ActualArgs->invokedWithVariadicArgument(Macro);
+
+  VAOptExpansionContext VCtx(PP);
+  
   for (unsigned I = 0, E = NumTokens; I != E; ++I) {
-    // If we found the stringify operator, get the argument stringified.  The
-    // preprocessor already verified that the following token is a macro name
-    // when the #define was parsed.
+    
     const Token &CurTok = Tokens[I];
     // We don't want a space for the next token after a paste
     // operator.  In valid code, the token will get smooshed onto the
@@ -192,10 +255,98 @@ void TokenLexer::ExpandFunctionArguments() {
     if (I != 0 && !Tokens[I-1].is(tok::hashhash) && CurTok.hasLeadingSpace())
       NextTokGetsSpace = true;
 
+    if (VCtx.isVAOptToken(CurTok)) {
+      MadeChange = true;
+      assert(Tokens[I + 1].is(tok::l_paren) &&
+             "__VA_OPT__ must be followed by '('");
+
+      ++I;             // Skip the l_paren
+      VCtx.sawVAOptFollowedByOpeningParens(CurTok.getLocation(),
+                                           ResultToks.size());
+      
+      continue;
+    }
+
+    // We have entered into the __VA_OPT__ context, so handle tokens
+    // appropriately.
+    if (VCtx.isInVAOpt()) {
+      // If we are about to process a token that is either an argument to
+      // __VA_OPT__ or its closing rparen, then:
+      //  1) If the token is the closing rparen that exits us out of __VA_OPT__,
+      //  perform any necessary stringification or placemarker processing,
+      //  and/or skip to the next token.
+      //  2) else if macro was invoked without variadic arguments skip this
+      //  token.
+      //  3) else (macro was invoked with variadic arguments) process the token
+      //  normally.
+
+      if (Tokens[I].is(tok::l_paren))
+        VCtx.sawOpeningParen(Tokens[I].getLocation());
+      // Continue skipping tokens within __VA_OPT__ if the macro was not
+      // called with variadic arguments, else let the rest of the loop handle
+      // this token. Note sawClosingParen() returns true only if the r_paren matches
+      // the closing r_paren of the __VA_OPT__.
+      if (!Tokens[I].is(tok::r_paren) || !VCtx.sawClosingParen()) {
+        if (!CalledWithVariadicArguments) {
+          // Skip this token.
+          continue;
+        }
+        // ... else the macro was called with variadic arguments, and we do not
+        // have a closing rparen - so process this token normally.
+
+      } else {
+        // Current token is the closing r_paren which marks the end of the
+        // __VA_OPT__ invocation, so handle any place-marker pasting (if
+        // empty) by removing hashhash either before (if exists) or after. And
+        // also stringify the entire contents if VAOPT was preceded by a hash,
+        // but do so only after any token concatenation that needs to occur
+        // within the contents of VAOPT.
+
+        if (VCtx.hasStringifyOrCharifyBefore()) {
+          // Replace all the tokens just added from within VAOPT into a single
+          // stringified token. This requires token-pasting to eagerly occur
+          // within these tokens. If either the contents of VAOPT were empty
+          // or the macro wasn't called with any variadic arguments, the result
+          // is a token that represents an empty string.
+          stringifyVAOPTContents(ResultToks, VCtx,
+                                 /*ClosingParenLoc*/ Tokens[I].getLocation());
+
+        } else if (/*No tokens within VAOPT*/ !(
+            ResultToks.size() - VCtx.getNumberOfTokensPriorToVAOpt())) {
+          // Treat VAOPT as a placemarker token.  Eat either the '##' before the
+          // RHS/VAOPT (if one exists, suggesting that the LHS (if any) to that
+          // hashhash was not a placemarker) or the '##'
+          // after VAOPT, but not both.
+
+          if (ResultToks.size() && ResultToks.back().is(tok::hashhash)) {
+            ResultToks.pop_back();
+          } else if ((I + 1 != E) && Tokens[I + 1].is(tok::hashhash)) {
+            ++I; // Skip the following hashhash.
+          }
+        }
+        VCtx.reset();
+        // We processed __VA_OPT__'s closing paren (and the exit out of
+        // __VA_OPT__), so skip to the next token.
+        continue;
+      }
+    }
+
+    // If we found the stringify operator, get the argument stringified.  The
+    // preprocessor already verified that the following token is a macro 
+    // parameter or __VA_OPT__ when the #define was lexed.
+    
     if (CurTok.isOneOf(tok::hash, tok::hashat)) {
       int ArgNo = Macro->getParameterNum(Tokens[I+1].getIdentifierInfo());
-      assert(ArgNo != -1 && "Token following # is not an argument?");
-
+      assert((ArgNo != -1 || VCtx.isVAOptToken(Tokens[I + 1])) &&
+             "Token following # is not an argument or __VA_OPT__!");
+      
+      if (ArgNo == -1) {
+        // Handle the __VA_OPT__ case.
+        VCtx.sawHashOrHashAtBefore(NextTokGetsSpace,
+                                   CurTok.is(tok::hashat));
+        continue;
+      }
+      // Else handle the simple argument case.
       SourceLocation ExpansionLocStart =
           getExpansionLocForMacroDefLoc(CurTok.getLocation());
       SourceLocation ExpansionLocEnd =
@@ -232,7 +383,9 @@ void TokenLexer::ExpandFunctionArguments() {
       !ResultToks.empty() && ResultToks.back().is(tok::hashhash);
     bool PasteBefore = I != 0 && Tokens[I-1].is(tok::hashhash);
     bool PasteAfter = I+1 != E && Tokens[I+1].is(tok::hashhash);
-    assert(!NonEmptyPasteBefore || PasteBefore);
+
+    assert((!NonEmptyPasteBefore || PasteBefore || VCtx.isInVAOpt()) &&
+           "unexpected ## in ResultToks");
 
     // Otherwise, if this is not an argument token, just add the token to the
     // output buffer.
@@ -384,7 +537,13 @@ void TokenLexer::ExpandFunctionArguments() {
     assert(PasteBefore);
     if (NonEmptyPasteBefore) {
       assert(ResultToks.back().is(tok::hashhash));
-      ResultToks.pop_back();
+      // Do not remove the paste operator if it is the one before __VA_OPT__
+      // (and we are still processing tokens within VA_OPT).  We handle the case
+      // of removing the paste operator if __VA_OPT__ reduces to the notional
+      // placemarker above when we encounter the closing paren of VA_OPT.
+      if (!VCtx.isInVAOpt() ||
+          ResultToks.size() > VCtx.getNumberOfTokensPriorToVAOpt())
+        ResultToks.pop_back();
     }
 
     // If this is the __VA_ARGS__ token, and if the argument wasn't provided,