Fix whitespace handling in empty macro arguments

When a function-like macro definition ends with one of the macro's
parameters, and the argument is empty, any whitespace before the
parameter name in the macro definition needs to be preserved. Promoting
the existing NextTokGetsSpace to a preserved bit-field and checking it
at the end of the macro expansion allows it to be moved to the first
token following the macro expansion result.

Patch by Harald van Dijk!

llvm-svn: 200786

3 files changed, 35 insertions, 18 deletions

diff --git a/clang/include/clang/Lex/TokenLexer.h b/clang/include/clang/Lex/TokenLexer.h
index 7c8cfd028a2..659643dedf9 100644
--- a/clang/include/clang/Lex/TokenLexer.h
+++ b/clang/include/clang/Lex/TokenLexer.h
@@ -81,6 +81,14 @@ class TokenLexer {
   bool AtStartOfLine : 1;
   bool HasLeadingSpace : 1;
 
+  // NextTokGetsSpace - When this is true, the next token appended to the
+  // output list during function argument expansion will get a leading space,
+  // regardless of whether it had one to begin with or not. This is used for
+  // placemarker support. If still true after function argument expansion, the
+  // leading space will be applied to the first token following the macro
+  // expansion.
+  bool NextTokGetsSpace : 1;
+
   /// OwnsTokens - This is true if this TokenLexer allocated the Tokens
   /// array, and thus needs to free it when destroyed.  For simple object-like
   /// macros (for example) we just point into the token buffer of the macro
@@ -182,6 +190,13 @@ private:
   void updateLocForMacroArgTokens(SourceLocation ArgIdSpellLoc,
                                   Token *begin_tokens, Token *end_tokens);
 
+  /// Remove comma ahead of __VA_ARGS__, if present, according to compiler
+  /// dialect settings.  Returns true if the comma is removed.
+  bool MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks,
+                                    bool HasPasteOperator,
+                                    MacroInfo *Macro, unsigned MacroArgNo,
+                                    Preprocessor &PP);
+
   void PropagateLineStartLeadingSpaceInfo(Token &Result);
 };
 
diff --git a/clang/lib/Lex/TokenLexer.cpp b/clang/lib/Lex/TokenLexer.cpp
index d41af3f41a9..2f29f7c9c8c 100644
--- a/clang/lib/Lex/TokenLexer.cpp
+++ b/clang/lib/Lex/TokenLexer.cpp
@@ -37,6 +37,7 @@ void TokenLexer::Init(Token &Tok, SourceLocation ELEnd, MacroInfo *MI,
   ExpandLocEnd = ELEnd;
   AtStartOfLine = Tok.isAtStartOfLine();
   HasLeadingSpace = Tok.hasLeadingSpace();
+  NextTokGetsSpace = false;
   Tokens = &*Macro->tokens_begin();
   OwnsTokens = false;
   DisableMacroExpansion = false;
@@ -95,6 +96,7 @@ void TokenLexer::Init(const Token *TokArray, unsigned NumToks,
   ExpandLocStart = ExpandLocEnd = SourceLocation();
   AtStartOfLine = false;
   HasLeadingSpace = false;
+  NextTokGetsSpace = false;
   MacroExpansionStart = SourceLocation();
 
   // Set HasLeadingSpace/AtStartOfLine so that the first token will be
@@ -119,13 +121,10 @@ void TokenLexer::destroy() {
   if (ActualArgs) ActualArgs->destroy(PP);
 }
 
-/// Remove comma ahead of __VA_ARGS__, if present, according to compiler dialect
-/// settings.  Returns true if the comma is removed.
-static bool MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks,
-                                         bool &NextTokGetsSpace,
-                                         bool HasPasteOperator,
-                                         MacroInfo *Macro, unsigned MacroArgNo,
-                                         Preprocessor &PP) {
+bool TokenLexer::MaybeRemoveCommaBeforeVaArgs(SmallVectorImpl<Token> &ResultToks,
+                                              bool HasPasteOperator,
+                                              MacroInfo *Macro, unsigned MacroArgNo,
+                                              Preprocessor &PP) {
   // Is the macro argument __VA_ARGS__?
   if (!Macro->isVariadic() || MacroArgNo != Macro->getNumArgs()-1)
     return false;
@@ -179,11 +178,6 @@ void TokenLexer::ExpandFunctionArguments() {
   // we install the newly expanded sequence as the new 'Tokens' list.
   bool MadeChange = false;
 
-  // NextTokGetsSpace - When this is true, the next token appended to the
-  // output list will get a leading space, regardless of whether it had one to
-  // begin with or not.  This is used for placemarker support.
-  bool NextTokGetsSpace = false;
-
   for (unsigned i = 0, e = NumTokens; i != e; ++i) {
     // If we found the stringify operator, get the argument stringified.  The
     // preprocessor already verified that the following token is a macro name
@@ -256,7 +250,7 @@ void TokenLexer::ExpandFunctionArguments() {
     // In Microsoft mode, remove the comma before __VA_ARGS__ to ensure there
     // are no trailing commas if __VA_ARGS__ is empty.
     if (!PasteBefore && ActualArgs->isVarargsElidedUse() &&
-        MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace,
+        MaybeRemoveCommaBeforeVaArgs(ResultToks,
                                      /*HasPasteOperator=*/false,
                                      Macro, ArgNo, PP))
       continue;
@@ -311,9 +305,10 @@ void TokenLexer::ExpandFunctionArguments() {
                                              NextTokGetsSpace);
         NextTokGetsSpace = false;
       } else {
-        // If this is an empty argument, and if there was whitespace before the
-        // formal token, make sure the next token gets whitespace before it.
-        NextTokGetsSpace = CurTok.hasLeadingSpace();
+        // If this is an empty argument, if there was whitespace before the
+        // formal token, and this is not the first token in the macro
+        // definition, make sure the next token gets whitespace before it.
+        NextTokGetsSpace |= i != 0 && CurTok.hasLeadingSpace();
       }
       continue;
     }
@@ -396,7 +391,7 @@ void TokenLexer::ExpandFunctionArguments() {
     // the ## was a comma, remove the comma.  This is a GCC extension which is
     // disabled when using -std=c99.
     if (ActualArgs->isVarargsElidedUse())
-      MaybeRemoveCommaBeforeVaArgs(ResultToks, NextTokGetsSpace,
+      MaybeRemoveCommaBeforeVaArgs(ResultToks,
                                    /*HasPasteOperator=*/true,
                                    Macro, ArgNo, PP);
 
@@ -428,7 +423,7 @@ bool TokenLexer::Lex(Token &Tok) {
 
     Tok.startToken();
     Tok.setFlagValue(Token::StartOfLine , AtStartOfLine);
-    Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace);
+    Tok.setFlagValue(Token::LeadingSpace, HasLeadingSpace || NextTokGetsSpace);
     if (CurToken == 0)
       Tok.setFlag(Token::LeadingEmptyMacro);
     return PP.HandleEndOfTokenLexer(Tok);
diff --git a/clang/test/Preprocessor/macro_arg_empty.c b/clang/test/Preprocessor/macro_arg_empty.c
new file mode 100644
index 00000000000..b5ecaa27ba1
--- /dev/null
+++ b/clang/test/Preprocessor/macro_arg_empty.c
@@ -0,0 +1,7 @@
+// RUN: %clang_cc1 -E %s | FileCheck --strict-whitespace %s
+
+#define FOO(x) x
+#define BAR(x) x x
+#define BAZ(x) [x] [ x] [x ]
+[FOO()] [ FOO()] [FOO() ] [BAR()] [ BAR()] [BAR() ] BAZ()
+// CHECK: [] [ ] [ ] [ ] [ ] [ ] [] [ ] [ ]