Comment parsing: fix a bug where a line with whitespace between two paragraphs

would cause us to concatenate these paragraphs into a single one.

The no-op whitespace churn in test/Index test happened because these tests
don't use the correct approach for testing and are more strict than required
for they are testing.

llvm-svn: 189126

4 files changed, 57 insertions, 11 deletions

diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index d89c79b3d93..03e01015b95 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -16,6 +16,15 @@
 #include "llvm/Support/ErrorHandling.h"
 
 namespace clang {
+
+static inline bool isWhitespace(llvm::StringRef S) {
+  for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
+    if (!isWhitespace(*I))
+      return false;
+  }
+  return true;
+}
+
 namespace comments {
 
 /// Re-lexes a sequence of tok::text tokens.
@@ -594,6 +603,18 @@ BlockContentComment *Parser::parseParagraphOrBlockCommand() {
         consumeToken();
         break; // Two newlines -- end of paragraph.
       }
+      // Also allow [tok::newline, tok::text, tok::newline] if the middle
+      // tok::text is just whitespace.
+      if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
+        Token WhitespaceTok = Tok;
+        consumeToken();
+        if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
+          consumeToken();
+          break;
+        }
+        // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
+        putBack(WhitespaceTok);
+      }
       if (Content.size() > 0)
         Content.back()->addTrailingNewline();
       continue;
diff --git a/clang/test/Index/comment-misc-tags.m b/clang/test/Index/comment-misc-tags.m
index a1875272236..bb16ea97b6d 100644
--- a/clang/test/Index/comment-misc-tags.m
+++ b/clang/test/Index/comment-misc-tags.m
@@ -45,7 +45,7 @@
 // CHECK-NEXT:     (CXComment_Text Text=[     ] IsWhitespace)))
 // CHECK:       (CXComment_BlockCommand CommandName=[seealso]
 // CHECK-NEXT:     (CXComment_Paragraph
-// CHECK-NEXT:     (CXComment_Text Text=[ //k_ref/doc/uid/XX30000905-CH204 Programming] HasTrailingNewline)
+// CHECK-NEXT:     (CXComment_Text Text=[ //k_ref/doc/uid/XX30000905-CH204 Programming])
 
 // rdar://12379053
 /*!
diff --git a/clang/test/Index/overriding-method-comments.mm b/clang/test/Index/overriding-method-comments.mm
index 46ead4f6425..9285693396e 100644
--- a/clang/test/Index/overriding-method-comments.mm
+++ b/clang/test/Index/overriding-method-comments.mm
@@ -19,7 +19,7 @@
 - (void)METH:(id)AAA;
 @end
 
-// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>METH:</Name><USR>c:objc(cs)Root(im)METH:</USR><Declaration>- (void)METH:(id)AAA;</Declaration><Parameters><Parameter><Name>AAA</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> ZZZ </Para></Discussion></Parameter></Parameters></Function>]
+// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>METH:</Name><USR>c:objc(cs)Root(im)METH:</USR><Declaration>- (void)METH:(id)AAA;</Declaration><Parameters><Parameter><Name>AAA</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> ZZZ</Para></Discussion></Parameter></Parameters></Function>]
 
 @interface Sub : Root
 @end
@@ -28,13 +28,13 @@
 - (void)METH:(id)BBB;
 @end
 
-// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>METH:</Name><USR>c:objc(cs)Root(im)METH:</USR><Declaration>- (void)METH:(id)BBB;</Declaration><Parameters><Parameter><Name>BBB</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> ZZZ </Para></Discussion></Parameter></Parameters></Function>]
+// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>METH:</Name><USR>c:objc(cs)Root(im)METH:</USR><Declaration>- (void)METH:(id)BBB;</Declaration><Parameters><Parameter><Name>BBB</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> ZZZ</Para></Discussion></Parameter></Parameters></Function>]
 
 @implementation Sub(CAT)
 - (void)METH:(id)III {}
 @end
 
-// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>METH:</Name><USR>c:objc(cs)Root(im)METH:</USR><Declaration>- (void)METH:(id)III;</Declaration><Parameters><Parameter><Name>III</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> ZZZ </Para></Discussion></Parameter></Parameters></Function>]
+// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>METH:</Name><USR>c:objc(cs)Root(im)METH:</USR><Declaration>- (void)METH:(id)III;</Declaration><Parameters><Parameter><Name>III</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> ZZZ</Para></Discussion></Parameter></Parameters></Function>]
 
 @interface Redec : Root
 @end
@@ -48,13 +48,13 @@
 - (void)EXT_METH:(id)AAA : (double)BBB : (int)CCC;
 @end
 
-// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>EXT_METH:::</Name><USR>c:objc(cs)Redec(im)EXT_METH:::</USR><Declaration>- (void)EXT_METH:(id)AAA:(double)BBB:(int)CCC;</Declaration><Parameters><Parameter><Name>AAA</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> input value </Para></Discussion></Parameter><Parameter><Name>BBB</Name><Index>1</Index><Direction isExplicit="1">in</Direction><Discussion><Para> 2nd input value is double </Para></Discussion></Parameter><Parameter><Name>CCC</Name><Index>2</Index><Direction isExplicit="1">out</Direction><Discussion><Para> output value is int </Para></Discussion></Parameter></Parameters></Function>]
+// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>EXT_METH:::</Name><USR>c:objc(cs)Redec(im)EXT_METH:::</USR><Declaration>- (void)EXT_METH:(id)AAA:(double)BBB:(int)CCC;</Declaration><Parameters><Parameter><Name>AAA</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> input value </Para></Discussion></Parameter><Parameter><Name>BBB</Name><Index>1</Index><Direction isExplicit="1">in</Direction><Discussion><Para> 2nd input value is double</Para></Discussion></Parameter><Parameter><Name>CCC</Name><Index>2</Index><Direction isExplicit="1">out</Direction><Discussion><Para> output value is int </Para></Discussion></Parameter></Parameters></Function>]
 
 @implementation Redec
 - (void)EXT_METH:(id)PPP : (double)QQQ : (int)RRR {}
 @end
 
-// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>EXT_METH:::</Name><USR>c:objc(cs)Redec(im)EXT_METH:::</USR><Declaration>- (void)EXT_METH:(id)PPP:(double)QQQ:(int)RRR;</Declaration><Parameters><Parameter><Name>PPP</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> input value </Para></Discussion></Parameter><Parameter><Name>QQQ</Name><Index>1</Index><Direction isExplicit="1">in</Direction><Discussion><Para> 2nd input value is double </Para></Discussion></Parameter><Parameter><Name>RRR</Name><Index>2</Index><Direction isExplicit="1">out</Direction><Discussion><Para> output value is int </Para></Discussion></Parameter></Parameters></Function>]
+// CHECK: FullCommentAsXML=[<Function isInstanceMethod="1" file="{{[^"]+}}overriding-method-comments.mm" line="[[@LINE-3]]" column="1"><Name>EXT_METH:::</Name><USR>c:objc(cs)Redec(im)EXT_METH:::</USR><Declaration>- (void)EXT_METH:(id)PPP:(double)QQQ:(int)RRR;</Declaration><Parameters><Parameter><Name>PPP</Name><Index>0</Index><Direction isExplicit="1">in</Direction><Discussion><Para> input value </Para></Discussion></Parameter><Parameter><Name>QQQ</Name><Index>1</Index><Direction isExplicit="1">in</Direction><Discussion><Para> 2nd input value is double</Para></Discussion></Parameter><Parameter><Name>RRR</Name><Index>2</Index><Direction isExplicit="1">out</Direction><Discussion><Para> output value is int </Para></Discussion></Parameter></Parameters></Function>]
 
 struct Base {
   /// \brief Does something.
diff --git a/clang/unittests/AST/CommentParser.cpp b/clang/unittests/AST/CommentParser.cpp
index d05fb58fafa..f75c636e01d 100644
--- a/clang/unittests/AST/CommentParser.cpp
+++ b/clang/unittests/AST/CommentParser.cpp
@@ -628,18 +628,43 @@ TEST_F(CommentParserTest, Basic3) {
   }
 }
 
-TEST_F(CommentParserTest, Paragraph1) {
+TEST_F(CommentParserTest, ParagraphSplitting1) {
   const char *Sources[] = {
     "// Aaa\n"
     "//\n"
     "// Bbb",
 
     "// Aaa\n"
+    "// \n"
+    "// Bbb",
+
+    "// Aaa\n"
+    "//\t\n"
+    "// Bbb",
+
+    "// Aaa\n"
     "//\n"
     "//\n"
     "// Bbb",
-  };
 
+    "/**\n"
+    " Aaa\n"
+    "\n"
+    " Bbb\n"
+    "*/",
+
+    "/**\n"
+    " Aaa\n"
+    " \n"
+    " Bbb\n"
+    "*/",
+
+    "/**\n"
+    " Aaa\n"
+    "\t \n"
+    " Bbb\n"
+    "*/",
+  };
 
   for (size_t i = 0, e = array_lengthof(Sources); i != e; i++) {
     FullComment *FC = parseString(Sources[i]);
@@ -650,7 +675,7 @@ TEST_F(CommentParserTest, Paragraph1) {
   }
 }
 
-TEST_F(CommentParserTest, Paragraph2) {
+TEST_F(CommentParserTest, Paragraph1) {
   const char *Source =
     "// \\brief Aaa\n"
     "//\n"
@@ -670,7 +695,7 @@ TEST_F(CommentParserTest, Paragraph2) {
   ASSERT_TRUE(HasParagraphCommentAt(FC, 2, " Bbb"));
 }
 
-TEST_F(CommentParserTest, Paragraph3) {
+TEST_F(CommentParserTest, Paragraph2) {
   const char *Source = "// \\brief \\author";
 
   FullComment *FC = parseString(Source);
@@ -694,7 +719,7 @@ TEST_F(CommentParserTest, Paragraph3) {
   }
 }
 
-TEST_F(CommentParserTest, Paragraph4) {
+TEST_F(CommentParserTest, Paragraph3) {
   const char *Source =
     "// \\brief Aaa\n"
     "// Bbb \\author\n"