Comment parsing: fix a bug where a line with whitespace between two paragraphs

would cause us to concatenate these paragraphs into a single one.

The no-op whitespace churn in test/Index test happened because these tests
don't use the correct approach for testing and are more strict than required
for they are testing.

llvm-svn: 189126

1 files changed, 21 insertions, 0 deletions

diff --git a/clang/lib/AST/CommentParser.cpp b/clang/lib/AST/CommentParser.cpp
index d89c79b3d93..03e01015b95 100644
--- a/clang/lib/AST/CommentParser.cpp
+++ b/clang/lib/AST/CommentParser.cpp
@@ -16,6 +16,15 @@
 #include "llvm/Support/ErrorHandling.h"
 
 namespace clang {
+
+static inline bool isWhitespace(llvm::StringRef S) {
+  for (StringRef::const_iterator I = S.begin(), E = S.end(); I != E; ++I) {
+    if (!isWhitespace(*I))
+      return false;
+  }
+  return true;
+}
+
 namespace comments {
 
 /// Re-lexes a sequence of tok::text tokens.
@@ -594,6 +603,18 @@ BlockContentComment *Parser::parseParagraphOrBlockCommand() {
         consumeToken();
         break; // Two newlines -- end of paragraph.
       }
+      // Also allow [tok::newline, tok::text, tok::newline] if the middle
+      // tok::text is just whitespace.
+      if (Tok.is(tok::text) && isWhitespace(Tok.getText())) {
+        Token WhitespaceTok = Tok;
+        consumeToken();
+        if (Tok.is(tok::newline) || Tok.is(tok::eof)) {
+          consumeToken();
+          break;
+        }
+        // We have [tok::newline, tok::text, non-newline].  Put back tok::text.
+        putBack(WhitespaceTok);
+      }
       if (Content.size() > 0)
         Content.back()->addTrailingNewline();
       continue;