[clang-format] Implement comment reflowing.

Summary:
This presents a version of the comment reflowing with less mutable state inside
the comment breakable token subclasses. The state has been pushed into the
driving breakProtrudingToken method. For this, the API of BreakableToken is enriched
by the methods getSplitBefore and getLineLengthAfterSplitBefore.

Reviewers: klimek

Reviewed By: klimek

Subscribers: djasper, klimek, mgorny, cfe-commits, ioeric

Differential Revision: https://reviews.llvm.org/D28764

llvm-svn: 293055

1 files changed, 478 insertions, 3 deletions

diff --git a/clang/unittests/Format/FormatTest.cpp b/clang/unittests/Format/FormatTest.cpp
index a7c2f7c9df2..2be5062d6f6 100644
--- a/clang/unittests/Format/FormatTest.cpp
+++ b/clang/unittests/Format/FormatTest.cpp
@@ -1783,6 +1783,463 @@ TEST_F(FormatTest, CommentsInStaticInitializers) {
                "    0x00, 0x00, 0x00, 0x00};            // comment\n");
 }
 
+TEST_F(FormatTest, ReflowsComments) {
+  // Break a long line and reflow with the full next line.
+  EXPECT_EQ("// long long long\n"
+            "// long long",
+            format("// long long long long\n"
+                   "// long",
+                   getLLVMStyleWithColumns(20)));
+
+  // Keep the trailing newline while reflowing.
+  EXPECT_EQ("// long long long\n"
+            "// long long\n",
+            format("// long long long long\n"
+                   "// long\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Break a long line and reflow with a part of the next line.
+  EXPECT_EQ("// long long long\n"
+            "// long long\n"
+            "// long_long",
+            format("// long long long long\n"
+                   "// long long_long",
+                   getLLVMStyleWithColumns(20)));
+
+  // Break but do not reflow if the first word from the next line is too long.
+  EXPECT_EQ("// long long long\n"
+            "// long\n"
+            "// long_long_long\n",
+            format("// long long long long\n"
+                   "// long_long_long\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't break or reflow short lines.
+  verifyFormat("// long\n"
+               "// long long long lo\n"
+               "// long long long lo\n"
+               "// long",
+               getLLVMStyleWithColumns(20));
+
+  // Keep prefixes and decorations while reflowing.
+  EXPECT_EQ("/// long long long\n"
+            "/// long long\n",
+            format("/// long long long long\n"
+                   "/// long\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("//! long long long\n"
+            "//! long long\n",
+            format("//! long long long long\n"
+                   "//! long\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/* long long long\n"
+            " * long long */",
+            format("/* long long long long\n"
+                   " * long */",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't bring leading whitespace up while reflowing.
+  EXPECT_EQ("/*  long long long\n"
+            " * long long long\n"
+            " */",
+            format("/*  long long long long\n"
+                   " *  long long\n"
+                   " */",
+                   getLLVMStyleWithColumns(20)));
+
+  // Reflow the last line of a block comment with its trailing '*/'.
+  EXPECT_EQ("/* long long long\n"
+            "   long long */",
+            format("/* long long long long\n"
+                   "   long */",
+                   getLLVMStyleWithColumns(20)));
+
+  // Reflow two short lines; keep the postfix of the last one.
+  EXPECT_EQ("/* long long long\n"
+            " * long long long */",
+            format("/* long long long long\n"
+                   " * long\n"
+                   " * long */",
+                   getLLVMStyleWithColumns(20)));
+
+  // Put the postfix of the last short reflow line on a newline if it doesn't
+  // fit.
+  EXPECT_EQ("/* long long long\n"
+            " * long long longg\n"
+            " */",
+            format("/* long long long long\n"
+                   " * long\n"
+                   " * longg */",
+                   getLLVMStyleWithColumns(20)));
+
+  // Break single line block comments that are first in the line with ' *'
+  // decoration.
+  EXPECT_EQ("/* long long long\n"
+            " * long */",
+            format("/* long long long long */", getLLVMStyleWithColumns(20)));
+
+  // Break single line block comment that are not first in the line with '  '
+  // decoration.
+  EXPECT_EQ("int i; /* long long\n"
+            "          long */",
+            format("int i; /* long long long */", getLLVMStyleWithColumns(20)));
+
+  // Reflow a line that goes just over the column limit.
+  EXPECT_EQ("// long long long\n"
+            "// lon long",
+            format("// long long long lon\n"
+                   "// long",
+                   getLLVMStyleWithColumns(20)));
+
+  // Stop reflowing if the next line has a different indentation than the
+  // previous line.
+  EXPECT_EQ("// long long long\n"
+            "// long\n"
+            "//  long long\n"
+            "//  long",
+            format("// long long long long\n"
+                   "//  long long\n"
+                   "//  long",
+                   getLLVMStyleWithColumns(20)));
+
+  // Reflow into the last part of a really long line that has been broken into
+  // multiple lines.
+  EXPECT_EQ("// long long long\n"
+            "// long long long\n"
+            "// long long long\n",
+            format("// long long long long long long long long\n"
+                   "// long\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Break the first line, then reflow the beginning of the second and third
+  // line up.
+  EXPECT_EQ("// long long long\n"
+            "// lon1 lon2 lon2\n"
+            "// lon2 lon3 lon3",
+            format("// long long long lon1\n"
+                   "// lon2 lon2 lon2\n"
+                   "// lon3 lon3",
+                   getLLVMStyleWithColumns(20)));
+
+  // Reflow the beginning of the second line, then break the rest.
+  EXPECT_EQ("// long long long\n"
+            "// lon1 lon2 lon2\n"
+            "// lon2 lon2 lon2\n"
+            "// lon3",
+            format("// long long long lon1\n"
+                   "// lon2 lon2 lon2 lon2 lon2 lon3",
+                   getLLVMStyleWithColumns(20)));
+
+  // Shrink the first line, then reflow the second line up.
+  EXPECT_EQ("// long long long", format("// long              long\n"
+                                        "// long",
+                                        getLLVMStyleWithColumns(20)));
+
+  // Don't shrink leading whitespace.
+  EXPECT_EQ("int i; ///           a",
+            format("int i; ///           a", getLLVMStyleWithColumns(20)));
+
+  // Shrink trailing whitespace if there is no postfix and reflow.
+  EXPECT_EQ("// long long long\n"
+            "// long long",
+            format("// long long long long    \n"
+                   "// long",
+                   getLLVMStyleWithColumns(20)));
+
+  // Shrink trailing whitespace to a single one if there is postfix.
+  EXPECT_EQ("/* long long long */",
+            format("/* long long long     */", getLLVMStyleWithColumns(20)));
+
+  // Break a block comment postfix if exceeding the line limit.
+  EXPECT_EQ("/*               long\n"
+            " */",
+            format("/*               long */", getLLVMStyleWithColumns(20)));
+
+  // Reflow indented comments.
+  EXPECT_EQ("{\n"
+            "  // long long long\n"
+            "  // long long\n"
+            "  int i; /* long lon\n"
+            "            g long\n"
+            "          */\n"
+            "}",
+            format("{\n"
+                   "  // long long long long\n"
+                   "  // long\n"
+                   "  int i; /* long lon g\n"
+                   "            long */\n"
+                   "}",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't realign trailing comments after reflow has happened.
+  EXPECT_EQ("// long long long\n"
+            "// long long\n"
+            "long i; // long",
+            format("// long long long long\n"
+                   "// long\n"
+                   "long i; // long",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("// long long long\n"
+            "// longng long long\n"
+            "// long lo",
+            format("// long long long longng\n"
+                   "// long long long\n"
+                   "// lo",
+                   getLLVMStyleWithColumns(20)));
+
+  // Reflow lines after a broken line.
+  EXPECT_EQ("int a; // Trailing\n"
+            "       // comment on\n"
+            "       // 2 or 3\n"
+            "       // lines.\n",
+            format("int a; // Trailing comment\n"
+                   "       // on 2\n"
+                   "       // or 3\n"
+                   "       // lines.\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/// This long line\n"
+            "/// gets reflown.\n",
+            format("/// This long line gets\n"
+                   "/// reflown.\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("//! This long line\n"
+            "//! gets reflown.\n",
+            format(" //! This long line gets\n"
+                   " //! reflown.\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/* This long line\n"
+            " * gets reflown.\n"
+            " */\n",
+            format("/* This long line gets\n"
+                   " * reflown.\n"
+                   " */\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Reflow after indentation makes a line too long.
+  EXPECT_EQ("{\n"
+            "  // long long long\n"
+            "  // lo long\n"
+            "}\n",
+            format("{\n"
+                   "// long long long lo\n"
+                   "// long\n"
+                   "}\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Break and reflow multiple lines.
+  EXPECT_EQ("/*\n"
+            " * Reflow the end of\n"
+            " * line by 11 22 33\n"
+            " * 4.\n"
+            " */\n",
+            format("/*\n"
+                   " * Reflow the end of line\n"
+                   " * by\n"
+                   " * 11\n"
+                   " * 22\n"
+                   " * 33\n"
+                   " * 4.\n"
+                   " */\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/// First line gets\n"
+            "/// broken. Second\n"
+            "/// line gets\n"
+            "/// reflown and\n"
+            "/// broken. Third\n"
+            "/// gets reflown.\n",
+            format("/// First line gets broken.\n"
+                   "/// Second line gets reflown and broken.\n"
+                   "/// Third gets reflown.\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("int i; // first long\n"
+            "       // long snd\n"
+            "       // long.\n",
+            format("int i; // first long long\n"
+                   "       // snd long.\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("{\n"
+            "  // first long line\n"
+            "  // line second\n"
+            "  // long line line\n"
+            "  // third long line\n"
+            "  // line\n"
+            "}\n",
+            format("{\n"
+                   "  // first long line line\n"
+                   "  // second long line line\n"
+                   "  // third long line line\n"
+                   "}\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("int i; /* first line\n"
+            "        * second\n"
+            "        * line third\n"
+            "        * line\n"
+            "        */",
+            format("int i; /* first line\n"
+                   "        * second line\n"
+                   "        * third line\n"
+                   "        */",
+                   getLLVMStyleWithColumns(20)));
+
+  // Reflow the last two lines of a section that starts with a line having
+  // different indentation.
+  EXPECT_EQ(
+      "//     long\n"
+      "// long long long\n"
+      "// long long",
+      format("//     long\n"
+             "// long long long long\n"
+             "// long",
+             getLLVMStyleWithColumns(20)));
+
+  // Keep the block comment endling '*/' while reflowing.
+  EXPECT_EQ("/* Long long long\n"
+            " * line short */\n",
+            format("/* Long long long line\n"
+                   " * short */\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't reflow between separate blocks of comments.
+  EXPECT_EQ("/* First comment\n"
+            " * block will */\n"
+            "/* Snd\n"
+            " */\n",
+            format("/* First comment block\n"
+                   " * will */\n"
+                   "/* Snd\n"
+                   " */\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't reflow across blank comment lines.
+  EXPECT_EQ("int i; // This long\n"
+            "       // line gets\n"
+            "       // broken.\n"
+            "       //  \n"
+            "       // keep.\n",
+            format("int i; // This long line gets broken.\n"
+                   "       //  \n"
+                   "       // keep.\n",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("{\n"
+            "  /// long long long\n"
+            "  /// long long\n"
+            "  ///\n"
+            "  /// long\n"
+            "}",
+            format("{\n"
+                   "  /// long long long long\n"
+                   "  /// long\n"
+                   "  ///\n"
+                   "  /// long\n"
+                   "}",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("//! long long long\n"
+            "//! long\n"
+            "\n"
+            "//! long",
+            format("//! long long long long\n"
+                   "\n"
+                   "//! long",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/* long long long\n"
+            "   long\n"
+            "\n"
+            "   long */",
+            format("/* long long long long\n"
+                   "\n"
+                   "   long */",
+                   getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/* long long long\n"
+            " * long\n"
+            " *\n"
+            " * long */",
+            format("/* long long long long\n"
+                   " *\n"
+                   " * long */",
+                   getLLVMStyleWithColumns(20)));
+  
+  // Don't reflow lines having content that is a single character.
+  EXPECT_EQ("// long long long\n"
+            "// long\n"
+            "// l",
+            format("// long long long long\n"
+                   "// l",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't reflow lines starting with two punctuation characters.
+  EXPECT_EQ("// long long long\n"
+            "// long\n"
+            "// ... --- ...",
+            format(
+                "// long long long long\n"
+                "// ... --- ...",
+                getLLVMStyleWithColumns(20)));
+  // Reflow lines that have a non-punctuation character among their first 2
+  // characters.
+  EXPECT_EQ("// long long long\n"
+            "// long 'long'",
+            format(
+                "// long long long long\n"
+                "// 'long'",
+                getLLVMStyleWithColumns(20)));
+
+  // Don't reflow between separate blocks of comments.
+  EXPECT_EQ("/* First comment\n"
+            " * block will */\n"
+            "/* Snd\n"
+            " */\n",
+            format("/* First comment block\n"
+                   " * will */\n"
+                   "/* Snd\n"
+                   " */\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't reflow lines having different indentation.
+  EXPECT_EQ("// long long long\n"
+            "// long\n"
+            "//  long",
+            format("// long long long long\n"
+                   "//  long",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't break or reflow after implicit string literals.
+  verifyFormat("#include <t> // l l l\n"
+               "             // l",
+               getLLVMStyleWithColumns(20));
+
+  // Don't break or reflow comments on import lines.
+  EXPECT_EQ("#include \"t\" /* l l l\n"
+            "                * l */",
+            format("#include \"t\" /* l l l\n"
+                   "                * l */",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't reflow between different trailing comment sections.
+  EXPECT_EQ("int i; // long long\n"
+            "       // long\n"
+            "int j; // long long\n"
+            "       // long\n",
+            format("int i; // long long long\n"
+                   "int j; // long long long\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Don't reflow if the first word on the next line is longer than the
+  // available space at current line.
+  EXPECT_EQ("int i; // trigger\n"
+            "       // reflow\n"
+            "       // longsec\n",
+            format("int i; // trigger reflow\n"
+                   "       // longsec\n",
+                   getLLVMStyleWithColumns(20)));
+
+  // Keep empty comment lines.
+  EXPECT_EQ("/**/", format(" /**/", getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/* */", format(" /* */", getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("/*  */", format(" /*  */", getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("//", format(" //  ", getLLVMStyleWithColumns(20)));
+  EXPECT_EQ("///", format(" ///  ", getLLVMStyleWithColumns(20)));
+}
+
 TEST_F(FormatTest, IgnoresIf0Contents) {
   EXPECT_EQ("#if 0\n"
             "}{)(&*(^%%#%@! fsadj f;ldjs ,:;| <<<>>>][)(][\n"
@@ -7238,11 +7695,9 @@ TEST_F(FormatTest, BlockCommentsAtEndOfLine) {
             format("a = {1111 /*      */\n"
                    "};",
                    getLLVMStyleWithColumns(15)));
-
-  // FIXME: The formatting is still wrong here.
   EXPECT_EQ("a = {\n"
             "    1111 /*      a\n"
-            "            */\n"
+            "          */\n"
             "};",
             format("a = {1111 /*      a */\n"
                    "};",
@@ -10933,6 +11388,26 @@ TEST_F(FormatTest, DisableRegions) {
                    "  int j;\n"
                    " /* clang-format on */\n"
                    "   int   k;"));
+
+  // Don't reflow comments within disabled regions.
+  EXPECT_EQ(
+      "// clang-format off\n"
+      "// long long long long long long line\n"
+      "/* clang-format on */\n"
+      "/* long long long\n"
+      " * long long long\n"
+      " * line */\n"
+      "int i;\n"
+      "/* clang-format off */\n"
+      "/* long long long long long long line */\n",
+      format("// clang-format off\n"
+             "// long long long long long long line\n"
+             "/* clang-format on */\n"
+             "/* long long long long long long line */\n"
+             "int i;\n"
+             "/* clang-format off */\n"
+             "/* long long long long long long line */\n",
+             getLLVMStyleWithColumns(20)));
 }
 
 TEST_F(FormatTest, DoNotCrashOnInvalidInput) {