Major refactoring of BreakableToken.

Unify handling of whitespace when breaking protruding tokens with other
whitespace replacements.

As a side effect, the BreakableToken structure changed significantly:
- have a common base class for single-line breakable tokens, as they are
  much more similar
- revamp handling of multi-line comments; we now calculate the
  information about lines in multi-line comments similar to normal
  tokens, and always issue replacements

As a result, we were able to get rid of special casing of trailing
whitespace deletion for comments in the whitespace manager and the
BreakableToken and fixed bugs related to tab handling and escaped
newlines.

llvm-svn: 182738

1 files changed, 26 insertions, 19 deletions

diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index b4ea90462a8..ea8a0f9f7d1 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -802,6 +802,9 @@ private:
     unsigned UnbreakableTailLength = Current.UnbreakableTailLength;
     llvm::OwningPtr<BreakableToken> Token;
     unsigned StartColumn = State.Column - Current.FormatTok.TokenLength;
+    unsigned OriginalStartColumn = SourceMgr.getSpellingColumnNumber(
+        Current.FormatTok.getStartOfNonWhitespace()) - 1;
+
     if (Current.is(tok::string_literal) &&
         Current.Type != TT_ImplicitStringLiteral) {
       // Only break up default narrow strings.
@@ -810,18 +813,16 @@ private:
       if (!LiteralData || *LiteralData != '"')
         return 0;
 
-      Token.reset(new BreakableStringLiteral(SourceMgr, Current.FormatTok,
-                                             StartColumn));
+      Token.reset(new BreakableStringLiteral(Current.FormatTok, StartColumn));
     } else if (Current.Type == TT_BlockComment) {
       BreakableBlockComment *BBC =
-          new BreakableBlockComment(SourceMgr, Current, StartColumn);
-      if (!DryRun)
-        BBC->alignLines(Whitespaces);
+          new BreakableBlockComment(Style, Current.FormatTok, StartColumn,
+                                    OriginalStartColumn, !Current.Parent);
       Token.reset(BBC);
     } else if (Current.Type == TT_LineComment &&
                (Current.Parent == NULL ||
                 Current.Parent->Type != TT_ImplicitStringLiteral)) {
-      Token.reset(new BreakableLineComment(SourceMgr, Current, StartColumn));
+      Token.reset(new BreakableLineComment(Current.FormatTok, StartColumn));
     } else {
       return 0;
     }
@@ -832,8 +833,12 @@ private:
     bool BreakInserted = false;
     unsigned Penalty = 0;
     unsigned PositionAfterLastLineInToken = 0;
-    for (unsigned LineIndex = 0; LineIndex < Token->getLineCount();
-         ++LineIndex) {
+    for (unsigned LineIndex = 0, EndIndex = Token->getLineCount();
+         LineIndex != EndIndex; ++LineIndex) {
+      if (!DryRun) {
+        Token->replaceWhitespaceBefore(LineIndex, Line.InPPDirective,
+                                       Whitespaces);
+      }
       unsigned TailOffset = 0;
       unsigned RemainingTokenLength =
           Token->getLineLengthAfterSplit(LineIndex, TailOffset);
@@ -845,8 +850,7 @@ private:
         assert(Split.first != 0);
         unsigned NewRemainingTokenLength = Token->getLineLengthAfterSplit(
             LineIndex, TailOffset + Split.first + Split.second);
-        if (NewRemainingTokenLength >= RemainingTokenLength)
-          break;
+        assert(NewRemainingTokenLength < RemainingTokenLength);
         if (!DryRun) {
           Token->insertBreak(LineIndex, TailOffset, Split, Line.InPPDirective,
                              Whitespaces);
@@ -857,9 +861,6 @@ private:
         BreakInserted = true;
       }
       PositionAfterLastLineInToken = RemainingTokenLength;
-      if (!DryRun) {
-        Token->trimLine(LineIndex, TailOffset, Line.InPPDirective, Whitespaces);
-      }
     }
 
     if (BreakInserted) {
@@ -1084,8 +1085,8 @@ private:
 class LexerBasedFormatTokenSource : public FormatTokenSource {
 public:
   LexerBasedFormatTokenSource(Lexer &Lex, SourceManager &SourceMgr)
-      : GreaterStashed(false), Lex(Lex), SourceMgr(SourceMgr),
-        IdentTable(Lex.getLangOpts()) {
+      : GreaterStashed(false), TrailingWhitespace(0), Lex(Lex),
+        SourceMgr(SourceMgr), IdentTable(Lex.getLangOpts()) {
     Lex.SetKeepWhitespaceMode(true);
   }
 
@@ -1102,12 +1103,13 @@ public:
     FormatTok = FormatToken();
     Lex.LexFromRawLexer(FormatTok.Tok);
     StringRef Text = rawTokenText(FormatTok.Tok);
-    SourceLocation WhitespaceStart = FormatTok.Tok.getLocation();
+    SourceLocation WhitespaceStart =
+        FormatTok.Tok.getLocation().getLocWithOffset(-TrailingWhitespace);
     if (SourceMgr.getFileOffset(WhitespaceStart) == 0)
       FormatTok.IsFirst = true;
 
     // Consume and record whitespace until we find a significant token.
-    unsigned WhitespaceLength = 0;
+    unsigned WhitespaceLength = TrailingWhitespace;
     while (FormatTok.Tok.is(tok::unknown)) {
       unsigned Newlines = Text.count('\n');
       if (Newlines > 0)
@@ -1130,9 +1132,10 @@ public:
     // Now FormatTok is the next non-whitespace token.
     FormatTok.TokenLength = Text.size();
 
+    TrailingWhitespace = 0;
     if (FormatTok.Tok.is(tok::comment)) {
-      FormatTok.TrailingWhiteSpaceLength = Text.size() - Text.rtrim().size();
-      FormatTok.TokenLength -= FormatTok.TrailingWhiteSpaceLength;
+      TrailingWhitespace = Text.size() - Text.rtrim().size();
+      FormatTok.TokenLength -= TrailingWhitespace;
     }
 
     // In case the token starts with escaped newlines, we want to
@@ -1163,6 +1166,9 @@ public:
 
     FormatTok.WhitespaceRange = SourceRange(
         WhitespaceStart, WhitespaceStart.getLocWithOffset(WhitespaceLength));
+    FormatTok.TokenText = StringRef(
+        SourceMgr.getCharacterData(FormatTok.getStartOfNonWhitespace()),
+        FormatTok.TokenLength);
     return FormatTok;
   }
 
@@ -1171,6 +1177,7 @@ public:
 private:
   FormatToken FormatTok;
   bool GreaterStashed;
+  unsigned TrailingWhitespace;
   Lexer &Lex;
   SourceManager &SourceMgr;
   IdentifierTable IdentTable;