Calculate and store ColumnWidth instead of CodePointCount in FormatTokens.

Summary:
This fixes various issues with mixed tabs and spaces handling, e.g.
when realigning block comments.

Reviewers: klimek, djasper

Reviewed By: djasper

CC: cfe-commits

Differential Revision: http://llvm-reviews.chandlerc.com/D1608

llvm-svn: 190395

1 files changed, 18 insertions, 24 deletions

diff --git a/clang/lib/Format/ContinuationIndenter.cpp b/clang/lib/Format/ContinuationIndenter.cpp
index 40d9d2f7d29..0b380041dfe 100644
--- a/clang/lib/Format/ContinuationIndenter.cpp
+++ b/clang/lib/Format/ContinuationIndenter.cpp
@@ -201,7 +201,7 @@ unsigned ContinuationIndenter::addTokenToState(LineState &State, bool Newline,
                                State.NextToken->WhitespaceRange.getEnd()) -
                            SourceMgr.getSpellingColumnNumber(
                                State.NextToken->WhitespaceRange.getBegin());
-    State.Column += WhitespaceLength + State.NextToken->CodePointCount;
+    State.Column += WhitespaceLength + State.NextToken->ColumnWidth;
     State.NextToken = State.NextToken->Next;
     return 0;
   }
@@ -257,11 +257,11 @@ unsigned ContinuationIndenter::addTokenToState(LineState &State, bool Newline,
                  State.Line->StartsDefinition))) {
       State.Column = State.Stack.back().Indent;
     } else if (Current.Type == TT_ObjCSelectorName) {
-      if (State.Stack.back().ColonPos > Current.CodePointCount) {
-        State.Column = State.Stack.back().ColonPos - Current.CodePointCount;
+      if (State.Stack.back().ColonPos > Current.ColumnWidth) {
+        State.Column = State.Stack.back().ColonPos - Current.ColumnWidth;
       } else {
         State.Column = State.Stack.back().Indent;
-        State.Stack.back().ColonPos = State.Column + Current.CodePointCount;
+        State.Stack.back().ColonPos = State.Column + Current.ColumnWidth;
       }
     } else if (Current.is(tok::l_square) && Current.Type != TT_ObjCMethodExpr &&
                Current.Type != TT_LambdaLSquare) {
@@ -307,7 +307,7 @@ unsigned ContinuationIndenter::addTokenToState(LineState &State, bool Newline,
     if (!Current.isTrailingComment())
       State.Stack.back().LastSpace = State.Column;
     if (Current.isMemberAccess())
-      State.Stack.back().LastSpace += Current.CodePointCount;
+      State.Stack.back().LastSpace += Current.ColumnWidth;
     State.StartOfLineLevel = State.ParenLevel;
     State.LowestLevelOnLine = State.ParenLevel;
 
@@ -343,8 +343,8 @@ unsigned ContinuationIndenter::addTokenToState(LineState &State, bool Newline,
       State.Stack.back().VariablePos = State.Column;
       // Move over * and & if they are bound to the variable name.
       const FormatToken *Tok = &Previous;
-      while (Tok && State.Stack.back().VariablePos >= Tok->CodePointCount) {
-        State.Stack.back().VariablePos -= Tok->CodePointCount;
+      while (Tok && State.Stack.back().VariablePos >= Tok->ColumnWidth) {
+        State.Stack.back().VariablePos -= Tok->ColumnWidth;
         if (Tok->SpacesRequiredBefore != 0)
           break;
         Tok = Tok->Previous;
@@ -361,12 +361,12 @@ unsigned ContinuationIndenter::addTokenToState(LineState &State, bool Newline,
     if (Current.Type == TT_ObjCSelectorName &&
         State.Stack.back().ColonPos == 0) {
       if (State.Stack.back().Indent + Current.LongestObjCSelectorName >
-          State.Column + Spaces + Current.CodePointCount)
+          State.Column + Spaces + Current.ColumnWidth)
         State.Stack.back().ColonPos =
             State.Stack.back().Indent + Current.LongestObjCSelectorName;
       else
         State.Stack.back().ColonPos =
-            State.Column + Spaces + Current.CodePointCount;
+            State.Column + Spaces + Current.ColumnWidth;
     }
 
     if (Previous.opensScope() && Previous.Type != TT_ObjCMethodExpr &&
@@ -436,7 +436,7 @@ unsigned ContinuationIndenter::moveStateToNextToken(LineState &State,
         std::min(State.LowestLevelOnLine, State.ParenLevel);
   if (Current.isMemberAccess())
     State.Stack.back().StartOfFunctionCall =
-        Current.LastInChainOfCalls ? 0 : State.Column + Current.CodePointCount;
+        Current.LastInChainOfCalls ? 0 : State.Column + Current.ColumnWidth;
   if (Current.Type == TT_CtorInitializerColon) {
     // Indent 2 from the column, so:
     // SomeClass::SomeClass()
@@ -592,7 +592,7 @@ unsigned ContinuationIndenter::moveStateToNextToken(LineState &State,
     State.StartOfStringLiteral = 0;
   }
 
-  State.Column += Current.CodePointCount;
+  State.Column += Current.ColumnWidth;
   State.NextToken = State.NextToken->Next;
   unsigned Penalty = breakProtrudingToken(Current, State, DryRun);
   if (State.Column > getColumnLimit(State)) {
@@ -618,8 +618,7 @@ ContinuationIndenter::addMultilineStringLiteral(const FormatToken &Current,
   for (unsigned i = 0, e = State.Stack.size(); i != e; ++i)
     State.Stack[i].BreakBeforeParameter = true;
 
-  unsigned ColumnsUsed =
-      State.Column - Current.CodePointCount + Current.FirstLineColumnWidth;
+  unsigned ColumnsUsed = State.Column;
   // We can only affect layout of the first and the last line, so the penalty
   // for all other lines is constant, and we ignore it.
   State.Column = Current.LastLineColumnWidth;
@@ -636,14 +635,14 @@ unsigned ContinuationIndenter::breakProtrudingToken(const FormatToken &Current,
     return 0;
 
   llvm::OwningPtr<BreakableToken> Token;
-  unsigned StartColumn = State.Column - Current.CodePointCount;
+  unsigned StartColumn = State.Column - Current.ColumnWidth;
 
   if (Current.is(tok::string_literal) &&
       Current.Type != TT_ImplicitStringLiteral) {
     // Don't break string literals with (in case of non-raw strings, escaped)
     // newlines. As clang-format must not change the string's content, it is
     // unlikely that we'll end up with a better format.
-    if (Current.isMultiline())
+    if (Current.IsMultiline)
       return addMultilineStringLiteral(Current, State);
 
     // Only break up default narrow strings.
@@ -657,11 +656,8 @@ unsigned ContinuationIndenter::breakProtrudingToken(const FormatToken &Current,
     Token.reset(new BreakableStringLiteral(
         Current, StartColumn, State.Line->InPPDirective, Encoding, Style));
   } else if (Current.Type == TT_BlockComment && Current.isTrailingComment()) {
-    unsigned OriginalStartColumn =
-        SourceMgr.getSpellingColumnNumber(Current.getStartOfNonWhitespace()) -
-        1;
     Token.reset(new BreakableBlockComment(
-        Current, StartColumn, OriginalStartColumn, !Current.Previous,
+        Current, StartColumn, Current.OriginalColumn, !Current.Previous,
         State.Line->InPPDirective, Encoding, Style));
   } else if (Current.Type == TT_LineComment &&
              (Current.Previous == NULL ||
@@ -673,10 +669,8 @@ unsigned ContinuationIndenter::breakProtrudingToken(const FormatToken &Current,
     // FIXME: If we want to handle them correctly, we'll need to adjust
     // leading whitespace in consecutive lines when changing indentation of
     // the first line similar to what we do with block comments.
-    if (Current.isMultiline()) {
-      State.Column = StartColumn + Current.FirstLineColumnWidth;
+    if (Current.IsMultiline)
       return 0;
-    }
 
     Token.reset(new BreakableLineComment(
         Current, StartColumn, State.Line->InPPDirective, Encoding, Style));
@@ -759,12 +753,12 @@ bool ContinuationIndenter::NextIsMultilineString(const LineState &State) {
   // AlwaysBreakBeforeMultilineStrings implementation.
   if (Current.TokenText.startswith("R\""))
     return false;
-  if (Current.isMultiline())
+  if (Current.IsMultiline)
     return true;
   if (Current.getNextNonComment() &&
       Current.getNextNonComment()->is(tok::string_literal))
     return true; // Implicit concatenation.
-  if (State.Column + Current.CodePointCount + Current.UnbreakableTailLength >
+  if (State.Column + Current.ColumnWidth + Current.UnbreakableTailLength >
       Style.ColumnLimit)
     return true; // String will be split.
   return false;