UTF-8 support for clang-format.

Summary:
Detect if the file is valid UTF-8, and if this is the case, count code
points instead of just using number of bytes in all (hopefully) places, where
number of columns is needed. In particular, use the new
FormatToken.CodePointCount instead of TokenLength where appropriate.
Changed BreakableToken implementations to respect utf-8 character boundaries
when in utf-8 mode.

Reviewers: klimek, djasper

Reviewed By: djasper

CC: cfe-commits, rsmith, gribozavr

Differential Revision: http://llvm-reviews.chandlerc.com/D918

llvm-svn: 183312

1 files changed, 70 insertions, 87 deletions

diff --git a/clang/lib/Format/BreakableToken.cpp b/clang/lib/Format/BreakableToken.cpp
index 7d7fe3f032e..5e5604c597f 100644
--- a/clang/lib/Format/BreakableToken.cpp
+++ b/clang/lib/Format/BreakableToken.cpp
@@ -25,66 +25,22 @@ namespace clang {
 namespace format {
 namespace {
 
-// FIXME: Move helper string functions to where it makes sense.
-
-unsigned getOctalLength(StringRef Text) {
-  unsigned I = 1;
-  while (I < Text.size() && I < 4 && (Text[I] >= '0' && Text[I] <= '7')) {
-    ++I;
-  }
-  return I;
-}
-
-unsigned getHexLength(StringRef Text) {
-  unsigned I = 2; // Point after '\x'.
-  while (I < Text.size() && ((Text[I] >= '0' && Text[I] <= '9') ||
-                             (Text[I] >= 'a' && Text[I] <= 'f') ||
-                             (Text[I] >= 'A' && Text[I] <= 'F'))) {
-    ++I;
-  }
-  return I;
-}
-
-unsigned getEscapeSequenceLength(StringRef Text) {
-  assert(Text[0] == '\\');
-  if (Text.size() < 2)
-    return 1;
-
-  switch (Text[1]) {
-  case 'u':
-    return 6;
-  case 'U':
-    return 10;
-  case 'x':
-    return getHexLength(Text);
-  default:
-    if (Text[1] >= '0' && Text[1] <= '7')
-      return getOctalLength(Text);
-    return 2;
-  }
-}
-
-StringRef::size_type getStartOfCharacter(StringRef Text,
-                                         StringRef::size_type Offset) {
-  StringRef::size_type NextEscape = Text.find('\\');
-  while (NextEscape != StringRef::npos && NextEscape < Offset) {
-    StringRef::size_type SequenceLength =
-        getEscapeSequenceLength(Text.substr(NextEscape));
-    if (Offset < NextEscape + SequenceLength)
-      return NextEscape;
-    NextEscape = Text.find('\\', NextEscape + SequenceLength);
-  }
-  return Offset;
-}
-
 BreakableToken::Split getCommentSplit(StringRef Text,
                                       unsigned ContentStartColumn,
-                                      unsigned ColumnLimit) {
+                                      unsigned ColumnLimit,
+                                      encoding::Encoding Encoding) {
   if (ColumnLimit <= ContentStartColumn + 1)
     return BreakableToken::Split(StringRef::npos, 0);
 
   unsigned MaxSplit = ColumnLimit - ContentStartColumn + 1;
-  StringRef::size_type SpaceOffset = Text.rfind(' ', MaxSplit);
+  unsigned MaxSplitBytes = 0;
+
+  for (unsigned NumChars = 0;
+       NumChars < MaxSplit && MaxSplitBytes < Text.size(); ++NumChars)
+    MaxSplitBytes +=
+        encoding::getCodePointNumBytes(Text[MaxSplitBytes], Encoding);
+
+  StringRef::size_type SpaceOffset = Text.rfind(' ', MaxSplitBytes);
   if (SpaceOffset == StringRef::npos ||
       // Don't break at leading whitespace.
       Text.find_last_not_of(' ', SpaceOffset) == StringRef::npos) {
@@ -95,7 +51,7 @@ BreakableToken::Split getCommentSplit(StringRef Text,
       // If the comment is only whitespace, we cannot split.
       return BreakableToken::Split(StringRef::npos, 0);
     SpaceOffset =
-        Text.find(' ', std::max<unsigned>(MaxSplit, FirstNonWhitespace));
+        Text.find(' ', std::max<unsigned>(MaxSplitBytes, FirstNonWhitespace));
   }
   if (SpaceOffset != StringRef::npos && SpaceOffset != 0) {
     StringRef BeforeCut = Text.substr(0, SpaceOffset).rtrim();
@@ -108,25 +64,48 @@ BreakableToken::Split getCommentSplit(StringRef Text,
 
 BreakableToken::Split getStringSplit(StringRef Text,
                                      unsigned ContentStartColumn,
-                                     unsigned ColumnLimit) {
-
-  if (ColumnLimit <= ContentStartColumn)
-    return BreakableToken::Split(StringRef::npos, 0);
-  unsigned MaxSplit = ColumnLimit - ContentStartColumn;
+                                     unsigned ColumnLimit,
+                                     encoding::Encoding Encoding) {
   // FIXME: Reduce unit test case.
   if (Text.empty())
     return BreakableToken::Split(StringRef::npos, 0);
-  MaxSplit = std::min<unsigned>(MaxSplit, Text.size() - 1);
-  StringRef::size_type SpaceOffset = Text.rfind(' ', MaxSplit);
-  if (SpaceOffset != StringRef::npos && SpaceOffset != 0)
+  if (ColumnLimit <= ContentStartColumn)
+    return BreakableToken::Split(StringRef::npos, 0);
+  unsigned MaxSplit =
+      std::min<unsigned>(ColumnLimit - ContentStartColumn,
+                         encoding::getCodePointCount(Text, Encoding) - 1);
+  StringRef::size_type SpaceOffset = 0;
+  StringRef::size_type SlashOffset = 0;
+  StringRef::size_type SplitPoint = 0;
+  for (unsigned Chars = 0;;) {
+    unsigned Advance;
+    if (Text[0] == '\\') {
+      Advance = encoding::getEscapeSequenceLength(Text);
+      Chars += Advance;
+    } else {
+      Advance = encoding::getCodePointNumBytes(Text[0], Encoding);
+      Chars += 1;
+    }
+
+    if (Chars > MaxSplit)
+      break;
+
+    if (Text[0] == ' ')
+      SpaceOffset = SplitPoint;
+    if (Text[0] == '/')
+      SlashOffset = SplitPoint;
+
+    SplitPoint += Advance;
+    Text = Text.substr(Advance);
+  }
+
+  if (SpaceOffset != 0)
     return BreakableToken::Split(SpaceOffset + 1, 0);
-  StringRef::size_type SlashOffset = Text.rfind('/', MaxSplit);
-  if (SlashOffset != StringRef::npos && SlashOffset != 0)
+  if (SlashOffset != 0)
     return BreakableToken::Split(SlashOffset + 1, 0);
-  StringRef::size_type SplitPoint = getStartOfCharacter(Text, MaxSplit);
-  if (SplitPoint == StringRef::npos || SplitPoint == 0)
-    return BreakableToken::Split(StringRef::npos, 0);
-  return BreakableToken::Split(SplitPoint, 0);
+  if (SplitPoint != 0)
+    return BreakableToken::Split(SplitPoint, 0);
+  return BreakableToken::Split(StringRef::npos, 0);
 }
 
 } // namespace
@@ -136,8 +115,8 @@ unsigned BreakableSingleLineToken::getLineCount() const { return 1; }
 unsigned
 BreakableSingleLineToken::getLineLengthAfterSplit(unsigned LineIndex,
                                                   unsigned TailOffset) const {
-  return StartColumn + Prefix.size() + Postfix.size() + Line.size() -
-         TailOffset;
+  return StartColumn + Prefix.size() + Postfix.size() +
+         encoding::getCodePointCount(Line.substr(TailOffset), Encoding);
 }
 
 void BreakableSingleLineToken::insertBreak(unsigned LineIndex,
@@ -152,8 +131,9 @@ void BreakableSingleLineToken::insertBreak(unsigned LineIndex,
 BreakableSingleLineToken::BreakableSingleLineToken(const FormatToken &Tok,
                                                    unsigned StartColumn,
                                                    StringRef Prefix,
-                                                   StringRef Postfix)
-    : BreakableToken(Tok), StartColumn(StartColumn), Prefix(Prefix),
+                                                   StringRef Postfix,
+                                                   encoding::Encoding Encoding)
+    : BreakableToken(Tok, Encoding), StartColumn(StartColumn), Prefix(Prefix),
       Postfix(Postfix) {
   assert(Tok.TokenText.startswith(Prefix) && Tok.TokenText.endswith(Postfix));
   Line = Tok.TokenText.substr(
@@ -161,13 +141,15 @@ BreakableSingleLineToken::BreakableSingleLineToken(const FormatToken &Tok,
 }
 
 BreakableStringLiteral::BreakableStringLiteral(const FormatToken &Tok,
-                                               unsigned StartColumn)
-    : BreakableSingleLineToken(Tok, StartColumn, "\"", "\"") {}
+                                               unsigned StartColumn,
+                                               encoding::Encoding Encoding)
+    : BreakableSingleLineToken(Tok, StartColumn, "\"", "\"", Encoding) {}
 
 BreakableToken::Split
 BreakableStringLiteral::getSplit(unsigned LineIndex, unsigned TailOffset,
                                  unsigned ColumnLimit) const {
-  return getStringSplit(Line.substr(TailOffset), StartColumn + 2, ColumnLimit);
+  return getStringSplit(Line.substr(TailOffset), StartColumn + 2, ColumnLimit,
+                        Encoding);
 }
 
 static StringRef getLineCommentPrefix(StringRef Comment) {
@@ -179,23 +161,23 @@ static StringRef getLineCommentPrefix(StringRef Comment) {
 }
 
 BreakableLineComment::BreakableLineComment(const FormatToken &Token,
-                                           unsigned StartColumn)
+                                           unsigned StartColumn,
+                                           encoding::Encoding Encoding)
     : BreakableSingleLineToken(Token, StartColumn,
-                               getLineCommentPrefix(Token.TokenText), "") {}
+                               getLineCommentPrefix(Token.TokenText), "",
+                               Encoding) {}
 
 BreakableToken::Split
 BreakableLineComment::getSplit(unsigned LineIndex, unsigned TailOffset,
                                unsigned ColumnLimit) const {
   return getCommentSplit(Line.substr(TailOffset), StartColumn + Prefix.size(),
-                         ColumnLimit);
+                         ColumnLimit, Encoding);
 }
 
-BreakableBlockComment::BreakableBlockComment(const FormatStyle &Style,
-                                             const FormatToken &Token,
-                                             unsigned StartColumn,
-                                             unsigned OriginalStartColumn,
-                                             bool FirstInLine)
-    : BreakableToken(Token) {
+BreakableBlockComment::BreakableBlockComment(
+    const FormatStyle &Style, const FormatToken &Token, unsigned StartColumn,
+    unsigned OriginalStartColumn, bool FirstInLine, encoding::Encoding Encoding)
+    : BreakableToken(Token, Encoding) {
   StringRef TokenText(Token.TokenText);
   assert(TokenText.startswith("/*") && TokenText.endswith("*/"));
   TokenText.substr(2, TokenText.size() - 4).split(Lines, "\n");
@@ -290,7 +272,8 @@ unsigned
 BreakableBlockComment::getLineLengthAfterSplit(unsigned LineIndex,
                                                unsigned TailOffset) const {
   return getContentStartColumn(LineIndex, TailOffset) +
-         (Lines[LineIndex].size() - TailOffset) +
+         encoding::getCodePointCount(Lines[LineIndex].substr(TailOffset),
+                                     Encoding) +
          // The last line gets a "*/" postfix.
          (LineIndex + 1 == Lines.size() ? 2 : 0);
 }
@@ -300,7 +283,7 @@ BreakableBlockComment::getSplit(unsigned LineIndex, unsigned TailOffset,
                                 unsigned ColumnLimit) const {
   return getCommentSplit(Lines[LineIndex].substr(TailOffset),
                          getContentStartColumn(LineIndex, TailOffset),
-                         ColumnLimit);
+                         ColumnLimit, Encoding);
 }
 
 void BreakableBlockComment::insertBreak(unsigned LineIndex, unsigned TailOffset,