UTF-8 support for clang-format.

Summary:
Detect if the file is valid UTF-8, and if this is the case, count code
points instead of just using number of bytes in all (hopefully) places, where
number of columns is needed. In particular, use the new
FormatToken.CodePointCount instead of TokenLength where appropriate.
Changed BreakableToken implementations to respect utf-8 character boundaries
when in utf-8 mode.

Reviewers: klimek, djasper

Reviewed By: djasper

CC: cfe-commits, rsmith, gribozavr

Differential Revision: http://llvm-reviews.chandlerc.com/D918

llvm-svn: 183312

1 files changed, 2 insertions, 7 deletions

diff --git a/clang/lib/Format/TokenAnnotator.h b/clang/lib/Format/TokenAnnotator.h
index a0d680c91df..28d55a007c2 100644
--- a/clang/lib/Format/TokenAnnotator.h
+++ b/clang/lib/Format/TokenAnnotator.h
@@ -21,7 +21,6 @@
 #include <string>
 
 namespace clang {
-class Lexer;
 class SourceManager;
 
 namespace format {
@@ -71,10 +70,8 @@ public:
 /// \c UnwrappedLine.
 class TokenAnnotator {
 public:
-  TokenAnnotator(const FormatStyle &Style, SourceManager &SourceMgr, Lexer &Lex,
-                 IdentifierInfo &Ident_in)
-      : Style(Style), SourceMgr(SourceMgr), Lex(Lex), Ident_in(Ident_in) {
-  }
+  TokenAnnotator(const FormatStyle &Style, IdentifierInfo &Ident_in)
+      : Style(Style), Ident_in(Ident_in) {}
 
   void annotate(AnnotatedLine &Line);
   void calculateFormattingInformation(AnnotatedLine &Line);
@@ -95,8 +92,6 @@ private:
   void calculateUnbreakableTailLengths(AnnotatedLine &Line);
 
   const FormatStyle &Style;
-  SourceManager &SourceMgr;
-  Lexer &Lex;
 
   // Contextual keywords:
   IdentifierInfo &Ident_in;