clang-format: Don't remove 'unknown' tokens.

In certain macros or incorrect string literals, the token stream can
contain 'unknown' tokens, e.g. a single backslash or a set of empty
ticks. clang-format simply treated them as whitespace and removed them
prior to this patch.

This fixes llvm.org/PR17215

llvm-svn: 192490

1 files changed, 14 insertions, 0 deletions

diff --git a/clang/lib/Format/Format.cpp b/clang/lib/Format/Format.cpp
index 129c119532e..7fc66167f08 100644
--- a/clang/lib/Format/Format.cpp
+++ b/clang/lib/Format/Format.cpp
@@ -691,18 +691,32 @@ private:
           FormatTok->LastNewlineOffset = WhitespaceLength + i + 1;
           Column = 0;
           break;
+        case '\r':
+        case '\f':
+        case '\v':
+          Column = 0;
+          break;
         case ' ':
           ++Column;
           break;
         case '\t':
           Column += Style.TabWidth - Column % Style.TabWidth;
           break;
+        case '\\':
+          ++Column;
+          if (i + 1 == e || (FormatTok->TokenText[i + 1] != '\r' &&
+                             FormatTok->TokenText[i + 1] != '\n'))
+            FormatTok->Type = TT_ImplicitStringLiteral;
+          break;
         default:
+          FormatTok->Type = TT_ImplicitStringLiteral;
           ++Column;
           break;
         }
       }
 
+      if (FormatTok->Type == TT_ImplicitStringLiteral)
+        break;
       WhitespaceLength += FormatTok->Tok.getLength();
 
       readRawToken(*FormatTok);