Add a new -dump-raw-tokens option, which allows us to see raw tokens.

Rename -dumptokens to -dump-tokens.

llvm-svn: 57405

2 files changed, 27 insertions, 4 deletions

diff --git a/clang/Driver/clang.cpp b/clang/Driver/clang.cpp
index f415de11ceb..91d521246ab 100644
--- a/clang/Driver/clang.cpp
+++ b/clang/Driver/clang.cpp
@@ -80,7 +80,8 @@ enum ProgActions {
   ParseNoop,                    // Parse with noop callbacks.
   RunPreprocessorOnly,          // Just lex, no output.
   PrintPreprocessedInput,       // -E mode.
-  DumpTokens,                   // Token dump mode.
+  DumpTokens,                   // Dump out preprocessed tokens.
+  DumpRawTokens,                // Dump out raw tokens.
   RunAnalysis                   // Run one or more source code analyses. 
 };
 
@@ -92,7 +93,9 @@ ProgAction(llvm::cl::desc("Choose output type:"), llvm::cl::ZeroOrMore,
                         "Just run preprocessor, no output (for timings)"),
              clEnumValN(PrintPreprocessedInput, "E",
                         "Run preprocessor, emit preprocessed file"),
-             clEnumValN(DumpTokens, "dumptokens",
+             clEnumValN(DumpRawTokens, "dump-raw-tokens",
+                        "Lex file in raw mode and dump raw tokens"),
+             clEnumValN(DumpTokens, "dump-tokens",
                         "Run preprocessor, dump internal rep of tokens"),
              clEnumValN(ParseNoop, "parse-noop",
                         "Run parser with noop callbacks (for timings)"),
@@ -1104,9 +1107,29 @@ static void ProcessInputFile(Preprocessor &PP, PreprocessorFactory &PPF,
 
     break;
       
+  case DumpRawTokens: {
+    SourceManager &SM = PP.getSourceManager();
+    std::pair<const char*,const char*> File =
+      SM.getBufferData(SM.getMainFileID());
+    // Start lexing the specified input file.
+    Lexer RawLex(SourceLocation::getFileLoc(SM.getMainFileID(), 0),
+                 PP.getLangOptions(), File.first, File.second);
+    RawLex.SetKeepWhitespaceMode(true);
+
+    Token RawTok;
+
+    RawLex.LexFromRawLexer(RawTok);
+    while (RawTok.isNot(tok::eof)) {
+      PP.DumpToken(RawTok, true);
+      fprintf(stderr, "\n");
+      RawLex.LexFromRawLexer(RawTok);
+    }
+    ClearSourceMgr = true;
+    break;
+  }
   case DumpTokens: {                 // Token dump mode.
     Token Tok;
-    // Start parsing the specified input file.
+    // Start preprocessing the specified input file.
     PP.EnterMainSourceFile();
     do {
       PP.Lex(Tok);
diff --git a/clang/test/Preprocessor/dumptokens_phyloc.c b/clang/test/Preprocessor/dumptokens_phyloc.c
index ea468728bb4..b5c4d7bae3a 100644
--- a/clang/test/Preprocessor/dumptokens_phyloc.c
+++ b/clang/test/Preprocessor/dumptokens_phyloc.c
@@ -1,4 +1,4 @@
-// RUN: clang -dumptokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
+// RUN: clang -dump-tokens %s 2>&1 | grep "PhysLoc=[-_.a-zA-Z/\\]*:3:20"
 
 #define TESTPHYLOC 10