[llvm-rc] Handle C preprocessor output

When preprocessing resource scripts (which can easily be done outside
of llvm-rc), included headers can leave behind C declarations (despite
preprocessing with -DRC_INVOKED), that can't be parsed by a resource
compiler.

This is handled in all of rc.exe, by parsing the preprocessor output
line markers and ignoring content from files named *.h and *.c,
documented at [1].

In addition to this filtering, strip out any other preprocessor directive
that is left behind (like pragmas) which also can't be handled by the
tokenizer.

The added test uses both standard #line markers (supported by rc.exe) and
GNU style extended line markers, thus this test doesn't pass with rc.exe,
but passes with GNU windres. (Windres on the other hand doesn't filter
out files named *.c, only *.h.)

Differential Revision: https://reviews.llvm.org/D46579

[1] https://msdn.microsoft.com/en-us/library/windows/desktop/aa381033(v=vs.85).aspx

llvm-svn: 331903

4 files changed, 151 insertions, 1 deletions

diff --git a/llvm/tools/llvm-rc/CMakeLists.txt b/llvm/tools/llvm-rc/CMakeLists.txt
index e5c0eb25d7b..4cadc176691 100644
--- a/llvm/tools/llvm-rc/CMakeLists.txt
+++ b/llvm/tools/llvm-rc/CMakeLists.txt
@@ -11,6 +11,7 @@ add_public_tablegen_target(RcTableGen)
 add_llvm_tool(llvm-rc
   llvm-rc.cpp
   ResourceFileWriter.cpp
+  ResourceScriptCppFilter.cpp
   ResourceScriptParser.cpp
   ResourceScriptStmt.cpp
   ResourceScriptToken.cpp
diff --git a/llvm/tools/llvm-rc/ResourceScriptCppFilter.cpp b/llvm/tools/llvm-rc/ResourceScriptCppFilter.cpp
new file mode 100644
index 00000000000..07740f9e8bf
--- /dev/null
+++ b/llvm/tools/llvm-rc/ResourceScriptCppFilter.cpp
@@ -0,0 +1,112 @@
+//===-- ResourceScriptCppFilter.cpp ----------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This file implements an interface defined in ResourceScriptCppFilter.h.
+//
+//===---------------------------------------------------------------------===//
+
+#include "ResourceScriptCppFilter.h"
+#include "llvm/ADT/StringExtras.h"
+
+#include <vector>
+
+using namespace llvm;
+
+namespace {
+
+class Filter {
+public:
+  explicit Filter(StringRef Input) : Data(Input), DataLength(Input.size()) {}
+
+  std::string run();
+
+private:
+  // Parse the line, returning whether the line should be included in
+  // the output.
+  bool parseLine(StringRef Line);
+
+  bool streamEof() const;
+
+  StringRef Data;
+  size_t DataLength;
+
+  size_t Pos = 0;
+  bool Outputting = true;
+};
+
+std::string Filter::run() {
+  std::vector<StringRef> Output;
+
+  while (!streamEof() && Pos != StringRef::npos) {
+    size_t LineStart = Pos;
+    Pos = Data.find_first_of("\r\n", Pos);
+    Pos = Data.find_first_not_of("\r\n", Pos);
+    StringRef Line = Data.take_front(Pos).drop_front(LineStart);
+
+    if (parseLine(Line))
+      Output.push_back(Line);
+  }
+
+  return llvm::join(Output, "");
+}
+
+bool Filter::parseLine(StringRef Line) {
+  Line = Line.ltrim();
+
+  if (!Line.consume_front("#")) {
+    // A normal content line, filtered according to the current mode.
+    return Outputting;
+  }
+
+  // Found a preprocessing directive line. From here on, we always return
+  // false since the preprocessing directives should be filtered out.
+
+  Line.consume_front("line");
+  if (!Line.startswith(" "))
+    return false; // Not a line directive (pragma etc).
+
+  // #line 123 "path/file.h"
+  // # 123 "path/file.h" 1
+
+  Line =
+      Line.ltrim(); // There could be multiple spaces after the #line directive
+
+  size_t N;
+  if (Line.consumeInteger(10, N)) // Returns true to signify an error
+    return false;
+
+  Line = Line.ltrim();
+
+  if (!Line.consume_front("\""))
+    return false; // Malformed line, no quote found.
+
+  // Split the string at the last quote (in case the path name had
+  // escaped quotes as well).
+  Line = Line.rsplit('"').first;
+
+  StringRef Ext = Line.rsplit('.').second;
+
+  if (Ext.equals_lower("h") || Ext.equals_lower("c")) {
+    Outputting = false;
+  } else {
+    Outputting = true;
+  }
+
+  return false;
+}
+
+bool Filter::streamEof() const { return Pos == DataLength; }
+
+} // anonymous namespace
+
+namespace llvm {
+
+std::string filterCppOutput(StringRef Input) { return Filter(Input).run(); }
+
+} // namespace llvm
diff --git a/llvm/tools/llvm-rc/ResourceScriptCppFilter.h b/llvm/tools/llvm-rc/ResourceScriptCppFilter.h
new file mode 100644
index 00000000000..2e61a075fb8
--- /dev/null
+++ b/llvm/tools/llvm-rc/ResourceScriptCppFilter.h
@@ -0,0 +1,35 @@
+//===-- ResourceScriptCppFilter.h ------------------------------*- C++ -*-===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===---------------------------------------------------------------------===//
+//
+// This filters the input to llvm-rc for preprocessor markers, removing
+// preprocessing directives that a preprocessor can output or leave behind.
+//
+// It also filters out any contribution from files named *.h or *.c, based
+// on preprocessor line markers. When preprocessing RC files, the included
+// headers can leave behind C declarations, that RC doesn't understand.
+// Rc.exe simply discards anything that comes from files named *.h or *.h.
+//
+// https://msdn.microsoft.com/en-us/library/windows/desktop/aa381033(v=vs.85).aspx
+//
+//===---------------------------------------------------------------------===//
+
+#ifndef LLVM_TOOLS_LLVMRC_RESOURCESCRIPTCPPFILTER_H
+#define LLVM_TOOLS_LLVMRC_RESOURCESCRIPTCPPFILTER_H
+
+#include "llvm/ADT/StringRef.h"
+
+#include <string>
+
+namespace llvm {
+
+std::string filterCppOutput(StringRef Input);
+
+} // namespace llvm
+
+#endif
diff --git a/llvm/tools/llvm-rc/llvm-rc.cpp b/llvm/tools/llvm-rc/llvm-rc.cpp
index 6495052e8ab..41360d2a7bd 100644
--- a/llvm/tools/llvm-rc/llvm-rc.cpp
+++ b/llvm/tools/llvm-rc/llvm-rc.cpp
@@ -13,6 +13,7 @@
 //===----------------------------------------------------------------------===//
 
 #include "ResourceFileWriter.h"
+#include "ResourceScriptCppFilter.h"
 #include "ResourceScriptParser.h"
 #include "ResourceScriptStmt.h"
 #include "ResourceScriptToken.h"
@@ -111,7 +112,8 @@ int main(int Argc, const char **Argv) {
   std::unique_ptr<MemoryBuffer> FileContents = std::move(*File);
   StringRef Contents = FileContents->getBuffer();
 
-  std::vector<RCToken> Tokens = ExitOnErr(tokenizeRC(Contents));
+  std::string FilteredContents = filterCppOutput(Contents);
+  std::vector<RCToken> Tokens = ExitOnErr(tokenizeRC(FilteredContents));
 
   if (BeVerbose) {
     const Twine TokenNames[] = {