Add a tool executor that runs actions on all TUs in the compilation database.

Summary: Tool results are deduplicated by the result key.

Reviewers: hokein

Subscribers: klimek, mgorny, cfe-commits

Differential Revision: https://reviews.llvm.org/D41729

llvm-svn: 321864

3 files changed, 170 insertions, 1 deletions

diff --git a/clang/lib/Tooling/AllTUsExecution.cpp b/clang/lib/Tooling/AllTUsExecution.cpp
new file mode 100644
index 00000000000..d35f82f40c5
--- /dev/null
+++ b/clang/lib/Tooling/AllTUsExecution.cpp
@@ -0,0 +1,165 @@
+//===- lib/Tooling/AllTUsExecution.cpp - Execute actions on all TUs. ------===//
+//
+//                     The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+
+#include "clang/Tooling/AllTUsExecution.h"
+#include "clang/Tooling/ToolExecutorPluginRegistry.h"
+#include "llvm/Support/ThreadPool.h"
+
+namespace clang {
+namespace tooling {
+
+const char *AllTUsToolExecutor::ExecutorName = "AllTUsToolExecutor";
+
+namespace {
+llvm::Error make_string_error(const llvm::Twine &Message) {
+  return llvm::make_error<llvm::StringError>(Message,
+                                             llvm::inconvertibleErrorCode());
+}
+
+ArgumentsAdjuster getDefaultArgumentsAdjusters() {
+  return combineAdjusters(
+      getClangStripOutputAdjuster(),
+      combineAdjusters(getClangSyntaxOnlyAdjuster(),
+                       getClangStripDependencyFileAdjuster()));
+}
+
+class ThreadSafeToolResults : public ToolResults {
+public:
+  void addResult(StringRef Key, StringRef Value) override {
+    std::unique_lock<std::mutex> LockGuard(Mutex);
+    Results[Key] = Value;
+  }
+
+  std::vector<std::pair<std::string, std::string>> AllKVResults() override {
+    std::vector<std::pair<std::string, std::string>> KVs;
+    for (const auto &Pair : Results)
+      KVs.emplace_back(Pair.first().str(), Pair.second);
+    return KVs;
+  }
+
+  void forEachResult(llvm::function_ref<void(StringRef Key, StringRef Value)>
+                         Callback) override {
+    for (const auto &Pair : Results)
+      Callback(Pair.first(), Pair.second);
+  }
+
+private:
+  llvm::StringMap<std::string> Results;
+  std::mutex Mutex;
+};
+
+} // namespace
+
+AllTUsToolExecutor::AllTUsToolExecutor(
+    const CompilationDatabase &Compilations, unsigned ThreadCount,
+    std::shared_ptr<PCHContainerOperations> PCHContainerOps)
+    : Compilations(Compilations), Results(new ThreadSafeToolResults),
+      Context(Results.get()), ThreadCount(ThreadCount) {}
+
+AllTUsToolExecutor::AllTUsToolExecutor(
+    CommonOptionsParser Options, unsigned ThreadCount,
+    std::shared_ptr<PCHContainerOperations> PCHContainerOps)
+    : OptionsParser(std::move(Options)),
+      Compilations(OptionsParser->getCompilations()),
+      Results(new ThreadSafeToolResults), Context(Results.get()),
+      ThreadCount(ThreadCount) {}
+
+llvm::Error AllTUsToolExecutor::execute(
+    llvm::ArrayRef<
+        std::pair<std::unique_ptr<FrontendActionFactory>, ArgumentsAdjuster>>
+        Actions) {
+  if (Actions.empty())
+    return make_string_error("No action to execute.");
+
+  if (Actions.size() != 1)
+    return make_string_error(
+        "Only support executing exactly 1 action at this point.");
+
+  std::string ErrorMsg;
+  std::mutex TUMutex;
+  auto AppendError = [&](llvm::Twine Err) {
+    std::unique_lock<std::mutex> LockGuard(TUMutex);
+    ErrorMsg += Err.str();
+  };
+
+  auto Log = [&](llvm::Twine Msg) {
+    std::unique_lock<std::mutex> LockGuard(TUMutex);
+    llvm::errs() << Msg.str() << "\n";
+  };
+
+  auto Files = Compilations.getAllFiles();
+  // Add a counter to track the progress.
+  const std::string TotalNumStr = std::to_string(Files.size());
+  unsigned Counter = 0;
+  auto Count = [&]() {
+    std::unique_lock<std::mutex> LockGuard(TUMutex);
+    return ++Counter;
+  };
+
+  auto &Action = Actions.front();
+
+  {
+    llvm::ThreadPool Pool(ThreadCount == 0 ? llvm::hardware_concurrency()
+                                           : ThreadCount);
+
+    for (std::string File : Files) {
+      Pool.async(
+          [&](std::string Path) {
+            Log("[" + std::to_string(Count()) + "/" + TotalNumStr +
+                "] Processing file " + Path);
+            ClangTool Tool(Compilations, {Path});
+            Tool.appendArgumentsAdjuster(Action.second);
+            Tool.appendArgumentsAdjuster(getDefaultArgumentsAdjusters());
+            for (const auto &FileAndContent : OverlayFiles)
+              Tool.mapVirtualFile(FileAndContent.first(),
+                                  FileAndContent.second);
+            if (Tool.run(Action.first.get()))
+              AppendError(llvm::Twine("Failed to run action on ") + Path +
+                          "\n");
+          },
+          File);
+    }
+  }
+
+  if (!ErrorMsg.empty())
+    return make_string_error(ErrorMsg);
+
+  return llvm::Error::success();
+}
+
+static llvm::cl::opt<unsigned> ExecutorConcurrency(
+    "execute-concurrency",
+    llvm::cl::desc("The number of threads used to process all files in "
+                   "parallel. Set to 0 for hardware concurrency."),
+    llvm::cl::init(0));
+
+class AllTUsToolExecutorPlugin : public ToolExecutorPlugin {
+public:
+  llvm::Expected<std::unique_ptr<ToolExecutor>>
+  create(CommonOptionsParser &OptionsParser) override {
+    if (OptionsParser.getSourcePathList().empty())
+      return make_string_error(
+          "[AllTUsToolExecutorPlugin] Please provide a directory/file path in "
+          "the compilation database.");
+    return llvm::make_unique<AllTUsToolExecutor>(std::move(OptionsParser),
+                                                 ExecutorConcurrency);
+  }
+};
+
+static ToolExecutorPluginRegistry::Add<AllTUsToolExecutorPlugin>
+    X("all-TUs",
+      "Runs FrontendActions on all TUs in the compilation database. "
+      "Tool results are deduplicated by the result key and stored in memory.");
+
+// This anchor is used to force the linker to link in the generated object file
+// and thus register the plugin.
+volatile int AllTUsToolExecutorAnchorSource = 0;
+
+} // end namespace tooling
+} // end namespace clang
diff --git a/clang/lib/Tooling/CMakeLists.txt b/clang/lib/Tooling/CMakeLists.txt
index ee681bbb45a..1999430aa3d 100644
--- a/clang/lib/Tooling/CMakeLists.txt
+++ b/clang/lib/Tooling/CMakeLists.txt
@@ -8,6 +8,7 @@ add_subdirectory(Refactoring)
 add_subdirectory(ASTDiff)
 
 add_clang_library(clangTooling
+  AllTUsExecution.cpp
   ArgumentsAdjusters.cpp
   CommonOptionsParser.cpp
   CompilationDatabase.cpp
diff --git a/clang/lib/Tooling/Execution.cpp b/clang/lib/Tooling/Execution.cpp
index 498d683f892..ff68f85adb2 100644
--- a/clang/lib/Tooling/Execution.cpp
+++ b/clang/lib/Tooling/Execution.cpp
@@ -96,10 +96,13 @@ createExecutorFromCommandLineArgs(int &argc, const char **argv,
 }
 
 // This anchor is used to force the linker to link in the generated object file
-// and thus register the StandaloneToolExecutorPlugin.
+// and thus register the StandaloneToolExecutorPlugin etc.
 extern volatile int StandaloneToolExecutorAnchorSource;
+extern volatile int AllTUsToolExecutorAnchorSource;
 static int LLVM_ATTRIBUTE_UNUSED StandaloneToolExecutorAnchorDest =
     StandaloneToolExecutorAnchorSource;
+static int LLVM_ATTRIBUTE_UNUSED AllTUsToolExecutorAnchorDest =
+    AllTUsToolExecutorAnchorSource;
 
 } // end namespace tooling
 } // end namespace clang