[clang][clang-scan-deps] Aggregate the full dependency information.

Differential Revision: https://reviews.llvm.org/D70268

3 files changed, 143 insertions, 80 deletions

diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
index f643c538f8f..31b8346b4ef 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningTool.cpp
@@ -8,24 +8,25 @@
 
 #include "clang/Tooling/DependencyScanning/DependencyScanningTool.h"
 #include "clang/Frontend/Utils.h"
-#include "llvm/Support/JSON.h"
-
-static llvm::json::Array toJSONSorted(const llvm::StringSet<> &Set) {
-  std::vector<llvm::StringRef> Strings;
-  for (auto &&I : Set)
-    Strings.push_back(I.getKey());
-  std::sort(Strings.begin(), Strings.end());
-  return llvm::json::Array(Strings);
-}
 
 namespace clang{
 namespace tooling{
 namespace dependencies{
 
+std::vector<std::string> FullDependencies::getAdditionalCommandLine(
+    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
+    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const {
+  std::vector<std::string> Ret = AdditionalNonPathCommandLine;
+
+  dependencies::detail::appendCommonModuleArguments(
+      ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret);
+
+  return Ret;
+}
+
 DependencyScanningTool::DependencyScanningTool(
     DependencyScanningService &Service)
-    : Format(Service.getFormat()), Worker(Service) {
-}
+    : Worker(Service) {}
 
 llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
     const tooling::CompilationDatabase &Compilations, StringRef CWD) {
@@ -75,8 +76,33 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
     std::vector<std::string> Dependencies;
   };
 
+  // We expect a single command here because if a source file occurs multiple
+  // times in the original CDB, then `computeDependencies` would run the
+  // `DependencyScanningAction` once for every time the input occured in the
+  // CDB. Instead we split up the CDB into single command chunks to avoid this
+  // behavior.
+  assert(Compilations.getAllCompileCommands().size() == 1 &&
+         "Expected a compilation database with a single command!");
+  std::string Input = Compilations.getAllCompileCommands().front().Filename;
+
+  MakeDependencyPrinterConsumer Consumer;
+  auto Result = Worker.computeDependencies(Input, CWD, Compilations, Consumer);
+  if (Result)
+    return std::move(Result);
+  std::string Output;
+  Consumer.printDependencies(Output);
+  return Output;
+}
+
+llvm::Expected<FullDependenciesResult>
+DependencyScanningTool::getFullDependencies(
+    const tooling::CompilationDatabase &Compilations, StringRef CWD,
+    const llvm::StringSet<> &AlreadySeen) {
   class FullDependencyPrinterConsumer : public DependencyConsumer {
   public:
+    FullDependencyPrinterConsumer(const llvm::StringSet<> &AlreadySeen)
+        : AlreadySeen(AlreadySeen) {}
+
     void handleFileDependency(const DependencyOutputOptions &Opts,
                               StringRef File) override {
       Dependencies.push_back(File);
@@ -90,55 +116,41 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
       ContextHash = std::move(Hash);
     }
 
-    void printDependencies(std::string &S, StringRef MainFile) {
-      // Sort the modules by name to get a deterministic order.
-      std::vector<StringRef> Modules;
-      for (auto &&Dep : ClangModuleDeps)
-        Modules.push_back(Dep.first);
-      std::sort(Modules.begin(), Modules.end());
+    FullDependenciesResult getFullDependencies() const {
+      FullDependencies FD;
 
-      llvm::raw_string_ostream OS(S);
+      FD.ContextHash = std::move(ContextHash);
 
-      using namespace llvm::json;
+      FD.FileDeps.assign(Dependencies.begin(), Dependencies.end());
 
-      Array Imports;
-      for (auto &&ModName : Modules) {
-        auto &MD = ClangModuleDeps[ModName];
+      for (auto &&M : ClangModuleDeps) {
+        auto &MD = M.second;
         if (MD.ImportedByMainFile)
-          Imports.push_back(MD.ModuleName);
+          FD.ClangModuleDeps.push_back({MD.ModuleName, ContextHash});
       }
 
-      Array Mods;
-      for (auto &&ModName : Modules) {
-        auto &MD = ClangModuleDeps[ModName];
-        Object Mod{
-            {"name", MD.ModuleName},
-            {"file-deps", toJSONSorted(MD.FileDeps)},
-            {"clang-module-deps", toJSONSorted(MD.ClangModuleDeps)},
-            {"clang-modulemap-file", MD.ClangModuleMapFile},
-        };
-        Mods.push_back(std::move(Mod));
-      }
+      FullDependenciesResult FDR;
 
-      Object O{
-          {"input-file", MainFile},
-          {"clang-context-hash", ContextHash},
-          {"file-deps", Dependencies},
-          {"clang-module-deps", std::move(Imports)},
-          {"clang-modules", std::move(Mods)},
-      };
+      for (auto &&M : ClangModuleDeps) {
+        // TODO: Avoid handleModuleDependency even being called for modules
+        //   we've already seen.
+        if (AlreadySeen.count(M.first))
+          continue;
+        FDR.DiscoveredModules.push_back(std::move(M.second));
+      }
 
-      S = llvm::formatv("{0:2},\n", Value(std::move(O))).str();
-      return;
+      FDR.FullDeps = std::move(FD);
+      return FDR;
     }
 
   private:
     std::vector<std::string> Dependencies;
     std::unordered_map<std::string, ModuleDeps> ClangModuleDeps;
     std::string ContextHash;
+    std::vector<std::string> OutputPaths;
+    const llvm::StringSet<> &AlreadySeen;
   };
 
-  
   // We expect a single command here because if a source file occurs multiple
   // times in the original CDB, then `computeDependencies` would run the
   // `DependencyScanningAction` once for every time the input occured in the
@@ -147,26 +159,13 @@ llvm::Expected<std::string> DependencyScanningTool::getDependencyFile(
   assert(Compilations.getAllCompileCommands().size() == 1 &&
          "Expected a compilation database with a single command!");
   std::string Input = Compilations.getAllCompileCommands().front().Filename;
-  
-  if (Format == ScanningOutputFormat::Make) {
-    MakeDependencyPrinterConsumer Consumer;
-    auto Result =
-        Worker.computeDependencies(Input, CWD, Compilations, Consumer);
-    if (Result)
-      return std::move(Result);
-    std::string Output;
-    Consumer.printDependencies(Output);
-    return Output;
-  } else {
-    FullDependencyPrinterConsumer Consumer;
-    auto Result =
-        Worker.computeDependencies(Input, CWD, Compilations, Consumer);
-    if (Result)
-      return std::move(Result);
-    std::string Output;
-    Consumer.printDependencies(Output, Input);
-    return Output;
-  }
+
+  FullDependencyPrinterConsumer Consumer(AlreadySeen);
+  llvm::Error Result =
+      Worker.computeDependencies(Input, CWD, Compilations, Consumer);
+  if (Result)
+    return std::move(Result);
+  return Consumer.getFullDependencies();
 }
 
 } // end namespace dependencies
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index edf2cf8bd70..51bd8ec3957 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -142,11 +142,18 @@ public:
                                                         Consumer));
       break;
     case ScanningOutputFormat::Full:
-      Compiler.addDependencyCollector(
-          std::make_shared<ModuleDepCollector>(Compiler, Consumer));
+      Compiler.addDependencyCollector(std::make_shared<ModuleDepCollector>(
+          std::move(Opts), Compiler, Consumer));
       break;
     }
 
+    // Consider different header search and diagnostic options to create
+    // different modules. This avoids the unsound aliasing of module PCMs.
+    //
+    // TODO: Implement diagnostic bucketing and header search pruning to reduce
+    // the impact of strict context hashing.
+    Compiler.getHeaderSearchOpts().ModulesStrictContextHash = false;
+
     Consumer.handleContextHash(Compiler.getInvocation().getModuleHash());
 
     auto Action = std::make_unique<PreprocessOnlyAction>();
diff --git a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
index 422940047f2..0d3f9cda634 100644
--- a/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
+++ b/clang/lib/Tooling/DependencyScanning/ModuleDepCollector.cpp
@@ -17,6 +17,47 @@ using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
+std::vector<std::string> ModuleDeps::getFullCommandLine(
+    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
+    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps) const {
+  std::vector<std::string> Ret = NonPathCommandLine;
+
+  // TODO: Build full command line. That also means capturing the original
+  //       command line into NonPathCommandLine.
+
+  dependencies::detail::appendCommonModuleArguments(
+      ClangModuleDeps, LookupPCMPath, LookupModuleDeps, Ret);
+
+  return Ret;
+}
+
+void dependencies::detail::appendCommonModuleArguments(
+    llvm::ArrayRef<ClangModuleDep> Modules,
+    std::function<StringRef(ClangModuleDep)> LookupPCMPath,
+    std::function<const ModuleDeps &(ClangModuleDep)> LookupModuleDeps,
+    std::vector<std::string> &Result) {
+  llvm::StringSet<> AlreadyAdded;
+
+  std::function<void(llvm::ArrayRef<ClangModuleDep>)> AddArgs =
+      [&](llvm::ArrayRef<ClangModuleDep> Modules) {
+        for (const ClangModuleDep &CMD : Modules) {
+          if (!AlreadyAdded.insert(CMD.ModuleName + CMD.ContextHash).second)
+            continue;
+          const ModuleDeps &M = LookupModuleDeps(CMD);
+          // Depth first traversal.
+          AddArgs(M.ClangModuleDeps);
+          Result.push_back(("-fmodule-file=" + LookupPCMPath(CMD)).str());
+          if (!M.ClangModuleMapFile.empty()) {
+            Result.push_back("-fmodule-map-file=" + M.ClangModuleMapFile);
+          }
+        }
+      };
+
+  Result.push_back("-fno-implicit-modules");
+  Result.push_back("-fno-implicit-module-maps");
+  AddArgs(Modules);
+}
+
 void ModuleDepCollectorPP::FileChanged(SourceLocation Loc,
                                        FileChangeReason Reason,
                                        SrcMgr::CharacteristicKind FileType,
@@ -50,7 +91,16 @@ void ModuleDepCollectorPP::InclusionDirective(
     // here as `FileChanged` will never see it.
     MDC.MainDeps.push_back(FileName);
   }
+  handleImport(Imported);
+}
 
+void ModuleDepCollectorPP::moduleImport(SourceLocation ImportLoc,
+                                        ModuleIdPath Path,
+                                        const Module *Imported) {
+  handleImport(Imported);
+}
+
+void ModuleDepCollectorPP::handleImport(const Module *Imported) {
   if (!Imported)
     return;
 
@@ -71,9 +121,8 @@ void ModuleDepCollectorPP::EndOfMainFile() {
   for (auto &&I : MDC.Deps)
     MDC.Consumer.handleModuleDependency(I.second);
 
-  DependencyOutputOptions Opts;
   for (auto &&I : MDC.MainDeps)
-    MDC.Consumer.handleFileDependency(Opts, I);
+    MDC.Consumer.handleFileDependency(*MDC.Opts, I);
 }
 
 void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
@@ -94,7 +143,7 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
 
   MD.ClangModuleMapFile = ModuleMap ? ModuleMap->getName() : "";
   MD.ModuleName = M->getFullModuleName();
-  MD.ModulePCMPath = M->getASTFile()->getName();
+  MD.ImplicitModulePCMPath = M->getASTFile()->getName();
   MD.ContextHash = MDC.ContextHash;
   serialization::ModuleFile *MF =
       MDC.Instance.getASTReader()->getModuleManager().lookup(M->getASTFile());
@@ -103,30 +152,38 @@ void ModuleDepCollectorPP::handleTopLevelModule(const Module *M) {
         MD.FileDeps.insert(IF.getFile()->getName());
       });
 
-  addAllSubmoduleDeps(M, MD);
+  llvm::DenseSet<const Module *> AddedModules;
+  addAllSubmoduleDeps(M, MD, AddedModules);
 }
 
-void ModuleDepCollectorPP::addAllSubmoduleDeps(const Module *M,
-                                               ModuleDeps &MD) {
-  addModuleDep(M, MD);
+void ModuleDepCollectorPP::addAllSubmoduleDeps(
+    const Module *M, ModuleDeps &MD,
+    llvm::DenseSet<const Module *> &AddedModules) {
+  addModuleDep(M, MD, AddedModules);
 
   for (const Module *SubM : M->submodules())
-    addAllSubmoduleDeps(SubM, MD);
+    addAllSubmoduleDeps(SubM, MD, AddedModules);
 }
 
-void ModuleDepCollectorPP::addModuleDep(const Module *M, ModuleDeps &MD) {
+void ModuleDepCollectorPP::addModuleDep(
+    const Module *M, ModuleDeps &MD,
+    llvm::DenseSet<const Module *> &AddedModules) {
   for (const Module *Import : M->Imports) {
     if (Import->getTopLevelModule() != M->getTopLevelModule()) {
-      MD.ClangModuleDeps.insert(Import->getTopLevelModuleName());
+      if (AddedModules.insert(Import->getTopLevelModule()).second)
+        MD.ClangModuleDeps.push_back(
+            {Import->getTopLevelModuleName(),
+             Instance.getInvocation().getModuleHash()});
       handleTopLevelModule(Import->getTopLevelModule());
     }
   }
 }
 
-ModuleDepCollector::ModuleDepCollector(CompilerInstance &I,
-                                       DependencyConsumer &C)
-    : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()) {
-}
+ModuleDepCollector::ModuleDepCollector(
+    std::unique_ptr<DependencyOutputOptions> Opts, CompilerInstance &I,
+    DependencyConsumer &C)
+    : Instance(I), Consumer(C), ContextHash(I.getInvocation().getModuleHash()),
+      Opts(std::move(Opts)) {}
 
 void ModuleDepCollector::attachToPreprocessor(Preprocessor &PP) {
   PP.addPPCallbacks(std::make_unique<ModuleDepCollectorPP>(Instance, *this));