[clang-scan-deps] reuse the file manager across invocations of

the dependency scanner on a single worker thread

This behavior can be controlled using the new `-reuse-filemanager` clang-scan-deps
option. By default the file manager is reused.

The added test/ClangScanDeps/symlink.cpp is able to pass with
the reused filemanager after the related FileEntryRef changes
landed earlier. The test test/ClangScanDeps/subframework_header_dir_symlink.m
still fails when the file manager is reused (I run the FileCheck with not to
make it PASS). I will address this in a follow-up patch that improves
the DirectoryEntry name modelling in the FileManager.

llvm-svn: 370420

11 files changed, 101 insertions, 8 deletions

diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
index 0dde0ad35df..c49f92d082c 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningService.h
@@ -34,16 +34,19 @@ enum class ScanningMode {
 /// the invidual dependency scanning workers.
 class DependencyScanningService {
 public:
-  DependencyScanningService(ScanningMode Mode);
+  DependencyScanningService(ScanningMode Mode, bool ReuseFileManager = true);
 
   ScanningMode getMode() const { return Mode; }
 
+  bool canReuseFileManager() const { return ReuseFileManager; }
+
   DependencyScanningFilesystemSharedCache &getSharedCache() {
     return SharedCache;
   }
 
 private:
   const ScanningMode Mode;
+  const bool ReuseFileManager;
   /// The global file system cache.
   DependencyScanningFilesystemSharedCache SharedCache;
 };
diff --git a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
index 79c652ef5a9..d56f5395da1 100644
--- a/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
+++ b/clang/include/clang/Tooling/DependencyScanning/DependencyScanningWorker.h
@@ -54,6 +54,9 @@ private:
   /// dependencies. This filesystem persists accross multiple compiler
   /// invocations.
   llvm::IntrusiveRefCntPtr<DependencyScanningWorkerFilesystem> DepFS;
+  /// The file manager that is reused accross multiple invocations by this
+  /// worker. If null, the file manager will not be reused.
+  llvm::IntrusiveRefCntPtr<FileManager> Files;
 };
 
 } // end namespace dependencies
diff --git a/clang/include/clang/Tooling/Tooling.h b/clang/include/clang/Tooling/Tooling.h
index 5df816e6715..b8c7435c5e2 100644
--- a/clang/include/clang/Tooling/Tooling.h
+++ b/clang/include/clang/Tooling/Tooling.h
@@ -349,12 +349,15 @@ public:
   /// clang modules.
   /// \param BaseFS VFS used for all underlying file accesses when running the
   /// tool.
+  /// \param Files The file manager to use for underlying file operations when
+  /// running the tool.
   ClangTool(const CompilationDatabase &Compilations,
             ArrayRef<std::string> SourcePaths,
             std::shared_ptr<PCHContainerOperations> PCHContainerOps =
                 std::make_shared<PCHContainerOperations>(),
             IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS =
-                llvm::vfs::getRealFileSystem());
+                llvm::vfs::getRealFileSystem(),
+            IntrusiveRefCntPtr<FileManager> Files = nullptr);
 
   ~ClangTool();
 
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
index 48aa68218c3..6ddce0dcee8 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningService.cpp
@@ -12,5 +12,6 @@ using namespace clang;
 using namespace tooling;
 using namespace dependencies;
 
-DependencyScanningService::DependencyScanningService(ScanningMode Mode)
-    : Mode(Mode) {}
+DependencyScanningService::DependencyScanningService(ScanningMode Mode,
+                                                     bool ReuseFileManager)
+    : Mode(Mode), ReuseFileManager(ReuseFileManager) {}
diff --git a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
index c80a55645eb..2d49e0d794a 100644
--- a/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
+++ b/clang/lib/Tooling/DependencyScanning/DependencyScanningWorker.cpp
@@ -148,6 +148,8 @@ DependencyScanningWorker::DependencyScanningWorker(
   if (Service.getMode() == ScanningMode::MinimizedSourcePreprocessing)
     DepFS = new DependencyScanningWorkerFilesystem(Service.getSharedCache(),
                                                    RealFS);
+  if (Service.canReuseFileManager())
+    Files = new FileManager(FileSystemOptions(), RealFS);
 }
 
 llvm::Expected<std::string>
@@ -164,7 +166,7 @@ DependencyScanningWorker::getDependencyFile(const std::string &Input,
   /// Create the tool that uses the underlying file system to ensure that any
   /// file system requests that are made by the driver do not go through the
   /// dependency scanning filesystem.
-  tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS);
+  tooling::ClangTool Tool(CDB, Input, PCHContainerOps, RealFS, Files);
   Tool.clearArgumentsAdjusters();
   Tool.setRestoreWorkingDir(false);
   Tool.setPrintErrorMessage(false);
diff --git a/clang/lib/Tooling/Tooling.cpp b/clang/lib/Tooling/Tooling.cpp
index 8c0d13d243d..472a3ab57c3 100644
--- a/clang/lib/Tooling/Tooling.cpp
+++ b/clang/lib/Tooling/Tooling.cpp
@@ -378,16 +378,20 @@ bool FrontendActionFactory::runInvocation(
 ClangTool::ClangTool(const CompilationDatabase &Compilations,
                      ArrayRef<std::string> SourcePaths,
                      std::shared_ptr<PCHContainerOperations> PCHContainerOps,
-                     IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS)
+                     IntrusiveRefCntPtr<llvm::vfs::FileSystem> BaseFS,
+                     IntrusiveRefCntPtr<FileManager> Files)
     : Compilations(Compilations), SourcePaths(SourcePaths),
       PCHContainerOps(std::move(PCHContainerOps)),
       OverlayFileSystem(new llvm::vfs::OverlayFileSystem(std::move(BaseFS))),
       InMemoryFileSystem(new llvm::vfs::InMemoryFileSystem),
-      Files(new FileManager(FileSystemOptions(), OverlayFileSystem)) {
+      Files(Files ? Files
+                  : new FileManager(FileSystemOptions(), OverlayFileSystem)) {
   OverlayFileSystem->pushOverlay(InMemoryFileSystem);
   appendArgumentsAdjuster(getClangStripOutputAdjuster());
   appendArgumentsAdjuster(getClangSyntaxOnlyAdjuster());
   appendArgumentsAdjuster(getClangStripDependencyFileAdjuster());
+  if (Files)
+    Files->setVirtualFileSystem(OverlayFileSystem);
 }
 
 ClangTool::~ClangTool() = default;
diff --git a/clang/test/ClangScanDeps/Inputs/subframework_header_dir_symlink_cdb.json b/clang/test/ClangScanDeps/Inputs/subframework_header_dir_symlink_cdb.json
new file mode 100644
index 00000000000..a405c6b3af1
--- /dev/null
+++ b/clang/test/ClangScanDeps/Inputs/subframework_header_dir_symlink_cdb.json
@@ -0,0 +1,12 @@
+[
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/subframework_header_dir_symlink.m -D EMPTY -iframework Inputs/frameworks",
+  "file": "DIR/subframework_header_dir_symlink.m"
+},
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/subframework_header_dir_symlink2.m -FInputs/frameworks_symlink -iframework Inputs/frameworks",
+  "file": "DIR/subframework_header_dir_symlink2.m"
+}
+]
diff --git a/clang/test/ClangScanDeps/Inputs/symlink_cdb.json b/clang/test/ClangScanDeps/Inputs/symlink_cdb.json
new file mode 100644
index 00000000000..43bb418adb1
--- /dev/null
+++ b/clang/test/ClangScanDeps/Inputs/symlink_cdb.json
@@ -0,0 +1,12 @@
+[
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/symlink.cpp -IInputs",
+  "file": "DIR/symlink.cpp"
+},
+{
+  "directory": "DIR",
+  "command": "clang -E DIR/symlink2.cpp -IInputs",
+  "file": "DIR/symlink2.cpp"
+}
+]
diff --git a/clang/test/ClangScanDeps/subframework_header_dir_symlink.m b/clang/test/ClangScanDeps/subframework_header_dir_symlink.m
new file mode 100644
index 00000000000..5cc17851e7c
--- /dev/null
+++ b/clang/test/ClangScanDeps/subframework_header_dir_symlink.m
@@ -0,0 +1,25 @@
+// REQUIRES: shell
+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/subframework_header_dir_symlink.m
+// RUN: cp %s %t.dir/subframework_header_dir_symlink2.m
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp -R %S/Inputs/frameworks %t.dir/Inputs/frameworks
+// RUN: ln -s %t.dir/Inputs/frameworks %t.dir/Inputs/frameworks_symlink
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/subframework_header_dir_symlink_cdb.json > %t.cdb
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1  -reuse-filemanager=0 | \
+// RUN:   FileCheck %s
+// FIXME: Make this work when the filemanager is reused:
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -reuse-filemanager=1 | \
+// RUN:   not FileCheck %s
+
+#ifndef EMPTY
+#include "Framework/Framework.h"
+#endif
+
+// CHECK: clang-scan-deps dependency
+// CHECK-NEXT: subframework_header_dir_symlink.m
+// CHECK: clang-scan-deps dependency
+// CHECK-NEXT: subframework_header_dir_symlink.m
+// CHECK-NEXT: Inputs{{/|\\}}frameworks_symlink{{/|\\}}Framework.framework{{/|\\}}Headers{{/|\\}}Framework.h
diff --git a/clang/test/ClangScanDeps/symlink.cpp b/clang/test/ClangScanDeps/symlink.cpp
new file mode 100644
index 00000000000..e1a6ac931aa
--- /dev/null
+++ b/clang/test/ClangScanDeps/symlink.cpp
@@ -0,0 +1,23 @@
+// REQUIRES: shell
+// RUN: rm -rf %t.dir
+// RUN: rm -rf %t.cdb
+// RUN: mkdir -p %t.dir
+// RUN: cp %s %t.dir/symlink.cpp
+// RUN: cp %s %t.dir/symlink2.cpp
+// RUN: mkdir %t.dir/Inputs
+// RUN: cp %S/Inputs/header.h %t.dir/Inputs/header.h
+// RUN: ln -s %t.dir/Inputs/header.h %t.dir/Inputs/symlink.h
+// RUN: sed -e "s|DIR|%/t.dir|g" %S/Inputs/symlink_cdb.json > %t.cdb
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -reuse-filemanager=0 | FileCheck %s
+// RUN: clang-scan-deps -compilation-database %t.cdb -j 1 -reuse-filemanager=1 | FileCheck %s
+
+#include "symlink.h"
+#include "header.h"
+
+// CHECK: symlink.cpp
+// CHECK-NEXT: Inputs{{/|\\}}symlink.h
+// CHECK-NEXT: Inputs{{/|\\}}header.h
+
+// CHECK: symlink2.cpp
+// CHECK-NEXT: Inputs{{/|\\}}symlink.h
+// CHECK-NEXT: Inputs{{/|\\}}header.h
diff --git a/clang/tools/clang-scan-deps/ClangScanDeps.cpp b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
index b838845c448..5e567fef9da 100644
--- a/clang/tools/clang-scan-deps/ClangScanDeps.cpp
+++ b/clang/tools/clang-scan-deps/ClangScanDeps.cpp
@@ -107,6 +107,11 @@ llvm::cl::opt<std::string>
                   llvm::cl::desc("Compilation database"), llvm::cl::Required,
                   llvm::cl::cat(DependencyScannerCategory));
 
+llvm::cl::opt<bool> ReuseFileManager(
+    "reuse-filemanager",
+    llvm::cl::desc("Reuse the file manager and its cache between invocations."),
+    llvm::cl::init(true), llvm::cl::cat(DependencyScannerCategory));
+
 } // end anonymous namespace
 
 int main(int argc, const char **argv) {
@@ -153,7 +158,7 @@ int main(int argc, const char **argv) {
   // Print out the dependency results to STDOUT by default.
   SharedStream DependencyOS(llvm::outs());
 
-  DependencyScanningService Service(ScanMode);
+  DependencyScanningService Service(ScanMode, ReuseFileManager);
 #if LLVM_ENABLE_THREADS
   unsigned NumWorkers =
       NumThreads == 0 ? llvm::hardware_concurrency() : NumThreads;