Modules: Cache PCMs in memory and avoid a use-after-free

Clang's internal build system for implicit modules uses lock files to
ensure that after a process writes a PCM it will read the same one back
in (without contention from other -cc1 commands).  Since PCMs are read
from disk repeatedly while invalidating, building, and importing, the
lock is not released quickly.  Furthermore, the LockFileManager is not
robust in every environment.  Other -cc1 commands can stall until
timeout (after about eight minutes).

This commit changes the lock file from being necessary for correctness
to a (possibly dubious) performance hack.  The remaining benefit is to
reduce duplicate work in competing -cc1 commands which depend on the
same module.  Follow-up commits will change the internal build system to
continue after a timeout, and reduce the timeout.  Perhaps we should
reconsider blocking at all.

This also fixes a use-after-free, when one part of a compilation
validates a PCM and starts using it, and another tries to swap out the
PCM for something new.

The PCMCache is a new type called MemoryBufferCache, which saves memory
buffers based on their filename.  Its ownership is shared by the
CompilerInstance and ModuleManager.

  - The ModuleManager stores PCMs there that it loads from disk, never
    touching the disk if the cache is hot.

  - When modules fail to validate, they're removed from the cache.

  - When a CompilerInstance is spawned to build a new module, each
    already-loaded PCM is assumed to be valid, and is frozen to avoid
    the use-after-free.

  - Any newly-built module is written directly to the cache to avoid the
    round-trip to the filesystem, making lock files unnecessary for
    correctness.

Original patch by Manman Ren; most testcases by Adrian Prantl!

llvm-svn: 298165

10 files changed, 70 insertions, 0 deletions

diff --git a/clang/test/Modules/Inputs/system-out-of-date/X.h b/clang/test/Modules/Inputs/system-out-of-date/X.h
new file mode 100644
index 00000000000..edcfa18f5af
--- /dev/null
+++ b/clang/test/Modules/Inputs/system-out-of-date/X.h
@@ -0,0 +1 @@
+#import <Y.h>
diff --git a/clang/test/Modules/Inputs/system-out-of-date/Y.h b/clang/test/Modules/Inputs/system-out-of-date/Y.h
new file mode 100644
index 00000000000..90fe1bcc585
--- /dev/null
+++ b/clang/test/Modules/Inputs/system-out-of-date/Y.h
@@ -0,0 +1 @@
+//empty
diff --git a/clang/test/Modules/Inputs/system-out-of-date/Z.h b/clang/test/Modules/Inputs/system-out-of-date/Z.h
new file mode 100644
index 00000000000..edcfa18f5af
--- /dev/null
+++ b/clang/test/Modules/Inputs/system-out-of-date/Z.h
@@ -0,0 +1 @@
+#import <Y.h>
diff --git a/clang/test/Modules/Inputs/system-out-of-date/module.map b/clang/test/Modules/Inputs/system-out-of-date/module.map
new file mode 100644
index 00000000000..0c0f42a5d01
--- /dev/null
+++ b/clang/test/Modules/Inputs/system-out-of-date/module.map
@@ -0,0 +1,12 @@
+module X [system] {
+  header "X.h" // imports Y
+  export *
+}
+module Y {
+  header "Y.h"
+  export *
+}
+module Z {
+  header "Z.h" // imports Y
+  export *
+}
diff --git a/clang/test/Modules/Inputs/warning-mismatch/Mismatch.h b/clang/test/Modules/Inputs/warning-mismatch/Mismatch.h
new file mode 100644
index 00000000000..a07b0ee31ea
--- /dev/null
+++ b/clang/test/Modules/Inputs/warning-mismatch/Mismatch.h
@@ -0,0 +1 @@
+struct Mismatch { int i; };
diff --git a/clang/test/Modules/Inputs/warning-mismatch/System.h b/clang/test/Modules/Inputs/warning-mismatch/System.h
new file mode 100644
index 00000000000..8e69e704c75
--- /dev/null
+++ b/clang/test/Modules/Inputs/warning-mismatch/System.h
@@ -0,0 +1,2 @@
+#import "Mismatch.h"
+struct System { int i; };
diff --git a/clang/test/Modules/Inputs/warning-mismatch/module.modulemap b/clang/test/Modules/Inputs/warning-mismatch/module.modulemap
new file mode 100644
index 00000000000..c22cde45978
--- /dev/null
+++ b/clang/test/Modules/Inputs/warning-mismatch/module.modulemap
@@ -0,0 +1,7 @@
+module System [system] {
+  header "System.h"
+}
+
+module Mismatch {
+  header "Mismatch.h"
+}
diff --git a/clang/test/Modules/outofdate-rebuild.m b/clang/test/Modules/outofdate-rebuild.m
new file mode 100644
index 00000000000..510325f62dc
--- /dev/null
+++ b/clang/test/Modules/outofdate-rebuild.m
@@ -0,0 +1,15 @@
+// RUN: rm -rf %t.cache
+// RUN: echo "@import CoreText;" > %t.m
+// RUN: %clang_cc1 -fdisable-module-hash -fmodules-cache-path=%t.cache \
+// RUN:   -fmodules -fimplicit-module-maps -I%S/Inputs/outofdate-rebuild %s \
+// RUN:   -fsyntax-only
+// RUN: %clang_cc1 -DMISMATCH -Werror -fdisable-module-hash \
+// RUN:   -fmodules-cache-path=%t.cache -fmodules -fimplicit-module-maps \
+// RUN:   -I%S/Inputs/outofdate-rebuild %t.m -fsyntax-only
+// RUN: %clang_cc1 -fdisable-module-hash -fmodules-cache-path=%t.cache \
+// RUN:   -fmodules -fimplicit-module-maps -I%S/Inputs/outofdate-rebuild %s \
+// RUN:   -fsyntax-only
+
+// This testcase reproduces a use-after-free in when ModuleManager removes an
+// entry from the PCMCache without notifying its parent ASTReader.
+@import Cocoa;
diff --git a/clang/test/Modules/system-out-of-date-test.m b/clang/test/Modules/system-out-of-date-test.m
new file mode 100644
index 00000000000..e78df7b3b3e
--- /dev/null
+++ b/clang/test/Modules/system-out-of-date-test.m
@@ -0,0 +1,17 @@
+// RUN: rm -rf %t.cache
+// RUN: echo '@import X;' | \
+// RUN:   %clang_cc1 -fmodules -fimplicit-module-maps \
+// RUN:     -fmodules-cache-path=%t.cache -I%S/Inputs/system-out-of-date \
+// RUN:     -fsyntax-only -x objective-c -
+//
+// Build something with different diagnostic options.
+// RUN: %clang_cc1 -fmodules -fimplicit-module-maps \
+// RUN:   -fmodules-cache-path=%t.cache -I%S/Inputs/system-out-of-date \
+// RUN:   -fsyntax-only %s -Wnon-modular-include-in-framework-module \
+// RUN:   -Werror=non-modular-include-in-framework-module 2>&1 \
+// RUN: | FileCheck %s
+@import X;
+
+#import <Z.h>
+// CHECK: While building module 'Z' imported from
+// CHECK: {{.*}}Y-{{.*}}pcm' was validated as a system module and is now being imported as a non-system module
diff --git a/clang/test/Modules/warning-mismatch.m b/clang/test/Modules/warning-mismatch.m
new file mode 100644
index 00000000000..dd7c7f82ec6
--- /dev/null
+++ b/clang/test/Modules/warning-mismatch.m
@@ -0,0 +1,13 @@
+// RUN: rm -rf %t.cache
+// RUN: echo "@import Mismatch;" >%t.m
+// RUN: %clang_cc1 -Wno-system-headers -fdisable-module-hash \
+// RUN:   -fmodules-cache-path=%t.cache -fmodules -fimplicit-module-maps \
+// RUN:   -I%S/Inputs/warning-mismatch %t.m -fsyntax-only
+// RUN: %clang_cc1 -Wsystem-headers -fdisable-module-hash \
+// RUN:   -fmodules-cache-path=%t.cache -fmodules -fimplicit-module-maps \
+// RUN:   -I%S/Inputs/warning-mismatch %s -fsyntax-only
+
+// This testcase triggers a warning flag mismatch in an already validated
+// header.
+@import Mismatch;
+@import System;