If a declaration is loaded, and then a module import adds a redeclaration, then

ensure that querying the first declaration for its most recent declaration
checks for redeclarations from the imported module.

This works as follows:
 * The 'most recent' pointer on a canonical declaration grows a pointer to the
   external AST source and a generation number (space- and time-optimized for
   the case where there is no external source).
 * Each time the 'most recent' pointer is queried, if it has an external source,
   we check whether it's up to date, and update it if not.
 * The ancillary data stored on the canonical declaration is allocated lazily
   to avoid filling it in for declarations that end up being non-canonical.
   We'll still perform a redundant (ASTContext) allocation if someone asks for
   the most recent declaration from a decl before setPreviousDecl is called,
   but such cases are probably all bugs, and are now easy to find.

Some finessing is still in order here -- in particular, we use a very general
mechanism for handling the DefinitionData pointer on CXXRecordData, and a more
targeted approach would be more compact.

Also, the MayHaveOutOfDateDef mechanism should now be expunged, since it was
addressing only a corner of the full problem space here. That's not covered
by this patch.

Early performance benchmarks show that this makes no measurable difference to 
Clang performance without modules enabled (and fixes a major correctness issue
with modules enabled). I'll revert if a full performance comparison shows any
problems.

llvm-svn: 209046

4 files changed, 103 insertions, 13 deletions

diff --git a/clang/test/Modules/Inputs/module.map b/clang/test/Modules/Inputs/module.map
index d6effcf0ad0..061abbd24d5 100644
--- a/clang/test/Modules/Inputs/module.map
+++ b/clang/test/Modules/Inputs/module.map
@@ -69,6 +69,8 @@ module redeclarations_left { header "redeclarations_left.h" }
 module redeclarations_right { header "redeclarations_right.h" }
 module redecl_namespaces_left { header "redecl_namespaces_left.h" }
 module redecl_namespaces_right { header "redecl_namespaces_right.h" }
+module redecl_add_after_load_top { header "redecl-add-after-load-top.h" }
+module redecl_add_after_load { header "redecl-add-after-load.h" }
 module load_failure { header "load_failure.h" }
 
 module decldef {
diff --git a/clang/test/Modules/Inputs/redecl-add-after-load.h b/clang/test/Modules/Inputs/redecl-add-after-load.h
new file mode 100644
index 00000000000..6951a76289f
--- /dev/null
+++ b/clang/test/Modules/Inputs/redecl-add-after-load.h
@@ -0,0 +1,23 @@
+struct A {};
+extern const int variable = 0;
+extern constexpr int function() { return 0; }
+
+namespace N {
+  struct A {};
+  extern const int variable = 0;
+  extern constexpr int function() { return 0; }
+}
+
+@import redecl_add_after_load_top;
+struct C::A {};
+const int C::variable = 0;
+constexpr int C::function() { return 0; }
+
+struct D {
+  struct A;
+  static const int variable;
+  static constexpr int function();
+};
+struct D::A {};
+const int D::variable = 0;
+constexpr int D::function() { return 0; }
diff --git a/clang/test/Modules/decldef.mm b/clang/test/Modules/decldef.mm
index 35694935dfb..2e2bd8a75e2 100644
--- a/clang/test/Modules/decldef.mm
+++ b/clang/test/Modules/decldef.mm
@@ -1,14 +1,18 @@
 // RUN: rm -rf %t
-// RUN: %clang_cc1 -fmodules -fobjc-arc -I %S/Inputs -fmodules-cache-path=%t %s -verify -DUSE_EARLY
-// RUN: %clang_cc1 -fmodules -fobjc-arc -I %S/Inputs -fmodules-cache-path=%t %s -verify
+// RUN: %clang_cc1 -fmodules -fobjc-arc -I %S/Inputs -fmodules-cache-path=%t %s -verify -DUSE_1 -DUSE_2 -DUSE_3 -DUSE_4
+// RUN: %clang_cc1 -fmodules -fobjc-arc -I %S/Inputs -fmodules-cache-path=%t %s -verify -DUSE_2 -DUSE_3 -DUSE_4
+// RUN: %clang_cc1 -fmodules -fobjc-arc -I %S/Inputs -fmodules-cache-path=%t %s -verify -DUSE_3 -DUSE_4
+// RUN: %clang_cc1 -fmodules -fobjc-arc -I %S/Inputs -fmodules-cache-path=%t %s -verify -DUSE_4
 
-// expected-note@Inputs/def.h:5 {{previous}}
+// expected-note@Inputs/def.h:5 0-1{{previous}}
+// expected-note@Inputs/def.h:16 0-1{{previous}}
+// expected-note@Inputs/def-include.h:11 0-1{{previous}}
 
 @class Def;
 Def *def;
-class Def2; // expected-note {{forward decl}}
+class Def2; // expected-note 0-1{{forward decl}}
 Def2 *def2;
-namespace Def3NS { class Def3; } // expected-note {{forward decl}}
+namespace Def3NS { class Def3; } // expected-note 0-1{{forward decl}}
 Def3NS::Def3 *def3;
 
 @interface Unrelated
@@ -16,9 +20,10 @@ Def3NS::Def3 *def3;
 @end
 
 @import decldef;
-#ifdef USE_EARLY
+#ifdef USE_1
 A *a1; // expected-error{{declaration of 'A' must be imported from module 'decldef.Def'}}
 B *b1;
+#define USED
 #endif
 @import decldef.Decl;
 
@@ -26,14 +31,23 @@ A *a2;
 B *b;
 
 void testA(A *a) {
+#ifdef USE_2
   a->ivar = 17;
-#ifndef USE_EARLY
+  #ifndef USED
   // expected-error@-2{{definition of 'A' must be imported from module 'decldef.Def' before it is required}}
+  #define USED
+  #endif
 #endif
 }
 
 void testB() {
-  B b; // Note: redundant error silenced
+#ifdef USE_3
+  B b;
+  #ifndef USED
+  // expected-error@-2{{definition of 'B' must be imported from module 'decldef.Def' before it is required}}
+  #define USED
+  #endif
+#endif
 }
 
 void testDef() {
@@ -41,9 +55,12 @@ void testDef() {
 }
 
 void testDef2() {
-  // FIXME: These should both work, since we've (implicitly) imported
-  // decldef.Def here, but they don't, because nothing has triggered the lazy
-  // loading of the definitions of these classes.
-  def2->func(); // expected-error {{incomplete}}
-  def3->func(); // expected-error {{incomplete}}
+#ifdef USE_4
+  def2->func();
+  def3->func();
+  #ifndef USED
+  // expected-error@-3 {{definition of 'Def2' must be imported}}
+  #define USED
+  #endif
+#endif
 }
diff --git a/clang/test/Modules/redecl-add-after-load.cpp b/clang/test/Modules/redecl-add-after-load.cpp
new file mode 100644
index 00000000000..4ee63b5d815
--- /dev/null
+++ b/clang/test/Modules/redecl-add-after-load.cpp
@@ -0,0 +1,48 @@
+// RUN: rm -rf %t
+// RUN: %clang_cc1 -x objective-c++ -fmodules -fno-modules-error-recovery -fmodules-cache-path=%t -I %S/Inputs %s -verify -std=c++11
+
+typedef struct A B;
+extern const int variable;
+extern constexpr int function();
+constexpr int test(bool b) { return b ? variable : function(); }
+
+namespace N {
+  typedef struct A B;
+  extern const int variable;
+  extern constexpr int function();
+}
+typedef N::B NB;
+constexpr int N_test(bool b) { return b ? N::variable : N::function(); }
+
+@import redecl_add_after_load_top;
+typedef C::A CB;
+constexpr int C_test(bool b) { return b ? C::variable : C::function(); }
+
+struct D {
+  struct A; // expected-note {{forward}}
+  static const int variable;
+  static constexpr int function(); // expected-note {{here}}
+};
+typedef D::A DB;
+constexpr int D_test(bool b) { return b ? D::variable : D::function(); } // expected-note {{subexpression}} expected-note {{undefined}}
+
+@import redecl_add_after_load;
+
+B tu_struct_test;
+constexpr int tu_variable_test = test(true);
+constexpr int tu_function_test = test(false);
+
+NB ns_struct_test;
+constexpr int ns_variable_test = N_test(true);
+constexpr int ns_function_test = N_test(false);
+
+CB struct_struct_test;
+constexpr int struct_variable_test = C_test(true);
+constexpr int struct_function_test = C_test(false);
+
+// FIXME: We should accept this, but we're currently too lazy when merging class
+// definitions to determine that the definitions in redecl_add_after_load are
+// definitions of these entities.
+DB merged_struct_struct_test; // expected-error {{incomplete}}
+constexpr int merged_struct_variable_test = D_test(true); // expected-error {{constant}} expected-note {{in call to}}
+constexpr int merged_struct_function_test = D_test(false); // expected-error {{constant}} expected-note {{in call to}}