Wrap clang module files in a Mach-O, ELF, or COFF container.

This is a necessary prerequisite for debugging with modules.
The .pcm files become containers that hold the serialized AST which allows
us to store debug information in the module file that can be shared by all
object files that were built importing the module.

rdar://problem/19104245

This reapplies r230044 with a fixed configure+make build and updated
dependencies. Take 3.

llvm-svn: 230305

5 files changed, 90 insertions, 12 deletions

diff --git a/clang/lib/Frontend/ASTUnit.cpp b/clang/lib/Frontend/ASTUnit.cpp
index bfb1efe3522..58a5f980085 100644
--- a/clang/lib/Frontend/ASTUnit.cpp
+++ b/clang/lib/Frontend/ASTUnit.cpp
@@ -914,13 +914,20 @@ class PrecompilePreambleConsumer : public PCHGenerator {
   unsigned &Hash;
   std::vector<Decl *> TopLevelDecls;
   PrecompilePreambleAction *Action;
+  raw_ostream *Out;
+  SmallVectorImpl<char> *SerializedASTBuffer;
 
 public:
   PrecompilePreambleConsumer(ASTUnit &Unit, PrecompilePreambleAction *Action,
                              const Preprocessor &PP, StringRef isysroot,
                              raw_ostream *Out)
-    : PCHGenerator(PP, "", nullptr, isysroot, Out, /*AllowASTWithErrors=*/true),
-      Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action) {
+    : PCHGenerator(PP, "", nullptr, isysroot, /*AllowASTWithErrors=*/true),
+      Unit(Unit), Hash(Unit.getCurrentTopLevelHashValue()), Action(Action),
+      Out(Out) {
+    RegisterSerializationFinishedCallback(
+      [&](SmallVectorImpl<char> *Buf){
+        SerializedASTBuffer = Buf;
+      });
     Hash = 0;
   }
 
@@ -941,6 +948,13 @@ public:
   void HandleTranslationUnit(ASTContext &Ctx) override {
     PCHGenerator::HandleTranslationUnit(Ctx);
     if (hasEmittedPCH()) {
+      // Write the generated bitstream to "Out".
+      Out->write((char *)&SerializedASTBuffer->front(),
+                 SerializedASTBuffer->size());
+      // Make sure it hits disk now.
+      Out->flush();
+      SerializedASTBuffer->clear();
+
       // Translate the top-level declarations we captured during
       // parsing into declaration IDs in the precompiled
       // preamble. This will allow us to deserialize those top-level
diff --git a/clang/lib/Frontend/CMakeLists.txt b/clang/lib/Frontend/CMakeLists.txt
index 7c5fca54d1e..6c9085d65dc 100644
--- a/clang/lib/Frontend/CMakeLists.txt
+++ b/clang/lib/Frontend/CMakeLists.txt
@@ -45,6 +45,7 @@ add_clang_library(clangFrontend
   LINK_LIBS
   clangAST
   clangBasic
+  clangCodeGen
   clangDriver
   clangEdit
   clangLex
diff --git a/clang/lib/Frontend/ChainedIncludesSource.cpp b/clang/lib/Frontend/ChainedIncludesSource.cpp
index cb260b4f4c6..1ecc0bfc6b4 100644
--- a/clang/lib/Frontend/ChainedIncludesSource.cpp
+++ b/clang/lib/Frontend/ChainedIncludesSource.cpp
@@ -156,11 +156,13 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource(
                                                  &Clang->getPreprocessor());
     Clang->createASTContext();
 
-    SmallVector<char, 256> serialAST;
-    llvm::raw_svector_ostream OS(serialAST);
-    auto consumer =
-        llvm::make_unique<PCHGenerator>(Clang->getPreprocessor(), "-", nullptr,
-                                        /*isysroot=*/"", &OS);
+    auto consumer = llvm::make_unique<PCHGenerator>(Clang->getPreprocessor(),
+                                       "-", nullptr, /*isysroot=*/"");
+    SmallVectorImpl<char> *serialAST;
+    consumer->RegisterSerializationFinishedCallback(
+      [&](SmallVectorImpl<char> *Buf){
+        serialAST = Buf;
+      });
     Clang->getASTContext().setASTMutationListener(
                                             consumer->GetASTMutationListener());
     Clang->setASTConsumer(std::move(consumer));
@@ -197,7 +199,9 @@ IntrusiveRefCntPtr<ExternalSemaSource> clang::createChainedIncludesSource(
 
     ParseAST(Clang->getSema());
     Clang->getDiagnosticClient().EndSourceFile();
-    SerialBufs.push_back(llvm::MemoryBuffer::getMemBufferCopy(OS.str()));
+    SerialBufs.push_back(llvm::MemoryBuffer::
+        getMemBufferCopy(StringRef(serialAST->data(), serialAST->size())));
+    serialAST->clear();
     source->CIs.push_back(Clang.release());
   }
 
diff --git a/clang/lib/Frontend/FrontendActions.cpp b/clang/lib/Frontend/FrontendActions.cpp
index a55a3257851..3e0f525e653 100644
--- a/clang/lib/Frontend/FrontendActions.cpp
+++ b/clang/lib/Frontend/FrontendActions.cpp
@@ -10,10 +10,13 @@
 #include "clang/Frontend/FrontendActions.h"
 #include "clang/AST/ASTConsumer.h"
 #include "clang/Basic/FileManager.h"
+#include "clang/Basic/TargetInfo.h"
+#include "clang/CodeGen/CodeGenModuleContainer.h"
 #include "clang/Frontend/ASTConsumers.h"
 #include "clang/Frontend/ASTUnit.h"
 #include "clang/Frontend/CompilerInstance.h"
 #include "clang/Frontend/FrontendDiagnostic.h"
+#include "clang/Frontend/MultiplexConsumer.h"
 #include "clang/Frontend/Utils.h"
 #include "clang/Lex/HeaderSearch.h"
 #include "clang/Lex/Pragma.h"
@@ -85,8 +88,23 @@ GeneratePCHAction::CreateASTConsumer(CompilerInstance &CI, StringRef InFile) {
 
   if (!CI.getFrontendOpts().RelocatablePCH)
     Sysroot.clear();
-  return llvm::make_unique<PCHGenerator>(CI.getPreprocessor(), OutputFile,
-                                         nullptr, Sysroot, OS);
+
+  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+  Consumers.push_back(llvm::make_unique<PCHGenerator>(CI.getPreprocessor(),
+                                                      OutputFile, nullptr,
+                                                      Sysroot));
+
+  auto CGOpts = CI.getCodeGenOpts();
+  // The debug info emitted by ModuleContainerGenerator is not affected by the
+  // optimization level.
+  CGOpts.OptimizationLevel = 0;
+  CGOpts.setDebugInfo(CodeGenOptions::LimitedDebugInfo);
+  Consumers.push_back(std::unique_ptr<ASTConsumer>(
+      CreateModuleContainerGenerator(CI.getDiagnostics(), "PCH", CGOpts,
+                                     CI.getTargetOpts(), CI.getLangOpts(), OS,
+                                     cast<PCHGenerator>(Consumers[0].get()))));
+
+  return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
 }
 
 bool GeneratePCHAction::ComputeASTConsumerArguments(CompilerInstance &CI,
@@ -122,8 +140,22 @@ GenerateModuleAction::CreateASTConsumer(CompilerInstance &CI,
   if (ComputeASTConsumerArguments(CI, InFile, Sysroot, OutputFile, OS))
     return nullptr;
 
-  return llvm::make_unique<PCHGenerator>(CI.getPreprocessor(), OutputFile,
-                                         Module, Sysroot, OS);
+  std::vector<std::unique_ptr<ASTConsumer>> Consumers;
+  Consumers.push_back(llvm::make_unique<PCHGenerator>(CI.getPreprocessor(),
+                                                      OutputFile, Module,
+                                                      Sysroot));
+
+  auto CGOpts = CI.getCodeGenOpts();
+  // The debug info emitted by ModuleContainerGenerator is not affected by the
+  // optimization level.
+  CGOpts.OptimizationLevel = 0;
+  CGOpts.setDebugInfo(CodeGenOptions::LimitedDebugInfo);
+  Consumers.push_back(
+      std::unique_ptr<ASTConsumer>(CreateModuleContainerGenerator(
+          CI.getDiagnostics(), Module->getFullModuleName(), CGOpts,
+          CI.getTargetOpts(), CI.getLangOpts(), OS,
+          cast<PCHGenerator>(Consumers[0].get()))));
+  return llvm::make_unique<MultiplexConsumer>(std::move(Consumers));
 }
 
 static SmallVectorImpl<char> &
diff --git a/clang/lib/Frontend/MultiplexConsumer.cpp b/clang/lib/Frontend/MultiplexConsumer.cpp
index 3c4fed1d18e..bb9c907b2e3 100644
--- a/clang/lib/Frontend/MultiplexConsumer.cpp
+++ b/clang/lib/Frontend/MultiplexConsumer.cpp
@@ -33,11 +33,14 @@ public:
   void ReaderInitialized(ASTReader *Reader) override;
   void IdentifierRead(serialization::IdentID ID,
                       IdentifierInfo *II) override;
+  void MacroRead(serialization::MacroID ID, MacroInfo *MI) override;
   void TypeRead(serialization::TypeIdx Idx, QualType T) override;
   void DeclRead(serialization::DeclID ID, const Decl *D) override;
   void SelectorRead(serialization::SelectorID iD, Selector Sel) override;
   void MacroDefinitionRead(serialization::PreprocessedEntityID,
                            MacroDefinition *MD) override;
+  void ModuleRead(serialization::SubmoduleID ID, Module *Mod) override;
+
 private:
   std::vector<ASTDeserializationListener*> Listeners;
 };
@@ -59,6 +62,12 @@ void MultiplexASTDeserializationListener::IdentifierRead(
     Listeners[i]->IdentifierRead(ID, II);
 }
 
+void MultiplexASTDeserializationListener::MacroRead(
+    serialization::MacroID ID, MacroInfo *MI) {
+  for (auto &Listener : Listeners)
+    Listener->MacroRead(ID, MI);
+}
+
 void MultiplexASTDeserializationListener::TypeRead(
     serialization::TypeIdx Idx, QualType T) {
   for (size_t i = 0, e = Listeners.size(); i != e; ++i)
@@ -83,6 +92,12 @@ void MultiplexASTDeserializationListener::MacroDefinitionRead(
     Listeners[i]->MacroDefinitionRead(ID, MD);
 }
 
+void MultiplexASTDeserializationListener::ModuleRead(
+    serialization::SubmoduleID ID, Module *Mod) {
+  for (auto &Listener : Listeners)
+    Listener->ModuleRead(ID, Mod);
+}
+
 // This ASTMutationListener forwards its notifications to a set of
 // child listeners.
 class MultiplexASTMutationListener : public ASTMutationListener {
@@ -98,11 +113,13 @@ public:
                                const VarTemplateSpecializationDecl *D) override;
   void AddedCXXTemplateSpecialization(const FunctionTemplateDecl *TD,
                                       const FunctionDecl *D) override;
+  void ResolvedExceptionSpec(const FunctionDecl *FD) override;
   void DeducedReturnType(const FunctionDecl *FD, QualType ReturnType) override;
   void CompletedImplicitDefinition(const FunctionDecl *D) override;
   void StaticDataMemberInstantiated(const VarDecl *D) override;
   void AddedObjCCategoryToInterface(const ObjCCategoryDecl *CatD,
                                     const ObjCInterfaceDecl *IFD) override;
+  void FunctionDefinitionInstantiated(const FunctionDecl *D) override;
   void AddedObjCPropertyInClassExtension(const ObjCPropertyDecl *Prop,
                                     const ObjCPropertyDecl *OrigProp,
                                     const ObjCCategoryDecl *ClassExt) override;
@@ -149,6 +166,11 @@ void MultiplexASTMutationListener::AddedCXXTemplateSpecialization(
   for (size_t i = 0, e = Listeners.size(); i != e; ++i)
     Listeners[i]->AddedCXXTemplateSpecialization(TD, D);
 }
+void MultiplexASTMutationListener::ResolvedExceptionSpec(
+    const FunctionDecl *FD) {
+  for (auto &Listener : Listeners)
+    Listener->ResolvedExceptionSpec(FD);
+}
 void MultiplexASTMutationListener::DeducedReturnType(const FunctionDecl *FD,
                                                      QualType ReturnType) {
   for (size_t i = 0, e = Listeners.size(); i != e; ++i)
@@ -170,6 +192,11 @@ void MultiplexASTMutationListener::AddedObjCCategoryToInterface(
   for (size_t i = 0, e = Listeners.size(); i != e; ++i)
     Listeners[i]->AddedObjCCategoryToInterface(CatD, IFD);
 }
+void MultiplexASTMutationListener::FunctionDefinitionInstantiated(
+    const FunctionDecl *D) {
+  for (auto &Listener : Listeners)
+    Listener->FunctionDefinitionInstantiated(D);
+} 
 void MultiplexASTMutationListener::AddedObjCPropertyInClassExtension(
                                              const ObjCPropertyDecl *Prop,
                                              const ObjCPropertyDecl *OrigProp,