Restore "[ThinLTO] Emit individual index files for distributed backends"

This restores commit r268627:
    Summary:
    When launching ThinLTO backends in a distributed build (currently
    supported in gold via the thinlto-index-only plugin option), emit
    an individual index file for each backend process as described here:
    http://lists.llvm.org/pipermail/llvm-dev/2016-April/098272.html

    ...

    Differential Revision: http://reviews.llvm.org/D19556

Address msan failures by avoiding std::prev on map.end(), the
theory is that this is causing issues due to some known UB problems
in __tree.

llvm-svn: 269059

2 files changed, 107 insertions, 29 deletions

diff --git a/llvm/tools/gold/gold-plugin.cpp b/llvm/tools/gold/gold-plugin.cpp
index ec27c26cce9..fb433beb658 100644
--- a/llvm/tools/gold/gold-plugin.cpp
+++ b/llvm/tools/gold/gold-plugin.cpp
@@ -41,6 +41,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Support/thread.h"
 #include "llvm/Transforms/IPO.h"
+#include "llvm/Transforms/IPO/FunctionImport.h"
 #include "llvm/Transforms/IPO/PassManagerBuilder.h"
 #include "llvm/Transforms/Utils/FunctionImportUtils.h"
 #include "llvm/Transforms/Utils/GlobalStatus.h"
@@ -179,7 +180,8 @@ namespace options {
   static bool thinlto = false;
   // If false, all ThinLTO backend compilations through code gen are performed
   // using multiple threads in the gold-plugin, before handing control back to
-  // gold. If true, exit after creating the combined index, the assuming is
+  // gold. If true, write individual backend index files which reflect
+  // the import decisions, and exit afterwards. The assumption is
   // that the build system will launch the backend processes.
   static bool thinlto_index_only = false;
   // Additional options to pass into the code generator.
@@ -1190,33 +1192,66 @@ static void thinLTOBackends(raw_fd_ostream *ApiFile,
     Task.cleanup();
 }
 
-/// gold informs us that all symbols have been read. At this point, we use
-/// get_symbols to see if any of our definitions have been overridden by a
-/// native object file. Then, perform optimization and codegen.
-static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) {
-  if (Modules.empty())
-    return LDPS_OK;
+/// Perform ThinLTO link, which creates the combined index file.
+/// Also, either launch backend threads or (under thinlto-index-only)
+/// emit individual index files for distributed backends and exit.
+static ld_plugin_status thinLTOLink(raw_fd_ostream *ApiFile) {
+  ModuleSummaryIndex CombinedIndex;
+  uint64_t NextModuleId = 0;
+  for (claimed_file &F : Modules) {
+    PluginInputFile InputFile(F.handle);
 
-  if (unsigned NumOpts = options::extra.size())
-    cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
+    std::unique_ptr<ModuleSummaryIndex> Index =
+        getModuleSummaryIndexForFile(F, InputFile.file());
+
+    // Skip files without a module summary.
+    if (Index)
+      CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
+  }
 
-  // If we are doing ThinLTO compilation, simply build the combined
-  // module index/summary and emit it. We don't need to parse the modules
-  // and link them in this case.
-  if (options::thinlto) {
-    ModuleSummaryIndex CombinedIndex;
-    uint64_t NextModuleId = 0;
+  if (options::thinlto_index_only) {
+    // Collect for each module the list of function it defines (GUID ->
+    // Summary).
+    StringMap<std::map<GlobalValue::GUID, GlobalValueSummary *>>
+        ModuleToDefinedGVSummaries(NextModuleId);
+    CombinedIndex.collectDefinedGVSummariesPerModule(
+        ModuleToDefinedGVSummaries);
+
+    // FIXME: We want to do this for the case where the threads are launched
+    // from gold as well, in which case this will be moved out of the
+    // thinlto_index_only handling, and the function importer will be invoked
+    // directly using the Lists.
+    StringMap<FunctionImporter::ImportMapTy> ImportLists(NextModuleId);
+    StringMap<FunctionImporter::ExportSetTy> ExportLists(NextModuleId);
+    ComputeCrossModuleImport(CombinedIndex, ModuleToDefinedGVSummaries,
+                             ImportLists, ExportLists);
+
+    // For each input bitcode file, generate an individual index that
+    // contains summaries only for its own global values, and for any that
+    // should be imported.
     for (claimed_file &F : Modules) {
       PluginInputFile InputFile(F.handle);
-
-      std::unique_ptr<ModuleSummaryIndex> Index =
-          getModuleSummaryIndexForFile(F, InputFile.file());
-
-      // Skip files without a module summary.
-      if (Index)
-        CombinedIndex.mergeFrom(std::move(Index), ++NextModuleId);
+      std::error_code EC;
+      raw_fd_ostream OS((Twine(InputFile.file().name) + ".thinlto.bc").str(),
+                        EC, sys::fs::OpenFlags::F_None);
+      if (EC)
+        message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s",
+                InputFile.file().name, EC.message().c_str());
+      // Build a map of module to the GUIDs and summary objects that should
+      // be written to its index.
+      std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
+      gatherImportedSummariesForModule(InputFile.file().name,
+                                       ModuleToDefinedGVSummaries, ImportLists,
+                                       ModuleToSummariesForIndex);
+      WriteIndexToFile(CombinedIndex, OS, &ModuleToSummariesForIndex);
     }
 
+    cleanup_hook();
+    exit(0);
+  }
+
+  // Create OS in nested scope so that it will be closed on destruction.
+  {
     std::error_code EC;
     raw_fd_ostream OS(output_name + ".thinlto.bc", EC,
                       sys::fs::OpenFlags::F_None);
@@ -1224,16 +1259,24 @@ static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) {
       message(LDPL_FATAL, "Unable to open %s.thinlto.bc for writing: %s",
               output_name.data(), EC.message().c_str());
     WriteIndexToFile(CombinedIndex, OS);
-    OS.close();
+  }
 
-    if (options::thinlto_index_only) {
-      cleanup_hook();
-      exit(0);
-    }
+  thinLTOBackends(ApiFile, CombinedIndex);
+  return LDPS_OK;
+}
 
-    thinLTOBackends(ApiFile, CombinedIndex);
+/// gold informs us that all symbols have been read. At this point, we use
+/// get_symbols to see if any of our definitions have been overridden by a
+/// native object file. Then, perform optimization and codegen.
+static ld_plugin_status allSymbolsReadHook(raw_fd_ostream *ApiFile) {
+  if (Modules.empty())
     return LDPS_OK;
-  }
+
+  if (unsigned NumOpts = options::extra.size())
+    cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
+
+  if (options::thinlto)
+    return thinLTOLink(ApiFile);
 
   LLVMContext Context;
   Context.setDiscardValueNames(options::TheOutputType !=
diff --git a/llvm/tools/llvm-lto/llvm-lto.cpp b/llvm/tools/llvm-lto/llvm-lto.cpp
index abede09a1e5..395988c0ebf 100644
--- a/llvm/tools/llvm-lto/llvm-lto.cpp
+++ b/llvm/tools/llvm-lto/llvm-lto.cpp
@@ -66,6 +66,7 @@ static cl::opt<bool>
 
 enum ThinLTOModes {
   THINLINK,
+  THINDISTRIBUTE,
   THINPROMOTE,
   THINIMPORT,
   THININTERNALIZE,
@@ -80,6 +81,8 @@ cl::opt<ThinLTOModes> ThinLTOMode(
         clEnumValN(
             THINLINK, "thinlink",
             "ThinLink: produces the index by linking only the summaries."),
+        clEnumValN(THINDISTRIBUTE, "distributedindexes",
+                   "Produces individual indexes for distributed backends."),
         clEnumValN(THINPROMOTE, "promote",
                    "Perform pre-import promotion (requires -thinlto-index)."),
         clEnumValN(THINIMPORT, "import", "Perform both promotion and "
@@ -354,6 +357,8 @@ public:
     switch (ThinLTOMode) {
     case THINLINK:
       return thinLink();
+    case THINDISTRIBUTE:
+      return distributedIndexes();
     case THINPROMOTE:
       return promote();
     case THINIMPORT:
@@ -396,6 +401,36 @@ private:
     return;
   }
 
+  /// Load the combined index from disk, then compute and generate
+  /// individual index files suitable for ThinLTO distributed backend builds
+  /// on the files mentioned on the command line (these must match the index
+  /// content).
+  void distributedIndexes() {
+    if (InputFilenames.size() != 1 && !OutputFilename.empty())
+      report_fatal_error("Can't handle a single output filename and multiple "
+                         "input files, do not provide an output filename and "
+                         "the output files will be suffixed from the input "
+                         "ones.");
+
+    auto Index = loadCombinedIndex();
+    for (auto &Filename : InputFilenames) {
+      // Build a map of module to the GUIDs and summary objects that should
+      // be written to its index.
+      std::map<std::string, GVSummaryMapTy> ModuleToSummariesForIndex;
+      ThinLTOCodeGenerator::gatherImportedSummariesForModule(
+          Filename, *Index, ModuleToSummariesForIndex);
+
+      std::string OutputName = OutputFilename;
+      if (OutputName.empty()) {
+        OutputName = Filename + ".thinlto.bc";
+      }
+      std::error_code EC;
+      raw_fd_ostream OS(OutputName, EC, sys::fs::OpenFlags::F_None);
+      error(EC, "error opening the file '" + OutputName + "'");
+      WriteIndexToFile(*Index, OS, &ModuleToSummariesForIndex);
+    }
+  }
+
   /// Load the combined index from disk, then load every file referenced by
   /// the index and add them to the generator, finally perform the promotion
   /// on the files mentioned on the command line (these must match the index