Revert r343058 "[ORC] Add support for multithreaded compiles to LLJIT and LLLazyJIT."

This doesn't work well in builds configured with LLVM_ENABLE_THREADS=OFF,
causing the following assert when running
ExecutionEngine/OrcLazy/multiple-compile-threads-basic.ll:

  lib/ExecutionEngine/Orc/Core.cpp:1748: Expected<llvm::JITEvaluatedSymbol>
  llvm::orc::lookup(const llvm::orc::JITDylibList &, llvm::orc::SymbolStringPtr):
  Assertion `ResultMap->size() == 1 && "Unexpected number of results"' failed.

> LLJIT and LLLazyJIT can now be constructed with an optional NumCompileThreads
> arguments. If this is non-zero then a thread-pool will be created with the
> given number of threads, and compile tasks will be dispatched to the thread
> pool.
>
> To enable testing of this feature, two new flags are added to lli:
>
> (1) -compile-threads=N (N = 0 by default) controls the number of compile threads
> to use.
>
> (2) -thread-entry can be used to execute code on additional threads. For each
> -thread-entry argument supplied (multiple are allowed) a new thread will be
> created and the given symbol called. These additional thread entry points are
> called after static constructors are run, but before main.

llvm-svn: 343099

4 files changed, 20 insertions, 181 deletions

diff --git a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
index 61c5cd29de7..8575d20e885 100644
--- a/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
+++ b/llvm/include/llvm/ExecutionEngine/Orc/LLJIT.h
@@ -22,7 +22,7 @@
 #include "llvm/ExecutionEngine/Orc/ObjectTransformLayer.h"
 #include "llvm/ExecutionEngine/Orc/RTDyldObjectLinkingLayer.h"
 #include "llvm/ExecutionEngine/Orc/ThreadSafeModule.h"
-#include "llvm/Support/ThreadPool.h"
+#include "llvm/Target/TargetMachine.h"
 
 namespace llvm {
 namespace orc {
@@ -30,19 +30,11 @@ namespace orc {
 /// A pre-fabricated ORC JIT stack that can serve as an alternative to MCJIT.
 class LLJIT {
 public:
-
-  /// Destruct this instance. If a multi-threaded instance, waits for all
-  /// compile threads to complete.
-  ~LLJIT();
-
   /// Create an LLJIT instance.
-  /// If NumCompileThreads is not equal to zero, creates a multi-threaded
-  /// LLJIT with the given number of compile threads.
   static Expected<std::unique_ptr<LLJIT>>
-  Create(JITTargetMachineBuilder JTMB, DataLayout DL,
-         unsigned NumCompileThreads = 0);
+  Create(std::unique_ptr<TargetMachine> TM, DataLayout DL);
 
-  /// Returns the ExecutionSession for this instance.
+  /// Returns a reference to the ExecutionSession for this JIT instance.
   ExecutionSession &getExecutionSession() { return *ES; }
 
   /// Returns a reference to the JITDylib representing the JIT'd main program.
@@ -99,15 +91,9 @@ public:
   RTDyldObjectLinkingLayer2 &getObjLinkingLayer() { return ObjLinkingLayer; }
 
 protected:
-
-  /// Create an LLJIT instance with a single compile thread.
   LLJIT(std::unique_ptr<ExecutionSession> ES, std::unique_ptr<TargetMachine> TM,
         DataLayout DL);
 
-  /// Create an LLJIT instance with multiple compile threads.
-  LLJIT(std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB,
-        DataLayout DL, unsigned NumCompileThreads);
-
   std::unique_ptr<RuntimeDyld::MemoryManager> getMemoryManager(VModuleKey K);
 
   std::string mangle(StringRef UnmangledName);
@@ -119,8 +105,8 @@ protected:
   std::unique_ptr<ExecutionSession> ES;
   JITDylib &Main;
 
+  std::unique_ptr<TargetMachine> TM;
   DataLayout DL;
-  std::unique_ptr<ThreadPool> CompileThreads;
 
   RTDyldObjectLinkingLayer2 ObjLinkingLayer;
   IRCompileLayer2 CompileLayer;
@@ -132,13 +118,9 @@ protected:
 /// compilation of LLVM IR.
 class LLLazyJIT : public LLJIT {
 public:
-
   /// Create an LLLazyJIT instance.
-  /// If NumCompileThreads is not equal to zero, creates a multi-threaded
-  /// LLLazyJIT with the given number of compile threads.
   static Expected<std::unique_ptr<LLLazyJIT>>
-  Create(JITTargetMachineBuilder JTMB, DataLayout DL,
-         unsigned NumCompileThreads = 0);
+  Create(std::unique_ptr<TargetMachine> TM, DataLayout DL);
 
   /// Set an IR transform (e.g. pass manager pipeline) to run on each function
   /// when it is compiled.
@@ -155,19 +137,11 @@ public:
   }
 
 private:
-
-  // Create a single-threaded LLLazyJIT instance.
   LLLazyJIT(std::unique_ptr<ExecutionSession> ES,
             std::unique_ptr<TargetMachine> TM, DataLayout DL,
             std::unique_ptr<LazyCallThroughManager> LCTMgr,
             std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder);
 
-  // Create a multi-threaded LLLazyJIT instance.
-  LLLazyJIT(std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB,
-            DataLayout DL, unsigned NumCompileThreads,
-            std::unique_ptr<LazyCallThroughManager> LCTMgr,
-            std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder);
-
   std::unique_ptr<LazyCallThroughManager> LCTMgr;
   std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder;
 
diff --git a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
index 0a0faaa681c..34402506a60 100644
--- a/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
+++ b/llvm/lib/ExecutionEngine/Orc/LLJIT.cpp
@@ -12,45 +12,13 @@
 #include "llvm/ExecutionEngine/SectionMemoryManager.h"
 #include "llvm/IR/Mangler.h"
 
-namespace {
-
-  // A SimpleCompiler that owns its TargetMachine.
-  class TMOwningSimpleCompiler : public llvm::orc::SimpleCompiler {
-  public:
-    TMOwningSimpleCompiler(std::unique_ptr<llvm::TargetMachine> TM)
-      : llvm::orc::SimpleCompiler(*TM), TM(std::move(TM)) {}
-  private:
-    // FIXME: shared because std::functions (and thus
-    // IRCompileLayer2::CompileFunction) are not moveable.
-    std::shared_ptr<llvm::TargetMachine> TM;
-  };
-
-} // end anonymous namespace
-
 namespace llvm {
 namespace orc {
 
-LLJIT::~LLJIT() {
-  if (CompileThreads)
-    CompileThreads->wait();
-}
-
 Expected<std::unique_ptr<LLJIT>>
-LLJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL,
-              unsigned NumCompileThreads) {
-
-  if (NumCompileThreads == 0) {
-    // If NumCompileThreads == 0 then create a single-threaded LLJIT instance.
-    auto TM = JTMB.createTargetMachine();
-    if (!TM)
-      return TM.takeError();
-    return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(),
-                                            std::move(*TM), std::move(DL)));
-  }
-
+LLJIT::Create(std::unique_ptr<TargetMachine> TM, DataLayout DL) {
   return std::unique_ptr<LLJIT>(new LLJIT(llvm::make_unique<ExecutionSession>(),
-                                          std::move(JTMB), std::move(DL),
-                                          NumCompileThreads));
+                                          std::move(TM), std::move(DL)));
 }
 
 Error LLJIT::defineAbsolute(StringRef Name, JITEvaluatedSymbol Sym) {
@@ -84,35 +52,12 @@ Expected<JITEvaluatedSymbol> LLJIT::lookupLinkerMangled(JITDylib &JD,
 LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES,
              std::unique_ptr<TargetMachine> TM, DataLayout DL)
     : ES(std::move(ES)), Main(this->ES->createJITDylib("main")),
-      DL(std::move(DL)),
+      TM(std::move(TM)), DL(std::move(DL)),
       ObjLinkingLayer(*this->ES,
                       [this](VModuleKey K) { return getMemoryManager(K); }),
-      CompileLayer(*this->ES, ObjLinkingLayer, TMOwningSimpleCompiler(std::move(TM))),
+      CompileLayer(*this->ES, ObjLinkingLayer, SimpleCompiler(*this->TM)),
       CtorRunner(Main), DtorRunner(Main) {}
 
-LLJIT::LLJIT(std::unique_ptr<ExecutionSession> ES,
-             JITTargetMachineBuilder JTMB, DataLayout DL,
-             unsigned NumCompileThreads)
-    : ES(std::move(ES)), Main(this->ES->createJITDylib("main")),
-      DL(std::move(DL)),
-      ObjLinkingLayer(*this->ES,
-                      [this](VModuleKey K) { return getMemoryManager(K); }),
-      CompileLayer(*this->ES, ObjLinkingLayer, MultiThreadedSimpleCompiler(std::move(JTMB))),
-      CtorRunner(Main), DtorRunner(Main) {
-  assert(NumCompileThreads != 0 &&
-         "Multithreaded LLJIT instance can not be created with 0 threads");
-
-  CompileThreads = llvm::make_unique<ThreadPool>(NumCompileThreads);
-  this->ES->setDispatchMaterialization([this](JITDylib &JD, std::unique_ptr<MaterializationUnit> MU) {
-      // FIXME: Switch to move capture once we have c++14.
-      auto SharedMU = std::shared_ptr<MaterializationUnit>(std::move(MU));
-      auto Work = [SharedMU, &JD]() {
-        SharedMU->doMaterialize(JD);
-      };
-      CompileThreads->async(std::move(Work));
-    });
-}
-
 std::unique_ptr<RuntimeDyld::MemoryManager>
 LLJIT::getMemoryManager(VModuleKey K) {
   return llvm::make_unique<SectionMemoryManager>();
@@ -145,11 +90,10 @@ void LLJIT::recordCtorDtors(Module &M) {
 }
 
 Expected<std::unique_ptr<LLLazyJIT>>
-  LLLazyJIT::Create(JITTargetMachineBuilder JTMB, DataLayout DL,
-                    unsigned NumCompileThreads) {
+LLLazyJIT::Create(std::unique_ptr<TargetMachine> TM, DataLayout DL) {
   auto ES = llvm::make_unique<ExecutionSession>();
 
-  const Triple &TT = JTMB.getTargetTriple();
+  const Triple &TT = TM->getTargetTriple();
 
   auto LCTMgr = createLocalLazyCallThroughManager(TT, *ES, 0);
   if (!LCTMgr)
@@ -161,18 +105,9 @@ Expected<std::unique_ptr<LLLazyJIT>>
         std::string("No indirect stubs manager builder for ") + TT.str(),
         inconvertibleErrorCode());
 
-  if (NumCompileThreads == 0) {
-    auto TM = JTMB.createTargetMachine();
-    if (!TM)
-      return TM.takeError();
-    return std::unique_ptr<LLLazyJIT>(
-        new LLLazyJIT(std::move(ES), std::move(*TM), std::move(DL),
-                      std::move(*LCTMgr), std::move(ISMBuilder)));
-  }
-
-  return std::unique_ptr<LLLazyJIT>(new LLLazyJIT(
-      std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads,
-      std::move(*LCTMgr), std::move(ISMBuilder)));
+  return std::unique_ptr<LLLazyJIT>(
+      new LLLazyJIT(std::move(ES), std::move(TM), std::move(DL),
+                    std::move(*LCTMgr), std::move(ISMBuilder)));
 }
 
 Error LLLazyJIT::addLazyIRModule(JITDylib &JD, ThreadSafeModule TSM) {
@@ -198,15 +133,5 @@ LLLazyJIT::LLLazyJIT(
       CODLayer(*this->ES, TransformLayer, *this->LCTMgr,
                std::move(ISMBuilder)) {}
 
-LLLazyJIT::LLLazyJIT(
-    std::unique_ptr<ExecutionSession> ES, JITTargetMachineBuilder JTMB,
-    DataLayout DL, unsigned NumCompileThreads,
-    std::unique_ptr<LazyCallThroughManager> LCTMgr,
-    std::function<std::unique_ptr<IndirectStubsManager>()> ISMBuilder)
-    : LLJIT(std::move(ES), std::move(JTMB), std::move(DL), NumCompileThreads),
-      LCTMgr(std::move(LCTMgr)), TransformLayer(*this->ES, CompileLayer),
-      CODLayer(*this->ES, TransformLayer, *this->LCTMgr,
-               std::move(ISMBuilder)) {}
-
 } // End namespace orc.
 } // End namespace llvm.
diff --git a/llvm/test/ExecutionEngine/OrcLazy/multiple-compile-threads-basic.ll b/llvm/test/ExecutionEngine/OrcLazy/multiple-compile-threads-basic.ll
deleted file mode 100644
index 24379128cc7..00000000000
--- a/llvm/test/ExecutionEngine/OrcLazy/multiple-compile-threads-basic.ll
+++ /dev/null
@@ -1,18 +0,0 @@
-; RUN: lli -jit-kind=orc-lazy -compile-threads=2 -thread-entry hello %s | FileCheck %s
-;
-; CHECK: Hello
-
-@.str = private unnamed_addr constant [7 x i8] c"Hello\0A\00", align 1
-
-define void @hello() {
-entry:
-  %call = call i32 (i8*, ...) @printf(i8* getelementptr inbounds ([7 x i8], [7 x i8]* @.str, i32 0, i32 0))
-  ret void
-}
-
-declare i32 @printf(i8*, ...)
-
-define i32 @main(i32 %argc, i8** %argv) {
-entry:
-  ret i32 0
-}
diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp
index 49d5db13326..2312d770348 100644
--- a/llvm/tools/lli/lli.cpp
+++ b/llvm/tools/lli/lli.cpp
@@ -97,17 +97,6 @@ namespace {
                                            "orc-lazy",
                                            "Orc-based lazy JIT.")));
 
-  cl::opt<unsigned>
-  LazyJITCompileThreads("compile-threads",
-                        cl::desc("Choose the number of compile threads "
-                                 "(jit-kind=orc-lazy only)"),
-                        cl::init(0));
-
-  cl::list<std::string>
-  ThreadEntryPoints("thread-entry",
-                    cl::desc("calls the given entry-point on a new thread "
-                             "(jit-kind=orc-lazy only)"));
-
   // The MCJIT supports building for a target address space separate from
   // the JIT compilation process. Use a forked process and a copying
   // memory manager with IPC to execute using this functionality.
@@ -374,19 +363,6 @@ int main(int argc, char **argv, char * const *envp) {
 
   if (UseJITKind == JITKind::OrcLazy)
     return runOrcLazyJIT(argv[0]);
-  else {
-    // Make sure nobody used an orc-lazy specific option accidentally.
-
-    if (LazyJITCompileThreads != 0) {
-      errs() << "-compile-threads requires -jit-kind=orc-lazy\n";
-      exit(1);
-    }
-
-    if (!ThreadEntryPoints.empty()) {
-      errs() << "-thread-entry requires -jit-kind=orc-lazy\n";
-      exit(1);
-    }
-  }
 
   LLVMContext Context;
 
@@ -769,11 +745,11 @@ int runOrcLazyJIT(const char *ProgName) {
     reportError(Err, ProgName);
 
   const auto &TT = MainModule.getModule()->getTargetTriple();
-  orc::JITTargetMachineBuilder JTMB =
+  orc::JITTargetMachineBuilder TMD =
       TT.empty() ? ExitOnErr(orc::JITTargetMachineBuilder::detectHost())
                  : orc::JITTargetMachineBuilder(Triple(TT));
 
-  JTMB.setArch(MArch)
+  TMD.setArch(MArch)
       .setCPU(getCPUStr())
       .addFeatures(getFeatureList())
       .setRelocationModel(RelocModel.getNumOccurrences()
@@ -782,13 +758,9 @@ int runOrcLazyJIT(const char *ProgName) {
       .setCodeModel(CMModel.getNumOccurrences()
                         ? Optional<CodeModel::Model>(CMModel)
                         : None);
-  DataLayout DL("");
-  {
-    // Create a throwaway TargetMachine to get the data layout.
-    auto TM = ExitOnErr(JTMB.createTargetMachine());
-    DL = TM->createDataLayout();
-  }
-  auto J = ExitOnErr(orc::LLLazyJIT::Create(std::move(JTMB), DL, LazyJITCompileThreads));
+  auto TM = ExitOnErr(TMD.createTargetMachine());
+  auto DL = TM->createDataLayout();
+  auto J = ExitOnErr(orc::LLLazyJIT::Create(std::move(TM), DL));
 
   auto Dump = createDebugDumper();
 
@@ -835,16 +807,6 @@ int runOrcLazyJIT(const char *ProgName) {
   // Run any static constructors.
   ExitOnErr(J->runConstructors());
 
-  // Run any -thread-entry points.
-  std::vector<std::thread> AltEntryThreads;
-  for (auto &ThreadEntryPoint : ThreadEntryPoints) {
-    auto EntryPointSym = ExitOnErr(J->lookup(ThreadEntryPoint));
-    typedef void (*EntryPointPtr)();
-    auto EntryPoint =
-      reinterpret_cast<EntryPointPtr>(static_cast<uintptr_t>(EntryPointSym.getAddress()));
-    AltEntryThreads.push_back(std::thread([EntryPoint]() { EntryPoint(); }));
-  }
-
   // Run main.
   auto MainSym = ExitOnErr(J->lookup("main"));
   typedef int (*MainFnPtr)(int, const char *[]);
@@ -855,12 +817,8 @@ int runOrcLazyJIT(const char *ProgName) {
       reinterpret_cast<MainFnPtr>(static_cast<uintptr_t>(MainSym.getAddress()));
   auto Result = Main(ArgV.size(), (const char **)ArgV.data());
 
-  // Wait for -entry-point threads.
-  for (auto &AltEntryThread : AltEntryThreads)
-    AltEntryThread.join();
-
-  // Run destructors.
   ExitOnErr(J->runDestructors());
+
   CXXRuntimeOverrides.runDestructors();
 
   return Result;