diff options
17 files changed, 251 insertions, 232 deletions
diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h index 212dc1c42ff..0db97866cef 100644 --- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h +++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter3/KaleidoscopeJIT.h @@ -76,8 +76,8 @@ public: [this](std::unique_ptr<Module> M) { return optimizeModule(std::move(M)); }), - CompileCallbackManager( - orc::createLocalCompileCallbackManager(TM->getTargetTriple(), 0)), + CompileCallbackManager(orc::createLocalCompileCallbackManager( + TM->getTargetTriple(), ES, 0)), CODLayer(ES, OptimizeLayer, [&](orc::VModuleKey K) { return Resolvers[K]; }, [&](orc::VModuleKey K, std::shared_ptr<SymbolResolver> R) { diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h index dc8e936afd2..ddd469b9ae2 100644 --- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h +++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter4/KaleidoscopeJIT.h @@ -116,8 +116,8 @@ public: [this](std::unique_ptr<Module> M) { return optimizeModule(std::move(M)); }), - CompileCallbackMgr( - orc::createLocalCompileCallbackManager(TM->getTargetTriple(), 0)) { + CompileCallbackMgr(orc::createLocalCompileCallbackManager( + TM->getTargetTriple(), ES, 0)) { auto IndirectStubsMgrBuilder = orc::createLocalIndirectStubsManagerBuilder(TM->getTargetTriple()); IndirectStubsMgr = IndirectStubsMgrBuilder(); @@ -134,22 +134,6 @@ public: } Error addFunctionAST(std::unique_ptr<FunctionAST> FnAST) { - // Create a CompileCallback - this is the re-entry point into the compiler - // for functions that haven't been compiled yet. - auto CCInfo = cantFail(CompileCallbackMgr->getCompileCallback()); - - // Create an indirect stub. This serves as the functions "canonical - // definition" - an unchanging (constant address) entry point to the - // function implementation. - // Initially we point the stub's function-pointer at the compile callback - // that we just created. In the compile action for the callback (see below) - // we will update the stub's function pointer to point at the function - // implementation that we just implemented. - if (auto Err = IndirectStubsMgr->createStub(mangle(FnAST->getName()), - CCInfo.getAddress(), - JITSymbolFlags::Exported)) - return Err; - // Move ownership of FnAST to a shared pointer - C++11 lambdas don't support // capture-by-move, which is be required for unique_ptr. auto SharedFnAST = std::shared_ptr<FunctionAST>(std::move(FnAST)); @@ -170,23 +154,37 @@ public: // The JIT runtime (the resolver block) will use the return address of // this function as the address to continue at once it has reset the // CPU state to what it was immediately before the call. - CCInfo.setCompileAction( - [this, SharedFnAST]() { - auto M = irgenAndTakeOwnership(*SharedFnAST, "$impl"); - addModule(std::move(M)); - auto Sym = findSymbol(SharedFnAST->getName() + "$impl"); - assert(Sym && "Couldn't find compiled function?"); - JITTargetAddress SymAddr = cantFail(Sym.getAddress()); - if (auto Err = - IndirectStubsMgr->updatePointer(mangle(SharedFnAST->getName()), - SymAddr)) { - logAllUnhandledErrors(std::move(Err), errs(), - "Error updating function pointer: "); - exit(1); - } - - return SymAddr; - }); + auto CompileAction = [this, SharedFnAST]() { + auto M = irgenAndTakeOwnership(*SharedFnAST, "$impl"); + addModule(std::move(M)); + auto Sym = findSymbol(SharedFnAST->getName() + "$impl"); + assert(Sym && "Couldn't find compiled function?"); + JITTargetAddress SymAddr = cantFail(Sym.getAddress()); + if (auto Err = IndirectStubsMgr->updatePointer( + mangle(SharedFnAST->getName()), SymAddr)) { + logAllUnhandledErrors(std::move(Err), errs(), + "Error updating function pointer: "); + exit(1); + } + + return SymAddr; + }; + + // Create a CompileCallback using the CompileAction - this is the re-entry + // point into the compiler for functions that haven't been compiled yet. + auto CCAddr = cantFail( + CompileCallbackMgr->getCompileCallback(std::move(CompileAction))); + + // Create an indirect stub. This serves as the functions "canonical + // definition" - an unchanging (constant address) entry point to the + // function implementation. + // Initially we point the stub's function-pointer at the compile callback + // that we just created. When the compile action for the callback is run we + // will update the stub's function pointer to point at the function + // implementation that we just implemented. + if (auto Err = IndirectStubsMgr->createStub( + mangle(SharedFnAST->getName()), CCAddr, JITSymbolFlags::Exported)) + return Err; return Error::success(); } diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h index e86f3c3c633..010f5436377 100644 --- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h +++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/KaleidoscopeJIT.h @@ -78,7 +78,7 @@ using MyRemote = remote::OrcRemoteTargetClient; class KaleidoscopeJIT { private: - ExecutionSession ES; + ExecutionSession &ES; std::shared_ptr<SymbolResolver> Resolver; std::unique_ptr<TargetMachine> TM; const DataLayout DL; @@ -95,8 +95,9 @@ private: MyRemote &Remote; public: - KaleidoscopeJIT(MyRemote &Remote) - : Resolver(createLegacyLookupResolver( + KaleidoscopeJIT(ExecutionSession &ES, MyRemote &Remote) + : ES(ES), + Resolver(createLegacyLookupResolver( ES, [this](const std::string &Name) -> JITSymbol { if (auto Sym = IndirectStubsMgr->findStub(Name, false)) @@ -146,22 +147,6 @@ public: } Error addFunctionAST(std::unique_ptr<FunctionAST> FnAST) { - // Create a CompileCallback - this is the re-entry point into the compiler - // for functions that haven't been compiled yet. - auto CCInfo = cantFail(CompileCallbackMgr->getCompileCallback()); - - // Create an indirect stub. This serves as the functions "canonical - // definition" - an unchanging (constant address) entry point to the - // function implementation. - // Initially we point the stub's function-pointer at the compile callback - // that we just created. In the compile action for the callback (see below) - // we will update the stub's function pointer to point at the function - // implementation that we just implemented. - if (auto Err = IndirectStubsMgr->createStub(mangle(FnAST->getName()), - CCInfo.getAddress(), - JITSymbolFlags::Exported)) - return Err; - // Move ownership of FnAST to a shared pointer - C++11 lambdas don't support // capture-by-move, which is be required for unique_ptr. auto SharedFnAST = std::shared_ptr<FunctionAST>(std::move(FnAST)); @@ -182,23 +167,37 @@ public: // The JIT runtime (the resolver block) will use the return address of // this function as the address to continue at once it has reset the // CPU state to what it was immediately before the call. - CCInfo.setCompileAction( - [this, SharedFnAST]() { - auto M = irgenAndTakeOwnership(*SharedFnAST, "$impl"); - addModule(std::move(M)); - auto Sym = findSymbol(SharedFnAST->getName() + "$impl"); - assert(Sym && "Couldn't find compiled function?"); - JITTargetAddress SymAddr = cantFail(Sym.getAddress()); - if (auto Err = - IndirectStubsMgr->updatePointer(mangle(SharedFnAST->getName()), - SymAddr)) { - logAllUnhandledErrors(std::move(Err), errs(), - "Error updating function pointer: "); - exit(1); - } - - return SymAddr; - }); + auto CompileAction = [this, SharedFnAST]() { + auto M = irgenAndTakeOwnership(*SharedFnAST, "$impl"); + addModule(std::move(M)); + auto Sym = findSymbol(SharedFnAST->getName() + "$impl"); + assert(Sym && "Couldn't find compiled function?"); + JITTargetAddress SymAddr = cantFail(Sym.getAddress()); + if (auto Err = IndirectStubsMgr->updatePointer( + mangle(SharedFnAST->getName()), SymAddr)) { + logAllUnhandledErrors(std::move(Err), errs(), + "Error updating function pointer: "); + exit(1); + } + + return SymAddr; + }; + + // Create a CompileCallback suing the CompileAction - this is the re-entry + // point into the compiler for functions that haven't been compiled yet. + auto CCAddr = cantFail( + CompileCallbackMgr->getCompileCallback(std::move(CompileAction))); + + // Create an indirect stub. This serves as the functions "canonical + // definition" - an unchanging (constant address) entry point to the + // function implementation. + // Initially we point the stub's function-pointer at the compile callback + // that we just created. In the compile action for the callback we will + // update the stub's function pointer to point at the function + // implementation that we just implemented. + if (auto Err = IndirectStubsMgr->createStub( + mangle(SharedFnAST->getName()), CCAddr, JITSymbolFlags::Exported)) + return Err; return Error::success(); } diff --git a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/toy.cpp b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/toy.cpp index 10b60bd12d1..415cc751277 100644 --- a/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/toy.cpp +++ b/llvm/examples/Kaleidoscope/BuildingAJIT/Chapter5/toy.cpp @@ -1243,7 +1243,9 @@ std::unique_ptr<FDRPCChannel> connect() { sockaddr_in servAddr; memset(&servAddr, 0, sizeof(servAddr)); servAddr.sin_family = PF_INET; - memcpy(&servAddr.sin_addr.s_addr, server->h_addr, server->h_length); + char *src; + memcpy(&src, &server->h_addr, sizeof(char *)); + memcpy(&servAddr.sin_addr.s_addr, src, server->h_length); servAddr.sin_port = htons(Port); if (connect(sockfd, reinterpret_cast<sockaddr*>(&servAddr), sizeof(servAddr)) < 0) { @@ -1276,9 +1278,10 @@ int main(int argc, char *argv[]) { BinopPrecedence['-'] = 20; BinopPrecedence['*'] = 40; // highest. + ExecutionSession ES; auto TCPChannel = connect(); - auto Remote = ExitOnErr(MyRemote::Create(*TCPChannel, ExitOnErr)); - TheJIT = llvm::make_unique<KaleidoscopeJIT>(*Remote); + auto Remote = ExitOnErr(MyRemote::Create(*TCPChannel, ES)); + TheJIT = llvm::make_unique<KaleidoscopeJIT>(ES, *Remote); // Automatically inject a definition for 'printExprResult'. FunctionProtos["printExprResult"] = diff --git a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h index f99dedc97fa..622a847d0d0 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/CompileOnDemandLayer.h @@ -349,22 +349,21 @@ private: // Create a callback, associate it with the stub for the function, // and set the compile action to compile the partition containing the // function. - if (auto CCInfoOrErr = CompileCallbackMgr.getCompileCallback()) { - auto &CCInfo = *CCInfoOrErr; + auto CompileAction = [this, &LD, LMId, &F]() -> JITTargetAddress { + if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) + return *FnImplAddrOrErr; + else { + // FIXME: Report error, return to 'abort' or something similar. + consumeError(FnImplAddrOrErr.takeError()); + return 0; + } + }; + if (auto CCAddr = + CompileCallbackMgr.getCompileCallback(std::move(CompileAction))) StubInits[MangledName] = - std::make_pair(CCInfo.getAddress(), - JITSymbolFlags::fromGlobalValue(F)); - CCInfo.setCompileAction([this, &LD, LMId, &F]() -> JITTargetAddress { - if (auto FnImplAddrOrErr = this->extractAndCompile(LD, LMId, F)) - return *FnImplAddrOrErr; - else { - // FIXME: Report error, return to 'abort' or something similar. - consumeError(FnImplAddrOrErr.takeError()); - return 0; - } - }); - } else - return CCInfoOrErr.takeError(); + std::make_pair(*CCAddr, JITSymbolFlags::fromGlobalValue(F)); + else + return CCAddr.takeError(); } if (auto Err = LD.StubsMgr->createStubs(StubInits)) diff --git a/llvm/include/llvm/ExecutionEngine/Orc/Core.h b/llvm/include/llvm/ExecutionEngine/Orc/Core.h index ca099e50ad2..2b714849017 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/Core.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/Core.h @@ -610,10 +610,10 @@ private: /// VSOs will be searched in order and no VSO pointer may be null. /// All symbols must be found within the given VSOs or an error /// will be returned. -Expected<SymbolMap> lookup(const std::vector<VSO *> &VSOs, SymbolNameSet Names); +Expected<SymbolMap> lookup(const VSO::VSOList &VSOs, SymbolNameSet Names); /// Look up a symbol by searching a list of VSOs. -Expected<JITEvaluatedSymbol> lookup(const std::vector<VSO *> VSOs, +Expected<JITEvaluatedSymbol> lookup(const VSO::VSOList &VSOs, SymbolStringPtr Name); } // End namespace orc diff --git a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h index 8b4ed9092ae..0e400885a76 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/IndirectionUtils.h @@ -18,6 +18,7 @@ #include "llvm/ADT/StringRef.h" #include "llvm/ADT/Twine.h" #include "llvm/ExecutionEngine/JITSymbol.h" +#include "llvm/ExecutionEngine/Orc/Core.h" #include "llvm/Support/Error.h" #include "llvm/Support/Memory.h" #include "llvm/Support/Process.h" @@ -49,95 +50,26 @@ namespace orc { /// Target-independent base class for compile callback management. class JITCompileCallbackManager { public: - using CompileFtor = std::function<JITTargetAddress()>; - - /// Handle to a newly created compile callback. Can be used to get an - /// IR constant representing the address of the trampoline, and to set - /// the compile action for the callback. - class CompileCallbackInfo { - public: - CompileCallbackInfo(JITTargetAddress Addr, CompileFtor &Compile) - : Addr(Addr), Compile(Compile) {} - - JITTargetAddress getAddress() const { return Addr; } - void setCompileAction(CompileFtor Compile) { - this->Compile = std::move(Compile); - } - - private: - JITTargetAddress Addr; - CompileFtor &Compile; - }; + using CompileFunction = std::function<JITTargetAddress()>; /// Construct a JITCompileCallbackManager. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - JITCompileCallbackManager(JITTargetAddress ErrorHandlerAddress) - : ErrorHandlerAddress(ErrorHandlerAddress) {} + JITCompileCallbackManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress) + : ES(ES), CallbacksVSO(ES.createVSO("<Callbacks>")), + ErrorHandlerAddress(ErrorHandlerAddress) {} virtual ~JITCompileCallbackManager() = default; - /// Execute the callback for the given trampoline id. Called by the JIT - /// to compile functions on demand. - JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr) { - auto I = ActiveTrampolines.find(TrampolineAddr); - // FIXME: Also raise an error in the Orc error-handler when we finally have - // one. - if (I == ActiveTrampolines.end()) - return ErrorHandlerAddress; - - // Found a callback handler. Yank this trampoline out of the active list and - // put it back in the available trampolines list, then try to run the - // handler's compile and update actions. - // Moving the trampoline ID back to the available list first means there's - // at - // least one available trampoline if the compile action triggers a request - // for - // a new one. - auto Compile = std::move(I->second); - ActiveTrampolines.erase(I); - AvailableTrampolines.push_back(TrampolineAddr); - - if (auto Addr = Compile()) - return Addr; - - return ErrorHandlerAddress; - } - /// Reserve a compile callback. - Expected<CompileCallbackInfo> getCompileCallback() { - if (auto TrampolineAddrOrErr = getAvailableTrampolineAddr()) { - const auto &TrampolineAddr = *TrampolineAddrOrErr; - auto &Compile = this->ActiveTrampolines[TrampolineAddr]; - return CompileCallbackInfo(TrampolineAddr, Compile); - } else - return TrampolineAddrOrErr.takeError(); - } - - /// Get a CompileCallbackInfo for an existing callback. - CompileCallbackInfo getCompileCallbackInfo(JITTargetAddress TrampolineAddr) { - auto I = ActiveTrampolines.find(TrampolineAddr); - assert(I != ActiveTrampolines.end() && "Not an active trampoline."); - return CompileCallbackInfo(I->first, I->second); - } + Expected<JITTargetAddress> getCompileCallback(CompileFunction Compile); - /// Release a compile callback. - /// - /// Note: Callbacks are auto-released after they execute. This method should - /// only be called to manually release a callback that is not going to - /// execute. - void releaseCompileCallback(JITTargetAddress TrampolineAddr) { - auto I = ActiveTrampolines.find(TrampolineAddr); - assert(I != ActiveTrampolines.end() && "Not an active trampoline."); - ActiveTrampolines.erase(I); - AvailableTrampolines.push_back(TrampolineAddr); - } + /// Execute the callback for the given trampoline id. Called by the JIT + /// to compile functions on demand. + JITTargetAddress executeCompileCallback(JITTargetAddress TrampolineAddr); protected: - JITTargetAddress ErrorHandlerAddress; - - using TrampolineMapT = std::map<JITTargetAddress, CompileFtor>; - TrampolineMapT ActiveTrampolines; std::vector<JITTargetAddress> AvailableTrampolines; private: @@ -156,6 +88,13 @@ private: virtual Error grow() = 0; virtual void anchor(); + + std::mutex CCMgrMutex; + ExecutionSession &ES; + VSO &CallbacksVSO; + JITTargetAddress ErrorHandlerAddress; + std::map<JITTargetAddress, SymbolStringPtr> AddrToSymbol; + size_t NextCallbackId = 0; }; /// Manage compile callbacks for in-process JITs. @@ -165,8 +104,9 @@ public: /// Construct a InProcessJITCompileCallbackManager. /// @param ErrorHandlerAddress The address of an error handler in the target /// process to be used if a compile callback fails. - LocalJITCompileCallbackManager(JITTargetAddress ErrorHandlerAddress) - : JITCompileCallbackManager(ErrorHandlerAddress) { + LocalJITCompileCallbackManager(ExecutionSession &ES, + JITTargetAddress ErrorHandlerAddress) + : JITCompileCallbackManager(ES, ErrorHandlerAddress) { /// Set up the resolver block. std::error_code EC; ResolverBlock = sys::OwningMemoryBlock(sys::Memory::allocateMappedMemory( @@ -360,7 +300,7 @@ private: /// ErrorHandlerAddress will be used by the resulting compile callback /// manager if a compile callback fails. std::unique_ptr<JITCompileCallbackManager> -createLocalCompileCallbackManager(const Triple &T, +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress); /// Create a local indriect stubs manager builder. diff --git a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h index 686d9a4aa04..a1c9dac876a 100644 --- a/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h +++ b/llvm/include/llvm/ExecutionEngine/Orc/OrcRemoteTargetClient.h @@ -451,8 +451,9 @@ public: class RemoteCompileCallbackManager : public JITCompileCallbackManager { public: RemoteCompileCallbackManager(OrcRemoteTargetClient &Client, + ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) - : JITCompileCallbackManager(ErrorHandlerAddress), Client(Client) {} + : JITCompileCallbackManager(ES, ErrorHandlerAddress), Client(Client) {} private: Error grow() override { @@ -477,10 +478,10 @@ public: /// Channel is the ChannelT instance to communicate on. It is assumed that /// the channel is ready to be read from and written to. static Expected<std::unique_ptr<OrcRemoteTargetClient>> - Create(rpc::RawByteChannel &Channel, std::function<void(Error)> ReportError) { + Create(rpc::RawByteChannel &Channel, ExecutionSession &ES) { Error Err = Error::success(); auto Client = std::unique_ptr<OrcRemoteTargetClient>( - new OrcRemoteTargetClient(Channel, std::move(ReportError), Err)); + new OrcRemoteTargetClient(Channel, ES, Err)); if (Err) return std::move(Err); return std::move(Client); @@ -534,12 +535,14 @@ public: Expected<RemoteCompileCallbackManager &> enableCompileCallbacks(JITTargetAddress ErrorHandlerAddress) { + assert(!CallbackManager && "CallbackManager already obtained"); + // Emit the resolver block on the JIT server. if (auto Err = callB<stubs::EmitResolverBlock>()) return std::move(Err); // Create the callback manager. - CallbackManager.emplace(*this, ErrorHandlerAddress); + CallbackManager.emplace(*this, ES, ErrorHandlerAddress); RemoteCompileCallbackManager &Mgr = *CallbackManager; return Mgr; } @@ -557,10 +560,10 @@ public: Error terminateSession() { return callB<utils::TerminateSession>(); } private: - OrcRemoteTargetClient(rpc::RawByteChannel &Channel, - std::function<void(Error)> ReportError, Error &Err) + OrcRemoteTargetClient(rpc::RawByteChannel &Channel, ExecutionSession &ES, + Error &Err) : rpc::SingleThreadedRPCEndpoint<rpc::RawByteChannel>(Channel, true), - ReportError(std::move(ReportError)) { + ES(ES) { ErrorAsOutParameter EAO(&Err); addHandler<utils::RequestCompile>( @@ -580,7 +583,7 @@ private: void deregisterEHFrames(JITTargetAddress Addr, uint32_t Size) { if (auto Err = callB<eh::RegisterEHFrames>(Addr, Size)) - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); } void destroyRemoteAllocator(ResourceIdMgr::ResourceId Id) { @@ -595,7 +598,7 @@ private: void destroyIndirectStubsManager(ResourceIdMgr::ResourceId Id) { IndirectStubOwnerIds.release(Id); if (auto Err = callB<stubs::DestroyIndirectStubsOwner>(Id)) - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); } Expected<std::tuple<JITTargetAddress, JITTargetAddress, uint32_t>> @@ -628,7 +631,7 @@ private: if (auto AddrOrErr = callB<mem::ReserveMem>(Id, Size, Align)) return *AddrOrErr; else { - ReportError(AddrOrErr.takeError()); + ES.reportError(AddrOrErr.takeError()); return 0; } } @@ -636,7 +639,7 @@ private: bool setProtections(ResourceIdMgr::ResourceId Id, JITTargetAddress RemoteSegAddr, unsigned ProtFlags) { if (auto Err = callB<mem::SetProtections>(Id, RemoteSegAddr, ProtFlags)) { - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); return true; } else return false; @@ -644,7 +647,7 @@ private: bool writeMem(JITTargetAddress Addr, const char *Src, uint64_t Size) { if (auto Err = callB<mem::WriteMem>(DirectBufferWriter(Src, Addr, Size))) { - ReportError(std::move(Err)); + ES.reportError(std::move(Err)); return true; } else return false; @@ -656,6 +659,7 @@ private: static Error doNothing() { return Error::success(); } + ExecutionSession &ES; std::function<void(Error)> ReportError; std::string RemoteTargetTriple; uint32_t RemotePointerSize = 0; diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index bae4977f36a..2fb2e5f4f98 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -869,7 +869,7 @@ VSO &ExecutionSession::createVSO(std::string Name) { }); } -Expected<SymbolMap> lookup(const std::vector<VSO *> &VSOs, SymbolNameSet Names) { +Expected<SymbolMap> lookup(const VSO::VSOList &VSOs, SymbolNameSet Names) { #if LLVM_ENABLE_THREADS // In the threaded case we use promises to return the results. std::promise<SymbolMap> PromisedResult; @@ -975,7 +975,7 @@ Expected<SymbolMap> lookup(const std::vector<VSO *> &VSOs, SymbolNameSet Names) } /// Look up a symbol by searching a list of VSOs. -Expected<JITEvaluatedSymbol> lookup(const std::vector<VSO *> VSOs, +Expected<JITEvaluatedSymbol> lookup(const VSO::VSOList &VSOs, SymbolStringPtr Name) { SymbolNameSet Names({Name}); if (auto ResultMap = lookup(VSOs, std::move(Names))) { diff --git a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp index 08aa3a8ce52..4aa5f3badec 100644 --- a/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/IndirectionUtils.cpp @@ -13,38 +13,123 @@ #include "llvm/ExecutionEngine/Orc/OrcABISupport.h" #include "llvm/IR/CallSite.h" #include "llvm/IR/IRBuilder.h" +#include "llvm/Support/Format.h" #include "llvm/Transforms/Utils/Cloning.h" #include <sstream> +using namespace llvm; +using namespace llvm::orc; + +namespace { + +class CompileCallbackMaterializationUnit : public orc::MaterializationUnit { +public: + using CompileFunction = JITCompileCallbackManager::CompileFunction; + + CompileCallbackMaterializationUnit(SymbolStringPtr Name, + CompileFunction Compile) + : MaterializationUnit(SymbolFlagsMap({{Name, JITSymbolFlags::Exported}})), + Name(std::move(Name)), Compile(std::move(Compile)) {} + +private: + void materialize(MaterializationResponsibility R) { + SymbolMap Result; + Result[Name] = JITEvaluatedSymbol(Compile(), JITSymbolFlags::Exported); + R.resolve(Result); + R.finalize(); + } + + void discard(const VSO &V, SymbolStringPtr Name) { + llvm_unreachable("Discard should never occur on a LMU?"); + } + + SymbolStringPtr Name; + CompileFunction Compile; +}; + +} // namespace + namespace llvm { namespace orc { void JITCompileCallbackManager::anchor() {} void IndirectStubsManager::anchor() {} +Expected<JITTargetAddress> +JITCompileCallbackManager::getCompileCallback(CompileFunction Compile) { + if (auto TrampolineAddr = getAvailableTrampolineAddr()) { + auto CallbackName = ES.getSymbolStringPool().intern( + std::string("cc") + std::to_string(++NextCallbackId)); + + std::lock_guard<std::mutex> Lock(CCMgrMutex); + AddrToSymbol[*TrampolineAddr] = CallbackName; + cantFail( + CallbacksVSO.define(make_unique<CompileCallbackMaterializationUnit>( + std::move(CallbackName), std::move(Compile)))); + return *TrampolineAddr; + } else + return TrampolineAddr.takeError(); +} + +JITTargetAddress JITCompileCallbackManager::executeCompileCallback( + JITTargetAddress TrampolineAddr) { + SymbolStringPtr Name; + + { + std::unique_lock<std::mutex> Lock(CCMgrMutex); + auto I = AddrToSymbol.find(TrampolineAddr); + + // If this address is not associated with a compile callback then report an + // error to the execution session and return ErrorHandlerAddress to the + // callee. + if (I == AddrToSymbol.end()) { + Lock.unlock(); + std::string ErrMsg; + { + raw_string_ostream ErrMsgStream(ErrMsg); + ErrMsgStream << "No compile callback for trampoline at " + << format("0x%016x", TrampolineAddr); + } + ES.reportError( + make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode())); + return ErrorHandlerAddress; + } else + Name = I->second; + } + + if (auto Sym = lookup({&CallbacksVSO}, Name)) + return Sym->getAddress(); + else { + // If anything goes wrong materializing Sym then report it to the session + // and return the ErrorHandlerAddress; + ES.reportError(Sym.takeError()); + return ErrorHandlerAddress; + } +} + std::unique_ptr<JITCompileCallbackManager> -createLocalCompileCallbackManager(const Triple &T, +createLocalCompileCallbackManager(const Triple &T, ExecutionSession &ES, JITTargetAddress ErrorHandlerAddress) { switch (T.getArch()) { default: return nullptr; case Triple::aarch64: { typedef orc::LocalJITCompileCallbackManager<orc::OrcAArch64> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } case Triple::x86: { typedef orc::LocalJITCompileCallbackManager<orc::OrcI386> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } case Triple::x86_64: { if ( T.getOS() == Triple::OSType::Win32 ) { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_Win32> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } else { typedef orc::LocalJITCompileCallbackManager<orc::OrcX86_64_SysV> CCMgrT; - return llvm::make_unique<CCMgrT>(ErrorHandlerAddress); + return llvm::make_unique<CCMgrT>(ES, ErrorHandlerAddress); } } diff --git a/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp b/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp index 09ad3e6b11c..d6005d24a64 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp +++ b/llvm/lib/ExecutionEngine/Orc/OrcCBindings.cpp @@ -18,12 +18,11 @@ LLVMOrcJITStackRef LLVMOrcCreateInstance(LLVMTargetMachineRef TM) { Triple T(TM2->getTargetTriple()); - auto CompileCallbackMgr = orc::createLocalCompileCallbackManager(T, 0); auto IndirectStubsMgrBuilder = orc::createLocalIndirectStubsManagerBuilder(T); - OrcCBindingsStack *JITStack = new OrcCBindingsStack( - *TM2, std::move(CompileCallbackMgr), IndirectStubsMgrBuilder); + OrcCBindingsStack *JITStack = + new OrcCBindingsStack(*TM2, std::move(IndirectStubsMgrBuilder)); return wrap(JITStack); } diff --git a/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h b/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h index 984adbae1b8..fd35ec46015 100644 --- a/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h +++ b/llvm/lib/ExecutionEngine/Orc/OrcCBindingsStack.h @@ -200,12 +200,10 @@ private: }; public: - OrcCBindingsStack(TargetMachine &TM, - std::unique_ptr<CompileCallbackMgr> CCMgr, IndirectStubsManagerBuilder IndirectStubsMgrBuilder) - : DL(TM.createDataLayout()), - IndirectStubsMgr(IndirectStubsMgrBuilder()), CCMgr(std::move(CCMgr)), + : CCMgr(createLocalCompileCallbackManager(TM.getTargetTriple(), ES, 0)), + DL(TM.createDataLayout()), IndirectStubsMgr(IndirectStubsMgrBuilder()), ObjectLayer(ES, [this](orc::VModuleKey K) { auto ResolverI = Resolvers.find(K); @@ -216,13 +214,14 @@ public: return ObjLayerT::Resources{ std::make_shared<SectionMemoryManager>(), Resolver}; }, - nullptr, - [this](orc::VModuleKey K, const object::ObjectFile &Obj, const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { + nullptr, + [this](orc::VModuleKey K, const object::ObjectFile &Obj, + const RuntimeDyld::LoadedObjectInfo &LoadedObjInfo) { this->notifyFinalized(K, Obj, LoadedObjInfo); - }, - [this](orc::VModuleKey K, const object::ObjectFile &Obj) { + }, + [this](orc::VModuleKey K, const object::ObjectFile &Obj) { this->notifyFreed(K, Obj); - }), + }), CompileLayer(ObjectLayer, orc::SimpleCompiler(TM)), CODLayer(ES, CompileLayer, [this](orc::VModuleKey K) { @@ -270,15 +269,15 @@ public: createLazyCompileCallback(JITTargetAddress &RetAddr, LLVMOrcLazyCompileCallbackFn Callback, void *CallbackCtx) { - if (auto CCInfoOrErr = CCMgr->getCompileCallback()) { - auto &CCInfo = *CCInfoOrErr; - CCInfo.setCompileAction([=]() -> JITTargetAddress { - return Callback(wrap(this), CallbackCtx); - }); - RetAddr = CCInfo.getAddress(); + auto WrappedCallback = [=]() -> JITTargetAddress { + return Callback(wrap(this), CallbackCtx); + }; + + if (auto CCAddr = CCMgr->getCompileCallback(std::move(WrappedCallback))) { + RetAddr = *CCAddr; return LLVMOrcErrSuccess; } else - return mapError(CCInfoOrErr.takeError()); + return mapError(CCAddr.takeError()); } LLVMOrcErrorCode createIndirectStub(StringRef StubName, @@ -484,6 +483,7 @@ private: } orc::ExecutionSession ES; + std::unique_ptr<CompileCallbackMgr> CCMgr; std::vector<JITEventListener *> EventListeners; @@ -492,7 +492,6 @@ private: std::unique_ptr<orc::IndirectStubsManager> IndirectStubsMgr; - std::unique_ptr<CompileCallbackMgr> CCMgr; ObjLayerT ObjectLayer; CompileLayerT CompileLayer; CODLayerT CODLayer; diff --git a/llvm/tools/lli/OrcLazyJIT.cpp b/llvm/tools/lli/OrcLazyJIT.cpp index a7f0e338b9c..31edabd6a29 100644 --- a/llvm/tools/lli/OrcLazyJIT.cpp +++ b/llvm/tools/lli/OrcLazyJIT.cpp @@ -121,15 +121,6 @@ int llvm::runOrcLazyJIT(std::vector<std::unique_ptr<Module>> Ms, EB.setOptLevel(getOptLevel()); auto TM = std::unique_ptr<TargetMachine>(EB.selectTarget()); Triple T(TM->getTargetTriple()); - auto CompileCallbackMgr = orc::createLocalCompileCallbackManager(T, 0); - - // If we couldn't build the factory function then there must not be a callback - // manager for this target. Bail out. - if (!CompileCallbackMgr) { - errs() << "No callback manager available for target '" - << TM->getTargetTriple().str() << "'.\n"; - return 1; - } auto IndirectStubsMgrBuilder = orc::createLocalIndirectStubsManagerBuilder(T); @@ -141,8 +132,7 @@ int llvm::runOrcLazyJIT(std::vector<std::unique_ptr<Module>> Ms, } // Everything looks good. Build the JIT. - OrcLazyJIT J(std::move(TM), std::move(CompileCallbackMgr), - std::move(IndirectStubsMgrBuilder), + OrcLazyJIT J(std::move(TM), std::move(IndirectStubsMgrBuilder), OrcInlineStubs); // Add the module, look up main and run it. diff --git a/llvm/tools/lli/OrcLazyJIT.h b/llvm/tools/lli/OrcLazyJIT.h index a421dc23e7d..33778cef74a 100644 --- a/llvm/tools/lli/OrcLazyJIT.h +++ b/llvm/tools/lli/OrcLazyJIT.h @@ -57,12 +57,11 @@ public: using IndirectStubsManagerBuilder = CODLayerT::IndirectStubsManagerBuilderT; OrcLazyJIT(std::unique_ptr<TargetMachine> TM, - std::unique_ptr<CompileCallbackMgr> CCMgr, IndirectStubsManagerBuilder IndirectStubsMgrBuilder, bool InlineStubs) - : TM(std::move(TM)), - DL(this->TM->createDataLayout()), - CCMgr(std::move(CCMgr)), + : TM(std::move(TM)), DL(this->TM->createDataLayout()), + CCMgr(orc::createLocalCompileCallbackManager( + this->TM->getTargetTriple(), ES, 0)), ObjectLayer(ES, [this](orc::VModuleKey K) { auto ResolverI = Resolvers.find(K); diff --git a/llvm/tools/lli/lli.cpp b/llvm/tools/lli/lli.cpp index 17957c97e2c..07075c2496d 100644 --- a/llvm/tools/lli/lli.cpp +++ b/llvm/tools/lli/lli.cpp @@ -612,8 +612,10 @@ int main(int argc, char **argv, char * const *envp) { } // Create a remote target client running over the channel. + llvm::orc::ExecutionSession ES; + ES.setErrorReporter([&](Error Err) { ExitOnErr(std::move(Err)); }); typedef orc::remote::OrcRemoteTargetClient MyRemote; - auto R = ExitOnErr(MyRemote::Create(*C, ExitOnErr)); + auto R = ExitOnErr(MyRemote::Create(*C, ES)); // Create a remote memory manager. auto RemoteMM = ExitOnErr(R->createRemoteMemoryManager()); diff --git a/llvm/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp b/llvm/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp index 833b7621674..0d2a712a9e8 100644 --- a/llvm/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp +++ b/llvm/unittests/ExecutionEngine/Orc/CompileOnDemandLayerTest.cpp @@ -18,7 +18,8 @@ namespace { class DummyCallbackManager : public orc::JITCompileCallbackManager { public: - DummyCallbackManager() : JITCompileCallbackManager(0) {} + DummyCallbackManager(ExecutionSession &ES) + : JITCompileCallbackManager(ES, 0) {} public: Error grow() override { llvm_unreachable("not implemented"); } @@ -57,9 +58,9 @@ TEST(CompileOnDemandLayerTest, FindSymbol) { return JITSymbol(nullptr); }; - DummyCallbackManager CallbackMgr; ExecutionSession ES(std::make_shared<SymbolStringPool>()); + DummyCallbackManager CallbackMgr(ES); auto GetResolver = [](orc::VModuleKey) -> std::shared_ptr<llvm::orc::SymbolResolver> { diff --git a/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h b/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h index 6b2743b9853..c4424e009b4 100644 --- a/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h +++ b/llvm/unittests/ExecutionEngine/Orc/OrcTestCommon.h @@ -71,11 +71,12 @@ public: // Use ability to create callback manager to detect whether Orc // has indirection support on this platform. This way the test // and Orc code do not get out of sync. - SupportsIndirection = !!orc::createLocalCompileCallbackManager(TT, 0); + SupportsIndirection = !!orc::createLocalCompileCallbackManager(TT, ES, 0); } }; protected: + orc::ExecutionSession ES; LLVMContext Context; std::unique_ptr<TargetMachine> TM; bool SupportsJIT = false; |
