diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine')
18 files changed, 3140 insertions, 65 deletions
diff --git a/llvm/lib/ExecutionEngine/CMakeLists.txt b/llvm/lib/ExecutionEngine/CMakeLists.txt index c0dea0550fb..ccee3460339 100644 --- a/llvm/lib/ExecutionEngine/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/CMakeLists.txt @@ -19,6 +19,7 @@ if(BUILD_SHARED_LIBS) endif() add_subdirectory(Interpreter) +add_subdirectory(JITLink) add_subdirectory(MCJIT) add_subdirectory(Orc) add_subdirectory(RuntimeDyld) diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt new file mode 100644 index 00000000000..2a4196535cf --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -0,0 +1,11 @@ +add_llvm_library(LLVMJITLink + JITLink.cpp + JITLinkGeneric.cpp + JITLink_EHFrameSupport.cpp + JITLink_MachO.cpp + JITLink_MachO_x86_64.cpp + MachOAtomGraphBuilder.cpp + + DEPENDS + intrinsics_gen + ) diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp new file mode 100644 index 00000000000..05901a1668b --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -0,0 +1,261 @@ +//===------------- JITLink.cpp - Core Run-time JIT linker APIs ------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "llvm/BinaryFormat/Magic.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/ManagedStatic.h" +#include "llvm/Support/MemoryBuffer.h" +#include "llvm/Support/Process.h" +#include "llvm/Support/raw_ostream.h" + +using namespace llvm; +using namespace llvm::object; + +#define DEBUG_TYPE "jitlink" + +namespace { + +enum JITLinkErrorCode { GenericJITLinkError = 1 }; + +// FIXME: This class is only here to support the transition to llvm::Error. It +// will be removed once this transition is complete. Clients should prefer to +// deal with the Error value directly, rather than converting to error_code. +class JITLinkerErrorCategory : public std::error_category { +public: + const char *name() const noexcept override { return "runtimedyld"; } + + std::string message(int Condition) const override { + switch (static_cast<JITLinkErrorCode>(Condition)) { + case GenericJITLinkError: + return "Generic JITLink error"; + } + llvm_unreachable("Unrecognized JITLinkErrorCode"); + } +}; + +static ManagedStatic<JITLinkerErrorCategory> JITLinkerErrorCategory; + +} // namespace + +namespace llvm { +namespace jitlink { + +char JITLinkError::ID = 0; + +void JITLinkError::log(raw_ostream &OS) const { OS << ErrMsg << "\n"; } + +std::error_code JITLinkError::convertToErrorCode() const { + return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); +} + +JITLinkMemoryManager::~JITLinkMemoryManager() = default; + +JITLinkMemoryManager::Allocation::~Allocation() = default; + +const StringRef getGenericEdgeKindName(Edge::Kind K) { + switch (K) { + case Edge::Invalid: + return "INVALID RELOCATION"; + case Edge::KeepAlive: + return "Keep-Alive"; + case Edge::LayoutNext: + return "Layout-Next"; + default: + llvm_unreachable("Unrecognized relocation kind"); + } +} + +raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { + OS << "<"; + if (A.getName().empty()) + OS << "anon@" << format("0x%016" PRIx64, A.getAddress()); + else + OS << A.getName(); + OS << " ["; + if (A.isDefined()) { + auto &DA = static_cast<const DefinedAtom &>(A); + OS << " section=" << DA.getSection().getName(); + if (DA.isLive()) + OS << " live"; + if (DA.shouldDiscard()) + OS << " should-discard"; + } else + OS << " external"; + OS << " ]>"; + return OS; +} + +void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, + StringRef EdgeKindName) { + OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) + << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName + << " -> " << E.getTarget() << " + " << E.getAddend(); +} + +Section::~Section() { + for (auto *DA : DefinedAtoms) + DA->~DefinedAtom(); +} + +void AtomGraph::dump(raw_ostream &OS, + std::function<StringRef(Edge::Kind)> EdgeKindToName) { + if (!EdgeKindToName) + EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; + + OS << "Defined atoms:\n"; + for (auto *DA : defined_atoms()) { + OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + << "\n"; + for (auto &E : DA->edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, *DA, E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, *DA, E, EdgeNumberString); + } + OS << "\n"; + } + } + + OS << "Absolute atoms:\n"; + for (auto *A : absolute_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; + + OS << "External atoms:\n"; + for (auto *A : external_atoms()) + OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + << "\n"; +} + +Expected<std::unique_ptr<JITLinkMemoryManager::Allocation>> +InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { + + using AllocationMap = DenseMap<unsigned, sys::MemoryBlock>; + + // Local class for allocation. + class IPMMAlloc : public Allocation { + public: + IPMMAlloc(AllocationMap SegBlocks) : SegBlocks(std::move(SegBlocks)) {} + MutableArrayRef<char> getWorkingMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return {static_cast<char *>(SegBlocks[Seg].base()), + SegBlocks[Seg].size()}; + } + JITTargetAddress getTargetMemory(ProtectionFlags Seg) override { + assert(SegBlocks.count(Seg) && "No allocation for segment"); + return reinterpret_cast<JITTargetAddress>(SegBlocks[Seg].base()); + } + void finalizeAsync(FinalizeContinuation OnFinalize) override { + OnFinalize(applyProtections()); + } + Error deallocate() override { + for (auto &KV : SegBlocks) + if (auto EC = sys::Memory::releaseMappedMemory(KV.second)) + return errorCodeToError(EC); + return Error::success(); + } + + private: + Error applyProtections() { + for (auto &KV : SegBlocks) { + auto &Prot = KV.first; + auto &Block = KV.second; + if (auto EC = sys::Memory::protectMappedMemory(Block, Prot)) + return errorCodeToError(EC); + if (Prot & sys::Memory::MF_EXEC) + sys::Memory::InvalidateInstructionCache(Block.base(), Block.size()); + } + return Error::success(); + } + + AllocationMap SegBlocks; + }; + + AllocationMap Blocks; + const sys::Memory::ProtectionFlags ReadWrite = + static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + for (auto &KV : Request) { + auto &Seg = KV.second; + + if (Seg.getContentAlignment() > sys::Process::getPageSize()) + return make_error<StringError>("Cannot request higher than page " + "alignment", + inconvertibleErrorCode()); + + if (sys::Process::getPageSize() % Seg.getContentAlignment() != 0) + return make_error<StringError>("Page size is not a multiple of " + "alignment", + inconvertibleErrorCode()); + + uint64_t ZeroFillStart = + alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); + uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); + + std::error_code EC; + auto SegMem = + sys::Memory::allocateMappedMemory(SegmentSize, nullptr, ReadWrite, EC); + + if (EC) + return errorCodeToError(EC); + + // Zero out the zero-fill memory. + bzero(static_cast<char *>(SegMem.base()) + ZeroFillStart, + Seg.getZeroFillSize()); + + // Record the block for this segment. + Blocks[KV.first] = std::move(SegMem); + } + return std::unique_ptr<InProcessMemoryManager::Allocation>( + new IPMMAlloc(std::move(Blocks))); +} + +JITLinkContext::~JITLinkContext() {} + +bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { + return true; +} + +AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return AtomGraphPassFunction(); +} + +Error JITLinkContext::modifyPassConfig(const Triple &TT, + PassConfiguration &Config) { + return Error::success(); +} + +Error markAllAtomsLive(AtomGraph &G) { + for (auto *DA : G.defined_atoms()) + DA->setLive(true); + return Error::success(); +} + +void jitLink(std::unique_ptr<JITLinkContext> Ctx) { + auto Magic = identify_magic(Ctx->getObjectBuffer().getBuffer()); + switch (Magic) { + case file_magic::macho_object: + return jitLink_MachO(std::move(Ctx)); + default: + Ctx->notifyFailed(make_error<JITLinkError>("Unsupported file format")); + }; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp new file mode 100644 index 00000000000..f6a31dc0515 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -0,0 +1,464 @@ +//===--------- JITLinkGeneric.cpp - Generic JIT linker utilities ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utility class. +// +//===----------------------------------------------------------------------===// + +#include "JITLinkGeneric.h" +#include "JITLink_EHFrameSupportImpl.h" + +#include "llvm/Support/BinaryStreamReader.h" +#include "llvm/Support/MemoryBuffer.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +JITLinkerBase::~JITLinkerBase() {} + +void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { + + // Build the atom graph. + if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) + G = std::move(*GraphOrErr); + else + return Ctx->notifyFailed(GraphOrErr.takeError()); + assert(G && "Graph should have been created by buildGraph above"); + + // Prune and optimize the graph. + if (auto Err = runPasses(Passes.PrePrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n"; + dumpGraph(dbgs()); + }); + + prune(*G); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n"; + dumpGraph(dbgs()); + }); + + // Run post-pruning passes. + if (auto Err = runPasses(Passes.PostPrunePasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + // Sort atoms into segments. + layOutAtoms(); + + // Allocate memory for segments. + if (auto Err = allocateSegments(Layout)) + return Ctx->notifyFailed(std::move(Err)); + + // Notify client that the defined atoms have been assigned addresses. + Ctx->notifyResolved(*G); + + auto ExternalSymbols = getExternalSymbolNames(); + + // We're about to hand off ownership of ourself to the continuation. Grab a + // pointer to the context so that we can call it to initiate the lookup. + // + // FIXME: Once callee expressions are defined to be sequenced before argument + // expressions (c++17) we can simplify all this to: + // + // Ctx->lookup(std::move(UnresolvedExternals), + // [Self=std::move(Self)](Expected<AsyncLookupResult> Result) { + // Self->linkPhase2(std::move(Self), std::move(Result)); + // }); + // + // FIXME: Use move capture once we have c++14. + auto *TmpCtx = Ctx.get(); + auto *UnownedSelf = Self.release(); + auto Phase2Continuation = + [UnownedSelf](Expected<AsyncLookupResult> LookupResult) { + std::unique_ptr<JITLinkerBase> Self(UnownedSelf); + UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); + }; + TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation)); +} + +void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, + Expected<AsyncLookupResult> LR) { + // If the lookup failed, bail out. + if (!LR) + return Ctx->notifyFailed(LR.takeError()); + + // Assign addresses to external atoms. + applyLookupResult(*LR); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + // Copy atom content to working memory and fix up. + if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + return Ctx->notifyFailed(std::move(Err)); + + LLVM_DEBUG({ + dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dumpGraph(dbgs()); + }); + + if (auto Err = runPasses(Passes.PostFixupPasses, *G)) + return Ctx->notifyFailed(std::move(Err)); + + // FIXME: Use move capture once we have c++14. + auto *UnownedSelf = Self.release(); + auto Phase3Continuation = [UnownedSelf](Error Err) { + std::unique_ptr<JITLinkerBase> Self(UnownedSelf); + UnownedSelf->linkPhase3(std::move(Self), std::move(Err)); + }; + + Alloc->finalizeAsync(std::move(Phase3Continuation)); +} + +void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) { + if (Err) + return Ctx->notifyFailed(std::move(Err)); + Ctx->notifyFinalized(std::move(Alloc)); +} + +Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) { + for (auto &P : Passes) + if (auto Err = P(G)) + return Err; + return Error::success(); +} + +void JITLinkerBase::layOutAtoms() { + // Group sections by protections, and whether or not they're zero-fill. + for (auto &S : G->sections()) { + + // Skip empty sections. + if (S.atoms_empty()) + continue; + + auto &SL = Layout[S.getProtectionFlags()]; + if (S.isZeroFill()) + SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S)); + else + SL.ContentSections.push_back(SegmentLayout::SectionLayout(S)); + } + + // Sort sections within the layout by ordinal. + { + auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS, + const SegmentLayout::SectionLayout &RHS) { + return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal(); + }; + for (auto &KV : Layout) { + auto &SL = KV.second; + std::sort(SL.ContentSections.begin(), SL.ContentSections.end(), + CompareByOrdinal); + std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(), + CompareByOrdinal); + } + } + + // Add atoms to the sections. + for (auto &KV : Layout) { + auto &SL = KV.second; + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) { + for (auto &SI : *SIList) { + std::vector<DefinedAtom *> LayoutHeads; + LayoutHeads.reserve(SI.S->atoms_size()); + + // First build the list of layout-heads (i.e. "heads" of layout-next + // chains). + DenseSet<DefinedAtom *> AlreadyLayedOut; + for (auto *DA : SI.S->atoms()) { + if (AlreadyLayedOut.count(DA)) + continue; + LayoutHeads.push_back(DA); + while (DA->hasLayoutNext()) { + auto &Next = DA->getLayoutNext(); + AlreadyLayedOut.insert(&Next); + DA = &Next; + } + } + + // Now sort the list of layout heads by address. + std::sort(LayoutHeads.begin(), LayoutHeads.end(), + [](const DefinedAtom *LHS, const DefinedAtom *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Now populate the SI.Atoms field by appending each of the chains. + for (auto *DA : LayoutHeads) { + SI.Atoms.push_back(DA); + while (DA->hasLayoutNext()) { + auto &Next = DA->getLayoutNext(); + SI.Atoms.push_back(&Next); + DA = &Next; + } + } + } + } + } + + LLVM_DEBUG({ + dbgs() << "Segment ordering:\n"; + for (auto &KV : Layout) { + dbgs() << " Segment " + << static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n"; + auto &SL = KV.second; + for (auto &SIEntry : + {std::make_pair(&SL.ContentSections, "content sections"), + std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) { + auto &SIList = *SIEntry.first; + dbgs() << " " << SIEntry.second << ":\n"; + for (auto &SI : SIList) { + dbgs() << " " << SI.S->getName() << ":\n"; + for (auto *DA : SI.Atoms) + dbgs() << " " << *DA << "\n"; + } + } + } + }); +} + +Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { + + // Compute segment sizes and allocate memory. + LLVM_DEBUG(dbgs() << "JIT linker requesting: { "); + JITLinkMemoryManager::SegmentsRequestMap Segments; + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + // Calculate segment content size. + size_t SegContentSize = 0; + for (auto &SI : SegLayout.ContentSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + for (auto *DA : SI.Atoms) { + SegContentSize = alignTo(SegContentSize, DA->getAlignment()); + SegContentSize += DA->getSize(); + } + } + + // Get segment content alignment. + unsigned SegContentAlign = 1; + if (!SegLayout.ContentSections.empty()) + SegContentAlign = + SegLayout.ContentSections.front().Atoms.front()->getAlignment(); + + // Calculate segment zero-fill size. + uint64_t SegZeroFillSize = 0; + for (auto &SI : SegLayout.ZeroFillSections) { + assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); + assert(!SI.Atoms.empty() && "Section layouts must not be empty"); + for (auto *DA : SI.Atoms) { + SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment()); + SegZeroFillSize += DA->getSize(); + } + } + + // Calculate segment zero-fill alignment. + uint32_t SegZeroFillAlign = 1; + if (!SegLayout.ZeroFillSections.empty()) + SegZeroFillAlign = + SegLayout.ZeroFillSections.front().Atoms.front()->getAlignment(); + + if (SegContentSize == 0) + SegContentAlign = SegZeroFillAlign; + + if (SegContentAlign % SegZeroFillAlign != 0) + return make_error<JITLinkError>("First content atom alignment does not " + "accommodate first zero-fill atom " + "alignment"); + + Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize, + SegZeroFillAlign}; + + LLVM_DEBUG({ + dbgs() << (&KV == &*Layout.begin() ? "" : "; ") + << static_cast<sys::Memory::ProtectionFlags>(Prot) << ": " + << SegContentSize << " content bytes (alignment " + << SegContentAlign << ") + " << SegZeroFillSize + << " zero-fill bytes (alignment " << SegZeroFillAlign << ")"; + }); + } + LLVM_DEBUG(dbgs() << " }\n"); + + if (auto AllocOrErr = Ctx->getMemoryManager().allocate(Segments)) + Alloc = std::move(*AllocOrErr); + else + return AllocOrErr.takeError(); + + LLVM_DEBUG({ + dbgs() << "JIT linker got working memory:\n"; + for (auto &KV : Layout) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>(KV.first); + dbgs() << " " << Prot << ": " + << (const void *)Alloc->getWorkingMemory(Prot).data() << "\n"; + } + }); + + // Update atom target addresses. + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SL = KV.second; + + JITTargetAddress AtomTargetAddr = + Alloc->getTargetMemory(static_cast<sys::Memory::ProtectionFlags>(Prot)); + + for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) + for (auto &SI : *SIList) + for (auto *DA : SI.Atoms) { + AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment()); + DA->setAddress(AtomTargetAddr); + AtomTargetAddr += DA->getSize(); + } + } + + return Error::success(); +} + +DenseSet<StringRef> JITLinkerBase::getExternalSymbolNames() const { + // Identify unresolved external atoms. + DenseSet<StringRef> UnresolvedExternals; + for (auto *DA : G->external_atoms()) { + assert(DA->getAddress() == 0 && + "External has already been assigned an address"); + assert(DA->getName() != StringRef() && DA->getName() != "" && + "Externals must be named"); + UnresolvedExternals.insert(DA->getName()); + } + return UnresolvedExternals; +} + +void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { + for (auto &KV : Result) { + Atom &A = G->getAtomByName(KV.first); + assert(A.getAddress() == 0 && "Atom already resolved"); + A.setAddress(KV.second.getAddress()); + } + + assert(llvm::all_of(G->external_atoms(), + [](Atom *A) { return A->getAddress() != 0; }) && + "All atoms should have been resolved by this point"); +} + +void JITLinkerBase::dumpGraph(raw_ostream &OS) { + assert(G && "Graph is not set yet"); + G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); +} + +void prune(AtomGraph &G) { + std::vector<DefinedAtom *> Worklist; + DenseMap<DefinedAtom *, std::vector<Edge *>> EdgesToUpdate; + + // Build the initial worklist from all atoms initially live. + for (auto *DA : G.defined_atoms()) { + if (!DA->isLive() || DA->shouldDiscard()) + continue; + + for (auto &E : DA->edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Propagate live flags to all atoms reachable from the initial live set. + while (!Worklist.empty()) { + DefinedAtom &NextLive = *Worklist.back(); + Worklist.pop_back(); + + assert(!NextLive.shouldDiscard() && + "should-discard nodes should never make it into the worklist"); + + // If this atom has already been marked as live, or is marked to be + // discarded, then skip it. + if (NextLive.isLive()) + continue; + + // Otherwise set it as live and add any non-live atoms that it points to + // to the worklist. + NextLive.setLive(true); + + for (auto &E : NextLive.edges()) { + if (!E.getTarget().isDefined()) + continue; + + auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); + + if (EDT.shouldDiscard()) + EdgesToUpdate[&EDT].push_back(&E); + else if (E.isKeepAlive() && !EDT.isLive()) + Worklist.push_back(&EDT); + } + } + + // Collect atoms to remove, then remove them from the graph. + std::vector<DefinedAtom *> AtomsToRemove; + for (auto *DA : G.defined_atoms()) + if (DA->shouldDiscard() || !DA->isLive()) + AtomsToRemove.push_back(DA); + + LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); + for (auto *DA : AtomsToRemove) { + LLVM_DEBUG(dbgs() << " " << *DA << "... "); + + // Check whether we need to replace this atom with an external atom. + // + // We replace if all of the following hold: + // (1) The atom is marked should-discard, + // (2) it is live, and + // (3) it has edges pointing to it. + // + // Otherwise we simply delete the atom. + bool ReplaceWithExternal = DA->isLive() && DA->shouldDiscard(); + std::vector<Edge *> *EdgesToUpdateForDA = nullptr; + if (ReplaceWithExternal) { + auto ETUItr = EdgesToUpdate.find(DA); + if (ETUItr == EdgesToUpdate.end()) + ReplaceWithExternal = false; + else + EdgesToUpdateForDA = &ETUItr->second; + } + + G.removeDefinedAtom(*DA); + + if (ReplaceWithExternal) { + assert(EdgesToUpdateForDA && + "Replacing atom: There should be edges to update"); + + auto &ExternalReplacement = G.addExternalAtom(DA->getName()); + for (auto *EdgeToUpdate : *EdgesToUpdateForDA) + EdgeToUpdate->setTarget(ExternalReplacement); + LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n"); + } else + LLVM_DEBUG(dbgs() << "deleted\n"); + } + + // Finally, discard any absolute symbols that were marked should-discard. + { + std::vector<Atom *> AbsoluteAtomsToRemove; + for (auto *A : G.absolute_atoms()) + if (A->shouldDiscard() || A->isLive()) + AbsoluteAtomsToRemove.push_back(A); + for (auto *A : AbsoluteAtomsToRemove) + G.removeAbsoluteAtom(*A); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h new file mode 100644 index 00000000000..74db48fdff2 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -0,0 +1,246 @@ +//===------ JITLinkGeneric.h - Generic JIT linker utilities -----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic JITLinker utilities. E.g. graph pruning, eh-frame parsing. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H +#define LIB_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H + +#include "llvm/ADT/DenseSet.h" +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { + +class MemoryBufferRef; + +namespace jitlink { + +/// Base class for a JIT linker. +/// +/// A JITLinkerBase instance links one object file into an ongoing JIT +/// session. Symbol resolution and finalization operations are pluggable, +/// and called using continuation passing (passing a continuation for the +/// remaining linker work) to allow them to be performed asynchronously. +class JITLinkerBase { +public: + JITLinkerBase(std::unique_ptr<JITLinkContext> Ctx, PassConfiguration Passes) + : Ctx(std::move(Ctx)), Passes(std::move(Passes)) { + assert(this->Ctx && "Ctx can not be null"); + } + + virtual ~JITLinkerBase(); + +protected: + struct SegmentLayout { + using SectionAtomsList = std::vector<DefinedAtom *>; + struct SectionLayout { + SectionLayout(Section &S) : S(&S) {} + + Section *S; + SectionAtomsList Atoms; + }; + + using SectionLayoutList = std::vector<SectionLayout>; + + SectionLayoutList ContentSections; + SectionLayoutList ZeroFillSections; + }; + + using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>; + + // Phase 1: + // 1.1: Build atom graph + // 1.2: Run pre-prune passes + // 1.2: Prune graph + // 1.3: Run post-prune passes + // 1.4: Sort atoms into segments + // 1.5: Allocate segment memory + // 1.6: Identify externals and make an async call to resolve function + void linkPhase1(std::unique_ptr<JITLinkerBase> Self); + + // Phase 2: + // 2.1: Apply resolution results + // 2.2: Fix up atom contents + // 2.3: Call OnResolved callback + // 2.3: Make an async call to transfer and finalize memory. + void linkPhase2(std::unique_ptr<JITLinkerBase> Self, + Expected<AsyncLookupResult> LookupResult); + + // Phase 3: + // 3.1: Call OnFinalized callback, handing off allocation. + void linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err); + + // Build a graph from the given object buffer. + // To be implemented by the client. + virtual Expected<std::unique_ptr<AtomGraph>> + buildGraph(MemoryBufferRef ObjBuffer) = 0; + + // For debug dumping of the atom graph. + virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + +private: + // Run all passes in the given pass list, bailing out immediately if any pass + // returns an error. + Error runPasses(AtomGraphPassList &Passes, AtomGraph &G); + + // Copy atom contents and apply relocations. + // Implemented in JITLinker. + virtual Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; + + void layOutAtoms(); + Error allocateSegments(const SegmentLayoutMap &Layout); + DenseSet<StringRef> getExternalSymbolNames() const; + void applyLookupResult(AsyncLookupResult LR); + + void dumpGraph(raw_ostream &OS); + + std::unique_ptr<JITLinkContext> Ctx; + PassConfiguration Passes; + std::unique_ptr<AtomGraph> G; + SegmentLayoutMap Layout; + std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc; +}; + +template <typename LinkerImpl> class JITLinker : public JITLinkerBase { +public: + using JITLinkerBase::JITLinkerBase; + + /// Link constructs a LinkerImpl instance and calls linkPhase1. + /// Link should be called with the constructor arguments for LinkerImpl, which + /// will be forwarded to the constructor. + template <typename... ArgTs> static void link(ArgTs &&... Args) { + auto L = llvm::make_unique<LinkerImpl>(std::forward<ArgTs>(Args)...); + + // Ownership of the linker is passed into the linker's doLink function to + // allow it to be passed on to async continuations. + // + // FIXME: Remove LTmp once we have c++17. + // C++17 sequencing rules guarantee that function name expressions are + // sequenced before arguments, so L->linkPhase1(std::move(L), ...) will be + // well formed. + auto <mp = *L; + LTmp.linkPhase1(std::move(L)); + } + +private: + const LinkerImpl &impl() const { + return static_cast<const LinkerImpl &>(*this); + } + + Error + copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + for (auto &KV : Layout) { + auto &Prot = KV.first; + auto &SegLayout = KV.second; + + auto SegMem = Alloc.getWorkingMemory( + static_cast<sys::Memory::ProtectionFlags>(Prot)); + char *LastAtomEnd = SegMem.data(); + char *AtomDataPtr = nullptr; + + LLVM_DEBUG({ + dbgs() << " Processing segment " + << static_cast<sys::Memory::ProtectionFlags>(Prot) << " [ " + << (const void *)SegMem.data() << " .. " + << (const void *)((char *)SegMem.data() + SegMem.size()) + << " ]\n Processing content sections:\n"; + }); + + for (auto &SI : SegLayout.ContentSections) { + LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n"); + for (auto *DA : SI.Atoms) { + AtomDataPtr = LastAtomEnd; + + // Align. + AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment()); + LLVM_DEBUG({ + dbgs() << " Bumped atom pointer to " + << (const void *)AtomDataPtr << " to meet alignment of " + << DA->getAlignment() << "\n"; + }); + + // Zero pad up to alignment. + LLVM_DEBUG({ + if (LastAtomEnd != AtomDataPtr) + dbgs() << " Zero padding from " << (const void *)LastAtomEnd + << " to " << (const void *)AtomDataPtr << "\n"; + }); + while (LastAtomEnd != AtomDataPtr) + *LastAtomEnd++ = 0; + + // Copy initial atom content. + LLVM_DEBUG({ + dbgs() << " Copying atom " << *DA << " content, " + << DA->getContent().size() << " bytes, from " + << (const void *)DA->getContent().data() << " to " + << (const void *)AtomDataPtr << "\n"; + }); + memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size()); + + // Copy atom data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : DA->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr)) + return Err; + } + + // Point the atom's content to the fixed up buffer. + DA->setContent(StringRef(AtomDataPtr, DA->getContent().size())); + + // Update atom end pointer. + LastAtomEnd = AtomDataPtr + DA->getContent().size(); + } + } + + // Zero pad the rest of the segment. + LLVM_DEBUG({ + dbgs() << " Zero padding end of segment from " + << (const void *)LastAtomEnd << " to " + << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; + }); + while (LastAtomEnd != SegMem.data() + SegMem.size()) + *LastAtomEnd++ = 0; + } + + return Error::success(); + } +}; + +/// Dead strips and replaces discarded definitions with external atoms. +/// +/// Finds the set of nodes reachable from any node initially marked live +/// (nodes marked should-discard are treated as not live, even if they are +/// reachable). All nodes not marked as live at the end of this process, +/// are deleted. Nodes that are live, but marked should-discard are replaced +/// with external atoms and all edges to them are re-written. +void prune(AtomGraph &G); + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#undef DEBUG_TYPE // "jitlink" + +#endif // LLVM_EXECUTIONENGINE_JITLINK_JITLINKGENERIC_H diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupport.cpp new file mode 100644 index 00000000000..bf051474b2d --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupport.cpp @@ -0,0 +1,537 @@ +//===-------- JITLink_EHFrameSupport.cpp - JITLink eh-frame utils ---------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "JITLink_EHFrameSupportImpl.h" + +#include "llvm/BinaryFormat/Dwarf.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), + EHFrameAddress(EHFrameAddress), + EHFrameReader(EHFrameContent, G.getEndianness()), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + +Error EHFrameParser::atomize() { + while (!EHFrameReader.empty()) { + size_t RecordOffset = EHFrameReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << "Processing eh-frame record at " + << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) + << " (offset " << RecordOffset << ")\n"; + }); + + size_t CIELength = 0; + uint32_t CIELengthField; + if (auto Err = EHFrameReader.readInteger(CIELengthField)) + return Err; + + // Process CIE length/extended-length fields to build the atom. + // + // The value of these fields describe the length of the *rest* of the CIE + // (not including data up to the end of the field itself) so we have to + // bump CIELength to include the data up to the end of the field: 4 bytes + // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. + if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + break; + + // If the regular length field's value is 0xffffffff, use extended length. + if (CIELengthField == 0xffffffff) { + uint64_t CIEExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + return Err; + if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + return make_error<JITLinkError>("CIE record extends past the end of " + "the __eh_frame section"); + if (CIEExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) + return make_error<JITLinkError>("CIE record too large to process"); + CIELength = CIEExtendedLengthField + 12; + } else { + if (CIELengthField > EHFrameReader.bytesRemaining()) + return make_error<JITLinkError>("CIE record extends past the end of " + "the __eh_frame section"); + CIELength = CIELengthField + 4; + } + + LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); + + // Add an atom for this record. + CurRecordAtom = &G.addAnonymousAtom( + EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); + CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength)); + + // Read the CIE Pointer. + size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); + uint32_t CIEPointer; + if (auto Err = EHFrameReader.readInteger(CIEPointer)) + return Err; + + // Based on the CIE pointer value, parse this as a CIE or FDE record. + if (CIEPointer == 0) { + if (auto Err = processCIE()) + return Err; + } else { + if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + return Err; + } + + EHFrameReader.setOffset(RecordOffset + CIELength); + } + + return Error::success(); +} + +Expected<EHFrameParser::AugmentationInfo> +EHFrameParser::parseAugmentationString() { + AugmentationInfo AugInfo; + uint8_t NextChar; + uint8_t *NextField = &AugInfo.Fields[0]; + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + + while (NextChar != 0) { + switch (NextChar) { + case 'z': + AugInfo.AugmentationDataPresent = true; + break; + case 'e': + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + if (NextChar != 'h') + return make_error<JITLinkError>("Unrecognized substring e" + + Twine(NextChar) + + " in augmentation string"); + AugInfo.EHDataFieldPresent = true; + break; + case 'L': + case 'P': + case 'R': + *NextField++ = NextChar; + break; + default: + return make_error<JITLinkError>("Unrecognized character " + + Twine(NextChar) + + " in augmentation string"); + } + + if (auto Err = EHFrameReader.readInteger(NextChar)) + return std::move(Err); + } + + return std::move(AugInfo); +} + +Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() { + static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), + "Result must be able to hold a uint64_t"); + JITTargetAddress Addr; + if (G.getPointerSize() == 8) { + if (auto Err = EHFrameReader.readInteger(Addr)) + return std::move(Err); + } else if (G.getPointerSize() == 4) { + uint32_t Addr32; + if (auto Err = EHFrameReader.readInteger(Addr32)) + return std::move(Err); + Addr = Addr32; + } else + llvm_unreachable("Pointer size is not 32-bit or 64-bit"); + return Addr; +} + +Error EHFrameParser::processCIE() { + // Use the dwarf namespace for convenient access to pointer encoding + // constants. + using namespace dwarf; + + LLVM_DEBUG(dbgs() << " Record is CIE\n"); + + /// Reset state for the new CIE. + MostRecentCIE = CurRecordAtom; + LSDAFieldPresent = false; + + uint8_t Version = 0; + if (auto Err = EHFrameReader.readInteger(Version)) + return Err; + + if (Version != 0x01) + return make_error<JITLinkError>("Bad CIE version " + Twine(Version) + + " (should be 0x01) in eh-frame"); + + auto AugInfo = parseAugmentationString(); + if (!AugInfo) + return AugInfo.takeError(); + + // Skip the EH Data field if present. + if (AugInfo->EHDataFieldPresent) + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + // Read and sanity check the code alignment factor. + { + uint64_t CodeAlignmentFactor = 0; + if (auto Err = EHFrameReader.readULEB128(CodeAlignmentFactor)) + return Err; + if (CodeAlignmentFactor != 1) + return make_error<JITLinkError>("Unsupported CIE code alignment factor " + + Twine(CodeAlignmentFactor) + + " (expected 1)"); + } + + // Read and sanity check the data alignment factor. + { + int64_t DataAlignmentFactor = 0; + if (auto Err = EHFrameReader.readSLEB128(DataAlignmentFactor)) + return Err; + if (DataAlignmentFactor != -8) + return make_error<JITLinkError>("Unsupported CIE data alignment factor " + + Twine(DataAlignmentFactor) + + " (expected -8)"); + } + + // Skip the return address register field. + if (auto Err = EHFrameReader.skip(1)) + return Err; + + uint64_t AugmentationDataLength = 0; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataLength)) + return Err; + + uint32_t AugmentationDataStartOffset = EHFrameReader.getOffset(); + + uint8_t *NextField = &AugInfo->Fields[0]; + while (uint8_t Field = *NextField++) { + switch (Field) { + case 'L': { + LSDAFieldPresent = true; + uint8_t LSDAPointerEncoding; + if (auto Err = EHFrameReader.readInteger(LSDAPointerEncoding)) + return Err; + if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error<JITLinkError>( + "Unsupported LSDA pointer encoding " + + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + case 'P': { + uint8_t PersonalityPointerEncoding = 0; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerEncoding)) + return Err; + if (PersonalityPointerEncoding != + (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4)) + return make_error<JITLinkError>( + "Unspported personality pointer " + "encoding " + + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + uint32_t PersonalityPointerAddress; + if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) + return Err; + break; + } + case 'R': { + uint8_t FDEPointerEncoding; + if (auto Err = EHFrameReader.readInteger(FDEPointerEncoding)) + return Err; + if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) + return make_error<JITLinkError>( + "Unsupported FDE address pointer " + "encoding " + + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + + formatv("{0:x16}", CurRecordAtom->getAddress())); + break; + } + default: + llvm_unreachable("Invalid augmentation string field"); + } + } + + if (EHFrameReader.getOffset() - AugmentationDataStartOffset > + AugmentationDataLength) + return make_error<JITLinkError>("Read past the end of the augmentation " + "data while parsing fields"); + + return Error::success(); +} + +Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, + uint32_t CIEPointer) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); + + // Sanity check the CIE pointer: if this is an FDE it must be proceeded by + // a CIE. + if (MostRecentCIE == nullptr) + return make_error<JITLinkError>("__eh_frame must start with CIE, not " + "FDE"); + + LLVM_DEBUG({ + dbgs() << " CIE pointer: " + << format("0x%016" PRIx64, CIEPointerAddress - CIEPointer) << "\n"; + }); + + // Verify that this FDE's CIE pointer points to the most recent CIE entry. + if (CIEPointerAddress - CIEPointer != MostRecentCIE->getAddress()) + return make_error<JITLinkError>("__eh_frame FDE's CIE Pointer does not " + "point at the most recent CIE"); + + // The CIEPointer looks good. Add a relocation. + CurRecordAtom->addEdge(FDEToCIERelocKind, + CIEPointerAddress - CurRecordAtom->getAddress(), + *MostRecentCIE, 0); + + // Read and sanity check the PC-start pointer and size. + JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); + + auto PCBeginDelta = readAbsolutePointer(); + if (!PCBeginDelta) + return PCBeginDelta.takeError(); + + JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; + LLVM_DEBUG({ + dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; + }); + + auto *TargetAtom = G.getAtomByAddress(PCBegin); + + if (!TargetAtom) + return make_error<JITLinkError>("FDE PC-begin " + + formatv("{0:x16}", PCBegin) + + " does not point at atom"); + + if (TargetAtom->getAddress() != PCBegin) + return make_error<JITLinkError>( + "FDE PC-begin " + formatv("{0:x16}", PCBegin) + + " does not point to start of atom at " + + formatv("{0:x16}", TargetAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); + + // The PC-start pointer and size look good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + PCBeginAddress - CurRecordAtom->getAddress(), + *TargetAtom, 0); + + // Add a keep-alive relocation from the function to the FDE to ensure it is + // not dead stripped. + TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + + // Skip over the PC range size field. + if (auto Err = EHFrameReader.skip(G.getPointerSize())) + return Err; + + if (LSDAFieldPresent) { + uint64_t AugmentationDataSize; + if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) + return Err; + if (AugmentationDataSize != G.getPointerSize()) + return make_error<JITLinkError>("Unexpected FDE augmentation data size " + "(expected " + + Twine(G.getPointerSize()) + ", got " + + Twine(AugmentationDataSize) + ")"); + JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); + auto LSDADelta = readAbsolutePointer(); + if (!LSDADelta) + return LSDADelta.takeError(); + + JITTargetAddress LSDA = LSDAAddress + *LSDADelta; + + auto *LSDAAtom = G.getAtomByAddress(LSDA); + + if (!LSDAAtom) + return make_error<JITLinkError>("FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point at atom"); + + if (LSDAAtom->getAddress() != LSDA) + return make_error<JITLinkError>( + "FDE LSDA " + formatv("{0:x16}", LSDA) + + " does not point to start of atom at " + + formatv("{0:x16}", LSDAAtom->getAddress())); + + LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n"); + + // LSDA looks good. Add relocations. + CurRecordAtom->addEdge(FDEToTargetRelocKind, + LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom, + 0); + } + + return Error::success(); +} + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) { + return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, + FDEToCIERelocKind, FDEToTargetRelocKind) + .atomize(); +} + +// Determine whether we can register EH tables. +#if (defined(__GNUC__) && !defined(__ARM_EABI__) && !defined(__ia64__) && \ + !defined(__SEH__) && !defined(__USING_SJLJ_EXCEPTIONS__)) +#define HAVE_EHTABLE_SUPPORT 1 +#else +#define HAVE_EHTABLE_SUPPORT 0 +#endif + +#if HAVE_EHTABLE_SUPPORT +extern "C" void __register_frame(const void *); +extern "C" void __deregister_frame(const void *); + +Error registerFrameWrapper(const void *P) { + __register_frame(P); + return Error::success(); +} + +Error deregisterFrameWrapper(const void *P) { + __deregister_frame(P); + return Error::success(); +} + +#else + +// The building compiler does not have __(de)register_frame but +// it may be found at runtime in a dynamically-loaded library. +// For example, this happens when building LLVM with Visual C++ +// but using the MingW runtime. +static Error registerFrameWrapper(const void *P) { + static void((*RegisterFrame)(const void *)) = 0; + + if (!RegisterFrame) + *(void **)&RegisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol("__register_frame"); + + if (RegisterFrame) { + RegisterFrame(P); + return Error::success(); + } + + return make_error<JITLinkError>("could not register eh-frame: " + "__register_frame function not found"); +} + +static void deregisterFrameWrapper(const void *P) { + static void((*DeregisterFrame)(const void *)) = 0; + + if (!DeregisterFrame) + *(void **)&DeregisterFrame = + llvm::sys::DynamicLibrary::SearchForAddressOfSymbol( + "__deregister_frame"); + + if (DeregisterFrame) { + DeregisterFrame(P); + return Error::success(); + } + + return make_error<JITLinkError>("could not deregister eh-frame: " + "__deregister_frame function not found"); +} +#endif + +#ifdef __APPLE__ + +template <typename HandleFDEFn> +Error walkAppleEHFrameSection(const char *const SectionStart, + HandleFDEFn HandleFDE) { + const char *CurCFIRecord = SectionStart; + uint64_t Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + + while (Size != 0) { + const char *OffsetField = CurCFIRecord + (Size == 0xffffffff ? 12 : 4); + if (Size == 0xffffffff) + Size = *reinterpret_cast<const uint64_t *>(CurCFIRecord + 4) + 12; + else + Size += 4; + uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + + LLVM_DEBUG({ + dbgs() << "Registering eh-frame section:\n"; + dbgs() << "Processing " << (Offset ? "FDE" : "CIE") << " @" + << (void *)CurCFIRecord << ": ["; + for (unsigned I = 0; I < Size; ++I) + dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); + dbgs() << " ]\n"; + }); + CurCFIRecord += Size; + + Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); + } + + return Error::success(); +} + +#endif // __APPLE__ + +Error registerEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + // On Darwin __register_frame has to be called for each FDE entry. + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + registerFrameWrapper); +#else + // On Linux __register_frame takes a single argument: + // a pointer to the start of the .eh_frame section. + + // How can it find the end? Because crtendS.o is linked + // in and it has an .eh_frame section with four zero chars. + return registerFrameWrapper(EHFrameSectionAddr); +#endif +} + +Error deregisterEHFrameSection(const void *EHFrameSectionAddr) { +#ifdef __APPLE__ + return walkAppleEHFrameSection(static_cast<const char *>(EHFrameSectionAddr), + deregisterFrameWrapper); +#else + return deregisterFrameWrapper(EHFrameSectionAddr); +#endif +} + +AtomGraphPassFunction createEHFrameRecorderPass(const Triple &TT, + JITTargetAddress &EHFrameAddr) { + const char *EHFrameSectionName = nullptr; + if (TT.getObjectFormat() == Triple::MachO) + EHFrameSectionName = "__eh_frame"; + else + EHFrameSectionName = ".eh_frame"; + + auto RecordEHFrame = [EHFrameSectionName, + &EHFrameAddr](AtomGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first atom + // in it. + JITTargetAddress Addr = 0; + for (auto &S : G.sections()) + if (S.getName() == EHFrameSectionName && !S.atoms_empty()) { + Addr = (*S.atoms().begin())->getAddress(); + break; + } + + EHFrameAddr = Addr; + return Error::success(); + }; + + return RecordEHFrame; +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupportImpl.h new file mode 100644 index 00000000000..85b1b803c5d --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink_EHFrameSupportImpl.h @@ -0,0 +1,66 @@ +//===----- JITLink_EHFrameSupport.h - JITLink eh-frame utils ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// EHFrame registration support for JITLink. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_LIB_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORTIMPL_H +#define LLVM_LIB_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORTIMPL_H + +#include "llvm/ExecutionEngine/JITLink/JITLink_EHFrameSupport.h" + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" +#include "llvm/Support/BinaryStreamReader.h" + +namespace llvm { +namespace jitlink { + +/// A generic parser for eh-frame sections. +/// +/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and +/// FDEToTarget relocation kinds. +class EHFrameParser { +public: + EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, + JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind); + Error atomize(); + +private: + struct AugmentationInfo { + bool AugmentationDataPresent = false; + bool EHDataFieldPresent = false; + uint8_t Fields[4] = {0x0, 0x0, 0x0, 0x0}; + }; + + Expected<AugmentationInfo> parseAugmentationString(); + Expected<JITTargetAddress> readAbsolutePointer(); + Error processCIE(); + Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer); + + AtomGraph &G; + Section &EHFrameSection; + StringRef EHFrameContent; + JITTargetAddress EHFrameAddress; + BinaryStreamReader EHFrameReader; + DefinedAtom *CurRecordAtom = nullptr; + DefinedAtom *MostRecentCIE = nullptr; + bool LSDAFieldPresent = false; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; +}; + +Error addEHFrame(AtomGraph &G, Section &EHFrameSection, + StringRef EHFrameContent, JITTargetAddress EHFrameAddress, + Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); + +} // end namespace jitlink +} // end namespace llvm + +#endif // LLVM_LIB_EXECUTIONENGINE_JITLINK_JITLINK_EHFRAMESUPPORTIMPL_H diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink_MachO.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink_MachO.cpp new file mode 100644 index 00000000000..7f4364d0b34 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink_MachO.cpp @@ -0,0 +1,73 @@ +//===------------ JITLink.cpp - Run-time JIT linker for MachO -------------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO.h" + +#include "llvm/BinaryFormat/MachO.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h" +#include "llvm/Support/Endian.h" +#include "llvm/Support/Format.h" +#include "llvm/Support/MemoryBuffer.h" + +using namespace llvm; + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +void jitLink_MachO(std::unique_ptr<JITLinkContext> Ctx) { + + // We don't want to do full MachO validation here. Just parse enough of the + // header to find out what MachO linker to use. + + StringRef Data = Ctx->getObjectBuffer().getBuffer(); + if (Data.size() < 4) { + Ctx->notifyFailed(make_error<JITLinkError>("Truncated MachO buffer")); + return; + } + + uint32_t Magic; + memcpy(&Magic, Data.data(), sizeof(uint32_t)); + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: magic = " << format("0x%08" PRIx32, Magic) + << "\n"; + }); + + if (Magic == MachO::MH_MAGIC || Magic == MachO::MH_CIGAM) { + Ctx->notifyFailed( + make_error<JITLinkError>("MachO 32-bit platforms not supported")); + return; + } else if (Magic == MachO::MH_MAGIC_64 || Magic == MachO::MH_CIGAM_64) { + MachO::mach_header_64 Header; + + memcpy(&Header, Data.data(), sizeof(MachO::mach_header_64)); + if (Magic == MachO::MH_CIGAM_64) + swapStruct(Header); + + LLVM_DEBUG({ + dbgs() << "jitLink_MachO: cputype = " + << format("0x%08" PRIx32, Header.cputype) + << ", cpusubtype = " << format("0x%08" PRIx32, Header.cpusubtype) + << "\n"; + }); + + switch (Header.cputype) { + case MachO::CPU_TYPE_X86_64: + return jitLink_MachO_x86_64(std::move(Ctx)); + } + Ctx->notifyFailed(make_error<JITLinkError>("MachO-64 CPU type not valid")); + return; + } + + Ctx->notifyFailed(make_error<JITLinkError>("MachO magic not valid")); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp new file mode 100644 index 00000000000..fd058fa3116 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink_MachO_x86_64.cpp @@ -0,0 +1,621 @@ +//===------- JITLink_MachO_x86_64.cpp - JIT linker functionality ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// MachO jit-link implementation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/JITLink/JITLink_MachO_x86_64.h" + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::jitlink::MachO_x86_64_Edges; + +namespace { + +class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +public: + MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOAtomGraphBuilder(Obj), + NumSymbols(Obj.getSymtabLoadCommand().nsyms) { + addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { + return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), + EHFrameSection.getContent(), + EHFrameSection.getAddress(), NegDelta32, Delta64); + }); + } + +private: + static Expected<MachOX86RelocationKind> + getRelocationKind(const MachO::relocation_info &RI) { + switch (RI.r_type) { + case MachO::X86_64_RELOC_UNSIGNED: + if (!RI.r_pcrel && RI.r_length == 3) + return RI.r_extern ? Pointer64 : Pointer64Anon; + break; + case MachO::X86_64_RELOC_SIGNED: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32 : PCRel32Anon; + break; + case MachO::X86_64_RELOC_BRANCH: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return Branch32; + break; + case MachO::X86_64_RELOC_GOT_LOAD: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOTLoad; + break; + case MachO::X86_64_RELOC_GOT: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32GOT; + break; + case MachO::X86_64_RELOC_SUBTRACTOR: + // SUBTRACTOR must be non-pc-rel, extern, with length 2 or 3. + // Initially represent SUBTRACTOR relocations with 'Delta<W>'. They may + // be turned into NegDelta<W> by parsePairRelocation. + if (!RI.r_pcrel && RI.r_extern) { + if (RI.r_length == 2) + return Delta32; + else if (RI.r_length == 3) + return Delta64; + } + break; + case MachO::X86_64_RELOC_SIGNED_1: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus1 : PCRel32Minus1Anon; + break; + case MachO::X86_64_RELOC_SIGNED_2: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus2 : PCRel32Minus2Anon; + break; + case MachO::X86_64_RELOC_SIGNED_4: + if (RI.r_pcrel && RI.r_length == 2) + return RI.r_extern ? PCRel32Minus4 : PCRel32Minus4Anon; + break; + case MachO::X86_64_RELOC_TLV: + if (RI.r_pcrel && RI.r_extern && RI.r_length == 2) + return PCRel32TLV; + break; + } + + return make_error<JITLinkError>("Unsupported x86-64 relocation kind"); + } + + Expected<Atom &> findAtomBySymbolIndex(const MachO::relocation_info &RI) { + auto &Obj = getObject(); + if (RI.r_symbolnum >= NumSymbols) + return make_error<JITLinkError>("Symbol index out of range"); + auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); + auto Name = SymI->getName(); + if (!Name) + return Name.takeError(); + return getGraph().getAtomByName(*Name); + } + + MachO::relocation_info + getRelocationInfo(const object::relocation_iterator RelItr) { + MachO::any_relocation_info ARI = + getObject().getRelocation(RelItr->getRawDataRefImpl()); + MachO::relocation_info RI; + memcpy(&RI, &ARI, sizeof(MachO::relocation_info)); + return RI; + } + + using PairRelocInfo = std::tuple<MachOX86RelocationKind, Atom *, uint64_t>; + + // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, + // returns the edge kind and addend to be used. + Expected<PairRelocInfo> + parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + const MachO::relocation_info &SubRI, + JITTargetAddress FixupAddress, const char *FixupContent, + object::relocation_iterator &UnsignedRelItr, + object::relocation_iterator &RelEnd) { + using namespace support; + + assert(((SubtractorKind == Delta32 && SubRI.r_length == 2) || + (SubtractorKind == Delta64 && SubRI.r_length == 3)) && + "Subtractor kind should match length"); + assert(SubRI.r_extern && "SUBTRACTOR reloc symbol should be extern"); + assert(!SubRI.r_pcrel && "SUBTRACTOR reloc should not be PCRel"); + + if (UnsignedRelItr == RelEnd) + return make_error<JITLinkError>("x86_64 SUBTRACTOR without paired " + "UNSIGNED relocation"); + + auto UnsignedRI = getRelocationInfo(UnsignedRelItr); + + if (SubRI.r_address != UnsignedRI.r_address) + return make_error<JITLinkError>("x86_64 SUBTRACTOR and paired UNSIGNED " + "point to different addresses"); + + if (SubRI.r_length != UnsignedRI.r_length) + return make_error<JITLinkError>("length of x86_64 SUBTRACTOR and paired " + "UNSIGNED reloc must match"); + + auto FromAtom = findAtomBySymbolIndex(SubRI); + if (!FromAtom) + return FromAtom.takeError(); + + // Read the current fixup value. + uint64_t FixupValue = 0; + if (SubRI.r_length == 3) + FixupValue = *(const ulittle64_t *)FixupContent; + else + FixupValue = *(const ulittle32_t *)FixupContent; + + // Find 'ToAtom' using symbol number or address, depending on whether the + // paired UNSIGNED relocation is extern. + Atom *ToAtom = nullptr; + if (UnsignedRI.r_extern) { + // Find target atom by symbol index. + if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + } else { + if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue)) + ToAtom = &*ToAtomOrErr; + else + return ToAtomOrErr.takeError(); + FixupValue -= ToAtom->getAddress(); + } + + MachOX86RelocationKind DeltaKind; + Atom *TargetAtom; + uint64_t Addend; + if (&AtomToFix == &*FromAtom) { + TargetAtom = ToAtom; + DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; + Addend = FixupValue + (FixupAddress - FromAtom->getAddress()); + // FIXME: handle extern 'from'. + } else if (&AtomToFix == ToAtom) { + TargetAtom = &*FromAtom; + DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; + Addend = FixupValue - (FixupAddress - ToAtom->getAddress()); + } else { + // AtomToFix was neither FromAtom nor ToAtom. + return make_error<JITLinkError>("SUBTRACTOR relocation must fix up " + "either 'A' or 'B'"); + } + + return PairRelocInfo(DeltaKind, TargetAtom, Addend); + } + + Error addRelocations() override { + using namespace support; + auto &G = getGraph(); + auto &Obj = getObject(); + + for (auto &S : Obj.sections()) { + + JITTargetAddress SectionAddress = S.getAddress(); + + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); + RelItr != RelEnd; ++RelItr) { + + MachO::relocation_info RI = getRelocationInfo(RelItr); + + // Sanity check the relocation kind. + auto Kind = getRelocationKind(RI); + if (!Kind) + return Kind.takeError(); + + // Find the address of the value to fix up. + JITTargetAddress FixupAddress = SectionAddress + (uint32_t)RI.r_address; + + LLVM_DEBUG({ + dbgs() << "Processing relocation at " + << format("0x%016" PRIx64, FixupAddress) << "\n"; + }); + + // Find the atom that the fixup points to. + DefinedAtom *AtomToFix = nullptr; + { + auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); + if (!AtomToFixOrErr) + return AtomToFixOrErr.takeError(); + AtomToFix = &*AtomToFixOrErr; + } + + if (FixupAddress + (1 << RI.r_length) > + AtomToFix->getAddress() + AtomToFix->getContent().size()) + return make_error<JITLinkError>( + "Relocation content extends past end of fixup atom"); + + // Get a pointer to the fixup content. + const char *FixupContent = AtomToFix->getContent().data() + + (FixupAddress - AtomToFix->getAddress()); + + // The target atom and addend will be populated by the switch below. + Atom *TargetAtom = nullptr; + uint64_t Addend = 0; + + switch (*Kind) { + case Branch32: + case PCRel32: + case PCRel32GOTLoad: + case PCRel32GOT: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent; + break; + case Pointer64: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle64_t *)FixupContent; + break; + case Pointer64Anon: { + JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: + if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = *(const ulittle32_t *)FixupContent + + (1 << (*Kind - PCRel32Minus1)); + break; + case PCRel32Anon: { + JITTargetAddress TargetAddress = + FixupAddress + 4 + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + JITTargetAddress Delta = 1 << (*Kind - PCRel32Minus1Anon); + JITTargetAddress TargetAddress = + FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; + if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) + TargetAtom = &*TargetAtomOrErr; + else + return TargetAtomOrErr.takeError(); + Addend = TargetAddress - TargetAtom->getAddress(); + break; + } + case Delta32: + case Delta64: { + // We use Delta32/Delta64 to represent SUBTRACTOR relocations. + // parsePairRelocation handles the paired reloc, and returns the + // edge kind to be used (either Delta32/Delta64, or + // NegDelta32/NegDelta64, depending on the direction of the + // subtraction) along with the addend. + auto PairInfo = + parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + FixupContent, ++RelItr, RelEnd); + if (!PairInfo) + return PairInfo.takeError(); + std::tie(*Kind, TargetAtom, Addend) = *PairInfo; + assert(TargetAtom && "No target atom from parsePairRelocation?"); + break; + } + default: + llvm_unreachable("Special relocation kind should not appear in " + "mach-o file"); + } + + LLVM_DEBUG({ + Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom, + Addend); + printEdge(dbgs(), *AtomToFix, GE, + getMachOX86RelocationKindName(*Kind)); + dbgs() << "\n"; + }); + AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), + *TargetAtom, Addend); + } + } + return Error::success(); + } + + unsigned NumSymbols = 0; +}; + +class MachOInPlaceGOTAndStubsBuilder { +public: + MachOInPlaceGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + + void run() { + // We're going to be adding new atoms, but we don't want to iterate over + // the newly added ones, so just copy the existing atoms out. + std::vector<DefinedAtom *> DAs(G.defined_atoms().begin(), + G.defined_atoms().end()); + + for (auto *DA : DAs) + for (auto &E : DA->edges()) + if (E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) + fixGOTEdge(E); + else if (E.getKind() == Branch32 && !E.getTarget().isDefined()) + fixExternalBranchEdge(E); + } + + Atom &getGOTEntryAtom(Atom &Target) { + assert(!Target.getName().empty() && + "GOT load edge cannot point to anonymous target"); + + auto GOTEntryI = GOTEntries.find(Target.getName()); + + // Build the entry if it doesn't exist. + if (GOTEntryI == GOTEntries.end()) { + // Build a GOT section if we don't have one already. + if (!GOTSection) + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ, false); + + auto &GOTEntryAtom = G.addAnonymousAtom(*GOTSection, 0x0, 8); + GOTEntryAtom.setContent( + StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), 8)); + GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); + GOTEntryI = + GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntryAtom)) + .first; + } + + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + return *GOTEntryI->second; + } + + void fixGOTEdge(Edge &E) { + assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && + "Not a GOT edge?"); + auto &GOTEntryAtom = getGOTEntryAtom(E.getTarget()); + E.setKind(PCRel32); + E.setTarget(GOTEntryAtom); + // Leave the edge addend as-is. + } + + Atom &getStubAtom(Atom &Target) { + assert(!Target.getName().empty() && + "Branch edge can not point to an anonymous target"); + auto StubI = Stubs.find(Target.getName()); + + if (StubI == Stubs.end()) { + // Build a Stubs section if we don't have one already. + if (!StubsSection) { + auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_EXEC); + StubsSection = &G.createSection("$__STUBS", StubsProt, false); + } + + auto &StubAtom = G.addAnonymousAtom(*StubsSection, 0x0, 2); + StubAtom.setContent( + StringRef(reinterpret_cast<const char *>(StubContent), 6)); + + // Re-use GOT entries for stub targets. + auto &GOTEntryAtom = getGOTEntryAtom(Target); + StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); + + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + } + + assert(StubI != Stubs.end() && "Count not get stub atom"); + return *StubI->second; + } + + void fixExternalBranchEdge(Edge &E) { + assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); + assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); + E.setTarget(getStubAtom(E.getTarget())); + } + + AtomGraph &G; + DenseMap<StringRef, DefinedAtom *> GOTEntries; + DenseMap<StringRef, DefinedAtom *> Stubs; + static const uint8_t NullGOTEntryContent[8]; + static const uint8_t StubContent[6]; + Section *GOTSection = nullptr; + Section *StubsSection = nullptr; +}; + +const uint8_t MachOInPlaceGOTAndStubsBuilder::NullGOTEntryContent[8] = { + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00}; +const uint8_t MachOInPlaceGOTAndStubsBuilder::StubContent[6] = { + 0xFF, 0x25, 0x00, 0x00, 0x00, 0x00}; +} // namespace + +namespace llvm { +namespace jitlink { + +class MachOJITLinker_x86_64 : public JITLinker<MachOJITLinker_x86_64> { + friend class JITLinker<MachOJITLinker_x86_64>; + +public: + MachOJITLinker_x86_64(std::unique_ptr<JITLinkContext> Ctx, + PassConfiguration PassConfig) + : JITLinker(std::move(Ctx), std::move(PassConfig)) {} + +private: + StringRef getEdgeKindName(Edge::Kind R) const override { + return getMachOX86RelocationKindName(R); + } + + Expected<std::unique_ptr<AtomGraph>> + buildGraph(MemoryBufferRef ObjBuffer) override { + auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); + if (!MachOObj) + return MachOObj.takeError(); + return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + } + + static Error targetOutOfRangeError(const Edge &E) { + std::string ErrMsg; + { + raw_string_ostream ErrStream(ErrMsg); + ErrStream << "Target \"" << E.getTarget() << "\" out of range"; + } + return make_error<JITLinkError>(std::move(ErrMsg)); + } + + Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + using namespace support; + + char *FixupPtr = AtomWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + + switch (E.getKind()) { + case Branch32: + case PCRel32: + case PCRel32Anon: { + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + break; + } + case Pointer64: + case Pointer64Anon: { + uint64_t Value = E.getTarget().getAddress() + E.getAddend(); + *(ulittle64_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1: + case PCRel32Minus2: + case PCRel32Minus4: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + break; + } + case PCRel32Minus1Anon: + case PCRel32Minus2Anon: + case PCRel32Minus4Anon: { + int Delta = 4 + (1 << (E.getKind() - PCRel32Minus1Anon)); + int64_t Value = + E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + break; + } + case Delta32: + case Delta64: + case NegDelta32: + case NegDelta64: { + int64_t Value; + if (E.getKind() == Delta32 || E.getKind() == Delta64) + Value = E.getTarget().getAddress() - FixupAddress + E.getAddend(); + else + Value = FixupAddress - E.getTarget().getAddress() + E.getAddend(); + + if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { + if (Value < std::numeric_limits<int32_t>::min() || + Value > std::numeric_limits<int32_t>::max()) + return targetOutOfRangeError(E); + *(little32_t *)FixupPtr = Value; + } else + *(little64_t *)FixupPtr = Value; + break; + } + default: + llvm_unreachable("Unrecognized edge kind"); + } + + return Error::success(); + } + + uint64_t NullValue = 0; +}; + +void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { + PassConfiguration Config; + Triple TT("x86_64-apple-macosx"); + + if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add a mark-live pass. + if (auto MarkLive = Ctx->getMarkLivePass(TT)) + Config.PrePrunePasses.push_back(std::move(MarkLive)); + else + Config.PrePrunePasses.push_back(markAllAtomsLive); + + // Add an in-place GOT/Stubs pass. + Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error { + MachOInPlaceGOTAndStubsBuilder(G).run(); + return Error::success(); + }); + } + + if (auto Err = Ctx->modifyPassConfig(TT, Config)) + return Ctx->notifyFailed(std::move(Err)); + + // Construct a JITLinker and run the link function. + MachOJITLinker_x86_64::link(std::move(Ctx), std::move(Config)); +} + +StringRef getMachOX86RelocationKindName(Edge::Kind R) { + switch (R) { + case Branch32: + return "Branch32"; + case Pointer64: + return "Pointer64"; + case Pointer64Anon: + return "Pointer64Anon"; + case PCRel32: + return "PCRel32"; + case PCRel32Minus1: + return "PCRel32Minus1"; + case PCRel32Minus2: + return "PCRel32Minus2"; + case PCRel32Minus4: + return "PCRel32Minus4"; + case PCRel32Anon: + return "PCRel32Anon"; + case PCRel32Minus1Anon: + return "PCRel32Minus1Anon"; + case PCRel32Minus2Anon: + return "PCRel32Minus2Anon"; + case PCRel32Minus4Anon: + return "PCRel32Minus4Anon"; + case PCRel32GOTLoad: + return "PCRel32GOTLoad"; + case PCRel32GOT: + return "PCRel32GOT"; + case PCRel32TLV: + return "PCRel32TLV"; + case Delta32: + return "Delta32"; + case Delta64: + return "Delta64"; + case NegDelta32: + return "NegDelta32"; + case NegDelta64: + return "NegDelta64"; + default: + return getGenericEdgeKindName(static_cast<Edge::Kind>(R)); + } +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/LLVMBuild.txt b/llvm/lib/ExecutionEngine/JITLink/LLVMBuild.txt new file mode 100644 index 00000000000..d342bf96427 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/LLVMBuild.txt @@ -0,0 +1,21 @@ +;===----- ./lib/ExecutionEngine/JTILink/LLVMBuild.txt ----------*- Conf -*--===; +; +; Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +; See https://llvm.org/LICENSE.txt for license information. +; SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +; +;===------------------------------------------------------------------------===; +; +; This is an LLVMBuild description file for the components in this subdirectory. +; +; For more information on the LLVMBuild system, please see: +; +; http://llvm.org/docs/LLVMBuild.html +; +;===------------------------------------------------------------------------===; + +[component_0] +type = Library +name = JITLink +parent = ExecutionEngine +required_libraries = Object Support diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp new file mode 100644 index 00000000000..670fc3ecf7a --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp @@ -0,0 +1,289 @@ +//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOAtomGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +namespace llvm { +namespace jitlink { + +MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} + +Expected<std::unique_ptr<AtomGraph>> MachOAtomGraphBuilder::buildGraph() { + if (auto Err = parseSections()) + return std::move(Err); + + if (auto Err = addAtoms()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(llvm::make_unique<AtomGraph>(Obj.getFileName(), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, + CustomAtomizeFunction Atomizer) { + assert(!CustomAtomizeFunctions.count(SectionName) && + "Custom atomizer for this section already exists"); + CustomAtomizeFunctions[SectionName] = std::move(Atomizer); +} + +unsigned +MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() { + if (!CommonSymbolsSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + auto &GenericSection = G->createSection("<common>", Prot, true); + CommonSymbolsSection = MachOSection(GenericSection); + } + return *CommonSymbolsSection; +} + +Error MachOAtomGraphBuilder::parseSections() { + for (auto &SecRef : Obj.sections()) { + assert((SecRef.getAlignment() <= std::numeric_limits<uint32_t>::max()) && + "Section alignment does not fit in 32 bits"); + + StringRef Name; + if (auto EC = SecRef.getName(Name)) + return errorCodeToError(EC); + + StringRef Content; + + // If this is a virtual section, leave its content empty. + if (!SecRef.isVirtual()) { + if (auto EC = SecRef.getContents(Content)) + return errorCodeToError(EC); + if (Content.size() != SecRef.getSize()) + return make_error<JITLinkError>("Section content size does not match " + "declared size for " + + Name); + } + + unsigned SectionIndex = SecRef.getIndex() + 1; + + LLVM_DEBUG({ + dbgs() << "Adding section " << Name << ": " + << format("0x%016" PRIx64, SecRef.getAddress()) + << ", size: " << Content.size() + << ", align: " << SecRef.getAlignment() << "\n"; + }); + + // FIXME: Get real section permissions + // How, exactly, on MachO? + sys::Memory::ProtectionFlags Prot; + if (SecRef.isText()) + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + auto &GenericSection = G->createSection(Name, Prot, SecRef.isBSS()); + if (SecRef.isVirtual()) + Sections[SectionIndex] = + MachOSection(GenericSection, SecRef.getAddress(), + SecRef.getAlignment(), SecRef.getSize()); + Sections[SectionIndex] = MachOSection(GenericSection, SecRef.getAddress(), + SecRef.getAlignment(), Content); + } + + return Error::success(); +} + +// Adds atoms with identified start addresses (but not lengths) for all named +// atoms. +// Also, for every section that contains named atoms, but does not have an +// atom at offset zero of that section, constructs an anonymous atom covering +// that range. +Error MachOAtomGraphBuilder::addNonCustomAtoms() { + using AddrToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; + DenseMap<MachOSection *, AddrToAtomMap> SecToAtoms; + + DenseMap<MachOSection *, unsigned> FirstOrdinal; + + for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; + ++SymI) { + object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); + + auto Name = Sym.getName(); + if (!Name) + return Name.takeError(); + + auto Addr = Sym.getAddress(); + if (!Addr) + return Addr.takeError(); + + auto SymType = Sym.getType(); + if (!SymType) + return SymType.takeError(); + + auto Flags = Sym.getFlags(); + + if (Flags & object::SymbolRef::SF_Undefined) { + LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); + G->addExternalAtom(*Name); + continue; + } else if (Flags & object::SymbolRef::SF_Absolute) { + LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " + << format("0x%016" PRIx64, *Addr) << "\n"); + auto &A = G->addAbsoluteAtom(*Name, *Addr); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + A.setWeak(Flags & object::SymbolRef::SF_Weak); + continue; + } else if (Flags & object::SymbolRef::SF_Common) { + LLVM_DEBUG({ + dbgs() << "Adding common \"" << *Name + << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n"; + }); + auto &A = + G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U), + Obj.getCommonSymbolSize(Sym.getRawDataRefImpl())); + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + continue; + } + + LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); + + // This atom is neither undefined nor absolute, so it must be defined in + // this object. Get its section index. + auto SecItr = Sym.getSection(); + if (!SecItr) + return SecItr.takeError(); + + uint64_t SectionIndex = (*SecItr)->getIndex() + 1; + + LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n"); + + auto SecByIndexItr = Sections.find(SectionIndex); + if (SecByIndexItr == Sections.end()) + return make_error<JITLinkError>("Unrecognized section index in macho"); + + auto &Sec = SecByIndexItr->second; + + auto &A = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, + std::max(Sym.getAlignment(), 1U)); + + A.setGlobal(Flags & object::SymbolRef::SF_Global); + A.setExported(Flags & object::SymbolRef::SF_Exported); + A.setWeak(Flags & object::SymbolRef::SF_Weak); + + A.setCallable(*SymType & object::SymbolRef::ST_Function); + + LLVM_DEBUG({ + dbgs() << " Added " << *Name + << " addr: " << format("0x%016" PRIx64, *Addr) + << ", align: " << A.getAlignment() + << ", section: " << Sec.getGenericSection().getName() << "\n"; + }); + + auto &SecAtoms = SecToAtoms[&Sec]; + SecAtoms[A.getAddress() - Sec.getAddress()] = &A; + } + + // Add anonymous atoms. + for (auto &KV : Sections) { + auto &S = KV.second; + + // Skip empty sections. + if (S.empty()) + continue; + + // Skip sections with custom handling. + if (CustomAtomizeFunctions.count(S.getName())) + continue; + + auto SAI = SecToAtoms.find(&S); + + // If S is not in the SecToAtoms map then it contained no named atom. Add + // one anonymous atom to cover the whole section. + if (SAI == SecToAtoms.end()) { + SecToAtoms[&S][0] = &G->addAnonymousAtom( + S.getGenericSection(), S.getAddress(), S.getAlignment()); + continue; + } + + // Otherwise, check whether this section had an atom covering offset zero. + // If not, add one. + auto &SecAtoms = SAI->second; + if (!SecAtoms.count(0)) + SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), + S.getAlignment()); + } + + LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); + + // Set atom contents. + for (auto &KV : SecToAtoms) { + auto &S = *KV.first; + auto &SecAtoms = KV.second; + + // Iterate the atoms in reverse order and set up their contents. + JITTargetAddress LastAtomAddr = S.getSize(); + for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { + auto Offset = I->first; + auto &A = *I->second; + LLVM_DEBUG({ + dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. " + << S.getAddress() + LastAtomAddr << " ]\n"; + }); + if (S.isZeroFill()) + A.setZeroFill(LastAtomAddr - Offset); + else + A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); + LastAtomAddr = Offset; + } + } + + return Error::success(); +} + +Error MachOAtomGraphBuilder::addAtoms() { + // Add all named atoms. + if (auto Err = addNonCustomAtoms()) + return Err; + + // Process special sections. + for (auto &KV : Sections) { + auto &S = KV.second; + auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); + if (HI != CustomAtomizeFunctions.end()) { + auto &Atomize = HI->second; + if (auto Err = Atomize(S)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h new file mode 100644 index 00000000000..340a11dfc0e --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h @@ -0,0 +1,119 @@ +//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO AtomGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "JITLinkGeneric.h" + +#include "llvm/Object/MachO.h" + +namespace llvm { +namespace jitlink { + +class MachOAtomGraphBuilder { +public: + virtual ~MachOAtomGraphBuilder(); + Expected<std::unique_ptr<AtomGraph>> buildGraph(); + +protected: + using OffsetToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; + + class MachOSection { + public: + MachOSection() = default; + + /// Create a MachO section with the given content. + MachOSection(Section &GenericSection, JITTargetAddress Address, + unsigned Alignment, StringRef Content) + : Address(Address), GenericSection(&GenericSection), + ContentPtr(Content.data()), Size(Content.size()), + Alignment(Alignment) {} + + /// Create a zero-fill MachO section with the given size. + MachOSection(Section &GenericSection, JITTargetAddress Address, + unsigned Alignment, size_t ZeroFillSize) + : Address(Address), GenericSection(&GenericSection), Size(ZeroFillSize), + Alignment(Alignment) {} + + /// Create a section without address, content or size (used for common + /// symbol sections). + MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {} + + Section &getGenericSection() const { + assert(GenericSection && "Section is null"); + return *GenericSection; + } + + StringRef getName() const { + assert(GenericSection && "No generic section attached"); + return GenericSection->getName(); + } + + bool isZeroFill() const { return !ContentPtr; } + + bool empty() const { return getSize() == 0; } + + size_t getSize() const { return Size; } + + StringRef getContent() const { + assert(ContentPtr && "getContent() called on zero-fill section"); + return {ContentPtr, Size}; + } + + JITTargetAddress getAddress() const { return Address; } + + unsigned getAlignment() const { return Alignment; } + + private: + JITTargetAddress Address = 0; + Section *GenericSection = nullptr; + const char *ContentPtr = nullptr; + size_t Size = 0; + unsigned Alignment = 0; + }; + + using CustomAtomizeFunction = std::function<Error(MachOSection &S)>; + + MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); + + AtomGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); + + virtual Error addRelocations() = 0; + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + MachOSection &getCommonSection(); + + Error parseSections(); + Error addNonCustomAtoms(); + Error addAtoms(); + + const object::MachOObjectFile &Obj; + std::unique_ptr<AtomGraph> G; + DenseMap<unsigned, MachOSection> Sections; + StringMap<CustomAtomizeFunction> CustomAtomizeFunctions; + Optional<MachOSection> CommonSymbolsSection; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H diff --git a/llvm/lib/ExecutionEngine/LLVMBuild.txt b/llvm/lib/ExecutionEngine/LLVMBuild.txt index 4a568e8cf04..13872da4a56 100644 --- a/llvm/lib/ExecutionEngine/LLVMBuild.txt +++ b/llvm/lib/ExecutionEngine/LLVMBuild.txt @@ -15,7 +15,8 @@ ;===------------------------------------------------------------------------===; [common] -subdirectories = Interpreter MCJIT RuntimeDyld IntelJITEvents OProfileJIT Orc PerfJITEvents +subdirectories = Interpreter MCJIT JITLink RuntimeDyld IntelJITEvents + OProfileJIT Orc PerfJITEvents [component_0] type = Library diff --git a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt index 9ca409f81cd..8fcf77c7a8a 100644 --- a/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/Orc/CMakeLists.txt @@ -11,6 +11,7 @@ add_llvm_library(LLVMOrcJIT Layer.cpp LLJIT.cpp NullResolver.cpp + ObjectLinkingLayer.cpp ObjectTransformLayer.cpp OrcABISupport.cpp OrcCBindings.cpp diff --git a/llvm/lib/ExecutionEngine/Orc/Core.cpp b/llvm/lib/ExecutionEngine/Orc/Core.cpp index 99504585d5b..cc52b564bec 100644 --- a/llvm/lib/ExecutionEngine/Orc/Core.cpp +++ b/llvm/lib/ExecutionEngine/Orc/Core.cpp @@ -26,17 +26,17 @@ namespace { #ifndef NDEBUG -cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(false), +cl::opt<bool> PrintHidden("debug-orc-print-hidden", cl::init(true), cl::desc("debug print hidden symbols defined by " "materialization units"), cl::Hidden); -cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(false), +cl::opt<bool> PrintCallable("debug-orc-print-callable", cl::init(true), cl::desc("debug print callable symbols defined by " "materialization units"), cl::Hidden); -cl::opt<bool> PrintData("debug-orc-print-data", cl::init(false), +cl::opt<bool> PrintData("debug-orc-print-data", cl::init(true), cl::desc("debug print data symbols defined by " "materialization units"), cl::Hidden); @@ -1051,9 +1051,11 @@ void JITDylib::notifyFailed(const SymbolNameSet &FailedSymbols) { void JITDylib::setSearchOrder(JITDylibSearchList NewSearchOrder, bool SearchThisJITDylibFirst, bool MatchNonExportedInThisDylib) { - if (SearchThisJITDylibFirst && NewSearchOrder.front().first != this) - NewSearchOrder.insert(NewSearchOrder.begin(), - {this, MatchNonExportedInThisDylib}); + if (SearchThisJITDylibFirst) { + if (NewSearchOrder.empty() || NewSearchOrder.front().first != this) + NewSearchOrder.insert(NewSearchOrder.begin(), + {this, MatchNonExportedInThisDylib}); + } ES.runSessionLocked([&]() { SearchOrder = std::move(NewSearchOrder); }); } @@ -1450,77 +1452,55 @@ JITDylib::JITDylib(ExecutionSession &ES, std::string Name) Error JITDylib::defineImpl(MaterializationUnit &MU) { SymbolNameSet Duplicates; - SymbolNameSet MUDefsOverridden; - - struct ExistingDefOverriddenEntry { - SymbolMap::iterator ExistingDefItr; - JITSymbolFlags NewFlags; - }; - std::vector<ExistingDefOverriddenEntry> ExistingDefsOverridden; + std::vector<SymbolStringPtr> ExistingDefsOverridden; + std::vector<SymbolStringPtr> MUDefsOverridden; - for (auto &KV : MU.getSymbols()) { + for (const auto &KV : MU.getSymbols()) { assert(!KV.second.isLazy() && "Lazy flag should be managed internally."); assert(!KV.second.isMaterializing() && "Materializing flags should be managed internally."); - SymbolMap::iterator EntryItr; - bool Added; - - auto NewFlags = KV.second; - NewFlags |= JITSymbolFlags::Lazy; - - std::tie(EntryItr, Added) = Symbols.insert( - std::make_pair(KV.first, JITEvaluatedSymbol(0, NewFlags))); + auto I = Symbols.find(KV.first); - if (!Added) { + if (I != Symbols.end()) { if (KV.second.isStrong()) { - if (EntryItr->second.getFlags().isStrong() || - (EntryItr->second.getFlags() & JITSymbolFlags::Materializing)) + if (I->second.getFlags().isStrong() || + I->second.getFlags().isMaterializing()) Duplicates.insert(KV.first); - else - ExistingDefsOverridden.push_back({EntryItr, NewFlags}); + else { + assert(I->second.getFlags().isLazy() && + !I->second.getFlags().isMaterializing() && + "Overridden existing def should be in the Lazy state"); + ExistingDefsOverridden.push_back(KV.first); + } } else - MUDefsOverridden.insert(KV.first); + MUDefsOverridden.push_back(KV.first); } } - if (!Duplicates.empty()) { - // We need to remove the symbols we added. - for (auto &KV : MU.getSymbols()) { - if (Duplicates.count(KV.first)) - continue; - - bool Found = false; - for (const auto &EDO : ExistingDefsOverridden) - if (EDO.ExistingDefItr->first == KV.first) - Found = true; - - if (!Found) - Symbols.erase(KV.first); - } - - // FIXME: Return all duplicates. + // If there were any duplicate definitions then bail out. + if (!Duplicates.empty()) return make_error<DuplicateDefinition>(**Duplicates.begin()); - } - // Update flags on existing defs and call discard on their materializers. - for (auto &EDO : ExistingDefsOverridden) { - assert(EDO.ExistingDefItr->second.getFlags().isLazy() && - !EDO.ExistingDefItr->second.getFlags().isMaterializing() && - "Overridden existing def should be in the Lazy state"); + // Discard any overridden defs in this MU. + for (auto &S : MUDefsOverridden) + MU.doDiscard(*this, S); - EDO.ExistingDefItr->second.setFlags(EDO.NewFlags); + // Discard existing overridden defs. + for (auto &S : ExistingDefsOverridden) { - auto UMII = UnmaterializedInfos.find(EDO.ExistingDefItr->first); + auto UMII = UnmaterializedInfos.find(S); assert(UMII != UnmaterializedInfos.end() && "Overridden existing def should have an UnmaterializedInfo"); - - UMII->second->MU->doDiscard(*this, EDO.ExistingDefItr->first); + UMII->second->MU->doDiscard(*this, S); } - // Discard overridden symbols povided by MU. - for (auto &Sym : MUDefsOverridden) - MU.doDiscard(*this, Sym); + // Finally, add the defs from this MU. + for (auto &KV : MU.getSymbols()) { + auto NewFlags = KV.second; + NewFlags |= JITSymbolFlags::Lazy; + Symbols[KV.first] = JITEvaluatedSymbol(0, NewFlags); + } return Error::success(); } diff --git a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp index 0ef85fd6857..195e9396671 100644 --- a/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp +++ b/llvm/lib/ExecutionEngine/Orc/ExecutionUtils.cpp @@ -178,18 +178,19 @@ Error LocalCXXRuntimeOverrides::enable(JITDylib &JD, } DynamicLibrarySearchGenerator::DynamicLibrarySearchGenerator( - sys::DynamicLibrary Dylib, const DataLayout &DL, SymbolPredicate Allow) + sys::DynamicLibrary Dylib, char GlobalPrefix, SymbolPredicate Allow) : Dylib(std::move(Dylib)), Allow(std::move(Allow)), - GlobalPrefix(DL.getGlobalPrefix()) {} + GlobalPrefix(GlobalPrefix) {} Expected<DynamicLibrarySearchGenerator> -DynamicLibrarySearchGenerator::Load(const char *FileName, const DataLayout &DL, +DynamicLibrarySearchGenerator::Load(const char *FileName, char GlobalPrefix, SymbolPredicate Allow) { std::string ErrMsg; auto Lib = sys::DynamicLibrary::getPermanentLibrary(FileName, &ErrMsg); if (!Lib.isValid()) return make_error<StringError>(std::move(ErrMsg), inconvertibleErrorCode()); - return DynamicLibrarySearchGenerator(std::move(Lib), DL, std::move(Allow)); + return DynamicLibrarySearchGenerator(std::move(Lib), GlobalPrefix, + std::move(Allow)); } SymbolNameSet DynamicLibrarySearchGenerator:: @@ -209,7 +210,8 @@ operator()(JITDylib &JD, const SymbolNameSet &Names) { if (HasGlobalPrefix && (*Name).front() != GlobalPrefix) continue; - std::string Tmp((*Name).data() + (HasGlobalPrefix ? 1 : 0), (*Name).size()); + std::string Tmp((*Name).data() + HasGlobalPrefix, + (*Name).size() - HasGlobalPrefix); if (void *Addr = Dylib.getAddressOfSymbol(Tmp.c_str())) { Added.insert(Name); NewSymbols[Name] = JITEvaluatedSymbol( diff --git a/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp new file mode 100644 index 00000000000..7b85f654bde --- /dev/null +++ b/llvm/lib/ExecutionEngine/Orc/ObjectLinkingLayer.cpp @@ -0,0 +1,382 @@ +//===------- ObjectLinkingLayer.cpp - JITLink backed ORC ObjectLayer ------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// + +#include "llvm/ExecutionEngine/Orc/ObjectLinkingLayer.h" + +#include "llvm/ADT/Optional.h" +#include "llvm/ExecutionEngine/JITLink/JITLink_EHFrameSupport.h" + +#include <vector> + +#define DEBUG_TYPE "orc" + +using namespace llvm; +using namespace llvm::jitlink; +using namespace llvm::orc; + +namespace llvm { +namespace orc { + +class ObjectLinkingLayerJITLinkContext final : public JITLinkContext { +public: + ObjectLinkingLayerJITLinkContext(ObjectLinkingLayer &Layer, + MaterializationResponsibility MR, + std::unique_ptr<MemoryBuffer> ObjBuffer) + : Layer(Layer), MR(std::move(MR)), ObjBuffer(std::move(ObjBuffer)) {} + + JITLinkMemoryManager &getMemoryManager() override { return Layer.MemMgr; } + + MemoryBufferRef getObjectBuffer() const override { + return ObjBuffer->getMemBufferRef(); + } + + void notifyFailed(Error Err) override { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + } + + void lookup(const DenseSet<StringRef> &Symbols, + JITLinkAsyncLookupContinuation LookupContinuation) override { + + JITDylibSearchList SearchOrder; + MR.getTargetJITDylib().withSearchOrderDo( + [&](const JITDylibSearchList &JDs) { SearchOrder = JDs; }); + + auto &ES = Layer.getExecutionSession(); + + SymbolNameSet InternedSymbols; + for (auto &S : Symbols) + InternedSymbols.insert(ES.intern(S)); + + // OnResolve -- De-intern the symbols and pass the result to the linker. + // FIXME: Capture LookupContinuation by move once we have c++14. + auto SharedLookupContinuation = + std::make_shared<JITLinkAsyncLookupContinuation>( + std::move(LookupContinuation)); + auto OnResolve = [SharedLookupContinuation](Expected<SymbolMap> Result) { + if (!Result) + (*SharedLookupContinuation)(Result.takeError()); + else { + AsyncLookupResult LR; + for (auto &KV : *Result) + LR[*KV.first] = KV.second; + (*SharedLookupContinuation)(std::move(LR)); + } + }; + + ES.lookup( + SearchOrder, std::move(InternedSymbols), std::move(OnResolve), + // OnReady: + [&ES](Error Err) { ES.reportError(std::move(Err)); }, + // RegisterDependencies: + [this](const SymbolDependenceMap &Deps) { + registerDependencies(Deps); + }); + } + + void notifyResolved(AtomGraph &G) override { + auto &ES = Layer.getExecutionSession(); + + SymbolFlagsMap ExtraSymbolsToClaim; + bool AutoClaim = Layer.AutoClaimObjectSymbols; + + SymbolMap InternedResult; + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && DA->isGlobal()) { + auto InternedName = ES.intern(DA->getName()); + JITSymbolFlags Flags; + + if (DA->isExported()) + Flags |= JITSymbolFlags::Exported; + if (DA->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (DA->isCallable()) + Flags |= JITSymbolFlags::Callable; + if (DA->isCommon()) + Flags |= JITSymbolFlags::Common; + + InternedResult[InternedName] = + JITEvaluatedSymbol(DA->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + for (auto *A : G.absolute_atoms()) + if (A->hasName()) { + auto InternedName = ES.intern(A->getName()); + JITSymbolFlags Flags; + Flags |= JITSymbolFlags::Absolute; + if (A->isWeak()) + Flags |= JITSymbolFlags::Weak; + if (A->isCallable()) + Flags |= JITSymbolFlags::Callable; + InternedResult[InternedName] = + JITEvaluatedSymbol(A->getAddress(), Flags); + if (AutoClaim && !MR.getSymbols().count(InternedName)) { + assert(!ExtraSymbolsToClaim.count(InternedName) && + "Duplicate symbol to claim?"); + ExtraSymbolsToClaim[InternedName] = Flags; + } + } + + if (!ExtraSymbolsToClaim.empty()) + if (auto Err = MR.defineMaterializing(ExtraSymbolsToClaim)) + return notifyFailed(std::move(Err)); + + MR.resolve(InternedResult); + + if (Layer.NotifyLoaded) + Layer.NotifyLoaded(MR.getVModuleKey()); + } + + void notifyFinalized( + std::unique_ptr<JITLinkMemoryManager::Allocation> A) override { + + if (EHFrameAddr) { + // If there is an eh-frame then try to register it. + if (auto Err = registerEHFrameSection((void *)EHFrameAddr)) { + Layer.getExecutionSession().reportError(std::move(Err)); + MR.failMaterialization(); + return; + } + } + + MR.emit(); + Layer.notifyFinalized( + ObjectLinkingLayer::ObjectResources(std::move(A), EHFrameAddr)); + } + + AtomGraphPassFunction getMarkLivePass(const Triple &TT) const override { + return [this](AtomGraph &G) { return markResponsibilitySymbolsLive(G); }; + } + + Error modifyPassConfig(const Triple &TT, PassConfiguration &Config) override { + // Add passes to mark duplicate defs as should-discard, and to walk the + // atom graph to build the symbol dependence graph. + Config.PrePrunePasses.push_back( + [this](AtomGraph &G) { return markSymbolsToDiscard(G); }); + Config.PostPrunePasses.push_back( + [this](AtomGraph &G) { return computeNamedSymbolDependencies(G); }); + + Config.PostFixupPasses.push_back( + createEHFrameRecorderPass(TT, EHFrameAddr)); + + if (Layer.ModifyPassConfig) + Layer.ModifyPassConfig(TT, Config); + + return Error::success(); + } + +private: + using AnonAtomNamedDependenciesMap = + DenseMap<const DefinedAtom *, SymbolNameSet>; + + Error markSymbolsToDiscard(AtomGraph &G) { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->isWeak() && DA->hasName()) { + auto S = ES.intern(DA->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + DA->setShouldDiscard(true); + } + + for (auto *A : G.absolute_atoms()) + if (A->isWeak() && A->hasName()) { + auto S = ES.intern(A->getName()); + auto I = MR.getSymbols().find(S); + if (I == MR.getSymbols().end()) + A->setShouldDiscard(true); + } + + return Error::success(); + } + + Error markResponsibilitySymbolsLive(AtomGraph &G) const { + auto &ES = Layer.getExecutionSession(); + for (auto *DA : G.defined_atoms()) + if (DA->hasName() && + MR.getSymbols().count(ES.intern(DA->getName()))) + DA->setLive(true); + return Error::success(); + } + + Error computeNamedSymbolDependencies(AtomGraph &G) { + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + auto AnonDeps = computeAnonDeps(G); + + for (auto *DA : G.defined_atoms()) { + + // Skip anonymous and non-global atoms: we do not need dependencies for + // these. + if (!DA->hasName() || !DA->isGlobal()) + continue; + + auto DAName = ES.intern(DA->getName()); + SymbolNameSet &DADeps = NamedSymbolDeps[DAName]; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + + if (TA.hasName()) + DADeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + auto &DTA = static_cast<DefinedAtom &>(TA); + auto I = AnonDeps.find(&DTA); + if (I != AnonDeps.end()) + for (auto &S : I->second) + DADeps.insert(S); + } + } + } + + return Error::success(); + } + + AnonAtomNamedDependenciesMap computeAnonDeps(AtomGraph &G) { + + auto &ES = MR.getTargetJITDylib().getExecutionSession(); + AnonAtomNamedDependenciesMap DepMap; + + // For all anonymous atoms: + // (1) Add their named dependencies. + // (2) Add them to the worklist for further iteration if they have any + // depend on any other anonymous atoms. + struct WorklistEntry { + WorklistEntry(DefinedAtom *DA, DenseSet<DefinedAtom *> DAAnonDeps) + : DA(DA), DAAnonDeps(std::move(DAAnonDeps)) {} + + DefinedAtom *DA = nullptr; + DenseSet<DefinedAtom *> DAAnonDeps; + }; + std::vector<WorklistEntry> Worklist; + for (auto *DA : G.defined_atoms()) + if (!DA->hasName()) { + auto &DANamedDeps = DepMap[DA]; + DenseSet<DefinedAtom *> DAAnonDeps; + + for (auto &E : DA->edges()) { + auto &TA = E.getTarget(); + if (TA.hasName()) + DANamedDeps.insert(ES.intern(TA.getName())); + else { + assert(TA.isDefined() && "Anonymous atoms must be defined"); + DAAnonDeps.insert(static_cast<DefinedAtom *>(&TA)); + } + } + + if (!DAAnonDeps.empty()) + Worklist.push_back(WorklistEntry(DA, std::move(DAAnonDeps))); + } + + // Loop over all anonymous atoms with anonymous dependencies, propagating + // their respective *named* dependencies. Iterate until we hit a stable + // state. + bool Changed; + do { + Changed = false; + for (auto &WLEntry : Worklist) { + auto *DA = WLEntry.DA; + auto &DANamedDeps = DepMap[DA]; + auto &DAAnonDeps = WLEntry.DAAnonDeps; + + for (auto *TA : DAAnonDeps) { + auto I = DepMap.find(TA); + if (I != DepMap.end()) + for (const auto &S : I->second) + Changed |= DANamedDeps.insert(S).second; + } + } + } while (Changed); + + return DepMap; + } + + void registerDependencies(const SymbolDependenceMap &QueryDeps) { + for (auto &NamedDepsEntry : NamedSymbolDeps) { + auto &Name = NamedDepsEntry.first; + auto &NameDeps = NamedDepsEntry.second; + SymbolDependenceMap SymbolDeps; + + for (const auto &QueryDepsEntry : QueryDeps) { + JITDylib &SourceJD = *QueryDepsEntry.first; + const SymbolNameSet &Symbols = QueryDepsEntry.second; + auto &DepsForJD = SymbolDeps[&SourceJD]; + + for (const auto &S : Symbols) + if (NameDeps.count(S)) + DepsForJD.insert(S); + + if (DepsForJD.empty()) + SymbolDeps.erase(&SourceJD); + } + + MR.addDependencies(Name, SymbolDeps); + } + } + + ObjectLinkingLayer &Layer; + MaterializationResponsibility MR; + std::unique_ptr<MemoryBuffer> ObjBuffer; + DenseMap<SymbolStringPtr, SymbolNameSet> NamedSymbolDeps; + JITTargetAddress EHFrameAddr = 0; +}; + +ObjectLinkingLayer::ObjectLinkingLayer( + ExecutionSession &ES, JITLinkMemoryManager &MemMgr, + NotifyLoadedFunction NotifyLoaded, NotifyEmittedFunction NotifyEmitted, + ModifyPassConfigFunction ModifyPassConfig) + : ObjectLayer(ES), MemMgr(MemMgr), NotifyLoaded(std::move(NotifyLoaded)), + NotifyEmitted(std::move(NotifyEmitted)), + ModifyPassConfig(std::move(ModifyPassConfig)) {} + +void ObjectLinkingLayer::emit(MaterializationResponsibility R, + std::unique_ptr<MemoryBuffer> O) { + assert(O && "Object must not be null"); + jitLink(llvm::make_unique<ObjectLinkingLayerJITLinkContext>( + *this, std::move(R), std::move(O))); +} + +ObjectLinkingLayer::ObjectResources::ObjectResources( + AllocPtr Alloc, JITTargetAddress EHFrameAddr) + : Alloc(std::move(Alloc)), EHFrameAddr(EHFrameAddr) {} + +ObjectLinkingLayer::ObjectResources::ObjectResources(ObjectResources &&Other) + : Alloc(std::move(Other.Alloc)), EHFrameAddr(Other.EHFrameAddr) { + Other.EHFrameAddr = 0; +} + +ObjectLinkingLayer::ObjectResources & +ObjectLinkingLayer::ObjectResources::operator=(ObjectResources &&Other) { + std::swap(Alloc, Other.Alloc); + std::swap(EHFrameAddr, Other.EHFrameAddr); + return *this; +} + +ObjectLinkingLayer::ObjectResources::~ObjectResources() { + const char *ErrBanner = + "ObjectLinkingLayer received error deallocating object resources:"; + + assert((EHFrameAddr == 0 || Alloc) && + "Non-null EHFrameAddr must have an associated allocation"); + + if (EHFrameAddr) + if (auto Err = deregisterEHFrameSection((void *)EHFrameAddr)) + logAllUnhandledErrors(std::move(Err), llvm::errs(), ErrBanner); + + if (Alloc) + if (auto Err = Alloc->deallocate()) + logAllUnhandledErrors(std::move(Err), llvm::errs(), ErrBanner); +} + +} // End namespace orc. +} // End namespace llvm. diff --git a/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp b/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp index 0a19f7e6c52..0363cd5e3b2 100644 --- a/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/RuntimeDyld/RTDyldMemoryManager.cpp @@ -47,7 +47,7 @@ extern "C" void __deregister_frame(void *); // it may be found at runtime in a dynamically-loaded library. // For example, this happens when building LLVM with Visual C++ // but using the MingW runtime. -void __register_frame(void *p) { +static void __register_frame(void *p) { static bool Searched = false; static void((*rf)(void *)) = 0; @@ -60,7 +60,7 @@ void __register_frame(void *p) { rf(p); } -void __deregister_frame(void *p) { +static void __deregister_frame(void *p) { static bool Searched = false; static void((*df)(void *)) = 0; |
