diff options
Diffstat (limited to 'llvm/lib/ExecutionEngine/JITLink')
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h | 35 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt | 2 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp | 195 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h | 50 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLink.cpp | 150 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp | 372 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h | 183 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp | 17 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp | 412 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h | 138 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp | 535 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h | 269 | ||||
| -rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp | 261 |
13 files changed, 1396 insertions, 1223 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h index 1271ad962b3..b47a798c760 100644 --- a/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/BasicGOTAndStubsBuilder.h @@ -20,24 +20,23 @@ namespace jitlink { template <typename BuilderImpl> class BasicGOTAndStubsBuilder { public: - BasicGOTAndStubsBuilder(AtomGraph &G) : G(G) {} + BasicGOTAndStubsBuilder(LinkGraph &G) : G(G) {} void run() { - // We're going to be adding new atoms, but we don't want to iterate over - // the newly added ones, so just copy the existing atoms out. - std::vector<DefinedAtom *> DAs(G.defined_atoms().begin(), - G.defined_atoms().end()); + // We're going to be adding new blocks, but we don't want to iterate over + // the newly added ones, so just copy the existing blocks out. + std::vector<Block *> Blocks(G.blocks().begin(), G.blocks().end()); - for (auto *DA : DAs) - for (auto &E : DA->edges()) + for (auto *B : Blocks) + for (auto &E : B->edges()) if (impl().isGOTEdge(E)) - impl().fixGOTEdge(E, getGOTEntryAtom(E.getTarget())); + impl().fixGOTEdge(E, getGOTEntrySymbol(E.getTarget())); else if (impl().isExternalBranchEdge(E)) - impl().fixExternalBranchEdge(E, getStubAtom(E.getTarget())); + impl().fixExternalBranchEdge(E, getStubSymbol(E.getTarget())); } protected: - Atom &getGOTEntryAtom(Atom &Target) { + Symbol &getGOTEntrySymbol(Symbol &Target) { assert(Target.hasName() && "GOT edge cannot point to anonymous target"); auto GOTEntryI = GOTEntries.find(Target.getName()); @@ -49,31 +48,31 @@ protected: GOTEntries.insert(std::make_pair(Target.getName(), &GOTEntry)).first; } - assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry atom"); + assert(GOTEntryI != GOTEntries.end() && "Could not get GOT entry symbol"); return *GOTEntryI->second; } - Atom &getStubAtom(Atom &Target) { + Symbol &getStubSymbol(Symbol &Target) { assert(Target.hasName() && "External branch edge can not point to an anonymous target"); auto StubI = Stubs.find(Target.getName()); if (StubI == Stubs.end()) { - auto &StubAtom = impl().createStub(Target); - StubI = Stubs.insert(std::make_pair(Target.getName(), &StubAtom)).first; + auto &StubSymbol = impl().createStub(Target); + StubI = Stubs.insert(std::make_pair(Target.getName(), &StubSymbol)).first; } - assert(StubI != Stubs.end() && "Count not get stub atom"); + assert(StubI != Stubs.end() && "Count not get stub symbol"); return *StubI->second; } - AtomGraph &G; + LinkGraph &G; private: BuilderImpl &impl() { return static_cast<BuilderImpl &>(*this); } - DenseMap<StringRef, DefinedAtom *> GOTEntries; - DenseMap<StringRef, DefinedAtom *> Stubs; + DenseMap<StringRef, Symbol *> GOTEntries; + DenseMap<StringRef, Symbol *> Stubs; }; } // end namespace jitlink diff --git a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt index e81648311cf..ad3427fdfe3 100644 --- a/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt +++ b/llvm/lib/ExecutionEngine/JITLink/CMakeLists.txt @@ -5,7 +5,7 @@ add_llvm_library(LLVMJITLink EHFrameSupport.cpp MachO.cpp MachO_x86_64.cpp - MachOAtomGraphBuilder.cpp + MachOLinkGraphBuilder.cpp DEPENDS intrinsics_gen diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index f373f2d92b0..355a000bb98 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -17,65 +17,57 @@ namespace llvm { namespace jitlink { -EHFrameParser::EHFrameParser(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, - JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) - : G(G), EHFrameSection(EHFrameSection), EHFrameContent(EHFrameContent), - EHFrameAddress(EHFrameAddress), - EHFrameReader(EHFrameContent, G.getEndianness()), - FDEToCIERelocKind(FDEToCIERelocKind), - FDEToTargetRelocKind(FDEToTargetRelocKind) {} - -Error EHFrameParser::atomize() { +EHFrameBinaryParser::EHFrameBinaryParser(JITTargetAddress EHFrameAddress, + StringRef EHFrameContent, + unsigned PointerSize, + support::endianness Endianness) + : EHFrameAddress(EHFrameAddress), EHFrameContent(EHFrameContent), + PointerSize(PointerSize), EHFrameReader(EHFrameContent, Endianness) {} + +Error EHFrameBinaryParser::addToGraph() { while (!EHFrameReader.empty()) { size_t RecordOffset = EHFrameReader.getOffset(); + JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset; LLVM_DEBUG({ dbgs() << "Processing eh-frame record at " - << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) - << " (offset " << RecordOffset << ")\n"; + << format("0x%016" PRIx64, RecordAddress) << " (offset " + << RecordOffset << ")\n"; }); - size_t CIELength = 0; - uint32_t CIELengthField; - if (auto Err = EHFrameReader.readInteger(CIELengthField)) + size_t RecordLength = 0; + uint32_t RecordLengthField; + if (auto Err = EHFrameReader.readInteger(RecordLengthField)) return Err; - // Process CIE length/extended-length fields to build the atom. + // Process CIE/FDE length/extended-length fields to build the blocks. // // The value of these fields describe the length of the *rest* of the CIE // (not including data up to the end of the field itself) so we have to - // bump CIELength to include the data up to the end of the field: 4 bytes + // bump RecordLength to include the data up to the end of the field: 4 bytes // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. - if (CIELengthField == 0) // Length 0 means end of __eh_frame section. + if (RecordLengthField == 0) // Length 0 means end of __eh_frame section. break; // If the regular length field's value is 0xffffffff, use extended length. - if (CIELengthField == 0xffffffff) { - uint64_t CIEExtendedLengthField; - if (auto Err = EHFrameReader.readInteger(CIEExtendedLengthField)) + if (RecordLengthField == 0xffffffff) { + uint64_t ExtendedLengthField; + if (auto Err = EHFrameReader.readInteger(ExtendedLengthField)) return Err; - if (CIEExtendedLengthField > EHFrameReader.bytesRemaining()) + if (ExtendedLengthField > EHFrameReader.bytesRemaining()) return make_error<JITLinkError>("CIE record extends past the end of " "the __eh_frame section"); - if (CIEExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) + if (ExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) return make_error<JITLinkError>("CIE record too large to process"); - CIELength = CIEExtendedLengthField + 12; + RecordLength = ExtendedLengthField + 12; } else { - if (CIELengthField > EHFrameReader.bytesRemaining()) + if (RecordLengthField > EHFrameReader.bytesRemaining()) return make_error<JITLinkError>("CIE record extends past the end of " "the __eh_frame section"); - CIELength = CIELengthField + 4; + RecordLength = RecordLengthField + 4; } - LLVM_DEBUG(dbgs() << " length: " << CIELength << "\n"); - - // Add an atom for this record. - CurRecordAtom = &G.addAnonymousAtom( - EHFrameSection, EHFrameAddress + RecordOffset, G.getPointerSize()); - CurRecordAtom->setContent(EHFrameContent.substr(RecordOffset, CIELength)); + LLVM_DEBUG(dbgs() << " length: " << RecordLength << "\n"); // Read the CIE Pointer. size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); @@ -85,21 +77,24 @@ Error EHFrameParser::atomize() { // Based on the CIE pointer value, parse this as a CIE or FDE record. if (CIEPointer == 0) { - if (auto Err = processCIE()) + if (auto Err = processCIE(RecordOffset, RecordLength)) return Err; } else { - if (auto Err = processFDE(CIEPointerAddress, CIEPointer)) + if (auto Err = processFDE(RecordOffset, RecordLength, CIEPointerAddress, + CIEPointer)) return Err; } - EHFrameReader.setOffset(RecordOffset + CIELength); + EHFrameReader.setOffset(RecordOffset + RecordLength); } return Error::success(); } -Expected<EHFrameParser::AugmentationInfo> -EHFrameParser::parseAugmentationString() { +void EHFrameBinaryParser::anchor() {} + +Expected<EHFrameBinaryParser::AugmentationInfo> +EHFrameBinaryParser::parseAugmentationString() { AugmentationInfo AugInfo; uint8_t NextChar; uint8_t *NextField = &AugInfo.Fields[0]; @@ -139,14 +134,14 @@ EHFrameParser::parseAugmentationString() { return std::move(AugInfo); } -Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() { +Expected<JITTargetAddress> EHFrameBinaryParser::readAbsolutePointer() { static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), "Result must be able to hold a uint64_t"); JITTargetAddress Addr; - if (G.getPointerSize() == 8) { + if (PointerSize == 8) { if (auto Err = EHFrameReader.readInteger(Addr)) return std::move(Err); - } else if (G.getPointerSize() == 4) { + } else if (PointerSize == 4) { uint32_t Addr32; if (auto Err = EHFrameReader.readInteger(Addr32)) return std::move(Err); @@ -156,14 +151,19 @@ Expected<JITTargetAddress> EHFrameParser::readAbsolutePointer() { return Addr; } -Error EHFrameParser::processCIE() { +Error EHFrameBinaryParser::processCIE(size_t RecordOffset, + size_t RecordLength) { // Use the dwarf namespace for convenient access to pointer encoding // constants. using namespace dwarf; LLVM_DEBUG(dbgs() << " Record is CIE\n"); - CIEInformation CIEInfo(*CurRecordAtom); + auto &CIESymbol = + createCIERecord(EHFrameAddress + RecordOffset, + EHFrameContent.substr(RecordOffset, RecordLength)); + + CIEInformation CIEInfo(CIESymbol); uint8_t Version = 0; if (auto Err = EHFrameReader.readInteger(Version)) @@ -179,7 +179,7 @@ Error EHFrameParser::processCIE() { // Skip the EH Data field if present. if (AugInfo->EHDataFieldPresent) - if (auto Err = EHFrameReader.skip(G.getPointerSize())) + if (auto Err = EHFrameReader.skip(PointerSize)) return Err; // Read and sanity check the code alignment factor. @@ -226,7 +226,7 @@ Error EHFrameParser::processCIE() { return make_error<JITLinkError>( "Unsupported LSDA pointer encoding " + formatv("{0:x2}", LSDAPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); break; } case 'P': { @@ -239,7 +239,7 @@ Error EHFrameParser::processCIE() { "Unspported personality pointer " "encoding " + formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); uint32_t PersonalityPointerAddress; if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) return Err; @@ -254,7 +254,7 @@ Error EHFrameParser::processCIE() { "Unsupported FDE address pointer " "encoding " + formatv("{0:x2}", FDEPointerEncoding) + " in CIE at " + - formatv("{0:x16}", CurRecordAtom->getAddress())); + formatv("{0:x16}", CIESymbol.getAddress())); break; } default: @@ -267,15 +267,16 @@ Error EHFrameParser::processCIE() { return make_error<JITLinkError>("Read past the end of the augmentation " "data while parsing fields"); - assert(!CIEInfos.count(CurRecordAtom->getAddress()) && + assert(!CIEInfos.count(CIESymbol.getAddress()) && "Multiple CIEs recorded at the same address?"); - CIEInfos[CurRecordAtom->getAddress()] = std::move(CIEInfo); + CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); return Error::success(); } -Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, - uint32_t CIEPointer) { +Error EHFrameBinaryParser::processFDE(size_t RecordOffset, size_t RecordLength, + JITTargetAddress CIEPointerAddress, + uint32_t CIEPointer) { LLVM_DEBUG(dbgs() << " Record is FDE\n"); LLVM_DEBUG({ @@ -286,16 +287,11 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer); if (CIEInfoItr == CIEInfos.end()) return make_error<JITLinkError>( - "FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress()) + + "FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset) + " points to non-existant CIE at " + formatv("{0:x16}", CIEPointerAddress - CIEPointer)); auto &CIEInfo = CIEInfoItr->second; - // The CIEPointer looks good. Add a relocation. - CurRecordAtom->addEdge(FDEToCIERelocKind, - CIEPointerAddress - CurRecordAtom->getAddress(), - *CIEInfo.CIEAtom, 0); - // Read and sanity check the PC-start pointer and size. JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); @@ -305,83 +301,68 @@ Error EHFrameParser::processFDE(JITTargetAddress CIEPointerAddress, JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; LLVM_DEBUG({ - dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; + dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; }); - auto *TargetAtom = G.getAtomByAddress(PCBegin); + auto *TargetSymbol = getSymbolAtAddress(PCBegin); - if (!TargetAtom) + if (!TargetSymbol) return make_error<JITLinkError>("FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point at atom"); + " does not point at symbol"); - if (TargetAtom->getAddress() != PCBegin) + if (TargetSymbol->getAddress() != PCBegin) return make_error<JITLinkError>( "FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point to start of atom at " + - formatv("{0:x16}", TargetAtom->getAddress())); - - LLVM_DEBUG(dbgs() << " FDE target: " << *TargetAtom << "\n"); + " does not point to start of symbol at " + + formatv("{0:x16}", TargetSymbol->getAddress())); - // The PC-start pointer and size look good. Add relocations. - CurRecordAtom->addEdge(FDEToTargetRelocKind, - PCBeginAddress - CurRecordAtom->getAddress(), - *TargetAtom, 0); - - // Add a keep-alive relocation from the function to the FDE to ensure it is - // not dead stripped. - TargetAtom->addEdge(Edge::KeepAlive, 0, *CurRecordAtom, 0); + LLVM_DEBUG(dbgs() << " FDE target: " << *TargetSymbol << "\n"); // Skip over the PC range size field. - if (auto Err = EHFrameReader.skip(G.getPointerSize())) + if (auto Err = EHFrameReader.skip(PointerSize)) return Err; + Symbol *LSDASymbol = nullptr; + JITTargetAddress LSDAAddress = 0; if (CIEInfo.FDEsHaveLSDAField) { uint64_t AugmentationDataSize; if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) return Err; - if (AugmentationDataSize != G.getPointerSize()) + if (AugmentationDataSize != PointerSize) return make_error<JITLinkError>( "Unexpected FDE augmentation data size (expected " + - Twine(G.getPointerSize()) + ", got " + Twine(AugmentationDataSize) + - ") for FDE at " + formatv("{0:x16}", CurRecordAtom->getAddress())); - JITTargetAddress LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); + Twine(PointerSize) + ", got " + Twine(AugmentationDataSize) + + ") for FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset)); + LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); auto LSDADelta = readAbsolutePointer(); if (!LSDADelta) return LSDADelta.takeError(); JITTargetAddress LSDA = LSDAAddress + *LSDADelta; - auto *LSDAAtom = G.getAtomByAddress(LSDA); + LSDASymbol = getSymbolAtAddress(LSDA); - if (!LSDAAtom) + if (!LSDASymbol) return make_error<JITLinkError>("FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point at atom"); + " does not point at symbol"); - if (LSDAAtom->getAddress() != LSDA) + if (LSDASymbol->getAddress() != LSDA) return make_error<JITLinkError>( "FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point to start of atom at " + - formatv("{0:x16}", LSDAAtom->getAddress())); - - LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDAAtom << "\n"); + " does not point to start of symbol at " + + formatv("{0:x16}", LSDASymbol->getAddress())); - // LSDA looks good. Add relocations. - CurRecordAtom->addEdge(FDEToTargetRelocKind, - LSDAAddress - CurRecordAtom->getAddress(), *LSDAAtom, - 0); + LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDASymbol << "\n"); } - return Error::success(); -} + JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset; + auto FDESymbol = createFDERecord( + RecordAddress, EHFrameContent.substr(RecordOffset, RecordLength), + *CIEInfo.CIESymbol, CIEPointerAddress - RecordAddress, *TargetSymbol, + PCBeginAddress - RecordAddress, LSDASymbol, LSDAAddress - RecordAddress); -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) { - return EHFrameParser(G, EHFrameSection, EHFrameContent, EHFrameAddress, - FDEToCIERelocKind, FDEToTargetRelocKind) - .atomize(); + return FDESymbol.takeError(); } // Determine whether we can register EH tables. @@ -523,7 +504,7 @@ InProcessEHFrameRegistrar &InProcessEHFrameRegistrar::getInstance() { InProcessEHFrameRegistrar::InProcessEHFrameRegistrar() {} -AtomGraphPassFunction +LinkGraphPassFunction createEHFrameRecorderPass(const Triple &TT, StoreFrameRangeFunction StoreRangeAddress) { const char *EHFrameSectionName = nullptr; @@ -533,14 +514,14 @@ createEHFrameRecorderPass(const Triple &TT, EHFrameSectionName = ".eh_frame"; auto RecordEHFrame = - [EHFrameSectionName, - StoreFrameRange = std::move(StoreRangeAddress)](AtomGraph &G) -> Error { - // Search for a non-empty eh-frame and record the address of the first atom - // in it. + [EHFrameSectionName, + StoreFrameRange = std::move(StoreRangeAddress)](LinkGraph &G) -> Error { + // Search for a non-empty eh-frame and record the address of the first + // symbol in it. JITTargetAddress Addr = 0; size_t Size = 0; if (auto *S = G.findSectionByName(EHFrameSectionName)) { - auto R = S->getRange(); + auto R = SectionRange(*S); Addr = R.getStart(); Size = R.getSize(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index d679edef7ea..6f9f68ad838 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,18 +21,31 @@ namespace llvm { namespace jitlink { -/// A generic parser for eh-frame sections. +/// A generic binary parser for eh-frame sections. /// -/// Adds atoms representing CIE and FDE entries, using the given FDE-to-CIE and -/// FDEToTarget relocation kinds. -class EHFrameParser { +/// Adds blocks and symbols representing CIE and FDE entries to a JITLink graph. +/// +/// This parser assumes that the user has already verified that the EH-frame's +/// address range does not overlap any other section/symbol, so that generated +/// CIE/FDE records do not overlap other sections/symbols. +class EHFrameBinaryParser { public: - EHFrameParser(AtomGraph &G, Section &EHFrameSection, StringRef EHFrameContent, - JITTargetAddress EHFrameAddress, Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind); - Error atomize(); + EHFrameBinaryParser(JITTargetAddress EHFrameAddress, StringRef EHFrameContent, + unsigned PointerSize, support::endianness Endianness); + virtual ~EHFrameBinaryParser() {} + + Error addToGraph(); private: + virtual void anchor(); + virtual Symbol *getSymbolAtAddress(JITTargetAddress Addr) = 0; + virtual Symbol &createCIERecord(JITTargetAddress RecordAddr, + StringRef RecordContent) = 0; + virtual Expected<Symbol &> + createFDERecord(JITTargetAddress RecordAddr, StringRef RecordContent, + Symbol &CIE, size_t CIEOffset, Symbol &Func, + size_t FuncOffset, Symbol *LSDA, size_t LSDAOffset) = 0; + struct AugmentationInfo { bool AugmentationDataPresent = false; bool EHDataFieldPresent = false; @@ -41,31 +54,24 @@ private: Expected<AugmentationInfo> parseAugmentationString(); Expected<JITTargetAddress> readAbsolutePointer(); - Error processCIE(); - Error processFDE(JITTargetAddress CIEPointerAddress, uint32_t CIEPointer); + Error processCIE(size_t RecordOffset, size_t RecordLength); + Error processFDE(size_t RecordOffset, size_t RecordLength, + JITTargetAddress CIEPointerOffset, uint32_t CIEPointer); struct CIEInformation { CIEInformation() = default; - CIEInformation(DefinedAtom &CIEAtom) : CIEAtom(&CIEAtom) {} - DefinedAtom *CIEAtom = nullptr; + CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} + Symbol *CIESymbol = nullptr; bool FDEsHaveLSDAField = false; }; - AtomGraph &G; - Section &EHFrameSection; - StringRef EHFrameContent; JITTargetAddress EHFrameAddress; + StringRef EHFrameContent; + unsigned PointerSize; BinaryStreamReader EHFrameReader; - DefinedAtom *CurRecordAtom = nullptr; DenseMap<JITTargetAddress, CIEInformation> CIEInfos; - Edge::Kind FDEToCIERelocKind; - Edge::Kind FDEToTargetRelocKind; }; -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); - } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp index 9d0a7459dc0..54324f74521 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLink.cpp @@ -56,95 +56,143 @@ std::error_code JITLinkError::convertToErrorCode() const { return std::error_code(GenericJITLinkError, *JITLinkerErrorCategory); } -const StringRef getGenericEdgeKindName(Edge::Kind K) { +const char *getGenericEdgeKindName(Edge::Kind K) { switch (K) { case Edge::Invalid: return "INVALID RELOCATION"; case Edge::KeepAlive: return "Keep-Alive"; - case Edge::LayoutNext: - return "Layout-Next"; default: llvm_unreachable("Unrecognized relocation kind"); } } -raw_ostream &operator<<(raw_ostream &OS, const Atom &A) { +const char *getLinkageName(Linkage L) { + switch (L) { + case Linkage::Strong: + return "strong"; + case Linkage::Weak: + return "weak"; + } +} + +const char *getScopeName(Scope S) { + switch (S) { + case Scope::Default: + return "default"; + case Scope::Hidden: + return "hidden"; + case Scope::Local: + return "local"; + } +} + +raw_ostream &operator<<(raw_ostream &OS, const Block &B) { + return OS << formatv("{0:x16}", B.getAddress()) << " -- " + << formatv("{0:x16}", B.getAddress() + B.getSize()) << ": " + << (B.isZeroFill() ? "zero-fill" : "content") + << ", align = " << B.getAlignment() + << ", align-ofs = " << B.getAlignmentOffset() + << ", section = " << B.getSection().getName(); +} + +raw_ostream &operator<<(raw_ostream &OS, const Symbol &Sym) { OS << "<"; - if (A.getName().empty()) - OS << "anon@" << format("0x%016" PRIx64, A.getAddress()); + if (Sym.getName().empty()) + OS << "*anon*"; else - OS << A.getName(); - OS << " ["; - if (A.isDefined()) { - auto &DA = static_cast<const DefinedAtom &>(A); - OS << " section=" << DA.getSection().getName(); - if (DA.isLive()) - OS << " live"; - if (DA.shouldDiscard()) - OS << " should-discard"; - } else - OS << " external"; - OS << " ]>"; + OS << Sym.getName(); + OS << ": flags = "; + switch (Sym.getLinkage()) { + case Linkage::Strong: + OS << 'S'; + break; + case Linkage::Weak: + OS << 'W'; + break; + } + switch (Sym.getScope()) { + case Scope::Default: + OS << 'D'; + break; + case Scope::Hidden: + OS << 'H'; + break; + case Scope::Local: + OS << 'L'; + break; + } + OS << (Sym.isLive() ? '+' : '-') + << ", size = " << formatv("{0:x8}", Sym.getSize()) + << ", addr = " << formatv("{0:x16}", Sym.getAddress()) << " (" + << formatv("{0:x16}", Sym.getAddressable().getAddress()) << " + " + << formatv("{0:x8}", Sym.getOffset()); + if (Sym.isDefined()) + OS << " " << Sym.getBlock().getSection().getName(); + OS << ")>"; return OS; } -void printEdge(raw_ostream &OS, const Atom &FixupAtom, const Edge &E, +void printEdge(raw_ostream &OS, const Block &B, const Edge &E, StringRef EdgeKindName) { - OS << "edge@" << formatv("{0:x16}", FixupAtom.getAddress() + E.getOffset()) - << ": " << FixupAtom << " + " << E.getOffset() << " -- " << EdgeKindName - << " -> " << E.getTarget() << " + " << E.getAddend(); + OS << "edge@" << formatv("{0:x16}", B.getAddress() + E.getOffset()) << ": " + << formatv("{0:x16}", B.getAddress()) << " + " << E.getOffset() << " -- " + << EdgeKindName << " -> " << E.getTarget() << " + " << E.getAddend(); } Section::~Section() { - for (auto *DA : DefinedAtoms) - DA->~DefinedAtom(); + for (auto *Sym : Symbols) + Sym->~Symbol(); } -void AtomGraph::dump(raw_ostream &OS, +void LinkGraph::dump(raw_ostream &OS, std::function<StringRef(Edge::Kind)> EdgeKindToName) { if (!EdgeKindToName) EdgeKindToName = [](Edge::Kind K) { return StringRef(); }; - OS << "Defined atoms:\n"; - for (auto *DA : defined_atoms()) { - OS << " " << format("0x%016" PRIx64, DA->getAddress()) << ": " << *DA + OS << "Symbols:\n"; + for (auto *Sym : defined_symbols()) { + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; - for (auto &E : DA->edges()) { - OS << " "; - StringRef EdgeName = (E.getKind() < Edge::FirstRelocation - ? getGenericEdgeKindName(E.getKind()) - : EdgeKindToName(E.getKind())); - - if (!EdgeName.empty()) - printEdge(OS, *DA, E, EdgeName); - else { - auto EdgeNumberString = std::to_string(E.getKind()); - printEdge(OS, *DA, E, EdgeNumberString); + if (Sym->isDefined()) { + for (auto &E : Sym->getBlock().edges()) { + OS << " "; + StringRef EdgeName = (E.getKind() < Edge::FirstRelocation + ? getGenericEdgeKindName(E.getKind()) + : EdgeKindToName(E.getKind())); + + if (!EdgeName.empty()) + printEdge(OS, Sym->getBlock(), E, EdgeName); + else { + auto EdgeNumberString = std::to_string(E.getKind()); + printEdge(OS, Sym->getBlock(), E, EdgeNumberString); + } + OS << "\n"; } - OS << "\n"; } } - OS << "Absolute atoms:\n"; - for (auto *A : absolute_atoms()) - OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + OS << "Absolute symbols:\n"; + for (auto *Sym : absolute_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; - OS << "External atoms:\n"; - for (auto *A : external_atoms()) - OS << " " << format("0x%016" PRIx64, A->getAddress()) << ": " << *A + OS << "External symbols:\n"; + for (auto *Sym : external_symbols()) + OS << " " << format("0x%016" PRIx64, Sym->getAddress()) << ": " << *Sym << "\n"; } +void JITLinkAsyncLookupContinuation::anchor() {} + JITLinkContext::~JITLinkContext() {} bool JITLinkContext::shouldAddDefaultTargetPasses(const Triple &TT) const { return true; } -AtomGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { - return AtomGraphPassFunction(); +LinkGraphPassFunction JITLinkContext::getMarkLivePass(const Triple &TT) const { + return LinkGraphPassFunction(); } Error JITLinkContext::modifyPassConfig(const Triple &TT, @@ -152,9 +200,9 @@ Error JITLinkContext::modifyPassConfig(const Triple &TT, return Error::success(); } -Error markAllAtomsLive(AtomGraph &G) { - for (auto *DA : G.defined_atoms()) - DA->setLive(true); +Error markAllSymbolsLive(LinkGraph &G) { + for (auto *Sym : G.defined_symbols()) + Sym->setLive(true); return Error::success(); } diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index 877107ffe25..d4270b5aa79 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -11,7 +11,6 @@ //===----------------------------------------------------------------------===// #include "JITLinkGeneric.h" -#include "EHFrameSupportImpl.h" #include "llvm/Support/BinaryStreamReader.h" #include "llvm/Support/MemoryBuffer.h" @@ -25,7 +24,7 @@ JITLinkerBase::~JITLinkerBase() {} void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { - // Build the atom graph. + // Build the link graph. if (auto GraphOrErr = buildGraph(Ctx->getObjectBuffer())) G = std::move(*GraphOrErr); else @@ -33,33 +32,33 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { assert(G && "Graph should have been created by buildGraph above"); // Prune and optimize the graph. - if (auto Err = runPasses(Passes.PrePrunePasses, *G)) + if (auto Err = runPasses(Passes.PrePrunePasses)) return Ctx->notifyFailed(std::move(Err)); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" pre-pruning:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" pre-pruning:\n"; dumpGraph(dbgs()); }); prune(*G); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" post-pruning:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" post-pruning:\n"; dumpGraph(dbgs()); }); // Run post-pruning passes. - if (auto Err = runPasses(Passes.PostPrunePasses, *G)) + if (auto Err = runPasses(Passes.PostPrunePasses)) return Ctx->notifyFailed(std::move(Err)); - // Sort atoms into segments. - layOutAtoms(); + // Sort blocks into segments. + auto Layout = layOutBlocks(); // Allocate memory for segments. if (auto Err = allocateSegments(Layout)) return Ctx->notifyFailed(std::move(Err)); - // Notify client that the defined atoms have been assigned addresses. + // Notify client that the defined symbols have been assigned addresses. Ctx->notifyResolved(*G); auto ExternalSymbols = getExternalSymbolNames(); @@ -74,42 +73,42 @@ void JITLinkerBase::linkPhase1(std::unique_ptr<JITLinkerBase> Self) { // [Self=std::move(Self)](Expected<AsyncLookupResult> Result) { // Self->linkPhase2(std::move(Self), std::move(Result)); // }); - // - // FIXME: Use move capture once we have c++14. auto *TmpCtx = Ctx.get(); - auto *UnownedSelf = Self.release(); - auto Phase2Continuation = - [UnownedSelf](Expected<AsyncLookupResult> LookupResult) { - std::unique_ptr<JITLinkerBase> Self(UnownedSelf); - UnownedSelf->linkPhase2(std::move(Self), std::move(LookupResult)); - }; - TmpCtx->lookup(std::move(ExternalSymbols), std::move(Phase2Continuation)); + TmpCtx->lookup(std::move(ExternalSymbols), + createLookupContinuation( + [S = std::move(Self), L = std::move(Layout)]( + Expected<AsyncLookupResult> LookupResult) mutable { + auto &TmpSelf = *S; + TmpSelf.linkPhase2(std::move(S), std::move(LookupResult), + std::move(L)); + })); } void JITLinkerBase::linkPhase2(std::unique_ptr<JITLinkerBase> Self, - Expected<AsyncLookupResult> LR) { + Expected<AsyncLookupResult> LR, + SegmentLayoutMap Layout) { // If the lookup failed, bail out. if (!LR) return deallocateAndBailOut(LR.takeError()); - // Assign addresses to external atoms. + // Assign addresses to external addressables. applyLookupResult(*LR); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" before copy-and-fixup:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" before copy-and-fixup:\n"; dumpGraph(dbgs()); }); - // Copy atom content to working memory and fix up. - if (auto Err = copyAndFixUpAllAtoms(Layout, *Alloc)) + // Copy block content to working memory and fix up. + if (auto Err = copyAndFixUpBlocks(Layout, *Alloc)) return deallocateAndBailOut(std::move(Err)); LLVM_DEBUG({ - dbgs() << "Atom graph \"" << G->getName() << "\" after copy-and-fixup:\n"; + dbgs() << "Link graph \"" << G->getName() << "\" after copy-and-fixup:\n"; dumpGraph(dbgs()); }); - if (auto Err = runPasses(Passes.PostFixupPasses, *G)) + if (auto Err = runPasses(Passes.PostFixupPasses)) return deallocateAndBailOut(std::move(Err)); // FIXME: Use move capture once we have c++14. @@ -128,82 +127,38 @@ void JITLinkerBase::linkPhase3(std::unique_ptr<JITLinkerBase> Self, Error Err) { Ctx->notifyFinalized(std::move(Alloc)); } -Error JITLinkerBase::runPasses(AtomGraphPassList &Passes, AtomGraph &G) { +Error JITLinkerBase::runPasses(LinkGraphPassList &Passes) { for (auto &P : Passes) - if (auto Err = P(G)) + if (auto Err = P(*G)) return Err; return Error::success(); } -void JITLinkerBase::layOutAtoms() { - // Group sections by protections, and whether or not they're zero-fill. - for (auto &S : G->sections()) { +JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { - // Skip empty sections. - if (S.atoms_empty()) - continue; + SegmentLayoutMap Layout; - auto &SL = Layout[S.getProtectionFlags()]; - if (S.isZeroFill()) - SL.ZeroFillSections.push_back(SegmentLayout::SectionLayout(S)); + /// Partition blocks based on permissions and content vs. zero-fill. + for (auto *B : G->blocks()) { + auto &SegLists = Layout[B->getSection().getProtectionFlags()]; + if (!B->isZeroFill()) + SegLists.ContentBlocks.push_back(B); else - SL.ContentSections.push_back(SegmentLayout::SectionLayout(S)); + SegLists.ZeroFillBlocks.push_back(B); } - // Sort sections within the layout by ordinal. - { - auto CompareByOrdinal = [](const SegmentLayout::SectionLayout &LHS, - const SegmentLayout::SectionLayout &RHS) { - return LHS.S->getSectionOrdinal() < RHS.S->getSectionOrdinal(); + /// Sort blocks within each list. + for (auto &KV : Layout) { + + auto CompareBlocks = [](const Block *LHS, const Block *RHS) { + if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal()) + return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal(); + return LHS->getOrdinal() < RHS->getOrdinal(); }; - for (auto &KV : Layout) { - auto &SL = KV.second; - std::sort(SL.ContentSections.begin(), SL.ContentSections.end(), - CompareByOrdinal); - std::sort(SL.ZeroFillSections.begin(), SL.ZeroFillSections.end(), - CompareByOrdinal); - } - } - // Add atoms to the sections. - for (auto &KV : Layout) { - auto &SL = KV.second; - for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) { - for (auto &SI : *SIList) { - // First build the set of layout-heads (i.e. "heads" of layout-next - // chains) by copying the section atoms, then eliminating any that - // appear as layout-next targets. - DenseSet<DefinedAtom *> LayoutHeads; - for (auto *DA : SI.S->atoms()) - LayoutHeads.insert(DA); - - for (auto *DA : SI.S->atoms()) - if (DA->hasLayoutNext()) - LayoutHeads.erase(&DA->getLayoutNext()); - - // Next, sort the layout heads by address order. - std::vector<DefinedAtom *> OrderedLayoutHeads; - OrderedLayoutHeads.reserve(LayoutHeads.size()); - for (auto *DA : LayoutHeads) - OrderedLayoutHeads.push_back(DA); - - // Now sort the list of layout heads by address. - std::sort(OrderedLayoutHeads.begin(), OrderedLayoutHeads.end(), - [](const DefinedAtom *LHS, const DefinedAtom *RHS) { - return LHS->getAddress() < RHS->getAddress(); - }); - - // Now populate the SI.Atoms field by appending each of the chains. - for (auto *DA : OrderedLayoutHeads) { - SI.Atoms.push_back(DA); - while (DA->hasLayoutNext()) { - auto &Next = DA->getLayoutNext(); - SI.Atoms.push_back(&Next); - DA = &Next; - } - } - } - } + auto &SegLists = KV.second; + llvm::sort(SegLists.ContentBlocks, CompareBlocks); + llvm::sort(SegLists.ZeroFillBlocks, CompareBlocks); } LLVM_DEBUG({ @@ -213,18 +168,16 @@ void JITLinkerBase::layOutAtoms() { << static_cast<sys::Memory::ProtectionFlags>(KV.first) << ":\n"; auto &SL = KV.second; for (auto &SIEntry : - {std::make_pair(&SL.ContentSections, "content sections"), - std::make_pair(&SL.ZeroFillSections, "zero-fill sections")}) { - auto &SIList = *SIEntry.first; + {std::make_pair(&SL.ContentBlocks, "content block"), + std::make_pair(&SL.ZeroFillBlocks, "zero-fill block")}) { dbgs() << " " << SIEntry.second << ":\n"; - for (auto &SI : SIList) { - dbgs() << " " << SI.S->getName() << ":\n"; - for (auto *DA : SI.Atoms) - dbgs() << " " << *DA << "\n"; - } + for (auto *B : *SIEntry.first) + dbgs() << " " << *B << "\n"; } } }); + + return Layout; } Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { @@ -234,61 +187,36 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { JITLinkMemoryManager::SegmentsRequestMap Segments; for (auto &KV : Layout) { auto &Prot = KV.first; - auto &SegLayout = KV.second; + auto &SegLists = KV.second; + + uint64_t SegAlign = 1; // Calculate segment content size. size_t SegContentSize = 0; - uint32_t SegContentAlign = 1; - for (auto &SI : SegLayout.ContentSections) { - assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); - assert(!SI.Atoms.empty() && "Section layouts must not be empty"); - - // Bump to section alignment before processing atoms. - SegContentSize = alignTo(SegContentSize, SI.S->getAlignment()); - SegContentAlign = std::max(SegContentAlign, SI.S->getAlignment()); - - for (auto *DA : SI.Atoms) { - SegContentSize = alignTo(SegContentSize, DA->getAlignment()); - SegContentSize += DA->getSize(); - SegContentAlign = std::max(SegContentAlign, DA->getAlignment()); - } + for (auto *B : SegLists.ContentBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegContentSize = alignToBlock(SegContentSize, *B); + SegContentSize += B->getSize(); } - // Calculate segment zero-fill size. - uint64_t SegZeroFillSize = 0; - uint32_t SegZeroFillAlign = 1; - - for (auto &SI : SegLayout.ZeroFillSections) { - assert(!SI.S->atoms_empty() && "Sections in layout must not be empty"); - assert(!SI.Atoms.empty() && "Section layouts must not be empty"); - - // Bump to section alignment before processing atoms. - SegZeroFillSize = alignTo(SegZeroFillSize, SI.S->getAlignment()); - SegZeroFillAlign = std::max(SegZeroFillAlign, SI.S->getAlignment()); + uint64_t SegZeroFillStart = SegContentSize; + uint64_t SegZeroFillEnd = SegZeroFillStart; - for (auto *DA : SI.Atoms) { - SegZeroFillSize = alignTo(SegZeroFillSize, DA->getAlignment()); - SegZeroFillSize += DA->getSize(); - SegZeroFillAlign = std::max(SegZeroFillAlign, SI.S->getAlignment()); - } + for (auto *B : SegLists.ZeroFillBlocks) { + SegAlign = std::max(SegAlign, B->getAlignment()); + SegZeroFillEnd = alignToBlock(SegZeroFillEnd, *B); + SegZeroFillEnd += B->getSize(); } - assert(isPowerOf2_32(SegContentAlign) && - "Expected content alignment to be power of 2"); - assert(isPowerOf2_32(SegZeroFillAlign) && - "Expected zero-fill alignment to be power of 2"); - // Round content alignment up to segment alignment. - SegContentAlign = std::max(SegContentAlign, SegZeroFillAlign); - - Segments[Prot] = {SegContentSize, SegContentAlign, SegZeroFillSize, - SegZeroFillAlign}; + Segments[Prot] = {SegAlign, SegContentSize, + SegZeroFillEnd - SegZeroFillStart}; LLVM_DEBUG({ dbgs() << (&KV == &*Layout.begin() ? "" : "; ") - << static_cast<sys::Memory::ProtectionFlags>(Prot) << ": " - << SegContentSize << " content bytes (alignment " - << SegContentAlign << ") + " << SegZeroFillSize - << " zero-fill bytes (alignment " << SegZeroFillAlign << ")"; + << static_cast<sys::Memory::ProtectionFlags>(Prot) + << ": alignment = " << SegAlign + << ", content size = " << SegContentSize + << ", zero-fill size = " << (SegZeroFillEnd - SegZeroFillStart); }); } LLVM_DEBUG(dbgs() << " }\n"); @@ -307,22 +235,19 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { } }); - // Update atom target addresses. + // Update block target addresses. for (auto &KV : Layout) { auto &Prot = KV.first; auto &SL = KV.second; - JITTargetAddress AtomTargetAddr = + JITTargetAddress NextBlockAddr = Alloc->getTargetMemory(static_cast<sys::Memory::ProtectionFlags>(Prot)); - for (auto *SIList : {&SL.ContentSections, &SL.ZeroFillSections}) - for (auto &SI : *SIList) { - AtomTargetAddr = alignTo(AtomTargetAddr, SI.S->getAlignment()); - for (auto *DA : SI.Atoms) { - AtomTargetAddr = alignTo(AtomTargetAddr, DA->getAlignment()); - DA->setAddress(AtomTargetAddr); - AtomTargetAddr += DA->getSize(); - } + for (auto *SIList : {&SL.ContentBlocks, &SL.ZeroFillBlocks}) + for (auto *B : *SIList) { + NextBlockAddr = alignToBlock(NextBlockAddr, *B); + B->setAddress(NextBlockAddr); + NextBlockAddr += B->getSize(); } } @@ -330,34 +255,35 @@ Error JITLinkerBase::allocateSegments(const SegmentLayoutMap &Layout) { } DenseSet<StringRef> JITLinkerBase::getExternalSymbolNames() const { - // Identify unresolved external atoms. + // Identify unresolved external symbols. DenseSet<StringRef> UnresolvedExternals; - for (auto *DA : G->external_atoms()) { - assert(DA->getAddress() == 0 && + for (auto *Sym : G->external_symbols()) { + assert(Sym->getAddress() == 0 && "External has already been assigned an address"); - assert(DA->getName() != StringRef() && DA->getName() != "" && + assert(Sym->getName() != StringRef() && Sym->getName() != "" && "Externals must be named"); - UnresolvedExternals.insert(DA->getName()); + UnresolvedExternals.insert(Sym->getName()); } return UnresolvedExternals; } void JITLinkerBase::applyLookupResult(AsyncLookupResult Result) { - for (auto &KV : Result) { - Atom &A = G->getAtomByName(KV.first); - assert(A.getAddress() == 0 && "Atom already resolved"); - A.setAddress(KV.second.getAddress()); + for (auto *Sym : G->external_symbols()) { + assert(Sym->getAddress() == 0 && "Symbol already resolved"); + assert(!Sym->isDefined() && "Symbol being resolved is already defined"); + assert(Result.count(Sym->getName()) && "Missing resolution for symbol"); + Sym->getAddressable().setAddress(Result[Sym->getName()].getAddress()); } LLVM_DEBUG({ dbgs() << "Externals after applying lookup result:\n"; - for (auto *A : G->external_atoms()) - dbgs() << " " << A->getName() << ": " - << formatv("{0:x16}", A->getAddress()) << "\n"; + for (auto *Sym : G->external_symbols()) + dbgs() << " " << Sym->getName() << ": " + << formatv("{0:x16}", Sym->getAddress()) << "\n"; }); - assert(llvm::all_of(G->external_atoms(), - [](Atom *A) { return A->getAddress() != 0; }) && - "All atoms should have been resolved by this point"); + assert(llvm::all_of(G->external_symbols(), + [](Symbol *Sym) { return Sym->getAddress() != 0; }) && + "All symbols should have been resolved by this point"); } void JITLinkerBase::deallocateAndBailOut(Error Err) { @@ -371,96 +297,60 @@ void JITLinkerBase::dumpGraph(raw_ostream &OS) { G->dump(dbgs(), [this](Edge::Kind K) { return getEdgeKindName(K); }); } -void prune(AtomGraph &G) { - std::vector<DefinedAtom *> Worklist; - DenseMap<DefinedAtom *, std::vector<Edge *>> EdgesToUpdate; +void prune(LinkGraph &G) { + std::vector<Symbol *> Worklist; + DenseSet<Block *> VisitedBlocks; - // Build the initial worklist from all atoms initially live. - for (auto *DA : G.defined_atoms()) { - if (!DA->isLive() || DA->shouldDiscard()) - continue; - - for (auto &E : DA->edges()) { - if (!E.getTarget().isDefined()) - continue; + // Build the initial worklist from all symbols initially live. + for (auto *Sym : G.defined_symbols()) + if (Sym->isLive()) + Worklist.push_back(Sym); - auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); - - if (EDT.shouldDiscard()) - EdgesToUpdate[&EDT].push_back(&E); - else if (E.isKeepAlive() && !EDT.isLive()) - Worklist.push_back(&EDT); - } - } - - // Propagate live flags to all atoms reachable from the initial live set. + // Propagate live flags to all symbols reachable from the initial live set. while (!Worklist.empty()) { - DefinedAtom &NextLive = *Worklist.back(); + auto *Sym = Worklist.back(); Worklist.pop_back(); - assert(!NextLive.shouldDiscard() && - "should-discard nodes should never make it into the worklist"); + auto &B = Sym->getBlock(); - // If this atom has already been marked as live, or is marked to be - // discarded, then skip it. - if (NextLive.isLive()) + // Skip addressables that we've visited before. + if (VisitedBlocks.count(&B)) continue; - // Otherwise set it as live and add any non-live atoms that it points to - // to the worklist. - NextLive.setLive(true); - - for (auto &E : NextLive.edges()) { - if (!E.getTarget().isDefined()) - continue; - - auto &EDT = static_cast<DefinedAtom &>(E.getTarget()); + VisitedBlocks.insert(&B); - if (EDT.shouldDiscard()) - EdgesToUpdate[&EDT].push_back(&E); - else if (E.isKeepAlive() && !EDT.isLive()) - Worklist.push_back(&EDT); + for (auto &E : Sym->getBlock().edges()) { + if (E.getTarget().isDefined() && !E.getTarget().isLive()) { + E.getTarget().setLive(true); + Worklist.push_back(&E.getTarget()); + } } } - // Collect atoms to remove, then remove them from the graph. - std::vector<DefinedAtom *> AtomsToRemove; - for (auto *DA : G.defined_atoms()) - if (DA->shouldDiscard() || !DA->isLive()) - AtomsToRemove.push_back(DA); - - LLVM_DEBUG(dbgs() << "Pruning atoms:\n"); - for (auto *DA : AtomsToRemove) { - LLVM_DEBUG(dbgs() << " " << *DA << "... "); - - // Check whether we need to replace this atom with an external atom. - // - // We replace if all of the following hold: - // (1) The atom is marked should-discard, - // (2) it has live edges (i.e. edges from live atoms) pointing to it. - // - // Otherwise we simply delete the atom. - - G.removeDefinedAtom(*DA); - - auto EdgesToUpdateItr = EdgesToUpdate.find(DA); - if (EdgesToUpdateItr != EdgesToUpdate.end()) { - auto &ExternalReplacement = G.addExternalAtom(DA->getName()); - for (auto *EdgeToUpdate : EdgesToUpdateItr->second) - EdgeToUpdate->setTarget(ExternalReplacement); - LLVM_DEBUG(dbgs() << "replaced with " << ExternalReplacement << "\n"); - } else - LLVM_DEBUG(dbgs() << "deleted\n"); + // Collect all the symbols to remove, then remove them. + { + LLVM_DEBUG(dbgs() << "Dead-stripping symbols:\n"); + std::vector<Symbol *> SymbolsToRemove; + for (auto *Sym : G.defined_symbols()) + if (!Sym->isLive()) + SymbolsToRemove.push_back(Sym); + for (auto *Sym : SymbolsToRemove) { + LLVM_DEBUG(dbgs() << " " << *Sym << "...\n"); + G.removeDefinedSymbol(*Sym); + } } - // Finally, discard any absolute symbols that were marked should-discard. + // Delete any unused blocks. { - std::vector<Atom *> AbsoluteAtomsToRemove; - for (auto *A : G.absolute_atoms()) - if (A->shouldDiscard() || A->isLive()) - AbsoluteAtomsToRemove.push_back(A); - for (auto *A : AbsoluteAtomsToRemove) - G.removeAbsoluteAtom(*A); + LLVM_DEBUG(dbgs() << "Dead-stripping blocks:\n"); + std::vector<Block *> BlocksToRemove; + for (auto *B : G.blocks()) + if (!VisitedBlocks.count(B)) + BlocksToRemove.push_back(B); + for (auto *B : BlocksToRemove) { + LLVM_DEBUG(dbgs() << " " << *B << "...\n"); + G.removeBlock(*B); + } } } diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h index eeb2527bd1b..07dee6cee20 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.h @@ -41,39 +41,32 @@ public: protected: struct SegmentLayout { - using SectionAtomsList = std::vector<DefinedAtom *>; - struct SectionLayout { - SectionLayout(Section &S) : S(&S) {} + using BlocksList = std::vector<Block *>; - Section *S; - SectionAtomsList Atoms; - }; - - using SectionLayoutList = std::vector<SectionLayout>; - - SectionLayoutList ContentSections; - SectionLayoutList ZeroFillSections; + BlocksList ContentBlocks; + BlocksList ZeroFillBlocks; }; using SegmentLayoutMap = DenseMap<unsigned, SegmentLayout>; // Phase 1: - // 1.1: Build atom graph + // 1.1: Build link graph // 1.2: Run pre-prune passes // 1.2: Prune graph // 1.3: Run post-prune passes - // 1.4: Sort atoms into segments + // 1.4: Sort blocks into segments // 1.5: Allocate segment memory // 1.6: Identify externals and make an async call to resolve function void linkPhase1(std::unique_ptr<JITLinkerBase> Self); // Phase 2: // 2.1: Apply resolution results - // 2.2: Fix up atom contents + // 2.2: Fix up block contents // 2.3: Call OnResolved callback // 2.3: Make an async call to transfer and finalize memory. void linkPhase2(std::unique_ptr<JITLinkerBase> Self, - Expected<AsyncLookupResult> LookupResult); + Expected<AsyncLookupResult> LookupResult, + SegmentLayoutMap Layout); // Phase 3: // 3.1: Call OnFinalized callback, handing off allocation. @@ -81,24 +74,37 @@ protected: // Build a graph from the given object buffer. // To be implemented by the client. - virtual Expected<std::unique_ptr<AtomGraph>> + virtual Expected<std::unique_ptr<LinkGraph>> buildGraph(MemoryBufferRef ObjBuffer) = 0; - // For debug dumping of the atom graph. + // For debug dumping of the link graph. virtual StringRef getEdgeKindName(Edge::Kind K) const = 0; + // Alight a JITTargetAddress to conform with block alignment requirements. + static JITTargetAddress alignToBlock(JITTargetAddress Addr, Block &B) { + uint64_t Delta = (B.getAlignmentOffset() - Addr) % B.getAlignment(); + return Addr + Delta; + } + + // Alight a pointer to conform with block alignment requirements. + static char *alignToBlock(char *P, Block &B) { + uint64_t PAddr = static_cast<uint64_t>(reinterpret_cast<uintptr_t>(P)); + uint64_t Delta = (B.getAlignmentOffset() - PAddr) % B.getAlignment(); + return P + Delta; + } + private: // Run all passes in the given pass list, bailing out immediately if any pass // returns an error. - Error runPasses(AtomGraphPassList &Passes, AtomGraph &G); + Error runPasses(LinkGraphPassList &Passes); - // Copy atom contents and apply relocations. + // Copy block contents and apply relocations. // Implemented in JITLinker. virtual Error - copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const = 0; + copyAndFixUpBlocks(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const = 0; - void layOutAtoms(); + SegmentLayoutMap layOutBlocks(); Error allocateSegments(const SegmentLayoutMap &Layout); DenseSet<StringRef> getExternalSymbolNames() const; void applyLookupResult(AsyncLookupResult LR); @@ -108,8 +114,7 @@ private: std::unique_ptr<JITLinkContext> Ctx; PassConfiguration Passes; - std::unique_ptr<AtomGraph> G; - SegmentLayoutMap Layout; + std::unique_ptr<LinkGraph> G; std::unique_ptr<JITLinkMemoryManager::Allocation> Alloc; }; @@ -140,17 +145,17 @@ private: } Error - copyAndFixUpAllAtoms(const SegmentLayoutMap &Layout, - JITLinkMemoryManager::Allocation &Alloc) const override { - LLVM_DEBUG(dbgs() << "Copying and fixing up atoms:\n"); + copyAndFixUpBlocks(const SegmentLayoutMap &Layout, + JITLinkMemoryManager::Allocation &Alloc) const override { + LLVM_DEBUG(dbgs() << "Copying and fixing up blocks:\n"); for (auto &KV : Layout) { auto &Prot = KV.first; auto &SegLayout = KV.second; auto SegMem = Alloc.getWorkingMemory( static_cast<sys::Memory::ProtectionFlags>(Prot)); - char *LastAtomEnd = SegMem.data(); - char *AtomDataPtr = LastAtomEnd; + char *LastBlockEnd = SegMem.data(); + char *BlockDataPtr = LastBlockEnd; LLVM_DEBUG({ dbgs() << " Processing segment " @@ -160,93 +165,79 @@ private: << " ]\n Processing content sections:\n"; }); - for (auto &SI : SegLayout.ContentSections) { - LLVM_DEBUG(dbgs() << " " << SI.S->getName() << ":\n"); + for (auto *B : SegLayout.ContentBlocks) { + LLVM_DEBUG(dbgs() << " " << *B << ":\n"); + + // Pad to alignment/alignment-offset. + BlockDataPtr = alignToBlock(BlockDataPtr, *B); - AtomDataPtr += alignmentAdjustment(AtomDataPtr, SI.S->getAlignment()); + LLVM_DEBUG({ + dbgs() << " Bumped block pointer to " + << (const void *)BlockDataPtr << " to meet block alignment " + << B->getAlignment() << " and alignment offset " + << B->getAlignmentOffset() << "\n"; + }); + // Zero pad up to alignment. LLVM_DEBUG({ - dbgs() << " Bumped atom pointer to " << (const void *)AtomDataPtr - << " to meet section alignment " - << " of " << SI.S->getAlignment() << "\n"; + if (LastBlockEnd != BlockDataPtr) + dbgs() << " Zero padding from " << (const void *)LastBlockEnd + << " to " << (const void *)BlockDataPtr << "\n"; }); - for (auto *DA : SI.Atoms) { - - // Align. - AtomDataPtr += alignmentAdjustment(AtomDataPtr, DA->getAlignment()); - LLVM_DEBUG({ - dbgs() << " Bumped atom pointer to " - << (const void *)AtomDataPtr << " to meet alignment of " - << DA->getAlignment() << "\n"; - }); - - // Zero pad up to alignment. - LLVM_DEBUG({ - if (LastAtomEnd != AtomDataPtr) - dbgs() << " Zero padding from " << (const void *)LastAtomEnd - << " to " << (const void *)AtomDataPtr << "\n"; - }); - while (LastAtomEnd != AtomDataPtr) - *LastAtomEnd++ = 0; - - // Copy initial atom content. - LLVM_DEBUG({ - dbgs() << " Copying atom " << *DA << " content, " - << DA->getContent().size() << " bytes, from " - << (const void *)DA->getContent().data() << " to " - << (const void *)AtomDataPtr << "\n"; - }); - memcpy(AtomDataPtr, DA->getContent().data(), DA->getContent().size()); - - // Copy atom data and apply fixups. - LLVM_DEBUG(dbgs() << " Applying fixups.\n"); - for (auto &E : DA->edges()) { - - // Skip non-relocation edges. - if (!E.isRelocation()) - continue; - - // Dispatch to LinkerImpl for fixup. - if (auto Err = impl().applyFixup(*DA, E, AtomDataPtr)) - return Err; - } - - // Point the atom's content to the fixed up buffer. - DA->setContent(StringRef(AtomDataPtr, DA->getContent().size())); - - // Update atom end pointer. - LastAtomEnd = AtomDataPtr + DA->getContent().size(); - AtomDataPtr = LastAtomEnd; + while (LastBlockEnd != BlockDataPtr) + *LastBlockEnd++ = 0; + + // Copy initial block content. + LLVM_DEBUG({ + dbgs() << " Copying block " << *B << " content, " + << B->getContent().size() << " bytes, from " + << (const void *)B->getContent().data() << " to " + << (const void *)BlockDataPtr << "\n"; + }); + memcpy(BlockDataPtr, B->getContent().data(), B->getContent().size()); + + // Copy Block data and apply fixups. + LLVM_DEBUG(dbgs() << " Applying fixups.\n"); + for (auto &E : B->edges()) { + + // Skip non-relocation edges. + if (!E.isRelocation()) + continue; + + // Dispatch to LinkerImpl for fixup. + if (auto Err = impl().applyFixup(*B, E, BlockDataPtr)) + return Err; } + + // Point the block's content to the fixed up buffer. + B->setContent(StringRef(BlockDataPtr, B->getContent().size())); + + // Update block end pointer. + LastBlockEnd = BlockDataPtr + B->getContent().size(); + BlockDataPtr = LastBlockEnd; } // Zero pad the rest of the segment. LLVM_DEBUG({ dbgs() << " Zero padding end of segment from " - << (const void *)LastAtomEnd << " to " + << (const void *)LastBlockEnd << " to " << (const void *)((char *)SegMem.data() + SegMem.size()) << "\n"; }); - while (LastAtomEnd != SegMem.data() + SegMem.size()) - *LastAtomEnd++ = 0; + while (LastBlockEnd != SegMem.data() + SegMem.size()) + *LastBlockEnd++ = 0; } return Error::success(); } }; -/// Dead strips and replaces discarded definitions with external atoms. +/// Removes dead symbols/blocks/addressables. /// -/// Finds the set of nodes reachable from any node initially marked live -/// (nodes marked should-discard are treated as not live, even if they are -/// reachable). All nodes not marked as live at the end of this process, -/// are deleted. Nodes that are live, but marked should-discard are replaced -/// with external atoms and all edges to them are re-written. -void prune(AtomGraph &G); - -Error addEHFrame(AtomGraph &G, Section &EHFrameSection, - StringRef EHFrameContent, JITTargetAddress EHFrameAddress, - Edge::Kind FDEToCIERelocKind, Edge::Kind FDEToTargetRelocKind); +/// Finds the set of symbols and addressables reachable from any symbol +/// initially marked live. All symbols/addressables not marked live at the end +/// of this process are removed. +void prune(LinkGraph &G); } // end namespace jitlink } // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp index 267307cfde0..ecc6793bbce 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkMemoryManager.cpp @@ -61,6 +61,10 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { AllocationMap SegBlocks; }; + if (!isPowerOf2_64((uint64_t)sys::Process::getPageSizeEstimate())) + return make_error<StringError>("Page size is not a power of 2", + inconvertibleErrorCode()); + AllocationMap Blocks; const sys::Memory::ProtectionFlags ReadWrite = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | @@ -69,19 +73,12 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { for (auto &KV : Request) { auto &Seg = KV.second; - if (Seg.getContentAlignment() > sys::Process::getPageSizeEstimate()) + if (Seg.getAlignment() > sys::Process::getPageSizeEstimate()) return make_error<StringError>("Cannot request higher than page " "alignment", inconvertibleErrorCode()); - if (sys::Process::getPageSizeEstimate() % Seg.getContentAlignment() != 0) - return make_error<StringError>("Page size is not a multiple of " - "alignment", - inconvertibleErrorCode()); - - uint64_t ZeroFillStart = - alignTo(Seg.getContentSize(), Seg.getZeroFillAlignment()); - uint64_t SegmentSize = ZeroFillStart + Seg.getZeroFillSize(); + uint64_t SegmentSize = Seg.getContentSize() + Seg.getZeroFillSize(); std::error_code EC; auto SegMem = @@ -91,7 +88,7 @@ InProcessMemoryManager::allocate(const SegmentsRequestMap &Request) { return errorCodeToError(EC); // Zero out the zero-fill memory. - memset(static_cast<char *>(SegMem.base()) + ZeroFillStart, 0, + memset(static_cast<char *>(SegMem.base()) + Seg.getContentSize(), 0, Seg.getZeroFillSize()); // Record the block for this segment. diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp deleted file mode 100644 index c1040c942b2..00000000000 --- a/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.cpp +++ /dev/null @@ -1,412 +0,0 @@ -//=--------- MachOAtomGraphBuilder.cpp - MachO AtomGraph builder ----------===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Generic MachO AtomGraph buliding code. -// -//===----------------------------------------------------------------------===// - -#include "MachOAtomGraphBuilder.h" - -#define DEBUG_TYPE "jitlink" - -namespace llvm { -namespace jitlink { - -MachOAtomGraphBuilder::~MachOAtomGraphBuilder() {} - -Expected<std::unique_ptr<AtomGraph>> MachOAtomGraphBuilder::buildGraph() { - if (auto Err = parseSections()) - return std::move(Err); - - if (auto Err = addAtoms()) - return std::move(Err); - - if (auto Err = addRelocations()) - return std::move(Err); - - return std::move(G); -} - -MachOAtomGraphBuilder::MachOAtomGraphBuilder(const object::MachOObjectFile &Obj) - : Obj(Obj), - G(std::make_unique<AtomGraph>(Obj.getFileName(), getPointerSize(Obj), - getEndianness(Obj))) {} - -void MachOAtomGraphBuilder::addCustomAtomizer(StringRef SectionName, - CustomAtomizeFunction Atomizer) { - assert(!CustomAtomizeFunctions.count(SectionName) && - "Custom atomizer for this section already exists"); - CustomAtomizeFunctions[SectionName] = std::move(Atomizer); -} - -bool MachOAtomGraphBuilder::areLayoutLocked(const Atom &A, const Atom &B) { - // If these atoms are the same then they're trivially "locked". - if (&A == &B) - return true; - - // If A and B are different, check whether either is undefined. (in which - // case they are not locked). - if (!A.isDefined() || !B.isDefined()) - return false; - - // A and B are different, but they're both defined atoms. We need to check - // whether they're part of the same alt_entry chain. - auto &DA = static_cast<const DefinedAtom &>(A); - auto &DB = static_cast<const DefinedAtom &>(B); - - auto AStartItr = AltEntryStarts.find(&DA); - if (AStartItr == AltEntryStarts.end()) // If A is not in a chain bail out. - return false; - - auto BStartItr = AltEntryStarts.find(&DB); - if (BStartItr == AltEntryStarts.end()) // If B is not in a chain bail out. - return false; - - // A and B are layout locked if they're in the same chain. - return AStartItr->second == BStartItr->second; -} - -unsigned -MachOAtomGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { - return Obj.is64Bit() ? 8 : 4; -} - -support::endianness -MachOAtomGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { - return Obj.isLittleEndian() ? support::little : support::big; -} - -MachOAtomGraphBuilder::MachOSection &MachOAtomGraphBuilder::getCommonSection() { - if (!CommonSymbolsSection) { - auto Prot = static_cast<sys::Memory::ProtectionFlags>( - sys::Memory::MF_READ | sys::Memory::MF_WRITE); - auto &GenericSection = G->createSection("<common>", 1, Prot, true); - CommonSymbolsSection = MachOSection(GenericSection); - } - return *CommonSymbolsSection; -} - -Error MachOAtomGraphBuilder::parseSections() { - for (auto &SecRef : Obj.sections()) { - assert((SecRef.getAlignment() <= std::numeric_limits<uint32_t>::max()) && - "Section alignment does not fit in 32 bits"); - - Expected<StringRef> NameOrErr = SecRef.getName(); - if (!NameOrErr) - return NameOrErr.takeError(); - StringRef Name = *NameOrErr; - - unsigned SectionIndex = SecRef.getIndex() + 1; - - uint32_t Align = SecRef.getAlignment(); - if (!isPowerOf2_32(Align)) - return make_error<JITLinkError>("Section " + Name + - " has non-power-of-2 " - "alignment"); - - // FIXME: Get real section permissions - // How, exactly, on MachO? - sys::Memory::ProtectionFlags Prot; - if (SecRef.isText()) - Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_EXEC); - else - Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | - sys::Memory::MF_WRITE); - - auto &GenericSection = G->createSection(Name, Align, Prot, SecRef.isBSS()); - - LLVM_DEBUG({ - dbgs() << "Adding section " << Name << ": " - << format("0x%016" PRIx64, SecRef.getAddress()) - << ", align: " << SecRef.getAlignment() << "\n"; - }); - - assert(!Sections.count(SectionIndex) && "Section index already in use"); - - auto &MachOSec = - Sections - .try_emplace(SectionIndex, GenericSection, SecRef.getAddress(), - SecRef.getAlignment()) - .first->second; - - if (!SecRef.isVirtual()) { - // If this section has content then record it. - Expected<StringRef> Content = SecRef.getContents(); - if (!Content) - return Content.takeError(); - if (Content->size() != SecRef.getSize()) - return make_error<JITLinkError>("Section content size does not match " - "declared size for " + - Name); - MachOSec.setContent(*Content); - } else { - // If this is a zero-fill section then just record the size. - MachOSec.setZeroFill(SecRef.getSize()); - } - - uint32_t SectionFlags = - Obj.is64Bit() ? Obj.getSection64(SecRef.getRawDataRefImpl()).flags - : Obj.getSection(SecRef.getRawDataRefImpl()).flags; - - MachOSec.setNoDeadStrip(SectionFlags & MachO::S_ATTR_NO_DEAD_STRIP); - } - - return Error::success(); -} - -// Adds atoms with identified start addresses (but not lengths) for all named -// atoms. -// Also, for every section that contains named atoms, but does not have an -// atom at offset zero of that section, constructs an anonymous atom covering -// that range. -Error MachOAtomGraphBuilder::addNonCustomAtoms() { - using AddrToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; - DenseMap<MachOSection *, AddrToAtomMap> SecToAtoms; - - DenseMap<MachOSection *, unsigned> FirstOrdinal; - std::vector<DefinedAtom *> AltEntryAtoms; - - DenseSet<StringRef> ProcessedSymbols; // Used to check for duplicate defs. - - for (auto SymI = Obj.symbol_begin(), SymE = Obj.symbol_end(); SymI != SymE; - ++SymI) { - object::SymbolRef Sym(SymI->getRawDataRefImpl(), &Obj); - - auto Name = Sym.getName(); - if (!Name) - return Name.takeError(); - - // Bail out on duplicate definitions: There should never be more than one - // definition for a symbol in a given object file. - if (ProcessedSymbols.count(*Name)) - return make_error<JITLinkError>("Duplicate definition within object: " + - *Name); - else - ProcessedSymbols.insert(*Name); - - auto Addr = Sym.getAddress(); - if (!Addr) - return Addr.takeError(); - - auto SymType = Sym.getType(); - if (!SymType) - return SymType.takeError(); - - auto Flags = Sym.getFlags(); - - if (Flags & object::SymbolRef::SF_Undefined) { - LLVM_DEBUG(dbgs() << "Adding undef atom \"" << *Name << "\"\n"); - G->addExternalAtom(*Name); - continue; - } else if (Flags & object::SymbolRef::SF_Absolute) { - LLVM_DEBUG(dbgs() << "Adding absolute \"" << *Name << "\" addr: " - << format("0x%016" PRIx64, *Addr) << "\n"); - auto &A = G->addAbsoluteAtom(*Name, *Addr); - A.setGlobal(Flags & object::SymbolRef::SF_Global); - A.setExported(Flags & object::SymbolRef::SF_Exported); - A.setWeak(Flags & object::SymbolRef::SF_Weak); - continue; - } else if (Flags & object::SymbolRef::SF_Common) { - LLVM_DEBUG({ - dbgs() << "Adding common \"" << *Name - << "\" addr: " << format("0x%016" PRIx64, *Addr) << "\n"; - }); - auto &A = - G->addCommonAtom(getCommonSection().getGenericSection(), *Name, *Addr, - std::max(Sym.getAlignment(), 1U), - Obj.getCommonSymbolSize(Sym.getRawDataRefImpl())); - A.setGlobal(Flags & object::SymbolRef::SF_Global); - A.setExported(Flags & object::SymbolRef::SF_Exported); - continue; - } - - LLVM_DEBUG(dbgs() << "Adding defined atom \"" << *Name << "\"\n"); - - // This atom is neither undefined nor absolute, so it must be defined in - // this object. Get its section index. - auto SecItr = Sym.getSection(); - if (!SecItr) - return SecItr.takeError(); - - uint64_t SectionIndex = (*SecItr)->getIndex() + 1; - - LLVM_DEBUG(dbgs() << " to section index " << SectionIndex << "\n"); - - auto SecByIndexItr = Sections.find(SectionIndex); - if (SecByIndexItr == Sections.end()) - return make_error<JITLinkError>("Unrecognized section index in macho"); - - auto &Sec = SecByIndexItr->second; - - auto &DA = G->addDefinedAtom(Sec.getGenericSection(), *Name, *Addr, - std::max(Sym.getAlignment(), 1U)); - - DA.setGlobal(Flags & object::SymbolRef::SF_Global); - DA.setExported(Flags & object::SymbolRef::SF_Exported); - DA.setWeak(Flags & object::SymbolRef::SF_Weak); - - DA.setCallable(*SymType & object::SymbolRef::ST_Function); - - // Check NDesc flags. - { - uint16_t NDesc = 0; - if (Obj.is64Bit()) - NDesc = Obj.getSymbol64TableEntry(SymI->getRawDataRefImpl()).n_desc; - else - NDesc = Obj.getSymbolTableEntry(SymI->getRawDataRefImpl()).n_desc; - - // Record atom for alt-entry post-processing (where the layout-next - // constraints will be added). - if (NDesc & MachO::N_ALT_ENTRY) - AltEntryAtoms.push_back(&DA); - - // If this atom has a no-dead-strip attr attached then mark it live. - if (NDesc & MachO::N_NO_DEAD_STRIP) - DA.setLive(true); - } - - LLVM_DEBUG({ - dbgs() << " Added " << *Name - << " addr: " << format("0x%016" PRIx64, *Addr) - << ", align: " << DA.getAlignment() - << ", section: " << Sec.getGenericSection().getName() << "\n"; - }); - - auto &SecAtoms = SecToAtoms[&Sec]; - SecAtoms[DA.getAddress() - Sec.getAddress()] = &DA; - } - - // Add anonymous atoms. - for (auto &KV : Sections) { - auto &S = KV.second; - - // Skip empty sections. - if (S.empty()) - continue; - - // Skip sections with custom handling. - if (CustomAtomizeFunctions.count(S.getName())) - continue; - - auto SAI = SecToAtoms.find(&S); - - // If S is not in the SecToAtoms map then it contained no named atom. Add - // one anonymous atom to cover the whole section. - if (SAI == SecToAtoms.end()) { - SecToAtoms[&S][0] = &G->addAnonymousAtom( - S.getGenericSection(), S.getAddress(), S.getAlignment()); - continue; - } - - // Otherwise, check whether this section had an atom covering offset zero. - // If not, add one. - auto &SecAtoms = SAI->second; - if (!SecAtoms.count(0)) - SecAtoms[0] = &G->addAnonymousAtom(S.getGenericSection(), S.getAddress(), - S.getAlignment()); - } - - LLVM_DEBUG(dbgs() << "MachOGraphBuilder setting atom content\n"); - - // Set atom contents and any section-based flags. - for (auto &KV : SecToAtoms) { - auto &S = *KV.first; - auto &SecAtoms = KV.second; - - // Iterate the atoms in reverse order and set up their contents. - JITTargetAddress LastAtomAddr = S.getSize(); - for (auto I = SecAtoms.rbegin(), E = SecAtoms.rend(); I != E; ++I) { - auto Offset = I->first; - auto &A = *I->second; - LLVM_DEBUG({ - dbgs() << " " << A << " to [ " << S.getAddress() + Offset << " .. " - << S.getAddress() + LastAtomAddr << " ]\n"; - }); - - if (S.isZeroFill()) - A.setZeroFill(LastAtomAddr - Offset); - else - A.setContent(S.getContent().substr(Offset, LastAtomAddr - Offset)); - - // If the section has no-dead-strip set then mark the atom as live. - if (S.isNoDeadStrip()) - A.setLive(true); - - LastAtomAddr = Offset; - } - } - - LLVM_DEBUG(dbgs() << "Adding alt-entry starts\n"); - - // Sort alt-entry atoms by address in ascending order. - llvm::sort(AltEntryAtoms.begin(), AltEntryAtoms.end(), - [](const DefinedAtom *LHS, const DefinedAtom *RHS) { - return LHS->getAddress() < RHS->getAddress(); - }); - - // Process alt-entry atoms in address order to build the table of alt-entry - // atoms to alt-entry chain starts. - for (auto *DA : AltEntryAtoms) { - assert(!AltEntryStarts.count(DA) && "Duplicate entry in AltEntryStarts"); - - // DA is an alt-entry atom. Look for the predecessor atom that it is locked - // to, bailing out if we do not find one. - auto AltEntryPred = G->findAtomByAddress(DA->getAddress() - 1); - if (!AltEntryPred) - return AltEntryPred.takeError(); - - // Add a LayoutNext edge from the predecessor to this atom. - AltEntryPred->setLayoutNext(*DA); - - // Check to see whether the predecessor itself is an alt-entry atom. - auto AltEntryStartItr = AltEntryStarts.find(&*AltEntryPred); - if (AltEntryStartItr != AltEntryStarts.end()) { - // If the predecessor was an alt-entry atom then re-use its value. - LLVM_DEBUG({ - dbgs() << " " << *DA << " -> " << *AltEntryStartItr->second - << " (based on existing entry for " << *AltEntryPred << ")\n"; - }); - AltEntryStarts[DA] = AltEntryStartItr->second; - } else { - // If the predecessor does not have an entry then add an entry for this - // atom (i.e. the alt_entry atom) and a self-reference entry for the - /// predecessory atom that is the start of this chain. - LLVM_DEBUG({ - dbgs() << " " << *AltEntryPred << " -> " << *AltEntryPred << "\n" - << " " << *DA << " -> " << *AltEntryPred << "\n"; - }); - AltEntryStarts[&*AltEntryPred] = &*AltEntryPred; - AltEntryStarts[DA] = &*AltEntryPred; - } - } - - return Error::success(); -} - -Error MachOAtomGraphBuilder::addAtoms() { - // Add all named atoms. - if (auto Err = addNonCustomAtoms()) - return Err; - - // Process special sections. - for (auto &KV : Sections) { - auto &S = KV.second; - auto HI = CustomAtomizeFunctions.find(S.getGenericSection().getName()); - if (HI != CustomAtomizeFunctions.end()) { - auto &Atomize = HI->second; - if (auto Err = Atomize(S)) - return Err; - } - } - - return Error::success(); -} - -} // end namespace jitlink -} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h deleted file mode 100644 index 72d441b24d0..00000000000 --- a/llvm/lib/ExecutionEngine/JITLink/MachOAtomGraphBuilder.h +++ /dev/null @@ -1,138 +0,0 @@ -//===----- MachOAtomGraphBuilder.h - MachO AtomGraph builder ----*- C++ -*-===// -// -// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. -// See https://llvm.org/LICENSE.txt for license information. -// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception -// -//===----------------------------------------------------------------------===// -// -// Generic MachO AtomGraph building code. -// -//===----------------------------------------------------------------------===// - -#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H -#define LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H - -#include "llvm/ExecutionEngine/JITLink/JITLink.h" - -#include "JITLinkGeneric.h" - -#include "llvm/Object/MachO.h" - -namespace llvm { -namespace jitlink { - -class MachOAtomGraphBuilder { -public: - virtual ~MachOAtomGraphBuilder(); - Expected<std::unique_ptr<AtomGraph>> buildGraph(); - -protected: - using OffsetToAtomMap = std::map<JITTargetAddress, DefinedAtom *>; - - class MachOSection { - public: - MachOSection() = default; - - /// Create a MachO section with the given address and alignment. - MachOSection(Section &GenericSection, JITTargetAddress Address, - unsigned Alignment) - : Address(Address), GenericSection(&GenericSection), - Alignment(Alignment) {} - - /// Create a section without address, content or size (used for common - /// symbol sections). - MachOSection(Section &GenericSection) : GenericSection(&GenericSection) {} - - Section &getGenericSection() const { - assert(GenericSection && "Section is null"); - return *GenericSection; - } - - StringRef getName() const { - assert(GenericSection && "No generic section attached"); - return GenericSection->getName(); - } - - MachOSection &setContent(StringRef Content) { - assert(!ContentPtr && !Size && "Content/zeroFill already set"); - ContentPtr = Content.data(); - Size = Content.size(); - return *this; - } - - MachOSection &setZeroFill(uint64_t Size) { - assert(!ContentPtr && !this->Size && "Content/zeroFill already set"); - this->Size = Size; - return *this; - } - - bool isZeroFill() const { return !ContentPtr; } - - bool empty() const { return getSize() == 0; } - - size_t getSize() const { return Size; } - - StringRef getContent() const { - assert(ContentPtr && "getContent() called on zero-fill section"); - return {ContentPtr, static_cast<size_t>(Size)}; - } - - JITTargetAddress getAddress() const { return Address; } - - unsigned getAlignment() const { return Alignment; } - - MachOSection &setNoDeadStrip(bool NoDeadStrip) { - this->NoDeadStrip = NoDeadStrip; - return *this; - } - - bool isNoDeadStrip() const { return NoDeadStrip; } - - private: - JITTargetAddress Address = 0; - Section *GenericSection = nullptr; - const char *ContentPtr = nullptr; - uint64_t Size = 0; - unsigned Alignment = 0; - bool NoDeadStrip = false; - }; - - using CustomAtomizeFunction = std::function<Error(MachOSection &S)>; - - MachOAtomGraphBuilder(const object::MachOObjectFile &Obj); - - AtomGraph &getGraph() const { return *G; } - - const object::MachOObjectFile &getObject() const { return Obj; } - - void addCustomAtomizer(StringRef SectionName, CustomAtomizeFunction Atomizer); - - virtual Error addRelocations() = 0; - - /// Returns true if Atom A and Atom B are at a fixed offset from one another - /// (i.e. if they're part of the same alt-entry chain). - bool areLayoutLocked(const Atom &A, const Atom &B); - -private: - static unsigned getPointerSize(const object::MachOObjectFile &Obj); - static support::endianness getEndianness(const object::MachOObjectFile &Obj); - - MachOSection &getCommonSection(); - - Error parseSections(); - Error addNonCustomAtoms(); - Error addAtoms(); - - const object::MachOObjectFile &Obj; - std::unique_ptr<AtomGraph> G; - DenseMap<const DefinedAtom *, const DefinedAtom *> AltEntryStarts; - DenseMap<unsigned, MachOSection> Sections; - StringMap<CustomAtomizeFunction> CustomAtomizeFunctions; - Optional<MachOSection> CommonSymbolsSection; -}; - -} // end namespace jitlink -} // end namespace llvm - -#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOATOMGRAPHBUILDER_H diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp new file mode 100644 index 00000000000..57fbc699757 --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.cpp @@ -0,0 +1,535 @@ +//=--------- MachOLinkGraphBuilder.cpp - MachO LinkGraph builder ----------===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph buliding code. +// +//===----------------------------------------------------------------------===// + +#include "MachOLinkGraphBuilder.h" + +#define DEBUG_TYPE "jitlink" + +static const char *CommonSectionName = "__common"; + +namespace llvm { +namespace jitlink { + +MachOLinkGraphBuilder::~MachOLinkGraphBuilder() {} + +Expected<std::unique_ptr<LinkGraph>> MachOLinkGraphBuilder::buildGraph() { + + // Sanity check: we only operate on relocatable objects. + if (!Obj.isRelocatableObject()) + return make_error<JITLinkError>("Object is not a relocatable MachO"); + + if (auto Err = createNormalizedSections()) + return std::move(Err); + + if (auto Err = createNormalizedSymbols()) + return std::move(Err); + + if (auto Err = graphifyRegularSymbols()) + return std::move(Err); + + if (auto Err = graphifySectionsWithCustomParsers()) + return std::move(Err); + + if (auto Err = addRelocations()) + return std::move(Err); + + return std::move(G); +} + +MachOLinkGraphBuilder::MachOLinkGraphBuilder(const object::MachOObjectFile &Obj) + : Obj(Obj), + G(std::make_unique<LinkGraph>(Obj.getFileName(), getPointerSize(Obj), + getEndianness(Obj))) {} + +void MachOLinkGraphBuilder::addCustomSectionParser( + StringRef SectionName, SectionParserFunction Parser) { + assert(!CustomSectionParserFunctions.count(SectionName) && + "Custom parser for this section already exists"); + CustomSectionParserFunctions[SectionName] = std::move(Parser); +} + +Linkage MachOLinkGraphBuilder::getLinkage(uint16_t Desc) { + if ((Desc & MachO::N_WEAK_DEF) || (Desc & MachO::N_WEAK_REF)) + return Linkage::Weak; + return Linkage::Strong; +} + +Scope MachOLinkGraphBuilder::getScope(StringRef Name, uint8_t Type) { + if (Name.startswith("l")) + return Scope::Local; + if (Type & MachO::N_PEXT) + return Scope::Hidden; + if (Type & MachO::N_EXT) + return Scope::Default; + return Scope::Local; +} + +bool MachOLinkGraphBuilder::isAltEntry(const NormalizedSymbol &NSym) { + return NSym.Desc & MachO::N_ALT_ENTRY; +} + +unsigned +MachOLinkGraphBuilder::getPointerSize(const object::MachOObjectFile &Obj) { + return Obj.is64Bit() ? 8 : 4; +} + +support::endianness +MachOLinkGraphBuilder::getEndianness(const object::MachOObjectFile &Obj) { + return Obj.isLittleEndian() ? support::little : support::big; +} + +Section &MachOLinkGraphBuilder::getCommonSection() { + if (!CommonSection) { + auto Prot = static_cast<sys::Memory::ProtectionFlags>( + sys::Memory::MF_READ | sys::Memory::MF_WRITE); + CommonSection = &G->createSection(CommonSectionName, Prot); + } + return *CommonSection; +} + +Error MachOLinkGraphBuilder::createNormalizedSections() { + // Build normalized sections. Verifies that section data is in-range (for + // sections with content) and that address ranges are non-overlapping. + + LLVM_DEBUG(dbgs() << "Creating normalized sections...\n"); + + for (auto &SecRef : Obj.sections()) { + NormalizedSection NSec; + uint32_t DataOffset = 0; + + auto SecIndex = Obj.getSectionIndex(SecRef.getRawDataRefImpl()); + + auto Name = SecRef.getName(); + if (!Name) + return Name.takeError(); + + if (Obj.is64Bit()) { + const MachO::section_64 &Sec64 = + Obj.getSection64(SecRef.getRawDataRefImpl()); + + NSec.Address = Sec64.addr; + NSec.Size = Sec64.size; + NSec.Alignment = 1ULL << Sec64.align; + NSec.Flags = Sec64.flags; + DataOffset = Sec64.offset; + } else { + const MachO::section &Sec32 = Obj.getSection(SecRef.getRawDataRefImpl()); + NSec.Address = Sec32.addr; + NSec.Size = Sec32.size; + NSec.Alignment = 1ULL << Sec32.align; + NSec.Flags = Sec32.flags; + DataOffset = Sec32.offset; + } + + LLVM_DEBUG({ + dbgs() << " " << *Name << ": " << formatv("{0:x16}", NSec.Address) + << " -- " << formatv("{0:x16}", NSec.Address + NSec.Size) + << ", align: " << NSec.Alignment << ", index: " << SecIndex + << "\n"; + }); + + // Get the section data if any. + { + unsigned SectionType = NSec.Flags & MachO::SECTION_TYPE; + if (SectionType != MachO::S_ZEROFILL && + SectionType != MachO::S_GB_ZEROFILL) { + + if (DataOffset + NSec.Size > Obj.getData().size()) + return make_error<JITLinkError>( + "Section data extends past end of file"); + + NSec.Data = Obj.getData().data() + DataOffset; + } + } + + // Get prot flags. + // FIXME: Make sure this test is correct (it's probably missing cases + // as-is). + sys::Memory::ProtectionFlags Prot; + if (NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS) + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_EXEC); + else + Prot = static_cast<sys::Memory::ProtectionFlags>(sys::Memory::MF_READ | + sys::Memory::MF_WRITE); + + NSec.GraphSection = &G->createSection(*Name, Prot); + IndexToSection.insert(std::make_pair(SecIndex, std::move(NSec))); + } + + std::vector<NormalizedSection *> Sections; + Sections.reserve(IndexToSection.size()); + for (auto &KV : IndexToSection) + Sections.push_back(&KV.second); + + // If we didn't end up creating any sections then bail out. The code below + // assumes that we have at least one section. + if (Sections.empty()) + return Error::success(); + + llvm::sort(Sections, + [](const NormalizedSection *LHS, const NormalizedSection *RHS) { + assert(LHS && RHS && "Null section?"); + return LHS->Address < RHS->Address; + }); + + for (unsigned I = 0, E = Sections.size() - 1; I != E; ++I) { + auto &Cur = *Sections[I]; + auto &Next = *Sections[I + 1]; + if (Next.Address < Cur.Address + Cur.Size) + return make_error<JITLinkError>( + "Address range for section " + Cur.GraphSection->getName() + + formatv(" [ {0:x16} -- {1:x16} ] ", Cur.Address, + Cur.Address + Cur.Size) + + "overlaps " + + formatv(" [ {0:x16} -- {1:x16} ] ", Next.Address, + Next.Address + Next.Size)); + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::createNormalizedSymbols() { + LLVM_DEBUG(dbgs() << "Creating normalized symbols...\n"); + + for (auto &SymRef : Obj.symbols()) { + + unsigned SymbolIndex = Obj.getSymbolIndex(SymRef.getRawDataRefImpl()); + uint64_t Value; + uint32_t NStrX; + uint8_t Type; + uint8_t Sect; + uint16_t Desc; + + if (Obj.is64Bit()) { + const MachO::nlist_64 &NL64 = + Obj.getSymbol64TableEntry(SymRef.getRawDataRefImpl()); + Value = NL64.n_value; + NStrX = NL64.n_strx; + Type = NL64.n_type; + Sect = NL64.n_sect; + Desc = NL64.n_desc; + } else { + const MachO::nlist &NL32 = + Obj.getSymbolTableEntry(SymRef.getRawDataRefImpl()); + Value = NL32.n_value; + NStrX = NL32.n_strx; + Type = NL32.n_type; + Sect = NL32.n_sect; + Desc = NL32.n_desc; + } + + // Skip stabs. + // FIXME: Are there other symbols we should be skipping? + if (Type & MachO::N_STAB) + continue; + + Optional<StringRef> Name; + if (NStrX) { + if (auto NameOrErr = SymRef.getName()) + Name = *NameOrErr; + else + return NameOrErr.takeError(); + } + + LLVM_DEBUG({ + dbgs() << " "; + if (!Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << *Name; + dbgs() << ": value = " << formatv("{0:x16}", Value) + << ", type = " << formatv("{0:x2}", Type) + << ", desc = " << formatv("{0:x4}", Desc) << ", sect = "; + if (Sect) + dbgs() << static_cast<unsigned>(Sect - 1); + else + dbgs() << "none"; + dbgs() << "\n"; + }); + + // If this symbol has a section, sanity check that the addresses line up. + NormalizedSection *NSec = nullptr; + if (Sect != 0) { + if (auto NSecOrErr = findSectionByIndex(Sect - 1)) + NSec = &*NSecOrErr; + else + return NSecOrErr.takeError(); + + if (Value < NSec->Address || Value > NSec->Address + NSec->Size) + return make_error<JITLinkError>("Symbol address does not fall within " + "section"); + } + + IndexToSymbol[SymbolIndex] = + &createNormalizedSymbol(*Name, Value, Type, Sect, Desc, + getLinkage(Type), getScope(*Name, Type)); + } + + return Error::success(); +} + +void MachOLinkGraphBuilder::addSectionStartSymAndBlock( + Section &GraphSec, uint64_t Address, const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive) { + Block &B = + Data ? G->createContentBlock(GraphSec, StringRef(Data, Size), Address, + Alignment, 0) + : G->createZeroFillBlock(GraphSec, Size, Address, Alignment, 0); + auto &Sym = G->addAnonymousSymbol(B, 0, Size, false, IsLive); + assert(!AddrToCanonicalSymbol.count(Sym.getAddress()) && + "Anonymous block start symbol clashes with existing symbol address"); + AddrToCanonicalSymbol[Sym.getAddress()] = &Sym; +} + +Error MachOLinkGraphBuilder::graphifyRegularSymbols() { + + LLVM_DEBUG(dbgs() << "Creating graph symbols...\n"); + + /// We only have 256 section indexes: Use a vector rather than a map. + std::vector<std::vector<NormalizedSymbol *>> SecIndexToSymbols; + SecIndexToSymbols.resize(256); + + // Create commons, externs, and absolutes, and partition all other symbols by + // section. + for (auto &KV : IndexToSymbol) { + auto &NSym = *KV.second; + + switch (NSym.Type & MachO::N_TYPE) { + case MachO::N_UNDF: + if (NSym.Value) { + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous common symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addCommonSymbol( + *NSym.Name, NSym.S, getCommonSection(), NSym.Value, 0, + 1U << MachO::GET_COMM_ALIGN(NSym.Desc), + NSym.Desc & MachO::N_NO_DEAD_STRIP); + } else { + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous external symbol at " + "index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addExternalSymbol(*NSym.Name, 0); + } + break; + case MachO::N_ABS: + if (!NSym.Name) + return make_error<JITLinkError>("Anonymous absolute symbol at index " + + Twine(KV.first)); + NSym.GraphSymbol = &G->addAbsoluteSymbol( + *NSym.Name, NSym.Value, 0, Linkage::Strong, Scope::Default, + NSym.Desc & MachO::N_NO_DEAD_STRIP); + break; + case MachO::N_SECT: + SecIndexToSymbols[NSym.Sect - 1].push_back(&NSym); + break; + case MachO::N_PBUD: + return make_error<JITLinkError>( + "Unupported N_PBUD symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + case MachO::N_INDR: + return make_error<JITLinkError>( + "Unupported N_INDR symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + default: + return make_error<JITLinkError>( + "Unrecognized symbol type " + Twine(NSym.Type & MachO::N_TYPE) + + " for symbol " + + (NSym.Name ? ("\"" + *NSym.Name + "\"") : Twine("<anon>")) + + " at index " + Twine(KV.first)); + } + } + + // Loop over sections performing regular graphification for those that + // don't have custom parsers. + for (auto &KV : IndexToSection) { + auto SecIndex = KV.first; + auto &NSec = KV.second; + + // Skip sections with custom parsers. + if (CustomSectionParserFunctions.count(NSec.GraphSection->getName())) { + LLVM_DEBUG({ + dbgs() << " Skipping section " << NSec.GraphSection->getName() + << " as it has a custom parser.\n"; + }); + continue; + } else + LLVM_DEBUG({ + dbgs() << " Processing section " << NSec.GraphSection->getName() + << "...\n"; + }); + + bool SectionIsNoDeadStrip = NSec.Flags & MachO::S_ATTR_NO_DEAD_STRIP; + bool SectionIsText = NSec.Flags & MachO::S_ATTR_PURE_INSTRUCTIONS; + + auto &SecNSymStack = SecIndexToSymbols[SecIndex]; + + // If this section is non-empty but there are no symbols covering it then + // create one block and anonymous symbol to cover the entire section. + if (SecNSymStack.empty()) { + if (NSec.Size > 0) { + LLVM_DEBUG({ + dbgs() << " Section non-empty, but contains no symbols. " + "Creating anonymous block to cover " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + NSec.Size) << "\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + NSec.Size, NSec.Alignment, + SectionIsNoDeadStrip); + } else + LLVM_DEBUG({ + dbgs() << " Section empty and contains no symbols. Skipping.\n"; + }); + continue; + } + + // Sort the symbol stack in by address, alt-entry status, scope, and name. + // We sort in reverse order so that symbols will be visited in the right + // order when we pop off the stack below. + llvm::sort(SecNSymStack, [](const NormalizedSymbol *LHS, + const NormalizedSymbol *RHS) { + if (LHS->Value != RHS->Value) + return LHS->Value > RHS->Value; + if (isAltEntry(*LHS) != isAltEntry(*RHS)) + return isAltEntry(*RHS); + if (LHS->S != RHS->S) + return static_cast<uint8_t>(LHS->S) < static_cast<uint8_t>(RHS->S); + return LHS->Name < RHS->Name; + }); + + // The first symbol in a section can not be an alt-entry symbol. + if (!SecNSymStack.empty() && isAltEntry(*SecNSymStack.back())) + return make_error<JITLinkError>( + "First symbol in " + NSec.GraphSection->getName() + " is alt-entry"); + + // If the section is non-empty but there is no symbol covering the start + // address then add an anonymous one. + if (SecNSymStack.back()->Value != NSec.Address) { + auto AnonBlockSize = SecNSymStack.back()->Value - NSec.Address; + LLVM_DEBUG({ + dbgs() << " Section start not covered by symbol. " + << "Creating anonymous block to cover [ " + << formatv("{0:x16}", NSec.Address) << " -- " + << formatv("{0:x16}", NSec.Address + AnonBlockSize) << " ]\n"; + }); + addSectionStartSymAndBlock(*NSec.GraphSection, NSec.Address, NSec.Data, + AnonBlockSize, NSec.Alignment, + SectionIsNoDeadStrip); + } + + // Visit section symbols in order by popping off the reverse-sorted stack, + // building blocks for each alt-entry chain and creating symbols as we go. + while (!SecNSymStack.empty()) { + SmallVector<NormalizedSymbol *, 8> BlockSyms; + + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + while (!SecNSymStack.empty() && + (isAltEntry(*SecNSymStack.back()) || + SecNSymStack.back()->Value == BlockSyms.back()->Value)) { + BlockSyms.push_back(SecNSymStack.back()); + SecNSymStack.pop_back(); + } + + // BlockNSyms now contains the block symbols in reverse canonical order. + JITTargetAddress BlockStart = BlockSyms.front()->Value; + JITTargetAddress BlockEnd = SecNSymStack.empty() + ? NSec.Address + NSec.Size + : SecNSymStack.back()->Value; + JITTargetAddress BlockOffset = BlockStart - NSec.Address; + JITTargetAddress BlockSize = BlockEnd - BlockStart; + + LLVM_DEBUG({ + dbgs() << " Creating block for " << formatv("{0:x16}", BlockStart) + << " -- " << formatv("{0:x16}", BlockEnd) << ": " + << NSec.GraphSection->getName() << " + " + << formatv("{0:x16}", BlockOffset) << " with " + << BlockSyms.size() << " symbol(s)...\n"; + }); + + Block &B = + NSec.Data + ? G->createContentBlock( + *NSec.GraphSection, + StringRef(NSec.Data + BlockOffset, BlockSize), BlockStart, + NSec.Alignment, BlockStart % NSec.Alignment) + : G->createZeroFillBlock(*NSec.GraphSection, BlockSize, + BlockStart, NSec.Alignment, + BlockStart % NSec.Alignment); + + Optional<JITTargetAddress> LastCanonicalAddr; + JITTargetAddress SymEnd = BlockEnd; + while (!BlockSyms.empty()) { + auto &NSym = *BlockSyms.back(); + BlockSyms.pop_back(); + + bool SymLive = + (NSym.Desc & MachO::N_NO_DEAD_STRIP) || SectionIsNoDeadStrip; + + LLVM_DEBUG({ + dbgs() << " " << formatv("{0:x16}", NSym.Value) << " -- " + << formatv("{0:x16}", SymEnd) << ": "; + if (!NSym.Name) + dbgs() << "<anonymous symbol>"; + else + dbgs() << NSym.Name; + if (SymLive) + dbgs() << " [no-dead-strip]"; + if (LastCanonicalAddr == NSym.Value) + dbgs() << " [non-canonical]"; + dbgs() << "\n"; + }); + + auto &Sym = + NSym.Name + ? G->addDefinedSymbol(B, NSym.Value - BlockStart, *NSym.Name, + SymEnd - NSym.Value, NSym.L, NSym.S, + SectionIsText, SymLive) + : G->addAnonymousSymbol(B, NSym.Value - BlockStart, + SymEnd - NSym.Value, SectionIsText, + SymLive); + NSym.GraphSymbol = &Sym; + if (LastCanonicalAddr != Sym.getAddress()) { + if (LastCanonicalAddr) + SymEnd = *LastCanonicalAddr; + LastCanonicalAddr = Sym.getAddress(); + setCanonicalSymbol(Sym); + } + } + } + } + + return Error::success(); +} + +Error MachOLinkGraphBuilder::graphifySectionsWithCustomParsers() { + // Graphify special sections. + for (auto &KV : IndexToSection) { + auto &NSec = KV.second; + + auto HI = CustomSectionParserFunctions.find(NSec.GraphSection->getName()); + if (HI != CustomSectionParserFunctions.end()) { + auto &Parse = HI->second; + if (auto Err = Parse(NSec)) + return Err; + } + } + + return Error::success(); +} + +} // end namespace jitlink +} // end namespace llvm diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h new file mode 100644 index 00000000000..cba26e9494b --- /dev/null +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -0,0 +1,269 @@ +//===----- MachOLinkGraphBuilder.h - MachO LinkGraph builder ----*- C++ -*-===// +// +// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. +// See https://llvm.org/LICENSE.txt for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +//===----------------------------------------------------------------------===// +// +// Generic MachO LinkGraph building code. +// +//===----------------------------------------------------------------------===// + +#ifndef LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H +#define LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H + +#include "llvm/ExecutionEngine/JITLink/JITLink.h" + +#include "EHFrameSupportImpl.h" +#include "JITLinkGeneric.h" +#include "llvm/Object/MachO.h" + +#include <list> + +namespace llvm { +namespace jitlink { + +class MachOLinkGraphBuilder { +public: + virtual ~MachOLinkGraphBuilder(); + Expected<std::unique_ptr<LinkGraph>> buildGraph(); + +protected: + class MachOEHFrameBinaryParser : public EHFrameBinaryParser { + public: + MachOEHFrameBinaryParser(MachOLinkGraphBuilder &Builder, + JITTargetAddress EHFrameAddress, + StringRef EHFrameContent, Section &EHFrameSection, + uint64_t CIEAlignment, uint64_t FDEAlignment, + Edge::Kind FDEToCIERelocKind, + Edge::Kind FDEToTargetRelocKind) + : EHFrameBinaryParser(EHFrameAddress, EHFrameContent, + Builder.getGraph().getPointerSize(), + Builder.getGraph().getEndianness()), + Builder(Builder), EHFrameSection(EHFrameSection), + CIEAlignment(CIEAlignment), FDEAlignment(FDEAlignment), + FDEToCIERelocKind(FDEToCIERelocKind), + FDEToTargetRelocKind(FDEToTargetRelocKind) {} + + Symbol *getSymbolAtAddress(JITTargetAddress Address) override { + if (auto *Sym = Builder.getSymbolByAddress(Address)) + if (Sym->getAddress() == Address) + return Sym; + return nullptr; + } + + Symbol &createCIERecord(JITTargetAddress RecordAddr, + StringRef RecordContent) override { + auto &G = Builder.getGraph(); + auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, + CIEAlignment, 0); + auto &CIESymbol = + G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); + Builder.setCanonicalSymbol(CIESymbol); + return CIESymbol; + } + + Expected<Symbol &> createFDERecord(JITTargetAddress RecordAddr, + StringRef RecordContent, Symbol &CIE, + size_t CIEOffset, Symbol &Func, + size_t FuncOffset, Symbol *LSDA, + size_t LSDAOffset) override { + auto &G = Builder.getGraph(); + auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, + FDEAlignment, 0); + + // Add edges to CIE, Func, and (conditionally) LSDA. + B.addEdge(FDEToCIERelocKind, CIEOffset, CIE, 0); + B.addEdge(FDEToTargetRelocKind, FuncOffset, Func, 0); + + if (LSDA) + B.addEdge(FDEToTargetRelocKind, LSDAOffset, *LSDA, 0); + + auto &FDESymbol = + G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); + + // Add a keep-alive relocation from the function to the FDE to ensure it + // is not dead stripped. + Func.getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + + return FDESymbol; + } + + private: + MachOLinkGraphBuilder &Builder; + Section &EHFrameSection; + uint64_t CIEAlignment; + uint64_t FDEAlignment; + Edge::Kind FDEToCIERelocKind; + Edge::Kind FDEToTargetRelocKind; + }; + + struct NormalizedSymbol { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSymbol(Optional<StringRef> Name, uint64_t Value, uint8_t Type, + uint8_t Sect, uint16_t Desc, Linkage L, Scope S) + : Name(Name), Value(Value), Type(Type), Sect(Sect), Desc(Desc), L(L), + S(S) { + assert(!Name || !Name->empty() && "Name must be none or non-empty"); + } + + public: + NormalizedSymbol(const NormalizedSymbol &) = delete; + NormalizedSymbol &operator=(const NormalizedSymbol &) = delete; + NormalizedSymbol(NormalizedSymbol &&) = delete; + NormalizedSymbol &operator=(NormalizedSymbol &&) = delete; + + Optional<StringRef> Name; + uint64_t Value = 0; + uint8_t Type = 0; + uint8_t Sect = 0; + uint16_t Desc = 0; + Linkage L = Linkage::Strong; + Scope S = Scope::Default; + Symbol *GraphSymbol = nullptr; + }; + + class NormalizedSection { + friend class MachOLinkGraphBuilder; + + private: + NormalizedSection() = default; + + public: + Section *GraphSection = nullptr; + uint64_t Address = 0; + uint64_t Size = 0; + uint64_t Alignment = 0; + uint32_t Flags = 0; + const char *Data = nullptr; + }; + + using SectionParserFunction = std::function<Error(NormalizedSection &S)>; + + MachOLinkGraphBuilder(const object::MachOObjectFile &Obj); + + LinkGraph &getGraph() const { return *G; } + + const object::MachOObjectFile &getObject() const { return Obj; } + + void addCustomSectionParser(StringRef SectionName, + SectionParserFunction Parse); + + virtual Error addRelocations() = 0; + + /// Create a symbol. + template <typename... ArgTs> + NormalizedSymbol &createNormalizedSymbol(ArgTs &&... Args) { + NormalizedSymbol *Sym = reinterpret_cast<NormalizedSymbol *>( + Allocator.Allocate<NormalizedSymbol>()); + new (Sym) NormalizedSymbol(std::forward<ArgTs>(Args)...); + return *Sym; + } + + /// Index is zero-based (MachO section indexes are usually one-based) and + /// assumed to be in-range. Client is responsible for checking. + NormalizedSection &getSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + assert(I != IndexToSection.end() && "No section recorded at index"); + return I->second; + } + + /// Try to get the section at the given index. Will return an error if the + /// given index is out of range, or if no section has been added for the given + /// index. + Expected<NormalizedSection &> findSectionByIndex(unsigned Index) { + auto I = IndexToSection.find(Index); + if (I == IndexToSection.end()) + return make_error<JITLinkError>("No section recorded for index " + + formatv("{0:u}", Index)); + return I->second; + } + + /// Try to get the symbol at the given index. Will return an error if the + /// given index is out of range, or if no symbol has been added for the given + /// index. + Expected<NormalizedSymbol &> findSymbolByIndex(uint64_t Index) { + if (Index >= IndexToSymbol.size()) + return make_error<JITLinkError>("Symbol index out of range"); + auto *Sym = IndexToSymbol[Index]; + if (!Sym) + return make_error<JITLinkError>("No symbol at index " + + formatv("{0:u}", Index)); + return *Sym; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or null if no such symbol exists. + Symbol *getSymbolByAddress(JITTargetAddress Address) { + auto I = AddrToCanonicalSymbol.upper_bound(Address); + if (I == AddrToCanonicalSymbol.begin()) + return nullptr; + return std::prev(I)->second; + } + + /// Returns the symbol with the highest address not greater than the search + /// address, or an error if no such symbol exists. + Expected<Symbol &> findSymbolByAddress(JITTargetAddress Address) { + auto *Sym = getSymbolByAddress(Address); + if (Sym) + if (Address < Sym->getAddress() + Sym->getSize()) + return *Sym; + return make_error<JITLinkError>("No symbol covering address " + + formatv("{0:x16}", Address)); + } + + static Linkage getLinkage(uint16_t Desc); + static Scope getScope(StringRef Name, uint8_t Type); + static bool isAltEntry(const NormalizedSymbol &NSym); + +private: + static unsigned getPointerSize(const object::MachOObjectFile &Obj); + static support::endianness getEndianness(const object::MachOObjectFile &Obj); + + void setCanonicalSymbol(Symbol &Sym) { + auto *&CanonicalSymEntry = AddrToCanonicalSymbol[Sym.getAddress()]; + // There should be no symbol at this address, or, if there is, + // it should be a zero-sized symbol from an empty section (which + // we can safely override). + assert((!CanonicalSymEntry || CanonicalSymEntry->getSize() == 0) && + "Duplicate canonical symbol at address"); + CanonicalSymEntry = &Sym; + } + + Section &getCommonSection(); + void addSectionStartSymAndBlock(Section &GraphSec, uint64_t Address, + const char *Data, uint64_t Size, + uint32_t Alignment, bool IsLive); + + Error createNormalizedSections(); + Error createNormalizedSymbols(); + + /// Create graph blocks and symbols for externals, absolutes, commons and + /// all defined symbols in sections without custom parsers. + Error graphifyRegularSymbols(); + + /// Create graph blocks and symbols for all sections. + Error graphifySectionsWithCustomParsers(); + + // Put the BumpPtrAllocator first so that we don't free any of the underlying + // memory until the Symbol/Addressable destructors have been run. + BumpPtrAllocator Allocator; + + const object::MachOObjectFile &Obj; + std::unique_ptr<LinkGraph> G; + + DenseMap<unsigned, NormalizedSection> IndexToSection; + Section *CommonSection = nullptr; + + DenseMap<uint32_t, NormalizedSymbol *> IndexToSymbol; + std::map<JITTargetAddress, Symbol *> AddrToCanonicalSymbol; + StringMap<SectionParserFunction> CustomSectionParserFunctions; +}; + +} // end namespace jitlink +} // end namespace llvm + +#endif // LIB_EXECUTIONENGINE_JITLINK_MACHOLINKGRAPHBUILDER_H diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index 52481f8436e..d83787ffd59 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -13,7 +13,7 @@ #include "llvm/ExecutionEngine/JITLink/MachO_x86_64.h" #include "BasicGOTAndStubsBuilder.h" -#include "MachOAtomGraphBuilder.h" +#include "MachOLinkGraphBuilder.h" #define DEBUG_TYPE "jitlink" @@ -23,16 +23,21 @@ using namespace llvm::jitlink::MachO_x86_64_Edges; namespace { -class MachOAtomGraphBuilder_x86_64 : public MachOAtomGraphBuilder { +class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: - MachOAtomGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOAtomGraphBuilder(Obj), - NumSymbols(Obj.getSymtabLoadCommand().nsyms) { - addCustomAtomizer("__eh_frame", [this](MachOSection &EHFrameSection) { - return addEHFrame(getGraph(), EHFrameSection.getGenericSection(), - EHFrameSection.getContent(), - EHFrameSection.getAddress(), NegDelta32, Delta64); - }); + MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) + : MachOLinkGraphBuilder(Obj) { + addCustomSectionParser( + "__eh_frame", [this](NormalizedSection &EHFrameSection) { + if (!EHFrameSection.Data) + return make_error<JITLinkError>( + "__eh_frame section is marked zero-fill"); + return MachOEHFrameBinaryParser( + *this, EHFrameSection.Address, + StringRef(EHFrameSection.Data, EHFrameSection.Size), + *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) + .addToGraph(); + }); } private: @@ -102,17 +107,6 @@ private: ", length=" + formatv("{0:d}", RI.r_length)); } - Expected<Atom &> findAtomBySymbolIndex(const MachO::relocation_info &RI) { - auto &Obj = getObject(); - if (RI.r_symbolnum >= NumSymbols) - return make_error<JITLinkError>("Symbol index out of range"); - auto SymI = Obj.getSymbolByIndex(RI.r_symbolnum); - auto Name = SymI->getName(); - if (!Name) - return Name.takeError(); - return getGraph().getAtomByName(*Name); - } - MachO::relocation_info getRelocationInfo(const object::relocation_iterator RelItr) { MachO::any_relocation_info ARI = @@ -122,12 +116,12 @@ private: return RI; } - using PairRelocInfo = std::tuple<MachOX86RelocationKind, Atom *, uint64_t>; + using PairRelocInfo = std::tuple<MachOX86RelocationKind, Symbol *, uint64_t>; // Parses paired SUBTRACTOR/UNSIGNED relocations and, on success, // returns the edge kind and addend to be used. Expected<PairRelocInfo> - parsePairRelocation(DefinedAtom &AtomToFix, Edge::Kind SubtractorKind, + parsePairRelocation(Block &BlockToFix, Edge::Kind SubtractorKind, const MachO::relocation_info &SubRI, JITTargetAddress FixupAddress, const char *FixupContent, object::relocation_iterator &UnsignedRelItr, @@ -154,9 +148,11 @@ private: return make_error<JITLinkError>("length of x86_64 SUBTRACTOR and paired " "UNSIGNED reloc must match"); - auto FromAtom = findAtomBySymbolIndex(SubRI); - if (!FromAtom) - return FromAtom.takeError(); + Symbol *FromSymbol; + if (auto FromSymbolOrErr = findSymbolByIndex(SubRI.r_symbolnum)) + FromSymbol = FromSymbolOrErr->GraphSymbol; + else + return FromSymbolOrErr.takeError(); // Read the current fixup value. uint64_t FixupValue = 0; @@ -165,54 +161,60 @@ private: else FixupValue = *(const little32_t *)FixupContent; - // Find 'ToAtom' using symbol number or address, depending on whether the + // Find 'ToSymbol' using symbol number or address, depending on whether the // paired UNSIGNED relocation is extern. - Atom *ToAtom = nullptr; + Symbol *ToSymbol = nullptr; if (UnsignedRI.r_extern) { - // Find target atom by symbol index. - if (auto ToAtomOrErr = findAtomBySymbolIndex(UnsignedRI)) - ToAtom = &*ToAtomOrErr; + // Find target symbol by symbol index. + if (auto ToSymbolOrErr = findSymbolByIndex(UnsignedRI.r_symbolnum)) + ToSymbol = ToSymbolOrErr->GraphSymbol; else - return ToAtomOrErr.takeError(); + return ToSymbolOrErr.takeError(); } else { - if (auto ToAtomOrErr = getGraph().findAtomByAddress(FixupValue)) - ToAtom = &*ToAtomOrErr; + if (auto ToSymbolOrErr = findSymbolByAddress(FixupValue)) + ToSymbol = &*ToSymbolOrErr; else - return ToAtomOrErr.takeError(); - FixupValue -= ToAtom->getAddress(); + return ToSymbolOrErr.takeError(); + FixupValue -= ToSymbol->getAddress(); } MachOX86RelocationKind DeltaKind; - Atom *TargetAtom; + Symbol *TargetSymbol; uint64_t Addend; - if (areLayoutLocked(AtomToFix, *FromAtom)) { - TargetAtom = ToAtom; + if (&BlockToFix == &FromSymbol->getAddressable()) { + TargetSymbol = ToSymbol; DeltaKind = (SubRI.r_length == 3) ? Delta64 : Delta32; - Addend = FixupValue + (FixupAddress - FromAtom->getAddress()); + Addend = FixupValue + (FixupAddress - FromSymbol->getAddress()); // FIXME: handle extern 'from'. - } else if (areLayoutLocked(AtomToFix, *ToAtom)) { - TargetAtom = &*FromAtom; + } else if (&BlockToFix == &ToSymbol->getAddressable()) { + TargetSymbol = FromSymbol; DeltaKind = (SubRI.r_length == 3) ? NegDelta64 : NegDelta32; - Addend = FixupValue - (FixupAddress - ToAtom->getAddress()); + Addend = FixupValue - (FixupAddress - ToSymbol->getAddress()); } else { - // AtomToFix was neither FromAtom nor ToAtom. + // BlockToFix was neither FromSymbol nor ToSymbol. return make_error<JITLinkError>("SUBTRACTOR relocation must fix up " - "either 'A' or 'B' (or an atom in one " - "of their alt-entry groups)"); + "either 'A' or 'B' (or a symbol in one " + "of their alt-entry chains)"); } - return PairRelocInfo(DeltaKind, TargetAtom, Addend); + return PairRelocInfo(DeltaKind, TargetSymbol, Addend); } Error addRelocations() override { using namespace support; - auto &G = getGraph(); auto &Obj = getObject(); for (auto &S : Obj.sections()) { JITTargetAddress SectionAddress = S.getAddress(); + if (S.isVirtual()) { + if (S.relocation_begin() != S.relocation_end()) + return make_error<JITLinkError>("Virtual section contains " + "relocations"); + continue; + } + for (auto RelItr = S.relocation_begin(), RelEnd = S.relocation_end(); RelItr != RelEnd; ++RelItr) { @@ -231,26 +233,26 @@ private: << format("0x%016" PRIx64, FixupAddress) << "\n"; }); - // Find the atom that the fixup points to. - DefinedAtom *AtomToFix = nullptr; + // Find the block that the fixup points to. + Block *BlockToFix = nullptr; { - auto AtomToFixOrErr = G.findAtomByAddress(FixupAddress); - if (!AtomToFixOrErr) - return AtomToFixOrErr.takeError(); - AtomToFix = &*AtomToFixOrErr; + auto SymbolToFixOrErr = findSymbolByAddress(FixupAddress); + if (!SymbolToFixOrErr) + return SymbolToFixOrErr.takeError(); + BlockToFix = &SymbolToFixOrErr->getBlock(); } if (FixupAddress + static_cast<JITTargetAddress>(1ULL << RI.r_length) > - AtomToFix->getAddress() + AtomToFix->getContent().size()) + BlockToFix->getAddress() + BlockToFix->getContent().size()) return make_error<JITLinkError>( - "Relocation content extends past end of fixup atom"); + "Relocation extends past end of fixup block"); // Get a pointer to the fixup content. - const char *FixupContent = AtomToFix->getContent().data() + - (FixupAddress - AtomToFix->getAddress()); + const char *FixupContent = BlockToFix->getContent().data() + + (FixupAddress - BlockToFix->getAddress()); - // The target atom and addend will be populated by the switch below. - Atom *TargetAtom = nullptr; + // The target symbol and addend will be populated by the switch below. + Symbol *TargetSymbol = nullptr; uint64_t Addend = 0; switch (*Kind) { @@ -258,53 +260,53 @@ private: case PCRel32: case PCRel32GOTLoad: case PCRel32GOT: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; break; case Pointer32: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent; break; case Pointer64: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle64_t *)FixupContent; break; case Pointer64Anon: { JITTargetAddress TargetAddress = *(const ulittle64_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case PCRel32Minus1: case PCRel32Minus2: case PCRel32Minus4: - if (auto TargetAtomOrErr = findAtomBySymbolIndex(RI)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByIndex(RI.r_symbolnum)) + TargetSymbol = TargetSymbolOrErr->GraphSymbol; else - return TargetAtomOrErr.takeError(); + return TargetSymbolOrErr.takeError(); Addend = *(const ulittle32_t *)FixupContent + (1 << (*Kind - PCRel32Minus1)); break; case PCRel32Anon: { JITTargetAddress TargetAddress = FixupAddress + 4 + *(const ulittle32_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case PCRel32Minus1Anon: @@ -314,11 +316,11 @@ private: static_cast<JITTargetAddress>(1ULL << (*Kind - PCRel32Minus1Anon)); JITTargetAddress TargetAddress = FixupAddress + 4 + Delta + *(const ulittle32_t *)FixupContent; - if (auto TargetAtomOrErr = G.findAtomByAddress(TargetAddress)) - TargetAtom = &*TargetAtomOrErr; + if (auto TargetSymbolOrErr = findSymbolByAddress(TargetAddress)) + TargetSymbol = &*TargetSymbolOrErr; else - return TargetAtomOrErr.takeError(); - Addend = TargetAddress - TargetAtom->getAddress(); + return TargetSymbolOrErr.takeError(); + Addend = TargetAddress - TargetSymbol->getAddress(); break; } case Delta32: @@ -329,12 +331,12 @@ private: // NegDelta32/NegDelta64, depending on the direction of the // subtraction) along with the addend. auto PairInfo = - parsePairRelocation(*AtomToFix, *Kind, RI, FixupAddress, + parsePairRelocation(*BlockToFix, *Kind, RI, FixupAddress, FixupContent, ++RelItr, RelEnd); if (!PairInfo) return PairInfo.takeError(); - std::tie(*Kind, TargetAtom, Addend) = *PairInfo; - assert(TargetAtom && "No target atom from parsePairRelocation?"); + std::tie(*Kind, TargetSymbol, Addend) = *PairInfo; + assert(TargetSymbol && "No target symbol from parsePairRelocation?"); break; } default: @@ -343,41 +345,38 @@ private: } LLVM_DEBUG({ - Edge GE(*Kind, FixupAddress - AtomToFix->getAddress(), *TargetAtom, + Edge GE(*Kind, FixupAddress - BlockToFix->getAddress(), *TargetSymbol, Addend); - printEdge(dbgs(), *AtomToFix, GE, + printEdge(dbgs(), *BlockToFix, GE, getMachOX86RelocationKindName(*Kind)); dbgs() << "\n"; }); - AtomToFix->addEdge(*Kind, FixupAddress - AtomToFix->getAddress(), - *TargetAtom, Addend); + BlockToFix->addEdge(*Kind, FixupAddress - BlockToFix->getAddress(), + *TargetSymbol, Addend); } } return Error::success(); } - - unsigned NumSymbols = 0; }; class MachO_x86_64_GOTAndStubsBuilder : public BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder> { public: - MachO_x86_64_GOTAndStubsBuilder(AtomGraph &G) + MachO_x86_64_GOTAndStubsBuilder(LinkGraph &G) : BasicGOTAndStubsBuilder<MachO_x86_64_GOTAndStubsBuilder>(G) {} bool isGOTEdge(Edge &E) const { return E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad; } - DefinedAtom &createGOTEntry(Atom &Target) { - auto &GOTEntryAtom = G.addAnonymousAtom(getGOTSection(), 0x0, 8); - GOTEntryAtom.setContent( - StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), 8)); - GOTEntryAtom.addEdge(Pointer64, 0, Target, 0); - return GOTEntryAtom; + Symbol &createGOTEntry(Symbol &Target) { + auto &GOTEntryBlock = G.createContentBlock( + getGOTSection(), getGOTEntryBlockContent(), 0, 8, 0); + GOTEntryBlock.addEdge(Pointer64, 0, Target, 0); + return G.addAnonymousSymbol(GOTEntryBlock, 0, 8, false, false); } - void fixGOTEdge(Edge &E, Atom &GOTEntry) { + void fixGOTEdge(Edge &E, Symbol &GOTEntry) { assert((E.getKind() == PCRel32GOT || E.getKind() == PCRel32GOTLoad) && "Not a GOT edge?"); E.setKind(PCRel32); @@ -389,19 +388,16 @@ public: return E.getKind() == Branch32 && !E.getTarget().isDefined(); } - DefinedAtom &createStub(Atom &Target) { - auto &StubAtom = G.addAnonymousAtom(getStubsSection(), 0x0, 2); - StubAtom.setContent( - StringRef(reinterpret_cast<const char *>(StubContent), 6)); - + Symbol &createStub(Symbol &Target) { + auto &StubContentBlock = + G.createContentBlock(getStubsSection(), getStubBlockContent(), 0, 1, 0); // Re-use GOT entries for stub targets. - auto &GOTEntryAtom = getGOTEntryAtom(Target); - StubAtom.addEdge(PCRel32, 2, GOTEntryAtom, 0); - - return StubAtom; + auto &GOTEntrySymbol = getGOTEntrySymbol(Target); + StubContentBlock.addEdge(PCRel32, 2, GOTEntrySymbol, 0); + return G.addAnonymousSymbol(StubContentBlock, 0, 6, true, false); } - void fixExternalBranchEdge(Edge &E, Atom &Stub) { + void fixExternalBranchEdge(Edge &E, Symbol &Stub) { assert(E.getKind() == Branch32 && "Not a Branch32 edge?"); assert(E.getAddend() == 0 && "Branch32 edge has non-zero addend?"); E.setTarget(Stub); @@ -410,7 +406,7 @@ public: private: Section &getGOTSection() { if (!GOTSection) - GOTSection = &G.createSection("$__GOT", 8, sys::Memory::MF_READ, false); + GOTSection = &G.createSection("$__GOT", sys::Memory::MF_READ); return *GOTSection; } @@ -418,11 +414,21 @@ private: if (!StubsSection) { auto StubsProt = static_cast<sys::Memory::ProtectionFlags>( sys::Memory::MF_READ | sys::Memory::MF_EXEC); - StubsSection = &G.createSection("$__STUBS", 8, StubsProt, false); + StubsSection = &G.createSection("$__STUBS", StubsProt); } return *StubsSection; } + StringRef getGOTEntryBlockContent() { + return StringRef(reinterpret_cast<const char *>(NullGOTEntryContent), + sizeof(NullGOTEntryContent)); + } + + StringRef getStubBlockContent() { + return StringRef(reinterpret_cast<const char *>(StubContent), + sizeof(StubContent)); + } + static const uint8_t NullGOTEntryContent[8]; static const uint8_t StubContent[6]; Section *GOTSection = nullptr; @@ -451,30 +457,31 @@ private: return getMachOX86RelocationKindName(R); } - Expected<std::unique_ptr<AtomGraph>> + Expected<std::unique_ptr<LinkGraph>> buildGraph(MemoryBufferRef ObjBuffer) override { auto MachOObj = object::ObjectFile::createMachOObjectFile(ObjBuffer); if (!MachOObj) return MachOObj.takeError(); - return MachOAtomGraphBuilder_x86_64(**MachOObj).buildGraph(); + return MachOLinkGraphBuilder_x86_64(**MachOObj).buildGraph(); } - static Error targetOutOfRangeError(const Atom &A, const Edge &E) { + static Error targetOutOfRangeError(const Block &B, const Edge &E) { std::string ErrMsg; { raw_string_ostream ErrStream(ErrMsg); ErrStream << "Relocation target out of range: "; - printEdge(ErrStream, A, E, getMachOX86RelocationKindName(E.getKind())); + printEdge(ErrStream, B, E, getMachOX86RelocationKindName(E.getKind())); ErrStream << "\n"; } return make_error<JITLinkError>(std::move(ErrMsg)); } - Error applyFixup(DefinedAtom &A, const Edge &E, char *AtomWorkingMem) const { + Error applyFixup(Block &B, const Edge &E, char *BlockWorkingMem) const { + using namespace support; - char *FixupPtr = AtomWorkingMem + E.getOffset(); - JITTargetAddress FixupAddress = A.getAddress() + E.getOffset(); + char *FixupPtr = BlockWorkingMem + E.getOffset(); + JITTargetAddress FixupAddress = B.getAddress() + E.getOffset(); switch (E.getKind()) { case Branch32: @@ -484,7 +491,7 @@ private: E.getTarget().getAddress() - (FixupAddress + 4) + E.getAddend(); if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -502,7 +509,7 @@ private: E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -514,7 +521,7 @@ private: E.getTarget().getAddress() - (FixupAddress + Delta) + E.getAddend(); if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; break; } @@ -531,7 +538,7 @@ private: if (E.getKind() == Delta32 || E.getKind() == NegDelta32) { if (Value < std::numeric_limits<int32_t>::min() || Value > std::numeric_limits<int32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(little32_t *)FixupPtr = Value; } else *(little64_t *)FixupPtr = Value; @@ -540,7 +547,7 @@ private: case Pointer32: { uint64_t Value = E.getTarget().getAddress() + E.getAddend(); if (Value > std::numeric_limits<uint32_t>::max()) - return targetOutOfRangeError(A, E); + return targetOutOfRangeError(B, E); *(ulittle32_t *)FixupPtr = Value; break; } @@ -563,10 +570,10 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); else - Config.PrePrunePasses.push_back(markAllAtomsLive); + Config.PrePrunePasses.push_back(markAllSymbolsLive); // Add an in-place GOT/Stubs pass. - Config.PostPrunePasses.push_back([](AtomGraph &G) -> Error { + Config.PostPrunePasses.push_back([](LinkGraph &G) -> Error { MachO_x86_64_GOTAndStubsBuilder(G).run(); return Error::success(); }); |
