diff options
author | Lang Hames <lhames@gmail.com> | 2019-11-04 16:00:09 -0800 |
---|---|---|
committer | Lang Hames <lhames@gmail.com> | 2019-11-06 14:30:26 -0800 |
commit | 76aee8a389447409905c58b178b0554c9bae8a0a (patch) | |
tree | 2959df9ba579588dffcab56a8bd2fb37fca4b79c /llvm/lib/ExecutionEngine | |
parent | 8243918f43c6eedc2b018c1edc9c6b72fe9b3c1e (diff) | |
download | bcm5719-llvm-76aee8a389447409905c58b178b0554c9bae8a0a.tar.gz bcm5719-llvm-76aee8a389447409905c58b178b0554c9bae8a0a.zip |
[JITLink] Refactor EH-frame handling to support eh-frames with existing relocs.
Some targets (E.g. MachO/arm64) use relocations to fix some CFI record fields
in the eh-frame section. When relocations are used the initial (pre-relocation)
content of the eh-frame section can no longer be interpreted by following the
eh-frame specification. This causes errors in the existing eh-frame parser.
This patch moves eh-frame handling into two LinkGraph passes that are run after
relocations have been parsed (but before they are applied). The first] pass
breaks up blocks in the eh-frame section into per-CFI-record blocks, and the
second parses blocks of (potentially multiple) CFI records and adds the
appropriate edges to any CFI fields that do not have existing relocations.
These passes can be run independently of one another. By handling eh-frame
splitting/fixing with LinkGraph passes we can both re-use existing relocations
for CFI record fields and avoid applying eh-frame fixups before parsing the
section (which would complicate the linker and require extra temporary
allocations of working memory).
Diffstat (limited to 'llvm/lib/ExecutionEngine')
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp | 655 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h | 96 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp | 5 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h | 68 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp | 14 | ||||
-rw-r--r-- | llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp | 19 |
6 files changed, 537 insertions, 320 deletions
diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp index f80b0e7f890..f1114e92c36 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupport.cpp @@ -17,174 +17,281 @@ namespace llvm { namespace jitlink { -EHFrameBinaryParser::EHFrameBinaryParser(JITTargetAddress EHFrameAddress, - StringRef EHFrameContent, - unsigned PointerSize, - support::endianness Endianness) - : EHFrameAddress(EHFrameAddress), EHFrameContent(EHFrameContent), - PointerSize(PointerSize), EHFrameReader(EHFrameContent, Endianness) {} +EHFrameSplitter::EHFrameSplitter(StringRef EHFrameSectionName) + : EHFrameSectionName(EHFrameSectionName) {} -Error EHFrameBinaryParser::addToGraph() { - while (!EHFrameReader.empty()) { - size_t RecordOffset = EHFrameReader.getOffset(); +Error EHFrameSplitter::operator()(LinkGraph &G) { + auto *EHFrame = G.findSectionByName(EHFrameSectionName); + if (!EHFrame) { LLVM_DEBUG({ - dbgs() << "Processing eh-frame record at " - << format("0x%016" PRIx64, EHFrameAddress + RecordOffset) - << " (offset " << RecordOffset << ")\n"; + dbgs() << "EHFrameSplitter: No " << EHFrameSectionName + << " section. Nothing to do\n"; }); + return Error::success(); + } - size_t RecordLength = 0; - uint32_t RecordLengthField; - if (auto Err = EHFrameReader.readInteger(RecordLengthField)) + LLVM_DEBUG({ + dbgs() << "EHFrameSplitter: Processing " << EHFrameSectionName << "...\n"; + }); + + DenseMap<Block *, LinkGraph::SplitBlockCache> Caches; + + { + // Pre-build the split caches. + for (auto *B : EHFrame->blocks()) + Caches[B] = LinkGraph::SplitBlockCache::value_type(); + for (auto *Sym : EHFrame->symbols()) + Caches[&Sym->getBlock()]->push_back(Sym); + for (auto *B : EHFrame->blocks()) + llvm::sort(*Caches[B], [](const Symbol *LHS, const Symbol *RHS) { + return LHS->getOffset() > RHS->getOffset(); + }); + } + + // Iterate over blocks (we do this by iterating over Caches entries rather + // than EHFrame->blocks() as we will be inserting new blocks along the way, + // which would invalidate iterators in the latter sequence. + for (auto &KV : Caches) { + auto &B = *KV.first; + auto &BCache = KV.second; + if (auto Err = processBlock(G, B, BCache)) return Err; + } - // Process CIE/FDE length/extended-length fields to build the blocks. - // - // The value of these fields describe the length of the *rest* of the CIE - // (not including data up to the end of the field itself) so we have to - // bump RecordLength to include the data up to the end of the field: 4 bytes - // for Length, or 12 bytes (4 bytes + 8 bytes) for ExtendedLength. - if (RecordLengthField == 0) // Length 0 means end of __eh_frame section. - break; + return Error::success(); +} - // If the regular length field's value is 0xffffffff, use extended length. - if (RecordLengthField == 0xffffffff) { - uint64_t ExtendedLengthField; - if (auto Err = EHFrameReader.readInteger(ExtendedLengthField)) - return Err; - if (ExtendedLengthField > EHFrameReader.bytesRemaining()) - return make_error<JITLinkError>("CIE record extends past the end of " - "the __eh_frame section"); - if (ExtendedLengthField + 12 > std::numeric_limits<size_t>::max()) - return make_error<JITLinkError>("CIE record too large to process"); - RecordLength = ExtendedLengthField + 12; - } else { - if (RecordLengthField > EHFrameReader.bytesRemaining()) - return make_error<JITLinkError>("CIE record extends past the end of " - "the __eh_frame section"); - RecordLength = RecordLengthField + 4; - } +Error EHFrameSplitter::processBlock(LinkGraph &G, Block &B, + LinkGraph::SplitBlockCache &Cache) { + LLVM_DEBUG({ + dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) + << "\n"; + }); - LLVM_DEBUG(dbgs() << " length: " << RecordLength << "\n"); + // eh-frame should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error<JITLinkError>("Unexpected zero-fill block in " + + EHFrameSectionName + " section"); - // Read the CIE Pointer. - size_t CIEPointerAddress = EHFrameAddress + EHFrameReader.getOffset(); - uint32_t CIEPointer; - if (auto Err = EHFrameReader.readInteger(CIEPointer)) - return Err; + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + BinaryStreamReader BlockReader(B.getContent(), G.getEndianness()); - // Based on the CIE pointer value, parse this as a CIE or FDE record. - if (CIEPointer == 0) { - if (auto Err = processCIE(RecordOffset, RecordLength)) + while (true) { + uint64_t RecordStartOffset = BlockReader.getOffset(); + + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress()) << "\n"; + }); + + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + if (Length != 0xffffffff) { + if (auto Err = BlockReader.skip(Length)) return Err; } else { - if (auto Err = processFDE(RecordOffset, RecordLength, CIEPointerAddress, - CIEPointer)) + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + if (auto Err = BlockReader.skip(ExtendedLength)) return Err; } - EHFrameReader.setOffset(RecordOffset + RecordLength); + // If this was the last block then there's nothing to split + if (BlockReader.empty()) { + LLVM_DEBUG(dbgs() << " Extracted " << B << "\n"); + return Error::success(); + } + + uint64_t BlockSize = BlockReader.getOffset() - RecordStartOffset; + auto &NewBlock = G.splitBlock(B, BlockSize); + (void)NewBlock; + LLVM_DEBUG(dbgs() << " Extracted " << NewBlock << "\n"); + } +} + +EHFrameEdgeFixer::EHFrameEdgeFixer(StringRef EHFrameSectionName, + Edge::Kind FDEToCIE, Edge::Kind FDEToPCBegin, + Edge::Kind FDEToLSDA) + : EHFrameSectionName(EHFrameSectionName), FDEToCIE(FDEToCIE), + FDEToPCBegin(FDEToPCBegin), FDEToLSDA(FDEToLSDA) {} + +Error EHFrameEdgeFixer::operator()(LinkGraph &G) { + auto *EHFrame = G.findSectionByName(EHFrameSectionName); + + if (!EHFrame) { + LLVM_DEBUG({ + dbgs() << "EHFrameEdgeFixer: No " << EHFrameSectionName + << " section. Nothing to do\n"; + }); + return Error::success(); + } + + LLVM_DEBUG({ + dbgs() << "EHFrameEdgeFixer: Processing " << EHFrameSectionName << "...\n"; + }); + + ParseContext PC(G); + + // Build a map of all blocks and symbols in the text sections. We will use + // these for finding / building edge targets when processing FDEs. + for (auto &Sec : G.sections()) { + PC.AddrToSyms.addSymbols(Sec.symbols()); + if (auto Err = PC.AddrToBlock.addBlocks(Sec.blocks(), + BlockAddressMap::includeNonNull)) + return Err; } + // Sort eh-frame blocks into address order to ensure we visit CIEs before + // their child FDEs. + std::vector<Block *> EHFrameBlocks; + for (auto *B : EHFrame->blocks()) + EHFrameBlocks.push_back(B); + llvm::sort(EHFrameBlocks, [](const Block *LHS, const Block *RHS) { + return LHS->getAddress() < RHS->getAddress(); + }); + + // Loop over the blocks in address order. + for (auto *B : EHFrameBlocks) + if (auto Err = processBlock(PC, *B)) + return Err; + return Error::success(); } -void EHFrameBinaryParser::anchor() {} +Error EHFrameEdgeFixer::processBlock(ParseContext &PC, Block &B) { -Expected<EHFrameBinaryParser::AugmentationInfo> -EHFrameBinaryParser::parseAugmentationString() { - AugmentationInfo AugInfo; - uint8_t NextChar; - uint8_t *NextField = &AugInfo.Fields[0]; + LLVM_DEBUG({ + dbgs() << " Processing block at " << formatv("{0:x16}", B.getAddress()) + << "\n"; + }); - if (auto Err = EHFrameReader.readInteger(NextChar)) - return std::move(Err); + // eh-frame should not contain zero-fill blocks. + if (B.isZeroFill()) + return make_error<JITLinkError>("Unexpected zero-fill block in " + + EHFrameSectionName + " section"); - while (NextChar != 0) { - switch (NextChar) { - case 'z': - AugInfo.AugmentationDataPresent = true; - break; - case 'e': - if (auto Err = EHFrameReader.readInteger(NextChar)) - return std::move(Err); - if (NextChar != 'h') - return make_error<JITLinkError>("Unrecognized substring e" + - Twine(NextChar) + - " in augmentation string"); - AugInfo.EHDataFieldPresent = true; - break; - case 'L': - case 'P': - case 'R': - *NextField++ = NextChar; - break; - default: - return make_error<JITLinkError>("Unrecognized character " + - Twine(NextChar) + - " in augmentation string"); + if (B.getSize() == 0) { + LLVM_DEBUG(dbgs() << " Block is empty. Skipping.\n"); + return Error::success(); + } + + // Find the offsets of any existing edges from this block. + BlockEdgeMap BlockEdges; + for (auto &E : B.edges()) + if (E.isRelocation()) { + if (BlockEdges.count(E.getOffset())) + return make_error<JITLinkError>( + "Multiple relocations at offset " + + formatv("{0:x16}", E.getOffset()) + " in " + EHFrameSectionName + + " block at address " + formatv("{0:x16}", B.getAddress())); + + BlockEdges[E.getOffset()] = EdgeTarget(E); } - if (auto Err = EHFrameReader.readInteger(NextChar)) - return std::move(Err); - } + CIEInfosMap CIEInfos; + BinaryStreamReader BlockReader(B.getContent(), PC.G.getEndianness()); + while (!BlockReader.empty()) { + size_t RecordStartOffset = BlockReader.getOffset(); - return std::move(AugInfo); -} + LLVM_DEBUG({ + dbgs() << " Processing CFI record at " + << formatv("{0:x16}", B.getAddress() + RecordStartOffset) << "\n"; + }); -Expected<JITTargetAddress> EHFrameBinaryParser::readAbsolutePointer() { - static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), - "Result must be able to hold a uint64_t"); - JITTargetAddress Addr; - if (PointerSize == 8) { - if (auto Err = EHFrameReader.readInteger(Addr)) - return std::move(Err); - } else if (PointerSize == 4) { - uint32_t Addr32; - if (auto Err = EHFrameReader.readInteger(Addr32)) - return std::move(Err); - Addr = Addr32; - } else - llvm_unreachable("Pointer size is not 32-bit or 64-bit"); - return Addr; + // Get the record length. + size_t RecordRemaining; + { + uint32_t Length; + if (auto Err = BlockReader.readInteger(Length)) + return Err; + // If Length < 0xffffffff then use the regular length field, otherwise + // read the extended length field. + if (Length != 0xffffffff) + RecordRemaining = Length; + else { + uint64_t ExtendedLength; + if (auto Err = BlockReader.readInteger(ExtendedLength)) + return Err; + RecordRemaining = ExtendedLength; + } + } + + if (BlockReader.bytesRemaining() < RecordRemaining) + return make_error<JITLinkError>( + "Incomplete CFI record at " + + formatv("{0:x16}", B.getAddress() + RecordStartOffset)); + + // Read the CIE delta for this record. + uint64_t CIEDeltaFieldOffset = BlockReader.getOffset() - RecordStartOffset; + uint32_t CIEDelta; + if (auto Err = BlockReader.readInteger(CIEDelta)) + return Err; + + if (CIEDelta == 0) { + if (auto Err = processCIE(PC, B, RecordStartOffset, + CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset)) + return Err; + } else { + if (auto Err = processFDE(PC, B, RecordStartOffset, + CIEDeltaFieldOffset + RecordRemaining, + CIEDeltaFieldOffset, CIEDelta, BlockEdges)) + return Err; + } + + // Move to the next record. + BlockReader.setOffset(RecordStartOffset + CIEDeltaFieldOffset + + RecordRemaining); + } + + return Error::success(); } -Error EHFrameBinaryParser::processCIE(size_t RecordOffset, - size_t RecordLength) { - // Use the dwarf namespace for convenient access to pointer encoding - // constants. +Error EHFrameEdgeFixer::processCIE(ParseContext &PC, Block &B, + size_t RecordOffset, size_t RecordLength, + size_t CIEDeltaFieldOffset) { using namespace dwarf; - LLVM_DEBUG(dbgs() << " Record is CIE\n"); + LLVM_DEBUG(dbgs() << " Record is CIE\n"); - auto &CIESymbol = - createCIERecord(EHFrameAddress + RecordOffset, - EHFrameContent.substr(RecordOffset, RecordLength)); + auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); + BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); + + // Skip past the CIE delta field: we've already processed this far. + RecordReader.setOffset(CIEDeltaFieldOffset + 4); + auto &CIESymbol = + PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); CIEInformation CIEInfo(CIESymbol); uint8_t Version = 0; - if (auto Err = EHFrameReader.readInteger(Version)) + if (auto Err = RecordReader.readInteger(Version)) return Err; if (Version != 0x01) return make_error<JITLinkError>("Bad CIE version " + Twine(Version) + " (should be 0x01) in eh-frame"); - auto AugInfo = parseAugmentationString(); + auto AugInfo = parseAugmentationString(RecordReader); if (!AugInfo) return AugInfo.takeError(); // Skip the EH Data field if present. if (AugInfo->EHDataFieldPresent) - if (auto Err = EHFrameReader.skip(PointerSize)) + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) return Err; // Read and sanity check the code alignment factor. { uint64_t CodeAlignmentFactor = 0; - if (auto Err = EHFrameReader.readULEB128(CodeAlignmentFactor)) + if (auto Err = RecordReader.readULEB128(CodeAlignmentFactor)) return Err; if (CodeAlignmentFactor != 1) return make_error<JITLinkError>("Unsupported CIE code alignment factor " + @@ -195,7 +302,7 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, // Read and sanity check the data alignment factor. { int64_t DataAlignmentFactor = 0; - if (auto Err = EHFrameReader.readSLEB128(DataAlignmentFactor)) + if (auto Err = RecordReader.readSLEB128(DataAlignmentFactor)) return Err; if (DataAlignmentFactor != -8) return make_error<JITLinkError>("Unsupported CIE data alignment factor " + @@ -204,14 +311,14 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, } // Skip the return address register field. - if (auto Err = EHFrameReader.skip(1)) + if (auto Err = RecordReader.skip(1)) return Err; uint64_t AugmentationDataLength = 0; - if (auto Err = EHFrameReader.readULEB128(AugmentationDataLength)) + if (auto Err = RecordReader.readULEB128(AugmentationDataLength)) return Err; - uint32_t AugmentationDataStartOffset = EHFrameReader.getOffset(); + uint32_t AugmentationDataStartOffset = RecordReader.getOffset(); uint8_t *NextField = &AugInfo->Fields[0]; while (uint8_t Field = *NextField++) { @@ -219,7 +326,7 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, case 'L': { CIEInfo.FDEsHaveLSDAField = true; uint8_t LSDAPointerEncoding; - if (auto Err = EHFrameReader.readInteger(LSDAPointerEncoding)) + if (auto Err = RecordReader.readInteger(LSDAPointerEncoding)) return Err; if (LSDAPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) return make_error<JITLinkError>( @@ -230,7 +337,7 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, } case 'P': { uint8_t PersonalityPointerEncoding = 0; - if (auto Err = EHFrameReader.readInteger(PersonalityPointerEncoding)) + if (auto Err = RecordReader.readInteger(PersonalityPointerEncoding)) return Err; if (PersonalityPointerEncoding != (DW_EH_PE_indirect | DW_EH_PE_pcrel | DW_EH_PE_sdata4)) @@ -240,13 +347,13 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, formatv("{0:x2}", PersonalityPointerEncoding) + " in CIE at " + formatv("{0:x16}", CIESymbol.getAddress())); uint32_t PersonalityPointerAddress; - if (auto Err = EHFrameReader.readInteger(PersonalityPointerAddress)) + if (auto Err = RecordReader.readInteger(PersonalityPointerAddress)) return Err; break; } case 'R': { uint8_t FDEPointerEncoding; - if (auto Err = EHFrameReader.readInteger(FDEPointerEncoding)) + if (auto Err = RecordReader.readInteger(FDEPointerEncoding)) return Err; if (FDEPointerEncoding != (DW_EH_PE_pcrel | DW_EH_PE_absptr)) return make_error<JITLinkError>( @@ -261,107 +368,265 @@ Error EHFrameBinaryParser::processCIE(size_t RecordOffset, } } - if (EHFrameReader.getOffset() - AugmentationDataStartOffset > + if (RecordReader.getOffset() - AugmentationDataStartOffset > AugmentationDataLength) return make_error<JITLinkError>("Read past the end of the augmentation " "data while parsing fields"); - assert(!CIEInfos.count(CIESymbol.getAddress()) && + assert(!PC.CIEInfos.count(CIESymbol.getAddress()) && "Multiple CIEs recorded at the same address?"); - CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); + PC.CIEInfos[CIESymbol.getAddress()] = std::move(CIEInfo); return Error::success(); } -Error EHFrameBinaryParser::processFDE(size_t RecordOffset, size_t RecordLength, - JITTargetAddress CIEPointerAddress, - uint32_t CIEPointer) { - LLVM_DEBUG(dbgs() << " Record is FDE\n"); +Error EHFrameEdgeFixer::processFDE(ParseContext &PC, Block &B, + size_t RecordOffset, size_t RecordLength, + size_t CIEDeltaFieldOffset, + uint32_t CIEDelta, + BlockEdgeMap &BlockEdges) { + LLVM_DEBUG(dbgs() << " Record is FDE\n"); - LLVM_DEBUG({ - dbgs() << " CIE pointer: " - << format("0x%016" PRIx64, CIEPointerAddress - CIEPointer) << "\n"; - }); - - auto CIEInfoItr = CIEInfos.find(CIEPointerAddress - CIEPointer); - if (CIEInfoItr == CIEInfos.end()) - return make_error<JITLinkError>( - "FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset) + - " points to non-existant CIE at " + - formatv("{0:x16}", CIEPointerAddress - CIEPointer)); - auto &CIEInfo = CIEInfoItr->second; + JITTargetAddress RecordAddress = B.getAddress() + RecordOffset; - // Read and sanity check the PC-start pointer and size. - JITTargetAddress PCBeginAddress = EHFrameAddress + EHFrameReader.getOffset(); + auto RecordContent = B.getContent().substr(RecordOffset, RecordLength); + BinaryStreamReader RecordReader(RecordContent, PC.G.getEndianness()); - auto PCBeginDelta = readAbsolutePointer(); - if (!PCBeginDelta) - return PCBeginDelta.takeError(); + // Skip past the CIE delta field: we've already read this far. + RecordReader.setOffset(CIEDeltaFieldOffset + 4); - JITTargetAddress PCBegin = PCBeginAddress + *PCBeginDelta; - LLVM_DEBUG({ - dbgs() << " PC begin: " << format("0x%016" PRIx64, PCBegin) << "\n"; - }); + auto &FDESymbol = + PC.G.addAnonymousSymbol(B, RecordOffset, RecordLength, false, false); - auto *TargetSymbol = getSymbolAtAddress(PCBegin); + CIEInformation *CIEInfo = nullptr; - if (!TargetSymbol) - return make_error<JITLinkError>("FDE PC-begin " + - formatv("{0:x16}", PCBegin) + - " does not point at symbol"); + { + // Process the CIE pointer field. + auto CIEEdgeItr = BlockEdges.find(RecordOffset + CIEDeltaFieldOffset); + JITTargetAddress CIEAddress = + RecordAddress + CIEDeltaFieldOffset - CIEDelta; + if (CIEEdgeItr == BlockEdges.end()) { + + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + << " to CIE at: " << formatv("{0:x16}", CIEAddress) << "\n"; + }); + if (auto CIEInfoOrErr = PC.findCIEInfo(CIEAddress)) + CIEInfo = *CIEInfoOrErr; + else + return CIEInfoOrErr.takeError(); + assert(CIEInfo->CIESymbol && "CIEInfo has no CIE symbol set"); + B.addEdge(FDEToCIE, RecordOffset + CIEDeltaFieldOffset, + *CIEInfo->CIESymbol, 0); + } else { + LLVM_DEBUG({ + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + << " to CIE at " << formatv("{0:x16}", CIEAddress) << "\n"; + }); + auto &EI = CIEEdgeItr->second; + if (EI.Addend) + return make_error<JITLinkError>( + "CIE edge at " + + formatv("{0:x16}", RecordAddress + CIEDeltaFieldOffset) + + " has non-zero addend"); + if (auto CIEInfoOrErr = PC.findCIEInfo(EI.Target->getAddress())) + CIEInfo = *CIEInfoOrErr; + else + return CIEInfoOrErr.takeError(); + } + } - if (TargetSymbol->getAddress() != PCBegin) - return make_error<JITLinkError>( - "FDE PC-begin " + formatv("{0:x16}", PCBegin) + - " does not point to start of symbol at " + - formatv("{0:x16}", TargetSymbol->getAddress())); + { + // Process the PC-Begin field. + Block *PCBeginBlock = nullptr; + JITTargetAddress PCBeginFieldOffset = RecordReader.getOffset(); + auto PCEdgeItr = BlockEdges.find(RecordOffset + PCBeginFieldOffset); + if (PCEdgeItr == BlockEdges.end()) { + auto PCBeginDelta = readAbsolutePointer(PC.G, RecordReader); + if (!PCBeginDelta) + return PCBeginDelta.takeError(); + JITTargetAddress PCBegin = + RecordAddress + PCBeginFieldOffset + *PCBeginDelta; + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) + << " to PC at " << formatv("{0:x16}", PCBegin) << "\n"; + }); + auto PCBeginSym = getOrCreateSymbol(PC, PCBegin); + if (!PCBeginSym) + return PCBeginSym.takeError(); + B.addEdge(FDEToPCBegin, RecordOffset + PCBeginFieldOffset, *PCBeginSym, + 0); + PCBeginBlock = &PCBeginSym->getBlock(); + } else { + auto &EI = PCEdgeItr->second; + LLVM_DEBUG({ + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + PCBeginFieldOffset) + << " to PC at " << formatv("{0:x16}", EI.Target->getAddress()); + if (EI.Addend) + dbgs() << " + " << formatv("{0:x16}", EI.Addend); + dbgs() << "\n"; + }); + + // Make sure the existing edge points at a defined block. + if (!EI.Target->isDefined()) { + auto EdgeAddr = RecordAddress + PCBeginFieldOffset; + return make_error<JITLinkError>("FDE edge at " + + formatv("{0:x16}", EdgeAddr) + + " points at external block"); + } + PCBeginBlock = &EI.Target->getBlock(); + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + return Err; + } - LLVM_DEBUG(dbgs() << " FDE target: " << *TargetSymbol << "\n"); + // Add a keep-alive edge from the FDE target to the FDE to ensure that the + // FDE is kept alive if its target is. + assert(PCBeginBlock && "PC-begin block not recorded"); + PCBeginBlock->addEdge(Edge::KeepAlive, 0, FDESymbol, 0); + } // Skip over the PC range size field. - if (auto Err = EHFrameReader.skip(PointerSize)) + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) return Err; - Symbol *LSDASymbol = nullptr; - JITTargetAddress LSDAAddress = 0; - if (CIEInfo.FDEsHaveLSDAField) { + if (CIEInfo->FDEsHaveLSDAField) { uint64_t AugmentationDataSize; - if (auto Err = EHFrameReader.readULEB128(AugmentationDataSize)) + if (auto Err = RecordReader.readULEB128(AugmentationDataSize)) return Err; - if (AugmentationDataSize != PointerSize) + if (AugmentationDataSize != PC.G.getPointerSize()) return make_error<JITLinkError>( "Unexpected FDE augmentation data size (expected " + - Twine(PointerSize) + ", got " + Twine(AugmentationDataSize) + - ") for FDE at " + formatv("{0:x16}", EHFrameAddress + RecordOffset)); - LSDAAddress = EHFrameAddress + EHFrameReader.getOffset(); - auto LSDADelta = readAbsolutePointer(); - if (!LSDADelta) - return LSDADelta.takeError(); + Twine(PC.G.getPointerSize()) + ", got " + + Twine(AugmentationDataSize) + ") for FDE at " + + formatv("{0:x16}", RecordAddress)); + + JITTargetAddress LSDAFieldOffset = RecordReader.getOffset(); + auto LSDAEdgeItr = BlockEdges.find(RecordOffset + LSDAFieldOffset); + if (LSDAEdgeItr == BlockEdges.end()) { + auto LSDADelta = readAbsolutePointer(PC.G, RecordReader); + if (!LSDADelta) + return LSDADelta.takeError(); + JITTargetAddress LSDA = RecordAddress + LSDAFieldOffset + *LSDADelta; + auto LSDASym = getOrCreateSymbol(PC, LSDA); + if (!LSDASym) + return LSDASym.takeError(); + LLVM_DEBUG({ + dbgs() << " Adding edge at " + << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) + << " to LSDA at " << formatv("{0:x16}", LSDA) << "\n"; + }); + B.addEdge(FDEToLSDA, RecordOffset + LSDAFieldOffset, *LSDASym, 0); + } else { + LLVM_DEBUG({ + auto &EI = LSDAEdgeItr->second; + dbgs() << " Already has edge at " + << formatv("{0:x16}", RecordAddress + LSDAFieldOffset) + << " to LSDA at " << formatv("{0:x16}", EI.Target->getAddress()); + if (EI.Addend) + dbgs() << " + " << formatv("{0:x16}", EI.Addend); + dbgs() << "\n"; + }); + if (auto Err = RecordReader.skip(PC.G.getPointerSize())) + return Err; + } + } else { + LLVM_DEBUG(dbgs() << " Record does not have LSDA field.\n"); + } - JITTargetAddress LSDA = LSDAAddress + *LSDADelta; + return Error::success(); +} - LSDASymbol = getSymbolAtAddress(LSDA); +Expected<EHFrameEdgeFixer::AugmentationInfo> +EHFrameEdgeFixer::parseAugmentationString(BinaryStreamReader &RecordReader) { + AugmentationInfo AugInfo; + uint8_t NextChar; + uint8_t *NextField = &AugInfo.Fields[0]; - if (!LSDASymbol) - return make_error<JITLinkError>("FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point at symbol"); + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); - if (LSDASymbol->getAddress() != LSDA) - return make_error<JITLinkError>( - "FDE LSDA " + formatv("{0:x16}", LSDA) + - " does not point to start of symbol at " + - formatv("{0:x16}", LSDASymbol->getAddress())); + while (NextChar != 0) { + switch (NextChar) { + case 'z': + AugInfo.AugmentationDataPresent = true; + break; + case 'e': + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); + if (NextChar != 'h') + return make_error<JITLinkError>("Unrecognized substring e" + + Twine(NextChar) + + " in augmentation string"); + AugInfo.EHDataFieldPresent = true; + break; + case 'L': + case 'P': + case 'R': + *NextField++ = NextChar; + break; + default: + return make_error<JITLinkError>("Unrecognized character " + + Twine(NextChar) + + " in augmentation string"); + } - LLVM_DEBUG(dbgs() << " FDE LSDA: " << *LSDASymbol << "\n"); + if (auto Err = RecordReader.readInteger(NextChar)) + return std::move(Err); } - JITTargetAddress RecordAddress = EHFrameAddress + RecordOffset; - auto FDESymbol = createFDERecord( - RecordAddress, EHFrameContent.substr(RecordOffset, RecordLength), - *CIEInfo.CIESymbol, CIEPointerAddress - RecordAddress, *TargetSymbol, - PCBeginAddress - RecordAddress, LSDASymbol, LSDAAddress - RecordAddress); + return std::move(AugInfo); +} + +Expected<JITTargetAddress> +EHFrameEdgeFixer::readAbsolutePointer(LinkGraph &G, + BinaryStreamReader &RecordReader) { + static_assert(sizeof(JITTargetAddress) == sizeof(uint64_t), + "Result must be able to hold a uint64_t"); + JITTargetAddress Addr; + if (G.getPointerSize() == 8) { + if (auto Err = RecordReader.readInteger(Addr)) + return std::move(Err); + } else if (G.getPointerSize() == 4) { + uint32_t Addr32; + if (auto Err = RecordReader.readInteger(Addr32)) + return std::move(Err); + Addr = Addr32; + } else + llvm_unreachable("Pointer size is not 32-bit or 64-bit"); + return Addr; +} + +Expected<Symbol &> EHFrameEdgeFixer::getOrCreateSymbol(ParseContext &PC, + JITTargetAddress Addr) { + Symbol *CanonicalSym = nullptr; + + auto UpdateCanonicalSym = [&](Symbol *Sym) { + if (!CanonicalSym || Sym->getLinkage() < CanonicalSym->getLinkage() || + Sym->getScope() < CanonicalSym->getScope() || + (Sym->hasName() && !CanonicalSym->hasName()) || + Sym->getName() < CanonicalSym->getName()) + CanonicalSym = Sym; + }; + + if (auto *SymbolsAtAddr = PC.AddrToSyms.getSymbolsAt(Addr)) + for (auto *Sym : *SymbolsAtAddr) + UpdateCanonicalSym(Sym); + + // If we found an existing symbol at the given address then use it. + if (CanonicalSym) + return *CanonicalSym; - return FDESymbol.takeError(); + // Otherwise search for a block covering the address and create a new symbol. + auto *B = PC.AddrToBlock.getBlockCovering(Addr); + if (!B) + return make_error<JITLinkError>("No symbol or block covering address " + + formatv("{0:x16}", Addr)); + + return PC.G.addAnonymousSymbol(*B, Addr - B->getAddress(), 0, false, false); } // Determine whether we can register EH tables. @@ -444,9 +709,6 @@ Error walkAppleEHFrameSection(const char *const SectionStart, else Size += 4; uint32_t Offset = *reinterpret_cast<const uint32_t *>(OffsetField); - if (Offset != 0) - if (auto Err = HandleFDE(CurCFIRecord)) - return Err; LLVM_DEBUG({ dbgs() << "Registering eh-frame section:\n"; @@ -456,6 +718,11 @@ Error walkAppleEHFrameSection(const char *const SectionStart, dbgs() << format(" 0x%02" PRIx8, *(CurCFIRecord + I)); dbgs() << " ]\n"; }); + + if (Offset != 0) + if (auto Err = HandleFDE(CurCFIRecord)) + return Err; + CurCFIRecord += Size; Size = *reinterpret_cast<const uint32_t *>(CurCFIRecord); diff --git a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h index 6f9f68ad838..a8cd32c664d 100644 --- a/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h +++ b/llvm/lib/ExecutionEngine/JITLink/EHFrameSupportImpl.h @@ -21,30 +21,30 @@ namespace llvm { namespace jitlink { -/// A generic binary parser for eh-frame sections. -/// -/// Adds blocks and symbols representing CIE and FDE entries to a JITLink graph. -/// -/// This parser assumes that the user has already verified that the EH-frame's -/// address range does not overlap any other section/symbol, so that generated -/// CIE/FDE records do not overlap other sections/symbols. -class EHFrameBinaryParser { +/// A LinkGraph pass that splits blocks in an eh-frame section into sub-blocks +/// representing individual eh-frames. +/// EHFrameSplitter should not be run without EHFrameEdgeFixer, which is +/// responsible for adding FDE-to-CIE edges. +class EHFrameSplitter { public: - EHFrameBinaryParser(JITTargetAddress EHFrameAddress, StringRef EHFrameContent, - unsigned PointerSize, support::endianness Endianness); - virtual ~EHFrameBinaryParser() {} + EHFrameSplitter(StringRef EHFrameSectionName); + Error operator()(LinkGraph &G); - Error addToGraph(); +private: + Error processBlock(LinkGraph &G, Block &B, LinkGraph::SplitBlockCache &Cache); + + StringRef EHFrameSectionName; +}; + +/// A LinkGraph pass that adds missing FDE-to-CIE, FDE-to-PC and FDE-to-LSDA +/// edges. +class EHFrameEdgeFixer { +public: + EHFrameEdgeFixer(StringRef EHFrameSectionName, Edge::Kind FDEToCIE, + Edge::Kind FDEToPCBegin, Edge::Kind FDEToLSDA); + Error operator()(LinkGraph &G); private: - virtual void anchor(); - virtual Symbol *getSymbolAtAddress(JITTargetAddress Addr) = 0; - virtual Symbol &createCIERecord(JITTargetAddress RecordAddr, - StringRef RecordContent) = 0; - virtual Expected<Symbol &> - createFDERecord(JITTargetAddress RecordAddr, StringRef RecordContent, - Symbol &CIE, size_t CIEOffset, Symbol &Func, - size_t FuncOffset, Symbol *LSDA, size_t LSDAOffset) = 0; struct AugmentationInfo { bool AugmentationDataPresent = false; @@ -52,12 +52,6 @@ private: uint8_t Fields[4] = {0x0, 0x0, 0x0, 0x0}; }; - Expected<AugmentationInfo> parseAugmentationString(); - Expected<JITTargetAddress> readAbsolutePointer(); - Error processCIE(size_t RecordOffset, size_t RecordLength); - Error processFDE(size_t RecordOffset, size_t RecordLength, - JITTargetAddress CIEPointerOffset, uint32_t CIEPointer); - struct CIEInformation { CIEInformation() = default; CIEInformation(Symbol &CIESymbol) : CIESymbol(&CIESymbol) {} @@ -65,11 +59,51 @@ private: bool FDEsHaveLSDAField = false; }; - JITTargetAddress EHFrameAddress; - StringRef EHFrameContent; - unsigned PointerSize; - BinaryStreamReader EHFrameReader; - DenseMap<JITTargetAddress, CIEInformation> CIEInfos; + struct EdgeTarget { + EdgeTarget() = default; + EdgeTarget(const Edge &E) : Target(&E.getTarget()), Addend(E.getAddend()) {} + + Symbol *Target = nullptr; + Edge::AddendT Addend = 0; + }; + + using BlockEdgeMap = DenseMap<Edge::OffsetT, EdgeTarget>; + using CIEInfosMap = DenseMap<JITTargetAddress, CIEInformation>; + + struct ParseContext { + ParseContext(LinkGraph &G) : G(G) {} + + Expected<CIEInformation *> findCIEInfo(JITTargetAddress Address) { + auto I = CIEInfos.find(Address); + if (I == CIEInfos.end()) + return make_error<JITLinkError>("No CIE found at address " + + formatv("{0:x16}", Address)); + return &I->second; + } + + LinkGraph &G; + CIEInfosMap CIEInfos; + BlockAddressMap AddrToBlock; + SymbolAddressMap AddrToSyms; + }; + + Error processBlock(ParseContext &PC, Block &B); + Error processCIE(ParseContext &PC, Block &B, size_t RecordOffset, + size_t RecordLength, size_t CIEDeltaFieldOffset); + Error processFDE(ParseContext &PC, Block &B, size_t RecordOffset, + size_t RecordLength, size_t CIEDeltaFieldOffset, + uint32_t CIEDelta, BlockEdgeMap &BlockEdges); + + Expected<AugmentationInfo> + parseAugmentationString(BinaryStreamReader &RecordReader); + Expected<JITTargetAddress> + readAbsolutePointer(LinkGraph &G, BinaryStreamReader &RecordReader); + Expected<Symbol &> getOrCreateSymbol(ParseContext &PC, JITTargetAddress Addr); + + StringRef EHFrameSectionName; + Edge::Kind FDEToCIE; + Edge::Kind FDEToPCBegin; + Edge::Kind FDEToLSDA; }; } // end namespace jitlink diff --git a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp index d4270b5aa79..9707b9624d9 100644 --- a/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/JITLinkGeneric.cpp @@ -151,9 +151,12 @@ JITLinkerBase::SegmentLayoutMap JITLinkerBase::layOutBlocks() { for (auto &KV : Layout) { auto CompareBlocks = [](const Block *LHS, const Block *RHS) { + // Sort by section, address and size if (LHS->getSection().getOrdinal() != RHS->getSection().getOrdinal()) return LHS->getSection().getOrdinal() < RHS->getSection().getOrdinal(); - return LHS->getOrdinal() < RHS->getOrdinal(); + if (LHS->getAddress() != RHS->getAddress()) + return LHS->getAddress() < RHS->getAddress(); + return LHS->getSize() < RHS->getSize(); }; auto &SegLists = KV.second; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h index e1123cd1104..91b1d5a2238 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h +++ b/llvm/lib/ExecutionEngine/JITLink/MachOLinkGraphBuilder.h @@ -30,74 +30,6 @@ public: Expected<std::unique_ptr<LinkGraph>> buildGraph(); protected: - class MachOEHFrameBinaryParser : public EHFrameBinaryParser { - public: - MachOEHFrameBinaryParser(MachOLinkGraphBuilder &Builder, - JITTargetAddress EHFrameAddress, - StringRef EHFrameContent, Section &EHFrameSection, - uint64_t CIEAlignment, uint64_t FDEAlignment, - Edge::Kind FDEToCIERelocKind, - Edge::Kind FDEToTargetRelocKind) - : EHFrameBinaryParser(EHFrameAddress, EHFrameContent, - Builder.getGraph().getPointerSize(), - Builder.getGraph().getEndianness()), - Builder(Builder), EHFrameSection(EHFrameSection), - CIEAlignment(CIEAlignment), FDEAlignment(FDEAlignment), - FDEToCIERelocKind(FDEToCIERelocKind), - FDEToTargetRelocKind(FDEToTargetRelocKind) {} - - Symbol *getSymbolAtAddress(JITTargetAddress Address) override { - if (auto *Sym = Builder.getSymbolByAddress(Address)) - if (Sym->getAddress() == Address) - return Sym; - return nullptr; - } - - Symbol &createCIERecord(JITTargetAddress RecordAddr, - StringRef RecordContent) override { - auto &G = Builder.getGraph(); - auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, - CIEAlignment, 0); - auto &CIESymbol = - G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); - Builder.setCanonicalSymbol(CIESymbol); - return CIESymbol; - } - - Expected<Symbol &> createFDERecord(JITTargetAddress RecordAddr, - StringRef RecordContent, Symbol &CIE, - size_t CIEOffset, Symbol &Func, - size_t FuncOffset, Symbol *LSDA, - size_t LSDAOffset) override { - auto &G = Builder.getGraph(); - auto &B = G.createContentBlock(EHFrameSection, RecordContent, RecordAddr, - FDEAlignment, 0); - - // Add edges to CIE, Func, and (conditionally) LSDA. - B.addEdge(FDEToCIERelocKind, CIEOffset, CIE, 0); - B.addEdge(FDEToTargetRelocKind, FuncOffset, Func, 0); - - if (LSDA) - B.addEdge(FDEToTargetRelocKind, LSDAOffset, *LSDA, 0); - - auto &FDESymbol = - G.addAnonymousSymbol(B, 0, RecordContent.size(), false, false); - - // Add a keep-alive relocation from the function to the FDE to ensure it - // is not dead stripped. - Func.getBlock().addEdge(Edge::KeepAlive, 0, FDESymbol, 0); - - return FDESymbol; - } - - private: - MachOLinkGraphBuilder &Builder; - Section &EHFrameSection; - uint64_t CIEAlignment; - uint64_t FDEAlignment; - Edge::Kind FDEToCIERelocKind; - Edge::Kind FDEToTargetRelocKind; - }; struct NormalizedSymbol { friend class MachOLinkGraphBuilder; diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp index 945343bff89..944767449ce 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_arm64.cpp @@ -27,19 +27,7 @@ class MachOLinkGraphBuilder_arm64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_arm64(const object::MachOObjectFile &Obj) : MachOLinkGraphBuilder(Obj), - NumSymbols(Obj.getSymtabLoadCommand().nsyms) { - addCustomSectionParser( - "__eh_frame", [this](NormalizedSection &EHFrameSection) { - if (!EHFrameSection.Data) - return make_error<JITLinkError>( - "__eh_frame section is marked zero-fill"); - return MachOEHFrameBinaryParser( - *this, EHFrameSection.Address, - StringRef(EHFrameSection.Data, EHFrameSection.Size), - *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) - .addToGraph(); - }); - } + NumSymbols(Obj.getSymtabLoadCommand().nsyms) {} private: static Expected<MachOARM64RelocationKind> diff --git a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp index d83787ffd59..9dbfb6556e3 100644 --- a/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp +++ b/llvm/lib/ExecutionEngine/JITLink/MachO_x86_64.cpp @@ -26,19 +26,7 @@ namespace { class MachOLinkGraphBuilder_x86_64 : public MachOLinkGraphBuilder { public: MachOLinkGraphBuilder_x86_64(const object::MachOObjectFile &Obj) - : MachOLinkGraphBuilder(Obj) { - addCustomSectionParser( - "__eh_frame", [this](NormalizedSection &EHFrameSection) { - if (!EHFrameSection.Data) - return make_error<JITLinkError>( - "__eh_frame section is marked zero-fill"); - return MachOEHFrameBinaryParser( - *this, EHFrameSection.Address, - StringRef(EHFrameSection.Data, EHFrameSection.Size), - *EHFrameSection.GraphSection, 8, 4, NegDelta32, Delta64) - .addToGraph(); - }); - } + : MachOLinkGraphBuilder(Obj) {} private: static Expected<MachOX86RelocationKind> @@ -566,6 +554,11 @@ void jitLink_MachO_x86_64(std::unique_ptr<JITLinkContext> Ctx) { Triple TT("x86_64-apple-macosx"); if (Ctx->shouldAddDefaultTargetPasses(TT)) { + // Add eh-frame passses. + Config.PrePrunePasses.push_back(EHFrameSplitter("__eh_frame")); + Config.PrePrunePasses.push_back( + EHFrameEdgeFixer("__eh_frame", NegDelta32, Delta64, Delta64)); + // Add a mark-live pass. if (auto MarkLive = Ctx->getMarkLivePass(TT)) Config.PrePrunePasses.push_back(std::move(MarkLive)); |