diff options
Diffstat (limited to 'lld/ELF')
-rw-r--r-- | lld/ELF/Config.h | 1 | ||||
-rw-r--r-- | lld/ELF/Driver.cpp | 1 | ||||
-rw-r--r-- | lld/ELF/InputFiles.h | 3 | ||||
-rw-r--r-- | lld/ELF/InputSection.cpp | 32 | ||||
-rw-r--r-- | lld/ELF/Options.td | 5 | ||||
-rw-r--r-- | lld/ELF/Relocations.cpp | 25 | ||||
-rw-r--r-- | lld/ELF/Symbols.h | 10 | ||||
-rw-r--r-- | lld/ELF/SyntheticSections.cpp | 536 | ||||
-rw-r--r-- | lld/ELF/SyntheticSections.h | 161 | ||||
-rw-r--r-- | lld/ELF/Writer.cpp | 3 |
10 files changed, 511 insertions, 266 deletions
diff --git a/lld/ELF/Config.h b/lld/ELF/Config.h index 60276f95ff3..07605b2aff3 100644 --- a/lld/ELF/Config.h +++ b/lld/ELF/Config.h @@ -194,6 +194,7 @@ struct Configuration { uint16_t EMachine = llvm::ELF::EM_NONE; llvm::Optional<uint64_t> ImageBase; uint64_t MaxPageSize; + uint64_t MipsGotSize; uint64_t ZStackSize; unsigned LTOPartitions; unsigned LTOO; diff --git a/lld/ELF/Driver.cpp b/lld/ELF/Driver.cpp index 0290084493f..0d3e0835c98 100644 --- a/lld/ELF/Driver.cpp +++ b/lld/ELF/Driver.cpp @@ -725,6 +725,7 @@ void LinkerDriver::readConfigs(opt::InputArgList &Args) { Config->LTOPartitions = args::getInteger(Args, OPT_lto_partitions, 1); Config->LTOSampleProfile = Args.getLastArgValue(OPT_lto_sample_profile); Config->MapFile = Args.getLastArgValue(OPT_Map); + Config->MipsGotSize = args::getInteger(Args, OPT_mips_got_size, 0xfff0); Config->MergeArmExidx = Args.hasFlag(OPT_merge_exidx_entries, OPT_no_merge_exidx_entries, true); Config->NoinhibitExec = Args.hasArg(OPT_noinhibit_exec); diff --git a/lld/ELF/InputFiles.h b/lld/ELF/InputFiles.h index 65de20483a9..4f5df6f224c 100644 --- a/lld/ELF/InputFiles.h +++ b/lld/ELF/InputFiles.h @@ -120,6 +120,9 @@ public: static bool IsInGroup; static uint32_t NextGroupId; + // Index of MIPS GOT built for this file. + llvm::Optional<size_t> MipsGotIndex; + protected: InputFile(Kind K, MemoryBufferRef M); std::vector<InputSectionBase *> Sections; diff --git a/lld/ELF/InputSection.cpp b/lld/ELF/InputSection.cpp index 32f1f4846f7..b8e4659b921 100644 --- a/lld/ELF/InputSection.cpp +++ b/lld/ELF/InputSection.cpp @@ -477,8 +477,8 @@ static uint64_t getARMStaticBase(const Symbol &Sym) { return OS->PtLoad->FirstSec->Addr; } -static uint64_t getRelocTargetVA(RelType Type, int64_t A, uint64_t P, - const Symbol &Sym, RelExpr Expr) { +static uint64_t getRelocTargetVA(const InputFile &File, RelType Type, int64_t A, + uint64_t P, const Symbol &Sym, RelExpr Expr) { switch (Expr) { case R_INVALID: return 0; @@ -516,9 +516,9 @@ static uint64_t getRelocTargetVA(RelType Type, int64_t A, uint64_t P, case R_TLSDESC_CALL: llvm_unreachable("cannot relocate hint relocs"); case R_MIPS_GOTREL: - return Sym.getVA(A) - InX::MipsGot->getGp(); + return Sym.getVA(A) - InX::MipsGot->getGp(&File); case R_MIPS_GOT_GP: - return InX::MipsGot->getGp() + A; + return InX::MipsGot->getGp(&File) + A; case R_MIPS_GOT_GP_PC: { // R_MIPS_LO16 expression has R_MIPS_GOT_GP_PC type iif the target // is _gp_disp symbol. In that case we should use the following @@ -527,7 +527,7 @@ static uint64_t getRelocTargetVA(RelType Type, int64_t A, uint64_t P, // microMIPS variants of these relocations use slightly different // expressions: AHL + GP - P + 3 for %lo() and AHL + GP - P - 1 for %hi() // to correctly handle less-sugnificant bit of the microMIPS symbol. - uint64_t V = InX::MipsGot->getGp() + A - P; + uint64_t V = InX::MipsGot->getGp(&File) + A - P; if (Type == R_MIPS_LO16 || Type == R_MICROMIPS_LO16) V += 4; if (Type == R_MICROMIPS_LO16 || Type == R_MICROMIPS_HI16) @@ -538,21 +538,24 @@ static uint64_t getRelocTargetVA(RelType Type, int64_t A, uint64_t P, // If relocation against MIPS local symbol requires GOT entry, this entry // should be initialized by 'page address'. This address is high 16-bits // of sum the symbol's value and the addend. - return InX::MipsGot->getVA() + InX::MipsGot->getPageEntryOffset(Sym, A) - - InX::MipsGot->getGp(); + return InX::MipsGot->getVA() + + InX::MipsGot->getPageEntryOffset(File, Sym, A) - + InX::MipsGot->getGp(&File); case R_MIPS_GOT_OFF: case R_MIPS_GOT_OFF32: // In case of MIPS if a GOT relocation has non-zero addend this addend // should be applied to the GOT entry content not to the GOT entry offset. // That is why we use separate expression type. - return InX::MipsGot->getVA() + InX::MipsGot->getSymEntryOffset(Sym, A) - - InX::MipsGot->getGp(); + return InX::MipsGot->getVA() + + InX::MipsGot->getSymEntryOffset(File, Sym, A) - + InX::MipsGot->getGp(&File); case R_MIPS_TLSGD: - return InX::MipsGot->getVA() + InX::MipsGot->getTlsOffset() + - InX::MipsGot->getGlobalDynOffset(Sym) - InX::MipsGot->getGp(); + return InX::MipsGot->getVA() + + InX::MipsGot->getGlobalDynOffset(File, Sym) - + InX::MipsGot->getGp(&File); case R_MIPS_TLSLD: - return InX::MipsGot->getVA() + InX::MipsGot->getTlsOffset() + - InX::MipsGot->getTlsIndexOff() - InX::MipsGot->getGp(); + return InX::MipsGot->getVA() + InX::MipsGot->getTlsIndexOffset(File) - + InX::MipsGot->getGp(&File); case R_PAGE_PC: case R_PLT_PAGE_PC: { uint64_t Dest; @@ -751,7 +754,8 @@ void InputSectionBase::relocateAlloc(uint8_t *Buf, uint8_t *BufEnd) { uint64_t AddrLoc = getOutputSection()->Addr + Offset; RelExpr Expr = Rel.Expr; uint64_t TargetVA = SignExtend64( - getRelocTargetVA(Type, Rel.Addend, AddrLoc, *Rel.Sym, Expr), Bits); + getRelocTargetVA(*File, Type, Rel.Addend, AddrLoc, *Rel.Sym, Expr), + Bits); switch (Expr) { case R_RELAX_GOT_PC: diff --git a/lld/ELF/Options.td b/lld/ELF/Options.td index 256dc5a4496..d9501587583 100644 --- a/lld/ELF/Options.td +++ b/lld/ELF/Options.td @@ -480,3 +480,8 @@ def: F<"EB">; def: F<"EL">; def: JoinedOrSeparate<["-"], "G">; def: F<"Qy">; + +// Hidden option used for testing MIPS multi-GOT implementation. +defm mips_got_size: + Eq<"mips-got-size", "Max size of a single MIPS GOT. 0x10000 by default.">, + Flags<[HelpHidden]>; diff --git a/lld/ELF/Relocations.cpp b/lld/ELF/Relocations.cpp index 2e3d4f192cc..64b23ab87de 100644 --- a/lld/ELF/Relocations.cpp +++ b/lld/ELF/Relocations.cpp @@ -85,27 +85,16 @@ static std::string getLocation(InputSectionBase &S, const Symbol &Sym, // pollute other `handleTlsRelocation` by MIPS `ifs` statements. // Mips has a custom MipsGotSection that handles the writing of GOT entries // without dynamic relocations. -template <class ELFT> static unsigned handleMipsTlsRelocation(RelType Type, Symbol &Sym, InputSectionBase &C, uint64_t Offset, int64_t Addend, RelExpr Expr) { if (Expr == R_MIPS_TLSLD) { - if (InX::MipsGot->addTlsIndex() && Config->Pic) - InX::RelaDyn->addReloc(Target->TlsModuleIndexRel, InX::MipsGot, - InX::MipsGot->getTlsIndexOff(), nullptr); + InX::MipsGot->addTlsIndex(*C.File); C.Relocations.push_back({Expr, Type, Offset, Addend, &Sym}); return 1; } - if (Expr == R_MIPS_TLSGD) { - if (InX::MipsGot->addDynTlsEntry(Sym) && Sym.IsPreemptible) { - uint64_t Off = InX::MipsGot->getGlobalDynOffset(Sym); - InX::RelaDyn->addReloc(Target->TlsModuleIndexRel, InX::MipsGot, Off, - &Sym); - if (Sym.IsPreemptible) - InX::RelaDyn->addReloc(Target->TlsOffsetRel, InX::MipsGot, - Off + Config->Wordsize, &Sym); - } + InX::MipsGot->addDynTlsEntry(*C.File, Sym); C.Relocations.push_back({Expr, Type, Offset, Addend, &Sym}); return 1; } @@ -184,7 +173,7 @@ handleTlsRelocation(RelType Type, Symbol &Sym, InputSectionBase &C, if (Config->EMachine == EM_ARM) return handleARMTlsRelocation<ELFT>(Type, Sym, C, Offset, Addend, Expr); if (Config->EMachine == EM_MIPS) - return handleMipsTlsRelocation<ELFT>(Type, Sym, C, Offset, Addend, Expr); + return handleMipsTlsRelocation(Type, Sym, C, Offset, Addend, Expr); if (isRelExprOneOf<R_TLSDESC, R_TLSDESC_PAGE, R_TLSDESC_CALL>(Expr) && Config->Shared) { @@ -476,7 +465,6 @@ static void replaceWithDefined(Symbol &Sym, SectionBase *Sec, uint64_t Value, Sym.PltIndex = Old.PltIndex; Sym.GotIndex = Old.GotIndex; Sym.VerdefIndex = Old.VerdefIndex; - Sym.IsInGlobalMipsGot = Old.IsInGlobalMipsGot; Sym.IsPreemptible = true; Sym.ExportDynamic = true; Sym.IsUsedInRegularObj = true; @@ -816,7 +804,7 @@ static void processRelocAux(InputSectionBase &Sec, RelExpr Expr, RelType Type, // a dynamic relocation. // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf p.4-19 if (Config->EMachine == EM_MIPS) - InX::MipsGot->addEntry(Sym, Addend, Expr); + InX::MipsGot->addEntry(*Sec.File, Sym, Addend, Expr); return; } } @@ -1002,10 +990,7 @@ static void scanReloc(InputSectionBase &Sec, OffsetGetter &GetOffset, RelTy *&I, // See "Global Offset Table" in Chapter 5 in the following document // for detailed description: // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf - InX::MipsGot->addEntry(Sym, Addend, Expr); - if (Sym.isTls() && Sym.IsPreemptible) - InX::RelaDyn->addReloc(Target->TlsGotRel, InX::MipsGot, - Sym.getGotOffset(), &Sym); + InX::MipsGot->addEntry(*Sec.File, Sym, Addend, Expr); } else if (!Sym.isInGot()) { addGotEntry<ELFT>(Sym); } diff --git a/lld/ELF/Symbols.h b/lld/ELF/Symbols.h index f0912d29b75..8c9513b9368 100644 --- a/lld/ELF/Symbols.h +++ b/lld/ELF/Symbols.h @@ -158,19 +158,13 @@ protected: uint8_t StOther, uint8_t Type) : File(File), NameData(Name.Data), NameSize(Name.Size), Binding(Binding), Type(Type), StOther(StOther), SymbolKind(K), NeedsPltAddr(false), - IsInGlobalMipsGot(false), Is32BitMipsGot(false), IsInIplt(false), - IsInIgot(false), IsPreemptible(false), Used(!Config->GcSections), - NeedsTocRestore(false) {} + IsInIplt(false), IsInIgot(false), IsPreemptible(false), + Used(!Config->GcSections), NeedsTocRestore(false) {} public: // True the symbol should point to its PLT entry. // For SharedSymbol only. unsigned NeedsPltAddr : 1; - // True if this symbol has an entry in the global part of MIPS GOT. - unsigned IsInGlobalMipsGot : 1; - - // True if this symbol is referenced by 32-bit GOT relocations. - unsigned Is32BitMipsGot : 1; // True if this symbol is in the Iplt sub-section of the Plt. unsigned IsInIplt : 1; diff --git a/lld/ELF/SyntheticSections.cpp b/lld/ELF/SyntheticSections.cpp index 1c66bcbe597..cd0d3fe525c 100644 --- a/lld/ELF/SyntheticSections.cpp +++ b/lld/ELF/SyntheticSections.cpp @@ -29,6 +29,7 @@ #include "lld/Common/Strings.h" #include "lld/Common/Threads.h" #include "lld/Common/Version.h" +#include "llvm/ADT/SetOperations.h" #include "llvm/BinaryFormat/Dwarf.h" #include "llvm/DebugInfo/DWARF/DWARFDebugPubTable.h" #include "llvm/Object/Decompressor.h" @@ -636,180 +637,344 @@ void GotSection::writeTo(uint8_t *Buf) { relocateAlloc(Buf - OutSecOff, Buf - OutSecOff + Size); } +static uint64_t getMipsPageAddr(uint64_t Addr) { + return (Addr + 0x8000) & ~0xffff; +} + +static uint64_t getMipsPageCount(uint64_t Size) { + return (Size + 0xfffe) / 0xffff + 1; +} + MipsGotSection::MipsGotSection() : SyntheticSection(SHF_ALLOC | SHF_WRITE | SHF_MIPS_GPREL, SHT_PROGBITS, 16, ".got") {} -void MipsGotSection::addEntry(Symbol &Sym, int64_t Addend, RelExpr Expr) { - // For "true" local symbols which can be referenced from the same module - // only compiler creates two instructions for address loading: - // - // lw $8, 0($gp) # R_MIPS_GOT16 - // addi $8, $8, 0 # R_MIPS_LO16 - // - // The first instruction loads high 16 bits of the symbol address while - // the second adds an offset. That allows to reduce number of required - // GOT entries because only one global offset table entry is necessary - // for every 64 KBytes of local data. So for local symbols we need to - // allocate number of GOT entries to hold all required "page" addresses. - // - // All global symbols (hidden and regular) considered by compiler uniformly. - // It always generates a single `lw` instruction and R_MIPS_GOT16 relocation - // to load address of the symbol. So for each such symbol we need to - // allocate dedicated GOT entry to store its address. - // - // If a symbol is preemptible we need help of dynamic linker to get its - // final address. The corresponding GOT entries are allocated in the - // "global" part of GOT. Entries for non preemptible global symbol allocated - // in the "local" part of GOT. - // - // See "Global Offset Table" in Chapter 5: - // ftp://www.linux-mips.org/pub/linux/mips/doc/ABI/mipsabi.pdf +void MipsGotSection::addEntry(InputFile &File, Symbol &Sym, int64_t Addend, + RelExpr Expr) { + FileGot &G = getGot(File); if (Expr == R_MIPS_GOT_LOCAL_PAGE) { - // At this point we do not know final symbol value so to reduce number - // of allocated GOT entries do the following trick. Save all output - // sections referenced by GOT relocations. Then later in the `finalize` - // method calculate number of "pages" required to cover all saved output - // section and allocate appropriate number of GOT entries. - PageIndexMap.insert({Sym.getOutputSection(), 0}); - return; - } - if (Sym.isTls()) { - // GOT entries created for MIPS TLS relocations behave like - // almost GOT entries from other ABIs. They go to the end - // of the global offset table. - Sym.GotIndex = TlsEntries.size(); - TlsEntries.push_back(&Sym); - return; - } - auto AddEntry = [&](Symbol &S, uint64_t A, GotEntries &Items) { - if (S.isInGot() && !A) - return; - size_t NewIndex = Items.size(); - if (!EntryIndexMap.insert({{&S, A}, NewIndex}).second) - return; - Items.emplace_back(&S, A); - if (!A) - S.GotIndex = NewIndex; - }; - if (Sym.IsPreemptible) { - // Ignore addends for preemptible symbols. They got single GOT entry anyway. - AddEntry(Sym, 0, GlobalEntries); - Sym.IsInGlobalMipsGot = true; - } else if (Expr == R_MIPS_GOT_OFF32) { - AddEntry(Sym, Addend, LocalEntries32); - Sym.Is32BitMipsGot = true; - } else { - // Hold local GOT entries accessed via a 16-bit index separately. - // That allows to write them in the beginning of the GOT and keep - // their indexes as less as possible to escape relocation's overflow. - AddEntry(Sym, Addend, LocalEntries); - } + if (const OutputSection *OS = Sym.getOutputSection()) + G.PagesMap.insert({OS, {}}); + else + G.Local16.insert({{nullptr, getMipsPageAddr(Sym.getVA(Addend))}, 0}); + } else if (Sym.isTls()) + G.Tls.insert({&Sym, 0}); + else if (Sym.IsPreemptible && Expr == R_ABS) + G.Relocs.insert({&Sym, 0}); + else if (Sym.IsPreemptible) + G.Global.insert({&Sym, 0}); + else if (Expr == R_MIPS_GOT_OFF32) + G.Local32.insert({{&Sym, Addend}, 0}); + else + G.Local16.insert({{&Sym, Addend}, 0}); } -bool MipsGotSection::addDynTlsEntry(Symbol &Sym) { - if (Sym.GlobalDynIndex != -1U) - return false; - Sym.GlobalDynIndex = TlsEntries.size(); - // Global Dynamic TLS entries take two GOT slots. - TlsEntries.push_back(nullptr); - TlsEntries.push_back(&Sym); - return true; +void MipsGotSection::addDynTlsEntry(InputFile &File, Symbol &Sym) { + getGot(File).DynTlsSymbols.insert({&Sym, 0}); } -// Reserves TLS entries for a TLS module ID and a TLS block offset. -// In total it takes two GOT slots. -bool MipsGotSection::addTlsIndex() { - if (TlsIndexOff != uint32_t(-1)) - return false; - TlsIndexOff = TlsEntries.size() * Config->Wordsize; - TlsEntries.push_back(nullptr); - TlsEntries.push_back(nullptr); - return true; +void MipsGotSection::addTlsIndex(InputFile &File) { + getGot(File).DynTlsSymbols.insert({nullptr, 0}); } -static uint64_t getMipsPageAddr(uint64_t Addr) { - return (Addr + 0x8000) & ~0xffff; +size_t MipsGotSection::FileGot::getEntriesNum() const { + return getPageEntriesNum() + Local16.size() + Global.size() + Relocs.size() + + Tls.size() + DynTlsSymbols.size() * 2; } -static uint64_t getMipsPageCount(uint64_t Size) { - return (Size + 0xfffe) / 0xffff + 1; +size_t MipsGotSection::FileGot::getPageEntriesNum() const { + size_t Num = 0; + for (const std::pair<const OutputSection *, FileGot::PageBlock> &P : PagesMap) + Num += P.second.Count; + return Num; } -uint64_t MipsGotSection::getPageEntryOffset(const Symbol &B, - int64_t Addend) const { - const OutputSection *OutSec = B.getOutputSection(); - uint64_t SecAddr = getMipsPageAddr(OutSec->Addr); - uint64_t SymAddr = getMipsPageAddr(B.getVA(Addend)); - uint64_t Index = PageIndexMap.lookup(OutSec) + (SymAddr - SecAddr) / 0xffff; - assert(Index < PageEntriesNum); - return (HeaderEntriesNum + Index) * Config->Wordsize; +size_t MipsGotSection::FileGot::getIndexedEntriesNum() const { + size_t Count = getPageEntriesNum() + Local16.size() + Global.size(); + // If there are relocation-only entries in the GOT, TLS entries + // are allocated after them. TLS entries should be addressable + // by 16-bit index so count both reloc-only and TLS entries. + if (!Tls.empty() || !DynTlsSymbols.empty()) + Count += Relocs.size() + Tls.size() + DynTlsSymbols.size() * 2; + return Count; } -uint64_t MipsGotSection::getSymEntryOffset(const Symbol &B, - int64_t Addend) const { - // Calculate offset of the GOT entries block: TLS, global, local. - uint64_t Index = HeaderEntriesNum + PageEntriesNum; - if (B.isTls()) - Index += LocalEntries.size() + LocalEntries32.size() + GlobalEntries.size(); - else if (B.IsInGlobalMipsGot) - Index += LocalEntries.size() + LocalEntries32.size(); - else if (B.Is32BitMipsGot) - Index += LocalEntries.size(); - // Calculate offset of the GOT entry in the block. - if (B.isInGot()) - Index += B.GotIndex; - else { - auto It = EntryIndexMap.find({&B, Addend}); - assert(It != EntryIndexMap.end()); - Index += It->second; +MipsGotSection::FileGot &MipsGotSection::getGot(InputFile &F) { + if (!F.MipsGotIndex.hasValue()) { + Gots.emplace_back(); + Gots.back().File = &F; + F.MipsGotIndex = Gots.size() - 1; + } + return Gots[*F.MipsGotIndex]; +} + +uint64_t MipsGotSection::getPageEntryOffset(const InputFile &F, + const Symbol &Sym, + int64_t Addend) const { + const FileGot &G = Gots[*F.MipsGotIndex]; + uint64_t Index = 0; + if (const OutputSection *OutSec = Sym.getOutputSection()) { + uint64_t SecAddr = getMipsPageAddr(OutSec->Addr); + uint64_t SymAddr = getMipsPageAddr(Sym.getVA(Addend)); + Index = G.PagesMap.lookup(OutSec).FirstIndex + (SymAddr - SecAddr) / 0xffff; + } else { + Index = G.Local16.lookup({nullptr, getMipsPageAddr(Sym.getVA(Addend))}); } return Index * Config->Wordsize; } -uint64_t MipsGotSection::getTlsOffset() const { - return (getLocalEntriesNum() + GlobalEntries.size()) * Config->Wordsize; +uint64_t MipsGotSection::getSymEntryOffset(const InputFile &F, const Symbol &S, + int64_t Addend) const { + const FileGot &G = Gots[*F.MipsGotIndex]; + Symbol *Sym = const_cast<Symbol *>(&S); + if (Sym->isTls()) + return G.Tls.find(Sym)->second * Config->Wordsize; + if (Sym->IsPreemptible) + return G.Global.find(Sym)->second * Config->Wordsize; + return G.Local16.find({Sym, Addend})->second * Config->Wordsize; } -uint64_t MipsGotSection::getGlobalDynOffset(const Symbol &B) const { - return B.GlobalDynIndex * Config->Wordsize; +uint64_t MipsGotSection::getTlsIndexOffset(const InputFile &F) const { + const FileGot &G = Gots[*F.MipsGotIndex]; + return G.DynTlsSymbols.find(nullptr)->second * Config->Wordsize; +} + +uint64_t MipsGotSection::getGlobalDynOffset(const InputFile &F, + const Symbol &S) const { + const FileGot &G = Gots[*F.MipsGotIndex]; + Symbol *Sym = const_cast<Symbol *>(&S); + return G.DynTlsSymbols.find(Sym)->second * Config->Wordsize; } const Symbol *MipsGotSection::getFirstGlobalEntry() const { - return GlobalEntries.empty() ? nullptr : GlobalEntries.front().first; + if (Gots.empty()) + return nullptr; + const FileGot &PrimGot = Gots.front(); + if (!PrimGot.Global.empty()) + return PrimGot.Global.front().first; + if (!PrimGot.Relocs.empty()) + return PrimGot.Relocs.front().first; + return nullptr; } unsigned MipsGotSection::getLocalEntriesNum() const { - return HeaderEntriesNum + PageEntriesNum + LocalEntries.size() + - LocalEntries32.size(); + if (Gots.empty()) + return HeaderEntriesNum; + return HeaderEntriesNum + Gots.front().getPageEntriesNum() + + Gots.front().Local16.size(); +} + +bool MipsGotSection::tryMergeGots(FileGot &Dst, FileGot &Src, bool IsPrimary) { + FileGot Tmp = Dst; + set_union(Tmp.PagesMap, Src.PagesMap); + set_union(Tmp.Local16, Src.Local16); + set_union(Tmp.Global, Src.Global); + set_union(Tmp.Relocs, Src.Relocs); + set_union(Tmp.Tls, Src.Tls); + set_union(Tmp.DynTlsSymbols, Src.DynTlsSymbols); + + size_t Count = IsPrimary ? HeaderEntriesNum : 0; + Count += Tmp.getIndexedEntriesNum(); + + if (Count * Config->Wordsize > Config->MipsGotSize) + return false; + + std::swap(Tmp, Dst); + return true; } void MipsGotSection::finalizeContents() { updateAllocSize(); } bool MipsGotSection::updateAllocSize() { - PageEntriesNum = 0; - for (std::pair<const OutputSection *, size_t> &P : PageIndexMap) { - // For each output section referenced by GOT page relocations calculate - // and save into PageIndexMap an upper bound of MIPS GOT entries required - // to store page addresses of local symbols. We assume the worst case - - // each 64kb page of the output section has at least one GOT relocation - // against it. And take in account the case when the section intersects - // page boundaries. - P.second = PageEntriesNum; - PageEntriesNum += getMipsPageCount(P.first->Size); - } - Size = (getLocalEntriesNum() + GlobalEntries.size() + TlsEntries.size()) * - Config->Wordsize; + Size = HeaderEntriesNum * Config->Wordsize; + for (const FileGot &G : Gots) + Size += G.getEntriesNum() * Config->Wordsize; return false; } +template <class ELFT> void MipsGotSection::build() { + if (Gots.empty()) + return; + + std::vector<FileGot> MergedGots(1); + + // For each GOT move non-preemptible symbols from the `Global` + // to `Local16` list. Preemptible symbol might become non-preemptible + // one if, for example, it gets a related copy relocation. + for (FileGot &Got : Gots) { + for (auto &P: Got.Global) + if (!P.first->IsPreemptible) + Got.Local16.insert({{P.first, 0}, 0}); + Got.Global.remove_if([&](const std::pair<Symbol *, size_t> &P) { + return !P.first->IsPreemptible; + }); + } + + // For each GOT remove "reloc-only" entry if there is "global" + // entry for the same symbol. And add local entries which indexed + // using 32-bit value at the end of 16-bit entries. + for (FileGot &Got : Gots) { + Got.Relocs.remove_if([&](const std::pair<Symbol *, size_t> &P) { + return Got.Global.count(P.first); + }); + set_union(Got.Local16, Got.Local32); + Got.Local32.clear(); + } + + // Evaluate number of "reloc-only" entries in the resulting GOT. + // To do that put all unique "reloc-only" and "global" entries + // from all GOTs to the future primary GOT. + FileGot *PrimGot = &MergedGots.front(); + for (FileGot &Got : Gots) { + set_union(PrimGot->Relocs, Got.Global); + set_union(PrimGot->Relocs, Got.Relocs); + Got.Relocs.clear(); + } + + // Evaluate number of "page" entries in each GOT. + for (FileGot &Got : Gots) { + for (std::pair<const OutputSection *, FileGot::PageBlock> &P : + Got.PagesMap) { + const OutputSection *OS = P.first; + uint64_t SecSize = 0; + for (BaseCommand *Cmd : OS->SectionCommands) { + if (auto *ISD = dyn_cast<InputSectionDescription>(Cmd)) + for (InputSection *IS : ISD->Sections) { + uint64_t Off = alignTo(SecSize, IS->Alignment); + SecSize = Off + IS->getSize(); + } + } + P.second.Count = getMipsPageCount(SecSize); + } + } + + // Merge GOTs. Try to join as much as possible GOTs but do not + // exceed maximum GOT size. In case of overflow create new GOT + // and continue merging. + for (FileGot &SrcGot : Gots) { + FileGot &DstGot = MergedGots.back(); + InputFile *File = SrcGot.File; + if (!tryMergeGots(DstGot, SrcGot, &DstGot == PrimGot)) { + MergedGots.emplace_back(); + std::swap(MergedGots.back(), SrcGot); + } + File->MipsGotIndex = MergedGots.size() - 1; + } + std::swap(Gots, MergedGots); + + // Reduce number of "reloc-only" entries in the primary GOT + // by substracting "global" entries exist in the primary GOT. + PrimGot = &Gots.front(); + PrimGot->Relocs.remove_if([&](const std::pair<Symbol *, size_t> &P) { + return PrimGot->Global.count(P.first); + }); + + // Calculate indexes for each GOT entry. + size_t Index = HeaderEntriesNum; + for (FileGot &Got : Gots) { + Got.StartIndex = &Got == PrimGot ? 0 : Index; + for (std::pair<const OutputSection *, FileGot::PageBlock> &P : + Got.PagesMap) { + // For each output section referenced by GOT page relocations calculate + // and save into PagesMap an upper bound of MIPS GOT entries required + // to store page addresses of local symbols. We assume the worst case - + // each 64kb page of the output section has at least one GOT relocation + // against it. And take in account the case when the section intersects + // page boundaries. + P.second.FirstIndex = Index; + Index += P.second.Count; + } + for (auto &P: Got.Local16) + P.second = Index++; + for (auto &P: Got.Global) + P.second = Index++; + for (auto &P: Got.Relocs) + P.second = Index++; + for (auto &P: Got.Tls) + P.second = Index++; + for (auto &P: Got.DynTlsSymbols) { + P.second = Index; + Index += 2; + } + } + + // Update Symbol::GotIndex field to use this + // value later in the `sortMipsSymbols` function. + for (auto &P : PrimGot->Global) + P.first->GotIndex = P.second; + for (auto &P : PrimGot->Relocs) + P.first->GotIndex = P.second; + + // Create dynamic relocations. + for (FileGot &Got : Gots) { + // Create dynamic relocations for TLS entries. + for (std::pair<Symbol *, size_t> &P : Got.Tls) { + Symbol *S = P.first; + uint64_t Offset = P.second * Config->Wordsize; + if (S->IsPreemptible) + InX::RelaDyn->addReloc(Target->TlsGotRel, this, Offset, S); + } + for (std::pair<Symbol *, size_t> &P : Got.DynTlsSymbols) { + Symbol *S = P.first; + uint64_t Offset = P.second * Config->Wordsize; + if (S == nullptr) { + if (!Config->Pic) + continue; + InX::RelaDyn->addReloc(Target->TlsModuleIndexRel, this, Offset, S); + } else { + if (!P.first->IsPreemptible) + continue; + InX::RelaDyn->addReloc(Target->TlsModuleIndexRel, this, Offset, S); + Offset += Config->Wordsize; + InX::RelaDyn->addReloc(Target->TlsOffsetRel, this, Offset, S); + } + } + + // Do not create dynamic relocations for non-TLS + // entries in the primary GOT. + if (&Got == PrimGot) + continue; + + // Dynamic relocations for "global" entries. + for (const std::pair<Symbol *, size_t> &P : Got.Global) { + uint64_t Offset = P.second * Config->Wordsize; + InX::RelaDyn->addReloc(Target->RelativeRel, this, Offset, P.first); + } + if (!Config->Pic) + continue; + // Dynamic relocations for "local" entries in case of PIC. + for (const std::pair<const OutputSection *, FileGot::PageBlock> &L : + Got.PagesMap) { + size_t PageCount = L.second.Count; + for (size_t PI = 0; PI < PageCount; ++PI) { + uint64_t Offset = (L.second.FirstIndex + PI) * Config->Wordsize; + InX::RelaDyn->addReloc({Target->RelativeRel, this, Offset, L.first, + int64_t(PI * 0x10000)}); + } + } + for (const std::pair<GotEntry, size_t> &P : Got.Local16) { + uint64_t Offset = P.second * Config->Wordsize; + InX::RelaDyn->addReloc({Target->RelativeRel, this, Offset, true, + P.first.first, P.first.second}); + } + } +} + bool MipsGotSection::empty() const { // We add the .got section to the result for dynamic MIPS target because // its address and properties are mentioned in the .dynamic section. return Config->Relocatable; } -uint64_t MipsGotSection::getGp() const { return ElfSym::MipsGp->getVA(0); } +uint64_t MipsGotSection::getGp(const InputFile *F) const { + // For files without related GOT or files refer a primary GOT + // returns "common" _gp value. For secondary GOTs calculate + // individual _gp values. + if (!F || !F->MipsGotIndex.hasValue() || *F->MipsGotIndex == 0) + return ElfSym::MipsGp->getVA(0); + return getVA() + Gots[*F->MipsGotIndex].StartIndex * Config->Wordsize + + 0x7ff0; +} void MipsGotSection::writeTo(uint8_t *Buf) { // Set the MSB of the second GOT slot. This is not required by any @@ -827,49 +992,47 @@ void MipsGotSection::writeTo(uint8_t *Buf) { // keep doing this for now. We really need to revisit this to see // if we had to do this. writeUint(Buf + Config->Wordsize, (uint64_t)1 << (Config->Wordsize * 8 - 1)); - Buf += HeaderEntriesNum * Config->Wordsize; - // Write 'page address' entries to the local part of the GOT. - for (std::pair<const OutputSection *, size_t> &L : PageIndexMap) { - size_t PageCount = getMipsPageCount(L.first->Size); - uint64_t FirstPageAddr = getMipsPageAddr(L.first->Addr); - for (size_t PI = 0; PI < PageCount; ++PI) { - uint8_t *Entry = Buf + (L.second + PI) * Config->Wordsize; - writeUint(Entry, FirstPageAddr + PI * 0x10000); - } - } - Buf += PageEntriesNum * Config->Wordsize; - auto AddEntry = [&](const GotEntry &SA) { - uint8_t *Entry = Buf; - Buf += Config->Wordsize; - const Symbol *Sym = SA.first; - uint64_t VA = Sym->getVA(SA.second); - if (Sym->StOther & STO_MIPS_MICROMIPS) - VA |= 1; - writeUint(Entry, VA); - }; - std::for_each(std::begin(LocalEntries), std::end(LocalEntries), AddEntry); - std::for_each(std::begin(LocalEntries32), std::end(LocalEntries32), AddEntry); - std::for_each(std::begin(GlobalEntries), std::end(GlobalEntries), AddEntry); - // Initialize TLS-related GOT entries. If the entry has a corresponding - // dynamic relocations, leave it initialized by zero. Write down adjusted - // TLS symbol's values otherwise. To calculate the adjustments use offsets - // for thread-local storage. - // https://www.linux-mips.org/wiki/NPTL - if (TlsIndexOff != -1U && !Config->Pic) - writeUint(Buf + TlsIndexOff, 1); - for (const Symbol *B : TlsEntries) { - if (!B || B->IsPreemptible) - continue; - uint64_t VA = B->getVA(); - if (B->GotIndex != -1U) { - uint8_t *Entry = Buf + B->GotIndex * Config->Wordsize; - writeUint(Entry, VA - 0x7000); + for (const FileGot &G : Gots) { + auto Write = [&](size_t I, const Symbol *S, int64_t A) { + uint64_t VA = A; + if (S) { + VA = S->getVA(A); + if (S->StOther & STO_MIPS_MICROMIPS) + VA |= 1; + } + writeUint(Buf + I * Config->Wordsize, VA); + }; + // Write 'page address' entries to the local part of the GOT. + for (const std::pair<const OutputSection *, FileGot::PageBlock> &L : + G.PagesMap) { + size_t PageCount = L.second.Count; + uint64_t FirstPageAddr = getMipsPageAddr(L.first->Addr); + for (size_t PI = 0; PI < PageCount; ++PI) + Write(L.second.FirstIndex + PI, nullptr, FirstPageAddr + PI * 0x10000); } - if (B->GlobalDynIndex != -1U) { - uint8_t *Entry = Buf + B->GlobalDynIndex * Config->Wordsize; - writeUint(Entry, 1); - Entry += Config->Wordsize; - writeUint(Entry, VA - 0x8000); + // Local, global, TLS, reloc-only entries. + // If TLS entry has a corresponding dynamic relocations, leave it + // initialized by zero. Write down adjusted TLS symbol's values otherwise. + // To calculate the adjustments use offsets for thread-local storage. + // https://www.linux-mips.org/wiki/NPTL + for (const std::pair<GotEntry, size_t> &P : G.Local16) + Write(P.second, P.first.first, P.first.second); + // Write VA to the primary GOT only. For secondary GOTs that + // will be done by REL32 dynamic relocations. + if (&G == &Gots.front()) + for (const std::pair<const Symbol *, size_t> &P : G.Global) + Write(P.second, P.first, 0); + for (const std::pair<Symbol *, size_t> &P : G.Relocs) + Write(P.second, P.first, 0); + for (const std::pair<Symbol *, size_t> &P : G.Tls) + Write(P.second, P.first, P.first->IsPreemptible ? 0 : -0x7000); + for (const std::pair<Symbol *, size_t> &P : G.DynTlsSymbols) { + if (P.first == nullptr && !Config->Pic) + Write(P.second, nullptr, 1); + else if (P.first && !P.first->IsPreemptible) { + Write(P.second, nullptr, 1); + Write(P.second + 1, P.first, -0x8000); + } } } } @@ -1234,7 +1397,10 @@ uint64_t DynamicReloc::getOffset() const { int64_t DynamicReloc::computeAddend() const { if (UseSymVA) return Sym->getVA(Addend); - return Addend; + if (!OutputSec) + return Addend; + // See the comment in the DynamicReloc ctor. + return getMipsPageAddr(OutputSec->Addr) + Addend; } uint32_t DynamicReloc::getSymIndex() const { @@ -1288,17 +1454,6 @@ static void encodeDynamicReloc(typename ELFT::Rela *P, if (Config->IsRela) P->r_addend = Rel.computeAddend(); P->r_offset = Rel.getOffset(); - if (Config->EMachine == EM_MIPS && Rel.getInputSec() == InX::MipsGot) - // The MIPS GOT section contains dynamic relocations that correspond to TLS - // entries. These entries are placed after the global and local sections of - // the GOT. At the point when we create these relocations, the size of the - // global and local sections is unknown, so the offset that we store in the - // TLS entry's DynamicReloc is relative to the start of the TLS section of - // the GOT, rather than being relative to the start of the GOT. This line of - // code adds the size of the global and local sections to the virtual - // address computed by getOffset() in order to adjust it into the TLS - // section. - P->r_offset += InX::MipsGot->getTlsOffset(); P->setSymbolAndType(Rel.getSymIndex(), Rel.Type, Config->IsMips64EL); } @@ -1537,10 +1692,8 @@ static bool sortMipsSymbols(const SymbolTableEntry &L, const SymbolTableEntry &R) { // Sort entries related to non-local preemptible symbols by GOT indexes. // All other entries go to the first part of GOT in arbitrary order. - bool LIsInLocalGot = !L.Sym->IsInGlobalMipsGot; - bool RIsInLocalGot = !R.Sym->IsInGlobalMipsGot; - if (LIsInLocalGot || RIsInLocalGot) - return !RIsInLocalGot; + if (!L.Sym->isInGot() || !R.Sym->isInGot()) + return !L.Sym->isInGot(); return L.Sym->GotIndex < R.Sym->GotIndex; } @@ -2746,6 +2899,11 @@ template void PltSection::addEntry<ELF32BE>(Symbol &Sym); template void PltSection::addEntry<ELF64LE>(Symbol &Sym); template void PltSection::addEntry<ELF64BE>(Symbol &Sym); +template void MipsGotSection::build<ELF32LE>(); +template void MipsGotSection::build<ELF32BE>(); +template void MipsGotSection::build<ELF64LE>(); +template void MipsGotSection::build<ELF64BE>(); + template class elf::MipsAbiFlagsSection<ELF32LE>; template class elf::MipsAbiFlagsSection<ELF32BE>; template class elf::MipsAbiFlagsSection<ELF64LE>; diff --git a/lld/ELF/SyntheticSections.h b/lld/ELF/SyntheticSections.h index fc5ffa3f274..9776f2a32d0 100644 --- a/lld/ELF/SyntheticSections.h +++ b/lld/ELF/SyntheticSections.h @@ -178,12 +178,21 @@ public: bool updateAllocSize() override; void finalizeContents() override; bool empty() const override; - void addEntry(Symbol &Sym, int64_t Addend, RelExpr Expr); - bool addDynTlsEntry(Symbol &Sym); - bool addTlsIndex(); - uint64_t getPageEntryOffset(const Symbol &B, int64_t Addend) const; - uint64_t getSymEntryOffset(const Symbol &B, int64_t Addend) const; - uint64_t getGlobalDynOffset(const Symbol &B) const; + + // Join separate GOTs built for each input file to generate + // primary and optional multiple secondary GOTs. + template <class ELFT> void build(); + + void addEntry(InputFile &File, Symbol &Sym, int64_t Addend, RelExpr Expr); + void addDynTlsEntry(InputFile &File, Symbol &Sym); + void addTlsIndex(InputFile &File); + + uint64_t getPageEntryOffset(const InputFile &F, const Symbol &S, + int64_t Addend) const; + uint64_t getSymEntryOffset(const InputFile &F, const Symbol &S, + int64_t Addend) const; + uint64_t getGlobalDynOffset(const InputFile &F, const Symbol &S) const; + uint64_t getTlsIndexOffset(const InputFile &F) const; // Returns the symbol which corresponds to the first entry of the global part // of GOT on MIPS platform. It is required to fill up MIPS-specific dynamic @@ -195,13 +204,8 @@ public: // the number of reserved entries. unsigned getLocalEntriesNum() const; - // Returns offset of TLS part of the MIPS GOT table. This part goes - // after 'local' and 'global' entries. - uint64_t getTlsOffset() const; - - uint32_t getTlsIndexOff() const { return TlsIndexOff; } - - uint64_t getGp() const; + // Return _gp value for primary GOT (nullptr) or particular input file. + uint64_t getGp(const InputFile *F = nullptr) const; private: // MIPS GOT consists of three parts: local, global and tls. Each part @@ -240,32 +244,110 @@ private: // addressing, but MIPS ABI requires that these entries be present in GOT. // TLS entries: // Entries created by TLS relocations. + // + // If the sum of local, global and tls entries is less than 64K only single + // got is enough. Otherwise, multi-got is created. Series of primary and + // multiple secondary GOTs have the following layout: + // - Primary GOT + // Header + // Local entries + // Global entries + // Relocation only entries + // TLS entries + // + // - Secondary GOT + // Local entries + // Global entries + // TLS entries + // ... + // + // All GOT entries required by relocations from a single input file entirely + // belong to either primary or one of secondary GOTs. To reference GOT entries + // each GOT has its own _gp value points to the "middle" of the GOT. + // In the code this value loaded to the register which is used for GOT access. + // + // MIPS 32 function's prologue: + // lui v0,0x0 + // 0: R_MIPS_HI16 _gp_disp + // addiu v0,v0,0 + // 4: R_MIPS_LO16 _gp_disp + // + // MIPS 64: + // lui at,0x0 + // 14: R_MIPS_GPREL16 main + // + // Dynamic linker does not know anything about secondary GOTs and cannot + // use a regular MIPS mechanism for GOT entries initialization. So we have + // to use an approach accepted by other architectures and create dynamic + // relocations R_MIPS_REL32 to initialize global entries (and local in case + // of PIC code) in secondary GOTs. But ironically MIPS dynamic linker + // requires GOT entries and correspondingly ordered dynamic symbol table + // entries to deal with dynamic relocations. To handle this problem + // relocation-only section in the primary GOT contains entries for all + // symbols referenced in global parts of secondary GOTs. Although the sum + // of local and normal global entries of the primary got should be less + // than 64K, the size of the primary got (including relocation-only entries + // can be greater than 64K, because parts of the primary got that overflow + // the 64K limit are used only by the dynamic linker at dynamic link-time + // and not by 16-bit gp-relative addressing at run-time. + // + // For complete multi-GOT description see the following link + // https://dmz-portal.mips.com/wiki/MIPS_Multi_GOT // Number of "Header" entries. static const unsigned HeaderEntriesNum = 2; - // Number of allocated "Page" entries. - uint32_t PageEntriesNum = 0; - // Map output sections referenced by MIPS GOT relocations - // to the first index of "Page" entries allocated for this section. - llvm::SmallMapVector<const OutputSection *, size_t, 16> PageIndexMap; - - typedef std::pair<const Symbol *, uint64_t> GotEntry; - typedef std::vector<GotEntry> GotEntries; - // Map from Symbol-Addend pair to the GOT index. - llvm::DenseMap<GotEntry, size_t> EntryIndexMap; - // Local entries (16-bit access). - GotEntries LocalEntries; - // Local entries (32-bit access). - GotEntries LocalEntries32; - - // Normal and reloc-only global entries. - GotEntries GlobalEntries; - - // TLS entries. - std::vector<const Symbol *> TlsEntries; - uint32_t TlsIndexOff = -1; uint64_t Size = 0; + + size_t LocalEntriesNum = 0; + + // Symbol and addend. + typedef std::pair<Symbol *, int64_t> GotEntry; + + struct FileGot { + InputFile *File = nullptr; + size_t StartIndex = 0; + + struct PageBlock { + size_t FirstIndex = 0; + size_t Count = 0; + }; + + // Map output sections referenced by MIPS GOT relocations + // to the description (index/count) "page" entries allocated + // for this section. + llvm::SmallMapVector<const OutputSection *, PageBlock, 16> PagesMap; + // Maps from Symbol+Addend pair or just Symbol to the GOT entry index. + llvm::MapVector<GotEntry, size_t> Local16; + llvm::MapVector<GotEntry, size_t> Local32; + llvm::MapVector<Symbol *, size_t> Global; + llvm::MapVector<Symbol *, size_t> Relocs; + llvm::MapVector<Symbol *, size_t> Tls; + // Set of symbols referenced by dynamic TLS relocations. + llvm::MapVector<Symbol *, size_t> DynTlsSymbols; + + // Total number of all entries. + size_t getEntriesNum() const; + // Number of "page" entries. + size_t getPageEntriesNum() const; + // Number of entries require 16-bit index to access. + size_t getIndexedEntriesNum() const; + + bool isOverflow() const; + }; + + // Container of GOT created for each input file. + // After building a final series of GOTs this container + // holds primary and secondary GOT's. + std::vector<FileGot> Gots; + + // Return (and create if necessary) `FileGot`. + FileGot &getGot(InputFile &F); + + // Try to merge two GOTs. In case of success the `Dst` contains + // result of merging and the function returns true. In case of + // ovwerflow the `Dst` is unchanged and the function returns false. + bool tryMergeGots(FileGot & Dst, FileGot & Src, bool IsPrimary); }; class GotPltSection final : public SyntheticSection { @@ -318,7 +400,15 @@ public: DynamicReloc(RelType Type, const InputSectionBase *InputSec, uint64_t OffsetInSec, bool UseSymVA, Symbol *Sym, int64_t Addend) : Type(Type), Sym(Sym), InputSec(InputSec), OffsetInSec(OffsetInSec), - UseSymVA(UseSymVA), Addend(Addend) {} + UseSymVA(UseSymVA), Addend(Addend), OutputSec(nullptr) {} + // This constructor records dynamic relocation settings used by MIPS + // multi-GOT implementation. It's to relocate addresses of 64kb pages + // lie inside the output section. + DynamicReloc(RelType Type, const InputSectionBase *InputSec, + uint64_t OffsetInSec, const OutputSection *OutputSec, + int64_t Addend) + : Type(Type), Sym(nullptr), InputSec(InputSec), OffsetInSec(OffsetInSec), + UseSymVA(false), Addend(Addend), OutputSec(OutputSec) {} uint64_t getOffset() const; uint32_t getSymIndex() const; @@ -341,6 +431,7 @@ private: // plus the original addend as the final relocation addend. bool UseSymVA; int64_t Addend; + const OutputSection *OutputSec; }; template <class ELFT> class DynamicSection final : public SyntheticSection { diff --git a/lld/ELF/Writer.cpp b/lld/ELF/Writer.cpp index 998b0dcbe57..0a23d467922 100644 --- a/lld/ELF/Writer.cpp +++ b/lld/ELF/Writer.cpp @@ -1611,6 +1611,9 @@ template <class ELFT> void Writer<ELFT>::finalizeSections() { if (errorCount()) return; + if (InX::MipsGot) + InX::MipsGot->build<ELFT>(); + removeUnusedSyntheticSections(); sortSections(); |