diff options
Diffstat (limited to 'llvm/lib/Transforms')
| -rw-r--r-- | llvm/lib/Transforms/IPO/LowerTypeTests.cpp | 217 |
1 files changed, 129 insertions, 88 deletions
diff --git a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp index 241f475cea9..e4e3f151073 100644 --- a/llvm/lib/Transforms/IPO/LowerTypeTests.cpp +++ b/llvm/lib/Transforms/IPO/LowerTypeTests.cpp @@ -206,6 +206,34 @@ struct ByteArrayInfo { Constant *Mask; }; +/// A POD-like structure that we use to store a global reference together with +/// its metadata types. In this pass we frequently need to query the set of +/// metadata types referenced by a global, which at the IR level is an expensive +/// operation involving a map lookup; this data structure helps to reduce the +/// number of times we need to do this lookup. +class GlobalTypeMember { + GlobalObject *GO; + size_t NTypes; + MDNode *Types[1]; // We treat this as a flexible array member. + +public: + static GlobalTypeMember *create(BumpPtrAllocator &Alloc, GlobalObject *GO, + ArrayRef<MDNode *> Types) { + auto GTM = Alloc.Allocate<GlobalTypeMember>( + sizeof(GlobalTypeMember) + (Types.size() - 1) * sizeof(MDNode *)); + GTM->GO = GO; + GTM->NTypes = Types.size(); + std::copy(Types.begin(), Types.end(), GTM->Types); + return GTM; + } + GlobalObject *getGlobal() const { + return GO; + } + ArrayRef<MDNode *> types() const { + return {&Types[0], &Types[NTypes]}; + } +}; + class LowerTypeTestsModule { Module &M; @@ -233,20 +261,20 @@ class LowerTypeTestsModule { BitSetInfo buildBitSet(Metadata *TypeId, - const DenseMap<GlobalObject *, uint64_t> &GlobalLayout); + const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout); ByteArrayInfo *createByteArray(BitSetInfo &BSI); void allocateByteArrays(); Value *createBitSetTest(IRBuilder<> &B, BitSetInfo &BSI, ByteArrayInfo *&BAI, Value *BitOffset); - void - lowerTypeTestCalls(ArrayRef<Metadata *> TypeIds, Constant *CombinedGlobalAddr, - const DenseMap<GlobalObject *, uint64_t> &GlobalLayout); + void lowerTypeTestCalls( + ArrayRef<Metadata *> TypeIds, Constant *CombinedGlobalAddr, + const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout); Value * lowerBitSetCall(CallInst *CI, BitSetInfo &BSI, ByteArrayInfo *&BAI, Constant *CombinedGlobal, const DenseMap<GlobalObject *, uint64_t> &GlobalLayout); void buildBitSetsFromGlobalVariables(ArrayRef<Metadata *> TypeIds, - ArrayRef<GlobalVariable *> Globals); + ArrayRef<GlobalTypeMember *> Globals); unsigned getJumpTableEntrySize(); Type *getJumpTableEntryType(); void createJumpTableEntry(raw_ostream &OS, Function *Dest, unsigned Distance); @@ -254,13 +282,13 @@ class LowerTypeTestsModule { GlobalVariable *JumpTable, unsigned Distance); void verifyTypeMDNode(GlobalObject *GO, MDNode *Type); void buildBitSetsFromFunctions(ArrayRef<Metadata *> TypeIds, - ArrayRef<Function *> Functions); + ArrayRef<GlobalTypeMember *> Functions); void buildBitSetsFromFunctionsNative(ArrayRef<Metadata *> TypeIds, - ArrayRef<Function *> Functions); + ArrayRef<GlobalTypeMember *> Functions); void buildBitSetsFromFunctionsWASM(ArrayRef<Metadata *> TypeIds, - ArrayRef<Function *> Functions); + ArrayRef<GlobalTypeMember *> Functions); void buildBitSetsFromDisjointSet(ArrayRef<Metadata *> TypeIds, - ArrayRef<GlobalObject *> Globals); + ArrayRef<GlobalTypeMember *> Globals); void replaceWeakDeclarationWithJumpTablePtr(Function *F, Constant *JT); void moveInitializerToModuleConstructor(GlobalVariable *GV); @@ -296,16 +324,14 @@ ModulePass *llvm::createLowerTypeTestsPass() { return new LowerTypeTests; } /// Build a bit set for TypeId using the object layouts in /// GlobalLayout. BitSetInfo LowerTypeTestsModule::buildBitSet( - Metadata *TypeId, const DenseMap<GlobalObject *, uint64_t> &GlobalLayout) { + Metadata *TypeId, + const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) { BitSetBuilder BSB; // Compute the byte offset of each address associated with this type // identifier. - SmallVector<MDNode *, 2> Types; for (auto &GlobalAndOffset : GlobalLayout) { - Types.clear(); - GlobalAndOffset.first->getMetadata(LLVMContext::MD_type, Types); - for (MDNode *Type : Types) { + for (MDNode *Type : GlobalAndOffset.first->types()) { if (Type->getOperand(1) != TypeId) continue; uint64_t Offset = @@ -521,16 +547,17 @@ Value *LowerTypeTestsModule::lowerBitSetCall( /// Given a disjoint set of type identifiers and globals, lay out the globals, /// build the bit sets and lower the llvm.type.test calls. void LowerTypeTestsModule::buildBitSetsFromGlobalVariables( - ArrayRef<Metadata *> TypeIds, ArrayRef<GlobalVariable *> Globals) { + ArrayRef<Metadata *> TypeIds, ArrayRef<GlobalTypeMember *> Globals) { // Build a new global with the combined contents of the referenced globals. // This global is a struct whose even-indexed elements contain the original // contents of the referenced globals and whose odd-indexed elements contain // any padding required to align the next element to the next power of 2. std::vector<Constant *> GlobalInits; const DataLayout &DL = M.getDataLayout(); - for (GlobalVariable *G : Globals) { - GlobalInits.push_back(G->getInitializer()); - uint64_t InitSize = DL.getTypeAllocSize(G->getValueType()); + for (GlobalTypeMember *G : Globals) { + GlobalVariable *GV = cast<GlobalVariable>(G->getGlobal()); + GlobalInits.push_back(GV->getInitializer()); + uint64_t InitSize = DL.getTypeAllocSize(GV->getValueType()); // Compute the amount of padding required. uint64_t Padding = NextPowerOf2(InitSize - 1) - InitSize; @@ -554,7 +581,7 @@ void LowerTypeTestsModule::buildBitSetsFromGlobalVariables( const StructLayout *CombinedGlobalLayout = DL.getStructLayout(NewTy); // Compute the offsets of the original globals within the new global. - DenseMap<GlobalObject *, uint64_t> GlobalLayout; + DenseMap<GlobalTypeMember *, uint64_t> GlobalLayout; for (unsigned I = 0; I != Globals.size(); ++I) // Multiply by 2 to account for padding elements. GlobalLayout[Globals[I]] = CombinedGlobalLayout->getElementOffset(I * 2); @@ -565,31 +592,36 @@ void LowerTypeTestsModule::buildBitSetsFromGlobalVariables( // global from which we built the combined global, and replace references // to the original globals with references to the aliases. for (unsigned I = 0; I != Globals.size(); ++I) { + GlobalVariable *GV = cast<GlobalVariable>(Globals[I]->getGlobal()); + // Multiply by 2 to account for padding elements. Constant *CombinedGlobalIdxs[] = {ConstantInt::get(Int32Ty, 0), ConstantInt::get(Int32Ty, I * 2)}; Constant *CombinedGlobalElemPtr = ConstantExpr::getGetElementPtr( NewInit->getType(), CombinedGlobal, CombinedGlobalIdxs); if (LinkerSubsectionsViaSymbols) { - Globals[I]->replaceAllUsesWith(CombinedGlobalElemPtr); + GV->replaceAllUsesWith(CombinedGlobalElemPtr); } else { - assert(Globals[I]->getType()->getAddressSpace() == 0); + assert(GV->getType()->getAddressSpace() == 0); GlobalAlias *GAlias = GlobalAlias::create(NewTy->getElementType(I * 2), 0, - Globals[I]->getLinkage(), "", + GV->getLinkage(), "", CombinedGlobalElemPtr, &M); - GAlias->setVisibility(Globals[I]->getVisibility()); - GAlias->takeName(Globals[I]); - Globals[I]->replaceAllUsesWith(GAlias); + GAlias->setVisibility(GV->getVisibility()); + GAlias->takeName(GV); + GV->replaceAllUsesWith(GAlias); } - Globals[I]->eraseFromParent(); + GV->eraseFromParent(); } } void LowerTypeTestsModule::lowerTypeTestCalls( ArrayRef<Metadata *> TypeIds, Constant *CombinedGlobalAddr, - const DenseMap<GlobalObject *, uint64_t> &GlobalLayout) { + const DenseMap<GlobalTypeMember *, uint64_t> &GlobalLayout) { Constant *CombinedGlobalIntAddr = ConstantExpr::getPtrToInt(CombinedGlobalAddr, IntPtrTy); + DenseMap<GlobalObject *, uint64_t> GlobalObjLayout; + for (auto &P : GlobalLayout) + GlobalObjLayout[P.first->getGlobal()] = P.second; // For each type identifier in this disjoint set... for (Metadata *TypeId : TypeIds) { @@ -609,7 +641,7 @@ void LowerTypeTestsModule::lowerTypeTestCalls( for (CallInst *CI : TypeTestCallSites[TypeId]) { ++NumTypeTestCallsLowered; Value *Lowered = - lowerBitSetCall(CI, BSI, BAI, CombinedGlobalIntAddr, GlobalLayout); + lowerBitSetCall(CI, BSI, BAI, CombinedGlobalIntAddr, GlobalObjLayout); CI->replaceAllUsesWith(Lowered); CI->eraseFromParent(); } @@ -730,7 +762,7 @@ Type *LowerTypeTestsModule::getJumpTableEntryType() { /// Given a disjoint set of type identifiers and functions, build the bit sets /// and lower the llvm.type.test calls, architecture dependently. void LowerTypeTestsModule::buildBitSetsFromFunctions( - ArrayRef<Metadata *> TypeIds, ArrayRef<Function *> Functions) { + ArrayRef<Metadata *> TypeIds, ArrayRef<GlobalTypeMember *> Functions) { if (Arch == Triple::x86 || Arch == Triple::x86_64 || Arch == Triple::arm || Arch == Triple::aarch64) buildBitSetsFromFunctionsNative(TypeIds, Functions); @@ -803,7 +835,7 @@ void LowerTypeTestsModule::replaceWeakDeclarationWithJumpTablePtr( /// Given a disjoint set of type identifiers and functions, build a jump table /// for the functions, build the bit sets and lower the llvm.type.test calls. void LowerTypeTestsModule::buildBitSetsFromFunctionsNative( - ArrayRef<Metadata *> TypeIds, ArrayRef<Function *> Functions) { + ArrayRef<Metadata *> TypeIds, ArrayRef<GlobalTypeMember *> Functions) { // Unlike the global bitset builder, the function bitset builder cannot // re-arrange functions in a particular order and base its calculations on the // layout of the functions' entry points, as we have no idea how large a @@ -884,7 +916,7 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative( assert(!Functions.empty()); // Build a simple layout based on the regular layout of jump tables. - DenseMap<GlobalObject *, uint64_t> GlobalLayout; + DenseMap<GlobalTypeMember *, uint64_t> GlobalLayout; unsigned EntrySize = getJumpTableEntrySize(); for (unsigned I = 0; I != Functions.size(); ++I) GlobalLayout[Functions[I]] = I * EntrySize; @@ -905,48 +937,48 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative( // Build aliases pointing to offsets into the jump table, and replace // references to the original functions with references to the aliases. for (unsigned I = 0; I != Functions.size(); ++I) { + Function *F = cast<Function>(Functions[I]->getGlobal()); + // Need a name for the asm label. Normally, unnamed functions get temporary // asm labels in TargetLoweringObjectFile but we don't have access to that // here. - if (!Functions[I]->hasName()) - Functions[I]->setName("unnamed"); - if (LinkerSubsectionsViaSymbols || Functions[I]->isDeclarationForLinker()) { + if (!F->hasName()) + F->setName("unnamed"); + if (LinkerSubsectionsViaSymbols || F->isDeclarationForLinker()) { Constant *CombinedGlobalElemPtr = ConstantExpr::getBitCast( ConstantExpr::getGetElementPtr( JumpTableType, JumpTable, ArrayRef<Constant *>{ConstantInt::get(IntPtrTy, 0), ConstantInt::get(IntPtrTy, I)}), - Functions[I]->getType()); - + F->getType()); - if (Functions[I]->isWeakForLinker()) { - AsmOS << ".weak " << Functions[I]->getName() << "\n"; - replaceWeakDeclarationWithJumpTablePtr(Functions[I], - CombinedGlobalElemPtr); + if (F->isWeakForLinker()) { + AsmOS << ".weak " << F->getName() << "\n"; + replaceWeakDeclarationWithJumpTablePtr(F, CombinedGlobalElemPtr); } else { - Functions[I]->replaceAllUsesWith(CombinedGlobalElemPtr); + F->replaceAllUsesWith(CombinedGlobalElemPtr); } } else { - assert(Functions[I]->getType()->getAddressSpace() == 0); + assert(F->getType()->getAddressSpace() == 0); - createJumpTableAlias(AsmOS, Functions[I], JumpTable, I); + createJumpTableAlias(AsmOS, F, JumpTable, I); Function *DeclAlias = - Function::Create(cast<FunctionType>(Functions[I]->getValueType()), + Function::Create(cast<FunctionType>(F->getValueType()), GlobalValue::ExternalLinkage, "", &M); // Since the alias (DeclAlias) is actually a declaration, it can not have // internal linkage. Compensate for that by giving it hidden visibility. // With this we end up with a GOT relocation against a local symbol. - DeclAlias->setVisibility(Functions[I]->hasLocalLinkage() + DeclAlias->setVisibility(F->hasLocalLinkage() ? GlobalValue::HiddenVisibility - : Functions[I]->getVisibility()); - DeclAlias->takeName(Functions[I]); + : F->getVisibility()); + DeclAlias->takeName(F); // Unnamed functions can not be added to llvm.used. - Functions[I]->setName(DeclAlias->getName() + ".cfi"); - Functions[I]->replaceAllUsesWith(DeclAlias); + F->setName(DeclAlias->getName() + ".cfi"); + F->replaceAllUsesWith(DeclAlias); } - if (!Functions[I]->isDeclarationForLinker()) - Functions[I]->setLinkage(GlobalValue::InternalLinkage); + if (!F->isDeclarationForLinker()) + F->setLinkage(GlobalValue::InternalLinkage); } // Try to emit the jump table at the end of the text segment. @@ -960,14 +992,14 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative( AsmOS << ".balign " << EntrySize << "\n"; AsmOS << JumpTable->getName() << ":\n"; for (unsigned I = 0; I != Functions.size(); ++I) - createJumpTableEntry(AsmOS, Functions[I], I); + createJumpTableEntry(AsmOS, cast<Function>(Functions[I]->getGlobal()), I); M.appendModuleInlineAsm(AsmOS.str()); SmallVector<GlobalValue *, 16> Used; Used.reserve(Functions.size()); for (auto *F : Functions) - Used.push_back(F); + Used.push_back(F->getGlobal()); appendToUsed(M, Used); } @@ -978,13 +1010,15 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsNative( /// Note: Dynamic linking is not supported, as the WebAssembly ABI has not yet /// been finalized. void LowerTypeTestsModule::buildBitSetsFromFunctionsWASM( - ArrayRef<Metadata *> TypeIds, ArrayRef<Function *> Functions) { + ArrayRef<Metadata *> TypeIds, ArrayRef<GlobalTypeMember *> Functions) { assert(!Functions.empty()); // Build consecutive monotonic integer ranges for each call target set - DenseMap<GlobalObject *, uint64_t> GlobalLayout; + DenseMap<GlobalTypeMember *, uint64_t> GlobalLayout; + + for (GlobalTypeMember *GTM : Functions) { + Function *F = cast<Function>(GTM->getGlobal()); - for (Function *F : Functions) { // Skip functions that are not address taken, to avoid bloating the table if (!F->hasAddressTaken()) continue; @@ -996,7 +1030,7 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsWASM( F->setMetadata("wasm.index", MD); // Assign the counter value - GlobalLayout[F] = IndirectIndex++; + GlobalLayout[GTM] = IndirectIndex++; } // The indirect function table index space starts at zero, so pass a NULL @@ -1006,7 +1040,7 @@ void LowerTypeTestsModule::buildBitSetsFromFunctionsWASM( } void LowerTypeTestsModule::buildBitSetsFromDisjointSet( - ArrayRef<Metadata *> TypeIds, ArrayRef<GlobalObject *> Globals) { + ArrayRef<Metadata *> TypeIds, ArrayRef<GlobalTypeMember *> Globals) { llvm::DenseMap<Metadata *, uint64_t> TypeIdIndices; for (unsigned I = 0; I != TypeIds.size(); ++I) TypeIdIndices[TypeIds[I]] = I; @@ -1014,12 +1048,9 @@ void LowerTypeTestsModule::buildBitSetsFromDisjointSet( // For each type identifier, build a set of indices that refer to members of // the type identifier. std::vector<std::set<uint64_t>> TypeMembers(TypeIds.size()); - SmallVector<MDNode *, 2> Types; unsigned GlobalIndex = 0; - for (GlobalObject *GO : Globals) { - Types.clear(); - GO->getMetadata(LLVMContext::MD_type, Types); - for (MDNode *Type : Types) { + for (GlobalTypeMember *GTM : Globals) { + for (MDNode *Type : GTM->types()) { // Type = { offset, type identifier } unsigned TypeIdIndex = TypeIdIndices[Type->getOperand(1)]; TypeMembers[TypeIdIndex].insert(GlobalIndex); @@ -1043,32 +1074,32 @@ void LowerTypeTestsModule::buildBitSetsFromDisjointSet( GLB.addFragment(MemSet); // Build the bitsets from this disjoint set. - if (Globals.empty() || isa<GlobalVariable>(Globals[0])) { + if (Globals.empty() || isa<GlobalVariable>(Globals[0]->getGlobal())) { // Build a vector of global variables with the computed layout. - std::vector<GlobalVariable *> OrderedGVs(Globals.size()); + std::vector<GlobalTypeMember *> OrderedGVs(Globals.size()); auto OGI = OrderedGVs.begin(); for (auto &&F : GLB.Fragments) { for (auto &&Offset : F) { - auto GV = dyn_cast<GlobalVariable>(Globals[Offset]); + auto GV = dyn_cast<GlobalVariable>(Globals[Offset]->getGlobal()); if (!GV) report_fatal_error("Type identifier may not contain both global " "variables and functions"); - *OGI++ = GV; + *OGI++ = Globals[Offset]; } } buildBitSetsFromGlobalVariables(TypeIds, OrderedGVs); } else { // Build a vector of functions with the computed layout. - std::vector<Function *> OrderedFns(Globals.size()); + std::vector<GlobalTypeMember *> OrderedFns(Globals.size()); auto OFI = OrderedFns.begin(); for (auto &&F : GLB.Fragments) { for (auto &&Offset : F) { - auto Fn = dyn_cast<Function>(Globals[Offset]); + auto Fn = dyn_cast<Function>(Globals[Offset]->getGlobal()); if (!Fn) report_fatal_error("Type identifier may not contain both global " "variables and functions"); - *OFI++ = Fn; + *OFI++ = Globals[Offset]; } } @@ -1093,22 +1124,37 @@ bool LowerTypeTestsModule::lower() { // Equivalence class set containing type identifiers and the globals that // reference them. This is used to partition the set of type identifiers in // the module into disjoint sets. - typedef EquivalenceClasses<PointerUnion<GlobalObject *, Metadata *>> + typedef EquivalenceClasses<PointerUnion<GlobalTypeMember *, Metadata *>> GlobalClassesTy; GlobalClassesTy GlobalClasses; - // Verify the type metadata and build a mapping from type identifiers to their - // last observed index in the list of globals. This will be used later to - // deterministically order the list of type identifiers. - llvm::DenseMap<Metadata *, unsigned> TypeIdIndices; + // Verify the type metadata and build a few data structures to let us + // efficiently enumerate the type identifiers associated with a global: + // a list of GlobalTypeMembers (a GlobalObject stored alongside a vector + // of associated type metadata) and a mapping from type identifiers to their + // list of GlobalTypeMembers and last observed index in the list of globals. + // The indices will be used later to deterministically order the list of type + // identifiers. + BumpPtrAllocator Alloc; + struct TIInfo { + unsigned Index; + std::vector<GlobalTypeMember *> RefGlobals; + }; + llvm::DenseMap<Metadata *, TIInfo> TypeIdInfo; unsigned I = 0; SmallVector<MDNode *, 2> Types; for (GlobalObject &GO : M.global_objects()) { Types.clear(); GO.getMetadata(LLVMContext::MD_type, Types); + if (Types.empty()) + continue; + + auto *GTM = GlobalTypeMember::create(Alloc, &GO, Types); for (MDNode *Type : Types) { verifyTypeMDNode(&GO, Type); - TypeIdIndices[cast<MDNode>(Type)->getOperand(1)] = ++I; + auto &Info = TypeIdInfo[cast<MDNode>(Type)->getOperand(1)]; + Info.Index = ++I; + Info.RefGlobals.push_back(GTM); } } @@ -1136,14 +1182,9 @@ bool LowerTypeTestsModule::lower() { GlobalClassesTy::member_iterator CurSet = GlobalClasses.findLeader(GCI); // Add the referenced globals to the type identifier's equivalence class. - for (GlobalObject &GO : M.global_objects()) { - Types.clear(); - GO.getMetadata(LLVMContext::MD_type, Types); - for (MDNode *Type : Types) - if (Type->getOperand(1) == BitSet) - CurSet = GlobalClasses.unionSets( - CurSet, GlobalClasses.findLeader(GlobalClasses.insert(&GO))); - } + for (GlobalTypeMember *GTM : TypeIdInfo[BitSet].RefGlobals) + CurSet = GlobalClasses.unionSets( + CurSet, GlobalClasses.findLeader(GlobalClasses.insert(GTM))); } if (GlobalClasses.empty()) @@ -1163,7 +1204,7 @@ bool LowerTypeTestsModule::lower() { for (GlobalClassesTy::member_iterator MI = GlobalClasses.member_begin(I); MI != GlobalClasses.member_end(); ++MI) { if ((*MI).is<Metadata *>()) - MaxIndex = std::max(MaxIndex, TypeIdIndices[MI->get<Metadata *>()]); + MaxIndex = std::max(MaxIndex, TypeIdInfo[MI->get<Metadata *>()].Index); } Sets.emplace_back(I, MaxIndex); } @@ -1177,20 +1218,20 @@ bool LowerTypeTestsModule::lower() { for (const auto &S : Sets) { // Build the list of type identifiers in this disjoint set. std::vector<Metadata *> TypeIds; - std::vector<GlobalObject *> Globals; + std::vector<GlobalTypeMember *> Globals; for (GlobalClassesTy::member_iterator MI = GlobalClasses.member_begin(S.first); MI != GlobalClasses.member_end(); ++MI) { if ((*MI).is<Metadata *>()) TypeIds.push_back(MI->get<Metadata *>()); else - Globals.push_back(MI->get<GlobalObject *>()); + Globals.push_back(MI->get<GlobalTypeMember *>()); } // Order type identifiers by global index for determinism. This ordering is // stable as there is a one-to-one mapping between metadata and indices. std::sort(TypeIds.begin(), TypeIds.end(), [&](Metadata *M1, Metadata *M2) { - return TypeIdIndices[M1] < TypeIdIndices[M2]; + return TypeIdInfo[M1].Index < TypeIdInfo[M2].Index; }); // Build bitsets for this disjoint set. |

