summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/ModuleSummaryAnalysis.cpp115
-rw-r--r--llvm/lib/AsmParser/LLLexer.cpp3
-rw-r--r--llvm/lib/AsmParser/LLParser.cpp168
-rw-r--r--llvm/lib/AsmParser/LLParser.h2
-rw-r--r--llvm/lib/AsmParser/LLToken.h3
-rw-r--r--llvm/lib/Bitcode/Reader/BitcodeReader.cpp53
-rw-r--r--llvm/lib/Bitcode/Writer/BitcodeWriter.cpp61
-rw-r--r--llvm/lib/IR/AsmWriter.cpp38
-rw-r--r--llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp45
9 files changed, 463 insertions, 25 deletions
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index 87f76d43bb1..af3d97a03cd 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -406,9 +406,98 @@ static void computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
Index.addGlobalValueSummary(F, std::move(FuncSummary));
}
-static void
-computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
- DenseSet<GlobalValue::GUID> &CantBePromoted) {
+/// Find function pointers referenced within the given vtable initializer
+/// (or subset of an initializer) \p I. The starting offset of \p I within
+/// the vtable initializer is \p StartingOffset. Any discovered function
+/// pointers are added to \p VTableFuncs along with their cumulative offset
+/// within the initializer.
+static void findFuncPointers(const Constant *I, uint64_t StartingOffset,
+ const Module &M, ModuleSummaryIndex &Index,
+ VTableFuncList &VTableFuncs) {
+ // First check if this is a function pointer.
+ if (I->getType()->isPointerTy()) {
+ auto Fn = dyn_cast<Function>(I->stripPointerCasts());
+ // We can disregard __cxa_pure_virtual as a possible call target, as
+ // calls to pure virtuals are UB.
+ if (Fn && Fn->getName() != "__cxa_pure_virtual")
+ VTableFuncs.push_back(
+ std::make_pair(Index.getOrInsertValueInfo(Fn), StartingOffset));
+ return;
+ }
+
+ // Walk through the elements in the constant struct or array and recursively
+ // look for virtual function pointers.
+ const DataLayout &DL = M.getDataLayout();
+ if (auto *C = dyn_cast<ConstantStruct>(I)) {
+ StructType *STy = dyn_cast<StructType>(C->getType());
+ assert(STy);
+ const StructLayout *SL = DL.getStructLayout(C->getType());
+
+ for (StructType::element_iterator EB = STy->element_begin(), EI = EB,
+ EE = STy->element_end();
+ EI != EE; ++EI) {
+ auto Offset = SL->getElementOffset(EI - EB);
+ unsigned Op = SL->getElementContainingOffset(Offset);
+ findFuncPointers(cast<Constant>(I->getOperand(Op)),
+ StartingOffset + Offset, M, Index, VTableFuncs);
+ }
+ } else if (auto *C = dyn_cast<ConstantArray>(I)) {
+ ArrayType *ATy = C->getType();
+ Type *EltTy = ATy->getElementType();
+ uint64_t EltSize = DL.getTypeAllocSize(EltTy);
+ for (unsigned i = 0, e = ATy->getNumElements(); i != e; ++i) {
+ findFuncPointers(cast<Constant>(I->getOperand(i)),
+ StartingOffset + i * EltSize, M, Index, VTableFuncs);
+ }
+ }
+}
+
+// Identify the function pointers referenced by vtable definition \p V.
+static void computeVTableFuncs(ModuleSummaryIndex &Index,
+ const GlobalVariable &V, const Module &M,
+ VTableFuncList &VTableFuncs) {
+ if (!V.isConstant())
+ return;
+
+ findFuncPointers(V.getInitializer(), /*StartingOffset=*/0, M, Index,
+ VTableFuncs);
+
+#ifndef NDEBUG
+ // Validate that the VTableFuncs list is ordered by offset.
+ uint64_t PrevOffset = 0;
+ for (auto &P : VTableFuncs) {
+ // The findVFuncPointers traversal should have encountered the
+ // functions in offset order. We need to use ">=" since PrevOffset
+ // starts at 0.
+ assert(P.second >= PrevOffset);
+ PrevOffset = P.second;
+ }
+#endif
+}
+
+/// Record vtable definition \p V for each type metadata it references.
+static void recordTypeIdMetadataReferences(ModuleSummaryIndex &Index,
+ const GlobalVariable &V,
+ SmallVectorImpl<MDNode *> &Types) {
+ for (MDNode *Type : Types) {
+ auto TypeID = Type->getOperand(1).get();
+
+ uint64_t Offset =
+ cast<ConstantInt>(
+ cast<ConstantAsMetadata>(Type->getOperand(0))->getValue())
+ ->getZExtValue();
+
+ if (auto *TypeId = dyn_cast<MDString>(TypeID))
+ Index.getOrInsertTypeIdMetadataSummary(TypeId->getString())
+ .push_back({Offset, Index.getOrInsertValueInfo(&V)});
+ }
+}
+
+static void computeVariableSummary(ModuleSummaryIndex &Index,
+ const GlobalVariable &V,
+ DenseSet<GlobalValue::GUID> &CantBePromoted,
+ const Module &M,
+ SmallVectorImpl<MDNode *> &Types) {
SetVector<ValueInfo> RefEdges;
SmallPtrSet<const User *, 8> Visited;
bool HasBlockAddress = findRefEdges(Index, &V, RefEdges, Visited);
@@ -416,6 +505,21 @@ computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
GlobalValueSummary::GVFlags Flags(V.getLinkage(), NonRenamableLocal,
/* Live = */ false, V.isDSOLocal());
+ VTableFuncList VTableFuncs;
+ // If splitting is not enabled, then we compute the summary information
+ // necessary for index-based whole program devirtualization.
+ if (!Index.enableSplitLTOUnit()) {
+ Types.clear();
+ V.getMetadata(LLVMContext::MD_type, Types);
+ if (!Types.empty()) {
+ // Identify the function pointers referenced by this vtable definition.
+ computeVTableFuncs(Index, V, M, VTableFuncs);
+
+ // Record this vtable definition for each type metadata it references.
+ recordTypeIdMetadataReferences(Index, V, Types);
+ }
+ }
+
// Don't mark variables we won't be able to internalize as read-only.
GlobalVarSummary::GVarFlags VarFlags(
!V.hasComdat() && !V.hasAppendingLinkage() && !V.isInterposable() &&
@@ -426,6 +530,8 @@ computeVariableSummary(ModuleSummaryIndex &Index, const GlobalVariable &V,
CantBePromoted.insert(V.getGUID());
if (HasBlockAddress)
GVarSummary->setNotEligibleToImport();
+ if (!VTableFuncs.empty())
+ GVarSummary->setVTableFuncs(VTableFuncs);
Index.addGlobalValueSummary(V, std::move(GVarSummary));
}
@@ -568,10 +674,11 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
// Compute summaries for all variables defined in module, and save in the
// index.
+ SmallVector<MDNode *, 2> Types;
for (const GlobalVariable &G : M.globals()) {
if (G.isDeclaration())
continue;
- computeVariableSummary(Index, G, CantBePromoted);
+ computeVariableSummary(Index, G, CantBePromoted, M, Types);
}
// Compute summaries for all aliases defined in module, and save in the
diff --git a/llvm/lib/AsmParser/LLLexer.cpp b/llvm/lib/AsmParser/LLLexer.cpp
index eab7ec81953..f5246d5480b 100644
--- a/llvm/lib/AsmParser/LLLexer.cpp
+++ b/llvm/lib/AsmParser/LLLexer.cpp
@@ -749,6 +749,8 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(critical);
KEYWORD(relbf);
KEYWORD(variable);
+ KEYWORD(vTableFuncs);
+ KEYWORD(virtFunc);
KEYWORD(aliasee);
KEYWORD(refs);
KEYWORD(typeIdInfo);
@@ -761,6 +763,7 @@ lltok::Kind LLLexer::LexIdentifier() {
KEYWORD(offset);
KEYWORD(args);
KEYWORD(typeid);
+ KEYWORD(typeidMetadata);
KEYWORD(summary);
KEYWORD(typeTestRes);
KEYWORD(kind);
diff --git a/llvm/lib/AsmParser/LLParser.cpp b/llvm/lib/AsmParser/LLParser.cpp
index 816bb4e9401..5ef171fe965 100644
--- a/llvm/lib/AsmParser/LLParser.cpp
+++ b/llvm/lib/AsmParser/LLParser.cpp
@@ -822,6 +822,9 @@ bool LLParser::ParseSummaryEntry() {
case lltok::kw_typeid:
return ParseTypeIdEntry(SummaryID);
break;
+ case lltok::kw_typeidMetadata:
+ return ParseTypeIdMetadataEntry(SummaryID);
+ break;
default:
return Error(Lex.getLoc(), "unexpected summary kind");
}
@@ -7257,6 +7260,90 @@ bool LLParser::ParseTypeIdSummary(TypeIdSummary &TIS) {
return false;
}
+static ValueInfo EmptyVI =
+ ValueInfo(false, (GlobalValueSummaryMapTy::value_type *)-8);
+
+/// TypeIdMetadataEntry
+/// ::= 'typeidMetadata' ':' '(' 'name' ':' STRINGCONSTANT ',' TypeIdGVInfo
+/// ')'
+bool LLParser::ParseTypeIdMetadataEntry(unsigned ID) {
+ assert(Lex.getKind() == lltok::kw_typeidMetadata);
+ Lex.Lex();
+
+ std::string Name;
+ if (ParseToken(lltok::colon, "expected ':' here") ||
+ ParseToken(lltok::lparen, "expected '(' here") ||
+ ParseToken(lltok::kw_name, "expected 'name' here") ||
+ ParseToken(lltok::colon, "expected ':' here") ||
+ ParseStringConstant(Name))
+ return true;
+
+ TypeIdGVInfo &TI = Index->getOrInsertTypeIdMetadataSummary(Name);
+ if (ParseToken(lltok::comma, "expected ',' here") ||
+ ParseToken(lltok::kw_summary, "expected 'summary' here") ||
+ ParseToken(lltok::colon, "expected ':' here") ||
+ ParseToken(lltok::lparen, "expected '(' here"))
+ return true;
+
+ IdToIndexMapType IdToIndexMap;
+ // Parse each call edge
+ do {
+ uint64_t Offset;
+ if (ParseToken(lltok::lparen, "expected '(' here") ||
+ ParseToken(lltok::kw_offset, "expected 'offset' here") ||
+ ParseToken(lltok::colon, "expected ':' here") || ParseUInt64(Offset) ||
+ ParseToken(lltok::comma, "expected ',' here"))
+ return true;
+
+ LocTy Loc = Lex.getLoc();
+ unsigned GVId;
+ ValueInfo VI;
+ if (ParseGVReference(VI, GVId))
+ return true;
+
+ // Keep track of the TypeIdGVInfo array index needing a forward reference.
+ // We will save the location of the ValueInfo needing an update, but
+ // can only do so once the std::vector is finalized.
+ if (VI == EmptyVI)
+ IdToIndexMap[GVId].push_back(std::make_pair(TI.size(), Loc));
+ TI.push_back({Offset, VI});
+
+ if (ParseToken(lltok::rparen, "expected ')' in call"))
+ return true;
+ } while (EatIfPresent(lltok::comma));
+
+ // Now that the TI vector is finalized, it is safe to save the locations
+ // of any forward GV references that need updating later.
+ for (auto I : IdToIndexMap) {
+ for (auto P : I.second) {
+ assert(TI[P.first].second == EmptyVI &&
+ "Forward referenced ValueInfo expected to be empty");
+ auto FwdRef = ForwardRefValueInfos.insert(std::make_pair(
+ I.first, std::vector<std::pair<ValueInfo *, LocTy>>()));
+ FwdRef.first->second.push_back(
+ std::make_pair(&TI[P.first].second, P.second));
+ }
+ }
+
+ if (ParseToken(lltok::rparen, "expected ')' here") ||
+ ParseToken(lltok::rparen, "expected ')' here"))
+ return true;
+
+ // Check if this ID was forward referenced, and if so, update the
+ // corresponding GUIDs.
+ auto FwdRefTIDs = ForwardRefTypeIds.find(ID);
+ if (FwdRefTIDs != ForwardRefTypeIds.end()) {
+ for (auto TIDRef : FwdRefTIDs->second) {
+ assert(!*TIDRef.first &&
+ "Forward referenced type id GUID expected to be 0");
+ *TIDRef.first = GlobalValue::getGUID(Name);
+ }
+ ForwardRefTypeIds.erase(FwdRefTIDs);
+ }
+
+ return false;
+}
+
/// TypeTestResolution
/// ::= 'typeTestRes' ':' '(' 'kind' ':'
/// ( 'unsat' | 'byteArray' | 'inline' | 'single' | 'allOnes' ) ','
@@ -7764,6 +7851,7 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID,
/*Live=*/false, /*IsLocal=*/false);
GlobalVarSummary::GVarFlags GVarFlags(/*ReadOnly*/ false);
std::vector<ValueInfo> Refs;
+ VTableFuncList VTableFuncs;
if (ParseToken(lltok::colon, "expected ':' here") ||
ParseToken(lltok::lparen, "expected '(' here") ||
ParseModuleReference(ModulePath) ||
@@ -7772,10 +7860,20 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID,
ParseGVarFlags(GVarFlags))
return true;
- // Parse optional refs field
- if (EatIfPresent(lltok::comma)) {
- if (ParseOptionalRefs(Refs))
- return true;
+ // Parse optional fields
+ while (EatIfPresent(lltok::comma)) {
+ switch (Lex.getKind()) {
+ case lltok::kw_vTableFuncs:
+ if (ParseOptionalVTableFuncs(VTableFuncs))
+ return true;
+ break;
+ case lltok::kw_refs:
+ if (ParseOptionalRefs(Refs))
+ return true;
+ break;
+ default:
+ return Error(Lex.getLoc(), "expected optional variable summary field");
+ }
}
if (ParseToken(lltok::rparen, "expected ')' here"))
@@ -7785,6 +7883,7 @@ bool LLParser::ParseVariableSummary(std::string Name, GlobalValue::GUID GUID,
llvm::make_unique<GlobalVarSummary>(GVFlags, GVarFlags, std::move(Refs));
GS->setModulePath(ModulePath);
+ GS->setVTableFuncs(std::move(VTableFuncs));
AddGlobalValueToIndex(Name, GUID, (GlobalValue::LinkageTypes)GVFlags.Linkage,
ID, std::move(GS));
@@ -8002,6 +8101,67 @@ bool LLParser::ParseHotness(CalleeInfo::HotnessType &Hotness) {
return false;
}
+/// OptionalVTableFuncs
+/// := 'vTableFuncs' ':' '(' VTableFunc [',' VTableFunc]* ')'
+/// VTableFunc ::= '(' 'virtFunc' ':' GVReference ',' 'offset' ':' UInt64 ')'
+bool LLParser::ParseOptionalVTableFuncs(VTableFuncList &VTableFuncs) {
+ assert(Lex.getKind() == lltok::kw_vTableFuncs);
+ Lex.Lex();
+
+ if (ParseToken(lltok::colon, "expected ':' in vTableFuncs") |
+ ParseToken(lltok::lparen, "expected '(' in vTableFuncs"))
+ return true;
+
+ IdToIndexMapType IdToIndexMap;
+ // Parse each virtual function pair
+ do {
+ ValueInfo VI;
+ if (ParseToken(lltok::lparen, "expected '(' in vTableFunc") ||
+ ParseToken(lltok::kw_virtFunc, "expected 'callee' in vTableFunc") ||
+ ParseToken(lltok::colon, "expected ':'"))
+ return true;
+
+ LocTy Loc = Lex.getLoc();
+ unsigned GVId;
+ if (ParseGVReference(VI, GVId))
+ return true;
+
+ uint64_t Offset;
+ if (ParseToken(lltok::comma, "expected comma") ||
+ ParseToken(lltok::kw_offset, "expected offset") ||
+ ParseToken(lltok::colon, "expected ':'") || ParseUInt64(Offset))
+ return true;
+
+ // Keep track of the VTableFuncs array index needing a forward reference.
+ // We will save the location of the ValueInfo needing an update, but
+ // can only do so once the std::vector is finalized.
+ if (VI == EmptyVI)
+ IdToIndexMap[GVId].push_back(std::make_pair(VTableFuncs.size(), Loc));
+ VTableFuncs.push_back(std::make_pair(VI, Offset));
+
+ if (ParseToken(lltok::rparen, "expected ')' in vTableFunc"))
+ return true;
+ } while (EatIfPresent(lltok::comma));
+
+ // Now that the VTableFuncs vector is finalized, it is safe to save the
+ // locations of any forward GV references that need updating later.
+ for (auto I : IdToIndexMap) {
+ for (auto P : I.second) {
+ assert(VTableFuncs[P.first].first == EmptyVI &&
+ "Forward referenced ValueInfo expected to be empty");
+ auto FwdRef = ForwardRefValueInfos.insert(std::make_pair(
+ I.first, std::vector<std::pair<ValueInfo *, LocTy>>()));
+ FwdRef.first->second.push_back(
+ std::make_pair(&VTableFuncs[P.first].first, P.second));
+ }
+ }
+
+ if (ParseToken(lltok::rparen, "expected ')' in vTableFuncs"))
+ return true;
+
+ return false;
+}
+
/// OptionalRefs
/// := 'refs' ':' '(' GVReference [',' GVReference]* ')'
bool LLParser::ParseOptionalRefs(std::vector<ValueInfo> &Refs) {
diff --git a/llvm/lib/AsmParser/LLParser.h b/llvm/lib/AsmParser/LLParser.h
index 5a0fc297265..272f1ac68a8 100644
--- a/llvm/lib/AsmParser/LLParser.h
+++ b/llvm/lib/AsmParser/LLParser.h
@@ -369,9 +369,11 @@ namespace llvm {
IdToIndexMapType &IdToIndexMap, unsigned Index);
bool ParseVFuncId(FunctionSummary::VFuncId &VFuncId,
IdToIndexMapType &IdToIndexMap, unsigned Index);
+ bool ParseOptionalVTableFuncs(VTableFuncList &VTableFuncs);
bool ParseOptionalRefs(std::vector<ValueInfo> &Refs);
bool ParseTypeIdEntry(unsigned ID);
bool ParseTypeIdSummary(TypeIdSummary &TIS);
+ bool ParseTypeIdMetadataEntry(unsigned ID);
bool ParseTypeTestResolution(TypeTestResolution &TTRes);
bool ParseOptionalWpdResolutions(
std::map<uint64_t, WholeProgramDevirtResolution> &WPDResMap);
diff --git a/llvm/lib/AsmParser/LLToken.h b/llvm/lib/AsmParser/LLToken.h
index c2e2795a946..13e574241a0 100644
--- a/llvm/lib/AsmParser/LLToken.h
+++ b/llvm/lib/AsmParser/LLToken.h
@@ -379,6 +379,8 @@ enum Kind {
kw_critical,
kw_relbf,
kw_variable,
+ kw_vTableFuncs,
+ kw_virtFunc,
kw_aliasee,
kw_refs,
kw_typeIdInfo,
@@ -391,6 +393,7 @@ enum Kind {
kw_offset,
kw_args,
kw_typeid,
+ kw_typeidMetadata,
kw_summary,
kw_typeTestRes,
kw_kind,
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index fe051e7a912..78504cc64e2 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -748,6 +748,9 @@ private:
bool HasRelBF);
Error parseEntireSummary(unsigned ID);
Error parseModuleStringTable();
+ void parseTypeIdMetadataSummaryRecord(ArrayRef<uint64_t> Record);
+ void parseTypeIdGVInfo(ArrayRef<uint64_t> Record, size_t &Slot,
+ TypeIdGVInfo &TypeId);
std::pair<ValueInfo, GlobalValue::GUID>
getValueInfoFromValueId(unsigned ValueId);
@@ -5224,6 +5227,24 @@ static void parseTypeIdSummaryRecord(ArrayRef<uint64_t> Record,
parseWholeProgramDevirtResolution(Record, Strtab, Slot, TypeId);
}
+void ModuleSummaryIndexBitcodeReader::parseTypeIdGVInfo(
+ ArrayRef<uint64_t> Record, size_t &Slot, TypeIdGVInfo &TypeId) {
+ uint64_t Offset = Record[Slot++];
+ ValueInfo Callee = getValueInfoFromValueId(Record[Slot++]).first;
+ TypeId.push_back({Offset, Callee});
+}
+
+void ModuleSummaryIndexBitcodeReader::parseTypeIdMetadataSummaryRecord(
+ ArrayRef<uint64_t> Record) {
+ size_t Slot = 0;
+ TypeIdGVInfo &TypeId = TheIndex.getOrInsertTypeIdMetadataSummary(
+ {Strtab.data() + Record[Slot], static_cast<size_t>(Record[Slot + 1])});
+ Slot += 2;
+
+ while (Slot < Record.size())
+ parseTypeIdGVInfo(Record, Slot, TypeId);
+}
+
static void setImmutableRefs(std::vector<ValueInfo> &Refs, unsigned Count) {
// Read-only refs are in the end of the refs list.
for (unsigned RefNo = Refs.size() - Count; RefNo < Refs.size(); ++RefNo)
@@ -5441,6 +5462,34 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
TheIndex.addGlobalValueSummary(GUID.first, std::move(FS));
break;
}
+ // FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS: [valueid, flags, varflags,
+ // numrefs, numrefs x valueid,
+ // n x (valueid, offset)]
+ case bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS: {
+ unsigned ValueID = Record[0];
+ uint64_t RawFlags = Record[1];
+ GlobalVarSummary::GVarFlags GVF = getDecodedGVarFlags(Record[2]);
+ unsigned NumRefs = Record[3];
+ unsigned RefListStartIndex = 4;
+ unsigned VTableListStartIndex = RefListStartIndex + NumRefs;
+ auto Flags = getDecodedGVSummaryFlags(RawFlags, Version);
+ std::vector<ValueInfo> Refs = makeRefList(
+ ArrayRef<uint64_t>(Record).slice(RefListStartIndex, NumRefs));
+ VTableFuncList VTableFuncs;
+ for (unsigned I = VTableListStartIndex, E = Record.size(); I != E; ++I) {
+ ValueInfo Callee = getValueInfoFromValueId(Record[I]).first;
+ uint64_t Offset = Record[++I];
+ VTableFuncs.push_back({Callee, Offset});
+ }
+ auto VS =
+ llvm::make_unique<GlobalVarSummary>(Flags, GVF, std::move(Refs));
+ VS->setModulePath(getThisModule()->first());
+ VS->setVTableFuncs(VTableFuncs);
+ auto GUID = getValueInfoFromValueId(ValueID);
+ VS->setOriginalName(GUID.second);
+ TheIndex.addGlobalValueSummary(GUID.first, std::move(VS));
+ break;
+ }
// FS_COMBINED: [valueid, modid, flags, instcount, fflags, numrefs,
// numrefs x valueid, n x (valueid)]
// FS_COMBINED_PROFILE: [valueid, modid, flags, instcount, fflags, numrefs,
@@ -5610,6 +5659,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(unsigned ID) {
case bitc::FS_TYPE_ID:
parseTypeIdSummaryRecord(Record, Strtab, TheIndex);
break;
+
+ case bitc::FS_TYPE_ID_METADATA:
+ parseTypeIdMetadataSummaryRecord(Record);
+ break;
}
}
llvm_unreachable("Exit infinite loop");
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index ba4f932e2e6..56656c74906 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -215,7 +215,8 @@ private:
const Function &F);
void writeModuleLevelReferences(const GlobalVariable &V,
SmallVector<uint64_t, 64> &NameVals,
- unsigned FSModRefsAbbrev);
+ unsigned FSModRefsAbbrev,
+ unsigned FSModVTableRefsAbbrev);
void assignValueId(GlobalValue::GUID ValGUID) {
GUIDToValueIdMap[ValGUID] = ++GlobalValueId;
@@ -3528,6 +3529,18 @@ static void writeTypeIdSummaryRecord(SmallVector<uint64_t, 64> &NameVals,
W.second);
}
+static void writeTypeIdMetadataSummaryRecord(
+ SmallVector<uint64_t, 64> &NameVals, StringTableBuilder &StrtabBuilder,
+ const std::string &Id, const TypeIdGVInfo &Summary, ValueEnumerator &VE) {
+ NameVals.push_back(StrtabBuilder.add(Id));
+ NameVals.push_back(Id.size());
+
+ for (auto &P : Summary) {
+ NameVals.push_back(P.first);
+ NameVals.push_back(VE.getValueID(P.second.getValue()));
+ }
+}
+
// Helper to emit a single function summary record.
void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
@@ -3572,7 +3585,7 @@ void ModuleBitcodeWriterBase::writePerModuleFunctionSummaryRecord(
// and emit them in a summary record.
void ModuleBitcodeWriterBase::writeModuleLevelReferences(
const GlobalVariable &V, SmallVector<uint64_t, 64> &NameVals,
- unsigned FSModRefsAbbrev) {
+ unsigned FSModRefsAbbrev, unsigned FSModVTableRefsAbbrev) {
auto VI = Index->getValueInfo(V.getGUID());
if (!VI || VI.getSummaryList().empty()) {
// Only declarations should not have a summary (a declaration might however
@@ -3586,6 +3599,10 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences(
NameVals.push_back(getEncodedGVSummaryFlags(VS->flags()));
NameVals.push_back(getEncodedGVarFlags(VS->varflags()));
+ auto VTableFuncs = VS->vTableFuncs();
+ if (!VTableFuncs.empty())
+ NameVals.push_back(VS->refs().size());
+
unsigned SizeBeforeRefs = NameVals.size();
for (auto &RI : VS->refs())
NameVals.push_back(VE.getValueID(RI.getValue()));
@@ -3593,8 +3610,20 @@ void ModuleBitcodeWriterBase::writeModuleLevelReferences(
// been initialized from a DenseSet.
llvm::sort(NameVals.begin() + SizeBeforeRefs, NameVals.end());
- Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals,
- FSModRefsAbbrev);
+ if (!VTableFuncs.empty()) {
+ // VTableFuncs pairs should already be sorted by offset.
+ for (auto &P : VTableFuncs) {
+ NameVals.push_back(VE.getValueID(P.first.getValue()));
+ NameVals.push_back(P.second);
+ }
+ }
+
+ if (VTableFuncs.empty())
+ Stream.EmitRecord(bitc::FS_PERMODULE_GLOBALVAR_INIT_REFS, NameVals,
+ FSModRefsAbbrev);
+ else
+ Stream.EmitRecord(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS, NameVals,
+ FSModVTableRefsAbbrev);
NameVals.clear();
}
@@ -3675,6 +3704,17 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
unsigned FSModRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
+ // Abbrev for FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS.
+ Abbv = std::make_shared<BitCodeAbbrev>();
+ Abbv->Add(BitCodeAbbrevOp(bitc::FS_PERMODULE_VTABLE_GLOBALVAR_INIT_REFS));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8)); // valueid
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 6)); // flags
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 4)); // numrefs
+ // numrefs x valueid, n x (valueid , offset)
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::Array));
+ Abbv->Add(BitCodeAbbrevOp(BitCodeAbbrevOp::VBR, 8));
+ unsigned FSModVTableRefsAbbrev = Stream.EmitAbbrev(std::move(Abbv));
+
// Abbrev for FS_ALIAS.
Abbv = std::make_shared<BitCodeAbbrev>();
Abbv->Add(BitCodeAbbrevOp(bitc::FS_ALIAS));
@@ -3707,7 +3747,8 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
// Capture references from GlobalVariable initializers, which are outside
// of a function scope.
for (const GlobalVariable &G : M.globals())
- writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev);
+ writeModuleLevelReferences(G, NameVals, FSModRefsAbbrev,
+ FSModVTableRefsAbbrev);
for (const GlobalAlias &A : M.aliases()) {
auto *Aliasee = A.getBaseObject();
@@ -3725,6 +3766,16 @@ void ModuleBitcodeWriterBase::writePerModuleGlobalValueSummary() {
NameVals.clear();
}
+ if (!Index->typeIdMetadataMap().empty()) {
+ SmallVector<uint64_t, 64> NameVals;
+ for (auto &S : Index->typeIdMetadataMap()) {
+ writeTypeIdMetadataSummaryRecord(NameVals, StrtabBuilder, S.first,
+ S.second, VE);
+ Stream.EmitRecord(bitc::FS_TYPE_ID_METADATA, NameVals);
+ NameVals.clear();
+ }
+ }
+
Stream.ExitBlock();
}
diff --git a/llvm/lib/IR/AsmWriter.cpp b/llvm/lib/IR/AsmWriter.cpp
index a5dc623e1a3..e431deb86f4 100644
--- a/llvm/lib/IR/AsmWriter.cpp
+++ b/llvm/lib/IR/AsmWriter.cpp
@@ -1038,6 +1038,9 @@ void SlotTracker::processIndex() {
TidIter != TheIndex->typeIds().end(); TidIter++)
CreateTypeIdSlot(TidIter->second.first);
+ for (auto &TId : TheIndex->typeIdMetadataMap())
+ CreateGUIDSlot(GlobalValue::getGUID(TId.first));
+
ST_DEBUG("end processIndex!\n");
}
@@ -2393,6 +2396,7 @@ public:
void printGlobalVarSummary(const GlobalVarSummary *GS);
void printFunctionSummary(const FunctionSummary *FS);
void printTypeIdSummary(const TypeIdSummary &TIS);
+ void printTypeIdMetadataSummary(const TypeIdGVInfo &TI);
void printTypeTestResolution(const TypeTestResolution &TTRes);
void printArgs(const std::vector<uint64_t> &Args);
void printWPDRes(const WholeProgramDevirtResolution &WPDRes);
@@ -2695,6 +2699,15 @@ void AssemblyWriter::printModuleSummaryIndex() {
printTypeIdSummary(TidIter->second.second);
Out << ") ; guid = " << TidIter->first << "\n";
}
+
+ // Print the TypeIdMetadataMap entries.
+ for (auto &TId : TheIndex->typeIdMetadataMap()) {
+ auto GUID = GlobalValue::getGUID(TId.first);
+ Out << "^" << Machine.getGUIDSlot(GUID) << " = typeidMetadata: (name: \""
+ << TId.first << "\"";
+ printTypeIdMetadataSummary(TId.second);
+ Out << ") ; guid = " << GUID << "\n";
+ }
}
static const char *
@@ -2777,6 +2790,18 @@ void AssemblyWriter::printTypeIdSummary(const TypeIdSummary &TIS) {
Out << ")";
}
+void AssemblyWriter::printTypeIdMetadataSummary(const TypeIdGVInfo &TI) {
+ Out << ", summary: (";
+ FieldSeparator FS;
+ for (auto &P : TI) {
+ Out << FS;
+ Out << "(offset: " << P.first << ", ";
+ Out << "^" << Machine.getGUIDSlot(P.second.getGUID());
+ Out << ")";
+ }
+ Out << ")";
+}
+
void AssemblyWriter::printArgs(const std::vector<uint64_t> &Args) {
Out << "args: (";
FieldSeparator FS;
@@ -2846,6 +2871,19 @@ void AssemblyWriter::printAliasSummary(const AliasSummary *AS) {
void AssemblyWriter::printGlobalVarSummary(const GlobalVarSummary *GS) {
Out << ", varFlags: (readonly: " << GS->VarFlags.ReadOnly << ")";
+
+ auto VTableFuncs = GS->vTableFuncs();
+ if (!VTableFuncs.empty()) {
+ Out << ", vTableFuncs: (";
+ FieldSeparator FS;
+ for (auto &P : VTableFuncs) {
+ Out << FS;
+ Out << "(virtFunc: ^" << Machine.getGUIDSlot(P.first.getGUID())
+ << ", offset: " << P.second;
+ Out << ")";
+ }
+ Out << ")";
+ }
}
static std::string getLinkageName(GlobalValue::LinkageTypes LT) {
diff --git a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
index 510ecb516dc..88b45e03492 100644
--- a/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
+++ b/llvm/lib/Transforms/IPO/ThinLTOBitcodeWriter.cpp
@@ -418,34 +418,55 @@ void splitAndWriteThinLTOBitcode(
}
}
-// Returns whether this module needs to be split because splitting is
-// enabled and it uses type metadata.
-bool requiresSplit(Module &M) {
- // First check if the LTO Unit splitting has been enabled.
+// Check if the LTO Unit splitting has been enabled.
+bool enableSplitLTOUnit(Module &M) {
bool EnableSplitLTOUnit = false;
if (auto *MD = mdconst::extract_or_null<ConstantInt>(
M.getModuleFlag("EnableSplitLTOUnit")))
EnableSplitLTOUnit = MD->getZExtValue();
- if (!EnableSplitLTOUnit)
- return false;
+ return EnableSplitLTOUnit;
+}
- // Module only needs to be split if it contains type metadata.
+// Returns whether this module needs to be split because it uses type metadata.
+bool hasTypeMetadata(Module &M) {
for (auto &GO : M.global_objects()) {
if (GO.hasMetadata(LLVMContext::MD_type))
return true;
}
-
return false;
}
void writeThinLTOBitcode(raw_ostream &OS, raw_ostream *ThinLinkOS,
function_ref<AAResults &(Function &)> AARGetter,
Module &M, const ModuleSummaryIndex *Index) {
- // Split module if splitting is enabled and it contains any type metadata.
- if (requiresSplit(M))
- return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
+ std::unique_ptr<ModuleSummaryIndex> NewIndex = nullptr;
+ // See if this module has any type metadata. If so, we try to split it
+ // or at least promote type ids to enable WPD.
+ if (hasTypeMetadata(M)) {
+ if (enableSplitLTOUnit(M))
+ return splitAndWriteThinLTOBitcode(OS, ThinLinkOS, AARGetter, M);
+ else {
+ // Promote type ids as needed for index-based WPD.
+ std::string ModuleId = getUniqueModuleId(&M);
+ if (!ModuleId.empty()) {
+ promoteTypeIds(M, ModuleId);
+ // Need to rebuild the index so that it contains type metadata
+ // for the newly promoted type ids.
+ // FIXME: Probably should not bother building the index at all
+ // in the caller of writeThinLTOBitcode (which does so via the
+ // ModuleSummaryIndexAnalysis pass), since we have to rebuild it
+ // anyway whenever there is type metadata (here or in
+ // splitAndWriteThinLTOBitcode). Just always build it once via the
+ // buildModuleSummaryIndex when Module(s) are ready.
+ ProfileSummaryInfo PSI(M);
+ NewIndex = llvm::make_unique<ModuleSummaryIndex>(
+ buildModuleSummaryIndex(M, nullptr, &PSI));
+ Index = NewIndex.get();
+ }
+ }
+ }
- // Otherwise we can just write it out as a regular module.
+ // Write it out as an unsplit ThinLTO module.
// Save the module hash produced for the full bitcode, which will
// be used in the backends, and use that in the minimized bitcode
OpenPOWER on IntegriCloud