summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/ModuleSummaryAnalysis.cpp125
-rw-r--r--llvm/lib/Bitcode/Reader/BitcodeReader.cpp44
-rw-r--r--llvm/lib/Bitcode/Writer/BitcodeWriter.cpp49
3 files changed, 191 insertions, 27 deletions
diff --git a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
index f5ba637e58e..70b55674c07 100644
--- a/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
+++ b/llvm/lib/Analysis/ModuleSummaryAnalysis.cpp
@@ -84,6 +84,92 @@ static bool isNonRenamableLocal(const GlobalValue &GV) {
return GV.hasSection() && GV.hasLocalLinkage();
}
+/// Determine whether this call has all constant integer arguments (excluding
+/// "this") and summarize it to VCalls or ConstVCalls as appropriate.
+static void addVCallToSet(DevirtCallSite Call, GlobalValue::GUID Guid,
+ SetVector<FunctionSummary::VFuncId> &VCalls,
+ SetVector<FunctionSummary::ConstVCall> &ConstVCalls) {
+ std::vector<uint64_t> Args;
+ // Start from the second argument to skip the "this" pointer.
+ for (auto &Arg : make_range(Call.CS.arg_begin() + 1, Call.CS.arg_end())) {
+ auto *CI = dyn_cast<ConstantInt>(Arg);
+ if (!CI || CI->getBitWidth() > 64) {
+ VCalls.insert({Guid, Call.Offset});
+ return;
+ }
+ Args.push_back(CI->getZExtValue());
+ }
+ ConstVCalls.insert({{Guid, Call.Offset}, std::move(Args)});
+}
+
+/// If this intrinsic call requires that we add information to the function
+/// summary, do so via the non-constant reference arguments.
+static void addIntrinsicToSummary(
+ const CallInst *CI, SetVector<GlobalValue::GUID> &TypeTests,
+ SetVector<FunctionSummary::VFuncId> &TypeTestAssumeVCalls,
+ SetVector<FunctionSummary::VFuncId> &TypeCheckedLoadVCalls,
+ SetVector<FunctionSummary::ConstVCall> &TypeTestAssumeConstVCalls,
+ SetVector<FunctionSummary::ConstVCall> &TypeCheckedLoadConstVCalls) {
+ switch (CI->getCalledFunction()->getIntrinsicID()) {
+ case Intrinsic::type_test: {
+ auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
+ auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
+ if (!TypeId)
+ break;
+ GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
+
+ // Produce a summary from type.test intrinsics. We only summarize type.test
+ // intrinsics that are used other than by an llvm.assume intrinsic.
+ // Intrinsics that are assumed are relevant only to the devirtualization
+ // pass, not the type test lowering pass.
+ bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
+ auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
+ if (!AssumeCI)
+ return true;
+ Function *F = AssumeCI->getCalledFunction();
+ return !F || F->getIntrinsicID() != Intrinsic::assume;
+ });
+ if (HasNonAssumeUses)
+ TypeTests.insert(Guid);
+
+ SmallVector<DevirtCallSite, 4> DevirtCalls;
+ SmallVector<CallInst *, 4> Assumes;
+ findDevirtualizableCallsForTypeTest(DevirtCalls, Assumes, CI);
+ for (auto &Call : DevirtCalls)
+ addVCallToSet(Call, Guid, TypeTestAssumeVCalls,
+ TypeTestAssumeConstVCalls);
+
+ break;
+ }
+
+ case Intrinsic::type_checked_load: {
+ auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(2));
+ auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata());
+ if (!TypeId)
+ break;
+ GlobalValue::GUID Guid = GlobalValue::getGUID(TypeId->getString());
+
+ SmallVector<DevirtCallSite, 4> DevirtCalls;
+ SmallVector<Instruction *, 4> LoadedPtrs;
+ SmallVector<Instruction *, 4> Preds;
+ bool HasNonCallUses = false;
+ findDevirtualizableCallsForTypeCheckedLoad(DevirtCalls, LoadedPtrs, Preds,
+ HasNonCallUses, CI);
+ // Any non-call uses of the result of llvm.type.checked.load will
+ // prevent us from optimizing away the llvm.type.test.
+ if (HasNonCallUses)
+ TypeTests.insert(Guid);
+ for (auto &Call : DevirtCalls)
+ addVCallToSet(Call, Guid, TypeCheckedLoadVCalls,
+ TypeCheckedLoadConstVCalls);
+
+ break;
+ }
+ default:
+ break;
+ }
+}
+
static void
computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
const Function &F, BlockFrequencyInfo *BFI,
@@ -99,6 +185,10 @@ computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
MapVector<ValueInfo, CalleeInfo> CallGraphEdges;
SetVector<ValueInfo> RefEdges;
SetVector<GlobalValue::GUID> TypeTests;
+ SetVector<FunctionSummary::VFuncId> TypeTestAssumeVCalls,
+ TypeCheckedLoadVCalls;
+ SetVector<FunctionSummary::ConstVCall> TypeTestAssumeConstVCalls,
+ TypeCheckedLoadConstVCalls;
ICallPromotionAnalysis ICallAnalysis;
bool HasInlineAsmMaybeReferencingInternal = false;
@@ -133,25 +223,11 @@ computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
// Check if this is a direct call to a known function or a known
// intrinsic, or an indirect call with profile data.
if (CalledFunction) {
- if (CalledFunction->isIntrinsic()) {
- if (CalledFunction->getIntrinsicID() != Intrinsic::type_test)
- continue;
- // Produce a summary from type.test intrinsics. We only summarize
- // type.test intrinsics that are used other than by an llvm.assume
- // intrinsic. Intrinsics that are assumed are relevant only to the
- // devirtualization pass, not the type test lowering pass.
- bool HasNonAssumeUses = llvm::any_of(CI->uses(), [](const Use &CIU) {
- auto *AssumeCI = dyn_cast<CallInst>(CIU.getUser());
- if (!AssumeCI)
- return true;
- Function *F = AssumeCI->getCalledFunction();
- return !F || F->getIntrinsicID() != Intrinsic::assume;
- });
- if (HasNonAssumeUses) {
- auto *TypeMDVal = cast<MetadataAsValue>(CI->getArgOperand(1));
- if (auto *TypeId = dyn_cast<MDString>(TypeMDVal->getMetadata()))
- TypeTests.insert(GlobalValue::getGUID(TypeId->getString()));
- }
+ if (CI && CalledFunction->isIntrinsic()) {
+ addIntrinsicToSummary(
+ CI, TypeTests, TypeTestAssumeVCalls, TypeCheckedLoadVCalls,
+ TypeTestAssumeConstVCalls, TypeCheckedLoadConstVCalls);
+ continue;
}
// We should have named any anonymous globals
assert(CalledFunction->hasName());
@@ -193,7 +269,10 @@ computeFunctionSummary(ModuleSummaryIndex &Index, const Module &M,
/* LiveRoot = */ false);
auto FuncSummary = llvm::make_unique<FunctionSummary>(
Flags, NumInsts, RefEdges.takeVector(), CallGraphEdges.takeVector(),
- TypeTests.takeVector());
+ TypeTests.takeVector(), TypeTestAssumeVCalls.takeVector(),
+ TypeCheckedLoadVCalls.takeVector(),
+ TypeTestAssumeConstVCalls.takeVector(),
+ TypeCheckedLoadConstVCalls.takeVector());
if (NonRenamableLocal)
CantBePromoted.insert(F.getGUID());
Index.addGlobalValueSummary(F.getName(), std::move(FuncSummary));
@@ -347,7 +426,11 @@ ModuleSummaryIndex llvm::buildModuleSummaryIndex(
llvm::make_unique<FunctionSummary>(
GVFlags, 0, ArrayRef<ValueInfo>{},
ArrayRef<FunctionSummary::EdgeTy>{},
- ArrayRef<GlobalValue::GUID>{});
+ ArrayRef<GlobalValue::GUID>{},
+ ArrayRef<FunctionSummary::VFuncId>{},
+ ArrayRef<FunctionSummary::VFuncId>{},
+ ArrayRef<FunctionSummary::ConstVCall>{},
+ ArrayRef<FunctionSummary::ConstVCall>{});
Index.addGlobalValueSummary(Name, std::move(Summary));
} else {
std::unique_ptr<GlobalVarSummary> Summary =
diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
index a46e49ccde8..fe73efef5a5 100644
--- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
+++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp
@@ -4848,6 +4848,10 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(
GlobalValueSummary *LastSeenSummary = nullptr;
bool Combined = false;
std::vector<GlobalValue::GUID> PendingTypeTests;
+ std::vector<FunctionSummary::VFuncId> PendingTypeTestAssumeVCalls,
+ PendingTypeCheckedLoadVCalls;
+ std::vector<FunctionSummary::ConstVCall> PendingTypeTestAssumeConstVCalls,
+ PendingTypeCheckedLoadConstVCalls;
while (true) {
BitstreamEntry Entry = Stream.advanceSkippingSubblocks();
@@ -4914,8 +4918,15 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(
IsOldProfileFormat, HasProfile);
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, std::move(Refs), std::move(Calls),
- std::move(PendingTypeTests));
+ std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls),
+ std::move(PendingTypeCheckedLoadVCalls),
+ std::move(PendingTypeTestAssumeConstVCalls),
+ std::move(PendingTypeCheckedLoadConstVCalls));
PendingTypeTests.clear();
+ PendingTypeTestAssumeVCalls.clear();
+ PendingTypeCheckedLoadVCalls.clear();
+ PendingTypeTestAssumeConstVCalls.clear();
+ PendingTypeCheckedLoadConstVCalls.clear();
auto GUID = getGUIDFromValueId(ValueID);
FS->setModulePath(TheIndex.addModulePath(ModulePath, 0)->first());
FS->setOriginalName(GUID.second);
@@ -4989,8 +5000,15 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(
GlobalValue::GUID GUID = getGUIDFromValueId(ValueID).first;
auto FS = llvm::make_unique<FunctionSummary>(
Flags, InstCount, std::move(Refs), std::move(Edges),
- std::move(PendingTypeTests));
+ std::move(PendingTypeTests), std::move(PendingTypeTestAssumeVCalls),
+ std::move(PendingTypeCheckedLoadVCalls),
+ std::move(PendingTypeTestAssumeConstVCalls),
+ std::move(PendingTypeCheckedLoadConstVCalls));
PendingTypeTests.clear();
+ PendingTypeTestAssumeVCalls.clear();
+ PendingTypeCheckedLoadVCalls.clear();
+ PendingTypeTestAssumeConstVCalls.clear();
+ PendingTypeCheckedLoadConstVCalls.clear();
LastSeenSummary = FS.get();
FS->setModulePath(ModuleIdMap[ModuleId]);
TheIndex.addGlobalValueSummary(GUID, std::move(FS));
@@ -5054,6 +5072,28 @@ Error ModuleSummaryIndexBitcodeReader::parseEntireSummary(
Record.end());
break;
}
+ case bitc::FS_TYPE_TEST_ASSUME_VCALLS: {
+ assert(PendingTypeTestAssumeVCalls.empty());
+ for (unsigned I = 0; I != Record.size(); I += 2)
+ PendingTypeTestAssumeVCalls.push_back({Record[I], Record[I+1]});
+ break;
+ }
+ case bitc::FS_TYPE_CHECKED_LOAD_VCALLS: {
+ assert(PendingTypeCheckedLoadVCalls.empty());
+ for (unsigned I = 0; I != Record.size(); I += 2)
+ PendingTypeCheckedLoadVCalls.push_back({Record[I], Record[I+1]});
+ break;
+ }
+ case bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL: {
+ PendingTypeTestAssumeConstVCalls.push_back(
+ {{Record[0], Record[1]}, {Record.begin() + 2, Record.end()}});
+ break;
+ }
+ case bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL: {
+ PendingTypeCheckedLoadConstVCalls.push_back(
+ {{Record[0], Record[1]}, {Record.begin() + 2, Record.end()}});
+ break;
+ }
}
}
llvm_unreachable("Exit infinite loop");
diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
index 4eac89c37a5..1281fca5f69 100644
--- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
+++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp
@@ -3368,6 +3368,49 @@ void IndexBitcodeWriter::writeModStrings() {
Stream.ExitBlock();
}
+/// Write the function type metadata related records that need to appear before
+/// a function summary entry (whether per-module or combined).
+static void writeFunctionTypeMetadataRecords(BitstreamWriter &Stream,
+ FunctionSummary *FS) {
+ if (!FS->type_tests().empty())
+ Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests());
+
+ SmallVector<uint64_t, 64> Record;
+
+ auto WriteVFuncIdVec = [&](uint64_t Ty,
+ ArrayRef<FunctionSummary::VFuncId> VFs) {
+ if (VFs.empty())
+ return;
+ Record.clear();
+ for (auto &VF : VFs) {
+ Record.push_back(VF.GUID);
+ Record.push_back(VF.Offset);
+ }
+ Stream.EmitRecord(Ty, Record);
+ };
+
+ WriteVFuncIdVec(bitc::FS_TYPE_TEST_ASSUME_VCALLS,
+ FS->type_test_assume_vcalls());
+ WriteVFuncIdVec(bitc::FS_TYPE_CHECKED_LOAD_VCALLS,
+ FS->type_checked_load_vcalls());
+
+ auto WriteConstVCallVec = [&](uint64_t Ty,
+ ArrayRef<FunctionSummary::ConstVCall> VCs) {
+ for (auto &VC : VCs) {
+ Record.clear();
+ Record.push_back(VC.VFunc.GUID);
+ Record.push_back(VC.VFunc.Offset);
+ Record.insert(Record.end(), VC.Args.begin(), VC.Args.end());
+ Stream.EmitRecord(Ty, Record);
+ }
+ };
+
+ WriteConstVCallVec(bitc::FS_TYPE_TEST_ASSUME_CONST_VCALL,
+ FS->type_test_assume_const_vcalls());
+ WriteConstVCallVec(bitc::FS_TYPE_CHECKED_LOAD_CONST_VCALL,
+ FS->type_checked_load_const_vcalls());
+}
+
// Helper to emit a single function summary record.
void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord(
SmallVector<uint64_t, 64> &NameVals, GlobalValueSummary *Summary,
@@ -3376,8 +3419,7 @@ void ModuleBitcodeWriter::writePerModuleFunctionSummaryRecord(
NameVals.push_back(ValueID);
FunctionSummary *FS = cast<FunctionSummary>(Summary);
- if (!FS->type_tests().empty())
- Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests());
+ writeFunctionTypeMetadataRecords(Stream, FS);
NameVals.push_back(getEncodedGVSummaryFlags(FS->flags()));
NameVals.push_back(FS->instCount());
@@ -3637,8 +3679,7 @@ void IndexBitcodeWriter::writeCombinedGlobalValueSummary() {
}
auto *FS = cast<FunctionSummary>(S);
- if (!FS->type_tests().empty())
- Stream.EmitRecord(bitc::FS_TYPE_TESTS, FS->type_tests());
+ writeFunctionTypeMetadataRecords(Stream, FS);
NameVals.push_back(ValueId);
NameVals.push_back(Index.getModuleId(FS->modulePath()));
OpenPOWER on IntegriCloud