summaryrefslogtreecommitdiffstats
path: root/llvm
diff options
context:
space:
mode:
Diffstat (limited to 'llvm')
-rw-r--r--llvm/include/llvm/ProfileData/SampleProf.h12
-rw-r--r--llvm/include/llvm/Transforms/SampleProfile.h4
-rw-r--r--llvm/lib/Passes/PassBuilder.cpp3
-rw-r--r--llvm/lib/Transforms/IPO/SampleProfile.cpp38
-rw-r--r--llvm/test/Transforms/SampleProfile/Inputs/import.prof2
-rw-r--r--llvm/test/Transforms/SampleProfile/import.ll8
6 files changed, 47 insertions, 20 deletions
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 7fc258831be..3e17df14807 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -331,7 +331,8 @@ public:
/// Recursively traverses all children, if the corresponding function is
/// not defined in module \p M, and its total sample is no less than
- /// \p Threshold, add its corresponding GUID to \p S.
+ /// \p Threshold, add its corresponding GUID to \p S. Also traverse the
+ /// BodySamples to add hot CallTarget's GUID to \p S.
void findImportedFunctions(DenseSet<GlobalValue::GUID> &S, const Module *M,
uint64_t Threshold) const {
if (TotalSamples <= Threshold)
@@ -339,6 +340,15 @@ public:
Function *F = M->getFunction(Name);
if (!F || !F->getSubprogram())
S.insert(Function::getGUID(Name));
+ // Import hot CallTargets, which may not be available in IR because full
+ // profile annotation cannot be done until backend compilation in ThinLTO.
+ for (const auto &BS : BodySamples)
+ for (const auto &TS : BS.second.getCallTargets())
+ if (TS.getValue() > Threshold) {
+ Function *Callee = M->getFunction(TS.getKey());
+ if (!Callee || !Callee->getSubprogram())
+ S.insert(Function::getGUID(TS.getKey()));
+ }
for (auto CS : CallsiteSamples)
for (const auto &NameFS : CS.second)
NameFS.second.findImportedFunctions(S, M, Threshold);
diff --git a/llvm/include/llvm/Transforms/SampleProfile.h b/llvm/include/llvm/Transforms/SampleProfile.h
index c984fe74ba9..8f970783565 100644
--- a/llvm/include/llvm/Transforms/SampleProfile.h
+++ b/llvm/include/llvm/Transforms/SampleProfile.h
@@ -21,10 +21,12 @@ namespace llvm {
class SampleProfileLoaderPass : public PassInfoMixin<SampleProfileLoaderPass> {
public:
PreservedAnalyses run(Module &M, ModuleAnalysisManager &AM);
- SampleProfileLoaderPass(std::string File = "") : ProfileFileName(File) {}
+ SampleProfileLoaderPass(std::string File = "", bool IsThinLTOPreLink = false)
+ : ProfileFileName(File), IsThinLTOPreLink(IsThinLTOPreLink) {}
private:
std::string ProfileFileName;
+ bool IsThinLTOPreLink;
};
} // End llvm namespace
diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp
index c277b5b14e7..c9cc0c5ae38 100644
--- a/llvm/lib/Passes/PassBuilder.cpp
+++ b/llvm/lib/Passes/PassBuilder.cpp
@@ -555,7 +555,8 @@ PassBuilder::buildModuleSimplificationPipeline(OptimizationLevel Level,
if (PGOOpt && !PGOOpt->SampleProfileFile.empty()) {
// Annotate sample profile right after early FPM to ensure freshness of
// the debug info.
- MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile));
+ MPM.addPass(SampleProfileLoaderPass(PGOOpt->SampleProfileFile,
+ Phase == ThinLTOPhase::PreLink));
// Do not invoke ICP in the ThinLTOPrelink phase as it makes it hard
// for the profile annotation to be accurate in the ThinLTO backend.
if (Phase != ThinLTOPhase::PreLink)
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 93e697c9a7e..7dce904ff15 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -149,13 +149,14 @@ private:
class SampleProfileLoader {
public:
SampleProfileLoader(
- StringRef Name,
+ StringRef Name, bool IsThinLTOPreLink,
std::function<AssumptionCache &(Function &)> GetAssumptionCache,
std::function<TargetTransformInfo &(Function &)> GetTargetTransformInfo)
: DT(nullptr), PDT(nullptr), LI(nullptr), GetAC(GetAssumptionCache),
GetTTI(GetTargetTransformInfo), Reader(), Samples(nullptr),
- Filename(Name), ProfileIsValid(false), TotalCollectedSamples(0),
- ORE(nullptr) {}
+ Filename(Name), ProfileIsValid(false),
+ IsThinLTOPreLink(IsThinLTOPreLink),
+ TotalCollectedSamples(0), ORE(nullptr) {}
bool doInitialization(Module &M);
bool runOnModule(Module &M, ModuleAnalysisManager *AM);
@@ -252,6 +253,12 @@ protected:
/// \brief Flag indicating whether the profile input loaded successfully.
bool ProfileIsValid;
+ /// \brief Flag indicating if the pass is invoked in ThinLTO compile phase.
+ ///
+ /// In this phase, in annotation, we should not promote indirect calls.
+ /// Instead, we will mark GUIDs that needs to be annotated to the function.
+ bool IsThinLTOPreLink;
+
/// \brief Total number of samples collected in this profile.
///
/// This is the sum of all the samples collected in all the functions executed
@@ -267,8 +274,9 @@ public:
// Class identification, replacement for typeinfo
static char ID;
- SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile)
- : ModulePass(ID), SampleLoader(Name,
+ SampleProfileLoaderLegacyPass(StringRef Name = SampleProfileFile,
+ bool IsThinLTOPreLink = false)
+ : ModulePass(ID), SampleLoader(Name, IsThinLTOPreLink,
[&](Function &F) -> AssumptionCache & {
return ACT->getAssumptionCache(F);
},
@@ -755,6 +763,12 @@ bool SampleProfileLoader::inlineHotFunctions(
if (PromotedInsns.count(I))
continue;
for (const auto *FS : findIndirectCallFunctionSamples(*I)) {
+ if (IsThinLTOPreLink) {
+ FS->findImportedFunctions(ImportGUIDs, F.getParent(),
+ Samples->getTotalSamples() *
+ SampleProfileHotThreshold / 100);
+ continue;
+ }
auto CalleeFunctionName = FS->getName();
// If it is a recursive call, we do not inline it as it could bloat
// the code exponentially. There is way to better handle this, e.g.
@@ -783,16 +797,16 @@ bool SampleProfileLoader::inlineHotFunctions(
inlineCallInstruction(DI))
LocalChanged = true;
} else {
- FS->findImportedFunctions(ImportGUIDs, F.getParent(),
- Samples->getTotalSamples() *
- SampleProfileHotThreshold / 100);
+ DEBUG(dbgs()
+ << "\nFailed to promote indirect call to "
+ << CalleeFunctionName << " because " << Reason << "\n");
}
}
} else if (CalledFunction && CalledFunction->getSubprogram() &&
!CalledFunction->isDeclaration()) {
if (inlineCallInstruction(I))
LocalChanged = true;
- } else {
+ } else if (IsThinLTOPreLink) {
findCalleeFunctionSamples(*I)->findImportedFunctions(
ImportGUIDs, F.getParent(),
Samples->getTotalSamples() * SampleProfileHotThreshold / 100);
@@ -1558,9 +1572,9 @@ PreservedAnalyses SampleProfileLoaderPass::run(Module &M,
return FAM.getResult<TargetIRAnalysis>(F);
};
- SampleProfileLoader SampleLoader(ProfileFileName.empty() ? SampleProfileFile
- : ProfileFileName,
- GetAssumptionCache, GetTTI);
+ SampleProfileLoader SampleLoader(
+ ProfileFileName.empty() ? SampleProfileFile : ProfileFileName,
+ IsThinLTOPreLink, GetAssumptionCache, GetTTI);
SampleLoader.doInitialization(M);
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/import.prof b/llvm/test/Transforms/SampleProfile/Inputs/import.prof
index aae072ac191..e09ee6bf060 100644
--- a/llvm/test/Transforms/SampleProfile/Inputs/import.prof
+++ b/llvm/test/Transforms/SampleProfile/Inputs/import.prof
@@ -5,4 +5,4 @@ test:10000:0
4: foo1:1000
1: 1000
4: foo2:1000
- 1: 1000
+ 1: 1000 foo3:1000
diff --git a/llvm/test/Transforms/SampleProfile/import.ll b/llvm/test/Transforms/SampleProfile/import.ll
index ad9c2d55aa9..8cc2338a048 100644
--- a/llvm/test/Transforms/SampleProfile/import.ll
+++ b/llvm/test/Transforms/SampleProfile/import.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/import.prof -S | FileCheck %s
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file=%S/Inputs/import.prof -S | FileCheck %s
; Tests whether the functions in the inline stack are added to the
; function_entry_count metadata.
@@ -15,9 +15,9 @@ define void @test(void ()*) !dbg !7 {
ret void
}
-; GUIDs of foo, bar, foo1 and foo2 should be included in the metadata to make
-; sure hot inline stacks are imported.
-; CHECK: !{!"function_entry_count", i64 1, i64 2494702099028631698, i64 6699318081062747564, i64 7682762345278052905, i64 -2012135647395072713}
+; GUIDs of foo, bar, foo1, foo2 and foo3 should be included in the metadata to
+; make sure hot inline stacks are imported.
+; CHECK: !{!"function_entry_count", i64 1, i64 2494702099028631698, i64 6699318081062747564, i64 7682762345278052905, i64 -7908226060800700466, i64 -2012135647395072713}
!llvm.dbg.cu = !{!0}
!llvm.module.flags = !{!8, !9}
OpenPOWER on IntegriCloud