summaryrefslogtreecommitdiffstats
path: root/llvm
diff options
context:
space:
mode:
Diffstat (limited to 'llvm')
-rw-r--r--llvm/include/llvm/ProfileData/SampleProf.h86
-rw-r--r--llvm/lib/ProfileData/SampleProf.cpp8
-rw-r--r--llvm/lib/ProfileData/SampleProfReader.cpp1
-rw-r--r--llvm/lib/Transforms/IPO/SampleProfile.cpp23
-rw-r--r--llvm/test/Transforms/SampleProfile/Inputs/function_metadata.compact.afdobin0 -> 280 bytes
-rw-r--r--llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof10
-rw-r--r--llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdobin0 -> 497 bytes
-rw-r--r--llvm/test/Transforms/SampleProfile/function_metadata.ll1
-rw-r--r--llvm/test/Transforms/SampleProfile/indirect-call.ll1
9 files changed, 108 insertions, 22 deletions
diff --git a/llvm/include/llvm/ProfileData/SampleProf.h b/llvm/include/llvm/ProfileData/SampleProf.h
index 0cd6dd2c2c0..f5e123f3c5c 100644
--- a/llvm/include/llvm/ProfileData/SampleProf.h
+++ b/llvm/include/llvm/ProfileData/SampleProf.h
@@ -293,6 +293,9 @@ public:
/// with the maximum total sample count.
const FunctionSamples *findFunctionSamplesAt(const LineLocation &Loc,
StringRef CalleeName) const {
+ std::string CalleeGUID;
+ CalleeName = getRepInFormat(CalleeName, Format, CalleeGUID);
+
auto iter = CallsiteSamples.find(Loc);
if (iter == CallsiteSamples.end())
return nullptr;
@@ -377,23 +380,23 @@ public:
/// GUID to \p S. Also traverse the BodySamples to add hot CallTarget's GUID
/// to \p S.
void findInlinedFunctions(DenseSet<GlobalValue::GUID> &S, const Module *M,
- uint64_t Threshold, bool isCompact) const {
+ uint64_t Threshold) const {
if (TotalSamples <= Threshold)
return;
- S.insert(Function::getGUID(Name));
+ S.insert(getGUID(Name));
// Import hot CallTargets, which may not be available in IR because full
// profile annotation cannot be done until backend compilation in ThinLTO.
for (const auto &BS : BodySamples)
for (const auto &TS : BS.second.getCallTargets())
if (TS.getValue() > Threshold) {
- Function *Callee = M->getFunction(TS.getKey());
+ const Function *Callee =
+ M->getFunction(getNameInModule(TS.getKey(), M));
if (!Callee || !Callee->getSubprogram())
- S.insert(isCompact ? std::stol(TS.getKey().data())
- : Function::getGUID(TS.getKey()));
+ S.insert(getGUID(TS.getKey()));
}
for (const auto &CS : CallsiteSamples)
for (const auto &NameFS : CS.second)
- NameFS.second.findInlinedFunctions(S, M, Threshold, isCompact);
+ NameFS.second.findInlinedFunctions(S, M, Threshold);
}
/// Set the name of the function.
@@ -402,6 +405,29 @@ public:
/// Return the function name.
const StringRef &getName() const { return Name; }
+ /// Return the original function name if it exists in Module \p M.
+ StringRef getFuncNameInModule(const Module *M) const {
+ return getNameInModule(Name, M);
+ }
+
+ /// Translate \p Name into its original name in Module.
+ /// When the Format is not SPF_Compact_Binary, \p Name needs no translation.
+ /// When the Format is SPF_Compact_Binary, \p Name in current FunctionSamples
+ /// is actually GUID of the original function name. getNameInModule will
+ /// translate \p Name in current FunctionSamples into its original name.
+ /// If the original name doesn't exist in \p M, return empty StringRef.
+ StringRef getNameInModule(StringRef Name, const Module *M) const {
+ if (Format != SPF_Compact_Binary)
+ return Name;
+ // Expect CurrentModule to be initialized by GUIDToFuncNameMapper.
+ if (M != CurrentModule)
+ llvm_unreachable("Input Module should be the same as CurrentModule");
+ auto iter = GUIDToFuncNameMap.find(std::stoul(Name.data()));
+ if (iter == GUIDToFuncNameMap.end())
+ return StringRef();
+ return iter->second;
+ }
+
/// Returns the line offset to the start line of the subprogram.
/// We assume that a single function will not exceed 65535 LOC.
static unsigned getOffset(const DILocation *DIL);
@@ -417,6 +443,54 @@ public:
/// \returns the FunctionSamples pointer to the inlined instance.
const FunctionSamples *findFunctionSamples(const DILocation *DIL) const;
+ static SampleProfileFormat Format;
+ /// GUIDToFuncNameMap saves the mapping from GUID to the symbol name, for
+ /// all the function symbols defined or declared in CurrentModule.
+ static DenseMap<uint64_t, StringRef> GUIDToFuncNameMap;
+ static Module *CurrentModule;
+
+ class GUIDToFuncNameMapper {
+ public:
+ GUIDToFuncNameMapper(Module &M) {
+ if (Format != SPF_Compact_Binary)
+ return;
+
+ for (const auto &F : M) {
+ StringRef OrigName = F.getName();
+ GUIDToFuncNameMap.insert({Function::getGUID(OrigName), OrigName});
+ /// Local to global var promotion used by optimization like thinlto
+ /// will rename the var and add suffix like ".llvm.xxx" to the
+ /// original local name. In sample profile, the suffixes of function
+ /// names are all stripped. Since it is possible that the mapper is
+ /// built in post-thin-link phase and var promotion has been done,
+ /// we need to add the substring of function name without the suffix
+ /// into the GUIDToFuncNameMap.
+ auto pos = OrigName.find('.');
+ if (pos != StringRef::npos) {
+ StringRef NewName = OrigName.substr(0, pos);
+ GUIDToFuncNameMap.insert({Function::getGUID(NewName), NewName});
+ }
+ }
+ CurrentModule = &M;
+ }
+
+ ~GUIDToFuncNameMapper() {
+ if (Format != SPF_Compact_Binary)
+ return;
+
+ GUIDToFuncNameMap.clear();
+ CurrentModule = nullptr;
+ }
+ };
+
+ // Assume the input \p Name is a name coming from FunctionSamples itself.
+ // If the format is SPF_Compact_Binary, the name is already a GUID and we
+ // don't want to return the GUID of GUID.
+ static uint64_t getGUID(const StringRef &Name) {
+ return (Format == SPF_Compact_Binary) ? std::stoul(Name.data())
+ : Function::getGUID(Name);
+ }
+
private:
/// Mangled name of the function.
StringRef Name;
diff --git a/llvm/lib/ProfileData/SampleProf.cpp b/llvm/lib/ProfileData/SampleProf.cpp
index 30438ba7962..b0818d12475 100644
--- a/llvm/lib/ProfileData/SampleProf.cpp
+++ b/llvm/lib/ProfileData/SampleProf.cpp
@@ -26,6 +26,14 @@
using namespace llvm;
using namespace sampleprof;
+namespace llvm {
+namespace sampleprof {
+SampleProfileFormat FunctionSamples::Format;
+DenseMap<uint64_t, StringRef> FunctionSamples::GUIDToFuncNameMap;
+Module *FunctionSamples::CurrentModule;
+} // namespace sampleprof
+} // namespace llvm
+
namespace {
// FIXME: This class is only here to support the transition to llvm::Error. It
diff --git a/llvm/lib/ProfileData/SampleProfReader.cpp b/llvm/lib/ProfileData/SampleProfReader.cpp
index 79335e67cd9..5503104c1ef 100644
--- a/llvm/lib/ProfileData/SampleProfReader.cpp
+++ b/llvm/lib/ProfileData/SampleProfReader.cpp
@@ -880,6 +880,7 @@ SampleProfileReader::create(std::unique_ptr<MemoryBuffer> &B, LLVMContext &C) {
else
return sampleprof_error::unrecognized_format;
+ FunctionSamples::Format = Reader->getFormat();
if (std::error_code EC = Reader->readHeader())
return EC;
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index dcd24595f7e..b9b055d36f7 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -643,8 +643,6 @@ SampleProfileLoader::findCalleeFunctionSamples(const Instruction &Inst) const {
if (FS == nullptr)
return nullptr;
- std::string CalleeGUID;
- CalleeName = getRepInFormat(CalleeName, Reader->getFormat(), CalleeGUID);
return FS->findFunctionSamplesAt(LineLocation(FunctionSamples::getOffset(DIL),
DIL->getBaseDiscriminator()),
CalleeName);
@@ -685,7 +683,10 @@ SampleProfileLoader::findIndirectCallFunctionSamples(
}
llvm::sort(R.begin(), R.end(),
[](const FunctionSamples *L, const FunctionSamples *R) {
- return L->getEntrySamples() > R->getEntrySamples();
+ if (L->getEntrySamples() != R->getEntrySamples())
+ return L->getEntrySamples() > R->getEntrySamples();
+ return FunctionSamples::getGUID(L->getName()) <
+ FunctionSamples::getGUID(R->getName());
});
}
return R;
@@ -760,7 +761,6 @@ bool SampleProfileLoader::inlineHotFunctions(
Function &F, DenseSet<GlobalValue::GUID> &InlinedGUIDs) {
DenseSet<Instruction *> PromotedInsns;
bool Changed = false;
- bool isCompact = (Reader->getFormat() == SPF_Compact_Binary);
while (true) {
bool LocalChanged = false;
SmallVector<Instruction *, 10> CIS;
@@ -792,19 +792,16 @@ bool SampleProfileLoader::inlineHotFunctions(
for (const auto *FS : findIndirectCallFunctionSamples(*I, Sum)) {
if (IsThinLTOPreLink) {
FS->findInlinedFunctions(InlinedGUIDs, F.getParent(),
- PSI->getOrCompHotCountThreshold(),
- isCompact);
+ PSI->getOrCompHotCountThreshold());
continue;
}
- auto CalleeFunctionName = FS->getName();
+ auto CalleeFunctionName = FS->getFuncNameInModule(F.getParent());
// If it is a recursive call, we do not inline it as it could bloat
// the code exponentially. There is way to better handle this, e.g.
// clone the caller first, and inline the cloned caller if it is
// recursive. As llvm does not inline recursive calls, we will
// simply ignore it instead of handling it explicitly.
- std::string FGUID;
- auto Fname = getRepInFormat(F.getName(), Reader->getFormat(), FGUID);
- if (CalleeFunctionName == Fname)
+ if (CalleeFunctionName == F.getName())
continue;
const char *Reason = "Callee function not available";
@@ -834,8 +831,7 @@ bool SampleProfileLoader::inlineHotFunctions(
LocalChanged = true;
} else if (IsThinLTOPreLink) {
findCalleeFunctionSamples(*I)->findInlinedFunctions(
- InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold(),
- isCompact);
+ InlinedGUIDs, F.getParent(), PSI->getOrCompHotCountThreshold());
}
}
if (LocalChanged) {
@@ -1177,7 +1173,7 @@ static SmallVector<InstrProfValueData, 2> SortCallTargets(
const SampleRecord::CallTargetMap &M) {
SmallVector<InstrProfValueData, 2> R;
for (auto I = M.begin(); I != M.end(); ++I)
- R.push_back({Function::getGUID(I->getKey()), I->getValue()});
+ R.push_back({FunctionSamples::getGUID(I->getKey()), I->getValue()});
llvm::sort(R.begin(), R.end(),
[](const InstrProfValueData &L, const InstrProfValueData &R) {
if (L.Count == R.Count)
@@ -1533,6 +1529,7 @@ ModulePass *llvm::createSampleProfileLoaderPass(StringRef Name) {
bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
ProfileSummaryInfo *_PSI) {
+ FunctionSamples::GUIDToFuncNameMapper Mapper(M);
if (!ProfileIsValid)
return false;
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.compact.afdo b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.compact.afdo
new file mode 100644
index 00000000000..16d7d008301
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.compact.afdo
Binary files differ
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof
index 239b5014b36..621bed722b7 100644
--- a/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof
+++ b/llvm/test/Transforms/SampleProfile/Inputs/function_metadata.prof
@@ -1,13 +1,17 @@
-test:10000:0
+test:3200:0
+ 1: 100
2: 100
- 3: 100
3: foo:1000
+ 1: 800
3: bar:200
+ 2: 190
4: baz:10
+ 2: 10
4: foo1:1000
1: 1000
4: foo2:1000
1: 1000 foo3:1000
-test_liveness:10000:0
+test_liveness:1000:0
1: foo:1000
1: foo_available:1000
+ 2: 1000
diff --git a/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo
new file mode 100644
index 00000000000..b43ac6fcd60
--- /dev/null
+++ b/llvm/test/Transforms/SampleProfile/Inputs/indirect-call.compact.afdo
Binary files differ
diff --git a/llvm/test/Transforms/SampleProfile/function_metadata.ll b/llvm/test/Transforms/SampleProfile/function_metadata.ll
index 18363514711..dca6660415e 100644
--- a/llvm/test/Transforms/SampleProfile/function_metadata.ll
+++ b/llvm/test/Transforms/SampleProfile/function_metadata.ll
@@ -1,4 +1,5 @@
; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.prof -S | FileCheck %s
+; RUN: opt < %s -passes='thinlto-pre-link<O2>' -pgo-kind=new-pm-pgo-sample-use-pipeline -profile-file=%S/Inputs/function_metadata.compact.afdo -S | FileCheck %s
; Tests whether the functions in the inline stack are added to the
; function_entry_count metadata.
diff --git a/llvm/test/Transforms/SampleProfile/indirect-call.ll b/llvm/test/Transforms/SampleProfile/indirect-call.ll
index 0c00639e6c0..95d0c473ae7 100644
--- a/llvm/test/Transforms/SampleProfile/indirect-call.ll
+++ b/llvm/test/Transforms/SampleProfile/indirect-call.ll
@@ -1,4 +1,5 @@
; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.prof -S | FileCheck %s
+; RUN: opt < %s -sample-profile -sample-profile-file=%S/Inputs/indirect-call.compact.afdo -S | FileCheck %s
; CHECK-LABEL: @test
define void @test(void ()*) !dbg !3 {
OpenPOWER on IntegriCloud