summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/ProfileSummaryInfo.cpp9
-rw-r--r--llvm/lib/IR/Module.cpp12
-rw-r--r--llvm/lib/IR/ProfileSummary.cpp6
-rw-r--r--llvm/lib/ProfileData/InstrProfReader.cpp22
-rw-r--r--llvm/lib/ProfileData/InstrProfWriter.cpp36
-rw-r--r--llvm/lib/Transforms/IPO/HotColdSplitting.cpp2
-rw-r--r--llvm/lib/Transforms/IPO/SampleProfile.cpp5
-rw-r--r--llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp8
8 files changed, 80 insertions, 20 deletions
diff --git a/llvm/lib/Analysis/ProfileSummaryInfo.cpp b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
index bd7817ce67f..1f9e9813c76 100644
--- a/llvm/lib/Analysis/ProfileSummaryInfo.cpp
+++ b/llvm/lib/Analysis/ProfileSummaryInfo.cpp
@@ -79,7 +79,14 @@ static const ProfileSummaryEntry &getEntryForPercentile(SummaryEntryVector &DS,
bool ProfileSummaryInfo::computeSummary() {
if (Summary)
return true;
- auto *SummaryMD = M.getProfileSummary();
+ // First try to get context sensitive ProfileSummary.
+ auto *SummaryMD = M.getProfileSummary(/* IsCS */ true);
+ if (SummaryMD) {
+ Summary.reset(ProfileSummary::getFromMD(SummaryMD));
+ return true;
+ }
+ // This will actually return PSK_Instr or PSK_Sample summary.
+ SummaryMD = M.getProfileSummary(/* IsCS */ false);
if (!SummaryMD)
return false;
Summary.reset(ProfileSummary::getFromMD(SummaryMD));
diff --git a/llvm/lib/IR/Module.cpp b/llvm/lib/IR/Module.cpp
index b6dd7ab70a8..dbf4035ac7c 100644
--- a/llvm/lib/IR/Module.cpp
+++ b/llvm/lib/IR/Module.cpp
@@ -531,12 +531,16 @@ void Module::setCodeModel(CodeModel::Model CL) {
addModuleFlag(ModFlagBehavior::Error, "Code Model", CL);
}
-void Module::setProfileSummary(Metadata *M) {
- addModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M);
+void Module::setProfileSummary(Metadata *M, ProfileSummary::Kind Kind) {
+ if (Kind == ProfileSummary::PSK_CSInstr)
+ addModuleFlag(ModFlagBehavior::Error, "CSProfileSummary", M);
+ else
+ addModuleFlag(ModFlagBehavior::Error, "ProfileSummary", M);
}
-Metadata *Module::getProfileSummary() {
- return getModuleFlag("ProfileSummary");
+Metadata *Module::getProfileSummary(bool IsCS) {
+ return (IsCS ? getModuleFlag("CSProfileSummary")
+ : getModuleFlag("ProfileSummary"));
}
void Module::setOwnedMemoryBuffer(std::unique_ptr<MemoryBuffer> MB) {
diff --git a/llvm/lib/IR/ProfileSummary.cpp b/llvm/lib/IR/ProfileSummary.cpp
index d8596924719..11d95ac19be 100644
--- a/llvm/lib/IR/ProfileSummary.cpp
+++ b/llvm/lib/IR/ProfileSummary.cpp
@@ -21,8 +21,6 @@
using namespace llvm;
-const char *ProfileSummary::KindStr[2] = {"InstrProf", "SampleProfile"};
-
// Return an MDTuple with two elements. The first element is a string Key and
// the second is a uint64_t Value.
static Metadata *getKeyValMD(LLVMContext &Context, const char *Key,
@@ -68,6 +66,7 @@ Metadata *ProfileSummary::getDetailedSummaryMD(LLVMContext &Context) {
// "SampleProfile"). The rest of the elements of the outer MDTuple are specific
// to the kind of profile summary as returned by getFormatSpecificMD.
Metadata *ProfileSummary::getMD(LLVMContext &Context) {
+ const char *KindStr[3] = {"InstrProf", "CSInstrProf", "SampleProfile"};
Metadata *Components[] = {
getKeyValMD(Context, "ProfileFormat", KindStr[PSK]),
getKeyValMD(Context, "TotalCount", getTotalCount()),
@@ -153,6 +152,9 @@ ProfileSummary *ProfileSummary::getFromMD(Metadata *MD) {
else if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat",
"InstrProf"))
SummaryKind = PSK_Instr;
+ else if (isKeyValuePair(dyn_cast_or_null<MDTuple>(FormatMD), "ProfileFormat",
+ "CSInstrProf"))
+ SummaryKind = PSK_CSInstr;
else
return nullptr;
diff --git a/llvm/lib/ProfileData/InstrProfReader.cpp b/llvm/lib/ProfileData/InstrProfReader.cpp
index 5919efa72d3..1852774eb8d 100644
--- a/llvm/lib/ProfileData/InstrProfReader.cpp
+++ b/llvm/lib/ProfileData/InstrProfReader.cpp
@@ -162,7 +162,10 @@ Error TextInstrProfReader::readHeader() {
IsIRInstr = true;
else if (Str.equals_lower("fe"))
IsIRInstr = false;
- else
+ else if (Str.equals_lower("csir")) {
+ IsIRInstr = true;
+ HasCSIRLevelProfile = true;
+ } else
return error(instrprof_error::bad_header);
++Line;
@@ -733,7 +736,7 @@ bool IndexedInstrProfReader::hasFormat(const MemoryBuffer &DataBuffer) {
const unsigned char *
IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
- const unsigned char *Cur) {
+ const unsigned char *Cur, bool UseCS) {
using namespace IndexedInstrProf;
using namespace support;
@@ -760,10 +763,13 @@ IndexedInstrProfReader::readSummary(IndexedInstrProf::ProfVersion Version,
DetailedSummary.emplace_back((uint32_t)Ent.Cutoff, Ent.MinBlockCount,
Ent.NumBlocks);
}
+ std::unique_ptr<llvm::ProfileSummary> &Summary =
+ UseCS ? this->CS_Summary : this->Summary;
+
// initialize InstrProfSummary using the SummaryData from disk.
- this->Summary = llvm::make_unique<ProfileSummary>(
- ProfileSummary::PSK_Instr, DetailedSummary,
- SummaryData->get(Summary::TotalBlockCount),
+ Summary = llvm::make_unique<ProfileSummary>(
+ UseCS ? ProfileSummary::PSK_CSInstr : ProfileSummary::PSK_Instr,
+ DetailedSummary, SummaryData->get(Summary::TotalBlockCount),
SummaryData->get(Summary::MaxBlockCount),
SummaryData->get(Summary::MaxInternalBlockCount),
SummaryData->get(Summary::MaxFunctionCount),
@@ -805,7 +811,11 @@ Error IndexedInstrProfReader::readHeader() {
IndexedInstrProf::ProfVersion::CurrentVersion)
return error(instrprof_error::unsupported_version);
- Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur);
+ Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur,
+ /* UseCS */ false);
+ if (Header->Version & VARIANT_MASK_CSIR_PROF)
+ Cur = readSummary((IndexedInstrProf::ProfVersion)FormatVersion, Cur,
+ /* UseCS */ true);
// Read the hash type and start offset.
IndexedInstrProf::HashT HashType = static_cast<IndexedInstrProf::HashT>(
diff --git a/llvm/lib/ProfileData/InstrProfWriter.cpp b/llvm/lib/ProfileData/InstrProfWriter.cpp
index c025cd38121..3aaa193d687 100644
--- a/llvm/lib/ProfileData/InstrProfWriter.cpp
+++ b/llvm/lib/ProfileData/InstrProfWriter.cpp
@@ -101,6 +101,7 @@ public:
support::endianness ValueProfDataEndianness = support::little;
InstrProfSummaryBuilder *SummaryBuilder;
+ InstrProfSummaryBuilder *CSSummaryBuilder;
InstrProfRecordWriterTrait() = default;
@@ -142,7 +143,10 @@ public:
endian::Writer LE(Out, little);
for (const auto &ProfileData : *V) {
const InstrProfRecord &ProfRecord = ProfileData.second;
- SummaryBuilder->addRecord(ProfRecord);
+ if (NamedInstrProfRecord::hasCSFlagInHash(ProfileData.first))
+ CSSummaryBuilder->addRecord(ProfRecord);
+ else
+ SummaryBuilder->addRecord(ProfRecord);
LE.write<uint64_t>(ProfileData.first); // Function hash
LE.write<uint64_t>(ProfRecord.Counts.size());
@@ -253,6 +257,8 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
InstrProfSummaryBuilder ISB(ProfileSummaryBuilder::DefaultCutoffs);
InfoObj->SummaryBuilder = &ISB;
+ InstrProfSummaryBuilder CSISB(ProfileSummaryBuilder::DefaultCutoffs);
+ InfoObj->CSSummaryBuilder = &CSISB;
// Populate the hash table generator.
for (const auto &I : FunctionData)
@@ -264,6 +270,10 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
Header.Version = IndexedInstrProf::ProfVersion::CurrentVersion;
if (ProfileKind == PF_IRLevel)
Header.Version |= VARIANT_MASK_IR_PROF;
+ if (ProfileKind == PF_IRLevelWithCS) {
+ Header.Version |= VARIANT_MASK_IR_PROF;
+ Header.Version |= VARIANT_MASK_CSIR_PROF;
+ }
Header.Unused = 0;
Header.HashType = static_cast<uint64_t>(IndexedInstrProf::HashType);
Header.HashOffset = 0;
@@ -287,6 +297,14 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
uint64_t SummaryOffset = OS.tell();
for (unsigned I = 0; I < SummarySize / sizeof(uint64_t); I++)
OS.write(0);
+ uint64_t CSSummaryOffset = 0;
+ uint64_t CSSummarySize = 0;
+ if (ProfileKind == PF_IRLevelWithCS) {
+ CSSummaryOffset = OS.tell();
+ CSSummarySize = SummarySize / sizeof(uint64_t);
+ for (unsigned I = 0; I < CSSummarySize; I++)
+ OS.write(0);
+ }
// Write the hash table.
uint64_t HashTableStart = Generator.Emit(OS.OS, *InfoObj);
@@ -300,13 +318,25 @@ void InstrProfWriter::writeImpl(ProfOStream &OS) {
setSummary(TheSummary.get(), *PS);
InfoObj->SummaryBuilder = nullptr;
+ // For Context Sensitive summary.
+ std::unique_ptr<IndexedInstrProf::Summary> TheCSSummary = nullptr;
+ if (ProfileKind == PF_IRLevelWithCS) {
+ TheCSSummary = IndexedInstrProf::allocSummary(SummarySize);
+ std::unique_ptr<ProfileSummary> CSPS = CSISB.getSummary();
+ setSummary(TheCSSummary.get(), *CSPS);
+ }
+ InfoObj->CSSummaryBuilder = nullptr;
+
// Now do the final patch:
PatchItem PatchItems[] = {
// Patch the Header.HashOffset field.
{HashTableStartFieldOffset, &HashTableStart, 1},
// Patch the summary data.
{SummaryOffset, reinterpret_cast<uint64_t *>(TheSummary.get()),
- (int)(SummarySize / sizeof(uint64_t))}};
+ (int)(SummarySize / sizeof(uint64_t))},
+ {CSSummaryOffset, reinterpret_cast<uint64_t *>(TheCSSummary.get()),
+ (int)CSSummarySize}};
+
OS.patch(PatchItems, sizeof(PatchItems) / sizeof(*PatchItems));
}
@@ -375,6 +405,8 @@ void InstrProfWriter::writeRecordInText(StringRef Name, uint64_t Hash,
Error InstrProfWriter::writeText(raw_fd_ostream &OS) {
if (ProfileKind == PF_IRLevel)
OS << "# IR level Instrumentation Flag\n:ir\n";
+ else if (ProfileKind == PF_IRLevelWithCS)
+ OS << "# CSIR level Instrumentation Flag\n:csir\n";
InstrProfSymtab Symtab;
for (const auto &I : FunctionData)
if (shouldEncodeData(I.getValue()))
diff --git a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
index a237eb7b4fc..5d6add54319 100644
--- a/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
+++ b/llvm/lib/Transforms/IPO/HotColdSplitting.cpp
@@ -664,7 +664,7 @@ bool HotColdSplitting::outlineColdRegions(Function &F, bool HasProfileSummary) {
bool HotColdSplitting::run(Module &M) {
bool Changed = false;
- bool HasProfileSummary = M.getProfileSummary();
+ bool HasProfileSummary = (M.getProfileSummary(/* IsCS */ false) != nullptr);
for (auto It = M.begin(), End = M.end(); It != End; ++It) {
Function &F = *It;
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 0b880b1e431..f3bad362237 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -1595,8 +1595,9 @@ bool SampleProfileLoader::runOnModule(Module &M, ModuleAnalysisManager *AM,
return false;
PSI = _PSI;
- if (M.getProfileSummary() == nullptr)
- M.setProfileSummary(Reader->getSummary().getMD(M.getContext()));
+ if (M.getProfileSummary(/* IsCS */ false) == nullptr)
+ M.setProfileSummary(Reader->getSummary().getMD(M.getContext()),
+ ProfileSummary::PSK_Sample);
// Compute the total number of samples collected in this profile.
for (const auto &I : Reader->getProfiles())
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index d44c2addb79..3cf3d90620f 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1147,7 +1147,7 @@ bool PGOUseFunc::readCounters(IndexedInstrProfReader *PGOReader, bool &AllZeros)
getBBInfo(nullptr).UnknownCountInEdge = 2;
setInstrumentedCounts(CountFromProfile);
- ProgramMaxCount = PGOReader->getMaximumFunctionCount();
+ ProgramMaxCount = PGOReader->getMaximumFunctionCount(IsCS);
return true;
}
@@ -1531,6 +1531,8 @@ static bool annotateAllFunctions(
StringRef("Cannot get PGOReader")));
return false;
}
+ if (!PGOReader->hasCSIRLevelProfile() && IsCS)
+ return false;
// TODO: might need to change the warning once the clang option is finalized.
if (!PGOReader->isIRLevelProfile()) {
@@ -1599,7 +1601,9 @@ static bool annotateAllFunctions(
}
}
}
- M.setProfileSummary(PGOReader->getSummary().getMD(M.getContext()));
+ M.setProfileSummary(PGOReader->getSummary(IsCS).getMD(M.getContext()),
+ IsCS ? ProfileSummary::PSK_CSInstr
+ : ProfileSummary::PSK_Instr);
// Set function hotness attribute from the profile.
// We have to apply these attributes at the end because their presence
OpenPOWER on IntegriCloud