summaryrefslogtreecommitdiffstats
path: root/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp')
-rw-r--r--llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp182
1 files changed, 121 insertions, 61 deletions
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index f8223496997..11a43e803a9 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -1,4 +1,4 @@
-//===-- PGOInstrumentation.cpp - MST-based PGO Instrumentation ------------===//
+//===- PGOInstrumentation.cpp - MST-based PGO Instrumentation -------------===//
//
// The LLVM Compiler Infrastructure
//
@@ -50,37 +50,69 @@
#include "llvm/Transforms/PGOInstrumentation.h"
#include "CFGMST.h"
+#include "llvm/ADT/APInt.h"
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/SmallVector.h"
#include "llvm/ADT/Statistic.h"
+#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/Triple.h"
+#include "llvm/ADT/Twine.h"
+#include "llvm/ADT/iterator.h"
+#include "llvm/ADT/iterator_range.h"
#include "llvm/Analysis/BlockFrequencyInfo.h"
#include "llvm/Analysis/BranchProbabilityInfo.h"
#include "llvm/Analysis/CFG.h"
#include "llvm/Analysis/IndirectCallSiteVisitor.h"
#include "llvm/Analysis/LoopInfo.h"
#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/IR/Attributes.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
#include "llvm/IR/CallSite.h"
+#include "llvm/IR/Comdat.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
#include "llvm/IR/DiagnosticInfo.h"
#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/GlobalAlias.h"
#include "llvm/IR/GlobalValue.h"
+#include "llvm/IR/GlobalVariable.h"
#include "llvm/IR/IRBuilder.h"
-#include "llvm/IR/InstIterator.h"
+#include "llvm/IR/InstVisitor.h"
+#include "llvm/IR/InstrTypes.h"
+#include "llvm/IR/Instruction.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Intrinsics.h"
+#include "llvm/IR/LLVMContext.h"
#include "llvm/IR/MDBuilder.h"
#include "llvm/IR/Module.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/IR/ProfileSummary.h"
+#include "llvm/IR/Type.h"
+#include "llvm/IR/Value.h"
#include "llvm/Pass.h"
+#include "llvm/ProfileData/InstrProf.h"
#include "llvm/ProfileData/InstrProfReader.h"
-#include "llvm/ProfileData/ProfileCommon.h"
#include "llvm/Support/BranchProbability.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
#include "llvm/Support/DOTGraphTraits.h"
#include "llvm/Support/Debug.h"
+#include "llvm/Support/Error.h"
+#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/GraphWriter.h"
#include "llvm/Support/JamCRC.h"
+#include "llvm/Support/raw_ostream.h"
#include "llvm/Transforms/Instrumentation.h"
#include "llvm/Transforms/Utils/BasicBlockUtils.h"
#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <memory>
+#include <numeric>
#include <string>
#include <unordered_map>
#include <utility>
@@ -202,11 +234,9 @@ extern cl::opt<PGOViewCountsType> PGOViewCounts;
// Defined in Analysis/BlockFrequencyInfo.cpp: -view-bfi-func-name=
extern cl::opt<std::string> ViewBlockFreqFuncName;
-namespace {
-
// Return a string describing the branch condition that can be
// used in static branch probability heuristics:
-std::string getBranchCondString(Instruction *TI) {
+static std::string getBranchCondString(Instruction *TI) {
BranchInst *BI = dyn_cast<BranchInst>(TI);
if (!BI || !BI->isConditional())
return std::string();
@@ -237,6 +267,8 @@ std::string getBranchCondString(Instruction *TI) {
return result;
}
+namespace {
+
/// The select instruction visitor plays three roles specified
/// by the mode. In \c VM_counting mode, it simply counts the number of
/// select instructions. In \c VM_instrument mode, it inserts code to count
@@ -263,6 +295,7 @@ struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
Mode = VM_counting;
visit(Func);
}
+
// Visit the IR stream and instrument all select instructions. \p
// Ind is a pointer to the counter index variable; \p TotalNC
// is the total number of counters; \p FNV is the pointer to the
@@ -287,8 +320,10 @@ struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
void instrumentOneSelectInst(SelectInst &SI);
void annotateOneSelectInst(SelectInst &SI);
+
// Visit \p SI instruction and perform tasks according to visit mode.
void visitSelectInst(SelectInst &SI);
+
// Return the number of select instructions. This needs be called after
// countSelects().
unsigned getNumOfSelectInsts() const { return NSIs; }
@@ -332,8 +367,10 @@ struct MemIntrinsicVisitor : public InstVisitor<MemIntrinsicVisitor> {
// Visit the IR stream and annotate all mem intrinsic call instructions.
void instrumentOneMemIntrinsic(MemIntrinsic &MI);
+
// Visit \p MI instruction and perform tasks according to visit mode.
void visitMemIntrinsic(MemIntrinsic &SI);
+
unsigned getNumOfMemIntrinsics() const { return NMemIs; }
};
@@ -375,6 +412,7 @@ private:
std::string ProfileFileName;
bool runOnModule(Module &M) override;
+
void getAnalysisUsage(AnalysisUsage &AU) const override {
AU.addRequired<BlockFrequencyInfoWrapperPass>();
}
@@ -383,6 +421,7 @@ private:
} // end anonymous namespace
char PGOInstrumentationGenLegacyPass::ID = 0;
+
INITIALIZE_PASS_BEGIN(PGOInstrumentationGenLegacyPass, "pgo-instr-gen",
"PGO instrumentation.", false, false)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
@@ -395,6 +434,7 @@ ModulePass *llvm::createPGOInstrumentationGenLegacyPass() {
}
char PGOInstrumentationUseLegacyPass::ID = 0;
+
INITIALIZE_PASS_BEGIN(PGOInstrumentationUseLegacyPass, "pgo-instr-use",
"Read PGO instrumentation profile.", false, false)
INITIALIZE_PASS_DEPENDENCY(BlockFrequencyInfoWrapperPass)
@@ -407,6 +447,7 @@ ModulePass *llvm::createPGOInstrumentationUseLegacyPass(StringRef Filename) {
}
namespace {
+
/// \brief An MST based instrumentation for PGO
///
/// Implements a Minimum Spanning Tree (MST) based instrumentation for PGO
@@ -417,12 +458,13 @@ struct PGOEdge {
const BasicBlock *SrcBB;
const BasicBlock *DestBB;
uint64_t Weight;
- bool InMST;
- bool Removed;
- bool IsCritical;
+ bool InMST = false;
+ bool Removed = false;
+ bool IsCritical = false;
+
PGOEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
- : SrcBB(Src), DestBB(Dest), Weight(W), InMST(false), Removed(false),
- IsCritical(false) {}
+ : SrcBB(Src), DestBB(Dest), Weight(W) {}
+
// Return the information string of an edge.
const std::string infoString() const {
return (Twine(Removed ? "-" : " ") + (InMST ? " " : "*") +
@@ -434,9 +476,9 @@ struct PGOEdge {
struct BBInfo {
BBInfo *Group;
uint32_t Index;
- uint32_t Rank;
+ uint32_t Rank = 0;
- BBInfo(unsigned IX) : Group(this), Index(IX), Rank(0) {}
+ BBInfo(unsigned IX) : Group(this), Index(IX) {}
// Return the information string of this object.
const std::string infoString() const {
@@ -448,19 +490,22 @@ struct BBInfo {
template <class Edge, class BBInfo> class FuncPGOInstrumentation {
private:
Function &F;
- void computeCFGHash();
- void renameComdatFunction();
+
// A map that stores the Comdat group in function F.
std::unordered_multimap<Comdat *, GlobalValue *> &ComdatMembers;
+ void computeCFGHash();
+ void renameComdatFunction();
+
public:
std::vector<std::vector<Instruction *>> ValueSites;
SelectInstVisitor SIVisitor;
MemIntrinsicVisitor MIVisitor;
std::string FuncName;
GlobalVariable *FuncNameVar;
+
// CFG hash value for this function.
- uint64_t FunctionHash;
+ uint64_t FunctionHash = 0;
// The Minimum Spanning Tree of function CFG.
CFGMST<Edge, BBInfo> MST;
@@ -487,8 +532,7 @@ public:
bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
BlockFrequencyInfo *BFI = nullptr)
: F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1),
- SIVisitor(Func), MIVisitor(Func), FunctionHash(0), MST(F, BPI, BFI) {
-
+ SIVisitor(Func), MIVisitor(Func), MST(F, BPI, BFI) {
// This should be done before CFG hash computation.
SIVisitor.countSelects(Func);
MIVisitor.countMemIntrinsics(Func);
@@ -499,7 +543,7 @@ public:
FuncName = getPGOFuncName(F);
computeCFGHash();
- if (ComdatMembers.size())
+ if (!ComdatMembers.empty())
renameComdatFunction();
DEBUG(dumpInfo("after CFGMST"));
@@ -527,6 +571,8 @@ public:
}
};
+} // end anonymous namespace
+
// Compute Hash value for the CFG: the lower 32 bits are CRC32 of the index
// value of each BB in the CFG. The higher 32 bits record the number of edges.
template <class Edge, class BBInfo>
@@ -686,7 +732,7 @@ static void instrumentOneFunc(
"Cannot get the Instrumentation point");
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment),
- {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
Builder.getInt64(FuncInfo.FunctionHash), Builder.getInt32(NumCounters),
Builder.getInt32(I++)});
}
@@ -710,7 +756,7 @@ static void instrumentOneFunc(
"Cannot get the Instrumentation point");
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
- {llvm::ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncInfo.FuncNameVar, I8PtrTy),
Builder.getInt64(FuncInfo.FunctionHash),
Builder.CreatePtrToInt(Callee, Builder.getInt64Ty()),
Builder.getInt32(IPVK_IndirectCallTarget),
@@ -723,12 +769,15 @@ static void instrumentOneFunc(
F, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash);
}
+namespace {
+
// This class represents a CFG edge in profile use compilation.
struct PGOUseEdge : public PGOEdge {
- bool CountValid;
- uint64_t CountValue;
+ bool CountValid = false;
+ uint64_t CountValue = 0;
+
PGOUseEdge(const BasicBlock *Src, const BasicBlock *Dest, unsigned W = 1)
- : PGOEdge(Src, Dest, W), CountValid(false), CountValue(0) {}
+ : PGOEdge(Src, Dest, W) {}
// Set edge count value
void setEdgeCount(uint64_t Value) {
@@ -745,22 +794,21 @@ struct PGOUseEdge : public PGOEdge {
}
};
-typedef SmallVector<PGOUseEdge *, 2> DirectEdges;
+using DirectEdges = SmallVector<PGOUseEdge *, 2>;
// This class stores the auxiliary information for each BB.
struct UseBBInfo : public BBInfo {
- uint64_t CountValue;
+ uint64_t CountValue = 0;
bool CountValid;
- int32_t UnknownCountInEdge;
- int32_t UnknownCountOutEdge;
+ int32_t UnknownCountInEdge = 0;
+ int32_t UnknownCountOutEdge = 0;
DirectEdges InEdges;
DirectEdges OutEdges;
- UseBBInfo(unsigned IX)
- : BBInfo(IX), CountValue(0), CountValid(false), UnknownCountInEdge(0),
- UnknownCountOutEdge(0) {}
+
+ UseBBInfo(unsigned IX) : BBInfo(IX), CountValid(false) {}
+
UseBBInfo(unsigned IX, uint64_t C)
- : BBInfo(IX), CountValue(C), CountValid(true), UnknownCountInEdge(0),
- UnknownCountOutEdge(0) {}
+ : BBInfo(IX), CountValue(C), CountValid(true) {}
// Set the profile count value for this BB.
void setBBInfoCount(uint64_t Value) {
@@ -776,6 +824,8 @@ struct UseBBInfo : public BBInfo {
}
};
+} // end anonymous namespace
+
// Sum up the count values for all the edges.
static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
uint64_t Total = 0;
@@ -787,6 +837,8 @@ static uint64_t sumEdgeCount(const ArrayRef<PGOUseEdge *> Edges) {
return Total;
}
+namespace {
+
class PGOUseFunc {
public:
PGOUseFunc(Function &Func, Module *Modu,
@@ -794,7 +846,7 @@ public:
BranchProbabilityInfo *BPI = nullptr,
BlockFrequencyInfo *BFI = nullptr)
: F(Func), M(Modu), FuncInfo(Func, ComdatMembers, false, BPI, BFI),
- CountPosition(0), ProfileCountSize(0), FreqAttr(FFA_Normal) {}
+ FreqAttr(FFA_Normal) {}
// Read counts for the instrumented BB from profile.
bool readCounters(IndexedInstrProfReader *PGOReader);
@@ -819,6 +871,7 @@ public:
// Return the function hash.
uint64_t getFuncHash() const { return FuncInfo.FunctionHash; }
+
// Return the profile record for this function;
InstrProfRecord &getProfileRecord() { return ProfileRecord; }
@@ -841,6 +894,7 @@ public:
private:
Function &F;
Module *M;
+
// This member stores the shared information with class PGOGenFunc.
FuncPGOInstrumentation<PGOUseEdge, UseBBInfo> FuncInfo;
@@ -849,10 +903,10 @@ private:
uint64_t ProgramMaxCount;
// Position of counter that remains to be read.
- uint32_t CountPosition;
+ uint32_t CountPosition = 0;
// Total size of the profile count for this function.
- uint32_t ProfileCountSize;
+ uint32_t ProfileCountSize = 0;
// ProfileRecord for this function.
InstrProfRecord ProfileRecord;
@@ -887,11 +941,12 @@ private:
}
};
+} // end anonymous namespace
+
// Visit all the edges and assign the count value for the instrumented
// edges and the BB.
void PGOUseFunc::setInstrumentedCounts(
const std::vector<uint64_t> &CountFromProfile) {
-
assert(FuncInfo.getNumCounters() == CountFromProfile.size());
// Use a worklist as we will update the vector during the iteration.
std::vector<PGOUseEdge *> WorkList;
@@ -1136,7 +1191,7 @@ void SelectInstVisitor::instrumentOneSelectInst(SelectInst &SI) {
auto *Step = Builder.CreateZExt(SI.getCondition(), Int64Ty);
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_increment_step),
- {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
Builder.getInt64(FuncHash), Builder.getInt32(TotalNumCtrs),
Builder.getInt32(*CurCtrIdx), Step});
++(*CurCtrIdx);
@@ -1191,7 +1246,7 @@ void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) {
assert(!dyn_cast<ConstantInt>(Length));
Builder.CreateCall(
Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
- {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+ {ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
Builder.getInt64(FuncHash), Builder.CreateZExtOrTrunc(Length, Int64Ty),
Builder.getInt32(IPVK_MemOPSize), Builder.getInt32(CurCtrId)});
++CurCtrId;
@@ -1257,7 +1312,6 @@ void PGOUseFunc::annotateValueSites(uint32_t Kind) {
ValueSiteIndex++;
}
}
-} // end anonymous namespace
// Create a COMDAT variable INSTR_PROF_RAW_VERSION_VAR to make the runtime
// aware this is an ir_level profile so it can set the version flag.
@@ -1327,7 +1381,6 @@ bool PGOInstrumentationGenLegacyPass::runOnModule(Module &M) {
PreservedAnalyses PGOInstrumentationGen::run(Module &M,
ModuleAnalysisManager &AM) {
-
auto &FAM = AM.getResult<FunctionAnalysisManagerModuleProxy>(M).getManager();
auto LookupBPI = [&FAM](Function &F) {
return &FAM.getResult<BranchProbabilityAnalysis>(F);
@@ -1428,12 +1481,12 @@ static bool annotateAllFunctions(
// can affect the BranchProbabilityInfo of any callers, resulting in an
// inconsistent MST between prof-gen and prof-use.
for (auto &F : HotFunctions) {
- F->addFnAttr(llvm::Attribute::InlineHint);
+ F->addFnAttr(Attribute::InlineHint);
DEBUG(dbgs() << "Set inline attribute to function: " << F->getName()
<< "\n");
}
for (auto &F : ColdFunctions) {
- F->addFnAttr(llvm::Attribute::Cold);
+ F->addFnAttr(Attribute::Cold);
DEBUG(dbgs() << "Set cold attribute to function: " << F->getName() << "\n");
}
return true;
@@ -1477,9 +1530,19 @@ bool PGOInstrumentationUseLegacyPass::runOnModule(Module &M) {
return annotateAllFunctions(M, ProfileFileName, LookupBPI, LookupBFI);
}
-namespace llvm {
-void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
- uint64_t MaxCount) {
+static std::string getSimpleNodeName(const BasicBlock *Node) {
+ if (!Node->getName().empty())
+ return Node->getName();
+
+ std::string SimpleNodeName;
+ raw_string_ostream OS(SimpleNodeName);
+ Node->printAsOperand(OS, false);
+ return OS.str();
+}
+
+void llvm::setProfMetadata(Module *M, Instruction *TI,
+ ArrayRef<uint64_t> EdgeCounts,
+ uint64_t MaxCount) {
MDBuilder MDB(M->getContext());
assert(MaxCount > 0 && "Bad max count");
uint64_t Scale = calculateCountScale(MaxCount);
@@ -1490,7 +1553,7 @@ void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
DEBUG(dbgs() << "Weight is: ";
for (const auto &W : Weights) { dbgs() << W << " "; }
dbgs() << "\n";);
- TI->setMetadata(llvm::LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
+ TI->setMetadata(LLVMContext::MD_prof, MDB.createBranchWeights(Weights));
if (EmitBranchProbability) {
std::string BrCondStr = getBranchCondString(TI);
if (BrCondStr.empty())
@@ -1517,36 +1580,32 @@ void setProfMetadata(Module *M, Instruction *TI, ArrayRef<uint64_t> EdgeCounts,
}
}
+namespace llvm {
+
template <> struct GraphTraits<PGOUseFunc *> {
- typedef const BasicBlock *NodeRef;
- typedef succ_const_iterator ChildIteratorType;
- typedef pointer_iterator<Function::const_iterator> nodes_iterator;
+ using NodeRef = const BasicBlock *;
+ using ChildIteratorType = succ_const_iterator;
+ using nodes_iterator = pointer_iterator<Function::const_iterator>;
static NodeRef getEntryNode(const PGOUseFunc *G) {
return &G->getFunc().front();
}
+
static ChildIteratorType child_begin(const NodeRef N) {
return succ_begin(N);
}
+
static ChildIteratorType child_end(const NodeRef N) { return succ_end(N); }
+
static nodes_iterator nodes_begin(const PGOUseFunc *G) {
return nodes_iterator(G->getFunc().begin());
}
+
static nodes_iterator nodes_end(const PGOUseFunc *G) {
return nodes_iterator(G->getFunc().end());
}
};
-static std::string getSimpleNodeName(const BasicBlock *Node) {
- if (!Node->getName().empty())
- return Node->getName();
-
- std::string SimpleNodeName;
- raw_string_ostream OS(SimpleNodeName);
- Node->printAsOperand(OS, false);
- return OS.str();
-}
-
template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits {
explicit DOTGraphTraits(bool isSimple = false)
: DefaultDOTGraphTraits(isSimple) {}
@@ -1586,4 +1645,5 @@ template <> struct DOTGraphTraits<PGOUseFunc *> : DefaultDOTGraphTraits {
return Result;
}
};
-} // namespace llvm
+
+} // end namespace llvm
OpenPOWER on IntegriCloud