summaryrefslogtreecommitdiffstats
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp87
-rw-r--r--llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp84
2 files changed, 158 insertions, 13 deletions
diff --git a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
index 2ff3bd178d4..3e21f399bb0 100644
--- a/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
+++ b/llvm/lib/Transforms/Instrumentation/InstrProfiling.cpp
@@ -77,6 +77,17 @@ cl::opt<double> NumCountersPerValueSite(
// is usually smaller than 2.
cl::init(1.0));
+cl::opt<std::string> MemOPSizeRange(
+ "memop-size-range",
+ cl::desc("Set the range of size in memory intrinsic calls to be profiled "
+ "precisely, in a format of <start_val>:<end_val>"),
+ cl::init(""));
+cl::opt<unsigned> MemOPSizeLarge(
+ "memop-size-large",
+ cl::desc("Set large value thresthold in memory intrinsic size profiling. "
+ "Value of 0 disables the large value profiling."),
+ cl::init(8192));
+
class InstrProfilingLegacyPass : public ModulePass {
InstrProfiling InstrProf;
@@ -165,6 +176,7 @@ bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
NamesSize = 0;
ProfileDataMap.clear();
UsedVars.clear();
+ getMemOPSizeOptions();
// We did not know how many value sites there would be inside
// the instrumented function. This is counting the number of instrumented
@@ -217,17 +229,34 @@ bool InstrProfiling::run(Module &M, const TargetLibraryInfo &TLI) {
}
static Constant *getOrInsertValueProfilingCall(Module &M,
- const TargetLibraryInfo &TLI) {
+ const TargetLibraryInfo &TLI,
+ bool IsRange = false) {
LLVMContext &Ctx = M.getContext();
auto *ReturnTy = Type::getVoidTy(M.getContext());
- Type *ParamTypes[] = {
+
+ Constant *Res;
+ if (!IsRange) {
+ Type *ParamTypes[] = {
#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
#include "llvm/ProfileData/InstrProfData.inc"
- };
- auto *ValueProfilingCallTy =
- FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
- Constant *Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
- ValueProfilingCallTy);
+ };
+ auto *ValueProfilingCallTy =
+ FunctionType::get(ReturnTy, makeArrayRef(ParamTypes), false);
+ Res = M.getOrInsertFunction(getInstrProfValueProfFuncName(),
+ ValueProfilingCallTy);
+ } else {
+ Type *RangeParamTypes[] = {
+#define VALUE_RANGE_PROF 1
+#define VALUE_PROF_FUNC_PARAM(ParamType, ParamName, ParamLLVMType) ParamLLVMType
+#include "llvm/ProfileData/InstrProfData.inc"
+#undef VALUE_RANGE_PROF
+ };
+ auto *ValueRangeProfilingCallTy =
+ FunctionType::get(ReturnTy, makeArrayRef(RangeParamTypes), false);
+ Res = M.getOrInsertFunction(getInstrProfValueRangeProfFuncName(),
+ ValueRangeProfilingCallTy);
+ }
+
if (Function *FunRes = dyn_cast<Function>(Res)) {
if (auto AK = TLI.getExtAttrForI32Param(false))
FunRes->addAttribute(3, AK);
@@ -261,11 +290,24 @@ void InstrProfiling::lowerValueProfileInst(InstrProfValueProfileInst *Ind) {
Index += It->second.NumValueSites[Kind];
IRBuilder<> Builder(Ind);
- Value *Args[3] = {Ind->getTargetValue(),
- Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
- Builder.getInt32(Index)};
- CallInst *Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI),
- Args);
+ bool IsRange = (Ind->getValueKind()->getZExtValue() ==
+ llvm::InstrProfValueKind::IPVK_MemOPSize);
+ CallInst *Call = nullptr;
+ if (!IsRange) {
+ Value *Args[3] = {Ind->getTargetValue(),
+ Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+ Builder.getInt32(Index)};
+ Call = Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI), Args);
+ } else {
+ Value *Args[6] = {Ind->getTargetValue(),
+ Builder.CreateBitCast(DataVar, Builder.getInt8PtrTy()),
+ Builder.getInt32(Index),
+ Builder.getInt64(MemOPSizeRangeStart),
+ Builder.getInt64(MemOPSizeRangeLast),
+ Builder.getInt64(MemOPSizeLargeVal)};
+ Call =
+ Builder.CreateCall(getOrInsertValueProfilingCall(*M, *TLI, true), Args);
+ }
if (auto AK = TLI->getExtAttrForI32Param(false))
Call->addAttribute(3, AK);
Ind->replaceAllUsesWith(Call);
@@ -658,3 +700,24 @@ void InstrProfiling::emitInitialization() {
appendToGlobalCtors(*M, F, 0);
}
+
+void InstrProfiling::getMemOPSizeOptions() {
+ // Parse the value profile options.
+ MemOPSizeRangeStart = DefaultMemOPSizeRangeStart;
+ MemOPSizeRangeLast = DefaultMemOPSizeRangeLast;
+ if (!MemOPSizeRange.empty()) {
+ auto Pos = MemOPSizeRange.find(":");
+ if (Pos != std::string::npos) {
+ if (Pos > 0)
+ MemOPSizeRangeStart = std::stoi(MemOPSizeRange.substr(0, Pos));
+ if (Pos < MemOPSizeRange.size() - 1)
+ MemOPSizeRangeLast = std::stoi(MemOPSizeRange.substr(Pos + 1));
+ } else
+ MemOPSizeRangeLast = std::stoi(MemOPSizeRange);
+ }
+ assert(MemOPSizeRangeLast >= MemOPSizeRangeStart);
+
+ MemOPSizeLargeVal = MemOPSizeLarge;
+ if (MemOPSizeLargeVal == 0)
+ MemOPSizeLargeVal = INT64_MIN;
+}
diff --git a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
index d33961813e8..d9c25f2e981 100644
--- a/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
+++ b/llvm/lib/Transforms/Instrumentation/PGOInstrumentation.cpp
@@ -91,6 +91,7 @@ using namespace llvm;
STATISTIC(NumOfPGOInstrument, "Number of edges instrumented.");
STATISTIC(NumOfPGOSelectInsts, "Number of select instruction instrumented.");
+STATISTIC(NumOfPGOMemIntrinsics, "Number of mem intrinsics instrumented.");
STATISTIC(NumOfPGOEdge, "Number of edges.");
STATISTIC(NumOfPGOBB, "Number of basic-blocks.");
STATISTIC(NumOfPGOSplit, "Number of critical edge splits.");
@@ -168,6 +169,10 @@ static cl::opt<bool>
"display to only one function, use "
"filtering option -view-bfi-func-name."));
+// Command line option to enable/disable memop intrinsic calls..
+static cl::opt<bool> PGOInstrMemOP("pgo-instr-memop", cl::init(true),
+ cl::Hidden);
+
// Command line option to turn on CFG dot dump after profile annotation.
// Defined in Analysis/BlockFrequencyInfo.cpp: -pgo-view-counts
extern cl::opt<bool> PGOViewCounts;
@@ -235,6 +240,40 @@ struct SelectInstVisitor : public InstVisitor<SelectInstVisitor> {
unsigned getNumOfSelectInsts() const { return NSIs; }
};
+/// Instruction Visitor class to visit memory intrinsic calls.
+struct MemIntrinsicVisitor : public InstVisitor<MemIntrinsicVisitor> {
+ Function &F;
+ unsigned NMemIs = 0; // Number of memIntrinsics instrumented.
+ VisitMode Mode = VM_counting; // Visiting mode.
+ unsigned CurCtrId = 0; // Current counter index.
+ unsigned TotalNumCtrs = 0; // Total number of counters
+ GlobalVariable *FuncNameVar = nullptr;
+ uint64_t FuncHash = 0;
+ PGOUseFunc *UseFunc = nullptr;
+
+ MemIntrinsicVisitor(Function &Func) : F(Func) {}
+
+ void countMemIntrinsics(Function &Func) {
+ NMemIs = 0;
+ Mode = VM_counting;
+ visit(Func);
+ }
+ void instrumentMemIntrinsics(Function &Func, unsigned TotalNC,
+ GlobalVariable *FNV, uint64_t FHash) {
+ Mode = VM_instrument;
+ TotalNumCtrs = TotalNC;
+ FuncHash = FHash;
+ FuncNameVar = FNV;
+ visit(Func);
+ }
+
+ // Visit the IR stream and annotate all mem intrinsic call instructions.
+ void instrumentOneMemIntrinsic(MemIntrinsic &MI);
+ // Visit \p MI instruction and perform tasks according to visit mode.
+ void visitMemIntrinsic(MemIntrinsic &SI);
+ unsigned getNumOfMemIntrinsics() const { return NMemIs; }
+};
+
class PGOInstrumentationGenLegacyPass : public ModulePass {
public:
static char ID;
@@ -354,6 +393,7 @@ private:
public:
std::vector<std::vector<Instruction *>> ValueSites;
SelectInstVisitor SIVisitor;
+ MemIntrinsicVisitor MIVisitor;
std::string FuncName;
GlobalVariable *FuncNameVar;
// CFG hash value for this function.
@@ -384,11 +424,13 @@ public:
bool CreateGlobalVar = false, BranchProbabilityInfo *BPI = nullptr,
BlockFrequencyInfo *BFI = nullptr)
: F(Func), ComdatMembers(ComdatMembers), ValueSites(IPVK_Last + 1),
- SIVisitor(Func), FunctionHash(0), MST(F, BPI, BFI) {
+ SIVisitor(Func), MIVisitor(Func), FunctionHash(0), MST(F, BPI, BFI) {
// This should be done before CFG hash computation.
SIVisitor.countSelects(Func);
+ MIVisitor.countMemIntrinsics(Func);
NumOfPGOSelectInsts += SIVisitor.getNumOfSelectInsts();
+ NumOfPGOMemIntrinsics += MIVisitor.getNumOfMemIntrinsics();
ValueSites[IPVK_IndirectCallTarget] = findIndirectCallSites(Func);
FuncName = getPGOFuncName(F);
@@ -605,6 +647,10 @@ static void instrumentOneFunc(
Builder.getInt32(NumIndirectCallSites++)});
}
NumOfPGOICall += NumIndirectCallSites;
+
+ // Now instrument memop intrinsic calls.
+ FuncInfo.MIVisitor.instrumentMemIntrinsics(
+ F, NumCounters, FuncInfo.FuncNameVar, FuncInfo.FunctionHash);
}
// This class represents a CFG edge in profile use compilation.
@@ -1076,6 +1122,42 @@ void SelectInstVisitor::visitSelectInst(SelectInst &SI) {
llvm_unreachable("Unknown visiting mode");
}
+void MemIntrinsicVisitor::instrumentOneMemIntrinsic(MemIntrinsic &MI) {
+ Module *M = F.getParent();
+ IRBuilder<> Builder(&MI);
+ Type *Int64Ty = Builder.getInt64Ty();
+ Type *I8PtrTy = Builder.getInt8PtrTy();
+ Value *Length = MI.getLength();
+ assert(!dyn_cast<ConstantInt>(Length));
+ Builder.CreateCall(
+ Intrinsic::getDeclaration(M, Intrinsic::instrprof_value_profile),
+ {llvm::ConstantExpr::getBitCast(FuncNameVar, I8PtrTy),
+ Builder.getInt64(FuncHash), Builder.CreatePtrToInt(Length, Int64Ty),
+ Builder.getInt32(IPVK_MemOPSize), Builder.getInt32(CurCtrId)});
+ ++CurCtrId;
+}
+
+void MemIntrinsicVisitor::visitMemIntrinsic(MemIntrinsic &MI) {
+ if (!PGOInstrMemOP)
+ return;
+ Value *Length = MI.getLength();
+ // Not instrument constant length calls.
+ if (dyn_cast<ConstantInt>(Length))
+ return;
+
+ NMemIs++;
+ switch (Mode) {
+ case VM_counting:
+ return;
+ case VM_instrument:
+ instrumentOneMemIntrinsic(MI);
+ return;
+ case VM_annotate:
+ break;
+ }
+ llvm_unreachable("Unknown visiting mode");
+}
+
// Traverse all valuesites and annotate the instructions for all value kind.
void PGOUseFunc::annotateValueSites() {
if (DisableValueProfiling)
OpenPOWER on IntegriCloud