diff options
| -rw-r--r-- | llvm/include/llvm/IR/Instruction.h | 5 | ||||
| -rw-r--r-- | llvm/lib/Analysis/InlineCost.cpp | 9 | ||||
| -rw-r--r-- | llvm/lib/IR/Metadata.cpp | 25 | ||||
| -rw-r--r-- | llvm/lib/Transforms/IPO/SampleProfile.cpp | 13 | ||||
| -rw-r--r-- | llvm/test/Transforms/Inline/inline-hot-callsite.ll | 52 |
5 files changed, 103 insertions, 1 deletions
diff --git a/llvm/include/llvm/IR/Instruction.h b/llvm/include/llvm/IR/Instruction.h index efcd26cc30f..748bb946775 100644 --- a/llvm/include/llvm/IR/Instruction.h +++ b/llvm/include/llvm/IR/Instruction.h @@ -222,6 +222,11 @@ public: /// Returns false if no metadata or invalid metadata was found. bool extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal); + /// Retrieve total raw weight values of a branch. + /// Returns true on success with profile total weights filled in. + /// Returns false if no metadata was found. + bool extractProfTotalWeight(uint64_t &TotalVal); + /// Set the debug location information for this instruction. void setDebugLoc(DebugLoc Loc) { DbgLoc = std::move(Loc); } diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp index f6a9edae88d..dcb724abc02 100644 --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -633,11 +633,18 @@ void CallAnalyzer::updateThreshold(CallSite CS, Function &Callee) { Threshold = OptSizeThreshold; } + bool HotCallsite = false; + uint64_t TotalWeight; + if (CS.getInstruction()->extractProfTotalWeight(TotalWeight) && + PSI->isHotCount(TotalWeight)) + HotCallsite = true; + // Listen to the inlinehint attribute or profile based hotness information // when it would increase the threshold and the caller does not need to // minimize its size. bool InlineHint = Callee.hasFnAttribute(Attribute::InlineHint) || - PSI->isHotFunction(&Callee); + PSI->isHotFunction(&Callee) || + HotCallsite; if (InlineHint && HintThreshold > Threshold && !Caller->optForMinSize()) Threshold = HintThreshold; diff --git a/llvm/lib/IR/Metadata.cpp b/llvm/lib/IR/Metadata.cpp index ed39fbafcb0..50b4088b5e3 100644 --- a/llvm/lib/IR/Metadata.cpp +++ b/llvm/lib/IR/Metadata.cpp @@ -1312,6 +1312,31 @@ bool Instruction::extractProfMetadata(uint64_t &TrueVal, uint64_t &FalseVal) { return true; } +bool Instruction::extractProfTotalWeight(uint64_t &TotalVal) { + assert((getOpcode() == Instruction::Br || + getOpcode() == Instruction::Select || + getOpcode() == Instruction::Call) && + "Looking for branch weights on something besides branch"); + + TotalVal = 0; + auto *ProfileData = getMetadata(LLVMContext::MD_prof); + if (!ProfileData) + return false; + + auto *ProfDataName = dyn_cast<MDString>(ProfileData->getOperand(0)); + if (!ProfDataName || !ProfDataName->getString().equals("branch_weights")) + return false; + + TotalVal = 0; + for (int i = 1; i < ProfileData->getNumOperands(); i++) { + auto *V = mdconst::dyn_extract<ConstantInt>(ProfileData->getOperand(i)); + if (!V) + return false; + TotalVal += V->getValue().getZExtValue(); + } + return true; +} + void Instruction::clearMetadataHashEntries() { assert(hasMetadataHashEntry() && "Caller should check"); getContext().pImpl->InstructionMetadata.erase(this); diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp index af86df798e8..39de108edc0 100644 --- a/llvm/lib/Transforms/IPO/SampleProfile.cpp +++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp @@ -987,6 +987,19 @@ void SampleProfileLoader::propagateWeights(Function &F) { MDBuilder MDB(Ctx); for (auto &BI : F) { BasicBlock *BB = &BI; + + if (BlockWeights[BB]) { + for (auto &I : BB->getInstList()) { + if (CallInst *CI = dyn_cast<CallInst>(&I)) { + if (!dyn_cast<IntrinsicInst>(&I)) { + SmallVector<uint32_t, 1> Weights; + Weights.push_back(BlockWeights[BB]); + CI->setMetadata(LLVMContext::MD_prof, + MDB.createBranchWeights(Weights)); + } + } + } + } TerminatorInst *TI = BB->getTerminator(); if (TI->getNumSuccessors() == 1) continue; diff --git a/llvm/test/Transforms/Inline/inline-hot-callsite.ll b/llvm/test/Transforms/Inline/inline-hot-callsite.ll new file mode 100644 index 00000000000..36d9407142e --- /dev/null +++ b/llvm/test/Transforms/Inline/inline-hot-callsite.ll @@ -0,0 +1,52 @@ +; RUN: opt < %s -inline -inline-threshold=0 -inlinehint-threshold=100 -S | FileCheck %s + +; This tests that a hot callsite gets the (higher) inlinehint-threshold even without +; without inline hints and gets inlined because the cost is less than +; inlinehint-threshold. A cold callee with identical body does not get inlined because +; cost exceeds the inline-threshold + +define i32 @callee1(i32 %x) { + %x1 = add i32 %x, 1 + %x2 = add i32 %x1, 1 + %x3 = add i32 %x2, 1 + + ret i32 %x3 +} + +define i32 @callee2(i32 %x) { +; CHECK-LABEL: @callee2( + %x1 = add i32 %x, 1 + %x2 = add i32 %x1, 1 + %x3 = add i32 %x2, 1 + + ret i32 %x3 +} + +define i32 @caller2(i32 %y1) { +; CHECK-LABEL: @caller2( +; CHECK: call i32 @callee2 +; CHECK-NOT: call i32 @callee1 +; CHECK: ret i32 %x3.i + %y2 = call i32 @callee2(i32 %y1), !prof !22 + %y3 = call i32 @callee1(i32 %y2), !prof !21 + ret i32 %y3 +} + +!llvm.module.flags = !{!1} +!21 = !{!"branch_weights", i64 300} +!22 = !{!"branch_weights", i64 1} + +!1 = !{i32 1, !"ProfileSummary", !2} +!2 = !{!3, !4, !5, !6, !7, !8, !9, !10} +!3 = !{!"ProfileFormat", !"InstrProf"} +!4 = !{!"TotalCount", i64 10000} +!5 = !{!"MaxCount", i64 1000} +!6 = !{!"MaxInternalCount", i64 1} +!7 = !{!"MaxFunctionCount", i64 1000} +!8 = !{!"NumCounts", i64 3} +!9 = !{!"NumFunctions", i64 3} +!10 = !{!"DetailedSummary", !11} +!11 = !{!12, !13, !14} +!12 = !{i32 10000, i64 100, i32 1} +!13 = !{i32 999000, i64 100, i32 1} +!14 = !{i32 999999, i64 1, i32 2} |
