summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/InlineCost.cpp58
-rw-r--r--llvm/lib/Transforms/IPO/InlineSimple.cpp7
-rw-r--r--llvm/lib/Transforms/IPO/Inliner.cpp81
-rw-r--r--llvm/lib/Transforms/IPO/SampleProfile.cpp4
-rw-r--r--llvm/lib/Transforms/Utils/InlineFunction.cpp10
5 files changed, 98 insertions, 62 deletions
diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp
index dcb724abc02..54205bcc341 100644
--- a/llvm/lib/Analysis/InlineCost.cpp
+++ b/llvm/lib/Analysis/InlineCost.cpp
@@ -75,8 +75,8 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
/// The TargetTransformInfo available for this compilation.
const TargetTransformInfo &TTI;
- /// The cache of @llvm.assume intrinsics.
- AssumptionCacheTracker *ACT;
+ /// Getter for the cache of @llvm.assume intrinsics.
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache;
/// Profile summary information.
ProfileSummaryInfo *PSI;
@@ -203,20 +203,21 @@ class CallAnalyzer : public InstVisitor<CallAnalyzer, bool> {
bool visitUnreachableInst(UnreachableInst &I);
public:
- CallAnalyzer(const TargetTransformInfo &TTI, AssumptionCacheTracker *ACT,
+ CallAnalyzer(const TargetTransformInfo &TTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
ProfileSummaryInfo *PSI, Function &Callee, int Threshold,
CallSite CSArg)
- : TTI(TTI), ACT(ACT), PSI(PSI), F(Callee), CandidateCS(CSArg),
- Threshold(Threshold), Cost(0), IsCallerRecursive(false),
- IsRecursiveCall(false), ExposesReturnsTwice(false),
- HasDynamicAlloca(false), ContainsNoDuplicateCall(false),
- HasReturn(false), HasIndirectBr(false), HasFrameEscape(false),
- AllocatedSize(0), NumInstructions(0), NumVectorInstructions(0),
- FiftyPercentVectorBonus(0), TenPercentVectorBonus(0), VectorBonus(0),
- NumConstantArgs(0), NumConstantOffsetPtrArgs(0), NumAllocaArgs(0),
- NumConstantPtrCmps(0), NumConstantPtrDiffs(0),
- NumInstructionsSimplified(0), SROACostSavings(0),
- SROACostSavingsLost(0) {}
+ : TTI(TTI), GetAssumptionCache(GetAssumptionCache), PSI(PSI), F(Callee),
+ CandidateCS(CSArg), Threshold(Threshold), Cost(0),
+ IsCallerRecursive(false), IsRecursiveCall(false),
+ ExposesReturnsTwice(false), HasDynamicAlloca(false),
+ ContainsNoDuplicateCall(false), HasReturn(false), HasIndirectBr(false),
+ HasFrameEscape(false), AllocatedSize(0), NumInstructions(0),
+ NumVectorInstructions(0), FiftyPercentVectorBonus(0),
+ TenPercentVectorBonus(0), VectorBonus(0), NumConstantArgs(0),
+ NumConstantOffsetPtrArgs(0), NumAllocaArgs(0), NumConstantPtrCmps(0),
+ NumConstantPtrDiffs(0), NumInstructionsSimplified(0),
+ SROACostSavings(0), SROACostSavingsLost(0) {}
bool analyzeCall(CallSite CS);
@@ -957,8 +958,8 @@ bool CallAnalyzer::visitCallSite(CallSite CS) {
// during devirtualization and so we want to give it a hefty bonus for
// inlining, but cap that bonus in the event that inlining wouldn't pan
// out. Pretend to inline the function, with a custom threshold.
- CallAnalyzer CA(TTI, ACT, PSI, *F, InlineConstants::IndirectCallThreshold,
- CS);
+ CallAnalyzer CA(TTI, GetAssumptionCache, PSI, *F,
+ InlineConstants::IndirectCallThreshold, CS);
if (CA.analyzeCall(CS)) {
// We were able to inline the indirect call! Subtract the cost from the
// threshold to get the bonus we want to apply, but don't go below zero.
@@ -1312,8 +1313,7 @@ bool CallAnalyzer::analyzeCall(CallSite CS) {
// the ephemeral values multiple times (and they're completely determined by
// the callee, so this is purely duplicate work).
SmallPtrSet<const Value *, 32> EphValues;
- CodeMetrics::collectEphemeralValues(&F, &ACT->getAssumptionCache(F),
- EphValues);
+ CodeMetrics::collectEphemeralValues(&F, &GetAssumptionCache(F), EphValues);
// The worklist of live basic blocks in the callee *after* inlining. We avoid
// adding basic blocks of the callee which can be proven to be dead for this
@@ -1444,12 +1444,12 @@ static bool functionsHaveCompatibleAttributes(Function *Caller,
AttributeFuncs::areInlineCompatible(*Caller, *Callee);
}
-InlineCost llvm::getInlineCost(CallSite CS, int DefaultThreshold,
- TargetTransformInfo &CalleeTTI,
- AssumptionCacheTracker *ACT,
- ProfileSummaryInfo *PSI) {
+InlineCost llvm::getInlineCost(
+ CallSite CS, int DefaultThreshold, TargetTransformInfo &CalleeTTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
+ ProfileSummaryInfo *PSI) {
return getInlineCost(CS, CS.getCalledFunction(), DefaultThreshold, CalleeTTI,
- ACT, PSI);
+ GetAssumptionCache, PSI);
}
int llvm::computeThresholdFromOptLevels(unsigned OptLevel,
@@ -1465,11 +1465,11 @@ int llvm::computeThresholdFromOptLevels(unsigned OptLevel,
int llvm::getDefaultInlineThreshold() { return DefaultInlineThreshold; }
-InlineCost llvm::getInlineCost(CallSite CS, Function *Callee,
- int DefaultThreshold,
- TargetTransformInfo &CalleeTTI,
- AssumptionCacheTracker *ACT,
- ProfileSummaryInfo *PSI) {
+InlineCost llvm::getInlineCost(
+ CallSite CS, Function *Callee, int DefaultThreshold,
+ TargetTransformInfo &CalleeTTI,
+ std::function<AssumptionCache &(Function &)> &GetAssumptionCache,
+ ProfileSummaryInfo *PSI) {
// Cannot inline indirect calls.
if (!Callee)
@@ -1503,7 +1503,7 @@ InlineCost llvm::getInlineCost(CallSite CS, Function *Callee,
DEBUG(llvm::dbgs() << " Analyzing call of " << Callee->getName()
<< "...\n");
- CallAnalyzer CA(CalleeTTI, ACT, PSI, *Callee, DefaultThreshold, CS);
+ CallAnalyzer CA(CalleeTTI, GetAssumptionCache, PSI, *Callee, DefaultThreshold, CS);
bool ShouldInline = CA.analyzeCall(CS);
DEBUG(CA.dump());
diff --git a/llvm/lib/Transforms/IPO/InlineSimple.cpp b/llvm/lib/Transforms/IPO/InlineSimple.cpp
index 2aa650bd219..0dc29506cf3 100644
--- a/llvm/lib/Transforms/IPO/InlineSimple.cpp
+++ b/llvm/lib/Transforms/IPO/InlineSimple.cpp
@@ -61,7 +61,12 @@ public:
InlineCost getInlineCost(CallSite CS) override {
Function *Callee = CS.getCalledFunction();
TargetTransformInfo &TTI = TTIWP->getTTI(*Callee);
- return llvm::getInlineCost(CS, DefaultThreshold, TTI, ACT, PSI);
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&](
+ Function &F) -> AssumptionCache & {
+ return ACT->getAssumptionCache(F);
+ };
+ return llvm::getInlineCost(CS, DefaultThreshold, TTI, GetAssumptionCache,
+ PSI);
}
bool runOnSCC(CallGraphSCC &SCC) override;
diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp
index 79535ca4978..baecbb38959 100644
--- a/llvm/lib/Transforms/IPO/Inliner.cpp
+++ b/llvm/lib/Transforms/IPO/Inliner.cpp
@@ -75,19 +75,15 @@ InlinedArrayAllocasTy;
/// available from other functions inlined into the caller. If we are able to
/// inline this call site we attempt to reuse already available allocas or add
/// any new allocas to the set if not possible.
-static bool InlineCallIfPossible(Pass &P, CallSite CS, InlineFunctionInfo &IFI,
- InlinedArrayAllocasTy &InlinedArrayAllocas,
- int InlineHistory, bool InsertLifetime) {
+static bool
+InlineCallIfPossible(CallSite CS, InlineFunctionInfo &IFI,
+ InlinedArrayAllocasTy &InlinedArrayAllocas,
+ int InlineHistory, bool InsertLifetime,
+ std::function<AAResults &(Function &)> &AARGetter) {
Function *Callee = CS.getCalledFunction();
Function *Caller = CS.getCaller();
- // We need to manually construct BasicAA directly in order to disable
- // its use of other function analyses.
- BasicAAResult BAR(createLegacyPMBasicAAResult(P, *Callee));
-
- // Construct our own AA results for this function. We do this manually to
- // work around the limitations of the legacy pass manager.
- AAResults AAR(createLegacyPMAAResults(P, *Callee, BAR));
+ AAResults &AAR = AARGetter(*Callee);
// Try to inline the function. Get the list of static allocas that were
// inlined.
@@ -229,8 +225,15 @@ static void emitAnalysis(CallSite CS, const Twine &Msg) {
emitOptimizationRemarkAnalysis(Ctx, DEBUG_TYPE, *Caller, DLoc, Msg);
}
-bool Inliner::shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
- int &TotalSecondaryCost) {
+/// Return true if inlining of CS can block the caller from being
+/// inlined which is proved to be more beneficial. \p IC is the
+/// estimated inline cost associated with callsite \p CS.
+/// \p TotalAltCost will be set to the estimated cost of inlining the caller
+/// if \p CS is suppressed for inlining.
+static bool
+shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
+ int &TotalSecondaryCost,
+ std::function<InlineCost(CallSite CS)> &GetInlineCost) {
// For now we only handle local or inline functions.
if (!Caller->hasLocalLinkage() && !Caller->hasLinkOnceODRLinkage())
@@ -269,7 +272,7 @@ bool Inliner::shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
continue;
}
- InlineCost IC2 = getInlineCost(CS2);
+ InlineCost IC2 = GetInlineCost(CS2);
++NumCallerCallersAnalyzed;
if (!IC2) {
callerWillBeRemoved = false;
@@ -300,8 +303,9 @@ bool Inliner::shouldBeDeferred(Function *Caller, CallSite CS, InlineCost IC,
}
/// Return true if the inliner should attempt to inline at the given CallSite.
-bool Inliner::shouldInline(CallSite CS) {
- InlineCost IC = getInlineCost(CS);
+static bool shouldInline(CallSite CS,
+ std::function<InlineCost(CallSite CS)> GetInlineCost) {
+ InlineCost IC = GetInlineCost(CS);
if (IC.isAlways()) {
DEBUG(dbgs() << " Inlining: cost=always"
@@ -332,7 +336,7 @@ bool Inliner::shouldInline(CallSite CS) {
}
int TotalSecondaryCost = 0;
- if (shouldBeDeferred(Caller, CS, IC, TotalSecondaryCost)) {
+ if (shouldBeDeferred(Caller, CS, IC, TotalSecondaryCost, GetInlineCost)) {
DEBUG(dbgs() << " NOT Inlining: " << *CS.getInstruction()
<< " Cost = " << IC.getCost()
<< ", outer Cost = " << TotalSecondaryCost << '\n');
@@ -370,15 +374,17 @@ static bool InlineHistoryIncludes(Function *F, int InlineHistoryID,
bool Inliner::runOnSCC(CallGraphSCC &SCC) {
if (skipSCC(SCC))
return false;
+
return inlineCalls(SCC);
}
-bool Inliner::inlineCalls(CallGraphSCC &SCC) {
- CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
- ACT = &getAnalysis<AssumptionCacheTracker>();
- PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(CG.getModule());
- auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
-
+static bool
+inlineCallsImpl(CallGraphSCC &SCC, CallGraph &CG,
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache,
+ ProfileSummaryInfo *PSI, TargetLibraryInfo &TLI,
+ bool InsertLifetime,
+ std::function<InlineCost(CallSite CS)> GetInlineCost,
+ std::function<AAResults &(Function &)> AARGetter) {
SmallPtrSet<Function*, 8> SCCFunctions;
DEBUG(dbgs() << "Inliner visiting SCC:");
for (CallGraphNode *Node : SCC) {
@@ -437,7 +443,7 @@ bool Inliner::inlineCalls(CallGraphSCC &SCC) {
InlinedArrayAllocasTy InlinedArrayAllocas;
- InlineFunctionInfo InlineInfo(&CG, ACT);
+ InlineFunctionInfo InlineInfo(&CG, &GetAssumptionCache);
// Now that we have all of the call sites, loop over them and inline them if
// it looks profitable to do so.
@@ -486,7 +492,7 @@ bool Inliner::inlineCalls(CallGraphSCC &SCC) {
// If the policy determines that we should inline this function,
// try to do so.
- if (!shouldInline(CS)) {
+ if (!shouldInline(CS, GetInlineCost)) {
emitOptimizationRemarkMissed(CallerCtx, DEBUG_TYPE, *Caller, DLoc,
Twine(Callee->getName() +
" will not be inlined into " +
@@ -495,8 +501,8 @@ bool Inliner::inlineCalls(CallGraphSCC &SCC) {
}
// Attempt to inline the function.
- if (!InlineCallIfPossible(*this, CS, InlineInfo, InlinedArrayAllocas,
- InlineHistoryID, InsertLifetime)) {
+ if (!InlineCallIfPossible(CS, InlineInfo, InlinedArrayAllocas,
+ InlineHistoryID, InsertLifetime, AARGetter)) {
emitOptimizationRemarkMissed(CallerCtx, DEBUG_TYPE, *Caller, DLoc,
Twine(Callee->getName() +
" will not be inlined into " +
@@ -565,6 +571,29 @@ bool Inliner::inlineCalls(CallGraphSCC &SCC) {
return Changed;
}
+bool Inliner::inlineCalls(CallGraphSCC &SCC) {
+ CallGraph &CG = getAnalysis<CallGraphWrapperPass>().getCallGraph();
+ ACT = &getAnalysis<AssumptionCacheTracker>();
+ PSI = getAnalysis<ProfileSummaryInfoWrapperPass>().getPSI(CG.getModule());
+ auto &TLI = getAnalysis<TargetLibraryInfoWrapperPass>().getTLI();
+ // We compute dedicated AA results for each function in the SCC as needed. We
+ // use a lambda referencing external objects so that they live long enough to
+ // be queried, but we re-use them each time.
+ Optional<BasicAAResult> BAR;
+ Optional<AAResults> AAR;
+ auto AARGetter = [&](Function &F) -> AAResults & {
+ BAR.emplace(createLegacyPMBasicAAResult(*this, F));
+ AAR.emplace(createLegacyPMAAResults(*this, F, *BAR));
+ return *AAR;
+ };
+ auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & {
+ return ACT->getAssumptionCache(F);
+ };
+ return inlineCallsImpl(SCC, CG, GetAssumptionCache, PSI, TLI, InsertLifetime,
+ [this](CallSite CS) { return getInlineCost(CS); },
+ AARGetter);
+}
+
/// Remove now-dead linkonce functions at the end of
/// processing to avoid breaking the SCC traversal.
bool Inliner::doFinalization(CallGraph &CG) {
diff --git a/llvm/lib/Transforms/IPO/SampleProfile.cpp b/llvm/lib/Transforms/IPO/SampleProfile.cpp
index 39de108edc0..6a74a938691 100644
--- a/llvm/lib/Transforms/IPO/SampleProfile.cpp
+++ b/llvm/lib/Transforms/IPO/SampleProfile.cpp
@@ -627,6 +627,8 @@ SampleProfileLoader::findFunctionSamples(const Instruction &Inst) const {
bool SampleProfileLoader::inlineHotFunctions(Function &F) {
bool Changed = false;
LLVMContext &Ctx = F.getContext();
+ std::function<AssumptionCache &(Function &)> GetAssumptionCache = [&](
+ Function &F) -> AssumptionCache & { return ACT->getAssumptionCache(F); };
while (true) {
bool LocalChanged = false;
SmallVector<CallInst *, 10> CIS;
@@ -638,7 +640,7 @@ bool SampleProfileLoader::inlineHotFunctions(Function &F) {
}
}
for (auto CI : CIS) {
- InlineFunctionInfo IFI(nullptr, ACT);
+ InlineFunctionInfo IFI(nullptr, ACT ? &GetAssumptionCache : nullptr);
Function *CalledFunction = CI->getCalledFunction();
DebugLoc DLoc = CI->getDebugLoc();
uint64_t NumSamples = findCalleeFunctionSamples(*CI)->getTotalSamples();
diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp
index 1fbb19d2b8a..e101412bf79 100644
--- a/llvm/lib/Transforms/Utils/InlineFunction.cpp
+++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp
@@ -1078,7 +1078,7 @@ static void AddAlignmentAssumptions(CallSite CS, InlineFunctionInfo &IFI) {
// caller, then don't bother inserting the assumption.
Value *Arg = CS.getArgument(I->getArgNo());
if (getKnownAlignment(Arg, DL, CS.getInstruction(),
- &IFI.ACT->getAssumptionCache(*CS.getCaller()),
+ &(*IFI.GetAssumptionCache)(*CS.getCaller()),
&DT) >= Align)
continue;
@@ -1199,7 +1199,7 @@ static Value *HandleByValArgument(Value *Arg, Instruction *TheCall,
// If the pointer is already known to be sufficiently aligned, or if we can
// round it up to a larger alignment, then we don't need a temporary.
if (getOrEnforceKnownAlignment(Arg, ByValAlignment, DL, TheCall,
- &IFI.ACT->getAssumptionCache(*Caller)) >=
+ &(*IFI.GetAssumptionCache)(*Caller)) >=
ByValAlignment)
return Arg;
@@ -1604,8 +1604,8 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
// FIXME: We could register any cloned assumptions instead of clearing the
// whole function's cache.
- if (IFI.ACT)
- IFI.ACT->getAssumptionCache(*Caller).clear();
+ if (IFI.GetAssumptionCache)
+ (*IFI.GetAssumptionCache)(*Caller).clear();
}
// If there are any alloca instructions in the block that used to be the entry
@@ -2125,7 +2125,7 @@ bool llvm::InlineFunction(CallSite CS, InlineFunctionInfo &IFI,
if (PHI) {
auto &DL = Caller->getParent()->getDataLayout();
if (Value *V = SimplifyInstruction(PHI, DL, nullptr, nullptr,
- &IFI.ACT->getAssumptionCache(*Caller))) {
+ &(*IFI.GetAssumptionCache)(*Caller))) {
PHI->replaceAllUsesWith(V);
PHI->eraseFromParent();
}
OpenPOWER on IntegriCloud