diff options
20 files changed, 135 insertions, 29 deletions
diff --git a/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h b/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h new file mode 100644 index 00000000000..724ff72f3b5 --- /dev/null +++ b/llvm/include/llvm/Transforms/IPO/ArgumentPromotion.h @@ -0,0 +1,31 @@ +//===- ArgumentPromotion.h - Promote by-reference arguments -----*- C++ -*-===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// + +#ifndef LLVM_TRANSFORMS_IPO_ARGUMENTPROMOTION_H +#define LLVM_TRANSFORMS_IPO_ARGUMENTPROMOTION_H + +#include "llvm/Analysis/CGSCCPassManager.h" +#include "llvm/Analysis/LazyCallGraph.h" + +namespace llvm { + +/// Argument promotion pass. +/// +/// This pass walks the functions in each SCC and for each one tries to +/// transform it and all of its callers to replace indirect arguments with +/// direct (by-value) arguments. +class ArgumentPromotionPass : public PassInfoMixin<ArgumentPromotionPass> { +public: + PreservedAnalyses run(LazyCallGraph::SCC &C, CGSCCAnalysisManager &AM, + LazyCallGraph &CG, CGSCCUpdateResult &UR); +}; + +} + +#endif diff --git a/llvm/lib/Passes/PassBuilder.cpp b/llvm/lib/Passes/PassBuilder.cpp index 520c9df3028..23d72a4d2ae 100644 --- a/llvm/lib/Passes/PassBuilder.cpp +++ b/llvm/lib/Passes/PassBuilder.cpp @@ -61,6 +61,7 @@ #include "llvm/Target/TargetMachine.h" #include "llvm/Transforms/GCOVProfiler.h" #include "llvm/Transforms/IPO/AlwaysInliner.h" +#include "llvm/Transforms/IPO/ArgumentPromotion.h" #include "llvm/Transforms/IPO/ConstantMerge.h" #include "llvm/Transforms/IPO/CrossDSOCFI.h" #include "llvm/Transforms/IPO/DeadArgumentElimination.h" diff --git a/llvm/lib/Passes/PassRegistry.def b/llvm/lib/Passes/PassRegistry.def index 5b2e63e8501..9234677ec00 100644 --- a/llvm/lib/Passes/PassRegistry.def +++ b/llvm/lib/Passes/PassRegistry.def @@ -85,6 +85,7 @@ CGSCC_ANALYSIS("fam-proxy", FunctionAnalysisManagerCGSCCProxy()) #ifndef CGSCC_PASS #define CGSCC_PASS(NAME, CREATE_PASS) #endif +CGSCC_PASS("argpromotion", ArgumentPromotionPass()) CGSCC_PASS("invalidate<all>", InvalidateAllAnalysesPass()) CGSCC_PASS("function-attrs", PostOrderFunctionAttrsPass()) CGSCC_PASS("inline", InlinerPass()) diff --git a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp index 2e2f131f7ed..c1f2c866549 100644 --- a/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp +++ b/llvm/lib/Transforms/IPO/ArgumentPromotion.cpp @@ -29,7 +29,9 @@ // //===----------------------------------------------------------------------===// +#include "llvm/Transforms/IPO/ArgumentPromotion.h" #include "llvm/ADT/DepthFirstIterator.h" +#include "llvm/ADT/Optional.h" #include "llvm/ADT/Statistic.h" #include "llvm/ADT/StringExtras.h" #include "llvm/Analysis/AliasAnalysis.h" @@ -37,6 +39,7 @@ #include "llvm/Analysis/BasicAliasAnalysis.h" #include "llvm/Analysis/CallGraph.h" #include "llvm/Analysis/CallGraphSCCPass.h" +#include "llvm/Analysis/LazyCallGraph.h" #include "llvm/Analysis/Loads.h" #include "llvm/Analysis/TargetLibraryInfo.h" #include "llvm/IR/CFG.h" @@ -67,9 +70,11 @@ typedef std::vector<uint64_t> IndicesVector; /// DoPromotion - This method actually performs the promotion of the specified /// arguments, and returns the new function. At this point, we know that it's /// safe to do so. -static CallGraphNode * +static Function * doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote, - SmallPtrSetImpl<Argument *> &ByValArgsToTransform, CallGraph &CG) { + SmallPtrSetImpl<Argument *> &ByValArgsToTransform, + Optional<function_ref<void(CallSite OldCS, CallSite NewCS)>> + ReplaceCallSite) { // Start by computing a new prototype for the function, which is the same as // the old function, but has modified arguments. @@ -207,9 +212,6 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote, F->getParent()->getFunctionList().insert(F->getIterator(), NF); NF->takeName(F); - // Get a new callgraph node for NF. - CallGraphNode *NF_CGN = CG.getOrInsertFunction(NF); - // Loop over all of the callers of the function, transforming the call sites // to pass in the loaded pointers. // @@ -334,8 +336,8 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote, AttributesVec.clear(); // Update the callgraph to know that the callsite has been transformed. - CallGraphNode *CalleeNode = CG[Call->getParent()->getParent()]; - CalleeNode->replaceCallEdge(CS, CallSite(New), NF_CGN); + if (ReplaceCallSite) + (*ReplaceCallSite)(CS, CallSite(New)); if (!Call->use_empty()) { Call->replaceAllUsesWith(New); @@ -463,18 +465,7 @@ doPromotion(Function *F, SmallPtrSetImpl<Argument *> &ArgsToPromote, std::advance(I2, ArgIndices.size()); } - NF_CGN->stealCalledFunctionsFrom(CG[F]); - - // Now that the old function is dead, delete it. If there is a dangling - // reference to the CallgraphNode, just leave the dead function around for - // someone else to nuke. - CallGraphNode *CGN = CG[F]; - if (CGN->getNumReferences() == 0) - delete CG.removeFunctionFromModule(CGN); - else - F->setLinkage(Function::ExternalLinkage); - - return NF_CGN; + return NF; } /// AllCallersPassInValidPointerForArgument - Return true if we can prove that @@ -818,14 +809,13 @@ static bool canPaddingBeAccessed(Argument *arg) { /// example, all callers are direct). If safe to promote some arguments, it /// calls the DoPromotion method. /// -static CallGraphNode * -promoteArguments(CallGraphNode *CGN, CallGraph &CG, - function_ref<AAResults &(Function &F)> AARGetter, - unsigned MaxElements) { - Function *F = CGN->getFunction(); - +static Function * +promoteArguments(Function *F, function_ref<AAResults &(Function &F)> AARGetter, + unsigned MaxElements, + Optional<function_ref<void(CallSite OldCS, CallSite NewCS)>> + ReplaceCallSite) { // Make sure that it is local to this module. - if (!F || !F->hasLocalLinkage()) + if (!F->hasLocalLinkage()) return nullptr; // Don't promote arguments for variadic functions. Adding, removing, or @@ -950,7 +940,52 @@ promoteArguments(CallGraphNode *CGN, CallGraph &CG, if (ArgsToPromote.empty() && ByValArgsToTransform.empty()) return nullptr; - return doPromotion(F, ArgsToPromote, ByValArgsToTransform, CG); + return doPromotion(F, ArgsToPromote, ByValArgsToTransform, ReplaceCallSite); +} + +PreservedAnalyses ArgumentPromotionPass::run(LazyCallGraph::SCC &C, + CGSCCAnalysisManager &AM, + LazyCallGraph &CG, + CGSCCUpdateResult &UR) { + bool Changed = false, LocalChange; + + // Iterate until we stop promoting from this SCC. + do { + LocalChange = false; + + for (LazyCallGraph::Node &N : C) { + Function &OldF = N.getFunction(); + + FunctionAnalysisManager &FAM = + AM.getResult<FunctionAnalysisManagerCGSCCProxy>(C, CG).getManager(); + // FIXME: This lambda must only be used with this function. We should + // skip the lambda and just get the AA results directly. + auto AARGetter = [&](Function &F) -> AAResults & { + assert(&F == &OldF && "Called with an unexpected function!"); + return FAM.getResult<AAManager>(F); + }; + + Function *NewF = promoteArguments(&OldF, AARGetter, 3u, None); + if (!NewF) + continue; + LocalChange = true; + + // Directly substitute the functions in the call graph. Note that this + // requires the old function to be completely dead and completely + // replaced by the new function. It does no call graph updates, it merely + // swaps out the particular function mapped to a particular node in the + // graph. + C.getOuterRefSCC().replaceNodeFunction(N, *NewF); + OldF.eraseFromParent(); + } + + Changed |= LocalChange; + } while (LocalChange); + + if (!Changed) + return PreservedAnalyses::all(); + + return PreservedAnalyses::none(); } namespace { @@ -1010,9 +1045,31 @@ bool ArgPromotion::runOnSCC(CallGraphSCC &SCC) { LocalChange = false; // Attempt to promote arguments from all functions in this SCC. for (CallGraphNode *OldNode : SCC) { - if (CallGraphNode *NewNode = - promoteArguments(OldNode, CG, AARGetter, MaxElements)) { + Function *OldF = OldNode->getFunction(); + if (!OldF) + continue; + + auto ReplaceCallSite = [&](CallSite OldCS, CallSite NewCS) { + Function *Caller = OldCS.getInstruction()->getParent()->getParent(); + CallGraphNode *NewCalleeNode = + CG.getOrInsertFunction(NewCS.getCalledFunction()); + CallGraphNode *CallerNode = CG[Caller]; + CallerNode->replaceCallEdge(OldCS, NewCS, NewCalleeNode); + }; + + if (Function *NewF = promoteArguments(OldF, AARGetter, MaxElements, + {ReplaceCallSite})) { LocalChange = true; + + // Update the call graph for the newly promoted function. + CallGraphNode *NewNode = CG.getOrInsertFunction(NewF); + NewNode->stealCalledFunctionsFrom(OldNode); + if (OldNode->getNumReferences() == 0) + delete CG.removeFunctionFromModule(OldNode); + else + OldF->setLinkage(Function::ExternalLinkage); + + // And updat ethe SCC we're iterating as well. SCC.ReplaceNode(OldNode, NewNode); } } diff --git a/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll b/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll index 694b89b3ad0..b0bab7784ed 100644 --- a/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll +++ b/llvm/test/Transforms/ArgumentPromotion/aggregate-promote.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s %T = type { i32, i32, i32, i32 } @G = constant %T { i32 0, i32 0, i32 17, i32 25 } diff --git a/llvm/test/Transforms/ArgumentPromotion/attrs.ll b/llvm/test/Transforms/ArgumentPromotion/attrs.ll index a85a088291a..29cef50fe80 100644 --- a/llvm/test/Transforms/ArgumentPromotion/attrs.ll +++ b/llvm/test/Transforms/ArgumentPromotion/attrs.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s %struct.ss = type { i32, i64 } diff --git a/llvm/test/Transforms/ArgumentPromotion/byval-2.ll b/llvm/test/Transforms/ArgumentPromotion/byval-2.ll index 6c0288f5f98..3e1fee8badd 100644 --- a/llvm/test/Transforms/ArgumentPromotion/byval-2.ll +++ b/llvm/test/Transforms/ArgumentPromotion/byval-2.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; Arg promotion eliminates the struct argument. ; FIXME: Should it eliminate the i32* argument? diff --git a/llvm/test/Transforms/ArgumentPromotion/byval.ll b/llvm/test/Transforms/ArgumentPromotion/byval.ll index b091b09a359..58475fc8960 100644 --- a/llvm/test/Transforms/ArgumentPromotion/byval.ll +++ b/llvm/test/Transforms/ArgumentPromotion/byval.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" diff --git a/llvm/test/Transforms/ArgumentPromotion/chained.ll b/llvm/test/Transforms/ArgumentPromotion/chained.ll index 53c2f69fbb1..028c6c426e5 100644 --- a/llvm/test/Transforms/ArgumentPromotion/chained.ll +++ b/llvm/test/Transforms/ArgumentPromotion/chained.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s @G1 = constant i32 0 @G2 = constant i32* @G1 diff --git a/llvm/test/Transforms/ArgumentPromotion/control-flow.ll b/llvm/test/Transforms/ArgumentPromotion/control-flow.ll index d26da10baa5..c3fe0c00e87 100644 --- a/llvm/test/Transforms/ArgumentPromotion/control-flow.ll +++ b/llvm/test/Transforms/ArgumentPromotion/control-flow.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; Don't promote around control flow. define internal i32 @callee(i1 %C, i32* %P) { diff --git a/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll b/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll index 7413f46a860..b75a32ddb33 100644 --- a/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll +++ b/llvm/test/Transforms/ArgumentPromotion/control-flow2.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; CHECK: load i32, i32* %A target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" diff --git a/llvm/test/Transforms/ArgumentPromotion/crash.ll b/llvm/test/Transforms/ArgumentPromotion/crash.ll index dfc83da1232..d46a48101e7 100644 --- a/llvm/test/Transforms/ArgumentPromotion/crash.ll +++ b/llvm/test/Transforms/ArgumentPromotion/crash.ll @@ -1,4 +1,5 @@ ; RUN: opt -S < %s -inline -argpromotion | FileCheck %s +; RUN: opt -S < %s -passes=inline,argpromotion | FileCheck %s %S = type { %S* } diff --git a/llvm/test/Transforms/ArgumentPromotion/dbg.ll b/llvm/test/Transforms/ArgumentPromotion/dbg.ll index 3d353db105f..7dac3ad8415 100644 --- a/llvm/test/Transforms/ArgumentPromotion/dbg.ll +++ b/llvm/test/Transforms/ArgumentPromotion/dbg.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s declare void @sink(i32) diff --git a/llvm/test/Transforms/ArgumentPromotion/fp80.ll b/llvm/test/Transforms/ArgumentPromotion/fp80.ll index 84ef603de82..bd780fa21ae 100644 --- a/llvm/test/Transforms/ArgumentPromotion/fp80.ll +++ b/llvm/test/Transforms/ArgumentPromotion/fp80.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-unknown-linux-gnu" diff --git a/llvm/test/Transforms/ArgumentPromotion/inalloca.ll b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll index 5bf57c8ff46..7ea3b4e4277 100644 --- a/llvm/test/Transforms/ArgumentPromotion/inalloca.ll +++ b/llvm/test/Transforms/ArgumentPromotion/inalloca.ll @@ -1,4 +1,5 @@ ; RUN: opt %s -argpromotion -sroa -S | FileCheck %s +; RUN: opt %s -passes='argpromotion,function(sroa)' -S | FileCheck %s target datalayout = "E-p:64:64:64-a0:0:8-f32:32:32-f64:64:64-i1:8:8-i8:8:8-i16:16:16-i32:32:32-i64:32:64-v64:64:64-v128:128:128" diff --git a/llvm/test/Transforms/ArgumentPromotion/pr27568.ll b/llvm/test/Transforms/ArgumentPromotion/pr27568.ll index 648317aee0d..1496780748d 100644 --- a/llvm/test/Transforms/ArgumentPromotion/pr27568.ll +++ b/llvm/test/Transforms/ArgumentPromotion/pr27568.ll @@ -1,4 +1,5 @@ ; RUN: opt -S -argpromotion < %s | FileCheck %s +; RUN: opt -S -passes=argpromotion < %s | FileCheck %s target triple = "x86_64-pc-windows-msvc" define internal void @callee(i8*) { diff --git a/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll b/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll index 35dd487da7b..3a3aa44b2a9 100644 --- a/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll +++ b/llvm/test/Transforms/ArgumentPromotion/reserve-tbaa.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; PR17906 ; When we promote two arguments in a single function with different types, diff --git a/llvm/test/Transforms/ArgumentPromotion/sret.ll b/llvm/test/Transforms/ArgumentPromotion/sret.ll index 8e5521f48d1..55fc036f177 100644 --- a/llvm/test/Transforms/ArgumentPromotion/sret.ll +++ b/llvm/test/Transforms/ArgumentPromotion/sret.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s target datalayout = "e-m:w-i64:64-f80:128-n8:16:32:64-S128" target triple = "x86_64-pc-windows-msvc" diff --git a/llvm/test/Transforms/ArgumentPromotion/tail.ll b/llvm/test/Transforms/ArgumentPromotion/tail.ll index 2ea387cd264..93de60afe91 100644 --- a/llvm/test/Transforms/ArgumentPromotion/tail.ll +++ b/llvm/test/Transforms/ArgumentPromotion/tail.ll @@ -1,4 +1,5 @@ ; RUN: opt %s -argpromotion -S -o - | FileCheck %s +; RUN: opt %s -passes=argpromotion -S -o - | FileCheck %s ; PR14710 target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128" diff --git a/llvm/test/Transforms/ArgumentPromotion/variadic.ll b/llvm/test/Transforms/ArgumentPromotion/variadic.ll index 0e03882d3b2..034f853883f 100644 --- a/llvm/test/Transforms/ArgumentPromotion/variadic.ll +++ b/llvm/test/Transforms/ArgumentPromotion/variadic.ll @@ -1,4 +1,5 @@ ; RUN: opt < %s -argpromotion -S | FileCheck %s +; RUN: opt < %s -passes=argpromotion -S | FileCheck %s ; Unused arguments from variadic functions cannot be eliminated as that changes ; their classiciation according to the SysV amd64 ABI. Clang and other frontends |
