summaryrefslogtreecommitdiffstats
path: root/llvm/lib/Transforms
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/Transforms')
-rw-r--r--llvm/lib/Transforms/IPO/PassManagerBuilder.cpp11
-rw-r--r--llvm/lib/Transforms/Scalar/CMakeLists.txt1
-rw-r--r--llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp447
-rw-r--r--llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp19
-rw-r--r--llvm/lib/Transforms/Scalar/Scalar.cpp5
-rw-r--r--llvm/lib/Transforms/Utils/CMakeLists.txt1
-rw-r--r--llvm/lib/Transforms/Utils/LoopUnroll.cpp22
-rw-r--r--llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp774
8 files changed, 1260 insertions, 20 deletions
diff --git a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
index a5357387f7d..52fdc737a75 100644
--- a/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
+++ b/llvm/lib/Transforms/IPO/PassManagerBuilder.cpp
@@ -96,6 +96,10 @@ static cl::opt<bool> EnableLoopInterchange(
"enable-loopinterchange", cl::init(false), cl::Hidden,
cl::desc("Enable the new, experimental LoopInterchange Pass"));
+static cl::opt<bool> EnableUnrollAndJam("enable-unroll-and-jam",
+ cl::init(false), cl::Hidden,
+ cl::desc("Enable Unroll And Jam Pass"));
+
static cl::opt<bool>
EnablePrepareForThinLTO("prepare-for-thinlto", cl::init(false), cl::Hidden,
cl::desc("Enable preparation for ThinLTO."));
@@ -669,6 +673,13 @@ void PassManagerBuilder::populateModulePassManager(
addInstructionCombiningPass(MPM);
if (!DisableUnrollLoops) {
+ if (EnableUnrollAndJam) {
+ // Unroll and Jam. We do this before unroll but need to be in a separate
+ // loop pass manager in order for the outer loop to be processed by
+ // unroll and jam before the inner loop is unrolled.
+ MPM.add(createLoopUnrollAndJamPass(OptLevel));
+ }
+
MPM.add(createLoopUnrollPass(OptLevel)); // Unroll small loops
// LoopUnroll may generate some redundency to cleanup.
diff --git a/llvm/lib/Transforms/Scalar/CMakeLists.txt b/llvm/lib/Transforms/Scalar/CMakeLists.txt
index 22adf9a5137..fce37d4bffb 100644
--- a/llvm/lib/Transforms/Scalar/CMakeLists.txt
+++ b/llvm/lib/Transforms/Scalar/CMakeLists.txt
@@ -39,6 +39,7 @@ add_llvm_library(LLVMScalarOpts
LoopSimplifyCFG.cpp
LoopStrengthReduce.cpp
LoopUnrollPass.cpp
+ LoopUnrollAndJamPass.cpp
LoopUnswitch.cpp
LoopVersioningLICM.cpp
LowerAtomic.cpp
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
new file mode 100644
index 00000000000..86c99aed441
--- /dev/null
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollAndJamPass.cpp
@@ -0,0 +1,447 @@
+//===- LoopUnrollAndJam.cpp - Loop unroll and jam pass --------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This pass implements an unroll and jam pass. Most of the work is done by
+// Utils/UnrollLoopAndJam.cpp.
+//===----------------------------------------------------------------------===//
+
+#include "llvm/Transforms/Scalar/LoopUnrollAndJamPass.h"
+#include "llvm/ADT/None.h"
+#include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/StringRef.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/CodeMetrics.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/LoopInfo.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/TargetTransformInfo.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/CFG.h"
+#include "llvm/IR/Constant.h"
+#include "llvm/IR/Constants.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/Function.h"
+#include "llvm/IR/Instruction.h"
+#include "llvm/IR/Instructions.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/Metadata.h"
+#include "llvm/IR/PassManager.h"
+#include "llvm/Pass.h"
+#include "llvm/Support/Casting.h"
+#include "llvm/Support/CommandLine.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/ErrorHandling.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Scalar.h"
+#include "llvm/Transforms/Scalar/LoopPassManager.h"
+#include "llvm/Transforms/Utils.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/UnrollLoop.h"
+#include <algorithm>
+#include <cassert>
+#include <cstdint>
+#include <string>
+
+using namespace llvm;
+
+#define DEBUG_TYPE "loop-unroll-and-jam"
+
+static cl::opt<bool>
+ AllowUnrollAndJam("allow-unroll-and-jam", cl::Hidden,
+ cl::desc("Allows loops to be unroll-and-jammed."));
+
+static cl::opt<unsigned> UnrollAndJamCount(
+ "unroll-and-jam-count", cl::Hidden,
+ cl::desc("Use this unroll count for all loops including those with "
+ "unroll_and_jam_count pragma values, for testing purposes"));
+
+static cl::opt<unsigned> UnrollAndJamThreshold(
+ "unroll-and-jam-threshold", cl::init(60), cl::Hidden,
+ cl::desc("Threshold to use for inner loop when doing unroll and jam."));
+
+static cl::opt<unsigned> PragmaUnrollAndJamThreshold(
+ "pragma-unroll-and-jam-threshold", cl::init(1024), cl::Hidden,
+ cl::desc("Unrolled size limit for loops with an unroll_and_jam(full) or "
+ "unroll_count pragma."));
+
+// Returns the loop hint metadata node with the given name (for example,
+// "llvm.loop.unroll.count"). If no such metadata node exists, then nullptr is
+// returned.
+static MDNode *GetUnrollMetadataForLoop(const Loop *L, StringRef Name) {
+ if (MDNode *LoopID = L->getLoopID())
+ return GetUnrollMetadata(LoopID, Name);
+ return nullptr;
+}
+
+// Returns true if the loop has any metadata starting with Prefix. For example a
+// Prefix of "llvm.loop.unroll." returns true if we have any unroll metadata.
+static bool HasAnyUnrollPragma(const Loop *L, StringRef Prefix) {
+ if (MDNode *LoopID = L->getLoopID()) {
+ // First operand should refer to the loop id itself.
+ assert(LoopID->getNumOperands() > 0 && "requires at least one operand");
+ assert(LoopID->getOperand(0) == LoopID && "invalid loop id");
+
+ for (unsigned i = 1, e = LoopID->getNumOperands(); i < e; ++i) {
+ MDNode *MD = dyn_cast<MDNode>(LoopID->getOperand(i));
+ if (!MD)
+ continue;
+
+ MDString *S = dyn_cast<MDString>(MD->getOperand(0));
+ if (!S)
+ continue;
+
+ if (S->getString().startswith(Prefix))
+ return true;
+ }
+ }
+ return false;
+}
+
+// Returns true if the loop has an unroll_and_jam(enable) pragma.
+static bool HasUnrollAndJamEnablePragma(const Loop *L) {
+ return GetUnrollMetadataForLoop(L, "llvm.loop.unroll_and_jam.enable");
+}
+
+// Returns true if the loop has an unroll_and_jam(disable) pragma.
+static bool HasUnrollAndJamDisablePragma(const Loop *L) {
+ return GetUnrollMetadataForLoop(L, "llvm.loop.unroll_and_jam.disable");
+}
+
+// If loop has an unroll_and_jam_count pragma return the (necessarily
+// positive) value from the pragma. Otherwise return 0.
+static unsigned UnrollAndJamCountPragmaValue(const Loop *L) {
+ MDNode *MD = GetUnrollMetadataForLoop(L, "llvm.loop.unroll_and_jam.count");
+ if (MD) {
+ assert(MD->getNumOperands() == 2 &&
+ "Unroll count hint metadata should have two operands.");
+ unsigned Count =
+ mdconst::extract<ConstantInt>(MD->getOperand(1))->getZExtValue();
+ assert(Count >= 1 && "Unroll count must be positive.");
+ return Count;
+ }
+ return 0;
+}
+
+// Returns loop size estimation for unrolled loop.
+static uint64_t
+getUnrollAndJammedLoopSize(unsigned LoopSize,
+ TargetTransformInfo::UnrollingPreferences &UP) {
+ assert(LoopSize >= UP.BEInsns && "LoopSize should not be less than BEInsns!");
+ return static_cast<uint64_t>(LoopSize - UP.BEInsns) * UP.Count + UP.BEInsns;
+}
+
+// Calculates unroll and jam count and writes it to UP.Count. Returns true if
+// unroll count was set explicitly.
+static bool computeUnrollAndJamCount(
+ Loop *L, Loop *SubLoop, const TargetTransformInfo &TTI, DominatorTree &DT,
+ LoopInfo *LI, ScalarEvolution &SE,
+ const SmallPtrSetImpl<const Value *> &EphValues,
+ OptimizationRemarkEmitter *ORE, unsigned OuterTripCount,
+ unsigned OuterTripMultiple, unsigned OuterLoopSize, unsigned InnerTripCount,
+ unsigned InnerLoopSize, TargetTransformInfo::UnrollingPreferences &UP) {
+ // Check for explicit Count from the "unroll-and-jam-count" option.
+ bool UserUnrollCount = UnrollAndJamCount.getNumOccurrences() > 0;
+ if (UserUnrollCount) {
+ UP.Count = UnrollAndJamCount;
+ UP.Force = true;
+ if (UP.AllowRemainder &&
+ getUnrollAndJammedLoopSize(OuterLoopSize, UP) < UP.Threshold &&
+ getUnrollAndJammedLoopSize(InnerLoopSize, UP) <
+ UP.UnrollAndJamInnerLoopThreshold)
+ return true;
+ }
+
+ // Check for unroll_and_jam pragmas
+ unsigned PragmaCount = UnrollAndJamCountPragmaValue(L);
+ if (PragmaCount > 0) {
+ UP.Count = PragmaCount;
+ UP.Runtime = true;
+ UP.Force = true;
+ if ((UP.AllowRemainder || (OuterTripMultiple % PragmaCount == 0)) &&
+ getUnrollAndJammedLoopSize(OuterLoopSize, UP) < UP.Threshold &&
+ getUnrollAndJammedLoopSize(InnerLoopSize, UP) <
+ UP.UnrollAndJamInnerLoopThreshold)
+ return true;
+ }
+
+ // Use computeUnrollCount from the loop unroller to get a sensible count
+ // for the unrolling the outer loop. This uses UP.Threshold /
+ // UP.PartialThreshold / UP.MaxCount to come up with sensible loop values.
+ // We have already checked that the loop has no unroll.* pragmas.
+ unsigned MaxTripCount = 0;
+ bool UseUpperBound = false;
+ bool ExplicitUnroll = computeUnrollCount(
+ L, TTI, DT, LI, SE, EphValues, ORE, OuterTripCount, MaxTripCount,
+ OuterTripMultiple, OuterLoopSize, UP, UseUpperBound);
+ if (ExplicitUnroll || UseUpperBound) {
+ // If the user explicitly set the loop as unrolled, dont UnJ it. Leave it
+ // for the unroller instead.
+ UP.Count = 0;
+ return false;
+ }
+
+ bool PragmaEnableUnroll = HasUnrollAndJamEnablePragma(L);
+ ExplicitUnroll = PragmaCount > 0 || PragmaEnableUnroll || UserUnrollCount;
+
+ // If the loop has an unrolling pragma, we want to be more aggressive with
+ // unrolling limits.
+ if (ExplicitUnroll && OuterTripCount != 0)
+ UP.UnrollAndJamInnerLoopThreshold = PragmaUnrollAndJamThreshold;
+
+ if (!UP.AllowRemainder && getUnrollAndJammedLoopSize(InnerLoopSize, UP) >=
+ UP.UnrollAndJamInnerLoopThreshold) {
+ UP.Count = 0;
+ return false;
+ }
+
+ // If the inner loop count is known and small, leave the entire loop nest to
+ // be the unroller
+ if (!ExplicitUnroll && InnerTripCount &&
+ InnerLoopSize * InnerTripCount < UP.Threshold) {
+ UP.Count = 0;
+ return false;
+ }
+
+ // We have a sensible limit for the outer loop, now adjust it for the inner
+ // loop and UP.UnrollAndJamInnerLoopThreshold.
+ while (UP.Count != 0 && UP.AllowRemainder &&
+ getUnrollAndJammedLoopSize(InnerLoopSize, UP) >=
+ UP.UnrollAndJamInnerLoopThreshold)
+ UP.Count--;
+
+ if (!ExplicitUnroll) {
+ // Check for situations where UnJ is likely to be unprofitable. Including
+ // subloops with more than 1 block.
+ if (SubLoop->getBlocks().size() != 1) {
+ UP.Count = 0;
+ return false;
+ }
+
+ // Limit to loops where there is something to gain from unrolling and
+ // jamming the loop. In this case, look for loads that are invariant in the
+ // outer loop and can become shared.
+ unsigned NumInvariant = 0;
+ for (BasicBlock *BB : SubLoop->getBlocks()) {
+ for (Instruction &I : *BB) {
+ if (auto *Ld = dyn_cast<LoadInst>(&I)) {
+ Value *V = Ld->getPointerOperand();
+ const SCEV *LSCEV = SE.getSCEVAtScope(V, L);
+ if (SE.isLoopInvariant(LSCEV, L))
+ NumInvariant++;
+ }
+ }
+ }
+ if (NumInvariant == 0) {
+ UP.Count = 0;
+ return false;
+ }
+ }
+
+ return ExplicitUnroll;
+}
+
+static LoopUnrollResult
+tryToUnrollAndJamLoop(Loop *L, DominatorTree &DT, LoopInfo *LI,
+ ScalarEvolution &SE, const TargetTransformInfo &TTI,
+ AssumptionCache &AC, DependenceInfo &DI,
+ OptimizationRemarkEmitter &ORE, int OptLevel) {
+ // Quick checks of the correct loop form
+ if (!L->isLoopSimplifyForm() || L->getSubLoops().size() != 1)
+ return LoopUnrollResult::Unmodified;
+ Loop *SubLoop = L->getSubLoops()[0];
+ if (!SubLoop->isLoopSimplifyForm())
+ return LoopUnrollResult::Unmodified;
+
+ BasicBlock *Latch = L->getLoopLatch();
+ BasicBlock *Exit = L->getExitingBlock();
+ BasicBlock *SubLoopLatch = SubLoop->getLoopLatch();
+ BasicBlock *SubLoopExit = SubLoop->getExitingBlock();
+
+ if (Latch != Exit || SubLoopLatch != SubLoopExit)
+ return LoopUnrollResult::Unmodified;
+
+ TargetTransformInfo::UnrollingPreferences UP = gatherUnrollingPreferences(
+ L, SE, TTI, OptLevel, None, None, None, None, None, None);
+ if (AllowUnrollAndJam.getNumOccurrences() > 0)
+ UP.UnrollAndJam = AllowUnrollAndJam;
+ if (UnrollAndJamThreshold.getNumOccurrences() > 0)
+ UP.UnrollAndJamInnerLoopThreshold = UnrollAndJamThreshold;
+ // Exit early if unrolling is disabled.
+ if (!UP.UnrollAndJam || UP.UnrollAndJamInnerLoopThreshold == 0)
+ return LoopUnrollResult::Unmodified;
+
+ LLVM_DEBUG(dbgs() << "Loop Unroll and Jam: F["
+ << L->getHeader()->getParent()->getName() << "] Loop %"
+ << L->getHeader()->getName() << "\n");
+
+ // A loop with any unroll pragma (enabling/disabling/count/etc) is left for
+ // the unroller, so long as it does not explicitly have unroll_and_jam
+ // metadata. This means #pragma nounroll will disable unroll and jam as well
+ // as unrolling
+ if (HasUnrollAndJamDisablePragma(L) ||
+ (HasAnyUnrollPragma(L, "llvm.loop.unroll.") &&
+ !HasAnyUnrollPragma(L, "llvm.loop.unroll_and_jam."))) {
+ LLVM_DEBUG(dbgs() << " Disabled due to pragma.\n");
+ return LoopUnrollResult::Unmodified;
+ }
+
+ if (!isSafeToUnrollAndJam(L, SE, DT, DI)) {
+ LLVM_DEBUG(dbgs() << " Disabled due to not being safe.\n");
+ return LoopUnrollResult::Unmodified;
+ }
+
+ // Approximate the loop size and collect useful info
+ unsigned NumInlineCandidates;
+ bool NotDuplicatable;
+ bool Convergent;
+ SmallPtrSet<const Value *, 32> EphValues;
+ CodeMetrics::collectEphemeralValues(L, &AC, EphValues);
+ unsigned InnerLoopSize =
+ ApproximateLoopSize(SubLoop, NumInlineCandidates, NotDuplicatable,
+ Convergent, TTI, EphValues, UP.BEInsns);
+ unsigned OuterLoopSize =
+ ApproximateLoopSize(L, NumInlineCandidates, NotDuplicatable, Convergent,
+ TTI, EphValues, UP.BEInsns);
+ LLVM_DEBUG(dbgs() << " Outer Loop Size: " << OuterLoopSize << "\n");
+ LLVM_DEBUG(dbgs() << " Inner Loop Size: " << InnerLoopSize << "\n");
+ if (NotDuplicatable) {
+ LLVM_DEBUG(dbgs() << " Not unrolling loop which contains non-duplicatable "
+ "instructions.\n");
+ return LoopUnrollResult::Unmodified;
+ }
+ if (NumInlineCandidates != 0) {
+ LLVM_DEBUG(dbgs() << " Not unrolling loop with inlinable calls.\n");
+ return LoopUnrollResult::Unmodified;
+ }
+ if (Convergent) {
+ LLVM_DEBUG(
+ dbgs() << " Not unrolling loop with convergent instructions.\n");
+ return LoopUnrollResult::Unmodified;
+ }
+
+ // Find trip count and trip multiple
+ unsigned OuterTripCount = SE.getSmallConstantTripCount(L, Latch);
+ unsigned OuterTripMultiple = SE.getSmallConstantTripMultiple(L, Latch);
+ unsigned InnerTripCount = SE.getSmallConstantTripCount(SubLoop, SubLoopLatch);
+
+ // Decide if, and by how much, to unroll
+ bool IsCountSetExplicitly = computeUnrollAndJamCount(
+ L, SubLoop, TTI, DT, LI, SE, EphValues, &ORE, OuterTripCount,
+ OuterTripMultiple, OuterLoopSize, InnerTripCount, InnerLoopSize, UP);
+ if (UP.Count <= 1)
+ return LoopUnrollResult::Unmodified;
+ // Unroll factor (Count) must be less or equal to TripCount.
+ if (OuterTripCount && UP.Count > OuterTripCount)
+ UP.Count = OuterTripCount;
+
+ LoopUnrollResult UnrollResult =
+ UnrollAndJamLoop(L, UP.Count, OuterTripCount, OuterTripMultiple,
+ UP.UnrollRemainder, LI, &SE, &DT, &AC, &ORE);
+
+ // If loop has an unroll count pragma or unrolled by explicitly set count
+ // mark loop as unrolled to prevent unrolling beyond that requested.
+ if (UnrollResult != LoopUnrollResult::FullyUnrolled && IsCountSetExplicitly)
+ L->setLoopAlreadyUnrolled();
+
+ return UnrollResult;
+}
+
+namespace {
+
+class LoopUnrollAndJam : public LoopPass {
+public:
+ static char ID; // Pass ID, replacement for typeid
+ unsigned OptLevel;
+
+ LoopUnrollAndJam(int OptLevel = 2) : LoopPass(ID), OptLevel(OptLevel) {
+ initializeLoopUnrollAndJamPass(*PassRegistry::getPassRegistry());
+ }
+
+ bool runOnLoop(Loop *L, LPPassManager &LPM) override {
+ if (skipLoop(L))
+ return false;
+
+ Function &F = *L->getHeader()->getParent();
+
+ auto &DT = getAnalysis<DominatorTreeWrapperPass>().getDomTree();
+ LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
+ ScalarEvolution &SE = getAnalysis<ScalarEvolutionWrapperPass>().getSE();
+ const TargetTransformInfo &TTI =
+ getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
+ auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
+ auto &DI = getAnalysis<DependenceAnalysisWrapperPass>().getDI();
+ // For the old PM, we can't use OptimizationRemarkEmitter as an analysis
+ // pass. Function analyses need to be preserved across loop transformations
+ // but ORE cannot be preserved (see comment before the pass definition).
+ OptimizationRemarkEmitter ORE(&F);
+
+ LoopUnrollResult Result =
+ tryToUnrollAndJamLoop(L, DT, LI, SE, TTI, AC, DI, ORE, OptLevel);
+
+ if (Result == LoopUnrollResult::FullyUnrolled)
+ LPM.markLoopAsDeleted(*L);
+
+ return Result != LoopUnrollResult::Unmodified;
+ }
+
+ /// This transformation requires natural loop information & requires that
+ /// loop preheaders be inserted into the CFG...
+ void getAnalysisUsage(AnalysisUsage &AU) const override {
+ AU.addRequired<AssumptionCacheTracker>();
+ AU.addRequired<TargetTransformInfoWrapperPass>();
+ AU.addRequired<DependenceAnalysisWrapperPass>();
+ getLoopAnalysisUsage(AU);
+ }
+};
+
+} // end anonymous namespace
+
+char LoopUnrollAndJam::ID = 0;
+
+INITIALIZE_PASS_BEGIN(LoopUnrollAndJam, "loop-unroll-and-jam",
+ "Unroll and Jam loops", false, false)
+INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker)
+INITIALIZE_PASS_DEPENDENCY(LoopPass)
+INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass)
+INITIALIZE_PASS_DEPENDENCY(DependenceAnalysisWrapperPass)
+INITIALIZE_PASS_END(LoopUnrollAndJam, "loop-unroll-and-jam",
+ "Unroll and Jam loops", false, false)
+
+Pass *llvm::createLoopUnrollAndJamPass(int OptLevel) {
+ return new LoopUnrollAndJam(OptLevel);
+}
+
+PreservedAnalyses LoopUnrollAndJamPass::run(Loop &L, LoopAnalysisManager &AM,
+ LoopStandardAnalysisResults &AR,
+ LPMUpdater &) {
+ const auto &FAM =
+ AM.getResult<FunctionAnalysisManagerLoopProxy>(L, AR).getManager();
+ Function *F = L.getHeader()->getParent();
+
+ auto *ORE = FAM.getCachedResult<OptimizationRemarkEmitterAnalysis>(*F);
+ // FIXME: This should probably be optional rather than required.
+ if (!ORE)
+ report_fatal_error(
+ "LoopUnrollAndJamPass: OptimizationRemarkEmitterAnalysis not cached at "
+ "a higher level");
+
+ DependenceInfo DI(F, &AR.AA, &AR.SE, &AR.LI);
+
+ LoopUnrollResult Result = tryToUnrollAndJamLoop(
+ &L, AR.DT, &AR.LI, AR.SE, AR.TTI, AR.AC, DI, *ORE, OptLevel);
+
+ if (Result == LoopUnrollResult::Unmodified)
+ return PreservedAnalyses::all();
+
+ return getLoopPassPreservedAnalyses();
+}
diff --git a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
index 797af471ed2..634215c9770 100644
--- a/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
+++ b/llvm/lib/Transforms/Scalar/LoopUnrollPass.cpp
@@ -165,7 +165,7 @@ static const unsigned NoThreshold = std::numeric_limits<unsigned>::max();
/// Gather the various unrolling parameters based on the defaults, compiler
/// flags, TTI overrides and user specified parameters.
-static TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
+TargetTransformInfo::UnrollingPreferences llvm::gatherUnrollingPreferences(
Loop *L, ScalarEvolution &SE, const TargetTransformInfo &TTI, int OptLevel,
Optional<unsigned> UserThreshold, Optional<unsigned> UserCount,
Optional<bool> UserAllowPartial, Optional<bool> UserRuntime,
@@ -192,6 +192,8 @@ static TargetTransformInfo::UnrollingPreferences gatherUnrollingPreferences(
UP.Force = false;
UP.UpperBound = false;
UP.AllowPeeling = true;
+ UP.UnrollAndJam = false;
+ UP.UnrollAndJamInnerLoopThreshold = 60;
// Override with any target specific settings
TTI.getUnrollingPreferences(L, SE, UP);
@@ -615,11 +617,10 @@ static Optional<EstimatedUnrollCost> analyzeLoopUnrollCost(
}
/// ApproximateLoopSize - Approximate the size of the loop.
-static unsigned
-ApproximateLoopSize(const Loop *L, unsigned &NumCalls, bool &NotDuplicatable,
- bool &Convergent, const TargetTransformInfo &TTI,
- const SmallPtrSetImpl<const Value *> &EphValues,
- unsigned BEInsns) {
+unsigned llvm::ApproximateLoopSize(
+ const Loop *L, unsigned &NumCalls, bool &NotDuplicatable, bool &Convergent,
+ const TargetTransformInfo &TTI,
+ const SmallPtrSetImpl<const Value *> &EphValues, unsigned BEInsns) {
CodeMetrics Metrics;
for (BasicBlock *BB : L->blocks())
Metrics.analyzeBasicBlock(BB, TTI, EphValues);
@@ -712,7 +713,7 @@ static uint64_t getUnrolledLoopSize(
// Returns true if unroll count was set explicitly.
// Calculates unroll count and writes it to UP.Count.
-static bool computeUnrollCount(
+bool llvm::computeUnrollCount(
Loop *L, const TargetTransformInfo &TTI, DominatorTree &DT, LoopInfo *LI,
ScalarEvolution &SE, const SmallPtrSetImpl<const Value *> &EphValues,
OptimizationRemarkEmitter *ORE, unsigned &TripCount, unsigned MaxTripCount,
@@ -753,8 +754,8 @@ static bool computeUnrollCount(
if (ExplicitUnroll && TripCount != 0) {
// If the loop has an unrolling pragma, we want to be more aggressive with
- // unrolling limits. Set thresholds to at least the PragmaThreshold value
- // which is larger than the default limits.
+ // unrolling limits. Set thresholds to at least the PragmaUnrollThreshold
+ // value which is larger than the default limits.
UP.Threshold = std::max<unsigned>(UP.Threshold, PragmaUnrollThreshold);
UP.PartialThreshold =
std::max<unsigned>(UP.PartialThreshold, PragmaUnrollThreshold);
diff --git a/llvm/lib/Transforms/Scalar/Scalar.cpp b/llvm/lib/Transforms/Scalar/Scalar.cpp
index 93c58fc3bb4..526487d3477 100644
--- a/llvm/lib/Transforms/Scalar/Scalar.cpp
+++ b/llvm/lib/Transforms/Scalar/Scalar.cpp
@@ -70,6 +70,7 @@ void llvm::initializeScalarOpts(PassRegistry &Registry) {
initializeLoopStrengthReducePass(Registry);
initializeLoopRerollPass(Registry);
initializeLoopUnrollPass(Registry);
+ initializeLoopUnrollAndJamPass(Registry);
initializeLoopUnswitchPass(Registry);
initializeLoopVersioningLICMPass(Registry);
initializeLoopIdiomRecognizeLegacyPassPass(Registry);
@@ -185,6 +186,10 @@ void LLVMAddLoopUnrollPass(LLVMPassManagerRef PM) {
unwrap(PM)->add(createLoopUnrollPass());
}
+void LLVMAddLoopUnrollAndJamPass(LLVMPassManagerRef PM) {
+ unwrap(PM)->add(createLoopUnrollAndJamPass());
+}
+
void LLVMAddLoopUnswitchPass(LLVMPassManagerRef PM) {
unwrap(PM)->add(createLoopUnswitchPass());
}
diff --git a/llvm/lib/Transforms/Utils/CMakeLists.txt b/llvm/lib/Transforms/Utils/CMakeLists.txt
index 48171efa0bd..c87b74f739f 100644
--- a/llvm/lib/Transforms/Utils/CMakeLists.txt
+++ b/llvm/lib/Transforms/Utils/CMakeLists.txt
@@ -28,6 +28,7 @@ add_llvm_library(LLVMTransformUtils
LoopRotationUtils.cpp
LoopSimplify.cpp
LoopUnroll.cpp
+ LoopUnrollAndJam.cpp
LoopUnrollPeel.cpp
LoopUnrollRuntime.cpp
LoopUtils.cpp
diff --git a/llvm/lib/Transforms/Utils/LoopUnroll.cpp b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
index c8cca70392d..04b8c1417e0 100644
--- a/llvm/lib/Transforms/Utils/LoopUnroll.cpp
+++ b/llvm/lib/Transforms/Utils/LoopUnroll.cpp
@@ -63,8 +63,7 @@ UnrollVerifyDomtree("unroll-verify-domtree", cl::Hidden,
/// Convert the instruction operands from referencing the current values into
/// those specified by VMap.
-static inline void remapInstruction(Instruction *I,
- ValueToValueMapTy &VMap) {
+void llvm::remapInstruction(Instruction *I, ValueToValueMapTy &VMap) {
for (unsigned op = 0, E = I->getNumOperands(); op != E; ++op) {
Value *Op = I->getOperand(op);
@@ -98,9 +97,9 @@ static inline void remapInstruction(Instruction *I,
/// Folds a basic block into its predecessor if it only has one predecessor, and
/// that predecessor only has one successor.
/// The LoopInfo Analysis that is passed will be kept consistent.
-static BasicBlock *
-foldBlockIntoPredecessor(BasicBlock *BB, LoopInfo *LI, ScalarEvolution *SE,
- DominatorTree *DT) {
+BasicBlock *llvm::foldBlockIntoPredecessor(BasicBlock *BB, LoopInfo *LI,
+ ScalarEvolution *SE,
+ DominatorTree *DT) {
// Merge basic blocks into their predecessor if there is only one distinct
// pred, and if there is only one distinct successor of the predecessor, and
// if there are no PHI nodes.
@@ -110,7 +109,8 @@ foldBlockIntoPredecessor(BasicBlock *BB, LoopInfo *LI, ScalarEvolution *SE,
if (OnlyPred->getTerminator()->getNumSuccessors() != 1)
return nullptr;
- LLVM_DEBUG(dbgs() << "Merging: " << *BB << "into: " << *OnlyPred);
+ LLVM_DEBUG(dbgs() << "Merging: " << BB->getName() << " into "
+ << OnlyPred->getName() << "\n");
// Resolve any PHI nodes at the start of the block. They are all
// guaranteed to have exactly one entry if they exist, unless there are
@@ -255,9 +255,9 @@ static bool isEpilogProfitable(Loop *L) {
/// Perform some cleanup and simplifications on loops after unrolling. It is
/// useful to simplify the IV's in the new loop, as well as do a quick
/// simplify/dce pass of the instructions.
-static void simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
- ScalarEvolution *SE, DominatorTree *DT,
- AssumptionCache *AC) {
+void llvm::simplifyLoopAfterUnroll(Loop *L, bool SimplifyIVs, LoopInfo *LI,
+ ScalarEvolution *SE, DominatorTree *DT,
+ AssumptionCache *AC) {
// Simplify any new induction variables in the partially unrolled loop.
if (SE && SimplifyIVs) {
SmallVector<WeakTrackingVH, 16> DeadInsts;
@@ -473,8 +473,8 @@ LoopUnrollResult llvm::UnrollLoop(
if (Force)
RuntimeTripCount = false;
else {
- LLVM_DEBUG(dbgs() << "Wont unroll; remainder loop could not be generated"
- "when assuming runtime trip count\n");
+ LLVM_DEBUG(dbgs() << "Won't unroll; remainder loop could not be "
+ "generated when assuming runtime trip count\n");
return LoopUnrollResult::Unmodified;
}
}
diff --git a/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
new file mode 100644
index 00000000000..b9ad8b03ed4
--- /dev/null
+++ b/llvm/lib/Transforms/Utils/LoopUnrollAndJam.cpp
@@ -0,0 +1,774 @@
+//===-- LoopUnrollAndJam.cpp - Loop unrolling utilities -------------------===//
+//
+// The LLVM Compiler Infrastructure
+//
+// This file is distributed under the University of Illinois Open Source
+// License. See LICENSE.TXT for details.
+//
+//===----------------------------------------------------------------------===//
+//
+// This file implements loop unroll and jam as a routine, much like
+// LoopUnroll.cpp implements loop unroll.
+//
+//===----------------------------------------------------------------------===//
+
+#include "llvm/ADT/SmallPtrSet.h"
+#include "llvm/ADT/Statistic.h"
+#include "llvm/Analysis/AssumptionCache.h"
+#include "llvm/Analysis/DependenceAnalysis.h"
+#include "llvm/Analysis/InstructionSimplify.h"
+#include "llvm/Analysis/LoopAnalysisManager.h"
+#include "llvm/Analysis/LoopIterator.h"
+#include "llvm/Analysis/LoopPass.h"
+#include "llvm/Analysis/OptimizationRemarkEmitter.h"
+#include "llvm/Analysis/ScalarEvolution.h"
+#include "llvm/Analysis/ScalarEvolutionExpander.h"
+#include "llvm/Analysis/Utils/Local.h"
+#include "llvm/IR/BasicBlock.h"
+#include "llvm/IR/DataLayout.h"
+#include "llvm/IR/DebugInfoMetadata.h"
+#include "llvm/IR/Dominators.h"
+#include "llvm/IR/IntrinsicInst.h"
+#include "llvm/IR/LLVMContext.h"
+#include "llvm/Support/Debug.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Transforms/Utils/BasicBlockUtils.h"
+#include "llvm/Transforms/Utils/Cloning.h"
+#include "llvm/Transforms/Utils/LoopSimplify.h"
+#include "llvm/Transforms/Utils/LoopUtils.h"
+#include "llvm/Transforms/Utils/SimplifyIndVar.h"
+#include "llvm/Transforms/Utils/UnrollLoop.h"
+using namespace llvm;
+
+#define DEBUG_TYPE "loop-unroll-and-jam"
+
+STATISTIC(NumUnrolledAndJammed, "Number of loops unroll and jammed");
+STATISTIC(NumCompletelyUnrolledAndJammed, "Number of loops unroll and jammed");
+
+static bool containsBB(std::vector<BasicBlock *> &V, BasicBlock *BB) {
+ return std::find(V.begin(), V.end(), BB) != V.end();
+}
+
+// Partition blocks in an outer/inner loop pair into blocks before and after
+// the loop
+static bool partitionOuterLoopBlocks(Loop *L, Loop *SubLoop,
+ std::vector<BasicBlock *> &ForeBlocks,
+ std::vector<BasicBlock *> &SubLoopBlocks,
+ std::vector<BasicBlock *> &AftBlocks,
+ DominatorTree *DT) {
+ BasicBlock *SubLoopLatch = SubLoop->getLoopLatch();
+ SubLoopBlocks = SubLoop->getBlocks();
+
+ for (BasicBlock *BB : L->blocks()) {
+ if (!SubLoop->contains(BB)) {
+ if (DT->dominates(SubLoopLatch, BB))
+ AftBlocks.push_back(BB);
+ else
+ ForeBlocks.push_back(BB);
+ }
+ }
+
+ // Check that all blocks in ForeBlocks together dominate the subloop
+ // TODO: This might ideally be done better with a dominator/postdominators.
+ BasicBlock *SubLoopPreHeader = SubLoop->getLoopPreheader();
+ for (BasicBlock *BB : ForeBlocks) {
+ if (BB == SubLoopPreHeader)
+ continue;
+ TerminatorInst *TI = BB->getTerminator();
+ for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
+ if (!containsBB(ForeBlocks, TI->getSuccessor(i)))
+ return false;
+ }
+
+ return true;
+}
+
+// Move the phi operands of Header from Latch out of AftBlocks to InsertLoc.
+static void
+moveHeaderPhiOperandsToForeBlocks(BasicBlock *Header, BasicBlock *Latch,
+ Instruction *InsertLoc,
+ std::vector<BasicBlock *> &AftBlocks) {
+ // We need to ensure we move the instructions in the correct order,
+ // starting with the earliest required instruction and moving forward.
+ std::vector<Instruction *> Worklist;
+ std::vector<Instruction *> Visited;
+ for (auto &Phi : Header->phis()) {
+ Value *V = Phi.getIncomingValueForBlock(Latch);
+ if (Instruction *I = dyn_cast<Instruction>(V))
+ Worklist.push_back(I);
+ }
+
+ while (!Worklist.empty()) {
+ Instruction *I = Worklist.back();
+ Worklist.pop_back();
+ if (!containsBB(AftBlocks, I->getParent()))
+ continue;
+
+ Visited.push_back(I);
+ for (auto &U : I->operands())
+ if (Instruction *II = dyn_cast<Instruction>(U))
+ Worklist.push_back(II);
+ }
+
+ // Move all instructions in program order to before the InsertLoc
+ BasicBlock *InsertLocBB = InsertLoc->getParent();
+ for (Instruction *I : reverse(Visited)) {
+ if (I->getParent() != InsertLocBB)
+ I->moveBefore(InsertLoc);
+ }
+}
+
+/*
+ This method performs Unroll and Jam. For a simple loop like:
+ for (i = ..)
+ Fore(i)
+ for (j = ..)
+ SubLoop(i, j)
+ Aft(i)
+
+ Instead of doing normal inner or outer unrolling, we do:
+ for (i = .., i+=2)
+ Fore(i)
+ Fore(i+1)
+ for (j = ..)
+ SubLoop(i, j)
+ SubLoop(i+1, j)
+ Aft(i)
+ Aft(i+1)
+
+ So the outer loop is essetially unrolled and then the inner loops are fused
+ ("jammed") together into a single loop. This can increase speed when there
+ are loads in SubLoop that are invariant to i, as they become shared between
+ the now jammed inner loops.
+
+ We do this by spliting the blocks in the loop into Fore, Subloop and Aft.
+ Fore blocks are those before the inner loop, Aft are those after. Normal
+ Unroll code is used to copy each of these sets of blocks and the results are
+ combined together into the final form above.
+
+ isSafeToUnrollAndJam should be used prior to calling this to make sure the
+ unrolling will be valid. Checking profitablility is also advisable.
+*/
+LoopUnrollResult
+llvm::UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
+ unsigned TripMultiple, bool UnrollRemainder,
+ LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
+ AssumptionCache *AC, OptimizationRemarkEmitter *ORE) {
+
+ // When we enter here we should have already checked that it is safe
+ BasicBlock *Header = L->getHeader();
+ assert(L->getSubLoops().size() == 1);
+ Loop *SubLoop = *L->begin();
+
+ // Don't enter the unroll code if there is nothing to do.
+ if (TripCount == 0 && Count < 2) {
+ LLVM_DEBUG(dbgs() << "Won't unroll; almost nothing to do\n");
+ return LoopUnrollResult::Unmodified;
+ }
+
+ assert(Count > 0);
+ assert(TripMultiple > 0);
+ assert(TripCount == 0 || TripCount % TripMultiple == 0);
+
+ // Are we eliminating the loop control altogether?
+ bool CompletelyUnroll = (Count == TripCount);
+
+ // We use the runtime remainder in cases where we don't know trip multiple
+ if (TripMultiple == 1 || TripMultiple % Count != 0) {
+ if (!UnrollRuntimeLoopRemainder(L, Count, /*AllowExpensiveTripCount*/ false,
+ /*UseEpilogRemainder*/ true,
+ UnrollRemainder, LI, SE, DT, AC, true)) {
+ LLVM_DEBUG(dbgs() << "Won't unroll-and-jam; remainder loop could not be "
+ "generated when assuming runtime trip count\n");
+ return LoopUnrollResult::Unmodified;
+ }
+ }
+
+ // Notify ScalarEvolution that the loop will be substantially changed,
+ // if not outright eliminated.
+ if (SE) {
+ SE->forgetLoop(L);
+ SE->forgetLoop(SubLoop);
+ }
+
+ using namespace ore;
+ // Report the unrolling decision.
+ if (CompletelyUnroll) {
+ LLVM_DEBUG(dbgs() << "COMPLETELY UNROLL AND JAMMING loop %"
+ << Header->getName() << " with trip count " << TripCount
+ << "!\n");
+ ORE->emit(OptimizationRemark(DEBUG_TYPE, "FullyUnrolled", L->getStartLoc(),
+ L->getHeader())
+ << "completely unroll and jammed loop with "
+ << NV("UnrollCount", TripCount) << " iterations");
+ } else {
+ auto DiagBuilder = [&]() {
+ OptimizationRemark Diag(DEBUG_TYPE, "PartialUnrolled", L->getStartLoc(),
+ L->getHeader());
+ return Diag << "unroll and jammed loop by a factor of "
+ << NV("UnrollCount", Count);
+ };
+
+ LLVM_DEBUG(dbgs() << "UNROLL AND JAMMING loop %" << Header->getName()
+ << " by " << Count);
+ if (TripMultiple != 1) {
+ LLVM_DEBUG(dbgs() << " with " << TripMultiple << " trips per branch");
+ ORE->emit([&]() {
+ return DiagBuilder() << " with " << NV("TripMultiple", TripMultiple)
+ << " trips per branch";
+ });
+ } else {
+ LLVM_DEBUG(dbgs() << " with run-time trip count");
+ ORE->emit([&]() { return DiagBuilder() << " with run-time trip count"; });
+ }
+ LLVM_DEBUG(dbgs() << "!\n");
+ }
+
+ BasicBlock *Preheader = L->getLoopPreheader();
+ BasicBlock *LatchBlock = L->getLoopLatch();
+ BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
+ assert(Preheader && LatchBlock && Header);
+ assert(BI && !BI->isUnconditional());
+ bool ContinueOnTrue = L->contains(BI->getSuccessor(0));
+ BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue);
+ bool SubLoopContinueOnTrue = SubLoop->contains(
+ SubLoop->getLoopLatch()->getTerminator()->getSuccessor(0));
+
+ // Partition blocks in an outer/inner loop pair into blocks before and after
+ // the loop
+ std::vector<BasicBlock *> SubLoopBlocks;
+ std::vector<BasicBlock *> ForeBlocks;
+ std::vector<BasicBlock *> AftBlocks;
+ partitionOuterLoopBlocks(L, SubLoop, ForeBlocks, SubLoopBlocks, AftBlocks,
+ DT);
+
+ // We keep track of the entering/first and exiting/last block of each of
+ // Fore/SubLoop/Aft in each iteration. This helps make the stapling up of
+ // blocks easier.
+ std::vector<BasicBlock *> ForeBlocksFirst;
+ std::vector<BasicBlock *> ForeBlocksLast;
+ std::vector<BasicBlock *> SubLoopBlocksFirst;
+ std::vector<BasicBlock *> SubLoopBlocksLast;
+ std::vector<BasicBlock *> AftBlocksFirst;
+ std::vector<BasicBlock *> AftBlocksLast;
+ ForeBlocksFirst.push_back(Header);
+ ForeBlocksLast.push_back(SubLoop->getLoopPreheader());
+ SubLoopBlocksFirst.push_back(SubLoop->getHeader());
+ SubLoopBlocksLast.push_back(SubLoop->getExitingBlock());
+ AftBlocksFirst.push_back(SubLoop->getExitBlock());
+ AftBlocksLast.push_back(L->getExitingBlock());
+ // Maps Blocks[0] -> Blocks[It]
+ ValueToValueMapTy LastValueMap;
+
+ // Move any instructions from fore phi operands from AftBlocks into Fore.
+ moveHeaderPhiOperandsToForeBlocks(
+ Header, LatchBlock, SubLoop->getLoopPreheader()->getTerminator(),
+ AftBlocks);
+
+ // The current on-the-fly SSA update requires blocks to be processed in
+ // reverse postorder so that LastValueMap contains the correct value at each
+ // exit.
+ LoopBlocksDFS DFS(L);
+ DFS.perform(LI);
+ // Stash the DFS iterators before adding blocks to the loop.
+ LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
+ LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
+
+ if (Header->getParent()->isDebugInfoForProfiling())
+ for (BasicBlock *BB : L->getBlocks())
+ for (Instruction &I : *BB)
+ if (!isa<DbgInfoIntrinsic>(&I))
+ if (const DILocation *DIL = I.getDebugLoc())
+ I.setDebugLoc(DIL->cloneWithDuplicationFactor(Count));
+
+ // Copy all blocks
+ for (unsigned It = 1; It != Count; ++It) {
+ std::vector<BasicBlock *> NewBlocks;
+ // Maps Blocks[It] -> Blocks[It-1]
+ DenseMap<Value *, Value *> PrevItValueMap;
+
+ for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
+ ValueToValueMapTy VMap;
+ BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
+ Header->getParent()->getBasicBlockList().push_back(New);
+
+ if (containsBB(ForeBlocks, *BB)) {
+ L->addBasicBlockToLoop(New, *LI);
+
+ if (*BB == ForeBlocksFirst[0])
+ ForeBlocksFirst.push_back(New);
+ if (*BB == ForeBlocksLast[0])
+ ForeBlocksLast.push_back(New);
+ } else if (containsBB(SubLoopBlocks, *BB)) {
+ SubLoop->addBasicBlockToLoop(New, *LI);
+
+ if (*BB == SubLoopBlocksFirst[0])
+ SubLoopBlocksFirst.push_back(New);
+ if (*BB == SubLoopBlocksLast[0])
+ SubLoopBlocksLast.push_back(New);
+ } else if (containsBB(AftBlocks, *BB)) {
+ L->addBasicBlockToLoop(New, *LI);
+
+ if (*BB == AftBlocksFirst[0])
+ AftBlocksFirst.push_back(New);
+ if (*BB == AftBlocksLast[0])
+ AftBlocksLast.push_back(New);
+ } else {
+ llvm_unreachable("BB being cloned should be in Fore/Sub/Aft");
+ }
+
+ // Update our running maps of newest clones
+ PrevItValueMap[New] = (It == 1 ? *BB : LastValueMap[*BB]);
+ LastValueMap[*BB] = New;
+ for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
+ VI != VE; ++VI) {
+ PrevItValueMap[VI->second] =
+ const_cast<Value *>(It == 1 ? VI->first : LastValueMap[VI->first]);
+ LastValueMap[VI->first] = VI->second;
+ }
+
+ NewBlocks.push_back(New);
+
+ // Update DomTree:
+ if (*BB == ForeBlocksFirst[0])
+ DT->addNewBlock(New, ForeBlocksLast[It - 1]);
+ else if (*BB == SubLoopBlocksFirst[0])
+ DT->addNewBlock(New, SubLoopBlocksLast[It - 1]);
+ else if (*BB == AftBlocksFirst[0])
+ DT->addNewBlock(New, AftBlocksLast[It - 1]);
+ else {
+ // Each set of blocks (Fore/Sub/Aft) will have the same internal domtree
+ // structure.
+ auto BBDomNode = DT->getNode(*BB);
+ auto BBIDom = BBDomNode->getIDom();
+ BasicBlock *OriginalBBIDom = BBIDom->getBlock();
+ assert(OriginalBBIDom);
+ assert(LastValueMap[cast<Value>(OriginalBBIDom)]);
+ DT->addNewBlock(
+ New, cast<BasicBlock>(LastValueMap[cast<Value>(OriginalBBIDom)]));
+ }
+ }
+
+ // Remap all instructions in the most recent iteration
+ for (BasicBlock *NewBlock : NewBlocks) {
+ for (Instruction &I : *NewBlock) {
+ ::remapInstruction(&I, LastValueMap);
+ if (auto *II = dyn_cast<IntrinsicInst>(&I))
+ if (II->getIntrinsicID() == Intrinsic::assume)
+ AC->registerAssumption(II);
+ }
+ }
+
+ // Alter the ForeBlocks phi's, pointing them at the latest version of the
+ // value from the previous iteration's phis
+ for (PHINode &Phi : ForeBlocksFirst[It]->phis()) {
+ Value *OldValue = Phi.getIncomingValueForBlock(AftBlocksLast[It]);
+ assert(OldValue && "should have incoming edge from Aft[It]");
+ Value *NewValue = OldValue;
+ if (Value *PrevValue = PrevItValueMap[OldValue])
+ NewValue = PrevValue;
+
+ assert(Phi.getNumOperands() == 2);
+ Phi.setIncomingBlock(0, ForeBlocksLast[It - 1]);
+ Phi.setIncomingValue(0, NewValue);
+ Phi.removeIncomingValue(1);
+ }
+ }
+
+ // Now that all the basic blocks for the unrolled iterations are in place,
+ // finish up connecting the blocks and phi nodes. At this point LastValueMap
+ // is the last unrolled iterations values.
+
+ // Update Phis in BB from OldBB to point to NewBB
+ auto updatePHIBlocks = [](BasicBlock *BB, BasicBlock *OldBB,
+ BasicBlock *NewBB) {
+ for (PHINode &Phi : BB->phis()) {
+ int I = Phi.getBasicBlockIndex(OldBB);
+ Phi.setIncomingBlock(I, NewBB);
+ }
+ };
+ // Update Phis in BB from OldBB to point to NewBB and use the latest value
+ // from LastValueMap
+ auto updatePHIBlocksAndValues = [](BasicBlock *BB, BasicBlock *OldBB,
+ BasicBlock *NewBB,
+ ValueToValueMapTy &LastValueMap) {
+ for (PHINode &Phi : BB->phis()) {
+ for (unsigned b = 0; b < Phi.getNumIncomingValues(); ++b) {
+ if (Phi.getIncomingBlock(b) == OldBB) {
+ Value *OldValue = Phi.getIncomingValue(b);
+ if (Value *LastValue = LastValueMap[OldValue])
+ Phi.setIncomingValue(b, LastValue);
+ Phi.setIncomingBlock(b, NewBB);
+ break;
+ }
+ }
+ }
+ };
+ // Move all the phis from Src into Dest
+ auto movePHIs = [](BasicBlock *Src, BasicBlock *Dest) {
+ Instruction *insertPoint = Dest->getFirstNonPHI();
+ while (PHINode *Phi = dyn_cast<PHINode>(Src->begin()))
+ Phi->moveBefore(insertPoint);
+ };
+
+ // Update the PHI values outside the loop to point to the last block
+ updatePHIBlocksAndValues(LoopExit, AftBlocksLast[0], AftBlocksLast.back(),
+ LastValueMap);
+
+ // Update ForeBlocks successors and phi nodes
+ BranchInst *ForeTerm =
+ cast<BranchInst>(ForeBlocksLast.back()->getTerminator());
+ BasicBlock *Dest = SubLoopBlocksFirst[0];
+ ForeTerm->setSuccessor(0, Dest);
+
+ if (CompletelyUnroll) {
+ while (PHINode *Phi = dyn_cast<PHINode>(ForeBlocksFirst[0]->begin())) {
+ Phi->replaceAllUsesWith(Phi->getIncomingValueForBlock(Preheader));
+ Phi->getParent()->getInstList().erase(Phi);
+ }
+ } else {
+ // Update the PHI values to point to the last aft block
+ updatePHIBlocksAndValues(ForeBlocksFirst[0], AftBlocksLast[0],
+ AftBlocksLast.back(), LastValueMap);
+ }
+
+ for (unsigned It = 1; It != Count; It++) {
+ // Remap ForeBlock successors from previous iteration to this
+ BranchInst *ForeTerm =
+ cast<BranchInst>(ForeBlocksLast[It - 1]->getTerminator());
+ BasicBlock *Dest = ForeBlocksFirst[It];
+ ForeTerm->setSuccessor(0, Dest);
+ }
+
+ // Subloop successors and phis
+ BranchInst *SubTerm =
+ cast<BranchInst>(SubLoopBlocksLast.back()->getTerminator());
+ SubTerm->setSuccessor(!SubLoopContinueOnTrue, SubLoopBlocksFirst[0]);
+ SubTerm->setSuccessor(SubLoopContinueOnTrue, AftBlocksFirst[0]);
+ updatePHIBlocks(SubLoopBlocksFirst[0], ForeBlocksLast[0],
+ ForeBlocksLast.back());
+ updatePHIBlocks(SubLoopBlocksFirst[0], SubLoopBlocksLast[0],
+ SubLoopBlocksLast.back());
+
+ for (unsigned It = 1; It != Count; It++) {
+ // Replace the conditional branch of the previous iteration subloop with an
+ // unconditional one to this one
+ BranchInst *SubTerm =
+ cast<BranchInst>(SubLoopBlocksLast[It - 1]->getTerminator());
+ BranchInst::Create(SubLoopBlocksFirst[It], SubTerm);
+ SubTerm->eraseFromParent();
+
+ updatePHIBlocks(SubLoopBlocksFirst[It], ForeBlocksLast[It],
+ ForeBlocksLast.back());
+ updatePHIBlocks(SubLoopBlocksFirst[It], SubLoopBlocksLast[It],
+ SubLoopBlocksLast.back());
+ movePHIs(SubLoopBlocksFirst[It], SubLoopBlocksFirst[0]);
+ }
+
+ // Aft blocks successors and phis
+ BranchInst *Term = cast<BranchInst>(AftBlocksLast.back()->getTerminator());
+ if (CompletelyUnroll) {
+ BranchInst::Create(LoopExit, Term);
+ Term->eraseFromParent();
+ } else {
+ Term->setSuccessor(!ContinueOnTrue, ForeBlocksFirst[0]);
+ }
+ updatePHIBlocks(AftBlocksFirst[0], SubLoopBlocksLast[0],
+ SubLoopBlocksLast.back());
+
+ for (unsigned It = 1; It != Count; It++) {
+ // Replace the conditional branch of the previous iteration subloop with an
+ // unconditional one to this one
+ BranchInst *AftTerm =
+ cast<BranchInst>(AftBlocksLast[It - 1]->getTerminator());
+ BranchInst::Create(AftBlocksFirst[It], AftTerm);
+ AftTerm->eraseFromParent();
+
+ updatePHIBlocks(AftBlocksFirst[It], SubLoopBlocksLast[It],
+ SubLoopBlocksLast.back());
+ movePHIs(AftBlocksFirst[It], AftBlocksFirst[0]);
+ }
+
+ // Dominator Tree. Remove the old links between Fore, Sub and Aft, adding the
+ // new ones required.
+ if (Count != 1) {
+ SmallVector<DominatorTree::UpdateType, 4> DTUpdates;
+ DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete, ForeBlocksLast[0],
+ SubLoopBlocksFirst[0]);
+ DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete,
+ SubLoopBlocksLast[0], AftBlocksFirst[0]);
+
+ DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
+ ForeBlocksLast.back(), SubLoopBlocksFirst[0]);
+ DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
+ SubLoopBlocksLast.back(), AftBlocksFirst[0]);
+ DT->applyUpdates(DTUpdates);
+ }
+
+ // Merge adjacent basic blocks, if possible.
+ SmallPtrSet<BasicBlock *, 16> MergeBlocks;
+ MergeBlocks.insert(ForeBlocksLast.begin(), ForeBlocksLast.end());
+ MergeBlocks.insert(SubLoopBlocksLast.begin(), SubLoopBlocksLast.end());
+ MergeBlocks.insert(AftBlocksLast.begin(), AftBlocksLast.end());
+ while (!MergeBlocks.empty()) {
+ BasicBlock *BB = *MergeBlocks.begin();
+ BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator());
+ if (Term && Term->isUnconditional() && L->contains(Term->getSuccessor(0))) {
+ BasicBlock *Dest = Term->getSuccessor(0);
+ if (BasicBlock *Fold = foldBlockIntoPredecessor(Dest, LI, SE, DT)) {
+ // Don't remove BB and add Fold as they are the same BB
+ assert(Fold == BB);
+ (void)Fold;
+ MergeBlocks.erase(Dest);
+ } else
+ MergeBlocks.erase(BB);
+ } else
+ MergeBlocks.erase(BB);
+ }
+
+ // At this point, the code is well formed. We now do a quick sweep over the
+ // inserted code, doing constant propagation and dead code elimination as we
+ // go.
+ simplifyLoopAfterUnroll(SubLoop, true, LI, SE, DT, AC);
+ simplifyLoopAfterUnroll(L, !CompletelyUnroll && Count > 1, LI, SE, DT, AC);
+
+ NumCompletelyUnrolledAndJammed += CompletelyUnroll;
+ ++NumUnrolledAndJammed;
+
+#ifndef NDEBUG
+ // We shouldn't have done anything to break loop simplify form or LCSSA.
+ Loop *OuterL = L->getParentLoop();
+ Loop *OutestLoop = OuterL ? OuterL : (!CompletelyUnroll ? L : SubLoop);
+ assert(OutestLoop->isRecursivelyLCSSAForm(*DT, *LI));
+ if (!CompletelyUnroll)
+ assert(L->isLoopSimplifyForm());
+ assert(SubLoop->isLoopSimplifyForm());
+ assert(DT->verify());
+#endif
+
+ // Update LoopInfo if the loop is completely removed.
+ if (CompletelyUnroll)
+ LI->erase(L);
+
+ return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled
+ : LoopUnrollResult::PartiallyUnrolled;
+}
+
+static bool getLoadsAndStores(std::vector<BasicBlock *> &Blocks,
+ SmallVector<Value *, 4> &MemInstr) {
+ // Scan the BBs and collect legal loads and stores.
+ // Returns false if non-simple loads/stores are found.
+ for (BasicBlock *BB : Blocks) {
+ for (Instruction &I : *BB) {
+ if (auto *Ld = dyn_cast<LoadInst>(&I)) {
+ if (!Ld->isSimple())
+ return false;
+ MemInstr.push_back(&I);
+ } else if (auto *St = dyn_cast<StoreInst>(&I)) {
+ if (!St->isSimple())
+ return false;
+ MemInstr.push_back(&I);
+ } else if (I.mayReadOrWriteMemory()) {
+ return false;
+ }
+ }
+ }
+ return true;
+}
+
+static bool checkDependencies(SmallVector<Value *, 4> &Earlier,
+ SmallVector<Value *, 4> &Later,
+ unsigned LoopDepth, bool InnerLoop,
+ DependenceInfo &DI) {
+ // Use DA to check for dependencies between loads and stores that make unroll
+ // and jam invalid
+ for (Value *I : Earlier) {
+ for (Value *J : Later) {
+ Instruction *Src = cast<Instruction>(I);
+ Instruction *Dst = cast<Instruction>(J);
+ if (Src == Dst)
+ continue;
+ // Ignore Input dependencies.
+ if (isa<LoadInst>(Src) && isa<LoadInst>(Dst))
+ continue;
+
+ // Track dependencies, and if we find them take a conservative approach
+ // by allowing only = or < (not >), altough some > would be safe
+ // (depending upon unroll width).
+ // For the inner loop, we need to disallow any (> <) dependencies
+ // FIXME: Allow > so long as distance is less than unroll width
+ if (auto D = DI.depends(Src, Dst, true)) {
+ assert(D->isOrdered() && "Expected an output, flow or anti dep.");
+
+ if (D->isConfused())
+ return false;
+ if (!InnerLoop) {
+ if (D->getDirection(LoopDepth) & Dependence::DVEntry::GT)
+ return false;
+ } else {
+ assert(LoopDepth + 1 <= D->getLevels());
+ if (D->getDirection(LoopDepth) & Dependence::DVEntry::GT &&
+ D->getDirection(LoopDepth + 1) & Dependence::DVEntry::LT)
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+}
+
+static bool checkDependencies(Loop *L, std::vector<BasicBlock *> &ForeBlocks,
+ std::vector<BasicBlock *> &SubLoopBlocks,
+ std::vector<BasicBlock *> &AftBlocks,
+ DependenceInfo &DI) {
+ // Get all loads/store pairs for each blocks
+ SmallVector<Value *, 4> ForeMemInstr;
+ SmallVector<Value *, 4> SubLoopMemInstr;
+ SmallVector<Value *, 4> AftMemInstr;
+ if (!getLoadsAndStores(ForeBlocks, ForeMemInstr) ||
+ !getLoadsAndStores(SubLoopBlocks, SubLoopMemInstr) ||
+ !getLoadsAndStores(AftBlocks, AftMemInstr))
+ return false;
+
+ // Check for dependencies between any blocks that may change order
+ unsigned LoopDepth = L->getLoopDepth();
+ return checkDependencies(ForeMemInstr, SubLoopMemInstr, LoopDepth, false,
+ DI) &&
+ checkDependencies(ForeMemInstr, AftMemInstr, LoopDepth, false, DI) &&
+ checkDependencies(SubLoopMemInstr, AftMemInstr, LoopDepth, false,
+ DI) &&
+ checkDependencies(SubLoopMemInstr, SubLoopMemInstr, LoopDepth, true,
+ DI);
+}
+
+bool llvm::isSafeToUnrollAndJam(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
+ DependenceInfo &DI) {
+ /* We currently handle outer loops like this:
+ |
+ ForeFirst <----\ }
+ Blocks | } ForeBlocks
+ ForeLast | }
+ | |
+ SubLoopFirst <\ | }
+ Blocks | | } SubLoopBlocks
+ SubLoopLast -/ | }
+ | |
+ AftFirst | }
+ Blocks | } AftBlocks
+ AftLast ------/ }
+ |
+
+ There are (theoretically) any number of blocks in ForeBlocks, SubLoopBlocks
+ and AftBlocks, providing that there is one edge from Fores to SubLoops,
+ one edge from SubLoops to Afts and a single outer loop exit (from Afts).
+ In practice we currently limit Aft blocks to a single block, and limit
+ things further in the profitablility checks of the unroll and jam pass.
+
+ Because of the way we rearrange basic blocks, we also require that
+ the Fore blocks on all unrolled iterations are safe to move before the
+ SubLoop blocks of all iterations. So we require that the phi node looping
+ operands of ForeHeader can be moved to at least the end of ForeEnd, so that
+ we can arrange cloned Fore Blocks before the subloop and match up Phi's
+ correctly.
+
+ i.e. The old order of blocks used to be F1 S1_1 S1_2 A1 F2 S2_1 S2_2 A2.
+ It needs to be safe to tranform this to F1 F2 S1_1 S2_1 S1_2 S2_2 A1 A2.
+
+ There are then a number of checks along the lines of no calls, no
+ exceptions, inner loop IV is consistent, etc. Note that for loops requiring
+ runtime unrolling, UnrollRuntimeLoopRemainder can also fail in
+ UnrollAndJamLoop if the trip count cannot be easily calculated.
+ */
+
+ if (!L->isLoopSimplifyForm() || L->getSubLoops().size() != 1)
+ return false;
+ Loop *SubLoop = L->getSubLoops()[0];
+ if (!SubLoop->isLoopSimplifyForm())
+ return false;
+
+ BasicBlock *Header = L->getHeader();
+ BasicBlock *Latch = L->getLoopLatch();
+ BasicBlock *Exit = L->getExitingBlock();
+ BasicBlock *SubLoopHeader = SubLoop->getHeader();
+ BasicBlock *SubLoopLatch = SubLoop->getLoopLatch();
+ BasicBlock *SubLoopExit = SubLoop->getExitingBlock();
+
+ if (Latch != Exit)
+ return false;
+ if (SubLoopLatch != SubLoopExit)
+ return false;
+
+ if (Header->hasAddressTaken() || SubLoopHeader->hasAddressTaken())
+ return false;
+
+ // Split blocks into Fore/SubLoop/Aft based on dominators
+ std::vector<BasicBlock *> SubLoopBlocks;
+ std::vector<BasicBlock *> ForeBlocks;
+ std::vector<BasicBlock *> AftBlocks;
+ if (!partitionOuterLoopBlocks(L, SubLoop, ForeBlocks, SubLoopBlocks,
+ AftBlocks, &DT))
+ return false;
+
+ // Aft blocks may need to move instructions to fore blocks, which becomes more
+ // difficult if there are multiple (potentially conditionally executed)
+ // blocks. For now we just exclude loops with multiple aft blocks.
+ if (AftBlocks.size() != 1)
+ return false;
+
+ // Check inner loop IV is consistent between all iterations
+ const SCEV *SubLoopBECountSC = SE.getExitCount(SubLoop, SubLoopLatch);
+ if (isa<SCEVCouldNotCompute>(SubLoopBECountSC) ||
+ !SubLoopBECountSC->getType()->isIntegerTy())
+ return false;
+ ScalarEvolution::LoopDisposition LD =
+ SE.getLoopDisposition(SubLoopBECountSC, L);
+ if (LD != ScalarEvolution::LoopInvariant)
+ return false;
+
+ // Check the loop safety info for exceptions.
+ LoopSafetyInfo LSI;
+ computeLoopSafetyInfo(&LSI, L);
+ if (LSI.MayThrow)
+ return false;
+
+ // We've ruled out the easy stuff and now need to check that there are no
+ // interdependencies which may prevent us from moving the:
+ // ForeBlocks before Subloop and AftBlocks.
+ // Subloop before AftBlocks.
+ // ForeBlock phi operands before the subloop
+
+ // Make sure we can move all instructions we need to before the subloop
+ SmallVector<Instruction *, 8> Worklist;
+ SmallPtrSet<Instruction *, 8> Visited;
+ for (auto &Phi : Header->phis()) {
+ Value *V = Phi.getIncomingValueForBlock(Latch);
+ if (Instruction *I = dyn_cast<Instruction>(V))
+ Worklist.push_back(I);
+ }
+ while (!Worklist.empty()) {
+ Instruction *I = Worklist.back();
+ Worklist.pop_back();
+ if (Visited.insert(I).second) {
+ if (SubLoop->contains(I->getParent()))
+ return false;
+ if (containsBB(AftBlocks, I->getParent())) {
+ // If we hit a phi node in afts we know we are done (probably LCSSA)
+ if (isa<PHINode>(I))
+ return false;
+ if (I->mayHaveSideEffects() || I->mayReadOrWriteMemory())
+ return false;
+ for (auto &U : I->operands())
+ if (Instruction *II = dyn_cast<Instruction>(U))
+ Worklist.push_back(II);
+ }
+ }
+ }
+
+ // Check for memory dependencies which prohibit the unrolling we are doing.
+ // Because of the way we are unrolling Fore/Sub/Aft blocks, we need to check
+ // there are no dependencies between Fore-Sub, Fore-Aft, Sub-Aft and Sub-Sub.
+ if (!checkDependencies(L, ForeBlocks, SubLoopBlocks, AftBlocks, DI))
+ return false;
+
+ return true;
+}
OpenPOWER on IntegriCloud