summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Analysis/LoopAccessAnalysis.cpp57
-rw-r--r--llvm/lib/Transforms/Vectorize/LoopVectorize.cpp30
2 files changed, 47 insertions, 40 deletions
diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
index 35c5807be08..c0cad5c0894 100644
--- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp
+++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp
@@ -25,6 +25,31 @@ using namespace llvm;
#define DEBUG_TYPE "loop-vectorize"
+static cl::opt<unsigned, true>
+VectorizationFactor("force-vector-width", cl::Hidden,
+ cl::desc("Sets the SIMD width. Zero is autoselect."),
+ cl::location(VectorizerParams::VectorizationFactor));
+unsigned VectorizerParams::VectorizationFactor = 0;
+
+static cl::opt<unsigned, true>
+VectorizationInterleave("force-vector-interleave", cl::Hidden,
+ cl::desc("Sets the vectorization interleave count. "
+ "Zero is autoselect."),
+ cl::location(
+ VectorizerParams::VectorizationInterleave));
+unsigned VectorizerParams::VectorizationInterleave = 0;
+
+/// When performing memory disambiguation checks at runtime do not make more
+/// than this number of comparisons.
+const unsigned VectorizerParams::RuntimeMemoryCheckThreshold = 8;
+
+/// Maximum SIMD width.
+const unsigned VectorizerParams::MaxVectorWidth = 64;
+
+bool VectorizerParams::isInterleaveForced() {
+ return ::VectorizationInterleave.getNumOccurrences() > 0;
+}
+
void VectorizationReport::emitAnalysis(VectorizationReport &Message,
const Function *TheFunction,
const Loop *TheLoop) {
@@ -454,10 +479,9 @@ public:
typedef PointerIntPair<Value *, 1, bool> MemAccessInfo;
typedef SmallPtrSet<MemAccessInfo, 8> MemAccessInfoSet;
- MemoryDepChecker(ScalarEvolution *Se, const DataLayout *Dl, const Loop *L,
- const LoopAccessInfo::VectorizerParams &VectParams)
+ MemoryDepChecker(ScalarEvolution *Se, const DataLayout *Dl, const Loop *L)
: SE(Se), DL(Dl), InnermostLoop(L), AccessIdx(0),
- ShouldRetryWithRuntimeCheck(false), VectParams(VectParams) {}
+ ShouldRetryWithRuntimeCheck(false) {}
/// \brief Register the location (instructions are given increasing numbers)
/// of a write access.
@@ -512,9 +536,6 @@ private:
/// vectorize this loop with runtime checks.
bool ShouldRetryWithRuntimeCheck;
- /// \brief Vectorizer parameters used by the analysis.
- LoopAccessInfo::VectorizerParams VectParams;
-
/// \brief Check whether there is a plausible dependence between the two
/// accesses.
///
@@ -638,7 +659,8 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance,
// Store-load forwarding distance.
const unsigned NumCyclesForStoreLoadThroughMemory = 8*TypeByteSize;
// Maximum vector factor.
- unsigned MaxVFWithoutSLForwardIssues = VectParams.MaxVectorWidth*TypeByteSize;
+ unsigned MaxVFWithoutSLForwardIssues =
+ VectorizerParams::MaxVectorWidth * TypeByteSize;
if(MaxSafeDepDistBytes < MaxVFWithoutSLForwardIssues)
MaxVFWithoutSLForwardIssues = MaxSafeDepDistBytes;
@@ -657,7 +679,8 @@ bool MemoryDepChecker::couldPreventStoreLoadForward(unsigned Distance,
}
if (MaxVFWithoutSLForwardIssues < MaxSafeDepDistBytes &&
- MaxVFWithoutSLForwardIssues != VectParams.MaxVectorWidth*TypeByteSize)
+ MaxVFWithoutSLForwardIssues !=
+ VectorizerParams::MaxVectorWidth * TypeByteSize)
MaxSafeDepDistBytes = MaxVFWithoutSLForwardIssues;
return false;
}
@@ -762,10 +785,10 @@ bool MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx,
unsigned Distance = (unsigned) Val.getZExtValue();
// Bail out early if passed-in parameters make vectorization not feasible.
- unsigned ForcedFactor = (VectParams.VectorizationFactor ?
- VectParams.VectorizationFactor : 1);
- unsigned ForcedUnroll = (VectParams.VectorizationInterleave ?
- VectParams.VectorizationInterleave : 1);
+ unsigned ForcedFactor = (VectorizerParams::VectorizationFactor ?
+ VectorizerParams::VectorizationFactor : 1);
+ unsigned ForcedUnroll = (VectorizerParams::VectorizationInterleave ?
+ VectorizerParams::VectorizationInterleave : 1);
// The distance must be bigger than the size needed for a vectorized version
// of the operation and the size of the vectorized operation must not be
@@ -848,7 +871,7 @@ bool LoopAccessInfo::canVectorizeMemory(ValueToValueMap &Strides) {
PtrRtCheck.Need = false;
const bool IsAnnotatedParallel = TheLoop->isAnnotatedParallel();
- MemoryDepChecker DepChecker(SE, DL, TheLoop, VectParams);
+ MemoryDepChecker DepChecker(SE, DL, TheLoop);
// For each block.
for (Loop::block_iterator bb = TheLoop->block_begin(),
@@ -1017,7 +1040,8 @@ bool LoopAccessInfo::canVectorizeMemory(ValueToValueMap &Strides) {
// Check that we did not collect too many pointers or found an unsizeable
// pointer.
- if (!CanDoRT || NumComparisons > VectParams.RuntimeMemoryCheckThreshold) {
+ if (!CanDoRT ||
+ NumComparisons > VectorizerParams::RuntimeMemoryCheckThreshold) {
PtrRtCheck.reset();
CanDoRT = false;
}
@@ -1057,14 +1081,15 @@ bool LoopAccessInfo::canVectorizeMemory(ValueToValueMap &Strides) {
TheLoop, Strides, true);
// Check that we did not collect too many pointers or found an unsizeable
// pointer.
- if (!CanDoRT || NumComparisons > VectParams.RuntimeMemoryCheckThreshold) {
+ if (!CanDoRT ||
+ NumComparisons > VectorizerParams::RuntimeMemoryCheckThreshold) {
if (!CanDoRT && NumComparisons > 0)
emitAnalysis(VectorizationReport()
<< "cannot check memory dependencies at runtime");
else
emitAnalysis(VectorizationReport()
<< NumComparisons << " exceeds limit of "
- << VectParams.RuntimeMemoryCheckThreshold
+ << VectorizerParams::RuntimeMemoryCheckThreshold
<< " dependent memory operations checked at runtime");
DEBUG(dbgs() << "LV: Can't vectorize with memory checks\n");
PtrRtCheck.reset();
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
index 1ddb45e5df6..f53c913c647 100644
--- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
+++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
@@ -106,15 +106,6 @@ using namespace llvm::PatternMatch;
STATISTIC(LoopsVectorized, "Number of loops vectorized");
STATISTIC(LoopsAnalyzed, "Number of loops analyzed for vectorization");
-static cl::opt<unsigned>
-VectorizationFactor("force-vector-width", cl::init(0), cl::Hidden,
- cl::desc("Sets the SIMD width. Zero is autoselect."));
-
-static cl::opt<unsigned>
-VectorizationInterleave("force-vector-interleave", cl::init(0), cl::Hidden,
- cl::desc("Sets the vectorization interleave count. "
- "Zero is autoselect."));
-
static cl::opt<bool>
EnableIfConversion("enable-if-conversion", cl::init(true), cl::Hidden,
cl::desc("Enable if-conversion during vectorization."));
@@ -145,13 +136,6 @@ static cl::opt<bool> EnableMemAccessVersioning(
/// We don't unroll loops with a known constant trip count below this number.
static const unsigned TinyTripCountUnrollThreshold = 128;
-/// When performing memory disambiguation checks at runtime do not make more
-/// than this number of comparisons.
-static const unsigned RuntimeMemoryCheckThreshold = 8;
-
-/// Maximum simd width.
-static const unsigned MaxVectorWidth = 64;
-
static cl::opt<unsigned> ForceTargetNumScalarRegs(
"force-target-num-scalar-regs", cl::init(0), cl::Hidden,
cl::desc("A flag that overrides the target's number of scalar registers."));
@@ -551,10 +535,7 @@ public:
: NumPredStores(0), TheLoop(L), SE(SE), DL(DL),
TLI(TLI), TheFunction(F), TTI(TTI), DT(DT), Induction(nullptr),
WidestIndTy(nullptr),
- LAI(F, L, SE, DL, TLI, AA, DT,
- LoopAccessInfo::VectorizerParams(
- MaxVectorWidth, VectorizationFactor, VectorizationInterleave,
- RuntimeMemoryCheckThreshold)),
+ LAI(F, L, SE, DL, TLI, AA, DT),
HasFunNoNaNAttr(false) {}
/// This enum represents the kinds of reductions that we support.
@@ -1021,7 +1002,7 @@ class LoopVectorizeHints {
bool validate(unsigned Val) {
switch (Kind) {
case HK_WIDTH:
- return isPowerOf2_32(Val) && Val <= MaxVectorWidth;
+ return isPowerOf2_32(Val) && Val <= VectorizerParams::MaxVectorWidth;
case HK_UNROLL:
return isPowerOf2_32(Val) && Val <= MaxInterleaveFactor;
case HK_FORCE:
@@ -1049,7 +1030,8 @@ public:
};
LoopVectorizeHints(const Loop *L, bool DisableInterleaving)
- : Width("vectorize.width", VectorizationFactor, HK_WIDTH),
+ : Width("vectorize.width", VectorizerParams::VectorizationFactor,
+ HK_WIDTH),
Interleave("interleave.count", DisableInterleaving, HK_UNROLL),
Force("vectorize.enable", FK_Undefined, HK_FORCE),
TheLoop(L) {
@@ -1057,8 +1039,8 @@ public:
getHintsFromMetadata();
// force-vector-interleave overrides DisableInterleaving.
- if (VectorizationInterleave.getNumOccurrences() > 0)
- Interleave.Value = VectorizationInterleave;
+ if (VectorizerParams::isInterleaveForced())
+ Interleave.Value = VectorizerParams::VectorizationInterleave;
DEBUG(if (DisableInterleaving && Interleave.Value == 1) dbgs()
<< "LV: Interleaving disabled by the pass manager\n");
OpenPOWER on IntegriCloud