diff options
| -rw-r--r-- | llvm/include/llvm/Analysis/LoopAccessAnalysis.h | 19 | ||||
| -rw-r--r-- | llvm/lib/Analysis/LoopAccessAnalysis.cpp | 11 | ||||
| -rw-r--r-- | llvm/test/Transforms/LoopVectorize/runtime-check.ll | 6 |
3 files changed, 23 insertions, 13 deletions
diff --git a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h index b5a964de9eb..4ed00e20775 100644 --- a/llvm/include/llvm/Analysis/LoopAccessAnalysis.h +++ b/llvm/include/llvm/Analysis/LoopAccessAnalysis.h @@ -104,7 +104,9 @@ public: // Can vectorize safely without RT checks. All dependences are known to be // safe. Safe, - // Cannot vectorize due to unsafe or unknown dependencies. + // Can possibly vectorize with RT checks to overcome unknown dependencies. + PossiblySafeWithRtChecks, + // Cannot vectorize due to known unsafe dependencies. Unsafe, }; @@ -175,7 +177,7 @@ public: MemoryDepChecker(PredicatedScalarEvolution &PSE, const Loop *L) : PSE(PSE), InnermostLoop(L), AccessIdx(0), MaxSafeRegisterWidth(-1U), - ShouldRetryWithRuntimeCheck(false), + FoundNonConstantDistanceDependence(false), Status(VectorizationSafetyStatus::Safe), RecordDependences(true) {} /// Register the location (instructions are given increasing numbers) @@ -218,7 +220,10 @@ public: /// In same cases when the dependency check fails we can still /// vectorize the loop with a dynamic array access check. - bool shouldRetryWithRuntimeCheck() { return ShouldRetryWithRuntimeCheck; } + bool shouldRetryWithRuntimeCheck() const { + return FoundNonConstantDistanceDependence && + Status == VectorizationSafetyStatus::PossiblySafeWithRtChecks; + } /// Returns the memory dependences. If null is returned we exceeded /// the MaxDependences threshold and this information is not @@ -280,10 +285,11 @@ private: /// If we see a non-constant dependence distance we can still try to /// vectorize this loop with runtime checks. - bool ShouldRetryWithRuntimeCheck; + bool FoundNonConstantDistanceDependence; /// Result of the dependence checks, indicating whether the checked - /// dependences are safe for vectorization or not. + /// dependences are safe for vectorization, require RT checks or are known to + /// be unsafe. VectorizationSafetyStatus Status; //// True if Dependences reflects the dependences in the @@ -319,7 +325,8 @@ private: bool couldPreventStoreLoadForward(uint64_t Distance, uint64_t TypeByteSize); /// Updates the current safety status with \p S. We can go from Safe to - /// to Unsafe. + /// either PossiblySafeWithRtChecks or Unsafe and from + /// PossiblySafeWithRtChecks to Unsafe. void mergeInStatus(VectorizationSafetyStatus S); }; diff --git a/llvm/lib/Analysis/LoopAccessAnalysis.cpp b/llvm/lib/Analysis/LoopAccessAnalysis.cpp index 245f318e308..7f3480f512a 100644 --- a/llvm/lib/Analysis/LoopAccessAnalysis.cpp +++ b/llvm/lib/Analysis/LoopAccessAnalysis.cpp @@ -342,7 +342,7 @@ void RuntimePointerChecking::groupChecks( // // The above case requires that we have an UnknownDependence between // accesses to the same underlying object. This cannot happen unless - // ShouldRetryWithRuntimeCheck is set, and therefore UseDependencies + // FoundNonConstantDistanceDependence is set, and therefore UseDependencies // is also false. In this case we will use the fallback path and create // separate checking groups for all pointers. @@ -556,7 +556,7 @@ public: /// perform dependency checking. /// /// Note that this can later be cleared if we retry memcheck analysis without - /// dependency checking (i.e. ShouldRetryWithRuntimeCheck). + /// dependency checking (i.e. FoundNonConstantDistanceDependence). bool isDependencyCheckNeeded() { return !CheckDeps.empty(); } /// We decided that no dependence analysis would be used. Reset the state. @@ -604,8 +604,8 @@ private: /// /// Note that, this is different from isDependencyCheckNeeded. When we retry /// memcheck analysis without dependency checking - /// (i.e. ShouldRetryWithRuntimeCheck), isDependencyCheckNeeded is cleared - /// while this remains set if we have potentially dependent accesses. + /// (i.e. FoundNonConstantDistanceDependence), isDependencyCheckNeeded is + /// cleared while this remains set if we have potentially dependent accesses. bool IsRTCheckAnalysisNeeded; /// The SCEV predicate containing all the SCEV-related assumptions. @@ -1230,6 +1230,7 @@ MemoryDepChecker::Dependence::isSafeForVectorization(DepType Type) { return VectorizationSafetyStatus::Safe; case Unknown: + return VectorizationSafetyStatus::PossiblySafeWithRtChecks; case ForwardButPreventsForwarding: case Backward: case BackwardVectorizableButPreventsForwarding: @@ -1491,7 +1492,7 @@ MemoryDepChecker::isDependent(const MemAccessInfo &A, unsigned AIdx, return Dependence::NoDep; LLVM_DEBUG(dbgs() << "LAA: Dependence because of non-constant distance\n"); - ShouldRetryWithRuntimeCheck = true; + FoundNonConstantDistanceDependence = true; return Dependence::Unknown; } diff --git a/llvm/test/Transforms/LoopVectorize/runtime-check.ll b/llvm/test/Transforms/LoopVectorize/runtime-check.ll index 19bfa50f769..332e3efb65d 100644 --- a/llvm/test/Transforms/LoopVectorize/runtime-check.ll +++ b/llvm/test/Transforms/LoopVectorize/runtime-check.ll @@ -117,7 +117,9 @@ loopexit: ret void } -; Check we do generate unnecessary runtime checks. They will always fail. +; Check we do not generate runtime checks if we found a known dependence preventing +; vectorization. In this case, it is a read of c[i-1] followed by a write of c[i]. +; The runtime checks would always fail. ; void test_runtime_check2(float *a, float b, unsigned offset, unsigned offset2, unsigned n, float *c) { ; for (unsigned i = 1; i < n; i++) { @@ -127,7 +129,7 @@ loopexit: ; } ; ; CHECK-LABEL: test_runtime_check2 -; CHECK: <4 x float> +; CHECK-NOT: <4 x float> define void @test_runtime_check2(float* %a, float %b, i64 %offset, i64 %offset2, i64 %n, float* %c) { entry: br label %for.body |
