diff options
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp | 18 | ||||
| -rw-r--r-- | llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll | 28 |
2 files changed, 41 insertions, 5 deletions
diff --git a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp index 2e4707f916d..c1d3f925515 100644 --- a/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/LoadStoreVectorizer.cpp @@ -562,20 +562,28 @@ Vectorizer::getVectorizablePrefix(ArrayRef<Instruction *> Chain) { if (BarrierMemoryInstr && OBB.dominates(BarrierMemoryInstr, MemInstr)) break; - if (isa<LoadInst>(MemInstr) && isa<LoadInst>(ChainInstr)) + auto *MemLoad = dyn_cast<LoadInst>(MemInstr); + auto *ChainLoad = dyn_cast<LoadInst>(ChainInstr); + if (MemLoad && ChainLoad) continue; + // We can ignore the alias if the we have a load store pair and the load + // is known to be invariant. The load cannot be clobbered by the store. + auto IsInvariantLoad = [](const LoadInst *LI) -> bool { + return LI->getMetadata(LLVMContext::MD_invariant_load); + }; + // We can ignore the alias as long as the load comes before the store, // because that means we won't be moving the load past the store to // vectorize it (the vectorized load is inserted at the location of the // first load in the chain). - if (isa<StoreInst>(MemInstr) && isa<LoadInst>(ChainInstr) && - OBB.dominates(ChainInstr, MemInstr)) + if (isa<StoreInst>(MemInstr) && ChainLoad && + (IsInvariantLoad(ChainLoad) || OBB.dominates(ChainLoad, MemInstr))) continue; // Same case, but in reverse. - if (isa<LoadInst>(MemInstr) && isa<StoreInst>(ChainInstr) && - OBB.dominates(MemInstr, ChainInstr)) + if (MemLoad && isa<StoreInst>(ChainInstr) && + (IsInvariantLoad(MemLoad) || OBB.dominates(MemLoad, ChainInstr))) continue; if (!AA.isNoAlias(MemoryLocation::get(MemInstr), diff --git a/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll new file mode 100644 index 00000000000..ffec12acbe0 --- /dev/null +++ b/llvm/test/Transforms/LoadStoreVectorizer/AMDGPU/invariant-load.ll @@ -0,0 +1,28 @@ +; RUN: opt -mtriple=amdgcn-amd-amdhsa -basicaa -load-store-vectorizer -S -o - %s | FileCheck %s + +target datalayout = "e-p:32:32-p1:64:64-p2:64:64-p3:32:32-p4:64:64-p5:32:32-p24:64:64-i64:64-v16:16-v24:32-v32:32-v48:64-v96:128-v192:256-v256:256-v512:512-v1024:1024-v2048:2048-n32:64" + +; CHECK-LABEL: @interleave +; CHECK: load <2 x double>, <2 x double> addrspace(1)* %{{.}}, align 8{{$}} +; CHECK: store <2 x double> zeroinitializer +; CHECK: store double %add +define amdgpu_kernel void @interleave(double addrspace(1)* nocapture %a, double addrspace(1)* nocapture %b, double addrspace(1)* nocapture readonly %c) #0 { +entry: + %a.idx.1 = getelementptr inbounds double, double addrspace(1)* %a, i64 1 + %c.idx.1 = getelementptr inbounds double, double addrspace(1)* %c, i64 1 + + %ld.c = load double, double addrspace(1)* %c, align 8 + store double 0.0, double addrspace(1)* %a, align 8 ; Cannot alias invariant load + + %ld.c.idx.1 = load double, double addrspace(1)* %c.idx.1, align 8, !invariant.load !0 + store double 0.0, double addrspace(1)* %a.idx.1, align 8 + + %add = fadd double %ld.c, %ld.c.idx.1 + store double %add, double addrspace(1)* %b + + ret void +} + +attributes #0 = { nounwind } + +!0 = !{} |
