diff options
Diffstat (limited to 'llvm/lib')
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 19 |
1 files changed, 11 insertions, 8 deletions
diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index fbe042118de..38ddeede84a 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -2786,24 +2786,27 @@ void InnerLoopVectorizer::vectorizeMemoryInstruction(Instruction *Instr) { // At this point we should vector version of GEP for Gather or Scatter assert(CreateGatherScatter && "The instruction should be scalarized"); if (Gep) { + // Vectorizing GEP, across UF parts. We want to get a vector value for base + // and each index that's defined inside the loop, even if it is + // loop-invariant but wasn't hoisted out. Otherwise we want to keep them + // scalar. SmallVector<VectorParts, 4> OpsV; - // Vectorizing GEP, across UF parts, we want to keep each loop-invariant - // base or index of GEP scalar for (Value *Op : Gep->operands()) { - if (PSE.getSE()->isLoopInvariant(PSE.getSCEV(Op), OrigLoop)) - OpsV.push_back(VectorParts(UF, Op)); - else + Instruction *SrcInst = dyn_cast<Instruction>(Op); + if (SrcInst && OrigLoop->contains(SrcInst)) OpsV.push_back(getVectorValue(Op)); + else + OpsV.push_back(VectorParts(UF, Op)); } - for (unsigned Part = 0; Part < UF; ++Part) { SmallVector<Value *, 4> Ops; Value *GEPBasePtr = OpsV[0][Part]; for (unsigned i = 1; i < Gep->getNumOperands(); i++) Ops.push_back(OpsV[i][Part]); - Value *NewGep = - Builder.CreateGEP(nullptr, GEPBasePtr, Ops, "VectorGep"); + Value *NewGep = Builder.CreateGEP(GEPBasePtr, Ops, "VectorGep"); + cast<GetElementPtrInst>(NewGep)->setIsInBounds(Gep->isInBounds()); assert(NewGep->getType()->isVectorTy() && "Expected vector GEP"); + NewGep = Builder.CreateBitCast(NewGep, VectorType::get(Ptr->getType(), VF)); VectorGep.push_back(NewGep); |
