From b02b0ad8c78eab94b2c24333c5c378ffe90ea309 Mon Sep 17 00:00:00 2001 From: Anna Thomas Date: Tue, 21 Aug 2018 14:40:27 +0000 Subject: [LV] Vectorize loops where non-phi instructions used outside loop Summary: Follow up change to rL339703, where we now vectorize loops with non-phi instructions used outside the loop. Note that the cyclic dependency identification occurs when identifying reduction/induction vars. We also need to identify that we do not allow users where the PSCEV information within and outside the loop are different. This was the fix added in rL307837 for PR33706. Reviewers: Ayal, mkuper, fhahn Subscribers: javed.absar, llvm-commits Differential Revision: https://reviews.llvm.org/D50778 llvm-svn: 340278 --- llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) (limited to 'llvm/lib/Transforms/Vectorize/LoopVectorize.cpp') diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 237c033ea64..42e24007da8 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -3721,11 +3721,18 @@ void InnerLoopVectorizer::fixLCSSAPHIs() { for (PHINode &LCSSAPhi : LoopExitBlock->phis()) { if (LCSSAPhi.getNumIncomingValues() == 1) { auto *IncomingValue = LCSSAPhi.getIncomingValue(0); + // Non-instruction incoming values will have only one value. + unsigned LastLane = 0; + if (isa(IncomingValue)) + LastLane = Cost->isUniformAfterVectorization( + cast(IncomingValue), VF) + ? 0 + : VF - 1; // Can be a loop invariant incoming value or the last scalar value to be // extracted from the vectorized loop. Builder.SetInsertPoint(LoopMiddleBlock->getTerminator()); Value *lastIncomingValue = - getOrCreateScalarValue(IncomingValue, {UF - 1, VF - 1}); + getOrCreateScalarValue(IncomingValue, { UF - 1, LastLane }); LCSSAPhi.addIncoming(lastIncomingValue, LoopMiddleBlock); } } @@ -4504,20 +4511,22 @@ void LoopVectorizationCostModel::collectLoopUniforms(unsigned VF) { } // Expand Worklist in topological order: whenever a new instruction - // is added , its users should be either already inside Worklist, or - // out of scope. It ensures a uniform instruction will only be used - // by uniform instructions or out of scope instructions. + // is added , its users should be already inside Worklist. It ensures + // a uniform instruction will only be used by uniform instructions. unsigned idx = 0; while (idx != Worklist.size()) { Instruction *I = Worklist[idx++]; for (auto OV : I->operand_values()) { + // isOutOfScope operands cannot be uniform instructions. if (isOutOfScope(OV)) continue; + // If all the users of the operand are uniform, then add the + // operand into the uniform worklist. auto *OI = cast(OV); if (llvm::all_of(OI->users(), [&](User *U) -> bool { auto *J = cast(U); - return !TheLoop->contains(J) || Worklist.count(J) || + return Worklist.count(J) || (OI == getLoadStorePointerOperand(J) && isUniformDecision(J, VF)); })) { -- cgit v1.2.3