diff options
| -rw-r--r-- | llvm/include/llvm/Transforms/Utils/LoopUtils.h | 115 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Scalar/LoopInterchange.cpp | 8 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Utils/LoopUtils.cpp | 132 | ||||
| -rw-r--r-- | llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 32 |
4 files changed, 150 insertions, 137 deletions
diff --git a/llvm/include/llvm/Transforms/Utils/LoopUtils.h b/llvm/include/llvm/Transforms/Utils/LoopUtils.h index 28791f5f43a..3936f1d5937 100644 --- a/llvm/include/llvm/Transforms/Utils/LoopUtils.h +++ b/llvm/include/llvm/Transforms/Utils/LoopUtils.h @@ -43,12 +43,23 @@ struct LICMSafetyInfo { {} }; -/// This POD struct holds information about a potential reduction operation. -class ReductionInstDesc { +/// The RecurrenceDescriptor is used to identify recurrences variables in a +/// loop. Reduction is a special case of recurrence that has uses of the +/// recurrence variable outside the loop. The method isReductionPHI identifies +/// reductions that are basic recurrences. +/// +/// Basic recurrences are defined as the summation, product, OR, AND, XOR, min, +/// or max of a set of terms. For example: for(i=0; i<n; i++) { total += +/// array[i]; } is a summation of array elements. Basic recurrences are a +/// special case of chains of recurrences (CR). See ScalarEvolution for CR +/// references. + +/// This POD struct holds information about a potential recurrence operation. +class RecurrenceInstDesc { public: - // This enum represents the kind of minmax reduction. - enum MinMaxReductionKind { + // This enum represents the kind of minmax recurrence. + enum MinMaxRecurrenceKind { MRK_Invalid, MRK_UIntMin, MRK_UIntMax, @@ -57,35 +68,35 @@ public: MRK_FloatMin, MRK_FloatMax }; - ReductionInstDesc(bool IsRedux, Instruction *I) - : IsReduction(IsRedux), PatternLastInst(I), MinMaxKind(MRK_Invalid) {} + RecurrenceInstDesc(bool IsRecur, Instruction *I) + : IsRecurrence(IsRecur), PatternLastInst(I), MinMaxKind(MRK_Invalid) {} + + RecurrenceInstDesc(Instruction *I, MinMaxRecurrenceKind K) + : IsRecurrence(true), PatternLastInst(I), MinMaxKind(K) {} - ReductionInstDesc(Instruction *I, MinMaxReductionKind K) - : IsReduction(true), PatternLastInst(I), MinMaxKind(K) {} + bool isRecurrence() { return IsRecurrence; } - bool isReduction() { return IsReduction; } + MinMaxRecurrenceKind getMinMaxKind() { return MinMaxKind; } - MinMaxReductionKind getMinMaxKind() { return MinMaxKind; } - Instruction *getPatternInst() { return PatternLastInst; } private: - // Is this instruction a reduction candidate. - bool IsReduction; + // Is this instruction a recurrence candidate. + bool IsRecurrence; // The last instruction in a min/max pattern (select of the select(icmp()) - // pattern), or the current reduction instruction otherwise. + // pattern), or the current recurrence instruction otherwise. Instruction *PatternLastInst; // If this is a min/max pattern the comparison predicate. - MinMaxReductionKind MinMaxKind; + MinMaxRecurrenceKind MinMaxKind; }; -/// This struct holds information about reduction variables. -class ReductionDescriptor { +/// This struct holds information about recurrence variables. +class RecurrenceDescriptor { public: - /// This enum represents the kinds of reductions that we support. - enum ReductionKind { - RK_NoReduction, ///< Not a reduction. + /// This enum represents the kinds of recurrences that we support. + enum RecurrenceKind { + RK_NoRecurrence, ///< Not a recurrence. RK_IntegerAdd, ///< Sum of integers. RK_IntegerMult, ///< Product of integers. RK_IntegerOr, ///< Bitwise or logical OR of numbers. @@ -97,22 +108,23 @@ public: RK_FloatMinMax ///< Min/max implemented in terms of select(cmp()). }; - ReductionDescriptor() - : StartValue(nullptr), LoopExitInstr(nullptr), Kind(RK_NoReduction), - MinMaxKind(ReductionInstDesc::MRK_Invalid) {} + RecurrenceDescriptor() + : StartValue(nullptr), LoopExitInstr(nullptr), Kind(RK_NoRecurrence), + MinMaxKind(RecurrenceInstDesc::MRK_Invalid) {} - ReductionDescriptor(Value *Start, Instruction *Exit, ReductionKind K, - ReductionInstDesc::MinMaxReductionKind MK) + RecurrenceDescriptor(Value *Start, Instruction *Exit, RecurrenceKind K, + RecurrenceInstDesc::MinMaxRecurrenceKind MK) : StartValue(Start), LoopExitInstr(Exit), Kind(K), MinMaxKind(MK) {} - /// Returns a struct describing if the instruction 'I' can be a reduction - /// variable of type 'Kind'. If the reduction is a min/max pattern of + /// Returns a struct describing if the instruction 'I' can be a recurrence + /// variable of type 'Kind'. If the recurrence is a min/max pattern of /// select(icmp()) this function advances the instruction pointer 'I' from the /// compare instruction to the select instruction and stores this pointer in /// 'PatternLastInst' member of the returned struct. - static ReductionInstDesc isReductionInstr(Instruction *I, ReductionKind Kind, - ReductionInstDesc &Prev, - bool HasFunNoNaNAttr); + static RecurrenceInstDesc isRecurrenceInstr(Instruction *I, + RecurrenceKind Kind, + RecurrenceInstDesc &Prev, + bool HasFunNoNaNAttr); /// Returns true if instuction I has multiple uses in Insts static bool hasMultipleUsesOf(Instruction *I, @@ -124,51 +136,52 @@ public: /// Returns a struct describing if the instruction if the instruction is a /// Select(ICmp(X, Y), X, Y) instruction pattern corresponding to a min(X, Y) /// or max(X, Y). - static ReductionInstDesc isMinMaxSelectCmpPattern(Instruction *I, - ReductionInstDesc &Prev); + static RecurrenceInstDesc isMinMaxSelectCmpPattern(Instruction *I, + RecurrenceInstDesc &Prev); - /// Returns identity corresponding to the ReductionKind. - static Constant *getReductionIdentity(ReductionKind K, Type *Tp); + /// Returns identity corresponding to the RecurrenceKind. + static Constant *getRecurrenceIdentity(RecurrenceKind K, Type *Tp); - /// Returns the opcode of binary operation corresponding to the ReductionKind. - static unsigned getReductionBinOp(ReductionKind Kind); + /// Returns the opcode of binary operation corresponding to the + /// RecurrenceKind. + static unsigned getRecurrenceBinOp(RecurrenceKind Kind); - /// Returns a Min/Max operation corresponding to MinMaxReductionKind. + /// Returns a Min/Max operation corresponding to MinMaxRecurrenceKind. static Value *createMinMaxOp(IRBuilder<> &Builder, - ReductionInstDesc::MinMaxReductionKind RK, + RecurrenceInstDesc::MinMaxRecurrenceKind RK, Value *Left, Value *Right); /// Returns true if Phi is a reduction of type Kind and adds it to the - /// ReductionDescriptor. - static bool AddReductionVar(PHINode *Phi, ReductionKind Kind, Loop *TheLoop, + /// RecurrenceDescriptor. + static bool AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Loop *TheLoop, bool HasFunNoNaNAttr, - ReductionDescriptor &RedDes); + RecurrenceDescriptor &RedDes); - /// Returns true if Phi is a reduction in TheLoop. The ReductionDescriptor is + /// Returns true if Phi is a reduction in TheLoop. The RecurrenceDescriptor is /// returned in RedDes. static bool isReductionPHI(PHINode *Phi, Loop *TheLoop, - ReductionDescriptor &RedDes); + RecurrenceDescriptor &RedDes); - ReductionKind getReductionKind() { return Kind; } + RecurrenceKind getRecurrenceKind() { return Kind; } - ReductionInstDesc::MinMaxReductionKind getMinMaxReductionKind() { + RecurrenceInstDesc::MinMaxRecurrenceKind getMinMaxRecurrenceKind() { return MinMaxKind; } - TrackingVH<Value> getReductionStartValue() { return StartValue; } + TrackingVH<Value> getRecurrenceStartValue() { return StartValue; } Instruction *getLoopExitInstr() { return LoopExitInstr; } private: - // The starting value of the reduction. + // The starting value of the recurrence. // It does not have to be zero! TrackingVH<Value> StartValue; // The instruction who's value is used outside the loop. Instruction *LoopExitInstr; - // The kind of the reduction. - ReductionKind Kind; - // If this a min/max reduction the kind of reduction. - ReductionInstDesc::MinMaxReductionKind MinMaxKind; + // The kind of the recurrence. + RecurrenceKind Kind; + // If this a min/max recurrence the kind of recurrence. + RecurrenceInstDesc::MinMaxRecurrenceKind MinMaxKind; }; BasicBlock *InsertPreheaderForLoop(Loop *L, Pass *P); diff --git a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp index f584018299d..25546553fd4 100644 --- a/llvm/lib/Transforms/Scalar/LoopInterchange.cpp +++ b/llvm/lib/Transforms/Scalar/LoopInterchange.cpp @@ -598,8 +598,8 @@ struct LoopInterchange : public FunctionPass { bool LoopInterchangeLegality::areAllUsesReductions(Instruction *Ins, Loop *L) { return !std::any_of(Ins->user_begin(), Ins->user_end(), [=](User *U) -> bool { PHINode *UserIns = dyn_cast<PHINode>(U); - ReductionDescriptor RD; - return !UserIns || !ReductionDescriptor::isReductionPHI(UserIns, L, RD); + RecurrenceDescriptor RD; + return !UserIns || !RecurrenceDescriptor::isReductionPHI(UserIns, L, RD); }); } @@ -697,12 +697,12 @@ bool LoopInterchangeLegality::findInductionAndReductions( if (!L->getLoopLatch() || !L->getLoopPredecessor()) return false; for (BasicBlock::iterator I = L->getHeader()->begin(); isa<PHINode>(I); ++I) { - ReductionDescriptor RD; + RecurrenceDescriptor RD; PHINode *PHI = cast<PHINode>(I); ConstantInt *StepValue = nullptr; if (isInductionPHI(PHI, SE, StepValue)) Inductions.push_back(PHI); - else if (ReductionDescriptor::isReductionPHI(PHI, L, RD)) + else if (RecurrenceDescriptor::isReductionPHI(PHI, L, RD)) Reductions.push_back(PHI); else { DEBUG( diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index 5f25e6b2cb6..c37ecfc881f 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -26,17 +26,17 @@ using namespace llvm::PatternMatch; #define DEBUG_TYPE "loop-utils" -bool ReductionDescriptor::areAllUsesIn(Instruction *I, - SmallPtrSetImpl<Instruction *> &Set) { +bool RecurrenceDescriptor::areAllUsesIn(Instruction *I, + SmallPtrSetImpl<Instruction *> &Set) { for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; ++Use) if (!Set.count(dyn_cast<Instruction>(*Use))) return false; return true; } -bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, - Loop *TheLoop, bool HasFunNoNaNAttr, - ReductionDescriptor &RedDes) { +bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, + Loop *TheLoop, bool HasFunNoNaNAttr, + RecurrenceDescriptor &RedDes) { if (Phi->getNumIncomingValues() != 2) return false; @@ -66,7 +66,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, // the number of instruction we saw from the recognized min/max pattern, // to make sure we only see exactly the two instructions. unsigned NumCmpSelectPatternInst = 0; - ReductionInstDesc ReduxDesc(false, nullptr); + RecurrenceInstDesc ReduxDesc(false, nullptr); SmallPtrSet<Instruction *, 8> VisitedInsts; SmallVector<Instruction *, 8> Worklist; @@ -111,8 +111,8 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, return false; // Any reduction instruction must be of one of the allowed kinds. - ReduxDesc = isReductionInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr); - if (!ReduxDesc.isReduction()) + ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr); + if (!ReduxDesc.isRecurrence()) return false; // A reduction operation must only have one use of the reduction value. @@ -164,7 +164,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, // Process instructions only once (termination). Each reduction cycle // value must only be used once, except by phi nodes and min/max // reductions which are represented as a cmp followed by a select. - ReductionInstDesc IgnoredVal(false, nullptr); + RecurrenceInstDesc IgnoredVal(false, nullptr); if (VisitedInsts.insert(UI).second) { if (isa<PHINode>(UI)) PHIs.push_back(UI); @@ -173,7 +173,7 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, } else if (!isa<PHINode>(UI) && ((!isa<FCmpInst>(UI) && !isa<ICmpInst>(UI) && !isa<SelectInst>(UI)) || - !isMinMaxSelectCmpPattern(UI, IgnoredVal).isReduction())) + !isMinMaxSelectCmpPattern(UI, IgnoredVal).isRecurrence())) return false; // Remember that we completed the cycle. @@ -197,11 +197,11 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, // only have a single instruction with out-of-loop users. // The ExitInstruction(Instruction which is allowed to have out-of-loop users) - // is saved as part of the ReductionDescriptor. + // is saved as part of the RecurrenceDescriptor. // Save the description of this reduction variable. - ReductionDescriptor RD(RdxStart, ExitInstruction, Kind, - ReduxDesc.getMinMaxKind()); + RecurrenceDescriptor RD(RdxStart, ExitInstruction, Kind, + ReduxDesc.getMinMaxKind()); RedDes = RD; @@ -210,9 +210,9 @@ bool ReductionDescriptor::AddReductionVar(PHINode *Phi, ReductionKind Kind, /// Returns true if the instruction is a Select(ICmp(X, Y), X, Y) instruction /// pattern corresponding to a min(X, Y) or max(X, Y). -ReductionInstDesc -ReductionDescriptor::isMinMaxSelectCmpPattern(Instruction *I, - ReductionInstDesc &Prev) { +RecurrenceInstDesc +RecurrenceDescriptor::isMinMaxSelectCmpPattern(Instruction *I, + RecurrenceInstDesc &Prev) { assert((isa<ICmpInst>(I) || isa<FCmpInst>(I) || isa<SelectInst>(I)) && "Expect a select instruction"); @@ -223,84 +223,84 @@ ReductionDescriptor::isMinMaxSelectCmpPattern(Instruction *I, // select. if ((Cmp = dyn_cast<ICmpInst>(I)) || (Cmp = dyn_cast<FCmpInst>(I))) { if (!Cmp->hasOneUse() || !(Select = dyn_cast<SelectInst>(*I->user_begin()))) - return ReductionInstDesc(false, I); - return ReductionInstDesc(Select, Prev.getMinMaxKind()); + return RecurrenceInstDesc(false, I); + return RecurrenceInstDesc(Select, Prev.getMinMaxKind()); } // Only handle single use cases for now. if (!(Select = dyn_cast<SelectInst>(I))) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); if (!(Cmp = dyn_cast<ICmpInst>(I->getOperand(0))) && !(Cmp = dyn_cast<FCmpInst>(I->getOperand(0)))) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); if (!Cmp->hasOneUse()) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); Value *CmpLeft; Value *CmpRight; // Look for a min/max pattern. if (m_UMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_UIntMin); else if (m_UMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_UIntMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_UIntMax); else if (m_SMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_SIntMax); else if (m_SMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_SIntMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_SIntMin); else if (m_OrdFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMin); else if (m_OrdFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMax); else if (m_UnordFMin(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMin); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMin); else if (m_UnordFMax(m_Value(CmpLeft), m_Value(CmpRight)).match(Select)) - return ReductionInstDesc(Select, ReductionInstDesc::MRK_FloatMax); + return RecurrenceInstDesc(Select, RecurrenceInstDesc::MRK_FloatMax); - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); } -ReductionInstDesc ReductionDescriptor::isReductionInstr(Instruction *I, - ReductionKind Kind, - ReductionInstDesc &Prev, - bool HasFunNoNaNAttr) { +RecurrenceInstDesc +RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, + RecurrenceInstDesc &Prev, + bool HasFunNoNaNAttr) { bool FP = I->getType()->isFloatingPointTy(); bool FastMath = FP && I->hasUnsafeAlgebra(); switch (I->getOpcode()) { default: - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); case Instruction::PHI: if (FP && (Kind != RK_FloatMult && Kind != RK_FloatAdd && Kind != RK_FloatMinMax)) - return ReductionInstDesc(false, I); - return ReductionInstDesc(I, Prev.getMinMaxKind()); + return RecurrenceInstDesc(false, I); + return RecurrenceInstDesc(I, Prev.getMinMaxKind()); case Instruction::Sub: case Instruction::Add: - return ReductionInstDesc(Kind == RK_IntegerAdd, I); + return RecurrenceInstDesc(Kind == RK_IntegerAdd, I); case Instruction::Mul: - return ReductionInstDesc(Kind == RK_IntegerMult, I); + return RecurrenceInstDesc(Kind == RK_IntegerMult, I); case Instruction::And: - return ReductionInstDesc(Kind == RK_IntegerAnd, I); + return RecurrenceInstDesc(Kind == RK_IntegerAnd, I); case Instruction::Or: - return ReductionInstDesc(Kind == RK_IntegerOr, I); + return RecurrenceInstDesc(Kind == RK_IntegerOr, I); case Instruction::Xor: - return ReductionInstDesc(Kind == RK_IntegerXor, I); + return RecurrenceInstDesc(Kind == RK_IntegerXor, I); case Instruction::FMul: - return ReductionInstDesc(Kind == RK_FloatMult && FastMath, I); + return RecurrenceInstDesc(Kind == RK_FloatMult && FastMath, I); case Instruction::FSub: case Instruction::FAdd: - return ReductionInstDesc(Kind == RK_FloatAdd && FastMath, I); + return RecurrenceInstDesc(Kind == RK_FloatAdd && FastMath, I); case Instruction::FCmp: case Instruction::ICmp: case Instruction::Select: if (Kind != RK_IntegerMinMax && (!HasFunNoNaNAttr || Kind != RK_FloatMinMax)) - return ReductionInstDesc(false, I); + return RecurrenceInstDesc(false, I); return isMinMaxSelectCmpPattern(I, Prev); } } -bool ReductionDescriptor::hasMultipleUsesOf( +bool RecurrenceDescriptor::hasMultipleUsesOf( Instruction *I, SmallPtrSetImpl<Instruction *> &Insts) { unsigned NumUses = 0; for (User::op_iterator Use = I->op_begin(), E = I->op_end(); Use != E; @@ -313,8 +313,8 @@ bool ReductionDescriptor::hasMultipleUsesOf( return false; } -bool ReductionDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, - ReductionDescriptor &RedDes) { +bool RecurrenceDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, + RecurrenceDescriptor &RedDes) { bool HasFunNoNaNAttr = false; BasicBlock *Header = TheLoop->getHeader(); @@ -366,7 +366,8 @@ bool ReductionDescriptor::isReductionPHI(PHINode *Phi, Loop *TheLoop, /// This function returns the identity element (or neutral element) for /// the operation K. -Constant *ReductionDescriptor::getReductionIdentity(ReductionKind K, Type *Tp) { +Constant *RecurrenceDescriptor::getRecurrenceIdentity(RecurrenceKind K, + Type *Tp) { switch (K) { case RK_IntegerXor: case RK_IntegerAdd: @@ -386,12 +387,12 @@ Constant *ReductionDescriptor::getReductionIdentity(ReductionKind K, Type *Tp) { // Adding zero to a number does not change it. return ConstantFP::get(Tp, 0.0L); default: - llvm_unreachable("Unknown reduction kind"); + llvm_unreachable("Unknown recurrence kind"); } } -/// This function translates the reduction kind to an LLVM binary operator. -unsigned ReductionDescriptor::getReductionBinOp(ReductionKind Kind) { +/// This function translates the recurrence kind to an LLVM binary operator. +unsigned RecurrenceDescriptor::getRecurrenceBinOp(RecurrenceKind Kind) { switch (Kind) { case RK_IntegerAdd: return Instruction::Add; @@ -412,41 +413,40 @@ unsigned ReductionDescriptor::getReductionBinOp(ReductionKind Kind) { case RK_FloatMinMax: return Instruction::FCmp; default: - llvm_unreachable("Unknown reduction operation"); + llvm_unreachable("Unknown recurrence operation"); } } -Value * -ReductionDescriptor::createMinMaxOp(IRBuilder<> &Builder, - ReductionInstDesc::MinMaxReductionKind RK, - Value *Left, Value *Right) { +Value *RecurrenceDescriptor::createMinMaxOp( + IRBuilder<> &Builder, RecurrenceInstDesc::MinMaxRecurrenceKind RK, + Value *Left, Value *Right) { CmpInst::Predicate P = CmpInst::ICMP_NE; switch (RK) { default: - llvm_unreachable("Unknown min/max reduction kind"); - case ReductionInstDesc::MRK_UIntMin: + llvm_unreachable("Unknown min/max recurrence kind"); + case RecurrenceInstDesc::MRK_UIntMin: P = CmpInst::ICMP_ULT; break; - case ReductionInstDesc::MRK_UIntMax: + case RecurrenceInstDesc::MRK_UIntMax: P = CmpInst::ICMP_UGT; break; - case ReductionInstDesc::MRK_SIntMin: + case RecurrenceInstDesc::MRK_SIntMin: P = CmpInst::ICMP_SLT; break; - case ReductionInstDesc::MRK_SIntMax: + case RecurrenceInstDesc::MRK_SIntMax: P = CmpInst::ICMP_SGT; break; - case ReductionInstDesc::MRK_FloatMin: + case RecurrenceInstDesc::MRK_FloatMin: P = CmpInst::FCMP_OLT; break; - case ReductionInstDesc::MRK_FloatMax: + case RecurrenceInstDesc::MRK_FloatMax: P = CmpInst::FCMP_OGT; break; } Value *Cmp; - if (RK == ReductionInstDesc::MRK_FloatMin || - RK == ReductionInstDesc::MRK_FloatMax) + if (RK == RecurrenceInstDesc::MRK_FloatMin || + RK == RecurrenceInstDesc::MRK_FloatMax) Cmp = Builder.CreateFCmp(P, Left, Right, "rdx.minmax.cmp"); else Cmp = Builder.CreateICmp(P, Left, Right, "rdx.minmax.cmp"); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index 95c9381985a..62de2008916 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -872,7 +872,7 @@ public: /// ReductionList contains the reduction descriptors for all /// of the reductions that were found in the loop. - typedef DenseMap<PHINode*, ReductionDescriptor> ReductionList; + typedef DenseMap<PHINode *, RecurrenceDescriptor> ReductionList; /// InductionList saves induction variables and maps them to the /// induction descriptor. @@ -3093,13 +3093,13 @@ void InnerLoopVectorizer::vectorizeLoop() { // Find the reduction variable descriptor. assert(Legal->getReductionVars()->count(RdxPhi) && "Unable to find the reduction variable"); - ReductionDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi]; + RecurrenceDescriptor RdxDesc = (*Legal->getReductionVars())[RdxPhi]; - ReductionDescriptor::ReductionKind RK = RdxDesc.getReductionKind(); - TrackingVH<Value> ReductionStartValue = RdxDesc.getReductionStartValue(); + RecurrenceDescriptor::RecurrenceKind RK = RdxDesc.getRecurrenceKind(); + TrackingVH<Value> ReductionStartValue = RdxDesc.getRecurrenceStartValue(); Instruction *LoopExitInst = RdxDesc.getLoopExitInstr(); - ReductionInstDesc::MinMaxReductionKind MinMaxKind = - RdxDesc.getMinMaxReductionKind(); + RecurrenceInstDesc::MinMaxRecurrenceKind MinMaxKind = + RdxDesc.getMinMaxRecurrenceKind(); setDebugLocFromInst(Builder, ReductionStartValue); // We need to generate a reduction vector from the incoming scalar. @@ -3116,8 +3116,8 @@ void InnerLoopVectorizer::vectorizeLoop() { // one for multiplication, -1 for And. Value *Identity; Value *VectorStart; - if (RK == ReductionDescriptor::RK_IntegerMinMax || - RK == ReductionDescriptor::RK_FloatMinMax) { + if (RK == RecurrenceDescriptor::RK_IntegerMinMax || + RK == RecurrenceDescriptor::RK_FloatMinMax) { // MinMax reduction have the start value as their identify. if (VF == 1) { VectorStart = Identity = ReductionStartValue; @@ -3127,8 +3127,8 @@ void InnerLoopVectorizer::vectorizeLoop() { } } else { // Handle other reduction kinds: - Constant *Iden = - ReductionDescriptor::getReductionIdentity(RK, VecTy->getScalarType()); + Constant *Iden = RecurrenceDescriptor::getRecurrenceIdentity( + RK, VecTy->getScalarType()); if (VF == 1) { Identity = Iden; // This vector is the Identity vector where the first element is the @@ -3185,7 +3185,7 @@ void InnerLoopVectorizer::vectorizeLoop() { // Reduce all of the unrolled parts into a single vector. Value *ReducedPartRdx = RdxParts[0]; - unsigned Op = ReductionDescriptor::getReductionBinOp(RK); + unsigned Op = RecurrenceDescriptor::getRecurrenceBinOp(RK); setDebugLocFromInst(Builder, ReducedPartRdx); for (unsigned part = 1; part < UF; ++part) { if (Op != Instruction::ICmp && Op != Instruction::FCmp) @@ -3194,7 +3194,7 @@ void InnerLoopVectorizer::vectorizeLoop() { Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxParts[part], ReducedPartRdx, "bin.rdx")); else - ReducedPartRdx = ReductionDescriptor::createMinMaxOp( + ReducedPartRdx = RecurrenceDescriptor::createMinMaxOp( Builder, MinMaxKind, ReducedPartRdx, RdxParts[part]); } @@ -3226,8 +3226,8 @@ void InnerLoopVectorizer::vectorizeLoop() { TmpVec = addFastMathFlag(Builder.CreateBinOp( (Instruction::BinaryOps)Op, TmpVec, Shuf, "bin.rdx")); else - TmpVec = ReductionDescriptor::createMinMaxOp(Builder, MinMaxKind, - TmpVec, Shuf); + TmpVec = RecurrenceDescriptor::createMinMaxOp(Builder, MinMaxKind, + TmpVec, Shuf); } // The result is in the first element of the vector. @@ -4040,8 +4040,8 @@ bool LoopVectorizationLegality::canVectorizeInstrs() { continue; } - if (ReductionDescriptor::isReductionPHI(Phi, TheLoop, - Reductions[Phi])) { + if (RecurrenceDescriptor::isReductionPHI(Phi, TheLoop, + Reductions[Phi])) { AllowedExit.insert(Reductions[Phi].getLoopExitInstr()); continue; } |
