diff options
Diffstat (limited to 'llvm/lib')
-rw-r--r-- | llvm/lib/Analysis/IVDescriptors.cpp | 10 | ||||
-rw-r--r-- | llvm/lib/CodeGen/ExpandReductions.cpp | 4 | ||||
-rw-r--r-- | llvm/lib/Transforms/Utils/LoopUtils.cpp | 46 | ||||
-rw-r--r-- | llvm/lib/Transforms/Vectorize/LoopVectorize.cpp | 16 | ||||
-rw-r--r-- | llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 3 |
5 files changed, 49 insertions, 30 deletions
diff --git a/llvm/lib/Analysis/IVDescriptors.cpp b/llvm/lib/Analysis/IVDescriptors.cpp index 555e3c9ed45..19f1a771b85 100644 --- a/llvm/lib/Analysis/IVDescriptors.cpp +++ b/llvm/lib/Analysis/IVDescriptors.cpp @@ -251,6 +251,10 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, Worklist.push_back(Start); VisitedInsts.insert(Start); + // Start with all flags set because we will intersect this with the reduction + // flags from all the reduction operations. + FastMathFlags FMF = FastMathFlags::getFast(); + // A value in the reduction can be used: // - By the reduction: // - Reduction operation: @@ -296,6 +300,8 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, ReduxDesc = isRecurrenceInstr(Cur, Kind, ReduxDesc, HasFunNoNaNAttr); if (!ReduxDesc.isRecurrence()) return false; + if (isa<FPMathOperator>(ReduxDesc.getPatternInst())) + FMF &= ReduxDesc.getPatternInst()->getFastMathFlags(); } bool IsASelect = isa<SelectInst>(Cur); @@ -441,7 +447,7 @@ bool RecurrenceDescriptor::AddReductionVar(PHINode *Phi, RecurrenceKind Kind, // Save the description of this reduction variable. RecurrenceDescriptor RD( - RdxStart, ExitInstruction, Kind, ReduxDesc.getMinMaxKind(), + RdxStart, ExitInstruction, Kind, FMF, ReduxDesc.getMinMaxKind(), ReduxDesc.getUnsafeAlgebraInst(), RecurrenceType, IsSigned, CastInsts); RedDes = RD; @@ -550,7 +556,7 @@ RecurrenceDescriptor::InstDesc RecurrenceDescriptor::isRecurrenceInstr(Instruction *I, RecurrenceKind Kind, InstDesc &Prev, bool HasFunNoNaNAttr) { Instruction *UAI = Prev.getUnsafeAlgebraInst(); - if (!UAI && isa<FPMathOperator>(I) && !I->isFast()) + if (!UAI && isa<FPMathOperator>(I) && !I->hasAllowReassoc()) UAI = I; // Found an unsafe (unvectorizable) algebra instruction. switch (I->getOpcode()) { diff --git a/llvm/lib/CodeGen/ExpandReductions.cpp b/llvm/lib/CodeGen/ExpandReductions.cpp index f44d89156b6..34858883298 100644 --- a/llvm/lib/CodeGen/ExpandReductions.cpp +++ b/llvm/lib/CodeGen/ExpandReductions.cpp @@ -118,9 +118,11 @@ bool expandReductions(Function &F, const TargetTransformInfo *TTI) { } if (!TTI->shouldExpandReduction(II)) continue; + FastMathFlags FMF = + isa<FPMathOperator>(II) ? II->getFastMathFlags() : FastMathFlags{}; Value *Rdx = IsOrdered ? getOrderedReduction(Builder, Acc, Vec, getOpcode(ID), MRK) - : getShuffleReduction(Builder, Vec, getOpcode(ID), MRK); + : getShuffleReduction(Builder, Vec, getOpcode(ID), MRK, FMF); II->replaceAllUsesWith(Rdx); II->eraseFromParent(); Changed = true; diff --git a/llvm/lib/Transforms/Utils/LoopUtils.cpp b/llvm/lib/Transforms/Utils/LoopUtils.cpp index c3a02685244..045cb6284c4 100644 --- a/llvm/lib/Transforms/Utils/LoopUtils.cpp +++ b/llvm/lib/Transforms/Utils/LoopUtils.cpp @@ -671,13 +671,9 @@ bool llvm::hasIterationCountInvariantInParent(Loop *InnerLoop, return true; } -/// Adds a 'fast' flag to floating point operations. -static Value *addFastMathFlag(Value *V) { - if (isa<FPMathOperator>(V)) { - FastMathFlags Flags; - Flags.setFast(); - cast<Instruction>(V)->setFastMathFlags(Flags); - } +static Value *addFastMathFlag(Value *V, FastMathFlags FMF) { + if (isa<FPMathOperator>(V)) + cast<Instruction>(V)->setFastMathFlags(FMF); return V; } @@ -761,7 +757,7 @@ llvm::getOrderedReduction(IRBuilder<> &Builder, Value *Acc, Value *Src, Value * llvm::getShuffleReduction(IRBuilder<> &Builder, Value *Src, unsigned Op, RecurrenceDescriptor::MinMaxRecurrenceKind MinMaxKind, - ArrayRef<Value *> RedOps) { + FastMathFlags FMF, ArrayRef<Value *> RedOps) { unsigned VF = Src->getType()->getVectorNumElements(); // VF is a power of 2 so we can emit the reduction using log2(VF) shuffles // and vector ops, reducing the set of values being computed by half each @@ -786,7 +782,8 @@ llvm::getShuffleReduction(IRBuilder<> &Builder, Value *Src, unsigned Op, if (Op != Instruction::ICmp && Op != Instruction::FCmp) { // Floating point operations had to be 'fast' to enable the reduction. TmpVec = addFastMathFlag(Builder.CreateBinOp((Instruction::BinaryOps)Op, - TmpVec, Shuf, "bin.rdx")); + TmpVec, Shuf, "bin.rdx"), + FMF); } else { assert(MinMaxKind != RecurrenceDescriptor::MRK_Invalid && "Invalid min/max"); @@ -803,7 +800,7 @@ llvm::getShuffleReduction(IRBuilder<> &Builder, Value *Src, unsigned Op, /// flags (if generating min/max reductions). Value *llvm::createSimpleTargetReduction( IRBuilder<> &Builder, const TargetTransformInfo *TTI, unsigned Opcode, - Value *Src, TargetTransformInfo::ReductionFlags Flags, + Value *Src, TargetTransformInfo::ReductionFlags Flags, FastMathFlags FMF, ArrayRef<Value *> RedOps) { assert(isa<VectorType>(Src->getType()) && "Type must be a vector"); @@ -873,7 +870,7 @@ Value *llvm::createSimpleTargetReduction( } if (TTI->useReductionIntrinsic(Opcode, Src->getType(), Flags)) return BuildFunc(); - return getShuffleReduction(Builder, Src, Opcode, MinMaxKind, RedOps); + return getShuffleReduction(Builder, Src, Opcode, MinMaxKind, FMF, RedOps); } /// Create a vector reduction using a given recurrence descriptor. @@ -888,28 +885,37 @@ Value *llvm::createTargetReduction(IRBuilder<> &B, Flags.NoNaN = NoNaN; switch (RecKind) { case RD::RK_FloatAdd: - return createSimpleTargetReduction(B, TTI, Instruction::FAdd, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::FAdd, Src, Flags, + Desc.getFastMathFlags()); case RD::RK_FloatMult: - return createSimpleTargetReduction(B, TTI, Instruction::FMul, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::FMul, Src, Flags, + Desc.getFastMathFlags()); case RD::RK_IntegerAdd: - return createSimpleTargetReduction(B, TTI, Instruction::Add, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::Add, Src, Flags, + Desc.getFastMathFlags()); case RD::RK_IntegerMult: - return createSimpleTargetReduction(B, TTI, Instruction::Mul, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::Mul, Src, Flags, + Desc.getFastMathFlags()); case RD::RK_IntegerAnd: - return createSimpleTargetReduction(B, TTI, Instruction::And, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::And, Src, Flags, + Desc.getFastMathFlags()); case RD::RK_IntegerOr: - return createSimpleTargetReduction(B, TTI, Instruction::Or, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::Or, Src, Flags, + Desc.getFastMathFlags()); case RD::RK_IntegerXor: - return createSimpleTargetReduction(B, TTI, Instruction::Xor, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::Xor, Src, Flags, + Desc.getFastMathFlags()); case RD::RK_IntegerMinMax: { RD::MinMaxRecurrenceKind MMKind = Desc.getMinMaxRecurrenceKind(); Flags.IsMaxOp = (MMKind == RD::MRK_SIntMax || MMKind == RD::MRK_UIntMax); Flags.IsSigned = (MMKind == RD::MRK_SIntMax || MMKind == RD::MRK_SIntMin); - return createSimpleTargetReduction(B, TTI, Instruction::ICmp, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::ICmp, Src, Flags, + Desc.getFastMathFlags()); } case RD::RK_FloatMinMax: { Flags.IsMaxOp = Desc.getMinMaxRecurrenceKind() == RD::MRK_FloatMax; - return createSimpleTargetReduction(B, TTI, Instruction::FCmp, Src, Flags); + return createSimpleTargetReduction(B, TTI, Instruction::FCmp, Src, Flags, + Desc.getFastMathFlags()); } default: llvm_unreachable("Unhandled RecKind"); diff --git a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp index b10ce86b682..b57012291df 100644 --- a/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp +++ b/llvm/lib/Transforms/Vectorize/LoopVectorize.cpp @@ -319,11 +319,14 @@ static unsigned getReciprocalPredBlockProb() { return 2; } /// A helper function that adds a 'fast' flag to floating-point operations. static Value *addFastMathFlag(Value *V) { - if (isa<FPMathOperator>(V)) { - FastMathFlags Flags; - Flags.setFast(); - cast<Instruction>(V)->setFastMathFlags(Flags); - } + if (isa<FPMathOperator>(V)) + cast<Instruction>(V)->setFastMathFlags(FastMathFlags::getFast()); + return V; +} + +static Value *addFastMathFlag(Value *V, FastMathFlags FMF) { + if (isa<FPMathOperator>(V)) + cast<Instruction>(V)->setFastMathFlags(FMF); return V; } @@ -3612,7 +3615,8 @@ void InnerLoopVectorizer::fixReduction(PHINode *Phi) { // Floating point operations had to be 'fast' to enable the reduction. ReducedPartRdx = addFastMathFlag( Builder.CreateBinOp((Instruction::BinaryOps)Op, RdxPart, - ReducedPartRdx, "bin.rdx")); + ReducedPartRdx, "bin.rdx"), + RdxDesc.getFastMathFlags()); else ReducedPartRdx = createMinMaxOp(Builder, MinMaxKind, ReducedPartRdx, RdxPart); diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 0016e10567e..28ed4e6ff1a 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -5929,7 +5929,8 @@ private: if (!IsPairwiseReduction) return createSimpleTargetReduction( Builder, TTI, ReductionData.getOpcode(), VectorizedValue, - ReductionData.getFlags(), ReductionOps.back()); + ReductionData.getFlags(), FastMathFlags::getFast(), + ReductionOps.back()); Value *TmpVec = VectorizedValue; for (unsigned i = ReduxWidth / 2; i != 0; i >>= 1) { |