summaryrefslogtreecommitdiffstats
path: root/llvm
diff options
context:
space:
mode:
Diffstat (limited to 'llvm')
-rw-r--r--llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp145
1 files changed, 73 insertions, 72 deletions
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 86e94f6d202..dfc3fe7373a 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -340,21 +340,22 @@ namespace {
/// Main data required for vectorization of instructions.
struct InstructionsState {
- /// The very first instruction in the list with the main opcode.
- Value *OpValue = nullptr;
-
- /// The main opcode for the list of instructions.
- unsigned Opcode = 0;
-
- /// Some of the instructions in the list have alternate opcodes.
- bool IsAltShuffle = false;
-
- InstructionsState() = default;
- InstructionsState(Value *OpValue, unsigned Opcode, bool IsAltShuffle)
- : OpValue(OpValue), Opcode(Opcode), IsAltShuffle(IsAltShuffle) {}
-};
-
-} // end anonymous namespace
+ /// The very first instruction in the list with the main opcode.
+ Value *OpValue = nullptr;
+
+ /// The main/alternate opcodes for the list of instructions.
+ unsigned Opcode = 0;
+ unsigned AltOpcode = 0;
+
+ /// Some of the instructions in the list have alternate opcodes.
+ bool isAltShuffle() const { return Opcode != AltOpcode; }
+
+ InstructionsState() = default;
+ InstructionsState(Value *OpValue, unsigned Opcode, unsigned AltOpcode)
+ : OpValue(OpValue), Opcode(Opcode), AltOpcode(AltOpcode) {}
+};
+
+} // end anonymous namespace
/// \returns analysis of the Instructions in \p VL described in
/// InstructionsState, the Opcode that we suppose the whole list
@@ -362,24 +363,25 @@ struct InstructionsState {
static InstructionsState getSameOpcode(ArrayRef<Value *> VL) {
// Make sure these are all Instructions.
if (llvm::any_of(VL, [](Value *V) { return !isa<Instruction>(V); }))
- return InstructionsState(VL[0], 0, false);
+ return InstructionsState(VL[0], 0, 0);
unsigned Opcode = cast<Instruction>(VL[0])->getOpcode();
+ unsigned AltOpcode = Opcode;
bool HasAltOpcodes = llvm::any_of(VL, [Opcode](Value *V) {
return Opcode != cast<Instruction>(V)->getOpcode();
});
// Check for an alternate opcode pattern.
if (HasAltOpcodes) {
- unsigned AltOpcode = getAltOpcode(Opcode);
+ AltOpcode = getAltOpcode(Opcode);
for (int Cnt = 0, E = VL.size(); Cnt < E; Cnt++) {
unsigned InstOpcode = cast<Instruction>(VL[Cnt])->getOpcode();
if (!sameOpcodeOrAlt(Opcode, AltOpcode, InstOpcode))
- return InstructionsState(VL[0], 0, false);
+ return InstructionsState(VL[0], 0, 0);
}
}
- return InstructionsState(VL[0], Opcode, HasAltOpcodes);
+ return InstructionsState(VL[0], Opcode, AltOpcode);
}
/// \returns true if all of the values in \p VL have the same type or false
@@ -1512,13 +1514,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
"tryScheduleBundle should cancelScheduling on failure");
newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
return;
- }
- LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
-
- unsigned ShuffleOrOp = S.IsAltShuffle ?
- (unsigned) Instruction::ShuffleVector : S.Opcode;
- switch (ShuffleOrOp) {
- case Instruction::PHI: {
+ }
+ LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
+
+ unsigned ShuffleOrOp = S.isAltShuffle() ?
+ (unsigned) Instruction::ShuffleVector : S.Opcode;
+ switch (ShuffleOrOp) {
+ case Instruction::PHI: {
PHINode *PH = dyn_cast<PHINode>(VL0);
// Check for terminator values (e.g. invoke).
@@ -1899,26 +1901,25 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
return;
}
- case Instruction::ShuffleVector:
- // If this is not an alternate sequence of opcode like add-sub
- // then do not vectorize this instruction.
- if (!S.IsAltShuffle) {
- BS.cancelScheduling(VL, VL0);
- newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
- LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
+ case Instruction::ShuffleVector:
+ // If this is not an alternate sequence of opcode like add-sub
+ // then do not vectorize this instruction.
+ if (!S.isAltShuffle()) {
+ BS.cancelScheduling(VL, VL0);
+ newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
+ LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
return;
}
newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
- // Reorder operands if reordering would enable vectorization.
- if (isa<BinaryOperator>(VL0)) {
- ValueList Left, Right;
- unsigned AltOpcode = getAltOpcode(S.Opcode);
- reorderAltShuffleOperands(S.Opcode, AltOpcode, VL, Left, Right);
- buildTree_rec(Left, Depth + 1, UserTreeIdx);
- buildTree_rec(Right, Depth + 1, UserTreeIdx);
- return;
+ // Reorder operands if reordering would enable vectorization.
+ if (isa<BinaryOperator>(VL0)) {
+ ValueList Left, Right;
+ reorderAltShuffleOperands(S.Opcode, S.AltOpcode, VL, Left, Right);
+ buildTree_rec(Left, Depth + 1, UserTreeIdx);
+ buildTree_rec(Right, Depth + 1, UserTreeIdx);
+ return;
}
for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
@@ -2091,13 +2092,13 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
}
return ReuseShuffleCost + getGatherCost(VL);
}
- InstructionsState S = getSameOpcode(VL);
- assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
- Instruction *VL0 = cast<Instruction>(S.OpValue);
- unsigned ShuffleOrOp = S.IsAltShuffle ?
- (unsigned) Instruction::ShuffleVector : S.Opcode;
- switch (ShuffleOrOp) {
- case Instruction::PHI:
+ InstructionsState S = getSameOpcode(VL);
+ assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
+ Instruction *VL0 = cast<Instruction>(S.OpValue);
+ unsigned ShuffleOrOp = S.isAltShuffle() ?
+ (unsigned) Instruction::ShuffleVector : S.Opcode;
+ switch (ShuffleOrOp) {
+ case Instruction::PHI:
return 0;
case Instruction::ExtractValue:
@@ -3048,13 +3049,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
}
E->VectorizedValue = V;
- return V;
- }
-
- unsigned ShuffleOrOp = S.IsAltShuffle ?
- (unsigned) Instruction::ShuffleVector : S.Opcode;
- switch (ShuffleOrOp) {
- case Instruction::PHI: {
+ return V;
+ }
+
+ unsigned ShuffleOrOp = S.isAltShuffle() ?
+ (unsigned) Instruction::ShuffleVector : S.Opcode;
+ switch (ShuffleOrOp) {
+ case Instruction::PHI: {
PHINode *PH = dyn_cast<PHINode>(VL0);
Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
Builder.SetCurrentDebugLocation(PH->getDebugLoc());
@@ -3480,14 +3481,14 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return V;
}
case Instruction::ShuffleVector: {
- ValueList LHSVL, RHSVL;
- assert(Instruction::isBinaryOp(S.Opcode) &&
- "Invalid Shuffle Vector Operand");
- unsigned AltOpcode = getAltOpcode(S.Opcode);
- reorderAltShuffleOperands(S.Opcode, AltOpcode, E->Scalars, LHSVL, RHSVL);
- setInsertPointAfterBundle(E->Scalars, VL0);
-
- Value *LHS = vectorizeTree(LHSVL);
+ ValueList LHSVL, RHSVL;
+ assert(Instruction::isBinaryOp(S.Opcode) &&
+ "Invalid Shuffle Vector Operand");
+ reorderAltShuffleOperands(S.Opcode, S.AltOpcode, E->Scalars, LHSVL,
+ RHSVL);
+ setInsertPointAfterBundle(E->Scalars, VL0);
+
+ Value *LHS = vectorizeTree(LHSVL);
Value *RHS = vectorizeTree(RHSVL);
if (E->VectorizedValue) {
@@ -3498,13 +3499,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
// Create a vector of LHS op1 RHS
Value *V0 = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
-
- // Create a vector of LHS op2 RHS
- Value *V1 = Builder.CreateBinOp(
- static_cast<Instruction::BinaryOps>(AltOpcode), LHS, RHS);
-
- // Create shuffle to take alternate operations from the vector.
- // Also, gather up odd and even scalar ops to propagate IR flags to
+
+ // Create a vector of LHS op2 RHS
+ Value *V1 = Builder.CreateBinOp(
+ static_cast<Instruction::BinaryOps>(S.AltOpcode), LHS, RHS);
+
+ // Create shuffle to take alternate operations from the vector.
+ // Also, gather up odd and even scalar ops to propagate IR flags to
// each vector operation.
ValueList OpScalars, AltScalars;
unsigned e = E->Scalars.size();
@@ -3512,9 +3513,9 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
for (unsigned i = 0; i < e; ++i) {
auto *OpInst = cast<Instruction>(E->Scalars[i]);
unsigned InstOpcode = OpInst->getOpcode();
- assert(sameOpcodeOrAlt(S.Opcode, AltOpcode, InstOpcode) &&
+ assert(sameOpcodeOrAlt(S.Opcode, S.AltOpcode, InstOpcode) &&
"Unexpected main/alternate opcode");
- if (InstOpcode == AltOpcode) {
+ if (InstOpcode == S.AltOpcode) {
Mask[i] = Builder.getInt32(e + i);
AltScalars.push_back(E->Scalars[i]);
} else {
OpenPOWER on IntegriCloud