summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp142
1 files changed, 71 insertions, 71 deletions
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index dfc3fe7373a..4168428c662 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -321,6 +321,27 @@ static bool sameOpcodeOrAlt(unsigned Opcode, unsigned AltOpcode,
return Opcode == CheckedOpcode || AltOpcode == CheckedOpcode;
}
+namespace {
+
+/// Main data required for vectorization of instructions.
+struct InstructionsState {
+ /// The very first instruction in the list with the main opcode.
+ Value *OpValue = nullptr;
+
+ /// The main/alternate opcodes for the list of instructions.
+ unsigned Opcode = 0;
+ unsigned AltOpcode = 0;
+
+ /// Some of the instructions in the list have alternate opcodes.
+ bool isAltShuffle() const { return Opcode != AltOpcode; }
+
+ InstructionsState() = default;
+ InstructionsState(Value *OpValue, unsigned Opcode, unsigned AltOpcode)
+ : OpValue(OpValue), Opcode(Opcode), AltOpcode(AltOpcode) {}
+};
+
+} // end anonymous namespace
+
/// Chooses the correct key for scheduling data. If \p Op has the same (or
/// alternate) opcode as \p OpValue, the key is \p Op. Otherwise the key is \p
/// OpValue.
@@ -336,27 +357,6 @@ static Value *isOneOf(Value *OpValue, Value *Op) {
return OpValue;
}
-namespace {
-
-/// Main data required for vectorization of instructions.
-struct InstructionsState {
- /// The very first instruction in the list with the main opcode.
- Value *OpValue = nullptr;
-
- /// The main/alternate opcodes for the list of instructions.
- unsigned Opcode = 0;
- unsigned AltOpcode = 0;
-
- /// Some of the instructions in the list have alternate opcodes.
- bool isAltShuffle() const { return Opcode != AltOpcode; }
-
- InstructionsState() = default;
- InstructionsState(Value *OpValue, unsigned Opcode, unsigned AltOpcode)
- : OpValue(OpValue), Opcode(Opcode), AltOpcode(AltOpcode) {}
-};
-
-} // end anonymous namespace
-
/// \returns analysis of the Instructions in \p VL described in
/// InstructionsState, the Opcode that we suppose the whole list
/// could be vectorized even if its structure is diverse.
@@ -1514,13 +1514,13 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
"tryScheduleBundle should cancelScheduling on failure");
newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
return;
- }
- LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
-
- unsigned ShuffleOrOp = S.isAltShuffle() ?
- (unsigned) Instruction::ShuffleVector : S.Opcode;
- switch (ShuffleOrOp) {
- case Instruction::PHI: {
+ }
+ LLVM_DEBUG(dbgs() << "SLP: We are able to schedule this bundle.\n");
+
+ unsigned ShuffleOrOp = S.isAltShuffle() ?
+ (unsigned) Instruction::ShuffleVector : S.Opcode;
+ switch (ShuffleOrOp) {
+ case Instruction::PHI: {
PHINode *PH = dyn_cast<PHINode>(VL0);
// Check for terminator values (e.g. invoke).
@@ -1901,25 +1901,25 @@ void BoUpSLP::buildTree_rec(ArrayRef<Value *> VL, unsigned Depth,
}
return;
}
- case Instruction::ShuffleVector:
- // If this is not an alternate sequence of opcode like add-sub
- // then do not vectorize this instruction.
- if (!S.isAltShuffle()) {
- BS.cancelScheduling(VL, VL0);
- newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
- LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
+ case Instruction::ShuffleVector:
+ // If this is not an alternate sequence of opcode like add-sub
+ // then do not vectorize this instruction.
+ if (!S.isAltShuffle()) {
+ BS.cancelScheduling(VL, VL0);
+ newTreeEntry(VL, false, UserTreeIdx, ReuseShuffleIndicies);
+ LLVM_DEBUG(dbgs() << "SLP: ShuffleVector are not vectorized.\n");
return;
}
newTreeEntry(VL, true, UserTreeIdx, ReuseShuffleIndicies);
LLVM_DEBUG(dbgs() << "SLP: added a ShuffleVector op.\n");
- // Reorder operands if reordering would enable vectorization.
- if (isa<BinaryOperator>(VL0)) {
- ValueList Left, Right;
- reorderAltShuffleOperands(S.Opcode, S.AltOpcode, VL, Left, Right);
- buildTree_rec(Left, Depth + 1, UserTreeIdx);
- buildTree_rec(Right, Depth + 1, UserTreeIdx);
- return;
+ // Reorder operands if reordering would enable vectorization.
+ if (isa<BinaryOperator>(VL0)) {
+ ValueList Left, Right;
+ reorderAltShuffleOperands(S.Opcode, S.AltOpcode, VL, Left, Right);
+ buildTree_rec(Left, Depth + 1, UserTreeIdx);
+ buildTree_rec(Right, Depth + 1, UserTreeIdx);
+ return;
}
for (unsigned i = 0, e = VL0->getNumOperands(); i < e; ++i) {
@@ -2092,13 +2092,13 @@ int BoUpSLP::getEntryCost(TreeEntry *E) {
}
return ReuseShuffleCost + getGatherCost(VL);
}
- InstructionsState S = getSameOpcode(VL);
- assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
- Instruction *VL0 = cast<Instruction>(S.OpValue);
- unsigned ShuffleOrOp = S.isAltShuffle() ?
- (unsigned) Instruction::ShuffleVector : S.Opcode;
- switch (ShuffleOrOp) {
- case Instruction::PHI:
+ InstructionsState S = getSameOpcode(VL);
+ assert(S.Opcode && allSameType(VL) && allSameBlock(VL) && "Invalid VL");
+ Instruction *VL0 = cast<Instruction>(S.OpValue);
+ unsigned ShuffleOrOp = S.isAltShuffle() ?
+ (unsigned) Instruction::ShuffleVector : S.Opcode;
+ switch (ShuffleOrOp) {
+ case Instruction::PHI:
return 0;
case Instruction::ExtractValue:
@@ -3049,13 +3049,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
}
}
E->VectorizedValue = V;
- return V;
- }
-
- unsigned ShuffleOrOp = S.isAltShuffle() ?
- (unsigned) Instruction::ShuffleVector : S.Opcode;
- switch (ShuffleOrOp) {
- case Instruction::PHI: {
+ return V;
+ }
+
+ unsigned ShuffleOrOp = S.isAltShuffle() ?
+ (unsigned) Instruction::ShuffleVector : S.Opcode;
+ switch (ShuffleOrOp) {
+ case Instruction::PHI: {
PHINode *PH = dyn_cast<PHINode>(VL0);
Builder.SetInsertPoint(PH->getParent()->getFirstNonPHI());
Builder.SetCurrentDebugLocation(PH->getDebugLoc());
@@ -3481,14 +3481,14 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
return V;
}
case Instruction::ShuffleVector: {
- ValueList LHSVL, RHSVL;
- assert(Instruction::isBinaryOp(S.Opcode) &&
- "Invalid Shuffle Vector Operand");
- reorderAltShuffleOperands(S.Opcode, S.AltOpcode, E->Scalars, LHSVL,
- RHSVL);
- setInsertPointAfterBundle(E->Scalars, VL0);
-
- Value *LHS = vectorizeTree(LHSVL);
+ ValueList LHSVL, RHSVL;
+ assert(Instruction::isBinaryOp(S.Opcode) &&
+ "Invalid Shuffle Vector Operand");
+ reorderAltShuffleOperands(S.Opcode, S.AltOpcode, E->Scalars, LHSVL,
+ RHSVL);
+ setInsertPointAfterBundle(E->Scalars, VL0);
+
+ Value *LHS = vectorizeTree(LHSVL);
Value *RHS = vectorizeTree(RHSVL);
if (E->VectorizedValue) {
@@ -3499,13 +3499,13 @@ Value *BoUpSLP::vectorizeTree(TreeEntry *E) {
// Create a vector of LHS op1 RHS
Value *V0 = Builder.CreateBinOp(
static_cast<Instruction::BinaryOps>(S.Opcode), LHS, RHS);
-
- // Create a vector of LHS op2 RHS
- Value *V1 = Builder.CreateBinOp(
- static_cast<Instruction::BinaryOps>(S.AltOpcode), LHS, RHS);
-
- // Create shuffle to take alternate operations from the vector.
- // Also, gather up odd and even scalar ops to propagate IR flags to
+
+ // Create a vector of LHS op2 RHS
+ Value *V1 = Builder.CreateBinOp(
+ static_cast<Instruction::BinaryOps>(S.AltOpcode), LHS, RHS);
+
+ // Create shuffle to take alternate operations from the vector.
+ // Also, gather up odd and even scalar ops to propagate IR flags to
// each vector operation.
ValueList OpScalars, AltScalars;
unsigned e = E->Scalars.size();
OpenPOWER on IntegriCloud