diff options
| -rw-r--r-- | llvm/include/llvm/CodeGen/SelectionDAG.h | 3 | ||||
| -rw-r--r-- | llvm/include/llvm/CodeGen/TargetLowering.h | 3 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h | 1 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp | 30 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 9 |
5 files changed, 44 insertions, 2 deletions
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index efbfb3331ce..ffd1f74287f 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1588,6 +1588,9 @@ public: return SplitVector(N->getOperand(OpNo), SDLoc(N)); } + /// Widen the vector up to the next power of two using INSERT_SUBVECTOR. + SDValue WidenVector(const SDValue &N, const SDLoc &DL); + /// Append the extracted elements from Start to Count out of the vector Op /// in Args. If Count is 0, all of the elements will be extracted. void ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args, diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h index 6d7569c223b..70fe7694834 100644 --- a/llvm/include/llvm/CodeGen/TargetLowering.h +++ b/llvm/include/llvm/CodeGen/TargetLowering.h @@ -294,6 +294,9 @@ public: // The default action for one element vectors is to scalarize if (VT.getVectorNumElements() == 1) return TypeScalarizeVector; + // The default action for an odd-width vector is to widen. + if (!VT.isPow2VectorType()) + return TypeWidenVector; // The default action for other vectors is to promote return TypePromoteInteger; } diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h index 001f879a957..b195401a04b 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h @@ -836,6 +836,7 @@ private: SDValue WidenVecOp_MGATHER(SDNode* N, unsigned OpNo); SDValue WidenVecOp_MSCATTER(SDNode* N, unsigned OpNo); SDValue WidenVecOp_SETCC(SDNode* N); + SDValue WidenVecOp_VSELECT(SDNode *N); SDValue WidenVecOp_Convert(SDNode *N); SDValue WidenVecOp_FCOPYSIGN(SDNode *N); diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp index d367ec4be27..7da04e81d4e 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp @@ -3831,8 +3831,15 @@ SDValue DAGTypeLegalizer::WidenVecRes_SETCC(SDNode *N) { return Res; } - InOp1 = GetWidenedVector(InOp1); - SDValue InOp2 = GetWidenedVector(N->getOperand(1)); + // If the inputs also widen, handle them directly. Otherwise widen by hand. + SDValue InOp2 = N->getOperand(1); + if (getTypeAction(InVT) == TargetLowering::TypeWidenVector) { + InOp1 = GetWidenedVector(InOp1); + InOp2 = GetWidenedVector(InOp2); + } else { + InOp1 = DAG.WidenVector(InOp1, SDLoc(N)); + InOp2 = DAG.WidenVector(InOp2, SDLoc(N)); + } // Assume that the input and output will be widen appropriately. If not, // we will have to unroll it at some point. @@ -3875,6 +3882,7 @@ bool DAGTypeLegalizer::WidenVectorOperand(SDNode *N, unsigned OpNo) { case ISD::MGATHER: Res = WidenVecOp_MGATHER(N, OpNo); break; case ISD::MSCATTER: Res = WidenVecOp_MSCATTER(N, OpNo); break; case ISD::SETCC: Res = WidenVecOp_SETCC(N); break; + case ISD::VSELECT: Res = WidenVecOp_VSELECT(N); break; case ISD::FCOPYSIGN: Res = WidenVecOp_FCOPYSIGN(N); break; case ISD::ANY_EXTEND: @@ -4311,6 +4319,24 @@ SDValue DAGTypeLegalizer::WidenVecOp_VECREDUCE(SDNode *N) { return DAG.getNode(N->getOpcode(), dl, N->getValueType(0), Op, N->getFlags()); } +SDValue DAGTypeLegalizer::WidenVecOp_VSELECT(SDNode *N) { + // This only gets called in the case that the left and right inputs and + // result are of a legal odd vector type, and the condition is illegal i1 of + // the same odd width that needs widening. + EVT VT = N->getValueType(0); + assert(VT.isVector() && !VT.isPow2VectorType() && isTypeLegal(VT)); + + SDValue Cond = GetWidenedVector(N->getOperand(0)); + SDValue LeftIn = DAG.WidenVector(N->getOperand(1), SDLoc(N)); + SDValue RightIn = DAG.WidenVector(N->getOperand(2), SDLoc(N)); + SDLoc DL(N); + + SDValue Select = DAG.getNode(N->getOpcode(), DL, LeftIn.getValueType(), Cond, + LeftIn, RightIn); + return DAG.getNode( + ISD::EXTRACT_SUBVECTOR, DL, VT, Select, + DAG.getConstant(0, DL, TLI.getVectorIdxTy(DAG.getDataLayout()))); +} //===----------------------------------------------------------------------===// // Vector Widening Utilities diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index fe3f2b99ba7..6e807556d37 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -9127,6 +9127,15 @@ SelectionDAG::SplitVector(const SDValue &N, const SDLoc &DL, const EVT &LoVT, return std::make_pair(Lo, Hi); } +/// Widen the vector up to the next power of two using INSERT_SUBVECTOR. +SDValue SelectionDAG::WidenVector(const SDValue &N, const SDLoc &DL) { + EVT VT = N.getValueType(); + EVT WideVT = EVT::getVectorVT(*getContext(), VT.getVectorElementType(), + NextPowerOf2(VT.getVectorNumElements())); + return getNode(ISD::INSERT_SUBVECTOR, DL, WideVT, getUNDEF(WideVT), N, + getConstant(0, DL, TLI->getVectorIdxTy(getDataLayout()))); +} + void SelectionDAG::ExtractVectorElements(SDValue Op, SmallVectorImpl<SDValue> &Args, unsigned Start, unsigned Count) { |
