diff options
Diffstat (limited to 'llvm/lib/Target/X86/X86ISelLowering.cpp')
| -rw-r--r-- | llvm/lib/Target/X86/X86ISelLowering.cpp | 77 |
1 files changed, 55 insertions, 22 deletions
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp index bc5e1e6b72e..04b068f834c 100644 --- a/llvm/lib/Target/X86/X86ISelLowering.cpp +++ b/llvm/lib/Target/X86/X86ISelLowering.cpp @@ -7747,13 +7747,8 @@ static SDValue lowerVectorShuffleAsDecomposedShuffleBlend(const SDLoc &DL, /// elements, and takes the low elements as the result. Note that while this is /// specified as a *right shift* because x86 is little-endian, it is a *left /// rotate* of the vector lanes. -static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, - SDValue V1, SDValue V2, - ArrayRef<int> Mask, - const X86Subtarget &Subtarget, - SelectionDAG &DAG) { - assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"); - +static int matchVectorShuffleAsByteRotate(MVT VT, SDValue &V1, SDValue &V2, + ArrayRef<int> Mask) { int NumElts = Mask.size(); int NumLanes = VT.getSizeInBits() / 128; int NumLaneElts = NumElts / NumLanes; @@ -7769,20 +7764,28 @@ static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, SDValue Lo, Hi; for (int l = 0; l < NumElts; l += NumLaneElts) { for (int i = 0; i < NumLaneElts; ++i) { - if (Mask[l + i] < 0) + int M = Mask[l + i]; + + if (M == SM_SentinelUndef) continue; + if (M == SM_SentinelZero) + return -1; + + assert(0 <= M && M < (2*NumElts) && "Unexpected mask index."); + // Get the mod-Size index and lane correct it. - int LaneIdx = (Mask[l + i] % NumElts) - l; + int LaneIdx = (M % NumElts) - l; + // Make sure it was in this lane. if (LaneIdx < 0 || LaneIdx >= NumLaneElts) - return SDValue(); + return -1; // Determine where a rotated vector would have started. int StartIdx = i - LaneIdx; if (StartIdx == 0) // The identity rotation isn't interesting, stop. - return SDValue(); + return -1; // If we found the tail of a vector the rotation must be the missing // front. If we found the head of a vector, it must be how much of the @@ -7793,10 +7796,10 @@ static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, Rotation = CandidateRotation; else if (Rotation != CandidateRotation) // The rotations don't match, so we can't match this mask. - return SDValue(); + return -1; // Compute which value this mask is pointing at. - SDValue MaskV = Mask[l + i] < NumElts ? V1 : V2; + SDValue MaskV = M < NumElts ? V1 : V2; // Compute which of the two target values this index should be assigned // to. This reflects whether the high elements are remaining or the low @@ -7810,7 +7813,7 @@ static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, else if (TargetV != MaskV) // This may be a rotation, but it pulls from the inputs in some // unsupported interleaving. - return SDValue(); + return -1; } } @@ -7822,15 +7825,32 @@ static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, else if (!Hi) Hi = Lo; - // Cast the inputs to i8 vector of correct length to match PALIGNR or - // PSLLDQ/PSRLDQ. - MVT ByteVT = MVT::getVectorVT(MVT::i8, 16 * NumLanes); - Lo = DAG.getBitcast(ByteVT, Lo); - Hi = DAG.getBitcast(ByteVT, Hi); + V1 = Lo; + V2 = Hi; // The actual rotate instruction rotates bytes, so we need to scale the // rotation based on how many bytes are in the vector lane. int Scale = 16 / NumLaneElts; + return Rotation * Scale; +} + +static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, + SDValue V1, SDValue V2, + ArrayRef<int> Mask, + const X86Subtarget &Subtarget, + SelectionDAG &DAG) { + assert(!isNoopShuffleMask(Mask) && "We shouldn't lower no-op shuffles!"); + + SDValue Lo = V1, Hi = V2; + int ByteRotation = matchVectorShuffleAsByteRotate(VT, Lo, Hi, Mask); + if (ByteRotation <= 0) + return SDValue(); + + // Cast the inputs to i8 vector of correct length to match PALIGNR or + // PSLLDQ/PSRLDQ. + MVT ByteVT = MVT::getVectorVT(MVT::i8, VT.getSizeInBits() / 8); + Lo = DAG.getBitcast(ByteVT, Lo); + Hi = DAG.getBitcast(ByteVT, Hi); // SSSE3 targets can use the palignr instruction. if (Subtarget.hasSSSE3()) { @@ -7838,7 +7858,7 @@ static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, "512-bit PALIGNR requires BWI instructions"); return DAG.getBitcast( VT, DAG.getNode(X86ISD::PALIGNR, DL, ByteVT, Lo, Hi, - DAG.getConstant(Rotation * Scale, DL, MVT::i8))); + DAG.getConstant(ByteRotation, DL, MVT::i8))); } assert(VT.is128BitVector() && @@ -7849,8 +7869,8 @@ static SDValue lowerVectorShuffleAsByteRotate(const SDLoc &DL, MVT VT, "SSE2 rotate lowering only needed for v16i8!"); // Default SSE2 implementation - int LoByteShift = 16 - Rotation * Scale; - int HiByteShift = Rotation * Scale; + int LoByteShift = 16 - ByteRotation; + int HiByteShift = ByteRotation; SDValue LoShift = DAG.getNode(X86ISD::VSHLDQ, DL, MVT::v16i8, Lo, DAG.getConstant(LoByteShift, DL, MVT::i8)); @@ -25198,6 +25218,19 @@ static bool matchBinaryPermuteVectorShuffle(MVT MaskVT, ArrayRef<int> Mask, unsigned &Shuffle, MVT &ShuffleVT, unsigned &PermuteImm) { unsigned NumMaskElts = Mask.size(); + bool FloatDomain = MaskVT.isFloatingPoint(); + + // Attempt to match against PALIGNR byte rotate. + if (!FloatDomain && ((MaskVT.is128BitVector() && Subtarget.hasSSSE3()) || + (MaskVT.is256BitVector() && Subtarget.hasAVX2()))) { + int ByteRotation = matchVectorShuffleAsByteRotate(MaskVT, V1, V2, Mask); + if (0 < ByteRotation) { + Shuffle = X86ISD::PALIGNR; + ShuffleVT = MVT::getVectorVT(MVT::i8, MaskVT.getSizeInBits() / 8); + PermuteImm = ByteRotation; + return true; + } + } // Attempt to blend with zero. if (NumMaskElts <= 8 && ((Subtarget.hasSSE41() && MaskVT.is128BitVector()) || |
