[DAGCombiner] prevent infinite looping by truncating 'and' (PR40793)

This fold can occur during legalization, so it can fight with promotion
to the larger type. It apparently takes a special sequence and subtarget
to avoid more basic simplifications that would hide the problem.

But there's a bigger question raised here: why does distributeTruncateThroughAnd()
even exist? It duplicates functionality from a more minimal pattern that we
already have. But getting rid of this function requires some preliminary steps.

https://bugs.llvm.org/show_bug.cgi?id=40793

llvm-svn: 354594

2 files changed, 29 insertions, 2 deletions

diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index e46cda729f3..dc208e10a28 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -6469,11 +6469,12 @@ SDValue DAGCombiner::distributeTruncateThroughAnd(SDNode *N) {
   assert(N->getOperand(0).getOpcode() == ISD::AND);
 
   // (truncate:TruncVT (and N00, N01C)) -> (and (truncate:TruncVT N00), TruncC)
-  if (N->hasOneUse() && N->getOperand(0).hasOneUse()) {
+  EVT TruncVT = N->getValueType(0);
+  if (N->hasOneUse() && N->getOperand(0).hasOneUse() &&
+      TLI.isTypeDesirableForOp(ISD::AND, TruncVT)) {
     SDValue N01 = N->getOperand(0).getOperand(1);
     if (isConstantOrConstantVector(N01, /* NoOpaques */ true)) {
       SDLoc DL(N);
-      EVT TruncVT = N->getValueType(0);
       SDValue N00 = N->getOperand(0).getOperand(0);
       SDValue Trunc00 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N00);
       SDValue Trunc01 = DAG.getNode(ISD::TRUNCATE, DL, TruncVT, N01);
diff --git a/llvm/test/CodeGen/X86/trunc-and.ll b/llvm/test/CodeGen/X86/trunc-and.ll
new file mode 100644
index 00000000000..28a55c257bd
--- /dev/null
+++ b/llvm/test/CodeGen/X86/trunc-and.ll
@@ -0,0 +1,26 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-- -mattr=avx512bw | FileCheck %s
+
+; This would infinite loop by trying to truncate to i16 and then promote back to i32.
+
+define i16 @PR40793(<8 x i16> %t1) {
+; CHECK-LABEL: PR40793:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    vpextrw $5, %xmm0, %ecx
+; CHECK-NEXT:    andb $15, %cl
+; CHECK-NEXT:    movl $32733, %eax # imm = 0x7FDD
+; CHECK-NEXT:    # kill: def $cl killed $cl killed $ecx
+; CHECK-NEXT:    shrl %cl, %eax
+; CHECK-NEXT:    # kill: def $ax killed $ax killed $eax
+; CHECK-NEXT:    retq
+  %t2 = and <8 x i16> %t1, <i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15, i16 15>
+  %t3 = lshr <8 x i16> <i16 25156, i16 -14844, i16 -26779, i16 -16956, i16 -13978, i16 32733, i16 -24894, i16 18530>, %t2
+  %t4 = bitcast <8 x i16> %t3 to <2 x double>
+  %t5 = call <2 x double> @llvm.fabs.v2f64(<2 x double> %t4)
+  %t6 = bitcast <2 x double> %t5 to <8 x i16>
+  %t7 = extractelement <8 x i16> %t6, i64 5
+  ret i16 %t7
+}
+
+declare <2 x double> @llvm.fabs.v2f64(<2 x double>)
+