[PatternMatch] allow undef elements when matching a vector zero

This is the last step in getting constant pattern matchers to allow
undef elements in constant vectors.

I'm adding a dedicated m_ZeroInt() function and building m_Zero() from
that. In most cases, calling code can be updated to use m_ZeroInt()
directly when there's no need to match pointers, but I'm leaving that
efficiency optimization as a follow-up step because it's not always
clear when that's ok.

There are just enough icmp folds in InstSimplify that can be used for 
integer or pointer types, that we probably still want a generic m_Zero()
for those cases. Otherwise, we could eliminate it (and possibly add a
m_NullPtr() as an alias for isa<ConstantPointerNull>()).

We're conservatively returning a full zero vector (zeroinitializer) in
InstSimplify/InstCombine on some of these folds (see diffs in InstSimplify),
but I'm not sure if that's actually necessary in all cases. We may be 
able to propagate an undef lane instead. One test where this happens is 
marked with 'TODO'.
 

llvm-svn: 330550

15 files changed, 79 insertions, 118 deletions

diff --git a/llvm/include/llvm/IR/PatternMatch.h b/llvm/include/llvm/IR/PatternMatch.h
index 48e07862c07..8b1c6b2f350 100644
--- a/llvm/include/llvm/IR/PatternMatch.h
+++ b/llvm/include/llvm/IR/PatternMatch.h
@@ -132,18 +132,6 @@ inline match_combine_and<LTy, RTy> m_CombineAnd(const LTy &L, const RTy &R) {
   return match_combine_and<LTy, RTy>(L, R);
 }
 
-struct match_zero {
-  template <typename ITy> bool match(ITy *V) {
-    if (const auto *C = dyn_cast<Constant>(V))
-      return C->isNullValue();
-    return false;
-  }
-};
-
-/// Match an arbitrary zero/null constant. This includes
-/// zero_initializer for vectors and ConstantPointerNull for pointers.
-inline match_zero m_Zero() { return match_zero(); }
-
 struct apint_match {
   const APInt *&Res;
 
@@ -365,6 +353,27 @@ inline cst_pred_ty<is_one> m_One() {
   return cst_pred_ty<is_one>();
 }
 
+struct is_zero_int {
+  bool isValue(const APInt &C) { return C.isNullValue(); }
+};
+/// Match an integer 0 or a vector with all elements equal to 0.
+/// For vectors, this includes constants with undefined elements.
+inline cst_pred_ty<is_zero_int> m_ZeroInt() {
+  return cst_pred_ty<is_zero_int>();
+}
+
+struct is_zero {
+  template <typename ITy> bool match(ITy *V) {
+    auto *C = dyn_cast<Constant>(V);
+    return C && (C->isNullValue() || cst_pred_ty<is_zero_int>().match(C));
+  }
+};
+/// Match any null constant or a vector with all elements equal to 0.
+/// For vectors, this includes constants with undefined elements.
+inline is_zero m_Zero() {
+  return is_zero();
+}
+
 struct is_power2 {
   bool isValue(const APInt &C) { return C.isPowerOf2(); }
 };
diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index 73c3c47e909..1e62b9d407e 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -680,14 +680,14 @@ static Value *SimplifySubInst(Value *Op0, Value *Op1, bool isNSW, bool isNUW,
   if (match(Op0, m_Zero())) {
     // 0 - X -> 0 if the sub is NUW.
     if (isNUW)
-      return Op0;
+      return Constant::getNullValue(Op0->getType());
 
     KnownBits Known = computeKnownBits(Op1, Q.DL, 0, Q.AC, Q.CxtI, Q.DT);
     if (Known.Zero.isMaxSignedValue()) {
       // Op1 is either 0 or the minimum signed value. If the sub is NSW, then
       // Op1 must be 0 because negating the minimum signed value is undefined.
       if (isNSW)
-        return Op0;
+        return Constant::getNullValue(Op0->getType());
 
       // 0 - X -> X if X is 0 or the minimum signed value.
       return Op1;
@@ -799,12 +799,9 @@ static Value *SimplifyMulInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
     return C;
 
   // X * undef -> 0
-  if (match(Op1, m_Undef()))
-    return Constant::getNullValue(Op0->getType());
-
   // X * 0 -> 0
-  if (match(Op1, m_Zero()))
-    return Op1;
+  if (match(Op1, m_CombineOr(m_Undef(), m_Zero())))
+    return Constant::getNullValue(Op0->getType());
 
   // X * 1 -> X
   if (match(Op1, m_One()))
@@ -888,7 +885,7 @@ static Value *simplifyDivRem(Value *Op0, Value *Op1, bool IsDiv) {
   // 0 / X -> 0
   // 0 % X -> 0
   if (match(Op0, m_Zero()))
-    return Op0;
+    return Constant::getNullValue(Op0->getType());
 
   // X / X -> 1
   // X % X -> 0
@@ -1147,7 +1144,7 @@ static Value *SimplifyShift(Instruction::BinaryOps Opcode, Value *Op0,
 
   // 0 shift by X -> 0
   if (match(Op0, m_Zero()))
-    return Op0;
+    return Constant::getNullValue(Op0->getType());
 
   // X shift by 0 -> X
   if (match(Op1, m_Zero()))
@@ -1689,7 +1686,7 @@ static Value *SimplifyAndInst(Value *Op0, Value *Op1, const SimplifyQuery &Q,
 
   // X & 0 = 0
   if (match(Op1, m_Zero()))
-    return Op1;
+    return Constant::getNullValue(Op0->getType());
 
   // X & -1 = X
   if (match(Op1, m_AllOnes()))
@@ -3068,8 +3065,7 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   Type *ITy = GetCompareTy(LHS); // The return type.
 
   // icmp X, X -> true/false
-  // X icmp undef -> true/false.  For example, icmp ugt %X, undef -> false
-  // because X could be 0.
+  // icmp X, undef -> true/false because undef could be X.
   if (LHS == RHS || isa<UndefValue>(RHS))
     return ConstantInt::get(ITy, CmpInst::isTrueWhenEqual(Pred));
 
diff --git a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
index 28d68a38b2b..1d90d2363aa 100644
--- a/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
+++ b/llvm/lib/Transforms/InstCombine/InstCombineAndOrXor.cpp
@@ -1699,7 +1699,6 @@ static bool areInverseVectorBitmasks(Constant *C1, Constant *C2) {
       return false;
 
     // One element must be all ones, and the other must be all zeros.
-    // FIXME: Allow undef elements.
     if (!((match(EltC1, m_Zero()) && match(EltC2, m_AllOnes())) ||
           (match(EltC2, m_Zero()) && match(EltC1, m_AllOnes()))))
       return false;
diff --git a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
index 07934fbdfe7..306577fae82 100644
--- a/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
+++ b/llvm/test/Transforms/InstCombine/X86/x86-vector-shifts.ll
@@ -2032,10 +2032,11 @@ define <4 x i64> @avx2_psrlv_q_256_allbig(<4 x i64> %v) {
   ret <4 x i64> %1
 }
 
+; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input.
+
 define <2 x i64> @avx2_psrlv_q_128_undef(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psrlv_q_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = lshr <2 x i64> %v, <i64 0, i64 undef>
-; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 1
   %2 = tail call <2 x i64> @llvm.x86.avx2.psrlv.q(<2 x i64> %v, <2 x i64> %1)
@@ -2432,10 +2433,11 @@ define <4 x i64> @avx2_psllv_q_256_allbig(<4 x i64> %v) {
   ret <4 x i64> %1
 }
 
+; The shift amount is 0 (the undef lane could be 0), so we return the unshifted input.
+
 define <2 x i64> @avx2_psllv_q_128_undef(<2 x i64> %v) {
 ; CHECK-LABEL: @avx2_psllv_q_128_undef(
-; CHECK-NEXT:    [[TMP1:%.*]] = shl <2 x i64> %v, <i64 0, i64 undef>
-; CHECK-NEXT:    ret <2 x i64> [[TMP1]]
+; CHECK-NEXT:    ret <2 x i64> [[V:%.*]]
 ;
   %1 = insertelement <2 x i64> <i64 0, i64 8>, i64 undef, i64 1
   %2 = tail call <2 x i64> @llvm.x86.avx2.psllv.q(<2 x i64> %v, <2 x i64> %1)
diff --git a/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll b/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
index 573c0a16293..9576c5c20ba 100644
--- a/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
+++ b/llvm/test/Transforms/InstCombine/cast-int-icmp-eq-0.ll
@@ -616,13 +616,11 @@ define <3 x i1> @i32_cast_cmp_ne_int_0_sitofp_double_vec(<3 x i32> %i) {
   ret <3 x i1> %cmp
 }
 
-; FIXME: Vector zero constant with undef is not matched.
+; TODO: Can we propagate the constant vector with undef element?
 
 define <3 x i1> @i32_cast_cmp_eq_int_0_sitofp_float_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_eq_int_0_sitofp_float_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = sitofp <3 x i32> [[I:%.*]] to <3 x float>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <3 x i32>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[B]], <i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
   %f = sitofp <3 x i32> %i to  <3 x float>
diff --git a/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll b/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll
index dde31749392..e1fa27256b1 100644
--- a/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll
+++ b/llvm/test/Transforms/InstCombine/cast-unsigned-icmp-eqcmp-0.ll
@@ -3,11 +3,8 @@
 
 ; This is related to https://bugs.llvm.org/show_bug.cgi?id=36682
 
-; FIXME: *all* of these are true tests.
 ; In *all* of these, uitofp and bitcast should be instcombine'd out.
 
-; FIXME: icmp eq/ne does not ignore undef elements in vectors.
-
 define i1 @i32_cast_cmp_eq_int_0_uitofp_float(i32 %i) {
 ; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_float(
 ; CHECK-NEXT:    [[CMP:%.*]] = icmp eq i32 [[I:%.*]], 0
@@ -32,9 +29,7 @@ define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_float_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_float_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_float_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x float>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <3 x i32>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[B]], <i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
   %f = uitofp <3 x i32> %i to <3 x float>
@@ -67,9 +62,7 @@ define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_float_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_float_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_float_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x float>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <3 x i32>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[B]], <i32 0, i32 undef, i32 0>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
   %f = uitofp <3 x i32> %i to <3 x float>
@@ -102,9 +95,7 @@ define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_double_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_double_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_double_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x double>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x double> [[F]] to <3 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i64> [[B]], <i64 0, i64 undef, i64 0>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
   %f = uitofp <3 x i32> %i to <3 x double>
@@ -137,9 +128,7 @@ define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_double_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_double_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_double_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x double>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x double> [[F]] to <3 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i64> [[B]], <i64 0, i64 undef, i64 0>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
   %f = uitofp <3 x i32> %i to <3 x double>
@@ -172,9 +161,7 @@ define <2 x i1> @i32_cast_cmp_eq_int_0_uitofp_half_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_eq_int_0_uitofp_half_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_eq_int_0_uitofp_half_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x half>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x half> [[F]] to <3 x i16>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i16> [[B]], <i16 0, i16 undef, i16 0>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp eq <3 x i32> [[I:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
   %f = uitofp <3 x i32> %i to <3 x half>
@@ -207,9 +194,7 @@ define <2 x i1> @i32_cast_cmp_ne_int_0_uitofp_half_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_ne_int_0_uitofp_half_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_ne_int_0_uitofp_half_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x half>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x half> [[F]] to <3 x i16>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i16> [[B]], <i16 0, i16 undef, i16 0>
+; CHECK-NEXT:    [[CMP:%.*]] = icmp ne <3 x i32> [[I:%.*]], zeroinitializer
 ; CHECK-NEXT:    ret <3 x i1> [[CMP]]
 ;
   %f = uitofp <3 x i32> %i to <3 x half>
diff --git a/llvm/test/Transforms/InstCombine/min-positive.ll b/llvm/test/Transforms/InstCombine/min-positive.ll
index 618795e79f6..51f98bc00dc 100644
--- a/llvm/test/Transforms/InstCombine/min-positive.ll
+++ b/llvm/test/Transforms/InstCombine/min-positive.ll
@@ -57,11 +57,7 @@ define <2 x i1> @smin_commute_vec(<2 x i32> %x, <2 x i32> %other) {
 
 define <2 x i1> @smin_commute_vec_undef_elts(<2 x i32> %x, <2 x i32> %other) {
 ; CHECK-LABEL: @smin_commute_vec_undef_elts(
-; CHECK-NEXT:    [[NOTNEG:%.*]] = and <2 x i32> [[X:%.*]], <i32 6, i32 6>
-; CHECK-NEXT:    [[POSITIVE:%.*]] = or <2 x i32> [[NOTNEG]], <i32 1, i32 1>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp sgt <2 x i32> [[POSITIVE]], [[OTHER:%.*]]
-; CHECK-NEXT:    [[SEL:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[OTHER]], <2 x i32> [[POSITIVE]]
-; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt <2 x i32> [[SEL]], <i32 0, i32 undef>
+; CHECK-NEXT:    [[TEST:%.*]] = icmp sgt <2 x i32> [[OTHER:%.*]], <i32 0, i32 undef>
 ; CHECK-NEXT:    ret <2 x i1> [[TEST]]
 ;
   %notneg = and <2 x i32> %x, <i32 7, i32 7>
diff --git a/llvm/test/Transforms/InstCombine/select-of-bittest.ll b/llvm/test/Transforms/InstCombine/select-of-bittest.ll
index 55d122b6301..d9bef00b2f7 100644
--- a/llvm/test/Transforms/InstCombine/select-of-bittest.ll
+++ b/llvm/test/Transforms/InstCombine/select-of-bittest.ll
@@ -82,11 +82,9 @@ define <2 x i32> @and_lshr_and_vec_v2(<2 x i32> %arg) {
 
 define <3 x i32> @and_lshr_and_vec_undef(<3 x i32> %arg) {
 ; CHECK-LABEL: @and_lshr_and_vec_undef(
-; CHECK-NEXT:    [[TMP:%.*]] = and <3 x i32> [[ARG:%.*]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i32> [[TMP]], <i32 0, i32 undef, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = lshr <3 x i32> [[ARG]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = and <3 x i32> [[TMP2]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[TMP3]], <3 x i32> <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i32> [[ARG:%.*]], <i32 3, i32 undef, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <3 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <3 x i1> [[TMP2]] to <3 x i32>
 ; CHECK-NEXT:    ret <3 x i32> [[TMP4]]
 ;
   %tmp = and <3 x i32> %arg, <i32 1, i32 undef, i32 1>
@@ -141,10 +139,9 @@ define <2 x i32> @and_and_vec(<2 x i32> %arg) {
 
 define <3 x i32> @and_and_vec_undef(<3 x i32> %arg) {
 ; CHECK-LABEL: @and_and_vec_undef(
-; CHECK-NEXT:    [[TMP:%.*]] = and <3 x i32> [[ARG:%.*]], <i32 2, i32 undef, i32 2>
-; CHECK-NEXT:    [[TMP1:%.*]] = icmp eq <3 x i32> [[TMP]], <i32 0, i32 undef, i32 0>
-; CHECK-NEXT:    [[TMP2:%.*]] = and <3 x i32> [[ARG]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP3:%.*]] = select <3 x i1> [[TMP1]], <3 x i32> [[TMP2]], <3 x i32> <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = and <3 x i32> [[ARG:%.*]], <i32 3, i32 -1, i32 3>
+; CHECK-NEXT:    [[TMP2:%.*]] = icmp ne <3 x i32> [[TMP1]], zeroinitializer
+; CHECK-NEXT:    [[TMP3:%.*]] = zext <3 x i1> [[TMP2]] to <3 x i32>
 ; CHECK-NEXT:    ret <3 x i32> [[TMP3]]
 ;
   %tmp = and <3 x i32> %arg, <i32 2, i32 undef, i32 2>
@@ -225,11 +222,10 @@ define <2 x i32> @f_var0_vec(<2 x i32> %arg, <2 x i32> %arg1) {
 
 define <3 x i32> @f_var0_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) {
 ; CHECK-LABEL: @f_var0_vec_undef(
-; CHECK-NEXT:    [[TMP:%.*]] = and <3 x i32> [[ARG:%.*]], [[ARG1:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <3 x i32> [[TMP]], <i32 0, i32 undef, i32 0>
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr <3 x i32> [[ARG]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = and <3 x i32> [[TMP3]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP5:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[TMP4]], <3 x i32> <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <3 x i32> [[ARG1:%.*]], <i32 2, i32 undef, i32 2>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <3 x i1> [[TMP3]] to <3 x i32>
 ; CHECK-NEXT:    ret <3 x i32> [[TMP5]]
 ;
   %tmp = and <3 x i32> %arg, %arg1
@@ -288,10 +284,10 @@ define <2 x i32> @f_var1_vec(<2 x i32> %arg, <2 x i32> %arg1) {
 
 define <3 x i32> @f_var1_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) {
 ; CHECK-LABEL: @f_var1_vec_undef(
-; CHECK-NEXT:    [[TMP:%.*]] = and <3 x i32> [[ARG:%.*]], [[ARG1:%.*]]
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <3 x i32> [[TMP]], <i32 0, i32 undef, i32 0>
-; CHECK-NEXT:    [[TMP3:%.*]] = and <3 x i32> [[ARG]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP4:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[TMP3]], <3 x i32> <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = or <3 x i32> [[ARG1:%.*]], <i32 1, i32 1, i32 1>
+; CHECK-NEXT:    [[TMP2:%.*]] = and <3 x i32> [[TMP1]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = icmp ne <3 x i32> [[TMP2]], zeroinitializer
+; CHECK-NEXT:    [[TMP4:%.*]] = zext <3 x i1> [[TMP3]] to <3 x i32>
 ; CHECK-NEXT:    ret <3 x i32> [[TMP4]]
 ;
   %tmp = and <3 x i32> %arg, %arg1
@@ -358,11 +354,11 @@ define <2 x i32> @f_var2_vec(<2 x i32> %arg, <2 x i32> %arg1) {
 
 define <3 x i32> @f_var2_vec_undef(<3 x i32> %arg, <3 x i32> %arg1) {
 ; CHECK-LABEL: @f_var2_vec_undef(
-; CHECK-NEXT:    [[TMP:%.*]] = and <3 x i32> [[ARG:%.*]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP2:%.*]] = icmp eq <3 x i32> [[TMP]], <i32 0, i32 undef, i32 0>
-; CHECK-NEXT:    [[TMP3:%.*]] = lshr <3 x i32> [[ARG]], [[ARG1:%.*]]
-; CHECK-NEXT:    [[TMP4:%.*]] = and <3 x i32> [[TMP3]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP5:%.*]] = select <3 x i1> [[TMP2]], <3 x i32> [[TMP4]], <3 x i32> <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <3 x i32> <i32 1, i32 1, i32 1>, [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <3 x i32> [[TMP1]], <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP3:%.*]] = and <3 x i32> [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <3 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP5:%.*]] = zext <3 x i1> [[TMP4]] to <3 x i32>
 ; CHECK-NEXT:    ret <3 x i32> [[TMP5]]
 ;
   %tmp = and <3 x i32> %arg, <i32 1, i32 undef, i32 1>
@@ -431,11 +427,11 @@ define <2 x i32> @f_var3_splatvec(<2 x i32> %arg, <2 x i32> %arg1, <2 x i32> %ar
 
 define <3 x i32> @f_var3_vec_undef(<3 x i32> %arg, <3 x i32> %arg1, <3 x i32> %arg2) {
 ; CHECK-LABEL: @f_var3_vec_undef(
-; CHECK-NEXT:    [[TMP:%.*]] = and <3 x i32> [[ARG:%.*]], [[ARG1:%.*]]
-; CHECK-NEXT:    [[TMP3:%.*]] = icmp eq <3 x i32> [[TMP]], <i32 0, i32 undef, i32 0>
-; CHECK-NEXT:    [[TMP4:%.*]] = lshr <3 x i32> [[ARG]], [[ARG2:%.*]]
-; CHECK-NEXT:    [[TMP5:%.*]] = and <3 x i32> [[TMP4]], <i32 1, i32 undef, i32 1>
-; CHECK-NEXT:    [[TMP6:%.*]] = select <3 x i1> [[TMP3]], <3 x i32> [[TMP5]], <3 x i32> <i32 1, i32 undef, i32 1>
+; CHECK-NEXT:    [[TMP1:%.*]] = shl <3 x i32> <i32 1, i32 1, i32 1>, [[ARG2:%.*]]
+; CHECK-NEXT:    [[TMP2:%.*]] = or <3 x i32> [[TMP1]], [[ARG1:%.*]]
+; CHECK-NEXT:    [[TMP3:%.*]] = and <3 x i32> [[TMP2]], [[ARG:%.*]]
+; CHECK-NEXT:    [[TMP4:%.*]] = icmp ne <3 x i32> [[TMP3]], zeroinitializer
+; CHECK-NEXT:    [[TMP6:%.*]] = zext <3 x i1> [[TMP4]] to <3 x i32>
 ; CHECK-NEXT:    ret <3 x i32> [[TMP6]]
 ;
   %tmp = and <3 x i32> %arg, %arg1
diff --git a/llvm/test/Transforms/InstSimplify/AndOrXor.ll b/llvm/test/Transforms/InstSimplify/AndOrXor.ll
index 859e7637501..8cb7be8aa1c 100644
--- a/llvm/test/Transforms/InstSimplify/AndOrXor.ll
+++ b/llvm/test/Transforms/InstSimplify/AndOrXor.ll
@@ -11,8 +11,7 @@ define i8 @and0(i8 %x) {
 
 define <2 x i8> @and0_vec_undef_elt(<2 x i8> %x) {
 ; CHECK-LABEL: @and0_vec_undef_elt(
-; CHECK-NEXT:    [[R:%.*]] = and <2 x i8> [[X:%.*]], <i8 undef, i8 0>
-; CHECK-NEXT:    ret <2 x i8> [[R]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
 ;
   %r = and <2 x i8> %x, <i8 undef, i8 0>
   ret <2 x i8> %r
diff --git a/llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll b/llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll
index 3e5a68ce782..3c068c8deb0 100644
--- a/llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll
+++ b/llvm/test/Transforms/InstSimplify/cast-unsigned-icmp-cmp-0.ll
@@ -7,8 +7,6 @@
 ;   * slt i32 %b, 0  -> false
 ;   * sgt i32 %b, -1 -> true
 
-; FIXME: m_Zero does not handle undef elements in vectors.
-
 define i1 @i32_cast_cmp_slt_int_0_uitofp_float(i32 %i) {
 ; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_float(
 ; CHECK-NEXT:    ret i1 false
@@ -31,10 +29,7 @@ define <2 x i1> @i32_cast_cmp_slt_int_0_uitofp_float_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_slt_int_0_uitofp_float_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_float_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x float>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x float> [[F]] to <3 x i32>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <3 x i32> [[B]], <i32 0, i32 undef, i32 0>
-; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
 ;
   %f = uitofp <3 x i32> %i to <3 x float>
   %b = bitcast <3 x float> %f to <3 x i32>
@@ -94,10 +89,7 @@ define <2 x i1> @i32_cast_cmp_slt_int_0_uitofp_double_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_slt_int_0_uitofp_double_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_double_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x double>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x double> [[F]] to <3 x i64>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <3 x i64> [[B]], <i64 0, i64 undef, i64 0>
-; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
 ;
   %f = uitofp <3 x i32> %i to <3 x double>
   %b = bitcast <3 x double> %f to <3 x i64>
@@ -157,10 +149,7 @@ define <2 x i1> @i32_cast_cmp_slt_int_0_uitofp_half_vec(<2 x i32> %i) {
 
 define <3 x i1> @i32_cast_cmp_slt_int_0_uitofp_half_vec_undef(<3 x i32> %i) {
 ; CHECK-LABEL: @i32_cast_cmp_slt_int_0_uitofp_half_vec_undef(
-; CHECK-NEXT:    [[F:%.*]] = uitofp <3 x i32> [[I:%.*]] to <3 x half>
-; CHECK-NEXT:    [[B:%.*]] = bitcast <3 x half> [[F]] to <3 x i16>
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <3 x i16> [[B]], <i16 0, i16 undef, i16 0>
-; CHECK-NEXT:    ret <3 x i1> [[CMP]]
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
 ;
   %f = uitofp <3 x i32> %i to <3 x half>
   %b = bitcast <3 x half> %f to <3 x i16>
diff --git a/llvm/test/Transforms/InstSimplify/div.ll b/llvm/test/Transforms/InstSimplify/div.ll
index b8ee7bc901b..146112a48c6 100644
--- a/llvm/test/Transforms/InstSimplify/div.ll
+++ b/llvm/test/Transforms/InstSimplify/div.ll
@@ -19,8 +19,7 @@ define <2 x i32> @zero_dividend_vector(<2 x i32> %A) {
 
 define <2 x i32> @zero_dividend_vector_undef_elt(<2 x i32> %A) {
 ; CHECK-LABEL: @zero_dividend_vector_undef_elt(
-; CHECK-NEXT:    [[B:%.*]] = sdiv <2 x i32> <i32 0, i32 undef>, [[A:%.*]]
-; CHECK-NEXT:    ret <2 x i32> [[B]]
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
 ;
   %B = sdiv <2 x i32> <i32 0, i32 undef>, %A
   ret <2 x i32> %B
diff --git a/llvm/test/Transforms/InstSimplify/mul.ll b/llvm/test/Transforms/InstSimplify/mul.ll
index 2c576383e25..71410cd0ca3 100644
--- a/llvm/test/Transforms/InstSimplify/mul.ll
+++ b/llvm/test/Transforms/InstSimplify/mul.ll
@@ -36,8 +36,7 @@ define <16 x i8> @mul_by_0_vec(<16 x i8> %a) {
 
 define <2 x i8> @mul_by_0_vec_undef_elt(<2 x i8> %a) {
 ; CHECK-LABEL: @mul_by_0_vec_undef_elt(
-; CHECK-NEXT:    [[B:%.*]] = mul <2 x i8> [[A:%.*]], <i8 undef, i8 0>
-; CHECK-NEXT:    ret <2 x i8> [[B]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
 ;
   %b = mul <2 x i8> %a, <i8 undef, i8 0>
   ret <2 x i8> %b
diff --git a/llvm/test/Transforms/InstSimplify/negate.ll b/llvm/test/Transforms/InstSimplify/negate.ll
index 49653cea263..ec18826073c 100644
--- a/llvm/test/Transforms/InstSimplify/negate.ll
+++ b/llvm/test/Transforms/InstSimplify/negate.ll
@@ -19,8 +19,7 @@ define <2 x i32> @negate_nuw_vec(<2 x i32> %x) {
 
 define <2 x i32> @negate_nuw_vec_undef_elt(<2 x i32> %x) {
 ; CHECK-LABEL: @negate_nuw_vec_undef_elt(
-; CHECK-NEXT:    [[NEG:%.*]] = sub nuw <2 x i32> <i32 0, i32 undef>, [[X:%.*]]
-; CHECK-NEXT:    ret <2 x i32> [[NEG]]
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
 ;
   %neg = sub nuw <2 x i32> <i32 0, i32 undef>, %x
   ret <2 x i32> %neg
@@ -46,9 +45,7 @@ define <2 x i8> @negate_zero_or_minsigned_nsw_vec(<2 x i8> %x) {
 
 define <2 x i8> @negate_zero_or_minsigned_nsw_vec_undef_elt(<2 x i8> %x) {
 ; CHECK-LABEL: @negate_zero_or_minsigned_nsw_vec_undef_elt(
-; CHECK-NEXT:    [[SIGNBIT:%.*]] = shl <2 x i8> [[X:%.*]], <i8 7, i8 7>
-; CHECK-NEXT:    [[NEG:%.*]] = sub nsw <2 x i8> <i8 undef, i8 0>, [[SIGNBIT]]
-; CHECK-NEXT:    ret <2 x i8> [[NEG]]
+; CHECK-NEXT:    ret <2 x i8> zeroinitializer
 ;
   %signbit = shl <2 x i8> %x, <i8 7, i8 7>
   %neg = sub nsw <2 x i8> <i8 undef, i8 0>, %signbit
diff --git a/llvm/test/Transforms/InstSimplify/rem.ll b/llvm/test/Transforms/InstSimplify/rem.ll
index 0d0f7281417..205fdedf246 100644
--- a/llvm/test/Transforms/InstSimplify/rem.ll
+++ b/llvm/test/Transforms/InstSimplify/rem.ll
@@ -19,8 +19,7 @@ define <2 x i32> @zero_dividend_vector(<2 x i32> %A) {
 
 define <2 x i32> @zero_dividend_vector_undef_elt(<2 x i32> %A) {
 ; CHECK-LABEL: @zero_dividend_vector_undef_elt(
-; CHECK-NEXT:    [[B:%.*]] = urem <2 x i32> <i32 undef, i32 0>, [[A:%.*]]
-; CHECK-NEXT:    ret <2 x i32> [[B]]
+; CHECK-NEXT:    ret <2 x i32> zeroinitializer
 ;
   %B = urem <2 x i32> <i32 undef, i32 0>, %A
   ret <2 x i32> %B
diff --git a/llvm/test/Transforms/InstSimplify/shift.ll b/llvm/test/Transforms/InstSimplify/shift.ll
index 87b3e8227c2..fe20939a3fa 100644
--- a/llvm/test/Transforms/InstSimplify/shift.ll
+++ b/llvm/test/Transforms/InstSimplify/shift.ll
@@ -19,8 +19,7 @@ define i41 @shl_0(i41 %X) {
 
 define <2 x i41> @shl_0_vec_undef_elt(<2 x i41> %X) {
 ; CHECK-LABEL: @shl_0_vec_undef_elt(
-; CHECK-NEXT:    [[B:%.*]] = shl <2 x i41> <i41 0, i41 undef>, [[X:%.*]]
-; CHECK-NEXT:    ret <2 x i41> [[B]]
+; CHECK-NEXT:    ret <2 x i41> zeroinitializer
 ;
   %B = shl <2 x i41> <i41 0, i41 undef>, %X
   ret <2 x i41> %B
@@ -44,8 +43,7 @@ define i39 @ashr_0(i39 %X) {
 
 define <2 x i141> @ashr_0_vec_undef_elt(<2 x i141> %X) {
 ; CHECK-LABEL: @ashr_0_vec_undef_elt(
-; CHECK-NEXT:    [[B:%.*]] = shl <2 x i141> <i141 undef, i141 0>, [[X:%.*]]
-; CHECK-NEXT:    ret <2 x i141> [[B]]
+; CHECK-NEXT:    ret <2 x i141> zeroinitializer
 ;
   %B = shl <2 x i141> <i141 undef, i141 0>, %X
   ret <2 x i141> %B