[InstSimplify] fold icmp based on range of abs/nabs (2nd try)

This is retrying the fold from rL345717 
(reverted at rL347780)
...with a fix for the miscompile
demonstrated by PR39510:
https://bugs.llvm.org/show_bug.cgi?id=39510

Original commit message:

This is a fix for PR39475:
https://bugs.llvm.org/show_bug.cgi?id=39475

We managed to get some of these patterns using computeKnownBits in https://reviews.llvm.org/D47041, but that
can't be used for nabs(). Instead, put in some range-based logic, so we can fold
both abs/nabs with icmp with a constant value.

Alive proofs:
https://rise4fun.com/Alive/21r

Name: abs_nsw_is_positive

  %cmp = icmp slt i32 %x, 0
  %negx = sub nsw i32 0, %x
  %abs = select i1 %cmp, i32 %negx, i32 %x
  %r = icmp sgt i32 %abs, -1
    =>
  %r = i1 true


Name: abs_nsw_is_not_negative

  %cmp = icmp slt i32 %x, 0
  %negx = sub nsw i32 0, %x
  %abs = select i1 %cmp, i32 %negx, i32 %x
  %r = icmp slt i32 %abs, 0
    =>
  %r = i1 false


Name: nabs_is_negative_or_0

  %cmp = icmp slt i32 %x, 0
  %negx = sub i32 0, %x
  %nabs = select i1 %cmp, i32 %x, i32 %negx
  %r = icmp slt i32 %nabs, 1
    =>
  %r = i1 true

Name: nabs_is_not_over_0

  %cmp = icmp slt i32 %x, 0
  %negx = sub i32 0, %x
  %nabs = select i1 %cmp, i32 %x, i32 %negx
  %r = icmp sgt i32 %nabs, 0
    =>
  %r = i1 false

Differential Revision: https://reviews.llvm.org/D53844

llvm-svn: 345832

2 files changed, 56 insertions, 75 deletions

diff --git a/llvm/lib/Analysis/InstructionSimplify.cpp b/llvm/lib/Analysis/InstructionSimplify.cpp
index c4b076341fc..db929aa7059 100644
--- a/llvm/lib/Analysis/InstructionSimplify.cpp
+++ b/llvm/lib/Analysis/InstructionSimplify.cpp
@@ -2996,6 +2996,44 @@ static Value *simplifyICmpWithBinOp(CmpInst::Predicate Pred, Value *LHS,
   return nullptr;
 }
 
+static Value *simplifyICmpWithAbsNabs(CmpInst::Predicate Pred, Value *Op0,
+                                      Value *Op1) {
+  // We need a comparison with a constant.
+  const APInt *C;
+  if (!match(Op1, m_APInt(C)))
+    return nullptr;
+
+  // matchSelectPattern returns the negation part of an abs pattern in SP1.
+  // If the negate has an NSW flag, abs(INT_MIN) is undefined. Without that
+  // constraint, we can't make a contiguous range for the result of abs.
+  ICmpInst::Predicate AbsPred = ICmpInst::BAD_ICMP_PREDICATE;
+  Value *SP0, *SP1;
+  SelectPatternFlavor SPF = matchSelectPattern(Op0, SP0, SP1).Flavor;
+  if (SPF == SelectPatternFlavor::SPF_ABS &&
+      cast<Instruction>(SP1)->hasNoSignedWrap())
+    // The result of abs(X) is >= 0 (with nsw).
+    AbsPred = ICmpInst::ICMP_SGE;
+  if (SPF == SelectPatternFlavor::SPF_NABS)
+    // The result of -abs(X) is <= 0.
+    AbsPred = ICmpInst::ICMP_SLE;
+
+  if (AbsPred == ICmpInst::BAD_ICMP_PREDICATE)
+    return nullptr;
+
+  // If there is no intersection between abs/nabs and the range of this icmp,
+  // the icmp must be false. If the abs/nabs range is a subset of the icmp
+  // range, the icmp must be true.
+  APInt Zero = APInt::getNullValue(C->getBitWidth());
+  ConstantRange AbsRange = ConstantRange::makeExactICmpRegion(AbsPred, Zero);
+  ConstantRange CmpRange = ConstantRange::makeExactICmpRegion(Pred, *C);
+  if (AbsRange.intersectWith(CmpRange).isEmptySet())
+    return getFalse(GetCompareTy(Op0));
+  if (CmpRange.contains(AbsRange))
+    return getTrue(GetCompareTy(Op0));
+
+  return nullptr;
+}
+
 /// Simplify integer comparisons where at least one operand of the compare
 /// matches an integer min/max idiom.
 static Value *simplifyICmpWithMinMax(CmpInst::Predicate Pred, Value *LHS,
@@ -3427,6 +3465,9 @@ static Value *SimplifyICmpInst(unsigned Predicate, Value *LHS, Value *RHS,
   if (Value *V = simplifyICmpWithMinMax(Pred, LHS, RHS, Q, MaxRecurse))
     return V;
 
+  if (Value *V = simplifyICmpWithAbsNabs(Pred, LHS, RHS))
+    return V;
+
   // Simplify comparisons of related pointers using a powerful, recursive
   // GEP-walk when we have target data available..
   if (LHS->getType()->isPointerTy())
diff --git a/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll b/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
index c2e24de885f..41ffc33ef57 100644
--- a/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
+++ b/llvm/test/Transforms/InstSimplify/icmp-abs-nabs.ll
@@ -5,11 +5,7 @@
 
 define i1 @abs_nsw_is_positive(i32 %x) {
 ; CHECK-LABEL: @abs_nsw_is_positive(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i32 0, [[X]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[ABS]], -1
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %cmp = icmp slt i32 %x, 0
   %negx = sub nsw i32 0, %x
@@ -35,11 +31,7 @@ define i1 @abs_nsw_is_positive_sge(i32 %x) {
 
 define i1 @abs_nsw_is_positive_reduced_range(i32 %x) {
 ; CHECK-LABEL: @abs_nsw_is_positive_reduced_range(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i32 0, [[X]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[ABS]], -42
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %cmp = icmp slt i32 %x, 0
   %negx = sub nsw i32 0, %x
@@ -99,11 +91,7 @@ define i1 @abs_nsw_is_not_negative(i32 %x) {
 
 define i1 @abs_nsw_is_not_negative_sle(i32 %x) {
 ; CHECK-LABEL: @abs_nsw_is_not_negative_sle(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i32 0, [[X]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sle i32 [[ABS]], -1
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i32 %x, 1
   %negx = sub nsw i32 0, %x
@@ -116,11 +104,7 @@ define i1 @abs_nsw_is_not_negative_sle(i32 %x) {
 
 define i1 @abs_nsw_is_not_negative_reduced_range(i32 %x) {
 ; CHECK-LABEL: @abs_nsw_is_not_negative_reduced_range(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i32 0, [[X]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i32 [[NEGX]], i32 [[X]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[ABS]], -24
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i32 %x, 0
   %negx = sub nsw i32 0, %x
@@ -167,11 +151,7 @@ define i1 @abs_nsw_is_not_negative_wrong_range(i32 %x) {
 
 define i1 @nabs_is_negative_or_0(i32 %x) {
 ; CHECK-LABEL: @nabs_is_negative_or_0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[NABS]], 1
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %cmp = icmp slt i32 %x, 0
   %negx = sub i32 0, %x
@@ -184,11 +164,7 @@ define i1 @nabs_is_negative_or_0(i32 %x) {
 
 define i1 @nabs_is_negative_or_0_sle(i32 %x) {
 ; CHECK-LABEL: @nabs_is_negative_or_0_sle(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sle i32 [[NABS]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %cmp = icmp slt i32 %x, 1
   %negx = sub i32 0, %x
@@ -201,11 +177,7 @@ define i1 @nabs_is_negative_or_0_sle(i32 %x) {
 
 define i1 @nabs_is_negative_or_0_reduced_range(i32 %x) {
 ; CHECK-LABEL: @nabs_is_negative_or_0_reduced_range(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt i32 [[NABS]], 421
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %cmp = icmp slt i32 %x, 1
   %negx = sub i32 0, %x
@@ -235,11 +207,7 @@ define i1 @nabs_is_negative_or_0_wrong_range(i32 %x) {
 
 define i1 @nabs_is_not_over_0(i32 %x) {
 ; CHECK-LABEL: @nabs_is_not_over_0(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[NABS]], 0
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i32 %x, 0
   %negx = sub i32 0, %x
@@ -252,11 +220,7 @@ define i1 @nabs_is_not_over_0(i32 %x) {
 
 define i1 @nabs_is_not_over_0_sle(i32 %x) {
 ; CHECK-LABEL: @nabs_is_not_over_0_sle(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sge i32 [[NABS]], 1
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i32 %x, 1
   %negx = sub i32 0, %x
@@ -269,11 +233,7 @@ define i1 @nabs_is_not_over_0_sle(i32 %x) {
 
 define i1 @nabs_is_not_over_0_reduced_range(i32 %x) {
 ; CHECK-LABEL: @nabs_is_not_over_0_reduced_range(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i32 [[X:%.*]], 1
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i32 0, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i32 [[X]], i32 [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sgt i32 [[NABS]], 4223
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i32 %x, 1
   %negx = sub i32 0, %x
@@ -318,11 +278,7 @@ define i1 @abs_nsw_is_positive_eq(i32 %x) {
 
 define i1 @abs_nsw_is_positive_ult(i8 %x) {
 ; CHECK-LABEL: @abs_nsw_is_positive_ult(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i8 0, [[X]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[NEGX]], i8 [[X]]
-; CHECK-NEXT:    [[R:%.*]] = icmp ult i8 [[ABS]], -117
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %cmp = icmp slt i8 %x, 0
   %negx = sub nsw i8 0, %x
@@ -335,11 +291,7 @@ define i1 @abs_nsw_is_positive_ult(i8 %x) {
 
 define i1 @abs_nsw_is_not_negative_ugt(i8 %x) {
 ; CHECK-LABEL: @abs_nsw_is_not_negative_ugt(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw i8 0, [[X]]
-; CHECK-NEXT:    [[ABS:%.*]] = select i1 [[CMP]], i8 [[NEGX]], i8 [[X]]
-; CHECK-NEXT:    [[R:%.*]] = icmp ugt i8 [[ABS]], 127
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 false
 ;
   %cmp = icmp slt i8 %x, 0
   %negx = sub nsw i8 0, %x
@@ -352,11 +304,7 @@ define i1 @abs_nsw_is_not_negative_ugt(i8 %x) {
 
 define <2 x i1> @abs_nsw_is_not_negative_vec_splat(<2 x i32> %x) {
 ; CHECK-LABEL: @abs_nsw_is_not_negative_vec_splat(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <2 x i32> [[X:%.*]], zeroinitializer
-; CHECK-NEXT:    [[NEGX:%.*]] = sub nsw <2 x i32> zeroinitializer, [[X]]
-; CHECK-NEXT:    [[ABS:%.*]] = select <2 x i1> [[CMP]], <2 x i32> [[NEGX]], <2 x i32> [[X]]
-; CHECK-NEXT:    [[R:%.*]] = icmp slt <2 x i32> [[ABS]], <i32 -8, i32 -8>
-; CHECK-NEXT:    ret <2 x i1> [[R]]
+; CHECK-NEXT:    ret <2 x i1> zeroinitializer
 ;
   %cmp = icmp slt <2 x i32> %x, zeroinitializer
   %negx = sub nsw <2 x i32> zeroinitializer, %x
@@ -369,11 +317,7 @@ define <2 x i1> @abs_nsw_is_not_negative_vec_splat(<2 x i32> %x) {
 
 define i1 @nabs_is_negative_or_0_ne(i8 %x) {
 ; CHECK-LABEL: @nabs_is_negative_or_0_ne(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt i8 [[X:%.*]], 0
-; CHECK-NEXT:    [[NEGX:%.*]] = sub i8 0, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select i1 [[CMP]], i8 [[X]], i8 [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp ne i8 [[NABS]], 12
-; CHECK-NEXT:    ret i1 [[R]]
+; CHECK-NEXT:    ret i1 true
 ;
   %cmp = icmp slt i8 %x, 0
   %negx = sub i8 0, %x
@@ -386,11 +330,7 @@ define i1 @nabs_is_negative_or_0_ne(i8 %x) {
 
 define <3 x i1> @nabs_is_not_over_0_sle_vec_splat(<3 x i33> %x) {
 ; CHECK-LABEL: @nabs_is_not_over_0_sle_vec_splat(
-; CHECK-NEXT:    [[CMP:%.*]] = icmp slt <3 x i33> [[X:%.*]], <i33 1, i33 1, i33 1>
-; CHECK-NEXT:    [[NEGX:%.*]] = sub <3 x i33> zeroinitializer, [[X]]
-; CHECK-NEXT:    [[NABS:%.*]] = select <3 x i1> [[CMP]], <3 x i33> [[X]], <3 x i33> [[NEGX]]
-; CHECK-NEXT:    [[R:%.*]] = icmp sge <3 x i33> [[NABS]], <i33 1, i33 1, i33 1>
-; CHECK-NEXT:    ret <3 x i1> [[R]]
+; CHECK-NEXT:    ret <3 x i1> zeroinitializer
 ;
   %cmp = icmp slt <3 x i33> %x, <i33 1, i33 1, i33 1>
   %negx = sub <3 x i33> zeroinitializer, %x