[Intrinsic] Unigned Saturation Addition Intrinsic

Add an intrinsic that takes 2 integers and perform unsigned saturation
addition on them.

This is a part of implementing fixed point arithmetic in clang where some of
the more complex operations will be implemented as intrinsics.

Differential Revision: https://reviews.llvm.org/D53340

llvm-svn: 344971

9 files changed, 62 insertions, 37 deletions

diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
index b73fc106a6b..7b25d9f98ff 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeDAG.cpp
@@ -1115,7 +1115,8 @@ void SelectionDAGLegalize::LegalizeOp(SDNode *Node) {
     Action = TLI.getStrictFPOperationAction(Node->getOpcode(),
                                             Node->getValueType(0));
     break;
-  case ISD::SADDSAT: {
+  case ISD::SADDSAT:
+  case ISD::UADDSAT: {
     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
     break;
   }
@@ -3460,8 +3461,9 @@ bool SelectionDAGLegalize::ExpandNode(SDNode *Node) {
     }
     break;
   }
-  case ISD::SADDSAT: {
-    Results.push_back(TLI.getExpandedSignedSaturationAddition(Node, DAG));
+  case ISD::SADDSAT:
+  case ISD::UADDSAT: {
+    Results.push_back(TLI.getExpandedSaturationAddition(Node, DAG));
     break;
   }
   case ISD::SADDO:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
index fffebaf194e..690a64e724b 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeIntegerTypes.cpp
@@ -141,7 +141,8 @@ void DAGTypeLegalizer::PromoteIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::ADDCARRY:
   case ISD::SUBCARRY:    Res = PromoteIntRes_ADDSUBCARRY(N, ResNo); break;
 
-  case ISD::SADDSAT:     Res = PromoteIntRes_SADDSAT(N); break;
+  case ISD::SADDSAT:
+  case ISD::UADDSAT:     Res = PromoteIntRes_ADDSAT(N); break;
 
   case ISD::ATOMIC_LOAD:
     Res = PromoteIntRes_Atomic0(cast<AtomicSDNode>(N)); break;
@@ -548,17 +549,22 @@ SDValue DAGTypeLegalizer::PromoteIntRes_Overflow(SDNode *N) {
   return SDValue(Res.getNode(), 1);
 }
 
-SDValue DAGTypeLegalizer::PromoteIntRes_SADDSAT(SDNode *N) {
+SDValue DAGTypeLegalizer::PromoteIntRes_ADDSAT(SDNode *N) {
   // For promoting iN -> iM, this can be expanded by
   // 1. ANY_EXTEND iN to iM
   // 2. SHL by M-N
-  // 3. SADDSAT
-  // 4. ASHR by M-N
+  // 3. U/SADDSAT
+  // 4. L/ASHR by M-N
   SDLoc dl(N);
   SDValue Op1 = N->getOperand(0);
   SDValue Op2 = N->getOperand(1);
   unsigned OldBits = Op1.getValueSizeInBits();
 
+  unsigned Opcode = N->getOpcode();
+  assert((Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) &&
+         "Expected opcode to be SADDSAT or UADDSAT");
+  unsigned ShiftOp = Opcode == ISD::SADDSAT ? ISD::SRA : ISD::SRL;
+
   SDValue Op1Promoted = GetPromotedInteger(Op1);
   SDValue Op2Promoted = GetPromotedInteger(Op2);
 
@@ -573,8 +579,8 @@ SDValue DAGTypeLegalizer::PromoteIntRes_SADDSAT(SDNode *N) {
       DAG.getNode(ISD::SHL, dl, PromotedType, Op2Promoted, ShiftAmount);
 
   SDValue Result =
-      DAG.getNode(ISD::SADDSAT, dl, PromotedType, Op1Promoted, Op2Promoted);
-  return DAG.getNode(ISD::SRA, dl, PromotedType, Result, ShiftAmount);
+      DAG.getNode(Opcode, dl, PromotedType, Op1Promoted, Op2Promoted);
+  return DAG.getNode(ShiftOp, dl, PromotedType, Result, ShiftAmount);
 }
 
 SDValue DAGTypeLegalizer::PromoteIntRes_SADDSUBO(SDNode *N, unsigned ResNo) {
@@ -1498,7 +1504,8 @@ void DAGTypeLegalizer::ExpandIntegerResult(SDNode *N, unsigned ResNo) {
   case ISD::UMULO:
   case ISD::SMULO: ExpandIntRes_XMULO(N, Lo, Hi); break;
 
-  case ISD::SADDSAT: ExpandIntRes_SADDSAT(N, Lo, Hi); break;
+  case ISD::SADDSAT:
+  case ISD::UADDSAT: ExpandIntRes_ADDSAT(N, Lo, Hi); break;
   }
 
   // If Lo/Hi is null, the sub-method took care of registering results etc.
@@ -2461,9 +2468,9 @@ void DAGTypeLegalizer::ExpandIntRes_READCYCLECOUNTER(SDNode *N, SDValue &Lo,
   ReplaceValueWith(SDValue(N, 1), R.getValue(2));
 }
 
-void DAGTypeLegalizer::ExpandIntRes_SADDSAT(SDNode *N, SDValue &Lo,
-                                            SDValue &Hi) {
-  SDValue Result = TLI.getExpandedSignedSaturationAddition(N, DAG);
+void DAGTypeLegalizer::ExpandIntRes_ADDSAT(SDNode *N, SDValue &Lo,
+                                           SDValue &Hi) {
+  SDValue Result = TLI.getExpandedSaturationAddition(N, DAG);
   SplitInteger(Result, Lo, Hi);
 }
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
index 83429ec6e98..f31b115bc2d 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeTypes.h
@@ -330,7 +330,7 @@ private:
   SDValue PromoteIntRes_UNDEF(SDNode *N);
   SDValue PromoteIntRes_VAARG(SDNode *N);
   SDValue PromoteIntRes_XMULO(SDNode *N, unsigned ResNo);
-  SDValue PromoteIntRes_SADDSAT(SDNode *N);
+  SDValue PromoteIntRes_ADDSAT(SDNode *N);
 
   // Integer Operand Promotion.
   bool PromoteIntegerOperand(SDNode *N, unsigned OpNo);
@@ -415,7 +415,7 @@ private:
   void ExpandIntRes_SADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_UADDSUBO          (SDNode *N, SDValue &Lo, SDValue &Hi);
   void ExpandIntRes_XMULO             (SDNode *N, SDValue &Lo, SDValue &Hi);
-  void ExpandIntRes_SADDSAT           (SDNode *N, SDValue &Lo, SDValue &Hi);
+  void ExpandIntRes_ADDSAT            (SDNode *N, SDValue &Lo, SDValue &Hi);
 
   void ExpandIntRes_ATOMIC_LOAD       (SDNode *N, SDValue &Lo, SDValue &Hi);
 
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
index e7edc0ef860..787091a7f4c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorOps.cpp
@@ -390,6 +390,7 @@ SDValue VectorLegalizer::LegalizeOp(SDValue Op) {
   case ISD::UMUL_LOHI:
   case ISD::FCANONICALIZE:
   case ISD::SADDSAT:
+  case ISD::UADDSAT:
     Action = TLI.getOperationAction(Node->getOpcode(), Node->getValueType(0));
     break;
   case ISD::FP_ROUND_INREG:
diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
index 2b5fd8d75f4..43b4bf0c497 100644
--- a/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeVectorTypes.cpp
@@ -123,6 +123,7 @@ void DAGTypeLegalizer::ScalarizeVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::UMAX:
 
   case ISD::SADDSAT:
+  case ISD::UADDSAT:
 
   case ISD::FPOW:
   case ISD::FREM:
@@ -805,6 +806,7 @@ void DAGTypeLegalizer::SplitVectorResult(SDNode *N, unsigned ResNo) {
   case ISD::UMIN:
   case ISD::UMAX:
   case ISD::SADDSAT:
+  case ISD::UADDSAT:
     SplitVecRes_BinOp(N, Lo, Hi);
     break;
   case ISD::FMA:
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
index be4a219efe5..71814d79098 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp
@@ -5777,6 +5777,12 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) {
     setValue(&I, DAG.getNode(ISD::SADDSAT, sdl, Op1.getValueType(), Op1, Op2));
     return nullptr;
   }
+  case Intrinsic::uadd_sat: {
+    SDValue Op1 = getValue(I.getArgOperand(0));
+    SDValue Op2 = getValue(I.getArgOperand(1));
+    setValue(&I, DAG.getNode(ISD::UADDSAT, sdl, Op1.getValueType(), Op1, Op2));
+    return nullptr;
+  }
   case Intrinsic::stacksave: {
     SDValue Op = getRoot();
     Res = DAG.getNode(
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
index 64a9764cce2..1b032ce456a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp
@@ -286,6 +286,7 @@ std::string SDNode::getOperationName(const SelectionDAG *G) const {
   case ISD::SRL_PARTS:                  return "srl_parts";
 
   case ISD::SADDSAT:                    return "saddsat";
+  case ISD::UADDSAT:                    return "uaddsat";
 
   // Conversion operators.
   case ISD::SIGN_EXTEND:                return "sign_extend";
diff --git a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
index ceedd06da1d..d31d6344519 100644
--- a/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/TargetLowering.cpp
@@ -4681,13 +4681,12 @@ SDValue TargetLowering::lowerCmpEqZeroToCtlzSrl(SDValue Op,
   return SDValue();
 }
 
-SDValue
-TargetLowering::getExpandedSignedSaturationAddition(SDNode *Node,
-                                                    SelectionDAG &DAG) const {
-  assert(Node->getOpcode() == ISD::SADDSAT &&
-         "Expected method to receive SADDSAT node.");
-  assert(Node->getNumOperands() == 2 &&
-         "Expected SADDSAT node to have 2 operands.");
+SDValue TargetLowering::getExpandedSaturationAddition(SDNode *Node,
+                                                      SelectionDAG &DAG) const {
+  unsigned Opcode = Node->getOpcode();
+  assert((Opcode == ISD::SADDSAT || Opcode == ISD::UADDSAT) &&
+         "Expected method to receive SADDSAT or UADDSAT node.");
+  assert(Node->getNumOperands() == 2 && "Expected node to have 2 operands.");
 
   SDLoc dl(Node);
   SDValue LHS = Node->getOperand(0);
@@ -4699,27 +4698,33 @@ TargetLowering::getExpandedSignedSaturationAddition(SDNode *Node,
          "Expected operands to be integers. Vector of int arguments should "
          "already be unrolled.");
   assert(LHS.getValueType() == RHS.getValueType() &&
-         "Expected both operands of SADDSAT to be the same type");
+         "Expected both operands to be the same type");
 
+  unsigned OverflowOp = Opcode == ISD::SADDSAT ? ISD::SADDO : ISD::UADDO;
   unsigned BitWidth = LHS.getValueSizeInBits();
   EVT ResultType = LHS.getValueType();
   EVT BoolVT =
       getSetCCResultType(DAG.getDataLayout(), *DAG.getContext(), ResultType);
   SDValue Result =
-      DAG.getNode(ISD::SADDO, dl, DAG.getVTList(ResultType, BoolVT), LHS, RHS);
+      DAG.getNode(OverflowOp, dl, DAG.getVTList(ResultType, BoolVT), LHS, RHS);
   SDValue Sum = Result.getValue(0);
   SDValue Overflow = Result.getValue(1);
-
-  // SatMax -> Overflow && Sum < 0
-  // SatMin -> Overflow && Sum > 0
-  SDValue Zero = DAG.getConstant(0, dl, LHS.getValueType());
-
-  SDValue SumNeg = DAG.getSetCC(dl, BoolVT, Sum, Zero, ISD::SETLT);
-  APInt MinVal = APInt::getSignedMinValue(BitWidth);
-  APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
-  SDValue SatMin = DAG.getConstant(MinVal, dl, ResultType);
-  SDValue SatMax = DAG.getConstant(MaxVal, dl, ResultType);
-
-  Result = DAG.getSelect(dl, ResultType, SumNeg, SatMax, SatMin);
-  return DAG.getSelect(dl, ResultType, Overflow, Result, Sum);
+  SDValue Zero = DAG.getConstant(0, dl, ResultType);
+
+  if (Opcode == ISD::SADDSAT) {
+    // SatMax -> Overflow && Sum < 0
+    // SatMin -> Overflow && Sum > 0
+    APInt MinVal = APInt::getSignedMinValue(BitWidth);
+    APInt MaxVal = APInt::getSignedMaxValue(BitWidth);
+    SDValue SatMin = DAG.getConstant(MinVal, dl, ResultType);
+    SDValue SatMax = DAG.getConstant(MaxVal, dl, ResultType);
+    SDValue SumNeg = DAG.getSetCC(dl, BoolVT, Sum, Zero, ISD::SETLT);
+    Result = DAG.getSelect(dl, ResultType, SumNeg, SatMax, SatMin);
+    return DAG.getSelect(dl, ResultType, Overflow, Result, Sum);
+  } else {
+    // Just need to check overflow for SatMax.
+    APInt MaxVal = APInt::getMaxValue(BitWidth);
+    SDValue SatMax = DAG.getConstant(MaxVal, dl, ResultType);
+    return DAG.getSelect(dl, ResultType, Overflow, SatMax, Sum);
+  }
 }
diff --git a/llvm/lib/CodeGen/TargetLoweringBase.cpp b/llvm/lib/CodeGen/TargetLoweringBase.cpp
index ddd5fc1df75..09c5b527956 100644
--- a/llvm/lib/CodeGen/TargetLoweringBase.cpp
+++ b/llvm/lib/CodeGen/TargetLoweringBase.cpp
@@ -611,6 +611,7 @@ void TargetLoweringBase::initActions() {
     setOperationAction(ISD::UMAX, VT, Expand);
     setOperationAction(ISD::ABS, VT, Expand);
     setOperationAction(ISD::SADDSAT, VT, Expand);
+    setOperationAction(ISD::UADDSAT, VT, Expand);
 
     // Overflow operations default to expand
     setOperationAction(ISD::SADDO, VT, Expand);