[ARM][MVE] Refactor complex vector intrinsics [NFCI]

Summary:
This patch refactors instruction selection of the complex vector
addition, multiplication and multiply-add intrinsics, so that it is
now based on TableGen patterns rather than C++ code.

It also changes the first parameter (halving vs non-halving) of the
arm_mve_vcaddq IR intrinsic to match the corresponding instruction
encoding, hence it requires some changes in the tests.

The patch addresses David's comment in https://reviews.llvm.org/D71190

Reviewers: dmgreen, ostannard, simon_tatham, MarkMurrayARM

Reviewed By: dmgreen

Subscribers: merge_guards_bot, kristof.beyls, hiraditya, cfe-commits, llvm-commits

Tags: #clang, #llvm

Differential Revision: https://reviews.llvm.org/D71245

2 files changed, 116 insertions, 197 deletions

diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 93e1a8fd2a7..887fd947bf3 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -233,22 +233,6 @@ private:
   void SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
                          uint16_t OpcodeWithNoCarry, bool Add, bool Predicated);
 
-  /// Select MVE complex vector addition intrinsic
-  /// OpcodesInt are opcodes for non-halving addition of complex integer vectors
-  /// OpcodesHInt are opcodes for halving addition of complex integer vectors
-  /// OpcodesFP are opcodes for addition of complex floating point vectors
-  void SelectMVE_VCADD(SDNode *N, const uint16_t *OpcodesInt,
-                       const uint16_t *OpcodesHInt, const uint16_t *OpcodesFP,
-                       bool Predicated);
-
-  /// Select MVE complex vector multiplication intrinsic
-  void SelectMVE_VCMUL(SDNode *N, uint16_t OpcodeF16, uint16_t OpcodeF32,
-                       bool Predicated);
-
-  /// Sekect NVE complex vector multiply-add intrinsic
-  void SelectMVE_VCMLA(SDNode *N, uint16_t OpcodeF16, uint16_t OpcodeF32,
-                       bool Predicated);
-
   /// SelectMVE_VLD - Select MVE interleaving load intrinsics. NumVecs
   /// should be 2 or 4. The opcode array specifies the instructions
   /// used for 8, 16 and 32-bit lane sizes respectively, and each
@@ -2533,138 +2517,6 @@ void ARMDAGToDAGISel::SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
   CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
 }
 
-/// Convert an SDValue to a boolean value. SDVal must be a compile-time constant
-static bool SDValueToConstBool(SDValue SDVal) {
-  ConstantSDNode *SDValConstant = dyn_cast<ConstantSDNode>(SDVal);
-  assert(SDValConstant && "expected a compile-time constant");
-  uint64_t Value = SDValConstant->getZExtValue();
-  assert((Value == 0 || Value == 1) && "expected value 0 or 1");
-  return Value;
-}
-
-/// Select an opcode based on a floating point vector type. One opcode
-/// corresponds to 16-bit floating point element type, the other to two 32-bit
-/// element type.
-/// Other types are not allowed
-static uint16_t SelectFPOpcode(EVT VT, uint16_t OpcodeF16, uint16_t OpcodeF32) {
-  assert(VT.isFloatingPoint() && VT.isVector() &&
-         "expected a floating-point vector");
-  switch (VT.getVectorElementType().getSizeInBits()) {
-  case 16:
-    return OpcodeF16;
-  case 32:
-    return OpcodeF32;
-  default:
-    llvm_unreachable("bad vector element size");
-  }
-}
-
-void ARMDAGToDAGISel::SelectMVE_VCADD(SDNode *N, const uint16_t *OpcodesInt,
-                                      const uint16_t *OpcodesHInt,
-                                      const uint16_t *OpcodesFP,
-                                      bool Predicated) {
-  EVT VT = N->getValueType(0);
-  SDLoc Loc(N);
-
-  bool IsHalved = SDValueToConstBool(N->getOperand(1));
-  bool IsAngle270 = SDValueToConstBool(N->getOperand(2));
-  bool IsFP = VT.isFloatingPoint();
-  if (IsHalved)
-    assert(!IsFP && "vhcaddq requires integer vector type");
-
-  uint16_t Opcode;
-  if (IsFP) {
-    Opcode = SelectFPOpcode(VT, OpcodesFP[0], OpcodesFP[1]);
-  } else {
-    const uint16_t *Opcodes = IsHalved ? OpcodesHInt : OpcodesInt;
-    switch (VT.getVectorElementType().getSizeInBits()) {
-    case 8:
-      Opcode = Opcodes[0];
-      break;
-    case 16:
-      Opcode = Opcodes[1];
-      break;
-    case 32:
-      Opcode = Opcodes[2];
-      break;
-    default:
-      llvm_unreachable("bad vector element size");
-   }
-  }
-
-  int FirstInputOp = Predicated ? 4 : 3;
-  SmallVector<SDValue, 8> Ops;
-  // Vectors
-  Ops.push_back(N->getOperand(FirstInputOp));
-  Ops.push_back(N->getOperand(FirstInputOp + 1));
-  // Rotation
-  Ops.push_back(CurDAG->getTargetConstant(IsAngle270, Loc, MVT::i32));
-
-  if (Predicated)
-    AddMVEPredicateToOps(Ops, Loc,
-                         N->getOperand(FirstInputOp + 2),  // predicate
-                         N->getOperand(FirstInputOp - 1)); // inactive
-  else
-    AddEmptyMVEPredicateToOps(Ops, Loc, VT);
-
-  CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
-}
-
-static uint32_t GetCMulRotation(SDValue V) {
-  const ConstantSDNode *RotConstant = dyn_cast<ConstantSDNode>(V);
-  assert(RotConstant && "expected a compile-time constant");
-  uint64_t RotValue = RotConstant->getZExtValue();
-  assert(RotValue < 4 && "expected value in range [0, 3]");
-  return RotValue;
-}
-
-void ARMDAGToDAGISel::SelectMVE_VCMUL(SDNode *N, uint16_t OpcodeF16,
-                                      uint16_t OpcodeF32, bool Predicated) {
-  EVT VT = N->getValueType(0);
-  SDLoc Loc(N);
-
-  int FirstInputOp = Predicated ? 3 : 2;
-  SmallVector<SDValue, 8> Ops;
-  // Vectors
-  Ops.push_back(N->getOperand(FirstInputOp));
-  Ops.push_back(N->getOperand(FirstInputOp + 1));
-  // Rotation
-  uint32_t RotValue = GetCMulRotation(N->getOperand(1));
-  Ops.push_back(CurDAG->getTargetConstant(RotValue, Loc, MVT::i32));
-
-  if (Predicated)
-    AddMVEPredicateToOps(Ops, Loc,
-                         N->getOperand(FirstInputOp + 2),  // predicate
-                         N->getOperand(FirstInputOp - 1)); // inactive
-  else
-    AddEmptyMVEPredicateToOps(Ops, Loc, VT);
-
-  uint16_t Opcode = SelectFPOpcode(VT, OpcodeF16, OpcodeF32);
-  CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
-}
-
-void ARMDAGToDAGISel::SelectMVE_VCMLA(SDNode *N, uint16_t OpcodeF16,
-                                      uint16_t OpcodeF32, bool Predicated) {
-  SDLoc Loc(N);
-
-  SmallVector<SDValue, 8> Ops;
-  // The 3 vector operands
-  for (int i = 2; i < 5; ++i)
-    Ops.push_back(N->getOperand(i));
-  // Rotation
-  uint32_t RotValue = GetCMulRotation(N->getOperand(1));
-  Ops.push_back(CurDAG->getTargetConstant(RotValue, Loc, MVT::i32));
-
-  if (Predicated)
-    AddMVEPredicateToOps(Ops, Loc, N->getOperand(5));
-  else
-    AddEmptyMVEPredicateToOps(Ops, Loc);
-
-  EVT VT = N->getValueType(0);
-  uint16_t Opcode = SelectFPOpcode(VT, OpcodeF16, OpcodeF32);
-  CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
-}
-
 void ARMDAGToDAGISel::SelectMVE_VLD(SDNode *N, unsigned NumVecs,
                                     const uint16_t *const *Opcodes) {
   EVT VT = N->getValueType(0);
@@ -4510,35 +4362,6 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
                         IntNo == Intrinsic::arm_mve_vadc_predicated);
       return;
 
-    case Intrinsic::arm_mve_vcaddq:
-    case Intrinsic::arm_mve_vcaddq_predicated: {
-      static const uint16_t OpcodesInt[] = {
-        ARM::MVE_VCADDi8, ARM::MVE_VCADDi16, ARM::MVE_VCADDi32,
-      };
-      static const uint16_t OpcodesHInt[] = {
-        ARM::MVE_VHCADDs8, ARM::MVE_VHCADDs16, ARM::MVE_VHCADDs32,
-      };
-      static const uint16_t OpcodesFP[] = {
-        ARM::MVE_VCADDf16, ARM::MVE_VCADDf32,
-      };
-
-      SelectMVE_VCADD(N, OpcodesInt, OpcodesHInt,
-                      OpcodesFP, IntNo == Intrinsic::arm_mve_vcaddq_predicated);
-      return;
-    }
-
-    case Intrinsic::arm_mve_vcmulq:
-    case Intrinsic::arm_mve_vcmulq_predicated:
-      SelectMVE_VCMUL(N, ARM::MVE_VCMULf16, ARM::MVE_VCMULf32,
-                      IntNo == Intrinsic::arm_mve_vcmulq_predicated);
-      return;
-
-    case Intrinsic::arm_mve_vcmlaq:
-    case Intrinsic::arm_mve_vcmlaq_predicated:
-      SelectMVE_VCMLA(N, ARM::MVE_VCMLAf16, ARM::MVE_VCMLAf32,
-                      IntNo == Intrinsic::arm_mve_vcmlaq_predicated);
-      return;
-
     }
     break;
   }
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index d351ae8905b..7bc067c25ef 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -2960,10 +2960,10 @@ multiclass MVE_VMUL_fp_m<MVEVectorVTInfo VTI>
 defm MVE_VMULf32 : MVE_VMUL_fp_m<MVE_v4f32>;
 defm MVE_VMULf16 : MVE_VMUL_fp_m<MVE_v8f16>;
 
-class MVE_VCMLA<string suffix, bit size, list<dag> pattern=[]>
+class MVE_VCMLA<string suffix, bit size>
   : MVEFloatArithNeon<"vcmla", suffix, size, (outs MQPR:$Qd),
                          (ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
-                         "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src", pattern> {
+                         "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src", []> {
   bits<4> Qd;
   bits<4> Qn;
   bits<2> rot;
@@ -2980,8 +2980,32 @@ class MVE_VCMLA<string suffix, bit size, list<dag> pattern=[]>
   let Inst{4} = 0b0;
 }
 
-def MVE_VCMLAf16 : MVE_VCMLA<"f16", 0b0>;
-def MVE_VCMLAf32 : MVE_VCMLA<"f32", 0b1>;
+multiclass MVE_VCMLA_m<MVEVectorVTInfo VTI, bit size> {
+  def "" : MVE_VCMLA<VTI.Suffix, size>;
+
+  let Predicates = [HasMVEFloat] in {
+    def : Pat<(VTI.Vec (int_arm_mve_vcmlaq
+                            imm:$rot, (VTI.Vec MQPR:$Qd_src),
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qd_src),
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            imm:$rot))>;
+
+    def : Pat<(VTI.Vec (int_arm_mve_vcmlaq_predicated
+                            imm:$rot, (VTI.Vec MQPR:$Qd_src),
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            (VTI.Pred VCCR:$mask))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qn),
+                            (VTI.Vec MQPR:$Qm), imm:$rot,
+                            ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
+
+  }
+}
+
+defm MVE_VCMLAf16 : MVE_VCMLA_m<MVE_v8f16, 0b0>;
+defm MVE_VCMLAf32 : MVE_VCMLA_m<MVE_v4f32, 0b1>;
 
 class MVE_VADDSUBFMA_fp<string iname, string suffix, bit size, bit bit_4,
                         bit bit_8, bit bit_21, dag iops=(ins),
@@ -3056,10 +3080,10 @@ defm MVE_VADDf16 : MVE_VADD_fp_m<MVE_v8f16>;
 defm MVE_VSUBf32 : MVE_VSUB_fp_m<MVE_v4f32>;
 defm MVE_VSUBf16 : MVE_VSUB_fp_m<MVE_v8f16>;
 
-class MVE_VCADD<string suffix, bit size, string cstr="", list<dag> pattern=[]>
+class MVE_VCADD<string suffix, bit size, string cstr="">
   : MVEFloatArithNeon<"vcadd", suffix, size, (outs MQPR:$Qd),
                          (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
-                         "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, pattern> {
+                         "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
   bits<4> Qd;
   bits<4> Qn;
   bit rot;
@@ -3077,8 +3101,31 @@ class MVE_VCADD<string suffix, bit size, string cstr="", list<dag> pattern=[]>
   let Inst{4} = 0b0;
 }
 
-def MVE_VCADDf16 : MVE_VCADD<"f16", 0b0>;
-def MVE_VCADDf32 : MVE_VCADD<"f32", 0b1, "@earlyclobber $Qd">;
+multiclass MVE_VCADD_m<MVEVectorVTInfo VTI, bit size, string cstr=""> {
+  def "" : MVE_VCADD<VTI.Suffix, size, cstr>;
+
+  let Predicates = [HasMVEFloat] in {
+    def : Pat<(VTI.Vec (int_arm_mve_vcaddq (i32 1),
+                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            imm:$rot))>;
+
+    def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated (i32 1),
+                            imm:$rot, (VTI.Vec MQPR:$inactive),
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            (VTI.Pred VCCR:$mask))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            imm:$rot,
+                            ARMVCCThen, (VTI.Pred VCCR:$mask),
+                            (VTI.Vec MQPR:$inactive)))>;
+
+  }
+}
+
+defm MVE_VCADDf16 : MVE_VCADD_m<MVE_v8f16, 0b0>;
+defm MVE_VCADDf32 : MVE_VCADD_m<MVE_v4f32, 0b1, "@earlyclobber $Qd">;
 
 class MVE_VABD_fp<string suffix, bit size>
   : MVE_float<"vabd", suffix, (outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
@@ -3690,10 +3737,10 @@ defm MVE_VQDMLSDHX  : MVE_VQxDMLxDH_multi<"vqdmlsdhx",  0b1, 0b0, 0b1>;
 defm MVE_VQRDMLSDH  : MVE_VQxDMLxDH_multi<"vqrdmlsdh",  0b0, 0b1, 0b1>;
 defm MVE_VQRDMLSDHX : MVE_VQxDMLxDH_multi<"vqrdmlsdhx", 0b1, 0b1, 0b1>;
 
-class MVE_VCMUL<string iname, string suffix, bit size, string cstr="", list<dag> pattern=[]>
+class MVE_VCMUL<string iname, string suffix, bit size, string cstr="">
   : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
                    (ins MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
-                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, pattern> {
+                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
   bits<4> Qn;
   bits<2> rot;
 
@@ -3709,8 +3756,33 @@ class MVE_VCMUL<string iname, string suffix, bit size, string cstr="", list<dag>
   let Predicates = [HasMVEFloat];
 }
 
-def MVE_VCMULf16 : MVE_VCMUL<"vcmul", "f16", 0b0>;
-def MVE_VCMULf32 : MVE_VCMUL<"vcmul", "f32", 0b1, "@earlyclobber $Qd">;
+multiclass MVE_VCMUL_m<string iname, MVEVectorVTInfo VTI,
+                       bit size, string cstr=""> {
+  def "" : MVE_VCMUL<iname, VTI.Suffix, size, cstr>;
+
+
+  let Predicates = [HasMVEFloat] in {
+    def : Pat<(VTI.Vec (int_arm_mve_vcmulq
+                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            imm:$rot))>;
+
+    def : Pat<(VTI.Vec (int_arm_mve_vcmulq_predicated
+                            imm:$rot, (VTI.Vec MQPR:$inactive),
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            (VTI.Pred VCCR:$mask))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            imm:$rot,
+                            ARMVCCThen, (VTI.Pred VCCR:$mask),
+                            (VTI.Vec MQPR:$inactive)))>;
+
+  }
+}
+
+defm MVE_VCMULf16 : MVE_VCMUL_m<"vcmul", MVE_v8f16, 0b0>;
+defm MVE_VCMULf32 : MVE_VCMUL_m<"vcmul", MVE_v4f32, 0b1, "@earlyclobber $Qd">;
 
 class MVE_VMULL<string iname, string suffix, bit bit_28, bits<2> bits_21_20,
                 bit T, string cstr, list<dag> pattern=[]>
@@ -3938,10 +4010,10 @@ defm MVE_VCVTf32f16bh : MVE_VCVT_h2f_m<"vcvtb", 0b0>;
 defm MVE_VCVTf32f16th : MVE_VCVT_h2f_m<"vcvtt", 0b1>;
 
 class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,
-                 string cstr="", list<dag> pattern=[]>
+                 string cstr="">
   : MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
                    (ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
-                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, pattern> {
+                   "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
   bits<4> Qn;
   bit rot;
 
@@ -3955,13 +4027,37 @@ class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,
   let Inst{0} = 0b0;
 }
 
-def MVE_VCADDi8   : MVE_VxCADD<"vcadd", "i8", 0b00, 0b1>;
-def MVE_VCADDi16  : MVE_VxCADD<"vcadd", "i16", 0b01, 0b1>;
-def MVE_VCADDi32  : MVE_VxCADD<"vcadd", "i32", 0b10, 0b1, "@earlyclobber $Qd">;
+multiclass MVE_VxCADD_m<string iname, MVEVectorVTInfo VTI,
+                        bit halve, string cstr=""> {
+  def "" : MVE_VxCADD<iname, VTI.Suffix, VTI.Size, halve, cstr>;
+
+  let Predicates = [HasMVEInt] in {
+    def : Pat<(VTI.Vec (int_arm_mve_vcaddq halve,
+                            imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            imm:$rot))>;
+
+    def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated halve,
+                            imm:$rot, (VTI.Vec MQPR:$inactive),
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            (VTI.Pred VCCR:$mask))),
+              (VTI.Vec (!cast<Instruction>(NAME)
+                            (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+                            imm:$rot,
+                            ARMVCCThen, (VTI.Pred VCCR:$mask),
+                            (VTI.Vec MQPR:$inactive)))>;
+
+  }
+}
+
+defm MVE_VCADDi8   : MVE_VxCADD_m<"vcadd", MVE_v16i8, 0b1>;
+defm MVE_VCADDi16  : MVE_VxCADD_m<"vcadd", MVE_v8i16, 0b1>;
+defm MVE_VCADDi32  : MVE_VxCADD_m<"vcadd", MVE_v4i32, 0b1, "@earlyclobber $Qd">;
 
-def MVE_VHCADDs8  : MVE_VxCADD<"vhcadd", "s8", 0b00, 0b0>;
-def MVE_VHCADDs16 : MVE_VxCADD<"vhcadd", "s16", 0b01, 0b0>;
-def MVE_VHCADDs32 : MVE_VxCADD<"vhcadd", "s32", 0b10, 0b0, "@earlyclobber $Qd">;
+defm MVE_VHCADDs8  : MVE_VxCADD_m<"vhcadd", MVE_v16s8, 0b0>;
+defm MVE_VHCADDs16 : MVE_VxCADD_m<"vhcadd", MVE_v8s16, 0b0>;
+defm MVE_VHCADDs32 : MVE_VxCADD_m<"vhcadd", MVE_v4s32, 0b0, "@earlyclobber $Qd">;
 
 class MVE_VADCSBC<string iname, bit I, bit subtract,
                   dag carryin, list<dag> pattern=[]>