summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp177
-rw-r--r--llvm/lib/Target/ARM/ARMInstrMVE.td136
2 files changed, 116 insertions, 197 deletions
diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
index 93e1a8fd2a7..887fd947bf3 100644
--- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
+++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp
@@ -233,22 +233,6 @@ private:
void SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
uint16_t OpcodeWithNoCarry, bool Add, bool Predicated);
- /// Select MVE complex vector addition intrinsic
- /// OpcodesInt are opcodes for non-halving addition of complex integer vectors
- /// OpcodesHInt are opcodes for halving addition of complex integer vectors
- /// OpcodesFP are opcodes for addition of complex floating point vectors
- void SelectMVE_VCADD(SDNode *N, const uint16_t *OpcodesInt,
- const uint16_t *OpcodesHInt, const uint16_t *OpcodesFP,
- bool Predicated);
-
- /// Select MVE complex vector multiplication intrinsic
- void SelectMVE_VCMUL(SDNode *N, uint16_t OpcodeF16, uint16_t OpcodeF32,
- bool Predicated);
-
- /// Sekect NVE complex vector multiply-add intrinsic
- void SelectMVE_VCMLA(SDNode *N, uint16_t OpcodeF16, uint16_t OpcodeF32,
- bool Predicated);
-
/// SelectMVE_VLD - Select MVE interleaving load intrinsics. NumVecs
/// should be 2 or 4. The opcode array specifies the instructions
/// used for 8, 16 and 32-bit lane sizes respectively, and each
@@ -2533,138 +2517,6 @@ void ARMDAGToDAGISel::SelectMVE_VADCSBC(SDNode *N, uint16_t OpcodeWithCarry,
CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
}
-/// Convert an SDValue to a boolean value. SDVal must be a compile-time constant
-static bool SDValueToConstBool(SDValue SDVal) {
- ConstantSDNode *SDValConstant = dyn_cast<ConstantSDNode>(SDVal);
- assert(SDValConstant && "expected a compile-time constant");
- uint64_t Value = SDValConstant->getZExtValue();
- assert((Value == 0 || Value == 1) && "expected value 0 or 1");
- return Value;
-}
-
-/// Select an opcode based on a floating point vector type. One opcode
-/// corresponds to 16-bit floating point element type, the other to two 32-bit
-/// element type.
-/// Other types are not allowed
-static uint16_t SelectFPOpcode(EVT VT, uint16_t OpcodeF16, uint16_t OpcodeF32) {
- assert(VT.isFloatingPoint() && VT.isVector() &&
- "expected a floating-point vector");
- switch (VT.getVectorElementType().getSizeInBits()) {
- case 16:
- return OpcodeF16;
- case 32:
- return OpcodeF32;
- default:
- llvm_unreachable("bad vector element size");
- }
-}
-
-void ARMDAGToDAGISel::SelectMVE_VCADD(SDNode *N, const uint16_t *OpcodesInt,
- const uint16_t *OpcodesHInt,
- const uint16_t *OpcodesFP,
- bool Predicated) {
- EVT VT = N->getValueType(0);
- SDLoc Loc(N);
-
- bool IsHalved = SDValueToConstBool(N->getOperand(1));
- bool IsAngle270 = SDValueToConstBool(N->getOperand(2));
- bool IsFP = VT.isFloatingPoint();
- if (IsHalved)
- assert(!IsFP && "vhcaddq requires integer vector type");
-
- uint16_t Opcode;
- if (IsFP) {
- Opcode = SelectFPOpcode(VT, OpcodesFP[0], OpcodesFP[1]);
- } else {
- const uint16_t *Opcodes = IsHalved ? OpcodesHInt : OpcodesInt;
- switch (VT.getVectorElementType().getSizeInBits()) {
- case 8:
- Opcode = Opcodes[0];
- break;
- case 16:
- Opcode = Opcodes[1];
- break;
- case 32:
- Opcode = Opcodes[2];
- break;
- default:
- llvm_unreachable("bad vector element size");
- }
- }
-
- int FirstInputOp = Predicated ? 4 : 3;
- SmallVector<SDValue, 8> Ops;
- // Vectors
- Ops.push_back(N->getOperand(FirstInputOp));
- Ops.push_back(N->getOperand(FirstInputOp + 1));
- // Rotation
- Ops.push_back(CurDAG->getTargetConstant(IsAngle270, Loc, MVT::i32));
-
- if (Predicated)
- AddMVEPredicateToOps(Ops, Loc,
- N->getOperand(FirstInputOp + 2), // predicate
- N->getOperand(FirstInputOp - 1)); // inactive
- else
- AddEmptyMVEPredicateToOps(Ops, Loc, VT);
-
- CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
-}
-
-static uint32_t GetCMulRotation(SDValue V) {
- const ConstantSDNode *RotConstant = dyn_cast<ConstantSDNode>(V);
- assert(RotConstant && "expected a compile-time constant");
- uint64_t RotValue = RotConstant->getZExtValue();
- assert(RotValue < 4 && "expected value in range [0, 3]");
- return RotValue;
-}
-
-void ARMDAGToDAGISel::SelectMVE_VCMUL(SDNode *N, uint16_t OpcodeF16,
- uint16_t OpcodeF32, bool Predicated) {
- EVT VT = N->getValueType(0);
- SDLoc Loc(N);
-
- int FirstInputOp = Predicated ? 3 : 2;
- SmallVector<SDValue, 8> Ops;
- // Vectors
- Ops.push_back(N->getOperand(FirstInputOp));
- Ops.push_back(N->getOperand(FirstInputOp + 1));
- // Rotation
- uint32_t RotValue = GetCMulRotation(N->getOperand(1));
- Ops.push_back(CurDAG->getTargetConstant(RotValue, Loc, MVT::i32));
-
- if (Predicated)
- AddMVEPredicateToOps(Ops, Loc,
- N->getOperand(FirstInputOp + 2), // predicate
- N->getOperand(FirstInputOp - 1)); // inactive
- else
- AddEmptyMVEPredicateToOps(Ops, Loc, VT);
-
- uint16_t Opcode = SelectFPOpcode(VT, OpcodeF16, OpcodeF32);
- CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
-}
-
-void ARMDAGToDAGISel::SelectMVE_VCMLA(SDNode *N, uint16_t OpcodeF16,
- uint16_t OpcodeF32, bool Predicated) {
- SDLoc Loc(N);
-
- SmallVector<SDValue, 8> Ops;
- // The 3 vector operands
- for (int i = 2; i < 5; ++i)
- Ops.push_back(N->getOperand(i));
- // Rotation
- uint32_t RotValue = GetCMulRotation(N->getOperand(1));
- Ops.push_back(CurDAG->getTargetConstant(RotValue, Loc, MVT::i32));
-
- if (Predicated)
- AddMVEPredicateToOps(Ops, Loc, N->getOperand(5));
- else
- AddEmptyMVEPredicateToOps(Ops, Loc);
-
- EVT VT = N->getValueType(0);
- uint16_t Opcode = SelectFPOpcode(VT, OpcodeF16, OpcodeF32);
- CurDAG->SelectNodeTo(N, Opcode, N->getVTList(), makeArrayRef(Ops));
-}
-
void ARMDAGToDAGISel::SelectMVE_VLD(SDNode *N, unsigned NumVecs,
const uint16_t *const *Opcodes) {
EVT VT = N->getValueType(0);
@@ -4510,35 +4362,6 @@ void ARMDAGToDAGISel::Select(SDNode *N) {
IntNo == Intrinsic::arm_mve_vadc_predicated);
return;
- case Intrinsic::arm_mve_vcaddq:
- case Intrinsic::arm_mve_vcaddq_predicated: {
- static const uint16_t OpcodesInt[] = {
- ARM::MVE_VCADDi8, ARM::MVE_VCADDi16, ARM::MVE_VCADDi32,
- };
- static const uint16_t OpcodesHInt[] = {
- ARM::MVE_VHCADDs8, ARM::MVE_VHCADDs16, ARM::MVE_VHCADDs32,
- };
- static const uint16_t OpcodesFP[] = {
- ARM::MVE_VCADDf16, ARM::MVE_VCADDf32,
- };
-
- SelectMVE_VCADD(N, OpcodesInt, OpcodesHInt,
- OpcodesFP, IntNo == Intrinsic::arm_mve_vcaddq_predicated);
- return;
- }
-
- case Intrinsic::arm_mve_vcmulq:
- case Intrinsic::arm_mve_vcmulq_predicated:
- SelectMVE_VCMUL(N, ARM::MVE_VCMULf16, ARM::MVE_VCMULf32,
- IntNo == Intrinsic::arm_mve_vcmulq_predicated);
- return;
-
- case Intrinsic::arm_mve_vcmlaq:
- case Intrinsic::arm_mve_vcmlaq_predicated:
- SelectMVE_VCMLA(N, ARM::MVE_VCMLAf16, ARM::MVE_VCMLAf32,
- IntNo == Intrinsic::arm_mve_vcmlaq_predicated);
- return;
-
}
break;
}
diff --git a/llvm/lib/Target/ARM/ARMInstrMVE.td b/llvm/lib/Target/ARM/ARMInstrMVE.td
index d351ae8905b..7bc067c25ef 100644
--- a/llvm/lib/Target/ARM/ARMInstrMVE.td
+++ b/llvm/lib/Target/ARM/ARMInstrMVE.td
@@ -2960,10 +2960,10 @@ multiclass MVE_VMUL_fp_m<MVEVectorVTInfo VTI>
defm MVE_VMULf32 : MVE_VMUL_fp_m<MVE_v4f32>;
defm MVE_VMULf16 : MVE_VMUL_fp_m<MVE_v8f16>;
-class MVE_VCMLA<string suffix, bit size, list<dag> pattern=[]>
+class MVE_VCMLA<string suffix, bit size>
: MVEFloatArithNeon<"vcmla", suffix, size, (outs MQPR:$Qd),
(ins MQPR:$Qd_src, MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
- "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src", pattern> {
+ "$Qd, $Qn, $Qm, $rot", vpred_n, "$Qd = $Qd_src", []> {
bits<4> Qd;
bits<4> Qn;
bits<2> rot;
@@ -2980,8 +2980,32 @@ class MVE_VCMLA<string suffix, bit size, list<dag> pattern=[]>
let Inst{4} = 0b0;
}
-def MVE_VCMLAf16 : MVE_VCMLA<"f16", 0b0>;
-def MVE_VCMLAf32 : MVE_VCMLA<"f32", 0b1>;
+multiclass MVE_VCMLA_m<MVEVectorVTInfo VTI, bit size> {
+ def "" : MVE_VCMLA<VTI.Suffix, size>;
+
+ let Predicates = [HasMVEFloat] in {
+ def : Pat<(VTI.Vec (int_arm_mve_vcmlaq
+ imm:$rot, (VTI.Vec MQPR:$Qd_src),
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qd_src),
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ imm:$rot))>;
+
+ def : Pat<(VTI.Vec (int_arm_mve_vcmlaq_predicated
+ imm:$rot, (VTI.Vec MQPR:$Qd_src),
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ (VTI.Pred VCCR:$mask))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qd_src), (VTI.Vec MQPR:$Qn),
+ (VTI.Vec MQPR:$Qm), imm:$rot,
+ ARMVCCThen, (VTI.Pred VCCR:$mask)))>;
+
+ }
+}
+
+defm MVE_VCMLAf16 : MVE_VCMLA_m<MVE_v8f16, 0b0>;
+defm MVE_VCMLAf32 : MVE_VCMLA_m<MVE_v4f32, 0b1>;
class MVE_VADDSUBFMA_fp<string iname, string suffix, bit size, bit bit_4,
bit bit_8, bit bit_21, dag iops=(ins),
@@ -3056,10 +3080,10 @@ defm MVE_VADDf16 : MVE_VADD_fp_m<MVE_v8f16>;
defm MVE_VSUBf32 : MVE_VSUB_fp_m<MVE_v4f32>;
defm MVE_VSUBf16 : MVE_VSUB_fp_m<MVE_v8f16>;
-class MVE_VCADD<string suffix, bit size, string cstr="", list<dag> pattern=[]>
+class MVE_VCADD<string suffix, bit size, string cstr="">
: MVEFloatArithNeon<"vcadd", suffix, size, (outs MQPR:$Qd),
(ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
- "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, pattern> {
+ "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
bits<4> Qd;
bits<4> Qn;
bit rot;
@@ -3077,8 +3101,31 @@ class MVE_VCADD<string suffix, bit size, string cstr="", list<dag> pattern=[]>
let Inst{4} = 0b0;
}
-def MVE_VCADDf16 : MVE_VCADD<"f16", 0b0>;
-def MVE_VCADDf32 : MVE_VCADD<"f32", 0b1, "@earlyclobber $Qd">;
+multiclass MVE_VCADD_m<MVEVectorVTInfo VTI, bit size, string cstr=""> {
+ def "" : MVE_VCADD<VTI.Suffix, size, cstr>;
+
+ let Predicates = [HasMVEFloat] in {
+ def : Pat<(VTI.Vec (int_arm_mve_vcaddq (i32 1),
+ imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ imm:$rot))>;
+
+ def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated (i32 1),
+ imm:$rot, (VTI.Vec MQPR:$inactive),
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ (VTI.Pred VCCR:$mask))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ imm:$rot,
+ ARMVCCThen, (VTI.Pred VCCR:$mask),
+ (VTI.Vec MQPR:$inactive)))>;
+
+ }
+}
+
+defm MVE_VCADDf16 : MVE_VCADD_m<MVE_v8f16, 0b0>;
+defm MVE_VCADDf32 : MVE_VCADD_m<MVE_v4f32, 0b1, "@earlyclobber $Qd">;
class MVE_VABD_fp<string suffix, bit size>
: MVE_float<"vabd", suffix, (outs MQPR:$Qd), (ins MQPR:$Qn, MQPR:$Qm),
@@ -3690,10 +3737,10 @@ defm MVE_VQDMLSDHX : MVE_VQxDMLxDH_multi<"vqdmlsdhx", 0b1, 0b0, 0b1>;
defm MVE_VQRDMLSDH : MVE_VQxDMLxDH_multi<"vqrdmlsdh", 0b0, 0b1, 0b1>;
defm MVE_VQRDMLSDHX : MVE_VQxDMLxDH_multi<"vqrdmlsdhx", 0b1, 0b1, 0b1>;
-class MVE_VCMUL<string iname, string suffix, bit size, string cstr="", list<dag> pattern=[]>
+class MVE_VCMUL<string iname, string suffix, bit size, string cstr="">
: MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
(ins MQPR:$Qn, MQPR:$Qm, complexrotateop:$rot),
- "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, pattern> {
+ "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
bits<4> Qn;
bits<2> rot;
@@ -3709,8 +3756,33 @@ class MVE_VCMUL<string iname, string suffix, bit size, string cstr="", list<dag>
let Predicates = [HasMVEFloat];
}
-def MVE_VCMULf16 : MVE_VCMUL<"vcmul", "f16", 0b0>;
-def MVE_VCMULf32 : MVE_VCMUL<"vcmul", "f32", 0b1, "@earlyclobber $Qd">;
+multiclass MVE_VCMUL_m<string iname, MVEVectorVTInfo VTI,
+ bit size, string cstr=""> {
+ def "" : MVE_VCMUL<iname, VTI.Suffix, size, cstr>;
+
+
+ let Predicates = [HasMVEFloat] in {
+ def : Pat<(VTI.Vec (int_arm_mve_vcmulq
+ imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ imm:$rot))>;
+
+ def : Pat<(VTI.Vec (int_arm_mve_vcmulq_predicated
+ imm:$rot, (VTI.Vec MQPR:$inactive),
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ (VTI.Pred VCCR:$mask))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ imm:$rot,
+ ARMVCCThen, (VTI.Pred VCCR:$mask),
+ (VTI.Vec MQPR:$inactive)))>;
+
+ }
+}
+
+defm MVE_VCMULf16 : MVE_VCMUL_m<"vcmul", MVE_v8f16, 0b0>;
+defm MVE_VCMULf32 : MVE_VCMUL_m<"vcmul", MVE_v4f32, 0b1, "@earlyclobber $Qd">;
class MVE_VMULL<string iname, string suffix, bit bit_28, bits<2> bits_21_20,
bit T, string cstr, list<dag> pattern=[]>
@@ -3938,10 +4010,10 @@ defm MVE_VCVTf32f16bh : MVE_VCVT_h2f_m<"vcvtb", 0b0>;
defm MVE_VCVTf32f16th : MVE_VCVT_h2f_m<"vcvtt", 0b1>;
class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,
- string cstr="", list<dag> pattern=[]>
+ string cstr="">
: MVE_qDest_qSrc<iname, suffix, (outs MQPR:$Qd),
(ins MQPR:$Qn, MQPR:$Qm, complexrotateopodd:$rot),
- "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, pattern> {
+ "$Qd, $Qn, $Qm, $rot", vpred_r, cstr, []> {
bits<4> Qn;
bit rot;
@@ -3955,13 +4027,37 @@ class MVE_VxCADD<string iname, string suffix, bits<2> size, bit halve,
let Inst{0} = 0b0;
}
-def MVE_VCADDi8 : MVE_VxCADD<"vcadd", "i8", 0b00, 0b1>;
-def MVE_VCADDi16 : MVE_VxCADD<"vcadd", "i16", 0b01, 0b1>;
-def MVE_VCADDi32 : MVE_VxCADD<"vcadd", "i32", 0b10, 0b1, "@earlyclobber $Qd">;
+multiclass MVE_VxCADD_m<string iname, MVEVectorVTInfo VTI,
+ bit halve, string cstr=""> {
+ def "" : MVE_VxCADD<iname, VTI.Suffix, VTI.Size, halve, cstr>;
+
+ let Predicates = [HasMVEInt] in {
+ def : Pat<(VTI.Vec (int_arm_mve_vcaddq halve,
+ imm:$rot, (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ imm:$rot))>;
+
+ def : Pat<(VTI.Vec (int_arm_mve_vcaddq_predicated halve,
+ imm:$rot, (VTI.Vec MQPR:$inactive),
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ (VTI.Pred VCCR:$mask))),
+ (VTI.Vec (!cast<Instruction>(NAME)
+ (VTI.Vec MQPR:$Qn), (VTI.Vec MQPR:$Qm),
+ imm:$rot,
+ ARMVCCThen, (VTI.Pred VCCR:$mask),
+ (VTI.Vec MQPR:$inactive)))>;
+
+ }
+}
+
+defm MVE_VCADDi8 : MVE_VxCADD_m<"vcadd", MVE_v16i8, 0b1>;
+defm MVE_VCADDi16 : MVE_VxCADD_m<"vcadd", MVE_v8i16, 0b1>;
+defm MVE_VCADDi32 : MVE_VxCADD_m<"vcadd", MVE_v4i32, 0b1, "@earlyclobber $Qd">;
-def MVE_VHCADDs8 : MVE_VxCADD<"vhcadd", "s8", 0b00, 0b0>;
-def MVE_VHCADDs16 : MVE_VxCADD<"vhcadd", "s16", 0b01, 0b0>;
-def MVE_VHCADDs32 : MVE_VxCADD<"vhcadd", "s32", 0b10, 0b0, "@earlyclobber $Qd">;
+defm MVE_VHCADDs8 : MVE_VxCADD_m<"vhcadd", MVE_v16s8, 0b0>;
+defm MVE_VHCADDs16 : MVE_VxCADD_m<"vhcadd", MVE_v8s16, 0b0>;
+defm MVE_VHCADDs32 : MVE_VxCADD_m<"vhcadd", MVE_v4s32, 0b0, "@earlyclobber $Qd">;
class MVE_VADCSBC<string iname, bit I, bit subtract,
dag carryin, list<dag> pattern=[]>
OpenPOWER on IntegriCloud