summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--llvm/lib/Target/X86/X86ISelLowering.cpp72
-rw-r--r--llvm/lib/Target/X86/X86InstrFMA.td131
-rw-r--r--llvm/lib/Target/X86/X86Subtarget.h2
3 files changed, 73 insertions, 132 deletions
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 1bbd4a76406..224cf01a8bd 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -10077,6 +10077,78 @@ X86TargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, SelectionDAG &DAG) const
SDVTList VTs = DAG.getVTList(Op.getValueType(), MVT::i32);
return DAG.getNode(Opcode, dl, VTs, NewOps.data(), NewOps.size());
}
+ case Intrinsic::x86_fma_vfmadd_ps:
+ case Intrinsic::x86_fma_vfmadd_pd:
+ case Intrinsic::x86_fma_vfmsub_ps:
+ case Intrinsic::x86_fma_vfmsub_pd:
+ case Intrinsic::x86_fma_vfnmadd_ps:
+ case Intrinsic::x86_fma_vfnmadd_pd:
+ case Intrinsic::x86_fma_vfnmsub_ps:
+ case Intrinsic::x86_fma_vfnmsub_pd:
+ case Intrinsic::x86_fma_vfmaddsub_ps:
+ case Intrinsic::x86_fma_vfmaddsub_pd:
+ case Intrinsic::x86_fma_vfmsubadd_ps:
+ case Intrinsic::x86_fma_vfmsubadd_pd:
+ case Intrinsic::x86_fma_vfmadd_ps_256:
+ case Intrinsic::x86_fma_vfmadd_pd_256:
+ case Intrinsic::x86_fma_vfmsub_ps_256:
+ case Intrinsic::x86_fma_vfmsub_pd_256:
+ case Intrinsic::x86_fma_vfnmadd_ps_256:
+ case Intrinsic::x86_fma_vfnmadd_pd_256:
+ case Intrinsic::x86_fma_vfnmsub_ps_256:
+ case Intrinsic::x86_fma_vfnmsub_pd_256:
+ case Intrinsic::x86_fma_vfmaddsub_ps_256:
+ case Intrinsic::x86_fma_vfmaddsub_pd_256:
+ case Intrinsic::x86_fma_vfmsubadd_ps_256:
+ case Intrinsic::x86_fma_vfmsubadd_pd_256: {
+ // Only lower intrinsics if FMA is enabled. FMA4 still uses patterns.
+ if (!Subtarget->hasFMA())
+ return SDValue();
+
+ unsigned Opc;
+ switch (IntNo) {
+ default: llvm_unreachable("Impossible intrinsic"); // Can't reach here.
+ case Intrinsic::x86_fma_vfmadd_ps:
+ case Intrinsic::x86_fma_vfmadd_pd:
+ case Intrinsic::x86_fma_vfmadd_ps_256:
+ case Intrinsic::x86_fma_vfmadd_pd_256:
+ Opc = X86ISD::FMADD;
+ break;
+ case Intrinsic::x86_fma_vfmsub_ps:
+ case Intrinsic::x86_fma_vfmsub_pd:
+ case Intrinsic::x86_fma_vfmsub_ps_256:
+ case Intrinsic::x86_fma_vfmsub_pd_256:
+ Opc = X86ISD::FMSUB;
+ break;
+ case Intrinsic::x86_fma_vfnmadd_ps:
+ case Intrinsic::x86_fma_vfnmadd_pd:
+ case Intrinsic::x86_fma_vfnmadd_ps_256:
+ case Intrinsic::x86_fma_vfnmadd_pd_256:
+ Opc = X86ISD::FNMADD;
+ break;
+ case Intrinsic::x86_fma_vfnmsub_ps:
+ case Intrinsic::x86_fma_vfnmsub_pd:
+ case Intrinsic::x86_fma_vfnmsub_ps_256:
+ case Intrinsic::x86_fma_vfnmsub_pd_256:
+ Opc = X86ISD::FNMSUB;
+ break;
+ case Intrinsic::x86_fma_vfmaddsub_ps:
+ case Intrinsic::x86_fma_vfmaddsub_pd:
+ case Intrinsic::x86_fma_vfmaddsub_ps_256:
+ case Intrinsic::x86_fma_vfmaddsub_pd_256:
+ Opc = X86ISD::FMADDSUB;
+ break;
+ case Intrinsic::x86_fma_vfmsubadd_ps:
+ case Intrinsic::x86_fma_vfmsubadd_pd:
+ case Intrinsic::x86_fma_vfmsubadd_ps_256:
+ case Intrinsic::x86_fma_vfmsubadd_pd_256:
+ Opc = X86ISD::FMSUBADD;
+ break;
+ }
+
+ return DAG.getNode(Opc, dl, Op.getValueType(), Op.getOperand(1),
+ Op.getOperand(2), Op.getOperand(3));
+ }
}
}
diff --git a/llvm/lib/Target/X86/X86InstrFMA.td b/llvm/lib/Target/X86/X86InstrFMA.td
index 95ee7e50ba1..a4e689e6bb3 100644
--- a/llvm/lib/Target/X86/X86InstrFMA.td
+++ b/llvm/lib/Target/X86/X86InstrFMA.td
@@ -112,137 +112,6 @@ let ExeDomain = SSEPackedDouble in {
v4f64>, VEX_W;
}
-let Predicates = [HasFMA] in {
- def : Pat<(int_x86_fma_vfmadd_ps VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMADDPSr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmadd_ps VR128:$src2, VR128:$src1,
- (memopv4f32 addr:$src3)),
- (VFMADDPSr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_ps VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMSUBPSr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_ps VR128:$src2, VR128:$src1,
- (memopv4f32 addr:$src3)),
- (VFMSUBPSr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_ps VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMADDSUBPSr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_ps VR128:$src2, VR128:$src1,
- (memopv4f32 addr:$src3)),
- (VFMADDSUBPSr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_ps VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMSUBADDPSr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_ps VR128:$src2, VR128:$src1,
- (memopv4f32 addr:$src3)),
- (VFMSUBADDPSr213m VR128:$src1, VR128:$src2, addr:$src3)>;
-
- def : Pat<(int_x86_fma_vfmadd_ps_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMADDPSr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmadd_ps_256 VR256:$src2, VR256:$src1,
- (memopv8f32 addr:$src3)),
- (VFMADDPSr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_ps_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMSUBPSr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_ps_256 VR256:$src2, VR256:$src1,
- (memopv8f32 addr:$src3)),
- (VFMSUBPSr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_ps_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMADDSUBPSr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_ps_256 VR256:$src2, VR256:$src1,
- (memopv8f32 addr:$src3)),
- (VFMADDSUBPSr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_ps_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMSUBADDPSr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_ps_256 VR256:$src2, VR256:$src1,
- (memopv8f32 addr:$src3)),
- (VFMSUBADDPSr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
-
- def : Pat<(int_x86_fma_vfmadd_pd VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMADDPDr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmadd_pd VR128:$src2, VR128:$src1,
- (memopv2f64 addr:$src3)),
- (VFMADDPDr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_pd VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMSUBPDr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_pd VR128:$src2, VR128:$src1,
- (memopv2f64 addr:$src3)),
- (VFMSUBPDr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_pd VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMADDSUBPDr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_pd VR128:$src2, VR128:$src1,
- (memopv2f64 addr:$src3)),
- (VFMADDSUBPDr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_pd VR128:$src2, VR128:$src1, VR128:$src3),
- (VFMSUBADDPDr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_pd VR128:$src2, VR128:$src1,
- (memopv2f64 addr:$src3)),
- (VFMSUBADDPDr213m VR128:$src1, VR128:$src2, addr:$src3)>;
-
- def : Pat<(int_x86_fma_vfmadd_pd_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMADDPDr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmadd_pd_256 VR256:$src2, VR256:$src1,
- (memopv4f64 addr:$src3)),
- (VFMADDPDr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_pd_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMSUBPDr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmsub_pd_256 VR256:$src2, VR256:$src1,
- (memopv4f64 addr:$src3)),
- (VFMSUBPDr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_pd_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMADDSUBPDr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmaddsub_pd_256 VR256:$src2, VR256:$src1,
- (memopv4f64 addr:$src3)),
- (VFMADDSUBPDr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_pd_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFMSUBADDPDr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfmsubadd_pd_256 VR256:$src2, VR256:$src1,
- (memopv4f64 addr:$src3)),
- (VFMSUBADDPDr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
-
- def : Pat<(int_x86_fma_vfnmadd_ps VR128:$src2, VR128:$src1, VR128:$src3),
- (VFNMADDPSr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfnmadd_ps VR128:$src2, VR128:$src1,
- (memopv4f32 addr:$src3)),
- (VFNMADDPSr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_ps VR128:$src2, VR128:$src1, VR128:$src3),
- (VFNMSUBPSr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_ps VR128:$src2, VR128:$src1,
- (memopv4f32 addr:$src3)),
- (VFNMSUBPSr213m VR128:$src1, VR128:$src2, addr:$src3)>;
-
- def : Pat<(int_x86_fma_vfnmadd_ps_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFNMADDPSr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfnmadd_ps_256 VR256:$src2, VR256:$src1,
- (memopv8f32 addr:$src3)),
- (VFNMADDPSr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_ps_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFNMSUBPSr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_ps_256 VR256:$src2, VR256:$src1,
- (memopv8f32 addr:$src3)),
- (VFNMSUBPSr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
-
- def : Pat<(int_x86_fma_vfnmadd_pd VR128:$src2, VR128:$src1, VR128:$src3),
- (VFNMADDPDr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfnmadd_pd VR128:$src2, VR128:$src1,
- (memopv2f64 addr:$src3)),
- (VFNMADDPDr213m VR128:$src1, VR128:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_pd VR128:$src2, VR128:$src1, VR128:$src3),
- (VFNMSUBPDr213r VR128:$src1, VR128:$src2, VR128:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_pd VR128:$src2, VR128:$src1,
- (memopv2f64 addr:$src3)),
- (VFNMSUBPDr213m VR128:$src1, VR128:$src2, addr:$src3)>;
-
- def : Pat<(int_x86_fma_vfnmadd_pd_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFNMADDPDr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfnmadd_pd_256 VR256:$src2, VR256:$src1,
- (memopv4f64 addr:$src3)),
- (VFNMADDPDr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_pd_256 VR256:$src2, VR256:$src1, VR256:$src3),
- (VFNMSUBPDr213rY VR256:$src1, VR256:$src2, VR256:$src3)>;
- def : Pat<(int_x86_fma_vfnmsub_pd_256 VR256:$src2, VR256:$src1,
- (memopv4f64 addr:$src3)),
- (VFNMSUBPDr213mY VR256:$src1, VR256:$src2, addr:$src3)>;
-
-} // Predicates = [HasFMA]
-
let Constraints = "$src1 = $dst" in {
multiclass fma3s_rm<bits<8> opc, string OpcodeStr, X86MemOperand x86memop,
RegisterClass RC, ValueType OpVT, PatFrag mem_frag,
diff --git a/llvm/lib/Target/X86/X86Subtarget.h b/llvm/lib/Target/X86/X86Subtarget.h
index 3280ce52fd6..8d3797ec3b4 100644
--- a/llvm/lib/Target/X86/X86Subtarget.h
+++ b/llvm/lib/Target/X86/X86Subtarget.h
@@ -205,7 +205,7 @@ public:
bool hasAES() const { return HasAES; }
bool hasPCLMUL() const { return HasPCLMUL; }
bool hasFMA() const { return HasFMA; }
- // FIXME: Favor FMA when both are enabled. Is this right?
+ // FIXME: Favor FMA when both are enabled. Is this the right thing to do?
bool hasFMA4() const { return HasFMA4 && !HasFMA; }
bool hasXOP() const { return HasXOP; }
bool hasMOVBE() const { return HasMOVBE; }
OpenPOWER on IntegriCloud