diff options
| -rw-r--r-- | llvm/lib/Target/R600/SIISelLowering.cpp | 37 | ||||
| -rw-r--r-- | llvm/lib/Target/R600/SIISelLowering.h | 1 | ||||
| -rw-r--r-- | llvm/lib/Target/R600/SIInstructions.td | 3 | ||||
| -rw-r--r-- | llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll | 3 | ||||
| -rw-r--r-- | llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.ll | 18 | ||||
| -rw-r--r-- | llvm/test/CodeGen/R600/rsq.ll | 10 |
6 files changed, 45 insertions, 27 deletions
diff --git a/llvm/lib/Target/R600/SIISelLowering.cpp b/llvm/lib/Target/R600/SIISelLowering.cpp index b107ed4238a..b3429b9748c 100644 --- a/llvm/lib/Target/R600/SIISelLowering.cpp +++ b/llvm/lib/Target/R600/SIISelLowering.cpp @@ -936,16 +936,27 @@ SDValue SITargetLowering::LowerSELECT(SDValue Op, SelectionDAG &DAG) const { return DAG.getNode(ISD::BITCAST, DL, MVT::i64, Res); } -static SDValue performUnsafeFDIV(SDValue Op, SelectionDAG &DAG) { +// Catch division cases where we can use shortcuts with rcp and rsq +// instructions. +SDValue SITargetLowering::LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const { SDLoc SL(Op); SDValue LHS = Op.getOperand(0); SDValue RHS = Op.getOperand(1); EVT VT = Op.getValueType(); + bool Unsafe = DAG.getTarget().Options.UnsafeFPMath; if (const ConstantFPSDNode *CLHS = dyn_cast<ConstantFPSDNode>(LHS)) { - if (CLHS->isExactlyValue(1.0)) { + if ((Unsafe || (VT == MVT::f32 && !Subtarget->hasFP32Denormals())) && + CLHS->isExactlyValue(1.0)) { + // v_rcp_f32 and v_rsq_f32 do not support denormals, and according to + // the CI documentation has a worst case error of 1 ulp. + // OpenCL requires <= 2.5 ulp for 1.0 / x, so it should always be OK to + // use it as long as we aren't trying to use denormals. // 1.0 / sqrt(x) -> rsq(x) + // + // XXX - Is UnsafeFPMath sufficient to do this for f64? The maximum ULP + // error seems really high at 2^29 ULP. if (RHS.getOpcode() == ISD::FSQRT) return DAG.getNode(AMDGPUISD::RSQ, SL, VT, RHS.getOperand(0)); @@ -954,15 +965,25 @@ static SDValue performUnsafeFDIV(SDValue Op, SelectionDAG &DAG) { } } - // Turn into multiply by the reciprocal - // x / y -> x * (1.0 / y) - SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); - return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip); + if (Unsafe) { + // Turn into multiply by the reciprocal. + // x / y -> x * (1.0 / y) + SDValue Recip = DAG.getNode(AMDGPUISD::RCP, SL, VT, RHS); + return DAG.getNode(ISD::FMUL, SL, VT, LHS, Recip); + } + + return SDValue(); } SDValue SITargetLowering::LowerFDIV32(SDValue Op, SelectionDAG &DAG) const { - if (DAG.getTarget().Options.UnsafeFPMath) - return performUnsafeFDIV(Op, DAG); + SDValue FastLowered = LowerFastFDIV(Op, DAG); + if (FastLowered.getNode()) + return FastLowered; + + // This uses v_rcp_f32 which does not handle denormals. Let this hit a + // selection error for now rather than do something incorrect. + if (Subtarget->hasFP32Denormals()) + return SDValue(); SDLoc SL(Op); SDValue LHS = Op.getOperand(0); diff --git a/llvm/lib/Target/R600/SIISelLowering.h b/llvm/lib/Target/R600/SIISelLowering.h index aa09d2c9f8e..9e9a0b05f53 100644 --- a/llvm/lib/Target/R600/SIISelLowering.h +++ b/llvm/lib/Target/R600/SIISelLowering.h @@ -27,6 +27,7 @@ class SITargetLowering : public AMDGPUTargetLowering { SelectionDAG &DAG) const; SDValue LowerLOAD(SDValue Op, SelectionDAG &DAG) const; SDValue LowerSELECT(SDValue Op, SelectionDAG &DAG) const; + SDValue LowerFastFDIV(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFDIV32(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFDIV64(SDValue Op, SelectionDAG &DAG) const; SDValue LowerFDIV(SDValue Op, SelectionDAG &DAG) const; diff --git a/llvm/lib/Target/R600/SIInstructions.td b/llvm/lib/Target/R600/SIInstructions.td index 1d2dd2f63a6..1a9be6c6157 100644 --- a/llvm/lib/Target/R600/SIInstructions.td +++ b/llvm/lib/Target/R600/SIInstructions.td @@ -1800,10 +1800,9 @@ def : Pat < // VOP1 Patterns //===----------------------------------------------------------------------===// +let Predicates = [UnsafeFPMath] in { def : RcpPat<V_RCP_F64_e32, f64>; defm : RsqPat<V_RSQ_F64_e32, f64>; - -let Predicates = [UnsafeFPMath] in { defm : RsqPat<V_RSQ_F32_e32, f32>; } diff --git a/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll b/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll index 62a30a79a2d..b5dda0ce81f 100644 --- a/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll +++ b/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.f64.ll @@ -20,7 +20,8 @@ define void @rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind { } ; FUNC-LABEL: @rsq_rcp_pat_f64 -; SI: V_RSQ_F64_e32 +; SI-UNSAFE: V_RSQ_F64_e32 +; SI-SAFE-NOT: V_RSQ_F64_e32 define void @rsq_rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind { %sqrt = call double @llvm.sqrt.f64(double %src) nounwind readnone %rcp = call double @llvm.AMDGPU.rcp.f64(double %sqrt) nounwind readnone diff --git a/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.ll b/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.ll index 42910a99afd..8d5d66e149b 100644 --- a/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.ll +++ b/llvm/test/CodeGen/R600/llvm.AMDGPU.rcp.ll @@ -1,5 +1,7 @@ -; RUN: llc -march=r600 -mcpu=SI -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s -; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -enable-unsafe-fp-math -verify-machineinstrs < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI -check-prefix=FUNC %s +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI -check-prefix=FUNC %s + +; XUN: llc -march=r600 -mcpu=SI -mattr=+fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE-SPDENORM -check-prefix=SI -check-prefix=FUNC %s declare float @llvm.AMDGPU.rcp.f32(float) nounwind readnone declare double @llvm.AMDGPU.rcp.f64(double) nounwind readnone @@ -25,15 +27,8 @@ define void @rcp_f64(double addrspace(1)* %out, double %src) nounwind { } ; FUNC-LABEL: @rcp_pat_f32 -; SI-UNSAFE-NOT: V_MUL_F32 -; SI-UNSAFE: V_RCP_F32_e32 -; SI-UNSAFE-NOT: V_MUL_F32 - -; Check for surrounding multiplies the correct divide has. -; SI-SAFE: V_MUL_F32 ; SI-SAFE: V_RCP_F32_e32 -; SI-SAFE: V_MUL_F32 - +; XSI-SAFE-SPDENORM-NOT: V_RCP_F32_e32 define void @rcp_pat_f32(float addrspace(1)* %out, float %src) nounwind { %rcp = fdiv float 1.0, %src store float %rcp, float addrspace(1)* %out, align 4 @@ -60,7 +55,8 @@ define void @rsq_rcp_pat_f32(float addrspace(1)* %out, float %src) nounwind { } ; FUNC-LABEL: @rsq_rcp_pat_f64 -; SI: V_RSQ_F64_e32 +; SI-UNSAFE: V_RSQ_F64_e32 +; SI-SAFE-NOT: V_RSQ_F64_e32 define void @rsq_rcp_pat_f64(double addrspace(1)* %out, double %src) nounwind { %sqrt = call double @llvm.sqrt.f64(double %src) nounwind readnone %rcp = call double @llvm.AMDGPU.rcp.f64(double %sqrt) nounwind readnone diff --git a/llvm/test/CodeGen/R600/rsq.ll b/llvm/test/CodeGen/R600/rsq.ll index 67177e9f68b..3069f62724b 100644 --- a/llvm/test/CodeGen/R600/rsq.ll +++ b/llvm/test/CodeGen/R600/rsq.ll @@ -1,12 +1,11 @@ -; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s -; RUN: llc -march=r600 -mcpu=SI -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs -enable-unsafe-fp-math < %s | FileCheck -check-prefix=SI-UNSAFE -check-prefix=SI %s +; RUN: llc -march=r600 -mcpu=SI -mattr=-fp32-denormals -verify-machineinstrs < %s | FileCheck -check-prefix=SI-SAFE -check-prefix=SI %s declare float @llvm.sqrt.f32(float) nounwind readnone declare double @llvm.sqrt.f64(double) nounwind readnone ; SI-LABEL: @rsq_f32 -; SI-UNSAFE: V_RSQ_F32_e32 -; SI-SAFE: V_SQRT_F32 +; SI: V_RSQ_F32_e32 ; SI: S_ENDPGM define void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noalias %in) nounwind { %val = load float addrspace(1)* %in, align 4 @@ -17,7 +16,8 @@ define void @rsq_f32(float addrspace(1)* noalias %out, float addrspace(1)* noali } ; SI-LABEL: @rsq_f64 -; SI: V_RSQ_F64_e32 +; SI-UNSAFE: V_RSQ_F64_e32 +; SI-SAFE: V_SQRT_F64_e32 ; SI: S_ENDPGM define void @rsq_f64(double addrspace(1)* noalias %out, double addrspace(1)* noalias %in) nounwind { %val = load double addrspace(1)* %in, align 4 |
