diff options
| -rw-r--r-- | llvm/lib/Target/AMDGPU/AMDGPUInstructions.td | 20 | ||||
| -rw-r--r-- | llvm/lib/Target/AMDGPU/DSInstructions.td | 54 | ||||
| -rw-r--r-- | llvm/lib/Target/AMDGPU/EvergreenInstructions.td | 2 | ||||
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIInstrInfo.td | 37 |
4 files changed, 51 insertions, 62 deletions
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td index 636dc7364b3..b73dd41fc51 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td +++ b/llvm/lib/Target/AMDGPU/AMDGPUInstructions.td @@ -399,26 +399,14 @@ def mskor_global : PatFrag<(ops node:$val, node:$ptr), return cast<MemSDNode>(N)->getAddressSpace() == AMDGPUASI.GLOBAL_ADDRESS; }]>; -multiclass AtomicCmpSwapLocal <SDNode cmp_swap_node> { - - def _32_local : PatFrag < +class AtomicCmpSwapLocal <SDNode cmp_swap_node> : PatFrag< (ops node:$ptr, node:$cmp, node:$swap), (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{ AtomicSDNode *AN = cast<AtomicSDNode>(N); - return AN->getMemoryVT() == MVT::i32 && - AN->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS; - }]>; - - def _64_local : PatFrag< - (ops node:$ptr, node:$cmp, node:$swap), - (cmp_swap_node node:$ptr, node:$cmp, node:$swap), [{ - AtomicSDNode *AN = cast<AtomicSDNode>(N); - return AN->getMemoryVT() == MVT::i64 && - AN->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS; - }]>; -} + return AN->getAddressSpace() == AMDGPUASI.LOCAL_ADDRESS; +}]>; -defm atomic_cmp_swap : AtomicCmpSwapLocal <atomic_cmp_swap>; +def atomic_cmp_swap_local : AtomicCmpSwapLocal <atomic_cmp_swap>; multiclass global_binary_atomic_op<SDNode atomic_op> { def "" : PatFrag< diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td index 4fbef5ca82d..15260d0bae1 100644 --- a/llvm/lib/Target/AMDGPU/DSInstructions.td +++ b/llvm/lib/Target/AMDGPU/DSInstructions.td @@ -631,35 +631,35 @@ class DSAtomicCmpXChg<DS_Pseudo inst, ValueType vt, PatFrag frag> : GCNPat < // 32-bit atomics. -def : DSAtomicRetPat<DS_WRXCHG_RTN_B32, i32, si_atomic_swap_local>; -def : DSAtomicRetPat<DS_ADD_RTN_U32, i32, si_atomic_load_add_local>; -def : DSAtomicRetPat<DS_SUB_RTN_U32, i32, si_atomic_load_sub_local>; -def : DSAtomicRetPat<DS_INC_RTN_U32, i32, si_atomic_inc_local>; -def : DSAtomicRetPat<DS_DEC_RTN_U32, i32, si_atomic_dec_local>; -def : DSAtomicRetPat<DS_AND_RTN_B32, i32, si_atomic_load_and_local>; -def : DSAtomicRetPat<DS_OR_RTN_B32, i32, si_atomic_load_or_local>; -def : DSAtomicRetPat<DS_XOR_RTN_B32, i32, si_atomic_load_xor_local>; -def : DSAtomicRetPat<DS_MIN_RTN_I32, i32, si_atomic_load_min_local>; -def : DSAtomicRetPat<DS_MAX_RTN_I32, i32, si_atomic_load_max_local>; -def : DSAtomicRetPat<DS_MIN_RTN_U32, i32, si_atomic_load_umin_local>; -def : DSAtomicRetPat<DS_MAX_RTN_U32, i32, si_atomic_load_umax_local>; -def : DSAtomicCmpXChg<DS_CMPST_RTN_B32, i32, si_atomic_cmp_swap_32_local>; +def : DSAtomicRetPat<DS_WRXCHG_RTN_B32, i32, atomic_swap_local_m0>; +def : DSAtomicRetPat<DS_ADD_RTN_U32, i32, atomic_load_add_local_m0>; +def : DSAtomicRetPat<DS_SUB_RTN_U32, i32, atomic_load_sub_local_m0>; +def : DSAtomicRetPat<DS_INC_RTN_U32, i32, atomic_inc_local_m0>; +def : DSAtomicRetPat<DS_DEC_RTN_U32, i32, atomic_dec_local_m0>; +def : DSAtomicRetPat<DS_AND_RTN_B32, i32, atomic_load_and_local_m0>; +def : DSAtomicRetPat<DS_OR_RTN_B32, i32, atomic_load_or_local_m0>; +def : DSAtomicRetPat<DS_XOR_RTN_B32, i32, atomic_load_xor_local_m0>; +def : DSAtomicRetPat<DS_MIN_RTN_I32, i32, atomic_load_min_local_m0>; +def : DSAtomicRetPat<DS_MAX_RTN_I32, i32, atomic_load_max_local_m0>; +def : DSAtomicRetPat<DS_MIN_RTN_U32, i32, atomic_load_umin_local_m0>; +def : DSAtomicRetPat<DS_MAX_RTN_U32, i32, atomic_load_umax_local_m0>; +def : DSAtomicCmpXChg<DS_CMPST_RTN_B32, i32, atomic_cmp_swap_local_m0>; // 64-bit atomics. -def : DSAtomicRetPat<DS_WRXCHG_RTN_B64, i64, si_atomic_swap_local>; -def : DSAtomicRetPat<DS_ADD_RTN_U64, i64, si_atomic_load_add_local>; -def : DSAtomicRetPat<DS_SUB_RTN_U64, i64, si_atomic_load_sub_local>; -def : DSAtomicRetPat<DS_INC_RTN_U64, i64, si_atomic_inc_local>; -def : DSAtomicRetPat<DS_DEC_RTN_U64, i64, si_atomic_dec_local>; -def : DSAtomicRetPat<DS_AND_RTN_B64, i64, si_atomic_load_and_local>; -def : DSAtomicRetPat<DS_OR_RTN_B64, i64, si_atomic_load_or_local>; -def : DSAtomicRetPat<DS_XOR_RTN_B64, i64, si_atomic_load_xor_local>; -def : DSAtomicRetPat<DS_MIN_RTN_I64, i64, si_atomic_load_min_local>; -def : DSAtomicRetPat<DS_MAX_RTN_I64, i64, si_atomic_load_max_local>; -def : DSAtomicRetPat<DS_MIN_RTN_U64, i64, si_atomic_load_umin_local>; -def : DSAtomicRetPat<DS_MAX_RTN_U64, i64, si_atomic_load_umax_local>; - -def : DSAtomicCmpXChg<DS_CMPST_RTN_B64, i64, si_atomic_cmp_swap_64_local>; +def : DSAtomicRetPat<DS_WRXCHG_RTN_B64, i64, atomic_swap_local_m0>; +def : DSAtomicRetPat<DS_ADD_RTN_U64, i64, atomic_load_add_local_m0>; +def : DSAtomicRetPat<DS_SUB_RTN_U64, i64, atomic_load_sub_local_m0>; +def : DSAtomicRetPat<DS_INC_RTN_U64, i64, atomic_inc_local_m0>; +def : DSAtomicRetPat<DS_DEC_RTN_U64, i64, atomic_dec_local_m0>; +def : DSAtomicRetPat<DS_AND_RTN_B64, i64, atomic_load_and_local_m0>; +def : DSAtomicRetPat<DS_OR_RTN_B64, i64, atomic_load_or_local_m0>; +def : DSAtomicRetPat<DS_XOR_RTN_B64, i64, atomic_load_xor_local_m0>; +def : DSAtomicRetPat<DS_MIN_RTN_I64, i64, atomic_load_min_local_m0>; +def : DSAtomicRetPat<DS_MAX_RTN_I64, i64, atomic_load_max_local_m0>; +def : DSAtomicRetPat<DS_MIN_RTN_U64, i64, atomic_load_umin_local_m0>; +def : DSAtomicRetPat<DS_MAX_RTN_U64, i64, atomic_load_umax_local_m0>; + +def : DSAtomicCmpXChg<DS_CMPST_RTN_B64, i64, atomic_cmp_swap_local_m0>; //===----------------------------------------------------------------------===// // Real instructions diff --git a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td index 31f1861f555..bccad826d18 100644 --- a/llvm/lib/Target/AMDGPU/EvergreenInstructions.td +++ b/llvm/lib/Target/AMDGPU/EvergreenInstructions.td @@ -660,7 +660,7 @@ def LDS_WRXCHG_RET : R600_LDS_1A1D_RET <0x2d, "LDS_WRXCHG", [(set i32:$dst, (atomic_swap_local i32:$src0, i32:$src1))] >; def LDS_CMPST_RET : R600_LDS_1A2D_RET <0x30, "LDS_CMPST", - [(set i32:$dst, (atomic_cmp_swap_32_local i32:$src0, i32:$src1, i32:$src2))] + [(set i32:$dst, (atomic_cmp_swap_local i32:$src0, i32:$src1, i32:$src2))] >; def LDS_READ_RET : R600_LDS_1A <0x32, "LDS_READ_RET", [(set (i32 R600_Reg32:$dst), (load_local R600_Reg32:$src0))] diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.td b/llvm/lib/Target/AMDGPU/SIInstrInfo.td index 7418a2bd902..4f9d1dbd05c 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.td +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.td @@ -254,27 +254,28 @@ multiclass SIAtomicM0Glue2 <string op_name, bit is_amdgpu = 0> { [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue] >; - def _local : local_binary_atomic_op <!cast<SDNode>(NAME#"_glue")>; -} - -defm si_atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">; -defm si_atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">; -defm si_atomic_inc : SIAtomicM0Glue2 <"INC", 1>; -defm si_atomic_dec : SIAtomicM0Glue2 <"DEC", 1>; -defm si_atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">; -defm si_atomic_load_min : SIAtomicM0Glue2 <"LOAD_MIN">; -defm si_atomic_load_max : SIAtomicM0Glue2 <"LOAD_MAX">; -defm si_atomic_load_or : SIAtomicM0Glue2 <"LOAD_OR">; -defm si_atomic_load_xor : SIAtomicM0Glue2 <"LOAD_XOR">; -defm si_atomic_load_umin : SIAtomicM0Glue2 <"LOAD_UMIN">; -defm si_atomic_load_umax : SIAtomicM0Glue2 <"LOAD_UMAX">; -defm si_atomic_swap : SIAtomicM0Glue2 <"SWAP">; - -def si_atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3, + def _local_m0 : local_binary_atomic_op <!cast<SDNode>(NAME#"_glue")>; +} + +defm atomic_load_add : SIAtomicM0Glue2 <"LOAD_ADD">; +defm atomic_load_sub : SIAtomicM0Glue2 <"LOAD_SUB">; +defm atomic_inc : SIAtomicM0Glue2 <"INC", 1>; +defm atomic_dec : SIAtomicM0Glue2 <"DEC", 1>; +defm atomic_load_and : SIAtomicM0Glue2 <"LOAD_AND">; +defm atomic_load_min : SIAtomicM0Glue2 <"LOAD_MIN">; +defm atomic_load_max : SIAtomicM0Glue2 <"LOAD_MAX">; +defm atomic_load_or : SIAtomicM0Glue2 <"LOAD_OR">; +defm atomic_load_xor : SIAtomicM0Glue2 <"LOAD_XOR">; +defm atomic_load_umin : SIAtomicM0Glue2 <"LOAD_UMIN">; +defm atomic_load_umax : SIAtomicM0Glue2 <"LOAD_UMAX">; +defm atomic_swap : SIAtomicM0Glue2 <"SWAP">; + +def atomic_cmp_swap_glue : SDNode <"ISD::ATOMIC_CMP_SWAP", SDTAtomic3, [SDNPHasChain, SDNPMayStore, SDNPMayLoad, SDNPMemOperand, SDNPInGlue] >; -defm si_atomic_cmp_swap : AtomicCmpSwapLocal <si_atomic_cmp_swap_glue>; +def atomic_cmp_swap_local_m0 : AtomicCmpSwapLocal<atomic_cmp_swap_glue>; + def as_i1imm : SDNodeXForm<imm, [{ return CurDAG->getTargetConstant(N->getZExtValue(), SDLoc(N), MVT::i1); |
