diff options
| -rw-r--r-- | llvm/include/llvm/IR/IntrinsicsAMDGPU.td | 4 | ||||
| -rw-r--r-- | llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 4 | ||||
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll | 9 | ||||
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll | 9 |
4 files changed, 24 insertions, 2 deletions
diff --git a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td index 24e119a3665..c9721011d88 100644 --- a/llvm/include/llvm/IR/IntrinsicsAMDGPU.td +++ b/llvm/include/llvm/IR/IntrinsicsAMDGPU.td @@ -417,7 +417,9 @@ class AMDGPUDSAppendConsumedIntrinsic : Intrinsic< [llvm_i32_ty], [llvm_anyptr_ty, // LDS or GDS ptr llvm_i1_ty], // isVolatile - [IntrConvergent, IntrArgMemOnly, NoCapture<0>, ImmArg<1>] + [IntrConvergent, IntrArgMemOnly, NoCapture<0>, ImmArg<1>], + "", + [SDNPMemOperand] >; def int_amdgcn_ds_ordered_add : AMDGPUDSOrderedIntrinsic; diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 3d692bcf915..d178cb252e8 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -1993,6 +1993,7 @@ void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) { SDValue Chain = N->getOperand(0); SDValue Ptr = N->getOperand(2); MemIntrinsicSDNode *M = cast<MemIntrinsicSDNode>(N); + MachineMemOperand *MMO = M->getMemOperand(); bool IsGDS = M->getAddressSpace() == AMDGPUAS::REGION_ADDRESS; SDValue Offset; @@ -2019,7 +2020,8 @@ void AMDGPUDAGToDAGISel::SelectDSAppendConsume(SDNode *N, unsigned IntrID) { N->getOperand(N->getNumOperands() - 1) // New glue }; - CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); + SDNode *Selected = CurDAG->SelectNodeTo(N, Opc, N->getVTList(), Ops); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Selected), {MMO}); } void AMDGPUDAGToDAGISel::SelectINTRINSIC_W_CHAIN(SDNode *N) { diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll index 132e3f717c5..9f7aa5ea3a1 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.append.ll @@ -7,6 +7,7 @@ ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_append_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false) @@ -18,6 +19,7 @@ define amdgpu_kernel void @ds_append_lds(i32 addrspace(3)* %lds, i32 addrspace(1 ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_append_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383 @@ -36,6 +38,7 @@ define amdgpu_kernel void @ds_append_lds_max_offset(i32 addrspace(3)* %lds, i32 ; CIPLUS: s_mov_b32 m0, [[PTR]] ; CIPLUS: ds_append [[RESULT:v[0-9]+]] offset:16{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_append_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 { %lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4 @@ -53,6 +56,7 @@ define amdgpu_kernel void @ds_append_no_fold_offset_si(i32 addrspace(3)* addrspa ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_append_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384 @@ -65,6 +69,7 @@ define amdgpu_kernel void @ds_append_lds_over_max_offset(i32 addrspace(3)* %lds, ; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0 ; GCN: s_mov_b32 m0, [[READLANE]] ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define void @ds_append_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %val = call i32 @llvm.amdgcn.ds.append.p3i32(i32 addrspace(3)* %lds, i1 false) @@ -76,6 +81,7 @@ define void @ds_append_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* % ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_append [[RESULT:v[0-9]+]] gds{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_append_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 { %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gds, i1 false) @@ -87,6 +93,7 @@ define amdgpu_kernel void @ds_append_gds(i32 addrspace(2)* %gds, i32 addrspace(1 ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_append [[RESULT:v[0-9]+]] offset:65532 gds{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_append_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383 @@ -96,6 +103,7 @@ define amdgpu_kernel void @ds_append_gds_max_offset(i32 addrspace(2)* %gds, i32 } ; GCN-LABEL: {{^}}ds_append_gds_over_max_offset: +; GCN-NOT: buffer_wbinvl1 define amdgpu_kernel void @ds_append_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384 %val = call i32 @llvm.amdgcn.ds.append.p2i32(i32 addrspace(2)* %gep, i1 false) @@ -107,6 +115,7 @@ define amdgpu_kernel void @ds_append_gds_over_max_offset(i32 addrspace(2)* %gds, ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_append [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; NOTGFX9: s_mov_b32 m0, -1 ; GFX9-NOT: m0 ; GCN: _store_dword diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll index e6cac3a3289..415b80a4b11 100644 --- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll +++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.ds.consume.ll @@ -7,6 +7,7 @@ ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_consume_lds(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false) @@ -18,6 +19,7 @@ define amdgpu_kernel void @ds_consume_lds(i32 addrspace(3)* %lds, i32 addrspace( ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_consume_lds_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16383 @@ -36,6 +38,7 @@ define amdgpu_kernel void @ds_consume_lds_max_offset(i32 addrspace(3)* %lds, i32 ; CIPLUS: s_mov_b32 m0, [[PTR]] ; CIPLUS: ds_consume [[RESULT:v[0-9]+]] offset:16{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_consume_no_fold_offset_si(i32 addrspace(3)* addrspace(4)* %lds.ptr, i32 addrspace(1)* %out) #0 { %lds = load i32 addrspace(3)*, i32 addrspace(3)* addrspace(4)* %lds.ptr, align 4 @@ -53,6 +56,7 @@ define amdgpu_kernel void @ds_consume_no_fold_offset_si(i32 addrspace(3)* addrsp ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_consume_lds_over_max_offset(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(3)* %lds, i32 16384 @@ -65,6 +69,7 @@ define amdgpu_kernel void @ds_consume_lds_over_max_offset(i32 addrspace(3)* %lds ; GCN: v_readfirstlane_b32 [[READLANE:s[0-9]+]], v0 ; GCN: s_mov_b32 m0, [[READLANE]] ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define void @ds_consume_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* %out) #0 { %val = call i32 @llvm.amdgcn.ds.consume.p3i32(i32 addrspace(3)* %lds, i1 false) @@ -76,6 +81,7 @@ define void @ds_consume_lds_vgpr_addr(i32 addrspace(3)* %lds, i32 addrspace(1)* ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_consume [[RESULT:v[0-9]+]] gds{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_consume_gds(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 { %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gds, i1 false) @@ -87,6 +93,7 @@ define amdgpu_kernel void @ds_consume_gds(i32 addrspace(2)* %gds, i32 addrspace( ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_consume [[RESULT:v[0-9]+]] offset:65532 gds{{$}} +; GCN-NOT: buffer_wbinvl1 ; GCN: {{.*}}store{{.*}} [[RESULT]] define amdgpu_kernel void @ds_consume_gds_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16383 @@ -96,6 +103,7 @@ define amdgpu_kernel void @ds_consume_gds_max_offset(i32 addrspace(2)* %gds, i32 } ; GCN-LABEL: {{^}}ds_consume_gds_over_max_offset: +; GCN-NOT: buffer_wbinvl1 define amdgpu_kernel void @ds_consume_gds_over_max_offset(i32 addrspace(2)* %gds, i32 addrspace(1)* %out) #0 { %gep = getelementptr inbounds i32, i32 addrspace(2)* %gds, i32 16384 %val = call i32 @llvm.amdgcn.ds.consume.p2i32(i32 addrspace(2)* %gep, i1 false) @@ -107,6 +115,7 @@ define amdgpu_kernel void @ds_consume_gds_over_max_offset(i32 addrspace(2)* %gds ; GCN: s_load_dword [[PTR:s[0-9]+]] ; GCN: s_mov_b32 m0, [[PTR]] ; GCN: ds_consume [[RESULT:v[0-9]+]]{{$}} +; GCN-NOT: buffer_wbinvl1 ; NOTGFX9: s_mov_b32 m0, -1 ; GFX9-NOT: m0 ; GCN: _store_dword |
