AMDGPU/GlobalISel: Fix load/store of types in other address spaces

There should probably be a size only matcher.

llvm-svn: 371155

5 files changed, 116 insertions, 95 deletions

diff --git a/llvm/lib/Target/AMDGPU/BUFInstructions.td b/llvm/lib/Target/AMDGPU/BUFInstructions.td
index ef64c481b9b..50089498289 100644
--- a/llvm/lib/Target/AMDGPU/BUFInstructions.td
+++ b/llvm/lib/Target/AMDGPU/BUFInstructions.td
@@ -1534,7 +1534,10 @@ defm : MUBUFScratchLoadPat <BUFFER_LOAD_SSHORT_OFFEN, BUFFER_LOAD_SSHORT_OFFSET,
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, extloadi16_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i32, zextloadi16_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_USHORT_OFFEN, BUFFER_LOAD_USHORT_OFFSET, i16, load_private>;
+
+foreach vt = [i32, f32, v2i16, v2f16, p2, p3, p5, p6] in {
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORD_OFFEN, BUFFER_LOAD_DWORD_OFFSET, i32, load_private>;
+}
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX2_OFFEN, BUFFER_LOAD_DWORDX2_OFFSET, v2i32, load_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX3_OFFEN, BUFFER_LOAD_DWORDX3_OFFSET, v3i32, load_private>;
 defm : MUBUFScratchLoadPat <BUFFER_LOAD_DWORDX4_OFFEN, BUFFER_LOAD_DWORDX4_OFFSET, v4i32, load_private>;
@@ -1609,7 +1612,11 @@ defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET,
 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i32, truncstorei16_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_BYTE_OFFEN, BUFFER_STORE_BYTE_OFFSET, i16, truncstorei8_private>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_SHORT_OFFEN, BUFFER_STORE_SHORT_OFFSET, i16, store_private>;
-defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, i32, store_private>;
+
+foreach vt = [i32, f32, v2i16, v2f16, p2, p3, p5, p6] in {
+defm : MUBUFScratchStorePat <BUFFER_STORE_DWORD_OFFEN, BUFFER_STORE_DWORD_OFFSET, vt, store_private>;
+}
+
 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX2_OFFEN, BUFFER_STORE_DWORDX2_OFFSET, v2i32, store_private, VReg_64>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX3_OFFEN, BUFFER_STORE_DWORDX3_OFFSET, v3i32, store_private, VReg_96>;
 defm : MUBUFScratchStorePat <BUFFER_STORE_DWORDX4_OFFEN, BUFFER_STORE_DWORDX4_OFFSET, v4i32, store_private, VReg_128>;
diff --git a/llvm/lib/Target/AMDGPU/DSInstructions.td b/llvm/lib/Target/AMDGPU/DSInstructions.td
index 1c1d81a6c66..53e567a69c2 100644
--- a/llvm/lib/Target/AMDGPU/DSInstructions.td
+++ b/llvm/lib/Target/AMDGPU/DSInstructions.td
@@ -639,13 +639,20 @@ defm : DSReadPat_mc <DS_READ_I16, i32, "sextloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i32, "extloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i32, "zextloadi16_local">;
 defm : DSReadPat_mc <DS_READ_U16, i16, "load_local">;
-defm : DSReadPat_mc <DS_READ_B32, i32, "load_local">;
+
+foreach vt = [i32, f32, v2i16, v2f16, p2, p3, p5, p6] in {
+defm : DSReadPat_mc <DS_READ_B32, vt, "load_local">;
+}
+
 defm : DSReadPat_mc <DS_READ_B32, i32, "atomic_load_32_local">;
 defm : DSReadPat_mc <DS_READ_B64, i64, "atomic_load_64_local">;
 
 let AddedComplexity = 100 in {
 
-defm : DSReadPat_mc <DS_READ_B64, v2i32, "load_align8_local">;
+foreach vt = VReg_64.RegTypes in {
+defm : DSReadPat_mc <DS_READ_B64, vt, "load_align8_local">;
+}
+
 defm : DSReadPat_mc <DS_READ_B128, v4i32, "load_align16_local">;
 
 } // End AddedComplexity = 100
@@ -702,7 +709,11 @@ defm : DSWritePat_mc <DS_WRITE_B8, i32, "truncstorei8_local">;
 defm : DSWritePat_mc <DS_WRITE_B16, i32, "truncstorei16_local">;
 defm : DSWritePat_mc <DS_WRITE_B8, i16, "truncstorei8_local">;
 defm : DSWritePat_mc <DS_WRITE_B16, i16, "store_local">;
-defm : DSWritePat_mc <DS_WRITE_B32, i32, "store_local">;
+
+foreach vt = VGPR_32.RegTypes in {
+defm : DSWritePat_mc <DS_WRITE_B32, vt, "store_local">;
+}
+
 defm : DSAtomicWritePat_mc <DS_WRITE_B32, i32, "atomic_store_local">;
 defm : DSAtomicWritePat_mc <DS_WRITE_B64, i64, "atomic_store_local">;
 
@@ -739,7 +750,10 @@ def : DS64Bit4ByteAlignedWritePat<DS_WRITE2_B32_gfx9, store_local>;
 
 let AddedComplexity = 100 in {
 
-defm : DSWritePat_mc <DS_WRITE_B64, v2i32, "store_align8_local">;
+foreach vt = VReg_64.RegTypes in {
+defm : DSWritePat_mc <DS_WRITE_B64, vt, "store_align8_local">;
+}
+
 defm : DSWritePat_mc <DS_WRITE_B128, v4i32, "store_align16_local">;
 
 } // End AddedComplexity = 100
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
index 8a8b77f1e4f..d9a33f0cf0d 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-load-local.mir
@@ -260,21 +260,21 @@ body: |
 
     ; GFX6-LABEL: name: load_local_s64
     ; GFX6: liveins: $vgpr0
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; GFX6: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
     ; GFX7-LABEL: name: load_local_s64
     ; GFX7: liveins: $vgpr0
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX7: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; GFX7: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX7: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
     ; GFX9-LABEL: name: load_local_s64
     ; GFX9: liveins: $vgpr0
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:vreg_64(s64) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[LOAD]](s64)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(s64) = G_LOAD %0 :: (load 8, align 8, addrspace 3)
     $vgpr0_vgpr1 = COPY %1
@@ -463,21 +463,21 @@ body: |
 
     ; GFX6-LABEL: name: load_local_p3_from_4
     ; GFX6: liveins: $vgpr0
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX6: $vgpr0 = COPY [[LOAD]](p3)
+    ; GFX6: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX6: $vgpr0 = COPY [[DS_READ_B32_]]
     ; GFX7-LABEL: name: load_local_p3_from_4
     ; GFX7: liveins: $vgpr0
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX7: $vgpr0 = COPY [[LOAD]](p3)
+    ; GFX7: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX7: $vgpr0 = COPY [[DS_READ_B32_]]
     ; GFX9-LABEL: name: load_local_p3_from_4
     ; GFX9: liveins: $vgpr0
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX9: $vgpr0 = COPY [[LOAD]](p3)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX9: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(p3) = G_LOAD %0 :: (load 4, align 4, addrspace 3)
     $vgpr0 = COPY %1
@@ -497,21 +497,21 @@ body: |
 
     ; GFX6-LABEL: name: load_local_p5_from_4
     ; GFX6: liveins: $vgpr0
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX6: $vgpr0 = COPY [[LOAD]](p3)
+    ; GFX6: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX6: $vgpr0 = COPY [[DS_READ_B32_]]
     ; GFX7-LABEL: name: load_local_p5_from_4
     ; GFX7: liveins: $vgpr0
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX7: $vgpr0 = COPY [[LOAD]](p3)
+    ; GFX7: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX7: $vgpr0 = COPY [[DS_READ_B32_]]
     ; GFX9-LABEL: name: load_local_p5_from_4
     ; GFX9: liveins: $vgpr0
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:vgpr_32(p3) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX9: $vgpr0 = COPY [[LOAD]](p3)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX9: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(p3) = G_LOAD %0 :: (load 4, align 4, addrspace 3)
     $vgpr0 = COPY %1
@@ -531,21 +531,21 @@ body: |
 
     ; GFX6-LABEL: name: load_local_p1_align8
     ; GFX6: liveins: $vgpr0
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: [[LOAD:%[0-9]+]]:vreg_64(p1) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; GFX6: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
     ; GFX7-LABEL: name: load_local_p1_align8
     ; GFX7: liveins: $vgpr0
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: [[LOAD:%[0-9]+]]:vreg_64(p1) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX7: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; GFX7: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX7: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
     ; GFX9-LABEL: name: load_local_p1_align8
     ; GFX9: liveins: $vgpr0
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:vreg_64(p1) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[LOAD]](p1)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(p1) = G_LOAD %0 :: (load 8, align 8, addrspace 3)
     $vgpr0_vgpr1 = COPY %1
@@ -667,21 +667,21 @@ body: |
 
     ; GFX6-LABEL: name: load_local_v2s16
     ; GFX6: liveins: $vgpr0
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX6: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; GFX6: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX6: $vgpr0 = COPY [[DS_READ_B32_]]
     ; GFX7-LABEL: name: load_local_v2s16
     ; GFX7: liveins: $vgpr0
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX7: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; GFX7: [[DS_READ_B32_:%[0-9]+]]:vgpr_32 = DS_READ_B32 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX7: $vgpr0 = COPY [[DS_READ_B32_]]
     ; GFX9-LABEL: name: load_local_v2s16
     ; GFX9: liveins: $vgpr0
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:vgpr_32(<2 x s16>) = G_LOAD [[COPY]](p3) :: (load 4, addrspace 3)
-    ; GFX9: $vgpr0 = COPY [[LOAD]](<2 x s16>)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[DS_READ_B32_gfx9_:%[0-9]+]]:vgpr_32 = DS_READ_B32_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 4, addrspace 3)
+    ; GFX9: $vgpr0 = COPY [[DS_READ_B32_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(<2 x s16>) = G_LOAD %0 :: (load 4, align 4, addrspace 3)
     $vgpr0 = COPY %1
@@ -701,21 +701,21 @@ body: |
 
     ; GFX6-LABEL: name: load_local_v4s16
     ; GFX6: liveins: $vgpr0
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX6: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; GFX6: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX6: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
     ; GFX7-LABEL: name: load_local_v4s16
     ; GFX7: liveins: $vgpr0
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX7: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; GFX7: [[DS_READ_B64_:%[0-9]+]]:vreg_64 = DS_READ_B64 [[COPY]], 0, 0, implicit $m0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX7: $vgpr0_vgpr1 = COPY [[DS_READ_B64_]]
     ; GFX9-LABEL: name: load_local_v4s16
     ; GFX9: liveins: $vgpr0
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[LOAD:%[0-9]+]]:vreg_64(<4 x s16>) = G_LOAD [[COPY]](p3) :: (load 8, addrspace 3)
-    ; GFX9: $vgpr0_vgpr1 = COPY [[LOAD]](<4 x s16>)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[DS_READ_B64_gfx9_:%[0-9]+]]:vreg_64 = DS_READ_B64_gfx9 [[COPY]], 0, 0, implicit $exec :: (load 8, addrspace 3)
+    ; GFX9: $vgpr0_vgpr1 = COPY [[DS_READ_B64_gfx9_]]
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(<4 x s16>) = G_LOAD %0 :: (load 8, align 8, addrspace 3)
     $vgpr0_vgpr1 = COPY %1
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-local.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-local.mir
index 54fd3b09000..8dd906172cc 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-local.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-local.mir
@@ -136,21 +136,21 @@ body: |
 
     ; GFX6-LABEL: name: store_local_v2s16
     ; GFX6: liveins: $vgpr0, $vgpr1
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p3) :: (store 4, addrspace 3)
+    ; GFX6: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
     ; GFX7-LABEL: name: store_local_v2s16
     ; GFX7: liveins: $vgpr0, $vgpr1
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; GFX7: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p3) :: (store 4, addrspace 3)
+    ; GFX7: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
     ; GFX9-LABEL: name: store_local_v2s16
     ; GFX9: liveins: $vgpr0, $vgpr1
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
-    ; GFX9: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p3) :: (store 4, addrspace 3)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: DS_WRITE_B32_gfx9 [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (store 4, addrspace 3)
     %0:vgpr(<2 x s16>) = COPY $vgpr0
     %1:vgpr(p3) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 3)
@@ -174,21 +174,21 @@ body: |
 
     ; GFX6-LABEL: name: store_local_p3
     ; GFX6: liveins: $vgpr0, $vgpr1
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX6: $m0 = S_MOV_B32 -1
-    ; GFX6: G_STORE [[COPY]](p3), [[COPY1]](p3) :: (store 4, addrspace 3)
+    ; GFX6: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
     ; GFX7-LABEL: name: store_local_p3
     ; GFX7: liveins: $vgpr0, $vgpr1
-    ; GFX7: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX7: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
+    ; GFX7: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX7: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
     ; GFX7: $m0 = S_MOV_B32 -1
-    ; GFX7: G_STORE [[COPY]](p3), [[COPY1]](p3) :: (store 4, addrspace 3)
+    ; GFX7: DS_WRITE_B32 [[COPY1]], [[COPY]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3)
     ; GFX9-LABEL: name: store_local_p3
     ; GFX9: liveins: $vgpr0, $vgpr1
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(p3) = COPY $vgpr1
-    ; GFX9: G_STORE [[COPY]](p3), [[COPY1]](p3) :: (store 4, addrspace 3)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: DS_WRITE_B32_gfx9 [[COPY1]], [[COPY]], 0, 0, implicit $exec :: (store 4, addrspace 3)
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(p3) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 3)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir
index 822a1412d16..2d8634025c9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/inst-select-store-private.mir
@@ -112,14 +112,14 @@ body: |
 
     ; GFX6-LABEL: name: store_private_v2s16
     ; GFX6: liveins: $vgpr0, $vgpr1
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(p5) = COPY $vgpr1
-    ; GFX6: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p5) :: (store 4, addrspace 5)
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     ; GFX9-LABEL: name: store_private_v2s16
     ; GFX9: liveins: $vgpr0, $vgpr1
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(<2 x s16>) = COPY $vgpr0
-    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(p5) = COPY $vgpr1
-    ; GFX9: G_STORE [[COPY]](<2 x s16>), [[COPY1]](p5) :: (store 4, addrspace 5)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     %0:vgpr(<2 x s16>) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 5)
@@ -143,14 +143,14 @@ body: |
 
     ; GFX6-LABEL: name: store_private_p3
     ; GFX6: liveins: $vgpr0, $vgpr1
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(p5) = COPY $vgpr1
-    ; GFX6: G_STORE [[COPY]](p3), [[COPY1]](p5) :: (store 4, addrspace 5)
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     ; GFX9-LABEL: name: store_private_p3
     ; GFX9: liveins: $vgpr0, $vgpr1
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p3) = COPY $vgpr0
-    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(p5) = COPY $vgpr1
-    ; GFX9: G_STORE [[COPY]](p3), [[COPY1]](p5) :: (store 4, addrspace 5)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     %0:vgpr(p3) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 5)
@@ -174,14 +174,14 @@ body: |
 
     ; GFX6-LABEL: name: store_private_p5
     ; GFX6: liveins: $vgpr0, $vgpr1
-    ; GFX6: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
-    ; GFX6: [[COPY1:%[0-9]+]]:vgpr(p5) = COPY $vgpr1
-    ; GFX6: G_STORE [[COPY]](p5), [[COPY1]](p5) :: (store 4, addrspace 5)
+    ; GFX6: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX6: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX6: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     ; GFX9-LABEL: name: store_private_p5
     ; GFX9: liveins: $vgpr0, $vgpr1
-    ; GFX9: [[COPY:%[0-9]+]]:vgpr(p5) = COPY $vgpr0
-    ; GFX9: [[COPY1:%[0-9]+]]:vgpr(p5) = COPY $vgpr1
-    ; GFX9: G_STORE [[COPY]](p5), [[COPY1]](p5) :: (store 4, addrspace 5)
+    ; GFX9: [[COPY:%[0-9]+]]:vgpr_32 = COPY $vgpr0
+    ; GFX9: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr1
+    ; GFX9: BUFFER_STORE_DWORD_OFFEN [[COPY]], [[COPY1]], $sgpr0_sgpr1_sgpr2_sgpr3, $sgpr4, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 5)
     %0:vgpr(p5) = COPY $vgpr0
     %1:vgpr(p5) = COPY $vgpr1
     G_STORE %0, %1 :: (store 4, align 4, addrspace 5)