3 files changed, 148 insertions, 13 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
index 6ec93c72ec5..b1e71722d80 100644
--- a/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
+++ b/llvm/test/CodeGen/AMDGPU/addrspacecast.ll
@@ -1,5 +1,5 @@
-; RUN: llc -march=amdgcn -mtriple=amdgcn-amd-amdhsa -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=HSA -check-prefix=CI %s
-; RUN: llc -march=amdgcn -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -check-prefix=HSA -check-prefix=GFX9 %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-amd-amdhsa -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=HSA -check-prefix=CI %s
+; RUN: llc -march=amdgcn -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -mattr=-promote-alloca -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=HSA -check-prefix=GFX9 %s
 
 ; HSA-LABEL: {{^}}use_group_to_flat_addrspacecast:
 ; HSA: enable_sgpr_private_segment_buffer = 1
@@ -223,9 +223,8 @@ define amdgpu_kernel void @cast_0_private_to_flat_addrspacecast() #0 {
 }
 
 ; HSA-LABEL: {{^}}cast_0_flat_to_private_addrspacecast:
-; HSA-DAG: v_mov_b32_e32 [[PTR:v[0-9]+]], 0{{$}}
-; HSA-DAG: v_mov_b32_e32 [[K:v[0-9]+]], 7{{$}}
-; HSA: buffer_store_dword [[K]], [[PTR]], s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+}} offen
+; HSA: v_mov_b32_e32 [[K:v[0-9]+]], 7{{$}}
+; HSA: buffer_store_dword [[K]], off, s{{\[[0-9]+:[0-9]+\]}}, s{{[0-9]+$}}
 define amdgpu_kernel void @cast_0_flat_to_private_addrspacecast() #0 {
   %cast = addrspacecast i32 addrspace(4)* null to i32 addrspace(0)*
   store volatile i32 7, i32* %cast
diff --git a/llvm/test/CodeGen/AMDGPU/mubuf-offset-private.ll b/llvm/test/CodeGen/AMDGPU/mubuf-offset-private.ll
new file mode 100644
index 00000000000..3a0605fa182
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/mubuf-offset-private.ll
@@ -0,0 +1,136 @@
+; RUN: llc -march=amdgcn -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=SI %s
+; RUN: llc -march=amdgcn -mcpu=fiji -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -mattr=+max-private-element-size-16 < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI %s
+
+; Test addressing modes when the scratch base is not a frame index.
+
+; GCN-LABEL: {{^}}store_private_offset_i8:
+; GCN: buffer_store_byte v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @store_private_offset_i8() #0 {
+  store volatile i8 5, i8* inttoptr (i32 8 to i8*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_offset_i16:
+; GCN: buffer_store_short v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @store_private_offset_i16() #0 {
+  store volatile i16 5, i16* inttoptr (i32 8 to i16*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_offset_i32:
+; GCN: buffer_store_dword v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @store_private_offset_i32() #0 {
+  store volatile i32 5, i32* inttoptr (i32 8 to i32*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_offset_v2i32:
+; GCN: buffer_store_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @store_private_offset_v2i32() #0 {
+  store volatile <2 x i32> <i32 5, i32 10>, <2 x i32>* inttoptr (i32 8 to <2 x i32>*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_offset_v4i32:
+; GCN: buffer_store_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @store_private_offset_v4i32() #0 {
+  store volatile <4 x i32> <i32 5, i32 10, i32 15, i32 0>, <4 x i32>* inttoptr (i32 8 to <4 x i32>*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_offset_i8:
+; GCN: buffer_load_ubyte v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @load_private_offset_i8() #0 {
+  %load = load volatile i8, i8* inttoptr (i32 8 to i8*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}sextload_private_offset_i8:
+; GCN: buffer_load_sbyte v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @sextload_private_offset_i8(i32 addrspace(1)* %out) #0 {
+  %load = load volatile i8, i8* inttoptr (i32 8 to i8*)
+  %sextload = sext i8 %load to i32
+  store i32 %sextload, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}zextload_private_offset_i8:
+; GCN: buffer_load_ubyte v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @zextload_private_offset_i8(i32 addrspace(1)* %out) #0 {
+  %load = load volatile i8, i8* inttoptr (i32 8 to i8*)
+  %zextload = zext i8 %load to i32
+  store i32 %zextload, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_offset_i16:
+; GCN: buffer_load_ushort v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @load_private_offset_i16() #0 {
+  %load = load volatile i16, i16* inttoptr (i32 8 to i16*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}sextload_private_offset_i16:
+; GCN: buffer_load_sshort v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @sextload_private_offset_i16(i32 addrspace(1)* %out) #0 {
+  %load = load volatile i16, i16* inttoptr (i32 8 to i16*)
+  %sextload = sext i16 %load to i32
+  store i32 %sextload, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}zextload_private_offset_i16:
+; GCN: buffer_load_ushort v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @zextload_private_offset_i16(i32 addrspace(1)* %out) #0 {
+  %load = load volatile i16, i16* inttoptr (i32 8 to i16*)
+  %zextload = zext i16 %load to i32
+  store i32 %zextload, i32 addrspace(1)* undef
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_offset_i32:
+; GCN: buffer_load_dword v{{[0-9]+}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @load_private_offset_i32() #0 {
+  %load = load volatile i32, i32* inttoptr (i32 8 to i32*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_offset_v2i32:
+; GCN: buffer_load_dwordx2 v{{\[[0-9]+:[0-9]+\]}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @load_private_offset_v2i32() #0 {
+  %load = load volatile <2 x i32>, <2 x i32>* inttoptr (i32 8 to <2 x i32>*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}load_private_offset_v4i32:
+; GCN: buffer_load_dwordx4 v{{\[[0-9]+:[0-9]+\]}}, off, s[4:7], s8 offset:8
+define amdgpu_kernel void @load_private_offset_v4i32() #0 {
+  %load = load volatile <4 x i32>, <4 x i32>* inttoptr (i32 8 to <4 x i32>*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_offset_i8_max_offset:
+; GCN: buffer_store_byte v{{[0-9]+}}, off, s[4:7], s8 offset:4095
+define amdgpu_kernel void @store_private_offset_i8_max_offset() #0 {
+  store volatile i8 5, i8* inttoptr (i32 4095 to i8*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_offset_i8_max_offset_plus1:
+; GCN: v_mov_b32_e32 [[OFFSET:v[0-9]+]], 0x1000
+; GCN: buffer_store_byte v{{[0-9]+}}, [[OFFSET]], s[4:7], s8 offen{{$}}
+define amdgpu_kernel void @store_private_offset_i8_max_offset_plus1() #0 {
+  store volatile i8 5, i8* inttoptr (i32 4096 to i8*)
+  ret void
+}
+
+; GCN-LABEL: {{^}}store_private_offset_i8_max_offset_plus2:
+; GCN: v_mov_b32_e32 [[OFFSET:v[0-9]+]], 0x1000
+; GCN: buffer_store_byte v{{[0-9]+}}, [[OFFSET]], s[4:7], s8 offen offset:1{{$}}
+define amdgpu_kernel void @store_private_offset_i8_max_offset_plus2() #0 {
+  store volatile i8 5, i8* inttoptr (i32 4097 to i8*)
+  ret void
+}
+
+attributes #0 = { nounwind }
diff --git a/llvm/test/CodeGen/AMDGPU/private-access-no-objects.ll b/llvm/test/CodeGen/AMDGPU/private-access-no-objects.ll
index af268351029..dcb089010e9 100644
--- a/llvm/test/CodeGen/AMDGPU/private-access-no-objects.ll
+++ b/llvm/test/CodeGen/AMDGPU/private-access-no-objects.ll
@@ -1,7 +1,7 @@
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=OPT %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=CI -check-prefix=OPT %s
-; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=iceland -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=VI -check-prefix=OPT %s
-; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefix=GCN -check-prefix=OPTNONE %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=OPT %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=hawaii -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=CI -check-prefix=OPT %s
+; RUN: llc -mtriple=amdgcn--amdhsa -mcpu=iceland -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=VI -check-prefix=OPT %s
+; RUN: llc -O0 -mtriple=amdgcn--amdhsa -mcpu=fiji -verify-machineinstrs < %s | FileCheck -enable-var-scope -check-prefix=GCN -check-prefix=OPTNONE %s
 
 ; There are no stack objects, but still a private memory access. The
 ; private access regiters need to be correctly initialized anyway, and
@@ -27,9 +27,9 @@ define amdgpu_kernel void @store_to_undef() #0 {
 ; OPT-DAG: s_mov_b64 s{{\[}}[[RSRC_LO:[0-9]+]]:{{[0-9]+\]}}, s[0:1]
 ; OPT-DAG: s_mov_b64 s{{\[[0-9]+}}:[[RSRC_HI:[0-9]+]]{{\]}}, s[2:3]
 ; OPT-DAG: s_mov_b32 [[SOFFSET:s[0-9]+]], s7{{$}}
-; OPT: buffer_store_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offen{{$}}
+; OPT: buffer_store_dword v{{[0-9]+}}, off, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offset:124{{$}}
 define amdgpu_kernel void @store_to_inttoptr() #0 {
- store volatile i32 0, i32* inttoptr (i32 123 to i32*)
+ store volatile i32 0, i32* inttoptr (i32 124 to i32*)
  ret void
 }
 
@@ -47,9 +47,9 @@ define amdgpu_kernel void @load_from_undef() #0 {
 ; OPT-DAG: s_mov_b64 s{{\[}}[[RSRC_LO:[0-9]+]]:{{[0-9]+\]}}, s[0:1]
 ; OPT-DAG: s_mov_b64 s{{\[[0-9]+}}:[[RSRC_HI:[0-9]+]]{{\]}}, s[2:3]
 ; OPT-DAG: s_mov_b32 [[SOFFSET:s[0-9]+]], s7{{$}}
-; OPT: buffer_load_dword v{{[0-9]+}}, v{{[0-9]+}}, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offen{{$}}
+; OPT: buffer_load_dword v{{[0-9]+}}, off, s{{\[}}[[RSRC_LO]]:[[RSRC_HI]]{{\]}}, [[SOFFSET]] offset:124{{$}}
 define amdgpu_kernel void @load_from_inttoptr() #0 {
-  %ld = load volatile i32, i32* inttoptr (i32 123 to i32*)
+  %ld = load volatile i32, i32* inttoptr (i32 124 to i32*)
   ret void
 }