diff options
Diffstat (limited to 'llvm/test/CodeGen/AMDGPU/idot4u.ll')
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/idot4u.ll | 912 |
1 files changed, 458 insertions, 454 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll index fddb14dbf97..6172d54b8e0 100644 --- a/llvm/test/CodeGen/AMDGPU/idot4u.ll +++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll @@ -17,22 +17,22 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1, ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 -; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_and_b32 s7, s4, s8 -; GFX7-NEXT: s_and_b32 s8, s5, s8 -; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008 -; GFX7-NEXT: v_mov_b32_e32 v0, s8 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 -; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010 -; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1 -; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 -; GFX7-NEXT: v_mov_b32_e32 v1, s10 -; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 -; GFX7-NEXT: v_mad_u32_u24 v0, s9, v1, v0 +; GFX7-NEXT: s_and_b32 s6, s4, s8 +; GFX7-NEXT: s_and_b32 s7, s5, s8 +; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 +; GFX7-NEXT: v_mov_b32_e32 v0, s7 ; GFX7-NEXT: v_mov_b32_e32 v1, s12 +; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 +; GFX7-NEXT: v_mad_u32_u24 v0, s6, v0, v1 +; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX7-NEXT: v_mov_b32_e32 v1, s9 +; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 +; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0 +; GFX7-NEXT: v_mov_b32_e32 v1, s11 ; GFX7-NEXT: s_lshr_b32 s5, s5, 24 -; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 @@ -47,22 +47,22 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1, ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_and_b32 s6, s3, s2 +; GFX8-NEXT: s_and_b32 s5, s3, s2 ; GFX8-NEXT: s_and_b32 s2, s4, s2 -; GFX8-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX8-NEXT: v_mov_b32_e32 v0, s2 -; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: s_bfe_u32 s10, s4, 0x80010 -; GFX8-NEXT: v_mad_u32_u24 v0, s6, v0, v1 -; GFX8-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX8-NEXT: v_mov_b32_e32 v1, s8 -; GFX8-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX8-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX8-NEXT: v_mov_b32_e32 v1, s10 +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1 +; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX8-NEXT: v_mov_b32_e32 v1, s7 +; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX8-NEXT: v_mad_u32_u24 v0, s6, v1, v0 +; GFX8-NEXT: v_mov_b32_e32 v1, s9 ; GFX8-NEXT: s_lshr_b32 s4, s4, 24 -; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0 +; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX8-NEXT: s_lshr_b32 s3, s3, 24 ; GFX8-NEXT: v_mov_b32_e32 v1, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0 @@ -79,22 +79,22 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX9-NODL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX9-NODL-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX9-NODL-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_and_b32 s6, s3, s2 +; GFX9-NODL-NEXT: s_and_b32 s5, s3, s2 ; GFX9-NODL-NEXT: s_and_b32 s2, s4, s2 -; GFX9-NODL-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX9-NODL-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NODL-NEXT: s_bfe_u32 s10, s4, 0x80010 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s6, v0, v1 -; GFX9-NODL-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8 -; GFX9-NODL-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s7, v1, v0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-NODL-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s5, v0, v1 +; GFX9-NODL-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s7 +; GFX9-NODL-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s6, v1, v0 +; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s9 ; GFX9-NODL-NEXT: s_lshr_b32 s4, s4, 24 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s9, v1, v0 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX9-NODL-NEXT: s_lshr_b32 s3, s3, 24 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s4 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s3, v1, v0 @@ -108,15 +108,15 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0 -; GFX9-DL-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0 +; GFX9-DL-NEXT: s_load_dword s3, s[0:1], 0x0 +; GFX9-DL-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: v_mov_b32_e32 v2, s3 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 -; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s2, v2, v3 +; GFX9-DL-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3 +; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s4, v2, v3 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; @@ -126,14 +126,14 @@ define amdgpu_kernel void @udot4_acc32(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0 -; GFX10-DL-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX10-DL-NEXT: s_load_dword s2, s[0:1], 0x0 +; GFX10-DL-NEXT: s_load_dword s3, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 +; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s2 +; GFX10-DL-NEXT: v_dot4_u32_u8 v2, s3, s4, v0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 -; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: v_mov_b32_e32 v2, s4 -; GFX10-DL-NEXT: v_dot4_u32_u8 v2, s2, s3, v2 ; GFX10-DL-NEXT: global_store_dword v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm <4 x i8> addrspace(1)* %src2, @@ -183,28 +183,28 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1, ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s2, -1 -; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 -; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 +; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0 +; GFX7-NEXT: s_movk_i32 s5, 0xff ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_and_b32 s7, s4, s8 ; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 -; GFX7-NEXT: s_and_b32 s6, s5, s8 -; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 -; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010 -; GFX7-NEXT: v_mov_b32_e32 v2, s8 ; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 -; GFX7-NEXT: s_lshr_b32 s5, s5, 24 +; GFX7-NEXT: s_and_b32 s7, s6, s5 +; GFX7-NEXT: s_and_b32 s5, s4, s5 +; GFX7-NEXT: s_bfe_u32 s8, s6, 0x80008 +; GFX7-NEXT: v_mov_b32_e32 v1, s7 +; GFX7-NEXT: s_bfe_u32 s10, s6, 0x80010 +; GFX7-NEXT: v_mov_b32_e32 v2, s8 +; GFX7-NEXT: s_lshr_b32 s6, s6, 24 ; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 -; GFX7-NEXT: v_mov_b32_e32 v1, s5 +; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: s_endpgm @@ -214,20 +214,20 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1, ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: flat_load_ushort v2, v[0:1] ; GFX8-NEXT: s_load_dword s1, s[4:5], 0x0 -; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010 +; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_and_b32 s3, s1, s0 ; GFX8-NEXT: s_and_b32 s0, s2, s0 -; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008 ; GFX8-NEXT: v_mov_b32_e32 v3, s0 -; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010 ; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008 -; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010 ; GFX8-NEXT: s_lshr_b32 s2, s2, 24 ; GFX8-NEXT: v_mov_b32_e32 v5, s7 @@ -246,20 +246,20 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NODL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: global_load_ushort v2, v[0:1], off ; GFX9-NODL-NEXT: s_load_dword s1, s[4:5], 0x0 -; GFX9-NODL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NODL-NEXT: s_bfe_u32 s5, s2, 0x80008 +; GFX9-NODL-NEXT: s_bfe_u32 s7, s2, 0x80010 +; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-NODL-NEXT: s_and_b32 s3, s1, s0 ; GFX9-NODL-NEXT: s_and_b32 s0, s2, s0 -; GFX9-NODL-NEXT: s_bfe_u32 s5, s2, 0x80008 ; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s0 -; GFX9-NODL-NEXT: s_bfe_u32 s7, s2, 0x80010 ; GFX9-NODL-NEXT: s_bfe_u32 s4, s1, 0x80008 -; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-NODL-NEXT: s_bfe_u32 s6, s1, 0x80010 ; GFX9-NODL-NEXT: s_lshr_b32 s2, s2, 24 ; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s7 @@ -278,15 +278,15 @@ define amdgpu_kernel void @udot4_acc16(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0 +; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off +; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3 +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) -; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s2, v3, v2 +; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s0, v3, v2 ; GFX9-DL-NEXT: global_store_short v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; @@ -353,28 +353,28 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1, ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s2, -1 -; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: buffer_load_ubyte v0, off, s[0:3], 0 -; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 +; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0 +; GFX7-NEXT: s_movk_i32 s5, 0xff ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_and_b32 s7, s4, s8 ; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 -; GFX7-NEXT: s_and_b32 s6, s5, s8 -; GFX7-NEXT: s_bfe_u32 s8, s5, 0x80008 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 -; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80010 -; GFX7-NEXT: v_mov_b32_e32 v2, s8 ; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 -; GFX7-NEXT: s_lshr_b32 s5, s5, 24 +; GFX7-NEXT: s_and_b32 s7, s6, s5 +; GFX7-NEXT: s_and_b32 s5, s4, s5 +; GFX7-NEXT: s_bfe_u32 s8, s6, 0x80008 +; GFX7-NEXT: v_mov_b32_e32 v1, s7 +; GFX7-NEXT: s_bfe_u32 s10, s6, 0x80010 +; GFX7-NEXT: v_mov_b32_e32 v2, s8 +; GFX7-NEXT: s_lshr_b32 s6, s6, 24 ; GFX7-NEXT: v_mov_b32_e32 v3, s10 ; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, s9, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 -; GFX7-NEXT: v_mov_b32_e32 v1, s5 +; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: buffer_store_byte v0, off, s[0:3], 0 ; GFX7-NEXT: s_endpgm @@ -383,31 +383,31 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1, ; GFX8: ; %bb.0: ; %entry ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 -; GFX8-NEXT: s_movk_i32 s2, 0xff ; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: flat_load_ubyte v2, v[0:1] -; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: s_load_dword s1, s[6:7], 0x0 +; GFX8-NEXT: s_movk_i32 s0, 0xff ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_bfe_u32 s5, s0, 0x80008 -; GFX8-NEXT: s_and_b32 s3, s1, s2 +; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80008 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x80010 +; GFX8-NEXT: s_and_b32 s3, s1, s0 +; GFX8-NEXT: s_and_b32 s0, s2, s0 ; GFX8-NEXT: s_bfe_u32 s4, s1, 0x80008 -; GFX8-NEXT: s_and_b32 s2, s0, s2 ; GFX8-NEXT: v_mov_b32_e32 v3, s3 ; GFX8-NEXT: s_bfe_u32 s6, s1, 0x80010 ; GFX8-NEXT: v_mov_b32_e32 v4, s4 -; GFX8-NEXT: s_bfe_u32 s7, s0, 0x80010 ; GFX8-NEXT: s_lshr_b32 s1, s1, 24 ; GFX8-NEXT: v_mov_b32_e32 v5, s6 -; GFX8-NEXT: s_lshr_b32 s0, s0, 24 +; GFX8-NEXT: s_lshr_b32 s2, s2, 24 ; GFX8-NEXT: s_waitcnt vmcnt(0) -; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s5, v4, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 -; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: s_endpgm ; @@ -415,31 +415,31 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL: ; %bb.0: ; %entry ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 -; GFX9-NODL-NEXT: s_movk_i32 s2, 0xff ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: global_load_ubyte v2, v[0:1], off -; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: s_load_dword s1, s[6:7], 0x0 +; GFX9-NODL-NEXT: s_movk_i32 s0, 0xff ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_bfe_u32 s5, s0, 0x80008 -; GFX9-NODL-NEXT: s_and_b32 s3, s1, s2 +; GFX9-NODL-NEXT: s_bfe_u32 s5, s2, 0x80008 +; GFX9-NODL-NEXT: s_bfe_u32 s7, s2, 0x80010 +; GFX9-NODL-NEXT: s_and_b32 s3, s1, s0 +; GFX9-NODL-NEXT: s_and_b32 s0, s2, s0 ; GFX9-NODL-NEXT: s_bfe_u32 s4, s1, 0x80008 -; GFX9-NODL-NEXT: s_and_b32 s2, s0, s2 ; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s3 ; GFX9-NODL-NEXT: s_bfe_u32 s6, s1, 0x80010 ; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s4 -; GFX9-NODL-NEXT: s_bfe_u32 s7, s0, 0x80010 ; GFX9-NODL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s6 -; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24 +; GFX9-NODL-NEXT: s_lshr_b32 s2, s2, 24 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) -; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 +; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s5, v4, v2 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s7, v5, v2 ; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1 -; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 +; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-NODL-NEXT: global_store_byte v[0:1], v2, off ; GFX9-NODL-NEXT: s_endpgm ; @@ -448,15 +448,15 @@ define amdgpu_kernel void @udot4_acc8(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0 +; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off +; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3 +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) -; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s2, v3, v2 +; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s0, v3, v2 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; @@ -613,19 +613,19 @@ define amdgpu_kernel void @udot2_8(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_movk_i32 s2, 0xff ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s3, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: s_load_dword s0, s[6:7], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[4:5], 0x0 ; GFX10-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_and_b32 s0, s3, s2 -; GFX10-DL-NEXT: s_and_b32 s1, s4, s2 -; GFX10-DL-NEXT: s_bfe_u32 s2, s4, 0x80008 -; GFX10-DL-NEXT: s_bfe_u32 s3, s3, 0x80008 +; GFX10-DL-NEXT: s_and_b32 s3, s0, s2 +; GFX10-DL-NEXT: s_and_b32 s2, s1, s2 +; GFX10-DL-NEXT: s_bfe_u32 s0, s0, 0x80008 +; GFX10-DL-NEXT: s_bfe_u32 s1, s1, 0x80008 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s3, s2, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s3, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s1, s0, v2 ; GFX10-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm <4 x i8> addrspace(1)* %src2, @@ -752,14 +752,14 @@ define amdgpu_kernel void @udot4_CommutationInsideMAD(<4 x i8> addrspace(1)* %sr ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off +; GFX9-DL-NEXT: s_load_dword s0, s[6:7], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) -; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s3, v3, v2 +; GFX9-DL-NEXT: v_dot4_u32_u8 v2, s0, v3, v2 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; @@ -948,24 +948,24 @@ define amdgpu_kernel void @udot4_CommutationAccrossMADs(<4 x i8> addrspace(1)* % ; GFX10-DL-NEXT: s_movk_i32 s2, 0xff ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s3, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_bfe_u32 s0, s3, 0x80008 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x80008 -; GFX10-DL-NEXT: s_and_b32 s5, s3, s2 -; GFX10-DL-NEXT: s_and_b32 s2, s4, s2 -; GFX10-DL-NEXT: s_bfe_u32 s6, s3, 0x80010 -; GFX10-DL-NEXT: s_bfe_u32 s7, s4, 0x80010 +; GFX10-DL-NEXT: s_bfe_u32 s3, s0, 0x80008 +; GFX10-DL-NEXT: s_bfe_u32 s4, s1, 0x80008 +; GFX10-DL-NEXT: s_and_b32 s5, s0, s2 +; GFX10-DL-NEXT: s_and_b32 s2, s1, s2 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s1, s0, v2 -; GFX10-DL-NEXT: s_lshr_b32 s0, s3, 24 -; GFX10-DL-NEXT: s_lshr_b32 s1, s4, 24 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s4, s3, v2 +; GFX10-DL-NEXT: s_bfe_u32 s3, s0, 0x80010 +; GFX10-DL-NEXT: s_bfe_u32 s4, s1, 0x80010 +; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24 +; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s5, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s6, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s4, s3, v2 ; GFX10-DL-NEXT: v_mad_u32_u24 v2, s1, s0, v2 ; GFX10-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm @@ -1012,23 +1012,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1, ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 -; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_and_b32 s7, s4, s8 -; GFX7-NEXT: s_and_b32 s8, s5, s8 -; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008 -; GFX7-NEXT: v_mov_b32_e32 v0, s8 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 -; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 -; GFX7-NEXT: v_mad_u32_u24 v1, s7, v0, v1 -; GFX7-NEXT: v_mov_b32_e32 v2, s10 -; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010 -; GFX7-NEXT: v_mad_u32_u24 v1, s9, v2, v1 -; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 -; GFX7-NEXT: v_mad_u32_u24 v0, s7, v0, v1 +; GFX7-NEXT: s_and_b32 s6, s4, s8 +; GFX7-NEXT: s_and_b32 s7, s5, s8 +; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 +; GFX7-NEXT: v_mov_b32_e32 v0, s7 ; GFX7-NEXT: v_mov_b32_e32 v1, s12 +; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX7-NEXT: v_mad_u32_u24 v1, s6, v0, v1 +; GFX7-NEXT: v_mov_b32_e32 v2, s9 +; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 +; GFX7-NEXT: v_mad_u32_u24 v1, s8, v2, v1 +; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 +; GFX7-NEXT: v_mad_u32_u24 v0, s6, v0, v1 +; GFX7-NEXT: v_mov_b32_e32 v1, s11 ; GFX7-NEXT: s_lshr_b32 s5, s5, 24 -; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 ; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_mov_b32_e32 v1, s5 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 @@ -1043,23 +1043,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1, ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_and_b32 s6, s3, s2 +; GFX8-NEXT: s_and_b32 s5, s3, s2 ; GFX8-NEXT: s_and_b32 s2, s4, s2 -; GFX8-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX8-NEXT: v_mov_b32_e32 v0, s2 -; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX8-NEXT: v_mad_u32_u24 v1, s6, v0, v1 -; GFX8-NEXT: v_mov_b32_e32 v2, s8 -; GFX8-NEXT: s_bfe_u32 s10, s4, 0x80010 -; GFX8-NEXT: v_mad_u32_u24 v1, s7, v2, v1 -; GFX8-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX8-NEXT: v_mad_u32_u24 v0, s6, v0, v1 ; GFX8-NEXT: v_mov_b32_e32 v1, s10 +; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX8-NEXT: v_mad_u32_u24 v1, s5, v0, v1 +; GFX8-NEXT: v_mov_b32_e32 v2, s7 +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX8-NEXT: v_mad_u32_u24 v1, s6, v2, v1 +; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX8-NEXT: v_mad_u32_u24 v0, s5, v0, v1 +; GFX8-NEXT: v_mov_b32_e32 v1, s9 ; GFX8-NEXT: s_lshr_b32 s4, s4, 24 -; GFX8-NEXT: v_mad_u32_u24 v0, s9, v1, v0 +; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX8-NEXT: s_lshr_b32 s3, s3, 24 ; GFX8-NEXT: v_mov_b32_e32 v1, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s3, v1, v0 @@ -1076,23 +1076,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX9-NODL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX9-NODL-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX9-NODL-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_and_b32 s6, s3, s2 +; GFX9-NODL-NEXT: s_and_b32 s5, s3, s2 ; GFX9-NODL-NEXT: s_and_b32 s2, s4, s2 -; GFX9-NODL-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX9-NODL-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NODL-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v0, v1 -; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s8 -; GFX9-NODL-NEXT: s_bfe_u32 s10, s4, 0x80010 -; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s7, v2, v1 -; GFX9-NODL-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s6, v0, v1 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-NODL-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s5, v0, v1 +; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s7 +; GFX9-NODL-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX9-NODL-NEXT: v_mad_u32_u24 v1, s6, v2, v1 +; GFX9-NODL-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s5, v0, v1 +; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s9 ; GFX9-NODL-NEXT: s_lshr_b32 s4, s4, 24 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s9, v1, v0 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX9-NODL-NEXT: s_lshr_b32 s3, s3, 24 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s4 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s3, v1, v0 @@ -1109,23 +1109,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX9-DL-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_and_b32 s6, s3, s2 +; GFX9-DL-NEXT: s_and_b32 s5, s3, s2 ; GFX9-DL-NEXT: s_and_b32 s2, s4, s2 -; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s2 -; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-DL-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v0, v1 -; GFX9-DL-NEXT: v_mov_b32_e32 v2, s8 -; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x80010 -; GFX9-DL-NEXT: v_mad_u32_u24 v1, s7, v2, v1 -; GFX9-DL-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX9-DL-NEXT: v_mad_u32_u24 v0, s6, v0, v1 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-DL-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX9-DL-NEXT: v_mad_u32_u24 v1, s5, v0, v1 +; GFX9-DL-NEXT: v_mov_b32_e32 v2, s7 +; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX9-DL-NEXT: v_mad_u32_u24 v1, s6, v2, v1 +; GFX9-DL-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX9-DL-NEXT: v_mad_u32_u24 v0, s5, v0, v1 +; GFX9-DL-NEXT: v_mov_b32_e32 v1, s9 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 24 -; GFX9-DL-NEXT: v_mad_u32_u24 v0, s9, v1, v0 +; GFX9-DL-NEXT: v_mad_u32_u24 v0, s8, v1, v0 ; GFX9-DL-NEXT: s_lshr_b32 s3, s3, 24 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s4 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s3, v1, v0 @@ -1144,23 +1144,23 @@ define amdgpu_kernel void @udot4_multiuse_mul1(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX10-DL-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-DL-NEXT: s_and_b32 s6, s3, s2 +; GFX10-DL-NEXT: s_and_b32 s2, s4, s2 +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s5 +; GFX10-DL-NEXT: s_bfe_u32 s5, s3, 0x80008 +; GFX10-DL-NEXT: s_bfe_u32 s7, s4, 0x80008 +; GFX10-DL-NEXT: v_mad_u32_u24 v0, s6, s2, v0 +; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s7, v0 +; GFX10-DL-NEXT: s_bfe_u32 s5, s3, 0x80010 +; GFX10-DL-NEXT: s_bfe_u32 s7, s4, 0x80010 +; GFX10-DL-NEXT: v_mad_u32_u24 v0, s6, s2, v0 +; GFX10-DL-NEXT: s_lshr_b32 s2, s3, 24 +; GFX10-DL-NEXT: s_lshr_b32 s3, s4, 24 +; GFX10-DL-NEXT: v_mad_u32_u24 v0, s5, s7, v0 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s3, v0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 -; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_and_b32 s0, s3, s2 -; GFX10-DL-NEXT: s_and_b32 s1, s4, s2 -; GFX10-DL-NEXT: v_mov_b32_e32 v2, s5 -; GFX10-DL-NEXT: s_bfe_u32 s2, s3, 0x80008 -; GFX10-DL-NEXT: s_bfe_u32 s5, s4, 0x80008 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s5, v2 -; GFX10-DL-NEXT: s_bfe_u32 s2, s3, 0x80010 -; GFX10-DL-NEXT: s_bfe_u32 s5, s4, 0x80010 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_lshr_b32 s0, s3, 24 -; GFX10-DL-NEXT: s_lshr_b32 s1, s4, 24 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s5, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 ; GFX10-DL-NEXT: global_store_dword v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm <4 x i8> addrspace(1)* %src2, @@ -1215,23 +1215,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1, ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 -; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX7-NEXT: s_load_dword s12, s[0:1], 0x0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_and_b32 s7, s4, s8 -; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008 -; GFX7-NEXT: s_and_b32 s8, s5, s8 -; GFX7-NEXT: s_bfe_u32 s9, s4, 0x80008 -; GFX7-NEXT: v_mov_b32_e32 v0, s10 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 -; GFX7-NEXT: v_mad_u32_u24 v0, s9, v0, v1 -; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010 -; GFX7-NEXT: v_mov_b32_e32 v2, s8 -; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 -; GFX7-NEXT: v_add_i32_e32 v1, vcc, s6, v0 -; GFX7-NEXT: v_mad_u32_u24 v0, s7, v2, v0 -; GFX7-NEXT: v_mov_b32_e32 v2, s12 +; GFX7-NEXT: s_and_b32 s6, s4, s8 +; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 +; GFX7-NEXT: s_and_b32 s7, s5, s8 +; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX7-NEXT: v_mov_b32_e32 v0, s9 +; GFX7-NEXT: v_mov_b32_e32 v1, s12 +; GFX7-NEXT: v_mad_u32_u24 v0, s8, v0, v1 +; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 +; GFX7-NEXT: v_mov_b32_e32 v2, s7 +; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 +; GFX7-NEXT: v_add_i32_e32 v1, vcc, s12, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s6, v2, v0 +; GFX7-NEXT: v_mov_b32_e32 v2, s11 ; GFX7-NEXT: s_lshr_b32 s5, s5, 24 -; GFX7-NEXT: v_mad_u32_u24 v0, s11, v2, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s10, v2, v0 ; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: v_mov_b32_e32 v2, s5 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v2, v0 @@ -1247,23 +1247,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1, ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX8-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_and_b32 s6, s3, s2 -; GFX8-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX8-NEXT: s_and_b32 s5, s3, s2 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX8-NEXT: s_and_b32 s2, s4, s2 -; GFX8-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX8-NEXT: v_mov_b32_e32 v0, s8 -; GFX8-NEXT: v_mov_b32_e32 v1, s5 -; GFX8-NEXT: v_mad_u32_u24 v0, s7, v0, v1 -; GFX8-NEXT: s_bfe_u32 s10, s4, 0x80010 +; GFX8-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX8-NEXT: v_mov_b32_e32 v0, s7 +; GFX8-NEXT: v_mov_b32_e32 v1, s10 +; GFX8-NEXT: v_mad_u32_u24 v0, s6, v0, v1 +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010 ; GFX8-NEXT: v_mov_b32_e32 v2, s2 -; GFX8-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX8-NEXT: v_add_u32_e32 v1, vcc, s5, v0 -; GFX8-NEXT: v_mad_u32_u24 v0, s6, v2, v0 -; GFX8-NEXT: v_mov_b32_e32 v2, s10 +; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX8-NEXT: v_add_u32_e32 v1, vcc, s10, v0 +; GFX8-NEXT: v_mad_u32_u24 v0, s5, v2, v0 +; GFX8-NEXT: v_mov_b32_e32 v2, s9 ; GFX8-NEXT: s_lshr_b32 s4, s4, 24 -; GFX8-NEXT: v_mad_u32_u24 v0, s9, v2, v0 +; GFX8-NEXT: v_mad_u32_u24 v0, s8, v2, v0 ; GFX8-NEXT: s_lshr_b32 s3, s3, 24 ; GFX8-NEXT: v_mov_b32_e32 v2, s4 ; GFX8-NEXT: v_mad_u32_u24 v0, s3, v2, v0 @@ -1281,23 +1281,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX9-NODL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX9-NODL-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX9-NODL-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_and_b32 s6, s3, s2 -; GFX9-NODL-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX9-NODL-NEXT: s_and_b32 s5, s3, s2 +; GFX9-NODL-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX9-NODL-NEXT: s_and_b32 s2, s4, s2 -; GFX9-NODL-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s8 -; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s7, v0, v1 -; GFX9-NODL-NEXT: s_bfe_u32 s10, s4, 0x80010 +; GFX9-NODL-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s7 +; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s6, v0, v1 +; GFX9-NODL-NEXT: s_bfe_u32 s9, s4, 0x80010 ; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s2 -; GFX9-NODL-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX9-NODL-NEXT: v_add_u32_e32 v1, s5, v0 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s6, v2, v0 -; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s10 +; GFX9-NODL-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX9-NODL-NEXT: v_add_u32_e32 v1, s10, v0 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s5, v2, v0 +; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s9 ; GFX9-NODL-NEXT: s_lshr_b32 s4, s4, 24 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s9, v2, v0 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s8, v2, v0 ; GFX9-NODL-NEXT: s_lshr_b32 s3, s3, 24 ; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s3, v2, v0 @@ -1315,23 +1315,23 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0 +; GFX9-DL-NEXT: s_load_dword s10, s[0:1], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_and_b32 s6, s3, s2 -; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x80008 +; GFX9-DL-NEXT: s_and_b32 s5, s3, s2 +; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x80008 ; GFX9-DL-NEXT: s_and_b32 s2, s4, s2 -; GFX9-DL-NEXT: s_bfe_u32 s7, s3, 0x80008 -; GFX9-DL-NEXT: v_mov_b32_e32 v0, s8 -; GFX9-DL-NEXT: v_mov_b32_e32 v1, s5 -; GFX9-DL-NEXT: v_mad_u32_u24 v0, s7, v0, v1 -; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x80010 +; GFX9-DL-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX9-DL-NEXT: v_mov_b32_e32 v0, s7 +; GFX9-DL-NEXT: v_mov_b32_e32 v1, s10 +; GFX9-DL-NEXT: v_mad_u32_u24 v0, s6, v0, v1 +; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x80010 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s2 -; GFX9-DL-NEXT: s_bfe_u32 s9, s3, 0x80010 -; GFX9-DL-NEXT: v_add_u32_e32 v1, s5, v0 -; GFX9-DL-NEXT: v_mad_u32_u24 v0, s6, v2, v0 -; GFX9-DL-NEXT: v_mov_b32_e32 v2, s10 +; GFX9-DL-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX9-DL-NEXT: v_add_u32_e32 v1, s10, v0 +; GFX9-DL-NEXT: v_mad_u32_u24 v0, s5, v2, v0 +; GFX9-DL-NEXT: v_mov_b32_e32 v2, s9 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 24 -; GFX9-DL-NEXT: v_mad_u32_u24 v0, s9, v2, v0 +; GFX9-DL-NEXT: v_mad_u32_u24 v0, s8, v2, v0 ; GFX9-DL-NEXT: s_lshr_b32 s3, s3, 24 ; GFX9-DL-NEXT: v_mov_b32_e32 v2, s4 ; GFX9-DL-NEXT: v_mad_u32_u24 v0, s3, v2, v0 @@ -1351,24 +1351,24 @@ define amdgpu_kernel void @udot4_multiuse_add1(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 ; GFX10-DL-NEXT: s_load_dword s5, s[0:1], 0x0 -; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 -; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_bfe_u32 s0, s3, 0x80008 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x80008 -; GFX10-DL-NEXT: v_mov_b32_e32 v2, s5 -; GFX10-DL-NEXT: s_and_b32 s6, s3, s2 +; GFX10-DL-NEXT: s_bfe_u32 s6, s3, 0x80008 +; GFX10-DL-NEXT: s_bfe_u32 s7, s4, 0x80008 +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s5 +; GFX10-DL-NEXT: s_and_b32 s8, s3, s2 ; GFX10-DL-NEXT: s_and_b32 s2, s4, s2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s0, s3, 0x80010 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x80010 -; GFX10-DL-NEXT: v_mad_u32_u24 v3, s6, s2, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v0, s6, s7, v0 +; GFX10-DL-NEXT: s_bfe_u32 s6, s3, 0x80010 +; GFX10-DL-NEXT: s_bfe_u32 s7, s4, 0x80010 +; GFX10-DL-NEXT: v_mad_u32_u24 v1, s8, s2, v0 ; GFX10-DL-NEXT: s_lshr_b32 s2, s3, 24 ; GFX10-DL-NEXT: s_lshr_b32 s3, s4, 24 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, s5, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v3, s0, s1, v3 -; GFX10-DL-NEXT: v_mad_u32_u24 v3, s2, s3, v3 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, v3, v2 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v0, s5, v0 +; GFX10-DL-NEXT: v_mad_u32_u24 v1, s6, s7, v1 +; GFX10-DL-NEXT: v_mad_u32_u24 v1, s2, s3, v1 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, v1, v0 +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 +; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX10-DL-NEXT: global_store_dword v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm <4 x i8> addrspace(1)* %src2, @@ -1421,28 +1421,28 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1, ; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_mov_b32 s8, 0xffff ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 +; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 -; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 +; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_sext_i32_i8 s6, s4 -; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80008 -; GFX7-NEXT: s_sext_i32_i8 s7, s5 -; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 +; GFX7-NEXT: s_sext_i32_i8 s7, s6 +; GFX7-NEXT: s_bfe_u32 s9, s6, 0x80008 +; GFX7-NEXT: s_sext_i32_i8 s5, s4 ; GFX7-NEXT: s_and_b32 s7, s7, s8 +; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80008 ; GFX7-NEXT: v_mov_b32_e32 v1, s9 -; GFX7-NEXT: s_bfe_u32 s11, s5, 0x80010 -; GFX7-NEXT: s_and_b32 s6, s6, s8 +; GFX7-NEXT: s_bfe_u32 s11, s6, 0x80010 +; GFX7-NEXT: s_and_b32 s5, s5, s8 ; GFX7-NEXT: v_mov_b32_e32 v3, s7 ; GFX7-NEXT: s_bfe_u32 s12, s4, 0x80010 -; GFX7-NEXT: s_lshr_b32 s5, s5, 24 +; GFX7-NEXT: s_lshr_b32 s6, s6, 24 ; GFX7-NEXT: v_mov_b32_e32 v2, s11 ; GFX7-NEXT: s_lshr_b32 s4, s4, 24 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 -; GFX7-NEXT: v_mad_u32_u24 v0, s6, v3, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s5, v3, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, s12, v2, v0 -; GFX7-NEXT: v_mov_b32_e32 v1, s5 +; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: s_endpgm @@ -1452,29 +1452,29 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1, ; GFX8-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX8-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX8-NEXT: s_load_dword s3, s[6:7], 0x0 +; GFX8-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: flat_load_ushort v2, v[0:1] +; GFX8-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_bfe_u32 s0, s2, 0x80008 -; GFX8-NEXT: s_bfe_u32 s1, s3, 0x80008 +; GFX8-NEXT: s_bfe_u32 s1, s2, 0x80008 +; GFX8-NEXT: s_sext_i32_i8 s3, s2 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 -; GFX8-NEXT: s_sext_i32_i8 s4, s3 -; GFX8-NEXT: s_bfe_u32 s5, s3, 0x80010 -; GFX8-NEXT: v_mov_b32_e32 v4, s4 -; GFX8-NEXT: s_sext_i32_i8 s1, s2 -; GFX8-NEXT: s_bfe_u32 s4, s2, 0x80010 -; GFX8-NEXT: s_lshr_b32 s3, s3, 24 -; GFX8-NEXT: v_mov_b32_e32 v5, s5 +; GFX8-NEXT: s_bfe_u32 s5, s2, 0x80010 +; GFX8-NEXT: s_bfe_u32 s1, s0, 0x80008 +; GFX8-NEXT: v_mov_b32_e32 v4, s3 +; GFX8-NEXT: s_sext_i32_i8 s4, s0 +; GFX8-NEXT: s_bfe_u32 s3, s0, 0x80010 ; GFX8-NEXT: s_lshr_b32 s2, s2, 24 +; GFX8-NEXT: v_mov_b32_e32 v5, s5 +; GFX8-NEXT: s_lshr_b32 s0, s0, 24 ; GFX8-NEXT: s_waitcnt vmcnt(0) +; GFX8-NEXT: v_mad_u32_u24 v2, s1, v3, v2 +; GFX8-NEXT: v_mad_i32_i24 v2, s4, v4, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s3, v5, v2 +; GFX8-NEXT: v_mov_b32_e32 v3, s2 ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 -; GFX8-NEXT: v_mad_i32_i24 v2, s1, v4, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s4, v5, v2 -; GFX8-NEXT: v_mov_b32_e32 v3, s3 -; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: s_endpgm ; @@ -1483,29 +1483,29 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-NODL-NEXT: s_load_dword s3, s[6:7], 0x0 +; GFX9-NODL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: global_load_ushort v2, v[0:1], off +; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_bfe_u32 s0, s2, 0x80008 -; GFX9-NODL-NEXT: s_bfe_u32 s1, s3, 0x80008 +; GFX9-NODL-NEXT: s_bfe_u32 s1, s2, 0x80008 +; GFX9-NODL-NEXT: s_sext_i32_i8 s3, s2 ; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s1 -; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s3 -; GFX9-NODL-NEXT: s_bfe_u32 s5, s3, 0x80010 -; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s4 -; GFX9-NODL-NEXT: s_sext_i32_i8 s1, s2 -; GFX9-NODL-NEXT: s_bfe_u32 s4, s2, 0x80010 -; GFX9-NODL-NEXT: s_lshr_b32 s3, s3, 24 -; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-NODL-NEXT: s_bfe_u32 s5, s2, 0x80010 +; GFX9-NODL-NEXT: s_bfe_u32 s1, s0, 0x80008 +; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s3 +; GFX9-NODL-NEXT: s_sext_i32_i8 s4, s0 +; GFX9-NODL-NEXT: s_bfe_u32 s3, s0, 0x80010 ; GFX9-NODL-NEXT: s_lshr_b32 s2, s2, 24 +; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-NODL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX9-NODL-NEXT: s_waitcnt vmcnt(0) +; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s1, v3, v2 +; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s4, v4, v2 +; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s3, v5, v2 +; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 -; GFX9-NODL-NEXT: v_mad_i32_i24 v2, s1, v4, v2 -; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s4, v5, v2 -; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s3 -; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-NODL-NEXT: global_store_short v[0:1], v2, off ; GFX9-NODL-NEXT: s_endpgm ; @@ -1514,29 +1514,29 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0 +; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off +; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_bfe_u32 s0, s2, 0x80008 -; GFX9-DL-NEXT: s_bfe_u32 s1, s3, 0x80008 +; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x80008 +; GFX9-DL-NEXT: s_sext_i32_i8 s3, s2 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1 -; GFX9-DL-NEXT: s_sext_i32_i8 s4, s3 -; GFX9-DL-NEXT: s_bfe_u32 s5, s3, 0x80010 -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s4 -; GFX9-DL-NEXT: s_sext_i32_i8 s1, s2 -; GFX9-DL-NEXT: s_bfe_u32 s4, s2, 0x80010 -; GFX9-DL-NEXT: s_lshr_b32 s3, s3, 24 -; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x80010 +; GFX9-DL-NEXT: s_bfe_u32 s1, s0, 0x80008 +; GFX9-DL-NEXT: v_mov_b32_e32 v4, s3 +; GFX9-DL-NEXT: s_sext_i32_i8 s4, s0 +; GFX9-DL-NEXT: s_bfe_u32 s3, s0, 0x80010 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 24 +; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5 +; GFX9-DL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v3, v2 +; GFX9-DL-NEXT: v_mad_i32_i24 v2, s4, v4, v2 +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s3, v5, v2 +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 -; GFX9-DL-NEXT: v_mad_i32_i24 v2, s1, v4, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s4, v5, v2 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-DL-NEXT: global_store_short v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; @@ -1546,24 +1546,24 @@ define amdgpu_kernel void @notdot4_mixedtypes(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_ushort v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x80008 -; GFX10-DL-NEXT: s_bfe_u32 s1, s3, 0x80008 -; GFX10-DL-NEXT: s_sext_i32_i8 s4, s2 -; GFX10-DL-NEXT: s_sext_i32_i8 s5, s3 -; GFX10-DL-NEXT: s_bfe_u32 s6, s2, 0x80010 -; GFX10-DL-NEXT: s_bfe_u32 s7, s3, 0x80010 +; GFX10-DL-NEXT: s_bfe_u32 s2, s0, 0x80008 +; GFX10-DL-NEXT: s_bfe_u32 s3, s1, 0x80008 +; GFX10-DL-NEXT: s_sext_i32_i8 s4, s0 +; GFX10-DL-NEXT: s_sext_i32_i8 s5, s1 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_lshr_b32 s0, s2, 24 -; GFX10-DL-NEXT: s_lshr_b32 s1, s3, 24 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s3, v2 +; GFX10-DL-NEXT: s_bfe_u32 s2, s0, 0x80010 +; GFX10-DL-NEXT: s_bfe_u32 s3, s1, 0x80010 +; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24 +; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24 ; GFX10-DL-NEXT: v_mad_i32_i24 v2, s4, s5, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s6, s7, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s3, v2 ; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 ; GFX10-DL-NEXT: global_store_short v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm @@ -1613,31 +1613,32 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX7: ; %bb.0: ; %entry ; GFX7-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x9 ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd -; GFX7-NEXT: s_movk_i32 s12, 0xff +; GFX7-NEXT: s_movk_i32 s11, 0xff ; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s2, -1 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 -; GFX7-NEXT: s_load_dword s6, s[0:1], 0x0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_lshr_b32 s7, s4, 24 -; GFX7-NEXT: s_lshr_b32 s9, s5, 24 -; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008 -; GFX7-NEXT: s_bfe_u32 s13, s5, 0x80010 -; GFX7-NEXT: s_and_b32 s5, s5, s12 -; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 -; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 -; GFX7-NEXT: s_and_b32 s4, s4, s12 +; GFX7-NEXT: s_lshr_b32 s6, s4, 24 +; GFX7-NEXT: s_bfe_u32 s7, s4, 0x80008 +; GFX7-NEXT: s_bfe_u32 s10, s4, 0x80010 +; GFX7-NEXT: s_lshr_b32 s8, s5, 24 +; GFX7-NEXT: s_bfe_u32 s9, s5, 0x80008 +; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010 +; GFX7-NEXT: s_and_b32 s5, s5, s11 +; GFX7-NEXT: s_and_b32 s4, s4, s11 +; GFX7-NEXT: s_load_dword s11, s[0:1], 0x0 ; GFX7-NEXT: v_mov_b32_e32 v0, s5 -; GFX7-NEXT: v_mov_b32_e32 v1, s6 +; GFX7-NEXT: s_waitcnt lgkmcnt(0) +; GFX7-NEXT: v_mov_b32_e32 v1, s11 ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v0, v1 -; GFX7-NEXT: v_mov_b32_e32 v1, s10 -; GFX7-NEXT: v_mad_u32_u24 v0, s8, v1, v0 -; GFX7-NEXT: v_mov_b32_e32 v1, s13 -; GFX7-NEXT: v_mad_u32_u24 v0, s11, v1, v0 ; GFX7-NEXT: v_mov_b32_e32 v1, s9 ; GFX7-NEXT: v_mad_u32_u24 v0, s7, v1, v0 +; GFX7-NEXT: v_mov_b32_e32 v1, s12 +; GFX7-NEXT: v_mad_u32_u24 v0, s10, v1, v0 +; GFX7-NEXT: v_mov_b32_e32 v1, s8 +; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 ; GFX7-NEXT: buffer_store_dword v0, off, s[0:3], 0 ; GFX7-NEXT: s_endpgm ; @@ -1649,24 +1650,25 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX8-NEXT: s_waitcnt lgkmcnt(0) ; GFX8-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX8-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX8-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX8-NEXT: s_waitcnt lgkmcnt(0) -; GFX8-NEXT: s_lshr_b32 s6, s3, 24 -; GFX8-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX8-NEXT: s_lshr_b32 s5, s3, 24 +; GFX8-NEXT: s_lshr_b32 s6, s4, 24 +; GFX8-NEXT: s_bfe_u32 s7, s3, 0x80010 ; GFX8-NEXT: v_lshrrev_b16_e64 v0, 8, s3 ; GFX8-NEXT: s_and_b32 s3, s3, s2 ; GFX8-NEXT: s_and_b32 s2, s4, s2 -; GFX8-NEXT: v_mov_b32_e32 v2, s2 -; GFX8-NEXT: v_mov_b32_e32 v3, s5 -; GFX8-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX8-NEXT: s_bfe_u32 s8, s4, 0x80010 ; GFX8-NEXT: v_lshrrev_b16_e64 v1, 8, s4 +; GFX8-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX8-NEXT: v_mov_b32_e32 v2, s2 +; GFX8-NEXT: s_waitcnt lgkmcnt(0) +; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s3, v2, v3 ; GFX8-NEXT: v_mad_u32_u24 v0, v0, v1, v2 -; GFX8-NEXT: v_mov_b32_e32 v1, s9 -; GFX8-NEXT: s_lshr_b32 s7, s4, 24 -; GFX8-NEXT: v_mad_u32_u24 v0, s8, v1, v0 -; GFX8-NEXT: v_mov_b32_e32 v1, s7 -; GFX8-NEXT: v_mad_u32_u24 v2, s6, v1, v0 +; GFX8-NEXT: v_mov_b32_e32 v1, s8 +; GFX8-NEXT: v_mad_u32_u24 v0, s7, v1, v0 +; GFX8-NEXT: v_mov_b32_e32 v1, s6 +; GFX8-NEXT: v_mad_u32_u24 v2, s5, v1, v0 ; GFX8-NEXT: v_mov_b32_e32 v0, s0 ; GFX8-NEXT: v_mov_b32_e32 v1, s1 ; GFX8-NEXT: flat_store_dword v[0:1], v2 @@ -1680,24 +1682,25 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-NODL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX9-NODL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX9-NODL-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_lshr_b32 s6, s3, 24 -; GFX9-NODL-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX9-NODL-NEXT: s_lshr_b32 s5, s3, 24 +; GFX9-NODL-NEXT: s_lshr_b32 s6, s4, 24 +; GFX9-NODL-NEXT: s_bfe_u32 s7, s3, 0x80010 ; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v0, 8, s3 ; GFX9-NODL-NEXT: s_and_b32 s3, s3, s2 ; GFX9-NODL-NEXT: s_and_b32 s2, s4, s2 -; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s2 -; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s5 -; GFX9-NODL-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX9-NODL-NEXT: s_bfe_u32 s8, s4, 0x80010 ; GFX9-NODL-NEXT: v_lshrrev_b16_e64 v1, 8, s4 +; GFX9-NODL-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX9-NODL-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s3, v2, v3 ; GFX9-NODL-NEXT: v_mad_u32_u24 v0, v0, v1, v2 -; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s9 -; GFX9-NODL-NEXT: s_lshr_b32 s7, s4, 24 -; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s8, v1, v0 -; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s7 -; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s6, v1, v0 +; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s8 +; GFX9-NODL-NEXT: v_mad_u32_u24 v0, s7, v1, v0 +; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-NODL-NEXT: v_mad_u32_u24 v2, s5, v1, v0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: global_store_dword v[0:1], v2, off @@ -1711,24 +1714,25 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) ; GFX9-DL-NEXT: s_load_dword s3, s[4:5], 0x0 ; GFX9-DL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX9-DL-NEXT: s_load_dword s5, s[0:1], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_lshr_b32 s6, s3, 24 -; GFX9-DL-NEXT: s_bfe_u32 s8, s3, 0x80010 +; GFX9-DL-NEXT: s_lshr_b32 s5, s3, 24 +; GFX9-DL-NEXT: s_lshr_b32 s6, s4, 24 +; GFX9-DL-NEXT: s_bfe_u32 s7, s3, 0x80010 ; GFX9-DL-NEXT: v_lshrrev_b16_e64 v0, 8, s3 ; GFX9-DL-NEXT: s_and_b32 s3, s3, s2 ; GFX9-DL-NEXT: s_and_b32 s2, s4, s2 -; GFX9-DL-NEXT: v_mov_b32_e32 v2, s2 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s5 -; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x80010 +; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x80010 ; GFX9-DL-NEXT: v_lshrrev_b16_e64 v1, 8, s4 +; GFX9-DL-NEXT: s_load_dword s4, s[0:1], 0x0 +; GFX9-DL-NEXT: v_mov_b32_e32 v2, s2 +; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s3, v2, v3 ; GFX9-DL-NEXT: v_mad_u32_u24 v0, v0, v1, v2 -; GFX9-DL-NEXT: v_mov_b32_e32 v1, s9 -; GFX9-DL-NEXT: s_lshr_b32 s7, s4, 24 -; GFX9-DL-NEXT: v_mad_u32_u24 v0, s8, v1, v0 -; GFX9-DL-NEXT: v_mov_b32_e32 v1, s7 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v1, v0 +; GFX9-DL-NEXT: v_mov_b32_e32 v1, s8 +; GFX9-DL-NEXT: v_mad_u32_u24 v0, s7, v1, v0 +; GFX9-DL-NEXT: v_mov_b32_e32 v1, s6 +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v1, v0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_store_dword v[0:1], v2, off @@ -1745,24 +1749,24 @@ define amdgpu_kernel void @udot4_acc32_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_load_dword s4, s[4:5], 0x0 ; GFX10-DL-NEXT: s_load_dword s5, s[6:7], 0x0 ; GFX10-DL-NEXT: s_load_dword s6, s[0:1], 0x0 +; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s4 +; GFX10-DL-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-DL-NEXT: s_and_b32 s7, s4, s3 +; GFX10-DL-NEXT: s_and_b32 s3, s5, s3 +; GFX10-DL-NEXT: v_mov_b32_e32 v2, s6 +; GFX10-DL-NEXT: v_and_b32_sdwa v0, s2, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1 +; GFX10-DL-NEXT: v_and_b32_sdwa v1, s2, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1 +; GFX10-DL-NEXT: s_bfe_u32 s2, s4, 0x80010 +; GFX10-DL-NEXT: s_lshr_b32 s4, s4, 24 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s3, v2 +; GFX10-DL-NEXT: s_bfe_u32 s3, s5, 0x80010 +; GFX10-DL-NEXT: s_lshr_b32 s5, s5, 24 +; GFX10-DL-NEXT: v_mad_u32_u24 v0, v0, v1, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v0, s2, s3, v0 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s4, s5, v0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 -; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: v_mov_b32_e32 v2, s4 -; GFX10-DL-NEXT: v_mov_b32_e32 v3, s5 -; GFX10-DL-NEXT: s_and_b32 s0, s4, s3 -; GFX10-DL-NEXT: s_and_b32 s1, s5, s3 -; GFX10-DL-NEXT: v_mov_b32_e32 v4, s6 -; GFX10-DL-NEXT: v_and_b32_sdwa v2, s2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1 -; GFX10-DL-NEXT: v_and_b32_sdwa v3, s2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1 -; GFX10-DL-NEXT: s_bfe_u32 s3, s4, 0x80010 -; GFX10-DL-NEXT: s_bfe_u32 s2, s5, 0x80010 -; GFX10-DL-NEXT: v_mad_u32_u24 v4, s0, s1, v4 -; GFX10-DL-NEXT: s_lshr_b32 s0, s4, 24 -; GFX10-DL-NEXT: s_lshr_b32 s1, s5, 24 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, v2, v3, v4 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s3, s2, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 ; GFX10-DL-NEXT: global_store_dword v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm <4 x i8> addrspace(1)* %src2, @@ -1798,29 +1802,29 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX7-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0xd ; GFX7-NEXT: s_mov_b32 s3, 0xf000 ; GFX7-NEXT: s_mov_b32 s2, -1 -; GFX7-NEXT: s_movk_i32 s8, 0xff ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 +; GFX7-NEXT: s_load_dword s6, s[6:7], 0x0 ; GFX7-NEXT: buffer_load_ushort v0, off, s[0:3], 0 -; GFX7-NEXT: s_load_dword s5, s[6:7], 0x0 +; GFX7-NEXT: s_load_dword s4, s[4:5], 0x0 +; GFX7-NEXT: s_movk_i32 s7, 0xff ; GFX7-NEXT: s_waitcnt lgkmcnt(0) -; GFX7-NEXT: s_lshr_b32 s6, s4, 24 -; GFX7-NEXT: s_bfe_u32 s7, s4, 0x80008 -; GFX7-NEXT: s_bfe_u32 s10, s5, 0x80008 -; GFX7-NEXT: s_bfe_u32 s12, s5, 0x80010 -; GFX7-NEXT: s_lshr_b32 s9, s5, 24 -; GFX7-NEXT: s_and_b32 s5, s5, s8 +; GFX7-NEXT: s_bfe_u32 s10, s6, 0x80008 +; GFX7-NEXT: s_bfe_u32 s12, s6, 0x80010 +; GFX7-NEXT: s_lshr_b32 s9, s6, 24 +; GFX7-NEXT: s_and_b32 s6, s6, s7 +; GFX7-NEXT: s_lshr_b32 s5, s4, 24 +; GFX7-NEXT: s_bfe_u32 s8, s4, 0x80008 ; GFX7-NEXT: s_bfe_u32 s11, s4, 0x80010 -; GFX7-NEXT: s_and_b32 s4, s4, s8 -; GFX7-NEXT: v_mov_b32_e32 v1, s5 +; GFX7-NEXT: s_and_b32 s4, s4, s7 +; GFX7-NEXT: v_mov_b32_e32 v1, s6 ; GFX7-NEXT: v_mov_b32_e32 v2, s10 ; GFX7-NEXT: v_mov_b32_e32 v3, s12 ; GFX7-NEXT: s_waitcnt vmcnt(0) ; GFX7-NEXT: v_mad_u32_u24 v0, s4, v1, v0 -; GFX7-NEXT: v_mad_u32_u24 v0, s7, v2, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s8, v2, v0 ; GFX7-NEXT: v_mad_u32_u24 v0, s11, v3, v0 ; GFX7-NEXT: v_mov_b32_e32 v1, s9 -; GFX7-NEXT: v_mad_u32_u24 v0, s6, v1, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s5, v1, v0 ; GFX7-NEXT: buffer_store_short v0, off, s[0:3], 0 ; GFX7-NEXT: s_endpgm ; @@ -1934,27 +1938,27 @@ define amdgpu_kernel void @udot4_acc16_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: v_mov_b32_e32 v2, 0xffff ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_ushort v3, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, s2 -; GFX10-DL-NEXT: v_and_b32_sdwa v7, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 -; GFX10-DL-NEXT: v_lshrrev_b16_e64 v5, 8, s3 -; GFX10-DL-NEXT: v_and_b32_sdwa v6, v2, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 -; GFX10-DL-NEXT: s_lshr_b32 s0, s2, 16 -; GFX10-DL-NEXT: s_lshr_b32 s1, s3, 16 +; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, s0 +; GFX10-DL-NEXT: v_and_b32_sdwa v7, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX10-DL-NEXT: v_lshrrev_b16_e64 v5, 8, s1 +; GFX10-DL-NEXT: v_and_b32_sdwa v6, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX10-DL-NEXT: s_lshr_b32 s2, s0, 16 +; GFX10-DL-NEXT: s_lshr_b32 s3, s1, 16 ; GFX10-DL-NEXT: v_lshl_or_b32 v4, v4, 16, v7 -; GFX10-DL-NEXT: s_lshr_b32 s2, s2, 24 +; GFX10-DL-NEXT: s_lshr_b32 s0, s0, 24 ; GFX10-DL-NEXT: v_lshl_or_b32 v5, v5, 16, v6 -; GFX10-DL-NEXT: s_lshr_b32 s3, s3, 24 -; GFX10-DL-NEXT: v_and_b32_sdwa v6, v2, s1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 -; GFX10-DL-NEXT: v_and_b32_sdwa v2, v2, s0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX10-DL-NEXT: s_lshr_b32 s1, s1, 24 +; GFX10-DL-NEXT: v_and_b32_sdwa v6, v2, s3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 +; GFX10-DL-NEXT: v_and_b32_sdwa v2, v2, s2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, v4, v5 -; GFX10-DL-NEXT: v_lshl_or_b32 v5, s3, 16, v6 -; GFX10-DL-NEXT: v_lshl_or_b32 v2, s2, 16, v2 +; GFX10-DL-NEXT: v_lshl_or_b32 v5, s1, 16, v6 +; GFX10-DL-NEXT: v_lshl_or_b32 v2, s0, 16, v2 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v2, v2, v5 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v4, v3 @@ -2081,25 +2085,25 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX9-NODL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-NODL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-NODL-NEXT: s_load_dword s3, s[6:7], 0x0 +; GFX9-NODL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-NODL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-NODL-NEXT: global_load_ubyte v2, v[0:1], off +; GFX9-NODL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-NODL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-NODL-NEXT: s_lshr_b32 s0, s2, 16 -; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s3 -; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s3 -; GFX9-NODL-NEXT: s_lshr_b32 s1, s3, 16 -; GFX9-NODL-NEXT: s_lshr_b32 s4, s3, 24 -; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v3, s2, v3 -; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v4, s2, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-NODL-NEXT: v_mov_b32_e32 v3, s2 +; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s2 +; GFX9-NODL-NEXT: s_lshr_b32 s1, s2, 16 +; GFX9-NODL-NEXT: s_lshr_b32 s3, s2, 24 +; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v3, s0, v3 +; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v4, s0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX9-NODL-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX9-NODL-NEXT: s_lshr_b32 s2, s0, 16 ; GFX9-NODL-NEXT: v_mov_b32_e32 v5, s1 -; GFX9-NODL-NEXT: s_lshr_b32 s5, s2, 24 -; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s4 -; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v5, s0, v5 +; GFX9-NODL-NEXT: s_lshr_b32 s4, s0, 24 +; GFX9-NODL-NEXT: v_mov_b32_e32 v4, s3 +; GFX9-NODL-NEXT: v_mul_lo_u16_sdwa v4, s4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-NODL-NEXT: v_mul_lo_u16_e32 v5, s2, v5 ; GFX9-NODL-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX9-NODL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-NODL-NEXT: v_or_b32_e32 v4, v3, v4 @@ -2117,25 +2121,25 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX9-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX9-DL-NEXT: s_load_dword s3, s[6:7], 0x0 +; GFX9-DL-NEXT: s_load_dword s2, s[6:7], 0x0 ; GFX9-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off +; GFX9-DL-NEXT: s_load_dword s0, s[4:5], 0x0 ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_lshr_b32 s0, s2, 16 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s3 -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s3 -; GFX9-DL-NEXT: s_lshr_b32 s1, s3, 16 -; GFX9-DL-NEXT: s_lshr_b32 s4, s3, 24 -; GFX9-DL-NEXT: v_mul_lo_u16_e32 v3, s2, v3 -; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, s2, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s2 +; GFX9-DL-NEXT: v_mov_b32_e32 v4, s2 +; GFX9-DL-NEXT: s_lshr_b32 s1, s2, 16 +; GFX9-DL-NEXT: s_lshr_b32 s3, s2, 24 +; GFX9-DL-NEXT: v_mul_lo_u16_e32 v3, s0, v3 +; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, s0, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:BYTE_1 src1_sel:BYTE_1 ; GFX9-DL-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX9-DL-NEXT: s_lshr_b32 s2, s0, 16 ; GFX9-DL-NEXT: v_mov_b32_e32 v5, s1 -; GFX9-DL-NEXT: s_lshr_b32 s5, s2, 24 -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s4 -; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, s5, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, s0, v5 +; GFX9-DL-NEXT: s_lshr_b32 s4, s0, 24 +; GFX9-DL-NEXT: v_mov_b32_e32 v4, s3 +; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, s4, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, s2, v5 ; GFX9-DL-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX9-DL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD ; GFX9-DL-NEXT: v_or_b32_e32 v4, v3, v4 @@ -2154,24 +2158,24 @@ define amdgpu_kernel void @udot4_acc8_vecMul(<4 x i8> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s3, s[6:7], 0x0 ; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 ; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 ; GFX10-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s2 -; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, s3 -; GFX10-DL-NEXT: s_lshr_b32 s0, s2, 24 -; GFX10-DL-NEXT: s_lshr_b32 s1, s3, 24 -; GFX10-DL-NEXT: s_lshr_b32 s4, s2, 16 -; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s2, s3 +; GFX10-DL-NEXT: v_lshrrev_b16_e64 v3, 8, s0 +; GFX10-DL-NEXT: v_lshrrev_b16_e64 v4, 8, s1 +; GFX10-DL-NEXT: s_lshr_b32 s2, s0, 24 +; GFX10-DL-NEXT: s_lshr_b32 s3, s1, 24 +; GFX10-DL-NEXT: s_lshr_b32 s4, s0, 16 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v3, v3, v4 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v4, s0, s1 -; GFX10-DL-NEXT: s_lshr_b32 s0, s3, 16 +; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s2, s3 +; GFX10-DL-NEXT: s_lshr_b32 s0, s1, 16 ; GFX10-DL-NEXT: v_lshlrev_b16_e64 v3, 8, v3 -; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v4 -; GFX10-DL-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX10-DL-NEXT: v_or_b32_sdwa v3, v4, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD +; GFX10-DL-NEXT: v_lshlrev_b16_e64 v4, 8, v5 ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s4, s0 ; GFX10-DL-NEXT: v_and_b32_e32 v3, 0xffff, v3 ; GFX10-DL-NEXT: v_or_b32_sdwa v4, v5, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD |
