diff options
Diffstat (limited to 'llvm/test/CodeGen/AMDGPU/idot8u.ll')
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/idot8u.ll | 1131 |
1 files changed, 434 insertions, 697 deletions
diff --git a/llvm/test/CodeGen/AMDGPU/idot8u.ll b/llvm/test/CodeGen/AMDGPU/idot8u.ll index 43099dbfbb4..e39c3556b94 100644 --- a/llvm/test/CodeGen/AMDGPU/idot8u.ll +++ b/llvm/test/CodeGen/AMDGPU/idot8u.ll @@ -314,35 +314,34 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1, ; GFX8-NEXT: s_and_b32 s1, s4, 15 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004 +; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008 ; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40008 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX8-NEXT: v_mov_b32_e32 v5, s6 +; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40008 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40014 -; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018 -; GFX8-NEXT: s_lshr_b32 s14, s4, 28 -; GFX8-NEXT: s_bfe_u32 s4, s4, 0x4000c -; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40008 -; GFX8-NEXT: v_mov_b32_e32 v5, s5 -; GFX8-NEXT: s_bfe_u32 s7, s2, 0x4000c -; GFX8-NEXT: v_mov_b32_e32 v6, s4 -; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40010 +; GFX8-NEXT: v_mov_b32_e32 v6, s7 +; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40014 ; GFX8-NEXT: v_mov_b32_e32 v7, s8 -; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40014 -; GFX8-NEXT: v_mov_b32_e32 v8, s10 -; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018 -; GFX8-NEXT: v_mov_b32_e32 v9, s12 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX8-NEXT: v_mov_b32_e32 v8, s9 +; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40018 +; GFX8-NEXT: s_lshr_b32 s4, s4, 28 +; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28 ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s7, v6, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s9, v7, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s11, v8, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s13, v9, v2 -; GFX8-NEXT: v_mov_b32_e32 v3, s14 +; GFX8-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s9, v9, v2 +; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: s_endpgm @@ -362,35 +361,34 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1, ; GFX9-NEXT: s_and_b32 s1, s4, 15 ; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004 +; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008 ; GFX9-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40008 +; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX9-NEXT: v_mov_b32_e32 v5, s6 +; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40008 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40014 -; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40018 -; GFX9-NEXT: s_lshr_b32 s14, s4, 28 -; GFX9-NEXT: s_bfe_u32 s4, s4, 0x4000c -; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40008 -; GFX9-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-NEXT: s_bfe_u32 s7, s2, 0x4000c -; GFX9-NEXT: v_mov_b32_e32 v6, s4 -; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40010 +; GFX9-NEXT: v_mov_b32_e32 v6, s7 +; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40014 ; GFX9-NEXT: v_mov_b32_e32 v7, s8 -; GFX9-NEXT: s_bfe_u32 s11, s2, 0x40014 -; GFX9-NEXT: v_mov_b32_e32 v8, s10 -; GFX9-NEXT: s_bfe_u32 s13, s2, 0x40018 -; GFX9-NEXT: v_mov_b32_e32 v9, s12 +; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX9-NEXT: v_mov_b32_e32 v8, s9 +; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40018 +; GFX9-NEXT: s_lshr_b32 s4, s4, 28 +; GFX9-NEXT: v_mov_b32_e32 v9, s10 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s6, v5, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s7, v6, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s9, v7, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s11, v8, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s13, v9, v2 -; GFX9-NEXT: v_mov_b32_e32 v3, s14 +; GFX9-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s9, v9, v2 +; GFX9-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-NEXT: global_store_short v[0:1], v2, off ; GFX9-NEXT: s_endpgm @@ -406,81 +404,26 @@ define amdgpu_kernel void @udot8_acc16(<8 x i4> addrspace(1)* %src1, ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ushort v2, v[0:1], off ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX9-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1 -; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004 -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5 -; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x40014 -; GFX9-DL-NEXT: s_bfe_u32 s12, s4, 0x40018 -; GFX9-DL-NEXT: s_lshr_b32 s14, s4, 28 -; GFX9-DL-NEXT: s_bfe_u32 s4, s4, 0x4000c -; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40008 -; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x4000c -; GFX9-DL-NEXT: v_mov_b32_e32 v6, s4 -; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40010 -; GFX9-DL-NEXT: v_mov_b32_e32 v7, s8 -; GFX9-DL-NEXT: s_bfe_u32 s11, s2, 0x40014 -; GFX9-DL-NEXT: v_mov_b32_e32 v8, s10 -; GFX9-DL-NEXT: s_bfe_u32 s13, s2, 0x40018 -; GFX9-DL-NEXT: v_mov_b32_e32 v9, s12 -; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28 +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v5, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v6, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v7, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s11, v8, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s13, v9, v2 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s14 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 +; GFX9-DL-NEXT: v_dot8_u32_u4 v2, s2, v3, v2 ; GFX9-DL-NEXT: global_store_short v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; ; GFX10-DL-LABEL: udot8_acc16: ; GFX10-DL: ; %bb.0: ; %entry -; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 -; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 -; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s4 +; GFX10-DL-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: global_load_ushort v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX10-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s7, s2, 0x40008 -; GFX10-DL-NEXT: s_bfe_u32 s8, s4, 0x40008 -; GFX10-DL-NEXT: s_bfe_u32 s9, s2, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s10, s4, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s11, s2, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s12, s4, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s13, s2, 0x40014 -; GFX10-DL-NEXT: s_bfe_u32 s14, s4, 0x40014 -; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40018 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40018 -; GFX10-DL-NEXT: s_lshr_b32 s2, s2, 28 -; GFX10-DL-NEXT: s_lshr_b32 s4, s4, 28 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 -; GFX10-DL-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s8, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s9, s10, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s11, s12, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s13, s14, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s4, v2 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 +; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-DL-NEXT: v_dot8_u32_u4 v2, s0, s1, v2 ; GFX10-DL-NEXT: global_store_short v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm <8 x i4> addrspace(1)* %src2, @@ -616,35 +559,34 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1, ; GFX8-NEXT: s_and_b32 s1, s4, 15 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004 +; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008 ; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40008 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX8-NEXT: v_mov_b32_e32 v5, s6 +; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40008 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40014 -; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018 -; GFX8-NEXT: s_lshr_b32 s14, s4, 28 -; GFX8-NEXT: s_bfe_u32 s4, s4, 0x4000c -; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40008 -; GFX8-NEXT: v_mov_b32_e32 v5, s5 -; GFX8-NEXT: s_bfe_u32 s7, s2, 0x4000c -; GFX8-NEXT: v_mov_b32_e32 v6, s4 -; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40010 +; GFX8-NEXT: v_mov_b32_e32 v6, s7 +; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40014 ; GFX8-NEXT: v_mov_b32_e32 v7, s8 -; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40014 -; GFX8-NEXT: v_mov_b32_e32 v8, s10 -; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018 -; GFX8-NEXT: v_mov_b32_e32 v9, s12 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX8-NEXT: v_mov_b32_e32 v8, s9 +; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40018 +; GFX8-NEXT: s_lshr_b32 s4, s4, 28 +; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28 ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX8-NEXT: v_and_b32_e32 v2, 0xff, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s7, v6, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s9, v7, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s11, v8, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s13, v9, v2 -; GFX8-NEXT: v_mov_b32_e32 v3, s14 +; GFX8-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s9, v9, v2 +; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: s_endpgm @@ -664,35 +606,34 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1, ; GFX9-NEXT: s_and_b32 s1, s4, 15 ; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004 +; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008 ; GFX9-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40008 +; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX9-NEXT: v_mov_b32_e32 v5, s6 +; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40008 ; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40014 -; GFX9-NEXT: s_bfe_u32 s12, s4, 0x40018 -; GFX9-NEXT: s_lshr_b32 s14, s4, 28 -; GFX9-NEXT: s_bfe_u32 s4, s4, 0x4000c -; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40008 -; GFX9-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-NEXT: s_bfe_u32 s7, s2, 0x4000c -; GFX9-NEXT: v_mov_b32_e32 v6, s4 -; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40010 +; GFX9-NEXT: v_mov_b32_e32 v6, s7 +; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40014 ; GFX9-NEXT: v_mov_b32_e32 v7, s8 -; GFX9-NEXT: s_bfe_u32 s11, s2, 0x40014 -; GFX9-NEXT: v_mov_b32_e32 v8, s10 -; GFX9-NEXT: s_bfe_u32 s13, s2, 0x40018 -; GFX9-NEXT: v_mov_b32_e32 v9, s12 +; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX9-NEXT: v_mov_b32_e32 v8, s9 +; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40018 +; GFX9-NEXT: s_lshr_b32 s4, s4, 28 +; GFX9-NEXT: v_mov_b32_e32 v9, s10 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-NEXT: v_and_b32_e32 v2, 0xff, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s6, v5, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s7, v6, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s9, v7, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s11, v8, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s13, v9, v2 -; GFX9-NEXT: v_mov_b32_e32 v3, s14 +; GFX9-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s9, v9, v2 +; GFX9-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-NEXT: global_store_byte v[0:1], v2, off ; GFX9-NEXT: s_endpgm @@ -708,81 +649,26 @@ define amdgpu_kernel void @udot8_acc8(<8 x i4> addrspace(1)* %src1, ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX9-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1 -; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004 -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5 -; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x40014 -; GFX9-DL-NEXT: s_bfe_u32 s12, s4, 0x40018 -; GFX9-DL-NEXT: s_lshr_b32 s14, s4, 28 -; GFX9-DL-NEXT: s_bfe_u32 s4, s4, 0x4000c -; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40008 -; GFX9-DL-NEXT: v_mov_b32_e32 v5, s5 -; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x4000c -; GFX9-DL-NEXT: v_mov_b32_e32 v6, s4 -; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40010 -; GFX9-DL-NEXT: v_mov_b32_e32 v7, s8 -; GFX9-DL-NEXT: s_bfe_u32 s11, s2, 0x40014 -; GFX9-DL-NEXT: v_mov_b32_e32 v8, s10 -; GFX9-DL-NEXT: s_bfe_u32 s13, s2, 0x40018 -; GFX9-DL-NEXT: v_mov_b32_e32 v9, s12 -; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28 +; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-DL-NEXT: v_and_b32_e32 v2, 0xff, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v5, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v6, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v7, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s11, v8, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s13, v9, v2 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s14 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 +; GFX9-DL-NEXT: v_dot8_u32_u4 v2, s2, v3, v2 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; ; GFX10-DL-LABEL: udot8_acc8: ; GFX10-DL: ; %bb.0: ; %entry -; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 -; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 -; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s4 +; GFX10-DL-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX10-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s7, s2, 0x40008 -; GFX10-DL-NEXT: s_bfe_u32 s8, s4, 0x40008 -; GFX10-DL-NEXT: s_bfe_u32 s9, s2, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s10, s4, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s11, s2, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s12, s4, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s13, s2, 0x40014 -; GFX10-DL-NEXT: s_bfe_u32 s14, s4, 0x40014 -; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40018 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40018 -; GFX10-DL-NEXT: s_lshr_b32 s2, s2, 28 -; GFX10-DL-NEXT: s_lshr_b32 s4, s4, 28 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 -; GFX10-DL-NEXT: v_and_b32_e32 v2, 0xff, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s8, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s9, s10, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s11, s12, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s13, s14, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s4, v2 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 +; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-DL-NEXT: v_dot8_u32_u4 v2, s0, s1, v2 ; GFX10-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm <8 x i4> addrspace(1)* %src2, @@ -920,35 +806,32 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1, ; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004 ; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c ; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX8-NEXT: v_mov_b32_e32 v5, s6 ; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX8-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX8-NEXT: v_mov_b32_e32 v5, s7 -; GFX8-NEXT: v_mov_b32_e32 v6, s6 -; GFX8-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX8-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX8-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX8-NEXT: v_mov_b32_e32 v7, s9 -; GFX8-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX8-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX8-NEXT: v_mov_b32_e32 v8, s11 -; GFX8-NEXT: s_bfe_u32 s14, s2, 0x40018 +; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010 +; GFX8-NEXT: v_mov_b32_e32 v6, s7 +; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40014 +; GFX8-NEXT: v_mov_b32_e32 v7, s8 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX8-NEXT: v_mov_b32_e32 v8, s9 +; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40018 ; GFX8-NEXT: s_lshr_b32 s4, s4, 28 -; GFX8-NEXT: v_mov_b32_e32 v9, s13 +; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28 ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s14, v9, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s9, v9, v2 ; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 @@ -971,35 +854,32 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1, ; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c ; GFX9-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004 +; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX9-NEXT: v_mov_b32_e32 v5, s6 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX9-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX9-NEXT: v_mov_b32_e32 v5, s7 -; GFX9-NEXT: v_mov_b32_e32 v6, s6 -; GFX9-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX9-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX9-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX9-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX9-NEXT: v_mov_b32_e32 v7, s9 -; GFX9-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX9-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX9-NEXT: v_mov_b32_e32 v8, s11 -; GFX9-NEXT: s_bfe_u32 s14, s2, 0x40018 +; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010 +; GFX9-NEXT: v_mov_b32_e32 v6, s7 +; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40014 +; GFX9-NEXT: v_mov_b32_e32 v7, s8 +; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX9-NEXT: v_mov_b32_e32 v8, s9 +; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40018 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28 -; GFX9-NEXT: v_mov_b32_e32 v9, s13 +; GFX9-NEXT: v_mov_b32_e32 v9, s10 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX9-NEXT: v_add_u32_e32 v2, v2, v5 -; GFX9-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s14, v9, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s9, v9, v2 ; GFX9-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 @@ -1017,86 +897,27 @@ define amdgpu_kernel void @udot8_acc4(<8 x i4> addrspace(1)* %src1, ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX9-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1 -; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004 -; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x4000c -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5 -; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX9-DL-NEXT: v_mov_b32_e32 v5, s7 -; GFX9-DL-NEXT: v_mov_b32_e32 v6, s6 -; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX9-DL-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX9-DL-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX9-DL-NEXT: v_mov_b32_e32 v7, s9 -; GFX9-DL-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX9-DL-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX9-DL-NEXT: v_mov_b32_e32 v8, s11 -; GFX9-DL-NEXT: s_bfe_u32 s14, s2, 0x40018 -; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28 -; GFX9-DL-NEXT: v_mov_b32_e32 v9, s13 -; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28 -; GFX9-DL-NEXT: s_waitcnt vmcnt(0) -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s14, v9, v2 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 +; GFX9-DL-NEXT: s_waitcnt vmcnt(0) +; GFX9-DL-NEXT: v_dot8_u32_u4 v2, s2, v3, v2 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; ; GFX10-DL-LABEL: udot8_acc4: ; GFX10-DL: ; %bb.0: ; %entry -; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 -; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 -; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s4 +; GFX10-DL-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX10-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s7, s2, 0x40008 -; GFX10-DL-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s9, s4, 0x40008 -; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s0, s4, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40010 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 -; GFX10-DL-NEXT: v_mul_u32_u24_e64 v3, s8, s0 -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40014 -; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40014 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s9, v2 -; GFX10-DL-NEXT: v_and_b32_e32 v3, 15, v3 -; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, v2, v3 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40018 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40018 -; GFX10-DL-NEXT: s_lshr_b32 s2, s2, 28 -; GFX10-DL-NEXT: s_lshr_b32 s4, s4, 28 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s4, v2 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 +; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-DL-NEXT: v_dot8_u32_u4 v2, s0, s1, v2 ; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX10-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm @@ -1219,35 +1040,32 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr ; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004 ; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c ; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX8-NEXT: v_mov_b32_e32 v5, s6 ; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX8-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX8-NEXT: v_mov_b32_e32 v5, s7 -; GFX8-NEXT: v_mov_b32_e32 v6, s6 -; GFX8-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX8-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX8-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX8-NEXT: v_mov_b32_e32 v7, s9 -; GFX8-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX8-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX8-NEXT: v_mov_b32_e32 v8, s11 -; GFX8-NEXT: s_bfe_u32 s14, s2, 0x40018 +; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010 +; GFX8-NEXT: v_mov_b32_e32 v6, s7 +; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40014 +; GFX8-NEXT: v_mov_b32_e32 v7, s8 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX8-NEXT: v_mov_b32_e32 v8, s9 +; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40018 ; GFX8-NEXT: s_lshr_b32 s4, s4, 28 -; GFX8-NEXT: v_mov_b32_e32 v9, s13 +; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28 ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v5 -; GFX8-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s14, v9, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s9, v9, v2 ; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 @@ -1270,35 +1088,32 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr ; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c ; GFX9-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004 +; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX9-NEXT: v_mov_b32_e32 v5, s6 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX9-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX9-NEXT: v_mov_b32_e32 v5, s7 -; GFX9-NEXT: v_mov_b32_e32 v6, s6 -; GFX9-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX9-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX9-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX9-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX9-NEXT: v_mov_b32_e32 v7, s9 -; GFX9-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX9-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX9-NEXT: v_mov_b32_e32 v8, s11 -; GFX9-NEXT: s_bfe_u32 s14, s2, 0x40018 +; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010 +; GFX9-NEXT: v_mov_b32_e32 v6, s7 +; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40014 +; GFX9-NEXT: v_mov_b32_e32 v7, s8 +; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX9-NEXT: v_mov_b32_e32 v8, s9 +; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40018 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28 -; GFX9-NEXT: v_mov_b32_e32 v9, s13 +; GFX9-NEXT: v_mov_b32_e32 v9, s10 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX9-NEXT: v_add_u32_e32 v2, v5, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s14, v9, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s9, v9, v2 ; GFX9-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 @@ -1321,35 +1136,32 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004 ; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x4000c ; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004 +; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX9-DL-NEXT: v_mov_b32_e32 v5, s6 ; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX9-DL-NEXT: v_mov_b32_e32 v5, s7 -; GFX9-DL-NEXT: v_mov_b32_e32 v6, s6 -; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX9-DL-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX9-DL-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX9-DL-NEXT: v_mov_b32_e32 v7, s9 -; GFX9-DL-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX9-DL-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX9-DL-NEXT: v_mov_b32_e32 v8, s11 -; GFX9-DL-NEXT: s_bfe_u32 s14, s2, 0x40018 +; GFX9-DL-NEXT: s_bfe_u32 s8, s4, 0x40010 +; GFX9-DL-NEXT: v_mov_b32_e32 v6, s7 +; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x40014 +; GFX9-DL-NEXT: v_mov_b32_e32 v7, s8 +; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX9-DL-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX9-DL-NEXT: v_mov_b32_e32 v8, s9 +; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX9-DL-NEXT: s_bfe_u32 s9, s2, 0x40018 ; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28 -; GFX9-DL-NEXT: v_mov_b32_e32 v9, s13 +; GFX9-DL-NEXT: v_mov_b32_e32 v9, s10 ; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX9-DL-NEXT: v_add_u32_e32 v2, v5, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s14, v9, v2 +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX9-DL-NEXT: v_mad_u32_u24 v2, s9, v9, v2 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 @@ -1373,27 +1185,24 @@ define amdgpu_kernel void @udot8_CommutationInsideMAD(<8 x i4> addrspace(1)* %sr ; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40004 ; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40004 ; GFX10-DL-NEXT: s_bfe_u32 s7, s2, 0x40008 -; GFX10-DL-NEXT: s_bfe_u32 s8, s2, 0x4000c +; GFX10-DL-NEXT: s_bfe_u32 s8, s4, 0x40008 +; GFX10-DL-NEXT: s_bfe_u32 s9, s2, 0x4000c +; GFX10-DL-NEXT: s_bfe_u32 s10, s4, 0x4000c +; GFX10-DL-NEXT: s_bfe_u32 s11, s2, 0x40010 +; GFX10-DL-NEXT: s_bfe_u32 s12, s4, 0x40010 +; GFX10-DL-NEXT: s_bfe_u32 s13, s2, 0x40014 +; GFX10-DL-NEXT: s_bfe_u32 s14, s4, 0x40014 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) ; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s0, s4, 0x40008 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 -; GFX10-DL-NEXT: v_mul_u32_u24_e64 v3, s8, s1 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40014 -; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40014 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s0, v2 -; GFX10-DL-NEXT: v_and_b32_e32 v3, 15, v3 -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40010 -; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, v3, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 ; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40018 ; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40018 ; GFX10-DL-NEXT: s_lshr_b32 s2, s2, 28 ; GFX10-DL-NEXT: s_lshr_b32 s4, s4, 28 ; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s8, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s9, s10, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s11, s12, v2 +; GFX10-DL-NEXT: v_mad_u32_u24 v2, s13, s14, v2 ; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 ; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s4, v2 ; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 @@ -1987,53 +1796,43 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX7-NEXT: s_bfe_u32 s20, s1, 0x4000c ; GFX7-NEXT: v_mov_b32_e32 v2, s20 ; GFX7-NEXT: v_mov_b32_e32 v4, s18 -; GFX7-NEXT: s_bfe_u32 s14, s1, 0x40014 -; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40010 -; GFX7-NEXT: s_lshr_b32 s16, s1, 28 -; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40018 +; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40018 +; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40014 +; GFX7-NEXT: s_bfe_u32 s17, s1, 0x40010 ; GFX7-NEXT: s_and_b32 s19, s1, 15 +; GFX7-NEXT: s_lshr_b32 s14, s1, 28 ; GFX7-NEXT: s_bfe_u32 s1, s1, 0x40008 ; GFX7-NEXT: v_mul_u32_u24_e32 v2, s13, v2 ; GFX7-NEXT: v_mul_u32_u24_e32 v4, s11, v4 -; GFX7-NEXT: s_bfe_u32 s2, s0, 0x40014 -; GFX7-NEXT: s_bfe_u32 s8, s0, 0x40010 -; GFX7-NEXT: s_lshr_b32 s9, s0, 28 -; GFX7-NEXT: v_mov_b32_e32 v6, s16 -; GFX7-NEXT: s_bfe_u32 s10, s0, 0x40018 +; GFX7-NEXT: s_lshr_b32 s2, s0, 28 +; GFX7-NEXT: s_bfe_u32 s8, s0, 0x40018 +; GFX7-NEXT: s_bfe_u32 s9, s0, 0x40014 +; GFX7-NEXT: s_bfe_u32 s10, s0, 0x40010 ; GFX7-NEXT: s_and_b32 s12, s0, 15 ; GFX7-NEXT: v_mov_b32_e32 v3, s19 ; GFX7-NEXT: s_bfe_u32 s0, s0, 0x40008 ; GFX7-NEXT: v_mov_b32_e32 v1, s1 -; GFX7-NEXT: v_mov_b32_e32 v5, s17 -; GFX7-NEXT: v_mul_u32_u24_e32 v6, s9, v6 -; GFX7-NEXT: v_mul_u32_u24_e32 v1, s0, v1 +; GFX7-NEXT: v_mul_u32_u24_e32 v8, s0, v1 ; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2 ; GFX7-NEXT: v_mul_u32_u24_e32 v3, s12, v3 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX7-NEXT: v_or_b32_e32 v1, v1, v2 -; GFX7-NEXT: v_or_b32_e32 v2, v3, v4 -; GFX7-NEXT: v_mul_u32_u24_e32 v5, s10, v5 -; GFX7-NEXT: v_lshlrev_b32_e32 v6, 16, v6 -; GFX7-NEXT: v_mov_b32_e32 v8, s14 -; GFX7-NEXT: v_or_b32_e32 v3, v5, v6 -; GFX7-NEXT: v_alignbit_b32 v5, v1, v2, 16 +; GFX7-NEXT: v_or_b32_e32 v3, v3, v4 +; GFX7-NEXT: v_or_b32_e32 v2, v8, v2 +; GFX7-NEXT: v_alignbit_b32 v4, v2, v3, 16 +; GFX7-NEXT: v_lshrrev_b32_e32 v2, 16, v2 +; GFX7-NEXT: v_mov_b32_e32 v5, s17 +; GFX7-NEXT: v_mov_b32_e32 v6, s16 ; GFX7-NEXT: v_mov_b32_e32 v7, s15 -; GFX7-NEXT: v_mul_u32_u24_e32 v8, s2, v8 -; GFX7-NEXT: v_mul_u32_u24_e32 v7, s8, v7 -; GFX7-NEXT: v_lshlrev_b32_e32 v8, 16, v8 -; GFX7-NEXT: v_lshrrev_b32_e32 v6, 16, v1 -; GFX7-NEXT: v_or_b32_e32 v4, v7, v8 -; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v4 -; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v3 ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0 +; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v3 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v7, v0 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v8, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0 +; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 +; GFX7-NEXT: v_mad_u32_u24 v0, s10, v5, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s9, v6, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s8, v7, v0 +; GFX7-NEXT: v_mov_b32_e32 v1, s14 +; GFX7-NEXT: v_mad_u32_u24 v0, s2, v1, v0 ; GFX7-NEXT: buffer_store_short v0, off, s[4:7], 0 ; GFX7-NEXT: s_endpgm ; @@ -2052,35 +1851,34 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX8-NEXT: s_and_b32 s1, s4, 15 ; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004 +; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008 ; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40008 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX8-NEXT: v_mov_b32_e32 v5, s6 +; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40008 ; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40014 -; GFX8-NEXT: s_bfe_u32 s12, s4, 0x40018 -; GFX8-NEXT: s_lshr_b32 s14, s4, 28 -; GFX8-NEXT: s_bfe_u32 s4, s4, 0x4000c -; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40008 -; GFX8-NEXT: v_mov_b32_e32 v5, s5 -; GFX8-NEXT: s_bfe_u32 s7, s2, 0x4000c -; GFX8-NEXT: v_mov_b32_e32 v6, s4 -; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40010 +; GFX8-NEXT: v_mov_b32_e32 v6, s7 +; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40014 ; GFX8-NEXT: v_mov_b32_e32 v7, s8 -; GFX8-NEXT: s_bfe_u32 s11, s2, 0x40014 -; GFX8-NEXT: v_mov_b32_e32 v8, s10 -; GFX8-NEXT: s_bfe_u32 s13, s2, 0x40018 -; GFX8-NEXT: v_mov_b32_e32 v9, s12 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX8-NEXT: v_mov_b32_e32 v8, s9 +; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40018 +; GFX8-NEXT: s_lshr_b32 s4, s4, 28 +; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28 ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s6, v5, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s7, v6, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s9, v7, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s11, v8, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s13, v9, v2 -; GFX8-NEXT: v_mov_b32_e32 v3, s14 +; GFX8-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s9, v9, v2 +; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: flat_store_short v[0:1], v2 ; GFX8-NEXT: s_endpgm @@ -2131,7 +1929,7 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 -; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0 +; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v5 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 @@ -2186,7 +1984,7 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: v_add_u32_e32 v2, v3, v2 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 -; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0 +; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 @@ -2237,7 +2035,7 @@ define amdgpu_kernel void @udot8_acc16_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v3, s1, s5 ; GFX10-DL-NEXT: s_pack_ll_b32_b16 s1, s6, s4 -; GFX10-DL-NEXT: v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:BYTE_0 +; GFX10-DL-NEXT: v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_0 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_pk_mul_lo_u16 v4, s0, s1 ; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, v2, v3 @@ -2293,64 +2091,64 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX7-NEXT: s_load_dword s1, s[10:11], 0x0 ; GFX7-NEXT: s_waitcnt lgkmcnt(0) ; GFX7-NEXT: s_bfe_u32 s2, s0, 0x4000c -; GFX7-NEXT: s_lshr_b32 s11, s0, 28 +; GFX7-NEXT: s_bfe_u32 s9, s0, 0x40004 ; GFX7-NEXT: s_bfe_u32 s14, s1, 0x4000c +; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40004 ; GFX7-NEXT: s_lshr_b32 s18, s1, 28 -; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40014 -; GFX7-NEXT: v_mov_b32_e32 v4, s18 +; GFX7-NEXT: v_mov_b32_e32 v6, s16 ; GFX7-NEXT: v_mov_b32_e32 v8, s14 ; GFX7-NEXT: s_bfe_u32 s15, s1, 0x40008 -; GFX7-NEXT: s_bfe_u32 s16, s1, 0x40004 ; GFX7-NEXT: s_and_b32 s17, s1, 15 ; GFX7-NEXT: s_bfe_u32 s19, s1, 0x40018 -; GFX7-NEXT: s_bfe_u32 s1, s1, 0x40010 -; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40014 -; GFX7-NEXT: v_mov_b32_e32 v2, s20 -; GFX7-NEXT: v_mul_u32_u24_e32 v2, s13, v2 +; GFX7-NEXT: s_bfe_u32 s20, s1, 0x40014 +; GFX7-NEXT: s_lshr_b32 s11, s0, 28 +; GFX7-NEXT: v_mov_b32_e32 v4, s18 ; GFX7-NEXT: v_mul_u32_u24_e32 v4, s11, v4 +; GFX7-NEXT: v_mul_u32_u24_e32 v6, s9, v6 ; GFX7-NEXT: v_mul_u32_u24_e32 v8, s2, v8 +; GFX7-NEXT: s_bfe_u32 s1, s1, 0x40010 ; GFX7-NEXT: s_bfe_u32 s8, s0, 0x40008 ; GFX7-NEXT: v_mov_b32_e32 v7, s15 -; GFX7-NEXT: s_bfe_u32 s9, s0, 0x40004 -; GFX7-NEXT: v_mov_b32_e32 v6, s16 ; GFX7-NEXT: s_and_b32 s10, s0, 15 +; GFX7-NEXT: v_mov_b32_e32 v5, s17 ; GFX7-NEXT: s_bfe_u32 s12, s0, 0x40018 ; GFX7-NEXT: v_mov_b32_e32 v3, s19 +; GFX7-NEXT: s_bfe_u32 s13, s0, 0x40014 +; GFX7-NEXT: v_mov_b32_e32 v2, s20 +; GFX7-NEXT: v_mul_u32_u24_e32 v2, s13, v2 ; GFX7-NEXT: s_bfe_u32 s0, s0, 0x40010 ; GFX7-NEXT: v_mov_b32_e32 v1, s1 -; GFX7-NEXT: v_mov_b32_e32 v5, s17 -; GFX7-NEXT: v_mul_u32_u24_e32 v6, s9, v6 -; GFX7-NEXT: v_mul_u32_u24_e32 v1, s0, v1 -; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2 ; GFX7-NEXT: v_mul_u32_u24_e32 v3, s12, v3 -; GFX7-NEXT: v_mul_u32_u24_e32 v7, s8, v7 ; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v4 -; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v8 -; GFX7-NEXT: v_or_b32_e32 v1, v1, v2 -; GFX7-NEXT: v_or_b32_e32 v2, v3, v4 -; GFX7-NEXT: v_or_b32_e32 v4, v7, v8 ; GFX7-NEXT: v_mul_u32_u24_e32 v5, s10, v5 +; GFX7-NEXT: v_mul_u32_u24_e32 v7, s8, v7 ; GFX7-NEXT: v_lshlrev_b32_e32 v6, 8, v6 -; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2 -; GFX7-NEXT: v_or_b32_e32 v3, v5, v6 -; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4 -; GFX7-NEXT: v_or_b32_e32 v1, v1, v2 -; GFX7-NEXT: v_or_b32_e32 v2, v3, v4 -; GFX7-NEXT: v_alignbit_b32 v3, v1, v2, 8 -; GFX7-NEXT: v_alignbit_b32 v4, v1, v2, 16 -; GFX7-NEXT: v_lshrrev_b32_e32 v5, 24, v2 -; GFX7-NEXT: v_lshrrev_b32_e32 v6, 8, v1 -; GFX7-NEXT: v_lshrrev_b32_e32 v7, 16, v1 -; GFX7-NEXT: v_lshrrev_b32_e32 v8, 24, v1 +; GFX7-NEXT: v_lshlrev_b32_e32 v8, 8, v8 +; GFX7-NEXT: v_or_b32_e32 v3, v3, v4 +; GFX7-NEXT: v_or_b32_e32 v4, v5, v6 +; GFX7-NEXT: v_or_b32_e32 v5, v7, v8 +; GFX7-NEXT: v_mul_u32_u24_e32 v9, s0, v1 +; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2 +; GFX7-NEXT: v_or_b32_e32 v2, v9, v2 +; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3 +; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5 +; GFX7-NEXT: v_or_b32_e32 v2, v2, v3 +; GFX7-NEXT: v_or_b32_e32 v3, v4, v5 +; GFX7-NEXT: v_alignbit_b32 v4, v2, v3, 8 +; GFX7-NEXT: v_alignbit_b32 v5, v2, v3, 16 +; GFX7-NEXT: v_lshrrev_b32_e32 v6, 24, v3 +; GFX7-NEXT: v_lshrrev_b32_e32 v7, 8, v2 +; GFX7-NEXT: v_lshrrev_b32_e32 v8, 16, v2 +; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2 ; GFX7-NEXT: s_waitcnt vmcnt(0) -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v3, v0 +; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v3 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v4, v0 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v5, v0 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v1 ; GFX7-NEXT: v_add_i32_e32 v0, vcc, v6, v0 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v7, v0 -; GFX7-NEXT: v_add_i32_e32 v0, vcc, v8, v0 +; GFX7-NEXT: v_mad_u32_u24 v0, s0, v1, v0 +; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v7 +; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v8 +; GFX7-NEXT: v_add_i32_e32 v0, vcc, v0, v2 ; GFX7-NEXT: buffer_store_byte v0, off, s[4:7], 0 ; GFX7-NEXT: s_endpgm ; @@ -2383,41 +2181,42 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX8-NEXT: v_mul_u32_u24_e32 v3, s10, v3 ; GFX8-NEXT: v_mul_u32_u24_e32 v5, s9, v6 ; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX8-NEXT: s_bfe_u32 s0, s2, 0x40014 +; GFX8-NEXT: s_bfe_u32 s1, s4, 0x40014 +; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40010 +; GFX8-NEXT: s_lshr_b32 s7, s2, 28 +; GFX8-NEXT: s_lshr_b32 s8, s4, 28 ; GFX8-NEXT: v_or_b32_e32 v5, v5, v6 ; GFX8-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX8-NEXT: v_or_b32_sdwa v3, v5, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: s_bfe_u32 s0, s2, 0x40014 -; GFX8-NEXT: s_lshr_b32 s1, s2, 28 -; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40014 -; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40010 -; GFX8-NEXT: s_lshr_b32 s7, s4, 28 +; GFX8-NEXT: s_bfe_u32 s6, s2, 0x40010 ; GFX8-NEXT: s_bfe_u32 s4, s4, 0x40018 -; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40010 -; GFX8-NEXT: s_bfe_u32 s2, s2, 0x40018 -; GFX8-NEXT: v_mov_b32_e32 v6, s4 +; GFX8-NEXT: v_mov_b32_e32 v6, s8 ; GFX8-NEXT: v_mov_b32_e32 v7, s7 -; GFX8-NEXT: v_mov_b32_e32 v8, s1 -; GFX8-NEXT: v_mov_b32_e32 v9, s6 -; GFX8-NEXT: v_mov_b32_e32 v10, s5 -; GFX8-NEXT: v_mov_b32_e32 v11, s0 +; GFX8-NEXT: v_mov_b32_e32 v8, s5 +; GFX8-NEXT: v_mov_b32_e32 v9, s1 +; GFX8-NEXT: v_mov_b32_e32 v10, s0 +; GFX8-NEXT: v_mul_u32_u24_sdwa v6, v7, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX8-NEXT: v_mul_u32_u24_e32 v7, s6, v8 +; GFX8-NEXT: v_mul_u32_u24_sdwa v8, v10, v9 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX8-NEXT: s_bfe_u32 s2, s2, 0x40018 +; GFX8-NEXT: v_mov_b32_e32 v4, s4 ; GFX8-NEXT: v_lshrrev_b32_e32 v5, 8, v3 -; GFX8-NEXT: v_mul_u32_u24_sdwa v7, v8, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX8-NEXT: v_mul_u32_u24_e32 v6, s2, v6 -; GFX8-NEXT: v_mul_u32_u24_e32 v8, s8, v9 -; GFX8-NEXT: v_mul_u32_u24_sdwa v9, v11, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX8-NEXT: v_or_b32_e32 v8, v8, v9 -; GFX8-NEXT: v_or_b32_sdwa v6, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX8-NEXT: v_or_b32_sdwa v4, v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX8-NEXT: v_lshrrev_b32_e32 v6, 8, v4 +; GFX8-NEXT: v_mul_u32_u24_e32 v4, s2, v4 +; GFX8-NEXT: v_or_b32_e32 v7, v7, v8 +; GFX8-NEXT: v_or_b32_sdwa v4, v4, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX8-NEXT: v_and_b32_e32 v6, 0xffff, v7 +; GFX8-NEXT: v_or_b32_e32 v4, v6, v4 +; GFX8-NEXT: v_lshrrev_b32_e32 v7, 8, v4 ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v3 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v2 -; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 +; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD ; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v4 ; GFX8-NEXT: v_add_u32_e32 v2, vcc, v2, v6 -; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 -; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 +; GFX8-NEXT: v_add_u32_e32 v2, vcc, v7, v2 +; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:DWORD +; GFX8-NEXT: v_add_u32_sdwa v2, vcc, v4, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_3 src1_sel:DWORD ; GFX8-NEXT: flat_store_byte v[0:1], v2 ; GFX8-NEXT: s_endpgm ; @@ -2448,35 +2247,36 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX9-NEXT: v_mul_lo_u16_sdwa v4, s8, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-NEXT: v_mul_lo_u16_e32 v5, s9, v5 ; GFX9-NEXT: v_mul_lo_u16_sdwa v6, s10, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-NEXT: s_bfe_u32 s0, s4, 0x40010 +; GFX9-NEXT: s_bfe_u32 s1, s4, 0x40014 ; GFX9-NEXT: v_or_b32_e32 v3, v3, v4 ; GFX9-NEXT: v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-NEXT: s_bfe_u32 s1, s4, 0x40014 -; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40018 -; GFX9-NEXT: s_bfe_u32 s0, s4, 0x40010 -; GFX9-NEXT: s_lshr_b32 s4, s4, 28 +; GFX9-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX9-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40010 -; GFX9-NEXT: v_mov_b32_e32 v4, s0 -; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40014 -; GFX9-NEXT: v_mov_b32_e32 v5, s1 -; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40018 -; GFX9-NEXT: v_mov_b32_e32 v6, s5 -; GFX9-NEXT: s_lshr_b32 s2, s2, 28 -; GFX9-NEXT: v_mov_b32_e32 v7, s4 -; GFX9-NEXT: v_mul_lo_u16_e32 v4, s6, v4 -; GFX9-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-NEXT: v_mul_lo_u16_e32 v6, s8, v6 -; GFX9-NEXT: v_mul_lo_u16_sdwa v7, s2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-NEXT: v_or_b32_e32 v4, v4, v5 -; GFX9-NEXT: v_or_b32_sdwa v5, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX9-NEXT: v_lshrrev_b32_e32 v5, 8, v3 +; GFX9-NEXT: s_lshr_b32 s4, s4, 28 +; GFX9-NEXT: v_mov_b32_e32 v7, s0 +; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40010 +; GFX9-NEXT: v_mov_b32_e32 v8, s1 +; GFX9-NEXT: s_bfe_u32 s6, s2, 0x40014 +; GFX9-NEXT: s_bfe_u32 s0, s2, 0x40018 +; GFX9-NEXT: v_mov_b32_e32 v9, s7 +; GFX9-NEXT: s_lshr_b32 s1, s2, 28 +; GFX9-NEXT: v_mov_b32_e32 v10, s4 +; GFX9-NEXT: v_mul_lo_u16_e32 v7, s5, v7 +; GFX9-NEXT: v_mul_lo_u16_sdwa v8, s6, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-NEXT: v_lshrrev_b32_e32 v6, 8, v3 +; GFX9-NEXT: v_or_b32_e32 v7, v7, v8 +; GFX9-NEXT: v_mul_lo_u16_e32 v9, s0, v9 +; GFX9-NEXT: v_mul_lo_u16_sdwa v10, s1, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-NEXT: v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-NEXT: v_and_b32_e32 v5, 0xffff, v7 +; GFX9-NEXT: v_or_b32_e32 v4, v5, v8 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_add_u32_e32 v2, v3, v2 -; GFX9-NEXT: v_add_u32_e32 v2, v2, v5 -; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 +; GFX9-NEXT: v_add_u32_e32 v2, v2, v6 +; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 -; GFX9-NEXT: v_add_u32_e32 v2, v2, v4 +; GFX9-NEXT: v_add_u32_e32 v2, v2, v5 ; GFX9-NEXT: v_lshrrev_b32_e32 v3, 8, v4 ; GFX9-NEXT: v_add_u32_e32 v2, v2, v3 ; GFX9-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 @@ -2511,35 +2311,36 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v4, s8, v4 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX9-DL-NEXT: v_mul_lo_u16_e32 v5, s9, v5 ; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v6, s10, v6 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-DL-NEXT: s_bfe_u32 s0, s4, 0x40010 +; GFX9-DL-NEXT: s_bfe_u32 s1, s4, 0x40014 ; GFX9-DL-NEXT: v_or_b32_e32 v3, v3, v4 ; GFX9-DL-NEXT: v_or_b32_sdwa v4, v5, v6 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-DL-NEXT: s_bfe_u32 s1, s4, 0x40014 -; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40018 -; GFX9-DL-NEXT: s_bfe_u32 s0, s4, 0x40010 -; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28 +; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x40018 ; GFX9-DL-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40010 -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s0 -; GFX9-DL-NEXT: s_bfe_u32 s7, s2, 0x40014 -; GFX9-DL-NEXT: v_mov_b32_e32 v5, s1 -; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x40018 -; GFX9-DL-NEXT: v_mov_b32_e32 v6, s5 -; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28 -; GFX9-DL-NEXT: v_mov_b32_e32 v7, s4 -; GFX9-DL-NEXT: v_mul_lo_u16_e32 v4, s6, v4 -; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v5, s7, v5 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-DL-NEXT: v_mul_lo_u16_e32 v6, s8, v6 -; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v7, s2, v7 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-DL-NEXT: v_or_b32_e32 v4, v4, v5 -; GFX9-DL-NEXT: v_or_b32_sdwa v5, v6, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX9-DL-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX9-DL-NEXT: v_lshrrev_b32_e32 v5, 8, v3 +; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28 +; GFX9-DL-NEXT: v_mov_b32_e32 v7, s0 +; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40010 +; GFX9-DL-NEXT: v_mov_b32_e32 v8, s1 +; GFX9-DL-NEXT: s_bfe_u32 s6, s2, 0x40014 +; GFX9-DL-NEXT: s_bfe_u32 s0, s2, 0x40018 +; GFX9-DL-NEXT: v_mov_b32_e32 v9, s7 +; GFX9-DL-NEXT: s_lshr_b32 s1, s2, 28 +; GFX9-DL-NEXT: v_mov_b32_e32 v10, s4 +; GFX9-DL-NEXT: v_mul_lo_u16_e32 v7, s5, v7 +; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v8, s6, v8 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v3 +; GFX9-DL-NEXT: v_or_b32_e32 v7, v7, v8 +; GFX9-DL-NEXT: v_mul_lo_u16_e32 v9, s0, v9 +; GFX9-DL-NEXT: v_mul_lo_u16_sdwa v10, s1, v10 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-DL-NEXT: v_or_b32_sdwa v8, v9, v10 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX9-DL-NEXT: v_and_b32_e32 v5, 0xffff, v7 +; GFX9-DL-NEXT: v_or_b32_e32 v4, v5, v8 ; GFX9-DL-NEXT: s_waitcnt vmcnt(0) ; GFX9-DL-NEXT: v_add_u32_e32 v2, v3, v2 -; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5 -; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 +; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v6 +; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 -; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v4 +; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5 ; GFX9-DL-NEXT: v_lshrrev_b32_e32 v3, 8, v4 ; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v3 ; GFX9-DL-NEXT: v_add_u32_sdwa v2, v2, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 @@ -2575,34 +2376,35 @@ define amdgpu_kernel void @udot8_acc8_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40014 ; GFX10-DL-NEXT: v_and_b32_sdwa v5, v5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD ; GFX10-DL-NEXT: v_mul_lo_u16_e64 v7, s8, s10 -; GFX10-DL-NEXT: s_lshr_b32 s1, s2, 28 +; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40014 ; GFX10-DL-NEXT: v_or_b32_sdwa v4, v6, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 -; GFX10-DL-NEXT: s_lshr_b32 s6, s4, 28 -; GFX10-DL-NEXT: s_bfe_u32 s5, s4, 0x40014 +; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40010 +; GFX10-DL-NEXT: s_lshr_b32 s6, s2, 28 ; GFX10-DL-NEXT: v_or_b32_sdwa v5, v7, v5 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 -; GFX10-DL-NEXT: s_bfe_u32 s7, s2, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s8, s4, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s2, s2, 0x40018 -; GFX10-DL-NEXT: s_bfe_u32 s4, s4, 0x40018 +; GFX10-DL-NEXT: v_mul_lo_u16_e64 v7, s0, s1 +; GFX10-DL-NEXT: s_bfe_u32 s7, s4, 0x40010 +; GFX10-DL-NEXT: s_lshr_b32 s8, s4, 28 +; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40018 ; GFX10-DL-NEXT: v_or_b32_sdwa v4, v4, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD -; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s0, s5 -; GFX10-DL-NEXT: v_mul_lo_u16_e64 v11, s1, s6 -; GFX10-DL-NEXT: v_mul_lo_u16_e64 v8, s7, s8 -; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, s2, s4 -; GFX10-DL-NEXT: v_lshrrev_b32_e32 v7, 8, v4 -; GFX10-DL-NEXT: v_and_b32_sdwa v5, v5, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX10-DL-NEXT: v_and_b32_sdwa v2, v11, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD -; GFX10-DL-NEXT: v_or_b32_sdwa v5, v8, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 +; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40018 +; GFX10-DL-NEXT: v_and_b32_sdwa v6, v7, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX10-DL-NEXT: v_mul_lo_u16_e64 v5, s5, s7 +; GFX10-DL-NEXT: v_mul_lo_u16_e64 v7, s6, s8 +; GFX10-DL-NEXT: v_lshrrev_b32_e32 v8, 8, v4 +; GFX10-DL-NEXT: v_mul_lo_u16_e64 v9, s0, s1 +; GFX10-DL-NEXT: v_or_b32_sdwa v5, v5, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 +; GFX10-DL-NEXT: v_and_b32_sdwa v2, v7, v2 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD +; GFX10-DL-NEXT: v_and_b32_e32 v5, 0xffff, v5 ; GFX10-DL-NEXT: v_or_b32_sdwa v2, v9, v2 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:WORD_0 -; GFX10-DL-NEXT: v_or_b32_sdwa v2, v5, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD +; GFX10-DL-NEXT: v_or_b32_e32 v2, v5, v2 +; GFX10-DL-NEXT: v_lshrrev_b32_e32 v6, 8, v2 ; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_add_nc_u32_e32 v6, v4, v3 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v6, v7 -; GFX10-DL-NEXT: v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:BYTE_2 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v4, v3 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v3, v8 +; GFX10-DL-NEXT: v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v3, v3, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 -; GFX10-DL-NEXT: v_lshrrev_b32_e32 v4, 8, v2 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v3, v2 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v3, v4 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v3, v5 +; GFX10-DL-NEXT: v_add_nc_u32_e32 v3, v3, v6 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v3, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1 ; GFX10-DL-NEXT: v_add_nc_u32_sdwa v2, v3, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_3 ; GFX10-DL-NEXT: global_store_byte v[0:1], v2, off @@ -2706,35 +2508,32 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX8-NEXT: v_mov_b32_e32 v3, s1 ; GFX8-NEXT: s_bfe_u32 s5, s4, 0x40004 ; GFX8-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c ; GFX8-NEXT: v_mov_b32_e32 v4, s5 ; GFX8-NEXT: s_bfe_u32 s1, s2, 0x40004 +; GFX8-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX8-NEXT: v_mov_b32_e32 v5, s6 ; GFX8-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX8-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX8-NEXT: v_mov_b32_e32 v5, s7 -; GFX8-NEXT: v_mov_b32_e32 v6, s6 -; GFX8-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX8-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX8-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX8-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX8-NEXT: v_mov_b32_e32 v7, s9 -; GFX8-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX8-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX8-NEXT: v_mov_b32_e32 v8, s11 -; GFX8-NEXT: s_bfe_u32 s14, s2, 0x40018 +; GFX8-NEXT: s_bfe_u32 s8, s4, 0x40010 +; GFX8-NEXT: v_mov_b32_e32 v6, s7 +; GFX8-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX8-NEXT: s_bfe_u32 s9, s4, 0x40014 +; GFX8-NEXT: v_mov_b32_e32 v7, s8 +; GFX8-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX8-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX8-NEXT: v_mov_b32_e32 v8, s9 +; GFX8-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX8-NEXT: s_bfe_u32 s9, s2, 0x40018 ; GFX8-NEXT: s_lshr_b32 s4, s4, 28 -; GFX8-NEXT: v_mov_b32_e32 v9, s13 +; GFX8-NEXT: v_mov_b32_e32 v9, s10 ; GFX8-NEXT: s_lshr_b32 s2, s2, 28 ; GFX8-NEXT: s_waitcnt vmcnt(0) ; GFX8-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX8-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX8-NEXT: v_add_u32_e32 v2, vcc, v5, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX8-NEXT: v_mad_u32_u24 v2, s14, v9, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX8-NEXT: v_mad_u32_u24 v2, s9, v9, v2 ; GFX8-NEXT: v_mov_b32_e32 v3, s4 ; GFX8-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX8-NEXT: v_and_b32_e32 v2, 15, v2 @@ -2757,35 +2556,32 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX9-NEXT: v_mov_b32_e32 v3, s1 ; GFX9-NEXT: s_bfe_u32 s5, s4, 0x40004 ; GFX9-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c ; GFX9-NEXT: v_mov_b32_e32 v4, s5 ; GFX9-NEXT: s_bfe_u32 s1, s2, 0x40004 +; GFX9-NEXT: s_bfe_u32 s7, s4, 0x4000c +; GFX9-NEXT: v_mov_b32_e32 v5, s6 ; GFX9-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX9-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX9-NEXT: v_mov_b32_e32 v5, s7 -; GFX9-NEXT: v_mov_b32_e32 v6, s6 -; GFX9-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX9-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX9-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX9-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX9-NEXT: v_mov_b32_e32 v7, s9 -; GFX9-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX9-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX9-NEXT: v_mov_b32_e32 v8, s11 -; GFX9-NEXT: s_bfe_u32 s14, s2, 0x40018 +; GFX9-NEXT: s_bfe_u32 s8, s4, 0x40010 +; GFX9-NEXT: v_mov_b32_e32 v6, s7 +; GFX9-NEXT: s_bfe_u32 s6, s2, 0x4000c +; GFX9-NEXT: s_bfe_u32 s9, s4, 0x40014 +; GFX9-NEXT: v_mov_b32_e32 v7, s8 +; GFX9-NEXT: s_bfe_u32 s7, s2, 0x40010 +; GFX9-NEXT: s_bfe_u32 s10, s4, 0x40018 +; GFX9-NEXT: v_mov_b32_e32 v8, s9 +; GFX9-NEXT: s_bfe_u32 s8, s2, 0x40014 +; GFX9-NEXT: s_bfe_u32 s9, s2, 0x40018 ; GFX9-NEXT: s_lshr_b32 s4, s4, 28 -; GFX9-NEXT: v_mov_b32_e32 v9, s13 +; GFX9-NEXT: v_mov_b32_e32 v9, s10 ; GFX9-NEXT: s_lshr_b32 s2, s2, 28 ; GFX9-NEXT: s_waitcnt vmcnt(0) ; GFX9-NEXT: v_mad_u32_u24 v2, s0, v3, v2 ; GFX9-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX9-NEXT: v_add_u32_e32 v2, v2, v5 -; GFX9-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX9-NEXT: v_mad_u32_u24 v2, s14, v9, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s5, v5, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s6, v6, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s7, v7, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s8, v8, v2 +; GFX9-NEXT: v_mad_u32_u24 v2, s9, v9, v2 ; GFX9-NEXT: v_mov_b32_e32 v3, s4 ; GFX9-NEXT: v_mad_u32_u24 v2, s2, v3, v2 ; GFX9-NEXT: v_and_b32_e32 v2, 15, v2 @@ -2803,86 +2599,27 @@ define amdgpu_kernel void @udot8_acc4_vecMul(<8 x i4> addrspace(1)* %src1, ; GFX9-DL-NEXT: v_mov_b32_e32 v1, s1 ; GFX9-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX9-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX9-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX9-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX9-DL-NEXT: v_mov_b32_e32 v3, s1 -; GFX9-DL-NEXT: s_bfe_u32 s5, s4, 0x40004 -; GFX9-DL-NEXT: s_bfe_u32 s6, s4, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s7, s4, 0x4000c -; GFX9-DL-NEXT: v_mov_b32_e32 v4, s5 -; GFX9-DL-NEXT: s_bfe_u32 s1, s2, 0x40004 -; GFX9-DL-NEXT: s_bfe_u32 s5, s2, 0x40008 -; GFX9-DL-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX9-DL-NEXT: v_mov_b32_e32 v5, s7 -; GFX9-DL-NEXT: v_mov_b32_e32 v6, s6 -; GFX9-DL-NEXT: v_mul_u32_u24_e32 v5, s8, v5 -; GFX9-DL-NEXT: s_bfe_u32 s9, s4, 0x40010 -; GFX9-DL-NEXT: v_and_b32_e32 v5, 15, v5 -; GFX9-DL-NEXT: s_bfe_u32 s11, s4, 0x40014 -; GFX9-DL-NEXT: s_bfe_u32 s10, s2, 0x40010 -; GFX9-DL-NEXT: v_mov_b32_e32 v7, s9 -; GFX9-DL-NEXT: s_bfe_u32 s13, s4, 0x40018 -; GFX9-DL-NEXT: s_bfe_u32 s12, s2, 0x40014 -; GFX9-DL-NEXT: v_mov_b32_e32 v8, s11 -; GFX9-DL-NEXT: s_bfe_u32 s14, s2, 0x40018 -; GFX9-DL-NEXT: s_lshr_b32 s4, s4, 28 -; GFX9-DL-NEXT: v_mov_b32_e32 v9, s13 -; GFX9-DL-NEXT: s_lshr_b32 s2, s2, 28 -; GFX9-DL-NEXT: s_waitcnt vmcnt(0) -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s0, v3, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s1, v4, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s5, v6, v2 -; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX9-DL-NEXT: v_add_u32_e32 v2, v2, v5 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s10, v7, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s12, v8, v2 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s14, v9, v2 ; GFX9-DL-NEXT: v_mov_b32_e32 v3, s4 -; GFX9-DL-NEXT: v_mad_u32_u24 v2, s2, v3, v2 +; GFX9-DL-NEXT: s_waitcnt vmcnt(0) +; GFX9-DL-NEXT: v_dot8_u32_u4 v2, s2, v3, v2 ; GFX9-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX9-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX9-DL-NEXT: s_endpgm ; ; GFX10-DL-LABEL: udot8_acc4_vecMul: ; GFX10-DL: ; %bb.0: ; %entry -; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 -; GFX10-DL-NEXT: s_load_dwordx2 s[0:1], s[0:1], 0x34 +; GFX10-DL-NEXT: s_load_dwordx2 s[4:5], s[0:1], 0x34 ; GFX10-DL-NEXT: ; implicit-def: $vcc_hi ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_load_dword s2, s[4:5], 0x0 -; GFX10-DL-NEXT: s_load_dword s4, s[6:7], 0x0 -; GFX10-DL-NEXT: v_mov_b32_e32 v0, s0 -; GFX10-DL-NEXT: v_mov_b32_e32 v1, s1 +; GFX10-DL-NEXT: v_mov_b32_e32 v0, s4 +; GFX10-DL-NEXT: v_mov_b32_e32 v1, s5 +; GFX10-DL-NEXT: s_load_dwordx4 s[4:7], s[0:1], 0x24 ; GFX10-DL-NEXT: global_load_ubyte v2, v[0:1], off ; GFX10-DL-NEXT: s_waitcnt lgkmcnt(0) -; GFX10-DL-NEXT: s_and_b32 s0, s2, 15 -; GFX10-DL-NEXT: s_and_b32 s1, s4, 15 -; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40004 -; GFX10-DL-NEXT: s_bfe_u32 s7, s2, 0x40008 -; GFX10-DL-NEXT: s_bfe_u32 s8, s2, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s9, s4, 0x40008 -; GFX10-DL-NEXT: s_waitcnt vmcnt(0) -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s0, s4, 0x4000c -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40010 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 -; GFX10-DL-NEXT: v_mul_u32_u24_e64 v3, s8, s0 -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40010 -; GFX10-DL-NEXT: s_bfe_u32 s5, s2, 0x40014 -; GFX10-DL-NEXT: s_bfe_u32 s6, s4, 0x40014 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s7, s9, v2 -; GFX10-DL-NEXT: v_and_b32_e32 v3, 15, v3 -; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 -; GFX10-DL-NEXT: v_add_nc_u32_e32 v2, v2, v3 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: s_bfe_u32 s0, s2, 0x40018 -; GFX10-DL-NEXT: s_bfe_u32 s1, s4, 0x40018 -; GFX10-DL-NEXT: s_lshr_b32 s2, s2, 28 -; GFX10-DL-NEXT: s_lshr_b32 s4, s4, 28 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s5, s6, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s0, s1, v2 -; GFX10-DL-NEXT: v_mad_u32_u24 v2, s2, s4, v2 +; GFX10-DL-NEXT: s_load_dword s0, s[4:5], 0x0 +; GFX10-DL-NEXT: s_load_dword s1, s[6:7], 0x0 +; GFX10-DL-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0) +; GFX10-DL-NEXT: v_dot8_u32_u4 v2, s0, s1, v2 ; GFX10-DL-NEXT: v_and_b32_e32 v2, 15, v2 ; GFX10-DL-NEXT: global_store_byte v[0:1], v2, off ; GFX10-DL-NEXT: s_endpgm |
