[AVX-512] Don't use unmasked VMOVDQU8/16 for 8-bit or 16-bit element stores even when BWI instructions are supported. Always use VMOVDQA32/VMOVDQU32.

We were already using the 32 bit element opcode if BWI isn't enabled, but there's no reason to change opcode if we have BWI. We will still use the 8/16 opcodes for masked stores though.

This allows us to use the aligned opcode when we can which makes our test output more consistent between different modes. It also reduces the number of isel patterns we need.

This is a slight inconsistency with loads which default to 64 bit element opcodes. I'll probably rectify that in a future patch.

Differential Revision: https://reviews.llvm.org/D35978

llvm-svn: 309693

9 files changed, 98 insertions, 195 deletions

diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index c0f49e45fe8..1a5aff7e295 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -3348,7 +3348,8 @@ multiclass avx512_load_vl<bits<8> opc, string OpcodeStr,
 }
 
 multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
-                        PatFrag st_frag, PatFrag mstore, string Name> {
+                        PatFrag st_frag, PatFrag mstore, string Name,
+                        bit NoMRPattern = 0> {
 
   let hasSideEffects = 0 in {
   def rr_REV  : AVX512PI<opc, MRMDestReg, (outs _.RC:$dst), (ins _.RC:$src),
@@ -3366,9 +3367,12 @@ multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
                           [], _.ExeDomain>, EVEX, EVEX_KZ, FoldGenData<Name#rrkz>;
   }
 
+  let hasSideEffects = 0, mayStore = 1 in
   def mr : AVX512PI<opc, MRMDestMem, (outs), (ins _.MemOp:$dst, _.RC:$src),
                     !strconcat(OpcodeStr, "\t{$src, $dst|$dst, $src}"),
-                    [(st_frag (_.VT _.RC:$src), addr:$dst)], _.ExeDomain>, EVEX;
+                    !if(NoMRPattern, [],
+                        [(st_frag (_.VT _.RC:$src), addr:$dst)]),
+                    _.ExeDomain>, EVEX;
   def mrk : AVX512PI<opc, MRMDestMem, (outs),
                      (ins _.MemOp:$dst, _.KRCWM:$mask, _.RC:$src),
               OpcodeStr # "\t{$src, ${dst} {${mask}}|${dst} {${mask}}, $src}",
@@ -3382,16 +3386,18 @@ multiclass avx512_store<bits<8> opc, string OpcodeStr, X86VectorVTInfo _,
 
 multiclass avx512_store_vl< bits<8> opc, string OpcodeStr,
                             AVX512VLVectorVTInfo _, Predicate prd,
-                            string Name> {
+                            string Name, bit NoMRPattern = 0> {
   let Predicates = [prd] in
   defm Z : avx512_store<opc, OpcodeStr, _.info512, store,
-                        masked_store_unaligned, Name#Z>, EVEX_V512;
+                        masked_store_unaligned, Name#Z, NoMRPattern>, EVEX_V512;
 
   let Predicates = [prd, HasVLX] in {
     defm Z256 : avx512_store<opc, OpcodeStr, _.info256, store,
-                             masked_store_unaligned, Name#Z256>, EVEX_V256;
+                             masked_store_unaligned, Name#Z256,
+                             NoMRPattern>, EVEX_V256;
     defm Z128 : avx512_store<opc, OpcodeStr, _.info128, store,
-                             masked_store_unaligned, Name#Z128>, EVEX_V128;
+                             masked_store_unaligned, Name#Z128,
+                             NoMRPattern>, EVEX_V128;
   }
 }
 
@@ -3448,12 +3454,12 @@ defm VMOVDQA64 : avx512_alignedload_vl<0x6F, "vmovdqa64", avx512vl_i64_info,
 
 defm VMOVDQU8 : avx512_load_vl<0x6F, "vmovdqu8", avx512vl_i8_info, HasBWI, 1>,
                 avx512_store_vl<0x7F, "vmovdqu8", avx512vl_i8_info,
-                                 HasBWI, "VMOVDQU8">,
+                                 HasBWI, "VMOVDQU8", 1>,
                 XD, EVEX_CD8<8, CD8VF>;
 
 defm VMOVDQU16 : avx512_load_vl<0x6F, "vmovdqu16", avx512vl_i16_info, HasBWI, 1>,
                  avx512_store_vl<0x7F, "vmovdqu16", avx512vl_i16_info,
-                                 HasBWI, "VMOVDQU16">,
+                                 HasBWI, "VMOVDQU16", 1>,
                  XD, VEX_W, EVEX_CD8<16, CD8VF>;
 
 defm VMOVDQU32 : avx512_load_vl<0x6F, "vmovdqu32", avx512vl_i32_info, HasAVX512,
@@ -3538,8 +3544,20 @@ def : Pat<(v8i32 (vselect (v8i1 VK8WM:$mask), (v8i32 VR256X:$src1),
            sub_ymm)>;
 }
 
-let Predicates = [HasVLX, NoBWI] in {
-  // 128-bit load/store without BWI.
+let Predicates = [HasAVX512] in {
+  // 512-bit store.
+  def : Pat<(alignedstore512 (v32i16 VR512:$src), addr:$dst),
+            (VMOVDQA32Zmr addr:$dst, VR512:$src)>;
+  def : Pat<(alignedstore512 (v64i8 VR512:$src), addr:$dst),
+            (VMOVDQA32Zmr addr:$dst, VR512:$src)>;
+  def : Pat<(store (v32i16 VR512:$src), addr:$dst),
+            (VMOVDQU32Zmr addr:$dst, VR512:$src)>;
+  def : Pat<(store (v64i8 VR512:$src), addr:$dst),
+            (VMOVDQU32Zmr addr:$dst, VR512:$src)>;
+}
+
+let Predicates = [HasVLX] in {
+  // 128-bit store.
   def : Pat<(alignedstore (v8i16 VR128X:$src), addr:$dst),
             (VMOVDQA32Z128mr addr:$dst, VR128X:$src)>;
   def : Pat<(alignedstore (v16i8 VR128X:$src), addr:$dst),
@@ -3549,7 +3567,7 @@ let Predicates = [HasVLX, NoBWI] in {
   def : Pat<(store (v16i8 VR128X:$src), addr:$dst),
             (VMOVDQU32Z128mr addr:$dst, VR128X:$src)>;
 
-  // 256-bit load/store without BWI.
+  // 256-bit store.
   def : Pat<(alignedstore256 (v16i16 VR256X:$src), addr:$dst),
             (VMOVDQA32Z256mr addr:$dst, VR256X:$src)>;
   def : Pat<(alignedstore256 (v32i8 VR256X:$src), addr:$dst),
@@ -3558,9 +3576,7 @@ let Predicates = [HasVLX, NoBWI] in {
             (VMOVDQU32Z256mr addr:$dst, VR256X:$src)>;
   def : Pat<(store (v32i8 VR256X:$src), addr:$dst),
             (VMOVDQU32Z256mr addr:$dst, VR256X:$src)>;
-}
 
-let Predicates = [HasVLX] in {
   // Special patterns for storing subvector extracts of lower 128-bits of 256.
   // Its cheaper to just use VMOVAPS/VMOVUPS instead of VEXTRACTF128mr
   def : Pat<(alignedstore (v2f64 (extract_subvector
diff --git a/llvm/test/CodeGen/X86/avg.ll b/llvm/test/CodeGen/X86/avg.ll
index cdbbecca822..a2454aa5a04 100644
--- a/llvm/test/CodeGen/X86/avg.ll
+++ b/llvm/test/CodeGen/X86/avg.ll
@@ -712,7 +712,7 @@ define void @avg_v64i8(<64 x i8>* %a, <64 x i8>* %b) {
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm0
 ; AVX512BW-NEXT:    vpavgb (%rdi), %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = load <64 x i8>, <64 x i8>* %a
@@ -1101,7 +1101,7 @@ define void @avg_v32i16(<32 x i16>* %a, <32 x i16>* %b) {
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm0
 ; AVX512BW-NEXT:    vpavgw (%rdi), %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = load <32 x i16>, <32 x i16>* %a
@@ -1734,7 +1734,7 @@ define void @avg_v64i8_2(<64 x i8>* %a, <64 x i8>* %b) {
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vmovdqa64 (%rsi), %zmm0
 ; AVX512BW-NEXT:    vpavgb %zmm0, %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = load <64 x i8>, <64 x i8>* %a
@@ -2124,7 +2124,7 @@ define void @avg_v32i16_2(<32 x i16>* %a, <32 x i16>* %b) {
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpavgw (%rsi), %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = load <32 x i16>, <32 x i16>* %a
@@ -2649,7 +2649,7 @@ define void @avg_v64i8_const(<64 x i8>* %a) {
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpavgb {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = load <64 x i8>, <64 x i8>* %a
@@ -2957,7 +2957,7 @@ define void @avg_v32i16_const(<32 x i16>* %a) {
 ; AVX512BW:       # BB#0:
 ; AVX512BW-NEXT:    vmovdqa64 (%rdi), %zmm0
 ; AVX512BW-NEXT:    vpavgw {{.*}}(%rip), %zmm0, %zmm0
-; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rax)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rax)
 ; AVX512BW-NEXT:    vzeroupper
 ; AVX512BW-NEXT:    retq
   %1 = load <32 x i16>, <32 x i16>* %a
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index 43c44702258..f75c329d10e 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -1839,21 +1839,13 @@ define float @test_extractelement_variable_v16f32(<16 x float> %t1, i32 %index)
 }
 
 define i16 @test_extractelement_variable_v8i16(<8 x i16> %t1, i32 %index) {
-; KNL-LABEL: test_extractelement_variable_v8i16:
-; KNL:       ## BB#0:
-; KNL-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; KNL-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; KNL-NEXT:    andl $7, %edi
-; KNL-NEXT:    movzwl -24(%rsp,%rdi,2), %eax
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: test_extractelement_variable_v8i16:
-; SKX:       ## BB#0:
-; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; SKX-NEXT:    vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
-; SKX-NEXT:    andl $7, %edi
-; SKX-NEXT:    movzwl -24(%rsp,%rdi,2), %eax
-; SKX-NEXT:    retq
+; CHECK-LABEL: test_extractelement_variable_v8i16:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
+; CHECK-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    andl $7, %edi
+; CHECK-NEXT:    movzwl -24(%rsp,%rdi,2), %eax
+; CHECK-NEXT:    retq
   %t2 = extractelement <8 x i16> %t1, i32 %index
   ret i16 %t2
 }
@@ -1892,7 +1884,7 @@ define i16 @test_extractelement_variable_v16i16(<16 x i16> %t1, i32 %index) {
 ; SKX-NEXT:    andq $-32, %rsp
 ; SKX-NEXT:    subq $64, %rsp
 ; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; SKX-NEXT:    vmovdqu %ymm0, (%rsp)
+; SKX-NEXT:    vmovaps %ymm0, (%rsp)
 ; SKX-NEXT:    andl $15, %edi
 ; SKX-NEXT:    movzwl (%rsp,%rdi,2), %eax
 ; SKX-NEXT:    movq %rbp, %rsp
@@ -1938,7 +1930,7 @@ define i16 @test_extractelement_variable_v32i16(<32 x i16> %t1, i32 %index) {
 ; SKX-NEXT:    andq $-64, %rsp
 ; SKX-NEXT:    subq $128, %rsp
 ; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; SKX-NEXT:    vmovdqu16 %zmm0, (%rsp)
+; SKX-NEXT:    vmovaps %zmm0, (%rsp)
 ; SKX-NEXT:    andl $31, %edi
 ; SKX-NEXT:    movzwl (%rsp,%rdi,2), %eax
 ; SKX-NEXT:    movq %rbp, %rsp
@@ -1950,23 +1942,14 @@ define i16 @test_extractelement_variable_v32i16(<32 x i16> %t1, i32 %index) {
 }
 
 define i8 @test_extractelement_variable_v16i8(<16 x i8> %t1, i32 %index) {
-; KNL-LABEL: test_extractelement_variable_v16i8:
-; KNL:       ## BB#0:
-; KNL-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; KNL-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
-; KNL-NEXT:    andl $15, %edi
-; KNL-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
-; KNL-NEXT:    movb (%rdi,%rax), %al
-; KNL-NEXT:    retq
-;
-; SKX-LABEL: test_extractelement_variable_v16i8:
-; SKX:       ## BB#0:
-; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; SKX-NEXT:    vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
-; SKX-NEXT:    andl $15, %edi
-; SKX-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
-; SKX-NEXT:    movb (%rdi,%rax), %al
-; SKX-NEXT:    retq
+; CHECK-LABEL: test_extractelement_variable_v16i8:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
+; CHECK-NEXT:    vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; CHECK-NEXT:    andl $15, %edi
+; CHECK-NEXT:    leaq -{{[0-9]+}}(%rsp), %rax
+; CHECK-NEXT:    movb (%rdi,%rax), %al
+; CHECK-NEXT:    retq
   %t2 = extractelement <16 x i8> %t1, i32 %index
   ret i8 %t2
 }
@@ -2006,7 +1989,7 @@ define i8 @test_extractelement_variable_v32i8(<32 x i8> %t1, i32 %index) {
 ; SKX-NEXT:    andq $-32, %rsp
 ; SKX-NEXT:    subq $64, %rsp
 ; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; SKX-NEXT:    vmovdqu %ymm0, (%rsp)
+; SKX-NEXT:    vmovaps %ymm0, (%rsp)
 ; SKX-NEXT:    andl $31, %edi
 ; SKX-NEXT:    movq %rsp, %rax
 ; SKX-NEXT:    movb (%rdi,%rax), %al
@@ -2055,7 +2038,7 @@ define i8 @test_extractelement_variable_v64i8(<64 x i8> %t1, i32 %index) {
 ; SKX-NEXT:    andq $-64, %rsp
 ; SKX-NEXT:    subq $128, %rsp
 ; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
-; SKX-NEXT:    vmovdqu8 %zmm0, (%rsp)
+; SKX-NEXT:    vmovaps %zmm0, (%rsp)
 ; SKX-NEXT:    andl $63, %edi
 ; SKX-NEXT:    movq %rsp, %rax
 ; SKX-NEXT:    movb (%rdi,%rax), %al
@@ -2105,7 +2088,7 @@ define i8 @test_extractelement_variable_v64i8_indexi8(<64 x i8> %t1, i8 %index)
 ; SKX-NEXT:    andq $-64, %rsp
 ; SKX-NEXT:    subq $128, %rsp
 ; SKX-NEXT:    addb %dil, %dil
-; SKX-NEXT:    vmovdqu8 %zmm0, (%rsp)
+; SKX-NEXT:    vmovaps %zmm0, (%rsp)
 ; SKX-NEXT:    movzbl %dil, %eax
 ; SKX-NEXT:    andl $63, %eax
 ; SKX-NEXT:    movq %rsp, %rcx
@@ -2330,7 +2313,7 @@ define zeroext i8 @test_extractelement_varible_v32i1(<32 x i8> %a, <32 x i8> %b,
 ; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
 ; SKX-NEXT:    vpcmpnleub %ymm1, %ymm0, %k0
 ; SKX-NEXT:    vpmovm2w %k0, %zmm0
-; SKX-NEXT:    vmovdqu16 %zmm0, (%rsp)
+; SKX-NEXT:    vmovdqa32 %zmm0, (%rsp)
 ; SKX-NEXT:    andl $31, %edi
 ; SKX-NEXT:    movzbl (%rsp,%rdi,2), %eax
 ; SKX-NEXT:    andl $1, %eax
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract_i1.ll b/llvm/test/CodeGen/X86/avx512-insert-extract_i1.ll
index a099b80898e..e36fad345b1 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract_i1.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract_i1.ll
@@ -19,7 +19,7 @@ define zeroext i8 @test_extractelement_varible_v64i1(<64 x i8> %a, <64 x i8> %b,
 ; SKX-NEXT:    ## kill: %EDI<def> %EDI<kill> %RDI<def>
 ; SKX-NEXT:    vpcmpnleub %zmm1, %zmm0, %k0
 ; SKX-NEXT:    vpmovm2b %k0, %zmm0
-; SKX-NEXT:    vmovdqu8 %zmm0, (%rsp)
+; SKX-NEXT:    vmovdqa32 %zmm0, (%rsp)
 ; SKX-NEXT:    andl $63, %edi
 ; SKX-NEXT:    movq %rsp, %rax
 ; SKX-NEXT:    movzbl (%rdi,%rax), %eax
diff --git a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
index 02cd9397a51..c6e1dbd8811 100644
--- a/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-intrinsics-upgrade.ll
@@ -9,7 +9,7 @@ define void@test_int_x86_avx512_mask_storeu_b_512(i8* %ptr1, i8* %ptr2, <64 x i8
 ; AVX512BW:       ## BB#0:
 ; AVX512BW-NEXT:    kmovq %rdx, %k1
 ; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rdi) {%k1}
-; AVX512BW-NEXT:    vmovdqu8 %zmm0, (%rsi)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rsi)
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512F-32-LABEL: test_int_x86_avx512_mask_storeu_b_512:
@@ -18,7 +18,7 @@ define void@test_int_x86_avx512_mask_storeu_b_512(i8* %ptr1, i8* %ptr2, <64 x i8
 ; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; AVX512F-32-NEXT:    kmovq {{[0-9]+}}(%esp), %k1
 ; AVX512F-32-NEXT:    vmovdqu8 %zmm0, (%ecx) {%k1}
-; AVX512F-32-NEXT:    vmovdqu8 %zmm0, (%eax)
+; AVX512F-32-NEXT:    vmovdqu32 %zmm0, (%eax)
 ; AVX512F-32-NEXT:    retl
   call void @llvm.x86.avx512.mask.storeu.b.512(i8* %ptr1, <64 x i8> %x1, i64 %x2)
   call void @llvm.x86.avx512.mask.storeu.b.512(i8* %ptr2, <64 x i8> %x1, i64 -1)
@@ -32,7 +32,7 @@ define void@test_int_x86_avx512_mask_storeu_w_512(i8* %ptr1, i8* %ptr2, <32 x i1
 ; AVX512BW:       ## BB#0:
 ; AVX512BW-NEXT:    kmovd %edx, %k1
 ; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rdi) {%k1}
-; AVX512BW-NEXT:    vmovdqu16 %zmm0, (%rsi)
+; AVX512BW-NEXT:    vmovdqu32 %zmm0, (%rsi)
 ; AVX512BW-NEXT:    retq
 ;
 ; AVX512F-32-LABEL: test_int_x86_avx512_mask_storeu_w_512:
@@ -41,7 +41,7 @@ define void@test_int_x86_avx512_mask_storeu_w_512(i8* %ptr1, i8* %ptr2, <32 x i1
 ; AVX512F-32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
 ; AVX512F-32-NEXT:    kmovd {{[0-9]+}}(%esp), %k1
 ; AVX512F-32-NEXT:    vmovdqu16 %zmm0, (%ecx) {%k1}
-; AVX512F-32-NEXT:    vmovdqu16 %zmm0, (%eax)
+; AVX512F-32-NEXT:    vmovdqu32 %zmm0, (%eax)
 ; AVX512F-32-NEXT:    retl
   call void @llvm.x86.avx512.mask.storeu.w.512(i8* %ptr1, <32 x i16> %x1, i32 %x2)
   call void @llvm.x86.avx512.mask.storeu.w.512(i8* %ptr2, <32 x i16> %x1, i32 -1)
diff --git a/llvm/test/CodeGen/X86/avx512bw-mov.ll b/llvm/test/CodeGen/X86/avx512bw-mov.ll
index c38d436db21..03d4314bd70 100644
--- a/llvm/test/CodeGen/X86/avx512bw-mov.ll
+++ b/llvm/test/CodeGen/X86/avx512bw-mov.ll
@@ -14,7 +14,7 @@ define <64 x i8> @test1(i8 * %addr) {
 define void @test2(i8 * %addr, <64 x i8> %data) {
 ; CHECK-LABEL: test2:
 ; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovdqu8 %zmm0, (%rdi)
+; CHECK-NEXT:    vmovups %zmm0, (%rdi)
 ; CHECK-NEXT:    retq
   %vaddr = bitcast i8* %addr to <64 x i8>*
   store <64 x i8>%data, <64 x i8>* %vaddr, align 1
@@ -62,7 +62,7 @@ define <32 x i16> @test5(i8 * %addr) {
 define void @test6(i8 * %addr, <32 x i16> %data) {
 ; CHECK-LABEL: test6:
 ; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovdqu16 %zmm0, (%rdi)
+; CHECK-NEXT:    vmovups %zmm0, (%rdi)
 ; CHECK-NEXT:    retq
   %vaddr = bitcast i8* %addr to <32 x i16>*
   store <32 x i16>%data, <32 x i16>* %vaddr, align 1
diff --git a/llvm/test/CodeGen/X86/avx512bwvl-mov.ll b/llvm/test/CodeGen/X86/avx512bwvl-mov.ll
index bb128d2be9e..8ceeed799c6 100644
--- a/llvm/test/CodeGen/X86/avx512bwvl-mov.ll
+++ b/llvm/test/CodeGen/X86/avx512bwvl-mov.ll
@@ -14,7 +14,7 @@ define <32 x i8> @test_256_1(i8 * %addr) {
 define void @test_256_2(i8 * %addr, <32 x i8> %data) {
 ; CHECK-LABEL: test_256_2:
 ; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovdqu %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfe,0x7f,0x07]
+; CHECK-NEXT:    vmovups %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %vaddr = bitcast i8* %addr to <32 x i8>*
   store <32 x i8>%data, <32 x i8>* %vaddr, align 1
@@ -62,7 +62,7 @@ define <16 x i16> @test_256_5(i8 * %addr) {
 define void @test_256_6(i8 * %addr, <16 x i16> %data) {
 ; CHECK-LABEL: test_256_6:
 ; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovdqu %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfe,0x7f,0x07]
+; CHECK-NEXT:    vmovups %ymm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfc,0x11,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %vaddr = bitcast i8* %addr to <16 x i16>*
   store <16 x i16>%data, <16 x i16>* %vaddr, align 1
@@ -110,7 +110,7 @@ define <16 x i8> @test_128_1(i8 * %addr) {
 define void @test_128_2(i8 * %addr, <16 x i8> %data) {
 ; CHECK-LABEL: test_128_2:
 ; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovdqu %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7f,0x07]
+; CHECK-NEXT:    vmovups %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %vaddr = bitcast i8* %addr to <16 x i8>*
   store <16 x i8>%data, <16 x i8>* %vaddr, align 1
@@ -158,7 +158,7 @@ define <8 x i16> @test_128_5(i8 * %addr) {
 define void @test_128_6(i8 * %addr, <8 x i16> %data) {
 ; CHECK-LABEL: test_128_6:
 ; CHECK:       ## BB#0:
-; CHECK-NEXT:    vmovdqu %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x7f,0x07]
+; CHECK-NEXT:    vmovups %xmm0, (%rdi) ## EVEX TO VEX Compression encoding: [0xc5,0xf8,0x11,0x07]
 ; CHECK-NEXT:    retq ## encoding: [0xc3]
   %vaddr = bitcast i8* %addr to <8 x i16>*
   store <8 x i16>%data, <8 x i16>* %vaddr, align 1
diff --git a/llvm/test/CodeGen/X86/subvector-broadcast.ll b/llvm/test/CodeGen/X86/subvector-broadcast.ll
index fd5031f8bd1..09c76e66165 100644
--- a/llvm/test/CodeGen/X86/subvector-broadcast.ll
+++ b/llvm/test/CodeGen/X86/subvector-broadcast.ll
@@ -894,69 +894,21 @@ define <8 x i32> @test_broadcast_4i32_8i32_reuse(<4 x i32>* %p0, <4 x i32>* %p1)
 }
 
 define <16 x i16> @test_broadcast_8i16_16i16_reuse(<8 x i16> *%p0, <8 x i16> *%p1) nounwind {
-; X32-AVX-LABEL: test_broadcast_8i16_16i16_reuse:
-; X32-AVX:       ## BB#0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX-NEXT:    vmovaps (%ecx), %xmm0
-; X32-AVX-NEXT:    vmovaps %xmm0, (%eax)
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX-NEXT:    retl
-;
-; X32-AVX512F-LABEL: test_broadcast_8i16_16i16_reuse:
-; X32-AVX512F:       ## BB#0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX512F-NEXT:    vmovaps (%ecx), %xmm0
-; X32-AVX512F-NEXT:    vmovaps %xmm0, (%eax)
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: test_broadcast_8i16_16i16_reuse:
-; X32-AVX512BW:       ## BB#0:
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX512BW-NEXT:    vmovdqa (%ecx), %xmm0
-; X32-AVX512BW-NEXT:    vmovdqu %xmm0, (%eax)
-; X32-AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: test_broadcast_8i16_16i16_reuse:
-; X32-AVX512DQ:       ## BB#0:
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX512DQ-NEXT:    vmovaps (%ecx), %xmm0
-; X32-AVX512DQ-NEXT:    vmovaps %xmm0, (%eax)
-; X32-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512DQ-NEXT:    retl
-;
-; X64-AVX-LABEL: test_broadcast_8i16_16i16_reuse:
-; X64-AVX:       ## BB#0:
-; X64-AVX-NEXT:    vmovaps (%rdi), %xmm0
-; X64-AVX-NEXT:    vmovaps %xmm0, (%rsi)
-; X64-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX-NEXT:    retq
-;
-; X64-AVX512F-LABEL: test_broadcast_8i16_16i16_reuse:
-; X64-AVX512F:       ## BB#0:
-; X64-AVX512F-NEXT:    vmovaps (%rdi), %xmm0
-; X64-AVX512F-NEXT:    vmovaps %xmm0, (%rsi)
-; X64-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: test_broadcast_8i16_16i16_reuse:
-; X64-AVX512BW:       ## BB#0:
-; X64-AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; X64-AVX512BW-NEXT:    vmovdqu %xmm0, (%rsi)
-; X64-AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512BW-NEXT:    retq
+; X32-LABEL: test_broadcast_8i16_16i16_reuse:
+; X32:       ## BB#0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    vmovaps (%ecx), %xmm0
+; X32-NEXT:    vmovaps %xmm0, (%eax)
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    retl
 ;
-; X64-AVX512DQ-LABEL: test_broadcast_8i16_16i16_reuse:
-; X64-AVX512DQ:       ## BB#0:
-; X64-AVX512DQ-NEXT:    vmovaps (%rdi), %xmm0
-; X64-AVX512DQ-NEXT:    vmovaps %xmm0, (%rsi)
-; X64-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512DQ-NEXT:    retq
+; X64-LABEL: test_broadcast_8i16_16i16_reuse:
+; X64:       ## BB#0:
+; X64-NEXT:    vmovaps (%rdi), %xmm0
+; X64-NEXT:    vmovaps %xmm0, (%rsi)
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-NEXT:    retq
  %1 = load <8 x i16>, <8 x i16> *%p0
  store <8 x i16> %1, <8 x i16>* %p1
  %2 = shufflevector <8 x i16> %1, <8 x i16> undef, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
@@ -964,69 +916,21 @@ define <16 x i16> @test_broadcast_8i16_16i16_reuse(<8 x i16> *%p0, <8 x i16> *%p
 }
 
 define <32 x i8> @test_broadcast_16i8_32i8_reuse(<16 x i8> *%p0, <16 x i8> *%p1) nounwind {
-; X32-AVX-LABEL: test_broadcast_16i8_32i8_reuse:
-; X32-AVX:       ## BB#0:
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX-NEXT:    vmovaps (%ecx), %xmm0
-; X32-AVX-NEXT:    vmovaps %xmm0, (%eax)
-; X32-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX-NEXT:    retl
-;
-; X32-AVX512F-LABEL: test_broadcast_16i8_32i8_reuse:
-; X32-AVX512F:       ## BB#0:
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512F-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX512F-NEXT:    vmovaps (%ecx), %xmm0
-; X32-AVX512F-NEXT:    vmovaps %xmm0, (%eax)
-; X32-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512F-NEXT:    retl
-;
-; X32-AVX512BW-LABEL: test_broadcast_16i8_32i8_reuse:
-; X32-AVX512BW:       ## BB#0:
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512BW-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX512BW-NEXT:    vmovdqa (%ecx), %xmm0
-; X32-AVX512BW-NEXT:    vmovdqu %xmm0, (%eax)
-; X32-AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512BW-NEXT:    retl
-;
-; X32-AVX512DQ-LABEL: test_broadcast_16i8_32i8_reuse:
-; X32-AVX512DQ:       ## BB#0:
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %eax
-; X32-AVX512DQ-NEXT:    movl {{[0-9]+}}(%esp), %ecx
-; X32-AVX512DQ-NEXT:    vmovaps (%ecx), %xmm0
-; X32-AVX512DQ-NEXT:    vmovaps %xmm0, (%eax)
-; X32-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X32-AVX512DQ-NEXT:    retl
-;
-; X64-AVX-LABEL: test_broadcast_16i8_32i8_reuse:
-; X64-AVX:       ## BB#0:
-; X64-AVX-NEXT:    vmovaps (%rdi), %xmm0
-; X64-AVX-NEXT:    vmovaps %xmm0, (%rsi)
-; X64-AVX-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX-NEXT:    retq
-;
-; X64-AVX512F-LABEL: test_broadcast_16i8_32i8_reuse:
-; X64-AVX512F:       ## BB#0:
-; X64-AVX512F-NEXT:    vmovaps (%rdi), %xmm0
-; X64-AVX512F-NEXT:    vmovaps %xmm0, (%rsi)
-; X64-AVX512F-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512F-NEXT:    retq
-;
-; X64-AVX512BW-LABEL: test_broadcast_16i8_32i8_reuse:
-; X64-AVX512BW:       ## BB#0:
-; X64-AVX512BW-NEXT:    vmovdqa (%rdi), %xmm0
-; X64-AVX512BW-NEXT:    vmovdqu %xmm0, (%rsi)
-; X64-AVX512BW-NEXT:    vinserti128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512BW-NEXT:    retq
+; X32-LABEL: test_broadcast_16i8_32i8_reuse:
+; X32:       ## BB#0:
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %eax
+; X32-NEXT:    movl {{[0-9]+}}(%esp), %ecx
+; X32-NEXT:    vmovaps (%ecx), %xmm0
+; X32-NEXT:    vmovaps %xmm0, (%eax)
+; X32-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X32-NEXT:    retl
 ;
-; X64-AVX512DQ-LABEL: test_broadcast_16i8_32i8_reuse:
-; X64-AVX512DQ:       ## BB#0:
-; X64-AVX512DQ-NEXT:    vmovaps (%rdi), %xmm0
-; X64-AVX512DQ-NEXT:    vmovaps %xmm0, (%rsi)
-; X64-AVX512DQ-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X64-AVX512DQ-NEXT:    retq
+; X64-LABEL: test_broadcast_16i8_32i8_reuse:
+; X64:       ## BB#0:
+; X64-NEXT:    vmovaps (%rdi), %xmm0
+; X64-NEXT:    vmovaps %xmm0, (%rsi)
+; X64-NEXT:    vinsertf128 $1, %xmm0, %ymm0, %ymm0
+; X64-NEXT:    retq
  %1 = load <16 x i8>, <16 x i8> *%p0
  store <16 x i8> %1, <16 x i8>* %p1
  %2 = shufflevector <16 x i8> %1, <16 x i8> undef, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
diff --git a/llvm/test/CodeGen/X86/x86-interleaved-access.ll b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
index 311868554ef..4bef4e2b58d 100644
--- a/llvm/test/CodeGen/X86/x86-interleaved-access.ll
+++ b/llvm/test/CodeGen/X86/x86-interleaved-access.ll
@@ -361,8 +361,8 @@ define void @interleaved_store_vf32_i8_stride4(<32 x i8> %x1, <32 x i8> %x2, <32
 ; AVX512-NEXT:    vperm2i128 {{.*#+}} ymm0 = ymm0[2,3],ymm5[2,3]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm4, %zmm2, %zmm2
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512-NEXT:    vmovdqu8 %zmm0, 64(%rdi)
-; AVX512-NEXT:    vmovdqu8 %zmm2, (%rdi)
+; AVX512-NEXT:    vmovdqa32 %zmm0, 64(%rdi)
+; AVX512-NEXT:    vmovdqa32 %zmm2, (%rdi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
   %v1 = shufflevector <32 x i8> %x1, <32 x i8> %x2, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
@@ -444,7 +444,7 @@ define void @interleaved_store_vf16_i8_stride4(<16 x i8> %x1, <16 x i8> %x2, <16
 ; AVX512-NEXT:    vpblendvb %ymm8, %ymm0, %ymm3, %ymm0
 ; AVX512-NEXT:    vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
 ; AVX512-NEXT:    vinserti64x4 $1, %ymm2, %zmm0, %zmm0
-; AVX512-NEXT:    vmovdqu8 %zmm0, (%rdi)
+; AVX512-NEXT:    vmovdqa32 %zmm0, (%rdi)
 ; AVX512-NEXT:    vzeroupper
 ; AVX512-NEXT:    retq
 %v1 = shufflevector <16 x i8> %x1, <16 x i8> %x2, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>