[GlobalISel][X86] G_LOAD/G_STORE vec256/512 support

Summary: mark G_LOAD/G_STORE vec256/512 legal for AVX/AVX512. Implement instruction selection.

Reviewers: zvi, guyblank

Reviewed By: zvi

Subscribers: rovka, llvm-commits, kristof.beyls

Differential Revision: https://reviews.llvm.org/D33268

llvm-svn: 303617

5 files changed, 530 insertions, 40 deletions

diff --git a/llvm/test/CodeGen/X86/GlobalISel/memop-vec.ll b/llvm/test/CodeGen/X86/GlobalISel/memop-vec.ll
index f1ffc15f4d0..870e812bbb6 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/memop-vec.ll
+++ b/llvm/test/CodeGen/X86/GlobalISel/memop-vec.ll
@@ -1,39 +1,116 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=x86_64-linux-gnu -mcpu=skx                       -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=SKX
-; RUN: llc -mtriple=x86_64-linux-gnu -mcpu=skx -regbankselect-greedy -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=SKX
+; RUN: llc -mtriple=x86_64-linux-gnu -mcpu=skx                       -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=SKX
+; RUN: llc -mtriple=x86_64-linux-gnu -mcpu=skx -regbankselect-greedy -global-isel -verify-machineinstrs < %s -o - | FileCheck %s --check-prefix=SKX
 
 define <4 x i32> @test_load_v4i32_noalign(<4 x i32> * %p1) {
-; ALL-LABEL: test_load_v4i32_noalign:
-; ALL:       # BB#0:
-; ALL-NEXT:    vmovups (%rdi), %xmm0
-; ALL-NEXT:    retq
+; SKX-LABEL: test_load_v4i32_noalign:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovups (%rdi), %xmm0
+; SKX-NEXT:    retq
   %r = load <4 x i32>, <4 x i32>* %p1, align 1
   ret <4 x i32> %r
 }
 
 define <4 x i32> @test_load_v4i32_align(<4 x i32> * %p1) {
-; ALL-LABEL: test_load_v4i32_align:
-; ALL:       # BB#0:
-; ALL-NEXT:    vmovaps (%rdi), %xmm0
-; ALL-NEXT:    retq
+; SKX-LABEL: test_load_v4i32_align:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovaps (%rdi), %xmm0
+; SKX-NEXT:    retq
   %r = load <4 x i32>, <4 x i32>* %p1, align 16
   ret <4 x i32> %r
 }
 
+define <8 x i32> @test_load_v8i32_noalign(<8 x i32> * %p1) {
+; SKX-LABEL: test_load_v8i32_noalign:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovups (%rdi), %ymm0
+; SKX-NEXT:    retq
+  %r = load <8 x i32>, <8 x i32>* %p1, align 1
+  ret <8 x i32> %r
+}
+
+define <8 x i32> @test_load_v8i32_align(<8 x i32> * %p1) {
+; SKX-LABEL: test_load_v8i32_align:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovaps (%rdi), %ymm0
+; SKX-NEXT:    retq
+  %r = load <8 x i32>, <8 x i32>* %p1, align 32
+  ret <8 x i32> %r
+}
+
+define <16 x i32> @test_load_v16i32_noalign(<16 x i32> * %p1) {
+; SKX-LABEL: test_load_v16i32_noalign:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovups (%rdi), %zmm0
+; SKX-NEXT:    retq
+  %r = load <16 x i32>, <16 x i32>* %p1, align 1
+  ret <16 x i32> %r
+}
+
+define <16 x i32> @test_load_v16i32_align(<16 x i32> * %p1) {
+; SKX-LABEL: test_load_v16i32_align:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovups (%rdi), %zmm0
+; SKX-NEXT:    retq
+  %r = load <16 x i32>, <16 x i32>* %p1, align 32
+  ret <16 x i32> %r
+}
+
 define void @test_store_v4i32_noalign(<4 x i32> %val, <4 x i32>* %p1) {
-; ALL-LABEL: test_store_v4i32_noalign:
-; ALL:       # BB#0:
-; ALL-NEXT:    vmovups %xmm0, (%rdi)
-; ALL-NEXT:    retq
+; SKX-LABEL: test_store_v4i32_noalign:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovups %xmm0, (%rdi)
+; SKX-NEXT:    retq
   store <4 x i32> %val, <4 x i32>* %p1, align 1
   ret void
 }
 
 define void @test_store_v4i32_align(<4 x i32> %val, <4 x i32>* %p1) {
-; ALL-LABEL: test_store_v4i32_align:
-; ALL:       # BB#0:
-; ALL-NEXT:    vmovaps %xmm0, (%rdi)
-; ALL-NEXT:    retq
+; SKX-LABEL: test_store_v4i32_align:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovaps %xmm0, (%rdi)
+; SKX-NEXT:    retq
   store <4 x i32> %val, <4 x i32>* %p1, align 16
   ret void
 }
+
+define void @test_store_v8i32_noalign(<8 x i32> %val, <8 x i32>* %p1) {
+; SKX-LABEL: test_store_v8i32_noalign:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovups %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  store <8 x i32> %val, <8 x i32>* %p1, align 1
+  ret void
+}
+
+define void @test_store_v8i32_align(<8 x i32> %val, <8 x i32>* %p1) {
+; SKX-LABEL: test_store_v8i32_align:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovaps %ymm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  store <8 x i32> %val, <8 x i32>* %p1, align 32
+  ret void
+}
+
+define void @test_store_v16i32_noalign(<16 x i32> %val, <16 x i32>* %p1) {
+; SKX-LABEL: test_store_v16i32_noalign:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovups %zmm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  store <16 x i32> %val, <16 x i32>* %p1, align 1
+  ret void
+}
+
+define void @test_store_v16i32_align(<16 x i32> %val, <16 x i32>* %p1) {
+; SKX-LABEL: test_store_v16i32_align:
+; SKX:       # BB#0:
+; SKX-NEXT:    vmovaps %zmm0, (%rdi)
+; SKX-NEXT:    vzeroupper
+; SKX-NEXT:    retq
+  store <16 x i32> %val, <16 x i32>* %p1, align 64
+  ret void
+}
+
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
index f925c836f3d..cc03f3a57f0 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX2.mir
@@ -14,7 +14,16 @@
     ret void
   }
 
-...
+  define <8 x i32> @test_load_v8i32_noalign(<8 x i32>* %p1) {
+    %r = load <8 x i32>, <8 x i32>* %p1, align 1
+    ret <8 x i32> %r
+  }
+
+  define void @test_store_v8i32_noalign(<8 x i32> %val, <8 x i32>* %p1) {
+    store <8 x i32> %val, <8 x i32>* %p1, align 1
+    ret void
+  }
+
 ---
 name:            test_mul_vec256
 alignment:       4
@@ -84,3 +93,47 @@ body:             |
     RET 0
 
 ...
+---
+name:            test_load_v8i32_noalign
+# CHECK-LABEL: name:  test_load_v8i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: false
+# CHECK:       registers:
+# CHECK-NEXT:    - { id: 0, class: gpr }
+# CHECK-NEXT:    - { id: 1, class: vecr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi
+
+    %0(p0) = COPY %rdi
+    %1(<8 x s32>) = G_LOAD %0(p0) :: (load 32 from %ir.p1, align 1)
+    %ymm0 = COPY %1(<8 x s32>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_store_v8i32_noalign
+# CHECK-LABEL: name:  test_store_v8i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: false
+# CHECK:       registers:
+# CHECK-NEXT:    - { id: 0, class: vecr }
+# CHECK-NEXT:    - { id: 1, class: gpr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %ymm0
+
+    %0(<8 x s32>) = COPY %ymm0
+    %1(p0) = COPY %rdi
+    G_STORE %0(<8 x s32>), %1(p0) :: (store 32 into %ir.p1, align 1)
+    RET 0
+
+...
diff --git a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
index e0c12ff44a2..278413ad38e 100644
--- a/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
+++ b/llvm/test/CodeGen/X86/GlobalISel/regbankselect-AVX512.mir
@@ -15,22 +15,29 @@
     ret void
   }
 
+  define <16 x i32> @test_load_v16i32_noalign(<16 x i32>* %p1) {
+    %r = load <16 x i32>, <16 x i32>* %p1, align 1
+    ret <16 x i32> %r
+  }
+
+  define void @test_store_v16i32_noalign(<16 x i32> %val, <16 x i32>* %p1) {
+    store <16 x i32> %val, <16 x i32>* %p1, align 1
+    ret void
+  }
+
 ...
 ---
 name:            test_mul_vec512
+# CHECK-LABEL: name:  test_mul_vec512
 alignment:       4
 legalized:       true
 regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-# CHECK-LABEL: name:            test_mul_vec512
-# CHECK: registers:
-# CHECK:  - { id: 0, class: vecr }
-# CHECK:  - { id: 1, class: vecr }
+# CHECK:       registers:
+# CHECK-NEXT:    - { id: 0, class: vecr }
+# CHECK-NEXT:    - { id: 1, class: vecr }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
-  - { id: 2, class: _ }
 body:             |
   bb.1 (%ir-block.0):
 
@@ -41,19 +48,16 @@ body:             |
 ...
 ---
 name:            test_add_vec512
+# CHECK-LABEL: name:  test_add_vec512
 alignment:       4
 legalized:       true
 regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-# CHECK-LABEL: name:            test_add_vec512
-# CHECK: registers:
-# CHECK:  - { id: 0, class: vecr }
-# CHECK:  - { id: 1, class: vecr }
+# CHECK:       registers:
+# CHECK-NEXT:    - { id: 0, class: vecr }
+# CHECK-NEXT:    - { id: 1, class: vecr }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
-  - { id: 2, class: _ }
 body:             |
   bb.1 (%ir-block.0):
 
@@ -64,24 +68,65 @@ body:             |
 ...
 ---
 name:            test_sub_vec512
+# CHECK-LABEL: name:  test_sub_vec512
 alignment:       4
 legalized:       true
 regBankSelected: false
-selected:        false
-tracksRegLiveness: true
-# CHECK-LABEL: name:            test_sub_vec512
-# CHECK: registers:
-# CHECK:  - { id: 0, class: vecr }
-# CHECK:  - { id: 1, class: vecr }
+# CHECK:       registers:
+# CHECK-NEXT:    - { id: 0, class: vecr }
+# CHECK-NEXT:    - { id: 1, class: vecr }
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
-  - { id: 2, class: _ }
 body:             |
   bb.1 (%ir-block.0):
 
     %0(<16 x s32>) = IMPLICIT_DEF
     %1(<16 x s32>) = G_SUB %0, %0
     RET 0
+...
+---
+
+name:            test_load_v16i32_noalign
+# CHECK-LABEL: name:  test_load_v16i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: false
+# CHECK:       registers:
+# CHECK-NEXT:    - { id: 0, class: gpr }
+# CHECK-NEXT:    - { id: 1, class: vecr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi
+
+    %0(p0) = COPY %rdi
+    %1(<16 x s32>) = G_LOAD %0(p0) :: (load 64 from %ir.p1, align 1)
+    %zmm0 = COPY %1(<16 x s32>)
+    RET 0, implicit %zmm0
+
+...
+---
+name:            test_store_v16i32_noalign
+# CHECK-LABEL: name:  test_store_v16i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: false
+# CHECK:       registers:
+# CHECK-NEXT:    - { id: 0, class: vecr }
+# CHECK-NEXT:    - { id: 1, class: gpr }
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %zmm0
+
+    %0(<16 x s32>) = COPY %zmm0
+    %1(p0) = COPY %rdi
+    G_STORE %0(<16 x s32>), %1(p0) :: (store 64 into %ir.p1, align 1)
+    RET 0
 
 ...
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
new file mode 100644
index 00000000000..b9a7e4a8cc4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v256.mir
@@ -0,0 +1,188 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx                      -global-isel -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=NO_AVX512VL --check-prefix=NO_AVX512F --check-prefix=AVX
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f                  -global-isel -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=NO_AVX512VL --check-prefix=AVX512ALL --check-prefix=AVX512F
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f -mattr=+avx512vl -global-isel -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=ALL --check-prefix=AVX512ALL --check-prefix=AVX512VL
+
+
+--- |
+  define <8 x i32> @test_load_v8i32_noalign(<8 x i32>* %p1) {
+    %r = load <8 x i32>, <8 x i32>* %p1, align 1
+    ret <8 x i32> %r
+  }
+
+  define <8 x i32> @test_load_v8i32_align(<8 x i32>* %p1) {
+    %r = load <8 x i32>, <8 x i32>* %p1, align 32
+    ret <8 x i32> %r
+  }
+
+  define void @test_store_v8i32_noalign(<8 x i32> %val, <8 x i32>* %p1) {
+    store <8 x i32> %val, <8 x i32>* %p1, align 1
+    ret void
+  }
+
+  define void @test_store_v8i32_align(<8 x i32> %val, <8 x i32>* %p1) {
+    store <8 x i32> %val, <8 x i32>* %p1, align 32
+    ret void
+  }
+
+
+...
+---
+name:            test_load_v8i32_noalign
+# ALL-LABEL: name:  test_load_v8i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: true
+# NO_AVX512F:       registers:
+# NO_AVX512F-NEXT:    - { id: 0, class: gr64 }
+# NO_AVX512F-NEXT:    - { id: 1, class: vr256 }
+#
+# AVX512ALL:        registers:
+# AVX512ALL-NEXT:     - { id: 0, class: gr64 }
+# AVX512ALL-NEXT:     - { id: 1, class: vr256x }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: vecr }
+# NO_AVX512F:           %0 = COPY %rdi
+# NO_AVX512F-NEXT:      %1 = VMOVUPSYrm %0, 1, _, 0, _ :: (load 32 from %ir.p1, align 1)
+# NO_AVX512F-NEXT:      %ymm0 = COPY %1
+# NO_AVX512F-NEXT:      RET 0, implicit %ymm0
+#
+# AVX512F:              %0 = COPY %rdi
+# AVX512F-NEXT:         %1 = VMOVUPSZ256rm_NOVLX %0, 1, _, 0, _ :: (load 32 from %ir.p1, align 1)
+# AVX512F-NEXT:         %ymm0 = COPY %1
+# AVX512F-NEXT:         RET 0, implicit %ymm0
+#
+# AVX512VL:             %0 = COPY %rdi
+# AVX512VL-NEXT:        %1 = VMOVUPSZ256rm %0, 1, _, 0, _ :: (load 32 from %ir.p1, align 1)
+# AVX512VL-NEXT:        %ymm0 = COPY %1
+# AVX512VL-NEXT:        RET 0, implicit %ymm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi
+
+    %0(p0) = COPY %rdi
+    %1(<8 x s32>) = G_LOAD %0(p0) :: (load 32 from %ir.p1, align 1)
+    %ymm0 = COPY %1(<8 x s32>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_load_v8i32_align
+# ALL-LABEL: name:  test_load_v8i32_align
+alignment:       4
+legalized:       true
+regBankSelected: true
+# NO_AVX512F:       registers:
+# NO_AVX512F-NEXT:    - { id: 0, class: gr64 }
+# NO_AVX512F-NEXT:    - { id: 1, class: vr256 }
+#
+# AVX512ALL:        registers:
+# AVX512ALL-NEXT:     - { id: 0, class: gr64 }
+# AVX512ALL-NEXT:     - { id: 1, class: vr256x }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: vecr }
+# NO_AVX512F:           %0 = COPY %rdi
+# NO_AVX512F-NEXT:      %1 = VMOVAPSYrm %0, 1, _, 0, _ :: (load 32 from %ir.p1)
+# NO_AVX512F-NEXT:      %ymm0 = COPY %1
+# NO_AVX512F-NEXT:      RET 0, implicit %ymm0
+#
+# AVX512F:              %0 = COPY %rdi
+# AVX512F-NEXT:         %1 = VMOVAPSZ256rm_NOVLX %0, 1, _, 0, _ :: (load 32 from %ir.p1)
+# AVX512F-NEXT:         %ymm0 = COPY %1
+# AVX512F-NEXT:         RET 0, implicit %ymm0
+#
+# AVX512VL:             %0 = COPY %rdi
+# AVX512VL-NEXT:        %1 = VMOVAPSZ256rm %0, 1, _, 0, _ :: (load 32 from %ir.p1)
+# AVX512VL-NEXT:        %ymm0 = COPY %1
+# AVX512VL-NEXT:        RET 0, implicit %ymm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi
+
+    %0(p0) = COPY %rdi
+    %1(<8 x s32>) = G_LOAD %0(p0) :: (load 32 from %ir.p1)
+    %ymm0 = COPY %1(<8 x s32>)
+    RET 0, implicit %ymm0
+
+...
+---
+name:            test_store_v8i32_noalign
+# ALL-LABEL: name:  test_store_v8i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: true
+# NO_AVX512F:       registers:
+# NO_AVX512F-NEXT:    - { id: 0, class: vr256 }
+# NO_AVX512F-NEXT:    - { id: 1, class: gr64 }
+#
+# AVX512ALL:        registers:
+# AVX512ALL-NEXT:     - { id: 0, class: vr256x }
+# AVX512ALL-NEXT:     - { id: 1, class: gr64 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: gpr }
+# NO_AVX512F:           %0 = COPY %ymm0
+# NO_AVX512F-NEXT:      %1 = COPY %rdi
+# NO_AVX512F-NEXT:      VMOVUPSYmr %1, 1, _, 0, _, %0 :: (store 32 into %ir.p1, align 1)
+# NO_AVX512F-NEXT:      RET 0
+#
+# AVX512F:              %0 = COPY %ymm0
+# AVX512F-NEXT:         %1 = COPY %rdi
+# AVX512F-NEXT:         VMOVUPSZ256mr_NOVLX %1, 1, _, 0, _, %0 :: (store 32 into %ir.p1, align 1)
+# AVX512F-NEXT:         RET 0
+#
+# AVX512VL:             %0 = COPY %ymm0
+# AVX512VL-NEXT:        %1 = COPY %rdi
+# AVX512VL-NEXT:        VMOVUPSZ256mr %1, 1, _, 0, _, %0 :: (store 32 into %ir.p1, align 1)
+# AVX512VL-NEXT:        RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %ymm0
+
+    %0(<8 x s32>) = COPY %ymm0
+    %1(p0) = COPY %rdi
+    G_STORE %0(<8 x s32>), %1(p0) :: (store 32 into %ir.p1, align 1)
+    RET 0
+
+...
+---
+name:            test_store_v8i32_align
+# ALL-LABEL: name:  test_store_v8i32_align
+alignment:       4
+legalized:       true
+regBankSelected: true
+# NO_AVX512F:       registers:
+# NO_AVX512F-NEXT:    - { id: 0, class: vr256 }
+# NO_AVX512F-NEXT:    - { id: 1, class: gr64 }
+#
+# AVX512ALL:        registers:
+# AVX512ALL-NEXT:     - { id: 0, class: vr256x }
+# AVX512ALL-NEXT:     - { id: 1, class: gr64 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: gpr }
+# NO_AVX512F:           %0 = COPY %ymm0
+# NO_AVX512F-NEXT:      %1 = COPY %rdi
+# NO_AVX512F-NEXT:      VMOVAPSYmr %1, 1, _, 0, _, %0 :: (store 32 into %ir.p1)
+# NO_AVX512F-NEXT:      RET 0
+#
+# AVX512F:              %0 = COPY %ymm0
+# AVX512F-NEXT:         %1 = COPY %rdi
+# AVX512F-NEXT:         VMOVAPSZ256mr_NOVLX %1, 1, _, 0, _, %0 :: (store 32 into %ir.p1)
+# AVX512F-NEXT:         RET 0
+#
+# AVX512VL:             %0 = COPY %ymm0
+# AVX512VL-NEXT:        %1 = COPY %rdi
+# AVX512VL-NEXT:        VMOVAPSZ256mr %1, 1, _, 0, _, %0 :: (store 32 into %ir.p1)
+# AVX512VL-NEXT:        RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %ymm0
+
+    %0(<8 x s32>) = COPY %ymm0
+    %1(p0) = COPY %rdi
+    G_STORE %0(<8 x s32>), %1(p0) :: (store 32 into %ir.p1)
+    RET 0
+
+...
diff --git a/llvm/test/CodeGen/X86/GlobalISel/select-memop-v512.mir b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v512.mir
new file mode 100644
index 00000000000..87978a684d4
--- /dev/null
+++ b/llvm/test/CodeGen/X86/GlobalISel/select-memop-v512.mir
@@ -0,0 +1,127 @@
+# RUN: llc -mtriple=x86_64-linux-gnu -mattr=+avx512f -global-isel -run-pass=instruction-select -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=AVX512F
+--- |
+  define <16 x i32> @test_load_v16i32_noalign(<16 x i32>* %p1) {
+    %r = load <16 x i32>, <16 x i32>* %p1, align 1
+    ret <16 x i32> %r
+  }
+
+  define <16 x i32> @test_load_v16i32_align(<16 x i32>* %p1) {
+    %r = load <16 x i32>, <16 x i32>* %p1, align 32
+    ret <16 x i32> %r
+  }
+
+  define void @test_store_v16i32_noalign(<16 x i32> %val, <16 x i32>* %p1) {
+    store <16 x i32> %val, <16 x i32>* %p1, align 1
+    ret void
+  }
+
+  define void @test_store_v16i32_align(<16 x i32> %val, <16 x i32>* %p1) {
+    store <16 x i32> %val, <16 x i32>* %p1, align 32
+    ret void
+  }
+
+...
+---
+name:            test_load_v16i32_noalign
+# AVX512F-LABEL: name:  test_load_v16i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: true
+# AVX512F:      registers:
+# AVX512F-NEXT:   - { id: 0, class: gr64 }
+# AVX512F-NEXT:   - { id: 1, class: vr512 }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: vecr }
+# AVX512F:          %0 = COPY %rdi
+# AVX512F-NEXT:     %1 = VMOVUPSZrm %0, 1, _, 0, _ :: (load 64 from %ir.p1, align 1)
+# AVX512F-NEXT:     %zmm0 = COPY %1
+# AVX512F-NEXT:     RET 0, implicit %zmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi
+
+    %0(p0) = COPY %rdi
+    %1(<16 x s32>) = G_LOAD %0(p0) :: (load 64 from %ir.p1, align 1)
+    %zmm0 = COPY %1(<16 x s32>)
+    RET 0, implicit %zmm0
+
+...
+---
+name:            test_load_v16i32_align
+# AVX512F-LABEL: name:  test_load_v16i32_align
+alignment:       4
+legalized:       true
+regBankSelected: true
+# AVX512F:      registers:
+# AVX512F-NEXT:   - { id: 0, class: gr64 }
+# AVX512F-NEXT:   - { id: 1, class: vr512 }
+registers:
+  - { id: 0, class: gpr }
+  - { id: 1, class: vecr }
+# AVX512F:          %0 = COPY %rdi
+# AVX512F-NEXT:     %1 = VMOVUPSZrm %0, 1, _, 0, _ :: (load 64 from %ir.p1, align 32)
+# AVX512F-NEXT:     %zmm0 = COPY %1
+# AVX512F-NEXT:     RET 0, implicit %zmm0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi
+
+    %0(p0) = COPY %rdi
+    %1(<16 x s32>) = G_LOAD %0(p0) :: (load 64 from %ir.p1, align 32)
+    %zmm0 = COPY %1(<16 x s32>)
+    RET 0, implicit %zmm0
+
+...
+---
+name:            test_store_v16i32_noalign
+# AVX512F-LABEL: name:  test_store_v16i32_noalign
+alignment:       4
+legalized:       true
+regBankSelected: true
+# AVX512F:      registers:
+# AVX512F-NEXT:   - { id: 0, class: vr512 }
+# AVX512F-NEXT:   - { id: 1, class: gr64 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: gpr }
+# AVX512F:          %0 = COPY %zmm0
+# AVX512F-NEXT:     %1 = COPY %rdi
+# AVX512F-NEXT:     VMOVUPSZmr %1, 1, _, 0, _, %0 :: (store 64 into %ir.p1, align 1)
+# AVX512F-NEXT:     RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %zmm0
+
+    %0(<16 x s32>) = COPY %zmm0
+    %1(p0) = COPY %rdi
+    G_STORE %0(<16 x s32>), %1(p0) :: (store 64 into %ir.p1, align 1)
+    RET 0
+
+...
+---
+name:            test_store_v16i32_align
+# AVX512F-LABEL: name:  test_store_v16i32_align
+alignment:       4
+legalized:       true
+regBankSelected: true
+# AVX512F:      registers:
+# AVX512F-NEXT:   - { id: 0, class: vr512 }
+# AVX512F-NEXT:   - { id: 1, class: gr64 }
+registers:
+  - { id: 0, class: vecr }
+  - { id: 1, class: gpr }
+# AVX512F:          %0 = COPY %zmm0
+# AVX512F-NEXT:     %1 = COPY %rdi
+# AVX512F-NEXT:     VMOVUPSZmr %1, 1, _, 0, _, %0 :: (store 64 into %ir.p1, align 32)
+# AVX512F-NEXT:     RET 0
+body:             |
+  bb.1 (%ir-block.0):
+    liveins: %rdi, %zmm0
+
+    %0(<16 x s32>) = COPY %zmm0
+    %1(p0) = COPY %rdi
+    G_STORE %0(<16 x s32>), %1(p0) :: (store 64 into %ir.p1, align 32)
+    RET 0
+
+...