7 files changed, 181 insertions, 70 deletions
diff --git a/llvm/test/CodeGen/X86/avx512cd-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512cd-intrinsics-fast-isel.ll
new file mode 100644
index 00000000000..ca5e5523a9d
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512cd-intrinsics-fast-isel.ll
@@ -0,0 +1,37 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512cd | FileCheck %s
+
+define <8 x i64> @test_mm512_broadcastmb_epi64(<8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: test_mm512_broadcastmb_epi64:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    vpcmpeqq %zmm1, %zmm0, %k0
+; CHECK-NEXT:    vpbroadcastmb2q %k0, %zmm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = icmp eq <8 x i64> %a, %b
+  %1 = bitcast <8 x i1> %0 to i8
+  %conv.i = zext i8 %1 to i64
+  %vecinit.i.i = insertelement <8 x i64> undef, i64 %conv.i, i32 0
+  %vecinit7.i.i = shufflevector <8 x i64> %vecinit.i.i, <8 x i64> undef, <8 x i32> zeroinitializer
+  ret <8 x i64> %vecinit7.i.i
+}
+
+define <8 x i64> @test_mm512_broadcastmw_epi32(<8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: test_mm512_broadcastmw_epi32:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; CHECK-NEXT:    vpbroadcastmw2d %k0, %zmm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %a to <16 x i32>
+  %1 = bitcast <8 x i64> %b to <16 x i32>
+  %2 = icmp eq <16 x i32> %0, %1
+  %3 = bitcast <16 x i1> %2 to i16
+  %conv.i = zext i16 %3 to i32
+  %vecinit.i.i = insertelement <16 x i32> undef, i32 %conv.i, i32 0
+  %vecinit15.i.i = shufflevector <16 x i32> %vecinit.i.i, <16 x i32> undef, <16 x i32> zeroinitializer
+  %4 = bitcast <16 x i32> %vecinit15.i.i to <8 x i64>
+  ret <8 x i64> %4
+}
+
+
diff --git a/llvm/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll
index e5dbff9ac51..92dfe1e087a 100644
--- a/llvm/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512cd-intrinsics-upgrade.ll
@@ -45,3 +45,26 @@ define <8 x i64> @test_mask_lzcnt_q(<8 x i64> %a, <8 x i64> %b, i8 %mask) {
   %res = call <8 x i64> @llvm.x86.avx512.mask.lzcnt.q.512(<8 x i64> %a, <8 x i64> %b, i8 %mask)
   ret <8 x i64> %res
 }
+
+define <16 x i32> @test_x86_vbroadcastmw_512(i16 %a0) {
+; CHECK-LABEL: test_x86_vbroadcastmw_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzwl %di, %eax
+; CHECK-NEXT:    vpbroadcastd %eax, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <16 x i32> @llvm.x86.avx512.broadcastmw.512(i16 %a0)
+  ret <16 x i32> %res
+}
+declare <16 x i32> @llvm.x86.avx512.broadcastmw.512(i16)
+
+define <8 x i64> @test_x86_broadcastmb_512(i8 %a0) {
+; CHECK-LABEL: test_x86_broadcastmb_512:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    vpbroadcastq %rax, %zmm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i64> @llvm.x86.avx512.broadcastmb.512(i8 %a0)
+  ret <8 x i64> %res
+}
+declare <8 x i64> @llvm.x86.avx512.broadcastmb.512(i8)
+
diff --git a/llvm/test/CodeGen/X86/avx512cd-intrinsics.ll b/llvm/test/CodeGen/X86/avx512cd-intrinsics.ll
index 7e5a3e8fe25..ab8c80f8dd3 100644
--- a/llvm/test/CodeGen/X86/avx512cd-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512cd-intrinsics.ll
@@ -1,28 +1,6 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=x86_64-apple-darwin -mcpu=knl -mattr=+avx512cd | FileCheck %s
 
-define <16 x i32> @test_x86_vbroadcastmw_512(i16 %a0) {
-; CHECK-LABEL: test_x86_vbroadcastmw_512:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k0
-; CHECK-NEXT:    vpbroadcastmw2d %k0, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <16 x i32> @llvm.x86.avx512.broadcastmw.512(i16 %a0)
-  ret <16 x i32> %res
-}
-declare <16 x i32> @llvm.x86.avx512.broadcastmw.512(i16)
-
-define <8 x i64> @test_x86_broadcastmb_512(i8 %a0) {
-; CHECK-LABEL: test_x86_broadcastmb_512:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k0
-; CHECK-NEXT:    vpbroadcastmb2q %k0, %zmm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i64> @llvm.x86.avx512.broadcastmb.512(i8 %a0)
-  ret <8 x i64> %res
-}
-declare <8 x i64> @llvm.x86.avx512.broadcastmb.512(i8)
-
 declare <16 x i32> @llvm.x86.avx512.mask.conflict.d.512(<16 x i32>, <16 x i32>, i16) nounwind readonly
 
 define <8 x i64> @test_conflict_q(<8 x i64> %a) {
diff --git a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
index f8f47c87100..0e310be3489 100644
--- a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
+++ b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics-upgrade.ll
@@ -69,3 +69,47 @@ define <4 x i64>@test_int_x86_avx512_mask_vplzcnt_q_256(<4 x i64> %x0, <4 x i64>
   ret <4 x i64> %res2
 }
 
+define <8 x i32> @test_x86_vbroadcastmw_256(i16 %a0) {
+; CHECK-LABEL: test_x86_vbroadcastmw_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzwl %di, %eax
+; CHECK-NEXT:    vpbroadcastd %eax, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <8 x i32> @llvm.x86.avx512.broadcastmw.256(i16 %a0) ;
+  ret <8 x i32> %res
+}
+declare <8 x i32> @llvm.x86.avx512.broadcastmw.256(i16)
+
+define <4 x i32> @test_x86_vbroadcastmw_128(i16 %a0) {
+; CHECK-LABEL: test_x86_vbroadcastmw_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzwl %di, %eax
+; CHECK-NEXT:    vpbroadcastd %eax, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i32> @llvm.x86.avx512.broadcastmw.128(i16 %a0) ;
+  ret <4 x i32> %res
+}
+declare <4 x i32> @llvm.x86.avx512.broadcastmw.128(i16)
+
+define <4 x i64> @test_x86_broadcastmb_256(i8 %a0) {
+; CHECK-LABEL: test_x86_broadcastmb_256:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    vpbroadcastq %rax, %ymm0
+; CHECK-NEXT:    retq
+  %res = call <4 x i64> @llvm.x86.avx512.broadcastmb.256(i8 %a0) ;
+  ret <4 x i64> %res
+}
+declare <4 x i64> @llvm.x86.avx512.broadcastmb.256(i8)
+
+define <2 x i64> @test_x86_broadcastmb_128(i8 %a0) {
+; CHECK-LABEL: test_x86_broadcastmb_128:
+; CHECK:       ## BB#0:
+; CHECK-NEXT:    movzbl %dil, %eax
+; CHECK-NEXT:    vpbroadcastq %rax, %xmm0
+; CHECK-NEXT:    retq
+  %res = call <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8 %a0) ;
+  ret <2 x i64> %res
+}
+declare <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8)
+
diff --git a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll
index 96254f7c95b..2fb50297c62 100644
--- a/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll
+++ b/llvm/test/CodeGen/X86/avx512cdvl-intrinsics.ll
@@ -147,46 +147,3 @@ define <4 x i64>@test_int_x86_avx512_mask_vpconflict_q_256(<4 x i64> %x0, <4 x i
   ret <4 x i64> %res2
 }
 
-define <8 x i32> @test_x86_vbroadcastmw_256(i16 %a0) {
-; CHECK-LABEL: test_x86_vbroadcastmw_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k0
-; CHECK-NEXT:    vpbroadcastmw2d %k0, %ymm0
-; CHECK-NEXT:    retq
-  %res = call <8 x i32> @llvm.x86.avx512.broadcastmw.256(i16 %a0) ;
-  ret <8 x i32> %res
-}
-declare <8 x i32> @llvm.x86.avx512.broadcastmw.256(i16)
-
-define <4 x i32> @test_x86_vbroadcastmw_128(i16 %a0) {
-; CHECK-LABEL: test_x86_vbroadcastmw_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k0
-; CHECK-NEXT:    vpbroadcastmw2d %k0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <4 x i32> @llvm.x86.avx512.broadcastmw.128(i16 %a0) ;
-  ret <4 x i32> %res
-}
-declare <4 x i32> @llvm.x86.avx512.broadcastmw.128(i16)
-
-define <4 x i64> @test_x86_broadcastmb_256(i8 %a0) {
-; CHECK-LABEL: test_x86_broadcastmb_256:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k0
-; CHECK-NEXT:    vpbroadcastmb2q %k0, %ymm0
-; CHECK-NEXT:    retq
-  %res = call <4 x i64> @llvm.x86.avx512.broadcastmb.256(i8 %a0) ;
-  ret <4 x i64> %res
-}
-declare <4 x i64> @llvm.x86.avx512.broadcastmb.256(i8)
-
-define <2 x i64> @test_x86_broadcastmb_128(i8 %a0) {
-; CHECK-LABEL: test_x86_broadcastmb_128:
-; CHECK:       ## BB#0:
-; CHECK-NEXT:    kmovw %edi, %k0
-; CHECK-NEXT:    vpbroadcastmb2q %k0, %xmm0
-; CHECK-NEXT:    retq
-  %res = call <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8 %a0) ;
-  ret <2 x i64> %res
-}
-declare <2 x i64> @llvm.x86.avx512.broadcastmb.128(i8)
diff --git a/llvm/test/CodeGen/X86/avx512vlcd-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512vlcd-intrinsics-fast-isel.ll
new file mode 100644
index 00000000000..ab4cbeb8d5e
--- /dev/null
+++ b/llvm/test/CodeGen/X86/avx512vlcd-intrinsics-fast-isel.ll
@@ -0,0 +1,75 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -fast-isel -mtriple=x86_64-unknown-unknown -mattr=+avx512cd,+avx512vl | FileCheck %s
+
+define <2 x i64> @test_mm_broadcastmb_epi64(<2 x i64> %a, <2 x i64> %b) {
+; CHECK-LABEL: test_mm_broadcastmb_epi64:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    vpcmpeqd %xmm1, %xmm0, %k0
+; CHECK-NEXT:    vpbroadcastmb2q %k0, %xmm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = bitcast <2 x i64> %a to <4 x i32>
+  %1 = bitcast <2 x i64> %b to <4 x i32>
+  %2 = icmp eq <4 x i32> %0, %1
+  %3 = shufflevector <4 x i1> %2, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %4 = bitcast <8 x i1> %3 to i8
+  %conv.i = zext i8 %4 to i64
+  %vecinit.i.i = insertelement <2 x i64> undef, i64 %conv.i, i32 0
+  %vecinit1.i.i = shufflevector <2 x i64> %vecinit.i.i, <2 x i64> undef, <2 x i32> zeroinitializer
+  ret <2 x i64> %vecinit1.i.i
+}
+
+define <4 x i64> @test_mm256_broadcastmb_epi64(<4 x i64> %a, <4 x i64> %b) {
+; CHECK-LABEL: test_mm256_broadcastmb_epi64:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    vpcmpeqq %ymm1, %ymm0, %k0
+; CHECK-NEXT:    vpbroadcastmb2q %k0, %ymm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = icmp eq <4 x i64> %a, %b
+  %1 = shufflevector <4 x i1> %0, <4 x i1> zeroinitializer, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+  %2 = bitcast <8 x i1> %1 to i8
+  %conv.i = zext i8 %2 to i64
+  %vecinit.i.i = insertelement <4 x i64> undef, i64 %conv.i, i32 0
+  %vecinit3.i.i = shufflevector <4 x i64> %vecinit.i.i, <4 x i64> undef, <4 x i32> zeroinitializer
+  ret <4 x i64> %vecinit3.i.i
+}
+
+define <2 x i64> @test_mm_broadcastmw_epi32(<8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: test_mm_broadcastmw_epi32:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; CHECK-NEXT:    vpbroadcastmw2d %k0, %xmm0
+; CHECK-NEXT:    vzeroupper
+; CHECK-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %a to <16 x i32>
+  %1 = bitcast <8 x i64> %b to <16 x i32>
+  %2 = icmp eq <16 x i32> %0, %1
+  %3 = bitcast <16 x i1> %2 to i16
+  %conv.i = zext i16 %3 to i32
+  %vecinit.i.i = insertelement <4 x i32> undef, i32 %conv.i, i32 0
+  %vecinit3.i.i = shufflevector <4 x i32> %vecinit.i.i, <4 x i32> undef, <4 x i32> zeroinitializer
+  %4 = bitcast <4 x i32> %vecinit3.i.i to <2 x i64>
+  ret <2 x i64> %4
+}
+
+define <4 x i64> @test_mm256_broadcastmw_epi32(<8 x i64> %a, <8 x i64> %b) {
+; CHECK-LABEL: test_mm256_broadcastmw_epi32:
+; CHECK:       # BB#0: # %entry
+; CHECK-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
+; CHECK-NEXT:    vpbroadcastmw2d %k0, %ymm0
+; CHECK-NEXT:    retq
+entry:
+  %0 = bitcast <8 x i64> %a to <16 x i32>
+  %1 = bitcast <8 x i64> %b to <16 x i32>
+  %2 = icmp eq <16 x i32> %0, %1
+  %3 = bitcast <16 x i1> %2 to i16
+  %conv.i = zext i16 %3 to i32
+  %vecinit.i.i = insertelement <8 x i32> undef, i32 %conv.i, i32 0
+  %vecinit7.i.i = shufflevector <8 x i32> %vecinit.i.i, <8 x i32> undef, <8 x i32> zeroinitializer
+  %4 = bitcast <8 x i32> %vecinit7.i.i to <4 x i64>
+  ret <4 x i64> %4
+}
+
+
diff --git a/llvm/test/CodeGen/X86/broadcastm-lowering.ll b/llvm/test/CodeGen/X86/broadcastm-lowering.ll
index 2a8236cf093..fc7b192c2f8 100644
--- a/llvm/test/CodeGen/X86/broadcastm-lowering.ll
+++ b/llvm/test/CodeGen/X86/broadcastm-lowering.ll
@@ -80,8 +80,7 @@ define <16 x i32> @test_mm512_epi32(<16 x i32> %a, <16 x i32> %b) {
 ; AVX512CD-LABEL: test_mm512_epi32:
 ; AVX512CD:       # BB#0: # %entry
 ; AVX512CD-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
-; AVX512CD-NEXT:    kmovw %k0, %eax
-; AVX512CD-NEXT:    vpbroadcastd %eax, %zmm0
+; AVX512CD-NEXT:    vpbroadcastmw2d %k0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512VLCDBW-LABEL: test_mm512_epi32:
@@ -110,9 +109,7 @@ define <8 x i64> @test_mm512_epi64(<8 x i32> %a, <8 x i32> %b) {
 ; AVX512CD-NEXT:    # kill: %YMM1<def> %YMM1<kill> %ZMM1<def>
 ; AVX512CD-NEXT:    # kill: %YMM0<def> %YMM0<kill> %ZMM0<def>
 ; AVX512CD-NEXT:    vpcmpeqd %zmm1, %zmm0, %k0
-; AVX512CD-NEXT:    kmovw %k0, %eax
-; AVX512CD-NEXT:    movzbl %al, %eax
-; AVX512CD-NEXT:    vpbroadcastq %rax, %zmm0
+; AVX512CD-NEXT:    vpbroadcastmb2q %k0, %zmm0
 ; AVX512CD-NEXT:    retq
 ;
 ; AVX512VLCDBW-LABEL: test_mm512_epi64: