; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi, -mattr=+mve.fp -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-MVE-FP
; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi, -mattr=+mve.fp -fp-contract=fast -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-MVE-VMLA
; RUN: llc -mtriple=thumbv8.1m.main-arm-none-eabi -mattr=+mve,+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefix=CHECK-MVE

define arm_aapcs_vfpcc <8 x half> @vfma16_v1(<8 x half> %src1, <8 x half> %src2, <8 x half> %src3) {
; CHECK-MVE-FP-LABEL: vfma16_v1:
; CHECK-MVE-FP:       @ %bb.0: @ %entry
; CHECK-MVE-FP-NEXT:    vmul.f16 q1, q1, q2
; CHECK-MVE-FP-NEXT:    vadd.f16 q0, q0, q1
; CHECK-MVE-FP-NEXT:    bx lr
;
; CHECK-MVE-VMLA-LABEL: vfma16_v1:
; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
; CHECK-MVE-VMLA-NEXT:    vfma.f16 q0, q1, q2
; CHECK-MVE-VMLA-NEXT:    bx lr
;
; CHECK-MVE-LABEL: vfma16_v1:
; CHECK-MVE:       @ %bb.0: @ %entry
; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
; CHECK-MVE-NEXT:    vmovx.f16 s13, s0
; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
; CHECK-MVE-NEXT:    vmovx.f16 s12, s8
; CHECK-MVE-NEXT:    vmovx.f16 s14, s4
; CHECK-MVE-NEXT:    vmov.f32 s16, s1
; CHECK-MVE-NEXT:    vmla.f16 s13, s14, s12
; CHECK-MVE-NEXT:    vmov r1, s0
; CHECK-MVE-NEXT:    vmla.f16 s16, s5, s9
; CHECK-MVE-NEXT:    vmov r0, s13
; CHECK-MVE-NEXT:    vmov.16 q3[0], r1
; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmovx.f16 s16, s9
; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
; CHECK-MVE-NEXT:    vmovx.f16 s20, s1
; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
; CHECK-MVE-NEXT:    vmov.f32 s16, s2
; CHECK-MVE-NEXT:    vmov r0, s20
; CHECK-MVE-NEXT:    vmla.f16 s16, s6, s10
; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmovx.f16 s16, s10
; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
; CHECK-MVE-NEXT:    vmovx.f16 s20, s2
; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
; CHECK-MVE-NEXT:    vmov.f32 s16, s3
; CHECK-MVE-NEXT:    vmov r0, s20
; CHECK-MVE-NEXT:    vmla.f16 s16, s7, s11
; CHECK-MVE-NEXT:    vmovx.f16 s8, s11
; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
; CHECK-MVE-NEXT:    vmov r0, s0
; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
; CHECK-MVE-NEXT:    vmov q0, q3
; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
; CHECK-MVE-NEXT:    bx lr
entry:
  %0 = fmul <8 x half> %src2, %src3
  %1 = fadd <8 x half> %src1, %0
  ret <8 x half> %1
}

define arm_aapcs_vfpcc <8 x half> @vfma16_v2(<8 x half> %src1, <8 x half> %src2, <8 x half> %src3) {
; CHECK-MVE-FP-LABEL: vfma16_v2:
; CHECK-MVE-FP:       @ %bb.0: @ %entry
; CHECK-MVE-FP-NEXT:    vmul.f16 q1, q1, q2
; CHECK-MVE-FP-NEXT:    vadd.f16 q0, q1, q0
; CHECK-MVE-FP-NEXT:    bx lr
;
; CHECK-MVE-VMLA-LABEL: vfma16_v2:
; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
; CHECK-MVE-VMLA-NEXT:    vfma.f16 q0, q1, q2
; CHECK-MVE-VMLA-NEXT:    bx lr
;
; CHECK-MVE-LABEL: vfma16_v2:
; CHECK-MVE:       @ %bb.0: @ %entry
; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
; CHECK-MVE-NEXT:    vmovx.f16 s13, s0
; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
; CHECK-MVE-NEXT:    vmovx.f16 s12, s8
; CHECK-MVE-NEXT:    vmovx.f16 s14, s4
; CHECK-MVE-NEXT:    vmov.f32 s16, s1
; CHECK-MVE-NEXT:    vmla.f16 s13, s14, s12
; CHECK-MVE-NEXT:    vmov r1, s0
; CHECK-MVE-NEXT:    vmla.f16 s16, s5, s9
; CHECK-MVE-NEXT:    vmov r0, s13
; CHECK-MVE-NEXT:    vmov.16 q3[0], r1
; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmovx.f16 s16, s9
; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
; CHECK-MVE-NEXT:    vmovx.f16 s20, s1
; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
; CHECK-MVE-NEXT:    vmov.f32 s16, s2
; CHECK-MVE-NEXT:    vmov r0, s20
; CHECK-MVE-NEXT:    vmla.f16 s16, s6, s10
; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmovx.f16 s16, s10
; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
; CHECK-MVE-NEXT:    vmovx.f16 s20, s2
; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
; CHECK-MVE-NEXT:    vmla.f16 s20, s18, s16
; CHECK-MVE-NEXT:    vmov.f32 s16, s3
; CHECK-MVE-NEXT:    vmov r0, s20
; CHECK-MVE-NEXT:    vmla.f16 s16, s7, s11
; CHECK-MVE-NEXT:    vmovx.f16 s8, s11
; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmla.f16 s0, s4, s8
; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
; CHECK-MVE-NEXT:    vmov r0, s0
; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
; CHECK-MVE-NEXT:    vmov q0, q3
; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
; CHECK-MVE-NEXT:    bx lr
entry:
  %0 = fmul <8 x half> %src2, %src3
  %1 = fadd <8 x half> %0, %src1
  ret <8 x half> %1
}

define arm_aapcs_vfpcc <8 x half> @vfms16(<8 x half> %src1, <8 x half> %src2, <8 x half> %src3) {
; CHECK-MVE-FP-LABEL: vfms16:
; CHECK-MVE-FP:       @ %bb.0: @ %entry
; CHECK-MVE-FP-NEXT:    vmul.f16 q1, q1, q2
; CHECK-MVE-FP-NEXT:    vsub.f16 q0, q0, q1
; CHECK-MVE-FP-NEXT:    bx lr
;
; CHECK-MVE-VMLA-LABEL: vfms16:
; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
; CHECK-MVE-VMLA-NEXT:    vfms.f16 q0, q1, q2
; CHECK-MVE-VMLA-NEXT:    bx lr
;
; CHECK-MVE-LABEL: vfms16:
; CHECK-MVE:       @ %bb.0: @ %entry
; CHECK-MVE-NEXT:    .vsave {d8, d9, d10}
; CHECK-MVE-NEXT:    vpush {d8, d9, d10}
; CHECK-MVE-NEXT:    vmovx.f16 s13, s0
; CHECK-MVE-NEXT:    vmls.f16 s0, s4, s8
; CHECK-MVE-NEXT:    vmovx.f16 s12, s8
; CHECK-MVE-NEXT:    vmovx.f16 s14, s4
; CHECK-MVE-NEXT:    vmov.f32 s16, s1
; CHECK-MVE-NEXT:    vmls.f16 s13, s14, s12
; CHECK-MVE-NEXT:    vmov r1, s0
; CHECK-MVE-NEXT:    vmls.f16 s16, s5, s9
; CHECK-MVE-NEXT:    vmov r0, s13
; CHECK-MVE-NEXT:    vmov.16 q3[0], r1
; CHECK-MVE-NEXT:    vmov.16 q3[1], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmovx.f16 s16, s9
; CHECK-MVE-NEXT:    vmovx.f16 s18, s5
; CHECK-MVE-NEXT:    vmovx.f16 s20, s1
; CHECK-MVE-NEXT:    vmov.16 q3[2], r0
; CHECK-MVE-NEXT:    vmls.f16 s20, s18, s16
; CHECK-MVE-NEXT:    vmov.f32 s16, s2
; CHECK-MVE-NEXT:    vmov r0, s20
; CHECK-MVE-NEXT:    vmls.f16 s16, s6, s10
; CHECK-MVE-NEXT:    vmov.16 q3[3], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmovx.f16 s16, s10
; CHECK-MVE-NEXT:    vmovx.f16 s18, s6
; CHECK-MVE-NEXT:    vmovx.f16 s20, s2
; CHECK-MVE-NEXT:    vmov.16 q3[4], r0
; CHECK-MVE-NEXT:    vmls.f16 s20, s18, s16
; CHECK-MVE-NEXT:    vmov.f32 s16, s3
; CHECK-MVE-NEXT:    vmov r0, s20
; CHECK-MVE-NEXT:    vmls.f16 s16, s7, s11
; CHECK-MVE-NEXT:    vmovx.f16 s8, s11
; CHECK-MVE-NEXT:    vmovx.f16 s4, s7
; CHECK-MVE-NEXT:    vmovx.f16 s0, s3
; CHECK-MVE-NEXT:    vmov.16 q3[5], r0
; CHECK-MVE-NEXT:    vmov r0, s16
; CHECK-MVE-NEXT:    vmls.f16 s0, s4, s8
; CHECK-MVE-NEXT:    vmov.16 q3[6], r0
; CHECK-MVE-NEXT:    vmov r0, s0
; CHECK-MVE-NEXT:    vmov.16 q3[7], r0
; CHECK-MVE-NEXT:    vmov q0, q3
; CHECK-MVE-NEXT:    vpop {d8, d9, d10}
; CHECK-MVE-NEXT:    bx lr
entry:
  %0 = fmul <8 x half> %src2, %src3
  %1 = fsub <8 x half> %src1, %0
  ret <8 x half> %1
}

define arm_aapcs_vfpcc <4 x float> @vfma32_v1(<4 x float> %src1, <4 x float> %src2, <4 x float> %src3) {
; CHECK-MVE-FP-LABEL: vfma32_v1:
; CHECK-MVE-FP:       @ %bb.0: @ %entry
; CHECK-MVE-FP-NEXT:    vmul.f32 q1, q1, q2
; CHECK-MVE-FP-NEXT:    vadd.f32 q0, q0, q1
; CHECK-MVE-FP-NEXT:    bx lr
;
; CHECK-MVE-VMLA-LABEL: vfma32_v1:
; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
; CHECK-MVE-VMLA-NEXT:    vfma.f32 q0, q1, q2
; CHECK-MVE-VMLA-NEXT:    bx lr
;
; CHECK-MVE-LABEL: vfma32_v1:
; CHECK-MVE:       @ %bb.0: @ %entry
; CHECK-MVE-NEXT:    vmla.f32 s3, s7, s11
; CHECK-MVE-NEXT:    vmla.f32 s2, s6, s10
; CHECK-MVE-NEXT:    vmla.f32 s1, s5, s9
; CHECK-MVE-NEXT:    vmla.f32 s0, s4, s8
; CHECK-MVE-NEXT:    bx lr
entry:
  %0 = fmul <4 x float> %src2, %src3
  %1 = fadd <4 x float> %src1, %0
  ret <4 x float> %1
}

define arm_aapcs_vfpcc <4 x float> @vfma32_v2(<4 x float> %src1, <4 x float> %src2, <4 x float> %src3) {
; CHECK-MVE-FP-LABEL: vfma32_v2:
; CHECK-MVE-FP:       @ %bb.0: @ %entry
; CHECK-MVE-FP-NEXT:    vmul.f32 q1, q1, q2
; CHECK-MVE-FP-NEXT:    vadd.f32 q0, q1, q0
; CHECK-MVE-FP-NEXT:    bx lr
;
; CHECK-MVE-VMLA-LABEL: vfma32_v2:
; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
; CHECK-MVE-VMLA-NEXT:    vfma.f32 q0, q1, q2
; CHECK-MVE-VMLA-NEXT:    bx lr
;
; CHECK-MVE-LABEL: vfma32_v2:
; CHECK-MVE:       @ %bb.0: @ %entry
; CHECK-MVE-NEXT:    vmla.f32 s3, s7, s11
; CHECK-MVE-NEXT:    vmla.f32 s2, s6, s10
; CHECK-MVE-NEXT:    vmla.f32 s1, s5, s9
; CHECK-MVE-NEXT:    vmla.f32 s0, s4, s8
; CHECK-MVE-NEXT:    bx lr
entry:
  %0 = fmul <4 x float> %src2, %src3
  %1 = fadd <4 x float> %0, %src1
  ret <4 x float> %1
}

define arm_aapcs_vfpcc <4 x float> @vfms32(<4 x float> %src1, <4 x float> %src2, <4 x float> %src3) {
; CHECK-MVE-FP-LABEL: vfms32:
; CHECK-MVE-FP:       @ %bb.0: @ %entry
; CHECK-MVE-FP-NEXT:    vmul.f32 q1, q1, q2
; CHECK-MVE-FP-NEXT:    vsub.f32 q0, q0, q1
; CHECK-MVE-FP-NEXT:    bx lr
;
; CHECK-MVE-VMLA-LABEL: vfms32:
; CHECK-MVE-VMLA:       @ %bb.0: @ %entry
; CHECK-MVE-VMLA-NEXT:    vfms.f32 q0, q1, q2
; CHECK-MVE-VMLA-NEXT:    bx lr
;
; CHECK-MVE-LABEL: vfms32:
; CHECK-MVE:       @ %bb.0: @ %entry
; CHECK-MVE-NEXT:    vmls.f32 s3, s7, s11
; CHECK-MVE-NEXT:    vmls.f32 s2, s6, s10
; CHECK-MVE-NEXT:    vmls.f32 s1, s5, s9
; CHECK-MVE-NEXT:    vmls.f32 s0, s4, s8
; CHECK-MVE-NEXT:    bx lr
entry:
  %0 = fmul <4 x float> %src2, %src3
  %1 = fsub <4 x float> %src1, %0
  ret <4 x float> %1
}