[AArch64][SVE] Asm: Support for (SQ)INCP/DECP (scalar, vector)

Increments/decrements the result with the number of active bits
from the predicate.

The inc/dec variants added are:
- incp   x0, p0.h     (scalar)
- incp   z0.h, p0     (vector)

The unsigned saturating inc/dec variants added are:
- uqincp x0, p0.h     (scalar)
- uqincp w0, p0.h     (scalar, 32bit)
- uqincp z0.h, p0     (vector)

The signed saturating inc/dec variants added are:
- sqincp x0, p0.h     (scalar)
- sqincp x0, p0.h, w0 (scalar, 32bit)
- sqincp z0.h, p0     (vector)

llvm-svn: 336091

13 files changed, 763 insertions, 0 deletions

diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index 50a84a0e525..a764ac6543e 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -593,6 +593,24 @@ let Predicates = [HasSVE] in {
   defm INCD_ZPiI   : sve_int_countvlv<0b11100, "incd",   ZPR64>;
   defm DECD_ZPiI   : sve_int_countvlv<0b11101, "decd",   ZPR64>;
 
+  defm SQINCP_XPWd : sve_int_count_r_s32<0b00000, "sqincp">;
+  defm SQINCP_XP   : sve_int_count_r_x64<0b00010, "sqincp">;
+  defm UQINCP_WP   : sve_int_count_r_u32<0b00100, "uqincp">;
+  defm UQINCP_XP   : sve_int_count_r_x64<0b00110, "uqincp">;
+  defm SQDECP_XPWd : sve_int_count_r_s32<0b01000, "sqdecp">;
+  defm SQDECP_XP   : sve_int_count_r_x64<0b01010, "sqdecp">;
+  defm UQDECP_WP   : sve_int_count_r_u32<0b01100, "uqdecp">;
+  defm UQDECP_XP   : sve_int_count_r_x64<0b01110, "uqdecp">;
+  defm INCP_XP     : sve_int_count_r_x64<0b10000, "incp">;
+  defm DECP_XP     : sve_int_count_r_x64<0b10100, "decp">;
+
+  defm SQINCP_ZP   : sve_int_count_v<0b00000, "sqincp">;
+  defm UQINCP_ZP   : sve_int_count_v<0b00100, "uqincp">;
+  defm SQDECP_ZP   : sve_int_count_v<0b01000, "sqdecp">;
+  defm UQDECP_ZP   : sve_int_count_v<0b01100, "uqdecp">;
+  defm INCP_ZP     : sve_int_count_v<0b10000, "incp">;
+  defm DECP_ZP     : sve_int_count_v<0b10100, "decp">;
+
   defm INDEX_RR : sve_int_index_rr<"index">;
   defm INDEX_IR : sve_int_index_ir<"index">;
   defm INDEX_RI : sve_int_index_ri<"index">;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 357ac8c8bc1..3700720fc7c 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -282,6 +282,82 @@ let Predicates = [HasSVE] in {
 
 
 //===----------------------------------------------------------------------===//
+// SVE Predicate Count Group
+//===----------------------------------------------------------------------===//
+
+class sve_int_count_r<bits<2> sz8_64, bits<5> opc, string asm,
+                      RegisterOperand dty, PPRRegOp pprty, RegisterOperand sty>
+: I<(outs dty:$Rdn), (ins pprty:$Pg, sty:$_Rdn),
+  asm, "\t$Rdn, $Pg",
+  "",
+  []>, Sched<[]> {
+  bits<5> Rdn;
+  bits<4> Pg;
+  let Inst{31-24} = 0b00100101;
+  let Inst{23-22} = sz8_64;
+  let Inst{21-19} = 0b101;
+  let Inst{18-16} = opc{4-2};
+  let Inst{15-11} = 0b10001;
+  let Inst{10-9}  = opc{1-0};
+  let Inst{8-5}   = Pg;
+  let Inst{4-0}   = Rdn;
+
+  // Signed 32bit forms require their GPR operand printed.
+  let AsmString = !if(!eq(opc{4,2-0}, 0b0000),
+                      !strconcat(asm, "\t$Rdn, $Pg, $_Rdn"),
+                      !strconcat(asm, "\t$Rdn, $Pg"));
+  let Constraints = "$Rdn = $_Rdn";
+}
+
+multiclass sve_int_count_r_s32<bits<5> opc, string asm> {
+  def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64as32>;
+  def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64as32>;
+  def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64as32>;
+  def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64as32>;
+}
+
+multiclass sve_int_count_r_u32<bits<5> opc, string asm> {
+  def _B : sve_int_count_r<0b00, opc, asm, GPR32z, PPR8, GPR32z>;
+  def _H : sve_int_count_r<0b01, opc, asm, GPR32z, PPR16, GPR32z>;
+  def _S : sve_int_count_r<0b10, opc, asm, GPR32z, PPR32, GPR32z>;
+  def _D : sve_int_count_r<0b11, opc, asm, GPR32z, PPR64, GPR32z>;
+}
+
+multiclass sve_int_count_r_x64<bits<5> opc, string asm> {
+  def _B : sve_int_count_r<0b00, opc, asm, GPR64z, PPR8, GPR64z>;
+  def _H : sve_int_count_r<0b01, opc, asm, GPR64z, PPR16, GPR64z>;
+  def _S : sve_int_count_r<0b10, opc, asm, GPR64z, PPR32, GPR64z>;
+  def _D : sve_int_count_r<0b11, opc, asm, GPR64z, PPR64, GPR64z>;
+}
+
+class sve_int_count_v<bits<2> sz8_64, bits<5> opc, string asm,
+                      ZPRRegOp zprty>
+: I<(outs zprty:$Zdn), (ins zprty:$_Zdn, PPRAny:$Pg),
+  asm, "\t$Zdn, $Pg",
+  "",
+  []>, Sched<[]> {
+  bits<4> Pg;
+  bits<5> Zdn;
+  let Inst{31-24} = 0b00100101;
+  let Inst{23-22} = sz8_64;
+  let Inst{21-19} = 0b101;
+  let Inst{18-16} = opc{4-2};
+  let Inst{15-11} = 0b10000;
+  let Inst{10-9}  = opc{1-0};
+  let Inst{8-5}   = Pg;
+  let Inst{4-0}   = Zdn;
+
+  let Constraints = "$Zdn = $_Zdn";
+}
+
+multiclass sve_int_count_v<bits<5> opc, string asm> {
+  def _H : sve_int_count_v<0b01, opc, asm, ZPR16>;
+  def _S : sve_int_count_v<0b10, opc, asm, ZPR32>;
+  def _D : sve_int_count_v<0b11, opc, asm, ZPR64>;
+}
+
+
+//===----------------------------------------------------------------------===//
 // SVE Element Count Group
 //===----------------------------------------------------------------------===//
 
diff --git a/llvm/test/MC/AArch64/SVE/decp-diagnostics.s b/llvm/test/MC/AArch64/SVE/decp-diagnostics.s
new file mode 100644
index 00000000000..2c8cc429331
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/decp-diagnostics.s
@@ -0,0 +1,38 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve  2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid result register
+
+decp sp, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: decp sp, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+decp z0.b, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: decp z0.b, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate operand
+
+decp x0, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: decp x0, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+decp x0, p0/z
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: decp x0, p0/z
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+decp x0, p0/m
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: decp x0, p0/m
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+decp x0, p0.q
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: decp x0, p0.q
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/decp.s b/llvm/test/MC/AArch64/SVE/decp.s
new file mode 100644
index 00000000000..4866c766aaf
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/decp.s
@@ -0,0 +1,74 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d -mattr=+sve - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+decp    x0, p0.b
+// CHECK-INST: decp    x0, p0.b
+// CHECK-ENCODING: [0x00,0x88,0x2d,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 2d 25 <unknown>
+
+decp    x0, p0.h
+// CHECK-INST: decp    x0, p0.h
+// CHECK-ENCODING: [0x00,0x88,0x6d,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 6d 25 <unknown>
+
+decp    x0, p0.s
+// CHECK-INST: decp    x0, p0.s
+// CHECK-ENCODING: [0x00,0x88,0xad,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 ad 25 <unknown>
+
+decp    x0, p0.d
+// CHECK-INST: decp    x0, p0.d
+// CHECK-ENCODING: [0x00,0x88,0xed,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 ed 25 <unknown>
+
+decp    xzr, p15.b
+// CHECK-INST: decp    xzr, p15.b
+// CHECK-ENCODING: [0xff,0x89,0x2d,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 2d 25 <unknown>
+
+decp    xzr, p15.h
+// CHECK-INST: decp    xzr, p15.h
+// CHECK-ENCODING: [0xff,0x89,0x6d,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 6d 25 <unknown>
+
+decp    xzr, p15.s
+// CHECK-INST: decp    xzr, p15.s
+// CHECK-ENCODING: [0xff,0x89,0xad,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 ad 25 <unknown>
+
+decp    xzr, p15.d
+// CHECK-INST: decp    xzr, p15.d
+// CHECK-ENCODING: [0xff,0x89,0xed,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 ed 25 <unknown>
+
+decp    z31.h, p15
+// CHECK-INST: decp    z31.h, p15
+// CHECK-ENCODING: [0xff,0x81,0x6d,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 81 6d 25 <unknown>
+
+decp    z31.s, p15
+// CHECK-INST: decp    z31.s, p15
+// CHECK-ENCODING: [0xff,0x81,0xad,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 81 ad 25 <unknown>
+
+decp    z31.d, p15
+// CHECK-INST: decp    z31.d, p15
+// CHECK-ENCODING: [0xff,0x81,0xed,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 81 ed 25 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE/incp-diagnostics.s b/llvm/test/MC/AArch64/SVE/incp-diagnostics.s
new file mode 100644
index 00000000000..71cc91f6d02
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/incp-diagnostics.s
@@ -0,0 +1,38 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve  2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid result register
+
+incp sp, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: incp sp, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+incp z0.b, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: incp z0.b, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate operand
+
+incp x0, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: incp x0, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+incp x0, p0/z
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: incp x0, p0/z
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+incp x0, p0/m
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: incp x0, p0/m
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+incp x0, p0.q
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: incp x0, p0.q
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/incp.s b/llvm/test/MC/AArch64/SVE/incp.s
new file mode 100644
index 00000000000..c897e846fd0
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/incp.s
@@ -0,0 +1,74 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d -mattr=+sve - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+incp    x0, p0.b
+// CHECK-INST: incp    x0, p0.b
+// CHECK-ENCODING: [0x00,0x88,0x2c,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 2c 25 <unknown>
+
+incp    x0, p0.h
+// CHECK-INST: incp    x0, p0.h
+// CHECK-ENCODING: [0x00,0x88,0x6c,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 6c 25 <unknown>
+
+incp    x0, p0.s
+// CHECK-INST: incp    x0, p0.s
+// CHECK-ENCODING: [0x00,0x88,0xac,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 ac 25 <unknown>
+
+incp    x0, p0.d
+// CHECK-INST: incp    x0, p0.d
+// CHECK-ENCODING: [0x00,0x88,0xec,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 88 ec 25 <unknown>
+
+incp    xzr, p15.b
+// CHECK-INST: incp    xzr, p15.b
+// CHECK-ENCODING: [0xff,0x89,0x2c,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 2c 25 <unknown>
+
+incp    xzr, p15.h
+// CHECK-INST: incp    xzr, p15.h
+// CHECK-ENCODING: [0xff,0x89,0x6c,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 6c 25 <unknown>
+
+incp    xzr, p15.s
+// CHECK-INST: incp    xzr, p15.s
+// CHECK-ENCODING: [0xff,0x89,0xac,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 ac 25 <unknown>
+
+incp    xzr, p15.d
+// CHECK-INST: incp    xzr, p15.d
+// CHECK-ENCODING: [0xff,0x89,0xec,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 ec 25 <unknown>
+
+incp    z31.h, p15
+// CHECK-INST: incp    z31.h, p15
+// CHECK-ENCODING: [0xff,0x81,0x6c,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 81 6c 25 <unknown>
+
+incp    z31.s, p15
+// CHECK-INST: incp    z31.s, p15
+// CHECK-ENCODING: [0xff,0x81,0xac,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 81 ac 25 <unknown>
+
+incp    z31.d, p15
+// CHECK-INST: incp    z31.d, p15
+// CHECK-ENCODING: [0xff,0x81,0xec,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 81 ec 25 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE/sqdecp-diagnostics.s b/llvm/test/MC/AArch64/SVE/sqdecp-diagnostics.s
new file mode 100644
index 00000000000..f3fca0f17c9
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/sqdecp-diagnostics.s
@@ -0,0 +1,53 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve  2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid result register
+
+sqdecp sp, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: sqdecp sp, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqdecp z0.b, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: sqdecp z0.b, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqdecp w0, p0.b
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: sqdecp w0, p0.b
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqdecp x0, p0.b, x1
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: sqdecp x0, p0.b, x1
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqdecp x0, p0.b, w1
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: operand must be 32-bit form of destination register
+// CHECK-NEXT: sqdecp x0, p0.b, w1
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate operand
+
+sqdecp x0, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: sqdecp x0, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqdecp x0, p0/z
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: sqdecp x0, p0/z
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqdecp x0, p0/m
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: sqdecp x0, p0/m
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+sqdecp x0, p0.q
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: sqdecp x0, p0.q
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/sqdecp.s b/llvm/test/MC/AArch64/SVE/sqdecp.s
new file mode 100644
index 00000000000..6b6abac8621
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/sqdecp.s
@@ -0,0 +1,74 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d -mattr=+sve - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+sqdecp  x0, p0.b
+// CHECK-INST: sqdecp x0, p0.b
+// CHECK-ENCODING: [0x00,0x8c,0x2a,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 2a 25 <unknown>
+
+sqdecp  x0, p0.h
+// CHECK-INST: sqdecp x0, p0.h
+// CHECK-ENCODING: [0x00,0x8c,0x6a,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 6a 25 <unknown>
+
+sqdecp  x0, p0.s
+// CHECK-INST: sqdecp x0, p0.s
+// CHECK-ENCODING: [0x00,0x8c,0xaa,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c aa 25 <unknown>
+
+sqdecp  x0, p0.d
+// CHECK-INST: sqdecp x0, p0.d
+// CHECK-ENCODING: [0x00,0x8c,0xea,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c ea 25 <unknown>
+
+sqdecp  xzr, p15.b, wzr
+// CHECK-INST: sqdecp xzr, p15.b, wzr
+// CHECK-ENCODING: [0xff,0x89,0x2a,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 2a 25 <unknown>
+
+sqdecp  xzr, p15.h, wzr
+// CHECK-INST: sqdecp xzr, p15.h, wzr
+// CHECK-ENCODING: [0xff,0x89,0x6a,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 6a 25 <unknown>
+
+sqdecp  xzr, p15.s, wzr
+// CHECK-INST: sqdecp xzr, p15.s, wzr
+// CHECK-ENCODING: [0xff,0x89,0xaa,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 aa 25 <unknown>
+
+sqdecp  xzr, p15.d, wzr
+// CHECK-INST: sqdecp xzr, p15.d, wzr
+// CHECK-ENCODING: [0xff,0x89,0xea,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 ea 25 <unknown>
+
+sqdecp  z0.h, p0
+// CHECK-INST: sqdecp z0.h, p0
+// CHECK-ENCODING: [0x00,0x80,0x6a,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 6a 25 <unknown>
+
+sqdecp  z0.s, p0
+// CHECK-INST: sqdecp z0.s, p0
+// CHECK-ENCODING: [0x00,0x80,0xaa,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 aa 25 <unknown>
+
+sqdecp  z0.d, p0
+// CHECK-INST: sqdecp z0.d, p0
+// CHECK-ENCODING: [0x00,0x80,0xea,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 ea 25 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE/sqincp-diagnostics.s b/llvm/test/MC/AArch64/SVE/sqincp-diagnostics.s
new file mode 100644
index 00000000000..8b98e62e220
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/sqincp-diagnostics.s
@@ -0,0 +1,48 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve  2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid result register
+
+uqdecp sp, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: uqdecp sp, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqdecp z0.b, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: uqdecp z0.b, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqdecp x0, p0.b, w0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: uqdecp x0, p0.b, w0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqdecp x0, p0.b, x1
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: uqdecp x0, p0.b, x1
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate operand
+
+uqdecp x0, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqdecp x0, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqdecp x0, p0/z
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqdecp x0, p0/z
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqdecp x0, p0/m
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqdecp x0, p0/m
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqdecp x0, p0.q
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqdecp x0, p0.q
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/sqincp.s b/llvm/test/MC/AArch64/SVE/sqincp.s
new file mode 100644
index 00000000000..6f812947df2
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/sqincp.s
@@ -0,0 +1,74 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d -mattr=+sve - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+sqincp  x0, p0.b
+// CHECK-INST: sqincp  x0, p0.b
+// CHECK-ENCODING: [0x00,0x8c,0x28,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 28 25 <unknown>
+
+sqincp  x0, p0.h
+// CHECK-INST: sqincp  x0, p0.h
+// CHECK-ENCODING: [0x00,0x8c,0x68,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 68 25 <unknown>
+
+sqincp  x0, p0.s
+// CHECK-INST: sqincp  x0, p0.s
+// CHECK-ENCODING: [0x00,0x8c,0xa8,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c a8 25 <unknown>
+
+sqincp  x0, p0.d
+// CHECK-INST: sqincp  x0, p0.d
+// CHECK-ENCODING: [0x00,0x8c,0xe8,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c e8 25 <unknown>
+
+sqincp  xzr, p15.b, wzr
+// CHECK-INST: sqincp  xzr, p15.b, wzr
+// CHECK-ENCODING: [0xff,0x89,0x28,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 28 25 <unknown>
+
+sqincp  xzr, p15.h, wzr
+// CHECK-INST: sqincp  xzr, p15.h, wzr
+// CHECK-ENCODING: [0xff,0x89,0x68,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 68 25 <unknown>
+
+sqincp  xzr, p15.s, wzr
+// CHECK-INST: sqincp  xzr, p15.s, wzr
+// CHECK-ENCODING: [0xff,0x89,0xa8,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 a8 25 <unknown>
+
+sqincp  xzr, p15.d, wzr
+// CHECK-INST: sqincp  xzr, p15.d, wzr
+// CHECK-ENCODING: [0xff,0x89,0xe8,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 e8 25 <unknown>
+
+sqincp  z0.h, p0
+// CHECK-INST: sqincp  z0.h, p0
+// CHECK-ENCODING: [0x00,0x80,0x68,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 68 25 <unknown>
+
+sqincp  z0.s, p0
+// CHECK-INST: sqincp  z0.s, p0
+// CHECK-ENCODING: [0x00,0x80,0xa8,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 a8 25 <unknown>
+
+sqincp  z0.d, p0
+// CHECK-INST: sqincp  z0.d, p0
+// CHECK-ENCODING: [0x00,0x80,0xe8,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 e8 25 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE/uqdecp.s b/llvm/test/MC/AArch64/SVE/uqdecp.s
new file mode 100644
index 00000000000..4f9a7af45d5
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/uqdecp.s
@@ -0,0 +1,74 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d -mattr=+sve - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+uqdecp  x0, p0.b
+// CHECK-INST: uqdecp x0, p0.b
+// CHECK-ENCODING: [0x00,0x8c,0x2b,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 2b 25 <unknown>
+
+uqdecp  x0, p0.h
+// CHECK-INST: uqdecp x0, p0.h
+// CHECK-ENCODING: [0x00,0x8c,0x6b,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 6b 25 <unknown>
+
+uqdecp  x0, p0.s
+// CHECK-INST: uqdecp x0, p0.s
+// CHECK-ENCODING: [0x00,0x8c,0xab,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c ab 25 <unknown>
+
+uqdecp  x0, p0.d
+// CHECK-INST: uqdecp x0, p0.d
+// CHECK-ENCODING: [0x00,0x8c,0xeb,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c eb 25 <unknown>
+
+uqdecp  wzr, p15.b
+// CHECK-INST: uqdecp wzr, p15.b
+// CHECK-ENCODING: [0xff,0x89,0x2b,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 2b 25 <unknown>
+
+uqdecp  wzr, p15.h
+// CHECK-INST: uqdecp wzr, p15.h
+// CHECK-ENCODING: [0xff,0x89,0x6b,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 6b 25 <unknown>
+
+uqdecp  wzr, p15.s
+// CHECK-INST: uqdecp wzr, p15.s
+// CHECK-ENCODING: [0xff,0x89,0xab,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 ab 25 <unknown>
+
+uqdecp  wzr, p15.d
+// CHECK-INST: uqdecp wzr, p15.d
+// CHECK-ENCODING: [0xff,0x89,0xeb,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 eb 25 <unknown>
+
+uqdecp  z0.h, p0
+// CHECK-INST: uqdecp z0.h, p0
+// CHECK-ENCODING: [0x00,0x80,0x6b,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 6b 25 <unknown>
+
+uqdecp  z0.s, p0
+// CHECK-INST: uqdecp z0.s, p0
+// CHECK-ENCODING: [0x00,0x80,0xab,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 ab 25 <unknown>
+
+uqdecp  z0.d, p0
+// CHECK-INST: uqdecp z0.d, p0
+// CHECK-ENCODING: [0x00,0x80,0xeb,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 eb 25 <unknown>
diff --git a/llvm/test/MC/AArch64/SVE/uqincp-diagnostics.s b/llvm/test/MC/AArch64/SVE/uqincp-diagnostics.s
new file mode 100644
index 00000000000..8a4aa25ac41
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/uqincp-diagnostics.s
@@ -0,0 +1,48 @@
+// RUN: not llvm-mc -triple=aarch64 -show-encoding -mattr=+sve  2>&1 < %s| FileCheck %s
+
+// ------------------------------------------------------------------------- //
+// Invalid result register
+
+uqincp sp, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: uqincp sp, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqincp z0.b, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid element width
+// CHECK-NEXT: uqincp z0.b, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqincp x0, p0.b, w0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: uqincp x0, p0.b, w0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqincp x0, p0.b, x1
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid operand
+// CHECK-NEXT: uqincp x0, p0.b, x1
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+
+// ------------------------------------------------------------------------- //
+// Invalid predicate operand
+
+uqincp x0, p0
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqincp x0, p0
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqincp x0, p0/z
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqincp x0, p0/z
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqincp x0, p0/m
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqincp x0, p0/m
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
+
+uqincp x0, p0.q
+// CHECK: [[@LINE-1]]:{{[0-9]+}}: error: invalid predicate register
+// CHECK-NEXT: uqincp x0, p0.q
+// CHECK-NOT: [[@LINE-1]]:{{[0-9]+}}:
diff --git a/llvm/test/MC/AArch64/SVE/uqincp.s b/llvm/test/MC/AArch64/SVE/uqincp.s
new file mode 100644
index 00000000000..e7c87efa4e0
--- /dev/null
+++ b/llvm/test/MC/AArch64/SVE/uqincp.s
@@ -0,0 +1,74 @@
+// RUN: llvm-mc -triple=aarch64 -show-encoding -mattr=+sve < %s \
+// RUN:        | FileCheck %s --check-prefixes=CHECK-ENCODING,CHECK-INST
+// RUN: not llvm-mc -triple=aarch64 -show-encoding < %s 2>&1 \
+// RUN:        | FileCheck %s --check-prefix=CHECK-ERROR
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d -mattr=+sve - | FileCheck %s --check-prefix=CHECK-INST
+// RUN: llvm-mc -triple=aarch64 -filetype=obj -mattr=+sve < %s \
+// RUN:        | llvm-objdump -d - | FileCheck %s --check-prefix=CHECK-UNKNOWN
+
+uqincp  x0, p0.b
+// CHECK-INST: uqincp  x0, p0.b
+// CHECK-ENCODING: [0x00,0x8c,0x29,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 29 25 <unknown>
+
+uqincp  x0, p0.h
+// CHECK-INST: uqincp  x0, p0.h
+// CHECK-ENCODING: [0x00,0x8c,0x69,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c 69 25 <unknown>
+
+uqincp  x0, p0.s
+// CHECK-INST: uqincp  x0, p0.s
+// CHECK-ENCODING: [0x00,0x8c,0xa9,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c a9 25 <unknown>
+
+uqincp  x0, p0.d
+// CHECK-INST: uqincp  x0, p0.d
+// CHECK-ENCODING: [0x00,0x8c,0xe9,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 8c e9 25 <unknown>
+
+uqincp  wzr, p15.b
+// CHECK-INST: uqincp  wzr, p15.b
+// CHECK-ENCODING: [0xff,0x89,0x29,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 29 25 <unknown>
+
+uqincp  wzr, p15.h
+// CHECK-INST: uqincp  wzr, p15.h
+// CHECK-ENCODING: [0xff,0x89,0x69,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 69 25 <unknown>
+
+uqincp  wzr, p15.s
+// CHECK-INST: uqincp  wzr, p15.s
+// CHECK-ENCODING: [0xff,0x89,0xa9,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 a9 25 <unknown>
+
+uqincp  wzr, p15.d
+// CHECK-INST: uqincp  wzr, p15.d
+// CHECK-ENCODING: [0xff,0x89,0xe9,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: ff 89 e9 25 <unknown>
+
+uqincp  z0.h, p0
+// CHECK-INST: uqincp  z0.h, p0
+// CHECK-ENCODING: [0x00,0x80,0x69,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 69 25 <unknown>
+
+uqincp  z0.s, p0
+// CHECK-INST: uqincp  z0.s, p0
+// CHECK-ENCODING: [0x00,0x80,0xa9,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 a9 25 <unknown>
+
+uqincp  z0.d, p0
+// CHECK-INST: uqincp  z0.d, p0
+// CHECK-ENCODING: [0x00,0x80,0xe9,0x25]
+// CHECK-ERROR: instruction requires: sve
+// CHECK-UNKNOWN: 00 80 e9 25 <unknown>