GlobalISel: Enable the legalization of G_MERGE_VALUES and G_UNMERGE_VALUES

Summary: LegalizerInfo assumes all G_MERGE_VALUES and G_UNMERGE_VALUES instructions are legal, so it is not possible to legalize vector operations on illegal vector types. This patch fixes the problem by removing the related check and adding default actions for G_MERGE_VALUES and G_UNMERGE_VALUES.

Reviewers: qcolombet, ab, dsanders, aditya_nandakumar, t.p.northover, kristof.beyls

Reviewed By: dsanders

Subscribers: rovka, javed.absar, igorb, llvm-commits

Differential Revision: https://reviews.llvm.org/D39823

llvm-svn: 319524

10 files changed, 193 insertions, 53 deletions

diff --git a/llvm/include/llvm/Target/GenericOpcodes.td b/llvm/include/llvm/Target/GenericOpcodes.td
index 96244d85278..489b569d990 100644
--- a/llvm/include/llvm/Target/GenericOpcodes.td
+++ b/llvm/include/llvm/Target/GenericOpcodes.td
@@ -607,8 +607,8 @@ def G_EXTRACT : Instruction {
 // indexes. This will almost certainly be mapped to sub-register COPYs after
 // register banks have been selected.
 def G_UNMERGE_VALUES : Instruction {
-  let OutOperandList = (outs);
-  let InOperandList = (ins variable_ops);
+  let OutOperandList = (outs type0:$dst0, variable_ops);
+  let InOperandList = (ins type1:$src);
   let hasSideEffects = 0;
 }
 
@@ -619,10 +619,10 @@ def G_INSERT : Instruction {
   let hasSideEffects = 0;
 }
 
-/// Concatenante multiple registers of the same size into a wider register.
+/// Concatenate multiple registers of the same size into a wider register.
 def G_MERGE_VALUES : Instruction {
   let OutOperandList = (outs type0:$dst);
-  let InOperandList = (ins variable_ops);
+  let InOperandList = (ins type1:$src0, variable_ops);
   let hasSideEffects = 0;
 }
 
diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
index 1a23b26e7ce..9c27c59a065 100644
--- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
+++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp
@@ -167,19 +167,25 @@ LegalizerInfo::getAction(const InstrAspect &Aspect) const {
   assert(TablesInitialized && "backend forgot to call computeTables");
   // These *have* to be implemented for now, they're the fundamental basis of
   // how everything else is transformed.
-
-  // FIXME: the long-term plan calls for expansion in terms of load/store (if
-  // they're not legal).
-  if (Aspect.Opcode == TargetOpcode::G_MERGE_VALUES ||
-      Aspect.Opcode == TargetOpcode::G_UNMERGE_VALUES)
-    return std::make_pair(Legal, Aspect.Type);
-
   if (Aspect.Type.isScalar() || Aspect.Type.isPointer())
     return findScalarLegalAction(Aspect);
   assert(Aspect.Type.isVector());
   return findVectorLegalAction(Aspect);
 }
 
+/// Helper function to get LLT for the given type index.
+static LLT getTypeFromTypeIdx(const MachineInstr &MI,
+                              const MachineRegisterInfo &MRI, unsigned OpIdx,
+                              unsigned TypeIdx) {
+  assert(TypeIdx < MI.getNumOperands() && "Unexpected TypeIdx");
+  // G_UNMERGE_VALUES has variable number of operands, but there is only
+  // one source type and one destination type as all destinations must be the
+  // same type. So, get the last operand if TypeIdx == 1.
+  if (MI.getOpcode() == TargetOpcode::G_UNMERGE_VALUES && TypeIdx == 1)
+    return MRI.getType(MI.getOperand(MI.getNumOperands() - 1).getReg());
+  return MRI.getType(MI.getOperand(OpIdx).getReg());
+}
+
 std::tuple<LegalizerInfo::LegalizeAction, unsigned, LLT>
 LegalizerInfo::getAction(const MachineInstr &MI,
                          const MachineRegisterInfo &MRI) const {
@@ -198,7 +204,7 @@ LegalizerInfo::getAction(const MachineInstr &MI,
 
     SeenTypes.set(TypeIdx);
 
-    LLT Ty = MRI.getType(MI.getOperand(i).getReg());
+    LLT Ty = getTypeFromTypeIdx(MI, MRI, i, TypeIdx);
     auto Action = getAction({MI.getOpcode(), TypeIdx, Ty});
     if (Action.first != Legal)
       return std::make_tuple(Action.first, TypeIdx, Action.second);
diff --git a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
index c7a42629c96..05df5120222 100644
--- a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp
@@ -368,6 +368,23 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) {
     }
   }
 
+  // Merge/Unmerge
+  for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES})
+    for (int Sz : {8, 16, 32, 64, 128, 192, 256, 384, 512}) {
+      LLT ScalarTy = LLT::scalar(Sz);
+      setAction({Op, ScalarTy}, Legal);
+      setAction({Op, 1, ScalarTy}, Legal);
+      if (Sz < 32)
+        continue;
+      for (int EltSize = 8; EltSize <= 64; EltSize *= 2) {
+        if (EltSize >= Sz)
+          continue;
+        LLT VecTy = LLT::vector(Sz / EltSize, EltSize);
+        setAction({Op, VecTy}, Legal);
+        setAction({Op, 1, VecTy}, Legal);
+      }
+    }
+
   computeTables();
 }
 
diff --git a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
index 51eae325c95..6a541991adb 100644
--- a/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
+++ b/llvm/lib/Target/ARM/ARMLegalizerInfo.cpp
@@ -177,6 +177,15 @@ ARMLegalizerInfo::ARMLegalizerInfo(const ARMSubtarget &ST) {
     for (auto Ty : {s32, s64})
       setAction({Op, Ty}, Libcall);
 
+  // Merge/Unmerge
+  for (const auto &Ty : {s32, s64}) {
+    setAction({G_MERGE_VALUES, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
+  }
+  for (const auto &Ty : {s16, s32}) {
+    setAction({G_MERGE_VALUES, 1, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, Ty}, Legal);
+  }
   computeTables();
 }
 
diff --git a/llvm/lib/Target/X86/X86LegalizerInfo.cpp b/llvm/lib/Target/X86/X86LegalizerInfo.cpp
index 4c86373f6f8..4108a58fa7a 100644
--- a/llvm/lib/Target/X86/X86LegalizerInfo.cpp
+++ b/llvm/lib/Target/X86/X86LegalizerInfo.cpp
@@ -90,6 +90,7 @@ void X86LegalizerInfo::setLegalizerInfo32bit() {
   const LLT s8 = LLT::scalar(8);
   const LLT s16 = LLT::scalar(16);
   const LLT s32 = LLT::scalar(32);
+  const LLT s64 = LLT::scalar(64);
 
   for (auto Ty : {p0, s1, s8, s16, s32})
     setAction({G_IMPLICIT_DEF, Ty}, Legal);
@@ -140,6 +141,16 @@ void X86LegalizerInfo::setLegalizerInfo32bit() {
 
   for (auto Ty : {s8, s16, s32, p0})
     setAction({G_ICMP, 1, Ty}, Legal);
+
+  // Merge/Unmerge
+  for (const auto &Ty : {s16, s32, s64}) {
+    setAction({G_MERGE_VALUES, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
+  }
+  for (const auto &Ty : {s8, s16, s32}) {
+    setAction({G_MERGE_VALUES, 1, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, Ty}, Legal);
+  }
 }
 
 void X86LegalizerInfo::setLegalizerInfo64bit() {
@@ -148,6 +159,7 @@ void X86LegalizerInfo::setLegalizerInfo64bit() {
     return;
 
   const LLT s64 = LLT::scalar(64);
+  const LLT s128 = LLT::scalar(128);
 
   setAction({G_IMPLICIT_DEF, s64}, Legal);
 
@@ -172,6 +184,12 @@ void X86LegalizerInfo::setLegalizerInfo64bit() {
 
   // Comparison
   setAction({G_ICMP, 1, s64}, Legal);
+
+  // Merge/Unmerge
+  setAction({G_MERGE_VALUES, s128}, Legal);
+  setAction({G_UNMERGE_VALUES, 1, s128}, Legal);
+  setAction({G_MERGE_VALUES, 1, s128}, Legal);
+  setAction({G_UNMERGE_VALUES, s128}, Legal);
 }
 
 void X86LegalizerInfo::setLegalizerInfoSSE1() {
@@ -179,6 +197,7 @@ void X86LegalizerInfo::setLegalizerInfoSSE1() {
     return;
 
   const LLT s32 = LLT::scalar(32);
+  const LLT s64 = LLT::scalar(64);
   const LLT v4s32 = LLT::vector(4, 32);
   const LLT v2s64 = LLT::vector(2, 64);
 
@@ -192,6 +211,14 @@ void X86LegalizerInfo::setLegalizerInfoSSE1() {
 
   // Constants
   setAction({TargetOpcode::G_FCONSTANT, s32}, Legal);
+
+  // Merge/Unmerge
+  for (const auto &Ty : {v4s32, v2s64}) {
+    setAction({G_MERGE_VALUES, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
+  }
+  setAction({G_MERGE_VALUES, 1, s64}, Legal);
+  setAction({G_UNMERGE_VALUES, s64}, Legal);
 }
 
 void X86LegalizerInfo::setLegalizerInfoSSE2() {
@@ -205,6 +232,11 @@ void X86LegalizerInfo::setLegalizerInfoSSE2() {
   const LLT v4s32 = LLT::vector(4, 32);
   const LLT v2s64 = LLT::vector(2, 64);
 
+  const LLT v32s8 = LLT::vector(32, 8);
+  const LLT v16s16 = LLT::vector(16, 16);
+  const LLT v8s32 = LLT::vector(8, 32);
+  const LLT v4s64 = LLT::vector(4, 64);
+
   for (unsigned BinOp : {G_FADD, G_FSUB, G_FMUL, G_FDIV})
     for (auto Ty : {s64, v2s64})
       setAction({BinOp, Ty}, Legal);
@@ -220,6 +252,17 @@ void X86LegalizerInfo::setLegalizerInfoSSE2() {
 
   // Constants
   setAction({TargetOpcode::G_FCONSTANT, s64}, Legal);
+
+  // Merge/Unmerge
+  for (const auto &Ty :
+       {v16s8, v32s8, v8s16, v16s16, v4s32, v8s32, v2s64, v4s64}) {
+    setAction({G_MERGE_VALUES, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
+  }
+  for (const auto &Ty : {v16s8, v8s16, v4s32, v2s64}) {
+    setAction({G_MERGE_VALUES, 1, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, Ty}, Legal);
+  }
 }
 
 void X86LegalizerInfo::setLegalizerInfoSSE41() {
@@ -241,9 +284,13 @@ void X86LegalizerInfo::setLegalizerInfoAVX() {
   const LLT v2s64 = LLT::vector(2, 64);
 
   const LLT v32s8 = LLT::vector(32, 8);
+  const LLT v64s8 = LLT::vector(64, 8);
   const LLT v16s16 = LLT::vector(16, 16);
+  const LLT v32s16 = LLT::vector(32, 16);
   const LLT v8s32 = LLT::vector(8, 32);
+  const LLT v16s32 = LLT::vector(16, 32);
   const LLT v4s64 = LLT::vector(4, 64);
+  const LLT v8s64 = LLT::vector(8, 64);
 
   for (unsigned MemOp : {G_LOAD, G_STORE})
     for (auto Ty : {v8s32, v4s64})
@@ -257,6 +304,17 @@ void X86LegalizerInfo::setLegalizerInfoAVX() {
     setAction({G_INSERT, 1, Ty}, Legal);
     setAction({G_EXTRACT, Ty}, Legal);
   }
+  // Merge/Unmerge
+  for (const auto &Ty :
+       {v32s8, v64s8, v16s16, v32s16, v8s32, v16s32, v4s64, v8s64}) {
+    setAction({G_MERGE_VALUES, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
+  }
+  for (const auto &Ty :
+       {v16s8, v32s8, v8s16, v16s16, v4s32, v8s32, v2s64, v4s64}) {
+    setAction({G_MERGE_VALUES, 1, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, Ty}, Legal);
+  }
 }
 
 void X86LegalizerInfo::setLegalizerInfoAVX2() {
@@ -268,12 +326,27 @@ void X86LegalizerInfo::setLegalizerInfoAVX2() {
   const LLT v8s32 = LLT::vector(8, 32);
   const LLT v4s64 = LLT::vector(4, 64);
 
+  const LLT v64s8 = LLT::vector(64, 8);
+  const LLT v32s16 = LLT::vector(32, 16);
+  const LLT v16s32 = LLT::vector(16, 32);
+  const LLT v8s64 = LLT::vector(8, 64);
+
   for (unsigned BinOp : {G_ADD, G_SUB})
     for (auto Ty : {v32s8, v16s16, v8s32, v4s64})
       setAction({BinOp, Ty}, Legal);
 
   for (auto Ty : {v16s16, v8s32})
     setAction({G_MUL, Ty}, Legal);
+
+  // Merge/Unmerge
+  for (const auto &Ty : {v64s8, v32s16, v16s32, v8s64}) {
+    setAction({G_MERGE_VALUES, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, 1, Ty}, Legal);
+  }
+  for (const auto &Ty : {v32s8, v16s16, v8s32, v4s64}) {
+    setAction({G_MERGE_VALUES, 1, Ty}, Legal);
+    setAction({G_UNMERGE_VALUES, Ty}, Legal);
+  }
 }
 
 void X86LegalizerInfo::setLegalizerInfoAVX512() {
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir
index 82594b8c476..9cf0f8fd0e7 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-combines.mir
@@ -8,7 +8,6 @@
   define void @test_combines_3() { ret void }
   define void @test_combines_4() { ret void }
   define void @test_combines_5() { ret void }
-  define void @test_combines_6() { ret void }
 ...
 
 ---
@@ -90,23 +89,3 @@ body: |
     %5:_(s32) = G_ADD %3, %4
     %w0 = COPY %5
 ...
-
----
-name:            test_combines_6
-body: |
-  bb.0:
-    liveins: %w0
-
-    ; Check that we replace all the uses of a G_EXTRACT.
-    ; CHECK-LABEL: name: test_combines_6
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %w0
-    ; CHECK: [[MUL:%[0-9]+]]:_(s32) = G_MUL [[COPY]], [[COPY]]
-    ; CHECK: [[ADD:%[0-9]+]]:_(s32) = G_ADD [[COPY]], [[MUL]]
-    %0:_(s32) = COPY %w0
-
-    %1:_(s32) = G_MERGE_VALUES %0
-    %2:_(s32) = G_UNMERGE_VALUES %1
-    %3:_(s32) = G_MUL %2, %2
-    %4:_(s32) = G_ADD %2, %3
-    %w0 = COPY %4
-...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
new file mode 100644
index 00000000000..e6171380344
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-merge-values.mir
@@ -0,0 +1,30 @@
+# RUN: llc -O0 -run-pass=legalizer -global-isel -global-isel-abort=0 -pass-remarks-missed='gisel*' %s -o - 2>&1 | FileCheck %s
+
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64--"
+  define void @test_merge_s4() {
+    ret void
+  }
+...
+
+---
+name:            test_merge_s4
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+  - { id: 4, class: _ }
+body: |
+  bb.0:
+    %0(s64) = G_CONSTANT i64 0
+    %1(s4) = G_TRUNC %0(s64)
+    ; Previously, LegalizerInfo was assuming all G_MERGE_VALUES and G_UNMERGE_VALUES
+    ; instructions are legal. Make sure that is no longer happening.
+    ; CHECK: unable to legalize instruction: {{.*}} G_MERGE_VALUES
+    %2(s8) = G_MERGE_VALUES %1(s4), %1(s4)
+    %3(s8) = COPY %2(s8)
+    %4(s64) = G_ANYEXT %3(s8)
+    %x0 = COPY %4(s64)
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-nonpowerof2eltsvec.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-nonpowerof2eltsvec.mir
index a586e69c855..168e1df0277 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-nonpowerof2eltsvec.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-nonpowerof2eltsvec.mir
@@ -4,33 +4,31 @@
 --- |
   target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
   target triple = "aarch64--"
-  define void @test_legalize_merge_v3s32() {
+  define void @test_legalize_merge_v3s64() {
     ret void
   }
 ...
 ---
-name:            test_legalize_merge_v3s32
+name:            test_legalize_merge_v3s64
 registers:
   - { id: 0, class: _ }
   - { id: 1, class: _ }
   - { id: 2, class: _ }
   - { id: 3, class: _ }
+  - { id: 4, class: _ }
+  - { id: 5, class: _ }
 body: |
   bb.0:
-    liveins: %w0, %w1, %w2
-    ; CHECK-LABEL: name: test_legalize_merge_v3s32
-    ; CHECK: [[COPY:%[0-9]+]]:_(s32) = COPY %w0
-    ; CHECK: [[COPY1:%[0-9]+]]:_(s32) = COPY %w1
-    ; CHECK: [[COPY2:%[0-9]+]]:_(s32) = COPY %w2
-    ; CHECK: %w0 = COPY [[COPY]](s32)
-    ; CHECK: %w1 = COPY [[COPY1]](s32)
-    ; CHECK: %w2 = COPY [[COPY2]](s32)
-    %0(s32) = COPY %w0
-    %1(s32) = COPY %w1
-    %2(s32) = COPY %w2
-    %3(<3 x s32>) = G_MERGE_VALUES %0(s32), %1(s32), %2(s32)
-    %4:_(s32), %5:_(s32), %6:_(s32) = G_UNMERGE_VALUES %3
-    %w0 = COPY %4
-    %w1 = COPY %5
-    %w2 = COPY %6
+    liveins: %w0
+    ; CHECK-LABEL: name: test_legalize_merge_v3s64
+    ; CHECK: [[COPY:%[0-9]+]]:_(s64) = COPY %x0
+    ; CHECK: [[MV:%[0-9]+]]:_(<3 x s64>) = G_MERGE_VALUES [[COPY]](s64), [[COPY]](s64), [[COPY]](s64)
+    ; CHECK: [[COPY1:%[0-9]+]]:_(<3 x s64>) = COPY [[MV]](<3 x s64>)
+    ; CHECK: [[UV:%[0-9]+]]:_(s64), [[UV1:%[0-9]+]]:_(s64), [[UV2:%[0-9]+]]:_(s64) = G_UNMERGE_VALUES [[COPY1]](<3 x s64>)
+    ; CHECK: %x0 = COPY [[UV]](s64)
+    %0(s64) = COPY %x0
+    %1(<3 x s64>) = G_MERGE_VALUES %0(s64), %0(s64), %0(s64)
+    %2(<3 x s64>) = COPY %1(<3 x s64>)
+    %3(s64), %4(s64), %5(s64) = G_UNMERGE_VALUES %2(<3 x s64>)
+    %x0 = COPY %3(s64)
 ...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
new file mode 100644
index 00000000000..85b65e94548
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/legalize-unmerge-values.mir
@@ -0,0 +1,28 @@
+# RUN: llc -O0 -run-pass=legalizer -global-isel -global-isel-abort=0 -pass-remarks-missed='gisel*' %s -o - 2>&1 | FileCheck %s
+
+--- |
+  target datalayout = "e-m:o-i64:64-i128:128-n32:64-S128"
+  target triple = "aarch64--"
+  define void @test_unmerge_s4() {
+    ret void
+  }
+...
+
+---
+name:            test_unmerge_s4
+registers:
+  - { id: 0, class: _ }
+  - { id: 1, class: _ }
+  - { id: 2, class: _ }
+  - { id: 3, class: _ }
+body: |
+  bb.0:
+    %0(s8) = G_CONSTANT i8 0
+    ; Previously, LegalizerInfo was assuming all G_MERGE_VALUES and G_UNMERGE_VALUES
+    ; instructions are legal. Make sure that is no longer happening.
+    ; CHECK: unable to legalize instruction: {{.*}} G_UNMERGE_VALUES
+    %1(s4), %2(s4)= G_UNMERGE_VALUES %0(s8)
+    %3(s64) = G_ANYEXT %1(s4)
+    %x0 = COPY %3(s64)
+
+...
diff --git a/llvm/test/CodeGen/AArch64/GlobalISel/no-regclass.mir b/llvm/test/CodeGen/AArch64/GlobalISel/no-regclass.mir
index d4d23142ab9..8732274fe03 100644
--- a/llvm/test/CodeGen/AArch64/GlobalISel/no-regclass.mir
+++ b/llvm/test/CodeGen/AArch64/GlobalISel/no-regclass.mir
@@ -25,7 +25,7 @@ body:             |
     ; CHECK: [[COPY:%[0-9]+]]:gpr32all = COPY %w0
     ; CHECK: %w0 = COPY [[COPY]]
     %0:gpr(s32) = COPY %w0
-    %1:gpr(s32) = G_MERGE_VALUES %0(s32)
-    %2:gpr(s32) = G_UNMERGE_VALUES %1(s32)
+    %1:gpr(s64) = G_MERGE_VALUES %0(s32), %0(s32)
+    %2:gpr(s32), %3:gpr(s32) = G_UNMERGE_VALUES %1(s64)
     %w0 = COPY %2(s32)
 ...