[MachineCopyPropagation] Extend pass to do COPY source forwarding

Summary:
This change extends MachineCopyPropagation to do COPY source forwarding
and adds an additional run of the pass to the default pass pipeline just
after register allocation.

This version of this patch uses the newly added
MachineOperand::isRenamable bit to avoid forwarding registers is such a
way as to violate constraints that aren't captured in the
Machine IR (e.g. ABI or ISA constraints).

This change is a continuation of the work started in D30751.

Reviewers: qcolombet, javed.absar, MatzeB, jonpa, tstellar

Subscribers: tpr, mgorny, mcrosier, nhaehnle, nemanjai, jyknight, hfinkel, arsenm, inouehrs, eraman, sdardis, guyblank, fedor.sergeev, aheejin, dschuff, jfb, myatsina, llvm-commits

Differential Revision: https://reviews.llvm.org/D41835

llvm-svn: 323991

10 files changed, 128 insertions, 25 deletions

diff --git a/llvm/test/CodeGen/AArch64/aarch64-fold-lslfast.ll b/llvm/test/CodeGen/AArch64/aarch64-fold-lslfast.ll
index 0dfe04b664d..55ddaf8b65f 100644
--- a/llvm/test/CodeGen/AArch64/aarch64-fold-lslfast.ll
+++ b/llvm/test/CodeGen/AArch64/aarch64-fold-lslfast.ll
@@ -9,7 +9,8 @@ define i16 @halfword(%struct.a* %ctx, i32 %xor72) nounwind {
 ; CHECK-LABEL: halfword:
 ; CHECK: ubfx [[REG:x[0-9]+]], x1, #9, #8
 ; CHECK: ldrh [[REG1:w[0-9]+]], [{{.*}}[[REG2:x[0-9]+]], [[REG]], lsl #1]
-; CHECK: strh [[REG1]], [{{.*}}[[REG2]], [[REG]], lsl #1]
+; CHECK: mov [[REG3:x[0-9]+]], [[REG2]]
+; CHECK: strh [[REG1]], [{{.*}}[[REG3]], [[REG]], lsl #1]
   %shr81 = lshr i32 %xor72, 9
   %conv82 = zext i32 %shr81 to i64
   %idxprom83 = and i64 %conv82, 255
@@ -24,7 +25,8 @@ define i32 @word(%struct.b* %ctx, i32 %xor72) nounwind {
 ; CHECK-LABEL: word:
 ; CHECK: ubfx [[REG:x[0-9]+]], x1, #9, #8
 ; CHECK: ldr [[REG1:w[0-9]+]], [{{.*}}[[REG2:x[0-9]+]], [[REG]], lsl #2]
-; CHECK: str [[REG1]], [{{.*}}[[REG2]], [[REG]], lsl #2]
+; CHECK: mov [[REG3:x[0-9]+]], [[REG2]]
+; CHECK: str [[REG1]], [{{.*}}[[REG3]], [[REG]], lsl #2]
   %shr81 = lshr i32 %xor72, 9
   %conv82 = zext i32 %shr81 to i64
   %idxprom83 = and i64 %conv82, 255
@@ -39,7 +41,8 @@ define i64 @doubleword(%struct.c* %ctx, i32 %xor72) nounwind {
 ; CHECK-LABEL: doubleword:
 ; CHECK: ubfx [[REG:x[0-9]+]], x1, #9, #8
 ; CHECK: ldr [[REG1:x[0-9]+]], [{{.*}}[[REG2:x[0-9]+]], [[REG]], lsl #3]
-; CHECK: str [[REG1]], [{{.*}}[[REG2]], [[REG]], lsl #3]
+; CHECK: mov [[REG3:x[0-9]+]], [[REG2]]
+; CHECK: str [[REG1]], [{{.*}}[[REG3]], [[REG]], lsl #3]
   %shr81 = lshr i32 %xor72, 9
   %conv82 = zext i32 %shr81 to i64
   %idxprom83 = and i64 %conv82, 255
diff --git a/llvm/test/CodeGen/AArch64/arm64-AdvSIMD-Scalar.ll b/llvm/test/CodeGen/AArch64/arm64-AdvSIMD-Scalar.ll
index 72e5ec6b89b..949de1a7c63 100644
--- a/llvm/test/CodeGen/AArch64/arm64-AdvSIMD-Scalar.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-AdvSIMD-Scalar.ll
@@ -8,15 +8,9 @@ define <2 x i64> @bar(<2 x i64> %a, <2 x i64> %b) nounwind readnone {
 ; CHECK: add.2d	v[[REG:[0-9]+]], v0, v1
 ; CHECK: add	d[[REG3:[0-9]+]], d[[REG]], d1
 ; CHECK: sub	d[[REG2:[0-9]+]], d[[REG]], d1
-; Without advanced copy optimization, we end up with cross register
-; banks copies that cannot be coalesced.
-; CHECK-NOOPT: fmov [[COPY_REG3:x[0-9]+]], d[[REG3]]
-; With advanced copy optimization, we end up with just one copy
-; to insert the computed high part into the V register. 
-; CHECK-OPT-NOT: fmov
+; CHECK-NOT: fmov
 ; CHECK: fmov [[COPY_REG2:x[0-9]+]], d[[REG2]]
-; CHECK-NOOPT: fmov d0, [[COPY_REG3]]
-; CHECK-OPT-NOT: fmov
+; CHECK-NOT: fmov
 ; CHECK: mov.d v0[1], [[COPY_REG2]]
 ; CHECK-NEXT: ret
 ;
@@ -24,11 +18,9 @@ define <2 x i64> @bar(<2 x i64> %a, <2 x i64> %b) nounwind readnone {
 ; GENERIC: add	v[[REG:[0-9]+]].2d, v0.2d, v1.2d
 ; GENERIC: add	d[[REG3:[0-9]+]], d[[REG]], d1
 ; GENERIC: sub	d[[REG2:[0-9]+]], d[[REG]], d1
-; GENERIC-NOOPT: fmov [[COPY_REG3:x[0-9]+]], d[[REG3]]
-; GENERIC-OPT-NOT: fmov
+; GENERIC-NOT: fmov
 ; GENERIC: fmov [[COPY_REG2:x[0-9]+]], d[[REG2]]
-; GENERIC-NOOPT: fmov d0, [[COPY_REG3]]
-; GENERIC-OPT-NOT: fmov
+; GENERIC-NOT: fmov
 ; GENERIC: mov v0.d[1], [[COPY_REG2]]
 ; GENERIC-NEXT: ret
   %add = add <2 x i64> %a, %b
diff --git a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll
index c56d607aa81..60a62030e44 100644
--- a/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll
+++ b/llvm/test/CodeGen/AArch64/arm64-zero-cycle-regmov.ll
@@ -4,8 +4,10 @@
 define i32 @t(i32 %a, i32 %b, i32 %c, i32 %d) nounwind ssp {
 entry:
 ; CHECK-LABEL: t:
-; CHECK: mov x0, [[REG1:x[0-9]+]]
-; CHECK: mov x1, [[REG2:x[0-9]+]]
+; CHECK: mov [[REG2:x[0-9]+]], x3
+; CHECK: mov [[REG1:x[0-9]+]], x2
+; CHECK: mov x0, x2
+; CHECK: mov x1, x3
 ; CHECK: bl _foo
 ; CHECK: mov x0, [[REG1]]
 ; CHECK: mov x1, [[REG2]]
diff --git a/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll b/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll
index da0f7073ace..5ff3ddfe09a 100644
--- a/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll
+++ b/llvm/test/CodeGen/AArch64/cmpxchg-idioms.ll
@@ -45,8 +45,7 @@ define i1 @test_return_bool(i8* %value, i8 %oldValue, i8 %newValue) {
 
 ; CHECK: [[FAILED]]:
 ; CHECK-NOT: cmp {{w[0-9]+}}, {{w[0-9]+}}
-; CHECK: mov [[TMP:w[0-9]+]], wzr
-; CHECK: eor w0, [[TMP]], #0x1
+; CHECK: eor w0, wzr, #0x1
 ; CHECK: ret
 
   %pair = cmpxchg i8* %value, i8 %oldValue, i8 %newValue acq_rel monotonic
diff --git a/llvm/test/CodeGen/AArch64/copyprop.mir b/llvm/test/CodeGen/AArch64/copyprop.mir
new file mode 100644
index 00000000000..e23002c5690
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/copyprop.mir
@@ -0,0 +1,104 @@
+# RUN: llc -mtriple=aarch64-linux-gnu -run-pass machine-cp -o - %s | FileCheck %s
+# Tests for MachineCopyPropagation copy forwarding.
+---
+# Simple forwarding.
+# CHECK-LABEL: name: test1
+# CHECK: $x0 = SUBXri $x0, 1, 0
+name:            test1
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0
+    renamable $x1 = COPY $x0
+    $x0 = SUBXri renamable $x1, 1, 0
+...
+---
+# Don't forward if not renamable.
+# CHECK-LABEL: name: test2
+# CHECK: $x0 = SUBXri $x1, 1, 0
+name:            test2
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0
+    $x1 = COPY $x0
+    $x0 = SUBXri $x1, 1, 0
+...
+---
+# Don't forward reserved non-constant reg values.
+# CHECK-LABEL: name: test4
+# CHECK: $x0 = SUBXri renamable $x1, 1, 0
+name:            test4
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0
+    $sp = SUBXri $sp, 16, 0
+    renamable $x1 = COPY $sp
+    $x0 = SUBXri renamable $x1, 1, 0
+    $sp = ADDXri $sp, 16, 0
+...
+---
+# Don't violate opcode constraints when forwarding.
+# CHECK-LABEL: name: test5
+# CHECK: $x0 = SUBXri renamable $x1, 1, 0
+name:            test5
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0
+    renamable $x1 = COPY $xzr
+    $x0 = SUBXri renamable $x1, 1, 0
+...
+---
+# Test cross-class COPY forwarding.
+# CHECK-LABEL: name: test6
+# CHECK: $x2 = COPY $x0
+name:            test6
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0
+    renamable $d1 = COPY $x0
+    $x2 = COPY renamable $d1
+    RET_ReallyLR implicit $x2
+...
+---
+# Don't forward if there are overlapping implicit operands.
+# CHECK-LABEL: name: test7
+# CHECK: $w0 = SUBWri killed renamable $w1, 1, 0
+name:            test7
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $w0
+    renamable $w1 = COPY $w0
+    $w0 = SUBWri killed renamable $w1, 1, 0, implicit killed $x1
+...
+---
+# Check that kill flags are cleared.
+# CHECK-LABEL: name: test8
+# CHECK: $x2 = ADDXri $x0, 1, 0
+# CHECK: $x0 = SUBXri $x0, 1, 0
+name:            test8
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0
+    renamable $x1 = COPY $x0
+    $x2 = ADDXri killed $x0, 1, 0
+    $x0 = SUBXri renamable $x1, 1, 0
+...
+---
+# Don't forward if value is clobbered.
+# CHECK-LABEL: name: test9
+# CHECK: $x2 = SUBXri renamable $x1, 1, 0
+name:            test9
+tracksRegLiveness: true
+body: |
+  bb.0:
+    liveins: $x0
+    renamable $x1 = COPY $x0
+    $x0 = ADDXri $x0, 1, 0
+    $x2 = SUBXri renamable $x1, 1, 0
+...
diff --git a/llvm/test/CodeGen/AArch64/f16-instructions.ll b/llvm/test/CodeGen/AArch64/f16-instructions.ll
index 1bec17f78ad..2aade50cddc 100644
--- a/llvm/test/CodeGen/AArch64/f16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/f16-instructions.ll
@@ -489,7 +489,7 @@ else:
 
 ; CHECK-COMMON-LABEL: test_phi:
 ; CHECK-COMMON: mov  x[[PTR:[0-9]+]], x0
-; CHECK-COMMON: ldr  h[[AB:[0-9]+]], [x[[PTR]]]
+; CHECK-COMMON: ldr  h[[AB:[0-9]+]], [x0]
 ; CHECK-COMMON: [[LOOP:LBB[0-9_]+]]:
 ; CHECK-COMMON: mov.16b  v[[R:[0-9]+]], v[[AB]]
 ; CHECK-COMMON: ldr  h[[AB]], [x[[PTR]]]
diff --git a/llvm/test/CodeGen/AArch64/flags-multiuse.ll b/llvm/test/CodeGen/AArch64/flags-multiuse.ll
index 0827fb8c9e8..a13f7e1e34a 100644
--- a/llvm/test/CodeGen/AArch64/flags-multiuse.ll
+++ b/llvm/test/CodeGen/AArch64/flags-multiuse.ll
@@ -17,6 +17,9 @@ define i32 @test_multiflag(i32 %n, i32 %m, i32 %o) {
   %val = zext i1 %test to i32
 ; CHECK: cset {{[xw][0-9]+}}, ne
 
+; CHECK: mov [[RHSCOPY:w[0-9]+]], [[RHS]]
+; CHECK: mov [[LHSCOPY:w[0-9]+]], [[LHS]]
+
   store i32 %val, i32* @var
 
   call void @bar()
@@ -25,7 +28,7 @@ define i32 @test_multiflag(i32 %n, i32 %m, i32 %o) {
   ; Currently, the comparison is emitted again. An MSR/MRS pair would also be
   ; acceptable, but assuming the call preserves NZCV is not.
   br i1 %test, label %iftrue, label %iffalse
-; CHECK: cmp [[LHS]], [[RHS]]
+; CHECK: cmp [[LHSCOPY]], [[RHSCOPY]]
 ; CHECK: b.eq
 
 iftrue:
diff --git a/llvm/test/CodeGen/AArch64/ldst-opt.ll b/llvm/test/CodeGen/AArch64/ldst-opt.ll
index e416dcb0f16..ae3f59ee8f5 100644
--- a/llvm/test/CodeGen/AArch64/ldst-opt.ll
+++ b/llvm/test/CodeGen/AArch64/ldst-opt.ll
@@ -1671,7 +1671,7 @@ entry:
 ; CHECK-LABEL: bug34674:
 ; CHECK: // %entry
 ; CHECK-NEXT: mov [[ZREG:x[0-9]+]], xzr
-; CHECK-DAG: stp [[ZREG]], [[ZREG]], [x0]
+; CHECK-DAG: stp xzr, xzr, [x0]
 ; CHECK-DAG: add x{{[0-9]+}}, [[ZREG]], #1
 define i64 @bug34674(<2 x i64>* %p) {
 entry:
diff --git a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
index 5bed63ef895..3b68cbb8c2a 100644
--- a/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
+++ b/llvm/test/CodeGen/AArch64/merge-store-dependency.ll
@@ -11,7 +11,7 @@ entry:
 ; A53: mov [[DATA:w[0-9]+]], w1
 ; A53: str q{{[0-9]+}}, {{.*}}
 ; A53: str q{{[0-9]+}}, {{.*}}
-; A53: str [[DATA]], {{.*}}
+; A53: str w1, {{.*}}
 
   %0 = bitcast %struct1* %fde to i8*
   tail call void @llvm.memset.p0i8.i64(i8* align 8 %0, i8 0, i64 40, i1 false)
diff --git a/llvm/test/CodeGen/AArch64/neg-imm.ll b/llvm/test/CodeGen/AArch64/neg-imm.ll
index 46bded78cc5..ee95f37c203 100644
--- a/llvm/test/CodeGen/AArch64/neg-imm.ll
+++ b/llvm/test/CodeGen/AArch64/neg-imm.ll
@@ -7,8 +7,8 @@ declare void @foo(i32)
 define void @test(i32 %px) {
 ; CHECK_LABEL: test:
 ; CHECK_LABEL: %entry
-; CHECK: subs
-; CHECK-NEXT: csel
+; CHECK: subs [[REG0:w[0-9]+]],
+; CHECK: csel {{w[0-9]+}}, wzr, [[REG0]]
 entry:
   %sub = add nsw i32 %px, -1
   %cmp = icmp slt i32 %px, 1