ARM: use a pseudo-instruction for cmpxchg at -O0.

The fast register-allocator cannot cope with inter-block dependencies without
spilling. This is fine for ldrex/strex loops coming from atomicrmw instructions
where any value produced within a block is dead by the end, but not for
cmpxchg. So we lower a cmpxchg at -O0 via a pseudo-inst that gets expanded
after regalloc.

Fortunately this is at -O0 so we don't have to care about performance. This
simplifies the various axes of expansion considerably: we assume a strong
seq_cst operation and ensure ordering via the always-present DMB instructions
rather than v8 acquire/release instructions.

Should fix the 32-bit part of PR25526.

llvm-svn: 266679

4 files changed, 84 insertions, 3 deletions

diff --git a/llvm/test/CodeGen/ARM/cmpxchg-O0.ll b/llvm/test/CodeGen/ARM/cmpxchg-O0.ll
new file mode 100644
index 00000000000..0bf22b05a76
--- /dev/null
+++ b/llvm/test/CodeGen/ARM/cmpxchg-O0.ll
@@ -0,0 +1,81 @@
+; RUN: llc -verify-machineinstrs -mtriple=armv7-linux-gnu -O0 %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=thumbv8-linux-gnu -O0 %s -o - | FileCheck %s
+; RUN: llc -verify-machineinstrs -mtriple=thumbv6m-none-eabi -O0 %s -o - | FileCheck %s --check-prefix=CHECK-T1
+
+; CHECK-T1-NOT: ldrex
+; CHECK-T1-NOT: strex
+
+define { i8, i1 } @test_cmpxchg_8(i8* %addr, i8 %desired, i8 %new) nounwind {
+; CHECK-LABEL: test_cmpxchg_8:
+; CHECK:     dmb ish
+; CHECK:     uxtb [[DESIRED:r[0-9]+]], [[DESIRED]]
+; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
+; CHECK:     ldrexb [[OLD:r[0-9]+]], [r0]
+; CHECK:     cmp [[OLD]], [[DESIRED]]
+; CHECK:     bne [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK:     strexb [[STATUS:r[0-9]+]], r2, [r0]
+; CHECK:     cmp{{(\.w)?}} [[STATUS]], #0
+; CHECK:     bne [[RETRY]]
+; CHECK: [[DONE]]:
+; CHECK:     cmp{{(\.w)?}} [[OLD]], [[DESIRED]]
+; CHECK:     {{moveq.w|movweq}} {{r[0-9]+}}, #1
+; CHECK:     dmb ish
+  %res = cmpxchg i8* %addr, i8 %desired, i8 %new seq_cst monotonic
+  ret { i8, i1 } %res
+}
+
+define { i16, i1 } @test_cmpxchg_16(i16* %addr, i16 %desired, i16 %new) nounwind {
+; CHECK-LABEL: test_cmpxchg_16:
+; CHECK:     dmb ish
+; CHECK:     uxth [[DESIRED:r[0-9]+]], [[DESIRED]]
+; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
+; CHECK:     ldrexh [[OLD:r[0-9]+]], [r0]
+; CHECK:     cmp [[OLD]], [[DESIRED]]
+; CHECK:     bne [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK:     strexh [[STATUS:r[0-9]+]], r2, [r0]
+; CHECK:     cmp{{(\.w)?}} [[STATUS]], #0
+; CHECK:     bne [[RETRY]]
+; CHECK: [[DONE]]:
+; CHECK:     cmp{{(\.w)?}} [[OLD]], [[DESIRED]]
+; CHECK:     {{moveq.w|movweq}} {{r[0-9]+}}, #1
+; CHECK:     dmb ish
+  %res = cmpxchg i16* %addr, i16 %desired, i16 %new seq_cst monotonic
+  ret { i16, i1 } %res
+}
+
+define { i32, i1 } @test_cmpxchg_32(i32* %addr, i32 %desired, i32 %new) nounwind {
+; CHECK-LABEL: test_cmpxchg_32:
+; CHECK:     dmb ish
+; CHECK-NOT:     uxt
+; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
+; CHECK:     ldrex [[OLD:r[0-9]+]], [r0]
+; CHECK:     cmp [[OLD]], [[DESIRED]]
+; CHECK:     bne [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK:     strex [[STATUS:r[0-9]+]], r2, [r0]
+; CHECK:     cmp{{(\.w)?}} [[STATUS]], #0
+; CHECK:     bne [[RETRY]]
+; CHECK: [[DONE]]:
+; CHECK:     cmp{{(\.w)?}} [[OLD]], [[DESIRED]]
+; CHECK:     {{moveq.w|movweq}} {{r[0-9]+}}, #1
+; CHECK:     dmb ish
+  %res = cmpxchg i32* %addr, i32 %desired, i32 %new seq_cst monotonic
+  ret { i32, i1 } %res
+}
+
+define { i64, i1 } @test_cmpxchg_64(i64* %addr, i64 %desired, i64 %new) nounwind {
+; CHECK-LABEL: test_cmpxchg_64:
+; CHECK:     dmb ish
+; CHECK-NOT: uxt
+; CHECK: [[RETRY:.LBB[0-9]+_[0-9]+]]:
+; CHECK:     ldrexd [[OLDLO:r[0-9]+]], [[OLDHI:r[0-9]+]], [r0]
+; CHECK:     cmp [[OLDLO]], r6
+; CHECK:     sbcs{{(\.w)?}} [[STATUS:r[0-9]+]], [[OLDHI]], r7
+; CHECK:     bne [[DONE:.LBB[0-9]+_[0-9]+]]
+; CHECK:     strexd [[STATUS]], r4, r5, [r0]
+; CHECK:     cmp{{(\.w)?}} [[STATUS]], #0
+; CHECK:     bne [[RETRY]]
+; CHECK: [[DONE]]:
+; CHECK:     dmb ish
+  %res = cmpxchg i64* %addr, i64 %desired, i64 %new seq_cst monotonic
+  ret { i64, i1 } %res
+}
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll
index 31d970f7ef1..5e84460b9c0 100644
--- a/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll
+++ b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v7.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -o - -mtriple=armv7-apple-ios7.0 -atomic-expand %s | FileCheck %s
+; RUN: opt -S -o - -mtriple=armv7-apple-ios7.0 -atomic-expand -codegen-opt-level=1 %s | FileCheck %s
 
 define i8 @test_atomic_xchg_i8(i8* %ptr, i8 %xchgend) {
 ; CHECK-LABEL: @test_atomic_xchg_i8
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll
index 7bb6ffed397..8397182e7e8 100644
--- a/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll
+++ b/llvm/test/Transforms/AtomicExpand/ARM/atomic-expansion-v8.ll
@@ -1,4 +1,4 @@
-; RUN: opt -S -o - -mtriple=armv8-linux-gnueabihf -atomic-expand %s | FileCheck %s
+; RUN: opt -S -o - -mtriple=armv8-linux-gnueabihf -atomic-expand %s -codegen-opt-level=1 | FileCheck %s
 
 define i8 @test_atomic_xchg_i8(i8* %ptr, i8 %xchgend) {
 ; CHECK-LABEL: @test_atomic_xchg_i8
diff --git a/llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll b/llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll
index 02e4dd1f1d5..375b41a26db 100644
--- a/llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll
+++ b/llvm/test/Transforms/AtomicExpand/ARM/cmpxchg-weak.ll
@@ -1,4 +1,4 @@
-; RUN: opt -atomic-expand -S -mtriple=thumbv7s-apple-ios7.0 %s | FileCheck %s
+; RUN: opt -atomic-expand -codegen-opt-level=1 -S -mtriple=thumbv7s-apple-ios7.0 %s | FileCheck %s
 
 define i32 @test_cmpxchg_seq_cst(i32* %addr, i32 %desired, i32 %new) {
 ; CHECK-LABEL: @test_cmpxchg_seq_cst