[SystemZ] Eliminate unnecessary serialization operations

We currently emit a serialization operation (bcr 14, 0) before every
atomic load and after every atomic store.  This is overly conservative.
The SystemZ architecture actually does not require any serialization
for atomic loads, and a serialization after an atomic store only if
we need to enforce sequential consistency.  This is what other compilers
for the platform implement as well.

llvm-svn: 310093

9 files changed, 45 insertions, 13 deletions

diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index c0e686e1370..42d4a8d3185 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -3348,28 +3348,28 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
   return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
 }
 
-// Op is an atomic load.  Lower it into a serialization followed
-// by a normal volatile load.
+// Op is an atomic load.  Lower it into a normal volatile load.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
                                                 SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
-  SDValue Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
-                                             MVT::Other, Node->getChain()), 0);
   return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
-                        Chain, Node->getBasePtr(),
+                        Node->getChain(), Node->getBasePtr(),
                         Node->getMemoryVT(), Node->getMemOperand());
 }
 
-// Op is an atomic store.  Lower it into a normal volatile store followed
-// by a serialization.
+// Op is an atomic store.  Lower it into a normal volatile store.
 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
                                                  SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
                                     Node->getBasePtr(), Node->getMemoryVT(),
                                     Node->getMemOperand());
-  return SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op), MVT::Other,
-                                    Chain), 0);
+  // We have to enforce sequential consistency by performing a
+  // serialization operation after the store.
+  if (Node->getOrdering() == AtomicOrdering::SequentiallyConsistent)
+    Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
+                                       MVT::Other, Chain), 0);
+  return Chain;
 }
 
 // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation.  Lower the first
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-01.ll b/llvm/test/CodeGen/SystemZ/atomic-load-01.ll
index b2f4ebe6639..4e228ac668b 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-01.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-01.ll
@@ -4,7 +4,6 @@
 
 define i8 @f1(i8 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lb %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i8 , i8 *%src seq_cst, align 1
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-02.ll b/llvm/test/CodeGen/SystemZ/atomic-load-02.ll
index b2b60f3d016..44e24d3cca4 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-02.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-02.ll
@@ -4,7 +4,6 @@
 
 define i16 @f1(i16 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lh %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i16 , i16 *%src seq_cst, align 2
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-03.ll b/llvm/test/CodeGen/SystemZ/atomic-load-03.ll
index d83c430bd0a..2f63bd65256 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-03.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-03.ll
@@ -4,7 +4,6 @@
 
 define i32 @f1(i32 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: l %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i32 , i32 *%src seq_cst, align 4
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-04.ll b/llvm/test/CodeGen/SystemZ/atomic-load-04.ll
index dc6b271e00e..6dba26390ec 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-04.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-04.ll
@@ -4,7 +4,6 @@
 
 define i64 @f1(i64 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lg %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i64 , i64 *%src seq_cst, align 8
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-01.ll b/llvm/test/CodeGen/SystemZ/atomic-store-01.ll
index 952e1a91216..617557fd1c2 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-01.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-01.ll
@@ -10,3 +10,12 @@ define void @f1(i8 %val, i8 *%src) {
   store atomic i8 %val, i8 *%src seq_cst, align 1
   ret void
 }
+
+define void @f2(i8 %val, i8 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: stc %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i8 %val, i8 *%src monotonic, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-02.ll b/llvm/test/CodeGen/SystemZ/atomic-store-02.ll
index c9576e55656..f23bac68289 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-02.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-02.ll
@@ -10,3 +10,12 @@ define void @f1(i16 %val, i16 *%src) {
   store atomic i16 %val, i16 *%src seq_cst, align 2
   ret void
 }
+
+define void @f2(i16 %val, i16 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: sth %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i16 %val, i16 *%src monotonic, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-03.ll b/llvm/test/CodeGen/SystemZ/atomic-store-03.ll
index 459cb6a94e1..2434bb27e7e 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-03.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-03.ll
@@ -10,3 +10,12 @@ define void @f1(i32 %val, i32 *%src) {
   store atomic i32 %val, i32 *%src seq_cst, align 4
   ret void
 }
+
+define void @f2(i32 %val, i32 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: st %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i32 %val, i32 *%src monotonic, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-04.ll b/llvm/test/CodeGen/SystemZ/atomic-store-04.ll
index 7f2406eb546..8b04cdd2036 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-04.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-04.ll
@@ -10,3 +10,12 @@ define void @f1(i64 %val, i64 *%src) {
   store atomic i64 %val, i64 *%src seq_cst, align 8
   ret void
 }
+
+define void @f2(i64 %val, i64 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: stg %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i64 %val, i64 *%src monotonic, align 8
+  ret void
+}