9 files changed, 45 insertions, 13 deletions
diff --git a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
index c0e686e1370..42d4a8d3185 100644
--- a/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
+++ b/llvm/lib/Target/SystemZ/SystemZISelLowering.cpp
@@ -3348,28 +3348,28 @@ SDValue SystemZTargetLowering::lowerATOMIC_FENCE(SDValue Op,
   return DAG.getNode(SystemZISD::MEMBARRIER, DL, MVT::Other, Op.getOperand(0));
 }
 
-// Op is an atomic load.  Lower it into a serialization followed
-// by a normal volatile load.
+// Op is an atomic load.  Lower it into a normal volatile load.
 SDValue SystemZTargetLowering::lowerATOMIC_LOAD(SDValue Op,
                                                 SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
-  SDValue Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
-                                             MVT::Other, Node->getChain()), 0);
   return DAG.getExtLoad(ISD::EXTLOAD, SDLoc(Op), Op.getValueType(),
-                        Chain, Node->getBasePtr(),
+                        Node->getChain(), Node->getBasePtr(),
                         Node->getMemoryVT(), Node->getMemOperand());
 }
 
-// Op is an atomic store.  Lower it into a normal volatile store followed
-// by a serialization.
+// Op is an atomic store.  Lower it into a normal volatile store.
 SDValue SystemZTargetLowering::lowerATOMIC_STORE(SDValue Op,
                                                  SelectionDAG &DAG) const {
   auto *Node = cast<AtomicSDNode>(Op.getNode());
   SDValue Chain = DAG.getTruncStore(Node->getChain(), SDLoc(Op), Node->getVal(),
                                     Node->getBasePtr(), Node->getMemoryVT(),
                                     Node->getMemOperand());
-  return SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op), MVT::Other,
-                                    Chain), 0);
+  // We have to enforce sequential consistency by performing a
+  // serialization operation after the store.
+  if (Node->getOrdering() == AtomicOrdering::SequentiallyConsistent)
+    Chain = SDValue(DAG.getMachineNode(SystemZ::Serialize, SDLoc(Op),
+                                       MVT::Other, Chain), 0);
+  return Chain;
 }
 
 // Op is an 8-, 16-bit or 32-bit ATOMIC_LOAD_* operation.  Lower the first
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-01.ll b/llvm/test/CodeGen/SystemZ/atomic-load-01.ll
index b2f4ebe6639..4e228ac668b 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-01.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-01.ll
@@ -4,7 +4,6 @@
 
 define i8 @f1(i8 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lb %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i8 , i8 *%src seq_cst, align 1
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-02.ll b/llvm/test/CodeGen/SystemZ/atomic-load-02.ll
index b2b60f3d016..44e24d3cca4 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-02.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-02.ll
@@ -4,7 +4,6 @@
 
 define i16 @f1(i16 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lh %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i16 , i16 *%src seq_cst, align 2
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-03.ll b/llvm/test/CodeGen/SystemZ/atomic-load-03.ll
index d83c430bd0a..2f63bd65256 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-03.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-03.ll
@@ -4,7 +4,6 @@
 
 define i32 @f1(i32 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: l %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i32 , i32 *%src seq_cst, align 4
diff --git a/llvm/test/CodeGen/SystemZ/atomic-load-04.ll b/llvm/test/CodeGen/SystemZ/atomic-load-04.ll
index dc6b271e00e..6dba26390ec 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-load-04.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-load-04.ll
@@ -4,7 +4,6 @@
 
 define i64 @f1(i64 *%src) {
 ; CHECK-LABEL: f1:
-; CHECK: bcr 1{{[45]}}, %r0
 ; CHECK: lg %r2, 0(%r2)
 ; CHECK: br %r14
   %val = load atomic i64 , i64 *%src seq_cst, align 8
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-01.ll b/llvm/test/CodeGen/SystemZ/atomic-store-01.ll
index 952e1a91216..617557fd1c2 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-01.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-01.ll
@@ -10,3 +10,12 @@ define void @f1(i8 %val, i8 *%src) {
   store atomic i8 %val, i8 *%src seq_cst, align 1
   ret void
 }
+
+define void @f2(i8 %val, i8 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: stc %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i8 %val, i8 *%src monotonic, align 1
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-02.ll b/llvm/test/CodeGen/SystemZ/atomic-store-02.ll
index c9576e55656..f23bac68289 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-02.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-02.ll
@@ -10,3 +10,12 @@ define void @f1(i16 %val, i16 *%src) {
   store atomic i16 %val, i16 *%src seq_cst, align 2
   ret void
 }
+
+define void @f2(i16 %val, i16 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: sth %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i16 %val, i16 *%src monotonic, align 2
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-03.ll b/llvm/test/CodeGen/SystemZ/atomic-store-03.ll
index 459cb6a94e1..2434bb27e7e 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-03.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-03.ll
@@ -10,3 +10,12 @@ define void @f1(i32 %val, i32 *%src) {
   store atomic i32 %val, i32 *%src seq_cst, align 4
   ret void
 }
+
+define void @f2(i32 %val, i32 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: st %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i32 %val, i32 *%src monotonic, align 4
+  ret void
+}
diff --git a/llvm/test/CodeGen/SystemZ/atomic-store-04.ll b/llvm/test/CodeGen/SystemZ/atomic-store-04.ll
index 7f2406eb546..8b04cdd2036 100644
--- a/llvm/test/CodeGen/SystemZ/atomic-store-04.ll
+++ b/llvm/test/CodeGen/SystemZ/atomic-store-04.ll
@@ -10,3 +10,12 @@ define void @f1(i64 %val, i64 *%src) {
   store atomic i64 %val, i64 *%src seq_cst, align 8
   ret void
 }
+
+define void @f2(i64 %val, i64 *%src) {
+; CHECK-LABEL: f2:
+; CHECK: stg %r2, 0(%r3)
+; CHECK-NOT: bcr 1{{[45]}}, %r0
+; CHECK: br %r14
+  store atomic i64 %val, i64 *%src monotonic, align 8
+  ret void
+}