PPC32 atomic operations.

llvm-svn: 49947

3 files changed, 184 insertions, 2 deletions

diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
index 83bea7e11c2..1d3bf224cda 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp
@@ -39,7 +39,8 @@ cl::desc("enable preincrement load/store generation on PPC (experimental)"),
                                      cl::Hidden);
 
 PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
-  : TargetLowering(TM), PPCSubTarget(*TM.getSubtargetImpl()) {
+  : TargetLowering(TM), PPCSubTarget(*TM.getSubtargetImpl()),
+    PPCAtomicLabelIndex(0) {
     
   setPow2DivIsCheap();
   
@@ -202,6 +203,10 @@ PPCTargetLowering::PPCTargetLowering(PPCTargetMachine &TM)
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i32  , Custom);
   setOperationAction(ISD::DYNAMIC_STACKALLOC, MVT::i64  , Custom);
 
+  setOperationAction(ISD::ATOMIC_LAS        , MVT::i32  , Custom);
+  setOperationAction(ISD::ATOMIC_LCS        , MVT::i32  , Custom);
+  setOperationAction(ISD::ATOMIC_SWAP       , MVT::i32  , Custom);
+
   // We want to custom lower some of our intrinsics.
   setOperationAction(ISD::INTRINSIC_WO_CHAIN, MVT::Other, Custom);
   
@@ -393,6 +398,9 @@ const char *PPCTargetLowering::getTargetNodeName(unsigned Opcode) const {
   case PPCISD::VCMPo:         return "PPCISD::VCMPo";
   case PPCISD::LBRX:          return "PPCISD::LBRX";
   case PPCISD::STBRX:         return "PPCISD::STBRX";
+  case PPCISD::LWARX:         return "PPCISD::LWARX";
+  case PPCISD::STWCX:         return "PPCISD::STWCX";
+  case PPCISD::CMP_UNRESERVE: return "PPCISD::CMP_UNRESERVE";
   case PPCISD::COND_BRANCH:   return "PPCISD::COND_BRANCH";
   case PPCISD::MFFS:          return "PPCISD::MFFS";
   case PPCISD::MTFSB0:        return "PPCISD::MTFSB0";
@@ -2295,6 +2303,117 @@ SDOperand PPCTargetLowering::LowerDYNAMIC_STACKALLOC(SDOperand Op,
   return DAG.getNode(PPCISD::DYNALLOC, VTs, Ops, 3);
 }
 
+SDOperand PPCTargetLowering::LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG) {
+  MVT::ValueType VT = Op.getValueType();
+  SDOperand Chain   = Op.getOperand(0);
+  SDOperand Ptr     = Op.getOperand(1);
+  SDOperand Incr    = Op.getOperand(2);
+
+  // Issue a "load and reserve".
+  std::vector<MVT::ValueType> VTs;
+  VTs.push_back(VT);
+  VTs.push_back(MVT::Other);
+
+  SDOperand Label  = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32);
+  SDOperand Ops[] = {
+    Chain,  // Chain
+    Ptr,    // Ptr
+    Label,  // Label
+  };
+  SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3);
+  Chain = Load.getValue(1);
+
+  // Compute new value.
+  SDOperand NewVal  = DAG.getNode(ISD::ADD, VT, Load, Incr);
+
+  // Issue a "store and check".
+  SDOperand Ops2[] = {
+    Chain,  // Chain
+    NewVal, // Value
+    Ptr,    // Ptr
+    Label,  // Label
+  };
+  SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops2, 4);
+  SDOperand OutOps[] = { Load, Store };
+  return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+                     OutOps, 2);
+}
+
+SDOperand PPCTargetLowering::LowerAtomicLCS(SDOperand Op, SelectionDAG &DAG) {
+  MVT::ValueType VT = Op.getValueType();
+  SDOperand Chain   = Op.getOperand(0);
+  SDOperand Ptr     = Op.getOperand(1);
+  SDOperand NewVal  = Op.getOperand(2);
+  SDOperand OldVal  = Op.getOperand(3);
+
+  // Issue a "load and reserve".
+  std::vector<MVT::ValueType> VTs;
+  VTs.push_back(VT);
+  VTs.push_back(MVT::Other);
+
+  SDOperand Label  = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32);
+  SDOperand Ops[] = {
+    Chain,  // Chain
+    Ptr,    // Ptr
+    Label,  // Label
+  };
+  SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3);
+  Chain = Load.getValue(1);
+
+  // Compare and unreserve if not equal.
+  SDOperand Ops2[] = {
+    Chain,  // Chain
+    OldVal, // Old value
+    Load,   // Value in memory
+    Label,  // Label
+  };
+  Chain = DAG.getNode(PPCISD::CMP_UNRESERVE, MVT::Other, Ops2, 4);
+
+  // Issue a "store and check".
+  SDOperand Ops3[] = {
+    Chain,  // Chain
+    NewVal, // Value
+    Ptr,    // Ptr
+    Label,  // Label
+  };
+  SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops3, 4);
+  SDOperand OutOps[] = { Load, Store };
+  return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+                     OutOps, 2);
+}
+
+SDOperand PPCTargetLowering::LowerAtomicSWAP(SDOperand Op, SelectionDAG &DAG) {
+  MVT::ValueType VT = Op.getValueType();
+  SDOperand Chain   = Op.getOperand(0);
+  SDOperand Ptr     = Op.getOperand(1);
+  SDOperand NewVal  = Op.getOperand(2);
+
+  // Issue a "load and reserve".
+  std::vector<MVT::ValueType> VTs;
+  VTs.push_back(VT);
+  VTs.push_back(MVT::Other);
+
+  SDOperand Label  = DAG.getConstant(PPCAtomicLabelIndex++, MVT::i32);
+  SDOperand Ops[] = {
+    Chain,  // Chain
+    Ptr,    // Ptr
+    Label,  // Label
+  };
+  SDOperand Load = DAG.getNode(PPCISD::LWARX, VTs, Ops, 3);
+  Chain = Load.getValue(1);
+
+  // Issue a "store and check".
+  SDOperand Ops2[] = {
+    Chain,  // Chain
+    NewVal, // Value
+    Ptr,    // Ptr
+    Label,  // Label
+  };
+  SDOperand Store = DAG.getNode(PPCISD::STWCX, MVT::Other, Ops2, 4);
+  SDOperand OutOps[] = { Load, Store };
+  return DAG.getNode(ISD::MERGE_VALUES, DAG.getVTList(VT, MVT::Other),
+                     OutOps, 2);
+}
 
 /// LowerSELECT_CC - Lower floating point select_cc's into fsel instruction when
 /// possible.
@@ -3404,6 +3523,10 @@ SDOperand PPCTargetLowering::LowerOperation(SDOperand Op, SelectionDAG &DAG) {
   case ISD::STACKRESTORE:       return LowerSTACKRESTORE(Op, DAG, PPCSubTarget);
   case ISD::DYNAMIC_STACKALLOC:
     return LowerDYNAMIC_STACKALLOC(Op, DAG, PPCSubTarget);
+
+  case ISD::ATOMIC_LAS:         return LowerAtomicLAS(Op, DAG);
+  case ISD::ATOMIC_LCS:         return LowerAtomicLCS(Op, DAG);
+  case ISD::ATOMIC_SWAP:        return LowerAtomicSWAP(Op, DAG);
     
   case ISD::SELECT_CC:          return LowerSELECT_CC(Op, DAG);
   case ISD::FP_TO_SINT:         return LowerFP_TO_SINT(Op, DAG);
diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h
index 61a2f255417..d9ced1b3366 100644
--- a/llvm/lib/Target/PowerPC/PPCISelLowering.h
+++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h
@@ -150,7 +150,19 @@ namespace llvm {
       FADDRTZ,
 
       /// MTFSF = F8RC, INFLAG - This moves the register into the FPSCR.
-      MTFSF
+      MTFSF,
+
+      /// LWARX = This corresponds to PPC lwarx instrcution: load word and
+      /// reserve indexed. This is used to implement atomic operations.
+      LWARX,
+
+      /// STWCX = This corresponds to PPC stwcx. instrcution: store word
+      /// conditional indexed. This is used to implement atomic operations.
+      STWCX,
+
+      /// CMP_UNRESERVE = Test for equality and "unreserve" if not true. This
+      /// is used to implement atomic operations.
+      CMP_UNRESERVE
     };
   }
 
@@ -296,6 +308,11 @@ namespace llvm {
     /// the offset of the target addressing mode.
     virtual bool isLegalAddressImmediate(GlobalValue *GV) const;
 
+  private:
+    /// PPCAtomicLabelIndex - Keep track the number of PPC atomic labels.
+    ///
+    unsigned PPCAtomicLabelIndex;
+
     SDOperand LowerRETURNADDR(SDOperand Op, SelectionDAG &DAG);
     SDOperand LowerFRAMEADDR(SDOperand Op, SelectionDAG &DAG);
     SDOperand LowerConstantPool(SDOperand Op, SelectionDAG &DAG);
@@ -324,6 +341,9 @@ namespace llvm {
     SDOperand LowerDYNAMIC_STACKALLOC(SDOperand Op, SelectionDAG &DAG,
                                       const PPCSubtarget &Subtarget);
     SDOperand LowerSELECT_CC(SDOperand Op, SelectionDAG &DAG);
+    SDOperand LowerAtomicLAS(SDOperand Op, SelectionDAG &DAG);
+    SDOperand LowerAtomicLCS(SDOperand Op, SelectionDAG &DAG);
+    SDOperand LowerAtomicSWAP(SDOperand Op, SelectionDAG &DAG);
     SDOperand LowerFP_TO_SINT(SDOperand Op, SelectionDAG &DAG);
     SDOperand LowerSINT_TO_FP(SDOperand Op, SelectionDAG &DAG);
     SDOperand LowerFP_ROUND_INREG(SDOperand Op, SelectionDAG &DAG);
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.td b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
index f51158d3b1a..7cbdac3c1c1 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.td
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.td
@@ -42,6 +42,16 @@ def SDT_PPCstbrx : SDTypeProfile<0, 4, [
   SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, OtherVT>, SDTCisVT<3, OtherVT>
 ]>;
 
+def SDT_PPClwarx : SDTypeProfile<1, 2, [
+  SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, i32>
+]>;
+def SDT_PPCstwcx : SDTypeProfile<0, 3, [
+  SDTCisVT<0, i32>, SDTCisPtrTy<1>, SDTCisVT<2, i32>
+]>;
+def SDT_PPCcmp_unres : SDTypeProfile<0, 3, [
+  SDTCisVT<0, i32>, SDTCisSameAs<0, 1>, SDTCisSameAs<1, 2>
+]>;
+
 //===----------------------------------------------------------------------===//
 // PowerPC specific DAG Nodes.
 //
@@ -122,6 +132,13 @@ def PPClbrx       : SDNode<"PPCISD::LBRX", SDT_PPClbrx,
 def PPCstbrx      : SDNode<"PPCISD::STBRX", SDT_PPCstbrx,
                            [SDNPHasChain, SDNPMayStore]>;
 
+def PPClwarx      : SDNode<"PPCISD::LWARX", SDT_PPClwarx,
+                           [SDNPHasChain, SDNPMayLoad]>;
+def PPCstwcx      : SDNode<"PPCISD::STWCX", SDT_PPCstwcx,
+                           [SDNPHasChain, SDNPMayStore]>;
+def PPCcmp_unres  : SDNode<"PPCISD::CMP_UNRESERVE", SDT_PPCcmp_unres,
+                           [SDNPHasChain]>;
+
 // Instructions to support dynamic alloca.
 def SDTDynOp  : SDTypeProfile<1, 2, []>;
 def PPCdynalloc   : SDNode<"PPCISD::DYNALLOC", SDTDynOp, [SDNPHasChain]>;
@@ -462,6 +479,24 @@ def DCBZL  : DCB_Form<1014, 1, (outs), (ins memrr:$dst),
                       "dcbzl $dst", LdStDCBF, [(int_ppc_dcbzl xoaddr:$dst)]>,
                       PPC970_DGroup_Single;
 
+// Atomic operations.
+def LWARX : Pseudo<(outs GPRC:$rD), (ins memrr:$ptr, i32imm:$label),
+                   "\nLa${label}_entry:\n\tlwarx $rD, $ptr",
+                   [(set GPRC:$rD, (PPClwarx xoaddr:$ptr, imm:$label))]>;
+
+let Defs = [CR0] in {
+def STWCX : Pseudo<(outs), (ins GPRC:$rS, memrr:$dst, i32imm:$label),
+                   "stwcx. $rS, $dst\n\tbne- La${label}_entry\nLa${label}_exit:",
+                   [(PPCstwcx GPRC:$rS, xoaddr:$dst, imm:$label)]>;
+
+def CMP_UNRESw : Pseudo<(outs), (ins GPRC:$rA, GPRC:$rB, i32imm:$label),
+                         "cmpw $rA, $rB\n\tbne- La${label}_exit",
+                         [(PPCcmp_unres GPRC:$rA, GPRC:$rB, imm:$label)]>;
+def CMP_UNRESwi : Pseudo<(outs), (ins GPRC:$rA, s16imm:$imm, i32imm:$label),
+                         "cmpwi $rA, $imm\n\tbne- La${label}_exit",
+                         [(PPCcmp_unres GPRC:$rA, imm:$imm, imm:$label)]>;
+}
+
 //===----------------------------------------------------------------------===//
 // PPC32 Load Instructions.
 //
@@ -1229,5 +1264,9 @@ def : Pat<(extloadf32 iaddr:$src),
 def : Pat<(extloadf32 xaddr:$src),
           (FMRSD (LFSX xaddr:$src))>;
 
+// Atomic operations
+def : Pat<(PPCcmp_unres imm:$imm, GPRC:$rA, imm:$label),
+          (CMP_UNRESwi GPRC:$rA, imm:$imm, imm:$label)>;
+
 include "PPCInstrAltivec.td"
 include "PPCInstr64Bit.td"