[AArch64] Add MMOs to callee-save load/store instructions.

Summary:
Without MMOs, the callee-save load/store instructions were treated as
volatile by the MI post-RA scheduler and AArch64LoadStoreOptimizer.

Reviewers: t.p.northover, mcrosier

Subscribers: aemerson, rengolin, mcrosier, llvm-commits

Differential Revision: http://reviews.llvm.org/D17661

llvm-svn: 266439

2 files changed, 38 insertions, 2 deletions

diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
index 57e320ab267..f567b624beb 100644
--- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp
@@ -871,6 +871,9 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
         .addReg(AArch64::SP)
         .addImm(Offset) // [sp, #offset * 8], where factor * 8 is implicit
         .setMIFlag(MachineInstr::FrameSetup);
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
+          MachineMemOperand::MOStore, 8, 8));
     } else {
       MBB.addLiveIn(Reg1);
       MIB.addReg(Reg1, getPrologueDeath(MF, Reg1))
@@ -878,6 +881,9 @@ bool AArch64FrameLowering::spillCalleeSavedRegisters(
         .addImm(BumpSP ? Offset * 8 : Offset) // pre-inc version is unscaled
         .setMIFlag(MachineInstr::FrameSetup);
     }
+    MIB.addMemOperand(MF.getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
+        MachineMemOperand::MOStore, 8, 8));
   }
   return true;
 }
@@ -935,18 +941,25 @@ bool AArch64FrameLowering::restoreCalleeSavedRegisters(
     if (BumpSP)
       MIB.addReg(AArch64::SP, RegState::Define);
 
-    if (RPI.isPaired())
+    if (RPI.isPaired()) {
       MIB.addReg(Reg2, getDefRegState(true))
         .addReg(Reg1, getDefRegState(true))
         .addReg(AArch64::SP)
         .addImm(Offset) // [sp], #offset * 8  or [sp, #offset * 8]
                         // where the factor * 8 is implicit
         .setMIFlag(MachineInstr::FrameDestroy);
-    else
+      MIB.addMemOperand(MF.getMachineMemOperand(
+          MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx + 1),
+          MachineMemOperand::MOLoad, 8, 8));
+    } else {
       MIB.addReg(Reg1, getDefRegState(true))
         .addReg(AArch64::SP)
         .addImm(BumpSP ? Offset * 8 : Offset) // post-dec version is unscaled
         .setMIFlag(MachineInstr::FrameDestroy);
+    }
+    MIB.addMemOperand(MF.getMachineMemOperand(
+        MachinePointerInfo::getFixedStack(MF, RPI.FrameIdx),
+        MachineMemOperand::MOLoad, 8, 8));
   }
   return true;
 }
diff --git a/llvm/test/CodeGen/AArch64/arm64-csldst-mmo.ll b/llvm/test/CodeGen/AArch64/arm64-csldst-mmo.ll
new file mode 100644
index 00000000000..69a9510c935
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/arm64-csldst-mmo.ll
@@ -0,0 +1,23 @@
+; RUN: llc < %s -mtriple=arm64-linux-gnu -mcpu=cortex-a57 -enable-misched=0 -debug-only=misched -o - 2>&1 > /dev/null | FileCheck %s
+
+@G = external global [0 x i32], align 4
+
+; Check that MMOs are added to epilogue calle-save restore loads so
+; that the store to G is not considered dependant on the callee-save
+; loads.
+;
+; CHECK: Before post-MI-sched:
+; CHECK-LABEL: # Machine code for function test1:
+; CHECK: SU(2):   STRWui %WZR
+; CHECK: SU(3):   %X21<def>, %X20<def> = LDPXi %SP
+; CHECK:  Predecessors:
+; CHECK-NEXT:   out SU(0)
+; CHECK-NEXT:   out SU(0)
+; CHECK-NEXT:   ch  SU(0)
+; CHECK-NEXT:  Successors:
+define void @test1() {
+entry:
+  tail call void asm sideeffect "nop", "~{x20},~{x21},~{x22},~{x23}"() nounwind
+  store i32 0, i32* getelementptr inbounds ([0 x i32], [0 x i32]* @G, i64 0, i64 0), align 4
+  ret void
+}