AMDGPU: Merge initial gfx9 support

llvm-svn: 295554

1 files changed, 72 insertions, 6 deletions

diff --git a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
index 16b0b788318..fd61530b718 100644
--- a/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNHazardRecognizer.cpp
@@ -71,6 +71,18 @@ static bool isRFE(unsigned Opcode) {
   return Opcode == AMDGPU::S_RFE_B64;
 }
 
+static bool isSMovRel(unsigned Opcode) {
+  return Opcode == AMDGPU::S_MOVRELS_B32 || AMDGPU::S_MOVRELS_B64 ||
+         Opcode == AMDGPU::S_MOVRELD_B32 || AMDGPU::S_MOVRELD_B64;
+}
+
+static bool isVInterp(unsigned Opcode) {
+  return Opcode == AMDGPU::V_INTERP_P1_F32 ||
+         Opcode == AMDGPU::V_INTERP_P1_F32_16bank ||
+         Opcode == AMDGPU::V_INTERP_P2_F32 ||
+         Opcode == AMDGPU::V_INTERP_MOV_F32;
+}
+
 static unsigned getHWReg(const SIInstrInfo *TII, const MachineInstr &RegInstr) {
   const MachineOperand *RegOp = TII->getNamedOperand(RegInstr,
                                                      AMDGPU::OpName::simm16);
@@ -108,6 +120,13 @@ GCNHazardRecognizer::getHazardType(SUnit *SU, int Stalls) {
   if (isRFE(MI->getOpcode()) && checkRFEHazards(MI) > 0)
     return NoopHazard;
 
+  if ((isVInterp(MI->getOpcode()) || isSMovRel(MI->getOpcode())) &&
+      checkReadM0Hazards(MI) > 0)
+    return NoopHazard;
+
+  if (checkAnyInstHazards(MI) > 0)
+    return NoopHazard;
+
   return NoHazard;
 }
 
@@ -116,11 +135,13 @@ unsigned GCNHazardRecognizer::PreEmitNoops(SUnit *SU) {
 }
 
 unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
+  int WaitStates = std::max(0, checkAnyInstHazards(MI));
+
   if (SIInstrInfo::isSMRD(*MI))
-    return std::max(0, checkSMRDHazards(MI));
+    return std::max(WaitStates, checkSMRDHazards(MI));
 
   if (SIInstrInfo::isVALU(*MI)) {
-    int WaitStates = std::max(0, checkVALUHazards(MI));
+      WaitStates = std::max(WaitStates, checkVALUHazards(MI));
 
     if (SIInstrInfo::isVMEM(*MI))
       WaitStates = std::max(WaitStates, checkVMEMHazards(MI));
@@ -134,19 +155,25 @@ unsigned GCNHazardRecognizer::PreEmitNoops(MachineInstr *MI) {
     if (isRWLane(MI->getOpcode()))
       WaitStates = std::max(WaitStates, checkRWLaneHazards(MI));
 
+    if (isVInterp(MI->getOpcode()))
+      WaitStates = std::max(WaitStates, checkReadM0Hazards(MI));
+
     return WaitStates;
   }
 
   if (isSGetReg(MI->getOpcode()))
-    return std::max(0, checkGetRegHazards(MI));
+    return std::max(WaitStates, checkGetRegHazards(MI));
 
   if (isSSetReg(MI->getOpcode()))
-    return std::max(0, checkSetRegHazards(MI));
+    return std::max(WaitStates, checkSetRegHazards(MI));
 
   if (isRFE(MI->getOpcode()))
-    return std::max(0, checkRFEHazards(MI));
+    return std::max(WaitStates, checkRFEHazards(MI));
 
-  return 0;
+  if (isSMovRel(MI->getOpcode()))
+    return std::max(WaitStates, checkReadM0Hazards(MI));
+
+  return WaitStates;
 }
 
 void GCNHazardRecognizer::EmitNoop() {
@@ -508,3 +535,42 @@ int GCNHazardRecognizer::checkRFEHazards(MachineInstr *RFE) {
   int WaitStatesNeeded = getWaitStatesSinceSetReg(IsHazardFn);
   return RFEWaitStates - WaitStatesNeeded;
 }
+
+int GCNHazardRecognizer::checkAnyInstHazards(MachineInstr *MI) {
+  if (MI->isDebugValue())
+    return 0;
+
+  const SIRegisterInfo *TRI = ST.getRegisterInfo();
+  if (!ST.hasSMovFedHazard())
+    return 0;
+
+  // Check for any instruction reading an SGPR after a write from
+  // s_mov_fed_b32.
+  int MovFedWaitStates = 1;
+  int WaitStatesNeeded = 0;
+
+  for (const MachineOperand &Use : MI->uses()) {
+    if (!Use.isReg() || TRI->isVGPR(MF.getRegInfo(), Use.getReg()))
+      continue;
+    auto IsHazardFn = [] (MachineInstr *MI) {
+      return MI->getOpcode() == AMDGPU::S_MOV_FED_B32;
+    };
+    int WaitStatesNeededForUse =
+        MovFedWaitStates - getWaitStatesSinceDef(Use.getReg(), IsHazardFn);
+    WaitStatesNeeded = std::max(WaitStatesNeeded, WaitStatesNeededForUse);
+  }
+
+  return WaitStatesNeeded;
+}
+
+int GCNHazardRecognizer::checkReadM0Hazards(MachineInstr *MI) {
+  if (!ST.hasReadM0Hazard())
+    return 0;
+
+  const SIInstrInfo *TII = ST.getInstrInfo();
+  int SMovRelWaitStates = 1;
+  auto IsHazardFn = [TII] (MachineInstr *MI) {
+    return TII->isSALU(*MI);
+  };
+  return SMovRelWaitStates - getWaitStatesSinceDef(AMDGPU::M0, IsHazardFn);
+}