[AMDGPU] Fix bundle scheduling

Bundles coming to scheduler considered free, i.e. zero latency.
Fixed.

Differential Revision: https://reviews.llvm.org/D72487

1 files changed, 40 insertions, 0 deletions

diff --git a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
index 9a726c320ff..46aea16a2be 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUSubtarget.cpp
@@ -755,6 +755,45 @@ struct MemOpClusterMutation : ScheduleDAGMutation {
   }
 };
 
+struct FixBundleLatencyMutation : ScheduleDAGMutation {
+  const SIInstrInfo *TII;
+
+  const TargetSchedModel *TSchedModel;
+
+  FixBundleLatencyMutation(const SIInstrInfo *tii) : TII(tii) {}
+
+  unsigned computeLatency(const MachineInstr &MI, unsigned Reg) const {
+    const SIRegisterInfo &TRI = TII->getRegisterInfo();
+    MachineBasicBlock::const_instr_iterator I(MI.getIterator());
+    MachineBasicBlock::const_instr_iterator E(MI.getParent()->instr_end());
+    unsigned Lat = 0;
+    for (++I; I != E && I->isBundledWithPred(); ++I) {
+      if (!I->modifiesRegister(Reg, &TRI))
+        continue;
+      Lat = TSchedModel->computeInstrLatency(&*I);
+      break;
+    }
+    return Lat;
+  }
+
+  void apply(ScheduleDAGInstrs *DAGInstrs) override {
+    ScheduleDAGMI *DAG = static_cast<ScheduleDAGMI*>(DAGInstrs);
+    TSchedModel = DAGInstrs->getSchedModel();
+    if (!TSchedModel || DAG->SUnits.empty())
+      return;
+
+    for (SUnit &SU : DAG->SUnits) {
+      if (!SU.isInstr() || !SU.getInstr()->isBundle())
+        continue;
+      for (SDep &Dep : SU.Succs) {
+        if (Dep.getKind() == SDep::Kind::Data && Dep.getReg())
+          if (unsigned Lat = computeLatency(*SU.getInstr(), Dep.getReg()))
+            Dep.setLatency(Lat);
+      }
+    }
+  }
+};
+
 struct FillMFMAShadowMutation : ScheduleDAGMutation {
   const SIInstrInfo *TII;
 
@@ -881,6 +920,7 @@ struct FillMFMAShadowMutation : ScheduleDAGMutation {
 
 void GCNSubtarget::getPostRAMutations(
     std::vector<std::unique_ptr<ScheduleDAGMutation>> &Mutations) const {
+  Mutations.push_back(std::make_unique<FixBundleLatencyMutation>(&InstrInfo));
   Mutations.push_back(std::make_unique<MemOpClusterMutation>(&InstrInfo));
   Mutations.push_back(std::make_unique<FillMFMAShadowMutation>(&InstrInfo));
 }