diff options
| -rw-r--r-- | llvm/include/llvm/MCA/Instruction.h | 52 | ||||
| -rw-r--r-- | llvm/lib/MCA/HardwareUnits/Scheduler.cpp | 72 | ||||
| -rw-r--r-- | llvm/lib/MCA/Instruction.cpp | 24 | ||||
| -rw-r--r-- | llvm/lib/MCA/Stages/DispatchStage.cpp | 3 |
4 files changed, 103 insertions, 48 deletions
diff --git a/llvm/include/llvm/MCA/Instruction.h b/llvm/include/llvm/MCA/Instruction.h index 9ac1fffb443..74be06e3c1c 100644 --- a/llvm/include/llvm/MCA/Instruction.h +++ b/llvm/include/llvm/MCA/Instruction.h @@ -80,11 +80,10 @@ struct ReadDescriptor { class ReadState; -/// Longest register dependency. +/// A critical data dependency descriptor. /// -/// Used internally by WriteState/ReadState/InstructionBase to help with the -/// computation of the longest register dependency for an instruction. -struct CriticalRegDep { +/// Field RegID is set to the invalid register for memory dependencies. +struct CriticalDependency { unsigned IID; unsigned RegID; unsigned Cycles; @@ -136,7 +135,7 @@ class WriteState { unsigned DependentWriteCyclesLeft; // Critical register dependency for this write. - CriticalRegDep CRD; + CriticalDependency CRD; // A list of dependent reads. Users is a set of dependent // reads. A dependent read is added to the set only if CyclesLeft @@ -166,7 +165,7 @@ public: return DependentWriteCyclesLeft; } const WriteState *getDependentWrite() const { return DependentWrite; } - const CriticalRegDep &getCriticalRegDep() const { return CRD; } + const CriticalDependency &getCriticalRegDep() const { return CRD; } // This method adds Use to the set of data dependent reads. IID is the // instruction identifier associated with this write. ReadAdvance is the @@ -244,7 +243,7 @@ class ReadState { // propagated to field CyclesLeft. unsigned TotalCycles; // Longest register dependency. - CriticalRegDep CRD; + CriticalDependency CRD; // This field is set to true only if there are no dependent writes, and // there are no `CyclesLeft' to wait. bool IsReady; @@ -263,7 +262,7 @@ public: unsigned getSchedClass() const { return RD->SchedClassID; } unsigned getRegisterID() const { return RegisterID; } unsigned getRegisterFileID() const { return PRFID; } - const CriticalRegDep &getCriticalRegDep() const { return CRD; } + const CriticalDependency &getCriticalRegDep() const { return CRD; } bool isPending() const { return !IndependentFromDef && CyclesLeft > 0; } bool isReady() const { return IsReady; } @@ -405,12 +404,8 @@ class InstructionBase { // One entry per each implicit and explicit register use. SmallVector<ReadState, 4> Uses; - // Critical register dependency. - CriticalRegDep CRD; - public: - InstructionBase(const InstrDesc &D) - : Desc(D), IsOptimizableMove(false), CRD() {} + InstructionBase(const InstrDesc &D) : Desc(D), IsOptimizableMove(false) {} SmallVectorImpl<WriteState> &getDefs() { return Defs; } const ArrayRef<WriteState> getDefs() const { return Defs; } @@ -420,9 +415,6 @@ public: unsigned getLatency() const { return Desc.MaxLatency; } - const CriticalRegDep &getCriticalRegDep() const { return CRD; } - const CriticalRegDep &computeCriticalRegDep(); - bool hasDependentUsers() const { return any_of(Defs, [](const WriteState &Def) { return Def.getNumUsers() > 0; }); @@ -466,14 +458,19 @@ class Instruction : public InstructionBase { // Retire Unit token ID for this instruction. unsigned RCUTokenID; + // Critical register dependency. + CriticalDependency CriticalRegDep; + + // Critical memory dependency. + CriticalDependency CriticalMemDep; + // A bitmask of busy processor resource units. // This field is set to zero only if execution is not delayed during this // cycle because of unavailable pipeline resources. uint64_t CriticalResourceMask; - // An instruction identifier. This field is only set if execution is delayed - // by a memory dependency. - unsigned CriticalMemDep; + // Used internally by the logic that computes the critical memory dependency. + const Instruction *CurrentMemDep; // True if this instruction has been optimized at register renaming stage. bool IsEliminated; @@ -481,8 +478,8 @@ class Instruction : public InstructionBase { public: Instruction(const InstrDesc &D) : InstructionBase(D), Stage(IS_INVALID), CyclesLeft(UNKNOWN_CYCLES), - RCUTokenID(0), CriticalResourceMask(0), CriticalMemDep(0), - IsEliminated(false) {} + RCUTokenID(0), CriticalRegDep(), CriticalMemDep(), + CriticalResourceMask(0), CurrentMemDep(nullptr), IsEliminated(false) {} unsigned getRCUTokenID() const { return RCUTokenID; } int getCyclesLeft() const { return CyclesLeft; } @@ -523,12 +520,21 @@ public: Stage = IS_RETIRED; } + const CriticalDependency &getCriticalRegDep() const { return CriticalRegDep; } + const CriticalDependency &getCriticalMemDep() const { return CriticalMemDep; } + const CriticalDependency &computeCriticalRegDep(); + + void setCriticalMemDep(unsigned IID, unsigned Cycles) { + CriticalMemDep.IID = IID; + CriticalMemDep.Cycles = Cycles; + } + const Instruction *getCurrentMemDep() const { return CurrentMemDep; } + void setCurrentMemDep(const Instruction *CMD) { CurrentMemDep = CMD; } + uint64_t getCriticalResourceMask() const { return CriticalResourceMask; } - unsigned getCriticalMemDep() const { return CriticalMemDep; } void setCriticalResourceMask(uint64_t ResourceMask) { CriticalResourceMask = ResourceMask; } - void setCriticalMemDep(unsigned IID) { CriticalMemDep = IID; } void cycleEvent(); }; diff --git a/llvm/lib/MCA/HardwareUnits/Scheduler.cpp b/llvm/lib/MCA/HardwareUnits/Scheduler.cpp index bf48d928899..b2928ed1b12 100644 --- a/llvm/lib/MCA/HardwareUnits/Scheduler.cpp +++ b/llvm/lib/MCA/HardwareUnits/Scheduler.cpp @@ -82,6 +82,8 @@ void Scheduler::issueInstructionImpl( // This updates the internal state of each write. IS->execute(IR.getSourceIndex()); + IS->computeCriticalRegDep(); + if (IS->isExecuting()) IssuedSet.emplace_back(IR); else if (IS->isExecuted()) @@ -107,6 +109,59 @@ void Scheduler::issueInstruction( promoteToReadySet(ReadyInstructions); } +static bool initializeCriticalMemDepInfo(InstRef &IR, const LSUnit &LSU) { + Instruction &IS = *IR.getInstruction(); + assert(IS.isMemOp() && "Not a memory operation!"); + + // Check if this instruction depends on another memory operation. + InstRef DependentMemOp = LSU.isReady(IR); + const Instruction *MemOp = DependentMemOp.getInstruction(); + IS.setCurrentMemDep(MemOp); + + // Initialize the CriticalMemDep structure. + unsigned Cycles = 0; + if (MemOp->isExecuting()) + Cycles = static_cast<unsigned>(MemOp->getCyclesLeft()); + IS.setCriticalMemDep(DependentMemOp.getSourceIndex(), Cycles); + return IR.getSourceIndex() == DependentMemOp.getSourceIndex(); +} + +static bool updateMemoryDependencyInfo(InstRef &IR, const LSUnit &LSU) { + Instruction &IS = *IR.getInstruction(); + assert(IS.isMemOp() && "Not a memory operation!"); + + const Instruction *MemOp = IS.getCurrentMemDep(); + if (!MemOp && initializeCriticalMemDepInfo(IR, LSU)) + return true; + + MemOp = IS.getCurrentMemDep(); + if (MemOp == IR.getInstruction()) + return true; + + const CriticalDependency &CMD = IS.getCriticalMemDep(); + if (MemOp->isExecuting() && !CMD.Cycles) { + // Update the critical memory dependency info. + IS.setCriticalMemDep(CMD.IID, MemOp->getCyclesLeft()); + return false; + } + + if (!MemOp->isExecuted() && !MemOp->isRetired()) + return false; + + // Check if there are still unsolved memory dependencies. + InstRef DependentMemOp = LSU.isReady(IR); + MemOp = DependentMemOp.getInstruction(); + IS.setCurrentMemDep(MemOp); + if (DependentMemOp == IR) + return true; + + unsigned Cycles = 0; + if (MemOp->isExecuting()) + Cycles = static_cast<unsigned>(MemOp->getCyclesLeft()); + IS.setCriticalMemDep(DependentMemOp.getSourceIndex(), Cycles); + return false; +} + bool Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) { // Scan the set of waiting instructions and promote them to the // ready set if operands are all ready. @@ -116,19 +171,14 @@ bool Scheduler::promoteToReadySet(SmallVectorImpl<InstRef> &Ready) { if (!IR) break; - // Check if there are still unsolved memory dependencies. + // Check if there are unsolved memory dependencies. Instruction &IS = *IR.getInstruction(); - if (IS.isMemOp()) { - const InstRef &CriticalMemDep = LSU.isReady(IR); - if (CriticalMemDep != IR) { - IS.setCriticalMemDep(CriticalMemDep.getSourceIndex()); - ++I; - continue; - } + if (IS.isMemOp() && !updateMemoryDependencyInfo(IR, LSU)) { + ++I; + continue; } - // Check if this instruction is now ready. In case, force - // a transition in state using method 'update()'. + // Check if there are unsolved register dependencies. if (!IS.isReady() && !IS.updatePending()) { ++I; continue; @@ -301,7 +351,7 @@ bool Scheduler::dispatch(const InstRef &IR) { } // Memory operations that are not in a ready state are initially assigned to - // the WaitSet. + // the WaitSet. if (!IS.isReady() || (IS.isMemOp() && LSU.isReady(IR) != IR)) { LLVM_DEBUG(dbgs() << "[SCHEDULER] Adding #" << IR << " to the WaitSet\n"); WaitSet.push_back(IR); diff --git a/llvm/lib/MCA/Instruction.cpp b/llvm/lib/MCA/Instruction.cpp index 5e2fb771e4f..001842bca31 100644 --- a/llvm/lib/MCA/Instruction.cpp +++ b/llvm/lib/MCA/Instruction.cpp @@ -18,7 +18,8 @@ namespace llvm { namespace mca { -void WriteState::writeStartEvent(unsigned IID, unsigned RegID, unsigned Cycles) { +void WriteState::writeStartEvent(unsigned IID, unsigned RegID, + unsigned Cycles) { CRD.IID = IID; CRD.RegID = RegID; CRD.Cycles = Cycles; @@ -134,23 +135,24 @@ void WriteRef::dump() const { } #endif -const CriticalRegDep &InstructionBase::computeCriticalRegDep() { - if (CRD.Cycles || (Defs.empty() && Uses.empty())) - return CRD; +const CriticalDependency &Instruction::computeCriticalRegDep() { + if (CriticalRegDep.Cycles) + return CriticalRegDep; + unsigned MaxLatency = 0; - for (const WriteState &WS : Defs) { - const CriticalRegDep &WriteCRD = WS.getCriticalRegDep(); + for (const WriteState &WS : getDefs()) { + const CriticalDependency &WriteCRD = WS.getCriticalRegDep(); if (WriteCRD.Cycles > MaxLatency) - CRD = WriteCRD; + CriticalRegDep = WriteCRD; } - for (const ReadState &RS : Uses) { - const CriticalRegDep &ReadCRD = RS.getCriticalRegDep(); + for (const ReadState &RS : getUses()) { + const CriticalDependency &ReadCRD = RS.getCriticalRegDep(); if (ReadCRD.Cycles > MaxLatency) - CRD = ReadCRD; + CriticalRegDep = ReadCRD; } - return CRD; + return CriticalRegDep; } void Instruction::dispatch(unsigned RCUToken) { diff --git a/llvm/lib/MCA/Stages/DispatchStage.cpp b/llvm/lib/MCA/Stages/DispatchStage.cpp index 80d6da09b5e..7334a268e9a 100644 --- a/llvm/lib/MCA/Stages/DispatchStage.cpp +++ b/llvm/lib/MCA/Stages/DispatchStage.cpp @@ -102,9 +102,6 @@ Error DispatchStage::dispatch(InstRef IR) { IS.setEliminated(); } - if (IS.isMemOp()) - IS.setCriticalMemDep(IR.getSourceIndex()); - // A dependency-breaking instruction doesn't have to wait on the register // input operands, and it is often optimized at register renaming stage. // Update RAW dependencies if this instruction is not a dependency-breaking |
