summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/MachinePipeliner.cpp16
-rw-r--r--llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp10
-rw-r--r--llvm/lib/Target/Hexagon/HexagonInstrInfo.h2
-rw-r--r--llvm/lib/Target/PowerPC/PPCInstrInfo.cpp74
-rw-r--r--llvm/lib/Target/PowerPC/PPCInstrInfo.h28
-rw-r--r--llvm/lib/Target/PowerPC/PPCSubtarget.cpp13
-rw-r--r--llvm/lib/Target/PowerPC/PPCSubtarget.h8
-rw-r--r--llvm/lib/Target/PowerPC/PPCTargetMachine.cpp3
8 files changed, 140 insertions, 14 deletions
diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp
index 639d124804c..81917abae08 100644
--- a/llvm/lib/CodeGen/MachinePipeliner.cpp
+++ b/llvm/lib/CodeGen/MachinePipeliner.cpp
@@ -187,6 +187,9 @@ bool MachinePipeliner::runOnMachineFunction(MachineFunction &mf) {
!EnableSWPOptSize.getPosition())
return false;
+ if (!mf.getSubtarget().enableMachinePipeliner())
+ return false;
+
// Cannot pipeline loops without instruction itineraries if we are using
// DFA for the pipeliner.
if (mf.getSubtarget().useDFAforSMS() &&
@@ -2026,6 +2029,10 @@ void SwingSchedulerDAG::generatePipelinedLoop(SMSchedule &Schedule) {
InstrMapTy InstrMap;
SmallVector<MachineBasicBlock *, 4> PrologBBs;
+
+ MachineBasicBlock *PreheaderBB = MLI->getLoopFor(BB)->getLoopPreheader();
+ assert(PreheaderBB != nullptr &&
+ "Need to add code to handle loops w/o preheader");
// Generate the prolog instructions that set up the pipeline.
generateProlog(Schedule, MaxStageCount, KernelBB, VRMap, PrologBBs);
MF.insert(BB->getIterator(), KernelBB);
@@ -2082,7 +2089,7 @@ void SwingSchedulerDAG::generatePipelinedLoop(SMSchedule &Schedule) {
removeDeadInstructions(KernelBB, EpilogBBs);
// Add branches between prolog and epilog blocks.
- addBranches(PrologBBs, KernelBB, EpilogBBs, Schedule, VRMap);
+ addBranches(*PreheaderBB, PrologBBs, KernelBB, EpilogBBs, Schedule, VRMap);
// Remove the original loop since it's no longer referenced.
for (auto &I : *BB)
@@ -2767,7 +2774,8 @@ static void removePhis(MachineBasicBlock *BB, MachineBasicBlock *Incoming) {
/// Create branches from each prolog basic block to the appropriate epilog
/// block. These edges are needed if the loop ends before reaching the
/// kernel.
-void SwingSchedulerDAG::addBranches(MBBVectorTy &PrologBBs,
+void SwingSchedulerDAG::addBranches(MachineBasicBlock &PreheaderBB,
+ MBBVectorTy &PrologBBs,
MachineBasicBlock *KernelBB,
MBBVectorTy &EpilogBBs,
SMSchedule &Schedule, ValueMapTy *VRMap) {
@@ -2794,8 +2802,8 @@ void SwingSchedulerDAG::addBranches(MBBVectorTy &PrologBBs,
// Check if the LOOP0 has already been removed. If so, then there is no need
// to reduce the trip count.
if (LC != 0)
- LC = TII->reduceLoopCount(*Prolog, IndVar, *Cmp, Cond, PrevInsts, j,
- MaxIter);
+ LC = TII->reduceLoopCount(*Prolog, PreheaderBB, IndVar, *Cmp, Cond,
+ PrevInsts, j, MaxIter);
// Record the value of the first trip count, which is used to determine if
// branches and blocks can be removed for constant trip counts.
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
index d525baa0301..a156de5ba12 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.cpp
@@ -697,11 +697,11 @@ bool HexagonInstrInfo::analyzeLoop(MachineLoop &L,
/// Generate code to reduce the loop iteration by one and check if the loop is
/// finished. Return the value/register of the new loop count. this function
/// assumes the nth iteration is peeled first.
-unsigned HexagonInstrInfo::reduceLoopCount(MachineBasicBlock &MBB,
- MachineInstr *IndVar, MachineInstr &Cmp,
- SmallVectorImpl<MachineOperand> &Cond,
- SmallVectorImpl<MachineInstr *> &PrevInsts,
- unsigned Iter, unsigned MaxIter) const {
+unsigned HexagonInstrInfo::reduceLoopCount(
+ MachineBasicBlock &MBB, MachineBasicBlock &PreHeader, MachineInstr *IndVar,
+ MachineInstr &Cmp, SmallVectorImpl<MachineOperand> &Cond,
+ SmallVectorImpl<MachineInstr *> &PrevInsts, unsigned Iter,
+ unsigned MaxIter) const {
// We expect a hardware loop currently. This means that IndVar is set
// to null, and the compare is the ENDLOOP instruction.
assert((!IndVar) && isEndLoopN(Cmp.getOpcode())
diff --git a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
index 6c4dbfedb31..e0a999d0f4c 100644
--- a/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
+++ b/llvm/lib/Target/Hexagon/HexagonInstrInfo.h
@@ -139,7 +139,7 @@ public:
/// is finished. Return the value/register of the new loop count. We need
/// this function when peeling off one or more iterations of a loop. This
/// function assumes the nth iteration is peeled first.
- unsigned reduceLoopCount(MachineBasicBlock &MBB,
+ unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineBasicBlock &PreHeader,
MachineInstr *IndVar, MachineInstr &Cmp,
SmallVectorImpl<MachineOperand> &Cond,
SmallVectorImpl<MachineInstr *> &PrevInsts,
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
index 0799c4281e3..7aede06b73d 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.cpp
@@ -3922,3 +3922,77 @@ PPCInstrInfo::isSignOrZeroExtended(const MachineInstr &MI, bool SignExt,
}
return false;
}
+
+bool PPCInstrInfo::isBDNZ(unsigned Opcode) const {
+ return (Opcode == (Subtarget.isPPC64() ? PPC::BDNZ8 : PPC::BDNZ));
+}
+
+bool PPCInstrInfo::analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst,
+ MachineInstr *&CmpInst) const {
+ MachineBasicBlock *LoopEnd = L.getBottomBlock();
+ MachineBasicBlock::iterator I = LoopEnd->getFirstTerminator();
+ // We really "analyze" only CTR loops right now.
+ if (I != LoopEnd->end() && isBDNZ(I->getOpcode())) {
+ IndVarInst = nullptr;
+ CmpInst = &*I;
+ return false;
+ }
+ return true;
+}
+
+MachineInstr *
+PPCInstrInfo::findLoopInstr(MachineBasicBlock &PreHeader) const {
+
+ unsigned LOOPi = (Subtarget.isPPC64() ? PPC::MTCTR8loop : PPC::MTCTRloop);
+
+ // The loop set-up instruction should be in preheader
+ for (auto &I : PreHeader.instrs())
+ if (I.getOpcode() == LOOPi)
+ return &I;
+ return nullptr;
+}
+
+unsigned PPCInstrInfo::reduceLoopCount(
+ MachineBasicBlock &MBB, MachineBasicBlock &PreHeader, MachineInstr *IndVar,
+ MachineInstr &Cmp, SmallVectorImpl<MachineOperand> &Cond,
+ SmallVectorImpl<MachineInstr *> &PrevInsts, unsigned Iter,
+ unsigned MaxIter) const {
+ // We expect a hardware loop currently. This means that IndVar is set
+ // to null, and the compare is the ENDLOOP instruction.
+ assert((!IndVar) && isBDNZ(Cmp.getOpcode()) && "Expecting a CTR loop");
+ MachineFunction *MF = MBB.getParent();
+ DebugLoc DL = Cmp.getDebugLoc();
+ MachineInstr *Loop = findLoopInstr(PreHeader);
+ if (!Loop)
+ return 0;
+ unsigned LoopCountReg = Loop->getOperand(0).getReg();
+ MachineRegisterInfo &MRI = MF->getRegInfo();
+ MachineInstr *LoopCount = MRI.getUniqueVRegDef(LoopCountReg);
+
+ if (!LoopCount)
+ return 0;
+ // If the loop trip count is a compile-time value, then just change the
+ // value.
+ if (LoopCount->getOpcode() == PPC::LI8 || LoopCount->getOpcode() == PPC::LI) {
+ int64_t Offset = LoopCount->getOperand(1).getImm();
+ if (Offset <= 1) {
+ LoopCount->eraseFromParent();
+ Loop->eraseFromParent();
+ return 0;
+ }
+ LoopCount->getOperand(1).setImm(Offset - 1);
+ return Offset - 1;
+ }
+
+ // The loop trip count is a run-time value.
+ // We need to subtract one from the trip count,
+ // and insert branch later to check if we're done with the loop.
+
+ // Since BDZ/BDZ8 that we will insert will also decrease the ctr by 1,
+ // so we don't need to generate any thing here.
+ Cond.push_back(MachineOperand::CreateImm(0));
+ Cond.push_back(MachineOperand::CreateReg(
+ Subtarget.isPPC64() ? PPC::CTR8 : PPC::CTR, true));
+ return LoopCountReg;
+}
+
diff --git a/llvm/lib/Target/PowerPC/PPCInstrInfo.h b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
index da34e70989d..78b04c2c1ca 100644
--- a/llvm/lib/Target/PowerPC/PPCInstrInfo.h
+++ b/llvm/lib/Target/PowerPC/PPCInstrInfo.h
@@ -457,6 +457,34 @@ public:
}
return Reg;
}
+
+ /// Check \p Opcode is BDNZ (Decrement CTR and branch if it is still nonzero).
+ bool isBDNZ(unsigned Opcode) const;
+
+ /// Find the hardware loop instruction used to set-up the specified loop.
+ /// On PPC, we have two instructions used to set-up the hardware loop
+ /// (MTCTRloop, MTCTR8loop) with corresponding endloop (BDNZ, BDNZ8)
+ /// instructions to indicate the end of a loop.
+ MachineInstr *findLoopInstr(MachineBasicBlock &PreHeader) const;
+
+ /// Analyze the loop code to find the loop induction variable and compare used
+ /// to compute the number of iterations. Currently, we analyze loop that are
+ /// controlled using hardware loops. In this case, the induction variable
+ /// instruction is null. For all other cases, this function returns true,
+ /// which means we're unable to analyze it. \p IndVarInst and \p CmpInst will
+ /// return new values when we can analyze the readonly loop \p L, otherwise,
+ /// nothing got changed
+ bool analyzeLoop(MachineLoop &L, MachineInstr *&IndVarInst,
+ MachineInstr *&CmpInst) const override;
+ /// Generate code to reduce the loop iteration by one and check if the loop
+ /// is finished. Return the value/register of the new loop count. We need
+ /// this function when peeling off one or more iterations of a loop. This
+ /// function assumes the last iteration is peeled first.
+ unsigned reduceLoopCount(MachineBasicBlock &MBB, MachineBasicBlock &PreHeader,
+ MachineInstr *IndVar, MachineInstr &Cmp,
+ SmallVectorImpl<MachineOperand> &Cond,
+ SmallVectorImpl<MachineInstr *> &PrevInsts,
+ unsigned Iter, unsigned MaxIter) const override;
};
}
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
index e3bc305be7a..0d2786e8443 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.cpp
@@ -39,6 +39,11 @@ static cl::opt<bool> QPXStackUnaligned("qpx-stack-unaligned",
cl::desc("Even when QPX is enabled the stack is not 32-byte aligned"),
cl::Hidden);
+static cl::opt<bool>
+ EnableMachinePipeliner("ppc-enable-pipeliner",
+ cl::desc("Enable Machine Pipeliner for PPC"),
+ cl::init(false), cl::Hidden);
+
PPCSubtarget &PPCSubtarget::initializeSubtargetDependencies(StringRef CPU,
StringRef FS) {
initializeEnvironment();
@@ -181,10 +186,14 @@ bool PPCSubtarget::hasLazyResolverStub(const GlobalValue *GV) const {
return false;
}
-bool PPCSubtarget::enableMachineScheduler() const {
- return true;
+bool PPCSubtarget::enableMachineScheduler() const { return true; }
+
+bool PPCSubtarget::enableMachinePipeliner() const {
+ return (DarwinDirective == PPC::DIR_PWR9) && EnableMachinePipeliner;
}
+bool PPCSubtarget::useDFAforSMS() const { return false; }
+
// This overrides the PostRAScheduler bit in the SchedModel for each CPU.
bool PPCSubtarget::enablePostRAScheduler() const { return true; }
diff --git a/llvm/lib/Target/PowerPC/PPCSubtarget.h b/llvm/lib/Target/PowerPC/PPCSubtarget.h
index e1f4a9680da..a59cbd60618 100644
--- a/llvm/lib/Target/PowerPC/PPCSubtarget.h
+++ b/llvm/lib/Target/PowerPC/PPCSubtarget.h
@@ -322,9 +322,13 @@ public:
/// but may expand the ISEL instruction later.
bool enableEarlyIfConversion() const override { return true; }
- // Scheduling customization.
+ /// Scheduling customization.
bool enableMachineScheduler() const override;
- // This overrides the PostRAScheduler bit in the SchedModel for each CPU.
+ /// Pipeliner customization.
+ bool enableMachinePipeliner() const override;
+ /// Machine Pipeliner customization
+ bool useDFAforSMS() const override;
+ /// This overrides the PostRAScheduler bit in the SchedModel for each CPU.
bool enablePostRAScheduler() const override;
AntiDepBreakMode getAntiDepBreakMode() const override;
void getCriticalPathRCs(RegClassVector &CriticalPathRCs) const override;
diff --git a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
index da1121bac9c..fcaf7d6e3ee 100644
--- a/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
+++ b/llvm/lib/Target/PowerPC/PPCTargetMachine.cpp
@@ -488,6 +488,9 @@ void PPCPassConfig::addPreRegAlloc() {
}
if (EnableExtraTOCRegDeps)
addPass(createPPCTOCRegDepsPass());
+
+ if (getOptLevel() != CodeGenOpt::None)
+ addPass(&MachinePipelinerID);
}
void PPCPassConfig::addPreSched2() {
OpenPOWER on IntegriCloud