diff options
Diffstat (limited to 'llvm/lib')
37 files changed, 632 insertions, 112 deletions
diff --git a/llvm/lib/CodeGen/CMakeLists.txt b/llvm/lib/CodeGen/CMakeLists.txt index ef5763855a8..6ce5b13ea39 100644 --- a/llvm/lib/CodeGen/CMakeLists.txt +++ b/llvm/lib/CodeGen/CMakeLists.txt @@ -93,6 +93,7 @@ add_llvm_library(LLVMCodeGen ScheduleDAGInstrs.cpp ScheduleDAGPrinter.cpp ScoreboardHazardRecognizer.cpp + ShrinkWrap.cpp ShadowStackGC.cpp ShadowStackGCLowering.cpp SjLjEHPrepare.cpp diff --git a/llvm/lib/CodeGen/CodeGen.cpp b/llvm/lib/CodeGen/CodeGen.cpp index da66639d02f..2c6eaf35a25 100644 --- a/llvm/lib/CodeGen/CodeGen.cpp +++ b/llvm/lib/CodeGen/CodeGen.cpp @@ -61,6 +61,7 @@ void llvm::initializeCodeGen(PassRegistry &Registry) { initializePostRASchedulerPass(Registry); initializeProcessImplicitDefsPass(Registry); initializeRegisterCoalescerPass(Registry); + initializeShrinkWrapPass(Registry); initializeSlotIndexesPass(Registry); initializeStackColoringPass(Registry); initializeStackMapLivenessPass(Registry); diff --git a/llvm/lib/CodeGen/MachineFunction.cpp b/llvm/lib/CodeGen/MachineFunction.cpp index 448531fb716..fb0553b9414 100644 --- a/llvm/lib/CodeGen/MachineFunction.cpp +++ b/llvm/lib/CodeGen/MachineFunction.cpp @@ -600,8 +600,8 @@ MachineFrameInfo::getPristineRegs(const MachineBasicBlock *MBB) const { for (const MCPhysReg *CSR = TRI->getCalleeSavedRegs(MF); CSR && *CSR; ++CSR) BV.set(*CSR); - // The entry MBB always has all CSRs pristine. - if (MBB == &MF->front()) + // Each MBB before the save point has all CSRs pristine. + if (isBeforeSavePoint(*MF, *MBB)) return BV; // On other MBBs the saved CSRs are not pristine. @@ -613,6 +613,40 @@ MachineFrameInfo::getPristineRegs(const MachineBasicBlock *MBB) const { return BV; } +// Note: We could use some sort of caching mecanism, but we lack the ability +// to know when the cache is invalid, i.e., the CFG changed. +// Assuming we have that, we can simply compute all the set of MBBs +// that are before the save point. +bool MachineFrameInfo::isBeforeSavePoint(const MachineFunction &MF, + const MachineBasicBlock &MBB) const { + // Early exit if shrink-wrapping did not kick. + if (!Save) + return &MBB == &MF.front(); + + // Starting from MBB, check if there is a path leading to Save that do + // not cross Restore. + SmallPtrSet<const MachineBasicBlock *, 8> Visited; + SmallVector<const MachineBasicBlock *, 8> WorkList; + WorkList.push_back(&MBB); + Visited.insert(&MBB); + do { + const MachineBasicBlock *CurBB = WorkList.pop_back_val(); + // By construction, the region that is after the save point is + // dominated by the Save and post-dominated by the Restore. + // If we do not reach Restore and still reach Save, this + // means MBB is before Save. + if (CurBB == Save) + return true; + if (CurBB == Restore) + continue; + // Enqueue all the successors not already visited. + for (MachineBasicBlock *SuccBB : CurBB->successors()) + if (Visited.insert(SuccBB).second) + WorkList.push_back(SuccBB); + } while (!WorkList.empty()); + return false; +} + unsigned MachineFrameInfo::estimateStackSize(const MachineFunction &MF) const { const TargetFrameLowering *TFI = MF.getSubtarget().getFrameLowering(); const TargetRegisterInfo *RegInfo = MF.getSubtarget().getRegisterInfo(); diff --git a/llvm/lib/CodeGen/Passes.cpp b/llvm/lib/CodeGen/Passes.cpp index c128414c0b3..690224342f6 100644 --- a/llvm/lib/CodeGen/Passes.cpp +++ b/llvm/lib/CodeGen/Passes.cpp @@ -52,7 +52,10 @@ static cl::opt<bool> DisableMachineLICM("disable-machine-licm", cl::Hidden, static cl::opt<bool> DisableMachineCSE("disable-machine-cse", cl::Hidden, cl::desc("Disable Machine Common Subexpression Elimination")); static cl::opt<cl::boolOrDefault> -OptimizeRegAlloc("optimize-regalloc", cl::Hidden, + EnableShrinkWrapOpt("enable-shrink-wrap", cl::Hidden, + cl::desc("enable the shrink-wrapping pass")); +static cl::opt<cl::boolOrDefault> OptimizeRegAlloc( + "optimize-regalloc", cl::Hidden, cl::desc("Enable optimized register allocation compilation path.")); static cl::opt<bool> DisablePostRAMachineLICM("disable-postra-machine-licm", cl::Hidden, @@ -206,10 +209,10 @@ TargetPassConfig::~TargetPassConfig() { // Out of line constructor provides default values for pass options and // registers all common codegen passes. TargetPassConfig::TargetPassConfig(TargetMachine *tm, PassManagerBase &pm) - : ImmutablePass(ID), PM(&pm), StartAfter(nullptr), StopAfter(nullptr), - Started(true), Stopped(false), AddingMachinePasses(false), TM(tm), - Impl(nullptr), Initialized(false), DisableVerify(false), - EnableTailMerge(true) { + : ImmutablePass(ID), PM(&pm), StartAfter(nullptr), StopAfter(nullptr), + Started(true), Stopped(false), AddingMachinePasses(false), TM(tm), + Impl(nullptr), Initialized(false), DisableVerify(false), + EnableTailMerge(true), EnableShrinkWrap(false) { Impl = new PassConfigImpl(); @@ -524,6 +527,8 @@ void TargetPassConfig::addMachinePasses() { addPostRegAlloc(); // Insert prolog/epilog code. Eliminate abstract frame index references... + if (getEnableShrinkWrap()) + addPass(&ShrinkWrapID); addPass(&PrologEpilogCodeInserterID); /// Add passes that optimize machine instructions after register allocation. @@ -599,6 +604,21 @@ void TargetPassConfig::addMachineSSAOptimization() { addPass(&DeadMachineInstructionElimID); } +bool TargetPassConfig::getEnableShrinkWrap() const { + switch (EnableShrinkWrapOpt) { + case cl::BOU_UNSET: + return EnableShrinkWrap && getOptLevel() != CodeGenOpt::None; + // If EnableShrinkWrap is set, it takes precedence on whatever the + // target sets. The rational is that we assume we want to test + // something related to shrink-wrapping. + case cl::BOU_TRUE: + return true; + case cl::BOU_FALSE: + return false; + } + llvm_unreachable("Invalid shrink-wrapping state"); +} + //===---------------------------------------------------------------------===// /// Register Allocation Pass Configuration //===---------------------------------------------------------------------===// diff --git a/llvm/lib/CodeGen/PrologEpilogInserter.cpp b/llvm/lib/CodeGen/PrologEpilogInserter.cpp index 5334a633ead..e4f359c213c 100644 --- a/llvm/lib/CodeGen/PrologEpilogInserter.cpp +++ b/llvm/lib/CodeGen/PrologEpilogInserter.cpp @@ -71,9 +71,9 @@ private: // stack frame indexes. unsigned MinCSFrameIndex, MaxCSFrameIndex; - // Entry and return blocks of the current function. - MachineBasicBlock *EntryBlock; - SmallVector<MachineBasicBlock *, 4> ReturnBlocks; + // Save and Restore blocks of the current function. + MachineBasicBlock *SaveBlock; + SmallVector<MachineBasicBlock *, 4> RestoreBlocks; // Flag to control whether to use the register scavenger to resolve // frame index materialization registers. Set according to @@ -133,20 +133,26 @@ bool PEI::isReturnBlock(MachineBasicBlock* MBB) { /// Compute the set of return blocks void PEI::calculateSets(MachineFunction &Fn) { - // Sets used to compute spill, restore placement sets. - const std::vector<CalleeSavedInfo> &CSI = - Fn.getFrameInfo()->getCalleeSavedInfo(); + const MachineFrameInfo *MFI = Fn.getFrameInfo(); - // If no CSRs used, we are done. - if (CSI.empty()) + // Even when we do not change any CSR, we still want to insert the + // prologue and epilogue of the function. + // So set the save points for those. + + // Use the points found by shrink-wrapping, if any. + if (MFI->getSavePoint()) { + SaveBlock = MFI->getSavePoint(); + assert(MFI->getRestorePoint() && "Both restore and save must be set"); + RestoreBlocks.push_back(MFI->getRestorePoint()); return; + } // Save refs to entry and return blocks. - EntryBlock = Fn.begin(); + SaveBlock = Fn.begin(); for (MachineFunction::iterator MBB = Fn.begin(), E = Fn.end(); MBB != E; ++MBB) if (isReturnBlock(MBB)) - ReturnBlocks.push_back(MBB); + RestoreBlocks.push_back(MBB); return; } @@ -226,7 +232,7 @@ bool PEI::runOnMachineFunction(MachineFunction &Fn) { } delete RS; - ReturnBlocks.clear(); + RestoreBlocks.clear(); return true; } @@ -372,6 +378,61 @@ void PEI::calculateCalleeSavedRegisters(MachineFunction &F) { MFI->setCalleeSavedInfo(CSI); } +/// Helper function to update the liveness information for the callee-saved +/// registers. +static void updateLiveness(MachineFunction &MF) { + MachineFrameInfo *MFI = MF.getFrameInfo(); + // Visited will contain all the basic blocks that are in the region + // where the callee saved registers are alive: + // - Anything that is not Save or Restore -> LiveThrough. + // - Save -> LiveIn. + // - Restore -> LiveOut. + // The live-out is not attached to the block, so no need to keep + // Restore in this set. + SmallPtrSet<MachineBasicBlock *, 8> Visited; + SmallVector<MachineBasicBlock *, 8> WorkList; + MachineBasicBlock *Entry = &MF.front(); + MachineBasicBlock *Save = MFI->getSavePoint(); + + if (!Save) + Save = Entry; + + if (Entry != Save) { + WorkList.push_back(Entry); + Visited.insert(Entry); + } + Visited.insert(Save); + + MachineBasicBlock *Restore = MFI->getRestorePoint(); + if (Restore) + // By construction Restore cannot be visited, otherwise it + // means there exists a path to Restore that does not go + // through Save. + WorkList.push_back(Restore); + + while (!WorkList.empty()) { + const MachineBasicBlock *CurBB = WorkList.pop_back_val(); + // By construction, the region that is after the save point is + // dominated by the Save and post-dominated by the Restore. + if (CurBB == Save) + continue; + // Enqueue all the successors not already visited. + // Those are by construction either before Save or after Restore. + for (MachineBasicBlock *SuccBB : CurBB->successors()) + if (Visited.insert(SuccBB).second) + WorkList.push_back(SuccBB); + } + + const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo(); + + for (unsigned i = 0, e = CSI.size(); i != e; ++i) { + for (MachineBasicBlock *MBB : Visited) + // Add the callee-saved register as live-in. + // It's killed at the spill. + MBB->addLiveIn(CSI[i].getReg()); + } +} + /// insertCSRSpillsAndRestores - Insert spill and restore code for /// callee saved registers used in the function. /// @@ -392,26 +453,22 @@ void PEI::insertCSRSpillsAndRestores(MachineFunction &Fn) { MachineBasicBlock::iterator I; // Spill using target interface. - I = EntryBlock->begin(); - if (!TFI->spillCalleeSavedRegisters(*EntryBlock, I, CSI, TRI)) { + I = SaveBlock->begin(); + if (!TFI->spillCalleeSavedRegisters(*SaveBlock, I, CSI, TRI)) { for (unsigned i = 0, e = CSI.size(); i != e; ++i) { - // Add the callee-saved register as live-in. - // It's killed at the spill. - EntryBlock->addLiveIn(CSI[i].getReg()); - // Insert the spill to the stack frame. unsigned Reg = CSI[i].getReg(); const TargetRegisterClass *RC = TRI->getMinimalPhysRegClass(Reg); - TII.storeRegToStackSlot(*EntryBlock, I, Reg, true, CSI[i].getFrameIdx(), + TII.storeRegToStackSlot(*SaveBlock, I, Reg, true, CSI[i].getFrameIdx(), RC, TRI); } } + // Update the live-in information of all the blocks up to the save point. + updateLiveness(Fn); // Restore using target interface. - for (unsigned ri = 0, re = ReturnBlocks.size(); ri != re; ++ri) { - MachineBasicBlock *MBB = ReturnBlocks[ri]; + for (MachineBasicBlock *MBB : RestoreBlocks) { I = MBB->end(); - --I; // Skip over all terminator instructions, which are part of the return // sequence. @@ -721,21 +778,18 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) { const TargetFrameLowering &TFI = *Fn.getSubtarget().getFrameLowering(); // Add prologue to the function... - TFI.emitPrologue(Fn); + TFI.emitPrologue(Fn, *SaveBlock); - // Add epilogue to restore the callee-save registers in each exiting block - for (MachineFunction::iterator I = Fn.begin(), E = Fn.end(); I != E; ++I) { - // If last instruction is a return instruction, add an epilogue - if (!I->empty() && I->back().isReturn()) - TFI.emitEpilogue(Fn, *I); - } + // Add epilogue to restore the callee-save registers in each exiting block. + for (MachineBasicBlock *RestoreBlock : RestoreBlocks) + TFI.emitEpilogue(Fn, *RestoreBlock); // Emit additional code that is required to support segmented stacks, if // we've been asked for it. This, when linked with a runtime with support // for segmented stacks (libgcc is one), will result in allocating stack // space in small chunks instead of one large contiguous block. if (Fn.shouldSplitStack()) - TFI.adjustForSegmentedStacks(Fn); + TFI.adjustForSegmentedStacks(Fn, *SaveBlock); // Emit additional code that is required to explicitly handle the stack in // HiPE native code (if needed) when loaded in the Erlang/OTP runtime. The @@ -743,7 +797,7 @@ void PEI::insertPrologEpilogCode(MachineFunction &Fn) { // different conditional check and another BIF for allocating more stack // space. if (Fn.getFunction()->getCallingConv() == CallingConv::HiPE) - TFI.adjustForHiPEPrologue(Fn); + TFI.adjustForHiPEPrologue(Fn, *SaveBlock); } /// replaceFrameIndices - Replace all MO_FrameIndex operands with physical diff --git a/llvm/lib/CodeGen/ShrinkWrap.cpp b/llvm/lib/CodeGen/ShrinkWrap.cpp new file mode 100644 index 00000000000..56f1200619c --- /dev/null +++ b/llvm/lib/CodeGen/ShrinkWrap.cpp @@ -0,0 +1,383 @@ +//===-- ShrinkWrap.cpp - Compute safe point for prolog/epilog insertion ---===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This pass looks for safe point where the prologue and epilogue can be +// inserted. +// The safe point for the prologue (resp. epilogue) is called Save +// (resp. Restore). +// A point is safe for prologue (resp. epilogue) if and only if +// it 1) dominates (resp. post-dominates) all the frame related operations and +// between 2) two executions of the Save (resp. Restore) point there is an +// execution of the Restore (resp. Save) point. +// +// For instance, the following points are safe: +// for (int i = 0; i < 10; ++i) { +// Save +// ... +// Restore +// } +// Indeed, the execution looks like Save -> Restore -> Save -> Restore ... +// And the following points are not: +// for (int i = 0; i < 10; ++i) { +// Save +// ... +// } +// for (int i = 0; i < 10; ++i) { +// ... +// Restore +// } +// Indeed, the execution looks like Save -> Save -> ... -> Restore -> Restore. +// +// This pass also ensures that the safe points are 3) cheaper than the regular +// entry and exits blocks. +// +// Property #1 is ensured via the use of MachineDominatorTree and +// MachinePostDominatorTree. +// Property #2 is ensured via property #1 and MachineLoopInfo, i.e., both +// points must be in the same loop. +// Property #3 is ensured via the MachineBlockFrequencyInfo. +// +// If this pass found points matching all this properties, then +// MachineFrameInfo is updated this that information. +//===----------------------------------------------------------------------===// +#include "llvm/ADT/Statistic.h" +// To check for profitability. +#include "llvm/CodeGen/MachineBlockFrequencyInfo.h" +// For property #1 for Save. +#include "llvm/CodeGen/MachineDominators.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +// To record the result of the analysis. +#include "llvm/CodeGen/MachineFrameInfo.h" +// For property #2. +#include "llvm/CodeGen/MachineLoopInfo.h" +// For property #1 for Restore. +#include "llvm/CodeGen/MachinePostDominators.h" +#include "llvm/CodeGen/Passes.h" +// To know about callee-saved. +#include "llvm/CodeGen/RegisterClassInfo.h" +#include "llvm/Support/Debug.h" +// To know about frame setup operation. +#include "llvm/Target/TargetInstrInfo.h" +// To access TargetInstrInfo. +#include "llvm/Target/TargetSubtargetInfo.h" + +#define DEBUG_TYPE "shrink-wrap" + +using namespace llvm; + +STATISTIC(NumFunc, "Number of functions"); +STATISTIC(NumCandidates, "Number of shrink-wrapping candidates"); +STATISTIC(NumCandidatesDropped, + "Number of shrink-wrapping candidates dropped because of frequency"); + +namespace { +/// \brief Class to determine where the safe point to insert the +/// prologue and epilogue are. +/// Unlike the paper from Fred C. Chow, PLDI'88, that introduces the +/// shrink-wrapping term for prologue/epilogue placement, this pass +/// does not rely on expensive data-flow analysis. Instead we use the +/// dominance properties and loop information to decide which point +/// are safe for such insertion. +class ShrinkWrap : public MachineFunctionPass { + /// Hold callee-saved information. + RegisterClassInfo RCI; + MachineDominatorTree *MDT; + MachinePostDominatorTree *MPDT; + /// Current safe point found for the prologue. + /// The prologue will be inserted before the first instruction + /// in this basic block. + MachineBasicBlock *Save; + /// Current safe point found for the epilogue. + /// The epilogue will be inserted before the first terminator instruction + /// in this basic block. + MachineBasicBlock *Restore; + /// Hold the information of the basic block frequency. + /// Use to check the profitability of the new points. + MachineBlockFrequencyInfo *MBFI; + /// Hold the loop information. Used to determine if Save and Restore + /// are in the same loop. + MachineLoopInfo *MLI; + /// Frequency of the Entry block. + uint64_t EntryFreq; + /// Current opcode for frame setup. + int FrameSetupOpcode; + /// Current opcode for frame destroy. + int FrameDestroyOpcode; + /// Entry block. + const MachineBasicBlock *Entry; + + /// \brief Check if \p MI uses or defines a callee-saved register or + /// a frame index. If this is the case, this means \p MI must happen + /// after Save and before Restore. + bool useOrDefCSROrFI(const MachineInstr &MI) const; + + /// \brief Update the Save and Restore points such that \p MBB is in + /// the region that is dominated by Save and post-dominated by Restore + /// and Save and Restore still match the safe point definition. + /// Such point may not exist and Save and/or Restore may be null after + /// this call. + void updateSaveRestorePoints(MachineBasicBlock &MBB); + + /// \brief Initialize the pass for \p MF. + void init(MachineFunction &MF) { + RCI.runOnMachineFunction(MF); + MDT = &getAnalysis<MachineDominatorTree>(); + MPDT = &getAnalysis<MachinePostDominatorTree>(); + Save = nullptr; + Restore = nullptr; + MBFI = &getAnalysis<MachineBlockFrequencyInfo>(); + MLI = &getAnalysis<MachineLoopInfo>(); + EntryFreq = MBFI->getEntryFreq(); + const TargetInstrInfo &TII = *MF.getSubtarget().getInstrInfo(); + FrameSetupOpcode = TII.getCallFrameSetupOpcode(); + FrameDestroyOpcode = TII.getCallFrameDestroyOpcode(); + Entry = &MF.front(); + + ++NumFunc; + } + + /// Check whether or not Save and Restore points are still interesting for + /// shrink-wrapping. + bool ArePointsInteresting() const { return Save != Entry && Save && Restore; } + +public: + static char ID; + + ShrinkWrap() : MachineFunctionPass(ID) { + initializeShrinkWrapPass(*PassRegistry::getPassRegistry()); + } + + void getAnalysisUsage(AnalysisUsage &AU) const override { + AU.setPreservesAll(); + AU.addRequired<MachineBlockFrequencyInfo>(); + AU.addRequired<MachineDominatorTree>(); + AU.addRequired<MachinePostDominatorTree>(); + AU.addRequired<MachineLoopInfo>(); + MachineFunctionPass::getAnalysisUsage(AU); + } + + const char *getPassName() const override { + return "Shrink Wrapping analysis"; + } + + /// \brief Perform the shrink-wrapping analysis and update + /// the MachineFrameInfo attached to \p MF with the results. + bool runOnMachineFunction(MachineFunction &MF) override; +}; +} // End anonymous namespace. + +char ShrinkWrap::ID = 0; +char &llvm::ShrinkWrapID = ShrinkWrap::ID; + +INITIALIZE_PASS_BEGIN(ShrinkWrap, "shrink-wrap", "Shrink Wrap Pass", false, + false) +INITIALIZE_PASS_DEPENDENCY(MachineBlockFrequencyInfo) +INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree) +INITIALIZE_PASS_DEPENDENCY(MachinePostDominatorTree) +INITIALIZE_PASS_DEPENDENCY(MachineLoopInfo) +INITIALIZE_PASS_END(ShrinkWrap, "shrink-wrap", "Shrink Wrap Pass", false, false) + +bool ShrinkWrap::useOrDefCSROrFI(const MachineInstr &MI) const { + if (MI.getOpcode() == FrameSetupOpcode || + MI.getOpcode() == FrameDestroyOpcode) { + DEBUG(dbgs() << "Frame instruction: " << MI << '\n'); + return true; + } + for (const MachineOperand &MO : MI.operands()) { + bool UseCSR = false; + if (MO.isReg()) { + unsigned PhysReg = MO.getReg(); + if (!PhysReg) + continue; + assert(TargetRegisterInfo::isPhysicalRegister(PhysReg) && + "Unallocated register?!"); + UseCSR = RCI.getLastCalleeSavedAlias(PhysReg); + } + // TODO: Handle regmask more accurately. + // For now, be conservative about them. + if (UseCSR || MO.isFI() || MO.isRegMask()) { + DEBUG(dbgs() << "Use or define CSR(" << UseCSR << ") or FI(" << MO.isFI() + << "): " << MI << '\n'); + return true; + } + } + return false; +} + +/// \brief Helper function to find the immediate (post) dominator. +template <typename ListOfBBs, typename DominanceAnalysis> +MachineBasicBlock *FindIDom(MachineBasicBlock &Block, ListOfBBs BBs, + DominanceAnalysis &Dom) { + MachineBasicBlock *IDom = &Block; + for (MachineBasicBlock *BB : BBs) { + IDom = Dom.findNearestCommonDominator(IDom, BB); + if (!IDom) + break; + } + return IDom; +} + +void ShrinkWrap::updateSaveRestorePoints(MachineBasicBlock &MBB) { + // Get rid of the easy cases first. + if (!Save) + Save = &MBB; + else + Save = MDT->findNearestCommonDominator(Save, &MBB); + + if (!Save) { + DEBUG(dbgs() << "Found a block that is not reachable from Entry\n"); + return; + } + + if (!Restore) + Restore = &MBB; + else + Restore = MPDT->findNearestCommonDominator(Restore, &MBB); + + // Make sure we would be able to insert the restore code before the + // terminator. + if (Restore == &MBB) { + for (const MachineInstr &Terminator : MBB.terminators()) { + if (!useOrDefCSROrFI(Terminator)) + continue; + // One of the terminator needs to happen before the restore point. + if (MBB.succ_empty()) { + Restore = nullptr; + break; + } + // Look for a restore point that post-dominates all the successors. + // The immediate post-dominator is what we are looking for. + Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT); + break; + } + } + + if (!Restore) { + DEBUG(dbgs() << "Restore point needs to be spanned on several blocks\n"); + return; + } + + // Make sure Save and Restore are suitable for shrink-wrapping: + // 1. all path from Save needs to lead to Restore before exiting. + // 2. all path to Restore needs to go through Save from Entry. + // We achieve that by making sure that: + // A. Save dominates Restore. + // B. Restore post-dominates Save. + // C. Save and Restore are in the same loop. + bool SaveDominatesRestore = false; + bool RestorePostDominatesSave = false; + while (Save && Restore && + (!(SaveDominatesRestore = MDT->dominates(Save, Restore)) || + !(RestorePostDominatesSave = MPDT->dominates(Restore, Save)) || + MLI->getLoopFor(Save) != MLI->getLoopFor(Restore))) { + // Fix (A). + if (!SaveDominatesRestore) { + Save = MDT->findNearestCommonDominator(Save, Restore); + continue; + } + // Fix (B). + if (!RestorePostDominatesSave) + Restore = MPDT->findNearestCommonDominator(Restore, Save); + + // Fix (C). + if (Save && Restore && Save != Restore && + MLI->getLoopFor(Save) != MLI->getLoopFor(Restore)) { + if (MLI->getLoopDepth(Save) > MLI->getLoopDepth(Restore)) + // Push Save outside of this loop. + Save = FindIDom<>(*Save, Save->predecessors(), *MDT); + else + // Push Restore outside of this loop. + Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT); + } + } +} + +bool ShrinkWrap::runOnMachineFunction(MachineFunction &MF) { + if (MF.empty()) + return false; + DEBUG(dbgs() << "**** Analysing " << MF.getName() << '\n'); + + init(MF); + + for (MachineBasicBlock &MBB : MF) { + DEBUG(dbgs() << "Look into: " << MBB.getNumber() << ' ' << MBB.getName() + << '\n'); + + for (const MachineInstr &MI : MBB) { + if (!useOrDefCSROrFI(MI)) + continue; + // Save (resp. restore) point must dominate (resp. post dominate) + // MI. Look for the proper basic block for those. + updateSaveRestorePoints(MBB); + // If we are at a point where we cannot improve the placement of + // save/restore instructions, just give up. + if (!ArePointsInteresting()) { + DEBUG(dbgs() << "No Shrink wrap candidate found\n"); + return false; + } + // No need to look for other instructions, this basic block + // will already be part of the handled region. + break; + } + } + if (!ArePointsInteresting()) { + // If the points are not interesting at this point, then they must be null + // because it means we did not encounter any frame/CSR related code. + // Otherwise, we would have returned from the previous loop. + assert(!Save && !Restore && "We miss a shrink-wrap opportunity?!"); + DEBUG(dbgs() << "Nothing to shrink-wrap\n"); + return false; + } + + DEBUG(dbgs() << "\n ** Results **\nFrequency of the Entry: " << EntryFreq + << '\n'); + + do { + DEBUG(dbgs() << "Shrink wrap candidates (#, Name, Freq):\nSave: " + << Save->getNumber() << ' ' << Save->getName() << ' ' + << MBFI->getBlockFreq(Save).getFrequency() << "\nRestore: " + << Restore->getNumber() << ' ' << Restore->getName() << ' ' + << MBFI->getBlockFreq(Restore).getFrequency() << '\n'); + + bool IsSaveCheap; + if ((IsSaveCheap = EntryFreq >= MBFI->getBlockFreq(Save).getFrequency()) && + EntryFreq >= MBFI->getBlockFreq(Restore).getFrequency()) + break; + DEBUG(dbgs() << "New points are too expensive\n"); + MachineBasicBlock *NewBB; + if (!IsSaveCheap) { + Save = FindIDom<>(*Save, Save->predecessors(), *MDT); + if (!Save) + break; + NewBB = Save; + } else { + // Restore is expensive. + Restore = FindIDom<>(*Restore, Restore->successors(), *MPDT); + if (!Restore) + break; + NewBB = Restore; + } + updateSaveRestorePoints(*NewBB); + } while (Save && Restore); + + if (!ArePointsInteresting()) { + ++NumCandidatesDropped; + return false; + } + + DEBUG(dbgs() << "Final shrink wrap candidates:\nSave: " << Save->getNumber() + << ' ' << Save->getName() << "\nRestore: " + << Restore->getNumber() << ' ' << Restore->getName() << '\n'); + + MachineFrameInfo *MFI = MF.getFrameInfo(); + MFI->setSavePoint(Save); + MFI->setRestorePoint(Restore); + ++NumCandidates; + return false; +} diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp index 6f133a35dce..7146f592a96 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.cpp @@ -275,8 +275,8 @@ static bool isCSSave(MachineInstr *MBBI) { MBBI->getOpcode() == AArch64::STPDpre; } -void AArch64FrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); // Prologue goes in entry BB. +void AArch64FrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { MachineBasicBlock::iterator MBBI = MBB.begin(); const MachineFrameInfo *MFI = MF.getFrameInfo(); const Function *Fn = MF.getFunction(); @@ -539,15 +539,19 @@ static bool isCSRestore(MachineInstr *MI, const MCPhysReg *CSRegs) { void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const { MachineBasicBlock::iterator MBBI = MBB.getLastNonDebugInstr(); - assert(MBBI->isReturn() && "Can only insert epilog into returning blocks"); MachineFrameInfo *MFI = MF.getFrameInfo(); const AArch64InstrInfo *TII = static_cast<const AArch64InstrInfo *>(MF.getSubtarget().getInstrInfo()); const AArch64RegisterInfo *RegInfo = static_cast<const AArch64RegisterInfo *>( MF.getSubtarget().getRegisterInfo()); - DebugLoc DL = MBBI->getDebugLoc(); - unsigned RetOpcode = MBBI->getOpcode(); - + DebugLoc DL; + bool IsTailCallReturn = false; + if (MBB.end() != MBBI) { + DL = MBBI->getDebugLoc(); + unsigned RetOpcode = MBBI->getOpcode(); + IsTailCallReturn = RetOpcode == AArch64::TCRETURNdi || + RetOpcode == AArch64::TCRETURNri; + } int NumBytes = MFI->getStackSize(); const AArch64FunctionInfo *AFI = MF.getInfo<AArch64FunctionInfo>(); @@ -559,7 +563,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, // Initial and residual are named for consistency with the prologue. Note that // in the epilogue, the residual adjustment is executed first. uint64_t ArgumentPopSize = 0; - if (RetOpcode == AArch64::TCRETURNdi || RetOpcode == AArch64::TCRETURNri) { + if (IsTailCallReturn) { MachineOperand &StackAdjust = MBBI->getOperand(1); // For a tail-call in a callee-pops-arguments environment, some or all of @@ -604,7 +608,7 @@ void AArch64FrameLowering::emitEpilogue(MachineFunction &MF, unsigned NumRestores = 0; // Move past the restores of the callee-saved registers. - MachineBasicBlock::iterator LastPopI = MBBI; + MachineBasicBlock::iterator LastPopI = MBB.getFirstTerminator(); const MCPhysReg *CSRegs = RegInfo->getCalleeSavedRegs(&MF); if (LastPopI != MBB.begin()) { do { diff --git a/llvm/lib/Target/AArch64/AArch64FrameLowering.h b/llvm/lib/Target/AArch64/AArch64FrameLowering.h index 1439bf32085..b496fccba34 100644 --- a/llvm/lib/Target/AArch64/AArch64FrameLowering.h +++ b/llvm/lib/Target/AArch64/AArch64FrameLowering.h @@ -34,7 +34,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; int getFrameIndexOffset(const MachineFunction &MF, int FI) const override; diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.cpp b/llvm/lib/Target/ARM/ARMFrameLowering.cpp index 92fe8fe5fbc..4eafd82af4c 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.cpp +++ b/llvm/lib/Target/ARM/ARMFrameLowering.cpp @@ -278,8 +278,9 @@ static void emitAligningInstructions(MachineFunction &MF, ARMFunctionInfo *AFI, } } -void ARMFrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); +void ARMFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MBB == &MF.front() && "Shrink-wrapping not yet implemented"); MachineBasicBlock::iterator MBBI = MBB.begin(); MachineFrameInfo *MFI = MF.getFrameInfo(); ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); @@ -1861,7 +1862,8 @@ static const uint64_t kSplitStackAvailable = 256; // ARM can be found at [1]. // // [1] - https://github.com/mozilla/rust/blob/86efd9/src/rt/arch/arm/morestack.S -void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { +void ARMFrameLowering::adjustForSegmentedStacks( + MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { unsigned Opcode; unsigned CFIIndex; const ARMSubtarget *ST = &MF.getSubtarget<ARMSubtarget>(); @@ -1874,7 +1876,7 @@ void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { if (!ST->isTargetAndroid() && !ST->isTargetLinux()) report_fatal_error("Segmented stacks not supported on this platform."); - MachineBasicBlock &prologueMBB = MF.front(); + assert(&PrologueMBB == &MF.front() && "Shrink-wrapping not yet implemented"); MachineFrameInfo *MFI = MF.getFrameInfo(); MachineModuleInfo &MMI = MF.getMMI(); MCContext &Context = MMI.getContext(); @@ -1902,8 +1904,8 @@ void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { MachineBasicBlock *GetMBB = MF.CreateMachineBasicBlock(); MachineBasicBlock *McrMBB = MF.CreateMachineBasicBlock(); - for (MachineBasicBlock::livein_iterator i = prologueMBB.livein_begin(), - e = prologueMBB.livein_end(); + for (MachineBasicBlock::livein_iterator i = PrologueMBB.livein_begin(), + e = PrologueMBB.livein_end(); i != e; ++i) { AllocMBB->addLiveIn(*i); GetMBB->addLiveIn(*i); @@ -2156,7 +2158,7 @@ void ARMFrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { .addCFIIndex(CFIIndex); // Organizing MBB lists - PostStackMBB->addSuccessor(&prologueMBB); + PostStackMBB->addSuccessor(&PrologueMBB); AllocMBB->addSuccessor(PostStackMBB); diff --git a/llvm/lib/Target/ARM/ARMFrameLowering.h b/llvm/lib/Target/ARM/ARMFrameLowering.h index b7be43642ad..ff3425795ae 100644 --- a/llvm/lib/Target/ARM/ARMFrameLowering.h +++ b/llvm/lib/Target/ARM/ARMFrameLowering.h @@ -28,7 +28,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void fixTCReturn(MachineFunction &MF, MachineBasicBlock &MBB) const; @@ -55,7 +55,8 @@ public: void processFunctionBeforeCalleeSavedScan(MachineFunction &MF, RegScavenger *RS) const override; - void adjustForSegmentedStacks(MachineFunction &MF) const override; + void adjustForSegmentedStacks(MachineFunction &MF, + MachineBasicBlock &MBB) const override; private: void emitPushInst(MachineBasicBlock &MBB, MachineBasicBlock::iterator MI, diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp index c496cd7c759..77cd890e4ca 100644 --- a/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp +++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.cpp @@ -82,8 +82,9 @@ eliminateCallFramePseudoInstr(MachineFunction &MF, MachineBasicBlock &MBB, MBB.erase(I); } -void Thumb1FrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); +void Thumb1FrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MBB == &MF.front() && "Shrink-wrapping not yet implemented"); MachineBasicBlock::iterator MBBI = MBB.begin(); MachineFrameInfo *MFI = MF.getFrameInfo(); ARMFunctionInfo *AFI = MF.getInfo<ARMFunctionInfo>(); diff --git a/llvm/lib/Target/ARM/Thumb1FrameLowering.h b/llvm/lib/Target/ARM/Thumb1FrameLowering.h index cf932035b8d..31d57325ebd 100644 --- a/llvm/lib/Target/ARM/Thumb1FrameLowering.h +++ b/llvm/lib/Target/ARM/Thumb1FrameLowering.h @@ -27,7 +27,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, diff --git a/llvm/lib/Target/BPF/BPFFrameLowering.cpp b/llvm/lib/Target/BPF/BPFFrameLowering.cpp index ae9f35520de..54c5ececc7d 100644 --- a/llvm/lib/Target/BPF/BPFFrameLowering.cpp +++ b/llvm/lib/Target/BPF/BPFFrameLowering.cpp @@ -23,7 +23,8 @@ using namespace llvm; bool BPFFrameLowering::hasFP(const MachineFunction &MF) const { return true; } -void BPFFrameLowering::emitPrologue(MachineFunction &MF) const {} +void BPFFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const {} void BPFFrameLowering::emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const {} diff --git a/llvm/lib/Target/BPF/BPFFrameLowering.h b/llvm/lib/Target/BPF/BPFFrameLowering.h index 833046ddab6..3b9fc443e05 100644 --- a/llvm/lib/Target/BPF/BPFFrameLowering.h +++ b/llvm/lib/Target/BPF/BPFFrameLowering.h @@ -24,7 +24,7 @@ public: explicit BPFFrameLowering(const BPFSubtarget &sti) : TargetFrameLowering(TargetFrameLowering::StackGrowsDown, 8, 0) {} - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool hasFP(const MachineFunction &MF) const override; diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp index ee025137dc0..3071b779413 100644 --- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp +++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.cpp @@ -344,16 +344,17 @@ void HexagonFrameLowering::findShrunkPrologEpilog(MachineFunction &MF, EpilogB = PDomB; } - /// Perform most of the PEI work here: /// - saving/restoring of the callee-saved registers, /// - stack frame creation and destruction. /// Normally, this work is distributed among various functions, but doing it /// in one place allows shrink-wrapping of the stack frame. -void HexagonFrameLowering::emitPrologue(MachineFunction &MF) const { +void HexagonFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { auto &HST = static_cast<const HexagonSubtarget&>(MF.getSubtarget()); auto &HRI = *HST.getRegisterInfo(); + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineFrameInfo *MFI = MF.getFrameInfo(); const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo(); diff --git a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h index 228a0442b47..89500cb8572 100644 --- a/llvm/lib/Target/Hexagon/HexagonFrameLowering.h +++ b/llvm/lib/Target/Hexagon/HexagonFrameLowering.h @@ -26,7 +26,8 @@ public: // All of the prolog/epilog functionality, including saving and restoring // callee-saved registers is handled in emitPrologue. This is to have the // logic for shrink-wrapping in one place. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const + override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override {} bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, diff --git a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp index d6cb9f6459a..eb720809e47 100644 --- a/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp +++ b/llvm/lib/Target/MSP430/MSP430FrameLowering.cpp @@ -39,8 +39,9 @@ bool MSP430FrameLowering::hasReservedCallFrame(const MachineFunction &MF) const return !MF.getFrameInfo()->hasVarSizedObjects(); } -void MSP430FrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); // Prolog goes in entry BB +void MSP430FrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineFrameInfo *MFI = MF.getFrameInfo(); MSP430MachineFunctionInfo *MSP430FI = MF.getInfo<MSP430MachineFunctionInfo>(); const MSP430InstrInfo &TII = diff --git a/llvm/lib/Target/MSP430/MSP430FrameLowering.h b/llvm/lib/Target/MSP430/MSP430FrameLowering.h index 1941af26e85..48c4dc866a6 100644 --- a/llvm/lib/Target/MSP430/MSP430FrameLowering.h +++ b/llvm/lib/Target/MSP430/MSP430FrameLowering.h @@ -27,7 +27,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void eliminateCallFramePseudoInstr(MachineFunction &MF, diff --git a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp index 5828fbdee40..db2a924a99f 100644 --- a/llvm/lib/Target/Mips/Mips16FrameLowering.cpp +++ b/llvm/lib/Target/Mips/Mips16FrameLowering.cpp @@ -32,8 +32,9 @@ using namespace llvm; Mips16FrameLowering::Mips16FrameLowering(const MipsSubtarget &STI) : MipsFrameLowering(STI, STI.stackAlignment()) {} -void Mips16FrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); +void Mips16FrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineFrameInfo *MFI = MF.getFrameInfo(); const Mips16InstrInfo &TII = *static_cast<const Mips16InstrInfo *>(STI.getInstrInfo()); diff --git a/llvm/lib/Target/Mips/Mips16FrameLowering.h b/llvm/lib/Target/Mips/Mips16FrameLowering.h index 0287e59243a..f281c927c1c 100644 --- a/llvm/lib/Target/Mips/Mips16FrameLowering.h +++ b/llvm/lib/Target/Mips/Mips16FrameLowering.h @@ -23,7 +23,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, diff --git a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp index e9a206170a6..19efa59e1fd 100644 --- a/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp +++ b/llvm/lib/Target/Mips/MipsSEFrameLowering.cpp @@ -364,8 +364,9 @@ bool ExpandPseudo::expandExtractElementF64(MachineBasicBlock &MBB, MipsSEFrameLowering::MipsSEFrameLowering(const MipsSubtarget &STI) : MipsFrameLowering(STI, STI.stackAlignment()) {} -void MipsSEFrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); +void MipsSEFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineFrameInfo *MFI = MF.getFrameInfo(); MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>(); diff --git a/llvm/lib/Target/Mips/MipsSEFrameLowering.h b/llvm/lib/Target/Mips/MipsSEFrameLowering.h index 22448a433b7..2fcd6bbb9a1 100644 --- a/llvm/lib/Target/Mips/MipsSEFrameLowering.h +++ b/llvm/lib/Target/Mips/MipsSEFrameLowering.h @@ -24,7 +24,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool spillCalleeSavedRegisters(MachineBasicBlock &MBB, diff --git a/llvm/lib/Target/NVPTX/NVPTXFrameLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXFrameLowering.cpp index 34d3a66adce..5503494fc3c 100644 --- a/llvm/lib/Target/NVPTX/NVPTXFrameLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXFrameLowering.cpp @@ -31,9 +31,10 @@ NVPTXFrameLowering::NVPTXFrameLowering() bool NVPTXFrameLowering::hasFP(const MachineFunction &MF) const { return true; } -void NVPTXFrameLowering::emitPrologue(MachineFunction &MF) const { +void NVPTXFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { if (MF.getFrameInfo()->hasStackObjects()) { - MachineBasicBlock &MBB = MF.front(); + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); // Insert "mov.u32 %SP, %Depot" MachineBasicBlock::iterator MBBI = MBB.begin(); // This instruction really occurs before first instruction diff --git a/llvm/lib/Target/NVPTX/NVPTXFrameLowering.h b/llvm/lib/Target/NVPTX/NVPTXFrameLowering.h index d1e0a5ceb30..14f8bb7b98f 100644 --- a/llvm/lib/Target/NVPTX/NVPTXFrameLowering.h +++ b/llvm/lib/Target/NVPTX/NVPTXFrameLowering.h @@ -23,7 +23,7 @@ public: explicit NVPTXFrameLowering(); bool hasFP(const MachineFunction &MF) const override; - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void diff --git a/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp b/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp index c1c67e3872e..5fd69a6815a 100644 --- a/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXPrologEpilogPass.cpp @@ -68,7 +68,7 @@ bool NVPTXPrologEpilogPass::runOnMachineFunction(MachineFunction &MF) { } // Add function prolog/epilog - TFI.emitPrologue(MF); + TFI.emitPrologue(MF, MF.front()); for (MachineFunction::iterator I = MF.begin(), E = MF.end(); I != E; ++I) { // If last instruction is a return instruction, add an epilogue diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp index f997fea4d93..b4008e4a886 100644 --- a/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.cpp @@ -555,8 +555,9 @@ void PPCFrameLowering::replaceFPWithRealFP(MachineFunction &MF) const { } } -void PPCFrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); // Prolog goes in entry BB +void PPCFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineBasicBlock::iterator MBBI = MBB.begin(); MachineFrameInfo *MFI = MF.getFrameInfo(); const PPCInstrInfo &TII = diff --git a/llvm/lib/Target/PowerPC/PPCFrameLowering.h b/llvm/lib/Target/PowerPC/PPCFrameLowering.h index dddabb80892..28d074ecd79 100644 --- a/llvm/lib/Target/PowerPC/PPCFrameLowering.h +++ b/llvm/lib/Target/PowerPC/PPCFrameLowering.h @@ -38,7 +38,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool hasFP(const MachineFunction &MF) const override; diff --git a/llvm/lib/Target/R600/AMDGPUFrameLowering.cpp b/llvm/lib/Target/R600/AMDGPUFrameLowering.cpp index 9e8302ec0a1..8175786fb9b 100644 --- a/llvm/lib/Target/R600/AMDGPUFrameLowering.cpp +++ b/llvm/lib/Target/R600/AMDGPUFrameLowering.cpp @@ -99,9 +99,8 @@ AMDGPUFrameLowering::getCalleeSavedSpillSlots(unsigned &NumEntries) const { NumEntries = 0; return nullptr; } -void -AMDGPUFrameLowering::emitPrologue(MachineFunction &MF) const { -} +void AMDGPUFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const {} void AMDGPUFrameLowering::emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const { diff --git a/llvm/lib/Target/R600/AMDGPUFrameLowering.h b/llvm/lib/Target/R600/AMDGPUFrameLowering.h index 15a6636a1ae..9f31be1af79 100644 --- a/llvm/lib/Target/R600/AMDGPUFrameLowering.h +++ b/llvm/lib/Target/R600/AMDGPUFrameLowering.h @@ -37,7 +37,7 @@ public: int getFrameIndexOffset(const MachineFunction &MF, int FI) const override; const SpillSlot * getCalleeSavedSpillSlots(unsigned &NumEntries) const override; - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool hasFP(const MachineFunction &MF) const override; }; diff --git a/llvm/lib/Target/Sparc/SparcFrameLowering.cpp b/llvm/lib/Target/Sparc/SparcFrameLowering.cpp index a065d3a0803..bccc6bdd53e 100644 --- a/llvm/lib/Target/Sparc/SparcFrameLowering.cpp +++ b/llvm/lib/Target/Sparc/SparcFrameLowering.cpp @@ -82,10 +82,11 @@ void SparcFrameLowering::emitSPAdjustment(MachineFunction &MF, .addReg(SP::O6).addReg(SP::G1); } -void SparcFrameLowering::emitPrologue(MachineFunction &MF) const { +void SparcFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { SparcMachineFunctionInfo *FuncInfo = MF.getInfo<SparcMachineFunctionInfo>(); - MachineBasicBlock &MBB = MF.front(); + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineFrameInfo *MFI = MF.getFrameInfo(); const SparcInstrInfo &TII = *static_cast<const SparcInstrInfo *>(MF.getSubtarget().getInstrInfo()); diff --git a/llvm/lib/Target/Sparc/SparcFrameLowering.h b/llvm/lib/Target/Sparc/SparcFrameLowering.h index 9e53994e3c3..bb3b78861cb 100644 --- a/llvm/lib/Target/Sparc/SparcFrameLowering.h +++ b/llvm/lib/Target/Sparc/SparcFrameLowering.h @@ -26,7 +26,7 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp index eff4ae3baf3..a636b35635c 100644 --- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp +++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.cpp @@ -309,8 +309,9 @@ static void emitIncrement(MachineBasicBlock &MBB, } } -void SystemZFrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); +void SystemZFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineFrameInfo *MFFrame = MF.getFrameInfo(); auto *ZII = static_cast<const SystemZInstrInfo *>(MF.getSubtarget().getInstrInfo()); diff --git a/llvm/lib/Target/SystemZ/SystemZFrameLowering.h b/llvm/lib/Target/SystemZ/SystemZFrameLowering.h index cefa56fd74e..60bad894ee4 100644 --- a/llvm/lib/Target/SystemZ/SystemZFrameLowering.h +++ b/llvm/lib/Target/SystemZ/SystemZFrameLowering.h @@ -40,7 +40,7 @@ public: override; void processFunctionBeforeFrameFinalized(MachineFunction &MF, RegScavenger *RS) const override; - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; bool hasFP(const MachineFunction &MF) const override; int getFrameIndexOffset(const MachineFunction &MF, int FI) const override; diff --git a/llvm/lib/Target/X86/X86FrameLowering.cpp b/llvm/lib/Target/X86/X86FrameLowering.cpp index 1d2c73c4308..a25643b2e2e 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.cpp +++ b/llvm/lib/Target/X86/X86FrameLowering.cpp @@ -565,8 +565,9 @@ static uint64_t calculateMaxStackAlign(const MachineFunction &MF) { - for 32-bit code, substitute %e?? registers for %r?? */ -void X86FrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); // Prologue goes in entry BB. +void X86FrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineBasicBlock::iterator MBBI = MBB.begin(); MachineFrameInfo *MFI = MF.getFrameInfo(); const Function *Fn = MF.getFunction(); @@ -1590,9 +1591,10 @@ GetScratchRegister(bool Is64Bit, bool IsLP64, const MachineFunction &MF, bool Pr // limit. static const uint64_t kSplitStackAvailable = 256; -void -X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { - MachineBasicBlock &prologueMBB = MF.front(); +void X86FrameLowering::adjustForSegmentedStacks( + MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { + assert(&PrologueMBB == &MF.front() && + "Shrink-wrapping is not implemented yet"); MachineFrameInfo *MFI = MF.getFrameInfo(); const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); const TargetInstrInfo &TII = *STI.getInstrInfo(); @@ -1634,8 +1636,9 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { // The MOV R10, RAX needs to be in a different block, since the RET we emit in // allocMBB needs to be last (terminating) instruction. - for (MachineBasicBlock::livein_iterator i = prologueMBB.livein_begin(), - e = prologueMBB.livein_end(); i != e; i++) { + for (MachineBasicBlock::livein_iterator i = PrologueMBB.livein_begin(), + e = PrologueMBB.livein_end(); + i != e; i++) { allocMBB->addLiveIn(*i); checkMBB->addLiveIn(*i); } @@ -1749,7 +1752,7 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { // This jump is taken if SP >= (Stacklet Limit + Stack Space required). // It jumps to normal execution of the function body. - BuildMI(checkMBB, DL, TII.get(X86::JA_1)).addMBB(&prologueMBB); + BuildMI(checkMBB, DL, TII.get(X86::JA_1)).addMBB(&PrologueMBB); // On 32 bit we first push the arguments size and then the frame size. On 64 // bit, we pass the stack frame size in r10 and the argument size in r11. @@ -1816,10 +1819,10 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { else BuildMI(allocMBB, DL, TII.get(X86::MORESTACK_RET)); - allocMBB->addSuccessor(&prologueMBB); + allocMBB->addSuccessor(&PrologueMBB); checkMBB->addSuccessor(allocMBB); - checkMBB->addSuccessor(&prologueMBB); + checkMBB->addSuccessor(&PrologueMBB); #ifdef XDEBUG MF.verify(); @@ -1841,7 +1844,8 @@ X86FrameLowering::adjustForSegmentedStacks(MachineFunction &MF) const { /// call inc_stack # doubles the stack space /// temp0 = sp - MaxStack /// if( temp0 < SP_LIMIT(P) ) goto IncStack else goto OldStart -void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const { +void X86FrameLowering::adjustForHiPEPrologue( + MachineFunction &MF, MachineBasicBlock &PrologueMBB) const { const X86Subtarget &STI = MF.getSubtarget<X86Subtarget>(); const TargetInstrInfo &TII = *STI.getInstrInfo(); MachineFrameInfo *MFI = MF.getFrameInfo(); @@ -1910,12 +1914,14 @@ void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const { // If the stack frame needed is larger than the guaranteed then runtime checks // and calls to "inc_stack_0" BIF should be inserted in the assembly prologue. if (MaxStack > Guaranteed) { - MachineBasicBlock &prologueMBB = MF.front(); + assert(&PrologueMBB == &MF.front() && + "Shrink-wrapping is not implemented yet"); MachineBasicBlock *stackCheckMBB = MF.CreateMachineBasicBlock(); MachineBasicBlock *incStackMBB = MF.CreateMachineBasicBlock(); - for (MachineBasicBlock::livein_iterator I = prologueMBB.livein_begin(), - E = prologueMBB.livein_end(); I != E; I++) { + for (MachineBasicBlock::livein_iterator I = PrologueMBB.livein_begin(), + E = PrologueMBB.livein_end(); + I != E; I++) { stackCheckMBB->addLiveIn(*I); incStackMBB->addLiveIn(*I); } @@ -1951,7 +1957,7 @@ void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const { // SPLimitOffset is in a fixed heap location (pointed by BP). addRegOffset(BuildMI(stackCheckMBB, DL, TII.get(CMPop)) .addReg(ScratchReg), PReg, false, SPLimitOffset); - BuildMI(stackCheckMBB, DL, TII.get(X86::JAE_1)).addMBB(&prologueMBB); + BuildMI(stackCheckMBB, DL, TII.get(X86::JAE_1)).addMBB(&PrologueMBB); // Create new MBB for IncStack: BuildMI(incStackMBB, DL, TII.get(CALLop)). @@ -1962,9 +1968,9 @@ void X86FrameLowering::adjustForHiPEPrologue(MachineFunction &MF) const { .addReg(ScratchReg), PReg, false, SPLimitOffset); BuildMI(incStackMBB, DL, TII.get(X86::JLE_1)).addMBB(incStackMBB); - stackCheckMBB->addSuccessor(&prologueMBB, 99); + stackCheckMBB->addSuccessor(&PrologueMBB, 99); stackCheckMBB->addSuccessor(incStackMBB, 1); - incStackMBB->addSuccessor(&prologueMBB, 99); + incStackMBB->addSuccessor(&PrologueMBB, 99); incStackMBB->addSuccessor(incStackMBB, 1); } #ifdef XDEBUG diff --git a/llvm/lib/Target/X86/X86FrameLowering.h b/llvm/lib/Target/X86/X86FrameLowering.h index 542bbbc79ae..a23bce9745e 100644 --- a/llvm/lib/Target/X86/X86FrameLowering.h +++ b/llvm/lib/Target/X86/X86FrameLowering.h @@ -35,12 +35,14 @@ public: /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; - void adjustForSegmentedStacks(MachineFunction &MF) const override; + void adjustForSegmentedStacks(MachineFunction &MF, + MachineBasicBlock &PrologueMBB) const override; - void adjustForHiPEPrologue(MachineFunction &MF) const override; + void adjustForHiPEPrologue(MachineFunction &MF, + MachineBasicBlock &PrologueMBB) const override; void processFunctionBeforeCalleeSavedScan(MachineFunction &MF, RegScavenger *RS = nullptr) const override; diff --git a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp index e0ac0e56781..bd834cc5be4 100644 --- a/llvm/lib/Target/XCore/XCoreFrameLowering.cpp +++ b/llvm/lib/Target/XCore/XCoreFrameLowering.cpp @@ -220,8 +220,9 @@ bool XCoreFrameLowering::hasFP(const MachineFunction &MF) const { MF.getFrameInfo()->hasVarSizedObjects(); } -void XCoreFrameLowering::emitPrologue(MachineFunction &MF) const { - MachineBasicBlock &MBB = MF.front(); // Prolog goes in entry BB +void XCoreFrameLowering::emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const { + assert(&MF.front() == &MBB && "Shrink-wrapping not yet supported"); MachineBasicBlock::iterator MBBI = MBB.begin(); MachineFrameInfo *MFI = MF.getFrameInfo(); MachineModuleInfo *MMI = &MF.getMMI(); diff --git a/llvm/lib/Target/XCore/XCoreFrameLowering.h b/llvm/lib/Target/XCore/XCoreFrameLowering.h index 7b169c2b8cd..607c7724895 100644 --- a/llvm/lib/Target/XCore/XCoreFrameLowering.h +++ b/llvm/lib/Target/XCore/XCoreFrameLowering.h @@ -27,7 +27,8 @@ namespace llvm { /// emitProlog/emitEpilog - These methods insert prolog and epilog code into /// the function. - void emitPrologue(MachineFunction &MF) const override; + void emitPrologue(MachineFunction &MF, + MachineBasicBlock &MBB) const override; void emitEpilogue(MachineFunction &MF, MachineBasicBlock &MBB) const override; |
