summaryrefslogtreecommitdiffstats
path: root/llvm/lib
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib')
-rw-r--r--llvm/lib/CodeGen/BranchFolding.cpp144
-rw-r--r--llvm/lib/CodeGen/BranchFolding.h8
-rw-r--r--llvm/lib/CodeGen/BranchRelaxation.cpp4
-rw-r--r--llvm/lib/CodeGen/LivePhysRegs.cpp22
-rw-r--r--llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp22
-rw-r--r--llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp22
6 files changed, 138 insertions, 84 deletions
diff --git a/llvm/lib/CodeGen/BranchFolding.cpp b/llvm/lib/CodeGen/BranchFolding.cpp
index 3c439e66944..fdd282ce124 100644
--- a/llvm/lib/CodeGen/BranchFolding.cpp
+++ b/llvm/lib/CodeGen/BranchFolding.cpp
@@ -31,6 +31,7 @@
#include "llvm/CodeGen/MachineFunction.h"
#include "llvm/CodeGen/MachineFunctionPass.h"
#include "llvm/CodeGen/MachineInstr.h"
+#include "llvm/CodeGen/MachineInstrBuilder.h"
#include "llvm/CodeGen/MachineJumpTableInfo.h"
#include "llvm/CodeGen/MachineLoopInfo.h"
#include "llvm/CodeGen/MachineModuleInfo.h"
@@ -365,15 +366,37 @@ static unsigned ComputeCommonTailLength(MachineBasicBlock *MBB1,
return TailLen;
}
-void BranchFolder::ReplaceTailWithBranchTo(MachineBasicBlock::iterator OldInst,
- MachineBasicBlock *NewDest) {
- TII->ReplaceTailWithBranchTo(OldInst, NewDest);
-
+void BranchFolder::replaceTailWithBranchTo(MachineBasicBlock::iterator OldInst,
+ MachineBasicBlock &NewDest) {
if (UpdateLiveIns) {
- NewDest->clearLiveIns();
- computeLiveIns(LiveRegs, *MRI, *NewDest);
+ // OldInst should always point to an instruction.
+ MachineBasicBlock &OldMBB = *OldInst->getParent();
+ LiveRegs.clear();
+ LiveRegs.addLiveOuts(OldMBB);
+ // Move backward to the place where will insert the jump.
+ MachineBasicBlock::iterator I = OldMBB.end();
+ do {
+ --I;
+ LiveRegs.stepBackward(*I);
+ } while (I != OldInst);
+
+ // Merging the tails may have switched some undef operand to non-undef ones.
+ // Add IMPLICIT_DEFS into OldMBB as necessary to have a definition of the
+ // register.
+ for (MachineBasicBlock::RegisterMaskPair P : NewDest.liveins()) {
+ // We computed the liveins with computeLiveIn earlier and should only see
+ // full registers:
+ assert(P.LaneMask == LaneBitmask::getAll() &&
+ "Can only handle full register.");
+ MCPhysReg Reg = P.PhysReg;
+ if (!LiveRegs.available(*MRI, Reg))
+ continue;
+ DebugLoc DL;
+ BuildMI(OldMBB, OldInst, DL, TII->get(TargetOpcode::IMPLICIT_DEF), Reg);
+ }
}
+ TII->ReplaceTailWithBranchTo(OldInst, &NewDest);
++NumTailMerge;
}
@@ -408,7 +431,7 @@ MachineBasicBlock *BranchFolder::SplitMBBAt(MachineBasicBlock &CurMBB,
MBBFreqInfo.setBlockFreq(NewMBB, MBBFreqInfo.getBlockFreq(&CurMBB));
if (UpdateLiveIns)
- computeLiveIns(LiveRegs, *MRI, *NewMBB);
+ computeAndAddLiveIns(LiveRegs, *NewMBB);
// Add the new block to the funclet.
const auto &FuncletI = FuncletMembership.find(&CurMBB);
@@ -766,43 +789,6 @@ bool BranchFolder::CreateCommonTailOnlyBlock(MachineBasicBlock *&PredBB,
return true;
}
-void BranchFolder::MergeCommonTailDebugLocs(unsigned commonTailIndex) {
- MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock();
-
- std::vector<MachineBasicBlock::iterator> NextCommonInsts(SameTails.size());
- for (unsigned int i = 0 ; i != SameTails.size() ; ++i) {
- if (i != commonTailIndex)
- NextCommonInsts[i] = SameTails[i].getTailStartPos();
- else {
- assert(SameTails[i].getTailStartPos() == MBB->begin() &&
- "MBB is not a common tail only block");
- }
- }
-
- for (auto &MI : *MBB) {
- if (MI.isDebugValue())
- continue;
- DebugLoc DL = MI.getDebugLoc();
- for (unsigned int i = 0 ; i < NextCommonInsts.size() ; i++) {
- if (i == commonTailIndex)
- continue;
-
- auto &Pos = NextCommonInsts[i];
- assert(Pos != SameTails[i].getBlock()->end() &&
- "Reached BB end within common tail");
- while (Pos->isDebugValue()) {
- ++Pos;
- assert(Pos != SameTails[i].getBlock()->end() &&
- "Reached BB end within common tail");
- }
- assert(MI.isIdenticalTo(*Pos) && "Expected matching MIIs!");
- DL = DILocation::getMergedLocation(DL, Pos->getDebugLoc());
- NextCommonInsts[i] = ++Pos;
- }
- MI.setDebugLoc(DL);
- }
-}
-
static void
mergeOperations(MachineBasicBlock::iterator MBBIStartPos,
MachineBasicBlock &MBBCommon) {
@@ -853,6 +839,67 @@ mergeOperations(MachineBasicBlock::iterator MBBIStartPos,
}
}
+void BranchFolder::mergeCommonTails(unsigned commonTailIndex) {
+ MachineBasicBlock *MBB = SameTails[commonTailIndex].getBlock();
+
+ std::vector<MachineBasicBlock::iterator> NextCommonInsts(SameTails.size());
+ for (unsigned int i = 0 ; i != SameTails.size() ; ++i) {
+ if (i != commonTailIndex) {
+ NextCommonInsts[i] = SameTails[i].getTailStartPos();
+ mergeOperations(SameTails[i].getTailStartPos(), *MBB);
+ } else {
+ assert(SameTails[i].getTailStartPos() == MBB->begin() &&
+ "MBB is not a common tail only block");
+ }
+ }
+
+ for (auto &MI : *MBB) {
+ if (MI.isDebugValue())
+ continue;
+ DebugLoc DL = MI.getDebugLoc();
+ for (unsigned int i = 0 ; i < NextCommonInsts.size() ; i++) {
+ if (i == commonTailIndex)
+ continue;
+
+ auto &Pos = NextCommonInsts[i];
+ assert(Pos != SameTails[i].getBlock()->end() &&
+ "Reached BB end within common tail");
+ while (Pos->isDebugValue()) {
+ ++Pos;
+ assert(Pos != SameTails[i].getBlock()->end() &&
+ "Reached BB end within common tail");
+ }
+ assert(MI.isIdenticalTo(*Pos) && "Expected matching MIIs!");
+ DL = DILocation::getMergedLocation(DL, Pos->getDebugLoc());
+ NextCommonInsts[i] = ++Pos;
+ }
+ MI.setDebugLoc(DL);
+ }
+
+ if (UpdateLiveIns) {
+ LivePhysRegs NewLiveIns(*TRI);
+ computeLiveIns(NewLiveIns, *MBB);
+
+ // The flag merging may lead to some register uses no longer using the
+ // <undef> flag, add IMPLICIT_DEFs in the predecessors as necessary.
+ for (MachineBasicBlock *Pred : MBB->predecessors()) {
+ LiveRegs.init(*TRI);
+ LiveRegs.addLiveOuts(*Pred);
+ MachineBasicBlock::iterator InsertBefore = Pred->getFirstTerminator();
+ for (unsigned Reg : NewLiveIns) {
+ if (!LiveRegs.available(*MRI, Reg))
+ continue;
+ DebugLoc DL;
+ BuildMI(*Pred, InsertBefore, DL, TII->get(TargetOpcode::IMPLICIT_DEF),
+ Reg);
+ }
+ }
+
+ MBB->clearLiveIns();
+ addLiveIns(*MBB, NewLiveIns);
+ }
+}
+
// See if any of the blocks in MergePotentials (which all have SuccBB as a
// successor, or all have no successor if it is null) can be tail-merged.
// If there is a successor, any blocks in MergePotentials that are not
@@ -955,8 +1002,9 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
// Recompute common tail MBB's edge weights and block frequency.
setCommonTailEdgeWeights(*MBB);
- // Merge debug locations across identical instructions for common tail.
- MergeCommonTailDebugLocs(commonTailIndex);
+ // Merge debug locations, MMOs and undef flags across identical instructions
+ // for common tail.
+ mergeCommonTails(commonTailIndex);
// MBB is common tail. Adjust all other BB's to jump to this one.
// Traversal must be forwards so erases work.
@@ -967,10 +1015,8 @@ bool BranchFolder::TryTailMergeBlocks(MachineBasicBlock *SuccBB,
continue;
DEBUG(dbgs() << "BB#" << SameTails[i].getBlock()->getNumber()
<< (i == e-1 ? "" : ", "));
- // Merge operations (MMOs, undef flags)
- mergeOperations(SameTails[i].getTailStartPos(), *MBB);
// Hack the end off BB i, making it jump to BB commonTailIndex instead.
- ReplaceTailWithBranchTo(SameTails[i].getTailStartPos(), MBB);
+ replaceTailWithBranchTo(SameTails[i].getTailStartPos(), *MBB);
// BB i is no longer a predecessor of SuccBB; remove it from the worklist.
MergePotentials.erase(SameTails[i].getMPIter());
}
diff --git a/llvm/lib/CodeGen/BranchFolding.h b/llvm/lib/CodeGen/BranchFolding.h
index 92681137e4c..f6efcb718c9 100644
--- a/llvm/lib/CodeGen/BranchFolding.h
+++ b/llvm/lib/CodeGen/BranchFolding.h
@@ -146,8 +146,8 @@ namespace llvm {
/// Delete the instruction OldInst and everything after it, replacing it
/// with an unconditional branch to NewDest.
- void ReplaceTailWithBranchTo(MachineBasicBlock::iterator OldInst,
- MachineBasicBlock *NewDest);
+ void replaceTailWithBranchTo(MachineBasicBlock::iterator OldInst,
+ MachineBasicBlock &NewDest);
/// Given a machine basic block and an iterator into it, split the MBB so
/// that the part before the iterator falls into the part starting at the
@@ -182,8 +182,8 @@ namespace llvm {
unsigned &commonTailIndex);
/// Create merged DebugLocs of identical instructions across SameTails and
- /// assign it to the instruction in common tail.
- void MergeCommonTailDebugLocs(unsigned commonTailIndex);
+ /// assign it to the instruction in common tail; merge MMOs and undef flags.
+ void mergeCommonTails(unsigned commonTailIndex);
bool OptimizeBranches(MachineFunction &MF);
diff --git a/llvm/lib/CodeGen/BranchRelaxation.cpp b/llvm/lib/CodeGen/BranchRelaxation.cpp
index 27ee12c4c5f..cbee43ecd20 100644
--- a/llvm/lib/CodeGen/BranchRelaxation.cpp
+++ b/llvm/lib/CodeGen/BranchRelaxation.cpp
@@ -259,7 +259,7 @@ MachineBasicBlock *BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI,
// Need to fix live-in lists if we track liveness.
if (TRI->trackLivenessAfterRegAlloc(*MF))
- computeLiveIns(LiveRegs, MF->getRegInfo(), *NewBB);
+ computeAndAddLiveIns(LiveRegs, *NewBB);
++NumSplit;
@@ -348,7 +348,7 @@ bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
// Need to fix live-in lists if we track liveness.
if (TRI->trackLivenessAfterRegAlloc(*MF))
- computeLiveIns(LiveRegs, MF->getRegInfo(), NewBB);
+ computeAndAddLiveIns(LiveRegs, NewBB);
}
// We now have an appropriate fall-through block in place (either naturally or
diff --git a/llvm/lib/CodeGen/LivePhysRegs.cpp b/llvm/lib/CodeGen/LivePhysRegs.cpp
index 870d8cc71c7..ed05acfac4d 100644
--- a/llvm/lib/CodeGen/LivePhysRegs.cpp
+++ b/llvm/lib/CodeGen/LivePhysRegs.cpp
@@ -218,16 +218,22 @@ void LivePhysRegs::addLiveIns(const MachineBasicBlock &MBB) {
}
void llvm::computeLiveIns(LivePhysRegs &LiveRegs,
- const MachineRegisterInfo &MRI,
- MachineBasicBlock &MBB) {
+ const MachineBasicBlock &MBB) {
+ const MachineFunction &MF = *MBB.getParent();
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
- assert(MBB.livein_empty());
LiveRegs.init(TRI);
LiveRegs.addLiveOutsNoPristines(MBB);
- for (MachineInstr &MI : make_range(MBB.rbegin(), MBB.rend()))
+ for (const MachineInstr &MI : make_range(MBB.rbegin(), MBB.rend()))
LiveRegs.stepBackward(MI);
+}
- for (unsigned Reg : LiveRegs) {
+void llvm::addLiveIns(MachineBasicBlock &MBB, const LivePhysRegs &LiveRegs) {
+ assert(MBB.livein_empty() && "Expected empty live-in list");
+ const MachineFunction &MF = *MBB.getParent();
+ const MachineRegisterInfo &MRI = MF.getRegInfo();
+ const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+ for (MCPhysReg Reg : LiveRegs) {
if (MRI.isReserved(Reg))
continue;
// Skip the register if we are about to add one of its super registers.
@@ -243,3 +249,9 @@ void llvm::computeLiveIns(LivePhysRegs &LiveRegs,
MBB.addLiveIn(Reg);
}
}
+
+void llvm::computeAndAddLiveIns(LivePhysRegs &LiveRegs,
+ MachineBasicBlock &MBB) {
+ computeLiveIns(LiveRegs, MBB);
+ addLiveIns(MBB, LiveRegs);
+}
diff --git a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
index fdb90f40aa8..f2530793959 100644
--- a/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ExpandPseudoInsts.cpp
@@ -672,16 +672,15 @@ bool AArch64ExpandPseudo::expandCMP_SWAP(
MI.eraseFromParent();
// Recompute livein lists.
- const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
LivePhysRegs LiveRegs;
- computeLiveIns(LiveRegs, MRI, *DoneBB);
- computeLiveIns(LiveRegs, MRI, *StoreBB);
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *DoneBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
// Do an extra pass around the loop to get loop carried registers right.
StoreBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
LoadCmpBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
return true;
}
@@ -766,16 +765,15 @@ bool AArch64ExpandPseudo::expandCMP_SWAP_128(
MI.eraseFromParent();
// Recompute liveness bottom up.
- const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
LivePhysRegs LiveRegs;
- computeLiveIns(LiveRegs, MRI, *DoneBB);
- computeLiveIns(LiveRegs, MRI, *StoreBB);
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *DoneBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
// Do an extra pass in the loop to get the loop carried dependencies right.
StoreBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
LoadCmpBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
return true;
}
diff --git a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
index 229346fdb82..be7afae5ea4 100644
--- a/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
+++ b/llvm/lib/Target/ARM/ARMExpandPseudoInsts.cpp
@@ -860,16 +860,15 @@ bool ARMExpandPseudo::ExpandCMP_SWAP(MachineBasicBlock &MBB,
MI.eraseFromParent();
// Recompute livein lists.
- const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
LivePhysRegs LiveRegs;
- computeLiveIns(LiveRegs, MRI, *DoneBB);
- computeLiveIns(LiveRegs, MRI, *StoreBB);
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *DoneBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
// Do an extra pass around the loop to get loop carried registers right.
StoreBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
LoadCmpBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
return true;
}
@@ -980,16 +979,15 @@ bool ARMExpandPseudo::ExpandCMP_SWAP_64(MachineBasicBlock &MBB,
MI.eraseFromParent();
// Recompute livein lists.
- const MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
LivePhysRegs LiveRegs;
- computeLiveIns(LiveRegs, MRI, *DoneBB);
- computeLiveIns(LiveRegs, MRI, *StoreBB);
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *DoneBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
// Do an extra pass around the loop to get loop carried registers right.
StoreBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *StoreBB);
+ computeAndAddLiveIns(LiveRegs, *StoreBB);
LoadCmpBB->clearLiveIns();
- computeLiveIns(LiveRegs, MRI, *LoadCmpBB);
+ computeAndAddLiveIns(LiveRegs, *LoadCmpBB);
return true;
}
OpenPOWER on IntegriCloud