summaryrefslogtreecommitdiffstats
path: root/llvm/lib/CodeGen
diff options
context:
space:
mode:
Diffstat (limited to 'llvm/lib/CodeGen')
-rw-r--r--llvm/lib/CodeGen/LiveInterval.cpp94
-rw-r--r--llvm/lib/CodeGen/LiveIntervalAnalysis.cpp26
-rw-r--r--llvm/lib/CodeGen/LiveRangeCalc.cpp191
-rw-r--r--llvm/lib/CodeGen/LiveRangeCalc.h51
-rw-r--r--llvm/lib/CodeGen/LiveRangeEdit.cpp10
-rw-r--r--llvm/lib/CodeGen/MachineVerifier.cpp15
-rw-r--r--llvm/lib/CodeGen/RegAllocBase.cpp1
-rw-r--r--llvm/lib/CodeGen/RegisterCoalescer.cpp116
-rw-r--r--llvm/lib/CodeGen/RenameIndependentSubregs.cpp5
-rw-r--r--llvm/lib/CodeGen/SplitKit.cpp271
-rw-r--r--llvm/lib/CodeGen/SplitKit.h27
-rw-r--r--llvm/lib/CodeGen/VirtRegMap.cpp1
12 files changed, 652 insertions, 156 deletions
diff --git a/llvm/lib/CodeGen/LiveInterval.cpp b/llvm/lib/CodeGen/LiveInterval.cpp
index df26ce6046a..509bf247948 100644
--- a/llvm/lib/CodeGen/LiveInterval.cpp
+++ b/llvm/lib/CodeGen/LiveInterval.cpp
@@ -59,18 +59,32 @@ public:
typedef LiveRange::Segment Segment;
typedef IteratorT iterator;
- VNInfo *createDeadDef(SlotIndex Def, VNInfo::Allocator &VNInfoAllocator) {
+ /// A counterpart of LiveRange::createDeadDef: Make sure the range has a
+ /// value defined at @p Def.
+ /// If @p ForVNI is null, and there is no value defined at @p Def, a new
+ /// value will be allocated using @p VNInfoAllocator.
+ /// If @p ForVNI is null, the return value is the value defined at @p Def,
+ /// either a pre-existing one, or the one newly created.
+ /// If @p ForVNI is not null, then @p Def should be the location where
+ /// @p ForVNI is defined. If the range does not have a value defined at
+ /// @p Def, the value @p ForVNI will be used instead of allocating a new
+ /// one. If the range already has a value defined at @p Def, it must be
+ /// same as @p ForVNI. In either case, @p ForVNI will be the return value.
+ VNInfo *createDeadDef(SlotIndex Def, VNInfo::Allocator *VNInfoAllocator,
+ VNInfo *ForVNI) {
assert(!Def.isDead() && "Cannot define a value at the dead slot");
-
+ assert((!ForVNI || ForVNI->def == Def) &&
+ "If ForVNI is specified, it must match Def");
iterator I = impl().find(Def);
if (I == segments().end()) {
- VNInfo *VNI = LR->getNextValue(Def, VNInfoAllocator);
+ VNInfo *VNI = ForVNI ? ForVNI : LR->getNextValue(Def, *VNInfoAllocator);
impl().insertAtEnd(Segment(Def, Def.getDeadSlot(), VNI));
return VNI;
}
Segment *S = segmentAt(I);
if (SlotIndex::isSameInstr(Def, S->start)) {
+ assert((!ForVNI || ForVNI == S->valno) && "Value number mismatch");
assert(S->valno->def == S->start && "Inconsistent existing value def");
// It is possible to have both normal and early-clobber defs of the same
@@ -84,7 +98,7 @@ public:
return S->valno;
}
assert(SlotIndex::isEarlierInstr(Def, S->start) && "Already live at def");
- VNInfo *VNI = LR->getNextValue(Def, VNInfoAllocator);
+ VNInfo *VNI = ForVNI ? ForVNI : LR->getNextValue(Def, *VNInfoAllocator);
segments().insert(I, Segment(Def, Def.getDeadSlot(), VNI));
return VNI;
}
@@ -93,7 +107,7 @@ public:
if (segments().empty())
return nullptr;
iterator I =
- impl().findInsertPos(Segment(Use.getPrevSlot(), Use, nullptr));
+ impl().findInsertPos(Segment(Use.getPrevSlot(), Use, nullptr));
if (I == segments().begin())
return nullptr;
--I;
@@ -104,6 +118,25 @@ public:
return I->valno;
}
+ std::pair<VNInfo*,bool> extendInBlock(ArrayRef<SlotIndex> Undefs,
+ SlotIndex StartIdx, SlotIndex Use) {
+ if (segments().empty())
+ return std::make_pair(nullptr, false);
+ SlotIndex BeforeUse = Use.getPrevSlot();
+ iterator I = impl().findInsertPos(Segment(BeforeUse, Use, nullptr));
+ if (I == segments().begin())
+ return std::make_pair(nullptr, LR->isUndefIn(Undefs, StartIdx, BeforeUse));
+ --I;
+ if (I->end <= StartIdx)
+ return std::make_pair(nullptr, LR->isUndefIn(Undefs, StartIdx, BeforeUse));
+ if (I->end < Use) {
+ if (LR->isUndefIn(Undefs, I->end, BeforeUse))
+ return std::make_pair(nullptr, true);
+ extendSegmentEndTo(I, Use);
+ }
+ return std::make_pair(I->valno, false);
+ }
+
/// This method is used when we want to extend the segment specified
/// by I to end at the specified endpoint. To do this, we should
/// merge and eliminate all segments that this will overlap
@@ -320,13 +353,20 @@ LiveRange::iterator LiveRange::find(SlotIndex Pos) {
return I;
}
-VNInfo *LiveRange::createDeadDef(SlotIndex Def,
- VNInfo::Allocator &VNInfoAllocator) {
+VNInfo *LiveRange::createDeadDef(SlotIndex Def, VNInfo::Allocator &VNIAlloc) {
+ // Use the segment set, if it is available.
+ if (segmentSet != nullptr)
+ return CalcLiveRangeUtilSet(this).createDeadDef(Def, &VNIAlloc, nullptr);
+ // Otherwise use the segment vector.
+ return CalcLiveRangeUtilVector(this).createDeadDef(Def, &VNIAlloc, nullptr);
+}
+
+VNInfo *LiveRange::createDeadDef(VNInfo *VNI) {
// Use the segment set, if it is available.
if (segmentSet != nullptr)
- return CalcLiveRangeUtilSet(this).createDeadDef(Def, VNInfoAllocator);
+ return CalcLiveRangeUtilSet(this).createDeadDef(VNI->def, nullptr, VNI);
// Otherwise use the segment vector.
- return CalcLiveRangeUtilVector(this).createDeadDef(Def, VNInfoAllocator);
+ return CalcLiveRangeUtilVector(this).createDeadDef(VNI->def, nullptr, VNI);
}
// overlaps - Return true if the intersection of the two live ranges is
@@ -507,9 +547,15 @@ void LiveRange::append(const Segment S) {
segments.push_back(S);
}
-/// extendInBlock - If this range is live before Kill in the basic
-/// block that starts at StartIdx, extend it to be live up to Kill and return
-/// the value. If there is no live range before Kill, return NULL.
+std::pair<VNInfo*,bool> LiveRange::extendInBlock(ArrayRef<SlotIndex> Undefs,
+ SlotIndex StartIdx, SlotIndex Kill) {
+ // Use the segment set, if it is available.
+ if (segmentSet != nullptr)
+ return CalcLiveRangeUtilSet(this).extendInBlock(Undefs, StartIdx, Kill);
+ // Otherwise use the segment vector.
+ return CalcLiveRangeUtilVector(this).extendInBlock(Undefs, StartIdx, Kill);
+}
+
VNInfo *LiveRange::extendInBlock(SlotIndex StartIdx, SlotIndex Kill) {
// Use the segment set, if it is available.
if (segmentSet != nullptr)
@@ -824,6 +870,30 @@ unsigned LiveInterval::getSize() const {
return Sum;
}
+void LiveInterval::computeSubRangeUndefs(SmallVectorImpl<SlotIndex> &Undefs,
+ LaneBitmask LaneMask,
+ const MachineRegisterInfo &MRI,
+ const SlotIndexes &Indexes) const {
+ assert(TargetRegisterInfo::isVirtualRegister(reg));
+ LaneBitmask VRegMask = MRI.getMaxLaneMaskForVReg(reg);
+ assert((VRegMask & LaneMask) != 0);
+ const TargetRegisterInfo &TRI = *MRI.getTargetRegisterInfo();
+ for (const MachineOperand &MO : MRI.def_operands(reg)) {
+ if (!MO.isUndef())
+ continue;
+ unsigned SubReg = MO.getSubReg();
+ assert(SubReg != 0 && "Undef should only be set on subreg defs");
+ LaneBitmask DefMask = TRI.getSubRegIndexLaneMask(SubReg);
+ LaneBitmask UndefMask = VRegMask & ~DefMask;
+ if ((UndefMask & LaneMask) != 0) {
+ const MachineInstr &MI = *MO.getParent();
+ bool EarlyClobber = MO.isEarlyClobber();
+ SlotIndex Pos = Indexes.getInstructionIndex(MI).getRegSlot(EarlyClobber);
+ Undefs.push_back(Pos);
+ }
+ }
+}
+
raw_ostream& llvm::operator<<(raw_ostream& os, const LiveRange::Segment &S) {
return os << '[' << S.start << ',' << S.end << ':' << S.valno->id << ')';
}
diff --git a/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp b/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
index 26b5cba3294..574684b19f2 100644
--- a/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
+++ b/llvm/lib/CodeGen/LiveIntervalAnalysis.cpp
@@ -504,8 +504,7 @@ bool LiveIntervals::computeDeadValues(LiveInterval &LI,
return MayHaveSplitComponents;
}
-void LiveIntervals::shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg)
-{
+void LiveIntervals::shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg) {
DEBUG(dbgs() << "Shrink: " << SR << '\n');
assert(TargetRegisterInfo::isVirtualRegister(Reg)
&& "Can only shrink virtual registers");
@@ -516,12 +515,14 @@ void LiveIntervals::shrinkToUses(LiveInterval::SubRange &SR, unsigned Reg)
SlotIndex LastIdx;
for (MachineOperand &MO : MRI->reg_operands(Reg)) {
MachineInstr *UseMI = MO.getParent();
- if (UseMI->isDebugValue())
+ if (UseMI->isDebugValue() || !MO.readsReg())
continue;
// Maybe the operand is for a subregister we don't care about.
unsigned SubReg = MO.getSubReg();
if (SubReg != 0) {
LaneBitmask LaneMask = TRI->getSubRegIndexLaneMask(SubReg);
+ if (MO.isDef())
+ LaneMask = ~LaneMask & MRI->getMaxLaneMaskForVReg(Reg);
if ((LaneMask & SR.LaneMask) == 0)
continue;
}
@@ -578,7 +579,7 @@ void LiveIntervals::extendToIndices(LiveRange &LR,
assert(LRCalc && "LRCalc not initialized.");
LRCalc->reset(MF, getSlotIndexes(), DomTree, &getVNInfoAllocator());
for (unsigned i = 0, e = Indices.size(); i != e; ++i)
- LRCalc->extend(LR, Indices[i]);
+ LRCalc->extend(LR, Indices[i], /*PhysReg=*/0, /*Undefs=*/{});
}
void LiveIntervals::pruneValue(LiveRange &LR, SlotIndex Kill,
@@ -954,7 +955,8 @@ public:
LiveInterval &LI = LIS.getInterval(Reg);
if (LI.hasSubRanges()) {
unsigned SubReg = MO.getSubReg();
- LaneBitmask LaneMask = TRI.getSubRegIndexLaneMask(SubReg);
+ LaneBitmask LaneMask = SubReg ? TRI.getSubRegIndexLaneMask(SubReg)
+ : MRI.getMaxLaneMaskForVReg(Reg);
for (LiveInterval::SubRange &S : LI.subranges()) {
if ((S.LaneMask & LaneMask) == 0)
continue;
@@ -1545,15 +1547,19 @@ void LiveIntervals::removePhysRegDefAt(unsigned Reg, SlotIndex Pos) {
}
void LiveIntervals::removeVRegDefAt(LiveInterval &LI, SlotIndex Pos) {
+ // LI may not have the main range computed yet, but its subranges may
+ // be present.
VNInfo *VNI = LI.getVNInfoAt(Pos);
- if (VNI == nullptr)
- return;
- LI.removeValNo(VNI);
+ if (VNI != nullptr) {
+ assert(VNI->def.getBaseIndex() == Pos.getBaseIndex());
+ LI.removeValNo(VNI);
+ }
- // Also remove the value in subranges.
+ // Also remove the value defined in subranges.
for (LiveInterval::SubRange &S : LI.subranges()) {
if (VNInfo *SVNI = S.getVNInfoAt(Pos))
- S.removeValNo(SVNI);
+ if (SVNI->def.getBaseIndex() == Pos.getBaseIndex())
+ S.removeValNo(SVNI);
}
LI.removeEmptySubRanges();
}
diff --git a/llvm/lib/CodeGen/LiveRangeCalc.cpp b/llvm/lib/CodeGen/LiveRangeCalc.cpp
index db91ca113dc..b480747b11c 100644
--- a/llvm/lib/CodeGen/LiveRangeCalc.cpp
+++ b/llvm/lib/CodeGen/LiveRangeCalc.cpp
@@ -12,6 +12,7 @@
//===----------------------------------------------------------------------===//
#include "LiveRangeCalc.h"
+#include "llvm/ADT/SetVector.h"
#include "llvm/CodeGen/MachineDominators.h"
#include "llvm/CodeGen/MachineRegisterInfo.h"
@@ -23,6 +24,7 @@ void LiveRangeCalc::resetLiveOutMap() {
unsigned NumBlocks = MF->getNumBlockIDs();
Seen.clear();
Seen.resize(NumBlocks);
+ EntryInfoMap.clear();
Map.resize(NumBlocks);
}
@@ -64,9 +66,9 @@ void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) {
unsigned SubReg = MO.getSubReg();
if (LI.hasSubRanges() || (SubReg != 0 && TrackSubRegs)) {
- LaneBitmask Mask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg)
- : MRI->getMaxLaneMaskForVReg(Reg);
-
+ LaneBitmask WholeMask = MRI->getMaxLaneMaskForVReg(Reg);
+ LaneBitmask SubMask = SubReg != 0 ? TRI.getSubRegIndexLaneMask(SubReg)
+ : WholeMask;
// If this is the first time we see a subregister def, initialize
// subranges by creating a copy of the main range.
if (!LI.hasSubRanges() && !LI.empty()) {
@@ -74,17 +76,18 @@ void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) {
LI.createSubRangeFrom(*Alloc, ClassMask, LI);
}
+ LaneBitmask Mask = SubMask;
for (LiveInterval::SubRange &S : LI.subranges()) {
// A Mask for subregs common to the existing subrange and current def.
LaneBitmask Common = S.LaneMask & Mask;
if (Common == 0)
continue;
- // A Mask for subregs covered by the subrange but not the current def.
- LaneBitmask LRest = S.LaneMask & ~Mask;
LiveInterval::SubRange *CommonRange;
- if (LRest != 0) {
- // Split current subrange into Common and LRest ranges.
- S.LaneMask = LRest;
+ // A Mask for subregs covered by the subrange but not the current def.
+ if (LaneBitmask RM = S.LaneMask & ~Mask) {
+ // Split the subrange S into two parts: one covered by the current
+ // def (CommonRange), and the one not affected by it (updated S).
+ S.LaneMask = RM;
CommonRange = LI.createSubRangeFrom(*Alloc, Common, S);
} else {
assert(Common == S.LaneMask);
@@ -116,8 +119,9 @@ void LiveRangeCalc::calculate(LiveInterval &LI, bool TrackSubRegs) {
// necessary.
if (LI.hasSubRanges()) {
for (LiveInterval::SubRange &S : LI.subranges()) {
- resetLiveOutMap();
- extendToUses(S, Reg, S.LaneMask);
+ LiveRangeCalc SubLRC;
+ SubLRC.reset(MF, Indexes, DomTree, Alloc);
+ SubLRC.extendToUses(S, Reg, S.LaneMask, &LI);
}
LI.clear();
constructMainRangeFromSubranges(LI);
@@ -139,9 +143,8 @@ void LiveRangeCalc::constructMainRangeFromSubranges(LiveInterval &LI) {
MainRange.createDeadDef(VNI->def, *Alloc);
}
}
-
resetLiveOutMap();
- extendToUses(MainRange, LI.reg);
+ extendToUses(MainRange, LI.reg, ~0U, &LI);
}
void LiveRangeCalc::createDeadDefs(LiveRange &LR, unsigned Reg) {
@@ -154,8 +157,12 @@ void LiveRangeCalc::createDeadDefs(LiveRange &LR, unsigned Reg) {
}
-void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg,
- LaneBitmask Mask) {
+void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask Mask,
+ LiveInterval *LI) {
+ SmallVector<SlotIndex, 4> Undefs;
+ if (LI != nullptr)
+ LI->computeSubRangeUndefs(Undefs, Mask, *MRI, *Indexes);
+
// Visit all operands that read Reg. This may include partial defs.
const TargetRegisterInfo &TRI = *MRI->getTargetRegisterInfo();
for (MachineOperand &MO : MRI->reg_nodbg_operands(Reg)) {
@@ -163,20 +170,16 @@ void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg,
// by LiveIntervalAnalysis::addKillFlags().
if (MO.isUse())
MO.setIsKill(false);
- else {
- // We only care about uses, but on the main range (mask ~0u) this includes
- // the "virtual" reads happening for subregister defs.
- if (Mask != ~0u)
- continue;
- }
-
if (!MO.readsReg())
continue;
+
unsigned SubReg = MO.getSubReg();
if (SubReg != 0) {
- LaneBitmask SubRegMask = TRI.getSubRegIndexLaneMask(SubReg);
+ LaneBitmask SLM = TRI.getSubRegIndexLaneMask(SubReg);
+ if (MO.isDef())
+ SLM = MRI->getMaxLaneMaskForVReg(Reg) & ~SLM;
// Ignore uses not covering the current subrange.
- if ((SubRegMask & Mask) == 0)
+ if ((SLM & Mask) == 0)
continue;
}
@@ -205,7 +208,7 @@ void LiveRangeCalc::extendToUses(LiveRange &LR, unsigned Reg,
// MI is reading Reg. We may have visited MI before if it happens to be
// reading Reg multiple times. That is OK, extend() is idempotent.
- extend(LR, UseIdx, Reg);
+ extend(LR, UseIdx, Reg, Undefs);
}
}
@@ -235,8 +238,8 @@ void LiveRangeCalc::updateFromLiveIns() {
LiveIn.clear();
}
-
-void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg) {
+void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs) {
assert(Use.isValid() && "Invalid SlotIndex");
assert(Indexes && "Missing SlotIndexes");
assert(DomTree && "Missing dominator tree");
@@ -245,14 +248,15 @@ void LiveRangeCalc::extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg) {
assert(UseMBB && "No MBB at Use");
// Is there a def in the same MBB we can extend?
- if (LR.extendInBlock(Indexes->getMBBStartIdx(UseMBB), Use))
+ auto EP = LR.extendInBlock(Undefs, Indexes->getMBBStartIdx(UseMBB), Use);
+ if (EP.first != nullptr || EP.second)
return;
// Find the single reaching def, or determine if Use is jointly dominated by
// multiple values, and we may need to create even more phi-defs to preserve
// VNInfo SSA form. Perform a search for all predecessor blocks where we
// know the dominating VNInfo.
- if (findReachingDefs(LR, *UseMBB, Use, PhysReg))
+ if (findReachingDefs(LR, *UseMBB, Use, PhysReg, Undefs))
return;
// When there were multiple different values, we may need new PHIs.
@@ -271,8 +275,72 @@ void LiveRangeCalc::calculateValues() {
}
+bool LiveRangeCalc::isDefOnEntry(LiveRange &LR, ArrayRef<SlotIndex> Undefs,
+ MachineBasicBlock &MBB, BitVector &DefOnEntry,
+ BitVector &UndefOnEntry) {
+ unsigned BN = MBB.getNumber();
+ if (DefOnEntry[BN])
+ return true;
+ if (UndefOnEntry[BN])
+ return false;
+
+ auto MarkDefined =
+ [this,BN,&DefOnEntry,&UndefOnEntry] (MachineBasicBlock &B) -> bool {
+ for (MachineBasicBlock *S : B.successors())
+ DefOnEntry[S->getNumber()] = true;
+ DefOnEntry[BN] = true;
+ return true;
+ };
+
+ SetVector<unsigned> WorkList;
+ // Checking if the entry of MBB is reached by some def: add all predecessors
+ // that are potentially defined-on-exit to the work list.
+ for (MachineBasicBlock *P : MBB.predecessors())
+ WorkList.insert(P->getNumber());
+
+ for (unsigned i = 0; i != WorkList.size(); ++i) {
+ // Determine if the exit from the block is reached by some def.
+ unsigned N = WorkList[i];
+ MachineBasicBlock &B = *MF->getBlockNumbered(N);
+ if (Seen[N] && Map[&B].first != nullptr)
+ return MarkDefined(B);
+ SlotIndex Begin, End;
+ std::tie(Begin, End) = Indexes->getMBBRange(&B);
+ LiveRange::iterator UB = std::upper_bound(LR.begin(), LR.end(), End);
+ if (UB != LR.begin()) {
+ LiveRange::Segment &Seg = *std::prev(UB);
+ if (Seg.end > Begin) {
+ // There is a segment that overlaps B. If the range is not explicitly
+ // undefined between the end of the segment and the end of the block,
+ // treat the block as defined on exit. If it is, go to the next block
+ // on the work list.
+ if (LR.isUndefIn(Undefs, Seg.end, End))
+ continue;
+ return MarkDefined(B);
+ }
+ }
+
+ // No segment overlaps with this block. If this block is not defined on
+ // entry, or it undefines the range, do not process its predecessors.
+ if (UndefOnEntry[N] || LR.isUndefIn(Undefs, Begin, End)) {
+ UndefOnEntry[N] = true;
+ continue;
+ }
+ if (DefOnEntry[N])
+ return MarkDefined(B);
+
+ // Still don't know: add all predecessors to the work list.
+ for (MachineBasicBlock *P : B.predecessors())
+ WorkList.insert(P->getNumber());
+ }
+
+ UndefOnEntry[BN] = true;
+ return false;
+}
+
bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
- SlotIndex Use, unsigned PhysReg) {
+ SlotIndex Use, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs) {
unsigned UseMBBNum = UseMBB.getNumber();
// Block numbers where LR should be live-in.
@@ -282,12 +350,14 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
bool UniqueVNI = true;
VNInfo *TheVNI = nullptr;
+ bool FoundUndef = false;
+
// Using Seen as a visited set, perform a BFS for all reaching defs.
for (unsigned i = 0; i != WorkList.size(); ++i) {
MachineBasicBlock *MBB = MF->getBlockNumbered(WorkList[i]);
#ifndef NDEBUG
- if (MBB->pred_empty()) {
+ if (Undefs.size() > 0 && MBB->pred_empty()) {
MBB->getParent()->verify();
errs() << "Use of " << PrintReg(PhysReg)
<< " does not have a corresponding definition on every path:\n";
@@ -306,6 +376,7 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
llvm_unreachable("Invalid global physical register");
}
#endif
+ FoundUndef |= MBB->pred_empty();
for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
PE = MBB->pred_end(); PI != PE; ++PI) {
@@ -326,18 +397,21 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
// First time we see Pred. Try to determine the live-out value, but set
// it as null if Pred is live-through with an unknown value.
- VNInfo *VNI = LR.extendInBlock(Start, End);
+ auto EP = LR.extendInBlock(Undefs, Start, End);
+ VNInfo *VNI = EP.first;
+ FoundUndef |= EP.second;
setLiveOutValue(Pred, VNI);
if (VNI) {
if (TheVNI && TheVNI != VNI)
UniqueVNI = false;
TheVNI = VNI;
- continue;
}
+ if (VNI || EP.second)
+ continue;
// No, we need a live-in value for Pred as well
if (Pred != &UseMBB)
- WorkList.push_back(Pred->getNumber());
+ WorkList.push_back(Pred->getNumber());
else
// Loopback to UseMBB, so value is really live through.
Use = SlotIndex();
@@ -345,6 +419,9 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
}
LiveIn.clear();
+ FoundUndef |= (TheVNI == nullptr);
+ if (Undefs.size() > 0 && FoundUndef)
+ UniqueVNI = false;
// Both updateSSA() and LiveRangeUpdater benefit from ordered blocks, but
// neither require it. Skip the sorting overhead for small updates.
@@ -353,27 +430,39 @@ bool LiveRangeCalc::findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
// If a unique reaching def was found, blit in the live ranges immediately.
if (UniqueVNI) {
+ assert(TheVNI != nullptr);
LiveRangeUpdater Updater(&LR);
- for (SmallVectorImpl<unsigned>::const_iterator I = WorkList.begin(),
- E = WorkList.end(); I != E; ++I) {
- SlotIndex Start, End;
- std::tie(Start, End) = Indexes->getMBBRange(*I);
- // Trim the live range in UseMBB.
- if (*I == UseMBBNum && Use.isValid())
- End = Use;
- else
- Map[MF->getBlockNumbered(*I)] = LiveOutPair(TheVNI, nullptr);
- Updater.add(Start, End, TheVNI);
+ for (unsigned BN : WorkList) {
+ SlotIndex Start, End;
+ std::tie(Start, End) = Indexes->getMBBRange(BN);
+ // Trim the live range in UseMBB.
+ if (BN == UseMBBNum && Use.isValid())
+ End = Use;
+ else
+ Map[MF->getBlockNumbered(BN)] = LiveOutPair(TheVNI, nullptr);
+ Updater.add(Start, End, TheVNI);
}
return true;
}
+ // Prepare the defined/undefined bit vectors.
+ auto EF = EntryInfoMap.find(&LR);
+ if (EF == EntryInfoMap.end()) {
+ unsigned N = MF->getNumBlockIDs();
+ EF = EntryInfoMap.insert({&LR, {BitVector(), BitVector()}}).first;
+ EF->second.first.resize(N);
+ EF->second.second.resize(N);
+ }
+ BitVector &DefOnEntry = EF->second.first;
+ BitVector &UndefOnEntry = EF->second.second;
+
// Multiple values were found, so transfer the work list to the LiveIn array
// where UpdateSSA will use it as a work list.
LiveIn.reserve(WorkList.size());
- for (SmallVectorImpl<unsigned>::const_iterator
- I = WorkList.begin(), E = WorkList.end(); I != E; ++I) {
- MachineBasicBlock *MBB = MF->getBlockNumbered(*I);
+ for (unsigned BN : WorkList) {
+ MachineBasicBlock *MBB = MF->getBlockNumbered(BN);
+ if (Undefs.size() > 0 && !isDefOnEntry(LR, Undefs, *MBB, DefOnEntry, UndefOnEntry))
+ continue;
addLiveInBlock(LR, DomTree->getNode(MBB));
if (MBB == &UseMBB)
LiveIn.back().Kill = Use;
@@ -458,10 +547,12 @@ void LiveRangeCalc::updateSSA() {
I.DomNode = nullptr;
// Add liveness since updateFromLiveIns now skips this node.
- if (I.Kill.isValid())
- LR.addSegment(LiveInterval::Segment(Start, I.Kill, VNI));
- else {
- LR.addSegment(LiveInterval::Segment(Start, End, VNI));
+ if (I.Kill.isValid()) {
+ if (VNI)
+ LR.addSegment(LiveInterval::Segment(Start, I.Kill, VNI));
+ } else {
+ if (VNI)
+ LR.addSegment(LiveInterval::Segment(Start, End, VNI));
LOP = LiveOutPair(VNI, Node);
}
} else if (IDomValue.first) {
diff --git a/llvm/lib/CodeGen/LiveRangeCalc.h b/llvm/lib/CodeGen/LiveRangeCalc.h
index 9de48b72288..84cc1cb2be7 100644
--- a/llvm/lib/CodeGen/LiveRangeCalc.h
+++ b/llvm/lib/CodeGen/LiveRangeCalc.h
@@ -22,6 +22,7 @@
#ifndef LLVM_LIB_CODEGEN_LIVERANGECALC_H
#define LLVM_LIB_CODEGEN_LIVERANGECALC_H
+#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/BitVector.h"
#include "llvm/ADT/IndexedMap.h"
#include "llvm/CodeGen/LiveInterval.h"
@@ -53,6 +54,19 @@ class LiveRangeCalc {
/// when switching live ranges.
BitVector Seen;
+ /// Map LiveRange to sets of blocks (represented by bit vectors) that
+ /// in the live range are defined on entry and undefined on entry.
+ /// A block is defined on entry if there is a path from at least one of
+ /// the defs in the live range to the entry of the block, and conversely,
+ /// a block is undefined on entry, if there is no such path (i.e. no
+ /// definition reaches the entry of the block). A single LiveRangeCalc
+ /// object is used to track live-out information for multiple registers
+ /// in live range splitting (which is ok, since the live ranges of these
+ /// registers do not overlap), but the defined/undefined information must
+ /// be kept separate for each individual range.
+ /// By convention, EntryInfoMap[&LR] = { Defined, Undefined }.
+ std::map<LiveRange*,std::pair<BitVector,BitVector>> EntryInfoMap;
+
/// Map each basic block where a live range is live out to the live-out value
/// and its defining block.
///
@@ -101,18 +115,31 @@ class LiveRangeCalc {
/// used to add entries directly.
SmallVector<LiveInBlock, 16> LiveIn;
- /// Assuming that @p LR is live-in to @p UseMBB, find the set of defs that can
- /// reach it.
+ /// Check if the entry to block @p MBB can be reached by any of the defs
+ /// in @p LR. Return true if none of the defs reach the entry to @p MBB.
+ bool isDefOnEntry(LiveRange &LR, ArrayRef<SlotIndex> Undefs,
+ MachineBasicBlock &MBB, BitVector &DefOnEntry,
+ BitVector &UndefOnEntry);
+
+ /// Find the set of defs that can reach @p Kill. @p Kill must belong to
+ /// @p UseMBB.
///
- /// If only one def can reach @p UseMBB, all paths from the def to @p UseMBB
- /// are added to @p LR, and the function returns true.
+ /// If exactly one def can reach @p UseMBB, and the def dominates @p Kill,
+ /// all paths from the def to @p UseMBB are added to @p LR, and the function
+ /// returns true.
///
/// If multiple values can reach @p UseMBB, the blocks that need @p LR to be
/// live in are added to the LiveIn array, and the function returns false.
///
+ /// The array @p Undef provides the locations where the range @p LR becomes
+ /// undefined by <def,read-undef> operands on other subranges. If @p Undef
+ /// is non-empty and @p Kill is jointly dominated only by the entries of
+ /// @p Undef, the function returns false.
+ ///
/// PhysReg, when set, is used to verify live-in lists on basic blocks.
bool findReachingDefs(LiveRange &LR, MachineBasicBlock &UseMBB,
- SlotIndex Kill, unsigned PhysReg);
+ SlotIndex Kill, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs);
/// updateSSA - Compute the values that will be live in to all requested
/// blocks in LiveIn. Create PHI-def values as required to preserve SSA form.
@@ -127,9 +154,14 @@ class LiveRangeCalc {
/// Extend the live range of @p LR to reach all uses of Reg.
///
- /// All uses must be jointly dominated by existing liveness. PHI-defs are
- /// inserted as needed to preserve SSA form.
- void extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask LaneMask);
+ /// If @p LR is a main range, or if @p LI is null, then all uses must be
+ /// jointly dominated by the definitions from @p LR. If @p LR is a subrange
+ /// of the live interval @p LI, corresponding to lane mask @p LaneMask,
+ /// all uses must be jointly dominated by the definitions from @p LR
+ /// together with definitions of other lanes where @p LR becomes undefined
+ /// (via <def,read-undef> operands).
+ void extendToUses(LiveRange &LR, unsigned Reg, LaneBitmask LaneMask,
+ LiveInterval *LI = nullptr);
/// Reset Map and Seen fields.
void resetLiveOutMap();
@@ -169,7 +201,8 @@ public:
/// inserted as required to preserve SSA form.
///
/// PhysReg, when set, is used to verify live-in lists on basic blocks.
- void extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg = 0);
+ void extend(LiveRange &LR, SlotIndex Use, unsigned PhysReg,
+ ArrayRef<SlotIndex> Undefs);
/// createDeadDefs - Create a dead def in LI for every def operand of Reg.
/// Each instruction defining Reg gets a new VNInfo with a corresponding
diff --git a/llvm/lib/CodeGen/LiveRangeEdit.cpp b/llvm/lib/CodeGen/LiveRangeEdit.cpp
index b35c0adfaca..6165689054d 100644
--- a/llvm/lib/CodeGen/LiveRangeEdit.cpp
+++ b/llvm/lib/CodeGen/LiveRangeEdit.cpp
@@ -37,6 +37,13 @@ LiveInterval &LiveRangeEdit::createEmptyIntervalFrom(unsigned OldReg) {
VRM->setIsSplitFromReg(VReg, VRM->getOriginal(OldReg));
}
LiveInterval &LI = LIS.createEmptyInterval(VReg);
+ // Create empty subranges if the OldReg's interval has them. Do not create
+ // the main range here---it will be constructed later after the subranges
+ // have been finalized.
+ LiveInterval &OldLI = LIS.getInterval(OldReg);
+ VNInfo::Allocator &Alloc = LIS.getVNInfoAllocator();
+ for (LiveInterval::SubRange &S : OldLI.subranges())
+ LI.createSubRange(Alloc, S.LaneMask);
return LI;
}
@@ -66,6 +73,8 @@ void LiveRangeEdit::scanRemattable(AliasAnalysis *aa) {
unsigned Original = VRM->getOriginal(getReg());
LiveInterval &OrigLI = LIS.getInterval(Original);
VNInfo *OrigVNI = OrigLI.getVNInfoAt(VNI->def);
+ if (!OrigVNI)
+ continue;
MachineInstr *DefMI = LIS.getInstructionFromIndex(OrigVNI->def);
if (!DefMI)
continue;
@@ -335,6 +344,7 @@ void LiveRangeEdit::eliminateDeadDef(MachineInstr *MI, ToShrinkSet &ToShrink,
// allocations of the func are done.
if (isOrigDef && DeadRemats && TII.isTriviallyReMaterializable(*MI, AA)) {
LiveInterval &NewLI = createEmptyIntervalFrom(Dest);
+ NewLI.removeEmptySubRanges();
VNInfo *VNI = NewLI.getNextValue(Idx, LIS.getVNInfoAllocator());
NewLI.addSegment(LiveInterval::Segment(Idx, Idx.getDeadSlot(), VNI));
pop_back();
diff --git a/llvm/lib/CodeGen/MachineVerifier.cpp b/llvm/lib/CodeGen/MachineVerifier.cpp
index 2b38ed36436..d75b90f96a5 100644
--- a/llvm/lib/CodeGen/MachineVerifier.cpp
+++ b/llvm/lib/CodeGen/MachineVerifier.cpp
@@ -1813,18 +1813,25 @@ void MachineVerifier::verifyLiveRangeSegment(const LiveRange &LR,
bool hasRead = false;
bool hasSubRegDef = false;
bool hasDeadDef = false;
+ LaneBitmask RLM = MRI->getMaxLaneMaskForVReg(Reg);
for (ConstMIBundleOperands MOI(*MI); MOI.isValid(); ++MOI) {
if (!MOI->isReg() || MOI->getReg() != Reg)
continue;
- if (LaneMask != 0 &&
- (LaneMask & TRI->getSubRegIndexLaneMask(MOI->getSubReg())) == 0)
- continue;
+ unsigned Sub = MOI->getSubReg();
+ LaneBitmask SLM = Sub != 0 ? TRI->getSubRegIndexLaneMask(Sub) : RLM;
if (MOI->isDef()) {
- if (MOI->getSubReg() != 0)
+ if (Sub != 0) {
hasSubRegDef = true;
+ // An operand vreg0:sub0<def> reads vreg0:sub1..n. Invert the lane
+ // mask for subregister defs. Read-undef defs will be handled by
+ // readsReg below.
+ SLM = ~SLM & RLM;
+ }
if (MOI->isDead())
hasDeadDef = true;
}
+ if (LaneMask != 0 && !(LaneMask & SLM))
+ continue;
if (MOI->readsReg())
hasRead = true;
}
diff --git a/llvm/lib/CodeGen/RegAllocBase.cpp b/llvm/lib/CodeGen/RegAllocBase.cpp
index 93eeb9cba45..c204665dfd5 100644
--- a/llvm/lib/CodeGen/RegAllocBase.cpp
+++ b/llvm/lib/CodeGen/RegAllocBase.cpp
@@ -143,6 +143,7 @@ void RegAllocBase::allocatePhysRegs() {
continue;
}
DEBUG(dbgs() << "queuing new interval: " << *SplitVirtReg << "\n");
+ assert(!SplitVirtReg->empty() && "expecting non-empty interval");
assert(TargetRegisterInfo::isVirtualRegister(SplitVirtReg->reg) &&
"expect split value in virtual register");
enqueue(SplitVirtReg);
diff --git a/llvm/lib/CodeGen/RegisterCoalescer.cpp b/llvm/lib/CodeGen/RegisterCoalescer.cpp
index bf7aaccf1a9..9477fafe8e7 100644
--- a/llvm/lib/CodeGen/RegisterCoalescer.cpp
+++ b/llvm/lib/CodeGen/RegisterCoalescer.cpp
@@ -1211,6 +1211,17 @@ bool RegisterCoalescer::eliminateUndefCopy(MachineInstr *CopyMI) {
MO.setIsUndef(true);
DEBUG(dbgs() << "\tnew undef: " << UseIdx << '\t' << MI);
}
+
+ // A def of a subregister may be a use of the other subregisters, so
+ // deleting a def of a subregister may also remove uses. Since CopyMI
+ // is still part of the function (but about to be erased), mark all
+ // defs of DstReg in it as <undef>, so that shrinkToUses would
+ // ignore them.
+ for (MachineOperand &MO : CopyMI->operands())
+ if (MO.isReg() && MO.isDef() && MO.getReg() == DstReg)
+ MO.setIsUndef(true);
+ LIS->shrinkToUses(&DstLI);
+
return true;
}
@@ -1890,12 +1901,22 @@ public:
/// no useful information and can be removed.
void pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask);
+ /// Pruning values in subranges can lead to removing segments in these
+ /// subranges started by IMPLICIT_DEFs. The corresponding segments in
+ /// the main range also need to be removed. This function will mark
+ /// the corresponding values in the main range as pruned, so that
+ /// eraseInstrs can do the final cleanup.
+ /// The parameter @p LI must be the interval whose main range is the
+ /// live range LR.
+ void pruneMainSegments(LiveInterval &LI, bool &ShrinkMainRange);
+
/// Erase any machine instructions that have been coalesced away.
/// Add erased instructions to ErasedInstrs.
/// Add foreign virtual registers to ShrinkRegs if their live range ended at
/// the erased instrs.
void eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
- SmallVectorImpl<unsigned> &ShrinkRegs);
+ SmallVectorImpl<unsigned> &ShrinkRegs,
+ LiveInterval *LI = nullptr);
/// Remove liverange defs at places where implicit defs will be removed.
void removeImplicitDefs();
@@ -2416,7 +2437,8 @@ void JoinVals::pruneValues(JoinVals &Other,
for (MachineOperand &MO :
Indexes->getInstructionFromIndex(Def)->operands()) {
if (MO.isReg() && MO.isDef() && MO.getReg() == Reg) {
- MO.setIsUndef(EraseImpDef);
+ if (MO.getSubReg() != 0)
+ MO.setIsUndef(EraseImpDef);
MO.setIsDead(false);
}
}
@@ -2449,8 +2471,7 @@ void JoinVals::pruneValues(JoinVals &Other,
}
}
-void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask)
-{
+void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask) {
// Look for values being erased.
bool DidPrune = false;
for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
@@ -2487,6 +2508,30 @@ void JoinVals::pruneSubRegValues(LiveInterval &LI, LaneBitmask &ShrinkMask)
LI.removeEmptySubRanges();
}
+/// Check if any of the subranges of @p LI contain a definition at @p Def.
+static bool isDefInSubRange(LiveInterval &LI, SlotIndex Def) {
+ for (LiveInterval::SubRange &SR : LI.subranges()) {
+ if (VNInfo *VNI = SR.Query(Def).valueOutOrDead())
+ if (VNI->def == Def)
+ return true;
+ }
+ return false;
+}
+
+void JoinVals::pruneMainSegments(LiveInterval &LI, bool &ShrinkMainRange) {
+ assert(&static_cast<LiveRange&>(LI) == &LR);
+
+ for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
+ if (Vals[i].Resolution != CR_Keep)
+ continue;
+ VNInfo *VNI = LR.getValNumInfo(i);
+ if (VNI->isUnused() || VNI->isPHIDef() || isDefInSubRange(LI, VNI->def))
+ continue;
+ Vals[i].Pruned = true;
+ ShrinkMainRange = true;
+ }
+}
+
void JoinVals::removeImplicitDefs() {
for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
Val &V = Vals[i];
@@ -2500,7 +2545,8 @@ void JoinVals::removeImplicitDefs() {
}
void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
- SmallVectorImpl<unsigned> &ShrinkRegs) {
+ SmallVectorImpl<unsigned> &ShrinkRegs,
+ LiveInterval *LI) {
for (unsigned i = 0, e = LR.getNumValNums(); i != e; ++i) {
// Get the def location before markUnused() below invalidates it.
SlotIndex Def = LR.getValNumInfo(i)->def;
@@ -2512,12 +2558,64 @@ void JoinVals::eraseInstrs(SmallPtrSetImpl<MachineInstr*> &ErasedInstrs,
if (!Vals[i].ErasableImplicitDef || !Vals[i].Pruned)
break;
// Remove value number i from LR.
+ // For intervals with subranges, removing a segment from the main range
+ // may require extending the previous segment: for each definition of
+ // a subregister, there will be a corresponding def in the main range.
+ // That def may fall in the middle of a segment from another subrange.
+ // In such cases, removing this def from the main range must be
+ // complemented by extending the main range to account for the liveness
+ // of the other subrange.
VNInfo *VNI = LR.getValNumInfo(i);
+ SlotIndex Def = VNI->def;
+ // The new end point of the main range segment to be extended.
+ SlotIndex NewEnd;
+ if (LI != nullptr) {
+ LiveRange::iterator I = LR.FindSegmentContaining(Def);
+ assert(I != LR.end());
+ // Do not extend beyond the end of the segment being removed.
+ // The segment may have been pruned in preparation for joining
+ // live ranges.
+ NewEnd = I->end;
+ }
+
LR.removeValNo(VNI);
// Note that this VNInfo is reused and still referenced in NewVNInfo,
// make it appear like an unused value number.
VNI->markUnused();
- DEBUG(dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n');
+
+ if (LI != nullptr && LI->hasSubRanges()) {
+ assert(static_cast<LiveRange*>(LI) == &LR);
+ // Determine the end point based on the subrange information:
+ // minimum of (earliest def of next segment,
+ // latest end point of containing segment)
+ SlotIndex ED, LE;
+ for (LiveInterval::SubRange &SR : LI->subranges()) {
+ LiveRange::iterator I = SR.find(Def);
+ if (I == SR.end())
+ continue;
+ if (I->start > Def)
+ ED = ED.isValid() ? std::min(ED, I->start) : I->start;
+ else
+ LE = LE.isValid() ? std::max(LE, I->end) : I->end;
+ }
+ if (LE.isValid())
+ NewEnd = std::min(NewEnd, LE);
+ if (ED.isValid())
+ NewEnd = std::min(NewEnd, ED);
+
+ // We only want to do the extension if there was a subrange that
+ // was live across Def.
+ if (LE.isValid()) {
+ LiveRange::iterator S = LR.find(Def);
+ if (S != LR.begin())
+ std::prev(S)->end = NewEnd;
+ }
+ }
+ DEBUG({
+ dbgs() << "\t\tremoved " << i << '@' << Def << ": " << LR << '\n';
+ if (LI != nullptr)
+ dbgs() << "\t\t LHS = " << *LI << '\n';
+ });
LLVM_FALLTHROUGH;
}
@@ -2698,6 +2796,10 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
}
DEBUG(dbgs() << "\tJoined SubRanges " << LHS << "\n");
+ // Pruning implicit defs from subranges may result in the main range
+ // having stale segments.
+ LHSVals.pruneMainSegments(LHS, ShrinkMainRange);
+
LHSVals.pruneSubRegValues(LHS, ShrinkMask);
RHSVals.pruneSubRegValues(LHS, ShrinkMask);
}
@@ -2713,7 +2815,7 @@ bool RegisterCoalescer::joinVirtRegs(CoalescerPair &CP) {
// Erase COPY and IMPLICIT_DEF instructions. This may cause some external
// registers to require trimming.
SmallVector<unsigned, 8> ShrinkRegs;
- LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
+ LHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs, &LHS);
RHSVals.eraseInstrs(ErasedInstrs, ShrinkRegs);
while (!ShrinkRegs.empty())
shrinkToUses(&LIS->getInterval(ShrinkRegs.pop_back_val()));
diff --git a/llvm/lib/CodeGen/RenameIndependentSubregs.cpp b/llvm/lib/CodeGen/RenameIndependentSubregs.cpp
index bd5a60836b7..9c189f90199 100644
--- a/llvm/lib/CodeGen/RenameIndependentSubregs.cpp
+++ b/llvm/lib/CodeGen/RenameIndependentSubregs.cpp
@@ -353,6 +353,11 @@ void RenameIndependentSubregs::computeMainRangesFixFlags(
if (I == 0)
LI.clear();
LIS->constructMainRangeFromSubranges(LI);
+ // A def of a subregister may be a use of other register lanes. Replacing
+ // such a def with a def of a different register will eliminate the use,
+ // and may cause the recorded live range to be larger than the actual
+ // liveness in the program IR.
+ LIS->shrinkToUses(&LI);
}
}
diff --git a/llvm/lib/CodeGen/SplitKit.cpp b/llvm/lib/CodeGen/SplitKit.cpp
index 07be24b18dd..b6430282b30 100644
--- a/llvm/lib/CodeGen/SplitKit.cpp
+++ b/llvm/lib/CodeGen/SplitKit.cpp
@@ -381,9 +381,59 @@ LLVM_DUMP_METHOD void SplitEditor::dump() const {
}
#endif
+LiveInterval::SubRange &SplitEditor::getSubRangeForMask(LaneBitmask LM,
+ LiveInterval &LI) {
+ for (LiveInterval::SubRange &S : LI.subranges())
+ if (S.LaneMask == LM)
+ return S;
+ llvm_unreachable("SubRange for this mask not found");
+};
+
+void SplitEditor::addDeadDef(LiveInterval &LI, VNInfo *VNI, bool Original) {
+ if (!LI.hasSubRanges()) {
+ LI.createDeadDef(VNI);
+ return;
+ }
+
+ SlotIndex Def = VNI->def;
+ if (Original) {
+ // If we are transferring a def from the original interval, make sure
+ // to only update the subranges for which the original subranges had
+ // a def at this location.
+ for (LiveInterval::SubRange &S : LI.subranges()) {
+ auto &PS = getSubRangeForMask(S.LaneMask, Edit->getParent());
+ VNInfo *PV = PS.getVNInfoAt(Def);
+ if (PV != nullptr && PV->def == Def)
+ S.createDeadDef(Def, LIS.getVNInfoAllocator());
+ }
+ } else {
+ // This is a new def: either from rematerialization, or from an inserted
+ // copy. Since rematerialization can regenerate a definition of a sub-
+ // register, we need to check which subranges need to be updated.
+ const MachineInstr *DefMI = LIS.getInstructionFromIndex(Def);
+ assert(DefMI != nullptr);
+ LaneBitmask LM = 0;
+ for (const MachineOperand &DefOp : DefMI->defs()) {
+ unsigned R = DefOp.getReg();
+ if (R != LI.reg)
+ continue;
+ if (unsigned SR = DefOp.getSubReg())
+ LM |= TRI.getSubRegIndexLaneMask(SR);
+ else {
+ LM = MRI.getMaxLaneMaskForVReg(R);
+ break;
+ }
+ }
+ for (LiveInterval::SubRange &S : LI.subranges())
+ if (S.LaneMask & LM)
+ S.createDeadDef(Def, LIS.getVNInfoAllocator());
+ }
+}
+
VNInfo *SplitEditor::defValue(unsigned RegIdx,
const VNInfo *ParentVNI,
- SlotIndex Idx) {
+ SlotIndex Idx,
+ bool Original) {
assert(ParentVNI && "Mapping NULL value");
assert(Idx.isValid() && "Invalid SlotIndex");
assert(Edit->getParent().getVNInfoAt(Idx) == ParentVNI && "Bad Parent VNI");
@@ -392,28 +442,28 @@ VNInfo *SplitEditor::defValue(unsigned RegIdx,
// Create a new value.
VNInfo *VNI = LI->getNextValue(Idx, LIS.getVNInfoAllocator());
+ bool Force = LI->hasSubRanges();
+ ValueForcePair FP(Force ? nullptr : VNI, Force);
// Use insert for lookup, so we can add missing values with a second lookup.
std::pair<ValueMap::iterator, bool> InsP =
- Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id),
- ValueForcePair(VNI, false)));
+ Values.insert(std::make_pair(std::make_pair(RegIdx, ParentVNI->id), FP));
- // This was the first time (RegIdx, ParentVNI) was mapped.
- // Keep it as a simple def without any liveness.
- if (InsP.second)
+ // This was the first time (RegIdx, ParentVNI) was mapped, and it is not
+ // forced. Keep it as a simple def without any liveness.
+ if (!Force && InsP.second)
return VNI;
// If the previous value was a simple mapping, add liveness for it now.
if (VNInfo *OldVNI = InsP.first->second.getPointer()) {
- SlotIndex Def = OldVNI->def;
- LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), OldVNI));
- // No longer a simple mapping. Switch to a complex, non-forced mapping.
- InsP.first->second = ValueForcePair();
+ addDeadDef(*LI, OldVNI, Original);
+
+ // No longer a simple mapping. Switch to a complex mapping. If the
+ // interval has subranges, make it a forced mapping.
+ InsP.first->second = ValueForcePair(nullptr, Force);
}
// This is a complex mapping, add liveness for VNI
- SlotIndex Def = VNI->def;
- LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI));
-
+ addDeadDef(*LI, VNI, Original);
return VNI;
}
@@ -431,9 +481,8 @@ void SplitEditor::forceRecompute(unsigned RegIdx, const VNInfo *ParentVNI) {
// This was previously a single mapping. Make sure the old def is represented
// by a trivial live range.
- SlotIndex Def = VNI->def;
- LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
- LI->addSegment(LiveInterval::Segment(Def, Def.getDeadSlot(), VNI));
+ addDeadDef(LIS.getInterval(Edit->get(RegIdx)), VNI, false);
+
// Mark as complex mapped, forced.
VFP = ValueForcePair(nullptr, true);
}
@@ -455,13 +504,18 @@ VNInfo *SplitEditor::defFromParent(unsigned RegIdx,
unsigned Original = VRM.getOriginal(Edit->get(RegIdx));
LiveInterval &OrigLI = LIS.getInterval(Original);
VNInfo *OrigVNI = OrigLI.getVNInfoAt(UseIdx);
- LiveRangeEdit::Remat RM(ParentVNI);
- RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
- if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) {
- Def = Edit->rematerializeAt(MBB, I, LI->reg, RM, TRI, Late);
- ++NumRemats;
- } else {
+ bool DidRemat = false;
+ if (OrigVNI) {
+ LiveRangeEdit::Remat RM(ParentVNI);
+ RM.OrigMI = LIS.getInstructionFromIndex(OrigVNI->def);
+ if (Edit->canRematerializeAt(RM, OrigVNI, UseIdx, true)) {
+ Def = Edit->rematerializeAt(MBB, I, LI->reg, RM, TRI, Late);
+ ++NumRemats;
+ DidRemat = true;
+ }
+ }
+ if (!DidRemat) {
// Can't remat, just insert a copy from parent.
CopyMI = BuildMI(MBB, I, DebugLoc(), TII.get(TargetOpcode::COPY), LI->reg)
.addReg(Edit->getReg());
@@ -472,7 +526,7 @@ VNInfo *SplitEditor::defFromParent(unsigned RegIdx,
}
// Define the value in Reg.
- return defValue(RegIdx, ParentVNI, Def);
+ return defValue(RegIdx, ParentVNI, Def, false);
}
/// Create a new virtual register and live interval.
@@ -944,14 +998,15 @@ bool SplitEditor::transferValues() {
}
// The interval [Start;End) is continuously mapped to RegIdx, ParentVNI.
- DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx);
- LiveRange &LR = LIS.getInterval(Edit->get(RegIdx));
+ DEBUG(dbgs() << " [" << Start << ';' << End << ")=" << RegIdx
+ << '(' << PrintReg(Edit->get(RegIdx)) << ')');
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
// Check for a simply defined value that can be blitted directly.
ValueForcePair VFP = Values.lookup(std::make_pair(RegIdx, ParentVNI->id));
if (VNInfo *VNI = VFP.getPointer()) {
DEBUG(dbgs() << ':' << VNI->id);
- LR.addSegment(LiveInterval::Segment(Start, End, VNI));
+ LI.addSegment(LiveInterval::Segment(Start, End, VNI));
Start = End;
continue;
}
@@ -975,7 +1030,7 @@ bool SplitEditor::transferValues() {
// The first block may be live-in, or it may have its own def.
if (Start != BlockStart) {
- VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End));
+ VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
assert(VNI && "Missing def for complex mapped value");
DEBUG(dbgs() << ':' << VNI->id << "*BB#" << MBB->getNumber());
// MBB has its own def. Is it also live-out?
@@ -995,7 +1050,7 @@ bool SplitEditor::transferValues() {
if (BlockStart == ParentVNI->def) {
// This block has the def of a parent PHI, so it isn't live-in.
assert(ParentVNI->isPHIDef() && "Non-phi defined at block start?");
- VNInfo *VNI = LR.extendInBlock(BlockStart, std::min(BlockEnd, End));
+ VNInfo *VNI = LI.extendInBlock(BlockStart, std::min(BlockEnd, End));
assert(VNI && "Missing def for complex mapped parent PHI");
if (End >= BlockEnd)
LRC.setLiveOutValue(&*MBB, VNI); // Live-out as well.
@@ -1003,10 +1058,10 @@ bool SplitEditor::transferValues() {
// This block needs a live-in value. The last block covered may not
// be live-out.
if (End < BlockEnd)
- LRC.addLiveInBlock(LR, MDT[&*MBB], End);
+ LRC.addLiveInBlock(LI, MDT[&*MBB], End);
else {
// Live-through, and we don't know the value.
- LRC.addLiveInBlock(LR, MDT[&*MBB]);
+ LRC.addLiveInBlock(LI, MDT[&*MBB]);
LRC.setLiveOutValue(&*MBB, nullptr);
}
}
@@ -1025,42 +1080,84 @@ bool SplitEditor::transferValues() {
return Skipped;
}
+static bool removeDeadSegment(SlotIndex Def, LiveRange &LR) {
+ const LiveRange::Segment *Seg = LR.getSegmentContaining(Def);
+ if (Seg == nullptr)
+ return true;
+ if (Seg->end != Def.getDeadSlot())
+ return false;
+ // This is a dead PHI. Remove it.
+ LR.removeSegment(*Seg, true);
+ return true;
+}
+
+void SplitEditor::extendPHIRange(MachineBasicBlock &B, LiveRangeCalc &LRC,
+ LiveRange &LR, ArrayRef<SlotIndex> Undefs) {
+ for (MachineBasicBlock *P : B.predecessors()) {
+ SlotIndex End = LIS.getMBBEndIdx(P);
+ SlotIndex LastUse = End.getPrevSlot();
+ // The predecessor may not have a live-out value. That is OK, like an
+ // undef PHI operand.
+ if (Edit->getParent().liveAt(LastUse))
+ LRC.extend(LR, End, /*PhysReg=*/0, Undefs);
+ }
+}
+
void SplitEditor::extendPHIKillRanges() {
// Extend live ranges to be live-out for successor PHI values.
- for (const VNInfo *PHIVNI : Edit->getParent().valnos) {
- if (PHIVNI->isUnused() || !PHIVNI->isPHIDef())
- continue;
- unsigned RegIdx = RegAssign.lookup(PHIVNI->def);
- LiveRange &LR = LIS.getInterval(Edit->get(RegIdx));
-
- // Check whether PHI is dead.
- const LiveRange::Segment *Segment = LR.getSegmentContaining(PHIVNI->def);
- assert(Segment != nullptr && "Missing segment for VNI");
- if (Segment->end == PHIVNI->def.getDeadSlot()) {
- // This is a dead PHI. Remove it.
- LR.removeSegment(*Segment, true);
+
+ // Visit each PHI def slot in the parent live interval. If the def is dead,
+ // remove it. Otherwise, extend the live interval to reach the end indexes
+ // of all predecessor blocks.
+
+ LiveInterval &ParentLI = Edit->getParent();
+ for (const VNInfo *V : ParentLI.valnos) {
+ if (V->isUnused() || !V->isPHIDef())
continue;
- }
+ unsigned RegIdx = RegAssign.lookup(V->def);
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
LiveRangeCalc &LRC = getLRCalc(RegIdx);
- MachineBasicBlock *MBB = LIS.getMBBFromIndex(PHIVNI->def);
- for (MachineBasicBlock::pred_iterator PI = MBB->pred_begin(),
- PE = MBB->pred_end(); PI != PE; ++PI) {
- SlotIndex End = LIS.getMBBEndIdx(*PI);
- SlotIndex LastUse = End.getPrevSlot();
- // The predecessor may not have a live-out value. That is OK, like an
- // undef PHI operand.
- if (Edit->getParent().liveAt(LastUse)) {
- assert(RegAssign.lookup(LastUse) == RegIdx &&
- "Different register assignment in phi predecessor");
- LRC.extend(LR, End);
- }
+ MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
+ if (!removeDeadSegment(V->def, LI))
+ extendPHIRange(B, LRC, LI, /*Undefs=*/{});
+ }
+
+ SmallVector<SlotIndex, 4> Undefs;
+ LiveRangeCalc SubLRC;
+
+ for (LiveInterval::SubRange &PS : ParentLI.subranges()) {
+ for (const VNInfo *V : PS.valnos) {
+ if (V->isUnused() || !V->isPHIDef())
+ continue;
+ unsigned RegIdx = RegAssign.lookup(V->def);
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
+ LiveInterval::SubRange &S = getSubRangeForMask(PS.LaneMask, LI);
+ if (removeDeadSegment(V->def, S))
+ continue;
+
+ MachineBasicBlock &B = *LIS.getMBBFromIndex(V->def);
+ SubLRC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
+ &LIS.getVNInfoAllocator());
+ Undefs.clear();
+ LI.computeSubRangeUndefs(Undefs, PS.LaneMask, MRI, *LIS.getSlotIndexes());
+ extendPHIRange(B, SubLRC, S, Undefs);
}
}
}
/// rewriteAssigned - Rewrite all uses of Edit->getReg().
void SplitEditor::rewriteAssigned(bool ExtendRanges) {
+ struct ExtPoint {
+ ExtPoint(const MachineOperand &O, unsigned R, SlotIndex N)
+ : MO(O), RegIdx(R), Next(N) {}
+ MachineOperand MO;
+ unsigned RegIdx;
+ SlotIndex Next;
+ };
+
+ SmallVector<ExtPoint,4> ExtPoints;
+
for (MachineRegisterInfo::reg_iterator RI = MRI.reg_begin(Edit->getReg()),
RE = MRI.reg_end(); RI != RE;) {
MachineOperand &MO = *RI;
@@ -1082,8 +1179,8 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) {
// Rewrite to the mapped register at Idx.
unsigned RegIdx = RegAssign.lookup(Idx);
- LiveInterval *LI = &LIS.getInterval(Edit->get(RegIdx));
- MO.setReg(LI->reg);
+ LiveInterval &LI = LIS.getInterval(Edit->get(RegIdx));
+ MO.setReg(LI.reg);
DEBUG(dbgs() << " rewr BB#" << MI->getParent()->getNumber() << '\t'
<< Idx << ':' << RegIdx << '\t' << *MI);
@@ -1095,7 +1192,7 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) {
if (MO.isDef()) {
if (!MO.getSubReg() && !MO.isEarlyClobber())
continue;
- // We may wan't to extend a live range for a partial redef, or for a use
+ // We may want to extend a live range for a partial redef, or for a use
// tied to an early clobber.
Idx = Idx.getPrevSlot();
if (!Edit->getParent().liveAt(Idx))
@@ -1103,7 +1200,56 @@ void SplitEditor::rewriteAssigned(bool ExtendRanges) {
} else
Idx = Idx.getRegSlot(true);
- getLRCalc(RegIdx).extend(*LI, Idx.getNextSlot());
+ SlotIndex Next = Idx.getNextSlot();
+ if (LI.hasSubRanges()) {
+ // We have to delay extending subranges until we have seen all operands
+ // defining the register. This is because a <def,read-undef> operand
+ // will create an "undef" point, and we cannot extend any subranges
+ // until all of them have been accounted for.
+ ExtPoints.push_back(ExtPoint(MO, RegIdx, Next));
+ } else {
+ LiveRangeCalc &LRC = getLRCalc(RegIdx);
+ LRC.extend(LI, Next, 0, ArrayRef<SlotIndex>());
+ }
+ }
+
+ for (ExtPoint &EP : ExtPoints) {
+ LiveInterval &LI = LIS.getInterval(Edit->get(EP.RegIdx));
+ assert(LI.hasSubRanges());
+
+ LiveRangeCalc SubLRC;
+ unsigned Reg = EP.MO.getReg(), Sub = EP.MO.getSubReg();
+ LaneBitmask LM = Sub != 0 ? TRI.getSubRegIndexLaneMask(Sub)
+ : MRI.getMaxLaneMaskForVReg(Reg);
+ // If this is a non-read-undef definition of a sub-register, extend
+ // subranges for everything except that sub-register.
+ if (Sub != 0 && EP.MO.isDef())
+ LM = MRI.getMaxLaneMaskForVReg(Reg) & ~LM;
+ for (LiveInterval::SubRange &S : LI.subranges()) {
+ if (!(S.LaneMask & LM))
+ continue;
+ // The problem here can be that the new register may have been created
+ // for a partially defined original register. For example:
+ // %vreg827:subreg_hireg<def,read-undef> = ...
+ // ...
+ // %vreg828<def> = COPY %vreg827
+ if (S.empty())
+ continue;
+ SubLRC.reset(&VRM.getMachineFunction(), LIS.getSlotIndexes(), &MDT,
+ &LIS.getVNInfoAllocator());
+ SmallVector<SlotIndex, 4> Undefs;
+ LI.computeSubRangeUndefs(Undefs, S.LaneMask, MRI, *LIS.getSlotIndexes());
+ SubLRC.extend(S, EP.Next, 0, Undefs);
+ }
+ }
+
+ for (unsigned R : *Edit) {
+ LiveInterval &LI = LIS.getInterval(R);
+ if (!LI.hasSubRanges())
+ continue;
+ LI.clear();
+ LI.removeEmptySubRanges();
+ LIS.constructMainRangeFromSubranges(LI);
}
}
@@ -1146,7 +1292,7 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
if (ParentVNI->isUnused())
continue;
unsigned RegIdx = RegAssign.lookup(ParentVNI->def);
- defValue(RegIdx, ParentVNI, ParentVNI->def);
+ defValue(RegIdx, ParentVNI, ParentVNI->def, true);
// Force rematted values to be recomputed everywhere.
// The new live ranges may be truncated.
@@ -1182,8 +1328,9 @@ void SplitEditor::finish(SmallVectorImpl<unsigned> *LRMap) {
deleteRematVictims();
// Get rid of unused values and set phi-kill flags.
- for (LiveRangeEdit::iterator I = Edit->begin(), E = Edit->end(); I != E; ++I) {
- LiveInterval &LI = LIS.getInterval(*I);
+ for (unsigned Reg : *Edit) {
+ LiveInterval &LI = LIS.getInterval(Reg);
+ LI.removeEmptySubRanges();
LI.RenumberValues();
}
diff --git a/llvm/lib/CodeGen/SplitKit.h b/llvm/lib/CodeGen/SplitKit.h
index a9684942885..94a1a28b9ff 100644
--- a/llvm/lib/CodeGen/SplitKit.h
+++ b/llvm/lib/CodeGen/SplitKit.h
@@ -325,12 +325,30 @@ private:
return LRCalc[SpillMode != SM_Partition && RegIdx != 0];
}
+ /// Find a subrange corresponding to the lane mask @p LM in the live
+ /// interval @p LI. The interval @p LI is assumed to contain such a subrange.
+ /// This function is used to find corresponding subranges between the
+ /// original interval and the new intervals.
+ LiveInterval::SubRange &getSubRangeForMask(LaneBitmask LM, LiveInterval &LI);
+
+ /// Add a segment to the interval LI for the value number VNI. If LI has
+ /// subranges, corresponding segments will be added to them as well, but
+ /// with newly created value numbers. If Original is true, dead def will
+ /// only be added a subrange of LI if the corresponding subrange of the
+ /// original interval has a def at this index. Otherwise, all subranges
+ /// of LI will be updated.
+ void addDeadDef(LiveInterval &LI, VNInfo *VNI, bool Original);
+
/// defValue - define a value in RegIdx from ParentVNI at Idx.
/// Idx does not have to be ParentVNI->def, but it must be contained within
/// ParentVNI's live range in ParentLI. The new value is added to the value
- /// map.
+ /// map. The value being defined may either come from rematerialization
+ /// (or an inserted copy), or it may be coming from the original interval.
+ /// The parameter Original should be true in the latter case, otherwise
+ /// it should be false.
/// Return the new LI value.
- VNInfo *defValue(unsigned RegIdx, const VNInfo *ParentVNI, SlotIndex Idx);
+ VNInfo *defValue(unsigned RegIdx, const VNInfo *ParentVNI, SlotIndex Idx,
+ bool Original);
/// forceRecompute - Force the live range of ParentVNI in RegIdx to be
/// recomputed by LiveRangeCalc::extend regardless of the number of defs.
@@ -368,6 +386,11 @@ private:
/// Return true if any ranges were skipped.
bool transferValues();
+ /// Live range @p LR has a live PHI def at the beginning of block @p B.
+ /// Extend the range @p LR of all predecessor values that reach this def.
+ void extendPHIRange(MachineBasicBlock &B, LiveRangeCalc &LRC,
+ LiveRange &LR, ArrayRef<SlotIndex>);
+
/// extendPHIKillRanges - Extend the ranges of all values killed by original
/// parent PHIDefs.
void extendPHIKillRanges();
diff --git a/llvm/lib/CodeGen/VirtRegMap.cpp b/llvm/lib/CodeGen/VirtRegMap.cpp
index ddbb72bf9d8..fcba8add32c 100644
--- a/llvm/lib/CodeGen/VirtRegMap.cpp
+++ b/llvm/lib/CodeGen/VirtRegMap.cpp
@@ -338,6 +338,7 @@ bool VirtRegRewriter::readsUndefSubreg(const MachineOperand &MO) const {
assert(LI.liveAt(BaseIndex) &&
"Reads of completely dead register should be marked undef already");
unsigned SubRegIdx = MO.getSubReg();
+ assert(SubRegIdx != 0 && LI.hasSubRanges());
LaneBitmask UseMask = TRI->getSubRegIndexLaneMask(SubRegIdx);
// See if any of the relevant subregister liveranges is defined at this point.
for (const LiveInterval::SubRange &SR : LI.subranges()) {
OpenPOWER on IntegriCloud