diff options
Diffstat (limited to 'llvm/lib/CodeGen')
5 files changed, 209 insertions, 45 deletions
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index c2404f963ce..78a6ea6d4fd 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -385,6 +385,7 @@ namespace { SDValue replaceStoreOfFPConstant(StoreSDNode *ST); SDValue visitSTORE(SDNode *N); + SDValue visitLIFETIME_END(SDNode *N); SDValue visitINSERT_VECTOR_ELT(SDNode *N); SDValue visitEXTRACT_VECTOR_ELT(SDNode *N); SDValue visitBUILD_VECTOR(SDNode *N); @@ -1589,6 +1590,7 @@ SDValue DAGCombiner::visit(SDNode *N) { case ISD::MLOAD: return visitMLOAD(N); case ISD::MSCATTER: return visitMSCATTER(N); case ISD::MSTORE: return visitMSTORE(N); + case ISD::LIFETIME_END: return visitLIFETIME_END(N); case ISD::FP_TO_FP16: return visitFP_TO_FP16(N); case ISD::FP16_TO_FP: return visitFP16_TO_FP(N); } @@ -15484,6 +15486,66 @@ SDValue DAGCombiner::visitSTORE(SDNode *N) { return ReduceLoadOpStoreWidth(N); } +SDValue DAGCombiner::visitLIFETIME_END(SDNode *N) { + const auto *LifetimeEnd = cast<LifetimeSDNode>(N); + if (!LifetimeEnd->hasOffset()) + return SDValue(); + + const BaseIndexOffset LifetimeEndBase(N->getOperand(1), SDValue(), + LifetimeEnd->getOffset(), false); + + // We walk up the chains to find stores. + SmallVector<SDValue, 8> Chains = {N->getOperand(0)}; + while (!Chains.empty()) { + SDValue Chain = Chains.back(); + Chains.pop_back(); + switch (Chain.getOpcode()) { + case ISD::TokenFactor: + for (unsigned Nops = Chain.getNumOperands(); Nops;) + Chains.push_back(Chain.getOperand(--Nops)); + break; + case ISD::LIFETIME_START: + case ISD::LIFETIME_END: { + // We can forward past any lifetime start/end that can be proven not to + // alias the node. + const auto *LifetimeStart = cast<LifetimeSDNode>(Chain); + if (!LifetimeStart->hasOffset()) + break; // Be conservative if we don't know the extents of the object. + + const BaseIndexOffset LifetimeStartBase( + LifetimeStart->getOperand(1), SDValue(), LifetimeStart->getOffset(), + false); + bool IsAlias; + if (BaseIndexOffset::computeAliasing( + LifetimeEndBase, LifetimeEnd->getSize(), LifetimeStartBase, + LifetimeStart->getSize(), DAG, IsAlias) && + !IsAlias) { + Chains.push_back(Chain.getOperand(0)); + } + break; + } + + case ISD::STORE: { + StoreSDNode *ST = dyn_cast<StoreSDNode>(Chain); + if (ST->isVolatile() || !ST->hasOneUse() || ST->isIndexed()) + continue; + const BaseIndexOffset StoreBase = BaseIndexOffset::match(ST, DAG); + // If we store purely within object bounds just before its lifetime ends, + // we can remove the store. + if (LifetimeEndBase.contains(LifetimeEnd->getSize(), StoreBase, + ST->getMemoryVT().getStoreSize(), DAG)) { + LLVM_DEBUG(dbgs() << "\nRemoving store:"; StoreBase.dump(); + dbgs() << "\nwithin LIFETIME_END of : "; + LifetimeEndBase.dump(); dbgs() << "\n"); + CombineTo(ST, ST->getChain()); + return SDValue(N, 0); + } + } + } + } + return SDValue(); +} + /// For the instruction sequence of store below, F and I values /// are bundled together as an i64 value before being stored into memory. /// Sometimes it is more efficent to generate separate stores for F and I, @@ -19235,41 +19297,11 @@ bool DAGCombiner::isAlias(LSBaseSDNode *Op0, LSBaseSDNode *Op1) const { unsigned NumBytes1 = Op1->getMemoryVT().getStoreSize(); // Check for BaseIndexOffset matching. - BaseIndexOffset BasePtr0 = BaseIndexOffset::match(Op0, DAG); - BaseIndexOffset BasePtr1 = BaseIndexOffset::match(Op1, DAG); - int64_t PtrDiff; - if (BasePtr0.getBase().getNode() && BasePtr1.getBase().getNode()) { - if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) - return !((NumBytes0 <= PtrDiff) || (PtrDiff + NumBytes1 <= 0)); - - // If both BasePtr0 and BasePtr1 are FrameIndexes, we will not be - // able to calculate their relative offset if at least one arises - // from an alloca. However, these allocas cannot overlap and we - // can infer there is no alias. - if (auto *A = dyn_cast<FrameIndexSDNode>(BasePtr0.getBase())) - if (auto *B = dyn_cast<FrameIndexSDNode>(BasePtr1.getBase())) { - MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); - // If the base are the same frame index but the we couldn't find a - // constant offset, (indices are different) be conservative. - if (A != B && (!MFI.isFixedObjectIndex(A->getIndex()) || - !MFI.isFixedObjectIndex(B->getIndex()))) - return false; - } - - bool IsFI0 = isa<FrameIndexSDNode>(BasePtr0.getBase()); - bool IsFI1 = isa<FrameIndexSDNode>(BasePtr1.getBase()); - bool IsGV0 = isa<GlobalAddressSDNode>(BasePtr0.getBase()); - bool IsGV1 = isa<GlobalAddressSDNode>(BasePtr1.getBase()); - bool IsCV0 = isa<ConstantPoolSDNode>(BasePtr0.getBase()); - bool IsCV1 = isa<ConstantPoolSDNode>(BasePtr1.getBase()); - - // If of mismatched base types or checkable indices we can check - // they do not alias. - if ((BasePtr0.getIndex() == BasePtr1.getIndex() || (IsFI0 != IsFI1) || - (IsGV0 != IsGV1) || (IsCV0 != IsCV1)) && - (IsFI0 || IsGV0 || IsCV0) && (IsFI1 || IsGV1 || IsCV1)) - return false; - } + bool IsAlias; + if (BaseIndexOffset::computeAliasing( + BaseIndexOffset::match(Op0, DAG), NumBytes0, + BaseIndexOffset::match(Op1, DAG), NumBytes1, DAG, IsAlias)) + return IsAlias; // If we know required SrcValue1 and SrcValue2 have relatively large // alignment compared to the size and offset of the access, we may be able @@ -19328,6 +19360,8 @@ void DAGCombiner::GatherAllAliases(LSBaseSDNode *N, SDValue OriginalChain, // Get alias information for node. bool IsLoad = isa<LoadSDNode>(N) && !N->isVolatile(); + const BaseIndexOffset LSBasePtr = BaseIndexOffset::match(N, DAG); + const unsigned LSNumBytes = N->getMemoryVT().getStoreSize(); // Starting off. Chains.push_back(OriginalChain); @@ -19398,6 +19432,27 @@ void DAGCombiner::GatherAllAliases(LSBaseSDNode *N, SDValue OriginalChain, ++Depth; break; + case ISD::LIFETIME_START: + case ISD::LIFETIME_END: { + // We can forward past any lifetime start/end that can be proven not to + // alias the memory access. + const auto *Lifetime = cast<LifetimeSDNode>(Chain); + if (!Lifetime->hasOffset()) + break; // Be conservative if we don't know the extents of the object. + + const BaseIndexOffset LifetimePtr(Lifetime->getOperand(1), SDValue(), + Lifetime->getOffset(), false); + bool IsAlias; + if (BaseIndexOffset::computeAliasing(LifetimePtr, Lifetime->getSize(), + LSBasePtr, LSNumBytes, DAG, + IsAlias) && + !IsAlias) { + Chains.push_back(Chain.getOperand(0)); + ++Depth; + } + break; + } + default: // For all other instructions we will just have to take what we can get. Aliases.push_back(Chain); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index d7d7b8b7191..05f733b5676 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -6502,6 +6502,36 @@ SDValue SelectionDAG::getMemIntrinsicNode(unsigned Opcode, const SDLoc &dl, return SDValue(N, 0); } +SDValue SelectionDAG::getLifetimeNode(bool IsStart, const SDLoc &dl, + SDValue Chain, int FrameIndex, + int64_t Size, int64_t Offset) { + const unsigned Opcode = IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END; + const auto VTs = getVTList(MVT::Other); + SDValue Ops[2] = { + Chain, + getFrameIndex(FrameIndex, + getTargetLoweringInfo().getFrameIndexTy(getDataLayout()), + true)}; + + FoldingSetNodeID ID; + AddNodeIDNode(ID, Opcode, VTs, Ops); + ID.AddInteger(FrameIndex); + ID.AddInteger(Size); + ID.AddInteger(Offset); + void *IP = nullptr; + if (SDNode *E = FindNodeOrInsertPos(ID, dl, IP)) + return SDValue(E, 0); + + LifetimeSDNode *N = newSDNode<LifetimeSDNode>( + Opcode, dl.getIROrder(), dl.getDebugLoc(), VTs, Size, Offset); + createOperands(N, Ops); + CSEMap.InsertNode(N, IP); + InsertNode(N); + SDValue V(N, 0); + NewSDValueDbgMsg(V, "Creating new node: ", this); + return V; +} + /// InferPointerInfo - If the specified ptr/offset is a frame index, infer a /// MachinePointerInfo record from it. This is particularly useful because the /// code generator has many cases where it doesn't bother passing in a diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp index 5666c1293e9..c21ecfca13a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGAddressAnalysis.cpp @@ -79,6 +79,81 @@ bool BaseIndexOffset::equalBaseIndex(const BaseIndexOffset &Other, return false; } +bool BaseIndexOffset::computeAliasing(const BaseIndexOffset &BasePtr0, + const int64_t NumBytes0, + const BaseIndexOffset &BasePtr1, + const int64_t NumBytes1, + const SelectionDAG &DAG, bool &IsAlias) { + if (!(BasePtr0.getBase().getNode() && BasePtr1.getBase().getNode())) + return false; + int64_t PtrDiff; + if (BasePtr0.equalBaseIndex(BasePtr1, DAG, PtrDiff)) { + // BasePtr1 is PtrDiff away from BasePtr0. They alias if none of the + // following situations arise: + IsAlias = !( + // [----BasePtr0----] + // [---BasePtr1--] + // ========PtrDiff========> + (NumBytes0 <= PtrDiff) || + // [----BasePtr0----] + // [---BasePtr1--] + // =====(-PtrDiff)====> + (PtrDiff + NumBytes1 <= 0)); // i.e. NumBytes1 < -PtrDiff. + return true; + } + // If both BasePtr0 and BasePtr1 are FrameIndexes, we will not be + // able to calculate their relative offset if at least one arises + // from an alloca. However, these allocas cannot overlap and we + // can infer there is no alias. + if (auto *A = dyn_cast<FrameIndexSDNode>(BasePtr0.getBase())) + if (auto *B = dyn_cast<FrameIndexSDNode>(BasePtr1.getBase())) { + MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); + // If the base are the same frame index but the we couldn't find a + // constant offset, (indices are different) be conservative. + if (A != B && (!MFI.isFixedObjectIndex(A->getIndex()) || + !MFI.isFixedObjectIndex(B->getIndex()))) { + IsAlias = false; + return true; + } + } + + bool IsFI0 = isa<FrameIndexSDNode>(BasePtr0.getBase()); + bool IsFI1 = isa<FrameIndexSDNode>(BasePtr1.getBase()); + bool IsGV0 = isa<GlobalAddressSDNode>(BasePtr0.getBase()); + bool IsGV1 = isa<GlobalAddressSDNode>(BasePtr1.getBase()); + bool IsCV0 = isa<ConstantPoolSDNode>(BasePtr0.getBase()); + bool IsCV1 = isa<ConstantPoolSDNode>(BasePtr1.getBase()); + + // If of mismatched base types or checkable indices we can check + // they do not alias. + if ((BasePtr0.getIndex() == BasePtr1.getIndex() || (IsFI0 != IsFI1) || + (IsGV0 != IsGV1) || (IsCV0 != IsCV1)) && + (IsFI0 || IsGV0 || IsCV0) && (IsFI1 || IsGV1 || IsCV1)) { + IsAlias = false; + return true; + } + return false; // Cannot determine whether the pointers alias. +} + +bool BaseIndexOffset::contains(int64_t Size, const BaseIndexOffset &Other, + int64_t OtherSize, + const SelectionDAG &DAG) const { + int64_t Offset; + if (!equalBaseIndex(Other, DAG, Offset)) + return false; + if (Offset >= 0) { + // Other is after *this: + // [-------*this---------] + // [---Other--] + // ==Offset==> + return Offset + OtherSize <= Size; + } + // Other starts strictly before *this, it cannot be fully contained. + // [-------*this---------] + // [--Other--] + return false; +} + /// Parses tree in Ptr for base, index, offset addresses. BaseIndexOffset BaseIndexOffset::match(const LSBaseSDNode *N, const SelectionDAG &DAG) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 75f7cc08aa9..1b75dc7d2e5 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -6373,8 +6373,11 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (TM.getOptLevel() == CodeGenOpt::None) return nullptr; + const int64_t ObjectSize = + cast<ConstantInt>(I.getArgOperand(0))->getSExtValue(); + Value *const ObjectPtr = I.getArgOperand(1); SmallVector<Value *, 4> Allocas; - GetUnderlyingObjects(I.getArgOperand(1), Allocas, *DL); + GetUnderlyingObjects(ObjectPtr, Allocas, *DL); for (SmallVectorImpl<Value*>::iterator Object = Allocas.begin(), E = Allocas.end(); Object != E; ++Object) { @@ -6390,15 +6393,13 @@ SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, unsigned Intrinsic) { if (SI == FuncInfo.StaticAllocaMap.end()) return nullptr; - int FI = SI->second; - - SDValue Ops[2]; - Ops[0] = getRoot(); - Ops[1] = - DAG.getFrameIndex(FI, TLI.getFrameIndexTy(DAG.getDataLayout()), true); - unsigned Opcode = (IsStart ? ISD::LIFETIME_START : ISD::LIFETIME_END); - - Res = DAG.getNode(Opcode, sdl, MVT::Other, Ops); + const int FrameIndex = SI->second; + int64_t Offset; + if (GetPointerBaseWithConstantOffset( + ObjectPtr, Offset, DAG.getDataLayout()) != LifetimeObject) + Offset = -1; // Cannot determine offset from alloca to lifetime object. + Res = DAG.getLifetimeNode(IsStart, sdl, getRoot(), FrameIndex, ObjectSize, + Offset); DAG.setRoot(Res); } return nullptr; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp index 490b2f9957e..365286309ea 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGDumper.cpp @@ -709,6 +709,9 @@ void SDNode::print_details(raw_ostream &OS, const SelectionDAG *G) const { << " -> " << ASC->getDestAddressSpace() << ']'; + } else if (const LifetimeSDNode *LN = dyn_cast<LifetimeSDNode>(this)) { + if (LN->hasOffset()) + OS << "<" << LN->getOffset() << " to " << LN->getOffset() + LN->getSize() << ">"; } if (VerboseDAGDumping) { |
