diff options
Diffstat (limited to 'llvm/lib')
22 files changed, 101 insertions, 62 deletions
diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.cpp b/llvm/lib/CodeGen/MIRParser/MILexer.cpp index 0850ae7badd..d94d6a5e6be 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.cpp +++ b/llvm/lib/CodeGen/MIRParser/MILexer.cpp @@ -222,6 +222,7 @@ static MIToken::TokenKind getIdentifierKind(StringRef Identifier) { .Case("target-flags", MIToken::kw_target_flags) .Case("volatile", MIToken::kw_volatile) .Case("non-temporal", MIToken::kw_non_temporal) + .Case("dereferenceable", MIToken::kw_dereferenceable) .Case("invariant", MIToken::kw_invariant) .Case("align", MIToken::kw_align) .Case("stack", MIToken::kw_stack) diff --git a/llvm/lib/CodeGen/MIRParser/MILexer.h b/llvm/lib/CodeGen/MIRParser/MILexer.h index c0d9561b2e8..6911700b4c8 100644 --- a/llvm/lib/CodeGen/MIRParser/MILexer.h +++ b/llvm/lib/CodeGen/MIRParser/MILexer.h @@ -54,6 +54,7 @@ struct MIToken { kw_implicit_define, kw_def, kw_dead, + kw_dereferenceable, kw_killed, kw_undef, kw_internal, @@ -166,7 +167,7 @@ public: bool isMemoryOperandFlag() const { return Kind == kw_volatile || Kind == kw_non_temporal || - Kind == kw_invariant; + Kind == kw_dereferenceable || Kind == kw_invariant; } bool is(TokenKind K) const { return Kind == K; } diff --git a/llvm/lib/CodeGen/MIRParser/MIParser.cpp b/llvm/lib/CodeGen/MIRParser/MIParser.cpp index 4dc964e1d80..696ba48e97c 100644 --- a/llvm/lib/CodeGen/MIRParser/MIParser.cpp +++ b/llvm/lib/CodeGen/MIRParser/MIParser.cpp @@ -1766,6 +1766,9 @@ bool MIParser::parseMemoryOperandFlag(MachineMemOperand::Flags &Flags) { case MIToken::kw_non_temporal: Flags |= MachineMemOperand::MONonTemporal; break; + case MIToken::kw_dereferenceable: + Flags |= MachineMemOperand::MODereferenceable; + break; case MIToken::kw_invariant: Flags |= MachineMemOperand::MOInvariant; break; diff --git a/llvm/lib/CodeGen/MIRPrinter.cpp b/llvm/lib/CodeGen/MIRPrinter.cpp index b981f597d57..e293a0743eb 100644 --- a/llvm/lib/CodeGen/MIRPrinter.cpp +++ b/llvm/lib/CodeGen/MIRPrinter.cpp @@ -892,6 +892,8 @@ void MIPrinter::print(const MachineMemOperand &Op) { OS << "volatile "; if (Op.isNonTemporal()) OS << "non-temporal "; + if (Op.isDereferenceable()) + OS << "dereferenceable "; if (Op.isInvariant()) OS << "invariant "; if (Op.isLoad()) diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index 90ebf2c532f..fb9bd8f6ec3 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -650,10 +650,10 @@ void MachineMemOperand::print(raw_ostream &OS, ModuleSlotTracker &MST) const { OS << ")"; } - // Print nontemporal info. if (isNonTemporal()) OS << "(nontemporal)"; - + if (isDereferenceable()) + OS << "(dereferenceable)"; if (isInvariant()) OS << "(invariant)"; } @@ -1580,7 +1580,8 @@ bool MachineInstr::isDereferenceableInvariantLoad(AliasAnalysis *AA) const { for (MachineMemOperand *MMO : memoperands()) { if (MMO->isVolatile()) return false; if (MMO->isStore()) return false; - if (MMO->isInvariant()) continue; + if (MMO->isInvariant() && MMO->isDereferenceable()) + continue; // A load from a constant PseudoSourceValue is invariant. if (const PseudoSourceValue *PSV = MMO->getPseudoValue()) diff --git a/llvm/lib/CodeGen/MachinePipeliner.cpp b/llvm/lib/CodeGen/MachinePipeliner.cpp index dc61ebaeae6..45e9d50805e 100644 --- a/llvm/lib/CodeGen/MachinePipeliner.cpp +++ b/llvm/lib/CodeGen/MachinePipeliner.cpp @@ -3082,7 +3082,8 @@ void SwingSchedulerDAG::updateMemOperands(MachineInstr &NewMI, MachineInstr::mmo_iterator NewMemRefs = MF.allocateMemRefsArray(NumRefs); unsigned Refs = 0; for (MachineMemOperand *MMO : NewMI.memoperands()) { - if (MMO->isVolatile() || MMO->isInvariant() || (!MMO->getValue())) { + if (MMO->isVolatile() || (MMO->isInvariant() && MMO->isDereferenceable()) || + (!MMO->getValue())) { NewMemRefs[Refs++] = MMO; continue; } diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 249abb998e8..8783bb24dae 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -14280,6 +14280,8 @@ bool DAGCombiner::SimplifySelectOps(SDNode *TheSelect, SDValue LHS, MachineMemOperand::Flags MMOFlags = LLD->getMemOperand()->getFlags(); if (!RLD->isInvariant()) MMOFlags &= ~MachineMemOperand::MOInvariant; + if (!RLD->isDereferenceable()) + MMOFlags &= ~MachineMemOperand::MODereferenceable; if (LLD->getExtensionType() == ISD::NON_EXTLOAD) { // FIXME: Discards pointer and AA info. Load = DAG.getLoad(TheSelect->getValueType(0), SDLoc(TheSelect), diff --git a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp index 1d961af70de..d25ea4e72eb 100644 --- a/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/FastISel.cpp @@ -2183,6 +2183,8 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const { bool IsNonTemporal = I->getMetadata(LLVMContext::MD_nontemporal) != nullptr; bool IsInvariant = I->getMetadata(LLVMContext::MD_invariant_load) != nullptr; + bool IsDereferenceable = + I->getMetadata(LLVMContext::MD_dereferenceable) != nullptr; const MDNode *Ranges = I->getMetadata(LLVMContext::MD_range); AAMDNodes AAInfo; @@ -2197,6 +2199,8 @@ FastISel::createMachineMemOperandFor(const Instruction *I) const { Flags |= MachineMemOperand::MOVolatile; if (IsNonTemporal) Flags |= MachineMemOperand::MONonTemporal; + if (IsDereferenceable) + Flags |= MachineMemOperand::MODereferenceable; if (IsInvariant) Flags |= MachineMemOperand::MOInvariant; diff --git a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp index 31ebf7bbec1..37163650474 100644 --- a/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/LegalizeFloatTypes.cpp @@ -632,7 +632,8 @@ SDValue DAGTypeLegalizer::SoftenFloatRes_LOAD(SDNode *N, unsigned ResNo) { SDLoc dl(N); auto MMOFlags = - L->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant; + L->getMemOperand()->getFlags() & + ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); SDValue NewL; if (L->getExtensionType() == ISD::NON_EXTLOAD) { NewL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), NVT, dl, @@ -2085,7 +2086,8 @@ SDValue DAGTypeLegalizer::PromoteFloatRes_LOAD(SDNode *N) { // Load the value as an integer value with the same number of bits. EVT IVT = EVT::getIntegerVT(*DAG.getContext(), VT.getSizeInBits()); auto MMOFlags = - L->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant; + L->getMemOperand()->getFlags() & + ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); SDValue newL = DAG.getLoad(L->getAddressingMode(), L->getExtensionType(), IVT, SDLoc(N), L->getChain(), L->getBasePtr(), L->getOffset(), L->getPointerInfo(), IVT, diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 991b19bf731..f050eb56146 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -5167,9 +5167,10 @@ SDValue SelectionDAG::getIndexedLoad(SDValue OrigLoad, const SDLoc &dl, ISD::MemIndexedMode AM) { LoadSDNode *LD = cast<LoadSDNode>(OrigLoad); assert(LD->getOffset().isUndef() && "Load is already a indexed load!"); - // Don't propagate the invariant flag. + // Don't propagate the invariant or dereferenceable flags. auto MMOFlags = - LD->getMemOperand()->getFlags() & ~MachineMemOperand::MOInvariant; + LD->getMemOperand()->getFlags() & + ~(MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable); return getLoad(AM, LD->getExtensionType(), OrigLoad.getValueType(), dl, LD->getChain(), Base, Offset, LD->getPointerInfo(), LD->getMemoryVT(), LD->getAlignment(), MMOFlags, @@ -6716,6 +6717,7 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, const DebugLoc &dl, : SDNode(Opc, Order, dl, VTs), MemoryVT(memvt), MMO(mmo) { MemSDNodeBits.IsVolatile = MMO->isVolatile(); MemSDNodeBits.IsNonTemporal = MMO->isNonTemporal(); + MemSDNodeBits.IsDereferenceable = MMO->isDereferenceable(); MemSDNodeBits.IsInvariant = MMO->isInvariant(); // We check here that the size of the memory operand fits within the size of diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 5e9bc0da871..e67915b2909 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2010,7 +2010,8 @@ static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, if (Global) { MachinePointerInfo MPInfo(Global); MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | + MachineMemOperand::MODereferenceable; *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy)); Node->setMemRefs(MemRefs, MemRefs + 1); @@ -3473,17 +3474,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { bool isVolatile = I.isVolatile(); bool isNonTemporal = I.getMetadata(LLVMContext::MD_nontemporal) != nullptr; - - // The IR notion of invariant_load only guarantees that all *non-faulting* - // invariant loads result in the same value. The MI notion of invariant load - // guarantees that the load can be legally moved to any location within its - // containing function. The MI notion of invariant_load is stronger than the - // IR notion of invariant_load -- an MI invariant_load is an IR invariant_load - // with a guarantee that the location being loaded from is dereferenceable - // throughout the function's lifetime. - - bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr && - isDereferenceablePointer(SV, DAG.getDataLayout()); + bool isInvariant = I.getMetadata(LLVMContext::MD_invariant_load) != nullptr; + bool isDereferenceable = isDereferenceablePointer(SV, DAG.getDataLayout()); unsigned Alignment = I.getAlignment(); AAMDNodes AAInfo; @@ -3551,6 +3543,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { MMOFlags |= MachineMemOperand::MONonTemporal; if (isInvariant) MMOFlags |= MachineMemOperand::MOInvariant; + if (isDereferenceable) + MMOFlags |= MachineMemOperand::MODereferenceable; SDValue L = DAG.getLoad(ValueVTs[i], dl, Root, A, MachinePointerInfo(SV, Offsets[i]), Alignment, diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index bdc9ca4444b..732baca0c99 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -3445,11 +3445,12 @@ AArch64TargetLowering::LowerDarwinGlobalTLSAddress(SDValue Op, // The first entry in the descriptor is a function pointer that we must call // to obtain the address of the variable. SDValue Chain = DAG.getEntryNode(); - SDValue FuncTLVGet = - DAG.getLoad(MVT::i64, DL, Chain, DescAddr, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), - /* Alignment = */ 8, MachineMemOperand::MONonTemporal | - MachineMemOperand::MOInvariant); + SDValue FuncTLVGet = DAG.getLoad( + MVT::i64, DL, Chain, DescAddr, + MachinePointerInfo::getGOT(DAG.getMachineFunction()), + /* Alignment = */ 8, + MachineMemOperand::MONonTemporal | MachineMemOperand::MOInvariant | + MachineMemOperand::MODereferenceable); Chain = FuncTLVGet.getValue(1); MachineFrameInfo &MFI = DAG.getMachineFunction().getFrameInfo(); diff --git a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp index da4cf387c8d..6b391de00d1 100644 --- a/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/R600ISelLowering.cpp @@ -1564,8 +1564,9 @@ SDValue R600TargetLowering::LowerFormalArguments( SDValue Arg = DAG.getLoad( ISD::UNINDEXED, Ext, VT, DL, Chain, DAG.getConstant(Offset, DL, MVT::i32), DAG.getUNDEF(MVT::i32), PtrInfo, - MemVT, /* Alignment = */ 4, - MachineMemOperand::MONonTemporal | MachineMemOperand::MOInvariant); + MemVT, /* Alignment = */ 4, MachineMemOperand::MONonTemporal | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); // 4 is the preferred alignment for the CONSTANT memory space. InVals.push_back(Arg); diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index b79bdeadef3..3d9c08046e5 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -581,8 +581,10 @@ SDValue SITargetLowering::LowerParameter(SelectionDAG &DAG, EVT VT, EVT MemVT, SDValue Ptr = LowerParameterPtr(DAG, SL, Chain, Offset); return DAG.getLoad(ISD::UNINDEXED, ExtTy, VT, SL, Chain, Ptr, PtrOffset, - PtrInfo, MemVT, Align, MachineMemOperand::MONonTemporal | - MachineMemOperand::MOInvariant); + PtrInfo, MemVT, Align, + MachineMemOperand::MONonTemporal | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } SDValue SITargetLowering::LowerFormalArguments( @@ -1792,7 +1794,8 @@ SDValue SITargetLowering::getSegmentAperture(unsigned AS, MachinePointerInfo PtrInfo(V, StructOffset); return DAG.getLoad(MVT::i32, SL, QueuePtr.getValue(1), Ptr, PtrInfo, MinAlign(64, StructOffset), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } SDValue SITargetLowering::lowerADDRSPACECAST(SDValue Op, @@ -1910,7 +1913,8 @@ SDValue SITargetLowering::LowerGlobalAddress(AMDGPUMachineFunction *MFI, MachinePointerInfo PtrInfo(UndefValue::get(PtrTy)); return DAG.getLoad(PtrVT, DL, DAG.getEntryNode(), GOTAddr, PtrInfo, Align, - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } SDValue SITargetLowering::lowerTRAP(SDValue Op, @@ -2130,9 +2134,10 @@ SDValue SITargetLowering::LowerINTRINSIC_WO_CHAIN(SDValue Op, }; MachineMemOperand *MMO = MF.getMachineMemOperand( - MachinePointerInfo(), - MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant, - VT.getStoreSize(), 4); + MachinePointerInfo(), + MachineMemOperand::MOLoad | MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant, + VT.getStoreSize(), 4); return DAG.getMemIntrinsicNode(AMDGPUISD::LOAD_CONSTANT, DL, Op->getVTList(), Ops, VT, MMO); } diff --git a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp index 79803fc595c..cd215b40d15 100644 --- a/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp +++ b/llvm/lib/Target/ARM/ARMBaseInstrInfo.cpp @@ -4155,7 +4155,9 @@ void ARMBaseInstrInfo::expandLoadStackGuardBase(MachineBasicBlock::iterator MI, if (Subtarget.isGVIndirectSymbol(GV)) { MIB = BuildMI(MBB, MI, DL, get(LoadOpc), Reg); MIB.addReg(Reg, RegState::Kill).addImm(0); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant; MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, 4); MIB.addMemOperand(MMO); diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 31f08b3b4af..3441688daa5 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -2040,10 +2040,11 @@ ARMTargetLowering::LowerCall(TargetLowering::CallLoweringInfo &CLI, Callee = DAG.getNode( ARMISD::WrapperPIC, dl, PtrVt, DAG.getTargetGlobalAddress(GV, dl, PtrVt, 0, ARMII::MO_NONLAZY)); - Callee = - DAG.getLoad(PtrVt, dl, DAG.getEntryNode(), Callee, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), - /* Alignment = */ 0, MachineMemOperand::MOInvariant); + Callee = DAG.getLoad( + PtrVt, dl, DAG.getEntryNode(), Callee, + MachinePointerInfo::getGOT(DAG.getMachineFunction()), + /* Alignment = */ 0, MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); } else if (Subtarget->isTargetCOFF()) { assert(Subtarget->isTargetWindows() && "Windows is the only supported COFF target"); @@ -2755,11 +2756,12 @@ ARMTargetLowering::LowerGlobalTLSAddressDarwin(SDValue Op, // The first entry in the descriptor is a function pointer that we must call // to obtain the address of the variable. SDValue Chain = DAG.getEntryNode(); - SDValue FuncTLVGet = - DAG.getLoad(MVT::i32, DL, Chain, DescAddr, - MachinePointerInfo::getGOT(DAG.getMachineFunction()), - /* Alignment = */ 4, MachineMemOperand::MONonTemporal | - MachineMemOperand::MOInvariant); + SDValue FuncTLVGet = DAG.getLoad( + MVT::i32, DL, Chain, DescAddr, + MachinePointerInfo::getGOT(DAG.getMachineFunction()), + /* Alignment = */ 4, + MachineMemOperand::MONonTemporal | MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); Chain = FuncTLVGet.getValue(1); MachineFunction &F = DAG.getMachineFunction(); diff --git a/llvm/lib/Target/ARM/ARMInstrInfo.cpp b/llvm/lib/Target/ARM/ARMInstrInfo.cpp index 98b1b4ca427..27b64322dfa 100644 --- a/llvm/lib/Target/ARM/ARMInstrInfo.cpp +++ b/llvm/lib/Target/ARM/ARMInstrInfo.cpp @@ -123,7 +123,9 @@ void ARMInstrInfo::expandLoadStackGuard(MachineBasicBlock::iterator MI) const { MIB = BuildMI(MBB, MI, DL, get(ARM::MOV_ga_pcrel_ldr), Reg) .addGlobalAddress(GV, 0, ARMII::MO_NONLAZY); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant; MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 4, 4); MIB.addMemOperand(MMO); diff --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp index 52eaf6e4c67..95556a71c9c 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp @@ -2238,7 +2238,8 @@ SDValue NVPTXTargetLowering::LowerFormalArguments( SDValue P = DAG.getLoad( EltVT, dl, Root, Arg, MachinePointerInfo(SrcValue), DL.getABITypeAlignment(EltVT.getTypeForEVT(F->getContext())), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); if (P.getNode()) P.getNode()->setIROrder(idx + 1); @@ -2255,7 +2256,8 @@ SDValue NVPTXTargetLowering::LowerFormalArguments( SDValue P = DAG.getLoad( VecVT, dl, Root, Arg, MachinePointerInfo(SrcValue), DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); if (P.getNode()) P.getNode()->setIROrder(idx + 1); @@ -2297,7 +2299,8 @@ SDValue NVPTXTargetLowering::LowerFormalArguments( SDValue P = DAG.getLoad( VecVT, dl, Root, SrcAddr, MachinePointerInfo(SrcValue), DL.getABITypeAlignment(VecVT.getTypeForEVT(F->getContext())), - MachineMemOperand::MOInvariant); + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant); if (P.getNode()) P.getNode()->setIROrder(idx + 1); diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp index 028f31a5f8c..170704958d4 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.cpp @@ -4444,7 +4444,8 @@ PrepareCall(SelectionDAG &DAG, SDValue &Callee, SDValue &InFlag, SDValue &Chain, LDChain = CallSeqStart.getValue(CallSeqStart->getNumValues()-2); auto MMOFlags = Subtarget.hasInvariantFunctionDescriptors() - ? MachineMemOperand::MOInvariant + ? (MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant) : MachineMemOperand::MONone; MachinePointerInfo MPI(CS ? CS->getCalledValue() : nullptr); @@ -6481,10 +6482,7 @@ SDValue PPCTargetLowering::LowerFP_TO_INT(SDValue Op, SelectionDAG &DAG, LowerFP_TO_INTForReuse(Op, RLI, DAG, dl); return DAG.getLoad(Op.getValueType(), dl, RLI.Chain, RLI.Ptr, RLI.MPI, - RLI.Alignment, - RLI.IsInvariant ? MachineMemOperand::MOInvariant - : MachineMemOperand::MONone, - RLI.AAInfo, RLI.Ranges); + RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); } // We're trying to insert a regular store, S, and then a load, L. If the @@ -6527,6 +6525,7 @@ bool PPCTargetLowering::canReuseLoadAddress(SDValue Op, EVT MemVT, RLI.Chain = LD->getChain(); RLI.MPI = LD->getPointerInfo(); + RLI.IsDereferenceable = LD->isDereferenceable(); RLI.IsInvariant = LD->isInvariant(); RLI.Alignment = LD->getAlignment(); RLI.AAInfo = LD->getAAInfo(); @@ -6719,11 +6718,8 @@ SDValue PPCTargetLowering::LowerINT_TO_FP(SDValue Op, MachineFunction &MF = DAG.getMachineFunction(); if (canReuseLoadAddress(SINT, MVT::i64, RLI, DAG)) { - Bits = - DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, RLI.Alignment, - RLI.IsInvariant ? MachineMemOperand::MOInvariant - : MachineMemOperand::MONone, - RLI.AAInfo, RLI.Ranges); + Bits = DAG.getLoad(MVT::f64, dl, RLI.Chain, RLI.Ptr, RLI.MPI, + RLI.Alignment, RLI.MMOFlags(), RLI.AAInfo, RLI.Ranges); spliceIntoChain(RLI.ResChain, Bits.getValue(1), DAG); } else if (Subtarget.hasLFIWAX() && canReuseLoadAddress(SINT, MVT::i32, RLI, DAG, ISD::SEXTLOAD)) { diff --git a/llvm/lib/Target/PowerPC/PPCISelLowering.h b/llvm/lib/Target/PowerPC/PPCISelLowering.h index e1dcd7d1d36..8b1c22b6bcd 100644 --- a/llvm/lib/Target/PowerPC/PPCISelLowering.h +++ b/llvm/lib/Target/PowerPC/PPCISelLowering.h @@ -761,12 +761,24 @@ namespace llvm { SDValue Chain; SDValue ResChain; MachinePointerInfo MPI; + bool IsDereferenceable; bool IsInvariant; unsigned Alignment; AAMDNodes AAInfo; const MDNode *Ranges; - ReuseLoadInfo() : IsInvariant(false), Alignment(0), Ranges(nullptr) {} + ReuseLoadInfo() + : IsDereferenceable(false), IsInvariant(false), Alignment(0), + Ranges(nullptr) {} + + MachineMemOperand::Flags MMOFlags() const { + MachineMemOperand::Flags F = MachineMemOperand::MONone; + if (IsDereferenceable) + F |= MachineMemOperand::MODereferenceable; + if (IsInvariant) + F |= MachineMemOperand::MOInvariant; + return F; + } }; bool canReuseLoadAddress(SDValue Op, EVT MemVT, ReuseLoadInfo &RLI, diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index 5677fdaaff2..1290fc1cb23 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -1176,7 +1176,7 @@ bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store, return false; // There's no chance of overlap if the load is invariant. - if (Load->isInvariant()) + if (Load->isInvariant() && Load->isDereferenceable()) return true; // Otherwise we need to check whether there's an alias. diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index 1813fb8f59b..6de41728971 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -5684,7 +5684,9 @@ static void expandLoadStackGuard(MachineInstrBuilder &MIB, unsigned Reg = MIB->getOperand(0).getReg(); const GlobalValue *GV = cast<GlobalValue>((*MIB->memoperands_begin())->getValue()); - auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant; + auto Flags = MachineMemOperand::MOLoad | + MachineMemOperand::MODereferenceable | + MachineMemOperand::MOInvariant; MachineMemOperand *MMO = MBB.getParent()->getMachineMemOperand( MachinePointerInfo::getGOT(*MBB.getParent()), Flags, 8, 8); MachineBasicBlock::iterator I = MIB.getInstr(); |