diff options
23 files changed, 244 insertions, 219 deletions
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index c691d445d19..70e69fcd677 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1156,6 +1156,11 @@ public: SDValue Op3, SDValue Op4, SDValue Op5); SDNode *UpdateNodeOperands(SDNode *N, ArrayRef<SDValue> Ops); + /// *Mutate* the specified machine node's memory references to the provided + /// list. + void setNodeMemRefs(MachineSDNode *N, + ArrayRef<MachineMemOperand *> NewMemRefs); + // Propagates the change in divergence to users void updateDivergence(SDNode * N); diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index a6d7e765295..01037d8e0b4 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -2246,32 +2246,60 @@ public: /// An SDNode that represents everything that will be needed /// to construct a MachineInstr. These nodes are created during the /// instruction selection proper phase. +/// +/// Note that the only supported way to set the `memoperands` is by calling the +/// `SelectionDAG::setNodeMemRefs` function as the memory management happens +/// inside the DAG rather than in the node. class MachineSDNode : public SDNode { -public: - using mmo_iterator = MachineMemOperand **; - private: friend class SelectionDAG; MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc &DL, SDVTList VTs) : SDNode(Opc, Order, DL, VTs) {} - /// Memory reference descriptions for this instruction. - mmo_iterator MemRefs = nullptr; - mmo_iterator MemRefsEnd = nullptr; + // We use a pointer union between a single `MachineMemOperand` pointer and + // a pointer to an array of `MachineMemOperand` pointers. This is null when + // the number of these is zero, the single pointer variant used when the + // number is one, and the array is used for larger numbers. + // + // The array is allocated via the `SelectionDAG`'s allocator and so will + // always live until the DAG is cleaned up and doesn't require ownership here. + // + // We can't use something simpler like `TinyPtrVector` here because `SDNode` + // subclasses aren't managed in a conforming C++ manner. See the comments on + // `SelectionDAG::MorphNodeTo` which details what all goes on, but the + // constraint here is that these don't manage memory with their constructor or + // destructor and can be initialized to a good state even if they start off + // uninitialized. + PointerUnion<MachineMemOperand *, MachineMemOperand **> MemRefs = {}; + + // Note that this could be folded into the above `MemRefs` member if doing so + // is advantageous at some point. We don't need to store this in most cases. + // However, at the moment this doesn't appear to make the allocation any + // smaller and makes the code somewhat simpler to read. + int NumMemRefs = 0; public: - mmo_iterator memoperands_begin() const { return MemRefs; } - mmo_iterator memoperands_end() const { return MemRefsEnd; } - bool memoperands_empty() const { return MemRefsEnd == MemRefs; } - - /// Assign this MachineSDNodes's memory reference descriptor - /// list. This does not transfer ownership. - void setMemRefs(mmo_iterator NewMemRefs, mmo_iterator NewMemRefsEnd) { - for (mmo_iterator MMI = NewMemRefs, MME = NewMemRefsEnd; MMI != MME; ++MMI) - assert(*MMI && "Null mem ref detected!"); - MemRefs = NewMemRefs; - MemRefsEnd = NewMemRefsEnd; + using mmo_iterator = ArrayRef<MachineMemOperand *>::const_iterator; + + ArrayRef<MachineMemOperand *> memoperands() const { + // Special case the common cases. + if (NumMemRefs == 0) + return {}; + if (NumMemRefs == 1) + return makeArrayRef(MemRefs.getAddrOfPtr1(), 1); + + // Otherwise we have an actual array. + return makeArrayRef(MemRefs.get<MachineMemOperand **>(), NumMemRefs); + } + mmo_iterator memoperands_begin() const { return memoperands().begin(); } + mmo_iterator memoperands_end() const { return memoperands().end(); } + bool memoperands_empty() const { return memoperands().empty(); } + + /// Clear out the memory reference descriptor list. + void clearMemRefs() { + MemRefs = nullptr; + NumMemRefs = 0; } static bool classof(const SDNode *N) { diff --git a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp index 238ea878510..1b0dac76613 100644 --- a/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/InstrEmitter.cpp @@ -887,8 +887,11 @@ EmitMachineNode(SDNode *Node, bool IsClone, bool IsCloned, RegState::EarlyClobber); // Transfer all of the memory reference descriptions of this instruction. - MIB.setMemRefs(cast<MachineSDNode>(Node)->memoperands_begin(), - cast<MachineSDNode>(Node)->memoperands_end()); + ArrayRef<MachineMemOperand *> SDNodeMemRefs = + cast<MachineSDNode>(Node)->memoperands(); + MachineMemOperand **MemRefs = MF->allocateMemRefsArray(SDNodeMemRefs.size()); + std::copy(SDNodeMemRefs.begin(), SDNodeMemRefs.end(), MemRefs); + MIB.setMemRefs({MemRefs, SDNodeMemRefs.size()}); // Insert the instruction into position in the block. This needs to // happen before any custom inserter hook is called so that the diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index 430d8fb3447..6563d195e1f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -145,20 +145,18 @@ static void CloneNodeWithValues(SDNode *N, SelectionDAG *DAG, ArrayRef<EVT> VTs, Ops.push_back(ExtraOper); SDVTList VTList = DAG->getVTList(VTs); - MachineSDNode::mmo_iterator Begin = nullptr, End = nullptr; MachineSDNode *MN = dyn_cast<MachineSDNode>(N); // Store memory references. - if (MN) { - Begin = MN->memoperands_begin(); - End = MN->memoperands_end(); - } + SmallVector<MachineMemOperand *, 2> MMOs; + if (MN) + MMOs.assign(MN->memoperands_begin(), MN->memoperands_end()); DAG->MorphNodeTo(N, N->getOpcode(), VTList, Ops); // Reset the memory references if (MN) - MN->setMemRefs(Begin, End); + DAG->setNodeMemRefs(MN, MMOs); } static bool AddGlue(SDNode *N, SDValue Glue, bool AddGlue, SelectionDAG *DAG) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index ac42ec39359..dc167a20253 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -7091,6 +7091,27 @@ void SDNode::DropOperands() { } } +void SelectionDAG::setNodeMemRefs(MachineSDNode *N, + ArrayRef<MachineMemOperand *> NewMemRefs) { + if (NewMemRefs.empty()) { + N->clearMemRefs(); + return; + } + + // Check if we can avoid allocating by storing a single reference directly. + if (NewMemRefs.size() == 1) { + N->MemRefs = NewMemRefs[0]; + N->NumMemRefs = 1; + return; + } + + MachineMemOperand **MemRefsBuffer = + Allocator.template Allocate<MachineMemOperand *>(NewMemRefs.size()); + std::copy(NewMemRefs.begin(), NewMemRefs.end(), MemRefsBuffer); + N->MemRefs = MemRefsBuffer; + N->NumMemRefs = static_cast<int>(NewMemRefs.size()); +} + /// SelectNodeTo - These are wrappers around MorphNodeTo that accept a /// machine opcode. /// @@ -7233,7 +7254,7 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, // For MachineNode, initialize the memory references information. if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) - MN->setMemRefs(nullptr, nullptr); + MN->clearMemRefs(); // Swap for an appropriately sized array from the recycler. removeOperands(N); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 4fb10fadc3c..f0efe4da6e6 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -2193,12 +2193,11 @@ static SDValue getLoadStackGuard(SelectionDAG &DAG, const SDLoc &DL, DAG.getMachineNode(TargetOpcode::LOAD_STACK_GUARD, DL, PtrTy, Chain); if (Global) { MachinePointerInfo MPInfo(Global); - MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1); auto Flags = MachineMemOperand::MOLoad | MachineMemOperand::MOInvariant | MachineMemOperand::MODereferenceable; - *MemRefs = MF.getMachineMemOperand(MPInfo, Flags, PtrTy.getSizeInBits() / 8, - DAG.getEVTAlignment(PtrTy)); - Node->setMemRefs(MemRefs, MemRefs + 1); + MachineMemOperand *MemRef = MF.getMachineMemOperand( + MPInfo, Flags, PtrTy.getSizeInBits() / 8, DAG.getEVTAlignment(PtrTy)); + DAG.setNodeMemRefs(Node, {MemRef}); } return SDValue(Node, 0); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index f7bd8847bee..8df30fb0349 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -3598,38 +3598,22 @@ void SelectionDAGISel::SelectCodeCommon(SDNode *NodeToMatch, bool mayLoad = MCID.mayLoad(); bool mayStore = MCID.mayStore(); - unsigned NumMemRefs = 0; - for (SmallVectorImpl<MachineMemOperand *>::const_iterator I = - MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) { - if ((*I)->isLoad()) { + // We expect to have relatively few of these so just filter them into a + // temporary buffer so that we can easily add them to the instruction. + SmallVector<MachineMemOperand *, 4> FilteredMemRefs; + for (MachineMemOperand *MMO : MatchedMemRefs) { + if (MMO->isLoad()) { if (mayLoad) - ++NumMemRefs; - } else if ((*I)->isStore()) { + FilteredMemRefs.push_back(MMO); + } else if (MMO->isStore()) { if (mayStore) - ++NumMemRefs; + FilteredMemRefs.push_back(MMO); } else { - ++NumMemRefs; + FilteredMemRefs.push_back(MMO); } } - MachineSDNode::mmo_iterator MemRefs = - MF->allocateMemRefsArray(NumMemRefs); - - MachineSDNode::mmo_iterator MemRefsPos = MemRefs; - for (SmallVectorImpl<MachineMemOperand *>::const_iterator I = - MatchedMemRefs.begin(), E = MatchedMemRefs.end(); I != E; ++I) { - if ((*I)->isLoad()) { - if (mayLoad) - *MemRefsPos++ = *I; - } else if ((*I)->isStore()) { - if (mayStore) - *MemRefsPos++ = *I; - } else { - *MemRefsPos++ = *I; - } - } - - Res->setMemRefs(MemRefs, MemRefs + NumMemRefs); + CurDAG->setNodeMemRefs(Res, FilteredMemRefs); } LLVM_DEBUG(if (!MatchedMemRefs.empty() && Res->memoperands_empty()) dbgs() diff --git a/llvm/lib/CodeGen/StackColoring.cpp b/llvm/lib/CodeGen/StackColoring.cpp index 81a41970f9e..ff7dc211025 100644 --- a/llvm/lib/CodeGen/StackColoring.cpp +++ b/llvm/lib/CodeGen/StackColoring.cpp @@ -1022,7 +1022,7 @@ void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) { } // We adjust AliasAnalysis information for merged stack slots. - MachineSDNode::mmo_iterator NewMemOps = + MachineInstr::mmo_iterator NewMemOps = MF->allocateMemRefsArray(I.getNumMemOperands()); unsigned MemOpIdx = 0; bool ReplaceMemOps = false; diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp index c1a9ee333b6..06035803a6b 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp @@ -1208,9 +1208,8 @@ void AArch64DAGToDAGISel::SelectLoad(SDNode *N, unsigned NumVecs, unsigned Opc, ReplaceUses(SDValue(N, NumVecs), SDValue(Ld, 1)); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp}); CurDAG->RemoveDeadNode(N); } @@ -1261,9 +1260,8 @@ void AArch64DAGToDAGISel::SelectStore(SDNode *N, unsigned NumVecs, SDNode *St = CurDAG->getMachineNode(Opc, dl, N->getValueType(0), Ops); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); ReplaceNode(N, St); } @@ -1441,9 +1439,8 @@ void AArch64DAGToDAGISel::SelectStoreLane(SDNode *N, unsigned NumVecs, SDNode *St = CurDAG->getMachineNode(Opc, dl, MVT::Other, Ops); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); ReplaceNode(N, St); } @@ -1476,9 +1473,8 @@ void AArch64DAGToDAGISel::SelectPostStoreLane(SDNode *N, unsigned NumVecs, SDNode *St = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); ReplaceNode(N, St); } @@ -2751,9 +2747,8 @@ bool AArch64DAGToDAGISel::SelectCMP_SWAP(SDNode *N) { Opcode, SDLoc(N), CurDAG->getVTList(RegTy, MVT::i32, MVT::Other), Ops); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp}); ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0)); ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2)); @@ -2923,9 +2918,9 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) { MVT::Other, MemAddr, Chain); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(Node)->getMemOperand(); - cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = + cast<MemIntrinsicSDNode>(Node)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp}); ReplaceNode(Node, Ld); return; } @@ -2944,9 +2939,9 @@ void AArch64DAGToDAGISel::Select(SDNode *Node) { SDNode *St = CurDAG->getMachineNode(Op, DL, MVT::i32, MVT::Other, Ops); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(Node)->getMemOperand(); - cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = + cast<MemIntrinsicSDNode>(Node)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); ReplaceNode(Node, St); return; diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp index cbe48a5dc42..b665debb929 100644 --- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp +++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp @@ -11175,12 +11175,10 @@ static void ReplaceCMP_SWAP_128Results(SDNode *N, N->getOperand(0), // Chain in }; - MachineFunction &MF = DAG.getMachineFunction(); - MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1); - MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); + MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); unsigned Opcode; - switch (MemOp[0]->getOrdering()) { + switch (MemOp->getOrdering()) { case AtomicOrdering::Monotonic: Opcode = AArch64::CASPX; break; @@ -11200,7 +11198,7 @@ static void ReplaceCMP_SWAP_128Results(SDNode *N, MachineSDNode *CmpSwap = DAG.getMachineNode( Opcode, SDLoc(N), DAG.getVTList(MVT::Untyped, MVT::Other), Ops); - CmpSwap->setMemRefs(MemOp, MemOp + 1); + DAG.setNodeMemRefs(CmpSwap, {MemOp}); unsigned SubReg1 = AArch64::sube64, SubReg2 = AArch64::subo64; if (DAG.getDataLayout().isBigEndian()) @@ -11221,10 +11219,8 @@ static void ReplaceCMP_SWAP_128Results(SDNode *N, AArch64::CMP_SWAP_128, SDLoc(N), DAG.getVTList(MVT::i64, MVT::i64, MVT::i32, MVT::Other), Ops); - MachineFunction &MF = DAG.getMachineFunction(); - MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1); - MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); + DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp}); Results.push_back(SDValue(CmpSwap, 0)); Results.push_back(SDValue(CmpSwap, 1)); diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp index 6defd97693f..a5626407719 100644 --- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp +++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp @@ -1849,9 +1849,8 @@ void AMDGPUDAGToDAGISel::SelectATOMIC_CMP_SWAP(SDNode *N) { return; } - MachineSDNode::mmo_iterator MMOs = MF->allocateMemRefsArray(1); - *MMOs = Mem->getMemOperand(); - CmpSwap->setMemRefs(MMOs, MMOs + 1); + MachineMemOperand *MMO = Mem->getMemOperand(); + CurDAG->setNodeMemRefs(CmpSwap, {MMO}); unsigned SubReg = Is32 ? AMDGPU::sub0 : AMDGPU::sub0_sub1; SDValue Extract diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp index ba8a3a512cb..16ebc8893ab 100644 --- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp +++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp @@ -4512,7 +4512,6 @@ SDValue SITargetLowering::lowerImage(SDValue Op, const AMDGPU::ImageDimIntrinsicInfo *Intr, SelectionDAG &DAG) const { SDLoc DL(Op); - MachineFunction &MF = DAG.getMachineFunction(); const AMDGPU::MIMGBaseOpcodeInfo *BaseOpcode = AMDGPU::getMIMGBaseOpcodeInfo(Intr->BaseOpcode); const AMDGPU::MIMGDimInfo *DimInfo = AMDGPU::getMIMGDimInfo(Intr->Dim); @@ -4684,9 +4683,8 @@ SDValue SITargetLowering::lowerImage(SDValue Op, MachineSDNode *NewNode = DAG.getMachineNode(Opcode, DL, ResultTypes, Ops); if (auto MemOp = dyn_cast<MemSDNode>(Op)) { - MachineInstr::mmo_iterator MemRefs = MF.allocateMemRefsArray(1); - *MemRefs = MemOp->getMemOperand(); - NewNode->setMemRefs(MemRefs, MemRefs + 1); + MachineMemOperand *MemRef = MemOp->getMemOperand(); + DAG.setNodeMemRefs(NewNode, {MemRef}); } if (BaseOpcode->AtomicX2) { @@ -8116,7 +8114,7 @@ SDNode *SITargetLowering::adjustWritemask(MachineSDNode *&Node, if (HasChain) { // Update chain. - NewNode->setMemRefs(Node->memoperands_begin(), Node->memoperands_end()); + DAG.setNodeMemRefs(NewNode, Node->memoperands()); DAG.ReplaceAllUsesOfValueWith(SDValue(Node, 1), SDValue(NewNode, 1)); } diff --git a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp index 665e67beddb..77622567d4b 100644 --- a/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp +++ b/llvm/lib/Target/ARM/ARMISelDAGToDAG.cpp @@ -1345,9 +1345,8 @@ static inline SDValue getAL(SelectionDAG *CurDAG, const SDLoc &dl) { } void ARMDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); } bool ARMDAGToDAGISel::tryARMIndexedLoad(SDNode *N) { @@ -1854,9 +1853,8 @@ void ARMDAGToDAGISel::SelectVLD(SDNode *N, bool isUpdating, unsigned NumVecs, } // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(VLd)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLd), {MemOp}); if (NumVecs == 1) { ReplaceNode(N, VLd); @@ -1893,8 +1891,7 @@ void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) return; - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); SDValue Chain = N->getOperand(0); EVT VT = N->getOperand(Vec0Idx).getValueType(); @@ -1983,7 +1980,7 @@ void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, SDNode *VSt = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); // Transfer memoperands. - cast<MachineSDNode>(VSt)->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(VSt), {MemOp}); ReplaceNode(N, VSt); return; @@ -2007,7 +2004,7 @@ void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, SDNode *VStA = CurDAG->getMachineNode(QOpcodes0[OpcodeIndex], dl, MemAddr.getValueType(), MVT::Other, OpsA); - cast<MachineSDNode>(VStA)->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStA), {MemOp}); Chain = SDValue(VStA, 1); // Store the odd D registers. @@ -2026,7 +2023,7 @@ void ARMDAGToDAGISel::SelectVST(SDNode *N, bool isUpdating, unsigned NumVecs, Ops.push_back(Chain); SDNode *VStB = CurDAG->getMachineNode(QOpcodes1[OpcodeIndex], dl, ResTys, Ops); - cast<MachineSDNode>(VStB)->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(VStB), {MemOp}); ReplaceNode(N, VStB); } @@ -2045,8 +2042,7 @@ void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating, if (!SelectAddrMode6(N, N->getOperand(AddrOpIdx), MemAddr, Align)) return; - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); SDValue Chain = N->getOperand(0); unsigned Lane = @@ -2135,7 +2131,7 @@ void ARMDAGToDAGISel::SelectVLDSTLane(SDNode *N, bool IsLoad, bool isUpdating, unsigned Opc = (is64BitVector ? DOpcodes[OpcodeIndex] : QOpcodes[OpcodeIndex]); SDNode *VLdLn = CurDAG->getMachineNode(Opc, dl, ResTys, Ops); - cast<MachineSDNode>(VLdLn)->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdLn), {MemOp}); if (!IsLoad) { ReplaceNode(N, VLdLn); return; @@ -2264,9 +2260,8 @@ void ARMDAGToDAGISel::SelectVLDDup(SDNode *N, bool IsIntrinsic, } // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(VLdDup)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(VLdDup), {MemOp}); // Extract the subregisters. if (NumVecs == 1) { @@ -2481,9 +2476,8 @@ void ARMDAGToDAGISel::SelectCMP_SWAP(SDNode *N) { Opcode, SDLoc(N), CurDAG->getVTList(MVT::i32, MVT::i32, MVT::Other), Ops); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp}); ReplaceUses(SDValue(N, 0), SDValue(CmpSwap, 0)); ReplaceUses(SDValue(N, 1), SDValue(CmpSwap, 2)); @@ -2632,12 +2626,11 @@ void ARMDAGToDAGISel::Select(SDNode *N) { // queries work properly. This e.g. gives the register allocation the // required information for rematerialization. MachineFunction& MF = CurDAG->getMachineFunction(); - MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1); - MemOp[0] = MF.getMachineMemOperand( - MachinePointerInfo::getConstantPool(MF), - MachineMemOperand::MOLoad, 4, 4); + MachineMemOperand *MemOp = + MF.getMachineMemOperand(MachinePointerInfo::getConstantPool(MF), + MachineMemOperand::MOLoad, 4, 4); - cast<MachineSDNode>(ResNode)->setMemRefs(MemOp, MemOp+1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemOp}); ReplaceNode(N, ResNode); return; @@ -3421,9 +3414,8 @@ void ARMDAGToDAGISel::Select(SDNode *N) { CurDAG->getRegister(0, MVT::i32), Chain}; SDNode *Ld = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Ld)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ld), {MemOp}); // Remap uses. SDValue OutChain = isThumb ? SDValue(Ld, 2) : SDValue(Ld, 1); @@ -3489,9 +3481,8 @@ void ARMDAGToDAGISel::Select(SDNode *N) { SDNode *St = CurDAG->getMachineNode(NewOpc, dl, ResTys, Ops); // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(St)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(St), {MemOp}); ReplaceNode(N, St); return; diff --git a/llvm/lib/Target/ARM/ARMISelLowering.cpp b/llvm/lib/Target/ARM/ARMISelLowering.cpp index 533ac02a15f..d0b3fb9c679 100644 --- a/llvm/lib/Target/ARM/ARMISelLowering.cpp +++ b/llvm/lib/Target/ARM/ARMISelLowering.cpp @@ -8030,10 +8030,8 @@ static void ReplaceCMP_SWAP_64Results(SDNode *N, ARM::CMP_SWAP_64, SDLoc(N), DAG.getVTList(MVT::Untyped, MVT::i32, MVT::Other), Ops); - MachineFunction &MF = DAG.getMachineFunction(); - MachineSDNode::mmo_iterator MemOp = MF.allocateMemRefsArray(1); - MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(CmpSwap)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); + DAG.setNodeMemRefs(cast<MachineSDNode>(CmpSwap), {MemOp}); bool isBigEndian = DAG.getDataLayout().isBigEndian(); diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp index efb4c2eb0fc..9a66aece579 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAG.cpp @@ -127,8 +127,7 @@ void HexagonDAGToDAGISel::SelectIndexedLoad(LoadSDNode *LD, const SDLoc &dl) { } SDValue IncV = CurDAG->getTargetConstant(Inc, dl, MVT::i32); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = LD->getMemOperand(); + MachineMemOperand *MemOp = LD->getMemOperand(); auto getExt64 = [this,ExtType] (MachineSDNode *N, const SDLoc &dl) -> MachineSDNode* { @@ -159,7 +158,7 @@ void HexagonDAGToDAGISel::SelectIndexedLoad(LoadSDNode *LD, const SDLoc &dl) { MachineSDNode *L = CurDAG->getMachineNode(Opcode, dl, ValueVT, MVT::i32, MVT::Other, Base, IncV, Chain); - L->setMemRefs(MemOp, MemOp+1); + CurDAG->setNodeMemRefs(L, {MemOp}); To[1] = SDValue(L, 1); // Next address. To[2] = SDValue(L, 2); // Chain. // Handle special case for extension to i64. @@ -170,7 +169,7 @@ void HexagonDAGToDAGISel::SelectIndexedLoad(LoadSDNode *LD, const SDLoc &dl) { SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32); MachineSDNode *L = CurDAG->getMachineNode(Opcode, dl, ValueVT, MVT::Other, Base, Zero, Chain); - L->setMemRefs(MemOp, MemOp+1); + CurDAG->setNodeMemRefs(L, {MemOp}); To[2] = SDValue(L, 1); // Chain. MachineSDNode *A = CurDAG->getMachineNode(Hexagon::A2_addi, dl, MVT::i32, Base, IncV); @@ -344,9 +343,8 @@ bool HexagonDAGToDAGISel::SelectBrevLdIntrinsic(SDNode *IntN) { FLI->second, dl, RTys, {IntN->getOperand(2), IntN->getOperand(3), IntN->getOperand(0)}); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(IntN)->getMemOperand(); - Res->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(IntN)->getMemOperand(); + CurDAG->setNodeMemRefs(Res, {MemOp}); ReplaceUses(SDValue(IntN, 0), SDValue(Res, 0)); ReplaceUses(SDValue(IntN, 1), SDValue(Res, 1)); @@ -525,8 +523,7 @@ void HexagonDAGToDAGISel::SelectIndexedStore(StoreSDNode *ST, const SDLoc &dl) { } SDValue IncV = CurDAG->getTargetConstant(Inc, dl, MVT::i32); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = ST->getMemOperand(); + MachineMemOperand *MemOp = ST->getMemOperand(); // Next address Chain SDValue From[2] = { SDValue(ST,0), SDValue(ST,1) }; @@ -537,14 +534,14 @@ void HexagonDAGToDAGISel::SelectIndexedStore(StoreSDNode *ST, const SDLoc &dl) { SDValue Ops[] = { Base, IncV, Value, Chain }; MachineSDNode *S = CurDAG->getMachineNode(Opcode, dl, MVT::i32, MVT::Other, Ops); - S->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(S, {MemOp}); To[0] = SDValue(S, 0); To[1] = SDValue(S, 1); } else { SDValue Zero = CurDAG->getTargetConstant(0, dl, MVT::i32); SDValue Ops[] = { Base, Zero, Value, Chain }; MachineSDNode *S = CurDAG->getMachineNode(Opcode, dl, MVT::Other, Ops); - S->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(S, {MemOp}); To[1] = SDValue(S, 0); MachineSDNode *A = CurDAG->getMachineNode(Hexagon::A2_addi, dl, MVT::i32, Base, IncV); diff --git a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp index 530f0622e8d..988ed92a047 100644 --- a/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp +++ b/llvm/lib/Target/Hexagon/HexagonISelDAGToDAGHVX.cpp @@ -2079,9 +2079,8 @@ void HexagonDAGToDAGISel::SelectV65GatherPred(SDNode *N) { SDValue Ops[] = { Address, Predicate, Base, Modifier, Offset, Chain }; SDNode *Result = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); ReplaceNode(N, Result); } @@ -2117,9 +2116,8 @@ void HexagonDAGToDAGISel::SelectV65Gather(SDNode *N) { SDValue Ops[] = { Address, Base, Modifier, Offset, Chain }; SDNode *Result = CurDAG->getMachineNode(Opcode, dl, VTs, Ops); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemIntrinsicSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemIntrinsicSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); ReplaceNode(N, Result); } diff --git a/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp b/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp index 005f5f44a63..b196c013902 100644 --- a/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp +++ b/llvm/lib/Target/MSP430/MSP430ISelDAGToDAG.cpp @@ -362,12 +362,11 @@ bool MSP430DAGToDAGISel::tryIndexedBinOp(SDNode *Op, SDValue N1, SDValue N2, MVT VT = LD->getMemoryVT().getSimpleVT(); unsigned Opc = (VT == MVT::i16 ? Opc16 : Opc8); - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = cast<MemSDNode>(N1)->getMemOperand(); + MachineMemOperand *MemRef = cast<MemSDNode>(N1)->getMemOperand(); SDValue Ops0[] = { N2, LD->getBasePtr(), LD->getChain() }; SDNode *ResNode = CurDAG->SelectNodeTo(Op, Opc, VT, MVT::i16, MVT::Other, Ops0); - cast<MachineSDNode>(ResNode)->setMemRefs(MemRefs0, MemRefs0 + 1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(ResNode), {MemRef}); // Transfer chain. ReplaceUses(SDValue(N1.getNode(), 2), SDValue(ResNode, 2)); // Transfer writeback. diff --git a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp index 21939d836dc..ffc6a59cd6c 100644 --- a/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp +++ b/llvm/lib/Target/NVPTX/NVPTXISelDAGToDAG.cpp @@ -981,9 +981,8 @@ bool NVPTXDAGToDAGISel::tryLoad(SDNode *N) { if (!NVPTXLD) return false; - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(NVPTXLD)->setMemRefs(MemRefs0, MemRefs0 + 1); + MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(NVPTXLD), {MemRef}); ReplaceNode(N, NVPTXLD); return true; @@ -1221,9 +1220,8 @@ bool NVPTXDAGToDAGISel::tryLoadVector(SDNode *N) { LD = CurDAG->getMachineNode(Opcode.getValue(), DL, N->getVTList(), Ops); } - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); + MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(LD), {MemRef}); ReplaceNode(N, LD); return true; @@ -1659,9 +1657,8 @@ bool NVPTXDAGToDAGISel::tryLDGLDU(SDNode *N) { LD = CurDAG->getMachineNode(Opcode.getValue(), DL, InstVTList, Ops); } - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = Mem->getMemOperand(); - cast<MachineSDNode>(LD)->setMemRefs(MemRefs0, MemRefs0 + 1); + MachineMemOperand *MemRef = Mem->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(LD), {MemRef}); // For automatic generation of LDG (through SelectLoad[Vector], not the // intrinsics), we may have an extending load like: @@ -1864,9 +1861,8 @@ bool NVPTXDAGToDAGISel::tryStore(SDNode *N) { if (!NVPTXST) return false; - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(NVPTXST)->setMemRefs(MemRefs0, MemRefs0 + 1); + MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(NVPTXST), {MemRef}); ReplaceNode(N, NVPTXST); return true; } @@ -2088,9 +2084,8 @@ bool NVPTXDAGToDAGISel::tryStoreVector(SDNode *N) { ST = CurDAG->getMachineNode(Opcode.getValue(), DL, MVT::Other, StOps); - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(ST)->setMemRefs(MemRefs0, MemRefs0 + 1); + MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(ST), {MemRef}); ReplaceNode(N, ST); return true; @@ -2236,9 +2231,8 @@ bool NVPTXDAGToDAGISel::tryStoreRetval(SDNode *N) { return false; SDNode *Ret = CurDAG->getMachineNode(Opcode.getValue(), DL, MVT::Other, Ops); - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); + MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ret), {MemRef}); ReplaceNode(N, Ret); return true; @@ -2341,9 +2335,8 @@ bool NVPTXDAGToDAGISel::tryStoreParam(SDNode *N) { SDVTList RetVTs = CurDAG->getVTList(MVT::Other, MVT::Glue); SDNode *Ret = CurDAG->getMachineNode(Opcode.getValue(), DL, RetVTs, Ops); - MachineSDNode::mmo_iterator MemRefs0 = MF->allocateMemRefsArray(1); - MemRefs0[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Ret)->setMemRefs(MemRefs0, MemRefs0 + 1); + MachineMemOperand *MemRef = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Ret), {MemRef}); ReplaceNode(N, Ret); return true; diff --git a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp index 6cec664d1e6..54239c228d9 100644 --- a/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp +++ b/llvm/lib/Target/PowerPC/PPCISelDAGToDAG.cpp @@ -4045,9 +4045,8 @@ bool PPCDAGToDAGISel::isOffsetMultipleOf(SDNode *N, unsigned Val) const { void PPCDAGToDAGISel::transferMemOperands(SDNode *N, SDNode *Result) { // Transfer memoperands. - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<MemSDNode>(N)->getMemOperand(); - cast<MachineSDNode>(Result)->setMemRefs(MemOp, MemOp + 1); + MachineMemOperand *MemOp = cast<MemSDNode>(N)->getMemOperand(); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(Result), {MemOp}); } /// This method returns a node after flipping the MSB of each element @@ -4683,11 +4682,10 @@ void PPCDAGToDAGISel::Select(SDNode *N) { SelectAddrIdxOnly(LD->getBasePtr(), Base, Offset)) { SDValue Chain = LD->getChain(); SDValue Ops[] = { Base, Offset, Chain }; - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = LD->getMemOperand(); + MachineMemOperand *MemOp = LD->getMemOperand(); SDNode *NewN = CurDAG->SelectNodeTo(N, PPC::LXVDSX, N->getValueType(0), Ops); - cast<MachineSDNode>(NewN)->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(NewN), {MemOp}); return; } } diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index 5425f1d16e5..5edfdf645e5 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -1354,11 +1354,8 @@ bool SystemZDAGToDAGISel::tryFoldLoadStoreIntoMemOperand(SDNode *Node) { SDValue Ops[] = { Base, Disp, Operand, InputChain }; MachineSDNode *Result = CurDAG->getMachineNode(NewOpc, DL, MVT::i32, MVT::Other, Ops); - - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(2); - MemOp[0] = StoreNode->getMemOperand(); - MemOp[1] = LoadNode->getMemOperand(); - Result->setMemRefs(MemOp, MemOp + 2); + CurDAG->setNodeMemRefs( + Result, {StoreNode->getMemOperand(), LoadNode->getMemOperand()}); ReplaceUses(SDValue(StoreNode, 0), SDValue(Result, 1)); ReplaceUses(SDValue(StoredVal.getNode(), 1), SDValue(Result, 0)); diff --git a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp index 74d1b6740e3..dbc68baf592 100644 --- a/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp +++ b/llvm/lib/Target/X86/X86ISelDAGToDAG.cpp @@ -2549,10 +2549,9 @@ bool X86DAGToDAGISel::foldLoadStoreIntoMemOperand(SDNode *Node) { llvm_unreachable("Invalid opcode!"); } - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(2); - MemOp[0] = StoreNode->getMemOperand(); - MemOp[1] = LoadNode->getMemOperand(); - Result->setMemRefs(MemOp, MemOp + 2); + MachineMemOperand *MemOps[] = {StoreNode->getMemOperand(), + LoadNode->getMemOperand()}; + CurDAG->setNodeMemRefs(Result, MemOps); // Update Load Chain uses as well. ReplaceUses(SDValue(LoadNode, 1), SDValue(Result, 1)); @@ -2642,9 +2641,7 @@ MachineSDNode *X86DAGToDAGISel::emitPCMPISTR(unsigned ROpc, unsigned MOpc, // Update the chain. ReplaceUses(Load.getValue(1), SDValue(CNode, 2)); // Record the mem-refs - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<LoadSDNode>(Load)->getMemOperand(); - CNode->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(Load)->getMemOperand()}); return CNode; } @@ -2682,9 +2679,7 @@ MachineSDNode *X86DAGToDAGISel::emitPCMPESTR(unsigned ROpc, unsigned MOpc, // Update the chain. ReplaceUses(Load.getValue(1), SDValue(CNode, 2)); // Record the mem-refs - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<LoadSDNode>(Load)->getMemOperand(); - CNode->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(Load)->getMemOperand()}); return CNode; } @@ -3036,9 +3031,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) { // Update the chain. ReplaceUses(N1.getValue(1), Chain); // Record the mem-refs - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<LoadSDNode>(N1)->getMemOperand(); - CNode->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); } else { SDValue Ops[] = { N1, InFlag }; if (Opc == X86::MULX32rr || Opc == X86::MULX64rr) { @@ -3206,9 +3199,7 @@ void X86DAGToDAGISel::Select(SDNode *Node) { // Update the chain. ReplaceUses(N1.getValue(1), SDValue(CNode, 0)); // Record the mem-refs - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = cast<LoadSDNode>(N1)->getMemOperand(); - CNode->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(CNode, {cast<LoadSDNode>(N1)->getMemOperand()}); } else { InFlag = SDValue(CurDAG->getMachineNode(Opc, dl, MVT::Glue, N1, InFlag), 0); diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp index 289faee0733..4073f455a81 100644 --- a/llvm/lib/Target/X86/X86InstrInfo.cpp +++ b/llvm/lib/Target/X86/X86InstrInfo.cpp @@ -5358,6 +5358,54 @@ MachineInstr *X86InstrInfo::foldMemoryOperandImpl( /*Size=*/0, Alignment, /*AllowCommute=*/true); } +static SmallVector<MachineMemOperand *, 2> +extractLoadMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { + SmallVector<MachineMemOperand *, 2> LoadMMOs; + + for (MachineMemOperand *MMO : MMOs) { + if (!MMO->isLoad()) + continue; + + if (!MMO->isStore()) { + // Reuse the MMO. + LoadMMOs.push_back(MMO); + } else { + // Clone the MMO and unset the store flag. + LoadMMOs.push_back(MF.getMachineMemOperand( + MMO->getPointerInfo(), MMO->getFlags() & ~MachineMemOperand::MOStore, + MMO->getSize(), MMO->getBaseAlignment(), MMO->getAAInfo(), nullptr, + MMO->getSyncScopeID(), MMO->getOrdering(), + MMO->getFailureOrdering())); + } + } + + return LoadMMOs; +} + +static SmallVector<MachineMemOperand *, 2> +extractStoreMMOs(ArrayRef<MachineMemOperand *> MMOs, MachineFunction &MF) { + SmallVector<MachineMemOperand *, 2> StoreMMOs; + + for (MachineMemOperand *MMO : MMOs) { + if (!MMO->isStore()) + continue; + + if (!MMO->isLoad()) { + // Reuse the MMO. + StoreMMOs.push_back(MMO); + } else { + // Clone the MMO and unset the load flag. + StoreMMOs.push_back(MF.getMachineMemOperand( + MMO->getPointerInfo(), MMO->getFlags() & ~MachineMemOperand::MOLoad, + MMO->getSize(), MMO->getBaseAlignment(), MMO->getAAInfo(), nullptr, + MMO->getSyncScopeID(), MMO->getOrdering(), + MMO->getFailureOrdering())); + } + } + + return StoreMMOs; +} + bool X86InstrInfo::unfoldMemoryOperand( MachineFunction &MF, MachineInstr &MI, unsigned Reg, bool UnfoldLoad, bool UnfoldStore, SmallVectorImpl<MachineInstr *> &NewMIs) const { @@ -5516,26 +5564,21 @@ X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, SDNode *Load = nullptr; if (FoldedLoad) { EVT VT = *TRI.legalclasstypes_begin(*RC); - std::pair<MachineInstr::mmo_iterator, - MachineInstr::mmo_iterator> MMOs = - MF.extractLoadMemRefs(cast<MachineSDNode>(N)->memoperands_begin(), - cast<MachineSDNode>(N)->memoperands_end()); - if (!(*MMOs.first) && - RC == &X86::VR128RegClass && + auto MMOs = extractLoadMMOs(cast<MachineSDNode>(N)->memoperands(), MF); + if (MMOs.empty() && RC == &X86::VR128RegClass && Subtarget.isUnalignedMem16Slow()) // Do not introduce a slow unaligned load. return false; // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte // memory access is slow above. unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); - bool isAligned = (*MMOs.first) && - (*MMOs.first)->getAlignment() >= Alignment; + bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; Load = DAG.getMachineNode(getLoadRegOpcode(0, RC, isAligned, Subtarget), dl, VT, MVT::Other, AddrOps); NewNodes.push_back(Load); // Preserve memory reference information. - cast<MachineSDNode>(Load)->setMemRefs(MMOs.first, MMOs.second); + DAG.setNodeMemRefs(cast<MachineSDNode>(Load), MMOs); } // Emit the data processing instruction. @@ -5585,27 +5628,22 @@ X86InstrInfo::unfoldMemoryOperand(SelectionDAG &DAG, SDNode *N, AddrOps.pop_back(); AddrOps.push_back(SDValue(NewNode, 0)); AddrOps.push_back(Chain); - std::pair<MachineInstr::mmo_iterator, - MachineInstr::mmo_iterator> MMOs = - MF.extractStoreMemRefs(cast<MachineSDNode>(N)->memoperands_begin(), - cast<MachineSDNode>(N)->memoperands_end()); - if (!(*MMOs.first) && - RC == &X86::VR128RegClass && + auto MMOs = extractStoreMMOs(cast<MachineSDNode>(N)->memoperands(), MF); + if (MMOs.empty() && RC == &X86::VR128RegClass && Subtarget.isUnalignedMem16Slow()) // Do not introduce a slow unaligned store. return false; // FIXME: If a VR128 can have size 32, we should be checking if a 32-byte // memory access is slow above. unsigned Alignment = std::max<uint32_t>(TRI.getSpillSize(*RC), 16); - bool isAligned = (*MMOs.first) && - (*MMOs.first)->getAlignment() >= Alignment; + bool isAligned = !MMOs.empty() && MMOs.front()->getAlignment() >= Alignment; SDNode *Store = DAG.getMachineNode(getStoreRegOpcode(0, DstRC, isAligned, Subtarget), dl, MVT::Other, AddrOps); NewNodes.push_back(Store); // Preserve memory reference information. - cast<MachineSDNode>(Store)->setMemRefs(MMOs.first, MMOs.second); + DAG.setNodeMemRefs(cast<MachineSDNode>(Store), MMOs); } return true; diff --git a/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp b/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp index 1a56d1fd6e2..1688c38efc1 100644 --- a/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp +++ b/llvm/lib/Target/XCore/XCoreISelDAGToDAG.cpp @@ -150,11 +150,10 @@ void XCoreDAGToDAGISel::Select(SDNode *N) { SDNode *node = CurDAG->getMachineNode(XCore::LDWCP_lru6, dl, MVT::i32, MVT::Other, CPIdx, CurDAG->getEntryNode()); - MachineSDNode::mmo_iterator MemOp = MF->allocateMemRefsArray(1); - MemOp[0] = + MachineMemOperand *MemOp = MF->getMachineMemOperand(MachinePointerInfo::getConstantPool(*MF), MachineMemOperand::MOLoad, 4, 4); - cast<MachineSDNode>(node)->setMemRefs(MemOp, MemOp + 1); + CurDAG->setNodeMemRefs(cast<MachineSDNode>(node), {MemOp}); ReplaceNode(N, node); return; } |
