diff options
| -rw-r--r-- | llvm/include/llvm/CodeGen/SelectionDAG.h | 31 | ||||
| -rw-r--r-- | llvm/include/llvm/CodeGen/SelectionDAGNodes.h | 313 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 261 |
3 files changed, 392 insertions, 213 deletions
diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 941de90ff4b..9a64444d8e7 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -23,7 +23,6 @@ #include "llvm/CodeGen/DAGCombine.h" #include "llvm/CodeGen/MachineFunction.h" #include "llvm/CodeGen/SelectionDAGNodes.h" -#include "llvm/Support/ArrayRecycler.h" #include "llvm/Support/RecyclingAllocator.h" #include "llvm/Target/TargetMachine.h" #include <cassert> @@ -209,7 +208,6 @@ class SelectionDAG { /// Pool allocation for machine-opcode SDNode operands. BumpPtrAllocator OperandAllocator; - ArrayRecycler<SDUse> OperandRecycler; /// Pool allocation for misc. objects that are created once per SelectionDAG. BumpPtrAllocator Allocator; @@ -276,30 +274,6 @@ private: SDNodeT(std::forward<ArgTypes>(Args)...); } - void createOperands(SDNode *Node, ArrayRef<SDValue> Vals) { - assert(!Node->OperandList && "Node already has operands"); - SDUse *Ops = OperandRecycler.allocate( - ArrayRecycler<SDUse>::Capacity::get(Vals.size()), OperandAllocator); - - for (unsigned I = 0; I != Vals.size(); ++I) { - Ops[I].setUser(Node); - Ops[I].setInitial(Vals[I]); - } - Node->NumOperands = Vals.size(); - Node->OperandList = Ops; - checkForCycles(Node); - } - - void removeOperands(SDNode *Node) { - if (!Node->OperandList) - return; - OperandRecycler.deallocate( - ArrayRecycler<SDUse>::Capacity::get(Node->NumOperands), - Node->OperandList); - Node->NumOperands = 0; - Node->OperandList = nullptr; - } - void operator=(const SelectionDAG&) = delete; SelectionDAG(const SelectionDAG&) = delete; @@ -1342,8 +1316,9 @@ private: void allnodes_clear(); - SDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, SDValue N1, - SDValue N2, const SDNodeFlags *Flags = nullptr); + BinarySDNode *GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, + SDValue N1, SDValue N2, + const SDNodeFlags *Flags = nullptr); /// Look up the node specified by ID in CSEMap. If it exists, return it. If /// not, return the insertion token that will make insertion faster. This diff --git a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h index c4482769ca2..375e238261d 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGNodes.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGNodes.h @@ -44,7 +44,7 @@ class GlobalValue; class MachineBasicBlock; class MachineConstantPoolValue; class SDNode; -class HandleSDNode; +class BinaryWithFlagsSDNode; class Value; class MCSymbol; template <typename T> struct DenseMapInfo; @@ -280,8 +280,6 @@ public: private: friend class SelectionDAG; friend class SDNode; - // TODO: unfriend HandleSDNode once we fix its operand handling. - friend class HandleSDNode; void setUser(SDNode *p) { User = p; } @@ -396,6 +394,10 @@ private: /// The operation that this node performs. int16_t NodeType; + /// This is true if OperandList was new[]'d. If true, + /// then they will be delete[]'d when the node is destroyed. + uint16_t OperandsNeedDelete : 1; + /// This tracks whether this node has one or more dbg_value /// nodes corresponding to it. uint16_t HasDebugValue : 1; @@ -404,7 +406,7 @@ protected: /// This member is defined by this class, but is not used for /// anything. Subclasses can use it to hold whatever state they find useful. /// This field is initialized to zero by the ctor. - uint16_t SubclassData : 15; + uint16_t SubclassData : 14; private: /// Unique id per SDNode in the DAG. @@ -438,8 +440,6 @@ private: friend class SelectionDAG; friend struct ilist_traits<SDNode>; - // TODO: unfriend HandleSDNode once we fix its operand handling. - friend class HandleSDNode; public: /// Unique and persistent id per SDNode in the DAG. @@ -792,20 +792,101 @@ protected: return Ret; } - /// Create an SDNode. - /// - /// SDNodes are created without any operands, and never own the operand - /// storage. To add operands, see SelectionDAG::createOperands. + SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + ArrayRef<SDValue> Ops) + : NodeType(Opc), OperandsNeedDelete(true), HasDebugValue(false), + SubclassData(0), NodeId(-1), + OperandList(Ops.size() ? new SDUse[Ops.size()] : nullptr), + ValueList(VTs.VTs), UseList(nullptr), NumOperands(Ops.size()), + NumValues(VTs.NumVTs), IROrder(Order), debugLoc(std::move(dl)) { + assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); + assert(NumOperands == Ops.size() && + "NumOperands wasn't wide enough for its operands!"); + assert(NumValues == VTs.NumVTs && + "NumValues wasn't wide enough for its operands!"); + for (unsigned i = 0; i != Ops.size(); ++i) { + assert(OperandList && "no operands available"); + OperandList[i].setUser(this); + OperandList[i].setInitial(Ops[i]); + } + checkForCycles(this); + } + + /// This constructor adds no operands itself; operands can be + /// set later with InitOperands. SDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs) - : NodeType(Opc), HasDebugValue(false), SubclassData(0), NodeId(-1), - OperandList(nullptr), ValueList(VTs.VTs), UseList(nullptr), - NumOperands(0), NumValues(VTs.NumVTs), IROrder(Order), - debugLoc(std::move(dl)) { + : NodeType(Opc), OperandsNeedDelete(false), HasDebugValue(false), + SubclassData(0), NodeId(-1), OperandList(nullptr), ValueList(VTs.VTs), + UseList(nullptr), NumOperands(0), NumValues(VTs.NumVTs), + IROrder(Order), debugLoc(std::move(dl)) { assert(debugLoc.hasTrivialDestructor() && "Expected trivial destructor"); assert(NumValues == VTs.NumVTs && "NumValues wasn't wide enough for its operands!"); } + /// Initialize the operands list of this with 1 operand. + void InitOperands(SDUse *Ops, const SDValue &Op0) { + Ops[0].setUser(this); + Ops[0].setInitial(Op0); + NumOperands = 1; + OperandList = Ops; + checkForCycles(this); + } + + /// Initialize the operands list of this with 2 operands. + void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1) { + Ops[0].setUser(this); + Ops[0].setInitial(Op0); + Ops[1].setUser(this); + Ops[1].setInitial(Op1); + NumOperands = 2; + OperandList = Ops; + checkForCycles(this); + } + + /// Initialize the operands list of this with 3 operands. + void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, + const SDValue &Op2) { + Ops[0].setUser(this); + Ops[0].setInitial(Op0); + Ops[1].setUser(this); + Ops[1].setInitial(Op1); + Ops[2].setUser(this); + Ops[2].setInitial(Op2); + NumOperands = 3; + OperandList = Ops; + checkForCycles(this); + } + + /// Initialize the operands list of this with 4 operands. + void InitOperands(SDUse *Ops, const SDValue &Op0, const SDValue &Op1, + const SDValue &Op2, const SDValue &Op3) { + Ops[0].setUser(this); + Ops[0].setInitial(Op0); + Ops[1].setUser(this); + Ops[1].setInitial(Op1); + Ops[2].setUser(this); + Ops[2].setInitial(Op2); + Ops[3].setUser(this); + Ops[3].setInitial(Op3); + NumOperands = 4; + OperandList = Ops; + checkForCycles(this); + } + + /// Initialize the operands list of this with N operands. + void InitOperands(SDUse *Ops, const SDValue *Vals, unsigned N) { + for (unsigned i = 0; i != N; ++i) { + Ops[i].setUser(this); + Ops[i].setInitial(Vals[i]); + } + NumOperands = N; + assert(NumOperands == N && + "NumOperands wasn't wide enough for its operands!"); + OperandList = Ops; + checkForCycles(this); + } + /// Release the operands and set this node to have zero operands. void DropOperands(); }; @@ -931,6 +1012,30 @@ inline void SDUse::setNode(SDNode *N) { if (N) N->addUse(*this); } +/// This class is used for single-operand SDNodes. This is solely +/// to allow co-allocation of node operands with the node itself. +class UnarySDNode : public SDNode { + SDUse Op; +public: + UnarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + SDValue X) + : SDNode(Opc, Order, dl, VTs) { + InitOperands(&Op, X); + } +}; + +/// This class is used for two-operand SDNodes. This is solely +/// to allow co-allocation of node operands with the node itself. +class BinarySDNode : public SDNode { + SDUse Ops[2]; +public: + BinarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + SDValue X, SDValue Y) + : SDNode(Opc, Order, dl, VTs) { + InitOperands(Ops, X, Y); + } +}; + /// Returns true if the opcode is a binary operation with flags. static bool isBinOpWithFlags(unsigned Opcode) { switch (Opcode) { @@ -955,17 +1060,30 @@ static bool isBinOpWithFlags(unsigned Opcode) { /// This class is an extension of BinarySDNode /// used from those opcodes that have associated extra flags. -class BinaryWithFlagsSDNode : public SDNode { +class BinaryWithFlagsSDNode : public BinarySDNode { public: SDNodeFlags Flags; BinaryWithFlagsSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - const SDNodeFlags &NodeFlags) - : SDNode(Opc, Order, dl, VTs), Flags(NodeFlags) {} + SDValue X, SDValue Y, const SDNodeFlags &NodeFlags) + : BinarySDNode(Opc, Order, dl, VTs, X, Y), Flags(NodeFlags) {} static bool classof(const SDNode *N) { return isBinOpWithFlags(N->getOpcode()); } }; +/// This class is used for three-operand SDNodes. This is solely +/// to allow co-allocation of node operands with the node itself. +class TernarySDNode : public SDNode { + SDUse Ops[3]; +public: + TernarySDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + SDValue X, SDValue Y, SDValue Z) + : SDNode(Opc, Order, dl, VTs) { + InitOperands(Ops, X, Y, Z); + } +}; + + /// This class is used to form a handle around another node that /// is persistent and is updated across invocations of replaceAllUsesWith on its /// operand. This node should be directly created by end-users and not added to @@ -978,28 +1096,20 @@ public: // HandleSDNodes are never inserted into the DAG, so they won't be // auto-numbered. Use ID 65535 as a sentinel. PersistentId = 0xffff; - - // Manually set up the operand list. This node type is special in that it's - // always stack allocated and SelectionDAG does not manage its operands. - // TODO: This should either (a) not be in the SDNode hierarchy, or (b) not - // be so special. - Op.setUser(this); - Op.setInitial(X); - NumOperands = 1; - OperandList = &Op; + InitOperands(&Op, X); } ~HandleSDNode(); const SDValue &getValue() const { return Op; } }; -class AddrSpaceCastSDNode : public SDNode { +class AddrSpaceCastSDNode : public UnarySDNode { private: unsigned SrcAddrSpace; unsigned DestAddrSpace; public: - AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, unsigned SrcAS, - unsigned DestAS); + AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, SDValue X, + unsigned SrcAS, unsigned DestAS); unsigned getSrcAddressSpace() const { return SrcAddrSpace; } unsigned getDestAddressSpace() const { return DestAddrSpace; } @@ -1023,6 +1133,9 @@ public: MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemoryVT, MachineMemOperand *MMO); + MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + ArrayRef<SDValue> Ops, EVT MemoryVT, MachineMemOperand *MMO); + bool readMem() const { return MMO->isLoad(); } bool writeMem() const { return MMO->isStore(); } @@ -1125,6 +1238,8 @@ public: /// This is an SDNode representing atomic operations. class AtomicSDNode : public MemSDNode { + SDUse Ops[4]; + /// For cmpxchg instructions, the ordering requirements when a store does not /// occur. AtomicOrdering FailureOrdering; @@ -1150,12 +1265,16 @@ class AtomicSDNode : public MemSDNode { } public: - AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, - EVT MemVT, MachineMemOperand *MMO, + AtomicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTL, EVT MemVT, + const SDValue* AllOps, SDUse *DynOps, unsigned NumOps, + MachineMemOperand *MMO, AtomicOrdering SuccessOrdering, AtomicOrdering FailureOrdering, SynchronizationScope SynchScope) - : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { + : MemSDNode(Opc, Order, dl, VTL, MemVT, MMO) { InitAtomic(SuccessOrdering, FailureOrdering, SynchScope); + assert((DynOps || NumOps <= array_lengthof(Ops)) && + "Too many ops for internal storage!"); + InitOperands(DynOps ? DynOps : Ops, AllOps, NumOps); } const SDValue &getBasePtr() const { return getOperand(1); } @@ -1203,8 +1322,9 @@ public: class MemIntrinsicSDNode : public MemSDNode { public: MemIntrinsicSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, - EVT MemoryVT, MachineMemOperand *MMO) - : MemSDNode(Opc, Order, dl, VTs, MemoryVT, MMO) { + ArrayRef<SDValue> Ops, EVT MemoryVT, + MachineMemOperand *MMO) + : MemSDNode(Opc, Order, dl, VTs, Ops, MemoryVT, MMO) { SubclassData |= 1u << 13; } @@ -1227,15 +1347,20 @@ public: /// An index of -1 is treated as undef, such that the code generator may put /// any value in the corresponding element of the result. class ShuffleVectorSDNode : public SDNode { + SDUse Ops[2]; + // The memory for Mask is owned by the SelectionDAG's OperandAllocator, and // is freed when the SelectionDAG object is destroyed. const int *Mask; protected: friend class SelectionDAG; - ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, const int *M) - : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) {} - + ShuffleVectorSDNode(EVT VT, unsigned Order, DebugLoc dl, SDValue N1, + SDValue N2, const int *M) + : SDNode(ISD::VECTOR_SHUFFLE, Order, dl, getSDVTList(VT)), Mask(M) { + InitOperands(Ops, N1, N2); + } public: + ArrayRef<int> getMask() const { EVT VT = getValueType(0); return makeArrayRef(Mask, VT.getVectorNumElements()); @@ -1666,11 +1791,13 @@ public: }; class EHLabelSDNode : public SDNode { + SDUse Chain; MCSymbol *Label; friend class SelectionDAG; - EHLabelSDNode(unsigned Order, DebugLoc dl, MCSymbol *L) - : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) {} - + EHLabelSDNode(unsigned Order, DebugLoc dl, SDValue ch, MCSymbol *L) + : SDNode(ISD::EH_LABEL, Order, dl, getSDVTList(MVT::Other)), Label(L) { + InitOperands(&Chain, ch); + } public: MCSymbol *getLabel() const { return Label; } @@ -1736,10 +1863,11 @@ class CvtRndSatSDNode : public SDNode { ISD::CvtCode CvtCode; friend class SelectionDAG; explicit CvtRndSatSDNode(EVT VT, unsigned Order, DebugLoc dl, - ISD::CvtCode Code) - : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT)), CvtCode(Code) { + ArrayRef<SDValue> Ops, ISD::CvtCode Code) + : SDNode(ISD::CONVERT_RNDSAT, Order, dl, getSDVTList(VT), Ops), + CvtCode(Code) { + assert(Ops.size() == 5 && "wrong number of operations"); } - public: ISD::CvtCode getCvtCode() const { return CvtCode; } @@ -1768,13 +1896,24 @@ public: /// Base class for LoadSDNode and StoreSDNode class LSBaseSDNode : public MemSDNode { + //! Operand array for load and store + /*! + \note Moving this array to the base class captures more + common functionality shared between LoadSDNode and + StoreSDNode + */ + SDUse Ops[4]; public: LSBaseSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, + SDValue *Operands, unsigned numOperands, SDVTList VTs, ISD::MemIndexedMode AM, EVT MemVT, MachineMemOperand *MMO) : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { SubclassData |= AM << 2; assert(getAddressingMode() == AM && "MemIndexedMode encoding error!"); + InitOperands(Ops, Operands, numOperands); + assert((getOffset().getOpcode() == ISD::UNDEF || isIndexed()) && + "Only indexed loads and stores have a non-undef offset operand"); } const SDValue &getOffset() const { @@ -1802,9 +1941,10 @@ public: /// This class is used to represent ISD::LOAD nodes. class LoadSDNode : public LSBaseSDNode { friend class SelectionDAG; - LoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, - ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) - : LSBaseSDNode(ISD::LOAD, Order, dl, VTs, AM, MemVT, MMO) { + LoadSDNode(SDValue *ChainPtrOff, unsigned Order, DebugLoc dl, SDVTList VTs, + ISD::MemIndexedMode AM, ISD::LoadExtType ETy, EVT MemVT, + MachineMemOperand *MMO) + : LSBaseSDNode(ISD::LOAD, Order, dl, ChainPtrOff, 3, VTs, AM, MemVT, MMO) { SubclassData |= (unsigned short)ETy; assert(getExtensionType() == ETy && "LoadExtType encoding error!"); assert(readMem() && "Load MachineMemOperand is not a load!"); @@ -1829,9 +1969,11 @@ public: /// This class is used to represent ISD::STORE nodes. class StoreSDNode : public LSBaseSDNode { friend class SelectionDAG; - StoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, ISD::MemIndexedMode AM, - bool isTrunc, EVT MemVT, MachineMemOperand *MMO) - : LSBaseSDNode(ISD::STORE, Order, dl, VTs, AM, MemVT, MMO) { + StoreSDNode(SDValue *ChainValuePtrOff, unsigned Order, DebugLoc dl, + SDVTList VTs, ISD::MemIndexedMode AM, bool isTrunc, EVT MemVT, + MachineMemOperand *MMO) + : LSBaseSDNode(ISD::STORE, Order, dl, ChainValuePtrOff, 4, + VTs, AM, MemVT, MMO) { SubclassData |= (unsigned short)isTrunc; assert(isTruncatingStore() == isTrunc && "isTrunc encoding error!"); assert(!readMem() && "Store MachineMemOperand is a load!"); @@ -1855,11 +1997,16 @@ public: /// This base class is used to represent MLOAD and MSTORE nodes class MaskedLoadStoreSDNode : public MemSDNode { + // Operands + SDUse Ops[4]; public: friend class SelectionDAG; MaskedLoadStoreSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, - SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} + SDValue *Operands, unsigned numOperands, SDVTList VTs, + EVT MemVT, MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + InitOperands(Ops, Operands, numOperands); + } // In the both nodes address is Op1, mask is Op2: // MaskedLoadSDNode (Chain, ptr, mask, src0), src0 is a passthru value @@ -1878,9 +2025,11 @@ public: class MaskedLoadSDNode : public MaskedLoadStoreSDNode { public: friend class SelectionDAG; - MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, - ISD::LoadExtType ETy, EVT MemVT, MachineMemOperand *MMO) - : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, VTs, MemVT, MMO) { + MaskedLoadSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, + unsigned numOperands, SDVTList VTs, ISD::LoadExtType ETy, + EVT MemVT, MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MLOAD, Order, dl, Operands, numOperands, + VTs, MemVT, MMO) { SubclassData |= (unsigned short)ETy; } @@ -1898,10 +2047,12 @@ class MaskedStoreSDNode : public MaskedLoadStoreSDNode { public: friend class SelectionDAG; - MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, bool isTrunc, - EVT MemVT, MachineMemOperand *MMO) - : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, VTs, MemVT, MMO) { - SubclassData |= (unsigned short)isTrunc; + MaskedStoreSDNode(unsigned Order, DebugLoc dl, SDValue *Operands, + unsigned numOperands, SDVTList VTs, bool isTrunc, EVT MemVT, + MachineMemOperand *MMO) + : MaskedLoadStoreSDNode(ISD::MSTORE, Order, dl, Operands, numOperands, + VTs, MemVT, MMO) { + SubclassData |= (unsigned short)isTrunc; } /// Return true if the op does a truncation before store. /// For integers this is the same as doing a TRUNCATE and storing the result. @@ -1919,11 +2070,17 @@ public: /// MGATHER and MSCATTER nodes /// class MaskedGatherScatterSDNode : public MemSDNode { + // Operands + SDUse Ops[5]; public: friend class SelectionDAG; MaskedGatherScatterSDNode(ISD::NodeType NodeTy, unsigned Order, DebugLoc dl, - SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) - : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) {} + ArrayRef<SDValue> Operands, SDVTList VTs, EVT MemVT, + MachineMemOperand *MMO) + : MemSDNode(NodeTy, Order, dl, VTs, MemVT, MMO) { + assert(Operands.size() == 5 && "Incompatible number of operands"); + InitOperands(Ops, Operands.data(), Operands.size()); + } // In the both nodes address is Op1, mask is Op2: // MaskedGatherSDNode (Chain, src0, mask, base, index), src0 is a passthru value @@ -1945,9 +2102,19 @@ public: class MaskedGatherSDNode : public MaskedGatherScatterSDNode { public: friend class SelectionDAG; - MaskedGatherSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT, - MachineMemOperand *MMO) - : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, VTs, MemVT, MMO) {} + MaskedGatherSDNode(unsigned Order, DebugLoc dl, ArrayRef<SDValue> Operands, + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MGATHER, Order, dl, Operands, VTs, MemVT, + MMO) { + assert(getValue().getValueType() == getValueType(0) && + "Incompatible type of the PassThru value in MaskedGatherSDNode"); + assert(getMask().getValueType().getVectorNumElements() == + getValueType(0).getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(getIndex().getValueType().getVectorNumElements() == + getValueType(0).getVectorNumElements() && + "Vector width mismatch between index and data"); + } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MGATHER; @@ -1960,9 +2127,17 @@ class MaskedScatterSDNode : public MaskedGatherScatterSDNode { public: friend class SelectionDAG; - MaskedScatterSDNode(unsigned Order, DebugLoc dl, SDVTList VTs, EVT MemVT, - MachineMemOperand *MMO) - : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, VTs, MemVT, MMO) {} + MaskedScatterSDNode(unsigned Order, DebugLoc dl,ArrayRef<SDValue> Operands, + SDVTList VTs, EVT MemVT, MachineMemOperand *MMO) + : MaskedGatherScatterSDNode(ISD::MSCATTER, Order, dl, Operands, VTs, MemVT, + MMO) { + assert(getMask().getValueType().getVectorNumElements() == + getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between mask and data"); + assert(getIndex().getValueType().getVectorNumElements() == + getValue().getValueType().getVectorNumElements() && + "Vector width mismatch between index and data"); + } static bool classof(const SDNode *N) { return N->getOpcode() == ISD::MSCATTER; @@ -1981,6 +2156,10 @@ private: MachineSDNode(unsigned Opc, unsigned Order, const DebugLoc DL, SDVTList VTs) : SDNode(Opc, Order, DL, VTs), MemRefs(nullptr), MemRefsEnd(nullptr) {} + /// Operands for this instruction, if they fit here. If + /// they don't, this field is unused. + SDUse LocalOperands[4]; + /// Memory reference descriptions for this instruction. mmo_iterator MemRefs; mmo_iterator MemRefsEnd; @@ -2056,7 +2235,7 @@ template <> struct GraphTraits<SDNode*> { }; /// The largest SDNode class. -typedef AtomicSDNode LargestSDNode; +typedef MaskedGatherScatterSDNode LargestSDNode; /// The SDNode class with the greatest alignment requirement. typedef GlobalAddressSDNode MostAlignedSDNode; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 592f5152558..03179b7c3d3 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -664,8 +664,8 @@ void SDDbgInfo::erase(const SDNode *Node) { } void SelectionDAG::DeallocateNode(SDNode *N) { - // If we have operands, deallocate them. - removeOperands(N); + if (N->OperandsNeedDelete) + delete[] N->OperandList; // Set the opcode to DELETED_NODE to help catch bugs when node // memory is reallocated. @@ -912,7 +912,6 @@ void SelectionDAG::init(MachineFunction &mf) { SelectionDAG::~SelectionDAG() { assert(!UpdateListeners && "Dangling registered DAGUpdateListeners"); allnodes_clear(); - OperandRecycler.clear(OperandAllocator); delete DbgInfo; } @@ -926,26 +925,24 @@ void SelectionDAG::allnodes_clear() { #endif } -SDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - SDValue N1, SDValue N2, - const SDNodeFlags *Flags) { - SDValue Ops[] = {N1, N2}; - +BinarySDNode *SelectionDAG::GetBinarySDNode(unsigned Opcode, SDLoc DL, + SDVTList VTs, SDValue N1, + SDValue N2, + const SDNodeFlags *Flags) { if (isBinOpWithFlags(Opcode)) { // If no flags were passed in, use a default flags object. SDNodeFlags F; if (Flags == nullptr) Flags = &F; - auto *FN = newSDNode<BinaryWithFlagsSDNode>(Opcode, DL.getIROrder(), - DL.getDebugLoc(), VTs, *Flags); - createOperands(FN, Ops); + auto *FN = newSDNode<BinaryWithFlagsSDNode>( + Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, N1, N2, *Flags); return FN; } - auto *N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); + auto *N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), + VTs, N1, N2); return N; } @@ -985,7 +982,6 @@ SDNode *SelectionDAG::FindNodeOrInsertPos(const FoldingSetNodeID &ID, void SelectionDAG::clear() { allnodes_clear(); - OperandRecycler.clear(OperandAllocator); OperandAllocator.Reset(); CSEMap.clear(); @@ -1642,9 +1638,7 @@ SDValue SelectionDAG::getVectorShuffle(EVT VT, SDLoc dl, SDValue N1, memcpy(MaskAlloc, &MaskVec[0], NElts * sizeof(int)); auto *N = newSDNode<ShuffleVectorSDNode>(VT, dl.getIROrder(), - dl.getDebugLoc(), MaskAlloc); - createOperands(N, Ops); - + dl.getDebugLoc(), N1, N2, MaskAlloc); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -1677,10 +1671,8 @@ SDValue SelectionDAG::getConvertRndSat(EVT VT, SDLoc dl, if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) return SDValue(E, 0); - auto *N = - newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(), Code); - createOperands(N, Ops); - + auto *N = newSDNode<CvtRndSatSDNode>(VT, dl.getIROrder(), dl.getDebugLoc(), + Ops, Code); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -1723,9 +1715,8 @@ SDValue SelectionDAG::getEHLabel(SDLoc dl, SDValue Root, MCSymbol *Label) { if (SDNode *E = FindNodeOrInsertPos(ID, IP)) return SDValue(E, 0); - auto *N = newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Label); - createOperands(N, Ops); - + auto *N = + newSDNode<EHLabelSDNode>(dl.getIROrder(), dl.getDebugLoc(), Root, Label); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -1808,9 +1799,7 @@ SDValue SelectionDAG::getAddrSpaceCast(SDLoc dl, EVT VT, SDValue Ptr, return SDValue(E, 0); auto *N = newSDNode<AddrSpaceCastSDNode>(dl.getIROrder(), dl.getDebugLoc(), - VT, SrcAS, DestAS); - createOperands(N, Ops); - + VT, Ptr, SrcAS, DestAS); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -3194,20 +3183,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDNode *N; SDVTList VTs = getVTList(VT); - SDValue Ops[] = {Operand}; if (VT != MVT::Glue) { // Don't CSE flag producing nodes FoldingSetNodeID ID; + SDValue Ops[1] = { Operand }; AddNodeIDNode(ID, Opcode, VTs, Ops); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); + N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, + Operand); CSEMap.InsertNode(N, IP); } else { - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); + N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, + Operand); } InsertNode(N); @@ -3906,7 +3895,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, } // Memoize this node if possible. - SDNode *N; + BinarySDNode *N; SDVTList VTs = getVTList(VT); if (VT != MVT::Glue) { SDValue Ops[] = {N1, N2}; @@ -3920,6 +3909,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, SDValue N1, } N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); + CSEMap.InsertNode(N, IP); } else { N = GetBinarySDNode(Opcode, DL, VTs, N1, N2, Flags); @@ -4010,20 +4000,20 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, // Memoize node if it doesn't produce a flag. SDNode *N; SDVTList VTs = getVTList(VT); - SDValue Ops[] = {N1, N2, N3}; if (VT != MVT::Glue) { + SDValue Ops[] = { N1, N2, N3 }; FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTs, Ops); void *IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); + N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, + N1, N2, N3); CSEMap.InsertNode(N, IP); } else { - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); + N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, + N1, N2, N3); } InsertNode(N); @@ -4815,11 +4805,18 @@ SDValue SelectionDAG::getAtomic(unsigned Opcode, SDLoc dl, EVT MemVT, return SDValue(E, 0); } - auto *N = newSDNode<AtomicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), - VTList, MemVT, MMO, SuccessOrdering, - FailureOrdering, SynchScope); - createOperands(N, Ops); + // Allocate the operands array for the node out of the BumpPtrAllocator, since + // SDNode doesn't have access to it. This memory will be "leaked" when + // the node is deallocated, but recovered when the allocator is released. + // If the number of operands is less than 5 we use AtomicSDNode's internal + // storage. + unsigned NumOps = Ops.size(); + SDUse *DynOps = NumOps > 4 ? OperandAllocator.Allocate<SDUse>(NumOps) + : nullptr; + auto *N = newSDNode<AtomicSDNode>( + Opcode, dl.getIROrder(), dl.getDebugLoc(), VTList, MemVT, Ops.data(), + DynOps, NumOps, MMO, SuccessOrdering, FailureOrdering, SynchScope); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5012,14 +5009,11 @@ SelectionDAG::getMemIntrinsicNode(unsigned Opcode, SDLoc dl, SDVTList VTList, } N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), - VTList, MemVT, MMO); - createOperands(N, Ops); - - CSEMap.InsertNode(N, IP); + VTList, Ops, MemVT, MMO); + CSEMap.InsertNode(N, IP); } else { N = newSDNode<MemIntrinsicSDNode>(Opcode, dl.getIROrder(), dl.getDebugLoc(), - VTList, MemVT, MMO); - createOperands(N, Ops); + VTList, Ops, MemVT, MMO); } InsertNode(N); return SDValue(N, 0); @@ -5137,10 +5131,8 @@ SelectionDAG::getLoad(ISD::MemIndexedMode AM, ISD::LoadExtType ExtType, cast<LoadSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode<LoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, - ExtType, MemVT, MMO); - createOperands(N, Ops); - + auto *N = newSDNode<LoadSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, + AM, ExtType, MemVT, MMO); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5246,10 +5238,8 @@ SDValue SelectionDAG::getStore(SDValue Chain, SDLoc dl, SDValue Val, cast<StoreSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, + auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED, false, VT, MMO); - createOperands(N, Ops); - CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5316,10 +5306,8 @@ SDValue SelectionDAG::getTruncStore(SDValue Chain, SDLoc dl, SDValue Val, cast<StoreSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, + auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, ISD::UNINDEXED, true, SVT, MMO); - createOperands(N, Ops); - CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5342,11 +5330,9 @@ SelectionDAG::getIndexedStore(SDValue OrigStore, SDLoc dl, SDValue Base, if (SDNode *E = FindNodeOrInsertPos(ID, dl.getDebugLoc(), IP)) return SDValue(E, 0); - auto *N = newSDNode<StoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, AM, - ST->isTruncatingStore(), ST->getMemoryVT(), - ST->getMemOperand()); - createOperands(N, Ops); - + auto *N = newSDNode<StoreSDNode>(Ops, dl.getIROrder(), dl.getDebugLoc(), VTs, + AM, ST->isTruncatingStore(), + ST->getMemoryVT(), ST->getMemOperand()); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5372,10 +5358,8 @@ SelectionDAG::getMaskedLoad(EVT VT, SDLoc dl, SDValue Chain, cast<MaskedLoadSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, - ExtTy, MemVT, MMO); - createOperands(N, Ops); - + auto *N = newSDNode<MaskedLoadSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops, + 4, VTs, ExtTy, MemVT, MMO); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5400,10 +5384,8 @@ SDValue SelectionDAG::getMaskedStore(SDValue Chain, SDLoc dl, SDValue Val, cast<MaskedStoreSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), VTs, - isTrunc, MemVT, MMO); - createOperands(N, Ops); - + auto *N = newSDNode<MaskedStoreSDNode>(dl.getIROrder(), dl.getDebugLoc(), Ops, + 4, VTs, isTrunc, MemVT, MMO); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5413,7 +5395,6 @@ SDValue SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO) { - assert(Ops.size() == 5 && "Incompatible number of operands"); FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MGATHER, VTs, Ops); @@ -5428,20 +5409,8 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, cast<MaskedGatherSDNode>(E)->refineAlignment(MMO); return SDValue(E, 0); } - auto *N = newSDNode<MaskedGatherSDNode>(dl.getIROrder(), dl.getDebugLoc(), - VTs, VT, MMO); - createOperands(N, Ops); - - assert(N->getValue().getValueType() == N->getValueType(0) && - "Incompatible type of the PassThru value in MaskedGatherSDNode"); - assert(N->getMask().getValueType().getVectorNumElements() == - N->getValueType(0).getVectorNumElements() && - "Vector width mismatch between mask and data"); - assert(N->getIndex().getValueType().getVectorNumElements() == - N->getValueType(0).getVectorNumElements() && - "Vector width mismatch between index and data"); - + Ops, VTs, VT, MMO); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5450,8 +5419,6 @@ SelectionDAG::getMaskedGather(SDVTList VTs, EVT VT, SDLoc dl, SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, ArrayRef<SDValue> Ops, MachineMemOperand *MMO) { - assert(Ops.size() == 5 && "Incompatible number of operands"); - FoldingSetNodeID ID; AddNodeIDNode(ID, ISD::MSCATTER, VTs, Ops); ID.AddInteger(VT.getRawBits()); @@ -5465,16 +5432,7 @@ SDValue SelectionDAG::getMaskedScatter(SDVTList VTs, EVT VT, SDLoc dl, return SDValue(E, 0); } auto *N = newSDNode<MaskedScatterSDNode>(dl.getIROrder(), dl.getDebugLoc(), - VTs, VT, MMO); - createOperands(N, Ops); - - assert(N->getMask().getValueType().getVectorNumElements() == - N->getValue().getValueType().getVectorNumElements() && - "Vector width mismatch between mask and data"); - assert(N->getIndex().getValueType().getVectorNumElements() == - N->getValue().getValueType().getVectorNumElements() && - "Vector width mismatch between index and data"); - + Ops, VTs, VT, MMO); CSEMap.InsertNode(N, IP); InsertNode(N); return SDValue(N, 0); @@ -5553,13 +5511,10 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, EVT VT, if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); - + N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops); CSEMap.InsertNode(N, IP); } else { - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); + N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs, Ops); } InsertNode(N); @@ -5601,6 +5556,7 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, // Memoize the node unless it returns a flag. SDNode *N; + unsigned NumOps = Ops.size(); if (VTList.VTs[VTList.NumVTs-1] != MVT::Glue) { FoldingSetNodeID ID; AddNodeIDNode(ID, Opcode, VTList, Ops); @@ -5608,12 +5564,34 @@ SDValue SelectionDAG::getNode(unsigned Opcode, SDLoc DL, SDVTList VTList, if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) return SDValue(E, 0); - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); - createOperands(N, Ops); + if (NumOps == 1) { + N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), + VTList, Ops[0]); + } else if (NumOps == 2) { + N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), + VTList, Ops[0], Ops[1]); + } else if (NumOps == 3) { + N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), + VTList, Ops[0], Ops[1], Ops[2]); + } else { + N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, + Ops); + } CSEMap.InsertNode(N, IP); } else { - N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList); - createOperands(N, Ops); + if (NumOps == 1) { + N = newSDNode<UnarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), + VTList, Ops[0]); + } else if (NumOps == 2) { + N = newSDNode<BinarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), + VTList, Ops[0], Ops[1]); + } else if (NumOps == 3) { + N = newSDNode<TernarySDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), + VTList, Ops[0], Ops[1], Ops[2]); + } else { + N = newSDNode<SDNode>(Opcode, DL.getIROrder(), DL.getDebugLoc(), VTList, + Ops); + } } InsertNode(N); return SDValue(N, 0); @@ -5999,6 +5977,7 @@ SDNode *SelectionDAG::UpdadeSDLocOnMergedSDNode(SDNode *N, SDLoc OLoc) { /// deleting things. SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, SDVTList VTs, ArrayRef<SDValue> Ops) { + unsigned NumOps = Ops.size(); // If an identical node already exists, use it. void *IP = nullptr; if (VTs.VTs[VTs.NumVTs-1] != MVT::Glue) { @@ -6027,13 +6006,36 @@ SDNode *SelectionDAG::MorphNodeTo(SDNode *N, unsigned Opc, DeadNodeSet.insert(Used); } - // For MachineNode, initialize the memory references information. - if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) + if (MachineSDNode *MN = dyn_cast<MachineSDNode>(N)) { + // Initialize the memory references information. MN->setMemRefs(nullptr, nullptr); - - // Swap for an appropriately sized array from the recycler. - removeOperands(N); - createOperands(N, Ops); + // If NumOps is larger than the # of operands we can have in a + // MachineSDNode, reallocate the operand list. + if (NumOps > MN->NumOperands || !MN->OperandsNeedDelete) { + if (MN->OperandsNeedDelete) + delete[] MN->OperandList; + if (NumOps > array_lengthof(MN->LocalOperands)) + // We're creating a final node that will live unmorphed for the + // remainder of the current SelectionDAG iteration, so we can allocate + // the operands directly out of a pool with no recycling metadata. + MN->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), + Ops.data(), NumOps); + else + MN->InitOperands(MN->LocalOperands, Ops.data(), NumOps); + MN->OperandsNeedDelete = false; + } else + MN->InitOperands(MN->OperandList, Ops.data(), NumOps); + } else { + // If NumOps is larger than the # of operands we currently have, reallocate + // the operand list. + if (NumOps > N->NumOperands) { + if (N->OperandsNeedDelete) + delete[] N->OperandList; + N->InitOperands(new SDUse[NumOps], Ops.data(), NumOps); + N->OperandsNeedDelete = true; + } else + N->InitOperands(N->OperandList, Ops.data(), NumOps); + } // Delete any nodes that are still dead after adding the uses for the // new operands. @@ -6176,14 +6178,16 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc dl, MachineSDNode * SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, - ArrayRef<SDValue> Ops) { + ArrayRef<SDValue> OpsArray) { bool DoCSE = VTs.VTs[VTs.NumVTs-1] != MVT::Glue; MachineSDNode *N; void *IP = nullptr; + const SDValue *Ops = OpsArray.data(); + unsigned NumOps = OpsArray.size(); if (DoCSE) { FoldingSetNodeID ID; - AddNodeIDNode(ID, ~Opcode, VTs, Ops); + AddNodeIDNode(ID, ~Opcode, VTs, OpsArray); IP = nullptr; if (SDNode *E = FindNodeOrInsertPos(ID, DL.getDebugLoc(), IP)) { return cast<MachineSDNode>(UpdadeSDLocOnMergedSDNode(E, DL)); @@ -6192,7 +6196,17 @@ SelectionDAG::getMachineNode(unsigned Opcode, SDLoc DL, SDVTList VTs, // Allocate a new MachineSDNode. N = newSDNode<MachineSDNode>(~Opcode, DL.getIROrder(), DL.getDebugLoc(), VTs); - createOperands(N, Ops); + + // Initialize the operands list. + if (NumOps > array_lengthof(N->LocalOperands)) + // We're creating a final node that will live unmorphed for the + // remainder of the current SelectionDAG iteration, so we can allocate + // the operands directly out of a pool with no recycling metadata. + N->InitOperands(OperandAllocator.Allocate<SDUse>(NumOps), + Ops, NumOps); + else + N->InitOperands(N->LocalOperands, Ops, NumOps); + N->OperandsNeedDelete = false; if (DoCSE) CSEMap.InsertNode(N, IP); @@ -6725,9 +6739,10 @@ GlobalAddressSDNode::GlobalAddressSDNode(unsigned Opc, unsigned Order, } AddrSpaceCastSDNode::AddrSpaceCastSDNode(unsigned Order, DebugLoc dl, EVT VT, - unsigned SrcAS, unsigned DestAS) - : SDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT)), - SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} + SDValue X, unsigned SrcAS, + unsigned DestAS) + : UnarySDNode(ISD::ADDRSPACECAST, Order, dl, getSDVTList(VT), X), + SrcAddrSpace(SrcAS), DestAddrSpace(DestAS) {} MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, EVT memvt, MachineMemOperand *mmo) @@ -6743,6 +6758,16 @@ MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); } +MemSDNode::MemSDNode(unsigned Opc, unsigned Order, DebugLoc dl, SDVTList VTs, + ArrayRef<SDValue> Ops, EVT memvt, MachineMemOperand *mmo) + : SDNode(Opc, Order, dl, VTs, Ops), + MemoryVT(memvt), MMO(mmo) { + SubclassData = encodeMemSDNodeFlags(0, ISD::UNINDEXED, MMO->isVolatile(), + MMO->isNonTemporal(), MMO->isInvariant()); + assert(isVolatile() == MMO->isVolatile() && "Volatile encoding error!"); + assert(memvt.getStoreSize() <= MMO->getSize() && "Size mismatch!"); +} + /// Profile - Gather unique data for the node. /// void SDNode::Profile(FoldingSetNodeID &ID) const { |
