diff options
4 files changed, 77 insertions, 10 deletions
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h index db9c5b713b7..3cc1d35cc5c 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelector.h @@ -107,6 +107,9 @@ enum { /// - InsnID - Instruction ID /// - The predicate to test GIM_CheckAPFloatImmPredicate, + /// Check a memory operation is non-atomic. + /// - InsnID - Instruction ID + GIM_CheckNonAtomic, /// Check the type for the specified operand /// - InsnID - Instruction ID diff --git a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h index f1bc8484096..d4cb1326fb8 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/InstructionSelectorImpl.h @@ -209,6 +209,25 @@ bool InstructionSelector::executeMatchTable( return false; break; } + case GIM_CheckNonAtomic: { + int64_t InsnID = MatchTable[CurrentIdx++]; + DEBUG(dbgs() << CurrentIdx << ": GIM_CheckNonAtomic(MIs[" << InsnID + << "])\n"); + assert(State.MIs[InsnID] != nullptr && "Used insn before defined"); + assert((State.MIs[InsnID]->getOpcode() == TargetOpcode::G_LOAD || + State.MIs[InsnID]->getOpcode() == TargetOpcode::G_STORE) && + "Expected G_LOAD/G_STORE"); + + if (!State.MIs[InsnID]->hasOneMemOperand()) + if (handleReject() == RejectAndGiveUp) + return false; + + for (const auto &MMO : State.MIs[InsnID]->memoperands()) + if (MMO->getOrdering() != AtomicOrdering::NotAtomic) + if (handleReject() == RejectAndGiveUp) + return false; + break; + } case GIM_CheckType: { int64_t InsnID = MatchTable[CurrentIdx++]; diff --git a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td index 96245b251f9..c012b20fd7b 100644 --- a/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td +++ b/llvm/include/llvm/Target/GlobalISel/SelectionDAGCompat.td @@ -23,6 +23,11 @@ class GINodeEquiv<Instruction i, SDNode node> { Instruction I = i; SDNode Node = node; + + // SelectionDAG has separate nodes for atomic and non-atomic memory operations + // (ISD::LOAD, ISD::ATOMIC_LOAD, ISD::STORE, ISD::ATOMIC_STORE) but GlobalISel + // stores this information in the MachineMemoryOperand. + bit CheckMMOIsNonAtomic = 0; } // These are defined in the same order as the G_* instructions. @@ -72,6 +77,23 @@ def : GINodeEquiv<G_INTRINSIC_W_SIDE_EFFECTS, intrinsic_w_chain>; def : GINodeEquiv<G_BR, br>; def : GINodeEquiv<G_BSWAP, bswap>; +// Broadly speaking G_LOAD is equivalent to ISD::LOAD but there are some +// complications that tablegen must take care of. For example, Predicates such +// as isSignExtLoad require that this is not a perfect 1:1 mapping since a +// sign-extending load is (G_SEXT (G_LOAD x)) in GlobalISel. Additionally, +// G_LOAD handles both atomic and non-atomic loads where as SelectionDAG had +// separate nodes for them. This GINodeEquiv maps the non-atomic loads to +// G_LOAD with a non-atomic MachineMemOperand. +def : GINodeEquiv<G_LOAD, ld> { let CheckMMOIsNonAtomic = 1; } +// Broadly speaking G_STORE is equivalent to ISD::STORE but there are some +// complications that tablegen must take care of. For example, predicates such +// as isTruncStore require that this is not a perfect 1:1 mapping since a +// truncating store is (G_STORE (G_TRUNCATE x)) in GlobalISel. Additionally, +// G_STORE handles both atomic and non-atomic stores where as SelectionDAG had +// separate nodes for them. This GINodeEquiv maps the non-atomic stores to +// G_STORE with a non-atomic MachineMemOperand. +def : GINodeEquiv<G_STORE, st> { let CheckMMOIsNonAtomic = 1; } + // Specifies the GlobalISel equivalents for SelectionDAG's ComplexPattern. // Should be used on defs that subclass GIComplexOperandMatcher<>. class GIComplexPatternEquiv<ComplexPattern seldag> { diff --git a/llvm/utils/TableGen/GlobalISelEmitter.cpp b/llvm/utils/TableGen/GlobalISelEmitter.cpp index 467ae40e51d..bbc396967c0 100644 --- a/llvm/utils/TableGen/GlobalISelEmitter.cpp +++ b/llvm/utils/TableGen/GlobalISelEmitter.cpp @@ -997,6 +997,7 @@ protected: enum PredicateKind { IPM_Opcode, IPM_ImmPredicate, + IPM_NonAtomicMMO, }; PredicateKind Kind; @@ -1125,6 +1126,24 @@ public: } }; +/// Generates code to check that a memory instruction has a non-atomic MachineMemoryOperand. +class NonAtomicMMOPredicateMatcher : public InstructionPredicateMatcher { +public: + NonAtomicMMOPredicateMatcher() + : InstructionPredicateMatcher(IPM_NonAtomicMMO) {} + + static bool classof(const InstructionPredicateMatcher *P) { + return P->getKind() == IPM_NonAtomicMMO; + } + + void emitPredicateOpcodes(MatchTable &Table, RuleMatcher &Rule, + unsigned InsnVarID) const override { + Table << MatchTable::Opcode("GIM_CheckNonAtomic") + << MatchTable::Comment("MI") << MatchTable::IntValue(InsnVarID) + << MatchTable::LineBreak; + } +}; + /// Generates code to check that a set of predicates and operands match for a /// particular instruction. /// @@ -1991,9 +2010,11 @@ private: const CodeGenTarget &Target; CodeGenRegBank CGRegs; - /// Keep track of the equivalence between SDNodes and Instruction. + /// Keep track of the equivalence between SDNodes and Instruction by mapping + /// SDNodes to the GINodeEquiv mapping. We need to map to the GINodeEquiv to + /// check for attributes on the relation such as CheckMMOIsNonAtomic. /// This is defined using 'GINodeEquiv' in the target description. - DenseMap<Record *, const CodeGenInstruction *> NodeEquivs; + DenseMap<Record *, Record *> NodeEquivs; /// Keep track of the equivalence between ComplexPattern's and /// GIComplexOperandMatcher. Map entries are specified by subclassing @@ -2004,7 +2025,7 @@ private: SubtargetFeatureInfoMap SubtargetFeatures; void gatherNodeEquivs(); - const CodeGenInstruction *findNodeEquiv(Record *N) const; + Record *findNodeEquiv(Record *N) const; Error importRulePredicates(RuleMatcher &M, ArrayRef<Predicate> Predicates); Expected<InstructionMatcher &> @@ -2041,8 +2062,7 @@ private: void GlobalISelEmitter::gatherNodeEquivs() { assert(NodeEquivs.empty()); for (Record *Equiv : RK.getAllDerivedDefinitions("GINodeEquiv")) - NodeEquivs[Equiv->getValueAsDef("Node")] = - &Target.getInstruction(Equiv->getValueAsDef("I")); + NodeEquivs[Equiv->getValueAsDef("Node")] = Equiv; assert(ComplexPatternEquivs.empty()); for (Record *Equiv : RK.getAllDerivedDefinitions("GIComplexPatternEquiv")) { @@ -2053,7 +2073,7 @@ void GlobalISelEmitter::gatherNodeEquivs() { } } -const CodeGenInstruction *GlobalISelEmitter::findNodeEquiv(Record *N) const { +Record *GlobalISelEmitter::findNodeEquiv(Record *N) const { return NodeEquivs.lookup(N); } @@ -2080,6 +2100,7 @@ Expected<InstructionMatcher &> GlobalISelEmitter::createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, const TreePatternNode *Src, unsigned &TempOpIdx) const { + Record *SrcGIEquivOrNull = nullptr; const CodeGenInstruction *SrcGIOrNull = nullptr; // Start with the defined operands (i.e., the results of the root operator). @@ -2095,14 +2116,14 @@ GlobalISelEmitter::createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, return failedImport( "Unable to deduce gMIR opcode to handle Src (which is a leaf)"); } else { - SrcGIOrNull = findNodeEquiv(Src->getOperator()); - if (!SrcGIOrNull) + SrcGIEquivOrNull = findNodeEquiv(Src->getOperator()); + if (!SrcGIEquivOrNull) return failedImport("Pattern operator lacks an equivalent Instruction" + explainOperator(Src->getOperator())); - auto &SrcGI = *SrcGIOrNull; + SrcGIOrNull = &Target.getInstruction(SrcGIEquivOrNull->getValueAsDef("I")); // The operators look good: match the opcode - InsnMatcher.addPredicate<InstructionOpcodeMatcher>(&SrcGI); + InsnMatcher.addPredicate<InstructionOpcodeMatcher>(SrcGIOrNull); } unsigned OpIdx = 0; @@ -2132,6 +2153,8 @@ GlobalISelEmitter::createAndImportSelDAGMatcher(InstructionMatcher &InsnMatcher, return failedImport("Src pattern child has predicate (" + explainPredicates(Src) + ")"); } + if (SrcGIEquivOrNull && SrcGIEquivOrNull->getValueAsBit("CheckMMOIsNonAtomic")) + InsnMatcher.addPredicate<NonAtomicMMOPredicateMatcher>(); if (Src->isLeaf()) { Init *SrcInit = Src->getLeafValue(); |
