diff options
| -rw-r--r-- | llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h | 340 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/GlobalISel/CMakeLists.txt | 4 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp | 78 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp | 44 | ||||
| -rw-r--r-- | llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp | 108 | ||||
| -rw-r--r-- | llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp | 554 | ||||
| -rw-r--r-- | llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp | 90 |
7 files changed, 843 insertions, 375 deletions
diff --git a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h index e9500343d4d..c35ef329996 100644 --- a/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h +++ b/llvm/include/llvm/CodeGen/GlobalISel/LegalizerInfo.h @@ -21,6 +21,7 @@ #include "llvm/ADT/STLExtras.h" #include "llvm/ADT/SmallVector.h" #include "llvm/CodeGen/TargetOpcodes.h" +#include "llvm/Support/raw_ostream.h" #include "llvm/Support/LowLevelTypeImpl.h" #include <cassert> #include <cstdint> @@ -80,6 +81,10 @@ enum LegalizeAction : std::uint8_t { /// Sentinel value for when no action was found in the specified table. NotFound, + + /// Fall back onto the old rules. + /// TODO: Remove this once we've migrated + UseLegacyRules, }; } // end namespace LegalizeActions @@ -109,34 +114,319 @@ struct InstrAspect { struct LegalityQuery { unsigned Opcode; ArrayRef<LLT> Types; + + raw_ostream &print(raw_ostream &OS) const; }; -class LegalizerInfo { +/// The result of a query. It either indicates a final answer of Legal or +/// Unsupported or describes an action that must be taken to make an operation +/// more legal. +struct LegalizeActionStep { + /// The action to take or the final answer. + LegalizeAction Action; + /// If describing an action, the type index to change. Otherwise zero. + unsigned TypeIdx; + /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. + LLT NewType; + + LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, + const LLT &NewType) + : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} + + bool operator==(const LegalizeActionStep &RHS) const { + return std::tie(Action, TypeIdx, NewType) == + std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); + } +}; + +using LegalityPredicate = std::function<bool (const LegalityQuery &)>; +using LegalizeMutation = + std::function<std::pair<unsigned, LLT>(const LegalityQuery &)>; + +namespace LegalityPredicates { +LegalityPredicate all(LegalityPredicate P0, LegalityPredicate P1); +LegalityPredicate typeInSet(unsigned TypeIdx, + std::initializer_list<LLT> TypesInit); +LegalityPredicate +typePairInSet(unsigned TypeIdx0, unsigned TypeIdx1, + std::initializer_list<std::pair<LLT, LLT>> TypesInit); +LegalityPredicate isScalar(unsigned TypeIdx); +LegalityPredicate narrowerThan(unsigned TypeIdx, unsigned Size); +LegalityPredicate widerThan(unsigned TypeIdx, unsigned Size); +LegalityPredicate sizeNotPow2(unsigned TypeIdx); +LegalityPredicate numElementsNotPow2(unsigned TypeIdx); +} // end namespace LegalityPredicates + +namespace LegalizeMutations { +LegalizeMutation identity(unsigned TypeIdx, LLT Ty); +LegalizeMutation widenScalarToNextPow2(unsigned TypeIdx, unsigned Min = 0); +LegalizeMutation moreElementsToNextPow2(unsigned TypeIdx, unsigned Min = 0); +} // end namespace LegalizeMutations + +class LegalizeRule { + LegalityPredicate Predicate; + LegalizeAction Action; + LegalizeMutation Mutation; + public: - /// The result of a query. It either indicates a final answer of Legal or - /// Unsupported or describes an action that must be taken to make an operation - /// more legal. - struct LegalizeActionStep { - /// The action to take or the final answer. - LegalizeAction Action; - /// If describing an action, the type index to change. Otherwise zero. - unsigned TypeIdx; - /// If describing an action, the new type for TypeIdx. Otherwise LLT{}. - LLT NewType; - - LegalizeActionStep(LegalizeAction Action, unsigned TypeIdx, - const LLT &NewType) - : Action(Action), TypeIdx(TypeIdx), NewType(NewType) {} - - bool operator==(const LegalizeActionStep &RHS) const { - return std::tie(Action, TypeIdx, NewType) == - std::tie(RHS.Action, RHS.TypeIdx, RHS.NewType); - } - }; + LegalizeRule(LegalityPredicate Predicate, LegalizeAction Action, + LegalizeMutation Mutation = nullptr) + : Predicate(Predicate), Action(Action), Mutation(Mutation) {} + + bool match(const LegalityQuery &Query) const { + return Predicate(Query); + } + + LegalizeAction getAction() const { return Action; } + std::pair<unsigned, LLT> determineMutation(const LegalityQuery &Query) const { + if (Mutation) + return Mutation(Query); + return std::make_pair(0, LLT{}); + } +}; + +class LegalizeRuleSet { + /// When non-zero, the opcode we are an alias of + unsigned AliasOf; + /// If true, there is another opcode that aliases this one + bool IsAliasedByAnother; + SmallVector<LegalizeRule, 2> Rules; + + void add(const LegalizeRule &Rule) { + assert(AliasOf == 0 && + "RuleSet is aliased, change the representative opcode instead"); + Rules.push_back(Rule); + } + + static bool always(const LegalityQuery &) { return true; } + + LegalizeRuleSet &actionIf(LegalizeAction Action, + LegalityPredicate Predicate) { + add({Predicate, Action}); + return *this; + } + LegalizeRuleSet &actionIf(LegalizeAction Action, LegalityPredicate Predicate, + LegalizeMutation Mutation) { + add({Predicate, Action, Mutation}); + return *this; + } + LegalizeRuleSet &actionFor(LegalizeAction Action, + std::initializer_list<LLT> Types) { + using namespace LegalityPredicates; + return actionIf(Action, typeInSet(0, Types)); + } + LegalizeRuleSet & + actionFor(LegalizeAction Action, + std::initializer_list<std::pair<LLT, LLT>> Types) { + using namespace LegalityPredicates; + return actionIf(Action, typePairInSet(0, 1, Types)); + } + LegalizeRuleSet &actionForCartesianProduct(LegalizeAction Action, + std::initializer_list<LLT> Types) { + using namespace LegalityPredicates; + return actionIf(Action, all(typeInSet(0, Types), typeInSet(1, Types))); + } + LegalizeRuleSet & + actionForCartesianProduct(LegalizeAction Action, + std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + using namespace LegalityPredicates; + return actionIf(Action, all(typeInSet(0, Types0), typeInSet(1, Types1))); + } + +public: + LegalizeRuleSet() : AliasOf(0), IsAliasedByAnother(false), Rules() {} + + bool isAliasedByAnother() { return IsAliasedByAnother; } + void setIsAliasedByAnother() { IsAliasedByAnother = true; } + void aliasTo(unsigned Opcode) { + assert((AliasOf == 0 || AliasOf == Opcode) && + "Opcode is already aliased to another opcode"); + assert(Rules.empty() && "Aliasing will discard rules"); + AliasOf = Opcode; + } + unsigned getAlias() const { return AliasOf; } + + // Primitive rule definition functions + LegalizeRuleSet &legalIf(LegalityPredicate Predicate) { + return actionIf(LegalizeAction::Legal, Predicate); + } + LegalizeRuleSet &legalFor(std::initializer_list<LLT> Types) { + return actionFor(LegalizeAction::Legal, Types); + } + LegalizeRuleSet &legalFor(std::initializer_list<std::pair<LLT, LLT>> Types) { + return actionFor(LegalizeAction::Legal, Types); + } + LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types) { + return actionForCartesianProduct(LegalizeAction::Legal, Types); + } + LegalizeRuleSet &legalForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + return actionForCartesianProduct(LegalizeAction::Legal, Types0, Types1); + } + + LegalizeRuleSet &libcallIf(LegalityPredicate Predicate) { + return actionIf(LegalizeAction::Libcall, Predicate); + } + LegalizeRuleSet &libcallFor(std::initializer_list<LLT> Types) { + return actionFor(LegalizeAction::Libcall, Types); + } + LegalizeRuleSet & + libcallFor(std::initializer_list<std::pair<LLT, LLT>> Types) { + return actionFor(LegalizeAction::Libcall, Types); + } + LegalizeRuleSet & + libcallForCartesianProduct(std::initializer_list<LLT> Types) { + return actionForCartesianProduct(LegalizeAction::Libcall, Types); + } + LegalizeRuleSet & + libcallForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + return actionForCartesianProduct(LegalizeAction::Libcall, Types0, Types1); + } + LegalizeRuleSet &widenScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizeAction::WidenScalar, Predicate, Mutation); + } + LegalizeRuleSet &narrowScalarIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizeAction::NarrowScalar, Predicate, Mutation); + } + + LegalizeRuleSet &moreElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizeAction::MoreElements, Predicate, Mutation); + } + LegalizeRuleSet &fewerElementsIf(LegalityPredicate Predicate, + LegalizeMutation Mutation) { + return actionIf(LegalizeAction::FewerElements, Predicate, Mutation); + } + + LegalizeRuleSet &unsupported() { + return actionIf(LegalizeAction::Unsupported, always); + } + LegalizeRuleSet &unsupportedIf(LegalityPredicate Predicate) { + return actionIf(LegalizeAction::Unsupported, Predicate); + } + + LegalizeRuleSet &customIf(LegalityPredicate Predicate) { + return actionIf(LegalizeAction::Custom, Predicate); + } + LegalizeRuleSet &customForCartesianProduct(std::initializer_list<LLT> Types) { + return actionForCartesianProduct(LegalizeAction::Custom, Types); + } + LegalizeRuleSet & + customForCartesianProduct(std::initializer_list<LLT> Types0, + std::initializer_list<LLT> Types1) { + return actionForCartesianProduct(LegalizeAction::Custom, Types0, Types1); + } + + // Convenience rule definition functions + LegalizeRuleSet &widenScalarToNextPow2(unsigned TypeIdx, unsigned MinSize = 0) { + using namespace LegalityPredicates; + return widenScalarIf( + sizeNotPow2(TypeIdx), + LegalizeMutations::widenScalarToNextPow2(TypeIdx, MinSize)); + } + + LegalizeRuleSet &narrowScalar(unsigned TypeIdx, LegalizeMutation Mutation) { + using namespace LegalityPredicates; + return narrowScalarIf(isScalar(TypeIdx), Mutation); + } + + LegalizeRuleSet &minScalar(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return widenScalarIf(narrowerThan(TypeIdx, Ty.getSizeInBits()), + LegalizeMutations::identity(TypeIdx, Ty)); + } + + LegalizeRuleSet &maxScalar(unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return narrowScalarIf(widerThan(TypeIdx, Ty.getSizeInBits()), + LegalizeMutations::identity(TypeIdx, Ty)); + } + + LegalizeRuleSet &maxScalarIf(LegalityPredicate Predicate, unsigned TypeIdx, const LLT &Ty) { + using namespace LegalityPredicates; + using namespace LegalizeMutations; + return narrowScalarIf( + [=](const LegalityQuery &Query) { + return widerThan(TypeIdx, Ty.getSizeInBits()) && + Predicate(Query); + }, + LegalizeMutations::identity(TypeIdx, Ty)); + } + + LegalizeRuleSet &clampScalar(unsigned TypeIdx, const LLT &MinTy, const LLT &MaxTy) { + assert(MinTy.isScalar() && MaxTy.isScalar() && "Expected scalar types"); + + return minScalar(TypeIdx, MinTy) + .maxScalar(TypeIdx, MaxTy); + } + + LegalizeRuleSet &moreElementsToNextPow2(unsigned TypeIdx) { + using namespace LegalityPredicates; + return moreElementsIf(numElementsNotPow2(TypeIdx), + LegalizeMutations::moreElementsToNextPow2(TypeIdx)); + } + + LegalizeRuleSet &clampMinNumElements(unsigned TypeIdx, const LLT &EltTy, + unsigned MinElements) { + return moreElementsIf( + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return VecTy.getElementType() == EltTy && + VecTy.getNumElements() < MinElements; + }, + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return std::make_pair( + TypeIdx, LLT::vector(MinElements, VecTy.getScalarSizeInBits())); + }); + } + LegalizeRuleSet &clampMaxNumElements(unsigned TypeIdx, const LLT &EltTy, + unsigned MaxElements) { + return fewerElementsIf( + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return VecTy.getElementType() == EltTy && + VecTy.getNumElements() > MaxElements; + }, + [=](const LegalityQuery &Query) { + LLT VecTy = Query.Types[TypeIdx]; + return std::make_pair( + TypeIdx, LLT::vector(MaxElements, VecTy.getScalarSizeInBits())); + }); + } + LegalizeRuleSet &clampNumElements(unsigned TypeIdx, const LLT &MinTy, + const LLT &MaxTy) { + assert(MinTy.getElementType() == MaxTy.getElementType() && + "Expected element types to agree"); + + const LLT &EltTy = MinTy.getElementType(); + return clampMinNumElements(TypeIdx, EltTy, MinTy.getNumElements()) + .clampMaxNumElements(TypeIdx, EltTy, MaxTy.getNumElements()); + } + + LegalizeRuleSet &fallback() { + add({always, LegalizeAction::UseLegacyRules}); + return *this; + } + + LegalizeActionStep apply(const LegalityQuery &Query) const; +}; + +class LegalizerInfo { +public: LegalizerInfo(); virtual ~LegalizerInfo() = default; + unsigned getOpcodeIdxForOpcode(unsigned Opcode) const; + unsigned getActionDefinitionsIdx(unsigned Opcode) const; + /// Compute any ancillary tables needed to quickly decide how an operation /// should be handled. This must be called after all "set*Action"methods but /// before any query is made or incorrect results may be returned. @@ -300,6 +590,12 @@ public: LegalizeAction DecreaseAction, LegalizeAction IncreaseAction); + const LegalizeRuleSet &getActionDefinitions(unsigned Opcode) const; + LegalizeRuleSet &getActionDefinitionsBuilder(unsigned Opcode); + LegalizeRuleSet & + getActionDefinitionsBuilder(std::initializer_list<unsigned> Opcodes); + void aliasActionDefinitions(unsigned OpcodeTo, unsigned OpcodeFrom); + /// Determine what action should be taken to legalize the described /// instruction. Requires computeTables to have been called. /// @@ -503,6 +799,8 @@ private: AddrSpace2PointerActions[LastOp - FirstOp + 1]; std::unordered_map<uint16_t, SmallVector<SizeAndActionsVec, 1>> NumElements2Actions[LastOp - FirstOp + 1]; + + LegalizeRuleSet RulesForOpcode[LastOp - FirstOp + 1]; }; } // end namespace llvm. diff --git a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt index e68ed179558..4c1da3756b1 100644 --- a/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt +++ b/llvm/lib/CodeGen/GlobalISel/CMakeLists.txt @@ -6,8 +6,10 @@ add_llvm_library(LLVMGlobalISel IRTranslator.cpp InstructionSelect.cpp InstructionSelector.cpp - LegalizerHelper.cpp + LegalityPredicates.cpp + LegalizeMutations.cpp Legalizer.cpp + LegalizerHelper.cpp LegalizerInfo.cpp Localizer.cpp MachineIRBuilder.cpp diff --git a/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp new file mode 100644 index 00000000000..0d30e0781cd --- /dev/null +++ b/llvm/lib/CodeGen/GlobalISel/LegalityPredicates.cpp @@ -0,0 +1,78 @@ +//===- lib/CodeGen/GlobalISel/LegalizerPredicates.cpp - Predicates --------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// A library of predicate factories to use for LegalityPredicate. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" + +using namespace llvm; + +LegalityPredicate +LegalityPredicates::all(LegalityPredicate P0, LegalityPredicate P1) { + return [=](const LegalityQuery &Query) { + return P0(Query) && P1(Query); + }; +} + +LegalityPredicate +LegalityPredicates::typeInSet(unsigned TypeIdx, + std::initializer_list<LLT> TypesInit) { + SmallVector<LLT, 4> Types = TypesInit; + return [=](const LegalityQuery &Query) { + return std::find(Types.begin(), Types.end(), Query.Types[TypeIdx]) != Types.end(); + }; +} + +LegalityPredicate LegalityPredicates::typePairInSet( + unsigned TypeIdx0, unsigned TypeIdx1, + std::initializer_list<std::pair<LLT, LLT>> TypesInit) { + SmallVector<std::pair<LLT, LLT>, 4> Types = TypesInit; + return [=](const LegalityQuery &Query) { + std::pair<LLT, LLT> Match = {Query.Types[TypeIdx0], Query.Types[TypeIdx1]}; + return std::find(Types.begin(), Types.end(), Match) != Types.end(); + }; +} + +LegalityPredicate LegalityPredicates::isScalar(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + return Query.Types[TypeIdx].isScalar(); + }; +} + +LegalityPredicate LegalityPredicates::narrowerThan(unsigned TypeIdx, + unsigned Size) { + return [=](const LegalityQuery &Query) { + const LLT &QueryTy = Query.Types[TypeIdx]; + return QueryTy.isScalar() && QueryTy.getSizeInBits() < Size; + }; +} + +LegalityPredicate LegalityPredicates::widerThan(unsigned TypeIdx, + unsigned Size) { + return [=](const LegalityQuery &Query) { + const LLT &QueryTy = Query.Types[TypeIdx]; + return QueryTy.isScalar() && QueryTy.getSizeInBits() > Size; + }; +} + +LegalityPredicate LegalityPredicates::sizeNotPow2(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + const LLT &QueryTy = Query.Types[TypeIdx]; + return QueryTy.isScalar() && !isPowerOf2_32(QueryTy.getSizeInBits()); + }; +} + +LegalityPredicate LegalityPredicates::numElementsNotPow2(unsigned TypeIdx) { + return [=](const LegalityQuery &Query) { + const LLT &QueryTy = Query.Types[TypeIdx]; + return QueryTy.isVector() && isPowerOf2_32(QueryTy.getNumElements()); + }; +} diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp new file mode 100644 index 00000000000..961988515e5 --- /dev/null +++ b/llvm/lib/CodeGen/GlobalISel/LegalizeMutations.cpp @@ -0,0 +1,44 @@ +//===- lib/CodeGen/GlobalISel/LegalizerMutations.cpp - Mutations ----------===// +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// A library of mutation factories to use for LegalityMutation. +// +//===----------------------------------------------------------------------===// + +#include "llvm/CodeGen/GlobalISel/LegalizerInfo.h" + +using namespace llvm; + +LegalizeMutation LegalizeMutations::identity(unsigned TypeIdx, LLT Ty) { + return + [=](const LegalityQuery &Query) { return std::make_pair(TypeIdx, Ty); }; +} + +LegalizeMutation LegalizeMutations::widenScalarToNextPow2(unsigned TypeIdx, + unsigned Min) { + return [=](const LegalityQuery &Query) { + unsigned NewSizeInBits = + 1 << Log2_32_Ceil(Query.Types[TypeIdx].getSizeInBits()); + if (NewSizeInBits < Min) + NewSizeInBits = Min; + return std::make_pair(TypeIdx, LLT::scalar(NewSizeInBits)); + }; +} + +LegalizeMutation LegalizeMutations::moreElementsToNextPow2(unsigned TypeIdx, + unsigned Min) { + return [=](const LegalityQuery &Query) { + const LLT &VecTy = Query.Types[TypeIdx]; + unsigned NewNumElements = 1 << Log2_32_Ceil(VecTy.getNumElements()); + if (NewNumElements < Min) + NewNumElements = Min; + return std::make_pair( + TypeIdx, LLT::vector(NewNumElements, VecTy.getScalarSizeInBits())); + }; +} diff --git a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp index 12f5524aa8c..054acc1d2fe 100644 --- a/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp +++ b/llvm/lib/CodeGen/GlobalISel/LegalizerInfo.cpp @@ -24,6 +24,7 @@ #include "llvm/CodeGen/MachineRegisterInfo.h" #include "llvm/CodeGen/TargetOpcodes.h" #include "llvm/MC/MCInstrDesc.h" +#include "llvm/Support/Debug.h" #include "llvm/Support/ErrorHandling.h" #include "llvm/Support/LowLevelTypeImpl.h" #include "llvm/Support/MathExtras.h" @@ -33,6 +34,40 @@ using namespace llvm; using namespace LegalizeActions; +#define DEBUG_TYPE "legalizer-info" + +raw_ostream &LegalityQuery::print(raw_ostream &OS) const { + OS << Opcode << ", {"; + for (const auto &Type : Types) { + OS << Type << ", "; + } + OS << "}"; + return OS; +} + +LegalizeActionStep LegalizeRuleSet::apply(const LegalityQuery &Query) const { + DEBUG(dbgs() << "Applying legalizer ruleset to: "; Query.print(dbgs()); + dbgs() << "\n"); + if (Rules.empty()) { + DEBUG(dbgs() << ".. fallback to legacy rules (no rules defined)\n"); + return {LegalizeAction::UseLegacyRules, 0, LLT{}}; + } + for (const auto &Rule : Rules) { + if (Rule.match(Query)) { + DEBUG(dbgs() << ".. match\n"); + std::pair<unsigned, LLT> Mutation = Rule.determineMutation(Query); + DEBUG(dbgs() << ".. .. " << (unsigned)Rule.getAction() << ", " + << Mutation.first << ", " << Mutation.second << "\n"); + assert(Query.Types[Mutation.first] != Mutation.second && + "Simple loop detected"); + return {Rule.getAction(), Mutation.first, Mutation.second}; + } else + DEBUG(dbgs() << ".. no match\n"); + } + DEBUG(dbgs() << ".. unsupported\n"); + return {LegalizeAction::Unsupported, 0, LLT{}}; +} + LegalizerInfo::LegalizerInfo() : TablesInitialized(false) { // Set defaults. // FIXME: these two (G_ANYEXT and G_TRUNC?) can be legalized to the @@ -188,17 +223,79 @@ static LLT getTypeFromTypeIdx(const MachineInstr &MI, return MRI.getType(MI.getOperand(OpIdx).getReg()); } -LegalizerInfo::LegalizeActionStep +unsigned LegalizerInfo::getOpcodeIdxForOpcode(unsigned Opcode) const { + assert(Opcode >= FirstOp && Opcode <= LastOp && "Unsupported opcode"); + return Opcode - FirstOp; +} + +unsigned LegalizerInfo::getActionDefinitionsIdx(unsigned Opcode) const { + unsigned OpcodeIdx = getOpcodeIdxForOpcode(Opcode); + if (unsigned Alias = RulesForOpcode[OpcodeIdx].getAlias()) { + DEBUG(dbgs() << ".. opcode " << Opcode << " is aliased to " << Alias + << "\n"); + OpcodeIdx = getOpcodeIdxForOpcode(Alias); + DEBUG(dbgs() << ".. opcode " << Alias << " is aliased to " + << RulesForOpcode[OpcodeIdx].getAlias() << "\n"); + assert(RulesForOpcode[OpcodeIdx].getAlias() == 0 && "Cannot chain aliases"); + } + + return OpcodeIdx; +} + +const LegalizeRuleSet & +LegalizerInfo::getActionDefinitions(unsigned Opcode) const { + unsigned OpcodeIdx = getActionDefinitionsIdx(Opcode); + return RulesForOpcode[OpcodeIdx]; +} + +LegalizeRuleSet &LegalizerInfo::getActionDefinitionsBuilder(unsigned Opcode) { + unsigned OpcodeIdx = getActionDefinitionsIdx(Opcode); + auto &Result = RulesForOpcode[OpcodeIdx]; + assert(!Result.isAliasedByAnother() && "Modifying this opcode will modify aliases"); + return Result; +} + +LegalizeRuleSet &LegalizerInfo::getActionDefinitionsBuilder( + std::initializer_list<unsigned> Opcodes) { + unsigned Representative = *Opcodes.begin(); + + for (auto I = Opcodes.begin() + 1, E = Opcodes.end(); I != E; ++I) + aliasActionDefinitions(Representative, *I); + + auto &Return = getActionDefinitionsBuilder(Representative); + Return.setIsAliasedByAnother(); + return Return; +} + +void LegalizerInfo::aliasActionDefinitions(unsigned OpcodeTo, + unsigned OpcodeFrom) { + assert(OpcodeTo != OpcodeFrom && "Cannot alias to self"); + assert(OpcodeTo >= FirstOp && OpcodeTo <= LastOp && "Unsupported opcode"); + const unsigned OpcodeFromIdx = getOpcodeIdxForOpcode(OpcodeFrom); + RulesForOpcode[OpcodeFromIdx].aliasTo(OpcodeTo); +} + +LegalizeActionStep LegalizerInfo::getAction(const LegalityQuery &Query) const { + LegalizeActionStep Step = getActionDefinitions(Query.Opcode).apply(Query); + if (Step.Action != LegalizeAction::UseLegacyRules) { + return Step; + } + for (unsigned i = 0; i < Query.Types.size(); ++i) { auto Action = getAspectAction({Query.Opcode, i, Query.Types[i]}); - if (Action.first != Legal) + if (Action.first != Legal) { + DEBUG(dbgs() << ".. (legacy) Type " << i << " Action=" + << (unsigned)Action.first << ", " << Action.second << "\n"); return {Action.first, i, Action.second}; + } else + DEBUG(dbgs() << ".. (legacy) Type " << i << " Legal\n"); } + DEBUG(dbgs() << ".. (legacy) Legal\n"); return {Legal, 0, LLT{}}; } -LegalizerInfo::LegalizeActionStep +LegalizeActionStep LegalizerInfo::getAction(const MachineInstr &MI, const MachineRegisterInfo &MRI) const { SmallVector<LLT, 2> Types; @@ -323,6 +420,7 @@ LegalizerInfo::findAction(const SizeAndActionsVec &Vec, const uint32_t Size) { case Unsupported: return {Size, Unsupported}; case NotFound: + case UseLegacyRules: llvm_unreachable("NotFound"); } llvm_unreachable("Action has an unknown enum value"); @@ -333,7 +431,7 @@ LegalizerInfo::findScalarLegalAction(const InstrAspect &Aspect) const { assert(Aspect.Type.isScalar() || Aspect.Type.isPointer()); if (Aspect.Opcode < FirstOp || Aspect.Opcode > LastOp) return {NotFound, LLT()}; - const unsigned OpcodeIdx = Aspect.Opcode - FirstOp; + const unsigned OpcodeIdx = getOpcodeIdxForOpcode(Aspect.Opcode); if (Aspect.Type.isPointer() && AddrSpace2PointerActions[OpcodeIdx].find(Aspect.Type.getAddressSpace()) == AddrSpace2PointerActions[OpcodeIdx].end()) { @@ -364,7 +462,7 @@ LegalizerInfo::findVectorLegalAction(const InstrAspect &Aspect) const { // lanes in the vector. if (Aspect.Opcode < FirstOp || Aspect.Opcode > LastOp) return {NotFound, Aspect.Type}; - const unsigned OpcodeIdx = Aspect.Opcode - FirstOp; + const unsigned OpcodeIdx = getOpcodeIdxForOpcode(Aspect.Opcode); const unsigned TypeIdx = Aspect.Idx; if (TypeIdx >= ScalarInVectorActions[OpcodeIdx].size()) return {NotFound, Aspect.Type}; diff --git a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp index 51a2bbb6ef7..16821fbe3d0 100644 --- a/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64LegalizerInfo.cpp @@ -24,110 +24,7 @@ using namespace llvm; using namespace LegalizeActions; - -/// FIXME: The following static functions are SizeChangeStrategy functions -/// that are meant to temporarily mimic the behaviour of the old legalization -/// based on doubling/halving non-legal types as closely as possible. This is -/// not entirly possible as only legalizing the types that are exactly a power -/// of 2 times the size of the legal types would require specifying all those -/// sizes explicitly. -/// In practice, not specifying those isn't a problem, and the below functions -/// should disappear quickly as we add support for legalizing non-power-of-2 -/// sized types further. -static void -addAndInterleaveWithUnsupported(LegalizerInfo::SizeAndActionsVec &result, - const LegalizerInfo::SizeAndActionsVec &v) { - for (unsigned i = 0; i < v.size(); ++i) { - result.push_back(v[i]); - if (i + 1 < v[i].first && i + 1 < v.size() && - v[i + 1].first != v[i].first + 1) - result.push_back({v[i].first + 1, Unsupported}); - } -} - -static LegalizerInfo::SizeAndActionsVec -widen_1_narrow_128_ToLargest(const LegalizerInfo::SizeAndActionsVec &v) { - assert(v.size() >= 1); - assert(v[0].first > 2); - LegalizerInfo::SizeAndActionsVec result = {{1, WidenScalar}, - {2, Unsupported}}; - addAndInterleaveWithUnsupported(result, v); - auto Largest = result.back().first; - assert(Largest + 1 < 128); - result.push_back({Largest + 1, Unsupported}); - result.push_back({128, NarrowScalar}); - result.push_back({129, Unsupported}); - return result; -} - -static LegalizerInfo::SizeAndActionsVec -widen_16(const LegalizerInfo::SizeAndActionsVec &v) { - assert(v.size() >= 1); - assert(v[0].first > 17); - LegalizerInfo::SizeAndActionsVec result = {{1, Unsupported}, - {16, WidenScalar}, - {17, Unsupported}}; - addAndInterleaveWithUnsupported(result, v); - auto Largest = result.back().first; - result.push_back({Largest + 1, Unsupported}); - return result; -} - -static LegalizerInfo::SizeAndActionsVec -widen_1_8(const LegalizerInfo::SizeAndActionsVec &v) { - assert(v.size() >= 1); - assert(v[0].first > 9); - LegalizerInfo::SizeAndActionsVec result = { - {1, WidenScalar}, {2, Unsupported}, - {8, WidenScalar}, {9, Unsupported}}; - addAndInterleaveWithUnsupported(result, v); - auto Largest = result.back().first; - result.push_back({Largest + 1, Unsupported}); - return result; -} - -static LegalizerInfo::SizeAndActionsVec -widen_1_8_16(const LegalizerInfo::SizeAndActionsVec &v) { - assert(v.size() >= 1); - assert(v[0].first > 17); - LegalizerInfo::SizeAndActionsVec result = { - {1, WidenScalar}, {2, Unsupported}, - {8, WidenScalar}, {9, Unsupported}, - {16, WidenScalar}, {17, Unsupported}}; - addAndInterleaveWithUnsupported(result, v); - auto Largest = result.back().first; - result.push_back({Largest + 1, Unsupported}); - return result; -} - -static LegalizerInfo::SizeAndActionsVec -widen_1_8_16_narrowToLargest(const LegalizerInfo::SizeAndActionsVec &v) { - assert(v.size() >= 1); - assert(v[0].first > 17); - LegalizerInfo::SizeAndActionsVec result = { - {1, WidenScalar}, {2, Unsupported}, - {8, WidenScalar}, {9, Unsupported}, - {16, WidenScalar}, {17, Unsupported}}; - addAndInterleaveWithUnsupported(result, v); - auto Largest = result.back().first; - result.push_back({Largest + 1, NarrowScalar}); - return result; -} - -static LegalizerInfo::SizeAndActionsVec -widen_1_8_16_32(const LegalizerInfo::SizeAndActionsVec &v) { - assert(v.size() >= 1); - assert(v[0].first > 33); - LegalizerInfo::SizeAndActionsVec result = { - {1, WidenScalar}, {2, Unsupported}, - {8, WidenScalar}, {9, Unsupported}, - {16, WidenScalar}, {17, Unsupported}, - {32, WidenScalar}, {33, Unsupported}}; - addAndInterleaveWithUnsupported(result, v); - auto Largest = result.back().first; - result.push_back({Largest + 1, Unsupported}); - return result; -} +using namespace LegalityPredicates; AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { using namespace TargetOpcode; @@ -138,45 +35,51 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { const LLT s32 = LLT::scalar(32); const LLT s64 = LLT::scalar(64); const LLT s128 = LLT::scalar(128); + const LLT s256 = LLT::scalar(256); + const LLT s512 = LLT::scalar(512); + const LLT v16s8 = LLT::vector(16, 8); + const LLT v8s8 = LLT::vector(8, 8); + const LLT v4s8 = LLT::vector(4, 8); + const LLT v8s16 = LLT::vector(8, 16); + const LLT v4s16 = LLT::vector(4, 16); + const LLT v2s16 = LLT::vector(2, 16); const LLT v2s32 = LLT::vector(2, 32); const LLT v4s32 = LLT::vector(4, 32); const LLT v2s64 = LLT::vector(2, 64); - for (auto Ty : {p0, s1, s8, s16, s32, s64}) - setAction({G_IMPLICIT_DEF, Ty}, Legal); + getActionDefinitionsBuilder(G_IMPLICIT_DEF) + .legalFor({p0, s1, s8, s16, s32, s64}) + .clampScalar(0, s1, s64) + .widenScalarToNextPow2(0, 8); - for (auto Ty : {s16, s32, s64, p0}) - setAction({G_PHI, Ty}, Legal); + getActionDefinitionsBuilder(G_PHI) + .legalFor({p0, s16, s32, s64}) + .clampScalar(0, s16, s64) + .widenScalarToNextPow2(0); - setLegalizeScalarToDifferentSizeStrategy(G_PHI, 0, widen_1_8); + getActionDefinitionsBuilder(G_BSWAP) + .legalFor({s32, s64}) + .clampScalar(0, s16, s64) + .widenScalarToNextPow2(0); - for (auto Ty : { s32, s64 }) - setAction({G_BSWAP, Ty}, Legal); + getActionDefinitionsBuilder({G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR, G_SHL}) + .legalFor({s32, s64, v2s32, v4s32, v2s64}) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0) + .clampNumElements(0, v2s32, v4s32) + .clampNumElements(0, v2s64, v2s64) + .moreElementsToNextPow2(0); - for (unsigned BinOp : {G_ADD, G_SUB, G_MUL, G_AND, G_OR, G_XOR, G_SHL}) { - // These operations naturally get the right answer when used on - // GPR32, even if the actual type is narrower. - for (auto Ty : {s32, s64, v2s32, v4s32, v2s64}) - setAction({BinOp, Ty}, Legal); + getActionDefinitionsBuilder(G_GEP) + .legalFor({{p0, s64}}) + .clampScalar(1, s64, s64); - if (BinOp != G_ADD) - setLegalizeScalarToDifferentSizeStrategy(BinOp, 0, - widen_1_8_16_narrowToLargest); - } + getActionDefinitionsBuilder(G_PTR_MASK).legalFor({p0}); - setAction({G_GEP, p0}, Legal); - setAction({G_GEP, 1, s64}, Legal); - - setLegalizeScalarToDifferentSizeStrategy(G_GEP, 1, widen_1_8_16_32); - - setAction({G_PTR_MASK, p0}, Legal); - - for (unsigned BinOp : {G_LSHR, G_ASHR, G_SDIV, G_UDIV}) { - for (auto Ty : {s32, s64}) - setAction({BinOp, Ty}, Legal); - - setLegalizeScalarToDifferentSizeStrategy(BinOp, 0, widen_1_8_16); - } + getActionDefinitionsBuilder({G_LSHR, G_ASHR, G_SDIV, G_UDIV}) + .legalFor({s32, s64}) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0); for (unsigned BinOp : {G_SREM, G_UREM}) for (auto Ty : { s1, s8, s16, s32, s64 }) @@ -187,204 +90,259 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST) { setAction({Op, 1, s1}, Legal); } - for (unsigned Op : {G_UADDE, G_USUBE, G_SADDO, G_SSUBO, G_SMULH, G_UMULH}) { - for (auto Ty : { s32, s64 }) - setAction({Op, Ty}, Legal); - - setAction({Op, 1, s1}, Legal); - } - - for (unsigned BinOp : {G_FADD, G_FSUB, G_FMA, G_FMUL, G_FDIV}) - for (auto Ty : {s32, s64}) - setAction({BinOp, Ty}, Legal); - - for (unsigned BinOp : {G_FREM, G_FPOW}) { - setAction({BinOp, s32}, Libcall); - setAction({BinOp, s64}, Libcall); - } - - for (auto Ty : {s32, s64, p0}) { - setAction({G_INSERT, Ty}, Legal); - setAction({G_INSERT, 1, Ty}, Legal); - } - setLegalizeScalarToDifferentSizeStrategy(G_INSERT, 0, - widen_1_8_16_narrowToLargest); - for (auto Ty : {s1, s8, s16}) { - setAction({G_INSERT, 1, Ty}, Legal); - // FIXME: Can't widen the sources because that violates the constraints on - // G_INSERT (It seems entirely reasonable that inputs shouldn't overlap). - } - - for (auto Ty : {s1, s8, s16, s32, s64, p0}) - setAction({G_EXTRACT, Ty}, Legal); - - for (auto Ty : {s32, s64}) - setAction({G_EXTRACT, 1, Ty}, Legal); - - for (unsigned MemOp : {G_LOAD, G_STORE}) { - for (auto Ty : {s8, s16, s32, s64, p0, v2s32}) - setAction({MemOp, Ty}, Legal); - - setLegalizeScalarToDifferentSizeStrategy(MemOp, 0, - widen_1_narrow_128_ToLargest); - - // And everything's fine in addrspace 0. - setAction({MemOp, 1, p0}, Legal); - } + getActionDefinitionsBuilder({G_SMULH, G_UMULH}).legalFor({s32, s64}); + + getActionDefinitionsBuilder({G_UADDE, G_USUBE, G_SADDO, G_SSUBO}) + .legalFor({{s32, s1}, {s64, s1}}); + + getActionDefinitionsBuilder({G_FADD, G_FSUB, G_FMA, G_FMUL, G_FDIV}) + .legalFor({s32, s64}); + + getActionDefinitionsBuilder({G_FREM, G_FPOW}).libcallFor({s32, s64}); + + getActionDefinitionsBuilder(G_INSERT) + .unsupportedIf([=](const LegalityQuery &Query) { + return Query.Types[0].getSizeInBits() <= Query.Types[1].getSizeInBits(); + }) + .legalIf([=](const LegalityQuery &Query) { + const LLT &Ty0 = Query.Types[0]; + const LLT &Ty1 = Query.Types[1]; + if (Ty0 != s32 && Ty0 != s64 && Ty0 != p0) + return false; + return isPowerOf2_32(Ty1.getSizeInBits()) && + (Ty1.getSizeInBits() == 1 || Ty1.getSizeInBits() >= 8); + }) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0) + .maxScalarIf(typeInSet(0, {s32}), 1, s16) + .maxScalarIf(typeInSet(0, {s64}), 1, s32) + .widenScalarToNextPow2(1); + + getActionDefinitionsBuilder(G_EXTRACT) + .unsupportedIf([=](const LegalityQuery &Query) { + return Query.Types[0].getSizeInBits() >= Query.Types[1].getSizeInBits(); + }) + .legalIf([=](const LegalityQuery &Query) { + const LLT &Ty0 = Query.Types[0]; + const LLT &Ty1 = Query.Types[1]; + if (Ty1 != s32 && Ty1 != s64) + return false; + if (Ty1 == p0) + return true; + return isPowerOf2_32(Ty0.getSizeInBits()) && + (Ty0.getSizeInBits() == 1 || Ty0.getSizeInBits() >= 8); + }) + .clampScalar(1, s32, s64) + .widenScalarToNextPow2(1) + .maxScalarIf(typeInSet(1, {s32}), 0, s16) + .maxScalarIf(typeInSet(1, {s64}), 0, s32) + .widenScalarToNextPow2(0); + + getActionDefinitionsBuilder({G_LOAD, G_STORE}) + .legalFor( + {{s8, p0}, {s16, p0}, {s32, p0}, {s64, p0}, {p0, p0}, {v2s32, p0}}) + .clampScalar(0, s8, s64) + .widenScalarToNextPow2(0) + .clampNumElements(0, v2s32, v2s32); // Constants - for (auto Ty : {s32, s64}) { - setAction({TargetOpcode::G_CONSTANT, Ty}, Legal); - setAction({TargetOpcode::G_FCONSTANT, Ty}, Legal); - } - - setAction({G_CONSTANT, p0}, Legal); - - setLegalizeScalarToDifferentSizeStrategy(G_CONSTANT, 0, widen_1_8_16); - setLegalizeScalarToDifferentSizeStrategy(G_FCONSTANT, 0, widen_16); - - setAction({G_ICMP, 1, s32}, Legal); - setAction({G_ICMP, 1, s64}, Legal); - setAction({G_ICMP, 1, p0}, Legal); - - setLegalizeScalarToDifferentSizeStrategy(G_ICMP, 0, widen_1_8_16); - setLegalizeScalarToDifferentSizeStrategy(G_FCMP, 0, widen_1_8_16); - setLegalizeScalarToDifferentSizeStrategy(G_ICMP, 1, widen_1_8_16); - - setAction({G_ICMP, s32}, Legal); - setAction({G_FCMP, s32}, Legal); - setAction({G_FCMP, 1, s32}, Legal); - setAction({G_FCMP, 1, s64}, Legal); + getActionDefinitionsBuilder(G_CONSTANT) + .legalFor({p0, s32, s64}) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0); + getActionDefinitionsBuilder(G_FCONSTANT) + .legalFor({s32, s64}) + .clampScalar(0, s32, s64); + + getActionDefinitionsBuilder(G_ICMP) + .legalFor({{s32, s32}, {s32, s64}, {s32, p0}}) + .clampScalar(0, s32, s32) + .clampScalar(1, s32, s64) + .widenScalarToNextPow2(1); + + getActionDefinitionsBuilder(G_FCMP) + .legalFor({{s32, s32}, {s32, s64}}) + .clampScalar(0, s32, s32) + .clampScalar(1, s32, s64) + .widenScalarToNextPow2(1); // Extensions - for (auto Ty : { s1, s8, s16, s32, s64 }) { - setAction({G_ZEXT, Ty}, Legal); - setAction({G_SEXT, Ty}, Legal); - setAction({G_ANYEXT, Ty}, Legal); - } + getActionDefinitionsBuilder({G_ZEXT, G_SEXT, G_ANYEXT}) + .legalFor({s1, s8, s16, s32, s64}) + .maxScalar(0, s64) + .widenScalarToNextPow2(0); // FP conversions - for (auto Ty : { s16, s32 }) { - setAction({G_FPTRUNC, Ty}, Legal); - setAction({G_FPEXT, 1, Ty}, Legal); - } - - for (auto Ty : { s32, s64 }) { - setAction({G_FPTRUNC, 1, Ty}, Legal); - setAction({G_FPEXT, Ty}, Legal); - } + getActionDefinitionsBuilder(G_FPTRUNC).legalFor( + {{s16, s32}, {s16, s64}, {s32, s64}}); + getActionDefinitionsBuilder(G_FPEXT).legalFor( + {{s32, s16}, {s64, s16}, {s64, s32}}); // Conversions - for (auto Ty : { s32, s64 }) { - setAction({G_FPTOSI, 0, Ty}, Legal); - setAction({G_FPTOUI, 0, Ty}, Legal); - setAction({G_SITOFP, 1, Ty}, Legal); - setAction({G_UITOFP, 1, Ty}, Legal); - } - setLegalizeScalarToDifferentSizeStrategy(G_FPTOSI, 0, widen_1_8_16); - setLegalizeScalarToDifferentSizeStrategy(G_FPTOUI, 0, widen_1_8_16); - setLegalizeScalarToDifferentSizeStrategy(G_SITOFP, 1, widen_1_8_16); - setLegalizeScalarToDifferentSizeStrategy(G_UITOFP, 1, widen_1_8_16); - - for (auto Ty : { s32, s64 }) { - setAction({G_FPTOSI, 1, Ty}, Legal); - setAction({G_FPTOUI, 1, Ty}, Legal); - setAction({G_SITOFP, 0, Ty}, Legal); - setAction({G_UITOFP, 0, Ty}, Legal); - } + getActionDefinitionsBuilder({G_FPTOSI, G_FPTOUI}) + .legalForCartesianProduct({s32, s64}) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0) + .clampScalar(1, s32, s64) + .widenScalarToNextPow2(1); + + getActionDefinitionsBuilder({G_SITOFP, G_UITOFP}) + .legalForCartesianProduct({s32, s64}) + .clampScalar(1, s32, s64) + .widenScalarToNextPow2(1) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0); // Control-flow - for (auto Ty : {s1, s8, s16, s32}) - setAction({G_BRCOND, Ty}, Legal); - setAction({G_BRINDIRECT, p0}, Legal); + getActionDefinitionsBuilder(G_BRCOND).legalFor({s1, s8, s16, s32}); + getActionDefinitionsBuilder(G_BRINDIRECT).legalFor({p0}); // Select - setLegalizeScalarToDifferentSizeStrategy(G_SELECT, 0, widen_1_8_16); - - for (auto Ty : {s32, s64, p0}) - setAction({G_SELECT, Ty}, Legal); - - setAction({G_SELECT, 1, s1}, Legal); + getActionDefinitionsBuilder(G_SELECT) + .legalFor({{s32, s1}, {s64, s1}, {p0, s1}}) + .clampScalar(0, s32, s64) + .widenScalarToNextPow2(0); // Pointer-handling - setAction({G_FRAME_INDEX, p0}, Legal); - setAction({G_GLOBAL_VALUE, p0}, Legal); + getActionDefinitionsBuilder(G_FRAME_INDEX).legalFor({p0}); + getActionDefinitionsBuilder(G_GLOBAL_VALUE).legalFor({p0}); - for (auto Ty : {s1, s8, s16, s32, s64}) - setAction({G_PTRTOINT, 0, Ty}, Legal); + getActionDefinitionsBuilder(G_PTRTOINT) + .legalForCartesianProduct({s1, s8, s16, s32, s64}, {p0}) + .maxScalar(0, s64) + .widenScalarToNextPow2(0, /*Min*/ 8); - setAction({G_PTRTOINT, 1, p0}, Legal); - - setAction({G_INTTOPTR, 0, p0}, Legal); - setAction({G_INTTOPTR, 1, s64}, Legal); + getActionDefinitionsBuilder(G_INTTOPTR) + .unsupportedIf([&](const LegalityQuery &Query) { + return Query.Types[0].getSizeInBits() != Query.Types[1].getSizeInBits(); + }) + .legalFor({s64, p0}); // Casts for 32 and 64-bit width type are just copies. // Same for 128-bit width type, except they are on the FPR bank. - for (auto Ty : {s1, s8, s16, s32, s64, s128}) { - setAction({G_BITCAST, 0, Ty}, Legal); - setAction({G_BITCAST, 1, Ty}, Legal); - } - - // For the sake of copying bits around, the type does not really - // matter as long as it fits a register. - for (int EltSize = 8; EltSize <= 64; EltSize *= 2) { - setAction({G_BITCAST, 0, LLT::vector(128/EltSize, EltSize)}, Legal); - setAction({G_BITCAST, 1, LLT::vector(128/EltSize, EltSize)}, Legal); - if (EltSize >= 64) - continue; - - setAction({G_BITCAST, 0, LLT::vector(64/EltSize, EltSize)}, Legal); - setAction({G_BITCAST, 1, LLT::vector(64/EltSize, EltSize)}, Legal); - if (EltSize >= 32) - continue; - - setAction({G_BITCAST, 0, LLT::vector(32/EltSize, EltSize)}, Legal); - setAction({G_BITCAST, 1, LLT::vector(32/EltSize, EltSize)}, Legal); - } + getActionDefinitionsBuilder(G_BITCAST) + // FIXME: This is wrong since G_BITCAST is not allowed to change the + // number of bits but it's what the previous code described and fixing + // it breaks tests. + .legalForCartesianProduct({s1, s8, s16, s32, s64, s128, v16s8, v8s8, v4s8, + v8s16, v4s16, v2s16, v4s32, v2s32, v2s64}); - setAction({G_VASTART, p0}, Legal); + getActionDefinitionsBuilder(G_VASTART).legalFor({p0}); // va_list must be a pointer, but most sized types are pretty easy to handle // as the destination. - setAction({G_VAARG, 1, p0}, Legal); + getActionDefinitionsBuilder(G_VAARG) + .customForCartesianProduct({s8, s16, s32, s64, p0}, {p0}) + .clampScalar(0, s8, s64) + .widenScalarToNextPow2(0, /*Min*/ 8); - for (auto Ty : {s8, s16, s32, s64, p0}) - setAction({G_VAARG, Ty}, Custom); + if (ST.hasLSE()) { + getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG) + .legalForCartesianProduct({s8, s16, s32, s64}, {p0}); + } + getActionDefinitionsBuilder(G_ATOMIC_CMPXCHG); if (ST.hasLSE()) { for (auto Ty : {s8, s16, s32, s64}) { setAction({G_ATOMIC_CMPXCHG_WITH_SUCCESS, Ty}, Lower); - setAction({G_ATOMIC_CMPXCHG, Ty}, Legal); - } - setAction({G_ATOMIC_CMPXCHG, 1, p0}, Legal); - - for (unsigned Op : - {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND, - G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, - G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX}) { - for (auto Ty : {s8, s16, s32, s64}) { - setAction({Op, Ty}, Legal); - } - setAction({Op, 1, p0}, Legal); } + + getActionDefinitionsBuilder( + {G_ATOMICRMW_XCHG, G_ATOMICRMW_ADD, G_ATOMICRMW_SUB, G_ATOMICRMW_AND, + G_ATOMICRMW_OR, G_ATOMICRMW_XOR, G_ATOMICRMW_MIN, G_ATOMICRMW_MAX, + G_ATOMICRMW_UMIN, G_ATOMICRMW_UMAX}) + .legalForCartesianProduct({s8, s16, s32, s64}, {p0}); } // Merge/Unmerge - for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) - for (int Sz : {8, 16, 32, 64, 128, 192, 256, 384, 512}) { - LLT ScalarTy = LLT::scalar(Sz); - setAction({Op, ScalarTy}, Legal); - setAction({Op, 1, ScalarTy}, Legal); - if (Sz < 32) - continue; - for (int EltSize = 8; EltSize <= 64; EltSize *= 2) { - if (EltSize >= Sz) - continue; - LLT VecTy = LLT::vector(Sz / EltSize, EltSize); - setAction({Op, VecTy}, Legal); - setAction({Op, 1, VecTy}, Legal); + for (unsigned Op : {G_MERGE_VALUES, G_UNMERGE_VALUES}) { + unsigned BigTyIdx = Op == G_MERGE_VALUES ? 0 : 1; + unsigned LitTyIdx = Op == G_MERGE_VALUES ? 1 : 0; + + auto notValidElt = [](const LegalityQuery &Query, unsigned TypeIdx) { + const LLT &Ty = Query.Types[TypeIdx]; + if (Ty.isVector()) { + const LLT &EltTy = Ty.getElementType(); + if (EltTy.getSizeInBits() < 8 || EltTy.getSizeInBits() > 64) + return true; + if (!isPowerOf2_32(EltTy.getSizeInBits())) + return true; } - } + return false; + }; + auto scalarize = + [](const LegalityQuery &Query, unsigned TypeIdx) { + const LLT &Ty = Query.Types[TypeIdx]; + return std::make_pair(TypeIdx, Ty.getElementType()); + }; + + // FIXME: This rule is horrible, but specifies the same as what we had + // before with the particularly strange definitions removed (e.g. + // s8 = G_MERGE_VALUES s32, s32). + // Part of the complexity comes from these ops being extremely flexible. For + // example, you can build/decompose vectors with it, concatenate vectors, + // etc. and in addition to this you can also bitcast with it at the same + // time. We've been considering breaking it up into multiple ops to make it + // more manageable throughout the backend. + getActionDefinitionsBuilder(Op) + // Break up vectors with weird elements into scalars + .fewerElementsIf( + [=](const LegalityQuery &Query) { return notValidElt(Query, 0); }, + [=](const LegalityQuery &Query) { return scalarize(Query, 0); }) + .fewerElementsIf( + [=](const LegalityQuery &Query) { return notValidElt(Query, 1); }, + [=](const LegalityQuery &Query) { return scalarize(Query, 1); }) + // Clamp the big scalar to s8-s512 and make it either a power of 2, 192, + // or 384. + .clampScalar(BigTyIdx, s8, s512) + .widenScalarIf( + [=](const LegalityQuery &Query) { + const LLT &Ty = Query.Types[BigTyIdx]; + return !isPowerOf2_32(Ty.getSizeInBits()) && + Ty.getSizeInBits() % 64 != 0; + }, + [=](const LegalityQuery &Query) { + // Pick the next power of 2, or a multiple of 64 over 128. + // Whichever is smaller. + const LLT &Ty = Query.Types[BigTyIdx]; + unsigned NewSizeInBits = 1 + << Log2_32_Ceil(Ty.getSizeInBits() + 1); + if (NewSizeInBits >= 256) { + unsigned RoundedTo = alignTo<64>(Ty.getSizeInBits() + 1); + if (RoundedTo < NewSizeInBits) + NewSizeInBits = RoundedTo; + } + return std::make_pair(BigTyIdx, LLT::scalar(NewSizeInBits)); + }) + // Clamp the little scalar to s8-s256 and make it a power of 2. It's not + // worth considering the multiples of 64 since 2*192 and 2*384 are not + // valid. + .clampScalar(LitTyIdx, s8, s256) + .widenScalarToNextPow2(LitTyIdx, /*Min*/ 8) + // So at this point, we have s8, s16, s32, s64, s128, s192, s256, s384, + // s512, <X x s8>, <X x s16>, <X x s32>, or <X x s64>. + // At this point it's simple enough to accept the legal types. + .legalIf([=](const LegalityQuery &Query) { + const LLT &BigTy = Query.Types[BigTyIdx]; + const LLT &LitTy = Query.Types[LitTyIdx]; + if (BigTy.isVector() && BigTy.getSizeInBits() < 32) + return false; + if (LitTy.isVector() && LitTy.getSizeInBits() < 32) + return false; + return BigTy.getSizeInBits() % LitTy.getSizeInBits() == 0; + }) + // Any vectors left are the wrong size. Scalarize them. + .fewerElementsIf([](const LegalityQuery &Query) { return true; }, + [](const LegalityQuery &Query) { + return std::make_pair( + 0, Query.Types[0].getElementType()); + }) + .fewerElementsIf([](const LegalityQuery &Query) { return true; }, + [](const LegalityQuery &Query) { + return std::make_pair( + 1, Query.Types[1].getElementType()); + }); + } computeTables(); } diff --git a/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp b/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp index 876fc5465bf..a661d92658b 100644 --- a/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp +++ b/llvm/unittests/CodeGen/GlobalISel/LegalizerInfoTest.cpp @@ -29,6 +29,7 @@ operator<<(std::ostream &OS, const LegalizeAction Act) { case Custom: OS << "Custom"; break; case Unsupported: OS << "Unsupported"; break; case NotFound: OS << "NotFound"; + case UseLegacyRules: OS << "UseLegacyRules"; break; } return OS; } @@ -62,34 +63,28 @@ TEST(LegalizerInfoTest, ScalarRISC) { for (unsigned opcode : {G_ADD, G_SUB}) { // Check we infer the correct types and actually do what we're told. ASSERT_EQ(L.getAction({opcode, {LLT::scalar(8)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); ASSERT_EQ(L.getAction({opcode, {LLT::scalar(16)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); ASSERT_EQ(L.getAction({opcode, {LLT::scalar(32)}}), - LegalizerInfo::LegalizeActionStep(Legal, 0, LLT{})); + LegalizeActionStep(Legal, 0, LLT{})); ASSERT_EQ(L.getAction({opcode, {LLT::scalar(64)}}), - LegalizerInfo::LegalizeActionStep(Legal, 0, LLT{})); + LegalizeActionStep(Legal, 0, LLT{})); // Make sure the default for over-sized types applies. - ASSERT_EQ( - L.getAction({opcode, {LLT::scalar(128)}}), - LegalizerInfo::LegalizeActionStep(NarrowScalar, 0, LLT::scalar(64))); + ASSERT_EQ(L.getAction({opcode, {LLT::scalar(128)}}), + LegalizeActionStep(NarrowScalar, 0, LLT::scalar(64))); // Make sure we also handle unusual sizes - ASSERT_EQ( - L.getAction({opcode, {LLT::scalar(1)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); - ASSERT_EQ( - L.getAction({opcode, {LLT::scalar(31)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); - ASSERT_EQ( - L.getAction({opcode, {LLT::scalar(33)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(64))); - ASSERT_EQ( - L.getAction({opcode, {LLT::scalar(63)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(64))); - ASSERT_EQ( - L.getAction({opcode, {LLT::scalar(65)}}), - LegalizerInfo::LegalizeActionStep(NarrowScalar, 0, LLT::scalar(64))); + ASSERT_EQ(L.getAction({opcode, {LLT::scalar(1)}}), + LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); + ASSERT_EQ(L.getAction({opcode, {LLT::scalar(31)}}), + LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); + ASSERT_EQ(L.getAction({opcode, {LLT::scalar(33)}}), + LegalizeActionStep(WidenScalar, 0, LLT::scalar(64))); + ASSERT_EQ(L.getAction({opcode, {LLT::scalar(63)}}), + LegalizeActionStep(WidenScalar, 0, LLT::scalar(64))); + ASSERT_EQ(L.getAction({opcode, {LLT::scalar(65)}}), + LegalizeActionStep(NarrowScalar, 0, LLT::scalar(64))); } } @@ -114,23 +109,18 @@ TEST(LegalizerInfoTest, VectorRISC) { // Check we infer the correct types and actually do what we're told for some // simple cases. ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(8, 8)}}), - LegalizerInfo::LegalizeActionStep(Legal, 0, LLT{})); - ASSERT_EQ( - L.getAction({G_ADD, {LLT::vector(8, 7)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::vector(8, 8))); - ASSERT_EQ( - L.getAction({G_ADD, {LLT::vector(2, 8)}}), - LegalizerInfo::LegalizeActionStep(MoreElements, 0, LLT::vector(8, 8))); - ASSERT_EQ( - L.getAction({G_ADD, {LLT::vector(8, 32)}}), - LegalizerInfo::LegalizeActionStep(FewerElements, 0, LLT::vector(4, 32))); + LegalizeActionStep(Legal, 0, LLT{})); + ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(8, 7)}}), + LegalizeActionStep(WidenScalar, 0, LLT::vector(8, 8))); + ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(2, 8)}}), + LegalizeActionStep(MoreElements, 0, LLT::vector(8, 8))); + ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(8, 32)}}), + LegalizeActionStep(FewerElements, 0, LLT::vector(4, 32))); // Check a few non-power-of-2 sizes: - ASSERT_EQ( - L.getAction({G_ADD, {LLT::vector(3, 3)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::vector(3, 8))); - ASSERT_EQ( - L.getAction({G_ADD, {LLT::vector(3, 8)}}), - LegalizerInfo::LegalizeActionStep(MoreElements, 0, LLT::vector(8, 8))); + ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(3, 3)}}), + LegalizeActionStep(WidenScalar, 0, LLT::vector(3, 8))); + ASSERT_EQ(L.getAction({G_ADD, {LLT::vector(3, 8)}}), + LegalizeActionStep(MoreElements, 0, LLT::vector(8, 8))); } TEST(LegalizerInfoTest, MultipleTypes) { @@ -150,15 +140,15 @@ TEST(LegalizerInfoTest, MultipleTypes) { // Check we infer the correct types and actually do what we're told. ASSERT_EQ(L.getAction({G_PTRTOINT, {s64, p0}}), - LegalizerInfo::LegalizeActionStep(Legal, 0, LLT{})); + LegalizeActionStep(Legal, 0, LLT{})); // Make sure we also handle unusual sizes ASSERT_EQ( L.getAction({G_PTRTOINT, {LLT::scalar(65), s64}}), - LegalizerInfo::LegalizeActionStep(NarrowScalar, 0, s64)); + LegalizeActionStep(NarrowScalar, 0, s64)); ASSERT_EQ( L.getAction({G_PTRTOINT, {s64, LLT::pointer(0, 32)}}), - LegalizerInfo::LegalizeActionStep(Unsupported, 1, LLT::pointer(0, 32))); + LegalizeActionStep(Unsupported, 1, LLT::pointer(0, 32))); } TEST(LegalizerInfoTest, MultipleSteps) { @@ -175,9 +165,9 @@ TEST(LegalizerInfoTest, MultipleSteps) { L.computeTables(); ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(16)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(32)}}), - LegalizerInfo::LegalizeActionStep(Lower, 0, LLT::scalar(32))); + LegalizeActionStep(Lower, 0, LLT::scalar(32))); } TEST(LegalizerInfoTest, SizeChangeStrategy) { @@ -193,19 +183,19 @@ TEST(LegalizerInfoTest, SizeChangeStrategy) { // Check we infer the correct types and actually do what we're told. for (unsigned Size : {1, 8, 16, 32}) { ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(Size)}}), - LegalizerInfo::LegalizeActionStep(Legal, 0, LLT{})); + LegalizeActionStep(Legal, 0, LLT{})); } ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(2)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(8))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(8))); ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(7)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(8))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(8))); ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(9)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(16))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(16))); ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(17)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(31)}}), - LegalizerInfo::LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); + LegalizeActionStep(WidenScalar, 0, LLT::scalar(32))); ASSERT_EQ(L.getAction({G_UREM, {LLT::scalar(33)}}), - LegalizerInfo::LegalizeActionStep(Unsupported, 0, LLT::scalar(33))); + LegalizeActionStep(Unsupported, 0, LLT::scalar(33))); } } |
