From 5a2583f0265007d27624e1089d40bf4b51f8718a Mon Sep 17 00:00:00 2001 From: Abderrazek Zaafrani Date: Fri, 8 Dec 2017 22:04:13 +0000 Subject: [AArch64] Rename AArch64VecorByElementOpt.cpp into AArch64SIMDInstrOpt.cpp to reflect the recently added features. The name change is dicsussed in https://reviews.llvm.org/D38196 llvm-svn: 320204 --- llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp | 735 +++++++++++++++++++++ .../Target/AArch64/AArch64VectorByElementOpt.cpp | 735 --------------------- llvm/lib/Target/AArch64/CMakeLists.txt | 2 +- 3 files changed, 736 insertions(+), 736 deletions(-) create mode 100644 llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp delete mode 100644 llvm/lib/Target/AArch64/AArch64VectorByElementOpt.cpp diff --git a/llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp b/llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp new file mode 100644 index 00000000000..3ff4155849a --- /dev/null +++ b/llvm/lib/Target/AArch64/AArch64SIMDInstrOpt.cpp @@ -0,0 +1,735 @@ +// +// The LLVM Compiler Infrastructure +// +// This file is distributed under the University of Illinois Open Source +// License. See LICENSE.TXT for details. +// +//===----------------------------------------------------------------------===// +// +// This file contains a pass that performs optimization on SIMD instructions +// with high latency by splitting them into more efficient series of +// instructions. +// +// 1. Rewrite certain SIMD instructions with vector element due to their +// inefficiency on some targets. +// Example: +// fmla v0.4s, v1.4s, v2.s[1] +// is rewritten into +// dup v3.4s, v2.s[1] +// fmla v0.4s, v1.4s, v3.4s +// +// 2. Rewrite Interleaved memory access instructions due to their +// inefficiency on some targets. +// Example: +// st2 {v0.4s, v1.4s}, addr +// is rewritten into +// zip1 v2.4s, v0.4s, v1.4s +// zip2 v3.4s, v0.4s, v1.4s +// stp q2, q3, addr +// +//===----------------------------------------------------------------------===// + +#include "AArch64InstrInfo.h" +#include "llvm/ADT/SmallVector.h" +#include "llvm/ADT/Statistic.h" +#include "llvm/ADT/StringRef.h" +#include "llvm/CodeGen/MachineBasicBlock.h" +#include "llvm/CodeGen/MachineFunction.h" +#include "llvm/CodeGen/MachineFunctionPass.h" +#include "llvm/CodeGen/MachineInstr.h" +#include "llvm/CodeGen/MachineInstrBuilder.h" +#include "llvm/CodeGen/MachineOperand.h" +#include "llvm/CodeGen/MachineRegisterInfo.h" +#include "llvm/CodeGen/TargetInstrInfo.h" +#include "llvm/CodeGen/TargetSchedule.h" +#include "llvm/CodeGen/TargetSubtargetInfo.h" +#include "llvm/MC/MCInstrDesc.h" +#include "llvm/MC/MCSchedule.h" +#include "llvm/Pass.h" +#include + +using namespace llvm; + +#define DEBUG_TYPE "aarch64-simdinstr-opt" + +STATISTIC(NumModifiedInstr, + "Number of SIMD instructions modified"); + +#define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \ + "AArch64 SIMD instructions optimization pass" + +namespace { + +struct AArch64SIMDInstrOpt : public MachineFunctionPass { + static char ID; + + const TargetInstrInfo *TII; + MachineRegisterInfo *MRI; + TargetSchedModel SchedModel; + + // The two maps below are used to cache decisions instead of recomputing: + // This is used to cache instruction replacement decisions within function + // units and across function units. + std::map, bool> SIMDInstrTable; + // This is used to cache the decision of whether to leave the Interleave-Store + // instructions replacement pass early or not for a particular target. + std::unordered_map InterlEarlyExit; + + typedef enum { + VectorElem, + Interleave + } Subpass; + + // Instruction represented by OrigOpc is replaced by instructions in ReplOpc. + struct InstReplInfo { + unsigned OrigOpc; + std::vector ReplOpc; + const TargetRegisterClass RC; + }; + +#define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \ + {OpcOrg, {OpcR0, OpcR1, OpcR2}, RC} +#define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \ + OpcR7, OpcR8, OpcR9, RC) \ + {OpcOrg, {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, \ + OpcR8, OpcR9}, RC} + + // The Instruction Replacement Table: + std::vector IRT = { + // ST2 instructions + RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, + AArch64::STPQi, AArch64::FPR128RegClass), + RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, + AArch64::STPQi, AArch64::FPR128RegClass), + RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, + AArch64::STPDi, AArch64::FPR64RegClass), + RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, + AArch64::STPQi, AArch64::FPR128RegClass), + RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, + AArch64::STPDi, AArch64::FPR64RegClass), + RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, + AArch64::STPQi, AArch64::FPR128RegClass), + RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, + AArch64::STPDi, AArch64::FPR64RegClass), + // ST4 instructions + RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, + AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, + AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, + AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), + RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, + AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, + AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, + AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), + RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, + AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, + AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, + AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), + RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, + AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, + AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, + AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), + RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, + AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, + AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, + AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), + RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, + AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, + AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, + AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), + RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, + AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, + AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, + AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass) + }; + + // A costly instruction is replaced in this work by N efficient instructions + // The maximum of N is curently 10 and it is for ST4 case. + static const unsigned MaxNumRepl = 10; + + AArch64SIMDInstrOpt() : MachineFunctionPass(ID) { + initializeAArch64SIMDInstrOptPass(*PassRegistry::getPassRegistry()); + } + + /// Based only on latency of instructions, determine if it is cost efficient + /// to replace the instruction InstDesc by the instructions stored in the + /// array InstDescRepl. + /// Return true if replacement is expected to be faster. + bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, + SmallVectorImpl &ReplInstrMCID); + + /// Determine if we need to exit the instruction replacement optimization + /// subpasses early. This makes sure that Targets with no need for this + /// optimization do not spend any compile time on this subpass other than the + /// simple check performed here. This simple check is done by comparing the + /// latency of the original instruction to the latency of the replacement + /// instructions. We only check for a representative instruction in the class + /// of instructions and not all concerned instructions. For the VectorElem + /// subpass, we check for the FMLA instruction while for the interleave subpass + /// we check for the st2.4s instruction. + /// Return true if early exit of the subpass is recommended. + bool shouldExitEarly(MachineFunction *MF, Subpass SP); + + /// Check whether an equivalent DUP instruction has already been + /// created or not. + /// Return true when the dup instruction already exists. In this case, + /// DestReg will point to the destination of the already created DUP. + bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg, + unsigned LaneNumber, unsigned *DestReg) const; + + /// Certain SIMD instructions with vector element operand are not efficient. + /// Rewrite them into SIMD instructions with vector operands. This rewrite + /// is driven by the latency of the instructions. + /// Return true if the SIMD instruction is modified. + bool optimizeVectElement(MachineInstr &MI); + + /// Process The REG_SEQUENCE instruction, and extract the source + /// operands of the st2/4 instruction from it. + /// Example of such instructions. + /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; + /// Return true when the instruction is processed successfully. + bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg, + unsigned* StRegKill, unsigned NumArg) const; + + /// Load/Store Interleaving instructions are not always beneficial. + /// Replace them by zip instructionand classical load/store. + /// Return true if the SIMD instruction is modified. + bool optimizeLdStInterleave(MachineInstr &MI); + + /// Return the number of useful source registers for this + /// instruction (2 for st2 and 4 for st4). + unsigned determineSrcReg(MachineInstr &MI) const; + + bool runOnMachineFunction(MachineFunction &Fn) override; + + StringRef getPassName() const override { + return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME; + } +}; + +char AArch64SIMDInstrOpt::ID = 0; + +} // end anonymous namespace + +INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt", + AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false) + +/// Based only on latency of instructions, determine if it is cost efficient +/// to replace the instruction InstDesc by the instructions stored in the +/// array InstDescRepl. +/// Return true if replacement is expected to be faster. +bool AArch64SIMDInstrOpt:: +shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, + SmallVectorImpl &InstDescRepl) { + // Check if replacement decision is already available in the cached table. + // if so, return it. + std::string Subtarget = SchedModel.getSubtargetInfo()->getCPU(); + std::pair InstID = std::make_pair(InstDesc->getOpcode(), Subtarget); + if (!SIMDInstrTable.empty() && + SIMDInstrTable.find(InstID) != SIMDInstrTable.end()) + return SIMDInstrTable[InstID]; + + unsigned SCIdx = InstDesc->getSchedClass(); + const MCSchedClassDesc *SCDesc = + SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx); + + // If a subtarget does not define resources for the instructions + // of interest, then return false for no replacement. + const MCSchedClassDesc *SCDescRepl; + if (!SCDesc->isValid() || SCDesc->isVariant()) + { + SIMDInstrTable[InstID] = false; + return false; + } + for (auto IDesc : InstDescRepl) + { + SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc( + IDesc->getSchedClass()); + if (!SCDescRepl->isValid() || SCDescRepl->isVariant()) + { + SIMDInstrTable[InstID] = false; + return false; + } + } + + // Replacement cost. + unsigned ReplCost = 0; + for (auto IDesc :InstDescRepl) + ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode()); + + if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost) + { + SIMDInstrTable[InstID] = true; + return true; + } + else + { + SIMDInstrTable[InstID] = false; + return false; + } +} + +/// Determine if we need to exit the instruction replacement optimization +/// subpasses early. This makes sure that Targets with no need for this +/// optimization do not spend any compile time on this subpass other than the +/// simple check performed here. This simple check is done by comparing the +/// latency of the original instruction to the latency of the replacement +/// instructions. We only check for a representative instruction in the class of +/// instructions and not all concerned instructions. For the VectorElem subpass, +/// we check for the FMLA instruction while for the interleave subpass we check +/// for the st2.4s instruction. +/// Return true if early exit of the subpass is recommended. +bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) { + const MCInstrDesc* OriginalMCID; + SmallVector ReplInstrMCID; + + switch (SP) { + case VectorElem: + OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed); + ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane)); + ReplInstrMCID.push_back(&TII->get(AArch64::FMULv4f32)); + if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) + return false; + break; + case Interleave: + // Check if early exit decision is already available in the cached + // table or not. + std::string Subtarget = SchedModel.getSubtargetInfo()->getCPU(); + if (InterlEarlyExit.find(Subtarget) != InterlEarlyExit.end()) + return InterlEarlyExit[Subtarget]; + + for (auto &I : IRT) { + OriginalMCID = &TII->get(I.OrigOpc); + for (auto &Repl : I.ReplOpc) + ReplInstrMCID.push_back(&TII->get(Repl)); + if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) { + InterlEarlyExit[Subtarget] = false; + return false; + } + ReplInstrMCID.clear(); + } + InterlEarlyExit[Subtarget] = true; + break; + } + + return true; +} + +/// Check whether an equivalent DUP instruction has already been +/// created or not. +/// Return true when the dup instruction already exists. In this case, +/// DestReg will point to the destination of the already created DUP. +bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode, + unsigned SrcReg, unsigned LaneNumber, + unsigned *DestReg) const { + for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin(); + MII != MIE;) { + MII--; + MachineInstr *CurrentMI = &*MII; + + if (CurrentMI->getOpcode() == DupOpcode && + CurrentMI->getNumOperands() == 3 && + CurrentMI->getOperand(1).getReg() == SrcReg && + CurrentMI->getOperand(2).getImm() == LaneNumber) { + *DestReg = CurrentMI->getOperand(0).getReg(); + return true; + } + } + + return false; +} + +/// Certain SIMD instructions with vector element operand are not efficient. +/// Rewrite them into SIMD instructions with vector operands. This rewrite +/// is driven by the latency of the instructions. +/// The instruction of concerns are for the time being fmla, fmls, fmul, +/// and fmulx and hence they are hardcoded. +/// +/// Example: +/// fmla v0.4s, v1.4s, v2.s[1] +/// is rewritten into +/// dup v3.4s, v2.s[1] // dup not necessary if redundant +/// fmla v0.4s, v1.4s, v3.4s +/// Return true if the SIMD instruction is modified. +bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) { + const MCInstrDesc *MulMCID, *DupMCID; + const TargetRegisterClass *RC = &AArch64::FPR128RegClass; + + switch (MI.getOpcode()) { + default: + return false; + + // 4X32 instructions + case AArch64::FMLAv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMLAv4f32); + break; + case AArch64::FMLSv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMLSv4f32); + break; + case AArch64::FMULXv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMULXv4f32); + break; + case AArch64::FMULv4i32_indexed: + DupMCID = &TII->get(AArch64::DUPv4i32lane); + MulMCID = &TII->get(AArch64::FMULv4f32); + break; + + // 2X64 instructions + case AArch64::FMLAv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMLAv2f64); + break; + case AArch64::FMLSv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMLSv2f64); + break; + case AArch64::FMULXv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMULXv2f64); + break; + case AArch64::FMULv2i64_indexed: + DupMCID = &TII->get(AArch64::DUPv2i64lane); + MulMCID = &TII->get(AArch64::FMULv2f64); + break; + + // 2X32 instructions + case AArch64::FMLAv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMLAv2f32); + break; + case AArch64::FMLSv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMLSv2f32); + break; + case AArch64::FMULXv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMULXv2f32); + break; + case AArch64::FMULv2i32_indexed: + RC = &AArch64::FPR64RegClass; + DupMCID = &TII->get(AArch64::DUPv2i32lane); + MulMCID = &TII->get(AArch64::FMULv2f32); + break; + } + + SmallVector ReplInstrMCID; + ReplInstrMCID.push_back(DupMCID); + ReplInstrMCID.push_back(MulMCID); + if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), + ReplInstrMCID)) + return false; + + const DebugLoc &DL = MI.getDebugLoc(); + MachineBasicBlock &MBB = *MI.getParent(); + MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); + + // get the operands of the current SIMD arithmetic instruction. + unsigned MulDest = MI.getOperand(0).getReg(); + unsigned SrcReg0 = MI.getOperand(1).getReg(); + unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill()); + unsigned SrcReg1 = MI.getOperand(2).getReg(); + unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill()); + unsigned DupDest; + + // Instructions of interest have either 4 or 5 operands. + if (MI.getNumOperands() == 5) { + unsigned SrcReg2 = MI.getOperand(3).getReg(); + unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill()); + unsigned LaneNumber = MI.getOperand(4).getImm(); + // Create a new DUP instruction. Note that if an equivalent DUP instruction + // has already been created before, then use that one instread of creating + // a new one. + if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) { + DupDest = MRI.createVirtualRegister(RC); + BuildMI(MBB, MI, DL, *DupMCID, DupDest) + .addReg(SrcReg2, Src2IsKill) + .addImm(LaneNumber); + } + BuildMI(MBB, MI, DL, *MulMCID, MulDest) + .addReg(SrcReg0, Src0IsKill) + .addReg(SrcReg1, Src1IsKill) + .addReg(DupDest, Src2IsKill); + } else if (MI.getNumOperands() == 4) { + unsigned LaneNumber = MI.getOperand(3).getImm(); + if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) { + DupDest = MRI.createVirtualRegister(RC); + BuildMI(MBB, MI, DL, *DupMCID, DupDest) + .addReg(SrcReg1, Src1IsKill) + .addImm(LaneNumber); + } + BuildMI(MBB, MI, DL, *MulMCID, MulDest) + .addReg(SrcReg0, Src0IsKill) + .addReg(DupDest, Src1IsKill); + } else { + return false; + } + + ++NumModifiedInstr; + return true; +} + +/// Load/Store Interleaving instructions are not always beneficial. +/// Replace them by zip instructions and classical load/store. +/// +/// Example: +/// st2 {v0.4s, v1.4s}, addr +/// is rewritten into +/// zip1 v2.4s, v0.4s, v1.4s +/// zip2 v3.4s, v0.4s, v1.4s +/// stp q2, q3, addr +// +/// Example: +/// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr +/// is rewritten into +/// zip1 v4.4s, v0.4s, v2.4s +/// zip2 v5.4s, v0.4s, v2.4s +/// zip1 v6.4s, v1.4s, v3.4s +/// zip2 v7.4s, v1.4s, v3.4s +/// zip1 v8.4s, v4.4s, v6.4s +/// zip2 v9.4s, v4.4s, v6.4s +/// zip1 v10.4s, v5.4s, v7.4s +/// zip2 v11.4s, v5.4s, v7.4s +/// stp q8, q9, addr +/// stp q10, q11, addr+32 +/// Currently only instructions related to st2 and st4 are considered. +/// Other may be added later. +/// Return true if the SIMD instruction is modified. +bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) { + + unsigned SeqReg, AddrReg; + unsigned StReg[4], StRegKill[4]; + MachineInstr *DefiningMI; + const DebugLoc &DL = MI.getDebugLoc(); + MachineBasicBlock &MBB = *MI.getParent(); + SmallVector ZipDest; + SmallVector ReplInstrMCID; + + // If current instruction matches any of the rewriting rules, then + // gather information about parameters of the new instructions. + bool Match = false; + for (auto &I : IRT) { + if (MI.getOpcode() == I.OrigOpc) { + SeqReg = MI.getOperand(0).getReg(); + AddrReg = MI.getOperand(1).getReg(); + DefiningMI = MRI->getUniqueVRegDef(SeqReg); + unsigned NumReg = determineSrcReg(MI); + if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg)) + return false; + + for (auto &Repl : I.ReplOpc) { + ReplInstrMCID.push_back(&TII->get(Repl)); + // Generate destination registers but only for non-store instruction. + if (Repl != AArch64::STPQi && Repl != AArch64::STPDi) + ZipDest.push_back(MRI->createVirtualRegister(&I.RC)); + } + Match = true; + break; + } + } + + if (!Match) + return false; + + // Determine if it is profitable to replace MI by the series of instructions + // represented in ReplInstrMCID. + if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), + ReplInstrMCID)) + return false; + + // Generate the replacement instructions composed of zip1, zip2, and stp (at + // this point, the code generation is hardcoded and does not rely on the IRT + // table used above given that code generation for ST2 replacement is somewhat + // different than for ST4 replacement. We could have added more info into the + // table related to how we build new instructions but we may be adding more + // complexity with that). + switch (MI.getOpcode()) { + default: + return false; + case AArch64::ST2Twov16b: + case AArch64::ST2Twov8b: + case AArch64::ST2Twov8h: + case AArch64::ST2Twov4h: + case AArch64::ST2Twov4s: + case AArch64::ST2Twov2s: + case AArch64::ST2Twov2d: + // zip instructions + BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) + .addReg(StReg[0]) + .addReg(StReg[1]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) + .addReg(StReg[0], StRegKill[0]) + .addReg(StReg[1], StRegKill[1]); + // stp instructions + BuildMI(MBB, MI, DL, *ReplInstrMCID[2]) + .addReg(ZipDest[0]) + .addReg(ZipDest[1]) + .addReg(AddrReg) + .addImm(0); + break; + case AArch64::ST4Fourv16b: + case AArch64::ST4Fourv8b: + case AArch64::ST4Fourv8h: + case AArch64::ST4Fourv4h: + case AArch64::ST4Fourv4s: + case AArch64::ST4Fourv2s: + case AArch64::ST4Fourv2d: + // zip instructions + BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) + .addReg(StReg[0]) + .addReg(StReg[2]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) + .addReg(StReg[0], StRegKill[0]) + .addReg(StReg[2], StRegKill[2]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2]) + .addReg(StReg[1]) + .addReg(StReg[3]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3]) + .addReg(StReg[1], StRegKill[1]) + .addReg(StReg[3], StRegKill[3]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4]) + .addReg(ZipDest[0]) + .addReg(ZipDest[2]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5]) + .addReg(ZipDest[0]) + .addReg(ZipDest[2]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6]) + .addReg(ZipDest[1]) + .addReg(ZipDest[3]); + BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7]) + .addReg(ZipDest[1]) + .addReg(ZipDest[3]); + // stp instructions + BuildMI(MBB, MI, DL, *ReplInstrMCID[8]) + .addReg(ZipDest[4]) + .addReg(ZipDest[5]) + .addReg(AddrReg) + .addImm(0); + BuildMI(MBB, MI, DL, *ReplInstrMCID[9]) + .addReg(ZipDest[6]) + .addReg(ZipDest[7]) + .addReg(AddrReg) + .addImm(2); + break; + } + + ++NumModifiedInstr; + return true; +} + +/// Process The REG_SEQUENCE instruction, and extract the source +/// operands of the st2/4 instruction from it. +/// Example of such instruction. +/// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; +/// Return true when the instruction is processed successfully. +bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI, + unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const { + assert (DefiningMI != NULL); + if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE) + return false; + + for (unsigned i=0; igetOperand(2*i+1).getReg(); + StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill()); + + // Sanity check for the other arguments. + if (DefiningMI->getOperand(2*i+2).isImm()) { + switch (DefiningMI->getOperand(2*i+2).getImm()) { + default: + return false; + case AArch64::dsub0: + case AArch64::dsub1: + case AArch64::dsub2: + case AArch64::dsub3: + case AArch64::qsub0: + case AArch64::qsub1: + case AArch64::qsub2: + case AArch64::qsub3: + break; + } + } + else + return false; + } + return true; +} + +/// Return the number of useful source registers for this instruction +/// (2 for ST2 and 4 for ST4). +unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const { + switch (MI.getOpcode()) { + default: + llvm_unreachable("Unsupported instruction for this pass"); + case AArch64::ST2Twov16b: + case AArch64::ST2Twov8b: + case AArch64::ST2Twov8h: + case AArch64::ST2Twov4h: + case AArch64::ST2Twov4s: + case AArch64::ST2Twov2s: + case AArch64::ST2Twov2d: + return 2; + case AArch64::ST4Fourv16b: + case AArch64::ST4Fourv8b: + case AArch64::ST4Fourv8h: + case AArch64::ST4Fourv4h: + case AArch64::ST4Fourv4s: + case AArch64::ST4Fourv2s: + case AArch64::ST4Fourv2d: + return 4; + } +} + +bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) { + if (skipFunction(*MF.getFunction())) + return false; + + TII = MF.getSubtarget().getInstrInfo(); + MRI = &MF.getRegInfo(); + const TargetSubtargetInfo &ST = MF.getSubtarget(); + const AArch64InstrInfo *AAII = + static_cast(ST.getInstrInfo()); + if (!AAII) + return false; + SchedModel.init(ST.getSchedModel(), &ST, AAII); + if (!SchedModel.hasInstrSchedModel()) + return false; + + bool Changed = false; + for (auto OptimizationKind : {VectorElem, Interleave}) { + if (!shouldExitEarly(&MF, OptimizationKind)) { + SmallVector RemoveMIs; + for (MachineBasicBlock &MBB : MF) { + for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end(); + MII != MIE;) { + MachineInstr &MI = *MII; + bool InstRewrite; + if (OptimizationKind == VectorElem) + InstRewrite = optimizeVectElement(MI) ; + else + InstRewrite = optimizeLdStInterleave(MI); + if (InstRewrite) { + // Add MI to the list of instructions to be removed given that it + // has been replaced. + RemoveMIs.push_back(&MI); + Changed = true; + } + ++MII; + } + } + for (MachineInstr *MI : RemoveMIs) + MI->eraseFromParent(); + } + } + + return Changed; +} + +/// createAArch64SIMDInstrOptPass - returns an instance of the +/// vector by element optimization pass. +FunctionPass *llvm::createAArch64SIMDInstrOptPass() { + return new AArch64SIMDInstrOpt(); +} diff --git a/llvm/lib/Target/AArch64/AArch64VectorByElementOpt.cpp b/llvm/lib/Target/AArch64/AArch64VectorByElementOpt.cpp deleted file mode 100644 index 3ff4155849a..00000000000 --- a/llvm/lib/Target/AArch64/AArch64VectorByElementOpt.cpp +++ /dev/null @@ -1,735 +0,0 @@ -// -// The LLVM Compiler Infrastructure -// -// This file is distributed under the University of Illinois Open Source -// License. See LICENSE.TXT for details. -// -//===----------------------------------------------------------------------===// -// -// This file contains a pass that performs optimization on SIMD instructions -// with high latency by splitting them into more efficient series of -// instructions. -// -// 1. Rewrite certain SIMD instructions with vector element due to their -// inefficiency on some targets. -// Example: -// fmla v0.4s, v1.4s, v2.s[1] -// is rewritten into -// dup v3.4s, v2.s[1] -// fmla v0.4s, v1.4s, v3.4s -// -// 2. Rewrite Interleaved memory access instructions due to their -// inefficiency on some targets. -// Example: -// st2 {v0.4s, v1.4s}, addr -// is rewritten into -// zip1 v2.4s, v0.4s, v1.4s -// zip2 v3.4s, v0.4s, v1.4s -// stp q2, q3, addr -// -//===----------------------------------------------------------------------===// - -#include "AArch64InstrInfo.h" -#include "llvm/ADT/SmallVector.h" -#include "llvm/ADT/Statistic.h" -#include "llvm/ADT/StringRef.h" -#include "llvm/CodeGen/MachineBasicBlock.h" -#include "llvm/CodeGen/MachineFunction.h" -#include "llvm/CodeGen/MachineFunctionPass.h" -#include "llvm/CodeGen/MachineInstr.h" -#include "llvm/CodeGen/MachineInstrBuilder.h" -#include "llvm/CodeGen/MachineOperand.h" -#include "llvm/CodeGen/MachineRegisterInfo.h" -#include "llvm/CodeGen/TargetInstrInfo.h" -#include "llvm/CodeGen/TargetSchedule.h" -#include "llvm/CodeGen/TargetSubtargetInfo.h" -#include "llvm/MC/MCInstrDesc.h" -#include "llvm/MC/MCSchedule.h" -#include "llvm/Pass.h" -#include - -using namespace llvm; - -#define DEBUG_TYPE "aarch64-simdinstr-opt" - -STATISTIC(NumModifiedInstr, - "Number of SIMD instructions modified"); - -#define AARCH64_VECTOR_BY_ELEMENT_OPT_NAME \ - "AArch64 SIMD instructions optimization pass" - -namespace { - -struct AArch64SIMDInstrOpt : public MachineFunctionPass { - static char ID; - - const TargetInstrInfo *TII; - MachineRegisterInfo *MRI; - TargetSchedModel SchedModel; - - // The two maps below are used to cache decisions instead of recomputing: - // This is used to cache instruction replacement decisions within function - // units and across function units. - std::map, bool> SIMDInstrTable; - // This is used to cache the decision of whether to leave the Interleave-Store - // instructions replacement pass early or not for a particular target. - std::unordered_map InterlEarlyExit; - - typedef enum { - VectorElem, - Interleave - } Subpass; - - // Instruction represented by OrigOpc is replaced by instructions in ReplOpc. - struct InstReplInfo { - unsigned OrigOpc; - std::vector ReplOpc; - const TargetRegisterClass RC; - }; - -#define RuleST2(OpcOrg, OpcR0, OpcR1, OpcR2, RC) \ - {OpcOrg, {OpcR0, OpcR1, OpcR2}, RC} -#define RuleST4(OpcOrg, OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, \ - OpcR7, OpcR8, OpcR9, RC) \ - {OpcOrg, {OpcR0, OpcR1, OpcR2, OpcR3, OpcR4, OpcR5, OpcR6, OpcR7, \ - OpcR8, OpcR9}, RC} - - // The Instruction Replacement Table: - std::vector IRT = { - // ST2 instructions - RuleST2(AArch64::ST2Twov2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, - AArch64::STPQi, AArch64::FPR128RegClass), - RuleST2(AArch64::ST2Twov4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, - AArch64::STPQi, AArch64::FPR128RegClass), - RuleST2(AArch64::ST2Twov2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, - AArch64::STPDi, AArch64::FPR64RegClass), - RuleST2(AArch64::ST2Twov8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, - AArch64::STPQi, AArch64::FPR128RegClass), - RuleST2(AArch64::ST2Twov4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, - AArch64::STPDi, AArch64::FPR64RegClass), - RuleST2(AArch64::ST2Twov16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, - AArch64::STPQi, AArch64::FPR128RegClass), - RuleST2(AArch64::ST2Twov8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, - AArch64::STPDi, AArch64::FPR64RegClass), - // ST4 instructions - RuleST4(AArch64::ST4Fourv2d, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, - AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, - AArch64::ZIP2v2i64, AArch64::ZIP1v2i64, AArch64::ZIP2v2i64, - AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), - RuleST4(AArch64::ST4Fourv4s, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, - AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, - AArch64::ZIP2v4i32, AArch64::ZIP1v4i32, AArch64::ZIP2v4i32, - AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), - RuleST4(AArch64::ST4Fourv2s, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, - AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, - AArch64::ZIP2v2i32, AArch64::ZIP1v2i32, AArch64::ZIP2v2i32, - AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), - RuleST4(AArch64::ST4Fourv8h, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, - AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, - AArch64::ZIP2v8i16, AArch64::ZIP1v8i16, AArch64::ZIP2v8i16, - AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), - RuleST4(AArch64::ST4Fourv4h, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, - AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, - AArch64::ZIP2v4i16, AArch64::ZIP1v4i16, AArch64::ZIP2v4i16, - AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass), - RuleST4(AArch64::ST4Fourv16b, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, - AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, - AArch64::ZIP2v16i8, AArch64::ZIP1v16i8, AArch64::ZIP2v16i8, - AArch64::STPQi, AArch64::STPQi, AArch64::FPR128RegClass), - RuleST4(AArch64::ST4Fourv8b, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, - AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, - AArch64::ZIP2v8i8, AArch64::ZIP1v8i8, AArch64::ZIP2v8i8, - AArch64::STPDi, AArch64::STPDi, AArch64::FPR64RegClass) - }; - - // A costly instruction is replaced in this work by N efficient instructions - // The maximum of N is curently 10 and it is for ST4 case. - static const unsigned MaxNumRepl = 10; - - AArch64SIMDInstrOpt() : MachineFunctionPass(ID) { - initializeAArch64SIMDInstrOptPass(*PassRegistry::getPassRegistry()); - } - - /// Based only on latency of instructions, determine if it is cost efficient - /// to replace the instruction InstDesc by the instructions stored in the - /// array InstDescRepl. - /// Return true if replacement is expected to be faster. - bool shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, - SmallVectorImpl &ReplInstrMCID); - - /// Determine if we need to exit the instruction replacement optimization - /// subpasses early. This makes sure that Targets with no need for this - /// optimization do not spend any compile time on this subpass other than the - /// simple check performed here. This simple check is done by comparing the - /// latency of the original instruction to the latency of the replacement - /// instructions. We only check for a representative instruction in the class - /// of instructions and not all concerned instructions. For the VectorElem - /// subpass, we check for the FMLA instruction while for the interleave subpass - /// we check for the st2.4s instruction. - /// Return true if early exit of the subpass is recommended. - bool shouldExitEarly(MachineFunction *MF, Subpass SP); - - /// Check whether an equivalent DUP instruction has already been - /// created or not. - /// Return true when the dup instruction already exists. In this case, - /// DestReg will point to the destination of the already created DUP. - bool reuseDUP(MachineInstr &MI, unsigned DupOpcode, unsigned SrcReg, - unsigned LaneNumber, unsigned *DestReg) const; - - /// Certain SIMD instructions with vector element operand are not efficient. - /// Rewrite them into SIMD instructions with vector operands. This rewrite - /// is driven by the latency of the instructions. - /// Return true if the SIMD instruction is modified. - bool optimizeVectElement(MachineInstr &MI); - - /// Process The REG_SEQUENCE instruction, and extract the source - /// operands of the st2/4 instruction from it. - /// Example of such instructions. - /// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; - /// Return true when the instruction is processed successfully. - bool processSeqRegInst(MachineInstr *DefiningMI, unsigned* StReg, - unsigned* StRegKill, unsigned NumArg) const; - - /// Load/Store Interleaving instructions are not always beneficial. - /// Replace them by zip instructionand classical load/store. - /// Return true if the SIMD instruction is modified. - bool optimizeLdStInterleave(MachineInstr &MI); - - /// Return the number of useful source registers for this - /// instruction (2 for st2 and 4 for st4). - unsigned determineSrcReg(MachineInstr &MI) const; - - bool runOnMachineFunction(MachineFunction &Fn) override; - - StringRef getPassName() const override { - return AARCH64_VECTOR_BY_ELEMENT_OPT_NAME; - } -}; - -char AArch64SIMDInstrOpt::ID = 0; - -} // end anonymous namespace - -INITIALIZE_PASS(AArch64SIMDInstrOpt, "aarch64-simdinstr-opt", - AARCH64_VECTOR_BY_ELEMENT_OPT_NAME, false, false) - -/// Based only on latency of instructions, determine if it is cost efficient -/// to replace the instruction InstDesc by the instructions stored in the -/// array InstDescRepl. -/// Return true if replacement is expected to be faster. -bool AArch64SIMDInstrOpt:: -shouldReplaceInst(MachineFunction *MF, const MCInstrDesc *InstDesc, - SmallVectorImpl &InstDescRepl) { - // Check if replacement decision is already available in the cached table. - // if so, return it. - std::string Subtarget = SchedModel.getSubtargetInfo()->getCPU(); - std::pair InstID = std::make_pair(InstDesc->getOpcode(), Subtarget); - if (!SIMDInstrTable.empty() && - SIMDInstrTable.find(InstID) != SIMDInstrTable.end()) - return SIMDInstrTable[InstID]; - - unsigned SCIdx = InstDesc->getSchedClass(); - const MCSchedClassDesc *SCDesc = - SchedModel.getMCSchedModel()->getSchedClassDesc(SCIdx); - - // If a subtarget does not define resources for the instructions - // of interest, then return false for no replacement. - const MCSchedClassDesc *SCDescRepl; - if (!SCDesc->isValid() || SCDesc->isVariant()) - { - SIMDInstrTable[InstID] = false; - return false; - } - for (auto IDesc : InstDescRepl) - { - SCDescRepl = SchedModel.getMCSchedModel()->getSchedClassDesc( - IDesc->getSchedClass()); - if (!SCDescRepl->isValid() || SCDescRepl->isVariant()) - { - SIMDInstrTable[InstID] = false; - return false; - } - } - - // Replacement cost. - unsigned ReplCost = 0; - for (auto IDesc :InstDescRepl) - ReplCost += SchedModel.computeInstrLatency(IDesc->getOpcode()); - - if (SchedModel.computeInstrLatency(InstDesc->getOpcode()) > ReplCost) - { - SIMDInstrTable[InstID] = true; - return true; - } - else - { - SIMDInstrTable[InstID] = false; - return false; - } -} - -/// Determine if we need to exit the instruction replacement optimization -/// subpasses early. This makes sure that Targets with no need for this -/// optimization do not spend any compile time on this subpass other than the -/// simple check performed here. This simple check is done by comparing the -/// latency of the original instruction to the latency of the replacement -/// instructions. We only check for a representative instruction in the class of -/// instructions and not all concerned instructions. For the VectorElem subpass, -/// we check for the FMLA instruction while for the interleave subpass we check -/// for the st2.4s instruction. -/// Return true if early exit of the subpass is recommended. -bool AArch64SIMDInstrOpt::shouldExitEarly(MachineFunction *MF, Subpass SP) { - const MCInstrDesc* OriginalMCID; - SmallVector ReplInstrMCID; - - switch (SP) { - case VectorElem: - OriginalMCID = &TII->get(AArch64::FMLAv4i32_indexed); - ReplInstrMCID.push_back(&TII->get(AArch64::DUPv4i32lane)); - ReplInstrMCID.push_back(&TII->get(AArch64::FMULv4f32)); - if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) - return false; - break; - case Interleave: - // Check if early exit decision is already available in the cached - // table or not. - std::string Subtarget = SchedModel.getSubtargetInfo()->getCPU(); - if (InterlEarlyExit.find(Subtarget) != InterlEarlyExit.end()) - return InterlEarlyExit[Subtarget]; - - for (auto &I : IRT) { - OriginalMCID = &TII->get(I.OrigOpc); - for (auto &Repl : I.ReplOpc) - ReplInstrMCID.push_back(&TII->get(Repl)); - if (shouldReplaceInst(MF, OriginalMCID, ReplInstrMCID)) { - InterlEarlyExit[Subtarget] = false; - return false; - } - ReplInstrMCID.clear(); - } - InterlEarlyExit[Subtarget] = true; - break; - } - - return true; -} - -/// Check whether an equivalent DUP instruction has already been -/// created or not. -/// Return true when the dup instruction already exists. In this case, -/// DestReg will point to the destination of the already created DUP. -bool AArch64SIMDInstrOpt::reuseDUP(MachineInstr &MI, unsigned DupOpcode, - unsigned SrcReg, unsigned LaneNumber, - unsigned *DestReg) const { - for (MachineBasicBlock::iterator MII = MI, MIE = MI.getParent()->begin(); - MII != MIE;) { - MII--; - MachineInstr *CurrentMI = &*MII; - - if (CurrentMI->getOpcode() == DupOpcode && - CurrentMI->getNumOperands() == 3 && - CurrentMI->getOperand(1).getReg() == SrcReg && - CurrentMI->getOperand(2).getImm() == LaneNumber) { - *DestReg = CurrentMI->getOperand(0).getReg(); - return true; - } - } - - return false; -} - -/// Certain SIMD instructions with vector element operand are not efficient. -/// Rewrite them into SIMD instructions with vector operands. This rewrite -/// is driven by the latency of the instructions. -/// The instruction of concerns are for the time being fmla, fmls, fmul, -/// and fmulx and hence they are hardcoded. -/// -/// Example: -/// fmla v0.4s, v1.4s, v2.s[1] -/// is rewritten into -/// dup v3.4s, v2.s[1] // dup not necessary if redundant -/// fmla v0.4s, v1.4s, v3.4s -/// Return true if the SIMD instruction is modified. -bool AArch64SIMDInstrOpt::optimizeVectElement(MachineInstr &MI) { - const MCInstrDesc *MulMCID, *DupMCID; - const TargetRegisterClass *RC = &AArch64::FPR128RegClass; - - switch (MI.getOpcode()) { - default: - return false; - - // 4X32 instructions - case AArch64::FMLAv4i32_indexed: - DupMCID = &TII->get(AArch64::DUPv4i32lane); - MulMCID = &TII->get(AArch64::FMLAv4f32); - break; - case AArch64::FMLSv4i32_indexed: - DupMCID = &TII->get(AArch64::DUPv4i32lane); - MulMCID = &TII->get(AArch64::FMLSv4f32); - break; - case AArch64::FMULXv4i32_indexed: - DupMCID = &TII->get(AArch64::DUPv4i32lane); - MulMCID = &TII->get(AArch64::FMULXv4f32); - break; - case AArch64::FMULv4i32_indexed: - DupMCID = &TII->get(AArch64::DUPv4i32lane); - MulMCID = &TII->get(AArch64::FMULv4f32); - break; - - // 2X64 instructions - case AArch64::FMLAv2i64_indexed: - DupMCID = &TII->get(AArch64::DUPv2i64lane); - MulMCID = &TII->get(AArch64::FMLAv2f64); - break; - case AArch64::FMLSv2i64_indexed: - DupMCID = &TII->get(AArch64::DUPv2i64lane); - MulMCID = &TII->get(AArch64::FMLSv2f64); - break; - case AArch64::FMULXv2i64_indexed: - DupMCID = &TII->get(AArch64::DUPv2i64lane); - MulMCID = &TII->get(AArch64::FMULXv2f64); - break; - case AArch64::FMULv2i64_indexed: - DupMCID = &TII->get(AArch64::DUPv2i64lane); - MulMCID = &TII->get(AArch64::FMULv2f64); - break; - - // 2X32 instructions - case AArch64::FMLAv2i32_indexed: - RC = &AArch64::FPR64RegClass; - DupMCID = &TII->get(AArch64::DUPv2i32lane); - MulMCID = &TII->get(AArch64::FMLAv2f32); - break; - case AArch64::FMLSv2i32_indexed: - RC = &AArch64::FPR64RegClass; - DupMCID = &TII->get(AArch64::DUPv2i32lane); - MulMCID = &TII->get(AArch64::FMLSv2f32); - break; - case AArch64::FMULXv2i32_indexed: - RC = &AArch64::FPR64RegClass; - DupMCID = &TII->get(AArch64::DUPv2i32lane); - MulMCID = &TII->get(AArch64::FMULXv2f32); - break; - case AArch64::FMULv2i32_indexed: - RC = &AArch64::FPR64RegClass; - DupMCID = &TII->get(AArch64::DUPv2i32lane); - MulMCID = &TII->get(AArch64::FMULv2f32); - break; - } - - SmallVector ReplInstrMCID; - ReplInstrMCID.push_back(DupMCID); - ReplInstrMCID.push_back(MulMCID); - if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), - ReplInstrMCID)) - return false; - - const DebugLoc &DL = MI.getDebugLoc(); - MachineBasicBlock &MBB = *MI.getParent(); - MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo(); - - // get the operands of the current SIMD arithmetic instruction. - unsigned MulDest = MI.getOperand(0).getReg(); - unsigned SrcReg0 = MI.getOperand(1).getReg(); - unsigned Src0IsKill = getKillRegState(MI.getOperand(1).isKill()); - unsigned SrcReg1 = MI.getOperand(2).getReg(); - unsigned Src1IsKill = getKillRegState(MI.getOperand(2).isKill()); - unsigned DupDest; - - // Instructions of interest have either 4 or 5 operands. - if (MI.getNumOperands() == 5) { - unsigned SrcReg2 = MI.getOperand(3).getReg(); - unsigned Src2IsKill = getKillRegState(MI.getOperand(3).isKill()); - unsigned LaneNumber = MI.getOperand(4).getImm(); - // Create a new DUP instruction. Note that if an equivalent DUP instruction - // has already been created before, then use that one instread of creating - // a new one. - if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg2, LaneNumber, &DupDest)) { - DupDest = MRI.createVirtualRegister(RC); - BuildMI(MBB, MI, DL, *DupMCID, DupDest) - .addReg(SrcReg2, Src2IsKill) - .addImm(LaneNumber); - } - BuildMI(MBB, MI, DL, *MulMCID, MulDest) - .addReg(SrcReg0, Src0IsKill) - .addReg(SrcReg1, Src1IsKill) - .addReg(DupDest, Src2IsKill); - } else if (MI.getNumOperands() == 4) { - unsigned LaneNumber = MI.getOperand(3).getImm(); - if (!reuseDUP(MI, DupMCID->getOpcode(), SrcReg1, LaneNumber, &DupDest)) { - DupDest = MRI.createVirtualRegister(RC); - BuildMI(MBB, MI, DL, *DupMCID, DupDest) - .addReg(SrcReg1, Src1IsKill) - .addImm(LaneNumber); - } - BuildMI(MBB, MI, DL, *MulMCID, MulDest) - .addReg(SrcReg0, Src0IsKill) - .addReg(DupDest, Src1IsKill); - } else { - return false; - } - - ++NumModifiedInstr; - return true; -} - -/// Load/Store Interleaving instructions are not always beneficial. -/// Replace them by zip instructions and classical load/store. -/// -/// Example: -/// st2 {v0.4s, v1.4s}, addr -/// is rewritten into -/// zip1 v2.4s, v0.4s, v1.4s -/// zip2 v3.4s, v0.4s, v1.4s -/// stp q2, q3, addr -// -/// Example: -/// st4 {v0.4s, v1.4s, v2.4s, v3.4s}, addr -/// is rewritten into -/// zip1 v4.4s, v0.4s, v2.4s -/// zip2 v5.4s, v0.4s, v2.4s -/// zip1 v6.4s, v1.4s, v3.4s -/// zip2 v7.4s, v1.4s, v3.4s -/// zip1 v8.4s, v4.4s, v6.4s -/// zip2 v9.4s, v4.4s, v6.4s -/// zip1 v10.4s, v5.4s, v7.4s -/// zip2 v11.4s, v5.4s, v7.4s -/// stp q8, q9, addr -/// stp q10, q11, addr+32 -/// Currently only instructions related to st2 and st4 are considered. -/// Other may be added later. -/// Return true if the SIMD instruction is modified. -bool AArch64SIMDInstrOpt::optimizeLdStInterleave(MachineInstr &MI) { - - unsigned SeqReg, AddrReg; - unsigned StReg[4], StRegKill[4]; - MachineInstr *DefiningMI; - const DebugLoc &DL = MI.getDebugLoc(); - MachineBasicBlock &MBB = *MI.getParent(); - SmallVector ZipDest; - SmallVector ReplInstrMCID; - - // If current instruction matches any of the rewriting rules, then - // gather information about parameters of the new instructions. - bool Match = false; - for (auto &I : IRT) { - if (MI.getOpcode() == I.OrigOpc) { - SeqReg = MI.getOperand(0).getReg(); - AddrReg = MI.getOperand(1).getReg(); - DefiningMI = MRI->getUniqueVRegDef(SeqReg); - unsigned NumReg = determineSrcReg(MI); - if (!processSeqRegInst(DefiningMI, StReg, StRegKill, NumReg)) - return false; - - for (auto &Repl : I.ReplOpc) { - ReplInstrMCID.push_back(&TII->get(Repl)); - // Generate destination registers but only for non-store instruction. - if (Repl != AArch64::STPQi && Repl != AArch64::STPDi) - ZipDest.push_back(MRI->createVirtualRegister(&I.RC)); - } - Match = true; - break; - } - } - - if (!Match) - return false; - - // Determine if it is profitable to replace MI by the series of instructions - // represented in ReplInstrMCID. - if (!shouldReplaceInst(MI.getParent()->getParent(), &TII->get(MI.getOpcode()), - ReplInstrMCID)) - return false; - - // Generate the replacement instructions composed of zip1, zip2, and stp (at - // this point, the code generation is hardcoded and does not rely on the IRT - // table used above given that code generation for ST2 replacement is somewhat - // different than for ST4 replacement. We could have added more info into the - // table related to how we build new instructions but we may be adding more - // complexity with that). - switch (MI.getOpcode()) { - default: - return false; - case AArch64::ST2Twov16b: - case AArch64::ST2Twov8b: - case AArch64::ST2Twov8h: - case AArch64::ST2Twov4h: - case AArch64::ST2Twov4s: - case AArch64::ST2Twov2s: - case AArch64::ST2Twov2d: - // zip instructions - BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) - .addReg(StReg[0]) - .addReg(StReg[1]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) - .addReg(StReg[0], StRegKill[0]) - .addReg(StReg[1], StRegKill[1]); - // stp instructions - BuildMI(MBB, MI, DL, *ReplInstrMCID[2]) - .addReg(ZipDest[0]) - .addReg(ZipDest[1]) - .addReg(AddrReg) - .addImm(0); - break; - case AArch64::ST4Fourv16b: - case AArch64::ST4Fourv8b: - case AArch64::ST4Fourv8h: - case AArch64::ST4Fourv4h: - case AArch64::ST4Fourv4s: - case AArch64::ST4Fourv2s: - case AArch64::ST4Fourv2d: - // zip instructions - BuildMI(MBB, MI, DL, *ReplInstrMCID[0], ZipDest[0]) - .addReg(StReg[0]) - .addReg(StReg[2]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[1], ZipDest[1]) - .addReg(StReg[0], StRegKill[0]) - .addReg(StReg[2], StRegKill[2]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[2], ZipDest[2]) - .addReg(StReg[1]) - .addReg(StReg[3]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[3], ZipDest[3]) - .addReg(StReg[1], StRegKill[1]) - .addReg(StReg[3], StRegKill[3]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[4], ZipDest[4]) - .addReg(ZipDest[0]) - .addReg(ZipDest[2]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[5], ZipDest[5]) - .addReg(ZipDest[0]) - .addReg(ZipDest[2]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[6], ZipDest[6]) - .addReg(ZipDest[1]) - .addReg(ZipDest[3]); - BuildMI(MBB, MI, DL, *ReplInstrMCID[7], ZipDest[7]) - .addReg(ZipDest[1]) - .addReg(ZipDest[3]); - // stp instructions - BuildMI(MBB, MI, DL, *ReplInstrMCID[8]) - .addReg(ZipDest[4]) - .addReg(ZipDest[5]) - .addReg(AddrReg) - .addImm(0); - BuildMI(MBB, MI, DL, *ReplInstrMCID[9]) - .addReg(ZipDest[6]) - .addReg(ZipDest[7]) - .addReg(AddrReg) - .addImm(2); - break; - } - - ++NumModifiedInstr; - return true; -} - -/// Process The REG_SEQUENCE instruction, and extract the source -/// operands of the st2/4 instruction from it. -/// Example of such instruction. -/// %dest = REG_SEQUENCE %st2_src1, dsub0, %st2_src2, dsub1; -/// Return true when the instruction is processed successfully. -bool AArch64SIMDInstrOpt::processSeqRegInst(MachineInstr *DefiningMI, - unsigned* StReg, unsigned* StRegKill, unsigned NumArg) const { - assert (DefiningMI != NULL); - if (DefiningMI->getOpcode() != AArch64::REG_SEQUENCE) - return false; - - for (unsigned i=0; igetOperand(2*i+1).getReg(); - StRegKill[i] = getKillRegState(DefiningMI->getOperand(2*i+1).isKill()); - - // Sanity check for the other arguments. - if (DefiningMI->getOperand(2*i+2).isImm()) { - switch (DefiningMI->getOperand(2*i+2).getImm()) { - default: - return false; - case AArch64::dsub0: - case AArch64::dsub1: - case AArch64::dsub2: - case AArch64::dsub3: - case AArch64::qsub0: - case AArch64::qsub1: - case AArch64::qsub2: - case AArch64::qsub3: - break; - } - } - else - return false; - } - return true; -} - -/// Return the number of useful source registers for this instruction -/// (2 for ST2 and 4 for ST4). -unsigned AArch64SIMDInstrOpt::determineSrcReg(MachineInstr &MI) const { - switch (MI.getOpcode()) { - default: - llvm_unreachable("Unsupported instruction for this pass"); - case AArch64::ST2Twov16b: - case AArch64::ST2Twov8b: - case AArch64::ST2Twov8h: - case AArch64::ST2Twov4h: - case AArch64::ST2Twov4s: - case AArch64::ST2Twov2s: - case AArch64::ST2Twov2d: - return 2; - case AArch64::ST4Fourv16b: - case AArch64::ST4Fourv8b: - case AArch64::ST4Fourv8h: - case AArch64::ST4Fourv4h: - case AArch64::ST4Fourv4s: - case AArch64::ST4Fourv2s: - case AArch64::ST4Fourv2d: - return 4; - } -} - -bool AArch64SIMDInstrOpt::runOnMachineFunction(MachineFunction &MF) { - if (skipFunction(*MF.getFunction())) - return false; - - TII = MF.getSubtarget().getInstrInfo(); - MRI = &MF.getRegInfo(); - const TargetSubtargetInfo &ST = MF.getSubtarget(); - const AArch64InstrInfo *AAII = - static_cast(ST.getInstrInfo()); - if (!AAII) - return false; - SchedModel.init(ST.getSchedModel(), &ST, AAII); - if (!SchedModel.hasInstrSchedModel()) - return false; - - bool Changed = false; - for (auto OptimizationKind : {VectorElem, Interleave}) { - if (!shouldExitEarly(&MF, OptimizationKind)) { - SmallVector RemoveMIs; - for (MachineBasicBlock &MBB : MF) { - for (MachineBasicBlock::iterator MII = MBB.begin(), MIE = MBB.end(); - MII != MIE;) { - MachineInstr &MI = *MII; - bool InstRewrite; - if (OptimizationKind == VectorElem) - InstRewrite = optimizeVectElement(MI) ; - else - InstRewrite = optimizeLdStInterleave(MI); - if (InstRewrite) { - // Add MI to the list of instructions to be removed given that it - // has been replaced. - RemoveMIs.push_back(&MI); - Changed = true; - } - ++MII; - } - } - for (MachineInstr *MI : RemoveMIs) - MI->eraseFromParent(); - } - } - - return Changed; -} - -/// createAArch64SIMDInstrOptPass - returns an instance of the -/// vector by element optimization pass. -FunctionPass *llvm::createAArch64SIMDInstrOptPass() { - return new AArch64SIMDInstrOpt(); -} diff --git a/llvm/lib/Target/AArch64/CMakeLists.txt b/llvm/lib/Target/AArch64/CMakeLists.txt index eb1079be730..3d4b9dcf7e8 100644 --- a/llvm/lib/Target/AArch64/CMakeLists.txt +++ b/llvm/lib/Target/AArch64/CMakeLists.txt @@ -53,7 +53,7 @@ add_llvm_target(AArch64CodeGen AArch64TargetMachine.cpp AArch64TargetObjectFile.cpp AArch64TargetTransformInfo.cpp - AArch64VectorByElementOpt.cpp + AArch64SIMDInstrOpt.cpp DEPENDS intrinsics_gen -- cgit v1.2.3