diff options
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 21 | ||||
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIInstrInfo.h | 4 | ||||
| -rw-r--r-- | llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp | 81 | ||||
| -rw-r--r-- | llvm/test/CodeGen/AMDGPU/collapse-endcf.mir | 125 |
4 files changed, 200 insertions, 31 deletions
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp index 7639a3fe138..370b8cf2cfb 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp @@ -2453,6 +2453,27 @@ bool SIInstrInfo::hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const return false; } +bool SIInstrInfo::mayReadEXEC(const MachineRegisterInfo &MRI, + const MachineInstr &MI) const { + if (MI.isMetaInstruction()) + return false; + + // This won't read exec if this is an SGPR->SGPR copy. + if (MI.isCopyLike()) { + if (!RI.isSGPRReg(MRI, MI.getOperand(0).getReg())) + return true; + + // Make sure this isn't copying exec as a normal operand + return MI.readsRegister(AMDGPU::EXEC, &RI); + } + + // Be conservative with any unhandled generic opcodes. + if (!isTargetSpecificOpcode(MI.getOpcode())) + return true; + + return !isSALU(MI) || MI.readsRegister(AMDGPU::EXEC, &RI); +} + bool SIInstrInfo::isInlineConstant(const APInt &Imm) const { switch (Imm.getBitWidth()) { case 32: diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.h b/llvm/lib/Target/AMDGPU/SIInstrInfo.h index 51b5df93fef..13e3dbd3cfe 100644 --- a/llvm/lib/Target/AMDGPU/SIInstrInfo.h +++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.h @@ -624,6 +624,10 @@ public: /// Whether we must prevent this instruction from executing with EXEC = 0. bool hasUnwantedEffectsWhenEXECEmpty(const MachineInstr &MI) const; + /// Returns true if the instruction could potentially depend on the value of + /// exec. If false, exec dependencies may safely be ignored. + bool mayReadEXEC(const MachineRegisterInfo &MRI, const MachineInstr &MI) const; + bool isInlineConstant(const APInt &Imm) const; bool isInlineConstant(const MachineOperand &MO, uint8_t OperandType) const; diff --git a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp index f5724a71e1d..df06ba5a692 100644 --- a/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp +++ b/llvm/lib/Target/AMDGPU/SIOptimizeExecMaskingPreRA.cpp @@ -33,10 +33,22 @@ using namespace llvm; namespace { class SIOptimizeExecMaskingPreRA : public MachineFunctionPass { +private: + const SIRegisterInfo *TRI; + const SIInstrInfo *TII; + MachineRegisterInfo *MRI; + public: - static char ID; + MachineBasicBlock::iterator skipIgnoreExecInsts( + MachineBasicBlock::iterator I, MachineBasicBlock::iterator E) const; + + MachineBasicBlock::iterator skipIgnoreExecInstsTrivialSucc( + MachineBasicBlock *&MBB, + MachineBasicBlock::iterator It) const; public: + static char ID; + SIOptimizeExecMaskingPreRA() : MachineFunctionPass(ID) { initializeSIOptimizeExecMaskingPreRAPass(*PassRegistry::getPassRegistry()); } @@ -102,6 +114,45 @@ static MachineInstr* getOrExecSource(const MachineInstr &MI, return SaveExecInst; } +/// Skip over instructions that don't care about the exec mask. +MachineBasicBlock::iterator SIOptimizeExecMaskingPreRA::skipIgnoreExecInsts( + MachineBasicBlock::iterator I, MachineBasicBlock::iterator E) const { + for ( ; I != E; ++I) { + if (TII->mayReadEXEC(*MRI, *I)) + break; + } + + return I; +} + +// Skip to the next instruction, ignoring debug instructions, and trivial block +// boundaries (blocks that have one (typically fallthrough) successor, and the +// successor has one predecessor. +MachineBasicBlock::iterator +SIOptimizeExecMaskingPreRA::skipIgnoreExecInstsTrivialSucc( + MachineBasicBlock *&MBB, + MachineBasicBlock::iterator It) const { + + do { + It = skipIgnoreExecInsts(It, MBB->end()); + if (It != MBB->end() || MBB->succ_size() != 1) + break; + + // If there is one trivial successor, advance to the next block. + MachineBasicBlock *Succ = *MBB->succ_begin(); + + // TODO: Is this really necessary? + if (!MBB->isLayoutSuccessor(Succ)) + break; + + It = Succ->begin(); + MBB = Succ; + } while (true); + + return It; +} + + // Optimize sequence // %sel = V_CNDMASK_B32_e64 0, 1, %cc // %cmp = V_CMP_NE_U32 1, %1 @@ -227,8 +278,10 @@ bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) { return false; const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>(); - const SIRegisterInfo *TRI = ST.getRegisterInfo(); - const SIInstrInfo *TII = ST.getInstrInfo(); + TRI = ST.getRegisterInfo(); + TII = ST.getInstrInfo(); + MRI = &MF.getRegInfo(); + MachineRegisterInfo &MRI = MF.getRegInfo(); LiveIntervals *LIS = &getAnalysis<LiveIntervals>(); DenseSet<unsigned> RecalcRegs({AMDGPU::EXEC_LO, AMDGPU::EXEC_HI}); @@ -313,25 +366,9 @@ bool SIOptimizeExecMaskingPreRA::runOnMachineFunction(MachineFunction &MF) { if (MBB.succ_size() != 1 || Lead == E || !isEndCF(*Lead, TRI)) continue; - const MachineBasicBlock* Succ = *MBB.succ_begin(); - if (!MBB.isLayoutSuccessor(Succ)) - continue; - - auto I = std::next(Lead); - - for ( ; I != E; ++I) { - if (I->isDebugInstr()) - continue; - - if (!TII->isSALU(*I) || I->readsRegister(AMDGPU::EXEC, TRI)) - break; - } - - if (I != E) - continue; - - auto NextLead = skipDebugInstructionsForward(Succ->begin(), Succ->end()); - if (NextLead == Succ->end() || !isEndCF(*NextLead, TRI) || + MachineBasicBlock *TmpMBB = &MBB; + auto NextLead = skipIgnoreExecInstsTrivialSucc(TmpMBB, std::next(Lead)); + if (NextLead == TmpMBB->end() || !isEndCF(*NextLead, TRI) || !getOrExecSource(*NextLead, *TII, MRI)) continue; diff --git a/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir b/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir index 2dd98b21907..d98b6cdc614 100644 --- a/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir +++ b/llvm/test/CodeGen/AMDGPU/collapse-endcf.mir @@ -143,8 +143,7 @@ body: | ; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0 ; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) ; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec - ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec - ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc + ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc ; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]] ; GCN: SI_MASK_BRANCH %bb.3, implicit $exec ; GCN: S_BRANCH %bb.2 @@ -156,7 +155,6 @@ body: | ; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) ; GCN: bb.3: ; GCN: successors: %bb.4(0x80000000) - ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc ; GCN: bb.4: ; GCN: successors: %bb.5(0x80000000) ; GCN: bb.5: @@ -253,8 +251,7 @@ body: | ; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0 ; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) ; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec - ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec - ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc + ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc ; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]] ; GCN: SI_MASK_BRANCH %bb.3, implicit $exec ; GCN: S_BRANCH %bb.2 @@ -266,7 +263,6 @@ body: | ; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) ; GCN: bb.3: ; GCN: successors: %bb.4(0x80000000) - ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc ; GCN: bb.4: ; GCN: successors: %bb.5(0x80000000) ; GCN: DBG_VALUE @@ -477,8 +473,7 @@ body: | ; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0 ; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) ; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec - ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec - ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc + ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 $exec, [[V_CMP_NE_U32_e64_]], implicit-def dead $scc ; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]] ; GCN: SI_MASK_BRANCH %bb.3, implicit $exec ; GCN: S_BRANCH %bb.2 @@ -490,7 +485,6 @@ body: | ; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) ; GCN: bb.3: ; GCN: successors: %bb.4(0x80000000) - ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc ; GCN: [[DEF:%[0-9]+]]:sgpr_32 = IMPLICIT_DEF ; GCN: [[S_BREV_B32_:%[0-9]+]]:sgpr_32 = S_BREV_B32 [[DEF]] ; GCN: KILL [[DEF]] @@ -773,3 +767,116 @@ body: | ... +# There's no real reason this can't be handled, but isn't now. +--- +name: simple_nested_if_not_layout_successor +tracksRegLiveness: true +liveins: + - { reg: '$vgpr0', virtual-reg: '%0' } + - { reg: '$sgpr0_sgpr1', virtual-reg: '%1' } +machineFunctionInfo: + isEntryFunction: true +body: | + ; GCN-LABEL: name: simple_nested_if_not_layout_successor + ; GCN: bb.0: + ; GCN: successors: %bb.1(0x40000000), %bb.4(0x40000000) + ; GCN: liveins: $vgpr0, $sgpr0_sgpr1 + ; GCN: [[COPY:%[0-9]+]]:sgpr_64 = COPY $sgpr0_sgpr1 + ; GCN: [[COPY1:%[0-9]+]]:vgpr_32 = COPY $vgpr0 + ; GCN: [[V_CMP_LT_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_LT_U32_e64 1, [[COPY1]], implicit $exec + ; GCN: [[COPY2:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec + ; GCN: [[S_AND_B64_:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY2]], [[V_CMP_LT_U32_e64_]], implicit-def dead $scc + ; GCN: $exec = S_MOV_B64_term [[S_AND_B64_]] + ; GCN: SI_MASK_BRANCH %bb.4, implicit $exec + ; GCN: S_BRANCH %bb.1 + ; GCN: bb.1: + ; GCN: successors: %bb.2(0x40000000), %bb.3(0x40000000) + ; GCN: undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM [[COPY]], 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4) + ; GCN: undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, [[COPY1]], implicit $exec + ; GCN: %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec + ; GCN: [[COPY3:%[0-9]+]]:vgpr_32 = COPY %5.sub1 + ; GCN: undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec + ; GCN: %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, [[COPY3]], %9, 0, implicit $exec + ; GCN: %5.sub3:sgpr_128 = S_MOV_B32 61440 + ; GCN: %5.sub2:sgpr_128 = S_MOV_B32 0 + ; GCN: BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) + ; GCN: [[V_CMP_NE_U32_e64_:%[0-9]+]]:sreg_64 = V_CMP_NE_U32_e64 2, [[COPY1]], implicit $exec + ; GCN: [[COPY4:%[0-9]+]]:sreg_64 = COPY $exec, implicit-def $exec + ; GCN: [[S_AND_B64_1:%[0-9]+]]:sreg_64 = S_AND_B64 [[COPY4]], [[V_CMP_NE_U32_e64_]], implicit-def dead $scc + ; GCN: $exec = S_MOV_B64_term [[S_AND_B64_1]] + ; GCN: SI_MASK_BRANCH %bb.3, implicit $exec + ; GCN: S_BRANCH %bb.2 + ; GCN: bb.2: + ; GCN: successors: %bb.3(0x80000000) + ; GCN: %5.sub0:sgpr_128 = COPY %5.sub2 + ; GCN: %5.sub1:sgpr_128 = COPY %5.sub2 + ; GCN: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + ; GCN: BUFFER_STORE_DWORD_ADDR64 [[V_MOV_B32_e32_]], %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) + ; GCN: bb.3: + ; GCN: successors: %bb.5(0x80000000) + ; GCN: $exec = S_OR_B64 $exec, [[COPY4]], implicit-def $scc + ; GCN: S_BRANCH %bb.5 + ; GCN: bb.4: + ; GCN: $exec = S_OR_B64 $exec, [[COPY2]], implicit-def $scc + ; GCN: [[V_MOV_B32_e32_1:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 3, implicit $exec + ; GCN: [[V_MOV_B32_e32_2:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + ; GCN: $m0 = S_MOV_B32 -1 + ; GCN: DS_WRITE_B32 [[V_MOV_B32_e32_2]], [[V_MOV_B32_e32_1]], 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3) + ; GCN: S_ENDPGM 0 + ; GCN: bb.5: + ; GCN: successors: %bb.4(0x80000000) + ; GCN: S_BRANCH %bb.4 + bb.0: + successors: %bb.1, %bb.4 + liveins: $vgpr0, $sgpr0_sgpr1 + + %1:sgpr_64 = COPY $sgpr0_sgpr1 + %0:vgpr_32 = COPY $vgpr0 + %2:sreg_64 = V_CMP_LT_U32_e64 1, %0, implicit $exec + %3:sreg_64 = COPY $exec, implicit-def $exec + %4:sreg_64 = S_AND_B64 %3, %2, implicit-def dead $scc + $exec = S_MOV_B64_term %4 + SI_MASK_BRANCH %bb.4, implicit $exec + S_BRANCH %bb.1 + + bb.1: + successors: %bb.2, %bb.3 + + undef %5.sub0_sub1:sgpr_128 = S_LOAD_DWORDX2_IMM %1, 9, 0 :: (dereferenceable invariant load 8, align 4, addrspace 4) + undef %6.sub0:vreg_64 = V_LSHLREV_B32_e32 2, %0, implicit $exec + %6.sub1:vreg_64 = V_MOV_B32_e32 0, implicit $exec + %7:vgpr_32 = COPY %5.sub1 + undef %8.sub0:vreg_64, %9:sreg_64_xexec = V_ADD_I32_e64 %5.sub0, %6.sub0, 0, implicit $exec + %8.sub1:vreg_64, dead %10:sreg_64_xexec = V_ADDC_U32_e64 0, %7, %9, 0, implicit $exec + %5.sub3:sgpr_128 = S_MOV_B32 61440 + %5.sub2:sgpr_128 = S_MOV_B32 0 + BUFFER_STORE_DWORD_ADDR64 %6.sub1, %6, %5, 0, 0, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) + %11:sreg_64 = V_CMP_NE_U32_e64 2, %0, implicit $exec + %12:sreg_64 = COPY $exec, implicit-def $exec + %13:sreg_64 = S_AND_B64 %12, %11, implicit-def dead $scc + $exec = S_MOV_B64_term %13 + SI_MASK_BRANCH %bb.3, implicit $exec + S_BRANCH %bb.2 + + bb.2: + %5.sub0:sgpr_128 = COPY %5.sub2 + %5.sub1:sgpr_128 = COPY %5.sub2 + %14:vgpr_32 = V_MOV_B32_e32 1, implicit $exec + BUFFER_STORE_DWORD_ADDR64 %14, %8, %5, 0, 4, 0, 0, 0, implicit $exec :: (store 4, addrspace 1) + + bb.3: + $exec = S_OR_B64 $exec, %12, implicit-def $scc + S_BRANCH %bb.5 + + bb.4: + $exec = S_OR_B64 $exec, %3, implicit-def $scc + %15:vgpr_32 = V_MOV_B32_e32 3, implicit $exec + %16:vgpr_32 = V_MOV_B32_e32 0, implicit $exec + $m0 = S_MOV_B32 -1 + DS_WRITE_B32 %16, %15, 0, 0, implicit $m0, implicit $exec :: (store 4, addrspace 3) + S_ENDPGM 0 + + bb.5: + S_BRANCH %bb.4 + +... |
