1 //===-- SILateBranchLowering.cpp - Final preparation of branches ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass mainly lowers early terminate pseudo instructions.
11 //
12 //===----------------------------------------------------------------------===//
13 
14 #include "AMDGPU.h"
15 #include "GCNSubtarget.h"
16 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
17 #include "SIMachineFunctionInfo.h"
18 #include "llvm/CodeGen/MachineDominators.h"
19 #include "llvm/InitializePasses.h"
20 
21 using namespace llvm;
22 
23 #define DEBUG_TYPE "si-late-branch-lowering"
24 
25 namespace {
26 
27 class SILateBranchLowering : public MachineFunctionPass {
28 private:
29   const SIRegisterInfo *TRI = nullptr;
30   const SIInstrInfo *TII = nullptr;
31   MachineDominatorTree *MDT = nullptr;
32 
33   void earlyTerm(MachineInstr &MI, MachineBasicBlock *EarlyExitBlock);
34 
35 public:
36   static char ID;
37 
38   unsigned MovOpc;
39   Register ExecReg;
40 
41   SILateBranchLowering() : MachineFunctionPass(ID) {}
42 
43   bool runOnMachineFunction(MachineFunction &MF) override;
44 
45   StringRef getPassName() const override {
46     return "SI Final Branch Preparation";
47   }
48 
49   void getAnalysisUsage(AnalysisUsage &AU) const override {
50     AU.addRequired<MachineDominatorTree>();
51     AU.addPreserved<MachineDominatorTree>();
52     MachineFunctionPass::getAnalysisUsage(AU);
53   }
54 };
55 
56 } // end anonymous namespace
57 
58 char SILateBranchLowering::ID = 0;
59 
60 INITIALIZE_PASS_BEGIN(SILateBranchLowering, DEBUG_TYPE,
61                       "SI insert s_cbranch_execz instructions", false, false)
62 INITIALIZE_PASS_DEPENDENCY(MachineDominatorTree)
63 INITIALIZE_PASS_END(SILateBranchLowering, DEBUG_TYPE,
64                     "SI insert s_cbranch_execz instructions", false, false)
65 
66 char &llvm::SILateBranchLoweringPassID = SILateBranchLowering::ID;
67 
68 static void generateEndPgm(MachineBasicBlock &MBB,
69                            MachineBasicBlock::iterator I, DebugLoc DL,
70                            const SIInstrInfo *TII, MachineFunction &MF) {
71   const Function &F = MF.getFunction();
72   bool IsPS = F.getCallingConv() == CallingConv::AMDGPU_PS;
73 
74   // Check if hardware has been configured to expect color or depth exports.
75   bool HasColorExports = AMDGPU::getHasColorExport(F);
76   bool HasDepthExports = AMDGPU::getHasDepthExport(F);
77   bool HasExports = HasColorExports || HasDepthExports;
78 
79   // Prior to GFX10, hardware always expects at least one export for PS.
80   bool MustExport = !AMDGPU::isGFX10Plus(TII->getSubtarget());
81 
82   if (IsPS && (HasExports || MustExport)) {
83     // Generate "null export" if hardware is expecting PS to export.
84     const GCNSubtarget &ST = MBB.getParent()->getSubtarget<GCNSubtarget>();
85     int Target =
86         ST.hasNullExportTarget()
87             ? AMDGPU::Exp::ET_NULL
88             : (HasColorExports ? AMDGPU::Exp::ET_MRT0 : AMDGPU::Exp::ET_MRTZ);
89     BuildMI(MBB, I, DL, TII->get(AMDGPU::EXP_DONE))
90         .addImm(Target)
91         .addReg(AMDGPU::VGPR0, RegState::Undef)
92         .addReg(AMDGPU::VGPR0, RegState::Undef)
93         .addReg(AMDGPU::VGPR0, RegState::Undef)
94         .addReg(AMDGPU::VGPR0, RegState::Undef)
95         .addImm(1)  // vm
96         .addImm(0)  // compr
97         .addImm(0); // en
98   }
99 
100   // s_endpgm
101   BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ENDPGM)).addImm(0);
102 }
103 
104 static void splitBlock(MachineBasicBlock &MBB, MachineInstr &MI,
105                        MachineDominatorTree *MDT) {
106   MachineBasicBlock *SplitBB = MBB.splitAt(MI, /*UpdateLiveIns*/ true);
107 
108   // Update dominator tree
109   using DomTreeT = DomTreeBase<MachineBasicBlock>;
110   SmallVector<DomTreeT::UpdateType, 16> DTUpdates;
111   for (MachineBasicBlock *Succ : SplitBB->successors()) {
112     DTUpdates.push_back({DomTreeT::Insert, SplitBB, Succ});
113     DTUpdates.push_back({DomTreeT::Delete, &MBB, Succ});
114   }
115   DTUpdates.push_back({DomTreeT::Insert, &MBB, SplitBB});
116   MDT->getBase().applyUpdates(DTUpdates);
117 }
118 
119 void SILateBranchLowering::earlyTerm(MachineInstr &MI,
120                                      MachineBasicBlock *EarlyExitBlock) {
121   MachineBasicBlock &MBB = *MI.getParent();
122   const DebugLoc DL = MI.getDebugLoc();
123 
124   auto BranchMI = BuildMI(MBB, MI, DL, TII->get(AMDGPU::S_CBRANCH_SCC0))
125                       .addMBB(EarlyExitBlock);
126   auto Next = std::next(MI.getIterator());
127 
128   if (Next != MBB.end() && !Next->isTerminator())
129     splitBlock(MBB, *BranchMI, MDT);
130 
131   MBB.addSuccessor(EarlyExitBlock);
132   MDT->getBase().insertEdge(&MBB, EarlyExitBlock);
133 }
134 
135 bool SILateBranchLowering::runOnMachineFunction(MachineFunction &MF) {
136   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
137   TII = ST.getInstrInfo();
138   TRI = &TII->getRegisterInfo();
139   MDT = &getAnalysis<MachineDominatorTree>();
140 
141   MovOpc = ST.isWave32() ? AMDGPU::S_MOV_B32 : AMDGPU::S_MOV_B64;
142   ExecReg = ST.isWave32() ? AMDGPU::EXEC_LO : AMDGPU::EXEC;
143 
144   SmallVector<MachineInstr *, 4> EarlyTermInstrs;
145   SmallVector<MachineInstr *, 1> EpilogInstrs;
146   bool MadeChange = false;
147 
148   for (MachineBasicBlock &MBB : MF) {
149     for (MachineInstr &MI : llvm::make_early_inc_range(MBB)) {
150       switch (MI.getOpcode()) {
151       case AMDGPU::S_BRANCH:
152         // Optimize out branches to the next block.
153         // This only occurs in -O0 when BranchFolding is not executed.
154         if (MBB.isLayoutSuccessor(MI.getOperand(0).getMBB())) {
155           assert(&MI == &MBB.back());
156           MI.eraseFromParent();
157           MadeChange = true;
158         }
159         break;
160 
161       case AMDGPU::SI_EARLY_TERMINATE_SCC0:
162         EarlyTermInstrs.push_back(&MI);
163         break;
164 
165       case AMDGPU::SI_RETURN_TO_EPILOG:
166         EpilogInstrs.push_back(&MI);
167         break;
168 
169       default:
170         break;
171       }
172     }
173   }
174 
175   // Lower any early exit branches first
176   if (!EarlyTermInstrs.empty()) {
177     MachineBasicBlock *EarlyExitBlock = MF.CreateMachineBasicBlock();
178     DebugLoc DL;
179 
180     MF.insert(MF.end(), EarlyExitBlock);
181     BuildMI(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII->get(MovOpc),
182             ExecReg)
183         .addImm(0);
184     generateEndPgm(*EarlyExitBlock, EarlyExitBlock->end(), DL, TII, MF);
185 
186     for (MachineInstr *Instr : EarlyTermInstrs) {
187       // Early termination in GS does nothing
188       if (MF.getFunction().getCallingConv() != CallingConv::AMDGPU_GS)
189         earlyTerm(*Instr, EarlyExitBlock);
190       Instr->eraseFromParent();
191     }
192 
193     EarlyTermInstrs.clear();
194     MadeChange = true;
195   }
196 
197   // Now check return to epilog instructions occur at function end
198   if (!EpilogInstrs.empty()) {
199     MachineBasicBlock *EmptyMBBAtEnd = nullptr;
200     assert(!MF.getInfo<SIMachineFunctionInfo>()->returnsVoid());
201 
202     // If there are multiple returns to epilog then all will
203     // become jumps to new empty end block.
204     if (EpilogInstrs.size() > 1) {
205       EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
206       MF.insert(MF.end(), EmptyMBBAtEnd);
207     }
208 
209     for (auto MI : EpilogInstrs) {
210       auto MBB = MI->getParent();
211       if (MBB == &MF.back() && MI == &MBB->back())
212         continue;
213 
214       // SI_RETURN_TO_EPILOG is not the last instruction.
215       // Jump to empty block at function end.
216       if (!EmptyMBBAtEnd) {
217         EmptyMBBAtEnd = MF.CreateMachineBasicBlock();
218         MF.insert(MF.end(), EmptyMBBAtEnd);
219       }
220 
221       MBB->addSuccessor(EmptyMBBAtEnd);
222       MDT->getBase().insertEdge(MBB, EmptyMBBAtEnd);
223       BuildMI(*MBB, MI, MI->getDebugLoc(), TII->get(AMDGPU::S_BRANCH))
224           .addMBB(EmptyMBBAtEnd);
225       MI->eraseFromParent();
226       MadeChange = true;
227     }
228 
229     EpilogInstrs.clear();
230   }
231 
232   return MadeChange;
233 }
234