1 //===-- SILowerControlFlow.cpp - Use predicates for control flow ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass lowers the pseudo control flow instructions to real
11 /// machine instructions.
12 ///
13 /// All control flow is handled using predicated instructions and
14 /// a predicate stack.  Each Scalar ALU controls the operations of 64 Vector
15 /// ALUs.  The Scalar ALU can update the predicate for any of the Vector ALUs
16 /// by writting to the 64-bit EXEC register (each bit corresponds to a
17 /// single vector ALU).  Typically, for predicates, a vector ALU will write
18 /// to its bit of the VCC register (like EXEC VCC is 64-bits, one for each
19 /// Vector ALU) and then the ScalarALU will AND the VCC register with the
20 /// EXEC to update the predicates.
21 ///
22 /// For example:
23 /// %vcc = V_CMP_GT_F32 %vgpr1, %vgpr2
24 /// %sgpr0 = SI_IF %vcc
25 ///   %vgpr0 = V_ADD_F32 %vgpr0, %vgpr0
26 /// %sgpr0 = SI_ELSE %sgpr0
27 ///   %vgpr0 = V_SUB_F32 %vgpr0, %vgpr0
28 /// SI_END_CF %sgpr0
29 ///
30 /// becomes:
31 ///
32 /// %sgpr0 = S_AND_SAVEEXEC_B64 %vcc  // Save and update the exec mask
33 /// %sgpr0 = S_XOR_B64 %sgpr0, %exec  // Clear live bits from saved exec mask
34 /// S_CBRANCH_EXECZ label0            // This instruction is an optional
35 ///                                   // optimization which allows us to
36 ///                                   // branch if all the bits of
37 ///                                   // EXEC are zero.
38 /// %vgpr0 = V_ADD_F32 %vgpr0, %vgpr0 // Do the IF block of the branch
39 ///
40 /// label0:
41 /// %sgpr0 = S_OR_SAVEEXEC_B64 %exec   // Restore the exec mask for the Then block
42 /// %exec = S_XOR_B64 %sgpr0, %exec    // Clear live bits from saved exec mask
43 /// S_BRANCH_EXECZ label1              // Use our branch optimization
44 ///                                    // instruction again.
45 /// %vgpr0 = V_SUB_F32 %vgpr0, %vgpr   // Do the THEN block
46 /// label1:
47 /// %exec = S_OR_B64 %exec, %sgpr0     // Re-enable saved exec mask bits
48 //===----------------------------------------------------------------------===//
49 
50 #include "AMDGPU.h"
51 #include "AMDGPUSubtarget.h"
52 #include "SIInstrInfo.h"
53 #include "MCTargetDesc/AMDGPUMCTargetDesc.h"
54 #include "llvm/ADT/SmallVector.h"
55 #include "llvm/ADT/StringRef.h"
56 #include "llvm/CodeGen/LiveIntervals.h"
57 #include "llvm/CodeGen/MachineBasicBlock.h"
58 #include "llvm/CodeGen/MachineFunction.h"
59 #include "llvm/CodeGen/MachineFunctionPass.h"
60 #include "llvm/CodeGen/MachineInstr.h"
61 #include "llvm/CodeGen/MachineInstrBuilder.h"
62 #include "llvm/CodeGen/MachineOperand.h"
63 #include "llvm/CodeGen/MachineRegisterInfo.h"
64 #include "llvm/CodeGen/Passes.h"
65 #include "llvm/CodeGen/SlotIndexes.h"
66 #include "llvm/CodeGen/TargetRegisterInfo.h"
67 #include "llvm/MC/MCRegisterInfo.h"
68 #include "llvm/Pass.h"
69 #include <cassert>
70 #include <iterator>
71 
72 using namespace llvm;
73 
74 #define DEBUG_TYPE "si-lower-control-flow"
75 
76 namespace {
77 
78 class SILowerControlFlow : public MachineFunctionPass {
79 private:
80   const SIRegisterInfo *TRI = nullptr;
81   const SIInstrInfo *TII = nullptr;
82   LiveIntervals *LIS = nullptr;
83   MachineRegisterInfo *MRI = nullptr;
84 
85   const TargetRegisterClass *BoolRC = nullptr;
86   unsigned AndOpc;
87   unsigned OrOpc;
88   unsigned XorOpc;
89   unsigned MovTermOpc;
90   unsigned Andn2TermOpc;
91   unsigned XorTermrOpc;
92   unsigned OrSaveExecOpc;
93   unsigned Exec;
94 
95   void emitIf(MachineInstr &MI);
96   void emitElse(MachineInstr &MI);
97   void emitIfBreak(MachineInstr &MI);
98   void emitLoop(MachineInstr &MI);
99   void emitEndCf(MachineInstr &MI);
100 
101   Register getSaveExec(MachineInstr* MI);
102 
103   void findMaskOperands(MachineInstr &MI, unsigned OpNo,
104                         SmallVectorImpl<MachineOperand> &Src) const;
105 
106   void combineMasks(MachineInstr &MI);
107 
108 public:
109   static char ID;
110 
SILowerControlFlow()111   SILowerControlFlow() : MachineFunctionPass(ID) {}
112 
113   bool runOnMachineFunction(MachineFunction &MF) override;
114 
getPassName() const115   StringRef getPassName() const override {
116     return "SI Lower control flow pseudo instructions";
117   }
118 
getAnalysisUsage(AnalysisUsage & AU) const119   void getAnalysisUsage(AnalysisUsage &AU) const override {
120     // Should preserve the same set that TwoAddressInstructions does.
121     AU.addPreserved<SlotIndexes>();
122     AU.addPreserved<LiveIntervals>();
123     AU.addPreservedID(LiveVariablesID);
124     AU.addPreservedID(MachineLoopInfoID);
125     AU.addPreservedID(MachineDominatorsID);
126     AU.setPreservesCFG();
127     MachineFunctionPass::getAnalysisUsage(AU);
128   }
129 };
130 
131 } // end anonymous namespace
132 
133 char SILowerControlFlow::ID = 0;
134 
135 INITIALIZE_PASS(SILowerControlFlow, DEBUG_TYPE,
136                "SI lower control flow", false, false)
137 
setImpSCCDefDead(MachineInstr & MI,bool IsDead)138 static void setImpSCCDefDead(MachineInstr &MI, bool IsDead) {
139   MachineOperand &ImpDefSCC = MI.getOperand(3);
140   assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
141 
142   ImpDefSCC.setIsDead(IsDead);
143 }
144 
145 char &llvm::SILowerControlFlowID = SILowerControlFlow::ID;
146 
isSimpleIf(const MachineInstr & MI,const MachineRegisterInfo * MRI,const SIInstrInfo * TII)147 static bool isSimpleIf(const MachineInstr &MI, const MachineRegisterInfo *MRI,
148                        const SIInstrInfo *TII) {
149   Register SaveExecReg = MI.getOperand(0).getReg();
150   auto U = MRI->use_instr_nodbg_begin(SaveExecReg);
151 
152   if (U == MRI->use_instr_nodbg_end() ||
153       std::next(U) != MRI->use_instr_nodbg_end() ||
154       U->getOpcode() != AMDGPU::SI_END_CF)
155     return false;
156 
157   // Check for SI_KILL_*_TERMINATOR on path from if to endif.
158   // if there is any such terminator simplififcations are not safe.
159   auto SMBB = MI.getParent();
160   auto EMBB = U->getParent();
161   DenseSet<const MachineBasicBlock*> Visited;
162   SmallVector<MachineBasicBlock*, 4> Worklist(SMBB->succ_begin(),
163                                               SMBB->succ_end());
164 
165   while (!Worklist.empty()) {
166     MachineBasicBlock *MBB = Worklist.pop_back_val();
167 
168     if (MBB == EMBB || !Visited.insert(MBB).second)
169       continue;
170     for(auto &Term : MBB->terminators())
171       if (TII->isKillTerminator(Term.getOpcode()))
172         return false;
173 
174     Worklist.append(MBB->succ_begin(), MBB->succ_end());
175   }
176 
177   return true;
178 }
179 
getSaveExec(MachineInstr * MI)180 Register SILowerControlFlow::getSaveExec(MachineInstr *MI) {
181   MachineBasicBlock *MBB = MI->getParent();
182   MachineOperand &SaveExec = MI->getOperand(0);
183   assert(SaveExec.getSubReg() == AMDGPU::NoSubRegister);
184 
185   Register SaveExecReg = SaveExec.getReg();
186   unsigned FalseTermOpc =
187       TII->isWave32() ? AMDGPU::S_MOV_B32_term : AMDGPU::S_MOV_B64_term;
188   MachineBasicBlock::iterator I = (MI);
189   MachineBasicBlock::iterator J = std::next(I);
190   if (J != MBB->end() && J->getOpcode() == FalseTermOpc &&
191       J->getOperand(1).isReg() && J->getOperand(1).getReg() == SaveExecReg) {
192     SaveExecReg = J->getOperand(0).getReg();
193     J->eraseFromParent();
194   }
195   return SaveExecReg;
196 }
197 
emitIf(MachineInstr & MI)198 void SILowerControlFlow::emitIf(MachineInstr &MI) {
199   MachineBasicBlock &MBB = *MI.getParent();
200   const DebugLoc &DL = MI.getDebugLoc();
201   MachineBasicBlock::iterator I(&MI);
202   Register SaveExecReg = getSaveExec(&MI);
203   MachineOperand& Cond = MI.getOperand(1);
204   assert(Cond.getSubReg() == AMDGPU::NoSubRegister);
205 
206   MachineOperand &ImpDefSCC = MI.getOperand(4);
207   assert(ImpDefSCC.getReg() == AMDGPU::SCC && ImpDefSCC.isDef());
208 
209   // If there is only one use of save exec register and that use is SI_END_CF,
210   // we can optimize SI_IF by returning the full saved exec mask instead of
211   // just cleared bits.
212   bool SimpleIf = isSimpleIf(MI, MRI, TII);
213 
214   // Add an implicit def of exec to discourage scheduling VALU after this which
215   // will interfere with trying to form s_and_saveexec_b64 later.
216   Register CopyReg = SimpleIf ? SaveExecReg
217                        : MRI->createVirtualRegister(BoolRC);
218   MachineInstr *CopyExec =
219     BuildMI(MBB, I, DL, TII->get(AMDGPU::COPY), CopyReg)
220     .addReg(Exec)
221     .addReg(Exec, RegState::ImplicitDefine);
222 
223   Register Tmp = MRI->createVirtualRegister(BoolRC);
224 
225   MachineInstr *And =
226     BuildMI(MBB, I, DL, TII->get(AndOpc), Tmp)
227     .addReg(CopyReg)
228     .add(Cond);
229 
230   setImpSCCDefDead(*And, true);
231 
232   MachineInstr *Xor = nullptr;
233   if (!SimpleIf) {
234     Xor =
235       BuildMI(MBB, I, DL, TII->get(XorOpc), SaveExecReg)
236       .addReg(Tmp)
237       .addReg(CopyReg);
238     setImpSCCDefDead(*Xor, ImpDefSCC.isDead());
239   }
240 
241   // Use a copy that is a terminator to get correct spill code placement it with
242   // fast regalloc.
243   MachineInstr *SetExec =
244     BuildMI(MBB, I, DL, TII->get(MovTermOpc), Exec)
245     .addReg(Tmp, RegState::Kill);
246 
247   // Insert a pseudo terminator to help keep the verifier happy. This will also
248   // be used later when inserting skips.
249   MachineInstr *NewBr = BuildMI(MBB, I, DL, TII->get(AMDGPU::SI_MASK_BRANCH))
250                             .add(MI.getOperand(2));
251 
252   if (!LIS) {
253     MI.eraseFromParent();
254     return;
255   }
256 
257   LIS->InsertMachineInstrInMaps(*CopyExec);
258 
259   // Replace with and so we don't need to fix the live interval for condition
260   // register.
261   LIS->ReplaceMachineInstrInMaps(MI, *And);
262 
263   if (!SimpleIf)
264     LIS->InsertMachineInstrInMaps(*Xor);
265   LIS->InsertMachineInstrInMaps(*SetExec);
266   LIS->InsertMachineInstrInMaps(*NewBr);
267 
268   LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
269   MI.eraseFromParent();
270 
271   // FIXME: Is there a better way of adjusting the liveness? It shouldn't be
272   // hard to add another def here but I'm not sure how to correctly update the
273   // valno.
274   LIS->removeInterval(SaveExecReg);
275   LIS->createAndComputeVirtRegInterval(SaveExecReg);
276   LIS->createAndComputeVirtRegInterval(Tmp);
277   if (!SimpleIf)
278     LIS->createAndComputeVirtRegInterval(CopyReg);
279 }
280 
emitElse(MachineInstr & MI)281 void SILowerControlFlow::emitElse(MachineInstr &MI) {
282   MachineBasicBlock &MBB = *MI.getParent();
283   const DebugLoc &DL = MI.getDebugLoc();
284 
285   Register DstReg = getSaveExec(&MI);
286 
287   bool ExecModified = MI.getOperand(3).getImm() != 0;
288   MachineBasicBlock::iterator Start = MBB.begin();
289 
290   // We are running before TwoAddressInstructions, and si_else's operands are
291   // tied. In order to correctly tie the registers, split this into a copy of
292   // the src like it does.
293   Register CopyReg = MRI->createVirtualRegister(BoolRC);
294   MachineInstr *CopyExec =
295     BuildMI(MBB, Start, DL, TII->get(AMDGPU::COPY), CopyReg)
296       .add(MI.getOperand(1)); // Saved EXEC
297 
298   // This must be inserted before phis and any spill code inserted before the
299   // else.
300   Register SaveReg = ExecModified ?
301     MRI->createVirtualRegister(BoolRC) : DstReg;
302   MachineInstr *OrSaveExec =
303     BuildMI(MBB, Start, DL, TII->get(OrSaveExecOpc), SaveReg)
304     .addReg(CopyReg);
305 
306   MachineBasicBlock *DestBB = MI.getOperand(2).getMBB();
307 
308   MachineBasicBlock::iterator ElsePt(MI);
309 
310   if (ExecModified) {
311     MachineInstr *And =
312       BuildMI(MBB, ElsePt, DL, TII->get(AndOpc), DstReg)
313       .addReg(Exec)
314       .addReg(SaveReg);
315 
316     if (LIS)
317       LIS->InsertMachineInstrInMaps(*And);
318   }
319 
320   MachineInstr *Xor =
321     BuildMI(MBB, ElsePt, DL, TII->get(XorTermrOpc), Exec)
322     .addReg(Exec)
323     .addReg(DstReg);
324 
325   MachineInstr *Branch =
326     BuildMI(MBB, ElsePt, DL, TII->get(AMDGPU::SI_MASK_BRANCH))
327     .addMBB(DestBB);
328 
329   if (!LIS) {
330     MI.eraseFromParent();
331     return;
332   }
333 
334   LIS->RemoveMachineInstrFromMaps(MI);
335   MI.eraseFromParent();
336 
337   LIS->InsertMachineInstrInMaps(*CopyExec);
338   LIS->InsertMachineInstrInMaps(*OrSaveExec);
339 
340   LIS->InsertMachineInstrInMaps(*Xor);
341   LIS->InsertMachineInstrInMaps(*Branch);
342 
343   // src reg is tied to dst reg.
344   LIS->removeInterval(DstReg);
345   LIS->createAndComputeVirtRegInterval(DstReg);
346   LIS->createAndComputeVirtRegInterval(CopyReg);
347   if (ExecModified)
348     LIS->createAndComputeVirtRegInterval(SaveReg);
349 
350   // Let this be recomputed.
351   LIS->removeAllRegUnitsForPhysReg(AMDGPU::EXEC);
352 }
353 
emitIfBreak(MachineInstr & MI)354 void SILowerControlFlow::emitIfBreak(MachineInstr &MI) {
355   MachineBasicBlock &MBB = *MI.getParent();
356   const DebugLoc &DL = MI.getDebugLoc();
357   auto Dst = getSaveExec(&MI);
358 
359   // Skip ANDing with exec if the break condition is already masked by exec
360   // because it is a V_CMP in the same basic block. (We know the break
361   // condition operand was an i1 in IR, so if it is a VALU instruction it must
362   // be one with a carry-out.)
363   bool SkipAnding = false;
364   if (MI.getOperand(1).isReg()) {
365     if (MachineInstr *Def = MRI->getUniqueVRegDef(MI.getOperand(1).getReg())) {
366       SkipAnding = Def->getParent() == MI.getParent()
367           && SIInstrInfo::isVALU(*Def);
368     }
369   }
370 
371   // AND the break condition operand with exec, then OR that into the "loop
372   // exit" mask.
373   MachineInstr *And = nullptr, *Or = nullptr;
374   if (!SkipAnding) {
375     Register AndReg = MRI->createVirtualRegister(BoolRC);
376     And = BuildMI(MBB, &MI, DL, TII->get(AndOpc), AndReg)
377              .addReg(Exec)
378              .add(MI.getOperand(1));
379     Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst)
380              .addReg(AndReg)
381              .add(MI.getOperand(2));
382     if (LIS)
383       LIS->createAndComputeVirtRegInterval(AndReg);
384   } else
385     Or = BuildMI(MBB, &MI, DL, TII->get(OrOpc), Dst)
386              .add(MI.getOperand(1))
387              .add(MI.getOperand(2));
388 
389   if (LIS) {
390     if (And)
391       LIS->InsertMachineInstrInMaps(*And);
392     LIS->ReplaceMachineInstrInMaps(MI, *Or);
393   }
394 
395   MI.eraseFromParent();
396 }
397 
emitLoop(MachineInstr & MI)398 void SILowerControlFlow::emitLoop(MachineInstr &MI) {
399   MachineBasicBlock &MBB = *MI.getParent();
400   const DebugLoc &DL = MI.getDebugLoc();
401 
402   MachineInstr *AndN2 =
403       BuildMI(MBB, &MI, DL, TII->get(Andn2TermOpc), Exec)
404           .addReg(Exec)
405           .add(MI.getOperand(0));
406 
407   MachineInstr *Branch =
408       BuildMI(MBB, &MI, DL, TII->get(AMDGPU::S_CBRANCH_EXECNZ))
409           .add(MI.getOperand(1));
410 
411   if (LIS) {
412     LIS->ReplaceMachineInstrInMaps(MI, *AndN2);
413     LIS->InsertMachineInstrInMaps(*Branch);
414   }
415 
416   MI.eraseFromParent();
417 }
418 
emitEndCf(MachineInstr & MI)419 void SILowerControlFlow::emitEndCf(MachineInstr &MI) {
420   MachineBasicBlock &MBB = *MI.getParent();
421   MachineRegisterInfo &MRI = MBB.getParent()->getRegInfo();
422   unsigned CFMask = MI.getOperand(0).getReg();
423   MachineInstr *Def = MRI.getUniqueVRegDef(CFMask);
424   const DebugLoc &DL = MI.getDebugLoc();
425 
426   MachineBasicBlock::iterator InsPt =
427       Def && Def->getParent() == &MBB ? std::next(MachineBasicBlock::iterator(Def))
428                                : MBB.begin();
429   MachineInstr *NewMI = BuildMI(MBB, InsPt, DL, TII->get(OrOpc), Exec)
430                             .addReg(Exec)
431                             .add(MI.getOperand(0));
432 
433   if (LIS)
434     LIS->ReplaceMachineInstrInMaps(MI, *NewMI);
435 
436   MI.eraseFromParent();
437 
438   if (LIS)
439     LIS->handleMove(*NewMI);
440 }
441 
442 // Returns replace operands for a logical operation, either single result
443 // for exec or two operands if source was another equivalent operation.
findMaskOperands(MachineInstr & MI,unsigned OpNo,SmallVectorImpl<MachineOperand> & Src) const444 void SILowerControlFlow::findMaskOperands(MachineInstr &MI, unsigned OpNo,
445        SmallVectorImpl<MachineOperand> &Src) const {
446   MachineOperand &Op = MI.getOperand(OpNo);
447   if (!Op.isReg() || !Register::isVirtualRegister(Op.getReg())) {
448     Src.push_back(Op);
449     return;
450   }
451 
452   MachineInstr *Def = MRI->getUniqueVRegDef(Op.getReg());
453   if (!Def || Def->getParent() != MI.getParent() ||
454       !(Def->isFullCopy() || (Def->getOpcode() == MI.getOpcode())))
455     return;
456 
457   // Make sure we do not modify exec between def and use.
458   // A copy with implcitly defined exec inserted earlier is an exclusion, it
459   // does not really modify exec.
460   for (auto I = Def->getIterator(); I != MI.getIterator(); ++I)
461     if (I->modifiesRegister(AMDGPU::EXEC, TRI) &&
462         !(I->isCopy() && I->getOperand(0).getReg() != Exec))
463       return;
464 
465   for (const auto &SrcOp : Def->explicit_operands())
466     if (SrcOp.isReg() && SrcOp.isUse() &&
467         (Register::isVirtualRegister(SrcOp.getReg()) || SrcOp.getReg() == Exec))
468       Src.push_back(SrcOp);
469 }
470 
471 // Search and combine pairs of equivalent instructions, like
472 // S_AND_B64 x, (S_AND_B64 x, y) => S_AND_B64 x, y
473 // S_OR_B64  x, (S_OR_B64  x, y) => S_OR_B64  x, y
474 // One of the operands is exec mask.
combineMasks(MachineInstr & MI)475 void SILowerControlFlow::combineMasks(MachineInstr &MI) {
476   assert(MI.getNumExplicitOperands() == 3);
477   SmallVector<MachineOperand, 4> Ops;
478   unsigned OpToReplace = 1;
479   findMaskOperands(MI, 1, Ops);
480   if (Ops.size() == 1) OpToReplace = 2; // First operand can be exec or its copy
481   findMaskOperands(MI, 2, Ops);
482   if (Ops.size() != 3) return;
483 
484   unsigned UniqueOpndIdx;
485   if (Ops[0].isIdenticalTo(Ops[1])) UniqueOpndIdx = 2;
486   else if (Ops[0].isIdenticalTo(Ops[2])) UniqueOpndIdx = 1;
487   else if (Ops[1].isIdenticalTo(Ops[2])) UniqueOpndIdx = 1;
488   else return;
489 
490   Register Reg = MI.getOperand(OpToReplace).getReg();
491   MI.RemoveOperand(OpToReplace);
492   MI.addOperand(Ops[UniqueOpndIdx]);
493   if (MRI->use_empty(Reg))
494     MRI->getUniqueVRegDef(Reg)->eraseFromParent();
495 }
496 
runOnMachineFunction(MachineFunction & MF)497 bool SILowerControlFlow::runOnMachineFunction(MachineFunction &MF) {
498   const GCNSubtarget &ST = MF.getSubtarget<GCNSubtarget>();
499   TII = ST.getInstrInfo();
500   TRI = &TII->getRegisterInfo();
501 
502   // This doesn't actually need LiveIntervals, but we can preserve them.
503   LIS = getAnalysisIfAvailable<LiveIntervals>();
504   MRI = &MF.getRegInfo();
505   BoolRC = TRI->getBoolRC();
506 
507   if (ST.isWave32()) {
508     AndOpc = AMDGPU::S_AND_B32;
509     OrOpc = AMDGPU::S_OR_B32;
510     XorOpc = AMDGPU::S_XOR_B32;
511     MovTermOpc = AMDGPU::S_MOV_B32_term;
512     Andn2TermOpc = AMDGPU::S_ANDN2_B32_term;
513     XorTermrOpc = AMDGPU::S_XOR_B32_term;
514     OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B32;
515     Exec = AMDGPU::EXEC_LO;
516   } else {
517     AndOpc = AMDGPU::S_AND_B64;
518     OrOpc = AMDGPU::S_OR_B64;
519     XorOpc = AMDGPU::S_XOR_B64;
520     MovTermOpc = AMDGPU::S_MOV_B64_term;
521     Andn2TermOpc = AMDGPU::S_ANDN2_B64_term;
522     XorTermrOpc = AMDGPU::S_XOR_B64_term;
523     OrSaveExecOpc = AMDGPU::S_OR_SAVEEXEC_B64;
524     Exec = AMDGPU::EXEC;
525   }
526 
527   MachineFunction::iterator NextBB;
528   for (MachineFunction::iterator BI = MF.begin(), BE = MF.end();
529        BI != BE; BI = NextBB) {
530     NextBB = std::next(BI);
531     MachineBasicBlock &MBB = *BI;
532 
533     MachineBasicBlock::iterator I, Next, Last;
534 
535     for (I = MBB.begin(), Last = MBB.end(); I != MBB.end(); I = Next) {
536       Next = std::next(I);
537       MachineInstr &MI = *I;
538 
539       switch (MI.getOpcode()) {
540       case AMDGPU::SI_IF:
541         emitIf(MI);
542         break;
543 
544       case AMDGPU::SI_ELSE:
545         emitElse(MI);
546         break;
547 
548       case AMDGPU::SI_IF_BREAK:
549         emitIfBreak(MI);
550         break;
551 
552       case AMDGPU::SI_LOOP:
553         emitLoop(MI);
554         break;
555 
556       case AMDGPU::SI_END_CF:
557         emitEndCf(MI);
558         break;
559 
560       case AMDGPU::S_AND_B64:
561       case AMDGPU::S_OR_B64:
562       case AMDGPU::S_AND_B32:
563       case AMDGPU::S_OR_B32:
564         // Cleanup bit manipulations on exec mask
565         combineMasks(MI);
566         Last = I;
567         continue;
568 
569       default:
570         Last = I;
571         continue;
572       }
573 
574       // Replay newly inserted code to combine masks
575       Next = (Last == MBB.end()) ? MBB.begin() : Last;
576     }
577   }
578 
579   return true;
580 }
581