1 //===- R600ControlFlowFinalizer.cpp - Finalize Control Flow Inst ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 /// \file
10 /// This pass compute turns all control flow pseudo instructions into native one
11 /// computing their address on the fly; it also sets STACK_SIZE info.
12 //
13 //===----------------------------------------------------------------------===//
14 
15 #include "MCTargetDesc/R600MCTargetDesc.h"
16 #include "R600.h"
17 #include "R600MachineFunctionInfo.h"
18 #include "R600Subtarget.h"
19 #include "llvm/CodeGen/MachineFunctionPass.h"
20 #include <set>
21 
22 using namespace llvm;
23 
24 #define DEBUG_TYPE "r600cf"
25 
26 namespace {
27 
28 struct CFStack {
29   enum StackItem {
30     ENTRY = 0,
31     SUB_ENTRY = 1,
32     FIRST_NON_WQM_PUSH = 2,
33     FIRST_NON_WQM_PUSH_W_FULL_ENTRY = 3
34   };
35 
36   const R600Subtarget *ST;
37   std::vector<StackItem> BranchStack;
38   std::vector<StackItem> LoopStack;
39   unsigned MaxStackSize;
40   unsigned CurrentEntries = 0;
41   unsigned CurrentSubEntries = 0;
42 
43   CFStack(const R600Subtarget *st, CallingConv::ID cc) : ST(st),
44       // We need to reserve a stack entry for CALL_FS in vertex shaders.
45       MaxStackSize(cc == CallingConv::AMDGPU_VS ? 1 : 0) {}
46 
47   unsigned getLoopDepth();
48   bool branchStackContains(CFStack::StackItem);
49   bool requiresWorkAroundForInst(unsigned Opcode);
50   unsigned getSubEntrySize(CFStack::StackItem Item);
51   void updateMaxStackSize();
52   void pushBranch(unsigned Opcode, bool isWQM = false);
53   void pushLoop();
54   void popBranch();
55   void popLoop();
56 };
57 
58 unsigned CFStack::getLoopDepth() {
59   return LoopStack.size();
60 }
61 
62 bool CFStack::branchStackContains(CFStack::StackItem Item) {
63   return llvm::is_contained(BranchStack, Item);
64 }
65 
66 bool CFStack::requiresWorkAroundForInst(unsigned Opcode) {
67   if (Opcode == R600::CF_ALU_PUSH_BEFORE && ST->hasCaymanISA() &&
68       getLoopDepth() > 1)
69     return true;
70 
71   if (!ST->hasCFAluBug())
72     return false;
73 
74   switch(Opcode) {
75   default: return false;
76   case R600::CF_ALU_PUSH_BEFORE:
77   case R600::CF_ALU_ELSE_AFTER:
78   case R600::CF_ALU_BREAK:
79   case R600::CF_ALU_CONTINUE:
80     if (CurrentSubEntries == 0)
81       return false;
82     if (ST->getWavefrontSize() == 64) {
83       // We are being conservative here.  We only require this work-around if
84       // CurrentSubEntries > 3 &&
85       // (CurrentSubEntries % 4 == 3 || CurrentSubEntries % 4 == 0)
86       //
87       // We have to be conservative, because we don't know for certain that
88       // our stack allocation algorithm for Evergreen/NI is correct.  Applying this
89       // work-around when CurrentSubEntries > 3 allows us to over-allocate stack
90       // resources without any problems.
91       return CurrentSubEntries > 3;
92     } else {
93       assert(ST->getWavefrontSize() == 32);
94       // We are being conservative here.  We only require the work-around if
95       // CurrentSubEntries > 7 &&
96       // (CurrentSubEntries % 8 == 7 || CurrentSubEntries % 8 == 0)
97       // See the comment on the wavefront size == 64 case for why we are
98       // being conservative.
99       return CurrentSubEntries > 7;
100     }
101   }
102 }
103 
104 unsigned CFStack::getSubEntrySize(CFStack::StackItem Item) {
105   switch(Item) {
106   default:
107     return 0;
108   case CFStack::FIRST_NON_WQM_PUSH:
109   assert(!ST->hasCaymanISA());
110   if (ST->getGeneration() <= AMDGPUSubtarget::R700) {
111     // +1 For the push operation.
112     // +2 Extra space required.
113     return 3;
114   } else {
115     // Some documentation says that this is not necessary on Evergreen,
116     // but experimentation has show that we need to allocate 1 extra
117     // sub-entry for the first non-WQM push.
118     // +1 For the push operation.
119     // +1 Extra space required.
120     return 2;
121   }
122   case CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY:
123     assert(ST->getGeneration() >= AMDGPUSubtarget::EVERGREEN);
124     // +1 For the push operation.
125     // +1 Extra space required.
126     return 2;
127   case CFStack::SUB_ENTRY:
128     return 1;
129   }
130 }
131 
132 void CFStack::updateMaxStackSize() {
133   unsigned CurrentStackSize = CurrentEntries + divideCeil(CurrentSubEntries, 4);
134   MaxStackSize = std::max(CurrentStackSize, MaxStackSize);
135 }
136 
137 void CFStack::pushBranch(unsigned Opcode, bool isWQM) {
138   CFStack::StackItem Item = CFStack::ENTRY;
139   switch(Opcode) {
140   case R600::CF_PUSH_EG:
141   case R600::CF_ALU_PUSH_BEFORE:
142     if (!isWQM) {
143       if (!ST->hasCaymanISA() &&
144           !branchStackContains(CFStack::FIRST_NON_WQM_PUSH))
145         Item = CFStack::FIRST_NON_WQM_PUSH;  // May not be required on Evergreen/NI
146                                              // See comment in
147                                              // CFStack::getSubEntrySize()
148       else if (CurrentEntries > 0 &&
149                ST->getGeneration() > AMDGPUSubtarget::EVERGREEN &&
150                !ST->hasCaymanISA() &&
151                !branchStackContains(CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY))
152         Item = CFStack::FIRST_NON_WQM_PUSH_W_FULL_ENTRY;
153       else
154         Item = CFStack::SUB_ENTRY;
155     } else
156       Item = CFStack::ENTRY;
157     break;
158   }
159   BranchStack.push_back(Item);
160   if (Item == CFStack::ENTRY)
161     CurrentEntries++;
162   else
163     CurrentSubEntries += getSubEntrySize(Item);
164   updateMaxStackSize();
165 }
166 
167 void CFStack::pushLoop() {
168   LoopStack.push_back(CFStack::ENTRY);
169   CurrentEntries++;
170   updateMaxStackSize();
171 }
172 
173 void CFStack::popBranch() {
174   CFStack::StackItem Top = BranchStack.back();
175   if (Top == CFStack::ENTRY)
176     CurrentEntries--;
177   else
178     CurrentSubEntries-= getSubEntrySize(Top);
179   BranchStack.pop_back();
180 }
181 
182 void CFStack::popLoop() {
183   CurrentEntries--;
184   LoopStack.pop_back();
185 }
186 
187 class R600ControlFlowFinalizer : public MachineFunctionPass {
188 private:
189   using ClauseFile = std::pair<MachineInstr *, std::vector<MachineInstr *>>;
190 
191   enum ControlFlowInstruction {
192     CF_TC,
193     CF_VC,
194     CF_CALL_FS,
195     CF_WHILE_LOOP,
196     CF_END_LOOP,
197     CF_LOOP_BREAK,
198     CF_LOOP_CONTINUE,
199     CF_JUMP,
200     CF_ELSE,
201     CF_POP,
202     CF_END
203   };
204 
205   const R600InstrInfo *TII = nullptr;
206   const R600RegisterInfo *TRI = nullptr;
207   unsigned MaxFetchInst;
208   const R600Subtarget *ST = nullptr;
209 
210   bool IsTrivialInst(MachineInstr &MI) const {
211     switch (MI.getOpcode()) {
212     case R600::KILL:
213     case R600::RETURN:
214       return true;
215     default:
216       return false;
217     }
218   }
219 
220   const MCInstrDesc &getHWInstrDesc(ControlFlowInstruction CFI) const {
221     unsigned Opcode = 0;
222     bool isEg = (ST->getGeneration() >= AMDGPUSubtarget::EVERGREEN);
223     switch (CFI) {
224     case CF_TC:
225       Opcode = isEg ? R600::CF_TC_EG : R600::CF_TC_R600;
226       break;
227     case CF_VC:
228       Opcode = isEg ? R600::CF_VC_EG : R600::CF_VC_R600;
229       break;
230     case CF_CALL_FS:
231       Opcode = isEg ? R600::CF_CALL_FS_EG : R600::CF_CALL_FS_R600;
232       break;
233     case CF_WHILE_LOOP:
234       Opcode = isEg ? R600::WHILE_LOOP_EG : R600::WHILE_LOOP_R600;
235       break;
236     case CF_END_LOOP:
237       Opcode = isEg ? R600::END_LOOP_EG : R600::END_LOOP_R600;
238       break;
239     case CF_LOOP_BREAK:
240       Opcode = isEg ? R600::LOOP_BREAK_EG : R600::LOOP_BREAK_R600;
241       break;
242     case CF_LOOP_CONTINUE:
243       Opcode = isEg ? R600::CF_CONTINUE_EG : R600::CF_CONTINUE_R600;
244       break;
245     case CF_JUMP:
246       Opcode = isEg ? R600::CF_JUMP_EG : R600::CF_JUMP_R600;
247       break;
248     case CF_ELSE:
249       Opcode = isEg ? R600::CF_ELSE_EG : R600::CF_ELSE_R600;
250       break;
251     case CF_POP:
252       Opcode = isEg ? R600::POP_EG : R600::POP_R600;
253       break;
254     case CF_END:
255       if (ST->hasCaymanISA()) {
256         Opcode = R600::CF_END_CM;
257         break;
258       }
259       Opcode = isEg ? R600::CF_END_EG : R600::CF_END_R600;
260       break;
261     }
262     assert (Opcode && "No opcode selected");
263     return TII->get(Opcode);
264   }
265 
266   bool isCompatibleWithClause(const MachineInstr &MI,
267                               std::set<unsigned> &DstRegs) const {
268     unsigned DstMI, SrcMI;
269     for (MachineInstr::const_mop_iterator I = MI.operands_begin(),
270                                           E = MI.operands_end();
271          I != E; ++I) {
272       const MachineOperand &MO = *I;
273       if (!MO.isReg())
274         continue;
275       if (MO.isDef()) {
276         Register Reg = MO.getReg();
277         if (R600::R600_Reg128RegClass.contains(Reg))
278           DstMI = Reg;
279         else
280           DstMI = TRI->getMatchingSuperReg(Reg,
281               R600RegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)),
282               &R600::R600_Reg128RegClass);
283       }
284       if (MO.isUse()) {
285         Register Reg = MO.getReg();
286         if (R600::R600_Reg128RegClass.contains(Reg))
287           SrcMI = Reg;
288         else
289           SrcMI = TRI->getMatchingSuperReg(Reg,
290               R600RegisterInfo::getSubRegFromChannel(TRI->getHWRegChan(Reg)),
291               &R600::R600_Reg128RegClass);
292       }
293     }
294     if ((DstRegs.find(SrcMI) == DstRegs.end())) {
295       DstRegs.insert(DstMI);
296       return true;
297     } else
298       return false;
299   }
300 
301   ClauseFile
302   MakeFetchClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
303       const {
304     MachineBasicBlock::iterator ClauseHead = I;
305     std::vector<MachineInstr *> ClauseContent;
306     unsigned AluInstCount = 0;
307     bool IsTex = TII->usesTextureCache(*ClauseHead);
308     std::set<unsigned> DstRegs;
309     for (MachineBasicBlock::iterator E = MBB.end(); I != E; ++I) {
310       if (IsTrivialInst(*I))
311         continue;
312       if (AluInstCount >= MaxFetchInst)
313         break;
314       if ((IsTex && !TII->usesTextureCache(*I)) ||
315           (!IsTex && !TII->usesVertexCache(*I)))
316         break;
317       if (!isCompatibleWithClause(*I, DstRegs))
318         break;
319       AluInstCount ++;
320       ClauseContent.push_back(&*I);
321     }
322     MachineInstr *MIb = BuildMI(MBB, ClauseHead, MBB.findDebugLoc(ClauseHead),
323         getHWInstrDesc(IsTex?CF_TC:CF_VC))
324         .addImm(0) // ADDR
325         .addImm(AluInstCount - 1); // COUNT
326     return ClauseFile(MIb, std::move(ClauseContent));
327   }
328 
329   void getLiteral(MachineInstr &MI, std::vector<MachineOperand *> &Lits) const {
330     static const unsigned LiteralRegs[] = {
331       R600::ALU_LITERAL_X,
332       R600::ALU_LITERAL_Y,
333       R600::ALU_LITERAL_Z,
334       R600::ALU_LITERAL_W
335     };
336     const SmallVector<std::pair<MachineOperand *, int64_t>, 3> Srcs =
337         TII->getSrcs(MI);
338     for (const auto &Src:Srcs) {
339       if (Src.first->getReg() != R600::ALU_LITERAL_X)
340         continue;
341       int64_t Imm = Src.second;
342       std::vector<MachineOperand *>::iterator It =
343           llvm::find_if(Lits, [&](MachineOperand *val) {
344             return val->isImm() && (val->getImm() == Imm);
345           });
346 
347       // Get corresponding Operand
348       MachineOperand &Operand = MI.getOperand(
349           TII->getOperandIdx(MI.getOpcode(), R600::OpName::literal));
350 
351       if (It != Lits.end()) {
352         // Reuse existing literal reg
353         unsigned Index = It - Lits.begin();
354         Src.first->setReg(LiteralRegs[Index]);
355       } else {
356         // Allocate new literal reg
357         assert(Lits.size() < 4 && "Too many literals in Instruction Group");
358         Src.first->setReg(LiteralRegs[Lits.size()]);
359         Lits.push_back(&Operand);
360       }
361     }
362   }
363 
364   MachineBasicBlock::iterator insertLiterals(
365       MachineBasicBlock::iterator InsertPos,
366       const std::vector<unsigned> &Literals) const {
367     MachineBasicBlock *MBB = InsertPos->getParent();
368     for (unsigned i = 0, e = Literals.size(); i < e; i+=2) {
369       unsigned LiteralPair0 = Literals[i];
370       unsigned LiteralPair1 = (i + 1 < e)?Literals[i + 1]:0;
371       InsertPos = BuildMI(MBB, InsertPos->getDebugLoc(),
372           TII->get(R600::LITERALS))
373           .addImm(LiteralPair0)
374           .addImm(LiteralPair1);
375     }
376     return InsertPos;
377   }
378 
379   ClauseFile
380   MakeALUClause(MachineBasicBlock &MBB, MachineBasicBlock::iterator &I)
381       const {
382     MachineInstr &ClauseHead = *I;
383     std::vector<MachineInstr *> ClauseContent;
384     I++;
385     for (MachineBasicBlock::instr_iterator E = MBB.instr_end(); I != E;) {
386       if (IsTrivialInst(*I)) {
387         ++I;
388         continue;
389       }
390       if (!I->isBundle() && !TII->isALUInstr(I->getOpcode()))
391         break;
392       std::vector<MachineOperand *>Literals;
393       if (I->isBundle()) {
394         MachineInstr &DeleteMI = *I;
395         MachineBasicBlock::instr_iterator BI = I.getInstrIterator();
396         while (++BI != E && BI->isBundledWithPred()) {
397           BI->unbundleFromPred();
398           for (MachineOperand &MO : BI->operands()) {
399             if (MO.isReg() && MO.isInternalRead())
400               MO.setIsInternalRead(false);
401           }
402           getLiteral(*BI, Literals);
403           ClauseContent.push_back(&*BI);
404         }
405         I = BI;
406         DeleteMI.eraseFromParent();
407       } else {
408         getLiteral(*I, Literals);
409         ClauseContent.push_back(&*I);
410         I++;
411       }
412       for (unsigned i = 0, e = Literals.size(); i < e; i += 2) {
413         MachineInstrBuilder MILit = BuildMI(MBB, I, I->getDebugLoc(),
414             TII->get(R600::LITERALS));
415         if (Literals[i]->isImm()) {
416             MILit.addImm(Literals[i]->getImm());
417         } else {
418             MILit.addGlobalAddress(Literals[i]->getGlobal(),
419                                    Literals[i]->getOffset());
420         }
421         if (i + 1 < e) {
422           if (Literals[i + 1]->isImm()) {
423             MILit.addImm(Literals[i + 1]->getImm());
424           } else {
425             MILit.addGlobalAddress(Literals[i + 1]->getGlobal(),
426                                    Literals[i + 1]->getOffset());
427           }
428         } else
429           MILit.addImm(0);
430         ClauseContent.push_back(MILit);
431       }
432     }
433     assert(ClauseContent.size() < 128 && "ALU clause is too big");
434     ClauseHead.getOperand(7).setImm(ClauseContent.size() - 1);
435     return ClauseFile(&ClauseHead, std::move(ClauseContent));
436   }
437 
438   void EmitFetchClause(MachineBasicBlock::iterator InsertPos,
439                        const DebugLoc &DL, ClauseFile &Clause,
440                        unsigned &CfCount) {
441     CounterPropagateAddr(*Clause.first, CfCount);
442     MachineBasicBlock *BB = Clause.first->getParent();
443     BuildMI(BB, DL, TII->get(R600::FETCH_CLAUSE)).addImm(CfCount);
444     for (MachineInstr *MI : Clause.second)
445       BB->splice(InsertPos, BB, MI);
446     CfCount += 2 * Clause.second.size();
447   }
448 
449   void EmitALUClause(MachineBasicBlock::iterator InsertPos, const DebugLoc &DL,
450                      ClauseFile &Clause, unsigned &CfCount) {
451     Clause.first->getOperand(0).setImm(0);
452     CounterPropagateAddr(*Clause.first, CfCount);
453     MachineBasicBlock *BB = Clause.first->getParent();
454     BuildMI(BB, DL, TII->get(R600::ALU_CLAUSE)).addImm(CfCount);
455     for (MachineInstr *MI : Clause.second)
456       BB->splice(InsertPos, BB, MI);
457     CfCount += Clause.second.size();
458   }
459 
460   void CounterPropagateAddr(MachineInstr &MI, unsigned Addr) const {
461     MI.getOperand(0).setImm(Addr + MI.getOperand(0).getImm());
462   }
463   void CounterPropagateAddr(const std::set<MachineInstr *> &MIs,
464                             unsigned Addr) const {
465     for (MachineInstr *MI : MIs) {
466       CounterPropagateAddr(*MI, Addr);
467     }
468   }
469 
470 public:
471   static char ID;
472 
473   R600ControlFlowFinalizer() : MachineFunctionPass(ID) {}
474 
475   bool runOnMachineFunction(MachineFunction &MF) override {
476     ST = &MF.getSubtarget<R600Subtarget>();
477     MaxFetchInst = ST->getTexVTXClauseSize();
478     TII = ST->getInstrInfo();
479     TRI = ST->getRegisterInfo();
480 
481     R600MachineFunctionInfo *MFI = MF.getInfo<R600MachineFunctionInfo>();
482 
483     CFStack CFStack(ST, MF.getFunction().getCallingConv());
484     for (MachineFunction::iterator MB = MF.begin(), ME = MF.end(); MB != ME;
485         ++MB) {
486       MachineBasicBlock &MBB = *MB;
487       unsigned CfCount = 0;
488       std::vector<std::pair<unsigned, std::set<MachineInstr *>>> LoopStack;
489       std::vector<MachineInstr * > IfThenElseStack;
490       if (MF.getFunction().getCallingConv() == CallingConv::AMDGPU_VS) {
491         BuildMI(MBB, MBB.begin(), MBB.findDebugLoc(MBB.begin()),
492             getHWInstrDesc(CF_CALL_FS));
493         CfCount++;
494       }
495       std::vector<ClauseFile> FetchClauses, AluClauses;
496       std::vector<MachineInstr *> LastAlu(1);
497       std::vector<MachineInstr *> ToPopAfter;
498 
499       for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end();
500           I != E;) {
501         if (TII->usesTextureCache(*I) || TII->usesVertexCache(*I)) {
502           LLVM_DEBUG(dbgs() << CfCount << ":"; I->dump(););
503           FetchClauses.push_back(MakeFetchClause(MBB, I));
504           CfCount++;
505           LastAlu.back() = nullptr;
506           continue;
507         }
508 
509         MachineBasicBlock::iterator MI = I;
510         if (MI->getOpcode() != R600::ENDIF)
511           LastAlu.back() = nullptr;
512         if (MI->getOpcode() == R600::CF_ALU)
513           LastAlu.back() = &*MI;
514         I++;
515         bool RequiresWorkAround =
516             CFStack.requiresWorkAroundForInst(MI->getOpcode());
517         switch (MI->getOpcode()) {
518         case R600::CF_ALU_PUSH_BEFORE:
519           if (RequiresWorkAround) {
520             LLVM_DEBUG(dbgs()
521                        << "Applying bug work-around for ALU_PUSH_BEFORE\n");
522             BuildMI(MBB, MI, MBB.findDebugLoc(MI), TII->get(R600::CF_PUSH_EG))
523                 .addImm(CfCount + 1)
524                 .addImm(1);
525             MI->setDesc(TII->get(R600::CF_ALU));
526             CfCount++;
527             CFStack.pushBranch(R600::CF_PUSH_EG);
528           } else
529             CFStack.pushBranch(R600::CF_ALU_PUSH_BEFORE);
530           [[fallthrough]];
531         case R600::CF_ALU:
532           I = MI;
533           AluClauses.push_back(MakeALUClause(MBB, I));
534           LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump(););
535           CfCount++;
536           break;
537         case R600::WHILELOOP: {
538           CFStack.pushLoop();
539           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
540               getHWInstrDesc(CF_WHILE_LOOP))
541               .addImm(1);
542           std::pair<unsigned, std::set<MachineInstr *>> Pair(CfCount,
543               std::set<MachineInstr *>());
544           Pair.second.insert(MIb);
545           LoopStack.push_back(std::move(Pair));
546           MI->eraseFromParent();
547           CfCount++;
548           break;
549         }
550         case R600::ENDLOOP: {
551           CFStack.popLoop();
552           std::pair<unsigned, std::set<MachineInstr *>> Pair =
553               std::move(LoopStack.back());
554           LoopStack.pop_back();
555           CounterPropagateAddr(Pair.second, CfCount);
556           BuildMI(MBB, MI, MBB.findDebugLoc(MI), getHWInstrDesc(CF_END_LOOP))
557               .addImm(Pair.first + 1);
558           MI->eraseFromParent();
559           CfCount++;
560           break;
561         }
562         case R600::IF_PREDICATE_SET: {
563           LastAlu.push_back(nullptr);
564           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
565               getHWInstrDesc(CF_JUMP))
566               .addImm(0)
567               .addImm(0);
568           IfThenElseStack.push_back(MIb);
569           LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
570           MI->eraseFromParent();
571           CfCount++;
572           break;
573         }
574         case R600::ELSE: {
575           MachineInstr * JumpInst = IfThenElseStack.back();
576           IfThenElseStack.pop_back();
577           CounterPropagateAddr(*JumpInst, CfCount);
578           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
579               getHWInstrDesc(CF_ELSE))
580               .addImm(0)
581               .addImm(0);
582           LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
583           IfThenElseStack.push_back(MIb);
584           MI->eraseFromParent();
585           CfCount++;
586           break;
587         }
588         case R600::ENDIF: {
589           CFStack.popBranch();
590           if (LastAlu.back()) {
591             ToPopAfter.push_back(LastAlu.back());
592           } else {
593             MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
594                 getHWInstrDesc(CF_POP))
595                 .addImm(CfCount + 1)
596                 .addImm(1);
597             (void)MIb;
598             LLVM_DEBUG(dbgs() << CfCount << ":"; MIb->dump(););
599             CfCount++;
600           }
601 
602           MachineInstr *IfOrElseInst = IfThenElseStack.back();
603           IfThenElseStack.pop_back();
604           CounterPropagateAddr(*IfOrElseInst, CfCount);
605           IfOrElseInst->getOperand(1).setImm(1);
606           LastAlu.pop_back();
607           MI->eraseFromParent();
608           break;
609         }
610         case R600::BREAK: {
611           CfCount ++;
612           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
613               getHWInstrDesc(CF_LOOP_BREAK))
614               .addImm(0);
615           LoopStack.back().second.insert(MIb);
616           MI->eraseFromParent();
617           break;
618         }
619         case R600::CONTINUE: {
620           MachineInstr *MIb = BuildMI(MBB, MI, MBB.findDebugLoc(MI),
621               getHWInstrDesc(CF_LOOP_CONTINUE))
622               .addImm(0);
623           LoopStack.back().second.insert(MIb);
624           MI->eraseFromParent();
625           CfCount++;
626           break;
627         }
628         case R600::RETURN: {
629           DebugLoc DL = MBB.findDebugLoc(MI);
630           BuildMI(MBB, MI, DL, getHWInstrDesc(CF_END));
631           CfCount++;
632           if (CfCount % 2) {
633             BuildMI(MBB, I, DL, TII->get(R600::PAD));
634             CfCount++;
635           }
636           MI->eraseFromParent();
637           for (ClauseFile &CF : FetchClauses)
638             EmitFetchClause(I, DL, CF, CfCount);
639           for (ClauseFile &CF : AluClauses)
640             EmitALUClause(I, DL, CF, CfCount);
641           break;
642         }
643         default:
644           if (TII->isExport(MI->getOpcode())) {
645             LLVM_DEBUG(dbgs() << CfCount << ":"; MI->dump(););
646             CfCount++;
647           }
648           break;
649         }
650       }
651       for (MachineInstr *Alu : ToPopAfter) {
652         BuildMI(MBB, Alu, MBB.findDebugLoc((MachineBasicBlock::iterator)Alu),
653             TII->get(R600::CF_ALU_POP_AFTER))
654             .addImm(Alu->getOperand(0).getImm())
655             .addImm(Alu->getOperand(1).getImm())
656             .addImm(Alu->getOperand(2).getImm())
657             .addImm(Alu->getOperand(3).getImm())
658             .addImm(Alu->getOperand(4).getImm())
659             .addImm(Alu->getOperand(5).getImm())
660             .addImm(Alu->getOperand(6).getImm())
661             .addImm(Alu->getOperand(7).getImm())
662             .addImm(Alu->getOperand(8).getImm());
663         Alu->eraseFromParent();
664       }
665       MFI->CFStackSize = CFStack.MaxStackSize;
666     }
667 
668     return false;
669   }
670 
671   StringRef getPassName() const override {
672     return "R600 Control Flow Finalizer Pass";
673   }
674 };
675 
676 } // end anonymous namespace
677 
678 INITIALIZE_PASS_BEGIN(R600ControlFlowFinalizer, DEBUG_TYPE,
679                      "R600 Control Flow Finalizer", false, false)
680 INITIALIZE_PASS_END(R600ControlFlowFinalizer, DEBUG_TYPE,
681                     "R600 Control Flow Finalizer", false, false)
682 
683 char R600ControlFlowFinalizer::ID = 0;
684 
685 char &llvm::R600ControlFlowFinalizerID = R600ControlFlowFinalizer::ID;
686 
687 FunctionPass *llvm::createR600ControlFlowFinalizer() {
688   return new R600ControlFlowFinalizer();
689 }
690