1 /*
2 * Copyright © 2019 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "aco_builder.h"
26 #include "aco_ir.h"
27
28 #include "util/u_math.h"
29
30 #include <set>
31 #include <vector>
32
33 namespace aco {
34
35 namespace {
36
37 enum WQMState : uint8_t {
38 Unspecified = 0,
39 Exact = 1 << 0,
40 WQM = 1 << 1, /* with control flow applied */
41 };
42
43 enum mask_type : uint8_t {
44 mask_type_global = 1 << 0,
45 mask_type_exact = 1 << 1,
46 mask_type_wqm = 1 << 2,
47 mask_type_loop = 1 << 3, /* active lanes of a loop */
48 };
49
50 struct wqm_ctx {
51 Program* program;
52 /* state for WQM propagation */
53 std::set<unsigned> worklist;
54 std::vector<bool> branch_wqm; /* true if the branch condition in this block should be in wqm */
wqm_ctxaco::__anonc52d909e0111::wqm_ctx55 wqm_ctx(Program* program_)
56 : program(program_), branch_wqm(program->blocks.size())
57 {
58 for (unsigned i = 0; i < program->blocks.size(); i++)
59 worklist.insert(i);
60 }
61 };
62
63 struct loop_info {
64 Block* loop_header;
65 uint16_t num_exec_masks;
66 bool has_divergent_break;
67 bool has_divergent_continue;
68 bool has_discard; /* has a discard or demote */
loop_infoaco::__anonc52d909e0111::loop_info69 loop_info(Block* b, uint16_t num, bool breaks, bool cont, bool discard)
70 : loop_header(b), num_exec_masks(num), has_divergent_break(breaks),
71 has_divergent_continue(cont), has_discard(discard)
72 {}
73 };
74
75 struct block_info {
76 std::vector<std::pair<Operand, uint8_t>>
77 exec; /* Vector of exec masks. Either a temporary or const -1. */
78 std::vector<WQMState> instr_needs;
79 uint8_t block_needs;
80 };
81
82 struct exec_ctx {
83 Program* program;
84 std::vector<block_info> info;
85 std::vector<loop_info> loop;
86 bool handle_wqm = false;
exec_ctxaco::__anonc52d909e0111::exec_ctx87 exec_ctx(Program* program_) : program(program_), info(program->blocks.size()) {}
88 };
89
90 bool
needs_exact(aco_ptr<Instruction> & instr)91 needs_exact(aco_ptr<Instruction>& instr)
92 {
93 if (instr->isMUBUF()) {
94 return instr->mubuf().disable_wqm;
95 } else if (instr->isMTBUF()) {
96 return instr->mtbuf().disable_wqm;
97 } else if (instr->isMIMG()) {
98 return instr->mimg().disable_wqm;
99 } else if (instr->isFlatLike()) {
100 return instr->flatlike().disable_wqm;
101 } else {
102 return instr->isEXP();
103 }
104 }
105
106 void
mark_block_wqm(wqm_ctx & ctx,unsigned block_idx)107 mark_block_wqm(wqm_ctx& ctx, unsigned block_idx)
108 {
109 if (ctx.branch_wqm[block_idx])
110 return;
111
112 for (Block& block : ctx.program->blocks) {
113 if (block.index >= block_idx && block.kind & block_kind_top_level)
114 break;
115 ctx.branch_wqm[block.index] = true;
116 ctx.worklist.insert(block.index);
117 }
118 }
119
120 void
get_block_needs(wqm_ctx & ctx,exec_ctx & exec_ctx,Block * block)121 get_block_needs(wqm_ctx& ctx, exec_ctx& exec_ctx, Block* block)
122 {
123 block_info& info = exec_ctx.info[block->index];
124
125 std::vector<WQMState> instr_needs(block->instructions.size());
126
127 bool propagate_wqm = ctx.branch_wqm[block->index];
128 for (int i = block->instructions.size() - 1; i >= 0; --i) {
129 aco_ptr<Instruction>& instr = block->instructions[i];
130
131 if (instr->opcode == aco_opcode::p_wqm)
132 propagate_wqm = true;
133
134 bool pred_by_exec = needs_exec_mask(instr.get()) ||
135 instr->opcode == aco_opcode::p_logical_end ||
136 instr->isBranch();
137
138 if (needs_exact(instr))
139 instr_needs[i] = Exact;
140 else if (propagate_wqm && pred_by_exec)
141 instr_needs[i] = WQM;
142 else
143 instr_needs[i] = Unspecified;
144
145 info.block_needs |= instr_needs[i];
146 }
147
148 info.instr_needs = instr_needs;
149
150 /* for "if (<cond>) <wqm code>" or "while (<cond>) <wqm code>",
151 * <cond> should be computed in WQM */
152 if (info.block_needs & WQM) {
153 mark_block_wqm(ctx, block->index);
154 }
155 }
156
157 void
calculate_wqm_needs(exec_ctx & exec_ctx)158 calculate_wqm_needs(exec_ctx& exec_ctx)
159 {
160 wqm_ctx ctx(exec_ctx.program);
161
162 while (!ctx.worklist.empty()) {
163 unsigned block_index = *std::prev(ctx.worklist.end());
164 ctx.worklist.erase(std::prev(ctx.worklist.end()));
165
166 Block& block = exec_ctx.program->blocks[block_index];
167 get_block_needs(ctx, exec_ctx, &block);
168 }
169
170 exec_ctx.handle_wqm = true;
171 }
172
173 Operand
get_exec_op(Operand t)174 get_exec_op(Operand t)
175 {
176 if (t.isUndefined())
177 return Operand(exec, t.regClass());
178 else
179 return t;
180 }
181
182 void
transition_to_WQM(exec_ctx & ctx,Builder bld,unsigned idx)183 transition_to_WQM(exec_ctx& ctx, Builder bld, unsigned idx)
184 {
185 if (ctx.info[idx].exec.back().second & mask_type_wqm)
186 return;
187 if (ctx.info[idx].exec.back().second & mask_type_global) {
188 Operand exec_mask = ctx.info[idx].exec.back().first;
189 if (exec_mask.isUndefined()) {
190 exec_mask = bld.copy(bld.def(bld.lm), Operand(exec, bld.lm));
191 ctx.info[idx].exec.back().first = exec_mask;
192 }
193
194 exec_mask = bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
195 get_exec_op(exec_mask));
196 ctx.info[idx].exec.emplace_back(exec_mask, mask_type_global | mask_type_wqm);
197 return;
198 }
199 /* otherwise, the WQM mask should be one below the current mask */
200 ctx.info[idx].exec.pop_back();
201 assert(ctx.info[idx].exec.back().second & mask_type_wqm);
202 assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
203 assert(ctx.info[idx].exec.back().first.isTemp());
204 ctx.info[idx].exec.back().first =
205 bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
206 }
207
208 void
transition_to_Exact(exec_ctx & ctx,Builder bld,unsigned idx)209 transition_to_Exact(exec_ctx& ctx, Builder bld, unsigned idx)
210 {
211 if (ctx.info[idx].exec.back().second & mask_type_exact)
212 return;
213 /* We can't remove the loop exec mask, because that can cause exec.size() to
214 * be less than num_exec_masks. The loop exec mask also needs to be kept
215 * around for various uses. */
216 if ((ctx.info[idx].exec.back().second & mask_type_global) &&
217 !(ctx.info[idx].exec.back().second & mask_type_loop)) {
218 ctx.info[idx].exec.pop_back();
219 assert(ctx.info[idx].exec.back().second & mask_type_exact);
220 assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
221 assert(ctx.info[idx].exec.back().first.isTemp());
222 ctx.info[idx].exec.back().first =
223 bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
224 return;
225 }
226 /* otherwise, we create an exact mask and push to the stack */
227 Operand wqm = ctx.info[idx].exec.back().first;
228 if (wqm.isUndefined()) {
229 wqm = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
230 Definition(exec, bld.lm), ctx.info[idx].exec[0].first, Operand(exec, bld.lm));
231 } else {
232 bld.sop2(Builder::s_and, Definition(exec, bld.lm), bld.def(s1, scc),
233 ctx.info[idx].exec[0].first, wqm);
234 }
235 ctx.info[idx].exec.back().first = Operand(wqm);
236 ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type_exact);
237 }
238
239 unsigned
add_coupling_code(exec_ctx & ctx,Block * block,std::vector<aco_ptr<Instruction>> & instructions)240 add_coupling_code(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions)
241 {
242 unsigned idx = block->index;
243 Builder bld(ctx.program, &instructions);
244 std::vector<unsigned>& preds = block->linear_preds;
245
246 /* start block */
247 if (idx == 0) {
248 aco_ptr<Instruction>& startpgm = block->instructions[0];
249 assert(startpgm->opcode == aco_opcode::p_startpgm);
250 bld.insert(std::move(startpgm));
251
252 Operand start_exec(bld.lm);
253
254 /* exec seems to need to be manually initialized with combined shaders */
255 if (ctx.program->stage.num_sw_stages() > 1 || ctx.program->stage.hw == HWStage::NGG) {
256 start_exec = Operand::c32_or_c64(-1u, bld.lm == s2);
257 bld.copy(Definition(exec, bld.lm), start_exec);
258 }
259
260 if (ctx.handle_wqm) {
261 ctx.info[0].exec.emplace_back(start_exec, mask_type_global | mask_type_exact);
262 /* if this block needs WQM, initialize already */
263 if (ctx.info[0].block_needs & WQM)
264 transition_to_WQM(ctx, bld, 0);
265 } else {
266 uint8_t mask = mask_type_global;
267 if (ctx.program->needs_wqm) {
268 bld.sop1(Builder::s_wqm, Definition(exec, bld.lm), bld.def(s1, scc),
269 Operand(exec, bld.lm));
270 mask |= mask_type_wqm;
271 } else {
272 mask |= mask_type_exact;
273 }
274 ctx.info[0].exec.emplace_back(start_exec, mask);
275 }
276
277 return 1;
278 }
279
280 /* loop entry block */
281 if (block->kind & block_kind_loop_header) {
282 assert(preds[0] == idx - 1);
283 ctx.info[idx].exec = ctx.info[idx - 1].exec;
284 loop_info& info = ctx.loop.back();
285 while (ctx.info[idx].exec.size() > info.num_exec_masks)
286 ctx.info[idx].exec.pop_back();
287
288 /* create ssa names for outer exec masks */
289 if (info.has_discard) {
290 aco_ptr<Pseudo_instruction> phi;
291 for (int i = 0; i < info.num_exec_masks - 1; i++) {
292 phi.reset(create_instruction<Pseudo_instruction>(aco_opcode::p_linear_phi,
293 Format::PSEUDO, preds.size(), 1));
294 phi->definitions[0] = bld.def(bld.lm);
295 phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[i].first);
296 ctx.info[idx].exec[i].first = bld.insert(std::move(phi));
297 }
298 }
299
300 /* create ssa name for restore mask */
301 if (info.has_divergent_break) {
302 /* this phi might be trivial but ensures a parallelcopy on the loop header */
303 aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
304 aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
305 phi->definitions[0] = bld.def(bld.lm);
306 phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec[info.num_exec_masks - 1].first);
307 ctx.info[idx].exec.back().first = bld.insert(std::move(phi));
308 }
309
310 /* create ssa name for loop active mask */
311 aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
312 aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
313 if (info.has_divergent_continue)
314 phi->definitions[0] = bld.def(bld.lm);
315 else
316 phi->definitions[0] = Definition(exec, bld.lm);
317 phi->operands[0] = get_exec_op(ctx.info[preds[0]].exec.back().first);
318 Temp loop_active = bld.insert(std::move(phi));
319
320 if (info.has_divergent_break) {
321 uint8_t mask_type =
322 (ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact)) | mask_type_loop;
323 ctx.info[idx].exec.emplace_back(loop_active, mask_type);
324 } else {
325 ctx.info[idx].exec.back().first = Operand(loop_active);
326 ctx.info[idx].exec.back().second |= mask_type_loop;
327 }
328
329 /* create a parallelcopy to move the active mask to exec */
330 unsigned i = 0;
331 if (info.has_divergent_continue) {
332 while (block->instructions[i]->opcode != aco_opcode::p_logical_start) {
333 bld.insert(std::move(block->instructions[i]));
334 i++;
335 }
336 uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
337 assert(ctx.info[idx].exec.back().first.size() == bld.lm.size());
338 ctx.info[idx].exec.emplace_back(
339 bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first), mask_type);
340 }
341
342 return i;
343 }
344
345 /* loop exit block */
346 if (block->kind & block_kind_loop_exit) {
347 Block* header = ctx.loop.back().loop_header;
348 loop_info& info = ctx.loop.back();
349
350 for (ASSERTED unsigned pred : preds)
351 assert(ctx.info[pred].exec.size() >= info.num_exec_masks);
352
353 /* fill the loop header phis */
354 std::vector<unsigned>& header_preds = header->linear_preds;
355 int instr_idx = 0;
356 if (info.has_discard) {
357 while (instr_idx < info.num_exec_masks - 1) {
358 aco_ptr<Instruction>& phi = header->instructions[instr_idx];
359 assert(phi->opcode == aco_opcode::p_linear_phi);
360 for (unsigned i = 1; i < phi->operands.size(); i++)
361 phi->operands[i] = get_exec_op(ctx.info[header_preds[i]].exec[instr_idx].first);
362 instr_idx++;
363 }
364 }
365
366 {
367 aco_ptr<Instruction>& phi = header->instructions[instr_idx++];
368 assert(phi->opcode == aco_opcode::p_linear_phi);
369 for (unsigned i = 1; i < phi->operands.size(); i++)
370 phi->operands[i] =
371 get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks - 1].first);
372 }
373
374 if (info.has_divergent_break) {
375 aco_ptr<Instruction>& phi = header->instructions[instr_idx];
376 assert(phi->opcode == aco_opcode::p_linear_phi);
377 for (unsigned i = 1; i < phi->operands.size(); i++)
378 phi->operands[i] =
379 get_exec_op(ctx.info[header_preds[i]].exec[info.num_exec_masks].first);
380 }
381
382 assert(!(block->kind & block_kind_top_level) || info.num_exec_masks <= 2);
383
384 /* create the loop exit phis if not trivial */
385 for (unsigned exec_idx = 0; exec_idx < info.num_exec_masks; exec_idx++) {
386 Operand same = ctx.info[preds[0]].exec[exec_idx].first;
387 uint8_t type = ctx.info[header_preds[0]].exec[exec_idx].second;
388 bool trivial = true;
389
390 for (unsigned i = 1; i < preds.size() && trivial; i++) {
391 if (ctx.info[preds[i]].exec[exec_idx].first != same)
392 trivial = false;
393 }
394
395 if (trivial) {
396 ctx.info[idx].exec.emplace_back(same, type);
397 } else {
398 /* create phi for loop footer */
399 aco_ptr<Pseudo_instruction> phi{create_instruction<Pseudo_instruction>(
400 aco_opcode::p_linear_phi, Format::PSEUDO, preds.size(), 1)};
401 phi->definitions[0] = bld.def(bld.lm);
402 if (exec_idx == info.num_exec_masks - 1u) {
403 phi->definitions[0] = Definition(exec, bld.lm);
404 }
405 for (unsigned i = 0; i < phi->operands.size(); i++)
406 phi->operands[i] = get_exec_op(ctx.info[preds[i]].exec[exec_idx].first);
407 ctx.info[idx].exec.emplace_back(bld.insert(std::move(phi)), type);
408 }
409 }
410
411 assert(ctx.info[idx].exec.size() == info.num_exec_masks);
412 ctx.loop.pop_back();
413
414 } else if (preds.size() == 1) {
415 ctx.info[idx].exec = ctx.info[preds[0]].exec;
416 } else {
417 assert(preds.size() == 2);
418 /* if one of the predecessors ends in exact mask, we pop it from stack */
419 unsigned num_exec_masks =
420 std::min(ctx.info[preds[0]].exec.size(), ctx.info[preds[1]].exec.size());
421
422 if (block->kind & block_kind_merge)
423 num_exec_masks--;
424 if (block->kind & block_kind_top_level)
425 num_exec_masks = std::min(num_exec_masks, 2u);
426
427 /* create phis for diverged exec masks */
428 for (unsigned i = 0; i < num_exec_masks; i++) {
429 /* skip trivial phis */
430 if (ctx.info[preds[0]].exec[i].first == ctx.info[preds[1]].exec[i].first) {
431 Operand t = ctx.info[preds[0]].exec[i].first;
432 /* discard/demote can change the state of the current exec mask */
433 assert(!t.isTemp() ||
434 ctx.info[preds[0]].exec[i].second == ctx.info[preds[1]].exec[i].second);
435 uint8_t mask = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
436 ctx.info[idx].exec.emplace_back(t, mask);
437 continue;
438 }
439
440 Temp phi = bld.pseudo(aco_opcode::p_linear_phi, bld.def(bld.lm),
441 get_exec_op(ctx.info[preds[0]].exec[i].first),
442 get_exec_op(ctx.info[preds[1]].exec[i].first));
443 uint8_t mask_type = ctx.info[preds[0]].exec[i].second & ctx.info[preds[1]].exec[i].second;
444 ctx.info[idx].exec.emplace_back(phi, mask_type);
445 }
446 }
447
448 unsigned i = 0;
449 while (block->instructions[i]->opcode == aco_opcode::p_phi ||
450 block->instructions[i]->opcode == aco_opcode::p_linear_phi) {
451 bld.insert(std::move(block->instructions[i]));
452 i++;
453 }
454
455 /* try to satisfy the block's needs */
456 if (ctx.handle_wqm) {
457 if (block->kind & block_kind_top_level && ctx.info[idx].exec.size() == 2) {
458 if (ctx.info[idx].block_needs == 0 || ctx.info[idx].block_needs == Exact) {
459 ctx.info[idx].exec.back().second |= mask_type_global;
460 transition_to_Exact(ctx, bld, idx);
461 ctx.handle_wqm = false;
462 }
463 }
464 }
465
466 /* restore exec mask after divergent control flow */
467 if (block->kind & (block_kind_loop_exit | block_kind_merge) &&
468 !ctx.info[idx].exec.back().first.isUndefined()) {
469 Operand restore = ctx.info[idx].exec.back().first;
470 assert(restore.size() == bld.lm.size());
471 bld.copy(Definition(exec, bld.lm), restore);
472 if (!restore.isConstant())
473 ctx.info[idx].exec.back().first = Operand(bld.lm);
474 }
475
476 return i;
477 }
478
479 void
process_instructions(exec_ctx & ctx,Block * block,std::vector<aco_ptr<Instruction>> & instructions,unsigned idx)480 process_instructions(exec_ctx& ctx, Block* block, std::vector<aco_ptr<Instruction>>& instructions,
481 unsigned idx)
482 {
483 WQMState state;
484 if (ctx.info[block->index].exec.back().second & mask_type_wqm) {
485 state = WQM;
486 } else {
487 assert(!ctx.handle_wqm || ctx.info[block->index].exec.back().second & mask_type_exact);
488 state = Exact;
489 }
490
491 /* if the block doesn't need both, WQM and Exact, we can skip processing the instructions */
492 bool process = (ctx.handle_wqm && (ctx.info[block->index].block_needs & state) !=
493 (ctx.info[block->index].block_needs & (WQM | Exact))) ||
494 block->kind & block_kind_uses_discard || block->kind & block_kind_needs_lowering;
495 if (!process) {
496 std::vector<aco_ptr<Instruction>>::iterator it = std::next(block->instructions.begin(), idx);
497 instructions.insert(instructions.end(),
498 std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(it),
499 std::move_iterator<std::vector<aco_ptr<Instruction>>::iterator>(
500 block->instructions.end()));
501 return;
502 }
503
504 Builder bld(ctx.program, &instructions);
505
506 for (; idx < block->instructions.size(); idx++) {
507 aco_ptr<Instruction> instr = std::move(block->instructions[idx]);
508
509 WQMState needs = ctx.handle_wqm ? ctx.info[block->index].instr_needs[idx] : Unspecified;
510
511 if (needs == WQM && state != WQM) {
512 transition_to_WQM(ctx, bld, block->index);
513 state = WQM;
514 } else if (needs == Exact && state != Exact) {
515 transition_to_Exact(ctx, bld, block->index);
516 state = Exact;
517 }
518
519 if (instr->opcode == aco_opcode::p_discard_if) {
520 Operand current_exec = Operand(exec, bld.lm);
521
522 if (ctx.info[block->index].exec.size() >= 2) {
523 if (needs == WQM) {
524 /* Preserve the WQM mask */
525 ctx.info[block->index].exec[1].second &= ~mask_type_global;
526 } else if (block->kind & block_kind_top_level) {
527 /* Transition to Exact without extra instruction. Since needs != WQM, we won't need
528 * WQM again.
529 */
530 ctx.info[block->index].exec.resize(1);
531 assert(ctx.info[block->index].exec[0].second == (mask_type_exact | mask_type_global));
532 current_exec = get_exec_op(ctx.info[block->index].exec.back().first);
533 ctx.info[block->index].exec[0].first = Operand(bld.lm);
534 }
535 }
536
537 Temp cond, exit_cond;
538 if (instr->operands[0].isConstant()) {
539 assert(instr->operands[0].constantValue() == -1u);
540 /* save condition and set exec to zero */
541 exit_cond = bld.tmp(s1);
542 cond =
543 bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
544 Definition(exec, bld.lm), Operand::zero(), Operand(exec, bld.lm));
545 } else {
546 cond = instr->operands[0].getTemp();
547 /* discard from current exec */
548 exit_cond = bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc),
549 current_exec, cond)
550 .def(1)
551 .getTemp();
552 }
553
554 /* discard from inner to outer exec mask on stack */
555 int num = ctx.info[block->index].exec.size() - 2;
556 for (int i = num; i >= 0; i--) {
557 Instruction* andn2 = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
558 ctx.info[block->index].exec[i].first, cond);
559 ctx.info[block->index].exec[i].first = Operand(andn2->definitions[0].getTemp());
560 exit_cond = andn2->definitions[1].getTemp();
561 }
562
563 instr->opcode = aco_opcode::p_exit_early_if;
564 instr->operands[0] = bld.scc(exit_cond);
565 assert(!ctx.handle_wqm || (ctx.info[block->index].exec[0].second & mask_type_wqm) == 0);
566
567 } else if (instr->opcode == aco_opcode::p_is_helper) {
568 Definition dst = instr->definitions[0];
569 assert(dst.size() == bld.lm.size());
570 if (state == Exact) {
571 instr.reset(create_instruction<SOP1_instruction>(bld.w64or32(Builder::s_mov),
572 Format::SOP1, 1, 1));
573 instr->operands[0] = Operand::zero();
574 instr->definitions[0] = dst;
575 } else {
576 std::pair<Operand, uint8_t>& exact_mask = ctx.info[block->index].exec[0];
577 assert(exact_mask.second & mask_type_exact);
578
579 instr.reset(create_instruction<SOP2_instruction>(bld.w64or32(Builder::s_andn2),
580 Format::SOP2, 2, 2));
581 instr->operands[0] = Operand(exec, bld.lm); /* current exec */
582 instr->operands[1] = Operand(exact_mask.first);
583 instr->definitions[0] = dst;
584 instr->definitions[1] = bld.def(s1, scc);
585 }
586 } else if (instr->opcode == aco_opcode::p_demote_to_helper) {
587 /* turn demote into discard_if with only exact masks */
588 assert(ctx.info[block->index].exec[0].second == (mask_type_exact | mask_type_global));
589
590 int num;
591 Temp cond, exit_cond;
592 if (instr->operands[0].isConstant()) {
593 assert(instr->operands[0].constantValue() == -1u);
594 /* transition to exact and set exec to zero */
595 exit_cond = bld.tmp(s1);
596 cond =
597 bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.scc(Definition(exit_cond)),
598 Definition(exec, bld.lm), Operand::zero(), Operand(exec, bld.lm));
599
600 num = ctx.info[block->index].exec.size() - 2;
601 if (!(ctx.info[block->index].exec.back().second & mask_type_exact)) {
602 ctx.info[block->index].exec.back().first = Operand(cond);
603 ctx.info[block->index].exec.emplace_back(Operand(bld.lm), mask_type_exact);
604 }
605 } else {
606 /* demote_if: transition to exact */
607 if (block->kind & block_kind_top_level && ctx.info[block->index].exec.size() == 2 &&
608 ctx.info[block->index].exec.back().second & mask_type_global) {
609 /* We don't need to actually copy anything into exact, since the s_andn2
610 * instructions later will do that.
611 */
612 ctx.info[block->index].exec.pop_back();
613 } else {
614 transition_to_Exact(ctx, bld, block->index);
615 }
616 assert(instr->operands[0].isTemp());
617 cond = instr->operands[0].getTemp();
618 num = ctx.info[block->index].exec.size() - 1;
619 }
620
621 for (int i = num; i >= 0; i--) {
622 if (ctx.info[block->index].exec[i].second & mask_type_exact) {
623 Instruction* andn2 =
624 bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.def(s1, scc),
625 get_exec_op(ctx.info[block->index].exec[i].first), cond);
626 if (i == (int)ctx.info[block->index].exec.size() - 1)
627 andn2->definitions[0] = Definition(exec, bld.lm);
628
629 ctx.info[block->index].exec[i].first = Operand(andn2->definitions[0].getTemp());
630 exit_cond = andn2->definitions[1].getTemp();
631 } else {
632 assert(i != 0);
633 }
634 }
635 instr->opcode = aco_opcode::p_exit_early_if;
636 instr->operands[0] = bld.scc(exit_cond);
637 state = Exact;
638
639 } else if (instr->opcode == aco_opcode::p_elect) {
640 bool all_lanes_enabled = ctx.info[block->index].exec.back().first.constantEquals(-1u);
641 Definition dst = instr->definitions[0];
642
643 if (all_lanes_enabled) {
644 bld.copy(Definition(dst), Operand::c32_or_c64(1u, dst.size() == 2));
645 } else {
646 Temp first_lane_idx = bld.sop1(Builder::s_ff1_i32, bld.def(s1), Operand(exec, bld.lm));
647 bld.sop2(Builder::s_lshl, Definition(dst), bld.def(s1, scc),
648 Operand::c32_or_c64(1u, dst.size() == 2), Operand(first_lane_idx));
649 }
650 instr.reset();
651 continue;
652 }
653
654 bld.insert(std::move(instr));
655 }
656 }
657
658 void
add_branch_code(exec_ctx & ctx,Block * block)659 add_branch_code(exec_ctx& ctx, Block* block)
660 {
661 unsigned idx = block->index;
662 Builder bld(ctx.program, block);
663
664 if (idx == ctx.program->blocks.size() - 1)
665 return;
666
667 /* try to disable wqm handling */
668 if (ctx.handle_wqm && block->kind & block_kind_top_level) {
669 if (ctx.info[idx].exec.size() == 3) {
670 assert(ctx.info[idx].exec[1].second == mask_type_wqm);
671 ctx.info[idx].exec.pop_back();
672 }
673 assert(ctx.info[idx].exec.size() <= 2);
674
675 if (!(ctx.info[idx].instr_needs.back() & WQM)) {
676 /* transition to Exact if the branch doesn't need WQM */
677 aco_ptr<Instruction> branch = std::move(block->instructions.back());
678 block->instructions.pop_back();
679 ctx.info[idx].exec.back().second |= mask_type_global;
680 transition_to_Exact(ctx, bld, idx);
681 bld.insert(std::move(branch));
682 ctx.handle_wqm = false;
683 }
684 }
685
686 if (block->kind & block_kind_loop_preheader) {
687 /* collect information about the succeeding loop */
688 bool has_divergent_break = false;
689 bool has_divergent_continue = false;
690 bool has_discard = false;
691 unsigned loop_nest_depth = ctx.program->blocks[idx + 1].loop_nest_depth;
692
693 for (unsigned i = idx + 1; ctx.program->blocks[i].loop_nest_depth >= loop_nest_depth; i++) {
694 Block& loop_block = ctx.program->blocks[i];
695
696 if (loop_block.kind & block_kind_uses_discard)
697 has_discard = true;
698 if (loop_block.loop_nest_depth != loop_nest_depth)
699 continue;
700
701 if (loop_block.kind & block_kind_uniform)
702 continue;
703 else if (loop_block.kind & block_kind_break)
704 has_divergent_break = true;
705 else if (loop_block.kind & block_kind_continue)
706 has_divergent_continue = true;
707 }
708
709 unsigned num_exec_masks = ctx.info[idx].exec.size();
710 if (block->kind & block_kind_top_level)
711 num_exec_masks = std::min(num_exec_masks, 2u);
712
713 ctx.loop.emplace_back(&ctx.program->blocks[block->linear_succs[0]], num_exec_masks,
714 has_divergent_break, has_divergent_continue, has_discard);
715 }
716
717 /* For normal breaks, this is the exec mask. For discard+break, it's the
718 * old exec mask before it was zero'd.
719 */
720 Operand break_cond = Operand(exec, bld.lm);
721
722 if (block->kind & block_kind_continue_or_break) {
723 assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[1]].linear_succs[0]].kind &
724 block_kind_loop_header);
725 assert(ctx.program->blocks[ctx.program->blocks[block->linear_succs[0]].linear_succs[0]].kind &
726 block_kind_loop_exit);
727 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
728 block->instructions.pop_back();
729
730 bool need_parallelcopy = false;
731 while (!(ctx.info[idx].exec.back().second & mask_type_loop)) {
732 ctx.info[idx].exec.pop_back();
733 need_parallelcopy = true;
734 }
735
736 if (need_parallelcopy)
737 ctx.info[idx].exec.back().first =
738 bld.copy(Definition(exec, bld.lm), ctx.info[idx].exec.back().first);
739 bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), Operand(exec, bld.lm),
740 block->linear_succs[1], block->linear_succs[0]);
741 return;
742 }
743
744 if (block->kind & block_kind_uniform) {
745 Pseudo_branch_instruction& branch = block->instructions.back()->branch();
746 if (branch.opcode == aco_opcode::p_branch) {
747 branch.target[0] = block->linear_succs[0];
748 } else {
749 branch.target[0] = block->linear_succs[1];
750 branch.target[1] = block->linear_succs[0];
751 }
752 return;
753 }
754
755 if (block->kind & block_kind_branch) {
756 // orig = s_and_saveexec_b64
757 assert(block->linear_succs.size() == 2);
758 assert(block->instructions.back()->opcode == aco_opcode::p_cbranch_z);
759 Temp cond = block->instructions.back()->operands[0].getTemp();
760 block->instructions.pop_back();
761
762 uint8_t mask_type = ctx.info[idx].exec.back().second & (mask_type_wqm | mask_type_exact);
763 if (ctx.info[idx].exec.back().first.constantEquals(-1u)) {
764 bld.copy(Definition(exec, bld.lm), cond);
765 } else {
766 Temp old_exec = bld.sop1(Builder::s_and_saveexec, bld.def(bld.lm), bld.def(s1, scc),
767 Definition(exec, bld.lm), cond, Operand(exec, bld.lm));
768
769 ctx.info[idx].exec.back().first = Operand(old_exec);
770 }
771
772 /* add next current exec to the stack */
773 ctx.info[idx].exec.emplace_back(Operand(bld.lm), mask_type);
774
775 bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
776 block->linear_succs[1], block->linear_succs[0]);
777 return;
778 }
779
780 if (block->kind & block_kind_invert) {
781 // exec = s_andn2_b64 (original_exec, exec)
782 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
783 block->instructions.pop_back();
784 assert(ctx.info[idx].exec.size() >= 2);
785 Operand orig_exec = ctx.info[idx].exec[ctx.info[idx].exec.size() - 2].first;
786 bld.sop2(Builder::s_andn2, Definition(exec, bld.lm), bld.def(s1, scc), orig_exec,
787 Operand(exec, bld.lm));
788
789 bld.branch(aco_opcode::p_cbranch_z, bld.def(s2), Operand(exec, bld.lm),
790 block->linear_succs[1], block->linear_succs[0]);
791 return;
792 }
793
794 if (block->kind & block_kind_break) {
795 // loop_mask = s_andn2_b64 (loop_mask, exec)
796 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
797 block->instructions.pop_back();
798
799 Temp cond = Temp();
800 for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
801 cond = bld.tmp(s1);
802 Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
803 exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
804 exec_mask, break_cond);
805 ctx.info[idx].exec[exec_idx].first = exec_mask;
806 if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
807 break;
808 }
809
810 /* check if the successor is the merge block, otherwise set exec to 0 */
811 // TODO: this could be done better by directly branching to the merge block
812 unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
813 Block& succ = ctx.program->blocks[succ_idx];
814 if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
815 bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
816 }
817
818 bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
819 block->linear_succs[0]);
820 return;
821 }
822
823 if (block->kind & block_kind_continue) {
824 assert(block->instructions.back()->opcode == aco_opcode::p_branch);
825 block->instructions.pop_back();
826
827 Temp cond = Temp();
828 for (int exec_idx = ctx.info[idx].exec.size() - 2; exec_idx >= 0; exec_idx--) {
829 if (ctx.info[idx].exec[exec_idx].second & mask_type_loop)
830 break;
831 cond = bld.tmp(s1);
832 Operand exec_mask = ctx.info[idx].exec[exec_idx].first;
833 exec_mask = bld.sop2(Builder::s_andn2, bld.def(bld.lm), bld.scc(Definition(cond)),
834 exec_mask, Operand(exec, bld.lm));
835 ctx.info[idx].exec[exec_idx].first = exec_mask;
836 }
837 assert(cond != Temp());
838
839 /* check if the successor is the merge block, otherwise set exec to 0 */
840 // TODO: this could be done better by directly branching to the merge block
841 unsigned succ_idx = ctx.program->blocks[block->linear_succs[1]].linear_succs[0];
842 Block& succ = ctx.program->blocks[succ_idx];
843 if (!(succ.kind & block_kind_invert || succ.kind & block_kind_merge)) {
844 bld.copy(Definition(exec, bld.lm), Operand::zero(bld.lm.bytes()));
845 }
846
847 bld.branch(aco_opcode::p_cbranch_nz, bld.def(s2), bld.scc(cond), block->linear_succs[1],
848 block->linear_succs[0]);
849 return;
850 }
851 }
852
853 void
process_block(exec_ctx & ctx,Block * block)854 process_block(exec_ctx& ctx, Block* block)
855 {
856 std::vector<aco_ptr<Instruction>> instructions;
857 instructions.reserve(block->instructions.size());
858
859 unsigned idx = add_coupling_code(ctx, block, instructions);
860
861 assert(block->index != ctx.program->blocks.size() - 1 ||
862 ctx.info[block->index].exec.size() <= 2);
863
864 process_instructions(ctx, block, instructions, idx);
865
866 block->instructions = std::move(instructions);
867
868 add_branch_code(ctx, block);
869 }
870
871 } /* end namespace */
872
873 void
insert_exec_mask(Program * program)874 insert_exec_mask(Program* program)
875 {
876 exec_ctx ctx(program);
877
878 if (program->needs_wqm && program->needs_exact)
879 calculate_wqm_needs(ctx);
880
881 for (Block& block : program->blocks)
882 process_block(ctx, &block);
883 }
884
885 } // namespace aco
886