1 /*
2 * Copyright © 2018 Valve Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 */
24
25 #include "aco_ir.h"
26
27 #include "util/memstream.h"
28
29 #include <array>
30 #include <map>
31 #include <set>
32 #include <vector>
33
34 namespace aco {
35
36 static void
aco_log(Program * program,enum radv_compiler_debug_level level,const char * prefix,const char * file,unsigned line,const char * fmt,va_list args)37 aco_log(Program* program, enum radv_compiler_debug_level level, const char* prefix,
38 const char* file, unsigned line, const char* fmt, va_list args)
39 {
40 char* msg;
41
42 if (program->debug.shorten_messages) {
43 msg = ralloc_vasprintf(NULL, fmt, args);
44 } else {
45 msg = ralloc_strdup(NULL, prefix);
46 ralloc_asprintf_append(&msg, " In file %s:%u\n", file, line);
47 ralloc_asprintf_append(&msg, " ");
48 ralloc_vasprintf_append(&msg, fmt, args);
49 }
50
51 if (program->debug.func)
52 program->debug.func(program->debug.private_data, level, msg);
53
54 fprintf(program->debug.output, "%s\n", msg);
55
56 ralloc_free(msg);
57 }
58
59 void
_aco_perfwarn(Program * program,const char * file,unsigned line,const char * fmt,...)60 _aco_perfwarn(Program* program, const char* file, unsigned line, const char* fmt, ...)
61 {
62 va_list args;
63
64 va_start(args, fmt);
65 aco_log(program, RADV_COMPILER_DEBUG_LEVEL_PERFWARN, "ACO PERFWARN:\n", file, line, fmt, args);
66 va_end(args);
67 }
68
69 void
_aco_err(Program * program,const char * file,unsigned line,const char * fmt,...)70 _aco_err(Program* program, const char* file, unsigned line, const char* fmt, ...)
71 {
72 va_list args;
73
74 va_start(args, fmt);
75 aco_log(program, RADV_COMPILER_DEBUG_LEVEL_ERROR, "ACO ERROR:\n", file, line, fmt, args);
76 va_end(args);
77 }
78
79 bool
validate_ir(Program * program)80 validate_ir(Program* program)
81 {
82 bool is_valid = true;
83 auto check = [&program, &is_valid](bool success, const char* msg,
84 aco::Instruction* instr) -> void
85 {
86 if (!success) {
87 char* out;
88 size_t outsize;
89 struct u_memstream mem;
90 u_memstream_open(&mem, &out, &outsize);
91 FILE* const memf = u_memstream_get(&mem);
92
93 fprintf(memf, "%s: ", msg);
94 aco_print_instr(instr, memf);
95 u_memstream_close(&mem);
96
97 aco_err(program, "%s", out);
98 free(out);
99
100 is_valid = false;
101 }
102 };
103
104 auto check_block = [&program, &is_valid](bool success, const char* msg,
105 aco::Block* block) -> void
106 {
107 if (!success) {
108 aco_err(program, "%s: BB%u", msg, block->index);
109 is_valid = false;
110 }
111 };
112
113 for (Block& block : program->blocks) {
114 for (aco_ptr<Instruction>& instr : block.instructions) {
115
116 /* check base format */
117 Format base_format = instr->format;
118 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
119 base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
120 if ((uint32_t)base_format & (uint32_t)Format::VOP1)
121 base_format = Format::VOP1;
122 else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
123 base_format = Format::VOP2;
124 else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
125 base_format = Format::VOPC;
126 else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) {
127 if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
128 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
129 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
130 instr->opcode == aco_opcode::v_interp_p2_f16) {
131 /* v_interp_*_fp16 are considered VINTRP by the compiler but
132 * they are emitted as VOP3.
133 */
134 base_format = Format::VOP3;
135 } else {
136 base_format = Format::VINTRP;
137 }
138 }
139 check(base_format == instr_info.format[(int)instr->opcode],
140 "Wrong base format for instruction", instr.get());
141
142 /* check VOP3 modifiers */
143 if (instr->isVOP3() && instr->format != Format::VOP3) {
144 check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
145 base_format == Format::VOPC || base_format == Format::VINTRP,
146 "Format cannot have VOP3/VOP3B applied", instr.get());
147 }
148
149 /* check SDWA */
150 if (instr->isSDWA()) {
151 check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
152 base_format == Format::VOPC,
153 "Format cannot have SDWA applied", instr.get());
154
155 check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get());
156
157 SDWA_instruction& sdwa = instr->sdwa();
158 check(sdwa.omod == 0 || program->chip_class >= GFX9,
159 "SDWA omod only supported on GFX9+", instr.get());
160 if (base_format == Format::VOPC) {
161 check(sdwa.clamp == false || program->chip_class == GFX8,
162 "SDWA VOPC clamp only supported on GFX8", instr.get());
163 check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
164 program->chip_class >= GFX9,
165 "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
166 } else {
167 const Definition& def = instr->definitions[0];
168 check(def.bytes() <= 4, "SDWA definitions must not be larger than 4 bytes",
169 instr.get());
170 check(def.bytes() >= sdwa.dst_sel.size() + sdwa.dst_sel.offset(),
171 "SDWA definition selection size must be at most definition size", instr.get());
172 check(
173 sdwa.dst_sel.size() == 1 || sdwa.dst_sel.size() == 2 || sdwa.dst_sel.size() == 4,
174 "SDWA definition selection size must be 1, 2 or 4 bytes", instr.get());
175 check(sdwa.dst_sel.offset() % sdwa.dst_sel.size() == 0, "Invalid selection offset",
176 instr.get());
177 check(def.bytes() == 4 || def.bytes() == sdwa.dst_sel.size(),
178 "SDWA dst_sel size must be definition size for subdword definitions",
179 instr.get());
180 check(def.bytes() == 4 || sdwa.dst_sel.offset() == 0,
181 "SDWA dst_sel offset must be 0 for subdword definitions", instr.get());
182 }
183
184 for (unsigned i = 0; i < std::min<unsigned>(2, instr->operands.size()); i++) {
185 const Operand& op = instr->operands[i];
186 check(op.bytes() <= 4, "SDWA operands must not be larger than 4 bytes", instr.get());
187 check(op.bytes() >= sdwa.sel[i].size() + sdwa.sel[i].offset(),
188 "SDWA operand selection size must be at most operand size", instr.get());
189 check(sdwa.sel[i].size() == 1 || sdwa.sel[i].size() == 2 || sdwa.sel[i].size() == 4,
190 "SDWA operand selection size must be 1, 2 or 4 bytes", instr.get());
191 check(sdwa.sel[i].offset() % sdwa.sel[i].size() == 0, "Invalid selection offset",
192 instr.get());
193 }
194 if (instr->operands.size() >= 3) {
195 check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
196 "3rd operand must be fixed to vcc with SDWA", instr.get());
197 }
198 if (instr->definitions.size() >= 2) {
199 check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
200 "2nd definition must be fixed to vcc with SDWA", instr.get());
201 }
202
203 const bool sdwa_opcodes =
204 instr->opcode != aco_opcode::v_fmac_f32 && instr->opcode != aco_opcode::v_fmac_f16 &&
205 instr->opcode != aco_opcode::v_fmamk_f32 &&
206 instr->opcode != aco_opcode::v_fmaak_f32 &&
207 instr->opcode != aco_opcode::v_fmamk_f16 &&
208 instr->opcode != aco_opcode::v_fmaak_f16 &&
209 instr->opcode != aco_opcode::v_madmk_f32 &&
210 instr->opcode != aco_opcode::v_madak_f32 &&
211 instr->opcode != aco_opcode::v_madmk_f16 &&
212 instr->opcode != aco_opcode::v_madak_f16 &&
213 instr->opcode != aco_opcode::v_readfirstlane_b32 &&
214 instr->opcode != aco_opcode::v_clrexcp && instr->opcode != aco_opcode::v_swap_b32;
215
216 const bool feature_mac =
217 program->chip_class == GFX8 &&
218 (instr->opcode == aco_opcode::v_mac_f32 && instr->opcode == aco_opcode::v_mac_f16);
219
220 check(sdwa_opcodes || feature_mac, "SDWA can't be used with this opcode", instr.get());
221 }
222
223 /* check opsel */
224 if (instr->isVOP3()) {
225 VOP3_instruction& vop3 = instr->vop3();
226 check(vop3.opsel == 0 || program->chip_class >= GFX9,
227 "Opsel is only supported on GFX9+", instr.get());
228
229 for (unsigned i = 0; i < 3; i++) {
230 if (i >= instr->operands.size() ||
231 (instr->operands[i].hasRegClass() &&
232 instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed()))
233 check((vop3.opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get());
234 }
235 if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed())
236 check((vop3.opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition",
237 instr.get());
238 } else if (instr->isVOP3P()) {
239 VOP3P_instruction& vop3p = instr->vop3p();
240 for (unsigned i = 0; i < instr->operands.size(); i++) {
241 if (instr->operands[i].hasRegClass() &&
242 instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed())
243 check((vop3p.opsel_lo & (1 << i)) == 0 && (vop3p.opsel_hi & (1 << i)) == 0,
244 "Unexpected opsel for subdword operand", instr.get());
245 }
246 check(instr->definitions[0].regClass() == v1, "VOP3P must have v1 definition",
247 instr.get());
248 }
249
250 /* check for undefs */
251 for (unsigned i = 0; i < instr->operands.size(); i++) {
252 if (instr->operands[i].isUndefined()) {
253 bool flat = instr->isFlatLike();
254 bool can_be_undef = is_phi(instr) || instr->isEXP() || instr->isReduction() ||
255 instr->opcode == aco_opcode::p_create_vector ||
256 (flat && i == 1) || (instr->isMIMG() && (i == 1 || i == 2)) ||
257 ((instr->isMUBUF() || instr->isMTBUF()) && i == 1);
258 check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
259 } else {
260 check(instr->operands[i].isFixed() || instr->operands[i].isTemp() ||
261 instr->operands[i].isConstant(),
262 "Uninitialized Operand", instr.get());
263 }
264 }
265
266 /* check subdword definitions */
267 for (unsigned i = 0; i < instr->definitions.size(); i++) {
268 if (instr->definitions[i].regClass().is_subdword())
269 check(instr->isPseudo() || instr->definitions[i].bytes() <= 4,
270 "Only Pseudo instructions can write subdword registers larger than 4 bytes",
271 instr.get());
272 }
273
274 if (instr->isSALU() || instr->isVALU()) {
275 /* check literals */
276 Operand literal(s1);
277 for (unsigned i = 0; i < instr->operands.size(); i++) {
278 Operand op = instr->operands[i];
279 if (!op.isLiteral())
280 continue;
281
282 check(!instr->isDPP() && !instr->isSDWA() &&
283 (!instr->isVOP3() || program->chip_class >= GFX10) &&
284 (!instr->isVOP3P() || program->chip_class >= GFX10),
285 "Literal applied on wrong instruction format", instr.get());
286
287 check(literal.isUndefined() || (literal.size() == op.size() &&
288 literal.constantValue() == op.constantValue()),
289 "Only 1 Literal allowed", instr.get());
290 literal = op;
291 check(instr->isSALU() || instr->isVOP3() || instr->isVOP3P() || i == 0 || i == 2,
292 "Wrong source position for Literal argument", instr.get());
293 }
294
295 /* check num sgprs for VALU */
296 if (instr->isVALU()) {
297 bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
298 instr->opcode == aco_opcode::v_lshrrev_b64 ||
299 instr->opcode == aco_opcode::v_ashrrev_i64;
300 unsigned const_bus_limit = 1;
301 if (program->chip_class >= GFX10 && !is_shift64)
302 const_bus_limit = 2;
303
304 uint32_t scalar_mask = instr->isVOP3() || instr->isVOP3P() ? 0x7 : 0x5;
305 if (instr->isSDWA())
306 scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4;
307 else if (instr->isDPP())
308 scalar_mask = 0x4;
309
310 if (instr->isVOPC() || instr->opcode == aco_opcode::v_readfirstlane_b32 ||
311 instr->opcode == aco_opcode::v_readlane_b32 ||
312 instr->opcode == aco_opcode::v_readlane_b32_e64) {
313 check(instr->definitions[0].getTemp().type() == RegType::sgpr,
314 "Wrong Definition type for VALU instruction", instr.get());
315 } else {
316 check(instr->definitions[0].getTemp().type() == RegType::vgpr,
317 "Wrong Definition type for VALU instruction", instr.get());
318 }
319
320 unsigned num_sgprs = 0;
321 unsigned sgpr[] = {0, 0};
322 for (unsigned i = 0; i < instr->operands.size(); i++) {
323 Operand op = instr->operands[i];
324 if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
325 instr->opcode == aco_opcode::v_readlane_b32 ||
326 instr->opcode == aco_opcode::v_readlane_b32_e64) {
327 check(i != 1 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
328 op.isConstant(),
329 "Must be a SGPR or a constant", instr.get());
330 check(i == 1 || (op.isTemp() && op.regClass().type() == RegType::vgpr &&
331 op.bytes() <= 4),
332 "Wrong Operand type for VALU instruction", instr.get());
333 continue;
334 }
335 if (instr->opcode == aco_opcode::v_permlane16_b32 ||
336 instr->opcode == aco_opcode::v_permlanex16_b32) {
337 check(i != 0 || (op.isTemp() && op.regClass().type() == RegType::vgpr),
338 "Operand 0 of v_permlane must be VGPR", instr.get());
339 check(i == 0 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
340 op.isConstant(),
341 "Lane select operands of v_permlane must be SGPR or constant",
342 instr.get());
343 }
344
345 if (instr->opcode == aco_opcode::v_writelane_b32 ||
346 instr->opcode == aco_opcode::v_writelane_b32_e64) {
347 check(i != 2 || (op.isTemp() && op.regClass().type() == RegType::vgpr &&
348 op.bytes() <= 4),
349 "Wrong Operand type for VALU instruction", instr.get());
350 check(i == 2 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
351 op.isConstant(),
352 "Must be a SGPR or a constant", instr.get());
353 continue;
354 }
355 if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
356 check(scalar_mask & (1 << i), "Wrong source position for SGPR argument",
357 instr.get());
358
359 if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
360 if (num_sgprs < 2)
361 sgpr[num_sgprs++] = op.tempId();
362 }
363 }
364
365 if (op.isConstant() && !op.isLiteral())
366 check(scalar_mask & (1 << i), "Wrong source position for constant argument",
367 instr.get());
368 }
369 check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit,
370 "Too many SGPRs/literals", instr.get());
371 }
372
373 if (instr->isSOP1() || instr->isSOP2()) {
374 check(instr->definitions[0].getTemp().type() == RegType::sgpr,
375 "Wrong Definition type for SALU instruction", instr.get());
376 for (const Operand& op : instr->operands) {
377 check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
378 "Wrong Operand type for SALU instruction", instr.get());
379 }
380 }
381 }
382
383 switch (instr->format) {
384 case Format::PSEUDO: {
385 if (instr->opcode == aco_opcode::p_create_vector) {
386 unsigned size = 0;
387 for (const Operand& op : instr->operands) {
388 check(op.bytes() < 4 || size % 4 == 0, "Operand is not aligned", instr.get());
389 size += op.bytes();
390 }
391 check(size == instr->definitions[0].bytes(),
392 "Definition size does not match operand sizes", instr.get());
393 if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
394 for (const Operand& op : instr->operands) {
395 check(op.isConstant() || op.regClass().type() == RegType::sgpr,
396 "Wrong Operand type for scalar vector", instr.get());
397 }
398 }
399 } else if (instr->opcode == aco_opcode::p_extract_vector) {
400 check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(),
401 "Wrong Operand types", instr.get());
402 check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <=
403 instr->operands[0].bytes(),
404 "Index out of range", instr.get());
405 check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
406 instr->operands[0].regClass().type() == RegType::sgpr,
407 "Cannot extract SGPR value from VGPR vector", instr.get());
408 check(program->chip_class >= GFX9 ||
409 !instr->definitions[0].regClass().is_subdword() ||
410 instr->operands[0].regClass().type() == RegType::vgpr,
411 "Cannot extract subdword from SGPR before GFX9+", instr.get());
412 } else if (instr->opcode == aco_opcode::p_split_vector) {
413 check(instr->operands[0].isTemp(), "Operand must be a temporary", instr.get());
414 unsigned size = 0;
415 for (const Definition& def : instr->definitions) {
416 size += def.bytes();
417 }
418 check(size == instr->operands[0].bytes(),
419 "Operand size does not match definition sizes", instr.get());
420 if (instr->operands[0].getTemp().type() == RegType::vgpr) {
421 for (const Definition& def : instr->definitions)
422 check(def.regClass().type() == RegType::vgpr,
423 "Wrong Definition type for VGPR split_vector", instr.get());
424 } else {
425 for (const Definition& def : instr->definitions)
426 check(program->chip_class >= GFX9 || !def.regClass().is_subdword(),
427 "Cannot split SGPR into subdword VGPRs before GFX9+", instr.get());
428 }
429 } else if (instr->opcode == aco_opcode::p_parallelcopy) {
430 check(instr->definitions.size() == instr->operands.size(),
431 "Number of Operands does not match number of Definitions", instr.get());
432 for (unsigned i = 0; i < instr->operands.size(); i++) {
433 check(instr->definitions[i].bytes() == instr->operands[i].bytes(),
434 "Operand and Definition size must match", instr.get());
435 if (instr->operands[i].isTemp()) {
436 check((instr->definitions[i].getTemp().type() ==
437 instr->operands[i].regClass().type()) ||
438 (instr->definitions[i].getTemp().type() == RegType::vgpr &&
439 instr->operands[i].regClass().type() == RegType::sgpr),
440 "Operand and Definition types do not match", instr.get());
441 check(instr->definitions[i].regClass().is_linear_vgpr() ==
442 instr->operands[i].regClass().is_linear_vgpr(),
443 "Operand and Definition types do not match", instr.get());
444 } else {
445 check(!instr->definitions[i].regClass().is_linear_vgpr(),
446 "Can only copy linear VGPRs into linear VGPRs, not constant/undef",
447 instr.get());
448 }
449 }
450 } else if (instr->opcode == aco_opcode::p_phi) {
451 check(instr->operands.size() == block.logical_preds.size(),
452 "Number of Operands does not match number of predecessors", instr.get());
453 check(instr->definitions[0].getTemp().type() == RegType::vgpr,
454 "Logical Phi Definition must be vgpr", instr.get());
455 for (const Operand& op : instr->operands)
456 check(instr->definitions[0].size() == op.size(),
457 "Operand sizes must match Definition size", instr.get());
458 } else if (instr->opcode == aco_opcode::p_linear_phi) {
459 for (const Operand& op : instr->operands) {
460 check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type",
461 instr.get());
462 check(instr->definitions[0].size() == op.size(),
463 "Operand sizes must match Definition size", instr.get());
464 }
465 check(instr->operands.size() == block.linear_preds.size(),
466 "Number of Operands does not match number of predecessors", instr.get());
467 } else if (instr->opcode == aco_opcode::p_extract ||
468 instr->opcode == aco_opcode::p_insert) {
469 check(instr->operands[0].isTemp(), "Data operand must be temporary", instr.get());
470 check(instr->operands[1].isConstant(), "Index must be constant", instr.get());
471 if (instr->opcode == aco_opcode::p_extract)
472 check(instr->operands[3].isConstant(), "Sign-extend flag must be constant",
473 instr.get());
474
475 check(instr->definitions[0].getTemp().type() != RegType::sgpr ||
476 instr->operands[0].getTemp().type() == RegType::sgpr,
477 "Can't extract/insert VGPR to SGPR", instr.get());
478
479 if (instr->opcode == aco_opcode::p_insert)
480 check(instr->operands[0].bytes() == instr->definitions[0].bytes(),
481 "Sizes of p_insert data operand and definition must match", instr.get());
482
483 if (instr->definitions[0].getTemp().type() == RegType::sgpr)
484 check(instr->definitions.size() >= 2 && instr->definitions[1].isFixed() &&
485 instr->definitions[1].physReg() == scc,
486 "SGPR extract/insert needs an SCC definition", instr.get());
487
488 unsigned data_bits = instr->operands[0].getTemp().bytes() * 8u;
489 unsigned op_bits = instr->operands[2].constantValue();
490
491 if (instr->opcode == aco_opcode::p_insert) {
492 check(op_bits == 8 || op_bits == 16, "Size must be 8 or 16", instr.get());
493 check(op_bits < data_bits, "Size must be smaller than source", instr.get());
494 } else if (instr->opcode == aco_opcode::p_extract) {
495 check(op_bits == 8 || op_bits == 16 || op_bits == 32,
496 "Size must be 8 or 16 or 32", instr.get());
497 check(data_bits >= op_bits, "Can't extract more bits than what the data has.",
498 instr.get());
499 }
500
501 unsigned comp = data_bits / MAX2(op_bits, 1);
502 check(instr->operands[1].constantValue() < comp, "Index must be in-bounds",
503 instr.get());
504 }
505 break;
506 }
507 case Format::PSEUDO_REDUCTION: {
508 for (const Operand& op : instr->operands)
509 check(op.regClass().type() == RegType::vgpr,
510 "All operands of PSEUDO_REDUCTION instructions must be in VGPRs.",
511 instr.get());
512
513 if (instr->opcode == aco_opcode::p_reduce &&
514 instr->reduction().cluster_size == program->wave_size)
515 check(instr->definitions[0].regClass().type() == RegType::sgpr ||
516 program->wave_size == 32,
517 "The result of unclustered reductions must go into an SGPR.", instr.get());
518 else
519 check(instr->definitions[0].regClass().type() == RegType::vgpr,
520 "The result of scans and clustered reductions must go into a VGPR.",
521 instr.get());
522
523 break;
524 }
525 case Format::SMEM: {
526 if (instr->operands.size() >= 1)
527 check((instr->operands[0].isFixed() && !instr->operands[0].isConstant()) ||
528 (instr->operands[0].isTemp() &&
529 instr->operands[0].regClass().type() == RegType::sgpr),
530 "SMEM operands must be sgpr", instr.get());
531 if (instr->operands.size() >= 2)
532 check(instr->operands[1].isConstant() ||
533 (instr->operands[1].isTemp() &&
534 instr->operands[1].regClass().type() == RegType::sgpr),
535 "SMEM offset must be constant or sgpr", instr.get());
536 if (!instr->definitions.empty())
537 check(instr->definitions[0].getTemp().type() == RegType::sgpr,
538 "SMEM result must be sgpr", instr.get());
539 break;
540 }
541 case Format::MTBUF:
542 case Format::MUBUF: {
543 check(instr->operands.size() > 1, "VMEM instructions must have at least one operand",
544 instr.get());
545 check(instr->operands[1].hasRegClass() &&
546 instr->operands[1].regClass().type() == RegType::vgpr,
547 "VADDR must be in vgpr for VMEM instructions", instr.get());
548 check(
549 instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr,
550 "VMEM resource constant must be sgpr", instr.get());
551 check(instr->operands.size() < 4 ||
552 (instr->operands[3].isTemp() &&
553 instr->operands[3].regClass().type() == RegType::vgpr),
554 "VMEM write data must be vgpr", instr.get());
555 break;
556 }
557 case Format::MIMG: {
558 check(instr->operands.size() >= 4, "MIMG instructions must have at least 4 operands",
559 instr.get());
560 check(instr->operands[0].hasRegClass() &&
561 (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
562 "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
563 if (instr->operands[1].hasRegClass())
564 check(instr->operands[1].regClass() == s4,
565 "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
566 if (!instr->operands[2].isUndefined()) {
567 bool is_cmpswap = instr->opcode == aco_opcode::image_atomic_cmpswap ||
568 instr->opcode == aco_opcode::image_atomic_fcmpswap;
569 check(instr->definitions.empty() ||
570 (instr->definitions[0].regClass() == instr->operands[2].regClass() ||
571 is_cmpswap),
572 "MIMG operands[2] (VDATA) must be the same as definitions[0] for atomics and "
573 "TFE/LWE loads",
574 instr.get());
575 }
576 check(instr->operands.size() == 4 || program->chip_class >= GFX10,
577 "NSA is only supported on GFX10+", instr.get());
578 for (unsigned i = 3; i < instr->operands.size(); i++) {
579 if (instr->operands.size() == 4) {
580 check(instr->operands[i].hasRegClass() &&
581 instr->operands[i].regClass().type() == RegType::vgpr,
582 "MIMG operands[3] (VADDR) must be VGPR", instr.get());
583 } else {
584 check(instr->operands[i].regClass() == v1, "MIMG VADDR must be v1 if NSA is used",
585 instr.get());
586 }
587 }
588 check(instr->definitions.empty() ||
589 (instr->definitions[0].isTemp() &&
590 instr->definitions[0].regClass().type() == RegType::vgpr),
591 "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
592 break;
593 }
594 case Format::DS: {
595 for (const Operand& op : instr->operands) {
596 check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
597 "Only VGPRs are valid DS instruction operands", instr.get());
598 }
599 if (!instr->definitions.empty())
600 check(instr->definitions[0].getTemp().type() == RegType::vgpr,
601 "DS instruction must return VGPR", instr.get());
602 break;
603 }
604 case Format::EXP: {
605 for (unsigned i = 0; i < 4; i++)
606 check(instr->operands[i].hasRegClass() &&
607 instr->operands[i].regClass().type() == RegType::vgpr,
608 "Only VGPRs are valid Export arguments", instr.get());
609 break;
610 }
611 case Format::FLAT:
612 check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR",
613 instr.get());
614 FALLTHROUGH;
615 case Format::GLOBAL:
616 case Format::SCRATCH: {
617 check(
618 instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr,
619 "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
620 check(instr->operands[1].hasRegClass() &&
621 instr->operands[1].regClass().type() == RegType::sgpr,
622 "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
623 if (!instr->definitions.empty())
624 check(instr->definitions[0].getTemp().type() == RegType::vgpr,
625 "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
626 else
627 check(instr->operands[2].regClass().type() == RegType::vgpr,
628 "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
629 break;
630 }
631 default: break;
632 }
633 }
634 }
635
636 /* validate CFG */
637 for (unsigned i = 0; i < program->blocks.size(); i++) {
638 Block& block = program->blocks[i];
639 check_block(block.index == i, "block.index must match actual index", &block);
640
641 /* predecessors/successors should be sorted */
642 for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
643 check_block(block.linear_preds[j] < block.linear_preds[j + 1],
644 "linear predecessors must be sorted", &block);
645 for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
646 check_block(block.logical_preds[j] < block.logical_preds[j + 1],
647 "logical predecessors must be sorted", &block);
648 for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
649 check_block(block.linear_succs[j] < block.linear_succs[j + 1],
650 "linear successors must be sorted", &block);
651 for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
652 check_block(block.logical_succs[j] < block.logical_succs[j + 1],
653 "logical successors must be sorted", &block);
654
655 /* critical edges are not allowed */
656 if (block.linear_preds.size() > 1) {
657 for (unsigned pred : block.linear_preds)
658 check_block(program->blocks[pred].linear_succs.size() == 1,
659 "linear critical edges are not allowed", &program->blocks[pred]);
660 for (unsigned pred : block.logical_preds)
661 check_block(program->blocks[pred].logical_succs.size() == 1,
662 "logical critical edges are not allowed", &program->blocks[pred]);
663 }
664 }
665
666 return is_valid;
667 }
668
669 /* RA validation */
670 namespace {
671
672 struct Location {
Locationaco::__anonf49a09840311::Location673 Location() : block(NULL), instr(NULL) {}
674
675 Block* block;
676 Instruction* instr; // NULL if it's the block's live-in
677 };
678
679 struct Assignment {
680 Location defloc;
681 Location firstloc;
682 PhysReg reg;
683 };
684
685 bool
ra_fail(Program * program,Location loc,Location loc2,const char * fmt,...)686 ra_fail(Program* program, Location loc, Location loc2, const char* fmt, ...)
687 {
688 va_list args;
689 va_start(args, fmt);
690 char msg[1024];
691 vsprintf(msg, fmt, args);
692 va_end(args);
693
694 char* out;
695 size_t outsize;
696 struct u_memstream mem;
697 u_memstream_open(&mem, &out, &outsize);
698 FILE* const memf = u_memstream_get(&mem);
699
700 fprintf(memf, "RA error found at instruction in BB%d:\n", loc.block->index);
701 if (loc.instr) {
702 aco_print_instr(loc.instr, memf);
703 fprintf(memf, "\n%s", msg);
704 } else {
705 fprintf(memf, "%s", msg);
706 }
707 if (loc2.block) {
708 fprintf(memf, " in BB%d:\n", loc2.block->index);
709 aco_print_instr(loc2.instr, memf);
710 }
711 fprintf(memf, "\n\n");
712 u_memstream_close(&mem);
713
714 aco_err(program, "%s", out);
715 free(out);
716
717 return true;
718 }
719
720 bool
validate_subdword_operand(chip_class chip,const aco_ptr<Instruction> & instr,unsigned index)721 validate_subdword_operand(chip_class chip, const aco_ptr<Instruction>& instr, unsigned index)
722 {
723 Operand op = instr->operands[index];
724 unsigned byte = op.physReg().byte();
725
726 if (instr->opcode == aco_opcode::p_as_uniform)
727 return byte == 0;
728 if (instr->isPseudo() && chip >= GFX8)
729 return true;
730 if (instr->isSDWA())
731 return byte + instr->sdwa().sel[index].offset() + instr->sdwa().sel[index].size() <= 4 &&
732 byte % instr->sdwa().sel[index].size() == 0;
733 if (instr->isVOP3P())
734 return ((instr->vop3p().opsel_lo >> index) & 1) == (byte >> 1) &&
735 ((instr->vop3p().opsel_hi >> index) & 1) == (byte >> 1);
736 if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1))
737 return true;
738
739 switch (instr->opcode) {
740 case aco_opcode::v_cvt_f32_ubyte1:
741 if (byte == 1)
742 return true;
743 break;
744 case aco_opcode::v_cvt_f32_ubyte2:
745 if (byte == 2)
746 return true;
747 break;
748 case aco_opcode::v_cvt_f32_ubyte3:
749 if (byte == 3)
750 return true;
751 break;
752 case aco_opcode::ds_write_b8_d16_hi:
753 case aco_opcode::ds_write_b16_d16_hi:
754 if (byte == 2 && index == 1)
755 return true;
756 break;
757 case aco_opcode::buffer_store_byte_d16_hi:
758 case aco_opcode::buffer_store_short_d16_hi:
759 if (byte == 2 && index == 3)
760 return true;
761 break;
762 case aco_opcode::flat_store_byte_d16_hi:
763 case aco_opcode::flat_store_short_d16_hi:
764 case aco_opcode::scratch_store_byte_d16_hi:
765 case aco_opcode::scratch_store_short_d16_hi:
766 case aco_opcode::global_store_byte_d16_hi:
767 case aco_opcode::global_store_short_d16_hi:
768 if (byte == 2 && index == 2)
769 return true;
770 break;
771 default: break;
772 }
773
774 return byte == 0;
775 }
776
777 bool
validate_subdword_definition(chip_class chip,const aco_ptr<Instruction> & instr)778 validate_subdword_definition(chip_class chip, const aco_ptr<Instruction>& instr)
779 {
780 Definition def = instr->definitions[0];
781 unsigned byte = def.physReg().byte();
782
783 if (instr->isPseudo() && chip >= GFX8)
784 return true;
785 if (instr->isSDWA())
786 return byte + instr->sdwa().dst_sel.offset() + instr->sdwa().dst_sel.size() <= 4 &&
787 byte % instr->sdwa().dst_sel.size() == 0;
788 if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1))
789 return true;
790
791 switch (instr->opcode) {
792 case aco_opcode::buffer_load_ubyte_d16_hi:
793 case aco_opcode::buffer_load_short_d16_hi:
794 case aco_opcode::flat_load_ubyte_d16_hi:
795 case aco_opcode::flat_load_short_d16_hi:
796 case aco_opcode::scratch_load_ubyte_d16_hi:
797 case aco_opcode::scratch_load_short_d16_hi:
798 case aco_opcode::global_load_ubyte_d16_hi:
799 case aco_opcode::global_load_short_d16_hi:
800 case aco_opcode::ds_read_u8_d16_hi:
801 case aco_opcode::ds_read_u16_d16_hi: return byte == 2;
802 default: break;
803 }
804
805 return byte == 0;
806 }
807
808 unsigned
get_subdword_bytes_written(Program * program,const aco_ptr<Instruction> & instr,unsigned index)809 get_subdword_bytes_written(Program* program, const aco_ptr<Instruction>& instr, unsigned index)
810 {
811 chip_class chip = program->chip_class;
812 Definition def = instr->definitions[index];
813
814 if (instr->isPseudo())
815 return chip >= GFX8 ? def.bytes() : def.size() * 4u;
816 if (instr->isVALU()) {
817 assert(def.bytes() <= 2);
818 if (instr->isSDWA())
819 return instr->sdwa().dst_sel.size();
820
821 if (instr_is_16bit(chip, instr->opcode))
822 return 2;
823
824 return 4;
825 }
826
827 switch (instr->opcode) {
828 case aco_opcode::buffer_load_ubyte_d16:
829 case aco_opcode::buffer_load_short_d16:
830 case aco_opcode::flat_load_ubyte_d16:
831 case aco_opcode::flat_load_short_d16:
832 case aco_opcode::scratch_load_ubyte_d16:
833 case aco_opcode::scratch_load_short_d16:
834 case aco_opcode::global_load_ubyte_d16:
835 case aco_opcode::global_load_short_d16:
836 case aco_opcode::ds_read_u8_d16:
837 case aco_opcode::ds_read_u16_d16:
838 case aco_opcode::buffer_load_ubyte_d16_hi:
839 case aco_opcode::buffer_load_short_d16_hi:
840 case aco_opcode::flat_load_ubyte_d16_hi:
841 case aco_opcode::flat_load_short_d16_hi:
842 case aco_opcode::scratch_load_ubyte_d16_hi:
843 case aco_opcode::scratch_load_short_d16_hi:
844 case aco_opcode::global_load_ubyte_d16_hi:
845 case aco_opcode::global_load_short_d16_hi:
846 case aco_opcode::ds_read_u8_d16_hi:
847 case aco_opcode::ds_read_u16_d16_hi: return program->dev.sram_ecc_enabled ? 4 : 2;
848 default: return def.size() * 4;
849 }
850 }
851
852 } /* end namespace */
853
854 bool
validate_ra(Program * program)855 validate_ra(Program* program)
856 {
857 if (!(debug_flags & DEBUG_VALIDATE_RA))
858 return false;
859
860 bool err = false;
861 aco::live live_vars = aco::live_var_analysis(program);
862 std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
863 uint16_t sgpr_limit = get_addr_sgpr_from_waves(program, program->num_waves);
864
865 std::map<unsigned, Assignment> assignments;
866 for (Block& block : program->blocks) {
867 Location loc;
868 loc.block = █
869 for (aco_ptr<Instruction>& instr : block.instructions) {
870 if (instr->opcode == aco_opcode::p_phi) {
871 for (unsigned i = 0; i < instr->operands.size(); i++) {
872 if (instr->operands[i].isTemp() &&
873 instr->operands[i].getTemp().type() == RegType::sgpr &&
874 instr->operands[i].isFirstKill())
875 phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
876 }
877 }
878
879 loc.instr = instr.get();
880 for (unsigned i = 0; i < instr->operands.size(); i++) {
881 Operand& op = instr->operands[i];
882 if (!op.isTemp())
883 continue;
884 if (!op.isFixed())
885 err |= ra_fail(program, loc, Location(), "Operand %d is not assigned a register", i);
886 if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
887 err |=
888 ra_fail(program, loc, assignments.at(op.tempId()).firstloc,
889 "Operand %d has an inconsistent register assignment with instruction", i);
890 if ((op.getTemp().type() == RegType::vgpr &&
891 op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) ||
892 (op.getTemp().type() == RegType::sgpr &&
893 op.physReg() + op.size() > program->config->num_sgprs &&
894 op.physReg() < sgpr_limit))
895 err |= ra_fail(program, loc, assignments.at(op.tempId()).firstloc,
896 "Operand %d has an out-of-bounds register assignment", i);
897 if (op.physReg() == vcc && !program->needs_vcc)
898 err |= ra_fail(program, loc, Location(),
899 "Operand %d fixed to vcc but needs_vcc=false", i);
900 if (op.regClass().is_subdword() &&
901 !validate_subdword_operand(program->chip_class, instr, i))
902 err |= ra_fail(program, loc, Location(), "Operand %d not aligned correctly", i);
903 if (!assignments[op.tempId()].firstloc.block)
904 assignments[op.tempId()].firstloc = loc;
905 if (!assignments[op.tempId()].defloc.block)
906 assignments[op.tempId()].reg = op.physReg();
907 }
908
909 for (unsigned i = 0; i < instr->definitions.size(); i++) {
910 Definition& def = instr->definitions[i];
911 if (!def.isTemp())
912 continue;
913 if (!def.isFixed())
914 err |=
915 ra_fail(program, loc, Location(), "Definition %d is not assigned a register", i);
916 if (assignments[def.tempId()].defloc.block)
917 err |= ra_fail(program, loc, assignments.at(def.tempId()).defloc,
918 "Temporary %%%d also defined by instruction", def.tempId());
919 if ((def.getTemp().type() == RegType::vgpr &&
920 def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) ||
921 (def.getTemp().type() == RegType::sgpr &&
922 def.physReg() + def.size() > program->config->num_sgprs &&
923 def.physReg() < sgpr_limit))
924 err |= ra_fail(program, loc, assignments.at(def.tempId()).firstloc,
925 "Definition %d has an out-of-bounds register assignment", i);
926 if (def.physReg() == vcc && !program->needs_vcc)
927 err |= ra_fail(program, loc, Location(),
928 "Definition %d fixed to vcc but needs_vcc=false", i);
929 if (def.regClass().is_subdword() &&
930 !validate_subdword_definition(program->chip_class, instr))
931 err |= ra_fail(program, loc, Location(), "Definition %d not aligned correctly", i);
932 if (!assignments[def.tempId()].firstloc.block)
933 assignments[def.tempId()].firstloc = loc;
934 assignments[def.tempId()].defloc = loc;
935 assignments[def.tempId()].reg = def.physReg();
936 }
937 }
938 }
939
940 for (Block& block : program->blocks) {
941 Location loc;
942 loc.block = █
943
944 std::array<unsigned, 2048> regs; /* register file in bytes */
945 regs.fill(0);
946
947 std::set<Temp> live;
948 for (unsigned id : live_vars.live_out[block.index])
949 live.insert(Temp(id, program->temp_rc[id]));
950 /* remove killed p_phi sgpr operands */
951 for (Temp tmp : phi_sgpr_ops[block.index])
952 live.erase(tmp);
953
954 /* check live out */
955 for (Temp tmp : live) {
956 PhysReg reg = assignments.at(tmp.id()).reg;
957 for (unsigned i = 0; i < tmp.bytes(); i++) {
958 if (regs[reg.reg_b + i]) {
959 err |= ra_fail(program, loc, Location(),
960 "Assignment of element %d of %%%d already taken by %%%d in live-out",
961 i, tmp.id(), regs[reg.reg_b + i]);
962 }
963 regs[reg.reg_b + i] = tmp.id();
964 }
965 }
966 regs.fill(0);
967
968 for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
969 aco_ptr<Instruction>& instr = *it;
970
971 /* check killed p_phi sgpr operands */
972 if (instr->opcode == aco_opcode::p_logical_end) {
973 for (Temp tmp : phi_sgpr_ops[block.index]) {
974 PhysReg reg = assignments.at(tmp.id()).reg;
975 for (unsigned i = 0; i < tmp.bytes(); i++) {
976 if (regs[reg.reg_b + i])
977 err |= ra_fail(
978 program, loc, Location(),
979 "Assignment of element %d of %%%d already taken by %%%d in live-out", i,
980 tmp.id(), regs[reg.reg_b + i]);
981 }
982 live.emplace(tmp);
983 }
984 }
985
986 for (const Definition& def : instr->definitions) {
987 if (!def.isTemp())
988 continue;
989 live.erase(def.getTemp());
990 }
991
992 /* don't count phi operands as live-in, since they are actually
993 * killed when they are copied at the predecessor */
994 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
995 for (const Operand& op : instr->operands) {
996 if (!op.isTemp())
997 continue;
998 live.insert(op.getTemp());
999 }
1000 }
1001 }
1002
1003 for (Temp tmp : live) {
1004 PhysReg reg = assignments.at(tmp.id()).reg;
1005 for (unsigned i = 0; i < tmp.bytes(); i++)
1006 regs[reg.reg_b + i] = tmp.id();
1007 }
1008
1009 for (aco_ptr<Instruction>& instr : block.instructions) {
1010 loc.instr = instr.get();
1011
1012 /* remove killed p_phi operands from regs */
1013 if (instr->opcode == aco_opcode::p_logical_end) {
1014 for (Temp tmp : phi_sgpr_ops[block.index]) {
1015 PhysReg reg = assignments.at(tmp.id()).reg;
1016 for (unsigned i = 0; i < tmp.bytes(); i++)
1017 regs[reg.reg_b + i] = 0;
1018 }
1019 }
1020
1021 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1022 for (const Operand& op : instr->operands) {
1023 if (!op.isTemp())
1024 continue;
1025 if (op.isFirstKillBeforeDef()) {
1026 for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1027 regs[op.physReg().reg_b + j] = 0;
1028 }
1029 }
1030 }
1031
1032 for (unsigned i = 0; i < instr->definitions.size(); i++) {
1033 Definition& def = instr->definitions[i];
1034 if (!def.isTemp())
1035 continue;
1036 Temp tmp = def.getTemp();
1037 PhysReg reg = assignments.at(tmp.id()).reg;
1038 for (unsigned j = 0; j < tmp.bytes(); j++) {
1039 if (regs[reg.reg_b + j])
1040 err |= ra_fail(
1041 program, loc, assignments.at(regs[reg.reg_b + j]).defloc,
1042 "Assignment of element %d of %%%d already taken by %%%d from instruction", i,
1043 tmp.id(), regs[reg.reg_b + j]);
1044 regs[reg.reg_b + j] = tmp.id();
1045 }
1046 if (def.regClass().is_subdword() && def.bytes() < 4) {
1047 unsigned written = get_subdword_bytes_written(program, instr, i);
1048 /* If written=4, the instruction still might write the upper half. In that case, it's
1049 * the lower half that isn't preserved */
1050 for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) {
1051 unsigned written_reg = reg.reg() * 4u + j;
1052 if (regs[written_reg] && regs[written_reg] != def.tempId())
1053 err |= ra_fail(program, loc, assignments.at(regs[written_reg]).defloc,
1054 "Assignment of element %d of %%%d overwrites the full register "
1055 "taken by %%%d from instruction",
1056 i, tmp.id(), regs[written_reg]);
1057 }
1058 }
1059 }
1060
1061 for (const Definition& def : instr->definitions) {
1062 if (!def.isTemp())
1063 continue;
1064 if (def.isKill()) {
1065 for (unsigned j = 0; j < def.getTemp().bytes(); j++)
1066 regs[def.physReg().reg_b + j] = 0;
1067 }
1068 }
1069
1070 if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1071 for (const Operand& op : instr->operands) {
1072 if (!op.isTemp())
1073 continue;
1074 if (op.isLateKill() && op.isFirstKill()) {
1075 for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1076 regs[op.physReg().reg_b + j] = 0;
1077 }
1078 }
1079 }
1080 }
1081 }
1082
1083 return err;
1084 }
1085 } // namespace aco
1086