1 /*
2  * Copyright © 2018 Valve Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24 
25 #include "aco_ir.h"
26 
27 #include "util/memstream.h"
28 
29 #include <array>
30 #include <map>
31 #include <set>
32 #include <vector>
33 
34 namespace aco {
35 
36 static void
aco_log(Program * program,enum radv_compiler_debug_level level,const char * prefix,const char * file,unsigned line,const char * fmt,va_list args)37 aco_log(Program* program, enum radv_compiler_debug_level level, const char* prefix,
38         const char* file, unsigned line, const char* fmt, va_list args)
39 {
40    char* msg;
41 
42    if (program->debug.shorten_messages) {
43       msg = ralloc_vasprintf(NULL, fmt, args);
44    } else {
45       msg = ralloc_strdup(NULL, prefix);
46       ralloc_asprintf_append(&msg, "    In file %s:%u\n", file, line);
47       ralloc_asprintf_append(&msg, "    ");
48       ralloc_vasprintf_append(&msg, fmt, args);
49    }
50 
51    if (program->debug.func)
52       program->debug.func(program->debug.private_data, level, msg);
53 
54    fprintf(program->debug.output, "%s\n", msg);
55 
56    ralloc_free(msg);
57 }
58 
59 void
_aco_perfwarn(Program * program,const char * file,unsigned line,const char * fmt,...)60 _aco_perfwarn(Program* program, const char* file, unsigned line, const char* fmt, ...)
61 {
62    va_list args;
63 
64    va_start(args, fmt);
65    aco_log(program, RADV_COMPILER_DEBUG_LEVEL_PERFWARN, "ACO PERFWARN:\n", file, line, fmt, args);
66    va_end(args);
67 }
68 
69 void
_aco_err(Program * program,const char * file,unsigned line,const char * fmt,...)70 _aco_err(Program* program, const char* file, unsigned line, const char* fmt, ...)
71 {
72    va_list args;
73 
74    va_start(args, fmt);
75    aco_log(program, RADV_COMPILER_DEBUG_LEVEL_ERROR, "ACO ERROR:\n", file, line, fmt, args);
76    va_end(args);
77 }
78 
79 bool
validate_ir(Program * program)80 validate_ir(Program* program)
81 {
82    bool is_valid = true;
83    auto check = [&program, &is_valid](bool success, const char* msg,
84                                       aco::Instruction* instr) -> void
85    {
86       if (!success) {
87          char* out;
88          size_t outsize;
89          struct u_memstream mem;
90          u_memstream_open(&mem, &out, &outsize);
91          FILE* const memf = u_memstream_get(&mem);
92 
93          fprintf(memf, "%s: ", msg);
94          aco_print_instr(instr, memf);
95          u_memstream_close(&mem);
96 
97          aco_err(program, "%s", out);
98          free(out);
99 
100          is_valid = false;
101       }
102    };
103 
104    auto check_block = [&program, &is_valid](bool success, const char* msg,
105                                             aco::Block* block) -> void
106    {
107       if (!success) {
108          aco_err(program, "%s: BB%u", msg, block->index);
109          is_valid = false;
110       }
111    };
112 
113    for (Block& block : program->blocks) {
114       for (aco_ptr<Instruction>& instr : block.instructions) {
115 
116          /* check base format */
117          Format base_format = instr->format;
118          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::SDWA);
119          base_format = (Format)((uint32_t)base_format & ~(uint32_t)Format::DPP);
120          if ((uint32_t)base_format & (uint32_t)Format::VOP1)
121             base_format = Format::VOP1;
122          else if ((uint32_t)base_format & (uint32_t)Format::VOP2)
123             base_format = Format::VOP2;
124          else if ((uint32_t)base_format & (uint32_t)Format::VOPC)
125             base_format = Format::VOPC;
126          else if ((uint32_t)base_format & (uint32_t)Format::VINTRP) {
127             if (instr->opcode == aco_opcode::v_interp_p1ll_f16 ||
128                 instr->opcode == aco_opcode::v_interp_p1lv_f16 ||
129                 instr->opcode == aco_opcode::v_interp_p2_legacy_f16 ||
130                 instr->opcode == aco_opcode::v_interp_p2_f16) {
131                /* v_interp_*_fp16 are considered VINTRP by the compiler but
132                 * they are emitted as VOP3.
133                 */
134                base_format = Format::VOP3;
135             } else {
136                base_format = Format::VINTRP;
137             }
138          }
139          check(base_format == instr_info.format[(int)instr->opcode],
140                "Wrong base format for instruction", instr.get());
141 
142          /* check VOP3 modifiers */
143          if (instr->isVOP3() && instr->format != Format::VOP3) {
144             check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
145                      base_format == Format::VOPC || base_format == Format::VINTRP,
146                   "Format cannot have VOP3/VOP3B applied", instr.get());
147          }
148 
149          /* check SDWA */
150          if (instr->isSDWA()) {
151             check(base_format == Format::VOP2 || base_format == Format::VOP1 ||
152                      base_format == Format::VOPC,
153                   "Format cannot have SDWA applied", instr.get());
154 
155             check(program->chip_class >= GFX8, "SDWA is GFX8+ only", instr.get());
156 
157             SDWA_instruction& sdwa = instr->sdwa();
158             check(sdwa.omod == 0 || program->chip_class >= GFX9,
159                   "SDWA omod only supported on GFX9+", instr.get());
160             if (base_format == Format::VOPC) {
161                check(sdwa.clamp == false || program->chip_class == GFX8,
162                      "SDWA VOPC clamp only supported on GFX8", instr.get());
163                check((instr->definitions[0].isFixed() && instr->definitions[0].physReg() == vcc) ||
164                         program->chip_class >= GFX9,
165                      "SDWA+VOPC definition must be fixed to vcc on GFX8", instr.get());
166             } else {
167                const Definition& def = instr->definitions[0];
168                check(def.bytes() <= 4, "SDWA definitions must not be larger than 4 bytes",
169                      instr.get());
170                check(def.bytes() >= sdwa.dst_sel.size() + sdwa.dst_sel.offset(),
171                      "SDWA definition selection size must be at most definition size", instr.get());
172                check(
173                   sdwa.dst_sel.size() == 1 || sdwa.dst_sel.size() == 2 || sdwa.dst_sel.size() == 4,
174                   "SDWA definition selection size must be 1, 2 or 4 bytes", instr.get());
175                check(sdwa.dst_sel.offset() % sdwa.dst_sel.size() == 0, "Invalid selection offset",
176                      instr.get());
177                check(def.bytes() == 4 || def.bytes() == sdwa.dst_sel.size(),
178                      "SDWA dst_sel size must be definition size for subdword definitions",
179                      instr.get());
180                check(def.bytes() == 4 || sdwa.dst_sel.offset() == 0,
181                      "SDWA dst_sel offset must be 0 for subdword definitions", instr.get());
182             }
183 
184             for (unsigned i = 0; i < std::min<unsigned>(2, instr->operands.size()); i++) {
185                const Operand& op = instr->operands[i];
186                check(op.bytes() <= 4, "SDWA operands must not be larger than 4 bytes", instr.get());
187                check(op.bytes() >= sdwa.sel[i].size() + sdwa.sel[i].offset(),
188                      "SDWA operand selection size must be at most operand size", instr.get());
189                check(sdwa.sel[i].size() == 1 || sdwa.sel[i].size() == 2 || sdwa.sel[i].size() == 4,
190                      "SDWA operand selection size must be 1, 2 or 4 bytes", instr.get());
191                check(sdwa.sel[i].offset() % sdwa.sel[i].size() == 0, "Invalid selection offset",
192                      instr.get());
193             }
194             if (instr->operands.size() >= 3) {
195                check(instr->operands[2].isFixed() && instr->operands[2].physReg() == vcc,
196                      "3rd operand must be fixed to vcc with SDWA", instr.get());
197             }
198             if (instr->definitions.size() >= 2) {
199                check(instr->definitions[1].isFixed() && instr->definitions[1].physReg() == vcc,
200                      "2nd definition must be fixed to vcc with SDWA", instr.get());
201             }
202 
203             const bool sdwa_opcodes =
204                instr->opcode != aco_opcode::v_fmac_f32 && instr->opcode != aco_opcode::v_fmac_f16 &&
205                instr->opcode != aco_opcode::v_fmamk_f32 &&
206                instr->opcode != aco_opcode::v_fmaak_f32 &&
207                instr->opcode != aco_opcode::v_fmamk_f16 &&
208                instr->opcode != aco_opcode::v_fmaak_f16 &&
209                instr->opcode != aco_opcode::v_madmk_f32 &&
210                instr->opcode != aco_opcode::v_madak_f32 &&
211                instr->opcode != aco_opcode::v_madmk_f16 &&
212                instr->opcode != aco_opcode::v_madak_f16 &&
213                instr->opcode != aco_opcode::v_readfirstlane_b32 &&
214                instr->opcode != aco_opcode::v_clrexcp && instr->opcode != aco_opcode::v_swap_b32;
215 
216             const bool feature_mac =
217                program->chip_class == GFX8 &&
218                (instr->opcode == aco_opcode::v_mac_f32 && instr->opcode == aco_opcode::v_mac_f16);
219 
220             check(sdwa_opcodes || feature_mac, "SDWA can't be used with this opcode", instr.get());
221          }
222 
223          /* check opsel */
224          if (instr->isVOP3()) {
225             VOP3_instruction& vop3 = instr->vop3();
226             check(vop3.opsel == 0 || program->chip_class >= GFX9,
227                   "Opsel is only supported on GFX9+", instr.get());
228 
229             for (unsigned i = 0; i < 3; i++) {
230                if (i >= instr->operands.size() ||
231                    (instr->operands[i].hasRegClass() &&
232                     instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed()))
233                   check((vop3.opsel & (1 << i)) == 0, "Unexpected opsel for operand", instr.get());
234             }
235             if (instr->definitions[0].regClass().is_subdword() && !instr->definitions[0].isFixed())
236                check((vop3.opsel & (1 << 3)) == 0, "Unexpected opsel for sub-dword definition",
237                      instr.get());
238          } else if (instr->isVOP3P()) {
239             VOP3P_instruction& vop3p = instr->vop3p();
240             for (unsigned i = 0; i < instr->operands.size(); i++) {
241                if (instr->operands[i].hasRegClass() &&
242                    instr->operands[i].regClass().is_subdword() && !instr->operands[i].isFixed())
243                   check((vop3p.opsel_lo & (1 << i)) == 0 && (vop3p.opsel_hi & (1 << i)) == 0,
244                         "Unexpected opsel for subdword operand", instr.get());
245             }
246             check(instr->definitions[0].regClass() == v1, "VOP3P must have v1 definition",
247                   instr.get());
248          }
249 
250          /* check for undefs */
251          for (unsigned i = 0; i < instr->operands.size(); i++) {
252             if (instr->operands[i].isUndefined()) {
253                bool flat = instr->isFlatLike();
254                bool can_be_undef = is_phi(instr) || instr->isEXP() || instr->isReduction() ||
255                                    instr->opcode == aco_opcode::p_create_vector ||
256                                    (flat && i == 1) || (instr->isMIMG() && (i == 1 || i == 2)) ||
257                                    ((instr->isMUBUF() || instr->isMTBUF()) && i == 1);
258                check(can_be_undef, "Undefs can only be used in certain operands", instr.get());
259             } else {
260                check(instr->operands[i].isFixed() || instr->operands[i].isTemp() ||
261                         instr->operands[i].isConstant(),
262                      "Uninitialized Operand", instr.get());
263             }
264          }
265 
266          /* check subdword definitions */
267          for (unsigned i = 0; i < instr->definitions.size(); i++) {
268             if (instr->definitions[i].regClass().is_subdword())
269                check(instr->isPseudo() || instr->definitions[i].bytes() <= 4,
270                      "Only Pseudo instructions can write subdword registers larger than 4 bytes",
271                      instr.get());
272          }
273 
274          if (instr->isSALU() || instr->isVALU()) {
275             /* check literals */
276             Operand literal(s1);
277             for (unsigned i = 0; i < instr->operands.size(); i++) {
278                Operand op = instr->operands[i];
279                if (!op.isLiteral())
280                   continue;
281 
282                check(!instr->isDPP() && !instr->isSDWA() &&
283                         (!instr->isVOP3() || program->chip_class >= GFX10) &&
284                         (!instr->isVOP3P() || program->chip_class >= GFX10),
285                      "Literal applied on wrong instruction format", instr.get());
286 
287                check(literal.isUndefined() || (literal.size() == op.size() &&
288                                                literal.constantValue() == op.constantValue()),
289                      "Only 1 Literal allowed", instr.get());
290                literal = op;
291                check(instr->isSALU() || instr->isVOP3() || instr->isVOP3P() || i == 0 || i == 2,
292                      "Wrong source position for Literal argument", instr.get());
293             }
294 
295             /* check num sgprs for VALU */
296             if (instr->isVALU()) {
297                bool is_shift64 = instr->opcode == aco_opcode::v_lshlrev_b64 ||
298                                  instr->opcode == aco_opcode::v_lshrrev_b64 ||
299                                  instr->opcode == aco_opcode::v_ashrrev_i64;
300                unsigned const_bus_limit = 1;
301                if (program->chip_class >= GFX10 && !is_shift64)
302                   const_bus_limit = 2;
303 
304                uint32_t scalar_mask = instr->isVOP3() || instr->isVOP3P() ? 0x7 : 0x5;
305                if (instr->isSDWA())
306                   scalar_mask = program->chip_class >= GFX9 ? 0x7 : 0x4;
307                else if (instr->isDPP())
308                   scalar_mask = 0x4;
309 
310                if (instr->isVOPC() || instr->opcode == aco_opcode::v_readfirstlane_b32 ||
311                    instr->opcode == aco_opcode::v_readlane_b32 ||
312                    instr->opcode == aco_opcode::v_readlane_b32_e64) {
313                   check(instr->definitions[0].getTemp().type() == RegType::sgpr,
314                         "Wrong Definition type for VALU instruction", instr.get());
315                } else {
316                   check(instr->definitions[0].getTemp().type() == RegType::vgpr,
317                         "Wrong Definition type for VALU instruction", instr.get());
318                }
319 
320                unsigned num_sgprs = 0;
321                unsigned sgpr[] = {0, 0};
322                for (unsigned i = 0; i < instr->operands.size(); i++) {
323                   Operand op = instr->operands[i];
324                   if (instr->opcode == aco_opcode::v_readfirstlane_b32 ||
325                       instr->opcode == aco_opcode::v_readlane_b32 ||
326                       instr->opcode == aco_opcode::v_readlane_b32_e64) {
327                      check(i != 1 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
328                               op.isConstant(),
329                            "Must be a SGPR or a constant", instr.get());
330                      check(i == 1 || (op.isTemp() && op.regClass().type() == RegType::vgpr &&
331                                       op.bytes() <= 4),
332                            "Wrong Operand type for VALU instruction", instr.get());
333                      continue;
334                   }
335                   if (instr->opcode == aco_opcode::v_permlane16_b32 ||
336                       instr->opcode == aco_opcode::v_permlanex16_b32) {
337                      check(i != 0 || (op.isTemp() && op.regClass().type() == RegType::vgpr),
338                            "Operand 0 of v_permlane must be VGPR", instr.get());
339                      check(i == 0 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
340                               op.isConstant(),
341                            "Lane select operands of v_permlane must be SGPR or constant",
342                            instr.get());
343                   }
344 
345                   if (instr->opcode == aco_opcode::v_writelane_b32 ||
346                       instr->opcode == aco_opcode::v_writelane_b32_e64) {
347                      check(i != 2 || (op.isTemp() && op.regClass().type() == RegType::vgpr &&
348                                       op.bytes() <= 4),
349                            "Wrong Operand type for VALU instruction", instr.get());
350                      check(i == 2 || (op.isTemp() && op.regClass().type() == RegType::sgpr) ||
351                               op.isConstant(),
352                            "Must be a SGPR or a constant", instr.get());
353                      continue;
354                   }
355                   if (op.isTemp() && instr->operands[i].regClass().type() == RegType::sgpr) {
356                      check(scalar_mask & (1 << i), "Wrong source position for SGPR argument",
357                            instr.get());
358 
359                      if (op.tempId() != sgpr[0] && op.tempId() != sgpr[1]) {
360                         if (num_sgprs < 2)
361                            sgpr[num_sgprs++] = op.tempId();
362                      }
363                   }
364 
365                   if (op.isConstant() && !op.isLiteral())
366                      check(scalar_mask & (1 << i), "Wrong source position for constant argument",
367                            instr.get());
368                }
369                check(num_sgprs + (literal.isUndefined() ? 0 : 1) <= const_bus_limit,
370                      "Too many SGPRs/literals", instr.get());
371             }
372 
373             if (instr->isSOP1() || instr->isSOP2()) {
374                check(instr->definitions[0].getTemp().type() == RegType::sgpr,
375                      "Wrong Definition type for SALU instruction", instr.get());
376                for (const Operand& op : instr->operands) {
377                   check(op.isConstant() || op.regClass().type() <= RegType::sgpr,
378                         "Wrong Operand type for SALU instruction", instr.get());
379                }
380             }
381          }
382 
383          switch (instr->format) {
384          case Format::PSEUDO: {
385             if (instr->opcode == aco_opcode::p_create_vector) {
386                unsigned size = 0;
387                for (const Operand& op : instr->operands) {
388                   check(op.bytes() < 4 || size % 4 == 0, "Operand is not aligned", instr.get());
389                   size += op.bytes();
390                }
391                check(size == instr->definitions[0].bytes(),
392                      "Definition size does not match operand sizes", instr.get());
393                if (instr->definitions[0].getTemp().type() == RegType::sgpr) {
394                   for (const Operand& op : instr->operands) {
395                      check(op.isConstant() || op.regClass().type() == RegType::sgpr,
396                            "Wrong Operand type for scalar vector", instr.get());
397                   }
398                }
399             } else if (instr->opcode == aco_opcode::p_extract_vector) {
400                check((instr->operands[0].isTemp()) && instr->operands[1].isConstant(),
401                      "Wrong Operand types", instr.get());
402                check((instr->operands[1].constantValue() + 1) * instr->definitions[0].bytes() <=
403                         instr->operands[0].bytes(),
404                      "Index out of range", instr.get());
405                check(instr->definitions[0].getTemp().type() == RegType::vgpr ||
406                         instr->operands[0].regClass().type() == RegType::sgpr,
407                      "Cannot extract SGPR value from VGPR vector", instr.get());
408                check(program->chip_class >= GFX9 ||
409                         !instr->definitions[0].regClass().is_subdword() ||
410                         instr->operands[0].regClass().type() == RegType::vgpr,
411                      "Cannot extract subdword from SGPR before GFX9+", instr.get());
412             } else if (instr->opcode == aco_opcode::p_split_vector) {
413                check(instr->operands[0].isTemp(), "Operand must be a temporary", instr.get());
414                unsigned size = 0;
415                for (const Definition& def : instr->definitions) {
416                   size += def.bytes();
417                }
418                check(size == instr->operands[0].bytes(),
419                      "Operand size does not match definition sizes", instr.get());
420                if (instr->operands[0].getTemp().type() == RegType::vgpr) {
421                   for (const Definition& def : instr->definitions)
422                      check(def.regClass().type() == RegType::vgpr,
423                            "Wrong Definition type for VGPR split_vector", instr.get());
424                } else {
425                   for (const Definition& def : instr->definitions)
426                      check(program->chip_class >= GFX9 || !def.regClass().is_subdword(),
427                            "Cannot split SGPR into subdword VGPRs before GFX9+", instr.get());
428                }
429             } else if (instr->opcode == aco_opcode::p_parallelcopy) {
430                check(instr->definitions.size() == instr->operands.size(),
431                      "Number of Operands does not match number of Definitions", instr.get());
432                for (unsigned i = 0; i < instr->operands.size(); i++) {
433                   check(instr->definitions[i].bytes() == instr->operands[i].bytes(),
434                         "Operand and Definition size must match", instr.get());
435                   if (instr->operands[i].isTemp()) {
436                      check((instr->definitions[i].getTemp().type() ==
437                             instr->operands[i].regClass().type()) ||
438                               (instr->definitions[i].getTemp().type() == RegType::vgpr &&
439                                instr->operands[i].regClass().type() == RegType::sgpr),
440                            "Operand and Definition types do not match", instr.get());
441                      check(instr->definitions[i].regClass().is_linear_vgpr() ==
442                               instr->operands[i].regClass().is_linear_vgpr(),
443                            "Operand and Definition types do not match", instr.get());
444                   } else {
445                      check(!instr->definitions[i].regClass().is_linear_vgpr(),
446                            "Can only copy linear VGPRs into linear VGPRs, not constant/undef",
447                            instr.get());
448                   }
449                }
450             } else if (instr->opcode == aco_opcode::p_phi) {
451                check(instr->operands.size() == block.logical_preds.size(),
452                      "Number of Operands does not match number of predecessors", instr.get());
453                check(instr->definitions[0].getTemp().type() == RegType::vgpr,
454                      "Logical Phi Definition must be vgpr", instr.get());
455                for (const Operand& op : instr->operands)
456                   check(instr->definitions[0].size() == op.size(),
457                         "Operand sizes must match Definition size", instr.get());
458             } else if (instr->opcode == aco_opcode::p_linear_phi) {
459                for (const Operand& op : instr->operands) {
460                   check(!op.isTemp() || op.getTemp().is_linear(), "Wrong Operand type",
461                         instr.get());
462                   check(instr->definitions[0].size() == op.size(),
463                         "Operand sizes must match Definition size", instr.get());
464                }
465                check(instr->operands.size() == block.linear_preds.size(),
466                      "Number of Operands does not match number of predecessors", instr.get());
467             } else if (instr->opcode == aco_opcode::p_extract ||
468                        instr->opcode == aco_opcode::p_insert) {
469                check(instr->operands[0].isTemp(), "Data operand must be temporary", instr.get());
470                check(instr->operands[1].isConstant(), "Index must be constant", instr.get());
471                if (instr->opcode == aco_opcode::p_extract)
472                   check(instr->operands[3].isConstant(), "Sign-extend flag must be constant",
473                         instr.get());
474 
475                check(instr->definitions[0].getTemp().type() != RegType::sgpr ||
476                         instr->operands[0].getTemp().type() == RegType::sgpr,
477                      "Can't extract/insert VGPR to SGPR", instr.get());
478 
479                if (instr->opcode == aco_opcode::p_insert)
480                   check(instr->operands[0].bytes() == instr->definitions[0].bytes(),
481                         "Sizes of p_insert data operand and definition must match", instr.get());
482 
483                if (instr->definitions[0].getTemp().type() == RegType::sgpr)
484                   check(instr->definitions.size() >= 2 && instr->definitions[1].isFixed() &&
485                            instr->definitions[1].physReg() == scc,
486                         "SGPR extract/insert needs an SCC definition", instr.get());
487 
488                unsigned data_bits = instr->operands[0].getTemp().bytes() * 8u;
489                unsigned op_bits = instr->operands[2].constantValue();
490 
491                if (instr->opcode == aco_opcode::p_insert) {
492                   check(op_bits == 8 || op_bits == 16, "Size must be 8 or 16", instr.get());
493                   check(op_bits < data_bits, "Size must be smaller than source", instr.get());
494                } else if (instr->opcode == aco_opcode::p_extract) {
495                   check(op_bits == 8 || op_bits == 16 || op_bits == 32,
496                         "Size must be 8 or 16 or 32", instr.get());
497                   check(data_bits >= op_bits, "Can't extract more bits than what the data has.",
498                         instr.get());
499                }
500 
501                unsigned comp = data_bits / MAX2(op_bits, 1);
502                check(instr->operands[1].constantValue() < comp, "Index must be in-bounds",
503                      instr.get());
504             }
505             break;
506          }
507          case Format::PSEUDO_REDUCTION: {
508             for (const Operand& op : instr->operands)
509                check(op.regClass().type() == RegType::vgpr,
510                      "All operands of PSEUDO_REDUCTION instructions must be in VGPRs.",
511                      instr.get());
512 
513             if (instr->opcode == aco_opcode::p_reduce &&
514                 instr->reduction().cluster_size == program->wave_size)
515                check(instr->definitions[0].regClass().type() == RegType::sgpr ||
516                         program->wave_size == 32,
517                      "The result of unclustered reductions must go into an SGPR.", instr.get());
518             else
519                check(instr->definitions[0].regClass().type() == RegType::vgpr,
520                      "The result of scans and clustered reductions must go into a VGPR.",
521                      instr.get());
522 
523             break;
524          }
525          case Format::SMEM: {
526             if (instr->operands.size() >= 1)
527                check((instr->operands[0].isFixed() && !instr->operands[0].isConstant()) ||
528                         (instr->operands[0].isTemp() &&
529                          instr->operands[0].regClass().type() == RegType::sgpr),
530                      "SMEM operands must be sgpr", instr.get());
531             if (instr->operands.size() >= 2)
532                check(instr->operands[1].isConstant() ||
533                         (instr->operands[1].isTemp() &&
534                          instr->operands[1].regClass().type() == RegType::sgpr),
535                      "SMEM offset must be constant or sgpr", instr.get());
536             if (!instr->definitions.empty())
537                check(instr->definitions[0].getTemp().type() == RegType::sgpr,
538                      "SMEM result must be sgpr", instr.get());
539             break;
540          }
541          case Format::MTBUF:
542          case Format::MUBUF: {
543             check(instr->operands.size() > 1, "VMEM instructions must have at least one operand",
544                   instr.get());
545             check(instr->operands[1].hasRegClass() &&
546                      instr->operands[1].regClass().type() == RegType::vgpr,
547                   "VADDR must be in vgpr for VMEM instructions", instr.get());
548             check(
549                instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::sgpr,
550                "VMEM resource constant must be sgpr", instr.get());
551             check(instr->operands.size() < 4 ||
552                      (instr->operands[3].isTemp() &&
553                       instr->operands[3].regClass().type() == RegType::vgpr),
554                   "VMEM write data must be vgpr", instr.get());
555             break;
556          }
557          case Format::MIMG: {
558             check(instr->operands.size() >= 4, "MIMG instructions must have at least 4 operands",
559                   instr.get());
560             check(instr->operands[0].hasRegClass() &&
561                      (instr->operands[0].regClass() == s4 || instr->operands[0].regClass() == s8),
562                   "MIMG operands[0] (resource constant) must be in 4 or 8 SGPRs", instr.get());
563             if (instr->operands[1].hasRegClass())
564                check(instr->operands[1].regClass() == s4,
565                      "MIMG operands[1] (sampler constant) must be 4 SGPRs", instr.get());
566             if (!instr->operands[2].isUndefined()) {
567                bool is_cmpswap = instr->opcode == aco_opcode::image_atomic_cmpswap ||
568                                  instr->opcode == aco_opcode::image_atomic_fcmpswap;
569                check(instr->definitions.empty() ||
570                         (instr->definitions[0].regClass() == instr->operands[2].regClass() ||
571                          is_cmpswap),
572                      "MIMG operands[2] (VDATA) must be the same as definitions[0] for atomics and "
573                      "TFE/LWE loads",
574                      instr.get());
575             }
576             check(instr->operands.size() == 4 || program->chip_class >= GFX10,
577                   "NSA is only supported on GFX10+", instr.get());
578             for (unsigned i = 3; i < instr->operands.size(); i++) {
579                if (instr->operands.size() == 4) {
580                   check(instr->operands[i].hasRegClass() &&
581                            instr->operands[i].regClass().type() == RegType::vgpr,
582                         "MIMG operands[3] (VADDR) must be VGPR", instr.get());
583                } else {
584                   check(instr->operands[i].regClass() == v1, "MIMG VADDR must be v1 if NSA is used",
585                         instr.get());
586                }
587             }
588             check(instr->definitions.empty() ||
589                      (instr->definitions[0].isTemp() &&
590                       instr->definitions[0].regClass().type() == RegType::vgpr),
591                   "MIMG definitions[0] (VDATA) must be VGPR", instr.get());
592             break;
593          }
594          case Format::DS: {
595             for (const Operand& op : instr->operands) {
596                check((op.isTemp() && op.regClass().type() == RegType::vgpr) || op.physReg() == m0,
597                      "Only VGPRs are valid DS instruction operands", instr.get());
598             }
599             if (!instr->definitions.empty())
600                check(instr->definitions[0].getTemp().type() == RegType::vgpr,
601                      "DS instruction must return VGPR", instr.get());
602             break;
603          }
604          case Format::EXP: {
605             for (unsigned i = 0; i < 4; i++)
606                check(instr->operands[i].hasRegClass() &&
607                         instr->operands[i].regClass().type() == RegType::vgpr,
608                      "Only VGPRs are valid Export arguments", instr.get());
609             break;
610          }
611          case Format::FLAT:
612             check(instr->operands[1].isUndefined(), "Flat instructions don't support SADDR",
613                   instr.get());
614             FALLTHROUGH;
615          case Format::GLOBAL:
616          case Format::SCRATCH: {
617             check(
618                instr->operands[0].isTemp() && instr->operands[0].regClass().type() == RegType::vgpr,
619                "FLAT/GLOBAL/SCRATCH address must be vgpr", instr.get());
620             check(instr->operands[1].hasRegClass() &&
621                      instr->operands[1].regClass().type() == RegType::sgpr,
622                   "FLAT/GLOBAL/SCRATCH sgpr address must be undefined or sgpr", instr.get());
623             if (!instr->definitions.empty())
624                check(instr->definitions[0].getTemp().type() == RegType::vgpr,
625                      "FLAT/GLOBAL/SCRATCH result must be vgpr", instr.get());
626             else
627                check(instr->operands[2].regClass().type() == RegType::vgpr,
628                      "FLAT/GLOBAL/SCRATCH data must be vgpr", instr.get());
629             break;
630          }
631          default: break;
632          }
633       }
634    }
635 
636    /* validate CFG */
637    for (unsigned i = 0; i < program->blocks.size(); i++) {
638       Block& block = program->blocks[i];
639       check_block(block.index == i, "block.index must match actual index", &block);
640 
641       /* predecessors/successors should be sorted */
642       for (unsigned j = 0; j + 1 < block.linear_preds.size(); j++)
643          check_block(block.linear_preds[j] < block.linear_preds[j + 1],
644                      "linear predecessors must be sorted", &block);
645       for (unsigned j = 0; j + 1 < block.logical_preds.size(); j++)
646          check_block(block.logical_preds[j] < block.logical_preds[j + 1],
647                      "logical predecessors must be sorted", &block);
648       for (unsigned j = 0; j + 1 < block.linear_succs.size(); j++)
649          check_block(block.linear_succs[j] < block.linear_succs[j + 1],
650                      "linear successors must be sorted", &block);
651       for (unsigned j = 0; j + 1 < block.logical_succs.size(); j++)
652          check_block(block.logical_succs[j] < block.logical_succs[j + 1],
653                      "logical successors must be sorted", &block);
654 
655       /* critical edges are not allowed */
656       if (block.linear_preds.size() > 1) {
657          for (unsigned pred : block.linear_preds)
658             check_block(program->blocks[pred].linear_succs.size() == 1,
659                         "linear critical edges are not allowed", &program->blocks[pred]);
660          for (unsigned pred : block.logical_preds)
661             check_block(program->blocks[pred].logical_succs.size() == 1,
662                         "logical critical edges are not allowed", &program->blocks[pred]);
663       }
664    }
665 
666    return is_valid;
667 }
668 
669 /* RA validation */
670 namespace {
671 
672 struct Location {
Locationaco::__anond617b2090311::Location673    Location() : block(NULL), instr(NULL) {}
674 
675    Block* block;
676    Instruction* instr; // NULL if it's the block's live-in
677 };
678 
679 struct Assignment {
680    Location defloc;
681    Location firstloc;
682    PhysReg reg;
683 };
684 
685 bool
ra_fail(Program * program,Location loc,Location loc2,const char * fmt,...)686 ra_fail(Program* program, Location loc, Location loc2, const char* fmt, ...)
687 {
688    va_list args;
689    va_start(args, fmt);
690    char msg[1024];
691    vsprintf(msg, fmt, args);
692    va_end(args);
693 
694    char* out;
695    size_t outsize;
696    struct u_memstream mem;
697    u_memstream_open(&mem, &out, &outsize);
698    FILE* const memf = u_memstream_get(&mem);
699 
700    fprintf(memf, "RA error found at instruction in BB%d:\n", loc.block->index);
701    if (loc.instr) {
702       aco_print_instr(loc.instr, memf);
703       fprintf(memf, "\n%s", msg);
704    } else {
705       fprintf(memf, "%s", msg);
706    }
707    if (loc2.block) {
708       fprintf(memf, " in BB%d:\n", loc2.block->index);
709       aco_print_instr(loc2.instr, memf);
710    }
711    fprintf(memf, "\n\n");
712    u_memstream_close(&mem);
713 
714    aco_err(program, "%s", out);
715    free(out);
716 
717    return true;
718 }
719 
720 bool
validate_subdword_operand(chip_class chip,const aco_ptr<Instruction> & instr,unsigned index)721 validate_subdword_operand(chip_class chip, const aco_ptr<Instruction>& instr, unsigned index)
722 {
723    Operand op = instr->operands[index];
724    unsigned byte = op.physReg().byte();
725 
726    if (instr->opcode == aco_opcode::p_as_uniform)
727       return byte == 0;
728    if (instr->isPseudo() && chip >= GFX8)
729       return true;
730    if (instr->isSDWA())
731       return byte + instr->sdwa().sel[index].offset() + instr->sdwa().sel[index].size() <= 4 &&
732              byte % instr->sdwa().sel[index].size() == 0;
733    if (instr->isVOP3P())
734       return ((instr->vop3p().opsel_lo >> index) & 1) == (byte >> 1) &&
735              ((instr->vop3p().opsel_hi >> index) & 1) == (byte >> 1);
736    if (byte == 2 && can_use_opsel(chip, instr->opcode, index, 1))
737       return true;
738 
739    switch (instr->opcode) {
740    case aco_opcode::v_cvt_f32_ubyte1:
741       if (byte == 1)
742          return true;
743       break;
744    case aco_opcode::v_cvt_f32_ubyte2:
745       if (byte == 2)
746          return true;
747       break;
748    case aco_opcode::v_cvt_f32_ubyte3:
749       if (byte == 3)
750          return true;
751       break;
752    case aco_opcode::ds_write_b8_d16_hi:
753    case aco_opcode::ds_write_b16_d16_hi:
754       if (byte == 2 && index == 1)
755          return true;
756       break;
757    case aco_opcode::buffer_store_byte_d16_hi:
758    case aco_opcode::buffer_store_short_d16_hi:
759       if (byte == 2 && index == 3)
760          return true;
761       break;
762    case aco_opcode::flat_store_byte_d16_hi:
763    case aco_opcode::flat_store_short_d16_hi:
764    case aco_opcode::scratch_store_byte_d16_hi:
765    case aco_opcode::scratch_store_short_d16_hi:
766    case aco_opcode::global_store_byte_d16_hi:
767    case aco_opcode::global_store_short_d16_hi:
768       if (byte == 2 && index == 2)
769          return true;
770       break;
771    default: break;
772    }
773 
774    return byte == 0;
775 }
776 
777 bool
validate_subdword_definition(chip_class chip,const aco_ptr<Instruction> & instr)778 validate_subdword_definition(chip_class chip, const aco_ptr<Instruction>& instr)
779 {
780    Definition def = instr->definitions[0];
781    unsigned byte = def.physReg().byte();
782 
783    if (instr->isPseudo() && chip >= GFX8)
784       return true;
785    if (instr->isSDWA())
786       return byte + instr->sdwa().dst_sel.offset() + instr->sdwa().dst_sel.size() <= 4 &&
787              byte % instr->sdwa().dst_sel.size() == 0;
788    if (byte == 2 && can_use_opsel(chip, instr->opcode, -1, 1))
789       return true;
790 
791    switch (instr->opcode) {
792    case aco_opcode::buffer_load_ubyte_d16_hi:
793    case aco_opcode::buffer_load_short_d16_hi:
794    case aco_opcode::flat_load_ubyte_d16_hi:
795    case aco_opcode::flat_load_short_d16_hi:
796    case aco_opcode::scratch_load_ubyte_d16_hi:
797    case aco_opcode::scratch_load_short_d16_hi:
798    case aco_opcode::global_load_ubyte_d16_hi:
799    case aco_opcode::global_load_short_d16_hi:
800    case aco_opcode::ds_read_u8_d16_hi:
801    case aco_opcode::ds_read_u16_d16_hi: return byte == 2;
802    default: break;
803    }
804 
805    return byte == 0;
806 }
807 
808 unsigned
get_subdword_bytes_written(Program * program,const aco_ptr<Instruction> & instr,unsigned index)809 get_subdword_bytes_written(Program* program, const aco_ptr<Instruction>& instr, unsigned index)
810 {
811    chip_class chip = program->chip_class;
812    Definition def = instr->definitions[index];
813 
814    if (instr->isPseudo())
815       return chip >= GFX8 ? def.bytes() : def.size() * 4u;
816    if (instr->isVALU()) {
817       assert(def.bytes() <= 2);
818       if (instr->isSDWA())
819          return instr->sdwa().dst_sel.size();
820 
821       if (instr_is_16bit(chip, instr->opcode))
822          return 2;
823 
824       return 4;
825    }
826 
827    switch (instr->opcode) {
828    case aco_opcode::buffer_load_ubyte_d16:
829    case aco_opcode::buffer_load_short_d16:
830    case aco_opcode::flat_load_ubyte_d16:
831    case aco_opcode::flat_load_short_d16:
832    case aco_opcode::scratch_load_ubyte_d16:
833    case aco_opcode::scratch_load_short_d16:
834    case aco_opcode::global_load_ubyte_d16:
835    case aco_opcode::global_load_short_d16:
836    case aco_opcode::ds_read_u8_d16:
837    case aco_opcode::ds_read_u16_d16:
838    case aco_opcode::buffer_load_ubyte_d16_hi:
839    case aco_opcode::buffer_load_short_d16_hi:
840    case aco_opcode::flat_load_ubyte_d16_hi:
841    case aco_opcode::flat_load_short_d16_hi:
842    case aco_opcode::scratch_load_ubyte_d16_hi:
843    case aco_opcode::scratch_load_short_d16_hi:
844    case aco_opcode::global_load_ubyte_d16_hi:
845    case aco_opcode::global_load_short_d16_hi:
846    case aco_opcode::ds_read_u8_d16_hi:
847    case aco_opcode::ds_read_u16_d16_hi: return program->dev.sram_ecc_enabled ? 4 : 2;
848    default: return def.size() * 4;
849    }
850 }
851 
852 } /* end namespace */
853 
854 bool
validate_ra(Program * program)855 validate_ra(Program* program)
856 {
857    if (!(debug_flags & DEBUG_VALIDATE_RA))
858       return false;
859 
860    bool err = false;
861    aco::live live_vars = aco::live_var_analysis(program);
862    std::vector<std::vector<Temp>> phi_sgpr_ops(program->blocks.size());
863    uint16_t sgpr_limit = get_addr_sgpr_from_waves(program, program->num_waves);
864 
865    std::map<unsigned, Assignment> assignments;
866    for (Block& block : program->blocks) {
867       Location loc;
868       loc.block = &block;
869       for (aco_ptr<Instruction>& instr : block.instructions) {
870          if (instr->opcode == aco_opcode::p_phi) {
871             for (unsigned i = 0; i < instr->operands.size(); i++) {
872                if (instr->operands[i].isTemp() &&
873                    instr->operands[i].getTemp().type() == RegType::sgpr &&
874                    instr->operands[i].isFirstKill())
875                   phi_sgpr_ops[block.logical_preds[i]].emplace_back(instr->operands[i].getTemp());
876             }
877          }
878 
879          loc.instr = instr.get();
880          for (unsigned i = 0; i < instr->operands.size(); i++) {
881             Operand& op = instr->operands[i];
882             if (!op.isTemp())
883                continue;
884             if (!op.isFixed())
885                err |= ra_fail(program, loc, Location(), "Operand %d is not assigned a register", i);
886             if (assignments.count(op.tempId()) && assignments[op.tempId()].reg != op.physReg())
887                err |=
888                   ra_fail(program, loc, assignments.at(op.tempId()).firstloc,
889                           "Operand %d has an inconsistent register assignment with instruction", i);
890             if ((op.getTemp().type() == RegType::vgpr &&
891                  op.physReg().reg_b + op.bytes() > (256 + program->config->num_vgprs) * 4) ||
892                 (op.getTemp().type() == RegType::sgpr &&
893                  op.physReg() + op.size() > program->config->num_sgprs &&
894                  op.physReg() < sgpr_limit))
895                err |= ra_fail(program, loc, assignments.at(op.tempId()).firstloc,
896                               "Operand %d has an out-of-bounds register assignment", i);
897             if (op.physReg() == vcc && !program->needs_vcc)
898                err |= ra_fail(program, loc, Location(),
899                               "Operand %d fixed to vcc but needs_vcc=false", i);
900             if (op.regClass().is_subdword() &&
901                 !validate_subdword_operand(program->chip_class, instr, i))
902                err |= ra_fail(program, loc, Location(), "Operand %d not aligned correctly", i);
903             if (!assignments[op.tempId()].firstloc.block)
904                assignments[op.tempId()].firstloc = loc;
905             if (!assignments[op.tempId()].defloc.block)
906                assignments[op.tempId()].reg = op.physReg();
907          }
908 
909          for (unsigned i = 0; i < instr->definitions.size(); i++) {
910             Definition& def = instr->definitions[i];
911             if (!def.isTemp())
912                continue;
913             if (!def.isFixed())
914                err |=
915                   ra_fail(program, loc, Location(), "Definition %d is not assigned a register", i);
916             if (assignments[def.tempId()].defloc.block)
917                err |= ra_fail(program, loc, assignments.at(def.tempId()).defloc,
918                               "Temporary %%%d also defined by instruction", def.tempId());
919             if ((def.getTemp().type() == RegType::vgpr &&
920                  def.physReg().reg_b + def.bytes() > (256 + program->config->num_vgprs) * 4) ||
921                 (def.getTemp().type() == RegType::sgpr &&
922                  def.physReg() + def.size() > program->config->num_sgprs &&
923                  def.physReg() < sgpr_limit))
924                err |= ra_fail(program, loc, assignments.at(def.tempId()).firstloc,
925                               "Definition %d has an out-of-bounds register assignment", i);
926             if (def.physReg() == vcc && !program->needs_vcc)
927                err |= ra_fail(program, loc, Location(),
928                               "Definition %d fixed to vcc but needs_vcc=false", i);
929             if (def.regClass().is_subdword() &&
930                 !validate_subdword_definition(program->chip_class, instr))
931                err |= ra_fail(program, loc, Location(), "Definition %d not aligned correctly", i);
932             if (!assignments[def.tempId()].firstloc.block)
933                assignments[def.tempId()].firstloc = loc;
934             assignments[def.tempId()].defloc = loc;
935             assignments[def.tempId()].reg = def.physReg();
936          }
937       }
938    }
939 
940    for (Block& block : program->blocks) {
941       Location loc;
942       loc.block = &block;
943 
944       std::array<unsigned, 2048> regs; /* register file in bytes */
945       regs.fill(0);
946 
947       std::set<Temp> live;
948       for (unsigned id : live_vars.live_out[block.index])
949          live.insert(Temp(id, program->temp_rc[id]));
950       /* remove killed p_phi sgpr operands */
951       for (Temp tmp : phi_sgpr_ops[block.index])
952          live.erase(tmp);
953 
954       /* check live out */
955       for (Temp tmp : live) {
956          PhysReg reg = assignments.at(tmp.id()).reg;
957          for (unsigned i = 0; i < tmp.bytes(); i++) {
958             if (regs[reg.reg_b + i]) {
959                err |= ra_fail(program, loc, Location(),
960                               "Assignment of element %d of %%%d already taken by %%%d in live-out",
961                               i, tmp.id(), regs[reg.reg_b + i]);
962             }
963             regs[reg.reg_b + i] = tmp.id();
964          }
965       }
966       regs.fill(0);
967 
968       for (auto it = block.instructions.rbegin(); it != block.instructions.rend(); ++it) {
969          aco_ptr<Instruction>& instr = *it;
970 
971          /* check killed p_phi sgpr operands */
972          if (instr->opcode == aco_opcode::p_logical_end) {
973             for (Temp tmp : phi_sgpr_ops[block.index]) {
974                PhysReg reg = assignments.at(tmp.id()).reg;
975                for (unsigned i = 0; i < tmp.bytes(); i++) {
976                   if (regs[reg.reg_b + i])
977                      err |= ra_fail(
978                         program, loc, Location(),
979                         "Assignment of element %d of %%%d already taken by %%%d in live-out", i,
980                         tmp.id(), regs[reg.reg_b + i]);
981                }
982                live.emplace(tmp);
983             }
984          }
985 
986          for (const Definition& def : instr->definitions) {
987             if (!def.isTemp())
988                continue;
989             live.erase(def.getTemp());
990          }
991 
992          /* don't count phi operands as live-in, since they are actually
993           * killed when they are copied at the predecessor */
994          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
995             for (const Operand& op : instr->operands) {
996                if (!op.isTemp())
997                   continue;
998                live.insert(op.getTemp());
999             }
1000          }
1001       }
1002 
1003       for (Temp tmp : live) {
1004          PhysReg reg = assignments.at(tmp.id()).reg;
1005          for (unsigned i = 0; i < tmp.bytes(); i++)
1006             regs[reg.reg_b + i] = tmp.id();
1007       }
1008 
1009       for (aco_ptr<Instruction>& instr : block.instructions) {
1010          loc.instr = instr.get();
1011 
1012          /* remove killed p_phi operands from regs */
1013          if (instr->opcode == aco_opcode::p_logical_end) {
1014             for (Temp tmp : phi_sgpr_ops[block.index]) {
1015                PhysReg reg = assignments.at(tmp.id()).reg;
1016                for (unsigned i = 0; i < tmp.bytes(); i++)
1017                   regs[reg.reg_b + i] = 0;
1018             }
1019          }
1020 
1021          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1022             for (const Operand& op : instr->operands) {
1023                if (!op.isTemp())
1024                   continue;
1025                if (op.isFirstKillBeforeDef()) {
1026                   for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1027                      regs[op.physReg().reg_b + j] = 0;
1028                }
1029             }
1030          }
1031 
1032          for (unsigned i = 0; i < instr->definitions.size(); i++) {
1033             Definition& def = instr->definitions[i];
1034             if (!def.isTemp())
1035                continue;
1036             Temp tmp = def.getTemp();
1037             PhysReg reg = assignments.at(tmp.id()).reg;
1038             for (unsigned j = 0; j < tmp.bytes(); j++) {
1039                if (regs[reg.reg_b + j])
1040                   err |= ra_fail(
1041                      program, loc, assignments.at(regs[reg.reg_b + j]).defloc,
1042                      "Assignment of element %d of %%%d already taken by %%%d from instruction", i,
1043                      tmp.id(), regs[reg.reg_b + j]);
1044                regs[reg.reg_b + j] = tmp.id();
1045             }
1046             if (def.regClass().is_subdword() && def.bytes() < 4) {
1047                unsigned written = get_subdword_bytes_written(program, instr, i);
1048                /* If written=4, the instruction still might write the upper half. In that case, it's
1049                 * the lower half that isn't preserved */
1050                for (unsigned j = reg.byte() & ~(written - 1); j < written; j++) {
1051                   unsigned written_reg = reg.reg() * 4u + j;
1052                   if (regs[written_reg] && regs[written_reg] != def.tempId())
1053                      err |= ra_fail(program, loc, assignments.at(regs[written_reg]).defloc,
1054                                     "Assignment of element %d of %%%d overwrites the full register "
1055                                     "taken by %%%d from instruction",
1056                                     i, tmp.id(), regs[written_reg]);
1057                }
1058             }
1059          }
1060 
1061          for (const Definition& def : instr->definitions) {
1062             if (!def.isTemp())
1063                continue;
1064             if (def.isKill()) {
1065                for (unsigned j = 0; j < def.getTemp().bytes(); j++)
1066                   regs[def.physReg().reg_b + j] = 0;
1067             }
1068          }
1069 
1070          if (instr->opcode != aco_opcode::p_phi && instr->opcode != aco_opcode::p_linear_phi) {
1071             for (const Operand& op : instr->operands) {
1072                if (!op.isTemp())
1073                   continue;
1074                if (op.isLateKill() && op.isFirstKill()) {
1075                   for (unsigned j = 0; j < op.getTemp().bytes(); j++)
1076                      regs[op.physReg().reg_b + j] = 0;
1077                }
1078             }
1079          }
1080       }
1081    }
1082 
1083    return err;
1084 }
1085 } // namespace aco
1086