1 /*
2 * Copyright 2017 Advanced Micro Devices, Inc.
3 * All Rights Reserved.
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * on the rights to use, copy, modify, merge, publish, distribute, sub
9 * license, and/or sell copies of the Software, and to permit persons to whom
10 * the Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
20 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
21 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
22 * USE OR OTHER DEALINGS IN THE SOFTWARE.
23 */
24
25 #include "ac_nir_to_llvm.h"
26 #include "ac_nir.h"
27 #include "compiler/nir/nir.h"
28 #include "compiler/nir/nir_builder.h"
29 #include "compiler/nir/nir_deref.h"
30 #include "compiler/nir_types.h"
31 #include "si_pipe.h"
32 #include "si_shader_internal.h"
33 #include "tgsi/tgsi_from_mesa.h"
34
get_texture_src(nir_tex_instr * instr,nir_tex_src_type type)35 static const nir_src *get_texture_src(nir_tex_instr *instr, nir_tex_src_type type)
36 {
37 for (unsigned i = 0; i < instr->num_srcs; i++) {
38 if (instr->src[i].src_type == type)
39 return &instr->src[i].src;
40 }
41 return NULL;
42 }
43
scan_io_usage(struct si_shader_info * info,nir_intrinsic_instr * intr,bool is_input)44 static void scan_io_usage(struct si_shader_info *info, nir_intrinsic_instr *intr,
45 bool is_input)
46 {
47 unsigned interp = INTERP_MODE_FLAT; /* load_input uses flat shading */
48
49 if (intr->intrinsic == nir_intrinsic_load_interpolated_input) {
50 nir_intrinsic_instr *baryc = nir_instr_as_intrinsic(intr->src[0].ssa->parent_instr);
51
52 if (baryc) {
53 if (nir_intrinsic_infos[baryc->intrinsic].index_map[NIR_INTRINSIC_INTERP_MODE] > 0)
54 interp = nir_intrinsic_interp_mode(baryc);
55 else
56 unreachable("unknown barycentric intrinsic");
57 } else {
58 unreachable("unknown barycentric expression");
59 }
60 }
61
62 unsigned mask, bit_size;
63 bool is_output_load;
64
65 if (nir_intrinsic_has_write_mask(intr)) {
66 mask = nir_intrinsic_write_mask(intr); /* store */
67 bit_size = nir_src_bit_size(intr->src[0]);
68 is_output_load = false;
69 } else {
70 mask = nir_ssa_def_components_read(&intr->dest.ssa); /* load */
71 bit_size = intr->dest.ssa.bit_size;
72 is_output_load = !is_input;
73 }
74 assert(bit_size != 64 && !(mask & ~0xf) && "64-bit IO should have been lowered");
75
76 /* Convert the 16-bit component mask to a 32-bit component mask except for VS inputs
77 * where the mask is untyped.
78 */
79 if (bit_size == 16 && !is_input) {
80 unsigned new_mask = 0;
81 for (unsigned i = 0; i < 4; i++) {
82 if (mask & (1 << i))
83 new_mask |= 0x1 << (i / 2);
84 }
85 mask = new_mask;
86 }
87
88 mask <<= nir_intrinsic_component(intr);
89
90 nir_src offset = *nir_get_io_offset_src(intr);
91 bool indirect = !nir_src_is_const(offset);
92 if (!indirect)
93 assert(nir_src_as_uint(offset) == 0);
94
95 unsigned semantic = 0;
96 /* VS doesn't have semantics. */
97 if (info->stage != MESA_SHADER_VERTEX || !is_input)
98 semantic = nir_intrinsic_io_semantics(intr).location;
99
100 if (info->stage == MESA_SHADER_FRAGMENT && !is_input) {
101 /* Never use FRAG_RESULT_COLOR directly. */
102 if (semantic == FRAG_RESULT_COLOR)
103 semantic = FRAG_RESULT_DATA0;
104 semantic += nir_intrinsic_io_semantics(intr).dual_source_blend_index;
105 }
106
107 unsigned driver_location = nir_intrinsic_base(intr);
108 unsigned num_slots = indirect ? nir_intrinsic_io_semantics(intr).num_slots : 1;
109
110 if (is_input) {
111 assert(driver_location + num_slots <= ARRAY_SIZE(info->input));
112
113 for (unsigned i = 0; i < num_slots; i++) {
114 unsigned loc = driver_location + i;
115
116 info->input[loc].semantic = semantic + i;
117
118 if (semantic == VARYING_SLOT_PRIMITIVE_ID)
119 info->input[loc].interpolate = INTERP_MODE_FLAT;
120 else
121 info->input[loc].interpolate = interp;
122
123 if (mask) {
124 info->input[loc].usage_mask |= mask;
125 if (bit_size == 16) {
126 if (nir_intrinsic_io_semantics(intr).high_16bits)
127 info->input[loc].fp16_lo_hi_valid |= 0x2;
128 else
129 info->input[loc].fp16_lo_hi_valid |= 0x1;
130 }
131 info->num_inputs = MAX2(info->num_inputs, loc + 1);
132 }
133 }
134 } else {
135 /* Outputs. */
136 assert(driver_location + num_slots <= ARRAY_SIZE(info->output_usagemask));
137
138 for (unsigned i = 0; i < num_slots; i++) {
139 unsigned loc = driver_location + i;
140
141 info->output_semantic[loc] = semantic + i;
142
143 if (is_output_load) {
144 /* Output loads have only a few things that we need to track. */
145 info->output_readmask[loc] |= mask;
146 } else if (mask) {
147 /* Output stores. */
148 unsigned gs_streams = (uint32_t)nir_intrinsic_io_semantics(intr).gs_streams <<
149 (nir_intrinsic_component(intr) * 2);
150 unsigned new_mask = mask & ~info->output_usagemask[loc];
151
152 for (unsigned i = 0; i < 4; i++) {
153 unsigned stream = (gs_streams >> (i * 2)) & 0x3;
154
155 if (new_mask & (1 << i)) {
156 info->output_streams[loc] |= stream << (i * 2);
157 info->num_stream_output_components[stream]++;
158 }
159 }
160
161 if (nir_intrinsic_has_src_type(intr))
162 info->output_type[loc] = nir_intrinsic_src_type(intr);
163 else if (nir_intrinsic_has_dest_type(intr))
164 info->output_type[loc] = nir_intrinsic_dest_type(intr);
165 else
166 info->output_type[loc] = nir_type_float32;
167
168 info->output_usagemask[loc] |= mask;
169 info->num_outputs = MAX2(info->num_outputs, loc + 1);
170
171 if (info->stage == MESA_SHADER_FRAGMENT &&
172 semantic >= FRAG_RESULT_DATA0 && semantic <= FRAG_RESULT_DATA7) {
173 unsigned index = semantic - FRAG_RESULT_DATA0;
174
175 if (nir_intrinsic_src_type(intr) == nir_type_float16)
176 info->output_color_types |= SI_TYPE_FLOAT16 << (index * 2);
177 else if (nir_intrinsic_src_type(intr) == nir_type_int16)
178 info->output_color_types |= SI_TYPE_INT16 << (index * 2);
179 else if (nir_intrinsic_src_type(intr) == nir_type_uint16)
180 info->output_color_types |= SI_TYPE_UINT16 << (index * 2);
181 }
182 }
183 }
184 }
185 }
186
is_bindless_handle_indirect(nir_instr * src)187 static bool is_bindless_handle_indirect(nir_instr *src)
188 {
189 /* Check if the bindless handle comes from indirect load_ubo. */
190 if (src->type == nir_instr_type_intrinsic &&
191 nir_instr_as_intrinsic(src)->intrinsic == nir_intrinsic_load_ubo) {
192 if (!nir_src_is_const(nir_instr_as_intrinsic(src)->src[0]))
193 return true;
194 } else {
195 /* Some other instruction. Return the worst-case result. */
196 return true;
197 }
198 return false;
199 }
200
scan_instruction(const struct nir_shader * nir,struct si_shader_info * info,nir_instr * instr)201 static void scan_instruction(const struct nir_shader *nir, struct si_shader_info *info,
202 nir_instr *instr)
203 {
204 if (instr->type == nir_instr_type_tex) {
205 nir_tex_instr *tex = nir_instr_as_tex(instr);
206 const nir_src *handle = get_texture_src(tex, nir_tex_src_texture_handle);
207
208 /* Gather the types of used VMEM instructions that return something. */
209 switch (tex->op) {
210 case nir_texop_tex:
211 case nir_texop_txb:
212 case nir_texop_txl:
213 case nir_texop_txd:
214 case nir_texop_lod:
215 case nir_texop_tg4:
216 info->uses_vmem_return_type_sampler_or_bvh = true;
217 break;
218 default:
219 info->uses_vmem_return_type_other = true;
220 break;
221 }
222
223 if (handle) {
224 info->uses_bindless_samplers = true;
225
226 if (is_bindless_handle_indirect(handle->ssa->parent_instr))
227 info->uses_indirect_descriptor = true;
228 } else {
229 const nir_src *deref = get_texture_src(tex, nir_tex_src_texture_deref);
230
231 if (nir_deref_instr_has_indirect(nir_src_as_deref(*deref)))
232 info->uses_indirect_descriptor = true;
233 }
234 } else if (instr->type == nir_instr_type_intrinsic) {
235 nir_intrinsic_instr *intr = nir_instr_as_intrinsic(instr);
236 const char *intr_name = nir_intrinsic_infos[intr->intrinsic].name;
237 bool is_ssbo = strstr(intr_name, "ssbo");
238 bool is_image = strstr(intr_name, "image_deref");
239 bool is_bindless_image = strstr(intr_name, "bindless_image");
240
241 /* Gather the types of used VMEM instructions that return something. */
242 if (nir_intrinsic_infos[intr->intrinsic].has_dest) {
243 switch (intr->intrinsic) {
244 case nir_intrinsic_load_ubo:
245 if (!nir_src_is_const(intr->src[1]))
246 info->uses_vmem_return_type_other = true;
247 break;
248 case nir_intrinsic_load_constant:
249 info->uses_vmem_return_type_other = true;
250 break;
251
252 case nir_intrinsic_load_barycentric_at_sample: /* This loads sample positions. */
253 case nir_intrinsic_load_tess_level_outer: /* TES input read from memory */
254 case nir_intrinsic_load_tess_level_inner: /* TES input read from memory */
255 info->uses_vmem_return_type_other = true;
256 break;
257
258 case nir_intrinsic_load_input:
259 case nir_intrinsic_load_input_vertex:
260 case nir_intrinsic_load_per_vertex_input:
261 if (nir->info.stage == MESA_SHADER_VERTEX ||
262 nir->info.stage == MESA_SHADER_TESS_EVAL)
263 info->uses_vmem_return_type_other = true;
264 break;
265
266 default:
267 if (is_image ||
268 is_bindless_image ||
269 is_ssbo ||
270 strstr(intr_name, "global") ||
271 strstr(intr_name, "scratch"))
272 info->uses_vmem_return_type_other = true;
273 break;
274 }
275 }
276
277 if (is_bindless_image)
278 info->uses_bindless_images = true;
279
280 if (strstr(intr_name, "image_atomic") ||
281 strstr(intr_name, "image_store") ||
282 strstr(intr_name, "image_deref_atomic") ||
283 strstr(intr_name, "image_deref_store") ||
284 strstr(intr_name, "ssbo_atomic") ||
285 intr->intrinsic == nir_intrinsic_store_ssbo)
286 info->num_memory_stores++;
287
288
289 if (is_image && nir_deref_instr_has_indirect(nir_src_as_deref(intr->src[0])))
290 info->uses_indirect_descriptor = true;
291
292 if (is_bindless_image && is_bindless_handle_indirect(intr->src[0].ssa->parent_instr))
293 info->uses_indirect_descriptor = true;
294
295 if (intr->intrinsic != nir_intrinsic_store_ssbo && is_ssbo &&
296 !nir_src_is_const(intr->src[0]))
297 info->uses_indirect_descriptor = true;
298
299 switch (intr->intrinsic) {
300 case nir_intrinsic_store_ssbo:
301 if (!nir_src_is_const(intr->src[1]))
302 info->uses_indirect_descriptor = true;
303 break;
304 case nir_intrinsic_load_ubo:
305 if (!nir_src_is_const(intr->src[0]))
306 info->uses_indirect_descriptor = true;
307 break;
308 case nir_intrinsic_load_local_invocation_id:
309 case nir_intrinsic_load_workgroup_id: {
310 unsigned mask = nir_ssa_def_components_read(&intr->dest.ssa);
311 while (mask) {
312 unsigned i = u_bit_scan(&mask);
313
314 if (intr->intrinsic == nir_intrinsic_load_workgroup_id)
315 info->uses_block_id[i] = true;
316 else
317 info->uses_thread_id[i] = true;
318 }
319 break;
320 }
321 case nir_intrinsic_load_color0:
322 case nir_intrinsic_load_color1: {
323 unsigned index = intr->intrinsic == nir_intrinsic_load_color1;
324 uint8_t mask = nir_ssa_def_components_read(&intr->dest.ssa);
325 info->colors_read |= mask << (index * 4);
326
327 switch (info->color_interpolate[index]) {
328 case INTERP_MODE_SMOOTH:
329 if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_SAMPLE)
330 info->uses_persp_sample = true;
331 else if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_CENTROID)
332 info->uses_persp_centroid = true;
333 else if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_CENTER)
334 info->uses_persp_center = true;
335 break;
336 case INTERP_MODE_NOPERSPECTIVE:
337 if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_SAMPLE)
338 info->uses_linear_sample = true;
339 else if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_CENTROID)
340 info->uses_linear_centroid = true;
341 else if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_CENTER)
342 info->uses_linear_center = true;
343 break;
344 case INTERP_MODE_COLOR:
345 /* We don't know the final value. This will be FLAT if flatshading is enabled
346 * in the rasterizer state, otherwise it will be SMOOTH.
347 */
348 info->uses_interp_color = true;
349 if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_SAMPLE)
350 info->uses_persp_sample_color = true;
351 else if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_CENTROID)
352 info->uses_persp_centroid_color = true;
353 else if (info->color_interpolate_loc[index] == TGSI_INTERPOLATE_LOC_CENTER)
354 info->uses_persp_center_color = true;
355 break;
356 }
357 break;
358 }
359 case nir_intrinsic_load_barycentric_at_offset: /* uses center */
360 case nir_intrinsic_load_barycentric_at_sample: /* uses center */
361 if (nir_intrinsic_interp_mode(intr) == INTERP_MODE_FLAT)
362 break;
363
364 if (nir_intrinsic_interp_mode(intr) == INTERP_MODE_NOPERSPECTIVE) {
365 info->uses_linear_center = true;
366 } else {
367 info->uses_persp_center = true;
368 }
369 if (intr->intrinsic == nir_intrinsic_load_barycentric_at_sample)
370 info->uses_interp_at_sample = true;
371 break;
372 case nir_intrinsic_load_input:
373 case nir_intrinsic_load_per_vertex_input:
374 case nir_intrinsic_load_input_vertex:
375 case nir_intrinsic_load_interpolated_input:
376 scan_io_usage(info, intr, true);
377 break;
378 case nir_intrinsic_load_output:
379 case nir_intrinsic_load_per_vertex_output:
380 case nir_intrinsic_store_output:
381 case nir_intrinsic_store_per_vertex_output:
382 scan_io_usage(info, intr, false);
383 break;
384 case nir_intrinsic_load_deref:
385 case nir_intrinsic_store_deref:
386 case nir_intrinsic_interp_deref_at_centroid:
387 case nir_intrinsic_interp_deref_at_sample:
388 case nir_intrinsic_interp_deref_at_offset:
389 unreachable("these opcodes should have been lowered");
390 break;
391 default:
392 break;
393 }
394 }
395 }
396
si_nir_scan_shader(const struct nir_shader * nir,struct si_shader_info * info)397 void si_nir_scan_shader(const struct nir_shader *nir, struct si_shader_info *info)
398 {
399 nir_function *func;
400
401 info->base = nir->info;
402 info->stage = nir->info.stage;
403
404 if (nir->info.stage == MESA_SHADER_TESS_EVAL) {
405 if (info->base.tess.primitive_mode == GL_ISOLINES)
406 info->base.tess.primitive_mode = GL_LINES;
407 }
408
409 if (nir->info.stage == MESA_SHADER_FRAGMENT) {
410 /* post_depth_coverage implies early_fragment_tests */
411 info->base.fs.early_fragment_tests |= info->base.fs.post_depth_coverage;
412
413 info->color_interpolate[0] = nir->info.fs.color0_interp;
414 info->color_interpolate[1] = nir->info.fs.color1_interp;
415 for (unsigned i = 0; i < 2; i++) {
416 if (info->color_interpolate[i] == INTERP_MODE_NONE)
417 info->color_interpolate[i] = INTERP_MODE_COLOR;
418 }
419
420 info->color_interpolate_loc[0] = nir->info.fs.color0_sample ? TGSI_INTERPOLATE_LOC_SAMPLE :
421 nir->info.fs.color0_centroid ? TGSI_INTERPOLATE_LOC_CENTROID :
422 TGSI_INTERPOLATE_LOC_CENTER;
423 info->color_interpolate_loc[1] = nir->info.fs.color1_sample ? TGSI_INTERPOLATE_LOC_SAMPLE :
424 nir->info.fs.color1_centroid ? TGSI_INTERPOLATE_LOC_CENTROID :
425 TGSI_INTERPOLATE_LOC_CENTER;
426 /* Set an invalid value. Will be determined at draw time if needed when the expected
427 * conditions are met.
428 */
429 info->writes_1_if_tex_is_1 = nir->info.writes_memory ? 0 : 0xff;
430 }
431
432 info->constbuf0_num_slots = nir->num_uniforms;
433
434 if (nir->info.stage == MESA_SHADER_TESS_CTRL) {
435 info->tessfactors_are_def_in_all_invocs = ac_are_tessfactors_def_in_all_invocs(nir);
436 }
437
438 info->uses_frontface = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_FRONT_FACE);
439 info->uses_instanceid = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_INSTANCE_ID);
440 info->uses_base_vertex = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BASE_VERTEX);
441 info->uses_base_instance = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BASE_INSTANCE);
442 info->uses_invocationid = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_INVOCATION_ID);
443 info->uses_grid_size = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_NUM_WORKGROUPS);
444 info->uses_subgroup_info = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_LOCAL_INVOCATION_INDEX) ||
445 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_SUBGROUP_ID) ||
446 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_NUM_SUBGROUPS);
447 info->uses_variable_block_size = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_WORKGROUP_SIZE);
448 info->uses_drawid = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_DRAW_ID);
449 info->uses_primid = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_PRIMITIVE_ID) ||
450 nir->info.inputs_read & VARYING_BIT_PRIMITIVE_ID;
451 info->reads_samplemask = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_SAMPLE_MASK_IN);
452 info->reads_tess_factors = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_TESS_LEVEL_INNER) ||
453 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_TESS_LEVEL_OUTER);
454 info->uses_linear_sample = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE);
455 info->uses_linear_centroid = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID);
456 info->uses_linear_center = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL);
457 info->uses_persp_sample = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE);
458 info->uses_persp_centroid = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID);
459 info->uses_persp_center = BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL);
460
461 if (nir->info.stage == MESA_SHADER_FRAGMENT) {
462 info->writes_z = nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_DEPTH);
463 info->writes_stencil = nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_STENCIL);
464 info->writes_samplemask = nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_SAMPLE_MASK);
465
466 info->colors_written = nir->info.outputs_written >> FRAG_RESULT_DATA0;
467 if (nir->info.outputs_written & BITFIELD64_BIT(FRAG_RESULT_COLOR)) {
468 info->color0_writes_all_cbufs = true;
469 info->colors_written |= 0x1;
470 }
471 if (nir->info.fs.color_is_dual_source)
472 info->colors_written |= 0x2;
473 } else {
474 info->writes_primid = nir->info.outputs_written & VARYING_BIT_PRIMITIVE_ID;
475 info->writes_viewport_index = nir->info.outputs_written & VARYING_BIT_VIEWPORT;
476 info->writes_layer = nir->info.outputs_written & VARYING_BIT_LAYER;
477 info->writes_psize = nir->info.outputs_written & VARYING_BIT_PSIZ;
478 info->writes_clipvertex = nir->info.outputs_written & VARYING_BIT_CLIP_VERTEX;
479 info->writes_edgeflag = nir->info.outputs_written & VARYING_BIT_EDGE;
480 info->writes_position = nir->info.outputs_written & VARYING_BIT_POS;
481 }
482
483 func = (struct nir_function *)exec_list_get_head_const(&nir->functions);
484 nir_foreach_block (block, func->impl) {
485 nir_foreach_instr (instr, block)
486 scan_instruction(nir, info, instr);
487 }
488
489 if (info->stage == MESA_SHADER_VERTEX || info->stage == MESA_SHADER_TESS_EVAL) {
490 /* Add the PrimitiveID output, but don't increment num_outputs.
491 * The driver inserts PrimitiveID only when it's used by the pixel shader,
492 * and si_emit_spi_map uses this unconditionally when such a pixel shader is used.
493 */
494 info->output_semantic[info->num_outputs] = VARYING_SLOT_PRIMITIVE_ID;
495 info->output_type[info->num_outputs] = nir_type_uint32;
496 info->output_usagemask[info->num_outputs] = 0x1;
497 }
498
499 if (nir->info.stage == MESA_SHADER_FRAGMENT) {
500 info->allow_flat_shading = !(info->uses_persp_center || info->uses_persp_centroid ||
501 info->uses_persp_sample || info->uses_linear_center ||
502 info->uses_linear_centroid || info->uses_linear_sample ||
503 info->uses_interp_at_sample || nir->info.writes_memory ||
504 nir->info.fs.uses_fbfetch_output ||
505 nir->info.fs.needs_quad_helper_invocations ||
506 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_FRAG_COORD) ||
507 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_POINT_COORD) ||
508 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_SAMPLE_ID) ||
509 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_SAMPLE_POS) ||
510 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_SAMPLE_MASK_IN) ||
511 BITSET_TEST(nir->info.system_values_read, SYSTEM_VALUE_HELPER_INVOCATION));
512
513 /* Add both front and back color inputs. */
514 unsigned num_inputs_with_colors = info->num_inputs;
515 for (unsigned back = 0; back < 2; back++) {
516 for (unsigned i = 0; i < 2; i++) {
517 if ((info->colors_read >> (i * 4)) & 0xf) {
518 unsigned index = num_inputs_with_colors;
519
520 info->input[index].semantic = (back ? VARYING_SLOT_BFC0 : VARYING_SLOT_COL0) + i;
521 info->input[index].interpolate = info->color_interpolate[i];
522 info->input[index].usage_mask = info->colors_read >> (i * 4);
523 num_inputs_with_colors++;
524
525 /* Back-face color don't increment num_inputs. si_emit_spi_map will use
526 * back-face colors conditionally only when they are needed.
527 */
528 if (!back)
529 info->num_inputs = num_inputs_with_colors;
530 }
531 }
532 }
533 }
534
535 /* Trim output read masks based on write masks. */
536 for (unsigned i = 0; i < info->num_outputs; i++)
537 info->output_readmask[i] &= info->output_usagemask[i];
538 }
539
si_alu_to_scalar_filter(const nir_instr * instr,const void * data)540 static bool si_alu_to_scalar_filter(const nir_instr *instr, const void *data)
541 {
542 struct si_screen *sscreen = (struct si_screen *)data;
543
544 if (sscreen->options.fp16 &&
545 instr->type == nir_instr_type_alu) {
546 nir_alu_instr *alu = nir_instr_as_alu(instr);
547
548 if (alu->dest.dest.is_ssa &&
549 alu->dest.dest.ssa.bit_size == 16 &&
550 alu->dest.dest.ssa.num_components == 2)
551 return false;
552 }
553
554 return true;
555 }
556
si_nir_opts(struct si_screen * sscreen,struct nir_shader * nir,bool first)557 void si_nir_opts(struct si_screen *sscreen, struct nir_shader *nir, bool first)
558 {
559 bool progress;
560
561 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
562 NIR_PASS_V(nir, nir_lower_alu_to_scalar, si_alu_to_scalar_filter, sscreen);
563 NIR_PASS_V(nir, nir_lower_phis_to_scalar, false);
564
565 do {
566 progress = false;
567 bool lower_alu_to_scalar = false;
568 bool lower_phis_to_scalar = false;
569
570 if (first) {
571 NIR_PASS(progress, nir, nir_split_array_vars, nir_var_function_temp);
572 NIR_PASS(lower_alu_to_scalar, nir, nir_shrink_vec_array_vars, nir_var_function_temp);
573 NIR_PASS(progress, nir, nir_opt_find_array_copies);
574 }
575 NIR_PASS(progress, nir, nir_opt_copy_prop_vars);
576 NIR_PASS(progress, nir, nir_opt_dead_write_vars);
577
578 NIR_PASS(lower_alu_to_scalar, nir, nir_opt_trivial_continues);
579 /* (Constant) copy propagation is needed for txf with offsets. */
580 NIR_PASS(progress, nir, nir_copy_prop);
581 NIR_PASS(progress, nir, nir_opt_remove_phis);
582 NIR_PASS(progress, nir, nir_opt_dce);
583 NIR_PASS(lower_phis_to_scalar, nir, nir_opt_if, true);
584 NIR_PASS(progress, nir, nir_opt_dead_cf);
585
586 if (lower_alu_to_scalar)
587 NIR_PASS_V(nir, nir_lower_alu_to_scalar, si_alu_to_scalar_filter, sscreen);
588 if (lower_phis_to_scalar)
589 NIR_PASS_V(nir, nir_lower_phis_to_scalar, false);
590 progress |= lower_alu_to_scalar | lower_phis_to_scalar;
591
592 NIR_PASS(progress, nir, nir_opt_cse);
593 NIR_PASS(progress, nir, nir_opt_peephole_select, 8, true, true);
594
595 /* Needed for algebraic lowering */
596 NIR_PASS(progress, nir, nir_opt_algebraic);
597 NIR_PASS(progress, nir, nir_opt_constant_folding);
598
599 if (!nir->info.flrp_lowered) {
600 unsigned lower_flrp = (nir->options->lower_flrp16 ? 16 : 0) |
601 (nir->options->lower_flrp32 ? 32 : 0) |
602 (nir->options->lower_flrp64 ? 64 : 0);
603 assert(lower_flrp);
604 bool lower_flrp_progress = false;
605
606 NIR_PASS(lower_flrp_progress, nir, nir_lower_flrp, lower_flrp, false /* always_precise */);
607 if (lower_flrp_progress) {
608 NIR_PASS(progress, nir, nir_opt_constant_folding);
609 progress = true;
610 }
611
612 /* Nothing should rematerialize any flrps, so we only
613 * need to do this lowering once.
614 */
615 nir->info.flrp_lowered = true;
616 }
617
618 NIR_PASS(progress, nir, nir_opt_undef);
619 NIR_PASS(progress, nir, nir_opt_conditional_discard);
620 if (nir->options->max_unroll_iterations) {
621 NIR_PASS(progress, nir, nir_opt_loop_unroll);
622 }
623
624 if (nir->info.stage == MESA_SHADER_FRAGMENT)
625 NIR_PASS_V(nir, nir_opt_move_discards_to_top);
626
627 if (sscreen->options.fp16)
628 NIR_PASS(progress, nir, nir_opt_vectorize, NULL, NULL);
629 } while (progress);
630
631 NIR_PASS_V(nir, nir_lower_var_copies);
632 }
633
si_nir_late_opts(nir_shader * nir)634 void si_nir_late_opts(nir_shader *nir)
635 {
636 bool more_late_algebraic = true;
637 while (more_late_algebraic) {
638 more_late_algebraic = false;
639 NIR_PASS(more_late_algebraic, nir, nir_opt_algebraic_late);
640 NIR_PASS_V(nir, nir_opt_constant_folding);
641 NIR_PASS_V(nir, nir_copy_prop);
642 NIR_PASS_V(nir, nir_opt_dce);
643 NIR_PASS_V(nir, nir_opt_cse);
644 }
645 }
646
si_late_optimize_16bit_samplers(struct si_screen * sscreen,nir_shader * nir)647 static void si_late_optimize_16bit_samplers(struct si_screen *sscreen, nir_shader *nir)
648 {
649 /* Optimize and fix types of image_sample sources and destinations.
650 *
651 * The image_sample constraints are:
652 * nir_tex_src_coord: has_a16 ? select 16 or 32 : 32
653 * nir_tex_src_comparator: 32
654 * nir_tex_src_offset: 32
655 * nir_tex_src_bias: 32
656 * nir_tex_src_lod: match coord
657 * nir_tex_src_min_lod: match coord
658 * nir_tex_src_ms_index: match coord
659 * nir_tex_src_ddx: has_g16 && coord == 32 ? select 16 or 32 : match coord
660 * nir_tex_src_ddy: match ddy
661 *
662 * coord and ddx are selected optimally. The types of the rest are legalized
663 * based on those two.
664 */
665 /* TODO: The constraints can't represent the ddx constraint. */
666 /*bool has_g16 = sscreen->info.chip_class >= GFX10 && LLVM_VERSION_MAJOR >= 12;*/
667 bool has_g16 = false;
668 nir_tex_src_type_constraints tex_constraints = {
669 [nir_tex_src_comparator] = {true, 32},
670 [nir_tex_src_offset] = {true, 32},
671 [nir_tex_src_bias] = {true, 32},
672 [nir_tex_src_lod] = {true, 0, nir_tex_src_coord},
673 [nir_tex_src_min_lod] = {true, 0, nir_tex_src_coord},
674 [nir_tex_src_ms_index] = {true, 0, nir_tex_src_coord},
675 [nir_tex_src_ddx] = {!has_g16, 0, nir_tex_src_coord},
676 [nir_tex_src_ddy] = {true, 0, has_g16 ? nir_tex_src_ddx : nir_tex_src_coord},
677 };
678 bool changed = false;
679
680 NIR_PASS(changed, nir, nir_fold_16bit_sampler_conversions,
681 (1 << nir_tex_src_coord) |
682 (has_g16 ? 1 << nir_tex_src_ddx : 0));
683 NIR_PASS(changed, nir, nir_legalize_16bit_sampler_srcs, tex_constraints);
684
685 if (changed) {
686 si_nir_opts(sscreen, nir, false);
687 si_nir_late_opts(nir);
688 }
689 }
690
type_size_vec4(const struct glsl_type * type,bool bindless)691 static int type_size_vec4(const struct glsl_type *type, bool bindless)
692 {
693 return glsl_count_attribute_slots(type, false);
694 }
695
si_nir_lower_color(nir_shader * nir)696 static void si_nir_lower_color(nir_shader *nir)
697 {
698 nir_function_impl *entrypoint = nir_shader_get_entrypoint(nir);
699
700 nir_builder b;
701 nir_builder_init(&b, entrypoint);
702
703 nir_foreach_block (block, entrypoint) {
704 nir_foreach_instr_safe (instr, block) {
705 if (instr->type != nir_instr_type_intrinsic)
706 continue;
707
708 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
709
710 if (intrin->intrinsic != nir_intrinsic_load_deref)
711 continue;
712
713 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
714 if (!nir_deref_mode_is(deref, nir_var_shader_in))
715 continue;
716
717 b.cursor = nir_before_instr(instr);
718 nir_variable *var = nir_deref_instr_get_variable(deref);
719 nir_ssa_def *def;
720
721 if (var->data.location == VARYING_SLOT_COL0) {
722 def = nir_load_color0(&b);
723 nir->info.fs.color0_interp = var->data.interpolation;
724 nir->info.fs.color0_sample = var->data.sample;
725 nir->info.fs.color0_centroid = var->data.centroid;
726 } else if (var->data.location == VARYING_SLOT_COL1) {
727 def = nir_load_color1(&b);
728 nir->info.fs.color1_interp = var->data.interpolation;
729 nir->info.fs.color1_sample = var->data.sample;
730 nir->info.fs.color1_centroid = var->data.centroid;
731 } else {
732 continue;
733 }
734
735 nir_ssa_def_rewrite_uses(&intrin->dest.ssa, def);
736 nir_instr_remove(instr);
737 }
738 }
739 }
740
si_lower_io(struct nir_shader * nir)741 static void si_lower_io(struct nir_shader *nir)
742 {
743 /* HW supports indirect indexing for: | Enabled in driver
744 * -------------------------------------------------------
745 * TCS inputs | Yes
746 * TES inputs | Yes
747 * GS inputs | No
748 * -------------------------------------------------------
749 * VS outputs before TCS | No
750 * TCS outputs | Yes
751 * VS/TES outputs before GS | No
752 */
753 bool has_indirect_inputs = nir->info.stage == MESA_SHADER_TESS_CTRL ||
754 nir->info.stage == MESA_SHADER_TESS_EVAL;
755 bool has_indirect_outputs = nir->info.stage == MESA_SHADER_TESS_CTRL;
756
757 if (!has_indirect_inputs || !has_indirect_outputs) {
758 NIR_PASS_V(nir, nir_lower_io_to_temporaries, nir_shader_get_entrypoint(nir),
759 !has_indirect_outputs, !has_indirect_inputs);
760
761 /* Since we're doing nir_lower_io_to_temporaries late, we need
762 * to lower all the copy_deref's introduced by
763 * lower_io_to_temporaries before calling nir_lower_io.
764 */
765 NIR_PASS_V(nir, nir_split_var_copies);
766 NIR_PASS_V(nir, nir_lower_var_copies);
767 NIR_PASS_V(nir, nir_lower_global_vars_to_local);
768 }
769
770 /* The vectorization must be done after nir_lower_io_to_temporaries, because
771 * nir_lower_io_to_temporaries after vectorization breaks:
772 * piglit/bin/arb_gpu_shader5-interpolateAtOffset -auto -fbo
773 * TODO: It's probably a bug in nir_lower_io_to_temporaries.
774 *
775 * The vectorizer can only vectorize this:
776 * op src0.x, src1.x
777 * op src0.y, src1.y
778 *
779 * So it requires that inputs are already vectors and it must be the same
780 * vector between instructions. The vectorizer doesn't create vectors
781 * from independent scalar sources, so vectorize inputs.
782 *
783 * TODO: The pass fails this for VS: assert(b.shader->info.stage != MESA_SHADER_VERTEX);
784 */
785 if (nir->info.stage != MESA_SHADER_VERTEX)
786 NIR_PASS_V(nir, nir_lower_io_to_vector, nir_var_shader_in);
787
788 /* Vectorize outputs, so that we don't split vectors before storing outputs. */
789 /* TODO: The pass fails an assertion for other shader stages. */
790 if (nir->info.stage == MESA_SHADER_TESS_CTRL ||
791 nir->info.stage == MESA_SHADER_FRAGMENT)
792 NIR_PASS_V(nir, nir_lower_io_to_vector, nir_var_shader_out);
793
794 if (nir->info.stage == MESA_SHADER_FRAGMENT)
795 si_nir_lower_color(nir);
796
797 NIR_PASS_V(nir, nir_lower_io, nir_var_shader_out | nir_var_shader_in,
798 type_size_vec4, nir_lower_io_lower_64bit_to_32);
799 nir->info.io_lowered = true;
800
801 /* This pass needs actual constants */
802 NIR_PASS_V(nir, nir_opt_constant_folding);
803 NIR_PASS_V(nir, nir_io_add_const_offset_to_base, nir_var_shader_in |
804 nir_var_shader_out);
805
806 /* Remove dead derefs, so that nir_validate doesn't fail. */
807 NIR_PASS_V(nir, nir_opt_dce);
808
809 /* Remove input and output nir_variables, because we don't need them
810 * anymore. Also remove uniforms, because those should have been lowered
811 * to UBOs already.
812 */
813 unsigned modes = nir_var_shader_in | nir_var_shader_out | nir_var_uniform;
814 nir_foreach_variable_with_modes_safe(var, nir, modes) {
815 if (var->data.mode == nir_var_uniform &&
816 (glsl_type_get_image_count(var->type) ||
817 glsl_type_get_sampler_count(var->type)))
818 continue;
819
820 exec_node_remove(&var->node);
821 }
822 }
823
824 /**
825 * Perform "lowering" operations on the NIR that are run once when the shader
826 * selector is created.
827 */
si_lower_nir(struct si_screen * sscreen,struct nir_shader * nir)828 static void si_lower_nir(struct si_screen *sscreen, struct nir_shader *nir)
829 {
830 /* Perform lowerings (and optimizations) of code.
831 *
832 * Performance considerations aside, we must:
833 * - lower certain ALU operations
834 * - ensure constant offsets for texture instructions are folded
835 * and copy-propagated
836 */
837
838 static const struct nir_lower_tex_options lower_tex_options = {
839 .lower_txp = ~0u,
840 .lower_txs_cube_array = true,
841 };
842 NIR_PASS_V(nir, nir_lower_tex, &lower_tex_options);
843
844 static const struct nir_lower_image_options lower_image_options = {
845 .lower_cube_size = true,
846 };
847 NIR_PASS_V(nir, nir_lower_image, &lower_image_options);
848
849 const nir_lower_subgroups_options subgroups_options = {
850 .subgroup_size = 64,
851 .ballot_bit_size = 64,
852 .ballot_components = 1,
853 .lower_to_scalar = true,
854 .lower_subgroup_masks = true,
855 .lower_vote_trivial = false,
856 .lower_vote_eq = true,
857 };
858 NIR_PASS_V(nir, nir_lower_subgroups, &subgroups_options);
859
860 NIR_PASS_V(nir, nir_lower_discard_or_demote,
861 (sscreen->debug_flags & DBG(FS_CORRECT_DERIVS_AFTER_KILL)) ||
862 nir->info.is_arb_asm);
863
864 /* Lower load constants to scalar and then clean up the mess */
865 NIR_PASS_V(nir, nir_lower_load_const_to_scalar);
866 NIR_PASS_V(nir, nir_lower_var_copies);
867 NIR_PASS_V(nir, nir_opt_intrinsics);
868 NIR_PASS_V(nir, nir_lower_system_values);
869 NIR_PASS_V(nir, nir_lower_compute_system_values, NULL);
870
871 if (nir->info.stage == MESA_SHADER_COMPUTE) {
872 if (nir->info.cs.derivative_group == DERIVATIVE_GROUP_QUADS) {
873 /* If we are shuffling local_invocation_id for quad derivatives, we
874 * need to derive local_invocation_index from local_invocation_id
875 * first, so that the value corresponds to the shuffled
876 * local_invocation_id.
877 */
878 nir_lower_compute_system_values_options options = {0};
879 options.lower_local_invocation_index = true;
880 NIR_PASS_V(nir, nir_lower_compute_system_values, &options);
881 }
882
883 nir_opt_cse(nir); /* CSE load_local_invocation_id */
884 nir_lower_compute_system_values_options options = {0};
885 options.shuffle_local_ids_for_quad_derivatives = true;
886 NIR_PASS_V(nir, nir_lower_compute_system_values, &options);
887 }
888
889 if (sscreen->b.get_shader_param(&sscreen->b, PIPE_SHADER_FRAGMENT, PIPE_SHADER_CAP_FP16)) {
890 NIR_PASS_V(nir, nir_lower_mediump_io,
891 /* TODO: LLVM fails to compile this test if VS inputs are 16-bit:
892 * dEQP-GLES31.functional.shaders.builtin_functions.integer.bitfieldinsert.uvec3_lowp_geometry
893 */
894 (nir->info.stage != MESA_SHADER_VERTEX ? nir_var_shader_in : 0) | nir_var_shader_out,
895 BITFIELD64_BIT(VARYING_SLOT_PNTC) | BITFIELD64_RANGE(VARYING_SLOT_VAR0, 32),
896 true);
897 }
898
899 si_nir_opts(sscreen, nir, true);
900
901 /* Lower large variables that are always constant with load_constant
902 * intrinsics, which get turned into PC-relative loads from a data
903 * section next to the shader.
904 *
905 * st/mesa calls finalize_nir twice, but we can't call this pass twice.
906 */
907 bool changed = false;
908 if (!nir->constant_data) {
909 /* The pass crashes if there are dead temps of lowered IO interface types. */
910 NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_function_temp, NULL);
911 NIR_PASS(changed, nir, nir_opt_large_constants, glsl_get_natural_size_align_bytes, 16);
912 }
913
914 changed |= ac_nir_lower_indirect_derefs(nir, sscreen->info.chip_class);
915 if (changed)
916 si_nir_opts(sscreen, nir, false);
917
918 /* Run late optimizations to fuse ffma and eliminate 16-bit conversions. */
919 si_nir_late_opts(nir);
920
921 if (sscreen->b.get_shader_param(&sscreen->b, PIPE_SHADER_FRAGMENT, PIPE_SHADER_CAP_FP16))
922 si_late_optimize_16bit_samplers(sscreen, nir);
923
924 NIR_PASS_V(nir, nir_remove_dead_variables, nir_var_function_temp, NULL);
925 }
926
si_finalize_nir(struct pipe_screen * screen,void * nirptr)927 char *si_finalize_nir(struct pipe_screen *screen, void *nirptr)
928 {
929 struct si_screen *sscreen = (struct si_screen *)screen;
930 struct nir_shader *nir = (struct nir_shader *)nirptr;
931
932 si_lower_io(nir);
933 si_lower_nir(sscreen, nir);
934 nir_shader_gather_info(nir, nir_shader_get_entrypoint(nir));
935
936 if (sscreen->options.inline_uniforms)
937 nir_find_inlinable_uniforms(nir);
938
939 return NULL;
940 }
941