1 /*
2 * Copyright © 2015 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Jason Ekstrand (jason@jlekstrand.net)
25 *
26 */
27
28 #include "vtn_private.h"
29 #include "spirv_info.h"
30 #include "nir_deref.h"
31 #include <vulkan/vulkan_core.h>
32
33 static void
ptr_decoration_cb(struct vtn_builder * b,struct vtn_value * val,int member,const struct vtn_decoration * dec,void * void_ptr)34 ptr_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
35 const struct vtn_decoration *dec, void *void_ptr)
36 {
37 struct vtn_pointer *ptr = void_ptr;
38
39 switch (dec->decoration) {
40 case SpvDecorationNonUniformEXT:
41 ptr->access |= ACCESS_NON_UNIFORM;
42 break;
43
44 default:
45 break;
46 }
47 }
48
49 static struct vtn_pointer*
vtn_decorate_pointer(struct vtn_builder * b,struct vtn_value * val,struct vtn_pointer * ptr)50 vtn_decorate_pointer(struct vtn_builder *b, struct vtn_value *val,
51 struct vtn_pointer *ptr)
52 {
53 struct vtn_pointer dummy = { .access = 0 };
54 vtn_foreach_decoration(b, val, ptr_decoration_cb, &dummy);
55
56 /* If we're adding access flags, make a copy of the pointer. We could
57 * probably just OR them in without doing so but this prevents us from
58 * leaking them any further than actually specified in the SPIR-V.
59 */
60 if (dummy.access & ~ptr->access) {
61 struct vtn_pointer *copy = ralloc(b, struct vtn_pointer);
62 *copy = *ptr;
63 copy->access |= dummy.access;
64 return copy;
65 }
66
67 return ptr;
68 }
69
70 struct vtn_value *
vtn_push_pointer(struct vtn_builder * b,uint32_t value_id,struct vtn_pointer * ptr)71 vtn_push_pointer(struct vtn_builder *b, uint32_t value_id,
72 struct vtn_pointer *ptr)
73 {
74 struct vtn_value *val = vtn_push_value(b, value_id, vtn_value_type_pointer);
75 val->pointer = vtn_decorate_pointer(b, val, ptr);
76 return val;
77 }
78
79 void
vtn_copy_value(struct vtn_builder * b,uint32_t src_value_id,uint32_t dst_value_id)80 vtn_copy_value(struct vtn_builder *b, uint32_t src_value_id,
81 uint32_t dst_value_id)
82 {
83 struct vtn_value *src = vtn_untyped_value(b, src_value_id);
84 struct vtn_value *dst = vtn_untyped_value(b, dst_value_id);
85 struct vtn_value src_copy = *src;
86
87 vtn_fail_if(dst->value_type != vtn_value_type_invalid,
88 "SPIR-V id %u has already been written by another instruction",
89 dst_value_id);
90
91 vtn_fail_if(dst->type->id != src->type->id,
92 "Result Type must equal Operand type");
93
94 src_copy.name = dst->name;
95 src_copy.decoration = dst->decoration;
96 src_copy.type = dst->type;
97 *dst = src_copy;
98
99 if (dst->value_type == vtn_value_type_pointer)
100 dst->pointer = vtn_decorate_pointer(b, dst, dst->pointer);
101 }
102
103 static struct vtn_access_chain *
vtn_access_chain_create(struct vtn_builder * b,unsigned length)104 vtn_access_chain_create(struct vtn_builder *b, unsigned length)
105 {
106 struct vtn_access_chain *chain;
107
108 /* Subtract 1 from the length since there's already one built in */
109 size_t size = sizeof(*chain) +
110 (MAX2(length, 1) - 1) * sizeof(chain->link[0]);
111 chain = rzalloc_size(b, size);
112 chain->length = length;
113
114 return chain;
115 }
116
117 bool
vtn_mode_uses_ssa_offset(struct vtn_builder * b,enum vtn_variable_mode mode)118 vtn_mode_uses_ssa_offset(struct vtn_builder *b,
119 enum vtn_variable_mode mode)
120 {
121 return ((mode == vtn_variable_mode_ubo ||
122 mode == vtn_variable_mode_ssbo) &&
123 b->options->lower_ubo_ssbo_access_to_offsets) ||
124 mode == vtn_variable_mode_push_constant;
125 }
126
127 static bool
vtn_mode_is_cross_invocation(struct vtn_builder * b,enum vtn_variable_mode mode)128 vtn_mode_is_cross_invocation(struct vtn_builder *b,
129 enum vtn_variable_mode mode)
130 {
131 return mode == vtn_variable_mode_ssbo ||
132 mode == vtn_variable_mode_ubo ||
133 mode == vtn_variable_mode_phys_ssbo ||
134 mode == vtn_variable_mode_push_constant ||
135 mode == vtn_variable_mode_workgroup ||
136 mode == vtn_variable_mode_cross_workgroup;
137 }
138
139 static bool
vtn_pointer_is_external_block(struct vtn_builder * b,struct vtn_pointer * ptr)140 vtn_pointer_is_external_block(struct vtn_builder *b,
141 struct vtn_pointer *ptr)
142 {
143 return ptr->mode == vtn_variable_mode_ssbo ||
144 ptr->mode == vtn_variable_mode_ubo ||
145 ptr->mode == vtn_variable_mode_phys_ssbo ||
146 ptr->mode == vtn_variable_mode_push_constant;
147 }
148
149 static nir_ssa_def *
vtn_access_link_as_ssa(struct vtn_builder * b,struct vtn_access_link link,unsigned stride,unsigned bit_size)150 vtn_access_link_as_ssa(struct vtn_builder *b, struct vtn_access_link link,
151 unsigned stride, unsigned bit_size)
152 {
153 vtn_assert(stride > 0);
154 if (link.mode == vtn_access_mode_literal) {
155 return nir_imm_intN_t(&b->nb, link.id * stride, bit_size);
156 } else {
157 nir_ssa_def *ssa = vtn_ssa_value(b, link.id)->def;
158 if (ssa->bit_size != bit_size)
159 ssa = nir_i2i(&b->nb, ssa, bit_size);
160 return nir_imul_imm(&b->nb, ssa, stride);
161 }
162 }
163
164 static VkDescriptorType
vk_desc_type_for_mode(struct vtn_builder * b,enum vtn_variable_mode mode)165 vk_desc_type_for_mode(struct vtn_builder *b, enum vtn_variable_mode mode)
166 {
167 switch (mode) {
168 case vtn_variable_mode_ubo:
169 return VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER;
170 case vtn_variable_mode_ssbo:
171 return VK_DESCRIPTOR_TYPE_STORAGE_BUFFER;
172 default:
173 vtn_fail("Invalid mode for vulkan_resource_index");
174 }
175 }
176
177 static nir_ssa_def *
vtn_variable_resource_index(struct vtn_builder * b,struct vtn_variable * var,nir_ssa_def * desc_array_index)178 vtn_variable_resource_index(struct vtn_builder *b, struct vtn_variable *var,
179 nir_ssa_def *desc_array_index)
180 {
181 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN);
182
183 if (!desc_array_index) {
184 vtn_assert(glsl_type_is_struct_or_ifc(var->type->type));
185 desc_array_index = nir_imm_int(&b->nb, 0);
186 }
187
188 nir_intrinsic_instr *instr =
189 nir_intrinsic_instr_create(b->nb.shader,
190 nir_intrinsic_vulkan_resource_index);
191 instr->src[0] = nir_src_for_ssa(desc_array_index);
192 nir_intrinsic_set_desc_set(instr, var->descriptor_set);
193 nir_intrinsic_set_binding(instr, var->binding);
194 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, var->mode));
195
196 vtn_fail_if(var->mode != vtn_variable_mode_ubo &&
197 var->mode != vtn_variable_mode_ssbo,
198 "Invalid mode for vulkan_resource_index");
199
200 nir_address_format addr_format = vtn_mode_to_address_format(b, var->mode);
201 const struct glsl_type *index_type =
202 b->options->lower_ubo_ssbo_access_to_offsets ?
203 glsl_uint_type() : nir_address_format_to_glsl_type(addr_format);
204
205 instr->num_components = glsl_get_vector_elements(index_type);
206 nir_ssa_dest_init(&instr->instr, &instr->dest, instr->num_components,
207 glsl_get_bit_size(index_type), NULL);
208 nir_builder_instr_insert(&b->nb, &instr->instr);
209
210 return &instr->dest.ssa;
211 }
212
213 static nir_ssa_def *
vtn_resource_reindex(struct vtn_builder * b,enum vtn_variable_mode mode,nir_ssa_def * base_index,nir_ssa_def * offset_index)214 vtn_resource_reindex(struct vtn_builder *b, enum vtn_variable_mode mode,
215 nir_ssa_def *base_index, nir_ssa_def *offset_index)
216 {
217 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN);
218
219 nir_intrinsic_instr *instr =
220 nir_intrinsic_instr_create(b->nb.shader,
221 nir_intrinsic_vulkan_resource_reindex);
222 instr->src[0] = nir_src_for_ssa(base_index);
223 instr->src[1] = nir_src_for_ssa(offset_index);
224 nir_intrinsic_set_desc_type(instr, vk_desc_type_for_mode(b, mode));
225
226 vtn_fail_if(mode != vtn_variable_mode_ubo && mode != vtn_variable_mode_ssbo,
227 "Invalid mode for vulkan_resource_reindex");
228
229 nir_address_format addr_format = vtn_mode_to_address_format(b, mode);
230 const struct glsl_type *index_type =
231 b->options->lower_ubo_ssbo_access_to_offsets ?
232 glsl_uint_type() : nir_address_format_to_glsl_type(addr_format);
233
234 instr->num_components = glsl_get_vector_elements(index_type);
235 nir_ssa_dest_init(&instr->instr, &instr->dest, instr->num_components,
236 glsl_get_bit_size(index_type), NULL);
237 nir_builder_instr_insert(&b->nb, &instr->instr);
238
239 return &instr->dest.ssa;
240 }
241
242 static nir_ssa_def *
vtn_descriptor_load(struct vtn_builder * b,enum vtn_variable_mode mode,nir_ssa_def * desc_index)243 vtn_descriptor_load(struct vtn_builder *b, enum vtn_variable_mode mode,
244 nir_ssa_def *desc_index)
245 {
246 vtn_assert(b->options->environment == NIR_SPIRV_VULKAN);
247
248 nir_intrinsic_instr *desc_load =
249 nir_intrinsic_instr_create(b->nb.shader,
250 nir_intrinsic_load_vulkan_descriptor);
251 desc_load->src[0] = nir_src_for_ssa(desc_index);
252 nir_intrinsic_set_desc_type(desc_load, vk_desc_type_for_mode(b, mode));
253
254 vtn_fail_if(mode != vtn_variable_mode_ubo && mode != vtn_variable_mode_ssbo,
255 "Invalid mode for load_vulkan_descriptor");
256
257 nir_address_format addr_format = vtn_mode_to_address_format(b, mode);
258 const struct glsl_type *ptr_type =
259 nir_address_format_to_glsl_type(addr_format);
260
261 desc_load->num_components = glsl_get_vector_elements(ptr_type);
262 nir_ssa_dest_init(&desc_load->instr, &desc_load->dest,
263 desc_load->num_components,
264 glsl_get_bit_size(ptr_type), NULL);
265 nir_builder_instr_insert(&b->nb, &desc_load->instr);
266
267 return &desc_load->dest.ssa;
268 }
269
270 /* Dereference the given base pointer by the access chain */
271 static struct vtn_pointer *
vtn_nir_deref_pointer_dereference(struct vtn_builder * b,struct vtn_pointer * base,struct vtn_access_chain * deref_chain)272 vtn_nir_deref_pointer_dereference(struct vtn_builder *b,
273 struct vtn_pointer *base,
274 struct vtn_access_chain *deref_chain)
275 {
276 struct vtn_type *type = base->type;
277 enum gl_access_qualifier access = base->access | deref_chain->access;
278 unsigned idx = 0;
279
280 nir_deref_instr *tail;
281 if (base->deref) {
282 tail = base->deref;
283 } else if (b->options->environment == NIR_SPIRV_VULKAN &&
284 vtn_pointer_is_external_block(b, base)) {
285 nir_ssa_def *block_index = base->block_index;
286
287 /* We dereferencing an external block pointer. Correctness of this
288 * operation relies on one particular line in the SPIR-V spec, section
289 * entitled "Validation Rules for Shader Capabilities":
290 *
291 * "Block and BufferBlock decorations cannot decorate a structure
292 * type that is nested at any level inside another structure type
293 * decorated with Block or BufferBlock."
294 *
295 * This means that we can detect the point where we cross over from
296 * descriptor indexing to buffer indexing by looking for the block
297 * decorated struct type. Anything before the block decorated struct
298 * type is a descriptor indexing operation and anything after the block
299 * decorated struct is a buffer offset operation.
300 */
301
302 /* Figure out the descriptor array index if any
303 *
304 * Some of the Vulkan CTS tests with hand-rolled SPIR-V have been known
305 * to forget the Block or BufferBlock decoration from time to time.
306 * It's more robust if we check for both !block_index and for the type
307 * to contain a block. This way there's a decent chance that arrays of
308 * UBOs/SSBOs will work correctly even if variable pointers are
309 * completley toast.
310 */
311 nir_ssa_def *desc_arr_idx = NULL;
312 if (!block_index || vtn_type_contains_block(b, type)) {
313 /* If our type contains a block, then we're still outside the block
314 * and we need to process enough levels of dereferences to get inside
315 * of it.
316 */
317 if (deref_chain->ptr_as_array) {
318 unsigned aoa_size = glsl_get_aoa_size(type->type);
319 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[idx],
320 MAX2(aoa_size, 1), 32);
321 idx++;
322 }
323
324 for (; idx < deref_chain->length; idx++) {
325 if (type->base_type != vtn_base_type_array) {
326 vtn_assert(type->base_type == vtn_base_type_struct);
327 break;
328 }
329
330 unsigned aoa_size = glsl_get_aoa_size(type->array_element->type);
331 nir_ssa_def *arr_offset =
332 vtn_access_link_as_ssa(b, deref_chain->link[idx],
333 MAX2(aoa_size, 1), 32);
334 if (desc_arr_idx)
335 desc_arr_idx = nir_iadd(&b->nb, desc_arr_idx, arr_offset);
336 else
337 desc_arr_idx = arr_offset;
338
339 type = type->array_element;
340 access |= type->access;
341 }
342 }
343
344 if (!block_index) {
345 vtn_assert(base->var && base->type);
346 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx);
347 } else if (desc_arr_idx) {
348 block_index = vtn_resource_reindex(b, base->mode,
349 block_index, desc_arr_idx);
350 }
351
352 if (idx == deref_chain->length) {
353 /* The entire deref was consumed in finding the block index. Return
354 * a pointer which just has a block index and a later access chain
355 * will dereference deeper.
356 */
357 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
358 ptr->mode = base->mode;
359 ptr->type = type;
360 ptr->block_index = block_index;
361 ptr->access = access;
362 return ptr;
363 }
364
365 /* If we got here, there's more access chain to handle and we have the
366 * final block index. Insert a descriptor load and cast to a deref to
367 * start the deref chain.
368 */
369 nir_ssa_def *desc = vtn_descriptor_load(b, base->mode, block_index);
370
371 assert(base->mode == vtn_variable_mode_ssbo ||
372 base->mode == vtn_variable_mode_ubo);
373 nir_variable_mode nir_mode =
374 base->mode == vtn_variable_mode_ssbo ? nir_var_mem_ssbo : nir_var_mem_ubo;
375
376 tail = nir_build_deref_cast(&b->nb, desc, nir_mode,
377 vtn_type_get_nir_type(b, type, base->mode),
378 base->ptr_type->stride);
379 } else {
380 assert(base->var && base->var->var);
381 tail = nir_build_deref_var(&b->nb, base->var->var);
382 if (base->ptr_type && base->ptr_type->type) {
383 tail->dest.ssa.num_components =
384 glsl_get_vector_elements(base->ptr_type->type);
385 tail->dest.ssa.bit_size = glsl_get_bit_size(base->ptr_type->type);
386 }
387 }
388
389 if (idx == 0 && deref_chain->ptr_as_array) {
390 /* We start with a deref cast to get the stride. Hopefully, we'll be
391 * able to delete that cast eventually.
392 */
393 tail = nir_build_deref_cast(&b->nb, &tail->dest.ssa, tail->mode,
394 tail->type, base->ptr_type->stride);
395
396 nir_ssa_def *index = vtn_access_link_as_ssa(b, deref_chain->link[0], 1,
397 tail->dest.ssa.bit_size);
398 tail = nir_build_deref_ptr_as_array(&b->nb, tail, index);
399 idx++;
400 }
401
402 for (; idx < deref_chain->length; idx++) {
403 if (glsl_type_is_struct_or_ifc(type->type)) {
404 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal);
405 unsigned field = deref_chain->link[idx].id;
406 tail = nir_build_deref_struct(&b->nb, tail, field);
407 type = type->members[field];
408 } else {
409 nir_ssa_def *arr_index =
410 vtn_access_link_as_ssa(b, deref_chain->link[idx], 1,
411 tail->dest.ssa.bit_size);
412 tail = nir_build_deref_array(&b->nb, tail, arr_index);
413 type = type->array_element;
414 }
415
416 access |= type->access;
417 }
418
419 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
420 ptr->mode = base->mode;
421 ptr->type = type;
422 ptr->var = base->var;
423 ptr->deref = tail;
424 ptr->access = access;
425
426 return ptr;
427 }
428
429 static struct vtn_pointer *
vtn_ssa_offset_pointer_dereference(struct vtn_builder * b,struct vtn_pointer * base,struct vtn_access_chain * deref_chain)430 vtn_ssa_offset_pointer_dereference(struct vtn_builder *b,
431 struct vtn_pointer *base,
432 struct vtn_access_chain *deref_chain)
433 {
434 nir_ssa_def *block_index = base->block_index;
435 nir_ssa_def *offset = base->offset;
436 struct vtn_type *type = base->type;
437 enum gl_access_qualifier access = base->access;
438
439 unsigned idx = 0;
440 if (base->mode == vtn_variable_mode_ubo ||
441 base->mode == vtn_variable_mode_ssbo) {
442 if (!block_index) {
443 vtn_assert(base->var && base->type);
444 nir_ssa_def *desc_arr_idx;
445 if (glsl_type_is_array(type->type)) {
446 if (deref_chain->length >= 1) {
447 desc_arr_idx =
448 vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32);
449 idx++;
450 /* This consumes a level of type */
451 type = type->array_element;
452 access |= type->access;
453 } else {
454 /* This is annoying. We've been asked for a pointer to the
455 * array of UBOs/SSBOs and not a specifc buffer. Return a
456 * pointer with a descriptor index of 0 and we'll have to do
457 * a reindex later to adjust it to the right thing.
458 */
459 desc_arr_idx = nir_imm_int(&b->nb, 0);
460 }
461 } else if (deref_chain->ptr_as_array) {
462 /* You can't have a zero-length OpPtrAccessChain */
463 vtn_assert(deref_chain->length >= 1);
464 desc_arr_idx = vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32);
465 } else {
466 /* We have a regular non-array SSBO. */
467 desc_arr_idx = NULL;
468 }
469 block_index = vtn_variable_resource_index(b, base->var, desc_arr_idx);
470 } else if (deref_chain->ptr_as_array &&
471 type->base_type == vtn_base_type_struct && type->block) {
472 /* We are doing an OpPtrAccessChain on a pointer to a struct that is
473 * decorated block. This is an interesting corner in the SPIR-V
474 * spec. One interpretation would be that they client is clearly
475 * trying to treat that block as if it's an implicit array of blocks
476 * repeated in the buffer. However, the SPIR-V spec for the
477 * OpPtrAccessChain says:
478 *
479 * "Base is treated as the address of the first element of an
480 * array, and the Element element’s address is computed to be the
481 * base for the Indexes, as per OpAccessChain."
482 *
483 * Taken literally, that would mean that your struct type is supposed
484 * to be treated as an array of such a struct and, since it's
485 * decorated block, that means an array of blocks which corresponds
486 * to an array descriptor. Therefore, we need to do a reindex
487 * operation to add the index from the first link in the access chain
488 * to the index we recieved.
489 *
490 * The downside to this interpretation (there always is one) is that
491 * this might be somewhat surprising behavior to apps if they expect
492 * the implicit array behavior described above.
493 */
494 vtn_assert(deref_chain->length >= 1);
495 nir_ssa_def *offset_index =
496 vtn_access_link_as_ssa(b, deref_chain->link[0], 1, 32);
497 idx++;
498
499 block_index = vtn_resource_reindex(b, base->mode,
500 block_index, offset_index);
501 }
502 }
503
504 if (!offset) {
505 if (base->mode == vtn_variable_mode_workgroup) {
506 /* SLM doesn't need nor have a block index */
507 vtn_assert(!block_index);
508
509 /* We need the variable for the base offset */
510 vtn_assert(base->var);
511
512 /* We need ptr_type for size and alignment */
513 vtn_assert(base->ptr_type);
514
515 /* Assign location on first use so that we don't end up bloating SLM
516 * address space for variables which are never statically used.
517 */
518 if (base->var->shared_location < 0) {
519 vtn_assert(base->ptr_type->length > 0 && base->ptr_type->align > 0);
520 b->shader->num_shared = vtn_align_u32(b->shader->num_shared,
521 base->ptr_type->align);
522 base->var->shared_location = b->shader->num_shared;
523 b->shader->num_shared += base->ptr_type->length;
524 }
525
526 offset = nir_imm_int(&b->nb, base->var->shared_location);
527 } else if (base->mode == vtn_variable_mode_push_constant) {
528 /* Push constants neither need nor have a block index */
529 vtn_assert(!block_index);
530
531 /* Start off with at the start of the push constant block. */
532 offset = nir_imm_int(&b->nb, 0);
533 } else {
534 /* The code above should have ensured a block_index when needed. */
535 vtn_assert(block_index);
536
537 /* Start off with at the start of the buffer. */
538 offset = nir_imm_int(&b->nb, 0);
539 }
540 }
541
542 if (deref_chain->ptr_as_array && idx == 0) {
543 /* We need ptr_type for the stride */
544 vtn_assert(base->ptr_type);
545
546 /* We need at least one element in the chain */
547 vtn_assert(deref_chain->length >= 1);
548
549 nir_ssa_def *elem_offset =
550 vtn_access_link_as_ssa(b, deref_chain->link[idx],
551 base->ptr_type->stride, offset->bit_size);
552 offset = nir_iadd(&b->nb, offset, elem_offset);
553 idx++;
554 }
555
556 for (; idx < deref_chain->length; idx++) {
557 switch (glsl_get_base_type(type->type)) {
558 case GLSL_TYPE_UINT:
559 case GLSL_TYPE_INT:
560 case GLSL_TYPE_UINT16:
561 case GLSL_TYPE_INT16:
562 case GLSL_TYPE_UINT8:
563 case GLSL_TYPE_INT8:
564 case GLSL_TYPE_UINT64:
565 case GLSL_TYPE_INT64:
566 case GLSL_TYPE_FLOAT:
567 case GLSL_TYPE_FLOAT16:
568 case GLSL_TYPE_DOUBLE:
569 case GLSL_TYPE_BOOL:
570 case GLSL_TYPE_ARRAY: {
571 nir_ssa_def *elem_offset =
572 vtn_access_link_as_ssa(b, deref_chain->link[idx],
573 type->stride, offset->bit_size);
574 offset = nir_iadd(&b->nb, offset, elem_offset);
575 type = type->array_element;
576 access |= type->access;
577 break;
578 }
579
580 case GLSL_TYPE_INTERFACE:
581 case GLSL_TYPE_STRUCT: {
582 vtn_assert(deref_chain->link[idx].mode == vtn_access_mode_literal);
583 unsigned member = deref_chain->link[idx].id;
584 offset = nir_iadd_imm(&b->nb, offset, type->offsets[member]);
585 type = type->members[member];
586 access |= type->access;
587 break;
588 }
589
590 default:
591 vtn_fail("Invalid type for deref");
592 }
593 }
594
595 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
596 ptr->mode = base->mode;
597 ptr->type = type;
598 ptr->block_index = block_index;
599 ptr->offset = offset;
600 ptr->access = access;
601
602 return ptr;
603 }
604
605 /* Dereference the given base pointer by the access chain */
606 static struct vtn_pointer *
vtn_pointer_dereference(struct vtn_builder * b,struct vtn_pointer * base,struct vtn_access_chain * deref_chain)607 vtn_pointer_dereference(struct vtn_builder *b,
608 struct vtn_pointer *base,
609 struct vtn_access_chain *deref_chain)
610 {
611 if (vtn_pointer_uses_ssa_offset(b, base)) {
612 return vtn_ssa_offset_pointer_dereference(b, base, deref_chain);
613 } else {
614 return vtn_nir_deref_pointer_dereference(b, base, deref_chain);
615 }
616 }
617
618 nir_deref_instr *
vtn_pointer_to_deref(struct vtn_builder * b,struct vtn_pointer * ptr)619 vtn_pointer_to_deref(struct vtn_builder *b, struct vtn_pointer *ptr)
620 {
621 vtn_assert(!vtn_pointer_uses_ssa_offset(b, ptr));
622 if (!ptr->deref) {
623 struct vtn_access_chain chain = {
624 .length = 0,
625 };
626 ptr = vtn_nir_deref_pointer_dereference(b, ptr, &chain);
627 }
628
629 return ptr->deref;
630 }
631
632 static void
_vtn_local_load_store(struct vtn_builder * b,bool load,nir_deref_instr * deref,struct vtn_ssa_value * inout,enum gl_access_qualifier access)633 _vtn_local_load_store(struct vtn_builder *b, bool load, nir_deref_instr *deref,
634 struct vtn_ssa_value *inout,
635 enum gl_access_qualifier access)
636 {
637 if (glsl_type_is_vector_or_scalar(deref->type)) {
638 if (load) {
639 inout->def = nir_load_deref_with_access(&b->nb, deref, access);
640 } else {
641 nir_store_deref_with_access(&b->nb, deref, inout->def, ~0, access);
642 }
643 } else if (glsl_type_is_array(deref->type) ||
644 glsl_type_is_matrix(deref->type)) {
645 unsigned elems = glsl_get_length(deref->type);
646 for (unsigned i = 0; i < elems; i++) {
647 nir_deref_instr *child =
648 nir_build_deref_array_imm(&b->nb, deref, i);
649 _vtn_local_load_store(b, load, child, inout->elems[i], access);
650 }
651 } else {
652 vtn_assert(glsl_type_is_struct_or_ifc(deref->type));
653 unsigned elems = glsl_get_length(deref->type);
654 for (unsigned i = 0; i < elems; i++) {
655 nir_deref_instr *child = nir_build_deref_struct(&b->nb, deref, i);
656 _vtn_local_load_store(b, load, child, inout->elems[i], access);
657 }
658 }
659 }
660
661 nir_deref_instr *
vtn_nir_deref(struct vtn_builder * b,uint32_t id)662 vtn_nir_deref(struct vtn_builder *b, uint32_t id)
663 {
664 struct vtn_pointer *ptr = vtn_value(b, id, vtn_value_type_pointer)->pointer;
665 return vtn_pointer_to_deref(b, ptr);
666 }
667
668 /*
669 * Gets the NIR-level deref tail, which may have as a child an array deref
670 * selecting which component due to OpAccessChain supporting per-component
671 * indexing in SPIR-V.
672 */
673 static nir_deref_instr *
get_deref_tail(nir_deref_instr * deref)674 get_deref_tail(nir_deref_instr *deref)
675 {
676 if (deref->deref_type != nir_deref_type_array)
677 return deref;
678
679 nir_deref_instr *parent =
680 nir_instr_as_deref(deref->parent.ssa->parent_instr);
681
682 if (glsl_type_is_vector(parent->type))
683 return parent;
684 else
685 return deref;
686 }
687
688 struct vtn_ssa_value *
vtn_local_load(struct vtn_builder * b,nir_deref_instr * src,enum gl_access_qualifier access)689 vtn_local_load(struct vtn_builder *b, nir_deref_instr *src,
690 enum gl_access_qualifier access)
691 {
692 nir_deref_instr *src_tail = get_deref_tail(src);
693 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src_tail->type);
694 _vtn_local_load_store(b, true, src_tail, val, access);
695
696 if (src_tail != src) {
697 val->type = src->type;
698 val->def = nir_vector_extract(&b->nb, val->def, src->arr.index.ssa);
699 }
700
701 return val;
702 }
703
704 void
vtn_local_store(struct vtn_builder * b,struct vtn_ssa_value * src,nir_deref_instr * dest,enum gl_access_qualifier access)705 vtn_local_store(struct vtn_builder *b, struct vtn_ssa_value *src,
706 nir_deref_instr *dest, enum gl_access_qualifier access)
707 {
708 nir_deref_instr *dest_tail = get_deref_tail(dest);
709
710 if (dest_tail != dest) {
711 struct vtn_ssa_value *val = vtn_create_ssa_value(b, dest_tail->type);
712 _vtn_local_load_store(b, true, dest_tail, val, access);
713
714 val->def = nir_vector_insert(&b->nb, val->def, src->def,
715 dest->arr.index.ssa);
716 _vtn_local_load_store(b, false, dest_tail, val, access);
717 } else {
718 _vtn_local_load_store(b, false, dest_tail, src, access);
719 }
720 }
721
722 nir_ssa_def *
vtn_pointer_to_offset(struct vtn_builder * b,struct vtn_pointer * ptr,nir_ssa_def ** index_out)723 vtn_pointer_to_offset(struct vtn_builder *b, struct vtn_pointer *ptr,
724 nir_ssa_def **index_out)
725 {
726 assert(vtn_pointer_uses_ssa_offset(b, ptr));
727 if (!ptr->offset) {
728 struct vtn_access_chain chain = {
729 .length = 0,
730 };
731 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain);
732 }
733 *index_out = ptr->block_index;
734 return ptr->offset;
735 }
736
737 /* Tries to compute the size of an interface block based on the strides and
738 * offsets that are provided to us in the SPIR-V source.
739 */
740 static unsigned
vtn_type_block_size(struct vtn_builder * b,struct vtn_type * type)741 vtn_type_block_size(struct vtn_builder *b, struct vtn_type *type)
742 {
743 enum glsl_base_type base_type = glsl_get_base_type(type->type);
744 switch (base_type) {
745 case GLSL_TYPE_UINT:
746 case GLSL_TYPE_INT:
747 case GLSL_TYPE_UINT16:
748 case GLSL_TYPE_INT16:
749 case GLSL_TYPE_UINT8:
750 case GLSL_TYPE_INT8:
751 case GLSL_TYPE_UINT64:
752 case GLSL_TYPE_INT64:
753 case GLSL_TYPE_FLOAT:
754 case GLSL_TYPE_FLOAT16:
755 case GLSL_TYPE_BOOL:
756 case GLSL_TYPE_DOUBLE: {
757 unsigned cols = type->row_major ? glsl_get_vector_elements(type->type) :
758 glsl_get_matrix_columns(type->type);
759 if (cols > 1) {
760 vtn_assert(type->stride > 0);
761 return type->stride * cols;
762 } else {
763 unsigned type_size = glsl_get_bit_size(type->type) / 8;
764 return glsl_get_vector_elements(type->type) * type_size;
765 }
766 }
767
768 case GLSL_TYPE_STRUCT:
769 case GLSL_TYPE_INTERFACE: {
770 unsigned size = 0;
771 unsigned num_fields = glsl_get_length(type->type);
772 for (unsigned f = 0; f < num_fields; f++) {
773 unsigned field_end = type->offsets[f] +
774 vtn_type_block_size(b, type->members[f]);
775 size = MAX2(size, field_end);
776 }
777 return size;
778 }
779
780 case GLSL_TYPE_ARRAY:
781 vtn_assert(type->stride > 0);
782 vtn_assert(glsl_get_length(type->type) > 0);
783 return type->stride * glsl_get_length(type->type);
784
785 default:
786 vtn_fail("Invalid block type");
787 return 0;
788 }
789 }
790
791 static void
_vtn_load_store_tail(struct vtn_builder * b,nir_intrinsic_op op,bool load,nir_ssa_def * index,nir_ssa_def * offset,unsigned access_offset,unsigned access_size,struct vtn_ssa_value ** inout,const struct glsl_type * type,enum gl_access_qualifier access)792 _vtn_load_store_tail(struct vtn_builder *b, nir_intrinsic_op op, bool load,
793 nir_ssa_def *index, nir_ssa_def *offset,
794 unsigned access_offset, unsigned access_size,
795 struct vtn_ssa_value **inout, const struct glsl_type *type,
796 enum gl_access_qualifier access)
797 {
798 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(b->nb.shader, op);
799 instr->num_components = glsl_get_vector_elements(type);
800
801 /* Booleans usually shouldn't show up in external memory in SPIR-V.
802 * However, they do for certain older GLSLang versions and can for shared
803 * memory when we lower access chains internally.
804 */
805 const unsigned data_bit_size = glsl_type_is_boolean(type) ? 32 :
806 glsl_get_bit_size(type);
807
808 int src = 0;
809 if (!load) {
810 nir_intrinsic_set_write_mask(instr, (1 << instr->num_components) - 1);
811 instr->src[src++] = nir_src_for_ssa((*inout)->def);
812 }
813
814 if (op == nir_intrinsic_load_push_constant) {
815 nir_intrinsic_set_base(instr, access_offset);
816 nir_intrinsic_set_range(instr, access_size);
817 }
818
819 if (op == nir_intrinsic_load_ubo ||
820 op == nir_intrinsic_load_ssbo ||
821 op == nir_intrinsic_store_ssbo) {
822 nir_intrinsic_set_access(instr, access);
823 }
824
825 /* With extensions like relaxed_block_layout, we really can't guarantee
826 * much more than scalar alignment.
827 */
828 if (op != nir_intrinsic_load_push_constant)
829 nir_intrinsic_set_align(instr, data_bit_size / 8, 0);
830
831 if (index)
832 instr->src[src++] = nir_src_for_ssa(index);
833
834 if (op == nir_intrinsic_load_push_constant) {
835 /* We need to subtract the offset from where the intrinsic will load the
836 * data. */
837 instr->src[src++] =
838 nir_src_for_ssa(nir_isub(&b->nb, offset,
839 nir_imm_int(&b->nb, access_offset)));
840 } else {
841 instr->src[src++] = nir_src_for_ssa(offset);
842 }
843
844 if (load) {
845 nir_ssa_dest_init(&instr->instr, &instr->dest,
846 instr->num_components, data_bit_size, NULL);
847 (*inout)->def = &instr->dest.ssa;
848 }
849
850 nir_builder_instr_insert(&b->nb, &instr->instr);
851
852 if (load && glsl_get_base_type(type) == GLSL_TYPE_BOOL)
853 (*inout)->def = nir_ine(&b->nb, (*inout)->def, nir_imm_int(&b->nb, 0));
854 }
855
856 static void
_vtn_block_load_store(struct vtn_builder * b,nir_intrinsic_op op,bool load,nir_ssa_def * index,nir_ssa_def * offset,unsigned access_offset,unsigned access_size,struct vtn_type * type,enum gl_access_qualifier access,struct vtn_ssa_value ** inout)857 _vtn_block_load_store(struct vtn_builder *b, nir_intrinsic_op op, bool load,
858 nir_ssa_def *index, nir_ssa_def *offset,
859 unsigned access_offset, unsigned access_size,
860 struct vtn_type *type, enum gl_access_qualifier access,
861 struct vtn_ssa_value **inout)
862 {
863 enum glsl_base_type base_type = glsl_get_base_type(type->type);
864 switch (base_type) {
865 case GLSL_TYPE_UINT:
866 case GLSL_TYPE_INT:
867 case GLSL_TYPE_UINT16:
868 case GLSL_TYPE_INT16:
869 case GLSL_TYPE_UINT8:
870 case GLSL_TYPE_INT8:
871 case GLSL_TYPE_UINT64:
872 case GLSL_TYPE_INT64:
873 case GLSL_TYPE_FLOAT:
874 case GLSL_TYPE_FLOAT16:
875 case GLSL_TYPE_DOUBLE:
876 case GLSL_TYPE_BOOL:
877 /* This is where things get interesting. At this point, we've hit
878 * a vector, a scalar, or a matrix.
879 */
880 if (glsl_type_is_matrix(type->type)) {
881 /* Loading the whole matrix */
882 struct vtn_ssa_value *transpose;
883 unsigned num_ops, vec_width, col_stride;
884 if (type->row_major) {
885 num_ops = glsl_get_vector_elements(type->type);
886 vec_width = glsl_get_matrix_columns(type->type);
887 col_stride = type->array_element->stride;
888 if (load) {
889 const struct glsl_type *transpose_type =
890 glsl_matrix_type(base_type, vec_width, num_ops);
891 *inout = vtn_create_ssa_value(b, transpose_type);
892 } else {
893 transpose = vtn_ssa_transpose(b, *inout);
894 inout = &transpose;
895 }
896 } else {
897 num_ops = glsl_get_matrix_columns(type->type);
898 vec_width = glsl_get_vector_elements(type->type);
899 col_stride = type->stride;
900 }
901
902 for (unsigned i = 0; i < num_ops; i++) {
903 nir_ssa_def *elem_offset =
904 nir_iadd_imm(&b->nb, offset, i * col_stride);
905 _vtn_load_store_tail(b, op, load, index, elem_offset,
906 access_offset, access_size,
907 &(*inout)->elems[i],
908 glsl_vector_type(base_type, vec_width),
909 type->access | access);
910 }
911
912 if (load && type->row_major)
913 *inout = vtn_ssa_transpose(b, *inout);
914 } else {
915 unsigned elems = glsl_get_vector_elements(type->type);
916 unsigned type_size = glsl_get_bit_size(type->type) / 8;
917 if (elems == 1 || type->stride == type_size) {
918 /* This is a tightly-packed normal scalar or vector load */
919 vtn_assert(glsl_type_is_vector_or_scalar(type->type));
920 _vtn_load_store_tail(b, op, load, index, offset,
921 access_offset, access_size,
922 inout, type->type,
923 type->access | access);
924 } else {
925 /* This is a strided load. We have to load N things separately.
926 * This is the single column of a row-major matrix case.
927 */
928 vtn_assert(type->stride > type_size);
929 vtn_assert(type->stride % type_size == 0);
930
931 nir_ssa_def *per_comp[4];
932 for (unsigned i = 0; i < elems; i++) {
933 nir_ssa_def *elem_offset =
934 nir_iadd_imm(&b->nb, offset, i * type->stride);
935 struct vtn_ssa_value *comp, temp_val;
936 if (!load) {
937 temp_val.def = nir_channel(&b->nb, (*inout)->def, i);
938 temp_val.type = glsl_scalar_type(base_type);
939 }
940 comp = &temp_val;
941 _vtn_load_store_tail(b, op, load, index, elem_offset,
942 access_offset, access_size,
943 &comp, glsl_scalar_type(base_type),
944 type->access | access);
945 per_comp[i] = comp->def;
946 }
947
948 if (load) {
949 if (*inout == NULL)
950 *inout = vtn_create_ssa_value(b, type->type);
951 (*inout)->def = nir_vec(&b->nb, per_comp, elems);
952 }
953 }
954 }
955 return;
956
957 case GLSL_TYPE_ARRAY: {
958 unsigned elems = glsl_get_length(type->type);
959 for (unsigned i = 0; i < elems; i++) {
960 nir_ssa_def *elem_off =
961 nir_iadd_imm(&b->nb, offset, i * type->stride);
962 _vtn_block_load_store(b, op, load, index, elem_off,
963 access_offset, access_size,
964 type->array_element,
965 type->array_element->access | access,
966 &(*inout)->elems[i]);
967 }
968 return;
969 }
970
971 case GLSL_TYPE_INTERFACE:
972 case GLSL_TYPE_STRUCT: {
973 unsigned elems = glsl_get_length(type->type);
974 for (unsigned i = 0; i < elems; i++) {
975 nir_ssa_def *elem_off =
976 nir_iadd_imm(&b->nb, offset, type->offsets[i]);
977 _vtn_block_load_store(b, op, load, index, elem_off,
978 access_offset, access_size,
979 type->members[i],
980 type->members[i]->access | access,
981 &(*inout)->elems[i]);
982 }
983 return;
984 }
985
986 default:
987 vtn_fail("Invalid block member type");
988 }
989 }
990
991 static struct vtn_ssa_value *
vtn_block_load(struct vtn_builder * b,struct vtn_pointer * src)992 vtn_block_load(struct vtn_builder *b, struct vtn_pointer *src)
993 {
994 nir_intrinsic_op op;
995 unsigned access_offset = 0, access_size = 0;
996 switch (src->mode) {
997 case vtn_variable_mode_ubo:
998 op = nir_intrinsic_load_ubo;
999 break;
1000 case vtn_variable_mode_ssbo:
1001 op = nir_intrinsic_load_ssbo;
1002 break;
1003 case vtn_variable_mode_push_constant:
1004 op = nir_intrinsic_load_push_constant;
1005 access_size = b->shader->num_uniforms;
1006 break;
1007 case vtn_variable_mode_workgroup:
1008 op = nir_intrinsic_load_shared;
1009 break;
1010 default:
1011 vtn_fail("Invalid block variable mode");
1012 }
1013
1014 nir_ssa_def *offset, *index = NULL;
1015 offset = vtn_pointer_to_offset(b, src, &index);
1016
1017 struct vtn_ssa_value *value = vtn_create_ssa_value(b, src->type->type);
1018 _vtn_block_load_store(b, op, true, index, offset,
1019 access_offset, access_size,
1020 src->type, src->access, &value);
1021 return value;
1022 }
1023
1024 static void
vtn_block_store(struct vtn_builder * b,struct vtn_ssa_value * src,struct vtn_pointer * dst)1025 vtn_block_store(struct vtn_builder *b, struct vtn_ssa_value *src,
1026 struct vtn_pointer *dst)
1027 {
1028 nir_intrinsic_op op;
1029 switch (dst->mode) {
1030 case vtn_variable_mode_ssbo:
1031 op = nir_intrinsic_store_ssbo;
1032 break;
1033 case vtn_variable_mode_workgroup:
1034 op = nir_intrinsic_store_shared;
1035 break;
1036 default:
1037 vtn_fail("Invalid block variable mode");
1038 }
1039
1040 nir_ssa_def *offset, *index = NULL;
1041 offset = vtn_pointer_to_offset(b, dst, &index);
1042
1043 _vtn_block_load_store(b, op, false, index, offset,
1044 0, 0, dst->type, dst->access, &src);
1045 }
1046
1047 static void
_vtn_variable_load_store(struct vtn_builder * b,bool load,struct vtn_pointer * ptr,enum gl_access_qualifier access,struct vtn_ssa_value ** inout)1048 _vtn_variable_load_store(struct vtn_builder *b, bool load,
1049 struct vtn_pointer *ptr,
1050 enum gl_access_qualifier access,
1051 struct vtn_ssa_value **inout)
1052 {
1053 if (ptr->mode == vtn_variable_mode_uniform) {
1054 if (ptr->type->base_type == vtn_base_type_image ||
1055 ptr->type->base_type == vtn_base_type_sampler) {
1056 /* See also our handling of OpTypeSampler and OpTypeImage */
1057 vtn_assert(load);
1058 (*inout)->def = vtn_pointer_to_ssa(b, ptr);
1059 return;
1060 } else if (ptr->type->base_type == vtn_base_type_sampled_image) {
1061 /* See also our handling of OpTypeSampledImage */
1062 vtn_assert(load);
1063 struct vtn_sampled_image si = {
1064 .image = vtn_pointer_to_deref(b, ptr),
1065 .sampler = vtn_pointer_to_deref(b, ptr),
1066 };
1067 (*inout)->def = vtn_sampled_image_to_nir_ssa(b, si);
1068 return;
1069 }
1070 }
1071
1072 enum glsl_base_type base_type = glsl_get_base_type(ptr->type->type);
1073 switch (base_type) {
1074 case GLSL_TYPE_UINT:
1075 case GLSL_TYPE_INT:
1076 case GLSL_TYPE_UINT16:
1077 case GLSL_TYPE_INT16:
1078 case GLSL_TYPE_UINT8:
1079 case GLSL_TYPE_INT8:
1080 case GLSL_TYPE_UINT64:
1081 case GLSL_TYPE_INT64:
1082 case GLSL_TYPE_FLOAT:
1083 case GLSL_TYPE_FLOAT16:
1084 case GLSL_TYPE_BOOL:
1085 case GLSL_TYPE_DOUBLE:
1086 if (glsl_type_is_vector_or_scalar(ptr->type->type)) {
1087 /* We hit a vector or scalar; go ahead and emit the load[s] */
1088 nir_deref_instr *deref = vtn_pointer_to_deref(b, ptr);
1089 if (vtn_mode_is_cross_invocation(b, ptr->mode)) {
1090 /* If it's cross-invocation, we call nir_load/store_deref
1091 * directly. The vtn_local_load/store helpers are too clever and
1092 * do magic to avoid array derefs of vectors. That magic is both
1093 * less efficient than the direct load/store and, in the case of
1094 * stores, is broken because it creates a race condition if two
1095 * threads are writing to different components of the same vector
1096 * due to the load+insert+store it uses to emulate the array
1097 * deref.
1098 */
1099 if (load) {
1100 (*inout)->def = nir_load_deref_with_access(&b->nb, deref,
1101 ptr->type->access | access);
1102 } else {
1103 nir_store_deref_with_access(&b->nb, deref, (*inout)->def, ~0,
1104 ptr->type->access | access);
1105 }
1106 } else {
1107 if (load) {
1108 *inout = vtn_local_load(b, deref, ptr->type->access | access);
1109 } else {
1110 vtn_local_store(b, *inout, deref, ptr->type->access | access);
1111 }
1112 }
1113 return;
1114 }
1115 /* Fall through */
1116
1117 case GLSL_TYPE_INTERFACE:
1118 case GLSL_TYPE_ARRAY:
1119 case GLSL_TYPE_STRUCT: {
1120 unsigned elems = glsl_get_length(ptr->type->type);
1121 struct vtn_access_chain chain = {
1122 .length = 1,
1123 .link = {
1124 { .mode = vtn_access_mode_literal, },
1125 }
1126 };
1127 for (unsigned i = 0; i < elems; i++) {
1128 chain.link[0].id = i;
1129 struct vtn_pointer *elem = vtn_pointer_dereference(b, ptr, &chain);
1130 _vtn_variable_load_store(b, load, elem, ptr->type->access | access,
1131 &(*inout)->elems[i]);
1132 }
1133 return;
1134 }
1135
1136 default:
1137 vtn_fail("Invalid access chain type");
1138 }
1139 }
1140
1141 struct vtn_ssa_value *
vtn_variable_load(struct vtn_builder * b,struct vtn_pointer * src)1142 vtn_variable_load(struct vtn_builder *b, struct vtn_pointer *src)
1143 {
1144 if (vtn_pointer_uses_ssa_offset(b, src)) {
1145 return vtn_block_load(b, src);
1146 } else {
1147 struct vtn_ssa_value *val = vtn_create_ssa_value(b, src->type->type);
1148 _vtn_variable_load_store(b, true, src, src->access, &val);
1149 return val;
1150 }
1151 }
1152
1153 void
vtn_variable_store(struct vtn_builder * b,struct vtn_ssa_value * src,struct vtn_pointer * dest)1154 vtn_variable_store(struct vtn_builder *b, struct vtn_ssa_value *src,
1155 struct vtn_pointer *dest)
1156 {
1157 if (vtn_pointer_uses_ssa_offset(b, dest)) {
1158 vtn_assert(dest->mode == vtn_variable_mode_ssbo ||
1159 dest->mode == vtn_variable_mode_workgroup);
1160 vtn_block_store(b, src, dest);
1161 } else {
1162 _vtn_variable_load_store(b, false, dest, dest->access, &src);
1163 }
1164 }
1165
1166 static void
_vtn_variable_copy(struct vtn_builder * b,struct vtn_pointer * dest,struct vtn_pointer * src)1167 _vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
1168 struct vtn_pointer *src)
1169 {
1170 vtn_assert(glsl_get_bare_type(src->type->type) ==
1171 glsl_get_bare_type(dest->type->type));
1172 enum glsl_base_type base_type = glsl_get_base_type(src->type->type);
1173 switch (base_type) {
1174 case GLSL_TYPE_UINT:
1175 case GLSL_TYPE_INT:
1176 case GLSL_TYPE_UINT16:
1177 case GLSL_TYPE_INT16:
1178 case GLSL_TYPE_UINT8:
1179 case GLSL_TYPE_INT8:
1180 case GLSL_TYPE_UINT64:
1181 case GLSL_TYPE_INT64:
1182 case GLSL_TYPE_FLOAT:
1183 case GLSL_TYPE_FLOAT16:
1184 case GLSL_TYPE_DOUBLE:
1185 case GLSL_TYPE_BOOL:
1186 /* At this point, we have a scalar, vector, or matrix so we know that
1187 * there cannot be any structure splitting still in the way. By
1188 * stopping at the matrix level rather than the vector level, we
1189 * ensure that matrices get loaded in the optimal way even if they
1190 * are storred row-major in a UBO.
1191 */
1192 vtn_variable_store(b, vtn_variable_load(b, src), dest);
1193 return;
1194
1195 case GLSL_TYPE_INTERFACE:
1196 case GLSL_TYPE_ARRAY:
1197 case GLSL_TYPE_STRUCT: {
1198 struct vtn_access_chain chain = {
1199 .length = 1,
1200 .link = {
1201 { .mode = vtn_access_mode_literal, },
1202 }
1203 };
1204 unsigned elems = glsl_get_length(src->type->type);
1205 for (unsigned i = 0; i < elems; i++) {
1206 chain.link[0].id = i;
1207 struct vtn_pointer *src_elem =
1208 vtn_pointer_dereference(b, src, &chain);
1209 struct vtn_pointer *dest_elem =
1210 vtn_pointer_dereference(b, dest, &chain);
1211
1212 _vtn_variable_copy(b, dest_elem, src_elem);
1213 }
1214 return;
1215 }
1216
1217 default:
1218 vtn_fail("Invalid access chain type");
1219 }
1220 }
1221
1222 static void
vtn_variable_copy(struct vtn_builder * b,struct vtn_pointer * dest,struct vtn_pointer * src)1223 vtn_variable_copy(struct vtn_builder *b, struct vtn_pointer *dest,
1224 struct vtn_pointer *src)
1225 {
1226 /* TODO: At some point, we should add a special-case for when we can
1227 * just emit a copy_var intrinsic.
1228 */
1229 _vtn_variable_copy(b, dest, src);
1230 }
1231
1232 static void
set_mode_system_value(struct vtn_builder * b,nir_variable_mode * mode)1233 set_mode_system_value(struct vtn_builder *b, nir_variable_mode *mode)
1234 {
1235 vtn_assert(*mode == nir_var_system_value || *mode == nir_var_shader_in);
1236 *mode = nir_var_system_value;
1237 }
1238
1239 static void
vtn_get_builtin_location(struct vtn_builder * b,SpvBuiltIn builtin,int * location,nir_variable_mode * mode)1240 vtn_get_builtin_location(struct vtn_builder *b,
1241 SpvBuiltIn builtin, int *location,
1242 nir_variable_mode *mode)
1243 {
1244 switch (builtin) {
1245 case SpvBuiltInPosition:
1246 *location = VARYING_SLOT_POS;
1247 break;
1248 case SpvBuiltInPointSize:
1249 *location = VARYING_SLOT_PSIZ;
1250 break;
1251 case SpvBuiltInClipDistance:
1252 *location = VARYING_SLOT_CLIP_DIST0; /* XXX CLIP_DIST1? */
1253 break;
1254 case SpvBuiltInCullDistance:
1255 *location = VARYING_SLOT_CULL_DIST0;
1256 break;
1257 case SpvBuiltInVertexId:
1258 case SpvBuiltInVertexIndex:
1259 /* The Vulkan spec defines VertexIndex to be non-zero-based and doesn't
1260 * allow VertexId. The ARB_gl_spirv spec defines VertexId to be the
1261 * same as gl_VertexID, which is non-zero-based, and removes
1262 * VertexIndex. Since they're both defined to be non-zero-based, we use
1263 * SYSTEM_VALUE_VERTEX_ID for both.
1264 */
1265 *location = SYSTEM_VALUE_VERTEX_ID;
1266 set_mode_system_value(b, mode);
1267 break;
1268 case SpvBuiltInInstanceIndex:
1269 *location = SYSTEM_VALUE_INSTANCE_INDEX;
1270 set_mode_system_value(b, mode);
1271 break;
1272 case SpvBuiltInInstanceId:
1273 *location = SYSTEM_VALUE_INSTANCE_ID;
1274 set_mode_system_value(b, mode);
1275 break;
1276 case SpvBuiltInPrimitiveId:
1277 if (b->shader->info.stage == MESA_SHADER_FRAGMENT) {
1278 vtn_assert(*mode == nir_var_shader_in);
1279 *location = VARYING_SLOT_PRIMITIVE_ID;
1280 } else if (*mode == nir_var_shader_out) {
1281 *location = VARYING_SLOT_PRIMITIVE_ID;
1282 } else {
1283 *location = SYSTEM_VALUE_PRIMITIVE_ID;
1284 set_mode_system_value(b, mode);
1285 }
1286 break;
1287 case SpvBuiltInInvocationId:
1288 *location = SYSTEM_VALUE_INVOCATION_ID;
1289 set_mode_system_value(b, mode);
1290 break;
1291 case SpvBuiltInLayer:
1292 *location = VARYING_SLOT_LAYER;
1293 if (b->shader->info.stage == MESA_SHADER_FRAGMENT)
1294 *mode = nir_var_shader_in;
1295 else if (b->shader->info.stage == MESA_SHADER_GEOMETRY)
1296 *mode = nir_var_shader_out;
1297 else if (b->options && b->options->caps.shader_viewport_index_layer &&
1298 (b->shader->info.stage == MESA_SHADER_VERTEX ||
1299 b->shader->info.stage == MESA_SHADER_TESS_EVAL))
1300 *mode = nir_var_shader_out;
1301 else
1302 vtn_fail("invalid stage for SpvBuiltInLayer");
1303 break;
1304 case SpvBuiltInViewportIndex:
1305 *location = VARYING_SLOT_VIEWPORT;
1306 if (b->shader->info.stage == MESA_SHADER_GEOMETRY)
1307 *mode = nir_var_shader_out;
1308 else if (b->options && b->options->caps.shader_viewport_index_layer &&
1309 (b->shader->info.stage == MESA_SHADER_VERTEX ||
1310 b->shader->info.stage == MESA_SHADER_TESS_EVAL))
1311 *mode = nir_var_shader_out;
1312 else if (b->shader->info.stage == MESA_SHADER_FRAGMENT)
1313 *mode = nir_var_shader_in;
1314 else
1315 vtn_fail("invalid stage for SpvBuiltInViewportIndex");
1316 break;
1317 case SpvBuiltInTessLevelOuter:
1318 *location = VARYING_SLOT_TESS_LEVEL_OUTER;
1319 break;
1320 case SpvBuiltInTessLevelInner:
1321 *location = VARYING_SLOT_TESS_LEVEL_INNER;
1322 break;
1323 case SpvBuiltInTessCoord:
1324 *location = SYSTEM_VALUE_TESS_COORD;
1325 set_mode_system_value(b, mode);
1326 break;
1327 case SpvBuiltInPatchVertices:
1328 *location = SYSTEM_VALUE_VERTICES_IN;
1329 set_mode_system_value(b, mode);
1330 break;
1331 case SpvBuiltInFragCoord:
1332 vtn_assert(*mode == nir_var_shader_in);
1333 if (b->options && b->options->frag_coord_is_sysval) {
1334 *mode = nir_var_system_value;
1335 *location = SYSTEM_VALUE_FRAG_COORD;
1336 } else {
1337 *location = VARYING_SLOT_POS;
1338 }
1339 break;
1340 case SpvBuiltInPointCoord:
1341 *location = VARYING_SLOT_PNTC;
1342 vtn_assert(*mode == nir_var_shader_in);
1343 break;
1344 case SpvBuiltInFrontFacing:
1345 *location = SYSTEM_VALUE_FRONT_FACE;
1346 set_mode_system_value(b, mode);
1347 break;
1348 case SpvBuiltInSampleId:
1349 *location = SYSTEM_VALUE_SAMPLE_ID;
1350 set_mode_system_value(b, mode);
1351 break;
1352 case SpvBuiltInSamplePosition:
1353 *location = SYSTEM_VALUE_SAMPLE_POS;
1354 set_mode_system_value(b, mode);
1355 break;
1356 case SpvBuiltInSampleMask:
1357 if (*mode == nir_var_shader_out) {
1358 *location = FRAG_RESULT_SAMPLE_MASK;
1359 } else {
1360 *location = SYSTEM_VALUE_SAMPLE_MASK_IN;
1361 set_mode_system_value(b, mode);
1362 }
1363 break;
1364 case SpvBuiltInFragDepth:
1365 *location = FRAG_RESULT_DEPTH;
1366 vtn_assert(*mode == nir_var_shader_out);
1367 break;
1368 case SpvBuiltInHelperInvocation:
1369 *location = SYSTEM_VALUE_HELPER_INVOCATION;
1370 set_mode_system_value(b, mode);
1371 break;
1372 case SpvBuiltInNumWorkgroups:
1373 *location = SYSTEM_VALUE_NUM_WORK_GROUPS;
1374 set_mode_system_value(b, mode);
1375 break;
1376 case SpvBuiltInWorkgroupSize:
1377 *location = SYSTEM_VALUE_LOCAL_GROUP_SIZE;
1378 set_mode_system_value(b, mode);
1379 break;
1380 case SpvBuiltInWorkgroupId:
1381 *location = SYSTEM_VALUE_WORK_GROUP_ID;
1382 set_mode_system_value(b, mode);
1383 break;
1384 case SpvBuiltInLocalInvocationId:
1385 *location = SYSTEM_VALUE_LOCAL_INVOCATION_ID;
1386 set_mode_system_value(b, mode);
1387 break;
1388 case SpvBuiltInLocalInvocationIndex:
1389 *location = SYSTEM_VALUE_LOCAL_INVOCATION_INDEX;
1390 set_mode_system_value(b, mode);
1391 break;
1392 case SpvBuiltInGlobalInvocationId:
1393 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_ID;
1394 set_mode_system_value(b, mode);
1395 break;
1396 case SpvBuiltInGlobalLinearId:
1397 *location = SYSTEM_VALUE_GLOBAL_INVOCATION_INDEX;
1398 set_mode_system_value(b, mode);
1399 break;
1400 case SpvBuiltInBaseVertex:
1401 /* OpenGL gl_BaseVertex (SYSTEM_VALUE_BASE_VERTEX) is not the same
1402 * semantic as Vulkan BaseVertex (SYSTEM_VALUE_FIRST_VERTEX).
1403 */
1404 if (b->options->environment == NIR_SPIRV_OPENGL)
1405 *location = SYSTEM_VALUE_BASE_VERTEX;
1406 else
1407 *location = SYSTEM_VALUE_FIRST_VERTEX;
1408 set_mode_system_value(b, mode);
1409 break;
1410 case SpvBuiltInBaseInstance:
1411 *location = SYSTEM_VALUE_BASE_INSTANCE;
1412 set_mode_system_value(b, mode);
1413 break;
1414 case SpvBuiltInDrawIndex:
1415 *location = SYSTEM_VALUE_DRAW_ID;
1416 set_mode_system_value(b, mode);
1417 break;
1418 case SpvBuiltInSubgroupSize:
1419 *location = SYSTEM_VALUE_SUBGROUP_SIZE;
1420 set_mode_system_value(b, mode);
1421 break;
1422 case SpvBuiltInSubgroupId:
1423 *location = SYSTEM_VALUE_SUBGROUP_ID;
1424 set_mode_system_value(b, mode);
1425 break;
1426 case SpvBuiltInSubgroupLocalInvocationId:
1427 *location = SYSTEM_VALUE_SUBGROUP_INVOCATION;
1428 set_mode_system_value(b, mode);
1429 break;
1430 case SpvBuiltInNumSubgroups:
1431 *location = SYSTEM_VALUE_NUM_SUBGROUPS;
1432 set_mode_system_value(b, mode);
1433 break;
1434 case SpvBuiltInDeviceIndex:
1435 *location = SYSTEM_VALUE_DEVICE_INDEX;
1436 set_mode_system_value(b, mode);
1437 break;
1438 case SpvBuiltInViewIndex:
1439 *location = SYSTEM_VALUE_VIEW_INDEX;
1440 set_mode_system_value(b, mode);
1441 break;
1442 case SpvBuiltInSubgroupEqMask:
1443 *location = SYSTEM_VALUE_SUBGROUP_EQ_MASK,
1444 set_mode_system_value(b, mode);
1445 break;
1446 case SpvBuiltInSubgroupGeMask:
1447 *location = SYSTEM_VALUE_SUBGROUP_GE_MASK,
1448 set_mode_system_value(b, mode);
1449 break;
1450 case SpvBuiltInSubgroupGtMask:
1451 *location = SYSTEM_VALUE_SUBGROUP_GT_MASK,
1452 set_mode_system_value(b, mode);
1453 break;
1454 case SpvBuiltInSubgroupLeMask:
1455 *location = SYSTEM_VALUE_SUBGROUP_LE_MASK,
1456 set_mode_system_value(b, mode);
1457 break;
1458 case SpvBuiltInSubgroupLtMask:
1459 *location = SYSTEM_VALUE_SUBGROUP_LT_MASK,
1460 set_mode_system_value(b, mode);
1461 break;
1462 case SpvBuiltInFragStencilRefEXT:
1463 *location = FRAG_RESULT_STENCIL;
1464 vtn_assert(*mode == nir_var_shader_out);
1465 break;
1466 case SpvBuiltInWorkDim:
1467 *location = SYSTEM_VALUE_WORK_DIM;
1468 set_mode_system_value(b, mode);
1469 break;
1470 case SpvBuiltInGlobalSize:
1471 *location = SYSTEM_VALUE_GLOBAL_GROUP_SIZE;
1472 set_mode_system_value(b, mode);
1473 break;
1474 case SpvBuiltInBaryCoordNoPerspAMD:
1475 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_PIXEL;
1476 set_mode_system_value(b, mode);
1477 break;
1478 case SpvBuiltInBaryCoordNoPerspCentroidAMD:
1479 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_CENTROID;
1480 set_mode_system_value(b, mode);
1481 break;
1482 case SpvBuiltInBaryCoordNoPerspSampleAMD:
1483 *location = SYSTEM_VALUE_BARYCENTRIC_LINEAR_SAMPLE;
1484 set_mode_system_value(b, mode);
1485 break;
1486 case SpvBuiltInBaryCoordSmoothAMD:
1487 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_PIXEL;
1488 set_mode_system_value(b, mode);
1489 break;
1490 case SpvBuiltInBaryCoordSmoothCentroidAMD:
1491 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_CENTROID;
1492 set_mode_system_value(b, mode);
1493 break;
1494 case SpvBuiltInBaryCoordSmoothSampleAMD:
1495 *location = SYSTEM_VALUE_BARYCENTRIC_PERSP_SAMPLE;
1496 set_mode_system_value(b, mode);
1497 break;
1498 case SpvBuiltInBaryCoordPullModelAMD:
1499 *location = SYSTEM_VALUE_BARYCENTRIC_PULL_MODEL;
1500 set_mode_system_value(b, mode);
1501 break;
1502 default:
1503 vtn_fail("Unsupported builtin: %s (%u)",
1504 spirv_builtin_to_string(builtin), builtin);
1505 }
1506 }
1507
1508 static void
apply_var_decoration(struct vtn_builder * b,struct nir_variable_data * var_data,const struct vtn_decoration * dec)1509 apply_var_decoration(struct vtn_builder *b,
1510 struct nir_variable_data *var_data,
1511 const struct vtn_decoration *dec)
1512 {
1513 switch (dec->decoration) {
1514 case SpvDecorationRelaxedPrecision:
1515 break; /* FIXME: Do nothing with this for now. */
1516 case SpvDecorationNoPerspective:
1517 var_data->interpolation = INTERP_MODE_NOPERSPECTIVE;
1518 break;
1519 case SpvDecorationFlat:
1520 var_data->interpolation = INTERP_MODE_FLAT;
1521 break;
1522 case SpvDecorationExplicitInterpAMD:
1523 var_data->interpolation = INTERP_MODE_EXPLICIT;
1524 break;
1525 case SpvDecorationCentroid:
1526 var_data->centroid = true;
1527 break;
1528 case SpvDecorationSample:
1529 var_data->sample = true;
1530 break;
1531 case SpvDecorationInvariant:
1532 var_data->invariant = true;
1533 break;
1534 case SpvDecorationConstant:
1535 var_data->read_only = true;
1536 break;
1537 case SpvDecorationNonReadable:
1538 var_data->access |= ACCESS_NON_READABLE;
1539 break;
1540 case SpvDecorationNonWritable:
1541 var_data->read_only = true;
1542 var_data->access |= ACCESS_NON_WRITEABLE;
1543 break;
1544 case SpvDecorationRestrict:
1545 var_data->access |= ACCESS_RESTRICT;
1546 break;
1547 case SpvDecorationAliased:
1548 var_data->access &= ~ACCESS_RESTRICT;
1549 break;
1550 case SpvDecorationVolatile:
1551 var_data->access |= ACCESS_VOLATILE;
1552 break;
1553 case SpvDecorationCoherent:
1554 var_data->access |= ACCESS_COHERENT;
1555 break;
1556 case SpvDecorationComponent:
1557 var_data->location_frac = dec->operands[0];
1558 break;
1559 case SpvDecorationIndex:
1560 var_data->index = dec->operands[0];
1561 break;
1562 case SpvDecorationBuiltIn: {
1563 SpvBuiltIn builtin = dec->operands[0];
1564
1565 nir_variable_mode mode = var_data->mode;
1566 vtn_get_builtin_location(b, builtin, &var_data->location, &mode);
1567 var_data->mode = mode;
1568
1569 switch (builtin) {
1570 case SpvBuiltInTessLevelOuter:
1571 case SpvBuiltInTessLevelInner:
1572 case SpvBuiltInClipDistance:
1573 case SpvBuiltInCullDistance:
1574 var_data->compact = true;
1575 break;
1576 default:
1577 break;
1578 }
1579 }
1580
1581 case SpvDecorationSpecId:
1582 case SpvDecorationRowMajor:
1583 case SpvDecorationColMajor:
1584 case SpvDecorationMatrixStride:
1585 case SpvDecorationUniform:
1586 case SpvDecorationUniformId:
1587 case SpvDecorationLinkageAttributes:
1588 break; /* Do nothing with these here */
1589
1590 case SpvDecorationPatch:
1591 var_data->patch = true;
1592 break;
1593
1594 case SpvDecorationLocation:
1595 vtn_fail("Handled above");
1596
1597 case SpvDecorationBlock:
1598 case SpvDecorationBufferBlock:
1599 case SpvDecorationArrayStride:
1600 case SpvDecorationGLSLShared:
1601 case SpvDecorationGLSLPacked:
1602 break; /* These can apply to a type but we don't care about them */
1603
1604 case SpvDecorationBinding:
1605 case SpvDecorationDescriptorSet:
1606 case SpvDecorationNoContraction:
1607 case SpvDecorationInputAttachmentIndex:
1608 vtn_warn("Decoration not allowed for variable or structure member: %s",
1609 spirv_decoration_to_string(dec->decoration));
1610 break;
1611
1612 case SpvDecorationXfbBuffer:
1613 var_data->explicit_xfb_buffer = true;
1614 var_data->xfb.buffer = dec->operands[0];
1615 var_data->always_active_io = true;
1616 break;
1617 case SpvDecorationXfbStride:
1618 var_data->explicit_xfb_stride = true;
1619 var_data->xfb.stride = dec->operands[0];
1620 break;
1621 case SpvDecorationOffset:
1622 var_data->explicit_offset = true;
1623 var_data->offset = dec->operands[0];
1624 break;
1625
1626 case SpvDecorationStream:
1627 var_data->stream = dec->operands[0];
1628 break;
1629
1630 case SpvDecorationCPacked:
1631 case SpvDecorationSaturatedConversion:
1632 case SpvDecorationFuncParamAttr:
1633 case SpvDecorationFPRoundingMode:
1634 case SpvDecorationFPFastMathMode:
1635 case SpvDecorationAlignment:
1636 if (b->shader->info.stage != MESA_SHADER_KERNEL) {
1637 vtn_warn("Decoration only allowed for CL-style kernels: %s",
1638 spirv_decoration_to_string(dec->decoration));
1639 }
1640 break;
1641
1642 case SpvDecorationUserSemantic:
1643 case SpvDecorationUserTypeGOOGLE:
1644 /* User semantic decorations can safely be ignored by the driver. */
1645 break;
1646
1647 case SpvDecorationRestrictPointerEXT:
1648 case SpvDecorationAliasedPointerEXT:
1649 /* TODO: We should actually plumb alias information through NIR. */
1650 break;
1651
1652 default:
1653 vtn_fail_with_decoration("Unhandled decoration", dec->decoration);
1654 }
1655 }
1656
1657 static void
var_is_patch_cb(struct vtn_builder * b,struct vtn_value * val,int member,const struct vtn_decoration * dec,void * out_is_patch)1658 var_is_patch_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1659 const struct vtn_decoration *dec, void *out_is_patch)
1660 {
1661 if (dec->decoration == SpvDecorationPatch) {
1662 *((bool *) out_is_patch) = true;
1663 }
1664 }
1665
1666 static void
var_decoration_cb(struct vtn_builder * b,struct vtn_value * val,int member,const struct vtn_decoration * dec,void * void_var)1667 var_decoration_cb(struct vtn_builder *b, struct vtn_value *val, int member,
1668 const struct vtn_decoration *dec, void *void_var)
1669 {
1670 struct vtn_variable *vtn_var = void_var;
1671
1672 /* Handle decorations that apply to a vtn_variable as a whole */
1673 switch (dec->decoration) {
1674 case SpvDecorationBinding:
1675 vtn_var->binding = dec->operands[0];
1676 vtn_var->explicit_binding = true;
1677 return;
1678 case SpvDecorationDescriptorSet:
1679 vtn_var->descriptor_set = dec->operands[0];
1680 return;
1681 case SpvDecorationInputAttachmentIndex:
1682 vtn_var->input_attachment_index = dec->operands[0];
1683 return;
1684 case SpvDecorationPatch:
1685 vtn_var->patch = true;
1686 break;
1687 case SpvDecorationOffset:
1688 vtn_var->offset = dec->operands[0];
1689 break;
1690 case SpvDecorationNonWritable:
1691 vtn_var->access |= ACCESS_NON_WRITEABLE;
1692 break;
1693 case SpvDecorationNonReadable:
1694 vtn_var->access |= ACCESS_NON_READABLE;
1695 break;
1696 case SpvDecorationVolatile:
1697 vtn_var->access |= ACCESS_VOLATILE;
1698 break;
1699 case SpvDecorationCoherent:
1700 vtn_var->access |= ACCESS_COHERENT;
1701 break;
1702 case SpvDecorationCounterBuffer:
1703 /* Counter buffer decorations can safely be ignored by the driver. */
1704 return;
1705 default:
1706 break;
1707 }
1708
1709 if (val->value_type == vtn_value_type_pointer) {
1710 assert(val->pointer->var == void_var);
1711 assert(member == -1);
1712 } else {
1713 assert(val->value_type == vtn_value_type_type);
1714 }
1715
1716 /* Location is odd. If applied to a split structure, we have to walk the
1717 * whole thing and accumulate the location. It's easier to handle as a
1718 * special case.
1719 */
1720 if (dec->decoration == SpvDecorationLocation) {
1721 unsigned location = dec->operands[0];
1722 if (b->shader->info.stage == MESA_SHADER_FRAGMENT &&
1723 vtn_var->mode == vtn_variable_mode_output) {
1724 location += FRAG_RESULT_DATA0;
1725 } else if (b->shader->info.stage == MESA_SHADER_VERTEX &&
1726 vtn_var->mode == vtn_variable_mode_input) {
1727 location += VERT_ATTRIB_GENERIC0;
1728 } else if (vtn_var->mode == vtn_variable_mode_input ||
1729 vtn_var->mode == vtn_variable_mode_output) {
1730 location += vtn_var->patch ? VARYING_SLOT_PATCH0 : VARYING_SLOT_VAR0;
1731 } else if (vtn_var->mode != vtn_variable_mode_uniform) {
1732 vtn_warn("Location must be on input, output, uniform, sampler or "
1733 "image variable");
1734 return;
1735 }
1736
1737 if (vtn_var->var->num_members == 0) {
1738 /* This handles the member and lone variable cases */
1739 vtn_var->var->data.location = location;
1740 } else {
1741 /* This handles the structure member case */
1742 assert(vtn_var->var->members);
1743
1744 if (member == -1)
1745 vtn_var->base_location = location;
1746 else
1747 vtn_var->var->members[member].location = location;
1748 }
1749
1750 return;
1751 } else {
1752 if (vtn_var->var) {
1753 if (vtn_var->var->num_members == 0) {
1754 /* We call this function on types as well as variables and not all
1755 * struct types get split so we can end up having stray member
1756 * decorations; just ignore them.
1757 */
1758 if (member == -1)
1759 apply_var_decoration(b, &vtn_var->var->data, dec);
1760 } else if (member >= 0) {
1761 /* Member decorations must come from a type */
1762 assert(val->value_type == vtn_value_type_type);
1763 apply_var_decoration(b, &vtn_var->var->members[member], dec);
1764 } else {
1765 unsigned length =
1766 glsl_get_length(glsl_without_array(vtn_var->type->type));
1767 for (unsigned i = 0; i < length; i++)
1768 apply_var_decoration(b, &vtn_var->var->members[i], dec);
1769 }
1770 } else {
1771 /* A few variables, those with external storage, have no actual
1772 * nir_variables associated with them. Fortunately, all decorations
1773 * we care about for those variables are on the type only.
1774 */
1775 vtn_assert(vtn_var->mode == vtn_variable_mode_ubo ||
1776 vtn_var->mode == vtn_variable_mode_ssbo ||
1777 vtn_var->mode == vtn_variable_mode_push_constant);
1778 }
1779 }
1780 }
1781
1782 enum vtn_variable_mode
vtn_storage_class_to_mode(struct vtn_builder * b,SpvStorageClass class,struct vtn_type * interface_type,nir_variable_mode * nir_mode_out)1783 vtn_storage_class_to_mode(struct vtn_builder *b,
1784 SpvStorageClass class,
1785 struct vtn_type *interface_type,
1786 nir_variable_mode *nir_mode_out)
1787 {
1788 enum vtn_variable_mode mode;
1789 nir_variable_mode nir_mode;
1790 switch (class) {
1791 case SpvStorageClassUniform:
1792 /* Assume it's an UBO if we lack the interface_type. */
1793 if (!interface_type || interface_type->block) {
1794 mode = vtn_variable_mode_ubo;
1795 nir_mode = nir_var_mem_ubo;
1796 } else if (interface_type->buffer_block) {
1797 mode = vtn_variable_mode_ssbo;
1798 nir_mode = nir_var_mem_ssbo;
1799 } else {
1800 /* Default-block uniforms, coming from gl_spirv */
1801 mode = vtn_variable_mode_uniform;
1802 nir_mode = nir_var_uniform;
1803 }
1804 break;
1805 case SpvStorageClassStorageBuffer:
1806 mode = vtn_variable_mode_ssbo;
1807 nir_mode = nir_var_mem_ssbo;
1808 break;
1809 case SpvStorageClassPhysicalStorageBuffer:
1810 mode = vtn_variable_mode_phys_ssbo;
1811 nir_mode = nir_var_mem_global;
1812 break;
1813 case SpvStorageClassUniformConstant:
1814 if (b->shader->info.stage == MESA_SHADER_KERNEL) {
1815 if (b->options->constant_as_global) {
1816 mode = vtn_variable_mode_cross_workgroup;
1817 nir_mode = nir_var_mem_global;
1818 } else {
1819 mode = vtn_variable_mode_ubo;
1820 nir_mode = nir_var_mem_ubo;
1821 }
1822 } else {
1823 mode = vtn_variable_mode_uniform;
1824 nir_mode = nir_var_uniform;
1825 }
1826 break;
1827 case SpvStorageClassPushConstant:
1828 mode = vtn_variable_mode_push_constant;
1829 nir_mode = nir_var_uniform;
1830 break;
1831 case SpvStorageClassInput:
1832 mode = vtn_variable_mode_input;
1833 nir_mode = nir_var_shader_in;
1834 break;
1835 case SpvStorageClassOutput:
1836 mode = vtn_variable_mode_output;
1837 nir_mode = nir_var_shader_out;
1838 break;
1839 case SpvStorageClassPrivate:
1840 mode = vtn_variable_mode_private;
1841 nir_mode = nir_var_shader_temp;
1842 break;
1843 case SpvStorageClassFunction:
1844 mode = vtn_variable_mode_function;
1845 nir_mode = nir_var_function_temp;
1846 break;
1847 case SpvStorageClassWorkgroup:
1848 mode = vtn_variable_mode_workgroup;
1849 nir_mode = nir_var_mem_shared;
1850 break;
1851 case SpvStorageClassAtomicCounter:
1852 mode = vtn_variable_mode_atomic_counter;
1853 nir_mode = nir_var_uniform;
1854 break;
1855 case SpvStorageClassCrossWorkgroup:
1856 mode = vtn_variable_mode_cross_workgroup;
1857 nir_mode = nir_var_mem_global;
1858 break;
1859 case SpvStorageClassImage:
1860 mode = vtn_variable_mode_image;
1861 nir_mode = nir_var_mem_ubo;
1862 break;
1863 case SpvStorageClassGeneric:
1864 default:
1865 vtn_fail("Unhandled variable storage class: %s (%u)",
1866 spirv_storageclass_to_string(class), class);
1867 }
1868
1869 if (nir_mode_out)
1870 *nir_mode_out = nir_mode;
1871
1872 return mode;
1873 }
1874
1875 nir_address_format
vtn_mode_to_address_format(struct vtn_builder * b,enum vtn_variable_mode mode)1876 vtn_mode_to_address_format(struct vtn_builder *b, enum vtn_variable_mode mode)
1877 {
1878 switch (mode) {
1879 case vtn_variable_mode_ubo:
1880 return b->options->ubo_addr_format;
1881
1882 case vtn_variable_mode_ssbo:
1883 return b->options->ssbo_addr_format;
1884
1885 case vtn_variable_mode_phys_ssbo:
1886 return b->options->phys_ssbo_addr_format;
1887
1888 case vtn_variable_mode_push_constant:
1889 return b->options->push_const_addr_format;
1890
1891 case vtn_variable_mode_workgroup:
1892 return b->options->shared_addr_format;
1893
1894 case vtn_variable_mode_cross_workgroup:
1895 return b->options->global_addr_format;
1896
1897 case vtn_variable_mode_function:
1898 if (b->physical_ptrs)
1899 return b->options->temp_addr_format;
1900 /* Fall through. */
1901
1902 case vtn_variable_mode_private:
1903 case vtn_variable_mode_uniform:
1904 case vtn_variable_mode_atomic_counter:
1905 case vtn_variable_mode_input:
1906 case vtn_variable_mode_output:
1907 case vtn_variable_mode_image:
1908 return nir_address_format_logical;
1909 }
1910
1911 unreachable("Invalid variable mode");
1912 }
1913
1914 nir_ssa_def *
vtn_pointer_to_ssa(struct vtn_builder * b,struct vtn_pointer * ptr)1915 vtn_pointer_to_ssa(struct vtn_builder *b, struct vtn_pointer *ptr)
1916 {
1917 if (vtn_pointer_uses_ssa_offset(b, ptr)) {
1918 /* This pointer needs to have a pointer type with actual storage */
1919 vtn_assert(ptr->ptr_type);
1920 vtn_assert(ptr->ptr_type->type);
1921
1922 if (!ptr->offset) {
1923 /* If we don't have an offset then we must be a pointer to the variable
1924 * itself.
1925 */
1926 vtn_assert(!ptr->offset && !ptr->block_index);
1927
1928 struct vtn_access_chain chain = {
1929 .length = 0,
1930 };
1931 ptr = vtn_ssa_offset_pointer_dereference(b, ptr, &chain);
1932 }
1933
1934 vtn_assert(ptr->offset);
1935 if (ptr->block_index) {
1936 vtn_assert(ptr->mode == vtn_variable_mode_ubo ||
1937 ptr->mode == vtn_variable_mode_ssbo);
1938 return nir_vec2(&b->nb, ptr->block_index, ptr->offset);
1939 } else {
1940 vtn_assert(ptr->mode == vtn_variable_mode_workgroup);
1941 return ptr->offset;
1942 }
1943 } else {
1944 if (vtn_pointer_is_external_block(b, ptr) &&
1945 vtn_type_contains_block(b, ptr->type) &&
1946 ptr->mode != vtn_variable_mode_phys_ssbo) {
1947 /* In this case, we're looking for a block index and not an actual
1948 * deref.
1949 *
1950 * For PhysicalStorageBuffer pointers, we don't have a block index
1951 * at all because we get the pointer directly from the client. This
1952 * assumes that there will never be a SSBO binding variable using the
1953 * PhysicalStorageBuffer storage class. This assumption appears
1954 * to be correct according to the Vulkan spec because the table,
1955 * "Shader Resource and Storage Class Correspondence," the only the
1956 * Uniform storage class with BufferBlock or the StorageBuffer
1957 * storage class with Block can be used.
1958 */
1959 if (!ptr->block_index) {
1960 /* If we don't have a block_index then we must be a pointer to the
1961 * variable itself.
1962 */
1963 vtn_assert(!ptr->deref);
1964
1965 struct vtn_access_chain chain = {
1966 .length = 0,
1967 };
1968 ptr = vtn_nir_deref_pointer_dereference(b, ptr, &chain);
1969 }
1970
1971 return ptr->block_index;
1972 } else {
1973 return &vtn_pointer_to_deref(b, ptr)->dest.ssa;
1974 }
1975 }
1976 }
1977
1978 struct vtn_pointer *
vtn_pointer_from_ssa(struct vtn_builder * b,nir_ssa_def * ssa,struct vtn_type * ptr_type)1979 vtn_pointer_from_ssa(struct vtn_builder *b, nir_ssa_def *ssa,
1980 struct vtn_type *ptr_type)
1981 {
1982 vtn_assert(ptr_type->base_type == vtn_base_type_pointer);
1983
1984 struct vtn_pointer *ptr = rzalloc(b, struct vtn_pointer);
1985 struct vtn_type *without_array =
1986 vtn_type_without_array(ptr_type->deref);
1987
1988 nir_variable_mode nir_mode;
1989 ptr->mode = vtn_storage_class_to_mode(b, ptr_type->storage_class,
1990 without_array, &nir_mode);
1991 ptr->type = ptr_type->deref;
1992 ptr->ptr_type = ptr_type;
1993
1994 if (vtn_pointer_uses_ssa_offset(b, ptr)) {
1995 /* This pointer type needs to have actual storage */
1996 vtn_assert(ptr_type->type);
1997 if (ptr->mode == vtn_variable_mode_ubo ||
1998 ptr->mode == vtn_variable_mode_ssbo) {
1999 vtn_assert(ssa->num_components == 2);
2000 ptr->block_index = nir_channel(&b->nb, ssa, 0);
2001 ptr->offset = nir_channel(&b->nb, ssa, 1);
2002 } else {
2003 vtn_assert(ssa->num_components == 1);
2004 ptr->block_index = NULL;
2005 ptr->offset = ssa;
2006 }
2007 } else {
2008 const struct glsl_type *deref_type =
2009 vtn_type_get_nir_type(b, ptr_type->deref, ptr->mode);
2010 if (!vtn_pointer_is_external_block(b, ptr)) {
2011 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode,
2012 deref_type, ptr_type->stride);
2013 } else if (vtn_type_contains_block(b, ptr->type) &&
2014 ptr->mode != vtn_variable_mode_phys_ssbo) {
2015 /* This is a pointer to somewhere in an array of blocks, not a
2016 * pointer to somewhere inside the block. Set the block index
2017 * instead of making a cast.
2018 */
2019 ptr->block_index = ssa;
2020 } else {
2021 /* This is a pointer to something internal or a pointer inside a
2022 * block. It's just a regular cast.
2023 *
2024 * For PhysicalStorageBuffer pointers, we don't have a block index
2025 * at all because we get the pointer directly from the client. This
2026 * assumes that there will never be a SSBO binding variable using the
2027 * PhysicalStorageBuffer storage class. This assumption appears
2028 * to be correct according to the Vulkan spec because the table,
2029 * "Shader Resource and Storage Class Correspondence," the only the
2030 * Uniform storage class with BufferBlock or the StorageBuffer
2031 * storage class with Block can be used.
2032 */
2033 ptr->deref = nir_build_deref_cast(&b->nb, ssa, nir_mode,
2034 deref_type, ptr_type->stride);
2035 ptr->deref->dest.ssa.num_components =
2036 glsl_get_vector_elements(ptr_type->type);
2037 ptr->deref->dest.ssa.bit_size = glsl_get_bit_size(ptr_type->type);
2038 }
2039 }
2040
2041 return ptr;
2042 }
2043
2044 static bool
is_per_vertex_inout(const struct vtn_variable * var,gl_shader_stage stage)2045 is_per_vertex_inout(const struct vtn_variable *var, gl_shader_stage stage)
2046 {
2047 if (var->patch || !glsl_type_is_array(var->type->type))
2048 return false;
2049
2050 if (var->mode == vtn_variable_mode_input) {
2051 return stage == MESA_SHADER_TESS_CTRL ||
2052 stage == MESA_SHADER_TESS_EVAL ||
2053 stage == MESA_SHADER_GEOMETRY;
2054 }
2055
2056 if (var->mode == vtn_variable_mode_output)
2057 return stage == MESA_SHADER_TESS_CTRL;
2058
2059 return false;
2060 }
2061
2062 static void
assign_missing_member_locations(struct vtn_variable * var)2063 assign_missing_member_locations(struct vtn_variable *var)
2064 {
2065 unsigned length =
2066 glsl_get_length(glsl_without_array(var->type->type));
2067 int location = var->base_location;
2068
2069 for (unsigned i = 0; i < length; i++) {
2070 /* From the Vulkan spec:
2071 *
2072 * “If the structure type is a Block but without a Location, then each
2073 * of its members must have a Location decoration.”
2074 *
2075 */
2076 if (var->type->block) {
2077 assert(var->base_location != -1 ||
2078 var->var->members[i].location != -1);
2079 }
2080
2081 /* From the Vulkan spec:
2082 *
2083 * “Any member with its own Location decoration is assigned that
2084 * location. Each remaining member is assigned the location after the
2085 * immediately preceding member in declaration order.”
2086 */
2087 if (var->var->members[i].location != -1)
2088 location = var->var->members[i].location;
2089 else
2090 var->var->members[i].location = location;
2091
2092 /* Below we use type instead of interface_type, because interface_type
2093 * is only available when it is a Block. This code also supports
2094 * input/outputs that are just structs
2095 */
2096 const struct glsl_type *member_type =
2097 glsl_get_struct_field(glsl_without_array(var->type->type), i);
2098
2099 location +=
2100 glsl_count_attribute_slots(member_type,
2101 false /* is_gl_vertex_input */);
2102 }
2103 }
2104
2105
2106 static void
vtn_create_variable(struct vtn_builder * b,struct vtn_value * val,struct vtn_type * ptr_type,SpvStorageClass storage_class,nir_constant * const_initializer,nir_variable * var_initializer)2107 vtn_create_variable(struct vtn_builder *b, struct vtn_value *val,
2108 struct vtn_type *ptr_type, SpvStorageClass storage_class,
2109 nir_constant *const_initializer, nir_variable *var_initializer)
2110 {
2111 vtn_assert(ptr_type->base_type == vtn_base_type_pointer);
2112 struct vtn_type *type = ptr_type->deref;
2113
2114 struct vtn_type *without_array = vtn_type_without_array(ptr_type->deref);
2115
2116 enum vtn_variable_mode mode;
2117 nir_variable_mode nir_mode;
2118 mode = vtn_storage_class_to_mode(b, storage_class, without_array, &nir_mode);
2119
2120 switch (mode) {
2121 case vtn_variable_mode_ubo:
2122 /* There's no other way to get vtn_variable_mode_ubo */
2123 vtn_assert(without_array->block);
2124 b->shader->info.num_ubos++;
2125 break;
2126 case vtn_variable_mode_ssbo:
2127 if (storage_class == SpvStorageClassStorageBuffer &&
2128 !without_array->block) {
2129 if (b->variable_pointers) {
2130 vtn_fail("Variables in the StorageBuffer storage class must "
2131 "have a struct type with the Block decoration");
2132 } else {
2133 /* If variable pointers are not present, it's still malformed
2134 * SPIR-V but we can parse it and do the right thing anyway.
2135 * Since some of the 8-bit storage tests have bugs in this are,
2136 * just make it a warning for now.
2137 */
2138 vtn_warn("Variables in the StorageBuffer storage class must "
2139 "have a struct type with the Block decoration");
2140 }
2141 }
2142 b->shader->info.num_ssbos++;
2143 break;
2144 case vtn_variable_mode_uniform:
2145 if (without_array->base_type == vtn_base_type_image) {
2146 if (glsl_type_is_image(without_array->glsl_image))
2147 b->shader->info.num_images++;
2148 else if (glsl_type_is_sampler(without_array->glsl_image))
2149 b->shader->info.num_textures++;
2150 }
2151 break;
2152 case vtn_variable_mode_push_constant:
2153 b->shader->num_uniforms = vtn_type_block_size(b, type);
2154 break;
2155
2156 case vtn_variable_mode_image:
2157 vtn_fail("Cannot create a variable with the Image storage class");
2158 break;
2159
2160 case vtn_variable_mode_phys_ssbo:
2161 vtn_fail("Cannot create a variable with the "
2162 "PhysicalStorageBuffer storage class");
2163 break;
2164
2165 default:
2166 /* No tallying is needed */
2167 break;
2168 }
2169
2170 struct vtn_variable *var = rzalloc(b, struct vtn_variable);
2171 var->type = type;
2172 var->mode = mode;
2173 var->base_location = -1;
2174
2175 val->pointer = rzalloc(b, struct vtn_pointer);
2176 val->pointer->mode = var->mode;
2177 val->pointer->type = var->type;
2178 val->pointer->ptr_type = ptr_type;
2179 val->pointer->var = var;
2180 val->pointer->access = var->type->access;
2181
2182 switch (var->mode) {
2183 case vtn_variable_mode_function:
2184 case vtn_variable_mode_private:
2185 case vtn_variable_mode_uniform:
2186 case vtn_variable_mode_atomic_counter:
2187 /* For these, we create the variable normally */
2188 var->var = rzalloc(b->shader, nir_variable);
2189 var->var->name = ralloc_strdup(var->var, val->name);
2190 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2191 var->var->data.mode = nir_mode;
2192 var->var->data.location = -1;
2193 var->var->interface_type = NULL;
2194 break;
2195
2196 case vtn_variable_mode_ubo:
2197 case vtn_variable_mode_ssbo:
2198 var->var = rzalloc(b->shader, nir_variable);
2199 var->var->name = ralloc_strdup(var->var, val->name);
2200
2201 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2202 var->var->interface_type = var->var->type;
2203
2204 var->var->data.mode = nir_mode;
2205 var->var->data.location = -1;
2206
2207 break;
2208
2209 case vtn_variable_mode_workgroup:
2210 /* Create the variable normally */
2211 var->var = rzalloc(b->shader, nir_variable);
2212 var->var->name = ralloc_strdup(var->var, val->name);
2213 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2214 var->var->data.mode = nir_var_mem_shared;
2215 break;
2216
2217 case vtn_variable_mode_input:
2218 case vtn_variable_mode_output: {
2219 /* In order to know whether or not we're a per-vertex inout, we need
2220 * the patch qualifier. This means walking the variable decorations
2221 * early before we actually create any variables. Not a big deal.
2222 *
2223 * GLSLang really likes to place decorations in the most interior
2224 * thing it possibly can. In particular, if you have a struct, it
2225 * will place the patch decorations on the struct members. This
2226 * should be handled by the variable splitting below just fine.
2227 *
2228 * If you have an array-of-struct, things get even more weird as it
2229 * will place the patch decorations on the struct even though it's
2230 * inside an array and some of the members being patch and others not
2231 * makes no sense whatsoever. Since the only sensible thing is for
2232 * it to be all or nothing, we'll call it patch if any of the members
2233 * are declared patch.
2234 */
2235 var->patch = false;
2236 vtn_foreach_decoration(b, val, var_is_patch_cb, &var->patch);
2237 if (glsl_type_is_array(var->type->type) &&
2238 glsl_type_is_struct_or_ifc(without_array->type)) {
2239 vtn_foreach_decoration(b, vtn_value(b, without_array->id,
2240 vtn_value_type_type),
2241 var_is_patch_cb, &var->patch);
2242 }
2243
2244 /* For inputs and outputs, we immediately split structures. This
2245 * is for a couple of reasons. For one, builtins may all come in
2246 * a struct and we really want those split out into separate
2247 * variables. For another, interpolation qualifiers can be
2248 * applied to members of the top-level struct ane we need to be
2249 * able to preserve that information.
2250 */
2251
2252 struct vtn_type *per_vertex_type = var->type;
2253 if (is_per_vertex_inout(var, b->shader->info.stage)) {
2254 /* In Geometry shaders (and some tessellation), inputs come
2255 * in per-vertex arrays. However, some builtins come in
2256 * non-per-vertex, hence the need for the is_array check. In
2257 * any case, there are no non-builtin arrays allowed so this
2258 * check should be sufficient.
2259 */
2260 per_vertex_type = var->type->array_element;
2261 }
2262
2263 var->var = rzalloc(b->shader, nir_variable);
2264 var->var->name = ralloc_strdup(var->var, val->name);
2265 var->var->type = vtn_type_get_nir_type(b, var->type, var->mode);
2266 var->var->data.mode = nir_mode;
2267 var->var->data.patch = var->patch;
2268
2269 /* Figure out the interface block type. */
2270 struct vtn_type *iface_type = per_vertex_type;
2271 if (var->mode == vtn_variable_mode_output &&
2272 (b->shader->info.stage == MESA_SHADER_VERTEX ||
2273 b->shader->info.stage == MESA_SHADER_TESS_EVAL ||
2274 b->shader->info.stage == MESA_SHADER_GEOMETRY)) {
2275 /* For vertex data outputs, we can end up with arrays of blocks for
2276 * transform feedback where each array element corresponds to a
2277 * different XFB output buffer.
2278 */
2279 while (iface_type->base_type == vtn_base_type_array)
2280 iface_type = iface_type->array_element;
2281 }
2282 if (iface_type->base_type == vtn_base_type_struct && iface_type->block)
2283 var->var->interface_type = vtn_type_get_nir_type(b, iface_type,
2284 var->mode);
2285
2286 if (per_vertex_type->base_type == vtn_base_type_struct &&
2287 per_vertex_type->block) {
2288 /* It's a struct. Set it up as per-member. */
2289 var->var->num_members = glsl_get_length(per_vertex_type->type);
2290 var->var->members = rzalloc_array(var->var, struct nir_variable_data,
2291 var->var->num_members);
2292
2293 for (unsigned i = 0; i < var->var->num_members; i++) {
2294 var->var->members[i].mode = nir_mode;
2295 var->var->members[i].patch = var->patch;
2296 var->var->members[i].location = -1;
2297 }
2298 }
2299
2300 /* For inputs and outputs, we need to grab locations and builtin
2301 * information from the per-vertex type.
2302 */
2303 vtn_foreach_decoration(b, vtn_value(b, per_vertex_type->id,
2304 vtn_value_type_type),
2305 var_decoration_cb, var);
2306 break;
2307 }
2308
2309 case vtn_variable_mode_push_constant:
2310 case vtn_variable_mode_cross_workgroup:
2311 /* These don't need actual variables. */
2312 break;
2313
2314 case vtn_variable_mode_image:
2315 case vtn_variable_mode_phys_ssbo:
2316 unreachable("Should have been caught before");
2317 }
2318
2319 /* We can only have one type of initializer */
2320 assert(!(const_initializer && var_initializer));
2321 if (const_initializer) {
2322 var->var->constant_initializer =
2323 nir_constant_clone(const_initializer, var->var);
2324 }
2325 if (var_initializer)
2326 var->var->pointer_initializer = var_initializer;
2327
2328 if (var->mode == vtn_variable_mode_uniform ||
2329 var->mode == vtn_variable_mode_ssbo) {
2330 /* SSBOs and images are assumed to not alias in the Simple, GLSL and Vulkan memory models */
2331 var->var->data.access |= b->mem_model != SpvMemoryModelOpenCL ? ACCESS_RESTRICT : 0;
2332 }
2333
2334 vtn_foreach_decoration(b, val, var_decoration_cb, var);
2335 vtn_foreach_decoration(b, val, ptr_decoration_cb, val->pointer);
2336
2337 /* Propagate access flags from the OpVariable decorations. */
2338 val->pointer->access |= var->access;
2339
2340 if ((var->mode == vtn_variable_mode_input ||
2341 var->mode == vtn_variable_mode_output) &&
2342 var->var->members) {
2343 assign_missing_member_locations(var);
2344 }
2345
2346 if (var->mode == vtn_variable_mode_uniform ||
2347 var->mode == vtn_variable_mode_ubo ||
2348 var->mode == vtn_variable_mode_ssbo ||
2349 var->mode == vtn_variable_mode_atomic_counter) {
2350 /* XXX: We still need the binding information in the nir_variable
2351 * for these. We should fix that.
2352 */
2353 var->var->data.binding = var->binding;
2354 var->var->data.explicit_binding = var->explicit_binding;
2355 var->var->data.descriptor_set = var->descriptor_set;
2356 var->var->data.index = var->input_attachment_index;
2357 var->var->data.offset = var->offset;
2358
2359 if (glsl_type_is_image(glsl_without_array(var->var->type)))
2360 var->var->data.image.format = without_array->image_format;
2361 }
2362
2363 if (var->mode == vtn_variable_mode_function) {
2364 vtn_assert(var->var != NULL && var->var->members == NULL);
2365 nir_function_impl_add_variable(b->nb.impl, var->var);
2366 } else if (var->var) {
2367 nir_shader_add_variable(b->shader, var->var);
2368 } else {
2369 vtn_assert(vtn_pointer_is_external_block(b, val->pointer));
2370 }
2371 }
2372
2373 static void
vtn_assert_types_equal(struct vtn_builder * b,SpvOp opcode,struct vtn_type * dst_type,struct vtn_type * src_type)2374 vtn_assert_types_equal(struct vtn_builder *b, SpvOp opcode,
2375 struct vtn_type *dst_type,
2376 struct vtn_type *src_type)
2377 {
2378 if (dst_type->id == src_type->id)
2379 return;
2380
2381 if (vtn_types_compatible(b, dst_type, src_type)) {
2382 /* Early versions of GLSLang would re-emit types unnecessarily and you
2383 * would end up with OpLoad, OpStore, or OpCopyMemory opcodes which have
2384 * mismatched source and destination types.
2385 *
2386 * https://github.com/KhronosGroup/glslang/issues/304
2387 * https://github.com/KhronosGroup/glslang/issues/307
2388 * https://bugs.freedesktop.org/show_bug.cgi?id=104338
2389 * https://bugs.freedesktop.org/show_bug.cgi?id=104424
2390 */
2391 vtn_warn("Source and destination types of %s do not have the same "
2392 "ID (but are compatible): %u vs %u",
2393 spirv_op_to_string(opcode), dst_type->id, src_type->id);
2394 return;
2395 }
2396
2397 vtn_fail("Source and destination types of %s do not match: %s vs. %s",
2398 spirv_op_to_string(opcode),
2399 glsl_get_type_name(dst_type->type),
2400 glsl_get_type_name(src_type->type));
2401 }
2402
2403 static nir_ssa_def *
nir_shrink_zero_pad_vec(nir_builder * b,nir_ssa_def * val,unsigned num_components)2404 nir_shrink_zero_pad_vec(nir_builder *b, nir_ssa_def *val,
2405 unsigned num_components)
2406 {
2407 if (val->num_components == num_components)
2408 return val;
2409
2410 nir_ssa_def *comps[NIR_MAX_VEC_COMPONENTS];
2411 for (unsigned i = 0; i < num_components; i++) {
2412 if (i < val->num_components)
2413 comps[i] = nir_channel(b, val, i);
2414 else
2415 comps[i] = nir_imm_intN_t(b, 0, val->bit_size);
2416 }
2417 return nir_vec(b, comps, num_components);
2418 }
2419
2420 static nir_ssa_def *
nir_sloppy_bitcast(nir_builder * b,nir_ssa_def * val,const struct glsl_type * type)2421 nir_sloppy_bitcast(nir_builder *b, nir_ssa_def *val,
2422 const struct glsl_type *type)
2423 {
2424 const unsigned num_components = glsl_get_vector_elements(type);
2425 const unsigned bit_size = glsl_get_bit_size(type);
2426
2427 /* First, zero-pad to ensure that the value is big enough that when we
2428 * bit-cast it, we don't loose anything.
2429 */
2430 if (val->bit_size < bit_size) {
2431 const unsigned src_num_components_needed =
2432 vtn_align_u32(val->num_components, bit_size / val->bit_size);
2433 val = nir_shrink_zero_pad_vec(b, val, src_num_components_needed);
2434 }
2435
2436 val = nir_bitcast_vector(b, val, bit_size);
2437
2438 return nir_shrink_zero_pad_vec(b, val, num_components);
2439 }
2440
2441 static bool
vtn_get_mem_operands(struct vtn_builder * b,const uint32_t * w,unsigned count,unsigned * idx,SpvMemoryAccessMask * access,unsigned * alignment,SpvScope * dest_scope,SpvScope * src_scope)2442 vtn_get_mem_operands(struct vtn_builder *b, const uint32_t *w, unsigned count,
2443 unsigned *idx, SpvMemoryAccessMask *access, unsigned *alignment,
2444 SpvScope *dest_scope, SpvScope *src_scope)
2445 {
2446 *access = 0;
2447 *alignment = 0;
2448 if (*idx >= count)
2449 return false;
2450
2451 *access = w[(*idx)++];
2452 if (*access & SpvMemoryAccessAlignedMask) {
2453 vtn_assert(*idx < count);
2454 *alignment = w[(*idx)++];
2455 }
2456
2457 if (*access & SpvMemoryAccessMakePointerAvailableMask) {
2458 vtn_assert(*idx < count);
2459 vtn_assert(dest_scope);
2460 *dest_scope = vtn_constant_uint(b, w[(*idx)++]);
2461 }
2462
2463 if (*access & SpvMemoryAccessMakePointerVisibleMask) {
2464 vtn_assert(*idx < count);
2465 vtn_assert(src_scope);
2466 *src_scope = vtn_constant_uint(b, w[(*idx)++]);
2467 }
2468
2469 return true;
2470 }
2471
2472 static void
ptr_nonuniform_workaround_cb(struct vtn_builder * b,struct vtn_value * val,int member,const struct vtn_decoration * dec,void * void_ptr)2473 ptr_nonuniform_workaround_cb(struct vtn_builder *b, struct vtn_value *val,
2474 int member, const struct vtn_decoration *dec, void *void_ptr)
2475 {
2476 enum gl_access_qualifier *access = void_ptr;
2477
2478 switch (dec->decoration) {
2479 case SpvDecorationNonUniformEXT:
2480 *access |= ACCESS_NON_UNIFORM;
2481 break;
2482
2483 default:
2484 break;
2485 }
2486 }
2487
2488 void
vtn_handle_variables(struct vtn_builder * b,SpvOp opcode,const uint32_t * w,unsigned count)2489 vtn_handle_variables(struct vtn_builder *b, SpvOp opcode,
2490 const uint32_t *w, unsigned count)
2491 {
2492 switch (opcode) {
2493 case SpvOpUndef: {
2494 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_undef);
2495 val->type = vtn_get_type(b, w[1]);
2496 break;
2497 }
2498
2499 case SpvOpVariable: {
2500 struct vtn_type *ptr_type = vtn_get_type(b, w[1]);
2501
2502 struct vtn_value *val = vtn_push_value(b, w[2], vtn_value_type_pointer);
2503
2504 SpvStorageClass storage_class = w[3];
2505 nir_constant *const_initializer = NULL;
2506 nir_variable *var_initializer = NULL;
2507 if (count > 4) {
2508 struct vtn_value *init = vtn_untyped_value(b, w[4]);
2509 switch (init->value_type) {
2510 case vtn_value_type_constant:
2511 const_initializer = init->constant;
2512 break;
2513 case vtn_value_type_pointer:
2514 var_initializer = init->pointer->var->var;
2515 break;
2516 default:
2517 vtn_fail("SPIR-V variable initializer %u must be constant or pointer",
2518 w[4]);
2519 }
2520 }
2521
2522 vtn_create_variable(b, val, ptr_type, storage_class, const_initializer, var_initializer);
2523
2524 break;
2525 }
2526
2527 case SpvOpAccessChain:
2528 case SpvOpPtrAccessChain:
2529 case SpvOpInBoundsAccessChain:
2530 case SpvOpInBoundsPtrAccessChain: {
2531 struct vtn_access_chain *chain = vtn_access_chain_create(b, count - 4);
2532 enum gl_access_qualifier access = 0;
2533 chain->ptr_as_array = (opcode == SpvOpPtrAccessChain || opcode == SpvOpInBoundsPtrAccessChain);
2534
2535 unsigned idx = 0;
2536 for (int i = 4; i < count; i++) {
2537 struct vtn_value *link_val = vtn_untyped_value(b, w[i]);
2538 if (link_val->value_type == vtn_value_type_constant) {
2539 chain->link[idx].mode = vtn_access_mode_literal;
2540 chain->link[idx].id = vtn_constant_int(b, w[i]);
2541 } else {
2542 chain->link[idx].mode = vtn_access_mode_id;
2543 chain->link[idx].id = w[i];
2544 }
2545
2546 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */
2547 vtn_foreach_decoration(b, link_val, ptr_nonuniform_workaround_cb, &access);
2548
2549 idx++;
2550 }
2551
2552 struct vtn_type *ptr_type = vtn_get_type(b, w[1]);
2553 struct vtn_pointer *base =
2554 vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
2555
2556 /* Workaround for https://gitlab.freedesktop.org/mesa/mesa/-/issues/3406 */
2557 access |= base->access & ACCESS_NON_UNIFORM;
2558
2559 struct vtn_pointer *ptr = vtn_pointer_dereference(b, base, chain);
2560 ptr->ptr_type = ptr_type;
2561 ptr->access |= access;
2562 vtn_push_pointer(b, w[2], ptr);
2563 break;
2564 }
2565
2566 case SpvOpCopyMemory: {
2567 struct vtn_value *dest = vtn_value(b, w[1], vtn_value_type_pointer);
2568 struct vtn_value *src = vtn_value(b, w[2], vtn_value_type_pointer);
2569
2570 vtn_assert_types_equal(b, opcode, dest->type->deref, src->type->deref);
2571
2572 vtn_variable_copy(b, dest->pointer, src->pointer);
2573 break;
2574 }
2575
2576 case SpvOpLoad: {
2577 struct vtn_type *res_type = vtn_get_type(b, w[1]);
2578 struct vtn_value *src_val = vtn_value(b, w[3], vtn_value_type_pointer);
2579 struct vtn_pointer *src = src_val->pointer;
2580
2581 vtn_assert_types_equal(b, opcode, res_type, src_val->type->deref);
2582
2583 unsigned idx = 4, alignment;
2584 SpvMemoryAccessMask access;
2585 SpvScope scope;
2586 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, NULL, &scope);
2587 if (access & SpvMemoryAccessMakePointerVisibleMask) {
2588 SpvMemorySemanticsMask semantics =
2589 SpvMemorySemanticsMakeVisibleMask |
2590 vtn_storage_class_to_memory_semantics(src->ptr_type->storage_class);
2591 vtn_emit_memory_barrier(b, scope, semantics);
2592 }
2593
2594 vtn_push_ssa_value(b, w[2], vtn_variable_load(b, src));
2595 break;
2596 }
2597
2598 case SpvOpStore: {
2599 struct vtn_value *dest_val = vtn_value(b, w[1], vtn_value_type_pointer);
2600 struct vtn_pointer *dest = dest_val->pointer;
2601 struct vtn_value *src_val = vtn_untyped_value(b, w[2]);
2602
2603 /* OpStore requires us to actually have a storage type */
2604 vtn_fail_if(dest->type->type == NULL,
2605 "Invalid destination type for OpStore");
2606
2607 if (glsl_get_base_type(dest->type->type) == GLSL_TYPE_BOOL &&
2608 glsl_get_base_type(src_val->type->type) == GLSL_TYPE_UINT) {
2609 /* Early versions of GLSLang would use uint types for UBOs/SSBOs but
2610 * would then store them to a local variable as bool. Work around
2611 * the issue by doing an implicit conversion.
2612 *
2613 * https://github.com/KhronosGroup/glslang/issues/170
2614 * https://bugs.freedesktop.org/show_bug.cgi?id=104424
2615 */
2616 vtn_warn("OpStore of value of type OpTypeInt to a pointer to type "
2617 "OpTypeBool. Doing an implicit conversion to work around "
2618 "the problem.");
2619 struct vtn_ssa_value *bool_ssa =
2620 vtn_create_ssa_value(b, dest->type->type);
2621 bool_ssa->def = nir_i2b(&b->nb, vtn_ssa_value(b, w[2])->def);
2622 vtn_variable_store(b, bool_ssa, dest);
2623 break;
2624 }
2625
2626 vtn_assert_types_equal(b, opcode, dest_val->type->deref, src_val->type);
2627
2628 unsigned idx = 3, alignment;
2629 SpvMemoryAccessMask access;
2630 SpvScope scope;
2631 vtn_get_mem_operands(b, w, count, &idx, &access, &alignment, &scope, NULL);
2632
2633 struct vtn_ssa_value *src = vtn_ssa_value(b, w[2]);
2634 vtn_variable_store(b, src, dest);
2635
2636 if (access & SpvMemoryAccessMakePointerAvailableMask) {
2637 SpvMemorySemanticsMask semantics =
2638 SpvMemorySemanticsMakeAvailableMask |
2639 vtn_storage_class_to_memory_semantics(dest->ptr_type->storage_class);
2640 vtn_emit_memory_barrier(b, scope, semantics);
2641 }
2642 break;
2643 }
2644
2645 case SpvOpArrayLength: {
2646 struct vtn_pointer *ptr =
2647 vtn_value(b, w[3], vtn_value_type_pointer)->pointer;
2648 const uint32_t field = w[4];
2649
2650 vtn_fail_if(ptr->type->base_type != vtn_base_type_struct,
2651 "OpArrayLength must take a pointer to a structure type");
2652 vtn_fail_if(field != ptr->type->length - 1 ||
2653 ptr->type->members[field]->base_type != vtn_base_type_array,
2654 "OpArrayLength must reference the last memeber of the "
2655 "structure and that must be an array");
2656
2657 const uint32_t offset = ptr->type->offsets[field];
2658 const uint32_t stride = ptr->type->members[field]->stride;
2659
2660 if (!ptr->block_index) {
2661 struct vtn_access_chain chain = {
2662 .length = 0,
2663 };
2664 ptr = vtn_pointer_dereference(b, ptr, &chain);
2665 vtn_assert(ptr->block_index);
2666 }
2667
2668 nir_intrinsic_instr *instr =
2669 nir_intrinsic_instr_create(b->nb.shader,
2670 nir_intrinsic_get_buffer_size);
2671 instr->src[0] = nir_src_for_ssa(ptr->block_index);
2672 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
2673 nir_builder_instr_insert(&b->nb, &instr->instr);
2674 nir_ssa_def *buf_size = &instr->dest.ssa;
2675
2676 /* array_length = max(buffer_size - offset, 0) / stride */
2677 nir_ssa_def *array_length =
2678 nir_idiv(&b->nb,
2679 nir_imax(&b->nb,
2680 nir_isub(&b->nb,
2681 buf_size,
2682 nir_imm_int(&b->nb, offset)),
2683 nir_imm_int(&b->nb, 0u)),
2684 nir_imm_int(&b->nb, stride));
2685
2686 vtn_push_nir_ssa(b, w[2], array_length);
2687 break;
2688 }
2689
2690 case SpvOpConvertPtrToU: {
2691 struct vtn_type *u_type = vtn_get_type(b, w[1]);
2692 struct vtn_type *ptr_type = vtn_get_value_type(b, w[3]);
2693
2694 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer ||
2695 ptr_type->type == NULL,
2696 "OpConvertPtrToU can only be used on physical pointers");
2697
2698 vtn_fail_if(u_type->base_type != vtn_base_type_vector &&
2699 u_type->base_type != vtn_base_type_scalar,
2700 "OpConvertPtrToU can only be used to cast to a vector or "
2701 "scalar type");
2702
2703 /* The pointer will be converted to an SSA value automatically */
2704 nir_ssa_def *ptr = vtn_get_nir_ssa(b, w[3]);
2705 nir_ssa_def *u = nir_sloppy_bitcast(&b->nb, ptr, u_type->type);
2706 vtn_push_nir_ssa(b, w[2], u);
2707 break;
2708 }
2709
2710 case SpvOpConvertUToPtr: {
2711 struct vtn_type *ptr_type = vtn_get_type(b, w[1]);
2712 struct vtn_type *u_type = vtn_get_value_type(b, w[3]);
2713
2714 vtn_fail_if(ptr_type->base_type != vtn_base_type_pointer ||
2715 ptr_type->type == NULL,
2716 "OpConvertUToPtr can only be used on physical pointers");
2717
2718 vtn_fail_if(u_type->base_type != vtn_base_type_vector &&
2719 u_type->base_type != vtn_base_type_scalar,
2720 "OpConvertUToPtr can only be used to cast from a vector or "
2721 "scalar type");
2722
2723 nir_ssa_def *u = vtn_get_nir_ssa(b, w[3]);
2724 nir_ssa_def *ptr = nir_sloppy_bitcast(&b->nb, u, ptr_type->type);
2725 vtn_push_pointer(b, w[2], vtn_pointer_from_ssa(b, ptr, ptr_type));
2726 break;
2727 }
2728
2729 case SpvOpCopyMemorySized:
2730 default:
2731 vtn_fail_with_opcode("Unhandled opcode", opcode);
2732 }
2733 }
2734