1 /*
2 * Copyright © 2014 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 *
23 * Authors:
24 * Connor Abbott (cwabbott0@gmail.com)
25 *
26 */
27
28 #include "float64_glsl.h"
29 #include "glsl_to_nir.h"
30 #include "ir_visitor.h"
31 #include "ir_hierarchical_visitor.h"
32 #include "ir.h"
33 #include "ir_optimization.h"
34 #include "program.h"
35 #include "compiler/nir/nir_control_flow.h"
36 #include "compiler/nir/nir_builder.h"
37 #include "compiler/nir/nir_builtin_builder.h"
38 #include "compiler/nir/nir_deref.h"
39 #include "main/errors.h"
40 #include "main/mtypes.h"
41 #include "main/shaderobj.h"
42 #include "main/context.h"
43 #include "util/u_math.h"
44
45 /*
46 * pass to lower GLSL IR to NIR
47 *
48 * This will lower variable dereferences to loads/stores of corresponding
49 * variables in NIR - the variables will be converted to registers in a later
50 * pass.
51 */
52
53 namespace {
54
55 class nir_visitor : public ir_visitor
56 {
57 public:
58 nir_visitor(gl_context *ctx, nir_shader *shader);
59 ~nir_visitor();
60
61 virtual void visit(ir_variable *);
62 virtual void visit(ir_function *);
63 virtual void visit(ir_function_signature *);
64 virtual void visit(ir_loop *);
65 virtual void visit(ir_if *);
66 virtual void visit(ir_discard *);
67 virtual void visit(ir_demote *);
68 virtual void visit(ir_loop_jump *);
69 virtual void visit(ir_return *);
70 virtual void visit(ir_call *);
71 virtual void visit(ir_assignment *);
72 virtual void visit(ir_emit_vertex *);
73 virtual void visit(ir_end_primitive *);
74 virtual void visit(ir_expression *);
75 virtual void visit(ir_swizzle *);
76 virtual void visit(ir_texture *);
77 virtual void visit(ir_constant *);
78 virtual void visit(ir_dereference_variable *);
79 virtual void visit(ir_dereference_record *);
80 virtual void visit(ir_dereference_array *);
81 virtual void visit(ir_barrier *);
82
83 void create_function(ir_function_signature *ir);
84
85 private:
86 void add_instr(nir_instr *instr, unsigned num_components, unsigned bit_size);
87 nir_ssa_def *evaluate_rvalue(ir_rvalue *ir);
88
89 nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def **srcs);
90 nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1);
91 nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
92 nir_ssa_def *src2);
93 nir_alu_instr *emit(nir_op op, unsigned dest_size, nir_ssa_def *src1,
94 nir_ssa_def *src2, nir_ssa_def *src3);
95
96 bool supports_std430;
97
98 nir_shader *shader;
99 nir_function_impl *impl;
100 nir_builder b;
101 nir_ssa_def *result; /* result of the expression tree last visited */
102
103 nir_deref_instr *evaluate_deref(ir_instruction *ir);
104
105 nir_constant *constant_copy(ir_constant *ir, void *mem_ctx);
106
107 /* most recent deref instruction created */
108 nir_deref_instr *deref;
109
110 /* whether the IR we're operating on is per-function or global */
111 bool is_global;
112
113 ir_function_signature *sig;
114
115 /* map of ir_variable -> nir_variable */
116 struct hash_table *var_table;
117
118 /* map of ir_function_signature -> nir_function_overload */
119 struct hash_table *overload_table;
120 };
121
122 /*
123 * This visitor runs before the main visitor, calling create_function() for
124 * each function so that the main visitor can resolve forward references in
125 * calls.
126 */
127
128 class nir_function_visitor : public ir_hierarchical_visitor
129 {
130 public:
nir_function_visitor(nir_visitor * v)131 nir_function_visitor(nir_visitor *v) : visitor(v)
132 {
133 }
134 virtual ir_visitor_status visit_enter(ir_function *);
135
136 private:
137 nir_visitor *visitor;
138 };
139
140 /* glsl_to_nir can only handle converting certain function paramaters
141 * to NIR. This visitor checks for parameters it can't currently handle.
142 */
143 class ir_function_param_visitor : public ir_hierarchical_visitor
144 {
145 public:
ir_function_param_visitor()146 ir_function_param_visitor()
147 : unsupported(false)
148 {
149 }
150
visit_enter(ir_function_signature * ir)151 virtual ir_visitor_status visit_enter(ir_function_signature *ir)
152 {
153
154 if (ir->is_intrinsic())
155 return visit_continue;
156
157 foreach_in_list(ir_variable, param, &ir->parameters) {
158 if (!param->type->is_vector() || !param->type->is_scalar()) {
159 unsupported = true;
160 return visit_stop;
161 }
162
163 if (param->data.mode == ir_var_function_inout) {
164 unsupported = true;
165 return visit_stop;
166 }
167 }
168
169 if (!glsl_type_is_vector_or_scalar(ir->return_type) &&
170 !ir->return_type->is_void()) {
171 unsupported = true;
172 return visit_stop;
173 }
174
175 return visit_continue;
176 }
177
178 bool unsupported;
179 };
180
181 } /* end of anonymous namespace */
182
183
184 static bool
has_unsupported_function_param(exec_list * ir)185 has_unsupported_function_param(exec_list *ir)
186 {
187 ir_function_param_visitor visitor;
188 visit_list_elements(&visitor, ir);
189 return visitor.unsupported;
190 }
191
192 nir_shader *
glsl_to_nir(struct gl_context * ctx,const struct gl_shader_program * shader_prog,gl_shader_stage stage,const nir_shader_compiler_options * options)193 glsl_to_nir(struct gl_context *ctx,
194 const struct gl_shader_program *shader_prog,
195 gl_shader_stage stage,
196 const nir_shader_compiler_options *options)
197 {
198 struct gl_linked_shader *sh = shader_prog->_LinkedShaders[stage];
199
200 const struct gl_shader_compiler_options *gl_options =
201 &ctx->Const.ShaderCompilerOptions[stage];
202
203 /* glsl_to_nir can only handle converting certain function paramaters
204 * to NIR. If we find something we can't handle then we get the GLSL IR
205 * opts to remove it before we continue on.
206 *
207 * TODO: add missing glsl ir to nir support and remove this loop.
208 */
209 while (has_unsupported_function_param(sh->ir)) {
210 do_common_optimization(sh->ir, true, true, gl_options,
211 ctx->Const.NativeIntegers);
212 }
213
214 nir_shader *shader = nir_shader_create(NULL, stage, options,
215 &sh->Program->info);
216
217 nir_visitor v1(ctx, shader);
218 nir_function_visitor v2(&v1);
219 v2.run(sh->ir);
220 visit_exec_list(sh->ir, &v1);
221
222 nir_validate_shader(shader, "after glsl to nir, before function inline");
223
224 /* We have to lower away local constant initializers right before we
225 * inline functions. That way they get properly initialized at the top
226 * of the function and not at the top of its caller.
227 */
228 nir_lower_variable_initializers(shader, nir_var_all);
229 nir_lower_returns(shader);
230 nir_inline_functions(shader);
231 nir_opt_deref(shader);
232
233 nir_validate_shader(shader, "after function inlining and return lowering");
234
235 /* Now that we have inlined everything remove all of the functions except
236 * main().
237 */
238 foreach_list_typed_safe(nir_function, function, node, &(shader)->functions){
239 if (strcmp("main", function->name) != 0) {
240 exec_node_remove(&function->node);
241 }
242 }
243
244 shader->info.name = ralloc_asprintf(shader, "GLSL%d", shader_prog->Name);
245 if (shader_prog->Label)
246 shader->info.label = ralloc_strdup(shader, shader_prog->Label);
247
248 /* Check for transform feedback varyings specified via the API */
249 shader->info.has_transform_feedback_varyings =
250 shader_prog->TransformFeedback.NumVarying > 0;
251
252 /* Check for transform feedback varyings specified in the Shader */
253 if (shader_prog->last_vert_prog)
254 shader->info.has_transform_feedback_varyings |=
255 shader_prog->last_vert_prog->sh.LinkedTransformFeedback->NumVarying > 0;
256
257 if (shader->info.stage == MESA_SHADER_FRAGMENT) {
258 shader->info.fs.pixel_center_integer = sh->Program->info.fs.pixel_center_integer;
259 shader->info.fs.origin_upper_left = sh->Program->info.fs.origin_upper_left;
260 shader->info.fs.advanced_blend_modes = sh->Program->info.fs.advanced_blend_modes;
261 }
262
263 return shader;
264 }
265
nir_visitor(gl_context * ctx,nir_shader * shader)266 nir_visitor::nir_visitor(gl_context *ctx, nir_shader *shader)
267 {
268 this->supports_std430 = ctx->Const.UseSTD430AsDefaultPacking;
269 this->shader = shader;
270 this->is_global = true;
271 this->var_table = _mesa_pointer_hash_table_create(NULL);
272 this->overload_table = _mesa_pointer_hash_table_create(NULL);
273 this->result = NULL;
274 this->impl = NULL;
275 this->deref = NULL;
276 this->sig = NULL;
277 memset(&this->b, 0, sizeof(this->b));
278 }
279
~nir_visitor()280 nir_visitor::~nir_visitor()
281 {
282 _mesa_hash_table_destroy(this->var_table, NULL);
283 _mesa_hash_table_destroy(this->overload_table, NULL);
284 }
285
286 nir_deref_instr *
evaluate_deref(ir_instruction * ir)287 nir_visitor::evaluate_deref(ir_instruction *ir)
288 {
289 ir->accept(this);
290 return this->deref;
291 }
292
293 nir_constant *
constant_copy(ir_constant * ir,void * mem_ctx)294 nir_visitor::constant_copy(ir_constant *ir, void *mem_ctx)
295 {
296 if (ir == NULL)
297 return NULL;
298
299 nir_constant *ret = rzalloc(mem_ctx, nir_constant);
300
301 const unsigned rows = ir->type->vector_elements;
302 const unsigned cols = ir->type->matrix_columns;
303 unsigned i;
304
305 ret->num_elements = 0;
306 switch (ir->type->base_type) {
307 case GLSL_TYPE_UINT:
308 /* Only float base types can be matrices. */
309 assert(cols == 1);
310
311 for (unsigned r = 0; r < rows; r++)
312 ret->values[r].u32 = ir->value.u[r];
313
314 break;
315
316 case GLSL_TYPE_UINT16:
317 /* Only float base types can be matrices. */
318 assert(cols == 1);
319
320 for (unsigned r = 0; r < rows; r++)
321 ret->values[r].u16 = ir->value.u16[r];
322 break;
323
324 case GLSL_TYPE_INT:
325 /* Only float base types can be matrices. */
326 assert(cols == 1);
327
328 for (unsigned r = 0; r < rows; r++)
329 ret->values[r].i32 = ir->value.i[r];
330
331 break;
332
333 case GLSL_TYPE_INT16:
334 /* Only float base types can be matrices. */
335 assert(cols == 1);
336
337 for (unsigned r = 0; r < rows; r++)
338 ret->values[r].i16 = ir->value.i16[r];
339 break;
340
341 case GLSL_TYPE_FLOAT:
342 case GLSL_TYPE_FLOAT16:
343 case GLSL_TYPE_DOUBLE:
344 if (cols > 1) {
345 ret->elements = ralloc_array(mem_ctx, nir_constant *, cols);
346 ret->num_elements = cols;
347 for (unsigned c = 0; c < cols; c++) {
348 nir_constant *col_const = rzalloc(mem_ctx, nir_constant);
349 col_const->num_elements = 0;
350 switch (ir->type->base_type) {
351 case GLSL_TYPE_FLOAT:
352 for (unsigned r = 0; r < rows; r++)
353 col_const->values[r].f32 = ir->value.f[c * rows + r];
354 break;
355
356 case GLSL_TYPE_FLOAT16:
357 for (unsigned r = 0; r < rows; r++)
358 col_const->values[r].u16 = ir->value.f16[c * rows + r];
359 break;
360
361 case GLSL_TYPE_DOUBLE:
362 for (unsigned r = 0; r < rows; r++)
363 col_const->values[r].f64 = ir->value.d[c * rows + r];
364 break;
365
366 default:
367 unreachable("Cannot get here from the first level switch");
368 }
369 ret->elements[c] = col_const;
370 }
371 } else {
372 switch (ir->type->base_type) {
373 case GLSL_TYPE_FLOAT:
374 for (unsigned r = 0; r < rows; r++)
375 ret->values[r].f32 = ir->value.f[r];
376 break;
377
378 case GLSL_TYPE_FLOAT16:
379 for (unsigned r = 0; r < rows; r++)
380 ret->values[r].u16 = ir->value.f16[r];
381 break;
382
383 case GLSL_TYPE_DOUBLE:
384 for (unsigned r = 0; r < rows; r++)
385 ret->values[r].f64 = ir->value.d[r];
386 break;
387
388 default:
389 unreachable("Cannot get here from the first level switch");
390 }
391 }
392 break;
393
394 case GLSL_TYPE_UINT64:
395 /* Only float base types can be matrices. */
396 assert(cols == 1);
397
398 for (unsigned r = 0; r < rows; r++)
399 ret->values[r].u64 = ir->value.u64[r];
400 break;
401
402 case GLSL_TYPE_INT64:
403 /* Only float base types can be matrices. */
404 assert(cols == 1);
405
406 for (unsigned r = 0; r < rows; r++)
407 ret->values[r].i64 = ir->value.i64[r];
408 break;
409
410 case GLSL_TYPE_BOOL:
411 /* Only float base types can be matrices. */
412 assert(cols == 1);
413
414 for (unsigned r = 0; r < rows; r++)
415 ret->values[r].b = ir->value.b[r];
416
417 break;
418
419 case GLSL_TYPE_STRUCT:
420 case GLSL_TYPE_ARRAY:
421 ret->elements = ralloc_array(mem_ctx, nir_constant *,
422 ir->type->length);
423 ret->num_elements = ir->type->length;
424
425 for (i = 0; i < ir->type->length; i++)
426 ret->elements[i] = constant_copy(ir->const_elements[i], mem_ctx);
427 break;
428
429 default:
430 unreachable("not reached");
431 }
432
433 return ret;
434 }
435
436 static const glsl_type *
wrap_type_in_array(const glsl_type * elem_type,const glsl_type * array_type)437 wrap_type_in_array(const glsl_type *elem_type, const glsl_type *array_type)
438 {
439 if (!array_type->is_array())
440 return elem_type;
441
442 elem_type = wrap_type_in_array(elem_type, array_type->fields.array);
443
444 return glsl_type::get_array_instance(elem_type, array_type->length);
445 }
446
447 static unsigned
get_nir_how_declared(unsigned how_declared)448 get_nir_how_declared(unsigned how_declared)
449 {
450 if (how_declared == ir_var_hidden)
451 return nir_var_hidden;
452
453 return nir_var_declared_normally;
454 }
455
456 void
visit(ir_variable * ir)457 nir_visitor::visit(ir_variable *ir)
458 {
459 /* TODO: In future we should switch to using the NIR lowering pass but for
460 * now just ignore these variables as GLSL IR should have lowered them.
461 * Anything remaining are just dead vars that weren't cleaned up.
462 */
463 if (ir->data.mode == ir_var_shader_shared)
464 return;
465
466 /* FINISHME: inout parameters */
467 assert(ir->data.mode != ir_var_function_inout);
468
469 if (ir->data.mode == ir_var_function_out)
470 return;
471
472 nir_variable *var = rzalloc(shader, nir_variable);
473 var->type = ir->type;
474 var->name = ralloc_strdup(var, ir->name);
475
476 var->data.always_active_io = ir->data.always_active_io;
477 var->data.read_only = ir->data.read_only;
478 var->data.centroid = ir->data.centroid;
479 var->data.sample = ir->data.sample;
480 var->data.patch = ir->data.patch;
481 var->data.how_declared = get_nir_how_declared(ir->data.how_declared);
482 var->data.invariant = ir->data.invariant;
483 var->data.location = ir->data.location;
484 var->data.stream = ir->data.stream;
485 if (ir->data.stream & (1u << 31))
486 var->data.stream |= NIR_STREAM_PACKED;
487
488 var->data.precision = ir->data.precision;
489 var->data.explicit_location = ir->data.explicit_location;
490 var->data.matrix_layout = ir->data.matrix_layout;
491 var->data.from_named_ifc_block = ir->data.from_named_ifc_block;
492 var->data.compact = false;
493
494 switch(ir->data.mode) {
495 case ir_var_auto:
496 case ir_var_temporary:
497 if (is_global)
498 var->data.mode = nir_var_shader_temp;
499 else
500 var->data.mode = nir_var_function_temp;
501 break;
502
503 case ir_var_function_in:
504 case ir_var_const_in:
505 var->data.mode = nir_var_function_temp;
506 break;
507
508 case ir_var_shader_in:
509 if (shader->info.stage == MESA_SHADER_GEOMETRY &&
510 ir->data.location == VARYING_SLOT_PRIMITIVE_ID) {
511 /* For whatever reason, GLSL IR makes gl_PrimitiveIDIn an input */
512 var->data.location = SYSTEM_VALUE_PRIMITIVE_ID;
513 var->data.mode = nir_var_system_value;
514 } else {
515 var->data.mode = nir_var_shader_in;
516
517 if (shader->info.stage == MESA_SHADER_TESS_EVAL &&
518 (ir->data.location == VARYING_SLOT_TESS_LEVEL_INNER ||
519 ir->data.location == VARYING_SLOT_TESS_LEVEL_OUTER)) {
520 var->data.compact = ir->type->without_array()->is_scalar();
521 }
522
523 if (shader->info.stage > MESA_SHADER_VERTEX &&
524 ir->data.location >= VARYING_SLOT_CLIP_DIST0 &&
525 ir->data.location <= VARYING_SLOT_CULL_DIST1) {
526 var->data.compact = ir->type->without_array()->is_scalar();
527 }
528 }
529 break;
530
531 case ir_var_shader_out:
532 var->data.mode = nir_var_shader_out;
533 if (shader->info.stage == MESA_SHADER_TESS_CTRL &&
534 (ir->data.location == VARYING_SLOT_TESS_LEVEL_INNER ||
535 ir->data.location == VARYING_SLOT_TESS_LEVEL_OUTER)) {
536 var->data.compact = ir->type->without_array()->is_scalar();
537 }
538
539 if (shader->info.stage <= MESA_SHADER_GEOMETRY &&
540 ir->data.location >= VARYING_SLOT_CLIP_DIST0 &&
541 ir->data.location <= VARYING_SLOT_CULL_DIST1) {
542 var->data.compact = ir->type->without_array()->is_scalar();
543 }
544 break;
545
546 case ir_var_uniform:
547 if (ir->get_interface_type())
548 var->data.mode = nir_var_mem_ubo;
549 else
550 var->data.mode = nir_var_uniform;
551 break;
552
553 case ir_var_shader_storage:
554 var->data.mode = nir_var_mem_ssbo;
555 break;
556
557 case ir_var_system_value:
558 var->data.mode = nir_var_system_value;
559 break;
560
561 default:
562 unreachable("not reached");
563 }
564
565 unsigned mem_access = 0;
566 if (ir->data.memory_read_only)
567 mem_access |= ACCESS_NON_WRITEABLE;
568 if (ir->data.memory_write_only)
569 mem_access |= ACCESS_NON_READABLE;
570 if (ir->data.memory_coherent)
571 mem_access |= ACCESS_COHERENT;
572 if (ir->data.memory_volatile)
573 mem_access |= ACCESS_VOLATILE;
574 if (ir->data.memory_restrict)
575 mem_access |= ACCESS_RESTRICT;
576
577 var->interface_type = ir->get_interface_type();
578
579 /* For UBO and SSBO variables, we need explicit types */
580 if (var->data.mode & (nir_var_mem_ubo | nir_var_mem_ssbo)) {
581 const glsl_type *explicit_ifc_type =
582 ir->get_interface_type()->get_explicit_interface_type(supports_std430);
583
584 var->interface_type = explicit_ifc_type;
585
586 if (ir->type->without_array()->is_interface()) {
587 /* If the type contains the interface, wrap the explicit type in the
588 * right number of arrays.
589 */
590 var->type = wrap_type_in_array(explicit_ifc_type, ir->type);
591 } else {
592 /* Otherwise, this variable is one entry in the interface */
593 UNUSED bool found = false;
594 for (unsigned i = 0; i < explicit_ifc_type->length; i++) {
595 const glsl_struct_field *field =
596 &explicit_ifc_type->fields.structure[i];
597 if (strcmp(ir->name, field->name) != 0)
598 continue;
599
600 var->type = field->type;
601 if (field->memory_read_only)
602 mem_access |= ACCESS_NON_WRITEABLE;
603 if (field->memory_write_only)
604 mem_access |= ACCESS_NON_READABLE;
605 if (field->memory_coherent)
606 mem_access |= ACCESS_COHERENT;
607 if (field->memory_volatile)
608 mem_access |= ACCESS_VOLATILE;
609 if (field->memory_restrict)
610 mem_access |= ACCESS_RESTRICT;
611
612 found = true;
613 break;
614 }
615 assert(found);
616 }
617 }
618
619 var->data.interpolation = ir->data.interpolation;
620 var->data.location_frac = ir->data.location_frac;
621
622 switch (ir->data.depth_layout) {
623 case ir_depth_layout_none:
624 var->data.depth_layout = nir_depth_layout_none;
625 break;
626 case ir_depth_layout_any:
627 var->data.depth_layout = nir_depth_layout_any;
628 break;
629 case ir_depth_layout_greater:
630 var->data.depth_layout = nir_depth_layout_greater;
631 break;
632 case ir_depth_layout_less:
633 var->data.depth_layout = nir_depth_layout_less;
634 break;
635 case ir_depth_layout_unchanged:
636 var->data.depth_layout = nir_depth_layout_unchanged;
637 break;
638 default:
639 unreachable("not reached");
640 }
641
642 var->data.index = ir->data.index;
643 var->data.descriptor_set = 0;
644 var->data.binding = ir->data.binding;
645 var->data.explicit_binding = ir->data.explicit_binding;
646 var->data.bindless = ir->data.bindless;
647 var->data.offset = ir->data.offset;
648 var->data.access = (gl_access_qualifier)mem_access;
649
650 if (var->type->without_array()->is_image()) {
651 var->data.image.format = ir->data.image_format;
652 } else if (var->data.mode == nir_var_shader_out) {
653 var->data.xfb.buffer = ir->data.xfb_buffer;
654 var->data.xfb.stride = ir->data.xfb_stride;
655 }
656
657 var->data.fb_fetch_output = ir->data.fb_fetch_output;
658 var->data.explicit_xfb_buffer = ir->data.explicit_xfb_buffer;
659 var->data.explicit_xfb_stride = ir->data.explicit_xfb_stride;
660
661 var->num_state_slots = ir->get_num_state_slots();
662 if (var->num_state_slots > 0) {
663 var->state_slots = rzalloc_array(var, nir_state_slot,
664 var->num_state_slots);
665
666 ir_state_slot *state_slots = ir->get_state_slots();
667 for (unsigned i = 0; i < var->num_state_slots; i++) {
668 for (unsigned j = 0; j < 4; j++)
669 var->state_slots[i].tokens[j] = state_slots[i].tokens[j];
670 var->state_slots[i].swizzle = state_slots[i].swizzle;
671 }
672 } else {
673 var->state_slots = NULL;
674 }
675
676 var->constant_initializer = constant_copy(ir->constant_initializer, var);
677
678 if (var->data.mode == nir_var_function_temp)
679 nir_function_impl_add_variable(impl, var);
680 else
681 nir_shader_add_variable(shader, var);
682
683 _mesa_hash_table_insert(var_table, ir, var);
684 }
685
686 ir_visitor_status
visit_enter(ir_function * ir)687 nir_function_visitor::visit_enter(ir_function *ir)
688 {
689 foreach_in_list(ir_function_signature, sig, &ir->signatures) {
690 visitor->create_function(sig);
691 }
692 return visit_continue_with_parent;
693 }
694
695 void
create_function(ir_function_signature * ir)696 nir_visitor::create_function(ir_function_signature *ir)
697 {
698 if (ir->is_intrinsic())
699 return;
700
701 nir_function *func = nir_function_create(shader, ir->function_name());
702 if (strcmp(ir->function_name(), "main") == 0)
703 func->is_entrypoint = true;
704
705 func->num_params = ir->parameters.length() +
706 (ir->return_type != glsl_type::void_type);
707 func->params = ralloc_array(shader, nir_parameter, func->num_params);
708
709 unsigned np = 0;
710
711 if (ir->return_type != glsl_type::void_type) {
712 /* The return value is a variable deref (basically an out parameter) */
713 func->params[np].num_components = 1;
714 func->params[np].bit_size = 32;
715 np++;
716 }
717
718 foreach_in_list(ir_variable, param, &ir->parameters) {
719 /* FINISHME: pass arrays, structs, etc by reference? */
720 assert(param->type->is_vector() || param->type->is_scalar());
721
722 if (param->data.mode == ir_var_function_in) {
723 func->params[np].num_components = param->type->vector_elements;
724 func->params[np].bit_size = glsl_get_bit_size(param->type);
725 } else {
726 func->params[np].num_components = 1;
727 func->params[np].bit_size = 32;
728 }
729 np++;
730 }
731 assert(np == func->num_params);
732
733 _mesa_hash_table_insert(this->overload_table, ir, func);
734 }
735
736 void
visit(ir_function * ir)737 nir_visitor::visit(ir_function *ir)
738 {
739 foreach_in_list(ir_function_signature, sig, &ir->signatures)
740 sig->accept(this);
741 }
742
743 void
visit(ir_function_signature * ir)744 nir_visitor::visit(ir_function_signature *ir)
745 {
746 if (ir->is_intrinsic())
747 return;
748
749 this->sig = ir;
750
751 struct hash_entry *entry =
752 _mesa_hash_table_search(this->overload_table, ir);
753
754 assert(entry);
755 nir_function *func = (nir_function *) entry->data;
756
757 if (ir->is_defined) {
758 nir_function_impl *impl = nir_function_impl_create(func);
759 this->impl = impl;
760
761 this->is_global = false;
762
763 nir_builder_init(&b, impl);
764 b.cursor = nir_after_cf_list(&impl->body);
765
766 unsigned i = (ir->return_type != glsl_type::void_type) ? 1 : 0;
767
768 foreach_in_list(ir_variable, param, &ir->parameters) {
769 nir_variable *var =
770 nir_local_variable_create(impl, param->type, param->name);
771
772 if (param->data.mode == ir_var_function_in) {
773 nir_store_var(&b, var, nir_load_param(&b, i), ~0);
774 }
775
776 _mesa_hash_table_insert(var_table, param, var);
777 i++;
778 }
779
780 visit_exec_list(&ir->body, this);
781
782 this->is_global = true;
783 } else {
784 func->impl = NULL;
785 }
786 }
787
788 void
visit(ir_loop * ir)789 nir_visitor::visit(ir_loop *ir)
790 {
791 nir_push_loop(&b);
792 visit_exec_list(&ir->body_instructions, this);
793 nir_pop_loop(&b, NULL);
794 }
795
796 void
visit(ir_if * ir)797 nir_visitor::visit(ir_if *ir)
798 {
799 nir_push_if(&b, evaluate_rvalue(ir->condition));
800 visit_exec_list(&ir->then_instructions, this);
801 nir_push_else(&b, NULL);
802 visit_exec_list(&ir->else_instructions, this);
803 nir_pop_if(&b, NULL);
804 }
805
806 void
visit(ir_discard * ir)807 nir_visitor::visit(ir_discard *ir)
808 {
809 /*
810 * discards aren't treated as control flow, because before we lower them
811 * they can appear anywhere in the shader and the stuff after them may still
812 * be executed (yay, crazy GLSL rules!). However, after lowering, all the
813 * discards will be immediately followed by a return.
814 */
815
816 if (ir->condition)
817 nir_discard_if(&b, evaluate_rvalue(ir->condition));
818 else
819 nir_discard(&b);
820 }
821
822 void
visit(ir_demote * ir)823 nir_visitor::visit(ir_demote *ir)
824 {
825 nir_demote(&b);
826 }
827
828 void
visit(ir_emit_vertex * ir)829 nir_visitor::visit(ir_emit_vertex *ir)
830 {
831 nir_emit_vertex(&b, (unsigned)ir->stream_id());
832 }
833
834 void
visit(ir_end_primitive * ir)835 nir_visitor::visit(ir_end_primitive *ir)
836 {
837 nir_end_primitive(&b, (unsigned)ir->stream_id());
838 }
839
840 void
visit(ir_loop_jump * ir)841 nir_visitor::visit(ir_loop_jump *ir)
842 {
843 nir_jump_type type;
844 switch (ir->mode) {
845 case ir_loop_jump::jump_break:
846 type = nir_jump_break;
847 break;
848 case ir_loop_jump::jump_continue:
849 type = nir_jump_continue;
850 break;
851 default:
852 unreachable("not reached");
853 }
854
855 nir_jump_instr *instr = nir_jump_instr_create(this->shader, type);
856 nir_builder_instr_insert(&b, &instr->instr);
857 }
858
859 void
visit(ir_return * ir)860 nir_visitor::visit(ir_return *ir)
861 {
862 if (ir->value != NULL) {
863 nir_deref_instr *ret_deref =
864 nir_build_deref_cast(&b, nir_load_param(&b, 0),
865 nir_var_function_temp, ir->value->type, 0);
866
867 nir_ssa_def *val = evaluate_rvalue(ir->value);
868 nir_store_deref(&b, ret_deref, val, ~0);
869 }
870
871 nir_jump_instr *instr = nir_jump_instr_create(this->shader, nir_jump_return);
872 nir_builder_instr_insert(&b, &instr->instr);
873 }
874
875 static void
intrinsic_set_std430_align(nir_intrinsic_instr * intrin,const glsl_type * type)876 intrinsic_set_std430_align(nir_intrinsic_instr *intrin, const glsl_type *type)
877 {
878 unsigned bit_size = type->is_boolean() ? 32 : glsl_get_bit_size(type);
879 unsigned pow2_components = util_next_power_of_two(type->vector_elements);
880 nir_intrinsic_set_align(intrin, (bit_size / 8) * pow2_components, 0);
881 }
882
883 /* Accumulate any qualifiers along the deref chain to get the actual
884 * load/store qualifier.
885 */
886
887 static enum gl_access_qualifier
deref_get_qualifier(nir_deref_instr * deref)888 deref_get_qualifier(nir_deref_instr *deref)
889 {
890 nir_deref_path path;
891 nir_deref_path_init(&path, deref, NULL);
892
893 unsigned qualifiers = path.path[0]->var->data.access;
894
895 const glsl_type *parent_type = path.path[0]->type;
896 for (nir_deref_instr **cur_ptr = &path.path[1]; *cur_ptr; cur_ptr++) {
897 nir_deref_instr *cur = *cur_ptr;
898
899 if (parent_type->is_interface()) {
900 const struct glsl_struct_field *field =
901 &parent_type->fields.structure[cur->strct.index];
902 if (field->memory_read_only)
903 qualifiers |= ACCESS_NON_WRITEABLE;
904 if (field->memory_write_only)
905 qualifiers |= ACCESS_NON_READABLE;
906 if (field->memory_coherent)
907 qualifiers |= ACCESS_COHERENT;
908 if (field->memory_volatile)
909 qualifiers |= ACCESS_VOLATILE;
910 if (field->memory_restrict)
911 qualifiers |= ACCESS_RESTRICT;
912 }
913
914 parent_type = cur->type;
915 }
916
917 nir_deref_path_finish(&path);
918
919 return (gl_access_qualifier) qualifiers;
920 }
921
922 void
visit(ir_call * ir)923 nir_visitor::visit(ir_call *ir)
924 {
925 if (ir->callee->is_intrinsic()) {
926 nir_intrinsic_op op;
927
928 switch (ir->callee->intrinsic_id) {
929 case ir_intrinsic_generic_atomic_add:
930 op = ir->return_deref->type->is_integer_32_64()
931 ? nir_intrinsic_deref_atomic_add : nir_intrinsic_deref_atomic_fadd;
932 break;
933 case ir_intrinsic_generic_atomic_and:
934 op = nir_intrinsic_deref_atomic_and;
935 break;
936 case ir_intrinsic_generic_atomic_or:
937 op = nir_intrinsic_deref_atomic_or;
938 break;
939 case ir_intrinsic_generic_atomic_xor:
940 op = nir_intrinsic_deref_atomic_xor;
941 break;
942 case ir_intrinsic_generic_atomic_min:
943 assert(ir->return_deref);
944 if (ir->return_deref->type == glsl_type::int_type ||
945 ir->return_deref->type == glsl_type::int64_t_type)
946 op = nir_intrinsic_deref_atomic_imin;
947 else if (ir->return_deref->type == glsl_type::uint_type ||
948 ir->return_deref->type == glsl_type::uint64_t_type)
949 op = nir_intrinsic_deref_atomic_umin;
950 else if (ir->return_deref->type == glsl_type::float_type)
951 op = nir_intrinsic_deref_atomic_fmin;
952 else
953 unreachable("Invalid type");
954 break;
955 case ir_intrinsic_generic_atomic_max:
956 assert(ir->return_deref);
957 if (ir->return_deref->type == glsl_type::int_type ||
958 ir->return_deref->type == glsl_type::int64_t_type)
959 op = nir_intrinsic_deref_atomic_imax;
960 else if (ir->return_deref->type == glsl_type::uint_type ||
961 ir->return_deref->type == glsl_type::uint64_t_type)
962 op = nir_intrinsic_deref_atomic_umax;
963 else if (ir->return_deref->type == glsl_type::float_type)
964 op = nir_intrinsic_deref_atomic_fmax;
965 else
966 unreachable("Invalid type");
967 break;
968 case ir_intrinsic_generic_atomic_exchange:
969 op = nir_intrinsic_deref_atomic_exchange;
970 break;
971 case ir_intrinsic_generic_atomic_comp_swap:
972 op = ir->return_deref->type->is_integer_32_64()
973 ? nir_intrinsic_deref_atomic_comp_swap
974 : nir_intrinsic_deref_atomic_fcomp_swap;
975 break;
976 case ir_intrinsic_atomic_counter_read:
977 op = nir_intrinsic_atomic_counter_read_deref;
978 break;
979 case ir_intrinsic_atomic_counter_increment:
980 op = nir_intrinsic_atomic_counter_inc_deref;
981 break;
982 case ir_intrinsic_atomic_counter_predecrement:
983 op = nir_intrinsic_atomic_counter_pre_dec_deref;
984 break;
985 case ir_intrinsic_atomic_counter_add:
986 op = nir_intrinsic_atomic_counter_add_deref;
987 break;
988 case ir_intrinsic_atomic_counter_and:
989 op = nir_intrinsic_atomic_counter_and_deref;
990 break;
991 case ir_intrinsic_atomic_counter_or:
992 op = nir_intrinsic_atomic_counter_or_deref;
993 break;
994 case ir_intrinsic_atomic_counter_xor:
995 op = nir_intrinsic_atomic_counter_xor_deref;
996 break;
997 case ir_intrinsic_atomic_counter_min:
998 op = nir_intrinsic_atomic_counter_min_deref;
999 break;
1000 case ir_intrinsic_atomic_counter_max:
1001 op = nir_intrinsic_atomic_counter_max_deref;
1002 break;
1003 case ir_intrinsic_atomic_counter_exchange:
1004 op = nir_intrinsic_atomic_counter_exchange_deref;
1005 break;
1006 case ir_intrinsic_atomic_counter_comp_swap:
1007 op = nir_intrinsic_atomic_counter_comp_swap_deref;
1008 break;
1009 case ir_intrinsic_image_load:
1010 op = nir_intrinsic_image_deref_load;
1011 break;
1012 case ir_intrinsic_image_store:
1013 op = nir_intrinsic_image_deref_store;
1014 break;
1015 case ir_intrinsic_image_atomic_add:
1016 op = ir->return_deref->type->is_integer_32_64()
1017 ? nir_intrinsic_image_deref_atomic_add
1018 : nir_intrinsic_image_deref_atomic_fadd;
1019 break;
1020 case ir_intrinsic_image_atomic_min:
1021 if (ir->return_deref->type == glsl_type::int_type)
1022 op = nir_intrinsic_image_deref_atomic_imin;
1023 else if (ir->return_deref->type == glsl_type::uint_type)
1024 op = nir_intrinsic_image_deref_atomic_umin;
1025 else
1026 unreachable("Invalid type");
1027 break;
1028 case ir_intrinsic_image_atomic_max:
1029 if (ir->return_deref->type == glsl_type::int_type)
1030 op = nir_intrinsic_image_deref_atomic_imax;
1031 else if (ir->return_deref->type == glsl_type::uint_type)
1032 op = nir_intrinsic_image_deref_atomic_umax;
1033 else
1034 unreachable("Invalid type");
1035 break;
1036 case ir_intrinsic_image_atomic_and:
1037 op = nir_intrinsic_image_deref_atomic_and;
1038 break;
1039 case ir_intrinsic_image_atomic_or:
1040 op = nir_intrinsic_image_deref_atomic_or;
1041 break;
1042 case ir_intrinsic_image_atomic_xor:
1043 op = nir_intrinsic_image_deref_atomic_xor;
1044 break;
1045 case ir_intrinsic_image_atomic_exchange:
1046 op = nir_intrinsic_image_deref_atomic_exchange;
1047 break;
1048 case ir_intrinsic_image_atomic_comp_swap:
1049 op = nir_intrinsic_image_deref_atomic_comp_swap;
1050 break;
1051 case ir_intrinsic_image_atomic_inc_wrap:
1052 op = nir_intrinsic_image_deref_atomic_inc_wrap;
1053 break;
1054 case ir_intrinsic_image_atomic_dec_wrap:
1055 op = nir_intrinsic_image_deref_atomic_dec_wrap;
1056 break;
1057 case ir_intrinsic_memory_barrier:
1058 op = nir_intrinsic_memory_barrier;
1059 break;
1060 case ir_intrinsic_image_size:
1061 op = nir_intrinsic_image_deref_size;
1062 break;
1063 case ir_intrinsic_image_samples:
1064 op = nir_intrinsic_image_deref_samples;
1065 break;
1066 case ir_intrinsic_ssbo_store:
1067 case ir_intrinsic_ssbo_load:
1068 case ir_intrinsic_ssbo_atomic_add:
1069 case ir_intrinsic_ssbo_atomic_and:
1070 case ir_intrinsic_ssbo_atomic_or:
1071 case ir_intrinsic_ssbo_atomic_xor:
1072 case ir_intrinsic_ssbo_atomic_min:
1073 case ir_intrinsic_ssbo_atomic_max:
1074 case ir_intrinsic_ssbo_atomic_exchange:
1075 case ir_intrinsic_ssbo_atomic_comp_swap:
1076 /* SSBO store/loads should only have been lowered in GLSL IR for
1077 * non-nir drivers, NIR drivers make use of gl_nir_lower_buffers()
1078 * instead.
1079 */
1080 unreachable("Invalid operation nir doesn't want lowered ssbo "
1081 "store/loads");
1082 case ir_intrinsic_shader_clock:
1083 op = nir_intrinsic_shader_clock;
1084 break;
1085 case ir_intrinsic_begin_invocation_interlock:
1086 op = nir_intrinsic_begin_invocation_interlock;
1087 break;
1088 case ir_intrinsic_end_invocation_interlock:
1089 op = nir_intrinsic_end_invocation_interlock;
1090 break;
1091 case ir_intrinsic_group_memory_barrier:
1092 op = nir_intrinsic_group_memory_barrier;
1093 break;
1094 case ir_intrinsic_memory_barrier_atomic_counter:
1095 op = nir_intrinsic_memory_barrier_atomic_counter;
1096 break;
1097 case ir_intrinsic_memory_barrier_buffer:
1098 op = nir_intrinsic_memory_barrier_buffer;
1099 break;
1100 case ir_intrinsic_memory_barrier_image:
1101 op = nir_intrinsic_memory_barrier_image;
1102 break;
1103 case ir_intrinsic_memory_barrier_shared:
1104 op = nir_intrinsic_memory_barrier_shared;
1105 break;
1106 case ir_intrinsic_shared_load:
1107 op = nir_intrinsic_load_shared;
1108 break;
1109 case ir_intrinsic_shared_store:
1110 op = nir_intrinsic_store_shared;
1111 break;
1112 case ir_intrinsic_shared_atomic_add:
1113 op = ir->return_deref->type->is_integer_32_64()
1114 ? nir_intrinsic_shared_atomic_add
1115 : nir_intrinsic_shared_atomic_fadd;
1116 break;
1117 case ir_intrinsic_shared_atomic_and:
1118 op = nir_intrinsic_shared_atomic_and;
1119 break;
1120 case ir_intrinsic_shared_atomic_or:
1121 op = nir_intrinsic_shared_atomic_or;
1122 break;
1123 case ir_intrinsic_shared_atomic_xor:
1124 op = nir_intrinsic_shared_atomic_xor;
1125 break;
1126 case ir_intrinsic_shared_atomic_min:
1127 assert(ir->return_deref);
1128 if (ir->return_deref->type == glsl_type::int_type ||
1129 ir->return_deref->type == glsl_type::int64_t_type)
1130 op = nir_intrinsic_shared_atomic_imin;
1131 else if (ir->return_deref->type == glsl_type::uint_type ||
1132 ir->return_deref->type == glsl_type::uint64_t_type)
1133 op = nir_intrinsic_shared_atomic_umin;
1134 else if (ir->return_deref->type == glsl_type::float_type)
1135 op = nir_intrinsic_shared_atomic_fmin;
1136 else
1137 unreachable("Invalid type");
1138 break;
1139 case ir_intrinsic_shared_atomic_max:
1140 assert(ir->return_deref);
1141 if (ir->return_deref->type == glsl_type::int_type ||
1142 ir->return_deref->type == glsl_type::int64_t_type)
1143 op = nir_intrinsic_shared_atomic_imax;
1144 else if (ir->return_deref->type == glsl_type::uint_type ||
1145 ir->return_deref->type == glsl_type::uint64_t_type)
1146 op = nir_intrinsic_shared_atomic_umax;
1147 else if (ir->return_deref->type == glsl_type::float_type)
1148 op = nir_intrinsic_shared_atomic_fmax;
1149 else
1150 unreachable("Invalid type");
1151 break;
1152 case ir_intrinsic_shared_atomic_exchange:
1153 op = nir_intrinsic_shared_atomic_exchange;
1154 break;
1155 case ir_intrinsic_shared_atomic_comp_swap:
1156 op = ir->return_deref->type->is_integer_32_64()
1157 ? nir_intrinsic_shared_atomic_comp_swap
1158 : nir_intrinsic_shared_atomic_fcomp_swap;
1159 break;
1160 case ir_intrinsic_vote_any:
1161 op = nir_intrinsic_vote_any;
1162 break;
1163 case ir_intrinsic_vote_all:
1164 op = nir_intrinsic_vote_all;
1165 break;
1166 case ir_intrinsic_vote_eq:
1167 op = nir_intrinsic_vote_ieq;
1168 break;
1169 case ir_intrinsic_ballot:
1170 op = nir_intrinsic_ballot;
1171 break;
1172 case ir_intrinsic_read_invocation:
1173 op = nir_intrinsic_read_invocation;
1174 break;
1175 case ir_intrinsic_read_first_invocation:
1176 op = nir_intrinsic_read_first_invocation;
1177 break;
1178 case ir_intrinsic_helper_invocation:
1179 op = nir_intrinsic_is_helper_invocation;
1180 break;
1181 default:
1182 unreachable("not reached");
1183 }
1184
1185 nir_intrinsic_instr *instr = nir_intrinsic_instr_create(shader, op);
1186 nir_ssa_def *ret = &instr->dest.ssa;
1187
1188 switch (op) {
1189 case nir_intrinsic_deref_atomic_add:
1190 case nir_intrinsic_deref_atomic_imin:
1191 case nir_intrinsic_deref_atomic_umin:
1192 case nir_intrinsic_deref_atomic_imax:
1193 case nir_intrinsic_deref_atomic_umax:
1194 case nir_intrinsic_deref_atomic_and:
1195 case nir_intrinsic_deref_atomic_or:
1196 case nir_intrinsic_deref_atomic_xor:
1197 case nir_intrinsic_deref_atomic_exchange:
1198 case nir_intrinsic_deref_atomic_comp_swap:
1199 case nir_intrinsic_deref_atomic_fadd:
1200 case nir_intrinsic_deref_atomic_fmin:
1201 case nir_intrinsic_deref_atomic_fmax:
1202 case nir_intrinsic_deref_atomic_fcomp_swap: {
1203 int param_count = ir->actual_parameters.length();
1204 assert(param_count == 2 || param_count == 3);
1205
1206 /* Deref */
1207 exec_node *param = ir->actual_parameters.get_head();
1208 ir_rvalue *rvalue = (ir_rvalue *) param;
1209 ir_dereference *deref = rvalue->as_dereference();
1210 ir_swizzle *swizzle = NULL;
1211 if (!deref) {
1212 /* We may have a swizzle to pick off a single vec4 component */
1213 swizzle = rvalue->as_swizzle();
1214 assert(swizzle && swizzle->type->vector_elements == 1);
1215 deref = swizzle->val->as_dereference();
1216 assert(deref);
1217 }
1218 nir_deref_instr *nir_deref = evaluate_deref(deref);
1219 if (swizzle) {
1220 nir_deref = nir_build_deref_array_imm(&b, nir_deref,
1221 swizzle->mask.x);
1222 }
1223 instr->src[0] = nir_src_for_ssa(&nir_deref->dest.ssa);
1224
1225 nir_intrinsic_set_access(instr, deref_get_qualifier(nir_deref));
1226
1227 /* data1 parameter (this is always present) */
1228 param = param->get_next();
1229 ir_instruction *inst = (ir_instruction *) param;
1230 instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1231
1232 /* data2 parameter (only with atomic_comp_swap) */
1233 if (param_count == 3) {
1234 assert(op == nir_intrinsic_deref_atomic_comp_swap ||
1235 op == nir_intrinsic_deref_atomic_fcomp_swap);
1236 param = param->get_next();
1237 inst = (ir_instruction *) param;
1238 instr->src[2] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1239 }
1240
1241 /* Atomic result */
1242 assert(ir->return_deref);
1243 if (ir->return_deref->type->is_integer_64()) {
1244 nir_ssa_dest_init(&instr->instr, &instr->dest,
1245 ir->return_deref->type->vector_elements, 64, NULL);
1246 } else {
1247 nir_ssa_dest_init(&instr->instr, &instr->dest,
1248 ir->return_deref->type->vector_elements, 32, NULL);
1249 }
1250 nir_builder_instr_insert(&b, &instr->instr);
1251 break;
1252 }
1253 case nir_intrinsic_atomic_counter_read_deref:
1254 case nir_intrinsic_atomic_counter_inc_deref:
1255 case nir_intrinsic_atomic_counter_pre_dec_deref:
1256 case nir_intrinsic_atomic_counter_add_deref:
1257 case nir_intrinsic_atomic_counter_min_deref:
1258 case nir_intrinsic_atomic_counter_max_deref:
1259 case nir_intrinsic_atomic_counter_and_deref:
1260 case nir_intrinsic_atomic_counter_or_deref:
1261 case nir_intrinsic_atomic_counter_xor_deref:
1262 case nir_intrinsic_atomic_counter_exchange_deref:
1263 case nir_intrinsic_atomic_counter_comp_swap_deref: {
1264 /* Set the counter variable dereference. */
1265 exec_node *param = ir->actual_parameters.get_head();
1266 ir_dereference *counter = (ir_dereference *)param;
1267
1268 instr->src[0] = nir_src_for_ssa(&evaluate_deref(counter)->dest.ssa);
1269 param = param->get_next();
1270
1271 /* Set the intrinsic destination. */
1272 if (ir->return_deref) {
1273 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 32, NULL);
1274 }
1275
1276 /* Set the intrinsic parameters. */
1277 if (!param->is_tail_sentinel()) {
1278 instr->src[1] =
1279 nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
1280 param = param->get_next();
1281 }
1282
1283 if (!param->is_tail_sentinel()) {
1284 instr->src[2] =
1285 nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
1286 param = param->get_next();
1287 }
1288
1289 nir_builder_instr_insert(&b, &instr->instr);
1290 break;
1291 }
1292 case nir_intrinsic_image_deref_load:
1293 case nir_intrinsic_image_deref_store:
1294 case nir_intrinsic_image_deref_atomic_add:
1295 case nir_intrinsic_image_deref_atomic_imin:
1296 case nir_intrinsic_image_deref_atomic_umin:
1297 case nir_intrinsic_image_deref_atomic_imax:
1298 case nir_intrinsic_image_deref_atomic_umax:
1299 case nir_intrinsic_image_deref_atomic_and:
1300 case nir_intrinsic_image_deref_atomic_or:
1301 case nir_intrinsic_image_deref_atomic_xor:
1302 case nir_intrinsic_image_deref_atomic_exchange:
1303 case nir_intrinsic_image_deref_atomic_comp_swap:
1304 case nir_intrinsic_image_deref_atomic_fadd:
1305 case nir_intrinsic_image_deref_samples:
1306 case nir_intrinsic_image_deref_size:
1307 case nir_intrinsic_image_deref_atomic_inc_wrap:
1308 case nir_intrinsic_image_deref_atomic_dec_wrap: {
1309 /* Set the image variable dereference. */
1310 exec_node *param = ir->actual_parameters.get_head();
1311 ir_dereference *image = (ir_dereference *)param;
1312 nir_deref_instr *deref = evaluate_deref(image);
1313 const glsl_type *type = deref->type;
1314
1315 nir_intrinsic_set_access(instr, deref_get_qualifier(deref));
1316
1317 instr->src[0] = nir_src_for_ssa(&deref->dest.ssa);
1318 param = param->get_next();
1319 nir_intrinsic_set_image_dim(instr,
1320 (glsl_sampler_dim)type->sampler_dimensionality);
1321 nir_intrinsic_set_image_array(instr, type->sampler_array);
1322
1323 /* Set the intrinsic destination. */
1324 if (ir->return_deref) {
1325 unsigned num_components = ir->return_deref->type->vector_elements;
1326 nir_ssa_dest_init(&instr->instr, &instr->dest,
1327 num_components, 32, NULL);
1328 }
1329
1330 if (op == nir_intrinsic_image_deref_size) {
1331 instr->num_components = instr->dest.ssa.num_components;
1332 } else if (op == nir_intrinsic_image_deref_load) {
1333 instr->num_components = 4;
1334 nir_intrinsic_set_dest_type(instr,
1335 nir_get_nir_type_for_glsl_base_type(type->sampled_type));
1336 } else if (op == nir_intrinsic_image_deref_store) {
1337 instr->num_components = 4;
1338 nir_intrinsic_set_src_type(instr,
1339 nir_get_nir_type_for_glsl_base_type(type->sampled_type));
1340 }
1341
1342 if (op == nir_intrinsic_image_deref_size ||
1343 op == nir_intrinsic_image_deref_samples) {
1344 /* image_deref_size takes an LOD parameter which is always 0
1345 * coming from GLSL.
1346 */
1347 if (op == nir_intrinsic_image_deref_size)
1348 instr->src[1] = nir_src_for_ssa(nir_imm_int(&b, 0));
1349 nir_builder_instr_insert(&b, &instr->instr);
1350 break;
1351 }
1352
1353 /* Set the address argument, extending the coordinate vector to four
1354 * components.
1355 */
1356 nir_ssa_def *src_addr =
1357 evaluate_rvalue((ir_dereference *)param);
1358 nir_ssa_def *srcs[4];
1359
1360 for (int i = 0; i < 4; i++) {
1361 if (i < type->coordinate_components())
1362 srcs[i] = nir_channel(&b, src_addr, i);
1363 else
1364 srcs[i] = nir_ssa_undef(&b, 1, 32);
1365 }
1366
1367 instr->src[1] = nir_src_for_ssa(nir_vec(&b, srcs, 4));
1368 param = param->get_next();
1369
1370 /* Set the sample argument, which is undefined for single-sample
1371 * images.
1372 */
1373 if (type->sampler_dimensionality == GLSL_SAMPLER_DIM_MS) {
1374 instr->src[2] =
1375 nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
1376 param = param->get_next();
1377 } else {
1378 instr->src[2] = nir_src_for_ssa(nir_ssa_undef(&b, 1, 32));
1379 }
1380
1381 /* Set the intrinsic parameters. */
1382 if (!param->is_tail_sentinel()) {
1383 instr->src[3] =
1384 nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
1385 param = param->get_next();
1386 } else if (op == nir_intrinsic_image_deref_load) {
1387 instr->src[3] = nir_src_for_ssa(nir_imm_int(&b, 0)); /* LOD */
1388 }
1389
1390 if (!param->is_tail_sentinel()) {
1391 instr->src[4] =
1392 nir_src_for_ssa(evaluate_rvalue((ir_dereference *)param));
1393 param = param->get_next();
1394 } else if (op == nir_intrinsic_image_deref_store) {
1395 instr->src[4] = nir_src_for_ssa(nir_imm_int(&b, 0)); /* LOD */
1396 }
1397
1398 nir_builder_instr_insert(&b, &instr->instr);
1399 break;
1400 }
1401 case nir_intrinsic_memory_barrier:
1402 case nir_intrinsic_group_memory_barrier:
1403 case nir_intrinsic_memory_barrier_atomic_counter:
1404 case nir_intrinsic_memory_barrier_buffer:
1405 case nir_intrinsic_memory_barrier_image:
1406 case nir_intrinsic_memory_barrier_shared:
1407 nir_builder_instr_insert(&b, &instr->instr);
1408 break;
1409 case nir_intrinsic_shader_clock:
1410 nir_ssa_dest_init(&instr->instr, &instr->dest, 2, 32, NULL);
1411 nir_intrinsic_set_memory_scope(instr, NIR_SCOPE_SUBGROUP);
1412 nir_builder_instr_insert(&b, &instr->instr);
1413 break;
1414 case nir_intrinsic_begin_invocation_interlock:
1415 nir_builder_instr_insert(&b, &instr->instr);
1416 break;
1417 case nir_intrinsic_end_invocation_interlock:
1418 nir_builder_instr_insert(&b, &instr->instr);
1419 break;
1420 case nir_intrinsic_store_ssbo: {
1421 exec_node *param = ir->actual_parameters.get_head();
1422 ir_rvalue *block = ((ir_instruction *)param)->as_rvalue();
1423
1424 param = param->get_next();
1425 ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
1426
1427 param = param->get_next();
1428 ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
1429
1430 param = param->get_next();
1431 ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
1432 assert(write_mask);
1433
1434 nir_ssa_def *nir_val = evaluate_rvalue(val);
1435 if (val->type->is_boolean())
1436 nir_val = nir_b2i32(&b, nir_val);
1437
1438 instr->src[0] = nir_src_for_ssa(nir_val);
1439 instr->src[1] = nir_src_for_ssa(evaluate_rvalue(block));
1440 instr->src[2] = nir_src_for_ssa(evaluate_rvalue(offset));
1441 intrinsic_set_std430_align(instr, val->type);
1442 nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
1443 instr->num_components = val->type->vector_elements;
1444
1445 nir_builder_instr_insert(&b, &instr->instr);
1446 break;
1447 }
1448 case nir_intrinsic_load_shared: {
1449 exec_node *param = ir->actual_parameters.get_head();
1450 ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
1451
1452 nir_intrinsic_set_base(instr, 0);
1453 instr->src[0] = nir_src_for_ssa(evaluate_rvalue(offset));
1454
1455 const glsl_type *type = ir->return_deref->var->type;
1456 instr->num_components = type->vector_elements;
1457 intrinsic_set_std430_align(instr, type);
1458
1459 /* Setup destination register */
1460 unsigned bit_size = type->is_boolean() ? 32 : glsl_get_bit_size(type);
1461 nir_ssa_dest_init(&instr->instr, &instr->dest,
1462 type->vector_elements, bit_size, NULL);
1463
1464 nir_builder_instr_insert(&b, &instr->instr);
1465
1466 /* The value in shared memory is a 32-bit value */
1467 if (type->is_boolean())
1468 ret = nir_b2b1(&b, &instr->dest.ssa);
1469 break;
1470 }
1471 case nir_intrinsic_store_shared: {
1472 exec_node *param = ir->actual_parameters.get_head();
1473 ir_rvalue *offset = ((ir_instruction *)param)->as_rvalue();
1474
1475 param = param->get_next();
1476 ir_rvalue *val = ((ir_instruction *)param)->as_rvalue();
1477
1478 param = param->get_next();
1479 ir_constant *write_mask = ((ir_instruction *)param)->as_constant();
1480 assert(write_mask);
1481
1482 nir_intrinsic_set_base(instr, 0);
1483 instr->src[1] = nir_src_for_ssa(evaluate_rvalue(offset));
1484
1485 nir_intrinsic_set_write_mask(instr, write_mask->value.u[0]);
1486
1487 nir_ssa_def *nir_val = evaluate_rvalue(val);
1488 /* The value in shared memory is a 32-bit value */
1489 if (val->type->is_boolean())
1490 nir_val = nir_b2b32(&b, nir_val);
1491
1492 instr->src[0] = nir_src_for_ssa(nir_val);
1493 instr->num_components = val->type->vector_elements;
1494 intrinsic_set_std430_align(instr, val->type);
1495
1496 nir_builder_instr_insert(&b, &instr->instr);
1497 break;
1498 }
1499 case nir_intrinsic_shared_atomic_add:
1500 case nir_intrinsic_shared_atomic_imin:
1501 case nir_intrinsic_shared_atomic_umin:
1502 case nir_intrinsic_shared_atomic_imax:
1503 case nir_intrinsic_shared_atomic_umax:
1504 case nir_intrinsic_shared_atomic_and:
1505 case nir_intrinsic_shared_atomic_or:
1506 case nir_intrinsic_shared_atomic_xor:
1507 case nir_intrinsic_shared_atomic_exchange:
1508 case nir_intrinsic_shared_atomic_comp_swap:
1509 case nir_intrinsic_shared_atomic_fadd:
1510 case nir_intrinsic_shared_atomic_fmin:
1511 case nir_intrinsic_shared_atomic_fmax:
1512 case nir_intrinsic_shared_atomic_fcomp_swap: {
1513 int param_count = ir->actual_parameters.length();
1514 assert(param_count == 2 || param_count == 3);
1515
1516 /* Offset */
1517 exec_node *param = ir->actual_parameters.get_head();
1518 ir_instruction *inst = (ir_instruction *) param;
1519 instr->src[0] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1520
1521 /* data1 parameter (this is always present) */
1522 param = param->get_next();
1523 inst = (ir_instruction *) param;
1524 instr->src[1] = nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1525
1526 /* data2 parameter (only with atomic_comp_swap) */
1527 if (param_count == 3) {
1528 assert(op == nir_intrinsic_shared_atomic_comp_swap ||
1529 op == nir_intrinsic_shared_atomic_fcomp_swap);
1530 param = param->get_next();
1531 inst = (ir_instruction *) param;
1532 instr->src[2] =
1533 nir_src_for_ssa(evaluate_rvalue(inst->as_rvalue()));
1534 }
1535
1536 /* Atomic result */
1537 assert(ir->return_deref);
1538 unsigned bit_size = glsl_get_bit_size(ir->return_deref->type);
1539 nir_ssa_dest_init(&instr->instr, &instr->dest,
1540 ir->return_deref->type->vector_elements,
1541 bit_size, NULL);
1542 nir_builder_instr_insert(&b, &instr->instr);
1543 break;
1544 }
1545 case nir_intrinsic_vote_ieq:
1546 instr->num_components = 1;
1547 FALLTHROUGH;
1548 case nir_intrinsic_vote_any:
1549 case nir_intrinsic_vote_all: {
1550 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 1, NULL);
1551
1552 ir_rvalue *value = (ir_rvalue *) ir->actual_parameters.get_head();
1553 instr->src[0] = nir_src_for_ssa(evaluate_rvalue(value));
1554
1555 nir_builder_instr_insert(&b, &instr->instr);
1556 break;
1557 }
1558
1559 case nir_intrinsic_ballot: {
1560 nir_ssa_dest_init(&instr->instr, &instr->dest,
1561 ir->return_deref->type->vector_elements, 64, NULL);
1562 instr->num_components = ir->return_deref->type->vector_elements;
1563
1564 ir_rvalue *value = (ir_rvalue *) ir->actual_parameters.get_head();
1565 instr->src[0] = nir_src_for_ssa(evaluate_rvalue(value));
1566
1567 nir_builder_instr_insert(&b, &instr->instr);
1568 break;
1569 }
1570 case nir_intrinsic_read_invocation: {
1571 nir_ssa_dest_init(&instr->instr, &instr->dest,
1572 ir->return_deref->type->vector_elements, 32, NULL);
1573 instr->num_components = ir->return_deref->type->vector_elements;
1574
1575 ir_rvalue *value = (ir_rvalue *) ir->actual_parameters.get_head();
1576 instr->src[0] = nir_src_for_ssa(evaluate_rvalue(value));
1577
1578 ir_rvalue *invocation = (ir_rvalue *) ir->actual_parameters.get_head()->next;
1579 instr->src[1] = nir_src_for_ssa(evaluate_rvalue(invocation));
1580
1581 nir_builder_instr_insert(&b, &instr->instr);
1582 break;
1583 }
1584 case nir_intrinsic_read_first_invocation: {
1585 nir_ssa_dest_init(&instr->instr, &instr->dest,
1586 ir->return_deref->type->vector_elements, 32, NULL);
1587 instr->num_components = ir->return_deref->type->vector_elements;
1588
1589 ir_rvalue *value = (ir_rvalue *) ir->actual_parameters.get_head();
1590 instr->src[0] = nir_src_for_ssa(evaluate_rvalue(value));
1591
1592 nir_builder_instr_insert(&b, &instr->instr);
1593 break;
1594 }
1595 case nir_intrinsic_is_helper_invocation: {
1596 nir_ssa_dest_init(&instr->instr, &instr->dest, 1, 1, NULL);
1597 nir_builder_instr_insert(&b, &instr->instr);
1598 break;
1599 }
1600 default:
1601 unreachable("not reached");
1602 }
1603
1604 if (ir->return_deref)
1605 nir_store_deref(&b, evaluate_deref(ir->return_deref), ret, ~0);
1606
1607 return;
1608 }
1609
1610 struct hash_entry *entry =
1611 _mesa_hash_table_search(this->overload_table, ir->callee);
1612 assert(entry);
1613 nir_function *callee = (nir_function *) entry->data;
1614
1615 nir_call_instr *call = nir_call_instr_create(this->shader, callee);
1616
1617 unsigned i = 0;
1618 nir_deref_instr *ret_deref = NULL;
1619 if (ir->return_deref) {
1620 nir_variable *ret_tmp =
1621 nir_local_variable_create(this->impl, ir->return_deref->type,
1622 "return_tmp");
1623 ret_deref = nir_build_deref_var(&b, ret_tmp);
1624 call->params[i++] = nir_src_for_ssa(&ret_deref->dest.ssa);
1625 }
1626
1627 foreach_two_lists(formal_node, &ir->callee->parameters,
1628 actual_node, &ir->actual_parameters) {
1629 ir_rvalue *param_rvalue = (ir_rvalue *) actual_node;
1630 ir_variable *sig_param = (ir_variable *) formal_node;
1631
1632 if (sig_param->data.mode == ir_var_function_out) {
1633 nir_deref_instr *out_deref = evaluate_deref(param_rvalue);
1634 call->params[i] = nir_src_for_ssa(&out_deref->dest.ssa);
1635 } else if (sig_param->data.mode == ir_var_function_in) {
1636 nir_ssa_def *val = evaluate_rvalue(param_rvalue);
1637 nir_src src = nir_src_for_ssa(val);
1638
1639 nir_src_copy(&call->params[i], &src);
1640 } else if (sig_param->data.mode == ir_var_function_inout) {
1641 unreachable("unimplemented: inout parameters");
1642 }
1643
1644 i++;
1645 }
1646
1647 nir_builder_instr_insert(&b, &call->instr);
1648
1649 if (ir->return_deref)
1650 nir_store_deref(&b, evaluate_deref(ir->return_deref), nir_load_deref(&b, ret_deref), ~0);
1651 }
1652
1653 void
visit(ir_assignment * ir)1654 nir_visitor::visit(ir_assignment *ir)
1655 {
1656 unsigned num_components = ir->lhs->type->vector_elements;
1657
1658 b.exact = ir->lhs->variable_referenced()->data.invariant ||
1659 ir->lhs->variable_referenced()->data.precise;
1660
1661 if ((ir->rhs->as_dereference() || ir->rhs->as_constant()) &&
1662 (ir->write_mask == (1 << num_components) - 1 || ir->write_mask == 0)) {
1663 nir_deref_instr *lhs = evaluate_deref(ir->lhs);
1664 nir_deref_instr *rhs = evaluate_deref(ir->rhs);
1665 enum gl_access_qualifier lhs_qualifiers = deref_get_qualifier(lhs);
1666 enum gl_access_qualifier rhs_qualifiers = deref_get_qualifier(rhs);
1667 if (ir->condition) {
1668 nir_push_if(&b, evaluate_rvalue(ir->condition));
1669 nir_copy_deref_with_access(&b, lhs, rhs, lhs_qualifiers,
1670 rhs_qualifiers);
1671 nir_pop_if(&b, NULL);
1672 } else {
1673 nir_copy_deref_with_access(&b, lhs, rhs, lhs_qualifiers,
1674 rhs_qualifiers);
1675 }
1676 return;
1677 }
1678
1679 assert(ir->rhs->type->is_scalar() || ir->rhs->type->is_vector());
1680
1681 ir->lhs->accept(this);
1682 nir_deref_instr *lhs_deref = this->deref;
1683 nir_ssa_def *src = evaluate_rvalue(ir->rhs);
1684
1685 if (ir->write_mask != (1 << num_components) - 1 && ir->write_mask != 0) {
1686 /* GLSL IR will give us the input to the write-masked assignment in a
1687 * single packed vector. So, for example, if the writemask is xzw, then
1688 * we have to swizzle x -> x, y -> z, and z -> w and get the y component
1689 * from the load.
1690 */
1691 unsigned swiz[4];
1692 unsigned component = 0;
1693 for (unsigned i = 0; i < 4; i++) {
1694 swiz[i] = ir->write_mask & (1 << i) ? component++ : 0;
1695 }
1696 src = nir_swizzle(&b, src, swiz, num_components);
1697 }
1698
1699 enum gl_access_qualifier qualifiers = deref_get_qualifier(lhs_deref);
1700 if (ir->condition) {
1701 nir_push_if(&b, evaluate_rvalue(ir->condition));
1702 nir_store_deref_with_access(&b, lhs_deref, src, ir->write_mask,
1703 qualifiers);
1704 nir_pop_if(&b, NULL);
1705 } else {
1706 nir_store_deref_with_access(&b, lhs_deref, src, ir->write_mask,
1707 qualifiers);
1708 }
1709 }
1710
1711 /*
1712 * Given an instruction, returns a pointer to its destination or NULL if there
1713 * is no destination.
1714 *
1715 * Note that this only handles instructions we generate at this level.
1716 */
1717 static nir_dest *
get_instr_dest(nir_instr * instr)1718 get_instr_dest(nir_instr *instr)
1719 {
1720 nir_alu_instr *alu_instr;
1721 nir_intrinsic_instr *intrinsic_instr;
1722 nir_tex_instr *tex_instr;
1723
1724 switch (instr->type) {
1725 case nir_instr_type_alu:
1726 alu_instr = nir_instr_as_alu(instr);
1727 return &alu_instr->dest.dest;
1728
1729 case nir_instr_type_intrinsic:
1730 intrinsic_instr = nir_instr_as_intrinsic(instr);
1731 if (nir_intrinsic_infos[intrinsic_instr->intrinsic].has_dest)
1732 return &intrinsic_instr->dest;
1733 else
1734 return NULL;
1735
1736 case nir_instr_type_tex:
1737 tex_instr = nir_instr_as_tex(instr);
1738 return &tex_instr->dest;
1739
1740 default:
1741 unreachable("not reached");
1742 }
1743
1744 return NULL;
1745 }
1746
1747 void
add_instr(nir_instr * instr,unsigned num_components,unsigned bit_size)1748 nir_visitor::add_instr(nir_instr *instr, unsigned num_components,
1749 unsigned bit_size)
1750 {
1751 nir_dest *dest = get_instr_dest(instr);
1752
1753 if (dest)
1754 nir_ssa_dest_init(instr, dest, num_components, bit_size, NULL);
1755
1756 nir_builder_instr_insert(&b, instr);
1757
1758 if (dest) {
1759 assert(dest->is_ssa);
1760 this->result = &dest->ssa;
1761 }
1762 }
1763
1764 nir_ssa_def *
evaluate_rvalue(ir_rvalue * ir)1765 nir_visitor::evaluate_rvalue(ir_rvalue* ir)
1766 {
1767 ir->accept(this);
1768 if (ir->as_dereference() || ir->as_constant()) {
1769 /*
1770 * A dereference is being used on the right hand side, which means we
1771 * must emit a variable load.
1772 */
1773
1774 enum gl_access_qualifier access = deref_get_qualifier(this->deref);
1775 this->result = nir_load_deref_with_access(&b, this->deref, access);
1776 }
1777
1778 return this->result;
1779 }
1780
1781 static bool
type_is_float(glsl_base_type type)1782 type_is_float(glsl_base_type type)
1783 {
1784 return type == GLSL_TYPE_FLOAT || type == GLSL_TYPE_DOUBLE ||
1785 type == GLSL_TYPE_FLOAT16;
1786 }
1787
1788 static bool
type_is_signed(glsl_base_type type)1789 type_is_signed(glsl_base_type type)
1790 {
1791 return type == GLSL_TYPE_INT || type == GLSL_TYPE_INT64 ||
1792 type == GLSL_TYPE_INT16;
1793 }
1794
1795 void
visit(ir_expression * ir)1796 nir_visitor::visit(ir_expression *ir)
1797 {
1798 /* Some special cases */
1799 switch (ir->operation) {
1800 case ir_unop_interpolate_at_centroid:
1801 case ir_binop_interpolate_at_offset:
1802 case ir_binop_interpolate_at_sample: {
1803 ir_dereference *deref = ir->operands[0]->as_dereference();
1804 ir_swizzle *swizzle = NULL;
1805 if (!deref) {
1806 /* the api does not allow a swizzle here, but the varying packing code
1807 * may have pushed one into here.
1808 */
1809 swizzle = ir->operands[0]->as_swizzle();
1810 assert(swizzle);
1811 deref = swizzle->val->as_dereference();
1812 assert(deref);
1813 }
1814
1815 deref->accept(this);
1816
1817 nir_intrinsic_op op;
1818 if (nir_deref_mode_is(this->deref, nir_var_shader_in)) {
1819 switch (ir->operation) {
1820 case ir_unop_interpolate_at_centroid:
1821 op = nir_intrinsic_interp_deref_at_centroid;
1822 break;
1823 case ir_binop_interpolate_at_offset:
1824 op = nir_intrinsic_interp_deref_at_offset;
1825 break;
1826 case ir_binop_interpolate_at_sample:
1827 op = nir_intrinsic_interp_deref_at_sample;
1828 break;
1829 default:
1830 unreachable("Invalid interpolation intrinsic");
1831 }
1832 } else {
1833 /* This case can happen if the vertex shader does not write the
1834 * given varying. In this case, the linker will lower it to a
1835 * global variable. Since interpolating a variable makes no
1836 * sense, we'll just turn it into a load which will probably
1837 * eventually end up as an SSA definition.
1838 */
1839 assert(nir_deref_mode_is(this->deref, nir_var_shader_temp));
1840 op = nir_intrinsic_load_deref;
1841 }
1842
1843 nir_intrinsic_instr *intrin = nir_intrinsic_instr_create(shader, op);
1844 intrin->num_components = deref->type->vector_elements;
1845 intrin->src[0] = nir_src_for_ssa(&this->deref->dest.ssa);
1846
1847 if (intrin->intrinsic == nir_intrinsic_interp_deref_at_offset ||
1848 intrin->intrinsic == nir_intrinsic_interp_deref_at_sample)
1849 intrin->src[1] = nir_src_for_ssa(evaluate_rvalue(ir->operands[1]));
1850
1851 unsigned bit_size = glsl_get_bit_size(deref->type);
1852 add_instr(&intrin->instr, deref->type->vector_elements, bit_size);
1853
1854 if (swizzle) {
1855 unsigned swiz[4] = {
1856 swizzle->mask.x, swizzle->mask.y, swizzle->mask.z, swizzle->mask.w
1857 };
1858
1859 result = nir_swizzle(&b, result, swiz,
1860 swizzle->type->vector_elements);
1861 }
1862
1863 return;
1864 }
1865
1866 case ir_unop_ssbo_unsized_array_length: {
1867 nir_intrinsic_instr *intrin =
1868 nir_intrinsic_instr_create(b.shader,
1869 nir_intrinsic_deref_buffer_array_length);
1870
1871 ir_dereference *deref = ir->operands[0]->as_dereference();
1872 intrin->src[0] = nir_src_for_ssa(&evaluate_deref(deref)->dest.ssa);
1873
1874 add_instr(&intrin->instr, 1, 32);
1875 return;
1876 }
1877
1878 case ir_binop_ubo_load:
1879 /* UBO loads should only have been lowered in GLSL IR for non-nir drivers,
1880 * NIR drivers make use of gl_nir_lower_buffers() instead.
1881 */
1882 unreachable("Invalid operation nir doesn't want lowered ubo loads");
1883 default:
1884 break;
1885 }
1886
1887 nir_ssa_def *srcs[4];
1888 for (unsigned i = 0; i < ir->num_operands; i++)
1889 srcs[i] = evaluate_rvalue(ir->operands[i]);
1890
1891 glsl_base_type types[4];
1892 for (unsigned i = 0; i < ir->num_operands; i++)
1893 types[i] = ir->operands[i]->type->base_type;
1894
1895 glsl_base_type out_type = ir->type->base_type;
1896
1897 switch (ir->operation) {
1898 case ir_unop_bit_not: result = nir_inot(&b, srcs[0]); break;
1899 case ir_unop_logic_not:
1900 result = nir_inot(&b, srcs[0]);
1901 break;
1902 case ir_unop_neg:
1903 result = type_is_float(types[0]) ? nir_fneg(&b, srcs[0])
1904 : nir_ineg(&b, srcs[0]);
1905 break;
1906 case ir_unop_abs:
1907 result = type_is_float(types[0]) ? nir_fabs(&b, srcs[0])
1908 : nir_iabs(&b, srcs[0]);
1909 break;
1910 case ir_unop_clz:
1911 result = nir_uclz(&b, srcs[0]);
1912 break;
1913 case ir_unop_saturate:
1914 assert(type_is_float(types[0]));
1915 result = nir_fsat(&b, srcs[0]);
1916 break;
1917 case ir_unop_sign:
1918 result = type_is_float(types[0]) ? nir_fsign(&b, srcs[0])
1919 : nir_isign(&b, srcs[0]);
1920 break;
1921 case ir_unop_rcp: result = nir_frcp(&b, srcs[0]); break;
1922 case ir_unop_rsq: result = nir_frsq(&b, srcs[0]); break;
1923 case ir_unop_sqrt: result = nir_fsqrt(&b, srcs[0]); break;
1924 case ir_unop_exp: unreachable("ir_unop_exp should have been lowered");
1925 case ir_unop_log: unreachable("ir_unop_log should have been lowered");
1926 case ir_unop_exp2: result = nir_fexp2(&b, srcs[0]); break;
1927 case ir_unop_log2: result = nir_flog2(&b, srcs[0]); break;
1928 case ir_unop_i2f:
1929 case ir_unop_u2f:
1930 case ir_unop_b2f:
1931 case ir_unop_f2i:
1932 case ir_unop_f2u:
1933 case ir_unop_f2b:
1934 case ir_unop_i2b:
1935 case ir_unop_b2i:
1936 case ir_unop_b2i64:
1937 case ir_unop_d2f:
1938 case ir_unop_f2d:
1939 case ir_unop_f162f:
1940 case ir_unop_f2f16:
1941 case ir_unop_f162b:
1942 case ir_unop_b2f16:
1943 case ir_unop_i2i:
1944 case ir_unop_u2u:
1945 case ir_unop_d2i:
1946 case ir_unop_d2u:
1947 case ir_unop_d2b:
1948 case ir_unop_i2d:
1949 case ir_unop_u2d:
1950 case ir_unop_i642i:
1951 case ir_unop_i642u:
1952 case ir_unop_i642f:
1953 case ir_unop_i642b:
1954 case ir_unop_i642d:
1955 case ir_unop_u642i:
1956 case ir_unop_u642u:
1957 case ir_unop_u642f:
1958 case ir_unop_u642d:
1959 case ir_unop_i2i64:
1960 case ir_unop_u2i64:
1961 case ir_unop_f2i64:
1962 case ir_unop_d2i64:
1963 case ir_unop_i2u64:
1964 case ir_unop_u2u64:
1965 case ir_unop_f2u64:
1966 case ir_unop_d2u64:
1967 case ir_unop_i2u:
1968 case ir_unop_u2i:
1969 case ir_unop_i642u64:
1970 case ir_unop_u642i64: {
1971 nir_alu_type src_type = nir_get_nir_type_for_glsl_base_type(types[0]);
1972 nir_alu_type dst_type = nir_get_nir_type_for_glsl_base_type(out_type);
1973 result = nir_build_alu(&b, nir_type_conversion_op(src_type, dst_type,
1974 nir_rounding_mode_undef),
1975 srcs[0], NULL, NULL, NULL);
1976 /* b2i and b2f don't have fixed bit-size versions so the builder will
1977 * just assume 32 and we have to fix it up here.
1978 */
1979 result->bit_size = nir_alu_type_get_type_size(dst_type);
1980 break;
1981 }
1982
1983 case ir_unop_f2fmp: {
1984 result = nir_build_alu(&b, nir_op_f2fmp, srcs[0], NULL, NULL, NULL);
1985 break;
1986 }
1987
1988 case ir_unop_i2imp: {
1989 result = nir_build_alu(&b, nir_op_i2imp, srcs[0], NULL, NULL, NULL);
1990 break;
1991 }
1992
1993 case ir_unop_u2ump: {
1994 result = nir_build_alu(&b, nir_op_i2imp, srcs[0], NULL, NULL, NULL);
1995 break;
1996 }
1997
1998 case ir_unop_bitcast_i2f:
1999 case ir_unop_bitcast_f2i:
2000 case ir_unop_bitcast_u2f:
2001 case ir_unop_bitcast_f2u:
2002 case ir_unop_bitcast_i642d:
2003 case ir_unop_bitcast_d2i64:
2004 case ir_unop_bitcast_u642d:
2005 case ir_unop_bitcast_d2u64:
2006 case ir_unop_subroutine_to_int:
2007 /* no-op */
2008 result = nir_mov(&b, srcs[0]);
2009 break;
2010 case ir_unop_trunc: result = nir_ftrunc(&b, srcs[0]); break;
2011 case ir_unop_ceil: result = nir_fceil(&b, srcs[0]); break;
2012 case ir_unop_floor: result = nir_ffloor(&b, srcs[0]); break;
2013 case ir_unop_fract: result = nir_ffract(&b, srcs[0]); break;
2014 case ir_unop_frexp_exp: result = nir_frexp_exp(&b, srcs[0]); break;
2015 case ir_unop_frexp_sig: result = nir_frexp_sig(&b, srcs[0]); break;
2016 case ir_unop_round_even: result = nir_fround_even(&b, srcs[0]); break;
2017 case ir_unop_sin: result = nir_fsin(&b, srcs[0]); break;
2018 case ir_unop_cos: result = nir_fcos(&b, srcs[0]); break;
2019 case ir_unop_dFdx: result = nir_fddx(&b, srcs[0]); break;
2020 case ir_unop_dFdy: result = nir_fddy(&b, srcs[0]); break;
2021 case ir_unop_dFdx_fine: result = nir_fddx_fine(&b, srcs[0]); break;
2022 case ir_unop_dFdy_fine: result = nir_fddy_fine(&b, srcs[0]); break;
2023 case ir_unop_dFdx_coarse: result = nir_fddx_coarse(&b, srcs[0]); break;
2024 case ir_unop_dFdy_coarse: result = nir_fddy_coarse(&b, srcs[0]); break;
2025 case ir_unop_pack_snorm_2x16:
2026 result = nir_pack_snorm_2x16(&b, srcs[0]);
2027 break;
2028 case ir_unop_pack_snorm_4x8:
2029 result = nir_pack_snorm_4x8(&b, srcs[0]);
2030 break;
2031 case ir_unop_pack_unorm_2x16:
2032 result = nir_pack_unorm_2x16(&b, srcs[0]);
2033 break;
2034 case ir_unop_pack_unorm_4x8:
2035 result = nir_pack_unorm_4x8(&b, srcs[0]);
2036 break;
2037 case ir_unop_pack_half_2x16:
2038 result = nir_pack_half_2x16(&b, srcs[0]);
2039 break;
2040 case ir_unop_unpack_snorm_2x16:
2041 result = nir_unpack_snorm_2x16(&b, srcs[0]);
2042 break;
2043 case ir_unop_unpack_snorm_4x8:
2044 result = nir_unpack_snorm_4x8(&b, srcs[0]);
2045 break;
2046 case ir_unop_unpack_unorm_2x16:
2047 result = nir_unpack_unorm_2x16(&b, srcs[0]);
2048 break;
2049 case ir_unop_unpack_unorm_4x8:
2050 result = nir_unpack_unorm_4x8(&b, srcs[0]);
2051 break;
2052 case ir_unop_unpack_half_2x16:
2053 result = nir_unpack_half_2x16(&b, srcs[0]);
2054 break;
2055 case ir_unop_pack_sampler_2x32:
2056 case ir_unop_pack_image_2x32:
2057 case ir_unop_pack_double_2x32:
2058 case ir_unop_pack_int_2x32:
2059 case ir_unop_pack_uint_2x32:
2060 result = nir_pack_64_2x32(&b, srcs[0]);
2061 break;
2062 case ir_unop_unpack_sampler_2x32:
2063 case ir_unop_unpack_image_2x32:
2064 case ir_unop_unpack_double_2x32:
2065 case ir_unop_unpack_int_2x32:
2066 case ir_unop_unpack_uint_2x32:
2067 result = nir_unpack_64_2x32(&b, srcs[0]);
2068 break;
2069 case ir_unop_bitfield_reverse:
2070 result = nir_bitfield_reverse(&b, srcs[0]);
2071 break;
2072 case ir_unop_bit_count:
2073 result = nir_bit_count(&b, srcs[0]);
2074 break;
2075 case ir_unop_find_msb:
2076 switch (types[0]) {
2077 case GLSL_TYPE_UINT:
2078 result = nir_ufind_msb(&b, srcs[0]);
2079 break;
2080 case GLSL_TYPE_INT:
2081 result = nir_ifind_msb(&b, srcs[0]);
2082 break;
2083 default:
2084 unreachable("Invalid type for findMSB()");
2085 }
2086 break;
2087 case ir_unop_find_lsb:
2088 result = nir_find_lsb(&b, srcs[0]);
2089 break;
2090
2091 case ir_unop_get_buffer_size: {
2092 nir_intrinsic_instr *load = nir_intrinsic_instr_create(
2093 this->shader,
2094 nir_intrinsic_get_ssbo_size);
2095 load->num_components = ir->type->vector_elements;
2096 load->src[0] = nir_src_for_ssa(evaluate_rvalue(ir->operands[0]));
2097 unsigned bit_size = glsl_get_bit_size(ir->type);
2098 add_instr(&load->instr, ir->type->vector_elements, bit_size);
2099 return;
2100 }
2101
2102 case ir_unop_atan:
2103 result = nir_atan(&b, srcs[0]);
2104 break;
2105
2106 case ir_binop_add:
2107 result = type_is_float(out_type) ? nir_fadd(&b, srcs[0], srcs[1])
2108 : nir_iadd(&b, srcs[0], srcs[1]);
2109 break;
2110 case ir_binop_add_sat:
2111 result = type_is_signed(out_type) ? nir_iadd_sat(&b, srcs[0], srcs[1])
2112 : nir_uadd_sat(&b, srcs[0], srcs[1]);
2113 break;
2114 case ir_binop_sub:
2115 result = type_is_float(out_type) ? nir_fsub(&b, srcs[0], srcs[1])
2116 : nir_isub(&b, srcs[0], srcs[1]);
2117 break;
2118 case ir_binop_sub_sat:
2119 result = type_is_signed(out_type) ? nir_isub_sat(&b, srcs[0], srcs[1])
2120 : nir_usub_sat(&b, srcs[0], srcs[1]);
2121 break;
2122 case ir_binop_abs_sub:
2123 /* out_type is always unsigned for ir_binop_abs_sub, so we have to key
2124 * on the type of the sources.
2125 */
2126 result = type_is_signed(types[0]) ? nir_uabs_isub(&b, srcs[0], srcs[1])
2127 : nir_uabs_usub(&b, srcs[0], srcs[1]);
2128 break;
2129 case ir_binop_avg:
2130 result = type_is_signed(out_type) ? nir_ihadd(&b, srcs[0], srcs[1])
2131 : nir_uhadd(&b, srcs[0], srcs[1]);
2132 break;
2133 case ir_binop_avg_round:
2134 result = type_is_signed(out_type) ? nir_irhadd(&b, srcs[0], srcs[1])
2135 : nir_urhadd(&b, srcs[0], srcs[1]);
2136 break;
2137 case ir_binop_mul_32x16:
2138 result = type_is_signed(out_type) ? nir_imul_32x16(&b, srcs[0], srcs[1])
2139 : nir_umul_32x16(&b, srcs[0], srcs[1]);
2140 break;
2141 case ir_binop_mul:
2142 if (type_is_float(out_type))
2143 result = nir_fmul(&b, srcs[0], srcs[1]);
2144 else if (out_type == GLSL_TYPE_INT64 &&
2145 (ir->operands[0]->type->base_type == GLSL_TYPE_INT ||
2146 ir->operands[1]->type->base_type == GLSL_TYPE_INT))
2147 result = nir_imul_2x32_64(&b, srcs[0], srcs[1]);
2148 else if (out_type == GLSL_TYPE_UINT64 &&
2149 (ir->operands[0]->type->base_type == GLSL_TYPE_UINT ||
2150 ir->operands[1]->type->base_type == GLSL_TYPE_UINT))
2151 result = nir_umul_2x32_64(&b, srcs[0], srcs[1]);
2152 else
2153 result = nir_imul(&b, srcs[0], srcs[1]);
2154 break;
2155 case ir_binop_div:
2156 if (type_is_float(out_type))
2157 result = nir_fdiv(&b, srcs[0], srcs[1]);
2158 else if (type_is_signed(out_type))
2159 result = nir_idiv(&b, srcs[0], srcs[1]);
2160 else
2161 result = nir_udiv(&b, srcs[0], srcs[1]);
2162 break;
2163 case ir_binop_mod:
2164 result = type_is_float(out_type) ? nir_fmod(&b, srcs[0], srcs[1])
2165 : nir_umod(&b, srcs[0], srcs[1]);
2166 break;
2167 case ir_binop_min:
2168 if (type_is_float(out_type))
2169 result = nir_fmin(&b, srcs[0], srcs[1]);
2170 else if (type_is_signed(out_type))
2171 result = nir_imin(&b, srcs[0], srcs[1]);
2172 else
2173 result = nir_umin(&b, srcs[0], srcs[1]);
2174 break;
2175 case ir_binop_max:
2176 if (type_is_float(out_type))
2177 result = nir_fmax(&b, srcs[0], srcs[1]);
2178 else if (type_is_signed(out_type))
2179 result = nir_imax(&b, srcs[0], srcs[1]);
2180 else
2181 result = nir_umax(&b, srcs[0], srcs[1]);
2182 break;
2183 case ir_binop_pow: result = nir_fpow(&b, srcs[0], srcs[1]); break;
2184 case ir_binop_bit_and: result = nir_iand(&b, srcs[0], srcs[1]); break;
2185 case ir_binop_bit_or: result = nir_ior(&b, srcs[0], srcs[1]); break;
2186 case ir_binop_bit_xor: result = nir_ixor(&b, srcs[0], srcs[1]); break;
2187 case ir_binop_logic_and:
2188 result = nir_iand(&b, srcs[0], srcs[1]);
2189 break;
2190 case ir_binop_logic_or:
2191 result = nir_ior(&b, srcs[0], srcs[1]);
2192 break;
2193 case ir_binop_logic_xor:
2194 result = nir_ixor(&b, srcs[0], srcs[1]);
2195 break;
2196 case ir_binop_lshift: result = nir_ishl(&b, srcs[0], nir_u2u32(&b, srcs[1])); break;
2197 case ir_binop_rshift:
2198 result = (type_is_signed(out_type)) ? nir_ishr(&b, srcs[0], nir_u2u32(&b, srcs[1]))
2199 : nir_ushr(&b, srcs[0], nir_u2u32(&b, srcs[1]));
2200 break;
2201 case ir_binop_imul_high:
2202 result = (out_type == GLSL_TYPE_INT) ? nir_imul_high(&b, srcs[0], srcs[1])
2203 : nir_umul_high(&b, srcs[0], srcs[1]);
2204 break;
2205 case ir_binop_carry: result = nir_uadd_carry(&b, srcs[0], srcs[1]); break;
2206 case ir_binop_borrow: result = nir_usub_borrow(&b, srcs[0], srcs[1]); break;
2207 case ir_binop_less:
2208 if (type_is_float(types[0]))
2209 result = nir_flt(&b, srcs[0], srcs[1]);
2210 else if (type_is_signed(types[0]))
2211 result = nir_ilt(&b, srcs[0], srcs[1]);
2212 else
2213 result = nir_ult(&b, srcs[0], srcs[1]);
2214 break;
2215 case ir_binop_gequal:
2216 if (type_is_float(types[0]))
2217 result = nir_fge(&b, srcs[0], srcs[1]);
2218 else if (type_is_signed(types[0]))
2219 result = nir_ige(&b, srcs[0], srcs[1]);
2220 else
2221 result = nir_uge(&b, srcs[0], srcs[1]);
2222 break;
2223 case ir_binop_equal:
2224 if (type_is_float(types[0]))
2225 result = nir_feq(&b, srcs[0], srcs[1]);
2226 else
2227 result = nir_ieq(&b, srcs[0], srcs[1]);
2228 break;
2229 case ir_binop_nequal:
2230 if (type_is_float(types[0]))
2231 result = nir_fneu(&b, srcs[0], srcs[1]);
2232 else
2233 result = nir_ine(&b, srcs[0], srcs[1]);
2234 break;
2235 case ir_binop_all_equal:
2236 if (type_is_float(types[0])) {
2237 switch (ir->operands[0]->type->vector_elements) {
2238 case 1: result = nir_feq(&b, srcs[0], srcs[1]); break;
2239 case 2: result = nir_ball_fequal2(&b, srcs[0], srcs[1]); break;
2240 case 3: result = nir_ball_fequal3(&b, srcs[0], srcs[1]); break;
2241 case 4: result = nir_ball_fequal4(&b, srcs[0], srcs[1]); break;
2242 default:
2243 unreachable("not reached");
2244 }
2245 } else {
2246 switch (ir->operands[0]->type->vector_elements) {
2247 case 1: result = nir_ieq(&b, srcs[0], srcs[1]); break;
2248 case 2: result = nir_ball_iequal2(&b, srcs[0], srcs[1]); break;
2249 case 3: result = nir_ball_iequal3(&b, srcs[0], srcs[1]); break;
2250 case 4: result = nir_ball_iequal4(&b, srcs[0], srcs[1]); break;
2251 default:
2252 unreachable("not reached");
2253 }
2254 }
2255 break;
2256 case ir_binop_any_nequal:
2257 if (type_is_float(types[0])) {
2258 switch (ir->operands[0]->type->vector_elements) {
2259 case 1: result = nir_fneu(&b, srcs[0], srcs[1]); break;
2260 case 2: result = nir_bany_fnequal2(&b, srcs[0], srcs[1]); break;
2261 case 3: result = nir_bany_fnequal3(&b, srcs[0], srcs[1]); break;
2262 case 4: result = nir_bany_fnequal4(&b, srcs[0], srcs[1]); break;
2263 default:
2264 unreachable("not reached");
2265 }
2266 } else {
2267 switch (ir->operands[0]->type->vector_elements) {
2268 case 1: result = nir_ine(&b, srcs[0], srcs[1]); break;
2269 case 2: result = nir_bany_inequal2(&b, srcs[0], srcs[1]); break;
2270 case 3: result = nir_bany_inequal3(&b, srcs[0], srcs[1]); break;
2271 case 4: result = nir_bany_inequal4(&b, srcs[0], srcs[1]); break;
2272 default:
2273 unreachable("not reached");
2274 }
2275 }
2276 break;
2277 case ir_binop_dot:
2278 result = nir_fdot(&b, srcs[0], srcs[1]);
2279 break;
2280 case ir_binop_vector_extract: {
2281 result = nir_channel(&b, srcs[0], 0);
2282 for (unsigned i = 1; i < ir->operands[0]->type->vector_elements; i++) {
2283 nir_ssa_def *swizzled = nir_channel(&b, srcs[0], i);
2284 result = nir_bcsel(&b, nir_ieq_imm(&b, srcs[1], i),
2285 swizzled, result);
2286 }
2287 break;
2288 }
2289
2290 case ir_binop_atan2:
2291 result = nir_atan2(&b, srcs[0], srcs[1]);
2292 break;
2293
2294 case ir_binop_ldexp: result = nir_ldexp(&b, srcs[0], srcs[1]); break;
2295 case ir_triop_fma:
2296 result = nir_ffma(&b, srcs[0], srcs[1], srcs[2]);
2297 break;
2298 case ir_triop_lrp:
2299 result = nir_flrp(&b, srcs[0], srcs[1], srcs[2]);
2300 break;
2301 case ir_triop_csel:
2302 result = nir_bcsel(&b, srcs[0], srcs[1], srcs[2]);
2303 break;
2304 case ir_triop_bitfield_extract:
2305 result = ir->type->is_int_16_32() ?
2306 nir_ibitfield_extract(&b, nir_i2i32(&b, srcs[0]), nir_i2i32(&b, srcs[1]), nir_i2i32(&b, srcs[2])) :
2307 nir_ubitfield_extract(&b, nir_u2u32(&b, srcs[0]), nir_i2i32(&b, srcs[1]), nir_i2i32(&b, srcs[2]));
2308 break;
2309 case ir_quadop_bitfield_insert:
2310 result = nir_bitfield_insert(&b,
2311 nir_u2u32(&b, srcs[0]), nir_u2u32(&b, srcs[1]),
2312 nir_i2i32(&b, srcs[2]), nir_i2i32(&b, srcs[3]));
2313 break;
2314 case ir_quadop_vector:
2315 result = nir_vec(&b, srcs, ir->type->vector_elements);
2316 break;
2317
2318 default:
2319 unreachable("not reached");
2320 }
2321 }
2322
2323 void
visit(ir_swizzle * ir)2324 nir_visitor::visit(ir_swizzle *ir)
2325 {
2326 unsigned swizzle[4] = { ir->mask.x, ir->mask.y, ir->mask.z, ir->mask.w };
2327 result = nir_swizzle(&b, evaluate_rvalue(ir->val), swizzle,
2328 ir->type->vector_elements);
2329 }
2330
2331 void
visit(ir_texture * ir)2332 nir_visitor::visit(ir_texture *ir)
2333 {
2334 unsigned num_srcs;
2335 nir_texop op;
2336 switch (ir->op) {
2337 case ir_tex:
2338 op = nir_texop_tex;
2339 num_srcs = 1; /* coordinate */
2340 break;
2341
2342 case ir_txb:
2343 case ir_txl:
2344 op = (ir->op == ir_txb) ? nir_texop_txb : nir_texop_txl;
2345 num_srcs = 2; /* coordinate, bias/lod */
2346 break;
2347
2348 case ir_txd:
2349 op = nir_texop_txd; /* coordinate, dPdx, dPdy */
2350 num_srcs = 3;
2351 break;
2352
2353 case ir_txf:
2354 op = nir_texop_txf;
2355 if (ir->lod_info.lod != NULL)
2356 num_srcs = 2; /* coordinate, lod */
2357 else
2358 num_srcs = 1; /* coordinate */
2359 break;
2360
2361 case ir_txf_ms:
2362 op = nir_texop_txf_ms;
2363 num_srcs = 2; /* coordinate, sample_index */
2364 break;
2365
2366 case ir_txs:
2367 op = nir_texop_txs;
2368 if (ir->lod_info.lod != NULL)
2369 num_srcs = 1; /* lod */
2370 else
2371 num_srcs = 0;
2372 break;
2373
2374 case ir_lod:
2375 op = nir_texop_lod;
2376 num_srcs = 1; /* coordinate */
2377 break;
2378
2379 case ir_tg4:
2380 op = nir_texop_tg4;
2381 num_srcs = 1; /* coordinate */
2382 break;
2383
2384 case ir_query_levels:
2385 op = nir_texop_query_levels;
2386 num_srcs = 0;
2387 break;
2388
2389 case ir_texture_samples:
2390 op = nir_texop_texture_samples;
2391 num_srcs = 0;
2392 break;
2393
2394 case ir_samples_identical:
2395 op = nir_texop_samples_identical;
2396 num_srcs = 1; /* coordinate */
2397 break;
2398
2399 default:
2400 unreachable("not reached");
2401 }
2402
2403 if (ir->projector != NULL)
2404 num_srcs++;
2405 if (ir->shadow_comparator != NULL)
2406 num_srcs++;
2407 /* offsets are constants we store inside nir_tex_intrs.offsets */
2408 if (ir->offset != NULL && !ir->offset->type->is_array())
2409 num_srcs++;
2410
2411 /* Add one for the texture deref */
2412 num_srcs += 2;
2413
2414 nir_tex_instr *instr = nir_tex_instr_create(this->shader, num_srcs);
2415
2416 instr->op = op;
2417 instr->sampler_dim =
2418 (glsl_sampler_dim) ir->sampler->type->sampler_dimensionality;
2419 instr->is_array = ir->sampler->type->sampler_array;
2420 instr->is_shadow = ir->sampler->type->sampler_shadow;
2421 if (instr->is_shadow)
2422 instr->is_new_style_shadow = (ir->type->vector_elements == 1);
2423 instr->dest_type = nir_get_nir_type_for_glsl_type(ir->type);
2424
2425 nir_deref_instr *sampler_deref = evaluate_deref(ir->sampler);
2426
2427 /* check for bindless handles */
2428 if (!nir_deref_mode_is(sampler_deref, nir_var_uniform) ||
2429 nir_deref_instr_get_variable(sampler_deref)->data.bindless) {
2430 nir_ssa_def *load = nir_load_deref(&b, sampler_deref);
2431 instr->src[0].src = nir_src_for_ssa(load);
2432 instr->src[0].src_type = nir_tex_src_texture_handle;
2433 instr->src[1].src = nir_src_for_ssa(load);
2434 instr->src[1].src_type = nir_tex_src_sampler_handle;
2435 } else {
2436 instr->src[0].src = nir_src_for_ssa(&sampler_deref->dest.ssa);
2437 instr->src[0].src_type = nir_tex_src_texture_deref;
2438 instr->src[1].src = nir_src_for_ssa(&sampler_deref->dest.ssa);
2439 instr->src[1].src_type = nir_tex_src_sampler_deref;
2440 }
2441
2442 unsigned src_number = 2;
2443
2444 if (ir->coordinate != NULL) {
2445 instr->coord_components = ir->coordinate->type->vector_elements;
2446 instr->src[src_number].src =
2447 nir_src_for_ssa(evaluate_rvalue(ir->coordinate));
2448 instr->src[src_number].src_type = nir_tex_src_coord;
2449 src_number++;
2450 }
2451
2452 if (ir->projector != NULL) {
2453 instr->src[src_number].src =
2454 nir_src_for_ssa(evaluate_rvalue(ir->projector));
2455 instr->src[src_number].src_type = nir_tex_src_projector;
2456 src_number++;
2457 }
2458
2459 if (ir->shadow_comparator != NULL) {
2460 instr->src[src_number].src =
2461 nir_src_for_ssa(evaluate_rvalue(ir->shadow_comparator));
2462 instr->src[src_number].src_type = nir_tex_src_comparator;
2463 src_number++;
2464 }
2465
2466 if (ir->offset != NULL) {
2467 if (ir->offset->type->is_array()) {
2468 for (int i = 0; i < ir->offset->type->array_size(); i++) {
2469 const ir_constant *c =
2470 ir->offset->as_constant()->get_array_element(i);
2471
2472 for (unsigned j = 0; j < 2; ++j) {
2473 int val = c->get_int_component(j);
2474 assert(val <= 31 && val >= -32);
2475 instr->tg4_offsets[i][j] = val;
2476 }
2477 }
2478 } else {
2479 assert(ir->offset->type->is_vector() || ir->offset->type->is_scalar());
2480
2481 instr->src[src_number].src =
2482 nir_src_for_ssa(evaluate_rvalue(ir->offset));
2483 instr->src[src_number].src_type = nir_tex_src_offset;
2484 src_number++;
2485 }
2486 }
2487
2488 switch (ir->op) {
2489 case ir_txb:
2490 instr->src[src_number].src =
2491 nir_src_for_ssa(evaluate_rvalue(ir->lod_info.bias));
2492 instr->src[src_number].src_type = nir_tex_src_bias;
2493 src_number++;
2494 break;
2495
2496 case ir_txl:
2497 case ir_txf:
2498 case ir_txs:
2499 if (ir->lod_info.lod != NULL) {
2500 instr->src[src_number].src =
2501 nir_src_for_ssa(evaluate_rvalue(ir->lod_info.lod));
2502 instr->src[src_number].src_type = nir_tex_src_lod;
2503 src_number++;
2504 }
2505 break;
2506
2507 case ir_txd:
2508 instr->src[src_number].src =
2509 nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdx));
2510 instr->src[src_number].src_type = nir_tex_src_ddx;
2511 src_number++;
2512 instr->src[src_number].src =
2513 nir_src_for_ssa(evaluate_rvalue(ir->lod_info.grad.dPdy));
2514 instr->src[src_number].src_type = nir_tex_src_ddy;
2515 src_number++;
2516 break;
2517
2518 case ir_txf_ms:
2519 instr->src[src_number].src =
2520 nir_src_for_ssa(evaluate_rvalue(ir->lod_info.sample_index));
2521 instr->src[src_number].src_type = nir_tex_src_ms_index;
2522 src_number++;
2523 break;
2524
2525 case ir_tg4:
2526 instr->component = ir->lod_info.component->as_constant()->value.u[0];
2527 break;
2528
2529 default:
2530 break;
2531 }
2532
2533 assert(src_number == num_srcs);
2534
2535 unsigned bit_size = glsl_get_bit_size(ir->type);
2536 add_instr(&instr->instr, nir_tex_instr_dest_size(instr), bit_size);
2537 }
2538
2539 void
visit(ir_constant * ir)2540 nir_visitor::visit(ir_constant *ir)
2541 {
2542 /*
2543 * We don't know if this variable is an array or struct that gets
2544 * dereferenced, so do the safe thing an make it a variable with a
2545 * constant initializer and return a dereference.
2546 */
2547
2548 nir_variable *var =
2549 nir_local_variable_create(this->impl, ir->type, "const_temp");
2550 var->data.read_only = true;
2551 var->constant_initializer = constant_copy(ir, var);
2552
2553 this->deref = nir_build_deref_var(&b, var);
2554 }
2555
2556 void
visit(ir_dereference_variable * ir)2557 nir_visitor::visit(ir_dereference_variable *ir)
2558 {
2559 if (ir->variable_referenced()->data.mode == ir_var_function_out) {
2560 unsigned i = (sig->return_type != glsl_type::void_type) ? 1 : 0;
2561
2562 foreach_in_list(ir_variable, param, &sig->parameters) {
2563 if (param == ir->variable_referenced()) {
2564 break;
2565 }
2566 i++;
2567 }
2568
2569 this->deref = nir_build_deref_cast(&b, nir_load_param(&b, i),
2570 nir_var_function_temp, ir->type, 0);
2571 return;
2572 }
2573
2574 assert(ir->variable_referenced()->data.mode != ir_var_function_inout);
2575
2576 struct hash_entry *entry =
2577 _mesa_hash_table_search(this->var_table, ir->var);
2578 assert(entry);
2579 nir_variable *var = (nir_variable *) entry->data;
2580
2581 this->deref = nir_build_deref_var(&b, var);
2582 }
2583
2584 void
visit(ir_dereference_record * ir)2585 nir_visitor::visit(ir_dereference_record *ir)
2586 {
2587 ir->record->accept(this);
2588
2589 int field_index = ir->field_idx;
2590 assert(field_index >= 0);
2591
2592 this->deref = nir_build_deref_struct(&b, this->deref, field_index);
2593 }
2594
2595 void
visit(ir_dereference_array * ir)2596 nir_visitor::visit(ir_dereference_array *ir)
2597 {
2598 nir_ssa_def *index = evaluate_rvalue(ir->array_index);
2599
2600 ir->array->accept(this);
2601
2602 this->deref = nir_build_deref_array(&b, this->deref, index);
2603 }
2604
2605 void
visit(ir_barrier *)2606 nir_visitor::visit(ir_barrier *)
2607 {
2608 if (shader->info.stage == MESA_SHADER_COMPUTE)
2609 nir_memory_barrier_shared(&b);
2610 else if (shader->info.stage == MESA_SHADER_TESS_CTRL)
2611 nir_memory_barrier_tcs_patch(&b);
2612
2613 nir_control_barrier(&b);
2614 }
2615
2616 nir_shader *
glsl_float64_funcs_to_nir(struct gl_context * ctx,const nir_shader_compiler_options * options)2617 glsl_float64_funcs_to_nir(struct gl_context *ctx,
2618 const nir_shader_compiler_options *options)
2619 {
2620 /* It's not possible to use float64 on GLSL ES, so don't bother trying to
2621 * build the support code. The support code depends on higher versions of
2622 * desktop GLSL, so it will fail to compile (below) anyway.
2623 */
2624 if (!_mesa_is_desktop_gl(ctx) || ctx->Const.GLSLVersion < 400)
2625 return NULL;
2626
2627 /* We pretend it's a vertex shader. Ultimately, the stage shouldn't
2628 * matter because we're not optimizing anything here.
2629 */
2630 struct gl_shader *sh = _mesa_new_shader(-1, MESA_SHADER_VERTEX);
2631 sh->Source = float64_source;
2632 sh->CompileStatus = COMPILE_FAILURE;
2633 _mesa_glsl_compile_shader(ctx, sh, false, false, true);
2634
2635 if (!sh->CompileStatus) {
2636 if (sh->InfoLog) {
2637 _mesa_problem(ctx,
2638 "fp64 software impl compile failed:\n%s\nsource:\n%s\n",
2639 sh->InfoLog, float64_source);
2640 }
2641 return NULL;
2642 }
2643
2644 nir_shader *nir = nir_shader_create(NULL, MESA_SHADER_VERTEX, options, NULL);
2645
2646 nir_visitor v1(ctx, nir);
2647 nir_function_visitor v2(&v1);
2648 v2.run(sh->ir);
2649 visit_exec_list(sh->ir, &v1);
2650
2651 /* _mesa_delete_shader will try to free sh->Source but it's static const */
2652 sh->Source = NULL;
2653 _mesa_delete_shader(ctx, sh);
2654
2655 nir_validate_shader(nir, "float64_funcs_to_nir");
2656
2657 NIR_PASS_V(nir, nir_lower_variable_initializers, nir_var_function_temp);
2658 NIR_PASS_V(nir, nir_lower_returns);
2659 NIR_PASS_V(nir, nir_inline_functions);
2660 NIR_PASS_V(nir, nir_opt_deref);
2661
2662 /* Do some optimizations to clean up the shader now. By optimizing the
2663 * functions in the library, we avoid having to re-do that work every
2664 * time we inline a copy of a function. Reducing basic blocks also helps
2665 * with compile times.
2666 */
2667 NIR_PASS_V(nir, nir_lower_vars_to_ssa);
2668 NIR_PASS_V(nir, nir_copy_prop);
2669 NIR_PASS_V(nir, nir_opt_dce);
2670 NIR_PASS_V(nir, nir_opt_cse);
2671 NIR_PASS_V(nir, nir_opt_gcm, true);
2672 NIR_PASS_V(nir, nir_opt_peephole_select, 1, false, false);
2673 NIR_PASS_V(nir, nir_opt_dce);
2674
2675 return nir;
2676 }
2677