1 /*
2 * Copyright (c) 2021 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "brw_nir_rt.h"
25 #include "brw_nir_rt_builder.h"
26
27 #include "nir_deref.h"
28
29 #include "util/macros.h"
30
31 struct lowering_state {
32 const struct intel_device_info *devinfo;
33
34 struct hash_table *queries;
35 uint32_t n_queries;
36
37 struct brw_nir_rt_globals_defs globals;
38 nir_ssa_def *rq_globals;
39 };
40
41 struct brw_ray_query {
42 nir_variable *opaque_var;
43 uint32_t id;
44 };
45
46 static bool
need_spill_fill(struct lowering_state * state)47 need_spill_fill(struct lowering_state *state)
48 {
49 return state->n_queries > 1;
50 }
51
52 /**
53 * This pass converts opaque RayQuery structures from SPIRV into a vec3 where
54 * the first 2 elements store a global address for the query and the third
55 * element is an incremented counter on the number of executed
56 * nir_intrinsic_rq_proceed.
57 */
58
59 static bool
maybe_create_brw_var(nir_instr * instr,struct lowering_state * state)60 maybe_create_brw_var(nir_instr *instr, struct lowering_state *state)
61 {
62 if (instr->type != nir_instr_type_deref)
63 return false;
64
65 nir_deref_instr *deref = nir_instr_as_deref(instr);
66 if (deref->deref_type != nir_deref_type_var &&
67 deref->deref_type != nir_deref_type_array)
68 return false;
69
70 nir_variable *opaque_var = nir_deref_instr_get_variable(deref);
71 if (!opaque_var || !opaque_var->data.ray_query)
72 return false;
73
74 struct hash_entry *entry = _mesa_hash_table_search(state->queries, opaque_var);
75 if (entry)
76 return false;
77
78 struct brw_ray_query *rq = rzalloc(state->queries, struct brw_ray_query);
79 rq->opaque_var = opaque_var;
80 rq->id = state->n_queries;
81
82 _mesa_hash_table_insert(state->queries, opaque_var, rq);
83
84 unsigned aoa_size = glsl_get_aoa_size(opaque_var->type);
85 state->n_queries += MAX2(1, aoa_size);
86
87 return true;
88 }
89
90 static nir_ssa_def *
get_ray_query_shadow_addr(nir_builder * b,nir_deref_instr * deref,struct lowering_state * state,nir_ssa_def ** out_state_addr)91 get_ray_query_shadow_addr(nir_builder *b,
92 nir_deref_instr *deref,
93 struct lowering_state *state,
94 nir_ssa_def **out_state_addr)
95 {
96 nir_deref_path path;
97 nir_deref_path_init(&path, deref, NULL);
98 assert(path.path[0]->deref_type == nir_deref_type_var);
99
100 nir_variable *opaque_var = nir_deref_instr_get_variable(path.path[0]);
101 struct hash_entry *entry = _mesa_hash_table_search(state->queries, opaque_var);
102 assert(entry);
103
104 struct brw_ray_query *rq = entry->data;
105
106 /* Base address in the shadow memory of the variable associated with this
107 * ray query variable.
108 */
109 nir_ssa_def *base_addr =
110 nir_iadd_imm(b, state->globals.resume_sbt_addr,
111 brw_rt_ray_queries_shadow_stack_size(state->devinfo) * rq->id);
112
113 bool spill_fill = need_spill_fill(state);
114 *out_state_addr =
115 spill_fill ?
116 nir_iadd_imm(b,
117 state->globals.resume_sbt_addr,
118 brw_rt_ray_queries_shadow_stack_size(state->devinfo) *
119 b->shader->info.ray_queries +
120 4 * rq->id) :
121 state->globals.resume_sbt_addr;
122
123 if (!spill_fill)
124 return NULL;
125
126 /* Just emit code and let constant-folding go to town */
127 nir_deref_instr **p = &path.path[1];
128 for (; *p; p++) {
129 if ((*p)->deref_type == nir_deref_type_array) {
130 nir_ssa_def *index = nir_ssa_for_src(b, (*p)->arr.index, 1);
131
132 /**/
133 uint32_t local_state_offset = 4 * MAX2(1, glsl_get_aoa_size((*p)->type));
134 *out_state_addr =
135 nir_iadd(b, *out_state_addr,
136 nir_i2i64(b,
137 nir_imul_imm(b, index, local_state_offset)));
138
139 /**/
140 uint64_t size = MAX2(1, glsl_get_aoa_size((*p)->type)) *
141 brw_rt_ray_queries_shadow_stack_size(state->devinfo);
142
143 nir_ssa_def *mul = nir_amul_imm(b, nir_i2i64(b, index), size);
144
145 base_addr = nir_iadd(b, base_addr, mul);
146 } else {
147 unreachable("Unsupported deref type");
148 }
149 }
150
151 nir_deref_path_finish(&path);
152
153 /* Add the lane offset to the shadow memory address */
154 nir_ssa_def *lane_offset =
155 nir_imul_imm(
156 b,
157 nir_iadd(
158 b,
159 nir_imul(
160 b,
161 brw_load_btd_dss_id(b),
162 brw_nir_rt_load_num_simd_lanes_per_dss(b, state->devinfo)),
163 brw_nir_rt_sync_stack_id(b)),
164 BRW_RT_SIZEOF_SHADOW_RAY_QUERY);
165
166 return nir_iadd(b, base_addr, nir_i2i64(b, lane_offset));
167 }
168
169 static void
update_trace_ctrl_level(nir_builder * b,nir_ssa_def * state_addr,nir_ssa_def ** out_old_ctrl,nir_ssa_def ** out_old_level,nir_ssa_def * new_ctrl,nir_ssa_def * new_level)170 update_trace_ctrl_level(nir_builder *b,
171 nir_ssa_def *state_addr,
172 nir_ssa_def **out_old_ctrl,
173 nir_ssa_def **out_old_level,
174 nir_ssa_def *new_ctrl,
175 nir_ssa_def *new_level)
176 {
177 nir_ssa_def *old_value = brw_nir_rt_load(b, state_addr, 4, 1, 32);
178 nir_ssa_def *old_ctrl = nir_ishr_imm(b, old_value, 2);
179 nir_ssa_def *old_level = nir_iand_imm(b, old_value, 0x3);
180
181 if (out_old_ctrl)
182 *out_old_ctrl = old_ctrl;
183 if (out_old_level)
184 *out_old_level = old_level;
185
186 if (new_ctrl || new_level) {
187 if (!new_ctrl)
188 new_ctrl = old_ctrl;
189 if (!new_level)
190 new_level = old_level;
191
192 nir_ssa_def *new_value = nir_ior(b, nir_ishl_imm(b, new_ctrl, 2), new_level);
193 brw_nir_rt_store(b, state_addr, 4, new_value, 0x1);
194 }
195 }
196
197 static void
fill_query(nir_builder * b,nir_ssa_def * hw_stack_addr,nir_ssa_def * shadow_stack_addr,nir_ssa_def * ctrl)198 fill_query(nir_builder *b,
199 nir_ssa_def *hw_stack_addr,
200 nir_ssa_def *shadow_stack_addr,
201 nir_ssa_def *ctrl)
202 {
203 brw_nir_memcpy_global(b,
204 brw_nir_rt_mem_hit_addr_from_addr(b, hw_stack_addr, false), 16,
205 brw_nir_rt_mem_hit_addr_from_addr(b, shadow_stack_addr, false), 16,
206 BRW_RT_SIZEOF_HIT_INFO);
207 brw_nir_memcpy_global(b,
208 brw_nir_rt_mem_hit_addr_from_addr(b, hw_stack_addr, true), 16,
209 brw_nir_rt_mem_hit_addr_from_addr(b, shadow_stack_addr, true), 16,
210 BRW_RT_SIZEOF_HIT_INFO);
211 brw_nir_memcpy_global(b,
212 brw_nir_rt_mem_ray_addr(b, hw_stack_addr,
213 BRW_RT_BVH_LEVEL_WORLD), 16,
214 brw_nir_rt_mem_ray_addr(b, shadow_stack_addr,
215 BRW_RT_BVH_LEVEL_WORLD), 16,
216 BRW_RT_SIZEOF_RAY);
217 }
218
219 static void
spill_query(nir_builder * b,nir_ssa_def * hw_stack_addr,nir_ssa_def * shadow_stack_addr)220 spill_query(nir_builder *b,
221 nir_ssa_def *hw_stack_addr,
222 nir_ssa_def *shadow_stack_addr)
223 {
224 struct brw_nir_rt_mem_hit_defs committed_hit = {};
225 brw_nir_rt_load_mem_hit_from_addr(b, &committed_hit, hw_stack_addr, true);
226
227 /* Always copy the potential hit back */
228 brw_nir_memcpy_global(b,
229 brw_nir_rt_mem_hit_addr_from_addr(b, shadow_stack_addr, false), 16,
230 brw_nir_rt_mem_hit_addr_from_addr(b, hw_stack_addr, false), 16,
231 BRW_RT_SIZEOF_HIT_INFO);
232
233 /* Also copy the committed hit back if it is valid */
234 nir_push_if(b, committed_hit.valid);
235 {
236 brw_nir_memcpy_global(b,
237 brw_nir_rt_mem_hit_addr_from_addr(b, shadow_stack_addr, true), 16,
238 brw_nir_rt_mem_hit_addr_from_addr(b, hw_stack_addr, true), 16,
239 BRW_RT_SIZEOF_HIT_INFO);
240 }
241 nir_pop_if(b, NULL);
242 }
243
244
245 static void
lower_ray_query_intrinsic(nir_builder * b,nir_intrinsic_instr * intrin,struct lowering_state * state)246 lower_ray_query_intrinsic(nir_builder *b,
247 nir_intrinsic_instr *intrin,
248 struct lowering_state *state)
249 {
250 nir_deref_instr *deref = nir_src_as_deref(intrin->src[0]);
251
252 b->cursor = nir_instr_remove(&intrin->instr);
253
254 nir_ssa_def *ctrl_level_addr;
255 nir_ssa_def *shadow_stack_addr =
256 get_ray_query_shadow_addr(b, deref, state, &ctrl_level_addr);
257 nir_ssa_def *hw_stack_addr =
258 brw_nir_rt_sync_stack_addr(b, state->globals.base_mem_addr, state->devinfo);
259 nir_ssa_def *stack_addr = shadow_stack_addr ? shadow_stack_addr : hw_stack_addr;
260
261 switch (intrin->intrinsic) {
262 case nir_intrinsic_rq_initialize: {
263 nir_ssa_def *as_addr = intrin->src[1].ssa;
264 nir_ssa_def *ray_flags = intrin->src[2].ssa;
265 /* From the SPIR-V spec:
266 *
267 * "Only the 8 least-significant bits of Cull Mask are used by
268 * this instruction - other bits are ignored.
269 *
270 * Only the 16 least-significant bits of Miss Index are used by
271 * this instruction - other bits are ignored."
272 */
273 nir_ssa_def *cull_mask = nir_iand_imm(b, intrin->src[3].ssa, 0xff);
274 nir_ssa_def *ray_orig = intrin->src[4].ssa;
275 nir_ssa_def *ray_t_min = intrin->src[5].ssa;
276 nir_ssa_def *ray_dir = intrin->src[6].ssa;
277 nir_ssa_def *ray_t_max = intrin->src[7].ssa;
278
279 nir_ssa_def *root_node_ptr =
280 brw_nir_rt_acceleration_structure_to_root_node(b, as_addr);
281
282 struct brw_nir_rt_mem_ray_defs ray_defs = {
283 .root_node_ptr = root_node_ptr,
284 .ray_flags = nir_u2u16(b, ray_flags),
285 .ray_mask = cull_mask,
286 .orig = ray_orig,
287 .t_near = ray_t_min,
288 .dir = ray_dir,
289 .t_far = ray_t_max,
290 };
291
292 nir_ssa_def *ray_addr =
293 brw_nir_rt_mem_ray_addr(b, stack_addr, BRW_RT_BVH_LEVEL_WORLD);
294
295 brw_nir_rt_query_mark_init(b, stack_addr);
296 brw_nir_rt_init_mem_hit_at_addr(b, stack_addr, false, ray_t_max);
297 brw_nir_rt_init_mem_hit_at_addr(b, stack_addr, true, ray_t_max);
298 brw_nir_rt_store_mem_ray_query_at_addr(b, ray_addr, &ray_defs);
299
300 update_trace_ctrl_level(b, ctrl_level_addr,
301 NULL, NULL,
302 nir_imm_int(b, GEN_RT_TRACE_RAY_INITAL),
303 nir_imm_int(b, BRW_RT_BVH_LEVEL_WORLD));
304 break;
305 }
306
307 case nir_intrinsic_rq_proceed: {
308 nir_ssa_def *not_done =
309 nir_inot(b, brw_nir_rt_query_done(b, stack_addr));
310 nir_ssa_def *not_done_then, *not_done_else;
311
312 nir_push_if(b, not_done);
313 {
314 nir_ssa_def *ctrl, *level;
315 update_trace_ctrl_level(b, ctrl_level_addr,
316 &ctrl, &level,
317 NULL,
318 NULL);
319
320 /* Mark the query as done because handing it over to the HW for
321 * processing. If the HW make any progress, it will write back some
322 * data and as a side effect, clear the "done" bit. If no progress is
323 * made, HW does not write anything back and we can use this bit to
324 * detect that.
325 */
326 brw_nir_rt_query_mark_done(b, stack_addr);
327
328 if (shadow_stack_addr)
329 fill_query(b, hw_stack_addr, shadow_stack_addr, ctrl);
330
331 nir_trace_ray_intel(b, state->rq_globals, level, ctrl, .synchronous = true);
332
333 struct brw_nir_rt_mem_hit_defs hit_in = {};
334 brw_nir_rt_load_mem_hit_from_addr(b, &hit_in, hw_stack_addr, false);
335
336 if (shadow_stack_addr)
337 spill_query(b, hw_stack_addr, shadow_stack_addr);
338
339 update_trace_ctrl_level(b, ctrl_level_addr,
340 NULL, NULL,
341 nir_imm_int(b, GEN_RT_TRACE_RAY_CONTINUE),
342 nir_imm_int(b, BRW_RT_BVH_LEVEL_OBJECT));
343
344 not_done_then = nir_inot(b, hit_in.done);
345 }
346 nir_push_else(b, NULL);
347 {
348 not_done_else = nir_imm_false(b);
349 }
350 nir_pop_if(b, NULL);
351 not_done = nir_if_phi(b, not_done_then, not_done_else);
352 nir_ssa_def_rewrite_uses(&intrin->dest.ssa, not_done);
353 break;
354 }
355
356 case nir_intrinsic_rq_confirm_intersection: {
357 brw_nir_memcpy_global(b,
358 brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, true), 16,
359 brw_nir_rt_mem_hit_addr_from_addr(b, stack_addr, false), 16,
360 BRW_RT_SIZEOF_HIT_INFO);
361 update_trace_ctrl_level(b, ctrl_level_addr,
362 NULL, NULL,
363 nir_imm_int(b, GEN_RT_TRACE_RAY_COMMIT),
364 nir_imm_int(b, BRW_RT_BVH_LEVEL_OBJECT));
365 break;
366 }
367
368 case nir_intrinsic_rq_generate_intersection: {
369 brw_nir_rt_generate_hit_addr(b, stack_addr, intrin->src[1].ssa);
370 update_trace_ctrl_level(b, ctrl_level_addr,
371 NULL, NULL,
372 nir_imm_int(b, GEN_RT_TRACE_RAY_COMMIT),
373 nir_imm_int(b, BRW_RT_BVH_LEVEL_OBJECT));
374 break;
375 }
376
377 case nir_intrinsic_rq_terminate: {
378 brw_nir_rt_query_mark_done(b, stack_addr);
379 break;
380 }
381
382 case nir_intrinsic_rq_load: {
383 const bool committed = nir_src_as_bool(intrin->src[1]);
384
385 struct brw_nir_rt_mem_ray_defs world_ray_in = {};
386 struct brw_nir_rt_mem_ray_defs object_ray_in = {};
387 struct brw_nir_rt_mem_hit_defs hit_in = {};
388 brw_nir_rt_load_mem_ray_from_addr(b, &world_ray_in, stack_addr,
389 BRW_RT_BVH_LEVEL_WORLD);
390 brw_nir_rt_load_mem_ray_from_addr(b, &object_ray_in, stack_addr,
391 BRW_RT_BVH_LEVEL_OBJECT);
392 brw_nir_rt_load_mem_hit_from_addr(b, &hit_in, stack_addr, committed);
393
394 nir_ssa_def *sysval = NULL;
395 switch (nir_intrinsic_base(intrin)) {
396 case nir_ray_query_value_intersection_type:
397 if (committed) {
398 /* Values we want to generate :
399 *
400 * RayQueryCommittedIntersectionNoneEXT = 0U <= hit_in.valid == false
401 * RayQueryCommittedIntersectionTriangleEXT = 1U <= hit_in.leaf_type == BRW_RT_BVH_NODE_TYPE_QUAD (4)
402 * RayQueryCommittedIntersectionGeneratedEXT = 2U <= hit_in.leaf_type == BRW_RT_BVH_NODE_TYPE_PROCEDURAL (3)
403 */
404 sysval =
405 nir_bcsel(b, nir_ieq(b, hit_in.leaf_type, nir_imm_int(b, 4)),
406 nir_imm_int(b, 1), nir_imm_int(b, 2));
407 sysval =
408 nir_bcsel(b, hit_in.valid,
409 sysval, nir_imm_int(b, 0));
410 } else {
411 /* 0 -> triangle, 1 -> AABB */
412 sysval =
413 nir_b2i32(b,
414 nir_ieq(b, hit_in.leaf_type,
415 nir_imm_int(b, BRW_RT_BVH_NODE_TYPE_PROCEDURAL)));
416 }
417 break;
418
419 case nir_ray_query_value_intersection_t:
420 sysval = hit_in.t;
421 break;
422
423 case nir_ray_query_value_intersection_instance_custom_index: {
424 struct brw_nir_rt_bvh_instance_leaf_defs leaf;
425 brw_nir_rt_load_bvh_instance_leaf(b, &leaf, hit_in.inst_leaf_ptr);
426 sysval = leaf.instance_id;
427 break;
428 }
429
430 case nir_ray_query_value_intersection_instance_id: {
431 struct brw_nir_rt_bvh_instance_leaf_defs leaf;
432 brw_nir_rt_load_bvh_instance_leaf(b, &leaf, hit_in.inst_leaf_ptr);
433 sysval = leaf.instance_index;
434 break;
435 }
436
437 case nir_ray_query_value_intersection_instance_sbt_index: {
438 struct brw_nir_rt_bvh_instance_leaf_defs leaf;
439 brw_nir_rt_load_bvh_instance_leaf(b, &leaf, hit_in.inst_leaf_ptr);
440 sysval = leaf.contribution_to_hit_group_index;
441 break;
442 }
443
444 case nir_ray_query_value_intersection_geometry_index: {
445 nir_ssa_def *geometry_index_dw =
446 nir_load_global(b, nir_iadd_imm(b, hit_in.prim_leaf_ptr, 4), 4,
447 1, 32);
448 sysval = nir_iand_imm(b, geometry_index_dw, BITFIELD_MASK(29));
449 break;
450 }
451
452 case nir_ray_query_value_intersection_primitive_index:
453 sysval = brw_nir_rt_load_primitive_id_from_hit(b, NULL /* is_procedural */, &hit_in);
454 break;
455
456 case nir_ray_query_value_intersection_barycentrics:
457 sysval = hit_in.tri_bary;
458 break;
459
460 case nir_ray_query_value_intersection_front_face:
461 sysval = hit_in.front_face;
462 break;
463
464 case nir_ray_query_value_intersection_object_ray_direction:
465 sysval = world_ray_in.dir;
466 break;
467
468 case nir_ray_query_value_intersection_object_ray_origin:
469 sysval = world_ray_in.orig;
470 break;
471
472 case nir_ray_query_value_intersection_object_to_world: {
473 struct brw_nir_rt_bvh_instance_leaf_defs leaf;
474 brw_nir_rt_load_bvh_instance_leaf(b, &leaf, hit_in.inst_leaf_ptr);
475 sysval = leaf.object_to_world[nir_intrinsic_column(intrin)];
476 break;
477 }
478
479 case nir_ray_query_value_intersection_world_to_object: {
480 struct brw_nir_rt_bvh_instance_leaf_defs leaf;
481 brw_nir_rt_load_bvh_instance_leaf(b, &leaf, hit_in.inst_leaf_ptr);
482 sysval = leaf.world_to_object[nir_intrinsic_column(intrin)];
483 break;
484 }
485
486 case nir_ray_query_value_intersection_candidate_aabb_opaque:
487 sysval = hit_in.front_face;
488 break;
489
490 case nir_ray_query_value_tmin:
491 sysval = world_ray_in.t_near;
492 break;
493
494 case nir_ray_query_value_flags:
495 sysval = nir_u2u32(b, world_ray_in.ray_flags);
496 break;
497
498 case nir_ray_query_value_world_ray_direction:
499 sysval = world_ray_in.dir;
500 break;
501
502 case nir_ray_query_value_world_ray_origin:
503 sysval = world_ray_in.orig;
504 break;
505
506 default:
507 unreachable("Invalid ray query");
508 }
509
510 assert(sysval);
511 nir_ssa_def_rewrite_uses(&intrin->dest.ssa, sysval);
512 break;
513 }
514
515 default:
516 unreachable("Invalid intrinsic");
517 }
518 }
519
520 static void
lower_ray_query_impl(nir_function_impl * impl,struct lowering_state * state)521 lower_ray_query_impl(nir_function_impl *impl, struct lowering_state *state)
522 {
523 nir_builder _b, *b = &_b;
524 nir_builder_init(&_b, impl);
525
526 b->cursor = nir_before_block(nir_start_block(b->impl));
527
528 state->rq_globals = nir_load_ray_query_global_intel(b);
529
530 brw_nir_rt_load_globals_addr(b, &state->globals, state->rq_globals);
531
532 nir_foreach_block_safe(block, impl) {
533 nir_foreach_instr_safe(instr, block) {
534 if (instr->type != nir_instr_type_intrinsic)
535 continue;
536
537 nir_intrinsic_instr *intrin = nir_instr_as_intrinsic(instr);
538 if (intrin->intrinsic != nir_intrinsic_rq_initialize &&
539 intrin->intrinsic != nir_intrinsic_rq_terminate &&
540 intrin->intrinsic != nir_intrinsic_rq_proceed &&
541 intrin->intrinsic != nir_intrinsic_rq_generate_intersection &&
542 intrin->intrinsic != nir_intrinsic_rq_confirm_intersection &&
543 intrin->intrinsic != nir_intrinsic_rq_load)
544 continue;
545
546 lower_ray_query_intrinsic(b, intrin, state);
547 }
548 }
549
550 nir_metadata_preserve(impl, nir_metadata_none);
551 }
552
553 bool
brw_nir_lower_ray_queries(nir_shader * shader,const struct intel_device_info * devinfo)554 brw_nir_lower_ray_queries(nir_shader *shader,
555 const struct intel_device_info *devinfo)
556 {
557 struct lowering_state state = {
558 .devinfo = devinfo,
559 .queries = _mesa_pointer_hash_table_create(NULL),
560 };
561
562 assert(exec_list_length(&shader->functions) == 1);
563
564 /* Find query variables */
565 nir_foreach_function(function, shader) {
566 if (!function->impl)
567 continue;
568
569 nir_foreach_block_safe(block, function->impl) {
570 nir_foreach_instr(instr, block)
571 maybe_create_brw_var(instr, &state);
572 }
573 }
574
575 if (_mesa_hash_table_num_entries(state.queries) > 0) {
576 nir_foreach_function(function, shader) {
577 if (function->impl)
578 lower_ray_query_impl(function->impl, &state);
579 }
580
581 nir_remove_dead_derefs(shader);
582 nir_remove_dead_variables(shader,
583 nir_var_shader_temp | nir_var_function_temp,
584 NULL);
585 }
586
587 ralloc_free(state.queries);
588
589 return true;
590 }
591