1 /*
2 * Copyright 2018 Collabora Ltd.
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * on the rights to use, copy, modify, merge, publish, distribute, sub
8 * license, and/or sell copies of the Software, and to permit persons to whom
9 * the Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHOR(S) AND/OR THEIR SUPPLIERS BE LIABLE FOR ANY CLAIM,
19 * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
20 * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
21 * USE OR OTHER DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "zink_context.h"
25
26 #include "zink_batch.h"
27 #include "zink_compiler.h"
28 #include "zink_fence.h"
29 #include "zink_format.h"
30 #include "zink_framebuffer.h"
31 #include "zink_helpers.h"
32 #include "zink_program.h"
33 #include "zink_pipeline.h"
34 #include "zink_query.h"
35 #include "zink_render_pass.h"
36 #include "zink_resource.h"
37 #include "zink_screen.h"
38 #include "zink_state.h"
39 #include "zink_surface.h"
40 #include "zink_inlines.h"
41
42 #include "util/u_blitter.h"
43 #include "util/u_debug.h"
44 #include "util/format_srgb.h"
45 #include "util/format/u_format.h"
46 #include "util/u_helpers.h"
47 #include "util/u_inlines.h"
48 #include "util/u_thread.h"
49 #include "util/u_cpu_detect.h"
50 #include "util/strndup.h"
51 #include "nir.h"
52
53 #include "util/u_memory.h"
54 #include "util/u_upload_mgr.h"
55
56 #define XXH_INLINE_ALL
57 #include "util/xxhash.h"
58
59 static void
calc_descriptor_hash_sampler_state(struct zink_sampler_state * sampler_state)60 calc_descriptor_hash_sampler_state(struct zink_sampler_state *sampler_state)
61 {
62 void *hash_data = &sampler_state->sampler;
63 size_t data_size = sizeof(VkSampler);
64 sampler_state->hash = XXH32(hash_data, data_size, 0);
65 }
66
67 void
debug_describe_zink_buffer_view(char * buf,const struct zink_buffer_view * ptr)68 debug_describe_zink_buffer_view(char *buf, const struct zink_buffer_view *ptr)
69 {
70 sprintf(buf, "zink_buffer_view");
71 }
72
73 ALWAYS_INLINE static void
check_resource_for_batch_ref(struct zink_context * ctx,struct zink_resource * res)74 check_resource_for_batch_ref(struct zink_context *ctx, struct zink_resource *res)
75 {
76 if (!zink_resource_has_binds(res))
77 zink_batch_reference_resource(&ctx->batch, res);
78 }
79
80 static void
zink_context_destroy(struct pipe_context * pctx)81 zink_context_destroy(struct pipe_context *pctx)
82 {
83 struct zink_context *ctx = zink_context(pctx);
84 struct zink_screen *screen = zink_screen(pctx->screen);
85
86 if (util_queue_is_initialized(&screen->flush_queue))
87 util_queue_finish(&screen->flush_queue);
88 if (screen->queue && !screen->device_lost && VKSCR(QueueWaitIdle)(screen->queue) != VK_SUCCESS)
89 debug_printf("vkQueueWaitIdle failed\n");
90
91 util_blitter_destroy(ctx->blitter);
92 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++)
93 pipe_surface_release(&ctx->base, &ctx->fb_state.cbufs[i]);
94 pipe_surface_release(&ctx->base, &ctx->fb_state.zsbuf);
95
96 pipe_resource_reference(&ctx->dummy_vertex_buffer, NULL);
97 pipe_resource_reference(&ctx->dummy_xfb_buffer, NULL);
98
99 for (unsigned i = 0; i < ARRAY_SIZE(ctx->dummy_surface); i++)
100 pipe_surface_release(&ctx->base, &ctx->dummy_surface[i]);
101 zink_buffer_view_reference(screen, &ctx->dummy_bufferview, NULL);
102
103 if (ctx->dd)
104 zink_descriptors_deinit_bindless(ctx);
105
106 simple_mtx_destroy(&ctx->batch_mtx);
107 if (ctx->batch.state) {
108 zink_clear_batch_state(ctx, ctx->batch.state);
109 zink_batch_state_destroy(screen, ctx->batch.state);
110 }
111 struct zink_batch_state *bs = ctx->batch_states;
112 while (bs) {
113 struct zink_batch_state *bs_next = bs->next;
114 zink_clear_batch_state(ctx, bs);
115 zink_batch_state_destroy(screen, bs);
116 bs = bs_next;
117 }
118 util_dynarray_foreach(&ctx->free_batch_states, struct zink_batch_state*, bs) {
119 zink_clear_batch_state(ctx, *bs);
120 zink_batch_state_destroy(screen, *bs);
121 }
122
123 for (unsigned i = 0; i < 2; i++) {
124 util_idalloc_fini(&ctx->di.bindless[i].tex_slots);
125 util_idalloc_fini(&ctx->di.bindless[i].img_slots);
126 free(ctx->di.bindless[i].buffer_infos);
127 free(ctx->di.bindless[i].img_infos);
128 util_dynarray_fini(&ctx->di.bindless[i].updates);
129 util_dynarray_fini(&ctx->di.bindless[i].resident);
130 }
131
132 if (screen->info.have_KHR_imageless_framebuffer) {
133 hash_table_foreach(&ctx->framebuffer_cache, he)
134 zink_destroy_framebuffer(screen, he->data);
135 } else if (ctx->framebuffer) {
136 simple_mtx_lock(&screen->framebuffer_mtx);
137 struct hash_entry *entry = _mesa_hash_table_search(&screen->framebuffer_cache, &ctx->framebuffer->state);
138 if (zink_framebuffer_reference(screen, &ctx->framebuffer, NULL))
139 _mesa_hash_table_remove(&screen->framebuffer_cache, entry);
140 simple_mtx_unlock(&screen->framebuffer_mtx);
141 }
142
143 hash_table_foreach(ctx->render_pass_cache, he)
144 zink_destroy_render_pass(screen, he->data);
145
146 u_upload_destroy(pctx->stream_uploader);
147 u_upload_destroy(pctx->const_uploader);
148 slab_destroy_child(&ctx->transfer_pool);
149 for (unsigned i = 0; i < ARRAY_SIZE(ctx->program_cache); i++)
150 _mesa_hash_table_clear(&ctx->program_cache[i], NULL);
151 _mesa_hash_table_clear(&ctx->compute_program_cache, NULL);
152 _mesa_hash_table_destroy(ctx->render_pass_cache, NULL);
153 slab_destroy_child(&ctx->transfer_pool_unsync);
154
155 if (ctx->dd)
156 screen->descriptors_deinit(ctx);
157
158 zink_descriptor_layouts_deinit(ctx);
159
160 p_atomic_dec(&screen->base.num_contexts);
161
162 ralloc_free(ctx);
163 }
164
165 static void
check_device_lost(struct zink_context * ctx)166 check_device_lost(struct zink_context *ctx)
167 {
168 if (!zink_screen(ctx->base.screen)->device_lost || ctx->is_device_lost)
169 return;
170 debug_printf("ZINK: device lost detected!\n");
171 if (ctx->reset.reset)
172 ctx->reset.reset(ctx->reset.data, PIPE_GUILTY_CONTEXT_RESET);
173 ctx->is_device_lost = true;
174 }
175
176 static enum pipe_reset_status
zink_get_device_reset_status(struct pipe_context * pctx)177 zink_get_device_reset_status(struct pipe_context *pctx)
178 {
179 struct zink_context *ctx = zink_context(pctx);
180
181 enum pipe_reset_status status = PIPE_NO_RESET;
182
183 if (ctx->is_device_lost) {
184 // Since we don't know what really happened to the hardware, just
185 // assume that we are in the wrong
186 status = PIPE_GUILTY_CONTEXT_RESET;
187
188 debug_printf("ZINK: device lost detected!\n");
189
190 if (ctx->reset.reset)
191 ctx->reset.reset(ctx->reset.data, status);
192 }
193
194 return status;
195 }
196
197 static void
zink_set_device_reset_callback(struct pipe_context * pctx,const struct pipe_device_reset_callback * cb)198 zink_set_device_reset_callback(struct pipe_context *pctx,
199 const struct pipe_device_reset_callback *cb)
200 {
201 struct zink_context *ctx = zink_context(pctx);
202
203 if (cb)
204 ctx->reset = *cb;
205 else
206 memset(&ctx->reset, 0, sizeof(ctx->reset));
207 }
208
209 static void
zink_set_context_param(struct pipe_context * pctx,enum pipe_context_param param,unsigned value)210 zink_set_context_param(struct pipe_context *pctx, enum pipe_context_param param,
211 unsigned value)
212 {
213 struct zink_context *ctx = zink_context(pctx);
214
215 switch (param) {
216 case PIPE_CONTEXT_PARAM_PIN_THREADS_TO_L3_CACHE:
217 util_set_thread_affinity(zink_screen(ctx->base.screen)->flush_queue.threads[0],
218 util_get_cpu_caps()->L3_affinity_mask[value],
219 NULL, util_get_cpu_caps()->num_cpu_mask_bits);
220 break;
221 default:
222 break;
223 }
224 }
225
226 static VkSamplerMipmapMode
sampler_mipmap_mode(enum pipe_tex_mipfilter filter)227 sampler_mipmap_mode(enum pipe_tex_mipfilter filter)
228 {
229 switch (filter) {
230 case PIPE_TEX_MIPFILTER_NEAREST: return VK_SAMPLER_MIPMAP_MODE_NEAREST;
231 case PIPE_TEX_MIPFILTER_LINEAR: return VK_SAMPLER_MIPMAP_MODE_LINEAR;
232 case PIPE_TEX_MIPFILTER_NONE:
233 unreachable("PIPE_TEX_MIPFILTER_NONE should be dealt with earlier");
234 }
235 unreachable("unexpected filter");
236 }
237
238 static VkSamplerAddressMode
sampler_address_mode(enum pipe_tex_wrap filter)239 sampler_address_mode(enum pipe_tex_wrap filter)
240 {
241 switch (filter) {
242 case PIPE_TEX_WRAP_REPEAT: return VK_SAMPLER_ADDRESS_MODE_REPEAT;
243 case PIPE_TEX_WRAP_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_EDGE;
244 case PIPE_TEX_WRAP_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_CLAMP_TO_BORDER;
245 case PIPE_TEX_WRAP_MIRROR_REPEAT: return VK_SAMPLER_ADDRESS_MODE_MIRRORED_REPEAT;
246 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_EDGE: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE;
247 case PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER: return VK_SAMPLER_ADDRESS_MODE_MIRROR_CLAMP_TO_EDGE; /* not technically correct, but kinda works */
248 default: break;
249 }
250 unreachable("unexpected wrap");
251 }
252
253 static VkCompareOp
compare_op(enum pipe_compare_func op)254 compare_op(enum pipe_compare_func op)
255 {
256 switch (op) {
257 case PIPE_FUNC_NEVER: return VK_COMPARE_OP_NEVER;
258 case PIPE_FUNC_LESS: return VK_COMPARE_OP_LESS;
259 case PIPE_FUNC_EQUAL: return VK_COMPARE_OP_EQUAL;
260 case PIPE_FUNC_LEQUAL: return VK_COMPARE_OP_LESS_OR_EQUAL;
261 case PIPE_FUNC_GREATER: return VK_COMPARE_OP_GREATER;
262 case PIPE_FUNC_NOTEQUAL: return VK_COMPARE_OP_NOT_EQUAL;
263 case PIPE_FUNC_GEQUAL: return VK_COMPARE_OP_GREATER_OR_EQUAL;
264 case PIPE_FUNC_ALWAYS: return VK_COMPARE_OP_ALWAYS;
265 }
266 unreachable("unexpected compare");
267 }
268
269 static inline bool
wrap_needs_border_color(unsigned wrap)270 wrap_needs_border_color(unsigned wrap)
271 {
272 return wrap == PIPE_TEX_WRAP_CLAMP || wrap == PIPE_TEX_WRAP_CLAMP_TO_BORDER ||
273 wrap == PIPE_TEX_WRAP_MIRROR_CLAMP || wrap == PIPE_TEX_WRAP_MIRROR_CLAMP_TO_BORDER;
274 }
275
276 static VkBorderColor
get_border_color(const union pipe_color_union * color,bool is_integer,bool need_custom)277 get_border_color(const union pipe_color_union *color, bool is_integer, bool need_custom)
278 {
279 if (is_integer) {
280 if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 0)
281 return VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
282 if (color->ui[0] == 0 && color->ui[1] == 0 && color->ui[2] == 0 && color->ui[3] == 1)
283 return VK_BORDER_COLOR_INT_OPAQUE_BLACK;
284 if (color->ui[0] == 1 && color->ui[1] == 1 && color->ui[2] == 1 && color->ui[3] == 1)
285 return VK_BORDER_COLOR_INT_OPAQUE_WHITE;
286 return need_custom ? VK_BORDER_COLOR_INT_CUSTOM_EXT : VK_BORDER_COLOR_INT_TRANSPARENT_BLACK;
287 }
288
289 if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 0)
290 return VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
291 if (color->f[0] == 0 && color->f[1] == 0 && color->f[2] == 0 && color->f[3] == 1)
292 return VK_BORDER_COLOR_FLOAT_OPAQUE_BLACK;
293 if (color->f[0] == 1 && color->f[1] == 1 && color->f[2] == 1 && color->f[3] == 1)
294 return VK_BORDER_COLOR_FLOAT_OPAQUE_WHITE;
295 return need_custom ? VK_BORDER_COLOR_FLOAT_CUSTOM_EXT : VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK;
296 }
297
298 static void *
zink_create_sampler_state(struct pipe_context * pctx,const struct pipe_sampler_state * state)299 zink_create_sampler_state(struct pipe_context *pctx,
300 const struct pipe_sampler_state *state)
301 {
302 struct zink_screen *screen = zink_screen(pctx->screen);
303 bool need_custom = false;
304
305 VkSamplerCreateInfo sci = {0};
306 VkSamplerCustomBorderColorCreateInfoEXT cbci = {0};
307 sci.sType = VK_STRUCTURE_TYPE_SAMPLER_CREATE_INFO;
308 sci.magFilter = zink_filter(state->mag_img_filter);
309 sci.minFilter = zink_filter(state->min_img_filter);
310
311 VkSamplerReductionModeCreateInfo rci;
312 rci.sType = VK_STRUCTURE_TYPE_SAMPLER_REDUCTION_MODE_CREATE_INFO;
313 rci.pNext = NULL;
314 switch (state->reduction_mode) {
315 case PIPE_TEX_REDUCTION_MIN:
316 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MIN;
317 break;
318 case PIPE_TEX_REDUCTION_MAX:
319 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_MAX;
320 break;
321 default:
322 rci.reductionMode = VK_SAMPLER_REDUCTION_MODE_WEIGHTED_AVERAGE;
323 break;
324 }
325 if (state->reduction_mode)
326 sci.pNext = &rci;
327
328 if (state->min_mip_filter != PIPE_TEX_MIPFILTER_NONE) {
329 sci.mipmapMode = sampler_mipmap_mode(state->min_mip_filter);
330 sci.minLod = state->min_lod;
331 sci.maxLod = state->max_lod;
332 } else {
333 sci.mipmapMode = VK_SAMPLER_MIPMAP_MODE_NEAREST;
334 sci.minLod = 0;
335 sci.maxLod = 0.25f;
336 }
337
338 sci.addressModeU = sampler_address_mode(state->wrap_s);
339 sci.addressModeV = sampler_address_mode(state->wrap_t);
340 sci.addressModeW = sampler_address_mode(state->wrap_r);
341 sci.mipLodBias = state->lod_bias;
342
343 need_custom |= wrap_needs_border_color(state->wrap_s);
344 need_custom |= wrap_needs_border_color(state->wrap_t);
345 need_custom |= wrap_needs_border_color(state->wrap_r);
346
347 if (state->compare_mode == PIPE_TEX_COMPARE_NONE)
348 sci.compareOp = VK_COMPARE_OP_NEVER;
349 else {
350 sci.compareOp = compare_op(state->compare_func);
351 sci.compareEnable = VK_TRUE;
352 }
353
354 bool is_integer = state->border_color_is_integer;
355
356 sci.borderColor = get_border_color(&state->border_color, is_integer, need_custom);
357 if (sci.borderColor > VK_BORDER_COLOR_INT_OPAQUE_WHITE && need_custom) {
358 if (screen->info.have_EXT_custom_border_color &&
359 screen->info.border_color_feats.customBorderColorWithoutFormat) {
360 cbci.sType = VK_STRUCTURE_TYPE_SAMPLER_CUSTOM_BORDER_COLOR_CREATE_INFO_EXT;
361 cbci.format = VK_FORMAT_UNDEFINED;
362 /* these are identical unions */
363 memcpy(&cbci.customBorderColor, &state->border_color, sizeof(union pipe_color_union));
364 cbci.pNext = sci.pNext;
365 sci.pNext = &cbci;
366 UNUSED uint32_t check = p_atomic_inc_return(&screen->cur_custom_border_color_samplers);
367 assert(check <= screen->info.border_color_props.maxCustomBorderColorSamplers);
368 } else
369 sci.borderColor = VK_BORDER_COLOR_FLOAT_TRANSPARENT_BLACK; // TODO with custom shader if we're super interested?
370 }
371
372 sci.unnormalizedCoordinates = !state->normalized_coords;
373
374 if (state->max_anisotropy > 1) {
375 sci.maxAnisotropy = state->max_anisotropy;
376 sci.anisotropyEnable = VK_TRUE;
377 }
378
379 struct zink_sampler_state *sampler = CALLOC_STRUCT(zink_sampler_state);
380 if (!sampler)
381 return NULL;
382
383 if (VKSCR(CreateSampler)(screen->dev, &sci, NULL, &sampler->sampler) != VK_SUCCESS) {
384 FREE(sampler);
385 return NULL;
386 }
387 util_dynarray_init(&sampler->desc_set_refs.refs, NULL);
388 calc_descriptor_hash_sampler_state(sampler);
389 sampler->custom_border_color = need_custom;
390
391 return sampler;
392 }
393
394 ALWAYS_INLINE static VkImageLayout
get_layout_for_binding(struct zink_resource * res,enum zink_descriptor_type type,bool is_compute)395 get_layout_for_binding(struct zink_resource *res, enum zink_descriptor_type type, bool is_compute)
396 {
397 if (res->obj->is_buffer)
398 return 0;
399 switch (type) {
400 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
401 return res->image_bind_count[is_compute] ?
402 VK_IMAGE_LAYOUT_GENERAL :
403 res->aspect & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT) ?
404 //Vulkan-Docs#1490
405 //(res->aspect == VK_IMAGE_ASPECT_DEPTH_BIT ? VK_IMAGE_LAYOUT_DEPTH_READ_ONLY_OPTIMAL :
406 //res->aspect == VK_IMAGE_ASPECT_STENCIL_BIT ? VK_IMAGE_LAYOUT_STENCIL_READ_ONLY_OPTIMAL :
407 (res->aspect == VK_IMAGE_ASPECT_DEPTH_BIT ? VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL :
408 res->aspect == VK_IMAGE_ASPECT_STENCIL_BIT ? VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL :
409 VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL) :
410 VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
411 case ZINK_DESCRIPTOR_TYPE_IMAGE:
412 return VK_IMAGE_LAYOUT_GENERAL;
413 default:
414 break;
415 }
416 return 0;
417 }
418
419 ALWAYS_INLINE static struct zink_surface *
get_imageview_for_binding(struct zink_context * ctx,enum pipe_shader_type stage,enum zink_descriptor_type type,unsigned idx)420 get_imageview_for_binding(struct zink_context *ctx, enum pipe_shader_type stage, enum zink_descriptor_type type, unsigned idx)
421 {
422 switch (type) {
423 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
424 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
425 return sampler_view->base.texture ? sampler_view->image_view : NULL;
426 }
427 case ZINK_DESCRIPTOR_TYPE_IMAGE: {
428 struct zink_image_view *image_view = &ctx->image_views[stage][idx];
429 return image_view->base.resource ? image_view->surface : NULL;
430 }
431 default:
432 break;
433 }
434 unreachable("ACK");
435 return VK_NULL_HANDLE;
436 }
437
438 ALWAYS_INLINE static struct zink_buffer_view *
get_bufferview_for_binding(struct zink_context * ctx,enum pipe_shader_type stage,enum zink_descriptor_type type,unsigned idx)439 get_bufferview_for_binding(struct zink_context *ctx, enum pipe_shader_type stage, enum zink_descriptor_type type, unsigned idx)
440 {
441 switch (type) {
442 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW: {
443 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[stage][idx]);
444 return sampler_view->base.texture ? sampler_view->buffer_view : NULL;
445 }
446 case ZINK_DESCRIPTOR_TYPE_IMAGE: {
447 struct zink_image_view *image_view = &ctx->image_views[stage][idx];
448 return image_view->base.resource ? image_view->buffer_view : NULL;
449 }
450 default:
451 break;
452 }
453 unreachable("ACK");
454 return VK_NULL_HANDLE;
455 }
456
457 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_ubo(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot,struct zink_resource * res)458 update_descriptor_state_ubo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res)
459 {
460 struct zink_screen *screen = zink_screen(ctx->base.screen);
461 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
462 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_UBO;
463 ctx->di.descriptor_res[type][shader][slot] = res;
464 ctx->di.ubos[shader][slot].offset = ctx->ubos[shader][slot].buffer_offset;
465 if (res) {
466 ctx->di.ubos[shader][slot].buffer = res->obj->buffer;
467 ctx->di.ubos[shader][slot].range = ctx->ubos[shader][slot].buffer_size;
468 assert(ctx->di.ubos[shader][slot].range <= screen->info.props.limits.maxUniformBufferRange);
469 } else {
470 VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
471 ctx->di.ubos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
472 ctx->di.ubos[shader][slot].range = VK_WHOLE_SIZE;
473 }
474 if (!slot) {
475 if (res)
476 ctx->di.push_valid |= BITFIELD64_BIT(shader);
477 else
478 ctx->di.push_valid &= ~BITFIELD64_BIT(shader);
479 }
480 return res;
481 }
482
483 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_ssbo(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot,struct zink_resource * res)484 update_descriptor_state_ssbo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res)
485 {
486 struct zink_screen *screen = zink_screen(ctx->base.screen);
487 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
488 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SSBO;
489 ctx->di.descriptor_res[type][shader][slot] = res;
490 ctx->di.ssbos[shader][slot].offset = ctx->ssbos[shader][slot].buffer_offset;
491 if (res) {
492 ctx->di.ssbos[shader][slot].buffer = res->obj->buffer;
493 ctx->di.ssbos[shader][slot].range = ctx->ssbos[shader][slot].buffer_size;
494 } else {
495 VkBuffer null_buffer = zink_resource(ctx->dummy_vertex_buffer)->obj->buffer;
496 ctx->di.ssbos[shader][slot].buffer = have_null_descriptors ? VK_NULL_HANDLE : null_buffer;
497 ctx->di.ssbos[shader][slot].range = VK_WHOLE_SIZE;
498 }
499 return res;
500 }
501
502 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_sampler(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot,struct zink_resource * res)503 update_descriptor_state_sampler(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res)
504 {
505 struct zink_screen *screen = zink_screen(ctx->base.screen);
506 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
507 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW;
508 ctx->di.descriptor_res[type][shader][slot] = res;
509 if (res) {
510 if (res->obj->is_buffer) {
511 struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
512 ctx->di.tbos[shader][slot] = bv->buffer_view;
513 ctx->di.sampler_surfaces[shader][slot].bufferview = bv;
514 ctx->di.sampler_surfaces[shader][slot].is_buffer = true;
515 } else {
516 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
517 ctx->di.textures[shader][slot].imageLayout = get_layout_for_binding(res, type, shader == PIPE_SHADER_COMPUTE);
518 ctx->di.textures[shader][slot].imageView = surface->image_view;
519 ctx->di.sampler_surfaces[shader][slot].surface = surface;
520 ctx->di.sampler_surfaces[shader][slot].is_buffer = false;
521 }
522 } else {
523 if (likely(have_null_descriptors)) {
524 ctx->di.textures[shader][slot].imageView = VK_NULL_HANDLE;
525 ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
526 ctx->di.tbos[shader][slot] = VK_NULL_HANDLE;
527 } else {
528 struct zink_surface *null_surface = zink_csurface(ctx->dummy_surface[0]);
529 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
530 ctx->di.textures[shader][slot].imageView = null_surface->image_view;
531 ctx->di.textures[shader][slot].imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
532 ctx->di.tbos[shader][slot] = null_bufferview->buffer_view;
533 }
534 memset(&ctx->di.sampler_surfaces[shader][slot], 0, sizeof(ctx->di.sampler_surfaces[shader][slot]));
535 }
536 return res;
537 }
538
539 ALWAYS_INLINE static struct zink_resource *
update_descriptor_state_image(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot,struct zink_resource * res)540 update_descriptor_state_image(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot, struct zink_resource *res)
541 {
542 struct zink_screen *screen = zink_screen(ctx->base.screen);
543 bool have_null_descriptors = screen->info.rb2_feats.nullDescriptor;
544 const enum zink_descriptor_type type = ZINK_DESCRIPTOR_TYPE_IMAGE;
545 ctx->di.descriptor_res[type][shader][slot] = res;
546 if (res) {
547 if (res->obj->is_buffer) {
548 struct zink_buffer_view *bv = get_bufferview_for_binding(ctx, shader, type, slot);
549 ctx->di.texel_images[shader][slot] = bv->buffer_view;
550 ctx->di.image_surfaces[shader][slot].bufferview = bv;
551 ctx->di.image_surfaces[shader][slot].is_buffer = true;
552 } else {
553 struct zink_surface *surface = get_imageview_for_binding(ctx, shader, type, slot);
554 ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
555 ctx->di.images[shader][slot].imageView = surface->image_view;
556 ctx->di.image_surfaces[shader][slot].surface = surface;
557 ctx->di.image_surfaces[shader][slot].is_buffer = false;
558 }
559 } else {
560 if (likely(have_null_descriptors)) {
561 memset(&ctx->di.images[shader][slot], 0, sizeof(ctx->di.images[shader][slot]));
562 ctx->di.texel_images[shader][slot] = VK_NULL_HANDLE;
563 } else {
564 struct zink_surface *null_surface = zink_csurface(ctx->dummy_surface[0]);
565 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
566 ctx->di.images[shader][slot].imageView = null_surface->image_view;
567 ctx->di.images[shader][slot].imageLayout = VK_IMAGE_LAYOUT_GENERAL;
568 ctx->di.texel_images[shader][slot] = null_bufferview->buffer_view;
569 }
570 memset(&ctx->di.image_surfaces[shader][slot], 0, sizeof(ctx->di.image_surfaces[shader][slot]));
571 }
572 return res;
573 }
574
575 static void
zink_bind_sampler_states(struct pipe_context * pctx,enum pipe_shader_type shader,unsigned start_slot,unsigned num_samplers,void ** samplers)576 zink_bind_sampler_states(struct pipe_context *pctx,
577 enum pipe_shader_type shader,
578 unsigned start_slot,
579 unsigned num_samplers,
580 void **samplers)
581 {
582 struct zink_context *ctx = zink_context(pctx);
583 for (unsigned i = 0; i < num_samplers; ++i) {
584 struct zink_sampler_state *state = samplers[i];
585 if (ctx->sampler_states[shader][start_slot + i] != state)
586 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, 1);
587 ctx->sampler_states[shader][start_slot + i] = state;
588 ctx->di.textures[shader][start_slot + i].sampler = state ? state->sampler : VK_NULL_HANDLE;
589 if (state)
590 zink_batch_usage_set(&state->batch_uses, ctx->batch.state);
591 }
592 ctx->di.num_samplers[shader] = start_slot + num_samplers;
593 }
594
595 static void
zink_delete_sampler_state(struct pipe_context * pctx,void * sampler_state)596 zink_delete_sampler_state(struct pipe_context *pctx,
597 void *sampler_state)
598 {
599 struct zink_sampler_state *sampler = sampler_state;
600 struct zink_batch *batch = &zink_context(pctx)->batch;
601 zink_descriptor_set_refs_clear(&sampler->desc_set_refs, sampler_state);
602 /* may be called if context_create fails */
603 if (batch->state)
604 util_dynarray_append(&batch->state->zombie_samplers, VkSampler,
605 sampler->sampler);
606 if (sampler->custom_border_color)
607 p_atomic_dec(&zink_screen(pctx->screen)->cur_custom_border_color_samplers);
608 FREE(sampler);
609 }
610
611 static VkImageAspectFlags
sampler_aspect_from_format(enum pipe_format fmt)612 sampler_aspect_from_format(enum pipe_format fmt)
613 {
614 if (util_format_is_depth_or_stencil(fmt)) {
615 const struct util_format_description *desc = util_format_description(fmt);
616 if (util_format_has_depth(desc))
617 return VK_IMAGE_ASPECT_DEPTH_BIT;
618 assert(util_format_has_stencil(desc));
619 return VK_IMAGE_ASPECT_STENCIL_BIT;
620 } else
621 return VK_IMAGE_ASPECT_COLOR_BIT;
622 }
623
624 static uint32_t
hash_bufferview(void * bvci)625 hash_bufferview(void *bvci)
626 {
627 size_t offset = offsetof(VkBufferViewCreateInfo, flags);
628 return _mesa_hash_data((char*)bvci + offset, sizeof(VkBufferViewCreateInfo) - offset);
629 }
630
631 static VkBufferViewCreateInfo
create_bvci(struct zink_context * ctx,struct zink_resource * res,enum pipe_format format,uint32_t offset,uint32_t range)632 create_bvci(struct zink_context *ctx, struct zink_resource *res, enum pipe_format format, uint32_t offset, uint32_t range)
633 {
634 struct zink_screen *screen = zink_screen(ctx->base.screen);
635 VkBufferViewCreateInfo bvci;
636 // Zero whole struct (including alignment holes), so hash_bufferview
637 // does not access potentially uninitialized data.
638 memset(&bvci, 0, sizeof(bvci));
639 bvci.sType = VK_STRUCTURE_TYPE_BUFFER_VIEW_CREATE_INFO;
640 bvci.pNext = NULL;
641 bvci.buffer = res->obj->buffer;
642 bvci.format = zink_get_format(screen, format);
643 assert(bvci.format);
644 bvci.offset = offset;
645 bvci.range = !offset && range == res->base.b.width0 ? VK_WHOLE_SIZE : range;
646 uint32_t clamp = util_format_get_blocksize(format) * screen->info.props.limits.maxTexelBufferElements;
647 if (bvci.range == VK_WHOLE_SIZE && res->base.b.width0 > clamp)
648 bvci.range = clamp;
649 bvci.flags = 0;
650 return bvci;
651 }
652
653 static struct zink_buffer_view *
get_buffer_view(struct zink_context * ctx,struct zink_resource * res,VkBufferViewCreateInfo * bvci)654 get_buffer_view(struct zink_context *ctx, struct zink_resource *res, VkBufferViewCreateInfo *bvci)
655 {
656 struct zink_screen *screen = zink_screen(ctx->base.screen);
657 struct zink_buffer_view *buffer_view = NULL;
658
659 uint32_t hash = hash_bufferview(bvci);
660 simple_mtx_lock(&res->bufferview_mtx);
661 struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, hash, bvci);
662 if (he) {
663 buffer_view = he->data;
664 p_atomic_inc(&buffer_view->reference.count);
665 } else {
666 VkBufferView view;
667 if (VKSCR(CreateBufferView)(screen->dev, bvci, NULL, &view) != VK_SUCCESS)
668 goto out;
669 buffer_view = CALLOC_STRUCT(zink_buffer_view);
670 if (!buffer_view) {
671 VKSCR(DestroyBufferView)(screen->dev, view, NULL);
672 goto out;
673 }
674 pipe_reference_init(&buffer_view->reference, 1);
675 pipe_resource_reference(&buffer_view->pres, &res->base.b);
676 util_dynarray_init(&buffer_view->desc_set_refs.refs, NULL);
677 buffer_view->bvci = *bvci;
678 buffer_view->buffer_view = view;
679 buffer_view->hash = hash;
680 _mesa_hash_table_insert_pre_hashed(&res->bufferview_cache, hash, &buffer_view->bvci, buffer_view);
681 }
682 out:
683 simple_mtx_unlock(&res->bufferview_mtx);
684 return buffer_view;
685 }
686
687 enum pipe_swizzle
zink_clamp_void_swizzle(const struct util_format_description * desc,enum pipe_swizzle swizzle)688 zink_clamp_void_swizzle(const struct util_format_description *desc, enum pipe_swizzle swizzle)
689 {
690 switch (swizzle) {
691 case PIPE_SWIZZLE_X:
692 case PIPE_SWIZZLE_Y:
693 case PIPE_SWIZZLE_Z:
694 case PIPE_SWIZZLE_W:
695 return desc->channel[swizzle].type == UTIL_FORMAT_TYPE_VOID ? PIPE_SWIZZLE_1 : swizzle;
696 default:
697 break;
698 }
699 return swizzle;
700 }
701
702 ALWAYS_INLINE static enum pipe_swizzle
clamp_zs_swizzle(enum pipe_swizzle swizzle)703 clamp_zs_swizzle(enum pipe_swizzle swizzle)
704 {
705 switch (swizzle) {
706 case PIPE_SWIZZLE_X:
707 case PIPE_SWIZZLE_Y:
708 case PIPE_SWIZZLE_Z:
709 case PIPE_SWIZZLE_W:
710 return PIPE_SWIZZLE_X;
711 default:
712 break;
713 }
714 return swizzle;
715 }
716
717 static struct pipe_sampler_view *
zink_create_sampler_view(struct pipe_context * pctx,struct pipe_resource * pres,const struct pipe_sampler_view * state)718 zink_create_sampler_view(struct pipe_context *pctx, struct pipe_resource *pres,
719 const struct pipe_sampler_view *state)
720 {
721 struct zink_screen *screen = zink_screen(pctx->screen);
722 struct zink_resource *res = zink_resource(pres);
723 struct zink_sampler_view *sampler_view = CALLOC_STRUCT(zink_sampler_view);
724 bool err;
725
726 sampler_view->base = *state;
727 sampler_view->base.texture = NULL;
728 pipe_resource_reference(&sampler_view->base.texture, pres);
729 sampler_view->base.reference.count = 1;
730 sampler_view->base.context = pctx;
731
732 if (state->target != PIPE_BUFFER) {
733 VkImageViewCreateInfo ivci;
734
735 struct pipe_surface templ = {0};
736 templ.u.tex.level = state->u.tex.first_level;
737 templ.format = state->format;
738 if (state->target != PIPE_TEXTURE_3D) {
739 templ.u.tex.first_layer = state->u.tex.first_layer;
740 templ.u.tex.last_layer = state->u.tex.last_layer;
741 }
742
743 ivci = create_ivci(screen, res, &templ, state->target);
744 ivci.subresourceRange.levelCount = state->u.tex.last_level - state->u.tex.first_level + 1;
745 ivci.subresourceRange.aspectMask = sampler_aspect_from_format(state->format);
746 /* samplers for stencil aspects of packed formats need to always use stencil swizzle */
747 if (ivci.subresourceRange.aspectMask & (VK_IMAGE_ASPECT_DEPTH_BIT | VK_IMAGE_ASPECT_STENCIL_BIT)) {
748 ivci.components.r = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_r));
749 ivci.components.g = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_g));
750 ivci.components.b = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_b));
751 ivci.components.a = zink_component_mapping(clamp_zs_swizzle(sampler_view->base.swizzle_a));
752 } else {
753 /* if we have e.g., R8G8B8X8, then we have to ignore alpha since we're just emulating
754 * these formats
755 */
756 if (zink_format_is_voidable_rgba_variant(state->format)) {
757 const struct util_format_description *desc = util_format_description(state->format);
758 sampler_view->base.swizzle_r = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_r);
759 sampler_view->base.swizzle_g = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_g);
760 sampler_view->base.swizzle_b = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_b);
761 sampler_view->base.swizzle_a = zink_clamp_void_swizzle(desc, sampler_view->base.swizzle_a);
762 }
763 ivci.components.r = zink_component_mapping(sampler_view->base.swizzle_r);
764 ivci.components.g = zink_component_mapping(sampler_view->base.swizzle_g);
765 ivci.components.b = zink_component_mapping(sampler_view->base.swizzle_b);
766 ivci.components.a = zink_component_mapping(sampler_view->base.swizzle_a);
767 }
768 assert(ivci.format);
769
770 sampler_view->image_view = (struct zink_surface*)zink_get_surface(zink_context(pctx), pres, &templ, &ivci);
771 err = !sampler_view->image_view;
772 } else {
773 VkBufferViewCreateInfo bvci = create_bvci(zink_context(pctx), res, state->format, state->u.buf.offset, state->u.buf.size);
774 sampler_view->buffer_view = get_buffer_view(zink_context(pctx), res, &bvci);
775 err = !sampler_view->buffer_view;
776 }
777 if (err) {
778 FREE(sampler_view);
779 return NULL;
780 }
781 return &sampler_view->base;
782 }
783
784 void
zink_destroy_buffer_view(struct zink_screen * screen,struct zink_buffer_view * buffer_view)785 zink_destroy_buffer_view(struct zink_screen *screen, struct zink_buffer_view *buffer_view)
786 {
787 struct zink_resource *res = zink_resource(buffer_view->pres);
788 simple_mtx_lock(&res->bufferview_mtx);
789 if (buffer_view->reference.count) {
790 /* got a cache hit during deletion */
791 simple_mtx_unlock(&res->bufferview_mtx);
792 return;
793 }
794 struct hash_entry *he = _mesa_hash_table_search_pre_hashed(&res->bufferview_cache, buffer_view->hash, &buffer_view->bvci);
795 assert(he);
796 _mesa_hash_table_remove(&res->bufferview_cache, he);
797 simple_mtx_unlock(&res->bufferview_mtx);
798 pipe_resource_reference(&buffer_view->pres, NULL);
799 VKSCR(DestroyBufferView)(screen->dev, buffer_view->buffer_view, NULL);
800 zink_descriptor_set_refs_clear(&buffer_view->desc_set_refs, buffer_view);
801 FREE(buffer_view);
802 }
803
804 static void
zink_sampler_view_destroy(struct pipe_context * pctx,struct pipe_sampler_view * pview)805 zink_sampler_view_destroy(struct pipe_context *pctx,
806 struct pipe_sampler_view *pview)
807 {
808 struct zink_sampler_view *view = zink_sampler_view(pview);
809 if (pview->texture->target == PIPE_BUFFER)
810 zink_buffer_view_reference(zink_screen(pctx->screen), &view->buffer_view, NULL);
811 else {
812 zink_surface_reference(zink_screen(pctx->screen), &view->image_view, NULL);
813 }
814 pipe_resource_reference(&pview->texture, NULL);
815 FREE(view);
816 }
817
818 static void
zink_get_sample_position(struct pipe_context * ctx,unsigned sample_count,unsigned sample_index,float * out_value)819 zink_get_sample_position(struct pipe_context *ctx,
820 unsigned sample_count,
821 unsigned sample_index,
822 float *out_value)
823 {
824 /* TODO: handle this I guess */
825 assert(zink_screen(ctx->screen)->info.props.limits.standardSampleLocations);
826 /* from 26.4. Multisampling */
827 switch (sample_count) {
828 case 0:
829 case 1: {
830 float pos[][2] = { {0.5,0.5}, };
831 out_value[0] = pos[sample_index][0];
832 out_value[1] = pos[sample_index][1];
833 break;
834 }
835 case 2: {
836 float pos[][2] = { {0.75,0.75},
837 {0.25,0.25}, };
838 out_value[0] = pos[sample_index][0];
839 out_value[1] = pos[sample_index][1];
840 break;
841 }
842 case 4: {
843 float pos[][2] = { {0.375, 0.125},
844 {0.875, 0.375},
845 {0.125, 0.625},
846 {0.625, 0.875}, };
847 out_value[0] = pos[sample_index][0];
848 out_value[1] = pos[sample_index][1];
849 break;
850 }
851 case 8: {
852 float pos[][2] = { {0.5625, 0.3125},
853 {0.4375, 0.6875},
854 {0.8125, 0.5625},
855 {0.3125, 0.1875},
856 {0.1875, 0.8125},
857 {0.0625, 0.4375},
858 {0.6875, 0.9375},
859 {0.9375, 0.0625}, };
860 out_value[0] = pos[sample_index][0];
861 out_value[1] = pos[sample_index][1];
862 break;
863 }
864 case 16: {
865 float pos[][2] = { {0.5625, 0.5625},
866 {0.4375, 0.3125},
867 {0.3125, 0.625},
868 {0.75, 0.4375},
869 {0.1875, 0.375},
870 {0.625, 0.8125},
871 {0.8125, 0.6875},
872 {0.6875, 0.1875},
873 {0.375, 0.875},
874 {0.5, 0.0625},
875 {0.25, 0.125},
876 {0.125, 0.75},
877 {0.0, 0.5},
878 {0.9375, 0.25},
879 {0.875, 0.9375},
880 {0.0625, 0.0}, };
881 out_value[0] = pos[sample_index][0];
882 out_value[1] = pos[sample_index][1];
883 break;
884 }
885 default:
886 unreachable("unhandled sample count!");
887 }
888 }
889
890 static void
zink_set_polygon_stipple(struct pipe_context * pctx,const struct pipe_poly_stipple * ps)891 zink_set_polygon_stipple(struct pipe_context *pctx,
892 const struct pipe_poly_stipple *ps)
893 {
894 }
895
896 ALWAYS_INLINE static void
update_res_bind_count(struct zink_context * ctx,struct zink_resource * res,bool is_compute,bool decrement)897 update_res_bind_count(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool decrement)
898 {
899 if (decrement) {
900 assert(res->bind_count[is_compute]);
901 if (!--res->bind_count[is_compute])
902 _mesa_set_remove_key(ctx->need_barriers[is_compute], res);
903 check_resource_for_batch_ref(ctx, res);
904 } else
905 res->bind_count[is_compute]++;
906 }
907
908 ALWAYS_INLINE static void
update_existing_vbo(struct zink_context * ctx,unsigned slot)909 update_existing_vbo(struct zink_context *ctx, unsigned slot)
910 {
911 if (!ctx->vertex_buffers[slot].buffer.resource)
912 return;
913 struct zink_resource *res = zink_resource(ctx->vertex_buffers[slot].buffer.resource);
914 res->vbo_bind_mask &= ~BITFIELD_BIT(slot);
915 update_res_bind_count(ctx, res, false, true);
916 }
917
918 static void
zink_set_vertex_buffers(struct pipe_context * pctx,unsigned start_slot,unsigned num_buffers,unsigned unbind_num_trailing_slots,bool take_ownership,const struct pipe_vertex_buffer * buffers)919 zink_set_vertex_buffers(struct pipe_context *pctx,
920 unsigned start_slot,
921 unsigned num_buffers,
922 unsigned unbind_num_trailing_slots,
923 bool take_ownership,
924 const struct pipe_vertex_buffer *buffers)
925 {
926 struct zink_context *ctx = zink_context(pctx);
927 const bool need_state_change = !zink_screen(pctx->screen)->info.have_EXT_extended_dynamic_state &&
928 !zink_screen(pctx->screen)->info.have_EXT_vertex_input_dynamic_state;
929 uint32_t enabled_buffers = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask;
930 enabled_buffers |= u_bit_consecutive(start_slot, num_buffers);
931 enabled_buffers &= ~u_bit_consecutive(start_slot + num_buffers, unbind_num_trailing_slots);
932
933 if (buffers) {
934 if (need_state_change)
935 ctx->vertex_state_changed = true;
936 for (unsigned i = 0; i < num_buffers; ++i) {
937 const struct pipe_vertex_buffer *vb = buffers + i;
938 struct pipe_vertex_buffer *ctx_vb = &ctx->vertex_buffers[start_slot + i];
939 update_existing_vbo(ctx, start_slot + i);
940 if (!take_ownership)
941 pipe_resource_reference(&ctx_vb->buffer.resource, vb->buffer.resource);
942 else {
943 pipe_resource_reference(&ctx_vb->buffer.resource, NULL);
944 ctx_vb->buffer.resource = vb->buffer.resource;
945 }
946 if (vb->buffer.resource) {
947 struct zink_resource *res = zink_resource(vb->buffer.resource);
948 res->vbo_bind_mask |= BITFIELD_BIT(start_slot + i);
949 update_res_bind_count(ctx, res, false, false);
950 ctx_vb->stride = vb->stride;
951 ctx_vb->buffer_offset = vb->buffer_offset;
952 /* always barrier before possible rebind */
953 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT,
954 VK_PIPELINE_STAGE_VERTEX_INPUT_BIT);
955 } else {
956 enabled_buffers &= ~BITFIELD_BIT(start_slot + i);
957 }
958 }
959 } else {
960 if (need_state_change)
961 ctx->vertex_state_changed = true;
962 for (unsigned i = 0; i < num_buffers; ++i) {
963 update_existing_vbo(ctx, start_slot + i);
964 pipe_resource_reference(&ctx->vertex_buffers[start_slot + i].buffer.resource, NULL);
965 }
966 }
967 for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
968 update_existing_vbo(ctx, start_slot + i);
969 pipe_resource_reference(&ctx->vertex_buffers[start_slot + i].buffer.resource, NULL);
970 }
971 ctx->gfx_pipeline_state.vertex_buffers_enabled_mask = enabled_buffers;
972 ctx->vertex_buffers_dirty = num_buffers > 0;
973 #ifndef NDEBUG
974 u_foreach_bit(b, enabled_buffers)
975 assert(ctx->vertex_buffers[b].buffer.resource);
976 #endif
977 }
978
979 static void
zink_set_viewport_states(struct pipe_context * pctx,unsigned start_slot,unsigned num_viewports,const struct pipe_viewport_state * state)980 zink_set_viewport_states(struct pipe_context *pctx,
981 unsigned start_slot,
982 unsigned num_viewports,
983 const struct pipe_viewport_state *state)
984 {
985 struct zink_context *ctx = zink_context(pctx);
986
987 for (unsigned i = 0; i < num_viewports; ++i)
988 ctx->vp_state.viewport_states[start_slot + i] = state[i];
989 ctx->vp_state.num_viewports = start_slot + num_viewports;
990
991 if (!zink_screen(pctx->screen)->info.have_EXT_extended_dynamic_state) {
992 if (ctx->gfx_pipeline_state.dyn_state1.num_viewports != ctx->vp_state.num_viewports)
993 ctx->gfx_pipeline_state.dirty = true;
994 ctx->gfx_pipeline_state.dyn_state1.num_viewports = ctx->vp_state.num_viewports;
995 }
996 ctx->vp_state_changed = true;
997 }
998
999 static void
zink_set_scissor_states(struct pipe_context * pctx,unsigned start_slot,unsigned num_scissors,const struct pipe_scissor_state * states)1000 zink_set_scissor_states(struct pipe_context *pctx,
1001 unsigned start_slot, unsigned num_scissors,
1002 const struct pipe_scissor_state *states)
1003 {
1004 struct zink_context *ctx = zink_context(pctx);
1005
1006 for (unsigned i = 0; i < num_scissors; i++)
1007 ctx->vp_state.scissor_states[start_slot + i] = states[i];
1008 ctx->scissor_changed = true;
1009 }
1010
1011 static void
zink_set_inlinable_constants(struct pipe_context * pctx,enum pipe_shader_type shader,uint num_values,uint32_t * values)1012 zink_set_inlinable_constants(struct pipe_context *pctx,
1013 enum pipe_shader_type shader,
1014 uint num_values, uint32_t *values)
1015 {
1016 struct zink_context *ctx = (struct zink_context *)pctx;
1017 const uint32_t bit = BITFIELD_BIT(shader);
1018 uint32_t *inlinable_uniforms;
1019 struct zink_shader_key *key = NULL;
1020
1021 if (shader == PIPE_SHADER_COMPUTE) {
1022 inlinable_uniforms = ctx->compute_inlinable_uniforms;
1023 } else {
1024 key = &ctx->gfx_pipeline_state.shader_keys.key[shader];
1025 inlinable_uniforms = key->base.inlined_uniform_values;
1026 }
1027 if (!(ctx->inlinable_uniforms_valid_mask & bit) ||
1028 memcmp(inlinable_uniforms, values, num_values * 4)) {
1029 memcpy(inlinable_uniforms, values, num_values * 4);
1030 ctx->dirty_shader_stages |= bit;
1031 ctx->inlinable_uniforms_valid_mask |= bit;
1032 if (key)
1033 key->inline_uniforms = true;
1034 }
1035 }
1036
1037 ALWAYS_INLINE static void
unbind_ubo(struct zink_context * ctx,struct zink_resource * res,enum pipe_shader_type pstage,unsigned slot)1038 unbind_ubo(struct zink_context *ctx, struct zink_resource *res, enum pipe_shader_type pstage, unsigned slot)
1039 {
1040 if (!res)
1041 return;
1042 res->ubo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1043 res->ubo_bind_count[pstage == PIPE_SHADER_COMPUTE]--;
1044 update_res_bind_count(ctx, res, pstage == PIPE_SHADER_COMPUTE, true);
1045 }
1046
1047 static void
invalidate_inlined_uniforms(struct zink_context * ctx,enum pipe_shader_type pstage)1048 invalidate_inlined_uniforms(struct zink_context *ctx, enum pipe_shader_type pstage)
1049 {
1050 unsigned bit = BITFIELD_BIT(pstage);
1051 if (!(ctx->inlinable_uniforms_valid_mask & bit))
1052 return;
1053 ctx->inlinable_uniforms_valid_mask &= ~bit;
1054 ctx->dirty_shader_stages |= bit;
1055 if (pstage == PIPE_SHADER_COMPUTE)
1056 return;
1057
1058 struct zink_shader_key *key = &ctx->gfx_pipeline_state.shader_keys.key[pstage];
1059 key->inline_uniforms = false;
1060 }
1061
1062 static void
zink_set_constant_buffer(struct pipe_context * pctx,enum pipe_shader_type shader,uint index,bool take_ownership,const struct pipe_constant_buffer * cb)1063 zink_set_constant_buffer(struct pipe_context *pctx,
1064 enum pipe_shader_type shader, uint index,
1065 bool take_ownership,
1066 const struct pipe_constant_buffer *cb)
1067 {
1068 struct zink_context *ctx = zink_context(pctx);
1069 bool update = false;
1070
1071 struct zink_resource *res = zink_resource(ctx->ubos[shader][index].buffer);
1072 if (cb) {
1073 struct pipe_resource *buffer = cb->buffer;
1074 unsigned offset = cb->buffer_offset;
1075 struct zink_screen *screen = zink_screen(pctx->screen);
1076 if (cb->user_buffer) {
1077 u_upload_data(ctx->base.const_uploader, 0, cb->buffer_size,
1078 screen->info.props.limits.minUniformBufferOffsetAlignment,
1079 cb->user_buffer, &offset, &buffer);
1080 }
1081 struct zink_resource *new_res = zink_resource(buffer);
1082 if (new_res) {
1083 if (new_res != res) {
1084 unbind_ubo(ctx, res, shader, index);
1085 new_res->ubo_bind_count[shader == PIPE_SHADER_COMPUTE]++;
1086 new_res->ubo_bind_mask[shader] |= BITFIELD_BIT(index);
1087 update_res_bind_count(ctx, new_res, shader == PIPE_SHADER_COMPUTE, false);
1088 }
1089 zink_batch_resource_usage_set(&ctx->batch, new_res, false);
1090 zink_resource_buffer_barrier(ctx, new_res, VK_ACCESS_UNIFORM_READ_BIT,
1091 zink_pipeline_flags_from_pipe_stage(shader));
1092 }
1093 update |= ((index || screen->descriptor_mode == ZINK_DESCRIPTOR_MODE_LAZY) && ctx->ubos[shader][index].buffer_offset != offset) ||
1094 !!res != !!buffer || (res && res->obj->buffer != new_res->obj->buffer) ||
1095 ctx->ubos[shader][index].buffer_size != cb->buffer_size;
1096
1097 if (take_ownership) {
1098 pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1099 ctx->ubos[shader][index].buffer = buffer;
1100 } else {
1101 pipe_resource_reference(&ctx->ubos[shader][index].buffer, buffer);
1102 }
1103 ctx->ubos[shader][index].buffer_offset = offset;
1104 ctx->ubos[shader][index].buffer_size = cb->buffer_size;
1105 ctx->ubos[shader][index].user_buffer = NULL;
1106
1107 if (cb->user_buffer)
1108 pipe_resource_reference(&buffer, NULL);
1109
1110 if (index + 1 >= ctx->di.num_ubos[shader])
1111 ctx->di.num_ubos[shader] = index + 1;
1112 update_descriptor_state_ubo(ctx, shader, index, new_res);
1113 } else {
1114 ctx->ubos[shader][index].buffer_offset = 0;
1115 ctx->ubos[shader][index].buffer_size = 0;
1116 ctx->ubos[shader][index].user_buffer = NULL;
1117 if (res) {
1118 unbind_ubo(ctx, res, shader, index);
1119 update_descriptor_state_ubo(ctx, shader, index, NULL);
1120 }
1121 update = !!ctx->ubos[shader][index].buffer;
1122
1123 pipe_resource_reference(&ctx->ubos[shader][index].buffer, NULL);
1124 if (ctx->di.num_ubos[shader] == index + 1)
1125 ctx->di.num_ubos[shader]--;
1126 }
1127 if (index == 0) {
1128 /* Invalidate current inlinable uniforms. */
1129 invalidate_inlined_uniforms(ctx, shader);
1130 }
1131
1132 if (update)
1133 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, index, 1);
1134 }
1135
1136 ALWAYS_INLINE static void
unbind_ssbo(struct zink_context * ctx,struct zink_resource * res,enum pipe_shader_type pstage,unsigned slot,bool writable)1137 unbind_ssbo(struct zink_context *ctx, struct zink_resource *res, enum pipe_shader_type pstage, unsigned slot, bool writable)
1138 {
1139 if (!res)
1140 return;
1141 res->ssbo_bind_mask[pstage] &= ~BITFIELD_BIT(slot);
1142 update_res_bind_count(ctx, res, pstage == PIPE_SHADER_COMPUTE, true);
1143 if (writable)
1144 res->write_bind_count[pstage == PIPE_SHADER_COMPUTE]--;
1145 }
1146
1147 static void
zink_set_shader_buffers(struct pipe_context * pctx,enum pipe_shader_type p_stage,unsigned start_slot,unsigned count,const struct pipe_shader_buffer * buffers,unsigned writable_bitmask)1148 zink_set_shader_buffers(struct pipe_context *pctx,
1149 enum pipe_shader_type p_stage,
1150 unsigned start_slot, unsigned count,
1151 const struct pipe_shader_buffer *buffers,
1152 unsigned writable_bitmask)
1153 {
1154 struct zink_context *ctx = zink_context(pctx);
1155 bool update = false;
1156 unsigned max_slot = 0;
1157
1158 unsigned modified_bits = u_bit_consecutive(start_slot, count);
1159 unsigned old_writable_mask = ctx->writable_ssbos[p_stage];
1160 ctx->writable_ssbos[p_stage] &= ~modified_bits;
1161 ctx->writable_ssbos[p_stage] |= writable_bitmask << start_slot;
1162
1163 for (unsigned i = 0; i < count; i++) {
1164 struct pipe_shader_buffer *ssbo = &ctx->ssbos[p_stage][start_slot + i];
1165 struct zink_resource *res = ssbo->buffer ? zink_resource(ssbo->buffer) : NULL;
1166 bool was_writable = old_writable_mask & BITFIELD64_BIT(start_slot + i);
1167 if (buffers && buffers[i].buffer) {
1168 struct zink_resource *new_res = zink_resource(buffers[i].buffer);
1169 if (new_res != res) {
1170 unbind_ssbo(ctx, res, p_stage, i, was_writable);
1171 new_res->ssbo_bind_mask[p_stage] |= BITFIELD_BIT(i);
1172 update_res_bind_count(ctx, new_res, p_stage == PIPE_SHADER_COMPUTE, false);
1173 }
1174 VkAccessFlags access = VK_ACCESS_SHADER_READ_BIT;
1175 if (ctx->writable_ssbos[p_stage] & BITFIELD64_BIT(start_slot + i)) {
1176 new_res->write_bind_count[p_stage == PIPE_SHADER_COMPUTE]++;
1177 access |= VK_ACCESS_SHADER_WRITE_BIT;
1178 }
1179 pipe_resource_reference(&ssbo->buffer, &new_res->base.b);
1180 zink_batch_resource_usage_set(&ctx->batch, new_res, access & VK_ACCESS_SHADER_WRITE_BIT);
1181 ssbo->buffer_offset = buffers[i].buffer_offset;
1182 ssbo->buffer_size = MIN2(buffers[i].buffer_size, new_res->base.b.width0 - ssbo->buffer_offset);
1183 util_range_add(&new_res->base.b, &new_res->valid_buffer_range, ssbo->buffer_offset,
1184 ssbo->buffer_offset + ssbo->buffer_size);
1185 zink_resource_buffer_barrier(ctx, new_res, access,
1186 zink_pipeline_flags_from_pipe_stage(p_stage));
1187 update = true;
1188 max_slot = MAX2(max_slot, start_slot + i);
1189 update_descriptor_state_ssbo(ctx, p_stage, start_slot + i, new_res);
1190 } else {
1191 update = !!res;
1192 ssbo->buffer_offset = 0;
1193 ssbo->buffer_size = 0;
1194 if (res) {
1195 unbind_ssbo(ctx, res, p_stage, i, was_writable);
1196 update_descriptor_state_ssbo(ctx, p_stage, start_slot + i, NULL);
1197 }
1198 pipe_resource_reference(&ssbo->buffer, NULL);
1199 }
1200 }
1201 if (start_slot + count >= ctx->di.num_ssbos[p_stage])
1202 ctx->di.num_ssbos[p_stage] = max_slot + 1;
1203 if (update)
1204 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_SSBO, start_slot, count);
1205 }
1206
1207 static void
update_binds_for_samplerviews(struct zink_context * ctx,struct zink_resource * res,bool is_compute)1208 update_binds_for_samplerviews(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1209 {
1210 VkImageLayout layout = get_layout_for_binding(res, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, is_compute);
1211 if (is_compute) {
1212 u_foreach_bit(slot, res->sampler_binds[PIPE_SHADER_COMPUTE]) {
1213 if (ctx->di.textures[PIPE_SHADER_COMPUTE][slot].imageLayout != layout) {
1214 update_descriptor_state_sampler(ctx, PIPE_SHADER_COMPUTE, slot, res);
1215 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, PIPE_SHADER_COMPUTE, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1216 }
1217 }
1218 } else {
1219 for (unsigned i = 0; i < ZINK_SHADER_COUNT; i++) {
1220 u_foreach_bit(slot, res->sampler_binds[i]) {
1221 if (ctx->di.textures[i][slot].imageLayout != layout) {
1222 update_descriptor_state_sampler(ctx, i, slot, res);
1223 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
1224 }
1225 }
1226 }
1227 }
1228 }
1229
1230 static void
flush_pending_clears(struct zink_context * ctx,struct zink_resource * res)1231 flush_pending_clears(struct zink_context *ctx, struct zink_resource *res)
1232 {
1233 if (res->fb_binds && ctx->clears_enabled)
1234 zink_fb_clears_apply(ctx, &res->base.b);
1235 }
1236
1237 static inline void
unbind_shader_image_counts(struct zink_context * ctx,struct zink_resource * res,bool is_compute,bool writable)1238 unbind_shader_image_counts(struct zink_context *ctx, struct zink_resource *res, bool is_compute, bool writable)
1239 {
1240 update_res_bind_count(ctx, res, is_compute, true);
1241 if (writable)
1242 res->write_bind_count[is_compute]--;
1243 res->image_bind_count[is_compute]--;
1244 /* if this was the last image bind, the sampler bind layouts must be updated */
1245 if (!res->obj->is_buffer && !res->image_bind_count[is_compute] && res->bind_count[is_compute])
1246 update_binds_for_samplerviews(ctx, res, is_compute);
1247 }
1248
1249 ALWAYS_INLINE static void
check_for_layout_update(struct zink_context * ctx,struct zink_resource * res,bool is_compute)1250 check_for_layout_update(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1251 {
1252 VkImageLayout layout = res->bind_count[is_compute] ? zink_descriptor_util_image_layout_eval(res, is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1253 VkImageLayout other_layout = res->bind_count[!is_compute] ? zink_descriptor_util_image_layout_eval(res, !is_compute) : VK_IMAGE_LAYOUT_UNDEFINED;
1254 if (res->bind_count[is_compute] && layout && res->layout != layout)
1255 _mesa_set_add(ctx->need_barriers[is_compute], res);
1256 if (res->bind_count[!is_compute] && other_layout && (layout != other_layout || res->layout != other_layout))
1257 _mesa_set_add(ctx->need_barriers[!is_compute], res);
1258 }
1259
1260 static void
unbind_shader_image(struct zink_context * ctx,enum pipe_shader_type stage,unsigned slot)1261 unbind_shader_image(struct zink_context *ctx, enum pipe_shader_type stage, unsigned slot)
1262 {
1263 struct zink_image_view *image_view = &ctx->image_views[stage][slot];
1264 bool is_compute = stage == PIPE_SHADER_COMPUTE;
1265 if (!image_view->base.resource)
1266 return;
1267
1268 struct zink_resource *res = zink_resource(image_view->base.resource);
1269 unbind_shader_image_counts(ctx, res, is_compute, image_view->base.access & PIPE_IMAGE_ACCESS_WRITE);
1270
1271 if (image_view->base.resource->target == PIPE_BUFFER) {
1272 if (zink_batch_usage_exists(image_view->buffer_view->batch_uses))
1273 zink_batch_reference_bufferview(&ctx->batch, image_view->buffer_view);
1274 zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
1275 } else {
1276 if (!res->image_bind_count[is_compute])
1277 check_for_layout_update(ctx, res, is_compute);
1278 if (zink_batch_usage_exists(image_view->surface->batch_uses))
1279 zink_batch_reference_surface(&ctx->batch, image_view->surface);
1280 zink_surface_reference(zink_screen(ctx->base.screen), &image_view->surface, NULL);
1281 }
1282 pipe_resource_reference(&image_view->base.resource, NULL);
1283 image_view->base.resource = NULL;
1284 image_view->surface = NULL;
1285 }
1286
1287 static struct zink_buffer_view *
create_image_bufferview(struct zink_context * ctx,const struct pipe_image_view * view)1288 create_image_bufferview(struct zink_context *ctx, const struct pipe_image_view *view)
1289 {
1290 struct zink_resource *res = zink_resource(view->resource);
1291 VkBufferViewCreateInfo bvci = create_bvci(ctx, res, view->format, view->u.buf.offset, view->u.buf.size);
1292 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1293 if (!buffer_view)
1294 return NULL;
1295 util_range_add(&res->base.b, &res->valid_buffer_range, view->u.buf.offset,
1296 view->u.buf.offset + view->u.buf.size);
1297 return buffer_view;
1298 }
1299
1300 static void
finalize_image_bind(struct zink_context * ctx,struct zink_resource * res,bool is_compute)1301 finalize_image_bind(struct zink_context *ctx, struct zink_resource *res, bool is_compute)
1302 {
1303 /* if this is the first image bind and there are sampler binds, the image's sampler layout
1304 * must be updated to GENERAL
1305 */
1306 if (res->image_bind_count[is_compute] == 1 &&
1307 res->bind_count[is_compute] > 1)
1308 update_binds_for_samplerviews(ctx, res, is_compute);
1309 check_for_layout_update(ctx, res, is_compute);
1310 }
1311
1312 static struct zink_surface *
create_image_surface(struct zink_context * ctx,const struct pipe_image_view * view,bool is_compute)1313 create_image_surface(struct zink_context *ctx, const struct pipe_image_view *view, bool is_compute)
1314 {
1315 struct zink_resource *res = zink_resource(view->resource);
1316 struct pipe_surface tmpl = {0};
1317 tmpl.format = view->format;
1318 tmpl.u.tex.level = view->u.tex.level;
1319 tmpl.u.tex.first_layer = view->u.tex.first_layer;
1320 tmpl.u.tex.last_layer = view->u.tex.last_layer;
1321 struct pipe_surface *psurf = ctx->base.create_surface(&ctx->base, &res->base.b, &tmpl);
1322 if (!psurf)
1323 return NULL;
1324 /* this is actually a zink_ctx_surface, but we just want the inner surface */
1325 struct zink_surface *surface = zink_csurface(psurf);
1326 FREE(psurf);
1327 flush_pending_clears(ctx, res);
1328 return surface;
1329 }
1330
1331 static void
zink_set_shader_images(struct pipe_context * pctx,enum pipe_shader_type p_stage,unsigned start_slot,unsigned count,unsigned unbind_num_trailing_slots,const struct pipe_image_view * images)1332 zink_set_shader_images(struct pipe_context *pctx,
1333 enum pipe_shader_type p_stage,
1334 unsigned start_slot, unsigned count,
1335 unsigned unbind_num_trailing_slots,
1336 const struct pipe_image_view *images)
1337 {
1338 struct zink_context *ctx = zink_context(pctx);
1339 bool update = false;
1340 for (unsigned i = 0; i < count; i++) {
1341 struct zink_image_view *image_view = &ctx->image_views[p_stage][start_slot + i];
1342 if (images && images[i].resource) {
1343 struct zink_resource *res = zink_resource(images[i].resource);
1344 struct zink_resource *old_res = zink_resource(image_view->base.resource);
1345 if (!zink_resource_object_init_storage(ctx, res)) {
1346 debug_printf("couldn't create storage image!");
1347 continue;
1348 }
1349 if (res != old_res) {
1350 if (old_res) {
1351 unbind_shader_image_counts(ctx, old_res, p_stage == PIPE_SHADER_COMPUTE, image_view->base.access & PIPE_IMAGE_ACCESS_WRITE);
1352 if (!old_res->obj->is_buffer && !old_res->image_bind_count[p_stage == PIPE_SHADER_COMPUTE])
1353 check_for_layout_update(ctx, old_res, p_stage == PIPE_SHADER_COMPUTE);
1354 }
1355 update_res_bind_count(ctx, res, p_stage == PIPE_SHADER_COMPUTE, false);
1356 }
1357 util_copy_image_view(&image_view->base, images + i);
1358 VkAccessFlags access = 0;
1359 if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) {
1360 zink_resource(image_view->base.resource)->write_bind_count[p_stage == PIPE_SHADER_COMPUTE]++;
1361 access |= VK_ACCESS_SHADER_WRITE_BIT;
1362 }
1363 if (image_view->base.access & PIPE_IMAGE_ACCESS_READ) {
1364 access |= VK_ACCESS_SHADER_READ_BIT;
1365 }
1366 res->image_bind_count[p_stage == PIPE_SHADER_COMPUTE]++;
1367 if (images[i].resource->target == PIPE_BUFFER) {
1368 image_view->buffer_view = create_image_bufferview(ctx, &images[i]);
1369 assert(image_view->buffer_view);
1370 zink_batch_usage_set(&image_view->buffer_view->batch_uses, ctx->batch.state);
1371 zink_resource_buffer_barrier(ctx, res, access,
1372 zink_pipeline_flags_from_pipe_stage(p_stage));
1373 } else {
1374 image_view->surface = create_image_surface(ctx, &images[i], p_stage == PIPE_SHADER_COMPUTE);
1375 assert(image_view->surface);
1376 finalize_image_bind(ctx, res, p_stage == PIPE_SHADER_COMPUTE);
1377 zink_batch_usage_set(&image_view->surface->batch_uses, ctx->batch.state);
1378 }
1379 zink_batch_resource_usage_set(&ctx->batch, zink_resource(image_view->base.resource),
1380 zink_resource_access_is_write(access));
1381 update = true;
1382 update_descriptor_state_image(ctx, p_stage, start_slot + i, res);
1383 } else if (image_view->base.resource) {
1384 update |= !!image_view->base.resource;
1385
1386 unbind_shader_image(ctx, p_stage, start_slot + i);
1387 update_descriptor_state_image(ctx, p_stage, start_slot + i, NULL);
1388 }
1389 }
1390 for (unsigned i = 0; i < unbind_num_trailing_slots; i++) {
1391 update |= !!ctx->image_views[p_stage][start_slot + count + i].base.resource;
1392 unbind_shader_image(ctx, p_stage, start_slot + count + i);
1393 update_descriptor_state_image(ctx, p_stage, start_slot + count + i, NULL);
1394 }
1395 ctx->di.num_images[p_stage] = start_slot + count;
1396 if (update)
1397 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, p_stage, ZINK_DESCRIPTOR_TYPE_IMAGE, start_slot, count);
1398 }
1399
1400 ALWAYS_INLINE static void
check_samplerview_for_batch_ref(struct zink_context * ctx,struct zink_sampler_view * sv)1401 check_samplerview_for_batch_ref(struct zink_context *ctx, struct zink_sampler_view *sv)
1402 {
1403 const struct zink_resource *res = zink_resource(sv->base.texture);
1404 if ((res->obj->is_buffer && zink_batch_usage_exists(sv->buffer_view->batch_uses)) ||
1405 (!res->obj->is_buffer && zink_batch_usage_exists(sv->image_view->batch_uses)))
1406 zink_batch_reference_sampler_view(&ctx->batch, sv);
1407 }
1408
1409 ALWAYS_INLINE static void
unbind_samplerview(struct zink_context * ctx,enum pipe_shader_type stage,unsigned slot)1410 unbind_samplerview(struct zink_context *ctx, enum pipe_shader_type stage, unsigned slot)
1411 {
1412 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[stage][slot]);
1413 if (!sv || !sv->base.texture)
1414 return;
1415 struct zink_resource *res = zink_resource(sv->base.texture);
1416 check_samplerview_for_batch_ref(ctx, sv);
1417 update_res_bind_count(ctx, res, stage == PIPE_SHADER_COMPUTE, true);
1418 res->sampler_binds[stage] &= ~BITFIELD_BIT(slot);
1419 }
1420
1421 static void
zink_set_sampler_views(struct pipe_context * pctx,enum pipe_shader_type shader_type,unsigned start_slot,unsigned num_views,unsigned unbind_num_trailing_slots,bool take_ownership,struct pipe_sampler_view ** views)1422 zink_set_sampler_views(struct pipe_context *pctx,
1423 enum pipe_shader_type shader_type,
1424 unsigned start_slot,
1425 unsigned num_views,
1426 unsigned unbind_num_trailing_slots,
1427 bool take_ownership,
1428 struct pipe_sampler_view **views)
1429 {
1430 struct zink_context *ctx = zink_context(pctx);
1431 unsigned i;
1432
1433 bool update = false;
1434 for (i = 0; i < num_views; ++i) {
1435 struct pipe_sampler_view *pview = views ? views[i] : NULL;
1436 struct zink_sampler_view *a = zink_sampler_view(ctx->sampler_views[shader_type][start_slot + i]);
1437 struct zink_sampler_view *b = zink_sampler_view(pview);
1438 struct zink_resource *res = b ? zink_resource(b->base.texture) : NULL;
1439 if (b && b->base.texture) {
1440 if (!a || zink_resource(a->base.texture) != res) {
1441 if (a)
1442 unbind_samplerview(ctx, shader_type, start_slot + i);
1443 update_res_bind_count(ctx, res, shader_type == PIPE_SHADER_COMPUTE, false);
1444 } else if (a != b) {
1445 check_samplerview_for_batch_ref(ctx, a);
1446 }
1447 if (res->base.b.target == PIPE_BUFFER) {
1448 if (b->buffer_view->bvci.buffer != res->obj->buffer) {
1449 /* if this resource has been rebound while it wasn't set here,
1450 * its backing resource will have changed and thus we need to update
1451 * the bufferview
1452 */
1453 VkBufferViewCreateInfo bvci = b->buffer_view->bvci;
1454 bvci.buffer = res->obj->buffer;
1455 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1456 assert(buffer_view != b->buffer_view);
1457 if (zink_batch_usage_exists(b->buffer_view->batch_uses))
1458 zink_batch_reference_bufferview(&ctx->batch, b->buffer_view);
1459 zink_buffer_view_reference(zink_screen(ctx->base.screen), &b->buffer_view, NULL);
1460 b->buffer_view = buffer_view;
1461 update = true;
1462 }
1463 zink_batch_usage_set(&b->buffer_view->batch_uses, ctx->batch.state);
1464 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT,
1465 zink_pipeline_flags_from_pipe_stage(shader_type));
1466 if (!a || a->buffer_view->buffer_view != b->buffer_view->buffer_view)
1467 update = true;
1468 } else if (!res->obj->is_buffer) {
1469 if (res->obj != b->image_view->obj) {
1470 struct pipe_surface *psurf = &b->image_view->base;
1471 VkImageView iv = b->image_view->image_view;
1472 zink_rebind_surface(ctx, &psurf);
1473 b->image_view = zink_surface(psurf);
1474 update |= iv != b->image_view->image_view;
1475 } else if (a != b)
1476 update = true;
1477 flush_pending_clears(ctx, res);
1478 check_for_layout_update(ctx, res, shader_type == PIPE_SHADER_COMPUTE);
1479 zink_batch_usage_set(&b->image_view->batch_uses, ctx->batch.state);
1480 if (!a)
1481 update = true;
1482 }
1483 res->sampler_binds[shader_type] |= BITFIELD_BIT(start_slot + i);
1484 zink_batch_resource_usage_set(&ctx->batch, res, false);
1485 } else if (a) {
1486 unbind_samplerview(ctx, shader_type, start_slot + i);
1487 update = true;
1488 }
1489 if (take_ownership) {
1490 pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], NULL);
1491 ctx->sampler_views[shader_type][start_slot + i] = pview;
1492 } else {
1493 pipe_sampler_view_reference(&ctx->sampler_views[shader_type][start_slot + i], pview);
1494 }
1495 update_descriptor_state_sampler(ctx, shader_type, start_slot + i, res);
1496 }
1497 for (; i < num_views + unbind_num_trailing_slots; ++i) {
1498 update |= !!ctx->sampler_views[shader_type][start_slot + i];
1499 unbind_samplerview(ctx, shader_type, start_slot + i);
1500 pipe_sampler_view_reference(
1501 &ctx->sampler_views[shader_type][start_slot + i],
1502 NULL);
1503 update_descriptor_state_sampler(ctx, shader_type, start_slot + i, NULL);
1504 }
1505 ctx->di.num_sampler_views[shader_type] = start_slot + num_views;
1506 if (update)
1507 zink_screen(pctx->screen)->context_invalidate_descriptor_state(ctx, shader_type, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, start_slot, num_views);
1508 }
1509
1510 static uint64_t
zink_create_texture_handle(struct pipe_context * pctx,struct pipe_sampler_view * view,const struct pipe_sampler_state * state)1511 zink_create_texture_handle(struct pipe_context *pctx, struct pipe_sampler_view *view, const struct pipe_sampler_state *state)
1512 {
1513 struct zink_context *ctx = zink_context(pctx);
1514 struct zink_resource *res = zink_resource(view->texture);
1515 struct zink_sampler_view *sv = zink_sampler_view(view);
1516 struct zink_bindless_descriptor *bd;
1517 bd = calloc(1, sizeof(struct zink_bindless_descriptor));
1518 if (!bd)
1519 return 0;
1520
1521 bd->sampler = pctx->create_sampler_state(pctx, state);
1522 if (!bd->sampler) {
1523 free(bd);
1524 return 0;
1525 }
1526
1527 bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
1528 if (res->base.b.target == PIPE_BUFFER)
1529 zink_buffer_view_reference(zink_screen(pctx->screen), &bd->ds.bufferview, sv->buffer_view);
1530 else
1531 zink_surface_reference(zink_screen(pctx->screen), &bd->ds.surface, sv->image_view);
1532 uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].tex_slots);
1533 if (bd->ds.is_buffer)
1534 handle += ZINK_MAX_BINDLESS_HANDLES;
1535 bd->handle = handle;
1536 _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].tex_handles, (void*)(uintptr_t)handle, bd);
1537 return handle;
1538 }
1539
1540 static void
zink_delete_texture_handle(struct pipe_context * pctx,uint64_t handle)1541 zink_delete_texture_handle(struct pipe_context *pctx, uint64_t handle)
1542 {
1543 struct zink_context *ctx = zink_context(pctx);
1544 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
1545 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
1546 assert(he);
1547 struct zink_bindless_descriptor *bd = he->data;
1548 struct zink_descriptor_surface *ds = &bd->ds;
1549 _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].tex_handles, he);
1550 uint32_t h = handle;
1551 util_dynarray_append(&ctx->batch.state->bindless_releases[0], uint32_t, h);
1552
1553 struct zink_resource *res = zink_descriptor_surface_resource(ds);
1554 if (ds->is_buffer) {
1555 if (zink_resource_has_usage(res))
1556 zink_batch_reference_bufferview(&ctx->batch, ds->bufferview);
1557 zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
1558 } else {
1559 if (zink_resource_has_usage(res))
1560 zink_batch_reference_surface(&ctx->batch, ds->surface);
1561 zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
1562 pctx->delete_sampler_state(pctx, bd->sampler);
1563 }
1564 free(ds);
1565 }
1566
1567 static void
rebind_bindless_bufferview(struct zink_context * ctx,struct zink_resource * res,struct zink_descriptor_surface * ds)1568 rebind_bindless_bufferview(struct zink_context *ctx, struct zink_resource *res, struct zink_descriptor_surface *ds)
1569 {
1570 /* if this resource has been rebound while it wasn't set here,
1571 * its backing resource will have changed and thus we need to update
1572 * the bufferview
1573 */
1574 VkBufferViewCreateInfo bvci = ds->bufferview->bvci;
1575 bvci.buffer = res->obj->buffer;
1576 struct zink_buffer_view *buffer_view = get_buffer_view(ctx, res, &bvci);
1577 assert(buffer_view != ds->bufferview);
1578 if (zink_resource_has_usage(res))
1579 zink_batch_reference_bufferview(&ctx->batch, ds->bufferview);
1580 zink_buffer_view_reference(zink_screen(ctx->base.screen), &ds->bufferview, NULL);
1581 ds->bufferview = buffer_view;
1582 }
1583
1584 static void
zero_bindless_descriptor(struct zink_context * ctx,uint32_t handle,bool is_buffer,bool is_image)1585 zero_bindless_descriptor(struct zink_context *ctx, uint32_t handle, bool is_buffer, bool is_image)
1586 {
1587 if (likely(zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor)) {
1588 if (is_buffer) {
1589 VkBufferView *bv = &ctx->di.bindless[is_image].buffer_infos[handle];
1590 *bv = VK_NULL_HANDLE;
1591 } else {
1592 VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
1593 memset(ii, 0, sizeof(*ii));
1594 }
1595 } else {
1596 if (is_buffer) {
1597 VkBufferView *bv = &ctx->di.bindless[is_image].buffer_infos[handle];
1598 struct zink_buffer_view *null_bufferview = ctx->dummy_bufferview;
1599 *bv = null_bufferview->buffer_view;
1600 } else {
1601 struct zink_surface *null_surface = zink_csurface(ctx->dummy_surface[is_image]);
1602 VkDescriptorImageInfo *ii = &ctx->di.bindless[is_image].img_infos[handle];
1603 ii->sampler = VK_NULL_HANDLE;
1604 ii->imageView = null_surface->image_view;
1605 ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
1606 }
1607 }
1608 }
1609
1610 static void
zink_make_texture_handle_resident(struct pipe_context * pctx,uint64_t handle,bool resident)1611 zink_make_texture_handle_resident(struct pipe_context *pctx, uint64_t handle, bool resident)
1612 {
1613 struct zink_context *ctx = zink_context(pctx);
1614 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
1615 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].tex_handles, (void*)(uintptr_t)handle);
1616 assert(he);
1617 struct zink_bindless_descriptor *bd = he->data;
1618 struct zink_descriptor_surface *ds = &bd->ds;
1619 struct zink_resource *res = zink_descriptor_surface_resource(ds);
1620 if (is_buffer)
1621 handle -= ZINK_MAX_BINDLESS_HANDLES;
1622 if (resident) {
1623 update_res_bind_count(ctx, res, false, false);
1624 update_res_bind_count(ctx, res, true, false);
1625 res->bindless[0]++;
1626 if (is_buffer) {
1627 if (ds->bufferview->bvci.buffer != res->obj->buffer)
1628 rebind_bindless_bufferview(ctx, res, ds);
1629 VkBufferView *bv = &ctx->di.bindless[0].buffer_infos[handle];
1630 *bv = ds->bufferview->buffer_view;
1631 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
1632 } else {
1633 VkDescriptorImageInfo *ii = &ctx->di.bindless[0].img_infos[handle];
1634 ii->sampler = bd->sampler->sampler;
1635 ii->imageView = ds->surface->image_view;
1636 ii->imageLayout = zink_descriptor_util_image_layout_eval(res, false);
1637 flush_pending_clears(ctx, res);
1638 check_for_layout_update(ctx, res, false);
1639 check_for_layout_update(ctx, res, true);
1640 }
1641 zink_batch_resource_usage_set(&ctx->batch, res, false);
1642 util_dynarray_append(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
1643 uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
1644 util_dynarray_append(&ctx->di.bindless[0].updates, uint32_t, h);
1645 } else {
1646 zero_bindless_descriptor(ctx, handle, is_buffer, false);
1647 util_dynarray_delete_unordered(&ctx->di.bindless[0].resident, struct zink_bindless_descriptor *, bd);
1648 update_res_bind_count(ctx, res, false, true);
1649 update_res_bind_count(ctx, res, true, true);
1650 res->bindless[0]--;
1651 for (unsigned i = 0; i < 2; i++) {
1652 if (!res->image_bind_count[i])
1653 check_for_layout_update(ctx, res, i);
1654 }
1655 }
1656 ctx->di.bindless_dirty[0] = true;
1657 }
1658
1659 static uint64_t
zink_create_image_handle(struct pipe_context * pctx,const struct pipe_image_view * view)1660 zink_create_image_handle(struct pipe_context *pctx, const struct pipe_image_view *view)
1661 {
1662 struct zink_context *ctx = zink_context(pctx);
1663 struct zink_resource *res = zink_resource(view->resource);
1664 struct zink_bindless_descriptor *bd;
1665 if (!zink_resource_object_init_storage(ctx, res)) {
1666 debug_printf("couldn't create storage image!");
1667 return 0;
1668 }
1669 bd = malloc(sizeof(struct zink_bindless_descriptor));
1670 if (!bd)
1671 return 0;
1672 bd->sampler = NULL;
1673
1674 bd->ds.is_buffer = res->base.b.target == PIPE_BUFFER;
1675 if (res->base.b.target == PIPE_BUFFER)
1676 bd->ds.bufferview = create_image_bufferview(ctx, view);
1677 else
1678 bd->ds.surface = create_image_surface(ctx, view, false);
1679 uint64_t handle = util_idalloc_alloc(&ctx->di.bindless[bd->ds.is_buffer].img_slots);
1680 if (bd->ds.is_buffer)
1681 handle += ZINK_MAX_BINDLESS_HANDLES;
1682 bd->handle = handle;
1683 _mesa_hash_table_insert(&ctx->di.bindless[bd->ds.is_buffer].img_handles, (void*)(uintptr_t)handle, bd);
1684 return handle;
1685 }
1686
1687 static void
zink_delete_image_handle(struct pipe_context * pctx,uint64_t handle)1688 zink_delete_image_handle(struct pipe_context *pctx, uint64_t handle)
1689 {
1690 struct zink_context *ctx = zink_context(pctx);
1691 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
1692 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
1693 assert(he);
1694 struct zink_descriptor_surface *ds = he->data;
1695 _mesa_hash_table_remove(&ctx->di.bindless[is_buffer].img_handles, he);
1696 uint32_t h = handle;
1697 util_dynarray_append(&ctx->batch.state->bindless_releases[1], uint32_t, h);
1698
1699 struct zink_resource *res = zink_descriptor_surface_resource(ds);
1700 if (ds->is_buffer) {
1701 if (zink_resource_has_usage(res))
1702 zink_batch_reference_bufferview(&ctx->batch, ds->bufferview);
1703 zink_buffer_view_reference(zink_screen(pctx->screen), &ds->bufferview, NULL);
1704 } else {
1705 if (zink_resource_has_usage(res))
1706 zink_batch_reference_surface(&ctx->batch, ds->surface);
1707 zink_surface_reference(zink_screen(pctx->screen), &ds->surface, NULL);
1708 }
1709 free(ds);
1710 }
1711
1712 static void
zink_make_image_handle_resident(struct pipe_context * pctx,uint64_t handle,unsigned paccess,bool resident)1713 zink_make_image_handle_resident(struct pipe_context *pctx, uint64_t handle, unsigned paccess, bool resident)
1714 {
1715 struct zink_context *ctx = zink_context(pctx);
1716 bool is_buffer = ZINK_BINDLESS_IS_BUFFER(handle);
1717 struct hash_entry *he = _mesa_hash_table_search(&ctx->di.bindless[is_buffer].img_handles, (void*)(uintptr_t)handle);
1718 assert(he);
1719 struct zink_bindless_descriptor *bd = he->data;
1720 struct zink_descriptor_surface *ds = &bd->ds;
1721 bd->access = paccess;
1722 struct zink_resource *res = zink_descriptor_surface_resource(ds);
1723 VkAccessFlags access = 0;
1724 if (paccess & PIPE_IMAGE_ACCESS_WRITE) {
1725 if (resident) {
1726 res->write_bind_count[0]++;
1727 res->write_bind_count[1]++;
1728 } else {
1729 res->write_bind_count[0]--;
1730 res->write_bind_count[1]--;
1731 }
1732 access |= VK_ACCESS_SHADER_WRITE_BIT;
1733 }
1734 if (paccess & PIPE_IMAGE_ACCESS_READ) {
1735 access |= VK_ACCESS_SHADER_READ_BIT;
1736 }
1737 if (is_buffer)
1738 handle -= ZINK_MAX_BINDLESS_HANDLES;
1739 if (resident) {
1740 update_res_bind_count(ctx, res, false, false);
1741 update_res_bind_count(ctx, res, true, false);
1742 res->image_bind_count[0]++;
1743 res->image_bind_count[1]++;
1744 res->bindless[1]++;
1745 if (is_buffer) {
1746 if (ds->bufferview->bvci.buffer != res->obj->buffer)
1747 rebind_bindless_bufferview(ctx, res, ds);
1748 VkBufferView *bv = &ctx->di.bindless[1].buffer_infos[handle];
1749 *bv = ds->bufferview->buffer_view;
1750 zink_resource_buffer_barrier(ctx, res, access, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT | VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT);
1751 } else {
1752 VkDescriptorImageInfo *ii = &ctx->di.bindless[1].img_infos[handle];
1753 ii->sampler = VK_NULL_HANDLE;
1754 ii->imageView = ds->surface->image_view;
1755 ii->imageLayout = VK_IMAGE_LAYOUT_GENERAL;
1756 finalize_image_bind(ctx, res, false);
1757 finalize_image_bind(ctx, res, true);
1758 }
1759 zink_batch_resource_usage_set(&ctx->batch, res, zink_resource_access_is_write(access));
1760 util_dynarray_append(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
1761 uint32_t h = is_buffer ? handle + ZINK_MAX_BINDLESS_HANDLES : handle;
1762 util_dynarray_append(&ctx->di.bindless[1].updates, uint32_t, h);
1763 } else {
1764 zero_bindless_descriptor(ctx, handle, is_buffer, true);
1765 util_dynarray_delete_unordered(&ctx->di.bindless[1].resident, struct zink_bindless_descriptor *, bd);
1766 unbind_shader_image_counts(ctx, res, false, false);
1767 unbind_shader_image_counts(ctx, res, true, false);
1768 res->bindless[1]--;
1769 for (unsigned i = 0; i < 2; i++) {
1770 if (!res->image_bind_count[i])
1771 check_for_layout_update(ctx, res, i);
1772 }
1773 }
1774 ctx->di.bindless_dirty[1] = true;
1775 }
1776
1777 static void
zink_set_stencil_ref(struct pipe_context * pctx,const struct pipe_stencil_ref ref)1778 zink_set_stencil_ref(struct pipe_context *pctx,
1779 const struct pipe_stencil_ref ref)
1780 {
1781 struct zink_context *ctx = zink_context(pctx);
1782 ctx->stencil_ref = ref;
1783 ctx->stencil_ref_changed = true;
1784 }
1785
1786 static void
zink_set_clip_state(struct pipe_context * pctx,const struct pipe_clip_state * pcs)1787 zink_set_clip_state(struct pipe_context *pctx,
1788 const struct pipe_clip_state *pcs)
1789 {
1790 }
1791
1792 static void
zink_set_tess_state(struct pipe_context * pctx,const float default_outer_level[4],const float default_inner_level[2])1793 zink_set_tess_state(struct pipe_context *pctx,
1794 const float default_outer_level[4],
1795 const float default_inner_level[2])
1796 {
1797 struct zink_context *ctx = zink_context(pctx);
1798 memcpy(&ctx->default_inner_level, default_inner_level, sizeof(ctx->default_inner_level));
1799 memcpy(&ctx->default_outer_level, default_outer_level, sizeof(ctx->default_outer_level));
1800 }
1801
1802 static void
zink_set_patch_vertices(struct pipe_context * pctx,uint8_t patch_vertices)1803 zink_set_patch_vertices(struct pipe_context *pctx, uint8_t patch_vertices)
1804 {
1805 struct zink_context *ctx = zink_context(pctx);
1806 ctx->gfx_pipeline_state.patch_vertices = patch_vertices;
1807 }
1808
1809 void
zink_update_fbfetch(struct zink_context * ctx)1810 zink_update_fbfetch(struct zink_context *ctx)
1811 {
1812 const bool had_fbfetch = ctx->di.fbfetch.imageLayout == VK_IMAGE_LAYOUT_GENERAL;
1813 if (!ctx->gfx_stages[PIPE_SHADER_FRAGMENT] ||
1814 !ctx->gfx_stages[PIPE_SHADER_FRAGMENT]->nir->info.fs.uses_fbfetch_output) {
1815 if (!had_fbfetch)
1816 return;
1817 ctx->rp_changed = true;
1818 zink_batch_no_rp(ctx);
1819 ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_UNDEFINED;
1820 ctx->di.fbfetch.imageView = zink_screen(ctx->base.screen)->info.rb2_feats.nullDescriptor ?
1821 VK_NULL_HANDLE :
1822 zink_csurface(ctx->dummy_surface[0])->image_view;
1823 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, PIPE_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
1824 return;
1825 }
1826
1827 bool changed = !had_fbfetch;
1828 if (ctx->fb_state.cbufs[0]) {
1829 VkImageView fbfetch = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
1830 changed |= fbfetch != ctx->di.fbfetch.imageView;
1831 ctx->di.fbfetch.imageView = zink_csurface(ctx->fb_state.cbufs[0])->image_view;
1832 }
1833 ctx->di.fbfetch.imageLayout = VK_IMAGE_LAYOUT_GENERAL;
1834 if (changed) {
1835 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, PIPE_SHADER_FRAGMENT, ZINK_DESCRIPTOR_TYPE_UBO, 0, 1);
1836 ctx->rp_changed = true;
1837 zink_batch_no_rp(ctx);
1838 }
1839 }
1840
1841 static size_t
rp_state_size(const struct zink_render_pass_pipeline_state * pstate)1842 rp_state_size(const struct zink_render_pass_pipeline_state *pstate)
1843 {
1844 return offsetof(struct zink_render_pass_pipeline_state, attachments) +
1845 sizeof(pstate->attachments[0]) * pstate->num_attachments;
1846 }
1847
1848 static uint32_t
hash_rp_state(const void * key)1849 hash_rp_state(const void *key)
1850 {
1851 const struct zink_render_pass_pipeline_state *s = key;
1852 return _mesa_hash_data(key, rp_state_size(s));
1853 }
1854
1855 static bool
equals_rp_state(const void * a,const void * b)1856 equals_rp_state(const void *a, const void *b)
1857 {
1858 return !memcmp(a, b, rp_state_size(a));
1859 }
1860
1861 static uint32_t
hash_render_pass_state(const void * key)1862 hash_render_pass_state(const void *key)
1863 {
1864 struct zink_render_pass_state* s = (struct zink_render_pass_state*)key;
1865 return _mesa_hash_data(key, offsetof(struct zink_render_pass_state, rts) + sizeof(s->rts[0]) * s->num_rts);
1866 }
1867
1868 static bool
equals_render_pass_state(const void * a,const void * b)1869 equals_render_pass_state(const void *a, const void *b)
1870 {
1871 const struct zink_render_pass_state *s_a = a, *s_b = b;
1872 if (s_a->num_rts != s_b->num_rts)
1873 return false;
1874 return memcmp(a, b, offsetof(struct zink_render_pass_state, rts) + sizeof(s_a->rts[0]) * s_a->num_rts) == 0;
1875 }
1876
1877 static struct zink_render_pass *
get_render_pass(struct zink_context * ctx)1878 get_render_pass(struct zink_context *ctx)
1879 {
1880 struct zink_screen *screen = zink_screen(ctx->base.screen);
1881 const struct pipe_framebuffer_state *fb = &ctx->fb_state;
1882 struct zink_render_pass_state state = {0};
1883 uint32_t clears = 0;
1884 state.swapchain_init = ctx->new_swapchain;
1885 state.samples = fb->samples > 0;
1886
1887 u_foreach_bit(i, ctx->fbfetch_outputs)
1888 state.rts[i].fbfetch = true;
1889
1890 for (int i = 0; i < fb->nr_cbufs; i++) {
1891 struct pipe_surface *surf = fb->cbufs[i];
1892 if (surf) {
1893 struct zink_surface *transient = zink_transient_surface(surf);
1894 state.rts[i].format = zink_get_format(screen, surf->format);
1895 state.rts[i].samples = MAX3(transient ? transient->base.nr_samples : 0, surf->texture->nr_samples, 1);
1896 state.rts[i].clear_color = zink_fb_clear_enabled(ctx, i) && !zink_fb_clear_first_needs_explicit(&ctx->fb_clears[i]);
1897 clears |= !!state.rts[i].clear_color ? PIPE_CLEAR_COLOR0 << i : 0;
1898 state.rts[i].swapchain = surf->texture->bind & PIPE_BIND_SCANOUT;
1899 if (transient) {
1900 state.num_cresolves++;
1901 state.rts[i].resolve = true;
1902 if (!state.rts[i].clear_color)
1903 state.msaa_expand_mask |= BITFIELD_BIT(i);
1904 }
1905 } else {
1906 state.rts[i].format = VK_FORMAT_R8_UINT;
1907 state.rts[i].samples = fb->samples;
1908 }
1909 state.num_rts++;
1910 }
1911 state.num_cbufs = fb->nr_cbufs;
1912 assert(!state.num_cresolves || state.num_cbufs == state.num_cresolves);
1913
1914 if (fb->zsbuf) {
1915 struct zink_resource *zsbuf = zink_resource(fb->zsbuf->texture);
1916 struct zink_framebuffer_clear *fb_clear = &ctx->fb_clears[PIPE_MAX_COLOR_BUFS];
1917 struct zink_surface *transient = zink_transient_surface(fb->zsbuf);
1918 state.rts[fb->nr_cbufs].format = zsbuf->format;
1919 state.rts[fb->nr_cbufs].samples = MAX3(transient ? transient->base.nr_samples : 0, fb->zsbuf->texture->nr_samples, 1);
1920 if (transient) {
1921 state.num_zsresolves = 1;
1922 state.rts[fb->nr_cbufs].resolve = true;
1923 }
1924 state.rts[fb->nr_cbufs].clear_color = zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS) &&
1925 !zink_fb_clear_first_needs_explicit(fb_clear) &&
1926 (zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_DEPTH);
1927 state.rts[fb->nr_cbufs].clear_stencil = zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS) &&
1928 !zink_fb_clear_first_needs_explicit(fb_clear) &&
1929 (zink_fb_clear_element(fb_clear, 0)->zs.bits & PIPE_CLEAR_STENCIL);
1930 if (state.rts[fb->nr_cbufs].clear_color)
1931 clears |= PIPE_CLEAR_DEPTH;
1932 if (state.rts[fb->nr_cbufs].clear_stencil)
1933 clears |= PIPE_CLEAR_STENCIL;
1934 const uint64_t outputs_written = ctx->gfx_stages[PIPE_SHADER_FRAGMENT] ?
1935 ctx->gfx_stages[PIPE_SHADER_FRAGMENT]->nir->info.outputs_written : 0;
1936 bool needs_write = (ctx->dsa_state && ctx->dsa_state->hw_state.depth_write) ||
1937 outputs_written & (BITFIELD64_BIT(FRAG_RESULT_DEPTH) | BITFIELD64_BIT(FRAG_RESULT_STENCIL));
1938 state.rts[fb->nr_cbufs].needs_write = needs_write || state.num_zsresolves || state.rts[fb->nr_cbufs].clear_color || state.rts[fb->nr_cbufs].clear_stencil;
1939 state.num_rts++;
1940 }
1941 state.have_zsbuf = fb->zsbuf != NULL;
1942 assert(clears == ctx->rp_clears_enabled);
1943 state.clears = clears;
1944 uint32_t hash = hash_render_pass_state(&state);
1945 struct hash_entry *entry = _mesa_hash_table_search_pre_hashed(ctx->render_pass_cache, hash,
1946 &state);
1947 struct zink_render_pass *rp;
1948 if (entry) {
1949 rp = entry->data;
1950 assert(rp->state.clears == clears);
1951 } else {
1952 struct zink_render_pass_pipeline_state pstate;
1953 pstate.samples = state.samples;
1954 rp = zink_create_render_pass(screen, &state, &pstate);
1955 if (!_mesa_hash_table_insert_pre_hashed(ctx->render_pass_cache, hash, &rp->state, rp))
1956 return NULL;
1957 bool found = false;
1958 struct set_entry *entry = _mesa_set_search_or_add(&ctx->render_pass_state_cache, &pstate, &found);
1959 struct zink_render_pass_pipeline_state *ppstate;
1960 if (!found) {
1961 entry->key = ralloc(ctx, struct zink_render_pass_pipeline_state);
1962 ppstate = (void*)entry->key;
1963 memcpy(ppstate, &pstate, rp_state_size(&pstate));
1964 ppstate->id = ctx->render_pass_state_cache.entries;
1965 }
1966 ppstate = (void*)entry->key;
1967 rp->pipeline_state = ppstate->id;
1968 }
1969 return rp;
1970 }
1971
1972 static uint32_t
hash_framebuffer_imageless(const void * key)1973 hash_framebuffer_imageless(const void *key)
1974 {
1975 struct zink_framebuffer_state* s = (struct zink_framebuffer_state*)key;
1976 return _mesa_hash_data(key, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments);
1977 }
1978
1979 static bool
equals_framebuffer_imageless(const void * a,const void * b)1980 equals_framebuffer_imageless(const void *a, const void *b)
1981 {
1982 struct zink_framebuffer_state *s = (struct zink_framebuffer_state*)a;
1983 return memcmp(a, b, offsetof(struct zink_framebuffer_state, infos) + sizeof(s->infos[0]) * s->num_attachments) == 0;
1984 }
1985
1986 static void
setup_framebuffer(struct zink_context * ctx)1987 setup_framebuffer(struct zink_context *ctx)
1988 {
1989 struct zink_screen *screen = zink_screen(ctx->base.screen);
1990 struct zink_render_pass *rp = ctx->gfx_pipeline_state.render_pass;
1991
1992 if (ctx->gfx_pipeline_state.sample_locations_enabled && ctx->sample_locations_changed) {
1993 unsigned samples = ctx->gfx_pipeline_state.rast_samples + 1;
1994 unsigned idx = util_logbase2_ceil(MAX2(samples, 1));
1995 VkExtent2D grid_size = screen->maxSampleLocationGridSize[idx];
1996
1997 for (unsigned pixel = 0; pixel < grid_size.width * grid_size.height; pixel++) {
1998 for (unsigned sample = 0; sample < samples; sample++) {
1999 unsigned pixel_x = pixel % grid_size.width;
2000 unsigned pixel_y = pixel / grid_size.width;
2001 unsigned wi = pixel * samples + sample;
2002 unsigned ri = (pixel_y * grid_size.width + pixel_x % grid_size.width);
2003 ri = ri * samples + sample;
2004 ctx->vk_sample_locations[wi].x = (ctx->sample_locations[ri] & 0xf) / 16.0f;
2005 ctx->vk_sample_locations[wi].y = (16 - (ctx->sample_locations[ri] >> 4)) / 16.0f;
2006 }
2007 }
2008 }
2009
2010 if (rp)
2011 ctx->rp_changed |= ctx->rp_clears_enabled != rp->state.clears;
2012 if (ctx->rp_changed)
2013 rp = get_render_pass(ctx);
2014
2015 ctx->fb_changed |= rp != ctx->gfx_pipeline_state.render_pass;
2016 if (rp->pipeline_state != ctx->gfx_pipeline_state.rp_state) {
2017 ctx->gfx_pipeline_state.rp_state = rp->pipeline_state;
2018 ctx->gfx_pipeline_state.dirty = true;
2019 }
2020
2021 ctx->rp_changed = false;
2022
2023 if (!ctx->fb_changed)
2024 return;
2025
2026 ctx->init_framebuffer(screen, ctx->framebuffer, rp);
2027 ctx->fb_changed = false;
2028 ctx->gfx_pipeline_state.render_pass = rp;
2029 }
2030
2031 static VkImageView
prep_fb_attachment(struct zink_context * ctx,struct zink_surface * surf,unsigned i)2032 prep_fb_attachment(struct zink_context *ctx, struct zink_surface *surf, unsigned i)
2033 {
2034 if (!surf)
2035 return zink_csurface(ctx->dummy_surface[util_logbase2_ceil(ctx->fb_state.samples)])->image_view;
2036
2037 zink_batch_resource_usage_set(&ctx->batch, zink_resource(surf->base.texture), true);
2038 zink_batch_usage_set(&surf->batch_uses, ctx->batch.state);
2039
2040 struct zink_resource *res = zink_resource(surf->base.texture);
2041 VkAccessFlags access;
2042 VkPipelineStageFlags pipeline;
2043 VkImageLayout layout = zink_render_pass_attachment_get_barrier_info(ctx->gfx_pipeline_state.render_pass,
2044 i, &pipeline, &access);
2045 zink_resource_image_barrier(ctx, res, layout, access, pipeline);
2046 return surf->image_view;
2047 }
2048
2049 static void
prep_fb_attachments(struct zink_context * ctx,VkImageView * att)2050 prep_fb_attachments(struct zink_context *ctx, VkImageView *att)
2051 {
2052 const unsigned cresolve_offset = ctx->fb_state.nr_cbufs + !!ctx->fb_state.zsbuf;
2053 unsigned num_resolves = 0;
2054 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2055 struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2056 struct zink_surface *transient = zink_transient_surface(ctx->fb_state.cbufs[i]);
2057 if (transient) {
2058 att[i] = prep_fb_attachment(ctx, transient, i);
2059 att[i + cresolve_offset] = prep_fb_attachment(ctx, surf, i);
2060 num_resolves++;
2061 } else {
2062 att[i] = prep_fb_attachment(ctx, surf, i);
2063 }
2064 }
2065 if (ctx->fb_state.zsbuf) {
2066 struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2067 struct zink_surface *transient = zink_transient_surface(ctx->fb_state.zsbuf);
2068 if (transient) {
2069 att[ctx->fb_state.nr_cbufs] = prep_fb_attachment(ctx, transient, ctx->fb_state.nr_cbufs);
2070 att[cresolve_offset + num_resolves] = prep_fb_attachment(ctx, surf, ctx->fb_state.nr_cbufs);
2071 } else {
2072 att[ctx->fb_state.nr_cbufs] = prep_fb_attachment(ctx, surf, ctx->fb_state.nr_cbufs);
2073 }
2074 }
2075 }
2076
2077 static void
update_framebuffer_state(struct zink_context * ctx,int old_w,int old_h)2078 update_framebuffer_state(struct zink_context *ctx, int old_w, int old_h)
2079 {
2080 if (ctx->fb_state.width != old_w || ctx->fb_state.height != old_h)
2081 ctx->scissor_changed = true;
2082 /* get_framebuffer adds a ref if the fb is reused or created;
2083 * always do get_framebuffer first to avoid deleting the same fb
2084 * we're about to use
2085 */
2086 struct zink_framebuffer *fb = ctx->get_framebuffer(ctx);
2087 struct zink_screen *screen = zink_screen(ctx->base.screen);
2088 if (ctx->framebuffer && !screen->info.have_KHR_imageless_framebuffer) {
2089 simple_mtx_lock(&screen->framebuffer_mtx);
2090 struct hash_entry *he = _mesa_hash_table_search(&screen->framebuffer_cache, &ctx->framebuffer->state);
2091 if (ctx->framebuffer && !ctx->framebuffer->state.num_attachments) {
2092 /* if this has no attachments then its lifetime has ended */
2093 _mesa_hash_table_remove(&screen->framebuffer_cache, he);
2094 he = NULL;
2095 /* ensure an unflushed fb doesn't get destroyed by deferring it */
2096 util_dynarray_append(&ctx->batch.state->dead_framebuffers, struct zink_framebuffer*, ctx->framebuffer);
2097 ctx->framebuffer = NULL;
2098 }
2099 /* a framebuffer loses 1 ref every time we unset it;
2100 * we do NOT add refs here, as the ref has already been added in
2101 * get_framebuffer()
2102 */
2103 if (zink_framebuffer_reference(screen, &ctx->framebuffer, NULL) && he)
2104 _mesa_hash_table_remove(&screen->framebuffer_cache, he);
2105 simple_mtx_unlock(&screen->framebuffer_mtx);
2106 }
2107 ctx->fb_changed |= ctx->framebuffer != fb;
2108 ctx->framebuffer = fb;
2109 }
2110
2111 static unsigned
begin_render_pass(struct zink_context * ctx)2112 begin_render_pass(struct zink_context *ctx)
2113 {
2114 struct zink_batch *batch = &ctx->batch;
2115 struct pipe_framebuffer_state *fb_state = &ctx->fb_state;
2116
2117 VkRenderPassBeginInfo rpbi = {0};
2118 rpbi.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
2119 rpbi.renderPass = ctx->gfx_pipeline_state.render_pass->render_pass;
2120 rpbi.renderArea.offset.x = 0;
2121 rpbi.renderArea.offset.y = 0;
2122 rpbi.renderArea.extent.width = fb_state->width;
2123 rpbi.renderArea.extent.height = fb_state->height;
2124
2125 VkClearValue clears[PIPE_MAX_COLOR_BUFS + 1] = {0};
2126 unsigned clear_buffers = 0;
2127 uint32_t clear_validate = 0;
2128 for (int i = 0; i < fb_state->nr_cbufs; i++) {
2129 /* these are no-ops */
2130 if (!fb_state->cbufs[i] || !zink_fb_clear_enabled(ctx, i))
2131 continue;
2132 /* these need actual clear calls inside the rp */
2133 struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(&ctx->fb_clears[i], 0);
2134 if (zink_fb_clear_needs_explicit(&ctx->fb_clears[i])) {
2135 clear_buffers |= (PIPE_CLEAR_COLOR0 << i);
2136 if (zink_fb_clear_count(&ctx->fb_clears[i]) < 2 ||
2137 zink_fb_clear_element_needs_explicit(clear))
2138 continue;
2139 }
2140 /* we now know there's one clear that can be done here */
2141 zink_fb_clear_util_unpack_clear_color(clear, fb_state->cbufs[i]->format, (void*)&clears[i].color);
2142 rpbi.clearValueCount = i + 1;
2143 clear_validate |= PIPE_CLEAR_COLOR0 << i;
2144 assert(ctx->framebuffer->rp->state.clears);
2145 }
2146 if (fb_state->zsbuf && zink_fb_clear_enabled(ctx, PIPE_MAX_COLOR_BUFS)) {
2147 struct zink_framebuffer_clear *fb_clear = &ctx->fb_clears[PIPE_MAX_COLOR_BUFS];
2148 struct zink_framebuffer_clear_data *clear = zink_fb_clear_element(fb_clear, 0);
2149 if (!zink_fb_clear_element_needs_explicit(clear)) {
2150 clears[fb_state->nr_cbufs].depthStencil.depth = clear->zs.depth;
2151 clears[fb_state->nr_cbufs].depthStencil.stencil = clear->zs.stencil;
2152 rpbi.clearValueCount = fb_state->nr_cbufs + 1;
2153 clear_validate |= clear->zs.bits;
2154 assert(ctx->framebuffer->rp->state.clears);
2155 }
2156 if (zink_fb_clear_needs_explicit(fb_clear)) {
2157 for (int j = !zink_fb_clear_element_needs_explicit(clear);
2158 (clear_buffers & PIPE_CLEAR_DEPTHSTENCIL) != PIPE_CLEAR_DEPTHSTENCIL && j < zink_fb_clear_count(fb_clear);
2159 j++)
2160 clear_buffers |= zink_fb_clear_element(fb_clear, j)->zs.bits;
2161 }
2162 }
2163 assert(clear_validate == ctx->framebuffer->rp->state.clears);
2164 rpbi.pClearValues = &clears[0];
2165 rpbi.framebuffer = ctx->framebuffer->fb;
2166
2167 assert(ctx->gfx_pipeline_state.render_pass && ctx->framebuffer);
2168
2169 VkRenderPassAttachmentBeginInfo infos;
2170 VkImageView att[2 * (PIPE_MAX_COLOR_BUFS + 1)];
2171 infos.sType = VK_STRUCTURE_TYPE_RENDER_PASS_ATTACHMENT_BEGIN_INFO;
2172 infos.pNext = NULL;
2173 infos.attachmentCount = ctx->framebuffer->state.num_attachments;
2174 infos.pAttachments = att;
2175 prep_fb_attachments(ctx, att);
2176 if (zink_screen(ctx->base.screen)->info.have_KHR_imageless_framebuffer) {
2177 #ifndef NDEBUG
2178 const unsigned cresolve_offset = ctx->fb_state.nr_cbufs + !!ctx->fb_state.zsbuf;
2179 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2180 if (ctx->fb_state.cbufs[i]) {
2181 struct zink_surface *surf = zink_csurface(ctx->fb_state.cbufs[i]);
2182 struct zink_surface *transient = zink_transient_surface(ctx->fb_state.cbufs[i]);
2183 if (transient) {
2184 assert(zink_resource(transient->base.texture)->obj->vkusage == ctx->framebuffer->state.infos[i].usage);
2185 assert(zink_resource(surf->base.texture)->obj->vkusage == ctx->framebuffer->state.infos[cresolve_offset].usage);
2186 } else {
2187 assert(zink_resource(surf->base.texture)->obj->vkusage == ctx->framebuffer->state.infos[i].usage);
2188 }
2189 }
2190 }
2191 if (ctx->fb_state.zsbuf) {
2192 struct zink_surface *surf = zink_csurface(ctx->fb_state.zsbuf);
2193 struct zink_surface *transient = zink_transient_surface(ctx->fb_state.zsbuf);
2194 if (transient) {
2195 assert(zink_resource(transient->base.texture)->obj->vkusage == ctx->framebuffer->state.infos[ctx->fb_state.nr_cbufs].usage);
2196 assert(zink_resource(surf->base.texture)->obj->vkusage == ctx->framebuffer->state.infos[cresolve_offset].usage);
2197 } else {
2198 assert(zink_resource(surf->base.texture)->obj->vkusage == ctx->framebuffer->state.infos[ctx->fb_state.nr_cbufs].usage);
2199 }
2200 }
2201 #endif
2202 rpbi.pNext = &infos;
2203 }
2204
2205 VKCTX(CmdBeginRenderPass)(batch->state->cmdbuf, &rpbi, VK_SUBPASS_CONTENTS_INLINE);
2206 batch->in_rp = true;
2207 ctx->new_swapchain = false;
2208 return clear_buffers;
2209 }
2210
2211 void
zink_init_vk_sample_locations(struct zink_context * ctx,VkSampleLocationsInfoEXT * loc)2212 zink_init_vk_sample_locations(struct zink_context *ctx, VkSampleLocationsInfoEXT *loc)
2213 {
2214 struct zink_screen *screen = zink_screen(ctx->base.screen);
2215 unsigned idx = util_logbase2_ceil(MAX2(ctx->gfx_pipeline_state.rast_samples + 1, 1));
2216 loc->sType = VK_STRUCTURE_TYPE_SAMPLE_LOCATIONS_INFO_EXT;
2217 loc->pNext = NULL;
2218 loc->sampleLocationsPerPixel = 1 << idx;
2219 loc->sampleLocationsCount = ctx->gfx_pipeline_state.rast_samples + 1;
2220 loc->sampleLocationGridSize = screen->maxSampleLocationGridSize[idx];
2221 loc->pSampleLocations = ctx->vk_sample_locations;
2222 }
2223
2224 static void
zink_evaluate_depth_buffer(struct pipe_context * pctx)2225 zink_evaluate_depth_buffer(struct pipe_context *pctx)
2226 {
2227 struct zink_context *ctx = zink_context(pctx);
2228
2229 if (!ctx->fb_state.zsbuf)
2230 return;
2231
2232 struct zink_resource *res = zink_resource(ctx->fb_state.zsbuf->texture);
2233 res->obj->needs_zs_evaluate = true;
2234 zink_init_vk_sample_locations(ctx, &res->obj->zs_evaluate);
2235 zink_batch_no_rp(ctx);
2236 }
2237
2238 void
zink_begin_render_pass(struct zink_context * ctx)2239 zink_begin_render_pass(struct zink_context *ctx)
2240 {
2241 setup_framebuffer(ctx);
2242 /* TODO: need replicate EXT */
2243 if (ctx->framebuffer->rp->state.msaa_expand_mask) {
2244 uint32_t rp_state = ctx->gfx_pipeline_state.rp_state;
2245 struct zink_render_pass *rp = ctx->gfx_pipeline_state.render_pass;
2246
2247 u_foreach_bit(i, ctx->framebuffer->rp->state.msaa_expand_mask) {
2248 struct zink_ctx_surface *csurf = (struct zink_ctx_surface*)ctx->fb_state.cbufs[i];
2249 if (csurf->transient_init)
2250 continue;
2251 struct pipe_surface *dst_view = (struct pipe_surface*)csurf->transient;
2252 assert(dst_view);
2253 struct pipe_sampler_view src_templ, *src_view;
2254 struct pipe_resource *src = ctx->fb_state.cbufs[i]->texture;
2255 struct pipe_box dstbox;
2256
2257 u_box_3d(0, 0, 0, ctx->fb_state.width, ctx->fb_state.height,
2258 1 + dst_view->u.tex.last_layer - dst_view->u.tex.first_layer, &dstbox);
2259
2260 util_blitter_default_src_texture(ctx->blitter, &src_templ, src, ctx->fb_state.cbufs[i]->u.tex.level);
2261 src_view = ctx->base.create_sampler_view(&ctx->base, src, &src_templ);
2262
2263 zink_blit_begin(ctx, ZINK_BLIT_SAVE_FB | ZINK_BLIT_SAVE_FS | ZINK_BLIT_SAVE_TEXTURES);
2264 util_blitter_blit_generic(ctx->blitter, dst_view, &dstbox,
2265 src_view, &dstbox, ctx->fb_state.width, ctx->fb_state.height,
2266 PIPE_MASK_RGBAZS, PIPE_TEX_FILTER_NEAREST, NULL,
2267 false, false);
2268
2269 pipe_sampler_view_reference(&src_view, NULL);
2270 csurf->transient_init = true;
2271 }
2272 ctx->fb_changed = ctx->rp_changed = false;
2273 ctx->gfx_pipeline_state.rp_state = rp_state;
2274 ctx->gfx_pipeline_state.render_pass = rp;
2275 }
2276 assert(ctx->gfx_pipeline_state.render_pass);
2277 unsigned clear_buffers = begin_render_pass(ctx);
2278
2279 if (ctx->render_condition.query)
2280 zink_start_conditional_render(ctx);
2281 zink_clear_framebuffer(ctx, clear_buffers);
2282 }
2283
2284 void
zink_end_render_pass(struct zink_context * ctx)2285 zink_end_render_pass(struct zink_context *ctx)
2286 {
2287 if (ctx->batch.in_rp) {
2288 if (ctx->render_condition.query)
2289 zink_stop_conditional_render(ctx);
2290 VKCTX(CmdEndRenderPass)(ctx->batch.state->cmdbuf);
2291 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2292 struct zink_ctx_surface *csurf = (struct zink_ctx_surface*)ctx->fb_state.cbufs[i];
2293 if (csurf)
2294 csurf->transient_init = true;
2295 }
2296 }
2297 ctx->batch.in_rp = false;
2298 }
2299
2300 static void
sync_flush(struct zink_context * ctx,struct zink_batch_state * bs)2301 sync_flush(struct zink_context *ctx, struct zink_batch_state *bs)
2302 {
2303 if (zink_screen(ctx->base.screen)->threaded)
2304 util_queue_fence_wait(&bs->flush_completed);
2305 }
2306
2307 static inline VkAccessFlags
get_access_flags_for_binding(struct zink_context * ctx,enum zink_descriptor_type type,enum pipe_shader_type stage,unsigned idx)2308 get_access_flags_for_binding(struct zink_context *ctx, enum zink_descriptor_type type, enum pipe_shader_type stage, unsigned idx)
2309 {
2310 VkAccessFlags flags = 0;
2311 switch (type) {
2312 case ZINK_DESCRIPTOR_TYPE_UBO:
2313 return VK_ACCESS_UNIFORM_READ_BIT;
2314 case ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW:
2315 return VK_ACCESS_SHADER_READ_BIT;
2316 case ZINK_DESCRIPTOR_TYPE_SSBO: {
2317 flags = VK_ACCESS_SHADER_READ_BIT;
2318 if (ctx->writable_ssbos[stage] & (1 << idx))
2319 flags |= VK_ACCESS_SHADER_WRITE_BIT;
2320 return flags;
2321 }
2322 case ZINK_DESCRIPTOR_TYPE_IMAGE: {
2323 struct zink_image_view *image_view = &ctx->image_views[stage][idx];
2324 if (image_view->base.access & PIPE_IMAGE_ACCESS_READ)
2325 flags |= VK_ACCESS_SHADER_READ_BIT;
2326 if (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE)
2327 flags |= VK_ACCESS_SHADER_WRITE_BIT;
2328 return flags;
2329 }
2330 default:
2331 break;
2332 }
2333 unreachable("ACK");
2334 return 0;
2335 }
2336
2337 static void
update_resource_refs_for_stage(struct zink_context * ctx,enum pipe_shader_type stage)2338 update_resource_refs_for_stage(struct zink_context *ctx, enum pipe_shader_type stage)
2339 {
2340 struct zink_batch *batch = &ctx->batch;
2341 unsigned max_slot[] = {
2342 [ZINK_DESCRIPTOR_TYPE_UBO] = ctx->di.num_ubos[stage],
2343 [ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW] = ctx->di.num_samplers[stage],
2344 [ZINK_DESCRIPTOR_TYPE_SSBO] = ctx->di.num_ssbos[stage],
2345 [ZINK_DESCRIPTOR_TYPE_IMAGE] = ctx->di.num_images[stage]
2346 };
2347 for (unsigned i = 0; i < ZINK_DESCRIPTOR_TYPES; i++) {
2348 for (unsigned j = 0; j < max_slot[i]; j++) {
2349 if (ctx->di.descriptor_res[i][stage][j]) {
2350 struct zink_resource *res = ctx->di.descriptor_res[i][stage][j];
2351 if (!res)
2352 continue;
2353 bool is_write = zink_resource_access_is_write(get_access_flags_for_binding(ctx, i, stage, j));
2354 zink_batch_resource_usage_set(batch, res, is_write);
2355
2356 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[stage][j]);
2357 struct zink_sampler_state *sampler_state = ctx->sampler_states[stage][j];
2358 struct zink_image_view *iv = &ctx->image_views[stage][j];
2359 if (sampler_state && i == ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW && j <= ctx->di.num_samplers[stage])
2360 zink_batch_usage_set(&sampler_state->batch_uses, ctx->batch.state);
2361 if (sv && i == ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW && j <= ctx->di.num_sampler_views[stage]) {
2362 if (res->obj->is_buffer)
2363 zink_batch_usage_set(&sv->buffer_view->batch_uses, ctx->batch.state);
2364 else
2365 zink_batch_usage_set(&sv->image_view->batch_uses, ctx->batch.state);
2366 zink_batch_reference_sampler_view(batch, sv);
2367 } else if (i == ZINK_DESCRIPTOR_TYPE_IMAGE && j <= ctx->di.num_images[stage]) {
2368 if (res->obj->is_buffer)
2369 zink_batch_usage_set(&iv->buffer_view->batch_uses, ctx->batch.state);
2370 else
2371 zink_batch_usage_set(&iv->surface->batch_uses, ctx->batch.state);
2372 zink_batch_reference_image_view(batch, iv);
2373 }
2374 }
2375 }
2376 }
2377 }
2378
2379 void
zink_update_descriptor_refs(struct zink_context * ctx,bool compute)2380 zink_update_descriptor_refs(struct zink_context *ctx, bool compute)
2381 {
2382 struct zink_batch *batch = &ctx->batch;
2383 if (compute) {
2384 update_resource_refs_for_stage(ctx, PIPE_SHADER_COMPUTE);
2385 if (ctx->curr_compute)
2386 zink_batch_reference_program(batch, &ctx->curr_compute->base);
2387 } else {
2388 for (unsigned i = 0; i < ZINK_SHADER_COUNT; i++)
2389 update_resource_refs_for_stage(ctx, i);
2390 unsigned vertex_buffers_enabled_mask = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask;
2391 unsigned last_vbo = util_last_bit(vertex_buffers_enabled_mask);
2392 for (unsigned i = 0; i < last_vbo + 1; i++) {
2393 if (ctx->vertex_buffers[i].buffer.resource)
2394 zink_batch_resource_usage_set(batch, zink_resource(ctx->vertex_buffers[i].buffer.resource), false);
2395 }
2396 if (ctx->curr_program)
2397 zink_batch_reference_program(batch, &ctx->curr_program->base);
2398 }
2399 if (ctx->di.bindless_refs_dirty) {
2400 ctx->di.bindless_refs_dirty = false;
2401 for (unsigned i = 0; i < 2; i++) {
2402 util_dynarray_foreach(&ctx->di.bindless[i].resident, struct zink_bindless_descriptor*, bd) {
2403 struct zink_resource *res = zink_descriptor_surface_resource(&(*bd)->ds);
2404 zink_batch_resource_usage_set(&ctx->batch, res, (*bd)->access & PIPE_IMAGE_ACCESS_WRITE);
2405 }
2406 }
2407 }
2408 }
2409
2410 static void
stall(struct zink_context * ctx)2411 stall(struct zink_context *ctx)
2412 {
2413 sync_flush(ctx, zink_batch_state(ctx->last_fence));
2414 zink_vkfence_wait(zink_screen(ctx->base.screen), ctx->last_fence, PIPE_TIMEOUT_INFINITE);
2415 zink_batch_reset_all(ctx);
2416 }
2417
2418 static void
flush_batch(struct zink_context * ctx,bool sync)2419 flush_batch(struct zink_context *ctx, bool sync)
2420 {
2421 struct zink_batch *batch = &ctx->batch;
2422 if (ctx->clears_enabled)
2423 /* start rp to do all the clears */
2424 zink_begin_render_pass(ctx);
2425 zink_end_render_pass(ctx);
2426 zink_end_batch(ctx, batch);
2427 ctx->deferred_fence = NULL;
2428
2429 if (sync)
2430 sync_flush(ctx, ctx->batch.state);
2431
2432 if (ctx->batch.state->is_device_lost) {
2433 check_device_lost(ctx);
2434 } else {
2435 zink_start_batch(ctx, batch);
2436 if (zink_screen(ctx->base.screen)->info.have_EXT_transform_feedback && ctx->num_so_targets)
2437 ctx->dirty_so_targets = true;
2438 ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
2439 zink_select_draw_vbo(ctx);
2440 zink_select_launch_grid(ctx);
2441
2442 if (ctx->oom_stall)
2443 stall(ctx);
2444 ctx->oom_flush = false;
2445 ctx->oom_stall = false;
2446 ctx->dd->bindless_bound = false;
2447 ctx->di.bindless_refs_dirty = true;
2448 }
2449 }
2450
2451 void
zink_flush_queue(struct zink_context * ctx)2452 zink_flush_queue(struct zink_context *ctx)
2453 {
2454 flush_batch(ctx, true);
2455 }
2456
2457 static bool
rebind_fb_surface(struct zink_context * ctx,struct pipe_surface ** surf,struct zink_resource * match_res)2458 rebind_fb_surface(struct zink_context *ctx, struct pipe_surface **surf, struct zink_resource *match_res)
2459 {
2460 if (!*surf)
2461 return false;
2462 struct zink_resource *surf_res = zink_resource((*surf)->texture);
2463 if ((match_res == surf_res) || surf_res->obj != zink_csurface(*surf)->obj)
2464 return zink_rebind_ctx_surface(ctx, surf);
2465 return false;
2466 }
2467
2468 static bool
rebind_fb_state(struct zink_context * ctx,struct zink_resource * match_res,bool from_set_fb)2469 rebind_fb_state(struct zink_context *ctx, struct zink_resource *match_res, bool from_set_fb)
2470 {
2471 bool rebind = false;
2472 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2473 rebind |= rebind_fb_surface(ctx, &ctx->fb_state.cbufs[i], match_res);
2474 if (from_set_fb && ctx->fb_state.cbufs[i] && ctx->fb_state.cbufs[i]->texture->bind & PIPE_BIND_SCANOUT)
2475 ctx->new_swapchain = true;
2476 }
2477 rebind |= rebind_fb_surface(ctx, &ctx->fb_state.zsbuf, match_res);
2478 return rebind;
2479 }
2480
2481 static void
unbind_fb_surface(struct zink_context * ctx,struct pipe_surface * surf,bool changed)2482 unbind_fb_surface(struct zink_context *ctx, struct pipe_surface *surf, bool changed)
2483 {
2484 if (!surf)
2485 return;
2486 struct zink_surface *transient = zink_transient_surface(surf);
2487 if (changed) {
2488 zink_fb_clears_apply(ctx, surf->texture);
2489 if (zink_batch_usage_exists(zink_csurface(surf)->batch_uses)) {
2490 zink_batch_reference_surface(&ctx->batch, zink_csurface(surf));
2491 if (transient)
2492 zink_batch_reference_surface(&ctx->batch, transient);
2493 }
2494 ctx->rp_changed = true;
2495 }
2496 struct zink_resource *res = zink_resource(surf->texture);
2497 res->fb_binds--;
2498 if (!res->fb_binds)
2499 check_resource_for_batch_ref(ctx, res);
2500 }
2501
2502 static void
zink_set_framebuffer_state(struct pipe_context * pctx,const struct pipe_framebuffer_state * state)2503 zink_set_framebuffer_state(struct pipe_context *pctx,
2504 const struct pipe_framebuffer_state *state)
2505 {
2506 struct zink_context *ctx = zink_context(pctx);
2507 unsigned samples = state->nr_cbufs || state->zsbuf ? 0 : state->samples;
2508
2509 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2510 struct pipe_surface *surf = ctx->fb_state.cbufs[i];
2511 if (i < state->nr_cbufs)
2512 ctx->rp_changed |= !!zink_transient_surface(surf) != !!zink_transient_surface(state->cbufs[i]);
2513 unbind_fb_surface(ctx, surf, i >= state->nr_cbufs || surf != state->cbufs[i]);
2514 }
2515 if (ctx->fb_state.zsbuf) {
2516 struct pipe_surface *surf = ctx->fb_state.zsbuf;
2517 struct zink_resource *res = zink_resource(surf->texture);
2518 bool changed = surf != state->zsbuf;
2519 unbind_fb_surface(ctx, surf, changed);
2520 if (!changed)
2521 ctx->rp_changed |= !!zink_transient_surface(surf) != !!zink_transient_surface(state->zsbuf);
2522 if (changed && unlikely(res->obj->needs_zs_evaluate))
2523 /* have to flush zs eval while the sample location data still exists,
2524 * so just throw some random barrier */
2525 zink_resource_image_barrier(ctx, res, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
2526 VK_ACCESS_SHADER_READ_BIT, VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
2527 }
2528 /* renderpass changes if the number or types of attachments change */
2529 ctx->rp_changed |= ctx->fb_state.nr_cbufs != state->nr_cbufs;
2530 ctx->rp_changed |= !!ctx->fb_state.zsbuf != !!state->zsbuf;
2531
2532 unsigned w = ctx->fb_state.width;
2533 unsigned h = ctx->fb_state.height;
2534
2535 util_copy_framebuffer_state(&ctx->fb_state, state);
2536 zink_update_fbfetch(ctx);
2537 unsigned prev_void_alpha_attachments = ctx->gfx_pipeline_state.void_alpha_attachments;
2538 ctx->gfx_pipeline_state.void_alpha_attachments = 0;
2539 for (int i = 0; i < ctx->fb_state.nr_cbufs; i++) {
2540 struct pipe_surface *surf = ctx->fb_state.cbufs[i];
2541 if (surf) {
2542 struct zink_surface *transient = zink_transient_surface(surf);
2543 if (!samples)
2544 samples = MAX3(transient ? transient->base.nr_samples : 1, surf->texture->nr_samples, 1);
2545 zink_resource(surf->texture)->fb_binds++;
2546 ctx->gfx_pipeline_state.void_alpha_attachments |= util_format_has_alpha1(surf->format) ? BITFIELD_BIT(i) : 0;
2547 }
2548 }
2549 if (ctx->gfx_pipeline_state.void_alpha_attachments != prev_void_alpha_attachments)
2550 ctx->gfx_pipeline_state.dirty = true;
2551 if (ctx->fb_state.zsbuf) {
2552 struct pipe_surface *surf = ctx->fb_state.zsbuf;
2553 struct zink_surface *transient = zink_transient_surface(surf);
2554 if (!samples)
2555 samples = MAX3(transient ? transient->base.nr_samples : 1, surf->texture->nr_samples, 1);
2556 zink_resource(surf->texture)->fb_binds++;
2557 }
2558 rebind_fb_state(ctx, NULL, true);
2559 ctx->fb_state.samples = MAX2(samples, 1);
2560 update_framebuffer_state(ctx, w, h);
2561
2562 uint8_t rast_samples = ctx->fb_state.samples - 1;
2563 if (rast_samples != ctx->gfx_pipeline_state.rast_samples)
2564 zink_update_fs_key_samples(ctx);
2565 if (ctx->gfx_pipeline_state.rast_samples != rast_samples) {
2566 ctx->sample_locations_changed |= ctx->gfx_pipeline_state.sample_locations_enabled;
2567 ctx->gfx_pipeline_state.dirty = true;
2568 }
2569 ctx->gfx_pipeline_state.rast_samples = rast_samples;
2570
2571 /* need to ensure we start a new rp on next draw */
2572 zink_batch_no_rp(ctx);
2573 /* this is an ideal time to oom flush since it won't split a renderpass */
2574 if (ctx->oom_flush)
2575 flush_batch(ctx, false);
2576 }
2577
2578 static void
zink_set_blend_color(struct pipe_context * pctx,const struct pipe_blend_color * color)2579 zink_set_blend_color(struct pipe_context *pctx,
2580 const struct pipe_blend_color *color)
2581 {
2582 struct zink_context *ctx = zink_context(pctx);
2583 memcpy(ctx->blend_constants, color->color, sizeof(float) * 4);
2584 }
2585
2586 static void
zink_set_sample_mask(struct pipe_context * pctx,unsigned sample_mask)2587 zink_set_sample_mask(struct pipe_context *pctx, unsigned sample_mask)
2588 {
2589 struct zink_context *ctx = zink_context(pctx);
2590 ctx->gfx_pipeline_state.sample_mask = sample_mask;
2591 ctx->gfx_pipeline_state.dirty = true;
2592 }
2593
2594 static void
zink_set_sample_locations(struct pipe_context * pctx,size_t size,const uint8_t * locations)2595 zink_set_sample_locations(struct pipe_context *pctx, size_t size, const uint8_t *locations)
2596 {
2597 struct zink_context *ctx = zink_context(pctx);
2598
2599 ctx->gfx_pipeline_state.sample_locations_enabled = size && locations;
2600 ctx->sample_locations_changed = ctx->gfx_pipeline_state.sample_locations_enabled;
2601 if (size > sizeof(ctx->sample_locations))
2602 size = sizeof(ctx->sample_locations);
2603
2604 if (locations)
2605 memcpy(ctx->sample_locations, locations, size);
2606 }
2607
2608 static VkAccessFlags
access_src_flags(VkImageLayout layout)2609 access_src_flags(VkImageLayout layout)
2610 {
2611 switch (layout) {
2612 case VK_IMAGE_LAYOUT_UNDEFINED:
2613 return 0;
2614
2615 case VK_IMAGE_LAYOUT_GENERAL:
2616 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2617
2618 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
2619 return VK_ACCESS_COLOR_ATTACHMENT_READ_BIT;
2620 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
2621 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT;
2622
2623 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
2624 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
2625 return VK_ACCESS_SHADER_READ_BIT;
2626
2627 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
2628 return VK_ACCESS_TRANSFER_READ_BIT;
2629
2630 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
2631 return VK_ACCESS_TRANSFER_WRITE_BIT;
2632
2633 case VK_IMAGE_LAYOUT_PREINITIALIZED:
2634 return VK_ACCESS_HOST_WRITE_BIT;
2635
2636 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
2637 return 0;
2638
2639 default:
2640 unreachable("unexpected layout");
2641 }
2642 }
2643
2644 static VkAccessFlags
access_dst_flags(VkImageLayout layout)2645 access_dst_flags(VkImageLayout layout)
2646 {
2647 switch (layout) {
2648 case VK_IMAGE_LAYOUT_UNDEFINED:
2649 return 0;
2650
2651 case VK_IMAGE_LAYOUT_GENERAL:
2652 return VK_ACCESS_TRANSFER_READ_BIT | VK_ACCESS_TRANSFER_WRITE_BIT;
2653
2654 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
2655 return VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
2656 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
2657 return VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
2658
2659 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
2660 return VK_ACCESS_SHADER_READ_BIT;
2661
2662 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
2663 return VK_ACCESS_TRANSFER_READ_BIT;
2664
2665 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
2666 return VK_ACCESS_SHADER_READ_BIT;
2667
2668 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
2669 return VK_ACCESS_TRANSFER_WRITE_BIT;
2670
2671 case VK_IMAGE_LAYOUT_PRESENT_SRC_KHR:
2672 return 0;
2673
2674 default:
2675 unreachable("unexpected layout");
2676 }
2677 }
2678
2679 static VkPipelineStageFlags
pipeline_dst_stage(VkImageLayout layout)2680 pipeline_dst_stage(VkImageLayout layout)
2681 {
2682 switch (layout) {
2683 case VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL:
2684 return VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
2685 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL:
2686 return VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT;
2687
2688 case VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL:
2689 return VK_PIPELINE_STAGE_TRANSFER_BIT;
2690 case VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL:
2691 return VK_PIPELINE_STAGE_TRANSFER_BIT;
2692
2693 case VK_IMAGE_LAYOUT_GENERAL:
2694 return VK_PIPELINE_STAGE_ALL_COMMANDS_BIT;
2695
2696 case VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL:
2697 case VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL:
2698 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
2699
2700 default:
2701 return VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT;
2702 }
2703 }
2704
2705 #define ALL_READ_ACCESS_FLAGS \
2706 (VK_ACCESS_INDIRECT_COMMAND_READ_BIT | \
2707 VK_ACCESS_INDEX_READ_BIT | \
2708 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT | \
2709 VK_ACCESS_UNIFORM_READ_BIT | \
2710 VK_ACCESS_INPUT_ATTACHMENT_READ_BIT | \
2711 VK_ACCESS_SHADER_READ_BIT | \
2712 VK_ACCESS_COLOR_ATTACHMENT_READ_BIT | \
2713 VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_READ_BIT | \
2714 VK_ACCESS_TRANSFER_READ_BIT |\
2715 VK_ACCESS_HOST_READ_BIT |\
2716 VK_ACCESS_MEMORY_READ_BIT |\
2717 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT |\
2718 VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT |\
2719 VK_ACCESS_COLOR_ATTACHMENT_READ_NONCOHERENT_BIT_EXT |\
2720 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_KHR |\
2721 VK_ACCESS_SHADING_RATE_IMAGE_READ_BIT_NV |\
2722 VK_ACCESS_FRAGMENT_DENSITY_MAP_READ_BIT_EXT |\
2723 VK_ACCESS_COMMAND_PREPROCESS_READ_BIT_NV |\
2724 VK_ACCESS_ACCELERATION_STRUCTURE_READ_BIT_NV |\
2725 VK_ACCESS_ACCELERATION_STRUCTURE_WRITE_BIT_NV)
2726
2727
2728 bool
zink_resource_access_is_write(VkAccessFlags flags)2729 zink_resource_access_is_write(VkAccessFlags flags)
2730 {
2731 return (flags & ALL_READ_ACCESS_FLAGS) != flags;
2732 }
2733
2734 bool
zink_resource_image_needs_barrier(struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)2735 zink_resource_image_needs_barrier(struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
2736 {
2737 if (!pipeline)
2738 pipeline = pipeline_dst_stage(new_layout);
2739 if (!flags)
2740 flags = access_dst_flags(new_layout);
2741 return res->layout != new_layout || (res->obj->access_stage & pipeline) != pipeline ||
2742 (res->obj->access & flags) != flags ||
2743 zink_resource_access_is_write(res->obj->access) ||
2744 zink_resource_access_is_write(flags);
2745 }
2746
2747 bool
zink_resource_image_barrier_init(VkImageMemoryBarrier * imb,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)2748 zink_resource_image_barrier_init(VkImageMemoryBarrier *imb, struct zink_resource *res, VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
2749 {
2750 if (!pipeline)
2751 pipeline = pipeline_dst_stage(new_layout);
2752 if (!flags)
2753 flags = access_dst_flags(new_layout);
2754
2755 VkImageSubresourceRange isr = {
2756 res->aspect,
2757 0, VK_REMAINING_MIP_LEVELS,
2758 0, VK_REMAINING_ARRAY_LAYERS
2759 };
2760 *imb = (VkImageMemoryBarrier){
2761 VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER,
2762 NULL,
2763 res->obj->access ? res->obj->access : access_src_flags(res->layout),
2764 flags,
2765 res->layout,
2766 new_layout,
2767 VK_QUEUE_FAMILY_IGNORED,
2768 VK_QUEUE_FAMILY_IGNORED,
2769 res->obj->image,
2770 isr
2771 };
2772 return res->obj->needs_zs_evaluate || zink_resource_image_needs_barrier(res, new_layout, flags, pipeline);
2773 }
2774
2775 static inline bool
is_shader_pipline_stage(VkPipelineStageFlags pipeline)2776 is_shader_pipline_stage(VkPipelineStageFlags pipeline)
2777 {
2778 return pipeline & (VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
2779 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
2780 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
2781 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
2782 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT);
2783 }
2784
2785 static void
resource_check_defer_buffer_barrier(struct zink_context * ctx,struct zink_resource * res,VkPipelineStageFlags pipeline)2786 resource_check_defer_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkPipelineStageFlags pipeline)
2787 {
2788 assert(res->obj->is_buffer);
2789 if (res->bind_count[0] - res->so_bind_count > 0) {
2790 if ((res->obj->is_buffer && res->vbo_bind_mask && !(pipeline & VK_PIPELINE_STAGE_VERTEX_INPUT_BIT)) ||
2791 ((!res->obj->is_buffer || util_bitcount(res->vbo_bind_mask) != res->bind_count[0]) && !is_shader_pipline_stage(pipeline)))
2792 /* gfx rebind */
2793 _mesa_set_add(ctx->need_barriers[0], res);
2794 }
2795 if (res->bind_count[1] && !(pipeline & VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT))
2796 /* compute rebind */
2797 _mesa_set_add(ctx->need_barriers[1], res);
2798 }
2799
2800 static inline VkCommandBuffer
get_cmdbuf(struct zink_context * ctx,struct zink_resource * res)2801 get_cmdbuf(struct zink_context *ctx, struct zink_resource *res)
2802 {
2803 if ((res->obj->access && !res->obj->unordered_barrier) || !ctx->batch.in_rp) {
2804 zink_batch_no_rp(ctx);
2805 res->obj->unordered_barrier = false;
2806 return ctx->batch.state->cmdbuf;
2807 }
2808 res->obj->unordered_barrier = true;
2809 ctx->batch.state->has_barriers = true;
2810 return ctx->batch.state->barrier_cmdbuf;
2811 }
2812
2813 static void
resource_check_defer_image_barrier(struct zink_context * ctx,struct zink_resource * res,VkImageLayout layout,VkPipelineStageFlags pipeline)2814 resource_check_defer_image_barrier(struct zink_context *ctx, struct zink_resource *res, VkImageLayout layout, VkPipelineStageFlags pipeline)
2815 {
2816 assert(!res->obj->is_buffer);
2817
2818 bool is_compute = pipeline == VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
2819 /* if this is a non-shader barrier and there are binds, always queue a shader barrier */
2820 bool is_shader = is_shader_pipline_stage(pipeline);
2821 if ((is_shader || !res->bind_count[is_compute]) &&
2822 /* if no layout change is needed between gfx and compute, do nothing */
2823 !res->bind_count[!is_compute] && (!is_compute || !res->fb_binds))
2824 return;
2825
2826 if (res->bind_count[!is_compute] && is_shader) {
2827 /* if the layout is the same between gfx and compute, do nothing */
2828 if (layout == zink_descriptor_util_image_layout_eval(res, !is_compute))
2829 return;
2830 }
2831 /* queue a layout change if a layout change will be needed */
2832 if (res->bind_count[!is_compute])
2833 _mesa_set_add(ctx->need_barriers[!is_compute], res);
2834 /* also queue a layout change if this is a non-shader layout */
2835 if (res->bind_count[is_compute] && !is_shader)
2836 _mesa_set_add(ctx->need_barriers[is_compute], res);
2837 }
2838
2839 void
zink_resource_image_barrier(struct zink_context * ctx,struct zink_resource * res,VkImageLayout new_layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)2840 zink_resource_image_barrier(struct zink_context *ctx, struct zink_resource *res,
2841 VkImageLayout new_layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
2842 {
2843 VkImageMemoryBarrier imb;
2844 if (!pipeline)
2845 pipeline = pipeline_dst_stage(new_layout);
2846
2847 if (!zink_resource_image_barrier_init(&imb, res, new_layout, flags, pipeline))
2848 return;
2849 /* only barrier if we're changing layout or doing something besides read -> read */
2850 VkCommandBuffer cmdbuf = get_cmdbuf(ctx, res);
2851 assert(new_layout);
2852 if (!res->obj->access_stage)
2853 imb.srcAccessMask = 0;
2854 if (res->obj->needs_zs_evaluate)
2855 imb.pNext = &res->obj->zs_evaluate;
2856 res->obj->needs_zs_evaluate = false;
2857 if (res->dmabuf_acquire) {
2858 imb.srcQueueFamilyIndex = VK_QUEUE_FAMILY_FOREIGN_EXT;
2859 imb.dstQueueFamilyIndex = zink_screen(ctx->base.screen)->gfx_queue;
2860 res->dmabuf_acquire = false;
2861 }
2862 VKCTX(CmdPipelineBarrier)(
2863 cmdbuf,
2864 res->obj->access_stage ? res->obj->access_stage : VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT,
2865 pipeline,
2866 0,
2867 0, NULL,
2868 0, NULL,
2869 1, &imb
2870 );
2871
2872 resource_check_defer_image_barrier(ctx, res, new_layout, pipeline);
2873
2874 if (res->obj->unordered_barrier) {
2875 res->obj->access |= imb.dstAccessMask;
2876 res->obj->access_stage |= pipeline;
2877 } else {
2878 res->obj->access = imb.dstAccessMask;
2879 res->obj->access_stage = pipeline;
2880 }
2881 res->layout = new_layout;
2882 }
2883
2884
2885 VkPipelineStageFlags
zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage)2886 zink_pipeline_flags_from_stage(VkShaderStageFlagBits stage)
2887 {
2888 switch (stage) {
2889 case VK_SHADER_STAGE_VERTEX_BIT:
2890 return VK_PIPELINE_STAGE_VERTEX_SHADER_BIT;
2891 case VK_SHADER_STAGE_FRAGMENT_BIT:
2892 return VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
2893 case VK_SHADER_STAGE_GEOMETRY_BIT:
2894 return VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT;
2895 case VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT:
2896 return VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT;
2897 case VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT:
2898 return VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT;
2899 case VK_SHADER_STAGE_COMPUTE_BIT:
2900 return VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
2901 default:
2902 unreachable("unknown shader stage bit");
2903 }
2904 }
2905
2906 ALWAYS_INLINE static VkPipelineStageFlags
pipeline_access_stage(VkAccessFlags flags)2907 pipeline_access_stage(VkAccessFlags flags)
2908 {
2909 if (flags & (VK_ACCESS_UNIFORM_READ_BIT |
2910 VK_ACCESS_SHADER_READ_BIT |
2911 VK_ACCESS_SHADER_WRITE_BIT))
2912 return VK_PIPELINE_STAGE_TASK_SHADER_BIT_NV |
2913 VK_PIPELINE_STAGE_MESH_SHADER_BIT_NV |
2914 VK_PIPELINE_STAGE_RAY_TRACING_SHADER_BIT_KHR |
2915 VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
2916 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
2917 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
2918 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
2919 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT |
2920 VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
2921 return VK_PIPELINE_STAGE_TRANSFER_BIT;
2922 }
2923
2924 ALWAYS_INLINE static bool
zink_resource_buffer_needs_barrier(struct zink_resource * res,VkAccessFlags flags,VkPipelineStageFlags pipeline)2925 zink_resource_buffer_needs_barrier(struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
2926 {
2927 if (!res->obj->access || !res->obj->access_stage)
2928 return true;
2929 if (!pipeline)
2930 pipeline = pipeline_access_stage(flags);
2931 return zink_resource_access_is_write(res->obj->access) ||
2932 zink_resource_access_is_write(flags) ||
2933 ((res->obj->access_stage & pipeline) != pipeline && !(res->obj->access_stage & (pipeline - 1))) ||
2934 (res->obj->access & flags) != flags;
2935 }
2936
2937 void
zink_resource_buffer_barrier(struct zink_context * ctx,struct zink_resource * res,VkAccessFlags flags,VkPipelineStageFlags pipeline)2938 zink_resource_buffer_barrier(struct zink_context *ctx, struct zink_resource *res, VkAccessFlags flags, VkPipelineStageFlags pipeline)
2939 {
2940 VkMemoryBarrier bmb;
2941 if (!pipeline)
2942 pipeline = pipeline_access_stage(flags);
2943 if (!zink_resource_buffer_needs_barrier(res, flags, pipeline))
2944 return;
2945
2946 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
2947 bmb.pNext = NULL;
2948 bmb.srcAccessMask = res->obj->access;
2949 bmb.dstAccessMask = flags;
2950 if (!res->obj->access_stage)
2951 bmb.srcAccessMask = 0;
2952 VkCommandBuffer cmdbuf = get_cmdbuf(ctx, res);
2953 /* only barrier if we're changing layout or doing something besides read -> read */
2954 VKCTX(CmdPipelineBarrier)(
2955 cmdbuf,
2956 res->obj->access_stage ? res->obj->access_stage : pipeline_access_stage(res->obj->access),
2957 pipeline,
2958 0,
2959 1, &bmb,
2960 0, NULL,
2961 0, NULL
2962 );
2963
2964 resource_check_defer_buffer_barrier(ctx, res, pipeline);
2965
2966 if (res->obj->unordered_barrier) {
2967 res->obj->access |= bmb.dstAccessMask;
2968 res->obj->access_stage |= pipeline;
2969 } else {
2970 res->obj->access = bmb.dstAccessMask;
2971 res->obj->access_stage = pipeline;
2972 }
2973 }
2974
2975 bool
zink_resource_needs_barrier(struct zink_resource * res,VkImageLayout layout,VkAccessFlags flags,VkPipelineStageFlags pipeline)2976 zink_resource_needs_barrier(struct zink_resource *res, VkImageLayout layout, VkAccessFlags flags, VkPipelineStageFlags pipeline)
2977 {
2978 if (res->base.b.target == PIPE_BUFFER)
2979 return zink_resource_buffer_needs_barrier(res, flags, pipeline);
2980 return zink_resource_image_needs_barrier(res, layout, flags, pipeline);
2981 }
2982
2983 VkShaderStageFlagBits
zink_shader_stage(enum pipe_shader_type type)2984 zink_shader_stage(enum pipe_shader_type type)
2985 {
2986 VkShaderStageFlagBits stages[] = {
2987 [PIPE_SHADER_VERTEX] = VK_SHADER_STAGE_VERTEX_BIT,
2988 [PIPE_SHADER_FRAGMENT] = VK_SHADER_STAGE_FRAGMENT_BIT,
2989 [PIPE_SHADER_GEOMETRY] = VK_SHADER_STAGE_GEOMETRY_BIT,
2990 [PIPE_SHADER_TESS_CTRL] = VK_SHADER_STAGE_TESSELLATION_CONTROL_BIT,
2991 [PIPE_SHADER_TESS_EVAL] = VK_SHADER_STAGE_TESSELLATION_EVALUATION_BIT,
2992 [PIPE_SHADER_COMPUTE] = VK_SHADER_STAGE_COMPUTE_BIT,
2993 };
2994 return stages[type];
2995 }
2996
2997 static void
zink_flush(struct pipe_context * pctx,struct pipe_fence_handle ** pfence,unsigned flags)2998 zink_flush(struct pipe_context *pctx,
2999 struct pipe_fence_handle **pfence,
3000 unsigned flags)
3001 {
3002 struct zink_context *ctx = zink_context(pctx);
3003 bool deferred = flags & PIPE_FLUSH_DEFERRED;
3004 bool deferred_fence = false;
3005 struct zink_batch *batch = &ctx->batch;
3006 struct zink_fence *fence = NULL;
3007 struct zink_screen *screen = zink_screen(ctx->base.screen);
3008 unsigned submit_count = 0;
3009
3010 /* triggering clears will force has_work */
3011 if (!deferred && ctx->clears_enabled)
3012 /* start rp to do all the clears */
3013 zink_begin_render_pass(ctx);
3014
3015 if (!batch->has_work) {
3016 if (pfence) {
3017 /* reuse last fence */
3018 fence = ctx->last_fence;
3019 }
3020 if (!deferred) {
3021 struct zink_batch_state *last = zink_batch_state(ctx->last_fence);
3022 if (last) {
3023 sync_flush(ctx, last);
3024 if (last->is_device_lost)
3025 check_device_lost(ctx);
3026 }
3027 }
3028 tc_driver_internal_flush_notify(ctx->tc);
3029 } else {
3030 fence = &batch->state->fence;
3031 submit_count = batch->state->submit_count;
3032 if (deferred && !(flags & PIPE_FLUSH_FENCE_FD) && pfence)
3033 deferred_fence = true;
3034 else
3035 flush_batch(ctx, true);
3036 }
3037
3038 if (pfence) {
3039 struct zink_tc_fence *mfence;
3040
3041 if (flags & TC_FLUSH_ASYNC) {
3042 mfence = zink_tc_fence(*pfence);
3043 assert(mfence);
3044 } else {
3045 mfence = zink_create_tc_fence();
3046
3047 screen->base.fence_reference(&screen->base, pfence, NULL);
3048 *pfence = (struct pipe_fence_handle *)mfence;
3049 }
3050
3051 mfence->fence = fence;
3052 if (fence)
3053 mfence->submit_count = submit_count;
3054
3055 if (deferred_fence) {
3056 assert(fence);
3057 mfence->deferred_ctx = pctx;
3058 assert(!ctx->deferred_fence || ctx->deferred_fence == fence);
3059 ctx->deferred_fence = fence;
3060 }
3061
3062 if (!fence || flags & TC_FLUSH_ASYNC) {
3063 if (!util_queue_fence_is_signalled(&mfence->ready))
3064 util_queue_fence_signal(&mfence->ready);
3065 }
3066 }
3067 if (fence) {
3068 if (!(flags & (PIPE_FLUSH_DEFERRED | PIPE_FLUSH_ASYNC)))
3069 sync_flush(ctx, zink_batch_state(fence));
3070
3071 if (flags & PIPE_FLUSH_END_OF_FRAME && !(flags & TC_FLUSH_ASYNC) && !deferred) {
3072 /* if the first frame has not yet occurred, we need an explicit fence here
3073 * in some cases in order to correctly draw the first frame, though it's
3074 * unknown at this time why this is the case
3075 */
3076 if (!ctx->first_frame_done)
3077 zink_vkfence_wait(screen, fence, PIPE_TIMEOUT_INFINITE);
3078 ctx->first_frame_done = true;
3079 }
3080 }
3081 }
3082
3083 void
zink_fence_wait(struct pipe_context * pctx)3084 zink_fence_wait(struct pipe_context *pctx)
3085 {
3086 struct zink_context *ctx = zink_context(pctx);
3087
3088 if (ctx->batch.has_work)
3089 pctx->flush(pctx, NULL, PIPE_FLUSH_HINT_FINISH);
3090 if (ctx->last_fence)
3091 stall(ctx);
3092 }
3093
3094 void
zink_wait_on_batch(struct zink_context * ctx,uint32_t batch_id)3095 zink_wait_on_batch(struct zink_context *ctx, uint32_t batch_id)
3096 {
3097 struct zink_batch_state *bs;
3098 if (!batch_id) {
3099 /* not submitted yet */
3100 flush_batch(ctx, true);
3101 bs = zink_batch_state(ctx->last_fence);
3102 assert(bs);
3103 batch_id = bs->fence.batch_id;
3104 }
3105 assert(batch_id);
3106 if (ctx->have_timelines) {
3107 if (!zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, UINT64_MAX))
3108 check_device_lost(ctx);
3109 return;
3110 }
3111 simple_mtx_lock(&ctx->batch_mtx);
3112 struct zink_fence *fence;
3113
3114 assert(ctx->last_fence);
3115 if (batch_id == zink_batch_state(ctx->last_fence)->fence.batch_id)
3116 fence = ctx->last_fence;
3117 else {
3118 for (bs = ctx->batch_states; bs; bs = bs->next) {
3119 if (bs->fence.batch_id < batch_id)
3120 continue;
3121 if (!bs->fence.batch_id || bs->fence.batch_id > batch_id)
3122 break;
3123 }
3124 if (!bs || bs->fence.batch_id != batch_id) {
3125 simple_mtx_unlock(&ctx->batch_mtx);
3126 /* if we can't find it, it either must have finished already or is on a different context */
3127 if (!zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id)) {
3128 /* if it hasn't finished, it's on another context, so force a flush so there's something to wait on */
3129 ctx->batch.has_work = true;
3130 zink_fence_wait(&ctx->base);
3131 }
3132 return;
3133 }
3134 fence = &bs->fence;
3135 }
3136 simple_mtx_unlock(&ctx->batch_mtx);
3137 assert(fence);
3138 sync_flush(ctx, zink_batch_state(fence));
3139 zink_vkfence_wait(zink_screen(ctx->base.screen), fence, PIPE_TIMEOUT_INFINITE);
3140 }
3141
3142 bool
zink_check_batch_completion(struct zink_context * ctx,uint32_t batch_id,bool have_lock)3143 zink_check_batch_completion(struct zink_context *ctx, uint32_t batch_id, bool have_lock)
3144 {
3145 assert(ctx->batch.state);
3146 if (!batch_id)
3147 /* not submitted yet */
3148 return false;
3149
3150 if (zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id))
3151 return true;
3152
3153 if (ctx->have_timelines) {
3154 bool success = zink_screen_timeline_wait(zink_screen(ctx->base.screen), batch_id, 0);
3155 if (!success)
3156 check_device_lost(ctx);
3157 return success;
3158 }
3159 struct zink_fence *fence;
3160
3161 if (!have_lock)
3162 simple_mtx_lock(&ctx->batch_mtx);
3163
3164 if (ctx->last_fence && batch_id == zink_batch_state(ctx->last_fence)->fence.batch_id)
3165 fence = ctx->last_fence;
3166 else {
3167 struct zink_batch_state *bs;
3168 for (bs = ctx->batch_states; bs; bs = bs->next) {
3169 if (bs->fence.batch_id < batch_id)
3170 continue;
3171 if (!bs->fence.batch_id || bs->fence.batch_id > batch_id)
3172 break;
3173 }
3174 if (!bs || bs->fence.batch_id != batch_id) {
3175 if (!have_lock)
3176 simple_mtx_unlock(&ctx->batch_mtx);
3177 /* return compare against last_finished, since this has info from all contexts */
3178 return zink_screen_check_last_finished(zink_screen(ctx->base.screen), batch_id);
3179 }
3180 fence = &bs->fence;
3181 }
3182 if (!have_lock)
3183 simple_mtx_unlock(&ctx->batch_mtx);
3184 assert(fence);
3185 if (zink_screen(ctx->base.screen)->threaded &&
3186 !util_queue_fence_is_signalled(&zink_batch_state(fence)->flush_completed))
3187 return false;
3188 return zink_vkfence_wait(zink_screen(ctx->base.screen), fence, 0);
3189 }
3190
3191 static void
zink_texture_barrier(struct pipe_context * pctx,unsigned flags)3192 zink_texture_barrier(struct pipe_context *pctx, unsigned flags)
3193 {
3194 struct zink_context *ctx = zink_context(pctx);
3195 if (!ctx->framebuffer || !ctx->framebuffer->state.num_attachments)
3196 return;
3197
3198 zink_batch_no_rp(ctx);
3199 if (ctx->fb_state.zsbuf) {
3200 VkMemoryBarrier dmb;
3201 dmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3202 dmb.pNext = NULL;
3203 dmb.srcAccessMask = VK_ACCESS_DEPTH_STENCIL_ATTACHMENT_WRITE_BIT;
3204 dmb.dstAccessMask = VK_ACCESS_SHADER_READ_BIT;
3205 VKCTX(CmdPipelineBarrier)(
3206 ctx->batch.state->cmdbuf,
3207 VK_PIPELINE_STAGE_EARLY_FRAGMENT_TESTS_BIT | VK_PIPELINE_STAGE_LATE_FRAGMENT_TESTS_BIT,
3208 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
3209 0,
3210 1, &dmb,
3211 0, NULL,
3212 0, NULL
3213 );
3214 }
3215 if (!ctx->fb_state.nr_cbufs)
3216 return;
3217
3218 VkMemoryBarrier bmb;
3219 bmb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3220 bmb.pNext = NULL;
3221 bmb.srcAccessMask = 0;
3222 bmb.dstAccessMask = 0;
3223 bmb.srcAccessMask |= VK_ACCESS_COLOR_ATTACHMENT_WRITE_BIT;
3224 bmb.dstAccessMask |= VK_ACCESS_SHADER_READ_BIT;
3225 VKCTX(CmdPipelineBarrier)(
3226 ctx->batch.state->cmdbuf,
3227 VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT,
3228 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT,
3229 0,
3230 1, &bmb,
3231 0, NULL,
3232 0, NULL
3233 );
3234 }
3235
3236 static inline void
mem_barrier(struct zink_context * ctx,VkPipelineStageFlags src_stage,VkPipelineStageFlags dst_stage,VkAccessFlags src,VkAccessFlags dst)3237 mem_barrier(struct zink_context *ctx, VkPipelineStageFlags src_stage, VkPipelineStageFlags dst_stage, VkAccessFlags src, VkAccessFlags dst)
3238 {
3239 struct zink_batch *batch = &ctx->batch;
3240 VkMemoryBarrier mb;
3241 mb.sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER;
3242 mb.pNext = NULL;
3243 mb.srcAccessMask = src;
3244 mb.dstAccessMask = dst;
3245 zink_end_render_pass(ctx);
3246 VKCTX(CmdPipelineBarrier)(batch->state->cmdbuf, src_stage, dst_stage, 0, 1, &mb, 0, NULL, 0, NULL);
3247 }
3248
3249 void
zink_flush_memory_barrier(struct zink_context * ctx,bool is_compute)3250 zink_flush_memory_barrier(struct zink_context *ctx, bool is_compute)
3251 {
3252 const VkPipelineStageFlags gfx_flags = VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
3253 VK_PIPELINE_STAGE_TESSELLATION_CONTROL_SHADER_BIT |
3254 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
3255 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT |
3256 VK_PIPELINE_STAGE_FRAGMENT_SHADER_BIT;
3257 const VkPipelineStageFlags cs_flags = VK_PIPELINE_STAGE_COMPUTE_SHADER_BIT;
3258 VkPipelineStageFlags src = ctx->batch.last_was_compute ? cs_flags : gfx_flags;
3259 VkPipelineStageFlags dst = is_compute ? cs_flags : gfx_flags;
3260
3261 if (ctx->memory_barrier & (PIPE_BARRIER_TEXTURE | PIPE_BARRIER_SHADER_BUFFER | PIPE_BARRIER_IMAGE))
3262 mem_barrier(ctx, src, dst, VK_ACCESS_SHADER_WRITE_BIT, VK_ACCESS_SHADER_READ_BIT);
3263
3264 if (ctx->memory_barrier & PIPE_BARRIER_CONSTANT_BUFFER)
3265 mem_barrier(ctx, src, dst,
3266 VK_ACCESS_SHADER_WRITE_BIT,
3267 VK_ACCESS_UNIFORM_READ_BIT);
3268
3269 if (!is_compute) {
3270 if (ctx->memory_barrier & PIPE_BARRIER_INDIRECT_BUFFER)
3271 mem_barrier(ctx, src, VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT,
3272 VK_ACCESS_SHADER_WRITE_BIT,
3273 VK_ACCESS_INDIRECT_COMMAND_READ_BIT);
3274 if (ctx->memory_barrier & PIPE_BARRIER_VERTEX_BUFFER)
3275 mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
3276 VK_ACCESS_SHADER_WRITE_BIT,
3277 VK_ACCESS_VERTEX_ATTRIBUTE_READ_BIT);
3278
3279 if (ctx->memory_barrier & PIPE_BARRIER_INDEX_BUFFER)
3280 mem_barrier(ctx, gfx_flags, VK_PIPELINE_STAGE_VERTEX_INPUT_BIT,
3281 VK_ACCESS_SHADER_WRITE_BIT,
3282 VK_ACCESS_INDEX_READ_BIT);
3283 if (ctx->memory_barrier & PIPE_BARRIER_FRAMEBUFFER)
3284 zink_texture_barrier(&ctx->base, 0);
3285 if (ctx->memory_barrier & PIPE_BARRIER_STREAMOUT_BUFFER)
3286 mem_barrier(ctx, VK_PIPELINE_STAGE_VERTEX_SHADER_BIT |
3287 VK_PIPELINE_STAGE_TESSELLATION_EVALUATION_SHADER_BIT |
3288 VK_PIPELINE_STAGE_GEOMETRY_SHADER_BIT,
3289 VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT,
3290 VK_ACCESS_SHADER_READ_BIT,
3291 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT |
3292 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT);
3293 }
3294 ctx->memory_barrier = 0;
3295 }
3296
3297 static void
zink_memory_barrier(struct pipe_context * pctx,unsigned flags)3298 zink_memory_barrier(struct pipe_context *pctx, unsigned flags)
3299 {
3300 struct zink_context *ctx = zink_context(pctx);
3301
3302 flags &= ~PIPE_BARRIER_UPDATE;
3303 if (!flags)
3304 return;
3305
3306 if (flags & PIPE_BARRIER_MAPPED_BUFFER) {
3307 /* TODO: this should flush all persistent buffers in use as I think */
3308 flags &= ~PIPE_BARRIER_MAPPED_BUFFER;
3309 }
3310 ctx->memory_barrier = flags;
3311 }
3312
3313 static void
zink_flush_resource(struct pipe_context * pctx,struct pipe_resource * pres)3314 zink_flush_resource(struct pipe_context *pctx,
3315 struct pipe_resource *pres)
3316 {
3317 struct zink_context *ctx = zink_context(pctx);
3318 /* TODO: this is not futureproof and should be updated once proper
3319 * WSI support is added
3320 */
3321 if (pres->bind & (PIPE_BIND_SHARED | PIPE_BIND_SCANOUT))
3322 pipe_resource_reference(&ctx->batch.state->flush_res, pres);
3323 }
3324
3325 void
zink_copy_buffer(struct zink_context * ctx,struct zink_resource * dst,struct zink_resource * src,unsigned dst_offset,unsigned src_offset,unsigned size)3326 zink_copy_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
3327 unsigned dst_offset, unsigned src_offset, unsigned size)
3328 {
3329 VkBufferCopy region;
3330 region.srcOffset = src_offset;
3331 region.dstOffset = dst_offset;
3332 region.size = size;
3333
3334 struct zink_batch *batch = &ctx->batch;
3335 zink_batch_no_rp(ctx);
3336 zink_batch_reference_resource_rw(batch, src, false);
3337 zink_batch_reference_resource_rw(batch, dst, true);
3338 util_range_add(&dst->base.b, &dst->valid_buffer_range, dst_offset, dst_offset + size);
3339 zink_resource_buffer_barrier(ctx, src, VK_ACCESS_TRANSFER_READ_BIT, 0);
3340 zink_resource_buffer_barrier(ctx, dst, VK_ACCESS_TRANSFER_WRITE_BIT, 0);
3341 VKCTX(CmdCopyBuffer)(batch->state->cmdbuf, src->obj->buffer, dst->obj->buffer, 1, ®ion);
3342 }
3343
3344 void
zink_copy_image_buffer(struct zink_context * ctx,struct zink_resource * dst,struct zink_resource * src,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,unsigned src_level,const struct pipe_box * src_box,enum pipe_map_flags map_flags)3345 zink_copy_image_buffer(struct zink_context *ctx, struct zink_resource *dst, struct zink_resource *src,
3346 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
3347 unsigned src_level, const struct pipe_box *src_box, enum pipe_map_flags map_flags)
3348 {
3349 struct zink_resource *img = dst->base.b.target == PIPE_BUFFER ? src : dst;
3350 struct zink_resource *buf = dst->base.b.target == PIPE_BUFFER ? dst : src;
3351 struct zink_batch *batch = &ctx->batch;
3352 zink_batch_no_rp(ctx);
3353
3354 bool buf2img = buf == src;
3355
3356 if (buf2img) {
3357 zink_resource_image_barrier(ctx, img, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, 0, 0);
3358 zink_resource_buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_READ_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
3359 } else {
3360 zink_resource_image_barrier(ctx, img, VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL, 0, 0);
3361 zink_resource_buffer_barrier(ctx, buf, VK_ACCESS_TRANSFER_WRITE_BIT, VK_PIPELINE_STAGE_TRANSFER_BIT);
3362 util_range_add(&dst->base.b, &dst->valid_buffer_range, dstx, dstx + src_box->width);
3363 }
3364
3365 VkBufferImageCopy region = {0};
3366 region.bufferOffset = buf2img ? src_box->x : dstx;
3367 region.bufferRowLength = 0;
3368 region.bufferImageHeight = 0;
3369 region.imageSubresource.mipLevel = buf2img ? dst_level : src_level;
3370 switch (img->base.b.target) {
3371 case PIPE_TEXTURE_CUBE:
3372 case PIPE_TEXTURE_CUBE_ARRAY:
3373 case PIPE_TEXTURE_2D_ARRAY:
3374 case PIPE_TEXTURE_1D_ARRAY:
3375 /* these use layer */
3376 region.imageSubresource.baseArrayLayer = buf2img ? dstz : src_box->z;
3377 region.imageSubresource.layerCount = src_box->depth;
3378 region.imageOffset.z = 0;
3379 region.imageExtent.depth = 1;
3380 break;
3381 case PIPE_TEXTURE_3D:
3382 /* this uses depth */
3383 region.imageSubresource.baseArrayLayer = 0;
3384 region.imageSubresource.layerCount = 1;
3385 region.imageOffset.z = buf2img ? dstz : src_box->z;
3386 region.imageExtent.depth = src_box->depth;
3387 break;
3388 default:
3389 /* these must only copy one layer */
3390 region.imageSubresource.baseArrayLayer = 0;
3391 region.imageSubresource.layerCount = 1;
3392 region.imageOffset.z = 0;
3393 region.imageExtent.depth = 1;
3394 }
3395 region.imageOffset.x = buf2img ? dstx : src_box->x;
3396 region.imageOffset.y = buf2img ? dsty : src_box->y;
3397
3398 region.imageExtent.width = src_box->width;
3399 region.imageExtent.height = src_box->height;
3400
3401 zink_batch_reference_resource_rw(batch, img, buf2img);
3402 zink_batch_reference_resource_rw(batch, buf, !buf2img);
3403
3404 /* we're using u_transfer_helper_deinterleave, which means we'll be getting PIPE_MAP_* usage
3405 * to indicate whether to copy either the depth or stencil aspects
3406 */
3407 unsigned aspects = 0;
3408 if (map_flags) {
3409 assert((map_flags & (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY)) !=
3410 (PIPE_MAP_DEPTH_ONLY | PIPE_MAP_STENCIL_ONLY));
3411 if (map_flags & PIPE_MAP_DEPTH_ONLY)
3412 aspects = VK_IMAGE_ASPECT_DEPTH_BIT;
3413 else if (map_flags & PIPE_MAP_STENCIL_ONLY)
3414 aspects = VK_IMAGE_ASPECT_STENCIL_BIT;
3415 }
3416 if (!aspects)
3417 aspects = img->aspect;
3418 while (aspects) {
3419 int aspect = 1 << u_bit_scan(&aspects);
3420 region.imageSubresource.aspectMask = aspect;
3421
3422 /* this may or may not work with multisampled depth/stencil buffers depending on the driver implementation:
3423 *
3424 * srcImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
3425 * - vkCmdCopyImageToBuffer spec
3426 *
3427 * dstImage must have a sample count equal to VK_SAMPLE_COUNT_1_BIT
3428 * - vkCmdCopyBufferToImage spec
3429 */
3430 if (buf2img)
3431 VKCTX(CmdCopyBufferToImage)(batch->state->cmdbuf, buf->obj->buffer, img->obj->image, img->layout, 1, ®ion);
3432 else
3433 VKCTX(CmdCopyImageToBuffer)(batch->state->cmdbuf, img->obj->image, img->layout, buf->obj->buffer, 1, ®ion);
3434 }
3435 }
3436
3437 static void
zink_resource_copy_region(struct pipe_context * pctx,struct pipe_resource * pdst,unsigned dst_level,unsigned dstx,unsigned dsty,unsigned dstz,struct pipe_resource * psrc,unsigned src_level,const struct pipe_box * src_box)3438 zink_resource_copy_region(struct pipe_context *pctx,
3439 struct pipe_resource *pdst,
3440 unsigned dst_level, unsigned dstx, unsigned dsty, unsigned dstz,
3441 struct pipe_resource *psrc,
3442 unsigned src_level, const struct pipe_box *src_box)
3443 {
3444 struct zink_resource *dst = zink_resource(pdst);
3445 struct zink_resource *src = zink_resource(psrc);
3446 struct zink_context *ctx = zink_context(pctx);
3447 if (dst->base.b.target != PIPE_BUFFER && src->base.b.target != PIPE_BUFFER) {
3448 VkImageCopy region = {0};
3449 if (util_format_get_num_planes(src->base.b.format) == 1 &&
3450 util_format_get_num_planes(dst->base.b.format) == 1) {
3451 /* If neither the calling command’s srcImage nor the calling command’s dstImage
3452 * has a multi-planar image format then the aspectMask member of srcSubresource
3453 * and dstSubresource must match
3454 *
3455 * -VkImageCopy spec
3456 */
3457 assert(src->aspect == dst->aspect);
3458 } else
3459 unreachable("planar formats not yet handled");
3460
3461 zink_fb_clears_apply_or_discard(ctx, pdst, (struct u_rect){dstx, dstx + src_box->width, dsty, dsty + src_box->height}, false);
3462 zink_fb_clears_apply_region(ctx, psrc, zink_rect_from_box(src_box));
3463
3464 region.srcSubresource.aspectMask = src->aspect;
3465 region.srcSubresource.mipLevel = src_level;
3466 switch (src->base.b.target) {
3467 case PIPE_TEXTURE_CUBE:
3468 case PIPE_TEXTURE_CUBE_ARRAY:
3469 case PIPE_TEXTURE_2D_ARRAY:
3470 case PIPE_TEXTURE_1D_ARRAY:
3471 /* these use layer */
3472 region.srcSubresource.baseArrayLayer = src_box->z;
3473 region.srcSubresource.layerCount = src_box->depth;
3474 region.srcOffset.z = 0;
3475 region.extent.depth = 1;
3476 break;
3477 case PIPE_TEXTURE_3D:
3478 /* this uses depth */
3479 region.srcSubresource.baseArrayLayer = 0;
3480 region.srcSubresource.layerCount = 1;
3481 region.srcOffset.z = src_box->z;
3482 region.extent.depth = src_box->depth;
3483 break;
3484 default:
3485 /* these must only copy one layer */
3486 region.srcSubresource.baseArrayLayer = 0;
3487 region.srcSubresource.layerCount = 1;
3488 region.srcOffset.z = 0;
3489 region.extent.depth = 1;
3490 }
3491
3492 region.srcOffset.x = src_box->x;
3493 region.srcOffset.y = src_box->y;
3494
3495 region.dstSubresource.aspectMask = dst->aspect;
3496 region.dstSubresource.mipLevel = dst_level;
3497 switch (dst->base.b.target) {
3498 case PIPE_TEXTURE_CUBE:
3499 case PIPE_TEXTURE_CUBE_ARRAY:
3500 case PIPE_TEXTURE_2D_ARRAY:
3501 case PIPE_TEXTURE_1D_ARRAY:
3502 /* these use layer */
3503 region.dstSubresource.baseArrayLayer = dstz;
3504 region.dstSubresource.layerCount = src_box->depth;
3505 region.dstOffset.z = 0;
3506 break;
3507 case PIPE_TEXTURE_3D:
3508 /* this uses depth */
3509 region.dstSubresource.baseArrayLayer = 0;
3510 region.dstSubresource.layerCount = 1;
3511 region.dstOffset.z = dstz;
3512 break;
3513 default:
3514 /* these must only copy one layer */
3515 region.dstSubresource.baseArrayLayer = 0;
3516 region.dstSubresource.layerCount = 1;
3517 region.dstOffset.z = 0;
3518 }
3519
3520 region.dstOffset.x = dstx;
3521 region.dstOffset.y = dsty;
3522 region.extent.width = src_box->width;
3523 region.extent.height = src_box->height;
3524
3525 struct zink_batch *batch = &ctx->batch;
3526 zink_batch_no_rp(ctx);
3527 zink_batch_reference_resource_rw(batch, src, false);
3528 zink_batch_reference_resource_rw(batch, dst, true);
3529
3530 zink_resource_setup_transfer_layouts(ctx, src, dst);
3531 VKCTX(CmdCopyImage)(batch->state->cmdbuf, src->obj->image, src->layout,
3532 dst->obj->image, dst->layout,
3533 1, ®ion);
3534 } else if (dst->base.b.target == PIPE_BUFFER &&
3535 src->base.b.target == PIPE_BUFFER) {
3536 zink_copy_buffer(ctx, dst, src, dstx, src_box->x, src_box->width);
3537 } else
3538 zink_copy_image_buffer(ctx, dst, src, dst_level, dstx, dsty, dstz, src_level, src_box, 0);
3539 }
3540
3541 static struct pipe_stream_output_target *
zink_create_stream_output_target(struct pipe_context * pctx,struct pipe_resource * pres,unsigned buffer_offset,unsigned buffer_size)3542 zink_create_stream_output_target(struct pipe_context *pctx,
3543 struct pipe_resource *pres,
3544 unsigned buffer_offset,
3545 unsigned buffer_size)
3546 {
3547 struct zink_so_target *t;
3548 t = CALLOC_STRUCT(zink_so_target);
3549 if (!t)
3550 return NULL;
3551
3552 /* using PIPE_BIND_CUSTOM here lets us create a custom pipe buffer resource,
3553 * which allows us to differentiate and use VK_BUFFER_USAGE_TRANSFORM_FEEDBACK_COUNTER_BUFFER_BIT_EXT
3554 * as we must for this case
3555 */
3556 t->counter_buffer = pipe_buffer_create(pctx->screen, PIPE_BIND_STREAM_OUTPUT | PIPE_BIND_CUSTOM, PIPE_USAGE_DEFAULT, 4);
3557 if (!t->counter_buffer) {
3558 FREE(t);
3559 return NULL;
3560 }
3561
3562 t->base.reference.count = 1;
3563 t->base.context = pctx;
3564 pipe_resource_reference(&t->base.buffer, pres);
3565 t->base.buffer_offset = buffer_offset;
3566 t->base.buffer_size = buffer_size;
3567
3568 zink_resource(t->base.buffer)->so_valid = true;
3569
3570 return &t->base;
3571 }
3572
3573 static void
zink_stream_output_target_destroy(struct pipe_context * pctx,struct pipe_stream_output_target * psot)3574 zink_stream_output_target_destroy(struct pipe_context *pctx,
3575 struct pipe_stream_output_target *psot)
3576 {
3577 struct zink_so_target *t = (struct zink_so_target *)psot;
3578 pipe_resource_reference(&t->counter_buffer, NULL);
3579 pipe_resource_reference(&t->base.buffer, NULL);
3580 FREE(t);
3581 }
3582
3583 static void
zink_set_stream_output_targets(struct pipe_context * pctx,unsigned num_targets,struct pipe_stream_output_target ** targets,const unsigned * offsets)3584 zink_set_stream_output_targets(struct pipe_context *pctx,
3585 unsigned num_targets,
3586 struct pipe_stream_output_target **targets,
3587 const unsigned *offsets)
3588 {
3589 struct zink_context *ctx = zink_context(pctx);
3590
3591 if (num_targets == 0) {
3592 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
3593 if (ctx->so_targets[i]) {
3594 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
3595 if (so) {
3596 so->so_bind_count--;
3597 update_res_bind_count(ctx, so, false, true);
3598 }
3599 }
3600 pipe_so_target_reference(&ctx->so_targets[i], NULL);
3601 }
3602 ctx->num_so_targets = 0;
3603 } else {
3604 for (unsigned i = 0; i < num_targets; i++) {
3605 struct zink_so_target *t = zink_so_target(targets[i]);
3606 pipe_so_target_reference(&ctx->so_targets[i], targets[i]);
3607 if (!t)
3608 continue;
3609 struct zink_resource *res = zink_resource(t->counter_buffer);
3610 if (offsets[0] == (unsigned)-1)
3611 ctx->xfb_barrier |= zink_resource_buffer_needs_barrier(res,
3612 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_READ_BIT_EXT,
3613 VK_PIPELINE_STAGE_DRAW_INDIRECT_BIT);
3614 else
3615 ctx->xfb_barrier |= zink_resource_buffer_needs_barrier(res,
3616 VK_ACCESS_TRANSFORM_FEEDBACK_COUNTER_WRITE_BIT_EXT,
3617 VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT);
3618 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
3619 if (so) {
3620 so->so_bind_count++;
3621 update_res_bind_count(ctx, so, false, false);
3622 }
3623 }
3624 for (unsigned i = num_targets; i < ctx->num_so_targets; i++) {
3625 if (ctx->so_targets[i]) {
3626 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
3627 if (so) {
3628 so->so_bind_count--;
3629 update_res_bind_count(ctx, so, false, true);
3630 }
3631 }
3632 pipe_so_target_reference(&ctx->so_targets[i], NULL);
3633 }
3634 ctx->num_so_targets = num_targets;
3635
3636 /* TODO: possibly avoid rebinding on resume if resuming from same buffers? */
3637 ctx->dirty_so_targets = true;
3638 }
3639 }
3640
3641 void
zink_rebind_framebuffer(struct zink_context * ctx,struct zink_resource * res)3642 zink_rebind_framebuffer(struct zink_context *ctx, struct zink_resource *res)
3643 {
3644 if (!ctx->framebuffer)
3645 return;
3646 bool did_rebind = false;
3647 if (res->aspect & VK_IMAGE_ASPECT_COLOR_BIT) {
3648 for (unsigned i = 0; i < ctx->fb_state.nr_cbufs; i++) {
3649 if (!ctx->fb_state.cbufs[i] ||
3650 zink_resource(ctx->fb_state.cbufs[i]->texture) != res)
3651 continue;
3652 zink_rebind_ctx_surface(ctx, &ctx->fb_state.cbufs[i]);
3653 did_rebind = true;
3654 }
3655 } else {
3656 if (ctx->fb_state.zsbuf && zink_resource(ctx->fb_state.zsbuf->texture) != res) {
3657 zink_rebind_ctx_surface(ctx, &ctx->fb_state.zsbuf);
3658 did_rebind = true;
3659 }
3660 }
3661
3662 did_rebind |= rebind_fb_state(ctx, res, false);
3663
3664 if (!did_rebind)
3665 return;
3666
3667 zink_batch_no_rp(ctx);
3668 if (zink_screen(ctx->base.screen)->info.have_KHR_imageless_framebuffer) {
3669 struct zink_framebuffer *fb = ctx->get_framebuffer(ctx);
3670 ctx->fb_changed |= ctx->framebuffer != fb;
3671 ctx->framebuffer = fb;
3672 }
3673 }
3674
3675 ALWAYS_INLINE static struct zink_resource *
rebind_ubo(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot)3676 rebind_ubo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot)
3677 {
3678 struct zink_resource *res = update_descriptor_state_ubo(ctx, shader, slot,
3679 ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_UBO][shader][slot]);
3680 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_UBO, slot, 1);
3681 return res;
3682 }
3683
3684 ALWAYS_INLINE static struct zink_resource *
rebind_ssbo(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot)3685 rebind_ssbo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot)
3686 {
3687 const struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
3688 struct zink_resource *res = zink_resource(ssbo->buffer);
3689 if (!res)
3690 return NULL;
3691 util_range_add(&res->base.b, &res->valid_buffer_range, ssbo->buffer_offset,
3692 ssbo->buffer_offset + ssbo->buffer_size);
3693 update_descriptor_state_ssbo(ctx, shader, slot, res);
3694 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SSBO, slot, 1);
3695 return res;
3696 }
3697
3698 ALWAYS_INLINE static struct zink_resource *
rebind_tbo(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot)3699 rebind_tbo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot)
3700 {
3701 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
3702 if (!sampler_view || sampler_view->base.texture->target != PIPE_BUFFER)
3703 return NULL;
3704 struct zink_resource *res = zink_resource(sampler_view->base.texture);
3705 if (zink_batch_usage_exists(sampler_view->buffer_view->batch_uses))
3706 zink_batch_reference_bufferview(&ctx->batch, sampler_view->buffer_view);
3707 VkBufferViewCreateInfo bvci = sampler_view->buffer_view->bvci;
3708 bvci.buffer = res->obj->buffer;
3709 zink_buffer_view_reference(zink_screen(ctx->base.screen), &sampler_view->buffer_view, NULL);
3710 sampler_view->buffer_view = get_buffer_view(ctx, res, &bvci);
3711 update_descriptor_state_sampler(ctx, shader, slot, res);
3712 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, slot, 1);
3713 return res;
3714 }
3715
3716 ALWAYS_INLINE static struct zink_resource *
rebind_ibo(struct zink_context * ctx,enum pipe_shader_type shader,unsigned slot)3717 rebind_ibo(struct zink_context *ctx, enum pipe_shader_type shader, unsigned slot)
3718 {
3719 struct zink_image_view *image_view = &ctx->image_views[shader][slot];
3720 struct zink_resource *res = zink_resource(image_view->base.resource);
3721 if (!res || res->base.b.target != PIPE_BUFFER)
3722 return NULL;
3723 zink_descriptor_set_refs_clear(&image_view->buffer_view->desc_set_refs, image_view->buffer_view);
3724 if (zink_batch_usage_exists(image_view->buffer_view->batch_uses))
3725 zink_batch_reference_bufferview(&ctx->batch, image_view->buffer_view);
3726 VkBufferViewCreateInfo bvci = image_view->buffer_view->bvci;
3727 bvci.buffer = res->obj->buffer;
3728 zink_buffer_view_reference(zink_screen(ctx->base.screen), &image_view->buffer_view, NULL);
3729 if (!zink_resource_object_init_storage(ctx, res)) {
3730 debug_printf("couldn't create storage image!");
3731 return NULL;
3732 }
3733 image_view->buffer_view = get_buffer_view(ctx, res, &bvci);
3734 assert(image_view->buffer_view);
3735 util_range_add(&res->base.b, &res->valid_buffer_range, image_view->base.u.buf.offset,
3736 image_view->base.u.buf.offset + image_view->base.u.buf.size);
3737 update_descriptor_state_image(ctx, shader, slot, res);
3738 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, shader, ZINK_DESCRIPTOR_TYPE_IMAGE, slot, 1);
3739 return res;
3740 }
3741
3742 static unsigned
rebind_buffer(struct zink_context * ctx,struct zink_resource * res,uint32_t rebind_mask,const unsigned expected_num_rebinds)3743 rebind_buffer(struct zink_context *ctx, struct zink_resource *res, uint32_t rebind_mask, const unsigned expected_num_rebinds)
3744 {
3745 unsigned num_rebinds = 0;
3746 bool has_write = false;
3747
3748 if (!zink_resource_has_binds(res))
3749 return 0;
3750
3751 assert(!res->bindless[1]); //TODO
3752 if ((rebind_mask & BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER)) || (!rebind_mask && res->so_bind_count && ctx->num_so_targets)) {
3753 for (unsigned i = 0; i < ctx->num_so_targets; i++) {
3754 if (ctx->so_targets[i]) {
3755 struct zink_resource *so = zink_resource(ctx->so_targets[i]->buffer);
3756 if (so && so == res) {
3757 ctx->dirty_so_targets = true;
3758 num_rebinds++;
3759 }
3760 }
3761 }
3762 rebind_mask &= ~BITFIELD_BIT(TC_BINDING_STREAMOUT_BUFFER);
3763 }
3764 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
3765 goto end;
3766
3767 if ((rebind_mask & BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER)) || (!rebind_mask && res->vbo_bind_mask)) {
3768 u_foreach_bit(slot, res->vbo_bind_mask) {
3769 if (ctx->vertex_buffers[slot].buffer.resource != &res->base.b) //wrong context
3770 goto end;
3771 num_rebinds++;
3772 }
3773 rebind_mask &= ~BITFIELD_BIT(TC_BINDING_VERTEX_BUFFER);
3774 ctx->vertex_buffers_dirty = true;
3775 }
3776 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
3777 goto end;
3778
3779 const uint32_t ubo_mask = rebind_mask ?
3780 rebind_mask & BITFIELD_RANGE(TC_BINDING_UBO_VS, PIPE_SHADER_TYPES) :
3781 ((res->ubo_bind_count[0] ? BITFIELD_RANGE(TC_BINDING_UBO_VS, (PIPE_SHADER_TYPES - 1)) : 0) |
3782 (res->ubo_bind_count[1] ? BITFIELD_BIT(TC_BINDING_UBO_CS) : 0));
3783 u_foreach_bit(shader, ubo_mask >> TC_BINDING_UBO_VS) {
3784 u_foreach_bit(slot, res->ubo_bind_mask[shader]) {
3785 if (&res->base.b != ctx->ubos[shader][slot].buffer) //wrong context
3786 goto end;
3787 rebind_ubo(ctx, shader, slot);
3788 num_rebinds++;
3789 }
3790 }
3791 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_UBO_VS, PIPE_SHADER_TYPES);
3792 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
3793 goto end;
3794
3795 const unsigned ssbo_mask = rebind_mask ?
3796 rebind_mask & BITFIELD_RANGE(TC_BINDING_SSBO_VS, PIPE_SHADER_TYPES) :
3797 BITFIELD_RANGE(TC_BINDING_SSBO_VS, PIPE_SHADER_TYPES);
3798 u_foreach_bit(shader, ssbo_mask >> TC_BINDING_SSBO_VS) {
3799 u_foreach_bit(slot, res->ssbo_bind_mask[shader]) {
3800 struct pipe_shader_buffer *ssbo = &ctx->ssbos[shader][slot];
3801 if (&res->base.b != ssbo->buffer) //wrong context
3802 goto end;
3803 rebind_ssbo(ctx, shader, slot);
3804 has_write |= (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0;
3805 num_rebinds++;
3806 }
3807 }
3808 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SSBO_VS, PIPE_SHADER_TYPES);
3809 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
3810 goto end;
3811 const unsigned sampler_mask = rebind_mask ?
3812 rebind_mask & BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, PIPE_SHADER_TYPES) :
3813 BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, PIPE_SHADER_TYPES);
3814 u_foreach_bit(shader, sampler_mask >> TC_BINDING_SAMPLERVIEW_VS) {
3815 u_foreach_bit(slot, res->sampler_binds[shader]) {
3816 struct zink_sampler_view *sampler_view = zink_sampler_view(ctx->sampler_views[shader][slot]);
3817 if (&res->base.b != sampler_view->base.texture) //wrong context
3818 goto end;
3819 rebind_tbo(ctx, shader, slot);
3820 num_rebinds++;
3821 }
3822 }
3823 rebind_mask &= ~BITFIELD_RANGE(TC_BINDING_SAMPLERVIEW_VS, PIPE_SHADER_TYPES);
3824 if (num_rebinds && expected_num_rebinds >= num_rebinds && !rebind_mask)
3825 goto end;
3826
3827 const unsigned image_mask = rebind_mask ?
3828 rebind_mask & BITFIELD_RANGE(TC_BINDING_IMAGE_VS, PIPE_SHADER_TYPES) :
3829 BITFIELD_RANGE(TC_BINDING_IMAGE_VS, PIPE_SHADER_TYPES);
3830 unsigned num_image_rebinds_remaining = rebind_mask ? expected_num_rebinds - num_rebinds : res->image_bind_count[0] + res->image_bind_count[1];
3831 u_foreach_bit(shader, image_mask >> TC_BINDING_IMAGE_VS) {
3832 for (unsigned slot = 0; num_image_rebinds_remaining && slot < ctx->di.num_images[shader]; slot++) {
3833 struct zink_resource *cres = ctx->di.descriptor_res[ZINK_DESCRIPTOR_TYPE_IMAGE][shader][slot];
3834 if (res != cres)
3835 continue;
3836
3837 rebind_ibo(ctx, shader, slot);
3838 const struct zink_image_view *image_view = &ctx->image_views[shader][slot];
3839 has_write |= (image_view->base.access & PIPE_IMAGE_ACCESS_WRITE) != 0;
3840 num_image_rebinds_remaining--;
3841 num_rebinds++;
3842 }
3843 }
3844 end:
3845 zink_batch_resource_usage_set(&ctx->batch, res, has_write);
3846 return num_rebinds;
3847 }
3848
3849 static bool
zink_resource_commit(struct pipe_context * pctx,struct pipe_resource * pres,unsigned level,struct pipe_box * box,bool commit)3850 zink_resource_commit(struct pipe_context *pctx, struct pipe_resource *pres, unsigned level, struct pipe_box *box, bool commit)
3851 {
3852 struct zink_context *ctx = zink_context(pctx);
3853 struct zink_resource *res = zink_resource(pres);
3854 struct zink_screen *screen = zink_screen(pctx->screen);
3855
3856 /* if any current usage exists, flush the queue */
3857 if (zink_resource_has_unflushed_usage(res))
3858 zink_flush_queue(ctx);
3859
3860 bool ret = zink_bo_commit(screen, res, box->x, box->width, commit);
3861 if (!ret)
3862 check_device_lost(ctx);
3863
3864 return ret;
3865 }
3866
3867 static void
rebind_image(struct zink_context * ctx,struct zink_resource * res)3868 rebind_image(struct zink_context *ctx, struct zink_resource *res)
3869 {
3870 zink_rebind_framebuffer(ctx, res);
3871 if (!zink_resource_has_binds(res))
3872 return;
3873 for (unsigned i = 0; i < PIPE_SHADER_TYPES; i++) {
3874 if (res->sampler_binds[i]) {
3875 for (unsigned j = 0; j < ctx->di.num_sampler_views[i]; j++) {
3876 struct zink_sampler_view *sv = zink_sampler_view(ctx->sampler_views[i][j]);
3877 if (sv && sv->base.texture == &res->base.b) {
3878 struct pipe_surface *psurf = &sv->image_view->base;
3879 zink_rebind_surface(ctx, &psurf);
3880 sv->image_view = zink_surface(psurf);
3881 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_SAMPLER_VIEW, j, 1);
3882 update_descriptor_state_sampler(ctx, i, j, res);
3883 }
3884 }
3885 }
3886 if (!res->image_bind_count[i == PIPE_SHADER_COMPUTE])
3887 continue;
3888 for (unsigned j = 0; j < ctx->di.num_images[i]; j++) {
3889 if (zink_resource(ctx->image_views[i][j].base.resource) == res) {
3890 zink_screen(ctx->base.screen)->context_invalidate_descriptor_state(ctx, i, ZINK_DESCRIPTOR_TYPE_IMAGE, j, 1);
3891 update_descriptor_state_sampler(ctx, i, j, res);
3892 _mesa_set_add(ctx->need_barriers[i == PIPE_SHADER_COMPUTE], res);
3893 }
3894 }
3895 }
3896 }
3897
3898 bool
zink_resource_rebind(struct zink_context * ctx,struct zink_resource * res)3899 zink_resource_rebind(struct zink_context *ctx, struct zink_resource *res)
3900 {
3901 if (res->base.b.target == PIPE_BUFFER) {
3902 /* force counter buffer reset */
3903 res->so_valid = false;
3904 return rebind_buffer(ctx, res, 0, 0) == res->bind_count[0] + res->bind_count[1];
3905 }
3906 rebind_image(ctx, res);
3907 return false;
3908 }
3909
3910 void
zink_rebind_all_buffers(struct zink_context * ctx)3911 zink_rebind_all_buffers(struct zink_context *ctx)
3912 {
3913 struct zink_batch *batch = &ctx->batch;
3914 ctx->vertex_buffers_dirty = ctx->gfx_pipeline_state.vertex_buffers_enabled_mask > 0;
3915 ctx->dirty_so_targets = ctx->num_so_targets > 0;
3916 if (ctx->num_so_targets)
3917 zink_resource_buffer_barrier(ctx, zink_resource(ctx->dummy_xfb_buffer),
3918 VK_ACCESS_TRANSFORM_FEEDBACK_WRITE_BIT_EXT, VK_PIPELINE_STAGE_TRANSFORM_FEEDBACK_BIT_EXT);
3919 for (unsigned shader = PIPE_SHADER_VERTEX; shader < PIPE_SHADER_TYPES; shader++) {
3920 for (unsigned slot = 0; slot < ctx->di.num_ubos[shader]; slot++) {
3921 struct zink_resource *res = rebind_ubo(ctx, shader, slot);
3922 if (res)
3923 zink_batch_resource_usage_set(batch, res, false);
3924 }
3925 for (unsigned slot = 0; slot < ctx->di.num_sampler_views[shader]; slot++) {
3926 struct zink_resource *res = rebind_tbo(ctx, shader, slot);
3927 if (res)
3928 zink_batch_resource_usage_set(batch, res, false);
3929 }
3930 for (unsigned slot = 0; slot < ctx->di.num_ssbos[shader]; slot++) {
3931 struct zink_resource *res = rebind_ssbo(ctx, shader, slot);
3932 if (res)
3933 zink_batch_resource_usage_set(batch, res, (ctx->writable_ssbos[shader] & BITFIELD64_BIT(slot)) != 0);
3934 }
3935 for (unsigned slot = 0; slot < ctx->di.num_images[shader]; slot++) {
3936 struct zink_resource *res = rebind_ibo(ctx, shader, slot);
3937 if (res)
3938 zink_batch_resource_usage_set(batch, res, (ctx->image_views[shader][slot].base.access & PIPE_IMAGE_ACCESS_WRITE) != 0);
3939 }
3940 }
3941 }
3942
3943 static void
zink_context_replace_buffer_storage(struct pipe_context * pctx,struct pipe_resource * dst,struct pipe_resource * src,unsigned num_rebinds,uint32_t rebind_mask,uint32_t delete_buffer_id)3944 zink_context_replace_buffer_storage(struct pipe_context *pctx, struct pipe_resource *dst,
3945 struct pipe_resource *src, unsigned num_rebinds,
3946 uint32_t rebind_mask, uint32_t delete_buffer_id)
3947 {
3948 struct zink_resource *d = zink_resource(dst);
3949 struct zink_resource *s = zink_resource(src);
3950 struct zink_context *ctx = zink_context(pctx);
3951 struct zink_screen *screen = zink_screen(pctx->screen);
3952
3953 assert(d->internal_format == s->internal_format);
3954 assert(d->obj);
3955 assert(s->obj);
3956 util_idalloc_mt_free(&screen->buffer_ids, delete_buffer_id);
3957 zink_descriptor_set_refs_clear(&d->obj->desc_set_refs, d->obj);
3958 /* add a ref just like check_resource_for_batch_ref() would've */
3959 if (zink_resource_has_binds(d) && zink_resource_has_usage(d))
3960 zink_batch_reference_resource(&ctx->batch, d);
3961 /* don't be too creative */
3962 zink_resource_object_reference(screen, &d->obj, s->obj);
3963 /* force counter buffer reset */
3964 d->so_valid = false;
3965 if (num_rebinds && rebind_buffer(ctx, d, rebind_mask, num_rebinds) < num_rebinds)
3966 ctx->buffer_rebind_counter = p_atomic_inc_return(&screen->buffer_rebind_counter);
3967 }
3968
3969 static bool
zink_context_is_resource_busy(struct pipe_screen * pscreen,struct pipe_resource * pres,unsigned usage)3970 zink_context_is_resource_busy(struct pipe_screen *pscreen, struct pipe_resource *pres, unsigned usage)
3971 {
3972 struct zink_screen *screen = zink_screen(pscreen);
3973 struct zink_resource *res = zink_resource(pres);
3974 uint32_t check_usage = 0;
3975 if (usage & PIPE_MAP_READ)
3976 check_usage |= ZINK_RESOURCE_ACCESS_WRITE;
3977 if (usage & PIPE_MAP_WRITE)
3978 check_usage |= ZINK_RESOURCE_ACCESS_RW;
3979 return !zink_resource_usage_check_completion(screen, res, check_usage);
3980 }
3981
3982 static void
zink_emit_string_marker(struct pipe_context * pctx,const char * string,int len)3983 zink_emit_string_marker(struct pipe_context *pctx,
3984 const char *string, int len)
3985 {
3986 struct zink_screen *screen = zink_screen(pctx->screen);
3987 struct zink_batch *batch = &zink_context(pctx)->batch;
3988
3989 /* make sure string is nul-terminated */
3990 char buf[512], *temp = NULL;
3991 if (len < ARRAY_SIZE(buf)) {
3992 memcpy(buf, string, len);
3993 buf[len] = '\0';
3994 string = buf;
3995 } else
3996 string = temp = strndup(string, len);
3997
3998 VkDebugUtilsLabelEXT label = {
3999 VK_STRUCTURE_TYPE_DEBUG_UTILS_LABEL_EXT, NULL,
4000 string,
4001 { 0 }
4002 };
4003 screen->vk.CmdInsertDebugUtilsLabelEXT(batch->state->cmdbuf, &label);
4004 free(temp);
4005 }
4006
4007 struct pipe_context *
zink_context_create(struct pipe_screen * pscreen,void * priv,unsigned flags)4008 zink_context_create(struct pipe_screen *pscreen, void *priv, unsigned flags)
4009 {
4010 struct zink_screen *screen = zink_screen(pscreen);
4011 struct zink_context *ctx = rzalloc(NULL, struct zink_context);
4012 if (!ctx)
4013 goto fail;
4014 ctx->have_timelines = screen->info.have_KHR_timeline_semaphore;
4015
4016 ctx->pipeline_changed[0] = ctx->pipeline_changed[1] = true;
4017 ctx->gfx_pipeline_state.dirty = true;
4018 ctx->compute_pipeline_state.dirty = true;
4019 ctx->fb_changed = ctx->rp_changed = true;
4020 ctx->gfx_pipeline_state.gfx_prim_mode = PIPE_PRIM_MAX;
4021
4022 zink_init_draw_functions(ctx, screen);
4023 zink_init_grid_functions(ctx);
4024
4025 ctx->base.screen = pscreen;
4026 ctx->base.priv = priv;
4027
4028 if (screen->info.have_KHR_imageless_framebuffer) {
4029 ctx->get_framebuffer = zink_get_framebuffer_imageless;
4030 ctx->init_framebuffer = zink_init_framebuffer_imageless;
4031 } else {
4032 ctx->get_framebuffer = zink_get_framebuffer;
4033 ctx->init_framebuffer = zink_init_framebuffer;
4034 }
4035
4036 ctx->base.destroy = zink_context_destroy;
4037 ctx->base.get_device_reset_status = zink_get_device_reset_status;
4038 ctx->base.set_device_reset_callback = zink_set_device_reset_callback;
4039
4040 zink_context_state_init(&ctx->base);
4041
4042 ctx->base.create_sampler_state = zink_create_sampler_state;
4043 ctx->base.bind_sampler_states = zink_bind_sampler_states;
4044 ctx->base.delete_sampler_state = zink_delete_sampler_state;
4045
4046 ctx->base.create_sampler_view = zink_create_sampler_view;
4047 ctx->base.set_sampler_views = zink_set_sampler_views;
4048 ctx->base.sampler_view_destroy = zink_sampler_view_destroy;
4049 ctx->base.get_sample_position = zink_get_sample_position;
4050 ctx->base.set_sample_locations = zink_set_sample_locations;
4051
4052 zink_program_init(ctx);
4053
4054 ctx->base.set_polygon_stipple = zink_set_polygon_stipple;
4055 ctx->base.set_vertex_buffers = zink_set_vertex_buffers;
4056 ctx->base.set_viewport_states = zink_set_viewport_states;
4057 ctx->base.set_scissor_states = zink_set_scissor_states;
4058 ctx->base.set_inlinable_constants = zink_set_inlinable_constants;
4059 ctx->base.set_constant_buffer = zink_set_constant_buffer;
4060 ctx->base.set_shader_buffers = zink_set_shader_buffers;
4061 ctx->base.set_shader_images = zink_set_shader_images;
4062 ctx->base.set_framebuffer_state = zink_set_framebuffer_state;
4063 ctx->base.set_stencil_ref = zink_set_stencil_ref;
4064 ctx->base.set_clip_state = zink_set_clip_state;
4065 ctx->base.set_blend_color = zink_set_blend_color;
4066 ctx->base.set_tess_state = zink_set_tess_state;
4067 ctx->base.set_patch_vertices = zink_set_patch_vertices;
4068
4069 ctx->base.set_sample_mask = zink_set_sample_mask;
4070
4071 ctx->base.clear = zink_clear;
4072 ctx->base.clear_texture = zink_clear_texture;
4073 ctx->base.clear_buffer = zink_clear_buffer;
4074 ctx->base.clear_render_target = zink_clear_render_target;
4075 ctx->base.clear_depth_stencil = zink_clear_depth_stencil;
4076
4077 ctx->base.fence_server_sync = zink_fence_server_sync;
4078 ctx->base.flush = zink_flush;
4079 ctx->base.memory_barrier = zink_memory_barrier;
4080 ctx->base.texture_barrier = zink_texture_barrier;
4081 ctx->base.evaluate_depth_buffer = zink_evaluate_depth_buffer;
4082
4083 ctx->base.resource_commit = zink_resource_commit;
4084 ctx->base.resource_copy_region = zink_resource_copy_region;
4085 ctx->base.blit = zink_blit;
4086 ctx->base.create_stream_output_target = zink_create_stream_output_target;
4087 ctx->base.stream_output_target_destroy = zink_stream_output_target_destroy;
4088
4089 ctx->base.set_stream_output_targets = zink_set_stream_output_targets;
4090 ctx->base.flush_resource = zink_flush_resource;
4091
4092 ctx->base.emit_string_marker = zink_emit_string_marker;
4093
4094 zink_context_surface_init(&ctx->base);
4095 zink_context_resource_init(&ctx->base);
4096 zink_context_query_init(&ctx->base);
4097
4098 _mesa_set_init(&ctx->update_barriers[0][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4099 _mesa_set_init(&ctx->update_barriers[1][0], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4100 _mesa_set_init(&ctx->update_barriers[0][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4101 _mesa_set_init(&ctx->update_barriers[1][1], ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4102 ctx->need_barriers[0] = &ctx->update_barriers[0][0];
4103 ctx->need_barriers[1] = &ctx->update_barriers[1][0];
4104
4105 util_dynarray_init(&ctx->free_batch_states, ctx);
4106
4107 ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state = screen->info.have_EXT_extended_dynamic_state;
4108 ctx->gfx_pipeline_state.have_EXT_extended_dynamic_state2 = screen->info.have_EXT_extended_dynamic_state2;
4109
4110 slab_create_child(&ctx->transfer_pool, &screen->transfer_pool);
4111 slab_create_child(&ctx->transfer_pool_unsync, &screen->transfer_pool);
4112
4113 ctx->base.stream_uploader = u_upload_create_default(&ctx->base);
4114 ctx->base.const_uploader = u_upload_create_default(&ctx->base);
4115 for (int i = 0; i < ARRAY_SIZE(ctx->fb_clears); i++)
4116 util_dynarray_init(&ctx->fb_clears[i].clears, ctx);
4117
4118 ctx->blitter = util_blitter_create(&ctx->base);
4119 if (!ctx->blitter)
4120 goto fail;
4121
4122 ctx->gfx_pipeline_state.shader_keys.last_vertex.key.vs_base.last_vertex_stage = true;
4123 ctx->last_vertex_stage_dirty = true;
4124 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_VERTEX].size = sizeof(struct zink_vs_key_base);
4125 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_TESS_EVAL].size = sizeof(struct zink_vs_key_base);
4126 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_GEOMETRY].size = sizeof(struct zink_vs_key_base);
4127 ctx->gfx_pipeline_state.shader_keys.key[PIPE_SHADER_FRAGMENT].size = sizeof(struct zink_fs_key);
4128 _mesa_hash_table_init(&ctx->compute_program_cache, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4129 _mesa_hash_table_init(&ctx->framebuffer_cache, ctx, hash_framebuffer_imageless, equals_framebuffer_imageless);
4130 _mesa_set_init(&ctx->render_pass_state_cache, ctx, hash_rp_state, equals_rp_state);
4131 ctx->render_pass_cache = _mesa_hash_table_create(NULL,
4132 hash_render_pass_state,
4133 equals_render_pass_state);
4134 if (!ctx->render_pass_cache)
4135 goto fail;
4136
4137 const uint8_t data[] = {0};
4138 ctx->dummy_vertex_buffer = pipe_buffer_create(&screen->base,
4139 PIPE_BIND_VERTEX_BUFFER | PIPE_BIND_SHADER_IMAGE, PIPE_USAGE_IMMUTABLE, sizeof(data));
4140 if (!ctx->dummy_vertex_buffer)
4141 goto fail;
4142 ctx->dummy_xfb_buffer = pipe_buffer_create(&screen->base,
4143 PIPE_BIND_STREAM_OUTPUT, PIPE_USAGE_DEFAULT, sizeof(data));
4144 if (!ctx->dummy_xfb_buffer)
4145 goto fail;
4146 for (unsigned i = 0; i < ARRAY_SIZE(ctx->dummy_surface); i++) {
4147 if (!(screen->info.props.limits.framebufferDepthSampleCounts & BITFIELD_BIT(i)))
4148 continue;
4149 ctx->dummy_surface[i] = zink_surface_create_null(ctx, PIPE_TEXTURE_2D, 1024, 1024, BITFIELD_BIT(i));
4150 if (!ctx->dummy_surface[i])
4151 goto fail;
4152 }
4153 VkBufferViewCreateInfo bvci = create_bvci(ctx, zink_resource(ctx->dummy_vertex_buffer), PIPE_FORMAT_R8_UNORM, 0, sizeof(data));
4154 ctx->dummy_bufferview = get_buffer_view(ctx, zink_resource(ctx->dummy_vertex_buffer), &bvci);
4155 if (!ctx->dummy_bufferview)
4156 goto fail;
4157
4158 if (!zink_descriptor_layouts_init(ctx))
4159 goto fail;
4160
4161 if (!screen->descriptors_init(ctx)) {
4162 zink_screen_init_descriptor_funcs(screen, true);
4163 if (!screen->descriptors_init(ctx))
4164 goto fail;
4165 }
4166
4167 ctx->base.create_texture_handle = zink_create_texture_handle;
4168 ctx->base.delete_texture_handle = zink_delete_texture_handle;
4169 ctx->base.make_texture_handle_resident = zink_make_texture_handle_resident;
4170 ctx->base.create_image_handle = zink_create_image_handle;
4171 ctx->base.delete_image_handle = zink_delete_image_handle;
4172 ctx->base.make_image_handle_resident = zink_make_image_handle_resident;
4173 for (unsigned i = 0; i < 2; i++) {
4174 _mesa_hash_table_init(&ctx->di.bindless[i].img_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4175 _mesa_hash_table_init(&ctx->di.bindless[i].tex_handles, ctx, _mesa_hash_pointer, _mesa_key_pointer_equal);
4176
4177 /* allocate 1024 slots and reserve slot 0 */
4178 util_idalloc_init(&ctx->di.bindless[i].tex_slots, ZINK_MAX_BINDLESS_HANDLES);
4179 util_idalloc_alloc(&ctx->di.bindless[i].tex_slots);
4180 util_idalloc_init(&ctx->di.bindless[i].img_slots, ZINK_MAX_BINDLESS_HANDLES);
4181 util_idalloc_alloc(&ctx->di.bindless[i].img_slots);
4182 ctx->di.bindless[i].buffer_infos = malloc(sizeof(VkImageView) * ZINK_MAX_BINDLESS_HANDLES);
4183 ctx->di.bindless[i].img_infos = malloc(sizeof(VkDescriptorImageInfo) * ZINK_MAX_BINDLESS_HANDLES);
4184 util_dynarray_init(&ctx->di.bindless[i].updates, NULL);
4185 util_dynarray_init(&ctx->di.bindless[i].resident, NULL);
4186 }
4187
4188 ctx->have_timelines = screen->info.have_KHR_timeline_semaphore;
4189 simple_mtx_init(&ctx->batch_mtx, mtx_plain);
4190 zink_start_batch(ctx, &ctx->batch);
4191 if (!ctx->batch.state)
4192 goto fail;
4193
4194 pipe_buffer_write(&ctx->base, ctx->dummy_vertex_buffer, 0, sizeof(data), data);
4195 pipe_buffer_write(&ctx->base, ctx->dummy_xfb_buffer, 0, sizeof(data), data);
4196
4197 for (unsigned i = 0; i < PIPE_SHADER_TYPES; i++) {
4198 /* need to update these based on screen config for null descriptors */
4199 for (unsigned j = 0; j < 32; j++) {
4200 update_descriptor_state_ubo(ctx, i, j, NULL);
4201 update_descriptor_state_sampler(ctx, i, j, NULL);
4202 update_descriptor_state_ssbo(ctx, i, j, NULL);
4203 update_descriptor_state_image(ctx, i, j, NULL);
4204 }
4205 }
4206 if (!screen->info.rb2_feats.nullDescriptor)
4207 ctx->di.fbfetch.imageView = zink_csurface(ctx->dummy_surface[0])->image_view;
4208 p_atomic_inc(&screen->base.num_contexts);
4209
4210 zink_select_draw_vbo(ctx);
4211 zink_select_launch_grid(ctx);
4212
4213 if (!(flags & PIPE_CONTEXT_PREFER_THREADED) || flags & PIPE_CONTEXT_COMPUTE_ONLY) {
4214 return &ctx->base;
4215 }
4216
4217 struct threaded_context *tc = (struct threaded_context*)threaded_context_create(&ctx->base, &screen->transfer_pool,
4218 zink_context_replace_buffer_storage,
4219 &(struct threaded_context_options){
4220 .create_fence = zink_create_tc_fence_for_tc,
4221 .is_resource_busy = zink_context_is_resource_busy,
4222 .driver_calls_flush_notify = true,
4223 .unsynchronized_get_device_reset_status = true,
4224 },
4225 &ctx->tc);
4226
4227 if (tc && (struct zink_context*)tc != ctx) {
4228 threaded_context_init_bytes_mapped_limit(tc, 4);
4229 ctx->base.set_context_param = zink_set_context_param;
4230 }
4231
4232 return (struct pipe_context*)tc;
4233
4234 fail:
4235 if (ctx)
4236 zink_context_destroy(&ctx->base);
4237 return NULL;
4238 }
4239