1 #include "zink_query.h"
2
3 #include "zink_context.h"
4 #include "zink_fence.h"
5 #include "zink_program.h"
6 #include "zink_resource.h"
7 #include "zink_screen.h"
8
9 #include "util/hash_table.h"
10 #include "util/set.h"
11 #include "util/u_dump.h"
12 #include "util/u_inlines.h"
13 #include "util/u_memory.h"
14
15 #if defined(PIPE_ARCH_X86_64) || defined(PIPE_ARCH_PPC_64) || defined(PIPE_ARCH_AARCH64) || defined(PIPE_ARCH_MIPS64)
16 #define NUM_QUERIES 5000
17 #else
18 #define NUM_QUERIES 500
19 #endif
20
21 struct zink_query_buffer {
22 struct list_head list;
23 unsigned num_results;
24 struct pipe_resource *buffer;
25 struct pipe_resource *xfb_buffers[PIPE_MAX_VERTEX_STREAMS - 1];
26 };
27
28 struct zink_query {
29 struct threaded_query base;
30 enum pipe_query_type type;
31
32 VkQueryPool query_pool;
33 VkQueryPool xfb_query_pool[PIPE_MAX_VERTEX_STREAMS - 1]; //stream 0 is in the base pool
34 unsigned curr_query, last_start;
35
36 VkQueryType vkqtype;
37 unsigned index;
38 bool precise;
39 bool xfb_running;
40 bool xfb_overflow;
41
42 bool active; /* query is considered active by vk */
43 bool needs_reset; /* query is considered active by vk and cannot be destroyed */
44 bool dead; /* query should be destroyed when its fence finishes */
45 bool needs_update; /* query needs to update its qbos */
46
47 struct list_head active_list;
48
49 struct list_head stats_list; /* when active, statistics queries are added to ctx->primitives_generated_queries */
50 bool have_gs[NUM_QUERIES]; /* geometry shaders use GEOMETRY_SHADER_PRIMITIVES_BIT */
51 bool have_xfb[NUM_QUERIES]; /* xfb was active during this query */
52 bool was_line_loop[NUM_QUERIES];
53 bool has_draws; /* have_gs and have_xfb are valid for idx=curr_query */
54
55 struct zink_batch_usage *batch_id; //batch that the query was started in
56
57 struct list_head buffers;
58 union {
59 struct zink_query_buffer *curr_qbo;
60 struct pipe_fence_handle *fence; //PIPE_QUERY_GPU_FINISHED
61 };
62
63 struct zink_resource *predicate;
64 bool predicate_dirty;
65 };
66
67 static void
68 update_qbo(struct zink_context *ctx, struct zink_query *q);
69 static void
70 reset_pool(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q);
71
72 static inline unsigned
get_num_results(enum pipe_query_type query_type)73 get_num_results(enum pipe_query_type query_type)
74 {
75 switch (query_type) {
76 case PIPE_QUERY_OCCLUSION_COUNTER:
77 case PIPE_QUERY_OCCLUSION_PREDICATE:
78 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
79 case PIPE_QUERY_TIME_ELAPSED:
80 case PIPE_QUERY_TIMESTAMP:
81 case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE:
82 return 1;
83 case PIPE_QUERY_PRIMITIVES_GENERATED:
84 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
85 case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
86 case PIPE_QUERY_PRIMITIVES_EMITTED:
87 return 2;
88 default:
89 debug_printf("unknown query: %s\n",
90 util_str_query_type(query_type, true));
91 unreachable("zink: unknown query type");
92 }
93 }
94
95 static VkQueryPipelineStatisticFlags
pipeline_statistic_convert(enum pipe_statistics_query_index idx)96 pipeline_statistic_convert(enum pipe_statistics_query_index idx)
97 {
98 unsigned map[] = {
99 [PIPE_STAT_QUERY_IA_VERTICES] = VK_QUERY_PIPELINE_STATISTIC_INPUT_ASSEMBLY_VERTICES_BIT,
100 [PIPE_STAT_QUERY_IA_PRIMITIVES] = VK_QUERY_PIPELINE_STATISTIC_INPUT_ASSEMBLY_PRIMITIVES_BIT,
101 [PIPE_STAT_QUERY_VS_INVOCATIONS] = VK_QUERY_PIPELINE_STATISTIC_VERTEX_SHADER_INVOCATIONS_BIT,
102 [PIPE_STAT_QUERY_GS_INVOCATIONS] = VK_QUERY_PIPELINE_STATISTIC_GEOMETRY_SHADER_INVOCATIONS_BIT,
103 [PIPE_STAT_QUERY_GS_PRIMITIVES] = VK_QUERY_PIPELINE_STATISTIC_GEOMETRY_SHADER_PRIMITIVES_BIT,
104 [PIPE_STAT_QUERY_C_INVOCATIONS] = VK_QUERY_PIPELINE_STATISTIC_CLIPPING_INVOCATIONS_BIT,
105 [PIPE_STAT_QUERY_C_PRIMITIVES] = VK_QUERY_PIPELINE_STATISTIC_CLIPPING_PRIMITIVES_BIT,
106 [PIPE_STAT_QUERY_PS_INVOCATIONS] = VK_QUERY_PIPELINE_STATISTIC_FRAGMENT_SHADER_INVOCATIONS_BIT,
107 [PIPE_STAT_QUERY_HS_INVOCATIONS] = VK_QUERY_PIPELINE_STATISTIC_TESSELLATION_CONTROL_SHADER_PATCHES_BIT,
108 [PIPE_STAT_QUERY_DS_INVOCATIONS] = VK_QUERY_PIPELINE_STATISTIC_TESSELLATION_EVALUATION_SHADER_INVOCATIONS_BIT,
109 [PIPE_STAT_QUERY_CS_INVOCATIONS] = VK_QUERY_PIPELINE_STATISTIC_COMPUTE_SHADER_INVOCATIONS_BIT
110 };
111 assert(idx < ARRAY_SIZE(map));
112 return map[idx];
113 }
114
115 static void
timestamp_to_nanoseconds(struct zink_screen * screen,uint64_t * timestamp)116 timestamp_to_nanoseconds(struct zink_screen *screen, uint64_t *timestamp)
117 {
118 /* The number of valid bits in a timestamp value is determined by
119 * the VkQueueFamilyProperties::timestampValidBits property of the queue on which the timestamp is written.
120 * - 17.5. Timestamp Queries
121 */
122 if (screen->timestamp_valid_bits < 64)
123 *timestamp &= (1ull << screen->timestamp_valid_bits) - 1;
124
125 /* The number of nanoseconds it takes for a timestamp value to be incremented by 1
126 * can be obtained from VkPhysicalDeviceLimits::timestampPeriod
127 * - 17.5. Timestamp Queries
128 */
129 *timestamp *= screen->info.props.limits.timestampPeriod;
130 }
131
132 static VkQueryType
convert_query_type(unsigned query_type,bool * precise)133 convert_query_type(unsigned query_type, bool *precise)
134 {
135 *precise = false;
136 switch (query_type) {
137 case PIPE_QUERY_OCCLUSION_COUNTER:
138 *precise = true;
139 FALLTHROUGH;
140 case PIPE_QUERY_OCCLUSION_PREDICATE:
141 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
142 return VK_QUERY_TYPE_OCCLUSION;
143 case PIPE_QUERY_TIME_ELAPSED:
144 case PIPE_QUERY_TIMESTAMP:
145 return VK_QUERY_TYPE_TIMESTAMP;
146 case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE:
147 case PIPE_QUERY_PRIMITIVES_GENERATED:
148 return VK_QUERY_TYPE_PIPELINE_STATISTICS;
149 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
150 case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
151 case PIPE_QUERY_PRIMITIVES_EMITTED:
152 return VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT;
153 default:
154 debug_printf("unknown query: %s\n",
155 util_str_query_type(query_type, true));
156 unreachable("zink: unknown query type");
157 }
158 }
159
160 static bool
needs_stats_list(struct zink_query * query)161 needs_stats_list(struct zink_query *query)
162 {
163 return query->type == PIPE_QUERY_PRIMITIVES_GENERATED ||
164 query->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE ||
165 query->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE;
166 }
167
168 static bool
is_time_query(struct zink_query * query)169 is_time_query(struct zink_query *query)
170 {
171 return query->type == PIPE_QUERY_TIMESTAMP || query->type == PIPE_QUERY_TIME_ELAPSED;
172 }
173
174 static bool
is_so_overflow_query(struct zink_query * query)175 is_so_overflow_query(struct zink_query *query)
176 {
177 return query->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE || query->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE;
178 }
179
180 static bool
is_bool_query(struct zink_query * query)181 is_bool_query(struct zink_query *query)
182 {
183 return is_so_overflow_query(query) ||
184 query->type == PIPE_QUERY_OCCLUSION_PREDICATE ||
185 query->type == PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE ||
186 query->type == PIPE_QUERY_GPU_FINISHED;
187 }
188
189 static bool
qbo_append(struct pipe_screen * screen,struct zink_query * query)190 qbo_append(struct pipe_screen *screen, struct zink_query *query)
191 {
192 if (query->curr_qbo && query->curr_qbo->list.next)
193 return true;
194 struct zink_query_buffer *qbo = CALLOC_STRUCT(zink_query_buffer);
195 if (!qbo)
196 return false;
197 qbo->buffer = pipe_buffer_create(screen, PIPE_BIND_QUERY_BUFFER,
198 PIPE_USAGE_STAGING,
199 /* this is the maximum possible size of the results in a given buffer */
200 NUM_QUERIES * get_num_results(query->type) * sizeof(uint64_t));
201 if (!qbo->buffer)
202 goto fail;
203 if (query->type == PIPE_QUERY_PRIMITIVES_GENERATED) {
204 /* need separate xfb buffer */
205 qbo->xfb_buffers[0] = pipe_buffer_create(screen, PIPE_BIND_QUERY_BUFFER,
206 PIPE_USAGE_STAGING,
207 /* this is the maximum possible size of the results in a given buffer */
208 NUM_QUERIES * get_num_results(query->type) * sizeof(uint64_t));
209 if (!qbo->xfb_buffers[0])
210 goto fail;
211 } else if (query->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
212 /* need to monitor all xfb streams */
213 for (unsigned i = 0; i < ARRAY_SIZE(qbo->xfb_buffers); i++) {
214 /* need separate xfb buffer */
215 qbo->xfb_buffers[i] = pipe_buffer_create(screen, PIPE_BIND_QUERY_BUFFER,
216 PIPE_USAGE_STAGING,
217 /* this is the maximum possible size of the results in a given buffer */
218 NUM_QUERIES * get_num_results(query->type) * sizeof(uint64_t));
219 if (!qbo->xfb_buffers[i])
220 goto fail;
221 }
222 }
223 list_addtail(&qbo->list, &query->buffers);
224
225 return true;
226 fail:
227 pipe_resource_reference(&qbo->buffer, NULL);
228 for (unsigned i = 0; i < ARRAY_SIZE(qbo->xfb_buffers); i++)
229 pipe_resource_reference(&qbo->xfb_buffers[i], NULL);
230 FREE(qbo);
231 return false;
232 }
233
234 static void
destroy_query(struct zink_screen * screen,struct zink_query * query)235 destroy_query(struct zink_screen *screen, struct zink_query *query)
236 {
237 assert(zink_screen_usage_check_completion(screen, query->batch_id));
238 if (query->query_pool)
239 VKSCR(DestroyQueryPool)(screen->dev, query->query_pool, NULL);
240 struct zink_query_buffer *qbo, *next;
241 LIST_FOR_EACH_ENTRY_SAFE(qbo, next, &query->buffers, list) {
242 pipe_resource_reference(&qbo->buffer, NULL);
243 for (unsigned i = 0; i < ARRAY_SIZE(qbo->xfb_buffers); i++)
244 pipe_resource_reference(&qbo->xfb_buffers[i], NULL);
245 FREE(qbo);
246 }
247 for (unsigned i = 0; i < ARRAY_SIZE(query->xfb_query_pool); i++) {
248 if (query->xfb_query_pool[i])
249 VKSCR(DestroyQueryPool)(screen->dev, query->xfb_query_pool[i], NULL);
250 }
251 pipe_resource_reference((struct pipe_resource**)&query->predicate, NULL);
252 FREE(query);
253 }
254
255 static void
reset_qbo(struct zink_query * q)256 reset_qbo(struct zink_query *q)
257 {
258 q->curr_qbo = list_first_entry(&q->buffers, struct zink_query_buffer, list);
259 q->curr_qbo->num_results = 0;
260 }
261
262 static struct pipe_query *
zink_create_query(struct pipe_context * pctx,unsigned query_type,unsigned index)263 zink_create_query(struct pipe_context *pctx,
264 unsigned query_type, unsigned index)
265 {
266 struct zink_screen *screen = zink_screen(pctx->screen);
267 struct zink_query *query = CALLOC_STRUCT(zink_query);
268 VkQueryPoolCreateInfo pool_create = {0};
269
270 if (!query)
271 return NULL;
272 list_inithead(&query->buffers);
273
274 query->index = index;
275 query->type = query_type;
276 if (query->type == PIPE_QUERY_GPU_FINISHED)
277 return (struct pipe_query *)query;
278 query->vkqtype = convert_query_type(query_type, &query->precise);
279 if (query->vkqtype == -1)
280 return NULL;
281
282 assert(!query->precise || query->vkqtype == VK_QUERY_TYPE_OCCLUSION);
283
284 query->curr_query = 0;
285
286 pool_create.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
287 pool_create.queryType = query->vkqtype;
288 pool_create.queryCount = NUM_QUERIES;
289 if (query_type == PIPE_QUERY_PRIMITIVES_GENERATED)
290 pool_create.pipelineStatistics = VK_QUERY_PIPELINE_STATISTIC_GEOMETRY_SHADER_PRIMITIVES_BIT |
291 VK_QUERY_PIPELINE_STATISTIC_CLIPPING_INVOCATIONS_BIT;
292 else if (query_type == PIPE_QUERY_PIPELINE_STATISTICS_SINGLE)
293 pool_create.pipelineStatistics = pipeline_statistic_convert(index);
294
295 VkResult status = VKSCR(CreateQueryPool)(screen->dev, &pool_create, NULL, &query->query_pool);
296 if (status != VK_SUCCESS) {
297 mesa_loge("ZINK: vkCreateQueryPool failed");
298 goto fail;
299 }
300 if (query_type == PIPE_QUERY_PRIMITIVES_GENERATED) {
301 /* if xfb is active, we need to use an xfb query, otherwise we need pipeline statistics */
302 pool_create.sType = VK_STRUCTURE_TYPE_QUERY_POOL_CREATE_INFO;
303 pool_create.queryType = VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT;
304 pool_create.queryCount = NUM_QUERIES;
305
306 status = VKSCR(CreateQueryPool)(screen->dev, &pool_create, NULL, &query->xfb_query_pool[0]);
307 if (status != VK_SUCCESS) {
308 mesa_loge("ZINK: vkCreateQueryPool failed");
309 goto fail;
310 }
311 } else if (query_type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
312 /* need to monitor all xfb streams */
313 for (unsigned i = 0; i < ARRAY_SIZE(query->xfb_query_pool); i++) {
314 status = VKSCR(CreateQueryPool)(screen->dev, &pool_create, NULL, &query->xfb_query_pool[i]);
315 if (status != VK_SUCCESS) {
316 mesa_loge("ZINK: vkCreateQueryPool failed");
317 goto fail;
318 }
319 }
320 }
321 if (!qbo_append(pctx->screen, query))
322 goto fail;
323 struct zink_batch *batch = &zink_context(pctx)->batch;
324 batch->has_work = true;
325 query->needs_reset = true;
326 if (query->type == PIPE_QUERY_TIMESTAMP) {
327 query->active = true;
328 /* defer pool reset until end_query since we're guaranteed to be threadsafe then */
329 reset_qbo(query);
330 }
331 return (struct pipe_query *)query;
332 fail:
333 destroy_query(screen, query);
334 return NULL;
335 }
336
337 static void
zink_destroy_query(struct pipe_context * pctx,struct pipe_query * q)338 zink_destroy_query(struct pipe_context *pctx,
339 struct pipe_query *q)
340 {
341 struct zink_screen *screen = zink_screen(pctx->screen);
342 struct zink_query *query = (struct zink_query *)q;
343
344 /* only destroy if this query isn't active on any batches,
345 * otherwise just mark dead and wait
346 */
347 if (query->batch_id) {
348 p_atomic_set(&query->dead, true);
349 return;
350 }
351
352 destroy_query(screen, query);
353 }
354
355 void
zink_prune_query(struct zink_screen * screen,struct zink_batch_state * bs,struct zink_query * query)356 zink_prune_query(struct zink_screen *screen, struct zink_batch_state *bs, struct zink_query *query)
357 {
358 if (!zink_batch_usage_matches(query->batch_id, bs))
359 return;
360 query->batch_id = NULL;
361 if (p_atomic_read(&query->dead))
362 destroy_query(screen, query);
363 }
364
365 static void
check_query_results(struct zink_query * query,union pipe_query_result * result,int num_results,uint64_t * results,uint64_t * xfb_results)366 check_query_results(struct zink_query *query, union pipe_query_result *result,
367 int num_results, uint64_t *results, uint64_t *xfb_results)
368 {
369 uint64_t last_val = 0;
370 int result_size = get_num_results(query->type);
371 for (int i = 0; i < num_results * result_size; i += result_size) {
372 switch (query->type) {
373 case PIPE_QUERY_OCCLUSION_PREDICATE:
374 case PIPE_QUERY_OCCLUSION_PREDICATE_CONSERVATIVE:
375 case PIPE_QUERY_GPU_FINISHED:
376 result->b |= results[i] != 0;
377 break;
378
379 case PIPE_QUERY_TIME_ELAPSED:
380 case PIPE_QUERY_TIMESTAMP:
381 /* the application can sum the differences between all N queries to determine the total execution time.
382 * - 17.5. Timestamp Queries
383 */
384 if (query->type != PIPE_QUERY_TIME_ELAPSED || i)
385 result->u64 += results[i] - last_val;
386 last_val = results[i];
387 break;
388 case PIPE_QUERY_OCCLUSION_COUNTER:
389 result->u64 += results[i];
390 break;
391 case PIPE_QUERY_PRIMITIVES_GENERATED:
392 if (query->have_xfb[query->last_start + i / 2] || query->index)
393 result->u64 += xfb_results[i + 1];
394 else
395 /* if a given draw had a geometry shader, we need to use the first result */
396 result->u64 += results[i + !query->have_gs[query->last_start + i / 2]];
397 break;
398 case PIPE_QUERY_PRIMITIVES_EMITTED:
399 /* A query pool created with this type will capture 2 integers -
400 * numPrimitivesWritten and numPrimitivesNeeded -
401 * for the specified vertex stream output from the last vertex processing stage.
402 * - from VK_EXT_transform_feedback spec
403 */
404 result->u64 += results[i];
405 break;
406 case PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE:
407 case PIPE_QUERY_SO_OVERFLOW_PREDICATE:
408 /* A query pool created with this type will capture 2 integers -
409 * numPrimitivesWritten and numPrimitivesNeeded -
410 * for the specified vertex stream output from the last vertex processing stage.
411 * - from VK_EXT_transform_feedback spec
412 */
413 if (query->have_xfb[query->last_start + i / 2])
414 result->b |= results[i] != results[i + 1];
415 break;
416 case PIPE_QUERY_PIPELINE_STATISTICS_SINGLE:
417 switch (query->index) {
418 case PIPE_STAT_QUERY_IA_VERTICES:
419 result->u64 += query->was_line_loop[query->last_start + i] ? results[i] / 2 : results[i];
420 break;
421 default:
422 result->u64 += results[i];
423 break;
424 }
425 break;
426
427 default:
428 debug_printf("unhandled query type: %s\n",
429 util_str_query_type(query->type, true));
430 unreachable("unexpected query type");
431 }
432 }
433 }
434
435 static bool
get_query_result(struct pipe_context * pctx,struct pipe_query * q,bool wait,union pipe_query_result * result)436 get_query_result(struct pipe_context *pctx,
437 struct pipe_query *q,
438 bool wait,
439 union pipe_query_result *result)
440 {
441 struct zink_screen *screen = zink_screen(pctx->screen);
442 struct zink_query *query = (struct zink_query *)q;
443 unsigned flags = PIPE_MAP_READ;
444
445 if (!wait)
446 flags |= PIPE_MAP_DONTBLOCK;
447 if (query->base.flushed)
448 /* this is not a context-safe operation; ensure map doesn't use slab alloc */
449 flags |= PIPE_MAP_THREAD_SAFE;
450
451 util_query_clear_result(result, query->type);
452
453 int num_results = query->curr_query - query->last_start;
454 int result_size = get_num_results(query->type) * sizeof(uint64_t);
455
456 struct zink_query_buffer *qbo;
457 struct pipe_transfer *xfer;
458 LIST_FOR_EACH_ENTRY(qbo, &query->buffers, list) {
459 uint64_t *xfb_results = NULL;
460 uint64_t *results;
461 bool is_timestamp = query->type == PIPE_QUERY_TIMESTAMP || query->type == PIPE_QUERY_TIMESTAMP_DISJOINT;
462 if (!qbo->num_results)
463 continue;
464 results = pipe_buffer_map_range(pctx, qbo->buffer, 0,
465 (is_timestamp ? 1 : qbo->num_results) * result_size, flags, &xfer);
466 if (!results) {
467 if (wait)
468 debug_printf("zink: qbo read failed!");
469 return false;
470 }
471 struct pipe_transfer *xfb_xfer = NULL;
472 if (query->type == PIPE_QUERY_PRIMITIVES_GENERATED) {
473 xfb_results = pipe_buffer_map_range(pctx, qbo->xfb_buffers[0], 0,
474 qbo->num_results * result_size, flags, &xfb_xfer);
475 if (!xfb_results) {
476 if (wait)
477 debug_printf("zink: xfb qbo read failed!");
478 pipe_buffer_unmap(pctx, xfer);
479 return false;
480 }
481 }
482 check_query_results(query, result, is_timestamp ? 1 : qbo->num_results, results, xfb_results);
483 pipe_buffer_unmap(pctx, xfer);
484 if (xfb_xfer)
485 pipe_buffer_unmap(pctx, xfb_xfer);
486 if (query->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
487 for (unsigned i = 0; i < ARRAY_SIZE(qbo->xfb_buffers) && !result->b; i++) {
488 uint64_t *results = pipe_buffer_map_range(pctx, qbo->xfb_buffers[i],
489 0,
490 qbo->num_results * result_size, flags, &xfer);
491 if (!results) {
492 if (wait)
493 debug_printf("zink: qbo read failed!");
494 return false;
495 }
496 check_query_results(query, result, num_results, results, xfb_results);
497 pipe_buffer_unmap(pctx, xfer);
498 }
499 /* if overflow is detected we can stop */
500 if (result->b)
501 break;
502 }
503 }
504
505 if (is_time_query(query))
506 timestamp_to_nanoseconds(screen, &result->u64);
507
508 return true;
509 }
510
511 static void
force_cpu_read(struct zink_context * ctx,struct pipe_query * pquery,enum pipe_query_value_type result_type,struct pipe_resource * pres,unsigned offset)512 force_cpu_read(struct zink_context *ctx, struct pipe_query *pquery, enum pipe_query_value_type result_type, struct pipe_resource *pres, unsigned offset)
513 {
514 struct pipe_context *pctx = &ctx->base;
515 unsigned result_size = result_type <= PIPE_QUERY_TYPE_U32 ? sizeof(uint32_t) : sizeof(uint64_t);
516 struct zink_query *query = (struct zink_query*)pquery;
517 union pipe_query_result result;
518
519 if (query->needs_update)
520 update_qbo(ctx, query);
521
522 bool success = get_query_result(pctx, pquery, true, &result);
523 if (!success) {
524 debug_printf("zink: getting query result failed\n");
525 return;
526 }
527
528 if (result_type <= PIPE_QUERY_TYPE_U32) {
529 uint32_t u32;
530 uint32_t limit;
531 if (result_type == PIPE_QUERY_TYPE_I32)
532 limit = INT_MAX;
533 else
534 limit = UINT_MAX;
535 if (is_bool_query(query))
536 u32 = result.b;
537 else
538 u32 = MIN2(limit, result.u64);
539 tc_buffer_write(pctx, pres, offset, result_size, &u32);
540 } else {
541 uint64_t u64;
542 if (is_bool_query(query))
543 u64 = result.b;
544 else
545 u64 = result.u64;
546 tc_buffer_write(pctx, pres, offset, result_size, &u64);
547 }
548 }
549
550 static void
copy_pool_results_to_buffer(struct zink_context * ctx,struct zink_query * query,VkQueryPool pool,unsigned query_id,struct zink_resource * res,unsigned offset,int num_results,VkQueryResultFlags flags)551 copy_pool_results_to_buffer(struct zink_context *ctx, struct zink_query *query, VkQueryPool pool,
552 unsigned query_id, struct zink_resource *res, unsigned offset,
553 int num_results, VkQueryResultFlags flags)
554 {
555 struct zink_batch *batch = &ctx->batch;
556 unsigned type_size = (flags & VK_QUERY_RESULT_64_BIT) ? sizeof(uint64_t) : sizeof(uint32_t);
557 unsigned base_result_size = get_num_results(query->type) * type_size;
558 unsigned result_size = base_result_size * num_results;
559 if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
560 result_size += type_size;
561 zink_batch_no_rp(ctx);
562 /* if it's a single query that doesn't need special handling, we can copy it and be done */
563 zink_batch_reference_resource_rw(batch, res, true);
564 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_TRANSFER_WRITE_BIT, 0);
565 util_range_add(&res->base.b, &res->valid_buffer_range, offset, offset + result_size);
566 assert(query_id < NUM_QUERIES);
567 VKCTX(CmdCopyQueryPoolResults)(batch->state->cmdbuf, pool, query_id, num_results, res->obj->buffer,
568 offset, base_result_size, flags);
569 }
570
571 static void
copy_results_to_buffer(struct zink_context * ctx,struct zink_query * query,struct zink_resource * res,unsigned offset,int num_results,VkQueryResultFlags flags)572 copy_results_to_buffer(struct zink_context *ctx, struct zink_query *query, struct zink_resource *res, unsigned offset, int num_results, VkQueryResultFlags flags)
573 {
574 copy_pool_results_to_buffer(ctx, query, query->query_pool, query->last_start, res, offset, num_results, flags);
575 }
576
577 static void
reset_pool(struct zink_context * ctx,struct zink_batch * batch,struct zink_query * q)578 reset_pool(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
579 {
580 /* This command must only be called outside of a render pass instance
581 *
582 * - vkCmdResetQueryPool spec
583 */
584 zink_batch_no_rp(ctx);
585 if (q->needs_update)
586 update_qbo(ctx, q);
587
588 VKCTX(CmdResetQueryPool)(batch->state->cmdbuf, q->query_pool, 0, NUM_QUERIES);
589 if (q->type == PIPE_QUERY_PRIMITIVES_GENERATED)
590 VKCTX(CmdResetQueryPool)(batch->state->cmdbuf, q->xfb_query_pool[0], 0, NUM_QUERIES);
591 else if (q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
592 for (unsigned i = 0; i < ARRAY_SIZE(q->xfb_query_pool); i++)
593 VKCTX(CmdResetQueryPool)(batch->state->cmdbuf, q->xfb_query_pool[i], 0, NUM_QUERIES);
594 }
595 memset(q->have_gs, 0, sizeof(q->have_gs));
596 memset(q->have_xfb, 0, sizeof(q->have_xfb));
597 memset(q->was_line_loop, 0, sizeof(q->was_line_loop));
598 q->last_start = q->curr_query = 0;
599 q->needs_reset = false;
600 /* create new qbo for non-timestamp queries:
601 * timestamp queries should never need more than 2 entries in the qbo
602 */
603 if (q->type == PIPE_QUERY_TIMESTAMP)
604 return;
605 if (qbo_append(ctx->base.screen, q))
606 reset_qbo(q);
607 else
608 debug_printf("zink: qbo alloc failed on reset!");
609 }
610
611 static inline unsigned
get_buffer_offset(struct zink_query * q,struct pipe_resource * pres,unsigned query_id)612 get_buffer_offset(struct zink_query *q, struct pipe_resource *pres, unsigned query_id)
613 {
614 return (query_id - q->last_start) * get_num_results(q->type) * sizeof(uint64_t);
615 }
616
617 static void
update_qbo(struct zink_context * ctx,struct zink_query * q)618 update_qbo(struct zink_context *ctx, struct zink_query *q)
619 {
620 struct zink_query_buffer *qbo = q->curr_qbo;
621 unsigned offset = 0;
622 uint32_t query_id = q->curr_query - 1;
623 bool is_timestamp = q->type == PIPE_QUERY_TIMESTAMP || q->type == PIPE_QUERY_TIMESTAMP_DISJOINT;
624 /* timestamp queries just write to offset 0 always */
625 if (!is_timestamp)
626 offset = get_buffer_offset(q, qbo->buffer, query_id);
627 copy_pool_results_to_buffer(ctx, q, q->query_pool, query_id, zink_resource(qbo->buffer),
628 offset,
629 1, VK_QUERY_RESULT_64_BIT);
630
631 if (q->type == PIPE_QUERY_PRIMITIVES_EMITTED ||
632 q->type == PIPE_QUERY_PRIMITIVES_GENERATED ||
633 q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) {
634 copy_pool_results_to_buffer(ctx, q,
635 q->xfb_query_pool[0] ? q->xfb_query_pool[0] : q->query_pool,
636 query_id,
637 zink_resource(qbo->xfb_buffers[0] ? qbo->xfb_buffers[0] : qbo->buffer),
638 get_buffer_offset(q, qbo->xfb_buffers[0] ? qbo->xfb_buffers[0] : qbo->buffer, query_id),
639 1, VK_QUERY_RESULT_64_BIT);
640 }
641
642 else if (q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
643 for (unsigned i = 0; i < ARRAY_SIZE(q->xfb_query_pool); i++) {
644 copy_pool_results_to_buffer(ctx, q, q->xfb_query_pool[i], query_id, zink_resource(qbo->xfb_buffers[i]),
645 get_buffer_offset(q, qbo->xfb_buffers[i], query_id),
646 1, VK_QUERY_RESULT_64_BIT);
647 }
648 }
649
650 if (!is_timestamp)
651 q->curr_qbo->num_results++;
652 else
653 q->curr_qbo->num_results = 1;
654 q->needs_update = false;
655 }
656
657 static void
begin_query(struct zink_context * ctx,struct zink_batch * batch,struct zink_query * q)658 begin_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
659 {
660 VkQueryControlFlags flags = 0;
661
662 q->predicate_dirty = true;
663 if (q->needs_reset)
664 reset_pool(ctx, batch, q);
665 assert(q->curr_query < NUM_QUERIES);
666 q->active = true;
667 batch->has_work = true;
668 if (q->type == PIPE_QUERY_TIME_ELAPSED) {
669 VKCTX(CmdWriteTimestamp)(batch->state->cmdbuf, VK_PIPELINE_STAGE_TOP_OF_PIPE_BIT, q->query_pool, q->curr_query);
670 q->curr_query++;
671 update_qbo(ctx, q);
672 zink_batch_usage_set(&q->batch_id, batch->state);
673 _mesa_set_add(batch->state->active_queries, q);
674 }
675 /* ignore the rest of begin_query for timestamps */
676 if (is_time_query(q))
677 return;
678 if (q->precise)
679 flags |= VK_QUERY_CONTROL_PRECISE_BIT;
680 if (q->type == PIPE_QUERY_PRIMITIVES_EMITTED ||
681 q->type == PIPE_QUERY_PRIMITIVES_GENERATED ||
682 q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) {
683 VKCTX(CmdBeginQueryIndexedEXT)(batch->state->cmdbuf,
684 q->xfb_query_pool[0] ? q->xfb_query_pool[0] : q->query_pool,
685 q->curr_query,
686 flags,
687 q->index);
688 q->xfb_running = true;
689 } else if (q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
690 VKCTX(CmdBeginQueryIndexedEXT)(batch->state->cmdbuf,
691 q->query_pool,
692 q->curr_query,
693 flags,
694 0);
695 for (unsigned i = 0; i < ARRAY_SIZE(q->xfb_query_pool); i++)
696 VKCTX(CmdBeginQueryIndexedEXT)(batch->state->cmdbuf,
697 q->xfb_query_pool[i],
698 q->curr_query,
699 flags,
700 i + 1);
701 q->xfb_running = true;
702 }
703 if (q->vkqtype != VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT)
704 VKCTX(CmdBeginQuery)(batch->state->cmdbuf, q->query_pool, q->curr_query, flags);
705 if (q->type == PIPE_QUERY_PIPELINE_STATISTICS_SINGLE && q->index == PIPE_STAT_QUERY_IA_VERTICES) {
706 assert(!ctx->vertices_query);
707 ctx->vertices_query = q;
708 }
709 if (needs_stats_list(q))
710 list_addtail(&q->stats_list, &ctx->primitives_generated_queries);
711 zink_batch_usage_set(&q->batch_id, batch->state);
712 _mesa_set_add(batch->state->active_queries, q);
713 if (q->type == PIPE_QUERY_PRIMITIVES_GENERATED) {
714 ctx->primitives_generated_active = true;
715 if (zink_set_rasterizer_discard(ctx, true))
716 zink_set_color_write_enables(ctx);
717 }
718 }
719
720 static bool
zink_begin_query(struct pipe_context * pctx,struct pipe_query * q)721 zink_begin_query(struct pipe_context *pctx,
722 struct pipe_query *q)
723 {
724 struct zink_query *query = (struct zink_query *)q;
725 struct zink_context *ctx = zink_context(pctx);
726 struct zink_batch *batch = &ctx->batch;
727
728 query->last_start = query->curr_query;
729 /* drop all past results */
730 reset_qbo(query);
731
732 begin_query(ctx, batch, query);
733
734 return true;
735 }
736
737 static void
update_query_id(struct zink_context * ctx,struct zink_query * q)738 update_query_id(struct zink_context *ctx, struct zink_query *q)
739 {
740 if (++q->curr_query == NUM_QUERIES) {
741 /* always reset on start; this ensures we can actually submit the batch that the current query is on */
742 q->needs_reset = true;
743 }
744 ctx->batch.has_work = true;
745 q->has_draws = false;
746
747 if (ctx->batch.in_rp)
748 q->needs_update = true;
749 else
750 update_qbo(ctx, q);
751 }
752
753 static void
end_query(struct zink_context * ctx,struct zink_batch * batch,struct zink_query * q)754 end_query(struct zink_context *ctx, struct zink_batch *batch, struct zink_query *q)
755 {
756 ASSERTED struct zink_query_buffer *qbo = q->curr_qbo;
757 assert(qbo);
758 assert(!is_time_query(q));
759 q->active = false;
760 if (q->type == PIPE_QUERY_PRIMITIVES_EMITTED ||
761 q->type == PIPE_QUERY_PRIMITIVES_GENERATED ||
762 q->type == PIPE_QUERY_SO_OVERFLOW_PREDICATE) {
763 VKCTX(CmdEndQueryIndexedEXT)(batch->state->cmdbuf,
764 q->xfb_query_pool[0] ? q->xfb_query_pool[0] : q->query_pool,
765 q->curr_query, q->index);
766 }
767
768 else if (q->type == PIPE_QUERY_SO_OVERFLOW_ANY_PREDICATE) {
769 VKCTX(CmdEndQueryIndexedEXT)(batch->state->cmdbuf, q->query_pool, q->curr_query, 0);
770 for (unsigned i = 0; i < ARRAY_SIZE(q->xfb_query_pool); i++) {
771 VKCTX(CmdEndQueryIndexedEXT)(batch->state->cmdbuf, q->xfb_query_pool[i], q->curr_query, i + 1);
772 }
773 }
774 if (q->vkqtype != VK_QUERY_TYPE_TRANSFORM_FEEDBACK_STREAM_EXT && !is_time_query(q))
775 VKCTX(CmdEndQuery)(batch->state->cmdbuf, q->query_pool, q->curr_query);
776
777 if (q->type == PIPE_QUERY_PIPELINE_STATISTICS_SINGLE &&
778 q->index == PIPE_STAT_QUERY_IA_VERTICES)
779 ctx->vertices_query = NULL;
780
781 if (needs_stats_list(q))
782 list_delinit(&q->stats_list);
783
784 update_query_id(ctx, q);
785 if (q->type == PIPE_QUERY_PRIMITIVES_GENERATED) {
786 ctx->primitives_generated_active = false;
787 if (zink_set_rasterizer_discard(ctx, false))
788 zink_set_color_write_enables(ctx);
789 }
790 }
791
792 static bool
zink_end_query(struct pipe_context * pctx,struct pipe_query * q)793 zink_end_query(struct pipe_context *pctx,
794 struct pipe_query *q)
795 {
796 struct zink_context *ctx = zink_context(pctx);
797 struct zink_query *query = (struct zink_query *)q;
798 struct zink_batch *batch = &ctx->batch;
799
800 if (query->type == PIPE_QUERY_GPU_FINISHED) {
801 pctx->flush(pctx, &query->fence, PIPE_FLUSH_DEFERRED);
802 return true;
803 }
804
805 /* FIXME: this can be called from a thread, but it needs to write to the cmdbuf */
806 threaded_context_unwrap_sync(pctx);
807
808 if (needs_stats_list(query))
809 list_delinit(&query->stats_list);
810 if (is_time_query(query)) {
811 if (query->needs_reset)
812 reset_pool(ctx, batch, query);
813 VKCTX(CmdWriteTimestamp)(batch->state->cmdbuf, VK_PIPELINE_STAGE_BOTTOM_OF_PIPE_BIT,
814 query->query_pool, query->curr_query);
815 zink_batch_usage_set(&query->batch_id, batch->state);
816 _mesa_set_add(batch->state->active_queries, query);
817 update_query_id(ctx, query);
818 } else if (query->active)
819 end_query(ctx, batch, query);
820
821 return true;
822 }
823
824 static bool
zink_get_query_result(struct pipe_context * pctx,struct pipe_query * q,bool wait,union pipe_query_result * result)825 zink_get_query_result(struct pipe_context *pctx,
826 struct pipe_query *q,
827 bool wait,
828 union pipe_query_result *result)
829 {
830 struct zink_query *query = (void*)q;
831 struct zink_context *ctx = zink_context(pctx);
832
833 if (query->type == PIPE_QUERY_GPU_FINISHED) {
834 struct pipe_screen *screen = pctx->screen;
835
836 result->b = screen->fence_finish(screen, query->base.flushed ? NULL : pctx,
837 query->fence, wait ? PIPE_TIMEOUT_INFINITE : 0);
838 return result->b;
839 }
840
841 if (query->needs_update)
842 update_qbo(ctx, query);
843
844 if (zink_batch_usage_is_unflushed(query->batch_id)) {
845 if (!threaded_query(q)->flushed)
846 pctx->flush(pctx, NULL, 0);
847 if (!wait)
848 return false;
849 } else if (!threaded_query(q)->flushed &&
850 /* timeline drivers can wait during buffer map */
851 !zink_screen(pctx->screen)->info.have_KHR_timeline_semaphore)
852 zink_batch_usage_check_completion(ctx, query->batch_id);
853
854 return get_query_result(pctx, q, wait, result);
855 }
856
857 static void
suspend_query(struct zink_context * ctx,struct zink_query * query)858 suspend_query(struct zink_context *ctx, struct zink_query *query)
859 {
860 /* if a query isn't active here then we don't need to reactivate it on the next batch */
861 if (query->active && !is_time_query(query))
862 end_query(ctx, &ctx->batch, query);
863 if (query->needs_update)
864 update_qbo(ctx, query);
865 /* do an implicit copy+reset if:
866 * - query is being read from and >50% of pool is used
867 * - >90% of pool is used
868 *
869 * this avoids overflow
870 */
871 if ((query->last_start && query->curr_query > NUM_QUERIES / 2) || (query->curr_query > NUM_QUERIES * 0.9))
872 reset_pool(ctx, &ctx->batch, query);
873 }
874
875 void
zink_suspend_queries(struct zink_context * ctx,struct zink_batch * batch)876 zink_suspend_queries(struct zink_context *ctx, struct zink_batch *batch)
877 {
878 set_foreach(batch->state->active_queries, entry) {
879 struct zink_query *query = (void*)entry->key;
880 if (query->active && !is_time_query(query))
881 /* the fence is going to steal the set off the batch, so we have to copy
882 * the active queries onto a list
883 */
884 list_addtail(&query->active_list, &ctx->suspended_queries);
885 suspend_query(ctx, query);
886 }
887 }
888
889 void
zink_resume_queries(struct zink_context * ctx,struct zink_batch * batch)890 zink_resume_queries(struct zink_context *ctx, struct zink_batch *batch)
891 {
892 struct zink_query *query, *next;
893 LIST_FOR_EACH_ENTRY_SAFE(query, next, &ctx->suspended_queries, active_list) {
894 begin_query(ctx, batch, query);
895 list_delinit(&query->active_list);
896 }
897 }
898
899 void
zink_query_update_gs_states(struct zink_context * ctx,bool was_line_loop)900 zink_query_update_gs_states(struct zink_context *ctx, bool was_line_loop)
901 {
902 struct zink_query *query;
903 LIST_FOR_EACH_ENTRY(query, &ctx->primitives_generated_queries, stats_list) {
904 assert(query->curr_query < ARRAY_SIZE(query->have_gs));
905 assert(query->active);
906 bool have_gs = !!ctx->gfx_stages[PIPE_SHADER_GEOMETRY];
907 bool have_xfb = !!ctx->num_so_targets;
908 if (query->has_draws) {
909 if (query->have_gs[query->curr_query] != have_gs ||
910 query->have_xfb[query->curr_query] != have_xfb) {
911 suspend_query(ctx, query);
912 begin_query(ctx, &ctx->batch, query);
913 }
914 }
915 query->have_gs[query->curr_query] = have_gs;
916 query->have_xfb[query->curr_query] = have_xfb;
917 query->has_draws = true;
918 }
919 if (ctx->vertices_query) {
920 query = ctx->vertices_query;
921 assert(query->curr_query < ARRAY_SIZE(query->have_gs));
922 assert(query->active);
923 if (query->was_line_loop[query->curr_query] != was_line_loop) {
924 suspend_query(ctx, query);
925 begin_query(ctx, &ctx->batch, query);
926 }
927 query->was_line_loop[query->curr_query] = was_line_loop;
928 query->has_draws = true;
929 }
930 }
931
932 static void
zink_set_active_query_state(struct pipe_context * pctx,bool enable)933 zink_set_active_query_state(struct pipe_context *pctx, bool enable)
934 {
935 struct zink_context *ctx = zink_context(pctx);
936 ctx->queries_disabled = !enable;
937
938 struct zink_batch *batch = &ctx->batch;
939 if (ctx->queries_disabled)
940 zink_suspend_queries(ctx, batch);
941 else
942 zink_resume_queries(ctx, batch);
943 }
944
945 void
zink_start_conditional_render(struct zink_context * ctx)946 zink_start_conditional_render(struct zink_context *ctx)
947 {
948 if (unlikely(!zink_screen(ctx->base.screen)->info.have_EXT_conditional_rendering) || ctx->render_condition.active)
949 return;
950 struct zink_batch *batch = &ctx->batch;
951 VkConditionalRenderingFlagsEXT begin_flags = 0;
952 if (ctx->render_condition.inverted)
953 begin_flags = VK_CONDITIONAL_RENDERING_INVERTED_BIT_EXT;
954 VkConditionalRenderingBeginInfoEXT begin_info = {0};
955 begin_info.sType = VK_STRUCTURE_TYPE_CONDITIONAL_RENDERING_BEGIN_INFO_EXT;
956 begin_info.buffer = ctx->render_condition.query->predicate->obj->buffer;
957 begin_info.flags = begin_flags;
958 VKCTX(CmdBeginConditionalRenderingEXT)(batch->state->cmdbuf, &begin_info);
959 zink_batch_reference_resource_rw(batch, ctx->render_condition.query->predicate, false);
960 ctx->render_condition.active = true;
961 }
962
963 void
zink_stop_conditional_render(struct zink_context * ctx)964 zink_stop_conditional_render(struct zink_context *ctx)
965 {
966 struct zink_batch *batch = &ctx->batch;
967 zink_clear_apply_conditionals(ctx);
968 if (unlikely(!zink_screen(ctx->base.screen)->info.have_EXT_conditional_rendering) || !ctx->render_condition.active)
969 return;
970 VKCTX(CmdEndConditionalRenderingEXT)(batch->state->cmdbuf);
971 ctx->render_condition.active = false;
972 }
973
974 bool
zink_check_conditional_render(struct zink_context * ctx)975 zink_check_conditional_render(struct zink_context *ctx)
976 {
977 if (!ctx->render_condition_active)
978 return true;
979 assert(ctx->render_condition.query);
980
981 union pipe_query_result result;
982 zink_get_query_result(&ctx->base, (struct pipe_query*)ctx->render_condition.query, true, &result);
983 return is_bool_query(ctx->render_condition.query) ?
984 ctx->render_condition.inverted != result.b :
985 ctx->render_condition.inverted != !!result.u64;
986 }
987
988 static void
zink_render_condition(struct pipe_context * pctx,struct pipe_query * pquery,bool condition,enum pipe_render_cond_flag mode)989 zink_render_condition(struct pipe_context *pctx,
990 struct pipe_query *pquery,
991 bool condition,
992 enum pipe_render_cond_flag mode)
993 {
994 struct zink_context *ctx = zink_context(pctx);
995 struct zink_query *query = (struct zink_query *)pquery;
996 zink_batch_no_rp(ctx);
997 VkQueryResultFlagBits flags = 0;
998
999 if (query == NULL) {
1000 /* force conditional clears if they exist */
1001 if (ctx->clears_enabled && !ctx->batch.in_rp)
1002 zink_batch_rp(ctx);
1003 zink_stop_conditional_render(ctx);
1004 ctx->render_condition_active = false;
1005 ctx->render_condition.query = NULL;
1006 return;
1007 }
1008
1009 if (!query->predicate) {
1010 struct pipe_resource *pres;
1011
1012 /* need to create a vulkan buffer to copy the data into */
1013 pres = pipe_buffer_create(pctx->screen, PIPE_BIND_QUERY_BUFFER, PIPE_USAGE_DEFAULT, sizeof(uint64_t));
1014 if (!pres)
1015 return;
1016
1017 query->predicate = zink_resource(pres);
1018 }
1019 if (query->predicate_dirty) {
1020 struct zink_resource *res = query->predicate;
1021
1022 if (mode == PIPE_RENDER_COND_WAIT || mode == PIPE_RENDER_COND_BY_REGION_WAIT)
1023 flags |= VK_QUERY_RESULT_WAIT_BIT;
1024
1025 flags |= VK_QUERY_RESULT_64_BIT;
1026 int num_results = query->curr_query - query->last_start;
1027 if (query->type != PIPE_QUERY_PRIMITIVES_GENERATED &&
1028 !is_so_overflow_query(query)) {
1029 copy_results_to_buffer(ctx, query, res, 0, num_results, flags);
1030 } else {
1031 /* these need special handling */
1032 force_cpu_read(ctx, pquery, PIPE_QUERY_TYPE_U32, &res->base.b, 0);
1033 }
1034 zink_resource_buffer_barrier(ctx, res, VK_ACCESS_CONDITIONAL_RENDERING_READ_BIT_EXT, VK_PIPELINE_STAGE_CONDITIONAL_RENDERING_BIT_EXT);
1035 query->predicate_dirty = false;
1036 }
1037 ctx->render_condition.inverted = condition;
1038 ctx->render_condition_active = true;
1039 ctx->render_condition.query = query;
1040 if (ctx->batch.in_rp)
1041 zink_start_conditional_render(ctx);
1042 }
1043
1044 static void
zink_get_query_result_resource(struct pipe_context * pctx,struct pipe_query * pquery,enum pipe_query_flags flags,enum pipe_query_value_type result_type,int index,struct pipe_resource * pres,unsigned offset)1045 zink_get_query_result_resource(struct pipe_context *pctx,
1046 struct pipe_query *pquery,
1047 enum pipe_query_flags flags,
1048 enum pipe_query_value_type result_type,
1049 int index,
1050 struct pipe_resource *pres,
1051 unsigned offset)
1052 {
1053 struct zink_context *ctx = zink_context(pctx);
1054 struct zink_screen *screen = zink_screen(pctx->screen);
1055 struct zink_query *query = (struct zink_query*)pquery;
1056 struct zink_resource *res = zink_resource(pres);
1057 unsigned result_size = result_type <= PIPE_QUERY_TYPE_U32 ? sizeof(uint32_t) : sizeof(uint64_t);
1058 VkQueryResultFlagBits size_flags = result_type <= PIPE_QUERY_TYPE_U32 ? 0 : VK_QUERY_RESULT_64_BIT;
1059 unsigned num_queries = query->curr_query - query->last_start;
1060 unsigned query_id = query->last_start;
1061
1062 if (index == -1) {
1063 /* VK_QUERY_RESULT_WITH_AVAILABILITY_BIT will ALWAYS write some kind of result data
1064 * in addition to the availability result, which is a problem if we're just trying to get availability data
1065 *
1066 * if we know that there's no valid buffer data in the preceding buffer range, then we can just
1067 * stomp on it with a glorious queued buffer copy instead of forcing a stall to manually write to the
1068 * buffer
1069 */
1070
1071 VkQueryResultFlags flag = is_time_query(query) ? 0 : VK_QUERY_RESULT_PARTIAL_BIT;
1072 unsigned src_offset = result_size * get_num_results(query->type);
1073 if (zink_batch_usage_check_completion(ctx, query->batch_id)) {
1074 uint64_t u64[4] = {0};
1075 if (VKCTX(GetQueryPoolResults)(screen->dev, query->query_pool, query_id, 1, sizeof(u64), u64,
1076 0, size_flags | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT | flag) == VK_SUCCESS) {
1077 tc_buffer_write(pctx, pres, offset, result_size, (unsigned char*)u64 + src_offset);
1078 return;
1079 } else {
1080 mesa_loge("ZINK: vkGetQueryPoolResults failed");
1081 }
1082 }
1083 struct pipe_resource *staging = pipe_buffer_create(pctx->screen, 0, PIPE_USAGE_STAGING, src_offset + result_size);
1084 copy_results_to_buffer(ctx, query, zink_resource(staging), 0, 1, size_flags | VK_QUERY_RESULT_WITH_AVAILABILITY_BIT | flag);
1085 zink_copy_buffer(ctx, res, zink_resource(staging), offset, result_size * get_num_results(query->type), result_size);
1086 pipe_resource_reference(&staging, NULL);
1087 return;
1088 }
1089
1090 if (!is_time_query(query) && !is_bool_query(query)) {
1091 if (num_queries == 1 && query->type != PIPE_QUERY_PRIMITIVES_GENERATED &&
1092 query->type != PIPE_QUERY_PRIMITIVES_EMITTED &&
1093 !is_bool_query(query)) {
1094 if (size_flags == VK_QUERY_RESULT_64_BIT) {
1095 if (query->needs_update)
1096 update_qbo(ctx, query);
1097 /* internal qbo always writes 64bit value so we can just direct copy */
1098 zink_copy_buffer(ctx, res, zink_resource(query->curr_qbo->buffer), offset,
1099 get_buffer_offset(query, query->curr_qbo->buffer, query->last_start),
1100 result_size);
1101 } else
1102 /* have to do a new copy for 32bit */
1103 copy_results_to_buffer(ctx, query, res, offset, 1, size_flags);
1104 return;
1105 }
1106 }
1107
1108 /* TODO: use CS to aggregate results */
1109
1110 /* unfortunately, there's no way to accumulate results from multiple queries on the gpu without either
1111 * clobbering all but the last result or writing the results sequentially, so we have to manually write the result
1112 */
1113 force_cpu_read(ctx, pquery, result_type, pres, offset);
1114 }
1115
1116 static uint64_t
zink_get_timestamp(struct pipe_context * pctx)1117 zink_get_timestamp(struct pipe_context *pctx)
1118 {
1119 struct zink_screen *screen = zink_screen(pctx->screen);
1120 uint64_t timestamp, deviation;
1121 assert(screen->info.have_EXT_calibrated_timestamps);
1122 VkCalibratedTimestampInfoEXT cti = {0};
1123 cti.sType = VK_STRUCTURE_TYPE_CALIBRATED_TIMESTAMP_INFO_EXT;
1124 cti.timeDomain = VK_TIME_DOMAIN_DEVICE_EXT;
1125 if (VKSCR(GetCalibratedTimestampsEXT)(screen->dev, 1, &cti, ×tamp, &deviation) != VK_SUCCESS) {
1126 mesa_loge("ZINK: vkGetCalibratedTimestampsEXT failed");
1127 }
1128 timestamp_to_nanoseconds(screen, ×tamp);
1129 return timestamp;
1130 }
1131
1132 void
zink_context_query_init(struct pipe_context * pctx)1133 zink_context_query_init(struct pipe_context *pctx)
1134 {
1135 struct zink_context *ctx = zink_context(pctx);
1136 list_inithead(&ctx->suspended_queries);
1137 list_inithead(&ctx->primitives_generated_queries);
1138
1139 pctx->create_query = zink_create_query;
1140 pctx->destroy_query = zink_destroy_query;
1141 pctx->begin_query = zink_begin_query;
1142 pctx->end_query = zink_end_query;
1143 pctx->get_query_result = zink_get_query_result;
1144 pctx->get_query_result_resource = zink_get_query_result_resource;
1145 pctx->set_active_query_state = zink_set_active_query_state;
1146 pctx->render_condition = zink_render_condition;
1147 pctx->get_timestamp = zink_get_timestamp;
1148 }
1149