1 /*
2 * Copyright 2019 Google LLC
3 * SPDX-License-Identifier: MIT
4 *
5 * based in part on anv and radv which are:
6 * Copyright © 2015 Intel Corporation
7 * Copyright © 2016 Red Hat.
8 * Copyright © 2016 Bas Nieuwenhuizen
9 */
10
11 #include "vn_queue.h"
12
13 #include "util/libsync.h"
14 #include "venus-protocol/vn_protocol_driver_event.h"
15 #include "venus-protocol/vn_protocol_driver_fence.h"
16 #include "venus-protocol/vn_protocol_driver_queue.h"
17 #include "venus-protocol/vn_protocol_driver_semaphore.h"
18
19 #include "vn_device.h"
20 #include "vn_device_memory.h"
21 #include "vn_renderer.h"
22 #include "vn_wsi.h"
23
24 /* queue commands */
25
26 void
vn_GetDeviceQueue(VkDevice device,uint32_t queueFamilyIndex,uint32_t queueIndex,VkQueue * pQueue)27 vn_GetDeviceQueue(VkDevice device,
28 uint32_t queueFamilyIndex,
29 uint32_t queueIndex,
30 VkQueue *pQueue)
31 {
32 struct vn_device *dev = vn_device_from_handle(device);
33
34 for (uint32_t i = 0; i < dev->queue_count; i++) {
35 struct vn_queue *queue = &dev->queues[i];
36 if (queue->family == queueFamilyIndex && queue->index == queueIndex) {
37 assert(!queue->flags);
38 *pQueue = vn_queue_to_handle(queue);
39 return;
40 }
41 }
42 unreachable("bad queue family/index");
43 }
44
45 void
vn_GetDeviceQueue2(VkDevice device,const VkDeviceQueueInfo2 * pQueueInfo,VkQueue * pQueue)46 vn_GetDeviceQueue2(VkDevice device,
47 const VkDeviceQueueInfo2 *pQueueInfo,
48 VkQueue *pQueue)
49 {
50 struct vn_device *dev = vn_device_from_handle(device);
51
52 for (uint32_t i = 0; i < dev->queue_count; i++) {
53 struct vn_queue *queue = &dev->queues[i];
54 if (queue->family == pQueueInfo->queueFamilyIndex &&
55 queue->index == pQueueInfo->queueIndex &&
56 queue->flags == pQueueInfo->flags) {
57 *pQueue = vn_queue_to_handle(queue);
58 return;
59 }
60 }
61 unreachable("bad queue family/index");
62 }
63
64 static void
65 vn_semaphore_reset_wsi(struct vn_device *dev, struct vn_semaphore *sem);
66
67 struct vn_queue_submission {
68 VkStructureType batch_type;
69 VkQueue queue;
70 uint32_t batch_count;
71 union {
72 const void *batches;
73 const VkSubmitInfo *submit_batches;
74 const VkBindSparseInfo *bind_sparse_batches;
75 };
76 VkFence fence;
77
78 uint32_t wait_semaphore_count;
79 uint32_t wait_wsi_count;
80
81 struct {
82 void *storage;
83
84 union {
85 void *batches;
86 VkSubmitInfo *submit_batches;
87 VkBindSparseInfo *bind_sparse_batches;
88 };
89 VkSemaphore *semaphores;
90 } temp;
91 };
92
93 static void
vn_queue_submission_count_batch_semaphores(struct vn_queue_submission * submit,uint32_t batch_index)94 vn_queue_submission_count_batch_semaphores(struct vn_queue_submission *submit,
95 uint32_t batch_index)
96 {
97 union {
98 const VkSubmitInfo *submit_batch;
99 const VkBindSparseInfo *bind_sparse_batch;
100 } u;
101 const VkSemaphore *wait_sems;
102 uint32_t wait_count;
103 switch (submit->batch_type) {
104 case VK_STRUCTURE_TYPE_SUBMIT_INFO:
105 u.submit_batch = &submit->submit_batches[batch_index];
106 wait_sems = u.submit_batch->pWaitSemaphores;
107 wait_count = u.submit_batch->waitSemaphoreCount;
108 break;
109 case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
110 u.bind_sparse_batch = &submit->bind_sparse_batches[batch_index];
111 wait_sems = u.bind_sparse_batch->pWaitSemaphores;
112 wait_count = u.bind_sparse_batch->waitSemaphoreCount;
113 break;
114 default:
115 unreachable("unexpected batch type");
116 break;
117 }
118
119 submit->wait_semaphore_count += wait_count;
120 for (uint32_t i = 0; i < wait_count; i++) {
121 struct vn_semaphore *sem = vn_semaphore_from_handle(wait_sems[i]);
122 const struct vn_sync_payload *payload = sem->payload;
123
124 if (payload->type == VN_SYNC_TYPE_WSI_SIGNALED)
125 submit->wait_wsi_count++;
126 }
127 }
128
129 static void
vn_queue_submission_count_semaphores(struct vn_queue_submission * submit)130 vn_queue_submission_count_semaphores(struct vn_queue_submission *submit)
131 {
132 submit->wait_semaphore_count = 0;
133 submit->wait_wsi_count = 0;
134
135 for (uint32_t i = 0; i < submit->batch_count; i++)
136 vn_queue_submission_count_batch_semaphores(submit, i);
137 }
138
139 static VkResult
vn_queue_submission_alloc_storage(struct vn_queue_submission * submit)140 vn_queue_submission_alloc_storage(struct vn_queue_submission *submit)
141 {
142 struct vn_queue *queue = vn_queue_from_handle(submit->queue);
143 const VkAllocationCallbacks *alloc = &queue->device->base.base.alloc;
144 size_t alloc_size = 0;
145 size_t semaphores_offset = 0;
146
147 /* we want to filter out VN_SYNC_TYPE_WSI_SIGNALED wait semaphores */
148 if (submit->wait_wsi_count) {
149 switch (submit->batch_type) {
150 case VK_STRUCTURE_TYPE_SUBMIT_INFO:
151 alloc_size += sizeof(VkSubmitInfo) * submit->batch_count;
152 break;
153 case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
154 alloc_size += sizeof(VkBindSparseInfo) * submit->batch_count;
155 break;
156 default:
157 unreachable("unexpected batch type");
158 break;
159 }
160
161 semaphores_offset = alloc_size;
162 alloc_size += sizeof(*submit->temp.semaphores) *
163 (submit->wait_semaphore_count - submit->wait_wsi_count);
164 }
165
166 if (!alloc_size) {
167 submit->temp.storage = NULL;
168 return VK_SUCCESS;
169 }
170
171 submit->temp.storage = vk_alloc(alloc, alloc_size, VN_DEFAULT_ALIGN,
172 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
173 if (!submit->temp.storage)
174 return VK_ERROR_OUT_OF_HOST_MEMORY;
175
176 submit->temp.batches = submit->temp.storage;
177 submit->temp.semaphores = submit->temp.storage + semaphores_offset;
178
179 return VK_SUCCESS;
180 }
181
182 static uint32_t
vn_queue_submission_filter_batch_wsi_semaphores(struct vn_queue_submission * submit,uint32_t batch_index,uint32_t sem_base)183 vn_queue_submission_filter_batch_wsi_semaphores(
184 struct vn_queue_submission *submit,
185 uint32_t batch_index,
186 uint32_t sem_base)
187 {
188 struct vn_queue *queue = vn_queue_from_handle(submit->queue);
189
190 union {
191 VkSubmitInfo *submit_batch;
192 VkBindSparseInfo *bind_sparse_batch;
193 } u;
194 const VkSemaphore *src_sems;
195 uint32_t src_count;
196 switch (submit->batch_type) {
197 case VK_STRUCTURE_TYPE_SUBMIT_INFO:
198 u.submit_batch = &submit->temp.submit_batches[batch_index];
199 src_sems = u.submit_batch->pWaitSemaphores;
200 src_count = u.submit_batch->waitSemaphoreCount;
201 break;
202 case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
203 u.bind_sparse_batch = &submit->temp.bind_sparse_batches[batch_index];
204 src_sems = u.bind_sparse_batch->pWaitSemaphores;
205 src_count = u.bind_sparse_batch->waitSemaphoreCount;
206 break;
207 default:
208 unreachable("unexpected batch type");
209 break;
210 }
211
212 VkSemaphore *dst_sems = &submit->temp.semaphores[sem_base];
213 uint32_t dst_count = 0;
214
215 /* filter out VN_SYNC_TYPE_WSI_SIGNALED wait semaphores */
216 for (uint32_t i = 0; i < src_count; i++) {
217 struct vn_semaphore *sem = vn_semaphore_from_handle(src_sems[i]);
218 const struct vn_sync_payload *payload = sem->payload;
219
220 if (payload->type == VN_SYNC_TYPE_WSI_SIGNALED)
221 vn_semaphore_reset_wsi(queue->device, sem);
222 else
223 dst_sems[dst_count++] = src_sems[i];
224 }
225
226 switch (submit->batch_type) {
227 case VK_STRUCTURE_TYPE_SUBMIT_INFO:
228 u.submit_batch->pWaitSemaphores = dst_sems;
229 u.submit_batch->waitSemaphoreCount = dst_count;
230 break;
231 case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
232 u.bind_sparse_batch->pWaitSemaphores = dst_sems;
233 u.bind_sparse_batch->waitSemaphoreCount = dst_count;
234 break;
235 default:
236 break;
237 }
238
239 return dst_count;
240 }
241
242 static void
vn_queue_submission_setup_batches(struct vn_queue_submission * submit)243 vn_queue_submission_setup_batches(struct vn_queue_submission *submit)
244 {
245 if (!submit->temp.storage)
246 return;
247
248 /* make a copy because we need to filter out WSI semaphores */
249 if (submit->wait_wsi_count) {
250 switch (submit->batch_type) {
251 case VK_STRUCTURE_TYPE_SUBMIT_INFO:
252 memcpy(submit->temp.submit_batches, submit->submit_batches,
253 sizeof(submit->submit_batches[0]) * submit->batch_count);
254 submit->submit_batches = submit->temp.submit_batches;
255 break;
256 case VK_STRUCTURE_TYPE_BIND_SPARSE_INFO:
257 memcpy(submit->temp.bind_sparse_batches, submit->bind_sparse_batches,
258 sizeof(submit->bind_sparse_batches[0]) * submit->batch_count);
259 submit->bind_sparse_batches = submit->temp.bind_sparse_batches;
260 break;
261 default:
262 unreachable("unexpected batch type");
263 break;
264 }
265 }
266
267 uint32_t wait_sem_base = 0;
268 for (uint32_t i = 0; i < submit->batch_count; i++) {
269 if (submit->wait_wsi_count) {
270 wait_sem_base += vn_queue_submission_filter_batch_wsi_semaphores(
271 submit, i, wait_sem_base);
272 }
273 }
274 }
275
276 static VkResult
vn_queue_submission_prepare_submit(struct vn_queue_submission * submit,VkQueue queue,uint32_t batch_count,const VkSubmitInfo * submit_batches,VkFence fence)277 vn_queue_submission_prepare_submit(struct vn_queue_submission *submit,
278 VkQueue queue,
279 uint32_t batch_count,
280 const VkSubmitInfo *submit_batches,
281 VkFence fence)
282 {
283 submit->batch_type = VK_STRUCTURE_TYPE_SUBMIT_INFO;
284 submit->queue = queue;
285 submit->batch_count = batch_count;
286 submit->submit_batches = submit_batches;
287 submit->fence = fence;
288
289 vn_queue_submission_count_semaphores(submit);
290
291 VkResult result = vn_queue_submission_alloc_storage(submit);
292 if (result != VK_SUCCESS)
293 return result;
294
295 vn_queue_submission_setup_batches(submit);
296
297 return VK_SUCCESS;
298 }
299
300 static VkResult
vn_queue_submission_prepare_bind_sparse(struct vn_queue_submission * submit,VkQueue queue,uint32_t batch_count,const VkBindSparseInfo * bind_sparse_batches,VkFence fence)301 vn_queue_submission_prepare_bind_sparse(
302 struct vn_queue_submission *submit,
303 VkQueue queue,
304 uint32_t batch_count,
305 const VkBindSparseInfo *bind_sparse_batches,
306 VkFence fence)
307 {
308 submit->batch_type = VK_STRUCTURE_TYPE_BIND_SPARSE_INFO;
309 submit->queue = queue;
310 submit->batch_count = batch_count;
311 submit->bind_sparse_batches = bind_sparse_batches;
312 submit->fence = fence;
313
314 vn_queue_submission_count_semaphores(submit);
315
316 VkResult result = vn_queue_submission_alloc_storage(submit);
317 if (result != VK_SUCCESS)
318 return result;
319
320 vn_queue_submission_setup_batches(submit);
321
322 return VK_SUCCESS;
323 }
324
325 static void
vn_queue_submission_cleanup(struct vn_queue_submission * submit)326 vn_queue_submission_cleanup(struct vn_queue_submission *submit)
327 {
328 struct vn_queue *queue = vn_queue_from_handle(submit->queue);
329 const VkAllocationCallbacks *alloc = &queue->device->base.base.alloc;
330
331 vk_free(alloc, submit->temp.storage);
332 }
333
334 VkResult
vn_QueueSubmit(VkQueue _queue,uint32_t submitCount,const VkSubmitInfo * pSubmits,VkFence fence)335 vn_QueueSubmit(VkQueue _queue,
336 uint32_t submitCount,
337 const VkSubmitInfo *pSubmits,
338 VkFence fence)
339 {
340 struct vn_queue *queue = vn_queue_from_handle(_queue);
341 struct vn_device *dev = queue->device;
342
343 struct vn_queue_submission submit;
344 VkResult result = vn_queue_submission_prepare_submit(
345 &submit, _queue, submitCount, pSubmits, fence);
346 if (result != VK_SUCCESS)
347 return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
348
349 const struct vn_device_memory *wsi_mem = NULL;
350 if (submit.batch_count == 1) {
351 const struct wsi_memory_signal_submit_info *info = vk_find_struct_const(
352 submit.submit_batches[0].pNext, WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA);
353 if (info) {
354 wsi_mem = vn_device_memory_from_handle(info->memory);
355 assert(!wsi_mem->base_memory && wsi_mem->base_bo);
356 }
357 }
358
359 result =
360 vn_call_vkQueueSubmit(dev->instance, submit.queue, submit.batch_count,
361 submit.submit_batches, submit.fence);
362 if (result != VK_SUCCESS) {
363 vn_queue_submission_cleanup(&submit);
364 return vn_error(dev->instance, result);
365 }
366
367 if (wsi_mem) {
368 /* XXX this is always false and kills the performance */
369 if (dev->instance->renderer_info.has_implicit_fencing) {
370 vn_renderer_submit(dev->renderer, &(const struct vn_renderer_submit){
371 .bos = &wsi_mem->base_bo,
372 .bo_count = 1,
373 });
374 } else {
375 if (VN_DEBUG(WSI)) {
376 static uint32_t ratelimit;
377 if (ratelimit < 10) {
378 vn_log(dev->instance,
379 "forcing vkQueueWaitIdle before presenting");
380 ratelimit++;
381 }
382 }
383
384 vn_QueueWaitIdle(submit.queue);
385 }
386 }
387
388 vn_queue_submission_cleanup(&submit);
389
390 return VK_SUCCESS;
391 }
392
393 VkResult
vn_QueueBindSparse(VkQueue _queue,uint32_t bindInfoCount,const VkBindSparseInfo * pBindInfo,VkFence fence)394 vn_QueueBindSparse(VkQueue _queue,
395 uint32_t bindInfoCount,
396 const VkBindSparseInfo *pBindInfo,
397 VkFence fence)
398 {
399 struct vn_queue *queue = vn_queue_from_handle(_queue);
400 struct vn_device *dev = queue->device;
401
402 struct vn_queue_submission submit;
403 VkResult result = vn_queue_submission_prepare_bind_sparse(
404 &submit, _queue, bindInfoCount, pBindInfo, fence);
405 if (result != VK_SUCCESS)
406 return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
407
408 result = vn_call_vkQueueBindSparse(
409 dev->instance, submit.queue, submit.batch_count,
410 submit.bind_sparse_batches, submit.fence);
411 if (result != VK_SUCCESS) {
412 vn_queue_submission_cleanup(&submit);
413 return vn_error(dev->instance, result);
414 }
415
416 vn_queue_submission_cleanup(&submit);
417
418 return VK_SUCCESS;
419 }
420
421 VkResult
vn_QueueWaitIdle(VkQueue _queue)422 vn_QueueWaitIdle(VkQueue _queue)
423 {
424 struct vn_queue *queue = vn_queue_from_handle(_queue);
425 VkDevice device = vn_device_to_handle(queue->device);
426
427 VkResult result = vn_QueueSubmit(_queue, 0, NULL, queue->wait_fence);
428 if (result != VK_SUCCESS)
429 return result;
430
431 result = vn_WaitForFences(device, 1, &queue->wait_fence, true, UINT64_MAX);
432 vn_ResetFences(device, 1, &queue->wait_fence);
433
434 return vn_result(queue->device->instance, result);
435 }
436
437 /* fence commands */
438
439 static void
vn_sync_payload_release(struct vn_device * dev,struct vn_sync_payload * payload)440 vn_sync_payload_release(struct vn_device *dev,
441 struct vn_sync_payload *payload)
442 {
443 payload->type = VN_SYNC_TYPE_INVALID;
444 }
445
446 static VkResult
vn_fence_init_payloads(struct vn_device * dev,struct vn_fence * fence,bool signaled,const VkAllocationCallbacks * alloc)447 vn_fence_init_payloads(struct vn_device *dev,
448 struct vn_fence *fence,
449 bool signaled,
450 const VkAllocationCallbacks *alloc)
451 {
452 fence->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
453 fence->temporary.type = VN_SYNC_TYPE_INVALID;
454 fence->payload = &fence->permanent;
455
456 return VK_SUCCESS;
457 }
458
459 void
vn_fence_signal_wsi(struct vn_device * dev,struct vn_fence * fence)460 vn_fence_signal_wsi(struct vn_device *dev, struct vn_fence *fence)
461 {
462 struct vn_sync_payload *temp = &fence->temporary;
463
464 vn_sync_payload_release(dev, temp);
465 temp->type = VN_SYNC_TYPE_WSI_SIGNALED;
466 fence->payload = temp;
467 }
468
469 VkResult
vn_CreateFence(VkDevice device,const VkFenceCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkFence * pFence)470 vn_CreateFence(VkDevice device,
471 const VkFenceCreateInfo *pCreateInfo,
472 const VkAllocationCallbacks *pAllocator,
473 VkFence *pFence)
474 {
475 struct vn_device *dev = vn_device_from_handle(device);
476 const VkAllocationCallbacks *alloc =
477 pAllocator ? pAllocator : &dev->base.base.alloc;
478
479 VkFenceCreateInfo local_create_info;
480 if (vk_find_struct_const(pCreateInfo->pNext, EXPORT_FENCE_CREATE_INFO)) {
481 local_create_info = *pCreateInfo;
482 local_create_info.pNext = NULL;
483 pCreateInfo = &local_create_info;
484 }
485
486 struct vn_fence *fence = vk_zalloc(alloc, sizeof(*fence), VN_DEFAULT_ALIGN,
487 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
488 if (!fence)
489 return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
490
491 vn_object_base_init(&fence->base, VK_OBJECT_TYPE_FENCE, &dev->base);
492
493 VkResult result = vn_fence_init_payloads(
494 dev, fence, pCreateInfo->flags & VK_FENCE_CREATE_SIGNALED_BIT, alloc);
495 if (result != VK_SUCCESS) {
496 vn_object_base_fini(&fence->base);
497 vk_free(alloc, fence);
498 return vn_error(dev->instance, result);
499 }
500
501 VkFence fence_handle = vn_fence_to_handle(fence);
502 vn_async_vkCreateFence(dev->instance, device, pCreateInfo, NULL,
503 &fence_handle);
504
505 *pFence = fence_handle;
506
507 return VK_SUCCESS;
508 }
509
510 void
vn_DestroyFence(VkDevice device,VkFence _fence,const VkAllocationCallbacks * pAllocator)511 vn_DestroyFence(VkDevice device,
512 VkFence _fence,
513 const VkAllocationCallbacks *pAllocator)
514 {
515 struct vn_device *dev = vn_device_from_handle(device);
516 struct vn_fence *fence = vn_fence_from_handle(_fence);
517 const VkAllocationCallbacks *alloc =
518 pAllocator ? pAllocator : &dev->base.base.alloc;
519
520 if (!fence)
521 return;
522
523 vn_async_vkDestroyFence(dev->instance, device, _fence, NULL);
524
525 vn_sync_payload_release(dev, &fence->permanent);
526 vn_sync_payload_release(dev, &fence->temporary);
527
528 vn_object_base_fini(&fence->base);
529 vk_free(alloc, fence);
530 }
531
532 VkResult
vn_ResetFences(VkDevice device,uint32_t fenceCount,const VkFence * pFences)533 vn_ResetFences(VkDevice device, uint32_t fenceCount, const VkFence *pFences)
534 {
535 struct vn_device *dev = vn_device_from_handle(device);
536
537 /* TODO if the fence is shared-by-ref, this needs to be synchronous */
538 if (false)
539 vn_call_vkResetFences(dev->instance, device, fenceCount, pFences);
540 else
541 vn_async_vkResetFences(dev->instance, device, fenceCount, pFences);
542
543 for (uint32_t i = 0; i < fenceCount; i++) {
544 struct vn_fence *fence = vn_fence_from_handle(pFences[i]);
545 struct vn_sync_payload *perm = &fence->permanent;
546
547 vn_sync_payload_release(dev, &fence->temporary);
548
549 assert(perm->type == VN_SYNC_TYPE_DEVICE_ONLY);
550 fence->payload = perm;
551 }
552
553 return VK_SUCCESS;
554 }
555
556 VkResult
vn_GetFenceStatus(VkDevice device,VkFence _fence)557 vn_GetFenceStatus(VkDevice device, VkFence _fence)
558 {
559 struct vn_device *dev = vn_device_from_handle(device);
560 struct vn_fence *fence = vn_fence_from_handle(_fence);
561 struct vn_sync_payload *payload = fence->payload;
562
563 VkResult result;
564 switch (payload->type) {
565 case VN_SYNC_TYPE_DEVICE_ONLY:
566 result = vn_call_vkGetFenceStatus(dev->instance, device, _fence);
567 break;
568 case VN_SYNC_TYPE_WSI_SIGNALED:
569 result = VK_SUCCESS;
570 break;
571 default:
572 unreachable("unexpected fence payload type");
573 break;
574 }
575
576 return vn_result(dev->instance, result);
577 }
578
579 static VkResult
vn_find_first_signaled_fence(VkDevice device,const VkFence * fences,uint32_t count)580 vn_find_first_signaled_fence(VkDevice device,
581 const VkFence *fences,
582 uint32_t count)
583 {
584 for (uint32_t i = 0; i < count; i++) {
585 VkResult result = vn_GetFenceStatus(device, fences[i]);
586 if (result == VK_SUCCESS || result < 0)
587 return result;
588 }
589 return VK_NOT_READY;
590 }
591
592 static VkResult
vn_remove_signaled_fences(VkDevice device,VkFence * fences,uint32_t * count)593 vn_remove_signaled_fences(VkDevice device, VkFence *fences, uint32_t *count)
594 {
595 uint32_t cur = 0;
596 for (uint32_t i = 0; i < *count; i++) {
597 VkResult result = vn_GetFenceStatus(device, fences[i]);
598 if (result != VK_SUCCESS) {
599 if (result < 0)
600 return result;
601 fences[cur++] = fences[i];
602 }
603 }
604
605 *count = cur;
606 return cur ? VK_NOT_READY : VK_SUCCESS;
607 }
608
609 static VkResult
vn_update_sync_result(VkResult result,int64_t abs_timeout,uint32_t * iter)610 vn_update_sync_result(VkResult result, int64_t abs_timeout, uint32_t *iter)
611 {
612 switch (result) {
613 case VK_NOT_READY:
614 if (abs_timeout != OS_TIMEOUT_INFINITE &&
615 os_time_get_nano() >= abs_timeout)
616 result = VK_TIMEOUT;
617 else
618 vn_relax(iter, "client");
619 break;
620 default:
621 assert(result == VK_SUCCESS || result < 0);
622 break;
623 }
624
625 return result;
626 }
627
628 VkResult
vn_WaitForFences(VkDevice device,uint32_t fenceCount,const VkFence * pFences,VkBool32 waitAll,uint64_t timeout)629 vn_WaitForFences(VkDevice device,
630 uint32_t fenceCount,
631 const VkFence *pFences,
632 VkBool32 waitAll,
633 uint64_t timeout)
634 {
635 struct vn_device *dev = vn_device_from_handle(device);
636 const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
637
638 const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
639 VkResult result = VK_NOT_READY;
640 uint32_t iter = 0;
641 if (fenceCount > 1 && waitAll) {
642 VkFence local_fences[8];
643 VkFence *fences = local_fences;
644 if (fenceCount > ARRAY_SIZE(local_fences)) {
645 fences =
646 vk_alloc(alloc, sizeof(*fences) * fenceCount, VN_DEFAULT_ALIGN,
647 VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
648 if (!fences)
649 return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
650 }
651 memcpy(fences, pFences, sizeof(*fences) * fenceCount);
652
653 while (result == VK_NOT_READY) {
654 result = vn_remove_signaled_fences(device, fences, &fenceCount);
655 result = vn_update_sync_result(result, abs_timeout, &iter);
656 }
657
658 if (fences != local_fences)
659 vk_free(alloc, fences);
660 } else {
661 while (result == VK_NOT_READY) {
662 result = vn_find_first_signaled_fence(device, pFences, fenceCount);
663 result = vn_update_sync_result(result, abs_timeout, &iter);
664 }
665 }
666
667 return vn_result(dev->instance, result);
668 }
669
670 static VkResult
vn_create_sync_file(struct vn_device * dev,int * out_fd)671 vn_create_sync_file(struct vn_device *dev, int *out_fd)
672 {
673 struct vn_renderer_sync *sync;
674 VkResult result = vn_renderer_sync_create(dev->renderer, 0,
675 VN_RENDERER_SYNC_BINARY, &sync);
676 if (result != VK_SUCCESS)
677 return vn_error(dev->instance, result);
678
679 const struct vn_renderer_submit submit = {
680 .batches =
681 &(const struct vn_renderer_submit_batch){
682 .syncs = &sync,
683 .sync_values = &(const uint64_t){ 1 },
684 .sync_count = 1,
685 },
686 .batch_count = 1,
687 };
688 result = vn_renderer_submit(dev->renderer, &submit);
689 if (result != VK_SUCCESS) {
690 vn_renderer_sync_destroy(dev->renderer, sync);
691 return vn_error(dev->instance, result);
692 }
693
694 *out_fd = vn_renderer_sync_export_syncobj(dev->renderer, sync, true);
695 vn_renderer_sync_destroy(dev->renderer, sync);
696
697 return *out_fd >= 0 ? VK_SUCCESS : VK_ERROR_TOO_MANY_OBJECTS;
698 }
699
700 VkResult
vn_ImportFenceFdKHR(VkDevice device,const VkImportFenceFdInfoKHR * pImportFenceFdInfo)701 vn_ImportFenceFdKHR(VkDevice device,
702 const VkImportFenceFdInfoKHR *pImportFenceFdInfo)
703 {
704 struct vn_device *dev = vn_device_from_handle(device);
705 struct vn_fence *fence = vn_fence_from_handle(pImportFenceFdInfo->fence);
706 ASSERTED const bool sync_file = pImportFenceFdInfo->handleType ==
707 VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
708 const int fd = pImportFenceFdInfo->fd;
709
710 assert(dev->instance->experimental.globalFencing);
711 assert(sync_file);
712 if (fd >= 0) {
713 if (sync_wait(fd, -1))
714 return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
715
716 close(fd);
717 }
718
719 /* abuse VN_SYNC_TYPE_WSI_SIGNALED */
720 vn_fence_signal_wsi(dev, fence);
721
722 return VK_SUCCESS;
723 }
724
725 VkResult
vn_GetFenceFdKHR(VkDevice device,const VkFenceGetFdInfoKHR * pGetFdInfo,int * pFd)726 vn_GetFenceFdKHR(VkDevice device,
727 const VkFenceGetFdInfoKHR *pGetFdInfo,
728 int *pFd)
729 {
730 struct vn_device *dev = vn_device_from_handle(device);
731 struct vn_fence *fence = vn_fence_from_handle(pGetFdInfo->fence);
732 const bool sync_file =
733 pGetFdInfo->handleType == VK_EXTERNAL_FENCE_HANDLE_TYPE_SYNC_FD_BIT;
734 struct vn_sync_payload *payload = fence->payload;
735
736 assert(dev->instance->experimental.globalFencing);
737 assert(sync_file);
738 int fd = -1;
739 if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
740 VkResult result = vn_create_sync_file(dev, &fd);
741 if (result != VK_SUCCESS)
742 return vn_error(dev->instance, result);
743 }
744
745 if (sync_file) {
746 vn_sync_payload_release(dev, &fence->temporary);
747 fence->payload = &fence->permanent;
748
749 /* XXX implies reset operation on the host fence */
750 }
751
752 *pFd = fd;
753 return VK_SUCCESS;
754 }
755
756 /* semaphore commands */
757
758 static VkResult
vn_semaphore_init_payloads(struct vn_device * dev,struct vn_semaphore * sem,uint64_t initial_val,const VkAllocationCallbacks * alloc)759 vn_semaphore_init_payloads(struct vn_device *dev,
760 struct vn_semaphore *sem,
761 uint64_t initial_val,
762 const VkAllocationCallbacks *alloc)
763 {
764 sem->permanent.type = VN_SYNC_TYPE_DEVICE_ONLY;
765 sem->temporary.type = VN_SYNC_TYPE_INVALID;
766 sem->payload = &sem->permanent;
767
768 return VK_SUCCESS;
769 }
770
771 static void
vn_semaphore_reset_wsi(struct vn_device * dev,struct vn_semaphore * sem)772 vn_semaphore_reset_wsi(struct vn_device *dev, struct vn_semaphore *sem)
773 {
774 struct vn_sync_payload *perm = &sem->permanent;
775
776 vn_sync_payload_release(dev, &sem->temporary);
777
778 sem->payload = perm;
779 }
780
781 void
vn_semaphore_signal_wsi(struct vn_device * dev,struct vn_semaphore * sem)782 vn_semaphore_signal_wsi(struct vn_device *dev, struct vn_semaphore *sem)
783 {
784 struct vn_sync_payload *temp = &sem->temporary;
785
786 vn_sync_payload_release(dev, temp);
787 temp->type = VN_SYNC_TYPE_WSI_SIGNALED;
788 sem->payload = temp;
789 }
790
791 VkResult
vn_CreateSemaphore(VkDevice device,const VkSemaphoreCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkSemaphore * pSemaphore)792 vn_CreateSemaphore(VkDevice device,
793 const VkSemaphoreCreateInfo *pCreateInfo,
794 const VkAllocationCallbacks *pAllocator,
795 VkSemaphore *pSemaphore)
796 {
797 struct vn_device *dev = vn_device_from_handle(device);
798 const VkAllocationCallbacks *alloc =
799 pAllocator ? pAllocator : &dev->base.base.alloc;
800
801 struct vn_semaphore *sem = vk_zalloc(alloc, sizeof(*sem), VN_DEFAULT_ALIGN,
802 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
803 if (!sem)
804 return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
805
806 vn_object_base_init(&sem->base, VK_OBJECT_TYPE_SEMAPHORE, &dev->base);
807
808 const VkSemaphoreTypeCreateInfo *type_info =
809 vk_find_struct_const(pCreateInfo->pNext, SEMAPHORE_TYPE_CREATE_INFO);
810 uint64_t initial_val = 0;
811 if (type_info && type_info->semaphoreType == VK_SEMAPHORE_TYPE_TIMELINE) {
812 sem->type = VK_SEMAPHORE_TYPE_TIMELINE;
813 initial_val = type_info->initialValue;
814 } else {
815 sem->type = VK_SEMAPHORE_TYPE_BINARY;
816 }
817
818 VkResult result = vn_semaphore_init_payloads(dev, sem, initial_val, alloc);
819 if (result != VK_SUCCESS) {
820 vn_object_base_fini(&sem->base);
821 vk_free(alloc, sem);
822 return vn_error(dev->instance, result);
823 }
824
825 VkSemaphore sem_handle = vn_semaphore_to_handle(sem);
826 vn_async_vkCreateSemaphore(dev->instance, device, pCreateInfo, NULL,
827 &sem_handle);
828
829 *pSemaphore = sem_handle;
830
831 return VK_SUCCESS;
832 }
833
834 void
vn_DestroySemaphore(VkDevice device,VkSemaphore semaphore,const VkAllocationCallbacks * pAllocator)835 vn_DestroySemaphore(VkDevice device,
836 VkSemaphore semaphore,
837 const VkAllocationCallbacks *pAllocator)
838 {
839 struct vn_device *dev = vn_device_from_handle(device);
840 struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
841 const VkAllocationCallbacks *alloc =
842 pAllocator ? pAllocator : &dev->base.base.alloc;
843
844 if (!sem)
845 return;
846
847 vn_async_vkDestroySemaphore(dev->instance, device, semaphore, NULL);
848
849 vn_sync_payload_release(dev, &sem->permanent);
850 vn_sync_payload_release(dev, &sem->temporary);
851
852 vn_object_base_fini(&sem->base);
853 vk_free(alloc, sem);
854 }
855
856 VkResult
vn_GetSemaphoreCounterValue(VkDevice device,VkSemaphore semaphore,uint64_t * pValue)857 vn_GetSemaphoreCounterValue(VkDevice device,
858 VkSemaphore semaphore,
859 uint64_t *pValue)
860 {
861 struct vn_device *dev = vn_device_from_handle(device);
862 struct vn_semaphore *sem = vn_semaphore_from_handle(semaphore);
863 ASSERTED struct vn_sync_payload *payload = sem->payload;
864
865 assert(payload->type == VN_SYNC_TYPE_DEVICE_ONLY);
866 return vn_call_vkGetSemaphoreCounterValue(dev->instance, device, semaphore,
867 pValue);
868 }
869
870 VkResult
vn_SignalSemaphore(VkDevice device,const VkSemaphoreSignalInfo * pSignalInfo)871 vn_SignalSemaphore(VkDevice device, const VkSemaphoreSignalInfo *pSignalInfo)
872 {
873 struct vn_device *dev = vn_device_from_handle(device);
874
875 /* TODO if the semaphore is shared-by-ref, this needs to be synchronous */
876 if (false)
877 vn_call_vkSignalSemaphore(dev->instance, device, pSignalInfo);
878 else
879 vn_async_vkSignalSemaphore(dev->instance, device, pSignalInfo);
880
881 return VK_SUCCESS;
882 }
883
884 static VkResult
vn_find_first_signaled_semaphore(VkDevice device,const VkSemaphore * semaphores,const uint64_t * values,uint32_t count)885 vn_find_first_signaled_semaphore(VkDevice device,
886 const VkSemaphore *semaphores,
887 const uint64_t *values,
888 uint32_t count)
889 {
890 for (uint32_t i = 0; i < count; i++) {
891 uint64_t val = 0;
892 VkResult result =
893 vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
894 if (result != VK_SUCCESS || val >= values[i])
895 return result;
896 }
897 return VK_NOT_READY;
898 }
899
900 static VkResult
vn_remove_signaled_semaphores(VkDevice device,VkSemaphore * semaphores,uint64_t * values,uint32_t * count)901 vn_remove_signaled_semaphores(VkDevice device,
902 VkSemaphore *semaphores,
903 uint64_t *values,
904 uint32_t *count)
905 {
906 uint32_t cur = 0;
907 for (uint32_t i = 0; i < *count; i++) {
908 uint64_t val = 0;
909 VkResult result =
910 vn_GetSemaphoreCounterValue(device, semaphores[i], &val);
911 if (result != VK_SUCCESS)
912 return result;
913 if (val < values[i])
914 semaphores[cur++] = semaphores[i];
915 }
916
917 *count = cur;
918 return cur ? VK_NOT_READY : VK_SUCCESS;
919 }
920
921 VkResult
vn_WaitSemaphores(VkDevice device,const VkSemaphoreWaitInfo * pWaitInfo,uint64_t timeout)922 vn_WaitSemaphores(VkDevice device,
923 const VkSemaphoreWaitInfo *pWaitInfo,
924 uint64_t timeout)
925 {
926 struct vn_device *dev = vn_device_from_handle(device);
927 const VkAllocationCallbacks *alloc = &dev->base.base.alloc;
928
929 const int64_t abs_timeout = os_time_get_absolute_timeout(timeout);
930 VkResult result = VK_NOT_READY;
931 uint32_t iter = 0;
932 if (pWaitInfo->semaphoreCount > 1 &&
933 !(pWaitInfo->flags & VK_SEMAPHORE_WAIT_ANY_BIT)) {
934 uint32_t semaphore_count = pWaitInfo->semaphoreCount;
935 VkSemaphore local_semaphores[8];
936 uint64_t local_values[8];
937 VkSemaphore *semaphores = local_semaphores;
938 uint64_t *values = local_values;
939 if (semaphore_count > ARRAY_SIZE(local_semaphores)) {
940 semaphores = vk_alloc(
941 alloc, (sizeof(*semaphores) + sizeof(*values)) * semaphore_count,
942 VN_DEFAULT_ALIGN, VK_SYSTEM_ALLOCATION_SCOPE_COMMAND);
943 if (!semaphores)
944 return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
945
946 values = (uint64_t *)&semaphores[semaphore_count];
947 }
948 memcpy(semaphores, pWaitInfo->pSemaphores,
949 sizeof(*semaphores) * semaphore_count);
950 memcpy(values, pWaitInfo->pValues, sizeof(*values) * semaphore_count);
951
952 while (result == VK_NOT_READY) {
953 result = vn_remove_signaled_semaphores(device, semaphores, values,
954 &semaphore_count);
955 result = vn_update_sync_result(result, abs_timeout, &iter);
956 }
957
958 if (semaphores != local_semaphores)
959 vk_free(alloc, semaphores);
960 } else {
961 while (result == VK_NOT_READY) {
962 result = vn_find_first_signaled_semaphore(
963 device, pWaitInfo->pSemaphores, pWaitInfo->pValues,
964 pWaitInfo->semaphoreCount);
965 result = vn_update_sync_result(result, abs_timeout, &iter);
966 }
967 }
968
969 return vn_result(dev->instance, result);
970 }
971
972 VkResult
vn_ImportSemaphoreFdKHR(VkDevice device,const VkImportSemaphoreFdInfoKHR * pImportSemaphoreFdInfo)973 vn_ImportSemaphoreFdKHR(
974 VkDevice device, const VkImportSemaphoreFdInfoKHR *pImportSemaphoreFdInfo)
975 {
976 struct vn_device *dev = vn_device_from_handle(device);
977 struct vn_semaphore *sem =
978 vn_semaphore_from_handle(pImportSemaphoreFdInfo->semaphore);
979 ASSERTED const bool sync_file =
980 pImportSemaphoreFdInfo->handleType ==
981 VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
982 const int fd = pImportSemaphoreFdInfo->fd;
983
984 assert(dev->instance->experimental.globalFencing);
985 assert(sync_file);
986 if (fd >= 0) {
987 if (sync_wait(fd, -1))
988 return vn_error(dev->instance, VK_ERROR_INVALID_EXTERNAL_HANDLE);
989
990 close(fd);
991 }
992
993 /* abuse VN_SYNC_TYPE_WSI_SIGNALED */
994 vn_semaphore_signal_wsi(dev, sem);
995
996 return VK_SUCCESS;
997 }
998
999 VkResult
vn_GetSemaphoreFdKHR(VkDevice device,const VkSemaphoreGetFdInfoKHR * pGetFdInfo,int * pFd)1000 vn_GetSemaphoreFdKHR(VkDevice device,
1001 const VkSemaphoreGetFdInfoKHR *pGetFdInfo,
1002 int *pFd)
1003 {
1004 struct vn_device *dev = vn_device_from_handle(device);
1005 struct vn_semaphore *sem = vn_semaphore_from_handle(pGetFdInfo->semaphore);
1006 const bool sync_file =
1007 pGetFdInfo->handleType == VK_EXTERNAL_SEMAPHORE_HANDLE_TYPE_SYNC_FD_BIT;
1008 struct vn_sync_payload *payload = sem->payload;
1009
1010 assert(dev->instance->experimental.globalFencing);
1011 assert(sync_file);
1012 int fd = -1;
1013 if (payload->type == VN_SYNC_TYPE_DEVICE_ONLY) {
1014 VkResult result = vn_create_sync_file(dev, &fd);
1015 if (result != VK_SUCCESS)
1016 return vn_error(dev->instance, result);
1017 }
1018
1019 if (sync_file) {
1020 vn_sync_payload_release(dev, &sem->temporary);
1021 sem->payload = &sem->permanent;
1022
1023 /* XXX implies wait operation on the host semaphore */
1024 }
1025
1026 *pFd = fd;
1027 return VK_SUCCESS;
1028 }
1029
1030 /* event commands */
1031
1032 VkResult
vn_CreateEvent(VkDevice device,const VkEventCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkEvent * pEvent)1033 vn_CreateEvent(VkDevice device,
1034 const VkEventCreateInfo *pCreateInfo,
1035 const VkAllocationCallbacks *pAllocator,
1036 VkEvent *pEvent)
1037 {
1038 struct vn_device *dev = vn_device_from_handle(device);
1039 const VkAllocationCallbacks *alloc =
1040 pAllocator ? pAllocator : &dev->base.base.alloc;
1041
1042 struct vn_event *ev = vk_zalloc(alloc, sizeof(*ev), VN_DEFAULT_ALIGN,
1043 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
1044 if (!ev)
1045 return vn_error(dev->instance, VK_ERROR_OUT_OF_HOST_MEMORY);
1046
1047 vn_object_base_init(&ev->base, VK_OBJECT_TYPE_EVENT, &dev->base);
1048
1049 VkEvent ev_handle = vn_event_to_handle(ev);
1050 vn_async_vkCreateEvent(dev->instance, device, pCreateInfo, NULL,
1051 &ev_handle);
1052
1053 *pEvent = ev_handle;
1054
1055 return VK_SUCCESS;
1056 }
1057
1058 void
vn_DestroyEvent(VkDevice device,VkEvent event,const VkAllocationCallbacks * pAllocator)1059 vn_DestroyEvent(VkDevice device,
1060 VkEvent event,
1061 const VkAllocationCallbacks *pAllocator)
1062 {
1063 struct vn_device *dev = vn_device_from_handle(device);
1064 struct vn_event *ev = vn_event_from_handle(event);
1065 const VkAllocationCallbacks *alloc =
1066 pAllocator ? pAllocator : &dev->base.base.alloc;
1067
1068 if (!ev)
1069 return;
1070
1071 vn_async_vkDestroyEvent(dev->instance, device, event, NULL);
1072
1073 vn_object_base_fini(&ev->base);
1074 vk_free(alloc, ev);
1075 }
1076
1077 VkResult
vn_GetEventStatus(VkDevice device,VkEvent event)1078 vn_GetEventStatus(VkDevice device, VkEvent event)
1079 {
1080 struct vn_device *dev = vn_device_from_handle(device);
1081
1082 /* TODO When the renderer supports it (requires a new vk extension), there
1083 * should be a coherent memory backing the event.
1084 */
1085 VkResult result = vn_call_vkGetEventStatus(dev->instance, device, event);
1086
1087 return vn_result(dev->instance, result);
1088 }
1089
1090 VkResult
vn_SetEvent(VkDevice device,VkEvent event)1091 vn_SetEvent(VkDevice device, VkEvent event)
1092 {
1093 struct vn_device *dev = vn_device_from_handle(device);
1094
1095 VkResult result = vn_call_vkSetEvent(dev->instance, device, event);
1096
1097 return vn_result(dev->instance, result);
1098 }
1099
1100 VkResult
vn_ResetEvent(VkDevice device,VkEvent event)1101 vn_ResetEvent(VkDevice device, VkEvent event)
1102 {
1103 struct vn_device *dev = vn_device_from_handle(device);
1104
1105 VkResult result = vn_call_vkResetEvent(dev->instance, device, event);
1106
1107 return vn_result(dev->instance, result);
1108 }
1109