1 /*
2 * Copyright © 2021 Intel Corporation
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "vk_alloc.h"
25 #include "vk_command_buffer.h"
26 #include "vk_common_entrypoints.h"
27 #include "vk_device.h"
28 #include "vk_queue.h"
29 #include "vk_util.h"
30 #include "../wsi/wsi_common.h"
31
32 VKAPI_ATTR void VKAPI_CALL
vk_common_CmdWriteTimestamp(VkCommandBuffer commandBuffer,VkPipelineStageFlagBits pipelineStage,VkQueryPool queryPool,uint32_t query)33 vk_common_CmdWriteTimestamp(
34 VkCommandBuffer commandBuffer,
35 VkPipelineStageFlagBits pipelineStage,
36 VkQueryPool queryPool,
37 uint32_t query)
38 {
39 VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
40 struct vk_device *device = cmd_buffer->base.device;
41
42 device->dispatch_table.CmdWriteTimestamp2KHR(commandBuffer,
43 (VkPipelineStageFlags2KHR) pipelineStage,
44 queryPool,
45 query);
46 }
47
48 static VkMemoryBarrier2KHR
upgrade_memory_barrier(const VkMemoryBarrier * barrier,VkPipelineStageFlags2KHR src_stage_mask2,VkPipelineStageFlags2KHR dst_stage_mask2)49 upgrade_memory_barrier(const VkMemoryBarrier *barrier,
50 VkPipelineStageFlags2KHR src_stage_mask2,
51 VkPipelineStageFlags2KHR dst_stage_mask2)
52 {
53 return (VkMemoryBarrier2KHR) {
54 .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2_KHR,
55 .srcStageMask = src_stage_mask2,
56 .srcAccessMask = (VkAccessFlags2KHR) barrier->srcAccessMask,
57 .dstStageMask = dst_stage_mask2,
58 .dstAccessMask = (VkAccessFlags2KHR) barrier->dstAccessMask,
59 };
60 }
61
62 static VkBufferMemoryBarrier2KHR
upgrade_buffer_memory_barrier(const VkBufferMemoryBarrier * barrier,VkPipelineStageFlags2KHR src_stage_mask2,VkPipelineStageFlags2KHR dst_stage_mask2)63 upgrade_buffer_memory_barrier(const VkBufferMemoryBarrier *barrier,
64 VkPipelineStageFlags2KHR src_stage_mask2,
65 VkPipelineStageFlags2KHR dst_stage_mask2)
66 {
67 return (VkBufferMemoryBarrier2KHR) {
68 .sType = VK_STRUCTURE_TYPE_BUFFER_MEMORY_BARRIER_2_KHR,
69 .srcStageMask = src_stage_mask2,
70 .srcAccessMask = (VkAccessFlags2KHR) barrier->srcAccessMask,
71 .dstStageMask = dst_stage_mask2,
72 .dstAccessMask = (VkAccessFlags2KHR) barrier->dstAccessMask,
73 .srcQueueFamilyIndex = barrier->srcQueueFamilyIndex,
74 .dstQueueFamilyIndex = barrier->dstQueueFamilyIndex,
75 .buffer = barrier->buffer,
76 .offset = barrier->offset,
77 .size = barrier->size,
78 };
79 }
80
81 static VkImageMemoryBarrier2KHR
upgrade_image_memory_barrier(const VkImageMemoryBarrier * barrier,VkPipelineStageFlags2KHR src_stage_mask2,VkPipelineStageFlags2KHR dst_stage_mask2)82 upgrade_image_memory_barrier(const VkImageMemoryBarrier *barrier,
83 VkPipelineStageFlags2KHR src_stage_mask2,
84 VkPipelineStageFlags2KHR dst_stage_mask2)
85 {
86 return (VkImageMemoryBarrier2KHR) {
87 .sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER_2_KHR,
88 .srcStageMask = src_stage_mask2,
89 .srcAccessMask = (VkAccessFlags2KHR) barrier->srcAccessMask,
90 .dstStageMask = dst_stage_mask2,
91 .dstAccessMask = (VkAccessFlags2KHR) barrier->dstAccessMask,
92 .oldLayout = barrier->oldLayout,
93 .newLayout = barrier->newLayout,
94 .srcQueueFamilyIndex = barrier->srcQueueFamilyIndex,
95 .dstQueueFamilyIndex = barrier->dstQueueFamilyIndex,
96 .image = barrier->image,
97 .subresourceRange = barrier->subresourceRange,
98 };
99 }
100
101 VKAPI_ATTR void VKAPI_CALL
vk_common_CmdPipelineBarrier(VkCommandBuffer commandBuffer,VkPipelineStageFlags srcStageMask,VkPipelineStageFlags dstStageMask,VkDependencyFlags dependencyFlags,uint32_t memoryBarrierCount,const VkMemoryBarrier * pMemoryBarriers,uint32_t bufferMemoryBarrierCount,const VkBufferMemoryBarrier * pBufferMemoryBarriers,uint32_t imageMemoryBarrierCount,const VkImageMemoryBarrier * pImageMemoryBarriers)102 vk_common_CmdPipelineBarrier(
103 VkCommandBuffer commandBuffer,
104 VkPipelineStageFlags srcStageMask,
105 VkPipelineStageFlags dstStageMask,
106 VkDependencyFlags dependencyFlags,
107 uint32_t memoryBarrierCount,
108 const VkMemoryBarrier* pMemoryBarriers,
109 uint32_t bufferMemoryBarrierCount,
110 const VkBufferMemoryBarrier* pBufferMemoryBarriers,
111 uint32_t imageMemoryBarrierCount,
112 const VkImageMemoryBarrier* pImageMemoryBarriers)
113 {
114 VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
115 struct vk_device *device = cmd_buffer->base.device;
116
117 STACK_ARRAY(VkMemoryBarrier2KHR, memory_barriers, memoryBarrierCount);
118 STACK_ARRAY(VkBufferMemoryBarrier2KHR, buffer_barriers, bufferMemoryBarrierCount);
119 STACK_ARRAY(VkImageMemoryBarrier2KHR, image_barriers, imageMemoryBarrierCount);
120
121 VkPipelineStageFlags2KHR src_stage_mask2 = (VkPipelineStageFlags2KHR) srcStageMask;
122 VkPipelineStageFlags2KHR dst_stage_mask2 = (VkPipelineStageFlags2KHR) dstStageMask;
123
124 for (uint32_t i = 0; i < memoryBarrierCount; i++) {
125 memory_barriers[i] = upgrade_memory_barrier(&pMemoryBarriers[i],
126 src_stage_mask2,
127 dst_stage_mask2);
128 }
129 for (uint32_t i = 0; i < bufferMemoryBarrierCount; i++) {
130 buffer_barriers[i] = upgrade_buffer_memory_barrier(&pBufferMemoryBarriers[i],
131 src_stage_mask2,
132 dst_stage_mask2);
133 }
134 for (uint32_t i = 0; i < imageMemoryBarrierCount; i++) {
135 image_barriers[i] = upgrade_image_memory_barrier(&pImageMemoryBarriers[i],
136 src_stage_mask2,
137 dst_stage_mask2);
138 }
139
140 VkDependencyInfoKHR dep_info = {
141 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO_KHR,
142 .memoryBarrierCount = memoryBarrierCount,
143 .pMemoryBarriers = memory_barriers,
144 .bufferMemoryBarrierCount = bufferMemoryBarrierCount,
145 .pBufferMemoryBarriers = buffer_barriers,
146 .imageMemoryBarrierCount = imageMemoryBarrierCount,
147 .pImageMemoryBarriers = image_barriers,
148 };
149
150 device->dispatch_table.CmdPipelineBarrier2KHR(commandBuffer, &dep_info);
151
152 STACK_ARRAY_FINISH(memory_barriers);
153 STACK_ARRAY_FINISH(buffer_barriers);
154 STACK_ARRAY_FINISH(image_barriers);
155 }
156
157 VKAPI_ATTR void VKAPI_CALL
vk_common_CmdSetEvent(VkCommandBuffer commandBuffer,VkEvent event,VkPipelineStageFlags stageMask)158 vk_common_CmdSetEvent(
159 VkCommandBuffer commandBuffer,
160 VkEvent event,
161 VkPipelineStageFlags stageMask)
162 {
163 VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
164 struct vk_device *device = cmd_buffer->base.device;
165
166 VkMemoryBarrier2KHR mem_barrier = {
167 .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2_KHR,
168 .srcStageMask = (VkPipelineStageFlags2KHR) stageMask,
169 .dstStageMask = (VkPipelineStageFlags2KHR) stageMask,
170 };
171 VkDependencyInfoKHR dep_info = {
172 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO_KHR,
173 .memoryBarrierCount = 1,
174 .pMemoryBarriers = &mem_barrier,
175 };
176
177 device->dispatch_table.CmdSetEvent2KHR(commandBuffer, event, &dep_info);
178 }
179
180 VKAPI_ATTR void VKAPI_CALL
vk_common_CmdResetEvent(VkCommandBuffer commandBuffer,VkEvent event,VkPipelineStageFlags stageMask)181 vk_common_CmdResetEvent(
182 VkCommandBuffer commandBuffer,
183 VkEvent event,
184 VkPipelineStageFlags stageMask)
185 {
186 VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
187 struct vk_device *device = cmd_buffer->base.device;
188
189 device->dispatch_table.CmdResetEvent2KHR(commandBuffer,
190 event,
191 (VkPipelineStageFlags2KHR) stageMask);
192 }
193
194 VKAPI_ATTR void VKAPI_CALL
vk_common_CmdWaitEvents(VkCommandBuffer commandBuffer,uint32_t eventCount,const VkEvent * pEvents,VkPipelineStageFlags srcStageMask,VkPipelineStageFlags destStageMask,uint32_t memoryBarrierCount,const VkMemoryBarrier * pMemoryBarriers,uint32_t bufferMemoryBarrierCount,const VkBufferMemoryBarrier * pBufferMemoryBarriers,uint32_t imageMemoryBarrierCount,const VkImageMemoryBarrier * pImageMemoryBarriers)195 vk_common_CmdWaitEvents(
196 VkCommandBuffer commandBuffer,
197 uint32_t eventCount,
198 const VkEvent* pEvents,
199 VkPipelineStageFlags srcStageMask,
200 VkPipelineStageFlags destStageMask,
201 uint32_t memoryBarrierCount,
202 const VkMemoryBarrier* pMemoryBarriers,
203 uint32_t bufferMemoryBarrierCount,
204 const VkBufferMemoryBarrier* pBufferMemoryBarriers,
205 uint32_t imageMemoryBarrierCount,
206 const VkImageMemoryBarrier* pImageMemoryBarriers)
207 {
208 VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
209 struct vk_device *device = cmd_buffer->base.device;
210
211 STACK_ARRAY(VkDependencyInfoKHR, deps, eventCount);
212
213 /* Note that dstStageMask and srcStageMask in the CmdWaitEvent2() call
214 * are the same. This is to match the CmdSetEvent2() call from
215 * vk_common_CmdSetEvent(). The actual src->dst stage barrier will
216 * happen as part of the CmdPipelineBarrier() call below.
217 */
218 VkMemoryBarrier2KHR stage_barrier = {
219 .sType = VK_STRUCTURE_TYPE_MEMORY_BARRIER_2_KHR,
220 .srcStageMask = srcStageMask,
221 .dstStageMask = srcStageMask,
222 };
223
224 for (uint32_t i = 0; i < eventCount; i++) {
225 deps[i] = (VkDependencyInfoKHR) {
226 .sType = VK_STRUCTURE_TYPE_DEPENDENCY_INFO_KHR,
227 .memoryBarrierCount = 1,
228 .pMemoryBarriers = &stage_barrier,
229 };
230 }
231 device->dispatch_table.CmdWaitEvents2KHR(commandBuffer, eventCount, pEvents, deps);
232
233 STACK_ARRAY_FINISH(deps);
234
235 /* Setting dependency to 0 because :
236 *
237 * - For BY_REGION_BIT and VIEW_LOCAL_BIT, events are not allowed inside a
238 * render pass so these don't apply.
239 *
240 * - For DEVICE_GROUP_BIT, we have the following bit of spec text:
241 *
242 * "Semaphore and event dependencies are device-local and only
243 * execute on the one physical device that performs the
244 * dependency."
245 */
246 const VkDependencyFlags dep_flags = 0;
247
248 device->dispatch_table.CmdPipelineBarrier(commandBuffer,
249 srcStageMask, destStageMask,
250 dep_flags,
251 memoryBarrierCount, pMemoryBarriers,
252 bufferMemoryBarrierCount, pBufferMemoryBarriers,
253 imageMemoryBarrierCount, pImageMemoryBarriers);
254 }
255
256 VKAPI_ATTR void VKAPI_CALL
vk_common_CmdWriteBufferMarkerAMD(VkCommandBuffer commandBuffer,VkPipelineStageFlagBits pipelineStage,VkBuffer dstBuffer,VkDeviceSize dstOffset,uint32_t marker)257 vk_common_CmdWriteBufferMarkerAMD(
258 VkCommandBuffer commandBuffer,
259 VkPipelineStageFlagBits pipelineStage,
260 VkBuffer dstBuffer,
261 VkDeviceSize dstOffset,
262 uint32_t marker)
263 {
264 VK_FROM_HANDLE(vk_command_buffer, cmd_buffer, commandBuffer);
265 struct vk_device *device = cmd_buffer->base.device;
266
267 device->dispatch_table.CmdWriteBufferMarker2AMD(commandBuffer,
268 (VkPipelineStageFlags2KHR) pipelineStage,
269 dstBuffer,
270 dstOffset,
271 marker);
272 }
273
274 VKAPI_ATTR void VKAPI_CALL
vk_common_GetQueueCheckpointDataNV(VkQueue queue,uint32_t * pCheckpointDataCount,VkCheckpointDataNV * pCheckpointData)275 vk_common_GetQueueCheckpointDataNV(
276 VkQueue queue,
277 uint32_t* pCheckpointDataCount,
278 VkCheckpointDataNV* pCheckpointData)
279 {
280 unreachable("Entrypoint not implemented");
281 }
282
283 VKAPI_ATTR VkResult VKAPI_CALL
vk_common_QueueSubmit(VkQueue _queue,uint32_t submitCount,const VkSubmitInfo * pSubmits,VkFence fence)284 vk_common_QueueSubmit(
285 VkQueue _queue,
286 uint32_t submitCount,
287 const VkSubmitInfo* pSubmits,
288 VkFence fence)
289 {
290 VK_FROM_HANDLE(vk_queue, queue, _queue);
291 struct vk_device *device = queue->base.device;
292
293 STACK_ARRAY(VkSubmitInfo2KHR, submit_info_2, submitCount);
294 STACK_ARRAY(VkPerformanceQuerySubmitInfoKHR, perf_query_submit_info, submitCount);
295 STACK_ARRAY(struct wsi_memory_signal_submit_info, wsi_mem_submit_info, submitCount);
296
297 uint32_t n_wait_semaphores = 0;
298 uint32_t n_command_buffers = 0;
299 uint32_t n_signal_semaphores = 0;
300 for (uint32_t s = 0; s < submitCount; s++) {
301 n_wait_semaphores += pSubmits[s].waitSemaphoreCount;
302 n_command_buffers += pSubmits[s].commandBufferCount;
303 n_signal_semaphores += pSubmits[s].signalSemaphoreCount;
304 }
305
306 STACK_ARRAY(VkSemaphoreSubmitInfoKHR, wait_semaphores, n_wait_semaphores);
307 STACK_ARRAY(VkCommandBufferSubmitInfoKHR, command_buffers, n_command_buffers);
308 STACK_ARRAY(VkSemaphoreSubmitInfoKHR, signal_semaphores, n_signal_semaphores);
309
310 n_wait_semaphores = 0;
311 n_command_buffers = 0;
312 n_signal_semaphores = 0;
313
314 for (uint32_t s = 0; s < submitCount; s++) {
315 const VkTimelineSemaphoreSubmitInfoKHR *timeline_info =
316 vk_find_struct_const(pSubmits[s].pNext,
317 TIMELINE_SEMAPHORE_SUBMIT_INFO_KHR);
318 const uint64_t *wait_values =
319 timeline_info && timeline_info->waitSemaphoreValueCount ?
320 timeline_info->pWaitSemaphoreValues : NULL;
321 const uint64_t *signal_values =
322 timeline_info && timeline_info->signalSemaphoreValueCount ?
323 timeline_info->pSignalSemaphoreValues : NULL;
324
325 const VkDeviceGroupSubmitInfo *group_info =
326 vk_find_struct_const(pSubmits[s].pNext, DEVICE_GROUP_SUBMIT_INFO);
327
328 for (uint32_t i = 0; i < pSubmits[s].waitSemaphoreCount; i++) {
329 wait_semaphores[n_wait_semaphores + i] = (VkSemaphoreSubmitInfoKHR) {
330 .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO_KHR,
331 .semaphore = pSubmits[s].pWaitSemaphores[i],
332 .value = wait_values ? wait_values[i] : 0,
333 .stageMask = pSubmits[s].pWaitDstStageMask[i],
334 .deviceIndex = group_info ? group_info->pWaitSemaphoreDeviceIndices[i] : 0,
335 };
336 }
337 for (uint32_t i = 0; i < pSubmits[s].commandBufferCount; i++) {
338 command_buffers[n_command_buffers + i] = (VkCommandBufferSubmitInfoKHR) {
339 .sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_SUBMIT_INFO_KHR,
340 .commandBuffer = pSubmits[s].pCommandBuffers[i],
341 .deviceMask = group_info ? group_info->pCommandBufferDeviceMasks[i] : 0,
342 };
343 }
344 for (uint32_t i = 0; i < pSubmits[s].signalSemaphoreCount; i++) {
345 signal_semaphores[n_signal_semaphores + i] = (VkSemaphoreSubmitInfoKHR) {
346 .sType = VK_STRUCTURE_TYPE_SEMAPHORE_SUBMIT_INFO_KHR,
347 .semaphore = pSubmits[s].pSignalSemaphores[i],
348 .value = signal_values ? signal_values[i] : 0,
349 .stageMask = 0,
350 .deviceIndex = group_info ? group_info->pSignalSemaphoreDeviceIndices[i] : 0,
351 };
352 }
353
354 const VkProtectedSubmitInfo *protected_info =
355 vk_find_struct_const(pSubmits[s].pNext, PROTECTED_SUBMIT_INFO);
356
357 submit_info_2[s] = (VkSubmitInfo2KHR) {
358 .sType = VK_STRUCTURE_TYPE_SUBMIT_INFO_2_KHR,
359 .flags = ((protected_info && protected_info->protectedSubmit) ?
360 VK_SUBMIT_PROTECTED_BIT_KHR : 0),
361 .waitSemaphoreInfoCount = pSubmits[s].waitSemaphoreCount,
362 .pWaitSemaphoreInfos = &wait_semaphores[n_wait_semaphores],
363 .commandBufferInfoCount = pSubmits[s].commandBufferCount,
364 .pCommandBufferInfos = &command_buffers[n_command_buffers],
365 .signalSemaphoreInfoCount = pSubmits[s].signalSemaphoreCount,
366 .pSignalSemaphoreInfos = &signal_semaphores[n_signal_semaphores],
367 };
368
369 const VkPerformanceQuerySubmitInfoKHR *query_info =
370 vk_find_struct_const(pSubmits[s].pNext,
371 PERFORMANCE_QUERY_SUBMIT_INFO_KHR);
372 if (query_info) {
373 perf_query_submit_info[s] = *query_info;
374 perf_query_submit_info[s].pNext = NULL;
375 __vk_append_struct(&submit_info_2[s], &perf_query_submit_info[s]);
376 }
377
378 const struct wsi_memory_signal_submit_info *mem_signal_info =
379 vk_find_struct_const(pSubmits[s].pNext,
380 WSI_MEMORY_SIGNAL_SUBMIT_INFO_MESA);
381 if (mem_signal_info) {
382 wsi_mem_submit_info[s] = *mem_signal_info;
383 wsi_mem_submit_info[s].pNext = NULL;
384 __vk_append_struct(&submit_info_2[s], &wsi_mem_submit_info[s]);
385 }
386
387 n_wait_semaphores += pSubmits[s].waitSemaphoreCount;
388 n_command_buffers += pSubmits[s].commandBufferCount;
389 n_signal_semaphores += pSubmits[s].signalSemaphoreCount;
390 }
391
392 VkResult result = device->dispatch_table.QueueSubmit2KHR(_queue,
393 submitCount,
394 submit_info_2,
395 fence);
396
397 STACK_ARRAY_FINISH(wait_semaphores);
398 STACK_ARRAY_FINISH(command_buffers);
399 STACK_ARRAY_FINISH(signal_semaphores);
400 STACK_ARRAY_FINISH(submit_info_2);
401 STACK_ARRAY_FINISH(perf_query_submit_info);
402 STACK_ARRAY_FINISH(wsi_mem_submit_info);
403
404 return result;
405 }
406