1 /*
2 * Copyright © 2020 Raspberry Pi Ltd
3 *
4 * Permission is hereby granted, free of charge, to any person obtaining a
5 * copy of this software and associated documentation files (the "Software"),
6 * to deal in the Software without restriction, including without limitation
7 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8 * and/or sell copies of the Software, and to permit persons to whom the
9 * Software is furnished to do so, subject to the following conditions:
10 *
11 * The above copyright notice and this permission notice (including the next
12 * paragraph) shall be included in all copies or substantial portions of the
13 * Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21 * IN THE SOFTWARE.
22 */
23
24 #include "v3dv_private.h"
25
26 VKAPI_ATTR VkResult VKAPI_CALL
v3dv_CreateQueryPool(VkDevice _device,const VkQueryPoolCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkQueryPool * pQueryPool)27 v3dv_CreateQueryPool(VkDevice _device,
28 const VkQueryPoolCreateInfo *pCreateInfo,
29 const VkAllocationCallbacks *pAllocator,
30 VkQueryPool *pQueryPool)
31 {
32 V3DV_FROM_HANDLE(v3dv_device, device, _device);
33
34 assert(pCreateInfo->queryType == VK_QUERY_TYPE_OCCLUSION ||
35 pCreateInfo->queryType == VK_QUERY_TYPE_TIMESTAMP);
36 assert(pCreateInfo->queryCount > 0);
37
38 struct v3dv_query_pool *pool =
39 vk_object_zalloc(&device->vk, pAllocator, sizeof(*pool),
40 VK_OBJECT_TYPE_QUERY_POOL);
41 if (pool == NULL)
42 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
43
44 pool->query_type = pCreateInfo->queryType;
45 pool->query_count = pCreateInfo->queryCount;
46
47 VkResult result;
48
49 const uint32_t pool_bytes = sizeof(struct v3dv_query) * pool->query_count;
50 pool->queries = vk_alloc2(&device->vk.alloc, pAllocator, pool_bytes, 8,
51 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
52 if (pool->queries == NULL) {
53 result = vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
54 goto fail;
55 }
56
57 if (pool->query_type == VK_QUERY_TYPE_OCCLUSION) {
58 /* The hardware allows us to setup groups of 16 queries in consecutive
59 * 4-byte addresses, requiring only that each group of 16 queries is
60 * aligned to a 1024 byte boundary.
61 */
62 const uint32_t query_groups = DIV_ROUND_UP(pool->query_count, 16);
63 const uint32_t bo_size = query_groups * 1024;
64 pool->bo = v3dv_bo_alloc(device, bo_size, "query", true);
65 if (!pool->bo) {
66 result = vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
67 goto fail;
68 }
69 if (!v3dv_bo_map(device, pool->bo, bo_size)) {
70 result = vk_error(device, VK_ERROR_OUT_OF_DEVICE_MEMORY);
71 goto fail;
72 }
73 }
74
75 uint32_t i;
76 for (i = 0; i < pool->query_count; i++) {
77 pool->queries[i].maybe_available = false;
78 switch (pool->query_type) {
79 case VK_QUERY_TYPE_OCCLUSION: {
80 const uint32_t query_group = i / 16;
81 const uint32_t query_offset = query_group * 1024 + (i % 16) * 4;
82 pool->queries[i].bo = pool->bo;
83 pool->queries[i].offset = query_offset;
84 break;
85 }
86 case VK_QUERY_TYPE_TIMESTAMP:
87 pool->queries[i].value = 0;
88 break;
89 default:
90 unreachable("Unsupported query type");
91 }
92 }
93
94 *pQueryPool = v3dv_query_pool_to_handle(pool);
95
96 return VK_SUCCESS;
97
98 fail:
99 if (pool->bo)
100 v3dv_bo_free(device, pool->bo);
101 if (pool->queries)
102 vk_free2(&device->vk.alloc, pAllocator, pool->queries);
103 vk_object_free(&device->vk, pAllocator, pool);
104
105 return result;
106 }
107
108 VKAPI_ATTR void VKAPI_CALL
v3dv_DestroyQueryPool(VkDevice _device,VkQueryPool queryPool,const VkAllocationCallbacks * pAllocator)109 v3dv_DestroyQueryPool(VkDevice _device,
110 VkQueryPool queryPool,
111 const VkAllocationCallbacks *pAllocator)
112 {
113 V3DV_FROM_HANDLE(v3dv_device, device, _device);
114 V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
115
116 if (!pool)
117 return;
118
119 if (pool->bo)
120 v3dv_bo_free(device, pool->bo);
121
122 if (pool->queries)
123 vk_free2(&device->vk.alloc, pAllocator, pool->queries);
124
125 vk_object_free(&device->vk, pAllocator, pool);
126 }
127
128 static void
write_query_result(void * dst,uint32_t idx,bool do_64bit,uint64_t value)129 write_query_result(void *dst, uint32_t idx, bool do_64bit, uint64_t value)
130 {
131 if (do_64bit) {
132 uint64_t *dst64 = (uint64_t *) dst;
133 dst64[idx] = value;
134 } else {
135 uint32_t *dst32 = (uint32_t *) dst;
136 dst32[idx] = (uint32_t) value;
137 }
138 }
139
140 static VkResult
get_occlusion_query_result(struct v3dv_device * device,struct v3dv_query_pool * pool,uint32_t query,bool do_wait,bool * available,uint64_t * value)141 get_occlusion_query_result(struct v3dv_device *device,
142 struct v3dv_query_pool *pool,
143 uint32_t query,
144 bool do_wait,
145 bool *available,
146 uint64_t *value)
147 {
148 assert(pool && pool->query_type == VK_QUERY_TYPE_OCCLUSION);
149
150 struct v3dv_query *q = &pool->queries[query];
151 assert(q->bo && q->bo->map);
152
153 if (do_wait) {
154 /* From the Vulkan 1.0 spec:
155 *
156 * "If VK_QUERY_RESULT_WAIT_BIT is set, (...) If the query does not
157 * become available in a finite amount of time (e.g. due to not
158 * issuing a query since the last reset), a VK_ERROR_DEVICE_LOST
159 * error may occur."
160 */
161 if (!q->maybe_available)
162 return vk_error(device, VK_ERROR_DEVICE_LOST);
163
164 if (!v3dv_bo_wait(device, q->bo, 0xffffffffffffffffull))
165 return vk_error(device, VK_ERROR_DEVICE_LOST);
166
167 *available = true;
168 } else {
169 *available = q->maybe_available && v3dv_bo_wait(device, q->bo, 0);
170 }
171
172 const uint8_t *query_addr = ((uint8_t *) q->bo->map) + q->offset;
173 *value = (uint64_t) *((uint32_t *)query_addr);
174 return VK_SUCCESS;
175 }
176
177 static VkResult
get_timestamp_query_result(struct v3dv_device * device,struct v3dv_query_pool * pool,uint32_t query,bool do_wait,bool * available,uint64_t * value)178 get_timestamp_query_result(struct v3dv_device *device,
179 struct v3dv_query_pool *pool,
180 uint32_t query,
181 bool do_wait,
182 bool *available,
183 uint64_t *value)
184 {
185 assert(pool && pool->query_type == VK_QUERY_TYPE_TIMESTAMP);
186
187 struct v3dv_query *q = &pool->queries[query];
188
189 if (do_wait) {
190 /* From the Vulkan 1.0 spec:
191 *
192 * "If VK_QUERY_RESULT_WAIT_BIT is set, (...) If the query does not
193 * become available in a finite amount of time (e.g. due to not
194 * issuing a query since the last reset), a VK_ERROR_DEVICE_LOST
195 * error may occur."
196 */
197 if (!q->maybe_available)
198 return vk_error(device, VK_ERROR_DEVICE_LOST);
199
200 *available = true;
201 } else {
202 *available = q->maybe_available;
203 }
204
205 *value = q->value;
206 return VK_SUCCESS;
207 }
208
209 static VkResult
get_query_result(struct v3dv_device * device,struct v3dv_query_pool * pool,uint32_t query,bool do_wait,bool * available,uint64_t * value)210 get_query_result(struct v3dv_device *device,
211 struct v3dv_query_pool *pool,
212 uint32_t query,
213 bool do_wait,
214 bool *available,
215 uint64_t *value)
216 {
217 switch (pool->query_type) {
218 case VK_QUERY_TYPE_OCCLUSION:
219 return get_occlusion_query_result(device, pool, query, do_wait,
220 available, value);
221 case VK_QUERY_TYPE_TIMESTAMP:
222 return get_timestamp_query_result(device, pool, query, do_wait,
223 available, value);
224 default:
225 unreachable("Unsupported query type");
226 }
227 }
228
229 VkResult
v3dv_get_query_pool_results_cpu(struct v3dv_device * device,struct v3dv_query_pool * pool,uint32_t first,uint32_t count,void * data,VkDeviceSize stride,VkQueryResultFlags flags)230 v3dv_get_query_pool_results_cpu(struct v3dv_device *device,
231 struct v3dv_query_pool *pool,
232 uint32_t first,
233 uint32_t count,
234 void *data,
235 VkDeviceSize stride,
236 VkQueryResultFlags flags)
237 {
238 assert(first < pool->query_count);
239 assert(first + count <= pool->query_count);
240 assert(data);
241
242 const bool do_64bit = flags & VK_QUERY_RESULT_64_BIT;
243 const bool do_wait = flags & VK_QUERY_RESULT_WAIT_BIT;
244 const bool do_partial = flags & VK_QUERY_RESULT_PARTIAL_BIT;
245
246 VkResult result = VK_SUCCESS;
247 for (uint32_t i = first; i < first + count; i++) {
248 bool available = false;
249 uint64_t value = 0;
250 VkResult query_result =
251 get_query_result(device, pool, i, do_wait, &available, &value);
252 if (query_result == VK_ERROR_DEVICE_LOST)
253 result = VK_ERROR_DEVICE_LOST;
254
255 /**
256 * From the Vulkan 1.0 spec:
257 *
258 * "If VK_QUERY_RESULT_WAIT_BIT and VK_QUERY_RESULT_PARTIAL_BIT are
259 * both not set then no result values are written to pData for queries
260 * that are in the unavailable state at the time of the call, and
261 * vkGetQueryPoolResults returns VK_NOT_READY. However, availability
262 * state is still written to pData for those queries if
263 * VK_QUERY_RESULT_WITH_AVAILABILITY_BIT is set."
264 */
265 uint32_t slot = 0;
266
267 const bool write_result = available || do_partial;
268 if (write_result)
269 write_query_result(data, slot, do_64bit, value);
270 slot++;
271
272 if (flags & VK_QUERY_RESULT_WITH_AVAILABILITY_BIT)
273 write_query_result(data, slot++, do_64bit, available ? 1u : 0u);
274
275 if (!write_result && result != VK_ERROR_DEVICE_LOST)
276 result = VK_NOT_READY;
277
278 data += stride;
279 }
280
281 return result;
282 }
283
284 VKAPI_ATTR VkResult VKAPI_CALL
v3dv_GetQueryPoolResults(VkDevice _device,VkQueryPool queryPool,uint32_t firstQuery,uint32_t queryCount,size_t dataSize,void * pData,VkDeviceSize stride,VkQueryResultFlags flags)285 v3dv_GetQueryPoolResults(VkDevice _device,
286 VkQueryPool queryPool,
287 uint32_t firstQuery,
288 uint32_t queryCount,
289 size_t dataSize,
290 void *pData,
291 VkDeviceSize stride,
292 VkQueryResultFlags flags)
293 {
294 V3DV_FROM_HANDLE(v3dv_device, device, _device);
295 V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
296
297 return v3dv_get_query_pool_results_cpu(device, pool, firstQuery, queryCount,
298 pData, stride, flags);
299 }
300
301 VKAPI_ATTR void VKAPI_CALL
v3dv_CmdResetQueryPool(VkCommandBuffer commandBuffer,VkQueryPool queryPool,uint32_t firstQuery,uint32_t queryCount)302 v3dv_CmdResetQueryPool(VkCommandBuffer commandBuffer,
303 VkQueryPool queryPool,
304 uint32_t firstQuery,
305 uint32_t queryCount)
306 {
307 V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
308 V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
309
310 v3dv_cmd_buffer_reset_queries(cmd_buffer, pool, firstQuery, queryCount);
311 }
312
313 VKAPI_ATTR void VKAPI_CALL
v3dv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer,VkQueryPool queryPool,uint32_t firstQuery,uint32_t queryCount,VkBuffer dstBuffer,VkDeviceSize dstOffset,VkDeviceSize stride,VkQueryResultFlags flags)314 v3dv_CmdCopyQueryPoolResults(VkCommandBuffer commandBuffer,
315 VkQueryPool queryPool,
316 uint32_t firstQuery,
317 uint32_t queryCount,
318 VkBuffer dstBuffer,
319 VkDeviceSize dstOffset,
320 VkDeviceSize stride,
321 VkQueryResultFlags flags)
322 {
323 V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
324 V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
325 V3DV_FROM_HANDLE(v3dv_buffer, dst, dstBuffer);
326
327 v3dv_cmd_buffer_copy_query_results(cmd_buffer, pool,
328 firstQuery, queryCount,
329 dst, dstOffset, stride, flags);
330 }
331
332 VKAPI_ATTR void VKAPI_CALL
v3dv_CmdBeginQuery(VkCommandBuffer commandBuffer,VkQueryPool queryPool,uint32_t query,VkQueryControlFlags flags)333 v3dv_CmdBeginQuery(VkCommandBuffer commandBuffer,
334 VkQueryPool queryPool,
335 uint32_t query,
336 VkQueryControlFlags flags)
337 {
338 V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
339 V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
340
341 v3dv_cmd_buffer_begin_query(cmd_buffer, pool, query, flags);
342 }
343
344 VKAPI_ATTR void VKAPI_CALL
v3dv_CmdEndQuery(VkCommandBuffer commandBuffer,VkQueryPool queryPool,uint32_t query)345 v3dv_CmdEndQuery(VkCommandBuffer commandBuffer,
346 VkQueryPool queryPool,
347 uint32_t query)
348 {
349 V3DV_FROM_HANDLE(v3dv_cmd_buffer, cmd_buffer, commandBuffer);
350 V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
351
352 v3dv_cmd_buffer_end_query(cmd_buffer, pool, query);
353 }
354
355 void
v3dv_reset_query_pools(struct v3dv_device * device,struct v3dv_query_pool * pool,uint32_t first,uint32_t count)356 v3dv_reset_query_pools(struct v3dv_device *device,
357 struct v3dv_query_pool *pool,
358 uint32_t first,
359 uint32_t count)
360 {
361 for (uint32_t i = first; i < first + count; i++) {
362 assert(i < pool->query_count);
363 struct v3dv_query *q = &pool->queries[i];
364 q->maybe_available = false;
365 switch (pool->query_type) {
366 case VK_QUERY_TYPE_OCCLUSION: {
367 const uint8_t *q_addr = ((uint8_t *) q->bo->map) + q->offset;
368 uint32_t *counter = (uint32_t *) q_addr;
369 *counter = 0;
370 break;
371 }
372 case VK_QUERY_TYPE_TIMESTAMP:
373 q->value = 0;
374 break;
375 default:
376 unreachable("Unsupported query type");
377 }
378 }
379 }
380
381 VKAPI_ATTR void VKAPI_CALL
v3dv_ResetQueryPool(VkDevice _device,VkQueryPool queryPool,uint32_t firstQuery,uint32_t queryCount)382 v3dv_ResetQueryPool(VkDevice _device,
383 VkQueryPool queryPool,
384 uint32_t firstQuery,
385 uint32_t queryCount)
386 {
387 V3DV_FROM_HANDLE(v3dv_device, device, _device);
388 V3DV_FROM_HANDLE(v3dv_query_pool, pool, queryPool);
389
390 v3dv_reset_query_pools(device, pool, firstQuery, queryCount);
391 }
392