1 /*
2 * Copyright © 2016 Red Hat.
3 * Copyright © 2016 Bas Nieuwenhuizen
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice (including the next
13 * paragraph) shall be included in all copies or substantial portions of the
14 * Software.
15 *
16 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
19 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
21 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
22 * DEALINGS IN THE SOFTWARE.
23 */
24
25 /**
26 * @file
27 *
28 * We use the bindless descriptor model, which maps fairly closely to how
29 * Vulkan descriptor sets work. The two exceptions are input attachments and
30 * dynamic descriptors, which have to be patched when recording command
31 * buffers. We reserve an extra descriptor set for these. This descriptor set
32 * contains all the input attachments in the pipeline, in order, and then all
33 * the dynamic descriptors. The dynamic descriptors are stored in the CPU-side
34 * datastructure for each tu_descriptor_set, and then combined into one big
35 * descriptor set at CmdBindDescriptors time/draw time.
36 */
37
38 #include "tu_private.h"
39
40 #include <assert.h>
41 #include <fcntl.h>
42 #include <stdbool.h>
43 #include <string.h>
44 #include <unistd.h>
45
46 #include "util/mesa-sha1.h"
47 #include "vk_descriptors.h"
48 #include "vk_util.h"
49
50 static inline uint8_t *
pool_base(struct tu_descriptor_pool * pool)51 pool_base(struct tu_descriptor_pool *pool)
52 {
53 return pool->host_bo ?: pool->bo.map;
54 }
55
56 static uint32_t
descriptor_size(VkDescriptorType type)57 descriptor_size(VkDescriptorType type)
58 {
59 switch (type) {
60 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
61 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
62 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
63 /* These are remapped to the special driver-managed descriptor set,
64 * hence they don't take up any space in the original descriptor set:
65 * Input attachment doesn't use descriptor sets at all
66 */
67 return 0;
68 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
69 /* We make offsets and sizes all 16 dwords, to match how the hardware
70 * interprets indices passed to sample/load/store instructions in
71 * multiples of 16 dwords. This means that "normal" descriptors are all
72 * of size 16, with padding for smaller descriptors like uniform storage
73 * descriptors which are less than 16 dwords. However combined images
74 * and samplers are actually two descriptors, so they have size 2.
75 */
76 return A6XX_TEX_CONST_DWORDS * 4 * 2;
77 default:
78 return A6XX_TEX_CONST_DWORDS * 4;
79 }
80 }
81
82 static uint32_t
mutable_descriptor_size(const VkMutableDescriptorTypeListVALVE * list)83 mutable_descriptor_size(const VkMutableDescriptorTypeListVALVE *list)
84 {
85 uint32_t max_size = 0;
86
87 /* Since we don't support VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER for
88 * mutable descriptors, max_size should be always A6XX_TEX_CONST_DWORDS * 4.
89 * But we leave this as-is and add an assert.
90 */
91 for (uint32_t i = 0; i < list->descriptorTypeCount; i++) {
92 uint32_t size = descriptor_size(list->pDescriptorTypes[i]);
93 max_size = MAX2(max_size, size);
94 }
95
96 assert(max_size == A6XX_TEX_CONST_DWORDS * 4);
97
98 return max_size;
99 }
100
101 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateDescriptorSetLayout(VkDevice _device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorSetLayout * pSetLayout)102 tu_CreateDescriptorSetLayout(
103 VkDevice _device,
104 const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
105 const VkAllocationCallbacks *pAllocator,
106 VkDescriptorSetLayout *pSetLayout)
107 {
108 TU_FROM_HANDLE(tu_device, device, _device);
109 struct tu_descriptor_set_layout *set_layout;
110
111 assert(pCreateInfo->sType ==
112 VK_STRUCTURE_TYPE_DESCRIPTOR_SET_LAYOUT_CREATE_INFO);
113 const VkDescriptorSetLayoutBindingFlagsCreateInfoEXT *variable_flags =
114 vk_find_struct_const(
115 pCreateInfo->pNext,
116 DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO_EXT);
117 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
118 vk_find_struct_const(
119 pCreateInfo->pNext,
120 MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
121
122 uint32_t num_bindings = 0;
123 uint32_t immutable_sampler_count = 0;
124 uint32_t ycbcr_sampler_count = 0;
125 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
126 num_bindings = MAX2(num_bindings, pCreateInfo->pBindings[j].binding + 1);
127 if ((pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
128 pCreateInfo->pBindings[j].descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
129 pCreateInfo->pBindings[j].pImmutableSamplers) {
130 immutable_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
131
132 bool has_ycbcr_sampler = false;
133 for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
134 if (tu_sampler_from_handle(pCreateInfo->pBindings[j].pImmutableSamplers[i])->ycbcr_sampler)
135 has_ycbcr_sampler = true;
136 }
137
138 if (has_ycbcr_sampler)
139 ycbcr_sampler_count += pCreateInfo->pBindings[j].descriptorCount;
140 }
141 }
142
143 uint32_t samplers_offset =
144 offsetof(struct tu_descriptor_set_layout, binding[num_bindings]);
145
146 /* note: only need to store TEX_SAMP_DWORDS for immutable samples,
147 * but using struct tu_sampler makes things simpler */
148 uint32_t size = samplers_offset +
149 immutable_sampler_count * sizeof(struct tu_sampler) +
150 ycbcr_sampler_count * sizeof(struct tu_sampler_ycbcr_conversion);
151
152 set_layout = vk_object_zalloc(&device->vk, pAllocator, size,
153 VK_OBJECT_TYPE_DESCRIPTOR_SET_LAYOUT);
154 if (!set_layout)
155 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
156
157 set_layout->flags = pCreateInfo->flags;
158
159 /* We just allocate all the immutable samplers at the end of the struct */
160 struct tu_sampler *samplers = (void*) &set_layout->binding[num_bindings];
161 struct tu_sampler_ycbcr_conversion *ycbcr_samplers =
162 (void*) &samplers[immutable_sampler_count];
163
164 VkDescriptorSetLayoutBinding *bindings = NULL;
165 VkResult result = vk_create_sorted_bindings(
166 pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
167 if (result != VK_SUCCESS) {
168 vk_object_free(&device->vk, pAllocator, set_layout);
169 return vk_error(device, result);
170 }
171
172 set_layout->binding_count = num_bindings;
173 set_layout->shader_stages = 0;
174 set_layout->has_immutable_samplers = false;
175 set_layout->size = 0;
176 set_layout->dynamic_ubo = 0;
177
178 uint32_t dynamic_offset_count = 0;
179
180 for (uint32_t j = 0; j < pCreateInfo->bindingCount; j++) {
181 const VkDescriptorSetLayoutBinding *binding = bindings + j;
182 uint32_t b = binding->binding;
183
184 set_layout->binding[b].type = binding->descriptorType;
185 set_layout->binding[b].array_size = binding->descriptorCount;
186 set_layout->binding[b].offset = set_layout->size;
187 set_layout->binding[b].dynamic_offset_offset = dynamic_offset_count;
188 set_layout->binding[b].shader_stages = binding->stageFlags;
189
190 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_VALVE) {
191 /* For mutable descriptor types we must allocate a size that fits the
192 * largest descriptor type that the binding can mutate to.
193 */
194 set_layout->binding[b].size =
195 mutable_descriptor_size(&mutable_info->pMutableDescriptorTypeLists[j]);
196 } else {
197 set_layout->binding[b].size = descriptor_size(binding->descriptorType);
198 }
199
200 if (variable_flags && binding->binding < variable_flags->bindingCount &&
201 (variable_flags->pBindingFlags[binding->binding] &
202 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT_EXT)) {
203 assert(!binding->pImmutableSamplers); /* Terribly ill defined how
204 many samplers are valid */
205 assert(binding->binding == num_bindings - 1);
206
207 set_layout->has_variable_descriptors = true;
208 }
209
210 if ((binding->descriptorType == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER ||
211 binding->descriptorType == VK_DESCRIPTOR_TYPE_SAMPLER) &&
212 binding->pImmutableSamplers) {
213 set_layout->binding[b].immutable_samplers_offset = samplers_offset;
214 set_layout->has_immutable_samplers = true;
215
216 for (uint32_t i = 0; i < binding->descriptorCount; i++)
217 samplers[i] = *tu_sampler_from_handle(binding->pImmutableSamplers[i]);
218
219 samplers += binding->descriptorCount;
220 samplers_offset += sizeof(struct tu_sampler) * binding->descriptorCount;
221
222 bool has_ycbcr_sampler = false;
223 for (unsigned i = 0; i < pCreateInfo->pBindings[j].descriptorCount; ++i) {
224 if (tu_sampler_from_handle(binding->pImmutableSamplers[i])->ycbcr_sampler)
225 has_ycbcr_sampler = true;
226 }
227
228 if (has_ycbcr_sampler) {
229 set_layout->binding[b].ycbcr_samplers_offset =
230 (const char*)ycbcr_samplers - (const char*)set_layout;
231 for (uint32_t i = 0; i < binding->descriptorCount; i++) {
232 struct tu_sampler *sampler = tu_sampler_from_handle(binding->pImmutableSamplers[i]);
233 if (sampler->ycbcr_sampler)
234 ycbcr_samplers[i] = *sampler->ycbcr_sampler;
235 else
236 ycbcr_samplers[i].ycbcr_model = VK_SAMPLER_YCBCR_MODEL_CONVERSION_RGB_IDENTITY;
237 }
238 ycbcr_samplers += binding->descriptorCount;
239 } else {
240 set_layout->binding[b].ycbcr_samplers_offset = 0;
241 }
242 }
243
244 set_layout->size +=
245 binding->descriptorCount * set_layout->binding[b].size;
246 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC ||
247 binding->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC) {
248 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC) {
249 STATIC_ASSERT(MAX_DYNAMIC_BUFFERS <= 8 * sizeof(set_layout->dynamic_ubo));
250 set_layout->dynamic_ubo |=
251 ((1u << binding->descriptorCount) - 1) << dynamic_offset_count;
252 }
253
254 dynamic_offset_count += binding->descriptorCount;
255 }
256
257 set_layout->shader_stages |= binding->stageFlags;
258 }
259
260 free(bindings);
261
262 set_layout->dynamic_offset_count = dynamic_offset_count;
263
264 *pSetLayout = tu_descriptor_set_layout_to_handle(set_layout);
265
266 return VK_SUCCESS;
267 }
268
269 VKAPI_ATTR void VKAPI_CALL
tu_DestroyDescriptorSetLayout(VkDevice _device,VkDescriptorSetLayout _set_layout,const VkAllocationCallbacks * pAllocator)270 tu_DestroyDescriptorSetLayout(VkDevice _device,
271 VkDescriptorSetLayout _set_layout,
272 const VkAllocationCallbacks *pAllocator)
273 {
274 TU_FROM_HANDLE(tu_device, device, _device);
275 TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout, _set_layout);
276
277 if (!set_layout)
278 return;
279
280 vk_object_free(&device->vk, pAllocator, set_layout);
281 }
282
283 VKAPI_ATTR void VKAPI_CALL
tu_GetDescriptorSetLayoutSupport(VkDevice device,const VkDescriptorSetLayoutCreateInfo * pCreateInfo,VkDescriptorSetLayoutSupport * pSupport)284 tu_GetDescriptorSetLayoutSupport(
285 VkDevice device,
286 const VkDescriptorSetLayoutCreateInfo *pCreateInfo,
287 VkDescriptorSetLayoutSupport *pSupport)
288 {
289 VkDescriptorSetLayoutBinding *bindings = NULL;
290 VkResult result = vk_create_sorted_bindings(
291 pCreateInfo->pBindings, pCreateInfo->bindingCount, &bindings);
292 if (result != VK_SUCCESS) {
293 pSupport->supported = false;
294 return;
295 }
296
297 const VkDescriptorSetLayoutBindingFlagsCreateInfoEXT *variable_flags =
298 vk_find_struct_const(
299 pCreateInfo->pNext,
300 DESCRIPTOR_SET_LAYOUT_BINDING_FLAGS_CREATE_INFO_EXT);
301 VkDescriptorSetVariableDescriptorCountLayoutSupportEXT *variable_count =
302 vk_find_struct(
303 (void *) pCreateInfo->pNext,
304 DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_LAYOUT_SUPPORT_EXT);
305 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
306 vk_find_struct_const(
307 pCreateInfo->pNext,
308 MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
309
310 if (variable_count) {
311 variable_count->maxVariableDescriptorCount = 0;
312 }
313
314 bool supported = true;
315 uint64_t size = 0;
316 for (uint32_t i = 0; i < pCreateInfo->bindingCount; i++) {
317 const VkDescriptorSetLayoutBinding *binding = bindings + i;
318
319 uint64_t descriptor_sz;
320
321 if (binding->descriptorType == VK_DESCRIPTOR_TYPE_MUTABLE_VALVE) {
322 const VkMutableDescriptorTypeListVALVE *list =
323 &mutable_info->pMutableDescriptorTypeLists[i];
324
325 for (uint32_t j = 0; j < list->descriptorTypeCount; j++) {
326 /* Don't support the input attachement and combined image sampler type
327 * for mutable descriptors */
328 if (list->pDescriptorTypes[j] == VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT ||
329 list->pDescriptorTypes[j] == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER) {
330 supported = false;
331 goto out;
332 }
333 }
334
335 descriptor_sz =
336 mutable_descriptor_size(&mutable_info->pMutableDescriptorTypeLists[i]);
337 } else {
338 descriptor_sz = descriptor_size(binding->descriptorType);
339 }
340 uint64_t descriptor_alignment = 8;
341
342 if (size && !ALIGN_POT(size, descriptor_alignment)) {
343 supported = false;
344 }
345 size = ALIGN_POT(size, descriptor_alignment);
346
347 uint64_t max_count = UINT64_MAX;
348 if (descriptor_sz)
349 max_count = (UINT64_MAX - size) / descriptor_sz;
350
351 if (max_count < binding->descriptorCount) {
352 supported = false;
353 }
354
355 if (variable_flags && binding->binding < variable_flags->bindingCount &&
356 variable_count &&
357 (variable_flags->pBindingFlags[binding->binding] &
358 VK_DESCRIPTOR_BINDING_VARIABLE_DESCRIPTOR_COUNT_BIT_EXT)) {
359 variable_count->maxVariableDescriptorCount =
360 MIN2(UINT32_MAX, max_count);
361 }
362 size += binding->descriptorCount * descriptor_sz;
363 }
364
365 out:
366 free(bindings);
367
368 pSupport->supported = supported;
369 }
370
371 /*
372 * Pipeline layouts. These have nothing to do with the pipeline. They are
373 * just multiple descriptor set layouts pasted together.
374 */
375
376 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreatePipelineLayout(VkDevice _device,const VkPipelineLayoutCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkPipelineLayout * pPipelineLayout)377 tu_CreatePipelineLayout(VkDevice _device,
378 const VkPipelineLayoutCreateInfo *pCreateInfo,
379 const VkAllocationCallbacks *pAllocator,
380 VkPipelineLayout *pPipelineLayout)
381 {
382 TU_FROM_HANDLE(tu_device, device, _device);
383 struct tu_pipeline_layout *layout;
384
385 assert(pCreateInfo->sType ==
386 VK_STRUCTURE_TYPE_PIPELINE_LAYOUT_CREATE_INFO);
387
388 layout = vk_object_alloc(&device->vk, pAllocator, sizeof(*layout),
389 VK_OBJECT_TYPE_PIPELINE_LAYOUT);
390 if (layout == NULL)
391 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
392
393 layout->num_sets = pCreateInfo->setLayoutCount;
394 layout->dynamic_offset_count = 0;
395
396 unsigned dynamic_offset_count = 0;
397
398 for (uint32_t set = 0; set < pCreateInfo->setLayoutCount; set++) {
399 TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout,
400 pCreateInfo->pSetLayouts[set]);
401 layout->set[set].layout = set_layout;
402 layout->set[set].dynamic_offset_start = dynamic_offset_count;
403 dynamic_offset_count += set_layout->dynamic_offset_count;
404 }
405
406 layout->dynamic_offset_count = dynamic_offset_count;
407 layout->push_constant_size = 0;
408
409 for (unsigned i = 0; i < pCreateInfo->pushConstantRangeCount; ++i) {
410 const VkPushConstantRange *range = pCreateInfo->pPushConstantRanges + i;
411 layout->push_constant_size =
412 MAX2(layout->push_constant_size, range->offset + range->size);
413 }
414
415 layout->push_constant_size = align(layout->push_constant_size, 16);
416 *pPipelineLayout = tu_pipeline_layout_to_handle(layout);
417
418 return VK_SUCCESS;
419 }
420
421 VKAPI_ATTR void VKAPI_CALL
tu_DestroyPipelineLayout(VkDevice _device,VkPipelineLayout _pipelineLayout,const VkAllocationCallbacks * pAllocator)422 tu_DestroyPipelineLayout(VkDevice _device,
423 VkPipelineLayout _pipelineLayout,
424 const VkAllocationCallbacks *pAllocator)
425 {
426 TU_FROM_HANDLE(tu_device, device, _device);
427 TU_FROM_HANDLE(tu_pipeline_layout, pipeline_layout, _pipelineLayout);
428
429 if (!pipeline_layout)
430 return;
431
432 vk_object_free(&device->vk, pAllocator, pipeline_layout);
433 }
434
435 #define EMPTY 1
436
437 static VkResult
tu_descriptor_set_create(struct tu_device * device,struct tu_descriptor_pool * pool,const struct tu_descriptor_set_layout * layout,const uint32_t * variable_count,struct tu_descriptor_set ** out_set)438 tu_descriptor_set_create(struct tu_device *device,
439 struct tu_descriptor_pool *pool,
440 const struct tu_descriptor_set_layout *layout,
441 const uint32_t *variable_count,
442 struct tu_descriptor_set **out_set)
443 {
444 struct tu_descriptor_set *set;
445 unsigned dynamic_offset = sizeof(struct tu_descriptor_set);
446 unsigned mem_size = dynamic_offset +
447 A6XX_TEX_CONST_DWORDS * 4 * layout->dynamic_offset_count;
448
449 if (pool->host_memory_base) {
450 if (pool->host_memory_end - pool->host_memory_ptr < mem_size)
451 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
452
453 set = (struct tu_descriptor_set*)pool->host_memory_ptr;
454 pool->host_memory_ptr += mem_size;
455 } else {
456 set = vk_alloc2(&device->vk.alloc, NULL, mem_size, 8,
457 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
458
459 if (!set)
460 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
461 }
462
463 memset(set, 0, mem_size);
464 vk_object_base_init(&device->vk, &set->base, VK_OBJECT_TYPE_DESCRIPTOR_SET);
465
466 if (layout->dynamic_offset_count) {
467 set->dynamic_descriptors = (uint32_t *)((uint8_t*)set + dynamic_offset);
468 }
469
470 set->layout = layout;
471 set->pool = pool;
472 uint32_t layout_size = layout->size;
473 if (variable_count) {
474 assert(layout->has_variable_descriptors);
475 uint32_t stride = layout->binding[layout->binding_count - 1].size;
476 layout_size = layout->binding[layout->binding_count - 1].offset +
477 *variable_count * stride;
478 }
479
480 if (layout_size) {
481 set->size = layout_size;
482
483 if (!pool->host_memory_base && pool->entry_count == pool->max_entry_count) {
484 vk_object_free(&device->vk, NULL, set);
485 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
486 }
487
488 /* try to allocate linearly first, so that we don't spend
489 * time looking for gaps if the app only allocates &
490 * resets via the pool. */
491 if (pool->current_offset + layout_size <= pool->size) {
492 set->mapped_ptr = (uint32_t*)(pool_base(pool) + pool->current_offset);
493 set->va = pool->host_bo ? 0 : pool->bo.iova + pool->current_offset;
494
495 if (!pool->host_memory_base) {
496 pool->entries[pool->entry_count].offset = pool->current_offset;
497 pool->entries[pool->entry_count].size = layout_size;
498 pool->entries[pool->entry_count].set = set;
499 pool->entry_count++;
500 }
501 pool->current_offset += layout_size;
502 } else if (!pool->host_memory_base) {
503 uint64_t offset = 0;
504 int index;
505
506 for (index = 0; index < pool->entry_count; ++index) {
507 if (pool->entries[index].offset - offset >= layout_size)
508 break;
509 offset = pool->entries[index].offset + pool->entries[index].size;
510 }
511
512 if (pool->size - offset < layout_size) {
513 vk_object_free(&device->vk, NULL, set);
514 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
515 }
516
517 set->mapped_ptr = (uint32_t*)(pool_base(pool) + offset);
518 set->va = pool->host_bo ? 0 : pool->bo.iova + offset;
519
520 memmove(&pool->entries[index + 1], &pool->entries[index],
521 sizeof(pool->entries[0]) * (pool->entry_count - index));
522 pool->entries[index].offset = offset;
523 pool->entries[index].size = layout_size;
524 pool->entries[index].set = set;
525 pool->entry_count++;
526 } else
527 return vk_error(device, VK_ERROR_OUT_OF_POOL_MEMORY);
528 }
529
530 if (layout->has_immutable_samplers) {
531 for (unsigned i = 0; i < layout->binding_count; ++i) {
532 if (!layout->binding[i].immutable_samplers_offset)
533 continue;
534
535 unsigned offset = layout->binding[i].offset / 4;
536 if (layout->binding[i].type == VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER)
537 offset += A6XX_TEX_CONST_DWORDS;
538
539 const struct tu_sampler *samplers =
540 (const struct tu_sampler *)((const char *)layout +
541 layout->binding[i].immutable_samplers_offset);
542 for (unsigned j = 0; j < layout->binding[i].array_size; ++j) {
543 memcpy(set->mapped_ptr + offset, samplers[j].descriptor,
544 sizeof(samplers[j].descriptor));
545 offset += layout->binding[i].size / 4;
546 }
547 }
548 }
549
550 *out_set = set;
551 return VK_SUCCESS;
552 }
553
554 static void
tu_descriptor_set_destroy(struct tu_device * device,struct tu_descriptor_pool * pool,struct tu_descriptor_set * set,bool free_bo)555 tu_descriptor_set_destroy(struct tu_device *device,
556 struct tu_descriptor_pool *pool,
557 struct tu_descriptor_set *set,
558 bool free_bo)
559 {
560 assert(!pool->host_memory_base);
561
562 if (free_bo && set->size && !pool->host_memory_base) {
563 uint32_t offset = (uint8_t*)set->mapped_ptr - pool_base(pool);
564
565 for (int i = 0; i < pool->entry_count; ++i) {
566 if (pool->entries[i].offset == offset) {
567 memmove(&pool->entries[i], &pool->entries[i+1],
568 sizeof(pool->entries[i]) * (pool->entry_count - i - 1));
569 --pool->entry_count;
570 break;
571 }
572 }
573 }
574
575 vk_object_free(&device->vk, NULL, set);
576 }
577
578 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateDescriptorPool(VkDevice _device,const VkDescriptorPoolCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorPool * pDescriptorPool)579 tu_CreateDescriptorPool(VkDevice _device,
580 const VkDescriptorPoolCreateInfo *pCreateInfo,
581 const VkAllocationCallbacks *pAllocator,
582 VkDescriptorPool *pDescriptorPool)
583 {
584 TU_FROM_HANDLE(tu_device, device, _device);
585 struct tu_descriptor_pool *pool;
586 uint64_t size = sizeof(struct tu_descriptor_pool);
587 uint64_t bo_size = 0, bo_count = 0, dynamic_count = 0;
588 VkResult ret;
589
590 const VkMutableDescriptorTypeCreateInfoVALVE *mutable_info =
591 vk_find_struct_const( pCreateInfo->pNext,
592 MUTABLE_DESCRIPTOR_TYPE_CREATE_INFO_VALVE);
593
594 for (unsigned i = 0; i < pCreateInfo->poolSizeCount; ++i) {
595 if (pCreateInfo->pPoolSizes[i].type != VK_DESCRIPTOR_TYPE_SAMPLER)
596 bo_count += pCreateInfo->pPoolSizes[i].descriptorCount;
597
598 switch(pCreateInfo->pPoolSizes[i].type) {
599 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
600 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
601 dynamic_count += pCreateInfo->pPoolSizes[i].descriptorCount;
602 break;
603 case VK_DESCRIPTOR_TYPE_MUTABLE_VALVE:
604 if (mutable_info && i < mutable_info->mutableDescriptorTypeListCount &&
605 mutable_info->pMutableDescriptorTypeLists[i].descriptorTypeCount > 0) {
606 bo_size +=
607 mutable_descriptor_size(&mutable_info->pMutableDescriptorTypeLists[i]) *
608 pCreateInfo->pPoolSizes[i].descriptorCount;
609 } else {
610 /* Allocate the maximum size possible.
611 * Since we don't support VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER for
612 * mutable descriptors, we can set the default size of descriptor types.
613 */
614 bo_size += A6XX_TEX_CONST_DWORDS * 4 *
615 pCreateInfo->pPoolSizes[i].descriptorCount;
616 }
617 continue;
618 default:
619 break;
620 }
621
622 bo_size += descriptor_size(pCreateInfo->pPoolSizes[i].type) *
623 pCreateInfo->pPoolSizes[i].descriptorCount;
624 }
625
626 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
627 uint64_t host_size = pCreateInfo->maxSets * sizeof(struct tu_descriptor_set);
628 host_size += sizeof(struct tu_bo*) * bo_count;
629 host_size += A6XX_TEX_CONST_DWORDS * 4 * dynamic_count;
630 size += host_size;
631 } else {
632 size += sizeof(struct tu_descriptor_pool_entry) * pCreateInfo->maxSets;
633 }
634
635 pool = vk_object_zalloc(&device->vk, pAllocator, size,
636 VK_OBJECT_TYPE_DESCRIPTOR_POOL);
637 if (!pool)
638 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
639
640 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_FREE_DESCRIPTOR_SET_BIT)) {
641 pool->host_memory_base = (uint8_t*)pool + sizeof(struct tu_descriptor_pool);
642 pool->host_memory_ptr = pool->host_memory_base;
643 pool->host_memory_end = (uint8_t*)pool + size;
644 }
645
646 if (bo_size) {
647 if (!(pCreateInfo->flags & VK_DESCRIPTOR_POOL_CREATE_HOST_ONLY_BIT_VALVE)) {
648 ret = tu_bo_init_new(device, &pool->bo, bo_size, TU_BO_ALLOC_ALLOW_DUMP);
649 if (ret)
650 goto fail_alloc;
651
652 ret = tu_bo_map(device, &pool->bo);
653 if (ret)
654 goto fail_map;
655 } else {
656 pool->host_bo = vk_alloc2(&device->vk.alloc, pAllocator, bo_size, 8,
657 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
658 if (!pool->host_bo) {
659 ret = VK_ERROR_OUT_OF_HOST_MEMORY;
660 goto fail_alloc;
661 }
662 }
663 }
664 pool->size = bo_size;
665 pool->max_entry_count = pCreateInfo->maxSets;
666
667 *pDescriptorPool = tu_descriptor_pool_to_handle(pool);
668 return VK_SUCCESS;
669
670 fail_map:
671 tu_bo_finish(device, &pool->bo);
672 fail_alloc:
673 vk_object_free(&device->vk, pAllocator, pool);
674 return ret;
675 }
676
677 VKAPI_ATTR void VKAPI_CALL
tu_DestroyDescriptorPool(VkDevice _device,VkDescriptorPool _pool,const VkAllocationCallbacks * pAllocator)678 tu_DestroyDescriptorPool(VkDevice _device,
679 VkDescriptorPool _pool,
680 const VkAllocationCallbacks *pAllocator)
681 {
682 TU_FROM_HANDLE(tu_device, device, _device);
683 TU_FROM_HANDLE(tu_descriptor_pool, pool, _pool);
684
685 if (!pool)
686 return;
687
688 if (!pool->host_memory_base) {
689 for(int i = 0; i < pool->entry_count; ++i) {
690 tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
691 }
692 }
693
694 if (pool->size) {
695 if (pool->host_bo)
696 vk_free2(&device->vk.alloc, pAllocator, pool->host_bo);
697 else
698 tu_bo_finish(device, &pool->bo);
699 }
700
701 vk_object_free(&device->vk, pAllocator, pool);
702 }
703
704 VKAPI_ATTR VkResult VKAPI_CALL
tu_ResetDescriptorPool(VkDevice _device,VkDescriptorPool descriptorPool,VkDescriptorPoolResetFlags flags)705 tu_ResetDescriptorPool(VkDevice _device,
706 VkDescriptorPool descriptorPool,
707 VkDescriptorPoolResetFlags flags)
708 {
709 TU_FROM_HANDLE(tu_device, device, _device);
710 TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
711
712 if (!pool->host_memory_base) {
713 for(int i = 0; i < pool->entry_count; ++i) {
714 tu_descriptor_set_destroy(device, pool, pool->entries[i].set, false);
715 }
716 pool->entry_count = 0;
717 }
718
719 pool->current_offset = 0;
720 pool->host_memory_ptr = pool->host_memory_base;
721
722 return VK_SUCCESS;
723 }
724
725 VKAPI_ATTR VkResult VKAPI_CALL
tu_AllocateDescriptorSets(VkDevice _device,const VkDescriptorSetAllocateInfo * pAllocateInfo,VkDescriptorSet * pDescriptorSets)726 tu_AllocateDescriptorSets(VkDevice _device,
727 const VkDescriptorSetAllocateInfo *pAllocateInfo,
728 VkDescriptorSet *pDescriptorSets)
729 {
730 TU_FROM_HANDLE(tu_device, device, _device);
731 TU_FROM_HANDLE(tu_descriptor_pool, pool, pAllocateInfo->descriptorPool);
732
733 VkResult result = VK_SUCCESS;
734 uint32_t i;
735 struct tu_descriptor_set *set = NULL;
736
737 const VkDescriptorSetVariableDescriptorCountAllocateInfoEXT *variable_counts =
738 vk_find_struct_const(pAllocateInfo->pNext, DESCRIPTOR_SET_VARIABLE_DESCRIPTOR_COUNT_ALLOCATE_INFO_EXT);
739 const uint32_t zero = 0;
740
741 /* allocate a set of buffers for each shader to contain descriptors */
742 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
743 TU_FROM_HANDLE(tu_descriptor_set_layout, layout,
744 pAllocateInfo->pSetLayouts[i]);
745
746 const uint32_t *variable_count = NULL;
747 if (variable_counts) {
748 if (i < variable_counts->descriptorSetCount)
749 variable_count = variable_counts->pDescriptorCounts + i;
750 else
751 variable_count = &zero;
752 }
753
754 assert(!(layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
755
756 result = tu_descriptor_set_create(device, pool, layout, variable_count, &set);
757 if (result != VK_SUCCESS)
758 break;
759
760 pDescriptorSets[i] = tu_descriptor_set_to_handle(set);
761 }
762
763 if (result != VK_SUCCESS) {
764 tu_FreeDescriptorSets(_device, pAllocateInfo->descriptorPool,
765 i, pDescriptorSets);
766 for (i = 0; i < pAllocateInfo->descriptorSetCount; i++) {
767 pDescriptorSets[i] = VK_NULL_HANDLE;
768 }
769 }
770 return result;
771 }
772
773 VKAPI_ATTR VkResult VKAPI_CALL
tu_FreeDescriptorSets(VkDevice _device,VkDescriptorPool descriptorPool,uint32_t count,const VkDescriptorSet * pDescriptorSets)774 tu_FreeDescriptorSets(VkDevice _device,
775 VkDescriptorPool descriptorPool,
776 uint32_t count,
777 const VkDescriptorSet *pDescriptorSets)
778 {
779 TU_FROM_HANDLE(tu_device, device, _device);
780 TU_FROM_HANDLE(tu_descriptor_pool, pool, descriptorPool);
781
782 for (uint32_t i = 0; i < count; i++) {
783 TU_FROM_HANDLE(tu_descriptor_set, set, pDescriptorSets[i]);
784
785 if (set && !pool->host_memory_base)
786 tu_descriptor_set_destroy(device, pool, set, true);
787 }
788 return VK_SUCCESS;
789 }
790
791 static void
write_texel_buffer_descriptor(uint32_t * dst,const VkBufferView buffer_view)792 write_texel_buffer_descriptor(uint32_t *dst, const VkBufferView buffer_view)
793 {
794 if (buffer_view == VK_NULL_HANDLE) {
795 memset(dst, 0, A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
796 } else {
797 TU_FROM_HANDLE(tu_buffer_view, view, buffer_view);
798
799 memcpy(dst, view->descriptor, sizeof(view->descriptor));
800 }
801 }
802
get_range(struct tu_buffer * buf,VkDeviceSize offset,VkDeviceSize range)803 static uint32_t get_range(struct tu_buffer *buf, VkDeviceSize offset,
804 VkDeviceSize range)
805 {
806 if (range == VK_WHOLE_SIZE) {
807 return buf->size - offset;
808 } else {
809 return range;
810 }
811 }
812
813 static void
write_buffer_descriptor(const struct tu_device * device,uint32_t * dst,const VkDescriptorBufferInfo * buffer_info)814 write_buffer_descriptor(const struct tu_device *device,
815 uint32_t *dst,
816 const VkDescriptorBufferInfo *buffer_info)
817 {
818 if (buffer_info->buffer == VK_NULL_HANDLE) {
819 memset(dst, 0, A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
820 return;
821 }
822
823 TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
824
825 assert((buffer_info->offset & 63) == 0); /* minStorageBufferOffsetAlignment */
826 uint64_t va = tu_buffer_iova(buffer) + buffer_info->offset;
827 uint32_t range = get_range(buffer, buffer_info->offset, buffer_info->range);
828
829 /* newer a6xx allows using 16-bit descriptor for both 16-bit and 32-bit access */
830 if (device->physical_device->info->a6xx.storage_16bit) {
831 dst[0] = A6XX_IBO_0_TILE_MODE(TILE6_LINEAR) | A6XX_IBO_0_FMT(FMT6_16_UINT);
832 dst[1] = DIV_ROUND_UP(range, 2);
833 } else {
834 dst[0] = A6XX_IBO_0_TILE_MODE(TILE6_LINEAR) | A6XX_IBO_0_FMT(FMT6_32_UINT);
835 dst[1] = DIV_ROUND_UP(range, 4);
836 }
837 dst[2] =
838 A6XX_IBO_2_UNK4 | A6XX_IBO_2_TYPE(A6XX_TEX_1D) | A6XX_IBO_2_UNK31;
839 dst[3] = 0;
840 dst[4] = A6XX_IBO_4_BASE_LO(va);
841 dst[5] = A6XX_IBO_5_BASE_HI(va >> 32);
842 for (int i = 6; i < A6XX_TEX_CONST_DWORDS; i++)
843 dst[i] = 0;
844 }
845
846 static void
write_ubo_descriptor(uint32_t * dst,const VkDescriptorBufferInfo * buffer_info)847 write_ubo_descriptor(uint32_t *dst, const VkDescriptorBufferInfo *buffer_info)
848 {
849 if (buffer_info->buffer == VK_NULL_HANDLE) {
850 dst[0] = dst[1] = 0;
851 return;
852 }
853
854 TU_FROM_HANDLE(tu_buffer, buffer, buffer_info->buffer);
855
856 uint32_t range = get_range(buffer, buffer_info->offset, buffer_info->range);
857 /* The HW range is in vec4 units */
858 range = ALIGN_POT(range, 16) / 16;
859 uint64_t va = tu_buffer_iova(buffer) + buffer_info->offset;
860
861 dst[0] = A6XX_UBO_0_BASE_LO(va);
862 dst[1] = A6XX_UBO_1_BASE_HI(va >> 32) | A6XX_UBO_1_SIZE(range);
863 }
864
865 static void
write_image_descriptor(uint32_t * dst,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info)866 write_image_descriptor(uint32_t *dst,
867 VkDescriptorType descriptor_type,
868 const VkDescriptorImageInfo *image_info)
869 {
870 if (image_info->imageView == VK_NULL_HANDLE) {
871 memset(dst, 0, A6XX_TEX_CONST_DWORDS * sizeof(uint32_t));
872 return;
873 }
874
875 TU_FROM_HANDLE(tu_image_view, iview, image_info->imageView);
876
877 if (descriptor_type == VK_DESCRIPTOR_TYPE_STORAGE_IMAGE) {
878 memcpy(dst, iview->storage_descriptor, sizeof(iview->storage_descriptor));
879 } else {
880 memcpy(dst, iview->descriptor, sizeof(iview->descriptor));
881 }
882 }
883
884 static void
write_combined_image_sampler_descriptor(uint32_t * dst,VkDescriptorType descriptor_type,const VkDescriptorImageInfo * image_info,bool has_sampler)885 write_combined_image_sampler_descriptor(uint32_t *dst,
886 VkDescriptorType descriptor_type,
887 const VkDescriptorImageInfo *image_info,
888 bool has_sampler)
889 {
890 write_image_descriptor(dst, descriptor_type, image_info);
891 /* copy over sampler state */
892 if (has_sampler) {
893 TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
894 memcpy(dst + A6XX_TEX_CONST_DWORDS, sampler->descriptor, sizeof(sampler->descriptor));
895 }
896 }
897
898 static void
write_sampler_descriptor(uint32_t * dst,const VkDescriptorImageInfo * image_info)899 write_sampler_descriptor(uint32_t *dst, const VkDescriptorImageInfo *image_info)
900 {
901 TU_FROM_HANDLE(tu_sampler, sampler, image_info->sampler);
902
903 memcpy(dst, sampler->descriptor, sizeof(sampler->descriptor));
904 }
905
906 /* note: this is used with immutable samplers in push descriptors */
907 static void
write_sampler_push(uint32_t * dst,const struct tu_sampler * sampler)908 write_sampler_push(uint32_t *dst, const struct tu_sampler *sampler)
909 {
910 memcpy(dst, sampler->descriptor, sizeof(sampler->descriptor));
911 }
912
913 void
tu_update_descriptor_sets(const struct tu_device * device,VkDescriptorSet dstSetOverride,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)914 tu_update_descriptor_sets(const struct tu_device *device,
915 VkDescriptorSet dstSetOverride,
916 uint32_t descriptorWriteCount,
917 const VkWriteDescriptorSet *pDescriptorWrites,
918 uint32_t descriptorCopyCount,
919 const VkCopyDescriptorSet *pDescriptorCopies)
920 {
921 uint32_t i, j;
922 for (i = 0; i < descriptorWriteCount; i++) {
923 const VkWriteDescriptorSet *writeset = &pDescriptorWrites[i];
924 TU_FROM_HANDLE(tu_descriptor_set, set, dstSetOverride ?: writeset->dstSet);
925 const struct tu_descriptor_set_binding_layout *binding_layout =
926 set->layout->binding + writeset->dstBinding;
927 uint32_t *ptr = set->mapped_ptr;
928 /* for immutable samplers with push descriptors: */
929 const bool copy_immutable_samplers =
930 dstSetOverride && binding_layout->immutable_samplers_offset;
931 const struct tu_sampler *samplers =
932 tu_immutable_samplers(set->layout, binding_layout);
933
934 ptr += binding_layout->offset / 4;
935
936 ptr += (binding_layout->size / 4) * writeset->dstArrayElement;
937 for (j = 0; j < writeset->descriptorCount; ++j) {
938 switch(writeset->descriptorType) {
939 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: {
940 assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
941 unsigned idx = writeset->dstArrayElement + j;
942 idx += binding_layout->dynamic_offset_offset;
943 write_ubo_descriptor(set->dynamic_descriptors + A6XX_TEX_CONST_DWORDS * idx,
944 writeset->pBufferInfo + j);
945 break;
946 }
947 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
948 write_ubo_descriptor(ptr, writeset->pBufferInfo + j);
949 break;
950 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
951 assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
952 unsigned idx = writeset->dstArrayElement + j;
953 idx += binding_layout->dynamic_offset_offset;
954 write_buffer_descriptor(device, set->dynamic_descriptors + A6XX_TEX_CONST_DWORDS * idx,
955 writeset->pBufferInfo + j);
956 break;
957 }
958 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
959 write_buffer_descriptor(device, ptr, writeset->pBufferInfo + j);
960 break;
961 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
962 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
963 write_texel_buffer_descriptor(ptr, writeset->pTexelBufferView[j]);
964 break;
965 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
966 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE:
967 write_image_descriptor(ptr, writeset->descriptorType, writeset->pImageInfo + j);
968 break;
969 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
970 write_combined_image_sampler_descriptor(ptr,
971 writeset->descriptorType,
972 writeset->pImageInfo + j,
973 !binding_layout->immutable_samplers_offset);
974
975 if (copy_immutable_samplers)
976 write_sampler_push(ptr + A6XX_TEX_CONST_DWORDS, &samplers[writeset->dstArrayElement + j]);
977 break;
978 case VK_DESCRIPTOR_TYPE_SAMPLER:
979 if (!binding_layout->immutable_samplers_offset)
980 write_sampler_descriptor(ptr, writeset->pImageInfo + j);
981 else if (copy_immutable_samplers)
982 write_sampler_push(ptr, &samplers[writeset->dstArrayElement + j]);
983 break;
984 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
985 /* nothing in descriptor set - framebuffer state is used instead */
986 break;
987 default:
988 unreachable("unimplemented descriptor type");
989 break;
990 }
991 ptr += binding_layout->size / 4;
992 }
993 }
994
995 for (i = 0; i < descriptorCopyCount; i++) {
996 const VkCopyDescriptorSet *copyset = &pDescriptorCopies[i];
997 TU_FROM_HANDLE(tu_descriptor_set, src_set,
998 copyset->srcSet);
999 TU_FROM_HANDLE(tu_descriptor_set, dst_set,
1000 copyset->dstSet);
1001 const struct tu_descriptor_set_binding_layout *src_binding_layout =
1002 src_set->layout->binding + copyset->srcBinding;
1003 const struct tu_descriptor_set_binding_layout *dst_binding_layout =
1004 dst_set->layout->binding + copyset->dstBinding;
1005 uint32_t *src_ptr = src_set->mapped_ptr;
1006 uint32_t *dst_ptr = dst_set->mapped_ptr;
1007
1008 src_ptr += src_binding_layout->offset / 4;
1009 dst_ptr += dst_binding_layout->offset / 4;
1010
1011 src_ptr += src_binding_layout->size * copyset->srcArrayElement / 4;
1012 dst_ptr += dst_binding_layout->size * copyset->dstArrayElement / 4;
1013
1014 /* In case of copies between mutable descriptor types
1015 * and non-mutable descriptor types.
1016 */
1017 uint32_t copy_size = MIN2(src_binding_layout->size, dst_binding_layout->size);
1018
1019 for (j = 0; j < copyset->descriptorCount; ++j) {
1020 switch (src_binding_layout->type) {
1021 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1022 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1023 unsigned src_idx = copyset->srcArrayElement + j;
1024 unsigned dst_idx = copyset->dstArrayElement + j;
1025 src_idx += src_binding_layout->dynamic_offset_offset;
1026 dst_idx += dst_binding_layout->dynamic_offset_offset;
1027
1028 uint32_t *src_dynamic, *dst_dynamic;
1029 src_dynamic = src_set->dynamic_descriptors + src_idx * A6XX_TEX_CONST_DWORDS;
1030 dst_dynamic = dst_set->dynamic_descriptors + dst_idx * A6XX_TEX_CONST_DWORDS;
1031 memcpy(dst_dynamic, src_dynamic, A6XX_TEX_CONST_DWORDS * 4);
1032 break;
1033 }
1034 default:
1035 memcpy(dst_ptr, src_ptr, copy_size);
1036 }
1037
1038 src_ptr += src_binding_layout->size / 4;
1039 dst_ptr += dst_binding_layout->size / 4;
1040 }
1041 }
1042 }
1043
1044 VKAPI_ATTR void VKAPI_CALL
tu_UpdateDescriptorSets(VkDevice _device,uint32_t descriptorWriteCount,const VkWriteDescriptorSet * pDescriptorWrites,uint32_t descriptorCopyCount,const VkCopyDescriptorSet * pDescriptorCopies)1045 tu_UpdateDescriptorSets(VkDevice _device,
1046 uint32_t descriptorWriteCount,
1047 const VkWriteDescriptorSet *pDescriptorWrites,
1048 uint32_t descriptorCopyCount,
1049 const VkCopyDescriptorSet *pDescriptorCopies)
1050 {
1051 TU_FROM_HANDLE(tu_device, device, _device);
1052 tu_update_descriptor_sets(device, VK_NULL_HANDLE,
1053 descriptorWriteCount, pDescriptorWrites,
1054 descriptorCopyCount, pDescriptorCopies);
1055 }
1056
1057 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateDescriptorUpdateTemplate(VkDevice _device,const VkDescriptorUpdateTemplateCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkDescriptorUpdateTemplate * pDescriptorUpdateTemplate)1058 tu_CreateDescriptorUpdateTemplate(
1059 VkDevice _device,
1060 const VkDescriptorUpdateTemplateCreateInfo *pCreateInfo,
1061 const VkAllocationCallbacks *pAllocator,
1062 VkDescriptorUpdateTemplate *pDescriptorUpdateTemplate)
1063 {
1064 TU_FROM_HANDLE(tu_device, device, _device);
1065 TU_FROM_HANDLE(tu_descriptor_set_layout, set_layout,
1066 pCreateInfo->descriptorSetLayout);
1067 const uint32_t entry_count = pCreateInfo->descriptorUpdateEntryCount;
1068 const size_t size =
1069 sizeof(struct tu_descriptor_update_template) +
1070 sizeof(struct tu_descriptor_update_template_entry) * entry_count;
1071 struct tu_descriptor_update_template *templ;
1072
1073 templ = vk_object_alloc(&device->vk, pAllocator, size,
1074 VK_OBJECT_TYPE_DESCRIPTOR_UPDATE_TEMPLATE);
1075 if (!templ)
1076 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1077
1078 templ->entry_count = entry_count;
1079
1080 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR) {
1081 TU_FROM_HANDLE(tu_pipeline_layout, pipeline_layout, pCreateInfo->pipelineLayout);
1082
1083 /* descriptorSetLayout should be ignored for push descriptors
1084 * and instead it refers to pipelineLayout and set.
1085 */
1086 assert(pCreateInfo->set < MAX_SETS);
1087 set_layout = pipeline_layout->set[pCreateInfo->set].layout;
1088
1089 templ->bind_point = pCreateInfo->pipelineBindPoint;
1090 }
1091
1092 for (uint32_t i = 0; i < entry_count; i++) {
1093 const VkDescriptorUpdateTemplateEntry *entry = &pCreateInfo->pDescriptorUpdateEntries[i];
1094
1095 const struct tu_descriptor_set_binding_layout *binding_layout =
1096 set_layout->binding + entry->dstBinding;
1097 uint32_t dst_offset, dst_stride;
1098 const struct tu_sampler *immutable_samplers = NULL;
1099
1100 /* dst_offset is an offset into dynamic_descriptors when the descriptor
1101 * is dynamic, and an offset into mapped_ptr otherwise.
1102 */
1103 switch (entry->descriptorType) {
1104 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC:
1105 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC:
1106 dst_offset = (binding_layout->dynamic_offset_offset +
1107 entry->dstArrayElement) * A6XX_TEX_CONST_DWORDS;
1108 dst_stride = A6XX_TEX_CONST_DWORDS;
1109 break;
1110 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1111 case VK_DESCRIPTOR_TYPE_SAMPLER:
1112 if (pCreateInfo->templateType == VK_DESCRIPTOR_UPDATE_TEMPLATE_TYPE_PUSH_DESCRIPTORS_KHR &&
1113 binding_layout->immutable_samplers_offset) {
1114 immutable_samplers =
1115 tu_immutable_samplers(set_layout, binding_layout) + entry->dstArrayElement;
1116 }
1117 FALLTHROUGH;
1118 default:
1119 dst_offset = binding_layout->offset / 4;
1120 dst_offset += (binding_layout->size * entry->dstArrayElement) / 4;
1121 dst_stride = binding_layout->size / 4;
1122 }
1123
1124 templ->entry[i] = (struct tu_descriptor_update_template_entry) {
1125 .descriptor_type = entry->descriptorType,
1126 .descriptor_count = entry->descriptorCount,
1127 .src_offset = entry->offset,
1128 .src_stride = entry->stride,
1129 .dst_offset = dst_offset,
1130 .dst_stride = dst_stride,
1131 .has_sampler = !binding_layout->immutable_samplers_offset,
1132 .immutable_samplers = immutable_samplers,
1133 };
1134 }
1135
1136 *pDescriptorUpdateTemplate =
1137 tu_descriptor_update_template_to_handle(templ);
1138
1139 return VK_SUCCESS;
1140 }
1141
1142 VKAPI_ATTR void VKAPI_CALL
tu_DestroyDescriptorUpdateTemplate(VkDevice _device,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const VkAllocationCallbacks * pAllocator)1143 tu_DestroyDescriptorUpdateTemplate(
1144 VkDevice _device,
1145 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1146 const VkAllocationCallbacks *pAllocator)
1147 {
1148 TU_FROM_HANDLE(tu_device, device, _device);
1149 TU_FROM_HANDLE(tu_descriptor_update_template, templ,
1150 descriptorUpdateTemplate);
1151
1152 if (!templ)
1153 return;
1154
1155 vk_object_free(&device->vk, pAllocator, templ);
1156 }
1157
1158 void
tu_update_descriptor_set_with_template(const struct tu_device * device,struct tu_descriptor_set * set,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1159 tu_update_descriptor_set_with_template(
1160 const struct tu_device *device,
1161 struct tu_descriptor_set *set,
1162 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1163 const void *pData)
1164 {
1165 TU_FROM_HANDLE(tu_descriptor_update_template, templ,
1166 descriptorUpdateTemplate);
1167
1168 for (uint32_t i = 0; i < templ->entry_count; i++) {
1169 uint32_t *ptr = set->mapped_ptr;
1170 const void *src = ((const char *) pData) + templ->entry[i].src_offset;
1171 const struct tu_sampler *samplers = templ->entry[i].immutable_samplers;
1172
1173 ptr += templ->entry[i].dst_offset;
1174 unsigned dst_offset = templ->entry[i].dst_offset;
1175 for (unsigned j = 0; j < templ->entry[i].descriptor_count; ++j) {
1176 switch(templ->entry[i].descriptor_type) {
1177 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER_DYNAMIC: {
1178 assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1179 write_ubo_descriptor(set->dynamic_descriptors + dst_offset, src);
1180 break;
1181 }
1182 case VK_DESCRIPTOR_TYPE_UNIFORM_BUFFER:
1183 write_ubo_descriptor(ptr, src);
1184 break;
1185 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER_DYNAMIC: {
1186 assert(!(set->layout->flags & VK_DESCRIPTOR_SET_LAYOUT_CREATE_PUSH_DESCRIPTOR_BIT_KHR));
1187 write_buffer_descriptor(device, set->dynamic_descriptors + dst_offset, src);
1188 break;
1189 }
1190 case VK_DESCRIPTOR_TYPE_STORAGE_BUFFER:
1191 write_buffer_descriptor(device, ptr, src);
1192 break;
1193 case VK_DESCRIPTOR_TYPE_UNIFORM_TEXEL_BUFFER:
1194 case VK_DESCRIPTOR_TYPE_STORAGE_TEXEL_BUFFER:
1195 write_texel_buffer_descriptor(ptr, *(VkBufferView *) src);
1196 break;
1197 case VK_DESCRIPTOR_TYPE_SAMPLED_IMAGE:
1198 case VK_DESCRIPTOR_TYPE_STORAGE_IMAGE: {
1199 write_image_descriptor(ptr, templ->entry[i].descriptor_type, src);
1200 break;
1201 }
1202 case VK_DESCRIPTOR_TYPE_COMBINED_IMAGE_SAMPLER:
1203 write_combined_image_sampler_descriptor(ptr,
1204 templ->entry[i].descriptor_type,
1205 src,
1206 templ->entry[i].has_sampler);
1207 if (samplers)
1208 write_sampler_push(ptr + A6XX_TEX_CONST_DWORDS, &samplers[j]);
1209 break;
1210 case VK_DESCRIPTOR_TYPE_SAMPLER:
1211 if (templ->entry[i].has_sampler)
1212 write_sampler_descriptor(ptr, src);
1213 else if (samplers)
1214 write_sampler_push(ptr, &samplers[j]);
1215 break;
1216 case VK_DESCRIPTOR_TYPE_INPUT_ATTACHMENT:
1217 /* nothing in descriptor set - framebuffer state is used instead */
1218 break;
1219 default:
1220 unreachable("unimplemented descriptor type");
1221 break;
1222 }
1223 src = (char *) src + templ->entry[i].src_stride;
1224 ptr += templ->entry[i].dst_stride;
1225 dst_offset += templ->entry[i].dst_stride;
1226 }
1227 }
1228 }
1229
1230 VKAPI_ATTR void VKAPI_CALL
tu_UpdateDescriptorSetWithTemplate(VkDevice _device,VkDescriptorSet descriptorSet,VkDescriptorUpdateTemplate descriptorUpdateTemplate,const void * pData)1231 tu_UpdateDescriptorSetWithTemplate(
1232 VkDevice _device,
1233 VkDescriptorSet descriptorSet,
1234 VkDescriptorUpdateTemplate descriptorUpdateTemplate,
1235 const void *pData)
1236 {
1237 TU_FROM_HANDLE(tu_device, device, _device);
1238 TU_FROM_HANDLE(tu_descriptor_set, set, descriptorSet);
1239
1240 tu_update_descriptor_set_with_template(device, set, descriptorUpdateTemplate, pData);
1241 }
1242
1243 VKAPI_ATTR VkResult VKAPI_CALL
tu_CreateSamplerYcbcrConversion(VkDevice _device,const VkSamplerYcbcrConversionCreateInfo * pCreateInfo,const VkAllocationCallbacks * pAllocator,VkSamplerYcbcrConversion * pYcbcrConversion)1244 tu_CreateSamplerYcbcrConversion(
1245 VkDevice _device,
1246 const VkSamplerYcbcrConversionCreateInfo *pCreateInfo,
1247 const VkAllocationCallbacks *pAllocator,
1248 VkSamplerYcbcrConversion *pYcbcrConversion)
1249 {
1250 TU_FROM_HANDLE(tu_device, device, _device);
1251 struct tu_sampler_ycbcr_conversion *conversion;
1252
1253 conversion = vk_object_alloc(&device->vk, pAllocator, sizeof(*conversion),
1254 VK_OBJECT_TYPE_SAMPLER_YCBCR_CONVERSION);
1255 if (!conversion)
1256 return vk_error(device, VK_ERROR_OUT_OF_HOST_MEMORY);
1257
1258 conversion->format = pCreateInfo->format;
1259 conversion->ycbcr_model = pCreateInfo->ycbcrModel;
1260 conversion->ycbcr_range = pCreateInfo->ycbcrRange;
1261 conversion->components = pCreateInfo->components;
1262 conversion->chroma_offsets[0] = pCreateInfo->xChromaOffset;
1263 conversion->chroma_offsets[1] = pCreateInfo->yChromaOffset;
1264 conversion->chroma_filter = pCreateInfo->chromaFilter;
1265
1266 *pYcbcrConversion = tu_sampler_ycbcr_conversion_to_handle(conversion);
1267 return VK_SUCCESS;
1268 }
1269
1270 VKAPI_ATTR void VKAPI_CALL
tu_DestroySamplerYcbcrConversion(VkDevice _device,VkSamplerYcbcrConversion ycbcrConversion,const VkAllocationCallbacks * pAllocator)1271 tu_DestroySamplerYcbcrConversion(VkDevice _device,
1272 VkSamplerYcbcrConversion ycbcrConversion,
1273 const VkAllocationCallbacks *pAllocator)
1274 {
1275 TU_FROM_HANDLE(tu_device, device, _device);
1276 TU_FROM_HANDLE(tu_sampler_ycbcr_conversion, ycbcr_conversion, ycbcrConversion);
1277
1278 if (!ycbcr_conversion)
1279 return;
1280
1281 vk_object_free(&device->vk, pAllocator, ycbcr_conversion);
1282 }
1283