1 //
2 // Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved.
3 //
4 // Permission is hereby granted, free of charge, to any person obtaining a copy
5 // of this software and associated documentation files (the "Software"), to deal
6 // in the Software without restriction, including without limitation the rights
7 // to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
8 // copies of the Software, and to permit persons to whom the Software is
9 // furnished to do so, subject to the following conditions:
10 //
11 // The above copyright notice and this permission notice shall be included in
12 // all copies or substantial portions of the Software.
13 //
14 // THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
15 // IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
16 // FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
17 // AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
18 // LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
19 // OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
20 // THE SOFTWARE.
21 //
22
23 #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H
24 #define AMD_VULKAN_MEMORY_ALLOCATOR_H
25
26 #ifdef __cplusplus
27 extern "C" {
28 #endif
29
30 /** \mainpage Vulkan Memory Allocator
31
32 <b>Version 2.0.0</b> (2018-03-19)
33
34 Copyright (c) 2017-2018 Advanced Micro Devices, Inc. All rights reserved. \n
35 License: MIT
36
37 Documentation of all members: vk_mem_alloc.h
38
39 \section main_table_of_contents Table of contents
40
41 - <b>User guide</b>
42 - \subpage quick_start
43 - [Project setup](@ref quick_start_project_setup)
44 - [Initialization](@ref quick_start_initialization)
45 - [Resource allocation](@ref quick_start_resource_allocation)
46 - \subpage choosing_memory_type
47 - [Usage](@ref choosing_memory_type_usage)
48 - [Required and preferred flags](@ref choosing_memory_type_required_preferred_flags)
49 - [Explicit memory types](@ref choosing_memory_type_explicit_memory_types)
50 - [Custom memory pools](@ref choosing_memory_type_custom_memory_pools)
51 - \subpage memory_mapping
52 - [Mapping functions](@ref memory_mapping_mapping_functions)
53 - [Persistently mapped memory](@ref memory_mapping_persistently_mapped_memory)
54 - [Cache control](@ref memory_mapping_cache_control)
55 - [Finding out if memory is mappable](@ref memory_mapping_finding_if_memory_mappable)
56 - \subpage custom_memory_pools
57 - [Choosing memory type index](@ref custom_memory_pools_MemTypeIndex)
58 - \subpage defragmentation
59 - \subpage lost_allocations
60 - \subpage statistics
61 - [Numeric statistics](@ref statistics_numeric_statistics)
62 - [JSON dump](@ref statistics_json_dump)
63 - \subpage allocation_annotation
64 - [Allocation user data](@ref allocation_user_data)
65 - [Allocation names](@ref allocation_names)
66 - \subpage usage_patterns
67 - [Simple patterns](@ref usage_patterns_simple)
68 - [Advanced patterns](@ref usage_patterns_advanced)
69 - \subpage configuration
70 - [Pointers to Vulkan functions](@ref config_Vulkan_functions)
71 - [Custom host memory allocator](@ref custom_memory_allocator)
72 - [Device memory allocation callbacks](@ref allocation_callbacks)
73 - [Device heap memory limit](@ref heap_memory_limit)
74 - \subpage vk_khr_dedicated_allocation
75 - \subpage general_considerations
76 - [Thread safety](@ref general_considerations_thread_safety)
77 - [Allocation algorithm](@ref general_considerations_allocation_algorithm)
78 - [Features not supported](@ref general_considerations_features_not_supported)
79
80 \section main_see_also See also
81
82 - [Product page on GPUOpen](https://gpuopen.com/gaming-product/vulkan-memory-allocator/)
83 - [Source repository on GitHub](https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator)
84
85
86
87
88 \page quick_start Quick start
89
90 \section quick_start_project_setup Project setup
91
92 Vulkan Memory Allocator comes in form of a single header file.
93 You don't need to build it as a separate library project.
94 You can add this file directly to your project and submit it to code repository next to your other source files.
95
96 "Single header" doesn't mean that everything is contained in C/C++ declarations,
97 like it tends to be in case of inline functions or C++ templates.
98 It means that implementation is bundled with interface in a single file and needs to be extracted using preprocessor macro.
99 If you don't do it properly, you will get linker errors.
100
101 To do it properly:
102
103 -# Include "vk_mem_alloc.h" file in each CPP file where you want to use the library.
104 This includes declarations of all members of the library.
105 -# In exacly one CPP file define following macro before this include.
106 It enables also internal definitions.
107
108 \code
109 #define VMA_IMPLEMENTATION
110 #include "vk_mem_alloc.h"
111 \endcode
112
113 It may be a good idea to create dedicated CPP file just for this purpose.
114
115 \section quick_start_initialization Initialization
116
117 At program startup:
118
119 -# Initialize Vulkan to have `VkPhysicalDevice` and `VkDevice` object.
120 -# Fill VmaAllocatorCreateInfo structure and create #VmaAllocator object by
121 calling vmaCreateAllocator().
122
123 \code
124 VmaAllocatorCreateInfo allocatorInfo = {};
125 allocatorInfo.physicalDevice = physicalDevice;
126 allocatorInfo.device = device;
127
128 VmaAllocator allocator;
129 vmaCreateAllocator(&allocatorInfo, &allocator);
130 \endcode
131
132 \section quick_start_resource_allocation Resource allocation
133
134 When you want to create a buffer or image:
135
136 -# Fill `VkBufferCreateInfo` / `VkImageCreateInfo` structure.
137 -# Fill VmaAllocationCreateInfo structure.
138 -# Call vmaCreateBuffer() / vmaCreateImage() to get `VkBuffer`/`VkImage` with memory
139 already allocated and bound to it.
140
141 \code
142 VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
143 bufferInfo.size = 65536;
144 bufferInfo.usage = VK_BUFFER_USAGE_VERTEX_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
145
146 VmaAllocationCreateInfo allocInfo = {};
147 allocInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
148
149 VkBuffer buffer;
150 VmaAllocation allocation;
151 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
152 \endcode
153
154 Don't forget to destroy your objects when no longer needed:
155
156 \code
157 vmaDestroyBuffer(allocator, buffer, allocation);
158 vmaDestroyAllocator(allocator);
159 \endcode
160
161
162 \page choosing_memory_type Choosing memory type
163
164 Physical devices in Vulkan support various combinations of memory heaps and
165 types. Help with choosing correct and optimal memory type for your specific
166 resource is one of the key features of this library. You can use it by filling
167 appropriate members of VmaAllocationCreateInfo structure, as described below.
168 You can also combine multiple methods.
169
170 -# If you just want to find memory type index that meets your requirements, you
171 can use function vmaFindMemoryTypeIndex().
172 -# If you want to allocate a region of device memory without association with any
173 specific image or buffer, you can use function vmaAllocateMemory(). Usage of
174 this function is not recommended and usually not needed.
175 -# If you already have a buffer or an image created, you want to allocate memory
176 for it and then you will bind it yourself, you can use function
177 vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage().
178 For binding you should use functions: vmaBindBufferMemory(), vmaBindImageMemory().
179 -# If you want to create a buffer or an image, allocate memory for it and bind
180 them together, all in one call, you can use function vmaCreateBuffer(),
181 vmaCreateImage(). This is the recommended way to use this library.
182
183 When using 3. or 4., the library internally queries Vulkan for memory types
184 supported for that buffer or image (function `vkGetBufferMemoryRequirements()`)
185 and uses only one of these types.
186
187 If no memory type can be found that meets all the requirements, these functions
188 return `VK_ERROR_FEATURE_NOT_PRESENT`.
189
190 You can leave VmaAllocationCreateInfo structure completely filled with zeros.
191 It means no requirements are specified for memory type.
192 It is valid, although not very useful.
193
194 \section choosing_memory_type_usage Usage
195
196 The easiest way to specify memory requirements is to fill member
197 VmaAllocationCreateInfo::usage using one of the values of enum #VmaMemoryUsage.
198 It defines high level, common usage types.
199 For more details, see description of this enum.
200
201 For example, if you want to create a uniform buffer that will be filled using
202 transfer only once or infrequently and used for rendering every frame, you can
203 do it using following code:
204
205 \code
206 VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
207 bufferInfo.size = 65536;
208 bufferInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
209
210 VmaAllocationCreateInfo allocInfo = {};
211 allocInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
212
213 VkBuffer buffer;
214 VmaAllocation allocation;
215 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
216 \endcode
217
218 \section choosing_memory_type_required_preferred_flags Required and preferred flags
219
220 You can specify more detailed requirements by filling members
221 VmaAllocationCreateInfo::requiredFlags and VmaAllocationCreateInfo::preferredFlags
222 with a combination of bits from enum `VkMemoryPropertyFlags`. For example,
223 if you want to create a buffer that will be persistently mapped on host (so it
224 must be `HOST_VISIBLE`) and preferably will also be `HOST_COHERENT` and `HOST_CACHED`,
225 use following code:
226
227 \code
228 VmaAllocationCreateInfo allocInfo = {};
229 allocInfo.requiredFlags = VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
230 allocInfo.preferredFlags = VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
231 allocInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
232
233 VkBuffer buffer;
234 VmaAllocation allocation;
235 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
236 \endcode
237
238 A memory type is chosen that has all the required flags and as many preferred
239 flags set as possible.
240
241 If you use VmaAllocationCreateInfo::usage, it is just internally converted to
242 a set of required and preferred flags.
243
244 \section choosing_memory_type_explicit_memory_types Explicit memory types
245
246 If you inspected memory types available on the physical device and you have
247 a preference for memory types that you want to use, you can fill member
248 VmaAllocationCreateInfo::memoryTypeBits. It is a bit mask, where each bit set
249 means that a memory type with that index is allowed to be used for the
250 allocation. Special value 0, just like `UINT32_MAX`, means there are no
251 restrictions to memory type index.
252
253 Please note that this member is NOT just a memory type index.
254 Still you can use it to choose just one, specific memory type.
255 For example, if you already determined that your buffer should be created in
256 memory type 2, use following code:
257
258 \code
259 uint32_t memoryTypeIndex = 2;
260
261 VmaAllocationCreateInfo allocInfo = {};
262 allocInfo.memoryTypeBits = 1u << memoryTypeIndex;
263
264 VkBuffer buffer;
265 VmaAllocation allocation;
266 vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr);
267 \endcode
268
269 \section choosing_memory_type_custom_memory_pools Custom memory pools
270
271 If you allocate from custom memory pool, all the ways of specifying memory
272 requirements described above are not applicable and the aforementioned members
273 of VmaAllocationCreateInfo structure are ignored. Memory type is selected
274 explicitly when creating the pool and then used to make all the allocations from
275 that pool. For further details, see \ref custom_memory_pools.
276
277
278 \page memory_mapping Memory mapping
279
280 To "map memory" in Vulkan means to obtain a CPU pointer to `VkDeviceMemory`,
281 to be able to read from it or write to it in CPU code.
282 Mapping is possible only of memory allocated from a memory type that has
283 `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag.
284 Functions `vkMapMemory()`, `vkUnmapMemory()` are designed for this purpose.
285 You can use them directly with memory allocated by this library,
286 but it is not recommended because of following issue:
287 Mapping the same `VkDeviceMemory` block multiple times is illegal - only one mapping at a time is allowed.
288 This includes mapping disjoint regions. Mapping is not reference-counted internally by Vulkan.
289 Because of this, Vulkan Memory Allocator provides following facilities:
290
291 \section memory_mapping_mapping_functions Mapping functions
292
293 The library provides following functions for mapping of a specific #VmaAllocation: vmaMapMemory(), vmaUnmapMemory().
294 They are safer and more convenient to use than standard Vulkan functions.
295 You can map an allocation multiple times simultaneously - mapping is reference-counted internally.
296 You can also map different allocations simultaneously regardless of whether they use the same `VkDeviceMemory` block.
297 They way it's implemented is that the library always maps entire memory block, not just region of the allocation.
298 For further details, see description of vmaMapMemory() function.
299 Example:
300
301 \code
302 // Having these objects initialized:
303
304 struct ConstantBuffer
305 {
306 ...
307 };
308 ConstantBuffer constantBufferData;
309
310 VmaAllocator allocator;
311 VmaBuffer constantBuffer;
312 VmaAllocation constantBufferAllocation;
313
314 // You can map and fill your buffer using following code:
315
316 void* mappedData;
317 vmaMapMemory(allocator, constantBufferAllocation, &mappedData);
318 memcpy(mappedData, &constantBufferData, sizeof(constantBufferData));
319 vmaUnmapMemory(allocator, constantBufferAllocation);
320 \endcode
321
322 \section memory_mapping_persistently_mapped_memory Persistently mapped memory
323
324 Kepping your memory persistently mapped is generally OK in Vulkan.
325 You don't need to unmap it before using its data on the GPU.
326 The library provides a special feature designed for that:
327 Allocations made with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag set in
328 VmaAllocationCreateInfo::flags stay mapped all the time,
329 so you can just access CPU pointer to it any time
330 without a need to call any "map" or "unmap" function.
331 Example:
332
333 \code
334 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
335 bufCreateInfo.size = sizeof(ConstantBuffer);
336 bufCreateInfo.usage = VK_BUFFER_USAGE_TRANSFER_SRC_BIT;
337
338 VmaAllocationCreateInfo allocCreateInfo = {};
339 allocCreateInfo.usage = VMA_MEMORY_USAGE_CPU_ONLY;
340 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
341
342 VkBuffer buf;
343 VmaAllocation alloc;
344 VmaAllocationInfo allocInfo;
345 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
346
347 // Buffer is already mapped. You can access its memory.
348 memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
349 \endcode
350
351 There are some exceptions though, when you should consider mapping memory only for a short period of time:
352
353 - When operating system is Windows 7 or 8.x (Windows 10 is not affected because it uses WDDM2),
354 device is discrete AMD GPU,
355 and memory type is the special 256 MiB pool of `DEVICE_LOCAL + HOST_VISIBLE` memory
356 (selected when you use #VMA_MEMORY_USAGE_CPU_TO_GPU),
357 then whenever a memory block allocated from this memory type stays mapped
358 for the time of any call to `vkQueueSubmit()` or `vkQueuePresentKHR()`, this
359 block is migrated by WDDM to system RAM, which degrades performance. It doesn't
360 matter if that particular memory block is actually used by the command buffer
361 being submitted.
362 - Keeping many large memory blocks mapped may impact performance or stability of some debugging tools.
363
364 \section memory_mapping_cache_control Cache control
365
366 Memory in Vulkan doesn't need to be unmapped before using it on GPU,
367 but unless a memory types has `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT` flag set,
368 you need to manually invalidate cache before reading of mapped pointer
369 using function `vkvkInvalidateMappedMemoryRanges()`
370 and flush cache after writing to mapped pointer
371 using function `vkFlushMappedMemoryRanges()`.
372 Example:
373
374 \code
375 memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
376
377 VkMemoryPropertyFlags memFlags;
378 vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags);
379 if((memFlags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) == 0)
380 {
381 VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE };
382 memRange.memory = allocInfo.deviceMemory;
383 memRange.offset = allocInfo.offset;
384 memRange.size = allocInfo.size;
385 vkFlushMappedMemoryRanges(device, 1, &memRange);
386 }
387 \endcode
388
389 Please note that memory allocated with #VMA_MEMORY_USAGE_CPU_ONLY is guaranteed to be host coherent.
390
391 Also, Windows drivers from all 3 PC GPU vendors (AMD, Intel, NVIDIA)
392 currently provide `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT` flag on all memory types that are
393 `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT`, so on this platform you may not need to bother.
394
395 \section memory_mapping_finding_if_memory_mappable Finding out if memory is mappable
396
397 It may happen that your allocation ends up in memory that is `HOST_VISIBLE` (available for mapping)
398 despite it wasn't explicitly requested.
399 For example, application may work on integrated graphics with unified memory (like Intel) or
400 allocation from video memory might have failed, so the library chose system memory as fallback.
401
402 You can detect this case and map such allocation to access its memory on CPU directly,
403 instead of launching a transfer operation.
404 In order to do that: inspect `allocInfo.memoryType`, call vmaGetMemoryTypeProperties(),
405 and look for `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag in properties of that memory type.
406
407 \code
408 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
409 bufCreateInfo.size = sizeof(ConstantBuffer);
410 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
411
412 VmaAllocationCreateInfo allocCreateInfo = {};
413 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
414
415 VkBuffer buf;
416 VmaAllocation alloc;
417 VmaAllocationInfo allocInfo;
418 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
419
420 VkMemoryPropertyFlags memFlags;
421 vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags);
422 if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
423 {
424 // Allocation ended up in mappable memory. You can map it and access it directly.
425 void* mappedData;
426 vmaMapMemory(allocator, alloc, &mappedData);
427 memcpy(mappedData, &constantBufferData, sizeof(constantBufferData));
428 vmaUnmapMemory(allocator, alloc);
429 }
430 else
431 {
432 // Allocation ended up in non-mappable memory.
433 // You need to create CPU-side buffer in VMA_MEMORY_USAGE_CPU_ONLY and make a transfer.
434 }
435 \endcode
436
437 You can even use #VMA_ALLOCATION_CREATE_MAPPED_BIT flag while creating allocations
438 that are not necessarily `HOST_VISIBLE` (e.g. using #VMA_MEMORY_USAGE_GPU_ONLY).
439 If the allocation ends up in memory type that is `HOST_VISIBLE`, it will be persistently mapped and you can use it directly.
440 If not, the flag is just ignored.
441 Example:
442
443 \code
444 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
445 bufCreateInfo.size = sizeof(ConstantBuffer);
446 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
447
448 VmaAllocationCreateInfo allocCreateInfo = {};
449 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
450 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_MAPPED_BIT;
451
452 VkBuffer buf;
453 VmaAllocation alloc;
454 VmaAllocationInfo allocInfo;
455 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
456
457 if(allocInfo.pUserData != nullptr)
458 {
459 // Allocation ended up in mappable memory.
460 // It's persistently mapped. You can access it directly.
461 memcpy(allocInfo.pMappedData, &constantBufferData, sizeof(constantBufferData));
462 }
463 else
464 {
465 // Allocation ended up in non-mappable memory.
466 // You need to create CPU-side buffer in VMA_MEMORY_USAGE_CPU_ONLY and make a transfer.
467 }
468 \endcode
469
470
471 \page custom_memory_pools Custom memory pools
472
473 A memory pool contains a number of `VkDeviceMemory` blocks.
474 The library automatically creates and manages default pool for each memory type available on the device.
475 Default memory pool automatically grows in size.
476 Size of allocated blocks is also variable and managed automatically.
477
478 You can create custom pool and allocate memory out of it.
479 It can be useful if you want to:
480
481 - Keep certain kind of allocations separate from others.
482 - Enforce particular, fixed size of Vulkan memory blocks.
483 - Limit maximum amount of Vulkan memory allocated for that pool.
484 - Reserve minimum or fixed amount of Vulkan memory always preallocated for that pool.
485
486 To use custom memory pools:
487
488 -# Fill VmaPoolCreateInfo structure.
489 -# Call vmaCreatePool() to obtain #VmaPool handle.
490 -# When making an allocation, set VmaAllocationCreateInfo::pool to this handle.
491 You don't need to specify any other parameters of this structure, like usage.
492
493 Example:
494
495 \code
496 // Create a pool that can have at most 2 blocks, 128 MiB each.
497 VmaPoolCreateInfo poolCreateInfo = {};
498 poolCreateInfo.memoryTypeIndex = ...
499 poolCreateInfo.blockSize = 128ull * 1024 * 1024;
500 poolCreateInfo.maxBlockCount = 2;
501
502 VmaPool pool;
503 vmaCreatePool(allocator, &poolCreateInfo, &pool);
504
505 // Allocate a buffer out of it.
506 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
507 bufCreateInfo.size = 1024;
508 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
509
510 VmaAllocationCreateInfo allocCreateInfo = {};
511 allocCreateInfo.pool = pool;
512
513 VkBuffer buf;
514 VmaAllocation alloc;
515 VmaAllocationInfo allocInfo;
516 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo);
517 \endcode
518
519 You have to free all allocations made from this pool before destroying it.
520
521 \code
522 vmaDestroyBuffer(allocator, buf, alloc);
523 vmaDestroyPool(allocator, pool);
524 \endcode
525
526 \section custom_memory_pools_MemTypeIndex Choosing memory type index
527
528 When creating a pool, you must explicitly specify memory type index.
529 To find the one suitable for your buffers or images, you can use helper functions
530 vmaFindMemoryTypeIndexForBufferInfo(), vmaFindMemoryTypeIndexForImageInfo().
531 You need to provide structures with example parameters of buffers or images
532 that you are going to create in that pool.
533
534 \code
535 VkBufferCreateInfo exampleBufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
536 exampleBufCreateInfo.size = 1024; // Whatever.
537 exampleBufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT; // Change if needed.
538
539 VmaAllocationCreateInfo allocCreateInfo = {};
540 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; // Change if needed.
541
542 uint32_t memTypeIndex;
543 vmaFindMemoryTypeIndexForBufferInfo(allocator, &exampleBufCreateInfo, &allocCreateInfo, &memTypeIndex);
544
545 VmaPoolCreateInfo poolCreateInfo = {};
546 poolCreateInfo.memoryTypeIndex = memTypeIndex;
547 // ...
548 \endcode
549
550 When creating buffers/images allocated in that pool, provide following parameters:
551
552 - `VkBufferCreateInfo`: Prefer to pass same parameters as above.
553 Otherwise you risk creating resources in a memory type that is not suitable for them, which may result in undefined behavior.
554 Using different `VK_BUFFER_USAGE_` flags may work, but you shouldn't create images in a pool intended for buffers
555 or the other way around.
556 - VmaAllocationCreateInfo: You don't need to pass same parameters. Fill only `pool` member.
557 Other members are ignored anyway.
558
559
560 \page defragmentation Defragmentation
561
562 Interleaved allocations and deallocations of many objects of varying size can
563 cause fragmentation, which can lead to a situation where the library is unable
564 to find a continuous range of free memory for a new allocation despite there is
565 enough free space, just scattered across many small free ranges between existing
566 allocations.
567
568 To mitigate this problem, you can use vmaDefragment(). Given set of allocations,
569 this function can move them to compact used memory, ensure more continuous free
570 space and possibly also free some `VkDeviceMemory`. It can work only on
571 allocations made from memory type that is `HOST_VISIBLE`. Allocations are
572 modified to point to the new `VkDeviceMemory` and offset. Data in this memory is
573 also `memmove`-ed to the new place. However, if you have images or buffers bound
574 to these allocations (and you certainly do), you need to destroy, recreate, and
575 bind them to the new place in memory.
576
577 For further details and example code, see documentation of function
578 vmaDefragment().
579
580 \page lost_allocations Lost allocations
581
582 If your game oversubscribes video memory, if may work OK in previous-generation
583 graphics APIs (DirectX 9, 10, 11, OpenGL) because resources are automatically
584 paged to system RAM. In Vulkan you can't do it because when you run out of
585 memory, an allocation just fails. If you have more data (e.g. textures) that can
586 fit into VRAM and you don't need it all at once, you may want to upload them to
587 GPU on demand and "push out" ones that are not used for a long time to make room
588 for the new ones, effectively using VRAM (or a cartain memory pool) as a form of
589 cache. Vulkan Memory Allocator can help you with that by supporting a concept of
590 "lost allocations".
591
592 To create an allocation that can become lost, include #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT
593 flag in VmaAllocationCreateInfo::flags. Before using a buffer or image bound to
594 such allocation in every new frame, you need to query it if it's not lost.
595 To check it, call vmaTouchAllocation().
596 If the allocation is lost, you should not use it or buffer/image bound to it.
597 You mustn't forget to destroy this allocation and this buffer/image.
598 vmaGetAllocationInfo() can also be used for checking status of the allocation.
599 Allocation is lost when returned VmaAllocationInfo::deviceMemory == `VK_NULL_HANDLE`.
600
601 To create an allocation that can make some other allocations lost to make room
602 for it, use #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag. You will
603 usually use both flags #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT and
604 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT at the same time.
605
606 Warning! Current implementation uses quite naive, brute force algorithm,
607 which can make allocation calls that use #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT
608 flag quite slow. A new, more optimal algorithm and data structure to speed this
609 up is planned for the future.
610
611 <b>Q: When interleaving creation of new allocations with usage of existing ones,
612 how do you make sure that an allocation won't become lost while it's used in the
613 current frame?</b>
614
615 It is ensured because vmaTouchAllocation() / vmaGetAllocationInfo() not only returns allocation
616 status/parameters and checks whether it's not lost, but when it's not, it also
617 atomically marks it as used in the current frame, which makes it impossible to
618 become lost in that frame. It uses lockless algorithm, so it works fast and
619 doesn't involve locking any internal mutex.
620
621 <b>Q: What if my allocation may still be in use by the GPU when it's rendering a
622 previous frame while I already submit new frame on the CPU?</b>
623
624 You can make sure that allocations "touched" by vmaTouchAllocation() / vmaGetAllocationInfo() will not
625 become lost for a number of additional frames back from the current one by
626 specifying this number as VmaAllocatorCreateInfo::frameInUseCount (for default
627 memory pool) and VmaPoolCreateInfo::frameInUseCount (for custom pool).
628
629 <b>Q: How do you inform the library when new frame starts?</b>
630
631 You need to call function vmaSetCurrentFrameIndex().
632
633 Example code:
634
635 \code
636 struct MyBuffer
637 {
638 VkBuffer m_Buf = nullptr;
639 VmaAllocation m_Alloc = nullptr;
640
641 // Called when the buffer is really needed in the current frame.
642 void EnsureBuffer();
643 };
644
645 void MyBuffer::EnsureBuffer()
646 {
647 // Buffer has been created.
648 if(m_Buf != VK_NULL_HANDLE)
649 {
650 // Check if its allocation is not lost + mark it as used in current frame.
651 if(vmaTouchAllocation(allocator, m_Alloc))
652 {
653 // It's all OK - safe to use m_Buf.
654 return;
655 }
656 }
657
658 // Buffer not yet exists or lost - destroy and recreate it.
659
660 vmaDestroyBuffer(allocator, m_Buf, m_Alloc);
661
662 VkBufferCreateInfo bufCreateInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
663 bufCreateInfo.size = 1024;
664 bufCreateInfo.usage = VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT | VK_BUFFER_USAGE_TRANSFER_DST_BIT;
665
666 VmaAllocationCreateInfo allocCreateInfo = {};
667 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
668 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT |
669 VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT;
670
671 vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &m_Buf, &m_Alloc, nullptr);
672 }
673 \endcode
674
675 When using lost allocations, you may see some Vulkan validation layer warnings
676 about overlapping regions of memory bound to different kinds of buffers and
677 images. This is still valid as long as you implement proper handling of lost
678 allocations (like in the example above) and don't use them.
679
680 You can create an allocation that is already in lost state from the beginning using function
681 vmaCreateLostAllocation(). It may be useful if you need a "dummy" allocation that is not null.
682
683 You can call function vmaMakePoolAllocationsLost() to set all eligible allocations
684 in a specified custom pool to lost state.
685 Allocations that have been "touched" in current frame or VmaPoolCreateInfo::frameInUseCount frames back
686 cannot become lost.
687
688
689 \page statistics Statistics
690
691 This library contains functions that return information about its internal state,
692 especially the amount of memory allocated from Vulkan.
693 Please keep in mind that these functions need to traverse all internal data structures
694 to gather these information, so they may be quite time-consuming.
695 Don't call them too often.
696
697 \section statistics_numeric_statistics Numeric statistics
698
699 You can query for overall statistics of the allocator using function vmaCalculateStats().
700 Information are returned using structure #VmaStats.
701 It contains #VmaStatInfo - number of allocated blocks, number of allocations
702 (occupied ranges in these blocks), number of unused (free) ranges in these blocks,
703 number of bytes used and unused (but still allocated from Vulkan) and other information.
704 They are summed across memory heaps, memory types and total for whole allocator.
705
706 You can query for statistics of a custom pool using function vmaGetPoolStats().
707 Information are returned using structure #VmaPoolStats.
708
709 You can query for information about specific allocation using function vmaGetAllocationInfo().
710 It fill structure #VmaAllocationInfo.
711
712 \section statistics_json_dump JSON dump
713
714 You can dump internal state of the allocator to a string in JSON format using function vmaBuildStatsString().
715 The result is guaranteed to be correct JSON.
716 It uses ANSI encoding.
717 Any strings provided by user (see [Allocation names](@ref allocation_names))
718 are copied as-is and properly escaped for JSON, so if they use UTF-8, ISO-8859-2 or any other encoding,
719 this JSON string can be treated as using this encoding.
720 It must be freed using function vmaFreeStatsString().
721
722 The format of this JSON string is not part of official documentation of the library,
723 but it will not change in backward-incompatible way without increasing library major version number
724 and appropriate mention in changelog.
725
726 The JSON string contains all the data that can be obtained using vmaCalculateStats().
727 It can also contain detailed map of allocated memory blocks and their regions -
728 free and occupied by allocations.
729 This allows e.g. to visualize the memory or assess fragmentation.
730
731
732 \page allocation_annotation Allocation names and user data
733
734 \section allocation_user_data Allocation user data
735
736 You can annotate allocations with your own information, e.g. for debugging purposes.
737 To do that, fill VmaAllocationCreateInfo::pUserData field when creating
738 an allocation. It's an opaque `void*` pointer. You can use it e.g. as a pointer,
739 some handle, index, key, ordinal number or any other value that would associate
740 the allocation with your custom metadata.
741
742 \code
743 VkBufferCreateInfo bufferInfo = { VK_STRUCTURE_TYPE_BUFFER_CREATE_INFO };
744 // Fill bufferInfo...
745
746 MyBufferMetadata* pMetadata = CreateBufferMetadata();
747
748 VmaAllocationCreateInfo allocCreateInfo = {};
749 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
750 allocCreateInfo.pUserData = pMetadata;
751
752 VkBuffer buffer;
753 VmaAllocation allocation;
754 vmaCreateBuffer(allocator, &bufferInfo, &allocCreateInfo, &buffer, &allocation, nullptr);
755 \endcode
756
757 The pointer may be later retrieved as VmaAllocationInfo::pUserData:
758
759 \code
760 VmaAllocationInfo allocInfo;
761 vmaGetAllocationInfo(allocator, allocation, &allocInfo);
762 MyBufferMetadata* pMetadata = (MyBufferMetadata*)allocInfo.pUserData;
763 \endcode
764
765 It can also be changed using function vmaSetAllocationUserData().
766
767 Values of (non-zero) allocations' `pUserData` are printed in JSON report created by
768 vmaBuildStatsString(), in hexadecimal form.
769
770 \section allocation_names Allocation names
771
772 There is alternative mode available where `pUserData` pointer is used to point to
773 a null-terminated string, giving a name to the allocation. To use this mode,
774 set #VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT flag in VmaAllocationCreateInfo::flags.
775 Then `pUserData` passed as VmaAllocationCreateInfo::pUserData or argument to
776 vmaSetAllocationUserData() must be either null or pointer to a null-terminated string.
777 The library creates internal copy of the string, so the pointer you pass doesn't need
778 to be valid for whole lifetime of the allocation. You can free it after the call.
779
780 \code
781 VkImageCreateInfo imageInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO };
782 // Fill imageInfo...
783
784 std::string imageName = "Texture: ";
785 imageName += fileName;
786
787 VmaAllocationCreateInfo allocCreateInfo = {};
788 allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY;
789 allocCreateInfo.flags = VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT;
790 allocCreateInfo.pUserData = imageName.c_str();
791
792 VkImage image;
793 VmaAllocation allocation;
794 vmaCreateImage(allocator, &imageInfo, &allocCreateInfo, &image, &allocation, nullptr);
795 \endcode
796
797 The value of `pUserData` pointer of the allocation will be different than the one
798 you passed when setting allocation's name - pointing to a buffer managed
799 internally that holds copy of the string.
800
801 \code
802 VmaAllocationInfo allocInfo;
803 vmaGetAllocationInfo(allocator, allocation, &allocInfo);
804 const char* imageName = (const char*)allocInfo.pUserData;
805 printf("Image name: %s\n", imageName);
806 \endcode
807
808 That string is also printed in JSON report created by vmaBuildStatsString().
809
810
811 \page usage_patterns Recommended usage patterns
812
813 \section usage_patterns_simple Simple patterns
814
815 \subsection usage_patterns_simple_render_targets Render targets
816
817 <b>When:</b>
818 Any resources that you frequently write and read on GPU,
819 e.g. images used as color attachments (aka "render targets"), depth-stencil attachments,
820 images/buffers used as storage image/buffer (aka "Unordered Access View (UAV)").
821
822 <b>What to do:</b>
823 Create them in video memory that is fastest to access from GPU using
824 #VMA_MEMORY_USAGE_GPU_ONLY.
825
826 Consider using [VK_KHR_dedicated_allocation](@ref vk_khr_dedicated_allocation) extension
827 and/or manually creating them as dedicated allocations using #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT,
828 especially if they are large or if you plan to destroy and recreate them e.g. when
829 display resolution changes.
830 Prefer to create such resources first and all other GPU resources (like textures and vertex buffers) later.
831
832 \subsection usage_patterns_simple_immutable_resources Immutable resources
833
834 <b>When:</b>
835 Any resources that you fill on CPU only once (aka "immutable") or infrequently
836 and then read frequently on GPU,
837 e.g. textures, vertex and index buffers, constant buffers that don't change often.
838
839 <b>What to do:</b>
840 Create them in video memory that is fastest to access from GPU using
841 #VMA_MEMORY_USAGE_GPU_ONLY.
842
843 To initialize content of such resource, create a CPU-side (aka "staging") copy of it
844 in system memory - #VMA_MEMORY_USAGE_CPU_ONLY, map it, fill it,
845 and submit a transfer from it to the GPU resource.
846 You can keep the staging copy if you need it for another upload transfer in the future.
847 If you don't, you can destroy it or reuse this buffer for uploading different resource
848 after the transfer finishes.
849
850 Prefer to create just buffers in system memory rather than images, even for uploading textures.
851 Use `vkCmdCopyBufferToImage()`.
852 Dont use images with `VK_IMAGE_TILING_LINEAR`.
853
854 \subsection usage_patterns_dynamic_resources Dynamic resources
855
856 <b>When:</b>
857 Any resources that change frequently (aka "dynamic"), e.g. every frame or every draw call,
858 written on CPU, read on GPU.
859
860 <b>What to do:</b>
861 Create them using #VMA_MEMORY_USAGE_CPU_TO_GPU.
862 You can map it and write to it directly on CPU, as well as read from it on GPU.
863
864 This is a more complex situation. Different solutions are possible,
865 and the best one depends on specific GPU type, but you can use this simple approach for the start.
866 Prefer to write to such resource sequentially (e.g. using `memcpy`).
867 Don't perform random access or any reads from it, as it may be very slow.
868
869 \subsection usage_patterns_readback Readback
870
871 <b>When:</b>
872 Resources that contain data written by GPU that you want to read back on CPU,
873 e.g. results of some computations.
874
875 <b>What to do:</b>
876 Create them using #VMA_MEMORY_USAGE_GPU_TO_CPU.
877 You can write to them directly on GPU, as well as map and read them on CPU.
878
879 \section usage_patterns_advanced Advanced patterns
880
881 \subsection usage_patterns_integrated_graphics Detecting integrated graphics
882
883 You can support integrated graphics (like Intel HD Graphics, AMD APU) better
884 by detecting it in Vulkan.
885 To do it, call `vkGetPhysicalDeviceProperties()`, inspect
886 `VkPhysicalDeviceProperties::deviceType` and look for `VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU`.
887 When you find it, you can assume that memory is unified and all memory types are equally fast
888 to access from GPU, regardless of `VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT`.
889
890 You can then sum up sizes of all available memory heaps and treat them as useful for
891 your GPU resources, instead of only `DEVICE_LOCAL` ones.
892 You can also prefer to create your resources in memory types that are `HOST_VISIBLE` to map them
893 directly instead of submitting explicit transfer (see below).
894
895 \subsection usage_patterns_direct_vs_transfer Direct access versus transfer
896
897 For resources that you frequently write on CPU and read on GPU, many solutions are possible:
898
899 -# Create one copy in video memory using #VMA_MEMORY_USAGE_GPU_ONLY,
900 second copy in system memory using #VMA_MEMORY_USAGE_CPU_ONLY and submit explicit tranfer each time.
901 -# Create just single copy using #VMA_MEMORY_USAGE_CPU_TO_GPU, map it and fill it on CPU,
902 read it directly on GPU.
903 -# Create just single copy using #VMA_MEMORY_USAGE_CPU_ONLY, map it and fill it on CPU,
904 read it directly on GPU.
905
906 Which solution is the most efficient depends on your resource and especially on the GPU.
907 It is best to measure it and then make the decision.
908 Some general recommendations:
909
910 - On integrated graphics use (2) or (3) to avoid unnecesary time and memory overhead
911 related to using a second copy.
912 - For small resources (e.g. constant buffers) use (2).
913 Discrete AMD cards have special 256 MiB pool of video memory that is directly mappable.
914 Even if the resource ends up in system memory, its data may be cached on GPU after first
915 fetch over PCIe bus.
916 - For larger resources (e.g. textures), decide between (1) and (2).
917 You may want to differentiate NVIDIA and AMD, e.g. by looking for memory type that is
918 both `DEVICE_LOCAL` and `HOST_VISIBLE`. When you find it, use (2), otherwise use (1).
919
920 Similarly, for resources that you frequently write on GPU and read on CPU, multiple
921 solutions are possible:
922
923 -# Create one copy in video memory using #VMA_MEMORY_USAGE_GPU_ONLY,
924 second copy in system memory using #VMA_MEMORY_USAGE_GPU_TO_CPU and submit explicit tranfer each time.
925 -# Create just single copy using #VMA_MEMORY_USAGE_GPU_TO_CPU, write to it directly on GPU,
926 map it and read it on CPU.
927
928 You should take some measurements to decide which option is faster in case of your specific
929 resource.
930
931 If you don't want to specialize your code for specific types of GPUs, yon can still make
932 an simple optimization for cases when your resource ends up in mappable memory to use it
933 directly in this case instead of creating CPU-side staging copy.
934 For details see [Finding out if memory is mappable](@ref memory_mapping_finding_if_memory_mappable).
935
936
937 \page configuration Configuration
938
939 Please check "CONFIGURATION SECTION" in the code to find macros that you can define
940 before each include of this file or change directly in this file to provide
941 your own implementation of basic facilities like assert, `min()` and `max()` functions,
942 mutex, atomic etc.
943 The library uses its own implementation of containers by default, but you can switch to using
944 STL containers instead.
945
946 \section config_Vulkan_functions Pointers to Vulkan functions
947
948 The library uses Vulkan functions straight from the `vulkan.h` header by default.
949 If you want to provide your own pointers to these functions, e.g. fetched using
950 `vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`:
951
952 -# Define `VMA_STATIC_VULKAN_FUNCTIONS 0`.
953 -# Provide valid pointers through VmaAllocatorCreateInfo::pVulkanFunctions.
954
955 \section custom_memory_allocator Custom host memory allocator
956
957 If you use custom allocator for CPU memory rather than default operator `new`
958 and `delete` from C++, you can make this library using your allocator as well
959 by filling optional member VmaAllocatorCreateInfo::pAllocationCallbacks. These
960 functions will be passed to Vulkan, as well as used by the library itself to
961 make any CPU-side allocations.
962
963 \section allocation_callbacks Device memory allocation callbacks
964
965 The library makes calls to `vkAllocateMemory()` and `vkFreeMemory()` internally.
966 You can setup callbacks to be informed about these calls, e.g. for the purpose
967 of gathering some statistics. To do it, fill optional member
968 VmaAllocatorCreateInfo::pDeviceMemoryCallbacks.
969
970 \section heap_memory_limit Device heap memory limit
971
972 If you want to test how your program behaves with limited amount of Vulkan device
973 memory available without switching your graphics card to one that really has
974 smaller VRAM, you can use a feature of this library intended for this purpose.
975 To do it, fill optional member VmaAllocatorCreateInfo::pHeapSizeLimit.
976
977
978
979 \page vk_khr_dedicated_allocation VK_KHR_dedicated_allocation
980
981 VK_KHR_dedicated_allocation is a Vulkan extension which can be used to improve
982 performance on some GPUs. It augments Vulkan API with possibility to query
983 driver whether it prefers particular buffer or image to have its own, dedicated
984 allocation (separate `VkDeviceMemory` block) for better efficiency - to be able
985 to do some internal optimizations.
986
987 The extension is supported by this library. It will be used automatically when
988 enabled. To enable it:
989
990 1 . When creating Vulkan device, check if following 2 device extensions are
991 supported (call `vkEnumerateDeviceExtensionProperties()`).
992 If yes, enable them (fill `VkDeviceCreateInfo::ppEnabledExtensionNames`).
993
994 - VK_KHR_get_memory_requirements2
995 - VK_KHR_dedicated_allocation
996
997 If you enabled these extensions:
998
999 2 . Use #VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT flag when creating
1000 your #VmaAllocator`to inform the library that you enabled required extensions
1001 and you want the library to use them.
1002
1003 \code
1004 allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT;
1005
1006 vmaCreateAllocator(&allocatorInfo, &allocator);
1007 \endcode
1008
1009 That's all. The extension will be automatically used whenever you create a
1010 buffer using vmaCreateBuffer() or image using vmaCreateImage().
1011
1012 When using the extension together with Vulkan Validation Layer, you will receive
1013 warnings like this:
1014
1015 vkBindBufferMemory(): Binding memory to buffer 0x33 but vkGetBufferMemoryRequirements() has not been called on that buffer.
1016
1017 It is OK, you should just ignore it. It happens because you use function
1018 `vkGetBufferMemoryRequirements2KHR()` instead of standard
1019 `vkGetBufferMemoryRequirements()`, while the validation layer seems to be
1020 unaware of it.
1021
1022 To learn more about this extension, see:
1023
1024 - [VK_KHR_dedicated_allocation in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html#VK_KHR_dedicated_allocation)
1025 - [VK_KHR_dedicated_allocation unofficial manual](http://asawicki.info/articles/VK_KHR_dedicated_allocation.php5)
1026
1027
1028
1029 \page general_considerations General considerations
1030
1031 \section general_considerations_thread_safety Thread safety
1032
1033 - The library has no global state, so separate #VmaAllocator objects can be used
1034 independently.
1035 There should be no need to create multiple such objects though - one per `VkDevice` is enough.
1036 - By default, all calls to functions that take #VmaAllocator as first parameter
1037 are safe to call from multiple threads simultaneously because they are
1038 synchronized internally when needed.
1039 - When the allocator is created with #VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT
1040 flag, calls to functions that take such #VmaAllocator object must be
1041 synchronized externally.
1042 - Access to a #VmaAllocation object must be externally synchronized. For example,
1043 you must not call vmaGetAllocationInfo() and vmaMapMemory() from different
1044 threads at the same time if you pass the same #VmaAllocation object to these
1045 functions.
1046
1047 \section general_considerations_allocation_algorithm Allocation algorithm
1048
1049 The library uses following algorithm for allocation, in order:
1050
1051 -# Try to find free range of memory in existing blocks.
1052 -# If failed, try to create a new block of `VkDeviceMemory`, with preferred block size.
1053 -# If failed, try to create such block with size/2, size/4, size/8.
1054 -# If failed and #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag was
1055 specified, try to find space in existing blocks, possilby making some other
1056 allocations lost.
1057 -# If failed, try to allocate separate `VkDeviceMemory` for this allocation,
1058 just like when you use #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT.
1059 -# If failed, choose other memory type that meets the requirements specified in
1060 VmaAllocationCreateInfo and go to point 1.
1061 -# If failed, return `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
1062
1063 \section general_considerations_features_not_supported Features not supported
1064
1065 Features deliberately excluded from the scope of this library:
1066
1067 - Data transfer - issuing commands that transfer data between buffers or images, any usage of
1068 `VkCommandList` or `VkCommandQueue` and related synchronization is responsibility of the user.
1069 - Support for any programming languages other than C/C++.
1070 Bindings to other languages are welcomed as external projects.
1071
1072 */
1073
1074 // For skia we don't include vulkan.h here. Before including this header we always include
1075 // GrVkDefines which has a user defined header. Additionally we don't require vulkan/vulkan.h to be
1076 // on the include path.
1077 // #include <vulkan/vulkan.h>
1078
1079 /** \struct VmaAllocator
1080 \brief Represents main object of this library initialized.
1081
1082 Fill structure VmaAllocatorCreateInfo and call function vmaCreateAllocator() to create it.
1083 Call function vmaDestroyAllocator() to destroy it.
1084
1085 It is recommended to create just one object of this type per `VkDevice` object,
1086 right after Vulkan is initialized and keep it alive until before Vulkan device is destroyed.
1087 */
1088 VK_DEFINE_HANDLE(VmaAllocator)
1089
1090 /// Callback function called after successful vkAllocateMemory.
1091 typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)(
1092 VmaAllocator allocator,
1093 uint32_t memoryType,
1094 VkDeviceMemory memory,
1095 VkDeviceSize size);
1096 /// Callback function called before vkFreeMemory.
1097 typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)(
1098 VmaAllocator allocator,
1099 uint32_t memoryType,
1100 VkDeviceMemory memory,
1101 VkDeviceSize size);
1102
1103 /** \brief Set of callbacks that the library will call for `vkAllocateMemory` and `vkFreeMemory`.
1104
1105 Provided for informative purpose, e.g. to gather statistics about number of
1106 allocations or total amount of memory allocated in Vulkan.
1107
1108 Used in VmaAllocatorCreateInfo::pDeviceMemoryCallbacks.
1109 */
1110 typedef struct VmaDeviceMemoryCallbacks {
1111 /// Optional, can be null.
1112 PFN_vmaAllocateDeviceMemoryFunction pfnAllocate;
1113 /// Optional, can be null.
1114 PFN_vmaFreeDeviceMemoryFunction pfnFree;
1115 } VmaDeviceMemoryCallbacks;
1116
1117 /// Flags for created #VmaAllocator.
1118 typedef enum VmaAllocatorCreateFlagBits {
1119 /** \brief Allocator and all objects created from it will not be synchronized internally, so you must guarantee they are used from only one thread at a time or synchronized externally by you.
1120
1121 Using this flag may increase performance because internal mutexes are not used.
1122 */
1123 VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT = 0x00000001,
1124 /** \brief Enables usage of VK_KHR_dedicated_allocation extension.
1125
1126 Using this extenion will automatically allocate dedicated blocks of memory for
1127 some buffers and images instead of suballocating place for them out of bigger
1128 memory blocks (as if you explicitly used #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT
1129 flag) when it is recommended by the driver. It may improve performance on some
1130 GPUs.
1131
1132 You may set this flag only if you found out that following device extensions are
1133 supported, you enabled them while creating Vulkan device passed as
1134 VmaAllocatorCreateInfo::device, and you want them to be used internally by this
1135 library:
1136
1137 - VK_KHR_get_memory_requirements2
1138 - VK_KHR_dedicated_allocation
1139
1140 When this flag is set, you can experience following warnings reported by Vulkan
1141 validation layer. You can ignore them.
1142
1143 > vkBindBufferMemory(): Binding memory to buffer 0x2d but vkGetBufferMemoryRequirements() has not been called on that buffer.
1144 */
1145 VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT = 0x00000002,
1146
1147 VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1148 } VmaAllocatorCreateFlagBits;
1149 typedef VkFlags VmaAllocatorCreateFlags;
1150
1151 /** \brief Pointers to some Vulkan functions - a subset used by the library.
1152
1153 Used in VmaAllocatorCreateInfo::pVulkanFunctions.
1154 */
1155 typedef struct VmaVulkanFunctions {
1156 PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties;
1157 PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties;
1158 PFN_vkAllocateMemory vkAllocateMemory;
1159 PFN_vkFreeMemory vkFreeMemory;
1160 PFN_vkMapMemory vkMapMemory;
1161 PFN_vkUnmapMemory vkUnmapMemory;
1162 PFN_vkBindBufferMemory vkBindBufferMemory;
1163 PFN_vkBindImageMemory vkBindImageMemory;
1164 PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements;
1165 PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements;
1166 PFN_vkCreateBuffer vkCreateBuffer;
1167 PFN_vkDestroyBuffer vkDestroyBuffer;
1168 PFN_vkCreateImage vkCreateImage;
1169 PFN_vkDestroyImage vkDestroyImage;
1170 PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR;
1171 PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR;
1172 } VmaVulkanFunctions;
1173
1174 /// Description of a Allocator to be created.
1175 typedef struct VmaAllocatorCreateInfo
1176 {
1177 /// Flags for created allocator. Use #VmaAllocatorCreateFlagBits enum.
1178 VmaAllocatorCreateFlags flags;
1179 /// Vulkan physical device.
1180 /** It must be valid throughout whole lifetime of created allocator. */
1181 VkPhysicalDevice physicalDevice;
1182 /// Vulkan device.
1183 /** It must be valid throughout whole lifetime of created allocator. */
1184 VkDevice device;
1185 /// Preferred size of a single `VkDeviceMemory` block to be allocated from large heaps > 1 GiB. Optional.
1186 /** Set to 0 to use default, which is currently 256 MiB. */
1187 VkDeviceSize preferredLargeHeapBlockSize;
1188 /// Custom CPU memory allocation callbacks. Optional.
1189 /** Optional, can be null. When specified, will also be used for all CPU-side memory allocations. */
1190 const VkAllocationCallbacks* pAllocationCallbacks;
1191 /// Informative callbacks for `vkAllocateMemory`, `vkFreeMemory`. Optional.
1192 /** Optional, can be null. */
1193 const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks;
1194 /** \brief Maximum number of additional frames that are in use at the same time as current frame.
1195
1196 This value is used only when you make allocations with
1197 VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocation cannot become
1198 lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount.
1199
1200 For example, if you double-buffer your command buffers, so resources used for
1201 rendering in previous frame may still be in use by the GPU at the moment you
1202 allocate resources needed for the current frame, set this value to 1.
1203
1204 If you want to allow any allocations other than used in the current frame to
1205 become lost, set this value to 0.
1206 */
1207 uint32_t frameInUseCount;
1208 /** \brief Either null or a pointer to an array of limits on maximum number of bytes that can be allocated out of particular Vulkan memory heap.
1209
1210 If not NULL, it must be a pointer to an array of
1211 `VkPhysicalDeviceMemoryProperties::memoryHeapCount` elements, defining limit on
1212 maximum number of bytes that can be allocated out of particular Vulkan memory
1213 heap.
1214
1215 Any of the elements may be equal to `VK_WHOLE_SIZE`, which means no limit on that
1216 heap. This is also the default in case of `pHeapSizeLimit` = NULL.
1217
1218 If there is a limit defined for a heap:
1219
1220 - If user tries to allocate more memory from that heap using this allocator,
1221 the allocation fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY`.
1222 - If the limit is smaller than heap size reported in `VkMemoryHeap::size`, the
1223 value of this limit will be reported instead when using vmaGetMemoryProperties().
1224
1225 Warning! Using this feature may not be equivalent to installing a GPU with
1226 smaller amount of memory, because graphics driver doesn't necessary fail new
1227 allocations with `VK_ERROR_OUT_OF_DEVICE_MEMORY` result when memory capacity is
1228 exceeded. It may return success and just silently migrate some device memory
1229 blocks to system RAM.
1230 */
1231 const VkDeviceSize* pHeapSizeLimit;
1232 /** \brief Pointers to Vulkan functions. Can be null if you leave define `VMA_STATIC_VULKAN_FUNCTIONS 1`.
1233
1234 If you leave define `VMA_STATIC_VULKAN_FUNCTIONS 1` in configuration section,
1235 you can pass null as this member, because the library will fetch pointers to
1236 Vulkan functions internally in a static way, like:
1237
1238 vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
1239
1240 Fill this member if you want to provide your own pointers to Vulkan functions,
1241 e.g. fetched using `vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`.
1242 */
1243 const VmaVulkanFunctions* pVulkanFunctions;
1244 } VmaAllocatorCreateInfo;
1245
1246 /// Creates Allocator object.
1247 VkResult vmaCreateAllocator(
1248 const VmaAllocatorCreateInfo* pCreateInfo,
1249 VmaAllocator* pAllocator);
1250
1251 /// Destroys allocator object.
1252 void vmaDestroyAllocator(
1253 VmaAllocator allocator);
1254
1255 /**
1256 PhysicalDeviceProperties are fetched from physicalDevice by the allocator.
1257 You can access it here, without fetching it again on your own.
1258 */
1259 void vmaGetPhysicalDeviceProperties(
1260 VmaAllocator allocator,
1261 const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties);
1262
1263 /**
1264 PhysicalDeviceMemoryProperties are fetched from physicalDevice by the allocator.
1265 You can access it here, without fetching it again on your own.
1266 */
1267 void vmaGetMemoryProperties(
1268 VmaAllocator allocator,
1269 const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties);
1270
1271 /**
1272 \brief Given Memory Type Index, returns Property Flags of this memory type.
1273
1274 This is just a convenience function. Same information can be obtained using
1275 vmaGetMemoryProperties().
1276 */
1277 void vmaGetMemoryTypeProperties(
1278 VmaAllocator allocator,
1279 uint32_t memoryTypeIndex,
1280 VkMemoryPropertyFlags* pFlags);
1281
1282 /** \brief Sets index of the current frame.
1283
1284 This function must be used if you make allocations with
1285 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT and
1286 #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flags to inform the allocator
1287 when a new frame begins. Allocations queried using vmaGetAllocationInfo() cannot
1288 become lost in the current frame.
1289 */
1290 void vmaSetCurrentFrameIndex(
1291 VmaAllocator allocator,
1292 uint32_t frameIndex);
1293
1294 /** \brief Calculated statistics of memory usage in entire allocator.
1295 */
1296 typedef struct VmaStatInfo
1297 {
1298 /// Number of `VkDeviceMemory` Vulkan memory blocks allocated.
1299 uint32_t blockCount;
1300 /// Number of #VmaAllocation allocation objects allocated.
1301 uint32_t allocationCount;
1302 /// Number of free ranges of memory between allocations.
1303 uint32_t unusedRangeCount;
1304 /// Total number of bytes occupied by all allocations.
1305 VkDeviceSize usedBytes;
1306 /// Total number of bytes occupied by unused ranges.
1307 VkDeviceSize unusedBytes;
1308 VkDeviceSize allocationSizeMin, allocationSizeAvg, allocationSizeMax;
1309 VkDeviceSize unusedRangeSizeMin, unusedRangeSizeAvg, unusedRangeSizeMax;
1310 } VmaStatInfo;
1311
1312 /// General statistics from current state of Allocator.
1313 typedef struct VmaStats
1314 {
1315 VmaStatInfo memoryType[VK_MAX_MEMORY_TYPES];
1316 VmaStatInfo memoryHeap[VK_MAX_MEMORY_HEAPS];
1317 VmaStatInfo total;
1318 } VmaStats;
1319
1320 /// Retrieves statistics from current state of the Allocator.
1321 void vmaCalculateStats(
1322 VmaAllocator allocator,
1323 VmaStats* pStats);
1324
1325 #define VMA_STATS_STRING_ENABLED 1
1326
1327 #if VMA_STATS_STRING_ENABLED
1328
1329 /// Builds and returns statistics as string in JSON format.
1330 /** @param[out] ppStatsString Must be freed using vmaFreeStatsString() function.
1331 */
1332 void vmaBuildStatsString(
1333 VmaAllocator allocator,
1334 char** ppStatsString,
1335 VkBool32 detailedMap);
1336
1337 void vmaFreeStatsString(
1338 VmaAllocator allocator,
1339 char* pStatsString);
1340
1341 #endif // #if VMA_STATS_STRING_ENABLED
1342
1343 /** \struct VmaPool
1344 \brief Represents custom memory pool
1345
1346 Fill structure VmaPoolCreateInfo and call function vmaCreatePool() to create it.
1347 Call function vmaDestroyPool() to destroy it.
1348
1349 For more information see [Custom memory pools](@ref choosing_memory_type_custom_memory_pools).
1350 */
1351 VK_DEFINE_HANDLE(VmaPool)
1352
1353 typedef enum VmaMemoryUsage
1354 {
1355 /** No intended memory usage specified.
1356 Use other members of VmaAllocationCreateInfo to specify your requirements.
1357 */
1358 VMA_MEMORY_USAGE_UNKNOWN = 0,
1359 /** Memory will be used on device only, so fast access from the device is preferred.
1360 It usually means device-local GPU (video) memory.
1361 No need to be mappable on host.
1362 It is roughly equivalent of `D3D12_HEAP_TYPE_DEFAULT`.
1363
1364 Usage:
1365
1366 - Resources written and read by device, e.g. images used as attachments.
1367 - Resources transferred from host once (immutable) or infrequently and read by
1368 device multiple times, e.g. textures to be sampled, vertex buffers, uniform
1369 (constant) buffers, and majority of other types of resources used by device.
1370
1371 Allocation may still end up in `HOST_VISIBLE` memory on some implementations.
1372 In such case, you are free to map it.
1373 You can use #VMA_ALLOCATION_CREATE_MAPPED_BIT with this usage type.
1374 */
1375 VMA_MEMORY_USAGE_GPU_ONLY = 1,
1376 /** Memory will be mappable on host.
1377 It usually means CPU (system) memory.
1378 Resources created in this pool may still be accessible to the device, but access to them can be slower.
1379 Guarantees to be `HOST_VISIBLE` and `HOST_COHERENT`.
1380 CPU read may be uncached.
1381 It is roughly equivalent of `D3D12_HEAP_TYPE_UPLOAD`.
1382
1383 Usage: Staging copy of resources used as transfer source.
1384 */
1385 VMA_MEMORY_USAGE_CPU_ONLY = 2,
1386 /**
1387 Memory that is both mappable on host (guarantees to be `HOST_VISIBLE`) and preferably fast to access by GPU.
1388 CPU reads may be uncached and very slow.
1389
1390 Usage: Resources written frequently by host (dynamic), read by device. E.g. textures, vertex buffers, uniform buffers updated every frame or every draw call.
1391 */
1392 VMA_MEMORY_USAGE_CPU_TO_GPU = 3,
1393 /** Memory mappable on host (guarantees to be `HOST_VISIBLE`) and cached.
1394 It is roughly equivalent of `D3D12_HEAP_TYPE_READBACK`.
1395
1396 Usage:
1397
1398 - Resources written by device, read by host - results of some computations, e.g. screen capture, average scene luminance for HDR tone mapping.
1399 - Any resources read or accessed randomly on host, e.g. CPU-side copy of vertex buffer used as source of transfer, but also used for collision detection.
1400 */
1401 VMA_MEMORY_USAGE_GPU_TO_CPU = 4,
1402 VMA_MEMORY_USAGE_MAX_ENUM = 0x7FFFFFFF
1403 } VmaMemoryUsage;
1404
1405 /// Flags to be passed as VmaAllocationCreateInfo::flags.
1406 typedef enum VmaAllocationCreateFlagBits {
1407 /** \brief Set this flag if the allocation should have its own memory block.
1408
1409 Use it for special, big resources, like fullscreen images used as attachments.
1410
1411 This flag must also be used for host visible resources that you want to map
1412 simultaneously because otherwise they might end up as regions of the same
1413 `VkDeviceMemory`, while mapping same `VkDeviceMemory` multiple times
1414 simultaneously is illegal.
1415
1416 You should not use this flag if VmaAllocationCreateInfo::pool is not null.
1417 */
1418 VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001,
1419
1420 /** \brief Set this flag to only try to allocate from existing `VkDeviceMemory` blocks and never create new such block.
1421
1422 If new allocation cannot be placed in any of the existing blocks, allocation
1423 fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY` error.
1424
1425 You should not use #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT and
1426 #VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT at the same time. It makes no sense.
1427
1428 If VmaAllocationCreateInfo::pool is not null, this flag is implied and ignored. */
1429 VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002,
1430 /** \brief Set this flag to use a memory that will be persistently mapped and retrieve pointer to it.
1431
1432 Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData.
1433
1434 Is it valid to use this flag for allocation made from memory type that is not
1435 `HOST_VISIBLE`. This flag is then ignored and memory is not mapped. This is
1436 useful if you need an allocation that is efficient to use on GPU
1437 (`DEVICE_LOCAL`) and still want to map it directly if possible on platforms that
1438 support it (e.g. Intel GPU).
1439
1440 You should not use this flag together with #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT.
1441 */
1442 VMA_ALLOCATION_CREATE_MAPPED_BIT = 0x00000004,
1443 /** Allocation created with this flag can become lost as a result of another
1444 allocation with #VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT flag, so you
1445 must check it before use.
1446
1447 To check if allocation is not lost, call vmaGetAllocationInfo() and check if
1448 VmaAllocationInfo::deviceMemory is not `VK_NULL_HANDLE`.
1449
1450 For details about supporting lost allocations, see Lost Allocations
1451 chapter of User Guide on Main Page.
1452
1453 You should not use this flag together with #VMA_ALLOCATION_CREATE_MAPPED_BIT.
1454 */
1455 VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT = 0x00000008,
1456 /** While creating allocation using this flag, other allocations that were
1457 created with flag #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT can become lost.
1458
1459 For details about supporting lost allocations, see Lost Allocations
1460 chapter of User Guide on Main Page.
1461 */
1462 VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT = 0x00000010,
1463 /** Set this flag to treat VmaAllocationCreateInfo::pUserData as pointer to a
1464 null-terminated string. Instead of copying pointer value, a local copy of the
1465 string is made and stored in allocation's `pUserData`. The string is automatically
1466 freed together with the allocation. It is also used in vmaBuildStatsString().
1467 */
1468 VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT = 0x00000020,
1469
1470 VMA_ALLOCATION_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1471 } VmaAllocationCreateFlagBits;
1472 typedef VkFlags VmaAllocationCreateFlags;
1473
1474 typedef struct VmaAllocationCreateInfo
1475 {
1476 /// Use #VmaAllocationCreateFlagBits enum.
1477 VmaAllocationCreateFlags flags;
1478 /** \brief Intended usage of memory.
1479
1480 You can leave #VMA_MEMORY_USAGE_UNKNOWN if you specify memory requirements in other way. \n
1481 If `pool` is not null, this member is ignored.
1482 */
1483 VmaMemoryUsage usage;
1484 /** \brief Flags that must be set in a Memory Type chosen for an allocation.
1485
1486 Leave 0 if you specify memory requirements in other way. \n
1487 If `pool` is not null, this member is ignored.*/
1488 VkMemoryPropertyFlags requiredFlags;
1489 /** \brief Flags that preferably should be set in a memory type chosen for an allocation.
1490
1491 Set to 0 if no additional flags are prefered. \n
1492 If `pool` is not null, this member is ignored. */
1493 VkMemoryPropertyFlags preferredFlags;
1494 /** \brief Bitmask containing one bit set for every memory type acceptable for this allocation.
1495
1496 Value 0 is equivalent to `UINT32_MAX` - it means any memory type is accepted if
1497 it meets other requirements specified by this structure, with no further
1498 restrictions on memory type index. \n
1499 If `pool` is not null, this member is ignored.
1500 */
1501 uint32_t memoryTypeBits;
1502 /** \brief Pool that this allocation should be created in.
1503
1504 Leave `VK_NULL_HANDLE` to allocate from default pool. If not null, members:
1505 `usage`, `requiredFlags`, `preferredFlags`, `memoryTypeBits` are ignored.
1506 */
1507 VmaPool pool;
1508 /** \brief Custom general-purpose pointer that will be stored in #VmaAllocation, can be read as VmaAllocationInfo::pUserData and changed using vmaSetAllocationUserData().
1509
1510 If #VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT is used, it must be either
1511 null or pointer to a null-terminated string. The string will be then copied to
1512 internal buffer, so it doesn't need to be valid after allocation call.
1513 */
1514 void* pUserData;
1515 } VmaAllocationCreateInfo;
1516
1517 /**
1518 \brief Helps to find memoryTypeIndex, given memoryTypeBits and VmaAllocationCreateInfo.
1519
1520 This algorithm tries to find a memory type that:
1521
1522 - Is allowed by memoryTypeBits.
1523 - Contains all the flags from pAllocationCreateInfo->requiredFlags.
1524 - Matches intended usage.
1525 - Has as many flags from pAllocationCreateInfo->preferredFlags as possible.
1526
1527 \return Returns VK_ERROR_FEATURE_NOT_PRESENT if not found. Receiving such result
1528 from this function or any other allocating function probably means that your
1529 device doesn't support any memory type with requested features for the specific
1530 type of resource you want to use it for. Please check parameters of your
1531 resource, like image layout (OPTIMAL versus LINEAR) or mip level count.
1532 */
1533 VkResult vmaFindMemoryTypeIndex(
1534 VmaAllocator allocator,
1535 uint32_t memoryTypeBits,
1536 const VmaAllocationCreateInfo* pAllocationCreateInfo,
1537 uint32_t* pMemoryTypeIndex);
1538
1539 /**
1540 \brief Helps to find memoryTypeIndex, given VkBufferCreateInfo and VmaAllocationCreateInfo.
1541
1542 It can be useful e.g. to determine value to be used as VmaPoolCreateInfo::memoryTypeIndex.
1543 It internally creates a temporary, dummy buffer that never has memory bound.
1544 It is just a convenience function, equivalent to calling:
1545
1546 - `vkCreateBuffer`
1547 - `vkGetBufferMemoryRequirements`
1548 - `vmaFindMemoryTypeIndex`
1549 - `vkDestroyBuffer`
1550 */
1551 VkResult vmaFindMemoryTypeIndexForBufferInfo(
1552 VmaAllocator allocator,
1553 const VkBufferCreateInfo* pBufferCreateInfo,
1554 const VmaAllocationCreateInfo* pAllocationCreateInfo,
1555 uint32_t* pMemoryTypeIndex);
1556
1557 /**
1558 \brief Helps to find memoryTypeIndex, given VkImageCreateInfo and VmaAllocationCreateInfo.
1559
1560 It can be useful e.g. to determine value to be used as VmaPoolCreateInfo::memoryTypeIndex.
1561 It internally creates a temporary, dummy image that never has memory bound.
1562 It is just a convenience function, equivalent to calling:
1563
1564 - `vkCreateImage`
1565 - `vkGetImageMemoryRequirements`
1566 - `vmaFindMemoryTypeIndex`
1567 - `vkDestroyImage`
1568 */
1569 VkResult vmaFindMemoryTypeIndexForImageInfo(
1570 VmaAllocator allocator,
1571 const VkImageCreateInfo* pImageCreateInfo,
1572 const VmaAllocationCreateInfo* pAllocationCreateInfo,
1573 uint32_t* pMemoryTypeIndex);
1574
1575 /// Flags to be passed as VmaPoolCreateInfo::flags.
1576 typedef enum VmaPoolCreateFlagBits {
1577 /** \brief Use this flag if you always allocate only buffers and linear images or only optimal images out of this pool and so Buffer-Image Granularity can be ignored.
1578
1579 This is na optional optimization flag.
1580
1581 If you always allocate using vmaCreateBuffer(), vmaCreateImage(),
1582 vmaAllocateMemoryForBuffer(), then you don't need to use it because allocator
1583 knows exact type of your allocations so it can handle Buffer-Image Granularity
1584 in the optimal way.
1585
1586 If you also allocate using vmaAllocateMemoryForImage() or vmaAllocateMemory(),
1587 exact type of such allocations is not known, so allocator must be conservative
1588 in handling Buffer-Image Granularity, which can lead to suboptimal allocation
1589 (wasted memory). In that case, if you can make sure you always allocate only
1590 buffers and linear images or only optimal images out of this pool, use this flag
1591 to make allocator disregard Buffer-Image Granularity and so make allocations
1592 more optimal.
1593 */
1594 VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT = 0x00000002,
1595
1596 VMA_POOL_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF
1597 } VmaPoolCreateFlagBits;
1598 typedef VkFlags VmaPoolCreateFlags;
1599
1600 /** \brief Describes parameter of created #VmaPool.
1601 */
1602 typedef struct VmaPoolCreateInfo {
1603 /** \brief Vulkan memory type index to allocate this pool from.
1604 */
1605 uint32_t memoryTypeIndex;
1606 /** \brief Use combination of #VmaPoolCreateFlagBits.
1607 */
1608 VmaPoolCreateFlags flags;
1609 /** \brief Size of a single `VkDeviceMemory` block to be allocated as part of this pool, in bytes.
1610
1611 Optional. Leave 0 to use default.
1612 */
1613 VkDeviceSize blockSize;
1614 /** \brief Minimum number of blocks to be always allocated in this pool, even if they stay empty.
1615
1616 Set to 0 to have no preallocated blocks and let the pool be completely empty.
1617 */
1618 size_t minBlockCount;
1619 /** \brief Maximum number of blocks that can be allocated in this pool. Optional.
1620
1621 Optional. Set to 0 to use `SIZE_MAX`, which means no limit.
1622
1623 Set to same value as minBlockCount to have fixed amount of memory allocated
1624 throuout whole lifetime of this pool.
1625 */
1626 size_t maxBlockCount;
1627 /** \brief Maximum number of additional frames that are in use at the same time as current frame.
1628
1629 This value is used only when you make allocations with
1630 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocation cannot become
1631 lost if allocation.lastUseFrameIndex >= allocator.currentFrameIndex - frameInUseCount.
1632
1633 For example, if you double-buffer your command buffers, so resources used for
1634 rendering in previous frame may still be in use by the GPU at the moment you
1635 allocate resources needed for the current frame, set this value to 1.
1636
1637 If you want to allow any allocations other than used in the current frame to
1638 become lost, set this value to 0.
1639 */
1640 uint32_t frameInUseCount;
1641 } VmaPoolCreateInfo;
1642
1643 /** \brief Describes parameter of existing #VmaPool.
1644 */
1645 typedef struct VmaPoolStats {
1646 /** \brief Total amount of `VkDeviceMemory` allocated from Vulkan for this pool, in bytes.
1647 */
1648 VkDeviceSize size;
1649 /** \brief Total number of bytes in the pool not used by any #VmaAllocation.
1650 */
1651 VkDeviceSize unusedSize;
1652 /** \brief Number of #VmaAllocation objects created from this pool that were not destroyed or lost.
1653 */
1654 size_t allocationCount;
1655 /** \brief Number of continuous memory ranges in the pool not used by any #VmaAllocation.
1656 */
1657 size_t unusedRangeCount;
1658 /** \brief Size of the largest continuous free memory region.
1659
1660 Making a new allocation of that size is not guaranteed to succeed because of
1661 possible additional margin required to respect alignment and buffer/image
1662 granularity.
1663 */
1664 VkDeviceSize unusedRangeSizeMax;
1665 } VmaPoolStats;
1666
1667 /** \brief Allocates Vulkan device memory and creates #VmaPool object.
1668
1669 @param allocator Allocator object.
1670 @param pCreateInfo Parameters of pool to create.
1671 @param[out] pPool Handle to created pool.
1672 */
1673 VkResult vmaCreatePool(
1674 VmaAllocator allocator,
1675 const VmaPoolCreateInfo* pCreateInfo,
1676 VmaPool* pPool);
1677
1678 /** \brief Destroys #VmaPool object and frees Vulkan device memory.
1679 */
1680 void vmaDestroyPool(
1681 VmaAllocator allocator,
1682 VmaPool pool);
1683
1684 /** \brief Retrieves statistics of existing #VmaPool object.
1685
1686 @param allocator Allocator object.
1687 @param pool Pool object.
1688 @param[out] pPoolStats Statistics of specified pool.
1689 */
1690 void vmaGetPoolStats(
1691 VmaAllocator allocator,
1692 VmaPool pool,
1693 VmaPoolStats* pPoolStats);
1694
1695 /** \brief Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now.
1696
1697 @param allocator Allocator object.
1698 @param pool Pool.
1699 @param[out] pLostAllocationCount Number of allocations marked as lost. Optional - pass null if you don't need this information.
1700 */
1701 void vmaMakePoolAllocationsLost(
1702 VmaAllocator allocator,
1703 VmaPool pool,
1704 size_t* pLostAllocationCount);
1705
1706 /** \struct VmaAllocation
1707 \brief Represents single memory allocation.
1708
1709 It may be either dedicated block of `VkDeviceMemory` or a specific region of a bigger block of this type
1710 plus unique offset.
1711
1712 There are multiple ways to create such object.
1713 You need to fill structure VmaAllocationCreateInfo.
1714 For more information see [Choosing memory type](@ref choosing_memory_type).
1715
1716 Although the library provides convenience functions that create Vulkan buffer or image,
1717 allocate memory for it and bind them together,
1718 binding of the allocation to a buffer or an image is out of scope of the allocation itself.
1719 Allocation object can exist without buffer/image bound,
1720 binding can be done manually by the user, and destruction of it can be done
1721 independently of destruction of the allocation.
1722
1723 The object also remembers its size and some other information.
1724 To retrieve this information, use function vmaGetAllocationInfo() and inspect
1725 returned structure VmaAllocationInfo.
1726
1727 Some kinds allocations can be in lost state.
1728 For more information, see [Lost allocations](@ref lost_allocations).
1729 */
1730 VK_DEFINE_HANDLE(VmaAllocation)
1731
1732 /** \brief Parameters of #VmaAllocation objects, that can be retrieved using function vmaGetAllocationInfo().
1733 */
1734 typedef struct VmaAllocationInfo {
1735 /** \brief Memory type index that this allocation was allocated from.
1736
1737 It never changes.
1738 */
1739 uint32_t memoryType;
1740 /** \brief Handle to Vulkan memory object.
1741
1742 Same memory object can be shared by multiple allocations.
1743
1744 It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost.
1745
1746 If the allocation is lost, it is equal to `VK_NULL_HANDLE`.
1747 */
1748 VkDeviceMemory deviceMemory;
1749 /** \brief Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation.
1750
1751 It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost.
1752 */
1753 VkDeviceSize offset;
1754 /** \brief Size of this allocation, in bytes.
1755
1756 It never changes, unless allocation is lost.
1757 */
1758 VkDeviceSize size;
1759 /** \brief Pointer to the beginning of this allocation as mapped data.
1760
1761 If the allocation hasn't been mapped using vmaMapMemory() and hasn't been
1762 created with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag, this value null.
1763
1764 It can change after call to vmaMapMemory(), vmaUnmapMemory().
1765 It can also change after call to vmaDefragment() if this allocation is passed to the function.
1766 */
1767 void* pMappedData;
1768 /** \brief Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vmaSetAllocationUserData().
1769
1770 It can change after call to vmaSetAllocationUserData() for this allocation.
1771 */
1772 void* pUserData;
1773 } VmaAllocationInfo;
1774
1775 /** \brief General purpose memory allocation.
1776
1777 @param[out] pAllocation Handle to allocated memory.
1778 @param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
1779
1780 You should free the memory using vmaFreeMemory().
1781
1782 It is recommended to use vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage(),
1783 vmaCreateBuffer(), vmaCreateImage() instead whenever possible.
1784 */
1785 VkResult vmaAllocateMemory(
1786 VmaAllocator allocator,
1787 const VkMemoryRequirements* pVkMemoryRequirements,
1788 const VmaAllocationCreateInfo* pCreateInfo,
1789 VmaAllocation* pAllocation,
1790 VmaAllocationInfo* pAllocationInfo);
1791
1792 /**
1793 @param[out] pAllocation Handle to allocated memory.
1794 @param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
1795
1796 You should free the memory using vmaFreeMemory().
1797 */
1798 VkResult vmaAllocateMemoryForBuffer(
1799 VmaAllocator allocator,
1800 VkBuffer buffer,
1801 const VmaAllocationCreateInfo* pCreateInfo,
1802 VmaAllocation* pAllocation,
1803 VmaAllocationInfo* pAllocationInfo);
1804
1805 /// Function similar to vmaAllocateMemoryForBuffer().
1806 VkResult vmaAllocateMemoryForImage(
1807 VmaAllocator allocator,
1808 VkImage image,
1809 const VmaAllocationCreateInfo* pCreateInfo,
1810 VmaAllocation* pAllocation,
1811 VmaAllocationInfo* pAllocationInfo);
1812
1813 /// Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage().
1814 void vmaFreeMemory(
1815 VmaAllocator allocator,
1816 VmaAllocation allocation);
1817
1818 /** \brief Returns current information about specified allocation and atomically marks it as used in current frame.
1819
1820 Current paramters of given allocation are returned in `pAllocationInfo`.
1821
1822 This function also atomically "touches" allocation - marks it as used in current frame,
1823 just like vmaTouchAllocation().
1824 If the allocation is in lost state, `pAllocationInfo->deviceMemory == VK_NULL_HANDLE`.
1825
1826 Although this function uses atomics and doesn't lock any mutex, so it should be quite efficient,
1827 you can avoid calling it too often.
1828
1829 - You can retrieve same VmaAllocationInfo structure while creating your resource, from function
1830 vmaCreateBuffer(), vmaCreateImage(). You can remember it if you are sure parameters don't change
1831 (e.g. due to defragmentation or allocation becoming lost).
1832 - If you just want to check if allocation is not lost, vmaTouchAllocation() will work faster.
1833 */
1834 void vmaGetAllocationInfo(
1835 VmaAllocator allocator,
1836 VmaAllocation allocation,
1837 VmaAllocationInfo* pAllocationInfo);
1838
1839 /** \brief Returns `VK_TRUE` if allocation is not lost and atomically marks it as used in current frame.
1840
1841 If the allocation has been created with #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag,
1842 this function returns `VK_TRUE` if it's not in lost state, so it can still be used.
1843 It then also atomically "touches" the allocation - marks it as used in current frame,
1844 so that you can be sure it won't become lost in current frame or next `frameInUseCount` frames.
1845
1846 If the allocation is in lost state, the function returns `VK_FALSE`.
1847 Memory of such allocation, as well as buffer or image bound to it, should not be used.
1848 Lost allocation and the buffer/image still need to be destroyed.
1849
1850 If the allocation has been created without #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag,
1851 this function always returns `VK_TRUE`.
1852 */
1853 VkBool32 vmaTouchAllocation(
1854 VmaAllocator allocator,
1855 VmaAllocation allocation);
1856
1857 /** \brief Sets pUserData in given allocation to new value.
1858
1859 If the allocation was created with VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT,
1860 pUserData must be either null, or pointer to a null-terminated string. The function
1861 makes local copy of the string and sets it as allocation's `pUserData`. String
1862 passed as pUserData doesn't need to be valid for whole lifetime of the allocation -
1863 you can free it after this call. String previously pointed by allocation's
1864 pUserData is freed from memory.
1865
1866 If the flag was not used, the value of pointer `pUserData` is just copied to
1867 allocation's `pUserData`. It is opaque, so you can use it however you want - e.g.
1868 as a pointer, ordinal number or some handle to you own data.
1869 */
1870 void vmaSetAllocationUserData(
1871 VmaAllocator allocator,
1872 VmaAllocation allocation,
1873 void* pUserData);
1874
1875 /** \brief Creates new allocation that is in lost state from the beginning.
1876
1877 It can be useful if you need a dummy, non-null allocation.
1878
1879 You still need to destroy created object using vmaFreeMemory().
1880
1881 Returned allocation is not tied to any specific memory pool or memory type and
1882 not bound to any image or buffer. It has size = 0. It cannot be turned into
1883 a real, non-empty allocation.
1884 */
1885 void vmaCreateLostAllocation(
1886 VmaAllocator allocator,
1887 VmaAllocation* pAllocation);
1888
1889 /** \brief Maps memory represented by given allocation and returns pointer to it.
1890
1891 Maps memory represented by given allocation to make it accessible to CPU code.
1892 When succeeded, `*ppData` contains pointer to first byte of this memory.
1893 If the allocation is part of bigger `VkDeviceMemory` block, the pointer is
1894 correctly offseted to the beginning of region assigned to this particular
1895 allocation.
1896
1897 Mapping is internally reference-counted and synchronized, so despite raw Vulkan
1898 function `vkMapMemory()` cannot be used to map same block of `VkDeviceMemory`
1899 multiple times simultaneously, it is safe to call this function on allocations
1900 assigned to the same memory block. Actual Vulkan memory will be mapped on first
1901 mapping and unmapped on last unmapping.
1902
1903 If the function succeeded, you must call vmaUnmapMemory() to unmap the
1904 allocation when mapping is no longer needed or before freeing the allocation, at
1905 the latest.
1906
1907 It also safe to call this function multiple times on the same allocation. You
1908 must call vmaUnmapMemory() same number of times as you called vmaMapMemory().
1909
1910 It is also safe to call this function on allocation created with
1911 #VMA_ALLOCATION_CREATE_MAPPED_BIT flag. Its memory stays mapped all the time.
1912 You must still call vmaUnmapMemory() same number of times as you called
1913 vmaMapMemory(). You must not call vmaUnmapMemory() additional time to free the
1914 "0-th" mapping made automatically due to #VMA_ALLOCATION_CREATE_MAPPED_BIT flag.
1915
1916 This function fails when used on allocation made in memory type that is not
1917 `HOST_VISIBLE`.
1918
1919 This function always fails when called for allocation that was created with
1920 #VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT flag. Such allocations cannot be
1921 mapped.
1922 */
1923 VkResult vmaMapMemory(
1924 VmaAllocator allocator,
1925 VmaAllocation allocation,
1926 void** ppData);
1927
1928 /** \brief Unmaps memory represented by given allocation, mapped previously using vmaMapMemory().
1929
1930 For details, see description of vmaMapMemory().
1931 */
1932 void vmaUnmapMemory(
1933 VmaAllocator allocator,
1934 VmaAllocation allocation);
1935
1936 /** \brief Optional configuration parameters to be passed to function vmaDefragment(). */
1937 typedef struct VmaDefragmentationInfo {
1938 /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places.
1939
1940 Default is `VK_WHOLE_SIZE`, which means no limit.
1941 */
1942 VkDeviceSize maxBytesToMove;
1943 /** \brief Maximum number of allocations that can be moved to different place.
1944
1945 Default is `UINT32_MAX`, which means no limit.
1946 */
1947 uint32_t maxAllocationsToMove;
1948 } VmaDefragmentationInfo;
1949
1950 /** \brief Statistics returned by function vmaDefragment(). */
1951 typedef struct VmaDefragmentationStats {
1952 /// Total number of bytes that have been copied while moving allocations to different places.
1953 VkDeviceSize bytesMoved;
1954 /// Total number of bytes that have been released to the system by freeing empty `VkDeviceMemory` objects.
1955 VkDeviceSize bytesFreed;
1956 /// Number of allocations that have been moved to different places.
1957 uint32_t allocationsMoved;
1958 /// Number of empty `VkDeviceMemory` objects that have been released to the system.
1959 uint32_t deviceMemoryBlocksFreed;
1960 } VmaDefragmentationStats;
1961
1962 /** \brief Compacts memory by moving allocations.
1963
1964 @param pAllocations Array of allocations that can be moved during this compation.
1965 @param allocationCount Number of elements in pAllocations and pAllocationsChanged arrays.
1966 @param[out] pAllocationsChanged Array of boolean values that will indicate whether matching allocation in pAllocations array has been moved. This parameter is optional. Pass null if you don't need this information.
1967 @param pDefragmentationInfo Configuration parameters. Optional - pass null to use default values.
1968 @param[out] pDefragmentationStats Statistics returned by the function. Optional - pass null if you don't need this information.
1969 @return VK_SUCCESS if completed, VK_INCOMPLETE if succeeded but didn't make all possible optimizations because limits specified in pDefragmentationInfo have been reached, negative error code in case of error.
1970
1971 This function works by moving allocations to different places (different
1972 `VkDeviceMemory` objects and/or different offsets) in order to optimize memory
1973 usage. Only allocations that are in pAllocations array can be moved. All other
1974 allocations are considered nonmovable in this call. Basic rules:
1975
1976 - Only allocations made in memory types that have
1977 `VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT` flag can be compacted. You may pass other
1978 allocations but it makes no sense - these will never be moved.
1979 - You may pass allocations made with #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT but
1980 it makes no sense - they will never be moved.
1981 - Both allocations made with or without #VMA_ALLOCATION_CREATE_MAPPED_BIT
1982 flag can be compacted. If not persistently mapped, memory will be mapped
1983 temporarily inside this function if needed.
1984 - You must not pass same #VmaAllocation object multiple times in pAllocations array.
1985
1986 The function also frees empty `VkDeviceMemory` blocks.
1987
1988 After allocation has been moved, its VmaAllocationInfo::deviceMemory and/or
1989 VmaAllocationInfo::offset changes. You must query them again using
1990 vmaGetAllocationInfo() if you need them.
1991
1992 If an allocation has been moved, data in memory is copied to new place
1993 automatically, but if it was bound to a buffer or an image, you must destroy
1994 that object yourself, create new one and bind it to the new memory pointed by
1995 the allocation. You must use `vkDestroyBuffer()`, `vkDestroyImage()`,
1996 `vkCreateBuffer()`, `vkCreateImage()` for that purpose and NOT vmaDestroyBuffer(),
1997 vmaDestroyImage(), vmaCreateBuffer(), vmaCreateImage()! Example:
1998
1999 \code
2000 VkDevice device = ...;
2001 VmaAllocator allocator = ...;
2002 std::vector<VkBuffer> buffers = ...;
2003 std::vector<VmaAllocation> allocations = ...;
2004
2005 std::vector<VkBool32> allocationsChanged(allocations.size());
2006 vmaDefragment(allocator, allocations.data(), allocations.size(), allocationsChanged.data(), nullptr, nullptr);
2007
2008 for(size_t i = 0; i < allocations.size(); ++i)
2009 {
2010 if(allocationsChanged[i])
2011 {
2012 VmaAllocationInfo allocInfo;
2013 vmaGetAllocationInfo(allocator, allocations[i], &allocInfo);
2014
2015 vkDestroyBuffer(device, buffers[i], nullptr);
2016
2017 VkBufferCreateInfo bufferInfo = ...;
2018 vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]);
2019
2020 // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning.
2021
2022 vkBindBufferMemory(device, buffers[i], allocInfo.deviceMemory, allocInfo.offset);
2023 }
2024 }
2025 \endcode
2026
2027 Note: Please don't expect memory to be fully compacted after this call.
2028 Algorithms inside are based on some heuristics that try to maximize number of Vulkan
2029 memory blocks to make totally empty to release them, as well as to maximimze continuous
2030 empty space inside remaining blocks, while minimizing the number and size of data that
2031 needs to be moved. Some fragmentation still remains after this call. This is normal.
2032
2033 Warning: This function is not 100% correct according to Vulkan specification. Use it
2034 at your own risk. That's because Vulkan doesn't guarantee that memory
2035 requirements (size and alignment) for a new buffer or image are consistent. They
2036 may be different even for subsequent calls with the same parameters. It really
2037 does happen on some platforms, especially with images.
2038
2039 Warning: This function may be time-consuming, so you shouldn't call it too often
2040 (like every frame or after every resource creation/destruction).
2041 You can call it on special occasions (like when reloading a game level or
2042 when you just destroyed a lot of objects).
2043 */
2044 VkResult vmaDefragment(
2045 VmaAllocator allocator,
2046 VmaAllocation* pAllocations,
2047 size_t allocationCount,
2048 VkBool32* pAllocationsChanged,
2049 const VmaDefragmentationInfo *pDefragmentationInfo,
2050 VmaDefragmentationStats* pDefragmentationStats);
2051
2052 /** \brief Binds buffer to allocation.
2053
2054 Binds specified buffer to region of memory represented by specified allocation.
2055 Gets `VkDeviceMemory` handle and offset from the allocation.
2056 If you want to create a buffer, allocate memory for it and bind them together separately,
2057 you should use this function for binding instead of standard `vkBindBufferMemory()`,
2058 because it ensures proper synchronization so that when a `VkDeviceMemory` object is used by multiple
2059 allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from multiple threads simultaneously
2060 (which is illegal in Vulkan).
2061
2062 It is recommended to use function vmaCreateBuffer() instead of this one.
2063 */
2064 VkResult vmaBindBufferMemory(
2065 VmaAllocator allocator,
2066 VmaAllocation allocation,
2067 VkBuffer buffer);
2068
2069 /** \brief Binds image to allocation.
2070
2071 Binds specified image to region of memory represented by specified allocation.
2072 Gets `VkDeviceMemory` handle and offset from the allocation.
2073 If you want to create an image, allocate memory for it and bind them together separately,
2074 you should use this function for binding instead of standard `vkBindImageMemory()`,
2075 because it ensures proper synchronization so that when a `VkDeviceMemory` object is used by multiple
2076 allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from multiple threads simultaneously
2077 (which is illegal in Vulkan).
2078
2079 It is recommended to use function vmaCreateImage() instead of this one.
2080 */
2081 VkResult vmaBindImageMemory(
2082 VmaAllocator allocator,
2083 VmaAllocation allocation,
2084 VkImage image);
2085
2086 /**
2087 @param[out] pBuffer Buffer that was created.
2088 @param[out] pAllocation Allocation that was created.
2089 @param[out] pAllocationInfo Optional. Information about allocated memory. It can be later fetched using function vmaGetAllocationInfo().
2090
2091 This function automatically:
2092
2093 -# Creates buffer.
2094 -# Allocates appropriate memory for it.
2095 -# Binds the buffer with the memory.
2096
2097 If any of these operations fail, buffer and allocation are not created,
2098 returned value is negative error code, *pBuffer and *pAllocation are null.
2099
2100 If the function succeeded, you must destroy both buffer and allocation when you
2101 no longer need them using either convenience function vmaDestroyBuffer() or
2102 separately, using `vkDestroyBuffer()` and vmaFreeMemory().
2103
2104 If VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT flag was used,
2105 VK_KHR_dedicated_allocation extension is used internally to query driver whether
2106 it requires or prefers the new buffer to have dedicated allocation. If yes,
2107 and if dedicated allocation is possible (VmaAllocationCreateInfo::pool is null
2108 and VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT is not used), it creates dedicated
2109 allocation for this buffer, just like when using
2110 VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT.
2111 */
2112 VkResult vmaCreateBuffer(
2113 VmaAllocator allocator,
2114 const VkBufferCreateInfo* pBufferCreateInfo,
2115 const VmaAllocationCreateInfo* pAllocationCreateInfo,
2116 VkBuffer* pBuffer,
2117 VmaAllocation* pAllocation,
2118 VmaAllocationInfo* pAllocationInfo);
2119
2120 /** \brief Destroys Vulkan buffer and frees allocated memory.
2121
2122 This is just a convenience function equivalent to:
2123
2124 \code
2125 vkDestroyBuffer(device, buffer, allocationCallbacks);
2126 vmaFreeMemory(allocator, allocation);
2127 \endcode
2128
2129 It it safe to pass null as buffer and/or allocation.
2130 */
2131 void vmaDestroyBuffer(
2132 VmaAllocator allocator,
2133 VkBuffer buffer,
2134 VmaAllocation allocation);
2135
2136 /// Function similar to vmaCreateBuffer().
2137 VkResult vmaCreateImage(
2138 VmaAllocator allocator,
2139 const VkImageCreateInfo* pImageCreateInfo,
2140 const VmaAllocationCreateInfo* pAllocationCreateInfo,
2141 VkImage* pImage,
2142 VmaAllocation* pAllocation,
2143 VmaAllocationInfo* pAllocationInfo);
2144
2145 /** \brief Destroys Vulkan image and frees allocated memory.
2146
2147 This is just a convenience function equivalent to:
2148
2149 \code
2150 vkDestroyImage(device, image, allocationCallbacks);
2151 vmaFreeMemory(allocator, allocation);
2152 \endcode
2153
2154 It it safe to pass null as image and/or allocation.
2155 */
2156 void vmaDestroyImage(
2157 VmaAllocator allocator,
2158 VkImage image,
2159 VmaAllocation allocation);
2160
2161 #ifdef __cplusplus
2162 }
2163 #endif
2164
2165 #endif // AMD_VULKAN_MEMORY_ALLOCATOR_H
2166
2167 // For Visual Studio IntelliSense.
2168 #ifdef __INTELLISENSE__
2169 #define VMA_IMPLEMENTATION
2170 #endif
2171
2172 #ifdef VMA_IMPLEMENTATION
2173 #undef VMA_IMPLEMENTATION
2174
2175 #include <cstdint>
2176 #include <cstdlib>
2177 #include <cstring>
2178
2179 /*******************************************************************************
2180 CONFIGURATION SECTION
2181
2182 Define some of these macros before each #include of this header or change them
2183 here if you need other then default behavior depending on your environment.
2184 */
2185
2186 /*
2187 Define this macro to 1 to make the library fetch pointers to Vulkan functions
2188 internally, like:
2189
2190 vulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
2191
2192 Define to 0 if you are going to provide you own pointers to Vulkan functions via
2193 VmaAllocatorCreateInfo::pVulkanFunctions.
2194 */
2195 #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES)
2196 #define VMA_STATIC_VULKAN_FUNCTIONS 1
2197 #endif
2198
2199 // Define this macro to 1 to make the library use STL containers instead of its own implementation.
2200 //#define VMA_USE_STL_CONTAINERS 1
2201
2202 /* Set this macro to 1 to make the library including and using STL containers:
2203 std::pair, std::vector, std::list, std::unordered_map.
2204
2205 Set it to 0 or undefined to make the library using its own implementation of
2206 the containers.
2207 */
2208 #if VMA_USE_STL_CONTAINERS
2209 #define VMA_USE_STL_VECTOR 1
2210 #define VMA_USE_STL_UNORDERED_MAP 1
2211 #define VMA_USE_STL_LIST 1
2212 #endif
2213
2214 #if VMA_USE_STL_VECTOR
2215 #include <vector>
2216 #endif
2217
2218 #if VMA_USE_STL_UNORDERED_MAP
2219 #include <unordered_map>
2220 #endif
2221
2222 #if VMA_USE_STL_LIST
2223 #include <list>
2224 #endif
2225
2226 /*
2227 Following headers are used in this CONFIGURATION section only, so feel free to
2228 remove them if not needed.
2229 */
2230 #include <cassert> // for assert
2231 #include <algorithm> // for min, max
2232 #include <mutex> // for std::mutex
2233 #include <atomic> // for std::atomic
2234
2235 #if !defined(_WIN32) && !defined(__APPLE__) && !defined(__FreeBSD__) && \
2236 !defined(__DragonFly__)
2237 #include <malloc.h> // for aligned_alloc()
2238 #endif
2239
2240 #ifndef VMA_NULL
2241 // Value used as null pointer. Define it to e.g.: nullptr, NULL, 0, (void*)0.
2242 #define VMA_NULL nullptr
2243 #endif
2244
2245 #if defined(__APPLE__) || defined(__ANDROID__)
2246 #include <cstdlib>
aligned_alloc(size_t alignment,size_t size)2247 void *aligned_alloc(size_t alignment, size_t size)
2248 {
2249 // alignment must be >= sizeof(void*)
2250 if(alignment < sizeof(void*))
2251 {
2252 alignment = sizeof(void*);
2253 }
2254
2255 void *pointer;
2256 if(posix_memalign(&pointer, alignment, size) == 0)
2257 return pointer;
2258 return VMA_NULL;
2259 }
2260 #endif
2261
2262 // Normal assert to check for programmer's errors, especially in Debug configuration.
2263 #ifndef VMA_ASSERT
2264 #ifdef _DEBUG
2265 #define VMA_ASSERT(expr) assert(expr)
2266 #else
2267 #define VMA_ASSERT(expr)
2268 #endif
2269 #endif
2270
2271 // Assert that will be called very often, like inside data structures e.g. operator[].
2272 // Making it non-empty can make program slow.
2273 #ifndef VMA_HEAVY_ASSERT
2274 #ifdef _DEBUG
2275 #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr)
2276 #else
2277 #define VMA_HEAVY_ASSERT(expr)
2278 #endif
2279 #endif
2280
2281 #ifndef VMA_ALIGN_OF
2282 #define VMA_ALIGN_OF(type) (__alignof(type))
2283 #endif
2284
2285 #ifndef VMA_SYSTEM_ALIGNED_MALLOC
2286 #if defined(_WIN32)
2287 #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment)))
2288 #else
2289 #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) ))
2290 #endif
2291 #endif
2292
2293 #ifndef VMA_SYSTEM_FREE
2294 #if defined(_WIN32)
2295 #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr)
2296 #else
2297 #define VMA_SYSTEM_FREE(ptr) free(ptr)
2298 #endif
2299 #endif
2300
2301 #ifndef VMA_MIN
2302 #define VMA_MIN(v1, v2) (std::min((v1), (v2)))
2303 #endif
2304
2305 #ifndef VMA_MAX
2306 #define VMA_MAX(v1, v2) (std::max((v1), (v2)))
2307 #endif
2308
2309 #ifndef VMA_SWAP
2310 #define VMA_SWAP(v1, v2) std::swap((v1), (v2))
2311 #endif
2312
2313 #ifndef VMA_SORT
2314 #define VMA_SORT(beg, end, cmp) std::sort(beg, end, cmp)
2315 #endif
2316
2317 #ifndef VMA_DEBUG_LOG
2318 #define VMA_DEBUG_LOG(format, ...)
2319 /*
2320 #define VMA_DEBUG_LOG(format, ...) do { \
2321 printf(format, __VA_ARGS__); \
2322 printf("\n"); \
2323 } while(false)
2324 */
2325 #endif
2326
2327 // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString.
2328 #if VMA_STATS_STRING_ENABLED
VmaUint32ToStr(char * outStr,size_t strLen,uint32_t num)2329 static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num)
2330 {
2331 snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num));
2332 }
VmaUint64ToStr(char * outStr,size_t strLen,uint64_t num)2333 static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num)
2334 {
2335 snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num));
2336 }
VmaPtrToStr(char * outStr,size_t strLen,const void * ptr)2337 static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr)
2338 {
2339 snprintf(outStr, strLen, "%p", ptr);
2340 }
2341 #endif
2342
2343 #ifndef VMA_MUTEX
2344 class VmaMutex
2345 {
2346 public:
VmaMutex()2347 VmaMutex() { }
~VmaMutex()2348 ~VmaMutex() { }
Lock()2349 void Lock() { m_Mutex.lock(); }
Unlock()2350 void Unlock() { m_Mutex.unlock(); }
2351 private:
2352 std::mutex m_Mutex;
2353 };
2354 #define VMA_MUTEX VmaMutex
2355 #endif
2356
2357 /*
2358 If providing your own implementation, you need to implement a subset of std::atomic:
2359
2360 - Constructor(uint32_t desired)
2361 - uint32_t load() const
2362 - void store(uint32_t desired)
2363 - bool compare_exchange_weak(uint32_t& expected, uint32_t desired)
2364 */
2365 #ifndef VMA_ATOMIC_UINT32
2366 #define VMA_ATOMIC_UINT32 std::atomic<uint32_t>
2367 #endif
2368
2369 #ifndef VMA_BEST_FIT
2370 /**
2371 Main parameter for function assessing how good is a free suballocation for a new
2372 allocation request.
2373
2374 - Set to 1 to use Best-Fit algorithm - prefer smaller blocks, as close to the
2375 size of requested allocations as possible.
2376 - Set to 0 to use Worst-Fit algorithm - prefer larger blocks, as large as
2377 possible.
2378
2379 Experiments in special testing environment showed that Best-Fit algorithm is
2380 better.
2381 */
2382 #define VMA_BEST_FIT (1)
2383 #endif
2384
2385 #ifndef VMA_DEBUG_ALWAYS_DEDICATED_MEMORY
2386 /**
2387 Every allocation will have its own memory block.
2388 Define to 1 for debugging purposes only.
2389 */
2390 #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0)
2391 #endif
2392
2393 #ifndef VMA_DEBUG_ALIGNMENT
2394 /**
2395 Minimum alignment of all suballocations, in bytes.
2396 Set to more than 1 for debugging purposes only. Must be power of two.
2397 */
2398 #define VMA_DEBUG_ALIGNMENT (1)
2399 #endif
2400
2401 #ifndef VMA_DEBUG_MARGIN
2402 /**
2403 Minimum margin between suballocations, in bytes.
2404 Set nonzero for debugging purposes only.
2405 */
2406 #define VMA_DEBUG_MARGIN (0)
2407 #endif
2408
2409 #ifndef VMA_DEBUG_GLOBAL_MUTEX
2410 /**
2411 Set this to 1 for debugging purposes only, to enable single mutex protecting all
2412 entry calls to the library. Can be useful for debugging multithreading issues.
2413 */
2414 #define VMA_DEBUG_GLOBAL_MUTEX (0)
2415 #endif
2416
2417 #ifndef VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY
2418 /**
2419 Minimum value for VkPhysicalDeviceLimits::bufferImageGranularity.
2420 Set to more than 1 for debugging purposes only. Must be power of two.
2421 */
2422 #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1)
2423 #endif
2424
2425 #ifndef VMA_SMALL_HEAP_MAX_SIZE
2426 /// Maximum size of a memory heap in Vulkan to consider it "small".
2427 #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024)
2428 #endif
2429
2430 #ifndef VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE
2431 /// Default size of a block allocated as single VkDeviceMemory from a "large" heap.
2432 #define VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE (256ull * 1024 * 1024)
2433 #endif
2434
2435 static const uint32_t VMA_FRAME_INDEX_LOST = UINT32_MAX;
2436
2437 /*******************************************************************************
2438 END OF CONFIGURATION
2439 */
2440
2441 static VkAllocationCallbacks VmaEmptyAllocationCallbacks = {
2442 VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL, VMA_NULL };
2443
2444 // Returns number of bits set to 1 in (v).
VmaCountBitsSet(uint32_t v)2445 static inline uint32_t VmaCountBitsSet(uint32_t v)
2446 {
2447 uint32_t c = v - ((v >> 1) & 0x55555555);
2448 c = ((c >> 2) & 0x33333333) + (c & 0x33333333);
2449 c = ((c >> 4) + c) & 0x0F0F0F0F;
2450 c = ((c >> 8) + c) & 0x00FF00FF;
2451 c = ((c >> 16) + c) & 0x0000FFFF;
2452 return c;
2453 }
2454
2455 // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16.
2456 // Use types like uint32_t, uint64_t as T.
2457 template <typename T>
VmaAlignUp(T val,T align)2458 static inline T VmaAlignUp(T val, T align)
2459 {
2460 return (val + align - 1) / align * align;
2461 }
2462
2463 // Division with mathematical rounding to nearest number.
2464 template <typename T>
VmaRoundDiv(T x,T y)2465 inline T VmaRoundDiv(T x, T y)
2466 {
2467 return (x + (y / (T)2)) / y;
2468 }
2469
2470 #ifndef VMA_SORT
2471
2472 template<typename Iterator, typename Compare>
VmaQuickSortPartition(Iterator beg,Iterator end,Compare cmp)2473 Iterator VmaQuickSortPartition(Iterator beg, Iterator end, Compare cmp)
2474 {
2475 Iterator centerValue = end; --centerValue;
2476 Iterator insertIndex = beg;
2477 for(Iterator memTypeIndex = beg; memTypeIndex < centerValue; ++memTypeIndex)
2478 {
2479 if(cmp(*memTypeIndex, *centerValue))
2480 {
2481 if(insertIndex != memTypeIndex)
2482 {
2483 VMA_SWAP(*memTypeIndex, *insertIndex);
2484 }
2485 ++insertIndex;
2486 }
2487 }
2488 if(insertIndex != centerValue)
2489 {
2490 VMA_SWAP(*insertIndex, *centerValue);
2491 }
2492 return insertIndex;
2493 }
2494
2495 template<typename Iterator, typename Compare>
VmaQuickSort(Iterator beg,Iterator end,Compare cmp)2496 void VmaQuickSort(Iterator beg, Iterator end, Compare cmp)
2497 {
2498 if(beg < end)
2499 {
2500 Iterator it = VmaQuickSortPartition<Iterator, Compare>(beg, end, cmp);
2501 VmaQuickSort<Iterator, Compare>(beg, it, cmp);
2502 VmaQuickSort<Iterator, Compare>(it + 1, end, cmp);
2503 }
2504 }
2505
2506 #define VMA_SORT(beg, end, cmp) VmaQuickSort(beg, end, cmp)
2507
2508 #endif // #ifndef VMA_SORT
2509
2510 /*
2511 Returns true if two memory blocks occupy overlapping pages.
2512 ResourceA must be in less memory offset than ResourceB.
2513
2514 Algorithm is based on "Vulkan 1.0.39 - A Specification (with all registered Vulkan extensions)"
2515 chapter 11.6 "Resource Memory Association", paragraph "Buffer-Image Granularity".
2516 */
VmaBlocksOnSamePage(VkDeviceSize resourceAOffset,VkDeviceSize resourceASize,VkDeviceSize resourceBOffset,VkDeviceSize pageSize)2517 static inline bool VmaBlocksOnSamePage(
2518 VkDeviceSize resourceAOffset,
2519 VkDeviceSize resourceASize,
2520 VkDeviceSize resourceBOffset,
2521 VkDeviceSize pageSize)
2522 {
2523 VMA_ASSERT(resourceAOffset + resourceASize <= resourceBOffset && resourceASize > 0 && pageSize > 0);
2524 VkDeviceSize resourceAEnd = resourceAOffset + resourceASize - 1;
2525 VkDeviceSize resourceAEndPage = resourceAEnd & ~(pageSize - 1);
2526 VkDeviceSize resourceBStart = resourceBOffset;
2527 VkDeviceSize resourceBStartPage = resourceBStart & ~(pageSize - 1);
2528 return resourceAEndPage == resourceBStartPage;
2529 }
2530
2531 enum VmaSuballocationType
2532 {
2533 VMA_SUBALLOCATION_TYPE_FREE = 0,
2534 VMA_SUBALLOCATION_TYPE_UNKNOWN = 1,
2535 VMA_SUBALLOCATION_TYPE_BUFFER = 2,
2536 VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN = 3,
2537 VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR = 4,
2538 VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL = 5,
2539 VMA_SUBALLOCATION_TYPE_MAX_ENUM = 0x7FFFFFFF
2540 };
2541
2542 /*
2543 Returns true if given suballocation types could conflict and must respect
2544 VkPhysicalDeviceLimits::bufferImageGranularity. They conflict if one is buffer
2545 or linear image and another one is optimal image. If type is unknown, behave
2546 conservatively.
2547 */
VmaIsBufferImageGranularityConflict(VmaSuballocationType suballocType1,VmaSuballocationType suballocType2)2548 static inline bool VmaIsBufferImageGranularityConflict(
2549 VmaSuballocationType suballocType1,
2550 VmaSuballocationType suballocType2)
2551 {
2552 if(suballocType1 > suballocType2)
2553 {
2554 VMA_SWAP(suballocType1, suballocType2);
2555 }
2556
2557 switch(suballocType1)
2558 {
2559 case VMA_SUBALLOCATION_TYPE_FREE:
2560 return false;
2561 case VMA_SUBALLOCATION_TYPE_UNKNOWN:
2562 return true;
2563 case VMA_SUBALLOCATION_TYPE_BUFFER:
2564 return
2565 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2566 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2567 case VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN:
2568 return
2569 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN ||
2570 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR ||
2571 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2572 case VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR:
2573 return
2574 suballocType2 == VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL;
2575 case VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL:
2576 return false;
2577 default:
2578 VMA_ASSERT(0);
2579 return true;
2580 }
2581 }
2582
2583 // Helper RAII class to lock a mutex in constructor and unlock it in destructor (at the end of scope).
2584 struct VmaMutexLock
2585 {
2586 public:
VmaMutexLockVmaMutexLock2587 VmaMutexLock(VMA_MUTEX& mutex, bool useMutex) :
2588 m_pMutex(useMutex ? &mutex : VMA_NULL)
2589 {
2590 if(m_pMutex)
2591 {
2592 m_pMutex->Lock();
2593 }
2594 }
2595
~VmaMutexLockVmaMutexLock2596 ~VmaMutexLock()
2597 {
2598 if(m_pMutex)
2599 {
2600 m_pMutex->Unlock();
2601 }
2602 }
2603
2604 private:
2605 VMA_MUTEX* m_pMutex;
2606 };
2607
2608 #if VMA_DEBUG_GLOBAL_MUTEX
2609 static VMA_MUTEX gDebugGlobalMutex;
2610 #define VMA_DEBUG_GLOBAL_MUTEX_LOCK VmaMutexLock debugGlobalMutexLock(gDebugGlobalMutex, true);
2611 #else
2612 #define VMA_DEBUG_GLOBAL_MUTEX_LOCK
2613 #endif
2614
2615 // Minimum size of a free suballocation to register it in the free suballocation collection.
2616 static const VkDeviceSize VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER = 16;
2617
2618 /*
2619 Performs binary search and returns iterator to first element that is greater or
2620 equal to (key), according to comparison (cmp).
2621
2622 Cmp should return true if first argument is less than second argument.
2623
2624 Returned value is the found element, if present in the collection or place where
2625 new element with value (key) should be inserted.
2626 */
2627 template <typename IterT, typename KeyT, typename CmpT>
VmaBinaryFindFirstNotLess(IterT beg,IterT end,const KeyT & key,CmpT cmp)2628 static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, CmpT cmp)
2629 {
2630 size_t down = 0, up = (end - beg);
2631 while(down < up)
2632 {
2633 const size_t mid = (down + up) / 2;
2634 if(cmp(*(beg+mid), key))
2635 {
2636 down = mid + 1;
2637 }
2638 else
2639 {
2640 up = mid;
2641 }
2642 }
2643 return beg + down;
2644 }
2645
2646 ////////////////////////////////////////////////////////////////////////////////
2647 // Memory allocation
2648
VmaMalloc(const VkAllocationCallbacks * pAllocationCallbacks,size_t size,size_t alignment)2649 static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment)
2650 {
2651 if((pAllocationCallbacks != VMA_NULL) &&
2652 (pAllocationCallbacks->pfnAllocation != VMA_NULL))
2653 {
2654 return (*pAllocationCallbacks->pfnAllocation)(
2655 pAllocationCallbacks->pUserData,
2656 size,
2657 alignment,
2658 VK_SYSTEM_ALLOCATION_SCOPE_OBJECT);
2659 }
2660 else
2661 {
2662 return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment);
2663 }
2664 }
2665
VmaFree(const VkAllocationCallbacks * pAllocationCallbacks,void * ptr)2666 static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr)
2667 {
2668 if((pAllocationCallbacks != VMA_NULL) &&
2669 (pAllocationCallbacks->pfnFree != VMA_NULL))
2670 {
2671 (*pAllocationCallbacks->pfnFree)(pAllocationCallbacks->pUserData, ptr);
2672 }
2673 else
2674 {
2675 VMA_SYSTEM_FREE(ptr);
2676 }
2677 }
2678
2679 template<typename T>
VmaAllocate(const VkAllocationCallbacks * pAllocationCallbacks)2680 static T* VmaAllocate(const VkAllocationCallbacks* pAllocationCallbacks)
2681 {
2682 return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T), VMA_ALIGN_OF(T));
2683 }
2684
2685 template<typename T>
VmaAllocateArray(const VkAllocationCallbacks * pAllocationCallbacks,size_t count)2686 static T* VmaAllocateArray(const VkAllocationCallbacks* pAllocationCallbacks, size_t count)
2687 {
2688 return (T*)VmaMalloc(pAllocationCallbacks, sizeof(T) * count, VMA_ALIGN_OF(T));
2689 }
2690
2691 #define vma_new(allocator, type) new(VmaAllocate<type>(allocator))(type)
2692
2693 #define vma_new_array(allocator, type, count) new(VmaAllocateArray<type>((allocator), (count)))(type)
2694
2695 template<typename T>
vma_delete(const VkAllocationCallbacks * pAllocationCallbacks,T * ptr)2696 static void vma_delete(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr)
2697 {
2698 ptr->~T();
2699 VmaFree(pAllocationCallbacks, ptr);
2700 }
2701
2702 template<typename T>
vma_delete_array(const VkAllocationCallbacks * pAllocationCallbacks,T * ptr,size_t count)2703 static void vma_delete_array(const VkAllocationCallbacks* pAllocationCallbacks, T* ptr, size_t count)
2704 {
2705 if(ptr != VMA_NULL)
2706 {
2707 for(size_t i = count; i--; )
2708 {
2709 ptr[i].~T();
2710 }
2711 VmaFree(pAllocationCallbacks, ptr);
2712 }
2713 }
2714
2715 // STL-compatible allocator.
2716 template<typename T>
2717 class VmaStlAllocator
2718 {
2719 public:
2720 const VkAllocationCallbacks* const m_pCallbacks;
2721 typedef T value_type;
2722
VmaStlAllocator(const VkAllocationCallbacks * pCallbacks)2723 VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { }
VmaStlAllocator(const VmaStlAllocator<U> & src)2724 template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { }
2725
allocate(size_t n)2726 T* allocate(size_t n) { return VmaAllocateArray<T>(m_pCallbacks, n); }
deallocate(T * p,size_t n)2727 void deallocate(T* p, size_t n) { VmaFree(m_pCallbacks, p); }
2728
2729 template<typename U>
2730 bool operator==(const VmaStlAllocator<U>& rhs) const
2731 {
2732 return m_pCallbacks == rhs.m_pCallbacks;
2733 }
2734 template<typename U>
2735 bool operator!=(const VmaStlAllocator<U>& rhs) const
2736 {
2737 return m_pCallbacks != rhs.m_pCallbacks;
2738 }
2739
2740 VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete;
2741 };
2742
2743 #if VMA_USE_STL_VECTOR
2744
2745 #define VmaVector std::vector
2746
2747 template<typename T, typename allocatorT>
VmaVectorInsert(std::vector<T,allocatorT> & vec,size_t index,const T & item)2748 static void VmaVectorInsert(std::vector<T, allocatorT>& vec, size_t index, const T& item)
2749 {
2750 vec.insert(vec.begin() + index, item);
2751 }
2752
2753 template<typename T, typename allocatorT>
VmaVectorRemove(std::vector<T,allocatorT> & vec,size_t index)2754 static void VmaVectorRemove(std::vector<T, allocatorT>& vec, size_t index)
2755 {
2756 vec.erase(vec.begin() + index);
2757 }
2758
2759 #else // #if VMA_USE_STL_VECTOR
2760
2761 /* Class with interface compatible with subset of std::vector.
2762 T must be POD because constructors and destructors are not called and memcpy is
2763 used for these objects. */
2764 template<typename T, typename AllocatorT>
2765 class VmaVector
2766 {
2767 public:
2768 typedef T value_type;
2769
VmaVector(const AllocatorT & allocator)2770 VmaVector(const AllocatorT& allocator) :
2771 m_Allocator(allocator),
2772 m_pArray(VMA_NULL),
2773 m_Count(0),
2774 m_Capacity(0)
2775 {
2776 }
2777
VmaVector(size_t count,const AllocatorT & allocator)2778 VmaVector(size_t count, const AllocatorT& allocator) :
2779 m_Allocator(allocator),
2780 m_pArray(count ? (T*)VmaAllocateArray<T>(allocator.m_pCallbacks, count) : VMA_NULL),
2781 m_Count(count),
2782 m_Capacity(count)
2783 {
2784 }
2785
VmaVector(const VmaVector<T,AllocatorT> & src)2786 VmaVector(const VmaVector<T, AllocatorT>& src) :
2787 m_Allocator(src.m_Allocator),
2788 m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL),
2789 m_Count(src.m_Count),
2790 m_Capacity(src.m_Count)
2791 {
2792 if(m_Count != 0)
2793 {
2794 memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T));
2795 }
2796 }
2797
~VmaVector()2798 ~VmaVector()
2799 {
2800 VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2801 }
2802
2803 VmaVector& operator=(const VmaVector<T, AllocatorT>& rhs)
2804 {
2805 if(&rhs != this)
2806 {
2807 resize(rhs.m_Count);
2808 if(m_Count != 0)
2809 {
2810 memcpy(m_pArray, rhs.m_pArray, m_Count * sizeof(T));
2811 }
2812 }
2813 return *this;
2814 }
2815
empty()2816 bool empty() const { return m_Count == 0; }
size()2817 size_t size() const { return m_Count; }
data()2818 T* data() { return m_pArray; }
data()2819 const T* data() const { return m_pArray; }
2820
2821 T& operator[](size_t index)
2822 {
2823 VMA_HEAVY_ASSERT(index < m_Count);
2824 return m_pArray[index];
2825 }
2826 const T& operator[](size_t index) const
2827 {
2828 VMA_HEAVY_ASSERT(index < m_Count);
2829 return m_pArray[index];
2830 }
2831
front()2832 T& front()
2833 {
2834 VMA_HEAVY_ASSERT(m_Count > 0);
2835 return m_pArray[0];
2836 }
front()2837 const T& front() const
2838 {
2839 VMA_HEAVY_ASSERT(m_Count > 0);
2840 return m_pArray[0];
2841 }
back()2842 T& back()
2843 {
2844 VMA_HEAVY_ASSERT(m_Count > 0);
2845 return m_pArray[m_Count - 1];
2846 }
back()2847 const T& back() const
2848 {
2849 VMA_HEAVY_ASSERT(m_Count > 0);
2850 return m_pArray[m_Count - 1];
2851 }
2852
2853 void reserve(size_t newCapacity, bool freeMemory = false)
2854 {
2855 newCapacity = VMA_MAX(newCapacity, m_Count);
2856
2857 if((newCapacity < m_Capacity) && !freeMemory)
2858 {
2859 newCapacity = m_Capacity;
2860 }
2861
2862 if(newCapacity != m_Capacity)
2863 {
2864 T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL;
2865 if(m_Count != 0)
2866 {
2867 memcpy(newArray, m_pArray, m_Count * sizeof(T));
2868 }
2869 VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2870 m_Capacity = newCapacity;
2871 m_pArray = newArray;
2872 }
2873 }
2874
2875 void resize(size_t newCount, bool freeMemory = false)
2876 {
2877 size_t newCapacity = m_Capacity;
2878 if(newCount > m_Capacity)
2879 {
2880 newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8));
2881 }
2882 else if(freeMemory)
2883 {
2884 newCapacity = newCount;
2885 }
2886
2887 if(newCapacity != m_Capacity)
2888 {
2889 T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator.m_pCallbacks, newCapacity) : VMA_NULL;
2890 const size_t elementsToCopy = VMA_MIN(m_Count, newCount);
2891 if(elementsToCopy != 0)
2892 {
2893 memcpy(newArray, m_pArray, elementsToCopy * sizeof(T));
2894 }
2895 VmaFree(m_Allocator.m_pCallbacks, m_pArray);
2896 m_Capacity = newCapacity;
2897 m_pArray = newArray;
2898 }
2899
2900 m_Count = newCount;
2901 }
2902
2903 void clear(bool freeMemory = false)
2904 {
2905 resize(0, freeMemory);
2906 }
2907
insert(size_t index,const T & src)2908 void insert(size_t index, const T& src)
2909 {
2910 VMA_HEAVY_ASSERT(index <= m_Count);
2911 const size_t oldCount = size();
2912 resize(oldCount + 1);
2913 if(index < oldCount)
2914 {
2915 memmove(m_pArray + (index + 1), m_pArray + index, (oldCount - index) * sizeof(T));
2916 }
2917 m_pArray[index] = src;
2918 }
2919
remove(size_t index)2920 void remove(size_t index)
2921 {
2922 VMA_HEAVY_ASSERT(index < m_Count);
2923 const size_t oldCount = size();
2924 if(index < oldCount - 1)
2925 {
2926 memmove(m_pArray + index, m_pArray + (index + 1), (oldCount - index - 1) * sizeof(T));
2927 }
2928 resize(oldCount - 1);
2929 }
2930
push_back(const T & src)2931 void push_back(const T& src)
2932 {
2933 const size_t newIndex = size();
2934 resize(newIndex + 1);
2935 m_pArray[newIndex] = src;
2936 }
2937
pop_back()2938 void pop_back()
2939 {
2940 VMA_HEAVY_ASSERT(m_Count > 0);
2941 resize(size() - 1);
2942 }
2943
push_front(const T & src)2944 void push_front(const T& src)
2945 {
2946 insert(0, src);
2947 }
2948
pop_front()2949 void pop_front()
2950 {
2951 VMA_HEAVY_ASSERT(m_Count > 0);
2952 remove(0);
2953 }
2954
2955 typedef T* iterator;
2956
begin()2957 iterator begin() { return m_pArray; }
end()2958 iterator end() { return m_pArray + m_Count; }
2959
2960 private:
2961 AllocatorT m_Allocator;
2962 T* m_pArray;
2963 size_t m_Count;
2964 size_t m_Capacity;
2965 };
2966
2967 template<typename T, typename allocatorT>
VmaVectorInsert(VmaVector<T,allocatorT> & vec,size_t index,const T & item)2968 static void VmaVectorInsert(VmaVector<T, allocatorT>& vec, size_t index, const T& item)
2969 {
2970 vec.insert(index, item);
2971 }
2972
2973 template<typename T, typename allocatorT>
VmaVectorRemove(VmaVector<T,allocatorT> & vec,size_t index)2974 static void VmaVectorRemove(VmaVector<T, allocatorT>& vec, size_t index)
2975 {
2976 vec.remove(index);
2977 }
2978
2979 #endif // #if VMA_USE_STL_VECTOR
2980
2981 template<typename CmpLess, typename VectorT>
VmaVectorInsertSorted(VectorT & vector,const typename VectorT::value_type & value)2982 size_t VmaVectorInsertSorted(VectorT& vector, const typename VectorT::value_type& value)
2983 {
2984 const size_t indexToInsert = VmaBinaryFindFirstNotLess(
2985 vector.data(),
2986 vector.data() + vector.size(),
2987 value,
2988 CmpLess()) - vector.data();
2989 VmaVectorInsert(vector, indexToInsert, value);
2990 return indexToInsert;
2991 }
2992
2993 template<typename CmpLess, typename VectorT>
VmaVectorRemoveSorted(VectorT & vector,const typename VectorT::value_type & value)2994 bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& value)
2995 {
2996 CmpLess comparator;
2997 typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
2998 vector.begin(),
2999 vector.end(),
3000 value,
3001 comparator);
3002 if((it != vector.end()) && !comparator(*it, value) && !comparator(value, *it))
3003 {
3004 size_t indexToRemove = it - vector.begin();
3005 VmaVectorRemove(vector, indexToRemove);
3006 return true;
3007 }
3008 return false;
3009 }
3010
3011 template<typename CmpLess, typename VectorT>
VmaVectorFindSorted(const VectorT & vector,const typename VectorT::value_type & value)3012 size_t VmaVectorFindSorted(const VectorT& vector, const typename VectorT::value_type& value)
3013 {
3014 CmpLess comparator;
3015 typename VectorT::iterator it = VmaBinaryFindFirstNotLess(
3016 vector.data(),
3017 vector.data() + vector.size(),
3018 value,
3019 comparator);
3020 if(it != vector.size() && !comparator(*it, value) && !comparator(value, *it))
3021 {
3022 return it - vector.begin();
3023 }
3024 else
3025 {
3026 return vector.size();
3027 }
3028 }
3029
3030 ////////////////////////////////////////////////////////////////////////////////
3031 // class VmaPoolAllocator
3032
3033 /*
3034 Allocator for objects of type T using a list of arrays (pools) to speed up
3035 allocation. Number of elements that can be allocated is not bounded because
3036 allocator can create multiple blocks.
3037 */
3038 template<typename T>
3039 class VmaPoolAllocator
3040 {
3041 public:
3042 VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock);
3043 ~VmaPoolAllocator();
3044 void Clear();
3045 T* Alloc();
3046 void Free(T* ptr);
3047
3048 private:
3049 union Item
3050 {
3051 uint32_t NextFreeIndex;
3052 T Value;
3053 };
3054
3055 struct ItemBlock
3056 {
3057 Item* pItems;
3058 uint32_t FirstFreeIndex;
3059 };
3060
3061 const VkAllocationCallbacks* m_pAllocationCallbacks;
3062 size_t m_ItemsPerBlock;
3063 VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks;
3064
3065 ItemBlock& CreateNewBlock();
3066 };
3067
3068 template<typename T>
VmaPoolAllocator(const VkAllocationCallbacks * pAllocationCallbacks,size_t itemsPerBlock)3069 VmaPoolAllocator<T>::VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, size_t itemsPerBlock) :
3070 m_pAllocationCallbacks(pAllocationCallbacks),
3071 m_ItemsPerBlock(itemsPerBlock),
3072 m_ItemBlocks(VmaStlAllocator<ItemBlock>(pAllocationCallbacks))
3073 {
3074 VMA_ASSERT(itemsPerBlock > 0);
3075 }
3076
3077 template<typename T>
~VmaPoolAllocator()3078 VmaPoolAllocator<T>::~VmaPoolAllocator()
3079 {
3080 Clear();
3081 }
3082
3083 template<typename T>
Clear()3084 void VmaPoolAllocator<T>::Clear()
3085 {
3086 for(size_t i = m_ItemBlocks.size(); i--; )
3087 vma_delete_array(m_pAllocationCallbacks, m_ItemBlocks[i].pItems, m_ItemsPerBlock);
3088 m_ItemBlocks.clear();
3089 }
3090
3091 template<typename T>
Alloc()3092 T* VmaPoolAllocator<T>::Alloc()
3093 {
3094 for(size_t i = m_ItemBlocks.size(); i--; )
3095 {
3096 ItemBlock& block = m_ItemBlocks[i];
3097 // This block has some free items: Use first one.
3098 if(block.FirstFreeIndex != UINT32_MAX)
3099 {
3100 Item* const pItem = &block.pItems[block.FirstFreeIndex];
3101 block.FirstFreeIndex = pItem->NextFreeIndex;
3102 return &pItem->Value;
3103 }
3104 }
3105
3106 // No block has free item: Create new one and use it.
3107 ItemBlock& newBlock = CreateNewBlock();
3108 Item* const pItem = &newBlock.pItems[0];
3109 newBlock.FirstFreeIndex = pItem->NextFreeIndex;
3110 return &pItem->Value;
3111 }
3112
3113 template<typename T>
Free(T * ptr)3114 void VmaPoolAllocator<T>::Free(T* ptr)
3115 {
3116 // Search all memory blocks to find ptr.
3117 for(size_t i = 0; i < m_ItemBlocks.size(); ++i)
3118 {
3119 ItemBlock& block = m_ItemBlocks[i];
3120
3121 // Casting to union.
3122 Item* pItemPtr;
3123 memcpy(&pItemPtr, &ptr, sizeof(pItemPtr));
3124
3125 // Check if pItemPtr is in address range of this block.
3126 if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + m_ItemsPerBlock))
3127 {
3128 const uint32_t index = static_cast<uint32_t>(pItemPtr - block.pItems);
3129 pItemPtr->NextFreeIndex = block.FirstFreeIndex;
3130 block.FirstFreeIndex = index;
3131 return;
3132 }
3133 }
3134 VMA_ASSERT(0 && "Pointer doesn't belong to this memory pool.");
3135 }
3136
3137 template<typename T>
CreateNewBlock()3138 typename VmaPoolAllocator<T>::ItemBlock& VmaPoolAllocator<T>::CreateNewBlock()
3139 {
3140 ItemBlock newBlock = {
3141 vma_new_array(m_pAllocationCallbacks, Item, m_ItemsPerBlock), 0 };
3142
3143 m_ItemBlocks.push_back(newBlock);
3144
3145 // Setup singly-linked list of all free items in this block.
3146 for(uint32_t i = 0; i < m_ItemsPerBlock - 1; ++i)
3147 newBlock.pItems[i].NextFreeIndex = i + 1;
3148 newBlock.pItems[m_ItemsPerBlock - 1].NextFreeIndex = UINT32_MAX;
3149 return m_ItemBlocks.back();
3150 }
3151
3152 ////////////////////////////////////////////////////////////////////////////////
3153 // class VmaRawList, VmaList
3154
3155 #if VMA_USE_STL_LIST
3156
3157 #define VmaList std::list
3158
3159 #else // #if VMA_USE_STL_LIST
3160
3161 template<typename T>
3162 struct VmaListItem
3163 {
3164 VmaListItem* pPrev;
3165 VmaListItem* pNext;
3166 T Value;
3167 };
3168
3169 // Doubly linked list.
3170 template<typename T>
3171 class VmaRawList
3172 {
3173 public:
3174 typedef VmaListItem<T> ItemType;
3175
3176 VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks);
3177 ~VmaRawList();
3178 void Clear();
3179
GetCount()3180 size_t GetCount() const { return m_Count; }
IsEmpty()3181 bool IsEmpty() const { return m_Count == 0; }
3182
Front()3183 ItemType* Front() { return m_pFront; }
Front()3184 const ItemType* Front() const { return m_pFront; }
Back()3185 ItemType* Back() { return m_pBack; }
Back()3186 const ItemType* Back() const { return m_pBack; }
3187
3188 ItemType* PushBack();
3189 ItemType* PushFront();
3190 ItemType* PushBack(const T& value);
3191 ItemType* PushFront(const T& value);
3192 void PopBack();
3193 void PopFront();
3194
3195 // Item can be null - it means PushBack.
3196 ItemType* InsertBefore(ItemType* pItem);
3197 // Item can be null - it means PushFront.
3198 ItemType* InsertAfter(ItemType* pItem);
3199
3200 ItemType* InsertBefore(ItemType* pItem, const T& value);
3201 ItemType* InsertAfter(ItemType* pItem, const T& value);
3202
3203 void Remove(ItemType* pItem);
3204
3205 private:
3206 const VkAllocationCallbacks* const m_pAllocationCallbacks;
3207 VmaPoolAllocator<ItemType> m_ItemAllocator;
3208 ItemType* m_pFront;
3209 ItemType* m_pBack;
3210 size_t m_Count;
3211
3212 // Declared not defined, to block copy constructor and assignment operator.
3213 VmaRawList(const VmaRawList<T>& src);
3214 VmaRawList<T>& operator=(const VmaRawList<T>& rhs);
3215 };
3216
3217 template<typename T>
VmaRawList(const VkAllocationCallbacks * pAllocationCallbacks)3218 VmaRawList<T>::VmaRawList(const VkAllocationCallbacks* pAllocationCallbacks) :
3219 m_pAllocationCallbacks(pAllocationCallbacks),
3220 m_ItemAllocator(pAllocationCallbacks, 128),
3221 m_pFront(VMA_NULL),
3222 m_pBack(VMA_NULL),
3223 m_Count(0)
3224 {
3225 }
3226
3227 template<typename T>
~VmaRawList()3228 VmaRawList<T>::~VmaRawList()
3229 {
3230 // Intentionally not calling Clear, because that would be unnecessary
3231 // computations to return all items to m_ItemAllocator as free.
3232 }
3233
3234 template<typename T>
Clear()3235 void VmaRawList<T>::Clear()
3236 {
3237 if(IsEmpty() == false)
3238 {
3239 ItemType* pItem = m_pBack;
3240 while(pItem != VMA_NULL)
3241 {
3242 ItemType* const pPrevItem = pItem->pPrev;
3243 m_ItemAllocator.Free(pItem);
3244 pItem = pPrevItem;
3245 }
3246 m_pFront = VMA_NULL;
3247 m_pBack = VMA_NULL;
3248 m_Count = 0;
3249 }
3250 }
3251
3252 template<typename T>
PushBack()3253 VmaListItem<T>* VmaRawList<T>::PushBack()
3254 {
3255 ItemType* const pNewItem = m_ItemAllocator.Alloc();
3256 pNewItem->pNext = VMA_NULL;
3257 if(IsEmpty())
3258 {
3259 pNewItem->pPrev = VMA_NULL;
3260 m_pFront = pNewItem;
3261 m_pBack = pNewItem;
3262 m_Count = 1;
3263 }
3264 else
3265 {
3266 pNewItem->pPrev = m_pBack;
3267 m_pBack->pNext = pNewItem;
3268 m_pBack = pNewItem;
3269 ++m_Count;
3270 }
3271 return pNewItem;
3272 }
3273
3274 template<typename T>
PushFront()3275 VmaListItem<T>* VmaRawList<T>::PushFront()
3276 {
3277 ItemType* const pNewItem = m_ItemAllocator.Alloc();
3278 pNewItem->pPrev = VMA_NULL;
3279 if(IsEmpty())
3280 {
3281 pNewItem->pNext = VMA_NULL;
3282 m_pFront = pNewItem;
3283 m_pBack = pNewItem;
3284 m_Count = 1;
3285 }
3286 else
3287 {
3288 pNewItem->pNext = m_pFront;
3289 m_pFront->pPrev = pNewItem;
3290 m_pFront = pNewItem;
3291 ++m_Count;
3292 }
3293 return pNewItem;
3294 }
3295
3296 template<typename T>
PushBack(const T & value)3297 VmaListItem<T>* VmaRawList<T>::PushBack(const T& value)
3298 {
3299 ItemType* const pNewItem = PushBack();
3300 pNewItem->Value = value;
3301 return pNewItem;
3302 }
3303
3304 template<typename T>
PushFront(const T & value)3305 VmaListItem<T>* VmaRawList<T>::PushFront(const T& value)
3306 {
3307 ItemType* const pNewItem = PushFront();
3308 pNewItem->Value = value;
3309 return pNewItem;
3310 }
3311
3312 template<typename T>
PopBack()3313 void VmaRawList<T>::PopBack()
3314 {
3315 VMA_HEAVY_ASSERT(m_Count > 0);
3316 ItemType* const pBackItem = m_pBack;
3317 ItemType* const pPrevItem = pBackItem->pPrev;
3318 if(pPrevItem != VMA_NULL)
3319 {
3320 pPrevItem->pNext = VMA_NULL;
3321 }
3322 m_pBack = pPrevItem;
3323 m_ItemAllocator.Free(pBackItem);
3324 --m_Count;
3325 }
3326
3327 template<typename T>
PopFront()3328 void VmaRawList<T>::PopFront()
3329 {
3330 VMA_HEAVY_ASSERT(m_Count > 0);
3331 ItemType* const pFrontItem = m_pFront;
3332 ItemType* const pNextItem = pFrontItem->pNext;
3333 if(pNextItem != VMA_NULL)
3334 {
3335 pNextItem->pPrev = VMA_NULL;
3336 }
3337 m_pFront = pNextItem;
3338 m_ItemAllocator.Free(pFrontItem);
3339 --m_Count;
3340 }
3341
3342 template<typename T>
Remove(ItemType * pItem)3343 void VmaRawList<T>::Remove(ItemType* pItem)
3344 {
3345 VMA_HEAVY_ASSERT(pItem != VMA_NULL);
3346 VMA_HEAVY_ASSERT(m_Count > 0);
3347
3348 if(pItem->pPrev != VMA_NULL)
3349 {
3350 pItem->pPrev->pNext = pItem->pNext;
3351 }
3352 else
3353 {
3354 VMA_HEAVY_ASSERT(m_pFront == pItem);
3355 m_pFront = pItem->pNext;
3356 }
3357
3358 if(pItem->pNext != VMA_NULL)
3359 {
3360 pItem->pNext->pPrev = pItem->pPrev;
3361 }
3362 else
3363 {
3364 VMA_HEAVY_ASSERT(m_pBack == pItem);
3365 m_pBack = pItem->pPrev;
3366 }
3367
3368 m_ItemAllocator.Free(pItem);
3369 --m_Count;
3370 }
3371
3372 template<typename T>
InsertBefore(ItemType * pItem)3373 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem)
3374 {
3375 if(pItem != VMA_NULL)
3376 {
3377 ItemType* const prevItem = pItem->pPrev;
3378 ItemType* const newItem = m_ItemAllocator.Alloc();
3379 newItem->pPrev = prevItem;
3380 newItem->pNext = pItem;
3381 pItem->pPrev = newItem;
3382 if(prevItem != VMA_NULL)
3383 {
3384 prevItem->pNext = newItem;
3385 }
3386 else
3387 {
3388 VMA_HEAVY_ASSERT(m_pFront == pItem);
3389 m_pFront = newItem;
3390 }
3391 ++m_Count;
3392 return newItem;
3393 }
3394 else
3395 return PushBack();
3396 }
3397
3398 template<typename T>
InsertAfter(ItemType * pItem)3399 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem)
3400 {
3401 if(pItem != VMA_NULL)
3402 {
3403 ItemType* const nextItem = pItem->pNext;
3404 ItemType* const newItem = m_ItemAllocator.Alloc();
3405 newItem->pNext = nextItem;
3406 newItem->pPrev = pItem;
3407 pItem->pNext = newItem;
3408 if(nextItem != VMA_NULL)
3409 {
3410 nextItem->pPrev = newItem;
3411 }
3412 else
3413 {
3414 VMA_HEAVY_ASSERT(m_pBack == pItem);
3415 m_pBack = newItem;
3416 }
3417 ++m_Count;
3418 return newItem;
3419 }
3420 else
3421 return PushFront();
3422 }
3423
3424 template<typename T>
InsertBefore(ItemType * pItem,const T & value)3425 VmaListItem<T>* VmaRawList<T>::InsertBefore(ItemType* pItem, const T& value)
3426 {
3427 ItemType* const newItem = InsertBefore(pItem);
3428 newItem->Value = value;
3429 return newItem;
3430 }
3431
3432 template<typename T>
InsertAfter(ItemType * pItem,const T & value)3433 VmaListItem<T>* VmaRawList<T>::InsertAfter(ItemType* pItem, const T& value)
3434 {
3435 ItemType* const newItem = InsertAfter(pItem);
3436 newItem->Value = value;
3437 return newItem;
3438 }
3439
3440 template<typename T, typename AllocatorT>
3441 class VmaList
3442 {
3443 public:
3444 class iterator
3445 {
3446 public:
iterator()3447 iterator() :
3448 m_pList(VMA_NULL),
3449 m_pItem(VMA_NULL)
3450 {
3451 }
3452
3453 T& operator*() const
3454 {
3455 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3456 return m_pItem->Value;
3457 }
3458 T* operator->() const
3459 {
3460 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3461 return &m_pItem->Value;
3462 }
3463
3464 iterator& operator++()
3465 {
3466 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3467 m_pItem = m_pItem->pNext;
3468 return *this;
3469 }
3470 iterator& operator--()
3471 {
3472 if(m_pItem != VMA_NULL)
3473 {
3474 m_pItem = m_pItem->pPrev;
3475 }
3476 else
3477 {
3478 VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3479 m_pItem = m_pList->Back();
3480 }
3481 return *this;
3482 }
3483
3484 iterator operator++(int)
3485 {
3486 iterator result = *this;
3487 ++*this;
3488 return result;
3489 }
3490 iterator operator--(int)
3491 {
3492 iterator result = *this;
3493 --*this;
3494 return result;
3495 }
3496
3497 bool operator==(const iterator& rhs) const
3498 {
3499 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3500 return m_pItem == rhs.m_pItem;
3501 }
3502 bool operator!=(const iterator& rhs) const
3503 {
3504 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3505 return m_pItem != rhs.m_pItem;
3506 }
3507
3508 private:
3509 VmaRawList<T>* m_pList;
3510 VmaListItem<T>* m_pItem;
3511
iterator(VmaRawList<T> * pList,VmaListItem<T> * pItem)3512 iterator(VmaRawList<T>* pList, VmaListItem<T>* pItem) :
3513 m_pList(pList),
3514 m_pItem(pItem)
3515 {
3516 }
3517
3518 friend class VmaList<T, AllocatorT>;
3519 };
3520
3521 class const_iterator
3522 {
3523 public:
const_iterator()3524 const_iterator() :
3525 m_pList(VMA_NULL),
3526 m_pItem(VMA_NULL)
3527 {
3528 }
3529
const_iterator(const iterator & src)3530 const_iterator(const iterator& src) :
3531 m_pList(src.m_pList),
3532 m_pItem(src.m_pItem)
3533 {
3534 }
3535
3536 const T& operator*() const
3537 {
3538 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3539 return m_pItem->Value;
3540 }
3541 const T* operator->() const
3542 {
3543 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3544 return &m_pItem->Value;
3545 }
3546
3547 const_iterator& operator++()
3548 {
3549 VMA_HEAVY_ASSERT(m_pItem != VMA_NULL);
3550 m_pItem = m_pItem->pNext;
3551 return *this;
3552 }
3553 const_iterator& operator--()
3554 {
3555 if(m_pItem != VMA_NULL)
3556 {
3557 m_pItem = m_pItem->pPrev;
3558 }
3559 else
3560 {
3561 VMA_HEAVY_ASSERT(!m_pList->IsEmpty());
3562 m_pItem = m_pList->Back();
3563 }
3564 return *this;
3565 }
3566
3567 const_iterator operator++(int)
3568 {
3569 const_iterator result = *this;
3570 ++*this;
3571 return result;
3572 }
3573 const_iterator operator--(int)
3574 {
3575 const_iterator result = *this;
3576 --*this;
3577 return result;
3578 }
3579
3580 bool operator==(const const_iterator& rhs) const
3581 {
3582 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3583 return m_pItem == rhs.m_pItem;
3584 }
3585 bool operator!=(const const_iterator& rhs) const
3586 {
3587 VMA_HEAVY_ASSERT(m_pList == rhs.m_pList);
3588 return m_pItem != rhs.m_pItem;
3589 }
3590
3591 private:
const_iterator(const VmaRawList<T> * pList,const VmaListItem<T> * pItem)3592 const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) :
3593 m_pList(pList),
3594 m_pItem(pItem)
3595 {
3596 }
3597
3598 const VmaRawList<T>* m_pList;
3599 const VmaListItem<T>* m_pItem;
3600
3601 friend class VmaList<T, AllocatorT>;
3602 };
3603
VmaList(const AllocatorT & allocator)3604 VmaList(const AllocatorT& allocator) : m_RawList(allocator.m_pCallbacks) { }
3605
empty()3606 bool empty() const { return m_RawList.IsEmpty(); }
size()3607 size_t size() const { return m_RawList.GetCount(); }
3608
begin()3609 iterator begin() { return iterator(&m_RawList, m_RawList.Front()); }
end()3610 iterator end() { return iterator(&m_RawList, VMA_NULL); }
3611
cbegin()3612 const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); }
cend()3613 const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); }
3614
clear()3615 void clear() { m_RawList.Clear(); }
push_back(const T & value)3616 void push_back(const T& value) { m_RawList.PushBack(value); }
erase(iterator it)3617 void erase(iterator it) { m_RawList.Remove(it.m_pItem); }
insert(iterator it,const T & value)3618 iterator insert(iterator it, const T& value) { return iterator(&m_RawList, m_RawList.InsertBefore(it.m_pItem, value)); }
3619
3620 private:
3621 VmaRawList<T> m_RawList;
3622 };
3623
3624 #endif // #if VMA_USE_STL_LIST
3625
3626 ////////////////////////////////////////////////////////////////////////////////
3627 // class VmaMap
3628
3629 // Unused in this version.
3630 #if 0
3631
3632 #if VMA_USE_STL_UNORDERED_MAP
3633
3634 #define VmaPair std::pair
3635
3636 #define VMA_MAP_TYPE(KeyT, ValueT) \
3637 std::unordered_map< KeyT, ValueT, std::hash<KeyT>, std::equal_to<KeyT>, VmaStlAllocator< std::pair<KeyT, ValueT> > >
3638
3639 #else // #if VMA_USE_STL_UNORDERED_MAP
3640
3641 template<typename T1, typename T2>
3642 struct VmaPair
3643 {
3644 T1 first;
3645 T2 second;
3646
3647 VmaPair() : first(), second() { }
3648 VmaPair(const T1& firstSrc, const T2& secondSrc) : first(firstSrc), second(secondSrc) { }
3649 };
3650
3651 /* Class compatible with subset of interface of std::unordered_map.
3652 KeyT, ValueT must be POD because they will be stored in VmaVector.
3653 */
3654 template<typename KeyT, typename ValueT>
3655 class VmaMap
3656 {
3657 public:
3658 typedef VmaPair<KeyT, ValueT> PairType;
3659 typedef PairType* iterator;
3660
3661 VmaMap(const VmaStlAllocator<PairType>& allocator) : m_Vector(allocator) { }
3662
3663 iterator begin() { return m_Vector.begin(); }
3664 iterator end() { return m_Vector.end(); }
3665
3666 void insert(const PairType& pair);
3667 iterator find(const KeyT& key);
3668 void erase(iterator it);
3669
3670 private:
3671 VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector;
3672 };
3673
3674 #define VMA_MAP_TYPE(KeyT, ValueT) VmaMap<KeyT, ValueT>
3675
3676 template<typename FirstT, typename SecondT>
3677 struct VmaPairFirstLess
3678 {
3679 bool operator()(const VmaPair<FirstT, SecondT>& lhs, const VmaPair<FirstT, SecondT>& rhs) const
3680 {
3681 return lhs.first < rhs.first;
3682 }
3683 bool operator()(const VmaPair<FirstT, SecondT>& lhs, const FirstT& rhsFirst) const
3684 {
3685 return lhs.first < rhsFirst;
3686 }
3687 };
3688
3689 template<typename KeyT, typename ValueT>
3690 void VmaMap<KeyT, ValueT>::insert(const PairType& pair)
3691 {
3692 const size_t indexToInsert = VmaBinaryFindFirstNotLess(
3693 m_Vector.data(),
3694 m_Vector.data() + m_Vector.size(),
3695 pair,
3696 VmaPairFirstLess<KeyT, ValueT>()) - m_Vector.data();
3697 VmaVectorInsert(m_Vector, indexToInsert, pair);
3698 }
3699
3700 template<typename KeyT, typename ValueT>
3701 VmaPair<KeyT, ValueT>* VmaMap<KeyT, ValueT>::find(const KeyT& key)
3702 {
3703 PairType* it = VmaBinaryFindFirstNotLess(
3704 m_Vector.data(),
3705 m_Vector.data() + m_Vector.size(),
3706 key,
3707 VmaPairFirstLess<KeyT, ValueT>());
3708 if((it != m_Vector.end()) && (it->first == key))
3709 {
3710 return it;
3711 }
3712 else
3713 {
3714 return m_Vector.end();
3715 }
3716 }
3717
3718 template<typename KeyT, typename ValueT>
3719 void VmaMap<KeyT, ValueT>::erase(iterator it)
3720 {
3721 VmaVectorRemove(m_Vector, it - m_Vector.begin());
3722 }
3723
3724 #endif // #if VMA_USE_STL_UNORDERED_MAP
3725
3726 #endif // #if 0
3727
3728 ////////////////////////////////////////////////////////////////////////////////
3729
3730 class VmaDeviceMemoryBlock;
3731
3732 struct VmaAllocation_T
3733 {
3734 private:
3735 static const uint8_t MAP_COUNT_FLAG_PERSISTENT_MAP = 0x80;
3736
3737 enum FLAGS
3738 {
3739 FLAG_USER_DATA_STRING = 0x01,
3740 };
3741
3742 public:
3743 enum ALLOCATION_TYPE
3744 {
3745 ALLOCATION_TYPE_NONE,
3746 ALLOCATION_TYPE_BLOCK,
3747 ALLOCATION_TYPE_DEDICATED,
3748 };
3749
VmaAllocation_TVmaAllocation_T3750 VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) :
3751 m_Alignment(1),
3752 m_Size(0),
3753 m_pUserData(VMA_NULL),
3754 m_LastUseFrameIndex(currentFrameIndex),
3755 m_Type((uint8_t)ALLOCATION_TYPE_NONE),
3756 m_SuballocationType((uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN),
3757 m_MapCount(0),
3758 m_Flags(userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0)
3759 {
3760 }
3761
~VmaAllocation_TVmaAllocation_T3762 ~VmaAllocation_T()
3763 {
3764 VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction.");
3765
3766 // Check if owned string was freed.
3767 VMA_ASSERT(m_pUserData == VMA_NULL);
3768 }
3769
InitBlockAllocationVmaAllocation_T3770 void InitBlockAllocation(
3771 VmaPool hPool,
3772 VmaDeviceMemoryBlock* block,
3773 VkDeviceSize offset,
3774 VkDeviceSize alignment,
3775 VkDeviceSize size,
3776 VmaSuballocationType suballocationType,
3777 bool mapped,
3778 bool canBecomeLost)
3779 {
3780 VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3781 VMA_ASSERT(block != VMA_NULL);
3782 m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3783 m_Alignment = alignment;
3784 m_Size = size;
3785 m_MapCount = mapped ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3786 m_SuballocationType = (uint8_t)suballocationType;
3787 m_BlockAllocation.m_hPool = hPool;
3788 m_BlockAllocation.m_Block = block;
3789 m_BlockAllocation.m_Offset = offset;
3790 m_BlockAllocation.m_CanBecomeLost = canBecomeLost;
3791 }
3792
InitLostVmaAllocation_T3793 void InitLost()
3794 {
3795 VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3796 VMA_ASSERT(m_LastUseFrameIndex.load() == VMA_FRAME_INDEX_LOST);
3797 m_Type = (uint8_t)ALLOCATION_TYPE_BLOCK;
3798 m_BlockAllocation.m_hPool = VK_NULL_HANDLE;
3799 m_BlockAllocation.m_Block = VMA_NULL;
3800 m_BlockAllocation.m_Offset = 0;
3801 m_BlockAllocation.m_CanBecomeLost = true;
3802 }
3803
3804 void ChangeBlockAllocation(
3805 VmaAllocator hAllocator,
3806 VmaDeviceMemoryBlock* block,
3807 VkDeviceSize offset);
3808
3809 // pMappedData not null means allocation is created with MAPPED flag.
InitDedicatedAllocationVmaAllocation_T3810 void InitDedicatedAllocation(
3811 uint32_t memoryTypeIndex,
3812 VkDeviceMemory hMemory,
3813 VmaSuballocationType suballocationType,
3814 void* pMappedData,
3815 VkDeviceSize size)
3816 {
3817 VMA_ASSERT(m_Type == ALLOCATION_TYPE_NONE);
3818 VMA_ASSERT(hMemory != VK_NULL_HANDLE);
3819 m_Type = (uint8_t)ALLOCATION_TYPE_DEDICATED;
3820 m_Alignment = 0;
3821 m_Size = size;
3822 m_SuballocationType = (uint8_t)suballocationType;
3823 m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0;
3824 m_DedicatedAllocation.m_MemoryTypeIndex = memoryTypeIndex;
3825 m_DedicatedAllocation.m_hMemory = hMemory;
3826 m_DedicatedAllocation.m_pMappedData = pMappedData;
3827 }
3828
GetTypeVmaAllocation_T3829 ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; }
GetAlignmentVmaAllocation_T3830 VkDeviceSize GetAlignment() const { return m_Alignment; }
GetSizeVmaAllocation_T3831 VkDeviceSize GetSize() const { return m_Size; }
IsUserDataStringVmaAllocation_T3832 bool IsUserDataString() const { return (m_Flags & FLAG_USER_DATA_STRING) != 0; }
GetUserDataVmaAllocation_T3833 void* GetUserData() const { return m_pUserData; }
3834 void SetUserData(VmaAllocator hAllocator, void* pUserData);
GetSuballocationTypeVmaAllocation_T3835 VmaSuballocationType GetSuballocationType() const { return (VmaSuballocationType)m_SuballocationType; }
3836
GetBlockVmaAllocation_T3837 VmaDeviceMemoryBlock* GetBlock() const
3838 {
3839 VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
3840 return m_BlockAllocation.m_Block;
3841 }
3842 VkDeviceSize GetOffset() const;
3843 VkDeviceMemory GetMemory() const;
3844 uint32_t GetMemoryTypeIndex() const;
IsPersistentMapVmaAllocation_T3845 bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; }
3846 void* GetMappedData() const;
3847 bool CanBecomeLost() const;
3848 VmaPool GetPool() const;
3849
GetLastUseFrameIndexVmaAllocation_T3850 uint32_t GetLastUseFrameIndex() const
3851 {
3852 return m_LastUseFrameIndex.load();
3853 }
CompareExchangeLastUseFrameIndexVmaAllocation_T3854 bool CompareExchangeLastUseFrameIndex(uint32_t& expected, uint32_t desired)
3855 {
3856 return m_LastUseFrameIndex.compare_exchange_weak(expected, desired);
3857 }
3858 /*
3859 - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex,
3860 makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true.
3861 - Else, returns false.
3862
3863 If hAllocation is already lost, assert - you should not call it then.
3864 If hAllocation was not created with CAN_BECOME_LOST_BIT, assert.
3865 */
3866 bool MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
3867
DedicatedAllocCalcStatsInfoVmaAllocation_T3868 void DedicatedAllocCalcStatsInfo(VmaStatInfo& outInfo)
3869 {
3870 VMA_ASSERT(m_Type == ALLOCATION_TYPE_DEDICATED);
3871 outInfo.blockCount = 1;
3872 outInfo.allocationCount = 1;
3873 outInfo.unusedRangeCount = 0;
3874 outInfo.usedBytes = m_Size;
3875 outInfo.unusedBytes = 0;
3876 outInfo.allocationSizeMin = outInfo.allocationSizeMax = m_Size;
3877 outInfo.unusedRangeSizeMin = UINT64_MAX;
3878 outInfo.unusedRangeSizeMax = 0;
3879 }
3880
3881 void BlockAllocMap();
3882 void BlockAllocUnmap();
3883 VkResult DedicatedAllocMap(VmaAllocator hAllocator, void** ppData);
3884 void DedicatedAllocUnmap(VmaAllocator hAllocator);
3885
3886 private:
3887 VkDeviceSize m_Alignment;
3888 VkDeviceSize m_Size;
3889 void* m_pUserData;
3890 VMA_ATOMIC_UINT32 m_LastUseFrameIndex;
3891 uint8_t m_Type; // ALLOCATION_TYPE
3892 uint8_t m_SuballocationType; // VmaSuballocationType
3893 // Bit 0x80 is set when allocation was created with VMA_ALLOCATION_CREATE_MAPPED_BIT.
3894 // Bits with mask 0x7F are reference counter for vmaMapMemory()/vmaUnmapMemory().
3895 uint8_t m_MapCount;
3896 uint8_t m_Flags; // enum FLAGS
3897
3898 // Allocation out of VmaDeviceMemoryBlock.
3899 struct BlockAllocation
3900 {
3901 VmaPool m_hPool; // Null if belongs to general memory.
3902 VmaDeviceMemoryBlock* m_Block;
3903 VkDeviceSize m_Offset;
3904 bool m_CanBecomeLost;
3905 };
3906
3907 // Allocation for an object that has its own private VkDeviceMemory.
3908 struct DedicatedAllocation
3909 {
3910 uint32_t m_MemoryTypeIndex;
3911 VkDeviceMemory m_hMemory;
3912 void* m_pMappedData; // Not null means memory is mapped.
3913 };
3914
3915 union
3916 {
3917 // Allocation out of VmaDeviceMemoryBlock.
3918 BlockAllocation m_BlockAllocation;
3919 // Allocation for an object that has its own private VkDeviceMemory.
3920 DedicatedAllocation m_DedicatedAllocation;
3921 };
3922
3923 void FreeUserDataString(VmaAllocator hAllocator);
3924 };
3925
3926 /*
3927 Represents a region of VmaDeviceMemoryBlock that is either assigned and returned as
3928 allocated memory block or free.
3929 */
3930 struct VmaSuballocation
3931 {
3932 VkDeviceSize offset;
3933 VkDeviceSize size;
3934 VmaAllocation hAllocation;
3935 VmaSuballocationType type;
3936 };
3937
3938 typedef VmaList< VmaSuballocation, VmaStlAllocator<VmaSuballocation> > VmaSuballocationList;
3939
3940 // Cost of one additional allocation lost, as equivalent in bytes.
3941 static const VkDeviceSize VMA_LOST_ALLOCATION_COST = 1048576;
3942
3943 /*
3944 Parameters of planned allocation inside a VmaDeviceMemoryBlock.
3945
3946 If canMakeOtherLost was false:
3947 - item points to a FREE suballocation.
3948 - itemsToMakeLostCount is 0.
3949
3950 If canMakeOtherLost was true:
3951 - item points to first of sequence of suballocations, which are either FREE,
3952 or point to VmaAllocations that can become lost.
3953 - itemsToMakeLostCount is the number of VmaAllocations that need to be made lost for
3954 the requested allocation to succeed.
3955 */
3956 struct VmaAllocationRequest
3957 {
3958 VkDeviceSize offset;
3959 VkDeviceSize sumFreeSize; // Sum size of free items that overlap with proposed allocation.
3960 VkDeviceSize sumItemSize; // Sum size of items to make lost that overlap with proposed allocation.
3961 VmaSuballocationList::iterator item;
3962 size_t itemsToMakeLostCount;
3963
CalcCostVmaAllocationRequest3964 VkDeviceSize CalcCost() const
3965 {
3966 return sumItemSize + itemsToMakeLostCount * VMA_LOST_ALLOCATION_COST;
3967 }
3968 };
3969
3970 /*
3971 Data structure used for bookkeeping of allocations and unused ranges of memory
3972 in a single VkDeviceMemory block.
3973 */
3974 class VmaBlockMetadata
3975 {
3976 public:
3977 VmaBlockMetadata(VmaAllocator hAllocator);
3978 ~VmaBlockMetadata();
3979 void Init(VkDeviceSize size);
3980
3981 // Validates all data structures inside this object. If not valid, returns false.
3982 bool Validate() const;
GetSize()3983 VkDeviceSize GetSize() const { return m_Size; }
GetAllocationCount()3984 size_t GetAllocationCount() const { return m_Suballocations.size() - m_FreeCount; }
GetSumFreeSize()3985 VkDeviceSize GetSumFreeSize() const { return m_SumFreeSize; }
3986 VkDeviceSize GetUnusedRangeSizeMax() const;
3987 // Returns true if this block is empty - contains only single free suballocation.
3988 bool IsEmpty() const;
3989
3990 void CalcAllocationStatInfo(VmaStatInfo& outInfo) const;
3991 void AddPoolStats(VmaPoolStats& inoutStats) const;
3992
3993 #if VMA_STATS_STRING_ENABLED
3994 void PrintDetailedMap(class VmaJsonWriter& json) const;
3995 #endif
3996
3997 // Creates trivial request for case when block is empty.
3998 void CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest);
3999
4000 // Tries to find a place for suballocation with given parameters inside this block.
4001 // If succeeded, fills pAllocationRequest and returns true.
4002 // If failed, returns false.
4003 bool CreateAllocationRequest(
4004 uint32_t currentFrameIndex,
4005 uint32_t frameInUseCount,
4006 VkDeviceSize bufferImageGranularity,
4007 VkDeviceSize allocSize,
4008 VkDeviceSize allocAlignment,
4009 VmaSuballocationType allocType,
4010 bool canMakeOtherLost,
4011 VmaAllocationRequest* pAllocationRequest);
4012
4013 bool MakeRequestedAllocationsLost(
4014 uint32_t currentFrameIndex,
4015 uint32_t frameInUseCount,
4016 VmaAllocationRequest* pAllocationRequest);
4017
4018 uint32_t MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount);
4019
4020 // Makes actual allocation based on request. Request must already be checked and valid.
4021 void Alloc(
4022 const VmaAllocationRequest& request,
4023 VmaSuballocationType type,
4024 VkDeviceSize allocSize,
4025 VmaAllocation hAllocation);
4026
4027 // Frees suballocation assigned to given memory region.
4028 void Free(const VmaAllocation allocation);
4029 void FreeAtOffset(VkDeviceSize offset);
4030
4031 private:
4032 VkDeviceSize m_Size;
4033 uint32_t m_FreeCount;
4034 VkDeviceSize m_SumFreeSize;
4035 VmaSuballocationList m_Suballocations;
4036 // Suballocations that are free and have size greater than certain threshold.
4037 // Sorted by size, ascending.
4038 VmaVector< VmaSuballocationList::iterator, VmaStlAllocator< VmaSuballocationList::iterator > > m_FreeSuballocationsBySize;
4039
4040 bool ValidateFreeSuballocationList() const;
4041
4042 // Checks if requested suballocation with given parameters can be placed in given pFreeSuballocItem.
4043 // If yes, fills pOffset and returns true. If no, returns false.
4044 bool CheckAllocation(
4045 uint32_t currentFrameIndex,
4046 uint32_t frameInUseCount,
4047 VkDeviceSize bufferImageGranularity,
4048 VkDeviceSize allocSize,
4049 VkDeviceSize allocAlignment,
4050 VmaSuballocationType allocType,
4051 VmaSuballocationList::const_iterator suballocItem,
4052 bool canMakeOtherLost,
4053 VkDeviceSize* pOffset,
4054 size_t* itemsToMakeLostCount,
4055 VkDeviceSize* pSumFreeSize,
4056 VkDeviceSize* pSumItemSize) const;
4057 // Given free suballocation, it merges it with following one, which must also be free.
4058 void MergeFreeWithNext(VmaSuballocationList::iterator item);
4059 // Releases given suballocation, making it free.
4060 // Merges it with adjacent free suballocations if applicable.
4061 // Returns iterator to new free suballocation at this place.
4062 VmaSuballocationList::iterator FreeSuballocation(VmaSuballocationList::iterator suballocItem);
4063 // Given free suballocation, it inserts it into sorted list of
4064 // m_FreeSuballocationsBySize if it's suitable.
4065 void RegisterFreeSuballocation(VmaSuballocationList::iterator item);
4066 // Given free suballocation, it removes it from sorted list of
4067 // m_FreeSuballocationsBySize if it's suitable.
4068 void UnregisterFreeSuballocation(VmaSuballocationList::iterator item);
4069 };
4070
4071 /*
4072 Represents a single block of device memory (`VkDeviceMemory`) with all the
4073 data about its regions (aka suballocations, #VmaAllocation), assigned and free.
4074
4075 Thread-safety: This class must be externally synchronized.
4076 */
4077 class VmaDeviceMemoryBlock
4078 {
4079 public:
4080 VmaBlockMetadata m_Metadata;
4081
4082 VmaDeviceMemoryBlock(VmaAllocator hAllocator);
4083
~VmaDeviceMemoryBlock()4084 ~VmaDeviceMemoryBlock()
4085 {
4086 VMA_ASSERT(m_MapCount == 0 && "VkDeviceMemory block is being destroyed while it is still mapped.");
4087 VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
4088 }
4089
4090 // Always call after construction.
4091 void Init(
4092 uint32_t newMemoryTypeIndex,
4093 VkDeviceMemory newMemory,
4094 VkDeviceSize newSize);
4095 // Always call before destruction.
4096 void Destroy(VmaAllocator allocator);
4097
GetDeviceMemory()4098 VkDeviceMemory GetDeviceMemory() const { return m_hMemory; }
GetMemoryTypeIndex()4099 uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
GetMappedData()4100 void* GetMappedData() const { return m_pMappedData; }
4101
4102 // Validates all data structures inside this object. If not valid, returns false.
4103 bool Validate() const;
4104
4105 // ppData can be null.
4106 VkResult Map(VmaAllocator hAllocator, uint32_t count, void** ppData);
4107 void Unmap(VmaAllocator hAllocator, uint32_t count);
4108
4109 VkResult BindBufferMemory(
4110 const VmaAllocator hAllocator,
4111 const VmaAllocation hAllocation,
4112 VkBuffer hBuffer);
4113 VkResult BindImageMemory(
4114 const VmaAllocator hAllocator,
4115 const VmaAllocation hAllocation,
4116 VkImage hImage);
4117
4118 private:
4119 uint32_t m_MemoryTypeIndex;
4120 VkDeviceMemory m_hMemory;
4121
4122 // Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory.
4123 // Also protects m_MapCount, m_pMappedData.
4124 VMA_MUTEX m_Mutex;
4125 uint32_t m_MapCount;
4126 void* m_pMappedData;
4127 };
4128
4129 struct VmaPointerLess
4130 {
operatorVmaPointerLess4131 bool operator()(const void* lhs, const void* rhs) const
4132 {
4133 return lhs < rhs;
4134 }
4135 };
4136
4137 class VmaDefragmentator;
4138
4139 /*
4140 Sequence of VmaDeviceMemoryBlock. Represents memory blocks allocated for a specific
4141 Vulkan memory type.
4142
4143 Synchronized internally with a mutex.
4144 */
4145 struct VmaBlockVector
4146 {
4147 VmaBlockVector(
4148 VmaAllocator hAllocator,
4149 uint32_t memoryTypeIndex,
4150 VkDeviceSize preferredBlockSize,
4151 size_t minBlockCount,
4152 size_t maxBlockCount,
4153 VkDeviceSize bufferImageGranularity,
4154 uint32_t frameInUseCount,
4155 bool isCustomPool);
4156 ~VmaBlockVector();
4157
4158 VkResult CreateMinBlocks();
4159
GetMemoryTypeIndexVmaBlockVector4160 uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; }
GetPreferredBlockSizeVmaBlockVector4161 VkDeviceSize GetPreferredBlockSize() const { return m_PreferredBlockSize; }
GetBufferImageGranularityVmaBlockVector4162 VkDeviceSize GetBufferImageGranularity() const { return m_BufferImageGranularity; }
GetFrameInUseCountVmaBlockVector4163 uint32_t GetFrameInUseCount() const { return m_FrameInUseCount; }
4164
4165 void GetPoolStats(VmaPoolStats* pStats);
4166
IsEmptyVmaBlockVector4167 bool IsEmpty() const { return m_Blocks.empty(); }
4168
4169 VkResult Allocate(
4170 VmaPool hCurrentPool,
4171 uint32_t currentFrameIndex,
4172 const VkMemoryRequirements& vkMemReq,
4173 const VmaAllocationCreateInfo& createInfo,
4174 VmaSuballocationType suballocType,
4175 VmaAllocation* pAllocation);
4176
4177 void Free(
4178 VmaAllocation hAllocation);
4179
4180 // Adds statistics of this BlockVector to pStats.
4181 void AddStats(VmaStats* pStats);
4182
4183 #if VMA_STATS_STRING_ENABLED
4184 void PrintDetailedMap(class VmaJsonWriter& json);
4185 #endif
4186
4187 void MakePoolAllocationsLost(
4188 uint32_t currentFrameIndex,
4189 size_t* pLostAllocationCount);
4190
4191 VmaDefragmentator* EnsureDefragmentator(
4192 VmaAllocator hAllocator,
4193 uint32_t currentFrameIndex);
4194
4195 VkResult Defragment(
4196 VmaDefragmentationStats* pDefragmentationStats,
4197 VkDeviceSize& maxBytesToMove,
4198 uint32_t& maxAllocationsToMove);
4199
4200 void DestroyDefragmentator();
4201
4202 private:
4203 friend class VmaDefragmentator;
4204
4205 const VmaAllocator m_hAllocator;
4206 const uint32_t m_MemoryTypeIndex;
4207 const VkDeviceSize m_PreferredBlockSize;
4208 const size_t m_MinBlockCount;
4209 const size_t m_MaxBlockCount;
4210 const VkDeviceSize m_BufferImageGranularity;
4211 const uint32_t m_FrameInUseCount;
4212 const bool m_IsCustomPool;
4213 VMA_MUTEX m_Mutex;
4214 // Incrementally sorted by sumFreeSize, ascending.
4215 VmaVector< VmaDeviceMemoryBlock*, VmaStlAllocator<VmaDeviceMemoryBlock*> > m_Blocks;
4216 /* There can be at most one allocation that is completely empty - a
4217 hysteresis to avoid pessimistic case of alternating creation and destruction
4218 of a VkDeviceMemory. */
4219 bool m_HasEmptyBlock;
4220 VmaDefragmentator* m_pDefragmentator;
4221
4222 size_t CalcMaxBlockSize() const;
4223
4224 // Finds and removes given block from vector.
4225 void Remove(VmaDeviceMemoryBlock* pBlock);
4226
4227 // Performs single step in sorting m_Blocks. They may not be fully sorted
4228 // after this call.
4229 void IncrementallySortBlocks();
4230
4231 VkResult CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex);
4232 };
4233
4234 struct VmaPool_T
4235 {
4236 public:
4237 VmaBlockVector m_BlockVector;
4238
4239 // Takes ownership.
4240 VmaPool_T(
4241 VmaAllocator hAllocator,
4242 const VmaPoolCreateInfo& createInfo);
4243 ~VmaPool_T();
4244
GetBlockVectorVmaPool_T4245 VmaBlockVector& GetBlockVector() { return m_BlockVector; }
4246
4247 #if VMA_STATS_STRING_ENABLED
4248 //void PrintDetailedMap(class VmaStringBuilder& sb);
4249 #endif
4250 };
4251
4252 class VmaDefragmentator
4253 {
4254 const VmaAllocator m_hAllocator;
4255 VmaBlockVector* const m_pBlockVector;
4256 uint32_t m_CurrentFrameIndex;
4257 VkDeviceSize m_BytesMoved;
4258 uint32_t m_AllocationsMoved;
4259
4260 struct AllocationInfo
4261 {
4262 VmaAllocation m_hAllocation;
4263 VkBool32* m_pChanged;
4264
AllocationInfoAllocationInfo4265 AllocationInfo() :
4266 m_hAllocation(VK_NULL_HANDLE),
4267 m_pChanged(VMA_NULL)
4268 {
4269 }
4270 };
4271
4272 struct AllocationInfoSizeGreater
4273 {
operatorAllocationInfoSizeGreater4274 bool operator()(const AllocationInfo& lhs, const AllocationInfo& rhs) const
4275 {
4276 return lhs.m_hAllocation->GetSize() > rhs.m_hAllocation->GetSize();
4277 }
4278 };
4279
4280 // Used between AddAllocation and Defragment.
4281 VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4282
4283 struct BlockInfo
4284 {
4285 VmaDeviceMemoryBlock* m_pBlock;
4286 bool m_HasNonMovableAllocations;
4287 VmaVector< AllocationInfo, VmaStlAllocator<AllocationInfo> > m_Allocations;
4288
BlockInfoBlockInfo4289 BlockInfo(const VkAllocationCallbacks* pAllocationCallbacks) :
4290 m_pBlock(VMA_NULL),
4291 m_HasNonMovableAllocations(true),
4292 m_Allocations(pAllocationCallbacks),
4293 m_pMappedDataForDefragmentation(VMA_NULL)
4294 {
4295 }
4296
CalcHasNonMovableAllocationsBlockInfo4297 void CalcHasNonMovableAllocations()
4298 {
4299 const size_t blockAllocCount = m_pBlock->m_Metadata.GetAllocationCount();
4300 const size_t defragmentAllocCount = m_Allocations.size();
4301 m_HasNonMovableAllocations = blockAllocCount != defragmentAllocCount;
4302 }
4303
SortAllocationsBySizeDescecndingBlockInfo4304 void SortAllocationsBySizeDescecnding()
4305 {
4306 VMA_SORT(m_Allocations.begin(), m_Allocations.end(), AllocationInfoSizeGreater());
4307 }
4308
4309 VkResult EnsureMapping(VmaAllocator hAllocator, void** ppMappedData);
4310 void Unmap(VmaAllocator hAllocator);
4311
4312 private:
4313 // Not null if mapped for defragmentation only, not originally mapped.
4314 void* m_pMappedDataForDefragmentation;
4315 };
4316
4317 struct BlockPointerLess
4318 {
operatorBlockPointerLess4319 bool operator()(const BlockInfo* pLhsBlockInfo, const VmaDeviceMemoryBlock* pRhsBlock) const
4320 {
4321 return pLhsBlockInfo->m_pBlock < pRhsBlock;
4322 }
operatorBlockPointerLess4323 bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4324 {
4325 return pLhsBlockInfo->m_pBlock < pRhsBlockInfo->m_pBlock;
4326 }
4327 };
4328
4329 // 1. Blocks with some non-movable allocations go first.
4330 // 2. Blocks with smaller sumFreeSize go first.
4331 struct BlockInfoCompareMoveDestination
4332 {
operatorBlockInfoCompareMoveDestination4333 bool operator()(const BlockInfo* pLhsBlockInfo, const BlockInfo* pRhsBlockInfo) const
4334 {
4335 if(pLhsBlockInfo->m_HasNonMovableAllocations && !pRhsBlockInfo->m_HasNonMovableAllocations)
4336 {
4337 return true;
4338 }
4339 if(!pLhsBlockInfo->m_HasNonMovableAllocations && pRhsBlockInfo->m_HasNonMovableAllocations)
4340 {
4341 return false;
4342 }
4343 if(pLhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize() < pRhsBlockInfo->m_pBlock->m_Metadata.GetSumFreeSize())
4344 {
4345 return true;
4346 }
4347 return false;
4348 }
4349 };
4350
4351 typedef VmaVector< BlockInfo*, VmaStlAllocator<BlockInfo*> > BlockInfoVector;
4352 BlockInfoVector m_Blocks;
4353
4354 VkResult DefragmentRound(
4355 VkDeviceSize maxBytesToMove,
4356 uint32_t maxAllocationsToMove);
4357
4358 static bool MoveMakesSense(
4359 size_t dstBlockIndex, VkDeviceSize dstOffset,
4360 size_t srcBlockIndex, VkDeviceSize srcOffset);
4361
4362 public:
4363 VmaDefragmentator(
4364 VmaAllocator hAllocator,
4365 VmaBlockVector* pBlockVector,
4366 uint32_t currentFrameIndex);
4367
4368 ~VmaDefragmentator();
4369
GetBytesMoved()4370 VkDeviceSize GetBytesMoved() const { return m_BytesMoved; }
GetAllocationsMoved()4371 uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; }
4372
4373 void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged);
4374
4375 VkResult Defragment(
4376 VkDeviceSize maxBytesToMove,
4377 uint32_t maxAllocationsToMove);
4378 };
4379
4380 // Main allocator object.
4381 struct VmaAllocator_T
4382 {
4383 bool m_UseMutex;
4384 bool m_UseKhrDedicatedAllocation;
4385 VkDevice m_hDevice;
4386 bool m_AllocationCallbacksSpecified;
4387 VkAllocationCallbacks m_AllocationCallbacks;
4388 VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks;
4389
4390 // Number of bytes free out of limit, or VK_WHOLE_SIZE if not limit for that heap.
4391 VkDeviceSize m_HeapSizeLimit[VK_MAX_MEMORY_HEAPS];
4392 VMA_MUTEX m_HeapSizeLimitMutex;
4393
4394 VkPhysicalDeviceProperties m_PhysicalDeviceProperties;
4395 VkPhysicalDeviceMemoryProperties m_MemProps;
4396
4397 // Default pools.
4398 VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES];
4399
4400 // Each vector is sorted by memory (handle value).
4401 typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType;
4402 AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES];
4403 VMA_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES];
4404
4405 VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo);
4406 ~VmaAllocator_T();
4407
GetAllocationCallbacksVmaAllocator_T4408 const VkAllocationCallbacks* GetAllocationCallbacks() const
4409 {
4410 return m_AllocationCallbacksSpecified ? &m_AllocationCallbacks : 0;
4411 }
GetVulkanFunctionsVmaAllocator_T4412 const VmaVulkanFunctions& GetVulkanFunctions() const
4413 {
4414 return m_VulkanFunctions;
4415 }
4416
GetBufferImageGranularityVmaAllocator_T4417 VkDeviceSize GetBufferImageGranularity() const
4418 {
4419 return VMA_MAX(
4420 static_cast<VkDeviceSize>(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY),
4421 m_PhysicalDeviceProperties.limits.bufferImageGranularity);
4422 }
4423
GetMemoryHeapCountVmaAllocator_T4424 uint32_t GetMemoryHeapCount() const { return m_MemProps.memoryHeapCount; }
GetMemoryTypeCountVmaAllocator_T4425 uint32_t GetMemoryTypeCount() const { return m_MemProps.memoryTypeCount; }
4426
MemoryTypeIndexToHeapIndexVmaAllocator_T4427 uint32_t MemoryTypeIndexToHeapIndex(uint32_t memTypeIndex) const
4428 {
4429 VMA_ASSERT(memTypeIndex < m_MemProps.memoryTypeCount);
4430 return m_MemProps.memoryTypes[memTypeIndex].heapIndex;
4431 }
4432
4433 void GetBufferMemoryRequirements(
4434 VkBuffer hBuffer,
4435 VkMemoryRequirements& memReq,
4436 bool& requiresDedicatedAllocation,
4437 bool& prefersDedicatedAllocation) const;
4438 void GetImageMemoryRequirements(
4439 VkImage hImage,
4440 VkMemoryRequirements& memReq,
4441 bool& requiresDedicatedAllocation,
4442 bool& prefersDedicatedAllocation) const;
4443
4444 // Main allocation function.
4445 VkResult AllocateMemory(
4446 const VkMemoryRequirements& vkMemReq,
4447 bool requiresDedicatedAllocation,
4448 bool prefersDedicatedAllocation,
4449 VkBuffer dedicatedBuffer,
4450 VkImage dedicatedImage,
4451 const VmaAllocationCreateInfo& createInfo,
4452 VmaSuballocationType suballocType,
4453 VmaAllocation* pAllocation);
4454
4455 // Main deallocation function.
4456 void FreeMemory(const VmaAllocation allocation);
4457
4458 void CalculateStats(VmaStats* pStats);
4459
4460 #if VMA_STATS_STRING_ENABLED
4461 void PrintDetailedMap(class VmaJsonWriter& json);
4462 #endif
4463
4464 VkResult Defragment(
4465 VmaAllocation* pAllocations,
4466 size_t allocationCount,
4467 VkBool32* pAllocationsChanged,
4468 const VmaDefragmentationInfo* pDefragmentationInfo,
4469 VmaDefragmentationStats* pDefragmentationStats);
4470
4471 void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo);
4472 bool TouchAllocation(VmaAllocation hAllocation);
4473
4474 VkResult CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool);
4475 void DestroyPool(VmaPool pool);
4476 void GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats);
4477
4478 void SetCurrentFrameIndex(uint32_t frameIndex);
4479
4480 void MakePoolAllocationsLost(
4481 VmaPool hPool,
4482 size_t* pLostAllocationCount);
4483
4484 void CreateLostAllocation(VmaAllocation* pAllocation);
4485
4486 VkResult AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory);
4487 void FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory);
4488
4489 VkResult Map(VmaAllocation hAllocation, void** ppData);
4490 void Unmap(VmaAllocation hAllocation);
4491
4492 VkResult BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer);
4493 VkResult BindImageMemory(VmaAllocation hAllocation, VkImage hImage);
4494
4495 private:
4496 VkDeviceSize m_PreferredLargeHeapBlockSize;
4497
4498 VkPhysicalDevice m_PhysicalDevice;
4499 VMA_ATOMIC_UINT32 m_CurrentFrameIndex;
4500
4501 VMA_MUTEX m_PoolsMutex;
4502 // Protected by m_PoolsMutex. Sorted by pointer value.
4503 VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools;
4504
4505 VmaVulkanFunctions m_VulkanFunctions;
4506
4507 void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions);
4508
4509 VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex);
4510
4511 VkResult AllocateMemoryOfType(
4512 const VkMemoryRequirements& vkMemReq,
4513 bool dedicatedAllocation,
4514 VkBuffer dedicatedBuffer,
4515 VkImage dedicatedImage,
4516 const VmaAllocationCreateInfo& createInfo,
4517 uint32_t memTypeIndex,
4518 VmaSuballocationType suballocType,
4519 VmaAllocation* pAllocation);
4520
4521 // Allocates and registers new VkDeviceMemory specifically for single allocation.
4522 VkResult AllocateDedicatedMemory(
4523 VkDeviceSize size,
4524 VmaSuballocationType suballocType,
4525 uint32_t memTypeIndex,
4526 bool map,
4527 bool isUserDataString,
4528 void* pUserData,
4529 VkBuffer dedicatedBuffer,
4530 VkImage dedicatedImage,
4531 VmaAllocation* pAllocation);
4532
4533 // Tries to free pMemory as Dedicated Memory. Returns true if found and freed.
4534 void FreeDedicatedMemory(VmaAllocation allocation);
4535 };
4536
4537 ////////////////////////////////////////////////////////////////////////////////
4538 // Memory allocation #2 after VmaAllocator_T definition
4539
VmaMalloc(VmaAllocator hAllocator,size_t size,size_t alignment)4540 static void* VmaMalloc(VmaAllocator hAllocator, size_t size, size_t alignment)
4541 {
4542 return VmaMalloc(&hAllocator->m_AllocationCallbacks, size, alignment);
4543 }
4544
VmaFree(VmaAllocator hAllocator,void * ptr)4545 static void VmaFree(VmaAllocator hAllocator, void* ptr)
4546 {
4547 VmaFree(&hAllocator->m_AllocationCallbacks, ptr);
4548 }
4549
4550 template<typename T>
VmaAllocate(VmaAllocator hAllocator)4551 static T* VmaAllocate(VmaAllocator hAllocator)
4552 {
4553 return (T*)VmaMalloc(hAllocator, sizeof(T), VMA_ALIGN_OF(T));
4554 }
4555
4556 template<typename T>
VmaAllocateArray(VmaAllocator hAllocator,size_t count)4557 static T* VmaAllocateArray(VmaAllocator hAllocator, size_t count)
4558 {
4559 return (T*)VmaMalloc(hAllocator, sizeof(T) * count, VMA_ALIGN_OF(T));
4560 }
4561
4562 template<typename T>
vma_delete(VmaAllocator hAllocator,T * ptr)4563 static void vma_delete(VmaAllocator hAllocator, T* ptr)
4564 {
4565 if(ptr != VMA_NULL)
4566 {
4567 ptr->~T();
4568 VmaFree(hAllocator, ptr);
4569 }
4570 }
4571
4572 template<typename T>
vma_delete_array(VmaAllocator hAllocator,T * ptr,size_t count)4573 static void vma_delete_array(VmaAllocator hAllocator, T* ptr, size_t count)
4574 {
4575 if(ptr != VMA_NULL)
4576 {
4577 for(size_t i = count; i--; )
4578 ptr[i].~T();
4579 VmaFree(hAllocator, ptr);
4580 }
4581 }
4582
4583 ////////////////////////////////////////////////////////////////////////////////
4584 // VmaStringBuilder
4585
4586 #if VMA_STATS_STRING_ENABLED
4587
4588 class VmaStringBuilder
4589 {
4590 public:
VmaStringBuilder(VmaAllocator alloc)4591 VmaStringBuilder(VmaAllocator alloc) : m_Data(VmaStlAllocator<char>(alloc->GetAllocationCallbacks())) { }
GetLength()4592 size_t GetLength() const { return m_Data.size(); }
GetData()4593 const char* GetData() const { return m_Data.data(); }
4594
Add(char ch)4595 void Add(char ch) { m_Data.push_back(ch); }
4596 void Add(const char* pStr);
AddNewLine()4597 void AddNewLine() { Add('\n'); }
4598 void AddNumber(uint32_t num);
4599 void AddNumber(uint64_t num);
4600 void AddPointer(const void* ptr);
4601
4602 private:
4603 VmaVector< char, VmaStlAllocator<char> > m_Data;
4604 };
4605
Add(const char * pStr)4606 void VmaStringBuilder::Add(const char* pStr)
4607 {
4608 const size_t strLen = strlen(pStr);
4609 if(strLen > 0)
4610 {
4611 const size_t oldCount = m_Data.size();
4612 m_Data.resize(oldCount + strLen);
4613 memcpy(m_Data.data() + oldCount, pStr, strLen);
4614 }
4615 }
4616
AddNumber(uint32_t num)4617 void VmaStringBuilder::AddNumber(uint32_t num)
4618 {
4619 char buf[11];
4620 VmaUint32ToStr(buf, sizeof(buf), num);
4621 Add(buf);
4622 }
4623
AddNumber(uint64_t num)4624 void VmaStringBuilder::AddNumber(uint64_t num)
4625 {
4626 char buf[21];
4627 VmaUint64ToStr(buf, sizeof(buf), num);
4628 Add(buf);
4629 }
4630
AddPointer(const void * ptr)4631 void VmaStringBuilder::AddPointer(const void* ptr)
4632 {
4633 char buf[21];
4634 VmaPtrToStr(buf, sizeof(buf), ptr);
4635 Add(buf);
4636 }
4637
4638 #endif // #if VMA_STATS_STRING_ENABLED
4639
4640 ////////////////////////////////////////////////////////////////////////////////
4641 // VmaJsonWriter
4642
4643 #if VMA_STATS_STRING_ENABLED
4644
4645 class VmaJsonWriter
4646 {
4647 public:
4648 VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb);
4649 ~VmaJsonWriter();
4650
4651 void BeginObject(bool singleLine = false);
4652 void EndObject();
4653
4654 void BeginArray(bool singleLine = false);
4655 void EndArray();
4656
4657 void WriteString(const char* pStr);
4658 void BeginString(const char* pStr = VMA_NULL);
4659 void ContinueString(const char* pStr);
4660 void ContinueString(uint32_t n);
4661 void ContinueString(uint64_t n);
4662 void ContinueString_Pointer(const void* ptr);
4663 void EndString(const char* pStr = VMA_NULL);
4664
4665 void WriteNumber(uint32_t n);
4666 void WriteNumber(uint64_t n);
4667 void WriteBool(bool b);
4668 void WriteNull();
4669
4670 private:
4671 static const char* const INDENT;
4672
4673 enum COLLECTION_TYPE
4674 {
4675 COLLECTION_TYPE_OBJECT,
4676 COLLECTION_TYPE_ARRAY,
4677 };
4678 struct StackItem
4679 {
4680 COLLECTION_TYPE type;
4681 uint32_t valueCount;
4682 bool singleLineMode;
4683 };
4684
4685 VmaStringBuilder& m_SB;
4686 VmaVector< StackItem, VmaStlAllocator<StackItem> > m_Stack;
4687 bool m_InsideString;
4688
4689 void BeginValue(bool isString);
4690 void WriteIndent(bool oneLess = false);
4691 };
4692
4693 const char* const VmaJsonWriter::INDENT = " ";
4694
VmaJsonWriter(const VkAllocationCallbacks * pAllocationCallbacks,VmaStringBuilder & sb)4695 VmaJsonWriter::VmaJsonWriter(const VkAllocationCallbacks* pAllocationCallbacks, VmaStringBuilder& sb) :
4696 m_SB(sb),
4697 m_Stack(VmaStlAllocator<StackItem>(pAllocationCallbacks)),
4698 m_InsideString(false)
4699 {
4700 }
4701
~VmaJsonWriter()4702 VmaJsonWriter::~VmaJsonWriter()
4703 {
4704 VMA_ASSERT(!m_InsideString);
4705 VMA_ASSERT(m_Stack.empty());
4706 }
4707
BeginObject(bool singleLine)4708 void VmaJsonWriter::BeginObject(bool singleLine)
4709 {
4710 VMA_ASSERT(!m_InsideString);
4711
4712 BeginValue(false);
4713 m_SB.Add('{');
4714
4715 StackItem item;
4716 item.type = COLLECTION_TYPE_OBJECT;
4717 item.valueCount = 0;
4718 item.singleLineMode = singleLine;
4719 m_Stack.push_back(item);
4720 }
4721
EndObject()4722 void VmaJsonWriter::EndObject()
4723 {
4724 VMA_ASSERT(!m_InsideString);
4725
4726 WriteIndent(true);
4727 m_SB.Add('}');
4728
4729 VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_OBJECT);
4730 m_Stack.pop_back();
4731 }
4732
BeginArray(bool singleLine)4733 void VmaJsonWriter::BeginArray(bool singleLine)
4734 {
4735 VMA_ASSERT(!m_InsideString);
4736
4737 BeginValue(false);
4738 m_SB.Add('[');
4739
4740 StackItem item;
4741 item.type = COLLECTION_TYPE_ARRAY;
4742 item.valueCount = 0;
4743 item.singleLineMode = singleLine;
4744 m_Stack.push_back(item);
4745 }
4746
EndArray()4747 void VmaJsonWriter::EndArray()
4748 {
4749 VMA_ASSERT(!m_InsideString);
4750
4751 WriteIndent(true);
4752 m_SB.Add(']');
4753
4754 VMA_ASSERT(!m_Stack.empty() && m_Stack.back().type == COLLECTION_TYPE_ARRAY);
4755 m_Stack.pop_back();
4756 }
4757
WriteString(const char * pStr)4758 void VmaJsonWriter::WriteString(const char* pStr)
4759 {
4760 BeginString(pStr);
4761 EndString();
4762 }
4763
BeginString(const char * pStr)4764 void VmaJsonWriter::BeginString(const char* pStr)
4765 {
4766 VMA_ASSERT(!m_InsideString);
4767
4768 BeginValue(true);
4769 m_SB.Add('"');
4770 m_InsideString = true;
4771 if(pStr != VMA_NULL && pStr[0] != '\0')
4772 {
4773 ContinueString(pStr);
4774 }
4775 }
4776
ContinueString(const char * pStr)4777 void VmaJsonWriter::ContinueString(const char* pStr)
4778 {
4779 VMA_ASSERT(m_InsideString);
4780
4781 const size_t strLen = strlen(pStr);
4782 for(size_t i = 0; i < strLen; ++i)
4783 {
4784 char ch = pStr[i];
4785 if(ch == '\'')
4786 {
4787 m_SB.Add("\\\\");
4788 }
4789 else if(ch == '"')
4790 {
4791 m_SB.Add("\\\"");
4792 }
4793 else if(ch >= 32)
4794 {
4795 m_SB.Add(ch);
4796 }
4797 else switch(ch)
4798 {
4799 case '\b':
4800 m_SB.Add("\\b");
4801 break;
4802 case '\f':
4803 m_SB.Add("\\f");
4804 break;
4805 case '\n':
4806 m_SB.Add("\\n");
4807 break;
4808 case '\r':
4809 m_SB.Add("\\r");
4810 break;
4811 case '\t':
4812 m_SB.Add("\\t");
4813 break;
4814 default:
4815 VMA_ASSERT(0 && "Character not currently supported.");
4816 break;
4817 }
4818 }
4819 }
4820
ContinueString(uint32_t n)4821 void VmaJsonWriter::ContinueString(uint32_t n)
4822 {
4823 VMA_ASSERT(m_InsideString);
4824 m_SB.AddNumber(n);
4825 }
4826
ContinueString(uint64_t n)4827 void VmaJsonWriter::ContinueString(uint64_t n)
4828 {
4829 VMA_ASSERT(m_InsideString);
4830 m_SB.AddNumber(n);
4831 }
4832
ContinueString_Pointer(const void * ptr)4833 void VmaJsonWriter::ContinueString_Pointer(const void* ptr)
4834 {
4835 VMA_ASSERT(m_InsideString);
4836 m_SB.AddPointer(ptr);
4837 }
4838
EndString(const char * pStr)4839 void VmaJsonWriter::EndString(const char* pStr)
4840 {
4841 VMA_ASSERT(m_InsideString);
4842 if(pStr != VMA_NULL && pStr[0] != '\0')
4843 {
4844 ContinueString(pStr);
4845 }
4846 m_SB.Add('"');
4847 m_InsideString = false;
4848 }
4849
WriteNumber(uint32_t n)4850 void VmaJsonWriter::WriteNumber(uint32_t n)
4851 {
4852 VMA_ASSERT(!m_InsideString);
4853 BeginValue(false);
4854 m_SB.AddNumber(n);
4855 }
4856
WriteNumber(uint64_t n)4857 void VmaJsonWriter::WriteNumber(uint64_t n)
4858 {
4859 VMA_ASSERT(!m_InsideString);
4860 BeginValue(false);
4861 m_SB.AddNumber(n);
4862 }
4863
WriteBool(bool b)4864 void VmaJsonWriter::WriteBool(bool b)
4865 {
4866 VMA_ASSERT(!m_InsideString);
4867 BeginValue(false);
4868 m_SB.Add(b ? "true" : "false");
4869 }
4870
WriteNull()4871 void VmaJsonWriter::WriteNull()
4872 {
4873 VMA_ASSERT(!m_InsideString);
4874 BeginValue(false);
4875 m_SB.Add("null");
4876 }
4877
BeginValue(bool isString)4878 void VmaJsonWriter::BeginValue(bool isString)
4879 {
4880 if(!m_Stack.empty())
4881 {
4882 StackItem& currItem = m_Stack.back();
4883 if(currItem.type == COLLECTION_TYPE_OBJECT &&
4884 currItem.valueCount % 2 == 0)
4885 {
4886 VMA_ASSERT(isString);
4887 }
4888
4889 if(currItem.type == COLLECTION_TYPE_OBJECT &&
4890 currItem.valueCount % 2 != 0)
4891 {
4892 m_SB.Add(": ");
4893 }
4894 else if(currItem.valueCount > 0)
4895 {
4896 m_SB.Add(", ");
4897 WriteIndent();
4898 }
4899 else
4900 {
4901 WriteIndent();
4902 }
4903 ++currItem.valueCount;
4904 }
4905 }
4906
WriteIndent(bool oneLess)4907 void VmaJsonWriter::WriteIndent(bool oneLess)
4908 {
4909 if(!m_Stack.empty() && !m_Stack.back().singleLineMode)
4910 {
4911 m_SB.AddNewLine();
4912
4913 size_t count = m_Stack.size();
4914 if(count > 0 && oneLess)
4915 {
4916 --count;
4917 }
4918 for(size_t i = 0; i < count; ++i)
4919 {
4920 m_SB.Add(INDENT);
4921 }
4922 }
4923 }
4924
4925 #endif // #if VMA_STATS_STRING_ENABLED
4926
4927 ////////////////////////////////////////////////////////////////////////////////
4928
SetUserData(VmaAllocator hAllocator,void * pUserData)4929 void VmaAllocation_T::SetUserData(VmaAllocator hAllocator, void* pUserData)
4930 {
4931 if(IsUserDataString())
4932 {
4933 VMA_ASSERT(pUserData == VMA_NULL || pUserData != m_pUserData);
4934
4935 FreeUserDataString(hAllocator);
4936
4937 if(pUserData != VMA_NULL)
4938 {
4939 const char* const newStrSrc = (char*)pUserData;
4940 const size_t newStrLen = strlen(newStrSrc);
4941 char* const newStrDst = vma_new_array(hAllocator, char, newStrLen + 1);
4942 memcpy(newStrDst, newStrSrc, newStrLen + 1);
4943 m_pUserData = newStrDst;
4944 }
4945 }
4946 else
4947 {
4948 m_pUserData = pUserData;
4949 }
4950 }
4951
ChangeBlockAllocation(VmaAllocator hAllocator,VmaDeviceMemoryBlock * block,VkDeviceSize offset)4952 void VmaAllocation_T::ChangeBlockAllocation(
4953 VmaAllocator hAllocator,
4954 VmaDeviceMemoryBlock* block,
4955 VkDeviceSize offset)
4956 {
4957 VMA_ASSERT(block != VMA_NULL);
4958 VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
4959
4960 // Move mapping reference counter from old block to new block.
4961 if(block != m_BlockAllocation.m_Block)
4962 {
4963 uint32_t mapRefCount = m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP;
4964 if(IsPersistentMap())
4965 ++mapRefCount;
4966 m_BlockAllocation.m_Block->Unmap(hAllocator, mapRefCount);
4967 block->Map(hAllocator, mapRefCount, VMA_NULL);
4968 }
4969
4970 m_BlockAllocation.m_Block = block;
4971 m_BlockAllocation.m_Offset = offset;
4972 }
4973
GetOffset()4974 VkDeviceSize VmaAllocation_T::GetOffset() const
4975 {
4976 switch(m_Type)
4977 {
4978 case ALLOCATION_TYPE_BLOCK:
4979 return m_BlockAllocation.m_Offset;
4980 case ALLOCATION_TYPE_DEDICATED:
4981 return 0;
4982 default:
4983 VMA_ASSERT(0);
4984 return 0;
4985 }
4986 }
4987
GetMemory()4988 VkDeviceMemory VmaAllocation_T::GetMemory() const
4989 {
4990 switch(m_Type)
4991 {
4992 case ALLOCATION_TYPE_BLOCK:
4993 return m_BlockAllocation.m_Block->GetDeviceMemory();
4994 case ALLOCATION_TYPE_DEDICATED:
4995 return m_DedicatedAllocation.m_hMemory;
4996 default:
4997 VMA_ASSERT(0);
4998 return VK_NULL_HANDLE;
4999 }
5000 }
5001
GetMemoryTypeIndex()5002 uint32_t VmaAllocation_T::GetMemoryTypeIndex() const
5003 {
5004 switch(m_Type)
5005 {
5006 case ALLOCATION_TYPE_BLOCK:
5007 return m_BlockAllocation.m_Block->GetMemoryTypeIndex();
5008 case ALLOCATION_TYPE_DEDICATED:
5009 return m_DedicatedAllocation.m_MemoryTypeIndex;
5010 default:
5011 VMA_ASSERT(0);
5012 return UINT32_MAX;
5013 }
5014 }
5015
GetMappedData()5016 void* VmaAllocation_T::GetMappedData() const
5017 {
5018 switch(m_Type)
5019 {
5020 case ALLOCATION_TYPE_BLOCK:
5021 if(m_MapCount != 0)
5022 {
5023 void* pBlockData = m_BlockAllocation.m_Block->GetMappedData();
5024 VMA_ASSERT(pBlockData != VMA_NULL);
5025 return (char*)pBlockData + m_BlockAllocation.m_Offset;
5026 }
5027 else
5028 {
5029 return VMA_NULL;
5030 }
5031 break;
5032 case ALLOCATION_TYPE_DEDICATED:
5033 VMA_ASSERT((m_DedicatedAllocation.m_pMappedData != VMA_NULL) == (m_MapCount != 0));
5034 return m_DedicatedAllocation.m_pMappedData;
5035 default:
5036 VMA_ASSERT(0);
5037 return VMA_NULL;
5038 }
5039 }
5040
CanBecomeLost()5041 bool VmaAllocation_T::CanBecomeLost() const
5042 {
5043 switch(m_Type)
5044 {
5045 case ALLOCATION_TYPE_BLOCK:
5046 return m_BlockAllocation.m_CanBecomeLost;
5047 case ALLOCATION_TYPE_DEDICATED:
5048 return false;
5049 default:
5050 VMA_ASSERT(0);
5051 return false;
5052 }
5053 }
5054
GetPool()5055 VmaPool VmaAllocation_T::GetPool() const
5056 {
5057 VMA_ASSERT(m_Type == ALLOCATION_TYPE_BLOCK);
5058 return m_BlockAllocation.m_hPool;
5059 }
5060
MakeLost(uint32_t currentFrameIndex,uint32_t frameInUseCount)5061 bool VmaAllocation_T::MakeLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5062 {
5063 VMA_ASSERT(CanBecomeLost());
5064
5065 /*
5066 Warning: This is a carefully designed algorithm.
5067 Do not modify unless you really know what you're doing :)
5068 */
5069 uint32_t localLastUseFrameIndex = GetLastUseFrameIndex();
5070 for(;;)
5071 {
5072 if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
5073 {
5074 VMA_ASSERT(0);
5075 return false;
5076 }
5077 else if(localLastUseFrameIndex + frameInUseCount >= currentFrameIndex)
5078 {
5079 return false;
5080 }
5081 else // Last use time earlier than current time.
5082 {
5083 if(CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, VMA_FRAME_INDEX_LOST))
5084 {
5085 // Setting hAllocation.LastUseFrameIndex atomic to VMA_FRAME_INDEX_LOST is enough to mark it as LOST.
5086 // Calling code just needs to unregister this allocation in owning VmaDeviceMemoryBlock.
5087 return true;
5088 }
5089 }
5090 }
5091 }
5092
FreeUserDataString(VmaAllocator hAllocator)5093 void VmaAllocation_T::FreeUserDataString(VmaAllocator hAllocator)
5094 {
5095 VMA_ASSERT(IsUserDataString());
5096 if(m_pUserData != VMA_NULL)
5097 {
5098 char* const oldStr = (char*)m_pUserData;
5099 const size_t oldStrLen = strlen(oldStr);
5100 vma_delete_array(hAllocator, oldStr, oldStrLen + 1);
5101 m_pUserData = VMA_NULL;
5102 }
5103 }
5104
BlockAllocMap()5105 void VmaAllocation_T::BlockAllocMap()
5106 {
5107 VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
5108
5109 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
5110 {
5111 ++m_MapCount;
5112 }
5113 else
5114 {
5115 VMA_ASSERT(0 && "Allocation mapped too many times simultaneously.");
5116 }
5117 }
5118
BlockAllocUnmap()5119 void VmaAllocation_T::BlockAllocUnmap()
5120 {
5121 VMA_ASSERT(GetType() == ALLOCATION_TYPE_BLOCK);
5122
5123 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
5124 {
5125 --m_MapCount;
5126 }
5127 else
5128 {
5129 VMA_ASSERT(0 && "Unmapping allocation not previously mapped.");
5130 }
5131 }
5132
DedicatedAllocMap(VmaAllocator hAllocator,void ** ppData)5133 VkResult VmaAllocation_T::DedicatedAllocMap(VmaAllocator hAllocator, void** ppData)
5134 {
5135 VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
5136
5137 if(m_MapCount != 0)
5138 {
5139 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) < 0x7F)
5140 {
5141 VMA_ASSERT(m_DedicatedAllocation.m_pMappedData != VMA_NULL);
5142 *ppData = m_DedicatedAllocation.m_pMappedData;
5143 ++m_MapCount;
5144 return VK_SUCCESS;
5145 }
5146 else
5147 {
5148 VMA_ASSERT(0 && "Dedicated allocation mapped too many times simultaneously.");
5149 return VK_ERROR_MEMORY_MAP_FAILED;
5150 }
5151 }
5152 else
5153 {
5154 VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
5155 hAllocator->m_hDevice,
5156 m_DedicatedAllocation.m_hMemory,
5157 0, // offset
5158 VK_WHOLE_SIZE,
5159 0, // flags
5160 ppData);
5161 if(result == VK_SUCCESS)
5162 {
5163 m_DedicatedAllocation.m_pMappedData = *ppData;
5164 m_MapCount = 1;
5165 }
5166 return result;
5167 }
5168 }
5169
DedicatedAllocUnmap(VmaAllocator hAllocator)5170 void VmaAllocation_T::DedicatedAllocUnmap(VmaAllocator hAllocator)
5171 {
5172 VMA_ASSERT(GetType() == ALLOCATION_TYPE_DEDICATED);
5173
5174 if((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) != 0)
5175 {
5176 --m_MapCount;
5177 if(m_MapCount == 0)
5178 {
5179 m_DedicatedAllocation.m_pMappedData = VMA_NULL;
5180 (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(
5181 hAllocator->m_hDevice,
5182 m_DedicatedAllocation.m_hMemory);
5183 }
5184 }
5185 else
5186 {
5187 VMA_ASSERT(0 && "Unmapping dedicated allocation not previously mapped.");
5188 }
5189 }
5190
5191 #if VMA_STATS_STRING_ENABLED
5192
5193 // Correspond to values of enum VmaSuballocationType.
5194 static const char* VMA_SUBALLOCATION_TYPE_NAMES[] = {
5195 "FREE",
5196 "UNKNOWN",
5197 "BUFFER",
5198 "IMAGE_UNKNOWN",
5199 "IMAGE_LINEAR",
5200 "IMAGE_OPTIMAL",
5201 };
5202
VmaPrintStatInfo(VmaJsonWriter & json,const VmaStatInfo & stat)5203 static void VmaPrintStatInfo(VmaJsonWriter& json, const VmaStatInfo& stat)
5204 {
5205 json.BeginObject();
5206
5207 json.WriteString("Blocks");
5208 json.WriteNumber(stat.blockCount);
5209
5210 json.WriteString("Allocations");
5211 json.WriteNumber(stat.allocationCount);
5212
5213 json.WriteString("UnusedRanges");
5214 json.WriteNumber(stat.unusedRangeCount);
5215
5216 json.WriteString("UsedBytes");
5217 json.WriteNumber(stat.usedBytes);
5218
5219 json.WriteString("UnusedBytes");
5220 json.WriteNumber(stat.unusedBytes);
5221
5222 if(stat.allocationCount > 1)
5223 {
5224 json.WriteString("AllocationSize");
5225 json.BeginObject(true);
5226 json.WriteString("Min");
5227 json.WriteNumber(stat.allocationSizeMin);
5228 json.WriteString("Avg");
5229 json.WriteNumber(stat.allocationSizeAvg);
5230 json.WriteString("Max");
5231 json.WriteNumber(stat.allocationSizeMax);
5232 json.EndObject();
5233 }
5234
5235 if(stat.unusedRangeCount > 1)
5236 {
5237 json.WriteString("UnusedRangeSize");
5238 json.BeginObject(true);
5239 json.WriteString("Min");
5240 json.WriteNumber(stat.unusedRangeSizeMin);
5241 json.WriteString("Avg");
5242 json.WriteNumber(stat.unusedRangeSizeAvg);
5243 json.WriteString("Max");
5244 json.WriteNumber(stat.unusedRangeSizeMax);
5245 json.EndObject();
5246 }
5247
5248 json.EndObject();
5249 }
5250
5251 #endif // #if VMA_STATS_STRING_ENABLED
5252
5253 struct VmaSuballocationItemSizeLess
5254 {
operatorVmaSuballocationItemSizeLess5255 bool operator()(
5256 const VmaSuballocationList::iterator lhs,
5257 const VmaSuballocationList::iterator rhs) const
5258 {
5259 return lhs->size < rhs->size;
5260 }
operatorVmaSuballocationItemSizeLess5261 bool operator()(
5262 const VmaSuballocationList::iterator lhs,
5263 VkDeviceSize rhsSize) const
5264 {
5265 return lhs->size < rhsSize;
5266 }
5267 };
5268
5269 ////////////////////////////////////////////////////////////////////////////////
5270 // class VmaBlockMetadata
5271
VmaBlockMetadata(VmaAllocator hAllocator)5272 VmaBlockMetadata::VmaBlockMetadata(VmaAllocator hAllocator) :
5273 m_Size(0),
5274 m_FreeCount(0),
5275 m_SumFreeSize(0),
5276 m_Suballocations(VmaStlAllocator<VmaSuballocation>(hAllocator->GetAllocationCallbacks())),
5277 m_FreeSuballocationsBySize(VmaStlAllocator<VmaSuballocationList::iterator>(hAllocator->GetAllocationCallbacks()))
5278 {
5279 }
5280
~VmaBlockMetadata()5281 VmaBlockMetadata::~VmaBlockMetadata()
5282 {
5283 }
5284
Init(VkDeviceSize size)5285 void VmaBlockMetadata::Init(VkDeviceSize size)
5286 {
5287 m_Size = size;
5288 m_FreeCount = 1;
5289 m_SumFreeSize = size;
5290
5291 VmaSuballocation suballoc = {};
5292 suballoc.offset = 0;
5293 suballoc.size = size;
5294 suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5295 suballoc.hAllocation = VK_NULL_HANDLE;
5296
5297 m_Suballocations.push_back(suballoc);
5298 VmaSuballocationList::iterator suballocItem = m_Suballocations.end();
5299 --suballocItem;
5300 m_FreeSuballocationsBySize.push_back(suballocItem);
5301 }
5302
Validate()5303 bool VmaBlockMetadata::Validate() const
5304 {
5305 if(m_Suballocations.empty())
5306 {
5307 return false;
5308 }
5309
5310 // Expected offset of new suballocation as calculates from previous ones.
5311 VkDeviceSize calculatedOffset = 0;
5312 // Expected number of free suballocations as calculated from traversing their list.
5313 uint32_t calculatedFreeCount = 0;
5314 // Expected sum size of free suballocations as calculated from traversing their list.
5315 VkDeviceSize calculatedSumFreeSize = 0;
5316 // Expected number of free suballocations that should be registered in
5317 // m_FreeSuballocationsBySize calculated from traversing their list.
5318 size_t freeSuballocationsToRegister = 0;
5319 // True if previous visisted suballocation was free.
5320 bool prevFree = false;
5321
5322 for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5323 suballocItem != m_Suballocations.cend();
5324 ++suballocItem)
5325 {
5326 const VmaSuballocation& subAlloc = *suballocItem;
5327
5328 // Actual offset of this suballocation doesn't match expected one.
5329 if(subAlloc.offset != calculatedOffset)
5330 {
5331 return false;
5332 }
5333
5334 const bool currFree = (subAlloc.type == VMA_SUBALLOCATION_TYPE_FREE);
5335 // Two adjacent free suballocations are invalid. They should be merged.
5336 if(prevFree && currFree)
5337 {
5338 return false;
5339 }
5340
5341 if(currFree != (subAlloc.hAllocation == VK_NULL_HANDLE))
5342 {
5343 return false;
5344 }
5345
5346 if(currFree)
5347 {
5348 calculatedSumFreeSize += subAlloc.size;
5349 ++calculatedFreeCount;
5350 if(subAlloc.size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5351 {
5352 ++freeSuballocationsToRegister;
5353 }
5354 }
5355 else
5356 {
5357 if(subAlloc.hAllocation->GetOffset() != subAlloc.offset)
5358 {
5359 return false;
5360 }
5361 if(subAlloc.hAllocation->GetSize() != subAlloc.size)
5362 {
5363 return false;
5364 }
5365 }
5366
5367 calculatedOffset += subAlloc.size;
5368 prevFree = currFree;
5369 }
5370
5371 // Number of free suballocations registered in m_FreeSuballocationsBySize doesn't
5372 // match expected one.
5373 if(m_FreeSuballocationsBySize.size() != freeSuballocationsToRegister)
5374 {
5375 return false;
5376 }
5377
5378 VkDeviceSize lastSize = 0;
5379 for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i)
5380 {
5381 VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i];
5382
5383 // Only free suballocations can be registered in m_FreeSuballocationsBySize.
5384 if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5385 {
5386 return false;
5387 }
5388 // They must be sorted by size ascending.
5389 if(suballocItem->size < lastSize)
5390 {
5391 return false;
5392 }
5393
5394 lastSize = suballocItem->size;
5395 }
5396
5397 // Check if totals match calculacted values.
5398 if(!ValidateFreeSuballocationList() ||
5399 (calculatedOffset != m_Size) ||
5400 (calculatedSumFreeSize != m_SumFreeSize) ||
5401 (calculatedFreeCount != m_FreeCount))
5402 {
5403 return false;
5404 }
5405
5406 return true;
5407 }
5408
GetUnusedRangeSizeMax()5409 VkDeviceSize VmaBlockMetadata::GetUnusedRangeSizeMax() const
5410 {
5411 if(!m_FreeSuballocationsBySize.empty())
5412 {
5413 return m_FreeSuballocationsBySize.back()->size;
5414 }
5415 else
5416 {
5417 return 0;
5418 }
5419 }
5420
IsEmpty()5421 bool VmaBlockMetadata::IsEmpty() const
5422 {
5423 return (m_Suballocations.size() == 1) && (m_FreeCount == 1);
5424 }
5425
CalcAllocationStatInfo(VmaStatInfo & outInfo)5426 void VmaBlockMetadata::CalcAllocationStatInfo(VmaStatInfo& outInfo) const
5427 {
5428 outInfo.blockCount = 1;
5429
5430 const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5431 outInfo.allocationCount = rangeCount - m_FreeCount;
5432 outInfo.unusedRangeCount = m_FreeCount;
5433
5434 outInfo.unusedBytes = m_SumFreeSize;
5435 outInfo.usedBytes = m_Size - outInfo.unusedBytes;
5436
5437 outInfo.allocationSizeMin = UINT64_MAX;
5438 outInfo.allocationSizeMax = 0;
5439 outInfo.unusedRangeSizeMin = UINT64_MAX;
5440 outInfo.unusedRangeSizeMax = 0;
5441
5442 for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5443 suballocItem != m_Suballocations.cend();
5444 ++suballocItem)
5445 {
5446 const VmaSuballocation& suballoc = *suballocItem;
5447 if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE)
5448 {
5449 outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size);
5450 outInfo.allocationSizeMax = VMA_MAX(outInfo.allocationSizeMax, suballoc.size);
5451 }
5452 else
5453 {
5454 outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, suballoc.size);
5455 outInfo.unusedRangeSizeMax = VMA_MAX(outInfo.unusedRangeSizeMax, suballoc.size);
5456 }
5457 }
5458 }
5459
AddPoolStats(VmaPoolStats & inoutStats)5460 void VmaBlockMetadata::AddPoolStats(VmaPoolStats& inoutStats) const
5461 {
5462 const uint32_t rangeCount = (uint32_t)m_Suballocations.size();
5463
5464 inoutStats.size += m_Size;
5465 inoutStats.unusedSize += m_SumFreeSize;
5466 inoutStats.allocationCount += rangeCount - m_FreeCount;
5467 inoutStats.unusedRangeCount += m_FreeCount;
5468 inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, GetUnusedRangeSizeMax());
5469 }
5470
5471 #if VMA_STATS_STRING_ENABLED
5472
PrintDetailedMap(class VmaJsonWriter & json)5473 void VmaBlockMetadata::PrintDetailedMap(class VmaJsonWriter& json) const
5474 {
5475 json.BeginObject();
5476
5477 json.WriteString("TotalBytes");
5478 json.WriteNumber(m_Size);
5479
5480 json.WriteString("UnusedBytes");
5481 json.WriteNumber(m_SumFreeSize);
5482
5483 json.WriteString("Allocations");
5484 json.WriteNumber((uint64_t)m_Suballocations.size() - m_FreeCount);
5485
5486 json.WriteString("UnusedRanges");
5487 json.WriteNumber(m_FreeCount);
5488
5489 json.WriteString("Suballocations");
5490 json.BeginArray();
5491 size_t i = 0;
5492 for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin();
5493 suballocItem != m_Suballocations.cend();
5494 ++suballocItem, ++i)
5495 {
5496 json.BeginObject(true);
5497
5498 json.WriteString("Type");
5499 json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[suballocItem->type]);
5500
5501 json.WriteString("Size");
5502 json.WriteNumber(suballocItem->size);
5503
5504 json.WriteString("Offset");
5505 json.WriteNumber(suballocItem->offset);
5506
5507 if(suballocItem->type != VMA_SUBALLOCATION_TYPE_FREE)
5508 {
5509 const void* pUserData = suballocItem->hAllocation->GetUserData();
5510 if(pUserData != VMA_NULL)
5511 {
5512 json.WriteString("UserData");
5513 if(suballocItem->hAllocation->IsUserDataString())
5514 {
5515 json.WriteString((const char*)pUserData);
5516 }
5517 else
5518 {
5519 json.BeginString();
5520 json.ContinueString_Pointer(pUserData);
5521 json.EndString();
5522 }
5523 }
5524 }
5525
5526 json.EndObject();
5527 }
5528 json.EndArray();
5529
5530 json.EndObject();
5531 }
5532
5533 #endif // #if VMA_STATS_STRING_ENABLED
5534
5535 /*
5536 How many suitable free suballocations to analyze before choosing best one.
5537 - Set to 1 to use First-Fit algorithm - first suitable free suballocation will
5538 be chosen.
5539 - Set to UINT32_MAX to use Best-Fit/Worst-Fit algorithm - all suitable free
5540 suballocations will be analized and best one will be chosen.
5541 - Any other value is also acceptable.
5542 */
5543 //static const uint32_t MAX_SUITABLE_SUBALLOCATIONS_TO_CHECK = 8;
5544
CreateFirstAllocationRequest(VmaAllocationRequest * pAllocationRequest)5545 void VmaBlockMetadata::CreateFirstAllocationRequest(VmaAllocationRequest* pAllocationRequest)
5546 {
5547 VMA_ASSERT(IsEmpty());
5548 pAllocationRequest->offset = 0;
5549 pAllocationRequest->sumFreeSize = m_SumFreeSize;
5550 pAllocationRequest->sumItemSize = 0;
5551 pAllocationRequest->item = m_Suballocations.begin();
5552 pAllocationRequest->itemsToMakeLostCount = 0;
5553 }
5554
CreateAllocationRequest(uint32_t currentFrameIndex,uint32_t frameInUseCount,VkDeviceSize bufferImageGranularity,VkDeviceSize allocSize,VkDeviceSize allocAlignment,VmaSuballocationType allocType,bool canMakeOtherLost,VmaAllocationRequest * pAllocationRequest)5555 bool VmaBlockMetadata::CreateAllocationRequest(
5556 uint32_t currentFrameIndex,
5557 uint32_t frameInUseCount,
5558 VkDeviceSize bufferImageGranularity,
5559 VkDeviceSize allocSize,
5560 VkDeviceSize allocAlignment,
5561 VmaSuballocationType allocType,
5562 bool canMakeOtherLost,
5563 VmaAllocationRequest* pAllocationRequest)
5564 {
5565 VMA_ASSERT(allocSize > 0);
5566 VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5567 VMA_ASSERT(pAllocationRequest != VMA_NULL);
5568 VMA_HEAVY_ASSERT(Validate());
5569
5570 // There is not enough total free space in this block to fullfill the request: Early return.
5571 if(canMakeOtherLost == false && m_SumFreeSize < allocSize)
5572 {
5573 return false;
5574 }
5575
5576 // New algorithm, efficiently searching freeSuballocationsBySize.
5577 const size_t freeSuballocCount = m_FreeSuballocationsBySize.size();
5578 if(freeSuballocCount > 0)
5579 {
5580 if(VMA_BEST_FIT)
5581 {
5582 // Find first free suballocation with size not less than allocSize.
5583 VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
5584 m_FreeSuballocationsBySize.data(),
5585 m_FreeSuballocationsBySize.data() + freeSuballocCount,
5586 allocSize,
5587 VmaSuballocationItemSizeLess());
5588 size_t index = it - m_FreeSuballocationsBySize.data();
5589 for(; index < freeSuballocCount; ++index)
5590 {
5591 if(CheckAllocation(
5592 currentFrameIndex,
5593 frameInUseCount,
5594 bufferImageGranularity,
5595 allocSize,
5596 allocAlignment,
5597 allocType,
5598 m_FreeSuballocationsBySize[index],
5599 false, // canMakeOtherLost
5600 &pAllocationRequest->offset,
5601 &pAllocationRequest->itemsToMakeLostCount,
5602 &pAllocationRequest->sumFreeSize,
5603 &pAllocationRequest->sumItemSize))
5604 {
5605 pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5606 return true;
5607 }
5608 }
5609 }
5610 else
5611 {
5612 // Search staring from biggest suballocations.
5613 for(size_t index = freeSuballocCount; index--; )
5614 {
5615 if(CheckAllocation(
5616 currentFrameIndex,
5617 frameInUseCount,
5618 bufferImageGranularity,
5619 allocSize,
5620 allocAlignment,
5621 allocType,
5622 m_FreeSuballocationsBySize[index],
5623 false, // canMakeOtherLost
5624 &pAllocationRequest->offset,
5625 &pAllocationRequest->itemsToMakeLostCount,
5626 &pAllocationRequest->sumFreeSize,
5627 &pAllocationRequest->sumItemSize))
5628 {
5629 pAllocationRequest->item = m_FreeSuballocationsBySize[index];
5630 return true;
5631 }
5632 }
5633 }
5634 }
5635
5636 if(canMakeOtherLost)
5637 {
5638 // Brute-force algorithm. TODO: Come up with something better.
5639
5640 pAllocationRequest->sumFreeSize = VK_WHOLE_SIZE;
5641 pAllocationRequest->sumItemSize = VK_WHOLE_SIZE;
5642
5643 VmaAllocationRequest tmpAllocRequest = {};
5644 for(VmaSuballocationList::iterator suballocIt = m_Suballocations.begin();
5645 suballocIt != m_Suballocations.end();
5646 ++suballocIt)
5647 {
5648 if(suballocIt->type == VMA_SUBALLOCATION_TYPE_FREE ||
5649 suballocIt->hAllocation->CanBecomeLost())
5650 {
5651 if(CheckAllocation(
5652 currentFrameIndex,
5653 frameInUseCount,
5654 bufferImageGranularity,
5655 allocSize,
5656 allocAlignment,
5657 allocType,
5658 suballocIt,
5659 canMakeOtherLost,
5660 &tmpAllocRequest.offset,
5661 &tmpAllocRequest.itemsToMakeLostCount,
5662 &tmpAllocRequest.sumFreeSize,
5663 &tmpAllocRequest.sumItemSize))
5664 {
5665 tmpAllocRequest.item = suballocIt;
5666
5667 if(tmpAllocRequest.CalcCost() < pAllocationRequest->CalcCost())
5668 {
5669 *pAllocationRequest = tmpAllocRequest;
5670 }
5671 }
5672 }
5673 }
5674
5675 if(pAllocationRequest->sumItemSize != VK_WHOLE_SIZE)
5676 {
5677 return true;
5678 }
5679 }
5680
5681 return false;
5682 }
5683
MakeRequestedAllocationsLost(uint32_t currentFrameIndex,uint32_t frameInUseCount,VmaAllocationRequest * pAllocationRequest)5684 bool VmaBlockMetadata::MakeRequestedAllocationsLost(
5685 uint32_t currentFrameIndex,
5686 uint32_t frameInUseCount,
5687 VmaAllocationRequest* pAllocationRequest)
5688 {
5689 while(pAllocationRequest->itemsToMakeLostCount > 0)
5690 {
5691 if(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE)
5692 {
5693 ++pAllocationRequest->item;
5694 }
5695 VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5696 VMA_ASSERT(pAllocationRequest->item->hAllocation != VK_NULL_HANDLE);
5697 VMA_ASSERT(pAllocationRequest->item->hAllocation->CanBecomeLost());
5698 if(pAllocationRequest->item->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5699 {
5700 pAllocationRequest->item = FreeSuballocation(pAllocationRequest->item);
5701 --pAllocationRequest->itemsToMakeLostCount;
5702 }
5703 else
5704 {
5705 return false;
5706 }
5707 }
5708
5709 VMA_HEAVY_ASSERT(Validate());
5710 VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end());
5711 VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE);
5712
5713 return true;
5714 }
5715
MakeAllocationsLost(uint32_t currentFrameIndex,uint32_t frameInUseCount)5716 uint32_t VmaBlockMetadata::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount)
5717 {
5718 uint32_t lostAllocationCount = 0;
5719 for(VmaSuballocationList::iterator it = m_Suballocations.begin();
5720 it != m_Suballocations.end();
5721 ++it)
5722 {
5723 if(it->type != VMA_SUBALLOCATION_TYPE_FREE &&
5724 it->hAllocation->CanBecomeLost() &&
5725 it->hAllocation->MakeLost(currentFrameIndex, frameInUseCount))
5726 {
5727 it = FreeSuballocation(it);
5728 ++lostAllocationCount;
5729 }
5730 }
5731 return lostAllocationCount;
5732 }
5733
Alloc(const VmaAllocationRequest & request,VmaSuballocationType type,VkDeviceSize allocSize,VmaAllocation hAllocation)5734 void VmaBlockMetadata::Alloc(
5735 const VmaAllocationRequest& request,
5736 VmaSuballocationType type,
5737 VkDeviceSize allocSize,
5738 VmaAllocation hAllocation)
5739 {
5740 VMA_ASSERT(request.item != m_Suballocations.end());
5741 VmaSuballocation& suballoc = *request.item;
5742 // Given suballocation is a free block.
5743 VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
5744 // Given offset is inside this suballocation.
5745 VMA_ASSERT(request.offset >= suballoc.offset);
5746 const VkDeviceSize paddingBegin = request.offset - suballoc.offset;
5747 VMA_ASSERT(suballoc.size >= paddingBegin + allocSize);
5748 const VkDeviceSize paddingEnd = suballoc.size - paddingBegin - allocSize;
5749
5750 // Unregister this free suballocation from m_FreeSuballocationsBySize and update
5751 // it to become used.
5752 UnregisterFreeSuballocation(request.item);
5753
5754 suballoc.offset = request.offset;
5755 suballoc.size = allocSize;
5756 suballoc.type = type;
5757 suballoc.hAllocation = hAllocation;
5758
5759 // If there are any free bytes remaining at the end, insert new free suballocation after current one.
5760 if(paddingEnd)
5761 {
5762 VmaSuballocation paddingSuballoc = {};
5763 paddingSuballoc.offset = request.offset + allocSize;
5764 paddingSuballoc.size = paddingEnd;
5765 paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5766 VmaSuballocationList::iterator next = request.item;
5767 ++next;
5768 const VmaSuballocationList::iterator paddingEndItem =
5769 m_Suballocations.insert(next, paddingSuballoc);
5770 RegisterFreeSuballocation(paddingEndItem);
5771 }
5772
5773 // If there are any free bytes remaining at the beginning, insert new free suballocation before current one.
5774 if(paddingBegin)
5775 {
5776 VmaSuballocation paddingSuballoc = {};
5777 paddingSuballoc.offset = request.offset - paddingBegin;
5778 paddingSuballoc.size = paddingBegin;
5779 paddingSuballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
5780 const VmaSuballocationList::iterator paddingBeginItem =
5781 m_Suballocations.insert(request.item, paddingSuballoc);
5782 RegisterFreeSuballocation(paddingBeginItem);
5783 }
5784
5785 // Update totals.
5786 m_FreeCount = m_FreeCount - 1;
5787 if(paddingBegin > 0)
5788 {
5789 ++m_FreeCount;
5790 }
5791 if(paddingEnd > 0)
5792 {
5793 ++m_FreeCount;
5794 }
5795 m_SumFreeSize -= allocSize;
5796 }
5797
Free(const VmaAllocation allocation)5798 void VmaBlockMetadata::Free(const VmaAllocation allocation)
5799 {
5800 for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5801 suballocItem != m_Suballocations.end();
5802 ++suballocItem)
5803 {
5804 VmaSuballocation& suballoc = *suballocItem;
5805 if(suballoc.hAllocation == allocation)
5806 {
5807 FreeSuballocation(suballocItem);
5808 VMA_HEAVY_ASSERT(Validate());
5809 return;
5810 }
5811 }
5812 VMA_ASSERT(0 && "Not found!");
5813 }
5814
FreeAtOffset(VkDeviceSize offset)5815 void VmaBlockMetadata::FreeAtOffset(VkDeviceSize offset)
5816 {
5817 for(VmaSuballocationList::iterator suballocItem = m_Suballocations.begin();
5818 suballocItem != m_Suballocations.end();
5819 ++suballocItem)
5820 {
5821 VmaSuballocation& suballoc = *suballocItem;
5822 if(suballoc.offset == offset)
5823 {
5824 FreeSuballocation(suballocItem);
5825 return;
5826 }
5827 }
5828 VMA_ASSERT(0 && "Not found!");
5829 }
5830
ValidateFreeSuballocationList()5831 bool VmaBlockMetadata::ValidateFreeSuballocationList() const
5832 {
5833 VkDeviceSize lastSize = 0;
5834 for(size_t i = 0, count = m_FreeSuballocationsBySize.size(); i < count; ++i)
5835 {
5836 const VmaSuballocationList::iterator it = m_FreeSuballocationsBySize[i];
5837
5838 if(it->type != VMA_SUBALLOCATION_TYPE_FREE)
5839 {
5840 VMA_ASSERT(0);
5841 return false;
5842 }
5843 if(it->size < VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
5844 {
5845 VMA_ASSERT(0);
5846 return false;
5847 }
5848 if(it->size < lastSize)
5849 {
5850 VMA_ASSERT(0);
5851 return false;
5852 }
5853
5854 lastSize = it->size;
5855 }
5856 return true;
5857 }
5858
CheckAllocation(uint32_t currentFrameIndex,uint32_t frameInUseCount,VkDeviceSize bufferImageGranularity,VkDeviceSize allocSize,VkDeviceSize allocAlignment,VmaSuballocationType allocType,VmaSuballocationList::const_iterator suballocItem,bool canMakeOtherLost,VkDeviceSize * pOffset,size_t * itemsToMakeLostCount,VkDeviceSize * pSumFreeSize,VkDeviceSize * pSumItemSize)5859 bool VmaBlockMetadata::CheckAllocation(
5860 uint32_t currentFrameIndex,
5861 uint32_t frameInUseCount,
5862 VkDeviceSize bufferImageGranularity,
5863 VkDeviceSize allocSize,
5864 VkDeviceSize allocAlignment,
5865 VmaSuballocationType allocType,
5866 VmaSuballocationList::const_iterator suballocItem,
5867 bool canMakeOtherLost,
5868 VkDeviceSize* pOffset,
5869 size_t* itemsToMakeLostCount,
5870 VkDeviceSize* pSumFreeSize,
5871 VkDeviceSize* pSumItemSize) const
5872 {
5873 VMA_ASSERT(allocSize > 0);
5874 VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE);
5875 VMA_ASSERT(suballocItem != m_Suballocations.cend());
5876 VMA_ASSERT(pOffset != VMA_NULL);
5877
5878 *itemsToMakeLostCount = 0;
5879 *pSumFreeSize = 0;
5880 *pSumItemSize = 0;
5881
5882 if(canMakeOtherLost)
5883 {
5884 if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5885 {
5886 *pSumFreeSize = suballocItem->size;
5887 }
5888 else
5889 {
5890 if(suballocItem->hAllocation->CanBecomeLost() &&
5891 suballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5892 {
5893 ++*itemsToMakeLostCount;
5894 *pSumItemSize = suballocItem->size;
5895 }
5896 else
5897 {
5898 return false;
5899 }
5900 }
5901
5902 // Remaining size is too small for this request: Early return.
5903 if(m_Size - suballocItem->offset < allocSize)
5904 {
5905 return false;
5906 }
5907
5908 // Start from offset equal to beginning of this suballocation.
5909 *pOffset = suballocItem->offset;
5910
5911 // Apply VMA_DEBUG_MARGIN at the beginning.
5912 if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
5913 {
5914 *pOffset += VMA_DEBUG_MARGIN;
5915 }
5916
5917 // Apply alignment.
5918 const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
5919 *pOffset = VmaAlignUp(*pOffset, alignment);
5920
5921 // Check previous suballocations for BufferImageGranularity conflicts.
5922 // Make bigger alignment if necessary.
5923 if(bufferImageGranularity > 1)
5924 {
5925 bool bufferImageGranularityConflict = false;
5926 VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
5927 while(prevSuballocItem != m_Suballocations.cbegin())
5928 {
5929 --prevSuballocItem;
5930 const VmaSuballocation& prevSuballoc = *prevSuballocItem;
5931 if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
5932 {
5933 if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
5934 {
5935 bufferImageGranularityConflict = true;
5936 break;
5937 }
5938 }
5939 else
5940 // Already on previous page.
5941 break;
5942 }
5943 if(bufferImageGranularityConflict)
5944 {
5945 *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
5946 }
5947 }
5948
5949 // Now that we have final *pOffset, check if we are past suballocItem.
5950 // If yes, return false - this function should be called for another suballocItem as starting point.
5951 if(*pOffset >= suballocItem->offset + suballocItem->size)
5952 {
5953 return false;
5954 }
5955
5956 // Calculate padding at the beginning based on current offset.
5957 const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset;
5958
5959 // Calculate required margin at the end if this is not last suballocation.
5960 VmaSuballocationList::const_iterator next = suballocItem;
5961 ++next;
5962 const VkDeviceSize requiredEndMargin =
5963 (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
5964
5965 const VkDeviceSize totalSize = paddingBegin + allocSize + requiredEndMargin;
5966 // Another early return check.
5967 if(suballocItem->offset + totalSize > m_Size)
5968 {
5969 return false;
5970 }
5971
5972 // Advance lastSuballocItem until desired size is reached.
5973 // Update itemsToMakeLostCount.
5974 VmaSuballocationList::const_iterator lastSuballocItem = suballocItem;
5975 if(totalSize > suballocItem->size)
5976 {
5977 VkDeviceSize remainingSize = totalSize - suballocItem->size;
5978 while(remainingSize > 0)
5979 {
5980 ++lastSuballocItem;
5981 if(lastSuballocItem == m_Suballocations.cend())
5982 {
5983 return false;
5984 }
5985 if(lastSuballocItem->type == VMA_SUBALLOCATION_TYPE_FREE)
5986 {
5987 *pSumFreeSize += lastSuballocItem->size;
5988 }
5989 else
5990 {
5991 VMA_ASSERT(lastSuballocItem->hAllocation != VK_NULL_HANDLE);
5992 if(lastSuballocItem->hAllocation->CanBecomeLost() &&
5993 lastSuballocItem->hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
5994 {
5995 ++*itemsToMakeLostCount;
5996 *pSumItemSize += lastSuballocItem->size;
5997 }
5998 else
5999 {
6000 return false;
6001 }
6002 }
6003 remainingSize = (lastSuballocItem->size < remainingSize) ?
6004 remainingSize - lastSuballocItem->size : 0;
6005 }
6006 }
6007
6008 // Check next suballocations for BufferImageGranularity conflicts.
6009 // If conflict exists, we must mark more allocations lost or fail.
6010 if(bufferImageGranularity > 1)
6011 {
6012 VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem;
6013 ++nextSuballocItem;
6014 while(nextSuballocItem != m_Suballocations.cend())
6015 {
6016 const VmaSuballocation& nextSuballoc = *nextSuballocItem;
6017 if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
6018 {
6019 if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
6020 {
6021 VMA_ASSERT(nextSuballoc.hAllocation != VK_NULL_HANDLE);
6022 if(nextSuballoc.hAllocation->CanBecomeLost() &&
6023 nextSuballoc.hAllocation->GetLastUseFrameIndex() + frameInUseCount < currentFrameIndex)
6024 {
6025 ++*itemsToMakeLostCount;
6026 }
6027 else
6028 {
6029 return false;
6030 }
6031 }
6032 }
6033 else
6034 {
6035 // Already on next page.
6036 break;
6037 }
6038 ++nextSuballocItem;
6039 }
6040 }
6041 }
6042 else
6043 {
6044 const VmaSuballocation& suballoc = *suballocItem;
6045 VMA_ASSERT(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE);
6046
6047 *pSumFreeSize = suballoc.size;
6048
6049 // Size of this suballocation is too small for this request: Early return.
6050 if(suballoc.size < allocSize)
6051 {
6052 return false;
6053 }
6054
6055 // Start from offset equal to beginning of this suballocation.
6056 *pOffset = suballoc.offset;
6057
6058 // Apply VMA_DEBUG_MARGIN at the beginning.
6059 if((VMA_DEBUG_MARGIN > 0) && suballocItem != m_Suballocations.cbegin())
6060 {
6061 *pOffset += VMA_DEBUG_MARGIN;
6062 }
6063
6064 // Apply alignment.
6065 const VkDeviceSize alignment = VMA_MAX(allocAlignment, static_cast<VkDeviceSize>(VMA_DEBUG_ALIGNMENT));
6066 *pOffset = VmaAlignUp(*pOffset, alignment);
6067
6068 // Check previous suballocations for BufferImageGranularity conflicts.
6069 // Make bigger alignment if necessary.
6070 if(bufferImageGranularity > 1)
6071 {
6072 bool bufferImageGranularityConflict = false;
6073 VmaSuballocationList::const_iterator prevSuballocItem = suballocItem;
6074 while(prevSuballocItem != m_Suballocations.cbegin())
6075 {
6076 --prevSuballocItem;
6077 const VmaSuballocation& prevSuballoc = *prevSuballocItem;
6078 if(VmaBlocksOnSamePage(prevSuballoc.offset, prevSuballoc.size, *pOffset, bufferImageGranularity))
6079 {
6080 if(VmaIsBufferImageGranularityConflict(prevSuballoc.type, allocType))
6081 {
6082 bufferImageGranularityConflict = true;
6083 break;
6084 }
6085 }
6086 else
6087 // Already on previous page.
6088 break;
6089 }
6090 if(bufferImageGranularityConflict)
6091 {
6092 *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity);
6093 }
6094 }
6095
6096 // Calculate padding at the beginning based on current offset.
6097 const VkDeviceSize paddingBegin = *pOffset - suballoc.offset;
6098
6099 // Calculate required margin at the end if this is not last suballocation.
6100 VmaSuballocationList::const_iterator next = suballocItem;
6101 ++next;
6102 const VkDeviceSize requiredEndMargin =
6103 (next != m_Suballocations.cend()) ? VMA_DEBUG_MARGIN : 0;
6104
6105 // Fail if requested size plus margin before and after is bigger than size of this suballocation.
6106 if(paddingBegin + allocSize + requiredEndMargin > suballoc.size)
6107 {
6108 return false;
6109 }
6110
6111 // Check next suballocations for BufferImageGranularity conflicts.
6112 // If conflict exists, allocation cannot be made here.
6113 if(bufferImageGranularity > 1)
6114 {
6115 VmaSuballocationList::const_iterator nextSuballocItem = suballocItem;
6116 ++nextSuballocItem;
6117 while(nextSuballocItem != m_Suballocations.cend())
6118 {
6119 const VmaSuballocation& nextSuballoc = *nextSuballocItem;
6120 if(VmaBlocksOnSamePage(*pOffset, allocSize, nextSuballoc.offset, bufferImageGranularity))
6121 {
6122 if(VmaIsBufferImageGranularityConflict(allocType, nextSuballoc.type))
6123 {
6124 return false;
6125 }
6126 }
6127 else
6128 {
6129 // Already on next page.
6130 break;
6131 }
6132 ++nextSuballocItem;
6133 }
6134 }
6135 }
6136
6137 // All tests passed: Success. pOffset is already filled.
6138 return true;
6139 }
6140
MergeFreeWithNext(VmaSuballocationList::iterator item)6141 void VmaBlockMetadata::MergeFreeWithNext(VmaSuballocationList::iterator item)
6142 {
6143 VMA_ASSERT(item != m_Suballocations.end());
6144 VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6145
6146 VmaSuballocationList::iterator nextItem = item;
6147 ++nextItem;
6148 VMA_ASSERT(nextItem != m_Suballocations.end());
6149 VMA_ASSERT(nextItem->type == VMA_SUBALLOCATION_TYPE_FREE);
6150
6151 item->size += nextItem->size;
6152 --m_FreeCount;
6153 m_Suballocations.erase(nextItem);
6154 }
6155
FreeSuballocation(VmaSuballocationList::iterator suballocItem)6156 VmaSuballocationList::iterator VmaBlockMetadata::FreeSuballocation(VmaSuballocationList::iterator suballocItem)
6157 {
6158 // Change this suballocation to be marked as free.
6159 VmaSuballocation& suballoc = *suballocItem;
6160 suballoc.type = VMA_SUBALLOCATION_TYPE_FREE;
6161 suballoc.hAllocation = VK_NULL_HANDLE;
6162
6163 // Update totals.
6164 ++m_FreeCount;
6165 m_SumFreeSize += suballoc.size;
6166
6167 // Merge with previous and/or next suballocation if it's also free.
6168 bool mergeWithNext = false;
6169 bool mergeWithPrev = false;
6170
6171 VmaSuballocationList::iterator nextItem = suballocItem;
6172 ++nextItem;
6173 if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE))
6174 {
6175 mergeWithNext = true;
6176 }
6177
6178 VmaSuballocationList::iterator prevItem = suballocItem;
6179 if(suballocItem != m_Suballocations.begin())
6180 {
6181 --prevItem;
6182 if(prevItem->type == VMA_SUBALLOCATION_TYPE_FREE)
6183 {
6184 mergeWithPrev = true;
6185 }
6186 }
6187
6188 if(mergeWithNext)
6189 {
6190 UnregisterFreeSuballocation(nextItem);
6191 MergeFreeWithNext(suballocItem);
6192 }
6193
6194 if(mergeWithPrev)
6195 {
6196 UnregisterFreeSuballocation(prevItem);
6197 MergeFreeWithNext(prevItem);
6198 RegisterFreeSuballocation(prevItem);
6199 return prevItem;
6200 }
6201 else
6202 {
6203 RegisterFreeSuballocation(suballocItem);
6204 return suballocItem;
6205 }
6206 }
6207
RegisterFreeSuballocation(VmaSuballocationList::iterator item)6208 void VmaBlockMetadata::RegisterFreeSuballocation(VmaSuballocationList::iterator item)
6209 {
6210 VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6211 VMA_ASSERT(item->size > 0);
6212
6213 // You may want to enable this validation at the beginning or at the end of
6214 // this function, depending on what do you want to check.
6215 VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6216
6217 if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6218 {
6219 if(m_FreeSuballocationsBySize.empty())
6220 {
6221 m_FreeSuballocationsBySize.push_back(item);
6222 }
6223 else
6224 {
6225 VmaVectorInsertSorted<VmaSuballocationItemSizeLess>(m_FreeSuballocationsBySize, item);
6226 }
6227 }
6228
6229 //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6230 }
6231
6232
UnregisterFreeSuballocation(VmaSuballocationList::iterator item)6233 void VmaBlockMetadata::UnregisterFreeSuballocation(VmaSuballocationList::iterator item)
6234 {
6235 VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE);
6236 VMA_ASSERT(item->size > 0);
6237
6238 // You may want to enable this validation at the beginning or at the end of
6239 // this function, depending on what do you want to check.
6240 VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6241
6242 if(item->size >= VMA_MIN_FREE_SUBALLOCATION_SIZE_TO_REGISTER)
6243 {
6244 VmaSuballocationList::iterator* const it = VmaBinaryFindFirstNotLess(
6245 m_FreeSuballocationsBySize.data(),
6246 m_FreeSuballocationsBySize.data() + m_FreeSuballocationsBySize.size(),
6247 item,
6248 VmaSuballocationItemSizeLess());
6249 for(size_t index = it - m_FreeSuballocationsBySize.data();
6250 index < m_FreeSuballocationsBySize.size();
6251 ++index)
6252 {
6253 if(m_FreeSuballocationsBySize[index] == item)
6254 {
6255 VmaVectorRemove(m_FreeSuballocationsBySize, index);
6256 return;
6257 }
6258 VMA_ASSERT((m_FreeSuballocationsBySize[index]->size == item->size) && "Not found.");
6259 }
6260 VMA_ASSERT(0 && "Not found.");
6261 }
6262
6263 //VMA_HEAVY_ASSERT(ValidateFreeSuballocationList());
6264 }
6265
6266 ////////////////////////////////////////////////////////////////////////////////
6267 // class VmaDeviceMemoryBlock
6268
VmaDeviceMemoryBlock(VmaAllocator hAllocator)6269 VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) :
6270 m_Metadata(hAllocator),
6271 m_MemoryTypeIndex(UINT32_MAX),
6272 m_hMemory(VK_NULL_HANDLE),
6273 m_MapCount(0),
6274 m_pMappedData(VMA_NULL)
6275 {
6276 }
6277
Init(uint32_t newMemoryTypeIndex,VkDeviceMemory newMemory,VkDeviceSize newSize)6278 void VmaDeviceMemoryBlock::Init(
6279 uint32_t newMemoryTypeIndex,
6280 VkDeviceMemory newMemory,
6281 VkDeviceSize newSize)
6282 {
6283 VMA_ASSERT(m_hMemory == VK_NULL_HANDLE);
6284
6285 m_MemoryTypeIndex = newMemoryTypeIndex;
6286 m_hMemory = newMemory;
6287
6288 m_Metadata.Init(newSize);
6289 }
6290
Destroy(VmaAllocator allocator)6291 void VmaDeviceMemoryBlock::Destroy(VmaAllocator allocator)
6292 {
6293 // This is the most important assert in the entire library.
6294 // Hitting it means you have some memory leak - unreleased VmaAllocation objects.
6295 VMA_ASSERT(m_Metadata.IsEmpty() && "Some allocations were not freed before destruction of this memory block!");
6296
6297 VMA_ASSERT(m_hMemory != VK_NULL_HANDLE);
6298 allocator->FreeVulkanMemory(m_MemoryTypeIndex, m_Metadata.GetSize(), m_hMemory);
6299 m_hMemory = VK_NULL_HANDLE;
6300 }
6301
Validate()6302 bool VmaDeviceMemoryBlock::Validate() const
6303 {
6304 if((m_hMemory == VK_NULL_HANDLE) ||
6305 (m_Metadata.GetSize() == 0))
6306 {
6307 return false;
6308 }
6309
6310 return m_Metadata.Validate();
6311 }
6312
Map(VmaAllocator hAllocator,uint32_t count,void ** ppData)6313 VkResult VmaDeviceMemoryBlock::Map(VmaAllocator hAllocator, uint32_t count, void** ppData)
6314 {
6315 if(count == 0)
6316 {
6317 return VK_SUCCESS;
6318 }
6319
6320 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6321 if(m_MapCount != 0)
6322 {
6323 m_MapCount += count;
6324 VMA_ASSERT(m_pMappedData != VMA_NULL);
6325 if(ppData != VMA_NULL)
6326 {
6327 *ppData = m_pMappedData;
6328 }
6329 return VK_SUCCESS;
6330 }
6331 else
6332 {
6333 VkResult result = (*hAllocator->GetVulkanFunctions().vkMapMemory)(
6334 hAllocator->m_hDevice,
6335 m_hMemory,
6336 0, // offset
6337 VK_WHOLE_SIZE,
6338 0, // flags
6339 &m_pMappedData);
6340 if(result == VK_SUCCESS)
6341 {
6342 if(ppData != VMA_NULL)
6343 {
6344 *ppData = m_pMappedData;
6345 }
6346 m_MapCount = count;
6347 }
6348 return result;
6349 }
6350 }
6351
Unmap(VmaAllocator hAllocator,uint32_t count)6352 void VmaDeviceMemoryBlock::Unmap(VmaAllocator hAllocator, uint32_t count)
6353 {
6354 if(count == 0)
6355 {
6356 return;
6357 }
6358
6359 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6360 if(m_MapCount >= count)
6361 {
6362 m_MapCount -= count;
6363 if(m_MapCount == 0)
6364 {
6365 m_pMappedData = VMA_NULL;
6366 (*hAllocator->GetVulkanFunctions().vkUnmapMemory)(hAllocator->m_hDevice, m_hMemory);
6367 }
6368 }
6369 else
6370 {
6371 VMA_ASSERT(0 && "VkDeviceMemory block is being unmapped while it was not previously mapped.");
6372 }
6373 }
6374
BindBufferMemory(const VmaAllocator hAllocator,const VmaAllocation hAllocation,VkBuffer hBuffer)6375 VkResult VmaDeviceMemoryBlock::BindBufferMemory(
6376 const VmaAllocator hAllocator,
6377 const VmaAllocation hAllocation,
6378 VkBuffer hBuffer)
6379 {
6380 VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
6381 hAllocation->GetBlock() == this);
6382 // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
6383 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6384 return hAllocator->GetVulkanFunctions().vkBindBufferMemory(
6385 hAllocator->m_hDevice,
6386 hBuffer,
6387 m_hMemory,
6388 hAllocation->GetOffset());
6389 }
6390
BindImageMemory(const VmaAllocator hAllocator,const VmaAllocation hAllocation,VkImage hImage)6391 VkResult VmaDeviceMemoryBlock::BindImageMemory(
6392 const VmaAllocator hAllocator,
6393 const VmaAllocation hAllocation,
6394 VkImage hImage)
6395 {
6396 VMA_ASSERT(hAllocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK &&
6397 hAllocation->GetBlock() == this);
6398 // This lock is important so that we don't call vkBind... and/or vkMap... simultaneously on the same VkDeviceMemory from multiple threads.
6399 VmaMutexLock lock(m_Mutex, hAllocator->m_UseMutex);
6400 return hAllocator->GetVulkanFunctions().vkBindImageMemory(
6401 hAllocator->m_hDevice,
6402 hImage,
6403 m_hMemory,
6404 hAllocation->GetOffset());
6405 }
6406
InitStatInfo(VmaStatInfo & outInfo)6407 static void InitStatInfo(VmaStatInfo& outInfo)
6408 {
6409 memset(&outInfo, 0, sizeof(outInfo));
6410 outInfo.allocationSizeMin = UINT64_MAX;
6411 outInfo.unusedRangeSizeMin = UINT64_MAX;
6412 }
6413
6414 // Adds statistics srcInfo into inoutInfo, like: inoutInfo += srcInfo.
VmaAddStatInfo(VmaStatInfo & inoutInfo,const VmaStatInfo & srcInfo)6415 static void VmaAddStatInfo(VmaStatInfo& inoutInfo, const VmaStatInfo& srcInfo)
6416 {
6417 inoutInfo.blockCount += srcInfo.blockCount;
6418 inoutInfo.allocationCount += srcInfo.allocationCount;
6419 inoutInfo.unusedRangeCount += srcInfo.unusedRangeCount;
6420 inoutInfo.usedBytes += srcInfo.usedBytes;
6421 inoutInfo.unusedBytes += srcInfo.unusedBytes;
6422 inoutInfo.allocationSizeMin = VMA_MIN(inoutInfo.allocationSizeMin, srcInfo.allocationSizeMin);
6423 inoutInfo.allocationSizeMax = VMA_MAX(inoutInfo.allocationSizeMax, srcInfo.allocationSizeMax);
6424 inoutInfo.unusedRangeSizeMin = VMA_MIN(inoutInfo.unusedRangeSizeMin, srcInfo.unusedRangeSizeMin);
6425 inoutInfo.unusedRangeSizeMax = VMA_MAX(inoutInfo.unusedRangeSizeMax, srcInfo.unusedRangeSizeMax);
6426 }
6427
VmaPostprocessCalcStatInfo(VmaStatInfo & inoutInfo)6428 static void VmaPostprocessCalcStatInfo(VmaStatInfo& inoutInfo)
6429 {
6430 inoutInfo.allocationSizeAvg = (inoutInfo.allocationCount > 0) ?
6431 VmaRoundDiv<VkDeviceSize>(inoutInfo.usedBytes, inoutInfo.allocationCount) : 0;
6432 inoutInfo.unusedRangeSizeAvg = (inoutInfo.unusedRangeCount > 0) ?
6433 VmaRoundDiv<VkDeviceSize>(inoutInfo.unusedBytes, inoutInfo.unusedRangeCount) : 0;
6434 }
6435
VmaPool_T(VmaAllocator hAllocator,const VmaPoolCreateInfo & createInfo)6436 VmaPool_T::VmaPool_T(
6437 VmaAllocator hAllocator,
6438 const VmaPoolCreateInfo& createInfo) :
6439 m_BlockVector(
6440 hAllocator,
6441 createInfo.memoryTypeIndex,
6442 createInfo.blockSize,
6443 createInfo.minBlockCount,
6444 createInfo.maxBlockCount,
6445 (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(),
6446 createInfo.frameInUseCount,
6447 true) // isCustomPool
6448 {
6449 }
6450
~VmaPool_T()6451 VmaPool_T::~VmaPool_T()
6452 {
6453 }
6454
6455 #if VMA_STATS_STRING_ENABLED
6456
6457 #endif // #if VMA_STATS_STRING_ENABLED
6458
VmaBlockVector(VmaAllocator hAllocator,uint32_t memoryTypeIndex,VkDeviceSize preferredBlockSize,size_t minBlockCount,size_t maxBlockCount,VkDeviceSize bufferImageGranularity,uint32_t frameInUseCount,bool isCustomPool)6459 VmaBlockVector::VmaBlockVector(
6460 VmaAllocator hAllocator,
6461 uint32_t memoryTypeIndex,
6462 VkDeviceSize preferredBlockSize,
6463 size_t minBlockCount,
6464 size_t maxBlockCount,
6465 VkDeviceSize bufferImageGranularity,
6466 uint32_t frameInUseCount,
6467 bool isCustomPool) :
6468 m_hAllocator(hAllocator),
6469 m_MemoryTypeIndex(memoryTypeIndex),
6470 m_PreferredBlockSize(preferredBlockSize),
6471 m_MinBlockCount(minBlockCount),
6472 m_MaxBlockCount(maxBlockCount),
6473 m_BufferImageGranularity(bufferImageGranularity),
6474 m_FrameInUseCount(frameInUseCount),
6475 m_IsCustomPool(isCustomPool),
6476 m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())),
6477 m_HasEmptyBlock(false),
6478 m_pDefragmentator(VMA_NULL)
6479 {
6480 }
6481
~VmaBlockVector()6482 VmaBlockVector::~VmaBlockVector()
6483 {
6484 VMA_ASSERT(m_pDefragmentator == VMA_NULL);
6485
6486 for(size_t i = m_Blocks.size(); i--; )
6487 {
6488 m_Blocks[i]->Destroy(m_hAllocator);
6489 vma_delete(m_hAllocator, m_Blocks[i]);
6490 }
6491 }
6492
CreateMinBlocks()6493 VkResult VmaBlockVector::CreateMinBlocks()
6494 {
6495 for(size_t i = 0; i < m_MinBlockCount; ++i)
6496 {
6497 VkResult res = CreateBlock(m_PreferredBlockSize, VMA_NULL);
6498 if(res != VK_SUCCESS)
6499 {
6500 return res;
6501 }
6502 }
6503 return VK_SUCCESS;
6504 }
6505
GetPoolStats(VmaPoolStats * pStats)6506 void VmaBlockVector::GetPoolStats(VmaPoolStats* pStats)
6507 {
6508 pStats->size = 0;
6509 pStats->unusedSize = 0;
6510 pStats->allocationCount = 0;
6511 pStats->unusedRangeCount = 0;
6512 pStats->unusedRangeSizeMax = 0;
6513
6514 VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6515
6516 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6517 {
6518 const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
6519 VMA_ASSERT(pBlock);
6520 VMA_HEAVY_ASSERT(pBlock->Validate());
6521 pBlock->m_Metadata.AddPoolStats(*pStats);
6522 }
6523 }
6524
6525 static const uint32_t VMA_ALLOCATION_TRY_COUNT = 32;
6526
Allocate(VmaPool hCurrentPool,uint32_t currentFrameIndex,const VkMemoryRequirements & vkMemReq,const VmaAllocationCreateInfo & createInfo,VmaSuballocationType suballocType,VmaAllocation * pAllocation)6527 VkResult VmaBlockVector::Allocate(
6528 VmaPool hCurrentPool,
6529 uint32_t currentFrameIndex,
6530 const VkMemoryRequirements& vkMemReq,
6531 const VmaAllocationCreateInfo& createInfo,
6532 VmaSuballocationType suballocType,
6533 VmaAllocation* pAllocation)
6534 {
6535 const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0;
6536 const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0;
6537
6538 VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6539
6540 // 1. Search existing allocations. Try to allocate without making other allocations lost.
6541 // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6542 for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6543 {
6544 VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6545 VMA_ASSERT(pCurrBlock);
6546 VmaAllocationRequest currRequest = {};
6547 if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6548 currentFrameIndex,
6549 m_FrameInUseCount,
6550 m_BufferImageGranularity,
6551 vkMemReq.size,
6552 vkMemReq.alignment,
6553 suballocType,
6554 false, // canMakeOtherLost
6555 &currRequest))
6556 {
6557 // Allocate from pCurrBlock.
6558 VMA_ASSERT(currRequest.itemsToMakeLostCount == 0);
6559
6560 if(mapped)
6561 {
6562 VkResult res = pCurrBlock->Map(m_hAllocator, 1, VMA_NULL);
6563 if(res != VK_SUCCESS)
6564 {
6565 return res;
6566 }
6567 }
6568
6569 // We no longer have an empty Allocation.
6570 if(pCurrBlock->m_Metadata.IsEmpty())
6571 {
6572 m_HasEmptyBlock = false;
6573 }
6574
6575 *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6576 pCurrBlock->m_Metadata.Alloc(currRequest, suballocType, vkMemReq.size, *pAllocation);
6577 (*pAllocation)->InitBlockAllocation(
6578 hCurrentPool,
6579 pCurrBlock,
6580 currRequest.offset,
6581 vkMemReq.alignment,
6582 vkMemReq.size,
6583 suballocType,
6584 mapped,
6585 (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6586 VMA_HEAVY_ASSERT(pCurrBlock->Validate());
6587 VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6588 (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6589 return VK_SUCCESS;
6590 }
6591 }
6592
6593 const bool canCreateNewBlock =
6594 ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) &&
6595 (m_Blocks.size() < m_MaxBlockCount);
6596
6597 // 2. Try to create new block.
6598 if(canCreateNewBlock)
6599 {
6600 // Calculate optimal size for new block.
6601 VkDeviceSize newBlockSize = m_PreferredBlockSize;
6602 uint32_t newBlockSizeShift = 0;
6603 const uint32_t NEW_BLOCK_SIZE_SHIFT_MAX = 3;
6604
6605 // Allocating blocks of other sizes is allowed only in default pools.
6606 // In custom pools block size is fixed.
6607 if(m_IsCustomPool == false)
6608 {
6609 // Allocate 1/8, 1/4, 1/2 as first blocks.
6610 const VkDeviceSize maxExistingBlockSize = CalcMaxBlockSize();
6611 for(uint32_t i = 0; i < NEW_BLOCK_SIZE_SHIFT_MAX; ++i)
6612 {
6613 const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6614 if(smallerNewBlockSize > maxExistingBlockSize && smallerNewBlockSize >= vkMemReq.size * 2)
6615 {
6616 newBlockSize = smallerNewBlockSize;
6617 ++newBlockSizeShift;
6618 }
6619 else
6620 {
6621 break;
6622 }
6623 }
6624 }
6625
6626 size_t newBlockIndex = 0;
6627 VkResult res = CreateBlock(newBlockSize, &newBlockIndex);
6628 // Allocation of this size failed? Try 1/2, 1/4, 1/8 of m_PreferredBlockSize.
6629 if(m_IsCustomPool == false)
6630 {
6631 while(res < 0 && newBlockSizeShift < NEW_BLOCK_SIZE_SHIFT_MAX)
6632 {
6633 const VkDeviceSize smallerNewBlockSize = newBlockSize / 2;
6634 if(smallerNewBlockSize >= vkMemReq.size)
6635 {
6636 newBlockSize = smallerNewBlockSize;
6637 ++newBlockSizeShift;
6638 res = CreateBlock(newBlockSize, &newBlockIndex);
6639 }
6640 else
6641 {
6642 break;
6643 }
6644 }
6645 }
6646
6647 if(res == VK_SUCCESS)
6648 {
6649 VmaDeviceMemoryBlock* const pBlock = m_Blocks[newBlockIndex];
6650 VMA_ASSERT(pBlock->m_Metadata.GetSize() >= vkMemReq.size);
6651
6652 if(mapped)
6653 {
6654 res = pBlock->Map(m_hAllocator, 1, VMA_NULL);
6655 if(res != VK_SUCCESS)
6656 {
6657 return res;
6658 }
6659 }
6660
6661 // Allocate from pBlock. Because it is empty, dstAllocRequest can be trivially filled.
6662 VmaAllocationRequest allocRequest;
6663 pBlock->m_Metadata.CreateFirstAllocationRequest(&allocRequest);
6664 *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6665 pBlock->m_Metadata.Alloc(allocRequest, suballocType, vkMemReq.size, *pAllocation);
6666 (*pAllocation)->InitBlockAllocation(
6667 hCurrentPool,
6668 pBlock,
6669 allocRequest.offset,
6670 vkMemReq.alignment,
6671 vkMemReq.size,
6672 suballocType,
6673 mapped,
6674 (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6675 VMA_HEAVY_ASSERT(pBlock->Validate());
6676 VMA_DEBUG_LOG(" Created new allocation Size=%llu", allocInfo.allocationSize);
6677 (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6678 return VK_SUCCESS;
6679 }
6680 }
6681
6682 const bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0;
6683
6684 // 3. Try to allocate from existing blocks with making other allocations lost.
6685 if(canMakeOtherLost)
6686 {
6687 uint32_t tryIndex = 0;
6688 for(; tryIndex < VMA_ALLOCATION_TRY_COUNT; ++tryIndex)
6689 {
6690 VmaDeviceMemoryBlock* pBestRequestBlock = VMA_NULL;
6691 VmaAllocationRequest bestRequest = {};
6692 VkDeviceSize bestRequestCost = VK_WHOLE_SIZE;
6693
6694 // 1. Search existing allocations.
6695 // Forward order in m_Blocks - prefer blocks with smallest amount of free space.
6696 for(size_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex )
6697 {
6698 VmaDeviceMemoryBlock* const pCurrBlock = m_Blocks[blockIndex];
6699 VMA_ASSERT(pCurrBlock);
6700 VmaAllocationRequest currRequest = {};
6701 if(pCurrBlock->m_Metadata.CreateAllocationRequest(
6702 currentFrameIndex,
6703 m_FrameInUseCount,
6704 m_BufferImageGranularity,
6705 vkMemReq.size,
6706 vkMemReq.alignment,
6707 suballocType,
6708 canMakeOtherLost,
6709 &currRequest))
6710 {
6711 const VkDeviceSize currRequestCost = currRequest.CalcCost();
6712 if(pBestRequestBlock == VMA_NULL ||
6713 currRequestCost < bestRequestCost)
6714 {
6715 pBestRequestBlock = pCurrBlock;
6716 bestRequest = currRequest;
6717 bestRequestCost = currRequestCost;
6718
6719 if(bestRequestCost == 0)
6720 {
6721 break;
6722 }
6723 }
6724 }
6725 }
6726
6727 if(pBestRequestBlock != VMA_NULL)
6728 {
6729 if(mapped)
6730 {
6731 VkResult res = pBestRequestBlock->Map(m_hAllocator, 1, VMA_NULL);
6732 if(res != VK_SUCCESS)
6733 {
6734 return res;
6735 }
6736 }
6737
6738 if(pBestRequestBlock->m_Metadata.MakeRequestedAllocationsLost(
6739 currentFrameIndex,
6740 m_FrameInUseCount,
6741 &bestRequest))
6742 {
6743 // We no longer have an empty Allocation.
6744 if(pBestRequestBlock->m_Metadata.IsEmpty())
6745 {
6746 m_HasEmptyBlock = false;
6747 }
6748 // Allocate from this pBlock.
6749 *pAllocation = vma_new(m_hAllocator, VmaAllocation_T)(currentFrameIndex, isUserDataString);
6750 pBestRequestBlock->m_Metadata.Alloc(bestRequest, suballocType, vkMemReq.size, *pAllocation);
6751 (*pAllocation)->InitBlockAllocation(
6752 hCurrentPool,
6753 pBestRequestBlock,
6754 bestRequest.offset,
6755 vkMemReq.alignment,
6756 vkMemReq.size,
6757 suballocType,
6758 mapped,
6759 (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0);
6760 VMA_HEAVY_ASSERT(pBestRequestBlock->Validate());
6761 VMA_DEBUG_LOG(" Returned from existing allocation #%u", (uint32_t)blockIndex);
6762 (*pAllocation)->SetUserData(m_hAllocator, createInfo.pUserData);
6763 return VK_SUCCESS;
6764 }
6765 // else: Some allocations must have been touched while we are here. Next try.
6766 }
6767 else
6768 {
6769 // Could not find place in any of the blocks - break outer loop.
6770 break;
6771 }
6772 }
6773 /* Maximum number of tries exceeded - a very unlike event when many other
6774 threads are simultaneously touching allocations making it impossible to make
6775 lost at the same time as we try to allocate. */
6776 if(tryIndex == VMA_ALLOCATION_TRY_COUNT)
6777 {
6778 return VK_ERROR_TOO_MANY_OBJECTS;
6779 }
6780 }
6781
6782 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
6783 }
6784
Free(VmaAllocation hAllocation)6785 void VmaBlockVector::Free(
6786 VmaAllocation hAllocation)
6787 {
6788 VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL;
6789
6790 // Scope for lock.
6791 {
6792 VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6793
6794 VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
6795
6796 if(hAllocation->IsPersistentMap())
6797 {
6798 pBlock->Unmap(m_hAllocator, 1);
6799 }
6800
6801 pBlock->m_Metadata.Free(hAllocation);
6802 VMA_HEAVY_ASSERT(pBlock->Validate());
6803
6804 VMA_DEBUG_LOG(" Freed from MemoryTypeIndex=%u", memTypeIndex);
6805
6806 // pBlock became empty after this deallocation.
6807 if(pBlock->m_Metadata.IsEmpty())
6808 {
6809 // Already has empty Allocation. We don't want to have two, so delete this one.
6810 if(m_HasEmptyBlock && m_Blocks.size() > m_MinBlockCount)
6811 {
6812 pBlockToDelete = pBlock;
6813 Remove(pBlock);
6814 }
6815 // We now have first empty Allocation.
6816 else
6817 {
6818 m_HasEmptyBlock = true;
6819 }
6820 }
6821 // pBlock didn't become empty, but we have another empty block - find and free that one.
6822 // (This is optional, heuristics.)
6823 else if(m_HasEmptyBlock)
6824 {
6825 VmaDeviceMemoryBlock* pLastBlock = m_Blocks.back();
6826 if(pLastBlock->m_Metadata.IsEmpty() && m_Blocks.size() > m_MinBlockCount)
6827 {
6828 pBlockToDelete = pLastBlock;
6829 m_Blocks.pop_back();
6830 m_HasEmptyBlock = false;
6831 }
6832 }
6833
6834 IncrementallySortBlocks();
6835 }
6836
6837 // Destruction of a free Allocation. Deferred until this point, outside of mutex
6838 // lock, for performance reason.
6839 if(pBlockToDelete != VMA_NULL)
6840 {
6841 VMA_DEBUG_LOG(" Deleted empty allocation");
6842 pBlockToDelete->Destroy(m_hAllocator);
6843 vma_delete(m_hAllocator, pBlockToDelete);
6844 }
6845 }
6846
CalcMaxBlockSize()6847 size_t VmaBlockVector::CalcMaxBlockSize() const
6848 {
6849 size_t result = 0;
6850 for(size_t i = m_Blocks.size(); i--; )
6851 {
6852 result = VMA_MAX((uint64_t)result, (uint64_t)m_Blocks[i]->m_Metadata.GetSize());
6853 if(result >= m_PreferredBlockSize)
6854 {
6855 break;
6856 }
6857 }
6858 return result;
6859 }
6860
Remove(VmaDeviceMemoryBlock * pBlock)6861 void VmaBlockVector::Remove(VmaDeviceMemoryBlock* pBlock)
6862 {
6863 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
6864 {
6865 if(m_Blocks[blockIndex] == pBlock)
6866 {
6867 VmaVectorRemove(m_Blocks, blockIndex);
6868 return;
6869 }
6870 }
6871 VMA_ASSERT(0);
6872 }
6873
IncrementallySortBlocks()6874 void VmaBlockVector::IncrementallySortBlocks()
6875 {
6876 // Bubble sort only until first swap.
6877 for(size_t i = 1; i < m_Blocks.size(); ++i)
6878 {
6879 if(m_Blocks[i - 1]->m_Metadata.GetSumFreeSize() > m_Blocks[i]->m_Metadata.GetSumFreeSize())
6880 {
6881 VMA_SWAP(m_Blocks[i - 1], m_Blocks[i]);
6882 return;
6883 }
6884 }
6885 }
6886
CreateBlock(VkDeviceSize blockSize,size_t * pNewBlockIndex)6887 VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex)
6888 {
6889 VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
6890 allocInfo.memoryTypeIndex = m_MemoryTypeIndex;
6891 allocInfo.allocationSize = blockSize;
6892 VkDeviceMemory mem = VK_NULL_HANDLE;
6893 VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem);
6894 if(res < 0)
6895 {
6896 return res;
6897 }
6898
6899 // New VkDeviceMemory successfully created.
6900
6901 // Create new Allocation for it.
6902 VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator);
6903 pBlock->Init(
6904 m_MemoryTypeIndex,
6905 mem,
6906 allocInfo.allocationSize);
6907
6908 m_Blocks.push_back(pBlock);
6909 if(pNewBlockIndex != VMA_NULL)
6910 {
6911 *pNewBlockIndex = m_Blocks.size() - 1;
6912 }
6913
6914 return VK_SUCCESS;
6915 }
6916
6917 #if VMA_STATS_STRING_ENABLED
6918
PrintDetailedMap(class VmaJsonWriter & json)6919 void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json)
6920 {
6921 VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
6922
6923 json.BeginObject();
6924
6925 if(m_IsCustomPool)
6926 {
6927 json.WriteString("MemoryTypeIndex");
6928 json.WriteNumber(m_MemoryTypeIndex);
6929
6930 json.WriteString("BlockSize");
6931 json.WriteNumber(m_PreferredBlockSize);
6932
6933 json.WriteString("BlockCount");
6934 json.BeginObject(true);
6935 if(m_MinBlockCount > 0)
6936 {
6937 json.WriteString("Min");
6938 json.WriteNumber((uint64_t)m_MinBlockCount);
6939 }
6940 if(m_MaxBlockCount < SIZE_MAX)
6941 {
6942 json.WriteString("Max");
6943 json.WriteNumber((uint64_t)m_MaxBlockCount);
6944 }
6945 json.WriteString("Cur");
6946 json.WriteNumber((uint64_t)m_Blocks.size());
6947 json.EndObject();
6948
6949 if(m_FrameInUseCount > 0)
6950 {
6951 json.WriteString("FrameInUseCount");
6952 json.WriteNumber(m_FrameInUseCount);
6953 }
6954 }
6955 else
6956 {
6957 json.WriteString("PreferredBlockSize");
6958 json.WriteNumber(m_PreferredBlockSize);
6959 }
6960
6961 json.WriteString("Blocks");
6962 json.BeginArray();
6963 for(size_t i = 0; i < m_Blocks.size(); ++i)
6964 {
6965 m_Blocks[i]->m_Metadata.PrintDetailedMap(json);
6966 }
6967 json.EndArray();
6968
6969 json.EndObject();
6970 }
6971
6972 #endif // #if VMA_STATS_STRING_ENABLED
6973
EnsureDefragmentator(VmaAllocator hAllocator,uint32_t currentFrameIndex)6974 VmaDefragmentator* VmaBlockVector::EnsureDefragmentator(
6975 VmaAllocator hAllocator,
6976 uint32_t currentFrameIndex)
6977 {
6978 if(m_pDefragmentator == VMA_NULL)
6979 {
6980 m_pDefragmentator = vma_new(m_hAllocator, VmaDefragmentator)(
6981 hAllocator,
6982 this,
6983 currentFrameIndex);
6984 }
6985
6986 return m_pDefragmentator;
6987 }
6988
Defragment(VmaDefragmentationStats * pDefragmentationStats,VkDeviceSize & maxBytesToMove,uint32_t & maxAllocationsToMove)6989 VkResult VmaBlockVector::Defragment(
6990 VmaDefragmentationStats* pDefragmentationStats,
6991 VkDeviceSize& maxBytesToMove,
6992 uint32_t& maxAllocationsToMove)
6993 {
6994 if(m_pDefragmentator == VMA_NULL)
6995 {
6996 return VK_SUCCESS;
6997 }
6998
6999 VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7000
7001 // Defragment.
7002 VkResult result = m_pDefragmentator->Defragment(maxBytesToMove, maxAllocationsToMove);
7003
7004 // Accumulate statistics.
7005 if(pDefragmentationStats != VMA_NULL)
7006 {
7007 const VkDeviceSize bytesMoved = m_pDefragmentator->GetBytesMoved();
7008 const uint32_t allocationsMoved = m_pDefragmentator->GetAllocationsMoved();
7009 pDefragmentationStats->bytesMoved += bytesMoved;
7010 pDefragmentationStats->allocationsMoved += allocationsMoved;
7011 VMA_ASSERT(bytesMoved <= maxBytesToMove);
7012 VMA_ASSERT(allocationsMoved <= maxAllocationsToMove);
7013 maxBytesToMove -= bytesMoved;
7014 maxAllocationsToMove -= allocationsMoved;
7015 }
7016
7017 // Free empty blocks.
7018 m_HasEmptyBlock = false;
7019 for(size_t blockIndex = m_Blocks.size(); blockIndex--; )
7020 {
7021 VmaDeviceMemoryBlock* pBlock = m_Blocks[blockIndex];
7022 if(pBlock->m_Metadata.IsEmpty())
7023 {
7024 if(m_Blocks.size() > m_MinBlockCount)
7025 {
7026 if(pDefragmentationStats != VMA_NULL)
7027 {
7028 ++pDefragmentationStats->deviceMemoryBlocksFreed;
7029 pDefragmentationStats->bytesFreed += pBlock->m_Metadata.GetSize();
7030 }
7031
7032 VmaVectorRemove(m_Blocks, blockIndex);
7033 pBlock->Destroy(m_hAllocator);
7034 vma_delete(m_hAllocator, pBlock);
7035 }
7036 else
7037 {
7038 m_HasEmptyBlock = true;
7039 }
7040 }
7041 }
7042
7043 return result;
7044 }
7045
DestroyDefragmentator()7046 void VmaBlockVector::DestroyDefragmentator()
7047 {
7048 if(m_pDefragmentator != VMA_NULL)
7049 {
7050 vma_delete(m_hAllocator, m_pDefragmentator);
7051 m_pDefragmentator = VMA_NULL;
7052 }
7053 }
7054
MakePoolAllocationsLost(uint32_t currentFrameIndex,size_t * pLostAllocationCount)7055 void VmaBlockVector::MakePoolAllocationsLost(
7056 uint32_t currentFrameIndex,
7057 size_t* pLostAllocationCount)
7058 {
7059 VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7060 size_t lostAllocationCount = 0;
7061 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7062 {
7063 VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7064 VMA_ASSERT(pBlock);
7065 lostAllocationCount += pBlock->m_Metadata.MakeAllocationsLost(currentFrameIndex, m_FrameInUseCount);
7066 }
7067 if(pLostAllocationCount != VMA_NULL)
7068 {
7069 *pLostAllocationCount = lostAllocationCount;
7070 }
7071 }
7072
AddStats(VmaStats * pStats)7073 void VmaBlockVector::AddStats(VmaStats* pStats)
7074 {
7075 const uint32_t memTypeIndex = m_MemoryTypeIndex;
7076 const uint32_t memHeapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(memTypeIndex);
7077
7078 VmaMutexLock lock(m_Mutex, m_hAllocator->m_UseMutex);
7079
7080 for(uint32_t blockIndex = 0; blockIndex < m_Blocks.size(); ++blockIndex)
7081 {
7082 const VmaDeviceMemoryBlock* const pBlock = m_Blocks[blockIndex];
7083 VMA_ASSERT(pBlock);
7084 VMA_HEAVY_ASSERT(pBlock->Validate());
7085 VmaStatInfo allocationStatInfo;
7086 pBlock->m_Metadata.CalcAllocationStatInfo(allocationStatInfo);
7087 VmaAddStatInfo(pStats->total, allocationStatInfo);
7088 VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7089 VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7090 }
7091 }
7092
7093 ////////////////////////////////////////////////////////////////////////////////
7094 // VmaDefragmentator members definition
7095
VmaDefragmentator(VmaAllocator hAllocator,VmaBlockVector * pBlockVector,uint32_t currentFrameIndex)7096 VmaDefragmentator::VmaDefragmentator(
7097 VmaAllocator hAllocator,
7098 VmaBlockVector* pBlockVector,
7099 uint32_t currentFrameIndex) :
7100 m_hAllocator(hAllocator),
7101 m_pBlockVector(pBlockVector),
7102 m_CurrentFrameIndex(currentFrameIndex),
7103 m_BytesMoved(0),
7104 m_AllocationsMoved(0),
7105 m_Allocations(VmaStlAllocator<AllocationInfo>(hAllocator->GetAllocationCallbacks())),
7106 m_Blocks(VmaStlAllocator<BlockInfo*>(hAllocator->GetAllocationCallbacks()))
7107 {
7108 }
7109
~VmaDefragmentator()7110 VmaDefragmentator::~VmaDefragmentator()
7111 {
7112 for(size_t i = m_Blocks.size(); i--; )
7113 {
7114 vma_delete(m_hAllocator, m_Blocks[i]);
7115 }
7116 }
7117
AddAllocation(VmaAllocation hAlloc,VkBool32 * pChanged)7118 void VmaDefragmentator::AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged)
7119 {
7120 AllocationInfo allocInfo;
7121 allocInfo.m_hAllocation = hAlloc;
7122 allocInfo.m_pChanged = pChanged;
7123 m_Allocations.push_back(allocInfo);
7124 }
7125
EnsureMapping(VmaAllocator hAllocator,void ** ppMappedData)7126 VkResult VmaDefragmentator::BlockInfo::EnsureMapping(VmaAllocator hAllocator, void** ppMappedData)
7127 {
7128 // It has already been mapped for defragmentation.
7129 if(m_pMappedDataForDefragmentation)
7130 {
7131 *ppMappedData = m_pMappedDataForDefragmentation;
7132 return VK_SUCCESS;
7133 }
7134
7135 // It is originally mapped.
7136 if(m_pBlock->GetMappedData())
7137 {
7138 *ppMappedData = m_pBlock->GetMappedData();
7139 return VK_SUCCESS;
7140 }
7141
7142 // Map on first usage.
7143 VkResult res = m_pBlock->Map(hAllocator, 1, &m_pMappedDataForDefragmentation);
7144 *ppMappedData = m_pMappedDataForDefragmentation;
7145 return res;
7146 }
7147
Unmap(VmaAllocator hAllocator)7148 void VmaDefragmentator::BlockInfo::Unmap(VmaAllocator hAllocator)
7149 {
7150 if(m_pMappedDataForDefragmentation != VMA_NULL)
7151 {
7152 m_pBlock->Unmap(hAllocator, 1);
7153 }
7154 }
7155
DefragmentRound(VkDeviceSize maxBytesToMove,uint32_t maxAllocationsToMove)7156 VkResult VmaDefragmentator::DefragmentRound(
7157 VkDeviceSize maxBytesToMove,
7158 uint32_t maxAllocationsToMove)
7159 {
7160 if(m_Blocks.empty())
7161 {
7162 return VK_SUCCESS;
7163 }
7164
7165 size_t srcBlockIndex = m_Blocks.size() - 1;
7166 size_t srcAllocIndex = SIZE_MAX;
7167 for(;;)
7168 {
7169 // 1. Find next allocation to move.
7170 // 1.1. Start from last to first m_Blocks - they are sorted from most "destination" to most "source".
7171 // 1.2. Then start from last to first m_Allocations - they are sorted from largest to smallest.
7172 while(srcAllocIndex >= m_Blocks[srcBlockIndex]->m_Allocations.size())
7173 {
7174 if(m_Blocks[srcBlockIndex]->m_Allocations.empty())
7175 {
7176 // Finished: no more allocations to process.
7177 if(srcBlockIndex == 0)
7178 {
7179 return VK_SUCCESS;
7180 }
7181 else
7182 {
7183 --srcBlockIndex;
7184 srcAllocIndex = SIZE_MAX;
7185 }
7186 }
7187 else
7188 {
7189 srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1;
7190 }
7191 }
7192
7193 BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex];
7194 AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex];
7195
7196 const VkDeviceSize size = allocInfo.m_hAllocation->GetSize();
7197 const VkDeviceSize srcOffset = allocInfo.m_hAllocation->GetOffset();
7198 const VkDeviceSize alignment = allocInfo.m_hAllocation->GetAlignment();
7199 const VmaSuballocationType suballocType = allocInfo.m_hAllocation->GetSuballocationType();
7200
7201 // 2. Try to find new place for this allocation in preceding or current block.
7202 for(size_t dstBlockIndex = 0; dstBlockIndex <= srcBlockIndex; ++dstBlockIndex)
7203 {
7204 BlockInfo* pDstBlockInfo = m_Blocks[dstBlockIndex];
7205 VmaAllocationRequest dstAllocRequest;
7206 if(pDstBlockInfo->m_pBlock->m_Metadata.CreateAllocationRequest(
7207 m_CurrentFrameIndex,
7208 m_pBlockVector->GetFrameInUseCount(),
7209 m_pBlockVector->GetBufferImageGranularity(),
7210 size,
7211 alignment,
7212 suballocType,
7213 false, // canMakeOtherLost
7214 &dstAllocRequest) &&
7215 MoveMakesSense(
7216 dstBlockIndex, dstAllocRequest.offset, srcBlockIndex, srcOffset))
7217 {
7218 VMA_ASSERT(dstAllocRequest.itemsToMakeLostCount == 0);
7219
7220 // Reached limit on number of allocations or bytes to move.
7221 if((m_AllocationsMoved + 1 > maxAllocationsToMove) ||
7222 (m_BytesMoved + size > maxBytesToMove))
7223 {
7224 return VK_INCOMPLETE;
7225 }
7226
7227 void* pDstMappedData = VMA_NULL;
7228 VkResult res = pDstBlockInfo->EnsureMapping(m_hAllocator, &pDstMappedData);
7229 if(res != VK_SUCCESS)
7230 {
7231 return res;
7232 }
7233
7234 void* pSrcMappedData = VMA_NULL;
7235 res = pSrcBlockInfo->EnsureMapping(m_hAllocator, &pSrcMappedData);
7236 if(res != VK_SUCCESS)
7237 {
7238 return res;
7239 }
7240
7241 // THE PLACE WHERE ACTUAL DATA COPY HAPPENS.
7242 memcpy(
7243 reinterpret_cast<char*>(pDstMappedData) + dstAllocRequest.offset,
7244 reinterpret_cast<char*>(pSrcMappedData) + srcOffset,
7245 static_cast<size_t>(size));
7246
7247 pDstBlockInfo->m_pBlock->m_Metadata.Alloc(dstAllocRequest, suballocType, size, allocInfo.m_hAllocation);
7248 pSrcBlockInfo->m_pBlock->m_Metadata.FreeAtOffset(srcOffset);
7249
7250 allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset);
7251
7252 if(allocInfo.m_pChanged != VMA_NULL)
7253 {
7254 *allocInfo.m_pChanged = VK_TRUE;
7255 }
7256
7257 ++m_AllocationsMoved;
7258 m_BytesMoved += size;
7259
7260 VmaVectorRemove(pSrcBlockInfo->m_Allocations, srcAllocIndex);
7261
7262 break;
7263 }
7264 }
7265
7266 // If not processed, this allocInfo remains in pBlockInfo->m_Allocations for next round.
7267
7268 if(srcAllocIndex > 0)
7269 {
7270 --srcAllocIndex;
7271 }
7272 else
7273 {
7274 if(srcBlockIndex > 0)
7275 {
7276 --srcBlockIndex;
7277 srcAllocIndex = SIZE_MAX;
7278 }
7279 else
7280 {
7281 return VK_SUCCESS;
7282 }
7283 }
7284 }
7285 }
7286
Defragment(VkDeviceSize maxBytesToMove,uint32_t maxAllocationsToMove)7287 VkResult VmaDefragmentator::Defragment(
7288 VkDeviceSize maxBytesToMove,
7289 uint32_t maxAllocationsToMove)
7290 {
7291 if(m_Allocations.empty())
7292 {
7293 return VK_SUCCESS;
7294 }
7295
7296 // Create block info for each block.
7297 const size_t blockCount = m_pBlockVector->m_Blocks.size();
7298 for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7299 {
7300 BlockInfo* pBlockInfo = vma_new(m_hAllocator, BlockInfo)(m_hAllocator->GetAllocationCallbacks());
7301 pBlockInfo->m_pBlock = m_pBlockVector->m_Blocks[blockIndex];
7302 m_Blocks.push_back(pBlockInfo);
7303 }
7304
7305 // Sort them by m_pBlock pointer value.
7306 VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockPointerLess());
7307
7308 // Move allocation infos from m_Allocations to appropriate m_Blocks[memTypeIndex].m_Allocations.
7309 for(size_t blockIndex = 0, allocCount = m_Allocations.size(); blockIndex < allocCount; ++blockIndex)
7310 {
7311 AllocationInfo& allocInfo = m_Allocations[blockIndex];
7312 // Now as we are inside VmaBlockVector::m_Mutex, we can make final check if this allocation was not lost.
7313 if(allocInfo.m_hAllocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7314 {
7315 VmaDeviceMemoryBlock* pBlock = allocInfo.m_hAllocation->GetBlock();
7316 BlockInfoVector::iterator it = VmaBinaryFindFirstNotLess(m_Blocks.begin(), m_Blocks.end(), pBlock, BlockPointerLess());
7317 if(it != m_Blocks.end() && (*it)->m_pBlock == pBlock)
7318 {
7319 (*it)->m_Allocations.push_back(allocInfo);
7320 }
7321 else
7322 {
7323 VMA_ASSERT(0);
7324 }
7325 }
7326 }
7327 m_Allocations.clear();
7328
7329 for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7330 {
7331 BlockInfo* pBlockInfo = m_Blocks[blockIndex];
7332 pBlockInfo->CalcHasNonMovableAllocations();
7333 pBlockInfo->SortAllocationsBySizeDescecnding();
7334 }
7335
7336 // Sort m_Blocks this time by the main criterium, from most "destination" to most "source" blocks.
7337 VMA_SORT(m_Blocks.begin(), m_Blocks.end(), BlockInfoCompareMoveDestination());
7338
7339 // Execute defragmentation rounds (the main part).
7340 VkResult result = VK_SUCCESS;
7341 for(size_t round = 0; (round < 2) && (result == VK_SUCCESS); ++round)
7342 {
7343 result = DefragmentRound(maxBytesToMove, maxAllocationsToMove);
7344 }
7345
7346 // Unmap blocks that were mapped for defragmentation.
7347 for(size_t blockIndex = 0; blockIndex < blockCount; ++blockIndex)
7348 {
7349 m_Blocks[blockIndex]->Unmap(m_hAllocator);
7350 }
7351
7352 return result;
7353 }
7354
MoveMakesSense(size_t dstBlockIndex,VkDeviceSize dstOffset,size_t srcBlockIndex,VkDeviceSize srcOffset)7355 bool VmaDefragmentator::MoveMakesSense(
7356 size_t dstBlockIndex, VkDeviceSize dstOffset,
7357 size_t srcBlockIndex, VkDeviceSize srcOffset)
7358 {
7359 if(dstBlockIndex < srcBlockIndex)
7360 {
7361 return true;
7362 }
7363 if(dstBlockIndex > srcBlockIndex)
7364 {
7365 return false;
7366 }
7367 if(dstOffset < srcOffset)
7368 {
7369 return true;
7370 }
7371 return false;
7372 }
7373
7374 ////////////////////////////////////////////////////////////////////////////////
7375 // VmaAllocator_T
7376
VmaAllocator_T(const VmaAllocatorCreateInfo * pCreateInfo)7377 VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) :
7378 m_UseMutex((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXTERNALLY_SYNCHRONIZED_BIT) == 0),
7379 m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0),
7380 m_hDevice(pCreateInfo->device),
7381 m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL),
7382 m_AllocationCallbacks(pCreateInfo->pAllocationCallbacks ?
7383 *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks),
7384 m_PreferredLargeHeapBlockSize(0),
7385 m_PhysicalDevice(pCreateInfo->physicalDevice),
7386 m_CurrentFrameIndex(0),
7387 m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks()))
7388 {
7389 VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device);
7390
7391 memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks));
7392 memset(&m_MemProps, 0, sizeof(m_MemProps));
7393 memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties));
7394
7395 memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors));
7396 memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations));
7397
7398 for(uint32_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7399 {
7400 m_HeapSizeLimit[i] = VK_WHOLE_SIZE;
7401 }
7402
7403 if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL)
7404 {
7405 m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate;
7406 m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree;
7407 }
7408
7409 ImportVulkanFunctions(pCreateInfo->pVulkanFunctions);
7410
7411 (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties);
7412 (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps);
7413
7414 m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ?
7415 pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE);
7416
7417 if(pCreateInfo->pHeapSizeLimit != VMA_NULL)
7418 {
7419 for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex)
7420 {
7421 const VkDeviceSize limit = pCreateInfo->pHeapSizeLimit[heapIndex];
7422 if(limit != VK_WHOLE_SIZE)
7423 {
7424 m_HeapSizeLimit[heapIndex] = limit;
7425 if(limit < m_MemProps.memoryHeaps[heapIndex].size)
7426 {
7427 m_MemProps.memoryHeaps[heapIndex].size = limit;
7428 }
7429 }
7430 }
7431 }
7432
7433 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7434 {
7435 const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(memTypeIndex);
7436
7437 m_pBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)(
7438 this,
7439 memTypeIndex,
7440 preferredBlockSize,
7441 0,
7442 SIZE_MAX,
7443 GetBufferImageGranularity(),
7444 pCreateInfo->frameInUseCount,
7445 false); // isCustomPool
7446 // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here,
7447 // becase minBlockCount is 0.
7448 m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks()));
7449 }
7450 }
7451
~VmaAllocator_T()7452 VmaAllocator_T::~VmaAllocator_T()
7453 {
7454 VMA_ASSERT(m_Pools.empty());
7455
7456 for(size_t i = GetMemoryTypeCount(); i--; )
7457 {
7458 vma_delete(this, m_pDedicatedAllocations[i]);
7459 vma_delete(this, m_pBlockVectors[i]);
7460 }
7461 }
7462
ImportVulkanFunctions(const VmaVulkanFunctions * pVulkanFunctions)7463 void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions)
7464 {
7465 #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7466 m_VulkanFunctions.vkGetPhysicalDeviceProperties = &vkGetPhysicalDeviceProperties;
7467 m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = &vkGetPhysicalDeviceMemoryProperties;
7468 m_VulkanFunctions.vkAllocateMemory = &vkAllocateMemory;
7469 m_VulkanFunctions.vkFreeMemory = &vkFreeMemory;
7470 m_VulkanFunctions.vkMapMemory = &vkMapMemory;
7471 m_VulkanFunctions.vkUnmapMemory = &vkUnmapMemory;
7472 m_VulkanFunctions.vkBindBufferMemory = &vkBindBufferMemory;
7473 m_VulkanFunctions.vkBindImageMemory = &vkBindImageMemory;
7474 m_VulkanFunctions.vkGetBufferMemoryRequirements = &vkGetBufferMemoryRequirements;
7475 m_VulkanFunctions.vkGetImageMemoryRequirements = &vkGetImageMemoryRequirements;
7476 m_VulkanFunctions.vkCreateBuffer = &vkCreateBuffer;
7477 m_VulkanFunctions.vkDestroyBuffer = &vkDestroyBuffer;
7478 m_VulkanFunctions.vkCreateImage = &vkCreateImage;
7479 m_VulkanFunctions.vkDestroyImage = &vkDestroyImage;
7480 if(m_UseKhrDedicatedAllocation)
7481 {
7482 m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR =
7483 (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR");
7484 m_VulkanFunctions.vkGetImageMemoryRequirements2KHR =
7485 (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR");
7486 }
7487 #endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1
7488
7489 #define VMA_COPY_IF_NOT_NULL(funcName) \
7490 if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName;
7491
7492 if(pVulkanFunctions != VMA_NULL)
7493 {
7494 VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties);
7495 VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties);
7496 VMA_COPY_IF_NOT_NULL(vkAllocateMemory);
7497 VMA_COPY_IF_NOT_NULL(vkFreeMemory);
7498 VMA_COPY_IF_NOT_NULL(vkMapMemory);
7499 VMA_COPY_IF_NOT_NULL(vkUnmapMemory);
7500 VMA_COPY_IF_NOT_NULL(vkBindBufferMemory);
7501 VMA_COPY_IF_NOT_NULL(vkBindImageMemory);
7502 VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements);
7503 VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements);
7504 VMA_COPY_IF_NOT_NULL(vkCreateBuffer);
7505 VMA_COPY_IF_NOT_NULL(vkDestroyBuffer);
7506 VMA_COPY_IF_NOT_NULL(vkCreateImage);
7507 VMA_COPY_IF_NOT_NULL(vkDestroyImage);
7508 VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR);
7509 VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR);
7510 }
7511
7512 #undef VMA_COPY_IF_NOT_NULL
7513
7514 // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1
7515 // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions.
7516 VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL);
7517 VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL);
7518 VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL);
7519 VMA_ASSERT(m_VulkanFunctions.vkFreeMemory != VMA_NULL);
7520 VMA_ASSERT(m_VulkanFunctions.vkMapMemory != VMA_NULL);
7521 VMA_ASSERT(m_VulkanFunctions.vkUnmapMemory != VMA_NULL);
7522 VMA_ASSERT(m_VulkanFunctions.vkBindBufferMemory != VMA_NULL);
7523 VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory != VMA_NULL);
7524 VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements != VMA_NULL);
7525 VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements != VMA_NULL);
7526 VMA_ASSERT(m_VulkanFunctions.vkCreateBuffer != VMA_NULL);
7527 VMA_ASSERT(m_VulkanFunctions.vkDestroyBuffer != VMA_NULL);
7528 VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL);
7529 VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL);
7530 if(m_UseKhrDedicatedAllocation)
7531 {
7532 VMA_ASSERT(m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR != VMA_NULL);
7533 VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL);
7534 }
7535 }
7536
CalcPreferredBlockSize(uint32_t memTypeIndex)7537 VkDeviceSize VmaAllocator_T::CalcPreferredBlockSize(uint32_t memTypeIndex)
7538 {
7539 const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7540 const VkDeviceSize heapSize = m_MemProps.memoryHeaps[heapIndex].size;
7541 const bool isSmallHeap = heapSize <= VMA_SMALL_HEAP_MAX_SIZE;
7542 return isSmallHeap ? (heapSize / 8) : m_PreferredLargeHeapBlockSize;
7543 }
7544
AllocateMemoryOfType(const VkMemoryRequirements & vkMemReq,bool dedicatedAllocation,VkBuffer dedicatedBuffer,VkImage dedicatedImage,const VmaAllocationCreateInfo & createInfo,uint32_t memTypeIndex,VmaSuballocationType suballocType,VmaAllocation * pAllocation)7545 VkResult VmaAllocator_T::AllocateMemoryOfType(
7546 const VkMemoryRequirements& vkMemReq,
7547 bool dedicatedAllocation,
7548 VkBuffer dedicatedBuffer,
7549 VkImage dedicatedImage,
7550 const VmaAllocationCreateInfo& createInfo,
7551 uint32_t memTypeIndex,
7552 VmaSuballocationType suballocType,
7553 VmaAllocation* pAllocation)
7554 {
7555 VMA_ASSERT(pAllocation != VMA_NULL);
7556 VMA_DEBUG_LOG(" AllocateMemory: MemoryTypeIndex=%u, Size=%llu", memTypeIndex, vkMemReq.size);
7557
7558 VmaAllocationCreateInfo finalCreateInfo = createInfo;
7559
7560 // If memory type is not HOST_VISIBLE, disable MAPPED.
7561 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7562 (m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0)
7563 {
7564 finalCreateInfo.flags &= ~VMA_ALLOCATION_CREATE_MAPPED_BIT;
7565 }
7566
7567 VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex];
7568 VMA_ASSERT(blockVector);
7569
7570 const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize();
7571 bool preferDedicatedMemory =
7572 VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ||
7573 dedicatedAllocation ||
7574 // Heuristics: Allocate dedicated memory if requested size if greater than half of preferred block size.
7575 vkMemReq.size > preferredBlockSize / 2;
7576
7577 if(preferDedicatedMemory &&
7578 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0 &&
7579 finalCreateInfo.pool == VK_NULL_HANDLE)
7580 {
7581 finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT;
7582 }
7583
7584 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0)
7585 {
7586 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7587 {
7588 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7589 }
7590 else
7591 {
7592 return AllocateDedicatedMemory(
7593 vkMemReq.size,
7594 suballocType,
7595 memTypeIndex,
7596 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7597 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7598 finalCreateInfo.pUserData,
7599 dedicatedBuffer,
7600 dedicatedImage,
7601 pAllocation);
7602 }
7603 }
7604 else
7605 {
7606 VkResult res = blockVector->Allocate(
7607 VK_NULL_HANDLE, // hCurrentPool
7608 m_CurrentFrameIndex.load(),
7609 vkMemReq,
7610 finalCreateInfo,
7611 suballocType,
7612 pAllocation);
7613 if(res == VK_SUCCESS)
7614 {
7615 return res;
7616 }
7617
7618 // 5. Try dedicated memory.
7619 if((finalCreateInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7620 {
7621 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7622 }
7623 else
7624 {
7625 res = AllocateDedicatedMemory(
7626 vkMemReq.size,
7627 suballocType,
7628 memTypeIndex,
7629 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0,
7630 (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0,
7631 finalCreateInfo.pUserData,
7632 dedicatedBuffer,
7633 dedicatedImage,
7634 pAllocation);
7635 if(res == VK_SUCCESS)
7636 {
7637 // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here.
7638 VMA_DEBUG_LOG(" Allocated as DedicatedMemory");
7639 return VK_SUCCESS;
7640 }
7641 else
7642 {
7643 // Everything failed: Return error code.
7644 VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7645 return res;
7646 }
7647 }
7648 }
7649 }
7650
AllocateDedicatedMemory(VkDeviceSize size,VmaSuballocationType suballocType,uint32_t memTypeIndex,bool map,bool isUserDataString,void * pUserData,VkBuffer dedicatedBuffer,VkImage dedicatedImage,VmaAllocation * pAllocation)7651 VkResult VmaAllocator_T::AllocateDedicatedMemory(
7652 VkDeviceSize size,
7653 VmaSuballocationType suballocType,
7654 uint32_t memTypeIndex,
7655 bool map,
7656 bool isUserDataString,
7657 void* pUserData,
7658 VkBuffer dedicatedBuffer,
7659 VkImage dedicatedImage,
7660 VmaAllocation* pAllocation)
7661 {
7662 VMA_ASSERT(pAllocation);
7663
7664 VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO };
7665 allocInfo.memoryTypeIndex = memTypeIndex;
7666 allocInfo.allocationSize = size;
7667
7668 VkMemoryDedicatedAllocateInfoKHR dedicatedAllocInfo = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_ALLOCATE_INFO_KHR };
7669 if(m_UseKhrDedicatedAllocation)
7670 {
7671 if(dedicatedBuffer != VK_NULL_HANDLE)
7672 {
7673 VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE);
7674 dedicatedAllocInfo.buffer = dedicatedBuffer;
7675 allocInfo.pNext = &dedicatedAllocInfo;
7676 }
7677 else if(dedicatedImage != VK_NULL_HANDLE)
7678 {
7679 dedicatedAllocInfo.image = dedicatedImage;
7680 allocInfo.pNext = &dedicatedAllocInfo;
7681 }
7682 }
7683
7684 // Allocate VkDeviceMemory.
7685 VkDeviceMemory hMemory = VK_NULL_HANDLE;
7686 VkResult res = AllocateVulkanMemory(&allocInfo, &hMemory);
7687 if(res < 0)
7688 {
7689 VMA_DEBUG_LOG(" vkAllocateMemory FAILED");
7690 return res;
7691 }
7692
7693 void* pMappedData = VMA_NULL;
7694 if(map)
7695 {
7696 res = (*m_VulkanFunctions.vkMapMemory)(
7697 m_hDevice,
7698 hMemory,
7699 0,
7700 VK_WHOLE_SIZE,
7701 0,
7702 &pMappedData);
7703 if(res < 0)
7704 {
7705 VMA_DEBUG_LOG(" vkMapMemory FAILED");
7706 FreeVulkanMemory(memTypeIndex, size, hMemory);
7707 return res;
7708 }
7709 }
7710
7711 *pAllocation = vma_new(this, VmaAllocation_T)(m_CurrentFrameIndex.load(), isUserDataString);
7712 (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size);
7713 (*pAllocation)->SetUserData(this, pUserData);
7714
7715 // Register it in m_pDedicatedAllocations.
7716 {
7717 VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7718 AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
7719 VMA_ASSERT(pDedicatedAllocations);
7720 VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, *pAllocation);
7721 }
7722
7723 VMA_DEBUG_LOG(" Allocated DedicatedMemory MemoryTypeIndex=#%u", memTypeIndex);
7724
7725 return VK_SUCCESS;
7726 }
7727
GetBufferMemoryRequirements(VkBuffer hBuffer,VkMemoryRequirements & memReq,bool & requiresDedicatedAllocation,bool & prefersDedicatedAllocation)7728 void VmaAllocator_T::GetBufferMemoryRequirements(
7729 VkBuffer hBuffer,
7730 VkMemoryRequirements& memReq,
7731 bool& requiresDedicatedAllocation,
7732 bool& prefersDedicatedAllocation) const
7733 {
7734 if(m_UseKhrDedicatedAllocation)
7735 {
7736 VkBufferMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_BUFFER_MEMORY_REQUIREMENTS_INFO_2_KHR };
7737 memReqInfo.buffer = hBuffer;
7738
7739 VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7740
7741 VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7742 memReq2.pNext = &memDedicatedReq;
7743
7744 (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7745
7746 memReq = memReq2.memoryRequirements;
7747 requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7748 prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7749 }
7750 else
7751 {
7752 (*m_VulkanFunctions.vkGetBufferMemoryRequirements)(m_hDevice, hBuffer, &memReq);
7753 requiresDedicatedAllocation = false;
7754 prefersDedicatedAllocation = false;
7755 }
7756 }
7757
GetImageMemoryRequirements(VkImage hImage,VkMemoryRequirements & memReq,bool & requiresDedicatedAllocation,bool & prefersDedicatedAllocation)7758 void VmaAllocator_T::GetImageMemoryRequirements(
7759 VkImage hImage,
7760 VkMemoryRequirements& memReq,
7761 bool& requiresDedicatedAllocation,
7762 bool& prefersDedicatedAllocation) const
7763 {
7764 if(m_UseKhrDedicatedAllocation)
7765 {
7766 VkImageMemoryRequirementsInfo2KHR memReqInfo = { VK_STRUCTURE_TYPE_IMAGE_MEMORY_REQUIREMENTS_INFO_2_KHR };
7767 memReqInfo.image = hImage;
7768
7769 VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR };
7770
7771 VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR };
7772 memReq2.pNext = &memDedicatedReq;
7773
7774 (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2);
7775
7776 memReq = memReq2.memoryRequirements;
7777 requiresDedicatedAllocation = (memDedicatedReq.requiresDedicatedAllocation != VK_FALSE);
7778 prefersDedicatedAllocation = (memDedicatedReq.prefersDedicatedAllocation != VK_FALSE);
7779 }
7780 else
7781 {
7782 (*m_VulkanFunctions.vkGetImageMemoryRequirements)(m_hDevice, hImage, &memReq);
7783 requiresDedicatedAllocation = false;
7784 prefersDedicatedAllocation = false;
7785 }
7786 }
7787
AllocateMemory(const VkMemoryRequirements & vkMemReq,bool requiresDedicatedAllocation,bool prefersDedicatedAllocation,VkBuffer dedicatedBuffer,VkImage dedicatedImage,const VmaAllocationCreateInfo & createInfo,VmaSuballocationType suballocType,VmaAllocation * pAllocation)7788 VkResult VmaAllocator_T::AllocateMemory(
7789 const VkMemoryRequirements& vkMemReq,
7790 bool requiresDedicatedAllocation,
7791 bool prefersDedicatedAllocation,
7792 VkBuffer dedicatedBuffer,
7793 VkImage dedicatedImage,
7794 const VmaAllocationCreateInfo& createInfo,
7795 VmaSuballocationType suballocType,
7796 VmaAllocation* pAllocation)
7797 {
7798 if((createInfo.flags & VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT) != 0 &&
7799 (createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7800 {
7801 VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT together with VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT makes no sense.");
7802 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7803 }
7804 if((createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 &&
7805 (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT) != 0)
7806 {
7807 VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_MAPPED_BIT together with VMA_ALLOCATION_CREATE_CAN_BECOME_LOST_BIT is invalid.");
7808 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7809 }
7810 if(requiresDedicatedAllocation)
7811 {
7812 if((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) != 0)
7813 {
7814 VMA_ASSERT(0 && "VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT specified while dedicated allocation is required.");
7815 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7816 }
7817 if(createInfo.pool != VK_NULL_HANDLE)
7818 {
7819 VMA_ASSERT(0 && "Pool specified while dedicated allocation is required.");
7820 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7821 }
7822 }
7823 if((createInfo.pool != VK_NULL_HANDLE) &&
7824 ((createInfo.flags & (VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT)) != 0))
7825 {
7826 VMA_ASSERT(0 && "Specifying VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT when pool != null is invalid.");
7827 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7828 }
7829
7830 if(createInfo.pool != VK_NULL_HANDLE)
7831 {
7832 return createInfo.pool->m_BlockVector.Allocate(
7833 createInfo.pool,
7834 m_CurrentFrameIndex.load(),
7835 vkMemReq,
7836 createInfo,
7837 suballocType,
7838 pAllocation);
7839 }
7840 else
7841 {
7842 // Bit mask of memory Vulkan types acceptable for this allocation.
7843 uint32_t memoryTypeBits = vkMemReq.memoryTypeBits;
7844 uint32_t memTypeIndex = UINT32_MAX;
7845 VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7846 if(res == VK_SUCCESS)
7847 {
7848 res = AllocateMemoryOfType(
7849 vkMemReq,
7850 requiresDedicatedAllocation || prefersDedicatedAllocation,
7851 dedicatedBuffer,
7852 dedicatedImage,
7853 createInfo,
7854 memTypeIndex,
7855 suballocType,
7856 pAllocation);
7857 // Succeeded on first try.
7858 if(res == VK_SUCCESS)
7859 {
7860 return res;
7861 }
7862 // Allocation from this memory type failed. Try other compatible memory types.
7863 else
7864 {
7865 for(;;)
7866 {
7867 // Remove old memTypeIndex from list of possibilities.
7868 memoryTypeBits &= ~(1u << memTypeIndex);
7869 // Find alternative memTypeIndex.
7870 res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex);
7871 if(res == VK_SUCCESS)
7872 {
7873 res = AllocateMemoryOfType(
7874 vkMemReq,
7875 requiresDedicatedAllocation || prefersDedicatedAllocation,
7876 dedicatedBuffer,
7877 dedicatedImage,
7878 createInfo,
7879 memTypeIndex,
7880 suballocType,
7881 pAllocation);
7882 // Allocation from this alternative memory type succeeded.
7883 if(res == VK_SUCCESS)
7884 {
7885 return res;
7886 }
7887 // else: Allocation from this memory type failed. Try next one - next loop iteration.
7888 }
7889 // No other matching memory type index could be found.
7890 else
7891 {
7892 // Not returning res, which is VK_ERROR_FEATURE_NOT_PRESENT, because we already failed to allocate once.
7893 return VK_ERROR_OUT_OF_DEVICE_MEMORY;
7894 }
7895 }
7896 }
7897 }
7898 // Can't find any single memory type maching requirements. res is VK_ERROR_FEATURE_NOT_PRESENT.
7899 else
7900 return res;
7901 }
7902 }
7903
FreeMemory(const VmaAllocation allocation)7904 void VmaAllocator_T::FreeMemory(const VmaAllocation allocation)
7905 {
7906 VMA_ASSERT(allocation);
7907
7908 if(allocation->CanBecomeLost() == false ||
7909 allocation->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST)
7910 {
7911 switch(allocation->GetType())
7912 {
7913 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
7914 {
7915 VmaBlockVector* pBlockVector = VMA_NULL;
7916 VmaPool hPool = allocation->GetPool();
7917 if(hPool != VK_NULL_HANDLE)
7918 {
7919 pBlockVector = &hPool->m_BlockVector;
7920 }
7921 else
7922 {
7923 const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
7924 pBlockVector = m_pBlockVectors[memTypeIndex];
7925 }
7926 pBlockVector->Free(allocation);
7927 }
7928 break;
7929 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
7930 FreeDedicatedMemory(allocation);
7931 break;
7932 default:
7933 VMA_ASSERT(0);
7934 }
7935 }
7936
7937 allocation->SetUserData(this, VMA_NULL);
7938 vma_delete(this, allocation);
7939 }
7940
CalculateStats(VmaStats * pStats)7941 void VmaAllocator_T::CalculateStats(VmaStats* pStats)
7942 {
7943 // Initialize.
7944 InitStatInfo(pStats->total);
7945 for(size_t i = 0; i < VK_MAX_MEMORY_TYPES; ++i)
7946 InitStatInfo(pStats->memoryType[i]);
7947 for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i)
7948 InitStatInfo(pStats->memoryHeap[i]);
7949
7950 // Process default pools.
7951 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7952 {
7953 VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex];
7954 VMA_ASSERT(pBlockVector);
7955 pBlockVector->AddStats(pStats);
7956 }
7957
7958 // Process custom pools.
7959 {
7960 VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
7961 for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex)
7962 {
7963 m_Pools[poolIndex]->GetBlockVector().AddStats(pStats);
7964 }
7965 }
7966
7967 // Process dedicated allocations.
7968 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
7969 {
7970 const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex);
7971 VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
7972 AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
7973 VMA_ASSERT(pDedicatedAllocVector);
7974 for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex)
7975 {
7976 VmaStatInfo allocationStatInfo;
7977 (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo);
7978 VmaAddStatInfo(pStats->total, allocationStatInfo);
7979 VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo);
7980 VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo);
7981 }
7982 }
7983
7984 // Postprocess.
7985 VmaPostprocessCalcStatInfo(pStats->total);
7986 for(size_t i = 0; i < GetMemoryTypeCount(); ++i)
7987 VmaPostprocessCalcStatInfo(pStats->memoryType[i]);
7988 for(size_t i = 0; i < GetMemoryHeapCount(); ++i)
7989 VmaPostprocessCalcStatInfo(pStats->memoryHeap[i]);
7990 }
7991
7992 static const uint32_t VMA_VENDOR_ID_AMD = 4098;
7993
Defragment(VmaAllocation * pAllocations,size_t allocationCount,VkBool32 * pAllocationsChanged,const VmaDefragmentationInfo * pDefragmentationInfo,VmaDefragmentationStats * pDefragmentationStats)7994 VkResult VmaAllocator_T::Defragment(
7995 VmaAllocation* pAllocations,
7996 size_t allocationCount,
7997 VkBool32* pAllocationsChanged,
7998 const VmaDefragmentationInfo* pDefragmentationInfo,
7999 VmaDefragmentationStats* pDefragmentationStats)
8000 {
8001 if(pAllocationsChanged != VMA_NULL)
8002 {
8003 memset(pAllocationsChanged, 0, sizeof(*pAllocationsChanged));
8004 }
8005 if(pDefragmentationStats != VMA_NULL)
8006 {
8007 memset(pDefragmentationStats, 0, sizeof(*pDefragmentationStats));
8008 }
8009
8010 const uint32_t currentFrameIndex = m_CurrentFrameIndex.load();
8011
8012 VmaMutexLock poolsLock(m_PoolsMutex, m_UseMutex);
8013
8014 const size_t poolCount = m_Pools.size();
8015
8016 // Dispatch pAllocations among defragmentators. Create them in BlockVectors when necessary.
8017 for(size_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex)
8018 {
8019 VmaAllocation hAlloc = pAllocations[allocIndex];
8020 VMA_ASSERT(hAlloc);
8021 const uint32_t memTypeIndex = hAlloc->GetMemoryTypeIndex();
8022 // DedicatedAlloc cannot be defragmented.
8023 if((hAlloc->GetType() == VmaAllocation_T::ALLOCATION_TYPE_BLOCK) &&
8024 // Only HOST_VISIBLE memory types can be defragmented.
8025 ((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) &&
8026 // Lost allocation cannot be defragmented.
8027 (hAlloc->GetLastUseFrameIndex() != VMA_FRAME_INDEX_LOST))
8028 {
8029 VmaBlockVector* pAllocBlockVector = VMA_NULL;
8030
8031 const VmaPool hAllocPool = hAlloc->GetPool();
8032 // This allocation belongs to custom pool.
8033 if(hAllocPool != VK_NULL_HANDLE)
8034 {
8035 pAllocBlockVector = &hAllocPool->GetBlockVector();
8036 }
8037 // This allocation belongs to general pool.
8038 else
8039 {
8040 pAllocBlockVector = m_pBlockVectors[memTypeIndex];
8041 }
8042
8043 VmaDefragmentator* const pDefragmentator = pAllocBlockVector->EnsureDefragmentator(this, currentFrameIndex);
8044
8045 VkBool32* const pChanged = (pAllocationsChanged != VMA_NULL) ?
8046 &pAllocationsChanged[allocIndex] : VMA_NULL;
8047 pDefragmentator->AddAllocation(hAlloc, pChanged);
8048 }
8049 }
8050
8051 VkResult result = VK_SUCCESS;
8052
8053 // ======== Main processing.
8054
8055 VkDeviceSize maxBytesToMove = SIZE_MAX;
8056 uint32_t maxAllocationsToMove = UINT32_MAX;
8057 if(pDefragmentationInfo != VMA_NULL)
8058 {
8059 maxBytesToMove = pDefragmentationInfo->maxBytesToMove;
8060 maxAllocationsToMove = pDefragmentationInfo->maxAllocationsToMove;
8061 }
8062
8063 // Process standard memory.
8064 for(uint32_t memTypeIndex = 0;
8065 (memTypeIndex < GetMemoryTypeCount()) && (result == VK_SUCCESS);
8066 ++memTypeIndex)
8067 {
8068 // Only HOST_VISIBLE memory types can be defragmented.
8069 if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8070 {
8071 result = m_pBlockVectors[memTypeIndex]->Defragment(
8072 pDefragmentationStats,
8073 maxBytesToMove,
8074 maxAllocationsToMove);
8075 }
8076 }
8077
8078 // Process custom pools.
8079 for(size_t poolIndex = 0; (poolIndex < poolCount) && (result == VK_SUCCESS); ++poolIndex)
8080 {
8081 result = m_Pools[poolIndex]->GetBlockVector().Defragment(
8082 pDefragmentationStats,
8083 maxBytesToMove,
8084 maxAllocationsToMove);
8085 }
8086
8087 // ======== Destroy defragmentators.
8088
8089 // Process custom pools.
8090 for(size_t poolIndex = poolCount; poolIndex--; )
8091 {
8092 m_Pools[poolIndex]->GetBlockVector().DestroyDefragmentator();
8093 }
8094
8095 // Process standard memory.
8096 for(uint32_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; )
8097 {
8098 if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8099 {
8100 m_pBlockVectors[memTypeIndex]->DestroyDefragmentator();
8101 }
8102 }
8103
8104 return result;
8105 }
8106
GetAllocationInfo(VmaAllocation hAllocation,VmaAllocationInfo * pAllocationInfo)8107 void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo)
8108 {
8109 if(hAllocation->CanBecomeLost())
8110 {
8111 /*
8112 Warning: This is a carefully designed algorithm.
8113 Do not modify unless you really know what you're doing :)
8114 */
8115 uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
8116 uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
8117 for(;;)
8118 {
8119 if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
8120 {
8121 pAllocationInfo->memoryType = UINT32_MAX;
8122 pAllocationInfo->deviceMemory = VK_NULL_HANDLE;
8123 pAllocationInfo->offset = 0;
8124 pAllocationInfo->size = hAllocation->GetSize();
8125 pAllocationInfo->pMappedData = VMA_NULL;
8126 pAllocationInfo->pUserData = hAllocation->GetUserData();
8127 return;
8128 }
8129 else if(localLastUseFrameIndex == localCurrFrameIndex)
8130 {
8131 pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
8132 pAllocationInfo->deviceMemory = hAllocation->GetMemory();
8133 pAllocationInfo->offset = hAllocation->GetOffset();
8134 pAllocationInfo->size = hAllocation->GetSize();
8135 pAllocationInfo->pMappedData = VMA_NULL;
8136 pAllocationInfo->pUserData = hAllocation->GetUserData();
8137 return;
8138 }
8139 else // Last use time earlier than current time.
8140 {
8141 if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
8142 {
8143 localLastUseFrameIndex = localCurrFrameIndex;
8144 }
8145 }
8146 }
8147 }
8148 else
8149 {
8150 pAllocationInfo->memoryType = hAllocation->GetMemoryTypeIndex();
8151 pAllocationInfo->deviceMemory = hAllocation->GetMemory();
8152 pAllocationInfo->offset = hAllocation->GetOffset();
8153 pAllocationInfo->size = hAllocation->GetSize();
8154 pAllocationInfo->pMappedData = hAllocation->GetMappedData();
8155 pAllocationInfo->pUserData = hAllocation->GetUserData();
8156 }
8157 }
8158
TouchAllocation(VmaAllocation hAllocation)8159 bool VmaAllocator_T::TouchAllocation(VmaAllocation hAllocation)
8160 {
8161 // This is a stripped-down version of VmaAllocator_T::GetAllocationInfo.
8162 if(hAllocation->CanBecomeLost())
8163 {
8164 uint32_t localCurrFrameIndex = m_CurrentFrameIndex.load();
8165 uint32_t localLastUseFrameIndex = hAllocation->GetLastUseFrameIndex();
8166 for(;;)
8167 {
8168 if(localLastUseFrameIndex == VMA_FRAME_INDEX_LOST)
8169 {
8170 return false;
8171 }
8172 else if(localLastUseFrameIndex == localCurrFrameIndex)
8173 {
8174 return true;
8175 }
8176 else // Last use time earlier than current time.
8177 {
8178 if(hAllocation->CompareExchangeLastUseFrameIndex(localLastUseFrameIndex, localCurrFrameIndex))
8179 {
8180 localLastUseFrameIndex = localCurrFrameIndex;
8181 }
8182 }
8183 }
8184 }
8185 else
8186 {
8187 return true;
8188 }
8189 }
8190
CreatePool(const VmaPoolCreateInfo * pCreateInfo,VmaPool * pPool)8191 VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPool* pPool)
8192 {
8193 VMA_DEBUG_LOG(" CreatePool: MemoryTypeIndex=%u", pCreateInfo->memoryTypeIndex);
8194
8195 VmaPoolCreateInfo newCreateInfo = *pCreateInfo;
8196
8197 if(newCreateInfo.maxBlockCount == 0)
8198 {
8199 newCreateInfo.maxBlockCount = SIZE_MAX;
8200 }
8201 if(newCreateInfo.blockSize == 0)
8202 {
8203 newCreateInfo.blockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex);
8204 }
8205
8206 *pPool = vma_new(this, VmaPool_T)(this, newCreateInfo);
8207
8208 VkResult res = (*pPool)->m_BlockVector.CreateMinBlocks();
8209 if(res != VK_SUCCESS)
8210 {
8211 vma_delete(this, *pPool);
8212 *pPool = VMA_NULL;
8213 return res;
8214 }
8215
8216 // Add to m_Pools.
8217 {
8218 VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8219 VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool);
8220 }
8221
8222 return VK_SUCCESS;
8223 }
8224
DestroyPool(VmaPool pool)8225 void VmaAllocator_T::DestroyPool(VmaPool pool)
8226 {
8227 // Remove from m_Pools.
8228 {
8229 VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8230 bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool);
8231 VMA_ASSERT(success && "Pool not found in Allocator.");
8232 }
8233
8234 vma_delete(this, pool);
8235 }
8236
GetPoolStats(VmaPool pool,VmaPoolStats * pPoolStats)8237 void VmaAllocator_T::GetPoolStats(VmaPool pool, VmaPoolStats* pPoolStats)
8238 {
8239 pool->m_BlockVector.GetPoolStats(pPoolStats);
8240 }
8241
SetCurrentFrameIndex(uint32_t frameIndex)8242 void VmaAllocator_T::SetCurrentFrameIndex(uint32_t frameIndex)
8243 {
8244 m_CurrentFrameIndex.store(frameIndex);
8245 }
8246
MakePoolAllocationsLost(VmaPool hPool,size_t * pLostAllocationCount)8247 void VmaAllocator_T::MakePoolAllocationsLost(
8248 VmaPool hPool,
8249 size_t* pLostAllocationCount)
8250 {
8251 hPool->m_BlockVector.MakePoolAllocationsLost(
8252 m_CurrentFrameIndex.load(),
8253 pLostAllocationCount);
8254 }
8255
CreateLostAllocation(VmaAllocation * pAllocation)8256 void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation)
8257 {
8258 *pAllocation = vma_new(this, VmaAllocation_T)(VMA_FRAME_INDEX_LOST, false);
8259 (*pAllocation)->InitLost();
8260 }
8261
AllocateVulkanMemory(const VkMemoryAllocateInfo * pAllocateInfo,VkDeviceMemory * pMemory)8262 VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory)
8263 {
8264 const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex);
8265
8266 VkResult res;
8267 if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
8268 {
8269 VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
8270 if(m_HeapSizeLimit[heapIndex] >= pAllocateInfo->allocationSize)
8271 {
8272 res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
8273 if(res == VK_SUCCESS)
8274 {
8275 m_HeapSizeLimit[heapIndex] -= pAllocateInfo->allocationSize;
8276 }
8277 }
8278 else
8279 {
8280 res = VK_ERROR_OUT_OF_DEVICE_MEMORY;
8281 }
8282 }
8283 else
8284 {
8285 res = (*m_VulkanFunctions.vkAllocateMemory)(m_hDevice, pAllocateInfo, GetAllocationCallbacks(), pMemory);
8286 }
8287
8288 if(res == VK_SUCCESS && m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL)
8289 {
8290 (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize);
8291 }
8292
8293 return res;
8294 }
8295
FreeVulkanMemory(uint32_t memoryType,VkDeviceSize size,VkDeviceMemory hMemory)8296 void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, VkDeviceMemory hMemory)
8297 {
8298 if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL)
8299 {
8300 (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size);
8301 }
8302
8303 (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks());
8304
8305 const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(memoryType);
8306 if(m_HeapSizeLimit[heapIndex] != VK_WHOLE_SIZE)
8307 {
8308 VmaMutexLock lock(m_HeapSizeLimitMutex, m_UseMutex);
8309 m_HeapSizeLimit[heapIndex] += size;
8310 }
8311 }
8312
Map(VmaAllocation hAllocation,void ** ppData)8313 VkResult VmaAllocator_T::Map(VmaAllocation hAllocation, void** ppData)
8314 {
8315 if(hAllocation->CanBecomeLost())
8316 {
8317 return VK_ERROR_MEMORY_MAP_FAILED;
8318 }
8319
8320 switch(hAllocation->GetType())
8321 {
8322 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8323 {
8324 VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
8325 char *pBytes = VMA_NULL;
8326 VkResult res = pBlock->Map(this, 1, (void**)&pBytes);
8327 if(res == VK_SUCCESS)
8328 {
8329 *ppData = pBytes + (ptrdiff_t)hAllocation->GetOffset();
8330 hAllocation->BlockAllocMap();
8331 }
8332 return res;
8333 }
8334 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8335 return hAllocation->DedicatedAllocMap(this, ppData);
8336 default:
8337 VMA_ASSERT(0);
8338 return VK_ERROR_MEMORY_MAP_FAILED;
8339 }
8340 }
8341
Unmap(VmaAllocation hAllocation)8342 void VmaAllocator_T::Unmap(VmaAllocation hAllocation)
8343 {
8344 switch(hAllocation->GetType())
8345 {
8346 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8347 {
8348 VmaDeviceMemoryBlock* const pBlock = hAllocation->GetBlock();
8349 hAllocation->BlockAllocUnmap();
8350 pBlock->Unmap(this, 1);
8351 }
8352 break;
8353 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8354 hAllocation->DedicatedAllocUnmap(this);
8355 break;
8356 default:
8357 VMA_ASSERT(0);
8358 }
8359 }
8360
BindBufferMemory(VmaAllocation hAllocation,VkBuffer hBuffer)8361 VkResult VmaAllocator_T::BindBufferMemory(VmaAllocation hAllocation, VkBuffer hBuffer)
8362 {
8363 VkResult res = VK_SUCCESS;
8364 switch(hAllocation->GetType())
8365 {
8366 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8367 res = GetVulkanFunctions().vkBindBufferMemory(
8368 m_hDevice,
8369 hBuffer,
8370 hAllocation->GetMemory(),
8371 0); //memoryOffset
8372 break;
8373 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8374 {
8375 VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
8376 VMA_ASSERT(pBlock && "Binding buffer to allocation that doesn't belong to any block. Is the allocation lost?");
8377 res = pBlock->BindBufferMemory(this, hAllocation, hBuffer);
8378 break;
8379 }
8380 default:
8381 VMA_ASSERT(0);
8382 }
8383 return res;
8384 }
8385
BindImageMemory(VmaAllocation hAllocation,VkImage hImage)8386 VkResult VmaAllocator_T::BindImageMemory(VmaAllocation hAllocation, VkImage hImage)
8387 {
8388 VkResult res = VK_SUCCESS;
8389 switch(hAllocation->GetType())
8390 {
8391 case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED:
8392 res = GetVulkanFunctions().vkBindImageMemory(
8393 m_hDevice,
8394 hImage,
8395 hAllocation->GetMemory(),
8396 0); //memoryOffset
8397 break;
8398 case VmaAllocation_T::ALLOCATION_TYPE_BLOCK:
8399 {
8400 VmaDeviceMemoryBlock* pBlock = hAllocation->GetBlock();
8401 VMA_ASSERT(pBlock && "Binding image to allocation that doesn't belong to any block. Is the allocation lost?");
8402 res = pBlock->BindImageMemory(this, hAllocation, hImage);
8403 break;
8404 }
8405 default:
8406 VMA_ASSERT(0);
8407 }
8408 return res;
8409 }
8410
FreeDedicatedMemory(VmaAllocation allocation)8411 void VmaAllocator_T::FreeDedicatedMemory(VmaAllocation allocation)
8412 {
8413 VMA_ASSERT(allocation && allocation->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED);
8414
8415 const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex();
8416 {
8417 VmaMutexLock lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
8418 AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex];
8419 VMA_ASSERT(pDedicatedAllocations);
8420 bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation);
8421 VMA_ASSERT(success);
8422 }
8423
8424 VkDeviceMemory hMemory = allocation->GetMemory();
8425
8426 if(allocation->GetMappedData() != VMA_NULL)
8427 {
8428 (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory);
8429 }
8430
8431 FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory);
8432
8433 VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex);
8434 }
8435
8436 #if VMA_STATS_STRING_ENABLED
8437
PrintDetailedMap(VmaJsonWriter & json)8438 void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json)
8439 {
8440 bool dedicatedAllocationsStarted = false;
8441 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8442 {
8443 VmaMutexLock dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex);
8444 AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex];
8445 VMA_ASSERT(pDedicatedAllocVector);
8446 if(pDedicatedAllocVector->empty() == false)
8447 {
8448 if(dedicatedAllocationsStarted == false)
8449 {
8450 dedicatedAllocationsStarted = true;
8451 json.WriteString("DedicatedAllocations");
8452 json.BeginObject();
8453 }
8454
8455 json.BeginString("Type ");
8456 json.ContinueString(memTypeIndex);
8457 json.EndString();
8458
8459 json.BeginArray();
8460
8461 for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i)
8462 {
8463 const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i];
8464 json.BeginObject(true);
8465
8466 json.WriteString("Type");
8467 json.WriteString(VMA_SUBALLOCATION_TYPE_NAMES[hAlloc->GetSuballocationType()]);
8468
8469 json.WriteString("Size");
8470 json.WriteNumber(hAlloc->GetSize());
8471
8472 const void* pUserData = hAlloc->GetUserData();
8473 if(pUserData != VMA_NULL)
8474 {
8475 json.WriteString("UserData");
8476 if(hAlloc->IsUserDataString())
8477 {
8478 json.WriteString((const char*)pUserData);
8479 }
8480 else
8481 {
8482 json.BeginString();
8483 json.ContinueString_Pointer(pUserData);
8484 json.EndString();
8485 }
8486 }
8487
8488 json.EndObject();
8489 }
8490
8491 json.EndArray();
8492 }
8493 }
8494 if(dedicatedAllocationsStarted)
8495 {
8496 json.EndObject();
8497 }
8498
8499 {
8500 bool allocationsStarted = false;
8501 for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex)
8502 {
8503 if(m_pBlockVectors[memTypeIndex]->IsEmpty() == false)
8504 {
8505 if(allocationsStarted == false)
8506 {
8507 allocationsStarted = true;
8508 json.WriteString("DefaultPools");
8509 json.BeginObject();
8510 }
8511
8512 json.BeginString("Type ");
8513 json.ContinueString(memTypeIndex);
8514 json.EndString();
8515
8516 m_pBlockVectors[memTypeIndex]->PrintDetailedMap(json);
8517 }
8518 }
8519 if(allocationsStarted)
8520 {
8521 json.EndObject();
8522 }
8523 }
8524
8525 {
8526 VmaMutexLock lock(m_PoolsMutex, m_UseMutex);
8527 const size_t poolCount = m_Pools.size();
8528 if(poolCount > 0)
8529 {
8530 json.WriteString("Pools");
8531 json.BeginArray();
8532 for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex)
8533 {
8534 m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json);
8535 }
8536 json.EndArray();
8537 }
8538 }
8539 }
8540
8541 #endif // #if VMA_STATS_STRING_ENABLED
8542
AllocateMemoryForImage(VmaAllocator allocator,VkImage image,const VmaAllocationCreateInfo * pAllocationCreateInfo,VmaSuballocationType suballocType,VmaAllocation * pAllocation)8543 static VkResult AllocateMemoryForImage(
8544 VmaAllocator allocator,
8545 VkImage image,
8546 const VmaAllocationCreateInfo* pAllocationCreateInfo,
8547 VmaSuballocationType suballocType,
8548 VmaAllocation* pAllocation)
8549 {
8550 VMA_ASSERT(allocator && (image != VK_NULL_HANDLE) && pAllocationCreateInfo && pAllocation);
8551
8552 VkMemoryRequirements vkMemReq = {};
8553 bool requiresDedicatedAllocation = false;
8554 bool prefersDedicatedAllocation = false;
8555 allocator->GetImageMemoryRequirements(image, vkMemReq,
8556 requiresDedicatedAllocation, prefersDedicatedAllocation);
8557
8558 return allocator->AllocateMemory(
8559 vkMemReq,
8560 requiresDedicatedAllocation,
8561 prefersDedicatedAllocation,
8562 VK_NULL_HANDLE, // dedicatedBuffer
8563 image, // dedicatedImage
8564 *pAllocationCreateInfo,
8565 suballocType,
8566 pAllocation);
8567 }
8568
8569 ////////////////////////////////////////////////////////////////////////////////
8570 // Public interface
8571
vmaCreateAllocator(const VmaAllocatorCreateInfo * pCreateInfo,VmaAllocator * pAllocator)8572 VkResult vmaCreateAllocator(
8573 const VmaAllocatorCreateInfo* pCreateInfo,
8574 VmaAllocator* pAllocator)
8575 {
8576 VMA_ASSERT(pCreateInfo && pAllocator);
8577 VMA_DEBUG_LOG("vmaCreateAllocator");
8578 *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo);
8579 return VK_SUCCESS;
8580 }
8581
vmaDestroyAllocator(VmaAllocator allocator)8582 void vmaDestroyAllocator(
8583 VmaAllocator allocator)
8584 {
8585 if(allocator != VK_NULL_HANDLE)
8586 {
8587 VMA_DEBUG_LOG("vmaDestroyAllocator");
8588 VkAllocationCallbacks allocationCallbacks = allocator->m_AllocationCallbacks;
8589 vma_delete(&allocationCallbacks, allocator);
8590 }
8591 }
8592
vmaGetPhysicalDeviceProperties(VmaAllocator allocator,const VkPhysicalDeviceProperties ** ppPhysicalDeviceProperties)8593 void vmaGetPhysicalDeviceProperties(
8594 VmaAllocator allocator,
8595 const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties)
8596 {
8597 VMA_ASSERT(allocator && ppPhysicalDeviceProperties);
8598 *ppPhysicalDeviceProperties = &allocator->m_PhysicalDeviceProperties;
8599 }
8600
vmaGetMemoryProperties(VmaAllocator allocator,const VkPhysicalDeviceMemoryProperties ** ppPhysicalDeviceMemoryProperties)8601 void vmaGetMemoryProperties(
8602 VmaAllocator allocator,
8603 const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties)
8604 {
8605 VMA_ASSERT(allocator && ppPhysicalDeviceMemoryProperties);
8606 *ppPhysicalDeviceMemoryProperties = &allocator->m_MemProps;
8607 }
8608
vmaGetMemoryTypeProperties(VmaAllocator allocator,uint32_t memoryTypeIndex,VkMemoryPropertyFlags * pFlags)8609 void vmaGetMemoryTypeProperties(
8610 VmaAllocator allocator,
8611 uint32_t memoryTypeIndex,
8612 VkMemoryPropertyFlags* pFlags)
8613 {
8614 VMA_ASSERT(allocator && pFlags);
8615 VMA_ASSERT(memoryTypeIndex < allocator->GetMemoryTypeCount());
8616 *pFlags = allocator->m_MemProps.memoryTypes[memoryTypeIndex].propertyFlags;
8617 }
8618
vmaSetCurrentFrameIndex(VmaAllocator allocator,uint32_t frameIndex)8619 void vmaSetCurrentFrameIndex(
8620 VmaAllocator allocator,
8621 uint32_t frameIndex)
8622 {
8623 VMA_ASSERT(allocator);
8624 VMA_ASSERT(frameIndex != VMA_FRAME_INDEX_LOST);
8625
8626 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8627
8628 allocator->SetCurrentFrameIndex(frameIndex);
8629 }
8630
vmaCalculateStats(VmaAllocator allocator,VmaStats * pStats)8631 void vmaCalculateStats(
8632 VmaAllocator allocator,
8633 VmaStats* pStats)
8634 {
8635 VMA_ASSERT(allocator && pStats);
8636 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8637 allocator->CalculateStats(pStats);
8638 }
8639
8640 #if VMA_STATS_STRING_ENABLED
8641
vmaBuildStatsString(VmaAllocator allocator,char ** ppStatsString,VkBool32 detailedMap)8642 void vmaBuildStatsString(
8643 VmaAllocator allocator,
8644 char** ppStatsString,
8645 VkBool32 detailedMap)
8646 {
8647 VMA_ASSERT(allocator && ppStatsString);
8648 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8649
8650 VmaStringBuilder sb(allocator);
8651 {
8652 VmaJsonWriter json(allocator->GetAllocationCallbacks(), sb);
8653 json.BeginObject();
8654
8655 VmaStats stats;
8656 allocator->CalculateStats(&stats);
8657
8658 json.WriteString("Total");
8659 VmaPrintStatInfo(json, stats.total);
8660
8661 for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex)
8662 {
8663 json.BeginString("Heap ");
8664 json.ContinueString(heapIndex);
8665 json.EndString();
8666 json.BeginObject();
8667
8668 json.WriteString("Size");
8669 json.WriteNumber(allocator->m_MemProps.memoryHeaps[heapIndex].size);
8670
8671 json.WriteString("Flags");
8672 json.BeginArray(true);
8673 if((allocator->m_MemProps.memoryHeaps[heapIndex].flags & VK_MEMORY_HEAP_DEVICE_LOCAL_BIT) != 0)
8674 {
8675 json.WriteString("DEVICE_LOCAL");
8676 }
8677 json.EndArray();
8678
8679 if(stats.memoryHeap[heapIndex].blockCount > 0)
8680 {
8681 json.WriteString("Stats");
8682 VmaPrintStatInfo(json, stats.memoryHeap[heapIndex]);
8683 }
8684
8685 for(uint32_t typeIndex = 0; typeIndex < allocator->GetMemoryTypeCount(); ++typeIndex)
8686 {
8687 if(allocator->MemoryTypeIndexToHeapIndex(typeIndex) == heapIndex)
8688 {
8689 json.BeginString("Type ");
8690 json.ContinueString(typeIndex);
8691 json.EndString();
8692
8693 json.BeginObject();
8694
8695 json.WriteString("Flags");
8696 json.BeginArray(true);
8697 VkMemoryPropertyFlags flags = allocator->m_MemProps.memoryTypes[typeIndex].propertyFlags;
8698 if((flags & VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT) != 0)
8699 {
8700 json.WriteString("DEVICE_LOCAL");
8701 }
8702 if((flags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0)
8703 {
8704 json.WriteString("HOST_VISIBLE");
8705 }
8706 if((flags & VK_MEMORY_PROPERTY_HOST_COHERENT_BIT) != 0)
8707 {
8708 json.WriteString("HOST_COHERENT");
8709 }
8710 if((flags & VK_MEMORY_PROPERTY_HOST_CACHED_BIT) != 0)
8711 {
8712 json.WriteString("HOST_CACHED");
8713 }
8714 if((flags & VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT) != 0)
8715 {
8716 json.WriteString("LAZILY_ALLOCATED");
8717 }
8718 json.EndArray();
8719
8720 if(stats.memoryType[typeIndex].blockCount > 0)
8721 {
8722 json.WriteString("Stats");
8723 VmaPrintStatInfo(json, stats.memoryType[typeIndex]);
8724 }
8725
8726 json.EndObject();
8727 }
8728 }
8729
8730 json.EndObject();
8731 }
8732 if(detailedMap == VK_TRUE)
8733 {
8734 allocator->PrintDetailedMap(json);
8735 }
8736
8737 json.EndObject();
8738 }
8739
8740 const size_t len = sb.GetLength();
8741 char* const pChars = vma_new_array(allocator, char, len + 1);
8742 if(len > 0)
8743 {
8744 memcpy(pChars, sb.GetData(), len);
8745 }
8746 pChars[len] = '\0';
8747 *ppStatsString = pChars;
8748 }
8749
vmaFreeStatsString(VmaAllocator allocator,char * pStatsString)8750 void vmaFreeStatsString(
8751 VmaAllocator allocator,
8752 char* pStatsString)
8753 {
8754 if(pStatsString != VMA_NULL)
8755 {
8756 VMA_ASSERT(allocator);
8757 size_t len = strlen(pStatsString);
8758 vma_delete_array(allocator, pStatsString, len + 1);
8759 }
8760 }
8761
8762 #endif // #if VMA_STATS_STRING_ENABLED
8763
8764 /*
8765 This function is not protected by any mutex because it just reads immutable data.
8766 */
vmaFindMemoryTypeIndex(VmaAllocator allocator,uint32_t memoryTypeBits,const VmaAllocationCreateInfo * pAllocationCreateInfo,uint32_t * pMemoryTypeIndex)8767 VkResult vmaFindMemoryTypeIndex(
8768 VmaAllocator allocator,
8769 uint32_t memoryTypeBits,
8770 const VmaAllocationCreateInfo* pAllocationCreateInfo,
8771 uint32_t* pMemoryTypeIndex)
8772 {
8773 VMA_ASSERT(allocator != VK_NULL_HANDLE);
8774 VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8775 VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8776
8777 if(pAllocationCreateInfo->memoryTypeBits != 0)
8778 {
8779 memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits;
8780 }
8781
8782 uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags;
8783 uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags;
8784
8785 // Convert usage to requiredFlags and preferredFlags.
8786 switch(pAllocationCreateInfo->usage)
8787 {
8788 case VMA_MEMORY_USAGE_UNKNOWN:
8789 break;
8790 case VMA_MEMORY_USAGE_GPU_ONLY:
8791 preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8792 break;
8793 case VMA_MEMORY_USAGE_CPU_ONLY:
8794 requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_COHERENT_BIT;
8795 break;
8796 case VMA_MEMORY_USAGE_CPU_TO_GPU:
8797 requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8798 preferredFlags |= VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT;
8799 break;
8800 case VMA_MEMORY_USAGE_GPU_TO_CPU:
8801 requiredFlags |= VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT;
8802 preferredFlags |= VK_MEMORY_PROPERTY_HOST_COHERENT_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT;
8803 break;
8804 default:
8805 break;
8806 }
8807
8808 *pMemoryTypeIndex = UINT32_MAX;
8809 uint32_t minCost = UINT32_MAX;
8810 for(uint32_t memTypeIndex = 0, memTypeBit = 1;
8811 memTypeIndex < allocator->GetMemoryTypeCount();
8812 ++memTypeIndex, memTypeBit <<= 1)
8813 {
8814 // This memory type is acceptable according to memoryTypeBits bitmask.
8815 if((memTypeBit & memoryTypeBits) != 0)
8816 {
8817 const VkMemoryPropertyFlags currFlags =
8818 allocator->m_MemProps.memoryTypes[memTypeIndex].propertyFlags;
8819 // This memory type contains requiredFlags.
8820 if((requiredFlags & ~currFlags) == 0)
8821 {
8822 // Calculate cost as number of bits from preferredFlags not present in this memory type.
8823 uint32_t currCost = VmaCountBitsSet(preferredFlags & ~currFlags);
8824 // Remember memory type with lowest cost.
8825 if(currCost < minCost)
8826 {
8827 *pMemoryTypeIndex = memTypeIndex;
8828 if(currCost == 0)
8829 {
8830 return VK_SUCCESS;
8831 }
8832 minCost = currCost;
8833 }
8834 }
8835 }
8836 }
8837 return (*pMemoryTypeIndex != UINT32_MAX) ? VK_SUCCESS : VK_ERROR_FEATURE_NOT_PRESENT;
8838 }
8839
vmaFindMemoryTypeIndexForBufferInfo(VmaAllocator allocator,const VkBufferCreateInfo * pBufferCreateInfo,const VmaAllocationCreateInfo * pAllocationCreateInfo,uint32_t * pMemoryTypeIndex)8840 VkResult vmaFindMemoryTypeIndexForBufferInfo(
8841 VmaAllocator allocator,
8842 const VkBufferCreateInfo* pBufferCreateInfo,
8843 const VmaAllocationCreateInfo* pAllocationCreateInfo,
8844 uint32_t* pMemoryTypeIndex)
8845 {
8846 VMA_ASSERT(allocator != VK_NULL_HANDLE);
8847 VMA_ASSERT(pBufferCreateInfo != VMA_NULL);
8848 VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8849 VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8850
8851 const VkDevice hDev = allocator->m_hDevice;
8852 VkBuffer hBuffer = VK_NULL_HANDLE;
8853 VkResult res = allocator->GetVulkanFunctions().vkCreateBuffer(
8854 hDev, pBufferCreateInfo, allocator->GetAllocationCallbacks(), &hBuffer);
8855 if(res == VK_SUCCESS)
8856 {
8857 VkMemoryRequirements memReq = {};
8858 allocator->GetVulkanFunctions().vkGetBufferMemoryRequirements(
8859 hDev, hBuffer, &memReq);
8860
8861 res = vmaFindMemoryTypeIndex(
8862 allocator,
8863 memReq.memoryTypeBits,
8864 pAllocationCreateInfo,
8865 pMemoryTypeIndex);
8866
8867 allocator->GetVulkanFunctions().vkDestroyBuffer(
8868 hDev, hBuffer, allocator->GetAllocationCallbacks());
8869 }
8870 return res;
8871 }
8872
vmaFindMemoryTypeIndexForImageInfo(VmaAllocator allocator,const VkImageCreateInfo * pImageCreateInfo,const VmaAllocationCreateInfo * pAllocationCreateInfo,uint32_t * pMemoryTypeIndex)8873 VkResult vmaFindMemoryTypeIndexForImageInfo(
8874 VmaAllocator allocator,
8875 const VkImageCreateInfo* pImageCreateInfo,
8876 const VmaAllocationCreateInfo* pAllocationCreateInfo,
8877 uint32_t* pMemoryTypeIndex)
8878 {
8879 VMA_ASSERT(allocator != VK_NULL_HANDLE);
8880 VMA_ASSERT(pImageCreateInfo != VMA_NULL);
8881 VMA_ASSERT(pAllocationCreateInfo != VMA_NULL);
8882 VMA_ASSERT(pMemoryTypeIndex != VMA_NULL);
8883
8884 const VkDevice hDev = allocator->m_hDevice;
8885 VkImage hImage = VK_NULL_HANDLE;
8886 VkResult res = allocator->GetVulkanFunctions().vkCreateImage(
8887 hDev, pImageCreateInfo, allocator->GetAllocationCallbacks(), &hImage);
8888 if(res == VK_SUCCESS)
8889 {
8890 VkMemoryRequirements memReq = {};
8891 allocator->GetVulkanFunctions().vkGetImageMemoryRequirements(
8892 hDev, hImage, &memReq);
8893
8894 res = vmaFindMemoryTypeIndex(
8895 allocator,
8896 memReq.memoryTypeBits,
8897 pAllocationCreateInfo,
8898 pMemoryTypeIndex);
8899
8900 allocator->GetVulkanFunctions().vkDestroyImage(
8901 hDev, hImage, allocator->GetAllocationCallbacks());
8902 }
8903 return res;
8904 }
8905
vmaCreatePool(VmaAllocator allocator,const VmaPoolCreateInfo * pCreateInfo,VmaPool * pPool)8906 VkResult vmaCreatePool(
8907 VmaAllocator allocator,
8908 const VmaPoolCreateInfo* pCreateInfo,
8909 VmaPool* pPool)
8910 {
8911 VMA_ASSERT(allocator && pCreateInfo && pPool);
8912
8913 VMA_DEBUG_LOG("vmaCreatePool");
8914
8915 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8916
8917 return allocator->CreatePool(pCreateInfo, pPool);
8918 }
8919
vmaDestroyPool(VmaAllocator allocator,VmaPool pool)8920 void vmaDestroyPool(
8921 VmaAllocator allocator,
8922 VmaPool pool)
8923 {
8924 VMA_ASSERT(allocator);
8925
8926 if(pool == VK_NULL_HANDLE)
8927 {
8928 return;
8929 }
8930
8931 VMA_DEBUG_LOG("vmaDestroyPool");
8932
8933 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8934
8935 allocator->DestroyPool(pool);
8936 }
8937
vmaGetPoolStats(VmaAllocator allocator,VmaPool pool,VmaPoolStats * pPoolStats)8938 void vmaGetPoolStats(
8939 VmaAllocator allocator,
8940 VmaPool pool,
8941 VmaPoolStats* pPoolStats)
8942 {
8943 VMA_ASSERT(allocator && pool && pPoolStats);
8944
8945 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8946
8947 allocator->GetPoolStats(pool, pPoolStats);
8948 }
8949
vmaMakePoolAllocationsLost(VmaAllocator allocator,VmaPool pool,size_t * pLostAllocationCount)8950 void vmaMakePoolAllocationsLost(
8951 VmaAllocator allocator,
8952 VmaPool pool,
8953 size_t* pLostAllocationCount)
8954 {
8955 VMA_ASSERT(allocator && pool);
8956
8957 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8958
8959 allocator->MakePoolAllocationsLost(pool, pLostAllocationCount);
8960 }
8961
vmaAllocateMemory(VmaAllocator allocator,const VkMemoryRequirements * pVkMemoryRequirements,const VmaAllocationCreateInfo * pCreateInfo,VmaAllocation * pAllocation,VmaAllocationInfo * pAllocationInfo)8962 VkResult vmaAllocateMemory(
8963 VmaAllocator allocator,
8964 const VkMemoryRequirements* pVkMemoryRequirements,
8965 const VmaAllocationCreateInfo* pCreateInfo,
8966 VmaAllocation* pAllocation,
8967 VmaAllocationInfo* pAllocationInfo)
8968 {
8969 VMA_ASSERT(allocator && pVkMemoryRequirements && pCreateInfo && pAllocation);
8970
8971 VMA_DEBUG_LOG("vmaAllocateMemory");
8972
8973 VMA_DEBUG_GLOBAL_MUTEX_LOCK
8974
8975 VkResult result = allocator->AllocateMemory(
8976 *pVkMemoryRequirements,
8977 false, // requiresDedicatedAllocation
8978 false, // prefersDedicatedAllocation
8979 VK_NULL_HANDLE, // dedicatedBuffer
8980 VK_NULL_HANDLE, // dedicatedImage
8981 *pCreateInfo,
8982 VMA_SUBALLOCATION_TYPE_UNKNOWN,
8983 pAllocation);
8984
8985 if(pAllocationInfo && result == VK_SUCCESS)
8986 {
8987 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
8988 }
8989
8990 return result;
8991 }
8992
vmaAllocateMemoryForBuffer(VmaAllocator allocator,VkBuffer buffer,const VmaAllocationCreateInfo * pCreateInfo,VmaAllocation * pAllocation,VmaAllocationInfo * pAllocationInfo)8993 VkResult vmaAllocateMemoryForBuffer(
8994 VmaAllocator allocator,
8995 VkBuffer buffer,
8996 const VmaAllocationCreateInfo* pCreateInfo,
8997 VmaAllocation* pAllocation,
8998 VmaAllocationInfo* pAllocationInfo)
8999 {
9000 VMA_ASSERT(allocator && buffer != VK_NULL_HANDLE && pCreateInfo && pAllocation);
9001
9002 VMA_DEBUG_LOG("vmaAllocateMemoryForBuffer");
9003
9004 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9005
9006 VkMemoryRequirements vkMemReq = {};
9007 bool requiresDedicatedAllocation = false;
9008 bool prefersDedicatedAllocation = false;
9009 allocator->GetBufferMemoryRequirements(buffer, vkMemReq,
9010 requiresDedicatedAllocation,
9011 prefersDedicatedAllocation);
9012
9013 VkResult result = allocator->AllocateMemory(
9014 vkMemReq,
9015 requiresDedicatedAllocation,
9016 prefersDedicatedAllocation,
9017 buffer, // dedicatedBuffer
9018 VK_NULL_HANDLE, // dedicatedImage
9019 *pCreateInfo,
9020 VMA_SUBALLOCATION_TYPE_BUFFER,
9021 pAllocation);
9022
9023 if(pAllocationInfo && result == VK_SUCCESS)
9024 {
9025 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
9026 }
9027
9028 return result;
9029 }
9030
vmaAllocateMemoryForImage(VmaAllocator allocator,VkImage image,const VmaAllocationCreateInfo * pCreateInfo,VmaAllocation * pAllocation,VmaAllocationInfo * pAllocationInfo)9031 VkResult vmaAllocateMemoryForImage(
9032 VmaAllocator allocator,
9033 VkImage image,
9034 const VmaAllocationCreateInfo* pCreateInfo,
9035 VmaAllocation* pAllocation,
9036 VmaAllocationInfo* pAllocationInfo)
9037 {
9038 VMA_ASSERT(allocator && image != VK_NULL_HANDLE && pCreateInfo && pAllocation);
9039
9040 VMA_DEBUG_LOG("vmaAllocateMemoryForImage");
9041
9042 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9043
9044 VkResult result = AllocateMemoryForImage(
9045 allocator,
9046 image,
9047 pCreateInfo,
9048 VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN,
9049 pAllocation);
9050
9051 if(pAllocationInfo && result == VK_SUCCESS)
9052 {
9053 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
9054 }
9055
9056 return result;
9057 }
9058
vmaFreeMemory(VmaAllocator allocator,VmaAllocation allocation)9059 void vmaFreeMemory(
9060 VmaAllocator allocator,
9061 VmaAllocation allocation)
9062 {
9063 VMA_ASSERT(allocator && allocation);
9064
9065 VMA_DEBUG_LOG("vmaFreeMemory");
9066
9067 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9068
9069 allocator->FreeMemory(allocation);
9070 }
9071
vmaGetAllocationInfo(VmaAllocator allocator,VmaAllocation allocation,VmaAllocationInfo * pAllocationInfo)9072 void vmaGetAllocationInfo(
9073 VmaAllocator allocator,
9074 VmaAllocation allocation,
9075 VmaAllocationInfo* pAllocationInfo)
9076 {
9077 VMA_ASSERT(allocator && allocation && pAllocationInfo);
9078
9079 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9080
9081 allocator->GetAllocationInfo(allocation, pAllocationInfo);
9082 }
9083
vmaTouchAllocation(VmaAllocator allocator,VmaAllocation allocation)9084 VkBool32 vmaTouchAllocation(
9085 VmaAllocator allocator,
9086 VmaAllocation allocation)
9087 {
9088 VMA_ASSERT(allocator && allocation);
9089
9090 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9091
9092 return allocator->TouchAllocation(allocation);
9093 }
9094
vmaSetAllocationUserData(VmaAllocator allocator,VmaAllocation allocation,void * pUserData)9095 void vmaSetAllocationUserData(
9096 VmaAllocator allocator,
9097 VmaAllocation allocation,
9098 void* pUserData)
9099 {
9100 VMA_ASSERT(allocator && allocation);
9101
9102 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9103
9104 allocation->SetUserData(allocator, pUserData);
9105 }
9106
vmaCreateLostAllocation(VmaAllocator allocator,VmaAllocation * pAllocation)9107 void vmaCreateLostAllocation(
9108 VmaAllocator allocator,
9109 VmaAllocation* pAllocation)
9110 {
9111 VMA_ASSERT(allocator && pAllocation);
9112
9113 VMA_DEBUG_GLOBAL_MUTEX_LOCK;
9114
9115 allocator->CreateLostAllocation(pAllocation);
9116 }
9117
vmaMapMemory(VmaAllocator allocator,VmaAllocation allocation,void ** ppData)9118 VkResult vmaMapMemory(
9119 VmaAllocator allocator,
9120 VmaAllocation allocation,
9121 void** ppData)
9122 {
9123 VMA_ASSERT(allocator && allocation && ppData);
9124
9125 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9126
9127 return allocator->Map(allocation, ppData);
9128 }
9129
vmaUnmapMemory(VmaAllocator allocator,VmaAllocation allocation)9130 void vmaUnmapMemory(
9131 VmaAllocator allocator,
9132 VmaAllocation allocation)
9133 {
9134 VMA_ASSERT(allocator && allocation);
9135
9136 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9137
9138 allocator->Unmap(allocation);
9139 }
9140
vmaDefragment(VmaAllocator allocator,VmaAllocation * pAllocations,size_t allocationCount,VkBool32 * pAllocationsChanged,const VmaDefragmentationInfo * pDefragmentationInfo,VmaDefragmentationStats * pDefragmentationStats)9141 VkResult vmaDefragment(
9142 VmaAllocator allocator,
9143 VmaAllocation* pAllocations,
9144 size_t allocationCount,
9145 VkBool32* pAllocationsChanged,
9146 const VmaDefragmentationInfo *pDefragmentationInfo,
9147 VmaDefragmentationStats* pDefragmentationStats)
9148 {
9149 VMA_ASSERT(allocator && pAllocations);
9150
9151 VMA_DEBUG_LOG("vmaDefragment");
9152
9153 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9154
9155 return allocator->Defragment(pAllocations, allocationCount, pAllocationsChanged, pDefragmentationInfo, pDefragmentationStats);
9156 }
9157
vmaBindBufferMemory(VmaAllocator allocator,VmaAllocation allocation,VkBuffer buffer)9158 VkResult vmaBindBufferMemory(
9159 VmaAllocator allocator,
9160 VmaAllocation allocation,
9161 VkBuffer buffer)
9162 {
9163 VMA_ASSERT(allocator && allocation && buffer);
9164
9165 VMA_DEBUG_LOG("vmaBindBufferMemory");
9166
9167 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9168
9169 return allocator->BindBufferMemory(allocation, buffer);
9170 }
9171
vmaBindImageMemory(VmaAllocator allocator,VmaAllocation allocation,VkImage image)9172 VkResult vmaBindImageMemory(
9173 VmaAllocator allocator,
9174 VmaAllocation allocation,
9175 VkImage image)
9176 {
9177 VMA_ASSERT(allocator && allocation && image);
9178
9179 VMA_DEBUG_LOG("vmaBindImageMemory");
9180
9181 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9182
9183 return allocator->BindImageMemory(allocation, image);
9184 }
9185
vmaCreateBuffer(VmaAllocator allocator,const VkBufferCreateInfo * pBufferCreateInfo,const VmaAllocationCreateInfo * pAllocationCreateInfo,VkBuffer * pBuffer,VmaAllocation * pAllocation,VmaAllocationInfo * pAllocationInfo)9186 VkResult vmaCreateBuffer(
9187 VmaAllocator allocator,
9188 const VkBufferCreateInfo* pBufferCreateInfo,
9189 const VmaAllocationCreateInfo* pAllocationCreateInfo,
9190 VkBuffer* pBuffer,
9191 VmaAllocation* pAllocation,
9192 VmaAllocationInfo* pAllocationInfo)
9193 {
9194 VMA_ASSERT(allocator && pBufferCreateInfo && pAllocationCreateInfo && pBuffer && pAllocation);
9195
9196 VMA_DEBUG_LOG("vmaCreateBuffer");
9197
9198 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9199
9200 *pBuffer = VK_NULL_HANDLE;
9201 *pAllocation = VK_NULL_HANDLE;
9202
9203 // 1. Create VkBuffer.
9204 VkResult res = (*allocator->GetVulkanFunctions().vkCreateBuffer)(
9205 allocator->m_hDevice,
9206 pBufferCreateInfo,
9207 allocator->GetAllocationCallbacks(),
9208 pBuffer);
9209 if(res >= 0)
9210 {
9211 // 2. vkGetBufferMemoryRequirements.
9212 VkMemoryRequirements vkMemReq = {};
9213 bool requiresDedicatedAllocation = false;
9214 bool prefersDedicatedAllocation = false;
9215 allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq,
9216 requiresDedicatedAllocation, prefersDedicatedAllocation);
9217
9218 // Make sure alignment requirements for specific buffer usages reported
9219 // in Physical Device Properties are included in alignment reported by memory requirements.
9220 if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0)
9221 {
9222 VMA_ASSERT(vkMemReq.alignment %
9223 allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0);
9224 }
9225 if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0)
9226 {
9227 VMA_ASSERT(vkMemReq.alignment %
9228 allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0);
9229 }
9230 if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0)
9231 {
9232 VMA_ASSERT(vkMemReq.alignment %
9233 allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0);
9234 }
9235
9236 // 3. Allocate memory using allocator.
9237 res = allocator->AllocateMemory(
9238 vkMemReq,
9239 requiresDedicatedAllocation,
9240 prefersDedicatedAllocation,
9241 *pBuffer, // dedicatedBuffer
9242 VK_NULL_HANDLE, // dedicatedImage
9243 *pAllocationCreateInfo,
9244 VMA_SUBALLOCATION_TYPE_BUFFER,
9245 pAllocation);
9246 if(res >= 0)
9247 {
9248 // 3. Bind buffer with memory.
9249 res = allocator->BindBufferMemory(*pAllocation, *pBuffer);
9250 if(res >= 0)
9251 {
9252 // All steps succeeded.
9253 if(pAllocationInfo != VMA_NULL)
9254 {
9255 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
9256 }
9257 return VK_SUCCESS;
9258 }
9259 allocator->FreeMemory(*pAllocation);
9260 *pAllocation = VK_NULL_HANDLE;
9261 (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
9262 *pBuffer = VK_NULL_HANDLE;
9263 return res;
9264 }
9265 (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, *pBuffer, allocator->GetAllocationCallbacks());
9266 *pBuffer = VK_NULL_HANDLE;
9267 return res;
9268 }
9269 return res;
9270 }
9271
vmaDestroyBuffer(VmaAllocator allocator,VkBuffer buffer,VmaAllocation allocation)9272 void vmaDestroyBuffer(
9273 VmaAllocator allocator,
9274 VkBuffer buffer,
9275 VmaAllocation allocation)
9276 {
9277 if(buffer != VK_NULL_HANDLE)
9278 {
9279 VMA_ASSERT(allocator);
9280
9281 VMA_DEBUG_LOG("vmaDestroyBuffer");
9282
9283 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9284
9285 (*allocator->GetVulkanFunctions().vkDestroyBuffer)(allocator->m_hDevice, buffer, allocator->GetAllocationCallbacks());
9286
9287 allocator->FreeMemory(allocation);
9288 }
9289 }
9290
vmaCreateImage(VmaAllocator allocator,const VkImageCreateInfo * pImageCreateInfo,const VmaAllocationCreateInfo * pAllocationCreateInfo,VkImage * pImage,VmaAllocation * pAllocation,VmaAllocationInfo * pAllocationInfo)9291 VkResult vmaCreateImage(
9292 VmaAllocator allocator,
9293 const VkImageCreateInfo* pImageCreateInfo,
9294 const VmaAllocationCreateInfo* pAllocationCreateInfo,
9295 VkImage* pImage,
9296 VmaAllocation* pAllocation,
9297 VmaAllocationInfo* pAllocationInfo)
9298 {
9299 VMA_ASSERT(allocator && pImageCreateInfo && pAllocationCreateInfo && pImage && pAllocation);
9300
9301 VMA_DEBUG_LOG("vmaCreateImage");
9302
9303 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9304
9305 *pImage = VK_NULL_HANDLE;
9306 *pAllocation = VK_NULL_HANDLE;
9307
9308 // 1. Create VkImage.
9309 VkResult res = (*allocator->GetVulkanFunctions().vkCreateImage)(
9310 allocator->m_hDevice,
9311 pImageCreateInfo,
9312 allocator->GetAllocationCallbacks(),
9313 pImage);
9314 if(res >= 0)
9315 {
9316 VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ?
9317 VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL :
9318 VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR;
9319
9320 // 2. Allocate memory using allocator.
9321 res = AllocateMemoryForImage(allocator, *pImage, pAllocationCreateInfo, suballocType, pAllocation);
9322 if(res >= 0)
9323 {
9324 // 3. Bind image with memory.
9325 res = allocator->BindImageMemory(*pAllocation, *pImage);
9326 if(res >= 0)
9327 {
9328 // All steps succeeded.
9329 if(pAllocationInfo != VMA_NULL)
9330 {
9331 allocator->GetAllocationInfo(*pAllocation, pAllocationInfo);
9332 }
9333 return VK_SUCCESS;
9334 }
9335 allocator->FreeMemory(*pAllocation);
9336 *pAllocation = VK_NULL_HANDLE;
9337 (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
9338 *pImage = VK_NULL_HANDLE;
9339 return res;
9340 }
9341 (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, *pImage, allocator->GetAllocationCallbacks());
9342 *pImage = VK_NULL_HANDLE;
9343 return res;
9344 }
9345 return res;
9346 }
9347
vmaDestroyImage(VmaAllocator allocator,VkImage image,VmaAllocation allocation)9348 void vmaDestroyImage(
9349 VmaAllocator allocator,
9350 VkImage image,
9351 VmaAllocation allocation)
9352 {
9353 if(image != VK_NULL_HANDLE)
9354 {
9355 VMA_ASSERT(allocator);
9356
9357 VMA_DEBUG_LOG("vmaDestroyImage");
9358
9359 VMA_DEBUG_GLOBAL_MUTEX_LOCK
9360
9361 (*allocator->GetVulkanFunctions().vkDestroyImage)(allocator->m_hDevice, image, allocator->GetAllocationCallbacks());
9362
9363 allocator->FreeMemory(allocation);
9364 }
9365 }
9366
9367 #endif // #ifdef VMA_IMPLEMENTATION
9368