1 //
2 // Copyright 2020 Pixar
3 //
4 // Licensed under the Apache License, Version 2.0 (the "Apache License")
5 // with the following modification; you may not use this file except in
6 // compliance with the Apache License and the following modification to it:
7 // Section 6. Trademarks. is deleted and replaced with:
8 //
9 // 6. Trademarks. This License does not grant permission to use the trade
10 // names, trademarks, service marks, or product names of the Licensor
11 // and its affiliates, except as required to comply with Section 4(c) of
12 // the License and to reproduce the content of the NOTICE file.
13 //
14 // You may obtain a copy of the Apache License at
15 //
16 // http://www.apache.org/licenses/LICENSE-2.0
17 //
18 // Unless required by applicable law or agreed to in writing, software
19 // distributed under the Apache License with the above modification is
20 // distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21 // KIND, either express or implied. See the Apache License for the specific
22 // language governing permissions and limitations under the Apache License.
23 //
24 #include "pxr/imaging/hgiVulkan/capabilities.h"
25 #include "pxr/imaging/hgiVulkan/commandQueue.h"
26 #include "pxr/imaging/hgiVulkan/device.h"
27 #include "pxr/imaging/hgiVulkan/diagnostic.h"
28 #include "pxr/imaging/hgiVulkan/hgi.h"
29 #include "pxr/imaging/hgiVulkan/instance.h"
30 #include "pxr/imaging/hgiVulkan/pipelineCache.h"
31
32 #include "pxr/base/tf/diagnostic.h"
33
34 #define VMA_IMPLEMENTATION
35 #include "pxr/imaging/hgiVulkan/vk_mem_alloc.h"
36 #undef VMA_IMPLEMENTATION
37
38 PXR_NAMESPACE_OPEN_SCOPE
39
40
41 static uint32_t
_GetGraphicsQueueFamilyIndex(VkPhysicalDevice physicalDevice)42 _GetGraphicsQueueFamilyIndex(VkPhysicalDevice physicalDevice)
43 {
44 uint32_t queueCount = 0;
45 vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, &queueCount, 0);
46
47 std::vector<VkQueueFamilyProperties> queues(queueCount);
48 vkGetPhysicalDeviceQueueFamilyProperties(
49 physicalDevice,
50 &queueCount,
51 queues.data());
52
53 for (uint32_t i = 0; i < queueCount; i++) {
54 if (queues[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) {
55 return i;
56 }
57 }
58
59 return VK_QUEUE_FAMILY_IGNORED;
60 }
61
62 static bool
_SupportsPresentation(VkPhysicalDevice physicalDevice,uint32_t familyIndex)63 _SupportsPresentation(
64 VkPhysicalDevice physicalDevice,
65 uint32_t familyIndex)
66 {
67 #if defined(VK_USE_PLATFORM_WIN32_KHR)
68 return vkGetPhysicalDeviceWin32PresentationSupportKHR(
69 physicalDevice, familyIndex);
70 #elif defined(VK_USE_PLATFORM_XLIB_KHR)
71 Display* dsp = XOpenDisplay(nullptr);
72 VisualID visualID = XVisualIDFromVisual(
73 DefaultVisual(dsp, DefaultScreen(dsp)));
74 return vkGetPhysicalDeviceXlibPresentationSupportKHR(
75 physicalDevice, familyIndex, dsp, visualID);
76 #elif defined(VK_USE_PLATFORM_MACOS_MVK)
77 // Presentation currently always supported on Metal / MoltenVk
78 return true;
79 #else
80 #error Unsupported Platform
81 return true;
82 #endif
83 }
84
HgiVulkanDevice(HgiVulkanInstance * instance)85 HgiVulkanDevice::HgiVulkanDevice(HgiVulkanInstance* instance)
86 : _vkPhysicalDevice(nullptr)
87 , _vkDevice(nullptr)
88 , _vmaAllocator(nullptr)
89 , _commandQueue(nullptr)
90 , _capabilities(nullptr)
91 {
92 //
93 // Determine physical device
94 //
95
96 const uint32_t maxDevices = 64;
97 VkPhysicalDevice physicalDevices[maxDevices];
98 uint32_t physicalDeviceCount = maxDevices;
99 TF_VERIFY(
100 vkEnumeratePhysicalDevices(
101 instance->GetVulkanInstance(),
102 &physicalDeviceCount,
103 physicalDevices) == VK_SUCCESS
104 );
105
106 for (uint32_t i = 0; i < physicalDeviceCount; i++) {
107 VkPhysicalDeviceProperties props;
108 vkGetPhysicalDeviceProperties(physicalDevices[i], &props);
109
110 uint32_t familyIndex =
111 _GetGraphicsQueueFamilyIndex(physicalDevices[i]);
112
113 if (familyIndex == VK_QUEUE_FAMILY_IGNORED) continue;
114
115 // Assume we always want a presentation capable device for now.
116 if (!_SupportsPresentation(physicalDevices[i], familyIndex)) {
117 continue;
118 }
119
120 if (props.apiVersion < VK_API_VERSION_1_0) continue;
121
122 // Try to find a discrete device. Until we find a discrete device,
123 // store the first non-discrete device as fallback in case we never
124 // find a discrete device at all.
125 if (props.deviceType==VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) {
126 _vkPhysicalDevice = physicalDevices[i];
127 _vkGfxsQueueFamilyIndex = familyIndex;
128 break;
129 } else if (!_vkPhysicalDevice) {
130 _vkPhysicalDevice = physicalDevices[i];
131 _vkGfxsQueueFamilyIndex = familyIndex;
132 }
133 }
134
135 if (!_vkPhysicalDevice) {
136 TF_CODING_ERROR("VULKAN_ERROR: Unable to determine physical device");
137 return;
138 }
139
140 //
141 // Query supported extensions for device
142 //
143
144 uint32_t extensionCount = 0;
145 TF_VERIFY(
146 vkEnumerateDeviceExtensionProperties(
147 _vkPhysicalDevice,
148 nullptr,
149 &extensionCount,
150 nullptr) == VK_SUCCESS
151 );
152
153 _vkExtensions.resize(extensionCount);
154
155 TF_VERIFY(
156 vkEnumerateDeviceExtensionProperties(
157 _vkPhysicalDevice,
158 nullptr,
159 &extensionCount,
160 _vkExtensions.data()) == VK_SUCCESS
161 );
162
163 //
164 // Create Device
165 //
166 _capabilities = new HgiVulkanCapabilities(this);
167
168 VkDeviceQueueCreateInfo queueInfo =
169 {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
170 float queuePriorities[] = {1.0f};
171 queueInfo.queueFamilyIndex = _vkGfxsQueueFamilyIndex;
172 queueInfo.queueCount = 1;
173 queueInfo.pQueuePriorities = queuePriorities;
174
175 std::vector<const char*> extensions = {
176 VK_KHR_SWAPCHAIN_EXTENSION_NAME
177 };
178
179 // Allow certain buffers/images to have dedicated memory allocations to
180 // improve performance on some GPUs.
181 bool dedicatedAllocations = false;
182 if (_IsSupportedExtension(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME)
183 && _IsSupportedExtension(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME))
184 {
185 dedicatedAllocations = true;
186 extensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME);
187 extensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME);
188 }
189
190 // Allow OpenGL interop - Note requires two extensions in HgiVulkanInstance.
191 if (_IsSupportedExtension(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME) &&
192 _IsSupportedExtension(VK_KHR_EXTERNAL_SEMAPHORE_EXTENSION_NAME))
193 {
194 extensions.push_back(VK_KHR_EXTERNAL_SEMAPHORE_EXTENSION_NAME);
195 extensions.push_back(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME);
196 }
197
198 // Memory budget query extension
199 bool supportsMemExtension = false;
200 if (_IsSupportedExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) {
201 supportsMemExtension = true;
202 extensions.push_back(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME);
203 }
204
205 // Resolve depth during render pass resolve extension
206 if (_IsSupportedExtension(VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME)) {
207 extensions.push_back(VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME);
208 extensions.push_back(VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME);
209 extensions.push_back(VK_KHR_MULTIVIEW_EXTENSION_NAME);
210 extensions.push_back(VK_KHR_MAINTENANCE2_EXTENSION_NAME);
211 }
212
213 // Allows the same layout in structs between c++ and glsl (share structs).
214 // This means instead of 'std430' you can now use 'scalar'.
215 if (_IsSupportedExtension(VK_EXT_SCALAR_BLOCK_LAYOUT_EXTENSION_NAME)) {
216 extensions.push_back(VK_EXT_SCALAR_BLOCK_LAYOUT_EXTENSION_NAME);
217 } else {
218 TF_WARN("Unsupported VK_EXT_scalar_block_layout."
219 "Update gfx driver?");
220 }
221
222 // This extension is needed to allow the viewport to be flipped in Y so that
223 // shaders and vertex data can remain the same between opengl and vulkan.
224 extensions.push_back(VK_KHR_MAINTENANCE1_EXTENSION_NAME);
225
226 // Enabling certain features may incure a performance hit
227 // (e.g. robustBufferAccess), so only enable the features we will use.
228 VkPhysicalDeviceFeatures2 features =
229 {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2};
230
231 //extension features
232 features.pNext = _capabilities->vkDeviceFeatures2.pNext;
233
234 features.features.multiDrawIndirect =
235 _capabilities->vkDeviceFeatures.multiDrawIndirect;
236 features.features.samplerAnisotropy =
237 _capabilities->vkDeviceFeatures.samplerAnisotropy;
238 features.features.shaderSampledImageArrayDynamicIndexing =
239 _capabilities->vkDeviceFeatures.shaderSampledImageArrayDynamicIndexing;
240 features.features.shaderStorageImageArrayDynamicIndexing =
241 _capabilities->vkDeviceFeatures.shaderStorageImageArrayDynamicIndexing;
242 features.features.sampleRateShading =
243 _capabilities->vkDeviceFeatures.sampleRateShading;
244 features.features.shaderClipDistance =
245 _capabilities->vkDeviceFeatures.shaderClipDistance;
246 features.features.tessellationShader =
247 _capabilities->vkDeviceFeatures.tessellationShader;
248
249 // Needed to write to storage buffers from vertex shader (eg. GPU culling).
250 features.features.vertexPipelineStoresAndAtomics =
251 _capabilities->vkDeviceFeatures.vertexPipelineStoresAndAtomics;
252
253 #if !defined(VK_USE_PLATFORM_MACOS_MVK)
254 // Needed for buffer address feature
255 features.features.shaderInt64 =
256 _capabilities->vkDeviceFeatures.shaderInt64;
257 // Needed for gl_primtiveID
258 features.features.geometryShader =
259 _capabilities->vkDeviceFeatures.geometryShader;
260 #endif
261
262 VkDeviceCreateInfo createInfo = {VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
263 createInfo.queueCreateInfoCount = 1;
264 createInfo.pQueueCreateInfos = &queueInfo;
265 createInfo.ppEnabledExtensionNames = extensions.data();
266 createInfo.enabledExtensionCount = (uint32_t) extensions.size();
267 createInfo.pNext = &features;
268
269 TF_VERIFY(
270 vkCreateDevice(
271 _vkPhysicalDevice,
272 &createInfo,
273 HgiVulkanAllocator(),
274 &_vkDevice) == VK_SUCCESS
275 );
276
277 HgiVulkanSetupDeviceDebug(instance, this);
278
279 //
280 // Extension function pointers
281 //
282
283 vkCreateRenderPass2KHR = (PFN_vkCreateRenderPass2KHR)
284 vkGetDeviceProcAddr(_vkDevice, "vkCreateRenderPass2KHR");
285
286 //
287 // Memory allocator
288 //
289
290 VmaAllocatorCreateInfo allocatorInfo = {};
291 allocatorInfo.instance = instance->GetVulkanInstance();
292 allocatorInfo.physicalDevice = _vkPhysicalDevice;
293 allocatorInfo.device = _vkDevice;
294 if (dedicatedAllocations) {
295 allocatorInfo.flags |=VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT;
296 }
297
298 if (supportsMemExtension) {
299 allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
300 }
301
302 TF_VERIFY(
303 vmaCreateAllocator(&allocatorInfo, &_vmaAllocator) == VK_SUCCESS
304 );
305
306 //
307 // Command Queue
308 //
309
310 _commandQueue = new HgiVulkanCommandQueue(this);
311
312 //
313 // Pipeline cache
314 //
315
316 _pipelineCache = new HgiVulkanPipelineCache(this);
317 }
318
~HgiVulkanDevice()319 HgiVulkanDevice::~HgiVulkanDevice()
320 {
321 // Make sure device is idle before destroying objects.
322 TF_VERIFY(vkDeviceWaitIdle(_vkDevice) == VK_SUCCESS);
323
324 delete _pipelineCache;
325 delete _commandQueue;
326 delete _capabilities;
327 vmaDestroyAllocator(_vmaAllocator);
328 vkDestroyDevice(_vkDevice, HgiVulkanAllocator());
329 }
330
331 VkDevice
GetVulkanDevice() const332 HgiVulkanDevice::GetVulkanDevice() const
333 {
334 return _vkDevice;
335 }
336
337 VmaAllocator
GetVulkanMemoryAllocator() const338 HgiVulkanDevice::GetVulkanMemoryAllocator() const
339 {
340 return _vmaAllocator;
341 }
342
343 HgiVulkanCommandQueue*
GetCommandQueue() const344 HgiVulkanDevice::GetCommandQueue() const
345 {
346 return _commandQueue;
347 }
348
349 HgiVulkanCapabilities const&
GetDeviceCapabilities() const350 HgiVulkanDevice::GetDeviceCapabilities() const
351 {
352 return *_capabilities;
353 }
354
355 uint32_t
GetGfxQueueFamilyIndex() const356 HgiVulkanDevice::GetGfxQueueFamilyIndex() const
357 {
358 return _vkGfxsQueueFamilyIndex;
359 }
360
361 VkPhysicalDevice
GetVulkanPhysicalDevice() const362 HgiVulkanDevice::GetVulkanPhysicalDevice() const
363 {
364 return _vkPhysicalDevice;
365 }
366
367 HgiVulkanPipelineCache*
GetPipelineCache() const368 HgiVulkanDevice::GetPipelineCache() const
369 {
370 return _pipelineCache;
371 }
372
373 void
WaitForIdle()374 HgiVulkanDevice::WaitForIdle()
375 {
376 TF_VERIFY(
377 vkDeviceWaitIdle(_vkDevice) == VK_SUCCESS
378 );
379 }
380
381 bool
_IsSupportedExtension(const char * extensionName) const382 HgiVulkanDevice::_IsSupportedExtension(const char* extensionName) const
383 {
384 for (VkExtensionProperties const& ext : _vkExtensions) {
385 if (!strcmp(extensionName, ext.extensionName)) {
386 return true;
387 }
388 }
389
390 return false;
391 }
392
393
394 PXR_NAMESPACE_CLOSE_SCOPE
395