1 //
2 // Copyright 2020 Pixar
3 //
4 // Licensed under the Apache License, Version 2.0 (the "Apache License")
5 // with the following modification; you may not use this file except in
6 // compliance with the Apache License and the following modification to it:
7 // Section 6. Trademarks. is deleted and replaced with:
8 //
9 // 6. Trademarks. This License does not grant permission to use the trade
10 //    names, trademarks, service marks, or product names of the Licensor
11 //    and its affiliates, except as required to comply with Section 4(c) of
12 //    the License and to reproduce the content of the NOTICE file.
13 //
14 // You may obtain a copy of the Apache License at
15 //
16 //     http://www.apache.org/licenses/LICENSE-2.0
17 //
18 // Unless required by applicable law or agreed to in writing, software
19 // distributed under the Apache License with the above modification is
20 // distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
21 // KIND, either express or implied. See the Apache License for the specific
22 // language governing permissions and limitations under the Apache License.
23 //
24 #include "pxr/imaging/hgiVulkan/capabilities.h"
25 #include "pxr/imaging/hgiVulkan/commandQueue.h"
26 #include "pxr/imaging/hgiVulkan/device.h"
27 #include "pxr/imaging/hgiVulkan/diagnostic.h"
28 #include "pxr/imaging/hgiVulkan/hgi.h"
29 #include "pxr/imaging/hgiVulkan/instance.h"
30 #include "pxr/imaging/hgiVulkan/pipelineCache.h"
31 
32 #include "pxr/base/tf/diagnostic.h"
33 
34 #define VMA_IMPLEMENTATION
35     #include "pxr/imaging/hgiVulkan/vk_mem_alloc.h"
36 #undef VMA_IMPLEMENTATION
37 
38 PXR_NAMESPACE_OPEN_SCOPE
39 
40 
41 static uint32_t
_GetGraphicsQueueFamilyIndex(VkPhysicalDevice physicalDevice)42 _GetGraphicsQueueFamilyIndex(VkPhysicalDevice physicalDevice)
43 {
44     uint32_t queueCount = 0;
45     vkGetPhysicalDeviceQueueFamilyProperties(physicalDevice, &queueCount, 0);
46 
47     std::vector<VkQueueFamilyProperties> queues(queueCount);
48     vkGetPhysicalDeviceQueueFamilyProperties(
49         physicalDevice,
50         &queueCount,
51         queues.data());
52 
53     for (uint32_t i = 0; i < queueCount; i++) {
54         if (queues[i].queueFlags & VK_QUEUE_GRAPHICS_BIT) {
55             return i;
56         }
57     }
58 
59     return VK_QUEUE_FAMILY_IGNORED;
60 }
61 
62 static bool
_SupportsPresentation(VkPhysicalDevice physicalDevice,uint32_t familyIndex)63 _SupportsPresentation(
64     VkPhysicalDevice physicalDevice,
65     uint32_t familyIndex)
66 {
67     #if defined(VK_USE_PLATFORM_WIN32_KHR)
68         return vkGetPhysicalDeviceWin32PresentationSupportKHR(
69                     physicalDevice, familyIndex);
70     #elif defined(VK_USE_PLATFORM_XLIB_KHR)
71         Display* dsp = XOpenDisplay(nullptr);
72         VisualID visualID = XVisualIDFromVisual(
73             DefaultVisual(dsp, DefaultScreen(dsp)));
74         return vkGetPhysicalDeviceXlibPresentationSupportKHR(
75                     physicalDevice, familyIndex, dsp, visualID);
76     #elif defined(VK_USE_PLATFORM_MACOS_MVK)
77         // Presentation currently always supported on Metal / MoltenVk
78         return true;
79     #else
80         #error Unsupported Platform
81         return true;
82     #endif
83 }
84 
HgiVulkanDevice(HgiVulkanInstance * instance)85 HgiVulkanDevice::HgiVulkanDevice(HgiVulkanInstance* instance)
86     : _vkPhysicalDevice(nullptr)
87     , _vkDevice(nullptr)
88     , _vmaAllocator(nullptr)
89     , _commandQueue(nullptr)
90     , _capabilities(nullptr)
91 {
92     //
93     // Determine physical device
94     //
95 
96     const uint32_t maxDevices = 64;
97     VkPhysicalDevice physicalDevices[maxDevices];
98     uint32_t physicalDeviceCount = maxDevices;
99     TF_VERIFY(
100         vkEnumeratePhysicalDevices(
101             instance->GetVulkanInstance(),
102             &physicalDeviceCount,
103             physicalDevices) == VK_SUCCESS
104     );
105 
106     for (uint32_t i = 0; i < physicalDeviceCount; i++) {
107         VkPhysicalDeviceProperties props;
108         vkGetPhysicalDeviceProperties(physicalDevices[i], &props);
109 
110         uint32_t familyIndex =
111             _GetGraphicsQueueFamilyIndex(physicalDevices[i]);
112 
113         if (familyIndex == VK_QUEUE_FAMILY_IGNORED) continue;
114 
115         // Assume we always want a presentation capable device for now.
116         if (!_SupportsPresentation(physicalDevices[i], familyIndex)) {
117             continue;
118         }
119 
120         if (props.apiVersion < VK_API_VERSION_1_0) continue;
121 
122         // Try to find a discrete device. Until we find a discrete device,
123         // store the first non-discrete device as fallback in case we never
124         // find a discrete device at all.
125         if (props.deviceType==VK_PHYSICAL_DEVICE_TYPE_DISCRETE_GPU) {
126             _vkPhysicalDevice = physicalDevices[i];
127             _vkGfxsQueueFamilyIndex = familyIndex;
128             break;
129         } else if (!_vkPhysicalDevice) {
130             _vkPhysicalDevice = physicalDevices[i];
131             _vkGfxsQueueFamilyIndex = familyIndex;
132         }
133     }
134 
135     if (!_vkPhysicalDevice) {
136         TF_CODING_ERROR("VULKAN_ERROR: Unable to determine physical device");
137         return;
138     }
139 
140     //
141     // Query supported extensions for device
142     //
143 
144     uint32_t extensionCount = 0;
145     TF_VERIFY(
146         vkEnumerateDeviceExtensionProperties(
147             _vkPhysicalDevice,
148             nullptr,
149             &extensionCount,
150             nullptr) == VK_SUCCESS
151     );
152 
153     _vkExtensions.resize(extensionCount);
154 
155     TF_VERIFY(
156         vkEnumerateDeviceExtensionProperties(
157             _vkPhysicalDevice,
158             nullptr,
159             &extensionCount,
160             _vkExtensions.data()) == VK_SUCCESS
161     );
162 
163     //
164     // Create Device
165     //
166     _capabilities = new HgiVulkanCapabilities(this);
167 
168     VkDeviceQueueCreateInfo queueInfo =
169         {VK_STRUCTURE_TYPE_DEVICE_QUEUE_CREATE_INFO};
170     float queuePriorities[] = {1.0f};
171     queueInfo.queueFamilyIndex = _vkGfxsQueueFamilyIndex;
172     queueInfo.queueCount = 1;
173     queueInfo.pQueuePriorities = queuePriorities;
174 
175     std::vector<const char*> extensions = {
176         VK_KHR_SWAPCHAIN_EXTENSION_NAME
177     };
178 
179     // Allow certain buffers/images to have dedicated memory allocations to
180     // improve performance on some GPUs.
181     bool dedicatedAllocations = false;
182     if (_IsSupportedExtension(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME)
183         && _IsSupportedExtension(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME))
184     {
185         dedicatedAllocations = true;
186         extensions.push_back(VK_KHR_GET_MEMORY_REQUIREMENTS_2_EXTENSION_NAME);
187         extensions.push_back(VK_KHR_DEDICATED_ALLOCATION_EXTENSION_NAME);
188     }
189 
190     // Allow OpenGL interop - Note requires two extensions in HgiVulkanInstance.
191     if (_IsSupportedExtension(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME) &&
192         _IsSupportedExtension(VK_KHR_EXTERNAL_SEMAPHORE_EXTENSION_NAME))
193     {
194         extensions.push_back(VK_KHR_EXTERNAL_SEMAPHORE_EXTENSION_NAME);
195         extensions.push_back(VK_KHR_EXTERNAL_MEMORY_EXTENSION_NAME);
196     }
197 
198     // Memory budget query extension
199     bool supportsMemExtension = false;
200     if (_IsSupportedExtension(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME)) {
201         supportsMemExtension = true;
202         extensions.push_back(VK_EXT_MEMORY_BUDGET_EXTENSION_NAME);
203     }
204 
205     // Resolve depth during render pass resolve extension
206     if (_IsSupportedExtension(VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME)) {
207         extensions.push_back(VK_KHR_DEPTH_STENCIL_RESOLVE_EXTENSION_NAME);
208         extensions.push_back(VK_KHR_CREATE_RENDERPASS_2_EXTENSION_NAME);
209         extensions.push_back(VK_KHR_MULTIVIEW_EXTENSION_NAME);
210         extensions.push_back(VK_KHR_MAINTENANCE2_EXTENSION_NAME);
211     }
212 
213     // Allows the same layout in structs between c++ and glsl (share structs).
214     // This means instead of 'std430' you can now use 'scalar'.
215     if (_IsSupportedExtension(VK_EXT_SCALAR_BLOCK_LAYOUT_EXTENSION_NAME)) {
216         extensions.push_back(VK_EXT_SCALAR_BLOCK_LAYOUT_EXTENSION_NAME);
217     } else {
218         TF_WARN("Unsupported VK_EXT_scalar_block_layout."
219                 "Update gfx driver?");
220     }
221 
222     // This extension is needed to allow the viewport to be flipped in Y so that
223     // shaders and vertex data can remain the same between opengl and vulkan.
224     extensions.push_back(VK_KHR_MAINTENANCE1_EXTENSION_NAME);
225 
226     // Enabling certain features may incure a performance hit
227     // (e.g. robustBufferAccess), so only enable the features we will use.
228     VkPhysicalDeviceFeatures2 features =
229         {VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_FEATURES_2};
230 
231     //extension features
232     features.pNext = _capabilities->vkDeviceFeatures2.pNext;
233 
234     features.features.multiDrawIndirect =
235         _capabilities->vkDeviceFeatures.multiDrawIndirect;
236     features.features.samplerAnisotropy =
237         _capabilities->vkDeviceFeatures.samplerAnisotropy;
238     features.features.shaderSampledImageArrayDynamicIndexing =
239         _capabilities->vkDeviceFeatures.shaderSampledImageArrayDynamicIndexing;
240     features.features.shaderStorageImageArrayDynamicIndexing =
241         _capabilities->vkDeviceFeatures.shaderStorageImageArrayDynamicIndexing;
242     features.features.sampleRateShading =
243         _capabilities->vkDeviceFeatures.sampleRateShading;
244     features.features.shaderClipDistance =
245         _capabilities->vkDeviceFeatures.shaderClipDistance;
246     features.features.tessellationShader =
247         _capabilities->vkDeviceFeatures.tessellationShader;
248 
249     // Needed to write to storage buffers from vertex shader (eg. GPU culling).
250     features.features.vertexPipelineStoresAndAtomics =
251         _capabilities->vkDeviceFeatures.vertexPipelineStoresAndAtomics;
252 
253     #if !defined(VK_USE_PLATFORM_MACOS_MVK)
254         // Needed for buffer address feature
255         features.features.shaderInt64 =
256             _capabilities->vkDeviceFeatures.shaderInt64;
257         // Needed for gl_primtiveID
258         features.features.geometryShader =
259             _capabilities->vkDeviceFeatures.geometryShader;
260     #endif
261 
262     VkDeviceCreateInfo createInfo = {VK_STRUCTURE_TYPE_DEVICE_CREATE_INFO};
263     createInfo.queueCreateInfoCount = 1;
264     createInfo.pQueueCreateInfos = &queueInfo;
265     createInfo.ppEnabledExtensionNames = extensions.data();
266     createInfo.enabledExtensionCount = (uint32_t) extensions.size();
267     createInfo.pNext = &features;
268 
269     TF_VERIFY(
270         vkCreateDevice(
271             _vkPhysicalDevice,
272             &createInfo,
273             HgiVulkanAllocator(),
274             &_vkDevice) == VK_SUCCESS
275     );
276 
277     HgiVulkanSetupDeviceDebug(instance, this);
278 
279     //
280     // Extension function pointers
281     //
282 
283     vkCreateRenderPass2KHR = (PFN_vkCreateRenderPass2KHR)
284     vkGetDeviceProcAddr(_vkDevice, "vkCreateRenderPass2KHR");
285 
286     //
287     // Memory allocator
288     //
289 
290     VmaAllocatorCreateInfo allocatorInfo = {};
291     allocatorInfo.instance = instance->GetVulkanInstance();
292     allocatorInfo.physicalDevice = _vkPhysicalDevice;
293     allocatorInfo.device = _vkDevice;
294     if (dedicatedAllocations) {
295         allocatorInfo.flags |=VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT;
296     }
297 
298     if (supportsMemExtension) {
299         allocatorInfo.flags |= VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT;
300     }
301 
302     TF_VERIFY(
303         vmaCreateAllocator(&allocatorInfo, &_vmaAllocator) == VK_SUCCESS
304     );
305 
306     //
307     // Command Queue
308     //
309 
310     _commandQueue = new HgiVulkanCommandQueue(this);
311 
312     //
313     // Pipeline cache
314     //
315 
316     _pipelineCache = new HgiVulkanPipelineCache(this);
317 }
318 
~HgiVulkanDevice()319 HgiVulkanDevice::~HgiVulkanDevice()
320 {
321     // Make sure device is idle before destroying objects.
322     TF_VERIFY(vkDeviceWaitIdle(_vkDevice) == VK_SUCCESS);
323 
324     delete _pipelineCache;
325     delete _commandQueue;
326     delete _capabilities;
327     vmaDestroyAllocator(_vmaAllocator);
328     vkDestroyDevice(_vkDevice, HgiVulkanAllocator());
329 }
330 
331 VkDevice
GetVulkanDevice() const332 HgiVulkanDevice::GetVulkanDevice() const
333 {
334     return _vkDevice;
335 }
336 
337 VmaAllocator
GetVulkanMemoryAllocator() const338 HgiVulkanDevice::GetVulkanMemoryAllocator() const
339 {
340     return _vmaAllocator;
341 }
342 
343 HgiVulkanCommandQueue*
GetCommandQueue() const344 HgiVulkanDevice::GetCommandQueue() const
345 {
346     return _commandQueue;
347 }
348 
349 HgiVulkanCapabilities const&
GetDeviceCapabilities() const350 HgiVulkanDevice::GetDeviceCapabilities() const
351 {
352     return *_capabilities;
353 }
354 
355 uint32_t
GetGfxQueueFamilyIndex() const356 HgiVulkanDevice::GetGfxQueueFamilyIndex() const
357 {
358     return _vkGfxsQueueFamilyIndex;
359 }
360 
361 VkPhysicalDevice
GetVulkanPhysicalDevice() const362 HgiVulkanDevice::GetVulkanPhysicalDevice() const
363 {
364     return _vkPhysicalDevice;
365 }
366 
367 HgiVulkanPipelineCache*
GetPipelineCache() const368 HgiVulkanDevice::GetPipelineCache() const
369 {
370     return _pipelineCache;
371 }
372 
373 void
WaitForIdle()374 HgiVulkanDevice::WaitForIdle()
375 {
376     TF_VERIFY(
377         vkDeviceWaitIdle(_vkDevice) == VK_SUCCESS
378     );
379 }
380 
381 bool
_IsSupportedExtension(const char * extensionName) const382 HgiVulkanDevice::_IsSupportedExtension(const char* extensionName) const
383 {
384     for (VkExtensionProperties const& ext : _vkExtensions) {
385         if (!strcmp(extensionName, ext.extensionName)) {
386             return true;
387         }
388     }
389 
390     return false;
391 }
392 
393 
394 PXR_NAMESPACE_CLOSE_SCOPE
395