1 /******************************************************************************* 2 Copyright (c) 2018-2023 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #include "uvm_ats_sva.h" 25 26 #if UVM_ATS_SVA_SUPPORTED() 27 28 #include "uvm_gpu.h" 29 #include "uvm_va_space.h" 30 #include "uvm_va_space_mm.h" 31 32 #include <linux/iommu.h> 33 #include <linux/mm_types.h> 34 35 // linux/sched/mm.h is needed for mmget_not_zero and mmput to get the mm 36 // reference required for the iommu_sva_bind_device() call. This header is not 37 // present in all the supported versions. Instead of adding a conftest just for 38 // this header file, use UVM_ATS_SVA_SUPPORTED(). 39 #include <linux/sched/mm.h> 40 41 // iommu_sva_bind_device() removed drvdata paramter with commit 42 // 942fd5435dccb273f90176b046ae6bbba60cfbd8 (10/31/2022). 43 #if defined(NV_IOMMU_SVA_BIND_DEVICE_HAS_DRVDATA_ARG) 44 #define UVM_IOMMU_SVA_BIND_DEVICE(dev, mm) iommu_sva_bind_device(dev, mm, NULL) 45 #else 46 #define UVM_IOMMU_SVA_BIND_DEVICE(dev, mm) iommu_sva_bind_device(dev, mm) 47 #endif 48 49 NV_STATUS uvm_ats_sva_add_gpu(uvm_parent_gpu_t *parent_gpu) 50 { 51 int ret; 52 53 ret = iommu_dev_enable_feature(&parent_gpu->pci_dev->dev, IOMMU_DEV_FEAT_SVA); 54 55 return errno_to_nv_status(ret); 56 } 57 58 void uvm_ats_sva_remove_gpu(uvm_parent_gpu_t *parent_gpu) 59 { 60 iommu_dev_disable_feature(&parent_gpu->pci_dev->dev, IOMMU_DEV_FEAT_SVA); 61 } 62 63 NV_STATUS uvm_ats_sva_bind_gpu(uvm_gpu_va_space_t *gpu_va_space) 64 { 65 NV_STATUS status = NV_OK; 66 struct iommu_sva *iommu_handle; 67 struct pci_dev *pci_dev = gpu_va_space->gpu->parent->pci_dev; 68 uvm_sva_gpu_va_space_t *sva_gpu_va_space = &gpu_va_space->ats.sva; 69 struct mm_struct *mm = gpu_va_space->va_space->va_space_mm.mm; 70 71 UVM_ASSERT(gpu_va_space->ats.enabled); 72 UVM_ASSERT(uvm_gpu_va_space_state(gpu_va_space) == UVM_GPU_VA_SPACE_STATE_INIT); 73 UVM_ASSERT(mm); 74 75 // The mmput() below may trigger the kernel's mm teardown with exit_mmap() 76 // and uvm_va_space_mm_shutdown() and uvm_vm_close_managed() in that path 77 // will try to grab the va_space lock and deadlock if va_space was already 78 // locked. 79 uvm_assert_unlocked_order(UVM_LOCK_ORDER_VA_SPACE); 80 81 // iommu_sva_bind_device() requires the mm reference to be acquired. Since 82 // the mm is already retained, mm is still valid but may be inactive since 83 // mm_users can still be zero since UVM doesn't use mm_users and maintains a 84 // separate refcount (retained_count) for the mm in va_space_mm. See the 85 // block comment in va_space_mm.c for more details. So, return an error if 86 // mm_users is zero. 87 if (!mmget_not_zero(mm)) 88 return NV_ERR_PAGE_TABLE_NOT_AVAIL; 89 90 // Multiple calls for the {same pci_dev, mm} pair are refcounted by the ARM 91 // SMMU Layer. 92 iommu_handle = UVM_IOMMU_SVA_BIND_DEVICE(&pci_dev->dev, mm); 93 if (IS_ERR(iommu_handle)) { 94 status = errno_to_nv_status(PTR_ERR(iommu_handle)); 95 goto out; 96 } 97 98 // If this is not the first bind of the gpu in the mm, then the previously 99 // stored iommu_handle in the gpu_va_space must match the handle returned by 100 // iommu_sva_bind_device(). 101 if (sva_gpu_va_space->iommu_handle) { 102 UVM_ASSERT(sva_gpu_va_space->iommu_handle == iommu_handle); 103 nv_kref_get(&sva_gpu_va_space->kref); 104 } 105 else { 106 sva_gpu_va_space->iommu_handle = iommu_handle; 107 nv_kref_init(&sva_gpu_va_space->kref); 108 } 109 110 out: 111 mmput(mm); 112 return status; 113 } 114 115 static void uvm_sva_reset_iommu_handle(nv_kref_t *nv_kref) 116 { 117 uvm_sva_gpu_va_space_t *sva_gpu_va_space = container_of(nv_kref, uvm_sva_gpu_va_space_t, kref); 118 sva_gpu_va_space->iommu_handle = NULL; 119 } 120 121 void uvm_ats_sva_unbind_gpu(uvm_gpu_va_space_t *gpu_va_space) 122 { 123 uvm_sva_gpu_va_space_t *sva_gpu_va_space = &gpu_va_space->ats.sva; 124 125 // ARM SMMU layer decrements the refcount for the {pci_dev, mm} pair. 126 // The actual unbind happens only when the refcount reaches zero. 127 if (sva_gpu_va_space->iommu_handle) { 128 iommu_sva_unbind_device(sva_gpu_va_space->iommu_handle); 129 nv_kref_put(&sva_gpu_va_space->kref, uvm_sva_reset_iommu_handle); 130 } 131 } 132 133 NV_STATUS uvm_ats_sva_register_gpu_va_space(uvm_gpu_va_space_t *gpu_va_space) 134 { 135 NvU32 pasid; 136 NV_STATUS status = NV_OK; 137 uvm_sva_gpu_va_space_t *sva_gpu_va_space = &gpu_va_space->ats.sva; 138 139 // A successful iommu_sva_bind_device() should have preceded this call. 140 UVM_ASSERT(sva_gpu_va_space->iommu_handle); 141 142 pasid = iommu_sva_get_pasid(sva_gpu_va_space->iommu_handle); 143 if (pasid == IOMMU_PASID_INVALID) 144 status = errno_to_nv_status(ENODEV); 145 else 146 gpu_va_space->ats.pasid = pasid; 147 148 return status; 149 } 150 151 void uvm_ats_sva_unregister_gpu_va_space(uvm_gpu_va_space_t *gpu_va_space) 152 { 153 gpu_va_space->ats.pasid = -1U; 154 } 155 156 #endif // UVM_ATS_SVA_SUPPORTED() 157