1 /******************************************************************************* 2 Copyright (c) 2015-2022 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #include "uvm_rm_mem.h" 25 #include "uvm_test.h" 26 #include "uvm_test_ioctl.h" 27 #include "uvm_va_space.h" 28 #include "uvm_kvmalloc.h" 29 30 static NV_STATUS map_cpu(uvm_rm_mem_t *rm_mem) 31 { 32 void *cpu_va; 33 34 // Map 35 TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem)); 36 TEST_CHECK_RET(uvm_rm_mem_mapped_on_cpu(rm_mem)); 37 38 // Mapping if already mapped is OK 39 TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem)); 40 41 // Unmap 42 uvm_rm_mem_unmap_cpu(rm_mem); 43 44 // Unmapping already unmapped also OK 45 uvm_rm_mem_unmap_cpu(rm_mem); 46 47 // Map again 48 TEST_NV_CHECK_RET(uvm_rm_mem_map_cpu(rm_mem)); 49 50 cpu_va = uvm_rm_mem_get_cpu_va(rm_mem); 51 TEST_CHECK_RET(cpu_va != NULL); 52 53 // Check that the CPU VA is writable. 54 // memset and memcpy might cause alignment faults on aarch64. 55 // See Bug 2668765 for more details. Since this is a test ioctl and 56 // therefore not a critical performance path, it's safe to use memset_io on 57 // all platforms. 58 memset_io(cpu_va, 0, rm_mem->size); 59 60 return NV_OK; 61 } 62 63 static NV_STATUS check_alignment(uvm_rm_mem_t *rm_mem, uvm_gpu_t *gpu, NvU64 alignment) 64 { 65 // Alignment requirements only apply to mappings in the UVM-owned VA space 66 if (alignment != 0) { 67 NvU64 gpu_uvm_va = uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu); 68 69 TEST_CHECK_RET(IS_ALIGNED(gpu_uvm_va, alignment)); 70 } 71 72 return NV_OK; 73 } 74 75 static NV_STATUS map_gpu_owner(uvm_rm_mem_t *rm_mem, NvU64 alignment) 76 { 77 uvm_gpu_t *gpu = rm_mem->gpu_owner; 78 NvU64 gpu_uvm_va; 79 NvU64 gpu_proxy_va = 0; 80 81 // The memory should have been automatically mapped in the GPU owner 82 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); 83 84 gpu_uvm_va = uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu); 85 86 // In SR-IOV heavy, there are two VA spaces per GPU, so there are two 87 // mappings for a single rm_mem object on a GPU, even if the memory is 88 // located in vidmem. 89 if (uvm_gpu_uses_proxy_channel_pool(gpu)) { 90 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); 91 92 gpu_proxy_va = uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu); 93 } 94 else { 95 TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); 96 } 97 98 TEST_NV_CHECK_RET(check_alignment(rm_mem, gpu, alignment)); 99 100 // Mappings are not ref counted, so additional map calls are no-ops; the 101 // GPU VA should remain the same for all the applicable VA spaces. 102 TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); 103 104 TEST_CHECK_RET(gpu_uvm_va == uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu)); 105 106 if (uvm_gpu_uses_proxy_channel_pool(gpu)) 107 TEST_CHECK_RET(gpu_proxy_va == uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu)); 108 109 // Unmapping the GPU owner is a no-op 110 uvm_rm_mem_unmap_gpu(rm_mem, gpu); 111 112 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); 113 TEST_CHECK_RET(gpu_uvm_va == uvm_rm_mem_get_gpu_uvm_va(rm_mem, gpu)); 114 115 if (uvm_gpu_uses_proxy_channel_pool(gpu)) { 116 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); 117 TEST_CHECK_RET(gpu_proxy_va == uvm_rm_mem_get_gpu_proxy_va(rm_mem, gpu)); 118 } 119 else { 120 TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); 121 } 122 123 return NV_OK; 124 } 125 126 static NV_STATUS map_other_gpus(uvm_rm_mem_t *rm_mem, uvm_va_space_t *va_space, NvU64 alignment) 127 { 128 uvm_gpu_t *gpu_owner = rm_mem->gpu_owner; 129 uvm_gpu_t *gpu; 130 131 for_each_va_space_gpu(gpu, va_space) { 132 if (gpu == gpu_owner) 133 continue; 134 135 TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); 136 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); 137 138 // Mappings are not ref counted, so additional map calls are no-ops 139 TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); 140 141 // The previous GPU map calls added mappings to the proxy VA space 142 // when in SR-IOV heavy mode 143 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu) == uvm_gpu_uses_proxy_channel_pool(gpu)); 144 145 // Unmapping removes all mappings 146 uvm_rm_mem_unmap_gpu(rm_mem, gpu); 147 TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); 148 TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); 149 150 // Additional unmappings are no-ops 151 uvm_rm_mem_unmap_gpu(rm_mem, gpu); 152 TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); 153 TEST_CHECK_RET(!uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu)); 154 155 // Subsequent mappings should behave as they did in the beginning. 156 TEST_NV_CHECK_RET(uvm_rm_mem_map_gpu(rm_mem, gpu, alignment)); 157 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu(rm_mem, gpu)); 158 159 TEST_CHECK_RET(uvm_rm_mem_mapped_on_gpu_proxy(rm_mem, gpu) == uvm_gpu_uses_proxy_channel_pool(gpu)); 160 161 TEST_NV_CHECK_RET(check_alignment(rm_mem, gpu, alignment)); 162 } 163 164 return NV_OK; 165 } 166 167 static NV_STATUS test_all_gpus_in_va(uvm_va_space_t *va_space) 168 { 169 uvm_gpu_t *gpu; 170 uvm_rm_mem_t *rm_mem = NULL; 171 NV_STATUS status = NV_OK; 172 173 // Create allocations of these types 174 static const uvm_rm_mem_type_t mem_types[] = { UVM_RM_MEM_TYPE_SYS, UVM_RM_MEM_TYPE_GPU }; 175 176 // Create allocations of these sizes 177 static const size_t sizes[] = { 1, 4, 16, 128, 1024, 4096, 1024 * 1024, 4 * 1024 * 1024 }; 178 static const NvU64 alignments[] = { 0, 179 8, 180 UVM_PAGE_SIZE_4K >> 1, 181 UVM_PAGE_SIZE_4K, 182 UVM_PAGE_SIZE_4K << 1, 183 UVM_PAGE_SIZE_64K, 184 UVM_PAGE_SIZE_2M, 185 UVM_PAGE_SIZE_2M << 3, 186 UVM_PAGE_SIZE_2M << 5 }; 187 188 uvm_assert_rwsem_locked(&va_space->lock); 189 190 TEST_CHECK_RET(!uvm_processor_mask_empty(&va_space->registered_gpus)); 191 192 for_each_va_space_gpu(gpu, va_space) { 193 int i, j, k; 194 195 for (i = 0; i < ARRAY_SIZE(sizes); ++i) { 196 for (j = 0; j < ARRAY_SIZE(mem_types); ++j) { 197 for (k = 0; k < ARRAY_SIZE(alignments); ++k) { 198 199 // Create an allocation in the GPU's address space 200 TEST_NV_CHECK_RET(uvm_rm_mem_alloc(gpu, mem_types[j], sizes[i], alignments[k], &rm_mem)); 201 202 // Test CPU mappings 203 TEST_NV_CHECK_GOTO(map_cpu(rm_mem), error); 204 205 // Test mappings in the GPU owning the allocation 206 TEST_NV_CHECK_GOTO(map_gpu_owner(rm_mem, alignments[k]), error); 207 208 // For sysmem allocations, test mappings on all other GPUs 209 if (rm_mem->type == UVM_RM_MEM_TYPE_SYS) 210 TEST_NV_CHECK_GOTO(map_other_gpus(rm_mem, va_space, alignments[k]), error); 211 212 uvm_rm_mem_free(rm_mem); 213 } 214 } 215 } 216 } 217 218 return NV_OK; 219 220 error: 221 uvm_rm_mem_free(rm_mem); 222 223 return status; 224 } 225 226 NV_STATUS uvm_test_rm_mem_sanity(UVM_TEST_RM_MEM_SANITY_PARAMS *params, struct file *filp) 227 { 228 NV_STATUS status; 229 uvm_va_space_t *va_space = uvm_va_space_get(filp); 230 231 uvm_va_space_down_read_rm(va_space); 232 233 status = test_all_gpus_in_va(va_space); 234 235 uvm_va_space_up_read_rm(va_space); 236 237 return status; 238 } 239