1 /******************************************************************************* 2 Copyright (c) 2021 NVIDIA Corporation 3 4 Permission is hereby granted, free of charge, to any person obtaining a copy 5 of this software and associated documentation files (the "Software"), to 6 deal in the Software without restriction, including without limitation the 7 rights to use, copy, modify, merge, publish, distribute, sublicense, and/or 8 sell copies of the Software, and to permit persons to whom the Software is 9 furnished to do so, subject to the following conditions: 10 11 The above copyright notice and this permission notice shall be 12 included in all copies or substantial portions of the Software. 13 14 THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 18 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 19 FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 20 DEALINGS IN THE SOFTWARE. 21 22 *******************************************************************************/ 23 24 #include "uvm_common.h" 25 #include "uvm_global.h" 26 #include "uvm_conf_computing.h" 27 #include "uvm_kvmalloc.h" 28 #include "uvm_gpu.h" 29 #include "uvm_mem.h" 30 #include "uvm_processors.h" 31 #include "uvm_tracker.h" 32 #include "nv_uvm_interface.h" 33 #include "uvm_va_block.h" 34 35 36 static UvmGpuConfComputeMode uvm_conf_computing_get_mode(const uvm_parent_gpu_t *parent) 37 { 38 return parent->rm_info.gpuConfComputeCaps.mode; 39 } 40 41 bool uvm_conf_computing_mode_enabled_parent(const uvm_parent_gpu_t *parent) 42 { 43 return uvm_conf_computing_get_mode(parent) != UVM_GPU_CONF_COMPUTE_MODE_NONE; 44 } 45 46 bool uvm_conf_computing_mode_enabled(const uvm_gpu_t *gpu) 47 { 48 return uvm_conf_computing_mode_enabled_parent(gpu->parent); 49 } 50 51 bool uvm_conf_computing_mode_is_hcc(const uvm_gpu_t *gpu) 52 { 53 return uvm_conf_computing_get_mode(gpu->parent) == UVM_GPU_CONF_COMPUTE_MODE_HCC; 54 } 55 56 NV_STATUS uvm_conf_computing_init_parent_gpu(const uvm_parent_gpu_t *parent) 57 { 58 UvmGpuConfComputeMode cc, sys_cc; 59 uvm_gpu_t *first; 60 61 uvm_assert_mutex_locked(&g_uvm_global.global_lock); 62 63 // TODO: Bug 2844714. 64 // Since we have no routine to traverse parent gpus, 65 // find first child GPU and get its parent. 66 first = uvm_global_processor_mask_find_first_gpu(&g_uvm_global.retained_gpus); 67 if (!first) 68 return NV_OK; 69 70 sys_cc = uvm_conf_computing_get_mode(first->parent); 71 cc = uvm_conf_computing_get_mode(parent); 72 73 return cc == sys_cc ? NV_OK : NV_ERR_NOT_SUPPORTED; 74 } 75 76 static void dma_buffer_destroy_locked(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool, 77 uvm_conf_computing_dma_buffer_t *dma_buffer) 78 { 79 uvm_assert_mutex_locked(&dma_buffer_pool->lock); 80 81 list_del(&dma_buffer->node); 82 uvm_tracker_wait_deinit(&dma_buffer->tracker); 83 84 uvm_mem_free(dma_buffer->alloc); 85 uvm_mem_free(dma_buffer->auth_tag); 86 uvm_kvfree(dma_buffer); 87 } 88 89 static uvm_gpu_t *dma_buffer_pool_to_gpu(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool) 90 { 91 return container_of(dma_buffer_pool, uvm_gpu_t, conf_computing.dma_buffer_pool); 92 } 93 94 // Allocate and map a new DMA stage buffer to CPU and GPU (VA) 95 static NV_STATUS dma_buffer_create(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool, 96 uvm_conf_computing_dma_buffer_t **dma_buffer_out) 97 { 98 uvm_gpu_t *dma_owner; 99 uvm_conf_computing_dma_buffer_t *dma_buffer; 100 uvm_mem_t *alloc = NULL; 101 NV_STATUS status = NV_OK; 102 size_t auth_tags_size = (UVM_CONF_COMPUTING_DMA_BUFFER_SIZE / PAGE_SIZE) * UVM_CONF_COMPUTING_AUTH_TAG_SIZE; 103 104 dma_buffer = uvm_kvmalloc_zero(sizeof(*dma_buffer)); 105 if (!dma_buffer) 106 return NV_ERR_NO_MEMORY; 107 108 dma_owner = dma_buffer_pool_to_gpu(dma_buffer_pool); 109 uvm_tracker_init(&dma_buffer->tracker); 110 INIT_LIST_HEAD(&dma_buffer->node); 111 112 status = uvm_mem_alloc_sysmem_dma_and_map_cpu_kernel(UVM_CONF_COMPUTING_DMA_BUFFER_SIZE, dma_owner, NULL, &alloc); 113 if (status != NV_OK) 114 goto err; 115 116 dma_buffer->alloc = alloc; 117 118 status = uvm_mem_map_gpu_kernel(alloc, dma_owner); 119 if (status != NV_OK) 120 goto err; 121 122 status = uvm_mem_alloc_sysmem_dma_and_map_cpu_kernel(auth_tags_size, dma_owner, NULL, &alloc); 123 if (status != NV_OK) 124 goto err; 125 126 dma_buffer->auth_tag = alloc; 127 128 status = uvm_mem_map_gpu_kernel(alloc, dma_owner); 129 if (status != NV_OK) 130 goto err; 131 132 *dma_buffer_out = dma_buffer; 133 134 return status; 135 136 err: 137 dma_buffer_destroy_locked(dma_buffer_pool, dma_buffer); 138 return status; 139 } 140 141 void uvm_conf_computing_dma_buffer_pool_sync(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool) 142 { 143 uvm_conf_computing_dma_buffer_t *dma_buffer; 144 145 if (dma_buffer_pool->num_dma_buffers == 0) 146 return; 147 148 uvm_mutex_lock(&dma_buffer_pool->lock); 149 list_for_each_entry(dma_buffer, &dma_buffer_pool->free_dma_buffers, node) 150 uvm_tracker_wait(&dma_buffer->tracker); 151 uvm_mutex_unlock(&dma_buffer_pool->lock); 152 } 153 154 static void conf_computing_dma_buffer_pool_deinit(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool) 155 { 156 uvm_conf_computing_dma_buffer_t *dma_buffer; 157 uvm_conf_computing_dma_buffer_t *next_buff; 158 159 if (dma_buffer_pool->num_dma_buffers == 0) 160 return; 161 162 // Because the pool is teared down at the same time the GPU is unregistered 163 // the lock is required only to quiet assertions not for functional reasons 164 // see dma_buffer_destroy_locked()). 165 uvm_mutex_lock(&dma_buffer_pool->lock); 166 167 list_for_each_entry_safe(dma_buffer, next_buff, &dma_buffer_pool->free_dma_buffers, node) { 168 dma_buffer_destroy_locked(dma_buffer_pool, dma_buffer); 169 dma_buffer_pool->num_dma_buffers--; 170 } 171 172 UVM_ASSERT(dma_buffer_pool->num_dma_buffers == 0); 173 UVM_ASSERT(list_empty(&dma_buffer_pool->free_dma_buffers)); 174 uvm_mutex_unlock(&dma_buffer_pool->lock); 175 } 176 177 static void dma_buffer_pool_add(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool, 178 uvm_conf_computing_dma_buffer_t *dma_buffer) 179 { 180 uvm_assert_mutex_locked(&dma_buffer_pool->lock); 181 list_add_tail(&dma_buffer->node, &dma_buffer_pool->free_dma_buffers); 182 } 183 184 static NV_STATUS conf_computing_dma_buffer_pool_init(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool) 185 { 186 size_t i; 187 uvm_gpu_t *gpu; 188 size_t num_dma_buffers = 32; 189 NV_STATUS status = NV_OK; 190 191 UVM_ASSERT(dma_buffer_pool->num_dma_buffers == 0); 192 193 gpu = dma_buffer_pool_to_gpu(dma_buffer_pool); 194 195 UVM_ASSERT(uvm_conf_computing_mode_enabled(gpu)); 196 197 INIT_LIST_HEAD(&dma_buffer_pool->free_dma_buffers); 198 uvm_mutex_init(&dma_buffer_pool->lock, UVM_LOCK_ORDER_CONF_COMPUTING_DMA_BUFFER_POOL); 199 dma_buffer_pool->num_dma_buffers = num_dma_buffers; 200 201 uvm_mutex_lock(&dma_buffer_pool->lock); 202 for (i = 0; i < num_dma_buffers; i++) { 203 uvm_conf_computing_dma_buffer_t *dma_buffer; 204 205 status = dma_buffer_create(dma_buffer_pool, &dma_buffer); 206 if (status != NV_OK) 207 break; 208 209 dma_buffer_pool_add(dma_buffer_pool, dma_buffer); 210 } 211 uvm_mutex_unlock(&dma_buffer_pool->lock); 212 213 if (i < num_dma_buffers) 214 conf_computing_dma_buffer_pool_deinit(dma_buffer_pool); 215 216 return status; 217 } 218 219 static NV_STATUS dma_buffer_pool_expand_locked(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool) 220 { 221 size_t i; 222 uvm_gpu_t *gpu; 223 size_t nb_to_alloc; 224 NV_STATUS status = NV_OK; 225 UVM_ASSERT(dma_buffer_pool->num_dma_buffers > 0); 226 227 gpu = dma_buffer_pool_to_gpu(dma_buffer_pool); 228 nb_to_alloc = dma_buffer_pool->num_dma_buffers; 229 for (i = 0; i < nb_to_alloc; ++i) { 230 uvm_conf_computing_dma_buffer_t *dma_buffer; 231 232 status = dma_buffer_create(dma_buffer_pool, &dma_buffer); 233 if (status != NV_OK) 234 break; 235 236 dma_buffer_pool_add(dma_buffer_pool, dma_buffer); 237 } 238 239 dma_buffer_pool->num_dma_buffers += i; 240 241 if (i == 0) 242 return status; 243 244 return NV_OK; 245 } 246 247 NV_STATUS uvm_conf_computing_dma_buffer_alloc(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool, 248 uvm_conf_computing_dma_buffer_t **dma_buffer_out, 249 uvm_tracker_t *out_tracker) 250 { 251 uvm_conf_computing_dma_buffer_t *dma_buffer = NULL; 252 NV_STATUS status; 253 254 UVM_ASSERT(dma_buffer_pool->num_dma_buffers > 0); 255 256 // TODO: Bug 3385623: Heuristically expand DMA memory pool 257 uvm_mutex_lock(&dma_buffer_pool->lock); 258 if (list_empty(&dma_buffer_pool->free_dma_buffers)) { 259 status = dma_buffer_pool_expand_locked(dma_buffer_pool); 260 261 if (status != NV_OK) { 262 uvm_mutex_unlock(&dma_buffer_pool->lock); 263 return status; 264 } 265 } 266 267 // We're guaranteed that at least one DMA stage buffer is available at this 268 // point. 269 dma_buffer = list_first_entry(&dma_buffer_pool->free_dma_buffers, uvm_conf_computing_dma_buffer_t, node); 270 list_del_init(&dma_buffer->node); 271 uvm_mutex_unlock(&dma_buffer_pool->lock); 272 273 status = uvm_tracker_wait_for_other_gpus(&dma_buffer->tracker, dma_buffer->alloc->dma_owner); 274 if (status != NV_OK) 275 goto error; 276 277 if (out_tracker) 278 status = uvm_tracker_add_tracker_safe(out_tracker, &dma_buffer->tracker); 279 else 280 status = uvm_tracker_wait(&dma_buffer->tracker); 281 282 if (status != NV_OK) 283 goto error; 284 285 uvm_page_mask_zero(&dma_buffer->encrypted_page_mask); 286 *dma_buffer_out = dma_buffer; 287 288 return status; 289 290 error: 291 uvm_tracker_deinit(&dma_buffer->tracker); 292 uvm_conf_computing_dma_buffer_free(dma_buffer_pool, dma_buffer, NULL); 293 return status; 294 } 295 296 void uvm_conf_computing_dma_buffer_free(uvm_conf_computing_dma_buffer_pool_t *dma_buffer_pool, 297 uvm_conf_computing_dma_buffer_t *dma_buffer, 298 uvm_tracker_t *tracker) 299 { 300 301 NV_STATUS status; 302 303 if (!dma_buffer) 304 return; 305 306 UVM_ASSERT(dma_buffer_pool->num_dma_buffers > 0); 307 308 uvm_tracker_remove_completed(&dma_buffer->tracker); 309 if (tracker) { 310 uvm_tracker_remove_completed(tracker); 311 status = uvm_tracker_add_tracker_safe(&dma_buffer->tracker, tracker); 312 if (status != NV_OK) 313 UVM_ASSERT(status == uvm_global_get_status()); 314 } 315 316 uvm_mutex_lock(&dma_buffer_pool->lock); 317 dma_buffer_pool_add(dma_buffer_pool, dma_buffer); 318 uvm_mutex_unlock(&dma_buffer_pool->lock); 319 } 320 321 static void dummy_iv_mem_deinit(uvm_gpu_t *gpu) 322 { 323 uvm_mem_free(gpu->conf_computing.iv_mem); 324 } 325 326 static NV_STATUS dummy_iv_mem_init(uvm_gpu_t *gpu) 327 { 328 NV_STATUS status; 329 330 if (!uvm_conf_computing_mode_is_hcc(gpu)) 331 return NV_OK; 332 333 status = uvm_mem_alloc_sysmem_dma(sizeof(UvmCslIv), gpu, NULL, &gpu->conf_computing.iv_mem); 334 if (status != NV_OK) 335 return status; 336 337 status = uvm_mem_map_gpu_kernel(gpu->conf_computing.iv_mem, gpu); 338 if (status != NV_OK) 339 goto error; 340 341 return NV_OK; 342 343 error: 344 dummy_iv_mem_deinit(gpu); 345 return status; 346 } 347 348 NV_STATUS uvm_conf_computing_gpu_init(uvm_gpu_t *gpu) 349 { 350 NV_STATUS status; 351 352 if (!uvm_conf_computing_mode_enabled(gpu)) 353 return NV_OK; 354 355 status = conf_computing_dma_buffer_pool_init(&gpu->conf_computing.dma_buffer_pool); 356 if (status != NV_OK) 357 return status; 358 359 status = dummy_iv_mem_init(gpu); 360 if (status != NV_OK) 361 goto error; 362 363 return NV_OK; 364 365 error: 366 uvm_conf_computing_gpu_deinit(gpu); 367 return status; 368 } 369 370 void uvm_conf_computing_gpu_deinit(uvm_gpu_t *gpu) 371 { 372 dummy_iv_mem_deinit(gpu); 373 conf_computing_dma_buffer_pool_deinit(&gpu->conf_computing.dma_buffer_pool); 374 } 375 376 void uvm_conf_computing_log_gpu_encryption(uvm_channel_t *channel, UvmCslIv *iv) 377 { 378 NV_STATUS status; 379 380 uvm_mutex_lock(&channel->csl.ctx_lock); 381 status = nvUvmInterfaceCslLogDeviceEncryption(&channel->csl.ctx, iv); 382 uvm_mutex_unlock(&channel->csl.ctx_lock); 383 384 // nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter 385 // overflows. This is not supposed to happen on CC. 386 UVM_ASSERT(status == NV_OK); 387 } 388 389 void uvm_conf_computing_acquire_encryption_iv(uvm_channel_t *channel, UvmCslIv *iv) 390 { 391 NV_STATUS status; 392 393 uvm_mutex_lock(&channel->csl.ctx_lock); 394 status = nvUvmInterfaceCslAcquireEncryptionIv(&channel->csl.ctx, iv); 395 uvm_mutex_unlock(&channel->csl.ctx_lock); 396 397 // nvUvmInterfaceLogDeviceEncryption fails when a 64-bit encryption counter 398 // overflows. This is not supposed to happen on CC. 399 UVM_ASSERT(status == NV_OK); 400 } 401 402 void uvm_conf_computing_cpu_encrypt(uvm_channel_t *channel, 403 void *dst_cipher, 404 const void *src_plain, 405 UvmCslIv *encrypt_iv, 406 size_t size, 407 void *auth_tag_buffer) 408 { 409 NV_STATUS status; 410 411 UVM_ASSERT(size); 412 413 uvm_mutex_lock(&channel->csl.ctx_lock); 414 status = nvUvmInterfaceCslEncrypt(&channel->csl.ctx, 415 size, 416 (NvU8 const *) src_plain, 417 encrypt_iv, 418 (NvU8 *) dst_cipher, 419 (NvU8 *) auth_tag_buffer); 420 uvm_mutex_unlock(&channel->csl.ctx_lock); 421 422 // nvUvmInterfaceCslEncrypt fails when a 64-bit encryption counter 423 // overflows. This is not supposed to happen on CC. 424 UVM_ASSERT(status == NV_OK); 425 } 426 427 NV_STATUS uvm_conf_computing_cpu_decrypt(uvm_channel_t *channel, 428 void *dst_plain, 429 const void *src_cipher, 430 const UvmCslIv *src_iv, 431 size_t size, 432 const void *auth_tag_buffer) 433 { 434 NV_STATUS status; 435 436 uvm_mutex_lock(&channel->csl.ctx_lock); 437 status = nvUvmInterfaceCslDecrypt(&channel->csl.ctx, 438 size, 439 (const NvU8 *) src_cipher, 440 src_iv, 441 (NvU8 *) dst_plain, 442 (const NvU8 *) auth_tag_buffer); 443 uvm_mutex_unlock(&channel->csl.ctx_lock); 444 445 return status; 446 } 447