1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2013-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /* 25 * This file sets up the communication between the UVM driver and RM. RM will 26 * call the UVM driver providing to it the set of OPS it supports. UVM will 27 * then return by filling out the structure with the callbacks it supports. 28 */ 29 30 #define __NO_VERSION__ 31 32 #include "os-interface.h" 33 #include "nv-linux.h" 34 35 #if defined(NV_UVM_ENABLE) 36 37 #include "nv_uvm_interface.h" 38 #include "nv_gpu_ops.h" 39 #include "rm-gpu-ops.h" 40 41 // This is really a struct UvmOpsUvmEvents *. It needs to be an atomic because 42 // it can be read outside of the g_pNvUvmEventsLock. Use getUvmEvents and 43 // setUvmEvents to access it. 44 static atomic_long_t g_pNvUvmEvents; 45 static struct semaphore g_pNvUvmEventsLock; 46 47 static struct UvmOpsUvmEvents *getUvmEvents(void) 48 { 49 return (struct UvmOpsUvmEvents *)atomic_long_read(&g_pNvUvmEvents); 50 } 51 52 static void setUvmEvents(struct UvmOpsUvmEvents *newEvents) 53 { 54 atomic_long_set(&g_pNvUvmEvents, (long)newEvents); 55 } 56 57 static nvidia_stack_t *g_sp; 58 static struct semaphore g_spLock; 59 60 // Use these to test g_sp usage. When DEBUG_GLOBAL_STACK, one out of every 61 // DEBUG_GLOBAL_STACK_THRESHOLD calls to nvUvmGetSafeStack will use g_sp. 62 #define DEBUG_GLOBAL_STACK 0 63 #define DEBUG_GLOBAL_STACK_THRESHOLD 2 64 65 static atomic_t g_debugGlobalStackCount = ATOMIC_INIT(0); 66 67 // Called at module load, not by an external client 68 int nv_uvm_init(void) 69 { 70 int rc = nv_kmem_cache_alloc_stack(&g_sp); 71 if (rc != 0) 72 return rc; 73 74 NV_INIT_MUTEX(&g_spLock); 75 NV_INIT_MUTEX(&g_pNvUvmEventsLock); 76 return 0; 77 } 78 79 void nv_uvm_exit(void) 80 { 81 // If this fires, the dependent driver never unregistered its callbacks with 82 // us before going away, leaving us potentially making callbacks to garbage 83 // memory. 84 WARN_ON(getUvmEvents() != NULL); 85 86 nv_kmem_cache_free_stack(g_sp); 87 } 88 89 90 // Testing code to force use of the global stack every now and then 91 static NvBool forceGlobalStack(void) 92 { 93 // Make sure that we do not try to allocate memory in interrupt or atomic 94 // context 95 if (DEBUG_GLOBAL_STACK || !NV_MAY_SLEEP()) 96 { 97 if ((atomic_inc_return(&g_debugGlobalStackCount) % 98 DEBUG_GLOBAL_STACK_THRESHOLD) == 0) 99 return NV_TRUE; 100 } 101 return NV_FALSE; 102 } 103 104 // Guaranteed to always return a valid stack. It first attempts to allocate one 105 // from the pool. If that fails, it falls back to the global pre-allocated 106 // stack. This fallback will serialize. 107 // 108 // This is required so paths that free resources do not themselves require 109 // allocation of resources. 110 static nvidia_stack_t *nvUvmGetSafeStack(void) 111 { 112 nvidia_stack_t *sp; 113 if (forceGlobalStack() || nv_kmem_cache_alloc_stack(&sp) != 0) 114 { 115 sp = g_sp; 116 down(&g_spLock); 117 } 118 return sp; 119 } 120 121 static void nvUvmFreeSafeStack(nvidia_stack_t *sp) 122 { 123 if (sp == g_sp) 124 up(&g_spLock); 125 else 126 nv_kmem_cache_free_stack(sp); 127 } 128 129 static NV_STATUS nvUvmDestroyFaultInfoAndStacks(nvidia_stack_t *sp, 130 uvmGpuDeviceHandle device, 131 UvmGpuFaultInfo *pFaultInfo) 132 { 133 nv_kmem_cache_free_stack(pFaultInfo->replayable.cslCtx.nvidia_stack); 134 nv_kmem_cache_free_stack(pFaultInfo->nonReplayable.isr_bh_sp); 135 nv_kmem_cache_free_stack(pFaultInfo->nonReplayable.isr_sp); 136 137 return rm_gpu_ops_destroy_fault_info(sp, 138 (gpuDeviceHandle)device, 139 pFaultInfo); 140 } 141 142 NV_STATUS nvUvmInterfaceRegisterGpu(const NvProcessorUuid *gpuUuid, UvmGpuPlatformInfo *gpuInfo) 143 { 144 nvidia_stack_t *sp = NULL; 145 NV_STATUS status; 146 int rc; 147 148 if (nv_kmem_cache_alloc_stack(&sp) != 0) 149 return NV_ERR_NO_MEMORY; 150 151 rc = nvidia_dev_get_uuid(gpuUuid->uuid, sp); 152 if (rc == 0) 153 { 154 rc = nvidia_dev_get_pci_info(gpuUuid->uuid, 155 &gpuInfo->pci_dev, 156 &gpuInfo->dma_addressable_start, 157 &gpuInfo->dma_addressable_limit); 158 159 // Block GPU from entering GC6 while used by UVM. 160 if (rc == 0) 161 rc = nvidia_dev_block_gc6(gpuUuid->uuid, sp); 162 163 // Avoid leaking reference on GPU if we failed. 164 if (rc != 0) 165 nvidia_dev_put_uuid(gpuUuid->uuid, sp); 166 } 167 168 switch (rc) 169 { 170 case 0: 171 status = NV_OK; 172 break; 173 case -ENOMEM: 174 status = NV_ERR_NO_MEMORY; 175 break; 176 case -ENODEV: 177 status = NV_ERR_GPU_UUID_NOT_FOUND; 178 break; 179 default: 180 status = NV_ERR_GENERIC; 181 break; 182 } 183 184 nv_kmem_cache_free_stack(sp); 185 return status; 186 } 187 EXPORT_SYMBOL(nvUvmInterfaceRegisterGpu); 188 189 void nvUvmInterfaceUnregisterGpu(const NvProcessorUuid *gpuUuid) 190 { 191 nvidia_stack_t *sp = nvUvmGetSafeStack(); 192 nvidia_dev_unblock_gc6(gpuUuid->uuid, sp); 193 nvidia_dev_put_uuid(gpuUuid->uuid, sp); 194 nvUvmFreeSafeStack(sp); 195 } 196 EXPORT_SYMBOL(nvUvmInterfaceUnregisterGpu); 197 198 NV_STATUS nvUvmInterfaceSessionCreate(uvmGpuSessionHandle *session, 199 UvmPlatformInfo *platformInfo) 200 { 201 nvidia_stack_t *sp = NULL; 202 NV_STATUS status; 203 204 if (nv_kmem_cache_alloc_stack(&sp) != 0) 205 { 206 return NV_ERR_NO_MEMORY; 207 } 208 209 memset(platformInfo, 0, sizeof(*platformInfo)); 210 platformInfo->atsSupported = nv_ats_supported; 211 212 platformInfo->sevEnabled = os_sev_enabled; 213 214 status = rm_gpu_ops_create_session(sp, (gpuSessionHandle *)session); 215 216 nv_kmem_cache_free_stack(sp); 217 return status; 218 } 219 EXPORT_SYMBOL(nvUvmInterfaceSessionCreate); 220 221 NV_STATUS nvUvmInterfaceSessionDestroy(uvmGpuSessionHandle session) 222 { 223 nvidia_stack_t *sp = nvUvmGetSafeStack(); 224 NV_STATUS status; 225 226 status = rm_gpu_ops_destroy_session(sp, (gpuSessionHandle)session); 227 228 nvUvmFreeSafeStack(sp); 229 return status; 230 } 231 EXPORT_SYMBOL(nvUvmInterfaceSessionDestroy); 232 233 NV_STATUS nvUvmInterfaceDeviceCreate(uvmGpuSessionHandle session, 234 const UvmGpuInfo *pGpuInfo, 235 const NvProcessorUuid *gpuUuid, 236 uvmGpuDeviceHandle *device, 237 NvBool bCreateSmcPartition) 238 { 239 nvidia_stack_t *sp = NULL; 240 NV_STATUS status; 241 242 if (nv_kmem_cache_alloc_stack(&sp) != 0) 243 { 244 return NV_ERR_NO_MEMORY; 245 } 246 247 status = rm_gpu_ops_device_create(sp, 248 (gpuSessionHandle)session, 249 (const gpuInfo *)pGpuInfo, 250 gpuUuid, 251 (gpuDeviceHandle *)device, 252 bCreateSmcPartition); 253 254 nv_kmem_cache_free_stack(sp); 255 return status; 256 } 257 EXPORT_SYMBOL(nvUvmInterfaceDeviceCreate); 258 259 void nvUvmInterfaceDeviceDestroy(uvmGpuDeviceHandle device) 260 { 261 nvidia_stack_t *sp = nvUvmGetSafeStack(); 262 263 rm_gpu_ops_device_destroy(sp, (gpuDeviceHandle)device); 264 265 nvUvmFreeSafeStack(sp); 266 } 267 EXPORT_SYMBOL(nvUvmInterfaceDeviceDestroy); 268 269 NV_STATUS nvUvmInterfaceDupAddressSpace(uvmGpuDeviceHandle device, 270 NvHandle hUserClient, 271 NvHandle hUserVASpace, 272 uvmGpuAddressSpaceHandle *vaSpace, 273 UvmGpuAddressSpaceInfo *vaSpaceInfo) 274 { 275 nvidia_stack_t *sp = NULL; 276 NV_STATUS status; 277 278 if (nv_kmem_cache_alloc_stack(&sp) != 0) 279 { 280 return NV_ERR_NO_MEMORY; 281 } 282 283 status = rm_gpu_ops_dup_address_space(sp, 284 (gpuDeviceHandle)device, 285 hUserClient, 286 hUserVASpace, 287 (gpuAddressSpaceHandle *)vaSpace, 288 vaSpaceInfo); 289 290 nv_kmem_cache_free_stack(sp); 291 return status; 292 } 293 EXPORT_SYMBOL(nvUvmInterfaceDupAddressSpace); 294 295 NV_STATUS nvUvmInterfaceAddressSpaceCreate(uvmGpuDeviceHandle device, 296 unsigned long long vaBase, 297 unsigned long long vaSize, 298 uvmGpuAddressSpaceHandle *vaSpace, 299 UvmGpuAddressSpaceInfo *vaSpaceInfo) 300 { 301 nvidia_stack_t *sp = NULL; 302 NV_STATUS status; 303 304 if (nv_kmem_cache_alloc_stack(&sp) != 0) 305 { 306 return NV_ERR_NO_MEMORY; 307 } 308 309 status = rm_gpu_ops_address_space_create(sp, 310 (gpuDeviceHandle)device, 311 vaBase, 312 vaSize, 313 (gpuAddressSpaceHandle *)vaSpace, 314 vaSpaceInfo); 315 316 nv_kmem_cache_free_stack(sp); 317 return status; 318 } 319 EXPORT_SYMBOL(nvUvmInterfaceAddressSpaceCreate); 320 321 void nvUvmInterfaceAddressSpaceDestroy(uvmGpuAddressSpaceHandle vaSpace) 322 { 323 nvidia_stack_t *sp = nvUvmGetSafeStack(); 324 325 rm_gpu_ops_address_space_destroy( 326 sp, (gpuAddressSpaceHandle)vaSpace); 327 328 nvUvmFreeSafeStack(sp); 329 } 330 EXPORT_SYMBOL(nvUvmInterfaceAddressSpaceDestroy); 331 332 NV_STATUS nvUvmInterfaceMemoryAllocFB(uvmGpuAddressSpaceHandle vaSpace, 333 NvLength length, UvmGpuPointer * gpuPointer, 334 UvmGpuAllocInfo * allocInfo) 335 { 336 nvidia_stack_t *sp = NULL; 337 NV_STATUS status; 338 339 if (nv_kmem_cache_alloc_stack(&sp) != 0) 340 { 341 return NV_ERR_NO_MEMORY; 342 } 343 344 status = rm_gpu_ops_memory_alloc_fb( 345 sp, (gpuAddressSpaceHandle)vaSpace, 346 length, (NvU64 *) gpuPointer, 347 allocInfo); 348 349 nv_kmem_cache_free_stack(sp); 350 return status; 351 } 352 EXPORT_SYMBOL(nvUvmInterfaceMemoryAllocFB); 353 354 NV_STATUS nvUvmInterfaceMemoryAllocSys(uvmGpuAddressSpaceHandle vaSpace, 355 NvLength length, UvmGpuPointer * gpuPointer, 356 UvmGpuAllocInfo * allocInfo) 357 { 358 nvidia_stack_t *sp = NULL; 359 NV_STATUS status; 360 361 if (nv_kmem_cache_alloc_stack(&sp) != 0) 362 { 363 return NV_ERR_NO_MEMORY; 364 } 365 366 status = rm_gpu_ops_memory_alloc_sys( 367 sp, (gpuAddressSpaceHandle)vaSpace, 368 length, (NvU64 *) gpuPointer, 369 allocInfo); 370 371 nv_kmem_cache_free_stack(sp); 372 return status; 373 } 374 375 EXPORT_SYMBOL(nvUvmInterfaceMemoryAllocSys); 376 377 NV_STATUS nvUvmInterfaceGetP2PCaps(uvmGpuDeviceHandle device1, 378 uvmGpuDeviceHandle device2, 379 UvmGpuP2PCapsParams * p2pCapsParams) 380 { 381 nvidia_stack_t *sp = NULL; 382 NV_STATUS status; 383 384 if (nv_kmem_cache_alloc_stack(&sp) != 0) 385 { 386 return NV_ERR_NO_MEMORY; 387 } 388 389 status = rm_gpu_ops_get_p2p_caps(sp, 390 (gpuDeviceHandle)device1, 391 (gpuDeviceHandle)device2, 392 p2pCapsParams); 393 nv_kmem_cache_free_stack(sp); 394 return status; 395 } 396 397 EXPORT_SYMBOL(nvUvmInterfaceGetP2PCaps); 398 399 NV_STATUS nvUvmInterfaceGetPmaObject(uvmGpuDeviceHandle device, 400 void **pPma, 401 const UvmPmaStatistics **pPmaPubStats) 402 { 403 nvidia_stack_t *sp = NULL; 404 NV_STATUS status; 405 406 if (nv_kmem_cache_alloc_stack(&sp) != 0) 407 { 408 return NV_ERR_NO_MEMORY; 409 } 410 411 status = rm_gpu_ops_get_pma_object(sp, (gpuDeviceHandle)device, pPma, (const nvgpuPmaStatistics_t *)pPmaPubStats); 412 413 nv_kmem_cache_free_stack(sp); 414 return status; 415 } 416 417 EXPORT_SYMBOL(nvUvmInterfaceGetPmaObject); 418 419 NV_STATUS nvUvmInterfacePmaRegisterEvictionCallbacks(void *pPma, 420 uvmPmaEvictPagesCallback evictPages, 421 uvmPmaEvictRangeCallback evictRange, 422 void *callbackData) 423 { 424 nvidia_stack_t *sp = NULL; 425 NV_STATUS status; 426 427 if (nv_kmem_cache_alloc_stack(&sp) != 0) 428 { 429 return NV_ERR_NO_MEMORY; 430 } 431 432 status = rm_gpu_ops_pma_register_callbacks(sp, pPma, evictPages, evictRange, callbackData); 433 434 nv_kmem_cache_free_stack(sp); 435 return status; 436 } 437 EXPORT_SYMBOL(nvUvmInterfacePmaRegisterEvictionCallbacks); 438 439 void nvUvmInterfacePmaUnregisterEvictionCallbacks(void *pPma) 440 { 441 nvidia_stack_t *sp = nvUvmGetSafeStack(); 442 443 rm_gpu_ops_pma_unregister_callbacks(sp, pPma); 444 445 nvUvmFreeSafeStack(sp); 446 } 447 EXPORT_SYMBOL(nvUvmInterfacePmaUnregisterEvictionCallbacks); 448 449 NV_STATUS nvUvmInterfacePmaAllocPages(void *pPma, 450 NvLength pageCount, 451 NvU64 pageSize, 452 UvmPmaAllocationOptions *pPmaAllocOptions, 453 NvU64 *pPages) 454 { 455 nvidia_stack_t *sp = NULL; 456 NV_STATUS status; 457 458 if (nv_kmem_cache_alloc_stack(&sp) != 0) 459 { 460 return NV_ERR_NO_MEMORY; 461 } 462 463 status = rm_gpu_ops_pma_alloc_pages( 464 sp, pPma, 465 pageCount, 466 pageSize, 467 (nvgpuPmaAllocationOptions_t)pPmaAllocOptions, 468 pPages); 469 470 nv_kmem_cache_free_stack(sp); 471 return status; 472 } 473 EXPORT_SYMBOL(nvUvmInterfacePmaAllocPages); 474 475 NV_STATUS nvUvmInterfacePmaPinPages(void *pPma, 476 NvU64 *pPages, 477 NvLength pageCount, 478 NvU64 pageSize, 479 NvU32 flags) 480 { 481 nvidia_stack_t *sp = NULL; 482 NV_STATUS status; 483 484 if (nv_kmem_cache_alloc_stack(&sp) != 0) 485 { 486 return NV_ERR_NO_MEMORY; 487 } 488 489 status = rm_gpu_ops_pma_pin_pages(sp, pPma, pPages, pageCount, pageSize, flags); 490 491 nv_kmem_cache_free_stack(sp); 492 return status; 493 } 494 EXPORT_SYMBOL(nvUvmInterfacePmaPinPages); 495 496 NV_STATUS nvUvmInterfacePmaUnpinPages(void *pPma, 497 NvU64 *pPages, 498 NvLength pageCount, 499 NvU64 pageSize) 500 { 501 nvidia_stack_t *sp = NULL; 502 NV_STATUS status; 503 504 if (nv_kmem_cache_alloc_stack(&sp) != 0) 505 { 506 return NV_ERR_NO_MEMORY; 507 } 508 509 status = rm_gpu_ops_pma_unpin_pages(sp, pPma, pPages, pageCount, pageSize); 510 511 nv_kmem_cache_free_stack(sp); 512 return status; 513 } 514 EXPORT_SYMBOL(nvUvmInterfacePmaUnpinPages); 515 516 void nvUvmInterfaceMemoryFree(uvmGpuAddressSpaceHandle vaSpace, 517 UvmGpuPointer gpuPointer) 518 { 519 nvidia_stack_t *sp = nvUvmGetSafeStack(); 520 521 rm_gpu_ops_memory_free( 522 sp, (gpuAddressSpaceHandle)vaSpace, 523 (NvU64) gpuPointer); 524 525 nvUvmFreeSafeStack(sp); 526 } 527 EXPORT_SYMBOL(nvUvmInterfaceMemoryFree); 528 529 void nvUvmInterfacePmaFreePages(void *pPma, 530 NvU64 *pPages, 531 NvLength pageCount, 532 NvU64 pageSize, 533 NvU32 flags) 534 { 535 nvidia_stack_t *sp = nvUvmGetSafeStack(); 536 537 rm_gpu_ops_pma_free_pages(sp, pPma, pPages, pageCount, pageSize, flags); 538 539 nvUvmFreeSafeStack(sp); 540 } 541 EXPORT_SYMBOL(nvUvmInterfacePmaFreePages); 542 543 NV_STATUS nvUvmInterfaceMemoryCpuMap(uvmGpuAddressSpaceHandle vaSpace, 544 UvmGpuPointer gpuPointer, NvLength length, void **cpuPtr, 545 NvU64 pageSize) 546 { 547 nvidia_stack_t *sp = NULL; 548 NV_STATUS status; 549 550 if (nv_kmem_cache_alloc_stack(&sp) != 0) 551 { 552 return NV_ERR_NO_MEMORY; 553 } 554 555 status = rm_gpu_ops_memory_cpu_map( 556 sp, (gpuAddressSpaceHandle)vaSpace, 557 (NvU64) gpuPointer, length, cpuPtr, pageSize); 558 559 nv_kmem_cache_free_stack(sp); 560 return status; 561 } 562 EXPORT_SYMBOL(nvUvmInterfaceMemoryCpuMap); 563 564 void nvUvmInterfaceMemoryCpuUnMap(uvmGpuAddressSpaceHandle vaSpace, 565 void *cpuPtr) 566 { 567 nvidia_stack_t *sp = nvUvmGetSafeStack(); 568 rm_gpu_ops_memory_cpu_ummap(sp, (gpuAddressSpaceHandle)vaSpace, cpuPtr); 569 nvUvmFreeSafeStack(sp); 570 } 571 EXPORT_SYMBOL(nvUvmInterfaceMemoryCpuUnMap); 572 573 NV_STATUS nvUvmInterfaceTsgAllocate(uvmGpuAddressSpaceHandle vaSpace, 574 const UvmGpuTsgAllocParams *allocParams, 575 uvmGpuTsgHandle *tsg) 576 { 577 nvidia_stack_t *sp = NULL; 578 NV_STATUS status; 579 580 if (nv_kmem_cache_alloc_stack(&sp) != 0) 581 { 582 return NV_ERR_NO_MEMORY; 583 } 584 585 status = rm_gpu_ops_tsg_allocate(sp, 586 (gpuAddressSpaceHandle)vaSpace, 587 allocParams, 588 (gpuTsgHandle *)tsg); 589 590 nv_kmem_cache_free_stack(sp); 591 592 return status; 593 } 594 EXPORT_SYMBOL(nvUvmInterfaceTsgAllocate); 595 596 void nvUvmInterfaceTsgDestroy(uvmGpuTsgHandle tsg) 597 { 598 nvidia_stack_t *sp = nvUvmGetSafeStack(); 599 rm_gpu_ops_tsg_destroy(sp, (gpuTsgHandle)tsg); 600 nvUvmFreeSafeStack(sp); 601 } 602 EXPORT_SYMBOL(nvUvmInterfaceTsgDestroy); 603 604 605 NV_STATUS nvUvmInterfaceChannelAllocate(const uvmGpuTsgHandle tsg, 606 const UvmGpuChannelAllocParams *allocParams, 607 uvmGpuChannelHandle *channel, 608 UvmGpuChannelInfo *channelInfo) 609 { 610 nvidia_stack_t *sp = NULL; 611 NV_STATUS status; 612 613 if (nv_kmem_cache_alloc_stack(&sp) != 0) 614 { 615 return NV_ERR_NO_MEMORY; 616 } 617 618 status = rm_gpu_ops_channel_allocate(sp, 619 (gpuTsgHandle)tsg, 620 allocParams, 621 (gpuChannelHandle *)channel, 622 channelInfo); 623 624 nv_kmem_cache_free_stack(sp); 625 626 return status; 627 } 628 EXPORT_SYMBOL(nvUvmInterfaceChannelAllocate); 629 630 void nvUvmInterfaceChannelDestroy(uvmGpuChannelHandle channel) 631 { 632 nvidia_stack_t *sp = nvUvmGetSafeStack(); 633 rm_gpu_ops_channel_destroy(sp, (gpuChannelHandle)channel); 634 nvUvmFreeSafeStack(sp); 635 } 636 EXPORT_SYMBOL(nvUvmInterfaceChannelDestroy); 637 638 NV_STATUS nvUvmInterfaceQueryCaps(uvmGpuDeviceHandle device, 639 UvmGpuCaps * caps) 640 { 641 nvidia_stack_t *sp = NULL; 642 NV_STATUS status; 643 644 if (nv_kmem_cache_alloc_stack(&sp) != 0) 645 { 646 return NV_ERR_NO_MEMORY; 647 } 648 649 status = rm_gpu_ops_query_caps(sp, (gpuDeviceHandle)device, caps); 650 651 nv_kmem_cache_free_stack(sp); 652 return status; 653 } 654 EXPORT_SYMBOL(nvUvmInterfaceQueryCaps); 655 656 NV_STATUS nvUvmInterfaceQueryCopyEnginesCaps(uvmGpuDeviceHandle device, 657 UvmGpuCopyEnginesCaps *caps) 658 { 659 nvidia_stack_t *sp = NULL; 660 NV_STATUS status; 661 662 if (nv_kmem_cache_alloc_stack(&sp) != 0) 663 { 664 return NV_ERR_NO_MEMORY; 665 } 666 667 status = rm_gpu_ops_query_ces_caps(sp, (gpuDeviceHandle)device, caps); 668 669 nv_kmem_cache_free_stack(sp); 670 return status; 671 } 672 EXPORT_SYMBOL(nvUvmInterfaceQueryCopyEnginesCaps); 673 674 NV_STATUS nvUvmInterfaceGetGpuInfo(const NvProcessorUuid *gpuUuid, 675 const UvmGpuClientInfo *pGpuClientInfo, 676 UvmGpuInfo *pGpuInfo) 677 { 678 nvidia_stack_t *sp = NULL; 679 NV_STATUS status; 680 681 if (nv_kmem_cache_alloc_stack(&sp) != 0) 682 { 683 return NV_ERR_NO_MEMORY; 684 } 685 686 status = rm_gpu_ops_get_gpu_info(sp, gpuUuid, pGpuClientInfo, pGpuInfo); 687 688 nv_kmem_cache_free_stack(sp); 689 return status; 690 } 691 EXPORT_SYMBOL(nvUvmInterfaceGetGpuInfo); 692 693 NV_STATUS nvUvmInterfaceServiceDeviceInterruptsRM(uvmGpuDeviceHandle device) 694 { 695 nvidia_stack_t *sp = NULL; 696 NV_STATUS status; 697 698 if (nv_kmem_cache_alloc_stack(&sp) != 0) 699 { 700 return NV_ERR_NO_MEMORY; 701 } 702 703 status = rm_gpu_ops_service_device_interrupts_rm(sp, 704 (gpuDeviceHandle)device); 705 706 nv_kmem_cache_free_stack(sp); 707 return status; 708 } 709 EXPORT_SYMBOL(nvUvmInterfaceServiceDeviceInterruptsRM); 710 711 NV_STATUS nvUvmInterfaceSetPageDirectory(uvmGpuAddressSpaceHandle vaSpace, 712 NvU64 physAddress, unsigned numEntries, 713 NvBool bVidMemAperture, NvU32 pasid) 714 { 715 nvidia_stack_t *sp = NULL; 716 NV_STATUS status; 717 718 if (nv_kmem_cache_alloc_stack(&sp) != 0) 719 { 720 return NV_ERR_NO_MEMORY; 721 } 722 723 status = rm_gpu_ops_set_page_directory(sp, (gpuAddressSpaceHandle)vaSpace, 724 physAddress, numEntries, bVidMemAperture, pasid); 725 726 nv_kmem_cache_free_stack(sp); 727 return status; 728 } 729 EXPORT_SYMBOL(nvUvmInterfaceSetPageDirectory); 730 731 NV_STATUS nvUvmInterfaceUnsetPageDirectory(uvmGpuAddressSpaceHandle vaSpace) 732 { 733 nvidia_stack_t *sp = nvUvmGetSafeStack(); 734 NV_STATUS status; 735 736 status = 737 rm_gpu_ops_unset_page_directory(sp, (gpuAddressSpaceHandle)vaSpace); 738 nvUvmFreeSafeStack(sp); 739 return status; 740 } 741 EXPORT_SYMBOL(nvUvmInterfaceUnsetPageDirectory); 742 743 NV_STATUS nvUvmInterfaceDupAllocation(uvmGpuAddressSpaceHandle srcVaSpace, 744 NvU64 srcAddress, 745 uvmGpuAddressSpaceHandle dstVaSpace, 746 NvU64 dstVaAlignment, 747 NvU64 *dstAddress) 748 { 749 nvidia_stack_t *sp = NULL; 750 NV_STATUS status; 751 752 if (nv_kmem_cache_alloc_stack(&sp) != 0) 753 { 754 return NV_ERR_NO_MEMORY; 755 } 756 757 status = rm_gpu_ops_dup_allocation(sp, 758 (gpuAddressSpaceHandle)srcVaSpace, 759 srcAddress, 760 (gpuAddressSpaceHandle)dstVaSpace, 761 dstVaAlignment, 762 dstAddress); 763 764 nv_kmem_cache_free_stack(sp); 765 return status; 766 } 767 EXPORT_SYMBOL(nvUvmInterfaceDupAllocation); 768 769 NV_STATUS nvUvmInterfaceDupMemory(uvmGpuDeviceHandle device, 770 NvHandle hClient, 771 NvHandle hPhysMemory, 772 NvHandle *hDupMemory, 773 UvmGpuMemoryInfo *pGpuMemoryInfo) 774 { 775 nvidia_stack_t *sp = NULL; 776 NV_STATUS status; 777 778 if (nv_kmem_cache_alloc_stack(&sp) != 0) 779 { 780 return NV_ERR_NO_MEMORY; 781 } 782 783 status = rm_gpu_ops_dup_memory(sp, 784 (gpuDeviceHandle)device, 785 hClient, 786 hPhysMemory, 787 hDupMemory, 788 pGpuMemoryInfo); 789 790 nv_kmem_cache_free_stack(sp); 791 return status; 792 } 793 EXPORT_SYMBOL(nvUvmInterfaceDupMemory); 794 795 796 NV_STATUS nvUvmInterfaceFreeDupedHandle(uvmGpuDeviceHandle device, 797 NvHandle hPhysHandle) 798 { 799 nvidia_stack_t *sp = nvUvmGetSafeStack(); 800 NV_STATUS status; 801 802 status = rm_gpu_ops_free_duped_handle(sp, 803 (gpuDeviceHandle)device, 804 hPhysHandle); 805 806 nvUvmFreeSafeStack(sp); 807 return status; 808 } 809 EXPORT_SYMBOL(nvUvmInterfaceFreeDupedHandle); 810 811 NV_STATUS nvUvmInterfaceGetFbInfo(uvmGpuDeviceHandle device, 812 UvmGpuFbInfo * fbInfo) 813 { 814 nvidia_stack_t *sp = NULL; 815 NV_STATUS status; 816 817 if (nv_kmem_cache_alloc_stack(&sp) != 0) 818 { 819 return NV_ERR_NO_MEMORY; 820 } 821 822 status = rm_gpu_ops_get_fb_info(sp, (gpuDeviceHandle)device, fbInfo); 823 824 nv_kmem_cache_free_stack(sp); 825 826 return status; 827 } 828 EXPORT_SYMBOL(nvUvmInterfaceGetFbInfo); 829 830 NV_STATUS nvUvmInterfaceGetEccInfo(uvmGpuDeviceHandle device, 831 UvmGpuEccInfo * eccInfo) 832 { 833 nvidia_stack_t *sp = NULL; 834 NV_STATUS status; 835 836 if (nv_kmem_cache_alloc_stack(&sp) != 0) 837 { 838 return NV_ERR_NO_MEMORY; 839 } 840 841 status = rm_gpu_ops_get_ecc_info(sp, (gpuDeviceHandle)device, eccInfo); 842 843 nv_kmem_cache_free_stack(sp); 844 845 return status; 846 } 847 EXPORT_SYMBOL(nvUvmInterfaceGetEccInfo); 848 849 NV_STATUS nvUvmInterfaceOwnPageFaultIntr(uvmGpuDeviceHandle device, NvBool bOwnInterrupts) 850 { 851 nvidia_stack_t *sp = NULL; 852 NV_STATUS status; 853 854 if (nv_kmem_cache_alloc_stack(&sp) != 0) 855 { 856 return NV_ERR_NO_MEMORY; 857 } 858 859 status = rm_gpu_ops_own_page_fault_intr(sp, (gpuDeviceHandle)device, bOwnInterrupts); 860 nv_kmem_cache_free_stack(sp); 861 return status; 862 } 863 EXPORT_SYMBOL(nvUvmInterfaceOwnPageFaultIntr); 864 865 866 NV_STATUS nvUvmInterfaceInitFaultInfo(uvmGpuDeviceHandle device, 867 UvmGpuFaultInfo *pFaultInfo) 868 { 869 nvidia_stack_t *sp = NULL; 870 NV_STATUS status; 871 int err; 872 873 if (nv_kmem_cache_alloc_stack(&sp) != 0) 874 { 875 return NV_ERR_NO_MEMORY; 876 } 877 878 status = rm_gpu_ops_init_fault_info(sp, 879 (gpuDeviceHandle)device, 880 pFaultInfo); 881 if (status != NV_OK) 882 { 883 goto done; 884 } 885 886 // Preallocate a stack for functions called from ISR top half 887 pFaultInfo->nonReplayable.isr_sp = NULL; 888 pFaultInfo->nonReplayable.isr_bh_sp = NULL; 889 pFaultInfo->replayable.cslCtx.nvidia_stack = NULL; 890 891 // NOTE: nv_kmem_cache_alloc_stack does not allocate a stack on PPC. 892 // Therefore, the pointer can be NULL on success. Always use the 893 // returned error code to determine if the operation was successful. 894 err = nv_kmem_cache_alloc_stack((nvidia_stack_t **)&pFaultInfo->nonReplayable.isr_sp); 895 if (err) 896 { 897 goto error; 898 } 899 900 err = nv_kmem_cache_alloc_stack((nvidia_stack_t **)&pFaultInfo->nonReplayable.isr_bh_sp); 901 if (err) 902 { 903 goto error; 904 } 905 906 // The cslCtx.ctx pointer is not NULL only when ConfidentialComputing is enabled. 907 if (pFaultInfo->replayable.cslCtx.ctx != NULL) 908 { 909 err = nv_kmem_cache_alloc_stack((nvidia_stack_t **)&pFaultInfo->replayable.cslCtx.nvidia_stack); 910 if (err) 911 { 912 goto error; 913 } 914 } 915 goto done; 916 917 error: 918 nvUvmDestroyFaultInfoAndStacks(sp, 919 device, 920 pFaultInfo); 921 status = NV_ERR_NO_MEMORY; 922 done: 923 nv_kmem_cache_free_stack(sp); 924 return status; 925 } 926 EXPORT_SYMBOL(nvUvmInterfaceInitFaultInfo); 927 928 NV_STATUS nvUvmInterfaceInitAccessCntrInfo(uvmGpuDeviceHandle device, 929 UvmGpuAccessCntrInfo *pAccessCntrInfo, 930 NvU32 accessCntrIndex) 931 { 932 nvidia_stack_t *sp = NULL; 933 NV_STATUS status; 934 935 if (nv_kmem_cache_alloc_stack(&sp) != 0) 936 { 937 return NV_ERR_NO_MEMORY; 938 } 939 940 status = rm_gpu_ops_init_access_cntr_info(sp, 941 (gpuDeviceHandle)device, 942 pAccessCntrInfo, 943 accessCntrIndex); 944 945 nv_kmem_cache_free_stack(sp); 946 return status; 947 } 948 EXPORT_SYMBOL(nvUvmInterfaceInitAccessCntrInfo); 949 950 NV_STATUS nvUvmInterfaceEnableAccessCntr(uvmGpuDeviceHandle device, 951 UvmGpuAccessCntrInfo *pAccessCntrInfo, 952 UvmGpuAccessCntrConfig *pAccessCntrConfig) 953 { 954 nvidia_stack_t *sp = NULL; 955 NV_STATUS status; 956 957 if (nv_kmem_cache_alloc_stack(&sp) != 0) 958 { 959 return NV_ERR_NO_MEMORY; 960 } 961 962 status = rm_gpu_ops_enable_access_cntr (sp, 963 (gpuDeviceHandle)device, 964 pAccessCntrInfo, 965 pAccessCntrConfig); 966 967 nv_kmem_cache_free_stack(sp); 968 return status; 969 } 970 EXPORT_SYMBOL(nvUvmInterfaceEnableAccessCntr); 971 972 NV_STATUS nvUvmInterfaceDestroyFaultInfo(uvmGpuDeviceHandle device, 973 UvmGpuFaultInfo *pFaultInfo) 974 { 975 nvidia_stack_t *sp = nvUvmGetSafeStack(); 976 NV_STATUS status; 977 978 status = nvUvmDestroyFaultInfoAndStacks(sp, 979 device, 980 pFaultInfo); 981 nvUvmFreeSafeStack(sp); 982 return status; 983 } 984 EXPORT_SYMBOL(nvUvmInterfaceDestroyFaultInfo); 985 986 NV_STATUS nvUvmInterfaceHasPendingNonReplayableFaults(UvmGpuFaultInfo *pFaultInfo, 987 NvBool *hasPendingFaults) 988 { 989 return rm_gpu_ops_has_pending_non_replayable_faults(pFaultInfo->nonReplayable.isr_sp, 990 pFaultInfo, 991 hasPendingFaults); 992 } 993 EXPORT_SYMBOL(nvUvmInterfaceHasPendingNonReplayableFaults); 994 995 NV_STATUS nvUvmInterfaceGetNonReplayableFaults(UvmGpuFaultInfo *pFaultInfo, 996 void *pFaultBuffer, 997 NvU32 *numFaults) 998 { 999 return rm_gpu_ops_get_non_replayable_faults(pFaultInfo->nonReplayable.isr_bh_sp, 1000 pFaultInfo, 1001 pFaultBuffer, 1002 numFaults); 1003 } 1004 EXPORT_SYMBOL(nvUvmInterfaceGetNonReplayableFaults); 1005 1006 NV_STATUS nvUvmInterfaceFlushReplayableFaultBuffer(uvmGpuDeviceHandle device) 1007 { 1008 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1009 NV_STATUS status; 1010 1011 status = rm_gpu_ops_flush_replayable_fault_buffer(sp, (gpuDeviceHandle)device); 1012 1013 nvUvmFreeSafeStack(sp); 1014 return status; 1015 } 1016 EXPORT_SYMBOL(nvUvmInterfaceFlushReplayableFaultBuffer); 1017 1018 NV_STATUS nvUvmInterfaceDestroyAccessCntrInfo(uvmGpuDeviceHandle device, 1019 UvmGpuAccessCntrInfo *pAccessCntrInfo) 1020 { 1021 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1022 NV_STATUS status; 1023 1024 status = rm_gpu_ops_destroy_access_cntr_info(sp, 1025 (gpuDeviceHandle)device, 1026 pAccessCntrInfo); 1027 1028 nvUvmFreeSafeStack(sp); 1029 return status; 1030 } 1031 EXPORT_SYMBOL(nvUvmInterfaceDestroyAccessCntrInfo); 1032 1033 NV_STATUS nvUvmInterfaceDisableAccessCntr(uvmGpuDeviceHandle device, 1034 UvmGpuAccessCntrInfo *pAccessCntrInfo) 1035 { 1036 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1037 NV_STATUS status; 1038 1039 status = rm_gpu_ops_disable_access_cntr(sp, 1040 (gpuDeviceHandle)device, 1041 pAccessCntrInfo); 1042 1043 nvUvmFreeSafeStack(sp); 1044 return status; 1045 } 1046 EXPORT_SYMBOL(nvUvmInterfaceDisableAccessCntr); 1047 1048 // this function is called by the UVM driver to register the ops 1049 NV_STATUS nvUvmInterfaceRegisterUvmCallbacks(struct UvmOpsUvmEvents *importedUvmOps) 1050 { 1051 NV_STATUS status = NV_OK; 1052 1053 if (!importedUvmOps) 1054 { 1055 return NV_ERR_INVALID_ARGUMENT; 1056 } 1057 1058 down(&g_pNvUvmEventsLock); 1059 if (getUvmEvents() != NULL) 1060 { 1061 status = NV_ERR_IN_USE; 1062 } 1063 else 1064 { 1065 // Be careful: as soon as the pointer is assigned, top half ISRs can 1066 // start reading it to make callbacks, even before we drop the lock. 1067 setUvmEvents(importedUvmOps); 1068 } 1069 up(&g_pNvUvmEventsLock); 1070 1071 return status; 1072 } 1073 EXPORT_SYMBOL(nvUvmInterfaceRegisterUvmCallbacks); 1074 1075 static void flush_top_half(void *info) 1076 { 1077 // Prior top halves on this core must have completed for this callback to 1078 // run at all, so we're done. 1079 return; 1080 } 1081 1082 void nvUvmInterfaceDeRegisterUvmOps(void) 1083 { 1084 // Taking the lock forces us to wait for non-interrupt callbacks to finish 1085 // up. 1086 down(&g_pNvUvmEventsLock); 1087 setUvmEvents(NULL); 1088 up(&g_pNvUvmEventsLock); 1089 1090 // We cleared the pointer so nv_uvm_event_interrupt can't invoke any new 1091 // top half callbacks, but prior ones could still be executing on other 1092 // cores. We can wait for them to finish by waiting for a context switch to 1093 // happen on every core. 1094 // 1095 // This is slow, but since nvUvmInterfaceDeRegisterUvmOps is very rare 1096 // (module unload) it beats having the top half synchronize with a spin lock 1097 // every time. 1098 // 1099 // Note that since we dropped the lock, another set of callbacks could have 1100 // already been registered. That's ok, since we just need to wait for old 1101 // ones to finish. 1102 on_each_cpu(flush_top_half, NULL, 1); 1103 } 1104 EXPORT_SYMBOL(nvUvmInterfaceDeRegisterUvmOps); 1105 1106 NV_STATUS nv_uvm_suspend(void) 1107 { 1108 NV_STATUS status = NV_OK; 1109 struct UvmOpsUvmEvents *events; 1110 1111 // Synchronize callbacks with unregistration 1112 down(&g_pNvUvmEventsLock); 1113 1114 // It's not strictly necessary to use a cached local copy of the events 1115 // pointer here since it can't change under the lock, but we'll do it for 1116 // consistency. 1117 events = getUvmEvents(); 1118 if (events && events->suspend) 1119 { 1120 status = events->suspend(); 1121 } 1122 1123 up(&g_pNvUvmEventsLock); 1124 1125 return status; 1126 } 1127 1128 NV_STATUS nv_uvm_resume(void) 1129 { 1130 NV_STATUS status = NV_OK; 1131 struct UvmOpsUvmEvents *events; 1132 1133 // Synchronize callbacks with unregistration 1134 down(&g_pNvUvmEventsLock); 1135 1136 // It's not strictly necessary to use a cached local copy of the events 1137 // pointer here since it can't change under the lock, but we'll do it for 1138 // consistency. 1139 events = getUvmEvents(); 1140 if (events && events->resume) 1141 { 1142 status = events->resume(); 1143 } 1144 1145 up(&g_pNvUvmEventsLock); 1146 1147 return status; 1148 } 1149 1150 void nv_uvm_notify_start_device(const NvU8 *pUuid) 1151 { 1152 NvProcessorUuid uvmUuid; 1153 struct UvmOpsUvmEvents *events; 1154 1155 memcpy(uvmUuid.uuid, pUuid, UVM_UUID_LEN); 1156 1157 // Synchronize callbacks with unregistration 1158 down(&g_pNvUvmEventsLock); 1159 1160 // It's not strictly necessary to use a cached local copy of the events 1161 // pointer here since it can't change under the lock, but we'll do it for 1162 // consistency. 1163 events = getUvmEvents(); 1164 if(events && events->startDevice) 1165 { 1166 events->startDevice(&uvmUuid); 1167 } 1168 up(&g_pNvUvmEventsLock); 1169 } 1170 1171 void nv_uvm_notify_stop_device(const NvU8 *pUuid) 1172 { 1173 NvProcessorUuid uvmUuid; 1174 struct UvmOpsUvmEvents *events; 1175 1176 memcpy(uvmUuid.uuid, pUuid, UVM_UUID_LEN); 1177 1178 // Synchronize callbacks with unregistration 1179 down(&g_pNvUvmEventsLock); 1180 1181 // It's not strictly necessary to use a cached local copy of the events 1182 // pointer here since it can't change under the lock, but we'll do it for 1183 // consistency. 1184 events = getUvmEvents(); 1185 if(events && events->stopDevice) 1186 { 1187 events->stopDevice(&uvmUuid); 1188 } 1189 up(&g_pNvUvmEventsLock); 1190 } 1191 1192 NV_STATUS nv_uvm_event_interrupt(const NvU8 *pUuid) 1193 { 1194 // 1195 // This is called from interrupt context, so we can't take 1196 // g_pNvUvmEventsLock to prevent the callbacks from being unregistered. Even 1197 // if we could take the lock, we don't want to slow down the ISR more than 1198 // absolutely necessary. 1199 // 1200 // Instead, we allow this function to be called concurrently with 1201 // nvUvmInterfaceDeRegisterUvmOps. That function will clear the events 1202 // pointer, then wait for all top halves to finish out. This means the 1203 // pointer may change out from under us, but the callbacks are still safe to 1204 // invoke while we're in this function. 1205 // 1206 // This requires that we read the pointer exactly once here so neither we 1207 // nor the compiler make assumptions about the pointer remaining valid while 1208 // in this function. 1209 // 1210 struct UvmOpsUvmEvents *events = getUvmEvents(); 1211 1212 if (events && events->isrTopHalf) 1213 return events->isrTopHalf((const NvProcessorUuid *)pUuid); 1214 1215 // 1216 // NV_OK means that the interrupt was for the UVM driver, so use 1217 // NV_ERR_NO_INTR_PENDING to tell the caller that we didn't do anything. 1218 // 1219 return NV_ERR_NO_INTR_PENDING; 1220 } 1221 1222 NV_STATUS nvUvmInterfaceP2pObjectCreate(uvmGpuDeviceHandle device1, 1223 uvmGpuDeviceHandle device2, 1224 NvHandle *hP2pObject) 1225 { 1226 nvidia_stack_t *sp = NULL; 1227 NV_STATUS status; 1228 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1229 { 1230 return NV_ERR_NO_MEMORY; 1231 } 1232 1233 status = rm_gpu_ops_p2p_object_create(sp, 1234 (gpuDeviceHandle)device1, 1235 (gpuDeviceHandle)device2, 1236 hP2pObject); 1237 1238 nv_kmem_cache_free_stack(sp); 1239 return status; 1240 } 1241 EXPORT_SYMBOL(nvUvmInterfaceP2pObjectCreate); 1242 1243 void nvUvmInterfaceP2pObjectDestroy(uvmGpuSessionHandle session, 1244 NvHandle hP2pObject) 1245 { 1246 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1247 1248 rm_gpu_ops_p2p_object_destroy(sp, (gpuSessionHandle)session, hP2pObject); 1249 1250 nvUvmFreeSafeStack(sp); 1251 } 1252 EXPORT_SYMBOL(nvUvmInterfaceP2pObjectDestroy); 1253 1254 NV_STATUS nvUvmInterfaceGetExternalAllocPtes(uvmGpuAddressSpaceHandle vaSpace, 1255 NvHandle hDupedMemory, 1256 NvU64 offset, 1257 NvU64 size, 1258 UvmGpuExternalMappingInfo *gpuExternalMappingInfo) 1259 { 1260 nvidia_stack_t *sp = NULL; 1261 NV_STATUS status; 1262 1263 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1264 { 1265 return NV_ERR_NO_MEMORY; 1266 } 1267 1268 status = rm_gpu_ops_get_external_alloc_ptes(sp, 1269 (gpuAddressSpaceHandle)vaSpace, 1270 hDupedMemory, 1271 offset, 1272 size, 1273 gpuExternalMappingInfo); 1274 1275 nv_kmem_cache_free_stack(sp); 1276 return status; 1277 } 1278 EXPORT_SYMBOL(nvUvmInterfaceGetExternalAllocPtes); 1279 1280 NV_STATUS nvUvmInterfaceRetainChannel(uvmGpuAddressSpaceHandle vaSpace, 1281 NvHandle hClient, 1282 NvHandle hChannel, 1283 void **retainedChannel, 1284 UvmGpuChannelInstanceInfo *channelInstanceInfo) 1285 { 1286 nvidia_stack_t *sp = NULL; 1287 NV_STATUS status; 1288 1289 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1290 { 1291 return NV_ERR_NO_MEMORY; 1292 } 1293 1294 status = rm_gpu_ops_retain_channel(sp, 1295 (gpuAddressSpaceHandle)vaSpace, 1296 hClient, 1297 hChannel, 1298 retainedChannel, 1299 channelInstanceInfo); 1300 1301 nv_kmem_cache_free_stack(sp); 1302 return status; 1303 } 1304 EXPORT_SYMBOL(nvUvmInterfaceRetainChannel); 1305 1306 NV_STATUS nvUvmInterfaceBindChannelResources(void *retainedChannel, 1307 UvmGpuChannelResourceBindParams *channelResourceBindParams) 1308 { 1309 nvidia_stack_t *sp = NULL; 1310 NV_STATUS status; 1311 1312 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1313 { 1314 return NV_ERR_NO_MEMORY; 1315 } 1316 1317 status = rm_gpu_ops_bind_channel_resources(sp, 1318 retainedChannel, 1319 channelResourceBindParams); 1320 1321 nv_kmem_cache_free_stack(sp); 1322 return status; 1323 } 1324 EXPORT_SYMBOL(nvUvmInterfaceBindChannelResources); 1325 1326 void nvUvmInterfaceReleaseChannel(void *retainedChannel) 1327 { 1328 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1329 1330 rm_gpu_ops_release_channel(sp, retainedChannel); 1331 1332 nvUvmFreeSafeStack(sp); 1333 } 1334 EXPORT_SYMBOL(nvUvmInterfaceReleaseChannel); 1335 1336 void nvUvmInterfaceStopChannel(void *retainedChannel, NvBool bImmediate) 1337 { 1338 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1339 1340 rm_gpu_ops_stop_channel(sp, retainedChannel, bImmediate); 1341 1342 nvUvmFreeSafeStack(sp); 1343 } 1344 EXPORT_SYMBOL(nvUvmInterfaceStopChannel); 1345 1346 NV_STATUS nvUvmInterfaceGetChannelResourcePtes(uvmGpuAddressSpaceHandle vaSpace, 1347 NvP64 resourceDescriptor, 1348 NvU64 offset, 1349 NvU64 size, 1350 UvmGpuExternalMappingInfo *externalMappingInfo) 1351 { 1352 nvidia_stack_t *sp = NULL; 1353 NV_STATUS status; 1354 1355 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1356 { 1357 return NV_ERR_NO_MEMORY; 1358 } 1359 1360 status = rm_gpu_ops_get_channel_resource_ptes(sp, 1361 (gpuAddressSpaceHandle)vaSpace, 1362 resourceDescriptor, 1363 offset, 1364 size, 1365 externalMappingInfo); 1366 1367 nv_kmem_cache_free_stack(sp); 1368 return status; 1369 } 1370 EXPORT_SYMBOL(nvUvmInterfaceGetChannelResourcePtes); 1371 1372 NV_STATUS nvUvmInterfaceReportNonReplayableFault(uvmGpuDeviceHandle device, 1373 const void *pFaultPacket) 1374 { 1375 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1376 NV_STATUS status; 1377 1378 status = rm_gpu_ops_report_non_replayable_fault(sp, (gpuDeviceHandle)device, pFaultPacket); 1379 1380 nvUvmFreeSafeStack(sp); 1381 return status; 1382 } 1383 EXPORT_SYMBOL(nvUvmInterfaceReportNonReplayableFault); 1384 1385 NV_STATUS nvUvmInterfacePagingChannelAllocate(uvmGpuDeviceHandle device, 1386 const UvmGpuPagingChannelAllocParams *allocParams, 1387 UvmGpuPagingChannelHandle *channel, 1388 UvmGpuPagingChannelInfo *channelInfo) 1389 { 1390 nvidia_stack_t *sp = NULL; 1391 nvidia_stack_t *pushStreamSp = NULL; 1392 NV_STATUS status; 1393 1394 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1395 return NV_ERR_NO_MEMORY; 1396 1397 if (nv_kmem_cache_alloc_stack(&pushStreamSp) != 0) 1398 { 1399 nv_kmem_cache_free_stack(sp); 1400 return NV_ERR_NO_MEMORY; 1401 } 1402 1403 status = rm_gpu_ops_paging_channel_allocate(sp, 1404 (gpuDeviceHandle)device, 1405 allocParams, 1406 (gpuPagingChannelHandle *)channel, 1407 channelInfo); 1408 1409 if (status == NV_OK) 1410 (*channel)->pushStreamSp = pushStreamSp; 1411 else 1412 nv_kmem_cache_free_stack(pushStreamSp); 1413 1414 nv_kmem_cache_free_stack(sp); 1415 1416 return status; 1417 } 1418 EXPORT_SYMBOL(nvUvmInterfacePagingChannelAllocate); 1419 1420 void nvUvmInterfacePagingChannelDestroy(UvmGpuPagingChannelHandle channel) 1421 { 1422 nvidia_stack_t *sp; 1423 1424 if (channel == NULL) 1425 return; 1426 1427 sp = nvUvmGetSafeStack(); 1428 nv_kmem_cache_free_stack(channel->pushStreamSp); 1429 rm_gpu_ops_paging_channel_destroy(sp, (gpuPagingChannelHandle)channel); 1430 nvUvmFreeSafeStack(sp); 1431 } 1432 EXPORT_SYMBOL(nvUvmInterfacePagingChannelDestroy); 1433 1434 NV_STATUS nvUvmInterfacePagingChannelsMap(uvmGpuAddressSpaceHandle srcVaSpace, 1435 UvmGpuPointer srcAddress, 1436 uvmGpuDeviceHandle device, 1437 NvU64 *dstAddress) 1438 { 1439 nvidia_stack_t *sp = NULL; 1440 NV_STATUS status; 1441 1442 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1443 return NV_ERR_NO_MEMORY; 1444 1445 status = rm_gpu_ops_paging_channels_map(sp, 1446 (gpuAddressSpaceHandle)srcVaSpace, 1447 (NvU64)srcAddress, 1448 (gpuDeviceHandle)device, 1449 dstAddress); 1450 1451 nv_kmem_cache_free_stack(sp); 1452 1453 return status; 1454 } 1455 EXPORT_SYMBOL(nvUvmInterfacePagingChannelsMap); 1456 1457 void nvUvmInterfacePagingChannelsUnmap(uvmGpuAddressSpaceHandle srcVaSpace, 1458 UvmGpuPointer srcAddress, 1459 uvmGpuDeviceHandle device) 1460 { 1461 nvidia_stack_t *sp = nvUvmGetSafeStack(); 1462 rm_gpu_ops_paging_channels_unmap(sp, 1463 (gpuAddressSpaceHandle)srcVaSpace, 1464 (NvU64)srcAddress, 1465 (gpuDeviceHandle)device); 1466 nvUvmFreeSafeStack(sp); 1467 } 1468 EXPORT_SYMBOL(nvUvmInterfacePagingChannelsUnmap); 1469 1470 NV_STATUS nvUvmInterfacePagingChannelPushStream(UvmGpuPagingChannelHandle channel, 1471 char *methodStream, 1472 NvU32 methodStreamSize) 1473 { 1474 return rm_gpu_ops_paging_channel_push_stream(channel->pushStreamSp, 1475 (gpuPagingChannelHandle)channel, 1476 methodStream, 1477 methodStreamSize); 1478 } 1479 EXPORT_SYMBOL(nvUvmInterfacePagingChannelPushStream); 1480 1481 NV_STATUS nvUvmInterfaceCslInitContext(UvmCslContext *uvmCslContext, 1482 uvmGpuChannelHandle channel) 1483 { 1484 nvidia_stack_t *sp = NULL; 1485 NV_STATUS status; 1486 1487 if (nv_kmem_cache_alloc_stack(&sp) != 0) 1488 { 1489 return NV_ERR_NO_MEMORY; 1490 } 1491 1492 status = rm_gpu_ops_ccsl_context_init(sp, &uvmCslContext->ctx, (gpuChannelHandle)channel); 1493 1494 // Saving the stack in the context allows UVM to safely use the CSL layer 1495 // in interrupt context without making new allocations. UVM serializes CSL 1496 // API usage for a given context so the stack pointer does not need 1497 // additional protection. 1498 if (status != NV_OK) 1499 { 1500 nv_kmem_cache_free_stack(sp); 1501 } 1502 else 1503 { 1504 uvmCslContext->nvidia_stack = sp; 1505 } 1506 1507 return status; 1508 } 1509 EXPORT_SYMBOL(nvUvmInterfaceCslInitContext); 1510 1511 void nvUvmInterfaceDeinitCslContext(UvmCslContext *uvmCslContext) 1512 { 1513 nvidia_stack_t *sp = uvmCslContext->nvidia_stack; 1514 rm_gpu_ops_ccsl_context_clear(sp, uvmCslContext->ctx); 1515 nvUvmFreeSafeStack(sp); 1516 } 1517 EXPORT_SYMBOL(nvUvmInterfaceDeinitCslContext); 1518 1519 NV_STATUS nvUvmInterfaceCslRotateIv(UvmCslContext *uvmCslContext, 1520 UvmCslOperation operation) 1521 { 1522 NV_STATUS status; 1523 nvidia_stack_t *sp = uvmCslContext->nvidia_stack; 1524 1525 status = rm_gpu_ops_ccsl_rotate_iv(sp, uvmCslContext->ctx, operation); 1526 1527 return status; 1528 } 1529 EXPORT_SYMBOL(nvUvmInterfaceCslRotateIv); 1530 1531 NV_STATUS nvUvmInterfaceCslEncrypt(UvmCslContext *uvmCslContext, 1532 NvU32 bufferSize, 1533 NvU8 const *inputBuffer, 1534 UvmCslIv *encryptIv, 1535 NvU8 *outputBuffer, 1536 NvU8 *authTagBuffer) 1537 { 1538 NV_STATUS status; 1539 nvidia_stack_t *sp = uvmCslContext->nvidia_stack; 1540 1541 if (encryptIv != NULL) 1542 status = rm_gpu_ops_ccsl_encrypt_with_iv(sp, uvmCslContext->ctx, bufferSize, inputBuffer, (NvU8*)encryptIv, outputBuffer, authTagBuffer); 1543 else 1544 status = rm_gpu_ops_ccsl_encrypt(sp, uvmCslContext->ctx, bufferSize, inputBuffer, outputBuffer, authTagBuffer); 1545 1546 return status; 1547 } 1548 EXPORT_SYMBOL(nvUvmInterfaceCslEncrypt); 1549 1550 NV_STATUS nvUvmInterfaceCslDecrypt(UvmCslContext *uvmCslContext, 1551 NvU32 bufferSize, 1552 NvU8 const *inputBuffer, 1553 UvmCslIv const *decryptIv, 1554 NvU8 *outputBuffer, 1555 NvU8 const *addAuthData, 1556 NvU32 addAuthDataSize, 1557 NvU8 const *authTagBuffer) 1558 { 1559 NV_STATUS status; 1560 nvidia_stack_t *sp = uvmCslContext->nvidia_stack; 1561 1562 status = rm_gpu_ops_ccsl_decrypt(sp, 1563 uvmCslContext->ctx, 1564 bufferSize, 1565 inputBuffer, 1566 (NvU8 *)decryptIv, 1567 outputBuffer, 1568 addAuthData, 1569 addAuthDataSize, 1570 authTagBuffer); 1571 1572 return status; 1573 } 1574 EXPORT_SYMBOL(nvUvmInterfaceCslDecrypt); 1575 1576 NV_STATUS nvUvmInterfaceCslSign(UvmCslContext *uvmCslContext, 1577 NvU32 bufferSize, 1578 NvU8 const *inputBuffer, 1579 NvU8 *authTagBuffer) 1580 { 1581 NV_STATUS status; 1582 nvidia_stack_t *sp = uvmCslContext->nvidia_stack; 1583 1584 status = rm_gpu_ops_ccsl_sign(sp, uvmCslContext->ctx, bufferSize, inputBuffer, authTagBuffer); 1585 1586 return status; 1587 } 1588 EXPORT_SYMBOL(nvUvmInterfaceCslSign); 1589 1590 NV_STATUS nvUvmInterfaceCslQueryMessagePool(UvmCslContext *uvmCslContext, 1591 UvmCslOperation operation, 1592 NvU64 *messageNum) 1593 { 1594 NV_STATUS status; 1595 nvidia_stack_t *sp = uvmCslContext->nvidia_stack; 1596 1597 status = rm_gpu_ops_ccsl_query_message_pool(sp, uvmCslContext->ctx, operation, messageNum); 1598 1599 return status; 1600 } 1601 EXPORT_SYMBOL(nvUvmInterfaceCslQueryMessagePool); 1602 1603 NV_STATUS nvUvmInterfaceCslIncrementIv(UvmCslContext *uvmCslContext, 1604 UvmCslOperation operation, 1605 NvU64 increment, 1606 UvmCslIv *iv) 1607 { 1608 NV_STATUS status; 1609 nvidia_stack_t *sp = uvmCslContext->nvidia_stack; 1610 1611 status = rm_gpu_ops_ccsl_increment_iv(sp, uvmCslContext->ctx, operation, increment, (NvU8 *)iv); 1612 1613 return status; 1614 } 1615 EXPORT_SYMBOL(nvUvmInterfaceCslIncrementIv); 1616 1617 #else // NV_UVM_ENABLE 1618 1619 NV_STATUS nv_uvm_suspend(void) 1620 { 1621 return NV_OK; 1622 } 1623 1624 NV_STATUS nv_uvm_resume(void) 1625 { 1626 return NV_OK; 1627 } 1628 1629 #endif // NV_UVM_ENABLE 1630