1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2013-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 // 25 // This file provides the interface that RM exposes to UVM. 26 // 27 28 #ifndef _NV_UVM_INTERFACE_H_ 29 #define _NV_UVM_INTERFACE_H_ 30 31 // Forward references, to break circular header file dependencies: 32 struct UvmOpsUvmEvents; 33 34 #if defined(NVIDIA_UVM_ENABLED) 35 36 // We are in the UVM build system, for a Linux target. 37 #include "uvm_linux.h" 38 39 #else 40 41 // We are in the RM build system, for a Linux target: 42 #include "nv-linux.h" 43 44 #endif // NVIDIA_UVM_ENABLED 45 46 #include "nvgputypes.h" 47 #include "nvstatus.h" 48 #include "nv_uvm_types.h" 49 50 51 // Define the type here as it's Linux specific, used only by the Linux specific 52 // nvUvmInterfaceRegisterGpu() API. 53 typedef struct 54 { 55 struct pci_dev *pci_dev; 56 57 // DMA addressable range of the device, mirrors fields in nv_state_t. 58 NvU64 dma_addressable_start; 59 NvU64 dma_addressable_limit; 60 } UvmGpuPlatformInfo; 61 62 /******************************************************************************* 63 nvUvmInterfaceRegisterGpu 64 65 Registers the GPU with the provided UUID for use. A GPU must be registered 66 before its UUID can be used with any other API. This call is ref-counted so 67 every nvUvmInterfaceRegisterGpu must be paired with a corresponding 68 nvUvmInterfaceUnregisterGpu. 69 70 You don't need to call nvUvmInterfaceSessionCreate before calling this. 71 72 Error codes: 73 NV_ERR_GPU_UUID_NOT_FOUND 74 NV_ERR_NO_MEMORY 75 NV_ERR_GENERIC 76 */ 77 NV_STATUS nvUvmInterfaceRegisterGpu(const NvProcessorUuid *gpuUuid, UvmGpuPlatformInfo *gpuInfo); 78 79 /******************************************************************************* 80 nvUvmInterfaceUnregisterGpu 81 82 Unregisters the GPU with the provided UUID. This drops the ref count from 83 nvUvmInterfaceRegisterGpu. Once the reference count goes to 0 the device may 84 no longer be accessible until the next nvUvmInterfaceRegisterGpu call. No 85 automatic resource freeing is performed, so only make the last unregister 86 call after destroying all your allocations associated with that UUID (such 87 as those from nvUvmInterfaceAddressSpaceCreate). 88 89 If the UUID is not found, no operation is performed. 90 */ 91 void nvUvmInterfaceUnregisterGpu(const NvProcessorUuid *gpuUuid); 92 93 /******************************************************************************* 94 nvUvmInterfaceSessionCreate 95 96 TODO: Creates session object. All allocations are tied to the session. 97 98 The platformInfo parameter is filled by the callee with miscellaneous system 99 information. Refer to the UvmPlatformInfo struct for details. 100 101 Error codes: 102 NV_ERR_GENERIC 103 NV_ERR_NO_MEMORY 104 */ 105 NV_STATUS nvUvmInterfaceSessionCreate(uvmGpuSessionHandle *session, 106 UvmPlatformInfo *platformInfo); 107 108 /******************************************************************************* 109 nvUvmInterfaceSessionDestroy 110 111 Destroys a session object. All allocations are tied to the session will 112 be destroyed. 113 114 Error codes: 115 NV_ERR_GENERIC 116 NV_ERR_NO_MEMORY 117 */ 118 NV_STATUS nvUvmInterfaceSessionDestroy(uvmGpuSessionHandle session); 119 120 /******************************************************************************* 121 nvUvmInterfaceDeviceCreate 122 123 Creates a device object under the given session for the GPU with the given 124 UUID. Also creates a partition object for the device iff bCreateSmcPartition 125 is true and pGpuInfo->smcEnabled is true. pGpuInfo->smcUserClientInfo will 126 be used to determine the SMC partition in this case. A device handle is 127 returned in the device output parameter. 128 129 Error codes: 130 NV_ERR_GENERIC 131 NV_ERR_NO_MEMORY 132 NV_ERR_INVALID_ARGUMENT 133 NV_ERR_INSUFFICIENT_RESOURCES 134 NV_ERR_OBJECT_NOT_FOUND 135 */ 136 NV_STATUS nvUvmInterfaceDeviceCreate(uvmGpuSessionHandle session, 137 const UvmGpuInfo *pGpuInfo, 138 const NvProcessorUuid *gpuUuid, 139 uvmGpuDeviceHandle *device, 140 NvBool bCreateSmcPartition); 141 142 /******************************************************************************* 143 nvUvmInterfaceDeviceDestroy 144 145 Destroys the device object for the given handle. The handle must have been 146 obtained in a prior call to nvUvmInterfaceDeviceCreate. 147 */ 148 void nvUvmInterfaceDeviceDestroy(uvmGpuDeviceHandle device); 149 150 /******************************************************************************* 151 nvUvmInterfaceAddressSpaceCreate 152 153 This function creates an address space. 154 This virtual address space is created on the GPU specified 155 by device. 156 157 Error codes: 158 NV_ERR_GENERIC 159 NV_ERR_NO_MEMORY 160 */ 161 NV_STATUS nvUvmInterfaceAddressSpaceCreate(uvmGpuDeviceHandle device, 162 unsigned long long vaBase, 163 unsigned long long vaSize, 164 uvmGpuAddressSpaceHandle *vaSpace, 165 UvmGpuAddressSpaceInfo *vaSpaceInfo); 166 167 /******************************************************************************* 168 nvUvmInterfaceDupAddressSpace 169 170 This function will dup the given vaspace from the users client to the 171 kernel client was created as an ops session. 172 173 By duping the vaspace it is guaranteed that RM will refcount the vaspace object. 174 175 Error codes: 176 NV_ERR_GENERIC 177 */ 178 NV_STATUS nvUvmInterfaceDupAddressSpace(uvmGpuDeviceHandle device, 179 NvHandle hUserClient, 180 NvHandle hUserVASpace, 181 uvmGpuAddressSpaceHandle *vaSpace, 182 UvmGpuAddressSpaceInfo *vaSpaceInfo); 183 184 /******************************************************************************* 185 nvUvmInterfaceAddressSpaceDestroy 186 187 Destroys an address space that was previously created via 188 nvUvmInterfaceAddressSpaceCreate. 189 */ 190 191 void nvUvmInterfaceAddressSpaceDestroy(uvmGpuAddressSpaceHandle vaSpace); 192 193 /******************************************************************************* 194 nvUvmInterfaceMemoryAllocFB 195 196 This function will allocate video memory and provide a mapped Gpu 197 virtual address to this allocation. It also returns the Gpu physical 198 offset if contiguous allocations are requested. 199 200 This function will allocate a minimum page size if the length provided is 0 201 and will return a unique GPU virtual address. 202 203 The default page size will be the small page size (as returned by query 204 caps). The physical alignment will also be enforced to small page 205 size(64K/128K). 206 207 Arguments: 208 vaSpace[IN] - Pointer to vaSpace object 209 length [IN] - Length of the allocation 210 gpuPointer[OUT] - GPU VA mapping 211 allocInfo[IN/OUT] - Pointer to allocation info structure which 212 contains below given fields 213 214 allocInfo Members: 215 gpuPhysOffset[OUT] - Physical offset of allocation returned only 216 if contiguous allocation is requested. 217 pageSize[IN] - Override the default page size (see above). 218 alignment[IN] - gpuPointer GPU VA alignment. 0 means 4KB 219 alignment. 220 bContiguousPhysAlloc[IN] - Flag to request contiguous allocation. Default 221 will follow the vidHeapControl default policy. 222 bMemGrowsDown[IN] 223 bPersistentVidmem[IN] - Allocate persistent vidmem. 224 hPhysHandle[IN/OUT] - The handle will be used in allocation if provided. 225 If not provided; allocator will return the handle 226 it used eventually. 227 Error codes: 228 NV_ERR_INVALID_ARGUMENT 229 NV_ERR_NO_MEMORY - Not enough physical memory to service 230 allocation request with provided constraints 231 NV_ERR_INSUFFICIENT_RESOURCES - Not enough available resources to satisfy allocation request 232 NV_ERR_INVALID_OWNER - Target memory not accessible by specified owner 233 NV_ERR_NOT_SUPPORTED - Operation not supported on broken FB 234 235 */ 236 NV_STATUS nvUvmInterfaceMemoryAllocFB(uvmGpuAddressSpaceHandle vaSpace, 237 NvLength length, 238 UvmGpuPointer * gpuPointer, 239 UvmGpuAllocInfo * allocInfo); 240 241 /******************************************************************************* 242 nvUvmInterfaceMemoryAllocSys 243 244 This function will allocate system memory and provide a mapped Gpu 245 virtual address to this allocation. 246 247 This function will allocate a minimum page size if the length provided is 0 248 and will return a unique GPU virtual address. 249 250 The default page size will be the small page size (as returned by query caps) 251 252 Arguments: 253 vaSpace[IN] - Pointer to vaSpace object 254 length [IN] - Length of the allocation 255 gpuPointer[OUT] - GPU VA mapping 256 allocInfo[IN/OUT] - Pointer to allocation info structure which 257 contains below given fields 258 259 allocInfo Members: 260 gpuPhysOffset[OUT] - Physical offset of allocation returned only 261 if contiguous allocation is requested. 262 pageSize[IN] - Override the default page size (see above). 263 alignment[IN] - gpuPointer GPU VA alignment. 0 means 4KB 264 alignment. 265 bContiguousPhysAlloc[IN] - Flag to request contiguous allocation. Default 266 will follow the vidHeapControl default policy. 267 bMemGrowsDown[IN] 268 bPersistentVidmem[IN] - Allocate persistent vidmem. 269 hPhysHandle[IN/OUT] - The handle will be used in allocation if provided. 270 If not provided; allocator will return the handle 271 it used eventually. 272 Error codes: 273 NV_ERR_INVALID_ARGUMENT 274 NV_ERR_NO_MEMORY - Not enough physical memory to service 275 allocation request with provided constraints 276 NV_ERR_INSUFFICIENT_RESOURCES - Not enough available resources to satisfy allocation request 277 NV_ERR_INVALID_OWNER - Target memory not accessible by specified owner 278 NV_ERR_NOT_SUPPORTED - Operation not supported 279 */ 280 NV_STATUS nvUvmInterfaceMemoryAllocSys(uvmGpuAddressSpaceHandle vaSpace, 281 NvLength length, 282 UvmGpuPointer * gpuPointer, 283 UvmGpuAllocInfo * allocInfo); 284 285 /******************************************************************************* 286 nvUvmInterfaceGetP2PCaps 287 288 Obtain the P2P capabilities between two devices. 289 290 Arguments: 291 device1[IN] - Device handle of the first GPU (required) 292 device2[IN] - Device handle of the second GPU (required) 293 p2pCapsParams [OUT] - P2P capabilities between the two GPUs 294 295 Error codes: 296 NV_ERR_INVALID_ARGUMENT 297 NV_ERR_GENERIC: 298 Unexpected error. We try hard to avoid returning this error 299 code,because it is not very informative. 300 301 */ 302 NV_STATUS nvUvmInterfaceGetP2PCaps(uvmGpuDeviceHandle device1, 303 uvmGpuDeviceHandle device2, 304 UvmGpuP2PCapsParams * p2pCapsParams); 305 306 /******************************************************************************* 307 nvUvmInterfaceGetPmaObject 308 309 This function will return pointer to PMA object for the given GPU. This 310 PMA object handle is required for page allocation. 311 312 Arguments: 313 device [IN] - Device handle allocated in 314 nvUvmInterfaceDeviceCreate 315 pPma [OUT] - Pointer to PMA object 316 pPmaPubStats [OUT] - Pointer to UvmPmaStatistics object 317 318 Error codes: 319 NV_ERR_NOT_SUPPORTED - Operation not supported on broken FB 320 NV_ERR_GENERIC: 321 Unexpected error. We try hard to avoid returning this error 322 code,because it is not very informative. 323 */ 324 NV_STATUS nvUvmInterfaceGetPmaObject(uvmGpuDeviceHandle device, 325 void **pPma, 326 const UvmPmaStatistics **pPmaPubStats); 327 328 // Mirrors pmaEvictPagesCb_t, see its documentation in pma.h. 329 typedef NV_STATUS (*uvmPmaEvictPagesCallback)(void *callbackData, 330 NvU64 pageSize, 331 NvU64 *pPages, 332 NvU32 count, 333 NvU64 physBegin, 334 NvU64 physEnd, 335 UVM_PMA_GPU_MEMORY_TYPE mem_type); 336 337 // Mirrors pmaEvictRangeCb_t, see its documentation in pma.h. 338 typedef NV_STATUS (*uvmPmaEvictRangeCallback)(void *callbackData, 339 NvU64 physBegin, 340 NvU64 physEnd, 341 UVM_PMA_GPU_MEMORY_TYPE mem_type); 342 343 /******************************************************************************* 344 nvUvmInterfacePmaRegisterEvictionCallbacks 345 346 Simple wrapper for pmaRegisterEvictionCb(), see its documentation in pma.h. 347 */ 348 NV_STATUS nvUvmInterfacePmaRegisterEvictionCallbacks(void *pPma, 349 uvmPmaEvictPagesCallback evictPages, 350 uvmPmaEvictRangeCallback evictRange, 351 void *callbackData); 352 353 /****************************************************************************** 354 nvUvmInterfacePmaUnregisterEvictionCallbacks 355 356 Simple wrapper for pmaUnregisterEvictionCb(), see its documentation in pma.h. 357 */ 358 void nvUvmInterfacePmaUnregisterEvictionCallbacks(void *pPma); 359 360 /******************************************************************************* 361 nvUvmInterfacePmaAllocPages 362 363 @brief Synchronous API for allocating pages from the PMA. 364 PMA will decide which pma regions to allocate from based on the provided 365 flags. PMA will also initiate UVM evictions to make room for this 366 allocation unless prohibited by PMA_FLAGS_DONT_EVICT. UVM callers must pass 367 this flag to avoid deadlock. Only UVM may allocated unpinned memory from 368 this API. 369 370 For broadcast methods, PMA will guarantee the same physical frames are 371 allocated on multiple GPUs, specified by the PMA objects passed in. 372 373 If allocation is contiguous, only one page in pPages will be filled. 374 Also, contiguous flag must be passed later to nvUvmInterfacePmaFreePages. 375 376 Arguments: 377 pPma[IN] - Pointer to PMA object 378 pageCount [IN] - Number of pages required to be allocated. 379 pageSize [IN] - 64kb, 128kb or 2mb. No other values are permissible. 380 pPmaAllocOptions[IN] - Pointer to PMA allocation info structure. 381 pPages[OUT] - Array of pointers, containing the PA base 382 address of each page. 383 384 Error codes: 385 NV_ERR_NO_MEMORY: 386 Internal memory allocation failed. 387 NV_ERR_GENERIC: 388 Unexpected error. We try hard to avoid returning this error 389 code,because it is not very informative. 390 */ 391 NV_STATUS nvUvmInterfacePmaAllocPages(void *pPma, 392 NvLength pageCount, 393 NvU64 pageSize, 394 UvmPmaAllocationOptions *pPmaAllocOptions, 395 NvU64 *pPages); 396 397 /******************************************************************************* 398 nvUvmInterfacePmaPinPages 399 400 This function will pin the physical memory allocated using PMA. The pages 401 passed as input must be unpinned else this function will return an error and 402 rollback any change if any page is not previously marked "unpinned". 403 404 Arguments: 405 pPma[IN] - Pointer to PMA object. 406 pPages[IN] - Array of pointers, containing the PA base 407 address of each page to be pinned. 408 pageCount [IN] - Number of pages required to be pinned. 409 pageSize [IN] - Page size of each page to be pinned. 410 flags [IN] - UVM_PMA_CALLED_FROM_PMA_EVICTION if called from 411 PMA eviction, 0 otherwise. 412 Error codes: 413 NV_ERR_INVALID_ARGUMENT - Invalid input arguments. 414 NV_ERR_GENERIC - Unexpected error. We try hard to avoid 415 returning this error code as is not very 416 informative. 417 NV_ERR_NOT_SUPPORTED - Operation not supported on broken FB 418 */ 419 NV_STATUS nvUvmInterfacePmaPinPages(void *pPma, 420 NvU64 *pPages, 421 NvLength pageCount, 422 NvU64 pageSize, 423 NvU32 flags); 424 425 /******************************************************************************* 426 nvUvmInterfacePmaUnpinPages 427 428 This function will unpin the physical memory allocated using PMA. The pages 429 passed as input must be already pinned, else this function will return an 430 error and rollback any change if any page is not previously marked "pinned". 431 Behaviour is undefined if any blacklisted pages are unpinned. 432 433 Arguments: 434 pPma[IN] - Pointer to PMA object. 435 pPages[IN] - Array of pointers, containing the PA base 436 address of each page to be unpinned. 437 pageCount [IN] - Number of pages required to be unpinned. 438 pageSize [IN] - Page size of each page to be unpinned. 439 440 Error codes: 441 NV_ERR_INVALID_ARGUMENT - Invalid input arguments. 442 NV_ERR_GENERIC - Unexpected error. We try hard to avoid 443 returning this error code as is not very 444 informative. 445 NV_ERR_NOT_SUPPORTED - Operation not supported on broken FB 446 */ 447 NV_STATUS nvUvmInterfacePmaUnpinPages(void *pPma, 448 NvU64 *pPages, 449 NvLength pageCount, 450 NvU64 pageSize); 451 452 /******************************************************************************* 453 nvUvmInterfaceMemoryFree 454 455 Free up a GPU allocation 456 */ 457 void nvUvmInterfaceMemoryFree(uvmGpuAddressSpaceHandle vaSpace, 458 UvmGpuPointer gpuPointer); 459 460 /******************************************************************************* 461 nvUvmInterfacePmaFreePages 462 463 This function will free physical memory allocated using PMA. It marks a list 464 of pages as free. This operation is also used by RM to mark pages as "scrubbed" 465 for the initial ECC sweep. This function does not fail. 466 467 When allocation was contiguous, an appropriate flag needs to be passed. 468 469 Arguments: 470 pPma[IN] - Pointer to PMA object 471 pPages[IN] - Array of pointers, containing the PA base 472 address of each page. 473 pageCount [IN] - Number of pages required to be allocated. 474 pageSize [IN] - Page size of each page 475 flags [IN] - Flags with information about allocation type 476 with the same meaning as flags in options for 477 nvUvmInterfacePmaAllocPages. When called from PMA 478 eviction, UVM_PMA_CALLED_FROM_PMA_EVICTION needs 479 to be added to flags. 480 Error codes: 481 NV_ERR_INVALID_ARGUMENT 482 NV_ERR_NO_MEMORY - Not enough physical memory to service 483 allocation request with provided constraints 484 NV_ERR_INSUFFICIENT_RESOURCES - Not enough available resources to satisfy allocation request 485 NV_ERR_INVALID_OWNER - Target memory not accessible by specified owner 486 NV_ERR_NOT_SUPPORTED - Operation not supported on broken FB 487 */ 488 void nvUvmInterfacePmaFreePages(void *pPma, 489 NvU64 *pPages, 490 NvLength pageCount, 491 NvU64 pageSize, 492 NvU32 flags); 493 494 /******************************************************************************* 495 nvUvmInterfaceMemoryCpuMap 496 497 This function creates a CPU mapping to the provided GPU address. 498 If the address is not the same as what is returned by the Alloc 499 function, then the function will map it from the address provided. 500 This offset will be relative to the gpu offset obtained from the 501 memory alloc functions. 502 503 Error codes: 504 NV_ERR_GENERIC 505 NV_ERR_NO_MEMORY 506 */ 507 NV_STATUS nvUvmInterfaceMemoryCpuMap(uvmGpuAddressSpaceHandle vaSpace, 508 UvmGpuPointer gpuPointer, 509 NvLength length, void **cpuPtr, 510 NvU64 pageSize); 511 512 /******************************************************************************* 513 uvmGpuMemoryCpuUnmap 514 515 Unmaps the cpuPtr provided from the process virtual address space. 516 */ 517 void nvUvmInterfaceMemoryCpuUnMap(uvmGpuAddressSpaceHandle vaSpace, 518 void *cpuPtr); 519 520 /******************************************************************************* 521 nvUvmInterfaceTsgAllocate 522 523 This function allocates a Time-Slice Group (TSG). 524 525 allocParams must contain an engineIndex as TSGs need to be bound to an 526 engine type at allocation time. The possible values are [0, 527 UVM_COPY_ENGINE_COUNT_MAX) for CE engine type. Notably only the copy engines 528 that have UvmGpuCopyEngineCaps::supported set to true can be allocated. 529 530 Note that TSG is not supported on all GPU architectures for all engine 531 types, e.g., pre-Volta GPUs only support TSG for the GR/Compute engine type. 532 On devices that do not support HW TSGs on the requested engine, this API is 533 still required, i.e., a TSG handle is required in 534 nvUvmInterfaceChannelAllocate(), due to information stored in it necessary 535 for channel allocation. However, when HW TSGs aren't supported, a TSG handle 536 is essentially a "fake" TSG with no HW scheduling impact. 537 538 tsg is filled with the address of the corresponding TSG handle. 539 540 Arguments: 541 vaSpace[IN] - VA space linked to a client and a device under which 542 the TSG is allocated. 543 allocParams[IN] - structure with allocation settings. 544 tsg[OUT] - pointer to the new TSG handle. 545 546 Error codes: 547 NV_ERR_GENERIC 548 NV_ERR_INVALID_ARGUMENT 549 NV_ERR_NO_MEMORY 550 NV_ERR_NOT_SUPPORTED 551 */ 552 NV_STATUS nvUvmInterfaceTsgAllocate(uvmGpuAddressSpaceHandle vaSpace, 553 const UvmGpuTsgAllocParams *allocParams, 554 uvmGpuTsgHandle *tsg); 555 556 /******************************************************************************* 557 nvUvmInterfaceTsgDestroy 558 559 This function destroys a given TSG. 560 561 Arguments: 562 tsg[IN] - Tsg handle 563 */ 564 void nvUvmInterfaceTsgDestroy(uvmGpuTsgHandle tsg); 565 566 /******************************************************************************* 567 nvUvmInterfaceChannelAllocate 568 569 This function will allocate a channel bound to a copy engine(CE) or a SEC2 570 engine. 571 572 allocParams contains information relative to GPFIFO and GPPut. 573 574 channel is filled with the address of the corresponding channel handle. 575 576 channelInfo is filled out with channel get/put. The errorNotifier is filled 577 out when the channel hits an RC error. On Volta+ devices, it also computes 578 the work submission token and the work submission offset to be used in the 579 Host channel submission doorbell. 580 581 Arguments: 582 tsg[IN] - Time-Slice Group that the channel will be a member. 583 allocParams[IN] - structure with allocation settings 584 channel[OUT] - pointer to the new channel handle 585 channelInfo[OUT] - structure filled with channel information 586 587 Error codes: 588 NV_ERR_GENERIC 589 NV_ERR_INVALID_ARGUMENT 590 NV_ERR_NO_MEMORY 591 NV_ERR_NOT_SUPPORTED 592 */ 593 NV_STATUS nvUvmInterfaceChannelAllocate(const uvmGpuTsgHandle tsg, 594 const UvmGpuChannelAllocParams *allocParams, 595 uvmGpuChannelHandle *channel, 596 UvmGpuChannelInfo *channelInfo); 597 598 /******************************************************************************* 599 nvUvmInterfaceChannelDestroy 600 601 This function destroys a given channel. 602 603 Arguments: 604 channel[IN] - channel handle 605 */ 606 void nvUvmInterfaceChannelDestroy(uvmGpuChannelHandle channel); 607 608 /******************************************************************************* 609 nvUvmInterfaceQueryCaps 610 611 Return capabilities for the provided GPU. 612 If GPU does not exist, an error will be returned. 613 614 If the client is only interested in the capabilities of the Copy Engines of 615 the given GPU, use nvUvmInterfaceQueryCopyEnginesCaps instead. 616 617 Error codes: 618 NV_ERR_GENERIC 619 NV_ERR_NO_MEMORY 620 */ 621 NV_STATUS nvUvmInterfaceQueryCaps(uvmGpuDeviceHandle device, 622 UvmGpuCaps *caps); 623 624 /******************************************************************************* 625 nvUvmInterfaceQueryCopyEnginesCaps 626 627 Return the capabilities of all the Copy Engines for the provided GPU. 628 If the GPU does not exist, an error will be returned. 629 630 Error codes: 631 NV_ERR_GENERIC 632 NV_ERR_NO_MEMORY 633 */ 634 NV_STATUS nvUvmInterfaceQueryCopyEnginesCaps(uvmGpuDeviceHandle device, 635 UvmGpuCopyEnginesCaps *caps); 636 637 /******************************************************************************* 638 nvUvmInterfaceGetGpuInfo 639 640 Return various gpu info, refer to the UvmGpuInfo struct for details. 641 If no gpu matching the uuid is found, an error will be returned. 642 643 On Ampere+ GPUs, pGpuClientInfo contains SMC information provided by the 644 client regarding the partition targeted in this operation. 645 646 Error codes: 647 NV_ERR_GENERIC 648 NV_ERR_INSUFFICIENT_RESOURCES 649 */ 650 NV_STATUS nvUvmInterfaceGetGpuInfo(const NvProcessorUuid *gpuUuid, 651 const UvmGpuClientInfo *pGpuClientInfo, 652 UvmGpuInfo *pGpuInfo); 653 654 /******************************************************************************* 655 nvUvmInterfaceServiceDeviceInterruptsRM 656 657 Tells RM to service all pending interrupts. This is helpful in ECC error 658 conditions when ECC error interrupt is set & error can be determined only 659 after ECC notifier will be set or reset. 660 661 Error codes: 662 NV_ERR_GENERIC 663 UVM_INVALID_ARGUMENTS 664 */ 665 NV_STATUS nvUvmInterfaceServiceDeviceInterruptsRM(uvmGpuDeviceHandle device); 666 667 /******************************************************************************* 668 nvUvmInterfaceSetPageDirectory 669 Sets pageDirectory in the provided location. Also moves the existing PDE to 670 the provided pageDirectory. 671 672 RM will propagate the update to all channels using the provided VA space. 673 All channels must be idle when this call is made. 674 675 Arguments: 676 vaSpace[IN} - VASpace Object 677 physAddress[IN] - Physical address of new page directory 678 numEntries[IN] - Number of entries including previous PDE which will be copied 679 bVidMemAperture[IN] - If set pageDirectory will reside in VidMem aperture else sysmem 680 pasid[IN] - PASID (Process Address Space IDentifier) of the process 681 corresponding to the VA space. Ignored unless the VA space 682 object has ATS enabled. 683 684 Error codes: 685 NV_ERR_GENERIC 686 NV_ERR_INVALID_ARGUMENT 687 */ 688 NV_STATUS nvUvmInterfaceSetPageDirectory(uvmGpuAddressSpaceHandle vaSpace, 689 NvU64 physAddress, unsigned numEntries, 690 NvBool bVidMemAperture, NvU32 pasid); 691 692 /******************************************************************************* 693 nvUvmInterfaceUnsetPageDirectory 694 Unsets/Restores pageDirectory to RM's defined location. 695 696 Arguments: 697 vaSpace[IN} - VASpace Object 698 699 Error codes: 700 NV_ERR_GENERIC 701 NV_ERR_INVALID_ARGUMENT 702 */ 703 NV_STATUS nvUvmInterfaceUnsetPageDirectory(uvmGpuAddressSpaceHandle vaSpace); 704 705 /******************************************************************************* 706 nvUvmInterfaceDupAllocation 707 708 Duplicate the given allocation in a different VA space. 709 710 The physical handle backing the source allocation is duplicated in 711 the GPU device associated with the destination VA space, and a new mapping 712 is created in that VA space. 713 714 The input allocation can be located in sysmem (i.e. allocated using 715 nvUvmInterfaceMemoryAllocSys) or vidmem (i.e. allocated using 716 nvUvmInterfaceMemoryAllocFB). If located in vidmem, duplication across 717 GPUs is not supported. 718 719 For duplication of physical memory use nvUvmInterfaceDupMemory. 720 721 Arguments: 722 srcVaSpace[IN] - Source VA space. 723 srcAddress[IN] - GPU VA in the source VA space. The provided address 724 should match one previously returned by 725 nvUvmInterfaceMemoryAllocFB or 726 nvUvmInterfaceMemoryAllocSys. 727 dstVaSpace[IN] - Destination VA space where the new mapping will be 728 created. 729 dstVaAlignment[IN] - Alignment of the GPU VA in the destination VA 730 space. 0 means 4KB alignment. 731 dstAddress[OUT] - Pointer to the GPU VA in the destination VA space. 732 733 Error codes: 734 NV_ERR_INVALID_ARGUMENT - If any of the inputs is invalid, or the source 735 and destination VA spaces are identical. 736 NV_ERR_OBJECT_NOT_FOUND - If the input allocation is not found in under 737 the provided VA space. 738 NV_ERR_NO_MEMORY - If there is no memory to back the duplicate, 739 or the associated metadata. 740 NV_ERR_NOT_SUPPORTED - If trying to duplicate vidmem across GPUs. 741 */ 742 NV_STATUS nvUvmInterfaceDupAllocation(uvmGpuAddressSpaceHandle srcVaSpace, 743 NvU64 srcAddress, 744 uvmGpuAddressSpaceHandle dstVaSpace, 745 NvU64 dstVaAlignment, 746 NvU64 *dstAddress); 747 748 /******************************************************************************* 749 nvUvmInterfaceDupMemory 750 751 Duplicates a physical memory allocation. If requested, provides information 752 about the allocation. 753 754 Arguments: 755 device[IN] - Device linked to a client under which 756 the phys memory needs to be duped. 757 hClient[IN] - Client owning the memory. 758 hPhysMemory[IN] - Phys memory which is to be duped. 759 hDupedHandle[OUT] - Handle of the duped memory object. 760 pGpuMemoryInfo[OUT] - see nv_uvm_types.h for more information. 761 This parameter can be NULL. (optional) 762 Error codes: 763 NV_ERR_INVALID_ARGUMENT - If the parameter/s is invalid. 764 NV_ERR_NOT_SUPPORTED - If the allocation is not a physical allocation. 765 NV_ERR_OBJECT_NOT_FOUND - If the allocation is not found in under the provided client. 766 */ 767 NV_STATUS nvUvmInterfaceDupMemory(uvmGpuDeviceHandle device, 768 NvHandle hClient, 769 NvHandle hPhysMemory, 770 NvHandle *hDupMemory, 771 UvmGpuMemoryInfo *pGpuMemoryInfo); 772 773 /******************************************************************************* 774 nvUvmInterfaceFreeDupedAllocation 775 776 Free the allocation represented by the physical handle used to create the 777 duped allocation. 778 779 Arguments: 780 device[IN] - Device handle used to dup the memory. 781 hPhysHandle[IN] - Handle representing the phys allocation. 782 783 Error codes: 784 NV_ERROR 785 NV_ERR_INVALID_ARGUMENT 786 */ 787 NV_STATUS nvUvmInterfaceFreeDupedHandle(uvmGpuDeviceHandle device, 788 NvHandle hPhysHandle); 789 790 /******************************************************************************* 791 nvUvmInterfaceGetFbInfo 792 793 Gets FB information from RM. 794 795 Arguments: 796 device[IN] - GPU device handle 797 fbInfo [OUT] - Pointer to FbInfo structure which contains 798 reservedHeapSize & heapSize 799 Error codes: 800 NV_ERROR 801 NV_ERR_INVALID_ARGUMENT 802 */ 803 NV_STATUS nvUvmInterfaceGetFbInfo(uvmGpuDeviceHandle device, 804 UvmGpuFbInfo * fbInfo); 805 806 /******************************************************************************* 807 nvUvmInterfaceGetEccInfo 808 809 Gets ECC information from RM. 810 811 Arguments: 812 device[IN] - GPU device handle 813 eccInfo [OUT] - Pointer to EccInfo structure 814 815 Error codes: 816 NV_ERROR 817 NV_ERR_INVALID_ARGUMENT 818 */ 819 NV_STATUS nvUvmInterfaceGetEccInfo(uvmGpuDeviceHandle device, 820 UvmGpuEccInfo * eccInfo); 821 822 /******************************************************************************* 823 nvUvmInterfaceOwnPageFaultIntr 824 825 This function transfers ownership of the replayable page fault interrupt, 826 between RM and UVM, for a particular GPU. 827 828 bOwnInterrupts == NV_TRUE: UVM is taking ownership from the RM. This causes 829 the following: RM will not service, enable or disable this interrupt and it 830 is up to the UVM driver to handle this interrupt. In this case, replayable 831 page fault interrupts are disabled by this function, before it returns. 832 833 bOwnInterrupts == NV_FALSE: UVM is returning ownership to the RM: in this 834 case, replayable page fault interrupts MUST BE DISABLED BEFORE CALLING this 835 function. 836 837 The cases above both result in transferring ownership of a GPU that has its 838 replayable page fault interrupts disabled. Doing otherwise would make it 839 very difficult to control which driver handles any interrupts that build up 840 during the hand-off. 841 842 The calling pattern should look like this: 843 844 UVM setting up a new GPU for operation: 845 UVM GPU LOCK 846 nvUvmInterfaceOwnPageFaultIntr(..., NV_TRUE) 847 UVM GPU UNLOCK 848 849 Enable replayable page faults for that GPU 850 851 UVM tearing down a GPU: 852 853 Disable replayable page faults for that GPU 854 855 UVM GPU GPU LOCK 856 nvUvmInterfaceOwnPageFaultIntr(..., NV_FALSE) 857 UVM GPU UNLOCK 858 859 Arguments: 860 gpuUuid[IN] - UUID of the GPU to operate on 861 bOwnInterrupts - Set to NV_TRUE for UVM to take ownership of the 862 replayable page fault interrupts. Set to NV_FALSE 863 to return ownership of the page fault interrupts 864 to RM. 865 Error codes: 866 NV_ERR_GENERIC 867 NV_ERR_INVALID_ARGUMENT 868 */ 869 NV_STATUS nvUvmInterfaceOwnPageFaultIntr(uvmGpuDeviceHandle device, NvBool bOwnInterrupts); 870 871 /******************************************************************************* 872 nvUvmInterfaceInitFaultInfo 873 874 This function obtains fault buffer address, size and a few register mappings 875 for replayable faults, and creates a shadow buffer to store non-replayable 876 faults if the GPU supports it. 877 878 Arguments: 879 device[IN] - Device handle associated with the gpu 880 pFaultInfo[OUT] - information provided by RM for fault handling 881 882 Error codes: 883 NV_ERR_GENERIC 884 NV_ERR_NO_MEMORY 885 NV_ERR_INVALID_ARGUMENT 886 */ 887 NV_STATUS nvUvmInterfaceInitFaultInfo(uvmGpuDeviceHandle device, 888 UvmGpuFaultInfo *pFaultInfo); 889 890 /******************************************************************************* 891 nvUvmInterfaceDestroyFaultInfo 892 893 This function obtains destroys unmaps the fault buffer and clears faultInfo 894 for replayable faults, and frees the shadow buffer for non-replayable faults. 895 896 Arguments: 897 device[IN] - Device handle associated with the gpu 898 pFaultInfo[OUT] - information provided by RM for fault handling 899 900 Error codes: 901 NV_ERR_GENERIC 902 NV_ERR_INVALID_ARGUMENT 903 */ 904 NV_STATUS nvUvmInterfaceDestroyFaultInfo(uvmGpuDeviceHandle device, 905 UvmGpuFaultInfo *pFaultInfo); 906 907 /******************************************************************************* 908 nvUvmInterfaceHasPendingNonReplayableFaults 909 910 This function tells whether there are pending non-replayable faults in the 911 client shadow fault buffer ready to be consumed. 912 913 NOTES: 914 - This function uses a pre-allocated stack per GPU (stored in the 915 UvmGpuFaultInfo object) for calls related to non-replayable faults from the 916 top half. 917 - Concurrent calls to this function using the same pFaultInfo are not 918 thread-safe due to pre-allocated stack. Therefore, locking is the caller's 919 responsibility. 920 - This function DOES NOT acquire the RM API or GPU locks. That is because 921 it is called during fault servicing, which could produce deadlocks. 922 923 Arguments: 924 pFaultInfo[IN] - information provided by RM for fault handling. 925 Contains a pointer to the shadow fault buffer 926 hasPendingFaults[OUT] - return value that tells if there are 927 non-replayable faults ready to be consumed by 928 the client 929 930 Error codes: 931 NV_ERR_INVALID_ARGUMENT 932 */ 933 NV_STATUS nvUvmInterfaceHasPendingNonReplayableFaults(UvmGpuFaultInfo *pFaultInfo, 934 NvBool *hasPendingFaults); 935 936 /******************************************************************************* 937 nvUvmInterfaceGetNonReplayableFaults 938 939 This function consumes all the non-replayable fault packets in the client 940 shadow fault buffer and copies them to the given buffer. It also returns the 941 number of faults that have been copied 942 943 NOTES: 944 - This function uses a pre-allocated stack per GPU (stored in the 945 UvmGpuFaultInfo object) for calls from the bottom half that handles 946 non-replayable faults. 947 - See nvUvmInterfaceHasPendingNonReplayableFaults for the implications of 948 using a shared stack. 949 - This function DOES NOT acquire the RM API or GPU locks. That is because 950 it is called during fault servicing, which could produce deadlocks. 951 952 Arguments: 953 pFaultInfo[IN] - information provided by RM for fault handling. 954 Contains a pointer to the shadow fault buffer 955 pFaultBuffer[OUT] - buffer provided by the client where fault buffers 956 are copied when they are popped out of the shadow 957 fault buffer (which is a circular queue). 958 numFaults[OUT] - return value that tells the number of faults copied 959 to the client's buffer 960 961 Error codes: 962 NV_ERR_INVALID_ARGUMENT 963 */ 964 NV_STATUS nvUvmInterfaceGetNonReplayableFaults(UvmGpuFaultInfo *pFaultInfo, 965 void *pFaultBuffer, 966 NvU32 *numFaults); 967 968 /******************************************************************************* 969 nvUvmInterfaceFlushReplayableFaultBuffer 970 971 This function sends an RPC to GSP in order to flush the HW replayable fault buffer. 972 973 NOTES: 974 - This function DOES NOT acquire the RM API or GPU locks. That is because 975 it is called during fault servicing, which could produce deadlocks. 976 977 Arguments: 978 device[IN] - Device handle associated with the gpu 979 980 Error codes: 981 NV_ERR_INVALID_ARGUMENT 982 */ 983 NV_STATUS nvUvmInterfaceFlushReplayableFaultBuffer(uvmGpuDeviceHandle device); 984 985 /******************************************************************************* 986 nvUvmInterfaceInitAccessCntrInfo 987 988 This function obtains access counter buffer address, size and a few register mappings 989 990 Arguments: 991 device[IN] - Device handle associated with the gpu 992 pAccessCntrInfo[OUT] - Information provided by RM for access counter handling 993 accessCntrIndex[IN] - Access counter index 994 995 Error codes: 996 NV_ERR_GENERIC 997 NV_ERR_INVALID_ARGUMENT 998 */ 999 NV_STATUS nvUvmInterfaceInitAccessCntrInfo(uvmGpuDeviceHandle device, 1000 UvmGpuAccessCntrInfo *pAccessCntrInfo, 1001 NvU32 accessCntrIndex); 1002 1003 /******************************************************************************* 1004 nvUvmInterfaceDestroyAccessCntrInfo 1005 1006 This function obtains, destroys, unmaps the access counter buffer and clears accessCntrInfo 1007 1008 Arguments: 1009 device[IN] - Device handle associated with the gpu 1010 pAccessCntrInfo[IN] - Information provided by RM for access counter handling 1011 1012 Error codes: 1013 NV_ERR_GENERIC 1014 NV_ERR_INVALID_ARGUMENT 1015 */ 1016 NV_STATUS nvUvmInterfaceDestroyAccessCntrInfo(uvmGpuDeviceHandle device, 1017 UvmGpuAccessCntrInfo *pAccessCntrInfo); 1018 1019 /******************************************************************************* 1020 nvUvmInterfaceEnableAccessCntr 1021 1022 This function enables access counters using the given configuration 1023 UVM is also taking ownership from the RM. 1024 This causes the following: RM will not service, enable or disable this 1025 interrupt and it is up to the UVM driver to handle this interrupt. In 1026 this case, access counter notificaion interrupts are enabled by this 1027 function before it returns. 1028 1029 Arguments: 1030 device[IN] - Device handle associated with the gpu 1031 pAccessCntrInfo[IN] - Pointer to structure filled out by nvUvmInterfaceInitAccessCntrInfo 1032 pAccessCntrConfig[IN] - Configuration for access counters 1033 1034 Error codes: 1035 NV_ERR_GENERIC 1036 NV_ERR_INVALID_ARGUMENT 1037 */ 1038 NV_STATUS nvUvmInterfaceEnableAccessCntr(uvmGpuDeviceHandle device, 1039 UvmGpuAccessCntrInfo *pAccessCntrInfo, 1040 UvmGpuAccessCntrConfig *pAccessCntrConfig); 1041 1042 /******************************************************************************* 1043 nvUvmInterfaceDisableAccessCntr 1044 1045 This function disables acccess counters 1046 UVM is also returning ownership to the RM: RM can service, enable or 1047 disable this interrupt. In this case, access counter notificaion interrupts 1048 are disabled by this function before it returns. 1049 1050 Arguments: 1051 device[IN] - Device handle associated with the gpu 1052 pAccessCntrInfo[IN] - Pointer to structure filled out by nvUvmInterfaceInitAccessCntrInfo 1053 1054 Error codes: 1055 NV_ERR_GENERIC 1056 NV_ERR_INVALID_ARGUMENT 1057 */ 1058 NV_STATUS nvUvmInterfaceDisableAccessCntr(uvmGpuDeviceHandle device, 1059 UvmGpuAccessCntrInfo *pAccessCntrInfo); 1060 1061 // 1062 // Called by the UVM driver to register operations with RM. Only one set of 1063 // callbacks can be registered by any driver at a time. If another set of 1064 // callbacks was already registered, NV_ERR_IN_USE is returned. 1065 // 1066 NV_STATUS nvUvmInterfaceRegisterUvmCallbacks(struct UvmOpsUvmEvents *importedUvmOps); 1067 1068 // 1069 // Counterpart to nvUvmInterfaceRegisterUvmCallbacks. This must only be called 1070 // if nvUvmInterfaceRegisterUvmCallbacks returned NV_OK. 1071 // 1072 // Upon return, the caller is guaranteed that any outstanding callbacks are done 1073 // and no new ones will be invoked. 1074 // 1075 void nvUvmInterfaceDeRegisterUvmOps(void); 1076 1077 /******************************************************************************* 1078 nvUvmInterfaceP2pObjectCreate 1079 1080 This API creates an NV50_P2P object for the GPUs with the given device 1081 handles, and returns the handle to the object. 1082 1083 Arguments: 1084 device1[IN] - first GPU device handle 1085 device2[IN] - second GPU device handle 1086 hP2pObject[OUT] - handle to the created P2p object. 1087 1088 Error codes: 1089 NV_ERR_INVALID_ARGUMENT 1090 NV_ERR_OBJECT_NOT_FOUND : If device object associated with the uuids aren't found. 1091 */ 1092 NV_STATUS nvUvmInterfaceP2pObjectCreate(uvmGpuDeviceHandle device1, 1093 uvmGpuDeviceHandle device2, 1094 NvHandle *hP2pObject); 1095 1096 /******************************************************************************* 1097 nvUvmInterfaceP2pObjectDestroy 1098 1099 This API destroys the NV50_P2P associated with the passed handle. 1100 1101 Arguments: 1102 session[IN] - Session handle. 1103 hP2pObject[IN] - handle to an P2p object. 1104 1105 Error codes: NONE 1106 */ 1107 void nvUvmInterfaceP2pObjectDestroy(uvmGpuSessionHandle session, 1108 NvHandle hP2pObject); 1109 1110 /******************************************************************************* 1111 nvUvmInterfaceGetExternalAllocPtes 1112 1113 The interface builds the RM PTEs using the provided input parameters. 1114 1115 Arguments: 1116 vaSpace[IN] - vaSpace handle. 1117 hMemory[IN] - Memory handle. 1118 offset [IN] - Offset from the beginning of the allocation 1119 where PTE mappings should begin. 1120 Should be aligned with mappingPagesize 1121 in gpuExternalMappingInfo associated 1122 with the allocation. 1123 size [IN] - Length of the allocation for which PTEs 1124 should be built. 1125 Should be aligned with mappingPagesize 1126 in gpuExternalMappingInfo associated 1127 with the allocation. 1128 size = 0 will be interpreted as the total size 1129 of the allocation. 1130 gpuExternalMappingInfo[IN/OUT] - See nv_uvm_types.h for more information. 1131 1132 Error codes: 1133 NV_ERR_INVALID_ARGUMENT - Invalid parameter/s is passed. 1134 NV_ERR_INVALID_OBJECT_HANDLE - Invalid memory handle is passed. 1135 NV_ERR_NOT_SUPPORTED - Functionality is not supported (see comments in nv_gpu_ops.c) 1136 NV_ERR_INVALID_BASE - offset is beyond the allocation size 1137 NV_ERR_INVALID_LIMIT - (offset + size) is beyond the allocation size. 1138 NV_ERR_BUFFER_TOO_SMALL - gpuExternalMappingInfo.pteBufferSize is insufficient to 1139 store single PTE. 1140 NV_ERR_NOT_READY - Returned when querying the PTEs requires a deferred setup 1141 which has not yet completed. It is expected that the caller 1142 will reattempt the call until a different code is returned. 1143 */ 1144 NV_STATUS nvUvmInterfaceGetExternalAllocPtes(uvmGpuAddressSpaceHandle vaSpace, 1145 NvHandle hMemory, 1146 NvU64 offset, 1147 NvU64 size, 1148 UvmGpuExternalMappingInfo *gpuExternalMappingInfo); 1149 1150 /******************************************************************************* 1151 nvUvmInterfaceRetainChannel 1152 1153 Validates and returns information about the user's channel and its resources 1154 (local CTX buffers + global CTX buffers). The state is refcounted and must be 1155 released by calling nvUvmInterfaceReleaseChannel. 1156 1157 Arguments: 1158 vaSpace[IN] - vaSpace handle. 1159 hClient[IN] - Client handle 1160 hChannel[IN] - Channel handle 1161 retainedChannel[OUT] - Opaque pointer to use to refer to this 1162 channel in other nvUvmInterface APIs. 1163 channelInstanceInfo[OUT] - Channel instance information to be filled out. 1164 See nv_uvm_types.h for details. 1165 1166 Error codes: 1167 NV_ERR_INVALID_ARGUMENT : If the parameter/s are invalid. 1168 NV_ERR_OBJECT_NOT_FOUND : If the object associated with the handle isn't found. 1169 NV_ERR_INVALID_CHANNEL : If the channel verification fails. 1170 NV_ERR_INSUFFICIENT_RESOURCES : If no memory available to store the resource information. 1171 */ 1172 NV_STATUS nvUvmInterfaceRetainChannel(uvmGpuAddressSpaceHandle vaSpace, 1173 NvHandle hClient, 1174 NvHandle hChannel, 1175 void **retainedChannel, 1176 UvmGpuChannelInstanceInfo *channelInstanceInfo); 1177 1178 /******************************************************************************* 1179 nvUvmInterfaceBindChannelResources 1180 1181 Associates the mapping address of the channel resources (VAs) provided by the 1182 caller with the channel. 1183 1184 Arguments: 1185 retainedChannel[IN] - Channel pointer returned by nvUvmInterfaceRetainChannel 1186 channelResourceBindParams[IN] - Buffer of initialized UvmGpuChannelInstanceInfo::resourceCount 1187 entries. See nv_uvm_types.h for details. 1188 1189 Error codes: 1190 NV_ERR_INVALID_ARGUMENT : If the parameter/s are invalid. 1191 NV_ERR_OBJECT_NOT_FOUND : If the object associated with the handle aren't found. 1192 NV_ERR_INSUFFICIENT_RESOURCES : If no memory available to store the resource information. 1193 */ 1194 NV_STATUS nvUvmInterfaceBindChannelResources(void *retainedChannel, 1195 UvmGpuChannelResourceBindParams *channelResourceBindParams); 1196 1197 /******************************************************************************* 1198 nvUvmInterfaceReleaseChannel 1199 1200 Releases state retained by nvUvmInterfaceRetainChannel. 1201 */ 1202 void nvUvmInterfaceReleaseChannel(void *retainedChannel); 1203 1204 /******************************************************************************* 1205 nvUvmInterfaceStopChannel 1206 1207 Idles the channel and takes it off the runlist. 1208 1209 Arguments: 1210 retainedChannel[IN] - Channel pointer returned by nvUvmInterfaceRetainChannel 1211 bImmediate[IN] - If true, kill the channel without attempting to wait for it to go idle. 1212 */ 1213 void nvUvmInterfaceStopChannel(void *retainedChannel, NvBool bImmediate); 1214 1215 /******************************************************************************* 1216 nvUvmInterfaceGetChannelResourcePtes 1217 1218 The interface builds the RM PTEs using the provided input parameters. 1219 1220 Arguments: 1221 vaSpace[IN] - vaSpace handle. 1222 resourceDescriptor[IN] - The channel resource descriptor returned by returned by 1223 nvUvmInterfaceRetainChannelResources. 1224 offset[IN] - Offset from the beginning of the allocation 1225 where PTE mappings should begin. 1226 Should be aligned with pagesize associated 1227 with the allocation. 1228 size[IN] - Length of the allocation for which PTEs 1229 should be built. 1230 Should be aligned with pagesize associated 1231 with the allocation. 1232 size = 0 will be interpreted as the total size 1233 of the allocation. 1234 gpuExternalMappingInfo[IN/OUT] - See nv_uvm_types.h for more information. 1235 1236 Error codes: 1237 NV_ERR_INVALID_ARGUMENT - Invalid parameter/s is passed. 1238 NV_ERR_INVALID_OBJECT_HANDLE - Invalid memory handle is passed. 1239 NV_ERR_NOT_SUPPORTED - Functionality is not supported. 1240 NV_ERR_INVALID_BASE - offset is beyond the allocation size 1241 NV_ERR_INVALID_LIMIT - (offset + size) is beyond the allocation size. 1242 NV_ERR_BUFFER_TOO_SMALL - gpuExternalMappingInfo.pteBufferSize is insufficient to 1243 store single PTE. 1244 */ 1245 NV_STATUS nvUvmInterfaceGetChannelResourcePtes(uvmGpuAddressSpaceHandle vaSpace, 1246 NvP64 resourceDescriptor, 1247 NvU64 offset, 1248 NvU64 size, 1249 UvmGpuExternalMappingInfo *externalMappingInfo); 1250 1251 /******************************************************************************* 1252 nvUvmInterfaceReportNonReplayableFault 1253 1254 The interface communicates a nonreplayable fault packet from UVM to RM, which 1255 will log the fault, notify the clients and then trigger RC on the channel. 1256 1257 Arguments: 1258 device[IN] - The device where the fault happened. 1259 pFaultPacket[IN] - The opaque pointer from UVM that will be later 1260 converted to a MMU_FAULT_PACKET type. 1261 Error codes: 1262 NV_ERR_INVALID_ARGUMENT - Invalid parameter/s is passed. 1263 NV_ERR_NOT_SUPPORTED - Functionality is not supported. 1264 */ 1265 NV_STATUS nvUvmInterfaceReportNonReplayableFault(uvmGpuDeviceHandle device, 1266 const void *pFaultPacket); 1267 1268 /******************************************************************************* 1269 nvUvmInterfacePagingChannelAllocate 1270 1271 In SR-IOV heavy, this function requests the allocation of a paging channel 1272 (i.e. a privileged CE channel) bound to a specified copy engine. Unlike 1273 channels allocated via nvUvmInterfaceChannelAllocate, the caller cannot push 1274 methods to a paging channel directly, but instead relies on the 1275 nvUvmInterfacePagingChannelPushStream API to do so. 1276 1277 SR-IOV heavy only. The implementation of this interface can acquire 1278 RM or GPU locks. 1279 1280 Arguments: 1281 device[IN] - device under which the paging channel will be allocated 1282 allocParams[IN] - structure with allocation settings 1283 channel[OUT] - pointer to the allocated paging channel handle 1284 channelInfo[OUT] - structure filled with channel information 1285 1286 Error codes: 1287 NV_ERR_INVALID_ARGUMENT - Invalid parameter/s is passed. 1288 NV_ERR_NO_MEMORY - Not enough memory to allocate 1289 paging channel/shadow notifier. 1290 NV_ERR_NOT_SUPPORTED - SR-IOV heavy mode is disabled. 1291 1292 */ 1293 NV_STATUS nvUvmInterfacePagingChannelAllocate(uvmGpuDeviceHandle device, 1294 const UvmGpuPagingChannelAllocParams *allocParams, 1295 UvmGpuPagingChannelHandle *channel, 1296 UvmGpuPagingChannelInfo *channelInfo); 1297 1298 /******************************************************************************* 1299 nvUvmInterfacePagingChannelDestroy 1300 1301 This function destroys a given paging channel. 1302 1303 SR-IOV heavy only. The implementation of this interface can acquire 1304 RM or GPU locks. 1305 1306 Arguments: 1307 channel[IN] - paging channel handle. If the passed handle is 1308 the NULL pointer, the function returns immediately. 1309 */ 1310 void nvUvmInterfacePagingChannelDestroy(UvmGpuPagingChannelHandle channel); 1311 1312 /******************************************************************************* 1313 1314 nvUvmInterfacePagingChannelsMap 1315 1316 Map a guest allocation in the address space associated with all the paging 1317 channels allocated under the given device. 1318 1319 SR-IOV heavy only. The implementation of this interface can acquire 1320 RM or GPU locks. 1321 1322 Arguments: 1323 srcVaSpace[IN] - VA space handle used to allocate the input pointer 1324 srcAddress. 1325 srcAddress[IN] - virtual address returned by nvUvmInterfaceMemoryAllocFB 1326 or nvUvmInterfaceMemoryAllocSys. The entire allocation 1327 backing this guest VA is mapped. 1328 device[IN] - device under which paging channels were allocated 1329 dstAddress[OUT] - a virtual address that is valid (i.e. is mapped) in 1330 all the paging channels allocated under the given vaSpace. 1331 1332 Error codes: 1333 NV_ERR_INVALID_ARGUMENT - Invalid parameter/s is passed. 1334 NV_ERR_NOT_SUPPORTED - SR-IOV heavy mode is disabled. 1335 */ 1336 NV_STATUS nvUvmInterfacePagingChannelsMap(uvmGpuAddressSpaceHandle srcVaSpace, 1337 UvmGpuPointer srcAddress, 1338 uvmGpuDeviceHandle device, 1339 NvU64 *dstAddress); 1340 1341 /******************************************************************************* 1342 1343 nvUvmInterfacePagingChannelsUnmap 1344 1345 Unmap a VA returned by nvUvmInterfacePagingChannelsMap. 1346 1347 SR-IOV heavy only. The implementation of this interface can acquire 1348 RM or GPU locks. 1349 1350 Arguments: 1351 srcVaSpace[IN] - VA space handle that was passed to prevous mapping. 1352 srcAddress[IN] - virtual address that was passed to prevous mapping. 1353 device[IN] - device under which paging channels were allocated. 1354 */ 1355 void nvUvmInterfacePagingChannelsUnmap(uvmGpuAddressSpaceHandle srcVaSpace, 1356 UvmGpuPointer srcAddress, 1357 uvmGpuDeviceHandle device); 1358 1359 1360 /******************************************************************************* 1361 nvUvmInterfacePagingChannelPushStream 1362 1363 Used for remote execution of the passed methods; the UVM driver uses this 1364 interface to ask the vGPU plugin to execute certain HW methods on its 1365 behalf. The callee should push the methods in the specified order i.e. is 1366 not allowed to do any reordering. 1367 1368 The API is asynchronous. The UVM driver can wait on the remote execution by 1369 inserting a semaphore release method at the end of the method stream, and 1370 then loop until the semaphore value reaches the completion value indicated 1371 in the release method. 1372 1373 The valid HW methods that can be passed by the UVM driver follow; the source 1374 functions listed contain the exact formatting (encoding) of the HW method 1375 used by the UVM driver for Ampere. 1376 1377 - TLB invalidation targeting a VA range. See 1378 uvm_hal_volta_host_tlb_invalidate_va. 1379 1380 - TLB invalidation targeting certain levels in the page tree (including 1381 the possibility of invalidating everything). 1382 See uvm_hal_pascal_host_tlb_invalidate_all. 1383 1384 - Replayable fault replay. See uvm_hal_volta_replay_faults. 1385 1386 - Replayable fault cancellation targeting a guest virtual address. See 1387 uvm_hal_volta_cancel_faults_va 1388 1389 - Membar, scoped to device or to the entire system. See 1390 uvm_hal_pascal_host_membar_gpu and uvm_hal_pascal_host_membar_sys 1391 1392 - Host semaphore acquire, see uvm_hal_turing_host_semaphore_acquire. The 1393 virtual address specified in the semaphore operation must lie within a 1394 buffer previously mapped by nvUvmInterfacePagingChannelsMap. 1395 1396 - CE semaphore release, see uvm_hal_pascal_ce_semaphore_release. The 1397 virtual address specified in the semaphore operation must lie within a 1398 buffer previously mapped by nvUvmInterfacePagingChannelsMap. 1399 1400 - 64 bits-wide memset, see uvm_hal_kepler_ce_memset_8. The destination 1401 address is a physical address in vidmem. 1402 1403 - No-op, see uvm_hal_kepler_host_noop. Used to store the source buffer 1404 of a memcopy method within the input stream itself. 1405 1406 - Memcopy, see uvm_hal_kepler_ce_memcopy. The destination address is a 1407 physical address in vidmem. The source address is an offset within 1408 methodStream, in bytes, indicating the location of the (inlined) source 1409 buffer. The copy size does not exceed 4KB. 1410 1411 - CE semaphore release with timestamp, see 1412 uvm_hal_kepler_ce_semaphore_timestamp. The virtual address specified in 1413 the semaphore operation must lie within a buffer previously mapped by 1414 nvUvmInterfacePagingChannelsMap. 1415 1416 - CE semaphore reduction, see uvm_hal_kepler_ce_semaphore_reduction_inc. 1417 The virtual address specified in the semaphore operation must lie within 1418 a buffer previously mapped by nvUvmInterfacePagingChannelsMap. 1419 1420 Only invoked in SR-IOV heavy mode. 1421 1422 NOTES: 1423 - This function uses a pre-allocated stack per paging channel 1424 (stored in the UvmGpuPagingChannel object) 1425 - This function DOES NOT acquire the RM API or GPU locks. That is because 1426 it is called during fault servicing, which could produce deadlocks. 1427 - Concurrent calls to this function using channels under same device are not 1428 allowed due to: 1429 a. pre-allocated stack 1430 b. the fact that internal RPC infrastructure doesn't acquire GPU lock. 1431 Therefore, locking is the caller's responsibility. 1432 1433 Arguments: 1434 channel[IN] - paging channel handle obtained via 1435 nvUvmInterfacePagingChannelAllocate 1436 1437 methodStream[IN] - HW methods to be pushed to the paging channel. 1438 1439 methodStreamSize[IN] - Size of methodStream, in bytes. The maximum push 1440 size is 128KB. 1441 1442 1443 Error codes: 1444 NV_ERR_INVALID_ARGUMENT - Invalid parameter/s is passed. 1445 NV_ERR_NOT_SUPPORTED - SR-IOV heavy mode is disabled. 1446 */ 1447 NV_STATUS nvUvmInterfacePagingChannelPushStream(UvmGpuPagingChannelHandle channel, 1448 char *methodStream, 1449 NvU32 methodStreamSize); 1450 1451 /******************************************************************************* 1452 CSL Interface and Locking 1453 1454 The following functions do not acquire the RM API or GPU locks and must not be called 1455 concurrently with the same UvmCslContext parameter in different threads. The caller must 1456 guarantee this exclusion. 1457 1458 * nvUvmInterfaceCslRotateIv 1459 * nvUvmInterfaceCslEncrypt 1460 * nvUvmInterfaceCslDecrypt 1461 * nvUvmInterfaceCslSign 1462 * nvUvmInterfaceCslQueryMessagePool 1463 * nvUvmInterfaceCslIncrementIv 1464 */ 1465 1466 /******************************************************************************* 1467 nvUvmInterfaceCslInitContext 1468 1469 Allocates and initializes a CSL context for a given secure channel. 1470 1471 The lifetime of the context is the same as the lifetime of the secure channel 1472 it is paired with. 1473 1474 Arguments: 1475 uvmCslContext[IN/OUT] - The CSL context. 1476 channel[IN] - Handle to a secure channel. 1477 1478 Error codes: 1479 NV_ERR_INVALID_STATE - The system is not operating in Confidential Compute mode. 1480 NV_ERR_INVALID_CHANNEL - The associated channel is not a secure channel. 1481 NV_ERR_IN_USE - The context has already been initialized. 1482 */ 1483 NV_STATUS nvUvmInterfaceCslInitContext(UvmCslContext *uvmCslContext, 1484 uvmGpuChannelHandle channel); 1485 1486 /******************************************************************************* 1487 nvUvmInterfaceDeinitCslContext 1488 1489 Securely deinitializes and clears the contents of a context. 1490 1491 If context is already deinitialized then function returns immediately. 1492 1493 Arguments: 1494 uvmCslContext[IN] - The CSL context. 1495 */ 1496 void nvUvmInterfaceDeinitCslContext(UvmCslContext *uvmCslContext); 1497 1498 /******************************************************************************* 1499 nvUvmInterfaceCslRotateIv 1500 1501 Rotates the IV for a given channel and operation. 1502 1503 This function will rotate the IV on both the CPU and the GPU. 1504 Outstanding messages that have been encrypted by the GPU should first be 1505 decrypted before calling this function with operation equal to 1506 UVM_CSL_OPERATION_DECRYPT. Similarly, outstanding messages that have been 1507 encrypted by the CPU should first be decrypted before calling this function 1508 with operation equal to UVM_CSL_OPERATION_ENCRYPT. For a given operation 1509 the channel must be idle before calling this function. This function can be 1510 called regardless of the value of the IV's message counter. 1511 1512 See "CSL Interface and Locking" for locking requirements. 1513 This function does not perform dynamic memory allocation. 1514 1515 Arguments: 1516 uvmCslContext[IN/OUT] - The CSL context. 1517 operation[IN] - Either 1518 - UVM_CSL_OPERATION_ENCRYPT 1519 - UVM_CSL_OPERATION_DECRYPT 1520 1521 Error codes: 1522 NV_ERR_INSUFFICIENT_RESOURCES - The rotate operation would cause a counter 1523 to overflow. 1524 NV_ERR_INVALID_ARGUMENT - Invalid value for operation. 1525 */ 1526 NV_STATUS nvUvmInterfaceCslRotateIv(UvmCslContext *uvmCslContext, 1527 UvmCslOperation operation); 1528 1529 /******************************************************************************* 1530 nvUvmInterfaceCslEncrypt 1531 1532 Encrypts data and produces an authentication tag. 1533 1534 Auth, input, and output buffers must not overlap. If they do then calling 1535 this function produces undefined behavior. Performance is typically 1536 maximized when the input and output buffers are 16-byte aligned. This is 1537 natural alignment for AES block. 1538 The encryptIV can be obtained from nvUvmInterfaceCslIncrementIv. 1539 However, it is optional. If it is NULL, the next IV in line will be used. 1540 1541 See "CSL Interface and Locking" for locking requirements. 1542 This function does not perform dynamic memory allocation. 1543 1544 Arguments: 1545 uvmCslContext[IN/OUT] - The CSL context. 1546 bufferSize[IN] - Size of the input and output buffers in 1547 units of bytes. Value can range from 1 byte 1548 to (2^32) - 1 bytes. 1549 inputBuffer[IN] - Address of plaintext input buffer. 1550 encryptIv[IN/OUT] - IV to use for encryption. Can be NULL. 1551 outputBuffer[OUT] - Address of ciphertext output buffer. 1552 authTagBuffer[OUT] - Address of authentication tag buffer. 1553 Its size is UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES. 1554 1555 Error codes: 1556 NV_ERR_INVALID_ARGUMENT - The size of the data is 0 bytes. 1557 - The encryptIv has already been used. 1558 */ 1559 NV_STATUS nvUvmInterfaceCslEncrypt(UvmCslContext *uvmCslContext, 1560 NvU32 bufferSize, 1561 NvU8 const *inputBuffer, 1562 UvmCslIv *encryptIv, 1563 NvU8 *outputBuffer, 1564 NvU8 *authTagBuffer); 1565 1566 /******************************************************************************* 1567 nvUvmInterfaceCslDecrypt 1568 1569 Verifies the authentication tag and decrypts data. 1570 1571 Auth, input, and output buffers must not overlap. If they do then calling 1572 this function produces undefined behavior. Performance is typically 1573 maximized when the input and output buffers are 16-byte aligned. This is 1574 natural alignment for AES block. 1575 1576 See "CSL Interface and Locking" for locking requirements. 1577 This function does not perform dynamic memory allocation. 1578 1579 Arguments: 1580 uvmCslContext[IN/OUT] - The CSL context. 1581 bufferSize[IN] - Size of the input and output buffers in units of bytes. 1582 Value can range from 1 byte to (2^32) - 1 bytes. 1583 decryptIv[IN] - IV used to decrypt the ciphertext. Its value can either be given by 1584 nvUvmInterfaceCslIncrementIv, or, if NULL, the CSL context's 1585 internal counter is used. 1586 inputBuffer[IN] - Address of ciphertext input buffer. 1587 outputBuffer[OUT] - Address of plaintext output buffer. 1588 addAuthData[IN] - Address of the plaintext additional authenticated data used to 1589 calculate the authentication tag. Can be NULL. 1590 addAuthDataSize[IN] - Size of the additional authenticated data in units of bytes. 1591 Value can range from 1 byte to (2^32) - 1 bytes. 1592 This parameter is ignored if addAuthData is NULL. 1593 authTagBuffer[IN] - Address of authentication tag buffer. 1594 Its size is UVM_CSL_CRYPT_AUTH_TAG_SIZE_BYTES. 1595 1596 Error codes: 1597 NV_ERR_INSUFFICIENT_RESOURCES - The decryption operation would cause a 1598 counter overflow to occur. 1599 NV_ERR_INVALID_ARGUMENT - The size of the data is 0 bytes. 1600 NV_ERR_INVALID_DATA - Verification of the authentication tag fails. 1601 */ 1602 NV_STATUS nvUvmInterfaceCslDecrypt(UvmCslContext *uvmCslContext, 1603 NvU32 bufferSize, 1604 NvU8 const *inputBuffer, 1605 UvmCslIv const *decryptIv, 1606 NvU8 *outputBuffer, 1607 NvU8 const *addAuthData, 1608 NvU32 addAuthDataSize, 1609 NvU8 const *authTagBuffer); 1610 1611 /******************************************************************************* 1612 nvUvmInterfaceCslSign 1613 1614 Generates an authentication tag for secure work launch. 1615 1616 Auth and input buffers must not overlap. If they do then calling this function produces 1617 undefined behavior. 1618 1619 See "CSL Interface and Locking" for locking requirements. 1620 This function does not perform dynamic memory allocation. 1621 1622 Arguments: 1623 uvmCslContext[IN/OUT] - The CSL context. 1624 bufferSize[IN] - Size of the input buffer in units of bytes. 1625 Value can range from 1 byte to (2^32) - 1 bytes. 1626 inputBuffer[IN] - Address of plaintext input buffer. 1627 authTagBuffer[OUT] - Address of authentication tag buffer. 1628 Its size is UVM_CSL_SIGN_AUTH_TAG_SIZE_BYTES. 1629 1630 Error codes: 1631 NV_ERR_INSUFFICIENT_RESOURCES - The signing operation would cause a counter overflow to occur. 1632 NV_ERR_INVALID_ARGUMENT - The size of the data is 0 bytes. 1633 */ 1634 NV_STATUS nvUvmInterfaceCslSign(UvmCslContext *uvmCslContext, 1635 NvU32 bufferSize, 1636 NvU8 const *inputBuffer, 1637 NvU8 *authTagBuffer); 1638 1639 /******************************************************************************* 1640 nvUvmInterfaceCslQueryMessagePool 1641 1642 Returns the number of messages that can be encrypted before the message counter will overflow. 1643 1644 See "CSL Interface and Locking" for locking requirements. 1645 This function does not perform dynamic memory allocation. 1646 1647 Arguments: 1648 uvmCslContext[IN/OUT] - The CSL context. 1649 operation[IN] - Either UVM_CSL_OPERATION_ENCRYPT or UVM_CSL_OPERATION_DECRYPT. 1650 messageNum[OUT] - Number of messages left before overflow. 1651 1652 Error codes: 1653 NV_ERR_INVALID_ARGUMENT - The value of the operation parameter is illegal. 1654 */ 1655 NV_STATUS nvUvmInterfaceCslQueryMessagePool(UvmCslContext *uvmCslContext, 1656 UvmCslOperation operation, 1657 NvU64 *messageNum); 1658 1659 /******************************************************************************* 1660 nvUvmInterfaceCslIncrementIv 1661 1662 Increments the message counter by the specified amount. 1663 1664 If iv is non-NULL then the incremented value is returned. 1665 If operation is UVM_CSL_OPERATION_ENCRYPT then the returned IV's "freshness" bit is set and 1666 can be used in nvUvmInterfaceCslEncrypt. If operation is UVM_CSL_OPERATION_DECRYPT then 1667 the returned IV can be used in nvUvmInterfaceCslDecrypt. 1668 1669 See "CSL Interface and Locking" for locking requirements. 1670 This function does not perform dynamic memory allocation. 1671 1672 Arguments: 1673 uvmCslContext[IN/OUT] - The CSL context. 1674 operation[IN] - Either 1675 - UVM_CSL_OPERATION_ENCRYPT 1676 - UVM_CSL_OPERATION_DECRYPT 1677 increment[IN] - The amount by which the IV is incremented. Can be 0. 1678 iv[out] - If non-NULL, a buffer to store the incremented IV. 1679 1680 Error codes: 1681 NV_ERR_INVALID_ARGUMENT - The value of the operation parameter is illegal. 1682 NV_ERR_INSUFFICIENT_RESOURCES - Incrementing the message counter would result 1683 in an overflow. 1684 */ 1685 NV_STATUS nvUvmInterfaceCslIncrementIv(UvmCslContext *uvmCslContext, 1686 UvmCslOperation operation, 1687 NvU64 increment, 1688 UvmCslIv *iv); 1689 1690 #endif // _NV_UVM_INTERFACE_H_ 1691