1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "kernel/gpu/fifo/kernel_channel.h" 25 #include "kernel/gpu/fifo/kernel_channel_group.h" 26 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 27 #include "kernel/mem_mgr/mem.h" 28 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 29 #include "gpu/mem_mgr/mem_mgr.h" 30 #include "gpu/mem_mgr/mem_desc.h" 31 32 #include "class/cl906f.h" 33 34 #include "published/maxwell/gm107/dev_ram.h" 35 #include "published/maxwell/gm107/dev_mmu.h" 36 37 static NV_STATUS _kchannelCreateRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel); 38 39 static NV_STATUS _kchannelDestroyRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel); 40 41 /*! 42 * The reason this is a hal method is because it primarily concerns with returning EngineID 43 * and unfortunately, the defines for these are not standard across chips. 44 * 45 * Reason we have a classEngineID concat is to present an opaque handle that clients can do 46 * setobject with directly. Some of them may also require to know the class, engine IDs. 47 */ 48 NV_STATUS 49 kchannelGetClassEngineID_GM107 50 ( 51 OBJGPU *pGpu, 52 KernelChannel *pKernelChannel, 53 NvHandle handle, 54 NvU32 *pClassEngineID, 55 NvU32 *pClassID, 56 RM_ENGINE_TYPE *pRmEngineID 57 ) 58 { 59 NV_STATUS status = NV_OK; 60 NvU32 halEngineTag; 61 NvU32 classID; 62 ChannelDescendant *pObject = NULL; 63 64 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 65 kchannelFindChildByHandle(pKernelChannel, handle, &pObject)); 66 NV_ASSERT_OR_RETURN(pObject != NULL, NV_ERR_OBJECT_NOT_FOUND); 67 68 *pClassID = classID = RES_GET_EXT_CLASS_ID(pObject); 69 halEngineTag = pObject->resourceDesc.engDesc; 70 71 if (halEngineTag == ENG_SW) 72 { 73 classID = pObject->classID; 74 } 75 76 status = gpuXlateEngDescToClientEngineId(pGpu, halEngineTag, pRmEngineID); 77 78 if (status == NV_OK) 79 { 80 *pClassEngineID = DRF_NUM(906F, _SET_OBJECT, _NVCLASS, classID); 81 } 82 83 NV_PRINTF(LEVEL_INFO, 84 "class ID: 0x%08x classEngine ID: 0x%08x\n", 85 classID, *pClassEngineID); 86 87 return status; 88 } 89 90 NV_STATUS 91 kchannelEnableVirtualContext_GM107 92 ( 93 KernelChannel *pKernelChannel 94 ) 95 { 96 pKernelChannel->bSkipCtxBufferAlloc = NV_TRUE; 97 98 return NV_OK; 99 } 100 101 /*! 102 * Create the sub memory descriptor from preallocated USERD memory 103 * allocated by RM for a channel 104 */ 105 static NV_STATUS 106 _kchannelCreateRMUserdMemDesc 107 ( 108 OBJGPU *pGpu, 109 KernelChannel *pKernelChannel 110 ) 111 { 112 NV_STATUS status = NV_OK; 113 NvU32 userdSize; 114 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 115 const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo); 116 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 117 MEMORY_DESCRIPTOR **ppUserdSubdevMemDesc = 118 &pKernelChannel->pUserdSubDeviceMemDesc[subdevInst]; 119 120 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL); 121 122 status = memdescCreateSubMem(ppUserdSubdevMemDesc, 123 pUserdInfo->userdPhysDesc[subdevInst], 124 pGpu, 125 pKernelChannel->ChID * userdSize, 126 userdSize ); 127 return status; 128 } 129 130 /*! Delete prealloc userd submemdesc for the channel */ 131 static NV_STATUS 132 _kchannelDestroyRMUserdMemDesc 133 ( 134 OBJGPU *pGpu, 135 KernelChannel *pKernelChannel 136 ) 137 { 138 MEMORY_DESCRIPTOR **ppUserdSubdevMemDesc = 139 &pKernelChannel->pUserdSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 140 if ((ppUserdSubdevMemDesc != NULL) && (*ppUserdSubdevMemDesc != NULL)) 141 { 142 memdescFree(*ppUserdSubdevMemDesc); 143 memdescDestroy(*ppUserdSubdevMemDesc); 144 *ppUserdSubdevMemDesc = NULL; 145 } 146 147 return NV_OK; 148 } 149 150 /*! 151 * @brief Create and alloc channel instance mem, 152 * ramfc and userd subdevice memdescs. 153 * 154 * @param pGpu[in] OBJGPU pointer 155 * @param pKernelChannel[in] KernelChannel pointer 156 * @param flags[in] Flags 157 * @param verifFlags[in] verifFlags 158 * 159 * @returns NV_STATUS 160 */ 161 NV_STATUS kchannelAllocMem_GM107 162 ( 163 OBJGPU *pGpu, 164 KernelChannel *pKernelChannel, 165 NvU32 Flags, 166 NvU32 verifFlags 167 ) 168 { 169 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 170 NV_STATUS status = NV_OK; 171 NvU32 CpuCacheAttrib; 172 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL; 173 NvU32 userdSize; 174 NvU64 instMemSize; 175 NvU64 instMemAlign; 176 NvBool bInstProtectedMem; 177 const NV_ADDRESS_SPACE *pInstAllocList; 178 CTX_BUF_POOL_INFO *pChannelBufPool = NULL; 179 NvU64 allocFlags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 180 NvU32 scgType; 181 NvU32 runqueue; 182 KernelChannelGroup *pKernelChannelGroup; 183 NvU32 subdevInst; 184 185 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 186 pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup; 187 188 scgType = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_THREAD, Flags); 189 runqueue = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, Flags); 190 191 if (!kfifoValidateSCGTypeAndRunqueue_HAL(pKernelFifo, scgType, runqueue)) 192 return NV_ERR_INVALID_ARGUMENT; 193 194 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL); 195 196 status = kfifoGetInstMemInfo_HAL(pKernelFifo, &instMemSize, &instMemAlign, 197 &bInstProtectedMem, &CpuCacheAttrib, &pInstAllocList); 198 if (status != NV_OK) 199 { 200 NV_PRINTF(LEVEL_ERROR, "Unable to get instance memory info!\n"); 201 goto fail; 202 } 203 204 /// Alloc Instance block 205 if (IsSLIEnabled(pGpu) || IS_GSP_CLIENT(pGpu)) 206 { 207 pInstAllocList = ADDRLIST_FBMEM_ONLY; 208 CpuCacheAttrib = NV_MEMORY_UNCACHED; 209 } 210 211 // check for allocating VPR memory 212 if (bInstProtectedMem) 213 allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED; 214 215 pChannelBufPool = pKernelChannelGroup->pChannelBufPool; 216 if (pChannelBufPool != NULL) 217 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 218 219 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 220 221 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 222 pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst]; 223 224 if (pInstanceBlock == NULL) 225 { 226 NV_PRINTF(LEVEL_ERROR, 227 "Instance block is NULL for hClient 0x%x Channel 0x%x!\n", 228 RES_GET_CLIENT_HANDLE(pKernelChannel), 229 RES_GET_HANDLE(pKernelChannel)); 230 SLI_LOOP_BREAK; 231 } 232 233 status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu, 234 instMemSize, instMemAlign, NV_TRUE, 235 ADDR_UNKNOWN, CpuCacheAttrib, allocFlags); 236 if (status != NV_OK) 237 { 238 NV_PRINTF(LEVEL_ERROR, "Unable to allocate instance memory descriptor!\n"); 239 SLI_LOOP_BREAK; 240 } 241 242 if ((memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc) == ADDR_SYSMEM) && 243 (gpuIsInstanceMemoryAlwaysCached(pGpu))) 244 { 245 memdescSetGpuCacheAttrib(pInstanceBlock->pInstanceBlockDesc, NV_MEMORY_CACHED); 246 } 247 248 if (pChannelBufPool != NULL) 249 { 250 status = memdescSetCtxBufPool(pInstanceBlock->pInstanceBlockDesc, pChannelBufPool); 251 if (status != NV_OK) 252 { 253 NV_ASSERT(status == NV_OK); 254 SLI_LOOP_BREAK; 255 } 256 } 257 258 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_116, 259 pInstanceBlock->pInstanceBlockDesc, pInstAllocList); 260 if (status == NV_OK) 261 { 262 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 263 264 // Initialize the instance block of the channel with zeros 265 status = memmgrMemDescMemSet(pMemoryManager, 266 pInstanceBlock->pInstanceBlockDesc, 267 0, 268 TRANSFER_FLAGS_NONE); 269 if (status != NV_OK) 270 { 271 NV_ASSERT(status == NV_OK); 272 SLI_LOOP_BREAK; 273 } 274 275 memdescSetName(pGpu, pInstanceBlock->pInstanceBlockDesc, NV_RM_SURF_NAME_INSTANCE_BLOCK, NULL); 276 } 277 else 278 { 279 NV_PRINTF(LEVEL_ERROR, 280 "Instance block allocation for hClient 0x%x hChannel 0x%x failed\n", 281 RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_HANDLE(pKernelChannel)); 282 SLI_LOOP_BREAK; 283 } 284 285 /// Alloc RAMFC Desc 286 status = memdescCreateSubMem(&pInstanceBlock->pRamfcDesc, 287 pInstanceBlock->pInstanceBlockDesc, 288 pGpu, 0, DRF_SIZE( NV_RAMIN_RAMFC ) / 8); 289 if (status != NV_OK) 290 { 291 NV_PRINTF(LEVEL_ERROR, "Could not allocate memdesc for RAMFC\n"); 292 SLI_LOOP_BREAK; 293 } 294 295 // TODO: Move this elsewhere. 296 if (!pKernelChannel->bClientAllocatedUserD) 297 { 298 NV_ASSERT(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL); 299 300 status = _kchannelCreateRMUserdMemDesc(pGpu, pKernelChannel); 301 if (status != NV_OK) 302 { 303 NV_PRINTF(LEVEL_ERROR, "Could not allocate sub memdesc for USERD\n"); // TODO SLI BREAK 304 SLI_LOOP_BREAK; 305 } 306 } 307 308 status = kchannelCreateUserMemDesc_HAL(pGpu, pKernelChannel); 309 if (status != NV_OK) 310 { 311 NV_PRINTF(LEVEL_ERROR, "kchannelCreateUserMemDesc failed \n"); 312 SLI_LOOP_BREAK; 313 } 314 315 NV_PRINTF(LEVEL_INFO, 316 "hChannel 0x%x hClient 0x%x, Class ID 0x%x " 317 "Instance Block @ 0x%llx (%s %x) " 318 "USERD @ 0x%llx " 319 "for subdevice %d\n", 320 RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel), 321 memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0), 322 memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 323 (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 324 (pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL) ? 0x0LL : 325 memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst], AT_GPU, 0LL), 326 subdevInst); 327 328 SLI_LOOP_END 329 330 fail: 331 // Just a note about our failure path, null and unallocated 332 // memdescFrees are allowed so this is not a bug. 333 if (status != NV_OK) 334 { 335 NV_PRINTF(LEVEL_ERROR, "Could not create Channel\n"); 336 DBG_BREAKPOINT(); 337 } 338 339 return status; 340 } 341 342 /*! 343 * @brief Free and destroy channel memdescs 344 * created during channel alloc mem 345 * 346 * @param pGpu[in] OBJGPU pointer 347 * @param pKernelChannel[in] KernelChannel pointer 348 * 349 * @return void 350 */ 351 void 352 kchannelDestroyMem_GM107 353 ( 354 OBJGPU *pGpu, 355 KernelChannel *pKernelChannel 356 ) 357 { 358 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL; 359 NvU32 subdevInst; 360 361 NV_ASSERT_OR_RETURN_VOID(pKernelChannel != NULL); 362 363 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 364 365 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 366 367 if (!pKernelChannel->bClientAllocatedUserD) 368 { 369 _kchannelDestroyRMUserdMemDesc(pGpu, pKernelChannel); 370 } 371 372 pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst]; 373 if (pInstanceBlock != NULL) 374 { 375 // Release RAMFC sub memdesc 376 if (pInstanceBlock->pRamfcDesc != NULL) 377 { 378 memdescFree(pInstanceBlock->pRamfcDesc); 379 memdescDestroy(pInstanceBlock->pRamfcDesc); 380 pInstanceBlock->pRamfcDesc = NULL; 381 } 382 383 // Release Inst block Desc 384 if (pInstanceBlock->pInstanceBlockDesc != NULL) 385 { 386 memdescFree(pInstanceBlock->pInstanceBlockDesc); 387 memdescDestroy(pInstanceBlock->pInstanceBlockDesc); 388 pInstanceBlock->pInstanceBlockDesc = NULL; 389 } 390 } 391 392 // Remove USERD memDescs 393 memdescDestroy(pKernelChannel->pInstSubDeviceMemDesc[subdevInst]); 394 pKernelChannel->pInstSubDeviceMemDesc[subdevInst] = NULL; 395 396 SLI_LOOP_END 397 398 return; 399 } 400 401 /** 402 * @brief reserves a hardware channel slot 403 * 404 * Only responsible for indicating a hardware channel is in use, does not set 405 * any other software state. 406 * 407 * @param pGpu 408 * @param[in] pKernelChannel the pre-allocated KernelChannel 409 * @param[in] hClient 410 * @param[in] allocMode CHANNEL_HW_ID_ALLC_MODE_* 411 * @param[in] ChID 412 * @param[in] bForceInternalIdx true if requesting specific index within USERD page 413 * @param[in] internalIdx requested index within USERD page when bForceInternalIdx 414 * true 415 */ 416 NV_STATUS 417 kchannelAllocHwID_GM107 418 ( 419 OBJGPU *pGpu, 420 KernelChannel *pKernelChannel, 421 NvHandle hClient, 422 NvU32 Flags, 423 NvU32 verifFlags2, 424 NvU32 ChID 425 ) 426 { 427 NV_STATUS status; 428 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 429 CHID_MGR *pChidMgr = NULL; 430 NvU32 internalIdx = 0; 431 NvU32 userdPageIdx = 0; 432 NvBool bForceInternalIdx = NV_FALSE; 433 NvBool bForceUserdPage = NV_FALSE; 434 CHANNEL_HW_ID_ALLOC_MODE allocMode = CHANNEL_HW_ID_ALLOC_MODE_GROW_UP; 435 436 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 437 { 438 // per runlist channel heap is supported for sriov only 439 NV_CHECK_OR_RETURN(LEVEL_ERROR, 440 !kfifoIsPerRunlistChramEnabled(pKernelFifo), 441 NV_ERR_INVALID_STATE); 442 443 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, 444 CHIDMGR_RUNLIST_ID_LEGACY); 445 446 NV_CHECK_OR_RETURN(LEVEL_INFO, 447 kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 448 pChidMgr, ChID) == NULL, 449 NV_OK); 450 allocMode = CHANNEL_HW_ID_ALLOC_MODE_PROVIDED; 451 } 452 else 453 { 454 if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_FIXED, _TRUE, Flags)) 455 { 456 bForceUserdPage = NV_TRUE; 457 userdPageIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_VALUE, Flags); 458 internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags); 459 460 NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS04, 461 _FLAGS, 462 _CHANNEL_USERD_INDEX_FIXED, 463 _FALSE, 464 Flags), 465 NV_ERR_INVALID_STATE); 466 } 467 468 if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_FIXED, _TRUE, Flags)) 469 { 470 bForceInternalIdx = NV_TRUE; 471 internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags); 472 } 473 } 474 475 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, kchannelGetRunlistId(pKernelChannel)); 476 477 status = kfifoChidMgrAllocChid(pGpu, 478 pKernelFifo, 479 pChidMgr, 480 hClient, 481 allocMode, 482 bForceInternalIdx, 483 internalIdx, 484 bForceUserdPage, 485 userdPageIdx, 486 ChID, 487 pKernelChannel); 488 489 return status; 490 } 491 492 /** 493 * @brief Releases a hardware channel ID. 494 * 495 * Not responsible for freeing any software state beyond that which indicates a 496 * hardware channel is in use. 497 * 498 * @param pGpu 499 * @param pKernelChannel 500 */ 501 NV_STATUS 502 kchannelFreeHwID_GM107 503 ( 504 OBJGPU *pGpu, 505 KernelChannel *pKernelChannel 506 ) 507 { 508 NV_STATUS status; 509 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 510 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, 511 kchannelGetRunlistId(pKernelChannel)); 512 EMEMBLOCK *pFifoDataBlock; 513 514 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 515 pChidMgr->pFifoDataHeap, 516 pKernelChannel->ChID, 517 NV_FALSE); 518 NV_ASSERT_OR_RETURN(pFifoDataBlock, NV_ERR_OBJECT_NOT_FOUND); 519 NV_ASSERT(pFifoDataBlock->pData == pKernelChannel); 520 521 status = kfifoChidMgrFreeChid(pGpu, pKernelFifo, pChidMgr, pKernelChannel->ChID); 522 if (status != NV_OK) 523 { 524 NV_PRINTF(LEVEL_ERROR, 525 "Unable to Free Channel From Heap: %d\n", 526 pKernelChannel->ChID); 527 DBG_BREAKPOINT(); 528 } 529 530 return status; 531 } 532 533 NV_STATUS 534 kchannelGetUserdInfo_GM107 535 ( 536 OBJGPU *pGpu, 537 KernelChannel *pKernelChannel, 538 NvU64 *userBase, 539 NvU64 *offset, 540 NvU64 *length 541 ) 542 { 543 NV_STATUS status; 544 NvU64 bar1MapOffset; 545 NvU32 bar1MapSize; 546 CLI_CHANNEL_CLASS_INFO classInfo; 547 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 548 549 NvBool bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING); 550 551 CliGetChannelClassInfo(RES_GET_EXT_CLASS_ID(pKernelChannel), &classInfo); 552 553 switch (classInfo.classType) 554 { 555 case CHANNEL_CLASS_TYPE_GPFIFO: 556 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 557 558 // USERD is not pre-allocated in BAR1 so there is no offset/userBase 559 NV_ASSERT_OR_RETURN(!pKernelChannel->bClientAllocatedUserD, 560 NV_ERR_INVALID_REQUEST); 561 562 status = kchannelGetUserdBar1MapOffset_HAL(pGpu, 563 pKernelChannel, 564 &bar1MapOffset, 565 &bar1MapSize); 566 if (status == NV_OK) 567 { 568 *offset = bar1MapOffset; 569 *length = bar1MapSize; 570 571 if (userBase) 572 { 573 if (bCoherentCpuMapping) 574 { 575 NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED)); 576 *userBase = pKernelMemorySystem->coherentCpuFbBase; 577 } 578 else 579 { 580 *userBase = gpumgrGetGpuPhysFbAddr(pGpu); 581 } 582 } 583 } 584 break; 585 586 default: 587 NV_PRINTF(LEVEL_ERROR, 588 "class = %x not supported for user base mapping\n", 589 RES_GET_EXT_CLASS_ID(pKernelChannel)); 590 status = NV_ERR_GENERIC; 591 break; 592 } 593 return status; 594 } 595 596 // 597 // Takes as input a Channel * and returns the BAR1 offset that this channel's 598 // USERD has been mapped to. Also returns the size of the BAR1 mapping that 599 // pertains to this channel. The BAR1 map of all USERDs should have already 600 // been setup before the first channel was created. 601 // 602 // For example, USERD of 40 channels have been mapped at BAR1 offset 0x100. 603 // USERD of one channel is of size 4k. In which case this function will return 604 // ( 0x100 + ( 0x1000 * 0xa ) ) if the input ChID = 0xa. 605 // 606 NV_STATUS 607 kchannelGetUserdBar1MapOffset_GM107 608 ( 609 OBJGPU *pGpu, 610 KernelChannel *pKernelChannel, 611 NvU64 *bar1MapOffset, 612 NvU32 *bar1MapSize 613 ) 614 { 615 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 616 const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo); 617 618 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 619 620 // 621 // only supported when bUsePerRunlistChannelRam is disabled. 622 // We don't pre-allocate userd for all channels across all runlists; we expect 623 // clients to have moved to client allocated userd. 624 // 625 NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo), 626 NV_ERR_NOT_SUPPORTED); 627 628 if (pUserdInfo->userdBar1MapSize == 0) 629 { 630 NV_PRINTF(LEVEL_ERROR, 631 "fifoGetUserdBar1Offset_GF100: BAR1 map of USERD has not " 632 "been setup yet\n"); 633 NV_ASSERT(0); 634 return NV_ERR_GENERIC; 635 } 636 637 kfifoGetUserdSizeAlign_HAL(pKernelFifo, bar1MapSize, NULL); 638 639 *bar1MapOffset = pKernelChannel->ChID * *bar1MapSize + 640 pUserdInfo->userdBar1MapStartOffset; 641 642 NV_ASSERT((*bar1MapOffset + *bar1MapSize) <= 643 (pUserdInfo->userdBar1MapStartOffset + 644 pUserdInfo->userdBar1MapSize)); 645 646 return NV_OK; 647 } 648 649 /*! 650 * @brief Creates a memory descriptor to be used for creating a GPU mapped MMIO 651 * region for a given channel. 652 */ 653 NV_STATUS 654 kchannelCreateUserMemDesc_GM107 655 ( 656 OBJGPU *pGpu, 657 KernelChannel *pKernelChannel 658 ) 659 { 660 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 661 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 662 NV_STATUS status; 663 MEMORY_DESCRIPTOR *pSubDevInstMemDesc = NULL; 664 MEMORY_DESCRIPTOR **ppMemDesc = 665 &pKernelChannel->pInstSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 666 667 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); 668 669 status = kfifoChannelGetFifoContextMemDesc_HAL(pGpu, pKernelFifo, pKernelChannel, 670 FIFO_CTX_INST_BLOCK, &pSubDevInstMemDesc); 671 672 if (status != NV_OK) 673 return status; 674 675 NV_ASSERT_OR_RETURN(pSubDevInstMemDesc, NV_ERR_OBJECT_NOT_FOUND); 676 677 status = memdescCreate(ppMemDesc, pGpu, RM_PAGE_SIZE, 0, 678 memdescGetContiguity(pSubDevInstMemDesc, AT_GPU), 679 memdescGetAddressSpace(pSubDevInstMemDesc), 680 memdescGetCpuCacheAttrib(pSubDevInstMemDesc), 681 MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE); 682 683 if (status != NV_OK) 684 return status; 685 686 NV_ASSERT(*ppMemDesc); 687 688 memdescDescribe(*ppMemDesc, memdescGetAddressSpace(pSubDevInstMemDesc), 689 memdescGetPhysAddr(pSubDevInstMemDesc, AT_GPU, 0), RM_PAGE_SIZE); 690 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB); 691 692 memdescSetPteKind(*ppMemDesc, NV_MMU_PTE_KIND_SMHOST_MESSAGE); 693 694 return NV_OK; 695 } 696 697 /** 698 * @brief Retrieves the engine ID (NV_PFIFO_ENGINE_*) a given channel is operating on. 699 * 700 * This value will not be valid for a channel that has not been scheduled. 701 * 702 * @param pGpu 703 * @param pKernelChannel 704 * @param[out] pEngDesc 705 */ 706 NV_STATUS 707 kchannelGetEngine_GM107 708 ( 709 OBJGPU *pGpu, 710 KernelChannel *pKernelChannel, 711 NvU32 *pEngDesc 712 ) 713 { 714 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 715 716 if (pEngDesc == NULL) 717 return NV_ERR_INVALID_ARGUMENT; 718 719 NV_PRINTF(LEVEL_INFO, "0x%x\n", kchannelGetDebugTag(pKernelChannel)); 720 721 *pEngDesc = kchannelGetRunlistId(pKernelChannel); 722 723 // This will pick the first engine on this runlist (may not be the only one). 724 NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 725 ENGINE_INFO_TYPE_RUNLIST, *pEngDesc, 726 ENGINE_INFO_TYPE_ENG_DESC, pEngDesc)); 727 728 return NV_OK; 729 } 730