1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "kernel/gpu/fifo/kernel_channel.h" 25 #include "kernel/gpu/fifo/kernel_channel_group.h" 26 #include "kernel/gpu/fifo/kernel_channel_group_api.h" 27 #include "kernel/mem_mgr/mem.h" 28 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 29 #include "gpu/mem_mgr/mem_mgr.h" 30 #include "gpu/mem_mgr/mem_desc.h" 31 #include "platform/sli/sli.h" 32 33 #include "class/cl906f.h" 34 35 #include "published/maxwell/gm107/dev_ram.h" 36 #include "published/maxwell/gm107/dev_mmu.h" 37 38 static NV_STATUS _kchannelCreateRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel); 39 40 static NV_STATUS _kchannelDestroyRMUserdMemDesc(OBJGPU *pGpu, KernelChannel *pKernelChannel); 41 42 /*! 43 * The reason this is a hal method is because it primarily concerns with returning EngineID 44 * and unfortunately, the defines for these are not standard across chips. 45 * 46 * Reason we have a classEngineID concat is to present an opaque handle that clients can do 47 * setobject with directly. Some of them may also require to know the class, engine IDs. 48 */ 49 NV_STATUS 50 kchannelGetClassEngineID_GM107 51 ( 52 OBJGPU *pGpu, 53 KernelChannel *pKernelChannel, 54 NvHandle handle, 55 NvU32 *pClassEngineID, 56 NvU32 *pClassID, 57 RM_ENGINE_TYPE *pRmEngineID 58 ) 59 { 60 NV_STATUS status = NV_OK; 61 NvU32 halEngineTag; 62 NvU32 classID; 63 ChannelDescendant *pObject = NULL; 64 65 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, 66 kchannelFindChildByHandle(pKernelChannel, handle, &pObject)); 67 NV_ASSERT_OR_RETURN(pObject != NULL, NV_ERR_OBJECT_NOT_FOUND); 68 69 *pClassID = classID = RES_GET_EXT_CLASS_ID(pObject); 70 halEngineTag = pObject->resourceDesc.engDesc; 71 72 if (halEngineTag == ENG_SW) 73 { 74 classID = pObject->classID; 75 } 76 77 status = gpuXlateEngDescToClientEngineId(pGpu, halEngineTag, pRmEngineID); 78 79 if (status == NV_OK) 80 { 81 *pClassEngineID = DRF_NUM(906F, _SET_OBJECT, _NVCLASS, classID); 82 } 83 84 NV_PRINTF(LEVEL_INFO, 85 "class ID: 0x%08x classEngine ID: 0x%08x\n", 86 classID, *pClassEngineID); 87 88 return status; 89 } 90 91 NV_STATUS 92 kchannelEnableVirtualContext_GM107 93 ( 94 KernelChannel *pKernelChannel 95 ) 96 { 97 pKernelChannel->bSkipCtxBufferAlloc = NV_TRUE; 98 99 return NV_OK; 100 } 101 102 /*! 103 * Create the sub memory descriptor from preallocated USERD memory 104 * allocated by RM for a channel 105 */ 106 static NV_STATUS 107 _kchannelCreateRMUserdMemDesc 108 ( 109 OBJGPU *pGpu, 110 KernelChannel *pKernelChannel 111 ) 112 { 113 NV_STATUS status = NV_OK; 114 NvU32 userdSize; 115 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 116 const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo); 117 NvU32 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 118 MEMORY_DESCRIPTOR **ppUserdSubdevMemDesc = 119 &pKernelChannel->pUserdSubDeviceMemDesc[subdevInst]; 120 121 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL); 122 123 status = memdescCreateSubMem(ppUserdSubdevMemDesc, 124 pUserdInfo->userdPhysDesc[subdevInst], 125 pGpu, 126 pKernelChannel->ChID * userdSize, 127 userdSize ); 128 return status; 129 } 130 131 /*! Delete prealloc userd submemdesc for the channel */ 132 static NV_STATUS 133 _kchannelDestroyRMUserdMemDesc 134 ( 135 OBJGPU *pGpu, 136 KernelChannel *pKernelChannel 137 ) 138 { 139 MEMORY_DESCRIPTOR **ppUserdSubdevMemDesc = 140 &pKernelChannel->pUserdSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 141 if ((ppUserdSubdevMemDesc != NULL) && (*ppUserdSubdevMemDesc != NULL)) 142 { 143 memdescFree(*ppUserdSubdevMemDesc); 144 memdescDestroy(*ppUserdSubdevMemDesc); 145 *ppUserdSubdevMemDesc = NULL; 146 } 147 148 return NV_OK; 149 } 150 151 /*! 152 * @brief Create and alloc channel instance mem, 153 * ramfc and userd subdevice memdescs. 154 * 155 * @param pGpu[in] OBJGPU pointer 156 * @param pKernelChannel[in] KernelChannel pointer 157 * @param flags[in] Flags 158 * @param verifFlags[in] verifFlags 159 * 160 * @returns NV_STATUS 161 */ 162 NV_STATUS kchannelAllocMem_GM107 163 ( 164 OBJGPU *pGpu, 165 KernelChannel *pKernelChannel, 166 NvU32 Flags, 167 NvU32 verifFlags 168 ) 169 { 170 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 171 NV_STATUS status = NV_OK; 172 NvU32 CpuCacheAttrib; 173 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL; 174 NvU32 userdSize; 175 NvU64 instMemSize; 176 NvU64 instMemAlign; 177 NvBool bInstProtectedMem; 178 const NV_ADDRESS_SPACE *pInstAllocList; 179 CTX_BUF_POOL_INFO *pChannelBufPool = NULL; 180 NvU64 allocFlags = MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE; 181 NvU32 scgType; 182 NvU32 runqueue; 183 KernelChannelGroup *pKernelChannelGroup; 184 NvU32 subdevInst; 185 186 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 187 pKernelChannelGroup = pKernelChannel->pKernelChannelGroupApi->pKernelChannelGroup; 188 189 scgType = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_THREAD, Flags); 190 runqueue = DRF_VAL(OS04, _FLAGS, _GROUP_CHANNEL_RUNQUEUE, Flags); 191 192 if (!kfifoValidateSCGTypeAndRunqueue_HAL(pKernelFifo, scgType, runqueue)) 193 return NV_ERR_INVALID_ARGUMENT; 194 195 kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL); 196 197 status = kfifoGetInstMemInfo_HAL(pKernelFifo, &instMemSize, &instMemAlign, 198 &bInstProtectedMem, &CpuCacheAttrib, &pInstAllocList); 199 if (status != NV_OK) 200 { 201 NV_PRINTF(LEVEL_ERROR, "Unable to get instance memory info!\n"); 202 goto fail; 203 } 204 205 /// Alloc Instance block 206 if (IsSLIEnabled(pGpu) || IS_GSP_CLIENT(pGpu)) 207 { 208 pInstAllocList = ADDRLIST_FBMEM_ONLY; 209 CpuCacheAttrib = NV_MEMORY_UNCACHED; 210 } 211 212 // check for allocating VPR memory 213 if (bInstProtectedMem) 214 allocFlags |= MEMDESC_ALLOC_FLAGS_PROTECTED; 215 216 pChannelBufPool = pKernelChannelGroup->pChannelBufPool; 217 if (pChannelBufPool != NULL) 218 allocFlags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL; 219 220 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 221 222 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 223 pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst]; 224 225 if (pInstanceBlock == NULL) 226 { 227 NV_PRINTF(LEVEL_ERROR, 228 "Instance block is NULL for hClient 0x%x Channel 0x%x!\n", 229 RES_GET_CLIENT_HANDLE(pKernelChannel), 230 RES_GET_HANDLE(pKernelChannel)); 231 SLI_LOOP_BREAK; 232 } 233 234 status = memdescCreate(&pInstanceBlock->pInstanceBlockDesc, pGpu, 235 instMemSize, instMemAlign, NV_TRUE, 236 ADDR_UNKNOWN, CpuCacheAttrib, allocFlags); 237 if (status != NV_OK) 238 { 239 NV_PRINTF(LEVEL_ERROR, "Unable to allocate instance memory descriptor!\n"); 240 SLI_LOOP_BREAK; 241 } 242 243 if ((memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc) == ADDR_SYSMEM) && 244 (gpuIsInstanceMemoryAlwaysCached(pGpu))) 245 { 246 memdescSetGpuCacheAttrib(pInstanceBlock->pInstanceBlockDesc, NV_MEMORY_CACHED); 247 } 248 249 if (pChannelBufPool != NULL) 250 { 251 status = memdescSetCtxBufPool(pInstanceBlock->pInstanceBlockDesc, pChannelBufPool); 252 if (status != NV_OK) 253 { 254 NV_ASSERT(status == NV_OK); 255 SLI_LOOP_BREAK; 256 } 257 } 258 259 memdescTagAllocList(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_116, 260 pInstanceBlock->pInstanceBlockDesc, pInstAllocList); 261 if (status == NV_OK) 262 { 263 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 264 265 // Initialize the instance block of the channel with zeros 266 status = memmgrMemDescMemSet(pMemoryManager, 267 pInstanceBlock->pInstanceBlockDesc, 268 0, 269 TRANSFER_FLAGS_NONE); 270 if (status != NV_OK) 271 { 272 NV_ASSERT(status == NV_OK); 273 SLI_LOOP_BREAK; 274 } 275 276 memdescSetName(pGpu, pInstanceBlock->pInstanceBlockDesc, NV_RM_SURF_NAME_INSTANCE_BLOCK, NULL); 277 } 278 else 279 { 280 NV_PRINTF(LEVEL_ERROR, 281 "Instance block allocation for hClient 0x%x hChannel 0x%x failed\n", 282 RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_HANDLE(pKernelChannel)); 283 SLI_LOOP_BREAK; 284 } 285 286 /// Alloc RAMFC Desc 287 status = memdescCreateSubMem(&pInstanceBlock->pRamfcDesc, 288 pInstanceBlock->pInstanceBlockDesc, 289 pGpu, 0, DRF_SIZE( NV_RAMIN_RAMFC ) / 8); 290 if (status != NV_OK) 291 { 292 NV_PRINTF(LEVEL_ERROR, "Could not allocate memdesc for RAMFC\n"); 293 SLI_LOOP_BREAK; 294 } 295 296 // TODO: Move this elsewhere. 297 if (!pKernelChannel->bClientAllocatedUserD) 298 { 299 NV_ASSERT(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL); 300 301 status = _kchannelCreateRMUserdMemDesc(pGpu, pKernelChannel); 302 if (status != NV_OK) 303 { 304 NV_PRINTF(LEVEL_ERROR, "Could not allocate sub memdesc for USERD\n"); // TODO SLI BREAK 305 SLI_LOOP_BREAK; 306 } 307 } 308 309 status = kchannelCreateUserMemDesc_HAL(pGpu, pKernelChannel); 310 if (status != NV_OK) 311 { 312 NV_PRINTF(LEVEL_ERROR, "kchannelCreateUserMemDesc failed \n"); 313 SLI_LOOP_BREAK; 314 } 315 316 NV_PRINTF(LEVEL_INFO, 317 "hChannel 0x%x hClient 0x%x, Class ID 0x%x " 318 "Instance Block @ 0x%llx (%s %x) " 319 "USERD @ 0x%llx " 320 "for subdevice %d\n", 321 RES_GET_HANDLE(pKernelChannel), RES_GET_CLIENT_HANDLE(pKernelChannel), RES_GET_EXT_CLASS_ID(pKernelChannel), 322 memdescGetPhysAddr(pInstanceBlock->pInstanceBlockDesc, AT_GPU, 0), 323 memdescGetApertureString(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 324 (NvU32)(memdescGetAddressSpace(pInstanceBlock->pInstanceBlockDesc)), 325 (pKernelChannel->pUserdSubDeviceMemDesc[subdevInst] == NULL) ? 0x0LL : 326 memdescGetPhysAddr(pKernelChannel->pUserdSubDeviceMemDesc[subdevInst], AT_GPU, 0LL), 327 subdevInst); 328 329 SLI_LOOP_END 330 331 fail: 332 // Just a note about our failure path, null and unallocated 333 // memdescFrees are allowed so this is not a bug. 334 if (status != NV_OK) 335 { 336 NV_PRINTF(LEVEL_ERROR, "Could not create Channel\n"); 337 DBG_BREAKPOINT(); 338 } 339 340 return status; 341 } 342 343 /*! 344 * @brief Free and destroy channel memdescs 345 * created during channel alloc mem 346 * 347 * @param pGpu[in] OBJGPU pointer 348 * @param pKernelChannel[in] KernelChannel pointer 349 * 350 * @return void 351 */ 352 void 353 kchannelDestroyMem_GM107 354 ( 355 OBJGPU *pGpu, 356 KernelChannel *pKernelChannel 357 ) 358 { 359 FIFO_INSTANCE_BLOCK *pInstanceBlock = NULL; 360 NvU32 subdevInst; 361 362 NV_ASSERT_OR_RETURN_VOID(pKernelChannel != NULL); 363 364 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 365 366 subdevInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu); 367 368 if (!pKernelChannel->bClientAllocatedUserD) 369 { 370 _kchannelDestroyRMUserdMemDesc(pGpu, pKernelChannel); 371 } 372 373 pInstanceBlock = pKernelChannel->pFifoHalData[subdevInst]; 374 if (pInstanceBlock != NULL) 375 { 376 // Release RAMFC sub memdesc 377 if (pInstanceBlock->pRamfcDesc != NULL) 378 { 379 memdescFree(pInstanceBlock->pRamfcDesc); 380 memdescDestroy(pInstanceBlock->pRamfcDesc); 381 pInstanceBlock->pRamfcDesc = NULL; 382 } 383 384 // Release Inst block Desc 385 if (pInstanceBlock->pInstanceBlockDesc != NULL) 386 { 387 memdescFree(pInstanceBlock->pInstanceBlockDesc); 388 memdescDestroy(pInstanceBlock->pInstanceBlockDesc); 389 pInstanceBlock->pInstanceBlockDesc = NULL; 390 } 391 } 392 393 // Remove USERD memDescs 394 memdescDestroy(pKernelChannel->pInstSubDeviceMemDesc[subdevInst]); 395 pKernelChannel->pInstSubDeviceMemDesc[subdevInst] = NULL; 396 397 SLI_LOOP_END 398 399 return; 400 } 401 402 /** 403 * @brief reserves a hardware channel slot 404 * 405 * Only responsible for indicating a hardware channel is in use, does not set 406 * any other software state. 407 * 408 * @param pGpu 409 * @param[in] pKernelChannel the pre-allocated KernelChannel 410 * @param[in] hClient 411 * @param[in] allocMode CHANNEL_HW_ID_ALLC_MODE_* 412 * @param[in] ChID 413 * @param[in] bForceInternalIdx true if requesting specific index within USERD page 414 * @param[in] internalIdx requested index within USERD page when bForceInternalIdx 415 * true 416 */ 417 NV_STATUS 418 kchannelAllocHwID_GM107 419 ( 420 OBJGPU *pGpu, 421 KernelChannel *pKernelChannel, 422 NvHandle hClient, 423 NvU32 Flags, 424 NvU32 verifFlags2, 425 NvU32 ChID 426 ) 427 { 428 NV_STATUS status; 429 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 430 CHID_MGR *pChidMgr = NULL; 431 NvU32 internalIdx = 0; 432 NvU32 userdPageIdx = 0; 433 NvBool bForceInternalIdx = NV_FALSE; 434 NvBool bForceUserdPage = NV_FALSE; 435 CHANNEL_HW_ID_ALLOC_MODE allocMode = CHANNEL_HW_ID_ALLOC_MODE_GROW_UP; 436 437 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 438 { 439 // per runlist channel heap is supported for sriov only 440 NV_CHECK_OR_RETURN(LEVEL_ERROR, 441 !kfifoIsPerRunlistChramEnabled(pKernelFifo), 442 NV_ERR_INVALID_STATE); 443 444 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, 445 CHIDMGR_RUNLIST_ID_LEGACY); 446 447 NV_CHECK_OR_RETURN(LEVEL_INFO, 448 kfifoChidMgrGetKernelChannel(pGpu, pKernelFifo, 449 pChidMgr, ChID) == NULL, 450 NV_OK); 451 allocMode = CHANNEL_HW_ID_ALLOC_MODE_PROVIDED; 452 } 453 else 454 { 455 if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_FIXED, _TRUE, Flags)) 456 { 457 bForceUserdPage = NV_TRUE; 458 userdPageIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_PAGE_VALUE, Flags); 459 internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags); 460 461 NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS04, 462 _FLAGS, 463 _CHANNEL_USERD_INDEX_FIXED, 464 _FALSE, 465 Flags), 466 NV_ERR_INVALID_STATE); 467 } 468 469 if (FLD_TEST_DRF(OS04, _FLAGS, _CHANNEL_USERD_INDEX_FIXED, _TRUE, Flags)) 470 { 471 bForceInternalIdx = NV_TRUE; 472 internalIdx = DRF_VAL(OS04, _FLAGS, _CHANNEL_USERD_INDEX_VALUE, Flags); 473 } 474 } 475 476 pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, kchannelGetRunlistId(pKernelChannel)); 477 478 status = kfifoChidMgrAllocChid(pGpu, 479 pKernelFifo, 480 pChidMgr, 481 hClient, 482 allocMode, 483 bForceInternalIdx, 484 internalIdx, 485 bForceUserdPage, 486 userdPageIdx, 487 ChID, 488 pKernelChannel); 489 490 return status; 491 } 492 493 /** 494 * @brief Releases a hardware channel ID. 495 * 496 * Not responsible for freeing any software state beyond that which indicates a 497 * hardware channel is in use. 498 * 499 * @param pGpu 500 * @param pKernelChannel 501 */ 502 NV_STATUS 503 kchannelFreeHwID_GM107 504 ( 505 OBJGPU *pGpu, 506 KernelChannel *pKernelChannel 507 ) 508 { 509 NV_STATUS status; 510 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 511 CHID_MGR *pChidMgr = kfifoGetChidMgr(pGpu, pKernelFifo, 512 kchannelGetRunlistId(pKernelChannel)); 513 EMEMBLOCK *pFifoDataBlock; 514 515 pFifoDataBlock = pChidMgr->pFifoDataHeap->eheapGetBlock( 516 pChidMgr->pFifoDataHeap, 517 pKernelChannel->ChID, 518 NV_FALSE); 519 NV_ASSERT_OR_RETURN(pFifoDataBlock, NV_ERR_OBJECT_NOT_FOUND); 520 NV_ASSERT(pFifoDataBlock->pData == pKernelChannel); 521 522 status = kfifoChidMgrFreeChid(pGpu, pKernelFifo, pChidMgr, pKernelChannel->ChID); 523 if (status != NV_OK) 524 { 525 NV_PRINTF(LEVEL_ERROR, 526 "Unable to Free Channel From Heap: %d\n", 527 pKernelChannel->ChID); 528 DBG_BREAKPOINT(); 529 } 530 531 return status; 532 } 533 534 NV_STATUS 535 kchannelGetUserdInfo_GM107 536 ( 537 OBJGPU *pGpu, 538 KernelChannel *pKernelChannel, 539 NvU64 *userBase, 540 NvU64 *offset, 541 NvU64 *length 542 ) 543 { 544 NV_STATUS status; 545 NvU64 bar1MapOffset; 546 NvU32 bar1MapSize; 547 CLI_CHANNEL_CLASS_INFO classInfo; 548 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 549 550 NvBool bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING); 551 552 CliGetChannelClassInfo(RES_GET_EXT_CLASS_ID(pKernelChannel), &classInfo); 553 554 switch (classInfo.classType) 555 { 556 case CHANNEL_CLASS_TYPE_GPFIFO: 557 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 558 559 // USERD is not pre-allocated in BAR1 so there is no offset/userBase 560 NV_ASSERT_OR_RETURN(!pKernelChannel->bClientAllocatedUserD, 561 NV_ERR_INVALID_REQUEST); 562 563 status = kchannelGetUserdBar1MapOffset_HAL(pGpu, 564 pKernelChannel, 565 &bar1MapOffset, 566 &bar1MapSize); 567 if (status == NV_OK) 568 { 569 *offset = bar1MapOffset; 570 *length = bar1MapSize; 571 572 if (userBase) 573 { 574 if (bCoherentCpuMapping) 575 { 576 NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED)); 577 *userBase = pKernelMemorySystem->coherentCpuFbBase; 578 } 579 else 580 { 581 *userBase = gpumgrGetGpuPhysFbAddr(pGpu); 582 } 583 } 584 } 585 break; 586 587 default: 588 NV_PRINTF(LEVEL_ERROR, 589 "class = %x not supported for user base mapping\n", 590 RES_GET_EXT_CLASS_ID(pKernelChannel)); 591 status = NV_ERR_GENERIC; 592 break; 593 } 594 return status; 595 } 596 597 // 598 // Takes as input a Channel * and returns the BAR1 offset that this channel's 599 // USERD has been mapped to. Also returns the size of the BAR1 mapping that 600 // pertains to this channel. The BAR1 map of all USERDs should have already 601 // been setup before the first channel was created. 602 // 603 // For example, USERD of 40 channels have been mapped at BAR1 offset 0x100. 604 // USERD of one channel is of size 4k. In which case this function will return 605 // ( 0x100 + ( 0x1000 * 0xa ) ) if the input ChID = 0xa. 606 // 607 NV_STATUS 608 kchannelGetUserdBar1MapOffset_GM107 609 ( 610 OBJGPU *pGpu, 611 KernelChannel *pKernelChannel, 612 NvU64 *bar1MapOffset, 613 NvU32 *bar1MapSize 614 ) 615 { 616 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 617 const PREALLOCATED_USERD_INFO *pUserdInfo = kfifoGetPreallocatedUserdInfo(pKernelFifo); 618 619 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 620 621 // 622 // only supported when bUsePerRunlistChannelRam is disabled. 623 // We don't pre-allocate userd for all channels across all runlists; we expect 624 // clients to have moved to client allocated userd. 625 // 626 NV_ASSERT_OR_RETURN(!kfifoIsPerRunlistChramEnabled(pKernelFifo), 627 NV_ERR_NOT_SUPPORTED); 628 629 if (pUserdInfo->userdBar1MapSize == 0) 630 { 631 NV_PRINTF(LEVEL_ERROR, 632 "fifoGetUserdBar1Offset_GF100: BAR1 map of USERD has not " 633 "been setup yet\n"); 634 NV_ASSERT(0); 635 return NV_ERR_GENERIC; 636 } 637 638 kfifoGetUserdSizeAlign_HAL(pKernelFifo, bar1MapSize, NULL); 639 640 *bar1MapOffset = pKernelChannel->ChID * *bar1MapSize + 641 pUserdInfo->userdBar1MapStartOffset; 642 643 NV_ASSERT((*bar1MapOffset + *bar1MapSize) <= 644 (pUserdInfo->userdBar1MapStartOffset + 645 pUserdInfo->userdBar1MapSize)); 646 647 return NV_OK; 648 } 649 650 /*! 651 * @brief Creates a memory descriptor to be used for creating a GPU mapped MMIO 652 * region for a given channel. 653 */ 654 NV_STATUS 655 kchannelCreateUserMemDesc_GM107 656 ( 657 OBJGPU *pGpu, 658 KernelChannel *pKernelChannel 659 ) 660 { 661 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 662 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 663 NV_STATUS status; 664 MEMORY_DESCRIPTOR *pSubDevInstMemDesc = NULL; 665 MEMORY_DESCRIPTOR **ppMemDesc = 666 &pKernelChannel->pInstSubDeviceMemDesc[gpumgrGetSubDeviceInstanceFromGpu(pGpu)]; 667 668 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); 669 670 status = kfifoChannelGetFifoContextMemDesc_HAL(pGpu, pKernelFifo, pKernelChannel, 671 FIFO_CTX_INST_BLOCK, &pSubDevInstMemDesc); 672 673 if (status != NV_OK) 674 return status; 675 676 NV_ASSERT_OR_RETURN(pSubDevInstMemDesc, NV_ERR_OBJECT_NOT_FOUND); 677 678 status = memdescCreate(ppMemDesc, pGpu, RM_PAGE_SIZE, 0, 679 memdescGetContiguity(pSubDevInstMemDesc, AT_GPU), 680 memdescGetAddressSpace(pSubDevInstMemDesc), 681 memdescGetCpuCacheAttrib(pSubDevInstMemDesc), 682 MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE); 683 684 if (status != NV_OK) 685 return status; 686 687 NV_ASSERT(*ppMemDesc); 688 689 memdescDescribe(*ppMemDesc, memdescGetAddressSpace(pSubDevInstMemDesc), 690 memdescGetPhysAddr(pSubDevInstMemDesc, AT_GPU, 0), RM_PAGE_SIZE); 691 memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, *ppMemDesc, AT_GPU, RM_ATTR_PAGE_SIZE_4KB); 692 693 memdescSetPteKind(*ppMemDesc, NV_MMU_PTE_KIND_SMHOST_MESSAGE); 694 695 return NV_OK; 696 } 697 698 /** 699 * @brief Retrieves the engine ID (NV_PFIFO_ENGINE_*) a given channel is operating on. 700 * 701 * This value will not be valid for a channel that has not been scheduled. 702 * 703 * @param pGpu 704 * @param pKernelChannel 705 * @param[out] pEngDesc 706 */ 707 NV_STATUS 708 kchannelGetEngine_GM107 709 ( 710 OBJGPU *pGpu, 711 KernelChannel *pKernelChannel, 712 NvU32 *pEngDesc 713 ) 714 { 715 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 716 717 if (pEngDesc == NULL) 718 return NV_ERR_INVALID_ARGUMENT; 719 720 NV_PRINTF(LEVEL_INFO, "0x%x\n", kchannelGetDebugTag(pKernelChannel)); 721 722 *pEngDesc = kchannelGetRunlistId(pKernelChannel); 723 724 // This will pick the first engine on this runlist (may not be the only one). 725 NV_ASSERT_OK_OR_RETURN(kfifoEngineInfoXlate_HAL(pGpu, pKernelFifo, 726 ENGINE_INFO_TYPE_RUNLIST, *pEngDesc, 727 ENGINE_INFO_TYPE_ENG_DESC, pEngDesc)); 728 729 return NV_OK; 730 } 731