1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 25 /***************************** HW State Routines ***************************\ 26 * * 27 * Fabric Virtual Address Space Function Definitions. * 28 * * 29 \***************************************************************************/ 30 31 #include "gpu/mmu/kern_gmmu.h" 32 #include "mem_mgr/vaspace.h" 33 #include "mem_mgr/fabric_vaspace.h" 34 #include "gpu/mem_mgr/mem_mgr.h" 35 #include "mem_mgr/gpu_vaspace.h" 36 #include "gpu/mem_mgr/virt_mem_allocator_common.h" 37 #include "os/os.h" 38 #include "gpu/bus/kern_bus.h" 39 #include "kernel/gpu/fifo/kernel_fifo.h" 40 #include "kernel/gpu/nvlink/kernel_nvlink.h" 41 #include "mmu/mmu_walk.h" 42 #include "lib/base_utils.h" 43 #include "class/cl90f1.h" // FERMI_VASPACE_A 44 #include "class/cl00fc.h" // FABRIC_VASPACE_A 45 #include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER 46 #include "class/cl0080.h" // NV01_DEVICE_0 47 #include "gpu/device/device.h" 48 #include "gpu/subdevice/subdevice.h" 49 #include "deprecated/rmapi_deprecated.h" 50 #include "rmapi/rs_utils.h" 51 #include "vgpu/vgpu_events.h" 52 #include "mem_mgr/virt_mem_mgr.h" 53 54 #include "published/ampere/ga100/dev_mmu.h" 55 #include "vgpu/rpc.h" 56 #include "virtualization/hypervisor/hypervisor.h" 57 58 59 60 // 61 // TODO: To be removed when legacy FLA VAS (pKernelBus->flaInfo.pFlaVAS) is removed" 62 // The instance block is setup during kbusAllocateFlaVaspace_HAL(). However, we 63 // lazily bind it to the new fabric VAS when the very first NV_FABRIC_MEMORY 64 // allocations happens. 65 // 66 static NV_STATUS 67 _fabricvaspaceBindInstBlk 68 ( 69 FABRIC_VASPACE *pFabricVAS 70 ) 71 { 72 OBJVASPACE *pVAS = staticCast(pFabricVAS, OBJVASPACE); 73 OBJGPU *pGpu = gpumgrGetGpu(gpumgrGetDefaultPrimaryGpu(pVAS->gpuMask)); 74 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 75 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 76 NV_STATUS status = NV_OK; 77 78 INST_BLK_INIT_PARAMS instblkParams; 79 80 if (!pKernelBus->flaInfo.bToggleBindPoint) 81 { 82 return NV_OK; 83 } 84 85 if (gvaspaceIsInUse(dynamicCast(pKernelBus->flaInfo.pFlaVAS, OBJGVASPACE))) 86 { 87 NV_PRINTF(LEVEL_ERROR, 88 "FabricVAS and FlaVAS cannot be used simultaneously! " 89 "Instance block setup for fabricVAS failed\n"); 90 return NV_ERR_INVALID_OPERATION; 91 } 92 93 // 94 // Check if this is the first fabric vaspace allocation. If this is not the 95 // first allocation, instance block is already setup. Return NV_OK. 96 // 97 if (gvaspaceIsInUse(dynamicCast(pFabricVAS->pGVAS, OBJGVASPACE))) 98 { 99 return NV_OK; 100 } 101 102 // Unbind the instance block for FLA vaspace. 103 status = kbusSetupUnbindFla_HAL(pGpu, pKernelBus); 104 if (status != NV_OK) 105 { 106 NV_PRINTF(LEVEL_ERROR, 107 "Failed to unbind instance block for FlaVAS, status=0x%x\n", 108 status); 109 return status; 110 } 111 112 // Instantiate the instance block for fabric vaspace. 113 portMemSet(&instblkParams, 0, sizeof(instblkParams)); 114 status = kgmmuInstBlkInit(pKernelGmmu, pKernelBus->flaInfo.pInstblkMemDesc, 115 pFabricVAS->pGVAS, FIFO_PDB_IDX_BASE, 116 &instblkParams); 117 if (status != NV_OK) 118 { 119 NV_PRINTF(LEVEL_ERROR, 120 "Failed to setup instance block for fabricVAS, status=0x%x\n", 121 status); 122 goto failed; 123 } 124 125 // Bind the instance block for fabric vaspace. 126 status = kbusSetupBindFla_HAL(pGpu, pKernelBus, pFabricVAS->gfid); 127 if (status != NV_OK) 128 { 129 NV_PRINTF(LEVEL_ERROR, 130 "Failed to bind instance block for fabricVAS, status=0x%x\n", 131 status); 132 goto failed; 133 } 134 135 return NV_OK; 136 137 failed: 138 // Instantiate the instance block for FLA vaspace. 139 portMemSet(&instblkParams, 0, sizeof(instblkParams)); 140 NV_ASSERT(kgmmuInstBlkInit(pKernelGmmu, pKernelBus->flaInfo.pInstblkMemDesc, 141 pKernelBus->flaInfo.pFlaVAS, FIFO_PDB_IDX_BASE, 142 &instblkParams) == NV_OK); 143 144 // Bind the instance block for FLA vaspace. 145 NV_ASSERT(kbusSetupBindFla_HAL(pGpu, pKernelBus, pFabricVAS->gfid) == NV_OK); 146 147 return status; 148 } 149 150 // 151 // TODO: To be removed when legacy FLA VAS (pKernelBus->flaInfo.pFlaVAS)is removed" 152 // The instance block is unbind during kbusDestroyFla_HAL(). However, we unbind 153 // it here and bind back the instance block for the legacy FLA VAS after the 154 // last NV_FABRIC_MEMORY allocation is freed. 155 // 156 static void 157 _fabricvaspaceUnbindInstBlk 158 ( 159 FABRIC_VASPACE *pFabricVAS 160 ) 161 { 162 OBJVASPACE *pVAS = staticCast(pFabricVAS, OBJVASPACE); 163 OBJGPU *pGpu = gpumgrGetGpu(gpumgrGetDefaultPrimaryGpu(pVAS->gpuMask)); 164 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 165 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 166 INST_BLK_INIT_PARAMS instblkParams = {0}; 167 168 if (!pKernelBus->flaInfo.bToggleBindPoint) 169 { 170 return; 171 } 172 173 // 174 // Check if there are any pending allocations for the fabric vaspace. 175 // If there are pending allocations, skip restore and return NV_OK. 176 // 177 if (gvaspaceIsInUse(dynamicCast(pFabricVAS->pGVAS, OBJGVASPACE))) 178 { 179 return; 180 } 181 182 // Unbind the instance block for fabric vaspace. 183 NV_ASSERT(kbusSetupUnbindFla_HAL(pGpu, pKernelBus) == NV_OK); 184 185 if (pKernelBus->flaInfo.pFlaVAS != NULL) 186 { 187 // Instantiate the instance block for FLA vaspace. 188 NV_ASSERT(kgmmuInstBlkInit(pKernelGmmu, 189 pKernelBus->flaInfo.pInstblkMemDesc, 190 pKernelBus->flaInfo.pFlaVAS, 191 FIFO_PDB_IDX_BASE, 192 &instblkParams) == NV_OK); 193 194 // Bind the instance block for FLA vaspace. 195 NV_ASSERT(kbusSetupBindFla_HAL(pGpu, pKernelBus, 196 pFabricVAS->gfid) == NV_OK); 197 } 198 } 199 200 NV_STATUS 201 fabricvaspaceConstruct__IMPL 202 ( 203 FABRIC_VASPACE *pFabricVAS, 204 NvU32 classId, 205 NvU32 vaspaceId, 206 NvU64 vaStart, 207 NvU64 vaLimit, 208 NvU64 vaStartInternal, 209 NvU64 vaLimitInternal, 210 NvU32 flags 211 ) 212 { 213 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 214 OBJSYS *pSys = SYS_GET_INSTANCE(); 215 OBJVMM *pVmm = SYS_GET_VMM(pSys); 216 OBJVASPACE *pVAS = staticCast(pFabricVAS, OBJVASPACE); 217 OBJGPU *pGpu = gpumgrGetGpu(gpumgrGetDefaultPrimaryGpu(pVAS->gpuMask)); 218 NV_STATUS status = NV_OK; 219 NvHandle hClient = 0; 220 NvHandle hDevice = 0; 221 NV0080_ALLOC_PARAMETERS devAllocParams = { 0 }; 222 NvU32 gfid = 0; 223 224 // Sanity check input parameters. 225 NV_ASSERT_OR_RETURN(FABRIC_VASPACE_A == classId, NV_ERR_INVALID_ARGUMENT); 226 NV_ASSERT_OR_RETURN(vaStart <= vaLimit, NV_ERR_INVALID_ARGUMENT); 227 NV_ASSERT_OR_RETURN(ONEBITSET(pVAS->gpuMask), NV_ERR_INVALID_ARGUMENT); 228 NV_ASSERT_OR_RETURN(vaspaceId == pGpu->gpuId, NV_ERR_INVALID_ARGUMENT); 229 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &gfid)); 230 231 status = pRmApi->AllocWithHandle(pRmApi, NV01_NULL_OBJECT, 232 NV01_NULL_OBJECT, NV01_NULL_OBJECT, 233 NV01_ROOT, &hClient, sizeof(hClient)); 234 if (status != NV_OK) 235 { 236 NV_PRINTF(LEVEL_ERROR, "failed creating client, status=0x%x\n", status); 237 return status; 238 } 239 240 status = serverutilGenResourceHandle(hClient, &hDevice); 241 if (status != NV_OK) 242 { 243 NV_PRINTF(LEVEL_ERROR, 244 "failed creating device handle, status=0x%x\n", status); 245 goto cleanup; 246 } 247 248 // Allocate a device handle 249 devAllocParams.deviceId = gpuGetDeviceInstance(pGpu); 250 status = pRmApi->AllocWithHandle(pRmApi, hClient, hClient, hDevice, 251 NV01_DEVICE_0, 252 &devAllocParams, sizeof(devAllocParams)); 253 if (status != NV_OK) 254 { 255 NV_PRINTF(LEVEL_ERROR, "failed creating device, status=0x%x\n", status); 256 goto cleanup; 257 } 258 259 // Save off flags. 260 pFabricVAS->flags = (flags | 261 VASPACE_FLAGS_ALLOW_ZERO_ADDRESS | 262 VASPACE_FLAGS_INVALIDATE_SCOPE_NVLINK_TLB | 263 VASPACE_FLAGS_DISABLE_SPLIT_VAS); 264 265 if (IS_GFID_VF(gfid)) 266 { 267 pFabricVAS->gfid = gfid; 268 pFabricVAS->flags |= VASPACE_FLAGS_ALLOW_PAGES_IN_PHYS_MEM_SUBALLOCATOR; 269 } 270 271 pFabricVAS->bRpcAlloc = IS_VIRTUAL(pGpu) && 272 gpuIsWarBug200577889SriovHeavyEnabled(pGpu); 273 274 // Create the GVASPACE object associated with this fabric vaspace. 275 status = vmmCreateVaspace(pVmm, FERMI_VASPACE_A, 0, pVAS->gpuMask, 276 vaStart, vaLimit, 0, 0, NULL, pFabricVAS->flags, 277 &pFabricVAS->pGVAS); 278 if (status != NV_OK) 279 { 280 NV_PRINTF(LEVEL_ERROR, 281 "Failed allocating gvaspace associated with the fabric vaspace, " 282 "status=0x%x\n", status); 283 goto cleanup; 284 } 285 286 pFabricVAS->hClient = hClient; 287 pFabricVAS->hDevice = hDevice; 288 289 // Capture the vasStart and vasLimit for the fabric vaspace. 290 pVAS->vasStart = pFabricVAS->pGVAS->vasStart; 291 pVAS->vasLimit = pFabricVAS->pGVAS->vasLimit; 292 293 return NV_OK; 294 295 cleanup: 296 NV_ASSERT(pRmApi->Free(pRmApi, hClient, hClient) == NV_OK); 297 298 return status; 299 } 300 301 void 302 fabricvaspaceDestruct_IMPL 303 ( 304 FABRIC_VASPACE *pFabricVAS 305 ) 306 { 307 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL); 308 OBJSYS *pSys = SYS_GET_INSTANCE(); 309 OBJVMM *pVmm = SYS_GET_VMM(pSys); 310 OBJVASPACE *pVAS = staticCast(pFabricVAS, OBJVASPACE); 311 312 if (pFabricVAS->pGVAS == NULL) 313 return; 314 315 NV_ASSERT(pRmApi->Free(pRmApi, pFabricVAS->hClient, 316 pFabricVAS->hClient) == NV_OK); 317 318 // There should be no vaspace allocations pending at this point. 319 NV_ASSERT(!gvaspaceIsInUse(dynamicCast(pFabricVAS->pGVAS, OBJGVASPACE))); 320 321 // Destroy the GVASPACE object associated with this fabric vaspace. 322 vmmDestroyVaspace(pVmm, pFabricVAS->pGVAS); 323 324 pFabricVAS->pGVAS = NULL; 325 pVAS->vasStart = 0; 326 pVAS->vasLimit = 0; 327 } 328 329 NV_STATUS 330 fabricvaspaceAlloc_IMPL 331 ( 332 FABRIC_VASPACE *pFabricVAS, 333 NvU64 size, 334 NvU64 align, 335 NvU64 rangeLo, 336 NvU64 rangeHi, 337 NvU64 pageSize, 338 VAS_ALLOC_FLAGS flags, 339 NvU64 *pAddr 340 ) 341 { 342 // 343 // TODO: If needed, can call into fabricvaspaceAllocNonContiguous_IMPL() 344 // by forcing contig flag. 345 // 346 return NV_ERR_NOT_SUPPORTED; 347 } 348 349 NV_STATUS 350 fabricvaspaceAllocNonContiguous_IMPL 351 ( 352 FABRIC_VASPACE *pFabricVAS, 353 NvU64 size, 354 NvU64 align, 355 NvU64 rangeLo, 356 NvU64 rangeHi, 357 NvU64 pageSize, 358 VAS_ALLOC_FLAGS flags, 359 NvU64 **ppAddr, 360 NvU32 *pNumAddr 361 ) 362 { 363 NV_STATUS status = NV_OK; 364 NvU64 freeSize = 0; 365 NvU32 pageCount = (size / pageSize); 366 NvU64 addr; 367 NvU32 idx; 368 NvBool bDefaultAllocMode; 369 370 // Sanity check the input parameters. 371 NV_ASSERT_OR_RETURN(pFabricVAS->pGVAS != NULL, NV_ERR_OBJECT_NOT_FOUND); 372 NV_ASSERT_OR_RETURN(ppAddr != NULL, NV_ERR_INVALID_ARGUMENT); 373 NV_ASSERT_OR_RETURN(pNumAddr != NULL, NV_ERR_INVALID_ARGUMENT); 374 NV_ASSERT_OR_RETURN(pageSize >= RM_PAGE_SIZE_HUGE, NV_ERR_INVALID_ARGUMENT); 375 NV_ASSERT_OR_RETURN(align != 0, NV_ERR_INVALID_ARGUMENT); 376 NV_ASSERT_OR_RETURN(size != 0, NV_ERR_INVALID_ARGUMENT); 377 378 // Check the alignment and size are pageSize aligned. 379 NV_ASSERT_OR_RETURN(NV_IS_ALIGNED64(align, pageSize), NV_ERR_INVALID_ARGUMENT); 380 NV_ASSERT_OR_RETURN(NV_IS_ALIGNED64(size, pageSize), NV_ERR_INVALID_ARGUMENT); 381 382 // Check if heap can satisfy the request. 383 NV_ASSERT_OK_OR_RETURN(fabricvaspaceGetFreeHeap(pFabricVAS, &freeSize)); 384 if (freeSize < size) 385 { 386 NV_PRINTF(LEVEL_ERROR, 387 "Not enough memory in eheap, size requested = 0x%llx, " 388 "free memory = 0x%llx\n", 389 size, freeSize); 390 return NV_ERR_NO_MEMORY; 391 } 392 393 if (flags.bForceNonContig && flags.bForceContig) 394 { 395 NV_PRINTF(LEVEL_ERROR, 396 "Forcing both contiguous and noncontiguous is not allowed\n"); 397 return NV_ERR_INVALID_ARGUMENT; 398 } 399 400 bDefaultAllocMode = (!flags.bForceNonContig && !flags.bForceContig); 401 402 // Adjust rangeLo and rangeHi. 403 rangeLo = NV_ALIGN_DOWN(rangeLo, pageSize); 404 rangeHi = NV_ALIGN_UP(rangeHi, pageSize); 405 406 *ppAddr = portMemAllocNonPaged(sizeof(NvU64) * pageCount); 407 if (*ppAddr == NULL) 408 { 409 return NV_ERR_NO_MEMORY; 410 } 411 portMemSet(*ppAddr, 0, sizeof(NvU64) * pageCount); 412 413 status = _fabricvaspaceBindInstBlk(pFabricVAS); 414 if (status != NV_OK) 415 { 416 NV_PRINTF(LEVEL_ERROR, "Failed to bind instance block for fabric vaspace." 417 " Alloc failed\n"); 418 goto failed; 419 } 420 421 // Initialize number of addresses to 0 422 *pNumAddr = 0; 423 424 // 425 // Attempt to allocate VA space of the size and alignment requested. 426 // 427 // RM_PAGE_SIZE_HUGE is passed since FLA->PA mappings support minimum 428 // 2MB pagesize. 429 // 430 if (flags.bForceContig || bDefaultAllocMode) 431 { 432 status = vaspaceAlloc(pFabricVAS->pGVAS, size, align, rangeLo, rangeHi, 433 RM_PAGE_SIZE_HUGE, flags, &addr); 434 if (status == NV_OK) 435 { 436 (*ppAddr)[0] = addr; 437 *pNumAddr = 1; 438 } 439 else if (flags.bForceContig) 440 { 441 NV_PRINTF(LEVEL_ERROR, "Failed to allocate contig vaspace\n"); 442 goto failed; 443 } 444 } 445 446 // 447 // If size could not be allocated in one memblock, break size into 448 // multiple pageSize chunks. 449 // 450 // RM_PAGE_SIZE_HUGE is passed since FLA->PA mappings support minimum 451 // 2MB pagesize. 452 // 453 if (flags.bForceNonContig || (bDefaultAllocMode && (status != NV_OK))) 454 { 455 for (idx = 0; idx < pageCount; idx++) 456 { 457 status = vaspaceAlloc(pFabricVAS->pGVAS, pageSize, align, rangeLo, 458 rangeHi, RM_PAGE_SIZE_HUGE, flags, &addr); 459 if (status == NV_OK) 460 { 461 // Assert that the address returned is pageSize aligned 462 NV_ASSERT(NV_IS_ALIGNED64(addr, pageSize)); 463 464 (*ppAddr)[idx] = addr; 465 *pNumAddr = *pNumAddr + 1; 466 } 467 else 468 { 469 NV_PRINTF(LEVEL_ERROR, "Failed to allocate vaspace\n"); 470 goto failed; 471 } 472 } 473 } 474 475 pFabricVAS->ucFabricFreeSize -= size; 476 pFabricVAS->ucFabricInUseSize += size; 477 478 return NV_OK; 479 480 failed: 481 482 fabricvaspaceBatchFree(pFabricVAS, *ppAddr, *pNumAddr, 1); 483 portMemFree(*ppAddr); 484 *ppAddr = NULL; 485 *pNumAddr = 0; 486 487 return status; 488 } 489 490 NV_STATUS 491 fabricvaspaceFree_IMPL 492 ( 493 FABRIC_VASPACE *pFabricVAS, 494 NvU64 vAddr 495 ) 496 { 497 OBJVASPACE *pVAS = staticCast(pFabricVAS, OBJVASPACE); 498 OBJGPU *pGpu = gpumgrGetGpu(gpumgrGetDefaultPrimaryGpu(pVAS->gpuMask)); 499 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 500 NvU64 blockSize; 501 NvBool bUcFla; 502 503 NV_ASSERT_OR_RETURN(pFabricVAS->pGVAS != NULL, NV_ERR_OBJECT_NOT_FOUND); 504 505 bUcFla = (vAddr >= fabricvaspaceGetUCFlaStart(pFabricVAS) && 506 vAddr < fabricvaspaceGetUCFlaLimit(pFabricVAS)); 507 508 NV_ASSERT(vaspaceFreeV2(pFabricVAS->pGVAS, vAddr, &blockSize) == NV_OK); 509 510 kbusFlush_HAL(pGpu, pKernelBus, (BUS_FLUSH_VIDEO_MEMORY | 511 BUS_FLUSH_SYSTEM_MEMORY)); 512 513 fabricvaspaceInvalidateTlb(pFabricVAS, pGpu, PTE_DOWNGRADE); 514 515 _fabricvaspaceUnbindInstBlk(pFabricVAS); 516 517 if (bUcFla) 518 { 519 pFabricVAS->ucFabricFreeSize += blockSize; 520 pFabricVAS->ucFabricInUseSize -= blockSize; 521 } 522 523 return NV_OK; 524 } 525 526 NV_STATUS 527 fabricvaspaceMap_IMPL 528 ( 529 FABRIC_VASPACE *pFabricVAS, 530 OBJGPU *pGpu, 531 const NvU64 vaLo, 532 const NvU64 vaHi, 533 const MMU_MAP_TARGET *pTarget, 534 const VAS_MAP_FLAGS flags 535 ) 536 { 537 return NV_ERR_NOT_SUPPORTED; 538 } 539 540 void 541 fabricvaspaceUnmap_IMPL 542 ( 543 FABRIC_VASPACE *pFabricVAS, 544 OBJGPU *pGpu, 545 const NvU64 vaLo, 546 const NvU64 vaHi 547 ) 548 { 549 return; 550 } 551 552 NV_STATUS 553 fabricvaspaceApplyDefaultAlignment_IMPL 554 ( 555 FABRIC_VASPACE *pFabricVAS, 556 const FB_ALLOC_INFO *pAllocInfo, 557 NvU64 *pAlign, 558 NvU64 *pSize, 559 NvU64 *pPageSizeLockMask 560 ) 561 { 562 return NV_ERR_NOT_SUPPORTED; 563 } 564 565 NV_STATUS 566 fabricvaspaceGetVasInfo_IMPL 567 ( 568 FABRIC_VASPACE *pFabricVAS, 569 NV0080_CTRL_DMA_ADV_SCHED_GET_VA_CAPS_PARAMS *pParams 570 ) 571 { 572 return NV_ERR_NOT_SUPPORTED; 573 } 574 575 NV_STATUS 576 fabricvaspacePinRootPageDir_IMPL 577 ( 578 FABRIC_VASPACE *pFabricVAS, 579 OBJGPU *pGpu 580 ) 581 { 582 NV_ASSERT_OR_RETURN(pFabricVAS->pGVAS != NULL, NV_ERR_OBJECT_NOT_FOUND); 583 584 return vaspacePinRootPageDir(pFabricVAS->pGVAS, pGpu); 585 } 586 587 void 588 fabricvaspaceUnpinRootPageDir_IMPL 589 ( 590 FABRIC_VASPACE *pFabricVAS, 591 OBJGPU *pGpu 592 ) 593 { 594 NV_ASSERT(pFabricVAS->pGVAS != NULL); 595 596 vaspaceUnpinRootPageDir(pFabricVAS->pGVAS, pGpu); 597 } 598 599 NV_STATUS 600 fabricvaspaceGetFreeHeap_IMPL 601 ( 602 FABRIC_VASPACE *pFabricVAS, 603 NvU64 *freeSize 604 ) 605 { 606 NV_ASSERT_OR_RETURN(pFabricVAS->pGVAS != NULL, NV_ERR_OBJECT_NOT_FOUND); 607 NV_ASSERT_OR_RETURN(freeSize != NULL, NV_ERR_INVALID_ARGUMENT); 608 609 *freeSize = pFabricVAS->ucFabricFreeSize; 610 return NV_OK; 611 } 612 613 void 614 fabricvaspaceBatchFree_IMPL 615 ( 616 FABRIC_VASPACE *pFabricVAS, 617 NvU64 *pAddr, 618 NvU32 numAddr, 619 NvU32 stride 620 ) 621 { 622 OBJVASPACE *pVAS = staticCast(pFabricVAS, OBJVASPACE); 623 OBJGPU *pGpu = gpumgrGetGpu(gpumgrGetDefaultPrimaryGpu(pVAS->gpuMask)); 624 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 625 NvU64 totalFreeSize = 0; 626 NvU64 freeSize; 627 NvU32 count = 0; 628 NvU32 idx = 0; 629 NvBool bUcFla; 630 631 632 for (count = 0; count < numAddr; count++) 633 { 634 bUcFla = (pAddr[idx] >= fabricvaspaceGetUCFlaStart(pFabricVAS) && 635 pAddr[idx] < fabricvaspaceGetUCFlaLimit(pFabricVAS)); 636 637 NV_ASSERT(vaspaceFreeV2(pFabricVAS->pGVAS, 638 pAddr[idx], &freeSize) == NV_OK); 639 640 idx += stride; 641 642 if (bUcFla) 643 totalFreeSize += freeSize; 644 } 645 646 kbusFlush_HAL(pGpu, pKernelBus, (BUS_FLUSH_VIDEO_MEMORY | 647 BUS_FLUSH_SYSTEM_MEMORY)); 648 649 fabricvaspaceInvalidateTlb(pFabricVAS, pGpu, PTE_DOWNGRADE); 650 651 _fabricvaspaceUnbindInstBlk(pFabricVAS); 652 653 pFabricVAS->ucFabricFreeSize += totalFreeSize; 654 pFabricVAS->ucFabricInUseSize -= totalFreeSize; 655 } 656 657 void 658 fabricvaspaceInvalidateTlb_IMPL 659 ( 660 FABRIC_VASPACE *pFabricVAS, 661 OBJGPU *pGpu, 662 VAS_PTE_UPDATE_TYPE type 663 ) 664 { 665 vaspaceInvalidateTlb(pFabricVAS->pGVAS, pGpu, type); 666 } 667 668 NV_STATUS 669 fabricvaspaceGetGpaMemdesc_IMPL 670 ( 671 FABRIC_VASPACE *pFabricVAS, 672 MEMORY_DESCRIPTOR *pFabricMemdesc, 673 OBJGPU *pMappingGpu, 674 MEMORY_DESCRIPTOR **ppAdjustedMemdesc 675 ) 676 { 677 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pMappingGpu); 678 MEMORY_DESCRIPTOR *pRootMemDesc = NULL; 679 NODE *pNode = NULL; 680 NV_STATUS status = NV_OK; 681 NvU64 rootOffset = 0; 682 NvBool bLoopbackSupported = NV_FALSE; 683 684 NV_ASSERT_OR_RETURN(ppAdjustedMemdesc != NULL, NV_ERR_INVALID_ARGUMENT); 685 686 { 687 bLoopbackSupported = pKernelNvlink != NULL && 688 (knvlinkIsP2pLoopbackSupported(pMappingGpu, pKernelNvlink) || 689 knvlinkIsForcedConfig(pMappingGpu, pKernelNvlink)); 690 } 691 692 if (memdescGetAddressSpace(pFabricMemdesc) != ADDR_FABRIC_V2 || 693 bLoopbackSupported) 694 { 695 *ppAdjustedMemdesc = pFabricMemdesc; 696 return NV_OK; 697 } 698 699 pRootMemDesc = memdescGetRootMemDesc(pFabricMemdesc, &rootOffset); 700 701 RmPhysAddr *pteArray = memdescGetPteArray(pRootMemDesc, AT_GPU); 702 703 // Check if pteArray[0] is within the VAS range for the mapping GPU. 704 if ((pteArray[0] < fabricvaspaceGetUCFlaStart(pFabricVAS)) || 705 (pteArray[0] > fabricvaspaceGetUCFlaLimit(pFabricVAS))) 706 { 707 *ppAdjustedMemdesc = pFabricMemdesc; 708 return NV_OK; 709 } 710 711 // 712 // If the address space is of type ADDR_FABRIC_V2 then determine if the 713 // FLA import is on the mapping GPU. If FLA import is on the mapping GPU 714 // and NVLink P2P over loopback is not supported, then map GVA->PA directly. 715 // For discontiguous fabric memory allocation, searching for the first entry 716 // in the pteArray should be fine to determine if FLA import is on the 717 // mapping GPU. 718 // 719 NV_ASSERT_OK_OR_RETURN(btreeSearch(pteArray[0], &pNode, 720 pFabricVAS->pFabricVaToGpaMap)); 721 722 FABRIC_VA_TO_GPA_MAP_NODE *pFabricNode = 723 (FABRIC_VA_TO_GPA_MAP_NODE *)pNode->Data; 724 725 // 726 // Create a sub-memdesc for the offset into the vidMemDesc where the GVA 727 // would be mapped. Note this includes two offsets: 728 // 1. Offset into the fabric memdesc where the GVA is mapped. 729 // 2. Offset into the physical vidmem memdesc where the fabric memory is 730 // mapped. 731 // 732 status = memdescCreateSubMem(ppAdjustedMemdesc, pFabricNode->pVidMemDesc, 733 pMappingGpu, 734 rootOffset + pFabricNode->offset, 735 memdescGetSize(pFabricMemdesc)); 736 if (status != NV_OK) 737 { 738 NV_PRINTF(LEVEL_ERROR, 739 "Failed to create submMemdesc for the GVA->PA mapping\n"); 740 return status; 741 } 742 743 return NV_OK; 744 } 745 746 void 747 fabricvaspacePutGpaMemdesc_IMPL 748 ( 749 FABRIC_VASPACE *pFabricVAS, 750 MEMORY_DESCRIPTOR *pMemDesc 751 ) 752 { 753 memdescDestroy(pMemDesc); 754 } 755 756 void 757 fabricvaspaceVaToGpaMapRemove_IMPL 758 ( 759 FABRIC_VASPACE *pFabricVAS, 760 NvU64 vAddr 761 ) 762 { 763 FABRIC_VA_TO_GPA_MAP_NODE *pFabricNode = NULL; 764 NODE *pNode = NULL; 765 766 if (btreeSearch(vAddr, &pNode, pFabricVAS->pFabricVaToGpaMap) == NV_OK) 767 { 768 pFabricNode = (FABRIC_VA_TO_GPA_MAP_NODE *)pNode->Data; 769 770 btreeUnlink(&pFabricNode->Node, &pFabricVAS->pFabricVaToGpaMap); 771 772 portMemFree(pFabricNode); 773 } 774 } 775 776 NV_STATUS 777 fabricvaspaceVaToGpaMapInsert_IMPL 778 ( 779 FABRIC_VASPACE *pFabricVAS, 780 NvU64 vAddr, 781 MEMORY_DESCRIPTOR *pVidMemDesc, 782 NvU64 offset 783 ) 784 { 785 FABRIC_VA_TO_GPA_MAP_NODE *pFabricNode = NULL; 786 NV_STATUS status = NV_OK; 787 788 pFabricNode = portMemAllocNonPaged(sizeof(FABRIC_VA_TO_GPA_MAP_NODE)); 789 if (pFabricNode == NULL) 790 return NV_ERR_NO_MEMORY; 791 792 portMemSet(pFabricNode, 0, sizeof(FABRIC_VA_TO_GPA_MAP_NODE)); 793 794 pFabricNode->pVidMemDesc = pVidMemDesc; 795 pFabricNode->offset = offset; 796 pFabricNode->Node.keyStart = vAddr; 797 pFabricNode->Node.keyEnd = vAddr; 798 pFabricNode->Node.Data = pFabricNode; 799 800 // Insert into the btree tracking memory fabric allocations for this GPU. 801 status = btreeInsert(&pFabricNode->Node, &pFabricVAS->pFabricVaToGpaMap); 802 if (status != NV_OK) 803 { 804 NV_PRINTF(LEVEL_ERROR, 805 "Failed to insert addr 0x%llx into the memory fabric tree\n", 806 pFabricNode->Node.keyStart); 807 808 portMemFree(pFabricNode); 809 return status; 810 } 811 812 return NV_OK; 813 } 814 815 NV_STATUS 816 fabricvaspaceAllocMulticast_IMPL 817 ( 818 FABRIC_VASPACE *pFabricVAS, 819 NvU64 pageSize, 820 NvU64 alignment, 821 VAS_ALLOC_FLAGS flags, 822 NvU64 base, 823 NvU64 size 824 ) 825 { 826 NvU64 rangeLo; 827 NvU64 rangeHi; 828 NvU64 addr = 0; 829 NV_STATUS status; 830 831 NV_ASSERT_OR_RETURN(pFabricVAS->pGVAS != NULL, NV_ERR_OBJECT_NOT_FOUND); 832 NV_ASSERT_OR_RETURN(pageSize >= RM_PAGE_SIZE_HUGE, NV_ERR_INVALID_ARGUMENT); 833 NV_ASSERT_OR_RETURN(alignment != 0, NV_ERR_INVALID_ARGUMENT); 834 NV_ASSERT_OR_RETURN(size != 0, NV_ERR_INVALID_ARGUMENT); 835 NV_ASSERT_OR_RETURN(NV_IS_ALIGNED64(alignment, pageSize), NV_ERR_INVALID_ARGUMENT); 836 NV_ASSERT_OR_RETURN(NV_IS_ALIGNED64(base, pageSize), NV_ERR_INVALID_ARGUMENT); 837 NV_ASSERT_OR_RETURN(NV_IS_ALIGNED64(size, pageSize), NV_ERR_INVALID_ARGUMENT); 838 839 rangeLo = base; 840 rangeHi = base + size - 1; 841 842 // 843 // RM_PAGE_SIZE_HUGE is passed since MCFLA->PA mappings support minimum 844 // 2MB pagesize. 845 // 846 status = vaspaceAlloc(pFabricVAS->pGVAS, size, alignment, rangeLo, 847 rangeHi, RM_PAGE_SIZE_HUGE, flags, &addr); 848 849 NV_ASSERT(addr == base); 850 851 return status; 852 } 853 854 static NV_STATUS 855 _fabricVaspaceValidateMapAttrs 856 ( 857 NvU64 fabricOffset, 858 NvU64 fabricAllocSize, 859 NvU64 fabricPageSize, 860 NvU64 physMapOffset, 861 NvU64 physMapLength, 862 NvU64 physAllocSize, 863 NvU64 physPageSize 864 ) 865 { 866 // Fabric mem offset should be at least phys page size aligned. 867 if (!NV_IS_ALIGNED64(fabricOffset, physPageSize) || 868 (fabricOffset >= fabricAllocSize)) 869 { 870 NV_PRINTF(LEVEL_ERROR, 871 "Invalid offset passed for the fabric handle\n"); 872 873 return NV_ERR_INVALID_OFFSET; 874 } 875 876 if (!NV_IS_ALIGNED64(physMapOffset, physPageSize) || 877 (physMapOffset >= physAllocSize)) 878 { 879 NV_PRINTF(LEVEL_ERROR, 880 "Invalid offset passed for the physmem handle\n"); 881 882 return NV_ERR_INVALID_OFFSET; 883 } 884 885 if ((physMapLength == 0) || 886 (!NV_IS_ALIGNED64(physMapLength, physPageSize)) || 887 (physMapLength > (physAllocSize - physMapOffset)) || 888 (physMapLength > (fabricAllocSize - fabricOffset))) 889 { 890 NV_PRINTF(LEVEL_ERROR, 891 "Invalid map length passed for the physmem handle\n"); 892 893 return NV_ERR_INVALID_ARGUMENT; 894 } 895 896 return NV_OK; 897 } 898 899 typedef struct FABRIC_VASPACE_MAPPING_REGION 900 { 901 NvU64 offset; 902 NvU64 length; 903 } FABRIC_VASPACE_MAPPING_REGION; 904 905 // 906 // In worst case, we can have three regions to map. Two partially filled fabric 907 // pages and one (or more) fully filled fabric page(s). 908 // 909 #define FABRIC_VASPACE_MAPPING_REGIONS_MAX 3 910 911 typedef struct FABRIC_VASPACE_MAPPING_REGIONS 912 { 913 FABRIC_VASPACE_MAPPING_REGION r[FABRIC_VASPACE_MAPPING_REGIONS_MAX]; 914 } FABRIC_VASPACE_MAPPING_REGIONS; 915 916 static void 917 _fabricvaspaceGetMappingRegions 918 ( 919 NvU64 fabricOffset, 920 NvU64 fabricPageSize, 921 NvU64 physMapLength, 922 FABRIC_VASPACE_MAPPING_REGIONS *pRegions, 923 NvU32 *pNumRegions 924 ) 925 { 926 NvU64 fabricOffsetAligned = NV_ALIGN_UP64(fabricOffset, fabricPageSize); 927 NvU64 mapLengthAligned = NV_ALIGN_DOWN64(physMapLength, fabricPageSize); 928 929 *pNumRegions = 0; 930 931 if ((fabricOffset < fabricOffsetAligned) && 932 (physMapLength >= (fabricOffsetAligned - fabricOffset))) 933 { 934 pRegions->r[*pNumRegions].offset = fabricOffset; 935 pRegions->r[*pNumRegions].length = fabricOffsetAligned - fabricOffset; 936 937 fabricOffset += pRegions->r[*pNumRegions].length; 938 physMapLength -= pRegions->r[*pNumRegions].length; 939 mapLengthAligned = NV_ALIGN_DOWN64(physMapLength, fabricPageSize); 940 941 (*pNumRegions)++; 942 } 943 944 if (physMapLength == 0) 945 return; 946 947 if ((fabricOffset == fabricOffsetAligned) && 948 (mapLengthAligned >= fabricPageSize)) 949 { 950 pRegions->r[*pNumRegions].offset = fabricOffset; 951 pRegions->r[*pNumRegions].length = mapLengthAligned; 952 953 fabricOffset += pRegions->r[*pNumRegions].length; 954 physMapLength -= pRegions->r[*pNumRegions].length; 955 956 (*pNumRegions)++; 957 } 958 959 if (physMapLength == 0) 960 return; 961 962 pRegions->r[*pNumRegions].offset = fabricOffset; 963 pRegions->r[*pNumRegions].length = physMapLength; 964 965 (*pNumRegions)++; 966 } 967 968 void 969 fabricvaspaceUnmapPhysMemdesc_IMPL 970 ( 971 FABRIC_VASPACE *pFabricVAS, 972 MEMORY_DESCRIPTOR *pFabricMemDesc, 973 NvU64 fabricOffset, 974 MEMORY_DESCRIPTOR *pPhysMemDesc, 975 NvU64 physMapLength 976 ) 977 { 978 OBJGPU *pGpu = pPhysMemDesc->pGpu; 979 NvU32 fabricPageCount; 980 NvU64 fabricAddr; 981 NvU64 fabricPageSize; 982 NvU32 i, j; 983 NvU64 mapLength; 984 FABRIC_VASPACE_MAPPING_REGIONS regions; 985 NvU32 numRegions; 986 987 fabricPageSize = memdescGetPageSize(pFabricMemDesc, AT_GPU); 988 989 NV_ASSERT_OR_RETURN_VOID(dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE) == 990 pFabricVAS); 991 992 _fabricvaspaceGetMappingRegions(fabricOffset, fabricPageSize, physMapLength, 993 ®ions, &numRegions); 994 NV_ASSERT_OR_RETURN_VOID(numRegions != 0); 995 996 for (i = 0; i < numRegions; i++) 997 { 998 fabricPageCount = 999 ((memdescGetPteArraySize(pFabricMemDesc, AT_GPU) == 1) || 1000 (regions.r[i].length < fabricPageSize)) ? 1001 1 : (regions.r[i].length / fabricPageSize); 1002 1003 mapLength = (fabricPageCount == 1) ? regions.r[i].length : fabricPageSize; 1004 1005 fabricOffset = regions.r[i].offset; 1006 1007 for (j = 0; j < fabricPageCount; j++) 1008 { 1009 if (fabricPageCount == 1) 1010 { 1011 fabricAddr = pFabricMemDesc->_pteArray[0] + fabricOffset; 1012 } 1013 else 1014 { 1015 fabricAddr = pFabricMemDesc->_pteArray[fabricOffset / 1016 pFabricMemDesc->pageArrayGranularity]; 1017 } 1018 1019 vaspaceUnmap(pFabricVAS->pGVAS, pPhysMemDesc->pGpu, fabricAddr, 1020 fabricAddr + mapLength - 1); 1021 1022 fabricOffset = fabricOffset + mapLength; 1023 } 1024 } 1025 1026 fabricvaspaceInvalidateTlb(pFabricVAS, pPhysMemDesc->pGpu, PTE_DOWNGRADE); 1027 } 1028 1029 NV_STATUS 1030 fabricvaspaceMapPhysMemdesc_IMPL 1031 ( 1032 FABRIC_VASPACE *pFabricVAS, 1033 MEMORY_DESCRIPTOR *pFabricMemDesc, 1034 NvU64 fabricOffset, 1035 MEMORY_DESCRIPTOR *pPhysMemDesc, 1036 NvU64 physOffset, 1037 NvU64 physMapLength, 1038 NvU32 flags 1039 ) 1040 { 1041 OBJGPU *pGpu = pPhysMemDesc->pGpu; 1042 VirtMemAllocator *pDma = GPU_GET_DMA(pGpu); 1043 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1044 NV_STATUS status; 1045 DMA_PAGE_ARRAY pageArray; 1046 NvU32 kind; 1047 COMPR_INFO comprInfo; 1048 NvU32 mapFlags = DMA_UPDATE_VASPACE_FLAGS_UPDATE_ALL | 1049 DMA_UPDATE_VASPACE_FLAGS_SKIP_4K_PTE_CHECK; 1050 NvU32 fabricPageCount; 1051 NvU64 fabricAddr; 1052 NvU64 physPageSize; 1053 NvU64 fabricPageSize; 1054 NvU64 physAddr; 1055 NvU32 i, j; 1056 NvU64 mapLength; 1057 NvBool bReadOnly = !!(flags & FABRIC_VASPACE_MAP_FLAGS_READ_ONLY); 1058 FABRIC_VASPACE_MAPPING_REGIONS regions; 1059 NvU32 numRegions; 1060 MEMORY_DESCRIPTOR *pTempMemdesc; 1061 NvU32 aperture; 1062 NvU32 peerNumber = BUS_INVALID_PEER; 1063 1064 NV_ASSERT_OR_RETURN(pFabricMemDesc != NULL, NV_ERR_INVALID_ARGUMENT); 1065 NV_ASSERT_OR_RETURN(pPhysMemDesc != NULL, NV_ERR_INVALID_ARGUMENT); 1066 1067 mapFlags |= bReadOnly ? DMA_UPDATE_VASPACE_FLAGS_READ_ONLY : 0; 1068 1069 NV_ASSERT_OR_RETURN(dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE) == pFabricVAS, 1070 NV_ERR_INVALID_ARGUMENT); 1071 1072 physPageSize = memdescGetPageSize(pPhysMemDesc, AT_GPU); 1073 fabricPageSize = memdescGetPageSize(pFabricMemDesc, AT_GPU); 1074 1075 status = _fabricVaspaceValidateMapAttrs(fabricOffset, 1076 memdescGetSize(pFabricMemDesc), 1077 fabricPageSize, 1078 physOffset, 1079 physMapLength, 1080 memdescGetSize(pPhysMemDesc), 1081 physPageSize); 1082 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, status); 1083 1084 if (pFabricVAS->bRpcAlloc) 1085 return NV_OK; 1086 1087 status = memmgrGetKindComprFromMemDesc(pMemoryManager, pPhysMemDesc, 1088 physOffset, &kind, &comprInfo); 1089 NV_ASSERT_OK_OR_RETURN(status); 1090 1091 if (memdescGetAddressSpace(pPhysMemDesc) == ADDR_FBMEM) 1092 { 1093 aperture = NV_MMU_PTE_APERTURE_VIDEO_MEMORY; 1094 } 1095 else if (memdescIsEgm(pPhysMemDesc)) 1096 { 1097 aperture = NV_MMU_PTE_APERTURE_PEER_MEMORY; 1098 // 1099 // Make sure that we receive a mapping request for EGM memory 1100 // only if local EGM is enabled. 1101 // 1102 NV_ASSERT_OR_RETURN(pMemoryManager->bLocalEgmEnabled, NV_ERR_INVALID_STATE); 1103 peerNumber = pMemoryManager->localEgmPeerId; 1104 } 1105 else if (memdescGetAddressSpace(pPhysMemDesc) == ADDR_SYSMEM) 1106 { 1107 if (memdescGetCpuCacheAttrib(pPhysMemDesc) == NV_MEMORY_CACHED) 1108 { 1109 aperture = NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY; 1110 } 1111 else 1112 { 1113 aperture = NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY; 1114 } 1115 } 1116 else 1117 { 1118 NV_PRINTF(LEVEL_ERROR, "Unsupported aperture\n"); 1119 NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_ARGUMENT); 1120 } 1121 1122 _fabricvaspaceGetMappingRegions(fabricOffset, fabricPageSize, physMapLength, 1123 ®ions, &numRegions); 1124 NV_ASSERT_OR_RETURN(numRegions != 0, NV_ERR_INVALID_ARGUMENT); 1125 1126 for (i = 0; i < numRegions; i++) 1127 { 1128 fabricPageCount = ((memdescGetPteArraySize(pFabricMemDesc, AT_GPU) == 1) || 1129 (regions.r[i].length < fabricPageSize)) ? \ 1130 1 : (regions.r[i].length / fabricPageSize); 1131 mapLength = (fabricPageCount == 1) ? regions.r[i].length : fabricPageSize; 1132 fabricOffset = regions.r[i].offset; 1133 1134 portMemSet(&pageArray, 0, sizeof(DMA_PAGE_ARRAY)); 1135 pageArray.count = (memdescGetPteArraySize(pPhysMemDesc, AT_GPU) == 1) ? \ 1136 1 : (mapLength / pPhysMemDesc->pageArrayGranularity); 1137 1138 for (j = 0; j < fabricPageCount; j++) 1139 { 1140 if (fabricPageCount == 1) 1141 { 1142 fabricAddr = pFabricMemDesc->_pteArray[0] + fabricOffset; 1143 } 1144 else 1145 { 1146 fabricAddr = pFabricMemDesc->_pteArray[fabricOffset / 1147 pFabricMemDesc->pageArrayGranularity]; 1148 } 1149 1150 if (pageArray.count == 1) 1151 { 1152 physAddr = pPhysMemDesc->_pteArray[0] + physOffset; 1153 pageArray.pData = &physAddr; 1154 } 1155 else 1156 { 1157 pageArray.pData = &pPhysMemDesc->_pteArray[physOffset / 1158 pPhysMemDesc->pageArrayGranularity]; 1159 } 1160 1161 // 1162 // When physPageSize is greater than fabricPageSize, to avoid fabric 1163 // VAs getting aligned using physPageSize by dmaUpdateVASpace_HAL, 1164 // create a tempMemdesc and override its pageSize. 1165 // 1166 if (fabricPageSize < physPageSize) 1167 { 1168 status = memdescCreateSubMem(&pTempMemdesc, pPhysMemDesc, 1169 pPhysMemDesc->pGpu, 1170 physOffset, mapLength); 1171 if (status != NV_OK) 1172 goto fail; 1173 1174 memdescSetPageSize(pTempMemdesc, AT_GPU, fabricPageSize); 1175 } 1176 else 1177 { 1178 pTempMemdesc = pPhysMemDesc; 1179 } 1180 1181 // Map the memory fabric object at the given physical memory offset. 1182 status = dmaUpdateVASpace_HAL(pGpu, pDma, pFabricVAS->pGVAS, pTempMemdesc, 1183 NULL, fabricAddr, fabricAddr + mapLength - 1, 1184 mapFlags, &pageArray, 0, &comprInfo, 0, 1185 NV_MMU_PTE_VALID_TRUE, 1186 aperture, 1187 peerNumber, NVLINK_INVALID_FABRIC_ADDR, 1188 DMA_DEFER_TLB_INVALIDATE, NV_FALSE, 1189 memdescGetPageSize(pTempMemdesc, AT_GPU)); 1190 1191 if (pTempMemdesc != pPhysMemDesc) 1192 memdescDestroy(pTempMemdesc); 1193 1194 if (status != NV_OK) 1195 goto fail; 1196 1197 physOffset = physOffset + mapLength; 1198 fabricOffset = fabricOffset + mapLength; 1199 } 1200 } 1201 1202 fabricvaspaceInvalidateTlb(pFabricVAS, pPhysMemDesc->pGpu, PTE_UPGRADE); 1203 1204 return NV_OK; 1205 1206 fail: 1207 for (j = 0; j < i; j++) 1208 fabricvaspaceUnmapPhysMemdesc(pFabricVAS, pFabricMemDesc, 1209 regions.r[j].offset, pPhysMemDesc, 1210 regions.r[j].length); 1211 1212 return status; 1213 } 1214 1215 NV_STATUS 1216 fabricvaspaceInitUCRange_IMPL 1217 ( 1218 FABRIC_VASPACE *pFabricVAS, 1219 OBJGPU *pGpu, 1220 NvU64 fabricBase, 1221 NvU64 fabricSize 1222 ) 1223 { 1224 if (fabricvaspaceGetUCFlaLimit(pFabricVAS) != 0) 1225 return NV_ERR_IN_USE; 1226 1227 if (fabricSize != 0) 1228 { 1229 NV_PRINTF(LEVEL_INFO, "Setting UC Base: %llx, size: %llx \n", 1230 fabricBase, fabricSize); 1231 pFabricVAS->ucFabricBase = fabricBase; 1232 pFabricVAS->ucFabricLimit = fabricBase + fabricSize - 1; 1233 pFabricVAS->ucFabricInUseSize = 0; 1234 pFabricVAS->ucFabricFreeSize = fabricSize; 1235 } 1236 1237 return NV_OK; 1238 } 1239 1240 void 1241 fabricvaspaceClearUCRange_IMPL 1242 ( 1243 FABRIC_VASPACE *pFabricVAS 1244 ) 1245 { 1246 pFabricVAS->ucFabricBase = 0; 1247 pFabricVAS->ucFabricLimit = 0; 1248 pFabricVAS->ucFabricInUseSize = 0; 1249 pFabricVAS->ucFabricFreeSize = 0; 1250 } 1251 1252 NV_STATUS 1253 fabricvaspaceGetPageLevelInfo_IMPL 1254 ( 1255 FABRIC_VASPACE *pFabricVAS, 1256 OBJGPU *pGpu, 1257 NV90F1_CTRL_VASPACE_GET_PAGE_LEVEL_INFO_PARAMS *pParams 1258 ) 1259 { 1260 OBJGVASPACE *pGVAS = dynamicCast(pFabricVAS->pGVAS, OBJGVASPACE); 1261 NV_ASSERT_OR_RETURN(pGVAS != NULL, NV_ERR_OBJECT_NOT_FOUND); 1262 1263 return gvaspaceGetPageLevelInfo(pGVAS, pGpu, pParams); 1264 } 1265 1266 NvBool 1267 fabricvaspaceIsInUse_IMPL 1268 ( 1269 FABRIC_VASPACE *pFabricVAS 1270 ) 1271 { 1272 return gvaspaceIsInUse(dynamicCast(pFabricVAS->pGVAS, OBJGVASPACE)); 1273 } 1274