1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2019-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "mem_mgr_internal.h" 25 #include "mem_mgr/mem_list.h" 26 #include "deprecated/rmapi_deprecated.h" 27 28 #include "mem_mgr/video_mem.h" 29 #include "gpu/mem_mgr/mem_desc.h" 30 #include "gpu/device/device.h" 31 #include "gpu/mem_mgr/mem_mgr.h" 32 #include "gpu/mem_sys/kern_mem_sys.h" 33 #include "gpu/mmu/kern_gmmu.h" 34 #include "gpu/bus/kern_bus.h" 35 #include "vgpu/vgpu_events.h" 36 #include "virtualization/hypervisor/hypervisor.h" 37 #include "gpu/mem_mgr/heap.h" 38 #include "os/os.h" 39 #include "rmapi/client.h" 40 41 #include "class/cl003e.h" // NV01_MEMORY_SYSTEM 42 #include "class/cl84a0.h" // NV01_MEMORY_LIST_XXX 43 #include "class/cl0040.h" // NV01_MEMORY_LOCAL_USER 44 45 NV_STATUS 46 memlistConstruct_IMPL 47 ( 48 MemoryList *pMemoryList, 49 CALL_CONTEXT *pCallContext, 50 RS_RES_ALLOC_PARAMS_INTERNAL *pParams 51 ) 52 { 53 NV_MEMORY_LIST_ALLOCATION_PARAMS *pAllocParams; 54 RsResourceRef *pResourceRef = pCallContext->pResourceRef; 55 NvHandle hClient = pCallContext->pClient->hClient; 56 NvHandle hParent = pResourceRef->pParentRef->hResource; 57 NvU32 externalClassId = pResourceRef->externalClassId; 58 Memory *pMemory = staticCast(pMemoryList, Memory); 59 OBJGPU *pGpu = pMemory->pGpu; 60 NvBool bUserModeArgs = (pCallContext->secInfo.paramLocation == PARAM_LOCATION_USER); 61 MEMORY_DESCRIPTOR *pMemDesc = NULL; 62 NV_STATUS status = NV_OK; 63 NvU64 memSize; 64 NvU32 i; 65 Memory *pMemoryInfo = NULL; // TODO: This needs a more descriptive name (MemoryInfo for what?) 66 NV_ADDRESS_SPACE addressSpace = ADDR_UNKNOWN; 67 NvU32 Cache = 0; 68 MEMORY_DESCRIPTOR *src_pMemDesc = NULL; 69 NvU32 src_hClient; 70 NvU32 src_hParent; 71 OBJGPU *src_pGpu; 72 RmPhysAddr *pPteArray = NULL; 73 NvBool bContig; 74 NvU32 src_hHwResClient; 75 NvU32 src_hHwResDevice; 76 NvU32 src_hHwResHandle; 77 NvU32 result; 78 RS_PRIV_LEVEL privLevel = pCallContext->secInfo.privLevel; 79 80 // Copy-construction has already been done by the base Memory class 81 if (RS_IS_COPY_CTOR(pParams)) 82 return NV_OK; 83 84 pAllocParams = pParams->pAllocParams; 85 86 // 87 // We use memory list support in the GSP/DCE firmware RM so we want 88 // to bypass the hypervisor check here. 89 // 90 if (!hypervisorIsVgxHyper()) 91 { 92 if (IS_VIRTUAL(pGpu) && privLevel >= RS_PRIV_LEVEL_KERNEL) 93 { 94 goto continue_alloc_object; 95 } 96 if (IS_GSP_CLIENT(pGpu)) 97 { 98 goto continue_alloc_object; 99 } 100 return NV_ERR_NOT_SUPPORTED; 101 } 102 continue_alloc_object: 103 104 if (DRF_VAL(OS02, _FLAGS, _COHERENCY, pAllocParams->flagsOs02) == NVOS02_FLAGS_COHERENCY_UNCACHED) 105 Cache = NV_MEMORY_UNCACHED; 106 else if (DRF_VAL(OS02, _FLAGS, _COHERENCY, pAllocParams->flagsOs02) == NVOS02_FLAGS_COHERENCY_CACHED) 107 Cache = NV_MEMORY_CACHED; 108 else if (DRF_VAL(OS02, _FLAGS, _COHERENCY, pAllocParams->flagsOs02) == NVOS02_FLAGS_COHERENCY_WRITE_COMBINE) 109 Cache = NV_MEMORY_WRITECOMBINED; 110 else if (DRF_VAL(OS02, _FLAGS, _COHERENCY, pAllocParams->flagsOs02) == NVOS02_FLAGS_COHERENCY_WRITE_THROUGH) 111 Cache = NV_MEMORY_CACHED; 112 else if (DRF_VAL(OS02, _FLAGS, _COHERENCY, pAllocParams->flagsOs02) == NVOS02_FLAGS_COHERENCY_WRITE_PROTECT) 113 Cache = NV_MEMORY_CACHED; 114 else if (DRF_VAL(OS02, _FLAGS, _COHERENCY, pAllocParams->flagsOs02) == NVOS02_FLAGS_COHERENCY_WRITE_BACK) 115 Cache = NV_MEMORY_CACHED; 116 117 src_hClient = pAllocParams->hClient; 118 src_hParent = pAllocParams->hParent; 119 src_hHwResClient = pAllocParams->hHwResClient; 120 src_hHwResDevice = pAllocParams->hHwResDevice; 121 src_hHwResHandle = pAllocParams->hHwResHandle; 122 123 if (src_hClient == NV01_NULL_OBJECT) 124 { 125 src_hClient = hClient; 126 if (src_hParent != NV01_NULL_OBJECT) 127 { 128 return NV_ERR_INVALID_OBJECT_PARENT; 129 } 130 src_hParent = hParent; 131 src_pGpu = pGpu; 132 } 133 else 134 { 135 RsClient *pClient; 136 137 status = serverGetClientUnderLock(&g_resServ, src_hClient, &pClient); 138 if (status != NV_OK) 139 return status; 140 141 status = gpuGetByHandle(pClient, src_hParent, NULL, &src_pGpu); 142 if (status != NV_OK) 143 return status; 144 145 if (src_pGpu != pGpu) 146 return NV_ERR_INVALID_OBJECT_PARENT; 147 } 148 149 if (externalClassId == NV01_MEMORY_LIST_OBJECT) 150 { 151 if (pAllocParams->hObject == NV01_NULL_OBJECT) 152 { 153 return NV_ERR_INVALID_ARGUMENT; 154 } 155 } 156 else 157 { 158 if (pAllocParams->hObject != NV01_NULL_OBJECT) 159 { 160 return NV_ERR_INVALID_ARGUMENT; 161 } 162 } 163 if (pAllocParams->reserved_0 || (pAllocParams->pteAdjust >= RM_PAGE_SIZE)) 164 { 165 return NV_ERR_INVALID_ARGUMENT; 166 } 167 168 NV_ASSERT_OR_RETURN(!(pAllocParams->flags & NVOS32_ALLOC_FLAGS_VIRTUAL), NV_ERR_NOT_SUPPORTED); 169 170 switch (externalClassId) 171 { 172 case NV01_MEMORY_LIST_SYSTEM: 173 addressSpace = ADDR_SYSMEM; 174 break; 175 case NV01_MEMORY_LIST_FBMEM: 176 addressSpace = ADDR_FBMEM; 177 if (src_hHwResHandle != 0) 178 { 179 RsClient *pHwResClient; 180 181 status = serverGetClientUnderLock(&g_resServ, src_hHwResClient, &pHwResClient); 182 if (status == NV_OK) 183 { 184 status = memGetByHandleAndDevice(pHwResClient, src_hHwResHandle, 185 src_hHwResDevice, &pMemoryInfo); 186 } 187 if (status != NV_OK) 188 { 189 src_hHwResHandle = 0; 190 } 191 } 192 break; 193 case NV01_MEMORY_LIST_OBJECT: 194 { 195 RsClient *pClient; 196 197 pMemoryInfo = NULL; 198 199 status = serverGetClientUnderLock(&g_resServ, src_hClient, &pClient); 200 if (status != NV_OK) 201 { 202 return NV_ERR_INVALID_CLIENT; 203 } 204 205 status = memGetByHandle(pClient, pAllocParams->hObject, &pMemoryInfo); 206 if (status != NV_OK) 207 { 208 return status; 209 } 210 src_pMemDesc = pMemoryInfo->pMemDesc; 211 212 if (src_pMemDesc == NULL) 213 { 214 return NV_ERR_INVALID_CLASS; 215 } 216 217 addressSpace = memdescGetAddressSpace(src_pMemDesc); 218 switch (addressSpace) 219 { 220 case ADDR_SYSMEM: 221 case ADDR_FBMEM: 222 break; 223 default: 224 return NV_ERR_INVALID_OBJECT; 225 } 226 227 if (!memdescGetContiguity(src_pMemDesc, AT_GPU) && 228 ((src_pMemDesc->PageCount << RM_PAGE_SHIFT) < src_pMemDesc->Size)) 229 { 230 return NV_ERR_BUFFER_TOO_SMALL; 231 } 232 233 /* 234 * get the client device memory info reference which will be used to 235 * retrieve the associated hardware resource information (pHwResource) 236 * for Win VMs 237 */ 238 if (src_hHwResHandle != 0) 239 { 240 RsClient *pHwResClient; 241 242 status = serverGetClientUnderLock(&g_resServ, src_hHwResClient, &pHwResClient); 243 if (status == NV_OK) 244 { 245 status = memGetByHandleAndDevice(pHwResClient, src_hHwResHandle, 246 src_hHwResDevice, &pMemoryInfo); 247 } 248 if (status != NV_OK) 249 { 250 src_hHwResHandle = 0; 251 } 252 } 253 break; 254 } 255 default: 256 break; 257 } 258 259 bContig = FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, pAllocParams->flagsOs02); 260 memSize = pAllocParams->limit + 1; 261 262 if (addressSpace == ADDR_SYSMEM) 263 { 264 NvU32 attr2 = 0; 265 266 if ((src_pMemDesc != NULL) || bContig) 267 { 268 NV_ASSERT(0); 269 return NV_ERR_NOT_SUPPORTED; 270 } 271 272 status = memdescCreate(&pMemDesc, pGpu, memSize, 0, 273 bContig, addressSpace, Cache, 274 MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE); 275 if (status != NV_OK) 276 { 277 return status; 278 } 279 280 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED, NV_TRUE); 281 282 pMemDesc->PteAdjust = pAllocParams->pteAdjust; 283 memdescSetPteKind(pMemDesc, pAllocParams->format); 284 memdescSetGuestId(pMemDesc, pAllocParams->guestId); 285 286 pPteArray = memdescGetPteArray(pMemDesc, AT_GPU); 287 288 if ((pAllocParams->pageCount > pMemDesc->PageCount) || 289 !portSafeMulU32(sizeof(NvU64), pAllocParams->pageCount, &result)) 290 { 291 memdescDestroy(pMemDesc); 292 return NV_ERR_INVALID_ARGUMENT; 293 } 294 295 // copy in the pages 296 status = rmapiParamsCopyIn(NULL, 297 pPteArray, 298 pAllocParams->pageNumberList, 299 result, 300 bUserModeArgs); 301 if (status != NV_OK) 302 { 303 memdescDestroy(pMemDesc); 304 return status; 305 } 306 307 if (RMCFG_MODULE_KERNEL_BUS) 308 { 309 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 310 311 for (i = 0; i < pKernelBus->totalPciBars; ++i) 312 { 313 RmPhysAddr barOffset = pKernelBus->pciBars[i]; 314 NvU64 barSize = pKernelBus->pciBarSizes[i]; 315 316 if ((pPteArray[0] >= barOffset >> RM_PAGE_SHIFT) && 317 (pPteArray[0] < (barOffset + barSize) >> RM_PAGE_SHIFT)) 318 { 319 /* 320 * For SYSMEM address space when creating descriptor for 321 * physical BAR range, mark the descriptor as CPU only. This 322 * access is generally done when NvWatch is enabled and we want 323 * to map physical BAR range in CPU addressable memory and 324 * these addresses are not used for DMA. 325 */ 326 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_CPU_ONLY, NV_TRUE); 327 328 if (i == 0) 329 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_BAR0_REFLECT, NV_TRUE); 330 else if (i == 1) 331 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_BAR1_REFLECT, NV_TRUE); 332 333 break; 334 } 335 } 336 } 337 338 // reformat the pages to addresses 339 for (i = pAllocParams->pageCount; i > 0;) 340 { 341 i--; 342 memdescSetPte(pMemDesc, AT_GPU, i, 343 pPteArray[i] << RM_PAGE_SHIFT); 344 } 345 346 // this will fake a memory allocation at 347 // the OS driver interface level - and also set pMemDesc->Allocated 348 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_46, 349 pMemDesc); 350 351 if (status != NV_OK) 352 { 353 NV_PRINTF(LEVEL_ERROR, 354 "*** Cannot fake guest sysmem allocation. status =0x%x\n", 355 status); 356 357 memdescDestroy(pMemDesc); 358 return status; 359 } 360 361 NvU32 os32Flags = 0; 362 NvU32 attr = 0; 363 NvBool bUseOs02flag = NV_FALSE; 364 365 if (bUseOs02flag == NV_FALSE) 366 { 367 status = RmDeprecatedConvertOs02ToOs32Flags(pAllocParams->flagsOs02, &attr, &attr2, &os32Flags); 368 if (status != NV_OK) 369 { 370 memdescDestroy(pMemDesc); 371 return status; 372 } 373 374 status = memConstructCommon(pMemory, NV01_MEMORY_SYSTEM, os32Flags, pMemDesc, 375 0, NULL, attr, attr2, 0, 0, 376 NVOS32_MEM_TAG_NONE, (HWRESOURCE_INFO *)NULL); 377 } 378 else 379 { 380 if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_USER_READ_ONLY, _YES, pAllocParams->flagsOs02)) 381 attr2 |= DRF_DEF(OS32, _ATTR2, _PROTECTION_USER, _READ_ONLY); 382 383 status = memConstructCommon(pMemory, NV01_MEMORY_SYSTEM, 0, pMemDesc, 384 0, NULL, 0, attr2, 0, 0, NVOS32_MEM_TAG_NONE, (HWRESOURCE_INFO *)NULL); 385 } 386 387 if (status != NV_OK) 388 { 389 memdescDestroy(pMemDesc); 390 } 391 } 392 else if (addressSpace == ADDR_FBMEM) 393 { 394 NvU64 newBase; 395 NvU64 baseOffset = 0; 396 NvU64 trueLength; 397 NvU32 hwResId = 0; 398 NvU64 pageSize = 0; 399 RM_ATTR_PAGE_SIZE pageSizeAttr; 400 FB_ALLOC_INFO *pFbAllocInfo = NULL; 401 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL; 402 HWRESOURCE_INFO *pHwResource = NULL; 403 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 404 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 405 Heap *pHeap; 406 NvBool bCallingContextPlugin; 407 RsResourceRef *pDeviceRef; 408 409 NV_ASSERT_OK_OR_RETURN( 410 refFindAncestorOfType(pResourceRef, classId(Device), &pDeviceRef)); 411 412 pHeap = vidmemGetHeap(pGpu, 413 dynamicCast(pDeviceRef->pResource, Device), 414 NV_FALSE, 415 NV_FALSE); 416 417 // 418 // When guest RM client doesn't subscribe to MIG partition and requests for vidmem allocation 419 // vidmemGetHeap() returns NULL for heap. Hence, add below assert for validation. 420 // 421 NV_ASSERT_OR_RETURN((pHeap != NULL), NV_ERR_INVALID_STATE); 422 423 // Must be of valid type, in FBMEM, one page for contig. 424 if ((pAllocParams->type >= NVOS32_NUM_MEM_TYPES) || 425 (bContig && (pAllocParams->pageCount > 1)) || 426 (!FLD_TEST_DRF(OS32, _ATTR, _LOCATION, _VIDMEM, pAllocParams->attr)) || 427 (pAllocParams->flags & (NVOS32_ALLOC_FLAGS_TURBO_CIPHER_ENCRYPTED | 428 NVOS32_ALLOC_FLAGS_ALIGNMENT_HINT | 429 NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE | 430 NVOS32_ALLOC_FLAGS_BANK_FORCE))) 431 { 432 return NV_ERR_INVALID_ARGUMENT; 433 } 434 435 pAllocParams->flags |= (NVOS32_ALLOC_FLAGS_MEMORY_HANDLE_PROVIDED | 436 NVOS32_ALLOC_FLAGS_IGNORE_BANK_PLACEMENT); 437 438 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO)); 439 if (pFbAllocInfo == NULL) 440 { 441 NV_ASSERT(0); 442 status = NV_ERR_NO_MEMORY; 443 goto done_fbmem; 444 } 445 446 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT)); 447 if (pFbAllocPageFormat == NULL) { 448 NV_ASSERT(0); 449 status = NV_ERR_NO_MEMORY; 450 goto done_fbmem; 451 } 452 453 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 454 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 455 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 456 457 if (src_pMemDesc != NULL) 458 { 459 trueLength = (src_pMemDesc->PteAdjust + 460 src_pMemDesc->Size); 461 baseOffset = memdescGetPhysAddr(src_pMemDesc, AT_GPU, 0); 462 } 463 else 464 { 465 NvU64 base = 0; 466 heapGetSize(pHeap, &trueLength); 467 heapGetBase(pHeap, &base); 468 trueLength = base + trueLength; 469 } 470 471 // pAllocParams->hHwResHandle can be non-zero only for Win VMs and 472 // at least one of NVOS32_ATTR_COMPR or NVOS32_ATTR_ZCULL can be 473 // set in pAllocParams->attr only for Linux VMs 474 NV_ASSERT((pAllocParams->hHwResHandle == 0) || 475 !(pAllocParams->attr & (DRF_SHIFTMASK(NVOS32_ATTR_COMPR) | 476 DRF_SHIFTMASK(NVOS32_ATTR_ZCULL)))); 477 478 status = memdescCreate(&pMemDesc, pGpu, memSize, 0, bContig, addressSpace, 479 Cache, MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE_FB_BC_ONLY(pGpu, addressSpace)); 480 if (status != NV_OK) 481 { 482 goto done_fbmem; 483 } 484 pPteArray = memdescGetPteArray(pMemDesc, AT_GPU); 485 // copy in the pages 486 // copy in the pages 487 status = rmapiParamsCopyIn(NULL, 488 pPteArray, 489 pAllocParams->pageNumberList, 490 sizeof(NvU64) * pAllocParams->pageCount, 491 bUserModeArgs); 492 493 if (status != NV_OK) 494 { 495 goto done_fbmem; 496 } 497 498 if (bContig) 499 { 500 newBase = (pPteArray[0] << RM_PAGE_SHIFT) + pAllocParams->pteAdjust; 501 502 if ((newBase + memSize) > trueLength) 503 { 504 NV_PRINTF(LEVEL_ERROR, 505 "Out of range contig memory at 0x%016llx of size 0x%016llx\n", 506 newBase, memSize); 507 status = NV_ERR_INVALID_ARGUMENT; 508 goto done_fbmem; 509 } 510 } 511 512 // reformat the pages to addresses 513 for (i = pAllocParams->pageCount; i > 0;) 514 { 515 i--; 516 newBase = (pPteArray[i] << RM_PAGE_SHIFT); 517 if ((newBase + RM_PAGE_SIZE) > trueLength) 518 { 519 NV_PRINTF(LEVEL_ERROR, 520 "Out of range page address 0x%016llx\n", newBase); 521 status = NV_ERR_BUFFER_TOO_SMALL; 522 goto done_fbmem; 523 } 524 memdescSetPte(pMemDesc, AT_GPU, i, newBase + baseOffset); 525 } 526 527 NV_ASSERT_OK_OR_GOTO(status, vgpuIsCallingContextPlugin(pMemDesc->pGpu, &bCallingContextPlugin), done_fbmem); 528 if (!bCallingContextPlugin) 529 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED, NV_TRUE); 530 531 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_LIST_MEMORY, NV_TRUE); 532 533 if (pAllocParams->attr & (DRF_SHIFTMASK(NVOS32_ATTR_COMPR) | 534 DRF_SHIFTMASK(NVOS32_ATTR_ZCULL))) 535 { 536 // 537 // Request any chip-specific resources for memory of this 538 // pAllocParams->type (e.g. tiles). This call may adjust size, pPitch 539 // and alignment as necessary. 540 // 541 pFbAllocInfo->pageFormat->type = pAllocParams->type; 542 pFbAllocInfo->hwResId = hwResId; 543 pFbAllocInfo->pad = 0; 544 pFbAllocInfo->alignPad = 0; 545 pFbAllocInfo->height = pAllocParams->height; 546 pFbAllocInfo->width = pAllocParams->width; 547 pFbAllocInfo->pitch = pAllocParams->pitch; 548 pFbAllocInfo->size = pAllocParams->size; 549 pFbAllocInfo->origSize = pAllocParams->size; 550 pFbAllocInfo->offset = ~0; 551 pFbAllocInfo->pageFormat->flags = pAllocParams->flags; 552 pFbAllocInfo->internalflags = 0; 553 pFbAllocInfo->pageFormat->attr = pAllocParams->attr; 554 pFbAllocInfo->retAttr = pAllocParams->attr; 555 pFbAllocInfo->pageFormat->attr2 = pAllocParams->attr2; 556 pFbAllocInfo->retAttr2 = pAllocParams->attr2; 557 pFbAllocInfo->format = pAllocParams->format; 558 pFbAllocInfo->comprCovg = pAllocParams->comprcovg; 559 pFbAllocInfo->zcullCovg = 0; 560 pFbAllocInfo->ctagOffset = pAllocParams->ctagOffset; 561 pFbAllocInfo->hClient = hClient; 562 pFbAllocInfo->hDevice = hParent; /* device */ 563 pFbAllocInfo->bIsKernelAlloc = NV_FALSE; 564 565 // only a kernel client can request for a protected allocation 566 if (pFbAllocInfo->pageFormat->flags & NVOS32_ALLOC_FLAGS_ALLOCATE_KERNEL_PRIVILEGED) 567 { 568 if (privLevel < RS_PRIV_LEVEL_KERNEL) 569 { 570 status = NV_ERR_INSUFFICIENT_PERMISSIONS; 571 NV_PRINTF(LEVEL_ERROR, "only kernel clients may request for a protected allocation\n"); 572 goto done_fbmem; 573 } 574 pFbAllocInfo->bIsKernelAlloc = NV_TRUE; 575 } 576 577 if ((pAllocParams->flags & NVOS32_ALLOC_FLAGS_ALIGNMENT_HINT) || 578 (pAllocParams->flags & NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE)) 579 { 580 pFbAllocInfo->align = pAllocParams->align; 581 } 582 else 583 { 584 pFbAllocInfo->align = RM_PAGE_SIZE; 585 } 586 587 // Fetch RM page size 588 pageSize = memmgrDeterminePageSize(pMemoryManager, pFbAllocInfo->hClient, pFbAllocInfo->size, 589 pFbAllocInfo->format, pFbAllocInfo->pageFormat->flags, 590 &pFbAllocInfo->retAttr, &pFbAllocInfo->retAttr2); 591 592 if (pageSize == 0) 593 { 594 status = NV_ERR_INVALID_STATE; 595 NV_PRINTF(LEVEL_ERROR, "memmgrDeterminePageSize failed\n"); 596 goto done_fbmem; 597 } 598 599 // Fetch memory alignment 600 status = memmgrAllocDetermineAlignment_HAL(pGpu, pMemoryManager, &pFbAllocInfo->size, &pFbAllocInfo->align, 601 pFbAllocInfo->alignPad, pFbAllocInfo->pageFormat->flags, 602 pFbAllocInfo->retAttr, pFbAllocInfo->retAttr2, 0); 603 604 if (status != NV_OK) 605 { 606 NV_PRINTF(LEVEL_ERROR, "memmgrAllocDetermineAlignment failed\n"); 607 goto done_fbmem; 608 } 609 610 // 611 // Call into HAL to reserve any hardware resources for 612 // the specified memory type. 613 // If the alignment was changed due to a HW limitation, and the 614 // flag NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE is set, bad_argument 615 // will be passed back from the HAL 616 // 617 status = memmgrAllocHwResources(pGpu, pMemoryManager, pFbAllocInfo); 618 if (status != NV_OK) 619 { 620 NV_PRINTF(LEVEL_ERROR, "memmgrAllocHwResources failure!\n"); 621 goto done_fbmem; 622 } 623 624 // No need to check format if comptag allocation is not requested 625 if (!(pAllocParams->flags & NVOS32_ALLOC_FLAGS_SKIP_RESOURCE_ALLOC)) 626 { 627 NV_ASSERT(pFbAllocInfo->format == pAllocParams->format); 628 } 629 630 NV_PRINTF(LEVEL_INFO, "fbAlloc for comptag successful!\n"); 631 632 /* Create hwResource from pFbAllocInfo */ 633 pHwResource = portMemAllocNonPaged(sizeof(HWRESOURCE_INFO)); 634 if (pHwResource == NULL) 635 { 636 (void)memmgrFreeHwResources(pGpu, pMemoryManager, pFbAllocInfo); 637 status = NV_ERR_NO_MEMORY; 638 goto done_fbmem; 639 } 640 641 portMemSet(pHwResource, 0x0, sizeof(HWRESOURCE_INFO)); 642 pHwResource->attr = pFbAllocInfo->retAttr; 643 pHwResource->attr2 = pFbAllocInfo->retAttr2; 644 pHwResource->comprCovg = pFbAllocInfo->comprCovg; 645 pHwResource->ctagOffset = pFbAllocInfo->ctagOffset; 646 pHwResource->hwResId = pFbAllocInfo->hwResId; 647 648 NV_PRINTF(LEVEL_INFO, "memmgrAllocHwResources result\n"); 649 NV_PRINTF(LEVEL_INFO, " Attr:0x%x\n", 650 pFbAllocInfo->retAttr); 651 NV_PRINTF(LEVEL_INFO, " Attr2:0x%x\n", 652 pFbAllocInfo->retAttr2); 653 NV_PRINTF(LEVEL_INFO, " comprCovg:0x%x\n", 654 pFbAllocInfo->comprCovg); 655 NV_PRINTF(LEVEL_INFO, " zcullCovg:0x%x\n", 656 pFbAllocInfo->zcullCovg); 657 NV_PRINTF(LEVEL_INFO, " ctagOffset:0x%x\n", 658 pFbAllocInfo->ctagOffset); 659 NV_PRINTF(LEVEL_INFO, " hwResId:0x%x\n", 660 pFbAllocInfo->hwResId); 661 662 hwResId = pFbAllocInfo->hwResId; 663 664 // 665 // For Linux Guest, we allocate hardware resources on the host through 666 // NV01_MEMORY_LIST_OBJECT or NV01_MEMORY_LIST_FBMEM class objects. 667 // isGuestAllocated flag when set TRUE indicates that hardware resource 668 // is allocated for Linux Guest and we will free it in 669 // memDestructCommon. 670 // 671 pHwResource->isGuestAllocated = NV_TRUE; 672 } 673 else if (src_hHwResHandle != 0) 674 { 675 // obtaining hardware resources info for Win VMs 676 pHwResource = pMemoryInfo->pHwResource; 677 hwResId = pHwResource->hwResId; 678 } 679 680 681 pageSizeAttr = dmaNvos32ToPageSizeAttr(pAllocParams->attr, pAllocParams->attr2); 682 683 switch (pageSizeAttr) 684 { 685 case RM_ATTR_PAGE_SIZE_4KB: 686 memdescSetPageSize(pMemDesc, AT_GPU, RM_PAGE_SIZE); 687 break; 688 case RM_ATTR_PAGE_SIZE_BIG: 689 memdescSetPageSize(pMemDesc, AT_GPU, RM_PAGE_SIZE_64K); 690 break; 691 case RM_ATTR_PAGE_SIZE_HUGE: 692 if (!kgmmuIsHugePageSupported(pKernelGmmu)) 693 { 694 NV_ASSERT(0); 695 status = NV_ERR_INVALID_ARGUMENT; 696 goto done_fbmem; 697 } 698 memdescSetPageSize(pMemDesc, AT_GPU, RM_PAGE_SIZE_HUGE); 699 break; 700 case RM_ATTR_PAGE_SIZE_512MB: 701 if (!kgmmuIsPageSize512mbSupported(pKernelGmmu)) 702 { 703 NV_ASSERT(0); 704 status = NV_ERR_INVALID_ARGUMENT; 705 goto done_fbmem; 706 } 707 memdescSetPageSize(pMemDesc, AT_GPU, RM_PAGE_SIZE_512M); 708 break; 709 case RM_ATTR_PAGE_SIZE_DEFAULT: 710 NV_PRINTF(LEVEL_INFO, "page size default doesn't have any impact \n"); 711 break; 712 case RM_ATTR_PAGE_SIZE_INVALID: 713 NV_PRINTF(LEVEL_INFO, "unexpected pageSizeAttr = 0x%x\n", pageSizeAttr); 714 status = NV_ERR_INVALID_STATE; 715 goto done_fbmem; 716 } 717 718 status = memConstructCommon(pMemory, 719 NV01_MEMORY_LOCAL_USER, 720 0, // flags 721 pMemDesc, 722 0, // heapOwner 723 pHeap, 724 pAllocParams->attr, 725 pAllocParams->attr2, 726 0, // pitch 727 pAllocParams->type, // type 728 NVOS32_MEM_TAG_NONE, 729 pHwResource); // pHwResource 730 if (status != NV_OK) 731 { 732 goto done_fbmem; 733 } 734 735 if ((pHwResource != NULL) && (src_hHwResHandle == 0)) 736 { 737 portMemFree(pHwResource); 738 pHwResource = NULL; 739 } 740 741 memdescSetPteKind(pMemory->pMemDesc, pAllocParams->format); 742 memdescSetHwResId(pMemory->pMemDesc, hwResId); 743 744 done_fbmem: 745 if (status != NV_OK) 746 { 747 memdescDestroy(pMemDesc); 748 749 if (src_hHwResHandle == 0) 750 { 751 portMemFree(pHwResource); 752 753 /* release hwResId resources */ 754 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO)); 755 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT)); 756 pFbAllocInfo->pageFormat = pFbAllocPageFormat; 757 pFbAllocInfo->pageFormat->type = pAllocParams->type; 758 pFbAllocInfo->hwResId = hwResId; 759 pFbAllocInfo->size = memSize; 760 pFbAllocInfo->hClient = hClient; 761 pFbAllocInfo->hDevice = hParent; 762 763 memmgrFreeHwResources(pGpu, pMemoryManager, pFbAllocInfo); 764 } 765 } 766 767 portMemFree(pFbAllocPageFormat); 768 portMemFree(pFbAllocInfo); 769 } 770 else 771 { 772 status = NV_ERR_INVALID_CLASS; 773 } 774 return status; 775 } 776 777 NvBool 778 memlistCanCopy_IMPL 779 ( 780 MemoryList *pMemoryList 781 ) 782 { 783 return NV_TRUE; 784 } 785