1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2005-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 25 /*! 26 * @file 27 * @brief The FERMI specific HAL VMA routines reside in this file 28 * 29 * =========================================================================== 30 * GLOSSARY OF INCONSISTENCIES 31 * =========================================================================== 32 * 33 * -------- 34 * LOW, MIN 35 * -------- 36 * (1) Synonyms for the first address or index in a range. 37 * e.g. In the inclusive range 37 to 509, the "low" or "min" is 37. 38 * 39 * --------------------- 40 * HIGH, MAX, LIMIT, END 41 * --------------------- 42 * (1) Synonyms for the last address or index in a range. 43 * e.g. In the inclusive range 37 to 509, the "limit" is 509. 44 * (2) Sometimes "high" or "end" are used for the "limit plus one" - e.g. 510. 45 * Currently this can only be determined by context. 46 * TODO: Decide on consistent terms and clean this up. 47 * 48 * --- 49 * PDB 50 * --- 51 * (1) Page Directory Base 52 * The base address of a page directory, 53 * e.g. written to the PD_BASE field of an instance block. 54 * (2) Page Directory Block 55 * The entire physical memory block of a page directory, 56 * e.g. described by a memdesc associated with a VASPACE object. 57 * (3) Property DataBase - e.g. in PDB_PROP_* 58 * The common meaning to the rest of RM - boolean values associated 59 * with an object. Completely unrelated to (1) and (2). 60 * 61 * --- 62 * PDE 63 * --- 64 * (1) Page Directory Entry 65 * An *ENTRY* within a page directory, containing the physical 66 * addresses and attributes of a single small/big page table pair. 67 * (2) !!!WRONG!!! The page direcory itself 68 * Somtimes also used in the plural form "PDEs". 69 * Use "page directory" or "PD" instead. 70 * 71 * -------------------------- 72 * PDE ENTRY !!!DO NOT USE!!! 73 * -------------------------- 74 * (1) !!!WRONG!!! Page Directory Entry Entry(?!) 75 * This is redundant - just use "PDE". 76 * (2) Page Dir*E*ctory Entry 77 * Desperate bacronym to justify current usage. 78 * 79 * -------- 80 * PDE SIZE 81 * -------- 82 * (1) Size or index corresponding to the NV_MMU_PDE_SIZE field of a PDE. 83 * This refers to the size of *page tables* that this 84 * PDE points to (1/8, 1/4, 1/2, full), not the size of the PDE itself. 85 * The more accurate term is "PT size" - most code has been cleaned up 86 * to use this instead, but some API params remain. 87 * (2) Size of the PDE itself (8 bytes), defined by the constant NV_MMU_PDE__SIZE. 88 * 89 * --- 90 * PTE 91 * --- 92 * (1) Page Table Entry 93 * An *ENTRY* within a page table, containing the physical 94 * address and attributes of a single page (small or big). 95 * (2) !!!WRONG!!! The page table itself 96 * Somtimes also used in the plural form "PTEs". 97 * Use "page table" or "PT" instead. 98 * 99 * -------------------------- 100 * PTE ENTRY !!!DO NOT USE!!! 101 * -------------------------- 102 * (1) !!!WRONG!!! Page Table Entry Entry(?!) 103 * This is redundant - just use "PTE". 104 * (2) Page Tabl*E* Entry 105 * Desperate bacronym to justify current usage. 106 * 107 */ 108 109 #include "core/core.h" 110 #include "gpu/gpu.h" 111 #include "lib/base_utils.h" 112 #include "gpu/mem_mgr/heap.h" 113 #include "os/os.h" 114 #include "rmapi/client.h" 115 #include "nvRmReg.h" 116 #include "gpu/mem_mgr/virt_mem_allocator.h" 117 #include "gpu/bif/kernel_bif.h" 118 #include "core/system.h" 119 #include "core/thread_state.h" 120 #include "mem_mgr/vaspace.h" 121 #include "mem_mgr/fabric_vaspace.h" 122 #include "mem_mgr/virt_mem_mgr.h" 123 124 #include "mem_mgr/fla_mem.h" 125 126 #include "gpu/mmu/kern_gmmu.h" 127 #include "gpu/mem_sys/kern_mem_sys.h" 128 #include "gpu_mgr/gpu_group.h" 129 #include "mmu/mmu_fmt.h" 130 #include "gpu/device/device.h" 131 #include "gpu/nvlink/kernel_nvlink.h" 132 #include "gpu/bus/kern_bus.h" 133 #include "gpu/mem_mgr/mem_mgr.h" 134 135 #include "gpu/mem_mgr/fermi_dma.h" 136 137 #include "published/maxwell/gm107/dev_mmu.h" 138 #include "published/maxwell/gm107/dev_bus.h" 139 140 #include "ctrl/ctrl0002.h" 141 142 #include "vgpu/rpc.h" 143 144 #define _MMUXLATEVADDR_FLAG_SHOW_INVALID NVBIT(0) 145 #define _MMUXLATEVADDR_FLAG_VALIDATE_ONLY NVBIT(1) // incomplete 146 #define _MMUXLATEVADDR_FLAG_VALIDATE_TERSELY NVBIT(2) // incomplete 147 // no trace output 148 #define _MMUXLATEVADDR_FLAG_XLATE_ONLY _MMUXLATEVADDR_FLAG_VALIDATE_TERSELY 149 150 static NV_STATUS _dmaGetFabricAddress(OBJGPU *pGpu, NvU32 aperture, NvU32 kind, NvU64 *fabricAddr); 151 152 static NV_STATUS 153 _dmaApplyWarForBug2720120 154 ( 155 OBJGVASPACE *pGVAS, 156 OBJGPU *pGpu, 157 const NvU64 vaLo, 158 const NvU64 vaHi 159 ); 160 161 // 162 // Virtual Address Space Block - Data tracked per virtual allocation 163 // 164 // only used with NV_REG_STR_RESERVE_PTE_SYSMEM_MB. Protential dead code. 165 typedef struct VASINFO_MAXWELL 166 { 167 PNODE pMapTree; // Tree of current mappings. 168 NvU32 pageSizeMask; // Mask of page size indices supported. 169 // See VAS_PAGESIZE_MASK. 170 VAS_ALLOC_FLAGS flags; 171 VA_MANAGEMENT management; // Level of management. 172 } VASINFO_MAXWELL, *PVASINFO_MAXWELL; 173 174 /*! 175 * @brief Allocate virtual memory and map it to physical memory. 176 * 177 * The virtual memory may already be allocated, in which case it is just 178 * initialized (backing page table tables allocated). 179 * 180 * VMM-TODO: If possible remove overloading - e.g. just map, never allocate. 181 * Definitely move MMU stuff down. 182 * 183 * @param[in] pGpu OBJGPU pointer 184 * @param[in] pDma VirtMemAllocator pointer 185 * @param[in] pVAS OBJVASPACE pointer 186 * @param[in] pMemDesc Physical memory descriptor 187 * @param[in/out] pVaddr Pointer to Virtual memory base address 188 * @param[in] flags Mapping options 189 * @param[in] pDmaMappingInfo CLI_DMA_MAPPING_INFO pointer (for RM Client mappings) 190 * @param[in] swizzId SMC swizzId (Only used for BAR1 mapping) 191 * 192 * @returns NV_STATUS status = NV_OK on success, or status upon failure. 193 */ 194 NV_STATUS 195 dmaAllocMapping_GM107 196 ( 197 OBJGPU *pGpu, 198 VirtMemAllocator *pDma, 199 OBJVASPACE *pVAS, 200 MEMORY_DESCRIPTOR *pMemDesc, 201 NvU64 *pVaddr, 202 NvU32 flags, 203 CLI_DMA_ALLOC_MAP_INFO *pCliMapInfo, 204 NvU32 swizzId 205 ) 206 { 207 NV_STATUS status = NV_OK; 208 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 209 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 210 OBJEHEAP *pVASpaceHeap = NULL; 211 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 212 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 213 FABRIC_VASPACE *pFabricVAS = dynamicCast(pGpu->pFabricVAS, FABRIC_VASPACE); 214 MEMORY_DESCRIPTOR *pAdjustedMemDesc = pMemDesc; 215 ADDRESS_TRANSLATION addressTranslation; 216 NvU32 gfid; 217 NvBool bCallingContextPlugin; 218 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 219 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 220 OBJGVASPACE *pGVAS = NULL; 221 222 struct 223 { 224 NvU32 pteCount; 225 NvU32 pageCount; 226 NvU32 overMap; 227 NvU64 vaLo; 228 NvU64 vaHi; 229 NvU64 mapLength; 230 NvU64 pageOffset; 231 NvU64 pageSize; 232 NvU64 vaRangeLo; 233 NvU64 vaRangeHi; 234 NvU32 kind; 235 NvU32 priv; 236 NvU32 cacheSnoop; 237 COMPR_INFO comprInfo; 238 NvU32 aperture; 239 NvU32 tlbLock; 240 NvU32 p2p; 241 NvU32 writeOnly; 242 NvU32 readOnly; 243 NvU32 subDevIdSrc; 244 NvU32 deferInvalidate; 245 NODE *pMapNode; 246 NvU32 shaderFlags; 247 NvU32 disableEncryption; 248 VASINFO_MAXWELL *pVASInfo; 249 OBJGPU *pSrcGpu; 250 NvU32 peerNumber; 251 NvBool bAllocVASpace; 252 NvBool bIsBarOrPerf; 253 NvBool bIsBar1; 254 NvBool bIsMIGMemPartitioningEnabled; 255 RmPhysAddr *pPteArray; 256 DMA_PAGE_ARRAY pageArray; 257 NvU64 vaspaceBigPageSize; 258 NvBool bIsMemContiguous; 259 NvU64 fabricAddr; 260 NvU32 indirectPeer; 261 NvBool bFlaImport; 262 NV_RANGE totalVaRange; 263 MEMORY_DESCRIPTOR *pRootMemDesc; 264 MEMORY_DESCRIPTOR *pTempMemDesc; 265 Memory *pMemory; 266 NvU32 pageArrayGranularity; 267 NvU8 pageShift; 268 NvU64 physPageSize; 269 } *pLocals = portMemAllocNonPaged(sizeof(*pLocals)); 270 // Heap Allocate to avoid stack overflow 271 272 if (pLocals == NULL) 273 return NV_ERR_NO_MEMORY; 274 275 portMemSet(pLocals, 0, sizeof(*pLocals)); 276 pLocals->pSrcGpu = pGpu; 277 pLocals->peerNumber = BUS_INVALID_PEER; 278 pLocals->totalVaRange = NV_RANGE_EMPTY; 279 280 if (pCliMapInfo != NULL) 281 pLocals->pMemory = pCliMapInfo->pMemory; 282 283 pLocals->vaspaceBigPageSize = vaspaceGetBigPageSize(pVAS); 284 pLocals->bIsBarOrPerf = (vaspaceGetFlags(pVAS) & 285 (VASPACE_FLAGS_BAR|VASPACE_FLAGS_PERFMON|VASPACE_FLAGS_HDA)) != 0; 286 pLocals->p2p = DRF_VAL(OS46, _FLAGS, _P2P_ENABLE, flags); 287 pLocals->subDevIdSrc = DRF_VAL(OS46, _FLAGS, _P2P_SUBDEV_ID_SRC, flags); 288 pLocals->deferInvalidate = FLD_TEST_DRF(OS46, _FLAGS, _DEFER_TLB_INVALIDATION, _TRUE, flags) ? 289 DMA_DEFER_TLB_INVALIDATE : DMA_TLB_INVALIDATE; 290 pLocals->bAllocVASpace = FLD_TEST_DRF(OS46, _FLAGS, _DMA_UNICAST_REUSE_ALLOC, _FALSE, flags); 291 pLocals->bIsBar1 = (vaspaceGetFlags(pVAS) & VASPACE_FLAGS_BAR_BAR1) != 0; 292 pLocals->bIsMIGMemPartitioningEnabled = (pKernelMIGManager != NULL) && kmigmgrIsMIGMemPartitioningEnabled(pGpu, pKernelMIGManager); 293 294 pLocals->cacheSnoop = (NVOS46_FLAGS_CACHE_SNOOP_ENABLE == DRF_VAL(OS46, _FLAGS, _CACHE_SNOOP, flags)); 295 pLocals->writeOnly = (NVOS46_FLAGS_ACCESS_WRITE_ONLY == DRF_VAL(OS46, _FLAGS, _ACCESS, flags)); 296 pLocals->readOnly = (NVOS46_FLAGS_ACCESS_READ_ONLY == DRF_VAL(OS46, _FLAGS, _ACCESS, flags)) ? 297 DMA_UPDATE_VASPACE_FLAGS_READ_ONLY : 0; 298 pLocals->tlbLock = (NVOS46_FLAGS_TLB_LOCK_ENABLE == DRF_VAL(OS46, _FLAGS, _TLB_LOCK, flags)) ? 299 DMA_UPDATE_VASPACE_FLAGS_TLB_LOCK : 0; 300 301 switch (DRF_VAL(OS46, _FLAGS, _SHADER_ACCESS, flags)) 302 { 303 default: 304 case NVOS46_FLAGS_SHADER_ACCESS_DEFAULT: 305 // The default (0) case we pick up the SHADER_ACCESS from ACCESS. 306 pLocals->shaderFlags = 0; 307 if (pLocals->readOnly) 308 pLocals->shaderFlags |= DMA_UPDATE_VASPACE_FLAGS_SHADER_READ_ONLY; 309 if (pLocals->writeOnly) 310 pLocals->shaderFlags |= DMA_UPDATE_VASPACE_FLAGS_SHADER_WRITE_ONLY; 311 break; 312 case NVOS46_FLAGS_SHADER_ACCESS_READ_WRITE: 313 pLocals->shaderFlags = 0; 314 break; 315 case NVOS46_FLAGS_SHADER_ACCESS_READ_ONLY: 316 pLocals->shaderFlags = DMA_UPDATE_VASPACE_FLAGS_SHADER_READ_ONLY; 317 break; 318 case NVOS46_FLAGS_SHADER_ACCESS_WRITE_ONLY: 319 pLocals->shaderFlags = DMA_UPDATE_VASPACE_FLAGS_SHADER_WRITE_ONLY; 320 break; 321 } 322 323 addressTranslation = VAS_ADDRESS_TRANSLATION(pVAS); 324 // In SRIOV-heavy plugin may map subheap allocations for itself using BAR1 325 NV_ASSERT_OK_OR_RETURN(vgpuIsCallingContextPlugin(pGpu, &bCallingContextPlugin)); 326 if (bCallingContextPlugin) 327 addressTranslation = FORCE_VMMU_TRANSLATION(pMemDesc, addressTranslation); 328 329 if (pFabricVAS != NULL) 330 { 331 status = fabricvaspaceGetGpaMemdesc(pFabricVAS, pMemDesc, pGpu, &pAdjustedMemDesc); 332 if (status != NV_OK) 333 { 334 NV_PRINTF(LEVEL_ERROR, "Failed to get the adjusted memdesc for the fabric memdesc\n"); 335 goto cleanup; 336 } 337 } 338 339 // Get pageSize 340 pLocals->pTempMemDesc = memdescGetMemDescFromGpu(pAdjustedMemDesc, pGpu); 341 342 // Get physical allocation granularity and page size. 343 pLocals->pageArrayGranularity = pLocals->pTempMemDesc->pageArrayGranularity; 344 pLocals->physPageSize = memdescGetPageSize64(pLocals->pTempMemDesc, addressTranslation); 345 346 // retrieve mapping page size from flags 347 switch(DRF_VAL(OS46, _FLAGS, _PAGE_SIZE, flags)) 348 { 349 case NVOS46_FLAGS_PAGE_SIZE_DEFAULT: 350 case NVOS46_FLAGS_PAGE_SIZE_BOTH: 351 pLocals->pageSize = memdescGetPageSize64(pLocals->pTempMemDesc, addressTranslation); 352 break; 353 case NVOS46_FLAGS_PAGE_SIZE_4KB: 354 pLocals->pageSize = RM_PAGE_SIZE; 355 break; 356 case NVOS46_FLAGS_PAGE_SIZE_BIG: 357 // case for arch specific 128K 358 pLocals->pageSize = pLocals->vaspaceBigPageSize; 359 break; 360 case NVOS46_FLAGS_PAGE_SIZE_HUGE: 361 pLocals->pageSize = RM_PAGE_SIZE_HUGE; 362 break; 363 default: 364 NV_PRINTF(LEVEL_ERROR, "Unknown page size flag encountered during mapping\n"); 365 return NV_ERR_INVALID_ARGUMENT; 366 } 367 368 NV_PRINTF(LEVEL_INFO, "Picked Page size based on flags: 0x%llx flagVal: 0x%x\n", 369 pLocals->pageSize, DRF_VAL(OS46, _FLAGS, _PAGE_SIZE, flags)); 370 371 if (pLocals->physPageSize < pLocals->pageSize) 372 { 373 NV_PRINTF(LEVEL_WARNING, "Requested mapping at larger page size than the physical granularity " 374 "PhysPageSize = 0x%llx MapPageSize = 0x%llx. Overriding to physical page granularity...\n", 375 pLocals->physPageSize, pLocals->pageSize); 376 pLocals->pageSize = pLocals->physPageSize; 377 } 378 379 if (memdescGetFlag(pLocals->pTempMemDesc, MEMDESC_FLAGS_DEVICE_READ_ONLY)) 380 { 381 NV_ASSERT_OR_ELSE((pLocals->readOnly == DMA_UPDATE_VASPACE_FLAGS_READ_ONLY), 382 status = NV_ERR_INVALID_ARGUMENT; goto cleanup); 383 } 384 385 // 386 // Force BAR1 VA pageSize at bigPageSize only if total BAR1 size is less 387 // than threshold(default: 256MB) to not waste BAR1. 388 // For large BAR1 SKUs, avoid forcing 64KB size and use the pagesize of 389 // the memdesc. 390 // 391 if (kgmmuIsVaspaceInteropSupported(pKernelGmmu) && 392 pLocals->bIsBar1) 393 { 394 if ((pLocals->pageSize > pLocals->vaspaceBigPageSize) && 395 kbusIsBar1Force64KBMappingEnabled(pKernelBus)) 396 { 397 pLocals->pageSize = pLocals->vaspaceBigPageSize; 398 399 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 400 memdescSetPageSize(memdescGetMemDescFromGpu(pAdjustedMemDesc, pGpu), 401 addressTranslation, (NvU32)pLocals->pageSize); 402 SLI_LOOP_END 403 } 404 } 405 406 pLocals->pageShift = BIT_IDX_32(pLocals->pageArrayGranularity); 407 408 // Get mapping params on current gpu memdesc 409 pLocals->pageOffset = memdescGetPhysAddr(pLocals->pTempMemDesc, addressTranslation, 0) & (pLocals->pageSize - 1); 410 pLocals->mapLength = RM_ALIGN_UP(pLocals->pageOffset + pLocals->pTempMemDesc->Size, pLocals->pageSize); 411 pLocals->pageCount = NvU64_LO32(pLocals->mapLength >> pLocals->pageShift); 412 pLocals->bIsMemContiguous = memdescGetContiguity(pLocals->pTempMemDesc, addressTranslation); 413 414 pLocals->kind = NV_MMU_PTE_KIND_PITCH; 415 416 // Get compression/pte pLocals->kind on current gpu memdesc 417 status = memmgrGetKindComprFromMemDesc(pMemoryManager, 418 pLocals->pTempMemDesc, 419 0, 420 &pLocals->kind, &pLocals->comprInfo); 421 422 if (NV_OK != status) 423 goto cleanup; 424 425 // 426 // When compression is enabled mapping at 4K is not supported due to 427 // RM allocating one comptagline per 64KB allocation (From Pascal to Turing). 428 // See bug 3909010 429 // 430 if ((pLocals->pageSize == RM_PAGE_SIZE) && 431 memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, pLocals->kind)) 432 { 433 NV_PRINTF(LEVEL_WARNING, "Requested 4K mapping on compressible sufrace. Overriding to physical page granularity..."); 434 pLocals->pageSize = pLocals->physPageSize; 435 } 436 437 #ifdef DEBUG 438 // Check for subdevices consistency if broadcast memdesc is passed in 439 if (memdescHasSubDeviceMemDescs(pAdjustedMemDesc)) 440 { 441 // Check pageOffset, pageSize consistency across subdevices 442 memdescCheckSubDevicePageSizeConsistency(pGpu, pAdjustedMemDesc, pVAS, pLocals->pageSize, pLocals->pageOffset); 443 444 // Check mem contiguity consistency across subdevices 445 memdescCheckSubDeviceMemContiguityConsistency(pGpu, pAdjustedMemDesc, pVAS, pLocals->bIsMemContiguous); 446 447 // Check compression/pte pLocals->kind consistency across subdevices 448 status = memdescCheckSubDeviceKindComprConsistency(pGpu, pAdjustedMemDesc, pVAS, 449 pLocals->kind, &pLocals->comprInfo); 450 NV_ASSERT(!status); 451 } 452 #endif 453 454 // 455 // +-- +-- +------------+ --+ 456 // | | | | |==> pageOffset 457 // pageSize <==| | | Page 0 | --+ 458 // | | | | | 459 // +-- | +------------+ | 460 // | | | | 461 // | | Page 1 | | 462 // | | | | 463 // mapLength <==| +------------+ |==> pMemDesc->Size 464 // | | | | 465 // | | ... | | 466 // | | | | 467 // | +------------+ | 468 // | | | | 469 // | | Page N-1 | --+ 470 // | | | 471 // +-- +------------+ 472 // 473 474 if (pLocals->bIsMemContiguous) 475 { 476 // FIXME: Throwing away physical length information is dangerous. 477 pLocals->pteCount = 1; 478 } 479 else 480 { 481 // FIXME: This is broken for page size > 4KB and page offset 482 // that crosses a page boundary (can overrun pPteArray). 483 // -- 484 // page count is one more than integral division in case of presence of offset hence being rounded up 485 pLocals->pteCount = RM_ALIGN_UP((pLocals->pTempMemDesc->Size + pLocals->pageOffset), pLocals->pageArrayGranularity) >> BIT_IDX_32(pLocals->pageArrayGranularity); 486 } 487 488 // Disable PLC Compression for FLA->PA Mapping because of the HW Bug: 3046774 489 if (pMemorySystemConfig->bUseRawModeComptaglineAllocation && 490 pMemorySystemConfig->bDisablePlcForCertainOffsetsBug3046774) 491 { 492 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 493 494 if (((vaspaceGetFlags(pVAS) & VASPACE_FLAGS_FLA) || (dynamicCast(pVAS, FABRIC_VASPACE) != NULL)) && 495 memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, pLocals->kind) && 496 !memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_DISALLOW_PLC, pLocals->kind)) 497 { 498 memmgrGetDisablePlcKind_HAL(pMemoryManager, &pLocals->kind); 499 } 500 } 501 502 if (pLocals->bIsBarOrPerf) 503 { 504 pLocals->totalVaRange = rangeMake(vaspaceGetVaStart(pVAS), vaspaceGetVaLimit(pVAS)); 505 506 // !!!! Nasty hack 507 // 508 // NVOS46_FLAGS_PTE_COALESCE_LEVEL_CAP used to get the encryption info from _busMapAperture_GF100(). 509 // Since we have no bit fields left in NVOS46_FLAGS_* to specify encryption info. 510 // This is applicable to FERMI+ chips. 511 // 512 // NVOS46_FLAGS_PTE_COALESCE_LEVEL_CAP is _NV50 specific, and is not used in FERMI+. 513 // NVOS46_FLAGS_PTE_COALESCE_LEVEL_CAP_DEFAULT means use default encryption status 514 // NVOS46_FLAGS_PTE_COALESCE_LEVEL_CAP_1 means disable encryption 515 // 516 // VMM-TODO: Add meaningful alias defines or just expand flag bits? 517 // 518 pLocals->disableEncryption = FLD_TEST_DRF(OS46, _FLAGS, _PTE_COALESCE_LEVEL_CAP, _1, flags) ? 519 DMA_UPDATE_VASPACE_FLAGS_DISABLE_ENCRYPTION : 0; 520 521 if (pLocals->bIsMemContiguous) 522 { 523 pLocals->overMap = pLocals->pageCount + NvU64_LO32((pLocals->pageOffset + (pLocals->pageSize - 1)) / pLocals->pageSize); 524 } 525 else 526 { 527 pLocals->overMap = pLocals->pageCount; 528 } 529 530 NV_ASSERT_OK_OR_GOTO(status, vgpuGetCallingContextGfid(pGpu, &gfid), cleanup); 531 532 // BAR1 VA space is split when MIG mem partitioning is enabled 533 if (pLocals->bIsBar1 && pLocals->bIsMIGMemPartitioningEnabled && IS_GFID_PF(gfid)) 534 { 535 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 536 537 pLocals->totalVaRange = memmgrGetMIGPartitionableBAR1Range(pGpu, pMemoryManager); 538 NV_ASSERT_OK_OR_GOTO(status, 539 kmemsysSwizzIdToMIGMemRange(pGpu, pKernelMemorySystem, swizzId, pLocals->totalVaRange, &pLocals->totalVaRange), 540 cleanup); 541 } 542 543 if (!FLD_TEST_DRF(OS46, _FLAGS, _DMA_OFFSET_FIXED, _TRUE, flags)) 544 { 545 pLocals->vaRangeLo = pLocals->totalVaRange.lo; 546 pLocals->vaRangeHi = pLocals->totalVaRange.hi; 547 } 548 } 549 else 550 { 551 552 NvU64 targetSpaceLength, targetSpaceBase, targetSpaceLimit; 553 554 NV_ASSERT((pLocals->pageSize == pLocals->vaspaceBigPageSize) || 555 (pLocals->pageSize == RM_PAGE_SIZE) || 556 (pLocals->pageSize == RM_PAGE_SIZE_HUGE) || 557 (pLocals->pageSize == RM_PAGE_SIZE_512M)); 558 559 pLocals->overMap = 0; 560 561 if (pCliMapInfo != NULL) 562 { 563 VirtualMemory *pVirtualMemory = pCliMapInfo->pVirtualMemory; 564 565 virtmemGetAddressAndSize(pVirtualMemory, &targetSpaceBase, &targetSpaceLength); 566 targetSpaceLimit = targetSpaceBase + targetSpaceLength - 1; 567 } 568 else 569 { 570 // RM internal mappings. Alt to dmaMapBuffer_HAL() 571 targetSpaceBase = vaspaceGetVaStart(pVAS); 572 targetSpaceLimit = vaspaceGetVaLimit(pVAS); 573 targetSpaceLength = targetSpaceLimit - targetSpaceBase + 1; 574 } 575 576 if (pLocals->pteCount > ((targetSpaceLength + (pLocals->pageArrayGranularity - 1)) / pLocals->pageArrayGranularity)) 577 { 578 NV_ASSERT(0); 579 status = NV_ERR_INVALID_ARGUMENT; 580 goto cleanup; 581 } 582 583 pVASpaceHeap = vaspaceGetHeap(pVAS); 584 585 if (!FLD_TEST_DRF(OS46, _FLAGS, _DMA_OFFSET_FIXED, _TRUE, flags)) 586 { 587 // offset of the context dma passed in when ctxdma allocated 588 // Virtual memory don't have any SMMU mapping. It is still OK to use the engine MMU context; it dont have any effect. 589 590 pLocals->vaRangeLo = NV_MAX(targetSpaceBase, vaspaceGetVaStart(pVAS)); 591 pLocals->vaRangeHi = NV_MIN(targetSpaceLimit, vaspaceGetVaLimit(pVAS)); 592 593 // 594 // Handle 32bit pointer requests. 32b pointers are forced below 32b 595 // on all chips. Non-32b requests are only forced on some chips, 596 // typically kepler, and only if there are no other address hints. 597 // 598 if (DRF_VAL(OS46, _FLAGS, _32BIT_POINTER, flags) == 599 NVOS46_FLAGS_32BIT_POINTER_ENABLE) 600 { 601 pLocals->vaRangeHi = NV_MIN(0xffffffff, pLocals->vaRangeHi); 602 } 603 else if (pDma->getProperty(pDma, PDB_PROP_DMA_ENFORCE_32BIT_POINTER) && 604 (pVASpaceHeap->free > NVBIT64(32))) // Pressured address spaces are exempt 605 { 606 pLocals->vaRangeLo = NV_MAX(NVBIT64(32), pLocals->vaRangeLo); 607 } 608 } 609 } 610 611 // 612 // Align the virtual address passed in down to the page size. 613 // 614 // There is no requirement that the physical offset of a mapping 615 // be page-aligned, so we need to map the entire page that contains 616 // the desired offset. We then add the page offset 617 // onto the returned virtual address. 618 // 619 pLocals->vaLo = RM_ALIGN_DOWN(*pVaddr, pLocals->pageSize); 620 621 if (pLocals->bAllocVASpace) 622 { 623 // 624 // TODO: This flag handling logic should be consolidated with dmaMapBuffer_GM107 625 // when old path removed. 626 // 627 VAS_ALLOC_FLAGS allocFlags = {0}; 628 NvU64 compAlign = NVBIT64(pLocals->comprInfo.compPageShift); 629 NvU64 vaAlign = NV_MAX(pLocals->pageSize, compAlign); 630 NvU64 vaSize = RM_ALIGN_UP(pLocals->mapLength, vaAlign); 631 NvU64 pageSizeLockMask = 0; 632 633 if (FLD_TEST_DRF(OS46, _FLAGS, _PAGE_SIZE, _BOTH, flags)) 634 { 635 vaAlign = NV_MAX(vaAlign, pLocals->vaspaceBigPageSize); 636 vaSize = RM_ALIGN_UP(pLocals->mapLength, vaAlign); 637 } 638 639 // 640 // Third party code path, nvidia_p2p_get_pages, expects on BAR1 VA to be 641 // always aligned at 64K. 642 // 643 // Also, RmMapMemory on PPC64LE expects BAR1 VA to be aligned at 64K. 644 // 645 if (pLocals->bIsBar1) 646 { 647 vaAlign = NV_MAX(vaAlign, pLocals->vaspaceBigPageSize); 648 vaSize = RM_ALIGN_UP(pLocals->mapLength, vaAlign); 649 } 650 if (FLD_TEST_DRF(OS46, _FLAGS, _DMA_OFFSET_FIXED, _TRUE, flags)) 651 { 652 pLocals->vaRangeLo = pLocals->vaLo; 653 pLocals->vaRangeHi = pLocals->vaLo + vaSize - 1; 654 if (pLocals->bIsBar1) 655 { 656 NV_RANGE requestedRange = rangeMake(pLocals->vaRangeLo, pLocals->vaRangeHi); 657 if (!rangeContains(pLocals->totalVaRange, requestedRange)) 658 { 659 NV_PRINTF(LEVEL_ERROR, "Requested BAR1 VA Lo=0x%llx Hi=0x%llx\n" 660 "total BAR1 VA range Lo=0x%llx Hi=0x%llx\n", 661 requestedRange.lo, requestedRange.hi, 662 pLocals->totalVaRange.lo, pLocals->totalVaRange.hi); 663 status = NV_ERR_INVALID_ARGUMENT; 664 DBG_BREAKPOINT(); 665 goto cleanup; 666 } 667 } 668 } 669 else if (pDma->getProperty(pDma, PDB_PROP_DMA_RESTRICT_VA_RANGE)) 670 { 671 // See comments in vaspaceFillAllocParams_IMPL. 672 pLocals->vaRangeHi = NV_MIN(pLocals->vaRangeHi, NVBIT64(40) - 1); 673 } 674 675 if (FLD_TEST_DRF(OS46, _FLAGS, _PAGE_SIZE, _BOTH, flags)) 676 { 677 NV_ASSERT(pLocals->pageSize <= pLocals->vaspaceBigPageSize); 678 pageSizeLockMask |= RM_PAGE_SIZE; 679 pageSizeLockMask |= pLocals->vaspaceBigPageSize; 680 } 681 else 682 { 683 pageSizeLockMask |= pLocals->pageSize; 684 } 685 686 allocFlags.bReverse = FLD_TEST_DRF(OS46, _FLAGS, _DMA_OFFSET_GROWS, _DOWN, flags); 687 688 // 689 // Feature requested for RM unlinked SLI: 690 // Clients can pass an allocation flag to the device or VA space constructor 691 // so that mappings and allocations will fail without an explicit address. 692 // 693 pGVAS = dynamicCast(pVAS, OBJGVASPACE); 694 if (pGVAS != NULL) 695 { 696 if ((pGVAS->flags & VASPACE_FLAGS_REQUIRE_FIXED_OFFSET) && 697 !FLD_TEST_DRF(OS46, _FLAGS, _DMA_OFFSET_FIXED, _TRUE, flags)) 698 { 699 status = NV_ERR_INVALID_ARGUMENT; 700 NV_PRINTF(LEVEL_ERROR, "The VA space requires all allocations to specify a fixed address\n"); 701 goto cleanup; 702 } 703 } 704 705 status = vaspaceAlloc(pVAS, vaSize, vaAlign, pLocals->vaRangeLo, pLocals->vaRangeHi, 706 pageSizeLockMask, allocFlags, &pLocals->vaLo); 707 if (NV_OK != status) 708 { 709 NV_PRINTF(LEVEL_ERROR, "can't alloc VA space for mapping.\n"); 710 goto cleanup; 711 } 712 NV_ASSERT_OR_ELSE(0 == (pLocals->vaLo & (pLocals->pageSize - 1)), 713 status = NV_ERR_INVALID_STATE; 714 goto cleanup; ); 715 NV_ASSERT_OR_ELSE(vaSize >= pLocals->mapLength, 716 status = NV_ERR_INVALID_STATE; 717 goto cleanup; ); 718 719 // 720 // Handle overmapping for BAR1. 721 // 722 // BAR1 VA is allocated at big page size granularity 723 // regardless of the physical memory size being mapped. 724 // Unmapped regions of BAR1 need to be mapped to dummy 725 // pages (or sparse) to avoid faults on PCIe prefetch. 726 // 727 // Overmap solves this by wrapping around the target physical 728 // memory for the remainder of the last big page so 729 // any left over 4K pages are "scratch invalidated." 730 // 731 // When this is used, the mapLength must be extended to 732 // to the entire VA range and dmaUpdateVASpace 733 // takes care of the overMap modulus. 734 // 735 // TODO: With VMM enabled BAR1 scratch invalidate is handled 736 // transparently with SW (or HW) sparse support. 737 // Removing this special overmap logic should be 738 // possible when the old VAS path is fully 739 // deprecated. 740 // 741 // See Bug 200090426. 742 // 743 if (pLocals->overMap != 0) 744 { 745 pLocals->mapLength = vaSize; 746 } 747 } 748 else 749 { 750 // 751 // We are mapping to an existing virtual memory allocation. 752 // 753 // The virtual offset passed in may or may not account for 754 // the page offset. Check for either the page-aligned case or 755 // the adjusted case to ensure clients are not requesting 756 // bogus offsets. 757 // 758 if (((*pVaddr - pLocals->vaLo) != 0) && 759 ((*pVaddr - pLocals->vaLo) != pLocals->pageOffset)) 760 { 761 NV_PRINTF(LEVEL_ERROR, 762 "Virtual address 0x%llX is not compatible with page size 0x%llX or page" 763 " offset 0x%llX.\n", *pVaddr, pLocals->pageSize, pLocals->pageOffset); 764 DBG_BREAKPOINT(); 765 status = NV_ERR_INVALID_OFFSET; 766 goto cleanup; 767 } 768 } 769 770 // 771 // Calculate mapping virtual address limit based on 772 // mapping length derived from number of physical pages going to map. 773 // 774 pLocals->vaHi = pLocals->vaLo + pLocals->mapLength - 1; 775 776 if (pLocals->p2p == NVOS46_FLAGS_P2P_ENABLE_NOSLI) 777 { 778 NV_ASSERT_OR_GOTO(pLocals->pMemory != NULL, fail_post_register); 779 780 FlaMemory *pFlaMemory = dynamicCast(pLocals->pMemory, FlaMemory); 781 if (pFlaMemory != NULL) 782 { 783 pLocals->pSrcGpu = gpumgrGetGpu(pFlaMemory->peerGpuInst); 784 pLocals->bFlaImport = NV_TRUE; 785 786 if (!pLocals->pSrcGpu) 787 { 788 NV_PRINTF(LEVEL_ERROR, "Cannot map FLA Memory without a valid srcGpu, failing....\n"); 789 status = NV_ERR_INVALID_ARGUMENT; 790 DBG_BREAKPOINT(); 791 goto fail_post_register; 792 } 793 } 794 else 795 { 796 pLocals->pSrcGpu = pLocals->pMemory->pGpu; 797 798 // XXX - is this required here if we disable SLI BC below? 799 GPU_RES_SET_THREAD_BC_STATE(pLocals->pMemory->pDevice); 800 } 801 802 if (IsSLIEnabled(pLocals->pSrcGpu)) 803 { 804 NvU32 deviceInstance = gpuGetDeviceInstance(pLocals->pSrcGpu); 805 806 pLocals->pSrcGpu = gpumgrGetGpuFromSubDeviceInst(deviceInstance, pLocals->subDevIdSrc); 807 gpumgrSetBcEnabledStatus(pLocals->pSrcGpu, NV_FALSE); 808 } 809 810 pLocals->peerNumber = kbusGetPeerId_HAL(pGpu, pKernelBus, pLocals->pSrcGpu); 811 812 // only needed pLocals->pSrcGpu for the one line above, swap back now. 813 if (IsSLIEnabled(pLocals->pSrcGpu)) 814 { 815 pLocals->pSrcGpu = gpumgrGetParentGPU(pLocals->pSrcGpu); 816 gpumgrSetBcEnabledStatus(pLocals->pSrcGpu, NV_TRUE); 817 } 818 819 NV_PRINTF(LEVEL_INFO, 820 "P2P LOOPBACK setup with physical vidmem at 0x%llx and virtual address " 821 "at 0x%llx\n", 822 memdescGetPhysAddr(pAdjustedMemDesc, addressTranslation, 0), pLocals->vaLo); 823 } 824 else if (pLocals->p2p == NVOS46_FLAGS_P2P_ENABLE_SLI) 825 { 826 // 827 // All the peer GPUs will have valid PTEs written as 828 // P2P mappings. The local GPU will have this region marked as 829 // invalid. 830 // 831 const NvU32 deviceInst = gpuGetDeviceInstance(pGpu); 832 pLocals->pSrcGpu = gpumgrGetGpuFromSubDeviceInst(deviceInst, pLocals->subDevIdSrc); 833 } 834 835 pLocals->pRootMemDesc = memdescGetRootMemDesc(pAdjustedMemDesc, NULL); 836 if (memdescGetAddressSpace(pLocals->pRootMemDesc) == ADDR_FBMEM) 837 { 838 if (gpumgrCheckIndirectPeer(pGpu, pLocals->pRootMemDesc->pGpu)) 839 { 840 pLocals->indirectPeer = DMA_UPDATE_VASPACE_FLAGS_INDIRECT_PEER; 841 } 842 } 843 844 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 845 { 846 if (pLocals->p2p) 847 { 848 if (pLocals->bFlaImport) 849 { 850 pLocals->pTempMemDesc = memdescGetMemDescFromGpu(pAdjustedMemDesc, pGpu); 851 } 852 else 853 { 854 pLocals->pTempMemDesc = memdescGetMemDescFromGpu(pAdjustedMemDesc, pLocals->pSrcGpu); 855 } 856 } 857 else 858 { 859 pLocals->pTempMemDesc = memdescGetMemDescFromGpu(pAdjustedMemDesc, pGpu); 860 } 861 862 // Commit the mapping update 863 pLocals->pPteArray = memdescGetPteArray(pLocals->pTempMemDesc, addressTranslation); 864 dmaPageArrayInit(&pLocals->pageArray, pLocals->pPteArray, pLocals->pteCount); 865 866 // Get pLocals->aperture 867 if (memdescGetAddressSpace(pLocals->pTempMemDesc) == ADDR_FBMEM) 868 { 869 if (pLocals->p2p) 870 { 871 pLocals->aperture = NV_MMU_PTE_APERTURE_PEER_MEMORY; 872 } 873 else if (pLocals->indirectPeer) 874 { 875 pLocals->aperture = NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY; 876 } 877 else 878 { 879 pLocals->aperture = NV_MMU_PTE_APERTURE_VIDEO_MEMORY; 880 } 881 } 882 else if ( 883 (memdescGetAddressSpace(pLocals->pTempMemDesc) == ADDR_FABRIC_MC) || 884 (memdescGetAddressSpace(pLocals->pTempMemDesc) == ADDR_FABRIC_V2)) 885 { 886 OBJGPU *pMappingGpu = pGpu; 887 OBJGPU *pPeerGpu; 888 pLocals->peerNumber = BUS_INVALID_PEER; 889 890 if (pLocals->pMemory == NULL) 891 { 892 status = NV_ERR_INVALID_STATE; 893 DBG_BREAKPOINT(); 894 SLI_LOOP_BREAK; 895 } 896 897 pPeerGpu = pLocals->pMemory->pGpu; 898 899 if (memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, pLocals->kind)) 900 { 901 NV_PRINTF(LEVEL_ERROR, 902 "Fabric memory should not be compressible.\n"); 903 status = NV_ERR_INVALID_STATE; 904 DBG_BREAKPOINT(); 905 SLI_LOOP_BREAK; 906 } 907 908 pLocals->aperture = NV_MMU_PTE_APERTURE_PEER_MEMORY; 909 910 if (!memIsGpuMapAllowed(pLocals->pMemory, pMappingGpu)) 911 { 912 NV_PRINTF(LEVEL_ERROR, 913 "Mapping Gpu is not attached to the given memory object\n"); 914 status = NV_ERR_INVALID_STATE; 915 DBG_BREAKPOINT(); 916 SLI_LOOP_BREAK; 917 } 918 919 if (pPeerGpu != NULL) 920 { 921 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pMappingGpu); 922 923 if ((pKernelNvlink != NULL) && 924 knvlinkIsNvlinkP2pSupported(pMappingGpu, pKernelNvlink, pPeerGpu)) 925 { 926 pLocals->peerNumber = kbusGetPeerId_HAL(pMappingGpu, GPU_GET_KERNEL_BUS(pMappingGpu), 927 pPeerGpu); 928 } 929 } 930 else 931 { 932 pLocals->peerNumber = kbusGetNvSwitchPeerId_HAL(pMappingGpu, 933 GPU_GET_KERNEL_BUS(pMappingGpu)); 934 } 935 936 if (pLocals->peerNumber == BUS_INVALID_PEER) 937 { 938 status = NV_ERR_INVALID_STATE; 939 DBG_BREAKPOINT(); 940 SLI_LOOP_BREAK; 941 } 942 } 943 else 944 { 945 // No P2P for system memory 946 if (pLocals->p2p) 947 { 948 status = NV_ERR_INVALID_ARGUMENT; 949 NV_PRINTF(LEVEL_ERROR, "No P2P for system memory.\n"); 950 SLI_LOOP_BREAK; 951 } 952 953 if (pLocals->cacheSnoop || memdescGetFlag(pAdjustedMemDesc, MEMDESC_FLAGS_MAP_SYSCOH_OVER_BAR1)) 954 { 955 pLocals->aperture = NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY; 956 } 957 else 958 { 959 pLocals->aperture = NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY; 960 } 961 } 962 963 if (pLocals->p2p == NVOS46_FLAGS_P2P_ENABLE_SLI) 964 { 965 if (pLocals->pSrcGpu == pGpu) 966 { 967 // Leave the local GPU VA range unmapped (invalid). 968 SLI_LOOP_CONTINUE; 969 } 970 else 971 { 972 pLocals->peerNumber = kbusGetPeerId_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pLocals->pSrcGpu); 973 } 974 } 975 976 if (pLocals->aperture == NV_MMU_PTE_APERTURE_PEER_MEMORY && 977 pLocals->peerNumber == BUS_INVALID_PEER) 978 { 979 status = NV_ERR_INVALID_STATE; 980 DBG_BREAKPOINT(); 981 SLI_LOOP_BREAK; 982 } 983 984 // 985 // Fabric memory descriptors are pre-encoded with the fabric base address 986 // use NVLINK_INVALID_FABRIC_ADDR to avoid encoding twice 987 // 988 if (pLocals->bFlaImport || 989 (memdescGetAddressSpace(pLocals->pTempMemDesc) == ADDR_FABRIC_MC) || 990 (memdescGetAddressSpace(pLocals->pTempMemDesc) == ADDR_FABRIC_V2)) 991 { 992 pLocals->fabricAddr = NVLINK_INVALID_FABRIC_ADDR; 993 } 994 else 995 { 996 status = _dmaGetFabricAddress(pLocals->pSrcGpu, pLocals->aperture, pLocals->kind, &pLocals->fabricAddr); 997 if (status != NV_OK) 998 { 999 DBG_BREAKPOINT(); 1000 SLI_LOOP_BREAK; 1001 } 1002 } 1003 1004 pDma = GPU_GET_DMA(pGpu); 1005 1006 status = dmaUpdateVASpace_HAL(pGpu, pDma, 1007 pVAS, 1008 pLocals->pTempMemDesc, 1009 NULL, 1010 pLocals->vaLo, pLocals->vaHi, 1011 DMA_UPDATE_VASPACE_FLAGS_UPDATE_ALL | pLocals->readOnly | pLocals->priv | 1012 pLocals->tlbLock | pLocals->shaderFlags | pLocals->disableEncryption | pLocals->indirectPeer, 1013 &pLocals->pageArray, pLocals->overMap, 1014 &pLocals->comprInfo, 1015 0, 1016 NV_MMU_PTE_VALID_TRUE, 1017 pLocals->aperture, 1018 pLocals->peerNumber, 1019 pLocals->fabricAddr, 1020 pLocals->deferInvalidate, 1021 NV_FALSE, 1022 pLocals->pageSize); 1023 if (NV_OK != status) 1024 { 1025 NV_PRINTF(LEVEL_ERROR, 1026 "can't update VA space for mapping @vaddr=0x%llx\n", 1027 pLocals->vaLo); 1028 DBG_BREAKPOINT(); 1029 SLI_LOOP_BREAK; 1030 } 1031 } 1032 SLI_LOOP_END 1033 1034 if (NV_OK == status) 1035 { 1036 // 1037 // Fill in the final virtual address of this mapping. 1038 // 1039 // This accounts for page offset for all cases, whether or not 1040 // the input *pVaddr accounted for it. 1041 // 1042 *pVaddr = pLocals->vaLo + pLocals->pageOffset; 1043 1044 // Fill in the final mapping page size for client mappings. 1045 if (pCliMapInfo != NULL) 1046 { 1047 pCliMapInfo->pDmaMappingInfo->mapPageSize = pLocals->pageSize; 1048 } 1049 1050 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 1051 // This is needed for cliDB tracking of the map. 1052 memdescSetPageSize(memdescGetMemDescFromGpu(pAdjustedMemDesc, pGpu), addressTranslation, pLocals->pageSize); 1053 SLI_LOOP_END 1054 } 1055 else 1056 { 1057 fail_post_register: 1058 if (pLocals->pMapNode) 1059 btreeUnlink(pLocals->pMapNode, &pLocals->pVASInfo->pMapTree); 1060 1061 portMemFree(pLocals->pMapNode); 1062 1063 // Only free the VA allocation if we allocated here. 1064 if (pLocals->bAllocVASpace) 1065 { 1066 vaspaceFree(pVAS, pLocals->vaLo); 1067 } 1068 } 1069 1070 cleanup: 1071 1072 if (pAdjustedMemDesc != pMemDesc) 1073 fabricvaspacePutGpaMemdesc(pFabricVAS, pAdjustedMemDesc); 1074 1075 portMemFree(pLocals); 1076 1077 return status; 1078 } 1079 1080 /*! 1081 * @brief Unmap a virtual allocation. 1082 * 1083 * For client allocations, invalidate the page tables, but don't bother freeing. 1084 * For internal allocations, free the allocation, but don't bother invalidating. 1085 * Wait, what? 1086 * 1087 * VMM-TODO: Split into two APIs - one for clients one for internal? 1088 * 1089 * @param[in] pGpu OBJGPU pointer 1090 * @param[in] pDma VirtMemAllocator pointer 1091 * @param[in] pVAS OBJVASPACE pointer 1092 * @param[in] vAddr Virtual memory base address 1093 * @param[in] pMemDesc Physical memory descriptor 1094 * @param[in] flags Unmap options 1095 * @param[in] pCliMapInfo PCLI_DMA_ALLOC_MAP_INFO pointer (for RM Client mappings) 1096 * 1097 * @returns NV_STATUS status = NV_OK on success, or status upon failure. 1098 */ 1099 NV_STATUS 1100 dmaFreeMapping_GM107 1101 ( 1102 OBJGPU *pGpu, 1103 VirtMemAllocator *pDma, 1104 OBJVASPACE *pVAS, 1105 NvU64 vAddr, 1106 MEMORY_DESCRIPTOR *pMemDesc, 1107 NvU32 flags, 1108 CLI_DMA_ALLOC_MAP_INFO *pCliMapInfo 1109 ) 1110 { 1111 VirtualMemory *pVirtualMemory = NULL; 1112 NvU32 p2p = NVOS46_FLAGS_P2P_ENABLE_NONE; 1113 NvU64 vaLo; 1114 NvU64 vaHi; 1115 NvU64 mapLength; 1116 NvU64 pageOffset; 1117 NvU64 pageSize; 1118 NvU32 deferInvalidate; 1119 NvU32 subDevIdSrc; 1120 OBJGPU *pLocalGpu = NULL; 1121 1122 NV_STATUS status = NV_OK; 1123 MEMORY_DESCRIPTOR *pTempMemDesc = NULL; 1124 1125 NV_ASSERT_OR_RETURN(NULL != pMemDesc, NV_ERR_INVALID_ARGUMENT); 1126 1127 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 1128 // ensure the page size has been set before continuing 1129 NV_ASSERT(memdescGetPageSize64(memdescGetMemDescFromGpu(pMemDesc, pGpu), VAS_ADDRESS_TRANSLATION(pVAS)) != 0); 1130 SLI_LOOP_END 1131 1132 if (pCliMapInfo) 1133 { 1134 p2p = DRF_VAL(OS46, _FLAGS, _P2P_ENABLE, pCliMapInfo->pDmaMappingInfo->Flags); 1135 subDevIdSrc = DRF_VAL(OS46, _FLAGS, _P2P_SUBDEV_ID_SRC, pCliMapInfo->pDmaMappingInfo->Flags); 1136 pVirtualMemory = pCliMapInfo->pVirtualMemory; 1137 } 1138 1139 if (p2p == NVOS46_FLAGS_P2P_ENABLE_SLI) 1140 { 1141 const NvU32 deviceInst = gpuGetDeviceInstance(pGpu); 1142 pLocalGpu = gpumgrGetGpuFromSubDeviceInst(deviceInst, subDevIdSrc); 1143 } 1144 1145 deferInvalidate = DRF_VAL(OS47, _FLAGS, _DEFER_TLB_INVALIDATION, flags) ? DMA_DEFER_TLB_INVALIDATE : DMA_TLB_INVALIDATE; 1146 1147 // Handle NV50_MEMORY_VIRTUAL use case 1148 if ((pVirtualMemory != NULL) && pVirtualMemory->bReserveVaOnAlloc) 1149 { 1150 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 1151 { 1152 if (p2p == NVOS46_FLAGS_P2P_ENABLE_SLI) 1153 { 1154 if (pLocalGpu == pGpu) 1155 { 1156 SLI_LOOP_CONTINUE; 1157 } 1158 } 1159 1160 pTempMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu); 1161 1162 NV_ASSERT_OR_RETURN(pCliMapInfo != NULL, NV_ERR_INVALID_STATE); 1163 NV_ASSERT_OR_RETURN(pCliMapInfo->pDmaMappingInfo->mapPageSize != 0, NV_ERR_INVALID_STATE); 1164 1165 pageSize = pCliMapInfo->pDmaMappingInfo->mapPageSize; 1166 pageOffset = memdescGetPhysAddr(pTempMemDesc, VAS_ADDRESS_TRANSLATION(pVAS), 0) & (pageSize - 1); 1167 mapLength = RM_ALIGN_UP(pageOffset + pTempMemDesc->Size, pageSize); 1168 vaLo = RM_ALIGN_DOWN(vAddr, pageSize); 1169 vaHi = vaLo + mapLength - 1; 1170 1171 pDma = GPU_GET_DMA(pGpu); 1172 if (vaspaceGetFlags(pVAS) & VASPACE_FLAGS_BAR_BAR1) 1173 { 1174 NV_PRINTF(LEVEL_ERROR, "Using dmaFreeMapping with sparse == False in BAR1 path!\n"); 1175 NV_ASSERT(0); 1176 return status; 1177 } 1178 1179 status = dmaUpdateVASpace_HAL(pGpu, pDma, 1180 pVAS, 1181 pTempMemDesc, 1182 NULL, 1183 vaLo, vaHi, 1184 DMA_UPDATE_VASPACE_FLAGS_UPDATE_VALID, // only change validity 1185 NULL, 0, 1186 NULL, 0, 1187 NV_MMU_PTE_VALID_FALSE, 1188 kgmmuGetHwPteApertureFromMemdesc(GPU_GET_KERNEL_GMMU(pGpu), pTempMemDesc), 0, 1189 NVLINK_INVALID_FABRIC_ADDR, 1190 deferInvalidate, 1191 NV_FALSE, 1192 pageSize); 1193 if (status != NV_OK) 1194 { 1195 NV_PRINTF(LEVEL_ERROR, "error updating VA space.\n"); 1196 vaspaceFree(pVAS, vaLo); 1197 return status; 1198 } 1199 } 1200 SLI_LOOP_END 1201 } 1202 else 1203 { 1204 vaspaceFree(pVAS, vAddr); 1205 } 1206 1207 // 1208 // invalidate any cached peer data if this memory was mapped p2p cached. 1209 // for SLI case - kmemsysCacheOp would loop through all GPUs 1210 // for non-SLI case pGpu is pointing to the P2P mapped GPU would 1211 // invalidate only on that GPU. 1212 // 1213 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY) 1214 if ((memdescGetGpuP2PCacheAttrib(memdescGetMemDescFromGpu(pMemDesc, pGpu)) == NV_MEMORY_CACHED) && 1215 (p2p != NVOS46_FLAGS_P2P_ENABLE_NONE)) 1216 { 1217 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 1218 kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, pMemDesc, FB_CACHE_PEER_MEMORY, 1219 FB_CACHE_INVALIDATE); 1220 } 1221 SLI_LOOP_END 1222 1223 return status; 1224 } 1225 1226 /*! 1227 * Defines the data needed to iterate over the last level duing map VA op. 1228 * Note: Used only in the new VMM code path. 1229 */ 1230 struct MMU_MAP_ITERATOR 1231 { 1232 /*! 1233 * @copydoc GMMU_FMT 1234 */ 1235 const GMMU_FMT *pFmt; 1236 1237 /*! 1238 * Physical aperture of the pages. 1239 */ 1240 GMMU_APERTURE aperture; 1241 1242 /*! 1243 * Opaque array of physical memory to map. Always points to 4K sized pages. 1244 */ 1245 DMA_PAGE_ARRAY *pPageArray; 1246 1247 /*! 1248 * Points to the index in pPageArray that needs to be mapped. 1249 */ 1250 NvU32 currIdx; 1251 1252 /*! 1253 * Base offset in bytes into the logical surface being mapped. 1254 */ 1255 NvU64 surfaceOffset; 1256 1257 /*! 1258 * Physical address of the last page mapped. 1259 */ 1260 NvU64 physAddr; 1261 1262 /*! 1263 * NvLink fabric address. Used for NVSwitch systems only! 1264 */ 1265 NvU64 fabricAddr; 1266 1267 /*! 1268 * @copydoc COMPR_INFO 1269 */ 1270 COMPR_INFO comprInfo; 1271 1272 1273 /*! 1274 * Non-compressed kind. 1275 */ 1276 NvU32 kindNoCompr; 1277 1278 /*! 1279 * Indicates whether compression is enabled. 1280 */ 1281 NvBool bCompr; 1282 1283 /*! 1284 * Template used to initialize the actual PTEs. Will have values that do not 1285 * change across one map operation. 1286 */ 1287 NvU8 pteTemplate[GMMU_FMT_MAX_ENTRY_SIZE] NV_ALIGN_BYTES(8); 1288 1289 /*! 1290 * The addr field that needs to be filled out, based on the 1291 * aperture. 1292 */ 1293 const GMMU_FIELD_ADDRESS *pAddrField; 1294 1295 /*! 1296 * Indicates after how many indexes in pPageArray, should the 1297 * map wrap around to the first mapped page. 1298 */ 1299 NvU32 overMapModulus; 1300 1301 /*! 1302 * Indicates to read-modify-write each PTE instead of 1303 * using the pteTemplate as the base value. 1304 */ 1305 NvBool bReadPtes; 1306 1307 /*! 1308 * Indicates to update physical address field of each PTE. 1309 */ 1310 NvBool bUpdatePhysAddr; 1311 1312 /*! 1313 * Indicates to update comptag line and kind of each PTE 1314 * that points to a compressed page. 1315 */ 1316 NvBool bUpdateCompr; 1317 1318 /*! 1319 * Indicates that we are writing PDEs for Bug 2720120. 1320 * Applicable only to GA100 1321 */ 1322 NvBool bApplyWarForBug2720120; 1323 1324 /*! 1325 * Current page table BAR2 aperture mapping (or user buffer). 1326 */ 1327 NvU8 *pMap; 1328 }; 1329 1330 static void 1331 _gmmuWalkCBMapNextEntries_Direct 1332 ( 1333 MMU_WALK_USER_CTX *pUserCtx, 1334 const MMU_MAP_TARGET *pTarget, 1335 const MMU_WALK_MEMDESC *pLevelMem, 1336 const NvU32 entryIndexLo, 1337 const NvU32 entryIndexHi, 1338 NvU32 *pProgress 1339 ) 1340 { 1341 NvU32 i; 1342 const MMU_FMT_LEVEL *pLevelFmt = pTarget->pLevelFmt; 1343 MMU_MAP_ITERATOR *pIter = pTarget->pIter; 1344 NvU8 *pMap = pIter->pMap; 1345 const NvU64 pageSize = mmuFmtLevelPageSize(pLevelFmt); 1346 NvU32 bufferAdjust = 0; 1347 GMMU_ENTRY_VALUE entry; 1348 1349 NV_ASSERT_OR_RETURN_VOID(NULL != pMap); 1350 1351 if (NULL == pLevelMem) 1352 { 1353 // 1354 // Calculate buffer adjustment to account for entryIndexLo in 1355 // buffered mode. 1356 // RM writes the user-supplied buffer starting at offset 0 1357 // even if the it is in middle of the page table. 1358 // 1359 bufferAdjust = entryIndexLo * pLevelFmt->entrySize; 1360 } 1361 1362 for (i = entryIndexLo; i <= entryIndexHi; ++i) 1363 { 1364 // Copy out current PTE if we are overwriting (Read-Modify-Write) 1365 if (pIter->bReadPtes) 1366 { 1367 portMemCopy(entry.v8, pLevelFmt->entrySize, pMap + (i * pLevelFmt->entrySize), pLevelFmt->entrySize); 1368 } 1369 else 1370 { 1371 // Copy the static fields passed in, if we aren't overwriting a subset of fields. 1372 portMemCopy(entry.v8, pLevelFmt->entrySize, pIter->pteTemplate, pLevelFmt->entrySize); 1373 } 1374 1375 if (pIter->bApplyWarForBug2720120) 1376 { 1377 // Commit to memory. 1378 portMemCopy(pMap + (i * pLevelFmt->entrySize) - bufferAdjust, pLevelFmt->entrySize, 1379 entry.v8, pLevelFmt->entrySize); 1380 continue; 1381 } 1382 1383 // Calculate the new physical address for the compression check below. 1384 if (pIter->bUpdatePhysAddr) 1385 { 1386 NvU32 currIdxMod = pIter->currIdx; 1387 1388 // Wrap the curr idx to the start offset for BAR1 overmapping. 1389 if (0 != pIter->overMapModulus) 1390 { 1391 currIdxMod %= pIter->overMapModulus; 1392 } 1393 1394 // Extract the physical address of the page to map. 1395 if (currIdxMod < pIter->pPageArray->count) 1396 { 1397 pIter->physAddr = dmaPageArrayGetPhysAddr(pIter->pPageArray, currIdxMod); 1398 // Hack to WAR submemesc mappings 1399 pIter->physAddr = NV_ALIGN_DOWN64(pIter->physAddr, pageSize); 1400 } 1401 else 1402 { 1403 // 1404 // Physically contiguous just increments physAddr 1405 // Should not be the first page (currIdxMod == 0) being mapped. 1406 // 1407 NV_ASSERT_OR_RETURN_VOID((pIter->pPageArray->count == 1) && 1408 (currIdxMod > 0)); 1409 pIter->physAddr += pageSize; 1410 } 1411 } 1412 1413 // Init comptag 1414 if (pIter->bUpdateCompr) 1415 { 1416 OBJGPU *pGpu = pUserCtx->pGpu; 1417 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1418 NvBool bCompressible = NV_TRUE; 1419 1420 // 1421 // Check if the FB physical address lands in a segment that 1422 // supports compression. 1423 // On WDDM neither RM or KMD has the physical address 1424 // information at compression allocation time. 1425 // On non-WDDM platforms, RM allocates compression before the 1426 // actual physical allocation. For non-contig allocations, the 1427 // physical pages can be spread across multiple regions 1428 // Therefore compression tags are always allocated and compression must 1429 // be disabled on a per-PTE basis at map time. 1430 // 1431 if ((pMemoryManager->Ram.numFBRegions > 1) && 1432 (gmmuFieldGetAperture(&pIter->pFmt->pPte->fldAperture, entry.v8) == 1433 GMMU_APERTURE_VIDEO)) 1434 { 1435 NvU32 iRegion; 1436 // Find the region in which the candidate block resides 1437 for (iRegion = 0; iRegion < pMemoryManager->Ram.numFBRegions; iRegion++) 1438 { 1439 // Does the block resides within this region? If so, then we are done searching. 1440 if ((pIter->physAddr >= pMemoryManager->Ram.fbRegion[iRegion].base) && 1441 (pIter->physAddr <= pMemoryManager->Ram.fbRegion[iRegion].limit)) 1442 { 1443 // Check if the region supports compression 1444 bCompressible = pMemoryManager->Ram.fbRegion[iRegion].bSupportCompressed; 1445 break; 1446 } 1447 } 1448 } 1449 1450 // 1451 // TODO: The flags that enable compression are confusing - 1452 // complicated by memsysReleaseReacquireCompr_GF100 usage. 1453 // Clean this up when removing old path and simplifying 1454 // the primitive "map" interface. 1455 // 1456 if (pIter->bCompr && bCompressible) 1457 { 1458 // 1459 // For VF, HW does 1 to 1 FB-comptag mapping. HW manages comptag 1460 // allocation, hence RM can skip the comptagline assignment to PTE. 1461 // Just updating the compressed kind is sufficient for VF. 1462 // 1463 if (!IS_VIRTUAL_WITH_SRIOV(pGpu) && pIter->pFmt->version <= GMMU_FMT_VERSION_2) 1464 { 1465 NvBool bIsWarApplied = NV_FALSE; 1466 NvU32 savedKind = pIter->comprInfo.kind; 1467 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu); 1468 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 1469 kmemsysGetStaticConfig(pGpu, pKernelMemorySystem); 1470 1471 if (pMemorySystemConfig->bUseRawModeComptaglineAllocation && 1472 pMemorySystemConfig->bDisablePlcForCertainOffsetsBug3046774 && 1473 !memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_DISALLOW_PLC, pIter->comprInfo.kind) && 1474 !kmemsysIsPagePLCable_HAL(pGpu, pKernelMemorySystem, (pIter->surfaceOffset + pIter->currIdx * pTarget->pageArrayGranularity), pageSize)) 1475 { 1476 bIsWarApplied = NV_TRUE; 1477 memmgrGetDisablePlcKind_HAL(pMemoryManager, &pIter->comprInfo.kind); 1478 } 1479 kgmmuFieldSetKindCompTags(GPU_GET_KERNEL_GMMU(pGpu), pIter->pFmt, pLevelFmt, &pIter->comprInfo, pIter->physAddr, 1480 pIter->surfaceOffset + pIter->currIdx * pTarget->pageArrayGranularity, 1481 i, entry.v8); 1482 // 1483 // restore the kind to PLC if changd, since kind is associated with entire surface, and the WAR applies to 1484 // individual pages in the surface. 1485 // 1486 if (bIsWarApplied) 1487 pIter->comprInfo.kind = savedKind; 1488 } 1489 else 1490 { 1491 nvFieldSet32(&pIter->pFmt->pPte->fldKind, pIter->comprInfo.kind, entry.v8); 1492 } 1493 } 1494 else 1495 { 1496 nvFieldSet32(&pIter->pFmt->pPte->fldKind, pIter->kindNoCompr, entry.v8); 1497 1498 if (pIter->pFmt->version <= GMMU_FMT_VERSION_2) 1499 { 1500 nvFieldSet32(&pIter->pFmt->pPte->fldCompTagLine, 0, entry.v8); 1501 if (nvFieldIsValid32(&pIter->pFmt->pPte->fldCompTagSubIndex)) 1502 { 1503 nvFieldSet32(&pIter->pFmt->pPte->fldCompTagSubIndex, 0, entry.v8); 1504 } 1505 } 1506 } 1507 } 1508 1509 // Fill the physical address field. 1510 if (pIter->bUpdatePhysAddr && (pIter->pAddrField != NULL)) 1511 { 1512 // Update the pte with the physical address 1513 gmmuFieldSetAddress(pIter->pAddrField, 1514 kgmmuEncodePhysAddr(GPU_GET_KERNEL_GMMU(pUserCtx->pGpu), pIter->aperture, pIter->physAddr, 1515 pIter->fabricAddr), 1516 entry.v8); 1517 } 1518 1519 // Commit to memory. 1520 portMemCopy(pMap + (i * pLevelFmt->entrySize) - bufferAdjust, pLevelFmt->entrySize, 1521 entry.v8, pLevelFmt->entrySize); 1522 1523 // 1524 // pPageArray deals in 4K pages. 1525 // So increment by the ratio of mapping page size to 4K 1526 // -- 1527 // The above assumption will be invalid upon implementation of memdesc dynamic page arrays 1528 // 1529 pIter->currIdx += (NvU32)(pageSize / pTarget->pageArrayGranularity); 1530 } 1531 1532 *pProgress = entryIndexHi - entryIndexLo + 1; 1533 } 1534 1535 static void 1536 _gmmuWalkCBMapNextEntries_RmAperture 1537 ( 1538 MMU_WALK_USER_CTX *pUserCtx, 1539 const MMU_MAP_TARGET *pTarget, 1540 const MMU_WALK_MEMDESC *pLevelMem, 1541 const NvU32 entryIndexLo, 1542 const NvU32 entryIndexHi, 1543 NvU32 *pProgress 1544 ) 1545 { 1546 OBJGPU *pGpu = pUserCtx->pGpu; 1547 MMU_MAP_ITERATOR *pIter = pTarget->pIter; 1548 MEMORY_DESCRIPTOR *pMemDesc = (MEMORY_DESCRIPTOR*)pLevelMem; 1549 1550 NV_PRINTF(LEVEL_INFO, "[GPU%u]: PA 0x%llX, Entries 0x%X-0x%X\n", 1551 pUserCtx->pGpu->gpuInstance, 1552 memdescGetPhysAddr(pMemDesc, AT_GPU, 0), entryIndexLo, 1553 entryIndexHi); 1554 1555 pIter->pMap = kbusMapRmAperture_HAL(pGpu, pMemDesc); 1556 NV_ASSERT_OR_RETURN_VOID(NULL != pIter->pMap); 1557 1558 _gmmuWalkCBMapNextEntries_Direct(pUserCtx, pTarget, pLevelMem, 1559 entryIndexLo, entryIndexHi, pProgress); 1560 1561 kbusUnmapRmAperture_HAL(pGpu, pMemDesc, &pIter->pMap, NV_TRUE); 1562 } 1563 1564 static NV_STATUS _dmaGetFabricAddress 1565 ( 1566 OBJGPU *pGpu, 1567 NvU32 aperture, 1568 NvU32 kind, 1569 NvU64 *fabricAddr 1570 ) 1571 { 1572 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1573 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 1574 1575 *fabricAddr = NVLINK_INVALID_FABRIC_ADDR; 1576 1577 if (pKernelNvlink == NULL) 1578 { 1579 return NV_OK; 1580 } 1581 1582 if (aperture != NV_MMU_PTE_APERTURE_PEER_MEMORY) 1583 { 1584 return NV_OK; 1585 } 1586 1587 // 1588 // Fabric address should be available for NVSwitch connected GPUs, 1589 // otherwise it is a NOP. 1590 // 1591 *fabricAddr = knvlinkGetUniqueFabricBaseAddress(pGpu, pKernelNvlink); 1592 if (*fabricAddr == NVLINK_INVALID_FABRIC_ADDR) 1593 { 1594 return NV_OK; 1595 } 1596 1597 if (memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind)) 1598 { 1599 NV_PRINTF(LEVEL_ERROR, 1600 "Nvswitch systems don't support compression.\n"); 1601 return NV_ERR_NOT_SUPPORTED; 1602 } 1603 1604 return NV_OK; 1605 } 1606 1607 // VMM-TODO: PL(N) mmuPageLevelUpdate - but major splits 1608 NV_STATUS 1609 dmaUpdateVASpace_GF100 1610 ( 1611 OBJGPU *pGpu, 1612 VirtMemAllocator *pDma, 1613 OBJVASPACE *pVAS, 1614 MEMORY_DESCRIPTOR *pMemDesc, 1615 NvU8 *pTgtPteMem, // CPU pointer to PTE memory for Vista updates 1616 NvU64 vAddr, 1617 NvU64 vAddrLimit, 1618 NvU32 flags, 1619 DMA_PAGE_ARRAY *pPageArray, 1620 NvU32 overmapPteMod, 1621 COMPR_INFO *pComprInfo, 1622 NvU64 surfaceOffset, 1623 NvU32 valid, 1624 NvU32 aperture, 1625 NvU32 peer, 1626 NvU64 fabricAddr, 1627 NvU32 deferInvalidate, 1628 NvBool bSparse, 1629 NvU64 pageSize 1630 ) 1631 { 1632 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 1633 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1634 NvBool readPte = NV_FALSE; 1635 NV_STATUS status = NV_OK; 1636 NvBool isVolatile = NV_TRUE; 1637 NvU32 encrypted = 0; 1638 NvU32 tlbLock; 1639 NvU32 readOnly; 1640 NvU32 priv; 1641 NvU32 writeDisable; 1642 NvU32 readDisable; 1643 NvU32 vaSpaceBigPageSize = 0; 1644 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = 1645 kmemsysGetStaticConfig(pGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu)); 1646 NvU32 alignSize = pMemorySystemConfig->comprPageSize; 1647 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 1648 NvBool bFillPteMem = !!(flags & DMA_UPDATE_VASPACE_FLAGS_FILL_PTE_MEM); 1649 NvBool bUnmap = !bFillPteMem && 1650 (flags & DMA_UPDATE_VASPACE_FLAGS_UPDATE_VALID) && 1651 (SF_VAL(_MMU, _PTE_VALID, valid) == NV_MMU_PTE_VALID_FALSE); 1652 NvBool bIsIndirectPeer; 1653 VAS_PTE_UPDATE_TYPE update_type; 1654 1655 priv = (flags & DMA_UPDATE_VASPACE_FLAGS_PRIV) ? NV_MMU_PTE_PRIVILEGE_TRUE : NV_MMU_PTE_PRIVILEGE_FALSE; 1656 tlbLock = (flags & DMA_UPDATE_VASPACE_FLAGS_TLB_LOCK) ? NV_MMU_PTE_LOCK_TRUE : NV_MMU_PTE_LOCK_FALSE; 1657 readOnly = (flags & DMA_UPDATE_VASPACE_FLAGS_READ_ONLY) ? NV_MMU_PTE_READ_ONLY_TRUE : NV_MMU_PTE_READ_ONLY_FALSE; 1658 writeDisable = !!(flags & DMA_UPDATE_VASPACE_FLAGS_SHADER_READ_ONLY); 1659 readDisable = !!(flags & DMA_UPDATE_VASPACE_FLAGS_SHADER_WRITE_ONLY); 1660 bIsIndirectPeer = !!(flags & DMA_UPDATE_VASPACE_FLAGS_INDIRECT_PEER); 1661 1662 NV_ASSERT_OR_RETURN(pageSize, NV_ERR_INVALID_ARGUMENT); 1663 1664 vaSpaceBigPageSize = vaspaceGetBigPageSize(pVAS); 1665 if ((pageSize == RM_PAGE_SIZE_64K) || (pageSize == RM_PAGE_SIZE_128K)) 1666 { 1667 NV_ASSERT_OR_RETURN(pageSize == vaSpaceBigPageSize, NV_ERR_INVALID_STATE); 1668 } 1669 1670 // 1671 // Determine whether we are invalidating or revoking privileges, so we know 1672 // whether to flush page accesses or not. ReadDisable and writeDisable have 1673 // been deprecated Pascal+, and we don't have the capability to guarantee 1674 // coherency post TLB invalidate on pre-Pascal, so we ignore them here. 1675 // 1676 update_type = (bUnmap || (NV_MMU_PTE_LOCK_FALSE == tlbLock) 1677 || (NV_MMU_PTE_READ_ONLY_TRUE == readOnly)) ? PTE_DOWNGRADE : PTE_UPGRADE; 1678 1679 if (pMemDesc == NULL) 1680 { 1681 NV_ASSERT(pMemDesc); 1682 return NV_ERR_INVALID_ARGUMENT; 1683 } 1684 1685 switch (aperture) 1686 { 1687 case NV_MMU_PTE_APERTURE_PEER_MEMORY: 1688 isVolatile = (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) ? NV_TRUE : NV_FALSE; 1689 break; 1690 case NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY: 1691 case NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY: 1692 if (bIsIndirectPeer) 1693 isVolatile = (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) ? NV_TRUE : NV_FALSE; 1694 else 1695 isVolatile = (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED) ? NV_TRUE : NV_FALSE; 1696 1697 break; 1698 default: 1699 case NV_MMU_PTE_APERTURE_VIDEO_MEMORY: 1700 isVolatile = NV_FALSE; 1701 break; 1702 } 1703 1704 encrypted = (flags & DMA_UPDATE_VASPACE_FLAGS_DISABLE_ENCRYPTION) ? 0 : 1705 memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ENCRYPTED); 1706 1707 // Check this here so we don't have to in the loop(s) below as necessary. 1708 if ((flags & DMA_UPDATE_VASPACE_FLAGS_UPDATE_PADDR) && (pPageArray == NULL)) 1709 { 1710 return NV_ERR_INVALID_ARGUMENT; 1711 } 1712 1713 // 1714 // Must get some attrs from existing PTE and Only if PTE is 1715 // going to be invalidated, no need to read it 1716 // 1717 if (DMA_UPDATE_VASPACE_FLAGS_UPDATE_ALL != (flags & DMA_UPDATE_VASPACE_FLAGS_UPDATE_ALL)) 1718 { 1719 readPte = !((flags & DMA_UPDATE_VASPACE_FLAGS_UPDATE_VALID) && 1720 (SF_VAL(_MMU, _PTE_VALID, valid) == NV_MMU_PTE_VALID_FALSE)); 1721 } 1722 1723 // 1724 // Compressed surfaces must be aligned to the compression page size 1725 // but don't check for LDDM which may pass in unaligned surfaces incrementally. 1726 // Chips that support full comp tag lines will support the VA being aligned to 1727 // the big page size. This is because the PA alignement chooses between even/odd pages 1728 // and SW programs the PA alignment. 1729 // 1730 if (pDma->getProperty(pDma, PDB_PROP_DMA_ENABLE_FULL_COMP_TAG_LINE)) 1731 { 1732 alignSize = vaSpaceBigPageSize; 1733 } 1734 1735 // 1736 // If we have dynamic granularity page arrays enabled we will never 1737 // encounter a case where a larger page granularity physical surface gets 1738 // represented by a smaller granularity pageArray. 1739 // 1740 if (!pMemoryManager->bEnableDynamicGranularityPageArrays) 1741 { 1742 // 1743 // VMM-TODO: Merge into PL1 traveral. 1744 // 1745 // If the pageSize of the mapping != 4K then be sure that the 4k pages 1746 // making up the big physical page are contiguous. This is currently 1747 // necessary since pMemDesc->PteArray is always in terms of 4KB pages. 1748 // Different large pages do not have to be contiguous with each other. 1749 // This check isn't needed for contig allocations. 1750 // 1751 if (pPageArray && (pageSize != RM_PAGE_SIZE) && (pPageArray->count > 1) && 1752 !(flags & DMA_UPDATE_VASPACE_FLAGS_SKIP_4K_PTE_CHECK)) 1753 { 1754 NvU32 i, j; 1755 RmPhysAddr pageAddr, pagePrevAddr; 1756 1757 for (i = 0; i < pPageArray->count; i += j) 1758 { 1759 for (j = i + 1; j < pPageArray->count; j++) 1760 { 1761 pagePrevAddr = dmaPageArrayGetPhysAddr(pPageArray, j - 1); 1762 pageAddr = dmaPageArrayGetPhysAddr(pPageArray, j); 1763 1764 if ((1 + (pagePrevAddr/(RM_PAGE_SIZE))) != 1765 (pageAddr/(RM_PAGE_SIZE))) 1766 { 1767 NV_PRINTF(LEVEL_ERROR, 1768 "MMU: given non-contig 4KB pages for %lldkB mapping\n", 1769 pageSize / 1024); 1770 DBG_BREAKPOINT(); 1771 return NV_ERR_GENERIC; 1772 } 1773 1774 // Are we at the pageSize boundary yet? 1775 if ((pageAddr + RM_PAGE_SIZE) 1776 % pageSize == 0) 1777 { 1778 j++; 1779 break; 1780 } 1781 } 1782 } 1783 } 1784 } 1785 1786 // Zero peer on non-peer requests to simplify pte construction 1787 if (aperture != NV_MMU_PTE_APERTURE_PEER_MEMORY) 1788 { 1789 peer = 0; 1790 } 1791 1792 MMU_MAP_TARGET mapTarget = {0}; 1793 MMU_MAP_ITERATOR mapIter = {0}; 1794 OBJGVASPACE *pGVAS = dynamicCast(pVAS, OBJGVASPACE); 1795 const NvU64 vaLo = NV_ALIGN_DOWN64(vAddr, pageSize); 1796 const NvU64 vaHi = NV_ALIGN_DOWN64(vAddrLimit + pageSize, pageSize) - 1; 1797 GVAS_GPU_STATE *pGpuState = gvaspaceGetGpuState(pGVAS, pGpu); 1798 const GMMU_FMT *pFmt = pGpuState->pFmt; 1799 1800 // Enforce unicast. 1801 NV_ASSERT_OR_RETURN(!gpumgrGetBcEnabledStatus(pGpu), NV_ERR_INVALID_STATE); 1802 1803 if (bUnmap) 1804 { 1805 gvaspaceUnmap(pGVAS, pGpu, vaLo, vaHi); 1806 } 1807 else 1808 { 1809 NvU32 kind = pComprInfo->kind; 1810 NvU32 kindNoCompression; 1811 1812 // 1813 // If the original kind is compressible we need to know what the non-compresible 1814 // kind is so we can fall back to that if we run out of compression tags. 1815 // 1816 if (memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, kind)) 1817 { 1818 kindNoCompression = memmgrGetUncompressedKind_HAL(pGpu, pMemoryManager, kind, NV_FALSE); 1819 } 1820 else 1821 { 1822 kindNoCompression = kind; 1823 } 1824 1825 if (!RMCFG_FEATURE_PLATFORM_WINDOWS_LDDM && 1826 memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, pComprInfo->kind) && 1827 ((vAddr & (alignSize-1)) != 0) && 1828 !(flags & DMA_UPDATE_VASPACE_FLAGS_UNALIGNED_COMP)) 1829 { 1830 return NV_ERR_INVALID_ARGUMENT; 1831 } 1832 1833 // MMU_MAP_CTX 1834 mapTarget.pLevelFmt = mmuFmtFindLevelWithPageShift(pFmt->pRoot, 1835 BIT_IDX_64(pageSize)); 1836 mapTarget.pIter = &mapIter; 1837 mapTarget.MapNextEntries = _gmmuWalkCBMapNextEntries_RmAperture; 1838 mapTarget.pageArrayGranularity = pMemDesc->pageArrayGranularity; 1839 1840 //MMU_MAP_ITER 1841 mapIter.pFmt = pFmt; 1842 mapIter.pPageArray = pPageArray; 1843 mapIter.surfaceOffset = surfaceOffset; 1844 mapIter.comprInfo = *pComprInfo; 1845 mapIter.overMapModulus = overmapPteMod; 1846 mapIter.bReadPtes = readPte; 1847 mapIter.kindNoCompr = kindNoCompression; 1848 mapIter.bCompr = memmgrIsKind_HAL(pMemoryManager, FB_IS_KIND_COMPRESSIBLE, pComprInfo->kind); 1849 mapIter.bUpdatePhysAddr = !!(flags & DMA_UPDATE_VASPACE_FLAGS_UPDATE_PADDR); 1850 mapIter.bUpdateCompr = !!(flags & DMA_UPDATE_VASPACE_FLAGS_UPDATE_COMPR); 1851 mapIter.fabricAddr = fabricAddr; 1852 1853 if ((pageSize == RM_PAGE_SIZE_512M) && kgmmuIsBug2720120WarEnabled(pKernelGmmu)) 1854 { 1855 NV_ASSERT_OK_OR_RETURN(_dmaApplyWarForBug2720120(pGVAS, pGpu, 1856 vaLo, vaHi)); 1857 } 1858 1859 // Build PTE template 1860 if (pFmt->version == GMMU_FMT_VERSION_3) 1861 { 1862 NvU32 ptePcfHw = 0; 1863 NvU32 ptePcfSw = 0; 1864 1865 // Set the new PTE PCF bits 1866 if (valid) 1867 { 1868 nvFieldSetBool(&pFmt->pPte->fldValid, NV_TRUE, 1869 mapIter.pteTemplate); 1870 nvFieldSet32(&pFmt->pPte->fldAperture._enum.desc, 1871 aperture, mapIter.pteTemplate); 1872 nvFieldSet32(&pFmt->pPte->fldPeerIndex, peer, 1873 mapIter.pteTemplate); 1874 nvFieldSet32(&pFmt->pPte->fldKind, kindNoCompression, 1875 mapIter.pteTemplate); 1876 // 1877 // Initializing the PTE V3 PCF bits whose default values are as follows: 1878 // 1.Regular vs Privilege : Regular (controlled by the priv flag) 1879 // 2.RO vs RW : RW (controlled by the readOnly flag) 1880 // 3.Atomic Enabled vs Atomic Disabled : Atomic Enabled 1881 // 4.Cached vs Uncached : Cached (controlled by the isVolatile flag) 1882 // 5.ACE vs ACD : ACD 1883 // 1884 ptePcfSw |= isVolatile ? (1 << SW_MMU_PCF_UNCACHED_IDX) : 0; 1885 ptePcfSw |= readOnly ? (1 << SW_MMU_PCF_RO_IDX) : 0; 1886 ptePcfSw |= tlbLock ? (1 << SW_MMU_PCF_NOATOMIC_IDX) : 0; 1887 ptePcfSw |= !priv ? (1 << SW_MMU_PCF_REGULAR_IDX) : 0; 1888 if ((memdescGetAddressSpace(pMemDesc) == ADDR_FABRIC_MC)) 1889 { 1890 ptePcfSw |= (1 << SW_MMU_PCF_ACE_IDX); 1891 } 1892 } 1893 else 1894 { 1895 // NV4K and NOMAPPING are not supported right now 1896 if (bSparse) 1897 { 1898 ptePcfSw |= (1 << SW_MMU_PCF_SPARSE_IDX); 1899 } 1900 else 1901 { 1902 ptePcfSw |= (1 << SW_MMU_PCF_INVALID_IDX); 1903 } 1904 } 1905 NV_CHECK_OR_RETURN(LEVEL_ERROR, 1906 (kgmmuTranslatePtePcfFromSw_HAL(pKernelGmmu, ptePcfSw, &ptePcfHw) == NV_OK), 1907 NV_ERR_INVALID_ARGUMENT); 1908 1909 nvFieldSet32(&pFmt->pPte->fldPtePcf, ptePcfHw, mapIter.pteTemplate); 1910 } 1911 else 1912 { 1913 if (bSparse) 1914 { 1915 const GMMU_FMT_FAMILY *pFmtFamily = 1916 kgmmuFmtGetFamily(pKernelGmmu, pFmt->version); 1917 NV_ASSERT_OR_RETURN(NULL != pFmtFamily, NV_ERR_INVALID_DATA); 1918 portMemCopy(mapIter.pteTemplate, 1919 mapTarget.pLevelFmt->entrySize, pFmtFamily->sparsePte.v8, 1920 mapTarget.pLevelFmt->entrySize); 1921 } 1922 else 1923 { 1924 nvFieldSetBool(&pFmt->pPte->fldValid, !!valid, 1925 mapIter.pteTemplate); 1926 nvFieldSet32(&pFmt->pPte->fldAperture._enum.desc, 1927 aperture, mapIter.pteTemplate); 1928 nvFieldSet32(&pFmt->pPte->fldPeerIndex, peer, 1929 mapIter.pteTemplate); 1930 1931 nvFieldSetBool(&pFmt->pPte->fldVolatile, !!isVolatile, 1932 mapIter.pteTemplate); 1933 nvFieldSet32(&pFmt->pPte->fldKind, kindNoCompression, 1934 mapIter.pteTemplate); 1935 nvFieldSetBool(&pFmt->pPte->fldReadOnly, !!readOnly, 1936 mapIter.pteTemplate); 1937 nvFieldSetBool(&pFmt->pPte->fldPrivilege, !!priv, 1938 mapIter.pteTemplate); 1939 nvFieldSetBool(&pFmt->pPte->fldEncrypted, !!encrypted, 1940 mapIter.pteTemplate); 1941 if (nvFieldIsValid32(&pFmt->pPte->fldReadDisable.desc)) 1942 { 1943 nvFieldSetBool(&pFmt->pPte->fldReadDisable, !!readDisable, 1944 mapIter.pteTemplate); 1945 } 1946 if (nvFieldIsValid32(&pFmt->pPte->fldWriteDisable.desc)) 1947 { 1948 nvFieldSetBool(&pFmt->pPte->fldWriteDisable, !!writeDisable, 1949 mapIter.pteTemplate); 1950 } 1951 if (nvFieldIsValid32(&pFmt->pPte->fldLocked.desc)) 1952 { 1953 nvFieldSetBool(&pFmt->pPte->fldLocked, !!tlbLock, 1954 mapIter.pteTemplate); 1955 } 1956 if (nvFieldIsValid32(&pFmt->pPte->fldAtomicDisable.desc)) 1957 { 1958 // tlbLock is overridden by atomic_disable 1959 nvFieldSetBool(&pFmt->pPte->fldAtomicDisable, !!tlbLock, 1960 mapIter.pteTemplate); 1961 } 1962 } 1963 } 1964 1965 // Extract the physical address field based on aperture. 1966 mapIter.aperture = 1967 gmmuFieldGetAperture(&pFmt->pPte->fldAperture, 1968 mapIter.pteTemplate); 1969 if (mapIter.aperture != GMMU_APERTURE_INVALID) 1970 { 1971 mapIter.pAddrField = 1972 gmmuFmtPtePhysAddrFld(pFmt->pPte, mapIter.aperture); 1973 } 1974 1975 // 1976 // FillPteMem case must be handled specially as it violates 1977 // internal VAS alignment and constistency checks. 1978 // 1979 if (bFillPteMem) 1980 { 1981 // If caller supplies buffer to write PTEs to, use that 1982 if (NULL != pTgtPteMem) 1983 { 1984 MMU_WALK_USER_CTX userCtx = {0}; 1985 NvU32 progress = 0; 1986 NvU32 entryIndexLo = mmuFmtVirtAddrToEntryIndex(mapTarget.pLevelFmt, vaLo); 1987 // Calculated to allow cross-page-table-boundary updates. 1988 NvU32 entryIndexHi = (NvU32)(vaHi >> mapTarget.pLevelFmt->virtAddrBitLo) - 1989 (NvU32)(vaLo >> mapTarget.pLevelFmt->virtAddrBitLo) + 1990 entryIndexLo; 1991 1992 // 1993 // Use pTgtPteMem directly as mapping and pass NULL memdesc to 1994 // indicate buffered mode. 1995 // 1996 userCtx.pGpu = pGpu; 1997 mapIter.pMap = pTgtPteMem; 1998 _gmmuWalkCBMapNextEntries_Direct(&userCtx, &mapTarget, NULL, 1999 entryIndexLo, entryIndexHi, &progress); 2000 NV_ASSERT(progress == entryIndexHi - entryIndexLo + 1); 2001 } 2002 // Otherwise use walker directly. 2003 else 2004 { 2005 GVAS_BLOCK *pVASBlock = NULL; 2006 EMEMBLOCK *pMemBlock = NULL; 2007 MMU_WALK_USER_CTX userCtx = {0}; 2008 2009 pMemBlock = pGVAS->pHeap->eheapGetBlock(pGVAS->pHeap, vaLo, 0); 2010 NV_ASSERT_OR_RETURN(NULL != pMemBlock, NV_ERR_INVALID_ARGUMENT); 2011 pVASBlock = pMemBlock->pData; 2012 2013 gvaspaceWalkUserCtxAcquire(pGVAS, pGpu, pVASBlock, &userCtx); 2014 status = mmuWalkMap(userCtx.pGpuState->pWalk, vaLo, vaHi, &mapTarget); 2015 NV_ASSERT(NV_OK == status); 2016 gvaspaceWalkUserCtxRelease(pGVAS, &userCtx); 2017 } 2018 } 2019 else 2020 { 2021 VAS_MAP_FLAGS mapFlags = {0}; 2022 mapFlags.bRemap = readPte || 2023 (flags & DMA_UPDATE_VASPACE_FLAGS_ALLOW_REMAP); 2024 status = gvaspaceMap(pGVAS, pGpu, vaLo, vaHi, &mapTarget, mapFlags); 2025 NV_ASSERT(NV_OK == status); 2026 } 2027 } 2028 2029 // Invalidate VAS TLB entries. 2030 if ((NULL == pTgtPteMem) && DMA_TLB_INVALIDATE == deferInvalidate) 2031 { 2032 kbusFlush_HAL(pGpu, pKernelBus, BUS_FLUSH_VIDEO_MEMORY | 2033 BUS_FLUSH_SYSTEM_MEMORY | 2034 BUS_FLUSH_USE_PCIE_READ); 2035 gvaspaceInvalidateTlb(pGVAS, pGpu, update_type); 2036 } 2037 2038 #if NV_PRINTF_LEVEL_ENABLED(LEVEL_INFO) 2039 if (DBG_RMMSG_CHECK(LEVEL_INFO)) 2040 { 2041 MMU_TRACE_ARG arg = {0}; 2042 MMU_TRACE_PARAM params = {0}; 2043 params.mode = MMU_TRACE_MODE_TRACE_VERBOSE; 2044 params.va = vAddr; 2045 params.vaLimit = vAddrLimit; 2046 params.pArg = &arg; 2047 2048 mmuTrace(pGpu, pVAS, ¶ms); 2049 } 2050 #endif 2051 return status; 2052 } 2053 2054 NV_STATUS 2055 dmaInit_GM107(OBJGPU *pGpu, VirtMemAllocator *pDma) 2056 { 2057 DMAHALINFO_FERMI *pDHPI = NULL; 2058 NvU32 data; 2059 2060 // Allocate and link in an 'info block' for this engine. 2061 if (NULL == (pDHPI = (PDMAHALINFO_FERMI)addInfoPtr(&pDma->infoList, HAL_IMPL_GF100, 2062 sizeof(DMAHALINFO_FERMI)))) 2063 { 2064 return NV_ERR_NO_MEMORY; 2065 } 2066 2067 if (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)) 2068 { 2069 pGpu->optimizeUseCaseOverride = 2070 NV_REG_STR_RM_OPTIMIZE_COMPUTE_OR_SPARSE_TEX_DEFAULT; 2071 } 2072 2073 pDHPI->vasReverse = !(!pDHPI->vasReverse); 2074 2075 pDHPI->compTagLineMultiplier = 1; 2076 2077 if (osReadRegistryDword(pGpu, NV_REG_STR_RM_RESTRICT_VA_RANGE, &data) 2078 == NV_OK) 2079 { 2080 if (NV_REG_STR_RM_RESTRICT_VA_RANGE_ON == data) 2081 { 2082 pDma->setProperty(pDma, PDB_PROP_DMA_RESTRICT_VA_RANGE, NV_TRUE); 2083 } 2084 } 2085 2086 return NV_OK; 2087 } 2088 2089 void 2090 dmaDestruct_GM107(VirtMemAllocator *pDma) 2091 { 2092 deleteInfoPtr(&pDma->infoList, HAL_IMPL_GF100); 2093 } 2094 2095 // Called when IsSLI = NV_TRUE and all linked GPUs are loaded 2096 NV_STATUS 2097 dmaStatePostLoad_GM107(OBJGPU *pGpu, VirtMemAllocator *pDma, NvU32 flags) 2098 { 2099 #ifdef DEBUG 2100 DMAHALINFO_FERMI *pDHPI = DMA_GET_FERMI_INFOBLK(pDma); 2101 DMAHALINFO_FERMI *pDHPIPeer; 2102 VirtMemAllocator *pPeerDma; 2103 2104 pPeerDma = GPU_GET_DMA(pGpu); 2105 pDHPIPeer = DMA_GET_FERMI_INFOBLK(pPeerDma); 2106 2107 // 2108 // Require these attributes to be symmetric for now. If we need to support 2109 // heterogeneous SLI across GPUs that don't match here we'll need to implement 2110 // dma[Get|Set]TunableState. 2111 // 2112 NV_ASSERT(pDHPIPeer->vasReverse == pDHPI->vasReverse); 2113 NV_ASSERT(pDHPIPeer->compTagLineMultiplier == pDHPI->compTagLineMultiplier); 2114 #endif 2115 return NV_OK; 2116 } 2117 2118 // VMM-TODO: Remove or merge with dmaAllocMapping_GF100. 2119 NV_STATUS 2120 dmaMapBuffer_GM107 2121 ( 2122 OBJGPU *pGpu, 2123 VirtMemAllocator *pDma, 2124 OBJVASPACE *pVAS, 2125 MEMORY_DESCRIPTOR *pMemDesc, 2126 NvU64 *pVaddr, 2127 NvU32 flagsForAlloc, 2128 NvU32 flagsForUpdate 2129 ) 2130 { 2131 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 2132 2133 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 2134 NvU32 kind; 2135 COMPR_INFO comprInfo; 2136 NvU32 pteCount, aperture; 2137 NvU64 mapLength; 2138 NvU64 vaddr; 2139 NV_STATUS status = NV_OK; 2140 NvU64 rangeLo = 0; 2141 NvU64 rangeHi = 0; 2142 NvU64 compAlign; 2143 NvU64 vaSize; 2144 NvU64 vaAlign; 2145 OBJEHEAP *pVASpaceHeap = vaspaceGetHeap(pVAS); 2146 NvU64 pageSize = 0; 2147 NvU64 pageSizeSubDev = 0; 2148 NvU64 pageOffs = 0; 2149 NvU64 pageOffsSubDev = 0; 2150 NvU32 flags; 2151 2152 DMA_PAGE_ARRAY pageArray; 2153 MEMORY_DESCRIPTOR *pSubDevMemDesc = NULL; 2154 VAS_ALLOC_FLAGS allocFlags = {0}; 2155 2156 NV_ASSERT(pVaddr); 2157 NV_ASSERT(pVAS); 2158 2159 // 2160 // Sets the page size for all subdevice memdescs when present. Since we don't support 2161 // different page size per subdevice, it asserts when the page size differs. 2162 // 2163 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2164 pSubDevMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu); 2165 if (memmgrSetMemDescPageSize_HAL(pGpu, pMemoryManager, pSubDevMemDesc, VAS_ADDRESS_TRANSLATION(pVAS), 2166 RM_ATTR_PAGE_SIZE_DEFAULT) != NV_OK) 2167 { 2168 SLI_LOOP_RETURN(NV_ERR_INVALID_ARGUMENT); 2169 } 2170 pageSizeSubDev = memdescGetPageSize(pSubDevMemDesc, VAS_ADDRESS_TRANSLATION(pVAS)); 2171 pageOffsSubDev = memdescGetPhysAddr(pSubDevMemDesc, VAS_ADDRESS_TRANSLATION(pVAS), 0) & 2172 (pageSizeSubDev - 1); 2173 if (0 == pageSize) 2174 { 2175 pageSize = pageSizeSubDev; 2176 pageOffs = pageOffsSubDev; 2177 } 2178 else 2179 { 2180 NV_ASSERT(pageSize == pageSizeSubDev); 2181 NV_ASSERT(pageOffs == pageOffsSubDev); 2182 } 2183 SLI_LOOP_END 2184 2185 status = memmgrGetKindComprFromMemDesc(pMemoryManager, pMemDesc, 0, &kind, &comprInfo); 2186 if (status != NV_OK) 2187 { 2188 NV_PRINTF(LEVEL_ERROR, "memmgrGetKindComprFromMemDesc failed\n"); 2189 return NV_ERR_GENERIC; 2190 } 2191 2192 if (kgmmuIsPerVaspaceBigPageEn(pKernelGmmu) && 2193 (pageSize >= RM_PAGE_SIZE_64K)) 2194 { 2195 NV_ASSERT(pageSize != RM_PAGE_SIZE_HUGE); 2196 pageSize = vaspaceGetBigPageSize(pVAS); 2197 } 2198 2199 mapLength = RM_ALIGN_UP(pageOffs + memdescGetSize(pMemDesc), pageSize); 2200 2201 vaddr = 0; 2202 compAlign = NVBIT64(comprInfo.compPageShift); 2203 vaAlign = NV_MAX(pageSize, compAlign); 2204 vaSize = RM_ALIGN_UP(mapLength, vaAlign); 2205 2206 if (flagsForAlloc & DMA_ALLOC_VASPACE_SIZE_ALIGNED) 2207 { 2208 NvU64 temp = vaSize; 2209 ROUNDUP_POW2_U64(temp); 2210 vaAlign = NV_MAX(vaAlign, temp); 2211 } 2212 2213 rangeLo = vaspaceGetVaStart(pVAS); 2214 rangeHi = vaspaceGetVaLimit(pVAS); 2215 2216 // If trying to conserve 32bit address space, map RM buffers at 4GB+ 2217 if (pDma->getProperty(pDma, PDB_PROP_DMA_ENFORCE_32BIT_POINTER) && 2218 (pVASpaceHeap->free > NVBIT64(32))) 2219 { 2220 rangeLo = NV_MAX(NVBIT64(32), rangeLo); 2221 } 2222 2223 if (flagsForAlloc & DMA_VA_LIMIT_57B) 2224 { 2225 rangeHi = NV_MIN(rangeHi, NVBIT64(57) - 1); 2226 } 2227 else if (flagsForAlloc & DMA_VA_LIMIT_49B) 2228 { 2229 rangeHi = NV_MIN(rangeHi, NVBIT64(49) - 1); 2230 } 2231 else if (pDma->getProperty(pDma, PDB_PROP_DMA_RESTRICT_VA_RANGE)) 2232 { 2233 // See comments in vaspaceFillAllocParams_IMPL. 2234 rangeHi = NV_MIN(rangeHi, NVBIT64(40) - 1); 2235 } 2236 2237 status = vaspaceAlloc(pVAS, vaSize, vaAlign, rangeLo, rangeHi, 2238 pageSize, allocFlags, &vaddr); 2239 if (status != NV_OK) 2240 { 2241 NV_PRINTF(LEVEL_ERROR, "vaspaceAlloc failed\n"); 2242 return status; 2243 } 2244 2245 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY) 2246 2247 pSubDevMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu); 2248 2249 pteCount = memdescGetContiguity(pSubDevMemDesc, VAS_ADDRESS_TRANSLATION(pVAS)) ? 1 : 2250 (NvU32)(mapLength >> RM_PAGE_SHIFT); 2251 2252 dmaPageArrayInit(&pageArray, 2253 memdescGetPteArray(pSubDevMemDesc, VAS_ADDRESS_TRANSLATION(pVAS)), 2254 pteCount); 2255 flags = flagsForUpdate; 2256 flags |= memdescGetFlag(pSubDevMemDesc, MEMDESC_FLAGS_GPU_PRIVILEGED) ? 2257 DMA_UPDATE_VASPACE_FLAGS_PRIV : 0; 2258 2259 if (memdescGetAddressSpace(pSubDevMemDesc) == ADDR_FBMEM) 2260 { 2261 aperture = NV_MMU_PTE_APERTURE_VIDEO_MEMORY; 2262 } 2263 else if (memdescGetCpuCacheAttrib(pSubDevMemDesc) == NV_MEMORY_CACHED) 2264 { 2265 aperture = NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY; 2266 } 2267 else 2268 { 2269 aperture = NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY; 2270 } 2271 2272 status = dmaUpdateVASpace_HAL(pGpu, pDma, pVAS, 2273 pSubDevMemDesc, 2274 NULL, 2275 vaddr, vaddr + mapLength - 1, 2276 flags | DMA_UPDATE_VASPACE_FLAGS_UPDATE_ALL, 2277 &pageArray, 0, &comprInfo, 2278 0, 2279 NV_MMU_PTE_VALID_TRUE, 2280 aperture, 0, 2281 NVLINK_INVALID_FABRIC_ADDR, 2282 NV_FALSE, NV_FALSE, 2283 pageSize); 2284 2285 if (status != NV_OK) 2286 { 2287 SLI_LOOP_BREAK; 2288 } 2289 2290 SLI_LOOP_END 2291 2292 if (status != NV_OK) 2293 { 2294 NV_PRINTF(LEVEL_ERROR, "dmaUpdateVASpace_GF100 failed\n"); 2295 vaspaceFree(pVAS, vaddr); 2296 return NV_ERR_GENERIC; 2297 } 2298 2299 if (pVaddr) 2300 { 2301 *pVaddr = vaddr; 2302 } 2303 2304 return NV_OK; 2305 } 2306 2307 void dmaUnmapBuffer_GM107(OBJGPU *pGpu, VirtMemAllocator *pDma, OBJVASPACE *pVAS, NvU64 vaddr) 2308 { 2309 NV_ASSERT_OR_RETURN_VOID(NULL != pVAS); 2310 2311 vaspaceFree(pVAS, vaddr); 2312 } 2313 2314 #ifdef DEBUG 2315 /* 2316 * These routines are not used by the RM proper. They are meant to be used by by 2317 * external debuggers. Because of this we do not have have a global prototype. 2318 */ 2319 NvU32 _mmuReadFb32(OBJGPU *pGpu, RmPhysAddr addr, NvU32 aperture); 2320 void _mmuWriteFb32(OBJGPU *pGpu, RmPhysAddr addr, NvU32 data, NvU32 aperture); 2321 2322 NvU32 _mmuReadFb32(OBJGPU *pGpu, RmPhysAddr addr, NvU32 aperture) 2323 { 2324 MEMORY_DESCRIPTOR memDesc = {0}; 2325 NvU8 *pOffset = NULL; 2326 NvU32 data = 0; 2327 2328 if (aperture == 0) 2329 aperture = ADDR_FBMEM; 2330 memdescCreateExisting(&memDesc, pGpu, 4, aperture, NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE); 2331 memdescDescribe(&memDesc, aperture, addr, 4); // Note that this will probably fail with MODS/sysmem 2332 pOffset = kbusMapRmAperture_HAL(pGpu, &memDesc); 2333 if (pOffset == NULL) 2334 { 2335 NV_ASSERT(pOffset != NULL); 2336 goto _mmuReadFb32_failed; 2337 } 2338 2339 data = MEM_RD32(pOffset); 2340 2341 kbusUnmapRmAperture_HAL(pGpu, &memDesc, &pOffset, NV_TRUE); 2342 _mmuReadFb32_failed: 2343 memdescDestroy(&memDesc); 2344 2345 return data; 2346 } 2347 2348 void _mmuWriteFb32(OBJGPU *pGpu, RmPhysAddr addr, NvU32 data, NvU32 aperture) 2349 { 2350 MEMORY_DESCRIPTOR memDesc = {0}; 2351 NvU8 *pOffset = NULL; 2352 2353 if (aperture == 0) 2354 aperture = ADDR_FBMEM; 2355 memdescCreateExisting(&memDesc, pGpu, 4, aperture, NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE); 2356 memdescDescribe(&memDesc, aperture, addr, 4); // Note that this will probably fail with MODS/sysmem 2357 pOffset = kbusMapRmAperture_HAL(pGpu, &memDesc); 2358 if (pOffset == NULL) 2359 { 2360 NV_ASSERT(pOffset != NULL); 2361 goto _mmuWriteFb32_failed; 2362 } 2363 2364 MEM_WR32(pOffset, data); 2365 2366 kbusUnmapRmAperture_HAL(pGpu, &memDesc, &pOffset, NV_TRUE); 2367 _mmuWriteFb32_failed: 2368 memdescDestroy(&memDesc); 2369 } 2370 2371 #endif // DEBUG 2372 2373 //-------------------------------------------------------------------------------- 2374 // dmaXlateVAtoPAforChannel_GM107 - this function translates virtual address 2375 // to physical address through page table walk for a given channel id. 2376 // 2377 // Returns NV_OK if translation was successful, NV_ERR_GENERIC otherwise. 2378 // 2379 // Output parameters: 2380 // pAddr - physical address 2381 // memType - memory type where this physical address belongs to 2382 // (ADDR_SYSMEM or ADDR_FBMEM) 2383 // 2384 //-------------------------------------------------------------------------------- 2385 NV_STATUS 2386 dmaXlateVAtoPAforChannel_GM107 2387 ( 2388 OBJGPU *pGpu, 2389 VirtMemAllocator *pDma, 2390 KernelChannel *pKernelChannel, 2391 NvU64 vAddr, 2392 NvU64 *pAddr, 2393 NvU32 *memType 2394 ) 2395 { 2396 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT); 2397 NV_ASSERT_OR_RETURN(pAddr != NULL, NV_ERR_INVALID_ARGUMENT); 2398 NV_ASSERT_OR_RETURN(memType != NULL, NV_ERR_INVALID_ARGUMENT); 2399 2400 MMU_TRACE_ARG arg = {0}; 2401 MMU_TRACE_PARAM params = {0}; 2402 NV_STATUS status; 2403 2404 params.mode = MMU_TRACE_MODE_TRANSLATE; 2405 params.va = vAddr; 2406 params.vaLimit = vAddr; 2407 params.pArg = &arg; 2408 2409 status = mmuTrace(pGpu, pKernelChannel->pVAS, ¶ms); 2410 if (status == NV_OK) 2411 { 2412 *memType = arg.aperture; 2413 *pAddr = arg.pa; 2414 } 2415 2416 return status; 2417 } 2418 2419 static NV_STATUS 2420 _dmaApplyWarForBug2720120 2421 ( 2422 OBJGVASPACE *pGVAS, 2423 OBJGPU *pGpu, 2424 const NvU64 vaLo, 2425 const NvU64 vaHi 2426 ) 2427 { 2428 KernelGmmu *pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 2429 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); 2430 GVAS_GPU_STATE *pGpuState = gvaspaceGetGpuState(pGVAS, pGpu); 2431 const GMMU_FMT *pFmt = pGpuState->pFmt; 2432 const GMMU_FMT_FAMILY *pFmtFamily = kgmmuFmtGetFamily(pKernelGmmu, pFmt->version); 2433 GVAS_BLOCK *pVASBlock = NULL; 2434 EMEMBLOCK *pMemBlock = NULL; 2435 MMU_WALK_USER_CTX userCtx = {0}; 2436 MMU_MAP_TARGET mapTarget = {0}; 2437 MMU_MAP_ITERATOR mapIter = {0}; 2438 2439 // MMU_MAP_CTX 2440 mapTarget.pLevelFmt = mmuFmtFindLevelWithPageShift(pFmt->pRoot, 29); 2441 mapTarget.pIter = &mapIter; 2442 mapTarget.MapNextEntries = _gmmuWalkCBMapNextEntries_RmAperture; 2443 2444 //MMU_MAP_ITER 2445 mapIter.pFmt = pFmt; 2446 mapIter.bApplyWarForBug2720120 = NV_TRUE; 2447 2448 // Copy the template 2449 portMemCopy(mapIter.pteTemplate, 2450 mapTarget.pLevelFmt->entrySize, pFmtFamily->bug2720120WarPde1.v8, 2451 mapTarget.pLevelFmt->entrySize); 2452 2453 pMemBlock = pGVAS->pHeap->eheapGetBlock(pGVAS->pHeap, vaLo, 0); 2454 NV_ASSERT_OR_RETURN(pMemBlock != NULL, NV_ERR_INVALID_ARGUMENT); 2455 pVASBlock = pMemBlock->pData; 2456 2457 gvaspaceWalkUserCtxAcquire(pGVAS, pGpu, pVASBlock, &userCtx); 2458 NV_ASSERT_OK_OR_RETURN(mmuWalkMap(userCtx.pGpuState->pWalk, 2459 vaLo, vaHi, &mapTarget)); 2460 gvaspaceWalkUserCtxRelease(pGVAS, &userCtx); 2461 2462 // Flush PTE writes to vidmem and issue TLB invalidate 2463 kbusFlush_HAL(pGpu, pKernelBus, BUS_FLUSH_VIDEO_MEMORY | 2464 BUS_FLUSH_SYSTEM_MEMORY | 2465 BUS_FLUSH_USE_PCIE_READ); 2466 gvaspaceInvalidateTlb(pGVAS, pGpu, PTE_UPGRADE); 2467 2468 return NV_OK; 2469 } 2470 2471 NV_STATUS 2472 dmaInitGart_GM107(OBJGPU *pGpu, VirtMemAllocator *pDma) 2473 { 2474 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); 2475 pDma->gpuGartCaps = DMA_GPU_GART_CAPS_NOSNOOP; 2476 2477 if ((pKernelBif != NULL) && FLD_TEST_REF(BIF_DMA_CAPS_SNOOP, _CTXDMA, 2478 kbifGetDmaCaps(pGpu, pKernelBif))) 2479 { 2480 pDma->gpuGartCaps |= DMA_GPU_GART_CAPS_SNOOP; 2481 } 2482 2483 return NV_OK; 2484 } 2485 2486 /*! 2487 * @brief This function returns the size of a large page 2488 * 2489 * @param[in] pGpu OBJGPU pointer 2490 * 2491 * @returns The size of GPU PTE 2492 */ 2493 NvU32 2494 dmaGetPTESize_GM107(OBJGPU *pGpu, VirtMemAllocator *pDma) 2495 { 2496 return NV_MMU_PTE__SIZE; 2497 } 2498