1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /*!
25 * @file
26 * @brief Memory descriptor handling utility routines.
27 */
28
29 #include "gpu/mem_mgr/mem_desc.h"
30
31 #include "gpu/bif/kernel_bif.h"
32
33 #include "os/os.h"
34
35 #include "gpu_mgr/gpu_mgr.h"
36 #include "core/locks.h"
37 #include "mem_mgr/io_vaspace.h"
38 #include "mem_mgr/virt_mem_mgr.h"
39 #include "core/system.h"
40 #include "vgpu/vgpu_util.h"
41 #include "platform/sli/sli.h"
42
43 #include "gpu/mem_mgr/virt_mem_allocator.h"
44
45 #include "rmconfig.h"
46 #include "vgpu/rpc.h"
47 #include "mem_mgr/mem.h"
48
49 #include "gpu/mem_mgr/mem_mgr.h"
50 #include "gpu/mem_mgr/mem_utils.h"
51
52 #include "gpu/mem_mgr/heap.h"
53
54 #include "gpu/mem_sys/kern_mem_sys.h"
55 #include "mem_mgr/video_mem.h"
56
57 #include "mem_mgr/ctx_buf_pool.h"
58
59 #include "nvrm_registry.h" // For memdescOverrideInstLoc*()
60
61 #include "deprecated/rmapi_deprecated.h"
62 #include "rmapi/rmapi.h"
63 #include "rmapi/rs_utils.h"
64 #include "class/cl0071.h" // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR
65
66 #include "gpu/bus/kern_bus.h"
67
68 // Structure for keeping track of BAR1 mappings
69 typedef struct
70 {
71 NvU64 FbAperture;
72 NvU64 FbApertureLen;
73 NvP64 pPriv;
74 } FB_MAPPING_INFO;
75
76 //
77 // Common address space lists
78 //
79 const NV_ADDRESS_SPACE ADDRLIST_FBMEM_PREFERRED[] = {ADDR_FBMEM, ADDR_SYSMEM, ADDR_UNKNOWN};
80 const NV_ADDRESS_SPACE ADDRLIST_SYSMEM_PREFERRED[] = {ADDR_SYSMEM, ADDR_FBMEM, ADDR_UNKNOWN};
81 const NV_ADDRESS_SPACE ADDRLIST_FBMEM_ONLY[] = {ADDR_FBMEM, ADDR_UNKNOWN};
82 const NV_ADDRESS_SPACE ADDRLIST_SYSMEM_ONLY[] = {ADDR_SYSMEM, ADDR_UNKNOWN};
83
84 // XXX These could probably encode the whole list in the u32 bits.
memdescAddrSpaceListToU32(const NV_ADDRESS_SPACE * addrlist)85 NvU32 memdescAddrSpaceListToU32(const NV_ADDRESS_SPACE *addrlist)
86 {
87 if (addrlist == ADDRLIST_FBMEM_PREFERRED)
88 return 1;
89 else if (addrlist == ADDRLIST_SYSMEM_PREFERRED)
90 return 2;
91 else if (addrlist == ADDRLIST_FBMEM_ONLY)
92 return 3;
93 else if (addrlist == ADDRLIST_SYSMEM_ONLY)
94 return 4;
95 else
96 return 0;
97 }
98
memdescU32ToAddrSpaceList(NvU32 index)99 const NV_ADDRESS_SPACE *memdescU32ToAddrSpaceList(NvU32 index)
100 {
101 switch (index)
102 {
103 case 1: return ADDRLIST_FBMEM_PREFERRED;
104 case 2: return ADDRLIST_SYSMEM_PREFERRED;
105 case 3: return ADDRLIST_FBMEM_ONLY;
106 case 4: return ADDRLIST_SYSMEM_ONLY;
107 default:
108 return NULL;
109 }
110 }
111
112 /*
113 * @brief Setting a MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE has to initialize
114 * pHeap and bUsingSubAllocator flags
115 */
_memdescSetSubAllocatorFlag(OBJGPU * pGpu,PMEMORY_DESCRIPTOR pMemDesc,NvBool bSet)116 static NV_STATUS _memdescSetSubAllocatorFlag
117 (
118 OBJGPU *pGpu,
119 PMEMORY_DESCRIPTOR pMemDesc,
120 NvBool bSet
121 )
122 {
123 NV_ASSERT_OR_RETURN(pGpu != NULL, NV_ERR_INVALID_ARGUMENT);
124
125 if (!bSet)
126 {
127 NV_PRINTF(LEVEL_ERROR,
128 "Unsetting MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE not supported\n");
129 NV_ASSERT(0);
130 return NV_ERR_INVALID_ARGUMENT;
131 }
132
133 NV_ASSERT(!(pMemDesc->_flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL));
134
135 // Set flag forcing the allocation to fall into suballocator
136 pMemDesc->_flags |= MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
137
138 {
139 Heap *pHeap = pMemDesc->pHeap;
140 NvBool bForceSubheap = NV_FALSE;
141
142 NV_ASSERT(pHeap == NULL || pHeap->heapType == HEAP_TYPE_PHYS_MEM_SUBALLOCATOR);
143
144 if (pMemDesc->_flags & MEMDESC_FLAGS_FORCE_ALLOC_FROM_SUBHEAP)
145 {
146 bForceSubheap = NV_TRUE;
147 }
148
149 if (pHeap == NULL)
150 pHeap = memmgrGetDeviceSuballocator(GPU_GET_MEMORY_MANAGER(pGpu), bForceSubheap);
151
152 NV_ASSERT_OR_RETURN(pHeap != NULL, NV_ERR_INVALID_STATE);
153
154 if (pHeap->heapType == HEAP_TYPE_PHYS_MEM_SUBALLOCATOR)
155 {
156 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &pMemDesc->gfid));
157 pMemDesc->bUsingSuballocator = NV_TRUE;
158 }
159 }
160
161 return NV_OK;
162 }
163
164 /*!
165 * @brief Initializing GFID for guest allocated memdescs
166 */
_memdescSetGuestAllocatedFlag(OBJGPU * pGpu,PMEMORY_DESCRIPTOR pMemDesc,NvBool bSet)167 static NV_STATUS _memdescSetGuestAllocatedFlag
168 (
169 OBJGPU *pGpu,
170 PMEMORY_DESCRIPTOR pMemDesc,
171 NvBool bSet
172 )
173 {
174 // for VGPU header scrubbing in Open Orin package
175
176 NV_ASSERT_OR_RETURN(pGpu != NULL, NV_ERR_INVALID_ARGUMENT);
177
178 if (!bSet)
179 {
180 NV_PRINTF(LEVEL_ERROR,
181 "Unsetting MEMDESC_FLAGS_GUEST_ALLOCATED not supported\n");
182 NV_ASSERT(0);
183 return NV_ERR_INVALID_ARGUMENT;
184 }
185
186 NV_ASSERT_OK_OR_RETURN(vgpuGetCallingContextGfid(pGpu, &pMemDesc->gfid));
187 pMemDesc->_flags |= MEMDESC_FLAGS_GUEST_ALLOCATED;
188
189 return NV_OK;
190 }
191
192 /*!
193 * @brief Allocate and initialize a new empty memory descriptor
194 *
195 * Allocate a new memory descriptor. This allocates the memory descriptor
196 * only. memdescAlloc or memdescDescribe are later used to allocate or associate
197 * memory to the memory descriptor.
198 *
199 * This routine takes size and the physical contiguous of the future allocation
200 * in order to size the PTE array for non-contiguous requests.
201 *
202 * memdescDestroy should be called to free a memory descriptor.
203 *
204 * If MEMDESC_FLAGS_PRE_ALLOCATED is specified, use the memory descriptor
205 * supplied by the client instead of allocating a new one.
206 *
207 * @param[out] ppMemDesc Return pointer to new memory descriptor
208 * @param[in] pGpu
209 * @param[in] Size Size of memory descriptor in bytes.
210 * @param[in] PhysicallyContiguous Need physical contig or can it be scattered?
211 * @param[in] AddressSpace NV_ADDRESS_SPACE requested
212 * @param[in] CpuCacheAttrib CPU cacheability requested
213 * @param[in] Flags MEMDESC_FLAGS_*
214 *
215 * @returns NV_OK on success
216 */
217 NV_STATUS
memdescCreate(MEMORY_DESCRIPTOR ** ppMemDesc,OBJGPU * pGpu,NvU64 Size,NvU64 Alignment,NvBool PhysicallyContiguous,NV_ADDRESS_SPACE AddressSpace,NvU32 CpuCacheAttrib,NvU64 Flags)218 memdescCreate
219 (
220 MEMORY_DESCRIPTOR **ppMemDesc,
221 OBJGPU *pGpu,
222 NvU64 Size,
223 NvU64 Alignment,
224 NvBool PhysicallyContiguous,
225 NV_ADDRESS_SPACE AddressSpace,
226 NvU32 CpuCacheAttrib,
227 NvU64 Flags
228 )
229 {
230 MEMORY_DESCRIPTOR *pMemDesc;
231 NvU64 allocSize, MdSize, PageCount;
232 NvU32 gpuCacheAttrib = NV_MEMORY_UNCACHED;
233 NV_STATUS status = NV_OK;
234
235
236 allocSize = Size;
237
238 //
239 // this memdesc may have gotten forced to sysmem if no carveout,
240 // but for VPR it needs to be in vidmem, so check and re-direct here,
241 // unless running with zero-FB
242 //
243 if ((AddressSpace != ADDR_UNKNOWN) &&
244 (Flags & MEMDESC_ALLOC_FLAGS_PROTECTED) &&
245 (!pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) ||
246 gpuIsCacheOnlyModeEnabled(pGpu)))
247 {
248 AddressSpace = ADDR_FBMEM;
249 }
250
251 if (pGpu != NULL)
252 {
253 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
254
255 if (((AddressSpace == ADDR_SYSMEM) || (AddressSpace == ADDR_UNKNOWN)) &&
256 !(Flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL))
257 {
258 if (pMemoryManager && pMemoryManager->sysmemPageSize)
259 {
260 allocSize = RM_ALIGN_UP(allocSize, pMemoryManager->sysmemPageSize);
261 if (allocSize < Size)
262 {
263 return NV_ERR_INVALID_ARGUMENT;
264 }
265 }
266 }
267
268 if (RMCFG_FEATURE_PLATFORM_MODS || IsT194(pGpu) || IsT234(pGpu))
269 {
270 if ( (AddressSpace == ADDR_FBMEM) &&
271 !(Flags & MEMDESC_ALLOC_FLAGS_PROTECTED) &&
272 memmgrGetUsableMemSizeMB_HAL(pGpu, pMemoryManager) == 0 &&
273 gpuIsUnifiedMemorySpaceEnabled(pGpu))
274 {
275 // On Tegra, force sysmem if carveout and SMMU are not available
276 AddressSpace = ADDR_SYSMEM;
277 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_IS_ALL_INST_IN_SYSMEM))
278 {
279 CpuCacheAttrib = pGpu->instCacheOverride;
280 }
281 }
282
283 //
284 // Support for aligned contiguous SYSMEM allocations.
285 //
286 if ((AddressSpace == ADDR_SYSMEM || AddressSpace == ADDR_UNKNOWN) &&
287 PhysicallyContiguous && (Alignment > RM_PAGE_SIZE))
288 {
289 if (!portSafeAddU64(allocSize, (Alignment - RM_PAGE_SIZE), &allocSize))
290 {
291 return NV_ERR_INVALID_ARGUMENT;
292 }
293 }
294 }
295 }
296
297 //
298 //
299 // Note that we allocate one extra PTE, since we don't know what the PteAdjust
300 // is yet; if the PteAdjust is zero, we simply won't use it. This is in the
301 // MEMORY_DESCRIPTOR structure definition.
302 //
303 // RM_PAGE_SIZE is 4k and RM_PAGE_SHIFT is 12, so shift operation can be
304 // modified from ((allocSize + RM_PAGE_SIZE-1) >> RM_PAGE_SHIFT) to below as
305 // (4k >> 12 = 1). This modification helps us to avoid overflow of variable
306 // allocSize, in case caller of this function passes highest value of NvU64.
307 //
308 // If allocSize is passed as 0, PageCount should be returned as 0.
309 //
310 if (allocSize == 0)
311 {
312 PageCount = 0;
313 }
314 else
315 {
316 PageCount = ((allocSize - 1) >> RM_PAGE_SHIFT) + 1;
317 }
318
319 if (PhysicallyContiguous)
320 {
321 MdSize = sizeof(MEMORY_DESCRIPTOR);
322 }
323 else
324 {
325 MdSize = sizeof(MEMORY_DESCRIPTOR) +
326 (sizeof(RmPhysAddr) * PageCount);
327 NV_ASSERT(MdSize <= 0xffffffffULL);
328 if (MdSize > 0xffffffffULL)
329 return NV_ERR_INSUFFICIENT_RESOURCES;
330 }
331
332 if (Flags & MEMDESC_FLAGS_PAGED_SYSMEM)
333 {
334 // The flag MEMDESC_FLAGS_PAGED_SYSMEM is only for Windows
335 return NV_ERR_NOT_SUPPORTED;
336 }
337
338 if (Flags & MEMDESC_FLAGS_PRE_ALLOCATED)
339 {
340 // Only fixed sized memDesc can be supported
341 if (PhysicallyContiguous == NV_FALSE)
342 {
343 return NV_ERR_BUFFER_TOO_SMALL;
344 }
345
346 NV_ASSERT_OR_RETURN(*ppMemDesc, NV_ERR_NOT_SUPPORTED);
347
348 pMemDesc = *ppMemDesc;
349 }
350 else
351 {
352 pMemDesc = portMemAllocNonPaged((NvU32)MdSize);
353 if (pMemDesc == NULL)
354 {
355 return NV_ERR_NO_MEMORY;
356 }
357 }
358
359 portMemSet(pMemDesc, 0, (NvU32)MdSize);
360
361 // Fill in initial non-zero parameters
362 pMemDesc->pGpu = pGpu;
363 pMemDesc->Size = Size;
364 pMemDesc->PageCount = PageCount;
365 pMemDesc->ActualSize = allocSize;
366 pMemDesc->_addressSpace = AddressSpace;
367 pMemDesc->RefCount = 1;
368 pMemDesc->DupCount = 1;
369 pMemDesc->_subDeviceAllocCount = 1;
370 pMemDesc->_flags = Flags;
371 pMemDesc->_gpuCacheAttrib = gpuCacheAttrib;
372 pMemDesc->_gpuP2PCacheAttrib = NV_MEMORY_UNCACHED;
373 pMemDesc->Alignment = Alignment;
374 pMemDesc->gfid = GPU_GFID_PF;
375 pMemDesc->bUsingSuballocator = NV_FALSE;
376 pMemDesc->bDeferredFree = NV_FALSE;
377 pMemDesc->numaNode = NV0000_CTRL_NO_NUMA_NODE;
378
379 pMemDesc->libosRegionHandle = 0;
380 pMemDesc->baseVirtualAddress = 0;
381
382 // parameter to determine page granularity
383 pMemDesc->pageArrayGranularity = RM_PAGE_SIZE;
384
385 memdescSetCpuCacheAttrib(pMemDesc, CpuCacheAttrib);
386
387 // Set any additional flags
388 pMemDesc->_flags |= MEMDESC_FLAGS_KERNEL_MODE;
389 if (PhysicallyContiguous)
390 pMemDesc->_flags |= MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS;
391 else
392 pMemDesc->_flags &= ~MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS;
393
394 // OBJHEAP may not be created at this time and pMemDesc->pHeap may be NULL after this if-else
395 if (Flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL)
396 {
397 pMemDesc->_flags |= MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL;
398 pMemDesc->_flags &= ~MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE;
399 }
400 else if (Flags & MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE)
401 {
402 NV_ASSERT_OK_OR_GOTO(status, _memdescSetSubAllocatorFlag(pGpu, pMemDesc, NV_TRUE), failed);
403 }
404
405 // In case of guest allocated memory, just initialize GFID
406 if (Flags & MEMDESC_FLAGS_GUEST_ALLOCATED)
407 {
408 NV_ASSERT_OK_OR_GOTO(status, _memdescSetGuestAllocatedFlag(pGpu, pMemDesc, NV_TRUE), failed);
409 }
410
411 failed:
412 if (status != NV_OK)
413 {
414 if (!(Flags & MEMDESC_FLAGS_PRE_ALLOCATED))
415 {
416 portMemFree(pMemDesc);
417 }
418 }
419 else
420 {
421 *ppMemDesc = pMemDesc;
422 }
423
424 return status;
425 }
426
427 /*!
428 * @brief Initialize an caller allocated memory descriptor
429 *
430 * Helper to make it easier to get the memDesc **, and typically used
431 * with memdescDescribe.
432 *
433 * Only can be used for physically contiguous regions with a fixed
434 * size PTE array.
435 *
436 * memdescDestroy should be called to free a memory descriptor.
437 *
438 * If MEMDESC_FLAGS_PRE_ALLOCATED is specified, use the memory descriptor
439 * supplied by the client instead of allocating a new one.
440 *
441 * @param[out] pMemDesc Return pointer to new memory descriptor
442 * @param[in] pGpu
443 * @param[in] Size Size of memory descriptor in bytes
444 * @param[in] AddressSpace NV_ADDRESS_SPACE requested
445 * @param[in] CpuCacheAttrib CPU cacheability requested
446 * @param[in] Flags MEMDESC_FLAGS_*
447 *
448 * @returns void with no malloc there should be no failure cases
449 */
450 void
memdescCreateExisting(MEMORY_DESCRIPTOR * pMemDesc,OBJGPU * pGpu,NvU64 Size,NV_ADDRESS_SPACE AddressSpace,NvU32 CpuCacheAttrib,NvU64 Flags)451 memdescCreateExisting
452 (
453 MEMORY_DESCRIPTOR *pMemDesc,
454 OBJGPU *pGpu,
455 NvU64 Size,
456 NV_ADDRESS_SPACE AddressSpace,
457 NvU32 CpuCacheAttrib,
458 NvU64 Flags
459 )
460 {
461 NV_STATUS status;
462 status = memdescCreate(&pMemDesc, pGpu, Size, 0, NV_TRUE, AddressSpace,
463 CpuCacheAttrib,
464 Flags | MEMDESC_FLAGS_PRE_ALLOCATED | MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE);
465 NV_ASSERT(status == NV_OK);
466 }
467
468
469 /*!
470 * Increment ref count
471 */
memdescAddRef(MEMORY_DESCRIPTOR * pMemDesc)472 void memdescAddRef
473 (
474 MEMORY_DESCRIPTOR *pMemDesc
475 )
476 {
477 NV_ASSERT(pMemDesc != NULL);
478 ++(pMemDesc->RefCount);
479 }
480
481 /*!
482 * Decrement ref count
483 */
memdescRemoveRef(MEMORY_DESCRIPTOR * pMemDesc)484 void memdescRemoveRef
485 (
486 MEMORY_DESCRIPTOR *pMemDesc
487 )
488 {
489 NV_ASSERT_OR_RETURN_VOID(pMemDesc != NULL);
490 --(pMemDesc->RefCount);
491 }
492
493 //
494 // Destroy all IOMMU mappings under this memdesc, including child
495 // mappings for root memdescs.
496 //
497 // TODO: merge the new IOMMU paths with the SMMU paths (see bug 1625121).
498 //
499 static void
_memdescFreeIommuMappings(PMEMORY_DESCRIPTOR pMemDesc)500 _memdescFreeIommuMappings(PMEMORY_DESCRIPTOR pMemDesc)
501 {
502 #if (RMCFG_FEATURE_PLATFORM_UNIX || RMCFG_FEATURE_PLATFORM_MODS) && !NVCPU_IS_ARM
503 PIOVAMAPPING pIovaMapping = pMemDesc->_pIommuMappings;
504
505 if (!pIovaMapping)
506 return;
507
508 if (memdescIsSubMemoryMemDesc(pMemDesc))
509 {
510 iovaMappingDestroy(pIovaMapping);
511 return;
512 }
513
514 while (pIovaMapping)
515 {
516 PIOVAMAPPING pTmpIovaMapping = pIovaMapping->pNext;
517 iovaMappingDestroy(pIovaMapping);
518 pIovaMapping = pTmpIovaMapping;
519 }
520
521 pMemDesc->_pIommuMappings = NULL;
522 #endif
523 }
524
525 /*!
526 * Destroy a memory descriptor if last reference is released
527 *
528 * If the memory descriptor is down to one reference, we need
529 * to check with the bus code check if that reference needs
530 * to be reclaimed.
531 *
532 * @param[in] pMemDesc Memory descriptor to be destroyed
533 *
534 * @returns None
535 */
536 void
memdescDestroy(MEMORY_DESCRIPTOR * pMemDesc)537 memdescDestroy
538 (
539 MEMORY_DESCRIPTOR *pMemDesc
540 )
541 {
542 // Allow null frees
543 if (!pMemDesc)
544 {
545 return;
546 }
547
548 memdescRemoveRef(pMemDesc);
549
550 // if still more references are there for pMemDesc (pMemDesc->RefCount != 0), then bail out.
551
552 if (pMemDesc->RefCount == 0)
553 {
554 MEM_DESC_DESTROY_CALLBACK *pCb = memdescGetDestroyCallbackList(pMemDesc);
555 MEM_DESC_DESTROY_CALLBACK *pNext;
556
557 if (pMemDesc->_flags & MEMDESC_FLAGS_RESTORE_PTE_KIND_ON_FREE)
558 {
559 NV_ASSERT_OK(kbusUpdateStaticBar1VAMapping_HAL(pMemDesc->pGpu,
560 GPU_GET_KERNEL_BUS(pMemDesc->pGpu), pMemDesc,
561 0, memdescGetSize(pMemDesc), NV_TRUE));
562 }
563
564 if (pMemDesc->_flags & MEMDESC_FLAGS_DUMMY_TOPLEVEL)
565 {
566 // When called from RmFreeFrameBuffer() and memdescFree could not do it because it is unallocated.
567 pMemDesc->_pNext = NULL;
568 pMemDesc->_subDeviceAllocCount = 1;
569 }
570
571 NV_ASSERT(pMemDesc->childDescriptorCnt == 0);
572 NV_ASSERT(pMemDesc->_addressSpace == ADDR_FBMEM || pMemDesc->pHeap == NULL);
573
574 //
575 // If there is private memdata, use the CB to free
576 //
577 if (pMemDesc->_pMemData && pMemDesc->_pMemDataReleaseCallback)
578 {
579 pMemDesc->_pMemDataReleaseCallback(pMemDesc);
580 }
581
582 if (pMemDesc->bDeferredFree)
583 {
584 memdescFree(pMemDesc);
585 }
586 else if (pMemDesc->Allocated != 0)
587 {
588 //
589 // The caller forgot to free the actual memory before destroying the memdesc.
590 // Please fix this by calling memdescFree().
591 // To prevent memory leaks, we explicitly free here until its fixed elsewhere.
592 //
593 NV_PRINTF(LEVEL_ERROR, "Destroying unfreed memory %p\n", pMemDesc);
594 NV_PRINTF(LEVEL_ERROR, "Please call memdescFree()\n");
595 memdescFree(pMemDesc);
596 NV_ASSERT(!pMemDesc->Allocated);
597 }
598
599 if (memdescGetStandbyBuffer(pMemDesc))
600 {
601 memdescFree(memdescGetStandbyBuffer(pMemDesc));
602 memdescDestroy(memdescGetStandbyBuffer(pMemDesc));
603 memdescSetStandbyBuffer(pMemDesc, NULL);
604 }
605
606 //
607 // Submemory descriptors will be destroyed without going through a free
608 // path, so we need to make sure that we remove the IOMMU submapping
609 // here. For root descriptors, we should already have removed all the
610 // associated IOVA mappings.
611 //
612 // However, for memory descriptors that weren't allocated by the RM,
613 // (e.g., were created from a user allocation), we won't go through a
614 // free path at all. In this case, mappings for other GPUs may still be
615 // attached to this root memory descriptor, so release them now.
616 //
617 _memdescFreeIommuMappings(pMemDesc);
618
619 // Notify all interested parties of destruction
620 while (pCb)
621 {
622 pNext = pCb->pNext;
623 pCb->destroyCallback(pMemDesc->pGpu, pCb->pObject, pMemDesc);
624 // pCb is now invalid
625 pCb = pNext;
626 }
627
628 portMemFree(pMemDesc->pPteSpaMappings);
629 pMemDesc->pPteSpaMappings = NULL;
630 portMemFree(pMemDesc->pSubMemDescList);
631 pMemDesc->pSubMemDescList = NULL;
632
633 if (pMemDesc->pPteEgmMappings != NULL)
634 {
635 portMemFree(pMemDesc->pPteEgmMappings);
636 pMemDesc->pPteEgmMappings = NULL;
637 }
638
639 if (pMemDesc->_pParentDescriptor)
640 {
641 if ((pMemDesc->_flags & MEMDESC_FLAGS_PRE_ALLOCATED) == 0)
642 pMemDesc->_pParentDescriptor->childDescriptorCnt--;
643 memdescDestroy(pMemDesc->_pParentDescriptor);
644 pMemDesc->_pParentDescriptor = NULL;
645 }
646
647 // Verify memdesc is not top
648 NV_ASSERT(memdescHasSubDeviceMemDescs(pMemDesc) == NV_FALSE);
649
650 if ((pMemDesc->_flags & MEMDESC_FLAGS_PRE_ALLOCATED) == 0)
651 {
652 portMemFree(pMemDesc);
653 }
654 }
655 }
656
657 /*!
658 * @brief Function that frees subdevice memory descriptors. If there are no
659 * subdevice memory descriptors function just simply resets memdesc structure.
660 * Top level memory descriptor is not destroyed.
661 *
662 * @param[in,out] pMemDesc Top level memory descriptor.
663 *
664 * @returns None
665 */
666 static void
_memSubDeviceFreeAndDestroy(MEMORY_DESCRIPTOR * pMemDesc)667 _memSubDeviceFreeAndDestroy
668 (
669 MEMORY_DESCRIPTOR *pMemDesc
670 )
671 {
672 MEMORY_DESCRIPTOR *pSubDevMemDesc = pMemDesc->_pNext;
673 MEMORY_DESCRIPTOR *pNextMemDesc;
674 OBJGPU *pGpu = pMemDesc->pGpu;
675 NvBool bBcState;
676
677 // No subdevice memdescs
678 if (pSubDevMemDesc == NULL || pGpu == NULL)
679 {
680 return;
681 }
682
683 bBcState = gpumgrGetBcEnabledStatus(pGpu);
684 gpumgrSetBcEnabledStatus(pGpu, NV_FALSE);
685
686 do
687 {
688 pNextMemDesc = pSubDevMemDesc->_pNext;
689 pSubDevMemDesc->_pNext = NULL;
690 memdescFree(pSubDevMemDesc);
691 memdescDestroy(pSubDevMemDesc);
692 pSubDevMemDesc = pNextMemDesc;
693 } while (pSubDevMemDesc != NULL);
694
695 gpumgrSetBcEnabledStatus(pGpu, bBcState);
696 }
697
698 /*!
699 * @brief Lower memdesc allocation layer for the special case of allocation
700 in the VPR region when MODS is managing it.
701 *
702 * @param[in] pMemDesc Memory descriptor to allocate
703 *
704 * @returns NV_OK on successful allocation.
705 * NV_ERR_NOT_SUPPORTED if not supported
706 */
707 static NV_STATUS
_memdescAllocVprRegion(MEMORY_DESCRIPTOR * pMemDesc)708 _memdescAllocVprRegion
709 (
710 MEMORY_DESCRIPTOR *pMemDesc
711 )
712 {
713 return NV_ERR_NOT_SUPPORTED;
714 }
715
716 /*!
717 * @brief Allocate and populate the EGM array based off of the already
718 * populated _pteArray of the memdesc
719 *
720 * @param[in] pMemDesc Memory descriptor to allocate EGM array in
721 *
722 * @returns NV_OK on successful allocation. NV_ERR if not.
723 */
724 static NV_INLINE NV_STATUS
_memdescAllocEgmArray(MEMORY_DESCRIPTOR * pMemDesc)725 _memdescAllocEgmArray
726 (
727 MEMORY_DESCRIPTOR *pMemDesc
728 )
729 {
730 //
731 // Since we allocate an extra PTE, we need to account for that in case we
732 // need it for the EGM array. This is why there is a '+ 1' for pageCount.
733 //
734 NvU64 i;
735 NvU64 pageCount = pMemDesc->PageCount + 1;
736
737 //
738 // Get the root memory descriptor's memory manager to be able to get the
739 // EGM base of that GPU, instead of the mapping GPU in the case of this
740 // array being used in a submemdesc. The submemdesc should always have the
741 // mapping of the root since it's a submemdesc derived from the root, and
742 // not based on the mapping GPU.
743 //
744 MEMORY_DESCRIPTOR *pRootMemDesc = memdescGetRootMemDesc(pMemDesc, NULL);
745 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pRootMemDesc->pGpu);
746
747 if (pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS)
748 {
749 pageCount = 1;
750 }
751
752 if (pMemDesc->pPteEgmMappings == NULL)
753 {
754 pMemDesc->pPteEgmMappings = portMemAllocNonPaged(sizeof(RmPhysAddr) * pageCount);
755 }
756
757 NV_ASSERT_OR_RETURN(pMemDesc->pPteEgmMappings != NULL, NV_ERR_NO_MEMORY);
758
759 for (i = 0; i < pageCount; i++)
760 {
761 pMemDesc->pPteEgmMappings[i] = pMemDesc->_pteArray[i] -
762 pMemoryManager->localEgmBasePhysAddr;
763 }
764
765 return NV_OK;
766 }
767
768 /*!
769 * @brief Lower memdesc allocation layer. Provides underlying allocation
770 * functionality.
771 *
772 * @param[in,out] pMemDesc Memory descriptor to allocate
773 *
774 * @returns NV_OK on successful allocation. Various NV_ERR_GENERIC codes otherwise.
775 */
776 static NV_STATUS
_memdescAllocInternal(MEMORY_DESCRIPTOR * pMemDesc)777 _memdescAllocInternal
778 (
779 MEMORY_DESCRIPTOR *pMemDesc
780 )
781 {
782 OBJGPU *pGpu = pMemDesc->pGpu;
783 NV_STATUS status = NV_OK;
784 FB_ALLOC_INFO *pFbAllocInfo = NULL;
785 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL;
786
787 if (pMemDesc->Allocated)
788 {
789 NV_ASSERT(!pMemDesc->Allocated);
790 return NV_ERR_INVALID_OBJECT_BUFFER;
791 }
792
793 // Special case of an allocation request in MODS managed VPR region.
794 status = _memdescAllocVprRegion(pMemDesc);
795 if (status != NV_ERR_NOT_SUPPORTED)
796 goto done;
797
798 switch (pMemDesc->_addressSpace)
799 {
800 case ADDR_EGM:
801 case ADDR_SYSMEM:
802 // System memory can be obtained from osAllocPages
803 status = osAllocPages(pMemDesc);
804 if (status != NV_OK)
805 {
806 goto done;
807 }
808 if (memdescIsEgm(pMemDesc))
809 {
810 NV_ASSERT_OK_OR_GOTO(status,
811 _memdescAllocEgmArray(pMemDesc),
812 done);
813 }
814
815 //
816 // The pages have been allocated, so mark the descriptor as
817 // allocated. The IOMMU-mapping code needs the memdesc to be
818 // allocated in order to create the mapping.
819 //
820 pMemDesc->Allocated = 1;
821
822 //
823 // TODO: merge new IOMMU paths with the SMMU paths below (see bug
824 // 1625121). For now they are parallel, and only one will be
825 // used.
826 //
827 if (!memdescGetFlag(pMemDesc, MEMDESC_FLAGS_CPU_ONLY) &&
828 !memdescIsEgm(pMemDesc) &&
829 !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_SKIP_IOMMU_MAPPING))
830 {
831 status = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
832 if (status != NV_OK)
833 {
834 pMemDesc->Allocated = 0;
835 osFreePages(pMemDesc);
836 goto done;
837 }
838 }
839
840 if (pMemDesc->_flags & MEMDESC_FLAGS_PROVIDE_IOMMU_MAP)
841 {
842 NV_PRINTF(LEVEL_ERROR, "SMMU mapping allocation is not supported for ARMv7.\n");
843 NV_ASSERT(0);
844 status = NV_ERR_NOT_SUPPORTED;
845 goto done;
846 }
847 else if ((pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS) &&
848 RMCFG_FEATURE_PLATFORM_MODS)
849 {
850 if (pMemDesc->Alignment > RM_PAGE_SIZE)
851 {
852 RmPhysAddr addr = memdescGetPhysAddr(pMemDesc, AT_CPU, 0);
853 NvU64 offset;
854
855 NV_ASSERT((addr & (RM_PAGE_SIZE - 1)) == 0);
856
857 NV_ASSERT((pMemDesc->Alignment & (pMemDesc->Alignment - 1)) == 0);
858 offset = addr & (pMemDesc->Alignment - 1);
859
860 if (offset)
861 {
862 NV_ASSERT((pMemDesc->PageCount * pMemDesc->pageArrayGranularity - pMemDesc->Size) >= offset);
863 NV_ASSERT(pMemDesc->PteAdjust == 0);
864 pMemDesc->PteAdjust += NvU64_LO32(pMemDesc->Alignment - offset);
865 }
866 }
867 }
868
869 break;
870 case ADDR_FBMEM:
871 {
872 Heap *pHeap = pMemDesc->pHeap;
873
874 if (RMCFG_FEATURE_PMA &&
875 (pMemDesc->_flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL))
876 {
877 CTX_BUF_POOL_INFO *pCtxBufPool = NULL;
878 pCtxBufPool = memdescGetCtxBufPool(pMemDesc);
879 NV_ASSERT_TRUE_OR_GOTO(status, pCtxBufPool != NULL, NV_ERR_INVALID_STATE, done);
880
881 // If pool is setup then allocate from pool
882 NV_ASSERT_OK_OR_GOTO(status, ctxBufPoolAllocate(pCtxBufPool, pMemDesc), done);
883 }
884 else
885 {
886 // XXX Hack!
887 MEMORY_ALLOCATION_REQUEST allocRequest = {0};
888 NV_MEMORY_ALLOCATION_PARAMS allocData = {0};
889 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
890 NvU64 requestedSize = pMemDesc->Size;
891
892 allocRequest.pUserParams = &allocData;
893
894 // Don't allow FB allocations if FB is broken unless running in L2 cache only mode
895 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) &&
896 !gpuIsCacheOnlyModeEnabled(pGpu))
897 {
898 DBG_BREAKPOINT();
899 status = NV_ERR_BROKEN_FB;
900 goto done;
901 }
902
903 allocData.owner = HEAP_OWNER_RM_CHANNEL_CTX_BUFFER;
904 allocData.type = NVOS32_TYPE_IMAGE;
905 allocData.flags = NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE;
906
907 // remove the "grows_down" flag when bReservedMemAtBottom is set so as to move RM memory to the bottom.
908 if (!pMemoryManager->bReservedMemAtBottom)
909 {
910 allocData.flags |= NVOS32_ALLOC_FLAGS_FORCE_MEM_GROWS_DOWN;
911 }
912
913 // Allocate in high priority memory?
914 if (pMemDesc->_flags & MEMDESC_FLAGS_HIGH_PRIORITY)
915 {
916 allocData.attr2 |= DRF_DEF(OS32, _ATTR2, _PRIORITY, _HIGH);
917 }
918 else if (pMemDesc->_flags & MEMDESC_FLAGS_LOW_PRIORITY)
919 {
920 allocData.attr2 |= DRF_DEF(OS32, _ATTR2, _PRIORITY, _LOW);
921 }
922
923 allocData.size = pMemDesc->Size;
924 allocData.alignment = pMemDesc->Alignment;
925 allocRequest.pMemDesc = pMemDesc;
926
927 if (pMemDesc->_pageSize == RM_PAGE_SIZE)
928 {
929 allocData.attr |= DRF_DEF(OS32, _ATTR, _PAGE_SIZE, _4KB);
930 }
931 else if (pMemDesc->_pageSize == RM_PAGE_SIZE_64K ||
932 pMemDesc->_pageSize == RM_PAGE_SIZE_128K)
933 {
934 allocData.attr |= DRF_DEF(OS32, _ATTR, _PAGE_SIZE, _BIG);
935 }
936
937 allocData.flags |= pMemDesc->Alignment ?
938 NVOS32_ALLOC_FLAGS_ALIGNMENT_FORCE :
939 NVOS32_ALLOC_FLAGS_FORCE_ALIGN_HOST_PAGE;
940
941 if (pMemDesc->_flags & MEMDESC_FLAGS_FIXED_ADDRESS_ALLOCATE)
942 {
943 allocData.flags |= NVOS32_ALLOC_FLAGS_FIXED_ADDRESS_ALLOCATE;
944 allocData.offset = pMemDesc->_pteArray[0];
945 }
946
947 if (pMemDesc->_gpuCacheAttrib == NV_MEMORY_CACHED)
948 {
949 allocData.attr2 |= DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _YES);
950 }
951 else
952 {
953 // Force internal allocations to uncached unless explicitly requested.
954 allocData.attr2 |= DRF_DEF(OS32, _ATTR2, _GPU_CACHEABLE, _NO);
955 }
956
957 allocData.attr2 = FLD_SET_DRF(OS32, _ATTR2, _INTERNAL, _YES, allocData.attr2);
958
959 if (pMemDesc->_flags & MEMDESC_ALLOC_FLAGS_PROTECTED)
960 {
961 allocData.flags |= NVOS32_ALLOC_FLAGS_PROTECTED;
962 }
963
964 //
965 // Assume all RM internal allocations to go into protected (CPR)
966 // video memory unless specified otherwise explicitly
967 //
968 if (gpuIsCCFeatureEnabled(pGpu))
969 {
970 if (pMemDesc->_flags & MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY)
971 {
972 //
973 // CC-TODO: Remove this check after non-CPR region is
974 // created. Not sure if RM will ever need to use non-CPR
975 // region for itself
976 //
977 NV_PRINTF(LEVEL_ERROR, "Non-CPR region still not created\n");
978 NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_ARGUMENT);
979 }
980 else
981 {
982 allocData.flags |= NVOS32_ALLOC_FLAGS_PROTECTED;
983 }
984 }
985
986 allocData.attr |= DRF_DEF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS);
987
988 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO));
989 NV_ASSERT_TRUE_OR_GOTO(status, pFbAllocInfo != NULL, NV_ERR_NO_MEMORY, done);
990
991 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT));
992 NV_ASSERT_TRUE_OR_GOTO(status, pFbAllocPageFormat != NULL, NV_ERR_NO_MEMORY, done);
993
994 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO));
995 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT));
996 pFbAllocInfo->pageFormat = pFbAllocPageFormat;
997
998 memUtilsInitFBAllocInfo(&allocData, pFbAllocInfo, 0, 0); // Client/device N/A
999
1000 status = memmgrAllocResources(pGpu, pMemoryManager, &allocRequest, pFbAllocInfo);
1001 if (status != NV_OK)
1002 goto done;
1003
1004 status = vidmemAllocResources(pGpu, pMemoryManager, &allocRequest, pFbAllocInfo, pHeap);
1005 if (status != NV_OK)
1006 goto done;
1007
1008 pMemDesc->Alignment = allocData.alignment;
1009
1010 // Update MemDesc GPU cacheability with results of allocation
1011 if (DRF_VAL(OS32, _ATTR2, _GPU_CACHEABLE, allocData.attr2) == NVOS32_ATTR2_GPU_CACHEABLE_YES)
1012 {
1013 pMemDesc->_gpuCacheAttrib = NV_MEMORY_CACHED;
1014 }
1015 else
1016 {
1017 pMemDesc->_gpuCacheAttrib = NV_MEMORY_UNCACHED;
1018 }
1019
1020 //
1021 // Adjust size to the requested size, not the heap rounded size. A number of callers
1022 // depend on this. In the future we will have the PageCount be accurate.
1023 //
1024 pMemDesc->Size = requestedSize;
1025 pMemDesc->PageCount = ((pMemDesc->Size + pMemDesc->PteAdjust + pMemDesc->pageArrayGranularity - 1) >>
1026 BIT_IDX_32(pMemDesc->pageArrayGranularity));
1027 }
1028 // We now have the memory
1029 pMemDesc->Allocated = 1;
1030
1031 // If the allocation succeeds and if its PhysMemSubAlloc, increment the refCount
1032 if ((status == NV_OK) && (pHeap != NULL) &&
1033 (pHeap->heapType == HEAP_TYPE_PHYS_MEM_SUBALLOCATOR))
1034 {
1035 heapAddRef(pHeap);
1036 }
1037 break;
1038 }
1039
1040 default:
1041 // Don't know how to do any other types of memory yet
1042 DBG_BREAKPOINT();
1043 status = NV_ERR_GENERIC;
1044 goto done;
1045 }
1046
1047 done:
1048 if (status == NV_OK)
1049 {
1050 memdescPrintMemdesc(pMemDesc, NV_TRUE, MAKE_NV_PRINTF_STR("memdesc allocated"));
1051 }
1052 else if (pMemDesc->pPteEgmMappings != NULL)
1053 {
1054 portMemFree(pMemDesc->pPteEgmMappings);
1055 pMemDesc->pPteEgmMappings = NULL;
1056 }
1057 portMemFree(pFbAllocPageFormat);
1058 portMemFree(pFbAllocInfo);
1059
1060 return status;
1061 }
1062
1063 /*!
1064 * @brief Upper memdesc allocation layer. Provides support for per-subdevice
1065 * sysmem buffers and lockless sysmem allocation.
1066 *
1067 * @param[in,out] pMemDesc Memory descriptor to allocate
1068 *
1069 * @returns NV_OK on successful allocation. Various NV_ERR_GENERIC codes otherwise.
1070 */
1071 NV_STATUS
memdescAlloc(MEMORY_DESCRIPTOR * pMemDesc)1072 memdescAlloc
1073 (
1074 MEMORY_DESCRIPTOR *pMemDesc
1075 )
1076 {
1077 OBJGPU *pGpu = pMemDesc->pGpu;
1078 NV_STATUS status = NV_OK;
1079 NvBool bcState = NV_FALSE;
1080 OBJSYS *pSys = SYS_GET_INSTANCE();
1081 NvBool reAcquire;
1082 NvU32 gpuMask = 0;
1083
1084 NV_ASSERT_OR_RETURN(!pMemDesc->Allocated, NV_ERR_INVALID_OBJECT_BUFFER);
1085
1086 switch (pMemDesc->_addressSpace)
1087 {
1088 case ADDR_SYSMEM:
1089 case ADDR_EGM:
1090 // Can't alloc sysmem on GSP firmware.
1091 if (RMCFG_FEATURE_PLATFORM_GSP && !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
1092 {
1093 //
1094 // TO DO: Make this an error once existing allocations are cleaned up.
1095 // After that pHeap selection can be moved to memdescAllocInternal()
1096 //
1097 NV_PRINTF(LEVEL_WARNING,
1098 "WARNING sysmem alloc on GSP firmware\n");
1099 pMemDesc->_addressSpace = ADDR_FBMEM;
1100 pMemDesc->pHeap = GPU_GET_HEAP(pGpu);
1101 }
1102 //
1103 // If AMD SEV is enabled but CC or APM is not enabled on the GPU,
1104 // all RM and client allocations must to to unprotected sysmem.
1105 // So, we override any unprotected/protected flag set by either RM
1106 // or client.
1107 // If APM is enabled and RM is allocating sysmem for its internal use
1108 // use such memory has to be unprotected as protected sysmem is not
1109 // accessible to GPU
1110 //
1111 if ((sysGetStaticConfig(pSys))->bOsCCEnabled)
1112 {
1113 if (!gpuIsCCorApmFeatureEnabled(pGpu) ||
1114 (gpuIsApmFeatureEnabled(pGpu) &&
1115 !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_SYSMEM_OWNED_BY_CLIENT)))
1116 {
1117 memdescSetFlag(pMemDesc,
1118 MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY, NV_TRUE);
1119 }
1120 }
1121 else
1122 {
1123 //
1124 // This flag has no meaning on non-SEV systems. So, unset it. The
1125 // OS layer currently honours this flag irrespective of whether
1126 // SEV is enabled or not
1127 //
1128 memdescSetFlag(pMemDesc,
1129 MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY, NV_FALSE);
1130 }
1131
1132 break;
1133 case ADDR_FBMEM:
1134 {
1135 //
1136 // When APM is enabled, all RM internal vidmem allocations go to
1137 // unprotected memory. There is an underlying assumption that
1138 // memdescAlloc won't be directly called in the client vidmem alloc
1139 // codepath. Note that memdescAlloc still gets called in the client
1140 // sysmem alloc codepath. See CONFCOMP-529
1141 //
1142 if (gpuIsApmFeatureEnabled(pGpu))
1143 {
1144 memdescSetFlag(pMemDesc,
1145 MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY, NV_TRUE);
1146 }
1147 // If FB is broken then don't allow the allocation, unless running in L2 cache only mode
1148 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) &&
1149 !gpuIsCacheOnlyModeEnabled(pGpu))
1150 {
1151 status = NV_ERR_BROKEN_FB;
1152 NV_PRINTF(LEVEL_ERROR, "Unsupported FB bound allocation on broken FB(0FB) platform\n");
1153 DBG_BREAKPOINT();
1154 }
1155
1156 NV_ASSERT(pMemDesc->pHeap == NULL);
1157 // Set the pHeap based on who owns this allocation
1158 if (pMemDesc->_flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL)
1159 {
1160 //
1161 // pHeap is not required in memdesc for ctx buf pools because each ctx buf
1162 // pool is tied to PMA and this pools is cached inside memdesc.
1163 //
1164 CTX_BUF_POOL_INFO *pCtxBufPool = memdescGetCtxBufPool(pMemDesc);
1165 NV_ASSERT_OR_RETURN(pCtxBufPool != NULL, NV_ERR_INVALID_STATE);
1166 }
1167 else if (pMemDesc->_flags & MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE)
1168 {
1169 NvBool bForceSubheap = NV_FALSE;
1170
1171 if (pMemDesc->_flags & MEMDESC_FLAGS_FORCE_ALLOC_FROM_SUBHEAP)
1172 {
1173 bForceSubheap = NV_TRUE;
1174 }
1175
1176 pMemDesc->pHeap = memmgrGetDeviceSuballocator(GPU_GET_MEMORY_MANAGER(pGpu), bForceSubheap);
1177 }
1178 else if (GPU_GET_MEMORY_MANAGER(pGpu) != NULL &&
1179 RMCFG_MODULE_HEAP &&
1180 pMemDesc->_addressSpace == ADDR_FBMEM)
1181 {
1182 pMemDesc->pHeap = GPU_GET_HEAP(pGpu);
1183 }
1184
1185 break;
1186 }
1187 default:
1188 // Don't know how to do any other types of memory yet
1189 DBG_BREAKPOINT();
1190 return NV_ERR_GENERIC;
1191 }
1192
1193 if (status != NV_OK)
1194 {
1195 return status;
1196 }
1197
1198 if (gpumgrGetBcEnabledStatus(pGpu))
1199 {
1200 // Broadcast memdescAlloc call with flag set to allocate per subdevice.
1201 if (pMemDesc->_flags & MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE)
1202 {
1203 NvU32 i;
1204 MEMORY_DESCRIPTOR *pSubDevMemDesc = pMemDesc;
1205 MEMORY_DESCRIPTOR *pPrev = pMemDesc;
1206 OBJGPU *pGpuChild;
1207
1208 pMemDesc->_subDeviceAllocCount = NumSubDevices(pGpu);
1209
1210 for (i = 0; i < pMemDesc->_subDeviceAllocCount; i++)
1211 {
1212 // Get pGpu for this subdeviceinst
1213 pGpuChild = gpumgrGetGpuFromSubDeviceInst(gpuGetDeviceInstance(pGpu), i);
1214 if (NULL == pGpuChild)
1215 {
1216 NV_ASSERT(0);
1217 status = NV_ERR_OBJECT_NOT_FOUND;
1218 goto subdeviceAlloc_failed;
1219 }
1220
1221 //
1222 // We are accessing the fields of the top level desc here directly without using the
1223 // accessor routines on purpose.
1224 //
1225 status = memdescCreate(&pSubDevMemDesc, pGpuChild, pMemDesc->Size, pMemDesc->Alignment,
1226 !!(pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS),
1227 pMemDesc->_addressSpace,
1228 pMemDesc->_cpuCacheAttrib,
1229 pMemDesc->_flags & ~MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE);
1230
1231 if (status != NV_OK)
1232 {
1233 NV_ASSERT(0);
1234 goto subdeviceAlloc_failed;
1235 }
1236
1237 pSubDevMemDesc->_gpuCacheAttrib = pMemDesc->_gpuCacheAttrib;
1238 pSubDevMemDesc->_pageSize = pMemDesc->_pageSize;
1239
1240 // Force broadcast state to false when allocating a subdevice memdesc
1241 gpumgrSetBcEnabledStatus(pGpuChild, NV_FALSE);
1242
1243 status = memdescAlloc(pSubDevMemDesc);
1244
1245 if (pMemDesc->_addressSpace == ADDR_FBMEM)
1246 {
1247 //
1248 // The top level memdesc could have flags that don't reflect contiguity which
1249 // is set after memdescAlloc.
1250 //
1251 pMemDesc->Alignment = pSubDevMemDesc->Alignment;
1252 pMemDesc->_flags = pSubDevMemDesc->_flags | MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE;
1253 pMemDesc->ActualSize = pSubDevMemDesc->ActualSize;
1254 }
1255
1256 // Restore broadcast state to true after allocating a subdevice memdesc
1257 gpumgrSetBcEnabledStatus(pGpuChild, NV_TRUE);
1258
1259 if (status != NV_OK)
1260 {
1261 memdescDestroy(pSubDevMemDesc);
1262 NV_ASSERT(0);
1263 goto subdeviceAlloc_failed;
1264 }
1265
1266 // Check for similarity in allocations for previous allocated subdev with current allocated subdev.
1267 // If subdev0 ~ subdev1 && subdev1~subdev2 then subdev0 ~ subdev2 and so on...Thus can check symmetry across all subdev allocations
1268 if (i > 0)
1269 {
1270 NV_ASSERT(pPrev->Size == pSubDevMemDesc->Size);
1271 NV_ASSERT(pPrev->PteAdjust == pSubDevMemDesc->PteAdjust);
1272 NV_ASSERT(pPrev->_addressSpace == pSubDevMemDesc->_addressSpace);
1273 NV_ASSERT(pPrev->_flags == pSubDevMemDesc->_flags);
1274 NV_ASSERT(pPrev->_pteKind == pSubDevMemDesc->_pteKind);
1275 NV_ASSERT(pPrev->_pteKindCompressed == pSubDevMemDesc->_pteKindCompressed);
1276 NV_ASSERT(pPrev->pHeap != pSubDevMemDesc->pHeap);
1277 }
1278
1279 pPrev->_pNext = pSubDevMemDesc;
1280 pPrev = pSubDevMemDesc;
1281 }
1282 pMemDesc->Allocated = 1;
1283 return NV_OK;
1284 }
1285 else if (pMemDesc->_addressSpace == ADDR_FBMEM)
1286 {
1287 // Broadcast memdescAlloc call on vidmem *without* flag set to allocate per subdevice
1288 NV_ASSERT(0);
1289 }
1290 }
1291
1292 // Unicast memdescAlloc call but with flag set to allocate per subdevice.
1293 NV_ASSERT(!((pMemDesc->_flags & MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE) && !gpumgrGetBcEnabledStatus(pGpu)));
1294
1295 reAcquire = NV_FALSE;
1296 bcState = NV_FALSE;
1297
1298 if ((pMemDesc->_flags & MEMDESC_FLAGS_LOCKLESS_SYSMEM_ALLOC) && (pMemDesc->_addressSpace != ADDR_FBMEM))
1299 {
1300 bcState = gpumgrGetBcEnabledStatus(pGpu);
1301 if (RMCFG_FEATURE_RM_BASIC_LOCK_MODEL)
1302 {
1303 //
1304 // There is no equivalent routine for osCondReleaseRmSema in
1305 // the new basic lock model.
1306
1307 //
1308 // However, we can't drop the RM system semaphore in this
1309 // path because on non-windows platforms (i.e. MODS) it
1310 // has undesirable consequences. So for now we must
1311 // bracket this section with a reference to the feature
1312 // flag until we can rework this interface.
1313 //
1314 //
1315 // Check to make sure we own the lock and that we are
1316 // not at elevated IRQL; this models the behavior
1317 // of osCondReleaseRmSema.
1318 //
1319 if (!osIsRaisedIRQL() &&
1320 (rmGpuGroupLockIsOwner(pGpu->gpuInstance, GPU_LOCK_GRP_DEVICE, &gpuMask) ||
1321 rmGpuGroupLockIsOwner(pGpu->gpuInstance, GPU_LOCK_GRP_SUBDEVICE, &gpuMask)))
1322 {
1323 //
1324 // Release all owned gpu locks rather than just the
1325 // device-related locks because the caller may be holding more
1326 // than the required device locks. All currently owned
1327 // locks will be re-acquired before returning.
1328 //
1329 // This prevents potential GPU locking violations (e.g., if the
1330 // caller is holding all the gpu locks but only releases the
1331 // first of two device locks, then attempting to re-acquire
1332 // the first device lock will be a locking violation with
1333 // respect to the second device lock.)
1334 //
1335 gpuMask = rmGpuLocksGetOwnedMask();
1336 rmGpuGroupLockRelease(gpuMask, GPUS_LOCK_FLAGS_NONE);
1337 reAcquire = NV_TRUE;
1338 }
1339 }
1340 else
1341 {
1342 reAcquire = osCondReleaseRmSema(pSys->pSema);
1343 }
1344 }
1345
1346 // Actually allocate the memory
1347 NV_CHECK_OK(status, LEVEL_ERROR, _memdescAllocInternal(pMemDesc));
1348
1349 if (status != NV_OK)
1350 {
1351 pMemDesc->pHeap = NULL;
1352 }
1353
1354 if (reAcquire)
1355 {
1356 if (osAcquireRmSema(pSys->pSema) != NV_OK)
1357 {
1358 DBG_BREAKPOINT();
1359
1360 }
1361
1362 if (rmGpuGroupLockAcquire(pGpu->gpuInstance, GPU_LOCK_GRP_MASK,
1363 GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_MEM,
1364 &gpuMask) != NV_OK)
1365 {
1366 DBG_BREAKPOINT();
1367 }
1368 // Releasing the semaphore allows another thread to enter RM and
1369 // modify broadcast state. We need to set it back (see bug 368643)
1370 gpumgrSetBcEnabledStatus(pGpu, bcState);
1371 }
1372
1373 if ((status == NV_OK) &&
1374 IS_VIRTUAL_WITH_SRIOV(pGpu) &&
1375 !gpuIsWarBug200577889SriovHeavyEnabled(pGpu) &&
1376 (pMemDesc->_addressSpace == ADDR_SYSMEM) &&
1377 !(pMemDesc->_flags & MEMDESC_FLAGS_CPU_ONLY))
1378 {
1379 status = vgpuUpdateSysmemPfnBitMap(pGpu, pMemDesc, NV_TRUE);
1380 if (status != NV_OK)
1381 {
1382 NV_PRINTF(LEVEL_INFO, "Failed to update sysmem PFN bitmap, error 0x%x\n", status);
1383 }
1384 }
1385
1386 return status;
1387
1388 subdeviceAlloc_failed:
1389 _memSubDeviceFreeAndDestroy(pMemDesc);
1390 pMemDesc->_subDeviceAllocCount = 1;
1391 pMemDesc->_pNext = NULL;
1392 return status;
1393 }
1394
1395 /*!
1396 * Allocate memory from one of the possible locations specified in pList.
1397 *
1398 * @param[in,out] pMemDesc Memory descriptor to allocate
1399 * @param[in] pList List of NV_ADDRESS_SPACE values. Terminated
1400 * by an ADDR_UNKNOWN entry.
1401 *
1402 * @returns NV_OK on successful allocation. Various NV_ERR_GENERIC codes otherwise.
1403 */
1404 NV_STATUS
memdescAllocList(MEMORY_DESCRIPTOR * pMemDesc,const NV_ADDRESS_SPACE * pList)1405 memdescAllocList
1406 (
1407 MEMORY_DESCRIPTOR *pMemDesc,
1408 const NV_ADDRESS_SPACE *pList
1409 )
1410 {
1411 NV_STATUS status = NV_ERR_INVALID_ARGUMENT;
1412 NvU32 i = 0;
1413
1414 if (!pList)
1415 {
1416 return status;
1417 }
1418
1419 //
1420 // this memdesc may have gotten forced to sysmem if no carveout,
1421 // but for VPR it needs to be in vidmem, so check and re-direct here
1422 //
1423 if (pMemDesc->_flags & MEMDESC_ALLOC_FLAGS_PROTECTED)
1424 {
1425 OBJGPU *pGpu = pMemDesc->pGpu;
1426
1427 // Only force to vidmem if not running with zero-FB.
1428 if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB) ||
1429 gpuIsCacheOnlyModeEnabled(pGpu))
1430 {
1431 pList = ADDRLIST_FBMEM_ONLY;
1432 }
1433 }
1434
1435 while (pList[i] != ADDR_UNKNOWN)
1436 {
1437 pMemDesc->_addressSpace = pList[i];
1438 status = memdescAlloc(pMemDesc);
1439
1440 if (status == NV_OK)
1441 {
1442 return status;
1443 }
1444
1445 i++;
1446 }
1447
1448 return status;
1449 }
1450
1451 /*!
1452 * @brief Lower memdesc free layer. Provides underlying free
1453 * functionality.
1454 *
1455 * @param[in,out] pMemDesc Memory descriptor to free
1456 *
1457 * @returns None
1458 */
1459 static void
_memdescFreeInternal(MEMORY_DESCRIPTOR * pMemDesc)1460 _memdescFreeInternal
1461 (
1462 MEMORY_DESCRIPTOR *pMemDesc
1463 )
1464 {
1465 MEM_DESC_DESTROY_CALLBACK *pCb, *pNext;
1466 NvU64 oldSize;
1467
1468 // Allow null frees
1469 if (!pMemDesc)
1470 {
1471 return;
1472 }
1473
1474 pCb = memdescGetDestroyCallbackList(pMemDesc);
1475
1476 // Notify all interested parties of destruction
1477 while (pCb)
1478 {
1479 pNext = pCb->pNext;
1480 pCb->destroyCallback(pMemDesc->pGpu, pCb->pObject, pMemDesc);
1481 // pCb is now invalid
1482 pCb = pNext;
1483 }
1484
1485 if (memdescHasSubDeviceMemDescs(pMemDesc))
1486 return;
1487
1488 memdescPrintMemdesc(pMemDesc, NV_FALSE, MAKE_NV_PRINTF_STR("memdesc being freed"));
1489
1490 // Bail our early in case this memdesc describes a MODS managed VPR region.
1491 if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_VPR_REGION_CLIENT_MANAGED))
1492 return;
1493
1494 switch (pMemDesc->_addressSpace)
1495 {
1496 case ADDR_SYSMEM:
1497 case ADDR_EGM:
1498 // invalidate if memory is cached in FB L2 cache.
1499 if (pMemDesc->_gpuCacheAttrib == NV_MEMORY_CACHED)
1500 {
1501 OBJGPU *pGpu = pMemDesc->pGpu;
1502
1503 //
1504 // If this memdesc managed to outlive its pGpu getting detached,
1505 // we're plenty off the rails already, but avoid using the pGpu
1506 // and carry on as best we can
1507 //
1508 if (gpumgrIsGpuPointerValid(pGpu))
1509 {
1510 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
1511 {
1512 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
1513 NV_ASSERT_OK(kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, pMemDesc,
1514 FB_CACHE_SYSTEM_MEMORY,
1515 FB_CACHE_INVALIDATE));
1516 }
1517 SLI_LOOP_END
1518 }
1519 else
1520 {
1521 NV_ASSERT_FAILED("Sysmemdesc outlived its attached pGpu");
1522 }
1523 }
1524
1525 oldSize = pMemDesc->Size;
1526 pMemDesc->Size = pMemDesc->ActualSize;
1527 pMemDesc->PageCount = ((pMemDesc->ActualSize + pMemDesc->pageArrayGranularity - 1) >> BIT_IDX_64(pMemDesc->pageArrayGranularity));
1528
1529 osFreePages(pMemDesc);
1530
1531 pMemDesc->Size = oldSize;
1532 pMemDesc->PageCount = ((oldSize + pMemDesc->pageArrayGranularity - 1) >> BIT_IDX_64(pMemDesc->pageArrayGranularity));
1533
1534 break;
1535
1536 case ADDR_FBMEM:
1537 {
1538 Heap *pHeap = pMemDesc->pHeap;
1539 NV_STATUS status = NV_OK;
1540 OBJGPU *pGpu = pMemDesc->pGpu;
1541
1542 if (RMCFG_FEATURE_PMA &&
1543 (pMemDesc->_flags & MEMDESC_FLAGS_OWNED_BY_CTX_BUF_POOL))
1544 {
1545 CTX_BUF_POOL_INFO *pCtxBufPool = memdescGetCtxBufPool(pMemDesc);
1546 if (pCtxBufPool == NULL)
1547 {
1548 DBG_BREAKPOINT();
1549 NV_PRINTF(LEVEL_ERROR, "ctx buf pool not found\n");
1550 return;
1551 }
1552 NV_STATUS status = ctxBufPoolFree(pCtxBufPool, pMemDesc);
1553 if (status != NV_OK)
1554 {
1555 DBG_BREAKPOINT();
1556 NV_PRINTF(LEVEL_ERROR, "Failed to free memdesc from context buffer pool\n");
1557 }
1558 }
1559 else
1560 {
1561 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
1562
1563 NV_ASSERT(pHeap != NULL);
1564
1565 if (!pHeap)
1566 return;
1567
1568 status = memmgrFree(pGpu, pMemoryManager, pHeap, 0x0, 0x0, 0x0,
1569 HEAP_OWNER_RM_CHANNEL_CTX_BUFFER,
1570 pMemDesc);
1571 NV_ASSERT(status == NV_OK);
1572 }
1573
1574 // If this heap is being used to manage PMSA memory, reduce the refcount accordingly
1575 if ((status == NV_OK) && (pHeap != NULL) &&
1576 (pHeap->heapType == HEAP_TYPE_PHYS_MEM_SUBALLOCATOR))
1577 {
1578 heapRemoveRef(pHeap);
1579 }
1580 break;
1581 }
1582
1583 default:
1584 // Don't know how to do any other types of memory yet
1585 DBG_BREAKPOINT();
1586 }
1587 }
1588
1589 /*!
1590 * @brief Upper memdesc free layer. Provides support for per-subdevice
1591 * sysmem buffers and lockless sysmem allocation. Because of SLI and subdevice
1592 * submem allocations (refer to submem chart) support, if memory has never
1593 * been allocated function will just unlink subdevice structure and destroy
1594 * subdevice descriptors.
1595 *
1596 * @param[in,out] pMemDesc Memory descriptor to free
1597 *
1598 * @returns None
1599 */
1600 void
memdescFree(MEMORY_DESCRIPTOR * pMemDesc)1601 memdescFree
1602 (
1603 MEMORY_DESCRIPTOR *pMemDesc
1604 )
1605 {
1606 // Allow null frees
1607 if (!pMemDesc)
1608 {
1609 return;
1610 }
1611
1612
1613 if (memdescIsSubMemoryMemDesc(pMemDesc))
1614 {
1615 NV_ASSERT(!pMemDesc->_pInternalMapping);
1616
1617 if (pMemDesc->_addressSpace == ADDR_SYSMEM)
1618 {
1619 // The memdesc is being freed so destroy all of its IOMMU mappings.
1620 _memdescFreeIommuMappings(pMemDesc);
1621 }
1622
1623 if (pMemDesc->_addressSpace != ADDR_FBMEM &&
1624 pMemDesc->_addressSpace != ADDR_EGM &&
1625 pMemDesc->_addressSpace != ADDR_SYSMEM)
1626 {
1627 return;
1628 }
1629
1630 _memSubDeviceFreeAndDestroy(pMemDesc);
1631 }
1632 else
1633 {
1634 //
1635 // In case RM attempts to free memory that has more than 1 refcount, the free is deferred until refcount reaches 0
1636 //
1637 // Bug 3307574 RM crashes when client's specify sysmem UserD location.
1638 // RM attempts to peek at the client allocated UserD when waiting for a channel to go idle.
1639 //
1640 if (pMemDesc->RefCount > 1 && pMemDesc->Allocated == 1)
1641 {
1642 pMemDesc->bDeferredFree = NV_TRUE;
1643 return;
1644 }
1645
1646 if (!pMemDesc->Allocated)
1647 {
1648 /*
1649 * For sysmem not allocated by RM but only registered to it, we
1650 * would need to update the shared sysmem pfn bitmap here
1651 */
1652 if (pMemDesc->pGpu && IS_VIRTUAL_WITH_SRIOV(pMemDesc->pGpu) &&
1653 !gpuIsWarBug200577889SriovHeavyEnabled(pMemDesc->pGpu) &&
1654 !(pMemDesc->_flags & MEMDESC_FLAGS_CPU_ONLY))
1655 {
1656 if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM) &&
1657 (pMemDesc->_addressSpace == ADDR_SYSMEM))
1658 {
1659 NV_STATUS status = vgpuUpdateSysmemPfnBitMap(pMemDesc->pGpu, pMemDesc, NV_FALSE);
1660 if (status != NV_OK)
1661 {
1662 NV_PRINTF(LEVEL_INFO, "Failed to update sysmem PFN bitmap\n");
1663 }
1664 }
1665 }
1666 return;
1667 }
1668 pMemDesc->Allocated--;
1669 if (0 != pMemDesc->Allocated)
1670 {
1671 return;
1672 }
1673
1674 // If standbyBuffer memory was allocated then free it
1675 if (pMemDesc->_pStandbyBuffer)
1676 {
1677 memdescFree(pMemDesc->_pStandbyBuffer);
1678 memdescDestroy(pMemDesc->_pStandbyBuffer);
1679 pMemDesc->_pStandbyBuffer = NULL;
1680 }
1681
1682 NV_ASSERT(!pMemDesc->_pInternalMapping);
1683
1684 if (pMemDesc->_addressSpace == ADDR_SYSMEM)
1685 {
1686 if (pMemDesc->pGpu && IS_VIRTUAL_WITH_SRIOV(pMemDesc->pGpu) &&
1687 !gpuIsWarBug200577889SriovHeavyEnabled(pMemDesc->pGpu) &&
1688 !(pMemDesc->_flags & MEMDESC_FLAGS_CPU_ONLY))
1689 {
1690 NV_STATUS status = vgpuUpdateSysmemPfnBitMap(pMemDesc->pGpu, pMemDesc, NV_FALSE);
1691 if (status != NV_OK)
1692 {
1693 NV_PRINTF(LEVEL_INFO, "Failed to update sysmem PFN bitmap\n");
1694 }
1695 }
1696 // The memdesc is being freed so destroy all of its IOMMU mappings.
1697 _memdescFreeIommuMappings(pMemDesc);
1698 }
1699
1700 if (pMemDesc->_addressSpace != ADDR_FBMEM &&
1701 pMemDesc->_addressSpace != ADDR_EGM &&
1702 pMemDesc->_addressSpace != ADDR_SYSMEM)
1703 {
1704 return;
1705 }
1706
1707 _memSubDeviceFreeAndDestroy(pMemDesc);
1708
1709 _memdescFreeInternal(pMemDesc);
1710 }
1711
1712 // Reset tracking state
1713 pMemDesc->_pNext = NULL;
1714 pMemDesc->_subDeviceAllocCount = 1;
1715
1716 //
1717 // Reset tracking state of parent
1718 // Why it is needed:
1719 // When a submemory toplevel memdesc with subdevices is freed,
1720 // the subdecice memdescs and their parent are destroyed or their
1721 // refcount decreased.
1722 // When the parent subdevice descriptors are destroyed, their
1723 // top level descriptor is left alone and has a dangling
1724 // _pNext pointer
1725 //
1726 if ((pMemDesc->_pParentDescriptor != NULL) &&
1727 (memdescHasSubDeviceMemDescs(pMemDesc->_pParentDescriptor)) &&
1728 (pMemDesc->_pParentDescriptor->RefCount == 1))
1729 {
1730 pMemDesc->_pParentDescriptor->_pNext = NULL;
1731 pMemDesc->_pParentDescriptor->_subDeviceAllocCount = 1;
1732 }
1733 }
1734
1735 /*!
1736 * @brief Lock the paged virtual memory descripted by the memory descriptor
1737 *
1738 * @param[in] pMemDesc Memory descriptor to lock
1739 *
1740 * @returns NV_OK on successful allocation. Various NV_ERR_GENERIC codes otherwise.
1741 */
1742 NV_STATUS
memdescLock(MEMORY_DESCRIPTOR * pMemDesc)1743 memdescLock
1744 (
1745 MEMORY_DESCRIPTOR *pMemDesc
1746 )
1747 {
1748
1749 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
1750 if (!(pMemDesc->_flags & MEMDESC_FLAGS_PAGED_SYSMEM))
1751 {
1752 return NV_ERR_ILLEGAL_ACTION;
1753 }
1754
1755 return osLockMem(pMemDesc);
1756 }
1757
1758 /*!
1759 * @brief Unlock the paged virtual memory descripted by the memory descriptor
1760 *
1761 * @param[in] pMemDesc Memory descriptor to unlock
1762 *
1763 * @returns NV_OK on successful allocation. Various NV_ERR_GENERIC codes otherwise.
1764 */
1765 NV_STATUS
memdescUnlock(MEMORY_DESCRIPTOR * pMemDesc)1766 memdescUnlock
1767 (
1768 MEMORY_DESCRIPTOR *pMemDesc
1769 )
1770 {
1771 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
1772 if (!(pMemDesc->_flags & MEMDESC_FLAGS_PAGED_SYSMEM))
1773 {
1774 return NV_ERR_ILLEGAL_ACTION;
1775 }
1776
1777 return osUnlockMem(pMemDesc);
1778 }
1779
1780 /*!
1781 * @brief Get a CPU mapping to the memory described by a memory descriptor
1782 *
1783 * This is for memory descriptors used by RM clients, not by the RM itself.
1784 * For internal mappings the busMapRmAperture() hal routines are used.
1785 *
1786 * @param[in] pMemDesc Memory descriptor to map
1787 * @param[in] Offset Offset into memory descriptor to start map
1788 * @param[in] Size Size of mapping
1789 * @param[in] Kernel Kernel or user address space
1790 * @param[in] Protect NV_PROTECT_*
1791 * @param[out] pAddress Return address
1792 * @param[out] pPriv Return cookie to be passed back to memdescUnmap
1793 *
1794 * @returns NV_STATUS
1795 */
1796
1797 NV_STATUS
memdescMapOld(MEMORY_DESCRIPTOR * pMemDesc,NvU64 Offset,NvU64 Size,NvBool Kernel,NvU32 Protect,void ** pAddress,void ** pPriv)1798 memdescMapOld
1799 (
1800 MEMORY_DESCRIPTOR *pMemDesc,
1801 NvU64 Offset,
1802 NvU64 Size,
1803 NvBool Kernel,
1804 NvU32 Protect,
1805 void **pAddress,
1806 void **pPriv
1807 )
1808 {
1809 NvP64 pAddressP64 = NV_PTR_TO_NvP64(*pAddress);
1810 NvP64 pPrivP64 = NV_PTR_TO_NvP64(*pPriv);
1811 NV_STATUS status;
1812
1813 #if !defined(NV_64_BITS)
1814 NV_ASSERT(Kernel);
1815 #endif
1816
1817 status = memdescMap(pMemDesc,
1818 Offset,
1819 Size,
1820 Kernel,
1821 Protect,
1822 &pAddressP64,
1823 &pPrivP64);
1824
1825 *pAddress = NvP64_VALUE(pAddressP64);
1826 *pPriv = NvP64_VALUE(pPrivP64);
1827
1828 return status;
1829 }
1830
1831 NV_STATUS
memdescMap(MEMORY_DESCRIPTOR * pMemDesc,NvU64 Offset,NvU64 Size,NvBool Kernel,NvU32 Protect,NvP64 * pAddress,NvP64 * pPriv)1832 memdescMap
1833 (
1834 MEMORY_DESCRIPTOR *pMemDesc,
1835 NvU64 Offset,
1836 NvU64 Size,
1837 NvBool Kernel,
1838 NvU32 Protect,
1839 NvP64 *pAddress,
1840 NvP64 *pPriv
1841 )
1842 {
1843 NV_STATUS status = NV_OK;
1844 NvU64 rootOffset = 0;
1845
1846 NV_ASSERT_OR_RETURN(((Offset + Size) <= memdescGetSize(pMemDesc)), NV_ERR_INVALID_ARGUMENT);
1847
1848 pMemDesc = memdescGetRootMemDesc(pMemDesc, &rootOffset);
1849 Offset += rootOffset;
1850
1851 if (pMemDesc->PteAdjust &&
1852 (pMemDesc->Alignment > RM_PAGE_SIZE) &&
1853 (pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS) &&
1854 RMCFG_FEATURE_PLATFORM_MODS)
1855 {
1856 Offset += pMemDesc->PteAdjust;
1857 }
1858
1859 NV_ASSERT_OR_RETURN(!memdescHasSubDeviceMemDescs(pMemDesc), NV_ERR_INVALID_OBJECT_BUFFER);
1860
1861 switch (pMemDesc->_addressSpace)
1862 {
1863 case ADDR_SYSMEM:
1864 case ADDR_EGM:
1865 {
1866 status = osMapSystemMemory(pMemDesc, Offset, Size,
1867 Kernel, Protect, pAddress, pPriv);
1868 if (status != NV_OK)
1869 {
1870 return status;
1871 }
1872 break;
1873 }
1874
1875 case ADDR_FBMEM:
1876 {
1877 OBJGPU *pGpu = pMemDesc->pGpu;
1878 NvU32 mode = NV_MEMORY_WRITECOMBINED;
1879 KernelBus *pKernelBus;
1880 FB_MAPPING_INFO *pMapping;
1881 RmPhysAddr bar1PhysAddr;
1882 NvBool bCoherentCpuMapping;
1883
1884 NV_ASSERT_OR_RETURN(pGpu != NULL, NV_ERR_INVALID_ARGUMENT);
1885
1886 pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
1887 bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING);
1888
1889 // Need struct to keep track of the info for this mapping
1890 pMapping = portMemAllocNonPaged(sizeof(FB_MAPPING_INFO));
1891 if (pMapping == NULL)
1892 {
1893 return NV_ERR_NO_MEMORY;
1894 }
1895
1896 if (bCoherentCpuMapping)
1897 {
1898 NV_ASSERT(pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS);
1899
1900 if (Kernel)
1901 {
1902 NvP64 tempCpuPtr = kbusMapCoherentCpuMapping_HAL(pGpu, pKernelBus, pMemDesc);
1903 if (tempCpuPtr == NULL)
1904 {
1905 status = NV_ERR_GENERIC;
1906 }
1907 else
1908 {
1909 status = NV_OK;
1910 tempCpuPtr = NvP64_PLUS_OFFSET(tempCpuPtr, Offset);
1911 }
1912 *pAddress = tempCpuPtr;
1913 }
1914 else
1915 {
1916 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
1917 NvU64 fbOffset = pMemDesc->_pteArray[0] +
1918 pMemDesc->PteAdjust + Offset;
1919 bar1PhysAddr = pKernelMemorySystem->coherentCpuFbBase + fbOffset;
1920 mode = NV_MEMORY_CACHED;
1921
1922 status = osMapPciMemoryUser(pGpu->pOsGpuInfo, bar1PhysAddr,
1923 Size, Protect, pAddress,
1924 &pMapping->pPriv,
1925 mode);
1926 }
1927
1928
1929 if (status != NV_OK)
1930 {
1931 portMemFree(pMapping);
1932 return status;
1933 }
1934
1935 NV_PRINTF(LEVEL_INFO, "Allocating coherent link mapping. VA: %p PA: 0x%llx size: 0x%llx\n",
1936 NvP64_VALUE(*pAddress),
1937 memdescGetPhysAddr(pMemDesc, AT_GPU, Offset), Size);
1938
1939 *pPriv = NV_PTR_TO_NvP64(pMapping);
1940 break;
1941 }
1942
1943 // Determine where in BAR1 the mapping will go
1944 pMapping->FbApertureLen = Size;
1945 status = kbusMapFbAperture_HAL(pGpu, pKernelBus,
1946 pMemDesc, Offset,
1947 &pMapping->FbAperture,
1948 &pMapping->FbApertureLen,
1949 BUS_MAP_FB_FLAGS_MAP_UNICAST,
1950 NULL);
1951 if (status != NV_OK)
1952 {
1953 portMemFree(pMapping);
1954 return status;
1955 }
1956
1957 bar1PhysAddr = gpumgrGetGpuPhysFbAddr(pGpu) + pMapping->FbAperture;
1958 mode = NV_MEMORY_WRITECOMBINED;
1959
1960 // Create the mapping
1961 if (Kernel)
1962 {
1963 status = osMapPciMemoryKernel64(pGpu, bar1PhysAddr,
1964 Size, Protect, pAddress,
1965 mode);
1966 }
1967 else
1968 {
1969 status = osMapPciMemoryUser(pGpu->pOsGpuInfo, bar1PhysAddr,
1970 Size, Protect, pAddress,
1971 &pMapping->pPriv,
1972 mode);
1973 }
1974
1975 if (status != NV_OK)
1976 {
1977 if (!bCoherentCpuMapping)
1978 {
1979 kbusUnmapFbAperture_HAL(pGpu, pKernelBus, pMemDesc,
1980 pMapping->FbAperture,
1981 pMapping->FbApertureLen,
1982 BUS_MAP_FB_FLAGS_MAP_UNICAST);
1983 }
1984 portMemFree(pMapping);
1985 return status;
1986 }
1987
1988 *pPriv = NV_PTR_TO_NvP64(pMapping);
1989 break;
1990 }
1991
1992 default:
1993 // Don't know how to do any other types of memory yet
1994 DBG_BREAKPOINT();
1995 return NV_ERR_GENERIC;
1996 }
1997 return NV_OK;
1998 }
1999 void
memdescUnmapOld(MEMORY_DESCRIPTOR * pMemDesc,NvBool Kernel,NvU32 ProcessId,void * Address,void * Priv)2000 memdescUnmapOld
2001 (
2002 MEMORY_DESCRIPTOR *pMemDesc,
2003 NvBool Kernel,
2004 NvU32 ProcessId,
2005 void *Address,
2006 void *Priv
2007 )
2008 {
2009 memdescUnmap(pMemDesc,
2010 Kernel,
2011 ProcessId,
2012 NV_PTR_TO_NvP64(Address),
2013 NV_PTR_TO_NvP64(Priv));
2014 }
2015
2016 /*!
2017 * @brief Remove a mapping for the memory descriptor, reversing memdescMap
2018 *
2019 * @param[in] pMemDesc Memory descriptor to unmap
2020 * @param[in] Kernel Kernel or user address space
2021 * @param[in] ProcessId Process ID if user space
2022 * @param[in] Address Mapped address
2023 * @param[in] Priv Return priv cookie from memdescMap
2024 *
2025 * @returns None
2026 */
2027 void
memdescUnmap(MEMORY_DESCRIPTOR * pMemDesc,NvBool Kernel,NvU32 ProcessId,NvP64 Address,NvP64 Priv)2028 memdescUnmap
2029 (
2030 MEMORY_DESCRIPTOR *pMemDesc,
2031 NvBool Kernel,
2032 NvU32 ProcessId,
2033 NvP64 Address,
2034 NvP64 Priv
2035 )
2036 {
2037 // Allow null unmaps
2038 if (!Address)
2039 return;
2040
2041 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
2042
2043
2044 // find first allocated parent descriptor
2045 while (!pMemDesc->Allocated && pMemDesc->_pParentDescriptor)
2046 {
2047 pMemDesc = pMemDesc->_pParentDescriptor;
2048 }
2049
2050 switch (pMemDesc->_addressSpace)
2051 {
2052 case ADDR_SYSMEM:
2053 case ADDR_EGM:
2054 {
2055 osUnmapSystemMemory(pMemDesc, Kernel, ProcessId, Address, Priv);
2056 break;
2057 }
2058
2059 case ADDR_FBMEM:
2060 {
2061 OBJGPU *pGpu = pMemDesc->pGpu;
2062 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
2063 FB_MAPPING_INFO *pMapping = (FB_MAPPING_INFO *)NvP64_VALUE(Priv);
2064 NvBool bCoherentCpuMapping = pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING);
2065 NvU64 Size = pMapping->FbApertureLen;
2066
2067 NV_ASSERT(!(pMemDesc->_flags & MEMDESC_FLAGS_CPU_ONLY));
2068
2069 if (bCoherentCpuMapping)
2070 {
2071 if (Kernel)
2072 {
2073 kbusUnmapCoherentCpuMapping_HAL(pGpu, pKernelBus, pMemDesc);
2074 }
2075 else
2076 {
2077 osUnmapPciMemoryUser(pGpu->pOsGpuInfo, Address, Size,
2078 pMapping->pPriv);
2079 }
2080
2081 portMemFree(pMapping);
2082 break;
2083 }
2084
2085 kbusUnmapFbAperture_HAL(pGpu, pKernelBus, pMemDesc,
2086 pMapping->FbAperture,
2087 Size,
2088 BUS_MAP_FB_FLAGS_MAP_UNICAST);
2089 if (Kernel)
2090 {
2091 osUnmapPciMemoryKernel64(pGpu, Address);
2092 }
2093 else
2094 {
2095 osUnmapPciMemoryUser(pGpu->pOsGpuInfo, Address, Size,
2096 pMapping->pPriv);
2097 }
2098
2099 portMemFree(pMapping);
2100 break;
2101 }
2102
2103 default:
2104 // Don't know how to do any other types of memory yet
2105 DBG_BREAKPOINT();
2106 }
2107 }
2108
2109 typedef enum
2110 {
2111 MEMDESC_MAP_INTERNAL_TYPE_GSP, // On GSP, use a pre-existing mapping
2112 MEMDESC_MAP_INTERNAL_TYPE_COHERENT_FBMEM, // For NVLINK, use a pre-existing mapping for fbmem
2113 MEMDESC_MAP_INTERNAL_TYPE_BAR2, // Use BAR2 (fbmem or reflected sysmem)
2114 MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT, // Use OS to map sysmem
2115 } MEMDESC_MAP_INTERNAL_TYPE;
2116
2117 static MEMDESC_MAP_INTERNAL_TYPE
memdescGetMapInternalType(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc)2118 memdescGetMapInternalType
2119 (
2120 OBJGPU *pGpu,
2121 MEMORY_DESCRIPTOR *pMemDesc
2122 )
2123 {
2124 if (RMCFG_FEATURE_PLATFORM_GSP)
2125 {
2126 return MEMDESC_MAP_INTERNAL_TYPE_GSP;
2127 }
2128 else if (pMemDesc->_addressSpace == ADDR_FBMEM &&
2129 pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING))
2130 {
2131 // Temporary hack to keep the same behavior on GV100F (dVOLTA & DFPGA)
2132 if (IsDFPGA(pGpu))
2133 return MEMDESC_MAP_INTERNAL_TYPE_BAR2;
2134
2135 return MEMDESC_MAP_INTERNAL_TYPE_COHERENT_FBMEM;
2136 }
2137 else
2138 {
2139 KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu);
2140 NvBool bUseDirectMap = NV_FALSE;
2141 NV_STATUS status;
2142
2143 status = kbusUseDirectSysmemMap_HAL(pGpu, pKernelBus, pMemDesc, &bUseDirectMap);
2144 NV_ASSERT_OR_RETURN(status == NV_OK, MEMDESC_MAP_INTERNAL_TYPE_BAR2);
2145
2146 return bUseDirectMap ? MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT : MEMDESC_MAP_INTERNAL_TYPE_BAR2;
2147 }
2148
2149 return MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT;
2150 }
2151
2152 void
memdescFlushGpuCaches(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc)2153 memdescFlushGpuCaches
2154 (
2155 OBJGPU *pGpu,
2156 MEMORY_DESCRIPTOR *pMemDesc
2157 )
2158 {
2159 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
2160
2161 if (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED)
2162 {
2163 //
2164 // Only the non-coherent memory path is available, so writeback GPU L2
2165 // invalidate the GPU L2
2166 //
2167 kmemsysCacheOp_HAL(pGpu, pKernelMemorySystem, pMemDesc, FB_CACHE_MEM_UNDEFINED, FB_CACHE_EVICT);
2168 }
2169 }
2170
2171 void
memdescFlushCpuCaches(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc)2172 memdescFlushCpuCaches
2173 (
2174 OBJGPU *pGpu,
2175 MEMORY_DESCRIPTOR *pMemDesc
2176 )
2177 {
2178 // Flush WC to get the data written to this mapping out to memory
2179 osFlushCpuWriteCombineBuffer();
2180
2181 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
2182
2183 // Special care is needed on SOC, where the GPU cannot snoop the CPU L2
2184 if ((pKernelBif != NULL) &&
2185 !kbifIsSnoopDmaCapable(pGpu, pKernelBif) &&
2186 (memdescGetCpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED))
2187 {
2188 // Flush CPU L2 so that the GPU will see any changes the CPU made
2189 osFlushCpuCache();
2190 }
2191 }
2192
2193 /*
2194 * @brief map memory descriptor for internal access
2195 *
2196 * flags - subset of TRANSFER_FLAGS_
2197 */
2198 void*
memdescMapInternal(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc,NvU32 flags)2199 memdescMapInternal
2200 (
2201 OBJGPU *pGpu,
2202 MEMORY_DESCRIPTOR *pMemDesc,
2203 NvU32 flags
2204 )
2205 {
2206 MEMDESC_MAP_INTERNAL_TYPE mapType;
2207 NV_STATUS status;
2208
2209 NV_ASSERT_OR_RETURN(pMemDesc != NULL, NULL);
2210
2211 if (pMemDesc->_addressSpace == ADDR_FBMEM)
2212 {
2213 pMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu);
2214 }
2215
2216 mapType = memdescGetMapInternalType(pGpu, pMemDesc);
2217
2218 // We need to flush & invalidate GPU L2 cache only for directed BAR mappings.
2219 // Reflected BAR mappings will access memory via GPU, and hence go through GPU L2 cache.
2220 if (mapType == MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT)
2221 memdescFlushGpuCaches(pGpu, pMemDesc);
2222
2223 if (pMemDesc->_pInternalMapping != NULL)
2224 {
2225 NV_ASSERT(pMemDesc->_internalMappingRefCount);
2226
2227 // Existing BAR2 mapping may be invalid due to GPU reset
2228 if (mapType == MEMDESC_MAP_INTERNAL_TYPE_BAR2)
2229 {
2230 pMemDesc->_pInternalMapping = kbusValidateBar2ApertureMapping_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pMemDesc,
2231 pMemDesc->_pInternalMapping);
2232 NV_CHECK_OR_RETURN(LEVEL_ERROR, pMemDesc->_pInternalMapping != NULL, NULL);
2233 }
2234
2235 pMemDesc->_internalMappingRefCount++;
2236 return pMemDesc->_pInternalMapping;
2237 }
2238
2239 switch (mapType)
2240 {
2241 case MEMDESC_MAP_INTERNAL_TYPE_GSP:
2242 NV_CHECK_OR_RETURN(LEVEL_ERROR, pMemDesc->_pInternalMapping != NULL, NULL);
2243 break;
2244 case MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT:
2245 {
2246 status = memdescMapOld(pMemDesc, 0, pMemDesc->Size, NV_TRUE, NV_PROTECT_READ_WRITE,
2247 &pMemDesc->_pInternalMapping, &pMemDesc->_pInternalMappingPriv);
2248 NV_CHECK_OR_RETURN(LEVEL_ERROR, status == NV_OK, NULL);
2249 break;
2250 }
2251 case MEMDESC_MAP_INTERNAL_TYPE_COHERENT_FBMEM:
2252 {
2253 NV_ASSERT(pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED));
2254 pMemDesc->_pInternalMapping = kbusMapCoherentCpuMapping_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pMemDesc);
2255 NV_CHECK_OR_RETURN(LEVEL_ERROR, pMemDesc->_pInternalMapping != NULL, NULL);
2256 break;
2257 }
2258 case MEMDESC_MAP_INTERNAL_TYPE_BAR2:
2259 pMemDesc->_pInternalMapping = kbusMapBar2Aperture_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pMemDesc, flags);
2260 NV_CHECK_OR_RETURN(LEVEL_ERROR, pMemDesc->_pInternalMapping != NULL, NULL);
2261 break;
2262
2263 default:
2264 DBG_BREAKPOINT();
2265 }
2266
2267 pMemDesc->_internalMappingRefCount = 1;
2268 return pMemDesc->_pInternalMapping;
2269 }
2270
memdescUnmapInternal(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc,NvU32 flags)2271 void memdescUnmapInternal
2272 (
2273 OBJGPU *pGpu,
2274 MEMORY_DESCRIPTOR *pMemDesc,
2275 NvU32 flags
2276 )
2277 {
2278 MEMDESC_MAP_INTERNAL_TYPE mapType;
2279
2280 NV_ASSERT_OR_RETURN_VOID(pMemDesc != NULL);
2281 NV_ASSERT_OR_RETURN_VOID(pMemDesc->_pInternalMapping != NULL && pMemDesc->_internalMappingRefCount != 0);
2282
2283 if (pMemDesc->_addressSpace == ADDR_FBMEM)
2284 {
2285 pMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu);
2286 }
2287
2288 mapType = memdescGetMapInternalType(pGpu, pMemDesc);
2289
2290 if (mapType == MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT || mapType == MEMDESC_MAP_INTERNAL_TYPE_BAR2)
2291 {
2292 memdescFlushCpuCaches(pGpu, pMemDesc);
2293 }
2294
2295 if (--pMemDesc->_internalMappingRefCount == 0)
2296 {
2297 switch (mapType)
2298 {
2299 case MEMDESC_MAP_INTERNAL_TYPE_GSP:
2300 break;
2301 case MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT:
2302 memdescUnmapOld(pMemDesc, NV_TRUE, 0,
2303 pMemDesc->_pInternalMapping, pMemDesc->_pInternalMappingPriv);
2304 break;
2305
2306 case MEMDESC_MAP_INTERNAL_TYPE_COHERENT_FBMEM:
2307 {
2308 kbusUnmapCoherentCpuMapping_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pMemDesc);
2309 break;
2310 }
2311 case MEMDESC_MAP_INTERNAL_TYPE_BAR2:
2312 {
2313 NvU8 *p = (NvU8 *)pMemDesc->_pInternalMapping;
2314
2315 kbusUnmapBar2ApertureWithFlags_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pMemDesc, &p, flags);
2316 break;
2317 }
2318
2319 default:
2320 DBG_BREAKPOINT();
2321 }
2322
2323 pMemDesc->_pInternalMapping = NULL;
2324 pMemDesc->_pInternalMappingPriv = NULL;
2325 pMemDesc->_internalMappingRefCount = 0;
2326 }
2327
2328 // Flush for direct mappings too to keep the behavior
2329 if (((flags & TRANSFER_FLAGS_DEFER_FLUSH) == 0) &&
2330 (mapType == MEMDESC_MAP_INTERNAL_TYPE_SYSMEM_DIRECT || mapType == MEMDESC_MAP_INTERNAL_TYPE_BAR2))
2331 {
2332 kbusFlush_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu),
2333 kbusGetFlushAperture(GPU_GET_KERNEL_BUS(pGpu), memdescGetAddressSpace(pMemDesc)));
2334 }
2335 }
2336
2337 /*!
2338 * Describe an existing region of memory in a memory descriptor
2339 *
2340 * Memory must be physically contiguous.
2341 *
2342 * The memory descriptor must be initialized with
2343 * memdescCreate*(), typically memdescCreateExisting()
2344 * prior to calling memdescDescribe.
2345 *
2346 * memdescDescribe() now only updates the fields needed in the call.
2347 *
2348 * @param[out] pMemDesc Memory descriptor to fill
2349 * @param[in] AddressSpace Address space of memory
2350 * @param[in] Base Physical address of region
2351 * @param[in] Size Size of region
2352 *
2353 * @returns None
2354 */
2355 void
memdescDescribe(MEMORY_DESCRIPTOR * pMemDesc,NV_ADDRESS_SPACE AddressSpace,RmPhysAddr Base,NvU64 Size)2356 memdescDescribe
2357 (
2358 MEMORY_DESCRIPTOR *pMemDesc,
2359 NV_ADDRESS_SPACE AddressSpace,
2360 RmPhysAddr Base,
2361 NvU64 Size
2362 )
2363 {
2364 // Some sanity checks to see if we went through MemCreate*() first
2365 NV_ASSERT((pMemDesc->RefCount == 1) &&
2366 (memdescGetDestroyCallbackList(pMemDesc) == NULL) &&
2367 (pMemDesc->PteAdjust == 0));
2368
2369 NV_ASSERT(pMemDesc->_pIommuMappings == NULL);
2370 NV_ASSERT(pMemDesc->Allocated == 0);
2371 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
2372
2373 //
2374 // Check if the base address accounts for the DMA window start address
2375 // (always in the high, unaddressable bits of the address) and add it
2376 // if necessary. On most platforms, the DMA window start address will
2377 // simply be 0.
2378 //
2379 // This is most likely to happen in cases where the Base address is
2380 // read directly from a register or MMU entry, which does not already
2381 // account for the DMA window.
2382 //
2383 if (pMemDesc->pGpu == NULL)
2384 {
2385 NV_PRINTF(LEVEL_WARNING,
2386 "unable to check Base 0x%016llx for DMA window\n", Base);
2387 }
2388 else if (AddressSpace == ADDR_SYSMEM)
2389 {
2390 OBJGPU *pGpu = pMemDesc->pGpu;
2391 if (pGpu)
2392 {
2393 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
2394 NvU32 physAddrWidth = gpuGetPhysAddrWidth_HAL(pGpu, ADDR_SYSMEM);
2395 if ((Base & ~(NVBIT64(physAddrWidth) - 1)) == 0)
2396 {
2397 Base += pKernelBif->dmaWindowStartAddress;
2398 }
2399 }
2400 }
2401
2402 if (pMemDesc->Alignment != 0)
2403 {
2404 NV_ASSERT(NV_FLOOR_TO_QUANTA(Base, pMemDesc->Alignment) == Base);
2405 }
2406
2407 pMemDesc->Size = Size;
2408 pMemDesc->ActualSize = NV_ROUNDUP(Size, pMemDesc->pageArrayGranularity);
2409 pMemDesc->_flags |= MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS;
2410 pMemDesc->_addressSpace = AddressSpace;
2411 pMemDesc->_pteArray[0] = Base & ~RM_PAGE_MASK;
2412 pMemDesc->_subDeviceAllocCount = 1;
2413 pMemDesc->PteAdjust = NvU64_LO32(Base) & RM_PAGE_MASK;
2414 pMemDesc->PageCount = ((Size + pMemDesc->PteAdjust + RM_PAGE_SIZE - 1) >> RM_PAGE_SHIFT);
2415 pMemDesc->_pParentDescriptor = NULL;
2416 pMemDesc->childDescriptorCnt = 0;
2417 }
2418
2419 /*!
2420 * Static helper called from memdescFillPages.
2421 * When dynamic granularity memdescs are enabled. We only need to copy over the pages
2422 * without worrying about converting them to 4K.
2423 *
2424 * @param[in] pMemDesc Memory descriptor to fill
2425 * @param[in] pageIndex Index into memory descriptor to fill from
2426 * @param[in] pPages Array of physical addresses
2427 * @param[in] pageCount Number of entries in pPages
2428 * @param[in] pageSize Size of each page in pPages
2429 *
2430 * @returns None
2431 */
2432 static void
_memdescFillPagesAtNativeGranularity(MEMORY_DESCRIPTOR * pMemDesc,NvU32 pageIndex,NvU64 * pPages,NvU32 pageCount,NvU64 pageSize)2433 _memdescFillPagesAtNativeGranularity
2434 (
2435 MEMORY_DESCRIPTOR *pMemDesc,
2436 NvU32 pageIndex,
2437 NvU64 *pPages,
2438 NvU32 pageCount,
2439 NvU64 pageSize
2440 )
2441 {
2442 NV_STATUS status;
2443
2444 NV_ASSERT(pageIndex + pageCount < pMemDesc->PageCount);
2445
2446 status = memdescSetPageArrayGranularity(pMemDesc, pageSize);
2447 if (status != NV_OK)
2448 {
2449 return;
2450 }
2451
2452 for (NvU32 i = 0; i < pageCount; i++)
2453 {
2454 pMemDesc->_pteArray[pageIndex + i] = pPages[i];
2455 }
2456
2457 pMemDesc->ActualSize = pageCount * pageSize;
2458 }
2459
2460 /*!
2461 * Fill the PTE array of a memory descriptor with an array of addresses
2462 * returned by pmaAllocatePages().
2463 *
2464 * Memory must be physically discontiguous. For the contiguous case
2465 * memdescDescribe() is more apt.
2466 *
2467 * The memory descriptor must be initialized with memdescCreate*(),
2468 * typically memdescCreateExisting() prior to calling
2469 * memdescFillPages().
2470 *
2471 * @param[in] pMemDesc Memory descriptor to fill
2472 * @param[in] pageIndex Index into memory descriptor to fill from
2473 * @param[in] pPages Array of physical addresses
2474 * @param[in] pageCount Number of entries in pPages
2475 * @param[in] pageSize Size of each page in pPages
2476 *
2477 * @returns None
2478 */
2479 void
memdescFillPages(MEMORY_DESCRIPTOR * pMemDesc,NvU32 pageIndex,NvU64 * pPages,NvU32 pageCount,NvU64 pageSize)2480 memdescFillPages
2481 (
2482 MEMORY_DESCRIPTOR *pMemDesc,
2483 NvU32 pageIndex,
2484 NvU64 *pPages,
2485 NvU32 pageCount,
2486 NvU64 pageSize
2487 )
2488 {
2489 OBJGPU *pGpu = gpumgrGetSomeGpu();
2490 NvU32 i, j, k;
2491 NvU32 numChunks4k = pageSize / RM_PAGE_SIZE;
2492 NvU32 offset4k = numChunks4k * pageIndex;
2493 NvU32 pageCount4k = numChunks4k * pageCount;
2494 NvU32 result4k, limit4k;
2495 NvU64 addr;
2496
2497 NV_ASSERT(pMemDesc != NULL);
2498
2499 if (GPU_GET_MEMORY_MANAGER(pGpu)->bEnableDynamicGranularityPageArrays)
2500 {
2501 _memdescFillPagesAtNativeGranularity(pMemDesc, pageIndex, pPages, pageCount, pageSize);
2502 return;
2503 }
2504
2505 NV_ASSERT(offset4k < pMemDesc->PageCount);
2506 NV_ASSERT(portSafeAddU32(offset4k, pageCount4k, &result4k));
2507
2508 //
2509 // There is a possibility that the pMemDesc was created using 4K aligned
2510 // allocSize, but the actual memory allocator could align up the allocation
2511 // size based on its supported pageSize, (e.g. PMA supports 64K pages). In
2512 // that case, pageCount4k would be greater than pMemdesc->pageCount. We
2513 // limit pageCount4k to stay within pMemdesc->pageCount in that case.
2514 //
2515 if (result4k > pMemDesc->PageCount)
2516 pageCount4k = pMemDesc->PageCount - offset4k;
2517
2518 NV_ASSERT(pageSize > 0);
2519 NV_ASSERT(0 == (pageSize & (RM_PAGE_SIZE - 1)));
2520 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
2521
2522 // Fill _pteArray array using numChunks4k as a stride
2523 for (i = 0, j = offset4k; i < pageCount; i++, j += numChunks4k)
2524 {
2525 pMemDesc->_pteArray[j] = addr = pPages[i];
2526
2527 // Fill _pteArray at 4K granularity
2528 limit4k = NV_MIN(j + numChunks4k, pageCount4k);
2529
2530 addr += RM_PAGE_SIZE;
2531 for (k = j + 1; k < limit4k; k++, addr += RM_PAGE_SIZE)
2532 pMemDesc->_pteArray[k] = addr;
2533 }
2534 }
2535
2536 /*!
2537 * @brief Acquire exclusive use for memdesc for RM.
2538 *
2539 * @param[inout] pMemDesc Memory descriptor
2540 *
2541 * @returns Boolean indicating whether we successfully acquired the memdesc for exclusive use
2542 */
2543 NvBool
memdescAcquireRmExclusiveUse(MEMORY_DESCRIPTOR * pMemDesc)2544 memdescAcquireRmExclusiveUse
2545 (
2546 MEMORY_DESCRIPTOR *pMemDesc
2547 )
2548 {
2549 NV_CHECK_OR_RETURN(LEVEL_ERROR, pMemDesc->_pParentDescriptor == NULL &&
2550 !pMemDesc->bRmExclusiveUse &&
2551 pMemDesc->DupCount == 1,
2552 NV_FALSE);
2553
2554 pMemDesc->bRmExclusiveUse = NV_TRUE;
2555 return NV_TRUE;
2556 }
2557
2558 //
2559 // SubMemory per subdevice chart: (MD - Memory Descriptor, SD - subdevice)
2560 //
2561 // If we try to create submemory of descriptor which has subdevices:
2562 //
2563 // [Top level MD]
2564 // ^ |
2565 // | +--------> [ Subdevice 0 MD ] --------> [Subdevice 1 MD]
2566 // | ^ ^
2567 // | | |
2568 // [SubMemory top level MD] | |
2569 // | | |
2570 // +--------> [Subdevice 0 SubMemory MD] --------> [Subdevice 1 SubMemory MD]
2571 //
2572 // Top Level MD : parent of SubMemoryTopLevelMD; has subdescriptors
2573 // for two subdevices
2574 // SubMemory top level MD : has pointer to parent memory descriptor; has two
2575 // subdevice MDs
2576 // Subdevice 0 MD : subdevice MD of topLevelMD and parent of SD0
2577 // submemory descriptor; has pointer to next in the
2578 // list of subdevice MDs
2579 // Subdevice 0 SubMemory MD : submemory of subdevice 0 MD; has pointer to
2580 // parent, subdevice 0 MD and to next in list of
2581 // submemory subdevice memory descriptors
2582 //
2583
2584
2585
2586 /*!
2587 * @brief Create a new memory descriptor that is a subset of pMemDesc. If
2588 * pMemDesc has subdevice memory descriptors subMemory will be created for all
2589 * subdevices and new memory descriptor will be top level for them (ASCII art)
2590 *
2591 * @param[out] ppMemDescNew New memory descriptor
2592 * @param[in] pMemDesc Original memory descriptor
2593 * @param[in] pGpu The GPU that this memory will be mapped to
2594 * @param[in] Offset Sub memory descriptor starts at pMemdesc+Offset
2595 * @param[in] Size For Size bytes
2596 *
2597 * @returns None
2598 */
2599 NV_STATUS
memdescCreateSubMem(MEMORY_DESCRIPTOR ** ppMemDescNew,MEMORY_DESCRIPTOR * pMemDesc,OBJGPU * pGpu,NvU64 Offset,NvU64 Size)2600 memdescCreateSubMem
2601 (
2602 MEMORY_DESCRIPTOR **ppMemDescNew,
2603 MEMORY_DESCRIPTOR *pMemDesc,
2604 OBJGPU *pGpu,
2605 NvU64 Offset,
2606 NvU64 Size
2607 )
2608 {
2609 NV_STATUS status;
2610 MEMORY_DESCRIPTOR *pMemDescNew;
2611 NvU32 subDevInst;
2612 NvU64 tmpSize = Size;
2613 MEMORY_DESCRIPTOR *pLast;
2614 MEMORY_DESCRIPTOR *pNew;
2615 OBJGPU *pGpuChild;
2616 const NvU64 pageArrayGranularity = pMemDesc->pageArrayGranularity;
2617 const NvU64 pageArrayGranularityMask = pMemDesc->pageArrayGranularity - 1;
2618 const NvU32 pageArrayGranularityShift = BIT_IDX_64(pMemDesc->pageArrayGranularity);
2619
2620 // Default to the original memdesc's GPU if none is specified
2621 if (pGpu == NULL)
2622 {
2623 pGpu = pMemDesc->pGpu;
2624 }
2625
2626 // Allocation size should be adjusted for the memory descriptor _pageSize.
2627 // Also note that the first 4k page may not be at _pageSize boundary so at
2628 // the time of the mapping, we maybe overmapping at the beginning or end of
2629 // the descriptor. To fix it in the right way, memory descriptor needs to
2630 // be further cleaned. Do not round to page size if client specifies so.
2631 if (!(pMemDesc->_flags & MEMDESC_FLAGS_PAGE_SIZE_ALIGN_IGNORE) &&
2632 pMemDesc->_pageSize != 0)
2633 {
2634 PMEMORY_DESCRIPTOR pTempMemDesc = pMemDesc;
2635 NvU64 pageOffset;
2636
2637 if (memdescHasSubDeviceMemDescs(pMemDesc))
2638 {
2639 NV_ASSERT(pGpu);
2640 pTempMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu);
2641 }
2642
2643 pageOffset = memdescGetPhysAddr(pTempMemDesc, AT_CPU, Offset) &
2644 (pTempMemDesc->_pageSize - 1);
2645
2646 // Check for integer overflow
2647 if (!portSafeAddU64(pageOffset, Size, &tmpSize))
2648 {
2649 return NV_ERR_INVALID_ARGUMENT;
2650 }
2651
2652 tmpSize = RM_ALIGN_UP(pageOffset + Size, pTempMemDesc->_pageSize);
2653
2654 // Check for integer overflow
2655 if (tmpSize < pageOffset + Size)
2656 {
2657 return NV_ERR_INVALID_ARGUMENT;
2658 }
2659 }
2660
2661 // Allocate the new MEMORY_DESCRIPTOR
2662 status = memdescCreate(&pMemDescNew, pGpu, tmpSize, 0,
2663 !!(pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS),
2664 pMemDesc->_addressSpace,
2665 pMemDesc->_cpuCacheAttrib,
2666 ((pMemDesc->_flags & ~MEMDESC_FLAGS_PRE_ALLOCATED) | MEMDESC_FLAGS_SKIP_RESOURCE_COMPUTE));
2667
2668 if (status != NV_OK)
2669 {
2670 return status;
2671 }
2672
2673 // Fill in various fields as best we can; XXX this can get sort of sketchy
2674 // in places, which should be all the more motivation to rip some of these
2675 // fields out of the MEMORY_DESCRIPTOR.
2676 if (pMemDesc->_flags & MEMDESC_FLAGS_KERNEL_MODE)
2677 pMemDescNew->_flags |= MEMDESC_FLAGS_KERNEL_MODE;
2678 else
2679 pMemDescNew->_flags &= ~MEMDESC_FLAGS_KERNEL_MODE;
2680
2681 //
2682 // This flag used to indicate the memdesc needs to restore pte kind
2683 // when it is freed. It should only not be set for any sub memDesc.
2684 //
2685 pMemDescNew->_flags &= ~MEMDESC_FLAGS_RESTORE_PTE_KIND_ON_FREE;
2686
2687 pMemDescNew->Size = Size;
2688 pMemDescNew->_pteKind = pMemDesc->_pteKind;
2689 pMemDescNew->_hwResId = pMemDesc->_hwResId;
2690 if (pMemDesc->_flags & MEMDESC_FLAGS_ENCRYPTED)
2691 pMemDescNew->_flags |= MEMDESC_FLAGS_ENCRYPTED;
2692 else
2693 pMemDescNew->_flags &= ~MEMDESC_FLAGS_ENCRYPTED;
2694 pMemDescNew->_pageSize = pMemDesc->_pageSize;
2695 pMemDescNew->pageArrayGranularity = pageArrayGranularity;
2696 pMemDescNew->_gpuCacheAttrib = pMemDesc->_gpuCacheAttrib;
2697 pMemDescNew->_gpuP2PCacheAttrib = pMemDesc->_gpuP2PCacheAttrib;
2698 pMemDescNew->gfid = pMemDesc->gfid;
2699 pMemDescNew->bUsingSuballocator = pMemDesc->bUsingSuballocator;
2700 pMemDescNew->_pParentDescriptor = pMemDesc;
2701 pMemDesc->childDescriptorCnt++;
2702 pMemDescNew->bRmExclusiveUse = pMemDesc->bRmExclusiveUse;
2703 pMemDescNew->numaNode = pMemDesc->numaNode;
2704
2705 pMemDescNew->subMemOffset = Offset;
2706
2707 // increase refCount of parent descriptor
2708 memdescAddRef(pMemDesc);
2709
2710 // Fill in the PteArray and PteAdjust
2711 if ((pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS) ||
2712 (pMemDesc->PageCount == 1))
2713 {
2714 // Compute the base address, then fill it in
2715 RmPhysAddr Base = pMemDesc->_pteArray[0] + pMemDesc->PteAdjust + Offset;
2716 pMemDescNew->_pteArray[0] = Base & ~pageArrayGranularityMask;
2717 pMemDescNew->PteAdjust = NvU64_LO32(Base) & pageArrayGranularityMask;
2718 if (memdescIsEgm(pMemDesc))
2719 {
2720 NV_ASSERT_OK_OR_GOTO(status,
2721 _memdescAllocEgmArray(pMemDescNew),
2722 fail);
2723 }
2724 }
2725 else
2726 {
2727 // More complicated...
2728 RmPhysAddr Adjust;
2729 NvU32 PageIndex, i;
2730
2731 // We start this many bytes into the memory alloc
2732 Adjust = pMemDesc->PteAdjust + Offset;
2733
2734 // Break it down into pages (PageIndex) and bytes (PteAdjust)
2735 PageIndex = (NvU32)(Adjust >> pageArrayGranularityShift);
2736 pMemDescNew->PteAdjust = NvU64_LO32(Adjust) & pageArrayGranularityMask;
2737
2738 // Fill in the PTEs; remember to copy the extra PTE, in case we need it
2739 if (pMemDesc->PageCount)
2740 {
2741 for (i = 0; i < pMemDescNew->PageCount+1; i++)
2742 {
2743 NvU32 j = i + PageIndex;
2744 if (j < pMemDesc->PageCount)
2745 {
2746 pMemDescNew->_pteArray[i] = pMemDesc->_pteArray[j];
2747 }
2748 else
2749 {
2750 //
2751 // This case can happen with page size greater than 4KB.
2752 // Since pages are always tracked at 4KB granularity the
2753 // subset description may overflow the parent memdesc.
2754 //
2755 // In this case the best we can do is describe the contiguous
2756 // memory after the last 4KB page in the sub-memdesc.
2757 //
2758 // TODO: Tracking memdesc pages at native page size would
2759 // remove the need for several hacks, including this one.
2760 //
2761 NV_ASSERT(i > 0);
2762 pMemDescNew->_pteArray[i] = pMemDescNew->_pteArray[i - 1] + pMemDescNew->pageArrayGranularity;
2763 }
2764 }
2765 if (memdescIsEgm(pMemDesc))
2766 {
2767 NV_ASSERT_OK_OR_GOTO(status,
2768 _memdescAllocEgmArray(pMemDescNew),
2769 fail);
2770 }
2771 }
2772 }
2773
2774 if ((pMemDesc->_addressSpace == ADDR_SYSMEM) &&
2775 !memdescIsEgm(pMemDesc) &&
2776 !memdescGetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_CPU_ONLY) &&
2777 !memdescGetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_MAP_SYSCOH_OVER_BAR1) &&
2778 !memdescGetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_SKIP_IOMMU_MAPPING))
2779 {
2780 //
2781 // For different IOVA spaces, the IOMMU mapping will often not be a
2782 // subrange of the original mapping.
2783 //
2784 // Request the submapping to be associated with the submemdesc.
2785 //
2786 // TODO: merge the new IOMMU paths with the SMMU path above (see bug
2787 // 1625121).
2788 //
2789 status = memdescMapIommu(pMemDescNew, pGpu->busInfo.iovaspaceId);
2790 if (status != NV_OK)
2791 {
2792 memdescDestroy(pMemDescNew);
2793 return status;
2794 }
2795 }
2796
2797 // Support for SLI submemory per-subdevice allocations (refer to chart)
2798 if (memdescHasSubDeviceMemDescs(pMemDesc))
2799 {
2800 NvBool bBcState = gpumgrGetBcEnabledStatus(pGpu);
2801
2802 if (gpumgrGetBcEnabledStatus(pGpu) && (pMemDesc->_addressSpace == ADDR_FBMEM))
2803 {
2804 NV_ASSERT(!!(pMemDesc->_flags & MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE));
2805 gpumgrSetBcEnabledStatus(pGpu, NV_FALSE);
2806 }
2807 pLast = pMemDescNew;
2808
2809 pMemDescNew->_subDeviceAllocCount = pMemDesc->_subDeviceAllocCount;
2810
2811 for (subDevInst = 0; subDevInst < pMemDesc->_subDeviceAllocCount; subDevInst++)
2812 {
2813 pGpuChild = gpumgrGetGpuFromSubDeviceInst(gpuGetDeviceInstance(pGpu), subDevInst);
2814 status = memdescCreateSubMem(&pNew, memdescGetMemDescFromGpu(pMemDesc, pGpuChild), pGpuChild, Offset, Size);
2815
2816 if (status != NV_OK)
2817 {
2818 while (NULL != pMemDescNew)
2819 {
2820 pNew = pMemDescNew;
2821 pMemDescNew = pMemDescNew->_pNext;
2822 memdescDestroy(pNew);
2823 }
2824 return status;
2825 }
2826
2827 pLast->_pNext = pNew;
2828 pLast = pNew;
2829 }
2830
2831 gpumgrSetBcEnabledStatus(pGpu, bBcState);
2832 }
2833
2834 *ppMemDescNew = pMemDescNew;
2835
2836 return NV_OK;
2837
2838 fail:
2839 memdescDestroy(pMemDescNew);
2840 return status;
2841 }
2842
2843 /*!
2844 * Given a memdesc, this checks if the allocated memory falls under subheap or in GPA address space
2845 */
2846 static NvBool
_memIsSriovMappingsEnabled(PMEMORY_DESCRIPTOR pMemDesc)2847 _memIsSriovMappingsEnabled
2848 (
2849 PMEMORY_DESCRIPTOR pMemDesc
2850 )
2851 {
2852 return gpuIsSriovEnabled(pMemDesc->pGpu) &&
2853 (((pMemDesc->_flags & MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE) && pMemDesc->bUsingSuballocator) ||
2854 (pMemDesc->_flags & MEMDESC_FLAGS_GUEST_ALLOCATED));
2855 }
2856
2857 /*!
2858 * Fills pGpaEntries with numEntries GPAs from pMemDesc->_pteArray starting at
2859 * the given starting index. For physically contiguous memdescs, fills with
2860 * RM_PAGE_SIZE strides.
2861 */
2862 static void
_memdescFillGpaEntriesForSpaTranslation(PMEMORY_DESCRIPTOR pMemDesc,RmPhysAddr * pGpaEntries,NvU32 start,NvU32 numEntries)2863 _memdescFillGpaEntriesForSpaTranslation
2864 (
2865 PMEMORY_DESCRIPTOR pMemDesc,
2866 RmPhysAddr *pGpaEntries,
2867 NvU32 start,
2868 NvU32 numEntries
2869 )
2870 {
2871 if (pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS)
2872 {
2873 NvU32 i;
2874
2875 for (i = 0; i < numEntries; i++)
2876 {
2877 pGpaEntries[i] = pMemDesc->_pteArray[0] + (((RmPhysAddr) (start + i)) * pMemDesc->pageArrayGranularity);
2878 }
2879 }
2880 else
2881 {
2882 portMemCopy(&pGpaEntries[0], numEntries * sizeof(pGpaEntries[0]),
2883 &pMemDesc->_pteArray[start], numEntries * sizeof(pGpaEntries[0]));
2884 }
2885 }
2886
2887 /*!
2888 * This function translates GPA -> SPA for a given memdesc and updates pPteSpaMappings with list of SPA addresses.
2889 * If memdesc is contiguous and if the translated SPA count > 1, this function fails for now.
2890 */
2891 NV_STATUS
_memdescUpdateSpaArray(PMEMORY_DESCRIPTOR pMemDesc)2892 _memdescUpdateSpaArray
2893 (
2894 PMEMORY_DESCRIPTOR pMemDesc
2895 )
2896 {
2897 NV_STATUS status = NV_OK;
2898 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pMemDesc->pGpu);
2899 NvU32 allocCnt;
2900 NvU32 i;
2901 NV2080_CTRL_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES_PARAMS *pParams = NULL;
2902
2903 if ((pMemDesc->pPteSpaMappings) || (!pMemDesc->PageCount))
2904 {
2905 status = NV_OK;
2906 goto _memUpdateSpArray_exit;
2907 }
2908
2909 allocCnt = memdescGetPteArraySize(pMemDesc, AT_PA);
2910
2911 // Allocate the array to hold pages up to PageCount
2912 pMemDesc->pPteSpaMappings = portMemAllocNonPaged(sizeof(RmPhysAddr) * allocCnt);
2913 if (pMemDesc->pPteSpaMappings == NULL)
2914 {
2915 status = NV_ERR_NO_MEMORY;
2916 goto _memUpdateSpArray_exit;
2917 }
2918
2919 pParams = portMemAllocStackOrHeap(sizeof(*pParams));
2920 if (pParams == NULL)
2921 {
2922 status = NV_ERR_NO_MEMORY;
2923 goto _memUpdateSpArray_exit;
2924 }
2925 portMemSet(pParams, 0, sizeof(*pParams));
2926
2927 pParams->gfid = pMemDesc->gfid;
2928
2929 for (i = 0; i < allocCnt; i += NV2080_CTRL_INTERNAL_VMMU_MAX_SPA_FOR_GPA_ENTRIES)
2930 {
2931 NvU32 numEntries = NV_MIN(allocCnt - i, NV2080_CTRL_INTERNAL_VMMU_MAX_SPA_FOR_GPA_ENTRIES);
2932 pParams->numEntries = numEntries;
2933
2934 _memdescFillGpaEntriesForSpaTranslation(pMemDesc, &pParams->gpaEntries[0],
2935 i, numEntries);
2936
2937 status = pRmApi->Control(pRmApi,
2938 pMemDesc->pGpu->hInternalClient,
2939 pMemDesc->pGpu->hInternalSubdevice,
2940 NV2080_CTRL_CMD_INTERNAL_VMMU_GET_SPA_FOR_GPA_ENTRIES,
2941 pParams,
2942 sizeof(*pParams));
2943 if (status != NV_OK)
2944 {
2945 NV_PRINTF(LEVEL_ERROR, "Getting SPA for GPA failed: GFID=%u, GPA=0x%llx\n",
2946 pMemDesc->gfid, pMemDesc->_pteArray[i]);
2947 goto _memUpdateSpArray_exit;
2948 }
2949
2950 portMemCopy(&pMemDesc->pPteSpaMappings[i], numEntries * sizeof(pParams->spaEntries[0]),
2951 &pParams->spaEntries[0], numEntries * sizeof(pParams->spaEntries[0]));
2952 }
2953
2954 _memUpdateSpArray_exit:
2955 if (status != NV_OK)
2956 {
2957 portMemFree(pMemDesc->pPteSpaMappings);
2958 pMemDesc->pPteSpaMappings = NULL;
2959 }
2960 portMemFreeStackOrHeap(pParams);
2961
2962 return status;
2963 }
2964
2965 /*!
2966 * @brief Return the physical addresses of pMemdesc
2967 *
2968 * @param[in] pMemDesc Memory descriptor used
2969 * @param[in] pGpu GPU to return the addresses for
2970 * @param[in] addressTranslation Address translation identifier
2971 * @param[in] offset Offset into memory descriptor
2972 * @param[in] stride How much to advance the offset for each
2973 * consecutive address
2974 * @param[in] count How many addresses to retrieve
2975 * @param[out] pAddresses Returned array of addresses
2976 *
2977 */
memdescGetPhysAddrsForGpu(MEMORY_DESCRIPTOR * pMemDesc,OBJGPU * pGpu,ADDRESS_TRANSLATION addressTranslation,NvU64 offset,NvU64 stride,NvU64 count,RmPhysAddr * pAddresses)2978 void memdescGetPhysAddrsForGpu(MEMORY_DESCRIPTOR *pMemDesc,
2979 OBJGPU *pGpu,
2980 ADDRESS_TRANSLATION addressTranslation,
2981 NvU64 offset,
2982 NvU64 stride,
2983 NvU64 count,
2984 RmPhysAddr *pAddresses)
2985 {
2986 //
2987 // Get the PTE array that we should use for phys addr lookups based on the
2988 // MMU context. (see bug 1625121)
2989 //
2990 NvU64 i;
2991 NvU64 pageIndex;
2992 RmPhysAddr *pteArray = memdescGetPteArrayForGpu(pMemDesc, pGpu, addressTranslation);
2993 const NvBool contiguous = (memdescGetPteArraySize(pMemDesc, addressTranslation) == 1);
2994 const NvU64 pageArrayGranularityMask = pMemDesc->pageArrayGranularity - 1;
2995 const NvU32 pageArrayGranularityShift = BIT_IDX_64(pMemDesc->pageArrayGranularity);
2996
2997 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
2998 offset += pMemDesc->PteAdjust;
2999
3000 for (i = 0; i < count; ++i)
3001 {
3002 if (contiguous)
3003 {
3004 pAddresses[i] = pteArray[0] + offset;
3005 }
3006 else
3007 {
3008 pageIndex = offset >> pageArrayGranularityShift;
3009 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR, pageIndex < pMemDesc->PageCount);
3010 pAddresses[i] = pteArray[pageIndex] + (offset & pageArrayGranularityMask);
3011 }
3012
3013 offset += stride;
3014 }
3015 }
3016
3017
3018 /*!
3019 * @brief Return the physical addresses of pMemdesc
3020 *
3021 * @param[in] pMemDesc Memory descriptor used
3022 * @param[in] addressTranslation Address translation identifier
3023 * @param[in] offset Offset into memory descriptor
3024 * @param[in] stride How much to advance the offset for each
3025 * consecutive address
3026 * @param[in] count How many addresses to retrieve
3027 * @param[out] pAddresses Returned array of addresses
3028 *
3029 */
memdescGetPhysAddrs(MEMORY_DESCRIPTOR * pMemDesc,ADDRESS_TRANSLATION addressTranslation,NvU64 offset,NvU64 stride,NvU64 count,RmPhysAddr * pAddresses)3030 void memdescGetPhysAddrs(MEMORY_DESCRIPTOR *pMemDesc,
3031 ADDRESS_TRANSLATION addressTranslation,
3032 NvU64 offset,
3033 NvU64 stride,
3034 NvU64 count,
3035 RmPhysAddr *pAddresses)
3036 {
3037 memdescGetPhysAddrsForGpu(pMemDesc, pMemDesc->pGpu, addressTranslation, offset, stride, count, pAddresses);
3038 }
3039
3040 /*!
3041 * @brief Return the physical address of pMemdesc+Offset
3042 *
3043 * "long description"
3044 *
3045 * @param[in] pMemDesc Memory descriptor used
3046 * @param[in] addressTranslation Address translation identifier
3047 * @param[in] offset Offset into memory descriptor
3048 *
3049 * @returns A physical address
3050 */
3051 RmPhysAddr
memdescGetPhysAddr(MEMORY_DESCRIPTOR * pMemDesc,ADDRESS_TRANSLATION addressTranslation,NvU64 offset)3052 memdescGetPhysAddr
3053 (
3054 MEMORY_DESCRIPTOR *pMemDesc,
3055 ADDRESS_TRANSLATION addressTranslation,
3056 NvU64 offset
3057 )
3058 {
3059 RmPhysAddr addr;
3060 memdescGetPhysAddrs(pMemDesc, addressTranslation, offset, 0, 1, &addr);
3061 return addr;
3062 }
3063
3064 /*!
3065 * @brief Return physical address for page specified by PteIndex
3066 *
3067 * @param[in] pMemDesc Memory descriptor to use
3068 * @param[in] addressTranslation Address translation identifier
3069 * @param[in] PteIndex Look up this PteIndex
3070 *
3071 * @returns A physical address
3072 */
3073 RmPhysAddr
memdescGetPte(PMEMORY_DESCRIPTOR pMemDesc,ADDRESS_TRANSLATION addressTranslation,NvU32 PteIndex)3074 memdescGetPte
3075 (
3076 PMEMORY_DESCRIPTOR pMemDesc,
3077 ADDRESS_TRANSLATION addressTranslation,
3078 NvU32 PteIndex
3079 )
3080 {
3081 //
3082 // Get the PTE array that we should use for phys addr lookups based on the
3083 // MMU context. (see bug 1625121)
3084 //
3085 RmPhysAddr *pteArray = memdescGetPteArray(pMemDesc, addressTranslation);
3086 RmPhysAddr PhysAddr;
3087
3088 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3089
3090 if (pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS)
3091 {
3092 PhysAddr = pteArray[0] + (PteIndex << RM_PAGE_SHIFT);
3093 }
3094 else
3095 {
3096 PhysAddr = pteArray[PteIndex];
3097 }
3098
3099 return PhysAddr;
3100 }
3101
3102 /*!
3103 * @brief Return physical address for page specified by PteIndex
3104 *
3105 * @param[in] pMemDesc Memory descriptor to use
3106 * @param[in] addressTranslation Address translation identifier
3107 * @param[in] PteIndex Look up this PteIndex
3108 * @param[in] PhysAddr PTE address
3109 *
3110 * @returns None
3111 */
3112 void
memdescSetPte(PMEMORY_DESCRIPTOR pMemDesc,ADDRESS_TRANSLATION addressTranslation,NvU32 PteIndex,RmPhysAddr PhysAddr)3113 memdescSetPte
3114 (
3115 PMEMORY_DESCRIPTOR pMemDesc,
3116 ADDRESS_TRANSLATION addressTranslation,
3117 NvU32 PteIndex,
3118 RmPhysAddr PhysAddr
3119 )
3120 {
3121 //
3122 // Get the PTE array that we should use for phys addr lookups based on the
3123 // MMU context. (see bug 1625121)
3124 //
3125 RmPhysAddr *pteArray = memdescGetPteArray(pMemDesc, addressTranslation);
3126 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3127
3128 if (pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS)
3129 {
3130 NV_ASSERT_OR_RETURN_VOID(PteIndex == 0);
3131 }
3132
3133 pteArray[PteIndex] = PhysAddr;
3134
3135 // Free pteArraySpa
3136 portMemFree(pMemDesc->pPteSpaMappings);
3137 pMemDesc->pPteSpaMappings = NULL;
3138 }
3139
3140 /*!
3141 * @brief Return page array size based on the MMU context
3142 * For SRIOV, the host context (AT_PA) will
3143 * have discontiguous view of the GPA in SPA space
3144 * This is treated similar to discontiguous memdescs
3145 *
3146 * @param[in] pMemDesc Memory descriptor to use
3147 * @param[in] addressTranslation Address translation identifier
3148 *
3149 * @returns PageArray
3150 */
memdescGetPteArraySize(MEMORY_DESCRIPTOR * pMemDesc,ADDRESS_TRANSLATION addressTranslation)3151 NvU32 memdescGetPteArraySize(MEMORY_DESCRIPTOR *pMemDesc, ADDRESS_TRANSLATION addressTranslation)
3152 {
3153 // Contiguous allocations in SPA domain can be non-contiguous at vmmusegment granularity.
3154 // Hence treat SPA domain allocations as non-contiguous by default.
3155 if (!(pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS) ||
3156 ((addressTranslation == AT_PA) && (pMemDesc->_addressSpace == ADDR_FBMEM) && _memIsSriovMappingsEnabled(pMemDesc)))
3157 {
3158 return NvU64_LO32(pMemDesc->PageCount);
3159 }
3160 return 1;
3161 }
3162
3163 /*!
3164 * @brief Return page array
3165 *
3166 * @param[in] pMemDesc Memory descriptor to use
3167 * @param[in] pGpu GPU to get the PTE array for.
3168 * @param[in] addressTranslation Address translation identifier
3169 *
3170 * @returns PageArray
3171 */
3172 RmPhysAddr *
memdescGetPteArrayForGpu(PMEMORY_DESCRIPTOR pMemDesc,OBJGPU * pGpu,ADDRESS_TRANSLATION addressTranslation)3173 memdescGetPteArrayForGpu
3174 (
3175 PMEMORY_DESCRIPTOR pMemDesc,
3176 OBJGPU *pGpu,
3177 ADDRESS_TRANSLATION addressTranslation
3178 )
3179 {
3180 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3181
3182 switch (AT_VALUE(addressTranslation))
3183 {
3184 //
3185 // In SRIOV systems, an access from guest has to go through the following translations
3186 // GVA -> GPA -> SPA
3187 //
3188 // Given HOST manages channel/memory management for guest, there are certain code paths that
3189 // expects VA -> GPA translations and some may need GPA -> SPA translations. We use addressTranslation
3190 // to differentiate between these cases.
3191 // Since GPA -> SPA is very similar to IOMMU xlation and since existing AT_PA is used only in
3192 // SYSMEM allocations, we decided to reuse AT_PA addressTranslation to fetch GPA -> SPA xlations.
3193 // In case of non-SRIOV systems, using AT_PA will fall back to AT_GPU or default context.
3194 //
3195 // pMemDesc -> _pteArray tracks GVA -> GPA translations
3196 // pMemDesc -> pPteSpaMappings tracks GPA -> SPA translations
3197 //
3198
3199 case AT_VALUE(AT_PA):
3200 {
3201 if (pGpu != NULL)
3202 {
3203 if (pMemDesc->_addressSpace == ADDR_FBMEM)
3204 {
3205 if (_memIsSriovMappingsEnabled(pMemDesc))
3206 {
3207 if (!pMemDesc->pPteSpaMappings)
3208 _memdescUpdateSpaArray(pMemDesc);
3209
3210 return pMemDesc->pPteSpaMappings;
3211 }
3212 }
3213 }
3214 }
3215 case AT_VALUE(AT_GPU):
3216 {
3217 // Imported ADDR_FABRIC_V2 memdescs are device-less.
3218 if (pGpu != NULL)
3219 {
3220 if (memdescIsEgm(pMemDesc) && (pMemDesc->pPteEgmMappings != NULL))
3221 {
3222 return pMemDesc->pPteEgmMappings;
3223 }
3224 {
3225 PIOVAMAPPING pIovaMap = memdescGetIommuMap(pMemDesc,
3226 pGpu->busInfo.iovaspaceId);
3227 if (pIovaMap != NULL)
3228 {
3229 return pIovaMap->iovaArray;
3230 }
3231 }
3232 }
3233
3234 //
3235 // If no IOMMU mapping exists in the default IOVASPACE, fall
3236 // through and use the physical memory descriptor instead.
3237 //
3238 }
3239 default:
3240 {
3241 return pMemDesc->_pteArray;
3242 }
3243 }
3244 }
3245
3246
3247
3248 /*!
3249 * @brief Convert aperture into a descriptive string.
3250 *
3251 * @param[in] addressSpace
3252 *
3253 * @returns String
3254 *
3255 * @todo "text"
3256 */
3257 const char *
memdescGetApertureString(NV_ADDRESS_SPACE addressSpace)3258 memdescGetApertureString
3259 (
3260 NV_ADDRESS_SPACE addressSpace
3261 )
3262 {
3263 static NV_PRINTF_STRING_SECTION const char ADDR_FBMEM_STR[] = "VIDEO MEMORY";
3264 static NV_PRINTF_STRING_SECTION const char ADDR_SYSMEM_STR[] = "SYSTEM MEMORY";
3265
3266 if (addressSpace == ADDR_FBMEM)
3267 {
3268 return ADDR_FBMEM_STR;
3269 }
3270
3271 if (addressSpace == ADDR_SYSMEM)
3272 {
3273 return ADDR_SYSMEM_STR;
3274 }
3275
3276 return NULL;
3277 }
3278
3279 /*!
3280 * @brief Compare two memory descriptors to see if the memory described the same
3281 *
3282 * @param[in] pMemDescOne
3283 * @param[in] pMemDescTwo
3284 *
3285 * @returns NV_TRUE if the memory descriptors refer to the same memory
3286 */
3287 NvBool
memdescDescIsEqual(MEMORY_DESCRIPTOR * pMemDescOne,MEMORY_DESCRIPTOR * pMemDescTwo)3288 memdescDescIsEqual
3289 (
3290 MEMORY_DESCRIPTOR *pMemDescOne,
3291 MEMORY_DESCRIPTOR *pMemDescTwo
3292 )
3293 {
3294 if ((pMemDescOne == NULL) || (pMemDescTwo == NULL))
3295 return NV_FALSE;
3296
3297 if (pMemDescOne->_addressSpace != pMemDescTwo->_addressSpace)
3298 return NV_FALSE;
3299
3300 // All the physical memory views should match.
3301 if ((memdescGetPhysAddr(pMemDescOne, AT_CPU, 0) != memdescGetPhysAddr(pMemDescTwo, AT_CPU, 0)) ||
3302 (memdescGetPhysAddr(pMemDescOne, AT_GPU, 0) != memdescGetPhysAddr(pMemDescTwo, AT_GPU, 0)))
3303 return NV_FALSE;
3304
3305 if (memdescGetCpuCacheAttrib(pMemDescOne) != memdescGetCpuCacheAttrib(pMemDescTwo))
3306 return NV_FALSE;
3307
3308 if (pMemDescOne->Size != pMemDescTwo->Size)
3309 return NV_FALSE;
3310
3311 if (pMemDescOne->Alignment != pMemDescTwo->Alignment)
3312 return NV_FALSE;
3313
3314 if (pMemDescOne->_pageSize != pMemDescTwo->_pageSize)
3315 return NV_FALSE;
3316
3317 return NV_TRUE;
3318 }
3319
3320 /*!
3321 * @brief Add callback block to the destroy callback queue
3322 *
3323 * @param[in] pMemDesc Memory descriptor to update
3324 * @param[in] pCb Callee allocated block with callback func/arg
3325 *
3326 * @returns nothing
3327 */
3328 void
memdescAddDestroyCallback(MEMORY_DESCRIPTOR * pMemDesc,MEM_DESC_DESTROY_CALLBACK * pCb)3329 memdescAddDestroyCallback
3330 (
3331 MEMORY_DESCRIPTOR *pMemDesc,
3332 MEM_DESC_DESTROY_CALLBACK *pCb
3333 )
3334 {
3335 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3336 pCb->pNext = memdescGetDestroyCallbackList(pMemDesc);
3337 memdescSetDestroyCallbackList(pMemDesc, pCb);
3338 }
3339
3340 /*!
3341 * @brief Remove callback block from the destroy callback queue
3342 *
3343 * @param[in] pMemDesc Memory descriptor to update
3344 * @param[in] pRemoveCb Callee allocated block with callback func/arg
3345 *
3346 * @returns nothing
3347 */
3348 void
memdescRemoveDestroyCallback(MEMORY_DESCRIPTOR * pMemDesc,MEM_DESC_DESTROY_CALLBACK * pRemoveCb)3349 memdescRemoveDestroyCallback
3350 (
3351 MEMORY_DESCRIPTOR *pMemDesc,
3352 MEM_DESC_DESTROY_CALLBACK *pRemoveCb
3353 )
3354 {
3355 MEM_DESC_DESTROY_CALLBACK *pCb = memdescGetDestroyCallbackList(pMemDesc);
3356 MEM_DESC_DESTROY_CALLBACK *pPrev = NULL;
3357
3358 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3359 while (pCb)
3360 {
3361 if (pCb == pRemoveCb)
3362 {
3363 if (pPrev == NULL)
3364 {
3365 memdescSetDestroyCallbackList(pMemDesc, pCb->pNext);
3366 }
3367 else
3368 {
3369 pPrev->pNext = pCb->pNext;
3370 }
3371 break;
3372 }
3373 pPrev = pCb;
3374 pCb = pCb->pNext;
3375 }
3376 }
3377
3378 /*!
3379 * @brief Retrieves a subdevice's memory descriptor by subdevice instance
3380 *
3381 * Subdevice memory descriptors are memory descriptors that describe
3382 * per-subdevice memory buffers. This functionality is required by our current
3383 * SLI programming model as our memdescAlloc() calls are primarily broadcast
3384 * operations. A singular memdesc works for video memory as the
3385 * heaps are symmetric. However, we run into trouble when dealing with system
3386 * memory as both GPUs then share the same address space and symmetric
3387 * addressing is no longer possible.
3388 *
3389 * N.B. The rational for exposing this routine is that it keeps SLI-isms out of
3390 * most of the RM -- the alternative approach would've been to pass in the
3391 * subdevice or a pGpu for all memdesc methods which would require more code
3392 * changes solely for SLI. Long term hopefully we can transition to a unicast
3393 * allocation model (SLI loops above memdescAlloc()/memdescCreate()) and the
3394 * subdevice support in memdesc can (easily) be deleted. This approach also
3395 * provides a safety net against misuse, e.g., if we added pGpu to
3396 * memdescGetPhysAddr, current code which utilizes that routine outside an SLI loop
3397 * would execute cleanly even though it's incorrect.
3398 *
3399 * @param[in] pMemDesc Memory descriptor to query
3400 * @param[in] subDeviceInst SLI subdevice instance (subdevice - 1)
3401 *
3402 * @returns Memory descriptor if one exist for the subdevice.
3403 * NULL if none is found.
3404 */
3405 MEMORY_DESCRIPTOR *
memdescGetMemDescFromSubDeviceInst(MEMORY_DESCRIPTOR * pMemDesc,NvU32 subDeviceInst)3406 memdescGetMemDescFromSubDeviceInst(MEMORY_DESCRIPTOR *pMemDesc, NvU32 subDeviceInst)
3407 {
3408 if (!memdescHasSubDeviceMemDescs(pMemDesc))
3409 {
3410 return pMemDesc;
3411 }
3412 else
3413 {
3414 return memdescGetMemDescFromIndex(pMemDesc, subDeviceInst);
3415 }
3416 }
3417
3418 /*!
3419 * @brief Retrieves a subdevice's memory descriptor by GPU object
3420 *
3421 * See memdescGetMemDescFromSubDeviceInst for an explanation of subdevice memory
3422 * descriptors
3423 *
3424 * @param[in] pMemDesc Memory descriptor to query
3425 * @param[in] pGpu
3426 *
3427 * @returns Memory descriptor if one exist for the GPU.
3428 * NULL if none is found.
3429 */
3430 MEMORY_DESCRIPTOR *
memdescGetMemDescFromGpu(MEMORY_DESCRIPTOR * pMemDesc,OBJGPU * pGpu)3431 memdescGetMemDescFromGpu(MEMORY_DESCRIPTOR *pMemDesc, OBJGPU *pGpu)
3432 {
3433 NvU32 subDeviceInst = gpumgrGetSubDeviceInstanceFromGpu(pGpu);
3434
3435 return memdescGetMemDescFromSubDeviceInst(pMemDesc, subDeviceInst);
3436 }
3437
3438 /*!
3439 * @brief Retrieves a subdevice's memory descriptor by memdesc index.
3440 *
3441 * See memdescGetMemDescFromSubDeviceInst for an explanation of subdevice memory
3442 * descriptors
3443 *
3444 * @param[in] pMemDesc Memory descriptor to query
3445 * @param[in] index Index into array of memdesc
3446 *
3447 * @returns Memory descriptor if one exist for the GPU.
3448 * NULL if none is found.
3449 */
3450 MEMORY_DESCRIPTOR *
memdescGetMemDescFromIndex(MEMORY_DESCRIPTOR * pMemDesc,NvU32 index)3451 memdescGetMemDescFromIndex(MEMORY_DESCRIPTOR *pMemDesc, NvU32 index)
3452 {
3453 if (!memdescHasSubDeviceMemDescs(pMemDesc))
3454 {
3455 return pMemDesc;
3456 }
3457 else
3458 {
3459 MEMORY_DESCRIPTOR *pSubDevMemDesc = pMemDesc->_pNext;
3460
3461 NV_ASSERT(pSubDevMemDesc);
3462
3463 while (index--)
3464 {
3465 pSubDevMemDesc = pSubDevMemDesc->_pNext;
3466
3467 if (!pSubDevMemDesc)
3468 {
3469 NV_ASSERT(0);
3470 return NULL;
3471 }
3472 }
3473
3474 return pSubDevMemDesc;
3475 }
3476 }
3477
3478 /*!
3479 * @brief Set address for a fixed heap allocation.
3480 *
3481 * Offset must refer to the heap. A later memdescAlloc() will
3482 * force this offset.
3483 *
3484 * @param[in] pMemDesc Memory descriptor to update
3485 * @param[in] fbOffset Offset to refer to
3486 *
3487 * @returns nothing
3488 */
3489 void
memdescSetHeapOffset(MEMORY_DESCRIPTOR * pMemDesc,RmPhysAddr fbOffset)3490 memdescSetHeapOffset
3491 (
3492 MEMORY_DESCRIPTOR *pMemDesc,
3493 RmPhysAddr fbOffset
3494 )
3495 {
3496 NV_ASSERT(pMemDesc->_addressSpace == ADDR_FBMEM);
3497 NV_ASSERT(pMemDesc->Allocated == NV_FALSE);
3498
3499 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3500 pMemDesc->_flags |= MEMDESC_FLAGS_FIXED_ADDRESS_ALLOCATE;
3501 pMemDesc->_pteArray[0] = fbOffset;
3502 }
3503
3504 /*!
3505 * @brief Set GPU cacheability
3506 *
3507 * A later memdescAlloc() will use this setting.
3508 *
3509 * @param[in] pMemDesc Memory descriptor to update
3510 * @param[in] cacheAttrib Set memory to GPU cacheable
3511 *
3512 * @returns nothing
3513 */
memdescSetGpuCacheAttrib(MEMORY_DESCRIPTOR * pMemDesc,NvU32 cacheAttrib)3514 void memdescSetGpuCacheAttrib
3515 (
3516 MEMORY_DESCRIPTOR *pMemDesc,
3517 NvU32 cacheAttrib
3518 )
3519 {
3520 NV_ASSERT(pMemDesc->Allocated == NV_FALSE);
3521
3522 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3523 pMemDesc->_gpuCacheAttrib = cacheAttrib;
3524 }
3525
3526 /*!
3527 * @brief Get GPU P2P cache attributes
3528 *
3529 * @param[in] pMemDesc Memory descriptor pointer
3530 *
3531 * @returns Current GPU P2P cache attributes
3532 */
memdescGetGpuP2PCacheAttrib(MEMORY_DESCRIPTOR * pMemDesc)3533 NvU32 memdescGetGpuP2PCacheAttrib
3534 (
3535 MEMORY_DESCRIPTOR *pMemDesc
3536 )
3537 {
3538 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3539 return pMemDesc->_gpuP2PCacheAttrib;
3540 }
3541
3542 /*!
3543 * @brief Set GPU P2P cacheability
3544 *
3545 * A later memdescAlloc() will use this setting.
3546 *
3547 * @param[in] pMemDesc Memory descriptor to update
3548 * @param[in] cacheAttrib Set memory to GPU P2P cacheable
3549 *
3550 * @returns nothing
3551 */
memdescSetGpuP2PCacheAttrib(MEMORY_DESCRIPTOR * pMemDesc,NvU32 cacheAttrib)3552 void memdescSetGpuP2PCacheAttrib
3553 (
3554 MEMORY_DESCRIPTOR *pMemDesc,
3555 NvU32 cacheAttrib
3556 )
3557 {
3558 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3559 pMemDesc->_gpuP2PCacheAttrib = cacheAttrib;
3560 }
3561
3562 /*!
3563 * @brief Set CPU cacheability
3564 *
3565 * A later memdescAlloc() will use this setting.
3566 *
3567 * @param[in] pMemDesc Memory descriptor to update
3568 * @param[in] cacheAttrib Set memory to CPU cacheable
3569 *
3570 * @returns nothing
3571 */
memdescSetCpuCacheAttrib(MEMORY_DESCRIPTOR * pMemDesc,NvU32 cpuCacheAttrib)3572 void memdescSetCpuCacheAttrib
3573 (
3574 MEMORY_DESCRIPTOR *pMemDesc,
3575 NvU32 cpuCacheAttrib
3576 )
3577 {
3578 //
3579 // When running 64-bit MODS on ARM v8, we need to force all CPU mappings as WC.
3580 // This seems to be an issue with glibc. See bug 1556221.
3581 //
3582 // Ideally, this should have been set based on a Core Logic (CL) property.
3583 // But chipset initialization will only happen during bifStateInit().
3584 // RM can makes sysmem CPU mappings before bifStateInit().
3585 //
3586 if (RMCFG_FEATURE_PLATFORM_MODS && NVCPU_IS_AARCH64)
3587 {
3588 if (cpuCacheAttrib == NV_MEMORY_UNCACHED)
3589 {
3590 cpuCacheAttrib = NV_MEMORY_WRITECOMBINED;
3591 }
3592 }
3593
3594 pMemDesc->_cpuCacheAttrib = cpuCacheAttrib;
3595 }
3596
3597 /*!
3598 * @brief Print contents of a MEMORY_DESCRIPTOR in a human readable format.
3599 *
3600 * @param[in] pMemDesc Memory Descriptor to print
3601 * @param[in] bPrintIndividualPages Individual pages will also be printed
3602 * iff they are discontiguous
3603 * @param[in] pPrefixMessage Message that will be printed before the contents
3604 * of the Memory Descriptor are printed.
3605 *
3606 * @returns nothing
3607 */
memdescPrintMemdesc(MEMORY_DESCRIPTOR * pMemDesc,NvBool bPrintIndividualPages,const char * pPrefixMessage)3608 void memdescPrintMemdesc
3609 (
3610 MEMORY_DESCRIPTOR *pMemDesc,
3611 NvBool bPrintIndividualPages,
3612 const char *pPrefixMessage
3613 )
3614 {
3615 #if 0
3616 NvU32 i;
3617
3618 if ((DBG_RMMSG_CHECK(DBG_LEVEL_INFO) == 0) || (pPrefixMessage == NULL) || (pMemDesc == NULL))
3619 {
3620 return;
3621 }
3622
3623 NV_PRINTF(LEVEL_INFO,
3624 "%s Aperture %s starting at 0x%llx and of size 0x%llx\n",
3625 pPrefixMessage,
3626 memdescGetApertureString(pMemDesc->_addressSpace),
3627 memdescGetPhysAddr(pMemDesc, AT_CPU, 0),
3628 pMemDesc->Size);
3629
3630 if ((bPrintIndividualPages == NV_TRUE) &&
3631 (pMemDesc->PageCount > 1) &&
3632 (!(pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS)))
3633 {
3634 for (i = 0; i < pMemDesc->PageCount; i++)
3635 {
3636 NV_PRINTF(LEVEL_INFO,
3637 " contains page starting @0x%llx\n",
3638 pMemDesc->_pteArray[i]);
3639 }
3640 }
3641
3642 // TODO: merge with SMMU path above (see bug 1625121).
3643 if (pMemDesc->_pIommuMappings != NULL)
3644 {
3645 if (!memdescIsSubMemoryMemDesc(pMemDesc))
3646 {
3647 PIOVAMAPPING pIovaMap = pMemDesc->_pIommuMappings;
3648 while (pIovaMap != NULL)
3649 {
3650 NV_PRINTF(LEVEL_INFO,
3651 "Has additional IOMMU mapping for IOVA space 0x%x starting @ 0x%llx\n",
3652 pIovaMap->iovaspaceId,
3653 pIovaMap->iovaArray[0]);
3654 pIovaMap = pIovaMap->pNext;
3655 }
3656 }
3657 else
3658 {
3659 NV_PRINTF(LEVEL_INFO,
3660 "Has additional IOMMU mapping starting @ 0x%llx\n",
3661 memdescGetPhysAddr(pMemDesc, AT_PA, 0));
3662 }
3663 }
3664 #endif // NV_PRINTF_ENABLED
3665 }
3666
3667 /*!
3668 * @brief Return page offset from a MEMORY_DESCRIPTOR for an arbitrary power of two page size
3669 *
3670 * PageAdjust covers the 4KB alignment, but must include bits from the address for big pages.
3671 *
3672 * @param[in] pMemDesc Memory Descriptor to print
3673 * @param[in] pageSize Page size (4096, 64K, 128K, etc)
3674 *
3675 * @returns nothing
3676 */
memdescGetPageOffset(MEMORY_DESCRIPTOR * pMemDesc,NvU64 pageSize)3677 NvU64 memdescGetPageOffset
3678 (
3679 MEMORY_DESCRIPTOR *pMemDesc,
3680 NvU64 pageSize
3681 )
3682 {
3683 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3684 return (pMemDesc->PteAdjust + pMemDesc->_pteArray[0]) & (pageSize-1);
3685 }
3686
3687 /*!
3688 * @brief Get PTE kind using GPU
3689 *
3690 * @param[in] pMemDesc Memory descriptor pointer
3691 * @param[in] pGpu GPU to be used get supported kind
3692 * @param[in] addressTranslation Address translation identifier
3693 *
3694 * @returns Current PTE kind value.
3695 */
memdescGetPteKindForGpu(PMEMORY_DESCRIPTOR pMemDesc,OBJGPU * pGpu)3696 NvU32 memdescGetPteKindForGpu
3697 (
3698 PMEMORY_DESCRIPTOR pMemDesc,
3699 OBJGPU *pGpu
3700 )
3701 {
3702 return memmgrGetHwPteKindFromSwPteKind_HAL(pGpu, GPU_GET_MEMORY_MANAGER(pGpu), pMemDesc->_pteKind);
3703 }
3704
3705 /*!
3706 * @brief Set PTE kind using GPU.
3707 *
3708 * @param[in] pMemDesc Memory descriptor pointer
3709 * @param[in] pGpu GPU to be used set supported kind
3710 * @param[in] addressTranslation Address translation identifier
3711 * @param[in] pteKind New PTE kind
3712 *
3713 * @returns nothing
3714 */
memdescSetPteKindForGpu(PMEMORY_DESCRIPTOR pMemDesc,OBJGPU * pGpu,NvU32 pteKind)3715 void memdescSetPteKindForGpu
3716 (
3717 PMEMORY_DESCRIPTOR pMemDesc,
3718 OBJGPU *pGpu,
3719 NvU32 pteKind
3720 )
3721 {
3722 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3723 pMemDesc->_pteKind = memmgrGetSwPteKindFromHwPteKind_HAL(pGpu, GPU_GET_MEMORY_MANAGER(pGpu), pteKind);
3724 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_SET_KIND, NV_TRUE);
3725 }
3726
3727 /*!
3728 * @brief Set PTE kind compressed value.
3729 *
3730 * @param[in] pMemDesc Memory descriptor pointer
3731 * @param[in] pteKind New PTE kind compressed value
3732 *
3733 * @returns nothing
3734 */
memdescSetPteKindCompressed(PMEMORY_DESCRIPTOR pMemDesc,NvU32 pteKindCmpr)3735 void memdescSetPteKindCompressed
3736 (
3737 PMEMORY_DESCRIPTOR pMemDesc,
3738 NvU32 pteKindCmpr
3739 )
3740 {
3741 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3742 pMemDesc->_pteKindCompressed = pteKindCmpr;
3743 }
3744
3745 /*!
3746 * @brief Get PTE kind compressed value.
3747 *
3748 * @param[in] pMemDesc Memory descriptor pointer
3749 * @param[in] addressTranslation Address translation identifier
3750 *
3751 * @returns Current PTE kind compressed value.
3752 */
memdescGetPteKindCompressed(PMEMORY_DESCRIPTOR pMemDesc)3753 NvU32 memdescGetPteKindCompressed
3754 (
3755 PMEMORY_DESCRIPTOR pMemDesc
3756 )
3757 {
3758 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3759 return pMemDesc->_pteKindCompressed;
3760 }
3761
3762 /*!
3763 * @brief Get kernel mapping
3764 *
3765 * @param[in] pMemDesc Memory descriptor pointer
3766 *
3767 * @returns Current kernel mapping
3768 */
memdescGetKernelMapping(MEMORY_DESCRIPTOR * pMemDesc)3769 NvP64 memdescGetKernelMapping
3770 (
3771 MEMORY_DESCRIPTOR *pMemDesc
3772 )
3773 {
3774 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3775 return pMemDesc->_kernelMapping;
3776 }
3777
3778 /*!
3779 * @brief Set kernel mapping
3780 *
3781 * @param[in] pMemDesc Memory descriptor pointer
3782 * @param[in] kernelMapping New kernel mapping
3783 *
3784 * @returns nothing
3785 */
memdescSetKernelMapping(MEMORY_DESCRIPTOR * pMemDesc,NvP64 kernelMapping)3786 void memdescSetKernelMapping
3787 (
3788 MEMORY_DESCRIPTOR *pMemDesc,
3789 NvP64 kernelMapping
3790 )
3791 {
3792 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3793 pMemDesc->_kernelMapping = kernelMapping;
3794 }
3795
3796 /*!
3797 * @brief Get privileged kernel mapping
3798 *
3799 * @param[in] pMemDesc Memory descriptor pointer
3800 *
3801 * @returns Current privileged kernel mapping
3802 */
memdescGetKernelMappingPriv(MEMORY_DESCRIPTOR * pMemDesc)3803 NvP64 memdescGetKernelMappingPriv
3804 (
3805 MEMORY_DESCRIPTOR *pMemDesc
3806 )
3807 {
3808 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3809 return pMemDesc->_kernelMappingPriv;
3810 }
3811
3812 /*!
3813 * @brief Set HW resource identifier (HwResId)
3814 *
3815 * @param[in] pMemDesc Memory descriptor pointer
3816 * @param[in] kernelMappingPriv New privileged kernel mapping
3817 *
3818 * @returns nothing
3819 */
memdescSetKernelMappingPriv(MEMORY_DESCRIPTOR * pMemDesc,NvP64 kernelMappingPriv)3820 void memdescSetKernelMappingPriv
3821 (
3822 MEMORY_DESCRIPTOR *pMemDesc,
3823 NvP64 kernelMappingPriv
3824 )
3825 {
3826 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3827 pMemDesc->_kernelMappingPriv = kernelMappingPriv;
3828 }
3829
3830
3831 /*!
3832 * @brief Set standby buffer memory descriptor
3833 *
3834 * @param[in] pMemDesc Memory descriptor pointer
3835 *
3836 * @returns Pointer to standby buffer memory descriptor
3837 */
memdescGetStandbyBuffer(MEMORY_DESCRIPTOR * pMemDesc)3838 MEMORY_DESCRIPTOR *memdescGetStandbyBuffer
3839 (
3840 MEMORY_DESCRIPTOR *pMemDesc
3841 )
3842 {
3843 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3844 return pMemDesc->_pStandbyBuffer;
3845 }
3846
3847 /*!
3848 * @brief Set standby buffer memory descriptor
3849 *
3850 * @param[in] pMemDesc Memory descriptor pointer
3851 * @param[in] pStandbyBuffer Standby buffer memory descriptor pointer
3852 *
3853 * @returns nothing
3854 */
memdescSetStandbyBuffer(MEMORY_DESCRIPTOR * pMemDesc,MEMORY_DESCRIPTOR * pStandbyBuffer)3855 void memdescSetStandbyBuffer
3856 (
3857 MEMORY_DESCRIPTOR *pMemDesc,
3858 MEMORY_DESCRIPTOR *pStandbyBuffer
3859 )
3860 {
3861 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3862 pMemDesc->_pStandbyBuffer = pStandbyBuffer;
3863 }
3864
3865 /*!
3866 * @brief Set mem destroy callback list pointer
3867 *
3868 * @param[in] pMemDesc Memory descriptor pointer
3869 * @param[in] pMemDestroyCallbackList Memory destroy callback list pointer
3870 *
3871 * @returns nothing
3872 */
memdescSetDestroyCallbackList(MEMORY_DESCRIPTOR * pMemDesc,MEM_DESC_DESTROY_CALLBACK * pCb)3873 void memdescSetDestroyCallbackList
3874 (
3875 MEMORY_DESCRIPTOR *pMemDesc,
3876 MEM_DESC_DESTROY_CALLBACK *pCb
3877 )
3878 {
3879 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3880 pMemDesc->_pMemDestroyCallbackList = pCb;
3881 }
3882
3883 /*!
3884 * @brief Get guest ID for specified memory descriptor
3885 *
3886 * @param[in] pMemDesc Memory descriptor pointer
3887 *
3888 * @returns Guest ID value
3889 */
memdescGetGuestId(MEMORY_DESCRIPTOR * pMemDesc)3890 NvU64 memdescGetGuestId
3891 (
3892 MEMORY_DESCRIPTOR *pMemDesc
3893 )
3894 {
3895 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3896 return pMemDesc->_guestId;
3897 }
3898
3899 /*!
3900 * @brief Set guest ID for memory descriptor
3901 *
3902 * @param[in] pMemDesc Memory descriptor pointer
3903 * @param[in] guestId New guest ID
3904 *
3905 * @returns nothing
3906 */
memdescSetGuestId(MEMORY_DESCRIPTOR * pMemDesc,NvU64 guestId)3907 void memdescSetGuestId
3908 (
3909 MEMORY_DESCRIPTOR *pMemDesc,
3910 NvU64 guestId
3911 )
3912 {
3913 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3914 pMemDesc->_guestId = guestId;
3915 }
3916
3917 /*!
3918 * @brief Get value of specified flag
3919 *
3920 * @param[in] pMemDesc Memory descriptor pointer
3921 * @param[in] flag MEMDESC_FLAGS_* value
3922 *
3923 * @returns Boolean value of specified flag
3924 */
memdescGetFlag(MEMORY_DESCRIPTOR * pMemDesc,NvU64 flag)3925 NvBool memdescGetFlag
3926 (
3927 MEMORY_DESCRIPTOR *pMemDesc,
3928 NvU64 flag
3929 )
3930 {
3931 // For checking contiguity, use the memdescGetContiguity() api
3932 NV_ASSERT(flag != MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS);
3933 // GPU_IN_RESET is set/got from top level memdesc always.
3934 if (flag != MEMDESC_FLAGS_GPU_IN_RESET)
3935 {
3936 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3937 }
3938 return !!(pMemDesc->_flags & flag);
3939 }
3940
3941 /*!
3942 * @brief Set value of specified flag
3943 *
3944 * @param[in] pMemDesc Memory descriptor pointer
3945 * @param[in] flag MEMDESC_FLAGS_* value
3946 * @param[in] bValue Boolean value of flag
3947 *
3948 * @returns nothing
3949 */
memdescSetFlag(MEMORY_DESCRIPTOR * pMemDesc,NvU64 flag,NvBool bValue)3950 void memdescSetFlag
3951 (
3952 MEMORY_DESCRIPTOR *pMemDesc,
3953 NvU64 flag,
3954 NvBool bValue
3955 )
3956 {
3957 // For setting contiguity, use the memdescSetContiguity() api
3958 NV_ASSERT(flag != MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS);
3959
3960 // GPU_IN_RESET is set/got from top level memdesc always.
3961 if (flag != MEMDESC_FLAGS_GPU_IN_RESET)
3962 {
3963 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3964 }
3965
3966 if (flag == MEMDESC_FLAGS_OWNED_BY_CURRENT_DEVICE)
3967 {
3968 NV_ASSERT_OK(_memdescSetSubAllocatorFlag(pMemDesc->pGpu, pMemDesc, bValue));
3969 return;
3970 }
3971 else if (flag == MEMDESC_FLAGS_GUEST_ALLOCATED)
3972 {
3973 NV_ASSERT_OK(_memdescSetGuestAllocatedFlag(pMemDesc->pGpu, pMemDesc, bValue));
3974 return;
3975 }
3976
3977 if (bValue)
3978 pMemDesc->_flags |= flag;
3979 else
3980 pMemDesc->_flags &= ~flag;
3981 }
3982
3983 /*!
3984 * @brief Return memory descriptor address pointer
3985 *
3986 * The address value is returned by osAllocPages
3987 *
3988 * @param[in] pMemDesc Memory descriptor pointer
3989 *
3990 * @returns Memory descriptor address pointer
3991 */
memdescGetAddress(MEMORY_DESCRIPTOR * pMemDesc)3992 NvP64 memdescGetAddress
3993 (
3994 MEMORY_DESCRIPTOR *pMemDesc
3995 )
3996 {
3997 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
3998 return pMemDesc->_address;
3999 }
4000
4001 /*!
4002 * @brief Set memory descriptor address pointer
4003 *
4004 * The address value is returned by osAllocPages
4005 *
4006 * @param[in] pMemDesc Memory descriptor pointer
4007 * @param[in] pAddress Pointer to address information
4008 *
4009 * @returns nothing
4010 */
memdescSetAddress(MEMORY_DESCRIPTOR * pMemDesc,NvP64 pAddress)4011 void memdescSetAddress
4012 (
4013 MEMORY_DESCRIPTOR *pMemDesc,
4014 NvP64 pAddress
4015 )
4016 {
4017 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
4018 pMemDesc->_address = pAddress;
4019 }
4020
4021 /*!
4022 * @brief Get memory descriptor os-specific memory data pointer
4023 *
4024 * The pMemData value is returned by osAllocPages
4025 *
4026 * @param[in] pMemDesc Memory descriptor pointer
4027 *
4028 * @returns Memory data pointer
4029 */
memdescGetMemData(MEMORY_DESCRIPTOR * pMemDesc)4030 void *memdescGetMemData
4031 (
4032 MEMORY_DESCRIPTOR *pMemDesc
4033 )
4034 {
4035 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
4036 return pMemDesc->_pMemData;
4037 }
4038
4039 /*!
4040 * @brief Set memory descriptor os-specific memory data pointer
4041 *
4042 * The pMemData value is returned by osAllocPages
4043 *
4044 * @param[in] pMemDesc Memory descriptor pointer
4045 * @param[in] pMemData Pointer to new os-specific memory data
4046 * @param[in] pMemDataReleaseCallback Pointer to CB to be called when memdesc
4047 * is freed.
4048 *
4049 * @returns nothing
4050 */
memdescSetMemData(MEMORY_DESCRIPTOR * pMemDesc,void * pMemData,MEM_DATA_RELEASE_CALL_BACK * pMemDataReleaseCallback)4051 void memdescSetMemData
4052 (
4053 MEMORY_DESCRIPTOR *pMemDesc,
4054 void *pMemData,
4055 MEM_DATA_RELEASE_CALL_BACK *pMemDataReleaseCallback
4056 )
4057 {
4058 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
4059 pMemDesc->_pMemData = pMemData;
4060 pMemDesc->_pMemDataReleaseCallback = pMemDataReleaseCallback;
4061 }
4062
4063 /*!
4064 * @brief Return memory descriptor volatile attribute
4065 *
4066 * @param[in] pMemDesc Memory descriptor pointer
4067 *
4068 * @returns Volatile or not
4069 */
memdescGetVolatility(PMEMORY_DESCRIPTOR pMemDesc)4070 NvBool memdescGetVolatility
4071 (
4072 PMEMORY_DESCRIPTOR pMemDesc
4073 )
4074 {
4075 NvBool bVolatile = NV_FALSE;
4076
4077 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
4078 if (pMemDesc->_addressSpace == ADDR_SYSMEM)
4079 {
4080 bVolatile = (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED);
4081 }
4082 else
4083 {
4084 NV_ASSERT(pMemDesc->_addressSpace == ADDR_FBMEM);
4085 }
4086
4087 return bVolatile;
4088 }
4089
4090 /*!
4091 * @brief Quick check whether the memory is contiguous or not
4092 *
4093 * @param[in] pMemDesc Memory descriptor used
4094 * @param[in] addressTranslation Address translation identifier
4095 *
4096 * @returns NV_TRUE if contiguous
4097 */
memdescGetContiguity(PMEMORY_DESCRIPTOR pMemDesc,ADDRESS_TRANSLATION addressTranslation)4098 NvBool memdescGetContiguity(PMEMORY_DESCRIPTOR pMemDesc, ADDRESS_TRANSLATION addressTranslation)
4099 {
4100 return !!(pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS);
4101 }
4102
4103 /*!
4104 * @brief Detailed Check whether the memory is contiguous or not
4105 *
4106 * @param[in] pMemDesc Memory descriptor used
4107 * @param[in] addressTranslation Address translation identifier
4108 *
4109 * @returns NV_TRUE if contiguous
4110 */
memdescCheckContiguity(PMEMORY_DESCRIPTOR pMemDesc,ADDRESS_TRANSLATION addressTranslation)4111 NvBool memdescCheckContiguity(PMEMORY_DESCRIPTOR pMemDesc, ADDRESS_TRANSLATION addressTranslation)
4112 {
4113 NvU32 i;
4114
4115 if (!(pMemDesc->_flags & MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS))
4116 {
4117 for (i = 0; i < (pMemDesc->PageCount - 1); i++)
4118 {
4119 if ((memdescGetPte(pMemDesc, addressTranslation, i) + pMemDesc->pageArrayGranularity) !=
4120 memdescGetPte(pMemDesc, addressTranslation, i + 1))
4121 return NV_FALSE;
4122 }
4123 }
4124
4125 return NV_TRUE;
4126 }
4127
4128 /*!
4129 * @brief Set the contiguity of the memory descriptor
4130 *
4131 * @param[in] pMemDesc Memory descriptor used
4132 * @param[in] addressTranslation Address translation identifier
4133 * @param[in] isContiguous Contiguity value
4134 *
4135 * @returns nothing
4136 */
memdescSetContiguity(PMEMORY_DESCRIPTOR pMemDesc,ADDRESS_TRANSLATION addressTranslation,NvBool isContiguous)4137 void memdescSetContiguity(PMEMORY_DESCRIPTOR pMemDesc, ADDRESS_TRANSLATION addressTranslation, NvBool isContiguous)
4138 {
4139 NV_ASSERT_OR_RETURN_VOID(pMemDesc);
4140
4141 if (isContiguous)
4142 pMemDesc->_flags |= MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS;
4143 else
4144 pMemDesc->_flags &= ~MEMDESC_FLAGS_PHYSICALLY_CONTIGUOUS;
4145 }
4146
4147 /*!
4148 * @brief Get the address space of the memory descriptor
4149 *
4150 * @param[in] pMemDesc Memory descriptor used
4151 * @param[in] addressTranslation Address translation identifier
4152 *
4153 * @returns addresspace
4154 */
memdescGetAddressSpace(PMEMORY_DESCRIPTOR pMemDesc)4155 NV_ADDRESS_SPACE memdescGetAddressSpace(PMEMORY_DESCRIPTOR pMemDesc)
4156 {
4157 NV_ASSERT_OR_RETURN(pMemDesc != NULL, 0);
4158 return pMemDesc->_addressSpace;
4159 }
4160
4161 /*!
4162 * @brief Get page size
4163 *
4164 * @param[in] pMemDesc Memory descriptor pointer
4165 * @param[in] addressTranslation Address translation identifier
4166 *
4167 * @returns Current page size.
4168 */
memdescGetPageSize(PMEMORY_DESCRIPTOR pMemDesc,ADDRESS_TRANSLATION addressTranslation)4169 NvU64 memdescGetPageSize
4170 (
4171 PMEMORY_DESCRIPTOR pMemDesc,
4172 ADDRESS_TRANSLATION addressTranslation
4173 )
4174 {
4175 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
4176 return pMemDesc->_pageSize;
4177 }
4178
4179 /*!
4180 * @brief Set page size
4181 *
4182 * @param[in] pMemDesc Memory descriptor pointer
4183 * @param[in] addressTranslation Address translation identifier
4184 * @param[in] pteKind New PTE kind
4185 *
4186 * @returns nothing
4187 */
memdescSetPageSize(PMEMORY_DESCRIPTOR pMemDesc,ADDRESS_TRANSLATION addressTranslation,NvU64 pageSize)4188 void memdescSetPageSize
4189 (
4190 PMEMORY_DESCRIPTOR pMemDesc,
4191 ADDRESS_TRANSLATION addressTranslation,
4192 NvU64 pageSize
4193 )
4194 {
4195 NV_ASSERT(!memdescHasSubDeviceMemDescs(pMemDesc));
4196 pMemDesc->_pageSize = pageSize;
4197 }
4198
4199 /*!
4200 * @brief Get the Root memory descriptor.
4201 *
4202 * This can also be used to get the root offset as well.
4203 *
4204 * Root memory descriptor is the top level memory descriptor with no parent,
4205 * from which this memory descriptor was derived
4206 *
4207 * @param[in] pMemDesc Pointer to memory descriptor.
4208 * @param[out] pRootOffset Pointer to the root offset parameter.
4209 *
4210 * @returns the Root memory descriptor.
4211 */
memdescGetRootMemDesc(PMEMORY_DESCRIPTOR pMemDesc,NvU64 * pRootOffset)4212 PMEMORY_DESCRIPTOR memdescGetRootMemDesc
4213 (
4214 PMEMORY_DESCRIPTOR pMemDesc,
4215 NvU64 *pRootOffset
4216 )
4217 {
4218 NvU64 offset = 0;
4219
4220 // Find the top-level parent descriptor
4221 while (pMemDesc->_pParentDescriptor)
4222 {
4223 // Sanity check, None of the child descriptors should be allocated
4224 NV_ASSERT(!pMemDesc->Allocated);
4225 offset += pMemDesc->subMemOffset;
4226 pMemDesc = pMemDesc->_pParentDescriptor;
4227 }
4228
4229 if (pRootOffset)
4230 {
4231 *pRootOffset = offset;
4232 }
4233
4234 return pMemDesc;
4235 }
4236 /*!
4237 * @brief Sets the CUSTOM_HEAP flag of MEMDESC.
4238 *
4239 * Since we have ACR region, Memory descriptor can be allocated in ACR region
4240 * in that case, we need to set this flag since we are using the custom ACR HEAP
4241 *
4242 * @param[in] pMemDesc Pointer to memory descriptor.
4243 *
4244 * @returns void.
4245 */
4246 void
memdescSetCustomHeap(PMEMORY_DESCRIPTOR pMemDesc)4247 memdescSetCustomHeap
4248 (
4249 PMEMORY_DESCRIPTOR pMemDesc
4250 )
4251 {
4252 NV_ASSERT(0);
4253 }
4254
4255 /*!
4256 * @brief Returns the ACR CUSTOM_HEAP flag.
4257 *
4258 *
4259 * @param[in] pMemDesc Pointer to memory descriptor.
4260 *
4261 * @returns NV_TRUE if flag MEMDESC_FLAGS_CUSTOM_HEAP_ACR is SET.
4262 */
4263 NvBool
memdescGetCustomHeap(PMEMORY_DESCRIPTOR pMemDesc)4264 memdescGetCustomHeap
4265 (
4266 PMEMORY_DESCRIPTOR pMemDesc
4267 )
4268 {
4269 return NV_FALSE;
4270 }
4271
memdescGetIommuMap(PMEMORY_DESCRIPTOR pMemDesc,NvU32 iovaspaceId)4272 PIOVAMAPPING memdescGetIommuMap
4273 (
4274 PMEMORY_DESCRIPTOR pMemDesc,
4275 NvU32 iovaspaceId
4276 )
4277 {
4278 PIOVAMAPPING pIommuMap = pMemDesc->_pIommuMappings;
4279 while (pIommuMap != NULL)
4280 {
4281 if (pIommuMap->iovaspaceId == iovaspaceId)
4282 {
4283 break;
4284 }
4285
4286 pIommuMap = pIommuMap->pNext;
4287 }
4288
4289 return pIommuMap;
4290 }
4291
memdescAddIommuMap(PMEMORY_DESCRIPTOR pMemDesc,PIOVAMAPPING pIommuMap)4292 NV_STATUS memdescAddIommuMap
4293 (
4294 PMEMORY_DESCRIPTOR pMemDesc,
4295 PIOVAMAPPING pIommuMap
4296 )
4297 {
4298 NV_ASSERT_OR_RETURN((pMemDesc->_pIommuMappings == NULL) ||
4299 (!memdescIsSubMemoryMemDesc(pMemDesc)), NV_ERR_INVALID_ARGUMENT);
4300
4301 //
4302 // Only root physical memdescs can have multiple IOMMU mappings.
4303 // Submemdescs can only have one, and the list linkage is used
4304 // instead to link it as a child of the root IOMMU mapping, so we
4305 // don't want to overwrite that here.
4306 //
4307 if (!memdescIsSubMemoryMemDesc(pMemDesc))
4308 {
4309 pIommuMap->pNext = pMemDesc->_pIommuMappings;
4310 }
4311
4312 pMemDesc->_pIommuMappings = pIommuMap;
4313
4314 return NV_OK;
4315 }
4316
memdescRemoveIommuMap(PMEMORY_DESCRIPTOR pMemDesc,PIOVAMAPPING pIommuMap)4317 void memdescRemoveIommuMap
4318 (
4319 PMEMORY_DESCRIPTOR pMemDesc,
4320 PIOVAMAPPING pIommuMap
4321 )
4322 {
4323 //
4324 // Only root physical memdescs can have multiple IOMMU mappings.
4325 // Submemdescs can only have one, and the list linkage is used
4326 // instead to link it as a child of the root IOMMU mapping, so we
4327 // don't want to overwrite that here.
4328 //
4329 if (!memdescIsSubMemoryMemDesc(pMemDesc))
4330 {
4331 PIOVAMAPPING *ppTmpIommuMap = &pMemDesc->_pIommuMappings;
4332 while ((*ppTmpIommuMap != NULL) && (*ppTmpIommuMap != pIommuMap))
4333 {
4334 ppTmpIommuMap = &(*ppTmpIommuMap)->pNext;
4335 }
4336
4337 if (*ppTmpIommuMap != NULL)
4338 {
4339 *ppTmpIommuMap = pIommuMap->pNext;
4340
4341 }
4342 else
4343 {
4344 NV_ASSERT(*ppTmpIommuMap != NULL);
4345 }
4346 }
4347 else if (pMemDesc->_pIommuMappings == pIommuMap)
4348 {
4349 pMemDesc->_pIommuMappings = NULL;
4350 }
4351 else
4352 {
4353 //
4354 // Trying to remove a submemory mapping that doesn't belong to this
4355 // descriptor?
4356 //
4357 NV_ASSERT(pMemDesc->_pIommuMappings == pIommuMap);
4358 }
4359 }
4360
memdescMapIommu(PMEMORY_DESCRIPTOR pMemDesc,NvU32 iovaspaceId)4361 NV_STATUS memdescMapIommu
4362 (
4363 PMEMORY_DESCRIPTOR pMemDesc,
4364 NvU32 iovaspaceId
4365 )
4366 {
4367 #if (RMCFG_FEATURE_PLATFORM_UNIX || RMCFG_FEATURE_PLATFORM_MODS) && !NVCPU_IS_ARM
4368 if (iovaspaceId != NV_IOVA_DOMAIN_NONE)
4369 {
4370 NV_ADDRESS_SPACE addrSpace = memdescGetAddressSpace(pMemDesc);
4371 OBJGPU *pMappingGpu = gpumgrGetGpuFromId(iovaspaceId);
4372 PMEMORY_DESCRIPTOR pRootMemDesc = memdescGetRootMemDesc(pMemDesc, NULL);
4373 if ((addrSpace == ADDR_SYSMEM) || gpumgrCheckIndirectPeer(pMappingGpu, pRootMemDesc->pGpu))
4374 {
4375 NV_STATUS status;
4376 OBJIOVASPACE *pIOVAS = iovaspaceFromId(iovaspaceId);
4377 NV_ASSERT_OR_RETURN(pIOVAS, NV_ERR_OBJECT_NOT_FOUND);
4378
4379 status = iovaspaceAcquireMapping(pIOVAS, pMemDesc);
4380 NV_ASSERT_OR_RETURN(status == NV_OK, status);
4381 }
4382 }
4383 #endif
4384
4385 //
4386 // Verify that the final physical addresses are indeed addressable by the
4387 // GPU. We only need to do this for internally allocated sysmem (RM owned)
4388 // as well as externally allocated/mapped sysmem. Note, addresses for peer
4389 // (P2P mailbox registers) BARs are actually not handled by the GMMU and
4390 // support a full 64-bit address width, hence validation is not needed.
4391 //
4392 if ((pMemDesc->Allocated ||
4393 memdescGetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM) ||
4394 memdescGetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM)) &&
4395 memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM)
4396 {
4397 // TODO This should look up the GPU corresponding to the IOVAS instead.
4398 OBJGPU *pGpu = pMemDesc->pGpu;
4399 RmPhysAddr dmaWindowStartAddr = gpuGetDmaStartAddress(pGpu);
4400 RmPhysAddr dmaWindowEndAddr = gpuGetDmaEndAddress_HAL(pGpu);
4401 RmPhysAddr physAddr;
4402
4403 if (memdescGetContiguity(pMemDesc, AT_GPU))
4404 {
4405 physAddr = memdescGetPhysAddr(pMemDesc, AT_GPU, 0);
4406 if ((physAddr < dmaWindowStartAddr) ||
4407 (physAddr + pMemDesc->Size - 1 > dmaWindowEndAddr))
4408 {
4409 NV_PRINTF(LEVEL_ERROR,
4410 "0x%llx-0x%llx is not addressable by GPU 0x%x [0x%llx-0x%llx]\n",
4411 physAddr, physAddr + pMemDesc->Size - 1,
4412 pGpu->gpuId, dmaWindowStartAddr, dmaWindowEndAddr);
4413 memdescUnmapIommu(pMemDesc, iovaspaceId);
4414 return NV_ERR_INVALID_ADDRESS;
4415 }
4416 }
4417 else
4418 {
4419 NvU32 i;
4420 for (i = 0; i < pMemDesc->PageCount; i++)
4421 {
4422 physAddr = memdescGetPte(pMemDesc, AT_GPU, i);
4423 if ((physAddr < dmaWindowStartAddr) ||
4424 (physAddr + (pMemDesc->pageArrayGranularity - 1) > dmaWindowEndAddr))
4425 {
4426 NV_PRINTF(LEVEL_ERROR,
4427 "0x%llx is not addressable by GPU 0x%x [0x%llx-0x%llx]\n",
4428 physAddr, pGpu->gpuId, dmaWindowStartAddr,
4429 dmaWindowEndAddr);
4430 memdescUnmapIommu(pMemDesc, iovaspaceId);
4431 return NV_ERR_INVALID_ADDRESS;
4432 }
4433 }
4434 }
4435 }
4436
4437 return NV_OK;
4438 }
4439
memdescUnmapIommu(PMEMORY_DESCRIPTOR pMemDesc,NvU32 iovaspaceId)4440 void memdescUnmapIommu
4441 (
4442 PMEMORY_DESCRIPTOR pMemDesc,
4443 NvU32 iovaspaceId
4444 )
4445 {
4446 #if (RMCFG_FEATURE_PLATFORM_UNIX || RMCFG_FEATURE_PLATFORM_MODS) && !NVCPU_IS_ARM
4447 PIOVAMAPPING pIovaMapping;
4448 OBJIOVASPACE *pIOVAS;
4449
4450 if (iovaspaceId == NV_IOVA_DOMAIN_NONE)
4451 return;
4452
4453 pIovaMapping = memdescGetIommuMap(pMemDesc, iovaspaceId);
4454 NV_ASSERT(pIovaMapping);
4455
4456 pIOVAS = iovaspaceFromMapping(pIovaMapping);
4457 iovaspaceReleaseMapping(pIOVAS, pIovaMapping);
4458 #endif
4459 }
4460
memdescCheckSubDevicePageSizeConsistency(OBJGPU * pGpu,PMEMORY_DESCRIPTOR pMemDesc,OBJVASPACE * pVAS,NvU64 pageSize,NvU64 pageOffset)4461 void memdescCheckSubDevicePageSizeConsistency
4462 (
4463 OBJGPU *pGpu,
4464 PMEMORY_DESCRIPTOR pMemDesc,
4465 OBJVASPACE *pVAS,
4466 NvU64 pageSize,
4467 NvU64 pageOffset
4468 )
4469 {
4470 NvU64 tempPageSize, tempPageOffset;
4471 PMEMORY_DESCRIPTOR pTempMemDesc = NULL;
4472
4473 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
4474 pTempMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu);
4475 tempPageSize = memdescGetPageSize(pTempMemDesc, VAS_ADDRESS_TRANSLATION(pVAS));
4476 tempPageOffset = memdescGetPhysAddr(pTempMemDesc, VAS_ADDRESS_TRANSLATION(pVAS), 0) & (tempPageSize - 1);
4477
4478 // Assert if inconsistent
4479 NV_ASSERT(pageSize == tempPageSize);
4480 NV_ASSERT(pageOffset == tempPageOffset);
4481 SLI_LOOP_END
4482 }
4483
memdescCheckSubDeviceMemContiguityConsistency(OBJGPU * pGpu,PMEMORY_DESCRIPTOR pMemDesc,OBJVASPACE * pVAS,NvBool bIsMemContiguous)4484 void memdescCheckSubDeviceMemContiguityConsistency
4485 (
4486 OBJGPU *pGpu,
4487 PMEMORY_DESCRIPTOR pMemDesc,
4488 OBJVASPACE *pVAS,
4489 NvBool bIsMemContiguous
4490 )
4491 {
4492 NvBool bTempIsMemContiguous = NV_FALSE;
4493
4494 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
4495 bTempIsMemContiguous = memdescGetContiguity(memdescGetMemDescFromGpu(pMemDesc, pGpu), VAS_ADDRESS_TRANSLATION(pVAS));
4496 // Assert if inconsistent
4497 NV_ASSERT(bIsMemContiguous == bTempIsMemContiguous);
4498 SLI_LOOP_END
4499 }
4500
memdescCheckSubDeviceKindComprConsistency(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc,OBJVASPACE * pVAS,NvU32 kind,COMPR_INFO * pComprInfo)4501 NV_STATUS memdescCheckSubDeviceKindComprConsistency
4502 (
4503 OBJGPU *pGpu,
4504 MEMORY_DESCRIPTOR *pMemDesc,
4505 OBJVASPACE *pVAS,
4506 NvU32 kind,
4507 COMPR_INFO *pComprInfo
4508 )
4509 {
4510 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
4511 {
4512 MemoryManager *MemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
4513 NvU32 tempKind;
4514 COMPR_INFO tempComprInfo = {0};
4515 NV_STATUS status;
4516
4517 status = memmgrGetKindComprFromMemDesc(MemoryManager,
4518 memdescGetMemDescFromGpu(pMemDesc, pGpu),
4519 0,
4520 &tempKind, &tempComprInfo);
4521
4522 if (NV_OK != status)
4523 SLI_LOOP_RETURN(status);
4524
4525 // Assert if inconsistent
4526 NV_ASSERT(kind == tempKind);
4527 NV_ASSERT(tempComprInfo.compPageShift == pComprInfo->compPageShift &&
4528 tempComprInfo.kind == pComprInfo->kind &&
4529 tempComprInfo.compPageIndexLo == pComprInfo->compPageIndexLo &&
4530 tempComprInfo.compPageIndexHi == pComprInfo->compPageIndexHi &&
4531 tempComprInfo.compTagLineMin == pComprInfo->compTagLineMin &&
4532 tempComprInfo.compTagLineMultiplier == pComprInfo->compTagLineMultiplier);
4533 }
4534 SLI_LOOP_END
4535
4536 return NV_OK;
4537 }
4538
4539 /* @brief Get GPA(guest physical addresses) for given GPU physical addresses.
4540 *
4541 * @param[in] pGpu GPU for which GPAs are needed
4542 * @param[in] pageCount Size of array. Should be 1 for contiguous mappings
4543 * @param[in/out] pGpa Array of GPU PAs to be converted to guest PAs
4544 *
4545 * @returns NV_STATUS
4546 */
memdescGetNvLinkGpa(OBJGPU * pGpu,NvU64 pageCount,RmPhysAddr * pGpa)4547 NV_STATUS memdescGetNvLinkGpa
4548 (
4549 OBJGPU *pGpu,
4550 NvU64 pageCount,
4551 RmPhysAddr *pGpa
4552 )
4553 {
4554 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
4555
4556 NV_ASSERT_OR_RETURN(pGpa, NV_ERR_INVALID_ARGUMENT);
4557
4558 NvU32 pageIndex;
4559 // For each page, do the GPU PA to GPA conversion
4560 for (pageIndex = 0; pageIndex < pageCount; pageIndex++)
4561 {
4562 pGpa[pageIndex] += pKernelMemorySystem->coherentCpuFbBase;
4563 }
4564
4565 return NV_OK;
4566 }
4567
4568 NV_STATUS
memdescSetCtxBufPool(PMEMORY_DESCRIPTOR pMemDesc,CTX_BUF_POOL_INFO * pCtxBufPool)4569 memdescSetCtxBufPool
4570 (
4571 PMEMORY_DESCRIPTOR pMemDesc,
4572 CTX_BUF_POOL_INFO *pCtxBufPool
4573 )
4574 {
4575
4576 NV_ASSERT_OR_RETURN(!pMemDesc->Allocated, NV_ERR_INVALID_STATE);
4577 NV_ASSERT_OR_RETURN(!memdescHasSubDeviceMemDescs(pMemDesc), NV_ERR_INVALID_ARGUMENT);
4578
4579 pMemDesc->pCtxBufPool = pCtxBufPool;
4580 return NV_OK;
4581 }
4582
4583 CTX_BUF_POOL_INFO*
memdescGetCtxBufPool(PMEMORY_DESCRIPTOR pMemDesc)4584 memdescGetCtxBufPool
4585 (
4586 PMEMORY_DESCRIPTOR pMemDesc
4587 )
4588 {
4589 NV_ASSERT_OR_RETURN(!memdescHasSubDeviceMemDescs(pMemDesc), NULL);
4590 return pMemDesc->pCtxBufPool;
4591 }
4592
4593 /*!
4594 * @brief Override the registry INST_LOC two-bit enum to an aperture (list) + cpu attr.
4595 *
4596 * Caller must set initial default values.
4597 */
4598 void
memdescOverrideInstLocList(NvU32 instLoc,const char * name,const NV_ADDRESS_SPACE ** ppAllocList,NvU32 * pCpuMappingAttr)4599 memdescOverrideInstLocList
4600 (
4601 NvU32 instLoc, // NV_REG_STR_RM_INST_LOC
4602 const char *name,
4603 const NV_ADDRESS_SPACE **ppAllocList,
4604 NvU32 *pCpuMappingAttr
4605 )
4606 {
4607 switch (instLoc)
4608 {
4609 case NV_REG_STR_RM_INST_LOC_COH:
4610 NV_PRINTF(LEVEL_INFO, "using coh system memory for %s\n", name);
4611 *ppAllocList = ADDRLIST_SYSMEM_ONLY;
4612 *pCpuMappingAttr = NV_MEMORY_CACHED;
4613 break;
4614 case NV_REG_STR_RM_INST_LOC_NCOH:
4615 NV_PRINTF(LEVEL_INFO, "using ncoh system memory for %s\n", name);
4616 *ppAllocList = ADDRLIST_SYSMEM_ONLY;
4617 *pCpuMappingAttr = NV_MEMORY_UNCACHED;
4618 break;
4619 case NV_REG_STR_RM_INST_LOC_VID:
4620 NV_PRINTF(LEVEL_INFO, "using video memory for %s\n", name);
4621 *ppAllocList = ADDRLIST_FBMEM_ONLY;
4622 *pCpuMappingAttr = NV_MEMORY_WRITECOMBINED;
4623 break;
4624 case NV_REG_STR_RM_INST_LOC_DEFAULT:
4625 default:
4626 // Do not update parameters
4627 break;
4628 }
4629 }
4630
4631 /*!
4632 * @brief Override wrapper for callers needed an aperture
4633 */
4634 void
memdescOverrideInstLoc(NvU32 instLoc,const char * name,NV_ADDRESS_SPACE * pAddrSpace,NvU32 * pCpuMappingAttr)4635 memdescOverrideInstLoc
4636 (
4637 NvU32 instLoc,
4638 const char *name,
4639 NV_ADDRESS_SPACE *pAddrSpace,
4640 NvU32 *pCpuMappingAttr
4641 )
4642 {
4643 const NV_ADDRESS_SPACE *pAllocList = NULL;
4644
4645 memdescOverrideInstLocList(instLoc, name, &pAllocList, pCpuMappingAttr);
4646 if (pAllocList != NULL)
4647 *pAddrSpace = pAllocList[0];
4648 }
4649 /*!
4650 * @brief override physical address width
4651 *
4652 * address width to be override in bits.
4653 * @param[in] pGpu
4654 * @param[in] pMemDesc Memory descriptor to update
4655 * @param[in] addresswidth Offset to refer to
4656 *
4657 * @returns nothing
4658 */
4659 void
memdescOverridePhysicalAddressWidthWindowsWAR(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc,NvU32 addressWidth)4660 memdescOverridePhysicalAddressWidthWindowsWAR
4661 (
4662 OBJGPU *pGpu,
4663 MEMORY_DESCRIPTOR *pMemDesc,
4664 NvU32 addressWidth
4665 )
4666 {
4667 if (RMCFG_FEATURE_PLATFORM_WINDOWS)
4668 {
4669 if (addressWidth < gpuGetPhysAddrWidth_HAL(pGpu, ADDR_SYSMEM))
4670 {
4671 pMemDesc->_flags |= MEMDESC_FLAGS_OVERRIDE_SYSTEM_ADDRESS_LIMIT;
4672 pMemDesc->_overridenAddressWidth = addressWidth;
4673 }
4674 }
4675 }
4676
4677 /*!
4678 * @brief Register MEMDESC to GSP
4679 * Life of the registration: until memdescDeregisterFromGSP is called,
4680 * always occurs when the memory is freed.
4681 * <GSP-TODO> Have argument as pMemory*; Move to NVOC
4682 *
4683 * @param[in] pGpu
4684 * @param[in] hClient NvHandle
4685 * @param[in] hDevice NvHandle
4686 * @param[in] hMemory NvHandle
4687 *
4688 * @returns NV_STATUS
4689 */
4690 NV_STATUS
memdescRegisterToGSP(OBJGPU * pGpu,NvHandle hClient,NvHandle hParent,NvHandle hMemory)4691 memdescRegisterToGSP
4692 (
4693 OBJGPU *pGpu,
4694 NvHandle hClient,
4695 NvHandle hParent,
4696 NvHandle hMemory
4697 )
4698 {
4699 NV_STATUS status = NV_OK;
4700 Memory *pMemory = NULL;
4701 RsResourceRef *pMemoryRef = NULL;
4702 MEMORY_DESCRIPTOR *pMemDesc = NULL;
4703 NvU32 hClass;
4704
4705 // Nothing to do without GSP
4706 if (!IS_GSP_CLIENT(pGpu))
4707 {
4708 return NV_OK;
4709 }
4710
4711 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, serverutilGetResourceRef(hClient, hMemory, &pMemoryRef));
4712
4713 pMemory = dynamicCast(pMemoryRef->pResource, Memory);
4714 NV_CHECK_OR_RETURN(LEVEL_ERROR, pMemory != NULL, NV_ERR_INVALID_OBJECT);
4715
4716 pMemDesc = pMemory->pMemDesc;
4717
4718 // Check: memory already registered
4719 if ((pMemDesc->_flags & MEMDESC_FLAGS_REGISTERED_TO_GSP) != 0)
4720 {
4721 return NV_OK;
4722 }
4723
4724 // Check: no subdevice memDescs
4725 NV_CHECK_OR_RETURN(LEVEL_ERROR,
4726 !memdescHasSubDeviceMemDescs(pMemDesc),
4727 NV_ERR_INVALID_STATE);
4728
4729 // Check: SYSMEM or FBMEM only
4730 if (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)
4731 hClass = NV01_MEMORY_LIST_FBMEM;
4732 else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM)
4733 hClass = NV01_MEMORY_LIST_SYSTEM;
4734 else
4735 return NV_ERR_INVALID_STATE;
4736
4737 NvU32 os02Flags = 0;
4738
4739 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
4740 RmDeprecatedConvertOs32ToOs02Flags(pMemory->Attr,
4741 pMemory->Attr2,
4742 pMemory->Flags,
4743 &os02Flags));
4744 NV_RM_RPC_ALLOC_MEMORY(pGpu,
4745 hClient,
4746 hParent,
4747 hMemory,
4748 hClass,
4749 os02Flags,
4750 pMemDesc,
4751 status);
4752
4753 if (status == NV_OK)
4754 {
4755 // Mark memory as registered in GSP
4756 pMemDesc->_flags |= MEMDESC_FLAGS_REGISTERED_TO_GSP;
4757 }
4758
4759 return status;
4760 }
4761
4762
4763 /*!
4764 * @brief Deregister MEMDESC from GSP
4765 * Is always called when the memory is freed.
4766 * <GSP-TODO> Have argument as pMemory*; Move to NVOC
4767 *
4768 * @param[in] pGpu
4769 * @param[in] hClient NvHandle
4770 * @param[in] hParent NvHandle
4771 * @param[in] hMemory NvHandle
4772 *
4773 * @returns NV_STATUS
4774 */
4775 NV_STATUS
memdescDeregisterFromGSP(OBJGPU * pGpu,NvHandle hClient,NvHandle hParent,NvHandle hMemory)4776 memdescDeregisterFromGSP
4777 (
4778 OBJGPU *pGpu,
4779 NvHandle hClient,
4780 NvHandle hParent,
4781 NvHandle hMemory
4782 )
4783 {
4784 NV_STATUS status = NV_OK;
4785 Memory *pMemory = NULL;
4786 RsResourceRef *pMemoryRef = NULL;
4787 MEMORY_DESCRIPTOR *pMemDesc = NULL;
4788
4789 // Nothing to do without GSP
4790 if ((pGpu == NULL) ||
4791 !IS_GSP_CLIENT(pGpu))
4792 {
4793 return NV_OK;
4794 }
4795
4796 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, serverutilGetResourceRef(hClient, hMemory, &pMemoryRef));
4797
4798 pMemory = dynamicCast(pMemoryRef->pResource, Memory);
4799 NV_CHECK_OR_RETURN(LEVEL_ERROR, pMemory != NULL, NV_ERR_INVALID_OBJECT);
4800
4801 pMemDesc = pMemory->pMemDesc;
4802
4803 // Nothing to do if memory is not registered to GSP
4804 if ((pMemDesc == NULL) ||
4805 (pMemDesc->_flags & MEMDESC_FLAGS_REGISTERED_TO_GSP) == 0)
4806 {
4807 return NV_OK;
4808 }
4809
4810 NV_RM_RPC_FREE(pGpu,
4811 hClient,
4812 hParent,
4813 hMemory,
4814 status);
4815
4816 if (status == NV_OK)
4817 {
4818 // Mark memory as not registered in GSP
4819 pMemDesc->_flags &= ~MEMDESC_FLAGS_REGISTERED_TO_GSP;
4820 }
4821
4822 return status;
4823 }
4824
4825 void
memdescSetName(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc,const char * name,const char * suffix)4826 memdescSetName(OBJGPU *pGpu, MEMORY_DESCRIPTOR *pMemDesc, const char *name, const char* suffix)
4827 {
4828 return;
4829 }
4830
4831 NV_STATUS
memdescSendMemDescToGSP(OBJGPU * pGpu,MEMORY_DESCRIPTOR * pMemDesc,NvHandle * pHandle)4832 memdescSendMemDescToGSP(OBJGPU *pGpu, MEMORY_DESCRIPTOR *pMemDesc, NvHandle *pHandle)
4833 {
4834 NV_STATUS status = NV_OK;
4835 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
4836 NvU32 flags = 0;
4837 NvU32 index = 0;
4838 NvU32 hClass;
4839 NvU64 *pageNumberList = NULL;
4840 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
4841 NV_MEMORY_LIST_ALLOCATION_PARAMS listAllocParams = {0};
4842
4843 // Nothing to do without GSP
4844 if (!IS_GSP_CLIENT(pGpu))
4845 {
4846 return NV_OK;
4847 }
4848
4849 switch (memdescGetAddressSpace(pMemDesc))
4850 {
4851
4852 case ADDR_FBMEM:
4853 hClass = NV01_MEMORY_LIST_FBMEM;
4854 break;
4855
4856 case ADDR_SYSMEM:
4857 hClass = NV01_MEMORY_LIST_SYSTEM;
4858 break;
4859
4860 default:
4861 return NV_ERR_NOT_SUPPORTED;
4862 }
4863
4864 // Initialize parameters with pMemDesc information
4865 listAllocParams.pteAdjust = pMemDesc->PteAdjust;
4866 listAllocParams.format = memdescGetPteKind(pMemDesc);
4867 listAllocParams.size = pMemDesc->Size;
4868 listAllocParams.hClient = NV01_NULL_OBJECT;
4869 listAllocParams.hParent = NV01_NULL_OBJECT;
4870 listAllocParams.hObject = NV01_NULL_OBJECT;
4871 listAllocParams.limit = pMemDesc->Size - 1;
4872 listAllocParams.flagsOs02 = (DRF_DEF(OS02,_FLAGS,_MAPPING,_NO_MAP) |
4873 (flags & DRF_SHIFTMASK(NVOS02_FLAGS_COHERENCY)));
4874
4875 // Handle pageCount based on pMemDesc contiguity
4876 if (!memdescGetContiguity(pMemDesc, AT_GPU))
4877 {
4878 listAllocParams.flagsOs02 |= DRF_DEF(OS02,_FLAGS,_PHYSICALITY,_NONCONTIGUOUS);
4879 listAllocParams.pageCount = pMemDesc->PageCount;
4880 }
4881 else
4882 {
4883 listAllocParams.pageCount = 1;
4884 }
4885
4886
4887 // Initialize pageNumberList
4888 pageNumberList = portMemAllocNonPaged(sizeof(NvU64) * listAllocParams.pageCount);
4889 for (index = 0; index < listAllocParams.pageCount; index++)
4890 pageNumberList[index] = memdescGetPte(pMemDesc, AT_GPU, index) >> RM_PAGE_SHIFT;
4891 listAllocParams.pageNumberList = pageNumberList;
4892
4893 // Create MemoryList object
4894 NV_ASSERT_OK_OR_GOTO(status,
4895 pRmApi->Alloc(pRmApi,
4896 pMemoryManager->hClient,
4897 pMemoryManager->hSubdevice,
4898 pHandle,
4899 hClass,
4900 &listAllocParams,
4901 sizeof(listAllocParams)),
4902 end);
4903
4904 // Register MemoryList object to GSP
4905 NV_ASSERT_OK_OR_GOTO(status,
4906 memdescRegisterToGSP(pGpu,
4907 pMemoryManager->hClient,
4908 pMemoryManager->hSubdevice,
4909 *pHandle),
4910 end);
4911
4912 end:
4913 if ((status != NV_OK) && (*pHandle != NV01_NULL_OBJECT))
4914 pRmApi->Free(pRmApi, pMemoryManager->hClient, *pHandle);
4915
4916 if (pageNumberList != NULL)
4917 portMemFree(pageNumberList);
4918
4919 return status;
4920 }
4921
4922 NV_STATUS
memdescSetPageArrayGranularity(MEMORY_DESCRIPTOR * pMemDesc,NvU64 pageArrayGranularity)4923 memdescSetPageArrayGranularity
4924 (
4925 MEMORY_DESCRIPTOR *pMemDesc,
4926 NvU64 pageArrayGranularity
4927 )
4928 {
4929 // Make sure pageArrayGranularity is a power of 2 value.
4930 NV_ASSERT_OR_RETURN((pageArrayGranularity & (pageArrayGranularity - 1)) == 0, NV_ERR_INVALID_ARGUMENT);
4931
4932 // Allow setting the same granularity.
4933 if (pMemDesc->pageArrayGranularity == pageArrayGranularity)
4934 {
4935 return NV_OK;
4936 }
4937
4938 // Make sure setting the page array happens before the pteArray is populated.
4939 NV_ASSERT_OR_RETURN(pMemDesc->_pteArray[0] == 0, NV_ERR_INVALID_STATE);
4940
4941 pMemDesc->pageArrayGranularity = pageArrayGranularity;
4942
4943 return NV_OK;
4944 }
4945
4946 NV_STATUS
memdescFillMemdescForPhysAttr(MEMORY_DESCRIPTOR * pMemDesc,ADDRESS_TRANSLATION addressTranslation,NvU64 * pOffset,NvU32 * pMemAperture,NvU32 * pMemKind,NvU32 * pZCullId,NvU32 * pGpuCacheAttr,NvU32 * pGpuP2PCacheAttr,NvU64 * contigSegmentSize)4947 memdescFillMemdescForPhysAttr
4948 (
4949 MEMORY_DESCRIPTOR *pMemDesc,
4950 ADDRESS_TRANSLATION addressTranslation,
4951 NvU64 *pOffset,
4952 NvU32 *pMemAperture,
4953 NvU32 *pMemKind,
4954 NvU32 *pZCullId,
4955 NvU32 *pGpuCacheAttr,
4956 NvU32 *pGpuP2PCacheAttr,
4957 NvU64 *contigSegmentSize
4958 )
4959 {
4960 NvU64 surfOffset = *pOffset, surfBase, surfLimit;
4961 NvU32 zcbitmap;
4962
4963 surfBase = memdescGetPhysAddr(pMemDesc, addressTranslation, 0);
4964 surfLimit = surfBase + pMemDesc->Size - 1;
4965 *pMemKind = memdescGetPteKind(pMemDesc);
4966
4967 *pOffset = memdescGetPhysAddr(pMemDesc, addressTranslation, surfOffset);
4968
4969 if (memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM )
4970 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_VIDMEM;
4971 else if (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM)
4972 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_SYSMEM;
4973 else if (memdescGetAddressSpace(pMemDesc) == ADDR_EGM)
4974 *pMemAperture = NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR_APERTURE_SYSMEM;
4975 else if (memdescGetAddressSpace(pMemDesc) == ADDR_VIRTUAL )
4976 {
4977 //
4978 // XXX we could theoretically find whatever phys mem object is plugged
4979 // in at surfOffset w/in the virt object... that'd mean scanning
4980 // pMemory->DmaMappingList
4981 //
4982 return NV_ERR_NOT_SUPPORTED;
4983 }
4984 else
4985 return NV_ERR_GENERIC;
4986
4987 if (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED)
4988 {
4989 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED;
4990 }
4991 else if (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED)
4992 {
4993 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_UNCACHED;
4994 }
4995 else
4996 {
4997 *pGpuCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED_UNKNOWN;
4998 }
4999
5000 if (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_CACHED)
5001 {
5002 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED;
5003 }
5004 else if (memdescGetGpuP2PCacheAttrib(pMemDesc) == NV_MEMORY_UNCACHED)
5005 {
5006 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_UNCACHED;
5007 }
5008 else
5009 {
5010 *pGpuP2PCacheAttr = NV0041_CTRL_GET_SURFACE_PHYS_ATTR_GPU_CACHED_UNKNOWN;
5011 }
5012
5013 zcbitmap = FB_HWRESID_ZCULL_VAL_FERMI(memdescGetHwResId(pMemDesc)); //bitmap form... need a scalar
5014 for ( *pZCullId = 0; zcbitmap; zcbitmap >>= 1, *pZCullId += 1) {;;;}
5015 *pZCullId -= 1; // side effect if there is no zcull id of setting ~0
5016
5017 *contigSegmentSize = surfLimit - (surfBase + surfOffset) + 1;
5018
5019 if ( !memdescGetContiguity(pMemDesc, addressTranslation))
5020 {
5021 // XXX overly conservative. we could scan the PTEs to find out if more pages are contig.
5022 NvU64 surfOffsetLimitSame4KBPage = (4*1024)*((surfBase + surfOffset)/(4*1024)) + (4*1024) - 1;
5023 if ( surfLimit >= surfOffsetLimitSame4KBPage )
5024 *contigSegmentSize = surfOffsetLimitSame4KBPage - (surfBase + surfOffset) + 1;
5025 }
5026
5027 return NV_OK;
5028 }
5029
5030 NvBool
memdescIsEgm(MEMORY_DESCRIPTOR * pMemDesc)5031 memdescIsEgm
5032 (
5033 MEMORY_DESCRIPTOR *pMemDesc
5034 )
5035 {
5036 NV_ADDRESS_SPACE addrSpace;
5037 MEMORY_DESCRIPTOR *pRootMemDesc;
5038 MemoryManager *pMemoryManager;
5039
5040 //
5041 // If memdesc is not device owned, we can't tell if local EGM is enabled
5042 // due to lack of memory manager.
5043 //
5044 if (pMemDesc->pGpu == NULL)
5045 {
5046 return NV_FALSE;
5047 }
5048
5049 addrSpace = memdescGetAddressSpace(pMemDesc);
5050 pRootMemDesc = memdescGetRootMemDesc(pMemDesc, NULL);
5051
5052 if ((pRootMemDesc == NULL) || (pRootMemDesc->pGpu == NULL))
5053 {
5054 return NV_FALSE;
5055 }
5056
5057 pMemoryManager = GPU_GET_MEMORY_MANAGER(pRootMemDesc->pGpu);
5058 if (pMemoryManager == NULL)
5059 {
5060 return NV_FALSE;
5061 }
5062
5063 if ((addrSpace == ADDR_EGM) ||
5064 (memmgrIsLocalEgmEnabled(pMemoryManager) &&
5065 (addrSpace == ADDR_SYSMEM) &&
5066 (memdescGetNumaNode(pMemDesc) == pMemoryManager->localEgmNodeId)))
5067 {
5068 return NV_TRUE;
5069 }
5070
5071 return NV_FALSE;
5072 }
5073
5074