1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "mem_mgr/mem.h"
25
26 #include "mem_mgr/fla_mem.h"
27
28 #include "gpu_mgr/gpu_mgr.h"
29 #include "gpu/gpu.h"
30 #include "gpu/mem_mgr/mem_mgr.h"
31 #include "gpu/disp/disp_objs.h"
32 #include "gpu/mem_mgr/mem_desc.h"
33 #include "os/os.h"
34 #include "core/locks.h"
35 #include "gpu/device/device.h"
36 #include "gpu/subdevice/subdevice.h"
37 #include "vgpu/rpc.h"
38 #include "platform/sli/sli.h"
39
40 #include "class/cl0041.h" // NV04_MEMORY
41 #include "class/cl003e.h" // NV01_MEMORY_SYSTEM
42 #include "class/cl0071.h" // NV01_MEMORY_SYSTEM_OS_DESCRIPTOR
43
44 NV_STATUS
memConstruct_IMPL(Memory * pMemory,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)45 memConstruct_IMPL
46 (
47 Memory *pMemory,
48 CALL_CONTEXT *pCallContext,
49 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
50 )
51 {
52 RsResourceRef *pResourceRef = pCallContext->pResourceRef;
53 RsResourceRef *pParentRef = pResourceRef->pParentRef;
54
55 //
56 // Common initialization used for both normal construction & copy
57 // constructor
58 //
59
60 // NULL if parent isn't a device
61 pMemory->pDevice = dynamicCast(pParentRef->pResource, Device);
62
63 // NULL if parent isn't a subdevice
64 pMemory->pSubDevice = dynamicCast(pParentRef->pResource, Subdevice);
65
66 // If parent subdevice, grandparent must be a device
67 if (pMemory->pSubDevice)
68 {
69 RsResourceRef *pGrandParentRef = pParentRef->pParentRef;
70
71 pMemory->pDevice = dynamicCast(pGrandParentRef->pResource, Device);
72
73 if (pMemory->pDevice == NULL)
74 return NV_ERR_INVALID_OBJECT_HANDLE;
75 }
76
77 // If child of device, we have a pGpu
78 if (pMemory->pDevice)
79 {
80 // NOTE: pGpu and pDevice be NULL for NoDeviceMemory
81 pMemory->pGpu = CliGetGpuFromContext(pResourceRef, &pMemory->bBcResource);
82
83 NV_ASSERT_OR_RETURN(pMemory->pGpu != NULL, NV_ERR_INVALID_ARGUMENT);
84
85 // Set thread BC state
86 gpuSetThreadBcState(pMemory->pGpu, pMemory->bBcResource);
87 }
88
89 if (RS_IS_COPY_CTOR(pParams))
90 {
91 //
92 // Copy constructor path (NvRmDupObject)
93 //
94 return memCopyConstruct_IMPL(pMemory, pCallContext, pParams);
95 }
96 else
97 {
98 //
99 // Default constructor path (NvRmAlloc)
100 //
101 }
102
103 return NV_OK;
104 }
105
106 NV_STATUS
memGetMapAddrSpace_IMPL(Memory * pMemory,CALL_CONTEXT * pCallContext,NvU32 mapFlags,NV_ADDRESS_SPACE * pAddrSpace)107 memGetMapAddrSpace_IMPL
108 (
109 Memory *pMemory,
110 CALL_CONTEXT *pCallContext,
111 NvU32 mapFlags,
112 NV_ADDRESS_SPACE *pAddrSpace
113 )
114 {
115 NV_ADDRESS_SPACE addrSpace;
116 OBJGPU *pGpu = pMemory->pGpu;
117 NvBool bBcResource = pMemory->bBcResource;
118 MEMORY_DESCRIPTOR *pMemDesc = NULL;
119
120 if (pGpu == NULL)
121 return NV_ERR_INVALID_OBJECT;
122
123 gpuSetThreadBcState(pGpu, bBcResource);
124
125 pMemDesc = memdescGetMemDescFromGpu(pMemory->pMemDesc, pGpu);
126
127 NV_ASSERT_OK_OR_RETURN(rmapiGetEffectiveAddrSpace(pGpu, pMemDesc, mapFlags, &addrSpace));
128
129 if (addrSpace == ADDR_SYSMEM)
130 {
131 if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_BAR0_REFLECT))
132 {
133 addrSpace = ADDR_REGMEM;
134 }
135 else if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_BAR1_REFLECT))
136 {
137 addrSpace = ADDR_FBMEM;
138 }
139 }
140
141 if (pAddrSpace)
142 *pAddrSpace = addrSpace;
143
144 return NV_OK;
145 }
146
147 void
memDestruct_IMPL(Memory * pMemory)148 memDestruct_IMPL
149 (
150 Memory *pMemory
151 )
152 {
153 OBJGPU *pGpu = pMemory->pGpu;
154 NvHandle hClient = RES_GET_CLIENT_HANDLE(pMemory);
155 NvHandle hParent = RES_GET_PARENT_HANDLE(pMemory);
156 NvHandle hMemory = RES_GET_HANDLE(pMemory);
157 NV_STATUS status = NV_OK;
158
159 //
160 // The default destructor is used when memConstructCommon() is called by
161 // the subclass but not memDestructCommon().
162 //
163 if (pMemory->bConstructed && pMemory->pMemDesc != NULL)
164 {
165 // Remove the system memory reference from the client
166 memDestructCommon(pMemory);
167 memdescFree(pMemory->pMemDesc);
168 memdescDestroy(pMemory->pMemDesc);
169 }
170
171 // if the allocation is RPC-ed, free using RPC
172 if (pMemory->bRpcAlloc && (IS_VIRTUAL(pGpu) || IS_GSP_CLIENT(pGpu)))
173 {
174 NV_RM_RPC_FREE(pGpu, hClient, hParent, hMemory, status);
175 NV_ASSERT(status == NV_OK);
176 }
177 }
178
179 NV_STATUS
memCreateMemDesc_IMPL(OBJGPU * pGpu,MEMORY_DESCRIPTOR ** ppMemDesc,NV_ADDRESS_SPACE addrSpace,NvU64 FBOffset,NvU64 length,NvU32 attr,NvU32 attr2)180 memCreateMemDesc_IMPL
181 (
182 OBJGPU *pGpu,
183 MEMORY_DESCRIPTOR **ppMemDesc,
184 NV_ADDRESS_SPACE addrSpace,
185 NvU64 FBOffset,
186 NvU64 length,
187 NvU32 attr,
188 NvU32 attr2
189 )
190 {
191 NV_STATUS status = NV_OK;
192 NvU32 CpuCacheAttrib, gpuCacheAttrib;
193 MEMORY_DESCRIPTOR *pMemDesc = NULL;
194
195 *ppMemDesc = NULL;
196
197 if (addrSpace == ADDR_SYSMEM)
198 NV_ASSERT_OR_RETURN(FLD_TEST_DRF(OS32, _ATTR, _PHYSICALITY, _CONTIGUOUS, attr), NV_ERR_INVALID_ARGUMENT);
199
200 // setup the CpuCacheAttrib as well.. (if the caller doesn't specify anything it will be 0=UNCACHED)
201 switch (DRF_VAL(OS32, _ATTR, _COHERENCY, attr))
202 {
203 case NVOS32_ATTR_COHERENCY_UNCACHED:
204 CpuCacheAttrib = NV_MEMORY_UNCACHED;
205 break;
206 case NVOS32_ATTR_COHERENCY_WRITE_COMBINE:
207 CpuCacheAttrib = NV_MEMORY_WRITECOMBINED;
208 break;
209 case NVOS32_ATTR_COHERENCY_CACHED:
210 case NVOS32_ATTR_COHERENCY_WRITE_THROUGH:
211 case NVOS32_ATTR_COHERENCY_WRITE_PROTECT:
212 case NVOS32_ATTR_COHERENCY_WRITE_BACK:
213 CpuCacheAttrib = NV_MEMORY_CACHED;
214 break;
215 default:
216 NV_ASSERT(0);
217 CpuCacheAttrib = NV_MEMORY_UNCACHED;
218 break;
219 }
220
221 gpuCacheAttrib = FLD_TEST_DRF(OS32, _ATTR2, _GPU_CACHEABLE, _YES, attr2) ? NV_MEMORY_CACHED : NV_MEMORY_UNCACHED;
222
223 // Create and fill in a memory descriptor
224 status = memdescCreate(&pMemDesc, pGpu, length, 0, NV_TRUE, addrSpace,
225 CpuCacheAttrib,
226 MEMDESC_FLAGS_ALLOC_PER_SUBDEVICE_FB_BC_ONLY(pGpu, addrSpace));
227 if (status == NV_OK)
228 {
229 if (memdescHasSubDeviceMemDescs(pMemDesc))
230 {
231 MEMORY_DESCRIPTOR *pMemDescNext = pMemDesc->_pNext;
232 while (pMemDescNext)
233 {
234 memdescDescribe(pMemDescNext, addrSpace, FBOffset, length);
235 memdescSetGpuCacheAttrib(pMemDescNext, gpuCacheAttrib);
236 pMemDescNext = pMemDescNext->_pNext;
237 }
238 }
239 else
240 {
241 memdescDescribe(pMemDesc, addrSpace, FBOffset, length);
242 memdescSetGpuCacheAttrib(pMemDesc, gpuCacheAttrib);
243 }
244
245 *ppMemDesc = pMemDesc;
246 }
247
248 return status;
249 }
250
251 NV_STATUS
memCreateKernelMapping_IMPL(Memory * pMemory,NvU32 Protect,NvBool bClear)252 memCreateKernelMapping_IMPL
253 (
254 Memory *pMemory,
255 NvU32 Protect,
256 NvBool bClear
257 )
258 {
259 NV_STATUS status;
260
261 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE));
262
263 if (pMemory->KernelVAddr == NvP64_NULL)
264 {
265 if (memdescGetAddressSpace(pMemory->pMemDesc) != ADDR_SYSMEM)
266 {
267 return NV_ERR_NOT_SUPPORTED;
268 }
269
270 status = memdescMap(pMemory->pMemDesc, 0, pMemory->Length, NV_TRUE,
271 Protect, &pMemory->KernelVAddr, &pMemory->KernelMapPriv);
272
273 if (status != NV_OK)
274 {
275 pMemory->KernelVAddr = NvP64_NULL;
276 pMemory->KernelMapPriv = NvP64_NULL;
277 return status;
278 }
279
280 memdescSetKernelMapping(pMemory->pMemDesc, pMemory->KernelVAddr);
281 memdescSetKernelMappingPriv(pMemory->pMemDesc, pMemory->KernelMapPriv);
282
283 if (bClear)
284 {
285 portMemSet(NvP64_VALUE(pMemory->KernelVAddr), 0, pMemory->Length);
286 }
287 }
288
289 return NV_OK;
290 }
291
292 RM_ATTR_PAGE_SIZE
dmaNvos32ToPageSizeAttr(NvU32 attr,NvU32 attr2)293 dmaNvos32ToPageSizeAttr
294 (
295 NvU32 attr,
296 NvU32 attr2
297 )
298 {
299 switch (DRF_VAL(OS32, _ATTR, _PAGE_SIZE, attr))
300 {
301 case NVOS32_ATTR_PAGE_SIZE_DEFAULT:
302 return RM_ATTR_PAGE_SIZE_DEFAULT;
303 case NVOS32_ATTR_PAGE_SIZE_4KB:
304 return RM_ATTR_PAGE_SIZE_4KB;
305 case NVOS32_ATTR_PAGE_SIZE_BIG:
306 return RM_ATTR_PAGE_SIZE_BIG;
307 case NVOS32_ATTR_PAGE_SIZE_HUGE:
308 switch (DRF_VAL(OS32, _ATTR2, _PAGE_SIZE_HUGE, attr2))
309 {
310 case NVOS32_ATTR2_PAGE_SIZE_HUGE_DEFAULT:
311 case NVOS32_ATTR2_PAGE_SIZE_HUGE_2MB:
312 return RM_ATTR_PAGE_SIZE_HUGE;
313 case NVOS32_ATTR2_PAGE_SIZE_HUGE_512MB:
314 return RM_ATTR_PAGE_SIZE_512MB;
315 }
316 break;
317 }
318
319 NV_ASSERT_FAILED("Invalid attr and attr2 page size arguments");
320 return RM_ATTR_PAGE_SIZE_DEFAULT;
321 }
322
323 NV_STATUS
memConstructCommon_IMPL(Memory * pMemory,NvU32 categoryClassId,NvU32 flags,MEMORY_DESCRIPTOR * pMemDesc,NvU32 heapOwner,Heap * pHeap,NvU32 attr,NvU32 attr2,NvU32 Pitch,NvU32 type,NvU32 tag,HWRESOURCE_INFO * pHwResource)324 memConstructCommon_IMPL
325 (
326 Memory *pMemory,
327 NvU32 categoryClassId,
328 NvU32 flags,
329 MEMORY_DESCRIPTOR *pMemDesc,
330 NvU32 heapOwner,
331 Heap *pHeap,
332 NvU32 attr,
333 NvU32 attr2,
334 NvU32 Pitch,
335 NvU32 type,
336 NvU32 tag,
337 HWRESOURCE_INFO *pHwResource
338 )
339 {
340 OBJGPU *pGpu = NULL;
341 NV_STATUS status = NV_OK;
342 NvHandle hClient = RES_GET_CLIENT_HANDLE(pMemory);
343 NvHandle hParent = RES_GET_PARENT_HANDLE(pMemory);
344 NvHandle hMemory = RES_GET_HANDLE(pMemory);
345
346 if (pMemDesc == NULL)
347 return NV_ERR_INVALID_ARGUMENT;
348
349 // initialize the memory description
350 pMemory->categoryClassId = categoryClassId;
351 pMemory->pMemDesc = pMemDesc;
352 pMemory->Length = pMemDesc->Size;
353 pMemory->RefCount = 1;
354 pMemory->HeapOwner = heapOwner;
355 pMemory->pHeap = pHeap;
356 pMemory->Attr = attr;
357 pMemory->Attr2 = attr2;
358 pMemory->Pitch = Pitch;
359 pMemory->Type = type;
360 pMemory->Flags = flags;
361 pMemory->tag = tag;
362 pMemory->isMemDescOwner = NV_TRUE;
363 pMemory->bRpcAlloc = NV_FALSE;
364
365 // We are finished if this instance is device-less
366 if (pMemory->pDevice == NULL)
367 {
368 goto done;
369 }
370
371 if (pMemDesc->pGpu == NULL)
372 {
373 return NV_ERR_INVALID_STATE;
374 }
375
376 // Memory has hw resources associated with it that need to be tracked.
377 if (pHwResource != NULL)
378 {
379 pMemory->pHwResource = portMemAllocNonPaged(sizeof(HWRESOURCE_INFO));
380 if (pMemory->pHwResource != NULL)
381 {
382 *pMemory->pHwResource = *pHwResource; // struct copy
383 pMemory->pHwResource->refCount = 1;
384 }
385 else
386 {
387 NV_PRINTF(LEVEL_ERROR,
388 "Unable to allocate HWRESOURCE_INFO tracking structure\n");
389 status = NV_ERR_NO_MEMORY;
390 goto done;
391 }
392 }
393
394 NV_ASSERT(status == NV_OK);
395
396 //
397 // Apply attr and flags to the memory descriptor. Ideally all should
398 // be handled before we get here.
399 //
400
401 // Check whether encryption should be enabled
402 if (flags & NVOS32_ALLOC_FLAGS_TURBO_CIPHER_ENCRYPTED)
403 {
404 pGpu = pMemDesc->pGpu;
405 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
406 memdescSetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_ENCRYPTED, NV_TRUE);
407 SLI_LOOP_END
408 }
409
410 if (FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_USER, _READ_ONLY, attr2))
411 {
412 pGpu = pMemDesc->pGpu;
413 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
414 memdescSetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_USER_READ_ONLY, NV_TRUE);
415 SLI_LOOP_END
416 }
417
418 if (FLD_TEST_DRF(OS32, _ATTR2, _PROTECTION_DEVICE, _READ_ONLY, attr2))
419 {
420 pGpu = pMemDesc->pGpu;
421 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
422 memdescSetFlag(memdescGetMemDescFromGpu(pMemDesc, pGpu), MEMDESC_FLAGS_DEVICE_READ_ONLY, NV_TRUE);
423 SLI_LOOP_END
424 }
425
426 // setup GpuP2PCacheAttrib
427 switch (DRF_VAL(OS32, _ATTR2, _P2P_GPU_CACHEABLE, attr2))
428 {
429 case NVOS32_ATTR2_P2P_GPU_CACHEABLE_YES:
430 pGpu = pMemDesc->pGpu;
431 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
432 memdescSetGpuP2PCacheAttrib(memdescGetMemDescFromGpu(pMemDesc, pGpu), NV_MEMORY_CACHED);
433 SLI_LOOP_END
434 break;
435 default:
436 NV_ASSERT(0);
437 /*FALLSTHRU*/
438 case NVOS32_ATTR2_P2P_GPU_CACHEABLE_NO:
439 case NVOS32_ATTR2_P2P_GPU_CACHEABLE_DEFAULT:
440 pGpu = pMemDesc->pGpu;
441 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
442 memdescSetGpuP2PCacheAttrib(memdescGetMemDescFromGpu(pMemDesc, pGpu), NV_MEMORY_UNCACHED);
443 SLI_LOOP_END
444 break;
445 }
446
447 //
448 // Page size may be specified at allocation. This if for fermi family
449 // chips and is a nop for previous generations. At this point the hal call
450 // to set the page size should never fail as the memory was just allocated.
451 //
452 if (pMemDesc->pGpu)
453 {
454 pGpu = pMemDesc->pGpu;
455 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY | SLI_LOOP_FLAGS_IGNORE_REENTRANCY)
456
457 RM_ATTR_PAGE_SIZE pageSizeAttr = dmaNvos32ToPageSizeAttr(attr, attr2);
458 status = memmgrSetMemDescPageSize_HAL(pGpu, GPU_GET_MEMORY_MANAGER(pGpu), memdescGetMemDescFromGpu(pMemDesc, pGpu),
459 AT_GPU, pageSizeAttr);
460 if (status != NV_OK)
461 {
462 SLI_LOOP_BREAK;
463 }
464 SLI_LOOP_END
465
466 if (status != NV_OK)
467 {
468 goto done;
469 }
470 }
471
472 pMemory->Node.keyStart = RES_GET_HANDLE(pMemory);
473 pMemory->Node.keyEnd = RES_GET_HANDLE(pMemory);
474 pMemory->Node.Data = pMemory;
475
476 status = btreeInsert(&pMemory->Node, &pMemory->pDevice->DevMemoryTable);
477 if (status != NV_OK)
478 goto done;
479
480 // Make GSP-RM aware of the memory descriptor so it can be used there
481 if (FLD_TEST_DRF(OS32, _ATTR2, _REGISTER_MEMDESC_TO_PHYS_RM, _TRUE, attr2))
482 {
483 status = memdescRegisterToGSP(pGpu, hClient, hParent, hMemory);
484 if (status != NV_OK)
485 goto done;
486 }
487
488 // Initialize the circular list item for tracking dup/sharing of pMemDesc
489 pMemory->dupListItem.pNext = pMemory->dupListItem.pPrev = pMemory;
490
491 done:
492 if (status != NV_OK)
493 {
494 if (pMemory->pHwResource != NULL)
495 {
496 portMemFree(pMemory->pHwResource);
497 }
498 }
499 else
500 {
501 pMemory->bConstructed = NV_TRUE;
502 }
503
504 return status;
505 }
506
507 static NvBool
_memCheckHostVgpuDeviceExists(OBJGPU * pGpu)508 _memCheckHostVgpuDeviceExists
509 (
510 OBJGPU *pGpu
511 )
512 {
513 NV_STATUS status;
514
515 KERNEL_HOST_VGPU_DEVICE *pKernelHostVgpuDevice = NULL;
516
517 NV_ASSERT_OK_OR_ELSE(status, vgpuGetCallingContextKernelHostVgpuDevice(pGpu, &pKernelHostVgpuDevice), return NV_FALSE);
518
519 return (pKernelHostVgpuDevice != NULL);
520 }
521
522 static void
_memDestructCommonWithDevice(Memory * pMemory)523 _memDestructCommonWithDevice
524 (
525 Memory *pMemory
526 )
527 {
528 NvHandle hMemory = RES_GET_HANDLE(pMemory);
529 OBJGPU *pGpu = pMemory->pGpu;
530 Device *pDevice = pMemory->pDevice;
531 RsResourceRef *pDeviceRef = RES_GET_REF(pDevice);
532 NvHandle hDevice = RES_GET_HANDLE(pDevice);
533 Subdevice *pSubDeviceInfo;
534 DispCommon *pDispCommon;
535 RsClient *pRsClient = RES_GET_CLIENT(pMemory);
536 NV_STATUS status;
537 RS_ITERATOR subDevIt;
538 FB_ALLOC_INFO *pFbAllocInfo = NULL;
539 FB_ALLOC_PAGE_FORMAT *pFbAllocPageFormat = NULL;
540
541 gpuSetThreadBcState(pGpu, pMemory->bBcResource);
542
543 subDevIt = clientRefIter(pRsClient, pDeviceRef, classId(Subdevice), RS_ITERATE_CHILDREN, NV_TRUE);
544 while (clientRefIterNext(pRsClient, &subDevIt))
545 {
546 pSubDeviceInfo = dynamicCast(subDevIt.pResourceRef->pResource, Subdevice);
547
548 if (hMemory == pSubDeviceInfo->hNotifierMemory)
549 {
550 pSubDeviceInfo->hNotifierMemory = NV01_NULL_OBJECT;
551 pSubDeviceInfo->pNotifierMemory = NULL;
552 }
553 }
554
555 dispcmnGetByDevice(pRsClient, hDevice, &pDispCommon);
556
557 if (pDispCommon != NULL)
558 {
559 DisplayApi *pDisplayApi = staticCast(pDispCommon, DisplayApi);
560 if (pDisplayApi->hNotifierMemory == hMemory)
561 {
562 pDisplayApi->hNotifierMemory = NV01_NULL_OBJECT;
563 pDisplayApi->pNotifierMemory = NULL;
564 }
565 }
566
567 //
568 // Release any FB HW resources
569 //
570 if (pMemory->pHwResource)
571 {
572 if (--pMemory->pHwResource->refCount == 0)
573 {
574 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
575 NvBool bHostVgpuDeviceExists = _memCheckHostVgpuDeviceExists(pGpu);
576
577 if ((pMemory->categoryClassId == NV01_MEMORY_SYSTEM && memmgrComprSupported(pMemoryManager, ADDR_SYSMEM)) ||
578 (bHostVgpuDeviceExists && (pMemory->pHwResource->isGuestAllocated)))
579 {
580 pFbAllocInfo = portMemAllocNonPaged(sizeof(FB_ALLOC_INFO));
581 if (pFbAllocInfo == NULL)
582 {
583 NV_ASSERT(0);
584 status = NV_ERR_NO_MEMORY;
585 goto done;
586 }
587
588 pFbAllocPageFormat = portMemAllocNonPaged(sizeof(FB_ALLOC_PAGE_FORMAT));
589 if (pFbAllocPageFormat == NULL) {
590 NV_ASSERT(0);
591 status = NV_ERR_NO_MEMORY;
592 goto done;
593 }
594
595 portMemSet(pFbAllocInfo, 0, sizeof(FB_ALLOC_INFO));
596 portMemSet(pFbAllocPageFormat, 0, sizeof(FB_ALLOC_PAGE_FORMAT));
597 pFbAllocInfo->pageFormat = pFbAllocPageFormat;
598
599 pFbAllocInfo->pageFormat->type = pMemory->Type;
600 pFbAllocInfo->pageFormat->attr = pMemory->Attr;
601 pFbAllocInfo->pageFormat->attr2 = pMemory->Attr2;
602 pFbAllocInfo->hwResId = memdescGetHwResId(pMemory->pMemDesc);
603 pFbAllocInfo->size = pMemory->Length;
604 pFbAllocInfo->format = memdescGetPteKind(pMemory->pMemDesc);
605 pFbAllocInfo->hClient = pRsClient->hClient;
606 pFbAllocInfo->hDevice = hDevice;
607
608 //
609 // Note that while freeing duped memory under a device, the
610 // device may not be the memory owning device. Hence, always use
611 // memory owning device (pMemDesc->pGpu) to free HW resources.
612 //
613 status = memmgrFreeHwResources(pMemory->pMemDesc->pGpu, pMemoryManager, pFbAllocInfo);
614 NV_ASSERT(status == NV_OK);
615 }
616 portMemFree(pMemory->pHwResource);
617 }
618 }
619
620 NV_ASSERT_OK_OR_GOTO(status, btreeUnlink(&pMemory->Node, &pDevice->DevMemoryTable), done);
621
622 pMemory->pMemDesc->DupCount--;
623
624 // Choose the new owner
625 if (pMemory->isMemDescOwner)
626 {
627 (pMemory->dupListItem.pNext)->isMemDescOwner = NV_TRUE;
628 }
629 // Remove from circular list tracking dup/sharing of pMemDesc
630 pMemory->dupListItem.pPrev->dupListItem.pNext = pMemory->dupListItem.pNext;
631 pMemory->dupListItem.pNext->dupListItem.pPrev = pMemory->dupListItem.pPrev;
632 pMemory->dupListItem.pNext = pMemory->dupListItem.pPrev = NULL;
633
634 pMemory->bConstructed = NV_FALSE;
635
636 done:
637 portMemFree(pFbAllocPageFormat);
638 portMemFree(pFbAllocInfo);
639
640 // The unmap call(s) above may have changed the broadcast state so restore it here
641 gpuSetThreadBcState(pGpu, pMemory->bBcResource);
642 }
643
644 void
memDestructCommon_IMPL(Memory * pMemory)645 memDestructCommon_IMPL
646 (
647 Memory *pMemory
648 )
649 {
650 OBJGPU *pGpu = pMemory->pGpu;
651 RsResourceRef *pResourceRef = RES_GET_REF(pMemory);
652 RsResourceRef *pParentRef = pResourceRef->pParentRef;
653 RsClient *pClient = RES_GET_CLIENT(pMemory);
654 NvHandle hClient = pClient->hClient;
655 NvHandle hParent = pParentRef->hResource;
656 NvHandle hMemory = RES_GET_HANDLE(pMemory);
657
658 if (!pMemory->bConstructed)
659 return;
660
661 NV_ASSERT_OK(memdescDeregisterFromGSP(pGpu, hClient, hParent, hMemory));
662
663 // Do device specific teardown if we have a device
664 if (pMemory->pDevice != NULL)
665 {
666 _memDestructCommonWithDevice(pMemory);
667 }
668 else
669 {
670 pMemory->bConstructed = NV_FALSE;
671 }
672
673 if (pMemory->KernelVAddr != NvP64_NULL)
674 {
675 memdescUnmap(pMemory->pMemDesc, NV_TRUE, osGetCurrentProcess(),
676 pMemory->KernelVAddr, pMemory->KernelMapPriv);
677 pMemory->KernelVAddr = NvP64_NULL;
678 pMemory->KernelMapPriv = NvP64_NULL;
679 }
680 }
681
682 NV_STATUS
memGetByHandleAndDevice_IMPL(RsClient * pClient,NvHandle hMemory,NvHandle hDevice,Memory ** ppMemory)683 memGetByHandleAndDevice_IMPL
684 (
685 RsClient *pClient,
686 NvHandle hMemory,
687 NvHandle hDevice,
688 Memory **ppMemory
689 )
690 {
691 NV_STATUS status;
692
693 status = memGetByHandle(pClient, hMemory, ppMemory);
694 if (status != NV_OK)
695 return status;
696
697 if (hDevice != RES_GET_HANDLE((*ppMemory)->pDevice))
698 {
699 *ppMemory = NULL;
700 return NV_ERR_OBJECT_NOT_FOUND;
701 }
702
703 return NV_OK;
704 }
705
706 NV_STATUS
memGetByHandle_IMPL(RsClient * pClient,NvHandle hMemory,Memory ** ppMemory)707 memGetByHandle_IMPL
708 (
709 RsClient *pClient,
710 NvHandle hMemory,
711 Memory **ppMemory
712 )
713 {
714 RsResourceRef *pResourceRef;
715 NV_STATUS status;
716
717 *ppMemory = NULL;
718
719 status = clientGetResourceRef(pClient, hMemory, &pResourceRef);
720 if (status != NV_OK)
721 return status;
722
723 *ppMemory = dynamicCast(pResourceRef->pResource, Memory);
724
725 if (*ppMemory == NULL)
726 return NV_ERR_INVALID_OBJECT_HANDLE;
727
728 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(*ppMemory, NV_FALSE));
729
730 return NV_OK;
731 }
732
733 NV_STATUS
memGetByHandleAndGroupedGpu_IMPL(RsClient * pClient,NvHandle hMemory,OBJGPU * pGpu,Memory ** ppMemory)734 memGetByHandleAndGroupedGpu_IMPL
735 (
736 RsClient *pClient,
737 NvHandle hMemory,
738 OBJGPU *pGpu,
739 Memory **ppMemory
740 )
741 {
742 Memory *pMemory;
743 NV_STATUS status;
744 Device *pDevice;
745
746 status = memGetByHandle(pClient, hMemory, &pMemory);
747 if (status != NV_OK)
748 return status;
749
750 pDevice = pMemory->pDevice;
751
752 if ((pDevice == NULL) ||
753 (gpumgrGetParentGPU(pGpu) != GPU_RES_GET_GPU(pDevice)))
754 {
755 *ppMemory = NULL;
756 return NV_ERR_OBJECT_NOT_FOUND;
757 }
758
759 *ppMemory = pMemory;
760 return NV_OK;
761 }
762
763 NV_STATUS
memIsReady_IMPL(Memory * pMemory,NvBool bCopyConstructorContext)764 memIsReady_IMPL
765 (
766 Memory *pMemory,
767 NvBool bCopyConstructorContext
768 )
769 {
770 if (pMemory->pMemDesc == NULL)
771 return NV_ERR_INVALID_OBJECT;
772
773 return NV_OK;
774 }
775
776 NV_STATUS
memControl_IMPL(Memory * pMemory,CALL_CONTEXT * pCallContext,RS_RES_CONTROL_PARAMS_INTERNAL * pParams)777 memControl_IMPL
778 (
779 Memory *pMemory,
780 CALL_CONTEXT *pCallContext,
781 RS_RES_CONTROL_PARAMS_INTERNAL *pParams
782 )
783 {
784 RmCtrlParams *pRmCtrlParams = pParams->pLegacyParams;
785
786 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE));
787
788 if (!pMemory->pGpu)
789 return NV_ERR_INVALID_OBJECT_PARENT;
790
791 if (REF_VAL(NVXXXX_CTRL_CMD_CLASS, pParams->cmd) == NV04_MEMORY)
792 {
793 //
794 // Tegra SOC import memory usecase uses NV01_MEMORY_SYSTEM_OS_DESCRIPTOR class for
795 // RM resource server registration of memory, RM can return the physical memory attributes
796 // for these imported buffers.
797 //
798 if ((pMemory->categoryClassId == NV01_MEMORY_SYSTEM_OS_DESCRIPTOR) &&
799 (pParams->cmd != NV0041_CTRL_CMD_GET_SURFACE_PHYS_ATTR))
800 {
801 return NV_ERR_NOT_SUPPORTED;
802 }
803 }
804
805 pRmCtrlParams->pGpu = pMemory->pGpu;
806
807 gpuSetThreadBcState(pMemory->pGpu, pMemory->bBcResource);
808
809 return resControl_IMPL(staticCast(pMemory, RsResource), pCallContext, pParams);
810 }
811
812 NV_STATUS
memCopyConstruct_IMPL(Memory * pMemory,CALL_CONTEXT * pCallContext,RS_RES_ALLOC_PARAMS_INTERNAL * pParams)813 memCopyConstruct_IMPL
814 (
815 Memory *pMemory,
816 CALL_CONTEXT *pCallContext,
817 RS_RES_ALLOC_PARAMS_INTERNAL *pParams
818 )
819 {
820 RsClient *pDstClient = pCallContext->pClient;
821 RsClient *pSrcClient = pParams->pSrcClient;
822 RsResourceRef *pDstRef = pCallContext->pResourceRef;
823 RsResourceRef *pSrcRef = pParams->pSrcRef;
824 Memory *pMemorySrc = dynamicCast(pSrcRef->pResource, Memory);
825 Memory *pMemoryDst = pMemory;
826 OBJGPU *pSrcGpu = NULL;
827 OBJGPU *pDstGpu = NULL;
828 NV_STATUS status = NV_OK;
829 NvBool bReleaseGpuLock = NV_FALSE;
830 Device *pSrcDevice = NULL;
831 Device *pDstDevice = NULL;
832 Subdevice *pSrcSubDevice = NULL;
833 Subdevice *pDstSubDevice = NULL;
834 RsResourceRef *pSrcParentRef = pSrcRef->pParentRef;
835 RsResourceRef *pDstParentRef = pDstRef->pParentRef;
836
837 NV_ASSERT_OR_RETURN(pSrcParentRef != NULL, NV_ERR_INVALID_OBJECT_PARENT);
838 NV_ASSERT_OR_RETURN(pDstParentRef != NULL, NV_ERR_INVALID_OBJECT_PARENT);
839 NV_ASSERT_OR_RETURN(pMemorySrc != NULL, NV_ERR_INVALID_OBJECT_HANDLE);
840
841 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemorySrc, NV_TRUE));
842
843 //
844 // Must return early when parent is Client.
845 // This copy constructor is very device-specific so it is up
846 // to the device-less Memory subclasses to define their own dup behavior.
847 //
848 if (RES_GET_CLIENT_HANDLE(pMemoryDst) == RES_GET_PARENT_HANDLE(pMemoryDst))
849 {
850 NV_ASSERT_OR_RETURN(RES_GET_CLIENT_HANDLE(pMemorySrc) ==
851 RES_GET_PARENT_HANDLE(pMemorySrc),
852 NV_ERR_INVALID_OBJECT_PARENT);
853 return NV_OK;
854 }
855
856 pSrcGpu = pMemorySrc->pGpu;
857 pDstGpu = pMemoryDst->pGpu;
858 pSrcDevice = pMemorySrc->pDevice;
859 pDstDevice = pMemoryDst->pDevice;
860 pSrcSubDevice = pMemorySrc->pSubDevice;
861 pDstSubDevice = pMemoryDst->pSubDevice;
862
863 // Only children of device are supported
864 NV_ASSERT_OR_RETURN(pSrcDevice != NULL, NV_ERR_INVALID_OBJECT_PARENT);
865 NV_ASSERT_OR_RETURN(pDstDevice != NULL, NV_ERR_INVALID_OBJECT_PARENT);
866
867 if (!!pSrcSubDevice != !!pDstSubDevice)
868 {
869 NV_PRINTF(LEVEL_INFO, "Parent type mismatch between Src and Dst objects"
870 "Both should be either device or subDevice\n");
871 return NV_ERR_INVALID_OBJECT_PARENT;
872 }
873
874 // RS-TODO: This should use pMemorySrc->bBcResource when adding full support for subdevice duping
875 gpuSetThreadBcState(pSrcGpu, NV_TRUE);
876
877 if (!rmGpuLockIsOwner() &&
878 !(rmDeviceGpuLockIsOwner(pSrcGpu->gpuInstance) &&
879 rmDeviceGpuLockIsOwner(pDstGpu->gpuInstance)))
880 {
881 // LOCK: acquire GPUs lock
882 if ((status = rmGpuLocksAcquire(GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_MEM)) != NV_OK)
883 {
884 NV_PRINTF(LEVEL_ERROR,
885 "Failed to acquire GPU locks, error 0x%x\n", status);
886 return status;
887 }
888
889 bReleaseGpuLock = NV_TRUE;
890 }
891
892 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR,
893 memCheckCopyPermissions(pMemorySrc, pDstGpu, pDstDevice), done);
894
895 // Initialize Memory
896 pMemoryDst->categoryClassId = pMemorySrc->categoryClassId;
897 pMemoryDst->Length = pMemorySrc->Length;
898 pMemoryDst->HeapOwner = pMemorySrc->HeapOwner;
899 pMemoryDst->pHeap = pMemorySrc->pHeap;
900 pMemoryDst->pMemDesc = pMemorySrc->pMemDesc;
901 pMemoryDst->KernelVAddr = NvP64_NULL;
902 pMemoryDst->KernelMapPriv = NvP64_NULL;
903 pMemoryDst->Attr = pMemorySrc->Attr;
904 pMemoryDst->Attr2 = pMemorySrc->Attr2;
905 pMemoryDst->Pitch = pMemorySrc->Pitch;
906 pMemoryDst->Type = pMemorySrc->Type;
907 pMemoryDst->Flags = pMemorySrc->Flags;
908 pMemoryDst->tag = pMemorySrc->tag;
909 pMemoryDst->pHwResource = pMemorySrc->pHwResource;
910 pMemoryDst->isMemDescOwner = NV_FALSE;
911 pMemoryDst->bRpcAlloc = pMemorySrc->bRpcAlloc;
912
913 // Link in the new device memory mapping
914 pMemoryDst->Node.keyStart = RES_GET_HANDLE(pMemoryDst);
915 pMemoryDst->Node.keyEnd = RES_GET_HANDLE(pMemoryDst);
916 pMemoryDst->Node.Data = pMemoryDst;
917
918 status = btreeInsert(&pMemoryDst->Node, &pDstDevice->DevMemoryTable);
919 if (status != NV_OK)
920 goto done;
921
922 {
923 OBJGPU *pGpu = pDstGpu; // Need pGpu for SLI loop
924
925 gpuSetThreadBcState(pDstGpu, NV_TRUE);
926 SLI_LOOP_START(SLI_LOOP_FLAGS_BC_ONLY)
927 if (memdescGetPageSize(memdescGetMemDescFromGpu(pMemoryDst->pMemDesc, pGpu), AT_GPU) == 0)
928 {
929 status = memmgrSetMemDescPageSize_HAL(pGpu, GPU_GET_MEMORY_MANAGER(pGpu),
930 memdescGetMemDescFromGpu(pMemoryDst->pMemDesc, pGpu),
931 AT_GPU, RM_ATTR_PAGE_SIZE_DEFAULT);
932 NV_ASSERT(status == NV_OK);
933 }
934 SLI_LOOP_END
935 }
936
937 //
938 // ref-count increments for shared structs after all places where we
939 // could return early.
940 //
941 if (pMemoryDst->pHwResource != NULL)
942 pMemoryDst->pHwResource->refCount++;
943
944 memdescAddRef(pMemoryDst->pMemDesc);
945 pMemoryDst->pMemDesc->DupCount++;
946 if (pMemoryDst->pMemDesc->Allocated)
947 pMemoryDst->pMemDesc->Allocated++;
948
949 // Insert pMemoryDst after pMemorySrc in circular list to track dup/sharing of pMemDesc
950 pMemoryDst->dupListItem.pNext = pMemorySrc->dupListItem.pNext;
951 pMemoryDst->dupListItem.pPrev = pMemorySrc;
952 pMemorySrc->dupListItem.pNext = pMemoryDst;
953 pMemoryDst->dupListItem.pNext->dupListItem.pPrev = pMemoryDst;
954
955 done:
956
957 // If the original allocation was RPCed, also send the Dup.
958 if (pMemory->bRpcAlloc && (IS_VIRTUAL(pSrcGpu) || IS_GSP_CLIENT(pSrcGpu)))
959 {
960 NV_RM_RPC_DUP_OBJECT(pSrcGpu, pDstClient->hClient, pDstParentRef->hResource, pDstRef->hResource,
961 pSrcClient->hClient, pSrcRef->hResource, 0,
962 NV_FALSE, // do not automatically issue RPC_FREE on object free
963 NULL,
964 status);
965 NV_ASSERT(status == NV_OK);
966 }
967
968 // UNLOCK: release GPUs lock
969 if (bReleaseGpuLock)
970 {
971 rmGpuLocksRelease(GPUS_LOCK_FLAGS_NONE, NULL);
972 }
973
974 pMemory->bConstructed = (status == NV_OK);
975 return status;
976 }
977
978 NV_STATUS
memGetMemInterMapParams_IMPL(Memory * pMemory,RMRES_MEM_INTER_MAP_PARAMS * pParams)979 memGetMemInterMapParams_IMPL
980 (
981 Memory *pMemory,
982 RMRES_MEM_INTER_MAP_PARAMS *pParams
983 )
984 {
985 OBJGPU *pGpu = pParams->pGpu;
986 RsResourceRef *pMemoryRef = pParams->pMemoryRef;
987
988 FlaMemory *pFlaMemory;
989
990 MEMORY_DESCRIPTOR *pSrcMemDesc = pMemory->pMemDesc;
991 Device *pDevice;
992 Subdevice *pSubdevice;
993 NvBool bcState = gpumgrGetBcEnabledStatus(pGpu);
994
995 // Don't expect to use default, but safe thing to do is set src=dest
996 NvHandle hMemoryDevice = 0;
997 OBJGPU *pSrcGpu = pGpu;
998
999 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE));
1000
1001 if (pMemoryRef->pParentRef != NULL)
1002 {
1003 pDevice = dynamicCast(pMemoryRef->pParentRef->pResource, Device);
1004 if (pDevice != NULL)
1005 {
1006 pSrcGpu = GPU_RES_GET_GPU(pDevice);
1007 hMemoryDevice = RES_GET_HANDLE(pDevice);
1008 GPU_RES_SET_THREAD_BC_STATE(pDevice);
1009 }
1010 else
1011 {
1012 pSubdevice = dynamicCast(pMemoryRef->pParentRef->pResource, Subdevice);
1013 if (pSubdevice != NULL)
1014 {
1015 pSrcGpu = GPU_RES_GET_GPU(pSubdevice);
1016 hMemoryDevice = RES_GET_HANDLE(pSubdevice->pDevice);
1017 GPU_RES_SET_THREAD_BC_STATE(pSubdevice);
1018 }
1019 }
1020 }
1021
1022 pParams->pSrcGpu = pSrcGpu;
1023 pParams->hMemoryDevice = hMemoryDevice;
1024
1025 //
1026 // Restore pGpu's bcState in case it was overwritten above (i.e.,
1027 // the case that hMemoryDevice and hBroadcastDevice are the same
1028 // device, but a unicast mapping was desired).
1029 //
1030 gpumgrSetBcEnabledStatus(pGpu, bcState);
1031
1032 pParams->pSrcMemDesc = pSrcMemDesc;
1033
1034 pFlaMemory = dynamicCast(pMemoryRef->pResource, FlaMemory);
1035 if (pFlaMemory != NULL)
1036 {
1037 pParams->pSrcGpu = gpumgrGetGpu(pFlaMemory->peerGpuInst);
1038 pParams->bFlaMapping = NV_TRUE;
1039
1040 NV_PRINTF(LEVEL_INFO, "FLA memory imported as (%s) with exportGpu:%x \n",
1041 (pParams->pSrcGpu != pGpu ? " P2P " : " LOOPBACK "),
1042 pFlaMemory->peerDeviceInst);
1043 }
1044
1045 return NV_OK;
1046 }
1047
1048 NV_STATUS
memGetMemoryMappingDescriptor_IMPL(Memory * pMemory,MEMORY_DESCRIPTOR ** ppMemDesc)1049 memGetMemoryMappingDescriptor_IMPL
1050 (
1051 Memory *pMemory,
1052 MEMORY_DESCRIPTOR **ppMemDesc
1053 )
1054 {
1055 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, memIsReady(pMemory, NV_FALSE));
1056 if (pMemory->pGpu != NULL)
1057 {
1058 *ppMemDesc = memdescGetMemDescFromGpu(pMemory->pMemDesc, pMemory->pGpu);
1059 }
1060 else
1061 {
1062 *ppMemDesc = pMemory->pMemDesc;
1063 }
1064 return NV_OK;
1065 }
1066
1067 NV_STATUS
memIsDuplicate_IMPL(Memory * pMemory,NvHandle hMemory,NvBool * pDuplicate)1068 memIsDuplicate_IMPL
1069 (
1070 Memory *pMemory,
1071 NvHandle hMemory,
1072 NvBool *pDuplicate
1073 )
1074 {
1075 RsClient *pClient = RES_GET_CLIENT(pMemory);
1076 Memory *pMemory1;
1077
1078 NV_CHECK_OK_OR_RETURN(LEVEL_SILENT,
1079 memIsReady(pMemory, NV_FALSE));
1080
1081 NV_CHECK_OK_OR_RETURN(LEVEL_SILENT,
1082 memGetByHandle(pClient, hMemory, &pMemory1));
1083
1084 //
1085 // Do not dereference pMemdesc here. We only take RMAPI RO lock and
1086 // client lock in this context.
1087 //
1088
1089 *pDuplicate = (pMemory->pMemDesc == pMemory1->pMemDesc);
1090
1091 return NV_OK;
1092 }
1093