1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2012-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /******************* OS Memory Descriptor APIS *****************************\
25 * *
26 * This contains routines to create and destroy OS memory descriptor *
27 * *
28 ****************************************************************************/
29
30 #include <nv.h> // NV device driver interface
31 #include <os/os.h>
32 #include <osapi.h>
33 #include <nvos.h>
34 #include <rmapi/client.h>
35 #include <rmapi/rs_utils.h>
36 #include <gpu_mgr/gpu_mgr.h>
37
38 #include "gpu/bif/kernel_bif.h"
39
40 static NV_STATUS osCreateOsDescriptorFromPageArray(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void **);
41 static void osDestroyOsDescriptorPageArray(PMEMORY_DESCRIPTOR);
42
43 static NV_STATUS osCreateOsDescriptorFromIoMemory(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
44 static void osDestroyOsDescriptorFromIoMemory(PMEMORY_DESCRIPTOR);
45
46 static NV_STATUS osCreateOsDescriptorFromPhysAddr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
47 static void osDestroyOsDescriptorFromPhysAddr(PMEMORY_DESCRIPTOR);
48
49 static NV_STATUS osCreateOsDescriptorFromFileHandle(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
50 static NV_STATUS osCreateOsDescriptorFromDmaBufPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
51 static void osDestroyOsDescriptorFromDmaBuf(PMEMORY_DESCRIPTOR);
52 static NV_STATUS osCreateOsDescriptorFromSgtPtr(OBJGPU *, NvP64, NvHandle, NvU32, NvU64 *, MEMORY_DESCRIPTOR **, void**);
53 static void osDestroyOsDescriptorFromSgt(PMEMORY_DESCRIPTOR);
54
55 static NV_STATUS osCheckGpuBarsOverlapAddrRange(NvRangeU64 addrRange);
56
57 NV_STATUS
osCreateMemFromOsDescriptor(OBJGPU * pGpu,NvP64 pDescriptor,NvHandle hClient,NvU32 flags,NvU64 * pLimit,MEMORY_DESCRIPTOR ** ppMemDesc,NvU32 descriptorType,RS_PRIV_LEVEL privilegeLevel)58 osCreateMemFromOsDescriptor
59 (
60 OBJGPU *pGpu,
61 NvP64 pDescriptor,
62 NvHandle hClient,
63 NvU32 flags,
64 NvU64 *pLimit,
65 MEMORY_DESCRIPTOR **ppMemDesc,
66 NvU32 descriptorType,
67 RS_PRIV_LEVEL privilegeLevel
68 )
69 {
70 RmClient* pClient;
71 NV_STATUS rmStatus;
72 void *pPrivate;
73
74 pClient = serverutilGetClientUnderLock(hClient);
75 if ((pDescriptor == NvP64_NULL) ||
76 (*pLimit == 0) ||
77 (pClient == NULL))
78 {
79 return NV_ERR_INVALID_PARAM_STRUCT;
80 }
81
82 //
83 // For the sake of simplicity, unmatched RM and OS page
84 // sizes are not currently supported in this path, except for
85 // PPC64LE and aarch64.
86 //
87 // Also, the nvmap handle is sent which can be any random number so
88 // the virtual address alignment sanity check can't be done here.
89 //
90 if (!NVCPU_IS_PPC64LE &&
91 !NVCPU_IS_AARCH64 &&
92 (NV_RM_PAGE_SIZE != os_page_size))
93 {
94 return NV_ERR_NOT_SUPPORTED;
95 }
96
97 //
98 // The two checks below use cached privilege because they
99 // concern the privilege level of the client, and not the
100 // privilege level of the calling context which may be
101 // overridden to KERNEL at some internal callsites.
102 //
103
104 //
105 // The RM cannot obtain a table of physical addresses
106 // for a kernel virtual address range on all of
107 // the supported UNIX platforms. Since this path is
108 // not really compelling for kernel allocations on any
109 // of those platforms, it is not supported.
110 // For UVM, they could have pre-allocated sysmem to register
111 // with RM so we put in an exception for that case.
112 //
113 if ((rmclientGetCachedPrivilege(pClient) >= RS_PRIV_LEVEL_KERNEL) &&
114 (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR) &&
115 (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE) &&
116 (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) &&
117 (descriptorType != NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR))
118 {
119 return NV_ERR_NOT_SUPPORTED;
120 }
121
122 //
123 // NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR and
124 // NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR can only be utilized by kernel space
125 // rm-clients.
126 //
127 if ((rmclientGetCachedPrivilege(pClient) < RS_PRIV_LEVEL_KERNEL) &&
128 ((descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR) ||
129 (descriptorType == NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR)))
130 {
131 return NV_ERR_NOT_SUPPORTED;
132 }
133
134 switch (descriptorType)
135 {
136 case NVOS32_DESCRIPTOR_TYPE_VIRTUAL_ADDRESS:
137 rmStatus = NV_ERR_NOT_SUPPORTED;
138 break;
139 case NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR:
140 if (privilegeLevel < RS_PRIV_LEVEL_KERNEL)
141 {
142 rmStatus = NV_ERR_NOT_SUPPORTED;
143 break;
144 }
145 rmStatus = osCreateOsDescriptorFromPhysAddr(pGpu, pDescriptor,
146 hClient, flags, pLimit, ppMemDesc, &pPrivate);
147 break;
148 case NVOS32_DESCRIPTOR_TYPE_OS_IO_MEMORY:
149 rmStatus = osCreateOsDescriptorFromIoMemory(pGpu, pDescriptor,
150 hClient, flags, pLimit, ppMemDesc, &pPrivate);
151 break;
152 case NVOS32_DESCRIPTOR_TYPE_OS_PAGE_ARRAY:
153 rmStatus = osCreateOsDescriptorFromPageArray(pGpu, pDescriptor,
154 hClient, flags, pLimit, ppMemDesc, &pPrivate);
155 break;
156 case NVOS32_DESCRIPTOR_TYPE_OS_FILE_HANDLE:
157 rmStatus = osCreateOsDescriptorFromFileHandle(pGpu, pDescriptor,
158 hClient, flags, pLimit, ppMemDesc, &pPrivate);
159 break;
160 case NVOS32_DESCRIPTOR_TYPE_OS_DMA_BUF_PTR:
161 rmStatus = osCreateOsDescriptorFromDmaBufPtr(pGpu, pDescriptor,
162 hClient, flags, pLimit, ppMemDesc, &pPrivate);
163 break;
164 case NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR:
165 rmStatus = osCreateOsDescriptorFromSgtPtr(pGpu, pDescriptor,
166 hClient, flags, pLimit, ppMemDesc, &pPrivate);
167 break;
168 default:
169 rmStatus = NV_ERR_INVALID_ARGUMENT;
170 break;
171 }
172
173 return rmStatus;
174 }
175
176 static NV_STATUS
osCreateMemdescFromPages(OBJGPU * pGpu,NvU64 size,NvU32 flags,NvU32 cacheType,MEMORY_DESCRIPTOR ** ppMemDesc,void * pImportPriv,void ** ppPrivate)177 osCreateMemdescFromPages
178 (
179 OBJGPU *pGpu,
180 NvU64 size,
181 NvU32 flags,
182 NvU32 cacheType,
183 MEMORY_DESCRIPTOR **ppMemDesc,
184 void *pImportPriv,
185 void **ppPrivate
186 )
187 {
188 NV_STATUS rmStatus;
189 MEMORY_DESCRIPTOR *pMemDesc;
190 NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
191 NvU32 gpuCachedFlags;
192
193 if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
194 {
195 memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
196 }
197
198 rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
199 NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM,
200 cacheType, memdescFlags);
201 if (rmStatus != NV_OK)
202 {
203 return rmStatus;
204 }
205
206 if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
207 gpuCachedFlags = NV_MEMORY_CACHED;
208 else
209 gpuCachedFlags = NV_MEMORY_UNCACHED;
210
211 pMemDesc = *ppMemDesc;
212 rmStatus = nv_register_user_pages(NV_GET_NV_STATE(pGpu),
213 NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
214 memdescGetPteArray(pMemDesc, AT_CPU), pImportPriv,
215 ppPrivate);
216 if (rmStatus != NV_OK)
217 {
218 memdescDestroy(pMemDesc);
219 return rmStatus;
220 }
221
222 memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
223 memdescSetAddress(pMemDesc, NvP64_NULL);
224 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
225 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
226
227 //
228 // If the OS layer doesn't think in RM page size, we need to inflate the
229 // PTE array into RM pages.
230 //
231 if ((NV_RM_PAGE_SIZE < os_page_size) &&
232 !memdescGetContiguity(pMemDesc, AT_CPU))
233 {
234 RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
235 pMemDesc->PageCount);
236 }
237
238 //
239 // memdescMapIommu() requires the OS-private data to be set on the memory
240 // descriptor, but we don't want to wire up the teardown callback just yet:
241 // that callback needs to unpin the pages, but that will already be done
242 // as part of failure handling further up the stack if memdescMapIommu()
243 // fails. So we only set up the priv-data cleanup callback once we're sure
244 // this call will succeed.
245 //
246 memdescSetMemData(pMemDesc, *ppPrivate, NULL);
247
248 rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
249 if (rmStatus != NV_OK)
250 {
251 if ((NV_RM_PAGE_SIZE < os_page_size) &&
252 !memdescGetContiguity(pMemDesc, AT_CPU))
253 {
254 RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
255 pMemDesc->PageCount);
256 }
257
258 nv_unregister_user_pages(NV_GET_NV_STATE(pGpu),
259 NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
260 NULL /* import_priv */, ppPrivate);
261 memdescDestroy(pMemDesc);
262 return rmStatus;
263 }
264
265 return NV_OK;
266 }
267
268 static NV_STATUS
osCreateOsDescriptorFromPageArray(OBJGPU * pGpu,NvP64 pDescriptor,NvHandle hClient,NvU32 flags,NvU64 * pLimit,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)269 osCreateOsDescriptorFromPageArray
270 (
271 OBJGPU *pGpu,
272 NvP64 pDescriptor,
273 NvHandle hClient,
274 NvU32 flags,
275 NvU64 *pLimit,
276 MEMORY_DESCRIPTOR **ppMemDesc,
277 void **ppPrivate
278 )
279 {
280 NV_STATUS rmStatus;
281
282 *ppPrivate = NvP64_VALUE(pDescriptor);
283
284 //
285 // Since the only type of memory permitted in this path
286 // is anonymous user memory, certain restrictions
287 // apply for the allocation flags:
288 //
289 // 1) anonymous memory is write-back cacheable, hence
290 // the _COHERENCY flag must match.
291 //
292 // 2) the RM has no control over the location of the
293 // associated pages in memory and thus cannot
294 // honor requests for contiguous memory.
295 //
296 // These restrictions are enforced here to avoid subtle
297 // bugs later on.
298 //
299 if ((!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _CACHED, flags) &&
300 !FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_BACK, flags)) ||
301 FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
302 {
303 return NV_ERR_INVALID_FLAGS;
304 }
305
306 rmStatus = osCreateMemdescFromPages(pGpu, (*pLimit + 1), flags,
307 NV_MEMORY_CACHED, ppMemDesc,
308 NULL /* pImportPriv */, ppPrivate);
309 if (rmStatus != NV_OK)
310 {
311 return rmStatus;
312 }
313
314 // All is well - wire up the cleanup callback now
315 memdescSetMemData(*ppMemDesc, memdescGetMemData(*ppMemDesc),
316 osDestroyOsDescriptorPageArray);
317
318 return NV_OK;
319 }
320
321 /*!
322 * @brief Checks if the given address range overlaps with the BARs for any of
323 * the GPUs.
324 */
325 static NV_STATUS
osCheckGpuBarsOverlapAddrRange(NvRangeU64 addrRange)326 osCheckGpuBarsOverlapAddrRange
327 (
328 NvRangeU64 addrRange
329 )
330 {
331 NvRangeU64 gpuPhysAddrRange;
332 NvRangeU64 gpuPhysFbAddrRange;
333 NvRangeU64 gpuPhysInstAddrRange;
334 NvU32 gpuInstance;
335 OBJGPU *pGpu;
336 NvU32 gpuMask;
337 NV_STATUS rmStatus;
338
339 rmStatus = gpumgrGetGpuAttachInfo(NULL, &gpuMask);
340 NV_ASSERT_OR_RETURN(rmStatus == NV_OK, rmStatus);
341
342 gpuInstance = 0;
343 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
344 {
345 NV_INIT_RANGE(gpuPhysFbAddrRange, gpumgrGetGpuPhysFbAddr(pGpu),
346 gpumgrGetGpuPhysFbAddr(pGpu) + pGpu->fbLength -1);
347
348 NV_INIT_RANGE(gpuPhysAddrRange, pGpu->busInfo.gpuPhysAddr,
349 pGpu->busInfo.gpuPhysAddr + pGpu->deviceMappings[0].gpuNvLength -1);
350
351 NV_INIT_RANGE(gpuPhysInstAddrRange, pGpu->busInfo.gpuPhysInstAddr,
352 pGpu->busInfo.gpuPhysInstAddr + pGpu->instLength -1);
353
354 if (NV_IS_OVERLAPPING_RANGE(gpuPhysFbAddrRange, addrRange) ||
355 NV_IS_OVERLAPPING_RANGE(gpuPhysAddrRange, addrRange) ||
356 NV_IS_OVERLAPPING_RANGE(gpuPhysInstAddrRange, addrRange))
357 {
358 return NV_ERR_INVALID_ADDRESS;
359 }
360 }
361
362 return NV_OK;
363 }
364
365 static NvU64
_doWarBug4040336(OBJGPU * pGpu,NvU64 addr)366 _doWarBug4040336
367 (
368 OBJGPU *pGpu,
369 NvU64 addr
370 )
371 {
372 if (gpuIsWarBug4040336Enabled(pGpu))
373 {
374 if ((addr & 0xffffffff00000000ULL) == 0x7fff00000000ULL)
375 {
376 addr = addr & 0xffffffffULL;
377 }
378 }
379 return addr;
380 }
381
382 static NV_STATUS
osCreateOsDescriptorFromIoMemory(OBJGPU * pGpu,NvP64 pDescriptor,NvHandle hClient,NvU32 flags,NvU64 * pLimit,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)383 osCreateOsDescriptorFromIoMemory
384 (
385 OBJGPU *pGpu,
386 NvP64 pDescriptor,
387 NvHandle hClient,
388 NvU32 flags,
389 NvU64 *pLimit,
390 MEMORY_DESCRIPTOR **ppMemDesc,
391 void **ppPrivate
392 )
393 {
394 NV_STATUS rmStatus;
395 NvU32 gpuCachedFlags;
396 MEMORY_DESCRIPTOR *pMemDesc;
397 NvU64 *pPteArray;
398 NvRangeU64 physAddrRange;
399 NvU64 *base = 0;
400 NvBool bAllowMmap;
401
402 //
403 // Unlike the page array path, this one deals exclusively
404 // with I/O memory, which is expected to be contiguous
405 // physically, and which may only be accessed with uncached
406 // transactions.
407 //
408 if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags) ||
409 !FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
410 {
411 return NV_ERR_INVALID_FLAGS;
412 }
413
414 //
415 // _PEER_MAP_OVERRIDE flag is controlled by the RM and not the client.
416 //
417 // RM will set the _PEER_MAP_OVERRIDE_REQUIRED flag itself for IO memory
418 // memory imported with RmVidHeapControl.
419 //
420 if (FLD_TEST_DRF(OS02, _FLAGS, _PEER_MAP_OVERRIDE, _REQUIRED, flags))
421 {
422 //
423 // Don't allow MMIO mappings for unprivileged users
424 // This is a temporary WAR for bug 1630288 "[PeerSync] threat related
425 // to GPU"
426 //
427 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
428 if (!pKernelBif->peerMappingOverride && !osIsAdministrator())
429 {
430 NV_PRINTF(LEVEL_ERROR,
431 "%s(): permission denied, allowPeermapping=%d\n",
432 __FUNCTION__, pKernelBif->peerMappingOverride);
433 return NV_ERR_INSUFFICIENT_PERMISSIONS;
434 }
435 }
436
437 bAllowMmap = !FLD_TEST_DRF(OS02, _FLAGS, _MAPPING, _NEVER_MAP, flags);
438
439 base = (void *)(NvUPtr)pDescriptor;
440
441 //
442 // There is an architectural deadlock scenario involved when full-duplex P2P
443 // enabled over BAR1. See #3 in the description of bug 1571948 which explains
444 // the classic deadlock. So, make sure to error out usermode's memory
445 // registration if a memory range falls within any of the available GPU's
446 // BAR window.
447 //
448 physAddrRange.min = *base;
449 physAddrRange.max = *base + *pLimit;
450
451 rmStatus = osCheckGpuBarsOverlapAddrRange(physAddrRange);
452 if (rmStatus != NV_OK)
453 {
454 NV_PRINTF(LEVEL_ERROR,
455 "%s(): phys range 0x%016llx-0x%016llx overlaps with GPU BARs",
456 __FUNCTION__, physAddrRange.min, physAddrRange.max);
457 return rmStatus;
458 }
459
460 //
461 // BF3's PCIe MMIO bus address at 0x800000000000(CPU PA 0x7fff00000000) is
462 // too high for Ampere to address. As a result, BF3's bus address is
463 // moved to < 4GB. Now, the CPU PA and the bus address are no longer 1:1
464 // and needs to be adjusted.
465 //
466 *base = _doWarBug4040336(pGpu, *base);
467
468 rmStatus = memdescCreate(ppMemDesc, pGpu, (*pLimit + 1), 0,
469 NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM,
470 NV_MEMORY_UNCACHED, MEMDESC_FLAGS_NONE);
471 if (rmStatus != NV_OK)
472 {
473 NV_PRINTF(LEVEL_ERROR,
474 "%s(): error %d while attempting to create the MMIO mapping\n",
475 __FUNCTION__, rmStatus);
476 return rmStatus;
477 }
478
479 pMemDesc = *ppMemDesc;
480
481 if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
482 gpuCachedFlags = NV_MEMORY_CACHED;
483 else
484 gpuCachedFlags = NV_MEMORY_UNCACHED;
485
486 memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
487 memdescSetAddress(pMemDesc, NvP64_NULL);
488 memdescSetMemData(pMemDesc, NULL, NULL);
489 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
490 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_PEER_IO_MEM, NV_TRUE);
491
492 pPteArray = memdescGetPteArray(pMemDesc, AT_CPU);
493 pPteArray[0] = *base;
494
495 *ppPrivate = NULL;
496
497 if (bAllowMmap)
498 {
499 rmStatus = nv_register_peer_io_mem(NV_GET_NV_STATE(pGpu), pPteArray,
500 NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
501 ppPrivate);
502 if (rmStatus != NV_OK)
503 {
504 memdescDestroy(pMemDesc);
505 return rmStatus;
506 }
507 }
508
509 memdescSetMemData(pMemDesc, *ppPrivate, NULL);
510
511 //
512 // memdescMapIommu() requires the OS-private data to be set on the memory
513 // descriptor, but we don't want to wire up the teardown callback just yet:
514 // that callback needs to unpin the pages, but that will already be done
515 // as part of failure handling further up the stack if memdescMapIommu()
516 // fails. So we only set up the priv-data cleanup callback once we're sure
517 // this call will succeed.
518 //
519 rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
520 if (rmStatus != NV_OK)
521 {
522 if (*ppPrivate != NULL)
523 {
524 nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), *ppPrivate);
525 }
526 memdescDestroy(pMemDesc);
527 return rmStatus;
528 }
529
530 // All is well - wire up the cleanup callback now
531 memdescSetMemData(pMemDesc, memdescGetMemData(pMemDesc),
532 osDestroyOsDescriptorFromIoMemory);
533
534 return NV_OK;
535 }
536
537 static NV_STATUS
osCreateOsDescriptorFromPhysAddr(OBJGPU * pGpu,NvP64 pDescriptor,NvHandle hClient,NvU32 flags,NvU64 * pLimit,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)538 osCreateOsDescriptorFromPhysAddr
539 (
540 OBJGPU *pGpu,
541 NvP64 pDescriptor,
542 NvHandle hClient,
543 NvU32 flags,
544 NvU64 *pLimit,
545 MEMORY_DESCRIPTOR **ppMemDesc,
546 void **ppPrivate
547 )
548 {
549 NV_STATUS rmStatus;
550 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
551 MEMORY_DESCRIPTOR *pMemDesc;
552 NvU64 *pPteArray;
553 NvU64 base = 0;
554 NvU32 cache_type = NV_MEMORY_CACHED;
555 NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
556 NvU64 *pPhys_addrs;
557 NvU64 num_os_pages;
558 NvU32 idx;
559
560 // Currently only work with contiguous sysmem allocations
561 if (!FLD_TEST_DRF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS, flags))
562 {
563 return NV_ERR_INVALID_FLAGS;
564 }
565
566 if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags))
567 {
568 // Syncpoint memory is uncached, DMA mapping needs to skip CPU sync.
569 cache_type = NV_MEMORY_UNCACHED;
570
571 //
572 // Syncpoint memory is NISO. Don't attempt to IOMMU map if the NISO
573 // IOMMU isn't enabled.
574 //
575 if (!NV_SOC_IS_NISO_IOMMU_PRESENT(nv))
576 {
577 memdescFlags |= MEMDESC_FLAGS_SKIP_IOMMU_MAPPING;
578 }
579 }
580
581 if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
582 {
583 memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
584 }
585
586 base = (NvU64)pDescriptor;
587 rmStatus = memdescCreate(ppMemDesc, pGpu, (*pLimit + 1), 0,
588 NV_MEMORY_CONTIGUOUS, ADDR_SYSMEM,
589 cache_type, memdescFlags);
590 if (rmStatus != NV_OK)
591 {
592 NV_PRINTF(LEVEL_ERROR,
593 "%s(): error %d while creating memdesc for kernel memory\n",
594 __FUNCTION__, rmStatus);
595 return rmStatus;
596 }
597
598 pMemDesc = *ppMemDesc;
599
600 memdescSetAddress(pMemDesc, NvP64_NULL);
601 memdescSetMemData(pMemDesc, NULL, NULL);
602 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
603
604 pPteArray = memdescGetPteArray(pMemDesc, AT_CPU);
605 pPteArray[0] = base;
606
607 num_os_pages = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
608 pPhys_addrs = portMemAllocNonPaged(sizeof(NvU64) * num_os_pages);
609 if (pPhys_addrs == NULL)
610 goto cleanup_memdesc;
611
612 for (idx = 0; idx < num_os_pages; idx++)
613 {
614 pPhys_addrs[idx] = base + (idx * os_page_size);
615 }
616
617 *ppPrivate = NULL;
618 rmStatus = nv_register_phys_pages(nv, pPhys_addrs, num_os_pages,
619 memdescGetCpuCacheAttrib(pMemDesc),
620 ppPrivate);
621 if (rmStatus != NV_OK)
622 goto cleanup_memdesc;
623
624 //
625 // For syncpoint memory, if IOMMU skip flag wasn't set earlier,
626 // create IOVA mapping.
627 //
628 if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_TYPE_SYNCPOINT, _APERTURE, flags) &&
629 !memdescGetFlag(pMemDesc, MEMDESC_FLAGS_SKIP_IOMMU_MAPPING))
630 {
631 //
632 // memdescMapIommu() requires the OS-private data to be set on the memory
633 // descriptor, but we don't want to wire up the teardown callback just yet:
634 // that callback needs to unpin the pages, but that will already be done
635 // as part of failure handling further up the stack if memdescMapIommu()
636 // fails. So we only set up the priv-data cleanup callback once we're sure
637 // this call will succeed.
638 //
639 memdescSetMemData(pMemDesc, *ppPrivate, NULL);
640
641 rmStatus = memdescMapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
642 if (rmStatus != NV_OK)
643 goto cleanup_pages;
644 }
645
646 // All is well - wire up the cleanup callback now
647 memdescSetMemData(pMemDesc, *ppPrivate,
648 osDestroyOsDescriptorFromPhysAddr);
649
650 portMemFree(pPhys_addrs);
651
652 return NV_OK;
653
654 cleanup_pages:
655 if (*ppPrivate != NULL)
656 {
657 nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), *ppPrivate);
658 }
659
660 cleanup_memdesc:
661 memdescDestroy(pMemDesc);
662
663 portMemFree(pPhys_addrs);
664
665 return rmStatus;
666 }
667
668 static NV_STATUS
_createMemdescFromDmaBufSgtHelper(OBJGPU * pGpu,NvU32 flags,void * pImportPriv,struct sg_table * pImportSgt,NvU32 size,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate,MEM_DATA_RELEASE_CALL_BACK * pMemDataReleaseCallback)669 _createMemdescFromDmaBufSgtHelper
670 (
671 OBJGPU *pGpu,
672 NvU32 flags,
673 void *pImportPriv,
674 struct sg_table *pImportSgt,
675 NvU32 size,
676 MEMORY_DESCRIPTOR **ppMemDesc,
677 void **ppPrivate,
678 MEM_DATA_RELEASE_CALL_BACK *pMemDataReleaseCallback
679 )
680 {
681 NV_STATUS rmStatus = NV_OK;
682 NvU32 cacheType = NV_MEMORY_UNCACHED;
683 MEMORY_DESCRIPTOR *pMemDesc;
684 NvU64 memdescFlags = MEMDESC_FLAGS_NONE;
685 NvU32 gpuCachedFlags;
686
687 NV_ASSERT((pMemDataReleaseCallback == osDestroyOsDescriptorFromDmaBuf) ||
688 (pMemDataReleaseCallback == osDestroyOsDescriptorFromSgt));
689
690 if (FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _WRITE_COMBINE, flags))
691 {
692 cacheType = NV_MEMORY_WRITECOMBINED;
693 }
694 else if (!FLD_TEST_DRF(OS02, _FLAGS, _COHERENCY, _UNCACHED, flags))
695 {
696 cacheType = NV_MEMORY_CACHED;
697 }
698
699 if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags))
700 {
701 memdescFlags |= MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO;
702 }
703
704 rmStatus = memdescCreate(ppMemDesc, pGpu, size, 0,
705 NV_MEMORY_NONCONTIGUOUS, ADDR_SYSMEM,
706 cacheType, memdescFlags);
707 if (rmStatus != NV_OK)
708 {
709 return rmStatus;
710 }
711
712 if (FLD_TEST_DRF(OS02, _FLAGS, _GPU_CACHEABLE, _YES, flags))
713 {
714 gpuCachedFlags = NV_MEMORY_CACHED;
715 }
716 else
717 {
718 gpuCachedFlags = NV_MEMORY_UNCACHED;
719 }
720
721 pMemDesc = *ppMemDesc;
722
723 memdescSetGpuCacheAttrib(pMemDesc, gpuCachedFlags);
724 memdescSetAddress(pMemDesc, NvP64_NULL);
725 memdescSetMemData(pMemDesc, NULL, NULL);
726 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_KERNEL_MODE, NV_FALSE);
727 memdescSetFlag(pMemDesc, MEMDESC_FLAGS_EXT_PAGE_ARRAY_MEM, NV_TRUE);
728
729 *ppPrivate = NULL;
730 rmStatus = nv_register_sgt(NV_GET_NV_STATE(pGpu),
731 memdescGetPteArray(pMemDesc, AT_CPU),
732 NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
733 memdescGetCpuCacheAttrib(pMemDesc),
734 ppPrivate,
735 pImportSgt,
736 pImportPriv);
737 if (rmStatus != NV_OK)
738 {
739 memdescDestroy(pMemDesc);
740 return rmStatus;
741 }
742
743 //
744 // If the OS layer doesn't think in RM page size, we need to inflate the
745 // PTE array into RM pages.
746 //
747 if ((NV_RM_PAGE_SIZE < os_page_size) &&
748 !memdescGetContiguity(pMemDesc, AT_CPU))
749 {
750 RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
751 pMemDesc->PageCount);
752 }
753
754 memdescSetMemData(*ppMemDesc, *ppPrivate, NULL);
755
756 //
757 // memdescMapIommu() requires the OS-private data to be set on the memory
758 // descriptor, but we don't want to wire up the teardown callback just yet:
759 // that callback does teardown that will already be done as part of failure
760 // handling further up the stack if memdescMapIommu() fails. So we only
761 // setup the priv-data cleanup callback once we're sure this call will
762 // succeed.
763 //
764 rmStatus = memdescMapIommu(*ppMemDesc, pGpu->busInfo.iovaspaceId);
765 if (rmStatus != NV_OK)
766 {
767 if ((NV_RM_PAGE_SIZE < os_page_size) &&
768 !memdescGetContiguity(pMemDesc, AT_CPU))
769 {
770 RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
771 pMemDesc->PageCount);
772 }
773 if (*ppPrivate != NULL)
774 {
775 nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
776 (void **) &pImportPriv, *ppPrivate);
777 }
778 memdescDestroy(pMemDesc);
779 return rmStatus;
780 }
781
782 // All is well - wire up the cleanup callback now
783 memdescSetMemData(*ppMemDesc, *ppPrivate, pMemDataReleaseCallback);
784
785 return rmStatus;
786 }
787
788 static NV_STATUS
_createMemdescFromDmaBuf(OBJGPU * pGpu,NvU32 flags,nv_dma_buf_t * pImportPriv,struct sg_table * pImportSgt,NvU32 size,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)789 _createMemdescFromDmaBuf
790 (
791 OBJGPU *pGpu,
792 NvU32 flags,
793 nv_dma_buf_t *pImportPriv,
794 struct sg_table *pImportSgt,
795 NvU32 size,
796 MEMORY_DESCRIPTOR **ppMemDesc,
797 void **ppPrivate
798 )
799 {
800 NV_STATUS rmStatus =
801 _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPriv, pImportSgt,
802 size, ppMemDesc, ppPrivate,
803 osDestroyOsDescriptorFromDmaBuf);
804 if (rmStatus != NV_OK)
805 {
806 nv_dma_release_dma_buf(pImportPriv);
807 }
808
809 return rmStatus;
810 }
811
812 static NV_STATUS
_createMemdescFromSgt(OBJGPU * pGpu,NvU32 flags,struct drm_gem_object * pImportPrivGem,struct sg_table * pImportSgt,NvU32 size,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)813 _createMemdescFromSgt
814 (
815 OBJGPU *pGpu,
816 NvU32 flags,
817 struct drm_gem_object *pImportPrivGem,
818 struct sg_table *pImportSgt,
819 NvU32 size,
820 MEMORY_DESCRIPTOR **ppMemDesc,
821 void **ppPrivate
822 )
823 {
824 NV_STATUS rmStatus =
825 _createMemdescFromDmaBufSgtHelper(pGpu, flags, pImportPrivGem,
826 pImportSgt, size, ppMemDesc,
827 ppPrivate,
828 osDestroyOsDescriptorFromSgt);
829 if (rmStatus != NV_OK)
830 {
831 nv_dma_release_sgt(pImportSgt, pImportPrivGem);
832 }
833
834 return rmStatus;
835 }
836
GetDmaDeviceForImport(nv_state_t * nv,NvU32 flags)837 static nv_dma_device_t *GetDmaDeviceForImport
838 (
839 nv_state_t *nv,
840 NvU32 flags
841 )
842 {
843 if (FLD_TEST_DRF(OS02, _FLAGS, _ALLOC_NISO_DISPLAY, _YES, flags) &&
844 (nv->niso_dma_dev != NULL))
845 {
846 return nv->niso_dma_dev;
847 }
848 else
849 {
850 return nv->dma_dev;
851 }
852 }
853
854 static NV_STATUS
osCreateOsDescriptorFromFileHandle(OBJGPU * pGpu,NvP64 pDescriptor,NvHandle hClient,NvU32 flags,NvU64 * pLimit,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)855 osCreateOsDescriptorFromFileHandle
856 (
857 OBJGPU *pGpu,
858 NvP64 pDescriptor,
859 NvHandle hClient,
860 NvU32 flags,
861 NvU64 *pLimit,
862 MEMORY_DESCRIPTOR **ppMemDesc,
863 void **ppPrivate
864 )
865 {
866 NV_STATUS rmStatus = NV_OK;
867 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
868 nv_dma_device_t *dma_dev = NULL;
869 NvU32 size = 0;
870 nv_dma_buf_t *pImportPriv = NULL;
871 struct sg_table *pImportSgt = NULL;
872 NvS32 fd;
873
874 fd = (NvS32)((NvU64)pDescriptor);
875 if ((NvU64)fd != (NvU64)pDescriptor)
876 {
877 NV_PRINTF(LEVEL_ERROR,
878 "%s(): fd must fit within a signed 32-bit integer!\n",
879 __FUNCTION__);
880 return NV_ERR_INVALID_ARGUMENT;
881 }
882
883 dma_dev = GetDmaDeviceForImport(nv, flags);
884 rmStatus = nv_dma_import_from_fd(dma_dev, fd, &size,
885 &pImportSgt, &pImportPriv);
886 if (rmStatus != NV_OK)
887 {
888 NV_PRINTF(LEVEL_ERROR,
889 "%s(): Error (%d) while trying to import fd!\n",
890 __FUNCTION__, rmStatus);
891 return rmStatus;
892 }
893
894 return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
895 pImportSgt,
896 size, ppMemDesc, ppPrivate);
897 }
898
899 static NV_STATUS
osCreateOsDescriptorFromSgtPtr(OBJGPU * pGpu,NvP64 pDescriptor,NvHandle hClient,NvU32 flags,NvU64 * pLimit,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)900 osCreateOsDescriptorFromSgtPtr
901 (
902 OBJGPU *pGpu,
903 NvP64 pDescriptor,
904 NvHandle hClient,
905 NvU32 flags,
906 NvU64 *pLimit,
907 MEMORY_DESCRIPTOR **ppMemDesc,
908 void **ppPrivate
909 )
910 {
911 NV_STATUS rmStatus = NV_OK;
912 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
913 NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS *params =
914 (NVOS32_DESCRIPTOR_TYPE_OS_SGT_PTR_PARAMETERS*)((NvUPtr) pDescriptor);
915
916 struct sg_table *sgt = params->sgt;
917 struct drm_gem_object *gem = params->gem;
918
919 rmStatus = nv_dma_import_sgt(nv->dma_dev, sgt, gem);
920 if (rmStatus != NV_OK)
921 {
922 NV_PRINTF(LEVEL_ERROR,
923 "%s(): Error (%d) while trying to import sgt!\n",
924 __FUNCTION__, rmStatus);
925 return rmStatus;
926 }
927
928 return _createMemdescFromSgt(pGpu, flags, gem, sgt,
929 (*pLimit + 1), ppMemDesc, ppPrivate);
930 }
931
932 static NV_STATUS
osCreateOsDescriptorFromDmaBufPtr(OBJGPU * pGpu,NvP64 pDescriptor,NvHandle hClient,NvU32 flags,NvU64 * pLimit,MEMORY_DESCRIPTOR ** ppMemDesc,void ** ppPrivate)933 osCreateOsDescriptorFromDmaBufPtr
934 (
935 OBJGPU *pGpu,
936 NvP64 pDescriptor,
937 NvHandle hClient,
938 NvU32 flags,
939 NvU64 *pLimit,
940 MEMORY_DESCRIPTOR **ppMemDesc,
941 void **ppPrivate
942 )
943 {
944 NV_STATUS rmStatus = NV_OK;
945 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
946 nv_dma_device_t *dma_dev = NULL;
947 NvU32 size = 0;
948 nv_dma_buf_t *pImportPriv = NULL;
949 struct sg_table *pImportSgt = NULL;
950 void *dmaBuf = (void*)((NvUPtr)pDescriptor);
951
952 dma_dev = GetDmaDeviceForImport(nv, flags);
953 rmStatus = nv_dma_import_dma_buf(dma_dev, dmaBuf, &size,
954 &pImportSgt, &pImportPriv);
955 if (rmStatus != NV_OK)
956 {
957 NV_PRINTF_COND(rmStatus == NV_ERR_NOT_SUPPORTED, LEVEL_INFO, LEVEL_ERROR,
958 "Error (%d) while trying to import dma_buf!\n", rmStatus);
959 return rmStatus;
960 }
961
962 return _createMemdescFromDmaBuf(pGpu, flags, pImportPriv,
963 pImportSgt,
964 size, ppMemDesc, ppPrivate);
965 }
966
967 static void
osDestroyOsDescriptorFromPhysAddr(PMEMORY_DESCRIPTOR pMemDesc)968 osDestroyOsDescriptorFromPhysAddr
969 (
970 PMEMORY_DESCRIPTOR pMemDesc
971 )
972 {
973 OBJGPU *pGpu = pMemDesc->pGpu;
974 void *pPrivate;
975
976 pPrivate = memdescGetMemData(pMemDesc);
977 NV_ASSERT(pPrivate != NULL);
978
979 nv_unregister_phys_pages(NV_GET_NV_STATE(pGpu), pPrivate);
980 }
981
982 static void
osDestroyOsDescriptorFromIoMemory(PMEMORY_DESCRIPTOR pMemDesc)983 osDestroyOsDescriptorFromIoMemory
984 (
985 PMEMORY_DESCRIPTOR pMemDesc
986 )
987 {
988 OBJGPU *pGpu = pMemDesc->pGpu;
989 void *pPrivate = memdescGetMemData(pMemDesc);
990
991 if (pPrivate == NULL)
992 {
993 return;
994 }
995
996 nv_unregister_peer_io_mem(NV_GET_NV_STATE(pGpu), pPrivate);
997 }
998
999 static void
osDestroyOsDescriptorPageArray(MEMORY_DESCRIPTOR * pMemDesc)1000 osDestroyOsDescriptorPageArray
1001 (
1002 MEMORY_DESCRIPTOR *pMemDesc
1003 )
1004 {
1005 OBJGPU *pGpu = pMemDesc->pGpu;
1006 NvU64 osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);
1007 NV_STATUS status;
1008 void *pPrivate;
1009
1010 pPrivate = memdescGetMemData(pMemDesc);
1011
1012 NV_ASSERT(pPrivate != NULL);
1013
1014 //
1015 // TODO: Bug 1811006: Notably skip any IOMMU mapping management as the
1016 // pMemDesc->pGpu might have been torn down already and the pGpu passed in
1017 // doesn't necessarily have IOMMU mappings. For now just allow memdescDestroy()
1018 // to clean up whatever is there (this may not work correctly either if any
1019 // of the IOMMU mappings have outlasted their VASPACEs). This should
1020 // be cleaned up once the fix for bug 1811006 is known.
1021 //
1022
1023 if ((NV_RM_PAGE_SIZE < os_page_size) &&
1024 !memdescGetContiguity(pMemDesc, AT_CPU))
1025 {
1026 RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
1027 pMemDesc->PageCount);
1028 }
1029
1030 nv_unregister_user_pages(NV_GET_NV_STATE(pGpu), osPageCount,
1031 NULL /* import_priv */, &pPrivate);
1032
1033 if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_FOREIGN_PAGE) == NV_FALSE)
1034 {
1035 status = os_unlock_user_pages(osPageCount, pPrivate);
1036 NV_ASSERT(status == NV_OK);
1037 }
1038 else
1039 {
1040 os_free_mem(pPrivate);
1041 }
1042 }
1043
1044 static void
osDestroyOsDescriptorFromDmaBuf(PMEMORY_DESCRIPTOR pMemDesc)1045 osDestroyOsDescriptorFromDmaBuf
1046 (
1047 PMEMORY_DESCRIPTOR pMemDesc
1048 )
1049 {
1050 OBJGPU *pGpu = pMemDesc->pGpu;
1051 void *pPrivate = memdescGetMemData(pMemDesc);
1052
1053 struct sg_table *pImportSgt;
1054 void *pImportPriv;
1055
1056 /*
1057 * Unmap IOMMU now or we will get a kernel crash when it is unmapped after
1058 * pImportSgt is freed.
1059 */
1060 memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
1061
1062 if ((NV_RM_PAGE_SIZE < os_page_size) &&
1063 !memdescGetContiguity(pMemDesc, AT_CPU))
1064 {
1065 RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
1066 pMemDesc->PageCount);
1067 }
1068
1069 nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
1070 &pImportPriv, pPrivate);
1071
1072 /*
1073 * pImportSgt doesn't need to be passed to nv_dma_release_dma_buf() because
1074 * the DMA-BUF associated with pImportPriv already has a reference to the
1075 * SGT.
1076 */
1077
1078 nv_dma_release_dma_buf(pImportPriv);
1079 }
1080
1081 static void
osDestroyOsDescriptorFromSgt(PMEMORY_DESCRIPTOR pMemDesc)1082 osDestroyOsDescriptorFromSgt
1083 (
1084 PMEMORY_DESCRIPTOR pMemDesc
1085 )
1086 {
1087 OBJGPU *pGpu = pMemDesc->pGpu;
1088 void *pPrivate = memdescGetMemData(pMemDesc);
1089
1090 struct sg_table *pImportSgt;
1091 struct drm_gem_object *pImportPrivGem;
1092
1093 NV_ASSERT(pPrivate != NULL);
1094
1095 /*
1096 * Unmap IOMMU now or we will get a kernel crash when it is unmapped after
1097 * pImportSgt is freed.
1098 */
1099 memdescUnmapIommu(pMemDesc, pGpu->busInfo.iovaspaceId);
1100
1101 if ((NV_RM_PAGE_SIZE < os_page_size) &&
1102 !memdescGetContiguity(pMemDesc, AT_CPU))
1103 {
1104 RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
1105 pMemDesc->PageCount);
1106 }
1107
1108 nv_unregister_sgt(NV_GET_NV_STATE(pGpu), &pImportSgt,
1109 (void **) &pImportPrivGem, pPrivate);
1110
1111 nv_dma_release_sgt(pImportSgt, pImportPrivGem);
1112 }
1113