1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1999-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include <nv.h> // NV device driver interface
25 #include <nv-priv.h>
26 #include <nv-caps.h>
27 #include <os/os.h>
28 #include <nvos.h>
29 #include <osapi.h>
30 #include <ctrl/ctrl0000/ctrl0000gpu.h>
31 #include <ctrl/ctrl0000/ctrl0000unix.h>
32
33 #include <nverror.h>
34 #include <gpu/device/device.h>
35
36 #include "gpu/gpu.h"
37 #include <gpu_mgr/gpu_mgr.h>
38 #include <osfuncs.h>
39 #include <platform/chipset/chipset.h>
40
41 #include "nverror.h"
42 #include "kernel/gpu/bif/kernel_bif.h"
43
44 #include "gpu/mig_mgr/kernel_mig_manager.h"
45
46 #include "gpu/mem_sys/kern_mem_sys.h"
47
48 #include "mem_mgr/io_vaspace.h"
49 #include <diagnostics/journal.h>
50 #include "gpu/mem_mgr/mem_desc.h"
51 #include "gpu/mem_mgr/mem_mgr.h"
52 #include "core/thread_state.h"
53 #include <nvacpitypes.h>
54 #include <platform/acpi_common.h>
55 #include <core/locks.h>
56 #include <ctrl/ctrl2080/ctrl2080gpu.h>
57 #include "virtualization/hypervisor/hypervisor.h"
58 #include "rmobjexportimport.h"
59 #include <nvi2c.h>
60 #include "rmapi/rs_utils.h"
61 #include "rmapi/client_resource.h"
62 #include "os/dce_rm_client_ipc.h"
63 #include "mem_mgr/mem.h"
64 #include "gpu/mem_mgr/virt_mem_allocator_common.h"
65
66 #include <acpidsmguids.h>
67 #include <pex.h>
68 #include "gps.h"
69 #include "jt.h"
70
71
72
73
74 extern const char *ppOsBugCheckBugcodeStr[];
75
76
77 ct_assert(NV_RM_PAGE_SIZE == RM_PAGE_SIZE);
78 ct_assert(NV_RM_PAGE_MASK == RM_PAGE_MASK);
79 ct_assert(NV_RM_PAGE_SHIFT == RM_PAGE_SHIFT);
80
81 typedef struct
82 {
83 NvU32 euid;
84 NvU32 pid;
85 } TOKEN_USER, *PTOKEN_USER;
86
87 struct OS_RM_CAPS
88 {
89 NvU32 count;
90
91 // This should be the last element
92 nv_cap_t **caps;
93 };
94
osIsRaisedIRQL(void)95 NvBool osIsRaisedIRQL(void)
96 {
97 return (!os_semaphore_may_sleep());
98 }
99
osIsISR(void)100 NvBool osIsISR(void)
101 {
102 return os_is_isr();
103 }
104
osGetDriverBlock(OS_GPU_INFO * pOsGpuInfo,OS_DRIVER_BLOCK * pBlock)105 NV_STATUS osGetDriverBlock
106 (
107 OS_GPU_INFO *pOsGpuInfo,
108 OS_DRIVER_BLOCK *pBlock
109 )
110 {
111 return NV_ERR_NOT_SUPPORTED;
112 }
113
osGetCurrentTick(NvU64 * pTimeInNs)114 NV_STATUS osGetCurrentTick(NvU64 *pTimeInNs)
115 {
116 *pTimeInNs = os_get_current_tick();
117 return NV_OK;
118 }
119
osGetTickResolution(void)120 NvU64 osGetTickResolution(void)
121 {
122 return os_get_tick_resolution();
123 }
124
osGetPerformanceCounter(NvU64 * pTimeInNs)125 NV_STATUS osGetPerformanceCounter(NvU64 *pTimeInNs)
126 {
127 *pTimeInNs = os_get_current_tick_hr();
128 return NV_OK;
129 }
130
osGetCurrentTime(NvU32 * pSeconds,NvU32 * pMicroSeconds)131 NV_STATUS osGetCurrentTime(
132 NvU32 *pSeconds,
133 NvU32 *pMicroSeconds
134 )
135 {
136 return os_get_current_time(pSeconds, pMicroSeconds);
137 }
138
139 /*!
140 * @brief Get timestamp for logging.
141 *
142 * Everything that logs a time stamp should use this routine for consistency.
143 *
144 * The returned value is OS dependent. We want the time stamp to use
145 * KeQueryPerformanceCounter on Windows so it matches the DirectX timestamps.
146 * Linux uses microseconds since 1970 (osGetCurrentTime), since matching DirectX
147 * is not a priority.
148 *
149 * osGetTimestampFreq returns the frequency required to decode the time stamps.
150 *
151 * @returns system dependent timestamp.
152 */
osGetTimestamp(void)153 NvU64 osGetTimestamp(void)
154 {
155 NvU32 sec = 0;
156 NvU32 usec = 0;
157 osGetCurrentTime(&sec, &usec);
158 return (NvU64)sec * 1000000 + usec;
159 }
160
161 /*!
162 * @brief Get timestamp frequency.
163 *
164 * Timestamps are OS dependent. This call returns the frequency
165 * required to decode them.
166 *
167 * @returns Timestamp frequency. For example, 1000000 for MHz.
168 */
osGetTimestampFreq(void)169 NvU64 osGetTimestampFreq(void)
170 {
171 return 1000000;
172 }
173
osDelay(NvU32 milliseconds)174 NV_STATUS osDelay(NvU32 milliseconds)
175 {
176 return os_delay(milliseconds);
177 }
178
osDelayUs(NvU32 microseconds)179 NV_STATUS osDelayUs(NvU32 microseconds)
180 {
181 return os_delay_us(microseconds);
182 }
183
osDelayNs(NvU32 nanoseconds)184 NV_STATUS osDelayNs(NvU32 nanoseconds)
185 {
186 NvU32 microseconds = NV_MAX(1, (nanoseconds / 1000));
187 return os_delay_us(microseconds);
188 }
189
osGetCpuFrequency(void)190 NvU32 osGetCpuFrequency(void)
191 {
192 /* convert os_get_cpu_frequency()'s return value from Hz to MHz */
193 return ((NvU32)(os_get_cpu_frequency() / 1000000ULL));
194 }
195
osPciInitHandle(NvU32 Domain,NvU8 Bus,NvU8 Slot,NvU8 Function,NvU16 * pVendor,NvU16 * pDevice)196 void* osPciInitHandle(
197 NvU32 Domain,
198 NvU8 Bus,
199 NvU8 Slot,
200 NvU8 Function,
201 NvU16 *pVendor,
202 NvU16 *pDevice
203 )
204 {
205 //
206 // Check if the BDF is for a GPU that's already been attached, for which
207 // we should already have a handle cached. This won't catch devices that
208 // have been probed but not yet attached, but that shouldn't be a common
209 // occurrence.
210 //
211 // More importantly, having this check here means we don't need to check
212 // a global list of devices in the kernel interface layer, which could
213 // have the implication of taking another lock, causing hairy lock
214 // ordering issues.
215 //
216 if (Function == 0)
217 {
218 OBJGPU *pGpu = gpumgrGetGpuFromBusInfo(Domain, Bus, Slot);
219 if (pGpu != NULL)
220 {
221 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
222 if (pVendor) *pVendor = nv->pci_info.vendor_id;
223 if (pDevice) *pDevice = nv->pci_info.device_id;
224 return nv->handle;
225 }
226 }
227
228 return os_pci_init_handle(Domain, Bus, Slot, Function, pVendor, pDevice);
229 }
230
osPciReadByte(void * pHandle,NvU32 Offset)231 NvU8 osPciReadByte(
232 void *pHandle,
233 NvU32 Offset
234 )
235 {
236 NvU8 val;
237 os_pci_read_byte(pHandle, Offset, &val);
238 return val;
239 }
240
osPciReadWord(void * pHandle,NvU32 Offset)241 NvU16 osPciReadWord(
242 void *pHandle,
243 NvU32 Offset
244 )
245 {
246 NvU16 val;
247 os_pci_read_word(pHandle, Offset, &val);
248 return val;
249 }
250
osPciReadDword(void * pHandle,NvU32 Offset)251 NvU32 osPciReadDword(
252 void *pHandle,
253 NvU32 Offset
254 )
255 {
256 NvU32 val;
257 os_pci_read_dword(pHandle, Offset, &val);
258 return val;
259 }
260
osPciWriteByte(void * pHandle,NvU32 Offset,NvU8 Value)261 void osPciWriteByte(
262 void *pHandle,
263 NvU32 Offset,
264 NvU8 Value
265 )
266 {
267 os_pci_write_byte(pHandle, Offset, Value);
268 }
269
osPciWriteWord(void * pHandle,NvU32 Offset,NvU16 Value)270 void osPciWriteWord(
271 void *pHandle,
272 NvU32 Offset,
273 NvU16 Value
274 )
275 {
276 os_pci_write_word(pHandle, Offset, Value);
277 }
278
osPciWriteDword(void * pHandle,NvU32 Offset,NvU32 Value)279 void osPciWriteDword(
280 void *pHandle,
281 NvU32 Offset,
282 NvU32 Value
283 )
284 {
285 os_pci_write_dword(pHandle, Offset, Value);
286 }
287
osMapKernelSpace(RmPhysAddr Start,NvU64 Size,NvU32 Mode,NvU32 Protect)288 void* osMapKernelSpace(
289 RmPhysAddr Start,
290 NvU64 Size,
291 NvU32 Mode,
292 NvU32 Protect
293 )
294 {
295 NvU64 offset;
296 NvU8 *ptr;
297
298 if (0 == Size)
299 {
300 NV_ASSERT(Size != 0);
301 return NULL;
302 }
303
304 offset = (Start & ~os_page_mask);
305 Start &= os_page_mask;
306
307 if (!portSafeAddU64(Size, offset, &Size) ||
308 !portSafeAddU64(Size, ~os_page_mask, &Size))
309 {
310 return NULL;
311 }
312 Size &= os_page_mask;
313
314 ptr = os_map_kernel_space(Start, Size, Mode);
315 if (ptr != NULL)
316 return (ptr + offset);
317
318 return NULL;
319 }
320
osUnmapKernelSpace(void * pAddress,NvU64 Size)321 void osUnmapKernelSpace(
322 void *pAddress,
323 NvU64 Size
324 )
325 {
326 NvU64 offset;
327 NvUPtr ptr = (NvUPtr)pAddress;
328
329 if (0 == Size)
330 {
331 NV_ASSERT(Size != 0);
332 return;
333 }
334
335 offset = (ptr & ~os_page_mask);
336 ptr &= os_page_mask;
337 Size = ((Size + offset + ~os_page_mask) & os_page_mask);
338 os_unmap_kernel_space((void *)ptr, Size);
339 }
340
setNumaPrivData(KernelMemorySystem * pKernelMemorySystem,nv_state_t * nv,MEMORY_DESCRIPTOR * pMemDesc)341 static NV_STATUS setNumaPrivData
342 (
343 KernelMemorySystem *pKernelMemorySystem,
344 nv_state_t *nv,
345 MEMORY_DESCRIPTOR *pMemDesc
346 )
347 {
348 NV_STATUS rmStatus = NV_OK;
349 void *pAllocPrivate = NULL;
350 NvU64 *addrArray = NULL;
351 NvU64 numPages = pMemDesc->PageCount;
352 NvU64 i;
353
354 addrArray = portMemAllocNonPaged(numPages * sizeof(NvU64));
355 if (addrArray == NULL)
356 {
357 return NV_ERR_NO_MEMORY;
358 }
359
360 portMemCopy((void*)addrArray,
361 (numPages * sizeof(NvU64)),
362 (void*)memdescGetPteArray(pMemDesc, AT_CPU),
363 (numPages * sizeof(NvU64)));
364
365 if (NV_RM_PAGE_SIZE < os_page_size)
366 {
367 RmDeflateRmToOsPageArray(addrArray, numPages);
368 numPages = NV_RM_PAGES_TO_OS_PAGES(numPages);
369 }
370
371 for (i = 0; i < numPages; i++)
372 {
373 // Update GPA to system physical address
374 addrArray[i] += pKernelMemorySystem->coherentCpuFbBase;
375 }
376
377 rmStatus = nv_register_phys_pages(nv, addrArray, numPages, NV_MEMORY_CACHED, &pAllocPrivate);
378 if (rmStatus != NV_OK)
379 {
380 goto errors;
381 }
382
383 memdescSetMemData(pMemDesc, pAllocPrivate, NULL);
384
385 errors:
386 portMemFree(addrArray);
387
388 return rmStatus;
389 }
390
osGetNumMemoryPages(MEMORY_DESCRIPTOR * pMemDesc,NvU32 * pNumPages)391 NV_STATUS osGetNumMemoryPages
392 (
393 MEMORY_DESCRIPTOR *pMemDesc,
394 NvU32 *pNumPages
395 )
396 {
397 void *pAllocPrivate = NULL;
398
399 pAllocPrivate = memdescGetMemData(pMemDesc);
400 if (pAllocPrivate == NULL)
401 {
402 NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
403 return NV_ERR_INVALID_STATE;
404 }
405
406 return nv_get_num_phys_pages(pAllocPrivate, pNumPages);
407 }
408
osGetMemoryPages(MEMORY_DESCRIPTOR * pMemDesc,void * pPages,NvU32 * pNumPages)409 NV_STATUS osGetMemoryPages
410 (
411 MEMORY_DESCRIPTOR *pMemDesc,
412 void *pPages,
413 NvU32 *pNumPages
414 )
415 {
416 void *pAllocPrivate = NULL;
417
418 pAllocPrivate = memdescGetMemData(pMemDesc);
419 if (pAllocPrivate == NULL)
420 {
421 NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
422 return NV_ERR_INVALID_STATE;
423 }
424
425 return nv_get_phys_pages(pAllocPrivate, pPages, pNumPages);
426 }
427
osMapSystemMemory(MEMORY_DESCRIPTOR * pMemDesc,NvU64 Offset,NvU64 Length,NvBool Kernel,NvU32 Protect,NvP64 * ppAddress,NvP64 * ppPrivate)428 NV_STATUS osMapSystemMemory
429 (
430 MEMORY_DESCRIPTOR *pMemDesc,
431 NvU64 Offset,
432 NvU64 Length,
433 NvBool Kernel,
434 NvU32 Protect,
435 NvP64 *ppAddress,
436 NvP64 *ppPrivate
437 )
438 {
439 OBJGPU *pGpu = pMemDesc->pGpu;
440
441 RmPhysAddr userAddress;
442 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
443 NV_STATUS rmStatus = NV_OK;
444 void *pAllocPrivate = NULL;
445 void *pAddress;
446 void *pPrivate = NULL;
447 NvU64 pageIndex;
448 NvU32 pageOffset;
449
450 if (pGpu != NULL &&
451 pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
452 memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM)
453 {
454 KernelMemorySystem *pKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pGpu);
455
456 rmStatus = setNumaPrivData(pKernelMemorySystem, nv, pMemDesc);
457 if (rmStatus != NV_OK)
458 return rmStatus;
459 }
460
461 *ppAddress = NvP64_NULL;
462 *ppPrivate = NvP64_NULL;
463
464 if ((Offset + Length) < Length)
465 return NV_ERR_INVALID_ARGUMENT;
466 if ((Offset + Length) > pMemDesc->Size)
467 return NV_ERR_INVALID_ARGUMENT;
468
469 pageIndex = (Offset >> os_page_shift);
470 pageOffset = (Offset & ~os_page_mask);
471
472 pAllocPrivate = memdescGetMemData(pMemDesc);
473 if (!pAllocPrivate)
474 {
475 NV_PRINTF(LEVEL_ERROR, "pAllocPrivate is NULL!\n");
476 return NV_ERR_INVALID_STATE;
477 }
478
479 if (Kernel)
480 {
481 pAddress = nv_alloc_kernel_mapping(nv, pAllocPrivate,
482 pageIndex, pageOffset, Length, &pPrivate);
483 if (pAddress == NULL)
484 {
485 NV_PRINTF(LEVEL_ERROR,
486 "failed to create system memory kernel mapping!\n");
487 rmStatus = NV_ERR_GENERIC;
488 }
489 else
490 {
491 *ppAddress = NV_PTR_TO_NvP64(pAddress);
492 *ppPrivate = NV_PTR_TO_NvP64(pPrivate);
493 }
494 }
495 else
496 {
497 rmStatus = nv_alloc_user_mapping(nv, pAllocPrivate,
498 pageIndex, pageOffset, Length, Protect, &userAddress,
499 &pPrivate);
500 if (rmStatus != NV_OK)
501 {
502 NV_PRINTF(LEVEL_ERROR,
503 "failed to create system memory user mapping!\n");
504 }
505 else
506 {
507 *ppAddress = (NvP64)(userAddress);
508 *ppPrivate = NV_PTR_TO_NvP64(pPrivate);
509 }
510 }
511
512 return rmStatus;
513 }
514
osUnmapSystemMemory(MEMORY_DESCRIPTOR * pMemDesc,NvBool Kernel,NvU32 ProcessId,NvP64 pAddress,NvP64 pPrivate)515 void osUnmapSystemMemory
516 (
517 MEMORY_DESCRIPTOR *pMemDesc,
518 NvBool Kernel,
519 NvU32 ProcessId,
520 NvP64 pAddress,
521 NvP64 pPrivate
522 )
523 {
524 NV_STATUS status;
525 void *pAllocPrivate = memdescGetMemData(pMemDesc);
526 OBJGPU *pGpu = pMemDesc->pGpu;
527 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
528
529 if (Kernel)
530 {
531 status = nv_free_kernel_mapping(nv, pAllocPrivate, NvP64_VALUE(pAddress),
532 NvP64_VALUE(pPrivate));
533 }
534 else
535 {
536 status = nv_free_user_mapping(nv, pAllocPrivate, (NvU64)pAddress,
537 NvP64_VALUE(pPrivate));
538 }
539
540 if (pGpu != NULL &&
541 pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) &&
542 memdescGetAddressSpace(pMemDesc) == ADDR_FBMEM &&
543 pAllocPrivate != NULL)
544 {
545 nv_unregister_phys_pages(nv, pAllocPrivate);
546 memdescSetMemData(pMemDesc, NULL, NULL);
547 }
548
549 NV_ASSERT(status == NV_OK);
550 }
551
osIoWriteByte(NvU32 Address,NvU8 Value)552 void osIoWriteByte(
553 NvU32 Address,
554 NvU8 Value
555 )
556 {
557 os_io_write_byte(Address, Value);
558 }
559
osIoReadWord(NvU32 Address)560 NvU16 osIoReadWord(
561 NvU32 Address
562 )
563 {
564 return os_io_read_word(Address);
565 }
566
osIoWriteWord(NvU32 Address,NvU16 Value)567 void osIoWriteWord(
568 NvU32 Address,
569 NvU16 Value
570 )
571 {
572 os_io_write_word(Address, Value);
573 }
574
osIoReadByte(NvU32 Address)575 NvU8 osIoReadByte(
576 NvU32 Address
577 )
578 {
579 return os_io_read_byte(Address);
580 }
581
osIsAdministrator(void)582 NvBool osIsAdministrator(void)
583 {
584 return os_is_administrator();
585 }
586
osAllowPriorityOverride(void)587 NvBool osAllowPriorityOverride(void)
588 {
589 return os_allow_priority_override();
590 }
591
osGetCurrentProcess(void)592 NvU32 osGetCurrentProcess(void)
593 {
594 return os_get_current_process();
595 }
596
osGetCurrentProcessName(char * ProcName,NvU32 Length)597 void osGetCurrentProcessName(char *ProcName, NvU32 Length)
598 {
599 return os_get_current_process_name(ProcName, Length);
600 }
601
osGetCurrentThread(OS_THREAD_HANDLE * pThreadId)602 NV_STATUS osGetCurrentThread(OS_THREAD_HANDLE *pThreadId)
603 {
604 NV_STATUS rmStatus;
605 NvU64 threadId = 0;
606
607 if (pThreadId == NULL)
608 {
609 return NV_ERR_INVALID_ARGUMENT;
610 }
611
612 rmStatus = os_get_current_thread(&threadId);
613 if (rmStatus == NV_OK)
614 {
615 *pThreadId = threadId;
616 }
617 else
618 {
619 *pThreadId = 0;
620 }
621
622 return rmStatus;
623 }
624
osGetPidInfo(void)625 void* osGetPidInfo(void)
626 {
627 return os_get_pid_info();
628 }
629
osPutPidInfo(void * pOsPidInfo)630 void osPutPidInfo(void *pOsPidInfo)
631 {
632 os_put_pid_info(pOsPidInfo);
633 }
634
osFindNsPid(void * pOsPidInfo,NvU32 * pNsPid)635 NV_STATUS osFindNsPid(void *pOsPidInfo, NvU32 *pNsPid)
636 {
637 return os_find_ns_pid(pOsPidInfo, pNsPid);
638 }
639
osAttachToProcess(void ** ppProcessInfo,NvU32 ProcessId)640 NV_STATUS osAttachToProcess(void** ppProcessInfo, NvU32 ProcessId)
641 {
642 //
643 // This function is used by RmUnmapMemory() to attach to the
644 // process for which a given device memory mapping was
645 // created, in order to be able to unmap it. On Linux/UNIX
646 // platforms, we can't "attach" to a random process, but
647 // since we don't create/destroy user mappings in the RM, we
648 // don't need to, either.
649 //
650 // Report success to the caller to keep RmUnmapMemory() from
651 // failing, and memory from being leaked as a result.
652 //
653 *ppProcessInfo = NULL;
654 return NV_OK;
655 }
656
osDetachFromProcess(void * pProcessInfo)657 void osDetachFromProcess(void* pProcessInfo)
658 {
659 // stub
660 return;
661 }
662
osAcquireRmSema(void * pSema)663 NV_STATUS osAcquireRmSema(void *pSema)
664 {
665 return NV_OK;
666 }
667
osCondAcquireRmSema(void * pSema)668 NV_STATUS osCondAcquireRmSema(void *pSema)
669 {
670 return NV_OK;
671 }
672
osReleaseRmSema(void * pSema,OBJGPU * pDpcGpu)673 NvU32 osReleaseRmSema(void *pSema, OBJGPU *pDpcGpu)
674 {
675 return NV_SEMA_RELEASE_SUCCEED;
676 }
677
osSpinLoop(void)678 void osSpinLoop(void)
679 {
680 // Enable this code to get debug prints from Libos.
681 }
682
osGetMaxUserVa(void)683 NvU64 osGetMaxUserVa(void)
684 {
685 return os_get_max_user_va();
686 }
687
osSchedule(void)688 NV_STATUS osSchedule(void)
689 {
690 return os_schedule();
691 }
692
osQueueWorkItemWithFlags(OBJGPU * pGpu,OSWorkItemFunction pFunction,void * pParams,NvU32 flags)693 NV_STATUS osQueueWorkItemWithFlags(
694 OBJGPU *pGpu,
695 OSWorkItemFunction pFunction,
696 void *pParams,
697 NvU32 flags
698 )
699 {
700 nv_work_item_t *pWi;
701 nv_state_t *nv;
702 NV_STATUS status;
703
704 pWi = portMemAllocNonPaged(sizeof(nv_work_item_t));
705
706 if (NULL == pWi)
707 {
708 return NV_ERR_NO_MEMORY;
709 }
710
711 pWi->flags = NV_WORK_ITEM_FLAGS_REQUIRES_GPU;
712 if (flags & OS_QUEUE_WORKITEM_FLAGS_DONT_FREE_PARAMS)
713 pWi->flags |= NV_WORK_ITEM_FLAGS_DONT_FREE_DATA;
714
715 if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA)
716 pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_SEMA;
717 if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW)
718 pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_API_RW;
719 if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW)
720 pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPUS_RW;
721 if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW)
722 pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW;
723 if (flags & OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW)
724 pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_SUBDEVICE_RW;
725
726 if (flags & OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY)
727 pWi->flags |= OS_QUEUE_WORKITEM_FLAGS_FULL_GPU_SANITY;
728
729 pWi->gpuInstance = gpuGetInstance(pGpu);
730 pWi->func.pGpuFunction = pFunction;
731 pWi->pData = pParams;
732 nv = NV_GET_NV_STATE(pGpu);
733
734 status = os_queue_work_item(nv ? nv->queue : NULL, pWi);
735
736 if (NV_OK != status)
737 {
738 portMemFree((void *)pWi);
739 }
740
741 return status;
742 }
743
osQueueSystemWorkItem(OSSystemWorkItemFunction pFunction,void * pParams)744 NV_STATUS osQueueSystemWorkItem(
745 OSSystemWorkItemFunction pFunction,
746 void *pParams
747 )
748 {
749 nv_work_item_t *pWi;
750 NV_STATUS status;
751
752 pWi = portMemAllocNonPaged(sizeof(nv_work_item_t));
753
754 if (NULL == pWi)
755 {
756 return NV_ERR_NO_MEMORY;
757 }
758
759 pWi->flags = NV_WORK_ITEM_FLAGS_NONE;
760 pWi->func.pSystemFunction = pFunction;
761 pWi->pData = pParams;
762
763 status = os_queue_work_item(NULL, pWi);
764
765 if (NV_OK != status)
766 {
767 portMemFree((void *)pWi);
768 }
769
770 return status;
771 }
772
osQueueMMUFaultHandler(OBJGPU * pGpu)773 void osQueueMMUFaultHandler(OBJGPU *pGpu)
774 {
775 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
776
777 nv_schedule_uvm_isr(nv);
778 }
779
osGetDmaDeviceForMemDesc(OS_GPU_INFO * pOsGpuInfo,MEMORY_DESCRIPTOR * pMemDesc)780 static inline nv_dma_device_t* osGetDmaDeviceForMemDesc(
781 OS_GPU_INFO *pOsGpuInfo,
782 MEMORY_DESCRIPTOR *pMemDesc
783 )
784 {
785 return (pOsGpuInfo->niso_dma_dev != NULL) &&
786 memdescGetFlag(pMemDesc, MEMDESC_FLAGS_MEMORY_TYPE_DISPLAY_NISO) ?
787 pOsGpuInfo->niso_dma_dev : pOsGpuInfo->dma_dev;
788 }
789
790 //
791 // Set the DMA address size for the given GPU
792 //
793 // This is a global device setting and care would need to be taken if it was to
794 // be modified outside of GPU initialization. At least on Linux other drivers,
795 // like UVM, might be requesting its own DMA mappings for the same GPU after
796 // the GPU has been initialized.
797 //
osDmaSetAddressSize(OS_GPU_INFO * pOsGpuInfo,NvU32 bits)798 void osDmaSetAddressSize(
799 OS_GPU_INFO *pOsGpuInfo,
800 NvU32 bits
801 )
802 {
803 nv_set_dma_address_size(pOsGpuInfo, bits);
804 }
805
osAllocPagesInternal(MEMORY_DESCRIPTOR * pMemDesc)806 NV_STATUS osAllocPagesInternal(
807 MEMORY_DESCRIPTOR *pMemDesc
808 )
809 {
810 OBJSYS *pSys = SYS_GET_INSTANCE();
811 OBJGPU *pGpu = pMemDesc->pGpu;
812 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
813 void *pMemData = NULL;
814 NV_STATUS status;
815 NvS32 nodeId = NV0000_CTRL_NO_NUMA_NODE;
816 NV_ADDRESS_SPACE addrSpace = memdescGetAddressSpace(pMemDesc);
817
818 memdescSetAddress(pMemDesc, NvP64_NULL);
819 memdescSetMemData(pMemDesc, NULL, NULL);
820
821 NV_ASSERT_OR_RETURN(pMemDesc->PageCount > 0, NV_ERR_INVALID_ARGUMENT);
822
823 if (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
824 {
825 if (NV_RM_PAGE_SIZE < os_page_size &&
826 !memdescGetContiguity(pMemDesc, AT_CPU))
827 {
828 RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
829 pMemDesc->PageCount);
830 }
831
832 status = nv_alias_pages(
833 NV_GET_NV_STATE(pGpu),
834 NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
835 memdescGetContiguity(pMemDesc, AT_CPU),
836 memdescGetCpuCacheAttrib(pMemDesc),
837 memdescGetGuestId(pMemDesc),
838 memdescGetPteArray(pMemDesc, AT_CPU),
839 &pMemData);
840 }
841 else
842 {
843 NvBool unencrypted = 0;
844
845 if (nv && (memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_32BIT_ADDRESSABLE)))
846 nv->force_dma32_alloc = NV_TRUE;
847
848 //
849 // If AMD SEV is enabled but APM(Ampere protected) or CC(Confidential
850 // Compute) mode is not enabled on the GPU, all RM and client
851 // allocations must be an unprotected sysmem. If APM is enabled and RM
852 // is allocating sysmem for its internal use, then it has to be
853 // unprotected as protected sysmem is not accessible to GPU.
854 //
855 unencrypted = memdescGetFlag(pMemDesc, MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY);
856
857 if (addrSpace == ADDR_SYSMEM)
858 {
859 nodeId = memdescGetNumaNode(pMemDesc);
860 }
861 else if (addrSpace == ADDR_EGM)
862 {
863 nodeId = GPU_GET_MEMORY_MANAGER(pGpu)->localEgmNodeId;
864 }
865
866 if (NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount) > NV_U32_MAX)
867 {
868 status = NV_ERR_INVALID_LIMIT;
869 }
870 else
871 {
872 NvU64 pageSize = osGetPageSize();
873
874 //
875 // Bug 4270864: Only non-contig EGM memory needs to specify order. Contig memory
876 // calculates it within nv_alloc_pages. The long term goal is to expand the ability
877 // to request large page size for all of sysmem.
878 //
879 if (memdescIsEgm(pMemDesc) && !memdescGetContiguity(pMemDesc, AT_CPU))
880 {
881 pageSize = memdescGetPageSize(pMemDesc, AT_GPU);
882 }
883
884 status = nv_alloc_pages(
885 NV_GET_NV_STATE(pGpu),
886 NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
887 pageSize,
888 memdescGetContiguity(pMemDesc, AT_CPU),
889 memdescGetCpuCacheAttrib(pMemDesc),
890 pSys->getProperty(pSys,
891 PDB_PROP_SYS_INITIALIZE_SYSTEM_MEMORY_ALLOCATIONS),
892 unencrypted,
893 nodeId,
894 memdescGetPteArray(pMemDesc, AT_CPU),
895 &pMemData);
896 }
897
898 if (nv && nv->force_dma32_alloc)
899 nv->force_dma32_alloc = NV_FALSE;
900 }
901
902 if (status != NV_OK)
903 {
904 return status;
905 }
906
907 //
908 // If the OS layer doesn't think in RM page size, we need to inflate the
909 // PTE array into RM pages.
910 //
911 if (NV_RM_PAGE_SIZE < os_page_size &&
912 !memdescGetContiguity(pMemDesc, AT_CPU))
913 {
914 RmInflateOsToRmPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
915 pMemDesc->PageCount);
916 }
917
918 memdescSetMemData(pMemDesc, pMemData, NULL);
919
920 return status;
921 }
922
osFreePagesInternal(MEMORY_DESCRIPTOR * pMemDesc)923 void osFreePagesInternal(
924 MEMORY_DESCRIPTOR *pMemDesc
925 )
926 {
927 OBJGPU *pGpu = pMemDesc->pGpu;
928 NV_STATUS rmStatus;
929
930 if (NV_RM_PAGE_SIZE < os_page_size &&
931 !memdescGetContiguity(pMemDesc, AT_CPU))
932 {
933 RmDeflateRmToOsPageArray(memdescGetPteArray(pMemDesc, AT_CPU),
934 pMemDesc->PageCount);
935 }
936
937 rmStatus = nv_free_pages(NV_GET_NV_STATE(pGpu),
938 NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount),
939 memdescGetContiguity(pMemDesc, AT_CPU),
940 memdescGetCpuCacheAttrib(pMemDesc),
941 memdescGetMemData(pMemDesc));
942 NV_ASSERT(rmStatus == NV_OK);
943 }
944
osLockMem(MEMORY_DESCRIPTOR * pMemDesc)945 NV_STATUS osLockMem(
946 MEMORY_DESCRIPTOR *pMemDesc
947 )
948 {
949 // Not supported on this OS.
950 DBG_BREAKPOINT();
951 return NV_ERR_NOT_SUPPORTED;
952 }
953
osUnlockMem(MEMORY_DESCRIPTOR * pMemDesc)954 NV_STATUS osUnlockMem(
955 MEMORY_DESCRIPTOR *pMemDesc
956 )
957 {
958 // Not supported on this OS.
959 DBG_BREAKPOINT();
960 return NV_ERR_NOT_SUPPORTED;
961 }
962
osMapPciMemoryUser(OS_GPU_INFO * pOsGpuInfo,RmPhysAddr busAddress,NvU64 length,NvU32 Protect,NvP64 * pVirtualAddress,NvP64 * pPriv,NvU32 modeFlag)963 NV_STATUS osMapPciMemoryUser(
964 OS_GPU_INFO *pOsGpuInfo,
965 RmPhysAddr busAddress,
966 NvU64 length,
967 NvU32 Protect,
968 NvP64 *pVirtualAddress,
969 NvP64 *pPriv,
970 NvU32 modeFlag
971 )
972 {
973 void *addr;
974 NvU64 offset = 0;
975
976 NV_ASSERT_OR_RETURN(length != 0, NV_ERR_INVALID_ARGUMENT);
977
978 offset = busAddress & (os_page_size - 1llu);
979 busAddress = NV_ALIGN_DOWN64(busAddress, os_page_size);
980 length = NV_ALIGN_UP64(busAddress + offset + length, os_page_size) - busAddress;
981
982 addr = os_map_user_space(busAddress, length, modeFlag, Protect, (void **) pPriv);
983
984 NV_ASSERT_OR_RETURN(addr != NULL, NV_ERR_INVALID_ADDRESS);
985
986 *pVirtualAddress = (NvP64)(((NvU64) addr) + offset);
987
988 return NV_OK;
989 }
990
osUnmapPciMemoryUser(OS_GPU_INFO * pOsGpuInfo,NvP64 virtualAddress,NvU64 length,NvP64 pPriv)991 void osUnmapPciMemoryUser(
992 OS_GPU_INFO *pOsGpuInfo,
993 NvP64 virtualAddress,
994 NvU64 length,
995 NvP64 pPriv
996 )
997 {
998 NvU64 addr;
999 void *priv;
1000
1001 addr = (NvU64)(virtualAddress);
1002 priv = (void*)(pPriv);
1003
1004 length = NV_ALIGN_UP64(addr + length, os_page_size);
1005 addr = NV_ALIGN_DOWN64(addr, os_page_size);
1006 length -= addr;
1007
1008 os_unmap_user_space((void *)addr, length, priv);
1009 }
1010
osMapPciMemoryKernelOld(OBJGPU * pGpu,RmPhysAddr busAddress,NvU64 length,NvU32 Protect,void ** pVirtualAddress,NvU32 modeFlag)1011 NV_STATUS osMapPciMemoryKernelOld
1012 (
1013 OBJGPU *pGpu,
1014 RmPhysAddr busAddress,
1015 NvU64 length,
1016 NvU32 Protect,
1017 void **pVirtualAddress,
1018 NvU32 modeFlag
1019 )
1020 {
1021 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1022 nv_kern_mapping_t *mapping;
1023
1024 if (pVirtualAddress == NULL)
1025 return NV_ERR_GENERIC;
1026
1027 *pVirtualAddress = os_map_kernel_space(busAddress, length, modeFlag);
1028 if (*pVirtualAddress == NULL)
1029 return NV_ERR_GENERIC;
1030
1031 mapping = portMemAllocNonPaged(sizeof(nv_kern_mapping_t));
1032 if (NULL == mapping)
1033 {
1034 os_unmap_kernel_space(*pVirtualAddress, length);
1035 *pVirtualAddress = 0;
1036 return NV_ERR_GENERIC;
1037 }
1038
1039 mapping->addr = *pVirtualAddress;
1040 mapping->size = length;
1041 mapping->modeFlag = modeFlag;
1042
1043 mapping->next = nv->kern_mappings;
1044 nv->kern_mappings = mapping;
1045
1046 return NV_OK;
1047 }
1048
osMapPciMemoryKernel64(OBJGPU * pGpu,RmPhysAddr busAddress,NvU64 length,NvU32 Protect,NvP64 * pVirtualAddress,NvU32 modeFlag)1049 NV_STATUS osMapPciMemoryKernel64
1050 (
1051 OBJGPU *pGpu,
1052 RmPhysAddr busAddress,
1053 NvU64 length,
1054 NvU32 Protect,
1055 NvP64 *pVirtualAddress,
1056 NvU32 modeFlag
1057 )
1058 {
1059 void *tmppVirtualAddress = NvP64_VALUE(pVirtualAddress);
1060 NV_STATUS rc;
1061
1062 rc = osMapPciMemoryKernelOld(pGpu,
1063 busAddress,
1064 length,
1065 Protect,
1066 &tmppVirtualAddress,
1067 modeFlag);
1068
1069 *pVirtualAddress = NV_PTR_TO_NvP64(tmppVirtualAddress);
1070
1071 return rc;
1072 }
1073
osUnmapPciMemoryKernelOld(OBJGPU * pGpu,void * virtualAddress)1074 void osUnmapPciMemoryKernelOld
1075 (
1076 OBJGPU *pGpu,
1077 void* virtualAddress
1078 )
1079 {
1080 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1081 nv_kern_mapping_t *mapping, *tmp;
1082
1083 // this can happen, for example, during a call to RmShutdownAdapter()
1084 // from a failed RmInitAdapter()
1085 if (virtualAddress == NULL)
1086 {
1087 return;
1088 }
1089
1090 tmp = mapping = nv->kern_mappings;
1091 while (mapping)
1092 {
1093 if (mapping->addr == virtualAddress)
1094 {
1095 if (mapping == nv->kern_mappings)
1096 {
1097 nv->kern_mappings = mapping->next;
1098 }
1099 else
1100 {
1101 tmp->next = mapping->next;
1102 }
1103
1104 os_unmap_kernel_space(mapping->addr, mapping->size);
1105
1106 portMemFree(mapping);
1107 return;
1108 }
1109 tmp = mapping;
1110 mapping = mapping->next;
1111 }
1112
1113 DBG_BREAKPOINT();
1114 }
1115
osUnmapPciMemoryKernel64(OBJGPU * pGpu,NvP64 virtualAddress)1116 void osUnmapPciMemoryKernel64
1117 (
1118 OBJGPU *pGpu,
1119 NvP64 virtualAddress
1120 )
1121 {
1122 osUnmapPciMemoryKernelOld(pGpu, NvP64_VALUE(virtualAddress));
1123 }
1124
osMapGPU(OBJGPU * pGpu,RS_PRIV_LEVEL privLevel,NvU64 offset,NvU64 length,NvU32 Protect,NvP64 * pAddress,NvP64 * pPriv)1125 NV_STATUS osMapGPU(
1126 OBJGPU *pGpu,
1127 RS_PRIV_LEVEL privLevel,
1128 NvU64 offset,
1129 NvU64 length,
1130 NvU32 Protect,
1131 NvP64 *pAddress,
1132 NvP64 *pPriv
1133 )
1134 {
1135 NV_STATUS rmStatus = NV_OK;
1136
1137 if (privLevel >= RS_PRIV_LEVEL_KERNEL)
1138 {
1139 if (!portSafeAddU64((NvUPtr)pGpu->deviceMappings[0].gpuNvAddr, offset, (NvU64*)pAddress))
1140 {
1141 rmStatus = NV_ERR_INVALID_LIMIT;
1142 }
1143 }
1144 else
1145 {
1146 RmPhysAddr busAddress;
1147 if (!portSafeAddU64(pGpu->busInfo.gpuPhysAddr, offset, &busAddress))
1148 {
1149 rmStatus = NV_ERR_INVALID_LIMIT;
1150 }
1151 else
1152 {
1153 rmStatus = osMapPciMemoryUser(pGpu->pOsGpuInfo,
1154 busAddress,
1155 length,
1156 Protect,
1157 pAddress,
1158 pPriv,
1159 NV_FALSE);
1160 }
1161 }
1162
1163 return rmStatus;
1164 }
1165
osUnmapGPU(OS_GPU_INFO * pOsGpuInfo,RS_PRIV_LEVEL privLevel,NvP64 address,NvU64 length,NvP64 priv)1166 void osUnmapGPU(
1167 OS_GPU_INFO *pOsGpuInfo,
1168 RS_PRIV_LEVEL privLevel,
1169 NvP64 address,
1170 NvU64 length,
1171 NvP64 priv
1172 )
1173 {
1174 if (privLevel < RS_PRIV_LEVEL_KERNEL)
1175 {
1176 osUnmapPciMemoryUser(pOsGpuInfo, address, length, priv);
1177 }
1178 }
1179
postEvent(nv_event_t * event,NvU32 hEvent,NvU32 notifyIndex,NvU32 info32,NvU16 info16,NvBool dataValid)1180 static void postEvent(
1181 nv_event_t *event,
1182 NvU32 hEvent,
1183 NvU32 notifyIndex,
1184 NvU32 info32,
1185 NvU16 info16,
1186 NvBool dataValid
1187 )
1188 {
1189 if (osReferenceObjectCount(event) != NV_OK)
1190 return;
1191 nv_post_event(event, hEvent, notifyIndex,
1192 info32, info16, dataValid);
1193 osDereferenceObjectCount(event);
1194 }
1195
osSetEvent(OBJGPU * pGpu,NvP64 eventID)1196 NvU32 osSetEvent
1197 (
1198 OBJGPU *pGpu,
1199 NvP64 eventID
1200 )
1201 {
1202 nv_event_t *event = NvP64_VALUE(eventID);
1203 postEvent(event, 0, 0, 0, 0, NV_FALSE);
1204 return 1;
1205 }
1206
osNotifyEvent(OBJGPU * pGpu,PEVENTNOTIFICATION NotifyEvent,NvU32 Method,NvU32 Data,NV_STATUS Status)1207 NV_STATUS osNotifyEvent(
1208 OBJGPU *pGpu,
1209 PEVENTNOTIFICATION NotifyEvent,
1210 NvU32 Method,
1211 NvU32 Data,
1212 NV_STATUS Status
1213 )
1214 {
1215 NV_STATUS rmStatus = NV_OK;
1216
1217 // notify the event
1218 switch (NotifyEvent->NotifyType)
1219 {
1220 case NV01_EVENT_OS_EVENT:
1221 {
1222 nv_event_t *event = NvP64_VALUE(NotifyEvent->Data);
1223 postEvent(event,
1224 NotifyEvent->hEvent,
1225 NotifyEvent->NotifyIndex,
1226 0, 0,
1227 NotifyEvent->bEventDataRequired);
1228 break;
1229 }
1230
1231 // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX.
1232 case NV01_EVENT_KERNEL_CALLBACK:
1233 {
1234 MINIPORT_CALLBACK callBackToMiniport =
1235 (MINIPORT_CALLBACK)NvP64_VALUE(NotifyEvent->Data);
1236
1237 // perform a direct callback to the miniport
1238 if (callBackToMiniport)
1239 callBackToMiniport(NV_GET_NV_STATE(pGpu));
1240 break;
1241 }
1242
1243 case NV01_EVENT_KERNEL_CALLBACK_EX:
1244 {
1245 NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(NotifyEvent->Data);
1246
1247 // passes two arguments (arg, params) to the kernel callback instead of one (arg).
1248 if (kc && kc->func)
1249 {
1250 kc->func(kc->arg, NULL, NotifyEvent->hEvent, Data, Status);
1251 }
1252 break;
1253 }
1254
1255
1256 default:
1257 {
1258 rmStatus = NV_ERR_GENERIC;
1259 break;
1260 }
1261 }
1262
1263 return rmStatus;
1264
1265 } // end of osNotifyEvent()
1266
1267 // Allow CPL Events to be callback or events
osEventNotification(OBJGPU * pGpu,PEVENTNOTIFICATION pNotifyEvent,NvU32 notifyIndex,void * pEventData,NvU32 eventDataSize)1268 NV_STATUS osEventNotification
1269 (
1270 OBJGPU *pGpu,
1271 PEVENTNOTIFICATION pNotifyEvent,
1272 NvU32 notifyIndex,
1273 void * pEventData,
1274 NvU32 eventDataSize
1275 )
1276 {
1277 return osEventNotificationWithInfo(pGpu, pNotifyEvent, notifyIndex, 0, 0,
1278 pEventData, eventDataSize);
1279 }
1280
osEventNotificationWithInfo(OBJGPU * pGpu,PEVENTNOTIFICATION pNotifyEvent,NvU32 notifyIndex,NvU32 info32,NvU16 info16,void * pEventData,NvU32 eventDataSize)1281 NV_STATUS osEventNotificationWithInfo
1282 (
1283 OBJGPU *pGpu,
1284 PEVENTNOTIFICATION pNotifyEvent,
1285 NvU32 notifyIndex,
1286 NvU32 info32,
1287 NvU16 info16,
1288 void * pEventData,
1289 NvU32 eventDataSize
1290 )
1291 {
1292 NV_STATUS rmStatus = NV_OK;
1293
1294 // walk this object's event list and find any matches for this specific notify
1295 for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next)
1296 {
1297 // notifyIndex must match if request isn't for all
1298 if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) &&
1299 (pNotifyEvent->NotifyIndex != notifyIndex))
1300 {
1301 continue;
1302 }
1303
1304 switch (pNotifyEvent->NotifyType)
1305 {
1306 case NV_EVENT_BUFFER_BIND:
1307 case NV01_EVENT_WIN32_EVENT:
1308 {
1309 nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data);
1310 postEvent(event,
1311 pNotifyEvent->hEvent,
1312 pNotifyEvent->NotifyIndex,
1313 info32, info16,
1314 pNotifyEvent->bEventDataRequired);
1315 break;
1316 }
1317
1318 // NOTE: NV01_EVENT_KERNEL_CALLBACK is deprecated. please use NV01_EVENT_KERNEL_CALLBACK_EX.
1319 case NV01_EVENT_KERNEL_CALLBACK:
1320 {
1321 MINIPORT_CALLBACK callBackToMiniport =
1322 (MINIPORT_CALLBACK)NvP64_VALUE(pNotifyEvent->Data);
1323
1324 // perform a direct callback to the miniport
1325 if (callBackToMiniport)
1326 callBackToMiniport(NV_GET_NV_STATE(pGpu));
1327 break;
1328 }
1329
1330 case NV01_EVENT_KERNEL_CALLBACK_EX:
1331 {
1332 NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data);
1333
1334 if (kc && kc->func)
1335 {
1336 kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK);
1337 }
1338 break;
1339 }
1340
1341 default:
1342 break;
1343 }
1344 }
1345
1346 return rmStatus;
1347 }
1348
1349 // Allow CPL Events to be callback or events
osObjectEventNotification(NvHandle hClient,NvHandle hObject,NvU32 hClass,PEVENTNOTIFICATION pNotifyEvent,NvU32 notifyIndex,void * pEventData,NvU32 eventDataSize)1350 NV_STATUS osObjectEventNotification
1351 (
1352 NvHandle hClient,
1353 NvHandle hObject,
1354 NvU32 hClass,
1355 PEVENTNOTIFICATION pNotifyEvent,
1356 NvU32 notifyIndex,
1357 void *pEventData,
1358 NvU32 eventDataSize
1359 )
1360 {
1361 NV_STATUS rmStatus = NV_OK;
1362
1363 NV_PRINTF(LEVEL_INFO, "%s()\n", __FUNCTION__);
1364 // walk this object's event list and find any matches for this specific notify
1365 for (; pNotifyEvent; pNotifyEvent = pNotifyEvent->Next)
1366 {
1367 // notifyIndex must match if request isn't for all
1368 if ((notifyIndex != OS_EVENT_NOTIFICATION_INDEX_ALL) &&
1369 (pNotifyEvent->NotifyIndex != notifyIndex))
1370 {
1371 continue;
1372 }
1373
1374 switch (pNotifyEvent->NotifyType)
1375 {
1376 case NV01_EVENT_OS_EVENT:
1377 {
1378 nv_event_t *event = NvP64_VALUE(pNotifyEvent->Data);
1379 postEvent(event,
1380 pNotifyEvent->hEvent,
1381 pNotifyEvent->NotifyIndex,
1382 0, 0,
1383 pNotifyEvent->bEventDataRequired);
1384 break;
1385 }
1386
1387 case NV01_EVENT_KERNEL_CALLBACK_EX:
1388 {
1389 NVOS10_EVENT_KERNEL_CALLBACK_EX *kc = (NVOS10_EVENT_KERNEL_CALLBACK_EX *)NvP64_VALUE(pNotifyEvent->Data);
1390
1391 if (kc && kc->func)
1392 {
1393 kc->func(kc->arg, pEventData, pNotifyEvent->hEvent, 0, NV_OK);
1394 }
1395 break;
1396 }
1397
1398 default:
1399 break;
1400 }
1401 }
1402
1403 return rmStatus;
1404 }
1405
osReferenceObjectCount(void * pEvent)1406 NV_STATUS osReferenceObjectCount(void *pEvent)
1407 {
1408 nv_state_t *nv = nv_get_ctl_state();
1409 nv_event_t *event = pEvent;
1410
1411 portSyncSpinlockAcquire(nv->event_spinlock);
1412 // If event->active is false, don't allow any more reference
1413 if (!event->active)
1414 {
1415 portSyncSpinlockRelease(nv->event_spinlock);
1416 return NV_ERR_INVALID_EVENT;
1417 }
1418 ++event->refcount;
1419 portSyncSpinlockRelease(nv->event_spinlock);
1420 return NV_OK;
1421 }
1422
osDereferenceObjectCount(void * pOSEvent)1423 NV_STATUS osDereferenceObjectCount(void *pOSEvent)
1424 {
1425 nv_state_t *nv = nv_get_ctl_state();
1426 nv_event_t *event = pOSEvent;
1427
1428 portSyncSpinlockAcquire(nv->event_spinlock);
1429 NV_ASSERT(event->refcount > 0);
1430 // If event->refcount == 0 but event->active is true, the client
1431 // has not yet freed the OS event. free_os_event will free its
1432 // memory when they do, or else when the client itself is freed.
1433 if (--event->refcount == 0 && !event->active)
1434 portMemFree(event);
1435 portSyncSpinlockRelease(nv->event_spinlock);
1436
1437 return NV_OK;
1438 }
1439
osUserHandleToKernelPtr(NvHandle hClient,NvP64 hEvent,NvP64 * pEvent)1440 NV_STATUS osUserHandleToKernelPtr(NvHandle hClient, NvP64 hEvent, NvP64 *pEvent)
1441 {
1442 nv_state_t *nv = nv_get_ctl_state();
1443 NvU32 fd = (NvU64)hEvent;
1444 NV_STATUS result;
1445
1446 portSyncSpinlockAcquire(nv->event_spinlock);
1447 nv_event_t *e = nv->event_list;
1448 while (e != NULL)
1449 {
1450 if (e->fd == fd && e->hParent == hClient)
1451 break;
1452 e = e->next;
1453 }
1454
1455 if (e != NULL)
1456 {
1457 ++e->refcount;
1458 *pEvent = NV_PTR_TO_NvP64(e);
1459 result = NV_OK;
1460 }
1461 else
1462 result = NV_ERR_OBJECT_NOT_FOUND;
1463 portSyncSpinlockRelease(nv->event_spinlock);
1464
1465 return result;
1466 }
1467
osFlushCpuCache(void)1468 NV_STATUS osFlushCpuCache(void)
1469 {
1470 return os_flush_cpu_cache_all();
1471 }
1472
osFlushCpuWriteCombineBuffer(void)1473 void osFlushCpuWriteCombineBuffer(void)
1474 {
1475 os_flush_cpu_write_combine_buffer();
1476 }
1477
1478
1479 //
1480 // Evict GPU memory range from the CPU caches.
1481 //
1482 // On some platforms (e.g. P9+V100), the CPU can coherently cache GPU memory
1483 // and RM takes advantage of that. Most everything is handled transparently,
1484 // but there are two exceptions that require explicitly flushing any CPU cache
1485 // lines of GPU memory. These are:
1486 //
1487 // 1) Flushing memory backing ACR regions before they get locked.
1488 //
1489 // Otherwise the cache could get flushed while the regions are locked causing a
1490 // region violation physical fault. See more details in
1491 // acrFlushRegionsFromGpuCoherentCpuCache_IMPL().
1492 //
1493 // 2) Flushing all of FB before GPU reset (NVLink going down)
1494 //
1495 // Leaving cache entries on the CPU causes fatal errors when the CPU tries
1496 // flushing them later while the link is down. See more details in
1497 // nvlinkStatePostUnload_IMPL().
1498 //
osFlushGpuCoherentCpuCacheRange(OS_GPU_INFO * pOsGpuInfo,NvU64 cpuVirtual,NvU64 size)1499 void osFlushGpuCoherentCpuCacheRange
1500 (
1501 OS_GPU_INFO *pOsGpuInfo,
1502 NvU64 cpuVirtual,
1503 NvU64 size
1504 )
1505 {
1506 nv_flush_coherent_cpu_cache_range(pOsGpuInfo, cpuVirtual, size);
1507 }
1508
osErrorLogV(OBJGPU * pGpu,NvU32 num,const char * pFormat,va_list arglist)1509 void osErrorLogV(OBJGPU *pGpu, NvU32 num, const char * pFormat, va_list arglist)
1510 {
1511 NV_STATUS rmStatus;
1512 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1513
1514 if ((pFormat == NULL) || (*pFormat == '\0'))
1515 {
1516 return;
1517 }
1518
1519 rmStatus = nv_log_error(nv, num, pFormat, arglist);
1520 NV_ASSERT(rmStatus == NV_OK);
1521 }
1522
osErrorLog(OBJGPU * pGpu,NvU32 num,const char * pFormat,...)1523 void osErrorLog(OBJGPU *pGpu, NvU32 num, const char* pFormat, ...)
1524 {
1525 va_list arglist;
1526 va_start(arglist, pFormat);
1527 osErrorLogV(pGpu, num, pFormat, arglist);
1528 va_end(arglist);
1529 }
1530
1531 NvU32
osPollHotkeyState(OBJGPU * pGpu)1532 osPollHotkeyState
1533 (
1534 OBJGPU *pGpu
1535 )
1536 {
1537 return 0;
1538 }
1539
osDevWriteReg008(OBJGPU * pGpu,DEVICE_MAPPING * pMapping,NvU32 thisAddress,NvV8 thisValue)1540 void osDevWriteReg008(
1541 OBJGPU *pGpu,
1542 DEVICE_MAPPING *pMapping,
1543 NvU32 thisAddress,
1544 NvV8 thisValue
1545 )
1546 {
1547 if (thisAddress >= pMapping->gpuNvLength)
1548 {
1549 NV_ASSERT(thisAddress < pMapping->gpuNvLength);
1550 return;
1551 }
1552
1553 NV_PRIV_REG_WR08(pMapping->gpuNvAddr, thisAddress, thisValue);
1554 }
1555
osDevWriteReg016(OBJGPU * pGpu,DEVICE_MAPPING * pMapping,NvU32 thisAddress,NvV16 thisValue)1556 void osDevWriteReg016(
1557 OBJGPU *pGpu,
1558 DEVICE_MAPPING *pMapping,
1559 NvU32 thisAddress,
1560 NvV16 thisValue
1561 )
1562 {
1563 if (thisAddress >= pMapping->gpuNvLength)
1564 {
1565 NV_ASSERT(thisAddress < pMapping->gpuNvLength);
1566 return;
1567 }
1568
1569 NV_PRIV_REG_WR16(pMapping->gpuNvAddr, thisAddress, thisValue);
1570 }
1571
osDevWriteReg032(OBJGPU * pGpu,DEVICE_MAPPING * pMapping,NvU32 thisAddress,NvV32 thisValue)1572 void osDevWriteReg032(
1573 OBJGPU *pGpu,
1574 DEVICE_MAPPING *pMapping,
1575 NvU32 thisAddress,
1576 NvV32 thisValue
1577 )
1578 {
1579 NvBool vgpuHandled = NV_FALSE;
1580
1581 vgpuDevWriteReg032(pGpu, thisAddress, thisValue, &vgpuHandled);
1582 if (vgpuHandled)
1583 {
1584 return;
1585 }
1586
1587 if (thisAddress >= pMapping->gpuNvLength)
1588 {
1589 NV_ASSERT(thisAddress < pMapping->gpuNvLength);
1590 return;
1591 }
1592
1593 NV_PRIV_REG_WR32(pMapping->gpuNvAddr, thisAddress, thisValue);
1594 }
1595
osDevReadReg008(OBJGPU * pGpu,DEVICE_MAPPING * pMapping,NvU32 thisAddress)1596 NvU8 osDevReadReg008(
1597 OBJGPU *pGpu,
1598 DEVICE_MAPPING *pMapping,
1599 NvU32 thisAddress
1600 )
1601 {
1602 NvU8 retval = 0;
1603
1604 if (thisAddress >= pMapping->gpuNvLength)
1605 {
1606 NV_ASSERT(thisAddress < pMapping->gpuNvLength);
1607 }
1608 else
1609 retval = NV_PRIV_REG_RD08(pMapping->gpuNvAddr, thisAddress);
1610
1611 return retval;
1612 }
1613
osDevReadReg016(OBJGPU * pGpu,DEVICE_MAPPING * pMapping,NvU32 thisAddress)1614 NvU16 osDevReadReg016(
1615 OBJGPU *pGpu,
1616 DEVICE_MAPPING *pMapping,
1617 NvU32 thisAddress
1618 )
1619 {
1620 NvU16 retval = 0;
1621
1622 if (thisAddress >= pMapping->gpuNvLength)
1623 {
1624 NV_ASSERT(thisAddress < pMapping->gpuNvLength);
1625 }
1626 else
1627 retval = NV_PRIV_REG_RD16(pMapping->gpuNvAddr, thisAddress);
1628
1629 return retval;
1630 }
1631
osDevReadReg032(OBJGPU * pGpu,DEVICE_MAPPING * pMapping,NvU32 thisAddress)1632 NvU32 osDevReadReg032(
1633 OBJGPU *pGpu,
1634 DEVICE_MAPPING *pMapping,
1635 NvU32 thisAddress
1636 )
1637 {
1638 NvU32 retval = 0;
1639 NvBool vgpuHandled = NV_FALSE;
1640
1641 retval = vgpuDevReadReg032(pGpu, thisAddress, &vgpuHandled);
1642 if (vgpuHandled)
1643 {
1644 return retval;
1645 }
1646
1647 if (thisAddress >= pMapping->gpuNvLength)
1648 {
1649 NV_ASSERT(thisAddress < pMapping->gpuNvLength);
1650 }
1651 else
1652 retval = NV_PRIV_REG_RD32(pMapping->gpuNvAddr, thisAddress);
1653
1654 return retval;
1655 }
1656
osReadRegistryDwordBase(OBJGPU * pGpu,const char * regParmStr,NvU32 * Data)1657 NV_STATUS osReadRegistryDwordBase(
1658 OBJGPU *pGpu,
1659 const char *regParmStr,
1660 NvU32 *Data
1661 )
1662 {
1663 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1664 return RmReadRegistryDword(nv, regParmStr, Data);
1665 }
1666
osWriteRegistryDword(OBJGPU * pGpu,const char * regParmStr,NvU32 Data)1667 NV_STATUS osWriteRegistryDword(
1668 OBJGPU *pGpu,
1669 const char *regParmStr,
1670 NvU32 Data
1671 )
1672 {
1673 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1674 return RmWriteRegistryDword(nv, regParmStr, Data);
1675 }
1676
osReadRegistryBinary(OBJGPU * pGpu,const char * regParmStr,NvU8 * Data,NvU32 * cbLen)1677 NV_STATUS osReadRegistryBinary(
1678 OBJGPU *pGpu,
1679 const char *regParmStr,
1680 NvU8 *Data,
1681 NvU32 *cbLen
1682 )
1683 {
1684 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1685 return RmReadRegistryBinary(nv, regParmStr, Data, cbLen);
1686 }
1687
osWriteRegistryBinary(OBJGPU * pGpu,const char * regParmStr,NvU8 * Data,NvU32 cbLen)1688 NV_STATUS osWriteRegistryBinary(
1689 OBJGPU *pGpu,
1690 const char *regParmStr,
1691 NvU8 *Data,
1692 NvU32 cbLen
1693 )
1694 {
1695 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1696 return RmWriteRegistryBinary(nv, regParmStr, Data, cbLen);
1697 }
1698
osWriteRegistryVolatile(OBJGPU * pGpu,const char * regParmStr,NvU8 * Data,NvU32 cbLen)1699 NV_STATUS osWriteRegistryVolatile(
1700 OBJGPU *pGpu,
1701 const char *regParmStr,
1702 NvU8 *Data,
1703 NvU32 cbLen
1704 )
1705 {
1706 return NV_ERR_NOT_SUPPORTED;
1707 }
1708
osReadRegistryVolatile(OBJGPU * pGpu,const char * regParmStr,NvU8 * Data,NvU32 cbLen)1709 NV_STATUS osReadRegistryVolatile
1710 (
1711 OBJGPU *pGpu,
1712 const char *regParmStr,
1713 NvU8 *Data,
1714 NvU32 cbLen
1715 )
1716 {
1717 return NV_ERR_NOT_SUPPORTED;
1718 }
1719
osReadRegistryVolatileSize(OBJGPU * pGpu,const char * regParmStr,NvU32 * pSize)1720 NV_STATUS osReadRegistryVolatileSize
1721 (
1722 OBJGPU *pGpu,
1723 const char *regParmStr,
1724 NvU32 *pSize
1725 )
1726 {
1727 return NV_ERR_NOT_SUPPORTED;
1728 }
1729
osReadRegistryStringBase(OBJGPU * pGpu,const char * regParmStr,NvU8 * buffer,NvU32 * pBufferLength)1730 NV_STATUS osReadRegistryStringBase(
1731 OBJGPU *pGpu,
1732 const char *regParmStr,
1733 NvU8 *buffer,
1734 NvU32 *pBufferLength
1735 )
1736 {
1737 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1738 return RmReadRegistryString(nv, regParmStr, buffer, pBufferLength);
1739 }
1740
osPackageRegistry(OBJGPU * pGpu,PACKED_REGISTRY_TABLE * pRegTable,NvU32 * pSize)1741 NV_STATUS osPackageRegistry(
1742 OBJGPU *pGpu,
1743 PACKED_REGISTRY_TABLE *pRegTable,
1744 NvU32 *pSize
1745 )
1746 {
1747 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
1748 return RmPackageRegistry(nv, pRegTable, pSize);
1749 }
1750
osGetCpuCount(void)1751 NvU32 osGetCpuCount(void)
1752 {
1753 return os_get_cpu_count(); // Total number of logical CPUs.
1754 }
1755
osGetCurrentProcessorNumber(void)1756 NvU32 osGetCurrentProcessorNumber(void)
1757 {
1758 return os_get_cpu_number();
1759 }
1760
osGetTimeoutParams(OBJGPU * pGpu,NvU32 * pTimeoutUs,NvU32 * pScale,NvU32 * pFlags)1761 void osGetTimeoutParams(OBJGPU *pGpu, NvU32 *pTimeoutUs, NvU32 *pScale, NvU32 *pFlags)
1762 {
1763 NvU32 gpuMode = gpuGetMode(pGpu);
1764
1765 NV_ASSERT((NV_GPU_MODE_GRAPHICS_MODE == gpuMode) ||
1766 (NV_GPU_MODE_COMPUTE_MODE == gpuMode));
1767
1768 if (hypervisorIsVgxHyper())
1769 {
1770 if (IS_GSP_CLIENT(pGpu) && pGpu->getProperty(pGpu, PDB_PROP_GPU_EXTENDED_GSP_RM_INITIALIZATION_TIMEOUT_FOR_VGX) &&
1771 !pGpu->gspRmInitialized)
1772 {
1773 //
1774 // For Hopper, 1.8 seconds is not enough to boot GSP-RM.
1775 // To avoid this issue, 4 seconds timeout is set on initialization,
1776 // and then it's going to be changed 1.8 seconds after GSP initialization.
1777 //
1778 *pTimeoutUs = 4 * 1000000;
1779 }
1780 else
1781 {
1782 //
1783 // 1.8 seconds is chosen because it is 90% of the overall hard limit of 2.0
1784 // seconds, imposed by WDDM driver rules.
1785 // Currently primary use case of VGX is Windows, so setting 1.8 as default
1786 //
1787 *pTimeoutUs = 1.8 * 1000000;
1788 }
1789 }
1790 else
1791 {
1792 switch (gpuMode)
1793 {
1794 default:
1795 case NV_GPU_MODE_GRAPHICS_MODE:
1796 *pTimeoutUs = 4 * 1000000;
1797 break;
1798
1799 case NV_GPU_MODE_COMPUTE_MODE:
1800 *pTimeoutUs = 30 * 1000000;
1801 break;
1802 }
1803 }
1804
1805 *pFlags = GPU_TIMEOUT_FLAGS_OSTIMER;
1806
1807 *pScale = 1;
1808 if (IS_EMULATION(pGpu) || IS_SIMULATION(pGpu))
1809 {
1810 *pScale = 60; // 1s -> 1m
1811 }
1812 return;
1813 }
1814
osFlushLog(void)1815 void osFlushLog(void)
1816 {
1817 // Not implemented
1818 }
1819
osGetSimulationMode(void)1820 NvU32 osGetSimulationMode(void)
1821 {
1822 return NV_SIM_MODE_HARDWARE;
1823 }
1824
1825 NV_STATUS
cliresCtrlCmdOsUnixFlushUserCache_IMPL(RmClientResource * pRmCliRes,NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS * pAddressSpaceParams)1826 cliresCtrlCmdOsUnixFlushUserCache_IMPL
1827 (
1828 RmClientResource *pRmCliRes,
1829 NV0000_CTRL_OS_UNIX_FLUSH_USER_CACHE_PARAMS *pAddressSpaceParams
1830 )
1831 {
1832 Memory *pMemory;
1833 MEMORY_DESCRIPTOR *pMemDesc;
1834 NvU64 start, end;
1835 NvBool bInvalidateOnly;
1836
1837 NV_CHECK_OK_OR_RETURN(LEVEL_SILENT,
1838 memGetByHandle(RES_GET_CLIENT(pRmCliRes),
1839 pAddressSpaceParams->hObject,
1840 &pMemory));
1841
1842 pMemDesc = pMemory->pMemDesc;
1843
1844 if (memdescGetAddressSpace(pMemDesc) != ADDR_SYSMEM)
1845 {
1846 NV_PRINTF(LEVEL_ERROR, "%s: wrong address space %d\n",
1847 __FUNCTION__, memdescGetAddressSpace(pMemDesc));
1848 return NV_ERR_INVALID_COMMAND;
1849 }
1850
1851 if (memdescGetCpuCacheAttrib(pMemDesc) != NV_MEMORY_CACHED)
1852 {
1853 NV_PRINTF(LEVEL_ERROR, "%s: wrong caching type %d\n",
1854 __FUNCTION__, memdescGetCpuCacheAttrib(pMemDesc));
1855 return NV_ERR_INVALID_COMMAND;
1856 }
1857
1858 start = pAddressSpaceParams->offset;
1859 end = start + pAddressSpaceParams->length;
1860
1861 switch(pAddressSpaceParams->cacheOps)
1862 {
1863 case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH_INVALIDATE:
1864 case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_FLUSH:
1865 bInvalidateOnly = NV_FALSE;
1866 break;
1867
1868 case NV0000_CTRL_OS_UNIX_FLAGS_USER_CACHE_INVALIDATE:
1869 bInvalidateOnly = NV_TRUE;
1870 break;
1871
1872 default:
1873 NV_PRINTF(LEVEL_ERROR, "%s: cacheOps not specified\n", __FUNCTION__);
1874 return NV_ERR_INVALID_COMMAND;
1875 }
1876
1877 if ((end - start) > pMemory->Length)
1878 {
1879 NV_PRINTF(LEVEL_ERROR,
1880 "%s: end address 0x%llx exceeded buffer length: 0x%llx\n",
1881 __FUNCTION__, end, pMemory->Length);
1882 return NV_ERR_INVALID_LIMIT;
1883 }
1884
1885 if (bInvalidateOnly)
1886 {
1887 //
1888 // XXX: this seems fishy - I'm not sure if invalidating by the kernel
1889 // VA only as nv_dma_cache_invalidate() does here is sufficient for
1890 // this control call.
1891 // pAddressSpaceParams->internalOnly is expected to be the RM client
1892 // VA for this control call; if we wanted to invalidate the user VA we
1893 // could do so using that.
1894 //
1895 // For I/O coherent platforms this won't actually do anything.
1896 // On non-I/O-coherent platforms, there's no need to do a second
1897 // invalidation after the full flush.
1898 //
1899 nv_state_t *nv = NV_GET_NV_STATE(pMemDesc->pGpu);
1900 if (nv->iovaspace_id != NV_IOVA_DOMAIN_NONE)
1901 {
1902 PIOVAMAPPING pIovaMapping = memdescGetIommuMap(pMemDesc, nv->iovaspace_id);
1903 //
1904 // This should only be called for devices that map memory descriptors
1905 // through the nv-dma library, where the memory descriptor data
1906 // contains all the kernel-specific context we need for the
1907 // invalidation.
1908 //
1909 // (These checks match those in osIovaUnmap() leading up to
1910 // nv_dma_unmap_alloc()).
1911 //
1912 if (pIovaMapping == NULL ||
1913 pIovaMapping->pOsData == NULL ||
1914 memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED) ||
1915 memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
1916 {
1917 return NV_ERR_INVALID_ARGUMENT;
1918 }
1919
1920 nv_dma_cache_invalidate(nv->dma_dev, pIovaMapping->pOsData);
1921 }
1922 else
1923 {
1924 return NV_ERR_INVALID_ARGUMENT;
1925 }
1926 }
1927 else
1928 {
1929 return os_flush_user_cache();
1930 }
1931
1932 return NV_OK;
1933 }
1934
1935 static NV_STATUS
_initializeExportObjectFd(nv_file_private_t * nvfp,NvHandle hClient,NvHandle hDevice,NvU16 maxObjects,NvU8 * metadata)1936 _initializeExportObjectFd
1937 (
1938 nv_file_private_t *nvfp,
1939 NvHandle hClient,
1940 NvHandle hDevice,
1941 NvU16 maxObjects,
1942 NvU8 *metadata
1943 )
1944 {
1945 NV_STATUS status;
1946 RsResourceRef *pResourceRef;
1947 Device *pDevice;
1948 NvU32 deviceInstance = NV_MAX_DEVICES;
1949 NvU32 gpuInstanceId = NV_U32_MAX;
1950
1951 if (nvfp->handles != NULL)
1952 {
1953 return NV_ERR_STATE_IN_USE;
1954 }
1955
1956 if (hDevice != 0)
1957 {
1958 OBJGPU *pGpu;
1959 MIG_INSTANCE_REF ref;
1960
1961 status = serverutilGetResourceRef(hClient, hDevice, &pResourceRef);
1962 if (status != NV_OK)
1963 {
1964 return status;
1965 }
1966
1967 pDevice = dynamicCast(pResourceRef->pResource, Device);
1968 if (pDevice == NULL)
1969 {
1970 return NV_ERR_INVALID_PARAMETER;
1971 }
1972
1973 deviceInstance = pDevice->deviceInst;
1974 pGpu = GPU_RES_GET_GPU(pDevice);
1975
1976 if (IS_MIG_IN_USE(pGpu))
1977 {
1978 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
1979 status = kmigmgrGetInstanceRefFromDevice(pGpu, pKernelMIGManager,
1980 pDevice, &ref);
1981 if (status == NV_OK)
1982 {
1983 gpuInstanceId = ref.pKernelMIGGpuInstance->swizzId;
1984 }
1985 }
1986 }
1987
1988 NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void **)&nvfp->handles,
1989 sizeof(nvfp->handles[0]) * maxObjects));
1990
1991 os_mem_set(nvfp->handles, 0,
1992 sizeof(nvfp->handles[0]) * maxObjects);
1993
1994 nvfp->maxHandles = maxObjects;
1995 nvfp->deviceInstance = deviceInstance;
1996 nvfp->gpuInstanceId = gpuInstanceId;
1997
1998 if (metadata != NULL)
1999 {
2000 os_mem_copy(nvfp->metadata, metadata, sizeof(nvfp->metadata));
2001 }
2002
2003 return NV_OK;
2004 }
2005
2006 NV_STATUS
cliresCtrlCmdOsUnixExportObjectToFd_IMPL(RmClientResource * pRmCliRes,NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS * pParams)2007 cliresCtrlCmdOsUnixExportObjectToFd_IMPL
2008 (
2009 RmClientResource *pRmCliRes,
2010 NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TO_FD_PARAMS *pParams
2011 )
2012 {
2013 NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
2014 RmObjExportHandle hExportHandle = 0;
2015 nv_file_private_t *nvfp = NULL;
2016 void *priv = NULL;
2017 NV_STATUS status = NV_OK;
2018
2019 /*
2020 * This flag is intended to be implemented entirely in the rmapi library in
2021 * userspace, we should never encounter it here.
2022 */
2023 if (FLD_TEST_DRF(0000_CTRL, _OS_UNIX_EXPORT_OBJECT_TO_FD_FLAGS,
2024 _EMPTY_FD, _TRUE, pParams->flags))
2025 {
2026 return NV_ERR_INVALID_PARAMETER;
2027 }
2028
2029 if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM ||
2030 pParams->fd == -1)
2031 {
2032 return NV_ERR_INVALID_PARAMETER;
2033 }
2034
2035 status = RmExportObject(hClient,
2036 pParams->object.data.rmObject.hObject,
2037 &hExportHandle, NULL);
2038 if (status != NV_OK)
2039 {
2040 goto done;
2041 }
2042 NV_ASSERT(hExportHandle != 0);
2043
2044 nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
2045 if (nvfp == NULL)
2046 {
2047 NV_ASSERT(priv == NULL);
2048 status = NV_ERR_INVALID_PARAMETER;
2049 goto done;
2050 }
2051
2052 status = _initializeExportObjectFd(nvfp, hClient,
2053 pParams->object.data.rmObject.hDevice,
2054 1, NULL);
2055 if (status != NV_OK)
2056 {
2057 goto done;
2058 }
2059
2060 nvfp->handles[0] = hExportHandle;
2061
2062 done:
2063
2064 if (status != NV_OK && hExportHandle != 0)
2065 {
2066 RmFreeObjExportHandle(hExportHandle);
2067 }
2068
2069 if (priv != NULL)
2070 {
2071 nv_put_file_private(priv);
2072 }
2073
2074 return status;
2075 }
2076
2077 // This control call has been deprecated. It will be deleted soon.
2078 NV_STATUS
cliresCtrlCmdOsUnixCreateExportObjectFd_IMPL(RmClientResource * pRmCliRes,NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS * pParams)2079 cliresCtrlCmdOsUnixCreateExportObjectFd_IMPL
2080 (
2081 RmClientResource *pRmCliRes,
2082 NV0000_CTRL_OS_UNIX_CREATE_EXPORT_OBJECT_FD_PARAMS *pParams
2083 )
2084 {
2085 NV_STATUS status;
2086 NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
2087 nv_file_private_t *nvfp = NULL;
2088 void *priv = NULL;
2089
2090 ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata));
2091
2092 if (pParams->maxObjects == 0)
2093 {
2094 return NV_ERR_INVALID_PARAMETER;
2095 }
2096
2097 nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
2098 if (nvfp == NULL)
2099 {
2100 NV_ASSERT(priv == NULL);
2101 status = NV_ERR_INVALID_PARAMETER;
2102 goto done;
2103 }
2104
2105 status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice,
2106 pParams->maxObjects, pParams->metadata);
2107
2108 done:
2109 if (priv != NULL)
2110 {
2111 nv_put_file_private(priv);
2112 }
2113
2114 return status;
2115 }
2116
2117 NV_STATUS
cliresCtrlCmdOsUnixExportObjectsToFd_IMPL(RmClientResource * pRmCliRes,NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS * pParams)2118 cliresCtrlCmdOsUnixExportObjectsToFd_IMPL
2119 (
2120 RmClientResource *pRmCliRes,
2121 NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_PARAMS *pParams
2122 )
2123 {
2124 NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
2125 RmObjExportHandle *pExportHandle;
2126 nv_file_private_t *nvfp = NULL;
2127 void *priv = NULL;
2128 NV_STATUS status = NV_OK;
2129 NvU32 i;
2130 NvU32 deviceInstance;
2131 NvU32 result;
2132 NvHandle *exportHandles = NULL;
2133 NvBool bFdSetup = NV_FALSE;
2134
2135 nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
2136 if (nvfp == NULL)
2137 {
2138 NV_ASSERT(priv == NULL);
2139 status = NV_ERR_INVALID_PARAMETER;
2140 goto done;
2141 }
2142
2143 ct_assert(sizeof(nvfp->metadata) == sizeof(pParams->metadata));
2144
2145 /* Setup export FD if not done */
2146 if (nvfp->handles == NULL)
2147 {
2148 if (pParams->maxObjects == 0)
2149 {
2150 status = NV_ERR_INVALID_PARAMETER;
2151 goto done;
2152 }
2153
2154 status = _initializeExportObjectFd(nvfp, hClient, pParams->hDevice,
2155 pParams->maxObjects,
2156 pParams->metadata);
2157 if (status != NV_OK)
2158 {
2159 goto done;
2160 }
2161
2162 bFdSetup = NV_TRUE;
2163 }
2164
2165 if ((nvfp->handles == NULL) ||
2166 (pParams->numObjects >
2167 NV0000_CTRL_OS_UNIX_EXPORT_OBJECTS_TO_FD_MAX_OBJECTS))
2168 {
2169 status = NV_ERR_INVALID_PARAMETER;
2170 goto done;
2171 }
2172
2173 if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) ||
2174 (result > nvfp->maxHandles))
2175 {
2176 status = NV_ERR_OUT_OF_RANGE;
2177 goto done;
2178 }
2179
2180 status = os_alloc_mem((void **)&exportHandles,
2181 sizeof(exportHandles[0]) *
2182 pParams->numObjects);
2183 if (status != NV_OK)
2184 {
2185 goto done;
2186 }
2187
2188 for (i = 0; i < pParams->numObjects; i++)
2189 {
2190 exportHandles[i] = 0;
2191
2192 if (pParams->objects[i] == 0)
2193 {
2194 continue;
2195 }
2196
2197 status = RmExportObject(hClient,
2198 pParams->objects[i],
2199 &exportHandles[i],
2200 &deviceInstance);
2201 if (status != NV_OK)
2202 {
2203 goto done;
2204 }
2205
2206 NV_ASSERT(exportHandles[i] != 0);
2207
2208 if (deviceInstance != nvfp->deviceInstance)
2209 {
2210 status = NV_ERR_INVALID_PARAMETER;
2211 goto done;
2212 }
2213 }
2214
2215 for (i = 0; i < pParams->numObjects; i++)
2216 {
2217 pExportHandle = &nvfp->handles[i + pParams->index];
2218
2219 // If the handle already exists in this position, free it
2220 if (*pExportHandle != 0)
2221 {
2222 RmFreeObjExportHandle(*pExportHandle);
2223 *pExportHandle = 0;
2224 }
2225
2226 *pExportHandle = exportHandles[i];
2227 }
2228
2229 done:
2230
2231 if ((status != NV_OK) && (exportHandles != NULL))
2232 {
2233 for (i = 0; i < pParams->numObjects; i++)
2234 {
2235 if (exportHandles[i] != 0)
2236 {
2237 RmFreeObjExportHandle(exportHandles[i]);
2238 }
2239 }
2240 }
2241
2242 if (exportHandles != NULL)
2243 {
2244 os_free_mem(exportHandles);
2245 }
2246
2247 if ((status != NV_OK) && bFdSetup)
2248 {
2249 os_free_mem(nvfp->handles);
2250 nvfp->handles = NULL;
2251 nvfp->maxHandles = 0;
2252 }
2253
2254 if (priv != NULL)
2255 {
2256 nv_put_file_private(priv);
2257 }
2258
2259 return status;
2260 }
2261
2262 NV_STATUS
cliresCtrlCmdOsUnixImportObjectFromFd_IMPL(RmClientResource * pRmCliRes,NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS * pParams)2263 cliresCtrlCmdOsUnixImportObjectFromFd_IMPL
2264 (
2265 RmClientResource *pRmCliRes,
2266 NV0000_CTRL_OS_UNIX_IMPORT_OBJECT_FROM_FD_PARAMS *pParams
2267 )
2268 {
2269 NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
2270 nv_file_private_t *nvfp = NULL;
2271 void *priv = NULL;
2272 NV_STATUS status = NV_OK;
2273
2274 if (pParams->object.type != NV0000_CTRL_OS_UNIX_EXPORT_OBJECT_TYPE_RM ||
2275 pParams->fd == -1)
2276 {
2277 return NV_ERR_INVALID_PARAMETER;
2278 }
2279
2280 nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
2281 if (nvfp == NULL)
2282 {
2283 NV_ASSERT(priv == NULL);
2284 status = NV_ERR_INVALID_PARAMETER;
2285 goto done;
2286 }
2287
2288 if ((nvfp->handles == NULL) || (nvfp->handles[0] == 0) ||
2289 (nvfp->maxHandles > 1))
2290 {
2291 status = NV_ERR_INVALID_PARAMETER;
2292 goto done;
2293 }
2294
2295 status = RmImportObject(hClient,
2296 pParams->object.data.rmObject.hParent,
2297 &pParams->object.data.rmObject.hObject,
2298 nvfp->handles[0], NULL);
2299
2300 done:
2301 if (priv != NULL)
2302 {
2303 nv_put_file_private(priv);
2304 }
2305
2306 return status;
2307 }
2308
2309 NV_STATUS
cliresCtrlCmdOsUnixImportObjectsFromFd_IMPL(RmClientResource * pRmCliRes,NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS * pParams)2310 cliresCtrlCmdOsUnixImportObjectsFromFd_IMPL
2311 (
2312 RmClientResource *pRmCliRes,
2313 NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_FROM_FD_PARAMS *pParams
2314 )
2315 {
2316 NvHandle hClient = RES_GET_CLIENT_HANDLE(pRmCliRes);
2317 nv_file_private_t *nvfp = NULL;
2318 void *priv = NULL;
2319 NV_STATUS status = NV_OK;
2320 NvU32 i = 0;
2321 RmObjExportHandle hImportHandle;
2322 NvU32 result;
2323 RM_API *pRmApi;
2324
2325 nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
2326 if (nvfp == NULL)
2327 {
2328 NV_ASSERT(priv == NULL);
2329 status = NV_ERR_INVALID_PARAMETER;
2330 goto done;
2331 }
2332
2333 if ((nvfp->handles == NULL) ||
2334 (pParams->numObjects >
2335 NV0000_CTRL_OS_UNIX_IMPORT_OBJECTS_TO_FD_MAX_OBJECTS))
2336 {
2337 status = NV_ERR_INVALID_PARAMETER;
2338 goto done;
2339 }
2340
2341 if (!portSafeAddU32(pParams->numObjects, pParams->index, &result) ||
2342 (result > nvfp->maxHandles))
2343 {
2344 status = NV_ERR_OUT_OF_RANGE;
2345 goto done;
2346 }
2347
2348 for (i = 0; i < pParams->numObjects; i++)
2349 {
2350 hImportHandle = nvfp->handles[i + pParams->index];
2351
2352 /* Nothing to import, just continue */
2353 if (hImportHandle == 0)
2354 {
2355 pParams->objectTypes[i] = \
2356 NV0000_CTRL_CMD_OS_UNIX_IMPORT_OBJECT_TYPE_NONE;
2357 continue;
2358 }
2359
2360 status = RmImportObject(hClient,
2361 pParams->hParent,
2362 &pParams->objects[i],
2363 hImportHandle,
2364 &pParams->objectTypes[i]);
2365 if (status != NV_OK)
2366 {
2367 NV_PRINTF(LEVEL_ERROR, "%s: Unable to import handle (%x, %x, %x)\n",
2368 __FUNCTION__, pParams->hParent, pParams->objects[i], hImportHandle);
2369 goto done;
2370 }
2371 }
2372
2373 done:
2374
2375 if (status != NV_OK)
2376 {
2377 pRmApi = rmapiGetInterface(RMAPI_API_LOCK_INTERNAL);
2378 while (i > 0)
2379 {
2380 i--;
2381
2382 if (pParams->objects[i] != 0)
2383 {
2384 pRmApi->Free(pRmApi, hClient, pParams->objects[i]);
2385 }
2386 }
2387 }
2388
2389 if (priv != NULL)
2390 {
2391 nv_put_file_private(priv);
2392 }
2393
2394 return status;
2395 }
2396
2397 NV_STATUS
cliresCtrlCmdOsUnixGetExportObjectInfo_IMPL(RmClientResource * pRmCliRes,NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS * pParams)2398 cliresCtrlCmdOsUnixGetExportObjectInfo_IMPL
2399 (
2400 RmClientResource *pRmCliRes,
2401 NV0000_CTRL_OS_UNIX_GET_EXPORT_OBJECT_INFO_PARAMS *pParams
2402 )
2403 {
2404 nv_file_private_t *nvfp = NULL;
2405 void *priv = NULL;
2406 NV_STATUS status = NV_OK;
2407
2408 if (pParams->fd < 0)
2409 {
2410 return NV_ERR_INVALID_PARAMETER;
2411 }
2412
2413 nvfp = nv_get_file_private(pParams->fd, NV_TRUE, &priv);
2414 if (nvfp == NULL)
2415 {
2416 NV_ASSERT(priv == NULL);
2417 status = NV_ERR_INVALID_PARAMETER;
2418 goto done;
2419 }
2420
2421 if (nvfp->handles == NULL)
2422 {
2423 status = NV_ERR_INVALID_PARAMETER;
2424 goto done;
2425 }
2426
2427 pParams->maxObjects = nvfp->maxHandles;
2428 pParams->deviceInstance = nvfp->deviceInstance;
2429 pParams->gpuInstanceId = nvfp->gpuInstanceId;
2430
2431 os_mem_copy(pParams->metadata, nvfp->metadata, sizeof(nvfp->metadata));
2432
2433 done:
2434 if (priv != NULL)
2435 {
2436 nv_put_file_private(priv);
2437 }
2438
2439 return status;
2440 }
2441
2442 /*!
2443 * osAcpiDsm
2444 *
2445 * @brief Handles os specific _DSM method function calls.
2446 *
2447 * Input parameters:
2448 * @param[in] pGpu : OBJGPU pointer
2449 * @param[in] acpiDsmFunction : ACPI DSM function
2450 * @param[in] acpiDsmSubFunction : ACPI DSM subfunction
2451 * @param[in/out] pInOut : in/out buffer, caller should make sure the buffer is large enough.
2452 * @param[in] pSize : when input, size of data that the caller wants to read, in bytes.
2453 * when output, size of valid data in pInOuta in bytes.
2454 */
osCallACPI_DSM(OBJGPU * pGpu,ACPI_DSM_FUNCTION acpiDsmFunction,NvU32 acpiDsmSubFunction,NvU32 * pInOut,NvU16 * pSize)2455 NV_STATUS osCallACPI_DSM
2456 (
2457 OBJGPU *pGpu,
2458 ACPI_DSM_FUNCTION acpiDsmFunction,
2459 NvU32 acpiDsmSubFunction,
2460 NvU32 *pInOut,
2461 NvU16 *pSize
2462 )
2463 {
2464 NV_STATUS status;
2465 NvU8 *pAcpiDsmGuid = NULL;
2466 NvU32 acpiDsmRev;
2467 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
2468 nv_priv_t *nvp = NV_GET_NV_PRIV(nv);
2469 NvU16 acpiDsmInArgSize = 4;
2470 NvBool acpiNvpcfDsmFunction = NV_FALSE;
2471
2472 // do any handling/remapping of guid needed.
2473 status = checkDsmCall(pGpu,
2474 (ACPI_DSM_FUNCTION *) &acpiDsmFunction,
2475 &acpiDsmSubFunction,
2476 pInOut,
2477 pSize);
2478
2479 // return if subfunction is not supported or we're returning cache data
2480 if (status != NV_WARN_MORE_PROCESSING_REQUIRED)
2481 {
2482 return status;
2483 }
2484
2485 switch ((NvU32) acpiDsmFunction)
2486 {
2487 case ACPI_DSM_FUNCTION_NBSI:
2488 pAcpiDsmGuid = (NvU8 *) &NBSI_DSM_GUID;
2489 acpiDsmRev = NBSI_REVISION_ID;
2490 break;
2491 case ACPI_DSM_FUNCTION_NVHG:
2492 pAcpiDsmGuid = (NvU8 *) &NVHG_DSM_GUID;
2493 acpiDsmRev = NVHG_REVISION_ID;
2494 break;
2495 case ACPI_DSM_FUNCTION_MXM:
2496 pAcpiDsmGuid = (NvU8 *) &DSM_MXM_GUID;
2497 acpiDsmRev = ACPI_MXM_REVISION_ID;
2498 break;
2499 case ACPI_DSM_FUNCTION_NBCI:
2500 pAcpiDsmGuid = (NvU8 *) &NBCI_DSM_GUID;
2501 acpiDsmRev = NBCI_REVISION_ID;
2502 break;
2503 case ACPI_DSM_FUNCTION_NVOP:
2504 pAcpiDsmGuid = (NvU8 *) &NVOP_DSM_GUID;
2505 acpiDsmRev = NVOP_REVISION_ID;
2506 break;
2507 case ACPI_DSM_FUNCTION_PCFG:
2508 pAcpiDsmGuid = (NvU8 *) &PCFG_DSM_GUID;
2509 acpiDsmRev = PCFG_REVISION_ID;
2510 break;
2511 case ACPI_DSM_FUNCTION_GPS_2X:
2512 pAcpiDsmGuid = (NvU8 *) &GPS_DSM_GUID;
2513 acpiDsmRev = GPS_2X_REVISION_ID;
2514 acpiDsmInArgSize = (*pSize);
2515 break;
2516 case ACPI_DSM_FUNCTION_GPS:
2517 if ((IsTU10X(pGpu)) ||
2518 ((gpuIsACPIPatchRequiredForBug2473619_HAL(pGpu)) &&
2519 ((acpiDsmSubFunction == GPS_FUNC_SUPPORT) ||
2520 (acpiDsmSubFunction == GPS_FUNC_GETCALLBACKS))))
2521 {
2522 pAcpiDsmGuid = (NvU8 *) &GPS_DSM_GUID;
2523 acpiDsmRev = GPS_REVISION_ID;
2524 acpiDsmInArgSize = (*pSize);
2525 }
2526 else
2527 {
2528 return NV_ERR_NOT_SUPPORTED;
2529 }
2530 break;
2531 case ACPI_DSM_FUNCTION_PEX:
2532 pAcpiDsmGuid = (NvU8 *) &PEX_DSM_GUID;
2533 acpiDsmRev = PEX_REVISION_ID;
2534 if (acpiDsmSubFunction == PEX_FUNC_SETLTRLATENCY)
2535 {
2536 acpiDsmInArgSize = (3 + *pSize);
2537 }
2538 break;
2539 case (ACPI_DSM_FUNCTION_JT):
2540 pAcpiDsmGuid = (NvU8 *) &JT_DSM_GUID;
2541 acpiDsmRev = JT_REVISION_ID;
2542 break;
2543 case ACPI_DSM_FUNCTION_NVPCF:
2544 {
2545 pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID;
2546 acpiDsmRev = NVPCF_ACPI_DSM_REVISION_ID;
2547 acpiDsmInArgSize = (*pSize);
2548 acpiNvpcfDsmFunction = NV_TRUE;
2549 break;
2550 }
2551 case ACPI_DSM_FUNCTION_NVPCF_2X:
2552 pAcpiDsmGuid = (NvU8 *)&NVPCF_ACPI_DSM_GUID;
2553 acpiDsmRev = NVPCF_2X_ACPI_DSM_REVISION_ID;
2554 acpiDsmInArgSize = (*pSize);
2555 if (!nv->nvpcf_dsm_in_gpu_scope)
2556 {
2557 acpiNvpcfDsmFunction = NV_TRUE;
2558 }
2559 break;
2560
2561 default:
2562 return NV_ERR_NOT_SUPPORTED;
2563 break;
2564 }
2565
2566 status = nv_acpi_dsm_method(nv,
2567 pAcpiDsmGuid,
2568 acpiDsmRev,
2569 acpiNvpcfDsmFunction,
2570 acpiDsmSubFunction,
2571 pInOut,
2572 acpiDsmInArgSize,
2573 NULL,
2574 pInOut,
2575 pSize);
2576
2577 if (status == NV_OK)
2578 {
2579 if (acpiDsmSubFunction == NV_ACPI_ALL_FUNC_SUPPORT)
2580 {
2581 // if handling get supported functions list... cache it for later calls
2582 cacheDsmSupportedFunction(pGpu, acpiDsmFunction, acpiDsmSubFunction, pInOut, *pSize);
2583 }
2584 }
2585 else if (nvp->b_mobile_config_enabled)
2586 {
2587 NV_PRINTF(LEVEL_ERROR,
2588 "osCallACPI_DSM: Error during 0x%x DSM subfunction 0x%x! status=0x%x\n",
2589 acpiDsmFunction, acpiDsmSubFunction, status);
2590 }
2591
2592 return status;
2593 }
2594
osCallACPI_DOD(OBJGPU * pGpu,NvU32 * pOut,NvU32 * pSize)2595 NV_STATUS osCallACPI_DOD
2596 (
2597 OBJGPU *pGpu,
2598 NvU32 *pOut,
2599 NvU32 *pSize
2600 )
2601 {
2602 NV_STATUS rmStatus;
2603 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
2604
2605 if ((pOut == NULL) || (pSize == NULL))
2606 {
2607 return NV_ERR_INVALID_POINTER;
2608 }
2609
2610 rmStatus = nv_acpi_dod_method(nv, pOut, pSize);
2611
2612 return rmStatus;
2613 }
2614
2615 //
2616 // osAcpiDdc
2617 //
2618 // Handles os specific _DDC method function calls. _DDC is to get EDID from SBIOS.
2619 //
osCallACPI_DDC(OBJGPU * pGpu,NvU32 ulAcpiId,NvU8 * pOutData,NvU32 * pOutSize,NvBool bReadMultiBlock)2620 NV_STATUS osCallACPI_DDC
2621 (
2622 OBJGPU *pGpu,
2623 NvU32 ulAcpiId,
2624 NvU8 *pOutData,
2625 NvU32 *pOutSize,
2626 NvBool bReadMultiBlock
2627 )
2628 {
2629 NV_STATUS rmStatus;
2630
2631 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
2632
2633 if ((pOutData == NULL) || (pOutSize == NULL))
2634 {
2635 return NV_ERR_INVALID_POINTER;
2636 }
2637
2638 portMemSet(pOutData, 0, *pOutSize);
2639
2640 rmStatus = nv_acpi_ddc_method(nv, pOutData, pOutSize, bReadMultiBlock);
2641
2642 return rmStatus;
2643 }
2644
2645 // osCallACPI_NVHG_ROM
2646 // Making ACPI Call into SBIOS with ROM method to get display device's ROM data.
2647 //
osCallACPI_NVHG_ROM(OBJGPU * pGpu,NvU32 * pInData,NvU32 * pOutData)2648 NV_STATUS osCallACPI_NVHG_ROM
2649 (
2650 OBJGPU *pGpu,
2651 NvU32 *pInData,
2652 NvU32 *pOutData
2653 )
2654 {
2655 NV_STATUS rmStatus;
2656 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
2657
2658 if ((pOutData == NULL) || (pInData == NULL))
2659 {
2660 return NV_ERR_INVALID_POINTER;
2661 }
2662
2663 if (pInData[1] > ROM_METHOD_MAX_RETURN_BUFFER_SIZE)
2664 {
2665 return NV_ERR_INVALID_ARGUMENT;
2666 }
2667
2668 rmStatus = nv_acpi_rom_method(nv, pInData, pOutData);
2669
2670 return rmStatus;
2671 }
2672
osInitSystemStaticConfig(SYS_STATIC_CONFIG * pConfig)2673 void osInitSystemStaticConfig(SYS_STATIC_CONFIG *pConfig)
2674 {
2675 pConfig->bIsNotebook = rm_is_system_notebook();
2676 pConfig->osType = nv_get_os_type();
2677 pConfig->bOsCCEnabled = os_cc_enabled;
2678 pConfig->bOsCCTdxEnabled = os_cc_tdx_enabled;
2679 }
2680
osApiLockAcquireConfigureFlags(NvU32 flags)2681 NvU32 osApiLockAcquireConfigureFlags(NvU32 flags)
2682 {
2683 return flags;
2684 }
2685
osGpuLocksQueueRelease(OBJGPU * pGpu,NvU32 dpcGpuLocksRelease)2686 NV_STATUS osGpuLocksQueueRelease(OBJGPU *pGpu, NvU32 dpcGpuLocksRelease)
2687 {
2688 return NV_SEMA_RELEASE_FAILED;
2689 }
2690
osSyncWithRmDestroy(void)2691 void osSyncWithRmDestroy(void)
2692 {
2693 }
2694
osSyncWithGpuDestroy(NvBool bEntry)2695 void osSyncWithGpuDestroy(NvBool bEntry)
2696 {
2697 }
2698
osModifyGpuSwStatePersistence(OS_GPU_INFO * pOsGpuInfo,NvBool bEnable)2699 void osModifyGpuSwStatePersistence
2700 (
2701 OS_GPU_INFO *pOsGpuInfo,
2702 NvBool bEnable
2703 )
2704 {
2705 if (bEnable)
2706 {
2707 pOsGpuInfo->flags |= NV_FLAG_PERSISTENT_SW_STATE;
2708 }
2709 else
2710 {
2711 pOsGpuInfo->flags &= ~NV_FLAG_PERSISTENT_SW_STATE;
2712 }
2713 }
2714
2715 NV_STATUS
osSystemGetBatteryDrain(NvS32 * pChargeRate)2716 osSystemGetBatteryDrain(NvS32 *pChargeRate)
2717 {
2718 NV_PRINTF(LEVEL_WARNING, "%s: Platform not supported!\n", __FUNCTION__);
2719 return NV_ERR_NOT_SUPPORTED;
2720 }
2721
2722 NV_STATUS
osPexRecoveryCallback(OS_GPU_INFO * pOsGpuInfo,OS_PEX_RECOVERY_STATUS Status)2723 osPexRecoveryCallback
2724 (
2725 OS_GPU_INFO *pOsGpuInfo,
2726 OS_PEX_RECOVERY_STATUS Status
2727 )
2728 {
2729 NV_ASSERT_FAILED("Not supported");
2730 return NV_ERR_NOT_SUPPORTED;
2731 }
2732
2733 //
2734 //osCallACPI_MXDS
2735 //
2736 //Handles OS specific MXDS function call.
2737 //
osCallACPI_MXDS(OBJGPU * pGpu,NvU32 acpiId,NvU32 * pInOut)2738 NV_STATUS osCallACPI_MXDS
2739 (
2740 OBJGPU *pGpu,
2741 NvU32 acpiId,
2742 NvU32 *pInOut
2743 )
2744 {
2745 NV_STATUS rmStatus;
2746
2747 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
2748
2749 if (pInOut == NULL)
2750 {
2751 return NV_ERR_INVALID_POINTER;
2752 }
2753
2754 rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDS");
2755
2756 return rmStatus;
2757 }
2758
2759 //
2760 //osCallACPI_MXDM
2761 //
2762 //Handles OS specific MXDM function call.
2763 //
osCallACPI_MXDM(OBJGPU * pGpu,NvU32 acpiId,NvU32 * pInOut)2764 NV_STATUS osCallACPI_MXDM
2765 (
2766 OBJGPU *pGpu,
2767 NvU32 acpiId,
2768 NvU32 *pInOut
2769 )
2770 {
2771 NV_STATUS rmStatus;
2772
2773 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
2774
2775 if (pInOut == NULL)
2776 {
2777 return NV_ERR_INVALID_POINTER;
2778 }
2779
2780 rmStatus = nv_acpi_mux_method(nv, pInOut, acpiId, "MXDM");
2781
2782 return rmStatus;
2783 }
2784
2785 #include "lib/protobuf/prb.h"
2786 #include "lib/protobuf/prb_util.h"
2787 #include "g_nvdebug_pb.h"
2788
osGetVersionDump(void * pVoid)2789 NV_STATUS osGetVersionDump(void * pVoid)
2790 {
2791 PRB_ENCODER * pPrbEnc = (PRB_ENCODER *)pVoid;
2792 NV_STATUS rmStatus;
2793 os_version_info * pOsVersionInfo = NULL;
2794 const char NV_UNKNOWN_BUILD_VERSION[] = "Unknown build version";
2795 const char NV_UNKNOWN_BUILD_DATE[] = "Unknown build date";
2796
2797 NV_ASSERT_OK_OR_RETURN(os_alloc_mem((void**)&pOsVersionInfo,
2798 sizeof(os_version_info)));
2799 portMemSet(pOsVersionInfo, 0, sizeof(os_version_info));
2800
2801 prbEncAddUInt32(pPrbEnc,
2802 NVDEBUG_SYSTEMINFO_OSINFO_FAMILY,
2803 NVDEBUG_OS_UNIX);
2804
2805 rmStatus = os_get_version_info(pOsVersionInfo);
2806 if (rmStatus != NV_OK)
2807 {
2808 goto cleanup;
2809 }
2810
2811 prbEncAddUInt32(pPrbEnc,
2812 NVDEBUG_SYSTEMINFO_OSINFO_OSMAJORVERSION,
2813 pOsVersionInfo->os_major_version);
2814
2815 prbEncAddUInt32(pPrbEnc,
2816 NVDEBUG_SYSTEMINFO_OSINFO_OSMINORVERSION,
2817 pOsVersionInfo->os_minor_version);
2818
2819 prbEncAddUInt32(pPrbEnc,
2820 NVDEBUG_SYSTEMINFO_OSINFO_OSBLDNUM,
2821 pOsVersionInfo->os_build_number);
2822
2823 if (NULL == pOsVersionInfo->os_build_version_str)
2824 {
2825 pOsVersionInfo->os_build_version_str = NV_UNKNOWN_BUILD_VERSION;
2826 }
2827
2828 prbEncAddString(pPrbEnc,
2829 NVDEBUG_SYSTEMINFO_OSINFO_BUILDVERSION,
2830 pOsVersionInfo->os_build_version_str);
2831
2832 if (NULL == pOsVersionInfo->os_build_date_plus_str)
2833 {
2834 pOsVersionInfo->os_build_date_plus_str = NV_UNKNOWN_BUILD_DATE;
2835 }
2836
2837 prbEncAddString(pPrbEnc,
2838 NVDEBUG_SYSTEMINFO_OSINFO_BUILDDATEPLUS,
2839 pOsVersionInfo->os_build_date_plus_str);
2840
2841 cleanup:
2842 os_free_mem(pOsVersionInfo);
2843 return rmStatus;
2844 }
2845
osGetVersion(NvU32 * majorVer,NvU32 * minorVer,NvU32 * buildNum,NvU16 * unusedPatchVersion,NvU16 * unusedProductType)2846 NV_STATUS osGetVersion(NvU32 *majorVer, NvU32 *minorVer, NvU32 *buildNum, NvU16 *unusedPatchVersion, NvU16 *unusedProductType)
2847 {
2848 os_version_info osVersionInfo;
2849 NV_STATUS rmStatus;
2850
2851 portMemSet(&osVersionInfo, 0, sizeof(osVersionInfo));
2852
2853 rmStatus = os_get_version_info(&osVersionInfo);
2854 if (rmStatus == NV_OK)
2855 {
2856 if (majorVer)
2857 *majorVer = osVersionInfo.os_major_version;
2858 if (minorVer)
2859 *minorVer = osVersionInfo.os_minor_version;
2860 if (buildNum)
2861 *buildNum = osVersionInfo.os_build_number;
2862 }
2863
2864 return rmStatus;
2865 }
2866
2867 NV_STATUS
osGetSystemCpuLogicalCoreCounts(NvU32 * pCpuCoreCount)2868 osGetSystemCpuLogicalCoreCounts
2869 (
2870 NvU32 *pCpuCoreCount
2871 )
2872 {
2873 return NV_ERR_NOT_SUPPORTED;
2874 }
2875
2876 NV_STATUS
osGetSystemCpuC0AndAPerfCounters(NvU32 coreIndex,POS_CPU_CORE_PERF_COUNTERS pCpuPerfData)2877 osGetSystemCpuC0AndAPerfCounters
2878 (
2879 NvU32 coreIndex,
2880 POS_CPU_CORE_PERF_COUNTERS pCpuPerfData
2881 )
2882 {
2883 return NV_ERR_NOT_SUPPORTED;
2884 }
2885
2886 void
osEnableCpuPerformanceCounters(OBJOS * pOS)2887 osEnableCpuPerformanceCounters
2888 (
2889 OBJOS *pOS
2890 )
2891 {
2892 NV_ASSERT_FAILED("Not supported");
2893 return;
2894 }
2895
2896 NV_STATUS
osCpuDpcObjInit(void ** ppCpuDpcObj,OBJGPU * pGpu,NvU32 coreCount)2897 osCpuDpcObjInit
2898 (
2899 void **ppCpuDpcObj,
2900 OBJGPU *pGpu,
2901 NvU32 coreCount
2902 )
2903 {
2904 NV_ASSERT_FAILED("Not supported");
2905 return NV_ERR_NOT_SUPPORTED;
2906 }
2907
2908 void
osCpuDpcObjQueue(void ** ppCpuDpcObj,NvU32 coreCount,POS_CPU_CORE_PERF_COUNTERS pCpuPerfData)2909 osCpuDpcObjQueue
2910 (
2911 void **ppCpuDpcObj,
2912 NvU32 coreCount,
2913 POS_CPU_CORE_PERF_COUNTERS pCpuPerfData
2914 )
2915 {
2916 NV_ASSERT_FAILED("Not supported");
2917 }
2918
2919 void
osCpuDpcObjFree(void ** ppCpuDpcObj)2920 osCpuDpcObjFree
2921 (
2922 void **ppCpuDpcObj
2923 )
2924 {
2925 NV_ASSERT_FAILED("Not supported");
2926 }
2927
2928 NV_STATUS
osGetCarveoutInfo(NvU64 * pAddr,NvU64 * pSize)2929 osGetCarveoutInfo
2930 (
2931 NvU64 *pAddr,
2932 NvU64 *pSize
2933 )
2934 {
2935 return NV_ERR_NOT_SUPPORTED;
2936 }
2937
2938 NV_STATUS
osGetVPRInfo(NvU64 * pAddr,NvU64 * pSize)2939 osGetVPRInfo
2940 (
2941 NvU64 *pAddr,
2942 NvU64 *pSize
2943 )
2944 {
2945 return NV_ERR_NOT_SUPPORTED;
2946 }
2947
2948 NV_STATUS
osAllocInVPR(MEMORY_DESCRIPTOR * pMemDesc)2949 osAllocInVPR
2950 (
2951 MEMORY_DESCRIPTOR *pMemDesc
2952 )
2953 {
2954 return NV_ERR_NOT_SUPPORTED;
2955 }
2956
2957 NV_STATUS
osGetGenCarveout(NvU64 * pAddr,NvU64 * pSize,NvU32 id,NvU64 align)2958 osGetGenCarveout
2959 (
2960 NvU64 *pAddr,
2961 NvU64 *pSize,
2962 NvU32 id,
2963 NvU64 align
2964 )
2965 {
2966 return NV_ERR_NOT_SUPPORTED;
2967 }
2968
2969 NV_STATUS
osI2CClosePorts(OS_GPU_INFO * pOsGpuInfo,NvU32 numPorts)2970 osI2CClosePorts
2971 (
2972 OS_GPU_INFO *pOsGpuInfo,
2973 NvU32 numPorts
2974 )
2975 {
2976 return NV_ERR_NOT_SUPPORTED;
2977 }
2978
2979 NV_STATUS
osI2CTransfer(OBJGPU * pGpu,NvU32 Port,NvU8 Address,nv_i2c_msg_t * nv_i2c_msgs,NvU32 count)2980 osI2CTransfer
2981 (
2982 OBJGPU *pGpu,
2983 NvU32 Port,
2984 NvU8 Address,
2985 nv_i2c_msg_t *nv_i2c_msgs,
2986 NvU32 count
2987 )
2988 {
2989 return NV_ERR_NOT_SUPPORTED;
2990 }
2991
2992 NV_STATUS
osTegraI2CGetBusState(OS_GPU_INFO * pOsGpuInfo,NvU32 port,NvS32 * scl,NvS32 * sda)2993 osTegraI2CGetBusState
2994 (
2995 OS_GPU_INFO *pOsGpuInfo,
2996 NvU32 port,
2997 NvS32 *scl,
2998 NvS32 *sda
2999 )
3000 {
3001 return NV_ERR_NOT_SUPPORTED;
3002 }
3003
3004 NV_STATUS
osReadI2CBufferDirect(OBJGPU * pGpu,NvU32 Port,NvU8 Address,void * pOutputBuffer,NvU32 OutputSize,void * pInputBuffer,NvU32 InputSize)3005 osReadI2CBufferDirect
3006 (
3007 OBJGPU *pGpu,
3008 NvU32 Port,
3009 NvU8 Address,
3010 void *pOutputBuffer,
3011 NvU32 OutputSize,
3012 void *pInputBuffer,
3013 NvU32 InputSize
3014 )
3015 {
3016 return NV_ERR_NOT_SUPPORTED;
3017 }
3018
3019 NV_STATUS
osWriteI2CBufferDirect(OBJGPU * pGpu,NvU32 Port,NvU8 Address,void * pOutputBuffer0,NvU32 OutputSize0,void * pOutputBuffer1,NvU32 OutputSize1)3020 osWriteI2CBufferDirect
3021 (
3022 OBJGPU *pGpu,
3023 NvU32 Port,
3024 NvU8 Address,
3025 void *pOutputBuffer0,
3026 NvU32 OutputSize0,
3027 void *pOutputBuffer1,
3028 NvU32 OutputSize1
3029 )
3030 {
3031 return NV_ERR_NOT_SUPPORTED;
3032 }
3033
3034 NV_STATUS
osGC6PowerControl(OBJGPU * pGpu,NvU32 cmd,NvU32 * pOut)3035 osGC6PowerControl
3036 (
3037 OBJGPU *pGpu,
3038 NvU32 cmd,
3039 NvU32 *pOut
3040 )
3041 {
3042 NV_STATUS status;
3043 NvU32 inOut = cmd;
3044 NvU16 rtnSize = sizeof(inOut);
3045
3046 if (FLD_TEST_DRF(_JT_FUNC, _POWERCONTROL, _GPU_POWER_CONTROL, _GSS, inOut))
3047 {
3048 if (!pOut)
3049 {
3050 return NV_ERR_INVALID_ARGUMENT;
3051 }
3052 }
3053
3054 status = osCallACPI_DSM(pGpu,
3055 ACPI_DSM_FUNCTION_JT,
3056 JT_FUNC_POWERCONTROL,
3057 &inOut,
3058 &rtnSize);
3059
3060 if ((status != NV_OK) || !pOut)
3061 {
3062 return status;
3063 }
3064
3065 *pOut = inOut;
3066
3067 return NV_OK;
3068 }
3069
osTestPcieExtendedConfigAccess(void * handle,NvU32 offset)3070 NvBool osTestPcieExtendedConfigAccess(void *handle, NvU32 offset)
3071 {
3072 OBJGPU *pGpu;
3073 KernelBif *pKernelBif;
3074 NvU32 nvXveId = 0;
3075 NvU32 nvXveVccapHdr = 0;
3076 NvU32 pciStart = 0;
3077 NvU32 pcieStart = 0;
3078
3079 static NvBool retryAllowed = NV_TRUE;
3080 static NvBool configAccess = NV_FALSE;
3081
3082 //
3083 // Return early for offset within PCI space
3084 // and does not requires extended config space access
3085 //
3086 if (offset < 0x100)
3087 {
3088 return NV_TRUE;
3089 }
3090
3091 if (!retryAllowed)
3092 {
3093 return configAccess;
3094 }
3095
3096 pGpu = gpumgrGetSomeGpu();
3097 if (pGpu == NULL)
3098 {
3099 return configAccess;
3100 }
3101
3102 retryAllowed = NV_FALSE;
3103
3104 pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
3105 if (pKernelBif == NULL || kbifGetBusIntfType_HAL(pKernelBif) !=
3106 NV2080_CTRL_BUS_INFO_TYPE_PCI_EXPRESS)
3107 {
3108 return configAccess;
3109 }
3110
3111 // Now verify PCI and PCIe config start registers.
3112 kbifGetPcieConfigAccessTestRegisters_HAL(pGpu, pKernelBif, &pciStart, &pcieStart);
3113 os_pci_read_dword(handle, pciStart, &nvXveId);
3114 os_pci_read_dword(handle, pcieStart, &nvXveVccapHdr);
3115
3116 if (NV_OK == kbifVerifyPcieConfigAccessTestRegisters_HAL(pGpu,
3117 pKernelBif,
3118 nvXveId,
3119 nvXveVccapHdr))
3120 {
3121 configAccess = NV_TRUE;
3122 }
3123
3124 return configAccess;
3125 }
3126
3127 /*!
3128 * @brief Map memory into an IOVA space according to the given mapping info.
3129 *
3130 * @param[in] pIovaMapping IOVA mapping info
3131 *
3132 * @return NV_ERR_NOT_SUPPORTED
3133 */
3134 NV_STATUS
osIovaMap(PIOVAMAPPING pIovaMapping)3135 osIovaMap
3136 (
3137 PIOVAMAPPING pIovaMapping
3138 )
3139 {
3140 OBJGPU *pGpu;
3141 nv_state_t *nv, *peer;
3142 NV_STATUS status;
3143 RmPhysAddr base;
3144 NvBool bIsBar0;
3145 PMEMORY_DESCRIPTOR pRootMemDesc;
3146 NvBool bIsFbOffset = NV_FALSE;
3147 NvBool bIsIndirectPeerMapping = NV_FALSE;
3148 NvBool bIsContig;
3149 NV_ADDRESS_SPACE addressSpace;
3150 NvU32 osPageCount;
3151
3152 if (pIovaMapping == NULL)
3153 {
3154 return NV_ERR_INVALID_ARGUMENT;
3155 }
3156
3157 pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId);
3158 if (pGpu == NULL)
3159 {
3160 return NV_ERR_INVALID_ARGUMENT;
3161 }
3162
3163 pRootMemDesc = memdescGetRootMemDesc(pIovaMapping->pPhysMemDesc, NULL);
3164 addressSpace = memdescGetAddressSpace(pIovaMapping->pPhysMemDesc);
3165 if (gpumgrCheckIndirectPeer(pGpu, pRootMemDesc->pGpu) &&
3166 (addressSpace == ADDR_FBMEM))
3167 {
3168 bIsIndirectPeerMapping = NV_TRUE;
3169 }
3170
3171 if ((addressSpace != ADDR_SYSMEM) && !bIsIndirectPeerMapping)
3172 {
3173 NV_PRINTF(LEVEL_INFO,
3174 "%s passed memory descriptor in an unsupported address space (%s)\n",
3175 __FUNCTION__,
3176 memdescGetApertureString(memdescGetAddressSpace(pIovaMapping->pPhysMemDesc)));
3177 return NV_ERR_NOT_SUPPORTED;
3178 }
3179
3180 //
3181 // For guest-allocated memory, we don't actually want to do any remapping,
3182 // since the physical address is already the DMA address to be used by the
3183 // GPU.
3184 //
3185 if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
3186 {
3187 return NV_OK;
3188 }
3189
3190 nv = NV_GET_NV_STATE(pGpu);
3191
3192 //
3193 // Intercept peer IO type memory. These are contiguous allocations, so no
3194 // need to adjust pages.
3195 //
3196 if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
3197 {
3198 NV_ASSERT(memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU));
3199
3200 status = nv_dma_map_mmio(nv->dma_dev,
3201 NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
3202 &pIovaMapping->iovaArray[0]);
3203
3204 if (status != NV_OK)
3205 {
3206 NV_PRINTF(LEVEL_ERROR,
3207 "%s: failed to map peer IO mem (status = 0x%x)\n",
3208 __FUNCTION__, status);
3209 }
3210
3211 return status;
3212 }
3213
3214 //
3215 // We need to check against the "root" GPU, e.g., the GPU that owns this
3216 // allocation. If we're trying to map one of its BARs for a peer, we need
3217 // to handle it differently because it wouldn't have gone through our system
3218 // memory page allocation paths, obviously, and wouldn't have alloc private
3219 // data associated with it.
3220 //
3221 peer = NV_GET_NV_STATE(pRootMemDesc->pGpu);
3222 bIsContig = memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU);
3223 if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig)
3224 {
3225 RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0],
3226 pIovaMapping->pPhysMemDesc->PageCount);
3227 }
3228
3229 base = memdescGetPhysAddr(pIovaMapping->pPhysMemDesc, AT_CPU, 0);
3230 bIsBar0 = IS_REG_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size);
3231
3232 bIsFbOffset = IS_FB_OFFSET(peer, base, pIovaMapping->pPhysMemDesc->Size);
3233
3234 //
3235 // For indirect peers bIsFbOffset should be NV_TRUE
3236 // TODO:IS_FB_OFFSET macro is currently broken for P9 systems
3237 // Bug 2010857 tracks fixing this
3238 //
3239 #if defined(NVCPU_PPC64LE)
3240 KernelMemorySystem *pRootKernelMemorySystem = GPU_GET_KERNEL_MEMORY_SYSTEM(pRootMemDesc->pGpu);
3241 if (bIsIndirectPeerMapping)
3242 {
3243 NvU64 atsBase = base + pRootKernelMemorySystem->coherentCpuFbBase;
3244 if ((atsBase >= pRootKernelMemorySystem->coherentCpuFbBase) &&
3245 (atsBase + pIovaMapping->pPhysMemDesc->Size <=
3246 pRootKernelMemorySystem->coherentCpuFbEnd))
3247 {
3248 bIsFbOffset = NV_TRUE;
3249 }
3250 else
3251 {
3252 NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_STATE);
3253 }
3254 }
3255 #endif
3256
3257 void *pPriv = memdescGetMemData(pIovaMapping->pPhysMemDesc);
3258 osPageCount = NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount);
3259
3260 if (!bIsBar0 && !bIsFbOffset)
3261 {
3262 if (pPriv == NULL)
3263 {
3264 return NV_ERR_INVALID_STATE;
3265 }
3266 }
3267 else if(bIsIndirectPeerMapping)
3268 {
3269 NV_ASSERT(!bIsBar0 && bIsFbOffset);
3270 //
3271 // TODO: Align onlined GPU memory allocation paths with system memory allocation
3272 // That way pMemDesc->pMemData is setup correctly when we try to create mapping
3273 // to onlined memory of indirect peer. After that we can also get rid of some
3274 // extra code in nv_dma_map_alloc. See bug 190324 for details
3275 //
3276
3277 status = memdescGetNvLinkGpa(pRootMemDesc->pGpu, (bIsContig ? 1 : osPageCount),
3278 &pIovaMapping->iovaArray[0]);
3279 if (status != NV_OK)
3280 {
3281 NV_PRINTF(LEVEL_ERROR, "%s Failed to get SPA\n", __FUNCTION__);
3282 return status;
3283 }
3284 }
3285
3286 if (!bIsBar0 && (!bIsFbOffset || bIsIndirectPeerMapping))
3287 {
3288 status = nv_dma_map_alloc(
3289 osGetDmaDeviceForMemDesc(nv, pIovaMapping->pPhysMemDesc),
3290 osPageCount,
3291 &pIovaMapping->iovaArray[0],
3292 bIsContig, &pPriv);
3293 if (status != NV_OK)
3294 {
3295 NV_PRINTF(LEVEL_ERROR,
3296 "%s: failed to map allocation (status = 0x%x)\n",
3297 __FUNCTION__, status);
3298 return status;
3299 }
3300
3301 pIovaMapping->pOsData = pPriv;
3302 }
3303 else if (peer != nv)
3304 {
3305 status = nv_dma_map_peer(nv->dma_dev, peer->dma_dev, bIsBar0 ? 0 : 1,
3306 osPageCount, &pIovaMapping->iovaArray[0]);
3307 if (status != NV_OK)
3308 {
3309 NV_PRINTF(LEVEL_INFO,
3310 "%s: failed to map peer (base = 0x%llx, status = 0x%x)\n",
3311 __FUNCTION__, base, status);
3312 return status;
3313 }
3314
3315 //
3316 // pOsData must be NULL to distinguish a peer DMA mapping from a
3317 // system memory mapping in osIovaUnmap(), so make sure to set it
3318 // accordingly here.
3319 //
3320 pIovaMapping->pOsData = NULL;
3321 }
3322 else
3323 {
3324 NV_PRINTF(LEVEL_INFO, "cannot map a GPU's BAR to itself\n");
3325 return NV_ERR_NOT_SUPPORTED;
3326 }
3327
3328 //
3329 // If the OS layer doesn't think in RM page size, we need to inflate the
3330 // PTE array into RM pages.
3331 //
3332 if (NV_RM_PAGE_SIZE < os_page_size && !bIsContig)
3333 {
3334 RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0],
3335 pIovaMapping->pPhysMemDesc->PageCount);
3336 }
3337
3338 return NV_OK;
3339 }
3340
3341 /*!
3342 * @brief Unmap memory from an IOVA space according to the given mapping info.
3343 *
3344 * This mapping info must have been previously mapped by osIovaMap().
3345 *
3346 * @param[in] pIovaMapping IOVA mapping info
3347 *
3348 */
3349 void
osIovaUnmap(PIOVAMAPPING pIovaMapping)3350 osIovaUnmap
3351 (
3352 PIOVAMAPPING pIovaMapping
3353 )
3354 {
3355 OBJGPU *pGpu;
3356 nv_state_t *nv;
3357 void *pPriv;
3358 NV_STATUS status;
3359
3360 if (pIovaMapping == NULL)
3361 {
3362 return;
3363 }
3364
3365 pGpu = gpumgrGetGpuFromId(pIovaMapping->iovaspaceId);
3366 if (pGpu == NULL)
3367 {
3368 return;
3369 }
3370
3371 //
3372 // For guest-allocated memory, we never actually remapped the memory, so we
3373 // shouldn't try to unmap it here.
3374 //
3375 if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_GUEST_ALLOCATED))
3376 {
3377 return;
3378 }
3379
3380 nv = NV_GET_NV_STATE(pGpu);
3381
3382 if (memdescGetFlag(pIovaMapping->pPhysMemDesc, MEMDESC_FLAGS_PEER_IO_MEM))
3383 {
3384 nv_dma_unmap_mmio(nv->dma_dev,
3385 NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
3386 pIovaMapping->iovaArray[0]);
3387
3388 return;
3389 }
3390
3391 //
3392 // TODO: Formalize the interface with the OS layers so we can use a common
3393 // definition of OS_IOVA_MAPPING_DATA.
3394 //
3395 pPriv = (void *)pIovaMapping->pOsData;
3396
3397 if (NV_RM_PAGE_SIZE < os_page_size &&
3398 !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU))
3399 {
3400 RmDeflateRmToOsPageArray(&pIovaMapping->iovaArray[0],
3401 pIovaMapping->pPhysMemDesc->PageCount);
3402 }
3403
3404 if (pPriv != NULL)
3405 {
3406 status = nv_dma_unmap_alloc(nv->dma_dev,
3407 NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
3408 &pIovaMapping->iovaArray[0], &pPriv);
3409 if (status != NV_OK)
3410 {
3411 NV_PRINTF(LEVEL_ERROR,
3412 "%s: failed to unmap allocation (status = 0x%x)\n",
3413 __FUNCTION__, status);
3414 }
3415 }
3416 else
3417 {
3418 nv_dma_unmap_peer(nv->dma_dev,
3419 NV_RM_PAGES_TO_OS_PAGES(pIovaMapping->pPhysMemDesc->PageCount),
3420 pIovaMapping->iovaArray[0]);
3421 }
3422
3423 //
3424 // If the OS layer doesn't think in RM page size, we need to fluff out the
3425 // PTE array into RM pages.
3426 //
3427 if (NV_RM_PAGE_SIZE < os_page_size &&
3428 !memdescGetContiguity(pIovaMapping->pPhysMemDesc, AT_CPU))
3429 {
3430 RmInflateOsToRmPageArray(&pIovaMapping->iovaArray[0],
3431 pIovaMapping->pPhysMemDesc->PageCount);
3432 }
3433
3434 pIovaMapping->pOsData = NULL;
3435 }
3436
3437 /*!
3438 * @brief Set the GPU Rail Voltage in Tegra SoC. Currently not supported
3439 *
3440 * @param[in] pGpu GPU object pointer
3441 * @param[in] reqVoltageuV Rail Voltage requested in uV
3442 * @param[out] pSetVoltageuV Rail Voltage set in uV
3443 *
3444 * @return NV_ERR_NOT_SUPPORTED
3445 */
3446 NV_STATUS
osSetGpuRailVoltage(OBJGPU * pGpu,NvU32 reqVoltageuV,NvU32 * pSetVoltageuV)3447 osSetGpuRailVoltage
3448 (
3449 OBJGPU *pGpu,
3450 NvU32 reqVoltageuV,
3451 NvU32 *pSetVoltageuV
3452 )
3453 {
3454 return NV_ERR_NOT_SUPPORTED;
3455 }
3456
3457 /*!
3458 * @brief Get the GPU Rail Voltage in Tegra SoC. Currently not supported
3459 *
3460 * @param[in] pGpu GPU object pointer
3461 * @param[out] voltageuV Rail Voltage in uV
3462 *
3463 * @return NV_ERR_NOT_SUPPORTED
3464 */
3465 NV_STATUS
osGetGpuRailVoltage(OBJGPU * pGpu,NvU32 * pVoltageuV)3466 osGetGpuRailVoltage
3467 (
3468 OBJGPU *pGpu,
3469 NvU32 *pVoltageuV
3470 )
3471 {
3472 return NV_ERR_NOT_SUPPORTED;
3473 }
3474
3475 /*!
3476 * @brief Bring down system in a controlled manner on known error conditions.
3477 *
3478 * @bugCode[in] Error code / reason.
3479 */
osBugCheck(NvU32 bugCode)3480 void osBugCheck(NvU32 bugCode)
3481 {
3482 if (bugCode > OS_BUG_CHECK_BUGCODE_LAST)
3483 {
3484 bugCode = OS_BUG_CHECK_BUGCODE_UNKNOWN;
3485 }
3486
3487 os_bug_check(bugCode, ppOsBugCheckBugcodeStr[bugCode]);
3488 }
3489
3490 /*!
3491 * @brief Perform an action at assertion failure.
3492 */
osAssertFailed(void)3493 void osAssertFailed(void)
3494 {
3495 os_dump_stack();
3496 }
3497
3498 /*!
3499 * @brief Get the GPU Chip Info - Speedo and IDDQ values
3500 *
3501 *
3502 * @param[in] pGpu GPU object pointer
3503 * @param[out] pGpuSpeedoHv Pointer to GPU Speedo value at high voltage corner.
3504 * @param[out] pGpuSpeedoLv Pointer to GPU Speedo value at low voltage corner.
3505 * @param[out] pGpuIddq Pointer to GPU Iddq Value
3506 * @param[out] pChipSkuId SKU ID for the chip
3507 *
3508 * @return NV_ERR_NOT_SUPPORTED
3509 */
3510 NV_STATUS
osGetChipInfo(OBJGPU * pGpu,NvU32 * pGpuSpeedoHv,NvU32 * pGpuSpeedoLv,NvU32 * pGpuIddq,NvU32 * pChipSkuId)3511 osGetChipInfo
3512 (
3513 OBJGPU *pGpu,
3514 NvU32 *pGpuSpeedoHv,
3515 NvU32 *pGpuSpeedoLv,
3516 NvU32 *pGpuIddq,
3517 NvU32 *pChipSkuId
3518 )
3519 {
3520 return NV_ERR_NOT_SUPPORTED;
3521 }
3522
3523 /*
3524 * @brief Get the GPU Rail Voltage Info (i.e. Min, Max and StepSize) in Tegra SoC.
3525 *
3526 * @param[in] pGpu GPU object pointer
3527 * @param[out] pMinVoltageuV Minimum Voltage supported on the Rail in Micro Volts
3528 * @param[out] pMaxVoltageuV Maximum Voltage supported on the Rail in Micro Volts
3529 * @param[out] pStepVoltageuV Voltage Step-size supported on the Rail in Micro Volts
3530 *
3531 * @return NV_ERR_NOT_SUPPORTED
3532 */
3533 NV_STATUS
osGetGpuRailVoltageInfo(OBJGPU * pGpu,NvU32 * pMinVoltageuV,NvU32 * pMaxVoltageuV,NvU32 * pStepVoltageuV)3534 osGetGpuRailVoltageInfo
3535 (
3536 OBJGPU *pGpu,
3537 NvU32 *pMinVoltageuV,
3538 NvU32 *pMaxVoltageuV,
3539 NvU32 *pStepVoltageuV
3540 )
3541 {
3542 return NV_ERR_NOT_SUPPORTED;
3543 }
3544
3545 /*!
3546 * @brief Get the current opaque security token.
3547 *
3548 * For Linux the security token is the effective UID of a process and process ID
3549 *
3550 * Note: This function allocates memory for the token. The onus is on the calling
3551 * function to free the memory associated with the token once its done with it.
3552 *
3553 * @return pointer to the security token.
3554 */
3555 PSECURITY_TOKEN
osGetSecurityToken(void)3556 osGetSecurityToken(void)
3557 {
3558 NV_STATUS rmStatus;
3559 TOKEN_USER *pTokenUser;
3560
3561 pTokenUser = portMemAllocNonPaged(sizeof(TOKEN_USER));
3562 if (pTokenUser == NULL)
3563 {
3564 return NULL;
3565 }
3566 rmStatus = os_get_euid(&pTokenUser->euid);
3567 if (rmStatus != NV_OK)
3568 {
3569 portMemFree(pTokenUser);
3570 return NULL;
3571 }
3572
3573 pTokenUser->pid = os_get_current_process();
3574
3575 return (PSECURITY_TOKEN)pTokenUser;
3576 }
3577
3578 PUID_TOKEN
osGetCurrentUidToken(void)3579 osGetCurrentUidToken(void)
3580 {
3581 NV_STATUS rmStatus;
3582 NvU32 *pUidToken;
3583
3584 pUidToken = portMemAllocNonPaged(sizeof(NvU32));
3585 if (pUidToken == NULL)
3586 {
3587 return NULL;
3588 }
3589
3590 rmStatus = os_get_euid(pUidToken);
3591 if (rmStatus != NV_OK)
3592 {
3593 portMemFree(pUidToken);
3594 return NULL;
3595 }
3596
3597 return (PUID_TOKEN)pUidToken;
3598 }
3599
3600 /*!
3601 * @brief Interface function to validate the token for the current client
3602 *
3603 * This function takes two tokens as parameters, validates them and checks
3604 * if either the PID or EUID from client database matches the current PID or EUID.
3605 *
3606 * @param[in] pClientSecurityToken security token cached in the client db
3607 * @param[in] pCurrentSecurityToken security token of the current client
3608 * @return NV_OK if the validation is successful
3609 * NV_ERR_INVALID_CLIENT if the tokens do not match
3610 * NV_ERR_INVALID_POINTER if the tokens are invalid
3611 */
3612 NV_STATUS
osValidateClientTokens(PSECURITY_TOKEN pClientSecurityToken,PSECURITY_TOKEN pCurrentSecurityToken)3613 osValidateClientTokens
3614 (
3615 PSECURITY_TOKEN pClientSecurityToken,
3616 PSECURITY_TOKEN pCurrentSecurityToken
3617 )
3618 {
3619 PTOKEN_USER pClientTokenUser = (PTOKEN_USER)pClientSecurityToken;
3620 PTOKEN_USER pCurrentTokenUser = (PTOKEN_USER)pCurrentSecurityToken;
3621
3622 if (pClientTokenUser == NULL || pCurrentTokenUser == NULL)
3623 return NV_ERR_INVALID_POINTER;
3624
3625 if ((pClientTokenUser->euid != pCurrentTokenUser->euid) &&
3626 (pClientTokenUser->pid != pCurrentTokenUser->pid))
3627 {
3628 NV_PRINTF(LEVEL_INFO,
3629 "NVRM: %s: Current security token doesn't match the one in the client database. "
3630 "Current EUID: %d, PID: %d; Client DB EUID: %d, PID: %d\n",
3631 __FUNCTION__, pCurrentTokenUser->euid, pCurrentTokenUser->pid,
3632 pClientTokenUser->euid, pClientTokenUser->pid);
3633 return NV_ERR_INVALID_CLIENT;
3634 }
3635
3636 return NV_OK;
3637 }
3638
3639 /*!
3640 * @brief Interface function to compare the tokens for two client
3641 *
3642 * This function takes two tokens as parameters, validates them and checks
3643 * if the EUIDs of each token match.
3644 *
3645 * @param[in] pToken1 Token to compare
3646 * @param[in] pToken2 Token to compare
3647 * @return NV_TRUE if the tokens match
3648 * NV_FALSE if the tokens do not match
3649 */
3650 NvBool
osUidTokensEqual(PUID_TOKEN pUidToken1,PUID_TOKEN pUidToken2)3651 osUidTokensEqual
3652 (
3653 PUID_TOKEN pUidToken1,
3654 PUID_TOKEN pUidToken2
3655 )
3656 {
3657 NvU32 * pTokenUser1 = (NvU32*)pUidToken1;
3658 NvU32 * pTokenUser2 = (NvU32*)pUidToken2;
3659
3660 NV_ASSERT_OR_RETURN((pTokenUser1 != NULL), NV_FALSE);
3661 NV_ASSERT_OR_RETURN((pTokenUser2 != NULL), NV_FALSE);
3662
3663 if (*pTokenUser1 != *pTokenUser2)
3664 {
3665 return NV_FALSE;
3666 }
3667
3668 return NV_TRUE;
3669 }
3670
3671 NvBool
osRemoveGpuSupported(void)3672 osRemoveGpuSupported
3673 (
3674 void
3675 )
3676 {
3677 return os_pci_remove_supported();
3678 }
3679
3680 /*
3681 * @brief Get the address ranges assigned to local or peer GPUs on a system that
3682 * supports hardware address translation services (ATS) over NVLink/C2C.
3683 *
3684 * @note
3685 * - All address values are in the System Physical Address (SPA) space
3686 * - Targets can either be "Local" (bIsPeer=False) or for a specified "Peer"
3687 * (bIsPeer=True, peerIndex=#) GPU
3688 * - Target address and mask values have a specified bit width, and represent
3689 * the higher order bits above the target address granularity
3690 *
3691 * @param[in] pGpu GPU object pointer
3692 * @param[out] pAddrSysPhys Pointer to hold SPA
3693 * @param[out] pAddrWidth Address range width value pointer
3694 * @param[out] pMask Mask value pointer
3695 * @param[out] pMaskWidth Mask width value pointer
3696 * @param[in] bIsPeer NV_TRUE if this is a peer, local GPU otherwise
3697 * @param[in] peerIndex Peer index
3698 *
3699 * @return NV_OK or NV_ERR_NOT_SUPPORTED
3700 *
3701 * A return value of NV_ERR_NOT_SUPPORTED for the local GPU would
3702 * indicate that the system does not support ATS over NVLink/C2C
3703 */
3704 NV_STATUS
osGetAtsTargetAddressRange(OBJGPU * pGpu,NvU64 * pAddrSysPhys,NvU32 * pAddrWidth,NvU32 * pMask,NvU32 * pMaskWidth,NvBool bIsPeer,NvU32 peerIndex)3705 osGetAtsTargetAddressRange
3706 (
3707 OBJGPU *pGpu,
3708 NvU64 *pAddrSysPhys,
3709 NvU32 *pAddrWidth,
3710 NvU32 *pMask,
3711 NvU32 *pMaskWidth,
3712 NvBool bIsPeer,
3713 NvU32 peerIndex
3714 )
3715 {
3716 #if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && (defined(NVCPU_PPC64LE) || defined(NVCPU_AARCH64))
3717 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
3718 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
3719 nv_state_t *nv;
3720 const int addrMaskWidth = 0x10;
3721
3722 if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP))
3723 return NV_ERR_INVALID_ARGUMENT;
3724
3725 nv = NV_GET_NV_STATE(pGpu);
3726
3727 //
3728 // TODO : Bug 1848958 restricts peer device tree parsing. Return early if
3729 // peer values are requested. This should be fixed by passing correct pGpu
3730 // pointer of the peer GPU retrieved using peerIds.
3731 //
3732 if (bIsPeer)
3733 {
3734 const int addrWidth = 0x10;
3735
3736 *pAddrSysPhys = 0;
3737 *pAddrWidth = addrWidth;
3738 *pMask = 0;
3739 *pMaskWidth = addrMaskWidth;
3740 return NV_OK;
3741 }
3742 else
3743 {
3744 NV_STATUS status = nv_get_device_memory_config(nv, pAddrSysPhys, NULL, NULL,
3745 pAddrWidth, NULL);
3746 if (status == NV_OK)
3747 {
3748 *pMask = NVBIT(*pAddrWidth) - 1U;
3749 *pMaskWidth = addrMaskWidth;
3750 }
3751 return status;
3752 }
3753
3754 return NV_OK;
3755 #endif
3756 return NV_ERR_NOT_SUPPORTED;
3757 }
3758
3759 /*
3760 * @brief Get the physical address in CPU address map and NUMA node id
3761 * of the GPU memory.
3762 *
3763 * @note
3764 * - The physical address is System Physical Address (SPA) in baremetal/host
3765 * and Intermediate Physical Address(IPA) or Guest Physical Address(GPA)
3766 * inside a VM.
3767 *
3768 * @param[in] pGpu GPU object pointer
3769 * @param[out] pAddrPhys Pointer to hold the physical address of FB in
3770 * CPU address map
3771 * @param[out] pNodeId NUMA nodeID of respective GPU memory
3772 *
3773 * @return NV_OK or NV_ERR_NOT_SUPPORTED
3774 *
3775 */
3776 NV_STATUS
osGetFbNumaInfo(OBJGPU * pGpu,NvU64 * pAddrPhys,NvU64 * pAddrRsvdPhys,NvS32 * pNodeId)3777 osGetFbNumaInfo
3778 (
3779 OBJGPU *pGpu,
3780 NvU64 *pAddrPhys,
3781 NvU64 *pAddrRsvdPhys,
3782 NvS32 *pNodeId
3783 )
3784 {
3785 #if RMCFG_MODULE_KERNEL_BIF && RMCFG_MODULE_KERNEL_NVLINK && (defined(NVCPU_PPC64LE) || defined(NVCPU_AARCH64))
3786 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu);
3787 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu);
3788 nv_state_t *nv;
3789
3790 *pNodeId = NV0000_CTRL_NO_NUMA_NODE;
3791
3792 if (!pKernelNvlink && !pKernelBif->getProperty(pKernelBif, PDB_PROP_KBIF_IS_C2C_LINK_UP))
3793 return NV_ERR_INVALID_ARGUMENT;
3794
3795 nv = NV_GET_NV_STATE(pGpu);
3796
3797 NV_STATUS status = nv_get_device_memory_config(nv, NULL, pAddrPhys,
3798 pAddrRsvdPhys, NULL, pNodeId);
3799
3800 return status;
3801 #endif
3802 return NV_ERR_NOT_SUPPORTED;
3803 }
3804
3805
3806 /*
3807 * @brief Verif only function to get the chiplib overrides for link connection
3808 * state for all C2C links.
3809 *
3810 * If chiplib overrides exist, each link can either be enabled (1) or disabled (0)
3811 *
3812 * @param[in] pGpu GPU object pointer
3813 * @param[in] maxLinks Size of pLinkConnection array
3814 * @param[out] pLinkConnection array of pLinkConnection values to be populated by MODS
3815 *
3816 * @return NV_OK or NV_ERR_NOT_SUPPORTED (no overrides available)
3817 */
3818 NV_STATUS
osGetForcedC2CConnection(OBJGPU * pGpu,NvU32 maxLinks,NvU32 * pLinkConnection)3819 osGetForcedC2CConnection
3820 (
3821 OBJGPU *pGpu,
3822 NvU32 maxLinks,
3823 NvU32 *pLinkConnection
3824 )
3825 {
3826 int i, ret;
3827 NV_STATUS status;
3828 char path[64];
3829
3830 NV_ASSERT_OR_RETURN((pLinkConnection != NULL), NV_ERR_INVALID_POINTER);
3831 NV_ASSERT_OR_RETURN((maxLinks > 0), NV_ERR_NOT_SUPPORTED);
3832 NV_ASSERT_OR_RETURN((pGpu != NULL), NV_ERR_INVALID_ARGUMENT);
3833
3834 for (i = 0; i < maxLinks; i++)
3835 {
3836 ret = os_snprintf(path, sizeof(path), "CPU_MODEL|CM_ATS_ADDRESS|C2C%u", i);
3837 NV_ASSERT((ret > 0) && (ret < (sizeof(path) - 1)));
3838
3839 status = gpuSimEscapeRead(pGpu, path, 0, 4, &pLinkConnection[i]);
3840 if (status == NV_OK)
3841 {
3842 NV_PRINTF(LEVEL_INFO, "%s: %s=0x%X\n", __FUNCTION__,
3843 path, pLinkConnection[i]);
3844 }
3845 else
3846 {
3847 NV_PRINTF(LEVEL_INFO, "%s: gpuSimEscapeRead for '%s' failed (%u)\n",
3848 __FUNCTION__, path, status);
3849 return NV_ERR_NOT_SUPPORTED;
3850 }
3851 }
3852 return NV_OK;
3853 }
3854
3855 static NV_STATUS
osGetSmbiosTableInfo(const NvU8 * pMappedAddr,NvU64 * pBaseAddr,NvU64 * pLength,NvU64 * pNumSubTypes,NvU32 * pVersion)3856 osGetSmbiosTableInfo
3857 (
3858 const NvU8 *pMappedAddr,
3859 NvU64 *pBaseAddr,
3860 NvU64 *pLength,
3861 NvU64 *pNumSubTypes,
3862 NvU32 *pVersion
3863 )
3864 {
3865 *pBaseAddr = 0;
3866 *pLength = 0;
3867 *pNumSubTypes = 0;
3868 *pVersion = 0;
3869
3870 if (portMemCmp(pMappedAddr, "_SM3_", 5) == 0)
3871 {
3872 *pVersion = (pMappedAddr[7] << 8) | pMappedAddr[8];
3873 portMemCopy(pLength, 4, pMappedAddr + 12, 4);
3874 portMemCopy(pBaseAddr, 8, pMappedAddr + 16, 8);
3875
3876 *pNumSubTypes = *pLength / 4;
3877
3878 return NV_OK;
3879 }
3880
3881 if (portMemCmp(pMappedAddr, "_SM_", 4) == 0)
3882 {
3883 *pVersion = (pMappedAddr[6] << 8) | pMappedAddr[7];
3884
3885 pMappedAddr += 16;
3886
3887 if (portMemCmp(pMappedAddr, "_DMI_", 5) == 0)
3888 {
3889 portMemCopy(pLength, 2, pMappedAddr + 6, 2);
3890 portMemCopy(pBaseAddr, 4, pMappedAddr + 8, 4);
3891 portMemCopy(pNumSubTypes, 2, pMappedAddr + 12, 2);
3892
3893 if (!*pVersion)
3894 *pVersion = (pMappedAddr[14] & 0xF0) << 4 |
3895 (pMappedAddr[14] & 0x0F);
3896
3897 return NV_OK;
3898 }
3899 }
3900
3901 return NV_ERR_INVALID_ADDRESS;
3902 }
3903
3904
3905 /*
3906 * @brief Function to export SMBIOS table. Also, maps table in kernel-space.
3907 *
3908 * @param[out] ppBaseVAddr Base virtual address of SMBIOS table.
3909 * @param[out] pLength Size of SMBIOS table.
3910 * @param[out] pNumSubTypes Count of structures (types) embedded in
3911 * the SMBIOS table.
3912 * @param[out] pVersion SMBIOS version
3913 *
3914 * @return NV_OK, NV_ERR_INSUFFICIENT_RESOURCES or NV_ERR_INVALID_ADDRESS
3915 * or errors from OS layer
3916 */
3917 NV_STATUS
osGetSmbiosTable(void ** ppBaseVAddr,NvU64 * pLength,NvU64 * pNumSubTypes,NvU32 * pVersion)3918 osGetSmbiosTable
3919 (
3920 void **ppBaseVAddr,
3921 NvU64 *pLength,
3922 NvU64 *pNumSubTypes,
3923 NvU32 *pVersion
3924 )
3925 {
3926 NV_STATUS status = NV_OK;
3927 NvU64 physSmbiosAddr = ~0ull;
3928 void *pMappedAddr = NULL;
3929 NvU64 basePAddr = 0;
3930
3931 if (!NVCPU_IS_X86_64 && !NVCPU_IS_AARCH64)
3932 {
3933 return NV_ERR_NOT_SUPPORTED;
3934 }
3935
3936 status = os_get_smbios_header(&physSmbiosAddr);
3937 if (status != NV_OK)
3938 {
3939 NV_PRINTF(LEVEL_INFO, "%s: Failed query SMBIOS table with error: %x \n",
3940 __FUNCTION__, status);
3941 return status;
3942 }
3943
3944 NV_ASSERT(physSmbiosAddr != ~0ull);
3945
3946 pMappedAddr = osMapKernelSpace(physSmbiosAddr,
3947 os_page_size,
3948 NV_MEMORY_CACHED,
3949 NV_PROTECT_READ_WRITE);
3950 if (!pMappedAddr)
3951 {
3952 return NV_ERR_INSUFFICIENT_RESOURCES;
3953 }
3954
3955 status = osGetSmbiosTableInfo(pMappedAddr,
3956 &basePAddr,
3957 pLength,
3958 pNumSubTypes,
3959 pVersion);
3960
3961 osUnmapKernelSpace(pMappedAddr, os_page_size);
3962
3963 if (status != NV_OK)
3964 {
3965 return status;
3966 }
3967
3968 *ppBaseVAddr = osMapKernelSpace(basePAddr,
3969 *pLength,
3970 NV_MEMORY_CACHED,
3971 NV_PROTECT_READ_WRITE);
3972 if (!*ppBaseVAddr)
3973 {
3974 return NV_ERR_INSUFFICIENT_RESOURCES;
3975 }
3976
3977 return NV_OK;
3978 }
3979
3980 /*
3981 * @brief Function to free SMBIOS table mappings
3982 *
3983 * @param[in] pBaseVAddr Base virtual address of SMBIOS table.
3984 * @param[in] length Size of SMBIOS table.
3985 *
3986 */
3987 void
osPutSmbiosTable(void * pBaseVAddr,NvU64 length)3988 osPutSmbiosTable
3989 (
3990 void *pBaseVAddr,
3991 NvU64 length
3992 )
3993 {
3994 osUnmapKernelSpace(pBaseVAddr, length);
3995 }
3996
3997 NV_STATUS
osGetAcpiRsdpFromUefi(NvU32 * pRsdpAddr)3998 osGetAcpiRsdpFromUefi
3999 (
4000 NvU32 *pRsdpAddr
4001 )
4002 {
4003 return os_get_acpi_rsdp_from_uefi(pRsdpAddr);
4004 }
4005
4006 /*
4007 * @brief Returns NV_TRUE if NvSwitch device is present in the system.
4008 */
4009 NvBool
osIsNvswitchPresent(void)4010 osIsNvswitchPresent
4011 (
4012 void
4013 )
4014 {
4015 return os_is_nvswitch_present();
4016 }
4017
4018 /*
4019 * @brief Function to add crashlog buffer entry.
4020 *
4021 * @param[in] pBuffer virt_addr of nvlog buffer
4022 * @param[in] length size of nvlog buffer
4023 */
4024 void
osAddRecordForCrashLog(void * pBuffer,NvU32 length)4025 osAddRecordForCrashLog
4026 (
4027 void *pBuffer,
4028 NvU32 length
4029 )
4030 {
4031 os_add_record_for_crashLog(pBuffer, length);
4032 }
4033
4034 /*
4035 * @brief Function to delete crashlog buffer entry.
4036 *
4037 * @param[in] pBuffer virt_addr of nvlog buffer
4038 */
4039 void
osDeleteRecordForCrashLog(void * pBuffer)4040 osDeleteRecordForCrashLog
4041 (
4042 void *pBuffer
4043 )
4044 {
4045 os_delete_record_for_crashLog(pBuffer);
4046 }
4047
4048 /*
4049 * @brief Queries the sysfs interface to get memblock size
4050 * @param[out] memblock_size Pointer to the memblock_size
4051 */
4052 NV_STATUS
osNumaMemblockSize(NvU64 * memblock_size)4053 osNumaMemblockSize
4054 (
4055 NvU64 *memblock_size
4056 )
4057 {
4058 return os_numa_memblock_size(memblock_size);
4059 }
4060
4061 NvBool
osNumaOnliningEnabled(OS_GPU_INFO * pOsGpuInfo)4062 osNumaOnliningEnabled
4063 (
4064 OS_GPU_INFO *pOsGpuInfo
4065 )
4066 {
4067 NvS32 numaNodeId = NV0000_CTRL_NO_NUMA_NODE;
4068
4069 //
4070 // Note that this numaNodeId value fetched from Linux layer might not be
4071 // accurate since it is possible to overwrite it with regkey on some configs
4072 //
4073 if (nv_get_device_memory_config(pOsGpuInfo, NULL, NULL, NULL, NULL,
4074 &numaNodeId) != NV_OK)
4075 {
4076 return NV_FALSE;
4077 }
4078
4079 return (numaNodeId != NV0000_CTRL_NO_NUMA_NODE);
4080 }
4081
4082 /*
4083 * @brief Function to call NUMA allocation entry.
4084 *
4085 * @param[in] nid NUMA node id
4086 * @param[in] size Allocation size
4087 * @param[in] flag Allocation flags
4088 * @param[out] pAddress Ptr to the allocated physical address
4089 */
4090 NV_STATUS
osAllocPagesNode(NvS32 nid,NvLength size,NvU32 flag,NvU64 * pAddress)4091 osAllocPagesNode
4092 (
4093 NvS32 nid,
4094 NvLength size,
4095 NvU32 flag,
4096 NvU64 *pAddress
4097 )
4098 {
4099 NV_STATUS status = NV_OK;
4100 NvU32 localFlag = NV_ALLOC_PAGES_NODE_NONE;
4101
4102 if (pAddress == NULL || nid < 0 || size > NV_U32_MAX)
4103 {
4104 return NV_ERR_INVALID_ARGUMENT;
4105 }
4106
4107 // Translate the flags
4108 if (flag & OS_ALLOC_PAGES_NODE_SKIP_RECLAIM)
4109 {
4110 localFlag |= NV_ALLOC_PAGES_NODE_SKIP_RECLAIM;
4111 }
4112
4113 status = os_alloc_pages_node(nid, (NvU32)size, localFlag, pAddress);
4114 return status;
4115 }
4116
4117 void
osAllocAcquirePage(NvU64 pAddress,NvU32 pageCount)4118 osAllocAcquirePage
4119 (
4120 NvU64 pAddress,
4121 NvU32 pageCount
4122 )
4123 {
4124 NvU32 i;
4125
4126 for (i = 0; i < pageCount; i++)
4127 {
4128 os_get_page(pAddress + (i << os_page_shift));
4129 }
4130 }
4131
4132 void
osAllocReleasePage(NvU64 pAddress,NvU32 pageCount)4133 osAllocReleasePage
4134 (
4135 NvU64 pAddress,
4136 NvU32 pageCount
4137 )
4138 {
4139 NvU32 i;
4140
4141 for (i = 0; i < pageCount; i++)
4142 {
4143 os_put_page(pAddress + (i << os_page_shift));
4144 }
4145 }
4146
4147 /*
4148 * @brief Function to return refcount on a page
4149 * @param[in] address The physical address of the page
4150 */
4151 NvU32
osGetPageRefcount(NvU64 pAddress)4152 osGetPageRefcount
4153 (
4154 NvU64 pAddress
4155 )
4156 {
4157 return os_get_page_refcount(pAddress);
4158 }
4159
4160 /*
4161 * @brief Function to return the number of tail pages if the address is
4162 * referring to a compound page; For non-compound pages, 1 is returned.
4163 * @param[in] address The physical address of the page
4164 */
4165 NvU32
osCountTailPages(NvU64 pAddress)4166 osCountTailPages
4167 (
4168 NvU64 pAddress
4169 )
4170 {
4171 return os_count_tail_pages(pAddress);
4172 }
4173
4174 /*
4175 * @brief Upon success, gets NPU register address range.
4176 *
4177 * @param[in] pOsGpuInfo OS specific GPU information pointer
4178 * @param[out] pBase base (physical) of NPU register address range
4179 * @param[out] pSize size of NPU register address range
4180 */
4181 NV_STATUS
osGetIbmnpuGenregInfo(OS_GPU_INFO * pOsGpuInfo,NvU64 * pBase,NvU64 * pSize)4182 osGetIbmnpuGenregInfo
4183 (
4184 OS_GPU_INFO *pOsGpuInfo,
4185 NvU64 *pBase,
4186 NvU64 *pSize
4187 )
4188 {
4189 return nv_get_ibmnpu_genreg_info(pOsGpuInfo, pBase, pSize, NULL);
4190 }
4191
4192 /*
4193 * @brief Upon success, gets NPU's relaxed ordering mode.
4194 *
4195 * @param[in] pOsGpuInfo OS specific GPU information pointer
4196 * @param[out] pMode relaxed ordering mode
4197 */
4198 NV_STATUS
osGetIbmnpuRelaxedOrderingMode(OS_GPU_INFO * pOsGpuInfo,NvBool * pMode)4199 osGetIbmnpuRelaxedOrderingMode
4200 (
4201 OS_GPU_INFO *pOsGpuInfo,
4202 NvBool *pMode
4203 )
4204 {
4205 return nv_get_ibmnpu_relaxed_ordering_mode(pOsGpuInfo, pMode);
4206 }
4207
4208 /*
4209 * @brief Waits for NVLink HW flush on an NPU associated with a GPU.
4210 *
4211 * @param[in] pOsGpuInfo OS specific GPU information pointer
4212 */
4213 void
osWaitForIbmnpuRsync(OS_GPU_INFO * pOsGpuInfo)4214 osWaitForIbmnpuRsync
4215 (
4216 OS_GPU_INFO *pOsGpuInfo
4217 )
4218 {
4219 nv_wait_for_ibmnpu_rsync(pOsGpuInfo);
4220 }
4221
4222 NvU64
osGetPageSize(void)4223 osGetPageSize(void)
4224 {
4225 return os_page_size;
4226 }
4227
4228 NvU8
osGetPageShift(void)4229 osGetPageShift(void)
4230 {
4231 return os_page_shift;
4232 }
4233
4234
4235
4236 /*
4237 * @brief Opens a new temporary file for reading and writing
4238 *
4239 * @param[in] ppFile void double pointer
4240 *
4241 * @returns NV_STATUS, NV_OK if success,
4242 NV_ERR_GENERIC, if error
4243 NV_ERR_NOT_SUPPORTED, for unsupported platforms
4244 */
4245 NV_STATUS
osOpenTemporaryFile(void ** ppFile)4246 osOpenTemporaryFile
4247 (
4248 void **ppFile
4249 )
4250 {
4251 return os_open_temporary_file(ppFile);
4252 }
4253
4254 /*
4255 * @brief Closes the specified temporary file
4256 *
4257 * @param[in] pFile Pointer to file
4258 *
4259 * @returns void
4260 */
4261 void
osCloseFile(void * pFile)4262 osCloseFile
4263 (
4264 void *pFile
4265 )
4266 {
4267 os_close_file(pFile);
4268 }
4269
4270 /*
4271 * @brief Writes the buffer to the specified file at the given offset
4272 *
4273 * @param[in] pFile Pointer to file (void)
4274 * @param[in] pBuffer Pointer to buffer from which to copy
4275 * @param[in] size Size of the copy
4276 * @parma[in] offset offset in the file
4277 *
4278 * @returns NV_STATUS, NV_OK if success,
4279 NV_ERR_GENERIC, if error
4280 NV_ERR_NOT_SUPPORTED, for unsupported platforms
4281 */
4282 NV_STATUS
osWriteToFile(void * pFile,NvU8 * pBuffer,NvU64 size,NvU64 offset)4283 osWriteToFile
4284 (
4285 void *pFile,
4286 NvU8 *pBuffer,
4287 NvU64 size,
4288 NvU64 offset
4289 )
4290 {
4291 return os_write_file(pFile, pBuffer, size, offset);
4292 }
4293
4294 /*
4295 * @brief Reads from the specified file at the given offset
4296 *
4297 * @param[in] pFile Pointer to file (void *)
4298 * @param[in] pBuffer Pointer to buffer to which the data is copied
4299 * @param[in] size Size of the copy
4300 * @parma[in] offset offset in the file
4301 *
4302 * @returns NV_STATUS, NV_OK if success,
4303 NV_ERR_GENERIC, if error
4304 NV_ERR_NOT_SUPPORTED, for unsupported platforms
4305 */
4306 NV_STATUS
osReadFromFile(void * pFile,NvU8 * pBuffer,NvU64 size,NvU64 offset)4307 osReadFromFile
4308 (
4309 void *pFile,
4310 NvU8 *pBuffer,
4311 NvU64 size,
4312 NvU64 offset
4313 )
4314 {
4315 return os_read_file(pFile, pBuffer, size, offset);
4316 }
4317
4318 /*
4319 * @brief Unregisters caps from the capability framework.
4320 * The function assumes that the caps are allocated and stored in the
4321 * hierarchical order. If they aren't, OS (Linux kernel) would warn and
4322 * leak the caps.
4323 *
4324 * @param[in] pOsRmCaps caps of interest
4325 */
4326 void
osRmCapUnregister(OS_RM_CAPS ** ppOsRmCaps)4327 osRmCapUnregister
4328 (
4329 OS_RM_CAPS **ppOsRmCaps
4330 )
4331 {
4332 OS_RM_CAPS *pOsRmCaps = *ppOsRmCaps;
4333 NvS32 i;
4334
4335 if (pOsRmCaps == NULL)
4336 {
4337 return;
4338 }
4339
4340 for (i = pOsRmCaps->count - 1; i >= 0; i--)
4341 {
4342 if (pOsRmCaps->caps[i] != NULL)
4343 {
4344 os_nv_cap_destroy_entry(pOsRmCaps->caps[i]);
4345 }
4346 }
4347
4348 os_free_mem(pOsRmCaps->caps);
4349 os_free_mem(pOsRmCaps);
4350
4351 *ppOsRmCaps = NULL;
4352 }
4353
4354 static NV_STATUS
_allocOsRmCaps(OS_RM_CAPS ** ppOsRmCaps,NvU32 count)4355 _allocOsRmCaps
4356 (
4357 OS_RM_CAPS **ppOsRmCaps,
4358 NvU32 count
4359 )
4360 {
4361 NV_STATUS status;
4362 OS_RM_CAPS *pOsRmCaps;
4363
4364 *ppOsRmCaps = NULL;
4365
4366 status = os_alloc_mem((void**)&pOsRmCaps, sizeof(OS_RM_CAPS));
4367 if (status != NV_OK)
4368 return status;
4369
4370 pOsRmCaps->count = count;
4371
4372 status = os_alloc_mem((void**)&pOsRmCaps->caps, sizeof(pOsRmCaps->caps[0]) * count);
4373 if (status != NV_OK)
4374 {
4375 os_free_mem(pOsRmCaps);
4376 return status;
4377 }
4378
4379 os_mem_set(pOsRmCaps->caps, 0, sizeof(pOsRmCaps->caps[0]) * count);
4380
4381 *ppOsRmCaps = pOsRmCaps;
4382 return NV_OK;
4383 }
4384
4385 #define OS_RM_CAP_GPU_DIR 0
4386 #define OS_RM_CAP_GPU_MIG_DIR 1
4387 #define OS_RM_CAP_GPU_COUNT 2
4388
4389 /*
4390 * @brief Registers OBJGPU with the capability framework.
4391 *
4392 * @param[in] pOsGpuInfo OS specific GPU information pointer
4393 * @param[out] ppOsRmCaps GPU OS specific capabilities pointer
4394 */
4395 NV_STATUS
osRmCapRegisterGpu(OS_GPU_INFO * pOsGpuInfo,OS_RM_CAPS ** ppOsRmCaps)4396 osRmCapRegisterGpu
4397 (
4398 OS_GPU_INFO *pOsGpuInfo,
4399 OS_RM_CAPS **ppOsRmCaps
4400 )
4401 {
4402 NvU32 minor = nv_get_dev_minor(pOsGpuInfo);
4403 char name[16];
4404 NV_STATUS status;
4405 OS_RM_CAPS *pOsRmCaps;
4406 nv_cap_t *parent;
4407 nv_cap_t *cap;
4408
4409 // Return success on the unsupported platforms.
4410 if (nvidia_caps_root == NULL)
4411 {
4412 return NV_OK;
4413 }
4414
4415 if (*ppOsRmCaps != NULL)
4416 {
4417 return NV_ERR_INVALID_ARGUMENT;
4418 }
4419
4420 status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_GPU_COUNT);
4421 if (status != NV_OK)
4422 return status;
4423
4424 *ppOsRmCaps = pOsRmCaps;
4425
4426 os_snprintf(name, sizeof(name), "gpu%u", minor);
4427 name[sizeof(name) - 1] = '\0';
4428 parent = nvidia_caps_root;
4429
4430 cap = os_nv_cap_create_dir_entry(parent, name, (OS_RUGO | OS_XUGO));
4431 if (cap == NULL)
4432 {
4433 NV_PRINTF(LEVEL_ERROR, "Failed to setup gpu%u directory\n", minor);
4434 status = NV_ERR_OPERATING_SYSTEM;
4435 goto failed;
4436 }
4437
4438 pOsRmCaps->caps[OS_RM_CAP_GPU_DIR] = cap;
4439 parent = cap;
4440
4441 // TODO: Bug 2679591: Add MIG directory only if SMC is enabled.
4442 // For now, always add "mig" directory.
4443 cap = os_nv_cap_create_dir_entry(parent, "mig", (OS_RUGO | OS_XUGO));
4444 if (cap == NULL)
4445 {
4446 NV_PRINTF(LEVEL_ERROR, "Failed to setup mig directory\n");
4447 status = NV_ERR_OPERATING_SYSTEM;
4448 goto failed;
4449 }
4450
4451 pOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR] = cap;
4452
4453 return NV_OK;
4454
4455 failed:
4456 osRmCapUnregister(ppOsRmCaps);
4457
4458 return status;
4459 }
4460
4461 #define OS_RM_CAP_SMC_PART_DIR 0
4462 #define OS_RM_CAP_SMC_PART_ACCESS_FILE 1
4463 #define OS_RM_CAP_SMC_PART_COUNT 2
4464
4465 /*
4466 * @brief Registers SMC partition (a.k.a. GPU instance) with the capability
4467 * framework
4468 *
4469 * @param[in] pGpuOsRmCaps GPU OS specific capabilities pointer
4470 * @param[out] ppPartitionOsRmCaps OS specific capabilities pointer for SMC partition
4471 * @param[in] swizzId SMC partition swizz ID
4472 */
4473 NV_STATUS
osRmCapRegisterSmcPartition(OS_RM_CAPS * pGpuOsRmCaps,OS_RM_CAPS ** ppPartitionOsRmCaps,NvU32 swizzId)4474 osRmCapRegisterSmcPartition
4475 (
4476 OS_RM_CAPS *pGpuOsRmCaps,
4477 OS_RM_CAPS **ppPartitionOsRmCaps,
4478 NvU32 swizzId
4479 )
4480 {
4481 char name[16];
4482 NV_STATUS status;
4483 nv_cap_t *parent;
4484 nv_cap_t *cap;
4485 OS_RM_CAPS *pOsRmCaps;
4486
4487 // Return success as there is nothing to do.
4488 if (pGpuOsRmCaps == NULL)
4489 {
4490 return NV_OK;
4491 }
4492
4493 if (*ppPartitionOsRmCaps != NULL || swizzId >= NV_U32_MAX)
4494 {
4495 return NV_ERR_INVALID_ARGUMENT;
4496 }
4497
4498 parent = pGpuOsRmCaps->caps[OS_RM_CAP_GPU_MIG_DIR];
4499 if (parent == NULL)
4500 {
4501 return NV_ERR_INVALID_STATE;
4502 }
4503
4504 status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_PART_COUNT);
4505 if (status != NV_OK)
4506 return status;
4507
4508 *ppPartitionOsRmCaps = pOsRmCaps;
4509
4510 os_snprintf(name, sizeof(name), "gi%u", swizzId);
4511 name[sizeof(name) - 1] = '\0';
4512
4513 cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO);
4514 if (cap == NULL)
4515 {
4516 NV_PRINTF(LEVEL_ERROR, "Failed to setup gi%u directory\n",
4517 swizzId);
4518 status = NV_ERR_OPERATING_SYSTEM;
4519 goto failed;
4520 }
4521
4522 pOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR] = cap;
4523 parent = cap;
4524
4525 cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO);
4526 if (cap == NULL)
4527 {
4528 NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n",
4529 swizzId);
4530 status = NV_ERR_OPERATING_SYSTEM;
4531 goto failed;
4532 }
4533
4534 pOsRmCaps->caps[OS_RM_CAP_SMC_PART_ACCESS_FILE] = cap;
4535
4536 return NV_OK;
4537
4538 failed:
4539 osRmCapUnregister(ppPartitionOsRmCaps);
4540
4541 return status;
4542 }
4543
4544 #define OS_RM_CAP_SMC_EXEC_PART_DIR 0
4545 #define OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE 1
4546 #define OS_RM_CAP_SMC_EXEC_PART_COUNT 2
4547
4548 /*
4549 * @brief Registers SMC execution partition (a.k.a. compute instance) with the
4550 * capability framework
4551 *
4552 * @param[in] pPartitionOsRmCaps OS specific capabilities pointer for SMC partition
4553 * @param[out] ppExecPartitionOsRmCaps OS specific capabilities pointer for SMC execution partition
4554 * @param[in] execPartitionId SMC execution partition ID
4555 */
4556 NV_STATUS
osRmCapRegisterSmcExecutionPartition(OS_RM_CAPS * pPartitionOsRmCaps,OS_RM_CAPS ** ppExecPartitionOsRmCaps,NvU32 execPartitionId)4557 osRmCapRegisterSmcExecutionPartition
4558 (
4559 OS_RM_CAPS *pPartitionOsRmCaps,
4560 OS_RM_CAPS **ppExecPartitionOsRmCaps,
4561 NvU32 execPartitionId
4562 )
4563 {
4564 char name[16];
4565 NV_STATUS status;
4566 nv_cap_t *parent;
4567 nv_cap_t *cap;
4568 OS_RM_CAPS *pOsRmCaps;
4569
4570 // Return success as there is nothing to do.
4571 if (pPartitionOsRmCaps == NULL)
4572 {
4573 return NV_OK;
4574 }
4575
4576 if ((*ppExecPartitionOsRmCaps != NULL) || (execPartitionId >= NV_U32_MAX))
4577 {
4578 return NV_ERR_INVALID_ARGUMENT;
4579 }
4580
4581 parent = pPartitionOsRmCaps->caps[OS_RM_CAP_SMC_PART_DIR];
4582 if (parent == NULL)
4583 {
4584 return NV_ERR_INVALID_STATE;
4585 }
4586
4587 status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SMC_EXEC_PART_COUNT);
4588 if (status != NV_OK)
4589 {
4590 return status;
4591 }
4592
4593 *ppExecPartitionOsRmCaps = pOsRmCaps;
4594
4595 os_snprintf(name, sizeof(name), "ci%u", execPartitionId);
4596 name[sizeof(name) - 1] = '\0';
4597
4598 cap = os_nv_cap_create_dir_entry(parent, name, OS_RUGO | OS_XUGO);
4599 if (cap == NULL)
4600 {
4601 NV_PRINTF(LEVEL_ERROR, "Failed to setup ci%u directory\n",
4602 execPartitionId);
4603 status = NV_ERR_OPERATING_SYSTEM;
4604 goto failed;
4605 }
4606
4607 pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_DIR] = cap;
4608 parent = cap;
4609
4610 cap = os_nv_cap_create_file_entry(parent, "access", OS_RUGO);
4611 if (cap == NULL)
4612 {
4613 NV_PRINTF(LEVEL_ERROR, "Failed to setup access file for ID:%u\n",
4614 execPartitionId);
4615 status = NV_ERR_OPERATING_SYSTEM;
4616 goto failed;
4617 }
4618
4619 pOsRmCaps->caps[OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE] = cap;
4620
4621 return NV_OK;
4622
4623 failed:
4624 osRmCapUnregister(ppExecPartitionOsRmCaps);
4625
4626 return status;
4627 }
4628
4629 /*
4630 * @brief Release the acquired capability
4631 *
4632 * @param[in] dupedCapDescriptor descriptor to be released
4633 */
4634 void
osRmCapRelease(NvU64 dupedCapDescriptor)4635 osRmCapRelease
4636 (
4637 NvU64 dupedCapDescriptor
4638 )
4639 {
4640 if (dupedCapDescriptor == NV_U64_MAX)
4641 {
4642 return;
4643 }
4644
4645 os_nv_cap_close_fd((int)dupedCapDescriptor);
4646 }
4647
4648 #define OS_RM_CAP_SYS_MIG_DIR 0
4649 #define OS_RM_CAP_SYS_SMC_CONFIG_FILE 1
4650 #define OS_RM_CAP_SYS_SMC_MONITOR_FILE 2
4651 #define OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE 3
4652 #define OS_RM_CAP_SYS_COUNT 4
4653
4654 NV_STATUS
osRmCapRegisterSys(OS_RM_CAPS ** ppOsRmCaps)4655 osRmCapRegisterSys
4656 (
4657 OS_RM_CAPS **ppOsRmCaps
4658 )
4659 {
4660 nv_cap_t **ppCaps;
4661 nv_cap_t *parent;
4662 nv_cap_t *cap;
4663 NV_STATUS status;
4664 OS_RM_CAPS *pOsRmCaps;
4665
4666 if (nvidia_caps_root == NULL)
4667 return NV_ERR_NOT_SUPPORTED;
4668
4669 status = _allocOsRmCaps(&pOsRmCaps, OS_RM_CAP_SYS_COUNT);
4670 if (status != NV_OK)
4671 return status;
4672
4673 *ppOsRmCaps = pOsRmCaps;
4674
4675 ppCaps = pOsRmCaps->caps;
4676
4677 parent = os_nv_cap_create_dir_entry(nvidia_caps_root, "mig", OS_RUGO | OS_XUGO);
4678 if (parent == NULL)
4679 {
4680 NV_PRINTF(LEVEL_ERROR, "Failed to create mig directory\n");
4681 status = NV_ERR_OPERATING_SYSTEM;
4682 goto failed;
4683 }
4684 ppCaps[OS_RM_CAP_SYS_MIG_DIR] = parent;
4685
4686 cap = os_nv_cap_create_file_entry(parent, "config", OS_RUSR);
4687 if (cap == NULL)
4688 {
4689 NV_PRINTF(LEVEL_ERROR, "Failed to create mig config file\n");
4690 status = NV_ERR_OPERATING_SYSTEM;
4691 goto failed;
4692 }
4693 ppCaps[OS_RM_CAP_SYS_SMC_CONFIG_FILE] = cap;
4694
4695 cap = os_nv_cap_create_file_entry(parent, "monitor", OS_RUGO);
4696 if (cap == NULL)
4697 {
4698 NV_PRINTF(LEVEL_ERROR, "Failed to create mig monitor file\n");
4699 status = NV_ERR_OPERATING_SYSTEM;
4700 goto failed;
4701 }
4702 ppCaps[OS_RM_CAP_SYS_SMC_MONITOR_FILE] = cap;
4703
4704 cap = os_nv_cap_create_file_entry(nvidia_caps_root, "fabric-imex-mgmt", OS_RUSR);
4705 if (cap == NULL)
4706 {
4707 NV_PRINTF(LEVEL_ERROR, "Failed to create imex file\n");
4708 status = NV_ERR_OPERATING_SYSTEM;
4709 goto failed;
4710 }
4711 ppCaps[OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE] = cap;
4712
4713 return NV_OK;
4714
4715 failed:
4716 osRmCapUnregister(ppOsRmCaps);
4717 return status;
4718 }
4719
4720 /*
4721 * @brief Acquire the requested capability
4722 *
4723 * @param[in] pOsRmCaps opaque pointer to the caps.
4724 * @param[in] rmCap the capability to be acquired.
4725 * @param[in] capDescriptor descriptor to be used for validation
4726 * @param[out] dupedCapDescriptor returns duplicated descriptor if validation
4727 * is successful
4728 *
4729 * Note: On Linux, duplicating fd is helpful to let administrators know about
4730 * the capability users. See https://linux.die.net/man/8/lsof usage.
4731 */
4732 NV_STATUS
osRmCapAcquire(OS_RM_CAPS * pOsRmCaps,NvU32 rmCap,NvU64 capDescriptor,NvU64 * dupedCapDescriptor)4733 osRmCapAcquire
4734 (
4735 OS_RM_CAPS *pOsRmCaps,
4736 NvU32 rmCap,
4737 NvU64 capDescriptor,
4738 NvU64 *dupedCapDescriptor
4739 )
4740 {
4741 nv_cap_t *cap;
4742 int fd = (int)capDescriptor;
4743 int duped_fd;
4744 NvU32 index;
4745 NV_STATUS status;
4746
4747 *dupedCapDescriptor = NV_U64_MAX;
4748
4749 switch (rmCap)
4750 {
4751 case NV_RM_CAP_SMC_PARTITION_ACCESS:
4752 {
4753 index = OS_RM_CAP_SMC_PART_ACCESS_FILE;
4754 break;
4755 }
4756 case NV_RM_CAP_EXT_FABRIC_MGMT:
4757 {
4758 status = nv_acquire_fabric_mgmt_cap(fd, &duped_fd);
4759 if (status != NV_OK)
4760 {
4761 return status;
4762 }
4763
4764 goto done;
4765 }
4766 case NV_RM_CAP_SMC_EXEC_PARTITION_ACCESS:
4767 {
4768 index = OS_RM_CAP_SMC_EXEC_PART_ACCESS_FILE;
4769 break;
4770 }
4771 case NV_RM_CAP_SYS_SMC_CONFIG:
4772 {
4773 index = OS_RM_CAP_SYS_SMC_CONFIG_FILE;
4774 break;
4775 }
4776 case NV_RM_CAP_SYS_SMC_MONITOR:
4777 {
4778 index = OS_RM_CAP_SYS_SMC_MONITOR_FILE;
4779 break;
4780 }
4781 case NV_RM_CAP_SYS_FABRIC_IMEX_MGMT:
4782 {
4783 index = OS_RM_CAP_SYS_FABRIC_IMEX_MGMT_FILE;
4784 break;
4785 }
4786 default:
4787 {
4788 return NV_ERR_INVALID_ARGUMENT;
4789 }
4790 }
4791
4792 if (pOsRmCaps == NULL)
4793 {
4794 return NV_ERR_NOT_SUPPORTED;
4795 }
4796
4797 if (index >= pOsRmCaps->count)
4798 {
4799 return NV_ERR_INVALID_ARGUMENT;
4800 }
4801
4802 cap = pOsRmCaps->caps[index];
4803
4804 duped_fd = os_nv_cap_validate_and_dup_fd(cap, fd);
4805 if (duped_fd < 0)
4806 {
4807 return NV_ERR_INSUFFICIENT_PERMISSIONS;
4808 }
4809
4810 done:
4811 *dupedCapDescriptor = duped_fd;
4812
4813 return NV_OK;
4814 }
4815
4816 /*
4817 * @brief Initializes capability descriptor
4818 *
4819 * @param[out] pCapDescriptor descriptor to be used
4820 *
4821 */
4822 void
osRmCapInitDescriptor(NvU64 * pCapDescriptor)4823 osRmCapInitDescriptor
4824 (
4825 NvU64 *pCapDescriptor
4826 )
4827 {
4828 *pCapDescriptor = NV_U64_MAX;
4829 }
4830
4831 /*
4832 * @brief Checks if IMEX channel support is present.
4833 */
4834 NvBool
osImexChannelIsSupported(void)4835 osImexChannelIsSupported(void)
4836 {
4837 return os_imex_channel_is_supported;
4838 }
4839
4840 /*
4841 * @brief Returns IMEX channel count.
4842 */
4843 NvS32
osImexChannelCount(void)4844 osImexChannelCount
4845 (
4846 void
4847 )
4848 {
4849 return os_imex_channel_count();
4850 }
4851
4852 /*
4853 * @brief Returns IMEX channel number.
4854 *
4855 * @param[in] descriptor OS specific descriptor to query channel number.
4856 *
4857 */
4858 NvS32
osImexChannelGet(NvU64 descriptor)4859 osImexChannelGet(NvU64 descriptor)
4860 {
4861 return os_imex_channel_get(descriptor);
4862 }
4863
4864 /*
4865 * @brief Generates random bytes which can be used as a universally unique
4866 * identifier.
4867 *
4868 * This function may sleep (interruptible).
4869 *
4870 * @param[out] pBytes Array of random bytes
4871 * @param[in] numBytes Size of the array
4872 */
4873 NV_STATUS
osGetRandomBytes(NvU8 * pBytes,NvU16 numBytes)4874 osGetRandomBytes
4875 (
4876 NvU8 *pBytes,
4877 NvU16 numBytes
4878 )
4879 {
4880 return os_get_random_bytes(pBytes, numBytes);
4881 }
4882
4883 /*
4884 * @brief Allocate wait queue
4885 *
4886 * @param[out] ppWq Wait queue
4887 */
4888 NV_STATUS
osAllocWaitQueue(OS_WAIT_QUEUE ** ppWq)4889 osAllocWaitQueue
4890 (
4891 OS_WAIT_QUEUE **ppWq
4892 )
4893 {
4894 return os_alloc_wait_queue(ppWq);
4895 }
4896
4897 /*
4898 * @brief Free wait queue
4899 *
4900 * @param[in] pWq Wait queue
4901 */
4902 void
osFreeWaitQueue(OS_WAIT_QUEUE * pWq)4903 osFreeWaitQueue
4904 (
4905 OS_WAIT_QUEUE *pWq
4906 )
4907 {
4908 os_free_wait_queue(pWq);
4909 }
4910
4911 /*
4912 * @brief Put thread to uninterruptible sleep
4913 *
4914 * @param[in] pWq Wait queue
4915 */
4916 void
osWaitUninterruptible(OS_WAIT_QUEUE * pWq)4917 osWaitUninterruptible
4918 (
4919 OS_WAIT_QUEUE *pWq
4920 )
4921 {
4922 os_wait_uninterruptible(pWq);
4923 }
4924
4925 /*
4926 * @brief Put thread to interruptible sleep
4927 *
4928 * @param[in] pWq Wait queue
4929 */
4930 void
osWaitInterruptible(OS_WAIT_QUEUE * pWq)4931 osWaitInterruptible
4932 (
4933 OS_WAIT_QUEUE *pWq
4934 )
4935 {
4936 os_wait_interruptible(pWq);
4937 }
4938
4939 /*
4940 * @brief Wake up thread from uninterruptible sleep
4941 *
4942 * @param[in] pWq Wait queue
4943 */
4944 void
osWakeUp(OS_WAIT_QUEUE * pWq)4945 osWakeUp
4946 (
4947 OS_WAIT_QUEUE *pWq
4948 )
4949 {
4950 os_wake_up(pWq);
4951 }
4952
4953 NV_STATUS
osReadPFPciConfigInVF(NvU32 addr,NvU32 * data)4954 osReadPFPciConfigInVF
4955 (
4956 NvU32 addr,
4957 NvU32 *data
4958 )
4959 {
4960 return NV_ERR_NOT_SUPPORTED;
4961 }
4962
4963 /*!
4964 * @brief Sends an MRQ (message-request) to BPMP
4965 *
4966 * The request, response, and ret parameters of this function correspond to the
4967 * components of the tegra_bpmp_message struct, which BPMP uses to receive
4968 * MRQs.
4969 *
4970 * @param[in] pOsGpuInfo OS specific GPU information pointer
4971 * @param[in] mrq MRQ_xxx ID specifying what is requested
4972 * @param[in] pRequestData Pointer to request input data
4973 * @param[in] requestDataSize Size of structure pointed to by pRequestData
4974 * @param[out] pResponseData Pointer to response output data
4975 * @param[in] responseDataSize Size of structure pointed to by pResponseData
4976 * @param[out] ret MRQ return code (from "ret" element of
4977 * tegra_bpmp_message struct)
4978 * @param[out] apiRet Return code from tegra_bpmp_transfer call
4979 *
4980 * @returns NV_OK if successful,
4981 * NV_ERR_NOT_SUPPORTED if the functionality is not available,
4982 * NV_ERR_INVALID_POINTER if the tegra_bpmp struct pointer could not
4983 * be obtained from nv, or
4984 * NV_ERR_GENERIC if the tegra_bpmp_transfer call failed (see apiRet
4985 * for Linux error code).
4986 */
4987 NV_STATUS
osTegraSocBpmpSendMrq(OS_GPU_INFO * pOsGpuInfo,NvU32 mrq,const void * pRequestData,NvU32 requestDataSize,void * pResponseData,NvU32 responseDataSize,NvS32 * pRet,NvS32 * pApiRet)4988 osTegraSocBpmpSendMrq
4989 (
4990 OS_GPU_INFO *pOsGpuInfo,
4991 NvU32 mrq,
4992 const void *pRequestData,
4993 NvU32 requestDataSize,
4994 void *pResponseData,
4995 NvU32 responseDataSize,
4996 NvS32 *pRet,
4997 NvS32 *pApiRet
4998 )
4999 {
5000 return NV_ERR_NOT_SUPPORTED;
5001 }
5002
5003 /*!
5004 * @brief Returns IMP-relevant data collected from other modules
5005 *
5006 * This function is basically a wrapper to call the unix/linux layer.
5007 *
5008 * @param[out] pTegraImpImportData Structure to receive the data
5009 *
5010 * @returns NV_OK if successful,
5011 * NV_ERR_BUFFER_TOO_SMALL if the array in TEGRA_IMP_IMPORT_DATA is
5012 * too small,
5013 * NV_ERR_NOT_SUPPORTED if the functionality is not available, or
5014 * other errors as may be returned by subfunctions.
5015 */
5016 NV_STATUS
osTegraSocGetImpImportData(TEGRA_IMP_IMPORT_DATA * pTegraImpImportData)5017 osTegraSocGetImpImportData
5018 (
5019 TEGRA_IMP_IMPORT_DATA *pTegraImpImportData
5020 )
5021 {
5022 return NV_ERR_NOT_SUPPORTED;
5023 }
5024
5025 /*!
5026 * @brief Tells BPMP whether or not RFL is valid
5027 *
5028 * Display HW generates an ok_to_switch signal which asserts when mempool
5029 * occupancy is high enough to be able to turn off memory long enough to
5030 * execute a dramclk frequency switch without underflowing display output.
5031 * ok_to_switch drives the RFL ("request for latency") signal in the memory
5032 * unit, and the switch sequencer waits for this signal to go active before
5033 * starting a dramclk switch. However, if the signal is not valid (e.g., if
5034 * display HW or SW has not been initialized yet), the switch sequencer ignores
5035 * the signal. This API tells BPMP whether or not the signal is valid.
5036 *
5037 * @param[in] pOsGpuInfo Per GPU Linux state
5038 * @param[in] bEnable True if RFL will be valid; false if invalid
5039 *
5040 * @returns NV_OK if successful,
5041 * NV_ERR_NOT_SUPPORTED if the functionality is not available, or
5042 * NV_ERR_GENERIC if some other kind of error occurred.
5043 */
5044 NV_STATUS
osTegraSocEnableDisableRfl(OS_GPU_INFO * pOsGpuInfo,NvBool bEnable)5045 osTegraSocEnableDisableRfl
5046 (
5047 OS_GPU_INFO *pOsGpuInfo,
5048 NvBool bEnable
5049 )
5050 {
5051 return NV_ERR_NOT_SUPPORTED;
5052 }
5053
5054 /*!
5055 * @brief Allocates a specified amount of ISO memory bandwidth for display
5056 *
5057 * floorBandwidthKBPS is the minimum required (i.e., floor) dramclk frequency
5058 * multiplied by the width of the pipe over which the display data will travel.
5059 * (It is understood that the bandwidth calculated by multiplying the clock
5060 * frequency by the pipe width will not be realistically achievable, due to
5061 * overhead in the memory subsystem. The infrastructure will not actually use
5062 * the bandwidth value, except to reverse the calculation to get the required
5063 * dramclk frequency.)
5064 *
5065 * This function is basically a wrapper to call the unix/linux layer.
5066 *
5067 * @param[in] pOsGpuInfo OS specific GPU information pointer
5068 * @param[in] averageBandwidthKBPS Amount of ISO memory bandwidth requested
5069 * @param[in] floorBandwidhtKBPS Min required dramclk freq * pipe width
5070 *
5071 * @returns NV_OK if successful,
5072 * NV_ERR_INSUFFICIENT_RESOURCES if one of the bandwidth values is too
5073 * high, and bandwidth cannot be allocated,
5074 * NV_ERR_NOT_SUPPORTED if the functionality is not available, or
5075 * NV_ERR_GENERIC if some other kind of error occurred.
5076 */
5077 NV_STATUS
osTegraAllocateDisplayBandwidth(OS_GPU_INFO * pOsGpuInfo,NvU32 averageBandwidthKBPS,NvU32 floorBandwidthKBPS)5078 osTegraAllocateDisplayBandwidth
5079 (
5080 OS_GPU_INFO *pOsGpuInfo,
5081 NvU32 averageBandwidthKBPS,
5082 NvU32 floorBandwidthKBPS
5083 )
5084 {
5085 return NV_ERR_NOT_SUPPORTED;
5086 }
5087
5088 /*!
5089 * @brief Creates or sets up platform specific nano second resolution timer
5090 *
5091 * @param[in] pOsGpuInfo OS specific GPU information pointer
5092 * @param[in] pTmrEvent Pointer to timer event information
5093 * @param[in/out] pTimer pointer to hold high resolution timer object
5094 */
5095 NV_STATUS
osCreateNanoTimer(OS_GPU_INFO * pOsGpuInfo,void * pTmrEvent,void ** pTimer)5096 osCreateNanoTimer
5097 (
5098 OS_GPU_INFO *pOsGpuInfo,
5099 void *pTmrEvent,
5100 void **pTimer
5101 )
5102 {
5103 nv_create_nano_timer(pOsGpuInfo, pTmrEvent, (nv_nano_timer_t **)pTimer);
5104 return NV_OK;
5105 }
5106
5107 /*!
5108 * @brief Starts platform specific nano second resolution timer
5109 *
5110 * @param[in] pOsGpuInfo OS specific GPU information pointer
5111 * @param[in] pTimer pointer to high resolution timer object
5112 * @param[in] timeNs Relative time in nano seconds
5113 */
5114 NV_STATUS
osStartNanoTimer(OS_GPU_INFO * pOsGpuInfo,void * pTimer,NvU64 timeNs)5115 osStartNanoTimer
5116 (
5117 OS_GPU_INFO *pOsGpuInfo,
5118 void *pTimer,
5119 NvU64 timeNs
5120 )
5121 {
5122 nv_start_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer, timeNs);
5123 return NV_OK;
5124 }
5125
5126 /*!
5127 * @brief Cancels platform specific nano second resolution timer
5128 *
5129 * @param[in] pOsGpuInfo OS specific GPU information pointer
5130 * @param[in] pTimer pointer to timer object
5131 */
5132 NV_STATUS
osCancelNanoTimer(OS_GPU_INFO * pOsGpuInfo,void * pTimer)5133 osCancelNanoTimer
5134 (
5135 OS_GPU_INFO *pOsGpuInfo,
5136 void *pTimer
5137 )
5138 {
5139 nv_cancel_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer);
5140 return NV_OK;
5141 }
5142
5143 /*!
5144 * @brief Destroys & cancels platform specific nano second resolution timer
5145 *
5146 *
5147 * @param[in] pGpu Device of interest
5148 * @param[in] pTimer pointer to timer object
5149 */
5150 NV_STATUS
osDestroyNanoTimer(OS_GPU_INFO * pOsGpuInfo,void * pTimer)5151 osDestroyNanoTimer
5152 (
5153 OS_GPU_INFO *pOsGpuInfo,
5154 void *pTimer
5155 )
5156 {
5157 nv_destroy_nano_timer(pOsGpuInfo, (nv_nano_timer_t *)pTimer);
5158 return NV_OK;
5159 }
5160
5161 /*!
5162 * @brief Get number of dpAux instances.
5163 * It is wrapper function to call unix/linux layer.
5164 *
5165 * @param[in] pGpu GPU object pointer
5166 * @param[out] pNumIntances Number of dpAux instances.
5167 *
5168 * @returns NV_STATUS, NV_OK if success,
5169 * NV_ERR_GENERIC, if error
5170 * NV_ERR_NOT_SUPPORTED, for unsupported platforms
5171 */
5172
5173 NV_STATUS
osGetTegraNumDpAuxInstances(OS_GPU_INFO * pOsGpuInfo,NvU32 * pNumIntances)5174 osGetTegraNumDpAuxInstances
5175 (
5176 OS_GPU_INFO *pOsGpuInfo,
5177 NvU32 *pNumIntances
5178 )
5179 {
5180 return NV_ERR_NOT_SUPPORTED;
5181 }
5182
5183 /*
5184 * @brief Return the priv Data of current IRQ.
5185 * It is wrapper function to call unix/linux layer.
5186 *
5187 * @param[in] pGpu Device of interest
5188 * @param[out] pPrivData privData of current IRQ
5189 *
5190 * @returns NV_STATUS, NV_OK if success,
5191 * NV_ERR_GENERIC, if error
5192 * NV_ERR_NOT_SUPPORTED, for unsupported platforms
5193 */
5194 NV_STATUS
osGetCurrentIrqPrivData(OS_GPU_INFO * pOsGpuInfo,NvU32 * pPrivData)5195 osGetCurrentIrqPrivData
5196 (
5197 OS_GPU_INFO *pOsGpuInfo,
5198 NvU32 *pPrivData
5199 )
5200 {
5201 return NV_ERR_NOT_SUPPORTED;
5202 }
5203
5204 /*!
5205 * @brief Get the brightness level
5206 * It is wrapper function to call unix/linux layer.
5207 *
5208 * @param[in] pGpu GPU object pointer
5209 * @param[out] brightness Pointer to brightness level
5210 *
5211 * @returns NV_STATUS, NV_OK if success,
5212 * NV_ERR_GENERIC, if error
5213 * NV_ERR_NOT_SUPPORTED, for unsupported platforms
5214 */
5215 NV_STATUS
osGetTegraBrightnessLevel(OS_GPU_INFO * pOsGpuInfo,NvU32 * brightness)5216 osGetTegraBrightnessLevel
5217 (
5218 OS_GPU_INFO *pOsGpuInfo,
5219 NvU32 *brightness
5220 )
5221 {
5222 return NV_ERR_NOT_SUPPORTED;
5223 }
5224
5225 /*!
5226 * @brief Set the brightness level
5227 * It is wrapper function to call unix/linux layer.
5228 *
5229 * @param[in] pGpu GPU object pointer
5230 * @param[out] brightness brightness level
5231 *
5232 * @returns NV_STATUS, NV_OK if success,
5233 * NV_ERR_GENERIC, if error
5234 * NV_ERR_NOT_SUPPORTED, for unsupported platforms
5235 */
5236 NV_STATUS
osSetTegraBrightnessLevel(OS_GPU_INFO * pOsGpuInfo,NvU32 brightness)5237 osSetTegraBrightnessLevel
5238 (
5239 OS_GPU_INFO *pOsGpuInfo,
5240 NvU32 brightness
5241 )
5242 {
5243 return NV_ERR_NOT_SUPPORTED;
5244 }
5245
5246 /* @brief Gets syncpoint aperture information
5247 *
5248 * @param[in] OS_GPU_INFO OS specific GPU information pointer
5249 * @param[in] syncpointId
5250 * @param[out] *physAddr
5251 * @param[out] *limit
5252 * @param[out] *offset
5253 */
5254 NV_STATUS
osGetSyncpointAperture(OS_GPU_INFO * pOsGpuInfo,NvU32 syncpointId,NvU64 * physAddr,NvU64 * limit,NvU32 * offset)5255 osGetSyncpointAperture
5256 (
5257 OS_GPU_INFO *pOsGpuInfo,
5258 NvU32 syncpointId,
5259 NvU64 *physAddr,
5260 NvU64 *limit,
5261 NvU32 *offset
5262 )
5263 {
5264 return NV_ERR_NOT_SUPPORTED;
5265 }
5266
5267 /*!
5268 * @brief Enable PCIe AtomicOp Requester Enable and return
5269 * the completer side capabilities that the requester can send.
5270 *
5271 * @param[in] pOsGpuInfo OS_GPU_INFO OS specific GPU information pointer
5272 * @param[out] pMask mask of supported atomic size, including one or more of:
5273 * OS_PCIE_CAP_MASK_REQ_ATOMICS_32
5274 * OS_PCIE_CAP_MASK_REQ_ATOMICS_64
5275 * OS_PCIE_CAP_MASK_REQ_ATOMICS_128
5276 *
5277 * @returns NV_STATUS, NV_OK if success
5278 * NV_ERR_NOT_SUPPORTED if platform doesn't support this
5279 * feature.
5280 * NV_ERR_GENERIC for any other error
5281 */
5282
5283 NV_STATUS
osConfigurePcieReqAtomics(OS_GPU_INFO * pOsGpuInfo,NvU32 * pMask)5284 osConfigurePcieReqAtomics
5285 (
5286 OS_GPU_INFO *pOsGpuInfo,
5287 NvU32 *pMask
5288 )
5289 {
5290 if (pMask)
5291 {
5292 *pMask = 0U;
5293 if (pOsGpuInfo)
5294 {
5295 if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
5296 OS_INTF_PCIE_REQ_ATOMICS_32BIT) == NV_OK)
5297 *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_32;
5298 if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
5299 OS_INTF_PCIE_REQ_ATOMICS_64BIT) == NV_OK)
5300 *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_64;
5301 if (os_enable_pci_req_atomics(pOsGpuInfo->handle,
5302 OS_INTF_PCIE_REQ_ATOMICS_128BIT) == NV_OK)
5303 *pMask |= OS_PCIE_CAP_MASK_REQ_ATOMICS_128;
5304
5305 if (*pMask != 0)
5306 return NV_OK;
5307 }
5308 }
5309 return NV_ERR_NOT_SUPPORTED;
5310 }
5311
5312 /*!
5313 * @brief Check GPU is accessible or not
5314 *
5315 * @param[in] pGpu GPU object pointer
5316 *
5317 * @returns NVBool, Returns TRUE if the GPU is accessible,
5318 * FALSE, if error
5319 */
5320 NvBool
osIsGpuAccessible(OBJGPU * pGpu)5321 osIsGpuAccessible
5322 (
5323 OBJGPU *pGpu
5324 )
5325 {
5326 return nv_is_gpu_accessible(NV_GET_NV_STATE(pGpu));
5327 }
5328
5329 /*!
5330 * @brief Check whether GPU has received a shutdown notification from the OS
5331 */
5332 NvBool
osIsGpuShutdown(OBJGPU * pGpu)5333 osIsGpuShutdown
5334 (
5335 OBJGPU *pGpu
5336 )
5337 {
5338 nv_state_t *nv = NV_GET_NV_STATE(pGpu);
5339 return nv ? nv->is_shutdown : NV_TRUE;
5340 }
5341
5342 /*!
5343 * @brief Check GPU OS info matches
5344 *
5345 * @param[in] pGpu GPU object pointer
5346 *
5347 * @returns NVBool, Returns TRUE if matched.
5348 */
5349 NvBool
osMatchGpuOsInfo(OBJGPU * pGpu,void * pOsInfo)5350 osMatchGpuOsInfo
5351 (
5352 OBJGPU *pGpu,
5353 void *pOsInfo
5354 )
5355 {
5356 return nv_match_gpu_os_info(NV_GET_NV_STATE(pGpu), pOsInfo);
5357 }
5358
5359 /*!
5360 * @brief Release GPU OS info.
5361 *
5362 * @param[in] pOsInfo GPU OS info pointer
5363 *
5364 * @returns void
5365 */
5366 void
osReleaseGpuOsInfo(void * pOsInfo)5367 osReleaseGpuOsInfo
5368 (
5369 void *pOsInfo
5370 )
5371 {
5372 nv_put_file_private(pOsInfo);
5373 }
5374
5375 /*!
5376 * @brief Get free, total memory of a NUMA node by NUMA node ID from kernel.
5377 *
5378 * @param[in] numaId NUMA node ID.
5379 * @param[out] free_memory_bytes free memory in bytes.
5380 * @param[out] total_memory_bytes total memory in bytes.
5381 *
5382 */
5383 void
osGetNumaMemoryUsage(NvS32 numaId,NvU64 * free_memory_bytes,NvU64 * total_memory_bytes)5384 osGetNumaMemoryUsage
5385 (
5386 NvS32 numaId,
5387 NvU64 *free_memory_bytes,
5388 NvU64 *total_memory_bytes
5389 )
5390 {
5391 NV_STATUS status = os_get_numa_node_memory_usage(numaId,
5392 free_memory_bytes,
5393 total_memory_bytes);
5394 NV_ASSERT(status == NV_OK);
5395 }
5396
5397 /*!
5398 * @brief Add GPU memory as a NUMA node.
5399 *
5400 * @param[in/out] pOsGpuInfo OS specific GPU information pointer
5401 * @param[in] offset start offset of the partition within FB
5402 * @param[in] size size of the partition
5403 * @param[out] pNumaNodeId OS NUMA node id for the added memory.
5404 *
5405 * @returns NV_OK if all is okay. Otherwise an error-specific value.
5406 *
5407 */
5408 NV_STATUS
osNumaAddGpuMemory(OS_GPU_INFO * pOsGpuInfo,NvU64 offset,NvU64 size,NvU32 * pNumaNodeId)5409 osNumaAddGpuMemory
5410 (
5411 OS_GPU_INFO *pOsGpuInfo,
5412 NvU64 offset,
5413 NvU64 size,
5414 NvU32 *pNumaNodeId
5415 )
5416 {
5417 nv_state_t *nv = pOsGpuInfo;
5418
5419 return os_numa_add_gpu_memory(nv->handle, offset, size, pNumaNodeId);
5420 }
5421
5422 /*!
5423 * @brief Remove a particular SMC partition's GPU memory from OS kernel.
5424 *
5425 * Remove GPU memory from the OS kernel that is earlier added as a NUMA node
5426 * to the kernel in platforms where GPU is coherently connected to the CPU.
5427 *
5428 * @param[in/out] pOsGpuInfo OS_GPU_INFO OS specific GPU information pointer
5429 * @param[in] offset start offset of the partition within FB
5430 * @param[in] size size of the partition
5431 * @param[in] numaNodeId OS NUMA node id of the memory to be removed.
5432 *
5433 */
5434 void
osNumaRemoveGpuMemory(OS_GPU_INFO * pOsGpuInfo,NvU64 offset,NvU64 size,NvU32 numaNodeId)5435 osNumaRemoveGpuMemory
5436 (
5437 OS_GPU_INFO *pOsGpuInfo,
5438 NvU64 offset,
5439 NvU64 size,
5440 NvU32 numaNodeId
5441 )
5442 {
5443 nv_state_t *nv = pOsGpuInfo;
5444
5445 NV_STATUS status = os_numa_remove_gpu_memory(nv->handle, offset, size, numaNodeId);
5446 NV_ASSERT(status == NV_OK);
5447
5448 return;
5449 }
5450
5451 NvBool
osDmabufIsSupported(void)5452 osDmabufIsSupported(void)
5453 {
5454 return os_dma_buf_enabled;
5455 }
5456
5457 NV_STATUS
osGetEgmInfo(OBJGPU * pGpu,NvU64 * pPhysAddr,NvU64 * pSize,NvS32 * pNodeId)5458 osGetEgmInfo
5459 (
5460 OBJGPU *pGpu,
5461 NvU64 *pPhysAddr,
5462 NvU64 *pSize,
5463 NvS32 *pNodeId
5464 )
5465 {
5466 return nv_get_egm_info(NV_GET_NV_STATE(pGpu), pPhysAddr, pSize, pNodeId);
5467 }
5468
5469 /*!
5470 * @brief Offline (i.e., blacklist) the page containing a given address from OS kernel.
5471 *
5472 * @param[in] address Address (SPA) of the page to be offlined
5473 *
5474 */
5475 NV_STATUS
osOfflinePageAtAddress(NvU64 address)5476 osOfflinePageAtAddress
5477 (
5478 NvU64 address
5479 )
5480 {
5481 return os_offline_page_at_address(address);
5482 }
5483
osAllocatedRmClient(void * pOsInfo)5484 void osAllocatedRmClient(void *pOsInfo)
5485 {
5486 nv_file_private_t* nvfp = (nv_file_private_t*)pOsInfo;
5487
5488 if (nvfp != NULL)
5489 nvfp->bCleanupRmapi = NV_TRUE;
5490 }
5491