1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021-2022 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include <nv.h> 25 #include <nv-priv.h> 26 #include <osapi.h> 27 #include <core/thread_state.h> 28 #include <core/locks.h> 29 #include <gpu/gpu.h> 30 #include "kernel/gpu/intr/intr.h" 31 #include "gpu/bif/kernel_bif.h" 32 #include "gpu/mmu/kern_gmmu.h" 33 #include "gpu/disp/kern_disp.h" 34 #include <nv_sriov_defines.h> 35 #include "objtmr.h" 36 37 38 static NvBool osInterruptPending( 39 OBJGPU *pGpu, 40 NvBool *serviced, 41 THREAD_STATE_NODE *pThreadState 42 ) 43 { 44 POBJDISP pDisp; 45 KernelDisplay *pKernelDisplay; 46 NvBool pending, sema_release; 47 THREAD_STATE_NODE threadState; 48 NvU32 gpuMask, gpuInstance; 49 Intr *pIntr = NULL; 50 MC_ENGINE_BITVECTOR intr0Pending; 51 MC_ENGINE_BITVECTOR intr1Pending; 52 53 *serviced = NV_FALSE; 54 pending = NV_FALSE; 55 sema_release = NV_TRUE; 56 OBJGPU *pDeviceLockGpu = pGpu; 57 NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer 58 PORT_MEM_ALLOCATOR *pIsrAllocator; 59 60 // 61 // GPU interrupt servicing ("top half") 62 // 63 // Top-level processing of GPU interrupts is performed using the 64 // steps below; although the code is straight forward, there 65 // are a few points to be aware of: 66 // 67 // 1) The GPUs lock is acquired for two reasons: to allow 68 // looping over GPUs atomically in SLI and to sanity 69 // check the PCI configuration space of any initialized 70 // GPUs. If the acquisition fails, the early return 71 // is acceptable since GPU interrupts are disabled while 72 // the lock is held; note that returning success 73 // in this case could interfere with the processing 74 // of third-party device interrupts if the IRQ is shared. 75 // Due to the above, some interrupts may be reported as 76 // unhandled if invocations of the ISR registered with 77 // the kernel are not serialized. This is bad, but 78 // ignored by currently supported kernels, provided most 79 // interrupts are handled. 80 // 81 // 2) Since acquisition of the lock disables interrupts 82 // on all initialized GPUs, NV_PMC_INTR_EN_0 can not be 83 // relied up on to determine whether interrupts are 84 // expected from a given GPU. The code below is therefore 85 // forced to rely on software state. NV_PMC_INTR_EN_0 86 // is read only as a sanity check to guard against 87 // invalid GPU state (lack of PCI memory access, etc.). 88 // 89 // 3) High priority interrupts (VBLANK, etc.), are serviced in 90 // this function, service of all other interrupts is 91 // deferred until a bottom half. If a bottom half needs 92 // to be scheduled, release of the GPUs lock is 93 // likewise deferred until completion of the bottom half. 94 // 95 // 4) To reduce the risk of starvation, an effort is made to 96 // consolidate processing of interrupts pending on 97 // all GPUs sharing a given IRQ. 98 // 99 // 5) Care is taken to ensure that the consolidated interrupt 100 // processing is performed in the context of a GPU 101 // that has interrupts pending. Else if additional ISR 102 // processing via a bottom-half is required, this 103 // bottom-half ISR might race against the GPU's shut-down 104 // path. 105 // 106 107 pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); 108 tlsIsrInit(pIsrAllocator); 109 110 // For SWRL granular locking process the countdown timer interrupt. 111 if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_SWRL_GRANULAR_LOCKING)) 112 { 113 threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); 114 115 gpuMask = gpumgrGetGpuMask(pDeviceLockGpu); 116 117 gpuInstance = 0; 118 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) 119 { 120 pIntr = GPU_GET_INTR(pGpu); 121 122 if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) 123 { 124 // If interrupt enable is garbage the GPU is probably in a bad state 125 if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX) 126 { 127 continue; 128 } 129 130 intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState); 131 POBJTMR pTmr = GPU_GET_TIMER(pGpu); 132 *serviced = tmrServiceSwrlWrapper(pGpu, pTmr, &intr0Pending, &threadState); 133 } 134 } 135 136 threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); 137 } 138 139 // LOCK: try to acquire GPUs lock 140 if (rmDeviceGpuLocksAcquire(pDeviceLockGpu, GPU_LOCK_FLAGS_COND_ACQUIRE, RM_LOCK_MODULES_ISR) == NV_OK) 141 { 142 threadStateInitISRAndDeferredIntHandler(&threadState, 143 pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR); 144 145 gpuMask = gpumgrGetGpuMask(pDeviceLockGpu); 146 147 gpuInstance = 0; 148 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) 149 { 150 pIntr = GPU_GET_INTR(pGpu); 151 pDisp = GPU_GET_DISP(pGpu); 152 pKernelDisplay = GPU_GET_KERNEL_DISPLAY(pGpu); 153 154 if ((pDisp != NULL) && pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) 155 { 156 } 157 else if ((pIntr != NULL) && INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) 158 { 159 // If interrupt enable is garbage the GPU is probably in a bad state 160 if (intrGetIntrEnFromHw_HAL(pGpu, pIntr, &threadState) > INTERRUPT_TYPE_MAX) 161 continue; 162 163 intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState); 164 if (bitVectorTest(&intr0Pending, MC_ENGINE_IDX_DISP)) 165 { 166 if (pKernelDisplay != NULL) 167 { 168 kdispServiceVblank_HAL(pGpu, pKernelDisplay, 0, 169 (VBLANK_STATE_PROCESS_LOW_LATENCY | 170 VBLANK_STATE_PROCESS_CALLED_FROM_ISR), 171 &threadState); 172 *serviced = NV_TRUE; 173 intrGetPendingStall_HAL(pGpu, pIntr, &intr0Pending, &threadState); 174 } 175 } 176 177 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) && 178 !pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) 179 { 180 pIntr = GPU_GET_INTR(pGpu); 181 if (pIntr != NULL) 182 { 183 NvBool bCtxswLog = NV_FALSE; 184 intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); 185 intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog); 186 } 187 } 188 189 if (!bitVectorTestAllCleared(&intr0Pending) || 190 !bitVectorTestAllCleared(&intr1Pending)) 191 { 192 pending = NV_TRUE; 193 sema_release = NV_FALSE; 194 } 195 } 196 } 197 threadStateFreeISRAndDeferredIntHandler(&threadState, 198 pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR); 199 200 if (sema_release) 201 { 202 NV_ASSERT(!pending); 203 204 // UNLOCK: release GPUs lock 205 rmDeviceGpuLocksRelease(pDeviceLockGpu, GPUS_LOCK_FLAGS_NONE, NULL); 206 } 207 else 208 { 209 rmDeviceGpuLockSetOwner(pDeviceLockGpu, GPUS_LOCK_OWNER_PENDING_DPC_REFRESH); 210 } 211 } 212 213 if (pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) && 214 pDeviceLockGpu->getProperty(pDeviceLockGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) 215 { 216 threadStateInitISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); 217 218 gpuMask = gpumgrGetGpuMask(pDeviceLockGpu); 219 gpuInstance = 0; 220 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) 221 { 222 pIntr = GPU_GET_INTR(pGpu); 223 if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))) 224 { 225 NvBool bCtxswLog = NV_FALSE; 226 intrGetPendingNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); 227 intrCheckFecsEventbufferPending(pGpu, pIntr, &intr1Pending, &bCtxswLog); 228 if (!bitVectorTestAllCleared(&intr1Pending)) 229 { 230 intrServiceNonStall_HAL(pGpu, pIntr, &intr1Pending, &threadState); 231 *serviced = NV_TRUE; 232 } 233 } 234 } 235 threadStateFreeISRLockless(&threadState, pDeviceLockGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); 236 } 237 238 tlsIsrDestroy(pIsrAllocator); 239 portMemAllocatorRelease(pIsrAllocator); 240 241 return pending; 242 } 243 244 NV_STATUS osIsr( 245 OBJGPU *pGpu 246 ) 247 { 248 NV_STATUS status = NV_OK; 249 nv_state_t *nv = NV_GET_NV_STATE(pGpu); 250 nv_priv_t *nvp = NV_GET_NV_PRIV(nv); 251 NvBool pending = NV_FALSE; 252 NvBool serviced = NV_FALSE; 253 Intr *pIntr; 254 255 if (nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) 256 { 257 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) 258 { 259 pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */); 260 } 261 else 262 { 263 pIntr = GPU_GET_INTR(pGpu); 264 if (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr)) 265 { 266 KernelBif *pKernelBif = GPU_GET_KERNEL_BIF(pGpu); 267 pending = osInterruptPending(pGpu, &serviced, NULL /* threadstate */); 268 kbifCheckAndRearmMSI(pGpu, pKernelBif); 269 } 270 } 271 } 272 273 if (!pending && (IS_VIRTUAL(pGpu) || !serviced)) 274 status = NV_ERR_NO_INTR_PENDING; 275 else if (pending) 276 status = NV_WARN_MORE_PROCESSING_REQUIRED; 277 278 return status; 279 } 280 281 /* 282 * Helper function to determine when the RM SEMA/GPUS LOCK should toggle 283 * interrupts. Based on the state of the GPU - we must add cases here as we 284 * discover them. 285 * 286 * Noteworthy special cases: 287 * 288 * - Suspend/resume: the GPU could still be suspended and not accessible 289 * on the bus, while passive-level threads need to grab the GPUs 290 * lock, or other GPUs are being resumed and triggering interrupts. 291 * 292 * - SLI state transitions: interrupts are disabled manually prior to 293 * removing GPUs from the lock mask leading up to SLI link/unlink 294 * operations on UNIX, but since the GPUs lock is not held by design in 295 * these paths, it needs to be ensured that GPUs lock acquisitions 296 * occurring aynchronously do not re-enable interrupts on any of the 297 * GPUs undergoing the SLI state transition. 298 * 299 * @param[in] pGpu OBJGPU pointer 300 * 301 * @return NV_TRUE if the RM SEMA/GPUS LOCK should toggle interrupts, NV_FALSE 302 * otherwise. 303 */ 304 NvBool osLockShouldToggleInterrupts(OBJGPU *pGpu) 305 { 306 return (!pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_PM_CODEPATH) && 307 gpuIsStateLoaded(pGpu) && 308 !pGpu->getProperty(pGpu, PDB_PROP_GPU_IN_SLI_LINK_CODEPATH)); 309 } 310 311 void osEnableInterrupts(OBJGPU *pGpu) 312 { 313 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) 314 { 315 // enable irq through os call 316 nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_TRUE); 317 return; 318 } 319 else 320 { 321 Intr *pIntr = GPU_GET_INTR(pGpu); 322 NvU32 intrEn; 323 324 if (!pIntr->getProperty(pIntr, PDB_PROP_INTR_USE_INTR_MASK_FOR_LOCKING)) 325 NV_ASSERT(intrGetIntrEnFromHw_HAL(pGpu, pIntr, NULL) == INTERRUPT_TYPE_DISABLED); 326 327 intrEn = intrGetIntrEn(pIntr); 328 intrSetIntrEnInHw_HAL(pGpu, pIntr, intrEn, NULL); 329 330 if (pIntr != NULL) 331 { 332 intrSetStall_HAL(pGpu, pIntr, intrEn, NULL); 333 } 334 335 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED)) 336 { 337 if (pIntr != NULL) 338 { 339 intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL); 340 } 341 } 342 343 } 344 } 345 346 void osDisableInterrupts( 347 OBJGPU *pGpu, 348 NvBool bIsr 349 ) 350 { 351 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) 352 { 353 // disable irq through os call 354 nv_control_soc_irqs(NV_GET_NV_STATE(pGpu), NV_FALSE); 355 return; 356 } 357 else 358 { 359 Intr *pIntr = GPU_GET_INTR(pGpu); 360 NvU32 new_intr_en_0 = INTERRUPT_TYPE_DISABLED; 361 362 intrSetIntrEnInHw_HAL(pGpu, pIntr, new_intr_en_0, NULL); 363 364 if (pIntr != NULL) 365 { 366 intrSetStall_HAL(pGpu, pIntr, new_intr_en_0, NULL); 367 } 368 369 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED)) 370 { 371 if (pIntr != NULL) 372 { 373 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) 374 { 375 intrRestoreNonStall_HAL(pGpu, pIntr, intrGetIntrEn(pIntr), NULL); 376 } 377 else 378 { 379 intrRestoreNonStall_HAL(pGpu, pIntr, new_intr_en_0, NULL); 380 } 381 } 382 } 383 } 384 } 385 386 static void RmIsrBottomHalf( 387 nv_state_t *pNv 388 ) 389 { 390 OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); 391 THREAD_STATE_NODE threadState; 392 OS_THREAD_HANDLE threadId; 393 NvU32 gpuMask, gpuInstance; 394 OBJGPU *pDeviceLockGpu = pGpu; 395 Intr *pIntr = NULL; 396 POBJDISP pDisp = NULL; 397 NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer 398 PORT_MEM_ALLOCATOR *pIsrAllocator; 399 400 pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); 401 tlsIsrInit(pIsrAllocator); 402 403 // 404 // The owning thread changes as the ISR acquires the GPUs lock, 405 // but the bottom half releases it. Refresh the ThreadId owner to be 406 // correct here for the bottom half context. 407 // 408 osGetCurrentThread(&threadId); 409 rmDeviceGpuLockSetOwner(pDeviceLockGpu, threadId); 410 411 gpuMask = gpumgrGetGpuMask(pGpu); 412 413 gpuInstance = 0; 414 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL) 415 { 416 417 threadStateInitISRAndDeferredIntHandler(&threadState, 418 pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER); 419 420 pIntr = GPU_GET_INTR(pGpu); 421 pDisp = GPU_GET_DISP(pGpu); 422 423 // 424 // Call disp service incase of SOC Display, 425 // TODO : with multi interrupt handling based on irq aux interrupts are serviced by dpAuxService 426 // See JIRA task TDS-4253. 427 // 428 if ((pDisp != NULL) && pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY)) 429 { 430 } 431 else if ((pIntr != NULL) && (INTERRUPT_TYPE_HARDWARE == intrGetIntrEn(pIntr))) 432 { 433 intrServiceStall_HAL(pGpu, pIntr); 434 435 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_ENABLED) && 436 !pGpu->getProperty(pGpu, PDB_PROP_GPU_ALTERNATE_TREE_HANDLE_LOCKLESS)) 437 { 438 MC_ENGINE_BITVECTOR intrPending; 439 intrServiceNonStall_HAL(pGpu, pIntr, &intrPending, &threadState); 440 } 441 } 442 443 threadStateFreeISRAndDeferredIntHandler(&threadState, 444 pGpu, THREAD_STATE_FLAGS_IS_ISR_DEFERRED_INT_HANDLER); 445 } 446 447 // UNLOCK: release GPUs lock 448 rmDeviceGpuLocksRelease(pDeviceLockGpu, GPUS_LOCK_FLAGS_NONE, NULL); 449 450 tlsIsrDestroy(pIsrAllocator); 451 portMemAllocatorRelease(pIsrAllocator); 452 } 453 454 static void RmIsrBottomHalfUnlocked( 455 nv_state_t *pNv 456 ) 457 { 458 OBJGPU *pGpu = NV_GET_NV_PRIV_PGPU(pNv); 459 Intr *pIntr; 460 THREAD_STATE_NODE threadState; 461 462 // In the GSP client scenario, the fatal fault interrupt is not shared 463 // by UVM and CPU-RM. Instead, it is handled entirely by GSP-RM. We 464 // therefore do not expect this function to be called. But if it is, bail 465 // without attempting to service interrupts. 466 if (IS_GSP_CLIENT(pGpu)) 467 { 468 return; 469 } 470 471 // Grab GPU lock here as this kthread-item was enqueued without grabbing GPU lock 472 if (rmDeviceGpuLocksAcquire(pGpu, GPUS_LOCK_FLAGS_NONE, RM_LOCK_MODULES_DPC) == NV_OK) 473 { 474 if (FULL_GPU_SANITY_CHECK(pGpu)) 475 { 476 pIntr = GPU_GET_INTR(pGpu); 477 478 threadStateInit(&threadState, THREAD_STATE_FLAGS_NONE); 479 480 if (intrGetIntrEn(pIntr) != INTERRUPT_TYPE_DISABLED) 481 { 482 MC_ENGINE_BITVECTOR intrPending; 483 intrGetPendingStall_HAL(pGpu, pIntr, &intrPending, &threadState); 484 intrServiceNonStallBottomHalf(pGpu, pIntr, &intrPending, &threadState); 485 } 486 487 threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); 488 } 489 490 rmDeviceGpuLocksRelease(pGpu, GPUS_LOCK_FLAGS_NONE, NULL); 491 } 492 } 493 494 NvBool NV_API_CALL rm_isr( 495 nvidia_stack_t *sp, 496 nv_state_t *nv, 497 NvU32 *NeedBottomHalf 498 ) 499 { 500 NV_STATUS status; 501 nv_priv_t *nvp = NV_GET_NV_PRIV(nv); 502 OBJGPU *pGpu; 503 NvBool retval; 504 void *fp; 505 506 if ((nvp->flags & NV_INIT_FLAG_GPU_STATE_LOAD) == 0) 507 { 508 return NV_FALSE; 509 } 510 511 pGpu = NV_GET_NV_PRIV_PGPU(nv); 512 if (pGpu == NULL) 513 { 514 return NV_FALSE; 515 } 516 517 NV_ENTER_RM_RUNTIME(sp,fp); 518 519 // call actual isr function here 520 status = isrWrapper(pGpu->testIntr, pGpu); 521 522 switch (status) 523 { 524 case NV_OK: 525 *NeedBottomHalf = NV_FALSE; 526 retval = NV_TRUE; 527 break; 528 case NV_WARN_MORE_PROCESSING_REQUIRED: 529 *NeedBottomHalf = NV_TRUE; 530 retval = NV_TRUE; 531 break; 532 case NV_ERR_NO_INTR_PENDING: 533 default: 534 *NeedBottomHalf = NV_FALSE; 535 retval = NV_FALSE; 536 break; 537 } 538 539 NV_EXIT_RM_RUNTIME(sp,fp); 540 541 return retval; 542 } 543 544 void NV_API_CALL rm_isr_bh( 545 nvidia_stack_t *sp, 546 nv_state_t *pNv 547 ) 548 { 549 void *fp; 550 551 NV_ENTER_RM_RUNTIME(sp,fp); 552 553 RmIsrBottomHalf(pNv); 554 555 NV_EXIT_RM_RUNTIME(sp,fp); 556 } 557 558 void NV_API_CALL rm_isr_bh_unlocked( 559 nvidia_stack_t *sp, 560 nv_state_t *pNv 561 ) 562 { 563 void *fp; 564 565 NV_ENTER_RM_RUNTIME(sp,fp); 566 567 RmIsrBottomHalfUnlocked(pNv); 568 569 NV_EXIT_RM_RUNTIME(sp,fp); 570 } 571 572 NV_STATUS NV_API_CALL rm_gpu_copy_mmu_faults( 573 nvidia_stack_t *sp, 574 nv_state_t *nv, 575 NvU32 *faultsCopied 576 ) 577 { 578 NV_STATUS status = NV_OK; 579 OBJGPU *pGpu; 580 void *fp; 581 582 NV_ENTER_RM_RUNTIME(sp,fp); 583 584 pGpu = NV_GET_NV_PRIV_PGPU(nv); 585 if (pGpu == NULL || faultsCopied == NULL) 586 { 587 status = NV_ERR_OBJECT_NOT_FOUND; 588 goto done; 589 } 590 591 if (IS_GSP_CLIENT(pGpu)) 592 { 593 // Non-replayable faults are copied to the client shadow buffer by GSP-RM. 594 status = NV_OK; 595 goto done; 596 } 597 598 done: 599 NV_EXIT_RM_RUNTIME(sp,fp); 600 601 return status; 602 } 603 604 // 605 // Use this call when MMU faults needs to be copied 606 // outisde of RM lock. 607 // 608 static NV_STATUS _rm_gpu_copy_mmu_faults_unlocked( 609 OBJGPU *pGpu, 610 NvU32 *pFaultsCopied, 611 THREAD_STATE_NODE *pThreadState 612 ) 613 { 614 615 return NV_OK; 616 } 617 618 // 619 // Wrapper to handle calls to copy mmu faults 620 // 621 NV_STATUS rm_gpu_handle_mmu_faults( 622 nvidia_stack_t *sp, 623 nv_state_t *nv, 624 NvU32 *faultsCopied 625 ) 626 { 627 NvU32 status = NV_OK; 628 OBJGPU *pGpu; 629 void *fp; 630 631 NV_ENTER_RM_RUNTIME(sp,fp); 632 633 *faultsCopied = 0; 634 pGpu = NV_GET_NV_PRIV_PGPU(nv); 635 636 if (pGpu == NULL) 637 { 638 NV_EXIT_RM_RUNTIME(sp,fp); 639 return NV_ERR_OBJECT_NOT_FOUND; 640 } 641 642 { 643 KernelGmmu *pKernelGmmu; 644 PORT_MEM_ALLOCATOR *pIsrAllocator; 645 THREAD_STATE_NODE threadState; 646 NvU8 stackAllocator[TLS_ISR_ALLOCATOR_SIZE]; // ISR allocations come from this buffer 647 648 pIsrAllocator = portMemAllocatorCreateOnExistingBlock(stackAllocator, sizeof(stackAllocator)); 649 tlsIsrInit(pIsrAllocator); 650 threadStateInitISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); 651 652 pKernelGmmu = GPU_GET_KERNEL_GMMU(pGpu); 653 654 if (IS_VGPU_GSP_PLUGIN_OFFLOAD_ENABLED(pGpu) && !IS_VIRTUAL(pGpu)) 655 { 656 Intr *pIntr = GPU_GET_INTR(pGpu); 657 658 if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState)) 659 { 660 // We have to clear the top level interrupt bit here since otherwise 661 // the bottom half will attempt to service the interrupt on the CPU 662 // side before GSP receives the notification and services it 663 kgmmuClearNonReplayableFaultIntr_HAL(pGpu, pKernelGmmu, &threadState); 664 status = intrTriggerPrivDoorbell_HAL(pGpu, pIntr, NV_DOORBELL_NOTIFY_LEAF_SERVICE_NON_REPLAYABLE_FAULT_HANDLE); 665 666 } 667 } 668 else if (IS_VIRTUAL_WITH_SRIOV(pGpu)) 669 { 670 if (kgmmuIsNonReplayableFaultPending_HAL(pGpu, pKernelGmmu, &threadState)) 671 { 672 status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState); 673 } 674 } 675 else 676 { 677 if (IS_GSP_CLIENT(pGpu)) 678 { 679 // Non-replayable faults are copied to the client shadow buffer by GSP-RM. 680 status = NV_OK; 681 } 682 else 683 { 684 status = _rm_gpu_copy_mmu_faults_unlocked(pGpu, faultsCopied, &threadState); 685 } 686 } 687 688 threadStateFreeISRLockless(&threadState, pGpu, THREAD_STATE_FLAGS_IS_ISR_LOCKLESS); 689 tlsIsrDestroy(pIsrAllocator); 690 portMemAllocatorRelease(pIsrAllocator); 691 } 692 693 NV_EXIT_RM_RUNTIME(sp,fp); 694 return status; 695 } 696 697