1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 /***************************************************************************\
25 * *
26 * Description: *
27 * This module contains Nv04Control support for Debugger object *
28 * represented by GT200_DEBUGGER class instantiations. *
29 * *
30 * *
31 \***************************************************************************/
32
33 #define NVOC_KERNEL_SM_DEBUGGER_SESSION_H_PRIVATE_ACCESS_ALLOWED
34
35 #include "kernel/rmapi/control.h"
36 #include "kernel/rmapi/rmapi.h"
37 #include "kernel/os/os.h"
38 #include "kernel/core/locks.h"
39 #include "vgpu/rpc.h"
40 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
41 #include "gpu/mem_mgr/mem_desc.h"
42 #include "kernel/rmapi/rs_utils.h"
43 #include "gpu/mmu/mmu_trace.h"
44 #include "kernel/gpu/gr/kernel_sm_debugger_session.h"
45 #include "kernel/gpu/fifo/kernel_channel.h"
46 #include "kernel/gpu/mem_mgr/virt_mem_allocator_common.h"
47 #include "kernel/gpu/gr/kernel_graphics_context.h"
48 #include "kernel/gpu/gr/kernel_graphics_object.h"
49 #include "kernel/gpu/mem_mgr/mem_mgr.h"
50 #include "kernel/gpu/bus/kern_bus.h"
51
52 #include "class/cl83de.h"
53 #include "class/clc637.h"
54 #include "ctrl/ctrl83de.h"
55
56 //
57 // _nv83deCtrlCmdFetchVAS
58 //
59 // Helper to fetch the the OBJVASPACE *from the hChannel without needing
60 // for the caller to explicitly pass in the handle corresponding to the VaSpaceApi:
61 //
62 static NV_STATUS
_nv83deCtrlCmdFetchVAS(RsClient * pClient,NvU32 hChannel,OBJVASPACE ** ppVASpace)63 _nv83deCtrlCmdFetchVAS(RsClient *pClient, NvU32 hChannel, OBJVASPACE **ppVASpace)
64 {
65 KernelChannel *pKernelChannel = NULL;
66
67 NV_ASSERT_OR_RETURN(ppVASpace != NULL, NV_ERR_INVALID_ARGUMENT);
68
69 // Fetch the corresponding Channel object from our handle
70 NV_ASSERT_OK_OR_RETURN(CliGetKernelChannel(pClient, hChannel, &pKernelChannel));
71 NV_ASSERT_OR_RETURN(pKernelChannel != NULL, NV_ERR_INVALID_ARGUMENT);
72
73 *ppVASpace = pKernelChannel->pVAS;
74
75 return NV_OK;
76 }
77
78 //
79 // _nv83deCtrlCmdValidateRange
80 //
81 // Helper to traverse through the virtual memory page hierarchy and
82 // determine whether or not the virtual address (VA) range provided as
83 // input has valid and allocated pages mapped to it in its entirety.
84 //
85 // This command's input is NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS which
86 // contains a buffer of NV83DE_CTRL_DEBUG_ACCESS_OPs
87 //
88 // Possible return values:
89 // NV_OK
90 // NV_ERR_INVALID_ARGUMENT
91 // NV_ERR_INVALID_XLATE
92 //
93 static NV_STATUS
_nv83deCtrlCmdValidateRange(OBJGPU * pGpu,OBJVASPACE * pVASpace,NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS * pParams)94 _nv83deCtrlCmdValidateRange
95 (
96 OBJGPU *pGpu,
97 OBJVASPACE *pVASpace,
98 NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS *pParams
99 )
100 {
101 NvU32 i;
102 NvU64 totalLength;
103 NV_STATUS status = NV_OK;
104
105 // Loop through to validate range for all provided ops
106 for (i = 0; i < pParams->count; i++)
107 {
108 MMU_TRACE_PARAM mmuParams;
109 MMU_TRACE_ARG traceArg = {0};
110
111 // Ensure that input gpuVA is 4-byte aligned. cpuVA is handled directly by portmemCopy.
112 NV_ASSERT_OR_RETURN((pParams->opsBuffer[i].gpuVA & 3) == 0, NV_ERR_INVALID_ARGUMENT);
113
114 // Sanity-check the requested size
115 if (pParams->opsBuffer[i].size == 0 || !portSafeAddU64(pParams->opsBuffer[i].gpuVA, pParams->opsBuffer[i].size, &totalLength))
116 return NV_ERR_INVALID_ARGUMENT;
117
118 mmuParams.mode = MMU_TRACE_MODE_VALIDATE;
119 mmuParams.va = pParams->opsBuffer[i].gpuVA;
120 mmuParams.vaLimit = pParams->opsBuffer[i].gpuVA + pParams->opsBuffer[i].size - 1;
121 mmuParams.pArg = &traceArg;
122
123 NV_ASSERT_OK_OR_RETURN(mmuTrace(pGpu, pVASpace, &mmuParams));
124
125 //
126 // mmuTrace may return NV_OK if the range is invalid but the translation did
127 // not otherwise cause errors. Use traceArg.valid to satisfy the output
128 // status needed for _nv83deCtrlCmdValidateRange.
129 //
130 if (traceArg.valid)
131 {
132 pParams->opsBuffer[i].valid = 1;
133 }
134 else
135 {
136 status = NV_ERR_INVALID_XLATE;
137 pParams->opsBuffer[i].valid = 0;
138 }
139 }
140
141 return status;
142 }
143
144 static NV_STATUS
_nv8deCtrlCmdReadWriteSurface(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS * pParams,NvBool bWrite)145 _nv8deCtrlCmdReadWriteSurface
146 (
147 KernelSMDebuggerSession *pKernelSMDebuggerSession,
148 NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS *pParams,
149 NvBool bWrite
150 )
151 {
152 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
153 RsClient *pClient = RES_GET_CLIENT(pKernelSMDebuggerSession);
154 OBJVASPACE *pVASpace = NULL;
155 NvU32 count = pParams->count;
156 NvU32 i;
157 NV_STATUS status = NV_OK;
158
159 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
160
161 if (count > MAX_ACCESS_OPS)
162 return NV_ERR_INVALID_ARGUMENT;
163
164 // Attempt to retrieve the VAS pointer
165 NV_ASSERT_OK_OR_RETURN(
166 _nv83deCtrlCmdFetchVAS(pClient, pKernelSMDebuggerSession->hChannel, &pVASpace));
167
168 // Validate VA range and fail if invalid
169 NV_ASSERT_OK_OR_RETURN(
170 _nv83deCtrlCmdValidateRange(pGpu, pVASpace, pParams));
171
172 for (i = 0; i < count; i++)
173 {
174 MEMORY_DESCRIPTOR *pMemDesc = NULL;
175 NvU64 virtAddr = pParams->opsBuffer[i].gpuVA;
176 NvP64 bufPtr = pParams->opsBuffer[i].pCpuVA;
177 NvU32 bufSize = pParams->opsBuffer[i].size;
178 NvU32 pageStartOffset;
179 NvU32 start4kPage;
180 NvU32 end4kPage;
181 NvU32 curSize;
182 NvU32 cur4kPage;
183 status = NV_OK;
184
185 // Break it up by 4K pages for now
186 pageStartOffset = NvOffset_LO32(virtAddr) & RM_PAGE_MASK;
187 start4kPage = (NvOffset_LO32(virtAddr) >> 12) & 0x1FFFF;
188 end4kPage = (NvOffset_LO32(virtAddr + bufSize - 1) >> 12) & 0x1FFFF;
189
190 curSize = RM_PAGE_SIZE - pageStartOffset;
191 virtAddr -= pageStartOffset;
192
193 for (cur4kPage = start4kPage; cur4kPage <= end4kPage; ++cur4kPage)
194 {
195 MMU_TRACE_PARAM mmuParams = {0};
196 MMU_TRACE_ARG traceArg = {0};
197 TRANSFER_SURFACE surf = {0};
198 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
199 NvU8 *pKernBuffer = NULL;
200
201 mmuParams.mode = MMU_TRACE_MODE_TRANSLATE;
202 mmuParams.va = virtAddr;
203 mmuParams.vaLimit = virtAddr;
204 mmuParams.pArg = &traceArg;
205
206 NV_ASSERT_OK_OR_RETURN(
207 mmuTrace(pGpu, pVASpace, &mmuParams));
208
209 if (curSize > bufSize)
210 {
211 curSize = bufSize;
212 }
213
214 pKernBuffer = portMemAllocNonPaged(curSize);
215 if (pKernBuffer == NULL)
216 {
217 return NV_ERR_INSUFFICIENT_RESOURCES;
218 }
219
220 if (traceArg.aperture == ADDR_SYSMEM)
221 {
222 NvP64 physAddr = NV_PTR_TO_NvP64(traceArg.pa);
223 NvU64 limit = RM_PAGE_SIZE - 1;
224
225 NvU32 os02Flags = DRF_DEF(OS02, _FLAGS, _LOCATION, _PCI) |
226 DRF_DEF(OS02, _FLAGS, _MAPPING, _NO_MAP) |
227 DRF_DEF(OS02, _FLAGS, _PHYSICALITY, _CONTIGUOUS) |
228 DRF_DEF(OS02, _FLAGS, _COHERENCY, _CACHED);
229
230 NV_ASSERT_OK_OR_ELSE(status,
231 osCreateMemFromOsDescriptor(pGpu,
232 physAddr,
233 pKernelSMDebuggerSession->hInternalClient,
234 os02Flags,
235 &limit,
236 &pMemDesc,
237 NVOS32_DESCRIPTOR_TYPE_OS_PHYS_ADDR,
238 RS_PRIV_LEVEL_KERNEL),
239 portMemFree(pKernBuffer); return status; );
240 }
241 else if (traceArg.aperture == ADDR_FBMEM)
242 {
243 NV_ASSERT_OK_OR_ELSE(status,
244 memdescCreate(&pMemDesc, pGpu, RM_PAGE_SIZE, 0, NV_TRUE,
245 traceArg.aperture, NV_MEMORY_UNCACHED,
246 MEMDESC_FLAGS_NONE),
247 portMemFree(pKernBuffer); return status; );
248 memdescDescribe(pMemDesc, traceArg.aperture, traceArg.pa, RM_PAGE_SIZE);
249 }
250
251 surf.pMemDesc = pMemDesc;
252 surf.offset = pageStartOffset;
253
254 if (bWrite)
255 {
256 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status,
257 portMemExCopyFromUser(bufPtr, pKernBuffer, curSize));
258
259 // Write out the buffer to memory
260 if (status == NV_OK)
261 {
262 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status,
263 memmgrMemWrite(pMemoryManager, &surf, pKernBuffer, curSize,
264 TRANSFER_FLAGS_DEFER_FLUSH));
265 }
266 }
267 else
268 {
269 // Read from memory
270 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status,
271 memmgrMemRead(pMemoryManager, &surf, pKernBuffer, curSize,
272 TRANSFER_FLAGS_DEFER_FLUSH));
273
274 if (status == NV_OK)
275 {
276 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status,
277 portMemExCopyToUser(pKernBuffer, bufPtr, curSize));
278 }
279 }
280
281 portMemFree(pKernBuffer);
282 memdescDestroy(pMemDesc);
283
284 if (status != NV_OK)
285 return status;
286
287 pageStartOffset = 0;
288 bufPtr = NvP64_PLUS_OFFSET(bufPtr,curSize);
289 bufSize -= curSize;
290 curSize = RM_PAGE_SIZE;
291 virtAddr += RM_PAGE_SIZE;
292 }
293 }
294
295 return status;
296 }
297
298 NV_STATUS
ksmdbgssnCtrlCmdReadSurface_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS * pParams)299 ksmdbgssnCtrlCmdReadSurface_IMPL
300 (
301 KernelSMDebuggerSession *pKernelSMDebuggerSession,
302 NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS *pParams
303 )
304 {
305 return _nv8deCtrlCmdReadWriteSurface(pKernelSMDebuggerSession, pParams, NV_FALSE);
306 }
307
308 NV_STATUS
ksmdbgssnCtrlCmdWriteSurface_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS * pParams)309 ksmdbgssnCtrlCmdWriteSurface_IMPL
310 (
311 KernelSMDebuggerSession *pKernelSMDebuggerSession,
312 NV83DE_CTRL_DEBUG_ACCESS_SURFACE_PARAMETERS *pParams
313 )
314 {
315 return _nv8deCtrlCmdReadWriteSurface(pKernelSMDebuggerSession, pParams, NV_TRUE);
316 }
317
318 NV_STATUS
ksmdbgssnCtrlCmdGetMappings_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS * pParams)319 ksmdbgssnCtrlCmdGetMappings_IMPL
320 (
321 KernelSMDebuggerSession *pKernelSMDebuggerSession,
322 NV83DE_CTRL_DEBUG_GET_MAPPINGS_PARAMETERS *pParams
323 )
324 {
325 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
326 RsClient *pClient = RES_GET_CLIENT(pKernelSMDebuggerSession);
327 OBJVASPACE *pVASpace = NULL;
328 MMU_TRACE_ARG traceArg = {0};
329 MMU_TRACE_PARAM mmuParams = {0};
330
331 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
332
333 // Attempt to retrieve the VAS pointer
334 NV_ASSERT_OK_OR_RETURN(
335 _nv83deCtrlCmdFetchVAS(pClient, pKernelSMDebuggerSession->hChannel, &pVASpace));
336
337 traceArg.pMapParams = pParams;
338
339 mmuParams.mode = MMU_TRACE_MODE_DUMP_RANGE;
340 mmuParams.va = pParams->vaLo;
341 mmuParams.vaLimit = pParams->vaHi;
342 mmuParams.pArg = &traceArg;
343
344 return mmuTrace(pGpu, pVASpace, &mmuParams);
345 }
346
347 typedef enum {
348 GRDBG_MEM_ACCESS_TYPE_INVALID,
349 GRDBG_MEM_ACCESS_TYPE_READ,
350 GRDBG_MEM_ACCESS_TYPE_WRITE,
351 } GrdbgMemoryAccessType;
352
353 static NV_STATUS
_nv83deFlushAllGpusL2Cache(MEMORY_DESCRIPTOR * pMemDesc)354 _nv83deFlushAllGpusL2Cache(MEMORY_DESCRIPTOR *pMemDesc)
355 {
356 NvU32 gpuCount;
357 NvU32 gpuMask;
358 NvU32 gpuInstance = 0;
359 OBJGPU *pTempGpu;
360 NV_STATUS rmStatus = NV_OK;
361
362 gpumgrGetGpuAttachInfo(&gpuCount, &gpuMask);
363
364 while ((pTempGpu = gpumgrGetNextGpu(gpuMask, &gpuInstance)) != NULL)
365 {
366 //
367 // On GPUs with write-back caches, FB_CACHE_INVALIDATE is reduced
368 // to FB_CACHE_EVICT, which first writes back dirty lines and then
369 // invalidates clean ones, exactly what we want. On write-through
370 // caches, it will invalidate clean lines as expected.
371 //
372 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
373 kmemsysCacheOp_HAL(pTempGpu, GPU_GET_KERNEL_MEMORY_SYSTEM(pTempGpu), pMemDesc,
374 FB_CACHE_SYSTEM_MEMORY, FB_CACHE_INVALIDATE));
375 }
376
377 return rmStatus;
378 }
379
380 static NV_STATUS
_nv83deUnmapMemoryFromGrdbgClient(OBJGPU * pTargetGpu,KernelSMDebuggerSession * pKernelSMDebuggerSession,NvP64 pCpuVirtAddr,NvU32 flags)381 _nv83deUnmapMemoryFromGrdbgClient
382 (
383 OBJGPU *pTargetGpu,
384 KernelSMDebuggerSession *pKernelSMDebuggerSession,
385 NvP64 pCpuVirtAddr,
386 NvU32 flags
387 )
388 {
389 NV_STATUS status;
390 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
391
392 // Unmap memory.
393 status = pRmApi->UnmapFromCpu(pRmApi,
394 pKernelSMDebuggerSession->hInternalClient,
395 pKernelSMDebuggerSession->hInternalDevice,
396 pKernelSMDebuggerSession->hInternalMemMapping,
397 NvP64_VALUE(pCpuVirtAddr),
398 flags,
399 osGetCurrentProcess());
400
401 // Free the memory handle
402 pRmApi->Free(pRmApi,
403 pKernelSMDebuggerSession->hInternalClient,
404 pKernelSMDebuggerSession->hInternalMemMapping);
405 pKernelSMDebuggerSession->hInternalMemMapping = NV01_NULL_OBJECT;
406
407 return status;
408 }
409
410 static NV_STATUS
_nv83deMapMemoryIntoGrdbgClient(OBJGPU * pTargetGpu,KernelSMDebuggerSession * pKernelSMDebuggerSession,NvHandle hClient,NvHandle hMemory,NvU64 offset,NvU32 length,NvP64 * ppCpuVirtAddr,NvU32 flags)411 _nv83deMapMemoryIntoGrdbgClient
412 (
413 OBJGPU *pTargetGpu,
414 KernelSMDebuggerSession *pKernelSMDebuggerSession,
415 NvHandle hClient,
416 NvHandle hMemory,
417 NvU64 offset,
418 NvU32 length,
419 NvP64 *ppCpuVirtAddr,
420 NvU32 flags
421 )
422 {
423 NV_STATUS rmStatus = NV_OK;
424 void *pCpuVirtAddr = NULL;
425 RM_API *pRmApi = rmapiGetInterface(RMAPI_GPU_LOCK_INTERNAL);
426
427 //
428 // We use our own internal client, dup the memory object into our client
429 // and then call RmMapMemory on it, so that the memory is never mapped
430 // into the client's address space.
431 // hInternalMemMappping must be freed after use, since we don't have space to
432 // store multiple memory handles.
433 //
434 NV_ASSERT_OR_RETURN(pKernelSMDebuggerSession->hInternalMemMapping == NV01_NULL_OBJECT,
435 NV_ERR_INVALID_STATE);
436
437 rmStatus = pRmApi->DupObject(pRmApi,
438 pKernelSMDebuggerSession->hInternalClient,
439 pKernelSMDebuggerSession->hInternalDevice,
440 &pKernelSMDebuggerSession->hInternalMemMapping,
441 hClient,
442 hMemory,
443 0);
444 if (rmStatus != NV_OK)
445 {
446 NV_PRINTF(LEVEL_WARNING,
447 "Unable to dup source memory (0x%x,0x%x) under device (status = 0x%x). Attempting subdevice dup.\n",
448 hClient, hMemory, rmStatus);
449
450 rmStatus = pRmApi->DupObject(pRmApi,
451 pKernelSMDebuggerSession->hInternalClient,
452 pKernelSMDebuggerSession->hInternalSubdevice,
453 &pKernelSMDebuggerSession->hInternalMemMapping,
454 hClient,
455 hMemory,
456 0);
457
458 if (rmStatus != NV_OK)
459 {
460 NV_PRINTF(LEVEL_WARNING,
461 "Unable to dup source memory (0x%x,0x%x) under subdevice (status = 0x%x). Aborting.\n",
462 hClient, hMemory, rmStatus);
463 return rmStatus;
464 }
465 }
466
467 // Map memory
468 rmStatus = pRmApi->MapToCpu(pRmApi,
469 pKernelSMDebuggerSession->hInternalClient,
470 pKernelSMDebuggerSession->hInternalSubdevice,
471 pKernelSMDebuggerSession->hInternalMemMapping,
472 offset,
473 length,
474 &pCpuVirtAddr,
475 flags);
476 if (NV_OK != rmStatus)
477 {
478 NV_PRINTF(LEVEL_WARNING, "RmMapMemory failed 0x%x\n", rmStatus);
479
480 // Free the memory handle
481 pRmApi->Free(pRmApi,
482 pKernelSMDebuggerSession->hInternalClient,
483 pKernelSMDebuggerSession->hInternalMemMapping);
484 pKernelSMDebuggerSession->hInternalMemMapping = NV01_NULL_OBJECT;
485
486 return rmStatus;
487 }
488
489 *ppCpuVirtAddr = NV_PTR_TO_NvP64(pCpuVirtAddr);
490
491 return rmStatus;
492 }
493
494 static NV_STATUS
_nv83deCtrlCmdDebugAccessMemory(OBJGPU * pTargetGpu,KernelSMDebuggerSession * pKernelSMDebuggerSession,NvHandle hClient,NvHandle hMemory,NvU64 offset,NvU32 length,NvP64 buffer,GrdbgMemoryAccessType accessType)495 _nv83deCtrlCmdDebugAccessMemory
496 (
497 OBJGPU *pTargetGpu,
498 KernelSMDebuggerSession *pKernelSMDebuggerSession,
499 NvHandle hClient,
500 NvHandle hMemory,
501 NvU64 offset,
502 NvU32 length,
503 NvP64 buffer,
504 GrdbgMemoryAccessType accessType
505 )
506 {
507 RsResourceRef *pResourceRef;
508 MEMORY_DESCRIPTOR *pMemDesc;
509 NvU64 totalLength;
510 NvP64 pCpuVirtAddr = NvP64_NULL;
511 NV_STATUS rmStatus = NV_OK;
512 NV_STATUS rmUnmapStatus = NV_OK;
513 NvU32 flags = 0;
514 NvBool bGpuCached;
515 NvBool bCpuMemory;
516
517 //
518 // SECURITY: Find the hMemory object in the RmCtrl caller's database.
519 // This ensures the RmCtrl caller has rights to access hMemory.
520 //
521 if (serverutilGetResourceRef(hClient, hMemory, &pResourceRef) != NV_OK)
522 return NV_ERR_INSUFFICIENT_PERMISSIONS;
523
524 // Get a memdesc for this object to determine its attributes
525 if (!dynamicCast(pResourceRef->pResource, Memory))
526 {
527 rmStatus = NV_ERR_INVALID_ARGUMENT;
528 NV_PRINTF(LEVEL_WARNING,
529 "Invalid handle: hMemory %x is not of type classId(Memory): (GPU 0x%llx, hClient 0x%x, hMemory %x, offset 0x%llx, length 0x%x, flags 0x%x)\n",
530 hMemory,
531 pTargetGpu->busInfo.gpuPhysAddr,
532 hClient,
533 hMemory,
534 offset,
535 length,
536 flags);
537 return rmStatus;
538 }
539
540 pMemDesc = dynamicCast(pResourceRef->pResource, Memory)->pMemDesc;
541 if (pMemDesc == NULL)
542 return NV_ERR_INVALID_STATE;
543
544 // Sanity-check the requested size
545 if ((length == 0) || (!portSafeAddU64(offset, length, &totalLength)))
546 return NV_ERR_INVALID_ARGUMENT;
547 if (totalLength > pMemDesc->Size)
548 return NV_ERR_INVALID_ARGUMENT;
549
550 // Setup mapping flags based on the kind of memory, access type etc.
551 if (accessType == GRDBG_MEM_ACCESS_TYPE_READ)
552 {
553 flags = FLD_SET_DRF(OS33, _FLAGS, _ACCESS, _READ_ONLY, flags);
554 }
555 else
556 {
557 flags = FLD_SET_DRF(OS33, _FLAGS, _ACCESS, _WRITE_ONLY, flags);
558 }
559
560 bCpuMemory = (memdescGetAddressSpace(pMemDesc) == ADDR_SYSMEM);
561 bGpuCached = (memdescGetGpuCacheAttrib(pMemDesc) == NV_MEMORY_CACHED);
562
563 //
564 // Ask for a direct mapping to this memory, to avoid getting a reflected
565 // mapping. We'll do explicit cache management to ensure coherence.
566 //
567 if (bCpuMemory)
568 {
569 flags = FLD_SET_DRF(OS33, _FLAGS, _MAPPING, _DIRECT, flags);
570 }
571
572 // Allow the mapping to happen successfully on HCC devtools mode
573 if (kbusIsBarAccessBlocked(GPU_GET_KERNEL_BUS(pTargetGpu)) &&
574 gpuIsCCDevToolsModeEnabled(pTargetGpu))
575 {
576 flags = FLD_SET_DRF(OS33, _FLAGS, _ALLOW_MAPPING_ON_HCC, _YES, flags);
577 }
578
579 // Map memory into the internal smdbg client
580 rmStatus = _nv83deMapMemoryIntoGrdbgClient(pTargetGpu,
581 pKernelSMDebuggerSession,
582 hClient,
583 hMemory,
584 offset,
585 length,
586 &pCpuVirtAddr,
587 flags);
588 if (NV_OK != rmStatus)
589 {
590 NV_PRINTF(LEVEL_WARNING,
591 "Failed to map memory into internal smdbg client (GPU 0x%llx, hClient 0x%x, hMemory %x, offset 0x%llx, length 0x%x, flags 0x%x): (rmStatus = %x)\n",
592 pTargetGpu->busInfo.gpuPhysAddr,
593 hClient,
594 hMemory,
595 offset,
596 length,
597 flags,
598 rmStatus);
599 return rmStatus;
600 }
601
602 // Fence to ensure previous in-flight accesses are complete
603 osFlushCpuWriteCombineBuffer();
604
605 //
606 // Flush and invalidate SYSMEM lines from L2s of all GPUs.
607 // Some GPUs have write-back caches, so this must be done both for
608 // accessType == READ and accessType == WRITE.
609 //
610 if (bCpuMemory && bGpuCached)
611 {
612 rmStatus = _nv83deFlushAllGpusL2Cache(pMemDesc);
613 if (NV_OK != rmStatus)
614 {
615 NV_PRINTF(LEVEL_WARNING,
616 "Failed to flush GPU L2 (GPU 0x%llx): (rmStatus = %x)\n",
617 pTargetGpu->busInfo.gpuPhysAddr, rmStatus);
618 goto cleanup_mapping;
619 }
620 }
621
622 // Perform the requested accessType operation
623 if (accessType == GRDBG_MEM_ACCESS_TYPE_READ)
624 {
625 if (!portMemCopy(NvP64_VALUE(buffer), length, NvP64_VALUE(pCpuVirtAddr), length))
626 {
627 rmStatus = NV_ERR_INVALID_ARGUMENT;
628 NV_PRINTF(LEVEL_WARNING,
629 "portMemCopy failed (from VA 0x" NvP64_fmt " to 0x" NvP64_fmt ", length 0x%x)\n",
630 pCpuVirtAddr, buffer, length);
631 goto cleanup_mapping;
632 }
633 NV_PRINTF(LEVEL_INFO, "Reading %d bytes of memory from 0x%x\n",
634 length, hMemory);
635 }
636 else
637 {
638 if (!portMemCopy(NvP64_VALUE(pCpuVirtAddr), length, NvP64_VALUE(buffer), length))
639 {
640 rmStatus = NV_ERR_INVALID_ARGUMENT;
641 NV_PRINTF(LEVEL_WARNING,
642 "portMemCopy failed (from VA 0x" NvP64_fmt " to 0x" NvP64_fmt ", length 0x%x)\n",
643 buffer, pCpuVirtAddr, length);
644 goto cleanup_mapping;
645 }
646
647 NV_PRINTF(LEVEL_INFO, "Writing %d bytes of memory to 0x%x\n", length,
648 hMemory);
649 }
650
651 // Another fence to ensure our own new accesses are complete
652 osFlushCpuWriteCombineBuffer();
653
654 cleanup_mapping:
655 // Unmap memory.
656 rmUnmapStatus = _nv83deUnmapMemoryFromGrdbgClient(pTargetGpu,
657 pKernelSMDebuggerSession,
658 pCpuVirtAddr,
659 flags);
660 // Return the first failure
661 return (rmStatus != NV_OK ? rmStatus: rmUnmapStatus);
662 }
663
664 NV_STATUS
ksmdbgssnCtrlCmdDebugReadMemory_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS * pParams)665 ksmdbgssnCtrlCmdDebugReadMemory_IMPL
666 (
667 KernelSMDebuggerSession *pKernelSMDebuggerSession,
668 NV83DE_CTRL_DEBUG_READ_MEMORY_PARAMS *pParams
669 )
670 {
671 return _nv83deCtrlCmdDebugAccessMemory(GPU_RES_GET_GPU(pKernelSMDebuggerSession),
672 pKernelSMDebuggerSession,
673 RES_GET_CLIENT_HANDLE(pKernelSMDebuggerSession),
674 pParams->hMemory,
675 pParams->offset,
676 pParams->length,
677 pParams->buffer,
678 GRDBG_MEM_ACCESS_TYPE_READ);
679 }
680
681 NV_STATUS
ksmdbgssnCtrlCmdDebugWriteMemory_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_WRITE_MEMORY_PARAMS * pParams)682 ksmdbgssnCtrlCmdDebugWriteMemory_IMPL
683 (
684 KernelSMDebuggerSession *pKernelSMDebuggerSession,
685 NV83DE_CTRL_DEBUG_WRITE_MEMORY_PARAMS *pParams
686 )
687 {
688 return _nv83deCtrlCmdDebugAccessMemory(GPU_RES_GET_GPU(pKernelSMDebuggerSession),
689 pKernelSMDebuggerSession,
690 RES_GET_CLIENT_HANDLE(pKernelSMDebuggerSession),
691 pParams->hMemory,
692 pParams->offset,
693 pParams->length,
694 pParams->buffer,
695 GRDBG_MEM_ACCESS_TYPE_WRITE);
696 }
697
698 NV_STATUS
ksmdbgssnCtrlCmdDebugGetHandles_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_GET_HANDLES_PARAMS * pParams)699 ksmdbgssnCtrlCmdDebugGetHandles_IMPL
700 (
701 KernelSMDebuggerSession *pKernelSMDebuggerSession,
702 NV83DE_CTRL_DEBUG_GET_HANDLES_PARAMS *pParams
703 )
704 {
705 pParams->hChannel = pKernelSMDebuggerSession->hChannel;
706 pParams->hSubdevice = pKernelSMDebuggerSession->hSubdevice;
707
708 return NV_OK;
709 }
710
ksmdbgssnCtrlCmdDebugExecRegOps_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_EXEC_REG_OPS_PARAMS * pParams)711 NV_STATUS ksmdbgssnCtrlCmdDebugExecRegOps_IMPL
712 (
713 KernelSMDebuggerSession *pKernelSMDebuggerSession,
714 NV83DE_CTRL_DEBUG_EXEC_REG_OPS_PARAMS *pParams
715 )
716 {
717 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
718 NvBool isClientGspPlugin = NV_FALSE;
719
720 NV_CHECK_OR_RETURN(LEVEL_ERROR,
721 pParams->regOpCount <= NV83DE_CTRL_GPU_EXEC_REG_OPS_MAX_OPS,
722 NV_ERR_INVALID_ARGUMENT);
723
724 // Check if User have permission to access register offset
725 NV_CHECK_OK_OR_RETURN(LEVEL_INFO,
726 gpuValidateRegOps(pGpu, pParams->regOps, pParams->regOpCount,
727 pParams->bNonTransactional, isClientGspPlugin));
728
729 if (IS_GSP_CLIENT(pGpu))
730 {
731 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
732 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
733 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
734
735 return pRmApi->Control(pRmApi,
736 pRmCtrlParams->hClient,
737 pRmCtrlParams->hObject,
738 pRmCtrlParams->cmd,
739 pRmCtrlParams->pParams,
740 pRmCtrlParams->paramsSize);
741 }
742
743 return NV_ERR_NOT_SUPPORTED;
744 }
745
746 NV_STATUS
ksmdbgssnCtrlCmdDebugReadBatchMemory_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS * pParams)747 ksmdbgssnCtrlCmdDebugReadBatchMemory_IMPL
748 (
749 KernelSMDebuggerSession *pKernelSMDebuggerSession,
750 NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS *pParams
751 )
752 {
753 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
754 NV_STATUS status = NV_OK;
755 NvU32 i;
756
757 NV_CHECK_OR_RETURN(LEVEL_ERROR, pParams->count <= MAX_ACCESS_MEMORY_OPS,
758 NV_ERR_INVALID_ARGUMENT);
759
760 for (i = 0; i < pParams->count; ++i)
761 {
762 NV_STATUS localStatus = NV_OK;
763 NvP64 pData = (NvP64)(((NvU8 *)pParams->pData) + pParams->entries[i].dataOffset);
764 NvU32 endingOffset;
765
766 NV_CHECK_OR_ELSE(LEVEL_ERROR,
767 portSafeAddU32(pParams->entries[i].dataOffset, pParams->entries[i].length, &endingOffset) &&
768 (endingOffset <= pParams->dataLength),
769 localStatus = NV_ERR_INVALID_OFFSET;
770 goto updateStatus; );
771
772 NV_CHECK_OK_OR_GOTO(localStatus, LEVEL_ERROR,
773 _nv83deCtrlCmdDebugAccessMemory(pGpu,
774 pKernelSMDebuggerSession,
775 RES_GET_CLIENT_HANDLE(pKernelSMDebuggerSession),
776 pParams->entries[i].hMemory,
777 pParams->entries[i].memOffset,
778 pParams->entries[i].length,
779 pData,
780 GRDBG_MEM_ACCESS_TYPE_READ),
781 updateStatus);
782
783 updateStatus:
784 pParams->entries[i].status = localStatus;
785 if ((status == NV_OK) && (localStatus != NV_OK))
786 status = localStatus;
787 }
788
789 return status;
790 }
791
792 NV_STATUS
ksmdbgssnCtrlCmdDebugWriteBatchMemory_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS * pParams)793 ksmdbgssnCtrlCmdDebugWriteBatchMemory_IMPL
794 (
795 KernelSMDebuggerSession *pKernelSMDebuggerSession,
796 NV83DE_CTRL_DEBUG_ACCESS_MEMORY_PARAMS *pParams
797 )
798 {
799 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
800 NV_STATUS status = NV_OK;
801 NvU32 i;
802
803 NV_CHECK_OR_RETURN(LEVEL_ERROR, pParams->count <= MAX_ACCESS_MEMORY_OPS,
804 NV_ERR_INVALID_ARGUMENT);
805
806 for (i = 0; i < pParams->count; ++i)
807 {
808 NV_STATUS localStatus = NV_OK;
809 NvP64 pData = (NvP64)(((NvU8 *)pParams->pData) + pParams->entries[i].dataOffset);
810 NvU32 endingOffset;
811
812 NV_CHECK_OR_ELSE(LEVEL_ERROR,
813 portSafeAddU32(pParams->entries[i].dataOffset, pParams->entries[i].length, &endingOffset) &&
814 (endingOffset <= pParams->dataLength),
815 localStatus = NV_ERR_INVALID_OFFSET;
816 goto updateStatus; );
817
818 NV_CHECK_OK_OR_GOTO(localStatus, LEVEL_ERROR,
819 _nv83deCtrlCmdDebugAccessMemory(pGpu,
820 pKernelSMDebuggerSession,
821 RES_GET_CLIENT_HANDLE(pKernelSMDebuggerSession),
822 pParams->entries[i].hMemory,
823 pParams->entries[i].memOffset,
824 pParams->entries[i].length,
825 pData,
826 GRDBG_MEM_ACCESS_TYPE_WRITE),
827 updateStatus);
828
829 updateStatus:
830 pParams->entries[i].status = localStatus;
831 if ((status == NV_OK) && (localStatus != NV_OK))
832 status = localStatus;
833 }
834
835 return status;
836 }
837
838 NV_STATUS
ksmdbgssnCtrlCmdDebugReadAllSmErrorStates_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS * pParams)839 ksmdbgssnCtrlCmdDebugReadAllSmErrorStates_IMPL
840 (
841 KernelSMDebuggerSession *pKernelSMDebuggerSession,
842 NV83DE_CTRL_DEBUG_READ_ALL_SM_ERROR_STATES_PARAMS *pParams
843 )
844 {
845 NV_STATUS rmStatus = NV_OK;
846 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
847
848 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
849
850 if (IS_VIRTUAL(pGpu))
851 {
852 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
853 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams->pLegacyParams;
854
855 NV_RM_RPC_CONTROL(pRmCtrlParams->pGpu,
856 pRmCtrlParams->hClient,
857 pRmCtrlParams->hObject,
858 pRmCtrlParams->cmd,
859 pRmCtrlParams->pParams,
860 pRmCtrlParams->paramsSize,
861 rmStatus);
862
863 //
864 // SR-IOV vGPU
865 //
866 // MMU fault info is to be managed from within the guest, since host is
867 // not aware of MMU fault info about the VF.
868 // SM exception info is still fetched from host via the RPC above.
869 //
870 if (IS_VIRTUAL_WITH_SRIOV(pGpu))
871 {
872 NV_ASSERT_OK(
873 kgrctxLookupMmuFault(pGpu,
874 kgrobjGetKernelGraphicsContext(pGpu, pKernelSMDebuggerSession->pObject),
875 &pParams->mmuFault));
876 }
877
878 return rmStatus;
879 }
880
881 return NV_ERR_NOT_SUPPORTED;
882 }
883
884 NV_STATUS
ksmdbgssnCtrlCmdDebugClearAllSmErrorStates_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS * pParams)885 ksmdbgssnCtrlCmdDebugClearAllSmErrorStates_IMPL
886 (
887 KernelSMDebuggerSession *pKernelSMDebuggerSession,
888 NV83DE_CTRL_DEBUG_CLEAR_ALL_SM_ERROR_STATES_PARAMS *pParams
889 )
890 {
891 NV_STATUS rmStatus = NV_OK;
892 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
893
894 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
895
896 if (IS_VIRTUAL(pGpu))
897 {
898 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
899 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams->pLegacyParams;
900 NV_RM_RPC_CONTROL(pRmCtrlParams->pGpu,
901 pRmCtrlParams->hClient,
902 pRmCtrlParams->hObject,
903 pRmCtrlParams->cmd,
904 pRmCtrlParams->pParams,
905 pRmCtrlParams->paramsSize,
906 rmStatus);
907
908 //
909 // SR-IOV vGPU
910 //
911 // MMU fault info is to be managed from within the guest, since host is
912 // not aware of MMU fault info about the VF.
913 // SM exception info is still fetched from host via the RPC above.
914 //
915 if (IS_VIRTUAL_WITH_SRIOV(pGpu))
916 {
917 NV_ASSERT_OK(
918 kgrctxClearMmuFault(pGpu, kgrobjGetKernelGraphicsContext(pGpu, pKernelSMDebuggerSession->pObject)));
919 }
920
921 return rmStatus;
922 }
923
924 return NV_ERR_NOT_SUPPORTED;
925 }
926
927 NV_STATUS
ksmdbgssnCtrlCmdDebugReadMMUFaultInfo_IMPL(KernelSMDebuggerSession * pKernelSMDebuggerSession,NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS * pParams)928 ksmdbgssnCtrlCmdDebugReadMMUFaultInfo_IMPL
929 (
930 KernelSMDebuggerSession *pKernelSMDebuggerSession,
931 NV83DE_CTRL_DEBUG_READ_MMU_FAULT_INFO_PARAMS *pParams
932 )
933 {
934 OBJGPU *pGpu = GPU_RES_GET_GPU(pKernelSMDebuggerSession);
935
936 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmGpuLockIsOwner());
937
938 //
939 // SR-IOV vGPU
940 //
941 // MMU fault info is to be managed from within the guest, since host is
942 // not aware of MMU fault info about the VF.
943 // SM exception info is still fetched from host via the RPC above.
944 //
945 if (IS_GSP_CLIENT(pGpu) || IS_VIRTUAL_WITHOUT_SRIOV(pGpu))
946 {
947 NV_STATUS status = NV_OK;
948 CALL_CONTEXT *pCallContext = resservGetTlsCallContext();
949 RmCtrlParams *pRmCtrlParams = pCallContext->pControlParams;
950
951 NV_RM_RPC_CONTROL(pGpu,
952 pRmCtrlParams->hClient,
953 pRmCtrlParams->hObject,
954 pRmCtrlParams->cmd,
955 pRmCtrlParams->pParams,
956 pRmCtrlParams->paramsSize,
957 status);
958 return status;
959 }
960
961 NV_ASSERT_OK_OR_RETURN(
962 kgrctxLookupMmuFaultInfo(pGpu,
963 kgrobjGetKernelGraphicsContext(pGpu, pKernelSMDebuggerSession->pObject),
964 pParams));
965
966 return NV_OK;
967 }
968
969