1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2016-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/core.h"
25 #include "gpu/gpu.h"
26 #include "gpu/ce/kernel_ce.h"
27 #include "gpu/mem_mgr/mem_mgr.h"
28 #include "gpu/mem_mgr/heap.h"
29 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h"
30 #include "gpu/bus/kern_bus.h"
31 #include "kernel/gpu/fifo/kernel_fifo.h"
32 #include "objtmr.h"
33 #include "gpu/mem_mgr/mem_desc.h"
34 #include "kernel/gpu/intr/intr.h"
35
36 #include "gpu/mem_mgr/channel_utils.h"
37 #include "gpu/mem_mgr/mem_scrub.h"
38 #include "os/os.h"
39 #include "gpu/mem_mgr/phys_mem_allocator/phys_mem_allocator.h"
40 #include "gpu/mem_mgr/mem_mgr.h"
41 #include "utils/nvprintf.h"
42 #include "utils/nvassert.h"
43 #include "nvgputypes.h"
44 #include "nvtypes.h"
45 #include "nvstatus.h"
46 #include "rmapi/rs_utils.h"
47 #include "core/locks.h"
48
49 #include "gpu/conf_compute/conf_compute.h"
50
51 #include "class/cl0050.h"
52 #include "class/clb0b5.h" // MAXWELL_DMA_COPY_A
53 #include "class/clc0b5.h" // PASCAL_DMA_COPY_A
54 #include "class/clc1b5.h" // PASCAL_DMA_COPY_B
55 #include "class/clc3b5.h" // VOLTA_DMA_COPY_A
56 #include "class/clc5b5.h" // TURING_DMA_COPY_A
57 #include "class/clc6b5.h" // AMPERE_DMA_COPY_A
58 #include "class/clc7b5.h" // AMPERE_DMA_COPY_B
59
60 #include "class/clc8b5.h" // HOPPER_DMA_COPY_A
61
62 #include "class/clc86f.h" // HOPPER_CHANNEL_GPFIFO_A
63
64 static NvU64 _scrubCheckProgress(OBJMEMSCRUB *pScrubber);
65 static NvU64 _searchScrubList(OBJMEMSCRUB *pScrubber, RmPhysAddr base, NvU64 size);
66 static void _waitForPayload(OBJMEMSCRUB *pScrubber, RmPhysAddr base, RmPhysAddr end);
67 static void _scrubAddWorkToList(OBJMEMSCRUB *pScrubber, RmPhysAddr base, NvU64 size, NvU64 newId);
68 static NvU32 _scrubMemory(OBJMEMSCRUB *pScrubber, RmPhysAddr base, NvU64 size,
69 NvU32 dstCpuCacheAttrib, NvU32 freeToken);
70 static void _scrubWaitAndSave(OBJMEMSCRUB *pScrubber, PSCRUB_NODE pList, NvLength itemsToSave);
71 static NvU64 _scrubGetFreeEntries(OBJMEMSCRUB *pScrubber);
72 static NvU64 _scrubCheckAndSubmit(OBJMEMSCRUB *pScrubber, NvU64 pageCount, PSCRUB_NODE pList,
73 PSCRUB_NODE pScrubListCopy, NvLength pagesToScrubCheck);
74 static void _scrubCopyListItems(OBJMEMSCRUB *pScrubber, PSCRUB_NODE pList, NvLength itemsToSave);
75
76 static NV_STATUS _scrubCheckLocked(OBJMEMSCRUB *pScrubber, PSCRUB_NODE *ppList, NvU64 *pSize);
77 static NV_STATUS _scrubCombinePages(NvU64 *pPages, NvU64 pageSize, NvU64 pageCount,
78 PSCRUB_NODE *ppScrubList, NvU64 *pSize);
79
80 /**
81 * Constructs the memory scrubber object and signals
82 * RM to create CE channels for submitting scrubbing work
83 *
84 * @param[in] pGpu OBJGPU pointer
85 * @param[in] pHeap Heap object pointer
86 *
87 * @returns NV_STATUS on success.
88 * error, if something fails
89 */
90 NV_STATUS
scrubberConstruct(OBJGPU * pGpu,Heap * pHeap)91 scrubberConstruct
92 (
93 OBJGPU *pGpu,
94 Heap *pHeap
95 )
96 {
97 OBJMEMSCRUB *pScrubber;
98 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
99 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu);
100 NV_STATUS status = NV_OK;
101 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu);
102 PMA *pPma = NULL;
103 KERNEL_MIG_GPU_INSTANCE *pKernelMIGGPUInstance = NULL;
104
105 if (pHeap == NULL)
106 {
107 return NV_ERR_INVALID_ARGUMENT;
108 }
109 pPma = &pHeap->pmaObject;
110
111 if (pPma->pScrubObj != NULL)
112 return NV_OK;
113
114 pScrubber = (OBJMEMSCRUB *)portMemAllocNonPaged(sizeof(OBJMEMSCRUB));
115 if (pScrubber == NULL)
116 {
117 return NV_ERR_INSUFFICIENT_RESOURCES;
118 }
119
120 portMemSet(pScrubber, 0, sizeof(OBJMEMSCRUB));
121
122 pScrubber->pScrubberMutex = (PORT_MUTEX *)portMemAllocNonPaged(portSyncMutexSize);
123 if (pScrubber->pScrubberMutex == NULL)
124 {
125 status = NV_ERR_INSUFFICIENT_RESOURCES;
126 goto error;
127 }
128
129 NV_ASSERT_OK_OR_GOTO(status,
130 portSyncMutexInitialize(pScrubber->pScrubberMutex), freemutex);
131
132 pScrubber->pScrubList = (PSCRUB_NODE)
133 portMemAllocNonPaged(sizeof(SCRUB_NODE) * MAX_SCRUB_ITEMS);
134 if (pScrubber->pScrubList == NULL)
135 {
136 status = NV_ERR_INSUFFICIENT_RESOURCES;
137 goto deinitmutex;
138 }
139 portMemSet(pScrubber->pScrubList, 0, sizeof(SCRUB_NODE) * MAX_SCRUB_ITEMS);
140
141 pScrubber->pGpu = pGpu;
142
143 {
144 NV_PRINTF(LEVEL_INFO, "Starting to init CeUtils for scrubber.\n");
145 NV0050_ALLOCATION_PARAMETERS ceUtilsAllocParams = {0};
146
147 if (memmgrUseVasForCeMemoryOps(pMemoryManager))
148 {
149 ceUtilsAllocParams.flags = DRF_DEF(0050, _CEUTILS_FLAGS, _VIRTUAL_MODE, _TRUE);
150 }
151
152 if (bMIGInUse)
153 {
154 KERNEL_MIG_GPU_INSTANCE *pCurrKernelMIGGPUInstance;
155
156 FOR_EACH_VALID_GPU_INSTANCE(pGpu, pKernelMIGManager, pCurrKernelMIGGPUInstance)
157 {
158 if (pCurrKernelMIGGPUInstance->pMemoryPartitionHeap == pHeap)
159 {
160 pKernelMIGGPUInstance = pCurrKernelMIGGPUInstance;
161 break;
162 }
163 }
164 FOR_EACH_VALID_GPU_INSTANCE_END();
165 }
166 ConfidentialCompute *pConfCompute = GPU_GET_CONF_COMPUTE(pGpu);
167 if ((pConfCompute != NULL) &&
168 (pConfCompute->getProperty(pCC, PDB_PROP_CONFCOMPUTE_CC_FEATURE_ENABLED)))
169 {
170 NV_ASSERT_OK_OR_GOTO(status, objCreate(&pScrubber->pSec2Utils, pHeap, Sec2Utils, pGpu, pKernelMIGGPUInstance),
171 destroyscrublist);
172 pScrubber->engineType = NV2080_ENGINE_TYPE_SEC2;
173 }
174 else
175 {
176 NV_ASSERT_OK_OR_GOTO(status, objCreate(&pScrubber->pCeUtils, pHeap, CeUtils, pGpu, pKernelMIGGPUInstance, &ceUtilsAllocParams),
177 destroyscrublist);
178 pScrubber->engineType = NV2080_ENGINE_TYPE_COPY2;
179 }
180 NV_ASSERT_OK_OR_GOTO(status, pmaRegMemScrub(pPma, pScrubber), destroyscrublist);
181 }
182
183 return status;
184
185 destroyscrublist:
186 portMemFree(pScrubber->pScrubList);
187
188 deinitmutex:
189 portSyncMutexDestroy(pScrubber->pScrubberMutex);
190
191 freemutex:
192 portMemFree(pScrubber->pScrubberMutex);
193 pScrubber->pScrubberMutex = NULL;
194
195 error:
196 portMemFree(pScrubber);
197 return status;
198 }
199
200 static NvBool
_isScrubWorkPending(OBJMEMSCRUB * pScrubber)201 _isScrubWorkPending(
202 OBJMEMSCRUB *pScrubber
203 )
204 {
205 NvBool workPending = NV_FALSE;
206
207 if (pScrubber->bVgpuScrubberEnabled)
208 {
209 if (pScrubber->lastSubmittedWorkId != pScrubber->vgpuScrubBuffRing.pScrubBuffRingHeader->lastSWSemaphoreDone)
210 workPending = NV_TRUE;
211 }
212 else
213 {
214 NvU64 lastCompleted;
215 if (pScrubber->engineType == NV2080_ENGINE_TYPE_SEC2)
216 {
217 lastCompleted = sec2utilsUpdateProgress(pScrubber->pSec2Utils);
218 }
219 else
220 {
221 lastCompleted = ceutilsUpdateProgress(pScrubber->pCeUtils);
222 }
223
224 if (pScrubber->lastSubmittedWorkId != lastCompleted)
225 workPending = NV_TRUE;
226 }
227 return workPending;
228 }
229
230 /**
231 * Destructs the scrubber
232 * 1. De-registers the scrubber from the PMA object
233 * 2. Free the scrubber list and scrubber lock
234 *
235 * @param[in] pGpu OBJGPU pointer
236 * @param[in] pHeap Heap object pointer
237 * @param[in] pScrubber OBJMEMSCRUB pointer
238 *
239 */
240 void
scrubberDestruct(OBJGPU * pGpu,Heap * pHeap,OBJMEMSCRUB * pScrubber)241 scrubberDestruct
242 (
243 OBJGPU *pGpu,
244 Heap *pHeap,
245 OBJMEMSCRUB *pScrubber
246 )
247 {
248 PMA *pPma = NULL;
249 PSCRUB_NODE pPmaScrubList = NULL;
250 NvU64 count = 0;
251 NV_STATUS status = NV_OK;
252
253 if (pHeap == NULL)
254 {
255 return;
256 }
257 pPma = &pHeap->pmaObject;
258
259 if (pScrubber == NULL)
260 return;
261
262 pmaUnregMemScrub(pPma);
263 portSyncMutexAcquire(pScrubber->pScrubberMutex);
264
265 if (!API_GPU_IN_RESET_SANITY_CHECK(pGpu))
266 {
267 RMTIMEOUT timeout;
268 gpuSetTimeout(pGpu, GPU_TIMEOUT_DEFAULT, &timeout, 0);
269
270 while (_isScrubWorkPending(pScrubber))
271 {
272 // just wait till it finishes
273 // Since the default RM Timeout is violated by this, added this for FMODEL
274 if (!IS_FMODEL(pGpu))
275 {
276 if (gpuCheckTimeout(pGpu, &timeout) == NV_ERR_TIMEOUT)
277 {
278 NV_PRINTF(LEVEL_FATAL,
279 " Timed out when waiting for the scrub to complete the pending work .\n");
280 DBG_BREAKPOINT();
281 break;
282 }
283 }
284 }
285 }
286
287 // check for the completed scrub work items
288 NV_ASSERT_OK_OR_CAPTURE_FIRST_ERROR(status, _scrubCheckLocked(pScrubber, &pPmaScrubList, &count));
289
290 // Make sure all scrubbed pages are returned to PMA
291 if (count > 0)
292 pmaClearScrubbedPages(pPma, pPmaScrubList, count);
293
294 portMemFree(pPmaScrubList);
295
296 portMemFree(pScrubber->pScrubList);
297 {
298 if (pScrubber->engineType == NV2080_ENGINE_TYPE_SEC2)
299 {
300 objDelete(pScrubber->pSec2Utils);
301 }
302 else
303 {
304 objDelete(pScrubber->pCeUtils);
305 }
306 }
307
308 portSyncMutexRelease(pScrubber->pScrubberMutex);
309 portSyncMutexDestroy(pScrubber->pScrubberMutex);
310 portMemFree(pScrubber->pScrubberMutex);
311 portMemFree(pScrubber);
312 }
313
314 static NV_STATUS
_scrubCheckLocked(OBJMEMSCRUB * pScrubber,PSCRUB_NODE * ppList,NvU64 * pSize)315 _scrubCheckLocked
316 (
317 OBJMEMSCRUB *pScrubber,
318 PSCRUB_NODE *ppList,
319 NvU64 *pSize
320 )
321 {
322 NV_STATUS status = NV_OK;
323 PSCRUB_NODE pList = NULL;
324 NvLength itemsToSave = 0;
325 NvU64 currentCompletedId = 0;
326
327 *ppList = NULL;
328 *pSize = 0;
329 currentCompletedId = _scrubCheckProgress(pScrubber);
330
331 itemsToSave = (NvLength)(currentCompletedId - pScrubber->lastSeenIdByClient);
332
333 NV_ASSERT(itemsToSave <= MAX_SCRUB_ITEMS);
334
335 if(itemsToSave == 0)
336 goto exit;
337
338 pList = (PSCRUB_NODE)portMemAllocNonPaged(itemsToSave * sizeof(SCRUB_NODE));
339 if (pList == NULL)
340 {
341 status = NV_ERR_INSUFFICIENT_RESOURCES;
342 goto exit;
343 }
344 portMemSet(pList, 0, sizeof(SCRUB_NODE) * itemsToSave);
345
346 _scrubCopyListItems(pScrubber, pList, itemsToSave);
347
348 exit:
349 *ppList = pList;
350 *pSize = itemsToSave;
351 return status;
352 }
353
354 /**
355 * This function checks for the completed scrub work items,
356 * and populates the SCRUB_NODE in the array.
357 * @param[in] pScrubber OBJMEMSCRUB pointer
358 * @param[out] ppList SCRUB_NODE double pointer
359 * @param[out] pSize NvU64 pointer
360 * @returns NV_OK on success,
361 * NV_ERR_INSUFFICIENT_RESOURCES when the list allocation fails.
362 */
363
364 NV_STATUS
scrubCheck(OBJMEMSCRUB * pScrubber,PSCRUB_NODE * ppList,NvU64 * pSize)365 scrubCheck
366 (
367 OBJMEMSCRUB *pScrubber,
368 PSCRUB_NODE *ppList,
369 NvU64 *pSize
370 )
371 {
372 NV_STATUS status;
373 portSyncMutexAcquire(pScrubber->pScrubberMutex);
374 status = _scrubCheckLocked(pScrubber, ppList, pSize);
375 portSyncMutexRelease(pScrubber->pScrubberMutex);
376 return status;
377 }
378
379 /**
380 * This function submits work to the memory scrubber.
381 * This function interface is changed to return a list of scrubbed pages to the
382 * client, since the scrubber work list resources are limited, if the submission
383 * page count is more than scrubber list resources the completed scrubbed pages
384 * are saved in the list and the submission progresses.
385 *
386 * @param[in] pScrubber OBJMEMSCRUB pointer
387 * @param[in] chunkSize NvU64 size of each page
388 * @param[in] pPages NvU64 array of base address
389 * @param[in] pageCount NvU64 number of pages
390 * @param[out] ppList SCRUB_NODE double pointer to hand off the list
391 * @param[out] pSize NvU64 pointer to store the size
392 *
393 * @returns NV_OK on success, NV_ERR_GENERIC on HW Failure
394 */
395 NV_STATUS
scrubSubmitPages(OBJMEMSCRUB * pScrubber,NvU64 chunkSize,NvU64 * pPages,NvU64 pageCount,PSCRUB_NODE * ppList,NvU64 * pSize)396 scrubSubmitPages
397 (
398 OBJMEMSCRUB *pScrubber,
399 NvU64 chunkSize,
400 NvU64 *pPages,
401 NvU64 pageCount,
402 PSCRUB_NODE *ppList,
403 NvU64 *pSize
404 )
405 {
406 NvU64 curPagesSaved = 0;
407 PSCRUB_NODE pScrubList = NULL;
408 PSCRUB_NODE pScrubListCopy = NULL;
409 NvU64 scrubListSize = 0;
410 NvLength pagesToScrubCheck = 0;
411 NvU64 totalSubmitted = 0;
412 NvU64 numFinished = 0;
413 NvU64 freeEntriesInList = 0;
414 NvU64 scrubCount = 0;
415 NvU64 numPagesToScrub = 0;
416 NV_STATUS status = NV_OK;
417
418 portSyncMutexAcquire(pScrubber->pScrubberMutex);
419 *pSize = 0;
420 *ppList = pScrubList;
421
422 NV_CHECK_OR_GOTO(LEVEL_INFO, pageCount > 0, cleanup);
423
424 NV_PRINTF(LEVEL_INFO, "submitting pages, pageCount = 0x%llx chunkSize = 0x%llx\n", pageCount, chunkSize);
425
426 freeEntriesInList = _scrubGetFreeEntries(pScrubber);
427
428 NV_ASSERT_OK_OR_GOTO(status,
429 _scrubCombinePages(pPages,
430 chunkSize,
431 pageCount,
432 &pScrubList,
433 &scrubListSize),
434 cleanup);
435
436 numPagesToScrub = scrubListSize;
437
438 if (freeEntriesInList < scrubListSize)
439 {
440 pScrubListCopy = (PSCRUB_NODE)
441 portMemAllocNonPaged((NvLength)(sizeof(SCRUB_NODE) * (scrubListSize - freeEntriesInList)));
442
443 if (pScrubListCopy == NULL)
444 {
445 status = NV_ERR_NO_MEMORY;
446 goto cleanup;
447 }
448
449 while (freeEntriesInList < scrubListSize)
450 {
451 if (scrubListSize > MAX_SCRUB_ITEMS)
452 {
453 pagesToScrubCheck = (NvLength)(MAX_SCRUB_ITEMS - freeEntriesInList);
454 scrubCount = MAX_SCRUB_ITEMS;
455 }
456 else
457 {
458 pagesToScrubCheck = (NvLength)(scrubListSize - freeEntriesInList);
459 scrubCount = scrubListSize;
460 }
461
462 numFinished = _scrubCheckAndSubmit(pScrubber, scrubCount,
463 &pScrubList[totalSubmitted],
464 &pScrubListCopy[curPagesSaved],
465 pagesToScrubCheck);
466
467 scrubListSize -= numFinished;
468 curPagesSaved += pagesToScrubCheck;
469 totalSubmitted += numFinished;
470 freeEntriesInList = _scrubGetFreeEntries(pScrubber);
471 }
472
473 *ppList = pScrubListCopy;
474 *pSize = curPagesSaved;
475 }
476 else
477 {
478 totalSubmitted = _scrubCheckAndSubmit(pScrubber, scrubListSize,
479 pScrubList, NULL, 0);
480 *ppList = NULL;
481 *pSize = 0;
482 }
483
484 cleanup:
485 portSyncMutexRelease(pScrubber->pScrubberMutex);
486
487 if (pScrubList != NULL)
488 {
489 portMemFree(pScrubList);
490 pScrubList = NULL;
491 }
492
493 NV_CHECK_OK_OR_RETURN(LEVEL_INFO, status);
494
495 if (totalSubmitted == numPagesToScrub)
496 {
497 status = NV_OK;
498 }
499 else
500 {
501 NV_PRINTF(LEVEL_FATAL, "totalSubmitted :%llx != pageCount: %llx\n",
502 totalSubmitted, pageCount);
503 DBG_BREAKPOINT();
504 status = NV_ERR_GENERIC;
505 }
506
507 return status;
508 }
509
510 /**
511 * This function waits for the memory scrubber to wait for the scrubbing of
512 * pages within the range [pagesStart, pagesEnd] for the for the array of pages
513 * of size pageCount
514 *
515 * @param[in] pScrubber OBJMEMSCRUB pointer
516 * @param[in] chunkSize NvU64 size of each page
517 * @param[in] pPages NvU64 pointer to store the base address
518 * @param[in] pageCount NvU64 number of pages in the array
519 *
520 * @returns NV_OK
521 */
522
523 NV_STATUS
scrubWaitPages(OBJMEMSCRUB * pScrubber,NvU64 chunkSize,NvU64 * pPages,NvU32 pageCount)524 scrubWaitPages
525 (
526 OBJMEMSCRUB *pScrubber,
527 NvU64 chunkSize,
528 NvU64 *pPages,
529 NvU32 pageCount
530 )
531 {
532
533 NvU32 iter = 0;
534 NV_STATUS status = NV_OK;
535 PSCRUB_NODE pScrubList = NULL;
536 NvU64 scrubListSize = 0;
537
538 NV_ASSERT_OK_OR_RETURN(_scrubCombinePages(pPages,
539 chunkSize,
540 pageCount,
541 &pScrubList,
542 &scrubListSize));
543
544 portSyncMutexAcquire(pScrubber->pScrubberMutex);
545
546 for (iter = 0; iter < scrubListSize; iter++)
547 {
548 _waitForPayload(pScrubber,
549 pScrubList[iter].base,
550 (pScrubList[iter].base + pScrubList[iter].size - 1));
551 }
552 portSyncMutexRelease(pScrubber->pScrubberMutex);
553
554 if (pScrubList != NULL)
555 {
556 portMemFree(pScrubList);
557 pScrubList = NULL;
558 }
559
560 return status;
561
562 }
563
564 /**
565 * This function waits for the scrubber to finish scrubbing enough items
566 * to have numPages fully scrubbed and then saves the work items to the list
567 * passed to the client.
568 *
569 * @param[in] pScrubber OBJMEMSCRUB pointer
570 * @param[in] numPages the number of pages we should wait to be scrubbed
571 * @param[in] pageSize the page size
572 * @param[out] ppList SCRUB_NODE double pointer to return the saved list pointer
573 * @param[out] pSize NvU64 pointer to return the size of saved work.
574 *
575 * @returns NV_OK if at least one work is pending in the scrubber list
576 * NV_ERR_NO_MEMORY when no work is pending in the scrubber list
577 */
578
579 NV_STATUS
scrubCheckAndWaitForSize(OBJMEMSCRUB * pScrubber,NvU64 numPages,NvU64 pageSize,PSCRUB_NODE * ppList,NvU64 * pSize)580 scrubCheckAndWaitForSize
581 (
582 OBJMEMSCRUB *pScrubber,
583 NvU64 numPages,
584 NvU64 pageSize,
585 PSCRUB_NODE *ppList,
586 NvU64 *pSize
587 )
588 {
589 PSCRUB_NODE pList = NULL;
590 NV_STATUS status = NV_OK;
591 NvLength totalItems = 0;
592 portSyncMutexAcquire(pScrubber->pScrubberMutex);
593 totalItems = (NvLength)pScrubber->scrubListSize;
594 *pSize = 0;
595 *ppList = pList;
596
597 NvLength startIdx = pScrubber->lastSeenIdByClient;
598 NvU64 totalScrubbedPages = 0;
599 NvLength requiredItemsToSave = 0;
600
601 for (; requiredItemsToSave < totalItems && totalScrubbedPages <= numPages; requiredItemsToSave++) {
602 totalScrubbedPages += (pScrubber->pScrubList[(startIdx + requiredItemsToSave) % MAX_SCRUB_ITEMS].size / pageSize);
603 }
604
605 if (requiredItemsToSave != 0) {
606 pList = (PSCRUB_NODE) portMemAllocNonPaged(sizeof(SCRUB_NODE) * requiredItemsToSave);
607 if (pList == NULL)
608 {
609 status = NV_ERR_INSUFFICIENT_RESOURCES;
610 goto exit;
611 }
612
613 _scrubWaitAndSave(pScrubber, pList, requiredItemsToSave);
614 }
615 else {
616 // since there is no scrub remaining, its upto the user about how to handle that.
617 status = NV_ERR_NO_MEMORY;
618 }
619
620 *pSize = (NvU64)requiredItemsToSave;
621 *ppList = pList;
622
623 exit:
624 portSyncMutexRelease(pScrubber->pScrubberMutex);
625 return status;
626 }
627
628 /**
629 * helper function to copy elements from scrub list to the temporary list to
630 * return to the caller.
631 * @param[in] pScrubber OBJMEMSCRUB pointer
632 * @param[in] SCRUB_NODE pointer to copy the element
633 * @param[in] NvLength number of elements to copy
634 *
635 */
636
637 static void
_scrubCopyListItems(OBJMEMSCRUB * pScrubber,PSCRUB_NODE pList,NvLength itemsToSave)638 _scrubCopyListItems
639 (
640 OBJMEMSCRUB *pScrubber,
641 PSCRUB_NODE pList,
642 NvLength itemsToSave
643 )
644 {
645 NvLength startIdx = pScrubber->lastSeenIdByClient%MAX_SCRUB_ITEMS;
646 NvLength endIdx = (pScrubber->lastSeenIdByClient + itemsToSave)%
647 MAX_SCRUB_ITEMS;
648
649 NV_ASSERT(pList != NULL);
650 NV_ASSERT(itemsToSave <= MAX_SCRUB_ITEMS);
651
652 if (startIdx < endIdx)
653 {
654 portMemCopy(pList,
655 sizeof(SCRUB_NODE) * itemsToSave,
656 &pScrubber->pScrubList[startIdx],
657 sizeof(SCRUB_NODE) * itemsToSave);
658 portMemSet(&pScrubber->pScrubList[startIdx], 0, sizeof(SCRUB_NODE) * itemsToSave);
659 }
660 else
661 {
662 NvLength itemsFromStartToLastItem = (NvLength)(MAX_SCRUB_ITEMS - startIdx);
663
664 // copy from startIdx to (MAX_SCRUB_ITEMS -1) idx
665 portMemCopy(pList,
666 sizeof(SCRUB_NODE) * itemsFromStartToLastItem,
667 &pScrubber->pScrubList[startIdx],
668 sizeof(SCRUB_NODE) * itemsFromStartToLastItem);
669 portMemSet(&pScrubber->pScrubList[startIdx], 0, sizeof(SCRUB_NODE) * itemsFromStartToLastItem);
670
671 // now copy from from 0 to endIdx
672 portMemCopy(&pList[itemsFromStartToLastItem],
673 sizeof(SCRUB_NODE) * endIdx,
674 &pScrubber->pScrubList[0],
675 sizeof(SCRUB_NODE) * endIdx);
676
677 portMemSet(&pScrubber->pScrubList[0], 0, sizeof(SCRUB_NODE) * endIdx);
678 }
679
680 pScrubber->lastSeenIdByClient += itemsToSave;
681 pScrubber->scrubListSize -= itemsToSave;
682 NV_ASSERT(_scrubGetFreeEntries(pScrubber) <= MAX_SCRUB_ITEMS);
683 }
684
685 /* This function is used to check and submit work items always within the
686 * available / maximum scrub list size.
687 *
688 * @param[in] pScrubber OBJMEMSCRUB pointer
689 * @param[in] pageCount number of pages in the array
690 * @param[in] pList pointer will store the return check array
691 * @param[in] pScrubListCopy List where pages are saved
692 * @param[in] pagesToScrubCheck How many pages will need to be saved
693 * @returns the number of work successfully submitted, else 0
694 */
695 static NvU64
_scrubCheckAndSubmit(OBJMEMSCRUB * pScrubber,NvU64 pageCount,PSCRUB_NODE pList,PSCRUB_NODE pScrubListCopy,NvLength pagesToScrubCheck)696 _scrubCheckAndSubmit
697 (
698 OBJMEMSCRUB *pScrubber,
699 NvU64 pageCount,
700 PSCRUB_NODE pList,
701 PSCRUB_NODE pScrubListCopy,
702 NvLength pagesToScrubCheck
703 )
704 {
705 NvU64 iter = 0;
706 NvU64 newId;
707 NV_STATUS status;
708
709 if (pScrubListCopy == NULL && pagesToScrubCheck != 0)
710 {
711 NV_PRINTF(LEVEL_ERROR,
712 "pages need to be saved off, but stash list is invalid\n");
713 goto exit;
714 }
715
716 _scrubWaitAndSave(pScrubber, pScrubListCopy, pagesToScrubCheck);
717
718 for (iter = 0; iter < pageCount; iter++)
719 {
720 newId = pScrubber->lastSubmittedWorkId + 1;
721
722 NV_PRINTF(LEVEL_INFO,
723 "Submitting work, Id: %llx, base: %llx, size: %llx\n",
724 newId, pList[iter].base, pList[iter].size);
725
726 {
727 status =_scrubMemory(pScrubber, pList[iter].base, pList[iter].size, NV_MEMORY_DEFAULT,
728 (NvU32)newId);
729 }
730
731 if(status != NV_OK)
732 {
733 NV_PRINTF(LEVEL_ERROR, "Failing because the work didn't submit.\n");
734 goto exit;
735 }
736 _scrubAddWorkToList(pScrubber, pList[iter].base, pList[iter].size, newId);
737 _scrubCheckProgress(pScrubber);
738 }
739
740 return iter;
741 exit:
742 return 0;
743
744 }
745
746 /**
747 * helper function to return the free space in scrub list
748 */
749 static NvU64
_scrubGetFreeEntries(OBJMEMSCRUB * pScrubber)750 _scrubGetFreeEntries
751 (
752 OBJMEMSCRUB *pScrubber
753 )
754 {
755 return MAX_SCRUB_ITEMS - pScrubber->scrubListSize;
756 }
757
758 /**
759 * helper function to return the max semaphore id that we need to wait for
760 * array of scrub works
761 *
762 * @returns 0, if no entry in list matched the base& end
763 */
764 static NvU64
_searchScrubList(OBJMEMSCRUB * pScrubber,RmPhysAddr base,RmPhysAddr end)765 _searchScrubList
766 (
767 OBJMEMSCRUB *pScrubber,
768 RmPhysAddr base,
769 RmPhysAddr end
770 )
771 {
772 NvU64 tempLastSeenIdByClient = pScrubber->lastSeenIdByClient;
773 NvU64 lastSubmittedWorkId = pScrubber->lastSubmittedWorkId;
774 NvU64 id = 0;
775 NvU64 maxId = 0;
776 RmPhysAddr blockStart = 0;
777 RmPhysAddr blockEnd = 0;
778
779 //
780 // we need not check for lastSubmittedWorkId, since lastSubmittedWorkId is always one more than
781 // the lastSubmittedWorkIdx.
782 //
783 while (tempLastSeenIdByClient != lastSubmittedWorkId)
784 {
785 blockStart = pScrubber->pScrubList[tempLastSeenIdByClient%MAX_SCRUB_ITEMS].base;
786 blockEnd = pScrubber->pScrubList[tempLastSeenIdByClient%MAX_SCRUB_ITEMS].base +
787 pScrubber->pScrubList[tempLastSeenIdByClient%MAX_SCRUB_ITEMS].size - 1;
788
789 // Check whether the page ranges overlap
790 if ( !(blockStart > end || blockEnd < base) )
791 {
792 id = pScrubber->pScrubList[tempLastSeenIdByClient%MAX_SCRUB_ITEMS].id;
793 maxId = (id > maxId) ? id : maxId;
794 }
795 tempLastSeenIdByClient++;
796 }
797 return maxId;
798 }
799
800
801 /**
802 * helper function which waits for a particular submission to complete and
803 * copies the completed work items from scrub list to temporary list
804 *
805 */
806
807 static void
_scrubWaitAndSave(OBJMEMSCRUB * pScrubber,PSCRUB_NODE pList,NvLength itemsToSave)808 _scrubWaitAndSave
809 (
810 OBJMEMSCRUB *pScrubber,
811 PSCRUB_NODE pList,
812 NvLength itemsToSave
813 )
814 {
815 NvU64 currentCompletedId = 0;
816
817 if (itemsToSave == 0)
818 return;
819
820 currentCompletedId = _scrubCheckProgress(pScrubber);
821
822 while (currentCompletedId < (pScrubber->lastSeenIdByClient + itemsToSave))
823 {
824 {
825 if (pScrubber->engineType == NV2080_ENGINE_TYPE_SEC2)
826 sec2utilsServiceInterrupts(pScrubber->pSec2Utils);
827 else
828 ceutilsServiceInterrupts(pScrubber->pCeUtils);
829 }
830 currentCompletedId = _scrubCheckProgress(pScrubber);
831 }
832
833 _scrubCopyListItems(pScrubber, pList, itemsToSave);
834 }
835
836
837 /**
838 * helper function to find and wait for a specific work to complete
839 */
840 static void
_waitForPayload(OBJMEMSCRUB * pScrubber,RmPhysAddr base,RmPhysAddr end)841 _waitForPayload
842 (
843 OBJMEMSCRUB *pScrubber,
844 RmPhysAddr base,
845 RmPhysAddr end
846 )
847 {
848 NvU64 idToWait;
849
850 //We need to look up in the range between [lastSeenIdByClient, lastSubmittedWorkId]
851 idToWait = _searchScrubList(pScrubber, base, end);
852
853 if (idToWait == 0)
854 {
855 return;
856 }
857
858 // Loop will break out, when the semaphore is equal to payload
859 while (_scrubCheckProgress(pScrubber) < idToWait)
860 {
861 portUtilSpin();
862 }
863 }
864
865 /**
866 * helper function to add a work to the scrub list
867 */
868 static void
_scrubAddWorkToList(OBJMEMSCRUB * pScrubber,RmPhysAddr base,NvU64 size,NvU64 newId)869 _scrubAddWorkToList
870 (
871 OBJMEMSCRUB *pScrubber,
872 RmPhysAddr base,
873 NvU64 size,
874 NvU64 newId
875 )
876 {
877 //since the Id works from [1,4k] range, the Idx in which it writes in 1 lesser
878 NvU32 idx = (newId-1) % MAX_SCRUB_ITEMS;
879
880 /*
881 * since this function is called after making sure that there is space
882 * available in the list, no check is needed
883 */
884 NV_ASSERT(pScrubber->pScrubList[idx].id == 0);
885 pScrubber->pScrubList[idx].base = base;
886 pScrubber->pScrubList[idx].size = size;
887 pScrubber->pScrubList[idx].id = newId;
888
889 pScrubber->lastSubmittedWorkId = newId;
890 pScrubber->scrubListSize++;
891 NV_ASSERT(_scrubGetFreeEntries(pScrubber) <= MAX_SCRUB_ITEMS);
892 }
893
894
895
896 /**
897 * Scrubber uses 64 bit index to track the work submitted. But HW supports
898 * only 32 bit semaphore. The current completed Id is calculated here, based
899 * on the lastSeenIdByClient and current HW semaphore value.
900 *
901 * @returns Current Completed 64 bit ID
902 */
903 static NvU64
_scrubCheckProgress(OBJMEMSCRUB * pScrubber)904 _scrubCheckProgress
905 (
906 OBJMEMSCRUB *pScrubber
907 )
908 {
909 NvU32 hwCurrentCompletedId;
910 NvU64 lastSWSemaphoreDone;
911
912 NV_ASSERT(pScrubber != NULL);
913
914 if (pScrubber->bVgpuScrubberEnabled)
915 {
916 hwCurrentCompletedId = pScrubber->vgpuScrubBuffRing.pScrubBuffRingHeader->lastSWSemaphoreDone;
917 lastSWSemaphoreDone = pScrubber->lastSWSemaphoreDone;
918
919 if (hwCurrentCompletedId == (NvU32)lastSWSemaphoreDone)
920 return lastSWSemaphoreDone;
921
922 // check for wrap around case. Increment the upper 32 bits
923 if (hwCurrentCompletedId < (NvU32)lastSWSemaphoreDone)
924 {
925 lastSWSemaphoreDone += 0x100000000ULL;
926 }
927
928 // update lower 32 bits
929 lastSWSemaphoreDone &= 0xFFFFFFFF00000000ULL;
930 lastSWSemaphoreDone |= (NvU64)hwCurrentCompletedId;
931
932 }
933 else
934 {
935 if (pScrubber->engineType == NV2080_ENGINE_TYPE_SEC2)
936 lastSWSemaphoreDone = sec2utilsUpdateProgress(pScrubber->pSec2Utils);
937 else
938 lastSWSemaphoreDone = ceutilsUpdateProgress(pScrubber->pCeUtils);
939 }
940
941 pScrubber->lastSWSemaphoreDone = lastSWSemaphoreDone;
942
943 return lastSWSemaphoreDone;
944 }
945
946
947 /** Single function to memset a surface mapped by GPU. This interface supports
948 both sysmem and vidmem surface, since it uses CE to memset a surface.
949 The user is notified by releasing semaphore with value "payload"
950 */
951 static NV_STATUS
_scrubMemory(OBJMEMSCRUB * pScrubber,RmPhysAddr base,NvU64 size,NvU32 dstCpuCacheAttrib,NvU32 payload)952 _scrubMemory
953 (
954 OBJMEMSCRUB *pScrubber,
955 RmPhysAddr base,
956 NvU64 size,
957 NvU32 dstCpuCacheAttrib,
958 NvU32 payload
959 )
960 {
961 NV_STATUS status = NV_OK;
962 MEMORY_DESCRIPTOR *pMemDesc = NULL;
963 NV_ASSERT_OK_OR_RETURN(memdescCreate(&pMemDesc, pScrubber->pGpu, size, 0, NV_TRUE,
964 ADDR_FBMEM, dstCpuCacheAttrib, MEMDESC_FLAGS_NONE));
965
966 memdescDescribe(pMemDesc, ADDR_FBMEM, base, size);
967
968 if (pScrubber->engineType == NV2080_ENGINE_TYPE_SEC2)
969 {
970 SEC2UTILS_MEMSET_PARAMS memsetParams = {0};
971 memsetParams.pMemDesc = pMemDesc;
972 memsetParams.length = size;
973
974 NV_ASSERT_OK_OR_GOTO(status, sec2utilsMemset(pScrubber->pSec2Utils, &memsetParams), cleanup);
975 pScrubber->lastSubmittedWorkId = memsetParams.submittedWorkId;
976 }
977 else
978 {
979 CEUTILS_MEMSET_PARAMS memsetParams = {0};
980 memsetParams.pMemDesc = pMemDesc;
981 memsetParams.length = size;
982 memsetParams.flags = NV0050_CTRL_MEMSET_FLAGS_ASYNC | NV0050_CTRL_MEMSET_FLAGS_PIPELINED;
983
984 NV_ASSERT_OK_OR_GOTO(status, ceutilsMemset(pScrubber->pCeUtils, &memsetParams), cleanup);
985 pScrubber->lastSubmittedWorkId = memsetParams.submittedWorkId;
986 }
987
988 cleanup:
989 memdescDestroy(pMemDesc);
990 return status;
991 }
992
993 static NV_STATUS
_scrubCombinePages(NvU64 * pPages,NvU64 pageSize,NvU64 pageCount,PSCRUB_NODE * ppScrubList,NvU64 * pSize)994 _scrubCombinePages
995 (
996 NvU64 *pPages,
997 NvU64 pageSize,
998 NvU64 pageCount,
999 PSCRUB_NODE *ppScrubList,
1000 NvU64 *pSize
1001 )
1002 {
1003 NvU64 i, j;
1004
1005 *ppScrubList = (PSCRUB_NODE)portMemAllocNonPaged(sizeof(SCRUB_NODE) * pageCount);
1006 NV_ASSERT_OR_RETURN(*ppScrubList != NULL, NV_ERR_NO_MEMORY);
1007
1008 // Copy first element from original list to new list
1009 (*ppScrubList)[0].base = pPages[0];
1010 (*ppScrubList)[0].size = pageSize;
1011
1012 for (i = 0, j = 0; i < (pageCount - 1); i++)
1013 {
1014 if ((((*ppScrubList)[j].size + pageSize) > SCRUB_MAX_BYTES_PER_LINE) ||
1015 ((pPages[i] + pageSize) != pPages[i+1]))
1016 {
1017 j++;
1018 (*ppScrubList)[j].base = pPages[i+1];
1019 (*ppScrubList)[j].size = pageSize;
1020 }
1021 else
1022 {
1023 (*ppScrubList)[j].size += pageSize;
1024 }
1025 }
1026
1027 *pSize = j + 1;
1028
1029 return NV_OK;
1030 }
1031