1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2009-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/core.h"
25 #include "os/os.h"
26 #include "gpu_mgr/gpu_mgr.h"
27 #include "gpu/mem_mgr/mem_mgr.h"
28 #include "gpu/mem_mgr/fbsr.h"
29 #include "gpu/bus/kern_bus.h"
30 #include "gpu/mem_mgr/mem_desc.h"
31 #include "published/maxwell/gm107/dev_ram.h"
32 #include "core/thread_state.h"
33 #include "nvrm_registry.h"
34
35 //
36 // Implementation notes:
37 //
38 // This file implements two mechanisms for FB save / restore on Fermi. I
39 // would've liked to split the implementation for Begin()/End()/CopyMemory()
40 // into seperate subclasses but our current object model makes inheritance a tad
41 // cumbersome.
42 //
43 // Mechanism #1 (TYPE_DMA): The data is DMA'ed (via CE) to a large non-paged
44 // system memory allocation. This is the preferred / performance path.
45 //
46 // Mechanism #2 (TYPE_CPU): The backup buffer is allocated in system memory
47 // chunks. The sysmem buffers are currently non-paged, long-term the plan is to
48 // switch the allocations to be paged. For this approach we pre-allocate a page
49 // size buffer for CE to DMA into. From that buffer we use CPU IO to move the
50 // data out into the intended storage buffer. The reason for chunks is that it
51 // has been noticed that large non-paged allocations tend to fail more often
52 // than multiple smaller non-paged allocations. Once we move over to paged
53 // allocations here this *might* not be needed. Bug 579780 and 579765 are
54 // tracking the RFE for paged memory. The reason for using CE here is bar2 isn't
55 // yet set up and BAR0 would be really really slow (not that we care about
56 // performance much for this sheme).
57 //
58 // Mechanism #3 (TYPE_PERSISTENT): The video Memory[Fb] data is transferred to
59 // sys_mem by means of DMA[CE engine], sys_mem allocation is pinned across S3
60 // transitions. Sys_mem allocations are done at first S3 cycle and release during
61 // driver unload. this approach reduces system VM fragmentation. Optimus systems,
62 // keeps GPU in D3 state, as long as there is no work for GPU. Because of frequent
63 // transitions between D0 & D3, system is running out of *CONTIGOUS* VM, with this
64 // approach Optimus system could avoid the above problem.
65 //
66 // Mechanism #4 (TYPE_PAGED_DMA): It is basically the same with the TYPE_DMA
67 // method except that we allocate the buffer from pagable memory pool. After the
68 // buffer is allocated, we need to use memdescLock to lock the buffer in physical
69 // memory so that CE can access it and then use memdescUnlock to unlock it.
70 //
71 // Mechanisms #5 and #6 are targetted for WDDM, a large VA section (Paged region)
72 // and a small pinned region are committed on boot.
73 //
74 // Mechanism #5 (TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED):
75 // For power save, map and pin the large va region and if the map succeeds,
76 // ce copy to this large pinned page. At this point it is similar to TYPE_DMA.
77 // If the map and pin fails, fall back to TYPE_WDDM_SLOW_CPU
78 //
79 // Mechanism #6 (TYPE_WDDM_SLOW_CPU_PAGED):
80 // When TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED fails, use a small 64k pinned page
81 // that was preallocated and dma to this pinned page from FB. Once the ce completes,
82 // the chunk is then copied to the paged CPU memory. A 64k chunk size is chosen
83 // because the Windows ZwMapViewOfSection requires 64K alignment
84 //
85 // While technically mechanism #2 can fail (even with paged memory) another
86 // approach worth considering would be to pre-allocate the save buffer in the
87 // video memory allocation path (memdescAlloc). However, with this approach we'd
88 // incur a memory overhead even if S/R was never used.
89 //
90
91 #ifdef DEBUG
92 #endif
93
94 #define CPU_PINNED_BUFFER_SIZE RM_PAGE_SIZE
95 #define CPU_MAX_PINNED_BUFFER_SIZE 0x10000
96
97 //
98 // Maximum data copy size in bytes for file operations (read/write)
99 // which can be transferred with default thread timeout.
100 //
101 #define MAX_FILE_COPY_SIZE_WITHIN_DEFAULT_THREAD_TIMEOUT (64 * 1024 * 1024)
102
_fbsrInitGsp(OBJGPU * pGpu,OBJFBSR * pFbsr)103 static NV_STATUS _fbsrInitGsp
104 (
105 OBJGPU *pGpu,
106 OBJFBSR *pFbsr
107 )
108 {
109 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
110 NvHandle hSysMem = NV01_NULL_OBJECT;
111 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
112 NV2080_CTRL_INTERNAL_FBSR_INIT_PARAMS params;
113
114 // Register sysmem memdesc with GSP. This creates memlist object
115 NV_ASSERT_OK_OR_RETURN(memdescSendMemDescToGSP(pGpu, pFbsr->pSysMemDesc, &hSysMem));
116
117 params.fbsrType = pFbsr->type;
118 params.numRegions = pFbsr->numRegions;
119 params.hClient = pMemoryManager->hClient;
120 params.hSysMem = hSysMem;
121 params.gspFbAllocsSysOffset = pFbsr->gspFbAllocsSysOffset;
122 params.bEnteringGcoffState = pGpu->getProperty(pGpu, PDB_PROP_GPU_GCOFF_STATE_ENTERING);
123
124 // Send S/R init information to GSP
125 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
126 pGpu->hInternalClient,
127 pGpu->hInternalSubdevice,
128 NV2080_CTRL_CMD_INTERNAL_FBSR_INIT,
129 ¶ms,
130 sizeof(params)));
131
132 // Free memlist object
133 pRmApi->Free(pRmApi, pMemoryManager->hClient, hSysMem);
134
135 //
136 // Clear numRegions for next S/R sequence
137 // Needed only to tell GSP how many regions
138 //
139 pFbsr->numRegions = 0;
140
141 return NV_OK;
142 }
143
_fbsrMemoryCopy(OBJGPU * pGpu,OBJFBSR * pFbsr,MEMORY_DESCRIPTOR * pDstMemDesc,NvU64 dstOffset,MEMORY_DESCRIPTOR * pSrcMemDesc,NvU64 srcOffset,NvU64 size)144 static NV_STATUS _fbsrMemoryCopy
145 (
146 OBJGPU *pGpu,
147 OBJFBSR *pFbsr,
148 MEMORY_DESCRIPTOR *pDstMemDesc,
149 NvU64 dstOffset,
150 MEMORY_DESCRIPTOR *pSrcMemDesc,
151 NvU64 srcOffset,
152 NvU64 size
153 )
154 {
155 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
156 NvHandle hVidMem = NV01_NULL_OBJECT;
157 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
158 NV2080_CTRL_INTERNAL_FBSR_SEND_REGION_INFO_PARAMS params;
159
160 // Register vidmem memdesc with GSP. This creates memlist object
161 NV_ASSERT_OK_OR_RETURN(memdescSendMemDescToGSP(pGpu, pSrcMemDesc, &hVidMem));
162
163 portMemSet(¶ms, 0, sizeof(params));
164
165 params.fbsrType = pFbsr->type;
166 params.hClient = pMemoryManager->hClient;
167 params.hVidMem = hVidMem;
168 params.vidOffset = srcOffset;
169 params.sysOffset = dstOffset;
170 params.size = size;
171
172 // Send region information to GSP
173 NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi,
174 pGpu->hInternalClient,
175 pGpu->hInternalSubdevice,
176 NV2080_CTRL_CMD_INTERNAL_FBSR_SEND_REGION_INFO,
177 ¶ms,
178 sizeof(params)));
179
180 // Free memlist object
181 pRmApi->Free(pRmApi, pMemoryManager->hClient, hVidMem);
182
183 return NV_OK;
184 }
185
186 /*!
187 * Init
188 *
189 * @param[in] pGpu OBJGPU pointer
190 * @param[in] pFbsr OBJFBSR pointer
191 *
192 * @returns None
193 */
194 NV_STATUS
fbsrInit_GM107(OBJGPU * pGpu,OBJFBSR * pFbsr)195 fbsrInit_GM107(OBJGPU *pGpu, OBJFBSR *pFbsr)
196 {
197 NV_STATUS status;
198 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
199
200 portMemSet(&pFbsr->pagedBufferInfo, 0, sizeof(pFbsr->pagedBufferInfo));
201
202 // Commit an upper bound VA for both slow cpu and fast dma.
203 if ((pFbsr->type == FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED ||
204 pFbsr->type == FBSR_TYPE_WDDM_SLOW_CPU_PAGED))
205 {
206 // We need it only once not per fbsr scheme though
207 pFbsr->pagedBufferInfo.maxLength = memmgrGetRsvdSizeForSr_HAL(pGpu, pMemoryManager);
208 if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT))
209 {
210 status = osReserveCpuAddressSpaceUpperBound(&pFbsr->pagedBufferInfo.sectionHandle,
211 pFbsr->pagedBufferInfo.maxLength);
212
213 if (status != NV_OK)
214 return status;
215 }
216 }
217
218 //
219 // Allocate a 64K/4k sized buffer for forward progress if the WDDM Fast path
220 // cannot pin the large buffer.
221 //
222 if (pFbsr->type == FBSR_TYPE_CPU ||
223 pFbsr->type == FBSR_TYPE_WDDM_SLOW_CPU_PAGED ||
224 pFbsr->type == FBSR_TYPE_FILE)
225 {
226 NvU32 memSize = 0;
227
228 if (RMCFG_FEATURE_PLATFORM_GSP)
229 {
230 // Can't allocate sysmem from GSP FW.
231 return NV_ERR_NOT_SUPPORTED;
232
233 // GSP FW TODO: Allocate this memory on the client side.
234 }
235
236 if (pFbsr->type == FBSR_TYPE_CPU)
237 {
238 memSize = CPU_PINNED_BUFFER_SIZE;
239 }
240 else
241 {
242 memSize = CPU_MAX_PINNED_BUFFER_SIZE;
243 }
244 //
245 // Pre-allocate a page size buffer for CE to DMA into.
246 // This buffer is accessed with the CPU so it is best
247 // to to use cached memory.
248 //
249 status = memdescCreate(&pFbsr->pSysMemDesc, pGpu, memSize,
250 0, NV_TRUE, ADDR_SYSMEM, NV_MEMORY_CACHED,
251 MEMDESC_FLAGS_NONE);
252 if (status != NV_OK)
253 {
254 NV_ASSERT(status == NV_OK);
255 goto fail;
256 }
257
258 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_126,
259 pFbsr->pSysMemDesc);
260 if (status != NV_OK)
261 {
262 NV_ASSERT(status == NV_OK);
263 memdescDestroy(pFbsr->pSysMemDesc);
264 pFbsr->pSysMemDesc = NULL;
265 goto fail;
266 }
267
268 status = memdescMapOld(pFbsr->pSysMemDesc, 0, memSize, NV_TRUE /*kernel*/ ,
269 NV_PROTECT_READ_WRITE,
270 (pFbsr->type == FBSR_TYPE_FILE ? (void**)&pFbsr->pDmaBuffer:
271 (void**)&pFbsr->pPinnedBuffer),
272 (void **)&pFbsr->pMapCookie);
273 if (status != NV_OK)
274 {
275 NV_ASSERT(0);
276 memdescFree(pFbsr->pSysMemDesc);
277 memdescDestroy(pFbsr->pSysMemDesc);
278 pFbsr->pSysMemDesc = NULL;
279 status = NV_ERR_INSUFFICIENT_RESOURCES;
280 goto fail;
281 }
282
283 NV_ASSERT(!pFbsr->pSysMemDesc->PteAdjust);
284 }
285 pFbsr->bInitialized = NV_TRUE;
286 return NV_OK;
287
288 fail:
289 if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT) &&
290 (pFbsr->type == FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED ||
291 pFbsr->type == FBSR_TYPE_WDDM_SLOW_CPU_PAGED))
292 {
293 osReleaseCpuAddressSpaceUpperBound(pFbsr->pagedBufferInfo.sectionHandle);
294 }
295
296 return status;
297 }
298
299 /*!
300 * Destroy
301 *
302 * @param[in] pGpu OBJGPU pointer
303 * @param[in] pFbsr OBJFBSR pointer
304 *
305 * @returns None
306 */
307 void
fbsrDestroy_GM107(OBJGPU * pGpu,OBJFBSR * pFbsr)308 fbsrDestroy_GM107(OBJGPU *pGpu, OBJFBSR *pFbsr)
309 {
310 if (pFbsr->type == FBSR_TYPE_CPU ||
311 pFbsr->type == FBSR_TYPE_WDDM_SLOW_CPU_PAGED ||
312 pFbsr->type == FBSR_TYPE_FILE)
313 {
314 if (pFbsr->pSysMemDesc)
315 {
316 memdescUnmapOld(pFbsr->pSysMemDesc, 1 /*kernel*/, 0,
317 (pFbsr->type == FBSR_TYPE_FILE) ? (void*)pFbsr->pDmaBuffer :
318 (void*)pFbsr->pPinnedBuffer,
319 pFbsr->pMapCookie);
320 memdescFree(pFbsr->pSysMemDesc);
321 memdescDestroy(pFbsr->pSysMemDesc);
322 pFbsr->pSysMemDesc = NULL;
323 }
324 }
325
326 if (pFbsr->type == FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED ||
327 pFbsr->type == FBSR_TYPE_WDDM_SLOW_CPU_PAGED)
328 {
329 osReleaseCpuAddressSpaceUpperBound(pFbsr->pagedBufferInfo.sectionHandle);
330 }
331 }
332
333 /*!
334 * Start save/restore operation
335 *
336 * @param[in] pGpu OBJGPU pointer
337 * @param[in] pFbsr OBJFBSR pointer
338 * @param[in] op Type of operation
339 *
340 * @returns NV_OK on success
341 */
342 NV_STATUS
fbsrBegin_GM107(OBJGPU * pGpu,OBJFBSR * pFbsr,FBSR_OP_TYPE op)343 fbsrBegin_GM107(OBJGPU *pGpu, OBJFBSR *pFbsr, FBSR_OP_TYPE op)
344 {
345 NV_STATUS status = NV_OK;
346 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
347
348 pFbsr->op = op;
349 pFbsr->bOperationFailed = NV_FALSE;
350 if (op != FBSR_OP_SIZE_BUF && op != FBSR_OP_DESTROY)
351 {
352 if (IS_GSP_CLIENT(pGpu) || IS_VIRTUAL(pGpu))
353 {
354 pFbsr->pCe = NULL;
355 }
356
357 //
358 // On guest, force re-initialize CeUtils with inst-in-sys.
359 // See bug 4414361 for details.
360 //
361 if (IS_VIRTUAL(pGpu))
362 {
363 NvU32 instLocOverrides = pGpu->instLocOverrides;
364 NvU32 instLocOverrides4 = pGpu->instLocOverrides4;
365
366 if (pMemoryManager->pCeUtils != NULL)
367 {
368 memmgrDestroyCeUtils(pMemoryManager, NV_FALSE);
369 }
370
371 pGpu->instLocOverrides = FLD_SET_DRF(_REG_STR, _RM_INST_LOC, _USERD, _NCOH, pGpu->instLocOverrides);
372 pGpu->instLocOverrides4 = FLD_SET_DRF(_REG_STR_RM, _INST_LOC_4, _CHANNEL_PUSHBUFFER, _NCOH, pGpu->instLocOverrides4);
373
374 NV_ASSERT_OK_OR_RETURN(memmgrInitCeUtils(pMemoryManager, NV_FALSE));
375
376 pGpu->instLocOverrides = instLocOverrides;
377 pGpu->instLocOverrides4 = instLocOverrides4;
378 }
379
380 NV_PRINTF(LEVEL_INFO, "%s %lld bytes of data\n",
381 pFbsr->op == FBSR_OP_SAVE ? "saving" : "restoring",
382 pFbsr->length);
383 }
384
385 if (op == FBSR_OP_SAVE)
386 {
387 switch (pFbsr->type)
388 {
389 case FBSR_TYPE_PAGED_DMA:
390 case FBSR_TYPE_DMA:
391 //
392 // Check if system memory is pre-allocated for DMA type FBSR
393 // and use the same for performing FBSR.
394 //
395 if (pFbsr->pSysReservedMemDesc)
396 {
397 //
398 // Validate if reserved system memory size is sufficient,
399 // Otherwise generate the assert and free the
400 // pre-allocated reserved system memory
401 //
402 if (pFbsr->pSysReservedMemDesc->Size >= pFbsr->length)
403 {
404 pFbsr->pSysMemDesc = pFbsr->pSysReservedMemDesc;
405 //
406 // The reference to the reserved memory is transferred
407 // and the pSysMemDesc pointer will be used for actual
408 // FBSR operations. The actual object will be freed
409 // during FBSR_OP_RESTORE operation.
410 //
411 pFbsr->pSysReservedMemDesc = NULL;
412 break;
413 }
414
415 NV_ASSERT(pFbsr->pSysReservedMemDesc->Size >= pFbsr->length);
416 memdescFree(pFbsr->pSysReservedMemDesc);
417 memdescDestroy(pFbsr->pSysReservedMemDesc);
418 pFbsr->pSysReservedMemDesc = NULL;
419 status = NV_ERR_GENERIC;
420 break;
421 }
422
423 if (pFbsr->length)
424 {
425 if (pFbsr->type == FBSR_TYPE_DMA)
426 {
427 // This buffer is never touched by the CPU, so it can be uncached.
428 status = memdescCreate(&pFbsr->pSysMemDesc, pGpu,
429 pFbsr->length, 0, NV_FALSE,
430 ADDR_SYSMEM, NV_MEMORY_UNCACHED,
431 MEMDESC_FLAGS_NONE);
432 }
433 else if (pFbsr->type == FBSR_TYPE_PAGED_DMA)
434 {
435 // On Windows, pageable memory is also cacheable.
436 status = memdescCreate(&pFbsr->pSysMemDesc, pGpu,
437 pFbsr->length, 0, NV_FALSE,
438 ADDR_SYSMEM, NV_MEMORY_CACHED,
439 MEMDESC_FLAGS_PAGED_SYSMEM);
440 }
441 if (status != NV_OK)
442 {
443 NV_ASSERT(status == NV_OK);
444 break;
445 }
446
447 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_127,
448 pFbsr->pSysMemDesc);
449 if (status != NV_OK)
450 {
451 NV_ASSERT(status == NV_OK);
452 memdescDestroy(pFbsr->pSysMemDesc);
453 pFbsr->pSysMemDesc = NULL;
454 break;
455 }
456
457 if (pFbsr->type == FBSR_TYPE_PAGED_DMA)
458 {
459 status = memdescLock(pFbsr->pSysMemDesc);
460 if (status != NV_OK)
461 {
462 NV_ASSERT(status == NV_OK);
463 memdescFree(pFbsr->pSysMemDesc);
464 memdescDestroy(pFbsr->pSysMemDesc);
465 pFbsr->pSysMemDesc = NULL;
466 break;
467 }
468 }
469 }
470
471 break;
472
473 case FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED:
474 {
475 NvBool bIommuEnabled = pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT);
476
477 if (pFbsr->length > pFbsr->pagedBufferInfo.maxLength)
478 {
479 status = NV_ERR_GENERIC;
480 break;
481 }
482
483 if (bIommuEnabled)
484 {
485 status = osSrPinSysmem(pGpu->pOsGpuInfo,
486 pFbsr->length,
487 &pFbsr->pagedBufferInfo.pMdl);
488
489 if (status != NV_OK)
490 break;
491
492 NV_ASSERT(pFbsr->pagedBufferInfo.pMdl);
493 status = osCreateMemFromOsDescriptorInternal(pGpu,
494 pFbsr->pagedBufferInfo.pMdl,
495 0,
496 pFbsr->length,
497 &pFbsr->pSysMemDesc,
498 NV_TRUE,
499 RS_PRIV_LEVEL_KERNEL
500 );
501 if (status != NV_OK)
502 (void) osSrUnpinSysmem(pGpu->pOsGpuInfo);
503 }
504 else
505 {
506 pFbsr->pagedBufferInfo.sysAddr = 0;
507 status = osMapViewToSection(pGpu->pOsGpuInfo,
508 pFbsr->pagedBufferInfo.sectionHandle,
509 (void **) (&pFbsr->pagedBufferInfo.sysAddr),
510 pFbsr->length, 0, bIommuEnabled);
511 NV_ASSERT(pFbsr->pagedBufferInfo.sysAddr);
512 if (status != NV_OK)
513 break;
514
515 status = osCreateMemFromOsDescriptorInternal(pGpu,
516 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
517 0,
518 pFbsr->length,
519 &pFbsr->pSysMemDesc,
520 NV_TRUE,
521 RS_PRIV_LEVEL_KERNEL);
522 // would return error
523 if (status != NV_OK)
524 {
525 NV_ASSERT(osUnmapViewFromSection(pGpu->pOsGpuInfo,
526 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
527 bIommuEnabled) == NV_OK);
528 }
529 }
530 }
531 break;
532
533 case FBSR_TYPE_WDDM_SLOW_CPU_PAGED:
534 if (pFbsr->length > pFbsr->pagedBufferInfo.maxLength ||
535 (!pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT) &&
536 !pFbsr->pagedBufferInfo.sectionHandle))
537 {
538 status = NV_ERR_GENERIC;
539 break;
540 }
541 pFbsr->pagedBufferInfo.avblViewSz = 0;
542 // fallthrough
543 case FBSR_TYPE_CPU:
544 if (!pFbsr->pSysMemDesc)
545 {
546 status = NV_ERR_GENERIC;
547 }
548
549 break;
550 case FBSR_TYPE_PERSISTENT:
551 break;
552 case FBSR_TYPE_FILE:
553 // XXX can this condition ever evaluate to true?
554 if (!pFbsr->pSysMemDesc)
555 {
556 status = NV_ERR_GENERIC;
557 break;
558 }
559
560 // Open a temporary file for writing
561 status = osOpenTemporaryFile(&pFbsr->pagedBufferInfo.sectionHandle);
562 if (status != NV_OK)
563 break;
564
565 pFbsr->pagedBufferInfo.avblViewSz = 0;
566 break;
567 default:
568 status = NV_ERR_GENERIC;
569 NV_ASSERT(0);
570 break;
571 }
572
573 // Initialize FBSR on GSP
574 if ((status == NV_OK) && IS_GSP_CLIENT(pGpu) && (pFbsr->pSysMemDesc != NULL))
575 {
576 NV_ASSERT_OK_OR_RETURN(_fbsrInitGsp(pGpu, pFbsr));
577 }
578 }
579 else if (pFbsr->op == FBSR_OP_RESTORE || pFbsr->op == FBSR_OP_DESTROY)
580 {
581 switch (pFbsr->type)
582 {
583 case FBSR_TYPE_PAGED_DMA:
584 status = memdescLock(pFbsr->pSysMemDesc);
585 NV_ASSERT(status == NV_OK);
586 break;
587 case FBSR_TYPE_FILE:
588 if (!pFbsr->pagedBufferInfo.sectionHandle)
589 {
590 status = NV_ERR_GENERIC;
591 break;
592 }
593 pFbsr->pagedBufferInfo.avblViewSz = 0;
594 break;
595 case FBSR_TYPE_WDDM_SLOW_CPU_PAGED:
596 if (pFbsr->length > pFbsr->pagedBufferInfo.maxLength ||
597 (!pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT) &&
598 !pFbsr->pagedBufferInfo.sectionHandle))
599 {
600 status = NV_ERR_GENERIC;
601 break;
602 }
603 pFbsr->pagedBufferInfo.avblViewSz = 0;
604 break;
605 case FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED:
606 {
607 NvBool bIommuEnabled = pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT);
608 // Error checked during SAVE
609 if (pFbsr->length > pFbsr->pagedBufferInfo.maxLength)
610 {
611 status = NV_ERR_GENERIC;
612 break;
613 }
614 if(bIommuEnabled)
615 {
616 status = osSrPinSysmem(pGpu->pOsGpuInfo,
617 pFbsr->length,
618 &pFbsr->pagedBufferInfo.pMdl);
619
620 if (status != NV_OK)
621 break;
622
623 NV_ASSERT(pFbsr->pagedBufferInfo.pMdl);
624 status = osCreateMemFromOsDescriptorInternal(pGpu,
625 pFbsr->pagedBufferInfo.pMdl,
626 0,
627 pFbsr->length,
628 &pFbsr->pSysMemDesc,
629 NV_TRUE,
630 RS_PRIV_LEVEL_KERNEL
631 );
632 if (status != NV_OK)
633 (void) osSrUnpinSysmem(pGpu->pOsGpuInfo);
634 }
635 else
636 {
637 pFbsr->pagedBufferInfo.sysAddr = 0;
638 status = osMapViewToSection(pGpu->pOsGpuInfo,
639 pFbsr->pagedBufferInfo.sectionHandle,
640 (void **)(&pFbsr->pagedBufferInfo.sysAddr),
641 pFbsr->length, 0, bIommuEnabled);
642
643 if (status != NV_OK)
644 break;
645
646 NV_ASSERT(pFbsr->pagedBufferInfo.sysAddr);
647 status = osCreateMemFromOsDescriptorInternal(pGpu,
648 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
649 0,
650 pFbsr->length,
651 &pFbsr->pSysMemDesc,
652 NV_TRUE,
653 RS_PRIV_LEVEL_KERNEL);
654 // would return error
655 if (status != NV_OK)
656 {
657 NV_ASSERT(osUnmapViewFromSection(pGpu->pOsGpuInfo,
658 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
659 bIommuEnabled) == NV_OK);
660 }
661 }
662 }
663
664 break;
665 }
666 }
667
668 pFbsr->pSysMemNodeCurrent = pFbsr->pSysMemNodeHead;
669 pFbsr->length = 0;
670 pFbsr->sysOffset = 0;
671
672 return status;
673 }
674
675 /*!
676 * End save/restore operation
677 *
678 * @param[in] pGpu OBJGPU pointer
679 * @param[in] pFbsr OBJFBSR pointer
680 *
681 * @returns NV_OK on success
682 */
683 NV_STATUS
fbsrEnd_GM107(OBJGPU * pGpu,OBJFBSR * pFbsr)684 fbsrEnd_GM107(OBJGPU *pGpu, OBJFBSR *pFbsr)
685 {
686 NvBool bIommuEnabled = pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT);
687 NV_STATUS status = NV_OK;
688 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
689
690 if (pFbsr->op != FBSR_OP_SIZE_BUF && pFbsr->op != FBSR_OP_DESTROY)
691 {
692
693 if (IS_VIRTUAL(pGpu) && pMemoryManager->pCeUtils != NULL)
694 {
695 memmgrDestroyCeUtils(pMemoryManager, NV_FALSE);
696 }
697 }
698
699 if (pFbsr->op == FBSR_OP_RESTORE || pFbsr->bOperationFailed || pFbsr->op == FBSR_OP_DESTROY)
700 {
701 switch (pFbsr->type)
702 {
703 case FBSR_TYPE_PAGED_DMA:
704 case FBSR_TYPE_DMA:
705 {
706 if (pFbsr->type == FBSR_TYPE_PAGED_DMA)
707 {
708 memdescUnlock(pFbsr->pSysMemDesc);
709 }
710 memdescFree(pFbsr->pSysMemDesc);
711 memdescDestroy(pFbsr->pSysMemDesc);
712 pFbsr->pSysMemDesc = NULL;
713 break;
714 }
715 case FBSR_TYPE_WDDM_SLOW_CPU_PAGED:
716 // Flush out writes
717 osFlushCpuWriteCombineBuffer();
718 if (pFbsr->pagedBufferInfo.avblViewSz)
719 {
720 if (osUnmapViewFromSection(pGpu->pOsGpuInfo,
721 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
722 bIommuEnabled) != NV_OK)
723 {
724 pFbsr->bOperationFailed = NV_TRUE;
725 }
726 }
727 break;
728 case FBSR_TYPE_FILE:
729 // Flush out writes
730 osFlushCpuWriteCombineBuffer();
731
732 // Close the file
733 osCloseFile(pFbsr->pagedBufferInfo.sectionHandle);
734 break;
735 case FBSR_TYPE_CPU:
736 {
737 PFBSR_NODE pNode;
738 PFBSR_NODE pNext;
739
740 // Flush out writes
741 osFlushCpuWriteCombineBuffer();
742
743 // Free up list
744 pNode = pFbsr->pSysMemNodeHead;
745
746 while (pNode)
747 {
748 pNext = pNode->pNext;
749 portMemFree(pNode);
750 pNode = pNext;
751 }
752
753 pFbsr->pSysMemNodeHead = NULL;
754
755 break;
756 }
757
758 case FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED:
759 memdescDestroy(pFbsr->pSysMemDesc);
760 pFbsr->pSysMemDesc = NULL;
761 if(bIommuEnabled)
762 {
763 status = osSrUnpinSysmem(pGpu->pOsGpuInfo);
764 }
765 else
766 {
767 status = osUnmapViewFromSection(pGpu->pOsGpuInfo,
768 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
769 bIommuEnabled);
770 }
771 if (status!= NV_OK)
772 {
773 pFbsr->bOperationFailed = NV_TRUE;
774 }
775 break;
776
777 }
778 }
779 else if (pFbsr->op == FBSR_OP_SAVE)
780 {
781 switch (pFbsr->type)
782 {
783 case FBSR_TYPE_PAGED_DMA:
784 memdescUnlock(pFbsr->pSysMemDesc);
785 break;
786 case FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED:
787 memdescDestroy(pFbsr->pSysMemDesc);
788 pFbsr->pSysMemDesc = NULL;
789 if(bIommuEnabled)
790 {
791 status = osSrUnpinSysmem(pGpu->pOsGpuInfo);
792 }
793 else
794 {
795 status = osUnmapViewFromSection(pGpu->pOsGpuInfo,
796 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
797 bIommuEnabled);
798 }
799 if (status != NV_OK)
800 {
801 pFbsr->bOperationFailed = NV_TRUE;
802 }
803 break;
804 case FBSR_TYPE_WDDM_SLOW_CPU_PAGED:
805 if (pFbsr->pagedBufferInfo.avblViewSz)
806 {
807 if (osUnmapViewFromSection(pGpu->pOsGpuInfo,
808 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
809 bIommuEnabled) != NV_OK)
810 {
811 pFbsr->bOperationFailed = NV_TRUE;
812 }
813 }
814 break;
815 }
816 }
817
818 return pFbsr->bOperationFailed ? NV_ERR_GENERIC : NV_OK;
819 }
820
821 /*!
822 * Saves or restores a region of video memory.
823 *
824 * @param[in] pGpu OBJGPU pointer
825 * @param[in] pFbsr OBJFBSR pointer
826 * @param[in] pVidMemDesc Memory descriptor for vidmem region
827 *
828 * @returns None
829 */
830 void
fbsrCopyMemoryMemDesc_GM107(OBJGPU * pGpu,OBJFBSR * pFbsr,MEMORY_DESCRIPTOR * pVidMemDesc)831 fbsrCopyMemoryMemDesc_GM107(OBJGPU *pGpu, OBJFBSR *pFbsr, MEMORY_DESCRIPTOR *pVidMemDesc)
832 {
833 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
834 NV_STATUS status = NV_OK;
835
836 NV_ASSERT(!gpumgrGetBcEnabledStatus(pGpu));
837
838 pVidMemDesc = memdescGetMemDescFromGpu(pVidMemDesc, pGpu);
839
840 if (pFbsr->bOperationFailed)
841 {
842 // If we hit a failure igonre the rest of the copy requests
843 return;
844 }
845
846 pFbsr->length += pVidMemDesc->Size;
847
848 // We should have nothing reserved when FB is broken
849 if (pGpu->getProperty(pGpu, PDB_PROP_GPU_BROKEN_FB))
850 {
851 NV_ASSERT(pVidMemDesc->Size == 0);
852 NV_PRINTF(LEVEL_WARNING, "return early since FB is broken!\n");
853 return;
854 }
855
856 if (pFbsr->op == FBSR_OP_SIZE_BUF)
857 {
858 pFbsr->numRegions++;
859
860 switch (pFbsr->type)
861 {
862 case FBSR_TYPE_CPU:
863 {
864 PFBSR_NODE pNode;
865
866 pNode = portMemAllocNonPaged(
867 sizeof(FBSR_NODE) + (NvU32)pVidMemDesc->Size - sizeof(pNode->data));
868
869 if (pNode == NULL)
870 {
871 NV_ASSERT(0);
872 pFbsr->bOperationFailed = NV_TRUE;
873 return;
874 }
875
876 pNode->pNext = NULL;
877
878 // Insert node
879 if (!pFbsr->pSysMemNodeHead)
880 {
881 pFbsr->pSysMemNodeHead = pNode;
882 }
883 else
884 {
885 pFbsr->pSysMemNodeCurrent->pNext = pNode;
886 }
887
888 pFbsr->pSysMemNodeCurrent = pNode;
889
890 break;
891 }
892 case FBSR_TYPE_PERSISTENT:
893 {
894 MEMORY_DESCRIPTOR *pStandbyBuffer;
895
896 if (memdescGetStandbyBuffer(pVidMemDesc) == NULL)
897 {
898 status = memdescCreate(&pStandbyBuffer, pGpu,
899 pVidMemDesc->Size, 0, NV_FALSE,
900 ADDR_SYSMEM,
901 NV_MEMORY_WRITECOMBINED,
902 MEMDESC_FLAGS_NONE);
903 if (status == NV_OK)
904 {
905 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_128,
906 pStandbyBuffer);
907 if (status != NV_OK )
908 {
909 memdescDestroy(pStandbyBuffer);
910 pFbsr->bOperationFailed = NV_TRUE;
911 break;
912 }
913 memdescSetStandbyBuffer(pVidMemDesc, pStandbyBuffer);
914 }
915 else
916 {
917 pFbsr->bOperationFailed = NV_TRUE;
918 }
919 }
920 break;
921 }
922 default:
923 break;
924 }
925 }
926 else
927 {
928 NV_PRINTF(LEVEL_INFO, "%s allocation %llx-%llx [%s]\n",
929 pFbsr->op == FBSR_OP_SAVE ? "saving" : "restoring",
930 memdescGetPhysAddr(pVidMemDesc, AT_GPU, 0),
931 memdescGetPhysAddr(pVidMemDesc, AT_GPU, 0) + pVidMemDesc->Size - 1,
932 pFbsr->type == FBSR_TYPE_DMA ? "DMA" : "CPU");
933
934 switch (pFbsr->type)
935 {
936 case FBSR_TYPE_WDDM_FAST_DMA_DEFERRED_NONPAGED:
937 case FBSR_TYPE_PAGED_DMA:
938 case FBSR_TYPE_DMA:
939 {
940 TRANSFER_SURFACE vidSurface = {0};
941 TRANSFER_SURFACE sysSurface = {0};
942
943 vidSurface.pMemDesc = pVidMemDesc;
944 sysSurface.pMemDesc = pFbsr->pSysMemDesc;
945 sysSurface.offset = pFbsr->sysOffset;
946
947 if (pFbsr->op == FBSR_OP_RESTORE)
948 {
949 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &vidSurface, &sysSurface, pVidMemDesc->Size,
950 TRANSFER_FLAGS_PREFER_CE | TRANSFER_FLAGS_CE_PRI_DEFER_FLUSH));
951 }
952 else
953 {
954 if (IS_GSP_CLIENT(pGpu))
955 {
956 _fbsrMemoryCopy(pGpu, pFbsr, pFbsr->pSysMemDesc,
957 pFbsr->sysOffset, pVidMemDesc, 0,
958 pVidMemDesc->Size);
959 }
960 else
961 {
962 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &sysSurface, &vidSurface, pVidMemDesc->Size,
963 TRANSFER_FLAGS_PREFER_CE | TRANSFER_FLAGS_CE_PRI_DEFER_FLUSH));
964 }
965 }
966 break;
967 }
968 case FBSR_TYPE_PERSISTENT:
969 {
970 TRANSFER_SURFACE vidSurface = {0};
971 TRANSFER_SURFACE sysSurface = {0};
972
973 vidSurface.pMemDesc = pVidMemDesc;
974 sysSurface.pMemDesc = memdescGetStandbyBuffer(pVidMemDesc);
975
976 if (pFbsr->op == FBSR_OP_RESTORE)
977 {
978 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &vidSurface, &sysSurface, pVidMemDesc->Size,
979 TRANSFER_FLAGS_PREFER_CE | TRANSFER_FLAGS_CE_PRI_DEFER_FLUSH));
980 }
981 else
982 {
983 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &sysSurface, &vidSurface, pVidMemDesc->Size,
984 TRANSFER_FLAGS_PREFER_CE | TRANSFER_FLAGS_CE_PRI_DEFER_FLUSH));
985 }
986 break;
987 }
988 case FBSR_TYPE_WDDM_SLOW_CPU_PAGED:
989 {
990 NvU64 totalCopySize = pVidMemDesc->Size;
991 NvU64 vidOffset = 0;
992 NvU64 copySize, i;
993 NvU32 *cpuCopyOffset = 0;
994 NvBool bIommuEnabled = pGpu->getProperty(pGpu, PDB_PROP_GPU_ENABLE_IOMMU_SUPPORT);
995
996 // Saves/restores a fbregion at memdesc granularity
997 while(totalCopySize)
998 {
999 // Backing system buffer (paged VA) is 64k and only map new 64k section
1000 // when backing buffer is consumed.
1001 if (pFbsr->pagedBufferInfo.avblViewSz == 0)
1002 {
1003 NV_ASSERT(((pFbsr->sysOffset + vidOffset)& 0xffff) == 0);
1004 pFbsr->pagedBufferInfo.avblViewSz = CPU_MAX_PINNED_BUFFER_SIZE;
1005 pFbsr->pagedBufferInfo.sysAddr = 0;
1006
1007 // Get VA to a 64K view in the section at the offset sysOffset + vidOffset
1008 // sysOffset tracks across memdesc, vidOffset tracks the 4k copy
1009 status = osMapViewToSection(pGpu->pOsGpuInfo,
1010 pFbsr->pagedBufferInfo.sectionHandle,
1011 (void **)(&pFbsr->pagedBufferInfo.sysAddr),
1012 pFbsr->pagedBufferInfo.avblViewSz,
1013 pFbsr->sysOffset + vidOffset,
1014 bIommuEnabled);
1015 if (status != NV_OK)
1016 {
1017 pFbsr->bOperationFailed = NV_TRUE;
1018 NV_ASSERT(0);
1019 break;
1020 }
1021 }
1022 // Compute the cpuOffset to copy to / from
1023 cpuCopyOffset = KERNEL_POINTER_FROM_NvP64(NvU32*, NvP64_PLUS_OFFSET(pFbsr->pagedBufferInfo.sysAddr,
1024 ((NvU64) CPU_MAX_PINNED_BUFFER_SIZE - pFbsr->pagedBufferInfo.avblViewSz)));
1025 copySize = NV_MIN(pFbsr->pagedBufferInfo.avblViewSz, totalCopySize);
1026
1027 if (pFbsr->op == FBSR_OP_RESTORE)
1028 {
1029 // pPinnedBuffer is the 64K scratch buffer
1030 // cpuCopyOffset is the mapped paged CPU VA
1031 // Restore from the backing store to the pinned 64k scratch buffer
1032 // We copy the region in 4byte granularity
1033 for (i = 0; i < (copySize / 4); i++)
1034 {
1035 pFbsr->pPinnedBuffer[i] = cpuCopyOffset[i];
1036 }
1037 }
1038
1039 TRANSFER_SURFACE vidSurface = {0};
1040 vidSurface.pMemDesc = pVidMemDesc;
1041 vidSurface.offset = vidOffset;
1042
1043 TRANSFER_SURFACE sysSurface = {0};
1044 sysSurface.pMemDesc = pFbsr->pSysMemDesc;
1045
1046 if (pFbsr->op == FBSR_OP_RESTORE)
1047 {
1048 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &vidSurface, &sysSurface, copySize, TRANSFER_FLAGS_PREFER_CE));
1049 }
1050 else
1051 {
1052 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &sysSurface, &vidSurface, copySize, TRANSFER_FLAGS_PREFER_CE));
1053 }
1054
1055 if (pFbsr->op == FBSR_OP_SAVE)
1056 {
1057 // Copy from the scratch buffer to the sysmem backing store
1058 for (i = 0; i < (copySize / 4); i++)
1059 {
1060 cpuCopyOffset[i] = pFbsr->pPinnedBuffer[i];
1061 }
1062 }
1063
1064 vidOffset += copySize;
1065 totalCopySize -= copySize;
1066 pFbsr->pagedBufferInfo.avblViewSz -= copySize;
1067 if (pFbsr->pagedBufferInfo.avblViewSz == 0)
1068 {
1069 status = osUnmapViewFromSection(pGpu->pOsGpuInfo,
1070 NvP64_VALUE(pFbsr->pagedBufferInfo.sysAddr),
1071 bIommuEnabled);
1072 if (status != NV_OK)
1073 {
1074 pFbsr->bOperationFailed = NV_TRUE;
1075 NV_ASSERT(0);
1076 break;
1077 }
1078 }
1079 }
1080 }
1081 break;
1082 case FBSR_TYPE_FILE:
1083 {
1084 NvU64 totalCopySize = pVidMemDesc->Size;
1085 NvU64 vidOffset = 0;
1086 NvU64 copySize;
1087 NvU64 threadTimeoutCopySize = 0;
1088
1089 //
1090 // File based operation can take longer time in completion, if the
1091 // FB usage is high. Also, the File read/write time is
1092 // system dependent (the temporary file location, Free
1093 // system RAM, secondary storage type, etc.). Reset the thread
1094 // timeout at the starting and at the end to prevent GPU
1095 // thread timeout errors. Also, keep track of file copy size
1096 // since last threadStateResetTimeout() in variable
1097 // threadTimeoutCopySize. When this copy size go above certain
1098 // threshold, then also reset the thread timeout and
1099 // threadTimeoutCopySize variable.
1100 //
1101 NV_ASSERT_OK(threadStateResetTimeout(pGpu));
1102
1103 // Saves/restores a fbregion at memdesc granularity
1104 while (totalCopySize)
1105 {
1106 // Backing system buffer (paged VA) is 64k and only map new 64k section
1107 // when backing buffer is consumed.
1108 if (pFbsr->pagedBufferInfo.avblViewSz == 0)
1109 {
1110 NV_ASSERT(((pFbsr->sysOffset + vidOffset) & (CPU_MAX_PINNED_BUFFER_SIZE - 1)) == 0);
1111 pFbsr->pagedBufferInfo.avblViewSz = CPU_MAX_PINNED_BUFFER_SIZE;
1112 }
1113
1114 copySize = NV_MIN(pFbsr->pagedBufferInfo.avblViewSz, totalCopySize);
1115
1116 if (threadTimeoutCopySize >= MAX_FILE_COPY_SIZE_WITHIN_DEFAULT_THREAD_TIMEOUT)
1117 {
1118 NV_ASSERT_OK(threadStateResetTimeout(pGpu));
1119 threadTimeoutCopySize = 0;
1120 }
1121
1122 threadTimeoutCopySize += copySize;
1123
1124 if (pFbsr->op == FBSR_OP_RESTORE)
1125 {
1126 // pPinnedBuffer is the 64K scratch buffer
1127 // cpuCopyOffset is the mapped paged CPU VA
1128 // Restore from the backing store to the pinned 64k scratch buffer
1129 // We copy the region in 4byte granularity
1130 // replace this with file read
1131 status = osReadFromFile(pFbsr->pagedBufferInfo.sectionHandle,
1132 &pFbsr->pDmaBuffer[0],
1133 copySize,
1134 pFbsr->sysOffset + vidOffset);
1135 if (status != NV_OK)
1136 {
1137 pFbsr->bOperationFailed = NV_TRUE;
1138 NV_ASSERT(0);
1139 break;
1140 }
1141 }
1142
1143 TRANSFER_SURFACE vidSurface = {0};
1144 vidSurface.pMemDesc = pVidMemDesc;
1145 vidSurface.offset = vidOffset;
1146
1147 TRANSFER_SURFACE sysSurface = {0};
1148 sysSurface.pMemDesc = pFbsr->pSysMemDesc;
1149
1150 if (pFbsr->op == FBSR_OP_RESTORE)
1151 {
1152 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &vidSurface, &sysSurface, copySize, TRANSFER_FLAGS_PREFER_CE));
1153 }
1154 else
1155 {
1156 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &sysSurface, &vidSurface, copySize, TRANSFER_FLAGS_PREFER_CE));
1157 }
1158
1159 if (pFbsr->op == FBSR_OP_SAVE)
1160 {
1161 status = osWriteToFile(pFbsr->pagedBufferInfo.sectionHandle,
1162 &pFbsr->pDmaBuffer[0],
1163 copySize,
1164 pFbsr->sysOffset + vidOffset);
1165 if (status != NV_OK)
1166 {
1167 pFbsr->bOperationFailed = NV_TRUE;
1168 NV_ASSERT(0);
1169 break;
1170 }
1171 }
1172
1173 vidOffset += copySize;
1174 totalCopySize -= copySize;
1175 pFbsr->pagedBufferInfo.avblViewSz -= copySize;
1176 }
1177 }
1178
1179 NV_ASSERT_OK(threadStateResetTimeout(pGpu));
1180 break;
1181 case FBSR_TYPE_CPU:
1182 {
1183 PFBSR_NODE pNode = pFbsr->pSysMemNodeCurrent;
1184 NvU64 vidOffset = 0;
1185 NvU64 copySize;
1186 NvU64 size = pVidMemDesc->Size;
1187 NvU64 i;
1188
1189 NV_ASSERT(pNode);
1190 NV_ASSERT((pFbsr->sysOffset & 3) == 0);
1191
1192 while (size)
1193 {
1194 copySize = NV_MIN(CPU_PINNED_BUFFER_SIZE, size);
1195
1196 if (pFbsr->op == FBSR_OP_RESTORE)
1197 {
1198 for (i = 0; i < (copySize / 4); i++)
1199 {
1200 pFbsr->pPinnedBuffer[i] = pNode->data[i];
1201 }
1202 }
1203
1204 TRANSFER_SURFACE vidSurface = {0};
1205 vidSurface.pMemDesc = pVidMemDesc;
1206 vidSurface.offset = vidOffset;
1207
1208 TRANSFER_SURFACE sysSurface = {0};
1209 sysSurface.pMemDesc = pFbsr->pSysMemDesc;
1210
1211 if (pFbsr->op == FBSR_OP_RESTORE)
1212 {
1213 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &vidSurface, &sysSurface, copySize, TRANSFER_FLAGS_PREFER_CE));
1214 }
1215 else
1216 {
1217 NV_ASSERT_OK(memmgrMemCopy(pMemoryManager, &sysSurface, &vidSurface, copySize, TRANSFER_FLAGS_PREFER_CE));
1218 }
1219
1220 if (pFbsr->op == FBSR_OP_SAVE)
1221 {
1222 for (i = 0; i < (copySize / 4); i++)
1223 {
1224 pNode->data[i] = pFbsr->pPinnedBuffer[i];
1225 }
1226 }
1227
1228 vidOffset += copySize;
1229 size -= copySize;
1230 }
1231
1232 pFbsr->pSysMemNodeCurrent = pFbsr->pSysMemNodeCurrent->pNext;
1233 }
1234
1235 break;
1236 }
1237
1238 pFbsr->sysOffset += pVidMemDesc->Size;
1239 }
1240 }
1241
1242 #ifdef DEBUG
1243 #endif
1244