1 /*
2 * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3 * SPDX-License-Identifier: MIT
4 *
5 * Permission is hereby granted, free of charge, to any person obtaining a
6 * copy of this software and associated documentation files (the "Software"),
7 * to deal in the Software without restriction, including without limitation
8 * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9 * and/or sell copies of the Software, and to permit persons to whom the
10 * Software is furnished to do so, subject to the following conditions:
11 *
12 * The above copyright notice and this permission notice shall be included in
13 * all copies or substantial portions of the Software.
14 *
15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21 * DEALINGS IN THE SOFTWARE.
22 */
23
24 #include "core/core.h"
25 #include "gpu/gpu.h"
26 #include "os/os.h"
27 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
28 #include "kernel/gpu/mem_mgr/mem_mgr.h"
29 #include "gpu/mem_mgr/mem_desc.h"
30 #include "ctrl/ctrl2080/ctrl2080fb.h"
31
32 #include "published/ampere/ga100/dev_fb.h"
33 #include "published/ampere/ga100/hwproject.h"
34
35 /*!
36 * @brief Write the sysmemFlushBuffer val into the NV_PFB_NISO_FLUSH_SYSMEM_ADDR register
37 *
38 * @param[in] pGpu OBJGPU pointer
39 * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
40 *
41 * @returns void
42 */
43 void
kmemsysProgramSysmemFlushBuffer_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)44 kmemsysProgramSysmemFlushBuffer_GA100
45 (
46 OBJGPU *pGpu,
47 KernelMemorySystem *pKernelMemorySystem
48 )
49 {
50 NvU64 alignedSysmemFlushBufferAddr = 0x0;
51 NvU32 alignedSysmemFlushBufferAddrHi = 0x0;
52
53 NV_ASSERT(pKernelMemorySystem->sysmemFlushBuffer != 0);
54
55 alignedSysmemFlushBufferAddr = pKernelMemorySystem->sysmemFlushBuffer >> kmemsysGetFlushSysmemBufferAddrShift_HAL(pGpu, pKernelMemorySystem);
56 alignedSysmemFlushBufferAddrHi = DRF_VAL(_PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40,
57 NvU64_HI32(alignedSysmemFlushBufferAddr));
58
59 NV_ASSERT((alignedSysmemFlushBufferAddrHi & (~NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI_MASK)) == 0);
60
61 alignedSysmemFlushBufferAddrHi &= NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI_MASK;
62
63 GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40,
64 alignedSysmemFlushBufferAddrHi);
65 GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR, _ADR_39_08,
66 NvU64_LO32(alignedSysmemFlushBufferAddr));
67 }
68
69 /*
70 * @brief Initialize the sysmem flush buffer
71 *
72 * Setting up the sysmem flush buffer needs to be done very early in some cases
73 * as it's required for the GPU to perform a system flush. One such case is
74 * resetting GPU FALCONs and in particular resetting the PMU as part of VBIOS
75 * init.
76 *
77 * @returns NV_OK if all is okay. Otherwise an error-specific value.
78 */
79 NV_STATUS
kmemsysInitFlushSysmemBuffer_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)80 kmemsysInitFlushSysmemBuffer_GA100
81 (
82 OBJGPU *pGpu,
83 KernelMemorySystem *pKernelMemorySystem
84 )
85 {
86 NV_STATUS status;
87
88 //
89 // In case of suspend/resume, the buffer might be already allocated, but
90 // the HW still needs to be programmed below.
91 //
92 if (pKernelMemorySystem->pSysmemFlushBufferMemDesc == NULL)
93 {
94 NvU64 flags = MEMDESC_FLAGS_NONE;
95
96 flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY;
97 //
98 // Sysmem flush buffer
99 // The sysmembar flush does a zero byte read of sysmem if there was a
100 // sysmem write since the last flush. The actual memory does have
101 // to be valid and allocated at all times because an actual read may
102 // be issued.
103 //
104 status = memdescCreate(&pKernelMemorySystem->pSysmemFlushBufferMemDesc,
105 pGpu, RM_PAGE_SIZE,
106 1 << kmemsysGetFlushSysmemBufferAddrShift_HAL(pGpu, pKernelMemorySystem),
107 NV_TRUE,
108 ADDR_SYSMEM,
109 NV_MEMORY_UNCACHED,
110 flags);
111 if (status != NV_OK)
112 return status;
113
114 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_139,
115 pKernelMemorySystem->pSysmemFlushBufferMemDesc);
116
117 if (status != NV_OK)
118 {
119 NV_PRINTF(LEVEL_ERROR,
120 "Could not allocate sysmem flush buffer: %x\n", status);
121 DBG_BREAKPOINT();
122 return status;
123 }
124
125 pKernelMemorySystem->sysmemFlushBuffer = memdescGetPhysAddr(pKernelMemorySystem->pSysmemFlushBufferMemDesc, AT_GPU, 0);
126 }
127
128 kmemsysProgramSysmemFlushBuffer_HAL(pGpu, pKernelMemorySystem);
129 return NV_OK;
130 }
131
132 /*!
133 * @brief Validate the sysmemFlushBuffer val and assert
134 *
135 * @param[in] pGpu OBJGPU pointer
136 * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
137 *
138 * @returns void
139 */
140 void
kmemsysAssertSysmemFlushBufferValid_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)141 kmemsysAssertSysmemFlushBufferValid_GA100
142 (
143 OBJGPU *pGpu,
144 KernelMemorySystem *pKernelMemorySystem
145 )
146 {
147 NV_ASSERT((GPU_REG_RD_DRF(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR, _ADR_39_08) != 0)
148 || (GPU_REG_RD_DRF(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40) != 0));
149 }
150
151 /*!
152 * @brief Read MIG Memory CFG register
153 */
154 NV_STATUS
kmemsysReadMIGMemoryCfg_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)155 kmemsysReadMIGMemoryCfg_GA100
156 (
157 OBJGPU *pGpu,
158 KernelMemorySystem *pKernelMemorySystem
159 )
160 {
161 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
162 NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS params = {0};
163
164 NV_ASSERT_OK_OR_RETURN(
165 pRmApi->Control(pRmApi,
166 pGpu->hInternalClient,
167 pGpu->hInternalSubdevice,
168 NV2080_CTRL_CMD_INTERNAL_KMEMSYS_GET_MIG_MEMORY_CONFIG,
169 ¶ms,
170 sizeof(params)));
171
172 pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA = params.memBoundaryCfgA;
173 pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgB = params.memBoundaryCfgB;
174 pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgC = params.memBoundaryCfgC;
175
176 return NV_OK;
177 }
178
179 /*!
180 * @brief Read MIG Memory partition table
181 */
182 NV_STATUS
kmemsysInitMIGMemoryPartitionTable_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)183 kmemsysInitMIGMemoryPartitionTable_GA100
184 (
185 OBJGPU *pGpu,
186 KernelMemorySystem *pKernelMemorySystem
187 )
188 {
189 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
190
191 if (!pKernelMemorySystem->bDisablePlcForCertainOffsetsBug3046774)
192 return NV_OK;
193
194 NV_ASSERT_OK_OR_RETURN(
195 pRmApi->Control(pRmApi,
196 pGpu->hInternalClient,
197 pGpu->hInternalSubdevice,
198 NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE,
199 &pKernelMemorySystem->migMemoryPartitionTable,
200 sizeof(pKernelMemorySystem->migMemoryPartitionTable)));
201
202 return NV_OK;
203 }
204
205 /*!
206 * @brief Function to map swizzId to mem range given total range in Fb
207 */
208 static NV_STATUS
_kmemsysSwizzIdToFbMemRange_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU64 vmmuSegmentSize,NV_RANGE totalRange,NV_RANGE * pAddrRange)209 _kmemsysSwizzIdToFbMemRange_GA100
210 (
211 OBJGPU *pGpu,
212 KernelMemorySystem *pKernelMemorySystem,
213 NvU32 swizzId,
214 NvU64 vmmuSegmentSize,
215 NV_RANGE totalRange,
216 NV_RANGE *pAddrRange
217 )
218 {
219 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
220 kmemsysSwizzIdToMIGMemRange(pGpu, pKernelMemorySystem, swizzId, totalRange, pAddrRange));
221
222 //
223 // If pAddrRange->lo is vmmuSegment aligned, then alignedUp
224 // by one segment else simply align it. We need to make sure we have
225 // atleast 1 VMMU segment delta between consecutive segments
226 //
227 if (pAddrRange->lo != 0)
228 {
229 pAddrRange->lo = NV_IS_ALIGNED64(pAddrRange->lo, vmmuSegmentSize) ?
230 pAddrRange->lo + vmmuSegmentSize :
231 NV_ALIGN_UP64(pAddrRange->lo, vmmuSegmentSize);
232 }
233
234 return NV_OK;
235 }
236
237 /*!
238 * @brief Function to map swizzId to VMMU Segments
239 */
240 NV_STATUS
kmemsysSwizzIdToVmmuSegmentsRange_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU32 vmmuSegmentSize,NvU32 totalVmmuSegments)241 kmemsysSwizzIdToVmmuSegmentsRange_GA100
242 (
243 OBJGPU *pGpu,
244 KernelMemorySystem *pKernelMemorySystem,
245 NvU32 swizzId,
246 NvU32 vmmuSegmentSize,
247 NvU32 totalVmmuSegments
248 )
249 {
250 //
251 // This parameter represents the number of boundaries drawn when a
252 // specific GPU instance type is created
253 //
254 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
255 NvU32 numBoundaries = 0;
256 NvU32 partitionDivFactor = 1;
257 NV_RANGE addrRange = NV_RANGE_EMPTY;
258 NV_RANGE partitionableMemoryRange = memmgrGetMIGPartitionableMemoryRange(pGpu, pMemoryManager);
259 NvU64 startingVmmuSegment;
260 NvU64 memSizeInVmmuSegment;
261
262 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
263 _kmemsysSwizzIdToFbMemRange_GA100(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, partitionableMemoryRange, &addrRange));
264
265 switch (swizzId)
266 {
267 case 0:
268 {
269 numBoundaries = 0;
270 partitionDivFactor = 1;
271 break;
272 }
273 case 1:
274 case 2:
275 {
276 numBoundaries = 1;
277 partitionDivFactor = 2;
278 break;
279 }
280 case 3:
281 case 4:
282 case 5:
283 case 6:
284 {
285 numBoundaries = 3;
286 partitionDivFactor = 4;
287 break;
288 }
289 case 7:
290 case 8:
291 case 9:
292 case 10:
293 case 11:
294 case 12:
295 case 13:
296 case 14:
297 {
298 numBoundaries = 7;
299 partitionDivFactor = 8;
300 break;
301 }
302 }
303
304 startingVmmuSegment = addrRange.lo / vmmuSegmentSize;
305 memSizeInVmmuSegment = (totalVmmuSegments - numBoundaries) / partitionDivFactor;
306
307 NV_ASSERT_OK_OR_RETURN(
308 kmemsysInitMIGGPUInstanceMemConfigForSwizzId(pGpu, pKernelMemorySystem, swizzId, startingVmmuSegment, memSizeInVmmuSegment));
309
310 return NV_OK;
311 }
312
313 NvBool
kmemsysIsPagePLCable_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU64 physAddr,NvU64 pageSize)314 kmemsysIsPagePLCable_GA100
315 (
316 OBJGPU *pGpu,
317 KernelMemorySystem *pKernelMemorySystem,
318 NvU64 physAddr,
319 NvU64 pageSize
320 )
321 {
322 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = kmemsysGetStaticConfig(pGpu, pKernelMemorySystem);
323 NvU64 topAddr = ((pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgB + pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA) *
324 pMemorySystemConfig->ltcCount * pMemorySystemConfig->ltsPerLtcCount) >> 4;
325 NvU64 bottomAddr = (pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA * pMemorySystemConfig->ltcCount *
326 pMemorySystemConfig->ltsPerLtcCount) >> 4;
327 NvU64 secureTopAddr = pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgC;
328 NvU64 revBottomAddr = pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA << 5;
329 NvBool bPageSize2M = (pageSize == (2 << 20));
330 NvU64 partitionedMemorySize = ((topAddr - bottomAddr) << 16);
331 NvU32 blackBlockOffset = 2048 - (NvU32)((revBottomAddr >> 1) % 2048);
332 NvU32 swizzId = 0;
333
334 if (pMemorySystemConfig->ltsPerLtcCount * pMemorySystemConfig->ltcCount != 80)
335 return NV_TRUE;
336
337 if (!((pMemorySystemConfig->ltsPerLtcCount != 4) || ((pMemorySystemConfig->ltcCount % 4) != 0) || (pMemorySystemConfig->ltcCount < 4)) &&
338 !(topAddr >= secureTopAddr) &&
339 !(physAddr >= (secureTopAddr << 16)) &&
340 !((physAddr < (bottomAddr << 16)) || (physAddr >= (topAddr << 16))))
341 {
342 NvU32 partition_id = (NvU32)((physAddr - (bottomAddr << 16)) / (partitionedMemorySize / 8));
343 NV_ASSERT_OR_RETURN(partition_id < 8, NV_TRUE);
344 swizzId = pKernelMemorySystem->migMemoryPartitionTable.data[partition_id];
345 }
346
347
348 switch (swizzId)
349 {
350 case 0:
351 if (!bPageSize2M)
352 {
353 return ((physAddr % (160 * 256 * 1024)) >= (3 * 256 * 1024));
354 }
355 else
356 {
357 NvBool noPLC = NV_FALSE;
358
359 for (NvU64 addr = physAddr; (addr < physAddr + (2 * 1024 * 1024)); addr += (256 * 1024))
360 {
361 noPLC |= ((addr % (160 * 256 * 1024)) < (3 * 256 * 1024));
362 }
363
364 return !noPLC;
365 }
366 case 1:
367 case 2:
368 {
369 NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 2) * (swizzId - 1)) +
370 160 * 64 * 1024 - (blackBlockOffset * 10 / 128) * (128 * 1024);
371
372 if (!bPageSize2M)
373 {
374 return ((rebasePA % (160 * 128 * 1024)) >= (4 * 128 * 1024));
375 }
376 else
377 {
378 NvBool noPLC = NV_FALSE;
379
380 for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (128 * 1024))
381 {
382 noPLC |= ((addr % (160 * 128 * 1024)) < (4 * 128 * 1024));
383 }
384
385 return !noPLC;
386 }
387 }
388 case 3:
389 case 4:
390 case 5:
391 case 6:
392 {
393 NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 4) * (swizzId - 3)) +
394 160 * 64 * 1024 - (blackBlockOffset * 10 / 128) * (64 * 1024);
395
396 if (!bPageSize2M)
397 {
398 return ((rebasePA % (160 * 64 * 1024)) >= (4 * 64 * 1024));
399 }
400 else
401 {
402 NvBool noPLC = NV_FALSE;
403
404 for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (64 * 1024))
405 {
406 noPLC |= ((addr % (160 * 64 * 1024)) < (4 * 64 * 1024));
407 }
408
409 return !noPLC;
410 }
411 }
412 case 7:
413 case 8:
414 case 9:
415 case 10:
416 case 11:
417 case 12:
418 case 13:
419 case 14:
420 {
421 NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 8) * (swizzId - 7)) +
422 80 * 64 * 1024 - (blackBlockOffset * 10 / 256) * (64 * 1024);
423
424 if (!bPageSize2M)
425 {
426 return ((rebasePA % (80 * 64 * 1024)) >= (3 * 64 * 1024));
427 }
428 else
429 {
430 NvBool noPLC = NV_FALSE;
431
432 for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (64 * 1024))
433 {
434 noPLC |= ((addr % (80 * 64 * 1024)) < (3 * 64 * 1024));
435 }
436
437 return !noPLC;
438 }
439 }
440 default:
441 return NV_TRUE;
442 }
443 }
444
445 NvU16
kmemsysGetMaximumBlacklistPages_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)446 kmemsysGetMaximumBlacklistPages_GA100
447 (
448 OBJGPU *pGpu,
449 KernelMemorySystem *pKernelMemorySystem
450 )
451 {
452 return NV2080_CTRL_FB_DYNAMIC_BLACKLIST_MAX_PAGES;
453 }
454
455 NvU32
kmemsysGetMaxFbpas_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)456 kmemsysGetMaxFbpas_GA100
457 (
458 OBJGPU *pGpu,
459 KernelMemorySystem *pKernelMemorySystem
460 )
461 {
462 return NV_SCAL_LITTER_NUM_FBPAS;
463 }
464