1 /*
2  * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3  * SPDX-License-Identifier: MIT
4  *
5  * Permission is hereby granted, free of charge, to any person obtaining a
6  * copy of this software and associated documentation files (the "Software"),
7  * to deal in the Software without restriction, including without limitation
8  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
9  * and/or sell copies of the Software, and to permit persons to whom the
10  * Software is furnished to do so, subject to the following conditions:
11  *
12  * The above copyright notice and this permission notice shall be included in
13  * all copies or substantial portions of the Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
21  * DEALINGS IN THE SOFTWARE.
22  */
23 
24 #include "core/core.h"
25 #include "gpu/gpu.h"
26 #include "os/os.h"
27 #include "kernel/gpu/mem_sys/kern_mem_sys.h"
28 #include "kernel/gpu/mem_mgr/mem_mgr.h"
29 #include "gpu/mem_mgr/mem_desc.h"
30 #include "ctrl/ctrl2080/ctrl2080fb.h"
31 
32 #include "published/ampere/ga100/dev_fb.h"
33 #include "published/ampere/ga100/hwproject.h"
34 
35 /*!
36  * @brief Write the sysmemFlushBuffer val into the NV_PFB_NISO_FLUSH_SYSMEM_ADDR register
37  *
38  * @param[in] pGpu                OBJGPU pointer
39  * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
40  *
41  * @returns void
42  */
43 void
kmemsysProgramSysmemFlushBuffer_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)44 kmemsysProgramSysmemFlushBuffer_GA100
45 (
46     OBJGPU *pGpu,
47     KernelMemorySystem *pKernelMemorySystem
48 )
49 {
50     NvU64 alignedSysmemFlushBufferAddr = 0x0;
51     NvU32 alignedSysmemFlushBufferAddrHi = 0x0;
52 
53     NV_ASSERT(pKernelMemorySystem->sysmemFlushBuffer != 0);
54 
55     alignedSysmemFlushBufferAddr = pKernelMemorySystem->sysmemFlushBuffer >> kmemsysGetFlushSysmemBufferAddrShift_HAL(pGpu, pKernelMemorySystem);
56     alignedSysmemFlushBufferAddrHi = DRF_VAL(_PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40,
57                                             NvU64_HI32(alignedSysmemFlushBufferAddr));
58 
59     NV_ASSERT((alignedSysmemFlushBufferAddrHi & (~NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI_MASK)) == 0);
60 
61     alignedSysmemFlushBufferAddrHi &= NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI_MASK;
62 
63     GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40,
64                     alignedSysmemFlushBufferAddrHi);
65     GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR, _ADR_39_08,
66                     NvU64_LO32(alignedSysmemFlushBufferAddr));
67 }
68 
69 /*
70  * @brief Initialize the sysmem flush buffer
71  *
72  * Setting up the sysmem flush buffer needs to be done very early in some cases
73  * as it's required for the GPU to perform a system flush. One such case is
74  * resetting GPU FALCONs and in particular resetting the PMU as part of VBIOS
75  * init.
76  *
77  * @returns NV_OK if all is okay.  Otherwise an error-specific value.
78  */
79 NV_STATUS
kmemsysInitFlushSysmemBuffer_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)80 kmemsysInitFlushSysmemBuffer_GA100
81 (
82     OBJGPU             *pGpu,
83     KernelMemorySystem *pKernelMemorySystem
84 )
85 {
86     NV_STATUS status;
87 
88     //
89     // In case of suspend/resume, the buffer might be already allocated, but
90     // the HW still needs to be programmed below.
91     //
92     if (pKernelMemorySystem->pSysmemFlushBufferMemDesc == NULL)
93     {
94         NvU64 flags = MEMDESC_FLAGS_NONE;
95 
96             flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY;
97         //
98         // Sysmem flush buffer
99         // The sysmembar flush does a zero byte read of sysmem if there was a
100         // sysmem write since the last flush. The actual memory does have
101         // to be valid and allocated at all times because an actual read may
102         // be issued.
103         //
104         status = memdescCreate(&pKernelMemorySystem->pSysmemFlushBufferMemDesc,
105                                pGpu, RM_PAGE_SIZE,
106                                1 << kmemsysGetFlushSysmemBufferAddrShift_HAL(pGpu, pKernelMemorySystem),
107                                NV_TRUE,
108                                ADDR_SYSMEM,
109                                NV_MEMORY_UNCACHED,
110                                flags);
111         if (status != NV_OK)
112             return status;
113 
114         memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_139,
115                         pKernelMemorySystem->pSysmemFlushBufferMemDesc);
116 
117         if (status != NV_OK)
118         {
119             NV_PRINTF(LEVEL_ERROR,
120                       "Could not allocate sysmem flush buffer: %x\n", status);
121             DBG_BREAKPOINT();
122             return status;
123         }
124 
125         pKernelMemorySystem->sysmemFlushBuffer = memdescGetPhysAddr(pKernelMemorySystem->pSysmemFlushBufferMemDesc, AT_GPU, 0);
126     }
127 
128     kmemsysProgramSysmemFlushBuffer_HAL(pGpu, pKernelMemorySystem);
129     return NV_OK;
130 }
131 
132 /*!
133  * @brief Validate the sysmemFlushBuffer val and assert
134  *
135  * @param[in] pGpu                OBJGPU pointer
136  * @param[in[ pKernelMemorySystem KernelMemorySystem pointer
137  *
138  * @returns void
139  */
140 void
kmemsysAssertSysmemFlushBufferValid_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)141 kmemsysAssertSysmemFlushBufferValid_GA100
142 (
143     OBJGPU *pGpu,
144     KernelMemorySystem *pKernelMemorySystem
145 )
146 {
147     NV_ASSERT((GPU_REG_RD_DRF(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR, _ADR_39_08) != 0)
148                || (GPU_REG_RD_DRF(pGpu, _PFB,  _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40) != 0));
149 }
150 
151 /*!
152  * @brief  Read MIG Memory CFG register
153  */
154 NV_STATUS
kmemsysReadMIGMemoryCfg_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)155 kmemsysReadMIGMemoryCfg_GA100
156 (
157     OBJGPU *pGpu,
158     KernelMemorySystem *pKernelMemorySystem
159 )
160 {
161     RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
162     NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS params = {0};
163 
164     NV_ASSERT_OK_OR_RETURN(
165         pRmApi->Control(pRmApi,
166                         pGpu->hInternalClient,
167                         pGpu->hInternalSubdevice,
168                         NV2080_CTRL_CMD_INTERNAL_KMEMSYS_GET_MIG_MEMORY_CONFIG,
169                         &params,
170                         sizeof(params)));
171 
172     pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA = params.memBoundaryCfgA;
173     pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgB = params.memBoundaryCfgB;
174     pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgC = params.memBoundaryCfgC;
175 
176     return NV_OK;
177 }
178 
179 /*!
180  * @brief  Read MIG Memory partition table
181  */
182 NV_STATUS
kmemsysInitMIGMemoryPartitionTable_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)183 kmemsysInitMIGMemoryPartitionTable_GA100
184 (
185     OBJGPU *pGpu,
186     KernelMemorySystem *pKernelMemorySystem
187 )
188 {
189     RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu);
190 
191     if (!pKernelMemorySystem->bDisablePlcForCertainOffsetsBug3046774)
192         return NV_OK;
193 
194     NV_ASSERT_OK_OR_RETURN(
195         pRmApi->Control(pRmApi,
196                         pGpu->hInternalClient,
197                         pGpu->hInternalSubdevice,
198                         NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE,
199                         &pKernelMemorySystem->migMemoryPartitionTable,
200                         sizeof(pKernelMemorySystem->migMemoryPartitionTable)));
201 
202     return NV_OK;
203 }
204 
205 /*!
206  * @brief   Function to map swizzId to mem range given total range in Fb
207  */
208 static NV_STATUS
_kmemsysSwizzIdToFbMemRange_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU64 vmmuSegmentSize,NV_RANGE totalRange,NV_RANGE * pAddrRange)209 _kmemsysSwizzIdToFbMemRange_GA100
210 (
211     OBJGPU *pGpu,
212     KernelMemorySystem *pKernelMemorySystem,
213     NvU32 swizzId,
214     NvU64 vmmuSegmentSize,
215     NV_RANGE totalRange,
216     NV_RANGE *pAddrRange
217 )
218 {
219     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
220         kmemsysSwizzIdToMIGMemRange(pGpu, pKernelMemorySystem, swizzId, totalRange, pAddrRange));
221 
222     //
223     // If pAddrRange->lo is vmmuSegment aligned, then alignedUp
224     // by one segment else simply align it. We need to make sure we have
225     // atleast 1 VMMU segment delta between consecutive segments
226     //
227     if (pAddrRange->lo != 0)
228     {
229         pAddrRange->lo = NV_IS_ALIGNED64(pAddrRange->lo, vmmuSegmentSize) ?
230                         pAddrRange->lo + vmmuSegmentSize :
231                         NV_ALIGN_UP64(pAddrRange->lo, vmmuSegmentSize);
232     }
233 
234     return NV_OK;
235 }
236 
237 /*!
238  * @brief   Function to map swizzId to VMMU Segments
239  */
240 NV_STATUS
kmemsysSwizzIdToVmmuSegmentsRange_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU32 swizzId,NvU32 vmmuSegmentSize,NvU32 totalVmmuSegments)241 kmemsysSwizzIdToVmmuSegmentsRange_GA100
242 (
243     OBJGPU *pGpu,
244     KernelMemorySystem *pKernelMemorySystem,
245     NvU32 swizzId,
246     NvU32 vmmuSegmentSize,
247     NvU32 totalVmmuSegments
248 )
249 {
250     //
251     // This parameter represents the number of boundaries drawn when a
252     // specific GPU instance type is created
253     //
254     MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu);
255     NvU32          numBoundaries = 0;
256     NvU32          partitionDivFactor = 1;
257     NV_RANGE       addrRange = NV_RANGE_EMPTY;
258     NV_RANGE       partitionableMemoryRange = memmgrGetMIGPartitionableMemoryRange(pGpu, pMemoryManager);
259     NvU64          startingVmmuSegment;
260     NvU64          memSizeInVmmuSegment;
261 
262     NV_CHECK_OK_OR_RETURN(LEVEL_ERROR,
263         _kmemsysSwizzIdToFbMemRange_GA100(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, partitionableMemoryRange, &addrRange));
264 
265     switch (swizzId)
266     {
267         case 0:
268         {
269             numBoundaries = 0;
270             partitionDivFactor = 1;
271             break;
272         }
273         case 1:
274         case 2:
275         {
276             numBoundaries = 1;
277             partitionDivFactor = 2;
278             break;
279         }
280         case 3:
281         case 4:
282         case 5:
283         case 6:
284         {
285             numBoundaries = 3;
286             partitionDivFactor = 4;
287             break;
288         }
289         case 7:
290         case 8:
291         case 9:
292         case 10:
293         case 11:
294         case 12:
295         case 13:
296         case 14:
297         {
298             numBoundaries = 7;
299             partitionDivFactor = 8;
300             break;
301         }
302     }
303 
304     startingVmmuSegment = addrRange.lo / vmmuSegmentSize;
305     memSizeInVmmuSegment = (totalVmmuSegments - numBoundaries) / partitionDivFactor;
306 
307     NV_ASSERT_OK_OR_RETURN(
308         kmemsysInitMIGGPUInstanceMemConfigForSwizzId(pGpu, pKernelMemorySystem, swizzId, startingVmmuSegment, memSizeInVmmuSegment));
309 
310     return NV_OK;
311 }
312 
313 NvBool
kmemsysIsPagePLCable_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem,NvU64 physAddr,NvU64 pageSize)314 kmemsysIsPagePLCable_GA100
315 (
316     OBJGPU             *pGpu,
317     KernelMemorySystem *pKernelMemorySystem,
318     NvU64               physAddr,
319     NvU64               pageSize
320 )
321 {
322     const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = kmemsysGetStaticConfig(pGpu, pKernelMemorySystem);
323     NvU64 topAddr       = ((pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgB + pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA) *
324                           pMemorySystemConfig->ltcCount * pMemorySystemConfig->ltsPerLtcCount) >> 4;
325     NvU64 bottomAddr    = (pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA * pMemorySystemConfig->ltcCount *
326                           pMemorySystemConfig->ltsPerLtcCount) >> 4;
327     NvU64 secureTopAddr = pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgC;
328     NvU64 revBottomAddr = pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA << 5;
329     NvBool bPageSize2M  = (pageSize == (2 << 20));
330     NvU64 partitionedMemorySize = ((topAddr - bottomAddr) << 16);
331     NvU32 blackBlockOffset = 2048 - (NvU32)((revBottomAddr >> 1) % 2048);
332     NvU32 swizzId = 0;
333 
334     if (pMemorySystemConfig->ltsPerLtcCount * pMemorySystemConfig->ltcCount != 80)
335         return NV_TRUE;
336 
337      if (!((pMemorySystemConfig->ltsPerLtcCount != 4) || ((pMemorySystemConfig->ltcCount % 4) != 0) || (pMemorySystemConfig->ltcCount < 4)) &&
338          !(topAddr >= secureTopAddr) &&
339          !(physAddr >= (secureTopAddr << 16)) &&
340          !((physAddr < (bottomAddr << 16)) || (physAddr >= (topAddr << 16))))
341      {
342          NvU32 partition_id = (NvU32)((physAddr - (bottomAddr << 16)) / (partitionedMemorySize / 8));
343          NV_ASSERT_OR_RETURN(partition_id < 8, NV_TRUE);
344          swizzId = pKernelMemorySystem->migMemoryPartitionTable.data[partition_id];
345      }
346 
347 
348     switch (swizzId)
349     {
350     case 0:
351         if (!bPageSize2M)
352         {
353             return ((physAddr % (160 * 256 * 1024)) >= (3 * 256 * 1024));
354         }
355         else
356         {
357             NvBool noPLC = NV_FALSE;
358 
359             for (NvU64 addr = physAddr; (addr < physAddr + (2 * 1024 * 1024)); addr += (256 * 1024))
360             {
361                 noPLC |= ((addr % (160 * 256 * 1024)) < (3 * 256 * 1024));
362             }
363 
364             return !noPLC;
365         }
366     case 1:
367     case 2:
368     {
369         NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 2) * (swizzId - 1)) +
370                          160 * 64 * 1024 - (blackBlockOffset * 10 / 128) * (128 * 1024);
371 
372         if (!bPageSize2M)
373         {
374             return ((rebasePA % (160 * 128 * 1024)) >= (4 * 128 * 1024));
375         }
376         else
377         {
378             NvBool noPLC = NV_FALSE;
379 
380             for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (128 * 1024))
381             {
382                 noPLC |= ((addr % (160 * 128 * 1024)) < (4 * 128 * 1024));
383             }
384 
385             return !noPLC;
386         }
387     }
388     case 3:
389     case 4:
390     case 5:
391     case 6:
392     {
393         NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 4) * (swizzId - 3)) +
394                          160 * 64 * 1024 - (blackBlockOffset * 10 / 128) * (64 * 1024);
395 
396         if (!bPageSize2M)
397         {
398             return ((rebasePA % (160 * 64 * 1024)) >= (4 * 64 * 1024));
399         }
400         else
401         {
402             NvBool noPLC = NV_FALSE;
403 
404             for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (64 * 1024))
405             {
406                 noPLC |= ((addr % (160 * 64 * 1024)) < (4 * 64 * 1024));
407             }
408 
409             return !noPLC;
410         }
411     }
412     case 7:
413     case 8:
414     case 9:
415     case 10:
416     case 11:
417     case 12:
418     case 13:
419     case 14:
420     {
421         NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 8) * (swizzId - 7)) +
422                          80 * 64 * 1024 - (blackBlockOffset * 10 / 256) * (64 * 1024);
423 
424         if (!bPageSize2M)
425         {
426             return ((rebasePA % (80 * 64 * 1024)) >= (3 * 64 * 1024));
427         }
428         else
429         {
430             NvBool noPLC = NV_FALSE;
431 
432             for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (64 * 1024))
433             {
434                 noPLC |= ((addr % (80 * 64 * 1024)) < (3 * 64 * 1024));
435             }
436 
437             return !noPLC;
438         }
439     }
440     default:
441         return NV_TRUE;
442     }
443 }
444 
445 NvU16
kmemsysGetMaximumBlacklistPages_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)446 kmemsysGetMaximumBlacklistPages_GA100
447 (
448     OBJGPU *pGpu,
449     KernelMemorySystem *pKernelMemorySystem
450 )
451 {
452     return NV2080_CTRL_FB_DYNAMIC_BLACKLIST_MAX_PAGES;
453 }
454 
455 NvU32
kmemsysGetMaxFbpas_GA100(OBJGPU * pGpu,KernelMemorySystem * pKernelMemorySystem)456 kmemsysGetMaxFbpas_GA100
457 (
458     OBJGPU             *pGpu,
459     KernelMemorySystem *pKernelMemorySystem
460 )
461 {
462     return NV_SCAL_LITTER_NUM_FBPAS;
463 }
464