1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "core/core.h" 25 #include "gpu/gpu.h" 26 #include "os/os.h" 27 #include "kernel/gpu/mem_sys/kern_mem_sys.h" 28 #include "kernel/gpu/mem_mgr/mem_mgr.h" 29 #include "kernel/gpu/subdevice/subdevice.h" 30 #include "gpu/mem_mgr/mem_desc.h" 31 32 #include "published/ampere/ga100/dev_fb.h" 33 34 /*! 35 * @brief Write the sysmemFlushBuffer val into the NV_PFB_NISO_FLUSH_SYSMEM_ADDR register 36 * 37 * @param[in] pGpu OBJGPU pointer 38 * @param[in[ pKernelMemorySystem KernelMemorySystem pointer 39 * 40 * @returns void 41 */ 42 void 43 kmemsysProgramSysmemFlushBuffer_GA100 44 ( 45 OBJGPU *pGpu, 46 KernelMemorySystem *pKernelMemorySystem 47 ) 48 { 49 NvU64 alignedSysmemFlushBufferAddr = 0x0; 50 NvU32 alignedSysmemFlushBufferAddrHi = 0x0; 51 52 NV_ASSERT(pKernelMemorySystem->sysmemFlushBuffer != 0); 53 54 alignedSysmemFlushBufferAddr = pKernelMemorySystem->sysmemFlushBuffer >> kmemsysGetFlushSysmemBufferAddrShift_HAL(pGpu, pKernelMemorySystem); 55 alignedSysmemFlushBufferAddrHi = DRF_VAL(_PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40, 56 NvU64_HI32(alignedSysmemFlushBufferAddr)); 57 58 NV_ASSERT((alignedSysmemFlushBufferAddrHi & (~NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI_MASK)) == 0); 59 60 alignedSysmemFlushBufferAddrHi &= NV_PFB_NISO_FLUSH_SYSMEM_ADDR_HI_MASK; 61 62 GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40, 63 alignedSysmemFlushBufferAddrHi); 64 GPU_FLD_WR_DRF_NUM(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR, _ADR_39_08, 65 NvU64_LO32(alignedSysmemFlushBufferAddr)); 66 } 67 68 /* 69 * @brief Initialize the sysmem flush buffer 70 * 71 * Setting up the sysmem flush buffer needs to be done very early in some cases 72 * as it's required for the GPU to perform a system flush. One such case is 73 * resetting GPU FALCONs and in particular resetting the PMU as part of VBIOS 74 * init. 75 * 76 * @returns NV_OK if all is okay. Otherwise an error-specific value. 77 */ 78 NV_STATUS 79 kmemsysInitFlushSysmemBuffer_GA100 80 ( 81 OBJGPU *pGpu, 82 KernelMemorySystem *pKernelMemorySystem 83 ) 84 { 85 NV_STATUS status; 86 87 // 88 // In case of suspend/resume, the buffer might be already allocated, but 89 // the HW still needs to be programmed below. 90 // 91 if (pKernelMemorySystem->pSysmemFlushBufferMemDesc == NULL) 92 { 93 NvU64 flags = MEMDESC_FLAGS_NONE; 94 95 flags |= MEMDESC_FLAGS_ALLOC_IN_UNPROTECTED_MEMORY; 96 // 97 // Sysmem flush buffer 98 // The sysmembar flush does a zero byte read of sysmem if there was a 99 // sysmem write since the last flush. The actual memory does have 100 // to be valid and allocated at all times because an actual read may 101 // be issued. 102 // 103 status = memdescCreate(&pKernelMemorySystem->pSysmemFlushBufferMemDesc, 104 pGpu, RM_PAGE_SIZE, 105 1 << kmemsysGetFlushSysmemBufferAddrShift_HAL(pGpu, pKernelMemorySystem), 106 NV_TRUE, 107 ADDR_SYSMEM, 108 NV_MEMORY_UNCACHED, 109 flags); 110 if (status != NV_OK) 111 return status; 112 113 memdescTagAlloc(status, NV_FB_ALLOC_RM_INTERNAL_OWNER_UNNAMED_TAG_139, 114 pKernelMemorySystem->pSysmemFlushBufferMemDesc); 115 116 if (status != NV_OK) 117 { 118 NV_PRINTF(LEVEL_ERROR, 119 "Could not allocate sysmem flush buffer: %x\n", status); 120 DBG_BREAKPOINT(); 121 return status; 122 } 123 124 pKernelMemorySystem->sysmemFlushBuffer = memdescGetPhysAddr(pKernelMemorySystem->pSysmemFlushBufferMemDesc, AT_GPU, 0); 125 } 126 127 kmemsysProgramSysmemFlushBuffer_HAL(pGpu, pKernelMemorySystem); 128 return NV_OK; 129 } 130 131 /*! 132 * @brief Validate the sysmemFlushBuffer val and assert 133 * 134 * @param[in] pGpu OBJGPU pointer 135 * @param[in[ pKernelMemorySystem KernelMemorySystem pointer 136 * 137 * @returns void 138 */ 139 void 140 kmemsysAssertSysmemFlushBufferValid_GA100 141 ( 142 OBJGPU *pGpu, 143 KernelMemorySystem *pKernelMemorySystem 144 ) 145 { 146 NV_ASSERT((GPU_REG_RD_DRF(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR, _ADR_39_08) != 0) 147 || (GPU_REG_RD_DRF(pGpu, _PFB, _NISO_FLUSH_SYSMEM_ADDR_HI, _ADR_63_40) != 0)); 148 } 149 150 /*! 151 * @brief Read MIG Memory CFG register 152 */ 153 NV_STATUS 154 kmemsysReadMIGMemoryCfg_GA100 155 ( 156 OBJGPU *pGpu, 157 KernelMemorySystem *pKernelMemorySystem 158 ) 159 { 160 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 161 NV2080_CTRL_INTERNAL_MEMSYS_GET_MIG_MEMORY_CONFIG_PARAMS params = {0}; 162 163 NV_ASSERT_OK_OR_RETURN( 164 pRmApi->Control(pRmApi, 165 pGpu->hInternalClient, 166 pGpu->hInternalSubdevice, 167 NV2080_CTRL_CMD_INTERNAL_KMEMSYS_GET_MIG_MEMORY_CONFIG, 168 ¶ms, 169 sizeof(params))); 170 171 pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA = params.memBoundaryCfgA; 172 pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgB = params.memBoundaryCfgB; 173 pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgC = params.memBoundaryCfgC; 174 175 return NV_OK; 176 } 177 178 /*! 179 * @brief Read MIG Memory partition table 180 */ 181 NV_STATUS 182 kmemsysInitMIGMemoryPartitionTable_GA100 183 ( 184 OBJGPU *pGpu, 185 KernelMemorySystem *pKernelMemorySystem 186 ) 187 { 188 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 189 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = kmemsysGetStaticConfig(pGpu, pKernelMemorySystem); 190 191 if (!pMemorySystemConfig->bDisablePlcForCertainOffsetsBug3046774) 192 return NV_OK; 193 194 NV_ASSERT_OK_OR_RETURN( 195 pRmApi->Control(pRmApi, 196 pGpu->hInternalClient, 197 pGpu->hInternalSubdevice, 198 NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_MIG_MEMORY_PARTITION_TABLE, 199 &pKernelMemorySystem->migMemoryPartitionTable, 200 sizeof(pKernelMemorySystem->migMemoryPartitionTable))); 201 202 return NV_OK; 203 } 204 205 /*! 206 * @brief Function to map swizzId to mem range given total range in Fb 207 */ 208 static NV_STATUS 209 _kmemsysSwizzIdToFbMemRange_GA100 210 ( 211 OBJGPU *pGpu, 212 KernelMemorySystem *pKernelMemorySystem, 213 NvU32 swizzId, 214 NvU64 vmmuSegmentSize, 215 NV_RANGE totalRange, 216 NV_RANGE *pAddrRange 217 ) 218 { 219 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 220 kmemsysSwizzIdToMIGMemRange(pGpu, pKernelMemorySystem, swizzId, totalRange, pAddrRange)); 221 222 // 223 // If pAddrRange->lo is vmmuSegment aligned, then alignedUp 224 // by one segment else simply align it. We need to make sure we have 225 // atleast 1 VMMU segment delta between consecutive segments 226 // 227 if (pAddrRange->lo != 0) 228 { 229 pAddrRange->lo = NV_IS_ALIGNED64(pAddrRange->lo, vmmuSegmentSize) ? 230 pAddrRange->lo + vmmuSegmentSize : 231 NV_ALIGN_UP64(pAddrRange->lo, vmmuSegmentSize); 232 } 233 234 return NV_OK; 235 } 236 237 /*! 238 * @brief Function to map swizzId to VMMU Segments 239 */ 240 NV_STATUS 241 kmemsysSwizzIdToVmmuSegmentsRange_GA100 242 ( 243 OBJGPU *pGpu, 244 KernelMemorySystem *pKernelMemorySystem, 245 NvU32 swizzId, 246 NvU32 vmmuSegmentSize, 247 NvU32 totalVmmuSegments 248 ) 249 { 250 // 251 // This parameter represents the number of boundaries drawn when a 252 // specific GPU instance type is created 253 // 254 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 255 NvU32 numBoundaries = 0; 256 NvU32 partitionDivFactor = 1; 257 NV_RANGE addrRange = NV_RANGE_EMPTY; 258 NV_RANGE partitionableMemoryRange = memmgrGetMIGPartitionableMemoryRange(pGpu, pMemoryManager); 259 NvU64 startingVmmuSegment; 260 NvU64 memSizeInVmmuSegment; 261 262 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 263 _kmemsysSwizzIdToFbMemRange_GA100(pGpu, pKernelMemorySystem, swizzId, vmmuSegmentSize, partitionableMemoryRange, &addrRange)); 264 265 switch (swizzId) 266 { 267 case 0: 268 { 269 numBoundaries = 0; 270 partitionDivFactor = 1; 271 break; 272 } 273 case 1: 274 case 2: 275 { 276 numBoundaries = 1; 277 partitionDivFactor = 2; 278 break; 279 } 280 case 3: 281 case 4: 282 case 5: 283 case 6: 284 { 285 numBoundaries = 3; 286 partitionDivFactor = 4; 287 break; 288 } 289 case 7: 290 case 8: 291 case 9: 292 case 10: 293 case 11: 294 case 12: 295 case 13: 296 case 14: 297 { 298 numBoundaries = 7; 299 partitionDivFactor = 8; 300 break; 301 } 302 } 303 304 startingVmmuSegment = addrRange.lo / vmmuSegmentSize; 305 memSizeInVmmuSegment = (totalVmmuSegments - numBoundaries) / partitionDivFactor; 306 307 NV_ASSERT_OK_OR_RETURN( 308 kmemsysInitMIGGPUInstanceMemConfigForSwizzId(pGpu, pKernelMemorySystem, swizzId, startingVmmuSegment, memSizeInVmmuSegment)); 309 310 return NV_OK; 311 } 312 313 NvBool 314 kmemsysIsPagePLCable_GA100 315 ( 316 OBJGPU *pGpu, 317 KernelMemorySystem *pKernelMemorySystem, 318 NvU64 physAddr, 319 NvU64 pageSize 320 ) 321 { 322 const MEMORY_SYSTEM_STATIC_CONFIG *pMemorySystemConfig = kmemsysGetStaticConfig(pGpu, pKernelMemorySystem); 323 NvU64 topAddr = ((pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgB + pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA) * 324 pMemorySystemConfig->ltcCount * pMemorySystemConfig->ltsPerLtcCount) >> 4; 325 NvU64 bottomAddr = (pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA * pMemorySystemConfig->ltcCount * 326 pMemorySystemConfig->ltsPerLtcCount) >> 4; 327 NvU64 secureTopAddr = pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgC; 328 NvU64 revBottomAddr = pKernelMemorySystem->memBoundaryCfgTable.memBoundaryCfgA << 5; 329 NvBool bPageSize2M = (pageSize == (2 << 20)); 330 NvU64 partitionedMemorySize = ((topAddr - bottomAddr) << 16); 331 NvU32 blackBlockOffset = 2048 - (NvU32)((revBottomAddr >> 1) % 2048); 332 NvU32 swizzId = 0; 333 334 if (pMemorySystemConfig->ltsPerLtcCount * pMemorySystemConfig->ltcCount != 80) 335 return NV_TRUE; 336 337 if (!((pMemorySystemConfig->ltsPerLtcCount != 4) || ((pMemorySystemConfig->ltcCount % 4) != 0) || (pMemorySystemConfig->ltcCount < 4)) && 338 !(topAddr >= secureTopAddr) && 339 !(physAddr >= (secureTopAddr << 16)) && 340 !((physAddr < (bottomAddr << 16)) || (physAddr >= (topAddr << 16)))) 341 { 342 NvU32 partition_id = (NvU32)((physAddr - (bottomAddr << 16)) / (partitionedMemorySize / 8)); 343 NV_ASSERT_OR_RETURN(partition_id < 8, NV_TRUE); 344 swizzId = pKernelMemorySystem->migMemoryPartitionTable.data[partition_id]; 345 } 346 347 348 switch (swizzId) 349 { 350 case 0: 351 if (!bPageSize2M) 352 { 353 return ((physAddr % (160 * 256 * 1024)) >= (3 * 256 * 1024)); 354 } 355 else 356 { 357 NvBool noPLC = NV_FALSE; 358 359 for (NvU64 addr = physAddr; (addr < physAddr + (2 * 1024 * 1024)); addr += (256 * 1024)) 360 { 361 noPLC |= ((addr % (160 * 256 * 1024)) < (3 * 256 * 1024)); 362 } 363 364 return !noPLC; 365 } 366 case 1: 367 case 2: 368 { 369 NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 2) * (swizzId - 1)) + 370 160 * 64 * 1024 - (blackBlockOffset * 10 / 128) * (128 * 1024); 371 372 if (!bPageSize2M) 373 { 374 return ((rebasePA % (160 * 128 * 1024)) >= (4 * 128 * 1024)); 375 } 376 else 377 { 378 NvBool noPLC = NV_FALSE; 379 380 for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (128 * 1024)) 381 { 382 noPLC |= ((addr % (160 * 128 * 1024)) < (4 * 128 * 1024)); 383 } 384 385 return !noPLC; 386 } 387 } 388 case 3: 389 case 4: 390 case 5: 391 case 6: 392 { 393 NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 4) * (swizzId - 3)) + 394 160 * 64 * 1024 - (blackBlockOffset * 10 / 128) * (64 * 1024); 395 396 if (!bPageSize2M) 397 { 398 return ((rebasePA % (160 * 64 * 1024)) >= (4 * 64 * 1024)); 399 } 400 else 401 { 402 NvBool noPLC = NV_FALSE; 403 404 for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (64 * 1024)) 405 { 406 noPLC |= ((addr % (160 * 64 * 1024)) < (4 * 64 * 1024)); 407 } 408 409 return !noPLC; 410 } 411 } 412 case 7: 413 case 8: 414 case 9: 415 case 10: 416 case 11: 417 case 12: 418 case 13: 419 case 14: 420 { 421 NvU64 rebasePA = physAddr - (bottomAddr << 16) - ((partitionedMemorySize / 8) * (swizzId - 7)) + 422 80 * 64 * 1024 - (blackBlockOffset * 10 / 256) * (64 * 1024); 423 424 if (!bPageSize2M) 425 { 426 return ((rebasePA % (80 * 64 * 1024)) >= (3 * 64 * 1024)); 427 } 428 else 429 { 430 NvBool noPLC = NV_FALSE; 431 432 for (NvU64 addr = rebasePA; (addr < rebasePA + (2 * 1024 * 1024)); addr += (64 * 1024)) 433 { 434 noPLC |= ((addr % (80 * 64 * 1024)) < (3 * 64 * 1024)); 435 } 436 437 return !noPLC; 438 } 439 } 440 default: 441 return NV_TRUE; 442 } 443 } 444