1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2014-2015 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /* ------------------------ Includes --------------------------------------- */ 25 #if defined(SRT_BUILD) 26 #include "shrdebug.h" 27 #endif 28 #include "mmu_walk_private.h" 29 30 static NV_STATUS _mmuWalkPostFillPTETasks 31 ( 32 const MMU_WALK *pWalk, 33 MMU_WALK_LEVEL *pLevel, 34 const NvU32 entryIndexLo, 35 const NvU32 entryIndexHi, 36 const MMU_WALK_FILL_STATE fillState, 37 const NvU64 vaLo 38 ); 39 40 /* ------------------------ Public Functions ------------------------------ */ 41 /** 42 * @brief Fill a VA range to a constant state for levels below the root. 43 * 44 * @details This function is of MmuWalkOp function type. Used by 45 * mmuWalkUnmap and mmuWalkSparsify, which fills INVALID and SPARSE 46 * states to the target page levels respectively. 47 * With NV4K introduced with VOLTA ATS, cross PT inconsistency 48 * during unmapping and sparsifying is handled here. 49 * 50 * @todo Recover from failure. It is difficult to do because rollbacks 51 * are costly and complex. Do we really want recovery or asserts? 52 * If the later one, we can replace those recovery codes with 53 * asserts. 54 * 55 * @copydoc MmuWalkCBOpFunc 56 * 57 * @return NV_OK on success 58 */ 59 NV_STATUS mmuWalkFill 60 ( 61 const MMU_WALK *pWalk, 62 const MMU_WALK_OP_PARAMS *pOpParams, 63 MMU_WALK_LEVEL *pLevel, 64 MMU_WALK_LEVEL_INST *pLevelInst, 65 NvU64 vaLo, 66 NvU64 vaHi 67 ) 68 { 69 const MMU_FILL_TARGET *pTarget = (const MMU_FILL_TARGET *) pOpParams->pOpCtx; 70 71 if (NULL == pLevelInst) 72 return NV_OK; 73 74 // If this level is a Page Directory, we keep walking down the tree. 75 if (0 != pLevel->pFmt->numSubLevels) 76 { 77 if (pWalk->flags.bUseIterative) 78 { 79 return NV_ERR_MORE_PROCESSING_REQUIRED; 80 } 81 else 82 { 83 // Process all the page level entries.falling within [vaLo, vaHi] 84 NV_ASSERT_OK_OR_RETURN( 85 mmuWalkProcessPdes(pWalk, 86 pOpParams, 87 pLevel, 88 pLevelInst, 89 vaLo, 90 vaHi)); 91 } 92 } 93 // We have reached a page table 94 else 95 { 96 const NvU32 entryIndexLo = mmuFmtVirtAddrToEntryIndex(pLevel->pFmt, vaLo); 97 const NvU32 entryIndexHi = mmuFmtVirtAddrToEntryIndex(pLevel->pFmt, vaHi); 98 NvU32 progress = 0; 99 NV_STATUS status = NV_OK; 100 // Calculate number of entries in the level 101 NvU64 numEntries = mmuFmtLevelEntryCount(pLevel->pFmt); 102 NvU32 entryIndex; 103 104 NV_ASSERT_OR_RETURN(pLevel->pFmt->bPageTable, NV_ERR_INVALID_ARGUMENT); 105 106 // Make sure all PTEs are contained within one page table 107 NV_ASSERT_OR_RETURN((entryIndexLo / numEntries) == 108 (entryIndexHi / numEntries), 109 NV_ERR_INVALID_ARGUMENT); 110 111 if (pWalk->pStagingBuffer != NULL && pWalk->bUseStagingBuffer) 112 { 113 // 114 // Clear out the PTEs modulo number of entries in table 115 // We do a modulo of number of entries in the table so that 116 // we do not exceed the allocated sysmem page. 117 // 118 pWalk->pCb->FillEntries(pWalk->pUserCtx, 119 pLevel->pFmt, 120 pWalk->pStagingBuffer, 121 entryIndexLo % numEntries, 122 entryIndexHi % numEntries, 123 pTarget->fillState, 124 &progress); 125 126 // Copy from staging buffer to final location 127 pWalk->pCb->WriteBuffer(pWalk->pUserCtx, 128 pWalk->pStagingBuffer, 129 pLevelInst->pMemDesc, 130 entryIndexLo, 131 entryIndexHi, 132 numEntries, 133 pLevel->pFmt->entrySize); 134 } 135 else 136 { 137 // Clear out the PTEs 138 pWalk->pCb->FillEntries(pWalk->pUserCtx, 139 pLevel->pFmt, 140 pLevelInst->pMemDesc, 141 entryIndexLo, 142 entryIndexHi, 143 pTarget->fillState, 144 &progress); 145 } 146 147 NV_ASSERT_OR_RETURN(progress == entryIndexHi - entryIndexLo + 1, NV_ERR_INVALID_STATE); 148 149 // Update the state tracker 150 for (entryIndex = entryIndexLo; entryIndex <= entryIndexHi; entryIndex++) 151 { 152 mmuWalkSetEntryState(pLevelInst, entryIndex, pTarget->entryState); 153 } 154 155 // Post fill tasks 156 status = _mmuWalkPostFillPTETasks(pWalk, pLevel, entryIndexLo, 157 entryIndexHi, pTarget->fillState, vaLo); 158 NV_ASSERT_OR_RETURN(status == NV_OK, NV_ERR_INVALID_STATE); 159 } 160 161 return NV_OK; 162 } 163 164 /*! 165 * TODO 166 */ 167 NV_STATUS 168 mmuWalkFillSelectSubLevel 169 ( 170 const void *pOpCtx, 171 const MMU_WALK_LEVEL *pLevel, 172 NvU32 *pSubLevel, 173 NvU64 vaLo, 174 NvU64 vaHi 175 ) 176 { 177 NvBool bFound = NV_FALSE; 178 NvU32 subLevel; 179 const MMU_FMT_LEVEL *pSubLevelFmt = NULL; 180 181 // If we've only one sublevel, it's at index 0. 182 if (pLevel->pFmt->numSubLevels == 1) 183 { 184 *pSubLevel = 0; 185 return NV_OK; 186 } 187 188 for (subLevel = 0; subLevel < pLevel->pFmt->numSubLevels; subLevel++) 189 { 190 pSubLevelFmt = pLevel->pFmt->subLevels + subLevel; 191 192 if ((( vaLo & mmuFmtEntryVirtAddrMask(pSubLevelFmt)) == 0) && 193 (((vaHi+1) & mmuFmtEntryVirtAddrMask(pSubLevelFmt)) == 0)) 194 { 195 MMU_WALK_LEVEL_INST *pSubLevelInst = NULL; 196 if (!bFound) 197 { 198 bFound = NV_TRUE; 199 *pSubLevel = subLevel; 200 } 201 if (NV_OK == btreeSearch(vaLo, (NODE**)&pSubLevelInst, 202 (NODE*)pLevel->subLevels[subLevel].pInstances)) 203 { 204 *pSubLevel = subLevel; 205 return NV_OK; 206 } 207 } 208 } 209 210 // Error if virt addresses are not aligned to any page size. 211 return bFound ? NV_OK : NV_ERR_INVALID_STATE; 212 } 213 214 /** 215 * @brief Determines if entries indexLo to indexHi (inclusive) are 216 * all invalid. 217 * 218 * @param pLevelInst The level instance 219 * @param[in] indexLo The index lower 220 * @param[in] indexHi The index higher 221 * 222 * @return True if no level instance or all entries are invalid, 223 * False otherwise. 224 */ 225 static NvBool 226 _isRangeAllInvalid 227 ( 228 MMU_WALK_LEVEL_INST *pLevelInst, 229 const NvU32 indexLo, 230 const NvU32 indexHi 231 ) 232 { 233 NvU32 i; 234 MMU_ENTRY_STATE entryState; 235 236 if (pLevelInst == NULL) 237 return NV_TRUE; 238 239 for (i = indexLo; i <= indexHi; i++) 240 { 241 entryState = mmuWalkGetEntryState(pLevelInst, i); 242 if (MMU_ENTRY_STATE_INVALID != entryState) 243 return NV_FALSE; 244 } 245 return NV_TRUE; 246 } 247 248 /** 249 * @brief Post PTE filling tasks to handle cross PTs inconsistency 250 * 251 * @details Helper function inside mmuWalkFill PT level to update 64K PTEs 252 * after mmuWalkFill operation complete. It gathers mmuWalkFill 253 * target entry index range and fillState as input and update 254 * 64K PTEs accordingly. The function doesn't handle extra page 255 * table allocations and deallocations. It relies on 256 * _mmuWalkPdeAcquire and _mmuWalkPdeRelease to prepare and 257 * cleanup page levels accordingly. 258 * 259 * @todo Recovery on failure. Same discussion as in mmuWalkFill. 260 * 261 * @param[in] pWalk The walk 262 * @param pLevel The level, used to get fmt and btree root 263 * @param[in] entryIndexLo The lower entry index (inclusive) 264 * @param[in] entryIndexHi The entry higher index (inclusive) 265 * @param[in] fillState The fill state 266 * @param[in] virtAddr The lower VA, to get the key for btrees 267 * 268 * @return NV_OK on success, NV_ERR_INVALID_ARGUMENT otherwise 269 */ 270 static NV_STATUS _mmuWalkPostFillPTETasks 271 ( 272 const MMU_WALK *pWalk, 273 MMU_WALK_LEVEL *pLevel, 274 const NvU32 entryIndexLo, 275 const NvU32 entryIndexHi, 276 const MMU_WALK_FILL_STATE fillState, 277 const NvU64 virtAddr 278 ) 279 { 280 const MMU_FMT_LEVEL *pFmtLevel = pLevel->pFmt; 281 282 // 283 // NV4K is only necessary for ATS 284 // Only update 64K PTEs on invalidation, not on sparsifying 285 // 286 if (pWalk->flags.bAtsEnabled && fillState == MMU_WALK_FILL_INVALID) 287 { 288 const NvU64 pageSize = mmuFmtLevelPageSize(pFmtLevel); 289 290 NvU32 progress = 0; 291 NvU32 entryIndex = 0; 292 NvU32 indexLo_4K, indexHi_4K, indexLo_64K, indexHi_64K; 293 294 MMU_WALK_LEVEL *pParent = pLevel->pParent; 295 MMU_WALK_LEVEL *pLevel64K = pParent->subLevels; 296 MMU_WALK_LEVEL *pLevel4K = pParent->subLevels + 1; 297 NvU64 vaLevelBase = mmuFmtLevelVirtAddrLo(pLevel64K->pFmt, virtAddr); 298 MMU_WALK_LEVEL_INST *pLevel64KInst = NULL; 299 MMU_WALK_LEVEL_INST *pLevel4KInst = NULL; 300 301 // search for the instances 302 btreeSearch(vaLevelBase, (NODE**)&pLevel64KInst, 303 (NODE*)pLevel64K->pInstances); 304 btreeSearch(vaLevelBase, (NODE**)&pLevel4KInst, 305 (NODE*)pLevel4K->pInstances); 306 307 // 308 // if 4K page table was modified in mmuWalkFill 309 // check the range and update 64K PTEs accordingly 310 // 311 if (pageSize == 0x1000) 312 { 313 // get involved 64K PTEs and 4K PTEs 314 mmuFmtCalcAlignedEntryIndices(pLevel4K->pFmt, entryIndexLo, 315 entryIndexHi, pLevel64K->pFmt, &indexLo_64K, &indexHi_64K); 316 mmuFmtCalcAlignedEntryIndices(pLevel64K->pFmt, indexLo_64K, 317 indexHi_64K, pLevel4K->pFmt, &indexLo_4K, &indexHi_4K); 318 319 // if only one 64K PTE involved, check a single 16 4K PTE group 320 if (indexLo_64K == indexHi_64K) 321 { 322 if (!_isRangeAllInvalid(pLevel4KInst, indexLo_4K, indexHi_4K)) 323 { 324 indexLo_64K++; 325 } 326 } 327 // otherwise check the head and tail groups 328 else 329 { 330 if (indexLo_4K < entryIndexLo && 331 !_isRangeAllInvalid(pLevel4KInst, indexLo_4K, 332 entryIndexLo - 1)) 333 { 334 indexLo_64K++; 335 } 336 if (indexHi_4K > entryIndexHi && 337 !_isRangeAllInvalid(pLevel4KInst, entryIndexHi + 1, 338 indexHi_4K)) 339 { 340 indexHi_64K--; 341 } 342 } 343 344 // update 64K PT given the indexes calculated above 345 if (indexLo_64K <= indexHi_64K) 346 { 347 pWalk->pCb->FillEntries(pWalk->pUserCtx, pLevel64K->pFmt, 348 pLevel64KInst->pMemDesc, indexLo_64K, indexHi_64K, 349 MMU_WALK_FILL_NV4K, &progress); 350 NV_ASSERT_OR_RETURN(progress == indexHi_64K - indexLo_64K + 1, 351 NV_ERR_INVALID_STATE); 352 // update entry states 353 for (entryIndex = indexLo_64K; entryIndex <= indexHi_64K; 354 entryIndex++) 355 { 356 mmuWalkSetEntryState(pLevel64KInst, entryIndex, 357 MMU_ENTRY_STATE_NV4K); 358 } 359 } 360 } 361 // 362 // if 64K page table is invalidated in mmuWalkFill 363 // correct the state as NV4K 364 // @todo move this portion to mmuWalkFill 365 // 366 else if (pageSize == 0x10000) 367 { 368 mmuFmtCalcAlignedEntryIndices(pLevel64K->pFmt, entryIndexLo, 369 entryIndexHi, pLevel4K->pFmt, &indexLo_4K, &indexHi_4K); 370 371 // the 4K PTE should have already been invalid 372 NV_ASSERT_OR_RETURN(_isRangeAllInvalid(pLevel4KInst, indexLo_4K, 373 indexHi_4K), NV_ERR_INVALID_STATE); 374 375 // Set 64K PTEs NV4K 376 pWalk->pCb->FillEntries(pWalk->pUserCtx, pLevel64K->pFmt, 377 pLevel64KInst->pMemDesc, entryIndexLo, entryIndexHi, 378 MMU_WALK_FILL_NV4K, &progress); 379 NV_ASSERT_OR_RETURN(progress == entryIndexHi - entryIndexLo + 1, 380 NV_ERR_INVALID_STATE); 381 382 for (entryIndex = entryIndexLo; entryIndex <= entryIndexHi; 383 entryIndex++) 384 { 385 mmuWalkSetEntryState(pLevel64KInst, entryIndex, 386 MMU_ENTRY_STATE_NV4K); 387 } 388 } 389 // NV4K only works with 64K PT + 4K PT comibination 390 else 391 { 392 NV_ASSERT_OR_RETURN(0, NV_ERR_INVALID_ARGUMENT); 393 } 394 } 395 return NV_OK; 396 } 397