1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 1993-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "core/core.h" 25 #include "os/os.h" 26 #include "mem_mgr/virt_mem_mgr.h" 27 #include "mem_mgr/vaspace.h" 28 #include "core/system.h" 29 #include "gpu/mem_mgr/mem_desc.h" 30 #include "gpu_mgr/gpu_group.h" 31 #include "class/cl00f2.h" // IO_VASPACE_A 32 #include "gpu/mem_mgr/vaspace_api.h" 33 #include "rmapi/rs_utils.h" 34 #include "gpu/device/device.h" 35 #include "gpu/subdevice/subdevice.h" 36 #include "gpu/mem_mgr/mem_mgr.h" 37 38 #include "gpu/mem_mgr/virt_mem_allocator.h" 39 40 #include "gpu_mgr/gpu_mgr.h" 41 42 #include "gpu/mmu/kern_gmmu.h" 43 44 /*! 45 * @brief Save client share allocation information for this device 46 * 47 * Save client share allocation information for this device. The 48 * client share is actually allocated as a result of CliGetVASpace() 49 * before the VAShare is actually used. 50 * 51 * @param[in] pDevice 52 * @param[in] hClientShare RM client specified share handle 53 * @param[in] deviceAllocFlags Allocation flags from RM client 54 * 55 * @returns NV_STATUS 56 */ 57 NV_STATUS 58 deviceSetClientShare_IMPL 59 ( 60 Device *pDevice, 61 NvHandle hClientShare, 62 NvU64 vaSize, 63 NvU64 vaStartInternal, 64 NvU64 vaLimitInternal, 65 NvU32 deviceAllocFlags 66 ) 67 { 68 pDevice->pVASpace = NULL; 69 pDevice->hClientShare = hClientShare; 70 pDevice->deviceAllocFlags = deviceAllocFlags; 71 pDevice->deviceInternalAllocFlags = 0; 72 pDevice->vaSize = vaSize; 73 74 if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_RESTRICT_RESERVED_VALIMITS) 75 { 76 pDevice->vaStartInternal = vaStartInternal; 77 pDevice->vaLimitInternal = vaLimitInternal; 78 } 79 80 if ((deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_SIZE) && (vaSize == 0)) 81 { 82 return NV_ERR_INVALID_ARGUMENT; 83 } 84 85 return NV_OK; 86 } 87 88 /*! 89 * @brief Initialize the device VASPACE 90 */ 91 static NV_STATUS 92 deviceInitClientShare 93 ( 94 Device *pDevice, 95 NvHandle hClientShare, 96 NvU64 vaSize, 97 NvU32 deviceAllocFlags, 98 NvU32 deviceAllocInternalFlags 99 ) 100 { 101 Device *pShareDevice; 102 RsClient *pClientShare; 103 OBJVASPACE *pVAS = NULL; 104 OBJSYS *pSys = SYS_GET_INSTANCE(); 105 OBJVMM *pVmm = SYS_GET_VMM(pSys); 106 NV_STATUS status; 107 OBJGPU *pGpu = GPU_RES_GET_GPU(pDevice); 108 NvU32 gpuMask = gpumgrGetGpuMask(pGpu); 109 NvU32 vaspaceClass = 0; 110 111 pDevice->pVASpace = NULL; 112 113 // Set broadcast state for thread 114 GPU_RES_SET_THREAD_BC_STATE(pDevice); 115 116 // 117 // Share "default" behavior is defined by "share w/null", which 118 // attaches to the global address space. 119 // 120 if (hClientShare == NV01_NULL_OBJECT) 121 { 122 OBJGPUGRP *pGpuGrp = gpumgrGetGpuGrpFromGpu(pGpu); 123 status = gpugrpGetGlobalVASpace(pGpuGrp, &pVAS); 124 NV_ASSERT_OR_RETURN(status == NV_OK, status); 125 126 vaspaceIncRefCnt(pVAS); 127 status = NV_OK; 128 } 129 130 // 131 // "Force a new share" behavior is defined by "share w/myself" 132 // 133 else if (hClientShare == RES_GET_CLIENT_HANDLE(pDevice)) 134 { 135 NvU32 flags = 0; 136 NvU64 vaLimit; 137 138 flags |= (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_SHARED_MANAGEMENT) ? 139 VASPACE_FLAGS_SHARED_MANAGEMENT : 0; 140 141 if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_MINIMIZE_PTETABLE_SIZE) 142 { 143 flags |= VASPACE_FLAGS_MINIMIZE_PTETABLE_SIZE; 144 } 145 if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_RETRY_PTE_ALLOC_IN_SYS) 146 { 147 flags |= VASPACE_FLAGS_RETRY_PTE_ALLOC_IN_SYS; 148 } 149 if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_SIZE) 150 { 151 vaLimit = pDevice->vaSize - 1; 152 } 153 else 154 { 155 vaLimit = 0; 156 } 157 158 if ( (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_BIG_PAGE_SIZE_64k) && 159 (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_BIG_PAGE_SIZE_128k) ) 160 { 161 return NV_ERR_INVALID_ARGUMENT; 162 } 163 164 if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_BIG_PAGE_SIZE_64k) 165 { 166 flags |= DRF_DEF(_VASPACE, _FLAGS, _BIG_PAGE_SIZE, _DEFAULT); 167 } 168 else if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_BIG_PAGE_SIZE_128k) 169 { 170 flags |= DRF_DEF(_VASPACE, _FLAGS, _BIG_PAGE_SIZE, _DEFAULT); 171 } 172 else 173 { 174 // will cause it to use the default size 175 flags |= DRF_DEF(_VASPACE, _FLAGS, _BIG_PAGE_SIZE, _DEFAULT); 176 } 177 178 if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_RESTRICT_RESERVED_VALIMITS) 179 { 180 flags |= VASPACE_FLAGS_RESTRICTED_RM_INTERNAL_VALIMITS; 181 NV_ASSERT(pDevice->vaStartInternal); 182 NV_ASSERT(pDevice->vaLimitInternal); 183 } 184 else 185 { 186 NV_ASSERT(!pDevice->vaStartInternal); 187 NV_ASSERT(!pDevice->vaLimitInternal); 188 } 189 190 // 191 // NV_DEVICE_ALLOCATION_FLAGS_VASPACE_IS_MIRRORED will be removed once CUDA phases out 192 // and uses the ctrl call NV0080_CTRL_DMA_ENABLE_PRIVILEGED_RANGE 193 // to set privileged address space 194 // 195 if ((deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_IS_MIRRORED) 196 || (deviceAllocInternalFlags & NV_DEVICE_INTERNAL_ALLOCATION_FLAGS_ENABLE_PRIVILEGED_VASPACE) 197 ) 198 { 199 flags |= VASPACE_FLAGS_SET_MIRRORED; 200 } 201 if (NULL != GPU_GET_KERNEL_GMMU(pGpu)) 202 vaspaceClass = kgmmuGetVaspaceClass_HAL(GPU_GET_KERNEL_GMMU(pGpu)); 203 if (NULL == GPU_GET_KERNEL_GMMU(pGpu) && (pGpu->getProperty(pGpu, PDB_PROP_GPU_TEGRA_SOC_NVDISPLAY) || IsDFPGA(pGpu))) 204 vaspaceClass = IO_VASPACE_A; 205 else if (vaspaceClass == 0) 206 { 207 NV_ASSERT(0); 208 return NV_ERR_OBJECT_NOT_FOUND; 209 } 210 211 flags |= VASPACE_FLAGS_ENABLE_VMM; 212 213 // 214 // Page tables are allocated in guest subheap only inside non SRIOV guests 215 // and on host RM. 216 // 217 if (!gpuIsSplitVasManagementServerClientRmEnabled(pGpu) || 218 !IS_VIRTUAL(pGpu)) 219 { 220 flags |= VASPACE_FLAGS_ALLOW_PAGES_IN_PHYS_MEM_SUBALLOCATOR; 221 } 222 223 // 224 // XXX NV_DEVICE_ALLOCATION_FLAGS_VASPACE_PTABLE_PMA_MANAGED should not 225 // be exposed to clients. It should be the default RM behavior. 226 // 227 // Until it is made the default, certain clients such as OpenGL 228 // might still need PTABLE allocations to go through PMA, so this 229 // flag has been temporary exposed. 230 // 231 // See bug 1880192 232 // 233 // Note: Some clients (including scrubber) depend on page tables not 234 // being PMA managed, so if this is made the default then an opt-out 235 // flag should still be exposed, or some other solution implemented. 236 // See bug 2844476 237 // 238 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 239 if (memmgrIsPmaInitialized(pMemoryManager) && 240 memmgrAreClientPageTablesPmaManaged(pMemoryManager) && 241 (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_PTABLE_PMA_MANAGED)) 242 { 243 flags |= VASPACE_FLAGS_PTETABLE_PMA_MANAGED; 244 } 245 246 // 247 // For RM unlinked SLI: the fixed offset requirement is enforced at the OBJGVASPACE 248 // level during allocations and mappings, so the Device flag must be converted 249 // into the internal VASPACE flag. 250 // 251 if (deviceAllocFlags & NV_DEVICE_ALLOCATION_FLAGS_VASPACE_REQUIRE_FIXED_OFFSET) 252 { 253 flags |= VASPACE_FLAGS_REQUIRE_FIXED_OFFSET; 254 } 255 256 status = vmmCreateVaspace(pVmm, vaspaceClass, 0, gpuMask, 0, 257 vaLimit, pDevice->vaStartInternal, 258 pDevice->vaLimitInternal, NULL, flags, &pVAS); 259 if (NV_OK != status) 260 { 261 NV_ASSERT(0); 262 return status; 263 } 264 } 265 266 // 267 // Try to attach to another clients VA Share. Validate client and pull the 268 // share information off the first device. 269 // 270 else 271 { 272 status = serverGetClientUnderLock(&g_resServ, hClientShare, &pClientShare); 273 if (status != NV_OK) 274 return status; 275 276 // 277 // If the share client doesn't have a device allocated for this GPU, 278 // there's no address space to share. 279 // 280 status = deviceGetByInstance(pClientShare, pDevice->deviceInst, &pShareDevice); 281 if (status != NV_OK) 282 return status; 283 284 // Init target share if needed 285 if (pShareDevice->pVASpace == NULL) 286 { 287 status = deviceInitClientShare(pShareDevice, 288 pShareDevice->hClientShare, 289 pShareDevice->vaSize, 290 pShareDevice->deviceAllocFlags, 291 pShareDevice->deviceInternalAllocFlags); 292 if (status != NV_OK) 293 return status; 294 } 295 296 pVAS = pShareDevice->pVASpace; 297 vaspaceIncRefCnt(pVAS); 298 } 299 300 pDevice->pVASpace = pVAS; 301 return status; 302 } 303 304 305 /*! 306 * @brief Detach this pDevice from the share group 307 */ 308 void 309 deviceRemoveFromClientShare_IMPL 310 ( 311 Device *pDevice 312 ) 313 { 314 OBJSYS *pSys = SYS_GET_INSTANCE(); 315 OBJVMM *pVmm = SYS_GET_VMM(pSys); 316 317 if (pDevice->pVASpace != NULL) 318 { 319 vmmDestroyVaspace(pVmm, pDevice->pVASpace); 320 pDevice->pVASpace = NULL; 321 } 322 } 323 324 NV_STATUS 325 deviceGetDefaultVASpace_IMPL 326 ( 327 Device *pDevice, 328 OBJVASPACE **ppVAS 329 ) 330 { 331 NV_STATUS status = NV_OK; 332 333 // 334 // There are some cases in SLI transitions where we allocate 335 // a device before the hal is initialized. 336 // 337 if (pDevice->pVASpace == NULL) 338 { 339 status = deviceInitClientShare(pDevice, 340 pDevice->hClientShare, 341 pDevice->vaSize, 342 pDevice->deviceAllocFlags, 343 pDevice->deviceInternalAllocFlags); 344 } 345 346 *ppVAS = pDevice->pVASpace; 347 348 return status; 349 } 350 351 /*! 352 * @brief Associate the given address space object as the default VASpace 353 * 354 * This function will associate the given address space object as the 355 * default vaspace of the parent device. 356 * 357 * @param[in] hClient RM client 358 * @param[in] hDevice RM device under this client 359 * @param[in] hVASpace VASpace object handle that is under this device 360 * 361 * @returns NV_STATUS or NV_ERR_INVALID_OBJECT_HANDLE 362 */ 363 NV_STATUS 364 deviceSetDefaultVASpace_IMPL 365 ( 366 Device *pDevice, 367 NvHandle hVASpace 368 ) 369 { 370 NV_STATUS status = NV_OK; 371 VaSpaceApi *pVaSpaceApi = NULL; 372 RsResourceRef *pResourceRef; 373 374 if (hVASpace == NV01_NULL_OBJECT) 375 return NV_ERR_INVALID_ARGUMENT; 376 377 status = serverutilGetResourceRefWithParent(RES_GET_CLIENT_HANDLE(pDevice), 378 RES_GET_HANDLE(pDevice), 379 hVASpace, 380 classId(VaSpaceApi), 381 &pResourceRef); 382 if (status != NV_OK) 383 { 384 NV_PRINTF(LEVEL_ERROR, "Invalid object handle 0x%x pEntry %p\n", 385 hVASpace, pVaSpaceApi); 386 return NV_ERR_INVALID_OBJECT_HANDLE; 387 } 388 389 pVaSpaceApi = dynamicCast(pResourceRef->pResource, VaSpaceApi); 390 391 if (pDevice->pVASpace != NULL) 392 { 393 NV_PRINTF(LEVEL_ERROR, "device already has an Associated VASPace\n"); 394 return NV_ERR_INVALID_OBJECT_HANDLE; 395 } 396 397 // associate the vaspace as default 398 pDevice->pVASpace = pVaSpaceApi->pVASpace; 399 vaspaceIncRefCnt(pVaSpaceApi->pVASpace); 400 return NV_OK; 401 } 402