1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "core/core.h" 25 #include "gpu/gpu.h" 26 #include "os/os.h" 27 #include "gpu/mem_sys/kern_mem_sys.h" 28 #include "gpu/bus/kern_bus.h" 29 #include "gpu/bus/p2p_api.h" 30 #include "gpu/nvlink/kernel_nvlink.h" 31 #include "published/volta/gv100/dev_mmu.h" 32 33 /*! 34 * @brief Get physical address of the FB memory on systems where GPU memory 35 * is onlined to the OS 36 * 37 * @param[in] pGpu OBJGPU pointer 38 * @param[in] pKernelMemorySystem pointer to the kernel side KernelMemorySystem instance. 39 * @param[in] physAddr Physical Address of FB memory 40 * @param[in] numaNodeId NUMA node id where FB memory is added to the 41 * kernel 42 * 43 * @return NV_OK on success 44 */ 45 NV_STATUS 46 kmemsysGetFbNumaInfo_GV100 47 ( 48 OBJGPU *pGpu, 49 KernelMemorySystem *pKernelMemorySystem, 50 NvU64 *physAddr, 51 NvS32 *numaNodeId 52 ) 53 { 54 NV_STATUS status; 55 56 status = osGetFbNumaInfo(pGpu, physAddr, numaNodeId); 57 if (status == NV_OK) 58 { 59 NV_PRINTF(LEVEL_INFO, "NUMA FB Physical address: 0x%llx Node ID: 0x%x\n", 60 *physAddr, *numaNodeId); 61 } 62 63 return status; 64 } 65 66 /*! 67 * @brief Determine whether RM needs to invalidate GPU L2 cache during map call 68 * 69 * @param[in] pGpu OBJGPU pointer 70 * @param[in] pKernelMemorySystem pointer to the kernel side KernelMemorySystem instance. 71 * @param[in] bIsVolatile Whether the map call is to create vol mapping 72 * @param[in] aperture Aperture of the memory being mapped 73 * 74 * @return NV_OK on success 75 */ 76 NvBool 77 kmemsysNeedInvalidateGpuCacheOnMap_GV100 78 ( 79 OBJGPU *pGpu, 80 KernelMemorySystem *pKernelMemorySystem, 81 NvBool bIsVolatile, 82 NvU32 aperture 83 ) 84 { 85 // 86 // Only need to invalidate L2 for cached (vol=0) mapping to sys/peer memory 87 // because GPU's L2 is not coherent with CPU updates to sysmem 88 // See bug 3342220 for more info 89 // 90 return (!bIsVolatile && (aperture == NV_MMU_PTE_APERTURE_PEER_MEMORY || 91 aperture == NV_MMU_PTE_APERTURE_SYSTEM_COHERENT_MEMORY || 92 aperture == NV_MMU_PTE_APERTURE_SYSTEM_NON_COHERENT_MEMORY)); 93 } 94 95 /*! 96 * @brief Configure local GPU's peer ATS config using peer GPU's local 97 * ATS config. 98 * 99 * @param[in] pLocalGpu Local GPU OBJGPU pointer 100 * @param[in] pLocalKernelMemorySystem Local GPU KernelMemorySystem pointer 101 * @param[in] pRemoteGpu Remote GPU OBJGPU pointer 102 * @param[in] peerId peer id from local GPU to remote GPU in 103 * local GPU 104 * 105 * @return NV_OK on success 106 */ 107 static 108 NV_STATUS 109 _kmemsysConfigureAtsPeers 110 ( 111 OBJGPU *pLocalGpu, 112 KernelMemorySystem *pLocalKernelMemorySystem, 113 OBJGPU *pRemoteGpu, 114 NvU32 peerId 115 ) 116 { 117 RM_API *pLocalRmApi = GPU_GET_PHYSICAL_RMAPI(pLocalGpu); 118 RM_API *pRemoteRmApi = GPU_GET_PHYSICAL_RMAPI(pRemoteGpu); 119 NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS getParams = { 0 }; 120 NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS setParams = { 0 }; 121 122 NV_ASSERT_OK_OR_RETURN(pRemoteRmApi->Control(pRemoteRmApi, 123 pRemoteGpu->hInternalClient, 124 pRemoteGpu->hInternalSubdevice, 125 NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG, 126 &getParams, 127 sizeof(NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS))); 128 setParams.peerId = peerId; 129 setParams.addrSysPhys = getParams.addrSysPhys; 130 setParams.addrWidth = getParams.addrWidth; 131 setParams.mask = getParams.mask; 132 setParams.maskWidth = getParams.maskWidth; 133 134 NV_ASSERT_OK_OR_RETURN(pLocalRmApi->Control(pLocalRmApi, 135 pLocalGpu->hInternalClient, 136 pLocalGpu->hInternalSubdevice, 137 NV2080_CTRL_CMD_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG, 138 &setParams, 139 sizeof(NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS))); 140 141 return NV_OK; 142 } 143 144 /*! 145 * @brief Remove local GPU's peer ATS config 146 * 147 * @param[in] pLocalGpu Local GPU OBJGPU pointer 148 * @param[in] pLocalKernelMemorySystem Local GPU KernelMemorySystem pointer 149 * @param[in] peerId peer id from local GPU to remote GPU in 150 * local GPU 151 * 152 * @return NV_OK on success 153 */ 154 static 155 NV_STATUS 156 _kmemsysResetAtsPeerConfiguration 157 ( 158 OBJGPU *pLocalGpu, 159 KernelMemorySystem *pLocalKernelMemorySystem, 160 NvU32 peerId 161 ) 162 { 163 RM_API *pLocalRmApi = GPU_GET_PHYSICAL_RMAPI(pLocalGpu); 164 NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS getParams = { 0 }; 165 NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS setParams = { 0 }; 166 167 NV_ASSERT_OK_OR_RETURN(pLocalRmApi->Control(pLocalRmApi, 168 pLocalGpu->hInternalClient, 169 pLocalGpu->hInternalSubdevice, 170 NV2080_CTRL_CMD_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG, 171 &getParams, 172 sizeof(NV2080_CTRL_INTERNAL_MEMSYS_GET_LOCAL_ATS_CONFIG_PARAMS))); 173 174 setParams.peerId = peerId; 175 setParams.addrSysPhys = 0; 176 setParams.addrWidth = getParams.addrWidth; 177 setParams.mask = 0; 178 setParams.maskWidth = getParams.maskWidth; 179 180 NV_ASSERT_OK_OR_RETURN(pLocalRmApi->Control(pLocalRmApi, 181 pLocalGpu->hInternalClient, 182 pLocalGpu->hInternalSubdevice, 183 NV2080_CTRL_CMD_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG, 184 &setParams, 185 sizeof(NV2080_CTRL_INTERNAL_MEMSYS_SET_PEER_ATS_CONFIG_PARAMS))); 186 187 return NV_OK; 188 } 189 190 /** 191 * @brief Setup one pair of ATS peers (non-chiplib configs) 192 * 193 * @param[in] pGpu OBJGPU pointer 194 * @param[in] pKernelMemorySystem Kernel Memory System pointer 195 * @param[in] pRemoteGpu OBJGPU pointer for the ATS peer 196 * 197 * @return NV_OK on success 198 */ 199 static 200 NV_STATUS 201 _kmemsysSetupAtsPeers 202 ( 203 OBJGPU *pGpu, 204 KernelMemorySystem *pKernelMemorySystem, 205 OBJGPU *pRemoteGpu 206 ) 207 { 208 NvU32 peer1 = BUS_INVALID_PEER; 209 NvU32 peer2 = BUS_INVALID_PEER; 210 NV_STATUS status = NV_OK; 211 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 212 KernelMemorySystem *pLocalKernelMs = NULL; 213 KernelMemorySystem *pRemoteKernelMs = NULL; 214 NvU32 attributes = DRF_DEF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _NVLINK) | 215 DRF_DEF(_P2PAPI, _ATTRIBUTES, _LINK_TYPE, _SPA); 216 217 // Set up P2P ATS configuration 218 pLocalKernelMs = pKernelMemorySystem; 219 pRemoteKernelMs = GPU_GET_KERNEL_MEMORY_SYSTEM(pRemoteGpu); 220 221 if (pKernelNvlink != NULL) 222 { 223 // Trigger P2P link training to HS before releasing credits on P9 224 knvlinkTrainP2pLinksToActive(pGpu, pRemoteGpu, pKernelNvlink); 225 } 226 227 status = kbusCreateP2PMapping_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pRemoteGpu, GPU_GET_KERNEL_BUS(pRemoteGpu), 228 &peer1, &peer2, attributes); 229 if (status != NV_OK) 230 { 231 return status; 232 } 233 234 if (pLocalKernelMs && pRemoteKernelMs && 235 pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED) && 236 pRemoteGpu->getProperty(pRemoteGpu, PDB_PROP_GPU_ATS_SUPPORTED)) 237 { 238 status = _kmemsysConfigureAtsPeers(pGpu, pLocalKernelMs, pRemoteGpu, peer1); 239 if (status != NV_OK) 240 { 241 NV_PRINTF(LEVEL_ERROR, 242 "Configuring ATS p2p config between GPU%u and GPU%u " 243 "failed with status %x\n", pGpu->gpuInstance, 244 pRemoteGpu->gpuInstance, status); 245 return status; 246 } 247 248 status = _kmemsysConfigureAtsPeers(pRemoteGpu, pRemoteKernelMs, pGpu, peer2); 249 if (status != NV_OK) 250 { 251 NV_PRINTF(LEVEL_ERROR, 252 "Configuring ATS p2p config between GPU%u and GPU%u " 253 "failed with status %x\n", pRemoteGpu->gpuInstance, 254 pGpu->gpuInstance, status); 255 return status; 256 } 257 } 258 259 return NV_OK; 260 } 261 262 /** 263 * @brief Remove one pair of ATS peers (non-chiplib configs) 264 * 265 * @param[in] pGpu OBJGPU pointer 266 * @param[in] pKernelMemorySystem Kernel Memory System pointer 267 * @param[in] pRemoteGpu OBJGPU pointer for the ATS peer 268 * 269 * @return NV_OK on success 270 */ 271 static 272 NV_STATUS 273 _kmemsysRemoveAtsPeers 274 ( 275 OBJGPU *pGpu, 276 KernelMemorySystem *pKernelMemorySystem, 277 OBJGPU *pRemoteGpu 278 ) 279 { 280 NvU32 peer1 = BUS_INVALID_PEER; 281 NvU32 peer2 = BUS_INVALID_PEER; 282 NV_STATUS status = NV_OK; 283 KernelMemorySystem *pLocalKernelMs = NULL; 284 KernelMemorySystem *pRemoteKernelMs = NULL; 285 NvU32 attributes = DRF_DEF(_P2PAPI, _ATTRIBUTES, _CONNECTION_TYPE, _NVLINK) | 286 DRF_DEF(_P2PAPI, _ATTRIBUTES, _LINK_TYPE, _SPA); 287 288 pLocalKernelMs = pKernelMemorySystem; 289 pRemoteKernelMs = GPU_GET_KERNEL_MEMORY_SYSTEM(pRemoteGpu); 290 291 peer1 = kbusGetPeerId_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pRemoteGpu); 292 peer2 = kbusGetPeerId_HAL(pRemoteGpu, GPU_GET_KERNEL_BUS(pRemoteGpu), pGpu); 293 294 status = kbusRemoveP2PMapping_HAL(pGpu, GPU_GET_KERNEL_BUS(pGpu), pRemoteGpu, GPU_GET_KERNEL_BUS(pRemoteGpu), 295 peer1, peer2, attributes); 296 if (status != NV_OK) 297 { 298 return status; 299 } 300 301 if (pLocalKernelMs && pRemoteKernelMs && 302 pGpu->getProperty(pGpu, PDB_PROP_GPU_ATS_SUPPORTED) && 303 pRemoteGpu->getProperty(pRemoteGpu, PDB_PROP_GPU_ATS_SUPPORTED)) 304 { 305 status = _kmemsysResetAtsPeerConfiguration(pGpu, pLocalKernelMs, peer1); 306 if (status != NV_OK) 307 { 308 NV_PRINTF(LEVEL_ERROR, 309 "Removing ATS p2p config between GPU%u and GPU%u " 310 "failed with status %x\n", pGpu->gpuInstance, 311 pRemoteGpu->gpuInstance, status); 312 } 313 314 status = _kmemsysResetAtsPeerConfiguration(pRemoteGpu, pRemoteKernelMs, peer2); 315 if (status != NV_OK) 316 { 317 NV_PRINTF(LEVEL_ERROR, 318 "Removinging ATS p2p config between GPU%u and GPU%u " 319 "failed with status %x\n", pRemoteGpu->gpuInstance, 320 pGpu->gpuInstance, status); 321 } 322 } 323 324 return NV_OK; 325 } 326 327 /** 328 * @brief Setup ATS peer access. On GV100 and GH180, ATS peers use NVLINK. 329 * 330 * @param[in] pGpu OBJGPU pointer 331 * @param[in] pKernelMemorySystem Kernel Memory System pointer 332 * 333 * @return NV_OK on success 334 */ 335 NV_STATUS 336 kmemsysSetupAllAtsPeers_GV100 337 ( 338 OBJGPU *pGpu, 339 KernelMemorySystem *pKernelMemorySystem 340 ) 341 { 342 KernelNvlink *pKernelNvlink = GPU_GET_KERNEL_NVLINK(pGpu); 343 NvU32 gpuAttachCnt, gpuAttachMask, gpuInstance = 0; 344 345 NV_STATUS status = NV_OK; 346 POBJGPU pRemoteGpu = NULL; 347 348 NV_CHECK_OR_RETURN(LEVEL_WARNING, pKernelNvlink != NULL, status); 349 350 // loop over all possible GPU pairs and setup the ATS config 351 gpumgrGetGpuAttachInfo(&gpuAttachCnt, &gpuAttachMask); 352 while ((pRemoteGpu = gpumgrGetNextGpu(gpuAttachMask, &gpuInstance)) != NULL) 353 { 354 if (pRemoteGpu == pGpu) 355 continue; 356 357 if (gpuIsGpuFullPower(pRemoteGpu) == NV_FALSE) 358 continue; 359 360 if (!knvlinkIsNvlinkP2pSupported(pGpu, pKernelNvlink, pRemoteGpu)) 361 continue; 362 363 status = _kmemsysSetupAtsPeers(pGpu, pKernelMemorySystem, pRemoteGpu); 364 if (status != NV_OK) 365 return status; 366 } 367 368 return NV_OK; 369 } 370 371 /** 372 * @brief Remove ATS peer access. On GV100 and GH180, ATS peers use NVLINK. 373 * 374 * @param[in] pGpu OBJGPU pointer 375 * @param[in] pKernelMemorySystem Kernel Memory System pointer 376 */ 377 void 378 kmemsysRemoveAllAtsPeers_GV100 379 ( 380 OBJGPU *pGpu, 381 KernelMemorySystem *pKernelMemorySystem 382 ) 383 { 384 NvU32 gpuAttachCnt, gpuAttachMask, gpuInstance = 0; 385 386 NV_STATUS status = NV_OK; 387 POBJGPU pRemoteGpu = NULL; 388 389 // loop over all possible GPU pairs and remove the ATS config 390 gpumgrGetGpuAttachInfo(&gpuAttachCnt, &gpuAttachMask); 391 while ((pRemoteGpu = gpumgrGetNextGpu(gpuAttachMask, &gpuInstance)) != NULL) 392 { 393 if (pRemoteGpu == pGpu) 394 continue; 395 396 if (gpuIsGpuFullPower(pRemoteGpu) == NV_FALSE) 397 continue; 398 399 status = _kmemsysRemoveAtsPeers(pGpu, pKernelMemorySystem, pRemoteGpu); 400 if (status != NV_OK) 401 { 402 NV_PRINTF(LEVEL_ERROR, "Failed to remove ATS peer access between GPU%d and GPU%d\n", 403 pGpu->gpuInstance, pRemoteGpu->gpuInstance); 404 } 405 } 406 } 407 408