1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /***************************************************************************\ 25 * * 26 * Module: fecs_event_list.c * 27 * Description: * 28 * This module contains an implementation of the Event Buffer * 29 * callback for FECS events * 30 * * 31 \***************************************************************************/ 32 33 #include "kernel/gpu/gr/kernel_graphics.h" 34 #include "kernel/rmapi/event.h" 35 #include "kernel/rmapi/event_buffer.h" 36 #include "libraries/resserv/rs_server.h" 37 #include "kernel/core/locks.h" 38 #include "kernel/os/os.h" 39 #include "kernel/gpu/gr/fecs_event_list.h" 40 #include "kernel/gpu/mig_mgr/gpu_instance_subscription.h" 41 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 42 #include "kernel/gpu/bus/kern_bus.h" 43 #include "kernel/gpu/mem_mgr/mem_mgr.h" 44 #include "kernel/gpu/fifo/kernel_channel.h" 45 #include "kernel/gpu/subdevice/subdevice.h" 46 #include "kernel/virtualization/hypervisor/hypervisor.h" 47 #include "rmapi/client.h" 48 #include "objtmr.h" 49 50 #include "class/cl90cdtypes.h" 51 #include "ctrl/ctrl90cd.h" 52 53 #define NV_FECS_TRACE_MAX_TIMESTAMPS 5 54 #define NV_FECS_TRACE_MAGIC_INVALIDATED 0xdededede // magic number for entries that have been read 55 #define NV_FECS_TRACE_CALLBACK_TIME_NS 33333333 // Approximating 30Hz callback 56 57 typedef struct 58 { 59 NvU32 magic_lo; 60 NvU32 magic_hi; 61 NvU32 context_id; 62 NvU32 context_ptr; 63 NvU32 new_context_id; 64 NvU32 new_context_ptr; 65 NvU64 ts[NV_FECS_TRACE_MAX_TIMESTAMPS]; 66 NvU32 reserved[13]; 67 NvU32 seqno; 68 } FECS_EVENT_RECORD; 69 70 /*! Opaque pointer to private data */ 71 typedef struct VGPU_FECS_TRACE_STAGING_BUFFER VGPU_FECS_TRACE_STAGING_BUFFER; 72 73 /*! Private FECS event buffer data stored per-KGR */ 74 struct KGRAPHICS_FECS_TRACE_INFO 75 { 76 NvU8 *pFecsBufferMapping; 77 NvU16 fecsCtxswLogRecordsPerIntr; 78 NvU16 fecsTraceRdOffset; 79 NvU16 fecsTraceCounter; 80 NvU32 fecsCtxswLogIntrPending; 81 NvU32 fecsLastSeqno; 82 83 #if PORT_IS_MODULE_SUPPORTED(crypto) 84 PORT_CRYPTO_PRNG *pFecsLogPrng; 85 #endif 86 87 // 88 // GR Routing information for GPU instance to which this engine is assigned if MIG is enabled. 89 // Will be 0/NULL for unassigned GR engines or if MIG is disabled 90 // 91 NvHandle hClient; 92 NvHandle hSubdevice; 93 NvU32 localGrEngineIdx; 94 95 // vGPU FECS staging eventbuffer (guest only) 96 VGPU_FECS_TRACE_STAGING_BUFFER *pVgpuStaging; 97 }; 98 99 /*! 100 * @brief Function to populate client/subdevice/grRouteInfo from cached 101 * information in order to make calls into the specific MIG GPU instance 102 * to which a GR engine is assigned. If MIG is not enabled, GPU 103 * internal client/subdevice handles will be used and grRouteInfo is 104 * cleared 105 * 106 * @param[in] pGpu 107 * @param[in] pKernelGraphics 108 * @param[out] phClient Client handle to populate 109 * @param[out] phSubdevice Subdevice handle to populate 110 * @param[out] pGrRouteInfo Internal GR Routing info to populate 111 */ 112 static NV_STATUS 113 _fecsLoadInternalRoutingInfo 114 ( 115 OBJGPU *pGpu, 116 KernelGraphics *pKernelGraphics, 117 NvHandle *phClient, 118 NvHandle *phSubdevice, 119 NV2080_CTRL_GR_ROUTE_INFO *pGrRouteInfo 120 ) 121 { 122 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 123 124 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_ERR_INVALID_STATE); 125 126 portMemSet(pGrRouteInfo, 0, sizeof(*pGrRouteInfo)); 127 128 if (!IS_MIG_IN_USE(pGpu)) 129 { 130 *phClient = pGpu->hInternalClient; 131 *phSubdevice = pGpu->hInternalSubdevice; 132 return NV_OK; 133 } 134 135 // GR Engines not allocated to any GPU instance will have null handles 136 NV_CHECK_OR_RETURN(LEVEL_INFO, pFecsTraceInfo->hClient != NV01_NULL_OBJECT, NV_ERR_INVALID_ARGUMENT); 137 138 kgrmgrCtrlSetEngineID(pFecsTraceInfo->localGrEngineIdx, pGrRouteInfo); 139 *phClient = pFecsTraceInfo->hClient; 140 *phSubdevice = pFecsTraceInfo->hSubdevice; 141 142 return NV_OK; 143 } 144 145 static NV_STATUS 146 fecsExtractTagAndTimestamp 147 ( 148 OBJGPU *pGpu, 149 KernelGraphics *pKernelGraphics, 150 NvU64 rawTimestamp, 151 NvU64 *pTimestampVal, 152 NvU8 *pTag 153 ) 154 { 155 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 156 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 157 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pFecsTraceDefines != NULL, NV_ERR_INVALID_STATE); 158 159 *pTag = ((NvU64_HI32(rawTimestamp)) >> pKernelGraphicsStaticInfo->pFecsTraceDefines->timestampHiTagShift) & pKernelGraphicsStaticInfo->pFecsTraceDefines->timestampHiTagMask; 160 *pTimestampVal = rawTimestamp & pKernelGraphicsStaticInfo->pFecsTraceDefines->timestampVMask; 161 162 // timestamp encoded as right shifted N bits, since they hold zeros. RM needs to reverse that here. 163 *pTimestampVal <<= pKernelGraphicsStaticInfo->pFecsTraceDefines->numLowerBitsZeroShift; 164 return NV_OK; 165 } 166 167 // 168 // The function formats the information from the FECS Buffer into a format 169 // suitable for EventBuffer, and then checks to see whether the subscriber 170 // needs to be notified. If so, the subscriber is notified. 171 // 172 // pGpu is used to retrieve data on the pid, and 173 // seqno provides the sequence number for the user to keep track of 174 // whether any entry has been dropped. 175 // pRecord is the current FECS entry. 176 // 177 static void 178 formatAndNotifyFecsRecord 179 ( 180 OBJGPU *pGpu, 181 KernelGraphics *pKernelGraphics, 182 FECS_EVENT_RECORD *pRecord 183 ) 184 { 185 FECS_EVENT_NOTIFICATION_DATA notifRecord; 186 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 187 KernelChannel *pKernelChannel = NULL; 188 KernelChannel *pKernelChannelNew = NULL; 189 MIG_INSTANCE_REF *pChannelRef; 190 MIG_INSTANCE_REF *pNewChannelRef; 191 INST_BLOCK_DESC inst; 192 NvU32 timestampId; 193 NvU64 noisyTimestampStart = 0; 194 NvU64 noisyTimestampRange = 0; 195 NvU32 instSize; 196 NvU32 instShift; 197 NV_STATUS status; 198 199 if (pRecord == NULL) 200 { 201 NV_PRINTF(LEVEL_ERROR, "Invalid FECS record!\n"); 202 DBG_BREAKPOINT(); 203 return; 204 } 205 206 kfifoGetInstBlkSizeAlign_HAL(pKernelFifo, &instSize, &instShift); 207 208 portMemSet(¬ifRecord, 0, sizeof(notifRecord)); 209 210 inst.address = ((NvU64)pRecord->context_ptr) << instShift; 211 inst.aperture = INST_BLOCK_APERTURE_VIDEO_MEMORY; 212 inst.gfid = GPU_GFID_PF; 213 if (pRecord->context_ptr && 214 (kfifoConvertInstToKernelChannel_HAL(pGpu, pKernelFifo, &inst, &pKernelChannel) != NV_OK)) 215 { 216 NV_PRINTF(LEVEL_INFO, "Error getting channel!\n"); 217 pKernelChannel = NULL; 218 } 219 220 inst.address = ((NvU64)pRecord->new_context_ptr) << instShift; 221 inst.aperture = INST_BLOCK_APERTURE_VIDEO_MEMORY; 222 inst.gfid = GPU_GFID_PF; 223 if (pRecord->new_context_ptr && 224 (kfifoConvertInstToKernelChannel_HAL(pGpu, pKernelFifo, &inst, &pKernelChannelNew) != NV_OK)) 225 { 226 NV_PRINTF(LEVEL_INFO, "Error getting new channel!\n"); 227 pKernelChannelNew = NULL; 228 } 229 230 pChannelRef = (pKernelChannel != NULL) ? kchannelGetMIGReference(pKernelChannel) : NULL; 231 pNewChannelRef = (pKernelChannelNew != NULL) ? kchannelGetMIGReference(pKernelChannelNew) : NULL; 232 233 if (kgraphicsIsFecsRecordUcodeSeqnoSupported(pGpu, pKernelGraphics)) 234 { 235 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 236 237 // Dropped at least 1 event 238 if ((pFecsTraceInfo->fecsLastSeqno + 1) != pRecord->seqno) 239 { 240 notifRecord.dropCount = pRecord->seqno - pFecsTraceInfo->fecsLastSeqno - 1; 241 } 242 243 pFecsTraceInfo->fecsLastSeqno = pRecord->seqno; 244 } 245 246 for (timestampId = 0; timestampId < NV_FECS_TRACE_MAX_TIMESTAMPS; timestampId++) 247 { 248 NV_ASSERT_OK_OR_ELSE(status, 249 fecsExtractTagAndTimestamp(pGpu, pKernelGraphics, 250 pRecord->ts[timestampId], 251 ¬ifRecord.timestamp, 252 ¬ifRecord.tag), 253 return); 254 255 // 256 // determine a few more fields of the current record by subevent type, 257 // before we notify the subscriber 258 // 259 switch (notifRecord.tag) 260 { 261 case NV_EVENT_BUFFER_FECS_CTXSWTAG_RESTORE_START: 262 case NV_EVENT_BUFFER_FECS_CTXSWTAG_CONTEXT_START: 263 if (pKernelChannelNew != NULL) 264 { 265 notifRecord.pid = pKernelChannelNew->ProcessID; 266 notifRecord.subpid = pKernelChannelNew->SubProcessID; 267 notifRecord.userInfo = (NvU64)(NvUPtr)pKernelChannelNew->pUserInfo; 268 notifRecord.context_id = kchannelGetCid(pKernelChannelNew); 269 270 if (kmigmgrIsMIGReferenceValid(pNewChannelRef)) 271 { 272 notifRecord.swizzId = pNewChannelRef->pKernelMIGGpuInstance->swizzId; 273 if (pNewChannelRef->pMIGComputeInstance) 274 notifRecord.computeInstanceId = pNewChannelRef->pMIGComputeInstance->id; 275 else 276 notifRecord.computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 277 } 278 279 if (notifRecord.tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_RESTORE_START) 280 { 281 noisyTimestampStart = notifRecord.timestamp; 282 } 283 else 284 { 285 noisyTimestampRange = notifRecord.timestamp - noisyTimestampStart; 286 } 287 } 288 break; 289 290 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_WFI: 291 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_GFXP: 292 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CTAP: 293 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CILP: 294 if (pKernelChannel != NULL) 295 { 296 notifRecord.pid = pKernelChannel->ProcessID; 297 notifRecord.subpid = pKernelChannel->SubProcessID; 298 notifRecord.userInfo = (NvU64)(NvUPtr)pKernelChannel->pUserInfo; 299 notifRecord.context_id = kchannelGetCid(pKernelChannel); 300 301 if (kmigmgrIsMIGReferenceValid(pChannelRef)) 302 { 303 notifRecord.swizzId = pChannelRef->pKernelMIGGpuInstance->swizzId; 304 if (pChannelRef->pMIGComputeInstance) 305 notifRecord.computeInstanceId = pChannelRef->pMIGComputeInstance->id; 306 else 307 notifRecord.computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 308 } 309 } 310 break; 311 312 case NV_EVENT_BUFFER_FECS_CTXSWTAG_CTXSW_REQ_BY_HOST: 313 case NV_EVENT_BUFFER_FECS_CTXSWTAG_SAVE_END: 314 if (pKernelChannel != NULL) 315 { 316 notifRecord.pid = pKernelChannel->ProcessID; 317 notifRecord.subpid = pKernelChannel->SubProcessID; 318 notifRecord.userInfo = (NvU64)(NvUPtr)pKernelChannel->pUserInfo; 319 notifRecord.context_id = kchannelGetCid(pKernelChannel); 320 321 if (kmigmgrIsMIGReferenceValid(pChannelRef)) 322 { 323 notifRecord.swizzId = pChannelRef->pKernelMIGGpuInstance->swizzId; 324 if (pChannelRef->pMIGComputeInstance != NULL) 325 notifRecord.computeInstanceId = pChannelRef->pMIGComputeInstance->id; 326 else 327 notifRecord.computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 328 } 329 330 if (notifRecord.tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_CTXSW_REQ_BY_HOST) 331 { 332 noisyTimestampStart = notifRecord.timestamp; 333 } 334 else 335 { 336 noisyTimestampRange = notifRecord.timestamp - noisyTimestampStart; 337 } 338 } 339 break; 340 341 default: 342 continue; 343 } 344 345 if ((pKernelChannel != NULL) || (pKernelChannelNew != NULL)) 346 { 347 FecsEventBufferBindMultiMapSubmap *pSubmap; 348 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 349 350 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 351 352 notifRecord.noisyTimestamp = 0; 353 if ((noisyTimestampRange > 0) && (pFecsTraceInfo->pFecsLogPrng != NULL)) 354 notifRecord.noisyTimestamp = noisyTimestampStart + portCryptoPseudoRandomGeneratorGetU32(pFecsTraceInfo->pFecsLogPrng) % noisyTimestampRange; 355 356 if (notifRecord.userInfo != 0) 357 { 358 // Notify event buffers listening for the current UID 359 pSubmap = multimapFindSubmap(&pGpu->fecsEventBufferBindingsUid, notifRecord.userInfo); 360 notifyEventBuffers(pGpu, pSubmap, ¬ifRecord); 361 } 362 363 // Notify event buffers listening for all UIDs 364 pSubmap = multimapFindSubmap(&pGpu->fecsEventBufferBindingsUid, 0); 365 notifyEventBuffers(pGpu, pSubmap, ¬ifRecord); 366 367 // Clear so we don't report drops for every event in this record 368 notifRecord.dropCount = 0; 369 } 370 } 371 } 372 373 static NV_STATUS 374 _fecsEventBufferAdd 375 ( 376 OBJGPU *pGpu, 377 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind, 378 NvU8 tag, 379 NvU32 pid, 380 NvU8 swizzId, 381 NvU8 computeInstanceId, 382 NvU32 context_id, 383 NvU64 timestamp 384 ) 385 { 386 NV_STATUS status; 387 NvBool bNotify; 388 NvP64 notificationHandle; 389 EVENT_BUFFER_PRODUCER_DATA notifyEvent; 390 NvU32 notifyIndex; 391 392 switch (pBind->version) 393 { 394 case 2: 395 notifyIndex = NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH_V2; 396 break; 397 case 1: 398 notifyIndex = NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH; 399 break; 400 default: 401 return NV_ERR_INVALID_ARGUMENT; 402 } 403 404 portMemSet(¬ifyEvent, 0, sizeof(notifyEvent)); 405 notifyEvent.pVardata = NV_PTR_TO_NvP64(NULL); 406 notifyEvent.vardataSize = 0; 407 408 NV_EVENT_BUFFER_FECS_RECORD_V2 fecsRecord; 409 portMemSet(&fecsRecord, 0, sizeof(fecsRecord)); 410 fecsRecord.tag = tag; 411 fecsRecord.pid = pid; 412 if (pBind->version >= 2) 413 { 414 fecsRecord.migGpuInstanceId = swizzId; 415 fecsRecord.migComputeInstanceId = computeInstanceId; 416 } 417 fecsRecord.context_id = context_id; 418 fecsRecord.timestamp = timestamp; 419 fecsRecord.seqno = pBind->pEventBuffer->seqNo++; 420 421 notifyEvent.pPayload = NV_PTR_TO_NvP64(&fecsRecord); 422 notifyEvent.payloadSize = sizeof(fecsRecord); 423 424 status = eventBufferAdd(pBind->pEventBuffer, ¬ifyEvent, notifyIndex, &bNotify, ¬ificationHandle); 425 426 if ((status == NV_OK) && bNotify && notificationHandle) 427 { 428 osEventNotification(pGpu, 429 pBind->pEventBuffer->pListeners, 430 notifyIndex, 431 ¬ifyEvent, 432 0); // Do not copy structure -- embedded pointers. 433 pBind->pEventBuffer->bNotifyPending = NV_TRUE; 434 } 435 436 return status; 437 } 438 439 void 440 notifyEventBuffers 441 ( 442 OBJGPU *pGpu, 443 FecsEventBufferBindMultiMapSubmap *pSubmap, 444 FECS_EVENT_NOTIFICATION_DATA const *pRecord 445 ) 446 { 447 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 448 449 if (pSubmap != NULL) 450 { 451 FecsEventBufferBindMultiMapIter iter = multimapSubmapIterItems(&pGpu->fecsEventBufferBindingsUid, pSubmap); 452 453 while (multimapItemIterNext(&iter)) 454 { 455 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind = iter.pValue; 456 NvBool bSanitizeKernel = (!pBind->bKernel) && (pRecord->userInfo == 0); 457 NvBool bSanitizeUser = (!pBind->bAdmin) && (pBind->pUserInfo != pRecord->userInfo); 458 NvBool bSanitize = bSanitizeKernel || bSanitizeUser; 459 NvU32 pid; 460 NvU32 context_id; 461 NvU64 timestamp; 462 NvU32 tag = pRecord->tag; 463 NvU8 swizzId = pRecord->swizzId; 464 NvU8 computeInstanceId = pRecord->computeInstanceId; 465 466 pBind->pEventBuffer->seqNo += pRecord->dropCount; 467 468 if (bSanitize || !(NVBIT(pRecord->tag) & pBind->eventMask)) 469 { 470 // 471 // Re-map CONTEXT_START as SIMPLE_START and SAVE_END as SIMPLE_END if 472 // the binding has simple level-of-detail or is being sanitized 473 // 474 if ((bSanitize || (pBind->eventMask & NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_START)) && 475 (tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_CONTEXT_START)) 476 { 477 tag = NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_START; 478 } 479 else if ((bSanitize || (pBind->eventMask & NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_END)) && 480 (tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_SAVE_END)) 481 { 482 tag = NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_END; 483 } 484 else if ((tag != NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_START) && 485 (tag != NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_END)) 486 { 487 continue; 488 } 489 } 490 491 // 492 // While MIG is enabled, if the bindpoint is registered for a specific MIG GPU instance 493 // then filter out records from other GPU instances 494 // 495 if (bMIGInUse && 496 ((pBind->swizzId != NV2080_CTRL_GPU_PARTITION_ID_INVALID) && 497 (pRecord->swizzId != pBind->swizzId))) 498 continue; 499 500 // While MIG is enabled, pause tracing of V1 bindpoints 501 if (bMIGInUse && (pBind->version < 2)) 502 continue; 503 504 if (bSanitizeKernel) 505 { 506 timestamp = pRecord->noisyTimestamp; 507 pid = NV_EVENT_BUFFER_KERNEL_PID; 508 context_id = NV_EVENT_BUFFER_KERNEL_CONTEXT; 509 swizzId = NV_EVENT_BUFFER_KERNEL_MIG_GI; 510 computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 511 } 512 else if (bSanitizeUser) 513 { 514 timestamp = pRecord->noisyTimestamp; 515 pid = NV_EVENT_BUFFER_HIDDEN_PID; 516 context_id = NV_EVENT_BUFFER_HIDDEN_CONTEXT; 517 swizzId = NV_EVENT_BUFFER_HIDDEN_MIG_GI; 518 computeInstanceId = NV_EVENT_BUFFER_HIDDEN_MIG_CI; 519 } 520 else 521 { 522 timestamp = pRecord->timestamp; 523 pid = pRecord->pid; 524 context_id = pRecord->context_id; 525 } 526 527 _fecsEventBufferAdd(pGpu, pBind, tag, 528 pid, 529 bMIGInUse ? swizzId : NV_EVENT_BUFFER_INVALID_MIG_GI, 530 bMIGInUse ? computeInstanceId : NV_EVENT_BUFFER_INVALID_MIG_CI, 531 context_id, timestamp); 532 } 533 } 534 } 535 536 static NV_STATUS 537 _getFecsMemDesc 538 ( 539 OBJGPU *pGpu, 540 KernelGraphics *pKernelGraphics, 541 MEMORY_DESCRIPTOR **ppFecsMemDesc 542 ) 543 { 544 MEMORY_DESCRIPTOR *pMemDesc; 545 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, GPU_GFID_PF); 546 547 *ppFecsMemDesc = NULL; 548 NV_CHECK_OR_RETURN(LEVEL_SILENT, pGlobalCtxBuffers != NULL, NV_ERR_INVALID_STATE); 549 pMemDesc = pGlobalCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT]; 550 if (pMemDesc != NULL) 551 pMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu); 552 553 *ppFecsMemDesc = pMemDesc; 554 555 return NV_OK; 556 } 557 558 static NV_STATUS 559 _getFecsEventListParameters 560 ( 561 OBJGPU *pGpu, 562 KernelGraphics *pKernelGraphics, 563 MEMORY_DESCRIPTOR **ppFecsMemDesc, 564 NvU32 *pFecsRecordSize 565 ) 566 { 567 const KGRAPHICS_STATIC_INFO *pStaticInfo; 568 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 569 _getFecsMemDesc(pGpu, pKernelGraphics, ppFecsMemDesc)); 570 571 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 572 NV_CHECK_OR_RETURN(LEVEL_ERROR, pStaticInfo != NULL, NV_ERR_INVALID_STATE); 573 NV_ASSERT_OR_RETURN(pStaticInfo->pFecsTraceDefines != NULL, NV_ERR_INVALID_STATE); 574 *pFecsRecordSize = pStaticInfo->pFecsTraceDefines->fecsRecordSize; 575 576 return NV_OK; 577 } 578 579 /*! 580 * @brief Set cached routing info for this KernelGraphics engine to make RPC calls 581 * into the specific MIG GPU instance to which this engine is assigned 582 * 583 * @param[in] pGpu 584 * @param[in] pKernelGraphics 585 * @param[in] hClient Client handle to make calls into MIG GPU instance 586 * @param[in] hSubdevice Subdevice handle to make calls into MIG GPU instance 587 * @param[in] localGrEngineIdx Local GR index for this GR engine in MIG GPU instance 588 */ 589 void fecsSetRoutingInfo 590 ( 591 OBJGPU *pGpu, 592 KernelGraphics *pKernelGraphics, 593 NvHandle hClient, 594 NvHandle hSubdevice, 595 NvU32 localGrEngineIdx 596 ) 597 { 598 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 599 600 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 601 602 pFecsTraceInfo->hClient = hClient; 603 pFecsTraceInfo->hSubdevice = hSubdevice; 604 pFecsTraceInfo->localGrEngineIdx = localGrEngineIdx; 605 } 606 607 // Clear cached routing info used to make GR calls into specific MIG GPU instance 608 void fecsClearRoutingInfo 609 ( 610 OBJGPU *pGpu, 611 KernelGraphics *pKernelGraphics 612 ) 613 { 614 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 615 616 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 617 618 pFecsTraceInfo->hClient = NV01_NULL_OBJECT; 619 pFecsTraceInfo->hSubdevice = NV01_NULL_OBJECT; 620 pFecsTraceInfo->localGrEngineIdx = 0; 621 } 622 623 NV_STATUS 624 fecsCtxswLoggingInit 625 ( 626 OBJGPU *pGpu, 627 KernelGraphics *pKernelGraphics, 628 KGRAPHICS_FECS_TRACE_INFO **ppFecsTraceInfo 629 ) 630 { 631 NvU64 seed; 632 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo; 633 634 NV_ASSERT_OR_RETURN(ppFecsTraceInfo != NULL, NV_ERR_NOT_SUPPORTED); 635 pFecsTraceInfo = portMemAllocNonPaged(sizeof(*pFecsTraceInfo)); 636 if (pFecsTraceInfo == NULL) 637 return NV_ERR_NO_MEMORY; 638 portMemSet(pFecsTraceInfo, 0, sizeof(*pFecsTraceInfo)); 639 640 *ppFecsTraceInfo = pFecsTraceInfo; 641 642 osGetCurrentTick(&seed); 643 pFecsTraceInfo->pFecsLogPrng = portCryptoPseudoRandomGeneratorCreate(seed); 644 multimapInit(&pGpu->fecsEventBufferBindingsUid, portMemAllocatorGetGlobalNonPaged()); 645 646 kgraphicsInitFecsRegistryOverrides_HAL(pGpu, pKernelGraphics); 647 648 return NV_OK; 649 } 650 651 void 652 fecsCtxswLoggingTeardown 653 ( 654 OBJGPU *pGpu, 655 KernelGraphics *pKernelGraphics 656 ) 657 { 658 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 659 660 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 661 662 multimapDestroy(&pGpu->fecsEventBufferBindingsUid); 663 664 portCryptoPseudoRandomGeneratorDestroy(pFecsTraceInfo->pFecsLogPrng); 665 pFecsTraceInfo->pFecsLogPrng = NULL; 666 portMemFree(pFecsTraceInfo); 667 } 668 669 /*! set num records to process per intr */ 670 void fecsSetRecordsPerIntr 671 ( 672 OBJGPU *pGpu, 673 KernelGraphics *pKernelGraphics, 674 NvU32 recordsPerIntr 675 ) 676 { 677 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 678 679 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 680 pFecsTraceInfo->fecsCtxswLogRecordsPerIntr = recordsPerIntr; 681 } 682 683 NvBool 684 fecsBufferChanged 685 ( 686 OBJGPU *pGpu, 687 KernelGraphics *pKernelGraphics 688 ) 689 { 690 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 691 NvU8 *pFecsBufferMapping; 692 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 693 NvU32 fecsRecordSize; 694 FECS_EVENT_RECORD *pPeekRecord; 695 NV_STATUS status; 696 697 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 698 pFecsBufferMapping = pFecsTraceInfo->pFecsBufferMapping; 699 700 status = _getFecsEventListParameters(pGpu, pKernelGraphics, &pFecsMemDesc, &fecsRecordSize); 701 if ((status != NV_OK) || (pFecsMemDesc == NULL) || (pFecsBufferMapping == NULL)) 702 return NV_FALSE; 703 704 pPeekRecord = (FECS_EVENT_RECORD*)(pFecsBufferMapping + 705 (pFecsTraceInfo->fecsTraceRdOffset * fecsRecordSize)); 706 707 if (pPeekRecord->magic_lo != NV_FECS_TRACE_MAGIC_INVALIDATED) 708 { 709 return NV_TRUE; 710 } 711 712 return NV_FALSE; 713 } 714 715 static void 716 _fecsOsWorkItem 717 ( 718 NvU32 gpuInstance, 719 void *data 720 ) 721 { 722 OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); 723 KernelGraphicsManager *pKernelGraphicsManager; 724 725 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR, pGpu != NULL); 726 pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 727 728 nvEventBufferFecsCallback(pGpu, NULL); 729 kgrmgrClearFecsCallbackScheduled(pGpu, pKernelGraphicsManager); 730 } 731 732 static NV_STATUS 733 _fecsTimerCallback 734 ( 735 OBJGPU *pGpu, 736 OBJTMR *pTmr, 737 TMR_EVENT *pTmrEvent 738 ) 739 { 740 NV_STATUS status = NV_OK; 741 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 742 NvU32 i; 743 744 // If any Kgraphics have events, schedule work item 745 for (i = 0; i < GPU_MAX_GRS; i++) 746 { 747 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, i); 748 749 if (pKernelGraphics == NULL) 750 continue; 751 752 if (fecsBufferChanged(pGpu, pKernelGraphics) && kgrmgrSignalFecsCallbackScheduled(pGpu, pKernelGraphicsManager)) 753 { 754 NV_CHECK_OK(status, LEVEL_ERROR, osQueueWorkItemWithFlags(pGpu, _fecsOsWorkItem, NULL, OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW)); 755 756 if (status != NV_OK) 757 kgrmgrClearFecsCallbackScheduled(pGpu, pKernelGraphicsManager); 758 759 break; 760 } 761 } 762 763 // TMR_FLAG_RECUR does not work, so reschedule it here. 764 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, LEVEL_ERROR, tmrEventScheduleRel(pTmr, pTmrEvent, NV_FECS_TRACE_CALLBACK_TIME_NS)); 765 766 return status; 767 } 768 769 static NV_STATUS 770 _fecsTimerCreate 771 ( 772 OBJGPU *pGpu 773 ) 774 { 775 OBJTMR *pTmr = GPU_GET_TIMER(pGpu); 776 NvU32 timerFlags = TMR_FLAG_RECUR; 777 // Unix needs to use the OS timer to avoid corrupting records, but Windows doesn't have an OS timer implementation 778 timerFlags |= TMR_FLAG_USE_OS_TIMER; 779 780 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 781 tmrEventCreate(pTmr, &pGpu->pFecsTimerEvent, _fecsTimerCallback, NULL, timerFlags)); 782 783 // This won't be a true 30Hz timer as the callbacks are scheduled from the time they're called 784 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 785 tmrEventScheduleRel(pTmr, pGpu->pFecsTimerEvent, NV_FECS_TRACE_CALLBACK_TIME_NS)); 786 787 return NV_OK; 788 } 789 790 static void 791 _fecsTimerDestroy 792 ( 793 OBJGPU *pGpu 794 ) 795 { 796 if (pGpu->pFecsTimerEvent != NULL) 797 { 798 OBJTMR *pTmr = GPU_GET_TIMER(pGpu); 799 800 tmrEventCancel(pTmr, pGpu->pFecsTimerEvent); 801 tmrEventDestroy(pTmr, pGpu->pFecsTimerEvent); 802 pGpu->pFecsTimerEvent = NULL; 803 } 804 } 805 806 /** 807 * @brief The callback function that transfers FECS Buffer entries to an EventBuffer 808 */ 809 void 810 nvEventBufferFecsCallback 811 ( 812 OBJGPU *pGpu, 813 void *pArgs 814 ) 815 { 816 KernelGraphics *pKernelGraphics = (KernelGraphics*)pArgs; 817 NvU32 fecsReadOffset; 818 NvU32 fecsReadOffsetPrev; 819 NvU64 fecsBufferSize; 820 NvU32 fecsRecordSize; 821 NvU32 i, j; 822 NvU8 *pFecsBufferMapping; 823 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 824 FECS_EVENT_RECORD *pPeekRecord; 825 NvU16 maxFecsRecordsPerIntr; 826 NV_STATUS status; 827 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 828 NvU8 numIterations = (pArgs == NULL) 829 ? KGRMGR_MAX_GR 830 : 1; 831 832 NV_ASSERT_OR_RETURN_VOID(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 833 834 NV_ASSERT_OR_RETURN_VOID(pGpu->fecsCtxswLogConsumerCount >= 0); 835 if (pGpu->fecsCtxswLogConsumerCount <= 0) 836 return; 837 838 for (j = 0; j < numIterations; j++) 839 { 840 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo; 841 842 if (pArgs == NULL) 843 { 844 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, j); 845 if (pKernelGraphics == NULL) 846 continue; 847 } 848 849 pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 850 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 851 852 pFecsBufferMapping = pFecsTraceInfo->pFecsBufferMapping; 853 maxFecsRecordsPerIntr = pFecsTraceInfo->fecsCtxswLogRecordsPerIntr; 854 855 if (pFecsBufferMapping == NULL) 856 continue; 857 858 status = _getFecsEventListParameters(pGpu, pKernelGraphics, &pFecsMemDesc, &fecsRecordSize); 859 if ((status != NV_OK) || (pFecsMemDesc == NULL)) 860 continue; 861 862 fecsBufferSize = memdescGetSize(pFecsMemDesc) / fecsRecordSize; 863 NV_ASSERT_OR_RETURN_VOID(fecsBufferSize > 0); 864 fecsReadOffset = pFecsTraceInfo->fecsTraceRdOffset; 865 866 if (!osIsRaisedIRQL()) 867 maxFecsRecordsPerIntr = fecsBufferSize; 868 869 // Bail out if the buffer has not changed 870 pPeekRecord = (FECS_EVENT_RECORD*)(pFecsBufferMapping + 871 (fecsReadOffset * fecsRecordSize)); 872 873 if (pPeekRecord->magic_lo == NV_FECS_TRACE_MAGIC_INVALIDATED) 874 continue; 875 876 // Get the read offset from hw if the buffer wrapped around 877 fecsReadOffsetPrev = (fecsReadOffset - 1) % fecsBufferSize; 878 pPeekRecord = (FECS_EVENT_RECORD*)(pFecsBufferMapping + 879 (fecsReadOffsetPrev * fecsRecordSize)); 880 881 if (pPeekRecord->magic_lo != NV_FECS_TRACE_MAGIC_INVALIDATED) 882 { 883 NvHandle hClient; 884 NvHandle hSubdevice; 885 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET_PARAMS params; 886 887 NV_PRINTF(LEVEL_ERROR, "FECS buffer overflow detected\n"); 888 889 NV_ASSERT_OK_OR_ELSE( 890 status, 891 _fecsLoadInternalRoutingInfo(pGpu, 892 pKernelGraphics, 893 &hClient, 894 &hSubdevice, 895 ¶ms.grRouteInfo), 896 return); 897 898 NV_ASSERT_OK_OR_ELSE( 899 status, 900 pRmApi->Control(pRmApi, 901 hClient, 902 hSubdevice, 903 NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET, 904 ¶ms, 905 sizeof(params)), 906 return); 907 fecsReadOffset = params.offset; 908 pFecsTraceInfo->fecsTraceCounter = 0; 909 } 910 911 // 912 // Over here we want to go through all EVENTNOTIFICATION nodes and 913 // loop through them in lockstep with the FECS_EVENT_RECORD records 914 // 915 for (i = 0; i < maxFecsRecordsPerIntr; ++i) 916 { 917 FECS_EVENT_RECORD *pCurrRecord = (FECS_EVENT_RECORD *)(pFecsBufferMapping + 918 (fecsReadOffset * fecsRecordSize)); 919 920 if (pCurrRecord->magic_lo == NV_FECS_TRACE_MAGIC_INVALIDATED) 921 break; 922 923 pCurrRecord->magic_lo = NV_FECS_TRACE_MAGIC_INVALIDATED; 924 osFlushCpuWriteCombineBuffer(); 925 926 // Loop through all bound event buffers and copy filtered data to user buffers 927 formatAndNotifyFecsRecord(pGpu, pKernelGraphics, pCurrRecord); 928 929 // Update read reg 930 pFecsTraceInfo->fecsTraceCounter++; 931 fecsReadOffset++; 932 if (fecsReadOffset >= fecsBufferSize) 933 { 934 fecsReadOffset = 0; 935 } 936 } 937 938 if (pFecsTraceInfo->fecsTraceCounter > 0) 939 { 940 NvHandle hClient; 941 NvHandle hSubdevice; 942 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS params; 943 944 params.offset = fecsReadOffset; 945 NV_ASSERT_OK_OR_ELSE( 946 status, 947 _fecsLoadInternalRoutingInfo(pGpu, 948 pKernelGraphics, 949 &hClient, 950 &hSubdevice, 951 ¶ms.grRouteInfo), 952 return); 953 954 NV_ASSERT_OK_OR_ELSE( 955 status, 956 pRmApi->Control(pRmApi, 957 hClient, 958 hSubdevice, 959 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET, 960 ¶ms, 961 sizeof(params)), 962 return); 963 pFecsTraceInfo->fecsTraceCounter = 0; 964 } 965 pFecsTraceInfo->fecsTraceRdOffset = fecsReadOffset; 966 967 // Re-arm interrupt if there may be more records 968 if (i == maxFecsRecordsPerIntr) 969 fecsSignalIntrPendingIfNotPending(pGpu, pKernelGraphics); 970 } 971 } 972 973 NV_STATUS 974 fecsAddBindpoint 975 ( 976 OBJGPU *pGpu, 977 RmClient *pClient, 978 RsResourceRef *pEventBufferRef, 979 Subdevice *pNotifier, 980 NvBool bAllUsers, 981 NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD levelOfDetail, 982 NvU32 eventFilter, 983 NvU8 version, 984 NvU32 *pReasonCode 985 ) 986 { 987 NV_STATUS status; 988 NvHandle hClient = staticCast(pClient, RsClient)->hClient; 989 NvHandle hEventBuffer = pEventBufferRef->hResource; 990 NvHandle hNotifier = RES_GET_HANDLE(pNotifier); 991 EventBuffer *pEventBuffer; 992 NvBool bAdmin = osIsAdministrator(); 993 NvU32 eventMask = 0; 994 NvU64 targetUser; 995 NvS32 gpuConsumerCount = pGpu->fecsCtxswLogConsumerCount; 996 NvBool bFecsBindingActive = (pGpu->fecsCtxswLogConsumerCount > 0); 997 NvBool bIntrDriven = NV_FALSE; 998 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 999 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 1000 NvU8 numIterations; 1001 NvU8 grIdx; 1002 NvBool bKernel; 1003 NvBool bSelectLOD; 1004 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind; 1005 1006 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 1007 NV_ASSERT_OR_RETURN(pCallContext != NULL, NV_ERR_INVALID_STATE); 1008 1009 bKernel = pCallContext->secInfo.privLevel >= RS_PRIV_LEVEL_KERNEL; 1010 1011 bSelectLOD = bKernel; 1012 1013 #if defined(DEBUG) || defined(DEVELOP) || defined(NV_VERIF_FEATURES) 1014 bSelectLOD = NV_TRUE; 1015 #endif 1016 1017 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 1018 1019 // On a hypervisor or VM: bail-out early if admin is required 1020 if (IS_VIRTUAL(pGpu) || hypervisorIsVgxHyper()) 1021 { 1022 if (pGpu->bRmProfilingPrivileged && !bAdmin) 1023 { 1024 if (pReasonCode != NULL) 1025 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NEED_ADMIN; 1026 1027 return NV_ERR_NOT_SUPPORTED; 1028 } 1029 } 1030 1031 if (bSelectLOD) 1032 { 1033 switch (levelOfDetail) 1034 { 1035 case NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_FULL: 1036 eventMask = ~0; 1037 break; 1038 case NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_COMPAT: 1039 eventMask |= NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_RESTORE_START | 1040 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_CONTEXT_START | 1041 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_CTXSW_REQ_BY_HOST; 1042 break; 1043 case NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_CUSTOM: 1044 eventMask = eventFilter; 1045 break; 1046 default: 1047 // Default to SIMPLIFIED level-of-detail 1048 eventMask |= NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_START | 1049 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_END; 1050 } 1051 } 1052 else 1053 { 1054 // Default to SIMPLIFIED level-of-detail 1055 eventMask |= NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_START | 1056 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_END; 1057 } 1058 1059 if (eventMask & NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_FE_ACK) 1060 { 1061 eventMask |= NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_WFI | 1062 NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_GFXP | 1063 NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CTAP | 1064 NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CILP; 1065 } 1066 1067 if (bAllUsers) 1068 { 1069 targetUser = 0; 1070 } 1071 else 1072 { 1073 targetUser = (NvU64)(NvUPtr)pClient->pUserInfo; 1074 1075 // Filtering UIDs is not yet implemented in legacy vGPU 1076 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1077 { 1078 if (pReasonCode != NULL) 1079 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED; 1080 1081 return NV_ERR_NOT_SUPPORTED; 1082 } 1083 } 1084 1085 pEventBuffer = dynamicCast(pEventBufferRef->pResource, EventBuffer); 1086 if (NULL == pEventBuffer) 1087 return NV_ERR_INVALID_ARGUMENT; 1088 1089 if (NULL == multimapFindSubmap(&pGpu->fecsEventBufferBindingsUid, targetUser)) 1090 { 1091 if (NULL == multimapInsertSubmap(&pGpu->fecsEventBufferBindingsUid, targetUser)) 1092 { 1093 NV_PRINTF(LEVEL_ERROR, "failed to add UID binding!\n"); 1094 return NV_ERR_INSUFFICIENT_RESOURCES; 1095 } 1096 } 1097 1098 // If the binding exists already, we're done 1099 if (NULL != multimapFindItem(&pGpu->fecsEventBufferBindingsUid, targetUser, (NvU64)(NvUPtr)pEventBuffer)) 1100 return NV_OK; 1101 1102 pBind = multimapInsertItemNew(&pGpu->fecsEventBufferBindingsUid, targetUser, (NvU64)(NvUPtr)pEventBuffer); 1103 if (pBind == NULL) 1104 return NV_ERR_INVALID_ARGUMENT; 1105 ++pGpu->fecsCtxswLogConsumerCount; 1106 1107 pBind->hClient = hClient; 1108 pBind->hNotifier = hNotifier; 1109 pBind->hEventBuffer = hEventBuffer; 1110 pBind->pEventBuffer = pEventBuffer; 1111 pBind->pUserInfo = (NvU64)(NvUPtr)pClient->pUserInfo; 1112 pBind->bAdmin = bAdmin; 1113 pBind->eventMask = eventMask; 1114 pBind->bKernel = bKernel; 1115 pBind->version = version; 1116 1117 status = registerEventNotification(&pEventBuffer->pListeners, 1118 staticCast(pClient, RsClient), 1119 hNotifier, 1120 hEventBuffer, 1121 (version == 2 ? 1122 NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH_V2 : 1123 NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH) | NV01_EVENT_WITHOUT_EVENT_DATA, 1124 NV_EVENT_BUFFER_BIND, 1125 pEventBuffer->producerInfo.notificationHandle, 1126 NV_FALSE); 1127 if (status != NV_OK) 1128 goto done; 1129 1130 if (bMIGInUse) 1131 { 1132 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, GPU_RES_GET_DEVICE(pNotifier))) 1133 { 1134 pBind->swizzId = NV2080_CTRL_GPU_PARTITION_ID_INVALID; 1135 } 1136 else 1137 { 1138 GPUInstanceSubscription *pGPUInstanceSubscription; 1139 status = gisubscriptionGetGPUInstanceSubscription( 1140 staticCast(pClient, RsClient), hNotifier, &pGPUInstanceSubscription); 1141 if (status != NV_OK) 1142 goto done; 1143 1144 if (gisubscriptionGetMIGGPUInstance(pGPUInstanceSubscription) == NULL) 1145 { 1146 if (pReasonCode != NULL) 1147 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED; 1148 1149 status = NV_ERR_NOT_SUPPORTED; 1150 goto done; 1151 } 1152 1153 pBind->swizzId = gisubscriptionGetMIGGPUInstance(pGPUInstanceSubscription)->swizzId; 1154 } 1155 } 1156 1157 numIterations = bMIGInUse ? GPU_MAX_GRS: 1; 1158 for (grIdx = 0; grIdx < numIterations; grIdx++) 1159 { 1160 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 1161 if (pKernelGraphics == NULL) 1162 continue; 1163 1164 if (!kgraphicsIsCtxswLoggingSupported(pGpu, pKernelGraphics)) 1165 { 1166 if (pReasonCode) 1167 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED_GPU; 1168 1169 status = NV_ERR_NOT_SUPPORTED; 1170 goto done; 1171 } 1172 1173 if (!bFecsBindingActive) 1174 { 1175 1176 fecsBufferReset(pGpu, pKernelGraphics); 1177 } 1178 1179 bIntrDriven |= kgraphicsIsIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics); 1180 } 1181 1182 if (!bFecsBindingActive && !bIntrDriven) 1183 { 1184 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, _fecsTimerCreate(pGpu), done); 1185 } 1186 1187 done: 1188 if (status != NV_OK) 1189 { 1190 if (gpuConsumerCount != pGpu->fecsCtxswLogConsumerCount) 1191 fecsRemoveBindpoint(pGpu, targetUser, pBind); 1192 1193 _fecsTimerDestroy(pGpu); 1194 } 1195 1196 return status; 1197 } 1198 1199 void 1200 fecsRemoveBindpoint 1201 ( 1202 OBJGPU *pGpu, 1203 NvU64 uid, 1204 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind 1205 ) 1206 { 1207 EventBuffer *pEventBuffer = pBind->pEventBuffer; 1208 1209 --pGpu->fecsCtxswLogConsumerCount; 1210 1211 unregisterEventNotificationWithData(&pEventBuffer->pListeners, 1212 pBind->hClient, 1213 pBind->hNotifier, 1214 pBind->hEventBuffer, 1215 NV_TRUE, 1216 pEventBuffer->producerInfo.notificationHandle); 1217 1218 multimapRemoveItemByKey(&pGpu->fecsEventBufferBindingsUid, 1219 uid, 1220 (NvU64)(NvUPtr)pEventBuffer); 1221 1222 if (pGpu->fecsCtxswLogConsumerCount == 0) 1223 { 1224 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 1225 NvU8 grIdx; 1226 NvBool bIntrDriven = NV_FALSE; 1227 1228 NvU8 numIterations = bMIGInUse ? GPU_MAX_GRS : 1; 1229 for (grIdx = 0; grIdx < numIterations; grIdx++) 1230 { 1231 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 1232 if (pKernelGraphics == NULL) 1233 continue; 1234 1235 // 1236 // Disable HW without unmapping buffer so that new event buffers still work properly 1237 // HW enable will happen on bindpoint creation. 1238 // Mapping only occurs on Graphics load/alloc, so unmapping should only occur when Graphics is destroyed. 1239 // 1240 fecsBufferDisableHw(pGpu, pKernelGraphics); 1241 bIntrDriven |= kgraphicsIsIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics); 1242 1243 } 1244 1245 if (!bIntrDriven) 1246 { 1247 _fecsTimerDestroy(pGpu); 1248 } 1249 } 1250 } 1251 1252 void 1253 fecsRemoveAllBindpoints 1254 ( 1255 EventBuffer *pEventBuffer 1256 ) 1257 { 1258 OBJGPU *pGpu = NULL; 1259 NvU32 gpuMask = 0; 1260 NvU32 gpuIndex = 0; 1261 FecsEventBufferBindMultiMapSupermapIter uidBindIter; 1262 1263 eventBufferSetEnable(&pEventBuffer->producerInfo, NV_FALSE); 1264 1265 gpumgrGetGpuAttachInfo(NULL, &gpuMask); 1266 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex)) != NULL) 1267 { 1268 uidBindIter = multimapSubmapIterAll(&pGpu->fecsEventBufferBindingsUid); 1269 while (multimapSubmapIterNext(&uidBindIter)) 1270 { 1271 FecsEventBufferBindMultiMapSubmap *pSubmap = uidBindIter.pValue; 1272 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind = NULL; 1273 NvU64 uid = mapKey_IMPL(uidBindIter.iter.pMap, pSubmap); 1274 1275 while ((pBind = multimapFindItem(&pGpu->fecsEventBufferBindingsUid, 1276 uid, 1277 (NvU64)(NvUPtr)pEventBuffer)) != NULL) 1278 { 1279 fecsRemoveBindpoint(pGpu, uid, pBind); 1280 } 1281 } 1282 1283 } 1284 } 1285 1286 void 1287 fecsBufferReset 1288 ( 1289 OBJGPU *pGpu, 1290 KernelGraphics *pKernelGraphics 1291 ) 1292 { 1293 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 1294 NV_STATUS status; 1295 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1296 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS getHwEnableParams; 1297 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 1298 NvHandle hClient; 1299 NvHandle hSubdevice; 1300 1301 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 1302 1303 if (pFecsTraceInfo->pFecsBufferMapping == NULL) 1304 return; 1305 1306 NV_ASSERT_OK_OR_ELSE( 1307 status, 1308 _fecsLoadInternalRoutingInfo(pGpu, 1309 pKernelGraphics, 1310 &hClient, 1311 &hSubdevice, 1312 &getHwEnableParams.grRouteInfo), 1313 return); 1314 1315 NV_ASSERT_OK_OR_ELSE( 1316 status, 1317 pRmApi->Control(pRmApi, 1318 hClient, 1319 hSubdevice, 1320 NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE, 1321 &getHwEnableParams, 1322 sizeof(getHwEnableParams)), 1323 return); 1324 1325 status = _getFecsMemDesc(pGpu, pKernelGraphics, &pFecsMemDesc); 1326 1327 if ((status == NV_OK) && (pFecsMemDesc != NULL) && (getHwEnableParams.bEnable != NV_TRUE)) 1328 { 1329 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET_PARAMS traceWrOffsetParams; 1330 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS traceRdOffsetParams; 1331 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS setHwEnableParams; 1332 1333 portMemSet(pFecsTraceInfo->pFecsBufferMapping, 1334 (NvU8)(NV_FECS_TRACE_MAGIC_INVALIDATED & 0xff), 1335 memdescGetSize(pFecsMemDesc)); 1336 1337 pFecsTraceInfo->fecsLastSeqno = 0; 1338 1339 // Routing info is the same for all future calls in this series 1340 traceWrOffsetParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1341 traceRdOffsetParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1342 setHwEnableParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1343 1344 traceWrOffsetParams.offset = 0; 1345 NV_ASSERT_OK_OR_ELSE( 1346 status, 1347 pRmApi->Control(pRmApi, 1348 hClient, 1349 hSubdevice, 1350 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET, 1351 &traceWrOffsetParams, 1352 sizeof(traceWrOffsetParams)), 1353 return); 1354 1355 traceRdOffsetParams.offset = 0; 1356 NV_ASSERT_OK_OR_ELSE( 1357 status, 1358 pRmApi->Control(pRmApi, 1359 hClient, 1360 hSubdevice, 1361 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET, 1362 &traceRdOffsetParams, 1363 sizeof(traceRdOffsetParams)), 1364 return); 1365 pFecsTraceInfo->fecsTraceRdOffset = 0; 1366 1367 setHwEnableParams.bEnable = NV_TRUE; 1368 NV_ASSERT_OK_OR_ELSE( 1369 status, 1370 pRmApi->Control(pRmApi, 1371 hClient, 1372 hSubdevice, 1373 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE, 1374 &setHwEnableParams, 1375 sizeof(setHwEnableParams)), 1376 return); 1377 } 1378 } 1379 1380 void 1381 fecsBufferDisableHw 1382 ( 1383 OBJGPU *pGpu, 1384 KernelGraphics *pKernelGraphics 1385 ) 1386 { 1387 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 1388 NvHandle hClient; 1389 NvHandle hSubdevice; 1390 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS getHwEnableParams; 1391 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS setHwEnableParams; 1392 NV_STATUS status; 1393 1394 // This function may be called with unused GR Engines 1395 NV_CHECK_OK_OR_ELSE( 1396 status, 1397 LEVEL_INFO, 1398 _fecsLoadInternalRoutingInfo(pGpu, 1399 pKernelGraphics, 1400 &hClient, 1401 &hSubdevice, 1402 &getHwEnableParams.grRouteInfo), 1403 return); 1404 1405 NV_ASSERT_OK_OR_ELSE( 1406 status, 1407 pRmApi->Control(pRmApi, 1408 hClient, 1409 hSubdevice, 1410 NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE, 1411 &getHwEnableParams, 1412 sizeof(getHwEnableParams)), 1413 return); 1414 1415 if (getHwEnableParams.bEnable) 1416 { 1417 // Copy previously loaded routing info 1418 setHwEnableParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1419 setHwEnableParams.bEnable = NV_FALSE; 1420 1421 NV_ASSERT_OK_OR_ELSE( 1422 status, 1423 pRmApi->Control(pRmApi, 1424 hClient, 1425 hSubdevice, 1426 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE, 1427 &setHwEnableParams, 1428 sizeof(setHwEnableParams)), 1429 return); 1430 } 1431 } 1432 1433 void 1434 fecsBufferTeardown 1435 ( 1436 OBJGPU *pGpu, 1437 KernelGraphics *pKernelGraphics 1438 ) 1439 { 1440 fecsBufferDisableHw(pGpu, pKernelGraphics); 1441 fecsBufferUnmap(pGpu, pKernelGraphics); 1442 } 1443 1444 /*! Is the FECS trace buffer mapped? */ 1445 NvBool 1446 fecsBufferIsMapped 1447 ( 1448 OBJGPU *pGpu, 1449 KernelGraphics *pKernelGraphics 1450 ) 1451 { 1452 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1453 1454 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1455 return pFecsTraceInfo->pFecsBufferMapping != NULL; 1456 } 1457 1458 NV_STATUS 1459 fecsBufferMap 1460 ( 1461 OBJGPU *pGpu, 1462 KernelGraphics *pKernelGraphics 1463 ) 1464 { 1465 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 1466 NvU8 *pFecsBufferMapping = NULL; 1467 NV_STATUS status; 1468 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1469 1470 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_ERR_INVALID_STATE); 1471 1472 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1473 return NV_OK; 1474 1475 if (pFecsTraceInfo->pFecsBufferMapping != NULL) 1476 return NV_OK; 1477 1478 status = _getFecsMemDesc(pGpu, pKernelGraphics, &pFecsMemDesc); 1479 if ((status != NV_OK) || (pFecsMemDesc == NULL)) 1480 return NV_ERR_INVALID_STATE; 1481 1482 pFecsBufferMapping = kbusMapRmAperture_HAL(pGpu, pFecsMemDesc); 1483 if (pFecsBufferMapping == NULL) 1484 return NV_ERR_INSUFFICIENT_RESOURCES; 1485 1486 pFecsTraceInfo->pFecsBufferMapping = pFecsBufferMapping; 1487 1488 return NV_OK; 1489 } 1490 1491 void 1492 fecsBufferUnmap 1493 ( 1494 OBJGPU *pGpu, 1495 KernelGraphics *pKernelGraphics 1496 ) 1497 { 1498 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 1499 NV_STATUS status; 1500 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1501 1502 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 1503 1504 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1505 return; 1506 1507 status = _getFecsMemDesc(pGpu, pKernelGraphics, &pFecsMemDesc); 1508 if ((status == NV_OK) && (pFecsMemDesc != NULL) && (pFecsTraceInfo->pFecsBufferMapping != NULL)) 1509 kbusUnmapRmAperture_HAL(pGpu, pFecsMemDesc, 1510 &pFecsTraceInfo->pFecsBufferMapping, 1511 NV_TRUE); 1512 } 1513 1514 /*! Atomically set intr callback pending, return NV_TRUE if wasn't pending prior */ 1515 NvBool 1516 fecsSignalIntrPendingIfNotPending 1517 ( 1518 OBJGPU *pGpu, 1519 KernelGraphics *pKernelGraphics 1520 ) 1521 { 1522 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1523 1524 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1525 1526 return portAtomicCompareAndSwapU32(&pFecsTraceInfo->fecsCtxswLogIntrPending, 1, 0); 1527 } 1528 1529 /*! Atomically clear intr callback pending, return NV_TRUE if was pending */ 1530 NvBool 1531 fecsClearIntrPendingIfPending 1532 ( 1533 OBJGPU *pGpu, 1534 KernelGraphics *pKernelGraphics 1535 ) 1536 { 1537 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1538 1539 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1540 1541 return portAtomicCompareAndSwapU32(&pFecsTraceInfo->fecsCtxswLogIntrPending, 0, 1); 1542 } 1543 1544 /*! Atomically check if intr callback pending */ 1545 NvBool fecsIsIntrPending 1546 ( 1547 OBJGPU *pGpu, 1548 KernelGraphics *pKernelGraphics 1549 ) 1550 { 1551 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1552 1553 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1554 1555 return portAtomicOrU32(&pFecsTraceInfo->fecsCtxswLogIntrPending, 0) != 0; 1556 } 1557 1558 /*! Retrieve the current VGPU staging buffer */ 1559 VGPU_FECS_TRACE_STAGING_BUFFER * 1560 fecsGetVgpuStagingBuffer 1561 ( 1562 OBJGPU *pGpu, 1563 KernelGraphics *pKernelGraphics 1564 ) 1565 { 1566 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1567 1568 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NULL); 1569 1570 return pFecsTraceInfo->pVgpuStaging; 1571 } 1572 1573 /*! Store the given VGPU staging buffer */ 1574 void 1575 fecsSetVgpuStagingBuffer 1576 ( 1577 OBJGPU *pGpu, 1578 KernelGraphics *pKernelGraphics, 1579 VGPU_FECS_TRACE_STAGING_BUFFER *pStagingBuffer 1580 ) 1581 { 1582 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1583 1584 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 1585 1586 pFecsTraceInfo->pVgpuStaging = pStagingBuffer; 1587 } 1588 1589