1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2018-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 /***************************************************************************\ 25 * * 26 * Module: fecs_event_list.c * 27 * Description: * 28 * This module contains an implementation of the Event Buffer * 29 * callback for FECS events * 30 * * 31 \***************************************************************************/ 32 33 #include "kernel/gpu/gr/kernel_graphics.h" 34 #include "kernel/rmapi/event.h" 35 #include "kernel/rmapi/event_buffer.h" 36 #include "libraries/resserv/rs_server.h" 37 #include "kernel/core/locks.h" 38 #include "kernel/os/os.h" 39 #include "kernel/gpu/gr/fecs_event_list.h" 40 #include "kernel/gpu/mig_mgr/gpu_instance_subscription.h" 41 #include "kernel/gpu/mig_mgr/kernel_mig_manager.h" 42 #include "kernel/gpu/bus/kern_bus.h" 43 #include "kernel/gpu/mem_mgr/mem_mgr.h" 44 #include "kernel/gpu/fifo/kernel_channel.h" 45 #include "kernel/gpu/subdevice/subdevice.h" 46 #include "kernel/virtualization/hypervisor/hypervisor.h" 47 #include "rmapi/client.h" 48 #include "objtmr.h" 49 #include "vgpu/sdk-structures.h" 50 51 #include "class/cl90cdtypes.h" 52 #include "ctrl/ctrl90cd.h" 53 54 #define NV_FECS_TRACE_MAX_TIMESTAMPS 5 55 #define NV_FECS_TRACE_MAGIC_INVALIDATED 0xdededede // magic number for entries that have been read 56 #define NV_FECS_TRACE_CALLBACK_TIME_NS 33333333 // Approximating 30Hz callback 57 58 typedef struct 59 { 60 NvU32 magic_lo; 61 NvU32 magic_hi; 62 NvU32 context_id; 63 NvU32 context_ptr; 64 NvU32 new_context_id; 65 NvU32 new_context_ptr; 66 NvU64 ts[NV_FECS_TRACE_MAX_TIMESTAMPS]; 67 NvU32 reserved[13]; 68 NvU32 seqno; 69 } FECS_EVENT_RECORD; 70 71 /*! Opaque pointer to private data */ 72 typedef struct VGPU_FECS_TRACE_STAGING_BUFFER VGPU_FECS_TRACE_STAGING_BUFFER; 73 74 /*! Private FECS event buffer data stored per-KGR */ 75 struct KGRAPHICS_FECS_TRACE_INFO 76 { 77 NvU8 *pFecsBufferMapping; 78 NvU16 fecsCtxswLogRecordsPerIntr; 79 NvU16 fecsTraceRdOffset; 80 NvU16 fecsTraceCounter; 81 NvU32 fecsCtxswLogIntrPending; 82 NvU32 fecsLastSeqno; 83 84 #if PORT_IS_MODULE_SUPPORTED(crypto) 85 PORT_CRYPTO_PRNG *pFecsLogPrng; 86 #endif 87 88 // 89 // GR Routing information for GPU instance to which this engine is assigned if MIG is enabled. 90 // Will be 0/NULL for unassigned GR engines or if MIG is disabled 91 // 92 NvHandle hClient; 93 NvHandle hSubdevice; 94 NvU32 localGrEngineIdx; 95 96 // vGPU FECS staging eventbuffer (guest only) 97 VGPU_FECS_TRACE_STAGING_BUFFER *pVgpuStaging; 98 }; 99 100 /*! Private FECS event buffer data stored for the GPU as a whole */ 101 struct KGRMGR_FECS_GLOBAL_TRACE_INFO 102 { 103 // map: { UserInfo* -> { pEventBuffer -> NV_EVENT_BUFFER_BIND_POINT* }} 104 FecsEventBufferBindMultiMap fecsEventBufferBindingsUid; 105 106 // Timer event to periodically the processing callback 107 TMR_EVENT *pFecsTimerEvent; 108 109 // Timer interval in nanoseconds 110 NvU32 fecsTimerInterval; 111 112 // Atomic for scheduling the fecs callback in timer mode 113 NvU32 fecsCallbackScheduled; 114 115 // Number of consumer clients 116 NvS16 fecsCtxswLogConsumerCount; 117 }; 118 119 /*! 120 * @brief Function to populate client/subdevice/grRouteInfo from cached 121 * information in order to make calls into the specific MIG GPU instance 122 * to which a GR engine is assigned. If MIG is not enabled, GPU 123 * internal client/subdevice handles will be used and grRouteInfo is 124 * cleared 125 * 126 * @param[in] pGpu 127 * @param[in] pKernelGraphics 128 * @param[out] phClient Client handle to populate 129 * @param[out] phSubdevice Subdevice handle to populate 130 * @param[out] pGrRouteInfo Internal GR Routing info to populate 131 */ 132 static NV_STATUS 133 _fecsLoadInternalRoutingInfo 134 ( 135 OBJGPU *pGpu, 136 KernelGraphics *pKernelGraphics, 137 NvHandle *phClient, 138 NvHandle *phSubdevice, 139 NV2080_CTRL_GR_ROUTE_INFO *pGrRouteInfo 140 ) 141 { 142 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 143 144 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_ERR_INVALID_STATE); 145 146 portMemSet(pGrRouteInfo, 0, sizeof(*pGrRouteInfo)); 147 148 if (!IS_MIG_IN_USE(pGpu)) 149 { 150 *phClient = pGpu->hInternalClient; 151 *phSubdevice = pGpu->hInternalSubdevice; 152 return NV_OK; 153 } 154 155 // GR Engines not allocated to any GPU instance will have null handles 156 NV_CHECK_OR_RETURN(LEVEL_INFO, pFecsTraceInfo->hClient != NV01_NULL_OBJECT, NV_ERR_INVALID_ARGUMENT); 157 158 kgrmgrCtrlSetEngineID(pFecsTraceInfo->localGrEngineIdx, pGrRouteInfo); 159 *phClient = pFecsTraceInfo->hClient; 160 *phSubdevice = pFecsTraceInfo->hSubdevice; 161 162 return NV_OK; 163 } 164 165 static NV_STATUS 166 fecsExtractTagAndTimestamp 167 ( 168 OBJGPU *pGpu, 169 KernelGraphics *pKernelGraphics, 170 NvU64 rawTimestamp, 171 NvU64 *pTimestampVal, 172 NvU8 *pTag 173 ) 174 { 175 const KGRAPHICS_STATIC_INFO *pKernelGraphicsStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 176 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo != NULL, NV_ERR_INVALID_STATE); 177 NV_ASSERT_OR_RETURN(pKernelGraphicsStaticInfo->pFecsTraceDefines != NULL, NV_ERR_INVALID_STATE); 178 179 *pTag = ((NvU64_HI32(rawTimestamp)) >> pKernelGraphicsStaticInfo->pFecsTraceDefines->timestampHiTagShift) & pKernelGraphicsStaticInfo->pFecsTraceDefines->timestampHiTagMask; 180 *pTimestampVal = rawTimestamp & pKernelGraphicsStaticInfo->pFecsTraceDefines->timestampVMask; 181 182 // timestamp encoded as right shifted N bits, since they hold zeros. RM needs to reverse that here. 183 *pTimestampVal <<= pKernelGraphicsStaticInfo->pFecsTraceDefines->numLowerBitsZeroShift; 184 return NV_OK; 185 } 186 187 // 188 // The function formats the information from the FECS Buffer into a format 189 // suitable for EventBuffer, and then checks to see whether the subscriber 190 // needs to be notified. If so, the subscriber is notified. 191 // 192 // pGpu is used to retrieve data on the pid, and 193 // seqno provides the sequence number for the user to keep track of 194 // whether any entry has been dropped. 195 // pRecord is the current FECS entry. 196 // 197 static void 198 formatAndNotifyFecsRecord 199 ( 200 OBJGPU *pGpu, 201 KernelGraphics *pKernelGraphics, 202 FECS_EVENT_RECORD *pRecord 203 ) 204 { 205 FECS_EVENT_NOTIFICATION_DATA notifRecord; 206 KernelFifo *pKernelFifo = GPU_GET_KERNEL_FIFO(pGpu); 207 KernelChannel *pKernelChannel = NULL; 208 KernelChannel *pKernelChannelNew = NULL; 209 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 210 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 211 MIG_INSTANCE_REF *pChannelRef; 212 MIG_INSTANCE_REF *pNewChannelRef; 213 INST_BLOCK_DESC inst; 214 NvU32 timestampId; 215 NvU64 noisyTimestampStart = 0; 216 NvU64 noisyTimestampRange = 0; 217 NvU32 instSize; 218 NvU32 instShift; 219 NV_STATUS status; 220 221 if (pRecord == NULL) 222 { 223 NV_PRINTF(LEVEL_ERROR, "Invalid FECS record!\n"); 224 DBG_BREAKPOINT(); 225 return; 226 } 227 228 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 229 230 kfifoGetInstBlkSizeAlign_HAL(pKernelFifo, &instSize, &instShift); 231 232 portMemSet(¬ifRecord, 0, sizeof(notifRecord)); 233 234 inst.address = ((NvU64)pRecord->context_ptr) << instShift; 235 inst.aperture = INST_BLOCK_APERTURE_VIDEO_MEMORY; 236 inst.gfid = GPU_GFID_PF; 237 if (pRecord->context_ptr && 238 (kfifoConvertInstToKernelChannel_HAL(pGpu, pKernelFifo, &inst, &pKernelChannel) != NV_OK)) 239 { 240 NV_PRINTF(LEVEL_INFO, "Error getting channel!\n"); 241 pKernelChannel = NULL; 242 } 243 244 inst.address = ((NvU64)pRecord->new_context_ptr) << instShift; 245 inst.aperture = INST_BLOCK_APERTURE_VIDEO_MEMORY; 246 inst.gfid = GPU_GFID_PF; 247 if (pRecord->new_context_ptr && 248 (kfifoConvertInstToKernelChannel_HAL(pGpu, pKernelFifo, &inst, &pKernelChannelNew) != NV_OK)) 249 { 250 NV_PRINTF(LEVEL_INFO, "Error getting new channel!\n"); 251 pKernelChannelNew = NULL; 252 } 253 254 pChannelRef = (pKernelChannel != NULL) ? kchannelGetMIGReference(pKernelChannel) : NULL; 255 pNewChannelRef = (pKernelChannelNew != NULL) ? kchannelGetMIGReference(pKernelChannelNew) : NULL; 256 257 if (kgraphicsIsFecsRecordUcodeSeqnoSupported(pGpu, pKernelGraphics)) 258 { 259 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 260 261 // Dropped at least 1 event 262 if ((pFecsTraceInfo->fecsLastSeqno + 1) != pRecord->seqno) 263 { 264 notifRecord.dropCount = pRecord->seqno - pFecsTraceInfo->fecsLastSeqno - 1; 265 } 266 267 pFecsTraceInfo->fecsLastSeqno = pRecord->seqno; 268 } 269 270 for (timestampId = 0; timestampId < NV_FECS_TRACE_MAX_TIMESTAMPS; timestampId++) 271 { 272 NV_ASSERT_OK_OR_ELSE(status, 273 fecsExtractTagAndTimestamp(pGpu, pKernelGraphics, 274 pRecord->ts[timestampId], 275 ¬ifRecord.timestamp, 276 ¬ifRecord.tag), 277 return); 278 279 // 280 // determine a few more fields of the current record by subevent type, 281 // before we notify the subscriber 282 // 283 switch (notifRecord.tag) 284 { 285 case NV_EVENT_BUFFER_FECS_CTXSWTAG_RESTORE_START: 286 case NV_EVENT_BUFFER_FECS_CTXSWTAG_CONTEXT_START: 287 if (pKernelChannelNew != NULL) 288 { 289 notifRecord.pid = pKernelChannelNew->ProcessID; 290 notifRecord.subpid = pKernelChannelNew->SubProcessID; 291 notifRecord.userInfo = (NvU64)(NvUPtr)pKernelChannelNew->pUserInfo; 292 notifRecord.context_id = kchannelGetCid(pKernelChannelNew); 293 294 if (kmigmgrIsMIGReferenceValid(pNewChannelRef)) 295 { 296 notifRecord.swizzId = pNewChannelRef->pKernelMIGGpuInstance->swizzId; 297 if (pNewChannelRef->pMIGComputeInstance) 298 notifRecord.computeInstanceId = pNewChannelRef->pMIGComputeInstance->id; 299 else 300 notifRecord.computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 301 } 302 303 if (notifRecord.tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_RESTORE_START) 304 { 305 noisyTimestampStart = notifRecord.timestamp; 306 } 307 else 308 { 309 noisyTimestampRange = notifRecord.timestamp - noisyTimestampStart; 310 } 311 } 312 break; 313 314 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_WFI: 315 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_GFXP: 316 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CTAP: 317 case NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CILP: 318 if (pKernelChannel != NULL) 319 { 320 notifRecord.pid = pKernelChannel->ProcessID; 321 notifRecord.subpid = pKernelChannel->SubProcessID; 322 notifRecord.userInfo = (NvU64)(NvUPtr)pKernelChannel->pUserInfo; 323 notifRecord.context_id = kchannelGetCid(pKernelChannel); 324 325 if (kmigmgrIsMIGReferenceValid(pChannelRef)) 326 { 327 notifRecord.swizzId = pChannelRef->pKernelMIGGpuInstance->swizzId; 328 if (pChannelRef->pMIGComputeInstance) 329 notifRecord.computeInstanceId = pChannelRef->pMIGComputeInstance->id; 330 else 331 notifRecord.computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 332 } 333 } 334 break; 335 336 case NV_EVENT_BUFFER_FECS_CTXSWTAG_CTXSW_REQ_BY_HOST: 337 case NV_EVENT_BUFFER_FECS_CTXSWTAG_SAVE_END: 338 if (pKernelChannel != NULL) 339 { 340 notifRecord.pid = pKernelChannel->ProcessID; 341 notifRecord.subpid = pKernelChannel->SubProcessID; 342 notifRecord.userInfo = (NvU64)(NvUPtr)pKernelChannel->pUserInfo; 343 notifRecord.context_id = kchannelGetCid(pKernelChannel); 344 345 if (kmigmgrIsMIGReferenceValid(pChannelRef)) 346 { 347 notifRecord.swizzId = pChannelRef->pKernelMIGGpuInstance->swizzId; 348 if (pChannelRef->pMIGComputeInstance != NULL) 349 notifRecord.computeInstanceId = pChannelRef->pMIGComputeInstance->id; 350 else 351 notifRecord.computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 352 } 353 354 if (notifRecord.tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_CTXSW_REQ_BY_HOST) 355 { 356 noisyTimestampStart = notifRecord.timestamp; 357 } 358 else 359 { 360 noisyTimestampRange = notifRecord.timestamp - noisyTimestampStart; 361 } 362 } 363 break; 364 365 default: 366 continue; 367 } 368 369 if ((pKernelChannel != NULL) || (pKernelChannelNew != NULL)) 370 { 371 FecsEventBufferBindMultiMapSubmap *pSubmap; 372 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 373 374 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 375 376 notifRecord.noisyTimestamp = 0; 377 if ((noisyTimestampRange > 0) && (pFecsTraceInfo->pFecsLogPrng != NULL)) 378 notifRecord.noisyTimestamp = noisyTimestampStart + portCryptoPseudoRandomGeneratorGetU32(pFecsTraceInfo->pFecsLogPrng) % noisyTimestampRange; 379 380 if (notifRecord.userInfo != 0) 381 { 382 // Notify event buffers listening for the current UID 383 pSubmap = multimapFindSubmap(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, notifRecord.userInfo); 384 notifyEventBuffers(pGpu, pSubmap, ¬ifRecord); 385 } 386 387 // Notify event buffers listening for all UIDs 388 pSubmap = multimapFindSubmap(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, 0); 389 notifyEventBuffers(pGpu, pSubmap, ¬ifRecord); 390 391 // Clear so we don't report drops for every event in this record 392 notifRecord.dropCount = 0; 393 } 394 } 395 } 396 397 static NV_STATUS 398 _fecsEventBufferAdd 399 ( 400 OBJGPU *pGpu, 401 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind, 402 NvU8 tag, 403 NvU32 pid, 404 NvU8 swizzId, 405 NvU8 computeInstanceId, 406 NvU32 context_id, 407 NvU64 timestamp 408 ) 409 { 410 NV_STATUS status; 411 NvBool bNotify; 412 NvP64 notificationHandle; 413 EVENT_BUFFER_PRODUCER_DATA notifyEvent; 414 NvU32 notifyIndex; 415 416 switch (pBind->version) 417 { 418 case 2: 419 notifyIndex = NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH_V2; 420 break; 421 case 1: 422 notifyIndex = NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH; 423 break; 424 default: 425 return NV_ERR_INVALID_ARGUMENT; 426 } 427 428 portMemSet(¬ifyEvent, 0, sizeof(notifyEvent)); 429 notifyEvent.pVardata = NV_PTR_TO_NvP64(NULL); 430 notifyEvent.vardataSize = 0; 431 432 NV_EVENT_BUFFER_FECS_RECORD_V2 fecsRecord; 433 portMemSet(&fecsRecord, 0, sizeof(fecsRecord)); 434 fecsRecord.tag = tag; 435 fecsRecord.pid = pid; 436 if (pBind->version >= 2) 437 { 438 fecsRecord.migGpuInstanceId = swizzId; 439 fecsRecord.migComputeInstanceId = computeInstanceId; 440 } 441 fecsRecord.context_id = context_id; 442 fecsRecord.timestamp = timestamp; 443 fecsRecord.seqno = pBind->pEventBuffer->seqNo++; 444 445 notifyEvent.pPayload = NV_PTR_TO_NvP64(&fecsRecord); 446 notifyEvent.payloadSize = sizeof(fecsRecord); 447 448 status = eventBufferAdd(pBind->pEventBuffer, ¬ifyEvent, notifyIndex, &bNotify, ¬ificationHandle); 449 450 if ((status == NV_OK) && bNotify && notificationHandle) 451 { 452 osEventNotification(pGpu, 453 pBind->pEventBuffer->pListeners, 454 notifyIndex, 455 ¬ifyEvent, 456 0); // Do not copy structure -- embedded pointers. 457 pBind->pEventBuffer->bNotifyPending = NV_TRUE; 458 } 459 460 return status; 461 } 462 463 void 464 notifyEventBuffers 465 ( 466 OBJGPU *pGpu, 467 FecsEventBufferBindMultiMapSubmap *pSubmap, 468 FECS_EVENT_NOTIFICATION_DATA const *pRecord 469 ) 470 { 471 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 472 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 473 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 474 475 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 476 477 if (pSubmap != NULL) 478 { 479 FecsEventBufferBindMultiMapIter iter = multimapSubmapIterItems(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, pSubmap); 480 481 while (multimapItemIterNext(&iter)) 482 { 483 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind = iter.pValue; 484 NvBool bSanitizeKernel = (!pBind->bKernel) && (pRecord->userInfo == 0); 485 NvBool bSanitizeUser = (!pBind->bAdmin) && (pBind->pUserInfo != pRecord->userInfo); 486 NvBool bSanitize = bSanitizeKernel || bSanitizeUser; 487 NvU32 pid; 488 NvU32 context_id; 489 NvU64 timestamp; 490 NvU32 tag = pRecord->tag; 491 NvU8 swizzId = pRecord->swizzId; 492 NvU8 computeInstanceId = pRecord->computeInstanceId; 493 494 pBind->pEventBuffer->seqNo += pRecord->dropCount; 495 496 if (bSanitize || !(NVBIT(pRecord->tag) & pBind->eventMask)) 497 { 498 // 499 // Re-map CONTEXT_START as SIMPLE_START and SAVE_END as SIMPLE_END if 500 // the binding has simple level-of-detail or is being sanitized 501 // 502 if ((bSanitize || (pBind->eventMask & NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_START)) && 503 (tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_CONTEXT_START)) 504 { 505 tag = NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_START; 506 } 507 else if ((bSanitize || (pBind->eventMask & NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_END)) && 508 (tag == NV_EVENT_BUFFER_FECS_CTXSWTAG_SAVE_END)) 509 { 510 tag = NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_END; 511 } 512 else if ((tag != NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_START) && 513 (tag != NV_EVENT_BUFFER_FECS_CTXSWTAG_SIMPLE_END)) 514 { 515 continue; 516 } 517 } 518 519 // 520 // While MIG is enabled, if the bindpoint is registered for a specific MIG GPU instance 521 // then filter out records from other GPU instances 522 // 523 if (bMIGInUse && 524 ((pBind->swizzId != NV2080_CTRL_GPU_PARTITION_ID_INVALID) && 525 (pRecord->swizzId != pBind->swizzId))) 526 continue; 527 528 // While MIG is enabled, pause tracing of V1 bindpoints 529 if (bMIGInUse && (pBind->version < 2)) 530 continue; 531 532 if (bSanitizeKernel) 533 { 534 timestamp = pRecord->noisyTimestamp; 535 pid = NV_EVENT_BUFFER_KERNEL_PID; 536 context_id = NV_EVENT_BUFFER_KERNEL_CONTEXT; 537 swizzId = NV_EVENT_BUFFER_KERNEL_MIG_GI; 538 computeInstanceId = NV_EVENT_BUFFER_KERNEL_MIG_CI; 539 } 540 else if (bSanitizeUser) 541 { 542 timestamp = pRecord->noisyTimestamp; 543 pid = NV_EVENT_BUFFER_HIDDEN_PID; 544 context_id = NV_EVENT_BUFFER_HIDDEN_CONTEXT; 545 swizzId = NV_EVENT_BUFFER_HIDDEN_MIG_GI; 546 computeInstanceId = NV_EVENT_BUFFER_HIDDEN_MIG_CI; 547 } 548 else 549 { 550 timestamp = pRecord->timestamp; 551 pid = pRecord->pid; 552 context_id = pRecord->context_id; 553 } 554 555 _fecsEventBufferAdd(pGpu, pBind, tag, 556 pid, 557 bMIGInUse ? swizzId : NV_EVENT_BUFFER_INVALID_MIG_GI, 558 bMIGInUse ? computeInstanceId : NV_EVENT_BUFFER_INVALID_MIG_CI, 559 context_id, timestamp); 560 } 561 } 562 } 563 564 static NV_STATUS 565 _getFecsMemDesc 566 ( 567 OBJGPU *pGpu, 568 KernelGraphics *pKernelGraphics, 569 MEMORY_DESCRIPTOR **ppFecsMemDesc 570 ) 571 { 572 MEMORY_DESCRIPTOR *pMemDesc; 573 GR_GLOBALCTX_BUFFERS *pGlobalCtxBuffers = kgraphicsGetGlobalCtxBuffers(pGpu, pKernelGraphics, GPU_GFID_PF); 574 575 *ppFecsMemDesc = NULL; 576 NV_CHECK_OR_RETURN(LEVEL_SILENT, pGlobalCtxBuffers != NULL, NV_ERR_INVALID_STATE); 577 pMemDesc = pGlobalCtxBuffers->memDesc[GR_GLOBALCTX_BUFFER_FECS_EVENT]; 578 if (pMemDesc != NULL) 579 pMemDesc = memdescGetMemDescFromGpu(pMemDesc, pGpu); 580 581 *ppFecsMemDesc = pMemDesc; 582 583 return NV_OK; 584 } 585 586 static NV_STATUS 587 _getFecsEventListParameters 588 ( 589 OBJGPU *pGpu, 590 KernelGraphics *pKernelGraphics, 591 MEMORY_DESCRIPTOR **ppFecsMemDesc, 592 NvU32 *pFecsRecordSize 593 ) 594 { 595 const KGRAPHICS_STATIC_INFO *pStaticInfo; 596 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 597 _getFecsMemDesc(pGpu, pKernelGraphics, ppFecsMemDesc)); 598 599 pStaticInfo = kgraphicsGetStaticInfo(pGpu, pKernelGraphics); 600 NV_CHECK_OR_RETURN(LEVEL_ERROR, pStaticInfo != NULL, NV_ERR_INVALID_STATE); 601 NV_ASSERT_OR_RETURN(pStaticInfo->pFecsTraceDefines != NULL, NV_ERR_INVALID_STATE); 602 *pFecsRecordSize = pStaticInfo->pFecsTraceDefines->fecsRecordSize; 603 604 return NV_OK; 605 } 606 607 /*! 608 * @brief Set cached routing info for this KernelGraphics engine to make RPC calls 609 * into the specific MIG GPU instance to which this engine is assigned 610 * 611 * @param[in] pGpu 612 * @param[in] pKernelGraphics 613 * @param[in] hClient Client handle to make calls into MIG GPU instance 614 * @param[in] hSubdevice Subdevice handle to make calls into MIG GPU instance 615 * @param[in] localGrEngineIdx Local GR index for this GR engine in MIG GPU instance 616 */ 617 void fecsSetRoutingInfo 618 ( 619 OBJGPU *pGpu, 620 KernelGraphics *pKernelGraphics, 621 NvHandle hClient, 622 NvHandle hSubdevice, 623 NvU32 localGrEngineIdx 624 ) 625 { 626 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 627 628 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 629 630 pFecsTraceInfo->hClient = hClient; 631 pFecsTraceInfo->hSubdevice = hSubdevice; 632 pFecsTraceInfo->localGrEngineIdx = localGrEngineIdx; 633 } 634 635 // Clear cached routing info used to make GR calls into specific MIG GPU instance 636 void fecsClearRoutingInfo 637 ( 638 OBJGPU *pGpu, 639 KernelGraphics *pKernelGraphics 640 ) 641 { 642 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 643 644 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 645 646 pFecsTraceInfo->hClient = NV01_NULL_OBJECT; 647 pFecsTraceInfo->hSubdevice = NV01_NULL_OBJECT; 648 pFecsTraceInfo->localGrEngineIdx = 0; 649 } 650 651 NV_STATUS 652 fecsCtxswLoggingInit 653 ( 654 OBJGPU *pGpu, 655 KernelGraphics *pKernelGraphics, 656 KGRAPHICS_FECS_TRACE_INFO **ppFecsTraceInfo 657 ) 658 { 659 NvU64 seed; 660 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo; 661 662 NV_ASSERT_OR_RETURN(ppFecsTraceInfo != NULL, NV_ERR_NOT_SUPPORTED); 663 pFecsTraceInfo = portMemAllocNonPaged(sizeof(*pFecsTraceInfo)); 664 if (pFecsTraceInfo == NULL) 665 return NV_ERR_NO_MEMORY; 666 portMemSet(pFecsTraceInfo, 0, sizeof(*pFecsTraceInfo)); 667 668 osGetCurrentTick(&seed); 669 pFecsTraceInfo->pFecsLogPrng = portCryptoPseudoRandomGeneratorCreate(seed); 670 671 *ppFecsTraceInfo = pFecsTraceInfo; 672 673 kgraphicsInitFecsRegistryOverrides_HAL(pGpu, pKernelGraphics); 674 675 return NV_OK; 676 } 677 678 void 679 fecsCtxswLoggingTeardown 680 ( 681 OBJGPU *pGpu, 682 KernelGraphics *pKernelGraphics 683 ) 684 { 685 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 686 687 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 688 689 portCryptoPseudoRandomGeneratorDestroy(pFecsTraceInfo->pFecsLogPrng); 690 pFecsTraceInfo->pFecsLogPrng = NULL; 691 portMemFree(pFecsTraceInfo); 692 } 693 694 NV_STATUS 695 fecsGlobalLoggingInit 696 ( 697 OBJGPU *pGpu, 698 KernelGraphicsManager *pKernelGraphicsManager, 699 KGRMGR_FECS_GLOBAL_TRACE_INFO **ppFecsGlobalTraceInfo 700 ) 701 { 702 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo; 703 704 pFecsGlobalTraceInfo = portMemAllocNonPaged(sizeof(*pFecsGlobalTraceInfo)); 705 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, NV_ERR_NO_MEMORY); 706 portMemSet(pFecsGlobalTraceInfo, 0, sizeof(*pFecsGlobalTraceInfo)); 707 708 multimapInit(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, portMemAllocatorGetGlobalNonPaged()); 709 710 *ppFecsGlobalTraceInfo = pFecsGlobalTraceInfo; 711 712 pFecsGlobalTraceInfo->fecsTimerInterval = NV_FECS_TRACE_CALLBACK_TIME_NS; 713 714 return NV_OK; 715 } 716 717 void 718 fecsGlobalLoggingTeardown 719 ( 720 OBJGPU *pGpu, 721 KernelGraphicsManager *pKernelGraphicsManager 722 ) 723 { 724 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 725 726 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 727 728 multimapDestroy(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid); 729 730 portMemFree(pFecsGlobalTraceInfo); 731 } 732 733 /*! set num records to process per intr */ 734 void fecsSetRecordsPerIntr 735 ( 736 OBJGPU *pGpu, 737 KernelGraphics *pKernelGraphics, 738 NvU32 recordsPerIntr 739 ) 740 { 741 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 742 743 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 744 pFecsTraceInfo->fecsCtxswLogRecordsPerIntr = recordsPerIntr; 745 } 746 747 NvBool 748 fecsBufferChanged 749 ( 750 OBJGPU *pGpu, 751 KernelGraphics *pKernelGraphics 752 ) 753 { 754 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 755 NvU8 *pFecsBufferMapping; 756 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 757 NvU32 fecsRecordSize; 758 FECS_EVENT_RECORD *pPeekRecord; 759 NV_STATUS status; 760 761 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 762 pFecsBufferMapping = pFecsTraceInfo->pFecsBufferMapping; 763 764 status = _getFecsEventListParameters(pGpu, pKernelGraphics, &pFecsMemDesc, &fecsRecordSize); 765 if ((status != NV_OK) || (pFecsMemDesc == NULL) || (pFecsBufferMapping == NULL)) 766 return NV_FALSE; 767 768 pPeekRecord = (FECS_EVENT_RECORD*)(pFecsBufferMapping + 769 (pFecsTraceInfo->fecsTraceRdOffset * fecsRecordSize)); 770 771 if (pPeekRecord->magic_lo != NV_FECS_TRACE_MAGIC_INVALIDATED) 772 { 773 return NV_TRUE; 774 } 775 776 return NV_FALSE; 777 } 778 779 static void 780 _fecsClearCallbackScheduled 781 ( 782 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo 783 ) 784 { 785 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 786 portAtomicSetU32(&pFecsGlobalTraceInfo->fecsCallbackScheduled, 0); 787 } 788 789 /*! 790 * @brief Atomically set fecs callback scheduled, return NV_TRUE if wasn't scheduled 791 */ 792 static NvBool 793 _fecsSignalCallbackScheduled 794 ( 795 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo 796 ) 797 { 798 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, 0); 799 return portAtomicCompareAndSwapU32(&pFecsGlobalTraceInfo->fecsCallbackScheduled, 1, 0); 800 } 801 802 static void 803 _fecsOsWorkItem 804 ( 805 NvU32 gpuInstance, 806 void *data 807 ) 808 { 809 OBJGPU *pGpu = gpumgrGetGpu(gpuInstance); 810 KernelGraphicsManager *pKernelGraphicsManager; 811 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo; 812 813 NV_CHECK_OR_RETURN_VOID(LEVEL_ERROR, pGpu != NULL); 814 pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 815 pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 816 817 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 818 819 nvEventBufferFecsCallback(pGpu, NULL); 820 _fecsClearCallbackScheduled(pFecsGlobalTraceInfo); 821 } 822 823 static NV_STATUS 824 _fecsTimerCallback 825 ( 826 OBJGPU *pGpu, 827 OBJTMR *pTmr, 828 TMR_EVENT *pTmrEvent 829 ) 830 { 831 NV_STATUS status = NV_OK; 832 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 833 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 834 NvU32 i; 835 836 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, NV_ERR_INVALID_STATE); 837 838 // If any Kgraphics have events, schedule work item 839 for (i = 0; i < GPU_MAX_GRS; i++) 840 { 841 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, i); 842 843 if (pKernelGraphics == NULL) 844 continue; 845 846 if (fecsBufferChanged(pGpu, pKernelGraphics) && _fecsSignalCallbackScheduled(pFecsGlobalTraceInfo)) 847 { 848 NV_CHECK_OK(status, LEVEL_ERROR, osQueueWorkItemWithFlags(pGpu, _fecsOsWorkItem, NULL, OS_QUEUE_WORKITEM_FLAGS_LOCK_GPU_GROUP_DEVICE_RW)); 849 850 if (status != NV_OK) 851 _fecsClearCallbackScheduled(pFecsGlobalTraceInfo); 852 853 break; 854 } 855 } 856 857 // TMR_FLAG_RECUR does not work, so reschedule it here. 858 NV_CHECK_OK_OR_CAPTURE_FIRST_ERROR(status, 859 LEVEL_ERROR, 860 tmrEventScheduleRel(pTmr, pTmrEvent, pFecsGlobalTraceInfo->fecsTimerInterval)); 861 862 return status; 863 } 864 865 static NV_STATUS 866 _fecsTimerCreate 867 ( 868 OBJGPU *pGpu 869 ) 870 { 871 OBJTMR *pTmr = GPU_GET_TIMER(pGpu); 872 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 873 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 874 NvU32 timerFlags = TMR_FLAG_RECUR; 875 // Unix needs to use the OS timer to avoid corrupting records, but Windows doesn't have an OS timer implementation 876 timerFlags |= TMR_FLAG_USE_OS_TIMER; 877 878 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, NV_ERR_INVALID_STATE); 879 880 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 881 tmrEventCreate(pTmr, &pFecsGlobalTraceInfo->pFecsTimerEvent, _fecsTimerCallback, NULL, timerFlags)); 882 883 // This won't be a true 30Hz timer as the callbacks are scheduled from the time they're called 884 NV_CHECK_OK_OR_RETURN(LEVEL_ERROR, 885 tmrEventScheduleRel(pTmr, pFecsGlobalTraceInfo->pFecsTimerEvent, pFecsGlobalTraceInfo->fecsTimerInterval)); 886 887 return NV_OK; 888 } 889 890 static void 891 _fecsTimerDestroy 892 ( 893 OBJGPU *pGpu 894 ) 895 { 896 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 897 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 898 899 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 900 901 if (pFecsGlobalTraceInfo->pFecsTimerEvent != NULL) 902 { 903 OBJTMR *pTmr = GPU_GET_TIMER(pGpu); 904 905 tmrEventCancel(pTmr, pFecsGlobalTraceInfo->pFecsTimerEvent); 906 tmrEventDestroy(pTmr, pFecsGlobalTraceInfo->pFecsTimerEvent); 907 pFecsGlobalTraceInfo->pFecsTimerEvent = NULL; 908 } 909 } 910 911 /** 912 * @brief The callback function that transfers FECS Buffer entries to an EventBuffer 913 */ 914 void 915 nvEventBufferFecsCallback 916 ( 917 OBJGPU *pGpu, 918 void *pArgs 919 ) 920 { 921 KernelGraphics *pKernelGraphics = (KernelGraphics*)pArgs; 922 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 923 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 924 NvU32 fecsReadOffset; 925 NvU32 fecsReadOffsetPrev; 926 NvU64 fecsBufferSize; 927 NvU32 fecsRecordSize; 928 NvU32 i, j; 929 NvU8 *pFecsBufferMapping; 930 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 931 FECS_EVENT_RECORD *pPeekRecord; 932 NvU16 maxFecsRecordsPerIntr; 933 NV_STATUS status; 934 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 935 NvU8 numIterations = (pArgs == NULL) 936 ? KGRMGR_MAX_GR 937 : 1; 938 939 NV_ASSERT_OR_RETURN_VOID(rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 940 941 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 942 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount >= 0); 943 if (pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount <= 0) 944 return; 945 946 for (j = 0; j < numIterations; j++) 947 { 948 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo; 949 950 if (pArgs == NULL) 951 { 952 pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, j); 953 if (pKernelGraphics == NULL) 954 continue; 955 } 956 957 pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 958 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 959 960 pFecsBufferMapping = pFecsTraceInfo->pFecsBufferMapping; 961 maxFecsRecordsPerIntr = pFecsTraceInfo->fecsCtxswLogRecordsPerIntr; 962 963 if (pFecsBufferMapping == NULL) 964 continue; 965 966 status = _getFecsEventListParameters(pGpu, pKernelGraphics, &pFecsMemDesc, &fecsRecordSize); 967 if ((status != NV_OK) || (pFecsMemDesc == NULL)) 968 continue; 969 970 fecsBufferSize = memdescGetSize(pFecsMemDesc) / fecsRecordSize; 971 NV_ASSERT_OR_RETURN_VOID(fecsBufferSize > 0); 972 fecsReadOffset = pFecsTraceInfo->fecsTraceRdOffset; 973 974 if (!osIsRaisedIRQL()) 975 maxFecsRecordsPerIntr = fecsBufferSize; 976 977 // Bail out if the buffer has not changed 978 pPeekRecord = (FECS_EVENT_RECORD*)(pFecsBufferMapping + 979 (fecsReadOffset * fecsRecordSize)); 980 981 if (pPeekRecord->magic_lo == NV_FECS_TRACE_MAGIC_INVALIDATED) 982 continue; 983 984 // Get the read offset from hw if the buffer wrapped around 985 fecsReadOffsetPrev = (fecsReadOffset - 1) % fecsBufferSize; 986 pPeekRecord = (FECS_EVENT_RECORD*)(pFecsBufferMapping + 987 (fecsReadOffsetPrev * fecsRecordSize)); 988 989 if (pPeekRecord->magic_lo != NV_FECS_TRACE_MAGIC_INVALIDATED) 990 { 991 NvHandle hClient; 992 NvHandle hSubdevice; 993 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET_PARAMS params; 994 995 NV_PRINTF(LEVEL_ERROR, "FECS buffer overflow detected\n"); 996 997 NV_ASSERT_OK_OR_ELSE( 998 status, 999 _fecsLoadInternalRoutingInfo(pGpu, 1000 pKernelGraphics, 1001 &hClient, 1002 &hSubdevice, 1003 ¶ms.grRouteInfo), 1004 return); 1005 1006 NV_ASSERT_OK_OR_ELSE( 1007 status, 1008 pRmApi->Control(pRmApi, 1009 hClient, 1010 hSubdevice, 1011 NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_RD_OFFSET, 1012 ¶ms, 1013 sizeof(params)), 1014 return); 1015 fecsReadOffset = params.offset; 1016 pFecsTraceInfo->fecsTraceCounter = 0; 1017 } 1018 1019 // 1020 // Over here we want to go through all EVENTNOTIFICATION nodes and 1021 // loop through them in lockstep with the FECS_EVENT_RECORD records 1022 // 1023 for (i = 0; i < maxFecsRecordsPerIntr; ++i) 1024 { 1025 FECS_EVENT_RECORD *pCurrRecord = (FECS_EVENT_RECORD *)(pFecsBufferMapping + 1026 (fecsReadOffset * fecsRecordSize)); 1027 1028 if (pCurrRecord->magic_lo == NV_FECS_TRACE_MAGIC_INVALIDATED) 1029 break; 1030 1031 pCurrRecord->magic_lo = NV_FECS_TRACE_MAGIC_INVALIDATED; 1032 osFlushCpuWriteCombineBuffer(); 1033 1034 // Loop through all bound event buffers and copy filtered data to user buffers 1035 formatAndNotifyFecsRecord(pGpu, pKernelGraphics, pCurrRecord); 1036 1037 // Update read reg 1038 pFecsTraceInfo->fecsTraceCounter++; 1039 fecsReadOffset++; 1040 if (fecsReadOffset >= fecsBufferSize) 1041 { 1042 fecsReadOffset = 0; 1043 } 1044 } 1045 1046 if (pFecsTraceInfo->fecsTraceCounter > 0) 1047 { 1048 NvHandle hClient; 1049 NvHandle hSubdevice; 1050 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS params; 1051 1052 params.offset = fecsReadOffset; 1053 NV_ASSERT_OK_OR_ELSE( 1054 status, 1055 _fecsLoadInternalRoutingInfo(pGpu, 1056 pKernelGraphics, 1057 &hClient, 1058 &hSubdevice, 1059 ¶ms.grRouteInfo), 1060 return); 1061 1062 NV_ASSERT_OK_OR_ELSE( 1063 status, 1064 pRmApi->Control(pRmApi, 1065 hClient, 1066 hSubdevice, 1067 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET, 1068 ¶ms, 1069 sizeof(params)), 1070 return); 1071 pFecsTraceInfo->fecsTraceCounter = 0; 1072 } 1073 pFecsTraceInfo->fecsTraceRdOffset = fecsReadOffset; 1074 1075 // Re-arm interrupt if there may be more records 1076 if (i == maxFecsRecordsPerIntr) 1077 fecsSignalIntrPendingIfNotPending(pGpu, pKernelGraphics); 1078 } 1079 } 1080 1081 NV_STATUS 1082 fecsAddBindpoint 1083 ( 1084 OBJGPU *pGpu, 1085 RmClient *pClient, 1086 RsResourceRef *pEventBufferRef, 1087 Subdevice *pNotifier, 1088 NvBool bAllUsers, 1089 NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD levelOfDetail, 1090 NvU32 eventFilter, 1091 NvU8 version, 1092 NvU32 *pReasonCode 1093 ) 1094 { 1095 NV_STATUS status; 1096 NvHandle hClient = staticCast(pClient, RsClient)->hClient; 1097 NvHandle hEventBuffer = pEventBufferRef->hResource; 1098 NvHandle hNotifier = RES_GET_HANDLE(pNotifier); 1099 EventBuffer *pEventBuffer; 1100 NvBool bAdmin = osIsAdministrator(); 1101 NvU32 eventMask = 0; 1102 NvU64 targetUser; 1103 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1104 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 1105 NvS32 gpuConsumerCount; 1106 NvBool bFecsBindingActive; 1107 NvBool bIntrDriven = NV_FALSE; 1108 KernelMIGManager *pKernelMIGManager = GPU_GET_KERNEL_MIG_MANAGER(pGpu); 1109 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 1110 NvU8 numIterations; 1111 NvU8 grIdx; 1112 NvBool bKernel; 1113 NvBool bSelectLOD; 1114 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind; 1115 1116 CALL_CONTEXT *pCallContext = resservGetTlsCallContext(); 1117 NV_ASSERT_OR_RETURN(pCallContext != NULL, NV_ERR_INVALID_STATE); 1118 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, NV_ERR_INVALID_STATE); 1119 1120 gpuConsumerCount = pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount; 1121 bFecsBindingActive = (pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount > 0); 1122 1123 bKernel = pCallContext->secInfo.privLevel >= RS_PRIV_LEVEL_KERNEL; 1124 1125 bSelectLOD = bKernel; 1126 1127 #if defined(DEBUG) || defined(DEVELOP) || defined(NV_VERIF_FEATURES) 1128 bSelectLOD = NV_TRUE; 1129 #endif 1130 1131 LOCK_ASSERT_AND_RETURN(rmapiLockIsOwner() && rmDeviceGpuLockIsOwner(pGpu->gpuInstance)); 1132 1133 // Early bail-out if profiling capability is not enabled on vGPU 1134 if (IS_VIRTUAL(pGpu)) 1135 { 1136 VGPU_STATIC_INFO *pVSI = GPU_GET_STATIC_INFO(pGpu); 1137 if ((pVSI == NULL) || !pVSI->vgpuStaticProperties.bProfilingTracingEnabled) 1138 { 1139 if (pReasonCode != NULL) 1140 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NEED_CAPABILITY; 1141 1142 return NV_ERR_NOT_SUPPORTED; 1143 } 1144 } 1145 1146 // On a hypervisor or VM: bail-out early if admin is required 1147 if (IS_VIRTUAL(pGpu) || hypervisorIsVgxHyper()) 1148 { 1149 if (pGpu->bRmProfilingPrivileged && !bAdmin) 1150 { 1151 if (pReasonCode != NULL) 1152 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NEED_ADMIN; 1153 1154 return NV_ERR_NOT_SUPPORTED; 1155 } 1156 } 1157 1158 if (bSelectLOD) 1159 { 1160 switch (levelOfDetail) 1161 { 1162 case NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_FULL: 1163 eventMask = ~0; 1164 break; 1165 case NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_COMPAT: 1166 eventMask |= NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_RESTORE_START | 1167 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_CONTEXT_START | 1168 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_CTXSW_REQ_BY_HOST; 1169 break; 1170 case NV2080_CTRL_GR_FECS_BIND_EVTBUF_LOD_CUSTOM: 1171 eventMask = eventFilter; 1172 break; 1173 default: 1174 // Default to SIMPLIFIED level-of-detail 1175 eventMask |= NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_START | 1176 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_END; 1177 } 1178 } 1179 else 1180 { 1181 // Default to SIMPLIFIED level-of-detail 1182 eventMask |= NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_START | 1183 NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_SIMPLE_END; 1184 } 1185 1186 if (eventMask & NV_EVENT_BUFFER_FECS_BITMASK_CTXSWTAG_FE_ACK) 1187 { 1188 eventMask |= NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_WFI | 1189 NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_GFXP | 1190 NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CTAP | 1191 NV_EVENT_BUFFER_FECS_CTXSWTAG_FE_ACK_CILP; 1192 } 1193 1194 if (bAllUsers) 1195 { 1196 targetUser = 0; 1197 } 1198 else 1199 { 1200 targetUser = (NvU64)(NvUPtr)pClient->pUserInfo; 1201 1202 // Filtering UIDs is not yet implemented in legacy vGPU 1203 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1204 { 1205 if (pReasonCode != NULL) 1206 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED; 1207 1208 return NV_ERR_NOT_SUPPORTED; 1209 } 1210 } 1211 1212 pEventBuffer = dynamicCast(pEventBufferRef->pResource, EventBuffer); 1213 if (pEventBuffer == NULL) 1214 return NV_ERR_INVALID_ARGUMENT; 1215 1216 if (multimapFindSubmap(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, targetUser) == NULL) 1217 { 1218 if (multimapInsertSubmap(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, targetUser) == NULL) 1219 { 1220 NV_PRINTF(LEVEL_ERROR, "failed to add UID binding!\n"); 1221 return NV_ERR_INSUFFICIENT_RESOURCES; 1222 } 1223 } 1224 1225 // If the binding exists already, we're done 1226 if (multimapFindItem(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, targetUser, (NvU64)(NvUPtr)pEventBuffer) != NULL) 1227 return NV_OK; 1228 1229 pBind = multimapInsertItemNew(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, targetUser, (NvU64)(NvUPtr)pEventBuffer); 1230 if (pBind == NULL) 1231 return NV_ERR_INVALID_ARGUMENT; 1232 ++pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount; 1233 1234 pBind->hClient = hClient; 1235 pBind->hNotifier = hNotifier; 1236 pBind->hEventBuffer = hEventBuffer; 1237 pBind->pEventBuffer = pEventBuffer; 1238 pBind->pUserInfo = (NvU64)(NvUPtr)pClient->pUserInfo; 1239 pBind->bAdmin = bAdmin; 1240 pBind->eventMask = eventMask; 1241 pBind->bKernel = bKernel; 1242 pBind->version = version; 1243 1244 status = registerEventNotification(&pEventBuffer->pListeners, 1245 staticCast(pClient, RsClient), 1246 hNotifier, 1247 hEventBuffer, 1248 (version == 2 ? 1249 NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH_V2 : 1250 NV_EVENT_BUFFER_RECORD_TYPE_FECS_CTX_SWITCH) | NV01_EVENT_WITHOUT_EVENT_DATA, 1251 NV_EVENT_BUFFER_BIND, 1252 pEventBuffer->producerInfo.notificationHandle, 1253 NV_FALSE); 1254 if (status != NV_OK) 1255 goto done; 1256 1257 if (bMIGInUse) 1258 { 1259 if (kmigmgrIsDeviceUsingDeviceProfiling(pGpu, pKernelMIGManager, GPU_RES_GET_DEVICE(pNotifier))) 1260 { 1261 pBind->swizzId = NV2080_CTRL_GPU_PARTITION_ID_INVALID; 1262 } 1263 else 1264 { 1265 GPUInstanceSubscription *pGPUInstanceSubscription; 1266 status = gisubscriptionGetGPUInstanceSubscription( 1267 staticCast(pClient, RsClient), hNotifier, &pGPUInstanceSubscription); 1268 if (status != NV_OK) 1269 goto done; 1270 1271 if (gisubscriptionGetMIGGPUInstance(pGPUInstanceSubscription) == NULL) 1272 { 1273 if (pReasonCode != NULL) 1274 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED; 1275 1276 status = NV_ERR_NOT_SUPPORTED; 1277 goto done; 1278 } 1279 1280 pBind->swizzId = gisubscriptionGetMIGGPUInstance(pGPUInstanceSubscription)->swizzId; 1281 } 1282 } 1283 1284 numIterations = bMIGInUse ? GPU_MAX_GRS: 1; 1285 for (grIdx = 0; grIdx < numIterations; grIdx++) 1286 { 1287 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 1288 if (pKernelGraphics == NULL) 1289 continue; 1290 1291 if (!kgraphicsIsCtxswLoggingSupported(pGpu, pKernelGraphics)) 1292 { 1293 if (pReasonCode) 1294 *pReasonCode = NV2080_CTRL_GR_FECS_BIND_REASON_CODE_NOT_ENABLED_GPU; 1295 1296 status = NV_ERR_NOT_SUPPORTED; 1297 goto done; 1298 } 1299 1300 if (!bFecsBindingActive) 1301 { 1302 1303 fecsBufferReset(pGpu, pKernelGraphics); 1304 } 1305 1306 bIntrDriven |= kgraphicsIsIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics); 1307 } 1308 1309 if (!bFecsBindingActive && !bIntrDriven) 1310 { 1311 NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, _fecsTimerCreate(pGpu), done); 1312 } 1313 1314 done: 1315 if (status != NV_OK) 1316 { 1317 if (gpuConsumerCount != pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount) 1318 fecsRemoveBindpoint(pGpu, targetUser, pBind); 1319 1320 _fecsTimerDestroy(pGpu); 1321 } 1322 1323 return status; 1324 } 1325 1326 void 1327 fecsRemoveBindpoint 1328 ( 1329 OBJGPU *pGpu, 1330 NvU64 uid, 1331 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind 1332 ) 1333 { 1334 EventBuffer *pEventBuffer = pBind->pEventBuffer; 1335 KernelGraphicsManager *pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1336 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 1337 1338 NV_ASSERT_OR_RETURN_VOID(pFecsGlobalTraceInfo != NULL); 1339 1340 --pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount; 1341 1342 unregisterEventNotificationWithData(&pEventBuffer->pListeners, 1343 pBind->hClient, 1344 pBind->hNotifier, 1345 pBind->hEventBuffer, 1346 NV_TRUE, 1347 pEventBuffer->producerInfo.notificationHandle); 1348 1349 multimapRemoveItemByKey(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, 1350 uid, 1351 (NvU64)(NvUPtr)pEventBuffer); 1352 1353 if (pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount == 0) 1354 { 1355 NvBool bMIGInUse = IS_MIG_IN_USE(pGpu); 1356 NvU8 grIdx; 1357 NvBool bIntrDriven = NV_FALSE; 1358 1359 NvU8 numIterations = bMIGInUse ? GPU_MAX_GRS : 1; 1360 for (grIdx = 0; grIdx < numIterations; grIdx++) 1361 { 1362 KernelGraphics *pKernelGraphics = GPU_GET_KERNEL_GRAPHICS(pGpu, grIdx); 1363 if (pKernelGraphics == NULL) 1364 continue; 1365 1366 // 1367 // Disable HW without unmapping buffer so that new event buffers still work properly 1368 // HW enable will happen on bindpoint creation. 1369 // Mapping only occurs on Graphics load/alloc, so unmapping should only occur when Graphics is destroyed. 1370 // 1371 fecsBufferDisableHw(pGpu, pKernelGraphics); 1372 bIntrDriven |= kgraphicsIsIntrDrivenCtxswLoggingEnabled(pGpu, pKernelGraphics); 1373 1374 } 1375 1376 if (!bIntrDriven) 1377 { 1378 _fecsTimerDestroy(pGpu); 1379 } 1380 } 1381 } 1382 1383 void 1384 fecsRemoveAllBindpoints 1385 ( 1386 EventBuffer *pEventBuffer 1387 ) 1388 { 1389 OBJGPU *pGpu = NULL; 1390 NvU32 gpuMask = 0; 1391 NvU32 gpuIndex = 0; 1392 KernelGraphicsManager *pKernelGraphicsManager; 1393 FecsEventBufferBindMultiMapSupermapIter uidBindIter; 1394 1395 eventBufferSetEnable(&pEventBuffer->producerInfo, NV_FALSE); 1396 1397 gpumgrGetGpuAttachInfo(NULL, &gpuMask); 1398 while ((pGpu = gpumgrGetNextGpu(gpuMask, &gpuIndex)) != NULL) 1399 { 1400 pKernelGraphicsManager = GPU_GET_KERNEL_GRAPHICS_MANAGER(pGpu); 1401 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 1402 1403 NV_CHECK_OR_ELSE(LEVEL_ERROR, pFecsGlobalTraceInfo != NULL, continue;); 1404 1405 uidBindIter = multimapSubmapIterAll(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid); 1406 while (multimapSubmapIterNext(&uidBindIter)) 1407 { 1408 FecsEventBufferBindMultiMapSubmap *pSubmap = uidBindIter.pValue; 1409 NV_EVENT_BUFFER_BIND_POINT_FECS *pBind = NULL; 1410 NvU64 uid = mapKey_IMPL(uidBindIter.iter.pMap, pSubmap); 1411 1412 while ((pBind = multimapFindItem(&pFecsGlobalTraceInfo->fecsEventBufferBindingsUid, 1413 uid, 1414 (NvU64)(NvUPtr)pEventBuffer)) != NULL) 1415 { 1416 fecsRemoveBindpoint(pGpu, uid, pBind); 1417 } 1418 } 1419 1420 } 1421 } 1422 1423 void 1424 fecsBufferReset 1425 ( 1426 OBJGPU *pGpu, 1427 KernelGraphics *pKernelGraphics 1428 ) 1429 { 1430 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 1431 NV_STATUS status; 1432 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1433 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS getHwEnableParams; 1434 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 1435 NvHandle hClient; 1436 NvHandle hSubdevice; 1437 1438 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 1439 1440 if (pFecsTraceInfo->pFecsBufferMapping == NULL) 1441 return; 1442 1443 NV_ASSERT_OK_OR_ELSE( 1444 status, 1445 _fecsLoadInternalRoutingInfo(pGpu, 1446 pKernelGraphics, 1447 &hClient, 1448 &hSubdevice, 1449 &getHwEnableParams.grRouteInfo), 1450 return); 1451 1452 NV_ASSERT_OK_OR_ELSE( 1453 status, 1454 pRmApi->Control(pRmApi, 1455 hClient, 1456 hSubdevice, 1457 NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE, 1458 &getHwEnableParams, 1459 sizeof(getHwEnableParams)), 1460 return); 1461 1462 status = _getFecsMemDesc(pGpu, pKernelGraphics, &pFecsMemDesc); 1463 1464 if ((status == NV_OK) && (pFecsMemDesc != NULL) && (getHwEnableParams.bEnable != NV_TRUE)) 1465 { 1466 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET_PARAMS traceWrOffsetParams; 1467 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET_PARAMS traceRdOffsetParams; 1468 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS setHwEnableParams; 1469 1470 portMemSet(pFecsTraceInfo->pFecsBufferMapping, 1471 (NvU8)(NV_FECS_TRACE_MAGIC_INVALIDATED & 0xff), 1472 memdescGetSize(pFecsMemDesc)); 1473 1474 pFecsTraceInfo->fecsLastSeqno = 0; 1475 1476 // Routing info is the same for all future calls in this series 1477 traceWrOffsetParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1478 traceRdOffsetParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1479 setHwEnableParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1480 1481 traceWrOffsetParams.offset = 0; 1482 NV_ASSERT_OK_OR_ELSE( 1483 status, 1484 pRmApi->Control(pRmApi, 1485 hClient, 1486 hSubdevice, 1487 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_WR_OFFSET, 1488 &traceWrOffsetParams, 1489 sizeof(traceWrOffsetParams)), 1490 return); 1491 1492 traceRdOffsetParams.offset = 0; 1493 NV_ASSERT_OK_OR_ELSE( 1494 status, 1495 pRmApi->Control(pRmApi, 1496 hClient, 1497 hSubdevice, 1498 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_RD_OFFSET, 1499 &traceRdOffsetParams, 1500 sizeof(traceRdOffsetParams)), 1501 return); 1502 pFecsTraceInfo->fecsTraceRdOffset = 0; 1503 1504 setHwEnableParams.bEnable = NV_TRUE; 1505 NV_ASSERT_OK_OR_ELSE( 1506 status, 1507 pRmApi->Control(pRmApi, 1508 hClient, 1509 hSubdevice, 1510 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE, 1511 &setHwEnableParams, 1512 sizeof(setHwEnableParams)), 1513 return); 1514 } 1515 } 1516 1517 void 1518 fecsBufferDisableHw 1519 ( 1520 OBJGPU *pGpu, 1521 KernelGraphics *pKernelGraphics 1522 ) 1523 { 1524 RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); 1525 NvHandle hClient; 1526 NvHandle hSubdevice; 1527 NV2080_CTRL_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE_PARAMS getHwEnableParams; 1528 NV2080_CTRL_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE_PARAMS setHwEnableParams; 1529 NV_STATUS status; 1530 1531 // This function may be called with unused GR Engines 1532 NV_CHECK_OK_OR_ELSE( 1533 status, 1534 LEVEL_INFO, 1535 _fecsLoadInternalRoutingInfo(pGpu, 1536 pKernelGraphics, 1537 &hClient, 1538 &hSubdevice, 1539 &getHwEnableParams.grRouteInfo), 1540 return); 1541 1542 NV_ASSERT_OK_OR_ELSE( 1543 status, 1544 pRmApi->Control(pRmApi, 1545 hClient, 1546 hSubdevice, 1547 NV2080_CTRL_CMD_INTERNAL_GR_GET_FECS_TRACE_HW_ENABLE, 1548 &getHwEnableParams, 1549 sizeof(getHwEnableParams)), 1550 return); 1551 1552 if (getHwEnableParams.bEnable) 1553 { 1554 // Copy previously loaded routing info 1555 setHwEnableParams.grRouteInfo = getHwEnableParams.grRouteInfo; 1556 setHwEnableParams.bEnable = NV_FALSE; 1557 1558 NV_ASSERT_OK_OR_ELSE( 1559 status, 1560 pRmApi->Control(pRmApi, 1561 hClient, 1562 hSubdevice, 1563 NV2080_CTRL_CMD_INTERNAL_GR_SET_FECS_TRACE_HW_ENABLE, 1564 &setHwEnableParams, 1565 sizeof(setHwEnableParams)), 1566 return); 1567 } 1568 } 1569 1570 void 1571 fecsBufferTeardown 1572 ( 1573 OBJGPU *pGpu, 1574 KernelGraphics *pKernelGraphics 1575 ) 1576 { 1577 fecsBufferDisableHw(pGpu, pKernelGraphics); 1578 fecsBufferUnmap(pGpu, pKernelGraphics); 1579 } 1580 1581 /*! Is the FECS trace buffer mapped? */ 1582 NvBool 1583 fecsBufferIsMapped 1584 ( 1585 OBJGPU *pGpu, 1586 KernelGraphics *pKernelGraphics 1587 ) 1588 { 1589 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1590 1591 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1592 return pFecsTraceInfo->pFecsBufferMapping != NULL; 1593 } 1594 1595 NV_STATUS 1596 fecsBufferMap 1597 ( 1598 OBJGPU *pGpu, 1599 KernelGraphics *pKernelGraphics 1600 ) 1601 { 1602 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 1603 NvU8 *pFecsBufferMapping = NULL; 1604 NV_STATUS status; 1605 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1606 TRANSFER_SURFACE surf = {0}; 1607 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1608 1609 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_ERR_INVALID_STATE); 1610 1611 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1612 return NV_OK; 1613 1614 if (pFecsTraceInfo->pFecsBufferMapping != NULL) 1615 return NV_OK; 1616 1617 status = _getFecsMemDesc(pGpu, pKernelGraphics, &pFecsMemDesc); 1618 if ((status != NV_OK) || (pFecsMemDesc == NULL)) 1619 return NV_ERR_INVALID_STATE; 1620 1621 surf.pMemDesc = pFecsMemDesc; 1622 surf.offset = 0; 1623 1624 pFecsBufferMapping = memmgrMemBeginTransfer(pMemoryManager, &surf, 1625 memdescGetSize(pFecsMemDesc), 1626 TRANSFER_FLAGS_PREFER_PROCESSOR | 1627 TRANSFER_FLAGS_PERSISTENT_CPU_MAPPING); 1628 if (pFecsBufferMapping == NULL) 1629 return NV_ERR_INSUFFICIENT_RESOURCES; 1630 1631 pFecsTraceInfo->pFecsBufferMapping = pFecsBufferMapping; 1632 1633 return NV_OK; 1634 } 1635 1636 void 1637 fecsBufferUnmap 1638 ( 1639 OBJGPU *pGpu, 1640 KernelGraphics *pKernelGraphics 1641 ) 1642 { 1643 MEMORY_DESCRIPTOR *pFecsMemDesc = NULL; 1644 NV_STATUS status; 1645 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1646 TRANSFER_SURFACE surf = {0}; 1647 MemoryManager *pMemoryManager = GPU_GET_MEMORY_MANAGER(pGpu); 1648 1649 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 1650 1651 if (IS_VIRTUAL_WITHOUT_SRIOV(pGpu)) 1652 return; 1653 1654 status = _getFecsMemDesc(pGpu, pKernelGraphics, &pFecsMemDesc); 1655 if ((status != NV_OK) || (pFecsMemDesc == NULL)) 1656 return; 1657 1658 surf.pMemDesc = pFecsMemDesc; 1659 surf.offset = 0; 1660 1661 if (pFecsTraceInfo->pFecsBufferMapping != NULL) 1662 { 1663 memmgrMemEndTransfer(pMemoryManager, &surf, 1664 memdescGetSize(pFecsMemDesc), 1665 TRANSFER_FLAGS_PREFER_PROCESSOR | 1666 TRANSFER_FLAGS_PERSISTENT_CPU_MAPPING); 1667 1668 pFecsTraceInfo->pFecsBufferMapping = NULL; 1669 } 1670 } 1671 1672 /*! Atomically set intr callback pending, return NV_TRUE if wasn't pending prior */ 1673 NvBool 1674 fecsSignalIntrPendingIfNotPending 1675 ( 1676 OBJGPU *pGpu, 1677 KernelGraphics *pKernelGraphics 1678 ) 1679 { 1680 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1681 1682 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1683 1684 return portAtomicCompareAndSwapU32(&pFecsTraceInfo->fecsCtxswLogIntrPending, 1, 0); 1685 } 1686 1687 /*! Atomically clear intr callback pending, return NV_TRUE if was pending */ 1688 NvBool 1689 fecsClearIntrPendingIfPending 1690 ( 1691 OBJGPU *pGpu, 1692 KernelGraphics *pKernelGraphics 1693 ) 1694 { 1695 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1696 1697 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1698 1699 return portAtomicCompareAndSwapU32(&pFecsTraceInfo->fecsCtxswLogIntrPending, 0, 1); 1700 } 1701 1702 /*! Atomically check if intr callback pending */ 1703 NvBool fecsIsIntrPending 1704 ( 1705 OBJGPU *pGpu, 1706 KernelGraphics *pKernelGraphics 1707 ) 1708 { 1709 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1710 1711 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NV_FALSE); 1712 1713 return portAtomicOrU32(&pFecsTraceInfo->fecsCtxswLogIntrPending, 0) != 0; 1714 } 1715 1716 NvS16 1717 fecsGetCtxswLogConsumerCount 1718 ( 1719 OBJGPU *pGpu, 1720 KernelGraphicsManager *pKernelGraphicsManager 1721 ) 1722 { 1723 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 1724 1725 NV_CHECK_OR_RETURN(LEVEL_ERROR, pFecsGlobalTraceInfo != NULL, 0); 1726 1727 return pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount; 1728 } 1729 1730 NV_STATUS 1731 fecsIncrementCtxswLogConsumerCount 1732 ( 1733 OBJGPU *pGpu, 1734 KernelGraphicsManager *pKernelGraphicsManager 1735 ) 1736 { 1737 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 1738 1739 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, NV_ERR_INVALID_STATE); 1740 1741 pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount++; 1742 1743 return NV_OK; 1744 } 1745 1746 NV_STATUS 1747 fecsDecrementCtxswLogConsumerCount 1748 ( 1749 OBJGPU *pGpu, 1750 KernelGraphicsManager *pKernelGraphicsManager 1751 ) 1752 { 1753 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 1754 1755 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, NV_ERR_INVALID_STATE); 1756 1757 pFecsGlobalTraceInfo->fecsCtxswLogConsumerCount--; 1758 1759 return NV_OK; 1760 } 1761 1762 /*! Retrieve the current VGPU staging buffer */ 1763 VGPU_FECS_TRACE_STAGING_BUFFER * 1764 fecsGetVgpuStagingBuffer 1765 ( 1766 OBJGPU *pGpu, 1767 KernelGraphics *pKernelGraphics 1768 ) 1769 { 1770 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1771 1772 NV_ASSERT_OR_RETURN(pFecsTraceInfo != NULL, NULL); 1773 1774 return pFecsTraceInfo->pVgpuStaging; 1775 } 1776 1777 /*! Store the given VGPU staging buffer */ 1778 void 1779 fecsSetVgpuStagingBuffer 1780 ( 1781 OBJGPU *pGpu, 1782 KernelGraphics *pKernelGraphics, 1783 VGPU_FECS_TRACE_STAGING_BUFFER *pStagingBuffer 1784 ) 1785 { 1786 KGRAPHICS_FECS_TRACE_INFO *pFecsTraceInfo = kgraphicsGetFecsTraceInfo(pGpu, pKernelGraphics); 1787 1788 NV_ASSERT_OR_RETURN_VOID(pFecsTraceInfo != NULL); 1789 1790 pFecsTraceInfo->pVgpuStaging = pStagingBuffer; 1791 } 1792 1793 FecsEventBufferBindMultiMap * 1794 fecsGetEventBufferBindMultiMap 1795 ( 1796 OBJGPU *pGpu, 1797 KernelGraphicsManager *pKernelGraphicsManager 1798 ) 1799 { 1800 KGRMGR_FECS_GLOBAL_TRACE_INFO *pFecsGlobalTraceInfo = kgrmgrGetFecsGlobalTraceInfo(pGpu, pKernelGraphicsManager); 1801 1802 NV_ASSERT_OR_RETURN(pFecsGlobalTraceInfo != NULL, NULL); 1803 1804 return &pFecsGlobalTraceInfo->fecsEventBufferBindingsUid; 1805 } 1806 1807