1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2017-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "common_nvswitch.h" 25 #include "rom_nvswitch.h" 26 #include "error_nvswitch.h" 27 #include "regkey_nvswitch.h" 28 #include "bios_nvswitch.h" 29 #include "haldef_nvswitch.h" 30 #include "flcn/haldefs_flcnable_nvswitch.h" 31 #include "flcn/flcn_nvswitch.h" 32 #include "soe/soe_nvswitch.h" 33 #include "nvVer.h" 34 #include "nvlink_inband_msg.h" 35 36 static NvlStatus _nvswitch_ctrl_inband_flush_data(nvswitch_device *device, NVSWITCH_INBAND_FLUSH_DATA_PARAMS *p); 37 38 #define NVSWITCH_DEV_CMD_CHECK_ADMIN NVBIT64(0) 39 #define NVSWITCH_DEV_CMD_CHECK_FM NVBIT64(1) 40 41 #define NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(cmd, function, type, private, flags)\ 42 case cmd: \ 43 { \ 44 if (sizeof(type) != size) \ 45 { \ 46 retval = -NVL_BAD_ARGS; \ 47 break; \ 48 } \ 49 \ 50 retval = _nvswitch_lib_validate_privileged_ctrl(private, flags); \ 51 if (retval != NVL_SUCCESS) \ 52 { \ 53 break; \ 54 } \ 55 \ 56 retval = function(device, params); \ 57 break; \ 58 } \ 59 60 #define NVSWITCH_DEV_CMD_DISPATCH_RESERVED(cmd) \ 61 case cmd: \ 62 { \ 63 retval = -NVL_ERR_NOT_IMPLEMENTED; \ 64 break; \ 65 } \ 66 67 const static NvU32 nvswitch_lr10_device_ids[] = 68 { 69 0x1AE8, 0x1AF0, 0x1AF1, 0x1AF2, 0x1AF3, 0x1AF4, 0x1AF5, 0x1AF6, 0x1AF7, 70 0x1AF8, 0x1AF9, 0x1AFA, 0x1AFB, 0x1AFC, 0x1AFD, 0x1AFE, 0x1AFF 71 }; 72 73 const static NvU32 nvswitch_ls10_device_ids[] = 74 { 75 // PCIE endpoint to manage the NVLink switch HW 76 0x22A0, 0x22A1, 0x22A2, 0x22A3, 0x22A4, 0x22A5, 0x22A6, 0x22A7, 77 // PCI-PCI Bridge, Laguna Switch Function 0 78 0x22A8, 0x22A9, 0x22AA, 0x22AB, 79 // Non-Transparent Bridge, Laguna Switch Function 1 80 0x22AC, 0x22AD, 0x22AE, 0x22AF 81 }; 82 83 nvlink_link_handlers link_handlers; 84 85 static NvBool 86 _nvswitch_is_device_id_present 87 ( 88 const NvU32 *array, 89 NvU32 array_len, 90 NvU32 device_id 91 ) 92 { 93 NvU32 i = 0; 94 95 for(i = 0; i < array_len; i++) 96 { 97 if (array[i] == device_id) 98 { 99 return NV_TRUE; 100 } 101 } 102 103 return NV_FALSE; 104 } 105 106 NvBool 107 nvswitch_is_lr10_device_id 108 ( 109 NvU32 device_id 110 ) 111 { 112 NvU32 count = (sizeof(nvswitch_lr10_device_ids) / 113 sizeof(nvswitch_lr10_device_ids[0])); 114 115 return _nvswitch_is_device_id_present(nvswitch_lr10_device_ids, count, device_id); 116 } 117 118 NvBool 119 nvswitch_is_ls10_device_id 120 ( 121 NvU32 device_id 122 ) 123 { 124 NvU32 count = (sizeof(nvswitch_ls10_device_ids) / 125 sizeof(nvswitch_ls10_device_ids[0])); 126 127 return _nvswitch_is_device_id_present(nvswitch_ls10_device_ids, count, device_id); 128 } 129 130 /* 131 * NVLink corelib callbacks are used by the NVLink library separate from the 132 * NVSwitch driver, therefore they do not take a device lock and can not modify 133 * nvswitch_device state or use error logging. 134 * 135 * These NVSwitch functions modify link state outside of the corelib: 136 * _nvswitch_ctrl_inject_link_error - injects asynchronous link errors (MODS-only) 137 */ 138 139 static NV_API_CALL NvlStatus 140 _nvswitch_corelib_add_link 141 ( 142 nvlink_link *link 143 ) 144 { 145 nvswitch_device *device = link->dev->pDevInfo; 146 return device->hal.nvswitch_corelib_add_link(link); 147 } 148 149 static NV_API_CALL NvlStatus 150 _nvswitch_corelib_remove_link 151 ( 152 nvlink_link *link 153 ) 154 { 155 nvswitch_device *device = link->dev->pDevInfo; 156 return device->hal.nvswitch_corelib_remove_link(link); 157 } 158 159 static NV_API_CALL NvlStatus 160 _nvswitch_corelib_set_dl_link_mode 161 ( 162 nvlink_link *link, 163 NvU64 mode, 164 NvU32 flags 165 ) 166 { 167 nvswitch_device *device = link->dev->pDevInfo; 168 return device->hal.nvswitch_corelib_set_dl_link_mode(link, mode, flags); 169 } 170 171 static NV_API_CALL NvlStatus 172 _nvswitch_corelib_get_dl_link_mode 173 ( 174 nvlink_link *link, 175 NvU64 *mode 176 ) 177 { 178 nvswitch_device *device = link->dev->pDevInfo; 179 return device->hal.nvswitch_corelib_get_dl_link_mode(link, mode); 180 } 181 182 static NV_API_CALL NvlStatus 183 _nvswitch_corelib_set_tl_link_mode 184 ( 185 nvlink_link *link, 186 NvU64 mode, 187 NvU32 flags 188 ) 189 { 190 nvswitch_device *device = link->dev->pDevInfo; 191 return device->hal.nvswitch_corelib_set_tl_link_mode(link, mode, flags); 192 } 193 194 static NV_API_CALL NvlStatus 195 _nvswitch_corelib_get_tl_link_mode 196 ( 197 nvlink_link *link, 198 NvU64 *mode 199 ) 200 { 201 nvswitch_device *device = link->dev->pDevInfo; 202 return device->hal.nvswitch_corelib_get_tl_link_mode(link, mode); 203 } 204 205 static NV_API_CALL NvlStatus 206 _nvswitch_corelib_set_tx_mode 207 ( 208 nvlink_link *link, 209 NvU64 mode, 210 NvU32 flags 211 ) 212 { 213 nvswitch_device *device = link->dev->pDevInfo; 214 return device->hal.nvswitch_corelib_set_tx_mode(link, mode, flags); 215 } 216 217 static NV_API_CALL NvlStatus 218 _nvswitch_corelib_get_tx_mode 219 ( 220 nvlink_link *link, 221 NvU64 *mode, 222 NvU32 *subMode 223 ) 224 { 225 nvswitch_device *device = link->dev->pDevInfo; 226 return device->hal.nvswitch_corelib_get_tx_mode(link, mode, subMode); 227 } 228 229 static NV_API_CALL NvlStatus 230 _nvswitch_corelib_set_rx_mode 231 ( 232 nvlink_link *link, 233 NvU64 mode, 234 NvU32 flags 235 ) 236 { 237 nvswitch_device *device = link->dev->pDevInfo; 238 return device->hal.nvswitch_corelib_set_rx_mode(link, mode, flags); 239 } 240 241 static NV_API_CALL NvlStatus 242 _nvswitch_corelib_get_rx_mode 243 ( 244 nvlink_link *link, 245 NvU64 *mode, 246 NvU32 *subMode 247 ) 248 { 249 nvswitch_device *device = link->dev->pDevInfo; 250 return device->hal.nvswitch_corelib_get_rx_mode(link, mode, subMode); 251 } 252 253 static NV_API_CALL NvlStatus 254 _nvswitch_corelib_set_rx_detect 255 ( 256 nvlink_link *link, 257 NvU32 flags 258 ) 259 { 260 nvswitch_device *device = link->dev->pDevInfo; 261 return device->hal.nvswitch_corelib_set_rx_detect(link, flags); 262 } 263 264 static NV_API_CALL NvlStatus 265 _nvswitch_corelib_get_rx_detect 266 ( 267 nvlink_link *link 268 ) 269 { 270 nvswitch_device *device = link->dev->pDevInfo; 271 return device->hal.nvswitch_corelib_get_rx_detect(link); 272 } 273 274 static NV_API_CALL void 275 _nvswitch_corelib_training_complete 276 ( 277 nvlink_link *link 278 ) 279 { 280 nvswitch_device *device = link->dev->pDevInfo; 281 device->hal.nvswitch_corelib_training_complete(link); 282 } 283 284 static NV_API_CALL void 285 _nvswitch_corelib_get_uphy_load 286 ( 287 nvlink_link *link, 288 NvBool *bUnlocked 289 ) 290 { 291 nvswitch_device *device = link->dev->pDevInfo; 292 return device->hal.nvswitch_corelib_get_uphy_load(link, bUnlocked); 293 } 294 295 296 static NV_API_CALL NvlStatus 297 _nvswitch_corelib_write_discovery_token 298 ( 299 nvlink_link *link, 300 NvU64 token 301 ) 302 { 303 nvswitch_device *device = link->dev->pDevInfo; 304 305 if (link->version >= NVLINK_DEVICE_VERSION_40) 306 { 307 nvswitch_store_topology_information(device, link); 308 return NVL_SUCCESS; 309 } 310 311 return NVL_SUCCESS; 312 } 313 314 static NV_API_CALL NvlStatus 315 _nvswitch_corelib_ali_training 316 ( 317 nvlink_link *link 318 ) 319 { 320 nvswitch_device *device = link->dev->pDevInfo; 321 return device->hal.nvswitch_launch_ALI_link_training(device, link, NV_FALSE); 322 } 323 324 void 325 nvswitch_get_link_handlers 326 ( 327 nvlink_link_handlers *nvswitch_link_handlers 328 ) 329 { 330 if (!nvswitch_link_handlers) 331 { 332 NVSWITCH_ASSERT(0); 333 return; 334 } 335 336 nvswitch_link_handlers->add = _nvswitch_corelib_add_link; 337 nvswitch_link_handlers->remove = _nvswitch_corelib_remove_link; 338 nvswitch_link_handlers->set_dl_link_mode = _nvswitch_corelib_set_dl_link_mode; 339 nvswitch_link_handlers->get_dl_link_mode = _nvswitch_corelib_get_dl_link_mode; 340 nvswitch_link_handlers->set_tl_link_mode = _nvswitch_corelib_set_tl_link_mode; 341 nvswitch_link_handlers->get_tl_link_mode = _nvswitch_corelib_get_tl_link_mode; 342 nvswitch_link_handlers->set_tx_mode = _nvswitch_corelib_set_tx_mode; 343 nvswitch_link_handlers->get_tx_mode = _nvswitch_corelib_get_tx_mode; 344 nvswitch_link_handlers->set_rx_mode = _nvswitch_corelib_set_rx_mode; 345 nvswitch_link_handlers->get_rx_mode = _nvswitch_corelib_get_rx_mode; 346 nvswitch_link_handlers->set_rx_detect = _nvswitch_corelib_set_rx_detect; 347 nvswitch_link_handlers->get_rx_detect = _nvswitch_corelib_get_rx_detect; 348 nvswitch_link_handlers->training_complete = _nvswitch_corelib_training_complete; 349 nvswitch_link_handlers->get_uphy_load = _nvswitch_corelib_get_uphy_load; 350 nvswitch_link_handlers->write_discovery_token = _nvswitch_corelib_write_discovery_token; 351 nvswitch_link_handlers->ali_training = _nvswitch_corelib_ali_training; 352 } 353 354 #define NVSWITCH_INIT_REGKEY(_private, _regkey, _string, _default_val) \ 355 do \ 356 { \ 357 NvU32 data; \ 358 \ 359 device->regkeys._regkey = _default_val; \ 360 if (NV_SWITCH_REGKEY_PRIVATE_ALLOWED || !NV_SWITCH_REGKEY##_private) \ 361 { \ 362 if (NVL_SUCCESS == \ 363 nvswitch_os_read_registry_dword(device->os_handle, _string, &data)) \ 364 { \ 365 NVSWITCH_PRINT(device, SETUP, \ 366 "%s: Applying regkey %s=0x%x\n", \ 367 __FUNCTION__, \ 368 _string, data); \ 369 device->regkeys._regkey = data; \ 370 } \ 371 } \ 372 } while(0) 373 374 static void 375 _nvswitch_init_device_regkeys 376 ( 377 nvswitch_device *device 378 ) 379 { 380 // 381 // Public external use regkeys 382 // 383 NVSWITCH_INIT_REGKEY(_PUBLIC, ato_control, 384 NV_SWITCH_REGKEY_ATO_CONTROL, 385 NV_SWITCH_REGKEY_ATO_CONTROL_DEFAULT); 386 387 NVSWITCH_INIT_REGKEY(_PUBLIC, sto_control, 388 NV_SWITCH_REGKEY_STO_CONTROL, 389 NV_SWITCH_REGKEY_STO_CONTROL_DEFAULT); 390 391 NVSWITCH_INIT_REGKEY(_PUBLIC, crc_bit_error_rate_short, 392 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_SHORT, 393 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_SHORT_DEFAULT); 394 395 NVSWITCH_INIT_REGKEY(_PUBLIC, crc_bit_error_rate_long, 396 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_LONG, 397 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_LONG_DEFAULT); 398 399 NVSWITCH_INIT_REGKEY(_PUBLIC, surpress_link_errors_for_gpu_reset, 400 NV_SWITCH_REGKEY_SURPRESS_LINK_ERRORS_FOR_GPU_RESET, 401 NV_SWITCH_REGKEY_SURPRESS_LINK_ERRORS_FOR_GPU_RESET_DISABLE); 402 403 // 404 // Debug use regkeys 405 // Not available on release build kernel drivers 406 // 407 NVSWITCH_INIT_REGKEY(_PRIVATE, external_fabric_mgmt, 408 NV_SWITCH_REGKEY_EXTERNAL_FABRIC_MGMT, 409 NV_SWITCH_REGKEY_EXTERNAL_FABRIC_MGMT_ENABLE); 410 411 NVSWITCH_INIT_REGKEY(_PRIVATE, txtrain_control, 412 NV_SWITCH_REGKEY_TXTRAIN_CONTROL, 413 NV_SWITCH_REGKEY_TXTRAIN_CONTROL_NOP); 414 415 NVSWITCH_INIT_REGKEY(_PRIVATE, crossbar_DBI, 416 NV_SWITCH_REGKEY_CROSSBAR_DBI, 417 NV_SWITCH_REGKEY_CROSSBAR_DBI_ENABLE); 418 419 NVSWITCH_INIT_REGKEY(_PRIVATE, link_DBI, 420 NV_SWITCH_REGKEY_LINK_DBI, 421 NV_SWITCH_REGKEY_LINK_DBI_ENABLE); 422 423 NVSWITCH_INIT_REGKEY(_PRIVATE, ac_coupled_mask, 424 NV_SWITCH_REGKEY_AC_COUPLED_MASK, 425 0); 426 427 NVSWITCH_INIT_REGKEY(_PRIVATE, ac_coupled_mask2, 428 NV_SWITCH_REGKEY_AC_COUPLED_MASK2, 429 0); 430 431 NVSWITCH_INIT_REGKEY(_PRIVATE, swap_clk, 432 NV_SWITCH_REGKEY_SWAP_CLK_OVERRIDE, 433 nvswitch_get_swap_clk_default(device)); 434 435 NVSWITCH_INIT_REGKEY(_PRIVATE, link_enable_mask, 436 NV_SWITCH_REGKEY_ENABLE_LINK_MASK, 437 NV_U32_MAX); 438 439 NVSWITCH_INIT_REGKEY(_PRIVATE, link_enable_mask2, 440 NV_SWITCH_REGKEY_ENABLE_LINK_MASK2, 441 NV_U32_MAX); 442 443 NVSWITCH_INIT_REGKEY(_PRIVATE, bandwidth_shaper, 444 NV_SWITCH_REGKEY_BANDWIDTH_SHAPER, 445 NV_SWITCH_REGKEY_BANDWIDTH_SHAPER_PROD); 446 447 NVSWITCH_INIT_REGKEY(_PRIVATE, ssg_control, 448 NV_SWITCH_REGKEY_SSG_CONTROL, 449 0); 450 451 NVSWITCH_INIT_REGKEY(_PRIVATE, skip_buffer_ready, 452 NV_SWITCH_REGKEY_SKIP_BUFFER_READY, 453 0); 454 455 NVSWITCH_INIT_REGKEY(_PRIVATE, enable_pm, 456 NV_SWITCH_REGKEY_ENABLE_PM, 457 NV_SWITCH_REGKEY_ENABLE_PM_YES); 458 459 NVSWITCH_INIT_REGKEY(_PRIVATE, chiplib_forced_config_link_mask, 460 NV_SWITCH_REGKEY_CHIPLIB_FORCED_LINK_CONFIG_MASK, 461 0); 462 463 NVSWITCH_INIT_REGKEY(_PRIVATE, chiplib_forced_config_link_mask2, 464 NV_SWITCH_REGKEY_CHIPLIB_FORCED_LINK_CONFIG_MASK2, 465 0); 466 467 NVSWITCH_INIT_REGKEY(_PRIVATE, soe_dma_self_test, 468 NV_SWITCH_REGKEY_SOE_DMA_SELFTEST, 469 NV_SWITCH_REGKEY_SOE_DMA_SELFTEST_ENABLE); 470 471 NVSWITCH_INIT_REGKEY(_PRIVATE, soe_disable, 472 NV_SWITCH_REGKEY_SOE_DISABLE, 473 NV_SWITCH_REGKEY_SOE_DISABLE_NO); 474 475 NVSWITCH_INIT_REGKEY(_PRIVATE, latency_counter, 476 NV_SWITCH_REGKEY_LATENCY_COUNTER_LOGGING, 477 NV_SWITCH_REGKEY_LATENCY_COUNTER_LOGGING_ENABLE); 478 479 NVSWITCH_INIT_REGKEY(_PRIVATE, nvlink_speed_control, 480 NV_SWITCH_REGKEY_SPEED_CONTROL, 481 NV_SWITCH_REGKEY_SPEED_CONTROL_SPEED_DEFAULT); 482 483 NVSWITCH_INIT_REGKEY(_PRIVATE, inforom_bbx_periodic_flush, 484 NV_SWITCH_REGKEY_INFOROM_BBX_ENABLE_PERIODIC_FLUSHING, 485 NV_SWITCH_REGKEY_INFOROM_BBX_ENABLE_PERIODIC_FLUSHING_DISABLE); 486 487 NVSWITCH_INIT_REGKEY(_PRIVATE, inforom_bbx_write_periodicity, 488 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_PERIODICITY, 489 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_PERIODICITY_DEFAULT); 490 491 NVSWITCH_INIT_REGKEY(_PRIVATE, inforom_bbx_write_min_duration, 492 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_MIN_DURATION, 493 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_MIN_DURATION_DEFAULT); 494 495 NVSWITCH_INIT_REGKEY(_PRIVATE, minion_disable, 496 NV_SWITCH_REGKEY_MINION_DISABLE, 497 NV_SWITCH_REGKEY_MINION_DISABLE_NO); 498 499 NVSWITCH_INIT_REGKEY(_PRIVATE, set_ucode_target, 500 NV_SWITCH_REGKEY_MINION_SET_UCODE_TARGET, 501 NV_SWITCH_REGKEY_MINION_SET_UCODE_TARGET_DEFAULT); 502 503 NVSWITCH_INIT_REGKEY(_PRIVATE, set_simmode, 504 NV_SWITCH_REGKEY_MINION_SET_SIMMODE, 505 NV_SWITCH_REGKEY_MINION_SET_SIMMODE_DEFAULT); 506 507 NVSWITCH_INIT_REGKEY(_PRIVATE, set_smf_settings, 508 NV_SWITCH_REGKEY_MINION_SET_SMF_SETTINGS, 509 NV_SWITCH_REGKEY_MINION_SET_SMF_SETTINGS_DEFAULT); 510 511 NVSWITCH_INIT_REGKEY(_PRIVATE, select_uphy_tables, 512 NV_SWITCH_REGKEY_MINION_SELECT_UPHY_TABLES, 513 NV_SWITCH_REGKEY_MINION_SELECT_UPHY_TABLES_DEFAULT); 514 515 NVSWITCH_INIT_REGKEY(_PRIVATE, link_training_mode, 516 NV_SWITCH_REGKEY_LINK_TRAINING_SELECT, 517 NV_SWITCH_REGKEY_LINK_TRAINING_SELECT_DEFAULT); 518 519 NVSWITCH_INIT_REGKEY(_PRIVATE, i2c_access_control, 520 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL, 521 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_DEFAULT); 522 523 NVSWITCH_INIT_REGKEY(_PRIVATE, force_kernel_i2c, 524 NV_SWITCH_REGKEY_FORCE_KERNEL_I2C, 525 NV_SWITCH_REGKEY_FORCE_KERNEL_I2C_DEFAULT); 526 527 NVSWITCH_INIT_REGKEY(_PRIVATE, link_recal_settings, 528 NV_SWITCH_REGKEY_LINK_RECAL_SETTINGS, 529 NV_SWITCH_REGKEY_LINK_RECAL_SETTINGS_NOP); 530 531 NVSWITCH_INIT_REGKEY(_PRIVATE, lp_threshold, 532 NV_SWITCH_REGKEY_SET_LP_THRESHOLD, 533 NV_SWITCH_REGKEY_SET_LP_THRESHOLD_DEFAULT); 534 535 NVSWITCH_INIT_REGKEY(_PUBLIC, minion_intr, 536 NV_SWITCH_REGKEY_MINION_INTERRUPTS, 537 NV_SWITCH_REGKEY_MINION_INTERRUPTS_DEFAULT); 538 539 NVSWITCH_INIT_REGKEY(_PRIVATE, block_code_mode, 540 NV_SWITCH_REGKEY_BLOCK_CODE_MODE, 541 NV_SWITCH_REGKEY_BLOCK_CODE_MODE_DEFAULT); 542 543 NVSWITCH_INIT_REGKEY(_PRIVATE, reference_clock_mode, 544 NV_SWITCH_REGKEY_REFERENCE_CLOCK_MODE, 545 NV_SWITCH_REGKEY_REFERENCE_CLOCK_MODE_DEFAULT); 546 } 547 NvU64 548 nvswitch_lib_deferred_task_dispatcher 549 ( 550 nvswitch_device *device 551 ) 552 { 553 NvU64 time_nsec; 554 NvU64 time_next_nsec = nvswitch_os_get_platform_time() + 100*NVSWITCH_INTERVAL_1MSEC_IN_NS; 555 NVSWITCH_TASK_TYPE *task; 556 NVSWITCH_TASK_TYPE *prev_task; 557 558 if (!NVSWITCH_IS_DEVICE_VALID(device)) 559 { 560 return NV_U64_MAX; 561 } 562 563 prev_task = NULL; 564 task = device->tasks; 565 566 // Walk the task list, executing those whose next execution interval is at hand 567 while (task) 568 { 569 // Get current time (nsec) for scheduling 570 time_nsec = nvswitch_os_get_platform_time(); 571 572 if (time_nsec >= task->last_run_nsec + task->period_nsec) 573 { 574 // 575 // The task has never been run or it is time to run 576 // Mark its last run time 577 // 578 task->last_run_nsec = time_nsec; 579 // Run the task 580 if (NVSWITCH_IS_DEVICE_INITIALIZED(device) || 581 (task->flags & NVSWITCH_TASK_TYPE_FLAGS_RUN_EVEN_IF_DEVICE_NOT_INITIALIZED)) 582 { 583 if(task->flags & NVSWITCH_TASK_TYPE_FLAGS_VOID_PTR_ARGS) 584 (*task->task_fn_vdptr)(device, task->task_args); // run task with provided args 585 else 586 (*task->task_fn_devptr)(device); 587 } 588 } 589 590 // Determine its next run time 591 time_next_nsec = NV_MIN(task->last_run_nsec + task->period_nsec, time_next_nsec); 592 593 // Advance pointer. If run once flag is set and task ran, remove task from list. 594 if((task->flags & NVSWITCH_TASK_TYPE_FLAGS_RUN_ONCE) && 595 (task->last_run_nsec == time_nsec)) 596 { 597 prev_task = task->prev; 598 599 // Removing from list head 600 if (prev_task == NULL) 601 { 602 device->tasks = task->next; 603 if (device->tasks != NULL) 604 { 605 device->tasks->prev = NULL; 606 } 607 nvswitch_os_free(task); 608 task = device->tasks; 609 } 610 else 611 { 612 prev_task->next = task->next; 613 if (prev_task->next != NULL) 614 { 615 prev_task->next->prev = prev_task; 616 } 617 nvswitch_os_free(task); 618 task = prev_task->next; 619 } 620 } 621 else 622 { 623 task = task->next; 624 } 625 } 626 627 time_nsec = nvswitch_os_get_platform_time(); 628 629 // Return to the OS layer how long to wait before calling again 630 return(time_next_nsec >= time_nsec ? time_next_nsec - time_nsec : 0); 631 } 632 633 static NvlStatus 634 _nvswitch_setup_hal 635 ( 636 nvswitch_device *device, 637 NvU32 pci_device_id 638 ) 639 { 640 if (nvswitch_is_lr10_device_id(pci_device_id)) 641 { 642 nvswitch_setup_hal_lr10(device); 643 return NVL_SUCCESS; 644 } 645 if (nvswitch_is_ls10_device_id(pci_device_id)) 646 { 647 nvswitch_setup_hal_ls10(device); 648 return NVL_SUCCESS; 649 } 650 NVSWITCH_PRINT(device, ERROR, 651 "NVSwitch HAL setup failed - Unrecognized PCI Device ID\n"); 652 return -NVL_ERR_NOT_SUPPORTED; 653 } 654 655 NvlStatus 656 nvswitch_lib_check_api_version 657 ( 658 const char *user_version, 659 char *kernel_version, 660 NvU32 length 661 ) 662 { 663 const NvLength VERSION_LENGTH = nvswitch_os_strlen(NV_VERSION_STRING); 664 665 if (kernel_version == NULL || user_version == NULL) 666 { 667 return -NVL_BAD_ARGS; 668 } 669 670 if (length < VERSION_LENGTH) 671 { 672 return -NVL_NO_MEM; 673 } 674 675 nvswitch_os_memset(kernel_version, 0x0, length); 676 nvswitch_os_strncpy(kernel_version, NV_VERSION_STRING, VERSION_LENGTH); 677 678 kernel_version[length - 1] = '\0'; 679 680 if (nvswitch_os_strncmp(user_version, kernel_version, VERSION_LENGTH)) 681 { 682 return -NVL_ERR_NOT_SUPPORTED; 683 } 684 685 return NVL_SUCCESS; 686 } 687 688 NvBool 689 nvswitch_is_inforom_supported 690 ( 691 nvswitch_device *device 692 ) 693 { 694 return device->hal.nvswitch_is_inforom_supported(device); 695 } 696 697 NvBool 698 nvswitch_is_spi_supported 699 ( 700 nvswitch_device *device 701 ) 702 { 703 return device->hal.nvswitch_is_spi_supported(device); 704 } 705 706 NvBool 707 nvswitch_is_smbpbi_supported 708 ( 709 nvswitch_device *device 710 ) 711 { 712 return device->hal.nvswitch_is_smbpbi_supported(device); 713 } 714 715 NvBool 716 nvswitch_is_soe_supported 717 ( 718 nvswitch_device *device 719 ) 720 { 721 if (device->regkeys.soe_disable == NV_SWITCH_REGKEY_SOE_DISABLE_YES) 722 { 723 NVSWITCH_PRINT(device, INFO, "SOE is disabled via regkey.\n"); 724 return NV_FALSE; 725 } 726 727 return device->hal.nvswitch_is_soe_supported(device); 728 } 729 730 731 NvlStatus 732 nvswitch_init_soe 733 ( 734 nvswitch_device *device 735 ) 736 { 737 if (device->regkeys.soe_disable == NV_SWITCH_REGKEY_SOE_DISABLE_YES) 738 { 739 NVSWITCH_PRINT(device, INFO, "SOE is disabled via regkey.\n"); 740 return NV_FALSE; 741 } 742 743 return device->hal.nvswitch_init_soe(device); 744 } 745 746 void 747 nvswitch_soe_init_l2_state 748 ( 749 nvswitch_device *device 750 ) 751 { 752 device->hal.nvswitch_soe_init_l2_state(device); 753 } 754 755 static NvlStatus 756 _nvswitch_construct_soe 757 ( 758 nvswitch_device *device 759 ) 760 { 761 FLCNABLE *pSoe = NULL; 762 NvlStatus retval; 763 764 device->pSoe = pSoe = (PFLCNABLE)soeAllocNew(); 765 if (pSoe == NULL) 766 { 767 NVSWITCH_PRINT(device, ERROR, "SOE allocation failed.\n"); 768 return -NVL_NO_MEM; 769 } 770 771 retval = soeInit(device, (PSOE)pSoe, device->nvlink_device->pciInfo.pciDeviceId); 772 if (retval != NVL_SUCCESS) 773 { 774 NVSWITCH_PRINT(device, ERROR, "SOE init failed.\n"); 775 goto soe_init_failed; 776 } 777 778 if (flcnableConstruct_HAL(device, pSoe) != NV_OK) 779 { 780 NVSWITCH_PRINT(device, ERROR, "FALCON construct failed.\n"); 781 retval = -NVL_ERR_INVALID_STATE; 782 goto flcn_construct_failed; 783 } 784 785 return NVL_SUCCESS; 786 787 flcn_construct_failed: 788 soeDestroy(device, (PSOE)pSoe); 789 790 soe_init_failed: 791 nvswitch_os_free(pSoe); 792 device->pSoe = NULL; 793 794 return retval; 795 } 796 797 static void 798 _nvswitch_destruct_soe 799 ( 800 nvswitch_device *device 801 ) 802 { 803 FLCNABLE *pSoe = device->pSoe; 804 805 if (pSoe == NULL) 806 { 807 return; 808 } 809 810 flcnableDestruct_HAL(device, pSoe); 811 soeDestroy(device, (PSOE)pSoe); 812 813 nvswitch_os_free(pSoe); 814 device->pSoe = NULL; 815 } 816 817 static NvlStatus 818 _nvswitch_initialize_device_state 819 ( 820 nvswitch_device *device 821 ) 822 { 823 return device->hal.nvswitch_initialize_device_state(device); 824 } 825 826 static NvlStatus 827 _nvswitch_post_init_device_setup 828 ( 829 nvswitch_device *device 830 ) 831 { 832 return device->hal.nvswitch_post_init_device_setup(device); 833 } 834 835 static NvlStatus 836 _nvswitch_setup_system_registers 837 ( 838 nvswitch_device *device 839 ) 840 { 841 return device->hal.nvswitch_setup_system_registers(device); 842 } 843 844 static void 845 _nvswitch_post_init_blacklist_device_setup 846 ( 847 nvswitch_device *device 848 ) 849 { 850 device->hal.nvswitch_post_init_blacklist_device_setup(device); 851 } 852 853 static void 854 _nvswitch_set_dma_mask 855 ( 856 nvswitch_device *device 857 ) 858 { 859 NvU32 hw_dma_width, retval; 860 861 hw_dma_width = device->hal.nvswitch_get_device_dma_width(device); 862 863 if (hw_dma_width == 0) 864 { 865 NVSWITCH_PRINT(device, INFO, "DMA is not supported on this device\n"); 866 return; 867 } 868 869 retval = nvswitch_os_set_dma_mask(device->os_handle, hw_dma_width); 870 if (retval == NVL_SUCCESS) 871 { 872 device->dma_addr_width = hw_dma_width; 873 return; 874 } 875 876 NVSWITCH_PRINT(device, SETUP, 877 "%s: Failed to set DMA mask, trying 32-bit fallback : %d\n", 878 __FUNCTION__, retval); 879 880 retval = nvswitch_os_set_dma_mask(device->os_handle, 32); 881 if (retval == NVL_SUCCESS) 882 { 883 device->dma_addr_width = 32; 884 return; 885 } 886 887 // failure is not fatal, the driver will just restrict DMA functionality 888 NVSWITCH_PRINT(device, ERROR, "Failed to set DMA mask : %d\n", retval); 889 } 890 891 NvlStatus 892 nvswitch_deassert_link_reset 893 ( 894 nvswitch_device *device, 895 nvlink_link *link 896 ) 897 { 898 return device->hal.nvswitch_deassert_link_reset(device, link); 899 } 900 901 NvU32 902 nvswitch_get_sublink_width 903 ( 904 nvswitch_device *device, 905 NvU32 linkNumber 906 ) 907 { 908 return device->hal.nvswitch_get_sublink_width(device, linkNumber); 909 } 910 911 static void 912 _nvswitch_unregister_links 913 ( 914 nvswitch_device *device 915 ) 916 { 917 nvlink_link *link = NULL; 918 NvU32 link_num; 919 NvBool is_blacklisted; 920 921 922 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 923 return; 924 925 device->nvlink_device->initialized = 0; 926 is_blacklisted = (device->device_fabric_state == NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED); 927 928 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 929 { 930 if (nvlink_lib_get_link(device->nvlink_device, link_num, &link) == NVL_SUCCESS) 931 { 932 nvlink_lib_unregister_link(link); 933 nvswitch_destroy_link(link); 934 } 935 } 936 937 if (!is_blacklisted) 938 nvswitch_inforom_nvlink_flush(device); 939 } 940 941 NvlStatus NV_API_CALL 942 nvswitch_lib_read_fabric_state 943 ( 944 nvswitch_device *device, 945 NVSWITCH_DEVICE_FABRIC_STATE *device_fabric_state, 946 NVSWITCH_DEVICE_BLACKLIST_REASON *device_blacklist_reason, 947 NVSWITCH_DRIVER_FABRIC_STATE *driver_fabric_state 948 ) 949 { 950 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 951 return -NVL_BAD_ARGS; 952 953 if (device_fabric_state != NULL) 954 *device_fabric_state = device->device_fabric_state; 955 956 if (device_blacklist_reason != NULL) 957 *device_blacklist_reason = device->device_blacklist_reason; 958 959 if (driver_fabric_state != NULL) 960 *driver_fabric_state = device->driver_fabric_state; 961 962 return NVL_SUCCESS; 963 } 964 965 static NvlStatus 966 nvswitch_lib_blacklist_device 967 ( 968 nvswitch_device *device, 969 NVSWITCH_DEVICE_BLACKLIST_REASON device_blacklist_reason 970 ) 971 { 972 NvlStatus status; 973 974 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 975 { 976 return -NVL_BAD_ARGS; 977 } 978 979 if (device->device_fabric_state == NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED) 980 { 981 NVSWITCH_PRINT(device, WARN, "Device is already blacklisted\n"); 982 return -NVL_ERR_NOT_SUPPORTED; 983 } 984 985 device->device_fabric_state = NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED; 986 device->device_blacklist_reason = device_blacklist_reason; 987 988 status = device->hal.nvswitch_write_fabric_state(device); 989 if (status != NVL_SUCCESS) 990 NVSWITCH_PRINT(device, INFO, "Cannot send fabric state to SOE\n"); 991 992 return NVL_SUCCESS; 993 } 994 995 static NvlStatus 996 nvswitch_ctrl_blacklist_device( 997 nvswitch_device *device, 998 NVSWITCH_BLACKLIST_DEVICE_PARAMS *p 999 ) 1000 { 1001 NvlStatus status; 1002 1003 status = nvswitch_lib_blacklist_device(device, p->deviceReason); 1004 if (status != NVL_SUCCESS) 1005 return status; 1006 1007 nvswitch_lib_disable_interrupts(device); 1008 1009 // Unregister links from NVLinkCoreLib, so that link training is not 1010 // attempted 1011 _nvswitch_unregister_links(device); 1012 1013 // Keep device registered for HAL access and Fabric State updates 1014 1015 return NVL_SUCCESS; 1016 } 1017 1018 static NvlStatus 1019 nvswitch_ctrl_set_fm_driver_state( 1020 nvswitch_device *device, 1021 NVSWITCH_SET_FM_DRIVER_STATE_PARAMS *p 1022 ) 1023 { 1024 NvU32 prev_fm_status; 1025 1026 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1027 { 1028 return -NVL_BAD_ARGS; 1029 } 1030 1031 prev_fm_status = device->driver_fabric_state; 1032 device->driver_fabric_state = p->driverState; 1033 device->fabric_state_timestamp = nvswitch_os_get_platform_time(); 1034 1035 if (prev_fm_status != p->driverState) 1036 { 1037 if (nvswitch_lib_notify_client_events(device, 1038 NVSWITCH_DEVICE_EVENT_FABRIC_STATE) != NVL_SUCCESS) 1039 { 1040 NVSWITCH_PRINT(device, ERROR, "%s: Failed to notify event\n", 1041 __FUNCTION__); 1042 } 1043 } 1044 1045 return NVL_SUCCESS; 1046 } 1047 1048 static NvlStatus 1049 nvswitch_ctrl_set_device_fabric_state( 1050 nvswitch_device *device, 1051 NVSWITCH_SET_DEVICE_FABRIC_STATE_PARAMS *p 1052 ) 1053 { 1054 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1055 { 1056 return -NVL_BAD_ARGS; 1057 } 1058 1059 if (device->device_fabric_state == NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED) 1060 return -NVL_ERR_NOT_SUPPORTED; 1061 1062 device->device_fabric_state = p->deviceState; 1063 device->fabric_state_timestamp = nvswitch_os_get_platform_time(); 1064 1065 // If FM had exceeded timeout, reset the status to not timed-out 1066 if (device->driver_fabric_state == NVSWITCH_DRIVER_FABRIC_STATE_MANAGER_TIMEOUT) 1067 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED; 1068 1069 return NVL_SUCCESS; 1070 } 1071 1072 static NvlStatus 1073 nvswitch_ctrl_set_fm_timeout( 1074 nvswitch_device *device, 1075 NVSWITCH_SET_FM_HEARTBEAT_TIMEOUT_PARAMS *p 1076 ) 1077 { 1078 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1079 { 1080 return -NVL_BAD_ARGS; 1081 } 1082 1083 device->fm_timeout = p->fmTimeout; 1084 1085 return NVL_SUCCESS; 1086 } 1087 1088 static NvlStatus 1089 _nvswitch_ctrl_register_events( 1090 nvswitch_device *device, 1091 NVSWITCH_REGISTER_EVENTS_PARAMS *p, 1092 void *osPrivate 1093 ) 1094 { 1095 NvlStatus status = NVL_SUCCESS; 1096 NvU32 i; 1097 NvBool many_events, os_descriptor; 1098 void *osDescriptor = osPrivate; 1099 1100 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1101 { 1102 return -NVL_BAD_ARGS; 1103 } 1104 1105 status = nvswitch_os_get_supported_register_events_params(&many_events, 1106 &os_descriptor); 1107 if (status != NVL_SUCCESS) 1108 { 1109 return status; 1110 } 1111 1112 if ((!many_events && (p->numEvents > 1)) || 1113 (p->numEvents == 0)) 1114 { 1115 return -NVL_BAD_ARGS; 1116 } 1117 1118 if (os_descriptor) 1119 { 1120 osDescriptor = (void *) p->osDescriptor; 1121 } 1122 1123 for (i = 0; i < p->numEvents; i++) 1124 { 1125 status = nvswitch_lib_add_client_event(device, osDescriptor, p->eventIds[i]); 1126 if (status != NVL_SUCCESS) 1127 { 1128 NVSWITCH_PRINT(device, ERROR, "%s: Failed to add client event.\n", __FUNCTION__); 1129 return status; 1130 } 1131 } 1132 1133 return NVL_SUCCESS; 1134 } 1135 1136 static NvlStatus 1137 _nvswitch_ctrl_unregister_events( 1138 nvswitch_device *device, 1139 NVSWITCH_UNREGISTER_EVENTS_PARAMS *p, 1140 void *osPrivate 1141 ) 1142 { 1143 NvlStatus status = NVL_SUCCESS; 1144 NvBool many_events, os_descriptor; 1145 void *osDescriptor = osPrivate; 1146 1147 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1148 { 1149 return -NVL_BAD_ARGS; 1150 } 1151 1152 status = nvswitch_os_get_supported_register_events_params(&many_events, 1153 &os_descriptor); 1154 if (status != NVL_SUCCESS) 1155 { 1156 return status; 1157 } 1158 1159 if (os_descriptor) 1160 { 1161 osDescriptor = (void *) p->osDescriptor; 1162 } 1163 1164 status = nvswitch_lib_remove_client_events(device, osDescriptor); 1165 if (status != NVL_SUCCESS) 1166 { 1167 NVSWITCH_PRINT(device, ERROR, "%s: Failed to remove client event.\n", __FUNCTION__); 1168 return status; 1169 } 1170 1171 return NVL_SUCCESS; 1172 } 1173 1174 /* 1175 * @Brief : Sends NACK or drops given inband msg based on message type 1176 * 1177 * @Description : 1178 * 1179 * @param[in] device NvSwitch device to contain this link 1180 * @param[in] linkId Link ID 1181 * @param[in] msghdr Header to the message 1182 * 1183 */ 1184 static void 1185 nvswitch_send_nack_or_drop 1186 ( 1187 nvswitch_device *device, 1188 NvU32 linkId, 1189 nvlink_inband_msg_header_t *msghdr 1190 ) 1191 { 1192 switch(msghdr->type) 1193 { 1194 case NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ: 1195 device->hal.nvswitch_send_inband_nack(device, (NvU32 *)msghdr, linkId); 1196 NVSWITCH_PRINT(device, ERROR, 1197 "Sending NACK for message (type 0x%x)\n", msghdr->type); 1198 return; 1199 default: 1200 // TODO: Add SXid in future if needed. 1201 NVSWITCH_PRINT(device, ERROR, 1202 "Dropping message (type 0x%x)\n", msghdr->type); 1203 return; 1204 } 1205 } 1206 1207 /* 1208 * @Brief : Deletes all the entries in persistent or non-persistent lists. 1209 * Send nacks if requested. 1210 * 1211 * @Description : 1212 * 1213 * @param[in] device NVSwitch device to contain this link 1214 * @param[in] linkId Link number 1215 * @param[in] bSendNack Send nacks if true 1216 * @param[in] bNonPersistentOnly Clear only non-persistent list 1217 */ 1218 static void 1219 _nvswitch_inband_clear_lists 1220 ( 1221 nvswitch_device *device, 1222 NvU32 linkId, 1223 NvBool bSendNack, 1224 NvBool bNonPersistentOnly 1225 ) 1226 { 1227 nvswitch_inband_data_list *curr = NULL; 1228 nvswitch_inband_data_list *next = NULL; 1229 nvlink_inband_msg_header_t *msghdr = NULL; 1230 1231 nvListForEachEntry_safe(curr, next, 1232 &device->link[linkId].inbandData.nonpersistent_list, entry) 1233 { 1234 if (bSendNack) 1235 { 1236 msghdr = (nvlink_inband_msg_header_t *)curr->data; 1237 nvswitch_send_nack_or_drop(device, linkId, msghdr); 1238 } 1239 1240 nvListDel(&curr->entry); 1241 nvswitch_os_free(curr); 1242 } 1243 1244 if (bNonPersistentOnly) 1245 return; 1246 1247 nvListForEachEntry_safe(curr, next, 1248 &device->link[linkId].inbandData.persistent_list, entry) 1249 { 1250 if (bSendNack) 1251 { 1252 msghdr = (nvlink_inband_msg_header_t *)curr->data; 1253 nvswitch_send_nack_or_drop(device, linkId, msghdr); 1254 } 1255 1256 nvListDel(&curr->entry); 1257 nvswitch_os_free(curr); 1258 } 1259 } 1260 1261 static void 1262 nvswitch_fabric_state_heartbeat( 1263 nvswitch_device *device 1264 ) 1265 { 1266 NvU64 age; 1267 NvU32 linkId; 1268 1269 if (!NVSWITCH_IS_DEVICE_VALID(device)) 1270 return; 1271 1272 age = nvswitch_os_get_platform_time() - device->fabric_state_timestamp; 1273 1274 // Check to see if we have exceeded the FM timeout 1275 if (device->driver_fabric_state == NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED && 1276 age > (NvU64)device->fm_timeout * 1000ULL * 1000ULL) 1277 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_MANAGER_TIMEOUT; 1278 1279 // 1280 // If FM is not running, clear pending non-persistent messages. Persistent 1281 // messages can be processed by the FM when it restarts. 1282 // 1283 if (device->driver_fabric_state != NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED) 1284 { 1285 for (linkId = 0; linkId < nvswitch_get_num_links(device); linkId++) 1286 _nvswitch_inband_clear_lists(device, linkId, 1287 NV_TRUE /* Nack */, 1288 NV_TRUE /* Non-persistent only */); 1289 } 1290 1291 (void)device->hal.nvswitch_write_fabric_state(device); 1292 } 1293 1294 static NvlStatus 1295 _nvswitch_ctrl_set_training_error_info 1296 ( 1297 nvswitch_device *device, 1298 NVSWITCH_SET_TRAINING_ERROR_INFO_PARAMS *p 1299 ) 1300 { 1301 return device->hal.nvswitch_set_training_error_info(device, p); 1302 } 1303 1304 static NvlStatus 1305 _nvswitch_ctrl_get_fatal_error_scope 1306 ( 1307 nvswitch_device *device, 1308 NVSWITCH_GET_FATAL_ERROR_SCOPE_PARAMS *pParams 1309 ) 1310 { 1311 return device->hal.nvswitch_ctrl_get_fatal_error_scope(device, pParams); 1312 } 1313 1314 static NvlStatus 1315 _nvswitch_ctrl_therm_get_temperature_limit 1316 ( 1317 nvswitch_device *device, 1318 NVSWITCH_CTRL_GET_TEMPERATURE_LIMIT_PARAMS *pParams 1319 ) 1320 { 1321 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1322 { 1323 return -NVL_BAD_ARGS; 1324 } 1325 1326 return device->hal.nvswitch_ctrl_therm_get_temperature_limit(device, pParams); 1327 } 1328 1329 NvlStatus 1330 nvswitch_lib_initialize_device 1331 ( 1332 nvswitch_device *device 1333 ) 1334 { 1335 NvlStatus retval = NVL_SUCCESS; 1336 NvU8 link_num; 1337 nvlink_link *link = NULL; 1338 NvBool is_blacklisted_by_os = NV_FALSE; 1339 NvU64 mode; 1340 1341 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1342 { 1343 return -NVL_BAD_ARGS; 1344 } 1345 1346 if (NVSWITCH_IS_DEVICE_INITIALIZED(device)) 1347 { 1348 NVSWITCH_PRINT(device, SETUP, "Device is already initialized!\n"); 1349 return NVL_SUCCESS; 1350 } 1351 1352 NVSWITCH_PRINT(device, SETUP, 1353 "Initializing nvswitch at (%04x:%02x:%02x.%02x)\n", 1354 device->nvlink_device->pciInfo.domain, 1355 device->nvlink_device->pciInfo.bus, 1356 device->nvlink_device->pciInfo.device, 1357 device->nvlink_device->pciInfo.function); 1358 1359 nvListInit(&device->client_events_list); 1360 1361 for (link_num=0; link_num < nvswitch_get_num_links(device); link_num++) 1362 { 1363 nvListInit(&device->link[link_num].inbandData.persistent_list); 1364 nvListInit(&device->link[link_num].inbandData.nonpersistent_list); 1365 } 1366 1367 retval = nvswitch_lib_load_platform_info(device); 1368 if (retval != NVL_SUCCESS) 1369 { 1370 NVSWITCH_PRINT(device, ERROR, "Failed to load platform information\n"); 1371 return retval; 1372 } 1373 1374 if (nvswitch_is_soe_supported(device)) 1375 { 1376 retval = _nvswitch_construct_soe(device); 1377 if (retval != NVL_SUCCESS) 1378 { 1379 return retval; 1380 } 1381 } 1382 else 1383 { 1384 NVSWITCH_PRINT(device, INFO, "SOE is not supported, skipping construct\n"); 1385 } 1386 1387 _nvswitch_set_dma_mask(device); 1388 1389 retval = _nvswitch_initialize_device_state(device); 1390 if (NVL_SUCCESS != retval) 1391 { 1392 NVSWITCH_PRINT(device, ERROR, 1393 "Failed to initialize device state: %d!\n", 1394 retval); 1395 goto nvswitch_initialize_device_state_fail; 1396 } 1397 1398 device->hal.nvswitch_load_uuid(device); 1399 1400 /* 1401 * Check module parameters for blacklisted device 1402 */ 1403 if (nvswitch_os_is_uuid_in_blacklist(&device->uuid) == NV_TRUE) 1404 { 1405 NVSWITCH_PRINT(device, SETUP, 1406 "Blacklisted nvswitch at (%04x:%02x:%02x.%02x)\n", 1407 device->nvlink_device->pciInfo.domain, 1408 device->nvlink_device->pciInfo.bus, 1409 device->nvlink_device->pciInfo.device, 1410 device->nvlink_device->pciInfo.function); 1411 is_blacklisted_by_os = NV_TRUE; 1412 // initialization continues until we have updated InfoROM... 1413 } 1414 1415 if (nvswitch_is_inforom_supported(device)) 1416 { 1417 retval = nvswitch_initialize_inforom(device); 1418 if (NVL_SUCCESS != retval) 1419 { 1420 NVSWITCH_PRINT(device, ERROR, 1421 "Failed to initialize InfoROM rc: %d\n", 1422 retval); 1423 goto nvswitch_initialize_device_state_fail; 1424 } 1425 1426 retval = nvswitch_initialize_inforom_objects(device); 1427 if (NVL_SUCCESS != retval) 1428 { 1429 NVSWITCH_PRINT(device, ERROR, 1430 "Failed to initialize InfoROM objects! rc:%d\n", 1431 retval); 1432 goto nvswitch_initialize_inforom_fail; 1433 } 1434 } 1435 else 1436 { 1437 NVSWITCH_PRINT(device, INFO, 1438 "InfoROM is not supported, skipping init\n"); 1439 } 1440 1441 (void)device->hal.nvswitch_read_oob_blacklist_state(device); 1442 (void)device->hal.nvswitch_write_fabric_state(device); 1443 1444 nvswitch_task_create(device, &nvswitch_fabric_state_heartbeat, 1445 NVSWITCH_HEARTBEAT_INTERVAL_NS, 1446 NVSWITCH_TASK_TYPE_FLAGS_RUN_EVEN_IF_DEVICE_NOT_INITIALIZED); 1447 1448 // 1449 // Blacklisted devices return successfully in order to preserve the fabric state heartbeat 1450 // and ensure OOB utilities don't think the driver has died 1451 // 1452 if (device->device_blacklist_reason == NVSWITCH_DEVICE_BLACKLIST_REASON_MANUAL_OUT_OF_BAND) 1453 { 1454 NVSWITCH_PRINT(device, SETUP, 1455 "Blacklisted nvswitch at (%04x:%02x:%02x.%02x)\n", 1456 device->nvlink_device->pciInfo.domain, 1457 device->nvlink_device->pciInfo.bus, 1458 device->nvlink_device->pciInfo.device, 1459 device->nvlink_device->pciInfo.function); 1460 return NVL_SUCCESS; 1461 } 1462 1463 if (is_blacklisted_by_os) 1464 { 1465 (void)nvswitch_lib_blacklist_device(device, NVSWITCH_DEVICE_BLACKLIST_REASON_MANUAL_IN_BAND); 1466 return NVL_SUCCESS; 1467 } 1468 1469 for (link_num=0; link_num < nvswitch_get_num_links(device); link_num++) 1470 { 1471 if (!nvswitch_is_link_valid(device, link_num)) 1472 { 1473 continue; 1474 } 1475 1476 retval = nvswitch_create_link(device, link_num, &link); 1477 if (NVL_SUCCESS != retval) 1478 { 1479 NVSWITCH_PRINT(device, ERROR, 1480 "Failed to create link %d : %d!\n", 1481 link_num, 1482 retval); 1483 goto nvswitch_link_fail; 1484 } 1485 1486 retval = nvlink_lib_register_link(device->nvlink_device, link); 1487 if (NVL_SUCCESS != retval) 1488 { 1489 NVSWITCH_PRINT(device, ERROR, 1490 "Failed to register link %d with the nvlink core : %d!\n", 1491 link_num, 1492 retval); 1493 1494 // Free the single dangling link. 1495 nvswitch_destroy_link(link); 1496 1497 goto nvswitch_link_fail; 1498 } 1499 1500 nvswitch_reset_persistent_link_hw_state(device, link_num); 1501 1502 if(_nvswitch_corelib_get_dl_link_mode(link, &mode) != NVL_SUCCESS) 1503 { 1504 NVSWITCH_PRINT(device, ERROR, "%s: nvlipt_lnk_status: Failed to check link mode! LinkId %d\n", 1505 __FUNCTION__, link_num); 1506 } 1507 else if(mode == NVLINK_LINKSTATE_FAULT) 1508 { 1509 NVSWITCH_PRINT(device, INFO, "%s: retraining LinkId %d\n", 1510 __FUNCTION__, link_num); 1511 nvswitch_reset_and_train_link(device, link); 1512 } 1513 1514 } 1515 1516 retval = nvswitch_set_training_mode(device); 1517 1518 if (retval != NVL_SUCCESS) 1519 { 1520 NVSWITCH_PRINT(device, ERROR, "Failed to determine link training mode! rc: %d\n", retval); 1521 goto nvswitch_link_fail; 1522 } 1523 // Initialize select scratch registers to 0x0 1524 device->hal.nvswitch_init_scratch(device); 1525 1526 retval = nvswitch_construct_error_log(&device->log_FATAL_ERRORS, 1024, NV_FALSE); 1527 if (retval != NVL_SUCCESS) 1528 { 1529 NVSWITCH_PRINT(device, ERROR, "Failed to construct log_FATAL_ERRORS! rc: %d\n", retval); 1530 goto nvswitch_construct_error_log_fail; 1531 } 1532 1533 retval = nvswitch_construct_error_log(&device->log_NONFATAL_ERRORS, 1024, NV_TRUE); 1534 if (retval != NVL_SUCCESS) 1535 { 1536 NVSWITCH_PRINT(device, ERROR, "Failed to construct log_NONFATAL_ERRORS! rc: %d\n", retval); 1537 goto nvswitch_construct_error_log_fail; 1538 } 1539 1540 if (device->regkeys.latency_counter == NV_SWITCH_REGKEY_LATENCY_COUNTER_LOGGING_ENABLE) 1541 { 1542 nvswitch_task_create(device, &nvswitch_internal_latency_bin_log, 1543 nvswitch_get_latency_sample_interval_msec(device) * NVSWITCH_INTERVAL_1MSEC_IN_NS * 9/10, 0); 1544 } 1545 1546 nvswitch_task_create(device, &nvswitch_ecc_writeback_task, 1547 (60 * NVSWITCH_INTERVAL_1SEC_IN_NS), 0); 1548 1549 if (IS_RTLSIM(device) || IS_EMULATION(device) || IS_FMODEL(device)) 1550 { 1551 NVSWITCH_PRINT(device, WARN, 1552 "%s: Skipping setup of NvSwitch thermal alert monitoring\n", 1553 __FUNCTION__); 1554 } 1555 else 1556 { 1557 nvswitch_task_create(device, &nvswitch_monitor_thermal_alert, 1558 100*NVSWITCH_INTERVAL_1MSEC_IN_NS, 0); 1559 } 1560 1561 device->nvlink_device->initialized = 1; 1562 1563 return NVL_SUCCESS; 1564 1565 nvswitch_construct_error_log_fail: 1566 //free allocated memory to avoid leaking 1567 nvswitch_destroy_error_log(device, &device->log_FATAL_ERRORS); 1568 nvswitch_destroy_error_log(device, &device->log_NONFATAL_ERRORS); 1569 1570 nvswitch_link_fail: 1571 // Track down all links that successfully registered. 1572 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 1573 { 1574 if (nvlink_lib_get_link(device->nvlink_device, link_num, &link) == NVL_SUCCESS) 1575 { 1576 nvlink_lib_unregister_link(link); 1577 nvswitch_destroy_link(link); 1578 } 1579 } 1580 1581 nvswitch_destroy_inforom_objects(device); 1582 1583 nvswitch_initialize_inforom_fail: 1584 nvswitch_destroy_inforom(device); 1585 1586 nvswitch_initialize_device_state_fail: 1587 _nvswitch_destruct_soe(device); 1588 nvswitch_tasks_destroy(device); 1589 1590 return retval; 1591 } 1592 1593 NvBool 1594 nvswitch_lib_validate_device_id 1595 ( 1596 NvU32 device_id 1597 ) 1598 { 1599 if (nvswitch_is_lr10_device_id(device_id)) 1600 { 1601 return NV_TRUE; 1602 } 1603 if (nvswitch_is_ls10_device_id(device_id)) 1604 { 1605 return NV_TRUE; 1606 } 1607 return NV_FALSE; 1608 } 1609 1610 NvlStatus 1611 nvswitch_lib_post_init_device 1612 ( 1613 nvswitch_device *device 1614 ) 1615 { 1616 NvlStatus retval; 1617 1618 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 1619 { 1620 return -NVL_ERR_INVALID_STATE; 1621 } 1622 1623 retval = _nvswitch_post_init_device_setup(device); 1624 if (retval != NVL_SUCCESS) 1625 { 1626 return retval; 1627 } 1628 1629 if (nvswitch_is_spi_supported(device)) 1630 { 1631 retval = nvswitch_bios_get_image(device); 1632 if (retval != NVL_SUCCESS) 1633 { 1634 return retval; 1635 } 1636 1637 retval = nvswitch_parse_bios_image(device); 1638 if (retval != NVL_SUCCESS) 1639 { 1640 return retval; 1641 } 1642 } 1643 else 1644 { 1645 NVSWITCH_PRINT(device, ERROR, 1646 "%s: Skipping BIOS parsing since SPI is unsupported.\n", 1647 __FUNCTION__); 1648 } 1649 1650 retval = _nvswitch_setup_system_registers(device); 1651 if (retval != NVL_SUCCESS) 1652 { 1653 return retval; 1654 } 1655 1656 nvswitch_smbpbi_post_init(device); 1657 1658 (void)nvswitch_launch_ALI(device); 1659 1660 return NVL_SUCCESS; 1661 } 1662 1663 void 1664 nvswitch_lib_post_init_blacklist_device 1665 ( 1666 nvswitch_device *device 1667 ) 1668 { 1669 _nvswitch_post_init_blacklist_device_setup(device); 1670 } 1671 1672 void 1673 _nvswitch_check_pending_data_and_notify 1674 ( 1675 nvswitch_device *device, 1676 NVSWITCH_CLIENT_EVENT *event 1677 ) 1678 { 1679 switch (event->eventId) 1680 { 1681 case NVSWITCH_DEVICE_EVENT_INBAND_DATA: 1682 { 1683 NvU32 i; 1684 1685 for (i = 0; i < nvswitch_get_num_links(device); i++) 1686 { 1687 if (!nvListIsEmpty(&device->link[i].inbandData.persistent_list) || 1688 !nvListIsEmpty(&device->link[i].inbandData.nonpersistent_list)) 1689 { 1690 (void)nvswitch_os_notify_client_event(device->os_handle, 1691 event->private_driver_data, 1692 event->eventId); 1693 } 1694 } 1695 break; 1696 } 1697 1698 default: 1699 return; 1700 } 1701 } 1702 1703 /*! 1704 * @brief: Gets the client event associated with the file descriptor 1705 * if it already exists in the Device's client event list. 1706 * 1707 * If found, and if there is pending data for the event, 1708 * the event is triggered before returning to unblock the 1709 * client right away. 1710 */ 1711 NvlStatus 1712 nvswitch_lib_get_client_event 1713 ( 1714 nvswitch_device *device, 1715 void *osPrivate, 1716 NVSWITCH_CLIENT_EVENT **ppClientEvent 1717 ) 1718 { 1719 NVSWITCH_CLIENT_EVENT *curr = NULL; 1720 1721 *ppClientEvent = NULL; 1722 1723 if(!NVSWITCH_IS_DEVICE_VALID(device)) 1724 { 1725 return -NVL_BAD_ARGS; 1726 } 1727 1728 nvListForEachEntry(curr, &device->client_events_list, entry) 1729 { 1730 if (curr->private_driver_data == osPrivate) 1731 { 1732 *ppClientEvent = curr; 1733 _nvswitch_check_pending_data_and_notify(device, curr); 1734 return NVL_SUCCESS; 1735 } 1736 } 1737 1738 return -NVL_NOT_FOUND; 1739 } 1740 1741 /*! 1742 * @brief: Adds an event to the front of the 1743 * Device's client event list. 1744 */ 1745 NvlStatus 1746 nvswitch_lib_add_client_event 1747 ( 1748 nvswitch_device *device, 1749 void *osPrivate, 1750 NvU32 eventId 1751 ) 1752 { 1753 NVSWITCH_CLIENT_EVENT *newEvent; 1754 NvlStatus status = NVL_SUCCESS; 1755 1756 if (!NVSWITCH_IS_DEVICE_VALID(device)) 1757 { 1758 return -NVL_BAD_ARGS; 1759 } 1760 1761 if (eventId >= NVSWITCH_DEVICE_EVENT_COUNT) 1762 { 1763 NVSWITCH_PRINT(device, ERROR, "%s: Invalid event Id.\n", __FUNCTION__); 1764 return -NVL_BAD_ARGS; 1765 } 1766 1767 // Invoke OS specific API to add event. 1768 status = nvswitch_os_add_client_event(device->os_handle, 1769 osPrivate, 1770 eventId); 1771 if (status != NVL_SUCCESS) 1772 { 1773 NVSWITCH_PRINT(device, ERROR, "%s: Failed to add client event.\n", __FUNCTION__); 1774 return status; 1775 } 1776 1777 newEvent = nvswitch_os_malloc(sizeof(*newEvent)); 1778 if (newEvent == NULL) 1779 { 1780 return -NVL_NO_MEM; 1781 } 1782 1783 newEvent->eventId = eventId; 1784 newEvent->private_driver_data = osPrivate; 1785 1786 nvListAdd(&newEvent->entry, &device->client_events_list); 1787 1788 return NVL_SUCCESS; 1789 } 1790 1791 /*! 1792 * @brief: Removes all events corresponding to osPrivate, 1793 * from the Device's client event list. 1794 */ 1795 NvlStatus 1796 nvswitch_lib_remove_client_events 1797 ( 1798 nvswitch_device *device, 1799 void *osPrivate 1800 ) 1801 { 1802 NVSWITCH_CLIENT_EVENT *curr = NULL; 1803 NVSWITCH_CLIENT_EVENT *next = NULL; 1804 NvlStatus status = NVL_SUCCESS; 1805 1806 // 1807 // Device shutdown may happen before this is called, so return 1808 // if device is gone 1809 // 1810 if (!NVSWITCH_IS_DEVICE_VALID(device)) 1811 { 1812 return NVL_SUCCESS; 1813 } 1814 1815 nvListForEachEntry_safe(curr, next, &device->client_events_list, entry) 1816 { 1817 if (curr->private_driver_data == osPrivate) 1818 { 1819 nvListDel(&curr->entry); 1820 nvswitch_os_free(curr); 1821 } 1822 } 1823 1824 // Invoke OS specific API to remove event. 1825 status = nvswitch_os_remove_client_event(device->os_handle, 1826 osPrivate); 1827 if (status != NVL_SUCCESS) 1828 { 1829 NVSWITCH_PRINT(device, ERROR, "%s: Failed to remove client events.\n", __FUNCTION__); 1830 return status; 1831 } 1832 1833 return NVL_SUCCESS; 1834 } 1835 1836 /*! 1837 * @brief: Notifies all events with matching event id in the 1838 * Device's client event list. 1839 */ 1840 NvlStatus 1841 nvswitch_lib_notify_client_events 1842 ( 1843 nvswitch_device *device, 1844 NvU32 eventId 1845 ) 1846 { 1847 NvlStatus status; 1848 NVSWITCH_CLIENT_EVENT *curr = NULL; 1849 1850 if (!NVSWITCH_IS_DEVICE_VALID(device)) 1851 { 1852 return -NVL_BAD_ARGS; 1853 } 1854 1855 if (eventId >= NVSWITCH_DEVICE_EVENT_COUNT) 1856 { 1857 NVSWITCH_PRINT(device, ERROR, "%s: Invalid event Id.\n", __FUNCTION__); 1858 return -NVL_BAD_ARGS; 1859 } 1860 1861 nvListForEachEntry(curr, &device->client_events_list, entry) 1862 { 1863 if (curr->eventId == eventId) 1864 { 1865 // OS specific event notification. 1866 status = nvswitch_os_notify_client_event(device->os_handle, 1867 curr->private_driver_data, 1868 eventId); 1869 if (status != NVL_SUCCESS) 1870 { 1871 return status; 1872 } 1873 } 1874 } 1875 1876 return NVL_SUCCESS; 1877 } 1878 1879 /*! 1880 @brief: Release ROM image from memory. 1881 */ 1882 void 1883 _nvswitch_destroy_rom(nvswitch_device *device) 1884 { 1885 if (device->biosImage.pImage != NULL) 1886 { 1887 nvswitch_os_free(device->biosImage.pImage); 1888 device->biosImage.pImage = NULL; 1889 } 1890 } 1891 1892 /*! 1893 * @brief: Free the device's client event list 1894 */ 1895 static void 1896 _nvswitch_destroy_event_list(nvswitch_device *device) 1897 { 1898 NVSWITCH_CLIENT_EVENT *curr = NULL; 1899 NVSWITCH_CLIENT_EVENT *next = NULL; 1900 1901 nvListForEachEntry_safe(curr, next, &device->client_events_list, entry) 1902 { 1903 nvListDel(&curr->entry); 1904 nvswitch_os_free(curr); 1905 } 1906 } 1907 1908 NvlStatus 1909 nvswitch_lib_shutdown_device 1910 ( 1911 nvswitch_device *device 1912 ) 1913 { 1914 NVSWITCH_INBAND_FLUSH_DATA_PARAMS p; 1915 1916 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1917 { 1918 return -NVL_BAD_ARGS; 1919 } 1920 1921 // 1922 // Set fabric state to offline 1923 // 1924 if (device->device_fabric_state != NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED) 1925 device->device_fabric_state = NVSWITCH_DEVICE_FABRIC_STATE_OFFLINE; 1926 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_OFFLINE; 1927 (void)device->hal.nvswitch_write_fabric_state(device); 1928 1929 nvswitch_hw_counter_shutdown(device); 1930 1931 // FLUSH any pending messages to avoid memory leaks 1932 p.linkMask = nvswitch_get_enabled_link_mask(device); 1933 _nvswitch_ctrl_inband_flush_data(device, &p); 1934 1935 _nvswitch_unregister_links(device); 1936 1937 nvswitch_destroy_error_log(device, &device->log_FATAL_ERRORS); 1938 nvswitch_destroy_error_log(device, &device->log_NONFATAL_ERRORS); 1939 1940 nvswitch_smbpbi_unload(device); 1941 _nvswitch_destroy_event_list(device); 1942 1943 nvswitch_destroy_inforom_objects(device); 1944 nvswitch_destroy_inforom(device); 1945 1946 nvswitch_smbpbi_destroy(device); 1947 1948 nvswitch_destroy_device_state(device); 1949 1950 _nvswitch_destroy_rom(device); 1951 1952 _nvswitch_destruct_soe(device); 1953 1954 nvswitch_tasks_destroy(device); 1955 1956 return NVL_SUCCESS; 1957 } 1958 1959 NvlStatus 1960 nvswitch_lib_get_log_count 1961 ( 1962 nvswitch_device *device, 1963 NvU32 *fatal, NvU32 *nonfatal 1964 ) 1965 { 1966 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device) || 1967 fatal == NULL || nonfatal == NULL) 1968 { 1969 return -NVL_BAD_ARGS; 1970 } 1971 1972 *fatal = device->log_FATAL_ERRORS.error_count; 1973 *nonfatal = device->log_NONFATAL_ERRORS.error_count; 1974 // No report of log_INFO currently 1975 1976 return NVL_SUCCESS; 1977 } 1978 1979 NvlStatus 1980 nvswitch_lib_load_platform_info 1981 ( 1982 nvswitch_device *device 1983 ) 1984 { 1985 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1986 { 1987 return -NVL_BAD_ARGS; 1988 } 1989 1990 device->hal.nvswitch_determine_platform(device); 1991 1992 return NVL_SUCCESS; 1993 } 1994 1995 void 1996 nvswitch_lib_get_device_info 1997 ( 1998 nvswitch_device *device, 1999 struct nvlink_pci_info **pciInfo 2000 ) 2001 { 2002 if (!NVSWITCH_IS_DEVICE_VALID(device) || pciInfo == NULL) 2003 { 2004 NVSWITCH_ASSERT(0); 2005 return; 2006 } 2007 2008 *pciInfo = &device->nvlink_device->pciInfo; 2009 } 2010 2011 NvlStatus 2012 nvswitch_lib_get_bios_version 2013 ( 2014 nvswitch_device *device, 2015 NvU64 *version 2016 ) 2017 { 2018 NVSWITCH_GET_BIOS_INFO_PARAMS p = { 0 }; 2019 NvlStatus ret; 2020 2021 if (!device) 2022 return -NVL_BAD_ARGS; 2023 2024 ret = device->hal.nvswitch_ctrl_get_bios_info(device, &p); 2025 2026 if (version != NULL) 2027 { 2028 *version = p.version; 2029 } 2030 2031 return ret; 2032 } 2033 2034 NvlStatus 2035 nvswitch_lib_use_pin_irq 2036 ( 2037 nvswitch_device *device 2038 ) 2039 { 2040 return IS_FMODEL(device); 2041 } 2042 2043 2044 NvlStatus 2045 nvswitch_lib_register_device 2046 ( 2047 NvU16 pci_domain, 2048 NvU8 pci_bus, 2049 NvU8 pci_device, 2050 NvU8 pci_func, 2051 NvU16 pci_device_id, 2052 void *os_handle, 2053 NvU32 os_instance, 2054 nvswitch_device **return_device 2055 ) 2056 { 2057 nvswitch_device *device = NULL; 2058 nvlink_device *coreDev = NULL; 2059 NvlStatus retval = NVL_SUCCESS; 2060 2061 if (!nvlink_lib_is_initialized()) 2062 { 2063 NVSWITCH_PRINT(device, ERROR, 2064 "NVLink core lib isn't initialized yet!\n"); 2065 return -NVL_INITIALIZATION_TOTAL_FAILURE; 2066 } 2067 2068 if (return_device == NULL || os_handle == NULL) 2069 { 2070 return -NVL_BAD_ARGS; 2071 } 2072 2073 *return_device = NULL; 2074 2075 device = nvswitch_os_malloc(sizeof(*device)); 2076 if (NULL == device) 2077 { 2078 NVSWITCH_PRINT(device, ERROR, 2079 "nvswitch_os_malloc during device creation failed!\n"); 2080 return -NVL_NO_MEM; 2081 } 2082 nvswitch_os_memset(device, 0, sizeof(*device)); 2083 2084 nvswitch_os_snprintf(device->name, sizeof(device->name), 2085 NVSWITCH_DEVICE_NAME "%d", os_instance); 2086 2087 coreDev = nvswitch_os_malloc(sizeof(*coreDev)); 2088 if (NULL == coreDev) 2089 { 2090 NVSWITCH_PRINT(device, ERROR, 2091 "nvswitch_os_malloc during device creation failed!\n"); 2092 2093 retval = -NVL_NO_MEM; 2094 goto nvlink_lib_register_device_fail; 2095 } 2096 nvswitch_os_memset(coreDev, 0, sizeof(*coreDev)); 2097 2098 coreDev->driverName = 2099 nvswitch_os_malloc(sizeof(NVSWITCH_DRIVER_NAME)); 2100 if (coreDev->driverName == NULL) 2101 { 2102 NVSWITCH_PRINT(device, ERROR, 2103 "nvswitch_os_malloc during device creation failed!\n"); 2104 2105 retval = -NVL_NO_MEM; 2106 goto nvlink_lib_register_device_fail; 2107 } 2108 nvswitch_os_memcpy(coreDev->driverName, NVSWITCH_DRIVER_NAME, 2109 sizeof(NVSWITCH_DRIVER_NAME)); 2110 2111 device->os_handle = os_handle; 2112 device->os_instance = os_instance; 2113 2114 device->nvlink_device = coreDev; 2115 device->nvlink_device->deviceName = device->name; 2116 device->nvlink_device->uuid = NULL; // No UUID support for switch 2117 2118 device->nvlink_device->pciInfo.domain = pci_domain; 2119 device->nvlink_device->pciInfo.bus = pci_bus; 2120 device->nvlink_device->pciInfo.device = pci_device; 2121 device->nvlink_device->pciInfo.function = pci_func; 2122 device->nvlink_device->pciInfo.pciDeviceId = pci_device_id; 2123 2124 // nvlink_device has a back pointer to nvswitch_device 2125 device->nvlink_device->pDevInfo = device; 2126 device->nvlink_device->type = NVLINK_DEVICE_TYPE_NVSWITCH; 2127 2128 // 2129 // Initialize the Fabric State 2130 // 2131 device->fm_timeout = NVSWITCH_DEFAULT_FM_HEARTBEAT_TIMEOUT_MSEC; 2132 device->fabric_state_sequence_number = 0; 2133 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_STANDBY; 2134 device->device_fabric_state = NVSWITCH_DEVICE_FABRIC_STATE_STANDBY; 2135 device->device_blacklist_reason = NVSWITCH_DEVICE_BLACKLIST_REASON_NONE; 2136 2137 // 2138 // Initialize HAL connectivity as early as possible so that other lib 2139 // interfaces can work. 2140 // 2141 retval = _nvswitch_setup_hal(device, device->nvlink_device->pciInfo.pciDeviceId); 2142 if (retval != NVL_SUCCESS) 2143 { 2144 goto nvlink_lib_register_device_fail; 2145 } 2146 2147 // 2148 // Initialize regkeys as early as possible so that most routines can take 2149 // advantage of them. 2150 // 2151 _nvswitch_init_device_regkeys(device); 2152 2153 // After regkeys have been set then only set the enableALI field. 2154 device->nvlink_device->enableALI = (device->regkeys.link_training_mode == 2155 NV_SWITCH_REGKEY_LINK_TRAINING_SELECT_ALI) ? NV_TRUE:NV_FALSE; 2156 2157 retval = nvlink_lib_register_device(device->nvlink_device); 2158 if (NVL_SUCCESS != retval) 2159 { 2160 NVSWITCH_PRINT(device, ERROR, 2161 "nvlinklib register device failed!\n"); 2162 goto nvlink_lib_register_device_fail; 2163 } 2164 2165 *return_device = device; 2166 2167 NVSWITCH_PRINT(device, SETUP, 2168 "Successfully registered with nvlinkcore\n"); 2169 2170 return retval; 2171 2172 nvlink_lib_register_device_fail: 2173 2174 if (NULL != coreDev) 2175 { 2176 nvswitch_os_free(coreDev->driverName); 2177 nvswitch_os_free(coreDev); 2178 } 2179 2180 if (NULL != device) 2181 nvswitch_os_free(device); 2182 2183 return retval; 2184 } 2185 2186 void 2187 nvswitch_lib_unregister_device 2188 ( 2189 nvswitch_device *device 2190 ) 2191 { 2192 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2193 { 2194 NVSWITCH_ASSERT(0); 2195 return; 2196 } 2197 2198 nvlink_lib_unregister_device(device->nvlink_device); 2199 2200 nvswitch_os_free(device->nvlink_device->driverName); 2201 nvswitch_os_free(device->nvlink_device); 2202 nvswitch_os_free(device); 2203 2204 return; 2205 } 2206 2207 /*! 2208 * @brief: Gets the mask of valid I2C ports on the 2209 * Device. 2210 */ 2211 NvlStatus 2212 nvswitch_lib_get_valid_ports_mask 2213 ( 2214 nvswitch_device *device, 2215 NvU32 *validPortsMask 2216 ) 2217 { 2218 NvU32 port_info; 2219 NvU32 i; 2220 NvU32 ports_mask = 0; 2221 NvBool is_i2c_access_allowed; 2222 NvBool is_port_allowed; 2223 2224 if (!NVSWITCH_IS_DEVICE_VALID(device) || 2225 (validPortsMask == NULL)) 2226 { 2227 return -NVL_BAD_ARGS; 2228 } 2229 2230 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 2231 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 2232 NV_TRUE : NV_FALSE; 2233 2234 for (i = 0; i < NVSWITCH_MAX_I2C_PORTS; i++) 2235 { 2236 port_info = nvswitch_i2c_get_port_info(device, i); 2237 2238 is_port_allowed = is_i2c_access_allowed ? NV_TRUE : 2239 FLD_TEST_DRF(_I2C, _PORTINFO, _ACCESS_ALLOWED, _TRUE, 2240 port_info); 2241 2242 if (is_port_allowed && 2243 FLD_TEST_DRF(_I2C, _PORTINFO, _DEFINED, _PRESENT, port_info)) 2244 { 2245 ports_mask |= NVBIT(i); 2246 } 2247 } 2248 2249 *validPortsMask = ports_mask; 2250 return NVL_SUCCESS; 2251 } 2252 2253 /*! 2254 * @brief: Returns if the I2C transactions are supported. 2255 */ 2256 NvBool 2257 nvswitch_lib_is_i2c_supported 2258 ( 2259 nvswitch_device *device 2260 ) 2261 { 2262 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2263 { 2264 NVSWITCH_ASSERT(0); 2265 return NV_FALSE; 2266 } 2267 2268 return nvswitch_is_i2c_supported(device); 2269 } 2270 2271 static NvlStatus 2272 _nvswitch_perform_i2c_transfer 2273 ( 2274 nvswitch_device *device, 2275 NvU32 client, 2276 NvU8 type, 2277 NvU16 addr, 2278 NvU8 port, 2279 NvU8 cmd, 2280 NvU32 msgLength, 2281 NvU8 *pData 2282 ) 2283 { 2284 NvlStatus status; 2285 NvU16 deviceAddr; 2286 NvU32 speedMode; 2287 NvBool bIsRead = NV_FALSE; 2288 NvU32 flags = 0; 2289 NVSWITCH_CTRL_I2C_INDEXED_PARAMS i2c_params = {0}; 2290 NvBool is_i2c_access_allowed; 2291 2292 if (!nvswitch_os_is_admin()) 2293 { 2294 return -NVL_ERR_INSUFFICIENT_PERMISSIONS; 2295 } 2296 2297 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 2298 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 2299 NV_TRUE : NV_FALSE; 2300 2301 // 2302 // The address needs to be shifted by 1, 2303 // See NVSWITCH_CTRL_I2C_INDEXED_PARAMS 2304 // 2305 deviceAddr = addr << 1; 2306 speedMode = device->pI2c->Ports[port].defaultSpeedMode; 2307 flags = DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _START, _SEND) | 2308 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _STOP, _SEND) | 2309 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _ADDRESS_MODE, _7BIT) | 2310 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _FLAVOR, _HW) | 2311 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _BLOCK_PROTOCOL, _DISABLED) | 2312 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _TRANSACTION_MODE, _NORMAL); 2313 2314 switch (speedMode) 2315 { 2316 case NVSWITCH_I2C_SPEED_MODE_1000KHZ: 2317 { 2318 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _1000KHZ, flags); 2319 break; 2320 } 2321 case NVSWITCH_I2C_SPEED_MODE_400KHZ: 2322 { 2323 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _400KHZ, flags); 2324 break; 2325 } 2326 case NVSWITCH_I2C_SPEED_MODE_300KHZ: 2327 { 2328 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _300KHZ, flags); 2329 break; 2330 } 2331 case NVSWITCH_I2C_SPEED_MODE_200KHZ: 2332 { 2333 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _200KHZ, flags); 2334 break; 2335 } 2336 case NVSWITCH_I2C_SPEED_MODE_100KHZ: 2337 { 2338 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _100KHZ, flags); 2339 break; 2340 } 2341 default: 2342 { 2343 NVSWITCH_PRINT(device, ERROR, "Invalid I2C speed!\n"); 2344 status = -NVL_BAD_ARGS; 2345 goto end; 2346 } 2347 } 2348 2349 switch (type) 2350 { 2351 case NVSWITCH_I2C_CMD_READ: 2352 bIsRead = NV_TRUE; 2353 // Fall through 2354 case NVSWITCH_I2C_CMD_WRITE: 2355 { 2356 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ZERO, flags); 2357 break; 2358 } 2359 case NVSWITCH_I2C_CMD_SMBUS_READ: 2360 { 2361 bIsRead = NV_TRUE; 2362 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _RESTART, _SEND, flags); 2363 // Fall through 2364 } 2365 case NVSWITCH_I2C_CMD_SMBUS_WRITE: 2366 { 2367 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ONE, flags); 2368 break; 2369 } 2370 case NVSWITCH_I2C_CMD_SMBUS_QUICK_READ: 2371 bIsRead = NV_TRUE; 2372 // Fall through 2373 case NVSWITCH_I2C_CMD_SMBUS_QUICK_WRITE: 2374 { 2375 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ZERO, flags); 2376 msgLength = 0; 2377 break; 2378 } 2379 default: 2380 { 2381 NVSWITCH_PRINT(device, ERROR, "Invalid SMBUS protocol! Protocol not supported.\n"); 2382 status = -NVL_BAD_ARGS; 2383 goto end; 2384 } 2385 } 2386 2387 if (!is_i2c_access_allowed && 2388 !nvswitch_i2c_is_device_access_allowed(device, port, deviceAddr, bIsRead)) 2389 { 2390 return -NVL_BAD_ARGS; 2391 } 2392 2393 if (msgLength > NVSWITCH_CTRL_I2C_MESSAGE_LENGTH_MAX) 2394 { 2395 NVSWITCH_PRINT(device, ERROR, 2396 "Length of buffer (0x%x bytes) provided larger than max (0x%x bytes)\n", 2397 msgLength, NVSWITCH_CTRL_I2C_MESSAGE_LENGTH_MAX); 2398 status = -NVL_BAD_ARGS; 2399 goto end; 2400 } 2401 2402 if (bIsRead) 2403 { 2404 i2c_params.bIsRead = NV_TRUE; 2405 } 2406 else 2407 { 2408 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _RESTART, _NONE, flags); 2409 nvswitch_os_memcpy(i2c_params.message, pData, msgLength); 2410 } 2411 2412 if (FLD_TEST_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ONE, flags)) 2413 { 2414 i2c_params.index[0] = cmd; 2415 } 2416 2417 i2c_params.port = port; 2418 i2c_params.address = deviceAddr; 2419 i2c_params.acquirer = client; 2420 i2c_params.flags = flags; 2421 i2c_params.messageLength = msgLength; 2422 2423 status = nvswitch_ctrl_i2c_indexed(device, &i2c_params); 2424 if (status != NVL_SUCCESS) 2425 { 2426 NVSWITCH_PRINT(device, ERROR, "I2C transfer Failed!\n"); 2427 goto end; 2428 } 2429 2430 if (bIsRead) 2431 { 2432 nvswitch_os_memcpy(pData, i2c_params.message, msgLength); 2433 } 2434 2435 end: 2436 return status; 2437 } 2438 2439 /*! 2440 * @brief: Performs an I2C transaction. 2441 */ 2442 NvlStatus 2443 nvswitch_lib_i2c_transfer 2444 ( 2445 nvswitch_device *device, 2446 NvU32 port, 2447 NvU8 type, 2448 NvU8 addr, 2449 NvU8 command, 2450 NvU32 len, 2451 NvU8 *pData 2452 ) 2453 { 2454 NvlStatus status; 2455 NvU32 port_info; 2456 NvBool is_i2c_access_allowed; 2457 NvBool is_port_allowed; 2458 2459 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2460 { 2461 NVSWITCH_ASSERT(0); 2462 return -NVL_ERR_INVALID_STATE; 2463 } 2464 2465 port_info = nvswitch_i2c_get_port_info(device, port); 2466 2467 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 2468 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 2469 NV_TRUE : NV_FALSE; 2470 is_port_allowed = is_i2c_access_allowed ? NV_TRUE : 2471 FLD_TEST_DRF(_I2C, _PORTINFO, _ACCESS_ALLOWED, _TRUE, 2472 port_info); 2473 2474 if (!is_port_allowed || 2475 !FLD_TEST_DRF(_I2C, _PORTINFO, _DEFINED, _PRESENT, port_info)) 2476 { 2477 NVSWITCH_PRINT(device, INFO, 2478 "%s: Invalid port access %d.\n", 2479 __FUNCTION__, port); 2480 return (-NVL_BAD_ARGS); 2481 } 2482 2483 status = _nvswitch_perform_i2c_transfer(device, NVSWITCH_I2C_ACQUIRER_EXTERNAL, 2484 type, (NvU16)addr, port, command, len, pData); 2485 if (status != NVL_SUCCESS) 2486 { 2487 NVSWITCH_PRINT(device, ERROR, "I2C transaction failed!\n"); 2488 return status; 2489 } 2490 2491 return NVL_SUCCESS; 2492 } 2493 2494 void 2495 nvswitch_timeout_create 2496 ( 2497 NvU64 timeout_ns, 2498 NVSWITCH_TIMEOUT *time 2499 ) 2500 { 2501 NvU64 time_current; 2502 2503 time_current = nvswitch_os_get_platform_time(); 2504 time->timeout_ns = time_current + timeout_ns; 2505 } 2506 2507 NvBool 2508 nvswitch_timeout_check 2509 ( 2510 NVSWITCH_TIMEOUT *time 2511 ) 2512 { 2513 NvU64 time_current; 2514 2515 time_current = nvswitch_os_get_platform_time(); 2516 return (time->timeout_ns <= time_current); 2517 } 2518 2519 NvlStatus 2520 nvswitch_task_create 2521 ( 2522 nvswitch_device *device, 2523 void (*task_fn)(nvswitch_device *device), 2524 NvU64 period_nsec, 2525 NvU32 flags 2526 ) 2527 { 2528 NVSWITCH_TASK_TYPE *task; 2529 task = nvswitch_os_malloc(sizeof(NVSWITCH_TASK_TYPE)); 2530 2531 if (task == NULL) 2532 { 2533 NVSWITCH_PRINT(device, ERROR, 2534 "%s: Unable to allocate task.\n", 2535 __FUNCTION__); 2536 return -NVL_NO_MEM; 2537 } 2538 else 2539 { 2540 task->task_fn_devptr = task_fn; 2541 task->task_args = NULL; 2542 task->period_nsec = period_nsec; 2543 task->last_run_nsec = nvswitch_os_get_platform_time(); // Prevent deferred tasks from being run immediately 2544 task->flags = flags; 2545 task->prev = NULL; 2546 task->next = device->tasks; 2547 if (device->tasks != NULL) 2548 { 2549 device->tasks->prev = task; 2550 } 2551 device->tasks = task; 2552 } 2553 2554 return NVL_SUCCESS; 2555 } 2556 2557 NvlStatus 2558 nvswitch_task_create_args 2559 ( 2560 nvswitch_device *device, 2561 void *fn_args, 2562 void (*task_fn)(nvswitch_device* device, void *fn_args), 2563 NvU64 period_nsec, 2564 NvU32 flags 2565 ) 2566 { 2567 NVSWITCH_TASK_TYPE *task; 2568 task = nvswitch_os_malloc(sizeof(NVSWITCH_TASK_TYPE)); 2569 2570 flags = flags | NVSWITCH_TASK_TYPE_FLAGS_VOID_PTR_ARGS; // ensure dispatcher always executes tasks passed through this function with args 2571 2572 if (task == NULL) 2573 { 2574 NVSWITCH_PRINT(device, ERROR, 2575 "%s: Unable to allocate task.\n", 2576 __FUNCTION__); 2577 return -NVL_NO_MEM; 2578 } 2579 else 2580 { 2581 task->task_fn_vdptr = task_fn; 2582 task->task_args = fn_args; 2583 task->period_nsec = period_nsec; 2584 task->last_run_nsec = nvswitch_os_get_platform_time(); // Prevent deferred tasks from being run immediately 2585 task->flags = flags; 2586 task->prev = NULL; 2587 task->next = device->tasks; 2588 if (device->tasks != NULL) 2589 { 2590 device->tasks->prev = task; 2591 } 2592 device->tasks = task; 2593 } 2594 2595 return NVL_SUCCESS; 2596 } 2597 2598 void 2599 nvswitch_tasks_destroy 2600 ( 2601 nvswitch_device *device 2602 ) 2603 { 2604 NVSWITCH_TASK_TYPE *task = device->tasks; 2605 NVSWITCH_TASK_TYPE *next_task; 2606 2607 device->tasks = NULL; 2608 2609 while (task) 2610 { 2611 next_task = task->next; 2612 nvswitch_os_free(task); 2613 task = next_task; 2614 } 2615 } 2616 2617 void 2618 nvswitch_destroy_device_state 2619 ( 2620 nvswitch_device *device 2621 ) 2622 { 2623 device->hal.nvswitch_destroy_device_state(device); 2624 } 2625 2626 static NvlStatus 2627 _nvswitch_ctrl_get_info 2628 ( 2629 nvswitch_device *device, 2630 NVSWITCH_GET_INFO *p 2631 ) 2632 { 2633 return device->hal.nvswitch_ctrl_get_info(device, p); 2634 } 2635 2636 static NvlStatus 2637 _nvswitch_ctrl_get_nvlink_status 2638 ( 2639 nvswitch_device *device, 2640 NVSWITCH_GET_NVLINK_STATUS_PARAMS *ret 2641 ) 2642 { 2643 return device->hal.nvswitch_ctrl_get_nvlink_status(device, ret); 2644 } 2645 2646 static NvlStatus 2647 _nvswitch_ctrl_get_counters 2648 ( 2649 nvswitch_device *device, 2650 NVSWITCH_NVLINK_GET_COUNTERS_PARAMS *ret 2651 ) 2652 { 2653 return device->hal.nvswitch_ctrl_get_counters(device, ret); 2654 } 2655 2656 NvlStatus 2657 nvswitch_set_nport_port_config 2658 ( 2659 nvswitch_device *device, 2660 NVSWITCH_SET_SWITCH_PORT_CONFIG *p 2661 ) 2662 { 2663 return device->hal.nvswitch_set_nport_port_config(device, p); 2664 } 2665 2666 static NvlStatus 2667 _nvswitch_ctrl_set_switch_port_config 2668 ( 2669 nvswitch_device *device, 2670 NVSWITCH_SET_SWITCH_PORT_CONFIG *p 2671 ) 2672 { 2673 return device->hal.nvswitch_ctrl_set_switch_port_config(device, p); 2674 } 2675 2676 static NvlStatus 2677 _nvswitch_ctrl_get_ingress_request_table 2678 ( 2679 nvswitch_device *device, 2680 NVSWITCH_GET_INGRESS_REQUEST_TABLE_PARAMS *params 2681 ) 2682 { 2683 return device->hal.nvswitch_ctrl_get_ingress_request_table(device, params); 2684 } 2685 2686 static NvlStatus 2687 _nvswitch_ctrl_set_ingress_request_table 2688 ( 2689 nvswitch_device *device, 2690 NVSWITCH_SET_INGRESS_REQUEST_TABLE *p 2691 ) 2692 { 2693 return device->hal.nvswitch_ctrl_set_ingress_request_table(device, p); 2694 } 2695 2696 static NvlStatus 2697 _nvswitch_ctrl_set_ingress_request_valid 2698 ( 2699 nvswitch_device *device, 2700 NVSWITCH_SET_INGRESS_REQUEST_VALID *p 2701 ) 2702 { 2703 return device->hal.nvswitch_ctrl_set_ingress_request_valid(device, p); 2704 } 2705 2706 static NvlStatus 2707 _nvswitch_ctrl_get_ingress_response_table 2708 ( 2709 nvswitch_device *device, 2710 NVSWITCH_GET_INGRESS_RESPONSE_TABLE_PARAMS *params 2711 ) 2712 { 2713 return device->hal.nvswitch_ctrl_get_ingress_response_table(device, params); 2714 } 2715 2716 static NvlStatus 2717 _nvswitch_ctrl_set_ingress_response_table 2718 ( 2719 nvswitch_device *device, 2720 NVSWITCH_SET_INGRESS_RESPONSE_TABLE *p 2721 ) 2722 { 2723 return device->hal.nvswitch_ctrl_set_ingress_response_table(device, p); 2724 } 2725 2726 static NvlStatus 2727 _nvswitch_ctrl_set_ganged_link_table 2728 ( 2729 nvswitch_device *device, 2730 NVSWITCH_SET_GANGED_LINK_TABLE *p 2731 ) 2732 { 2733 return device->hal.nvswitch_ctrl_set_ganged_link_table(device, p); 2734 } 2735 2736 void 2737 nvswitch_init_npg_multicast 2738 ( 2739 nvswitch_device *device 2740 ) 2741 { 2742 return device->hal.nvswitch_init_npg_multicast(device); 2743 } 2744 2745 void 2746 nvswitch_init_warm_reset 2747 ( 2748 nvswitch_device *device 2749 ) 2750 { 2751 return device->hal.nvswitch_init_warm_reset(device); 2752 } 2753 2754 static NvlStatus 2755 _nvswitch_ctrl_set_remap_policy 2756 ( 2757 nvswitch_device *device, 2758 NVSWITCH_SET_REMAP_POLICY *p 2759 ) 2760 { 2761 return device->hal.nvswitch_ctrl_set_remap_policy(device, p); 2762 } 2763 2764 static NvlStatus 2765 _nvswitch_ctrl_get_remap_policy 2766 ( 2767 nvswitch_device *device, 2768 NVSWITCH_GET_REMAP_POLICY_PARAMS *params 2769 ) 2770 { 2771 return device->hal.nvswitch_ctrl_get_remap_policy(device, params); 2772 } 2773 2774 static NvlStatus 2775 _nvswitch_ctrl_set_remap_policy_valid 2776 ( 2777 nvswitch_device *device, 2778 NVSWITCH_SET_REMAP_POLICY_VALID *p 2779 ) 2780 { 2781 return device->hal.nvswitch_ctrl_set_remap_policy_valid(device, p); 2782 } 2783 2784 static NvlStatus 2785 _nvswitch_ctrl_set_routing_id 2786 ( 2787 nvswitch_device *device, 2788 NVSWITCH_SET_ROUTING_ID *p 2789 ) 2790 { 2791 return device->hal.nvswitch_ctrl_set_routing_id(device, p); 2792 } 2793 2794 static NvlStatus 2795 _nvswitch_ctrl_get_routing_id 2796 ( 2797 nvswitch_device *device, 2798 NVSWITCH_GET_ROUTING_ID_PARAMS *params 2799 ) 2800 { 2801 return device->hal.nvswitch_ctrl_get_routing_id(device, params); 2802 } 2803 2804 static NvlStatus 2805 _nvswitch_ctrl_set_routing_id_valid 2806 ( 2807 nvswitch_device *device, 2808 NVSWITCH_SET_ROUTING_ID_VALID *p 2809 ) 2810 { 2811 return device->hal.nvswitch_ctrl_set_routing_id_valid(device, p); 2812 } 2813 2814 static NvlStatus 2815 _nvswitch_ctrl_set_routing_lan 2816 ( 2817 nvswitch_device *device, 2818 NVSWITCH_SET_ROUTING_LAN *p 2819 ) 2820 { 2821 return device->hal.nvswitch_ctrl_set_routing_lan(device, p); 2822 } 2823 2824 static NvlStatus 2825 _nvswitch_ctrl_get_routing_lan 2826 ( 2827 nvswitch_device *device, 2828 NVSWITCH_GET_ROUTING_LAN_PARAMS *params 2829 ) 2830 { 2831 return device->hal.nvswitch_ctrl_get_routing_lan(device, params); 2832 } 2833 2834 static NvlStatus 2835 _nvswitch_ctrl_set_routing_lan_valid 2836 ( 2837 nvswitch_device *device, 2838 NVSWITCH_SET_ROUTING_LAN_VALID *p 2839 ) 2840 { 2841 return device->hal.nvswitch_ctrl_set_routing_lan_valid(device, p); 2842 } 2843 2844 static NvlStatus 2845 _nvswitch_ctrl_get_internal_latency 2846 ( 2847 nvswitch_device *device, 2848 NVSWITCH_GET_INTERNAL_LATENCY *p 2849 ) 2850 { 2851 return device->hal.nvswitch_ctrl_get_internal_latency(device, p); 2852 } 2853 2854 static NvlStatus 2855 _nvswitch_ctrl_get_nvlipt_counters 2856 ( 2857 nvswitch_device *device, 2858 NVSWITCH_GET_NVLIPT_COUNTERS *p 2859 ) 2860 { 2861 // 2862 // This control call is now deprecated. 2863 // New control call to fetch throughput counters is: 2864 // nvswitch_ctrl_get_throughput_counters 2865 // 2866 return -NVL_ERR_NOT_SUPPORTED; 2867 } 2868 2869 static NvlStatus 2870 _nvswitch_ctrl_set_nvlipt_counter_config 2871 ( 2872 nvswitch_device *device, 2873 NVSWITCH_SET_NVLIPT_COUNTER_CONFIG *p 2874 ) 2875 { 2876 // 2877 // This control call is now deprecated. 2878 // New control call to fetch throughput counters is: 2879 // nvswitch_ctrl_get_throughput_counters_lr10 2880 // 2881 // Setting counter config is not allowed on these 2882 // non-configurable counters. These counters are 2883 // expected to be used by monitoring clients. 2884 // 2885 return -NVL_ERR_NOT_SUPPORTED; 2886 } 2887 2888 static NvlStatus 2889 _nvswitch_ctrl_get_nvlipt_counter_config 2890 ( 2891 nvswitch_device *device, 2892 NVSWITCH_GET_NVLIPT_COUNTER_CONFIG *p 2893 ) 2894 { 2895 // 2896 // This control call is now deprecated. 2897 // New control call to fetch throughput counters is: 2898 // nvswitch_ctrl_get_throughput_counters_lr10 2899 // 2900 // Getting counter config is useful if counters are 2901 // configurable. These counters are not configurable 2902 // and are expected to be used by monitoring clients. 2903 // 2904 return -NVL_ERR_NOT_SUPPORTED; 2905 } 2906 2907 static NvlStatus 2908 _nvswitch_ctrl_register_read 2909 ( 2910 nvswitch_device *device, 2911 NVSWITCH_REGISTER_READ *p 2912 ) 2913 { 2914 return device->hal.nvswitch_ctrl_register_read(device, p); 2915 } 2916 2917 static NvlStatus 2918 _nvswitch_ctrl_register_write 2919 ( 2920 nvswitch_device *device, 2921 NVSWITCH_REGISTER_WRITE *p 2922 ) 2923 { 2924 return device->hal.nvswitch_ctrl_register_write(device, p); 2925 } 2926 2927 NvU32 2928 nvswitch_i2c_get_port_info 2929 ( 2930 nvswitch_device *device, 2931 NvU32 port 2932 ) 2933 { 2934 return device->hal.nvswitch_i2c_get_port_info(device, port); 2935 } 2936 2937 NvlStatus 2938 nvswitch_ctrl_i2c_indexed 2939 ( 2940 nvswitch_device *device, 2941 NVSWITCH_CTRL_I2C_INDEXED_PARAMS *pParams 2942 ) 2943 { 2944 return device->hal.nvswitch_ctrl_i2c_indexed(device, pParams); 2945 } 2946 2947 static NvlStatus 2948 _nvswitch_ctrl_therm_read_temperature 2949 ( 2950 nvswitch_device *device, 2951 NVSWITCH_CTRL_GET_TEMPERATURE_PARAMS *info 2952 ) 2953 { 2954 return device->hal.nvswitch_ctrl_therm_read_temperature(device, info); 2955 } 2956 2957 static NvlStatus 2958 _nvswitch_ctrl_get_bios_info 2959 ( 2960 nvswitch_device *device, 2961 NVSWITCH_GET_BIOS_INFO_PARAMS *p 2962 ) 2963 { 2964 return device->hal.nvswitch_ctrl_get_bios_info(device, p); 2965 } 2966 2967 static NvlStatus 2968 _nvswitch_ctrl_get_inforom_version 2969 ( 2970 nvswitch_device *device, 2971 NVSWITCH_GET_INFOROM_VERSION_PARAMS *p 2972 ) 2973 { 2974 return device->hal.nvswitch_ctrl_get_inforom_version(device, p); 2975 } 2976 2977 NvlStatus 2978 nvswitch_ctrl_set_latency_bins 2979 ( 2980 nvswitch_device *device, 2981 NVSWITCH_SET_LATENCY_BINS *p 2982 ) 2983 { 2984 return device->hal.nvswitch_ctrl_set_latency_bins(device, p); 2985 } 2986 2987 static NvlStatus 2988 _nvswitch_ctrl_get_ingress_reqlinkid 2989 ( 2990 nvswitch_device *device, 2991 NVSWITCH_GET_INGRESS_REQLINKID_PARAMS *params 2992 ) 2993 { 2994 return device->hal.nvswitch_ctrl_get_ingress_reqlinkid(device, params); 2995 } 2996 2997 NvlStatus 2998 nvswitch_ctrl_get_throughput_counters 2999 ( 3000 nvswitch_device *device, 3001 NVSWITCH_GET_THROUGHPUT_COUNTERS_PARAMS *p 3002 ) 3003 { 3004 return device->hal.nvswitch_ctrl_get_throughput_counters(device, p); 3005 } 3006 3007 static NvlStatus 3008 _nvswitch_ctrl_unregister_link 3009 ( 3010 nvswitch_device *device, 3011 NVSWITCH_UNREGISTER_LINK_PARAMS *params 3012 ) 3013 { 3014 nvlink_link *link = nvswitch_get_link(device, (NvU8)params->portNum); 3015 3016 if (link == NULL) 3017 { 3018 return -NVL_BAD_ARGS; 3019 } 3020 3021 // With ALI in FW, links can be unregistered while Active 3022 if (!device->nvlink_device->enableALI) 3023 { 3024 3025 if (device->hal.nvswitch_is_link_in_use(device, params->portNum)) 3026 { 3027 return -NVL_ERR_STATE_IN_USE; 3028 } 3029 3030 } 3031 3032 nvlink_lib_unregister_link(link); 3033 nvswitch_destroy_link(link); 3034 3035 return NVL_SUCCESS; 3036 } 3037 3038 static NvlStatus 3039 _nvswitch_ctrl_acquire_capability 3040 ( 3041 nvswitch_device *device, 3042 NVSWITCH_ACQUIRE_CAPABILITY_PARAMS *params, 3043 void *osPrivate 3044 ) 3045 { 3046 return nvswitch_os_acquire_fabric_mgmt_cap(osPrivate, 3047 params->capDescriptor); 3048 } 3049 3050 static NvlStatus 3051 _nvswitch_ctrl_reset_and_drain_links 3052 ( 3053 nvswitch_device *device, 3054 NVSWITCH_RESET_AND_DRAIN_LINKS_PARAMS *params 3055 ) 3056 { 3057 return device->hal.nvswitch_reset_and_drain_links(device, params->linkMask); 3058 } 3059 3060 static NvlStatus 3061 _nvswitch_ctrl_get_fom_values 3062 ( 3063 nvswitch_device *device, 3064 NVSWITCH_GET_FOM_VALUES_PARAMS *ret 3065 ) 3066 { 3067 return device->hal.nvswitch_ctrl_get_fom_values(device, ret); 3068 } 3069 3070 static NvlStatus 3071 _nvswitch_ctrl_get_nvlink_ecc_errors 3072 ( 3073 nvswitch_device *device, 3074 NVSWITCH_GET_NVLINK_ECC_ERRORS_PARAMS *params 3075 ) 3076 { 3077 return device->hal.nvswitch_get_nvlink_ecc_errors(device, params); 3078 } 3079 3080 static NvlStatus 3081 _nvswitch_ctrl_set_mc_rid_table 3082 ( 3083 nvswitch_device *device, 3084 NVSWITCH_SET_MC_RID_TABLE_PARAMS *p 3085 ) 3086 { 3087 return device->hal.nvswitch_ctrl_set_mc_rid_table(device, p); 3088 } 3089 3090 static NvlStatus 3091 _nvswitch_ctrl_get_mc_rid_table 3092 ( 3093 nvswitch_device *device, 3094 NVSWITCH_GET_MC_RID_TABLE_PARAMS *p 3095 ) 3096 { 3097 return device->hal.nvswitch_ctrl_get_mc_rid_table(device, p); 3098 } 3099 3100 static NvlStatus 3101 _nvswitch_ctrl_set_residency_bins 3102 ( 3103 nvswitch_device *device, 3104 NVSWITCH_SET_RESIDENCY_BINS *p 3105 ) 3106 { 3107 return device->hal.nvswitch_ctrl_set_residency_bins(device, p); 3108 } 3109 3110 static NvlStatus 3111 _nvswitch_ctrl_get_residency_bins 3112 ( 3113 nvswitch_device *device, 3114 NVSWITCH_GET_RESIDENCY_BINS *p 3115 ) 3116 { 3117 return device->hal.nvswitch_ctrl_get_residency_bins(device, p); 3118 } 3119 3120 static NvlStatus 3121 _nvswitch_ctrl_get_rb_stall_busy 3122 ( 3123 nvswitch_device *device, 3124 NVSWITCH_GET_RB_STALL_BUSY *p 3125 ) 3126 { 3127 return device->hal.nvswitch_ctrl_get_rb_stall_busy(device, p); 3128 } 3129 3130 NvlStatus 3131 nvswitch_ctrl_get_multicast_id_error_vector 3132 ( 3133 nvswitch_device *device, 3134 NVSWITCH_GET_MULTICAST_ID_ERROR_VECTOR *p 3135 ) 3136 { 3137 return device->hal.nvswitch_ctrl_get_multicast_id_error_vector(device, p); 3138 } 3139 3140 NvlStatus 3141 nvswitch_ctrl_clear_multicast_id_error_vector 3142 ( 3143 nvswitch_device *device, 3144 NVSWITCH_CLEAR_MULTICAST_ID_ERROR_VECTOR *p 3145 ) 3146 { 3147 return device->hal.nvswitch_ctrl_clear_multicast_id_error_vector(device, p); 3148 } 3149 3150 static NvlStatus 3151 _nvswitch_ctrl_inband_send_data 3152 ( 3153 nvswitch_device *device, 3154 NVSWITCH_INBAND_SEND_DATA_PARAMS *p 3155 ) 3156 { 3157 return device->hal.nvswitch_ctrl_inband_send_data(device, p); 3158 } 3159 3160 static NvlStatus 3161 _nvswitch_ctrl_inband_read_data 3162 ( 3163 nvswitch_device *device, 3164 NVSWITCH_INBAND_READ_DATA_PARAMS *p 3165 ) 3166 { 3167 return device->hal.nvswitch_ctrl_inband_read_data(device, p); 3168 } 3169 3170 static NvlStatus 3171 _nvswitch_ctrl_inband_flush_data 3172 ( 3173 nvswitch_device *device, 3174 NVSWITCH_INBAND_FLUSH_DATA_PARAMS *p 3175 ) 3176 { 3177 NvU32 i; 3178 NvU64 enabledLinkMask; 3179 3180 if (p->linkMask == 0) 3181 { 3182 NVSWITCH_PRINT(device, ERROR, "Nothing to clear\n"); 3183 return NVL_SUCCESS; 3184 } 3185 3186 enabledLinkMask = nvswitch_get_enabled_link_mask(device); 3187 3188 FOR_EACH_INDEX_IN_MASK(64, i, p->linkMask) 3189 { 3190 if (nvswitch_is_link_valid(device, i) && 3191 (enabledLinkMask & NVBIT64(i))) 3192 { 3193 // 3194 // Flush is expected to clear both persistent and non-persistent 3195 // list. FM does flush when it wants to drop (ignore) all pending 3196 // messages w/o any NACKs. 3197 // 3198 _nvswitch_inband_clear_lists(device, i, 3199 NV_FALSE /* Nack */, 3200 NV_FALSE /* Non-persistent only */); 3201 } 3202 } 3203 FOR_EACH_INDEX_IN_MASK_END; 3204 3205 return NVL_SUCCESS; 3206 } 3207 3208 static NvlStatus 3209 _nvswitch_ctrl_inband_pending_data_stats 3210 ( 3211 nvswitch_device *device, 3212 NVSWITCH_INBAND_PENDING_DATA_STATS_PARAMS *p 3213 ) 3214 { 3215 NvU32 link_num; 3216 NvU64 enabledLinkMask, persistent_mask = 0, nonpersistent_mask = 0; 3217 3218 enabledLinkMask = nvswitch_get_enabled_link_mask(device); 3219 3220 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 3221 { 3222 if (nvswitch_is_link_valid(device, link_num) && 3223 (enabledLinkMask & NVBIT64(link_num))) 3224 { 3225 if (!nvListIsEmpty(&device->link[link_num].inbandData.persistent_list)) 3226 { 3227 persistent_mask |= NVBIT64(link_num); 3228 } 3229 3230 if (!nvListIsEmpty(&device->link[link_num].inbandData.nonpersistent_list)) 3231 { 3232 nonpersistent_mask |= NVBIT64(link_num); 3233 } 3234 } 3235 } 3236 3237 if (persistent_mask > 0) 3238 { 3239 p->linkMask = persistent_mask; 3240 } 3241 else 3242 { 3243 p->linkMask = nonpersistent_mask; 3244 } 3245 3246 return NVL_SUCCESS; 3247 } 3248 3249 static NvlStatus 3250 _nvswitch_ctrl_get_board_part_number 3251 ( 3252 nvswitch_device *device, 3253 NVSWITCH_GET_BOARD_PART_NUMBER_VECTOR *p 3254 ) 3255 { 3256 if (!nvswitch_is_inforom_supported(device)) 3257 { 3258 NVSWITCH_PRINT(device, ERROR, "InfoROM is not supported\n"); 3259 return -NVL_ERR_NOT_SUPPORTED; 3260 } 3261 3262 return device->hal.nvswitch_ctrl_get_board_part_number(device, p); 3263 } 3264 3265 static NvlStatus 3266 _nvswitch_ctrl_i2c_smbus_command 3267 ( 3268 nvswitch_device *device, 3269 NVSWITCH_I2C_SMBUS_COMMAND_PARAMS *pParams 3270 ) 3271 { 3272 NvU32 port_info; 3273 NvU32 port = pParams->port; 3274 NvU8 msgLen; 3275 NvU8 cmd; 3276 NvU16 addr; 3277 NvU8 cmdType; 3278 NvU8 *pData; 3279 NvBool is_i2c_access_allowed; 3280 NvBool is_port_allowed; 3281 3282 port_info = nvswitch_i2c_get_port_info(device, port); 3283 3284 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 3285 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 3286 NV_TRUE : NV_FALSE; 3287 is_port_allowed = is_i2c_access_allowed ? NV_TRUE : 3288 FLD_TEST_DRF(_I2C, _PORTINFO, _ACCESS_ALLOWED, _TRUE, 3289 port_info); 3290 3291 if (!is_port_allowed || 3292 !FLD_TEST_DRF(_I2C, _PORTINFO, _DEFINED, _PRESENT, port_info)) 3293 { 3294 NVSWITCH_PRINT(device, ERROR, "Invalid port access %d.\n", port); 3295 return NVL_BAD_ARGS; 3296 } 3297 3298 addr = pParams->deviceAddr; 3299 3300 switch (pParams->cmdType) 3301 { 3302 case NVSWITCH_I2C_SMBUS_CMD_QUICK: 3303 { 3304 cmd = 0; 3305 msgLen = 0; 3306 cmdType = pParams->bRead ? 3307 NVSWITCH_I2C_CMD_SMBUS_QUICK_READ : 3308 NVSWITCH_I2C_CMD_SMBUS_QUICK_WRITE; 3309 pData = NULL; 3310 break; 3311 } 3312 case NVSWITCH_I2C_SMBUS_CMD_BYTE: 3313 { 3314 cmd = 0; 3315 msgLen = 1; 3316 cmdType = pParams->bRead ? 3317 NVSWITCH_I2C_CMD_READ : 3318 NVSWITCH_I2C_CMD_WRITE; 3319 pData = (NvU8 *)&pParams->transactionData.smbusByte.message; 3320 break; 3321 } 3322 case NVSWITCH_I2C_SMBUS_CMD_BYTE_DATA: 3323 { 3324 msgLen = 1; 3325 cmd = pParams->transactionData.smbusByteData.cmd; 3326 cmdType = pParams->bRead ? 3327 NVSWITCH_I2C_CMD_SMBUS_READ : 3328 NVSWITCH_I2C_CMD_SMBUS_WRITE; 3329 pData = (NvU8 *)&pParams->transactionData.smbusByteData.message; 3330 break; 3331 } 3332 case NVSWITCH_I2C_SMBUS_CMD_WORD_DATA: 3333 { 3334 msgLen = 2; 3335 cmd = pParams->transactionData.smbusWordData.cmd; 3336 cmdType = pParams->bRead ? 3337 NVSWITCH_I2C_CMD_SMBUS_READ : 3338 NVSWITCH_I2C_CMD_SMBUS_WRITE; 3339 pData = (NvU8 *)&pParams->transactionData.smbusWordData.message; 3340 break; 3341 } 3342 default: 3343 { 3344 NVSWITCH_PRINT(device, ERROR, "Invalid Smbus command: %d.\n", port); 3345 return NVL_BAD_ARGS; 3346 } 3347 } 3348 3349 return _nvswitch_perform_i2c_transfer(device, NVSWITCH_I2C_ACQUIRER_IOCTL, 3350 cmdType, addr, port, cmd, msgLen, pData); 3351 } 3352 3353 static NvlStatus 3354 _nvswitch_ctrl_get_inforom_nvlink_max_correctable_error_rate 3355 ( 3356 nvswitch_device *device, 3357 NVSWITCH_GET_NVLINK_MAX_CORRECTABLE_ERROR_RATES_PARAMS *params 3358 ) 3359 { 3360 return nvswitch_inforom_nvlink_get_max_correctable_error_rate(device, params); 3361 } 3362 3363 static NvlStatus 3364 _nvswitch_ctrl_get_inforom_nvlink_errors 3365 ( 3366 nvswitch_device *device, 3367 NVSWITCH_GET_NVLINK_ERROR_COUNTS_PARAMS *params 3368 ) 3369 { 3370 return nvswitch_inforom_nvlink_get_errors(device, params); 3371 } 3372 3373 static NvlStatus 3374 _nvswitch_ctrl_get_inforom_ecc_errors 3375 ( 3376 nvswitch_device *device, 3377 NVSWITCH_GET_ECC_ERROR_COUNTS_PARAMS *params 3378 ) 3379 { 3380 return nvswitch_inforom_ecc_get_errors(device, params); 3381 } 3382 3383 static NvlStatus 3384 _nvswitch_ctrl_get_inforom_bbx_sxid 3385 ( 3386 nvswitch_device *device, 3387 NVSWITCH_GET_SXIDS_PARAMS *params 3388 ) 3389 { 3390 return nvswitch_inforom_bbx_get_sxid(device, params); 3391 } 3392 3393 static NvlStatus 3394 _nvswitch_ctrl_get_nvlink_lp_counters 3395 ( 3396 nvswitch_device *device, 3397 NVSWITCH_GET_NVLINK_LP_COUNTERS_PARAMS *params 3398 ) 3399 { 3400 return device->hal.nvswitch_ctrl_get_nvlink_lp_counters(device, params); 3401 } 3402 3403 static NvlStatus 3404 _nvswitch_ctrl_get_sw_info 3405 ( 3406 nvswitch_device *device, 3407 NVSWITCH_GET_SW_INFO_PARAMS *params 3408 ) 3409 { 3410 return device->hal.nvswitch_ctrl_get_sw_info(device, params); 3411 } 3412 3413 static NvlStatus 3414 _nvswitch_lib_validate_privileged_ctrl 3415 ( 3416 void *osPrivate, 3417 NvU64 flags 3418 ) 3419 { 3420 if (flags & NVSWITCH_DEV_CMD_CHECK_ADMIN) 3421 { 3422 if (nvswitch_os_is_admin()) 3423 { 3424 return NVL_SUCCESS; 3425 } 3426 } 3427 3428 if (flags & NVSWITCH_DEV_CMD_CHECK_FM) 3429 { 3430 if (nvswitch_os_is_fabric_manager(osPrivate)) 3431 { 3432 return NVL_SUCCESS; 3433 } 3434 } 3435 3436 return -NVL_ERR_INSUFFICIENT_PERMISSIONS; 3437 } 3438 3439 /* 3440 * @Brief : Copy the data from the persistant or nonpersistant list 3441 * 3442 * @Description : 3443 * 3444 * @param[in] device NvSwitch device to contain this link 3445 * @param[out] data Destination Data 3446 * @param[in] linkId link number of the link 3447 * @param[out] dataSize Size of data copied 3448 * 3449 * @returns NVL_SUCCESS if action succeeded, 3450 * -NVL_NOT_FOUND if link doesnt have data 3451 */ 3452 NvlStatus 3453 nvswitch_inband_read_data 3454 ( 3455 nvswitch_device *device, 3456 NvU8 *dest, 3457 NvU32 linkId, 3458 NvU32 *dataSize 3459 ) 3460 { 3461 nvswitch_inband_data_list *curr = NULL; 3462 NVListRec *list; 3463 3464 if (nvListIsEmpty(&device->link[linkId].inbandData.persistent_list) && 3465 nvListIsEmpty(&device->link[linkId].inbandData.nonpersistent_list)) 3466 { 3467 NVSWITCH_PRINT(device, ERROR, "%s: LinkId %d doesnt have any data to send\n", 3468 __FUNCTION__, linkId); 3469 *dataSize = 0; 3470 return -NVL_NOT_FOUND; 3471 } 3472 3473 list = nvListIsEmpty(&device->link[linkId].inbandData.persistent_list) ? 3474 &device->link[linkId].inbandData.nonpersistent_list : 3475 &device->link[linkId].inbandData.persistent_list; 3476 3477 nvListForEachEntry(curr, list, entry) 3478 { 3479 *dataSize = curr->dataSize; 3480 nvswitch_os_memcpy(dest, curr->data, curr->dataSize); 3481 nvListDel(&curr->entry); 3482 nvswitch_os_free(curr); 3483 break; 3484 } 3485 3486 return NVL_SUCCESS; 3487 } 3488 3489 /* 3490 * @Brief : Returns NV_TRUE if the given inband msg 3491 * needs to go to persistant list 3492 * 3493 * @Description : 3494 * 3495 * @param[in] device NvSwitch device to contain this link 3496 * @param[in] msghdr Header to the message 3497 * 3498 */ 3499 3500 static NvBool 3501 nvswitch_is_message_persistent 3502 ( 3503 nvswitch_device *device, 3504 nvlink_inband_msg_header_t *msghdr 3505 ) 3506 { 3507 // We expect only one message per received data 3508 switch(msghdr->type) 3509 { 3510 case NVLINK_INBAND_MSG_TYPE_MC_TEAM_RELEASE_REQ: 3511 return NV_TRUE; 3512 default: 3513 return NV_FALSE; 3514 } 3515 } 3516 3517 /* 3518 * @Brief : Moves the data into persistant or nonpersistant list 3519 * 3520 * @Description : 3521 * 3522 * @param[in] device NvSwitch device to contain this link 3523 * @param[in] linkId link number of the link 3524 * 3525 */ 3526 void 3527 nvswitch_filter_messages 3528 ( 3529 nvswitch_device *device, 3530 NvU32 linkId 3531 ) 3532 { 3533 NvlStatus status; 3534 nvlink_inband_msg_header_t *msghdr = NULL; 3535 nvswitch_inband_data_list *msg = device->link[linkId].inbandData.message; 3536 NvU8 *buffer = device->link[linkId].inbandData.message->data; 3537 NVSWITCH_DRIVER_FABRIC_STATE driver_fabric_state = 0; 3538 NvBool bSendNackOrDrop = NV_FALSE; 3539 3540 NVSWITCH_ASSERT(nvswitch_lib_read_fabric_state(device, NULL, NULL, 3541 &driver_fabric_state) == NVL_SUCCESS); 3542 3543 msghdr = (nvlink_inband_msg_header_t*)buffer; 3544 3545 if (nvswitch_is_message_persistent(device, msghdr)) 3546 { 3547 if (nvListCount(&device->link[linkId].inbandData.persistent_list) < 3548 device->hal.nvswitch_get_max_persistent_message_count(device)) 3549 { 3550 nvListAdd(&msg->entry, &device->link[linkId].inbandData.persistent_list); 3551 } 3552 else 3553 { 3554 bSendNackOrDrop = NV_TRUE; 3555 } 3556 } 3557 else 3558 { 3559 if (driver_fabric_state == NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED) 3560 { 3561 nvListAdd(&msg->entry, 3562 &device->link[linkId].inbandData.nonpersistent_list); 3563 } 3564 else 3565 { 3566 bSendNackOrDrop = NV_TRUE; 3567 } 3568 } 3569 3570 if (bSendNackOrDrop) 3571 { 3572 nvswitch_send_nack_or_drop(device, linkId, msghdr); 3573 nvswitch_os_free(msg); 3574 } 3575 else 3576 { 3577 status = nvswitch_lib_notify_client_events(device, 3578 NVSWITCH_DEVICE_EVENT_INBAND_DATA); 3579 if (status != NVL_SUCCESS) 3580 { 3581 NVSWITCH_PRINT(device, ERROR, "%s: Failed to notify INBAND_DATA event\n", 3582 __FUNCTION__); 3583 } 3584 } 3585 3586 device->link[linkId].inbandData.message = NULL; 3587 } 3588 3589 /* 3590 * @Brief : Constructs an NVS link struct with the given data 3591 * 3592 * @Description : 3593 * 3594 * @param[in] device NvSwitch device to contain this link 3595 * @param[in] link_num link number of the link 3596 * @param[out] link reference to store the created link into 3597 * 3598 * @returns NVL_SUCCESS if action succeeded, 3599 * -NVL_NO_MEM if memory allocation failed 3600 */ 3601 NvlStatus 3602 nvswitch_create_link 3603 ( 3604 nvswitch_device *device, 3605 NvU32 link_number, 3606 nvlink_link **link 3607 ) 3608 { 3609 NvlStatus retval = NVL_SUCCESS; 3610 nvlink_link *ret = NULL; 3611 LINK_INFO *link_info = NULL; 3612 NvU64 ac_coupled_mask; 3613 3614 NVSWITCH_ASSERT(nvswitch_get_num_links(device) <= NVSWITCH_MAX_NUM_LINKS); 3615 3616 ret = nvswitch_os_malloc(sizeof(*ret)); 3617 if (NULL == ret) 3618 { 3619 NVSWITCH_PRINT(device, ERROR, 3620 "nvswitch_os_malloc during link creation failed!\n"); 3621 retval = -NVL_NO_MEM; 3622 goto nvswitch_create_link_cleanup; 3623 } 3624 nvswitch_os_memset(ret, 0, sizeof(*ret)); 3625 3626 link_info = nvswitch_os_malloc(sizeof(*link_info)); 3627 if (NULL == link_info) 3628 { 3629 NVSWITCH_PRINT(device, ERROR, 3630 "nvswitch_os_malloc during link creation failed!\n"); 3631 retval = -NVL_NO_MEM; 3632 goto nvswitch_create_link_cleanup; 3633 } 3634 nvswitch_os_memset(link_info, 0, sizeof(*link_info)); 3635 nvswitch_os_snprintf(link_info->name, sizeof(link_info->name), NVSWITCH_LINK_NAME "%d", link_number); 3636 3637 ret->dev = device->nvlink_device; 3638 ret->linkName = link_info->name; 3639 ret->linkNumber = link_number; 3640 ret->state = NVLINK_LINKSTATE_OFF; 3641 ret->ac_coupled = NV_FALSE; 3642 ret->version = nvswitch_get_link_ip_version(device, link_number); 3643 3644 ac_coupled_mask = ((NvU64)device->regkeys.ac_coupled_mask2 << 32 | 3645 (NvU64)device->regkeys.ac_coupled_mask); 3646 3647 if (ac_coupled_mask) 3648 { 3649 if (ac_coupled_mask & NVBIT64(link_number)) 3650 { 3651 ret->ac_coupled = NV_TRUE; 3652 } 3653 } 3654 else if (device->firmware.nvlink.link_config_found) 3655 { 3656 if (device->firmware.nvlink.link_ac_coupled_mask & NVBIT64(link_number)) 3657 { 3658 ret->ac_coupled = NV_TRUE; 3659 } 3660 } 3661 3662 // Initialize NVLink corelib callbacks for switch 3663 nvswitch_get_link_handlers(&link_handlers); 3664 3665 ret->link_handlers = &link_handlers; 3666 3667 // 3668 // link_info is used to store private link information 3669 // 3670 3671 ret->link_info = link_info; 3672 3673 *link = ret; 3674 3675 return retval; 3676 3677 nvswitch_create_link_cleanup: 3678 if (NULL != ret) 3679 { 3680 nvswitch_os_free(ret); 3681 } 3682 if (NULL != link_info) 3683 { 3684 nvswitch_os_free(link_info); 3685 } 3686 3687 return retval; 3688 } 3689 3690 void 3691 nvswitch_destroy_link 3692 ( 3693 nvlink_link *link 3694 ) 3695 { 3696 if (NULL != link->link_info) 3697 { 3698 nvswitch_os_free(link->link_info); 3699 } 3700 3701 nvswitch_os_free(link); 3702 } 3703 3704 NvU32 3705 nvswitch_get_num_links 3706 ( 3707 nvswitch_device *device 3708 ) 3709 { 3710 return device->hal.nvswitch_get_num_links(device); 3711 } 3712 3713 NvBool 3714 nvswitch_is_link_valid 3715 ( 3716 nvswitch_device *device, 3717 NvU32 link_id 3718 ) 3719 { 3720 return device->hal.nvswitch_is_link_valid(device, link_id); 3721 } 3722 3723 nvlink_link* 3724 nvswitch_get_link(nvswitch_device *device, NvU8 link_id) 3725 { 3726 nvlink_link *link = NULL; 3727 3728 nvlink_lib_get_link(device->nvlink_device, link_id, &link); 3729 3730 return link; 3731 } 3732 3733 NvU64 3734 nvswitch_get_enabled_link_mask 3735 ( 3736 nvswitch_device *device 3737 ) 3738 { 3739 NvU64 ret; 3740 nvlink_link *link; 3741 NvU32 link_num; 3742 3743 ret = 0x0; 3744 3745 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 3746 { 3747 if (nvlink_lib_get_link(device->nvlink_device, link_num, &link) == NVL_SUCCESS) 3748 { 3749 ret |= NVBIT64(link_num); 3750 } 3751 } 3752 3753 return ret; 3754 } 3755 3756 void 3757 nvswitch_set_fatal_error 3758 ( 3759 nvswitch_device *device, 3760 NvBool device_fatal, 3761 NvU32 link_id 3762 ) 3763 { 3764 device->hal.nvswitch_set_fatal_error(device, device_fatal, link_id); 3765 } 3766 3767 NvU32 3768 nvswitch_get_swap_clk_default 3769 ( 3770 nvswitch_device *device 3771 ) 3772 { 3773 return device->hal.nvswitch_get_swap_clk_default(device); 3774 } 3775 3776 NvU32 3777 nvswitch_get_latency_sample_interval_msec 3778 ( 3779 nvswitch_device *device 3780 ) 3781 { 3782 return device->hal.nvswitch_get_latency_sample_interval_msec(device); 3783 } 3784 3785 void 3786 nvswitch_internal_latency_bin_log 3787 ( 3788 nvswitch_device *device 3789 ) 3790 { 3791 device->hal.nvswitch_internal_latency_bin_log(device); 3792 } 3793 3794 void 3795 nvswitch_ecc_writeback_task 3796 ( 3797 nvswitch_device *device 3798 ) 3799 { 3800 device->hal.nvswitch_ecc_writeback_task(device); 3801 } 3802 3803 void 3804 nvswitch_monitor_thermal_alert 3805 ( 3806 nvswitch_device *device 3807 ) 3808 { 3809 device->hal.nvswitch_monitor_thermal_alert(device); 3810 } 3811 3812 void 3813 nvswitch_hw_counter_shutdown 3814 ( 3815 nvswitch_device *device 3816 ) 3817 { 3818 device->hal.nvswitch_hw_counter_shutdown(device); 3819 } 3820 3821 NvlStatus 3822 nvswitch_get_rom_info 3823 ( 3824 nvswitch_device *device, 3825 NVSWITCH_EEPROM_TYPE *eeprom 3826 ) 3827 { 3828 return device->hal.nvswitch_get_rom_info(device, eeprom); 3829 } 3830 3831 void 3832 nvswitch_lib_enable_interrupts 3833 ( 3834 nvswitch_device *device 3835 ) 3836 { 3837 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 3838 { 3839 NVSWITCH_ASSERT(0); 3840 return; 3841 } 3842 3843 device->hal.nvswitch_lib_enable_interrupts(device); 3844 } 3845 3846 void 3847 nvswitch_lib_disable_interrupts 3848 ( 3849 nvswitch_device *device 3850 ) 3851 { 3852 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 3853 { 3854 NVSWITCH_ASSERT(0); 3855 return; 3856 } 3857 3858 device->hal.nvswitch_lib_disable_interrupts(device); 3859 } 3860 3861 NvlStatus 3862 nvswitch_lib_check_interrupts 3863 ( 3864 nvswitch_device *device 3865 ) 3866 { 3867 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 3868 { 3869 return -NVL_BAD_ARGS; 3870 } 3871 3872 return device->hal.nvswitch_lib_check_interrupts(device); 3873 } 3874 3875 NvlStatus 3876 nvswitch_lib_service_interrupts 3877 ( 3878 nvswitch_device *device 3879 ) 3880 { 3881 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 3882 { 3883 return -NVL_BAD_ARGS; 3884 } 3885 3886 return device->hal.nvswitch_lib_service_interrupts(device); 3887 } 3888 3889 NvU64 3890 nvswitch_hw_counter_read_counter 3891 ( 3892 nvswitch_device *device 3893 ) 3894 { 3895 return device->hal.nvswitch_hw_counter_read_counter(device); 3896 } 3897 3898 NvU32 3899 nvswitch_get_link_ip_version 3900 ( 3901 nvswitch_device *device, 3902 NvU32 link_id 3903 ) 3904 { 3905 return device->hal.nvswitch_get_link_ip_version(device, link_id); 3906 } 3907 3908 NvU32 3909 nvswitch_reg_read_32 3910 ( 3911 nvswitch_device *device, 3912 NvU32 offset 3913 ) 3914 { 3915 NvU32 val; 3916 3917 if (device->nvlink_device->pciInfo.bars[0].pBar == NULL) 3918 { 3919 NVSWITCH_PRINT(device, ERROR, 3920 "register read failed at offset 0x%x\n", offset); 3921 3922 return 0xFFFFFFFF; 3923 } 3924 3925 val = nvswitch_os_mem_read32((NvU8 *)device->nvlink_device->pciInfo.bars[0].pBar + offset); 3926 3927 if ((val & 0xFFFF0000) == 0xBADF0000) 3928 { 3929 NvU32 boot_0; 3930 NVSWITCH_PRINT(device, WARN, 3931 "Potential IO failure reading 0x%x (0x%x)\n", offset, val); 3932 boot_0 = nvswitch_os_mem_read32((NvU8 *)device->nvlink_device->pciInfo.bars[0].pBar + 0x0); 3933 3934 if ((boot_0 & 0xFFFF0000) == 0xBADF0000) 3935 { 3936 NVSWITCH_PRINT_SXID(device, NVSWITCH_ERR_HW_HOST_IO_FAILURE, 3937 "IO failure\n"); 3938 NVSWITCH_PRINT(device, ERROR, 3939 "IO failure reading 0x%x (0x%x)\n", offset, val); 3940 } 3941 } 3942 3943 #ifdef _VERBOSE_REG_ACCESS 3944 NVSWITCH_PRINT(device, SETUP, 3945 "NVSWITCH read 0x%6x+%6x = 0x%08x\n", 3946 device->nvlink_device->pciInfo.bars[0].baseAddr, offset, val); 3947 #endif 3948 3949 return val; 3950 } 3951 3952 void 3953 nvswitch_reg_write_32 3954 ( 3955 nvswitch_device *device, 3956 NvU32 offset, 3957 NvU32 data 3958 ) 3959 { 3960 if (device->nvlink_device->pciInfo.bars[0].pBar == NULL) 3961 { 3962 NVSWITCH_PRINT(device, ERROR, 3963 "register write failed at offset 0x%x\n", offset); 3964 3965 return; 3966 } 3967 3968 #ifdef _VERBOSE_REG_ACCESS 3969 NVSWITCH_PRINT(device, SETUP, 3970 "NVSWITCH write 0x%6x+%6x = 0x%08x\n", 3971 device->nvlink_device->pciInfo.bars[0].baseAddr, offset, data); 3972 #endif 3973 3974 // Write the register 3975 nvswitch_os_mem_write32((NvU8 *)device->nvlink_device->pciInfo.bars[0].pBar + offset, data); 3976 3977 return; 3978 } 3979 3980 NvU64 3981 nvswitch_read_64bit_counter 3982 ( 3983 nvswitch_device *device, 3984 NvU32 lo_offset, 3985 NvU32 hi_offset 3986 ) 3987 { 3988 NvU32 hi0; 3989 NvU32 hi1; 3990 NvU32 lo; 3991 3992 hi0 = nvswitch_reg_read_32(device, hi_offset); 3993 do 3994 { 3995 hi1 = hi0; 3996 lo = nvswitch_reg_read_32(device, lo_offset); 3997 hi0 = nvswitch_reg_read_32(device, hi_offset); 3998 } while (hi0 != hi1); 3999 4000 return (lo | ((NvU64)hi0 << 32)); 4001 } 4002 4003 NvlStatus 4004 nvswitch_validate_pll_config 4005 ( 4006 nvswitch_device *device, 4007 NVSWITCH_PLL_INFO *switch_pll, 4008 NVSWITCH_PLL_LIMITS default_pll_limits 4009 ) 4010 { 4011 NvU32 update_rate_khz; 4012 NvU32 vco_freq_khz; 4013 NVSWITCH_PLL_LIMITS pll_limits; 4014 4015 NVSWITCH_PRINT(device, SETUP, 4016 "%s: Validating PLL: %dkHz * %d / (%d * %d * (1 << %d))\n", 4017 __FUNCTION__, 4018 switch_pll->src_freq_khz, 4019 switch_pll->N, 4020 switch_pll->M, 4021 switch_pll->PL, 4022 switch_pll->dist_mode); 4023 4024 // 4025 // These parameters could come from schmoo'ing API, settings file or a ROM. 4026 // For now, hard code with POR. 4027 // 4028 if (device->firmware.firmware_size > 0 && 4029 device->firmware.clocks.clocks_found && 4030 device->firmware.clocks.sys_pll.valid) 4031 { 4032 pll_limits = device->firmware.clocks.sys_pll; 4033 } 4034 else 4035 { 4036 pll_limits = default_pll_limits; 4037 } 4038 4039 NVSWITCH_ASSERT(switch_pll->M != 0); 4040 NVSWITCH_ASSERT(switch_pll->PL != 0); 4041 4042 if ((switch_pll->src_freq_khz < pll_limits.ref_min_mhz * 1000) || 4043 (switch_pll->src_freq_khz > pll_limits.ref_max_mhz * 1000)) 4044 { 4045 NVSWITCH_PRINT(device, ERROR, 4046 "%s: ERROR: Ref(%d) out-of-range\n", 4047 __FUNCTION__, 4048 switch_pll->src_freq_khz); 4049 return -NVL_ERR_INVALID_STATE; 4050 } 4051 4052 if ((switch_pll->M < pll_limits.m_min) || 4053 (switch_pll->M > pll_limits.m_max)) 4054 { 4055 NVSWITCH_PRINT(device, ERROR, 4056 "%s: ERROR: M(%d) out-of-range\n", 4057 __FUNCTION__, 4058 switch_pll->M); 4059 return -NVL_ERR_INVALID_STATE; 4060 } 4061 4062 if ((switch_pll->N < pll_limits.n_min) || 4063 (switch_pll->N > pll_limits.n_max)) 4064 { 4065 NVSWITCH_PRINT(device, ERROR, 4066 "%s: ERROR: N(%d) out-of-range\n", 4067 __FUNCTION__, 4068 switch_pll->N); 4069 return -NVL_ERR_INVALID_STATE; 4070 } 4071 4072 if ((switch_pll->PL < pll_limits.pl_min) || 4073 (switch_pll->PL > pll_limits.pl_max)) 4074 { 4075 NVSWITCH_PRINT(device, ERROR, 4076 "%s: ERROR: PL(%d) out-of-range\n", 4077 __FUNCTION__, 4078 switch_pll->PL); 4079 return -NVL_ERR_INVALID_STATE; 4080 } 4081 4082 vco_freq_khz = switch_pll->src_freq_khz * switch_pll->N 4083 / switch_pll->M; 4084 if ((vco_freq_khz < pll_limits.vco_min_mhz * 1000) || 4085 (vco_freq_khz > pll_limits.vco_max_mhz * 1000)) 4086 { 4087 NVSWITCH_PRINT(device, ERROR, 4088 "%s: ERROR: VCO(%d) freq out-of-range\n", 4089 __FUNCTION__, 4090 vco_freq_khz); 4091 return -NVL_ERR_INVALID_STATE; 4092 } 4093 4094 update_rate_khz = switch_pll->src_freq_khz / switch_pll->M; 4095 if ((update_rate_khz < pll_limits.update_min_mhz * 1000) || 4096 (update_rate_khz > pll_limits.update_max_mhz * 1000)) 4097 { 4098 NVSWITCH_PRINT(device, ERROR, 4099 "%s: ERROR: update rate(%d) out-of-range\n", 4100 __FUNCTION__, 4101 update_rate_khz); 4102 return -NVL_ERR_INVALID_STATE; 4103 } 4104 4105 switch_pll->vco_freq_khz = vco_freq_khz; 4106 4107 switch_pll->freq_khz = 4108 switch_pll->src_freq_khz * switch_pll->N / 4109 (switch_pll->M * switch_pll->PL * (1 << switch_pll->dist_mode)); 4110 4111 NVSWITCH_PRINT(device, SETUP, 4112 "%s: Validated PLL: %dkHz * %d / (%d * %d * (1 << %d)) = %dkHz\n", 4113 __FUNCTION__, 4114 switch_pll->src_freq_khz, 4115 switch_pll->N, 4116 switch_pll->M, 4117 switch_pll->PL, 4118 switch_pll->dist_mode, 4119 switch_pll->freq_khz); 4120 4121 return NVL_SUCCESS; 4122 } 4123 4124 NvlStatus 4125 nvswitch_init_pll_config 4126 ( 4127 nvswitch_device *device 4128 ) 4129 { 4130 return device->hal.nvswitch_init_pll_config(device); 4131 } 4132 4133 NvlStatus 4134 nvswitch_init_pll 4135 ( 4136 nvswitch_device *device 4137 ) 4138 { 4139 return device->hal.nvswitch_init_pll(device); 4140 } 4141 4142 void 4143 nvswitch_init_clock_gating 4144 ( 4145 nvswitch_device *device 4146 ) 4147 { 4148 return device->hal.nvswitch_init_clock_gating(device); 4149 } 4150 4151 void 4152 nvswitch_lib_get_uuid 4153 ( 4154 nvswitch_device *device, 4155 NvUuid *uuid 4156 ) 4157 { 4158 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device) || (uuid == NULL)) 4159 { 4160 return; 4161 } 4162 4163 nvswitch_os_memcpy(uuid, &device->uuid, sizeof(device->uuid)); 4164 } 4165 4166 NvlStatus 4167 nvswitch_lib_get_physid 4168 ( 4169 nvswitch_device *device, 4170 NvU32 *phys_id 4171 ) 4172 { 4173 NVSWITCH_GET_INFO get_info; 4174 NvlStatus ret; 4175 4176 if (phys_id == NULL || !NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 4177 { 4178 return -NVL_BAD_ARGS; 4179 } 4180 4181 get_info.count=1; 4182 get_info.index[0] = NVSWITCH_GET_INFO_INDEX_PHYSICAL_ID; 4183 4184 ret = _nvswitch_ctrl_get_info(device, &get_info); 4185 if (ret != NVL_SUCCESS) 4186 { 4187 NVSWITCH_PRINT(device, ERROR, 4188 "Failed to get physical ID\n"); 4189 return ret; 4190 } 4191 4192 *phys_id = get_info.info[0]; 4193 4194 return NVL_SUCCESS; 4195 } 4196 4197 void 4198 nvswitch_i2c_set_hw_speed_mode 4199 ( 4200 nvswitch_device *device, 4201 NvU32 port, 4202 NvU32 speedMode 4203 ) 4204 { 4205 device->hal.nvswitch_i2c_set_hw_speed_mode(device, port, speedMode); 4206 return; 4207 } 4208 4209 void 4210 nvswitch_lib_smbpbi_log_sxid 4211 ( 4212 nvswitch_device *device, 4213 NvU32 sxid, 4214 const char *pFormat, 4215 ... 4216 ) 4217 { 4218 va_list arglist; 4219 int msglen; 4220 char string[RM_SOE_SMBPBI_CMD_LOG_MESSAGE_MAX_STRING + 1]; 4221 4222 nvswitch_os_memset(string, 0, (NvLength)sizeof(string)); 4223 4224 va_start(arglist, pFormat); 4225 msglen = nvswitch_os_vsnprintf(string, sizeof(string), pFormat, arglist); 4226 va_end(arglist); 4227 4228 if (!(msglen < 0)) 4229 { 4230 // 4231 // HALs will know that the string is being truncated by seeing that the 4232 // last byte in the buffer is not nul. 4233 // 4234 msglen = NV_MIN(msglen + 1, (int)RM_SOE_SMBPBI_CMD_LOG_MESSAGE_MAX_STRING); 4235 device->hal.nvswitch_smbpbi_log_message(device, sxid, msglen, (NvU8 *) string); 4236 } 4237 } 4238 4239 NvlStatus 4240 nvswitch_set_minion_initialized 4241 ( 4242 nvswitch_device *device, 4243 NvU32 idx_minion, 4244 NvBool initialized 4245 ) 4246 { 4247 return device->hal.nvswitch_set_minion_initialized(device, idx_minion, initialized); 4248 } 4249 4250 NvBool 4251 nvswitch_is_minion_initialized 4252 ( 4253 nvswitch_device *device, 4254 NvU32 idx_minion 4255 ) 4256 { 4257 return device->hal.nvswitch_is_minion_initialized(device, idx_minion); 4258 } 4259 4260 NvlStatus 4261 nvswitch_device_discovery 4262 ( 4263 nvswitch_device *device, 4264 NvU32 discovery_offset 4265 ) 4266 { 4267 return device->hal.nvswitch_device_discovery(device, discovery_offset); 4268 } 4269 4270 void 4271 nvswitch_filter_discovery 4272 ( 4273 nvswitch_device *device 4274 ) 4275 { 4276 device->hal.nvswitch_filter_discovery(device); 4277 } 4278 4279 NvlStatus 4280 nvswitch_process_discovery 4281 ( 4282 nvswitch_device *device 4283 ) 4284 { 4285 return device->hal.nvswitch_process_discovery(device); 4286 } 4287 4288 NvlStatus 4289 nvswitch_init_minion 4290 ( 4291 nvswitch_device *device 4292 ) 4293 { 4294 return device->hal.nvswitch_init_minion(device); 4295 } 4296 4297 NvU32 4298 nvswitch_get_link_eng_inst 4299 ( 4300 nvswitch_device *device, 4301 NvU32 link_id, 4302 NVSWITCH_ENGINE_ID eng_id 4303 ) 4304 { 4305 return device->hal.nvswitch_get_link_eng_inst(device, link_id, eng_id); 4306 } 4307 4308 void * 4309 nvswitch_alloc_chipdevice 4310 ( 4311 nvswitch_device *device 4312 ) 4313 { 4314 return(device->hal.nvswitch_alloc_chipdevice(device)); 4315 } 4316 4317 void 4318 nvswitch_free_chipdevice 4319 ( 4320 nvswitch_device *device 4321 ) 4322 { 4323 if (device->chip_device) 4324 { 4325 nvswitch_os_free(device->chip_device); 4326 device->chip_device = NULL; 4327 } 4328 } 4329 4330 NvlStatus 4331 nvswitch_init_thermal 4332 ( 4333 nvswitch_device *device 4334 ) 4335 { 4336 return(device->hal.nvswitch_init_thermal(device)); 4337 } 4338 4339 NvU32 4340 nvswitch_read_physical_id 4341 ( 4342 nvswitch_device *device 4343 ) 4344 { 4345 return(device->hal.nvswitch_read_physical_id(device)); 4346 } 4347 4348 NvU32 4349 nvswitch_get_caps_nvlink_version 4350 ( 4351 nvswitch_device *device 4352 ) 4353 { 4354 return(device->hal.nvswitch_get_caps_nvlink_version(device)); 4355 } 4356 4357 void 4358 nvswitch_initialize_interrupt_tree 4359 ( 4360 nvswitch_device *device 4361 ) 4362 { 4363 device->hal.nvswitch_initialize_interrupt_tree(device); 4364 } 4365 4366 void 4367 nvswitch_init_dlpl_interrupts 4368 ( 4369 nvlink_link *link 4370 ) 4371 { 4372 nvswitch_device *device = link->dev->pDevInfo; 4373 4374 device->hal.nvswitch_init_dlpl_interrupts(link); 4375 } 4376 4377 NvlStatus 4378 nvswitch_initialize_pmgr 4379 ( 4380 nvswitch_device *device 4381 ) 4382 { 4383 return(device->hal.nvswitch_initialize_pmgr(device)); 4384 } 4385 4386 NvlStatus 4387 nvswitch_initialize_ip_wrappers 4388 ( 4389 nvswitch_device *device 4390 ) 4391 { 4392 return(device->hal.nvswitch_initialize_ip_wrappers(device)); 4393 } 4394 4395 NvlStatus 4396 nvswitch_initialize_route 4397 ( 4398 nvswitch_device *device 4399 ) 4400 { 4401 return(device->hal.nvswitch_initialize_route(device)); 4402 } 4403 4404 void 4405 nvswitch_soe_unregister_events 4406 ( 4407 nvswitch_device *device 4408 ) 4409 { 4410 device->hal.nvswitch_soe_unregister_events(device); 4411 } 4412 4413 NvlStatus 4414 nvswitch_soe_register_event_callbacks 4415 ( 4416 nvswitch_device *device 4417 ) 4418 { 4419 return device->hal.nvswitch_soe_register_event_callbacks(device); 4420 } 4421 4422 NVSWITCH_BIOS_NVLINK_CONFIG * 4423 nvswitch_get_bios_nvlink_config 4424 ( 4425 nvswitch_device *device 4426 ) 4427 { 4428 return(device->hal.nvswitch_get_bios_nvlink_config(device)); 4429 } 4430 4431 NvlStatus 4432 nvswitch_minion_send_command 4433 ( 4434 nvswitch_device *device, 4435 NvU32 linkNumber, 4436 NvU32 command, 4437 NvU32 scratch0 4438 ) 4439 { 4440 return(device->hal.nvswitch_minion_send_command(device, linkNumber, 4441 command, scratch0)); 4442 } 4443 4444 NvlStatus 4445 nvswitch_init_nport 4446 ( 4447 nvswitch_device *device 4448 ) 4449 { 4450 return device->hal.nvswitch_init_nport(device); 4451 } 4452 4453 NvlStatus 4454 nvswitch_init_nxbar 4455 ( 4456 nvswitch_device *device 4457 ) 4458 { 4459 return device->hal.nvswitch_init_nxbar(device); 4460 } 4461 4462 NvlStatus 4463 nvswitch_clear_nport_rams 4464 ( 4465 nvswitch_device *device 4466 ) 4467 { 4468 return device->hal.nvswitch_clear_nport_rams(device); 4469 } 4470 4471 NvlStatus 4472 nvswitch_pri_ring_init 4473 ( 4474 nvswitch_device *device 4475 ) 4476 { 4477 return(device->hal.nvswitch_pri_ring_init(device)); 4478 } 4479 4480 NvlStatus 4481 nvswitch_get_remap_table_selector 4482 ( 4483 nvswitch_device *device, 4484 NVSWITCH_TABLE_SELECT_REMAP table_selector, 4485 NvU32 *remap_ram_sel 4486 ) 4487 { 4488 return device->hal.nvswitch_get_remap_table_selector(device, table_selector, remap_ram_sel); 4489 } 4490 4491 NvU32 4492 nvswitch_get_ingress_ram_size 4493 ( 4494 nvswitch_device *device, 4495 NvU32 ingress_ram_selector // NV_INGRESS_REQRSPMAPADDR_RAM_ADDRESS_* 4496 ) 4497 { 4498 return device->hal.nvswitch_get_ingress_ram_size(device, ingress_ram_selector); 4499 } 4500 4501 NvlStatus 4502 nvswitch_minion_get_dl_status 4503 ( 4504 nvswitch_device *device, 4505 NvU32 linkId, 4506 NvU32 statusIdx, 4507 NvU32 statusArgs, 4508 NvU32 *statusData 4509 ) 4510 { 4511 return device->hal.nvswitch_minion_get_dl_status(device, linkId, statusIdx, statusArgs, statusData); 4512 } 4513 4514 NvBool 4515 nvswitch_is_i2c_supported 4516 ( 4517 nvswitch_device *device 4518 ) 4519 { 4520 return device->hal.nvswitch_is_i2c_supported(device); 4521 } 4522 4523 4524 NvlStatus 4525 nvswitch_poll_sublink_state 4526 ( 4527 nvswitch_device *device, 4528 nvlink_link *link 4529 ) 4530 { 4531 return device->hal.nvswitch_poll_sublink_state(device, link); 4532 } 4533 4534 void 4535 nvswitch_setup_link_loopback_mode 4536 ( 4537 nvswitch_device *device, 4538 NvU32 linkNumber 4539 ) 4540 { 4541 return device->hal.nvswitch_setup_link_loopback_mode(device, linkNumber); 4542 } 4543 4544 void 4545 nvswitch_reset_persistent_link_hw_state 4546 ( 4547 nvswitch_device *device, 4548 NvU32 linkNumber 4549 ) 4550 { 4551 return device->hal.nvswitch_reset_persistent_link_hw_state(device, linkNumber); 4552 } 4553 4554 void 4555 nvswitch_store_topology_information 4556 ( 4557 nvswitch_device *device, 4558 nvlink_link *link 4559 ) 4560 { 4561 return device->hal.nvswitch_store_topology_information(device, link); 4562 } 4563 4564 void 4565 nvswitch_init_lpwr_regs 4566 ( 4567 nvlink_link *link 4568 ) 4569 { 4570 nvswitch_device *device = link->dev->pDevInfo; 4571 device->hal.nvswitch_init_lpwr_regs(link); 4572 } 4573 4574 NvlStatus 4575 nvswitch_launch_ALI 4576 ( 4577 nvswitch_device *device 4578 ) 4579 { 4580 return device->hal.nvswitch_launch_ALI(device); 4581 } 4582 4583 NvlStatus 4584 nvswitch_set_training_mode 4585 ( 4586 nvswitch_device *device 4587 ) 4588 { 4589 return device->hal.nvswitch_set_training_mode(device); 4590 } 4591 4592 NvBool 4593 nvswitch_is_link_in_reset 4594 ( 4595 nvswitch_device *device, 4596 nvlink_link *link 4597 ) 4598 { 4599 return device->hal.nvswitch_is_link_in_reset(device, link); 4600 } 4601 4602 NvBool 4603 nvswitch_i2c_is_device_access_allowed 4604 ( 4605 nvswitch_device *device, 4606 NvU32 port, 4607 NvU8 addr, 4608 NvBool bIsRead 4609 ) 4610 { 4611 return device->hal.nvswitch_i2c_is_device_access_allowed(device, port, addr, bIsRead); 4612 } 4613 4614 NvlStatus 4615 nvswitch_parse_bios_image 4616 ( 4617 nvswitch_device *device 4618 ) 4619 { 4620 return device->hal.nvswitch_parse_bios_image(device); 4621 } 4622 4623 void 4624 nvswitch_init_buffer_ready 4625 ( 4626 nvswitch_device *device, 4627 nvlink_link *link, 4628 NvBool bNportBufferReady 4629 ) 4630 { 4631 return device->hal.nvswitch_init_buffer_ready(device, link, bNportBufferReady); 4632 } 4633 4634 void 4635 nvswitch_apply_recal_settings 4636 ( 4637 nvswitch_device *device, 4638 nvlink_link *link 4639 ) 4640 { 4641 return device->hal.nvswitch_apply_recal_settings(device, link); 4642 } 4643 4644 NvlStatus 4645 nvswitch_launch_ALI_link_training 4646 ( 4647 nvswitch_device *device, 4648 nvlink_link *link, 4649 NvBool bSync 4650 ) 4651 { 4652 return device->hal.nvswitch_launch_ALI_link_training(device, link, bSync); 4653 } 4654 4655 NvlStatus 4656 nvswitch_reset_and_train_link 4657 ( 4658 nvswitch_device *device, 4659 nvlink_link *link 4660 ) 4661 { 4662 return device->hal.nvswitch_reset_and_train_link(device, link); 4663 } 4664 4665 static NvlStatus 4666 _nvswitch_ctrl_get_err_info 4667 ( 4668 nvswitch_device *device, 4669 NVSWITCH_NVLINK_GET_ERR_INFO_PARAMS *ret 4670 ) 4671 { 4672 return device->hal.nvswitch_ctrl_get_err_info(device, ret); 4673 } 4674 4675 static NvlStatus 4676 _nvswitch_ctrl_clear_counters 4677 ( 4678 nvswitch_device *device, 4679 NVSWITCH_NVLINK_CLEAR_COUNTERS_PARAMS *ret 4680 ) 4681 { 4682 return device->hal.nvswitch_ctrl_clear_counters(device, ret); 4683 } 4684 4685 void 4686 nvswitch_setup_link_system_registers 4687 ( 4688 nvswitch_device *device, 4689 nvlink_link *link 4690 ) 4691 { 4692 device->hal.nvswitch_setup_link_system_registers(device, link); 4693 } 4694 4695 void 4696 nvswitch_load_link_disable_settings 4697 ( 4698 nvswitch_device *device, 4699 nvlink_link *link 4700 ) 4701 { 4702 device->hal.nvswitch_load_link_disable_settings(device, link); 4703 } 4704 4705 static NvlStatus 4706 _nvswitch_ctrl_set_nvlink_error_threshold 4707 ( 4708 nvswitch_device *device, 4709 NVSWITCH_SET_NVLINK_ERROR_THRESHOLD_PARAMS *pParams 4710 ) 4711 { 4712 return device->hal.nvswitch_ctrl_set_nvlink_error_threshold(device, pParams); 4713 } 4714 4715 static NvlStatus 4716 _nvswitch_ctrl_get_nvlink_error_threshold 4717 ( 4718 nvswitch_device *device, 4719 NVSWITCH_GET_NVLINK_ERROR_THRESHOLD_PARAMS *pParams 4720 ) 4721 { 4722 return device->hal.nvswitch_ctrl_get_nvlink_error_threshold(device, pParams); 4723 } 4724 4725 static NvlStatus 4726 _nvswitch_ctrl_therm_read_voltage 4727 ( 4728 nvswitch_device *device, 4729 NVSWITCH_CTRL_GET_VOLTAGE_PARAMS *info 4730 ) 4731 { 4732 return device->hal.nvswitch_ctrl_therm_read_voltage(device, info); 4733 } 4734 4735 NvlStatus 4736 nvswitch_lib_ctrl 4737 ( 4738 nvswitch_device *device, 4739 NvU32 cmd, 4740 void *params, 4741 NvU64 size, 4742 void *osPrivate 4743 ) 4744 { 4745 NvlStatus retval; 4746 NvU64 flags = 0; 4747 4748 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device) || params == NULL) 4749 { 4750 return -NVL_BAD_ARGS; 4751 } 4752 4753 flags = NVSWITCH_DEV_CMD_CHECK_ADMIN | NVSWITCH_DEV_CMD_CHECK_FM; 4754 switch (cmd) 4755 { 4756 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_INFO, 4757 _nvswitch_ctrl_get_info, 4758 NVSWITCH_GET_INFO); 4759 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_INTERNAL_LATENCY, 4760 _nvswitch_ctrl_get_internal_latency, 4761 NVSWITCH_GET_INTERNAL_LATENCY); 4762 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLIPT_COUNTERS, 4763 _nvswitch_ctrl_get_nvlipt_counters, 4764 NVSWITCH_GET_NVLIPT_COUNTERS); 4765 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_ERRORS, 4766 nvswitch_ctrl_get_errors, 4767 NVSWITCH_GET_ERRORS_PARAMS); 4768 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLINK_STATUS, 4769 _nvswitch_ctrl_get_nvlink_status, 4770 NVSWITCH_GET_NVLINK_STATUS_PARAMS); 4771 NVSWITCH_DEV_CMD_DISPATCH_WITH_PRIVATE_DATA( 4772 CTRL_NVSWITCH_ACQUIRE_CAPABILITY, 4773 _nvswitch_ctrl_acquire_capability, 4774 NVSWITCH_ACQUIRE_CAPABILITY_PARAMS, 4775 osPrivate); 4776 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_TEMPERATURE, 4777 _nvswitch_ctrl_therm_read_temperature, 4778 NVSWITCH_CTRL_GET_TEMPERATURE_PARAMS); 4779 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_THROUGHPUT_COUNTERS, 4780 nvswitch_ctrl_get_throughput_counters, 4781 NVSWITCH_GET_THROUGHPUT_COUNTERS_PARAMS); 4782 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_FATAL_ERROR_SCOPE, 4783 _nvswitch_ctrl_get_fatal_error_scope, 4784 NVSWITCH_GET_FATAL_ERROR_SCOPE_PARAMS); 4785 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4786 CTRL_NVSWITCH_SET_SWITCH_PORT_CONFIG, 4787 _nvswitch_ctrl_set_switch_port_config, 4788 NVSWITCH_SET_SWITCH_PORT_CONFIG, 4789 osPrivate, flags); 4790 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4791 CTRL_NVSWITCH_GET_INGRESS_REQUEST_TABLE, 4792 _nvswitch_ctrl_get_ingress_request_table, 4793 NVSWITCH_GET_INGRESS_REQUEST_TABLE_PARAMS, 4794 osPrivate, flags); 4795 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4796 CTRL_NVSWITCH_SET_INGRESS_REQUEST_TABLE, 4797 _nvswitch_ctrl_set_ingress_request_table, 4798 NVSWITCH_SET_INGRESS_REQUEST_TABLE, 4799 osPrivate, flags); 4800 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4801 CTRL_NVSWITCH_SET_INGRESS_REQUEST_VALID, 4802 _nvswitch_ctrl_set_ingress_request_valid, 4803 NVSWITCH_SET_INGRESS_REQUEST_VALID, 4804 osPrivate, flags); 4805 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4806 CTRL_NVSWITCH_GET_INGRESS_RESPONSE_TABLE, 4807 _nvswitch_ctrl_get_ingress_response_table, 4808 NVSWITCH_GET_INGRESS_RESPONSE_TABLE_PARAMS, 4809 osPrivate, flags); 4810 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4811 CTRL_NVSWITCH_SET_INGRESS_RESPONSE_TABLE, 4812 _nvswitch_ctrl_set_ingress_response_table, 4813 NVSWITCH_SET_INGRESS_RESPONSE_TABLE, 4814 osPrivate, flags); 4815 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4816 CTRL_NVSWITCH_SET_GANGED_LINK_TABLE, 4817 _nvswitch_ctrl_set_ganged_link_table, 4818 NVSWITCH_SET_GANGED_LINK_TABLE, 4819 osPrivate, flags); 4820 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_LATENCY_BINS, 4821 nvswitch_ctrl_set_latency_bins, 4822 NVSWITCH_SET_LATENCY_BINS, 4823 osPrivate, flags); 4824 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4825 CTRL_NVSWITCH_SET_NVLIPT_COUNTER_CONFIG, 4826 _nvswitch_ctrl_set_nvlipt_counter_config, 4827 NVSWITCH_SET_NVLIPT_COUNTER_CONFIG, 4828 osPrivate, flags); 4829 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4830 CTRL_NVSWITCH_GET_NVLIPT_COUNTER_CONFIG, 4831 _nvswitch_ctrl_get_nvlipt_counter_config, 4832 NVSWITCH_GET_NVLIPT_COUNTER_CONFIG, 4833 osPrivate, flags); 4834 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_REMAP_POLICY, 4835 _nvswitch_ctrl_set_remap_policy, 4836 NVSWITCH_SET_REMAP_POLICY, 4837 osPrivate, flags); 4838 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_REMAP_POLICY, 4839 _nvswitch_ctrl_get_remap_policy, 4840 NVSWITCH_GET_REMAP_POLICY_PARAMS, 4841 osPrivate, flags); 4842 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4843 CTRL_NVSWITCH_SET_REMAP_POLICY_VALID, 4844 _nvswitch_ctrl_set_remap_policy_valid, 4845 NVSWITCH_SET_REMAP_POLICY_VALID, 4846 osPrivate, flags); 4847 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_ROUTING_ID, 4848 _nvswitch_ctrl_set_routing_id, 4849 NVSWITCH_SET_ROUTING_ID, 4850 osPrivate, flags); 4851 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_ROUTING_ID, 4852 _nvswitch_ctrl_get_routing_id, 4853 NVSWITCH_GET_ROUTING_ID_PARAMS, 4854 osPrivate, flags); 4855 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_ROUTING_ID_VALID, 4856 _nvswitch_ctrl_set_routing_id_valid, 4857 NVSWITCH_SET_ROUTING_LAN_VALID, 4858 osPrivate, flags); 4859 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_ROUTING_LAN, 4860 _nvswitch_ctrl_set_routing_lan, 4861 NVSWITCH_SET_ROUTING_LAN, 4862 osPrivate, flags); 4863 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_ROUTING_LAN, 4864 _nvswitch_ctrl_get_routing_lan, 4865 NVSWITCH_GET_ROUTING_LAN_PARAMS, 4866 osPrivate, flags); 4867 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4868 CTRL_NVSWITCH_SET_ROUTING_LAN_VALID, 4869 _nvswitch_ctrl_set_routing_lan_valid, 4870 NVSWITCH_SET_ROUTING_LAN_VALID, 4871 osPrivate, flags); 4872 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4873 CTRL_NVSWITCH_GET_INGRESS_REQLINKID, 4874 _nvswitch_ctrl_get_ingress_reqlinkid, 4875 NVSWITCH_GET_INGRESS_REQLINKID_PARAMS, 4876 osPrivate, flags); 4877 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_UNREGISTER_LINK, 4878 _nvswitch_ctrl_unregister_link, 4879 NVSWITCH_UNREGISTER_LINK_PARAMS, 4880 osPrivate, flags); 4881 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4882 CTRL_NVSWITCH_RESET_AND_DRAIN_LINKS, 4883 _nvswitch_ctrl_reset_and_drain_links, 4884 NVSWITCH_RESET_AND_DRAIN_LINKS_PARAMS, 4885 osPrivate, flags); 4886 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_BIOS_INFO, 4887 _nvswitch_ctrl_get_bios_info, 4888 NVSWITCH_GET_BIOS_INFO_PARAMS); 4889 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_INFOROM_VERSION, 4890 _nvswitch_ctrl_get_inforom_version, 4891 NVSWITCH_GET_INFOROM_VERSION_PARAMS); 4892 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4893 CTRL_NVSWITCH_BLACKLIST_DEVICE, 4894 nvswitch_ctrl_blacklist_device, 4895 NVSWITCH_BLACKLIST_DEVICE_PARAMS, 4896 osPrivate, flags); 4897 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4898 CTRL_NVSWITCH_SET_FM_DRIVER_STATE, 4899 nvswitch_ctrl_set_fm_driver_state, 4900 NVSWITCH_SET_FM_DRIVER_STATE_PARAMS, 4901 osPrivate, flags); 4902 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4903 CTRL_NVSWITCH_SET_DEVICE_FABRIC_STATE, 4904 nvswitch_ctrl_set_device_fabric_state, 4905 NVSWITCH_SET_DEVICE_FABRIC_STATE_PARAMS, 4906 osPrivate, flags); 4907 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4908 CTRL_NVSWITCH_SET_FM_HEARTBEAT_TIMEOUT, 4909 nvswitch_ctrl_set_fm_timeout, 4910 NVSWITCH_SET_FM_HEARTBEAT_TIMEOUT_PARAMS, 4911 osPrivate, flags); 4912 NVSWITCH_DEV_CMD_DISPATCH_WITH_PRIVATE_DATA( 4913 CTRL_NVSWITCH_REGISTER_EVENTS, 4914 _nvswitch_ctrl_register_events, 4915 NVSWITCH_REGISTER_EVENTS_PARAMS, 4916 osPrivate); 4917 NVSWITCH_DEV_CMD_DISPATCH_WITH_PRIVATE_DATA( 4918 CTRL_NVSWITCH_UNREGISTER_EVENTS, 4919 _nvswitch_ctrl_unregister_events, 4920 NVSWITCH_UNREGISTER_EVENTS_PARAMS, 4921 osPrivate); 4922 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4923 CTRL_NVSWITCH_SET_TRAINING_ERROR_INFO, 4924 _nvswitch_ctrl_set_training_error_info, 4925 NVSWITCH_SET_TRAINING_ERROR_INFO_PARAMS, 4926 osPrivate, flags); 4927 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4928 CTRL_NVSWITCH_SET_MC_RID_TABLE, 4929 _nvswitch_ctrl_set_mc_rid_table, 4930 NVSWITCH_SET_MC_RID_TABLE_PARAMS, 4931 osPrivate, flags); 4932 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4933 CTRL_NVSWITCH_GET_MC_RID_TABLE, 4934 _nvswitch_ctrl_get_mc_rid_table, 4935 NVSWITCH_GET_MC_RID_TABLE_PARAMS, 4936 osPrivate, flags); 4937 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4938 CTRL_NVSWITCH_GET_COUNTERS, 4939 _nvswitch_ctrl_get_counters, 4940 NVSWITCH_NVLINK_GET_COUNTERS_PARAMS, 4941 osPrivate, flags); 4942 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4943 CTRL_NVSWITCH_GET_NVLINK_ECC_ERRORS, 4944 _nvswitch_ctrl_get_nvlink_ecc_errors, 4945 NVSWITCH_GET_NVLINK_ECC_ERRORS_PARAMS, 4946 osPrivate, flags); 4947 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4948 CTRL_NVSWITCH_I2C_SMBUS_COMMAND, 4949 _nvswitch_ctrl_i2c_smbus_command, 4950 NVSWITCH_I2C_SMBUS_COMMAND_PARAMS, 4951 osPrivate, NVSWITCH_DEV_CMD_CHECK_ADMIN); 4952 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4953 CTRL_NVSWITCH_RESERVED_0); 4954 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4955 CTRL_NVSWITCH_RESERVED_1); 4956 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4957 CTRL_NVSWITCH_RESERVED_2); 4958 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4959 CTRL_NVSWITCH_RESERVED_3); 4960 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4961 CTRL_NVSWITCH_RESERVED_4); 4962 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4963 CTRL_NVSWITCH_RESERVED_5); 4964 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4965 CTRL_NVSWITCH_RESERVED_8); 4966 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4967 CTRL_NVSWITCH_RESERVED_9); 4968 NVSWITCH_DEV_CMD_DISPATCH_RESERVED( 4969 CTRL_NVSWITCH_RESERVED_10); 4970 NVSWITCH_DEV_CMD_DISPATCH( 4971 CTRL_NVSWITCH_GET_TEMPERATURE_LIMIT, 4972 _nvswitch_ctrl_therm_get_temperature_limit, 4973 NVSWITCH_CTRL_GET_TEMPERATURE_LIMIT_PARAMS); 4974 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4975 CTRL_NVSWITCH_GET_NVLINK_MAX_ERROR_RATES, 4976 _nvswitch_ctrl_get_inforom_nvlink_max_correctable_error_rate, 4977 NVSWITCH_GET_NVLINK_MAX_CORRECTABLE_ERROR_RATES_PARAMS, 4978 osPrivate, flags); 4979 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4980 CTRL_NVSWITCH_GET_NVLINK_ERROR_COUNTS, 4981 _nvswitch_ctrl_get_inforom_nvlink_errors, 4982 NVSWITCH_GET_NVLINK_ERROR_COUNTS_PARAMS, 4983 osPrivate, flags); 4984 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4985 CTRL_NVSWITCH_GET_ECC_ERROR_COUNTS, 4986 _nvswitch_ctrl_get_inforom_ecc_errors, 4987 NVSWITCH_GET_ECC_ERROR_COUNTS_PARAMS, 4988 osPrivate, flags); 4989 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4990 CTRL_NVSWITCH_GET_SXIDS, 4991 _nvswitch_ctrl_get_inforom_bbx_sxid, 4992 NVSWITCH_GET_SXIDS_PARAMS, 4993 osPrivate, flags); 4994 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 4995 CTRL_NVSWITCH_GET_FOM_VALUES, 4996 _nvswitch_ctrl_get_fom_values, 4997 NVSWITCH_GET_FOM_VALUES_PARAMS, 4998 osPrivate, flags); 4999 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5000 CTRL_NVSWITCH_GET_NVLINK_LP_COUNTERS, 5001 _nvswitch_ctrl_get_nvlink_lp_counters, 5002 NVSWITCH_GET_NVLINK_LP_COUNTERS_PARAMS, 5003 osPrivate, flags); 5004 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_RESIDENCY_BINS, 5005 _nvswitch_ctrl_get_residency_bins, 5006 NVSWITCH_GET_RESIDENCY_BINS); 5007 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_RESIDENCY_BINS, 5008 _nvswitch_ctrl_set_residency_bins, 5009 NVSWITCH_SET_RESIDENCY_BINS, 5010 osPrivate, flags); 5011 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_RB_STALL_BUSY, 5012 _nvswitch_ctrl_get_rb_stall_busy, 5013 NVSWITCH_GET_RB_STALL_BUSY); 5014 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_MULTICAST_ID_ERROR_VECTOR, 5015 nvswitch_ctrl_get_multicast_id_error_vector, 5016 NVSWITCH_GET_MULTICAST_ID_ERROR_VECTOR); 5017 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_CLEAR_MULTICAST_ID_ERROR_VECTOR, 5018 nvswitch_ctrl_clear_multicast_id_error_vector, 5019 NVSWITCH_CLEAR_MULTICAST_ID_ERROR_VECTOR); 5020 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5021 CTRL_NVSWITCH_INBAND_SEND_DATA, 5022 _nvswitch_ctrl_inband_send_data, 5023 NVSWITCH_INBAND_SEND_DATA_PARAMS, 5024 osPrivate, flags); 5025 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5026 CTRL_NVSWITCH_INBAND_READ_DATA, 5027 _nvswitch_ctrl_inband_read_data, 5028 NVSWITCH_INBAND_READ_DATA_PARAMS, 5029 osPrivate, flags); 5030 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5031 CTRL_NVSWITCH_INBAND_FLUSH_DATA, 5032 _nvswitch_ctrl_inband_flush_data, 5033 NVSWITCH_INBAND_FLUSH_DATA_PARAMS, 5034 osPrivate, flags); 5035 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5036 CTRL_NVSWITCH_INBAND_PENDING_DATA_STATS, 5037 _nvswitch_ctrl_inband_pending_data_stats, 5038 NVSWITCH_INBAND_PENDING_DATA_STATS_PARAMS, 5039 osPrivate, flags); 5040 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_BOARD_PART_NUMBER, 5041 _nvswitch_ctrl_get_board_part_number, 5042 NVSWITCH_GET_BOARD_PART_NUMBER_VECTOR); 5043 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5044 CTRL_NVSWITCH_GET_SW_INFO, 5045 _nvswitch_ctrl_get_sw_info, 5046 NVSWITCH_GET_SW_INFO_PARAMS, 5047 osPrivate, flags); 5048 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_REGISTER_READ, 5049 _nvswitch_ctrl_register_read, 5050 NVSWITCH_REGISTER_READ, 5051 osPrivate, flags); 5052 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_REGISTER_WRITE, 5053 _nvswitch_ctrl_register_write, 5054 NVSWITCH_REGISTER_WRITE, 5055 osPrivate, flags); 5056 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_ERR_INFO, 5057 _nvswitch_ctrl_get_err_info, 5058 NVSWITCH_NVLINK_GET_ERR_INFO_PARAMS, 5059 osPrivate, flags); 5060 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_CLEAR_COUNTERS, 5061 _nvswitch_ctrl_clear_counters, 5062 NVSWITCH_NVLINK_CLEAR_COUNTERS_PARAMS, 5063 osPrivate, flags); 5064 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_NVLINK_ERROR_THRESHOLD, 5065 _nvswitch_ctrl_set_nvlink_error_threshold, 5066 NVSWITCH_SET_NVLINK_ERROR_THRESHOLD_PARAMS, 5067 osPrivate, flags); 5068 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLINK_ERROR_THRESHOLD, 5069 _nvswitch_ctrl_get_nvlink_error_threshold, 5070 NVSWITCH_GET_NVLINK_ERROR_THRESHOLD_PARAMS); 5071 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_VOLTAGE, 5072 _nvswitch_ctrl_therm_read_voltage, 5073 NVSWITCH_CTRL_GET_VOLTAGE_PARAMS); 5074 5075 default: 5076 nvswitch_os_print(NVSWITCH_DBG_LEVEL_INFO, "unknown ioctl %x\n", cmd); 5077 retval = -NVL_BAD_ARGS; 5078 break; 5079 } 5080 5081 return retval; 5082 } 5083