1 /* 2 * SPDX-FileCopyrightText: Copyright (c) 2017-2023 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 * SPDX-License-Identifier: MIT 4 * 5 * Permission is hereby granted, free of charge, to any person obtaining a 6 * copy of this software and associated documentation files (the "Software"), 7 * to deal in the Software without restriction, including without limitation 8 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 9 * and/or sell copies of the Software, and to permit persons to whom the 10 * Software is furnished to do so, subject to the following conditions: 11 * 12 * The above copyright notice and this permission notice shall be included in 13 * all copies or substantial portions of the Software. 14 * 15 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 18 * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER 21 * DEALINGS IN THE SOFTWARE. 22 */ 23 24 #include "common_nvswitch.h" 25 #include "rom_nvswitch.h" 26 #include "error_nvswitch.h" 27 #include "regkey_nvswitch.h" 28 #include "bios_nvswitch.h" 29 #include "haldef_nvswitch.h" 30 #include "flcn/haldefs_flcnable_nvswitch.h" 31 #include "flcn/flcn_nvswitch.h" 32 #include "soe/soe_nvswitch.h" 33 #include "soe/soeififr.h" 34 #include "nvVer.h" 35 #include "nvlink_inband_msg.h" 36 #include "cci/cci_nvswitch.h" 37 38 static NvlStatus _nvswitch_ctrl_inband_flush_data(nvswitch_device *device, NVSWITCH_INBAND_FLUSH_DATA_PARAMS *p); 39 40 #define NVSWITCH_DEV_CMD_CHECK_ADMIN NVBIT64(0) 41 #define NVSWITCH_DEV_CMD_CHECK_FM NVBIT64(1) 42 43 #define NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(cmd, function, type, private, flags)\ 44 case cmd: \ 45 { \ 46 if (sizeof(type) != size) \ 47 { \ 48 retval = -NVL_BAD_ARGS; \ 49 break; \ 50 } \ 51 \ 52 retval = _nvswitch_lib_validate_privileged_ctrl(private, flags); \ 53 if (retval != NVL_SUCCESS) \ 54 { \ 55 break; \ 56 } \ 57 \ 58 retval = function(device, params); \ 59 break; \ 60 } \ 61 62 #define NVSWITCH_DEV_CMD_DISPATCH_RESERVED(cmd) \ 63 case cmd: \ 64 { \ 65 retval = -NVL_ERR_NOT_SUPPORTED; \ 66 break; \ 67 } \ 68 69 const static NvU32 nvswitch_lr10_device_ids[] = 70 { 71 0x1AE8, 0x1AF0, 0x1AF1, 0x1AF2, 0x1AF3, 0x1AF4, 0x1AF5, 0x1AF6, 0x1AF7, 72 0x1AF8, 0x1AF9, 0x1AFA, 0x1AFB, 0x1AFC, 0x1AFD, 0x1AFE, 0x1AFF 73 }; 74 75 const static NvU32 nvswitch_ls10_device_ids[] = 76 { 77 // PCIE endpoint to manage the NVLink switch HW 78 0x22A0, 0x22A1, 0x22A2, 0x22A3, 0x22A4, 0x22A5, 0x22A6, 0x22A7, 79 // PCI-PCI Bridge, Laguna Switch Function 0 80 0x22A8, 0x22A9, 0x22AA, 0x22AB, 81 // Non-Transparent Bridge, Laguna Switch Function 1 82 0x22AC, 0x22AD, 0x22AE, 0x22AF 83 }; 84 85 nvlink_link_handlers link_handlers; 86 87 static NvBool 88 _nvswitch_is_device_id_present 89 ( 90 const NvU32 *array, 91 NvU32 array_len, 92 NvU32 device_id 93 ) 94 { 95 NvU32 i = 0; 96 97 for(i = 0; i < array_len; i++) 98 { 99 if (array[i] == device_id) 100 { 101 return NV_TRUE; 102 } 103 } 104 105 return NV_FALSE; 106 } 107 108 NvBool 109 nvswitch_is_lr10_device_id 110 ( 111 NvU32 device_id 112 ) 113 { 114 NvU32 count = (sizeof(nvswitch_lr10_device_ids) / 115 sizeof(nvswitch_lr10_device_ids[0])); 116 117 return _nvswitch_is_device_id_present(nvswitch_lr10_device_ids, count, device_id); 118 } 119 120 NvBool 121 nvswitch_is_ls10_device_id 122 ( 123 NvU32 device_id 124 ) 125 { 126 NvU32 count = (sizeof(nvswitch_ls10_device_ids) / 127 sizeof(nvswitch_ls10_device_ids[0])); 128 129 return _nvswitch_is_device_id_present(nvswitch_ls10_device_ids, count, device_id); 130 } 131 132 /* 133 * NVLink corelib callbacks are used by the NVLink library separate from the 134 * NVSwitch driver, therefore they do not take a device lock and can not modify 135 * nvswitch_device state or use error logging. 136 * 137 * These NVSwitch functions modify link state outside of the corelib: 138 * _nvswitch_ctrl_inject_link_error - injects asynchronous link errors (MODS-only) 139 */ 140 141 static NV_API_CALL NvlStatus 142 _nvswitch_corelib_add_link 143 ( 144 nvlink_link *link 145 ) 146 { 147 nvswitch_device *device = link->dev->pDevInfo; 148 return device->hal.nvswitch_corelib_add_link(link); 149 } 150 151 static NV_API_CALL NvlStatus 152 _nvswitch_corelib_remove_link 153 ( 154 nvlink_link *link 155 ) 156 { 157 nvswitch_device *device = link->dev->pDevInfo; 158 return device->hal.nvswitch_corelib_remove_link(link); 159 } 160 161 static NV_API_CALL NvlStatus 162 _nvswitch_corelib_set_dl_link_mode 163 ( 164 nvlink_link *link, 165 NvU64 mode, 166 NvU32 flags 167 ) 168 { 169 nvswitch_device *device = link->dev->pDevInfo; 170 return device->hal.nvswitch_corelib_set_dl_link_mode(link, mode, flags); 171 } 172 173 static NV_API_CALL NvlStatus 174 _nvswitch_corelib_get_dl_link_mode 175 ( 176 nvlink_link *link, 177 NvU64 *mode 178 ) 179 { 180 nvswitch_device *device = link->dev->pDevInfo; 181 return device->hal.nvswitch_corelib_get_dl_link_mode(link, mode); 182 } 183 184 static NV_API_CALL NvlStatus 185 _nvswitch_corelib_set_tl_link_mode 186 ( 187 nvlink_link *link, 188 NvU64 mode, 189 NvU32 flags 190 ) 191 { 192 nvswitch_device *device = link->dev->pDevInfo; 193 return device->hal.nvswitch_corelib_set_tl_link_mode(link, mode, flags); 194 } 195 196 static NV_API_CALL NvlStatus 197 _nvswitch_corelib_get_tl_link_mode 198 ( 199 nvlink_link *link, 200 NvU64 *mode 201 ) 202 { 203 nvswitch_device *device = link->dev->pDevInfo; 204 return device->hal.nvswitch_corelib_get_tl_link_mode(link, mode); 205 } 206 207 static NV_API_CALL NvlStatus 208 _nvswitch_corelib_set_tx_mode 209 ( 210 nvlink_link *link, 211 NvU64 mode, 212 NvU32 flags 213 ) 214 { 215 nvswitch_device *device = link->dev->pDevInfo; 216 return device->hal.nvswitch_corelib_set_tx_mode(link, mode, flags); 217 } 218 219 static NV_API_CALL NvlStatus 220 _nvswitch_corelib_get_tx_mode 221 ( 222 nvlink_link *link, 223 NvU64 *mode, 224 NvU32 *subMode 225 ) 226 { 227 nvswitch_device *device = link->dev->pDevInfo; 228 return device->hal.nvswitch_corelib_get_tx_mode(link, mode, subMode); 229 } 230 231 static NV_API_CALL NvlStatus 232 _nvswitch_corelib_set_rx_mode 233 ( 234 nvlink_link *link, 235 NvU64 mode, 236 NvU32 flags 237 ) 238 { 239 nvswitch_device *device = link->dev->pDevInfo; 240 return device->hal.nvswitch_corelib_set_rx_mode(link, mode, flags); 241 } 242 243 static NV_API_CALL NvlStatus 244 _nvswitch_corelib_get_rx_mode 245 ( 246 nvlink_link *link, 247 NvU64 *mode, 248 NvU32 *subMode 249 ) 250 { 251 nvswitch_device *device = link->dev->pDevInfo; 252 return device->hal.nvswitch_corelib_get_rx_mode(link, mode, subMode); 253 } 254 255 static NV_API_CALL NvlStatus 256 _nvswitch_corelib_set_rx_detect 257 ( 258 nvlink_link *link, 259 NvU32 flags 260 ) 261 { 262 nvswitch_device *device = link->dev->pDevInfo; 263 return device->hal.nvswitch_corelib_set_rx_detect(link, flags); 264 } 265 266 static NV_API_CALL NvlStatus 267 _nvswitch_corelib_get_rx_detect 268 ( 269 nvlink_link *link 270 ) 271 { 272 nvswitch_device *device = link->dev->pDevInfo; 273 return device->hal.nvswitch_corelib_get_rx_detect(link); 274 } 275 276 static NV_API_CALL void 277 _nvswitch_corelib_training_complete 278 ( 279 nvlink_link *link 280 ) 281 { 282 nvswitch_device *device = link->dev->pDevInfo; 283 device->hal.nvswitch_corelib_training_complete(link); 284 } 285 286 static NV_API_CALL void 287 _nvswitch_corelib_get_uphy_load 288 ( 289 nvlink_link *link, 290 NvBool *bUnlocked 291 ) 292 { 293 nvswitch_device *device = link->dev->pDevInfo; 294 return device->hal.nvswitch_corelib_get_uphy_load(link, bUnlocked); 295 } 296 297 298 static NV_API_CALL NvlStatus 299 _nvswitch_corelib_write_discovery_token 300 ( 301 nvlink_link *link, 302 NvU64 token 303 ) 304 { 305 nvswitch_device *device = link->dev->pDevInfo; 306 307 if (link->version >= NVLINK_DEVICE_VERSION_40) 308 { 309 nvswitch_store_topology_information(device, link); 310 return NVL_SUCCESS; 311 } 312 313 return NVL_SUCCESS; 314 } 315 316 static NV_API_CALL NvlStatus 317 _nvswitch_corelib_read_discovery_token 318 ( 319 nvlink_link *link, 320 NvU64 *token 321 ) 322 { 323 if (link->version >= NVLINK_DEVICE_VERSION_40) 324 { 325 return NVL_SUCCESS; 326 } 327 return NVL_SUCCESS; 328 } 329 330 static NV_API_CALL NvlStatus 331 _nvswitch_corelib_get_cci_link_mode 332 ( 333 nvlink_link *link, 334 NvU64 *mode 335 ) 336 { 337 nvswitch_device *device = link->dev->pDevInfo; 338 return device->hal.nvswitch_cci_get_cci_link_mode(device, link->linkNumber, mode); 339 } 340 341 static NV_API_CALL NvlStatus 342 _nvswitch_corelib_ali_training 343 ( 344 nvlink_link *link 345 ) 346 { 347 nvswitch_device *device = link->dev->pDevInfo; 348 return device->hal.nvswitch_launch_ALI_link_training(device, link, NV_FALSE); 349 } 350 351 void 352 nvswitch_get_link_handlers 353 ( 354 nvlink_link_handlers *nvswitch_link_handlers 355 ) 356 { 357 if (!nvswitch_link_handlers) 358 { 359 NVSWITCH_ASSERT(0); 360 return; 361 } 362 363 nvswitch_link_handlers->add = _nvswitch_corelib_add_link; 364 nvswitch_link_handlers->remove = _nvswitch_corelib_remove_link; 365 nvswitch_link_handlers->set_dl_link_mode = _nvswitch_corelib_set_dl_link_mode; 366 nvswitch_link_handlers->get_dl_link_mode = _nvswitch_corelib_get_dl_link_mode; 367 nvswitch_link_handlers->set_tl_link_mode = _nvswitch_corelib_set_tl_link_mode; 368 nvswitch_link_handlers->get_tl_link_mode = _nvswitch_corelib_get_tl_link_mode; 369 nvswitch_link_handlers->set_tx_mode = _nvswitch_corelib_set_tx_mode; 370 nvswitch_link_handlers->get_tx_mode = _nvswitch_corelib_get_tx_mode; 371 nvswitch_link_handlers->set_rx_mode = _nvswitch_corelib_set_rx_mode; 372 nvswitch_link_handlers->get_rx_mode = _nvswitch_corelib_get_rx_mode; 373 nvswitch_link_handlers->set_rx_detect = _nvswitch_corelib_set_rx_detect; 374 nvswitch_link_handlers->get_rx_detect = _nvswitch_corelib_get_rx_detect; 375 nvswitch_link_handlers->training_complete = _nvswitch_corelib_training_complete; 376 nvswitch_link_handlers->get_uphy_load = _nvswitch_corelib_get_uphy_load; 377 nvswitch_link_handlers->write_discovery_token = _nvswitch_corelib_write_discovery_token; 378 nvswitch_link_handlers->read_discovery_token = _nvswitch_corelib_read_discovery_token; 379 nvswitch_link_handlers->ali_training = _nvswitch_corelib_ali_training; 380 nvswitch_link_handlers->get_cci_link_mode = _nvswitch_corelib_get_cci_link_mode; 381 } 382 383 #define NVSWITCH_INIT_REGKEY(_private, _regkey, _string, _default_val) \ 384 do \ 385 { \ 386 NvU32 data; \ 387 \ 388 device->regkeys._regkey = _default_val; \ 389 if (NV_SWITCH_REGKEY_PRIVATE_ALLOWED || !NV_SWITCH_REGKEY##_private) \ 390 { \ 391 if (NVL_SUCCESS == \ 392 nvswitch_os_read_registry_dword(device->os_handle, _string, &data)) \ 393 { \ 394 NVSWITCH_PRINT(device, SETUP, \ 395 "%s: Applying regkey %s=0x%x\n", \ 396 __FUNCTION__, \ 397 _string, data); \ 398 device->regkeys._regkey = data; \ 399 } \ 400 } \ 401 } while(0) 402 403 static void 404 _nvswitch_init_device_regkeys 405 ( 406 nvswitch_device *device 407 ) 408 { 409 // 410 // Public external use regkeys 411 // 412 NVSWITCH_INIT_REGKEY(_PUBLIC, ato_control, 413 NV_SWITCH_REGKEY_ATO_CONTROL, 414 NV_SWITCH_REGKEY_ATO_CONTROL_DEFAULT); 415 416 NVSWITCH_INIT_REGKEY(_PUBLIC, sto_control, 417 NV_SWITCH_REGKEY_STO_CONTROL, 418 NV_SWITCH_REGKEY_STO_CONTROL_DEFAULT); 419 420 NVSWITCH_INIT_REGKEY(_PUBLIC, crc_bit_error_rate_short, 421 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_SHORT, 422 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_SHORT_DEFAULT); 423 424 NVSWITCH_INIT_REGKEY(_PUBLIC, crc_bit_error_rate_long, 425 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_LONG, 426 NV_SWITCH_REGKEY_CRC_BIT_ERROR_RATE_LONG_DEFAULT); 427 428 NVSWITCH_INIT_REGKEY(_PUBLIC, surpress_link_errors_for_gpu_reset, 429 NV_SWITCH_REGKEY_SURPRESS_LINK_ERRORS_FOR_GPU_RESET, 430 NV_SWITCH_REGKEY_SURPRESS_LINK_ERRORS_FOR_GPU_RESET_DISABLE); 431 432 NVSWITCH_INIT_REGKEY(_PUBLIC, cci_control, 433 NV_SWITCH_REGKEY_CCI_CONTROL, 434 NV_SWITCH_REGKEY_CCI_CONTROL_DEFAULT); 435 NVSWITCH_INIT_REGKEY(_PRIVATE, cci_link_train_disable_mask, 436 NV_SWITCH_REGKEY_CCI_DISABLE_LINK_TRAIN_MASK, 437 NV_SWITCH_REGKEY_CCI_DISABLE_LINK_TRAIN_MASK_DEFAULT); 438 NVSWITCH_INIT_REGKEY(_PRIVATE, cci_link_train_disable_mask2, 439 NV_SWITCH_REGKEY_CCI_DISABLE_LINK_TRAIN_MASK2, 440 NV_SWITCH_REGKEY_CCI_DISABLE_LINK_TRAIN_MASK2_DEFAULT); 441 NVSWITCH_INIT_REGKEY(_PUBLIC, cci_max_onboard_attempts, 442 NV_SWITCH_REGKEY_CCI_MAX_ONBOARD_ATTEMPTS, 443 NV_SWITCH_REGKEY_CCI_MAX_ONBOARD_ATTEMPTS_DEFAULT); 444 NVSWITCH_INIT_REGKEY(_PUBLIC, cci_error_log_enable, 445 NV_SWITCH_REGKEY_CCI_ERROR_LOG_ENABLE, 446 NV_SWITCH_REGKEY_CCI_ERROR_LOG_ENABLE_DEFAULT); 447 // 448 // Debug use regkeys 449 // Not available on release build kernel drivers 450 // 451 NVSWITCH_INIT_REGKEY(_PRIVATE, external_fabric_mgmt, 452 NV_SWITCH_REGKEY_EXTERNAL_FABRIC_MGMT, 453 NV_SWITCH_REGKEY_EXTERNAL_FABRIC_MGMT_ENABLE); 454 455 NVSWITCH_INIT_REGKEY(_PRIVATE, txtrain_control, 456 NV_SWITCH_REGKEY_TXTRAIN_CONTROL, 457 NV_SWITCH_REGKEY_TXTRAIN_CONTROL_NOP); 458 459 NVSWITCH_INIT_REGKEY(_PRIVATE, crossbar_DBI, 460 NV_SWITCH_REGKEY_CROSSBAR_DBI, 461 NV_SWITCH_REGKEY_CROSSBAR_DBI_ENABLE); 462 463 NVSWITCH_INIT_REGKEY(_PRIVATE, link_DBI, 464 NV_SWITCH_REGKEY_LINK_DBI, 465 NV_SWITCH_REGKEY_LINK_DBI_ENABLE); 466 467 NVSWITCH_INIT_REGKEY(_PRIVATE, ac_coupled_mask, 468 NV_SWITCH_REGKEY_AC_COUPLED_MASK, 469 0); 470 471 NVSWITCH_INIT_REGKEY(_PRIVATE, ac_coupled_mask2, 472 NV_SWITCH_REGKEY_AC_COUPLED_MASK2, 473 0); 474 475 NVSWITCH_INIT_REGKEY(_PRIVATE, swap_clk, 476 NV_SWITCH_REGKEY_SWAP_CLK_OVERRIDE, 477 nvswitch_get_swap_clk_default(device)); 478 479 NVSWITCH_INIT_REGKEY(_PRIVATE, link_enable_mask, 480 NV_SWITCH_REGKEY_ENABLE_LINK_MASK, 481 NV_U32_MAX); 482 483 NVSWITCH_INIT_REGKEY(_PRIVATE, link_enable_mask2, 484 NV_SWITCH_REGKEY_ENABLE_LINK_MASK2, 485 NV_U32_MAX); 486 487 NVSWITCH_INIT_REGKEY(_PRIVATE, bandwidth_shaper, 488 NV_SWITCH_REGKEY_BANDWIDTH_SHAPER, 489 NV_SWITCH_REGKEY_BANDWIDTH_SHAPER_PROD); 490 491 NVSWITCH_INIT_REGKEY(_PRIVATE, ssg_control, 492 NV_SWITCH_REGKEY_SSG_CONTROL, 493 0); 494 495 NVSWITCH_INIT_REGKEY(_PRIVATE, skip_buffer_ready, 496 NV_SWITCH_REGKEY_SKIP_BUFFER_READY, 497 0); 498 499 NVSWITCH_INIT_REGKEY(_PUBLIC, enable_pm, 500 NV_SWITCH_REGKEY_ENABLE_PM, 501 NV_SWITCH_REGKEY_ENABLE_PM_YES); 502 503 NVSWITCH_INIT_REGKEY(_PRIVATE, chiplib_forced_config_link_mask, 504 NV_SWITCH_REGKEY_CHIPLIB_FORCED_LINK_CONFIG_MASK, 505 0); 506 507 NVSWITCH_INIT_REGKEY(_PRIVATE, chiplib_forced_config_link_mask2, 508 NV_SWITCH_REGKEY_CHIPLIB_FORCED_LINK_CONFIG_MASK2, 509 0); 510 511 NVSWITCH_INIT_REGKEY(_PRIVATE, soe_dma_self_test, 512 NV_SWITCH_REGKEY_SOE_DMA_SELFTEST, 513 NV_SWITCH_REGKEY_SOE_DMA_SELFTEST_ENABLE); 514 515 NVSWITCH_INIT_REGKEY(_PRIVATE, soe_disable, 516 NV_SWITCH_REGKEY_SOE_DISABLE, 517 NV_SWITCH_REGKEY_SOE_DISABLE_NO); 518 519 NVSWITCH_INIT_REGKEY(_PRIVATE, latency_counter, 520 NV_SWITCH_REGKEY_LATENCY_COUNTER_LOGGING, 521 NV_SWITCH_REGKEY_LATENCY_COUNTER_LOGGING_ENABLE); 522 523 NVSWITCH_INIT_REGKEY(_PRIVATE, nvlink_speed_control, 524 NV_SWITCH_REGKEY_SPEED_CONTROL, 525 NV_SWITCH_REGKEY_SPEED_CONTROL_SPEED_DEFAULT); 526 527 NVSWITCH_INIT_REGKEY(_PRIVATE, inforom_bbx_periodic_flush, 528 NV_SWITCH_REGKEY_INFOROM_BBX_ENABLE_PERIODIC_FLUSHING, 529 NV_SWITCH_REGKEY_INFOROM_BBX_ENABLE_PERIODIC_FLUSHING_DISABLE); 530 531 NVSWITCH_INIT_REGKEY(_PRIVATE, inforom_bbx_write_periodicity, 532 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_PERIODICITY, 533 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_PERIODICITY_DEFAULT); 534 535 NVSWITCH_INIT_REGKEY(_PRIVATE, inforom_bbx_write_min_duration, 536 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_MIN_DURATION, 537 NV_SWITCH_REGKEY_INFOROM_BBX_WRITE_MIN_DURATION_DEFAULT); 538 539 NVSWITCH_INIT_REGKEY(_PRIVATE, minion_disable, 540 NV_SWITCH_REGKEY_MINION_DISABLE, 541 NV_SWITCH_REGKEY_MINION_DISABLE_NO); 542 543 NVSWITCH_INIT_REGKEY(_PRIVATE, set_ucode_target, 544 NV_SWITCH_REGKEY_MINION_SET_UCODE_TARGET, 545 NV_SWITCH_REGKEY_MINION_SET_UCODE_TARGET_DEFAULT); 546 547 NVSWITCH_INIT_REGKEY(_PRIVATE, set_simmode, 548 NV_SWITCH_REGKEY_MINION_SET_SIMMODE, 549 NV_SWITCH_REGKEY_MINION_SET_SIMMODE_DEFAULT); 550 551 NVSWITCH_INIT_REGKEY(_PRIVATE, set_smf_settings, 552 NV_SWITCH_REGKEY_MINION_SET_SMF_SETTINGS, 553 NV_SWITCH_REGKEY_MINION_SET_SMF_SETTINGS_DEFAULT); 554 555 NVSWITCH_INIT_REGKEY(_PRIVATE, select_uphy_tables, 556 NV_SWITCH_REGKEY_MINION_SELECT_UPHY_TABLES, 557 NV_SWITCH_REGKEY_MINION_SELECT_UPHY_TABLES_DEFAULT); 558 559 NVSWITCH_INIT_REGKEY(_PRIVATE, link_training_mode, 560 NV_SWITCH_REGKEY_LINK_TRAINING_SELECT, 561 NV_SWITCH_REGKEY_LINK_TRAINING_SELECT_DEFAULT); 562 563 NVSWITCH_INIT_REGKEY(_PRIVATE, i2c_access_control, 564 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL, 565 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_DEFAULT); 566 567 NVSWITCH_INIT_REGKEY(_PRIVATE, force_kernel_i2c, 568 NV_SWITCH_REGKEY_FORCE_KERNEL_I2C, 569 NV_SWITCH_REGKEY_FORCE_KERNEL_I2C_DEFAULT); 570 571 NVSWITCH_INIT_REGKEY(_PRIVATE, link_recal_settings, 572 NV_SWITCH_REGKEY_LINK_RECAL_SETTINGS, 573 NV_SWITCH_REGKEY_LINK_RECAL_SETTINGS_NOP); 574 575 NVSWITCH_INIT_REGKEY(_PRIVATE, lp_threshold, 576 NV_SWITCH_REGKEY_SET_LP_THRESHOLD, 577 NV_SWITCH_REGKEY_SET_LP_THRESHOLD_DEFAULT); 578 579 NVSWITCH_INIT_REGKEY(_PUBLIC, minion_intr, 580 NV_SWITCH_REGKEY_MINION_INTERRUPTS, 581 NV_SWITCH_REGKEY_MINION_INTERRUPTS_DEFAULT); 582 583 NVSWITCH_INIT_REGKEY(_PRIVATE, block_code_mode, 584 NV_SWITCH_REGKEY_BLOCK_CODE_MODE, 585 NV_SWITCH_REGKEY_BLOCK_CODE_MODE_DEFAULT); 586 587 NVSWITCH_INIT_REGKEY(_PRIVATE, reference_clock_mode, 588 NV_SWITCH_REGKEY_REFERENCE_CLOCK_MODE, 589 NV_SWITCH_REGKEY_REFERENCE_CLOCK_MODE_DEFAULT); 590 591 NVSWITCH_INIT_REGKEY(_PRIVATE, debug_level, 592 NV_SWITCH_REGKEY_DBG_LEVEL, 593 NV_SWITCH_REGKEY_DBG_LEVEL_DEFAULT); 594 } 595 596 ct_assert(NVSWITCH_DBG_LEVEL_MMIO == NV_SWITCH_REGKEY_DBG_LEVEL_MMIO); 597 ct_assert(NVSWITCH_DBG_LEVEL_NOISY == NV_SWITCH_REGKEY_DBG_LEVEL_NOISY); 598 ct_assert(NVSWITCH_DBG_LEVEL_SETUP == NV_SWITCH_REGKEY_DBG_LEVEL_SETUP); 599 ct_assert(NVSWITCH_DBG_LEVEL_INFO == NV_SWITCH_REGKEY_DBG_LEVEL_INFO); 600 ct_assert(NVSWITCH_DBG_LEVEL_WARN == NV_SWITCH_REGKEY_DBG_LEVEL_WARN); 601 ct_assert(NVSWITCH_DBG_LEVEL_ERROR == NV_SWITCH_REGKEY_DBG_LEVEL_ERROR); 602 603 NvU64 604 nvswitch_lib_deferred_task_dispatcher 605 ( 606 nvswitch_device *device 607 ) 608 { 609 NvU64 time_nsec; 610 NvU64 time_next_nsec = nvswitch_os_get_platform_time() + 100*NVSWITCH_INTERVAL_1MSEC_IN_NS; 611 NVSWITCH_TASK_TYPE *task; 612 NVSWITCH_TASK_TYPE *prev_task; 613 614 if (!NVSWITCH_IS_DEVICE_VALID(device)) 615 { 616 return NV_U64_MAX; 617 } 618 619 prev_task = NULL; 620 task = device->tasks; 621 622 // Walk the task list, executing those whose next execution interval is at hand 623 while (task) 624 { 625 // Get current time (nsec) for scheduling 626 time_nsec = nvswitch_os_get_platform_time(); 627 628 if (time_nsec >= task->last_run_nsec + task->period_nsec) 629 { 630 // 631 // The task has never been run or it is time to run 632 // Mark its last run time 633 // 634 task->last_run_nsec = time_nsec; 635 // Run the task 636 if (NVSWITCH_IS_DEVICE_INITIALIZED(device) || 637 (task->flags & NVSWITCH_TASK_TYPE_FLAGS_RUN_EVEN_IF_DEVICE_NOT_INITIALIZED)) 638 { 639 if(task->flags & NVSWITCH_TASK_TYPE_FLAGS_VOID_PTR_ARGS) 640 (*task->task_fn_vdptr)(device, task->task_args); // run task with provided args 641 else 642 (*task->task_fn_devptr)(device); 643 } 644 } 645 646 // Determine its next run time 647 time_next_nsec = NV_MIN(task->last_run_nsec + task->period_nsec, time_next_nsec); 648 649 // Advance pointer. If run once flag is set and task ran, remove task from list. 650 if((task->flags & NVSWITCH_TASK_TYPE_FLAGS_RUN_ONCE) && 651 (task->last_run_nsec == time_nsec)) 652 { 653 prev_task = task->prev; 654 655 // Removing from list head 656 if (prev_task == NULL) 657 { 658 device->tasks = task->next; 659 if (device->tasks != NULL) 660 { 661 device->tasks->prev = NULL; 662 } 663 nvswitch_os_free(task); 664 task = device->tasks; 665 } 666 else 667 { 668 prev_task->next = task->next; 669 if (prev_task->next != NULL) 670 { 671 prev_task->next->prev = prev_task; 672 } 673 nvswitch_os_free(task); 674 task = prev_task->next; 675 } 676 } 677 else 678 { 679 task = task->next; 680 } 681 } 682 683 time_nsec = nvswitch_os_get_platform_time(); 684 685 // Return to the OS layer how long to wait before calling again 686 return(time_next_nsec >= time_nsec ? time_next_nsec - time_nsec : 0); 687 } 688 689 static NvlStatus 690 _nvswitch_setup_hal 691 ( 692 nvswitch_device *device, 693 NvU32 pci_device_id 694 ) 695 { 696 if (nvswitch_is_lr10_device_id(pci_device_id)) 697 { 698 nvswitch_setup_hal_lr10(device); 699 return NVL_SUCCESS; 700 } 701 if (nvswitch_is_ls10_device_id(pci_device_id)) 702 { 703 nvswitch_setup_hal_ls10(device); 704 return NVL_SUCCESS; 705 } 706 NVSWITCH_PRINT(device, ERROR, 707 "NVSwitch HAL setup failed - Unrecognized PCI Device ID\n"); 708 return -NVL_ERR_NOT_SUPPORTED; 709 } 710 711 NvlStatus 712 nvswitch_lib_check_api_version 713 ( 714 const char *user_version, 715 char *kernel_version, 716 NvU32 length 717 ) 718 { 719 const NvLength VERSION_LENGTH = nvswitch_os_strlen(NV_VERSION_STRING); 720 721 if (kernel_version == NULL || user_version == NULL) 722 { 723 return -NVL_BAD_ARGS; 724 } 725 726 if (length < VERSION_LENGTH) 727 { 728 return -NVL_NO_MEM; 729 } 730 731 nvswitch_os_memset(kernel_version, 0x0, length); 732 nvswitch_os_strncpy(kernel_version, NV_VERSION_STRING, VERSION_LENGTH); 733 734 kernel_version[length - 1] = '\0'; 735 736 if (nvswitch_os_strncmp(user_version, kernel_version, VERSION_LENGTH)) 737 { 738 return -NVL_ERR_NOT_SUPPORTED; 739 } 740 741 return NVL_SUCCESS; 742 } 743 744 NvBool 745 nvswitch_is_inforom_supported 746 ( 747 nvswitch_device *device 748 ) 749 { 750 return device->hal.nvswitch_is_inforom_supported(device); 751 } 752 753 NvBool 754 nvswitch_is_spi_supported 755 ( 756 nvswitch_device *device 757 ) 758 { 759 return device->hal.nvswitch_is_spi_supported(device); 760 } 761 762 NvBool 763 nvswitch_is_bios_supported 764 ( 765 nvswitch_device *device 766 ) 767 { 768 return device->hal.nvswitch_is_bios_supported(device); 769 } 770 771 NvBool 772 nvswitch_is_smbpbi_supported 773 ( 774 nvswitch_device *device 775 ) 776 { 777 return device->hal.nvswitch_is_smbpbi_supported(device); 778 } 779 780 NvBool 781 nvswitch_is_soe_supported 782 ( 783 nvswitch_device *device 784 ) 785 { 786 return device->hal.nvswitch_is_soe_supported(device); 787 } 788 789 790 NvlStatus 791 nvswitch_init_soe 792 ( 793 nvswitch_device *device 794 ) 795 { 796 return device->hal.nvswitch_init_soe(device); 797 } 798 799 void 800 nvswitch_soe_init_l2_state 801 ( 802 nvswitch_device *device 803 ) 804 { 805 device->hal.nvswitch_soe_init_l2_state(device); 806 } 807 808 void 809 nvswitch_fsp_update_cmdq_head_tail 810 ( 811 nvswitch_device *device, 812 NvU32 queueHead, 813 NvU32 queueTail 814 ) 815 { 816 return device->hal.nvswitch_fsp_update_cmdq_head_tail(device, queueHead, queueTail); 817 } 818 819 void 820 nvswitch_fsp_get_cmdq_head_tail 821 ( 822 nvswitch_device *device, 823 NvU32 *pQueueHead, 824 NvU32 *pQueueTail 825 ) 826 { 827 return device->hal.nvswitch_fsp_get_cmdq_head_tail(device, pQueueHead, pQueueTail); 828 } 829 830 void 831 nvswitch_fsp_update_msgq_head_tail 832 ( 833 nvswitch_device *device, 834 NvU32 msgqHead, 835 NvU32 msgqTail 836 ) 837 { 838 return device->hal.nvswitch_fsp_update_msgq_head_tail(device, msgqHead, msgqTail); 839 } 840 841 void 842 nvswitch_fsp_get_msgq_head_tail 843 ( 844 nvswitch_device *device, 845 NvU32 *pMsgqHead, 846 NvU32 *pMsgqTail 847 ) 848 { 849 return device->hal.nvswitch_fsp_get_msgq_head_tail(device, pMsgqHead, pMsgqTail); 850 } 851 852 NvU32 853 nvswitch_fsp_get_channel_size 854 ( 855 nvswitch_device *device 856 ) 857 { 858 return device->hal.nvswitch_fsp_get_channel_size(device); 859 } 860 861 NvU8 862 nvswitch_fsp_nvdm_to_seid 863 ( 864 nvswitch_device *device, 865 NvU8 nvdmType 866 ) 867 { 868 return device->hal.nvswitch_fsp_nvdm_to_seid(device, nvdmType); 869 } 870 871 NvU32 872 nvswitch_fsp_create_mctp_header 873 ( 874 nvswitch_device *device, 875 NvU8 som, 876 NvU8 eom, 877 NvU8 seid, 878 NvU8 seq 879 ) 880 { 881 return device->hal.nvswitch_fsp_create_mctp_header(device, som, eom, seid, seq); 882 } 883 884 NvU32 885 nvswitch_fsp_create_nvdm_header 886 ( 887 nvswitch_device *device, 888 NvU32 nvdmType 889 ) 890 { 891 return device->hal.nvswitch_fsp_create_nvdm_header(device, nvdmType); 892 } 893 894 NvlStatus 895 nvswitch_fsp_get_packet_info 896 ( 897 nvswitch_device *device, 898 NvU8 *pBuffer, 899 NvU32 size, 900 NvU8 *pPacketState, 901 NvU8 *pTag 902 ) 903 { 904 return device->hal.nvswitch_fsp_get_packet_info(device, pBuffer, size, pPacketState, pTag); 905 } 906 907 NvlStatus 908 nvswitch_fsp_validate_mctp_payload_header 909 ( 910 nvswitch_device *device, 911 NvU8 *pBuffer, 912 NvU32 size 913 ) 914 { 915 return device->hal.nvswitch_fsp_validate_mctp_payload_header(device, pBuffer, size); 916 } 917 918 NvlStatus 919 nvswitch_fsp_process_nvdm_msg 920 ( 921 nvswitch_device *device, 922 NvU8 *pBuffer, 923 NvU32 size 924 ) 925 { 926 return device->hal.nvswitch_fsp_process_nvdm_msg(device, pBuffer, size); 927 } 928 929 NvlStatus 930 nvswitch_fsp_process_cmd_response 931 ( 932 nvswitch_device *device, 933 NvU8 *pBuffer, 934 NvU32 size 935 ) 936 { 937 return device->hal.nvswitch_fsp_process_cmd_response(device, pBuffer, size); 938 } 939 940 NvlStatus 941 nvswitch_fsp_config_ememc 942 ( 943 nvswitch_device *device, 944 NvU32 offset, 945 NvBool bAincw, 946 NvBool bAincr 947 ) 948 { 949 return device->hal.nvswitch_fsp_config_ememc(device, offset, bAincw, bAincr); 950 } 951 952 NvlStatus 953 nvswitch_fsp_write_to_emem 954 ( 955 nvswitch_device *device, 956 NvU8 *pBuffer, 957 NvU32 size 958 ) 959 { 960 return device->hal.nvswitch_fsp_write_to_emem(device, pBuffer, size); 961 } 962 963 NvlStatus 964 nvswitch_fsp_read_from_emem 965 ( 966 nvswitch_device *device, 967 NvU8 *pBuffer, 968 NvU32 size 969 ) 970 { 971 return device->hal.nvswitch_fsp_read_from_emem(device, pBuffer, size); 972 } 973 974 NvlStatus 975 nvswitch_fsp_error_code_to_nvlstatus_map 976 ( 977 nvswitch_device *device, 978 NvU32 errorCode 979 ) 980 { 981 return device->hal.nvswitch_fsp_error_code_to_nvlstatus_map(device, errorCode); 982 } 983 984 static NvlStatus 985 _nvswitch_ctrl_fsprpc_get_caps 986 ( 987 nvswitch_device *device, 988 NVSWITCH_FSPRPC_GET_CAPS_PARAMS *params 989 ) 990 { 991 return device->hal.nvswitch_fsprpc_get_caps(device, params); 992 } 993 994 static NvlStatus 995 _nvswitch_construct_soe 996 ( 997 nvswitch_device *device 998 ) 999 { 1000 FLCNABLE *pSoe = NULL; 1001 NvlStatus retval; 1002 1003 device->pSoe = pSoe = (PFLCNABLE)soeAllocNew(); 1004 if (pSoe == NULL) 1005 { 1006 NVSWITCH_PRINT(device, ERROR, "SOE allocation failed.\n"); 1007 return -NVL_NO_MEM; 1008 } 1009 1010 retval = soeInit(device, (PSOE)pSoe, device->nvlink_device->pciInfo.pciDeviceId); 1011 if (retval != NVL_SUCCESS) 1012 { 1013 NVSWITCH_PRINT(device, ERROR, "SOE init failed.\n"); 1014 goto soe_init_failed; 1015 } 1016 1017 if (flcnableConstruct_HAL(device, pSoe) != NV_OK) 1018 { 1019 NVSWITCH_PRINT(device, ERROR, "FALCON construct failed.\n"); 1020 retval = -NVL_ERR_INVALID_STATE; 1021 goto flcn_construct_failed; 1022 } 1023 1024 return NVL_SUCCESS; 1025 1026 flcn_construct_failed: 1027 soeDestroy(device, (PSOE)pSoe); 1028 1029 soe_init_failed: 1030 nvswitch_os_free(pSoe); 1031 device->pSoe = NULL; 1032 1033 return retval; 1034 } 1035 1036 static void 1037 _nvswitch_destruct_soe 1038 ( 1039 nvswitch_device *device 1040 ) 1041 { 1042 FLCNABLE *pSoe = device->pSoe; 1043 1044 if (pSoe == NULL) 1045 { 1046 return; 1047 } 1048 1049 flcnableDestruct_HAL(device, pSoe); 1050 soeDestroy(device, (PSOE)pSoe); 1051 1052 nvswitch_os_free(pSoe); 1053 device->pSoe = NULL; 1054 } 1055 1056 static NvlStatus 1057 _nvswitch_construct_cci 1058 ( 1059 nvswitch_device *device 1060 ) 1061 { 1062 CCI *pCci = NULL; 1063 NvlStatus retval; 1064 1065 device->pCci = pCci = cciAllocNew(); 1066 if (pCci == NULL) 1067 { 1068 NVSWITCH_PRINT(device, ERROR, "CCI allocation failed.\n"); 1069 return -NVL_NO_MEM; 1070 } 1071 1072 retval = cciInit(device, pCci, device->nvlink_device->pciInfo.pciDeviceId); 1073 if (retval != NVL_SUCCESS) 1074 { 1075 NVSWITCH_PRINT(device, ERROR, "CCI init failed.\n"); 1076 goto cci_init_failed; 1077 } 1078 1079 return NVL_SUCCESS; 1080 1081 cci_init_failed: 1082 nvswitch_os_free(pCci); 1083 device->pCci = NULL; 1084 1085 return retval; 1086 } 1087 1088 static void 1089 _nvswitch_destruct_cci 1090 ( 1091 nvswitch_device *device 1092 ) 1093 { 1094 CCI *pCci = device->pCci; 1095 1096 if (pCci == NULL) 1097 { 1098 return; 1099 } 1100 1101 cciDestroy(device, pCci); 1102 1103 nvswitch_os_free(pCci); 1104 device->pCci = NULL; 1105 } 1106 1107 static void 1108 _nvswitch_update_link_state_led 1109 ( 1110 nvswitch_device *device 1111 ) 1112 { 1113 device->hal.nvswitch_update_link_state_led(device); 1114 } 1115 1116 static void 1117 _nvswitch_led_shutdown 1118 ( 1119 nvswitch_device *device 1120 ) 1121 { 1122 device->hal.nvswitch_led_shutdown(device); 1123 } 1124 1125 static NvlStatus 1126 _nvswitch_initialize_device_state 1127 ( 1128 nvswitch_device *device 1129 ) 1130 { 1131 return device->hal.nvswitch_initialize_device_state(device); 1132 } 1133 1134 static NvlStatus 1135 _nvswitch_post_init_device_setup 1136 ( 1137 nvswitch_device *device 1138 ) 1139 { 1140 return device->hal.nvswitch_post_init_device_setup(device); 1141 } 1142 1143 static NvlStatus 1144 _nvswitch_setup_system_registers 1145 ( 1146 nvswitch_device *device 1147 ) 1148 { 1149 return device->hal.nvswitch_setup_system_registers(device); 1150 } 1151 1152 static void 1153 _nvswitch_post_init_blacklist_device_setup 1154 ( 1155 nvswitch_device *device 1156 ) 1157 { 1158 device->hal.nvswitch_post_init_blacklist_device_setup(device); 1159 } 1160 1161 static void 1162 _nvswitch_set_dma_mask 1163 ( 1164 nvswitch_device *device 1165 ) 1166 { 1167 NvU32 hw_dma_width, retval; 1168 1169 hw_dma_width = device->hal.nvswitch_get_device_dma_width(device); 1170 1171 if (hw_dma_width == 0) 1172 { 1173 NVSWITCH_PRINT(device, INFO, "DMA is not supported on this device\n"); 1174 return; 1175 } 1176 1177 retval = nvswitch_os_set_dma_mask(device->os_handle, hw_dma_width); 1178 if (retval == NVL_SUCCESS) 1179 { 1180 device->dma_addr_width = hw_dma_width; 1181 return; 1182 } 1183 1184 NVSWITCH_PRINT(device, SETUP, 1185 "%s: Failed to set DMA mask, trying 32-bit fallback : %d\n", 1186 __FUNCTION__, retval); 1187 1188 retval = nvswitch_os_set_dma_mask(device->os_handle, 32); 1189 if (retval == NVL_SUCCESS) 1190 { 1191 device->dma_addr_width = 32; 1192 return; 1193 } 1194 1195 // failure is not fatal, the driver will just restrict DMA functionality 1196 NVSWITCH_PRINT(device, ERROR, "Failed to set DMA mask : %d\n", retval); 1197 } 1198 1199 NvlStatus 1200 nvswitch_deassert_link_reset 1201 ( 1202 nvswitch_device *device, 1203 nvlink_link *link 1204 ) 1205 { 1206 return device->hal.nvswitch_deassert_link_reset(device, link); 1207 } 1208 1209 NvU32 1210 nvswitch_get_sublink_width 1211 ( 1212 nvswitch_device *device, 1213 NvU32 linkNumber 1214 ) 1215 { 1216 return device->hal.nvswitch_get_sublink_width(device, linkNumber); 1217 } 1218 1219 static void 1220 _nvswitch_unregister_links 1221 ( 1222 nvswitch_device *device 1223 ) 1224 { 1225 nvlink_link *link = NULL; 1226 NvU32 link_num; 1227 NvBool is_blacklisted; 1228 1229 1230 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 1231 return; 1232 1233 device->nvlink_device->initialized = 0; 1234 is_blacklisted = (device->device_fabric_state == NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED); 1235 1236 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 1237 { 1238 if (nvlink_lib_get_link(device->nvlink_device, link_num, &link) == NVL_SUCCESS) 1239 { 1240 nvlink_lib_unregister_link(link); 1241 nvswitch_destroy_link(link); 1242 } 1243 } 1244 1245 if (!is_blacklisted) 1246 nvswitch_inforom_nvlink_flush(device); 1247 } 1248 1249 NvlStatus NV_API_CALL 1250 nvswitch_lib_read_fabric_state 1251 ( 1252 nvswitch_device *device, 1253 NVSWITCH_DEVICE_FABRIC_STATE *device_fabric_state, 1254 NVSWITCH_DEVICE_BLACKLIST_REASON *device_blacklist_reason, 1255 NVSWITCH_DRIVER_FABRIC_STATE *driver_fabric_state 1256 ) 1257 { 1258 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1259 return -NVL_BAD_ARGS; 1260 1261 if (device_fabric_state != NULL) 1262 *device_fabric_state = device->device_fabric_state; 1263 1264 if (device_blacklist_reason != NULL) 1265 *device_blacklist_reason = device->device_blacklist_reason; 1266 1267 if (driver_fabric_state != NULL) 1268 *driver_fabric_state = device->driver_fabric_state; 1269 1270 return NVL_SUCCESS; 1271 } 1272 1273 static NvlStatus 1274 nvswitch_lib_blacklist_device 1275 ( 1276 nvswitch_device *device, 1277 NVSWITCH_DEVICE_BLACKLIST_REASON device_blacklist_reason 1278 ) 1279 { 1280 NvlStatus status; 1281 1282 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1283 { 1284 return -NVL_BAD_ARGS; 1285 } 1286 1287 if (device->device_fabric_state == NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED) 1288 { 1289 NVSWITCH_PRINT(device, WARN, "Device is already blacklisted\n"); 1290 return -NVL_ERR_NOT_SUPPORTED; 1291 } 1292 1293 device->device_fabric_state = NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED; 1294 device->device_blacklist_reason = device_blacklist_reason; 1295 1296 status = device->hal.nvswitch_write_fabric_state(device); 1297 if (status != NVL_SUCCESS) 1298 NVSWITCH_PRINT(device, INFO, "Cannot send fabric state to SOE\n"); 1299 1300 return NVL_SUCCESS; 1301 } 1302 1303 static NvlStatus 1304 nvswitch_ctrl_blacklist_device( 1305 nvswitch_device *device, 1306 NVSWITCH_BLACKLIST_DEVICE_PARAMS *p 1307 ) 1308 { 1309 NvlStatus status; 1310 1311 status = nvswitch_lib_blacklist_device(device, p->deviceReason); 1312 if (status != NVL_SUCCESS) 1313 return status; 1314 1315 nvswitch_lib_disable_interrupts(device); 1316 1317 // Unregister links from NVLinkCoreLib, so that link training is not 1318 // attempted 1319 _nvswitch_unregister_links(device); 1320 1321 // Keep device registered for HAL access and Fabric State updates 1322 1323 return NVL_SUCCESS; 1324 } 1325 1326 static NvlStatus 1327 nvswitch_ctrl_set_fm_driver_state( 1328 nvswitch_device *device, 1329 NVSWITCH_SET_FM_DRIVER_STATE_PARAMS *p 1330 ) 1331 { 1332 NvU32 prev_fm_status; 1333 1334 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1335 { 1336 return -NVL_BAD_ARGS; 1337 } 1338 1339 prev_fm_status = device->driver_fabric_state; 1340 device->driver_fabric_state = p->driverState; 1341 device->fabric_state_timestamp = nvswitch_os_get_platform_time(); 1342 1343 if (prev_fm_status != p->driverState) 1344 { 1345 if (nvswitch_lib_notify_client_events(device, 1346 NVSWITCH_DEVICE_EVENT_FABRIC_STATE) != NVL_SUCCESS) 1347 { 1348 NVSWITCH_PRINT(device, ERROR, "%s: Failed to notify event\n", 1349 __FUNCTION__); 1350 } 1351 } 1352 1353 return NVL_SUCCESS; 1354 } 1355 1356 static NvlStatus 1357 nvswitch_ctrl_set_device_fabric_state( 1358 nvswitch_device *device, 1359 NVSWITCH_SET_DEVICE_FABRIC_STATE_PARAMS *p 1360 ) 1361 { 1362 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1363 { 1364 return -NVL_BAD_ARGS; 1365 } 1366 1367 if (device->device_fabric_state == NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED) 1368 return -NVL_ERR_NOT_SUPPORTED; 1369 1370 device->device_fabric_state = p->deviceState; 1371 device->fabric_state_timestamp = nvswitch_os_get_platform_time(); 1372 1373 // If FM had exceeded timeout, reset the status to not timed-out 1374 if (device->driver_fabric_state == NVSWITCH_DRIVER_FABRIC_STATE_MANAGER_TIMEOUT) 1375 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED; 1376 1377 return NVL_SUCCESS; 1378 } 1379 1380 static NvlStatus 1381 nvswitch_ctrl_set_fm_timeout( 1382 nvswitch_device *device, 1383 NVSWITCH_SET_FM_HEARTBEAT_TIMEOUT_PARAMS *p 1384 ) 1385 { 1386 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1387 { 1388 return -NVL_BAD_ARGS; 1389 } 1390 1391 device->fm_timeout = p->fmTimeout; 1392 1393 return NVL_SUCCESS; 1394 } 1395 1396 static NvlStatus 1397 _nvswitch_ctrl_register_events( 1398 nvswitch_device *device, 1399 NVSWITCH_REGISTER_EVENTS_PARAMS *p, 1400 void *osPrivate 1401 ) 1402 { 1403 NvlStatus status = NVL_SUCCESS; 1404 NvU32 i; 1405 NvBool many_events, os_descriptor; 1406 void *osDescriptor = osPrivate; 1407 1408 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1409 { 1410 return -NVL_BAD_ARGS; 1411 } 1412 1413 status = nvswitch_os_get_supported_register_events_params(&many_events, 1414 &os_descriptor); 1415 if (status != NVL_SUCCESS) 1416 { 1417 return status; 1418 } 1419 1420 if ((!many_events && (p->numEvents > 1)) || 1421 (p->numEvents == 0)) 1422 { 1423 return -NVL_BAD_ARGS; 1424 } 1425 1426 if (os_descriptor) 1427 { 1428 osDescriptor = (void *) p->osDescriptor; 1429 } 1430 1431 for (i = 0; i < p->numEvents; i++) 1432 { 1433 status = nvswitch_lib_add_client_event(device, osDescriptor, p->eventIds[i]); 1434 if (status != NVL_SUCCESS) 1435 { 1436 NVSWITCH_PRINT(device, ERROR, "%s: Failed to add client event.\n", __FUNCTION__); 1437 return status; 1438 } 1439 } 1440 1441 return NVL_SUCCESS; 1442 } 1443 1444 static NvlStatus 1445 _nvswitch_ctrl_unregister_events( 1446 nvswitch_device *device, 1447 NVSWITCH_UNREGISTER_EVENTS_PARAMS *p, 1448 void *osPrivate 1449 ) 1450 { 1451 NvlStatus status = NVL_SUCCESS; 1452 NvBool many_events, os_descriptor; 1453 void *osDescriptor = osPrivate; 1454 1455 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1456 { 1457 return -NVL_BAD_ARGS; 1458 } 1459 1460 status = nvswitch_os_get_supported_register_events_params(&many_events, 1461 &os_descriptor); 1462 if (status != NVL_SUCCESS) 1463 { 1464 return status; 1465 } 1466 1467 if (os_descriptor) 1468 { 1469 osDescriptor = (void *) p->osDescriptor; 1470 } 1471 1472 status = nvswitch_lib_remove_client_events(device, osDescriptor); 1473 if (status != NVL_SUCCESS) 1474 { 1475 NVSWITCH_PRINT(device, ERROR, "%s: Failed to remove client event.\n", __FUNCTION__); 1476 return status; 1477 } 1478 1479 return NVL_SUCCESS; 1480 } 1481 1482 /* 1483 * @Brief : Sends NACK or drops given inband msg based on message type 1484 * 1485 * @Description : 1486 * 1487 * @param[in] device NvSwitch device to contain this link 1488 * @param[in] linkId Link ID 1489 * @param[in] msghdr Header to the message 1490 * 1491 */ 1492 static void 1493 nvswitch_send_nack_or_drop 1494 ( 1495 nvswitch_device *device, 1496 NvU32 linkId, 1497 nvlink_inband_msg_header_t *msghdr 1498 ) 1499 { 1500 switch(msghdr->type) 1501 { 1502 case NVLINK_INBAND_MSG_TYPE_MC_TEAM_SETUP_REQ: 1503 device->hal.nvswitch_send_inband_nack(device, (NvU32 *)msghdr, linkId); 1504 NVSWITCH_PRINT(device, ERROR, 1505 "Sending NACK for message (type 0x%x)\n", msghdr->type); 1506 return; 1507 default: 1508 // TODO: Add SXid in future if needed. 1509 NVSWITCH_PRINT(device, ERROR, 1510 "Dropping message (type 0x%x)\n", msghdr->type); 1511 return; 1512 } 1513 } 1514 1515 /* 1516 * @Brief : Deletes all the entries in persistent or non-persistent lists. 1517 * Send nacks if requested. 1518 * 1519 * @Description : 1520 * 1521 * @param[in] device NVSwitch device to contain this link 1522 * @param[in] linkId Link number 1523 * @param[in] bSendNack Send nacks if true 1524 * @param[in] bNonPersistentOnly Clear only non-persistent list 1525 */ 1526 static void 1527 _nvswitch_inband_clear_lists 1528 ( 1529 nvswitch_device *device, 1530 NvU32 linkId, 1531 NvBool bSendNack, 1532 NvBool bNonPersistentOnly 1533 ) 1534 { 1535 nvswitch_inband_data_list *curr = NULL; 1536 nvswitch_inband_data_list *next = NULL; 1537 nvlink_inband_msg_header_t *msghdr = NULL; 1538 1539 nvListForEachEntry_safe(curr, next, 1540 &device->link[linkId].inbandData.nonpersistent_list, entry) 1541 { 1542 if (bSendNack) 1543 { 1544 msghdr = (nvlink_inband_msg_header_t *)curr->data; 1545 nvswitch_send_nack_or_drop(device, linkId, msghdr); 1546 } 1547 1548 nvListDel(&curr->entry); 1549 nvswitch_os_free(curr); 1550 } 1551 1552 if (bNonPersistentOnly) 1553 return; 1554 1555 nvListForEachEntry_safe(curr, next, 1556 &device->link[linkId].inbandData.persistent_list, entry) 1557 { 1558 if (bSendNack) 1559 { 1560 msghdr = (nvlink_inband_msg_header_t *)curr->data; 1561 nvswitch_send_nack_or_drop(device, linkId, msghdr); 1562 } 1563 1564 nvListDel(&curr->entry); 1565 nvswitch_os_free(curr); 1566 } 1567 } 1568 1569 static void 1570 nvswitch_fabric_state_heartbeat( 1571 nvswitch_device *device 1572 ) 1573 { 1574 NvU64 age; 1575 NvU32 linkId; 1576 1577 if (!NVSWITCH_IS_DEVICE_VALID(device)) 1578 return; 1579 1580 age = nvswitch_os_get_platform_time() - device->fabric_state_timestamp; 1581 1582 // Check to see if we have exceeded the FM timeout 1583 if (device->driver_fabric_state == NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED && 1584 age > (NvU64)device->fm_timeout * 1000ULL * 1000ULL) 1585 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_MANAGER_TIMEOUT; 1586 1587 // 1588 // If FM is not running, clear pending non-persistent messages. Persistent 1589 // messages can be processed by the FM when it restarts. 1590 // 1591 if (device->driver_fabric_state != NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED) 1592 { 1593 for (linkId = 0; linkId < nvswitch_get_num_links(device); linkId++) 1594 _nvswitch_inband_clear_lists(device, linkId, 1595 NV_TRUE /* Nack */, 1596 NV_TRUE /* Non-persistent only */); 1597 } 1598 1599 (void)device->hal.nvswitch_write_fabric_state(device); 1600 } 1601 1602 static NvlStatus 1603 _nvswitch_ctrl_set_training_error_info 1604 ( 1605 nvswitch_device *device, 1606 NVSWITCH_SET_TRAINING_ERROR_INFO_PARAMS *p 1607 ) 1608 { 1609 return device->hal.nvswitch_set_training_error_info(device, p); 1610 } 1611 1612 static NvlStatus 1613 _nvswitch_ctrl_get_fatal_error_scope 1614 ( 1615 nvswitch_device *device, 1616 NVSWITCH_GET_FATAL_ERROR_SCOPE_PARAMS *pParams 1617 ) 1618 { 1619 return device->hal.nvswitch_ctrl_get_fatal_error_scope(device, pParams); 1620 } 1621 1622 static NvlStatus 1623 _nvswitch_ctrl_therm_get_temperature_limit 1624 ( 1625 nvswitch_device *device, 1626 NVSWITCH_CTRL_GET_TEMPERATURE_LIMIT_PARAMS *pParams 1627 ) 1628 { 1629 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1630 { 1631 return -NVL_BAD_ARGS; 1632 } 1633 1634 return device->hal.nvswitch_ctrl_therm_get_temperature_limit(device, pParams); 1635 } 1636 1637 // 1638 // Construct an port event log 1639 // 1640 // If port_event_log_size > 0 a circular buffer is created to record port events 1641 // 1642 NvlStatus 1643 _nvswitch_construct_port_event_log 1644 ( 1645 NVSWITCH_PORT_EVENT_LOG_TYPE *port_events, 1646 NvU32 port_event_log_size, 1647 NvBool overwritable 1648 ) 1649 { 1650 NvlStatus retval = NVL_SUCCESS; 1651 1652 NVSWITCH_ASSERT(port_events != NULL); 1653 1654 port_events->port_event_start = 0; 1655 port_events->port_event_count = 0; 1656 port_events->port_event_total = 0; 1657 port_events->port_event_log_size = 0; 1658 port_events->port_event_log = NULL; 1659 port_events->overwritable = overwritable; 1660 port_events->bOverflow = NV_FALSE; 1661 1662 if (port_event_log_size > 0) 1663 { 1664 port_events->port_event_log = nvswitch_os_malloc(port_event_log_size * sizeof(NVSWITCH_PORT_EVENT_TYPE)); 1665 } 1666 1667 if (port_events->port_event_log != NULL) 1668 { 1669 port_events->port_event_log_size = port_event_log_size; 1670 nvswitch_os_memset(port_events->port_event_log, 0, port_events->port_event_log_size * sizeof(NVSWITCH_PORT_EVENT_TYPE)); 1671 } 1672 1673 if (port_event_log_size != port_events->port_event_log_size) 1674 { 1675 retval = -NVL_NO_MEM; 1676 } 1677 1678 return retval; 1679 } 1680 1681 // 1682 // Destroy an error log 1683 // 1684 void 1685 _nvswitch_destroy_port_event_log 1686 ( 1687 nvswitch_device *device, 1688 NVSWITCH_PORT_EVENT_LOG_TYPE *port_events 1689 ) 1690 { 1691 if (port_events == NULL) 1692 return; 1693 1694 port_events->port_event_start = 0; 1695 port_events->port_event_count = 0; 1696 port_events->port_event_log_size = 0; 1697 port_events->bOverflow = NV_FALSE; 1698 1699 if (port_events->port_event_log != NULL) 1700 { 1701 nvswitch_os_free(port_events->port_event_log); 1702 port_events->port_event_log = NULL; 1703 } 1704 } 1705 1706 NvlStatus 1707 nvswitch_lib_initialize_device 1708 ( 1709 nvswitch_device *device 1710 ) 1711 { 1712 NvlStatus retval = NVL_SUCCESS; 1713 NvU8 link_num; 1714 nvlink_link *link = NULL; 1715 NvBool is_blacklisted_by_os = NV_FALSE; 1716 1717 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 1718 { 1719 return -NVL_BAD_ARGS; 1720 } 1721 1722 if (NVSWITCH_IS_DEVICE_INITIALIZED(device)) 1723 { 1724 NVSWITCH_PRINT(device, SETUP, "Device is already initialized!\n"); 1725 return NVL_SUCCESS; 1726 } 1727 1728 NVSWITCH_PRINT(device, SETUP, 1729 "Initializing nvswitch at (%04x:%02x:%02x.%02x)\n", 1730 device->nvlink_device->pciInfo.domain, 1731 device->nvlink_device->pciInfo.bus, 1732 device->nvlink_device->pciInfo.device, 1733 device->nvlink_device->pciInfo.function); 1734 1735 nvListInit(&device->client_events_list); 1736 1737 for (link_num=0; link_num < nvswitch_get_num_links(device); link_num++) 1738 { 1739 nvListInit(&device->link[link_num].inbandData.persistent_list); 1740 nvListInit(&device->link[link_num].inbandData.nonpersistent_list); 1741 } 1742 1743 retval = nvswitch_lib_load_platform_info(device); 1744 if (retval != NVL_SUCCESS) 1745 { 1746 NVSWITCH_PRINT(device, ERROR, "Failed to load platform information\n"); 1747 return retval; 1748 } 1749 1750 if (nvswitch_is_soe_supported(device)) 1751 { 1752 retval = _nvswitch_construct_soe(device); 1753 if (retval != NVL_SUCCESS) 1754 { 1755 return retval; 1756 } 1757 } 1758 else 1759 { 1760 NVSWITCH_PRINT(device, INFO, "SOE is not supported, skipping construct\n"); 1761 } 1762 if (nvswitch_is_cci_supported(device)) 1763 { 1764 retval = _nvswitch_construct_cci(device); 1765 if (retval != NVL_SUCCESS) 1766 { 1767 NVSWITCH_PRINT(device, ERROR, "Failed to construct CCI: %d\n", retval); 1768 goto nvswitch_initialize_cci_fail; 1769 } 1770 } 1771 else 1772 { 1773 NVSWITCH_PRINT(device, INFO, "CCI is not supported, skipping construct\n"); 1774 } 1775 1776 _nvswitch_set_dma_mask(device); 1777 1778 retval = _nvswitch_initialize_device_state(device); 1779 if (NVL_SUCCESS != retval) 1780 { 1781 NVSWITCH_PRINT(device, ERROR, 1782 "Failed to initialize device state: %d!\n", 1783 retval); 1784 goto nvswitch_initialize_device_state_fail; 1785 } 1786 1787 device->hal.nvswitch_load_uuid(device); 1788 1789 /* 1790 * Check module parameters for blacklisted device 1791 */ 1792 if (nvswitch_os_is_uuid_in_blacklist(&device->uuid) == NV_TRUE) 1793 { 1794 NVSWITCH_PRINT(device, SETUP, 1795 "Blacklisted nvswitch at (%04x:%02x:%02x.%02x)\n", 1796 device->nvlink_device->pciInfo.domain, 1797 device->nvlink_device->pciInfo.bus, 1798 device->nvlink_device->pciInfo.device, 1799 device->nvlink_device->pciInfo.function); 1800 is_blacklisted_by_os = NV_TRUE; 1801 // initialization continues until we have updated InfoROM... 1802 } 1803 1804 if (nvswitch_is_inforom_supported(device)) 1805 { 1806 retval = nvswitch_initialize_inforom(device); 1807 if (NVL_SUCCESS != retval) 1808 { 1809 NVSWITCH_PRINT(device, ERROR, 1810 "Failed to initialize InfoROM rc: %d\n", 1811 retval); 1812 goto nvswitch_initialize_device_state_fail; 1813 } 1814 1815 retval = nvswitch_initialize_inforom_objects(device); 1816 if (NVL_SUCCESS != retval) 1817 { 1818 NVSWITCH_PRINT(device, ERROR, 1819 "Failed to initialize InfoROM objects! rc:%d\n", 1820 retval); 1821 goto nvswitch_initialize_inforom_fail; 1822 } 1823 } 1824 else 1825 { 1826 NVSWITCH_PRINT(device, INFO, 1827 "InfoROM is not supported, skipping init\n"); 1828 } 1829 1830 (void)device->hal.nvswitch_read_oob_blacklist_state(device); 1831 (void)device->hal.nvswitch_write_fabric_state(device); 1832 1833 nvswitch_task_create(device, &nvswitch_fabric_state_heartbeat, 1834 NVSWITCH_HEARTBEAT_INTERVAL_NS, 1835 NVSWITCH_TASK_TYPE_FLAGS_RUN_EVEN_IF_DEVICE_NOT_INITIALIZED); 1836 1837 // 1838 // Blacklisted devices return successfully in order to preserve the fabric state heartbeat 1839 // and ensure OOB utilities don't think the driver has died 1840 // 1841 if (device->device_blacklist_reason == NVSWITCH_DEVICE_BLACKLIST_REASON_MANUAL_OUT_OF_BAND) 1842 { 1843 NVSWITCH_PRINT(device, SETUP, 1844 "Blacklisted nvswitch at (%04x:%02x:%02x.%02x)\n", 1845 device->nvlink_device->pciInfo.domain, 1846 device->nvlink_device->pciInfo.bus, 1847 device->nvlink_device->pciInfo.device, 1848 device->nvlink_device->pciInfo.function); 1849 return NVL_SUCCESS; 1850 } 1851 1852 if (is_blacklisted_by_os) 1853 { 1854 (void)nvswitch_lib_blacklist_device(device, NVSWITCH_DEVICE_BLACKLIST_REASON_MANUAL_IN_BAND); 1855 return NVL_SUCCESS; 1856 } 1857 1858 for (link_num=0; link_num < nvswitch_get_num_links(device); link_num++) 1859 { 1860 if (!nvswitch_is_link_valid(device, link_num)) 1861 { 1862 continue; 1863 } 1864 1865 retval = nvswitch_create_link(device, link_num, &link); 1866 if (NVL_SUCCESS != retval) 1867 { 1868 NVSWITCH_PRINT(device, ERROR, 1869 "Failed to create link %d : %d!\n", 1870 link_num, 1871 retval); 1872 goto nvswitch_link_fail; 1873 } 1874 1875 retval = nvlink_lib_register_link(device->nvlink_device, link); 1876 if (NVL_SUCCESS != retval) 1877 { 1878 NVSWITCH_PRINT(device, ERROR, 1879 "Failed to register link %d with the nvlink core : %d!\n", 1880 link_num, 1881 retval); 1882 1883 // Free the single dangling link. 1884 nvswitch_destroy_link(link); 1885 1886 goto nvswitch_link_fail; 1887 } 1888 1889 nvswitch_reset_persistent_link_hw_state(device, link_num); 1890 1891 // 1892 // During Nvswitch initialization, the default L1 thresholds are programmed by the 1893 // BIOS from the BIOS tables. Save these L1 Threshold Values in scratch registers 1894 // for use when resetting the thresholds to default. 1895 // 1896 nvswitch_program_l1_scratch_reg(device, link_num); 1897 1898 // 1899 // WAR : Initializing the L1 threshold registers at this point as a WAR for 1900 // Bug 3963639 where it was discussed that the L1 threshold register should have 1901 // the default value for all available links and not just for active links. 1902 // 1903 nvswitch_init_lpwr_regs(link); 1904 } 1905 1906 retval = nvswitch_set_training_mode(device); 1907 1908 if (retval != NVL_SUCCESS) 1909 { 1910 NVSWITCH_PRINT(device, ERROR, "Failed to determine link training mode! rc: %d\n", retval); 1911 goto nvswitch_link_fail; 1912 } 1913 // Initialize select scratch registers to 0x0 1914 device->hal.nvswitch_init_scratch(device); 1915 1916 retval = nvswitch_construct_error_log(&device->log_FATAL_ERRORS, 1024, NV_FALSE); 1917 if (retval != NVL_SUCCESS) 1918 { 1919 NVSWITCH_PRINT(device, ERROR, "Failed to construct log_FATAL_ERRORS! rc: %d\n", retval); 1920 goto nvswitch_construct_error_log_fail; 1921 } 1922 1923 retval = nvswitch_construct_error_log(&device->log_NONFATAL_ERRORS, 1024, NV_TRUE); 1924 if (retval != NVL_SUCCESS) 1925 { 1926 NVSWITCH_PRINT(device, ERROR, "Failed to construct log_NONFATAL_ERRORS! rc: %d\n", retval); 1927 goto nvswitch_construct_error_log_fail; 1928 } 1929 1930 retval = _nvswitch_construct_port_event_log(&device->log_PORT_EVENTS, NVSWITCH_PORT_EVENT_LOG_SIZE, NV_TRUE); 1931 if (retval != NVL_SUCCESS) 1932 { 1933 NVSWITCH_PRINT(device, ERROR, "Failed to construct log_PORT_EVENTS! rc: %d\n", retval); 1934 goto nvswitch_construct_port_event_log_fail; 1935 } 1936 1937 if (device->regkeys.latency_counter == NV_SWITCH_REGKEY_LATENCY_COUNTER_LOGGING_ENABLE) 1938 { 1939 nvswitch_task_create(device, &nvswitch_internal_latency_bin_log, 1940 nvswitch_get_latency_sample_interval_msec(device) * NVSWITCH_INTERVAL_1MSEC_IN_NS * 9/10, 0); 1941 } 1942 1943 nvswitch_task_create(device, &nvswitch_ecc_writeback_task, 1944 (60 * NVSWITCH_INTERVAL_1SEC_IN_NS), 0); 1945 1946 if (IS_RTLSIM(device) || IS_EMULATION(device) || IS_FMODEL(device)) 1947 { 1948 NVSWITCH_PRINT(device, WARN, 1949 "%s: Skipping setup of NvSwitch thermal alert monitoring\n", 1950 __FUNCTION__); 1951 } 1952 else 1953 { 1954 nvswitch_task_create(device, &nvswitch_monitor_thermal_alert, 1955 100*NVSWITCH_INTERVAL_1MSEC_IN_NS, 0); 1956 } 1957 1958 device->nvlink_device->initialized = 1; 1959 1960 return NVL_SUCCESS; 1961 1962 nvswitch_construct_error_log_fail: 1963 //free allocated memory to avoid leaking 1964 nvswitch_destroy_error_log(device, &device->log_FATAL_ERRORS); 1965 nvswitch_destroy_error_log(device, &device->log_NONFATAL_ERRORS); 1966 1967 nvswitch_construct_port_event_log_fail: 1968 //free allocated memory to avoid leaking 1969 _nvswitch_destroy_port_event_log(device, &device->log_PORT_EVENTS); 1970 1971 nvswitch_link_fail: 1972 // Track down all links that successfully registered. 1973 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 1974 { 1975 if (nvlink_lib_get_link(device->nvlink_device, link_num, &link) == NVL_SUCCESS) 1976 { 1977 nvlink_lib_unregister_link(link); 1978 nvswitch_destroy_link(link); 1979 } 1980 } 1981 1982 nvswitch_destroy_inforom_objects(device); 1983 1984 nvswitch_initialize_inforom_fail: 1985 nvswitch_destroy_inforom(device); 1986 1987 nvswitch_initialize_device_state_fail: 1988 _nvswitch_destruct_cci(device); 1989 nvswitch_initialize_cci_fail: 1990 _nvswitch_destruct_soe(device); 1991 nvswitch_tasks_destroy(device); 1992 1993 return retval; 1994 } 1995 1996 NvBool 1997 nvswitch_lib_validate_device_id 1998 ( 1999 NvU32 device_id 2000 ) 2001 { 2002 if (nvswitch_is_lr10_device_id(device_id)) 2003 { 2004 return NV_TRUE; 2005 } 2006 if (nvswitch_is_ls10_device_id(device_id)) 2007 { 2008 return NV_TRUE; 2009 } 2010 return NV_FALSE; 2011 } 2012 2013 NvlStatus 2014 nvswitch_lib_post_init_device 2015 ( 2016 nvswitch_device *device 2017 ) 2018 { 2019 NvlStatus retval; 2020 NvlStatus status; 2021 NvU32 link_num; 2022 NvU64 mode; 2023 nvlink_link *link; 2024 NvU64 enabledLinkMaskNonCci; 2025 2026 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 2027 { 2028 return -NVL_ERR_INVALID_STATE; 2029 } 2030 2031 retval = _nvswitch_post_init_device_setup(device); 2032 if (retval != NVL_SUCCESS) 2033 { 2034 return retval; 2035 } 2036 2037 if (nvswitch_is_bios_supported(device)) 2038 { 2039 retval = nvswitch_bios_get_image(device); 2040 if (retval != NVL_SUCCESS) 2041 { 2042 return retval; 2043 } 2044 2045 retval = nvswitch_parse_bios_image(device); 2046 if (retval != NVL_SUCCESS) 2047 { 2048 return retval; 2049 } 2050 } 2051 else 2052 { 2053 NVSWITCH_PRINT(device, ERROR, 2054 "%s: Skipping BIOS parsing since BIOS is unsupported.\n", 2055 __FUNCTION__); 2056 } 2057 2058 if (nvswitch_is_cci_supported(device)) 2059 { 2060 retval = cciLoad(device); 2061 if (NVL_SUCCESS != retval) 2062 { 2063 NVSWITCH_PRINT(device, ERROR, "%s: Init CCI failed\n", 2064 __FUNCTION__); 2065 return retval; 2066 } 2067 2068 enabledLinkMaskNonCci = nvswitch_get_enabled_link_mask(device); 2069 2070 FOR_EACH_INDEX_IN_MASK(64, link_num, enabledLinkMaskNonCci) 2071 { 2072 if (cciIsLinkManaged(device, link_num)) 2073 { 2074 enabledLinkMaskNonCci = enabledLinkMaskNonCci & ~NVBIT64(link_num); 2075 } 2076 } 2077 FOR_EACH_INDEX_IN_MASK_END; 2078 2079 if (enabledLinkMaskNonCci != 0) 2080 { 2081 nvswitch_task_create(device, &_nvswitch_update_link_state_led, 2082 NVSWITCH_INTERVAL_1SEC_IN_NS, 0); 2083 } 2084 } 2085 else 2086 { 2087 NVSWITCH_PRINT(device, INFO, "%s: Skipping CCI init.\n", 2088 __FUNCTION__); 2089 } 2090 2091 retval = _nvswitch_setup_system_registers(device); 2092 if (retval != NVL_SUCCESS) 2093 { 2094 return retval; 2095 } 2096 2097 nvswitch_smbpbi_post_init(device); 2098 2099 // ALI launched by VBIOS on silicon 2100 if (IS_RTLSIM(device) || IS_EMULATION(device) || IS_FMODEL(device)) 2101 { 2102 (void)nvswitch_launch_ALI(device); 2103 } 2104 2105 // 2106 // There is an edge case where a hyperisor may not send same number 2107 // of reset to switch and GPUs, so try to re-train links in fault 2108 // if possible 2109 // 2110 for (link_num=0; link_num < nvswitch_get_num_links(device); link_num++) 2111 { 2112 // Sanity check 2113 if (!nvswitch_is_link_valid(device, link_num)) 2114 { 2115 continue; 2116 } 2117 2118 // CCI links are trained and managed by SOE, skip any driver based training 2119 if (cciIsLinkManaged(device, link_num)) 2120 { 2121 continue; 2122 } 2123 status = nvlink_lib_get_link(device->nvlink_device, link_num, &link); 2124 if (status != NVL_SUCCESS) 2125 { 2126 NVSWITCH_PRINT(device, ERROR, "%s: Failed to get link for LinkId %d\n", 2127 __FUNCTION__, link_num); 2128 continue; 2129 } 2130 2131 // If the link is in fault then re-train 2132 if(_nvswitch_corelib_get_dl_link_mode(link, &mode) != NVL_SUCCESS) 2133 { 2134 NVSWITCH_PRINT(device, ERROR, "%s: nvlipt_lnk_status: Failed to check link mode! LinkId %d\n", 2135 __FUNCTION__, link_num); 2136 } 2137 else if(mode == NVLINK_LINKSTATE_FAULT) 2138 { 2139 NVSWITCH_PRINT(device, INFO, "%s: retraining LinkId %d\n", 2140 __FUNCTION__, link_num); 2141 nvswitch_reset_and_train_link(device, link); 2142 } 2143 } 2144 2145 return NVL_SUCCESS; 2146 } 2147 2148 void 2149 nvswitch_lib_post_init_blacklist_device 2150 ( 2151 nvswitch_device *device 2152 ) 2153 { 2154 _nvswitch_post_init_blacklist_device_setup(device); 2155 } 2156 2157 void 2158 _nvswitch_check_pending_data_and_notify 2159 ( 2160 nvswitch_device *device, 2161 NVSWITCH_CLIENT_EVENT *event 2162 ) 2163 { 2164 switch (event->eventId) 2165 { 2166 case NVSWITCH_DEVICE_EVENT_INBAND_DATA: 2167 { 2168 NvU32 i; 2169 2170 for (i = 0; i < nvswitch_get_num_links(device); i++) 2171 { 2172 if (!nvListIsEmpty(&device->link[i].inbandData.persistent_list) || 2173 !nvListIsEmpty(&device->link[i].inbandData.nonpersistent_list)) 2174 { 2175 (void)nvswitch_os_notify_client_event(device->os_handle, 2176 event->private_driver_data, 2177 event->eventId); 2178 } 2179 } 2180 break; 2181 } 2182 2183 default: 2184 return; 2185 } 2186 } 2187 2188 /*! 2189 * @brief: Gets the client event associated with the file descriptor 2190 * if it already exists in the Device's client event list. 2191 * 2192 * If found, and if there is pending data for the event, 2193 * the event is triggered before returning to unblock the 2194 * client right away. 2195 */ 2196 NvlStatus 2197 nvswitch_lib_get_client_event 2198 ( 2199 nvswitch_device *device, 2200 void *osPrivate, 2201 NVSWITCH_CLIENT_EVENT **ppClientEvent 2202 ) 2203 { 2204 NVSWITCH_CLIENT_EVENT *curr = NULL; 2205 2206 *ppClientEvent = NULL; 2207 2208 if(!NVSWITCH_IS_DEVICE_VALID(device)) 2209 { 2210 return -NVL_BAD_ARGS; 2211 } 2212 2213 nvListForEachEntry(curr, &device->client_events_list, entry) 2214 { 2215 if (curr->private_driver_data == osPrivate) 2216 { 2217 *ppClientEvent = curr; 2218 _nvswitch_check_pending_data_and_notify(device, curr); 2219 return NVL_SUCCESS; 2220 } 2221 } 2222 2223 return -NVL_NOT_FOUND; 2224 } 2225 2226 /*! 2227 * @brief: Adds an event to the front of the 2228 * Device's client event list. 2229 */ 2230 NvlStatus 2231 nvswitch_lib_add_client_event 2232 ( 2233 nvswitch_device *device, 2234 void *osPrivate, 2235 NvU32 eventId 2236 ) 2237 { 2238 NVSWITCH_CLIENT_EVENT *newEvent; 2239 NvlStatus status = NVL_SUCCESS; 2240 2241 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2242 { 2243 return -NVL_BAD_ARGS; 2244 } 2245 2246 if (eventId >= NVSWITCH_DEVICE_EVENT_COUNT) 2247 { 2248 NVSWITCH_PRINT(device, ERROR, "%s: Invalid event Id.\n", __FUNCTION__); 2249 return -NVL_BAD_ARGS; 2250 } 2251 2252 // Invoke OS specific API to add event. 2253 status = nvswitch_os_add_client_event(device->os_handle, 2254 osPrivate, 2255 eventId); 2256 if (status != NVL_SUCCESS) 2257 { 2258 NVSWITCH_PRINT(device, ERROR, "%s: Failed to add client event.\n", __FUNCTION__); 2259 return status; 2260 } 2261 2262 newEvent = nvswitch_os_malloc(sizeof(*newEvent)); 2263 if (newEvent == NULL) 2264 { 2265 return -NVL_NO_MEM; 2266 } 2267 2268 newEvent->eventId = eventId; 2269 newEvent->private_driver_data = osPrivate; 2270 2271 nvListAdd(&newEvent->entry, &device->client_events_list); 2272 2273 return NVL_SUCCESS; 2274 } 2275 2276 /*! 2277 * @brief: Removes all events corresponding to osPrivate, 2278 * from the Device's client event list. 2279 */ 2280 NvlStatus 2281 nvswitch_lib_remove_client_events 2282 ( 2283 nvswitch_device *device, 2284 void *osPrivate 2285 ) 2286 { 2287 NVSWITCH_CLIENT_EVENT *curr = NULL; 2288 NVSWITCH_CLIENT_EVENT *next = NULL; 2289 NvlStatus status = NVL_SUCCESS; 2290 2291 // 2292 // Device shutdown may happen before this is called, so return 2293 // if device is gone 2294 // 2295 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2296 { 2297 return NVL_SUCCESS; 2298 } 2299 2300 nvListForEachEntry_safe(curr, next, &device->client_events_list, entry) 2301 { 2302 if (curr->private_driver_data == osPrivate) 2303 { 2304 nvListDel(&curr->entry); 2305 nvswitch_os_free(curr); 2306 } 2307 } 2308 2309 // Invoke OS specific API to remove event. 2310 status = nvswitch_os_remove_client_event(device->os_handle, 2311 osPrivate); 2312 if (status != NVL_SUCCESS) 2313 { 2314 NVSWITCH_PRINT(device, ERROR, "%s: Failed to remove client events.\n", __FUNCTION__); 2315 return status; 2316 } 2317 2318 return NVL_SUCCESS; 2319 } 2320 2321 /*! 2322 * @brief: Notifies all events with matching event id in the 2323 * Device's client event list. 2324 */ 2325 NvlStatus 2326 nvswitch_lib_notify_client_events 2327 ( 2328 nvswitch_device *device, 2329 NvU32 eventId 2330 ) 2331 { 2332 NvlStatus status; 2333 NVSWITCH_CLIENT_EVENT *curr = NULL; 2334 2335 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2336 { 2337 return -NVL_BAD_ARGS; 2338 } 2339 2340 if (eventId >= NVSWITCH_DEVICE_EVENT_COUNT) 2341 { 2342 NVSWITCH_PRINT(device, ERROR, "%s: Invalid event Id.\n", __FUNCTION__); 2343 return -NVL_BAD_ARGS; 2344 } 2345 2346 nvListForEachEntry(curr, &device->client_events_list, entry) 2347 { 2348 if (curr->eventId == eventId) 2349 { 2350 // OS specific event notification. 2351 status = nvswitch_os_notify_client_event(device->os_handle, 2352 curr->private_driver_data, 2353 eventId); 2354 if (status != NVL_SUCCESS) 2355 { 2356 return status; 2357 } 2358 } 2359 } 2360 2361 return NVL_SUCCESS; 2362 } 2363 2364 void 2365 nvswitch_record_port_event 2366 ( 2367 nvswitch_device *device, 2368 NVSWITCH_PORT_EVENT_LOG_TYPE *port_events, 2369 NvU32 link_id, 2370 NvU8 port_event_type 2371 ) 2372 { 2373 NvU32 idx; 2374 2375 NVSWITCH_ASSERT(port_events != NULL); 2376 2377 // If no port events log has been created, then don't log it. 2378 if ((port_events->port_event_log_size != 0) && 2379 (port_events->port_event_log != NULL)) 2380 { 2381 idx = (port_events->port_event_start + port_events->port_event_count) 2382 % port_events->port_event_log_size; 2383 2384 if (port_events->port_event_count == port_events->port_event_log_size) 2385 { 2386 // Error: ring buffer is already full/ 2387 if (port_events->overwritable) 2388 { 2389 port_events->port_event_start = (port_events->port_event_start + 1) 2390 % port_events->port_event_log_size; 2391 port_events->bOverflow = NV_TRUE; 2392 } 2393 else 2394 { 2395 // No logging, ring buffer is full 2396 return; 2397 } 2398 } 2399 else 2400 { 2401 port_events->port_event_count++; 2402 } 2403 // Log port event info 2404 port_events->port_event_log[idx].link_id = link_id; 2405 port_events->port_event_log[idx].port_event_type = port_event_type; 2406 2407 // Log tracking info 2408 port_events->port_event_log[idx].time = nvswitch_os_get_platform_time(); 2409 port_events->port_event_log[idx].local_port_event_num = port_events->port_event_total; 2410 } 2411 port_events->port_event_total++; 2412 } 2413 2414 /* 2415 * @Brief : Retrives a port event entry by index. 2416 * 2417 * @Description : Retrieves the port_event at index port_event_idx. If index is out 2418 * of range, returns an empty port event entry with port_event_type = 2 2419 * 2420 * @param[in] device NVSwitch device to contain this link 2421 * @param[in] port_events Log of all port events with metadata 2422 * @param[in] port_event_idx Index of entry to retrieve (0 = oldest port event) 2423 * @param[out] port_event_count Clear only non-persistent list 2424 */ 2425 void 2426 nvswitch_get_port_event 2427 ( 2428 nvswitch_device *device, 2429 NVSWITCH_PORT_EVENT_LOG_TYPE *port_events, 2430 NVSWITCH_PORT_EVENT_TYPE *port_event_entry, 2431 NvU32 port_event_idx, 2432 NvU32 *port_event_count 2433 ) 2434 { 2435 NvU32 idx; 2436 NVSWITCH_ASSERT(port_events != NULL); 2437 2438 if (port_event_entry != NULL) 2439 { 2440 // Index is out of range 2441 if (port_event_idx >= port_events->port_event_count) 2442 { 2443 nvswitch_os_memset(port_event_entry, 0, sizeof(*port_event_entry)); 2444 port_event_entry->port_event_type = NVSWITCH_PORT_EVENT_TYPE_INVALID; 2445 port_event_entry->time = nvswitch_os_get_platform_time(); 2446 } 2447 else 2448 { 2449 idx = (port_events->port_event_start + port_event_idx) % port_events->port_event_log_size; 2450 *port_event_entry = port_events->port_event_log[idx]; 2451 } 2452 } 2453 2454 if (port_event_count) 2455 { 2456 *port_event_count = port_events->port_event_count; 2457 } 2458 } 2459 2460 NvlStatus 2461 nvswitch_ctrl_get_port_events 2462 ( 2463 nvswitch_device *device, 2464 NVSWITCH_GET_PORT_EVENTS_PARAMS *p 2465 ) 2466 { 2467 NvU32 index = 0; 2468 NvU32 count = 0; 2469 NVSWITCH_PORT_EVENT_LOG_TYPE *port_events = &device->log_PORT_EVENTS; 2470 NVSWITCH_PORT_EVENT_TYPE port_event; 2471 2472 nvswitch_os_memset(p->portEvent, 0, sizeof(NVSWITCH_PORT_EVENT) 2473 *NVSWITCH_PORT_EVENT_COUNT_SIZE); 2474 p->nextPortEventIndex = port_events->port_event_total; 2475 p->portEventCount = 0; 2476 p->bOverflow = port_events->bOverflow; 2477 2478 // Return if there are no more port events to get 2479 nvswitch_get_port_event(device, port_events, &port_event, index, &count); 2480 if (count == 0) 2481 { 2482 return NVL_SUCCESS; 2483 } 2484 2485 // If port event's local_port_Event_num is smaller than the portEventIndex 2486 // passed in by the client, fast-forward index by the difference. 2487 // This will skip over port events that were previously read by the client. 2488 if (port_event.local_port_event_num < p->portEventIndex) 2489 { 2490 index = (NvU32) (p->portEventIndex - port_event.local_port_event_num); 2491 } 2492 2493 // Return if there are no more events after fast-forwarding. 2494 if (index >= count) 2495 { 2496 return NVL_SUCCESS; 2497 } 2498 2499 while ((p->portEventCount < NVSWITCH_PORT_EVENT_COUNT_SIZE) && (index < count)) 2500 { 2501 nvswitch_get_port_event(device, port_events, &port_event, index, NULL); 2502 2503 p->portEvent[p->portEventCount].port_event_type = port_event.port_event_type; 2504 p->portEvent[p->portEventCount].link_id = port_event.link_id; 2505 p->portEvent[p->portEventCount].time = port_event.time; 2506 2507 p->portEventCount++; 2508 index++; 2509 } 2510 2511 p->portEventIndex = port_event.local_port_event_num + 1; 2512 2513 return NVL_SUCCESS; 2514 } 2515 2516 /*! 2517 @brief: Release ROM image from memory. 2518 */ 2519 void 2520 _nvswitch_destroy_rom(nvswitch_device *device) 2521 { 2522 if (device->biosImage.pImage != NULL) 2523 { 2524 nvswitch_os_free(device->biosImage.pImage); 2525 device->biosImage.pImage = NULL; 2526 } 2527 } 2528 2529 /*! 2530 * @brief: Free the device's client event list 2531 */ 2532 static void 2533 _nvswitch_destroy_event_list(nvswitch_device *device) 2534 { 2535 NVSWITCH_CLIENT_EVENT *curr = NULL; 2536 NVSWITCH_CLIENT_EVENT *next = NULL; 2537 2538 nvListForEachEntry_safe(curr, next, &device->client_events_list, entry) 2539 { 2540 nvListDel(&curr->entry); 2541 nvswitch_os_free(curr); 2542 } 2543 } 2544 2545 NvlStatus 2546 nvswitch_lib_shutdown_device 2547 ( 2548 nvswitch_device *device 2549 ) 2550 { 2551 NVSWITCH_INBAND_FLUSH_DATA_PARAMS p; 2552 2553 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 2554 { 2555 return -NVL_BAD_ARGS; 2556 } 2557 2558 // 2559 // Set fabric state to offline 2560 // 2561 if (device->device_fabric_state != NVSWITCH_DEVICE_FABRIC_STATE_BLACKLISTED) 2562 device->device_fabric_state = NVSWITCH_DEVICE_FABRIC_STATE_OFFLINE; 2563 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_OFFLINE; 2564 (void)device->hal.nvswitch_write_fabric_state(device); 2565 2566 nvswitch_hw_counter_shutdown(device); 2567 2568 // FLUSH any pending messages to avoid memory leaks 2569 p.linkMask = nvswitch_get_enabled_link_mask(device); 2570 _nvswitch_ctrl_inband_flush_data(device, &p); 2571 2572 _nvswitch_destruct_cci(device); 2573 _nvswitch_led_shutdown(device); 2574 2575 _nvswitch_unregister_links(device); 2576 2577 nvswitch_destroy_error_log(device, &device->log_FATAL_ERRORS); 2578 nvswitch_destroy_error_log(device, &device->log_NONFATAL_ERRORS); 2579 2580 _nvswitch_destroy_port_event_log(device, &device->log_PORT_EVENTS); 2581 2582 nvswitch_smbpbi_unload(device); 2583 _nvswitch_destroy_event_list(device); 2584 2585 nvswitch_destroy_inforom_objects(device); 2586 nvswitch_destroy_inforom(device); 2587 2588 nvswitch_smbpbi_destroy(device); 2589 2590 nvswitch_destroy_device_state(device); 2591 2592 _nvswitch_destroy_rom(device); 2593 2594 _nvswitch_destruct_soe(device); 2595 2596 nvswitch_tasks_destroy(device); 2597 2598 return NVL_SUCCESS; 2599 } 2600 2601 NvlStatus 2602 nvswitch_lib_get_log_count 2603 ( 2604 nvswitch_device *device, 2605 NvU32 *fatal, NvU32 *nonfatal, NvU32 *portEvent 2606 ) 2607 { 2608 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device) || 2609 fatal == NULL || nonfatal == NULL || portEvent == NULL) 2610 { 2611 return -NVL_BAD_ARGS; 2612 } 2613 2614 *fatal = device->log_FATAL_ERRORS.error_count; 2615 *nonfatal = device->log_NONFATAL_ERRORS.error_count; 2616 *portEvent = device->log_PORT_EVENTS.port_event_count; 2617 // No report of log_INFO currently 2618 2619 return NVL_SUCCESS; 2620 } 2621 2622 NvlStatus 2623 nvswitch_lib_load_platform_info 2624 ( 2625 nvswitch_device *device 2626 ) 2627 { 2628 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 2629 { 2630 return -NVL_BAD_ARGS; 2631 } 2632 2633 device->hal.nvswitch_determine_platform(device); 2634 2635 return NVL_SUCCESS; 2636 } 2637 2638 void 2639 nvswitch_lib_get_device_info 2640 ( 2641 nvswitch_device *device, 2642 struct nvlink_pci_info **pciInfo 2643 ) 2644 { 2645 if (!NVSWITCH_IS_DEVICE_VALID(device) || pciInfo == NULL) 2646 { 2647 NVSWITCH_ASSERT(0); 2648 return; 2649 } 2650 2651 *pciInfo = &device->nvlink_device->pciInfo; 2652 } 2653 2654 NvlStatus 2655 nvswitch_lib_get_bios_version 2656 ( 2657 nvswitch_device *device, 2658 NvU64 *version 2659 ) 2660 { 2661 NVSWITCH_GET_BIOS_INFO_PARAMS p = { 0 }; 2662 NvlStatus ret; 2663 2664 if (!device) 2665 return -NVL_BAD_ARGS; 2666 2667 ret = device->hal.nvswitch_ctrl_get_bios_info(device, &p); 2668 2669 if (version != NULL) 2670 { 2671 *version = p.version; 2672 } 2673 2674 return ret; 2675 } 2676 2677 NvlStatus 2678 nvswitch_lib_use_pin_irq 2679 ( 2680 nvswitch_device *device 2681 ) 2682 { 2683 return IS_FMODEL(device); 2684 } 2685 2686 2687 NvlStatus 2688 nvswitch_lib_register_device 2689 ( 2690 NvU16 pci_domain, 2691 NvU8 pci_bus, 2692 NvU8 pci_device, 2693 NvU8 pci_func, 2694 NvU16 pci_device_id, 2695 void *os_handle, 2696 NvU32 os_instance, 2697 nvswitch_device **return_device 2698 ) 2699 { 2700 nvswitch_device *device = NULL; 2701 nvlink_device *coreDev = NULL; 2702 NvlStatus retval = NVL_SUCCESS; 2703 2704 if (!nvlink_lib_is_initialized()) 2705 { 2706 NVSWITCH_PRINT(device, ERROR, 2707 "NVLink core lib isn't initialized yet!\n"); 2708 return -NVL_INITIALIZATION_TOTAL_FAILURE; 2709 } 2710 2711 if (return_device == NULL || os_handle == NULL) 2712 { 2713 return -NVL_BAD_ARGS; 2714 } 2715 2716 *return_device = NULL; 2717 2718 device = nvswitch_os_malloc(sizeof(*device)); 2719 if (NULL == device) 2720 { 2721 NVSWITCH_PRINT(device, ERROR, 2722 "nvswitch_os_malloc during device creation failed!\n"); 2723 return -NVL_NO_MEM; 2724 } 2725 nvswitch_os_memset(device, 0, sizeof(*device)); 2726 2727 nvswitch_os_snprintf(device->name, sizeof(device->name), 2728 NVSWITCH_DEVICE_NAME "%d", os_instance); 2729 2730 coreDev = nvswitch_os_malloc(sizeof(*coreDev)); 2731 if (NULL == coreDev) 2732 { 2733 NVSWITCH_PRINT(device, ERROR, 2734 "nvswitch_os_malloc during device creation failed!\n"); 2735 2736 retval = -NVL_NO_MEM; 2737 goto nvlink_lib_register_device_fail; 2738 } 2739 nvswitch_os_memset(coreDev, 0, sizeof(*coreDev)); 2740 2741 coreDev->driverName = 2742 nvswitch_os_malloc(sizeof(NVSWITCH_DRIVER_NAME)); 2743 if (coreDev->driverName == NULL) 2744 { 2745 NVSWITCH_PRINT(device, ERROR, 2746 "nvswitch_os_malloc during device creation failed!\n"); 2747 2748 retval = -NVL_NO_MEM; 2749 goto nvlink_lib_register_device_fail; 2750 } 2751 nvswitch_os_memcpy(coreDev->driverName, NVSWITCH_DRIVER_NAME, 2752 sizeof(NVSWITCH_DRIVER_NAME)); 2753 2754 device->os_handle = os_handle; 2755 device->os_instance = os_instance; 2756 2757 device->nvlink_device = coreDev; 2758 device->nvlink_device->deviceName = device->name; 2759 device->nvlink_device->uuid = NULL; // No UUID support for switch 2760 2761 device->nvlink_device->pciInfo.domain = pci_domain; 2762 device->nvlink_device->pciInfo.bus = pci_bus; 2763 device->nvlink_device->pciInfo.device = pci_device; 2764 device->nvlink_device->pciInfo.function = pci_func; 2765 device->nvlink_device->pciInfo.pciDeviceId = pci_device_id; 2766 2767 // nvlink_device has a back pointer to nvswitch_device 2768 device->nvlink_device->pDevInfo = device; 2769 device->nvlink_device->type = NVLINK_DEVICE_TYPE_NVSWITCH; 2770 2771 // 2772 // Initialize the Fabric State 2773 // 2774 device->fm_timeout = NVSWITCH_DEFAULT_FM_HEARTBEAT_TIMEOUT_MSEC; 2775 device->fabric_state_sequence_number = 0; 2776 device->driver_fabric_state = NVSWITCH_DRIVER_FABRIC_STATE_STANDBY; 2777 device->device_fabric_state = NVSWITCH_DEVICE_FABRIC_STATE_STANDBY; 2778 device->device_blacklist_reason = NVSWITCH_DEVICE_BLACKLIST_REASON_NONE; 2779 2780 // 2781 // Initialize HAL connectivity as early as possible so that other lib 2782 // interfaces can work. 2783 // 2784 retval = _nvswitch_setup_hal(device, device->nvlink_device->pciInfo.pciDeviceId); 2785 if (retval != NVL_SUCCESS) 2786 { 2787 goto nvlink_lib_register_device_fail; 2788 } 2789 2790 // 2791 // Initialize regkeys as early as possible so that most routines can take 2792 // advantage of them. 2793 // 2794 _nvswitch_init_device_regkeys(device); 2795 2796 // After regkeys have been set then only set the enableALI field. 2797 device->nvlink_device->enableALI = (device->regkeys.link_training_mode == 2798 NV_SWITCH_REGKEY_LINK_TRAINING_SELECT_ALI) ? NV_TRUE:NV_FALSE; 2799 2800 retval = nvlink_lib_register_device(device->nvlink_device); 2801 if (NVL_SUCCESS != retval) 2802 { 2803 NVSWITCH_PRINT(device, ERROR, 2804 "nvlinklib register device failed!\n"); 2805 goto nvlink_lib_register_device_fail; 2806 } 2807 2808 *return_device = device; 2809 2810 NVSWITCH_PRINT(device, SETUP, 2811 "Successfully registered with nvlinkcore\n"); 2812 2813 return retval; 2814 2815 nvlink_lib_register_device_fail: 2816 2817 if (NULL != coreDev) 2818 { 2819 nvswitch_os_free(coreDev->driverName); 2820 nvswitch_os_free(coreDev); 2821 } 2822 2823 if (NULL != device) 2824 nvswitch_os_free(device); 2825 2826 return retval; 2827 } 2828 2829 void 2830 nvswitch_lib_unregister_device 2831 ( 2832 nvswitch_device *device 2833 ) 2834 { 2835 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2836 { 2837 NVSWITCH_ASSERT(0); 2838 return; 2839 } 2840 2841 nvlink_lib_unregister_device(device->nvlink_device); 2842 2843 nvswitch_os_free(device->nvlink_device->driverName); 2844 nvswitch_os_free(device->nvlink_device); 2845 nvswitch_os_free(device); 2846 2847 return; 2848 } 2849 2850 /*! 2851 * @brief: Gets the mask of valid I2C ports on the 2852 * Device. 2853 */ 2854 NvlStatus 2855 nvswitch_lib_get_valid_ports_mask 2856 ( 2857 nvswitch_device *device, 2858 NvU32 *validPortsMask 2859 ) 2860 { 2861 NvU32 port_info; 2862 NvU32 i; 2863 NvU32 ports_mask = 0; 2864 NvBool is_i2c_access_allowed; 2865 NvBool is_port_allowed; 2866 2867 if (!NVSWITCH_IS_DEVICE_VALID(device) || 2868 (validPortsMask == NULL)) 2869 { 2870 return -NVL_BAD_ARGS; 2871 } 2872 2873 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 2874 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 2875 NV_TRUE : NV_FALSE; 2876 2877 for (i = 0; i < NVSWITCH_MAX_I2C_PORTS; i++) 2878 { 2879 port_info = nvswitch_i2c_get_port_info(device, i); 2880 2881 is_port_allowed = is_i2c_access_allowed ? NV_TRUE : 2882 FLD_TEST_DRF(_I2C, _PORTINFO, _ACCESS_ALLOWED, _TRUE, 2883 port_info); 2884 2885 if (is_port_allowed && 2886 FLD_TEST_DRF(_I2C, _PORTINFO, _DEFINED, _PRESENT, port_info)) 2887 { 2888 ports_mask |= NVBIT(i); 2889 } 2890 } 2891 2892 *validPortsMask = ports_mask; 2893 return NVL_SUCCESS; 2894 } 2895 2896 /*! 2897 * @brief: Returns if the I2C transactions are supported. 2898 */ 2899 NvBool 2900 nvswitch_lib_is_i2c_supported 2901 ( 2902 nvswitch_device *device 2903 ) 2904 { 2905 if (!NVSWITCH_IS_DEVICE_VALID(device)) 2906 { 2907 NVSWITCH_ASSERT(0); 2908 return NV_FALSE; 2909 } 2910 2911 return nvswitch_is_i2c_supported(device); 2912 } 2913 2914 static NvlStatus 2915 _nvswitch_perform_i2c_transfer 2916 ( 2917 nvswitch_device *device, 2918 NvU32 client, 2919 NvU8 type, 2920 NvU16 addr, 2921 NvU8 port, 2922 NvU8 cmd, 2923 NvU32 msgLength, 2924 NvU8 *pData 2925 ) 2926 { 2927 NvlStatus status; 2928 NvU16 deviceAddr; 2929 NvU32 speedMode; 2930 NvBool bIsRead = NV_FALSE; 2931 NvU32 flags = 0; 2932 NVSWITCH_CTRL_I2C_INDEXED_PARAMS i2c_params = {0}; 2933 NvBool is_i2c_access_allowed; 2934 2935 if (!nvswitch_os_is_admin()) 2936 { 2937 return -NVL_ERR_INSUFFICIENT_PERMISSIONS; 2938 } 2939 2940 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 2941 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 2942 NV_TRUE : NV_FALSE; 2943 2944 // 2945 // The address needs to be shifted by 1, 2946 // See NVSWITCH_CTRL_I2C_INDEXED_PARAMS 2947 // 2948 deviceAddr = addr << 1; 2949 speedMode = device->pI2c->Ports[port].defaultSpeedMode; 2950 flags = DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _START, _SEND) | 2951 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _STOP, _SEND) | 2952 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _ADDRESS_MODE, _7BIT) | 2953 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _FLAVOR, _HW) | 2954 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _BLOCK_PROTOCOL, _DISABLED) | 2955 DRF_DEF(SWITCH_CTRL, _I2C_FLAGS, _TRANSACTION_MODE, _NORMAL); 2956 2957 switch (speedMode) 2958 { 2959 case NVSWITCH_I2C_SPEED_MODE_1000KHZ: 2960 { 2961 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _1000KHZ, flags); 2962 break; 2963 } 2964 case NVSWITCH_I2C_SPEED_MODE_400KHZ: 2965 { 2966 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _400KHZ, flags); 2967 break; 2968 } 2969 case NVSWITCH_I2C_SPEED_MODE_300KHZ: 2970 { 2971 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _300KHZ, flags); 2972 break; 2973 } 2974 case NVSWITCH_I2C_SPEED_MODE_200KHZ: 2975 { 2976 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _200KHZ, flags); 2977 break; 2978 } 2979 case NVSWITCH_I2C_SPEED_MODE_100KHZ: 2980 { 2981 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _SPEED_MODE, _100KHZ, flags); 2982 break; 2983 } 2984 default: 2985 { 2986 NVSWITCH_PRINT(device, ERROR, "Invalid I2C speed!\n"); 2987 status = -NVL_BAD_ARGS; 2988 goto end; 2989 } 2990 } 2991 2992 switch (type) 2993 { 2994 case NVSWITCH_I2C_CMD_READ: 2995 bIsRead = NV_TRUE; 2996 // Fall through 2997 case NVSWITCH_I2C_CMD_WRITE: 2998 { 2999 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ZERO, flags); 3000 break; 3001 } 3002 case NVSWITCH_I2C_CMD_SMBUS_READ: 3003 { 3004 bIsRead = NV_TRUE; 3005 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _RESTART, _SEND, flags); 3006 // Fall through 3007 } 3008 case NVSWITCH_I2C_CMD_SMBUS_WRITE: 3009 { 3010 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ONE, flags); 3011 break; 3012 } 3013 case NVSWITCH_I2C_CMD_SMBUS_QUICK_READ: 3014 bIsRead = NV_TRUE; 3015 // Fall through 3016 case NVSWITCH_I2C_CMD_SMBUS_QUICK_WRITE: 3017 { 3018 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ZERO, flags); 3019 msgLength = 0; 3020 break; 3021 } 3022 default: 3023 { 3024 NVSWITCH_PRINT(device, ERROR, "Invalid SMBUS protocol! Protocol not supported.\n"); 3025 status = -NVL_BAD_ARGS; 3026 goto end; 3027 } 3028 } 3029 3030 if (!is_i2c_access_allowed && 3031 !nvswitch_i2c_is_device_access_allowed(device, port, deviceAddr, bIsRead)) 3032 { 3033 return -NVL_BAD_ARGS; 3034 } 3035 3036 if (msgLength > NVSWITCH_CTRL_I2C_MESSAGE_LENGTH_MAX) 3037 { 3038 NVSWITCH_PRINT(device, ERROR, 3039 "Length of buffer (0x%x bytes) provided larger than max (0x%x bytes)\n", 3040 msgLength, NVSWITCH_CTRL_I2C_MESSAGE_LENGTH_MAX); 3041 status = -NVL_BAD_ARGS; 3042 goto end; 3043 } 3044 3045 if (bIsRead) 3046 { 3047 i2c_params.bIsRead = NV_TRUE; 3048 } 3049 else 3050 { 3051 flags = FLD_SET_DRF(SWITCH_CTRL, _I2C_FLAGS, _RESTART, _NONE, flags); 3052 nvswitch_os_memcpy(i2c_params.message, pData, msgLength); 3053 } 3054 3055 if (FLD_TEST_DRF(SWITCH_CTRL, _I2C_FLAGS, _INDEX_LENGTH, _ONE, flags)) 3056 { 3057 i2c_params.index[0] = cmd; 3058 } 3059 3060 i2c_params.port = port; 3061 i2c_params.address = deviceAddr; 3062 i2c_params.acquirer = client; 3063 i2c_params.flags = flags; 3064 i2c_params.messageLength = msgLength; 3065 3066 status = nvswitch_ctrl_i2c_indexed(device, &i2c_params); 3067 if (status != NVL_SUCCESS) 3068 { 3069 NVSWITCH_PRINT(device, ERROR, "I2C transfer Failed!\n"); 3070 goto end; 3071 } 3072 3073 if (bIsRead) 3074 { 3075 nvswitch_os_memcpy(pData, i2c_params.message, msgLength); 3076 } 3077 3078 end: 3079 return status; 3080 } 3081 3082 /*! 3083 * @brief: Performs an I2C transaction. 3084 */ 3085 NvlStatus 3086 nvswitch_lib_i2c_transfer 3087 ( 3088 nvswitch_device *device, 3089 NvU32 port, 3090 NvU8 type, 3091 NvU8 addr, 3092 NvU8 command, 3093 NvU32 len, 3094 NvU8 *pData 3095 ) 3096 { 3097 NvlStatus status; 3098 NvU32 port_info; 3099 NvBool is_i2c_access_allowed; 3100 NvBool is_port_allowed; 3101 3102 if (!NVSWITCH_IS_DEVICE_VALID(device)) 3103 { 3104 NVSWITCH_ASSERT(0); 3105 return -NVL_ERR_INVALID_STATE; 3106 } 3107 3108 port_info = nvswitch_i2c_get_port_info(device, port); 3109 3110 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 3111 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 3112 NV_TRUE : NV_FALSE; 3113 is_port_allowed = is_i2c_access_allowed ? NV_TRUE : 3114 FLD_TEST_DRF(_I2C, _PORTINFO, _ACCESS_ALLOWED, _TRUE, 3115 port_info); 3116 3117 if (!is_port_allowed || 3118 !FLD_TEST_DRF(_I2C, _PORTINFO, _DEFINED, _PRESENT, port_info)) 3119 { 3120 NVSWITCH_PRINT(device, INFO, 3121 "%s: Invalid port access %d.\n", 3122 __FUNCTION__, port); 3123 return (-NVL_BAD_ARGS); 3124 } 3125 3126 status = _nvswitch_perform_i2c_transfer(device, NVSWITCH_I2C_ACQUIRER_EXTERNAL, 3127 type, (NvU16)addr, port, command, len, pData); 3128 if (status != NVL_SUCCESS) 3129 { 3130 NVSWITCH_PRINT(device, ERROR, "I2C transaction failed!\n"); 3131 return status; 3132 } 3133 3134 return NVL_SUCCESS; 3135 } 3136 3137 void 3138 nvswitch_timeout_create 3139 ( 3140 NvU64 timeout_ns, 3141 NVSWITCH_TIMEOUT *time 3142 ) 3143 { 3144 NvU64 time_current; 3145 3146 time_current = nvswitch_os_get_platform_time(); 3147 time->timeout_ns = time_current + timeout_ns; 3148 } 3149 3150 NvBool 3151 nvswitch_timeout_check 3152 ( 3153 NVSWITCH_TIMEOUT *time 3154 ) 3155 { 3156 NvU64 time_current; 3157 3158 time_current = nvswitch_os_get_platform_time(); 3159 return (time->timeout_ns <= time_current); 3160 } 3161 3162 NvlStatus 3163 nvswitch_task_create 3164 ( 3165 nvswitch_device *device, 3166 void (*task_fn)(nvswitch_device *device), 3167 NvU64 period_nsec, 3168 NvU32 flags 3169 ) 3170 { 3171 NVSWITCH_TASK_TYPE *task; 3172 task = nvswitch_os_malloc(sizeof(NVSWITCH_TASK_TYPE)); 3173 3174 if (task == NULL) 3175 { 3176 NVSWITCH_PRINT(device, ERROR, 3177 "%s: Unable to allocate task.\n", 3178 __FUNCTION__); 3179 return -NVL_NO_MEM; 3180 } 3181 else 3182 { 3183 task->task_fn_devptr = task_fn; 3184 task->task_args = NULL; 3185 task->period_nsec = period_nsec; 3186 task->last_run_nsec = nvswitch_os_get_platform_time(); // Prevent deferred tasks from being run immediately 3187 task->flags = flags; 3188 task->prev = NULL; 3189 task->next = device->tasks; 3190 if (device->tasks != NULL) 3191 { 3192 device->tasks->prev = task; 3193 } 3194 device->tasks = task; 3195 } 3196 3197 return NVL_SUCCESS; 3198 } 3199 3200 NvlStatus 3201 nvswitch_task_create_args 3202 ( 3203 nvswitch_device *device, 3204 void *fn_args, 3205 void (*task_fn)(nvswitch_device* device, void *fn_args), 3206 NvU64 period_nsec, 3207 NvU32 flags 3208 ) 3209 { 3210 NVSWITCH_TASK_TYPE *task; 3211 task = nvswitch_os_malloc(sizeof(NVSWITCH_TASK_TYPE)); 3212 3213 flags = flags | NVSWITCH_TASK_TYPE_FLAGS_VOID_PTR_ARGS; // ensure dispatcher always executes tasks passed through this function with args 3214 3215 if (task == NULL) 3216 { 3217 NVSWITCH_PRINT(device, ERROR, 3218 "%s: Unable to allocate task.\n", 3219 __FUNCTION__); 3220 return -NVL_NO_MEM; 3221 } 3222 else 3223 { 3224 task->task_fn_vdptr = task_fn; 3225 task->task_args = fn_args; 3226 task->period_nsec = period_nsec; 3227 task->last_run_nsec = nvswitch_os_get_platform_time(); // Prevent deferred tasks from being run immediately 3228 task->flags = flags; 3229 task->prev = NULL; 3230 task->next = device->tasks; 3231 if (device->tasks != NULL) 3232 { 3233 device->tasks->prev = task; 3234 } 3235 device->tasks = task; 3236 } 3237 3238 return NVL_SUCCESS; 3239 } 3240 3241 void 3242 nvswitch_tasks_destroy 3243 ( 3244 nvswitch_device *device 3245 ) 3246 { 3247 NVSWITCH_TASK_TYPE *task = device->tasks; 3248 NVSWITCH_TASK_TYPE *next_task; 3249 3250 device->tasks = NULL; 3251 3252 while (task) 3253 { 3254 next_task = task->next; 3255 nvswitch_os_free(task); 3256 task = next_task; 3257 } 3258 } 3259 3260 void 3261 nvswitch_destroy_device_state 3262 ( 3263 nvswitch_device *device 3264 ) 3265 { 3266 device->hal.nvswitch_destroy_device_state(device); 3267 } 3268 3269 static NvlStatus 3270 _nvswitch_ctrl_get_info 3271 ( 3272 nvswitch_device *device, 3273 NVSWITCH_GET_INFO *p 3274 ) 3275 { 3276 return device->hal.nvswitch_ctrl_get_info(device, p); 3277 } 3278 3279 static NvlStatus 3280 _nvswitch_ctrl_get_nvlink_status 3281 ( 3282 nvswitch_device *device, 3283 NVSWITCH_GET_NVLINK_STATUS_PARAMS *ret 3284 ) 3285 { 3286 return device->hal.nvswitch_ctrl_get_nvlink_status(device, ret); 3287 } 3288 3289 static NvlStatus 3290 _nvswitch_ctrl_get_counters 3291 ( 3292 nvswitch_device *device, 3293 NVSWITCH_NVLINK_GET_COUNTERS_PARAMS *ret 3294 ) 3295 { 3296 return device->hal.nvswitch_ctrl_get_counters(device, ret); 3297 } 3298 3299 NvlStatus 3300 nvswitch_set_nport_port_config 3301 ( 3302 nvswitch_device *device, 3303 NVSWITCH_SET_SWITCH_PORT_CONFIG *p 3304 ) 3305 { 3306 return device->hal.nvswitch_set_nport_port_config(device, p); 3307 } 3308 3309 static NvlStatus 3310 _nvswitch_ctrl_set_switch_port_config 3311 ( 3312 nvswitch_device *device, 3313 NVSWITCH_SET_SWITCH_PORT_CONFIG *p 3314 ) 3315 { 3316 return device->hal.nvswitch_ctrl_set_switch_port_config(device, p); 3317 } 3318 3319 static NvlStatus 3320 _nvswitch_ctrl_get_ingress_request_table 3321 ( 3322 nvswitch_device *device, 3323 NVSWITCH_GET_INGRESS_REQUEST_TABLE_PARAMS *params 3324 ) 3325 { 3326 return device->hal.nvswitch_ctrl_get_ingress_request_table(device, params); 3327 } 3328 3329 static NvlStatus 3330 _nvswitch_ctrl_set_ingress_request_table 3331 ( 3332 nvswitch_device *device, 3333 NVSWITCH_SET_INGRESS_REQUEST_TABLE *p 3334 ) 3335 { 3336 return device->hal.nvswitch_ctrl_set_ingress_request_table(device, p); 3337 } 3338 3339 static NvlStatus 3340 _nvswitch_ctrl_set_ingress_request_valid 3341 ( 3342 nvswitch_device *device, 3343 NVSWITCH_SET_INGRESS_REQUEST_VALID *p 3344 ) 3345 { 3346 return device->hal.nvswitch_ctrl_set_ingress_request_valid(device, p); 3347 } 3348 3349 static NvlStatus 3350 _nvswitch_ctrl_get_ingress_response_table 3351 ( 3352 nvswitch_device *device, 3353 NVSWITCH_GET_INGRESS_RESPONSE_TABLE_PARAMS *params 3354 ) 3355 { 3356 return device->hal.nvswitch_ctrl_get_ingress_response_table(device, params); 3357 } 3358 3359 static NvlStatus 3360 _nvswitch_ctrl_set_ingress_response_table 3361 ( 3362 nvswitch_device *device, 3363 NVSWITCH_SET_INGRESS_RESPONSE_TABLE *p 3364 ) 3365 { 3366 return device->hal.nvswitch_ctrl_set_ingress_response_table(device, p); 3367 } 3368 3369 static NvlStatus 3370 _nvswitch_ctrl_set_ganged_link_table 3371 ( 3372 nvswitch_device *device, 3373 NVSWITCH_SET_GANGED_LINK_TABLE *p 3374 ) 3375 { 3376 return device->hal.nvswitch_ctrl_set_ganged_link_table(device, p); 3377 } 3378 3379 void 3380 nvswitch_init_npg_multicast 3381 ( 3382 nvswitch_device *device 3383 ) 3384 { 3385 return device->hal.nvswitch_init_npg_multicast(device); 3386 } 3387 3388 void 3389 nvswitch_init_warm_reset 3390 ( 3391 nvswitch_device *device 3392 ) 3393 { 3394 return device->hal.nvswitch_init_warm_reset(device); 3395 } 3396 3397 static NvlStatus 3398 _nvswitch_ctrl_set_remap_policy 3399 ( 3400 nvswitch_device *device, 3401 NVSWITCH_SET_REMAP_POLICY *p 3402 ) 3403 { 3404 return device->hal.nvswitch_ctrl_set_remap_policy(device, p); 3405 } 3406 3407 static NvlStatus 3408 _nvswitch_ctrl_get_remap_policy 3409 ( 3410 nvswitch_device *device, 3411 NVSWITCH_GET_REMAP_POLICY_PARAMS *params 3412 ) 3413 { 3414 return device->hal.nvswitch_ctrl_get_remap_policy(device, params); 3415 } 3416 3417 static NvlStatus 3418 _nvswitch_ctrl_set_remap_policy_valid 3419 ( 3420 nvswitch_device *device, 3421 NVSWITCH_SET_REMAP_POLICY_VALID *p 3422 ) 3423 { 3424 return device->hal.nvswitch_ctrl_set_remap_policy_valid(device, p); 3425 } 3426 3427 static NvlStatus 3428 _nvswitch_ctrl_set_routing_id 3429 ( 3430 nvswitch_device *device, 3431 NVSWITCH_SET_ROUTING_ID *p 3432 ) 3433 { 3434 return device->hal.nvswitch_ctrl_set_routing_id(device, p); 3435 } 3436 3437 static NvlStatus 3438 _nvswitch_ctrl_get_routing_id 3439 ( 3440 nvswitch_device *device, 3441 NVSWITCH_GET_ROUTING_ID_PARAMS *params 3442 ) 3443 { 3444 return device->hal.nvswitch_ctrl_get_routing_id(device, params); 3445 } 3446 3447 static NvlStatus 3448 _nvswitch_ctrl_set_routing_id_valid 3449 ( 3450 nvswitch_device *device, 3451 NVSWITCH_SET_ROUTING_ID_VALID *p 3452 ) 3453 { 3454 return device->hal.nvswitch_ctrl_set_routing_id_valid(device, p); 3455 } 3456 3457 static NvlStatus 3458 _nvswitch_ctrl_set_routing_lan 3459 ( 3460 nvswitch_device *device, 3461 NVSWITCH_SET_ROUTING_LAN *p 3462 ) 3463 { 3464 return device->hal.nvswitch_ctrl_set_routing_lan(device, p); 3465 } 3466 3467 static NvlStatus 3468 _nvswitch_ctrl_get_routing_lan 3469 ( 3470 nvswitch_device *device, 3471 NVSWITCH_GET_ROUTING_LAN_PARAMS *params 3472 ) 3473 { 3474 return device->hal.nvswitch_ctrl_get_routing_lan(device, params); 3475 } 3476 3477 static NvlStatus 3478 _nvswitch_ctrl_set_routing_lan_valid 3479 ( 3480 nvswitch_device *device, 3481 NVSWITCH_SET_ROUTING_LAN_VALID *p 3482 ) 3483 { 3484 return device->hal.nvswitch_ctrl_set_routing_lan_valid(device, p); 3485 } 3486 3487 static NvlStatus 3488 _nvswitch_ctrl_get_internal_latency 3489 ( 3490 nvswitch_device *device, 3491 NVSWITCH_GET_INTERNAL_LATENCY *p 3492 ) 3493 { 3494 return device->hal.nvswitch_ctrl_get_internal_latency(device, p); 3495 } 3496 3497 static NvlStatus 3498 _nvswitch_ctrl_get_nvlipt_counters 3499 ( 3500 nvswitch_device *device, 3501 NVSWITCH_GET_NVLIPT_COUNTERS *p 3502 ) 3503 { 3504 // 3505 // This control call is now deprecated. 3506 // New control call to fetch throughput counters is: 3507 // nvswitch_ctrl_get_throughput_counters 3508 // 3509 return -NVL_ERR_NOT_SUPPORTED; 3510 } 3511 3512 static NvlStatus 3513 _nvswitch_ctrl_set_nvlipt_counter_config 3514 ( 3515 nvswitch_device *device, 3516 NVSWITCH_SET_NVLIPT_COUNTER_CONFIG *p 3517 ) 3518 { 3519 // 3520 // This control call is now deprecated. 3521 // New control call to fetch throughput counters is: 3522 // nvswitch_ctrl_get_throughput_counters_lr10 3523 // 3524 // Setting counter config is not allowed on these 3525 // non-configurable counters. These counters are 3526 // expected to be used by monitoring clients. 3527 // 3528 return -NVL_ERR_NOT_SUPPORTED; 3529 } 3530 3531 static NvlStatus 3532 _nvswitch_ctrl_get_nvlipt_counter_config 3533 ( 3534 nvswitch_device *device, 3535 NVSWITCH_GET_NVLIPT_COUNTER_CONFIG *p 3536 ) 3537 { 3538 // 3539 // This control call is now deprecated. 3540 // New control call to fetch throughput counters is: 3541 // nvswitch_ctrl_get_throughput_counters_lr10 3542 // 3543 // Getting counter config is useful if counters are 3544 // configurable. These counters are not configurable 3545 // and are expected to be used by monitoring clients. 3546 // 3547 return -NVL_ERR_NOT_SUPPORTED; 3548 } 3549 3550 static NvlStatus 3551 _nvswitch_ctrl_register_read 3552 ( 3553 nvswitch_device *device, 3554 NVSWITCH_REGISTER_READ *p 3555 ) 3556 { 3557 return device->hal.nvswitch_ctrl_register_read(device, p); 3558 } 3559 3560 static NvlStatus 3561 _nvswitch_ctrl_register_write 3562 ( 3563 nvswitch_device *device, 3564 NVSWITCH_REGISTER_WRITE *p 3565 ) 3566 { 3567 return device->hal.nvswitch_ctrl_register_write(device, p); 3568 } 3569 3570 NvU32 3571 nvswitch_i2c_get_port_info 3572 ( 3573 nvswitch_device *device, 3574 NvU32 port 3575 ) 3576 { 3577 return device->hal.nvswitch_i2c_get_port_info(device, port); 3578 } 3579 3580 NvlStatus 3581 nvswitch_ctrl_i2c_indexed 3582 ( 3583 nvswitch_device *device, 3584 NVSWITCH_CTRL_I2C_INDEXED_PARAMS *pParams 3585 ) 3586 { 3587 return device->hal.nvswitch_ctrl_i2c_indexed(device, pParams); 3588 } 3589 3590 static NvlStatus 3591 _nvswitch_ctrl_therm_read_temperature 3592 ( 3593 nvswitch_device *device, 3594 NVSWITCH_CTRL_GET_TEMPERATURE_PARAMS *info 3595 ) 3596 { 3597 return device->hal.nvswitch_ctrl_therm_read_temperature(device, info); 3598 } 3599 3600 static NvlStatus 3601 _nvswitch_ctrl_get_bios_info 3602 ( 3603 nvswitch_device *device, 3604 NVSWITCH_GET_BIOS_INFO_PARAMS *p 3605 ) 3606 { 3607 return device->hal.nvswitch_ctrl_get_bios_info(device, p); 3608 } 3609 3610 static NvlStatus 3611 _nvswitch_ctrl_get_inforom_version 3612 ( 3613 nvswitch_device *device, 3614 NVSWITCH_GET_INFOROM_VERSION_PARAMS *p 3615 ) 3616 { 3617 return device->hal.nvswitch_ctrl_get_inforom_version(device, p); 3618 } 3619 3620 NvlStatus 3621 nvswitch_ctrl_set_latency_bins 3622 ( 3623 nvswitch_device *device, 3624 NVSWITCH_SET_LATENCY_BINS *p 3625 ) 3626 { 3627 return device->hal.nvswitch_ctrl_set_latency_bins(device, p); 3628 } 3629 3630 static NvlStatus 3631 _nvswitch_ctrl_get_ingress_reqlinkid 3632 ( 3633 nvswitch_device *device, 3634 NVSWITCH_GET_INGRESS_REQLINKID_PARAMS *params 3635 ) 3636 { 3637 return device->hal.nvswitch_ctrl_get_ingress_reqlinkid(device, params); 3638 } 3639 3640 NvlStatus 3641 nvswitch_ctrl_get_throughput_counters 3642 ( 3643 nvswitch_device *device, 3644 NVSWITCH_GET_THROUGHPUT_COUNTERS_PARAMS *p 3645 ) 3646 { 3647 return device->hal.nvswitch_ctrl_get_throughput_counters(device, p); 3648 } 3649 3650 static NvlStatus 3651 _nvswitch_ctrl_unregister_link 3652 ( 3653 nvswitch_device *device, 3654 NVSWITCH_UNREGISTER_LINK_PARAMS *params 3655 ) 3656 { 3657 nvlink_link *link = nvswitch_get_link(device, (NvU8)params->portNum); 3658 3659 if (link == NULL) 3660 { 3661 return -NVL_BAD_ARGS; 3662 } 3663 3664 // With ALI in FW, links can be unregistered while Active 3665 if (!device->nvlink_device->enableALI) 3666 { 3667 3668 if (device->hal.nvswitch_is_link_in_use(device, params->portNum)) 3669 { 3670 return -NVL_ERR_STATE_IN_USE; 3671 } 3672 3673 } 3674 3675 nvlink_lib_unregister_link(link); 3676 nvswitch_destroy_link(link); 3677 3678 return NVL_SUCCESS; 3679 } 3680 3681 static NvlStatus 3682 _nvswitch_ctrl_acquire_capability 3683 ( 3684 nvswitch_device *device, 3685 NVSWITCH_ACQUIRE_CAPABILITY_PARAMS *params, 3686 void *osPrivate 3687 ) 3688 { 3689 return nvswitch_os_acquire_fabric_mgmt_cap(osPrivate, 3690 params->capDescriptor); 3691 } 3692 3693 static NvlStatus 3694 _nvswitch_ctrl_reset_and_drain_links 3695 ( 3696 nvswitch_device *device, 3697 NVSWITCH_RESET_AND_DRAIN_LINKS_PARAMS *params 3698 ) 3699 { 3700 return device->hal.nvswitch_reset_and_drain_links(device, params->linkMask, NV_FALSE); 3701 } 3702 3703 static NvlStatus 3704 _nvswitch_ctrl_get_fom_values 3705 ( 3706 nvswitch_device *device, 3707 NVSWITCH_GET_FOM_VALUES_PARAMS *ret 3708 ) 3709 { 3710 return device->hal.nvswitch_ctrl_get_fom_values(device, ret); 3711 } 3712 3713 static NvlStatus 3714 _nvswitch_ctrl_get_nvlink_ecc_errors 3715 ( 3716 nvswitch_device *device, 3717 NVSWITCH_GET_NVLINK_ECC_ERRORS_PARAMS *params 3718 ) 3719 { 3720 return device->hal.nvswitch_get_nvlink_ecc_errors(device, params); 3721 } 3722 3723 static NvlStatus 3724 _nvswitch_ctrl_set_mc_rid_table 3725 ( 3726 nvswitch_device *device, 3727 NVSWITCH_SET_MC_RID_TABLE_PARAMS *p 3728 ) 3729 { 3730 return device->hal.nvswitch_ctrl_set_mc_rid_table(device, p); 3731 } 3732 3733 static NvlStatus 3734 _nvswitch_ctrl_get_mc_rid_table 3735 ( 3736 nvswitch_device *device, 3737 NVSWITCH_GET_MC_RID_TABLE_PARAMS *p 3738 ) 3739 { 3740 return device->hal.nvswitch_ctrl_get_mc_rid_table(device, p); 3741 } 3742 3743 static NvlStatus 3744 _nvswitch_ctrl_set_residency_bins 3745 ( 3746 nvswitch_device *device, 3747 NVSWITCH_SET_RESIDENCY_BINS *p 3748 ) 3749 { 3750 return device->hal.nvswitch_ctrl_set_residency_bins(device, p); 3751 } 3752 3753 static NvlStatus 3754 _nvswitch_ctrl_get_residency_bins 3755 ( 3756 nvswitch_device *device, 3757 NVSWITCH_GET_RESIDENCY_BINS *p 3758 ) 3759 { 3760 return device->hal.nvswitch_ctrl_get_residency_bins(device, p); 3761 } 3762 3763 static NvlStatus 3764 _nvswitch_ctrl_get_rb_stall_busy 3765 ( 3766 nvswitch_device *device, 3767 NVSWITCH_GET_RB_STALL_BUSY *p 3768 ) 3769 { 3770 return device->hal.nvswitch_ctrl_get_rb_stall_busy(device, p); 3771 } 3772 3773 NvlStatus 3774 nvswitch_ctrl_get_multicast_id_error_vector 3775 ( 3776 nvswitch_device *device, 3777 NVSWITCH_GET_MULTICAST_ID_ERROR_VECTOR *p 3778 ) 3779 { 3780 return device->hal.nvswitch_ctrl_get_multicast_id_error_vector(device, p); 3781 } 3782 3783 NvlStatus 3784 nvswitch_ctrl_clear_multicast_id_error_vector 3785 ( 3786 nvswitch_device *device, 3787 NVSWITCH_CLEAR_MULTICAST_ID_ERROR_VECTOR *p 3788 ) 3789 { 3790 return device->hal.nvswitch_ctrl_clear_multicast_id_error_vector(device, p); 3791 } 3792 3793 static NvlStatus 3794 _nvswitch_ctrl_inband_send_data 3795 ( 3796 nvswitch_device *device, 3797 NVSWITCH_INBAND_SEND_DATA_PARAMS *p 3798 ) 3799 { 3800 return device->hal.nvswitch_ctrl_inband_send_data(device, p); 3801 } 3802 3803 static NvlStatus 3804 _nvswitch_ctrl_inband_read_data 3805 ( 3806 nvswitch_device *device, 3807 NVSWITCH_INBAND_READ_DATA_PARAMS *p 3808 ) 3809 { 3810 return device->hal.nvswitch_ctrl_inband_read_data(device, p); 3811 } 3812 3813 static NvlStatus 3814 _nvswitch_ctrl_inband_flush_data 3815 ( 3816 nvswitch_device *device, 3817 NVSWITCH_INBAND_FLUSH_DATA_PARAMS *p 3818 ) 3819 { 3820 NvU32 i; 3821 NvU64 enabledLinkMask; 3822 3823 if (p->linkMask == 0) 3824 { 3825 NVSWITCH_PRINT(device, ERROR, "Nothing to clear\n"); 3826 return NVL_SUCCESS; 3827 } 3828 3829 enabledLinkMask = nvswitch_get_enabled_link_mask(device); 3830 3831 FOR_EACH_INDEX_IN_MASK(64, i, p->linkMask) 3832 { 3833 if (nvswitch_is_link_valid(device, i) && 3834 (enabledLinkMask & NVBIT64(i))) 3835 { 3836 // 3837 // Flush is expected to clear both persistent and non-persistent 3838 // list. FM does flush when it wants to drop (ignore) all pending 3839 // messages w/o any NACKs. 3840 // 3841 _nvswitch_inband_clear_lists(device, i, 3842 NV_FALSE /* Nack */, 3843 NV_FALSE /* Non-persistent only */); 3844 } 3845 } 3846 FOR_EACH_INDEX_IN_MASK_END; 3847 3848 return NVL_SUCCESS; 3849 } 3850 3851 static NvlStatus 3852 _nvswitch_ctrl_inband_pending_data_stats 3853 ( 3854 nvswitch_device *device, 3855 NVSWITCH_INBAND_PENDING_DATA_STATS_PARAMS *p 3856 ) 3857 { 3858 NvU32 link_num; 3859 NvU64 enabledLinkMask, persistent_mask = 0, nonpersistent_mask = 0; 3860 3861 enabledLinkMask = nvswitch_get_enabled_link_mask(device); 3862 3863 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 3864 { 3865 if (nvswitch_is_link_valid(device, link_num) && 3866 (enabledLinkMask & NVBIT64(link_num))) 3867 { 3868 if (!nvListIsEmpty(&device->link[link_num].inbandData.persistent_list)) 3869 { 3870 persistent_mask |= NVBIT64(link_num); 3871 } 3872 3873 if (!nvListIsEmpty(&device->link[link_num].inbandData.nonpersistent_list)) 3874 { 3875 nonpersistent_mask |= NVBIT64(link_num); 3876 } 3877 } 3878 } 3879 3880 if (persistent_mask > 0) 3881 { 3882 p->linkMask = persistent_mask; 3883 } 3884 else 3885 { 3886 p->linkMask = nonpersistent_mask; 3887 } 3888 3889 return NVL_SUCCESS; 3890 } 3891 3892 static NvlStatus 3893 _nvswitch_ctrl_get_board_part_number 3894 ( 3895 nvswitch_device *device, 3896 NVSWITCH_GET_BOARD_PART_NUMBER_VECTOR *p 3897 ) 3898 { 3899 if (IS_RTLSIM(device) || IS_EMULATION(device) || IS_FMODEL(device)) 3900 { 3901 NVSWITCH_PRINT(device, INFO, 3902 "%s: Skipping retrieval of board part number on FSF\n", 3903 __FUNCTION__); 3904 3905 nvswitch_os_memset(p, 0, sizeof(NVSWITCH_GET_BOARD_PART_NUMBER_VECTOR)); 3906 3907 return NVL_SUCCESS; 3908 } 3909 else 3910 { 3911 if (!nvswitch_is_inforom_supported(device)) 3912 { 3913 NVSWITCH_PRINT(device, ERROR, "InfoROM is not supported\n"); 3914 return -NVL_ERR_NOT_SUPPORTED; 3915 } 3916 3917 return device->hal.nvswitch_ctrl_get_board_part_number(device, p); 3918 } 3919 } 3920 3921 static NvlStatus 3922 _nvswitch_ctrl_i2c_smbus_command 3923 ( 3924 nvswitch_device *device, 3925 NVSWITCH_I2C_SMBUS_COMMAND_PARAMS *pParams 3926 ) 3927 { 3928 NvU32 port_info; 3929 NvU32 port = pParams->port; 3930 NvU8 msgLen; 3931 NvU8 cmd; 3932 NvU16 addr; 3933 NvU8 cmdType; 3934 NvU8 *pData; 3935 NvBool is_i2c_access_allowed; 3936 NvBool is_port_allowed; 3937 3938 port_info = nvswitch_i2c_get_port_info(device, port); 3939 3940 is_i2c_access_allowed = (device->regkeys.i2c_access_control == 3941 NV_SWITCH_REGKEY_I2C_ACCESS_CONTROL_ENABLE) ? 3942 NV_TRUE : NV_FALSE; 3943 is_port_allowed = is_i2c_access_allowed ? NV_TRUE : 3944 FLD_TEST_DRF(_I2C, _PORTINFO, _ACCESS_ALLOWED, _TRUE, 3945 port_info); 3946 3947 if (!is_port_allowed || 3948 !FLD_TEST_DRF(_I2C, _PORTINFO, _DEFINED, _PRESENT, port_info)) 3949 { 3950 NVSWITCH_PRINT(device, ERROR, "Invalid port access %d.\n", port); 3951 return NVL_BAD_ARGS; 3952 } 3953 3954 addr = pParams->deviceAddr; 3955 3956 switch (pParams->cmdType) 3957 { 3958 case NVSWITCH_I2C_SMBUS_CMD_QUICK: 3959 { 3960 cmd = 0; 3961 msgLen = 0; 3962 cmdType = pParams->bRead ? 3963 NVSWITCH_I2C_CMD_SMBUS_QUICK_READ : 3964 NVSWITCH_I2C_CMD_SMBUS_QUICK_WRITE; 3965 pData = NULL; 3966 break; 3967 } 3968 case NVSWITCH_I2C_SMBUS_CMD_BYTE: 3969 { 3970 cmd = 0; 3971 msgLen = 1; 3972 cmdType = pParams->bRead ? 3973 NVSWITCH_I2C_CMD_READ : 3974 NVSWITCH_I2C_CMD_WRITE; 3975 pData = (NvU8 *)&pParams->transactionData.smbusByte.message; 3976 break; 3977 } 3978 case NVSWITCH_I2C_SMBUS_CMD_BYTE_DATA: 3979 { 3980 msgLen = 1; 3981 cmd = pParams->transactionData.smbusByteData.cmd; 3982 cmdType = pParams->bRead ? 3983 NVSWITCH_I2C_CMD_SMBUS_READ : 3984 NVSWITCH_I2C_CMD_SMBUS_WRITE; 3985 pData = (NvU8 *)&pParams->transactionData.smbusByteData.message; 3986 break; 3987 } 3988 case NVSWITCH_I2C_SMBUS_CMD_WORD_DATA: 3989 { 3990 msgLen = 2; 3991 cmd = pParams->transactionData.smbusWordData.cmd; 3992 cmdType = pParams->bRead ? 3993 NVSWITCH_I2C_CMD_SMBUS_READ : 3994 NVSWITCH_I2C_CMD_SMBUS_WRITE; 3995 pData = (NvU8 *)&pParams->transactionData.smbusWordData.message; 3996 break; 3997 } 3998 default: 3999 { 4000 NVSWITCH_PRINT(device, ERROR, "Invalid Smbus command: %d.\n", port); 4001 return NVL_BAD_ARGS; 4002 } 4003 } 4004 4005 return _nvswitch_perform_i2c_transfer(device, NVSWITCH_I2C_ACQUIRER_IOCTL, 4006 cmdType, addr, port, cmd, msgLen, pData); 4007 } 4008 4009 NvBool 4010 nvswitch_does_link_need_termination_enabled 4011 ( 4012 nvswitch_device *device, 4013 nvlink_link *link 4014 ) 4015 { 4016 return device->hal.nvswitch_does_link_need_termination_enabled(device, link); 4017 } 4018 4019 NvlStatus 4020 nvswitch_link_termination_setup 4021 ( 4022 nvswitch_device *device, 4023 nvlink_link* link 4024 ) 4025 { 4026 return device->hal.nvswitch_link_termination_setup(device, link); 4027 } 4028 4029 static NvlStatus 4030 _nvswitch_ctrl_cci_cmis_presence 4031 ( 4032 nvswitch_device *device, 4033 NVSWITCH_CCI_CMIS_PRESENCE_PARAMS *pParams 4034 ) 4035 { 4036 nvswitch_os_memset(pParams, 0, sizeof(NVSWITCH_CCI_CMIS_PRESENCE_PARAMS)); 4037 if (device->pCci != NULL) 4038 { 4039 (void)cciGetXcvrMask(device, &pParams->cagesMask, &pParams->modulesMask); 4040 } 4041 4042 // IOCTL will always succeed 4043 return NVL_SUCCESS; 4044 } 4045 4046 static NvlStatus 4047 _nvswitch_ctrl_cci_nvlink_mappings 4048 ( 4049 nvswitch_device *device, 4050 NVSWITCH_CCI_CMIS_NVLINK_MAPPING_PARAMS *pParams 4051 ) 4052 { 4053 if (device->pCci == NULL) 4054 { 4055 NVSWITCH_PRINT(device, ERROR, 4056 "%s: CCI not supported\n", 4057 __FUNCTION__); 4058 return -NVL_ERR_NOT_SUPPORTED; 4059 } 4060 4061 return cciGetCageMapping(device, pParams->cageIndex, &pParams->linkMask, &pParams->encodedValue); 4062 } 4063 4064 static NvlStatus 4065 _nvswitch_ctrl_cci_memory_access_read 4066 ( 4067 nvswitch_device *device, 4068 NVSWITCH_CCI_CMIS_MEMORY_ACCESS_READ_PARAMS *pParams 4069 ) 4070 { 4071 NvlStatus retVal; 4072 4073 if (device->pCci == NULL) 4074 { 4075 NVSWITCH_PRINT(device, ERROR, 4076 "%s: CCI not supported\n", 4077 __FUNCTION__); 4078 return -NVL_ERR_NOT_SUPPORTED; 4079 } 4080 4081 if (!cciCmisAccessTryLock(device, pParams->cageIndex)) 4082 { 4083 return -NVL_ERR_STATE_IN_USE; 4084 } 4085 4086 retVal = cciCmisRead(device, pParams->cageIndex, pParams->bank, 4087 pParams->page, pParams->address, pParams->count, 4088 pParams->data); 4089 4090 if (!pParams->bSequenceLock) 4091 { 4092 cciCmisAccessReleaseLock(device, pParams->cageIndex); 4093 } 4094 4095 return retVal; 4096 } 4097 4098 static NvlStatus 4099 _nvswitch_ctrl_cci_memory_access_write 4100 ( 4101 nvswitch_device *device, 4102 NVSWITCH_CCI_CMIS_MEMORY_ACCESS_WRITE_PARAMS *pParams 4103 ) 4104 { 4105 NvlStatus retVal; 4106 4107 if (device->pCci == NULL) 4108 { 4109 NVSWITCH_PRINT(device, ERROR, 4110 "%s: CCI not supported\n", 4111 __FUNCTION__); 4112 return -NVL_ERR_NOT_SUPPORTED; 4113 } 4114 4115 if (!cciCmisAccessTryLock(device, pParams->cageIndex)) 4116 { 4117 return -NVL_ERR_STATE_IN_USE; 4118 } 4119 4120 retVal = cciCmisWrite(device, pParams->cageIndex, pParams->bank, 4121 pParams->page, pParams->address, pParams->count, 4122 pParams->data); 4123 4124 if (!pParams->bSequenceLock) 4125 { 4126 cciCmisAccessReleaseLock(device, pParams->cageIndex); 4127 } 4128 4129 return retVal; 4130 } 4131 4132 static NvlStatus 4133 _nvswitch_ctrl_cci_cage_bezel_marking 4134 ( 4135 nvswitch_device *device, 4136 NVSWITCH_CCI_CMIS_CAGE_BEZEL_MARKING_PARAMS *pParams 4137 ) 4138 { 4139 if (device->pCci == NULL) 4140 { 4141 NVSWITCH_PRINT(device, ERROR, 4142 "%s: CCI not supported\n", 4143 __FUNCTION__); 4144 return -NVL_ERR_NOT_SUPPORTED; 4145 } 4146 4147 return cciCmisCageBezelMarking(device, pParams->cageIndex, pParams->bezelMarking); 4148 } 4149 4150 static NvlStatus 4151 _nvswitch_ctrl_get_soe_heartbeat 4152 ( 4153 nvswitch_device *device, 4154 NVSWITCH_GET_SOE_HEARTBEAT_PARAMS *pParams 4155 ) 4156 { 4157 return device->hal.nvswitch_ctrl_get_soe_heartbeat(device, pParams); 4158 } 4159 4160 static NvlStatus 4161 _nvswitch_ctrl_set_continuous_ali 4162 ( 4163 nvswitch_device *device, 4164 NVSWITCH_SET_CONTINUOUS_ALI_PARAMS *pParams 4165 ) 4166 { 4167 device->bModeContinuousALI = pParams->bEnable; 4168 NVSWITCH_PRINT(device, INFO, 4169 "%s: Continuous ALI 0x%x\n", 4170 __FUNCTION__, device->bModeContinuousALI); 4171 return NVL_SUCCESS; 4172 } 4173 4174 static NvlStatus 4175 _nvswitch_ctrl_request_ali 4176 ( 4177 nvswitch_device *device, 4178 NVSWITCH_REQUEST_ALI_PARAMS *pParams 4179 ) 4180 { 4181 nvlink_link *link; 4182 NvU64 linkStateTl; 4183 NvU64 enabledLinkMask; 4184 NvU8 linkId; 4185 4186 if (device->bModeContinuousALI) 4187 { 4188 return -NVL_ERR_INVALID_STATE; 4189 } 4190 4191 // Only process enabled links 4192 enabledLinkMask = nvswitch_get_enabled_link_mask(device); 4193 pParams->linkMaskTrain &= enabledLinkMask; 4194 4195 NVSWITCH_PRINT(device, INFO, 4196 "%s: ALI requested for links 0x%llx\n", 4197 __FUNCTION__, pParams->linkMaskTrain); 4198 4199 // Handle access links 4200 FOR_EACH_INDEX_IN_MASK(64, linkId, pParams->linkMaskTrain) 4201 { 4202 // Only directly launch ALI on non-CCI managed links 4203 if (cciIsLinkManaged(device, linkId)) 4204 { 4205 continue; 4206 } 4207 4208 link = nvswitch_get_link(device, linkId); 4209 if ((link == NULL) || 4210 !NVSWITCH_IS_LINK_ENG_VALID(device, linkId, NVLIPT_LNK) || 4211 (linkId >= NVSWITCH_NVLINK_MAX_LINKS) || 4212 (device->hal.nvswitch_corelib_get_tl_link_mode(link, &linkStateTl) != NVL_SUCCESS)) 4213 { 4214 continue; 4215 } 4216 4217 if (linkStateTl == NVLINK_LINKSTATE_ACTIVE_PENDING) 4218 { 4219 continue; 4220 } 4221 4222 // Forcibly reset and re-train access links 4223 device->hal.nvswitch_reset_and_drain_links(device, 4224 NVBIT64(linkId), NV_TRUE); 4225 } 4226 FOR_EACH_INDEX_IN_MASK_END; 4227 4228 // Ask CCI to handle trunk links 4229 nvswitch_ctrl_cci_request_ali(device, pParams); 4230 4231 return NVL_SUCCESS; 4232 } 4233 4234 static NvlStatus 4235 _nvswitch_ctrl_get_inforom_nvlink_max_correctable_error_rate 4236 ( 4237 nvswitch_device *device, 4238 NVSWITCH_GET_NVLINK_MAX_CORRECTABLE_ERROR_RATES_PARAMS *params 4239 ) 4240 { 4241 return nvswitch_inforom_nvlink_get_max_correctable_error_rate(device, params); 4242 } 4243 4244 static NvlStatus 4245 _nvswitch_ctrl_get_inforom_nvlink_errors 4246 ( 4247 nvswitch_device *device, 4248 NVSWITCH_GET_NVLINK_ERROR_COUNTS_PARAMS *params 4249 ) 4250 { 4251 return nvswitch_inforom_nvlink_get_errors(device, params); 4252 } 4253 4254 static NvlStatus 4255 _nvswitch_ctrl_get_inforom_ecc_errors 4256 ( 4257 nvswitch_device *device, 4258 NVSWITCH_GET_ECC_ERROR_COUNTS_PARAMS *params 4259 ) 4260 { 4261 return nvswitch_inforom_ecc_get_errors(device, params); 4262 } 4263 4264 static NvlStatus 4265 _nvswitch_ctrl_get_inforom_bbx_sxid 4266 ( 4267 nvswitch_device *device, 4268 NVSWITCH_GET_SXIDS_PARAMS *params 4269 ) 4270 { 4271 return nvswitch_inforom_bbx_get_sxid(device, params); 4272 } 4273 4274 static NvlStatus 4275 _nvswitch_ctrl_get_inforom_bbx_sys_info 4276 ( 4277 nvswitch_device *device, 4278 NVSWITCH_GET_SYS_INFO_PARAMS *params 4279 ) 4280 { 4281 return nvswitch_inforom_bbx_get_data(device, RM_SOE_IFR_BBX_GET_SYS_INFO, (void *)params); 4282 } 4283 4284 static NvlStatus 4285 _nvswitch_ctrl_get_inforom_bbx_time_info 4286 ( 4287 nvswitch_device *device, 4288 NVSWITCH_GET_TIME_INFO_PARAMS *params 4289 ) 4290 { 4291 return nvswitch_inforom_bbx_get_data(device, RM_SOE_IFR_BBX_GET_TIME_INFO, (void *)params); 4292 } 4293 4294 static NvlStatus 4295 _nvswitch_ctrl_get_inforom_bbx_temp_data 4296 ( 4297 nvswitch_device *device, 4298 NVSWITCH_GET_TEMP_DATA_PARAMS *params 4299 ) 4300 { 4301 return nvswitch_inforom_bbx_get_data(device, RM_SOE_IFR_BBX_GET_TEMP_DATA, (void *)params); 4302 } 4303 4304 static NvlStatus 4305 _nvswitch_ctrl_get_inforom_bbx_temp_samples 4306 ( 4307 nvswitch_device *device, 4308 NVSWITCH_GET_TEMP_SAMPLES_PARAMS *params 4309 ) 4310 { 4311 return nvswitch_inforom_bbx_get_data(device, RM_SOE_IFR_BBX_GET_TEMP_SAMPLES, (void *)params); 4312 } 4313 4314 static NvlStatus 4315 _nvswitch_ctrl_get_nvlink_lp_counters 4316 ( 4317 nvswitch_device *device, 4318 NVSWITCH_GET_NVLINK_LP_COUNTERS_PARAMS *params 4319 ) 4320 { 4321 return device->hal.nvswitch_ctrl_get_nvlink_lp_counters(device, params); 4322 } 4323 4324 static NvlStatus 4325 _nvswitch_ctrl_get_sw_info 4326 ( 4327 nvswitch_device *device, 4328 NVSWITCH_GET_SW_INFO_PARAMS *params 4329 ) 4330 { 4331 return device->hal.nvswitch_ctrl_get_sw_info(device, params); 4332 } 4333 4334 static NvlStatus 4335 _nvswitch_lib_validate_privileged_ctrl 4336 ( 4337 void *osPrivate, 4338 NvU64 flags 4339 ) 4340 { 4341 if (flags & NVSWITCH_DEV_CMD_CHECK_ADMIN) 4342 { 4343 if (nvswitch_os_is_admin()) 4344 { 4345 return NVL_SUCCESS; 4346 } 4347 } 4348 4349 if (flags & NVSWITCH_DEV_CMD_CHECK_FM) 4350 { 4351 if (nvswitch_os_is_fabric_manager(osPrivate)) 4352 { 4353 return NVL_SUCCESS; 4354 } 4355 } 4356 4357 return -NVL_ERR_INSUFFICIENT_PERMISSIONS; 4358 } 4359 4360 /* 4361 * @Brief : Copy the data from the persistant or nonpersistant list 4362 * 4363 * @Description : 4364 * 4365 * @param[in] device NvSwitch device to contain this link 4366 * @param[out] data Destination Data 4367 * @param[in] linkId link number of the link 4368 * @param[out] dataSize Size of data copied 4369 * 4370 * @returns NVL_SUCCESS if action succeeded, 4371 * -NVL_NOT_FOUND if link doesnt have data 4372 */ 4373 NvlStatus 4374 nvswitch_inband_read_data 4375 ( 4376 nvswitch_device *device, 4377 NvU8 *dest, 4378 NvU32 linkId, 4379 NvU32 *dataSize 4380 ) 4381 { 4382 nvswitch_inband_data_list *curr = NULL; 4383 NVListRec *list; 4384 4385 if (nvListIsEmpty(&device->link[linkId].inbandData.persistent_list) && 4386 nvListIsEmpty(&device->link[linkId].inbandData.nonpersistent_list)) 4387 { 4388 NVSWITCH_PRINT(device, ERROR, "%s: LinkId %d doesnt have any data to send\n", 4389 __FUNCTION__, linkId); 4390 *dataSize = 0; 4391 return -NVL_NOT_FOUND; 4392 } 4393 4394 list = nvListIsEmpty(&device->link[linkId].inbandData.persistent_list) ? 4395 &device->link[linkId].inbandData.nonpersistent_list : 4396 &device->link[linkId].inbandData.persistent_list; 4397 4398 nvListForEachEntry(curr, list, entry) 4399 { 4400 *dataSize = curr->dataSize; 4401 nvswitch_os_memcpy(dest, curr->data, curr->dataSize); 4402 nvListDel(&curr->entry); 4403 nvswitch_os_free(curr); 4404 break; 4405 } 4406 4407 return NVL_SUCCESS; 4408 } 4409 4410 /* 4411 * @Brief : Returns NV_TRUE if the given inband msg 4412 * needs to go to persistant list 4413 * 4414 * @Description : 4415 * 4416 * @param[in] device NvSwitch device to contain this link 4417 * @param[in] msghdr Header to the message 4418 * 4419 */ 4420 4421 static NvBool 4422 nvswitch_is_message_persistent 4423 ( 4424 nvswitch_device *device, 4425 nvlink_inband_msg_header_t *msghdr 4426 ) 4427 { 4428 // We expect only one message per received data 4429 switch(msghdr->type) 4430 { 4431 case NVLINK_INBAND_MSG_TYPE_MC_TEAM_RELEASE_REQ: 4432 return NV_TRUE; 4433 default: 4434 return NV_FALSE; 4435 } 4436 } 4437 4438 /* 4439 * @Brief : Moves the data into persistant or nonpersistant list 4440 * 4441 * @Description : 4442 * 4443 * @param[in] device NvSwitch device to contain this link 4444 * @param[in] linkId link number of the link 4445 * 4446 */ 4447 void 4448 nvswitch_filter_messages 4449 ( 4450 nvswitch_device *device, 4451 NvU32 linkId 4452 ) 4453 { 4454 NvlStatus status; 4455 nvlink_inband_msg_header_t *msghdr = NULL; 4456 nvswitch_inband_data_list *msg = device->link[linkId].inbandData.message; 4457 NvU8 *buffer = device->link[linkId].inbandData.message->data; 4458 NVSWITCH_DRIVER_FABRIC_STATE driver_fabric_state = 0; 4459 NvBool bSendNackOrDrop = NV_FALSE; 4460 4461 NVSWITCH_ASSERT(nvswitch_lib_read_fabric_state(device, NULL, NULL, 4462 &driver_fabric_state) == NVL_SUCCESS); 4463 4464 msghdr = (nvlink_inband_msg_header_t*)buffer; 4465 4466 if (nvswitch_is_message_persistent(device, msghdr)) 4467 { 4468 if (nvListCount(&device->link[linkId].inbandData.persistent_list) < 4469 device->hal.nvswitch_get_max_persistent_message_count(device)) 4470 { 4471 nvListAdd(&msg->entry, &device->link[linkId].inbandData.persistent_list); 4472 } 4473 else 4474 { 4475 bSendNackOrDrop = NV_TRUE; 4476 } 4477 } 4478 else 4479 { 4480 if (driver_fabric_state == NVSWITCH_DRIVER_FABRIC_STATE_CONFIGURED) 4481 { 4482 nvListAdd(&msg->entry, 4483 &device->link[linkId].inbandData.nonpersistent_list); 4484 } 4485 else 4486 { 4487 bSendNackOrDrop = NV_TRUE; 4488 } 4489 } 4490 4491 if (bSendNackOrDrop) 4492 { 4493 nvswitch_send_nack_or_drop(device, linkId, msghdr); 4494 nvswitch_os_free(msg); 4495 } 4496 else 4497 { 4498 status = nvswitch_lib_notify_client_events(device, 4499 NVSWITCH_DEVICE_EVENT_INBAND_DATA); 4500 if (status != NVL_SUCCESS) 4501 { 4502 NVSWITCH_PRINT(device, ERROR, "%s: Failed to notify INBAND_DATA event\n", 4503 __FUNCTION__); 4504 } 4505 } 4506 4507 device->link[linkId].inbandData.message = NULL; 4508 } 4509 4510 /* 4511 * @Brief : Constructs an NVS link struct with the given data 4512 * 4513 * @Description : 4514 * 4515 * @param[in] device NvSwitch device to contain this link 4516 * @param[in] link_num link number of the link 4517 * @param[out] link reference to store the created link into 4518 * 4519 * @returns NVL_SUCCESS if action succeeded, 4520 * -NVL_NO_MEM if memory allocation failed 4521 */ 4522 NvlStatus 4523 nvswitch_create_link 4524 ( 4525 nvswitch_device *device, 4526 NvU32 link_number, 4527 nvlink_link **link 4528 ) 4529 { 4530 NvlStatus retval = NVL_SUCCESS; 4531 nvlink_link *ret = NULL; 4532 LINK_INFO *link_info = NULL; 4533 NvU64 ac_coupled_mask; 4534 4535 NVSWITCH_ASSERT(nvswitch_get_num_links(device) <= NVSWITCH_MAX_NUM_LINKS); 4536 4537 ret = nvswitch_os_malloc(sizeof(*ret)); 4538 if (NULL == ret) 4539 { 4540 NVSWITCH_PRINT(device, ERROR, 4541 "nvswitch_os_malloc during link creation failed!\n"); 4542 retval = -NVL_NO_MEM; 4543 goto nvswitch_create_link_cleanup; 4544 } 4545 nvswitch_os_memset(ret, 0, sizeof(*ret)); 4546 4547 link_info = nvswitch_os_malloc(sizeof(*link_info)); 4548 if (NULL == link_info) 4549 { 4550 NVSWITCH_PRINT(device, ERROR, 4551 "nvswitch_os_malloc during link creation failed!\n"); 4552 retval = -NVL_NO_MEM; 4553 goto nvswitch_create_link_cleanup; 4554 } 4555 nvswitch_os_memset(link_info, 0, sizeof(*link_info)); 4556 nvswitch_os_snprintf(link_info->name, sizeof(link_info->name), NVSWITCH_LINK_NAME "%d", link_number); 4557 4558 ret->dev = device->nvlink_device; 4559 ret->linkName = link_info->name; 4560 ret->linkNumber = link_number; 4561 ret->state = NVLINK_LINKSTATE_OFF; 4562 ret->ac_coupled = NV_FALSE; 4563 ret->version = nvswitch_get_link_ip_version(device, link_number); 4564 4565 ac_coupled_mask = ((NvU64)device->regkeys.ac_coupled_mask2 << 32 | 4566 (NvU64)device->regkeys.ac_coupled_mask); 4567 4568 if (ac_coupled_mask) 4569 { 4570 if (ac_coupled_mask & NVBIT64(link_number)) 4571 { 4572 ret->ac_coupled = NV_TRUE; 4573 } 4574 } 4575 else if (device->firmware.nvlink.link_config_found) 4576 { 4577 if (device->firmware.nvlink.link_ac_coupled_mask & NVBIT64(link_number)) 4578 { 4579 ret->ac_coupled = NV_TRUE; 4580 } 4581 } 4582 4583 // Initialize NVLink corelib callbacks for switch 4584 nvswitch_get_link_handlers(&link_handlers); 4585 4586 ret->link_handlers = &link_handlers; 4587 4588 // 4589 // link_info is used to store private link information 4590 // 4591 4592 ret->link_info = link_info; 4593 4594 *link = ret; 4595 4596 return retval; 4597 4598 nvswitch_create_link_cleanup: 4599 if (NULL != ret) 4600 { 4601 nvswitch_os_free(ret); 4602 } 4603 if (NULL != link_info) 4604 { 4605 nvswitch_os_free(link_info); 4606 } 4607 4608 return retval; 4609 } 4610 4611 void 4612 nvswitch_destroy_link 4613 ( 4614 nvlink_link *link 4615 ) 4616 { 4617 if (NULL != link->link_info) 4618 { 4619 nvswitch_os_free(link->link_info); 4620 } 4621 4622 nvswitch_os_free(link); 4623 } 4624 4625 NvU32 4626 nvswitch_get_num_links 4627 ( 4628 nvswitch_device *device 4629 ) 4630 { 4631 return device->hal.nvswitch_get_num_links(device); 4632 } 4633 4634 NvBool 4635 nvswitch_is_link_valid 4636 ( 4637 nvswitch_device *device, 4638 NvU32 link_id 4639 ) 4640 { 4641 return device->hal.nvswitch_is_link_valid(device, link_id); 4642 } 4643 4644 nvlink_link* 4645 nvswitch_get_link(nvswitch_device *device, NvU8 link_id) 4646 { 4647 nvlink_link *link = NULL; 4648 4649 nvlink_lib_get_link(device->nvlink_device, link_id, &link); 4650 4651 return link; 4652 } 4653 4654 NvU64 4655 nvswitch_get_enabled_link_mask 4656 ( 4657 nvswitch_device *device 4658 ) 4659 { 4660 NvU64 ret; 4661 nvlink_link *link; 4662 NvU32 link_num; 4663 4664 ret = 0x0; 4665 4666 for (link_num = 0; link_num < nvswitch_get_num_links(device); link_num++) 4667 { 4668 if (nvlink_lib_get_link(device->nvlink_device, link_num, &link) == NVL_SUCCESS) 4669 { 4670 ret |= NVBIT64(link_num); 4671 } 4672 } 4673 4674 return ret; 4675 } 4676 4677 void 4678 nvswitch_set_fatal_error 4679 ( 4680 nvswitch_device *device, 4681 NvBool device_fatal, 4682 NvU32 link_id 4683 ) 4684 { 4685 device->hal.nvswitch_set_fatal_error(device, device_fatal, link_id); 4686 } 4687 4688 NvU32 4689 nvswitch_get_swap_clk_default 4690 ( 4691 nvswitch_device *device 4692 ) 4693 { 4694 return device->hal.nvswitch_get_swap_clk_default(device); 4695 } 4696 4697 NvU32 4698 nvswitch_get_latency_sample_interval_msec 4699 ( 4700 nvswitch_device *device 4701 ) 4702 { 4703 return device->hal.nvswitch_get_latency_sample_interval_msec(device); 4704 } 4705 4706 void 4707 nvswitch_internal_latency_bin_log 4708 ( 4709 nvswitch_device *device 4710 ) 4711 { 4712 device->hal.nvswitch_internal_latency_bin_log(device); 4713 } 4714 4715 void 4716 nvswitch_ecc_writeback_task 4717 ( 4718 nvswitch_device *device 4719 ) 4720 { 4721 device->hal.nvswitch_ecc_writeback_task(device); 4722 } 4723 4724 void 4725 nvswitch_monitor_thermal_alert 4726 ( 4727 nvswitch_device *device 4728 ) 4729 { 4730 device->hal.nvswitch_monitor_thermal_alert(device); 4731 } 4732 4733 void 4734 nvswitch_hw_counter_shutdown 4735 ( 4736 nvswitch_device *device 4737 ) 4738 { 4739 device->hal.nvswitch_hw_counter_shutdown(device); 4740 } 4741 4742 NvlStatus 4743 nvswitch_get_rom_info 4744 ( 4745 nvswitch_device *device, 4746 NVSWITCH_EEPROM_TYPE *eeprom 4747 ) 4748 { 4749 return device->hal.nvswitch_get_rom_info(device, eeprom); 4750 } 4751 4752 void 4753 nvswitch_lib_enable_interrupts 4754 ( 4755 nvswitch_device *device 4756 ) 4757 { 4758 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 4759 { 4760 NVSWITCH_ASSERT(0); 4761 return; 4762 } 4763 4764 device->hal.nvswitch_lib_enable_interrupts(device); 4765 } 4766 4767 void 4768 nvswitch_lib_disable_interrupts 4769 ( 4770 nvswitch_device *device 4771 ) 4772 { 4773 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 4774 { 4775 NVSWITCH_ASSERT(0); 4776 return; 4777 } 4778 4779 device->hal.nvswitch_lib_disable_interrupts(device); 4780 } 4781 4782 NvlStatus 4783 nvswitch_lib_check_interrupts 4784 ( 4785 nvswitch_device *device 4786 ) 4787 { 4788 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 4789 { 4790 return -NVL_BAD_ARGS; 4791 } 4792 4793 return device->hal.nvswitch_lib_check_interrupts(device); 4794 } 4795 4796 NvlStatus 4797 nvswitch_lib_service_interrupts 4798 ( 4799 nvswitch_device *device 4800 ) 4801 { 4802 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device)) 4803 { 4804 return -NVL_BAD_ARGS; 4805 } 4806 4807 return device->hal.nvswitch_lib_service_interrupts(device); 4808 } 4809 4810 NvU64 4811 nvswitch_hw_counter_read_counter 4812 ( 4813 nvswitch_device *device 4814 ) 4815 { 4816 return device->hal.nvswitch_hw_counter_read_counter(device); 4817 } 4818 4819 NvU32 4820 nvswitch_get_link_ip_version 4821 ( 4822 nvswitch_device *device, 4823 NvU32 link_id 4824 ) 4825 { 4826 return device->hal.nvswitch_get_link_ip_version(device, link_id); 4827 } 4828 4829 NvU32 4830 nvswitch_reg_read_32 4831 ( 4832 nvswitch_device *device, 4833 NvU32 offset 4834 ) 4835 { 4836 NvU32 val; 4837 4838 if (device->nvlink_device->pciInfo.bars[0].pBar == NULL) 4839 { 4840 NVSWITCH_PRINT(device, ERROR, 4841 "register read failed at offset 0x%x\n", offset); 4842 4843 return 0xFFFFFFFF; 4844 } 4845 4846 val = nvswitch_os_mem_read32((NvU8 *)device->nvlink_device->pciInfo.bars[0].pBar + offset); 4847 4848 if ((val & 0xFFFF0000) == 0xBADF0000) 4849 { 4850 NvU32 boot_0; 4851 NVSWITCH_PRINT(device, WARN, 4852 "Potential IO failure reading 0x%x (0x%x)\n", offset, val); 4853 boot_0 = nvswitch_os_mem_read32((NvU8 *)device->nvlink_device->pciInfo.bars[0].pBar + 0x0); 4854 4855 if ((boot_0 & 0xFFFF0000) == 0xBADF0000) 4856 { 4857 NVSWITCH_PRINT_SXID(device, NVSWITCH_ERR_HW_HOST_IO_FAILURE, 4858 "IO failure\n"); 4859 NVSWITCH_PRINT(device, ERROR, 4860 "IO failure reading 0x%x (0x%x)\n", offset, val); 4861 } 4862 } 4863 4864 #ifdef _VERBOSE_REG_ACCESS 4865 NVSWITCH_PRINT(device, SETUP, 4866 "NVSWITCH read 0x%6x+%6x = 0x%08x\n", 4867 device->nvlink_device->pciInfo.bars[0].baseAddr, offset, val); 4868 #endif 4869 4870 return val; 4871 } 4872 4873 void 4874 nvswitch_reg_write_32 4875 ( 4876 nvswitch_device *device, 4877 NvU32 offset, 4878 NvU32 data 4879 ) 4880 { 4881 if (device->nvlink_device->pciInfo.bars[0].pBar == NULL) 4882 { 4883 NVSWITCH_PRINT(device, ERROR, 4884 "register write failed at offset 0x%x\n", offset); 4885 4886 return; 4887 } 4888 4889 #ifdef _VERBOSE_REG_ACCESS 4890 NVSWITCH_PRINT(device, SETUP, 4891 "NVSWITCH write 0x%6x+%6x = 0x%08x\n", 4892 device->nvlink_device->pciInfo.bars[0].baseAddr, offset, data); 4893 #endif 4894 4895 // Write the register 4896 nvswitch_os_mem_write32((NvU8 *)device->nvlink_device->pciInfo.bars[0].pBar + offset, data); 4897 4898 return; 4899 } 4900 4901 NvU64 4902 nvswitch_read_64bit_counter 4903 ( 4904 nvswitch_device *device, 4905 NvU32 lo_offset, 4906 NvU32 hi_offset 4907 ) 4908 { 4909 NvU32 hi0; 4910 NvU32 hi1; 4911 NvU32 lo; 4912 4913 hi0 = nvswitch_reg_read_32(device, hi_offset); 4914 do 4915 { 4916 hi1 = hi0; 4917 lo = nvswitch_reg_read_32(device, lo_offset); 4918 hi0 = nvswitch_reg_read_32(device, hi_offset); 4919 } while (hi0 != hi1); 4920 4921 return (lo | ((NvU64)hi0 << 32)); 4922 } 4923 4924 NvlStatus 4925 nvswitch_validate_pll_config 4926 ( 4927 nvswitch_device *device, 4928 NVSWITCH_PLL_INFO *switch_pll, 4929 NVSWITCH_PLL_LIMITS default_pll_limits 4930 ) 4931 { 4932 NvU32 update_rate_khz; 4933 NvU32 vco_freq_khz; 4934 NVSWITCH_PLL_LIMITS pll_limits; 4935 4936 NVSWITCH_PRINT(device, SETUP, 4937 "%s: Validating PLL: %dkHz * %d / (%d * %d * (1 << %d))\n", 4938 __FUNCTION__, 4939 switch_pll->src_freq_khz, 4940 switch_pll->N, 4941 switch_pll->M, 4942 switch_pll->PL, 4943 switch_pll->dist_mode); 4944 4945 // 4946 // These parameters could come from schmoo'ing API, settings file or a ROM. 4947 // For now, hard code with POR. 4948 // 4949 if (device->firmware.firmware_size > 0 && 4950 device->firmware.clocks.clocks_found && 4951 device->firmware.clocks.sys_pll.valid) 4952 { 4953 pll_limits = device->firmware.clocks.sys_pll; 4954 } 4955 else 4956 { 4957 pll_limits = default_pll_limits; 4958 } 4959 4960 NVSWITCH_ASSERT(switch_pll->M != 0); 4961 NVSWITCH_ASSERT(switch_pll->PL != 0); 4962 4963 if ((switch_pll->src_freq_khz < pll_limits.ref_min_mhz * 1000) || 4964 (switch_pll->src_freq_khz > pll_limits.ref_max_mhz * 1000)) 4965 { 4966 NVSWITCH_PRINT(device, ERROR, 4967 "%s: ERROR: Ref(%d) out-of-range\n", 4968 __FUNCTION__, 4969 switch_pll->src_freq_khz); 4970 return -NVL_ERR_INVALID_STATE; 4971 } 4972 4973 if ((switch_pll->M < pll_limits.m_min) || 4974 (switch_pll->M > pll_limits.m_max)) 4975 { 4976 NVSWITCH_PRINT(device, ERROR, 4977 "%s: ERROR: M(%d) out-of-range\n", 4978 __FUNCTION__, 4979 switch_pll->M); 4980 return -NVL_ERR_INVALID_STATE; 4981 } 4982 4983 if ((switch_pll->N < pll_limits.n_min) || 4984 (switch_pll->N > pll_limits.n_max)) 4985 { 4986 NVSWITCH_PRINT(device, ERROR, 4987 "%s: ERROR: N(%d) out-of-range\n", 4988 __FUNCTION__, 4989 switch_pll->N); 4990 return -NVL_ERR_INVALID_STATE; 4991 } 4992 4993 if ((switch_pll->PL < pll_limits.pl_min) || 4994 (switch_pll->PL > pll_limits.pl_max)) 4995 { 4996 NVSWITCH_PRINT(device, ERROR, 4997 "%s: ERROR: PL(%d) out-of-range\n", 4998 __FUNCTION__, 4999 switch_pll->PL); 5000 return -NVL_ERR_INVALID_STATE; 5001 } 5002 5003 vco_freq_khz = switch_pll->src_freq_khz * switch_pll->N 5004 / switch_pll->M; 5005 if ((vco_freq_khz < pll_limits.vco_min_mhz * 1000) || 5006 (vco_freq_khz > pll_limits.vco_max_mhz * 1000)) 5007 { 5008 NVSWITCH_PRINT(device, ERROR, 5009 "%s: ERROR: VCO(%d) freq out-of-range\n", 5010 __FUNCTION__, 5011 vco_freq_khz); 5012 return -NVL_ERR_INVALID_STATE; 5013 } 5014 5015 update_rate_khz = switch_pll->src_freq_khz / switch_pll->M; 5016 if ((update_rate_khz < pll_limits.update_min_mhz * 1000) || 5017 (update_rate_khz > pll_limits.update_max_mhz * 1000)) 5018 { 5019 NVSWITCH_PRINT(device, ERROR, 5020 "%s: ERROR: update rate(%d) out-of-range\n", 5021 __FUNCTION__, 5022 update_rate_khz); 5023 return -NVL_ERR_INVALID_STATE; 5024 } 5025 5026 switch_pll->vco_freq_khz = vco_freq_khz; 5027 5028 switch_pll->freq_khz = 5029 switch_pll->src_freq_khz * switch_pll->N / 5030 (switch_pll->M * switch_pll->PL * (1 << switch_pll->dist_mode)); 5031 5032 NVSWITCH_PRINT(device, SETUP, 5033 "%s: Validated PLL: %dkHz * %d / (%d * %d * (1 << %d)) = %dkHz\n", 5034 __FUNCTION__, 5035 switch_pll->src_freq_khz, 5036 switch_pll->N, 5037 switch_pll->M, 5038 switch_pll->PL, 5039 switch_pll->dist_mode, 5040 switch_pll->freq_khz); 5041 5042 return NVL_SUCCESS; 5043 } 5044 5045 NvlStatus 5046 nvswitch_init_pll_config 5047 ( 5048 nvswitch_device *device 5049 ) 5050 { 5051 return device->hal.nvswitch_init_pll_config(device); 5052 } 5053 5054 NvlStatus 5055 nvswitch_init_pll 5056 ( 5057 nvswitch_device *device 5058 ) 5059 { 5060 return device->hal.nvswitch_init_pll(device); 5061 } 5062 5063 void 5064 nvswitch_init_clock_gating 5065 ( 5066 nvswitch_device *device 5067 ) 5068 { 5069 return device->hal.nvswitch_init_clock_gating(device); 5070 } 5071 5072 void 5073 nvswitch_lib_get_uuid 5074 ( 5075 nvswitch_device *device, 5076 NvUuid *uuid 5077 ) 5078 { 5079 if (!NVSWITCH_IS_DEVICE_INITIALIZED(device) || (uuid == NULL)) 5080 { 5081 return; 5082 } 5083 5084 nvswitch_os_memcpy(uuid, &device->uuid, sizeof(device->uuid)); 5085 } 5086 5087 NvlStatus 5088 nvswitch_lib_get_physid 5089 ( 5090 nvswitch_device *device, 5091 NvU32 *phys_id 5092 ) 5093 { 5094 NVSWITCH_GET_INFO get_info; 5095 NvlStatus ret; 5096 5097 if (phys_id == NULL || !NVSWITCH_IS_DEVICE_ACCESSIBLE(device)) 5098 { 5099 return -NVL_BAD_ARGS; 5100 } 5101 5102 get_info.count=1; 5103 get_info.index[0] = NVSWITCH_GET_INFO_INDEX_PHYSICAL_ID; 5104 5105 ret = _nvswitch_ctrl_get_info(device, &get_info); 5106 if (ret != NVL_SUCCESS) 5107 { 5108 NVSWITCH_PRINT(device, ERROR, 5109 "Failed to get physical ID\n"); 5110 return ret; 5111 } 5112 5113 *phys_id = get_info.info[0]; 5114 5115 return NVL_SUCCESS; 5116 } 5117 5118 void 5119 nvswitch_i2c_set_hw_speed_mode 5120 ( 5121 nvswitch_device *device, 5122 NvU32 port, 5123 NvU32 speedMode 5124 ) 5125 { 5126 device->hal.nvswitch_i2c_set_hw_speed_mode(device, port, speedMode); 5127 return; 5128 } 5129 5130 void 5131 nvswitch_lib_smbpbi_log_sxid 5132 ( 5133 nvswitch_device *device, 5134 NvU32 sxid, 5135 const char *pFormat, 5136 ... 5137 ) 5138 { 5139 va_list arglist; 5140 int msglen; 5141 char string[RM_SOE_SMBPBI_CMD_LOG_MESSAGE_MAX_STRING + 1]; 5142 5143 nvswitch_os_memset(string, 0, (NvLength)sizeof(string)); 5144 5145 va_start(arglist, pFormat); 5146 msglen = nvswitch_os_vsnprintf(string, sizeof(string), pFormat, arglist); 5147 va_end(arglist); 5148 5149 if (!(msglen < 0)) 5150 { 5151 // 5152 // HALs will know that the string is being truncated by seeing that the 5153 // last byte in the buffer is not nul. 5154 // 5155 msglen = NV_MIN(msglen + 1, (int)RM_SOE_SMBPBI_CMD_LOG_MESSAGE_MAX_STRING); 5156 device->hal.nvswitch_smbpbi_log_message(device, sxid, msglen, (NvU8 *) string); 5157 } 5158 } 5159 5160 NvlStatus 5161 nvswitch_set_minion_initialized 5162 ( 5163 nvswitch_device *device, 5164 NvU32 idx_minion, 5165 NvBool initialized 5166 ) 5167 { 5168 return device->hal.nvswitch_set_minion_initialized(device, idx_minion, initialized); 5169 } 5170 5171 NvBool 5172 nvswitch_is_minion_initialized 5173 ( 5174 nvswitch_device *device, 5175 NvU32 idx_minion 5176 ) 5177 { 5178 return device->hal.nvswitch_is_minion_initialized(device, idx_minion); 5179 } 5180 5181 NvlStatus 5182 nvswitch_device_discovery 5183 ( 5184 nvswitch_device *device, 5185 NvU32 discovery_offset 5186 ) 5187 { 5188 return device->hal.nvswitch_device_discovery(device, discovery_offset); 5189 } 5190 5191 void 5192 nvswitch_filter_discovery 5193 ( 5194 nvswitch_device *device 5195 ) 5196 { 5197 device->hal.nvswitch_filter_discovery(device); 5198 } 5199 5200 NvlStatus 5201 nvswitch_process_discovery 5202 ( 5203 nvswitch_device *device 5204 ) 5205 { 5206 return device->hal.nvswitch_process_discovery(device); 5207 } 5208 5209 NvlStatus 5210 nvswitch_init_minion 5211 ( 5212 nvswitch_device *device 5213 ) 5214 { 5215 return device->hal.nvswitch_init_minion(device); 5216 } 5217 5218 NvU32 5219 nvswitch_get_link_eng_inst 5220 ( 5221 nvswitch_device *device, 5222 NvU32 link_id, 5223 NVSWITCH_ENGINE_ID eng_id 5224 ) 5225 { 5226 return device->hal.nvswitch_get_link_eng_inst(device, link_id, eng_id); 5227 } 5228 5229 void * 5230 nvswitch_alloc_chipdevice 5231 ( 5232 nvswitch_device *device 5233 ) 5234 { 5235 return(device->hal.nvswitch_alloc_chipdevice(device)); 5236 } 5237 5238 void 5239 nvswitch_free_chipdevice 5240 ( 5241 nvswitch_device *device 5242 ) 5243 { 5244 if (device->chip_device) 5245 { 5246 nvswitch_os_free(device->chip_device); 5247 device->chip_device = NULL; 5248 } 5249 } 5250 5251 NvlStatus 5252 nvswitch_init_thermal 5253 ( 5254 nvswitch_device *device 5255 ) 5256 { 5257 return(device->hal.nvswitch_init_thermal(device)); 5258 } 5259 5260 NvU32 5261 nvswitch_read_physical_id 5262 ( 5263 nvswitch_device *device 5264 ) 5265 { 5266 return(device->hal.nvswitch_read_physical_id(device)); 5267 } 5268 5269 NvU32 5270 nvswitch_get_caps_nvlink_version 5271 ( 5272 nvswitch_device *device 5273 ) 5274 { 5275 return(device->hal.nvswitch_get_caps_nvlink_version(device)); 5276 } 5277 5278 void 5279 nvswitch_initialize_interrupt_tree 5280 ( 5281 nvswitch_device *device 5282 ) 5283 { 5284 device->hal.nvswitch_initialize_interrupt_tree(device); 5285 } 5286 5287 void 5288 nvswitch_init_dlpl_interrupts 5289 ( 5290 nvlink_link *link 5291 ) 5292 { 5293 nvswitch_device *device = link->dev->pDevInfo; 5294 5295 device->hal.nvswitch_init_dlpl_interrupts(link); 5296 } 5297 5298 NvlStatus 5299 nvswitch_initialize_pmgr 5300 ( 5301 nvswitch_device *device 5302 ) 5303 { 5304 return(device->hal.nvswitch_initialize_pmgr(device)); 5305 } 5306 5307 NvlStatus 5308 nvswitch_initialize_ip_wrappers 5309 ( 5310 nvswitch_device *device 5311 ) 5312 { 5313 return(device->hal.nvswitch_initialize_ip_wrappers(device)); 5314 } 5315 5316 NvlStatus 5317 nvswitch_initialize_route 5318 ( 5319 nvswitch_device *device 5320 ) 5321 { 5322 return(device->hal.nvswitch_initialize_route(device)); 5323 } 5324 5325 void 5326 nvswitch_soe_unregister_events 5327 ( 5328 nvswitch_device *device 5329 ) 5330 { 5331 device->hal.nvswitch_soe_unregister_events(device); 5332 } 5333 5334 NvlStatus 5335 nvswitch_soe_register_event_callbacks 5336 ( 5337 nvswitch_device *device 5338 ) 5339 { 5340 return device->hal.nvswitch_soe_register_event_callbacks(device); 5341 } 5342 5343 NVSWITCH_BIOS_NVLINK_CONFIG * 5344 nvswitch_get_bios_nvlink_config 5345 ( 5346 nvswitch_device *device 5347 ) 5348 { 5349 return(device->hal.nvswitch_get_bios_nvlink_config(device)); 5350 } 5351 5352 NvlStatus 5353 nvswitch_minion_send_command 5354 ( 5355 nvswitch_device *device, 5356 NvU32 linkNumber, 5357 NvU32 command, 5358 NvU32 scratch0 5359 ) 5360 { 5361 return(device->hal.nvswitch_minion_send_command(device, linkNumber, 5362 command, scratch0)); 5363 } 5364 5365 NvlStatus 5366 nvswitch_init_nport 5367 ( 5368 nvswitch_device *device 5369 ) 5370 { 5371 return device->hal.nvswitch_init_nport(device); 5372 } 5373 5374 NvlStatus 5375 nvswitch_init_nxbar 5376 ( 5377 nvswitch_device *device 5378 ) 5379 { 5380 return device->hal.nvswitch_init_nxbar(device); 5381 } 5382 5383 NvlStatus 5384 nvswitch_clear_nport_rams 5385 ( 5386 nvswitch_device *device 5387 ) 5388 { 5389 return device->hal.nvswitch_clear_nport_rams(device); 5390 } 5391 5392 NvlStatus 5393 nvswitch_pri_ring_init 5394 ( 5395 nvswitch_device *device 5396 ) 5397 { 5398 return(device->hal.nvswitch_pri_ring_init(device)); 5399 } 5400 5401 NvlStatus 5402 nvswitch_get_remap_table_selector 5403 ( 5404 nvswitch_device *device, 5405 NVSWITCH_TABLE_SELECT_REMAP table_selector, 5406 NvU32 *remap_ram_sel 5407 ) 5408 { 5409 return device->hal.nvswitch_get_remap_table_selector(device, table_selector, remap_ram_sel); 5410 } 5411 5412 NvU32 5413 nvswitch_get_ingress_ram_size 5414 ( 5415 nvswitch_device *device, 5416 NvU32 ingress_ram_selector // NV_INGRESS_REQRSPMAPADDR_RAM_ADDRESS_* 5417 ) 5418 { 5419 return device->hal.nvswitch_get_ingress_ram_size(device, ingress_ram_selector); 5420 } 5421 5422 NvlStatus 5423 nvswitch_minion_get_dl_status 5424 ( 5425 nvswitch_device *device, 5426 NvU32 linkId, 5427 NvU32 statusIdx, 5428 NvU32 statusArgs, 5429 NvU32 *statusData 5430 ) 5431 { 5432 return device->hal.nvswitch_minion_get_dl_status(device, linkId, statusIdx, statusArgs, statusData); 5433 } 5434 5435 NvBool 5436 nvswitch_is_i2c_supported 5437 ( 5438 nvswitch_device *device 5439 ) 5440 { 5441 return device->hal.nvswitch_is_i2c_supported(device); 5442 } 5443 5444 5445 NvlStatus 5446 nvswitch_poll_sublink_state 5447 ( 5448 nvswitch_device *device, 5449 nvlink_link *link 5450 ) 5451 { 5452 return device->hal.nvswitch_poll_sublink_state(device, link); 5453 } 5454 5455 void 5456 nvswitch_setup_link_loopback_mode 5457 ( 5458 nvswitch_device *device, 5459 NvU32 linkNumber 5460 ) 5461 { 5462 return device->hal.nvswitch_setup_link_loopback_mode(device, linkNumber); 5463 } 5464 5465 void 5466 nvswitch_reset_persistent_link_hw_state 5467 ( 5468 nvswitch_device *device, 5469 NvU32 linkNumber 5470 ) 5471 { 5472 return device->hal.nvswitch_reset_persistent_link_hw_state(device, linkNumber); 5473 } 5474 5475 void 5476 nvswitch_store_topology_information 5477 ( 5478 nvswitch_device *device, 5479 nvlink_link *link 5480 ) 5481 { 5482 return device->hal.nvswitch_store_topology_information(device, link); 5483 } 5484 5485 void 5486 nvswitch_init_lpwr_regs 5487 ( 5488 nvlink_link *link 5489 ) 5490 { 5491 nvswitch_device *device = link->dev->pDevInfo; 5492 device->hal.nvswitch_init_lpwr_regs(link); 5493 } 5494 5495 void 5496 nvswitch_program_l1_scratch_reg 5497 ( 5498 nvswitch_device *device, 5499 NvU32 linkNumber 5500 ) 5501 { 5502 device->hal.nvswitch_program_l1_scratch_reg(device, linkNumber); 5503 } 5504 5505 NvlStatus 5506 nvswitch_check_io_sanity 5507 ( 5508 nvswitch_device *device 5509 ) 5510 { 5511 return device->hal.nvswitch_check_io_sanity(device); 5512 } 5513 5514 NvlStatus 5515 nvswitch_launch_ALI 5516 ( 5517 nvswitch_device *device 5518 ) 5519 { 5520 return device->hal.nvswitch_launch_ALI(device); 5521 } 5522 5523 NvlStatus 5524 nvswitch_set_training_mode 5525 ( 5526 nvswitch_device *device 5527 ) 5528 { 5529 return device->hal.nvswitch_set_training_mode(device); 5530 } 5531 5532 NvBool 5533 nvswitch_is_link_in_reset 5534 ( 5535 nvswitch_device *device, 5536 nvlink_link *link 5537 ) 5538 { 5539 return device->hal.nvswitch_is_link_in_reset(device, link); 5540 } 5541 5542 NvBool 5543 nvswitch_i2c_is_device_access_allowed 5544 ( 5545 nvswitch_device *device, 5546 NvU32 port, 5547 NvU8 addr, 5548 NvBool bIsRead 5549 ) 5550 { 5551 return device->hal.nvswitch_i2c_is_device_access_allowed(device, port, addr, bIsRead); 5552 } 5553 5554 NvlStatus 5555 nvswitch_parse_bios_image 5556 ( 5557 nvswitch_device *device 5558 ) 5559 { 5560 return device->hal.nvswitch_parse_bios_image(device); 5561 } 5562 5563 void 5564 nvswitch_init_buffer_ready 5565 ( 5566 nvswitch_device *device, 5567 nvlink_link *link, 5568 NvBool bNportBufferReady 5569 ) 5570 { 5571 return device->hal.nvswitch_init_buffer_ready(device, link, bNportBufferReady); 5572 } 5573 5574 void 5575 nvswitch_apply_recal_settings 5576 ( 5577 nvswitch_device *device, 5578 nvlink_link *link 5579 ) 5580 { 5581 return device->hal.nvswitch_apply_recal_settings(device, link); 5582 } 5583 5584 NvlStatus 5585 nvswitch_launch_ALI_link_training 5586 ( 5587 nvswitch_device *device, 5588 nvlink_link *link, 5589 NvBool bSync 5590 ) 5591 { 5592 return device->hal.nvswitch_launch_ALI_link_training(device, link, bSync); 5593 } 5594 5595 NvlStatus 5596 nvswitch_reset_and_train_link 5597 ( 5598 nvswitch_device *device, 5599 nvlink_link *link 5600 ) 5601 { 5602 return device->hal.nvswitch_reset_and_train_link(device, link); 5603 } 5604 5605 static NvlStatus 5606 _nvswitch_ctrl_get_err_info 5607 ( 5608 nvswitch_device *device, 5609 NVSWITCH_NVLINK_GET_ERR_INFO_PARAMS *ret 5610 ) 5611 { 5612 return device->hal.nvswitch_ctrl_get_err_info(device, ret); 5613 } 5614 5615 static NvlStatus 5616 _nvswitch_ctrl_clear_counters 5617 ( 5618 nvswitch_device *device, 5619 NVSWITCH_NVLINK_CLEAR_COUNTERS_PARAMS *ret 5620 ) 5621 { 5622 return device->hal.nvswitch_ctrl_clear_counters(device, ret); 5623 } 5624 5625 void 5626 nvswitch_setup_link_system_registers 5627 ( 5628 nvswitch_device *device, 5629 nvlink_link *link 5630 ) 5631 { 5632 device->hal.nvswitch_setup_link_system_registers(device, link); 5633 } 5634 5635 void 5636 nvswitch_load_link_disable_settings 5637 ( 5638 nvswitch_device *device, 5639 nvlink_link *link 5640 ) 5641 { 5642 device->hal.nvswitch_load_link_disable_settings(device, link); 5643 } 5644 5645 static NvlStatus 5646 _nvswitch_ctrl_set_nvlink_error_threshold 5647 ( 5648 nvswitch_device *device, 5649 NVSWITCH_SET_NVLINK_ERROR_THRESHOLD_PARAMS *pParams 5650 ) 5651 { 5652 return device->hal.nvswitch_ctrl_set_nvlink_error_threshold(device, pParams); 5653 } 5654 5655 static NvlStatus 5656 _nvswitch_ctrl_get_nvlink_error_threshold 5657 ( 5658 nvswitch_device *device, 5659 NVSWITCH_GET_NVLINK_ERROR_THRESHOLD_PARAMS *pParams 5660 ) 5661 { 5662 return device->hal.nvswitch_ctrl_get_nvlink_error_threshold(device, pParams); 5663 } 5664 5665 static NvlStatus 5666 _nvswitch_ctrl_therm_read_voltage 5667 ( 5668 nvswitch_device *device, 5669 NVSWITCH_CTRL_GET_VOLTAGE_PARAMS *info 5670 ) 5671 { 5672 return device->hal.nvswitch_ctrl_therm_read_voltage(device, info); 5673 } 5674 5675 static NvlStatus 5676 _nvswitch_ctrl_therm_read_power 5677 ( 5678 nvswitch_device *device, 5679 NVSWITCH_GET_POWER_PARAMS *info 5680 ) 5681 { 5682 return device->hal.nvswitch_ctrl_therm_read_power(device, info); 5683 } 5684 5685 NvlStatus 5686 nvswitch_get_board_id 5687 ( 5688 nvswitch_device *device, 5689 NvU16 *boardId 5690 ) 5691 { 5692 return device->hal.nvswitch_get_board_id(device, boardId); 5693 } 5694 5695 NvlStatus 5696 _nvswitch_ctrl_get_link_l1_capability 5697 ( 5698 nvswitch_device *device, 5699 NVSWITCH_GET_NVLINK_L1_CAPABILITY_PARAMS *p 5700 ) 5701 { 5702 NvlStatus status; 5703 nvlink_link *link; 5704 NvU8 i; 5705 5706 if (p->linkMask == 0) 5707 { 5708 NVSWITCH_PRINT(device, ERROR, "%s: No links in linkMask\n", __FUNCTION__); 5709 return -NVL_BAD_ARGS; 5710 } 5711 5712 FOR_EACH_INDEX_IN_MASK(64, i, p->linkMask) 5713 { 5714 NvU32 linkNum; 5715 5716 NVSWITCH_ASSERT(i < NVSWITCH_LINK_COUNT(device)); 5717 5718 link = nvswitch_get_link(device, i); 5719 if ((link == NULL) || 5720 (i >= NVSWITCH_NVLINK_MAX_LINKS)) 5721 { 5722 NVSWITCH_PRINT(device, ERROR, "%s: Invalid input link %d set in linkMask\n", 5723 __FUNCTION__, i); 5724 return -NVL_BAD_ARGS; 5725 } 5726 5727 linkNum = link->linkNumber; 5728 5729 status = device->hal.nvswitch_ctrl_get_link_l1_capability(device, linkNum, &(p->l1Capable[linkNum])); 5730 if (status != NVL_SUCCESS) 5731 { 5732 NVSWITCH_PRINT(device, ERROR, "%s: Failed to get l1 capability for link %d\n", 5733 __FUNCTION__, linkNum); 5734 return status; 5735 } 5736 } 5737 FOR_EACH_INDEX_IN_MASK_END; 5738 5739 return NVL_SUCCESS; 5740 } 5741 5742 NvlStatus 5743 _nvswitch_ctrl_get_link_l1_threshold 5744 ( 5745 nvswitch_device *device, 5746 NVSWITCH_GET_NVLINK_L1_THRESHOLD_PARAMS *p 5747 ) 5748 { 5749 NvlStatus status; 5750 nvlink_link *link; 5751 NvBool isL1Capable; 5752 NvU8 i; 5753 5754 if (device->regkeys.enable_pm == NV_SWITCH_REGKEY_ENABLE_PM_NO) 5755 { 5756 NVSWITCH_PRINT(device, INFO, "%s: L1 Threshold is disabled\n", __FUNCTION__); 5757 return -NVL_ERR_NOT_SUPPORTED; 5758 } 5759 5760 if (p->linkMask == 0) 5761 { 5762 NVSWITCH_PRINT(device, ERROR, "%s: No links in linkMask\n", __FUNCTION__); 5763 return -NVL_BAD_ARGS; 5764 } 5765 5766 FOR_EACH_INDEX_IN_MASK(64, i, p->linkMask) 5767 { 5768 NvU32 linkNum; 5769 5770 NVSWITCH_ASSERT(i < NVSWITCH_LINK_COUNT(device)); 5771 5772 link = nvswitch_get_link(device, i); 5773 if ((link == NULL) || 5774 (i >= NVSWITCH_NVLINK_MAX_LINKS)) 5775 { 5776 NVSWITCH_PRINT(device, ERROR, "%s: Invalid input link %d set in linkMask\n", 5777 __FUNCTION__, i); 5778 return -NVL_BAD_ARGS; 5779 } 5780 5781 linkNum = link->linkNumber; 5782 5783 status = device->hal.nvswitch_ctrl_get_link_l1_capability(device, linkNum, &isL1Capable); 5784 if (status != NVL_SUCCESS) 5785 { 5786 NVSWITCH_PRINT(device, ERROR, "%s: Failed to get l1 capability for link %d\n", 5787 __FUNCTION__, linkNum); 5788 return status; 5789 } 5790 5791 if (!isL1Capable) 5792 { 5793 NVSWITCH_PRINT(device, ERROR, "%s: Input link %d does not support L1\n", 5794 __FUNCTION__, i); 5795 return -NVL_ERR_NOT_SUPPORTED; 5796 } 5797 5798 // Get HAL 5799 status = device->hal.nvswitch_ctrl_get_link_l1_threshold(device, linkNum, &(p->l1Threshold[linkNum])); 5800 if (status != NVL_SUCCESS) 5801 { 5802 NVSWITCH_PRINT(device, ERROR, "%s: Failed to get L1 Threshold for link %d\n", 5803 __FUNCTION__, linkNum); 5804 return status; 5805 } 5806 } 5807 FOR_EACH_INDEX_IN_MASK_END; 5808 5809 return NVL_SUCCESS; 5810 } 5811 5812 NvlStatus 5813 _nvswitch_ctrl_set_link_l1_threshold 5814 ( 5815 nvswitch_device *device, 5816 NVSWITCH_SET_NVLINK_L1_THRESHOLD_PARAMS *p 5817 ) 5818 { 5819 NvlStatus status; 5820 nvlink_link *link; 5821 NvBool isL1Capable; 5822 NvU8 i; 5823 5824 if (device->regkeys.enable_pm == NV_SWITCH_REGKEY_ENABLE_PM_NO) 5825 { 5826 NVSWITCH_PRINT(device, INFO, "%s: L1 Threshold is disabled\n", __FUNCTION__); 5827 return -NVL_ERR_NOT_SUPPORTED; 5828 } 5829 5830 if (p->linkMask == 0) 5831 { 5832 NVSWITCH_PRINT(device, ERROR, "%s: No links in linkMask\n", __FUNCTION__); 5833 return -NVL_BAD_ARGS; 5834 } 5835 5836 FOR_EACH_INDEX_IN_MASK(64, i, p->linkMask) 5837 { 5838 NvU32 linkNum; 5839 NvU32 l1Threshold; 5840 5841 NVSWITCH_ASSERT(i < NVSWITCH_LINK_COUNT(device)); 5842 5843 link = nvswitch_get_link(device, i); 5844 if ((link == NULL) || 5845 (i >= NVSWITCH_NVLINK_MAX_LINKS)) 5846 { 5847 NVSWITCH_PRINT(device, ERROR, "%s: Invalid input link %d set in linkMask\n", 5848 __FUNCTION__, i); 5849 return -NVL_BAD_ARGS; 5850 } 5851 5852 linkNum = link->linkNumber; 5853 l1Threshold = p->l1Threshold[linkNum]; 5854 5855 status = device->hal.nvswitch_ctrl_get_link_l1_capability(device, linkNum, &isL1Capable); 5856 if (status != NVL_SUCCESS) 5857 { 5858 NVSWITCH_PRINT(device, ERROR, "%s: Failed to get l1 capability for link %d\n", 5859 __FUNCTION__, linkNum); 5860 return status; 5861 } 5862 5863 if (!isL1Capable) 5864 { 5865 NVSWITCH_PRINT(device, ERROR, "%s: Input link %d does not support L1\n", 5866 __FUNCTION__, i); 5867 return -NVL_ERR_NOT_SUPPORTED; 5868 } 5869 5870 if (((l1Threshold < NVSWITCH_SET_NVLINK_L1_THRESHOLD_MIN) || 5871 (l1Threshold > NVSWITCH_SET_NVLINK_L1_THRESHOLD_MAX)) && 5872 (l1Threshold != NVSWITCH_SET_NVLINK_L1_THRESHOLD_DEFAULT)) 5873 { 5874 return -NVL_BAD_ARGS; 5875 } 5876 5877 status = device->hal.nvswitch_ctrl_set_link_l1_threshold(link, 5878 p->l1Threshold[linkNum]); 5879 if (status != NVL_SUCCESS) 5880 { 5881 NVSWITCH_PRINT(device, ERROR, "%s: Failed to set L1 Threshold for link %d\n", 5882 __FUNCTION__, linkNum); 5883 return status; 5884 } 5885 } 5886 FOR_EACH_INDEX_IN_MASK_END 5887 5888 return NVL_SUCCESS; 5889 } 5890 5891 NvlStatus 5892 nvswitch_lib_ctrl 5893 ( 5894 nvswitch_device *device, 5895 NvU32 cmd, 5896 void *params, 5897 NvU64 size, 5898 void *osPrivate 5899 ) 5900 { 5901 NvlStatus retval; 5902 NvU64 flags = 0; 5903 5904 if (!NVSWITCH_IS_DEVICE_ACCESSIBLE(device) || params == NULL) 5905 { 5906 return -NVL_BAD_ARGS; 5907 } 5908 5909 flags = NVSWITCH_DEV_CMD_CHECK_ADMIN | NVSWITCH_DEV_CMD_CHECK_FM; 5910 switch (cmd) 5911 { 5912 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_INFO, 5913 _nvswitch_ctrl_get_info, 5914 NVSWITCH_GET_INFO); 5915 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_INTERNAL_LATENCY, 5916 _nvswitch_ctrl_get_internal_latency, 5917 NVSWITCH_GET_INTERNAL_LATENCY); 5918 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLIPT_COUNTERS, 5919 _nvswitch_ctrl_get_nvlipt_counters, 5920 NVSWITCH_GET_NVLIPT_COUNTERS); 5921 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_ERRORS, 5922 nvswitch_ctrl_get_errors, 5923 NVSWITCH_GET_ERRORS_PARAMS); 5924 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_PORT_EVENTS, 5925 nvswitch_ctrl_get_port_events, 5926 NVSWITCH_GET_PORT_EVENTS_PARAMS); 5927 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLINK_STATUS, 5928 _nvswitch_ctrl_get_nvlink_status, 5929 NVSWITCH_GET_NVLINK_STATUS_PARAMS); 5930 NVSWITCH_DEV_CMD_DISPATCH_WITH_PRIVATE_DATA( 5931 CTRL_NVSWITCH_ACQUIRE_CAPABILITY, 5932 _nvswitch_ctrl_acquire_capability, 5933 NVSWITCH_ACQUIRE_CAPABILITY_PARAMS, 5934 osPrivate); 5935 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_TEMPERATURE, 5936 _nvswitch_ctrl_therm_read_temperature, 5937 NVSWITCH_CTRL_GET_TEMPERATURE_PARAMS); 5938 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_THROUGHPUT_COUNTERS, 5939 nvswitch_ctrl_get_throughput_counters, 5940 NVSWITCH_GET_THROUGHPUT_COUNTERS_PARAMS); 5941 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_FATAL_ERROR_SCOPE, 5942 _nvswitch_ctrl_get_fatal_error_scope, 5943 NVSWITCH_GET_FATAL_ERROR_SCOPE_PARAMS); 5944 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5945 CTRL_NVSWITCH_SET_SWITCH_PORT_CONFIG, 5946 _nvswitch_ctrl_set_switch_port_config, 5947 NVSWITCH_SET_SWITCH_PORT_CONFIG, 5948 osPrivate, flags); 5949 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5950 CTRL_NVSWITCH_GET_INGRESS_REQUEST_TABLE, 5951 _nvswitch_ctrl_get_ingress_request_table, 5952 NVSWITCH_GET_INGRESS_REQUEST_TABLE_PARAMS, 5953 osPrivate, flags); 5954 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5955 CTRL_NVSWITCH_SET_INGRESS_REQUEST_TABLE, 5956 _nvswitch_ctrl_set_ingress_request_table, 5957 NVSWITCH_SET_INGRESS_REQUEST_TABLE, 5958 osPrivate, flags); 5959 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5960 CTRL_NVSWITCH_SET_INGRESS_REQUEST_VALID, 5961 _nvswitch_ctrl_set_ingress_request_valid, 5962 NVSWITCH_SET_INGRESS_REQUEST_VALID, 5963 osPrivate, flags); 5964 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5965 CTRL_NVSWITCH_GET_INGRESS_RESPONSE_TABLE, 5966 _nvswitch_ctrl_get_ingress_response_table, 5967 NVSWITCH_GET_INGRESS_RESPONSE_TABLE_PARAMS, 5968 osPrivate, flags); 5969 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5970 CTRL_NVSWITCH_SET_INGRESS_RESPONSE_TABLE, 5971 _nvswitch_ctrl_set_ingress_response_table, 5972 NVSWITCH_SET_INGRESS_RESPONSE_TABLE, 5973 osPrivate, flags); 5974 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5975 CTRL_NVSWITCH_SET_GANGED_LINK_TABLE, 5976 _nvswitch_ctrl_set_ganged_link_table, 5977 NVSWITCH_SET_GANGED_LINK_TABLE, 5978 osPrivate, flags); 5979 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_LATENCY_BINS, 5980 nvswitch_ctrl_set_latency_bins, 5981 NVSWITCH_SET_LATENCY_BINS, 5982 osPrivate, flags); 5983 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5984 CTRL_NVSWITCH_SET_NVLIPT_COUNTER_CONFIG, 5985 _nvswitch_ctrl_set_nvlipt_counter_config, 5986 NVSWITCH_SET_NVLIPT_COUNTER_CONFIG, 5987 osPrivate, flags); 5988 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 5989 CTRL_NVSWITCH_GET_NVLIPT_COUNTER_CONFIG, 5990 _nvswitch_ctrl_get_nvlipt_counter_config, 5991 NVSWITCH_GET_NVLIPT_COUNTER_CONFIG, 5992 osPrivate, flags); 5993 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_REMAP_POLICY, 5994 _nvswitch_ctrl_set_remap_policy, 5995 NVSWITCH_SET_REMAP_POLICY, 5996 osPrivate, flags); 5997 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_REMAP_POLICY, 5998 _nvswitch_ctrl_get_remap_policy, 5999 NVSWITCH_GET_REMAP_POLICY_PARAMS, 6000 osPrivate, flags); 6001 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6002 CTRL_NVSWITCH_SET_REMAP_POLICY_VALID, 6003 _nvswitch_ctrl_set_remap_policy_valid, 6004 NVSWITCH_SET_REMAP_POLICY_VALID, 6005 osPrivate, flags); 6006 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_ROUTING_ID, 6007 _nvswitch_ctrl_set_routing_id, 6008 NVSWITCH_SET_ROUTING_ID, 6009 osPrivate, flags); 6010 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_ROUTING_ID, 6011 _nvswitch_ctrl_get_routing_id, 6012 NVSWITCH_GET_ROUTING_ID_PARAMS, 6013 osPrivate, flags); 6014 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_ROUTING_ID_VALID, 6015 _nvswitch_ctrl_set_routing_id_valid, 6016 NVSWITCH_SET_ROUTING_LAN_VALID, 6017 osPrivate, flags); 6018 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_ROUTING_LAN, 6019 _nvswitch_ctrl_set_routing_lan, 6020 NVSWITCH_SET_ROUTING_LAN, 6021 osPrivate, flags); 6022 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_ROUTING_LAN, 6023 _nvswitch_ctrl_get_routing_lan, 6024 NVSWITCH_GET_ROUTING_LAN_PARAMS, 6025 osPrivate, flags); 6026 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6027 CTRL_NVSWITCH_SET_ROUTING_LAN_VALID, 6028 _nvswitch_ctrl_set_routing_lan_valid, 6029 NVSWITCH_SET_ROUTING_LAN_VALID, 6030 osPrivate, flags); 6031 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6032 CTRL_NVSWITCH_GET_INGRESS_REQLINKID, 6033 _nvswitch_ctrl_get_ingress_reqlinkid, 6034 NVSWITCH_GET_INGRESS_REQLINKID_PARAMS, 6035 osPrivate, flags); 6036 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_UNREGISTER_LINK, 6037 _nvswitch_ctrl_unregister_link, 6038 NVSWITCH_UNREGISTER_LINK_PARAMS, 6039 osPrivate, flags); 6040 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6041 CTRL_NVSWITCH_RESET_AND_DRAIN_LINKS, 6042 _nvswitch_ctrl_reset_and_drain_links, 6043 NVSWITCH_RESET_AND_DRAIN_LINKS_PARAMS, 6044 osPrivate, flags); 6045 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_BIOS_INFO, 6046 _nvswitch_ctrl_get_bios_info, 6047 NVSWITCH_GET_BIOS_INFO_PARAMS); 6048 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_INFOROM_VERSION, 6049 _nvswitch_ctrl_get_inforom_version, 6050 NVSWITCH_GET_INFOROM_VERSION_PARAMS); 6051 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6052 CTRL_NVSWITCH_BLACKLIST_DEVICE, 6053 nvswitch_ctrl_blacklist_device, 6054 NVSWITCH_BLACKLIST_DEVICE_PARAMS, 6055 osPrivate, flags); 6056 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6057 CTRL_NVSWITCH_SET_FM_DRIVER_STATE, 6058 nvswitch_ctrl_set_fm_driver_state, 6059 NVSWITCH_SET_FM_DRIVER_STATE_PARAMS, 6060 osPrivate, flags); 6061 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6062 CTRL_NVSWITCH_SET_DEVICE_FABRIC_STATE, 6063 nvswitch_ctrl_set_device_fabric_state, 6064 NVSWITCH_SET_DEVICE_FABRIC_STATE_PARAMS, 6065 osPrivate, flags); 6066 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6067 CTRL_NVSWITCH_SET_FM_HEARTBEAT_TIMEOUT, 6068 nvswitch_ctrl_set_fm_timeout, 6069 NVSWITCH_SET_FM_HEARTBEAT_TIMEOUT_PARAMS, 6070 osPrivate, flags); 6071 NVSWITCH_DEV_CMD_DISPATCH_WITH_PRIVATE_DATA( 6072 CTRL_NVSWITCH_REGISTER_EVENTS, 6073 _nvswitch_ctrl_register_events, 6074 NVSWITCH_REGISTER_EVENTS_PARAMS, 6075 osPrivate); 6076 NVSWITCH_DEV_CMD_DISPATCH_WITH_PRIVATE_DATA( 6077 CTRL_NVSWITCH_UNREGISTER_EVENTS, 6078 _nvswitch_ctrl_unregister_events, 6079 NVSWITCH_UNREGISTER_EVENTS_PARAMS, 6080 osPrivate); 6081 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6082 CTRL_NVSWITCH_SET_TRAINING_ERROR_INFO, 6083 _nvswitch_ctrl_set_training_error_info, 6084 NVSWITCH_SET_TRAINING_ERROR_INFO_PARAMS, 6085 osPrivate, flags); 6086 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6087 CTRL_NVSWITCH_SET_MC_RID_TABLE, 6088 _nvswitch_ctrl_set_mc_rid_table, 6089 NVSWITCH_SET_MC_RID_TABLE_PARAMS, 6090 osPrivate, flags); 6091 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6092 CTRL_NVSWITCH_GET_MC_RID_TABLE, 6093 _nvswitch_ctrl_get_mc_rid_table, 6094 NVSWITCH_GET_MC_RID_TABLE_PARAMS, 6095 osPrivate, flags); 6096 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6097 CTRL_NVSWITCH_GET_COUNTERS, 6098 _nvswitch_ctrl_get_counters, 6099 NVSWITCH_NVLINK_GET_COUNTERS_PARAMS, 6100 osPrivate, flags); 6101 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6102 CTRL_NVSWITCH_GET_NVLINK_ECC_ERRORS, 6103 _nvswitch_ctrl_get_nvlink_ecc_errors, 6104 NVSWITCH_GET_NVLINK_ECC_ERRORS_PARAMS, 6105 osPrivate, flags); 6106 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6107 CTRL_NVSWITCH_I2C_SMBUS_COMMAND, 6108 _nvswitch_ctrl_i2c_smbus_command, 6109 NVSWITCH_I2C_SMBUS_COMMAND_PARAMS, 6110 osPrivate, NVSWITCH_DEV_CMD_CHECK_ADMIN); 6111 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6112 CTRL_NVSWITCH_CCI_CMIS_PRESENCE, 6113 _nvswitch_ctrl_cci_cmis_presence, 6114 NVSWITCH_CCI_CMIS_PRESENCE_PARAMS, 6115 osPrivate, flags); 6116 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6117 CTRL_NVSWITCH_CCI_CMIS_NVLINK_MAPPING, 6118 _nvswitch_ctrl_cci_nvlink_mappings, 6119 NVSWITCH_CCI_CMIS_NVLINK_MAPPING_PARAMS, 6120 osPrivate, flags); 6121 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6122 CTRL_NVSWITCH_CCI_CMIS_MEMORY_ACCESS_READ, 6123 _nvswitch_ctrl_cci_memory_access_read, 6124 NVSWITCH_CCI_CMIS_MEMORY_ACCESS_READ_PARAMS, 6125 osPrivate, flags); 6126 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6127 CTRL_NVSWITCH_CCI_CMIS_MEMORY_ACCESS_WRITE, 6128 _nvswitch_ctrl_cci_memory_access_write, 6129 NVSWITCH_CCI_CMIS_MEMORY_ACCESS_WRITE_PARAMS, 6130 osPrivate, flags); 6131 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6132 CTRL_NVSWITCH_CCI_CMIS_CAGE_BEZEL_MARKING, 6133 _nvswitch_ctrl_cci_cage_bezel_marking, 6134 NVSWITCH_CCI_CMIS_CAGE_BEZEL_MARKING_PARAMS, 6135 osPrivate, flags); 6136 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6137 CTRL_NVSWITCH_CCI_GET_GRADING_VALUES, 6138 nvswitch_ctrl_get_grading_values, 6139 NVSWITCH_CCI_GET_GRADING_VALUES_PARAMS, 6140 osPrivate, flags); 6141 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6142 CTRL_NVSWITCH_CCI_GET_PORTS_CPLD_INFO, 6143 nvswitch_ctrl_get_ports_cpld_info, 6144 NVSWITCH_CCI_GET_PORTS_CPLD_INFO_PARAMS, 6145 osPrivate, flags); 6146 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6147 CTRL_NVSWITCH_CCI_GET_FW_REVISIONS, 6148 nvswitch_ctrl_get_cci_fw_revisions, 6149 NVSWITCH_CCI_GET_FW_REVISION_PARAMS, 6150 osPrivate, flags); 6151 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6152 CTRL_NVSWITCH_CCI_SET_LOCATE_LED, 6153 nvswitch_ctrl_set_locate_led, 6154 NVSWITCH_CCI_SET_LOCATE_LED_PARAMS, 6155 osPrivate, flags); 6156 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6157 CTRL_NVSWITCH_GET_SOE_HEARTBEAT, 6158 _nvswitch_ctrl_get_soe_heartbeat, 6159 NVSWITCH_GET_SOE_HEARTBEAT_PARAMS, 6160 osPrivate, flags); 6161 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6162 CTRL_NVSWITCH_SET_CONTINUOUS_ALI, 6163 _nvswitch_ctrl_set_continuous_ali, 6164 NVSWITCH_SET_CONTINUOUS_ALI_PARAMS, 6165 osPrivate, flags); 6166 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6167 CTRL_NVSWITCH_REQUEST_ALI, 6168 _nvswitch_ctrl_request_ali, 6169 NVSWITCH_REQUEST_ALI_PARAMS, 6170 osPrivate, flags); 6171 NVSWITCH_DEV_CMD_DISPATCH( 6172 CTRL_NVSWITCH_GET_TEMPERATURE_LIMIT, 6173 _nvswitch_ctrl_therm_get_temperature_limit, 6174 NVSWITCH_CTRL_GET_TEMPERATURE_LIMIT_PARAMS); 6175 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6176 CTRL_NVSWITCH_GET_NVLINK_MAX_ERROR_RATES, 6177 _nvswitch_ctrl_get_inforom_nvlink_max_correctable_error_rate, 6178 NVSWITCH_GET_NVLINK_MAX_CORRECTABLE_ERROR_RATES_PARAMS, 6179 osPrivate, flags); 6180 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6181 CTRL_NVSWITCH_GET_NVLINK_ERROR_COUNTS, 6182 _nvswitch_ctrl_get_inforom_nvlink_errors, 6183 NVSWITCH_GET_NVLINK_ERROR_COUNTS_PARAMS, 6184 osPrivate, flags); 6185 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6186 CTRL_NVSWITCH_GET_ECC_ERROR_COUNTS, 6187 _nvswitch_ctrl_get_inforom_ecc_errors, 6188 NVSWITCH_GET_ECC_ERROR_COUNTS_PARAMS, 6189 osPrivate, flags); 6190 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6191 CTRL_NVSWITCH_GET_SXIDS, 6192 _nvswitch_ctrl_get_inforom_bbx_sxid, 6193 NVSWITCH_GET_SXIDS_PARAMS, 6194 osPrivate, flags); 6195 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6196 CTRL_NVSWITCH_GET_FOM_VALUES, 6197 _nvswitch_ctrl_get_fom_values, 6198 NVSWITCH_GET_FOM_VALUES_PARAMS, 6199 osPrivate, flags); 6200 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6201 CTRL_NVSWITCH_GET_NVLINK_LP_COUNTERS, 6202 _nvswitch_ctrl_get_nvlink_lp_counters, 6203 NVSWITCH_GET_NVLINK_LP_COUNTERS_PARAMS, 6204 osPrivate, flags); 6205 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_RESIDENCY_BINS, 6206 _nvswitch_ctrl_get_residency_bins, 6207 NVSWITCH_GET_RESIDENCY_BINS); 6208 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_RESIDENCY_BINS, 6209 _nvswitch_ctrl_set_residency_bins, 6210 NVSWITCH_SET_RESIDENCY_BINS, 6211 osPrivate, flags); 6212 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_RB_STALL_BUSY, 6213 _nvswitch_ctrl_get_rb_stall_busy, 6214 NVSWITCH_GET_RB_STALL_BUSY); 6215 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_MULTICAST_ID_ERROR_VECTOR, 6216 nvswitch_ctrl_get_multicast_id_error_vector, 6217 NVSWITCH_GET_MULTICAST_ID_ERROR_VECTOR); 6218 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_CLEAR_MULTICAST_ID_ERROR_VECTOR, 6219 nvswitch_ctrl_clear_multicast_id_error_vector, 6220 NVSWITCH_CLEAR_MULTICAST_ID_ERROR_VECTOR); 6221 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6222 CTRL_NVSWITCH_INBAND_SEND_DATA, 6223 _nvswitch_ctrl_inband_send_data, 6224 NVSWITCH_INBAND_SEND_DATA_PARAMS, 6225 osPrivate, flags); 6226 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6227 CTRL_NVSWITCH_INBAND_READ_DATA, 6228 _nvswitch_ctrl_inband_read_data, 6229 NVSWITCH_INBAND_READ_DATA_PARAMS, 6230 osPrivate, flags); 6231 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6232 CTRL_NVSWITCH_INBAND_FLUSH_DATA, 6233 _nvswitch_ctrl_inband_flush_data, 6234 NVSWITCH_INBAND_FLUSH_DATA_PARAMS, 6235 osPrivate, flags); 6236 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6237 CTRL_NVSWITCH_INBAND_PENDING_DATA_STATS, 6238 _nvswitch_ctrl_inband_pending_data_stats, 6239 NVSWITCH_INBAND_PENDING_DATA_STATS_PARAMS, 6240 osPrivate, flags); 6241 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_BOARD_PART_NUMBER, 6242 _nvswitch_ctrl_get_board_part_number, 6243 NVSWITCH_GET_BOARD_PART_NUMBER_VECTOR); 6244 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6245 CTRL_NVSWITCH_GET_SW_INFO, 6246 _nvswitch_ctrl_get_sw_info, 6247 NVSWITCH_GET_SW_INFO_PARAMS, 6248 osPrivate, flags); 6249 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_REGISTER_READ, 6250 _nvswitch_ctrl_register_read, 6251 NVSWITCH_REGISTER_READ, 6252 osPrivate, flags); 6253 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_REGISTER_WRITE, 6254 _nvswitch_ctrl_register_write, 6255 NVSWITCH_REGISTER_WRITE, 6256 osPrivate, flags); 6257 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_GET_ERR_INFO, 6258 _nvswitch_ctrl_get_err_info, 6259 NVSWITCH_NVLINK_GET_ERR_INFO_PARAMS, 6260 osPrivate, flags); 6261 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_CLEAR_COUNTERS, 6262 _nvswitch_ctrl_clear_counters, 6263 NVSWITCH_NVLINK_CLEAR_COUNTERS_PARAMS, 6264 osPrivate, flags); 6265 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_NVLINK_ERROR_THRESHOLD, 6266 _nvswitch_ctrl_set_nvlink_error_threshold, 6267 NVSWITCH_SET_NVLINK_ERROR_THRESHOLD_PARAMS, 6268 osPrivate, flags); 6269 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLINK_ERROR_THRESHOLD, 6270 _nvswitch_ctrl_get_nvlink_error_threshold, 6271 NVSWITCH_GET_NVLINK_ERROR_THRESHOLD_PARAMS); 6272 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_VOLTAGE, 6273 _nvswitch_ctrl_therm_read_voltage, 6274 NVSWITCH_CTRL_GET_VOLTAGE_PARAMS); 6275 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_POWER, 6276 _nvswitch_ctrl_therm_read_power, 6277 NVSWITCH_GET_POWER_PARAMS); 6278 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6279 CTRL_NVSWITCH_GET_SYS_INFO, 6280 _nvswitch_ctrl_get_inforom_bbx_sys_info, 6281 NVSWITCH_GET_SYS_INFO_PARAMS, 6282 osPrivate, flags); 6283 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6284 CTRL_NVSWITCH_GET_TIME_INFO, 6285 _nvswitch_ctrl_get_inforom_bbx_time_info, 6286 NVSWITCH_GET_TIME_INFO_PARAMS, 6287 osPrivate, flags); 6288 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6289 CTRL_NVSWITCH_GET_TEMP_DATA, 6290 _nvswitch_ctrl_get_inforom_bbx_temp_data, 6291 NVSWITCH_GET_TEMP_DATA_PARAMS, 6292 osPrivate, flags); 6293 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED( 6294 CTRL_NVSWITCH_GET_TEMP_SAMPLES, 6295 _nvswitch_ctrl_get_inforom_bbx_temp_samples, 6296 NVSWITCH_GET_TEMP_SAMPLES_PARAMS, 6297 osPrivate, flags); 6298 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLINK_L1_CAPABILITY, 6299 _nvswitch_ctrl_get_link_l1_capability, 6300 NVSWITCH_GET_NVLINK_L1_CAPABILITY_PARAMS); 6301 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_GET_NVLINK_L1_THRESHOLD, 6302 _nvswitch_ctrl_get_link_l1_threshold, 6303 NVSWITCH_GET_NVLINK_L1_THRESHOLD_PARAMS); 6304 NVSWITCH_DEV_CMD_DISPATCH_PRIVILEGED(CTRL_NVSWITCH_SET_NVLINK_L1_THRESHOLD, 6305 _nvswitch_ctrl_set_link_l1_threshold, 6306 NVSWITCH_SET_NVLINK_L1_THRESHOLD_PARAMS, 6307 osPrivate, flags); 6308 NVSWITCH_DEV_CMD_DISPATCH(CTRL_NVSWITCH_FSPRPC_GET_CAPS, 6309 _nvswitch_ctrl_fsprpc_get_caps, 6310 NVSWITCH_FSPRPC_GET_CAPS_PARAMS); 6311 6312 default: 6313 nvswitch_os_print(NVSWITCH_DBG_LEVEL_INFO, "unknown ioctl %x\n", cmd); 6314 retval = -NVL_BAD_ARGS; 6315 break; 6316 } 6317 6318 return retval; 6319 } 6320 6321 #if defined(DEVELOP) || defined(DEBUG) || defined(NV_MODS) 6322 void nvswitch_assert_log 6323 ( 6324 const char *function, 6325 const char *file, 6326 NvU32 line 6327 ) 6328 { 6329 nvswitch_os_assert_log("NVSwitch: Assertion failed in %s() at %s:%d\n", 6330 function, file, line); 6331 } 6332 #else 6333 void nvswitch_assert_log(void) 6334 { 6335 nvswitch_os_assert_log("NVSwitch: Assertion failed\n"); 6336 } 6337 #endif 6338