1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright(c) 2007-2010 Intel Corporation. All rights reserved. 24 */ 25 26 /* 27 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved. 28 * Copyright (c) 2012, Joyent, Inc. All rights reserved. 29 * Copyright 2012 Nexenta Systems, Inc. All rights reserved. 30 * Copyright (c) 2013 Saso Kiselkov. All rights reserved. 31 * Copyright (c) 2013 OSN Online Service Nuernberg GmbH. All rights reserved. 32 */ 33 34 #include "ixgbe_sw.h" 35 36 static char ixgbe_ident[] = "Intel 10Gb Ethernet"; 37 static char ixgbe_version[] = "ixgbe 1.1.7"; 38 39 /* 40 * Local function protoypes 41 */ 42 static int ixgbe_register_mac(ixgbe_t *); 43 static int ixgbe_identify_hardware(ixgbe_t *); 44 static int ixgbe_regs_map(ixgbe_t *); 45 static void ixgbe_init_properties(ixgbe_t *); 46 static int ixgbe_init_driver_settings(ixgbe_t *); 47 static void ixgbe_init_locks(ixgbe_t *); 48 static void ixgbe_destroy_locks(ixgbe_t *); 49 static int ixgbe_init(ixgbe_t *); 50 static int ixgbe_chip_start(ixgbe_t *); 51 static void ixgbe_chip_stop(ixgbe_t *); 52 static int ixgbe_reset(ixgbe_t *); 53 static void ixgbe_tx_clean(ixgbe_t *); 54 static boolean_t ixgbe_tx_drain(ixgbe_t *); 55 static boolean_t ixgbe_rx_drain(ixgbe_t *); 56 static int ixgbe_alloc_rings(ixgbe_t *); 57 static void ixgbe_free_rings(ixgbe_t *); 58 static int ixgbe_alloc_rx_data(ixgbe_t *); 59 static void ixgbe_free_rx_data(ixgbe_t *); 60 static void ixgbe_setup_rings(ixgbe_t *); 61 static void ixgbe_setup_rx(ixgbe_t *); 62 static void ixgbe_setup_tx(ixgbe_t *); 63 static void ixgbe_setup_rx_ring(ixgbe_rx_ring_t *); 64 static void ixgbe_setup_tx_ring(ixgbe_tx_ring_t *); 65 static void ixgbe_setup_rss(ixgbe_t *); 66 static void ixgbe_setup_vmdq(ixgbe_t *); 67 static void ixgbe_setup_vmdq_rss(ixgbe_t *); 68 static void ixgbe_init_unicst(ixgbe_t *); 69 static int ixgbe_unicst_find(ixgbe_t *, const uint8_t *); 70 static void ixgbe_setup_multicst(ixgbe_t *); 71 static void ixgbe_get_hw_state(ixgbe_t *); 72 static void ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe); 73 static void ixgbe_get_conf(ixgbe_t *); 74 static void ixgbe_init_params(ixgbe_t *); 75 static int ixgbe_get_prop(ixgbe_t *, char *, int, int, int); 76 static void ixgbe_driver_link_check(ixgbe_t *); 77 static void ixgbe_sfp_check(void *); 78 static void ixgbe_overtemp_check(void *); 79 static void ixgbe_link_timer(void *); 80 static void ixgbe_local_timer(void *); 81 static void ixgbe_arm_watchdog_timer(ixgbe_t *); 82 static void ixgbe_restart_watchdog_timer(ixgbe_t *); 83 static void ixgbe_disable_adapter_interrupts(ixgbe_t *); 84 static void ixgbe_enable_adapter_interrupts(ixgbe_t *); 85 static boolean_t is_valid_mac_addr(uint8_t *); 86 static boolean_t ixgbe_stall_check(ixgbe_t *); 87 static boolean_t ixgbe_set_loopback_mode(ixgbe_t *, uint32_t); 88 static void ixgbe_set_internal_mac_loopback(ixgbe_t *); 89 static boolean_t ixgbe_find_mac_address(ixgbe_t *); 90 static int ixgbe_alloc_intrs(ixgbe_t *); 91 static int ixgbe_alloc_intr_handles(ixgbe_t *, int); 92 static int ixgbe_add_intr_handlers(ixgbe_t *); 93 static void ixgbe_map_rxring_to_vector(ixgbe_t *, int, int); 94 static void ixgbe_map_txring_to_vector(ixgbe_t *, int, int); 95 static void ixgbe_setup_ivar(ixgbe_t *, uint16_t, uint8_t, int8_t); 96 static void ixgbe_enable_ivar(ixgbe_t *, uint16_t, int8_t); 97 static void ixgbe_disable_ivar(ixgbe_t *, uint16_t, int8_t); 98 static uint32_t ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index); 99 static int ixgbe_map_intrs_to_vectors(ixgbe_t *); 100 static void ixgbe_setup_adapter_vector(ixgbe_t *); 101 static void ixgbe_rem_intr_handlers(ixgbe_t *); 102 static void ixgbe_rem_intrs(ixgbe_t *); 103 static int ixgbe_enable_intrs(ixgbe_t *); 104 static int ixgbe_disable_intrs(ixgbe_t *); 105 static uint_t ixgbe_intr_legacy(void *, void *); 106 static uint_t ixgbe_intr_msi(void *, void *); 107 static uint_t ixgbe_intr_msix(void *, void *); 108 static void ixgbe_intr_rx_work(ixgbe_rx_ring_t *); 109 static void ixgbe_intr_tx_work(ixgbe_tx_ring_t *); 110 static void ixgbe_intr_other_work(ixgbe_t *, uint32_t); 111 static void ixgbe_get_driver_control(struct ixgbe_hw *); 112 static int ixgbe_addmac(void *, const uint8_t *); 113 static int ixgbe_remmac(void *, const uint8_t *); 114 static void ixgbe_release_driver_control(struct ixgbe_hw *); 115 116 static int ixgbe_attach(dev_info_t *, ddi_attach_cmd_t); 117 static int ixgbe_detach(dev_info_t *, ddi_detach_cmd_t); 118 static int ixgbe_resume(dev_info_t *); 119 static int ixgbe_suspend(dev_info_t *); 120 static int ixgbe_quiesce(dev_info_t *); 121 static void ixgbe_unconfigure(dev_info_t *, ixgbe_t *); 122 static uint8_t *ixgbe_mc_table_itr(struct ixgbe_hw *, uint8_t **, uint32_t *); 123 static int ixgbe_cbfunc(dev_info_t *, ddi_cb_action_t, void *, void *, void *); 124 static int ixgbe_intr_cb_register(ixgbe_t *); 125 static int ixgbe_intr_adjust(ixgbe_t *, ddi_cb_action_t, int); 126 127 static int ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, 128 const void *impl_data); 129 static void ixgbe_fm_init(ixgbe_t *); 130 static void ixgbe_fm_fini(ixgbe_t *); 131 132 char *ixgbe_priv_props[] = { 133 "_tx_copy_thresh", 134 "_tx_recycle_thresh", 135 "_tx_overload_thresh", 136 "_tx_resched_thresh", 137 "_rx_copy_thresh", 138 "_rx_limit_per_intr", 139 "_intr_throttling", 140 "_adv_pause_cap", 141 "_adv_asym_pause_cap", 142 NULL 143 }; 144 145 #define IXGBE_MAX_PRIV_PROPS \ 146 (sizeof (ixgbe_priv_props) / sizeof (mac_priv_prop_t)) 147 148 static struct cb_ops ixgbe_cb_ops = { 149 nulldev, /* cb_open */ 150 nulldev, /* cb_close */ 151 nodev, /* cb_strategy */ 152 nodev, /* cb_print */ 153 nodev, /* cb_dump */ 154 nodev, /* cb_read */ 155 nodev, /* cb_write */ 156 nodev, /* cb_ioctl */ 157 nodev, /* cb_devmap */ 158 nodev, /* cb_mmap */ 159 nodev, /* cb_segmap */ 160 nochpoll, /* cb_chpoll */ 161 ddi_prop_op, /* cb_prop_op */ 162 NULL, /* cb_stream */ 163 D_MP | D_HOTPLUG, /* cb_flag */ 164 CB_REV, /* cb_rev */ 165 nodev, /* cb_aread */ 166 nodev /* cb_awrite */ 167 }; 168 169 static struct dev_ops ixgbe_dev_ops = { 170 DEVO_REV, /* devo_rev */ 171 0, /* devo_refcnt */ 172 NULL, /* devo_getinfo */ 173 nulldev, /* devo_identify */ 174 nulldev, /* devo_probe */ 175 ixgbe_attach, /* devo_attach */ 176 ixgbe_detach, /* devo_detach */ 177 nodev, /* devo_reset */ 178 &ixgbe_cb_ops, /* devo_cb_ops */ 179 NULL, /* devo_bus_ops */ 180 ddi_power, /* devo_power */ 181 ixgbe_quiesce, /* devo_quiesce */ 182 }; 183 184 static struct modldrv ixgbe_modldrv = { 185 &mod_driverops, /* Type of module. This one is a driver */ 186 ixgbe_ident, /* Discription string */ 187 &ixgbe_dev_ops /* driver ops */ 188 }; 189 190 static struct modlinkage ixgbe_modlinkage = { 191 MODREV_1, &ixgbe_modldrv, NULL 192 }; 193 194 /* 195 * Access attributes for register mapping 196 */ 197 ddi_device_acc_attr_t ixgbe_regs_acc_attr = { 198 DDI_DEVICE_ATTR_V1, 199 DDI_STRUCTURE_LE_ACC, 200 DDI_STRICTORDER_ACC, 201 DDI_FLAGERR_ACC 202 }; 203 204 /* 205 * Loopback property 206 */ 207 static lb_property_t lb_normal = { 208 normal, "normal", IXGBE_LB_NONE 209 }; 210 211 static lb_property_t lb_mac = { 212 internal, "MAC", IXGBE_LB_INTERNAL_MAC 213 }; 214 215 static lb_property_t lb_external = { 216 external, "External", IXGBE_LB_EXTERNAL 217 }; 218 219 #define IXGBE_M_CALLBACK_FLAGS \ 220 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP | MC_PROPINFO) 221 222 static mac_callbacks_t ixgbe_m_callbacks = { 223 IXGBE_M_CALLBACK_FLAGS, 224 ixgbe_m_stat, 225 ixgbe_m_start, 226 ixgbe_m_stop, 227 ixgbe_m_promisc, 228 ixgbe_m_multicst, 229 NULL, 230 NULL, 231 NULL, 232 ixgbe_m_ioctl, 233 ixgbe_m_getcapab, 234 NULL, 235 NULL, 236 ixgbe_m_setprop, 237 ixgbe_m_getprop, 238 ixgbe_m_propinfo 239 }; 240 241 /* 242 * Initialize capabilities of each supported adapter type 243 */ 244 static adapter_info_t ixgbe_82598eb_cap = { 245 64, /* maximum number of rx queues */ 246 1, /* minimum number of rx queues */ 247 64, /* default number of rx queues */ 248 16, /* maximum number of rx groups */ 249 1, /* minimum number of rx groups */ 250 1, /* default number of rx groups */ 251 32, /* maximum number of tx queues */ 252 1, /* minimum number of tx queues */ 253 8, /* default number of tx queues */ 254 16366, /* maximum MTU size */ 255 0xFFFF, /* maximum interrupt throttle rate */ 256 0, /* minimum interrupt throttle rate */ 257 200, /* default interrupt throttle rate */ 258 18, /* maximum total msix vectors */ 259 16, /* maximum number of ring vectors */ 260 2, /* maximum number of other vectors */ 261 IXGBE_EICR_LSC, /* "other" interrupt types handled */ 262 0, /* "other" interrupt types enable mask */ 263 (IXGBE_FLAG_DCA_CAPABLE /* capability flags */ 264 | IXGBE_FLAG_RSS_CAPABLE 265 | IXGBE_FLAG_VMDQ_CAPABLE) 266 }; 267 268 static adapter_info_t ixgbe_82599eb_cap = { 269 128, /* maximum number of rx queues */ 270 1, /* minimum number of rx queues */ 271 128, /* default number of rx queues */ 272 64, /* maximum number of rx groups */ 273 1, /* minimum number of rx groups */ 274 1, /* default number of rx groups */ 275 128, /* maximum number of tx queues */ 276 1, /* minimum number of tx queues */ 277 8, /* default number of tx queues */ 278 15500, /* maximum MTU size */ 279 0xFF8, /* maximum interrupt throttle rate */ 280 0, /* minimum interrupt throttle rate */ 281 200, /* default interrupt throttle rate */ 282 64, /* maximum total msix vectors */ 283 16, /* maximum number of ring vectors */ 284 2, /* maximum number of other vectors */ 285 (IXGBE_EICR_LSC 286 | IXGBE_EICR_GPI_SDP1 287 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 288 289 (IXGBE_SDP1_GPIEN 290 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 291 292 (IXGBE_FLAG_DCA_CAPABLE 293 | IXGBE_FLAG_RSS_CAPABLE 294 | IXGBE_FLAG_VMDQ_CAPABLE 295 | IXGBE_FLAG_RSC_CAPABLE 296 | IXGBE_FLAG_SFP_PLUG_CAPABLE) /* capability flags */ 297 }; 298 299 static adapter_info_t ixgbe_X540_cap = { 300 128, /* maximum number of rx queues */ 301 1, /* minimum number of rx queues */ 302 128, /* default number of rx queues */ 303 64, /* maximum number of rx groups */ 304 1, /* minimum number of rx groups */ 305 1, /* default number of rx groups */ 306 128, /* maximum number of tx queues */ 307 1, /* minimum number of tx queues */ 308 8, /* default number of tx queues */ 309 15500, /* maximum MTU size */ 310 0xFF8, /* maximum interrupt throttle rate */ 311 0, /* minimum interrupt throttle rate */ 312 200, /* default interrupt throttle rate */ 313 64, /* maximum total msix vectors */ 314 16, /* maximum number of ring vectors */ 315 2, /* maximum number of other vectors */ 316 (IXGBE_EICR_LSC 317 | IXGBE_EICR_GPI_SDP1 318 | IXGBE_EICR_GPI_SDP2), /* "other" interrupt types handled */ 319 320 (IXGBE_SDP1_GPIEN 321 | IXGBE_SDP2_GPIEN), /* "other" interrupt types enable mask */ 322 323 (IXGBE_FLAG_DCA_CAPABLE 324 | IXGBE_FLAG_RSS_CAPABLE 325 | IXGBE_FLAG_VMDQ_CAPABLE 326 | IXGBE_FLAG_RSC_CAPABLE) /* capability flags */ 327 }; 328 329 /* 330 * Module Initialization Functions. 331 */ 332 333 int 334 _init(void) 335 { 336 int status; 337 338 mac_init_ops(&ixgbe_dev_ops, MODULE_NAME); 339 340 status = mod_install(&ixgbe_modlinkage); 341 342 if (status != DDI_SUCCESS) { 343 mac_fini_ops(&ixgbe_dev_ops); 344 } 345 346 return (status); 347 } 348 349 int 350 _fini(void) 351 { 352 int status; 353 354 status = mod_remove(&ixgbe_modlinkage); 355 356 if (status == DDI_SUCCESS) { 357 mac_fini_ops(&ixgbe_dev_ops); 358 } 359 360 return (status); 361 } 362 363 int 364 _info(struct modinfo *modinfop) 365 { 366 int status; 367 368 status = mod_info(&ixgbe_modlinkage, modinfop); 369 370 return (status); 371 } 372 373 /* 374 * ixgbe_attach - Driver attach. 375 * 376 * This function is the device specific initialization entry 377 * point. This entry point is required and must be written. 378 * The DDI_ATTACH command must be provided in the attach entry 379 * point. When attach() is called with cmd set to DDI_ATTACH, 380 * all normal kernel services (such as kmem_alloc(9F)) are 381 * available for use by the driver. 382 * 383 * The attach() function will be called once for each instance 384 * of the device on the system with cmd set to DDI_ATTACH. 385 * Until attach() succeeds, the only driver entry points which 386 * may be called are open(9E) and getinfo(9E). 387 */ 388 static int 389 ixgbe_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 390 { 391 ixgbe_t *ixgbe; 392 struct ixgbe_osdep *osdep; 393 struct ixgbe_hw *hw; 394 int instance; 395 char taskqname[32]; 396 397 /* 398 * Check the command and perform corresponding operations 399 */ 400 switch (cmd) { 401 default: 402 return (DDI_FAILURE); 403 404 case DDI_RESUME: 405 return (ixgbe_resume(devinfo)); 406 407 case DDI_ATTACH: 408 break; 409 } 410 411 /* Get the device instance */ 412 instance = ddi_get_instance(devinfo); 413 414 /* Allocate memory for the instance data structure */ 415 ixgbe = kmem_zalloc(sizeof (ixgbe_t), KM_SLEEP); 416 417 ixgbe->dip = devinfo; 418 ixgbe->instance = instance; 419 420 hw = &ixgbe->hw; 421 osdep = &ixgbe->osdep; 422 hw->back = osdep; 423 osdep->ixgbe = ixgbe; 424 425 /* Attach the instance pointer to the dev_info data structure */ 426 ddi_set_driver_private(devinfo, ixgbe); 427 428 /* 429 * Initialize for fma support 430 */ 431 ixgbe->fm_capabilities = ixgbe_get_prop(ixgbe, PROP_FM_CAPABLE, 432 0, 0x0f, DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 433 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 434 ixgbe_fm_init(ixgbe); 435 ixgbe->attach_progress |= ATTACH_PROGRESS_FM_INIT; 436 437 /* 438 * Map PCI config space registers 439 */ 440 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 441 ixgbe_error(ixgbe, "Failed to map PCI configurations"); 442 goto attach_fail; 443 } 444 ixgbe->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 445 446 /* 447 * Identify the chipset family 448 */ 449 if (ixgbe_identify_hardware(ixgbe) != IXGBE_SUCCESS) { 450 ixgbe_error(ixgbe, "Failed to identify hardware"); 451 goto attach_fail; 452 } 453 454 /* 455 * Map device registers 456 */ 457 if (ixgbe_regs_map(ixgbe) != IXGBE_SUCCESS) { 458 ixgbe_error(ixgbe, "Failed to map device registers"); 459 goto attach_fail; 460 } 461 ixgbe->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 462 463 /* 464 * Initialize driver parameters 465 */ 466 ixgbe_init_properties(ixgbe); 467 ixgbe->attach_progress |= ATTACH_PROGRESS_PROPS; 468 469 /* 470 * Register interrupt callback 471 */ 472 if (ixgbe_intr_cb_register(ixgbe) != IXGBE_SUCCESS) { 473 ixgbe_error(ixgbe, "Failed to register interrupt callback"); 474 goto attach_fail; 475 } 476 477 /* 478 * Allocate interrupts 479 */ 480 if (ixgbe_alloc_intrs(ixgbe) != IXGBE_SUCCESS) { 481 ixgbe_error(ixgbe, "Failed to allocate interrupts"); 482 goto attach_fail; 483 } 484 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 485 486 /* 487 * Allocate rx/tx rings based on the ring numbers. 488 * The actual numbers of rx/tx rings are decided by the number of 489 * allocated interrupt vectors, so we should allocate the rings after 490 * interrupts are allocated. 491 */ 492 if (ixgbe_alloc_rings(ixgbe) != IXGBE_SUCCESS) { 493 ixgbe_error(ixgbe, "Failed to allocate rx and tx rings"); 494 goto attach_fail; 495 } 496 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 497 498 /* 499 * Map rings to interrupt vectors 500 */ 501 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 502 ixgbe_error(ixgbe, "Failed to map interrupts to vectors"); 503 goto attach_fail; 504 } 505 506 /* 507 * Add interrupt handlers 508 */ 509 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 510 ixgbe_error(ixgbe, "Failed to add interrupt handlers"); 511 goto attach_fail; 512 } 513 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 514 515 /* 516 * Create a taskq for sfp-change 517 */ 518 (void) sprintf(taskqname, "ixgbe%d_sfp_taskq", instance); 519 if ((ixgbe->sfp_taskq = ddi_taskq_create(devinfo, taskqname, 520 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 521 ixgbe_error(ixgbe, "sfp_taskq create failed"); 522 goto attach_fail; 523 } 524 ixgbe->attach_progress |= ATTACH_PROGRESS_SFP_TASKQ; 525 526 /* 527 * Create a taskq for over-temp 528 */ 529 (void) sprintf(taskqname, "ixgbe%d_overtemp_taskq", instance); 530 if ((ixgbe->overtemp_taskq = ddi_taskq_create(devinfo, taskqname, 531 1, TASKQ_DEFAULTPRI, 0)) == NULL) { 532 ixgbe_error(ixgbe, "overtemp_taskq create failed"); 533 goto attach_fail; 534 } 535 ixgbe->attach_progress |= ATTACH_PROGRESS_OVERTEMP_TASKQ; 536 537 /* 538 * Initialize driver parameters 539 */ 540 if (ixgbe_init_driver_settings(ixgbe) != IXGBE_SUCCESS) { 541 ixgbe_error(ixgbe, "Failed to initialize driver settings"); 542 goto attach_fail; 543 } 544 545 /* 546 * Initialize mutexes for this device. 547 * Do this before enabling the interrupt handler and 548 * register the softint to avoid the condition where 549 * interrupt handler can try using uninitialized mutex. 550 */ 551 ixgbe_init_locks(ixgbe); 552 ixgbe->attach_progress |= ATTACH_PROGRESS_LOCKS; 553 554 /* 555 * Initialize chipset hardware 556 */ 557 if (ixgbe_init(ixgbe) != IXGBE_SUCCESS) { 558 ixgbe_error(ixgbe, "Failed to initialize adapter"); 559 goto attach_fail; 560 } 561 ixgbe->link_check_complete = B_FALSE; 562 ixgbe->link_check_hrtime = gethrtime() + 563 (IXGBE_LINK_UP_TIME * 100000000ULL); 564 ixgbe->attach_progress |= ATTACH_PROGRESS_INIT; 565 566 if (ixgbe_check_acc_handle(ixgbe->osdep.cfg_handle) != DDI_FM_OK) { 567 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 568 goto attach_fail; 569 } 570 571 /* 572 * Initialize statistics 573 */ 574 if (ixgbe_init_stats(ixgbe) != IXGBE_SUCCESS) { 575 ixgbe_error(ixgbe, "Failed to initialize statistics"); 576 goto attach_fail; 577 } 578 ixgbe->attach_progress |= ATTACH_PROGRESS_STATS; 579 580 /* 581 * Register the driver to the MAC 582 */ 583 if (ixgbe_register_mac(ixgbe) != IXGBE_SUCCESS) { 584 ixgbe_error(ixgbe, "Failed to register MAC"); 585 goto attach_fail; 586 } 587 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 588 ixgbe->attach_progress |= ATTACH_PROGRESS_MAC; 589 590 ixgbe->periodic_id = ddi_periodic_add(ixgbe_link_timer, ixgbe, 591 IXGBE_CYCLIC_PERIOD, DDI_IPL_0); 592 if (ixgbe->periodic_id == 0) { 593 ixgbe_error(ixgbe, "Failed to add the link check timer"); 594 goto attach_fail; 595 } 596 ixgbe->attach_progress |= ATTACH_PROGRESS_LINK_TIMER; 597 598 /* 599 * Now that mutex locks are initialized, and the chip is also 600 * initialized, enable interrupts. 601 */ 602 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 603 ixgbe_error(ixgbe, "Failed to enable DDI interrupts"); 604 goto attach_fail; 605 } 606 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 607 608 ixgbe_log(ixgbe, "%s, %s", ixgbe_ident, ixgbe_version); 609 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_INITIALIZED); 610 611 return (DDI_SUCCESS); 612 613 attach_fail: 614 ixgbe_unconfigure(devinfo, ixgbe); 615 return (DDI_FAILURE); 616 } 617 618 /* 619 * ixgbe_detach - Driver detach. 620 * 621 * The detach() function is the complement of the attach routine. 622 * If cmd is set to DDI_DETACH, detach() is used to remove the 623 * state associated with a given instance of a device node 624 * prior to the removal of that instance from the system. 625 * 626 * The detach() function will be called once for each instance 627 * of the device for which there has been a successful attach() 628 * once there are no longer any opens on the device. 629 * 630 * Interrupts routine are disabled, All memory allocated by this 631 * driver are freed. 632 */ 633 static int 634 ixgbe_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 635 { 636 ixgbe_t *ixgbe; 637 638 /* 639 * Check detach command 640 */ 641 switch (cmd) { 642 default: 643 return (DDI_FAILURE); 644 645 case DDI_SUSPEND: 646 return (ixgbe_suspend(devinfo)); 647 648 case DDI_DETACH: 649 break; 650 } 651 652 /* 653 * Get the pointer to the driver private data structure 654 */ 655 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 656 if (ixgbe == NULL) 657 return (DDI_FAILURE); 658 659 /* 660 * If the device is still running, it needs to be stopped first. 661 * This check is necessary because under some specific circumstances, 662 * the detach routine can be called without stopping the interface 663 * first. 664 */ 665 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 666 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 667 mutex_enter(&ixgbe->gen_lock); 668 ixgbe_stop(ixgbe, B_TRUE); 669 mutex_exit(&ixgbe->gen_lock); 670 /* Disable and stop the watchdog timer */ 671 ixgbe_disable_watchdog_timer(ixgbe); 672 } 673 674 /* 675 * Check if there are still rx buffers held by the upper layer. 676 * If so, fail the detach. 677 */ 678 if (!ixgbe_rx_drain(ixgbe)) 679 return (DDI_FAILURE); 680 681 /* 682 * Do the remaining unconfigure routines 683 */ 684 ixgbe_unconfigure(devinfo, ixgbe); 685 686 return (DDI_SUCCESS); 687 } 688 689 /* 690 * quiesce(9E) entry point. 691 * 692 * This function is called when the system is single-threaded at high 693 * PIL with preemption disabled. Therefore, this function must not be 694 * blocked. 695 * 696 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 697 * DDI_FAILURE indicates an error condition and should almost never happen. 698 */ 699 static int 700 ixgbe_quiesce(dev_info_t *devinfo) 701 { 702 ixgbe_t *ixgbe; 703 struct ixgbe_hw *hw; 704 705 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 706 707 if (ixgbe == NULL) 708 return (DDI_FAILURE); 709 710 hw = &ixgbe->hw; 711 712 /* 713 * Disable the adapter interrupts 714 */ 715 ixgbe_disable_adapter_interrupts(ixgbe); 716 717 /* 718 * Tell firmware driver is no longer in control 719 */ 720 ixgbe_release_driver_control(hw); 721 722 /* 723 * Reset the chipset 724 */ 725 (void) ixgbe_reset_hw(hw); 726 727 /* 728 * Reset PHY 729 */ 730 (void) ixgbe_reset_phy(hw); 731 732 return (DDI_SUCCESS); 733 } 734 735 static void 736 ixgbe_unconfigure(dev_info_t *devinfo, ixgbe_t *ixgbe) 737 { 738 /* 739 * Disable interrupt 740 */ 741 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 742 (void) ixgbe_disable_intrs(ixgbe); 743 } 744 745 /* 746 * remove the link check timer 747 */ 748 if (ixgbe->attach_progress & ATTACH_PROGRESS_LINK_TIMER) { 749 if (ixgbe->periodic_id != NULL) { 750 ddi_periodic_delete(ixgbe->periodic_id); 751 ixgbe->periodic_id = NULL; 752 } 753 } 754 755 /* 756 * Unregister MAC 757 */ 758 if (ixgbe->attach_progress & ATTACH_PROGRESS_MAC) { 759 (void) mac_unregister(ixgbe->mac_hdl); 760 } 761 762 /* 763 * Free statistics 764 */ 765 if (ixgbe->attach_progress & ATTACH_PROGRESS_STATS) { 766 kstat_delete((kstat_t *)ixgbe->ixgbe_ks); 767 } 768 769 /* 770 * Remove interrupt handlers 771 */ 772 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 773 ixgbe_rem_intr_handlers(ixgbe); 774 } 775 776 /* 777 * Remove taskq for sfp-status-change 778 */ 779 if (ixgbe->attach_progress & ATTACH_PROGRESS_SFP_TASKQ) { 780 ddi_taskq_destroy(ixgbe->sfp_taskq); 781 } 782 783 /* 784 * Remove taskq for over-temp 785 */ 786 if (ixgbe->attach_progress & ATTACH_PROGRESS_OVERTEMP_TASKQ) { 787 ddi_taskq_destroy(ixgbe->overtemp_taskq); 788 } 789 790 /* 791 * Remove interrupts 792 */ 793 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 794 ixgbe_rem_intrs(ixgbe); 795 } 796 797 /* 798 * Unregister interrupt callback handler 799 */ 800 (void) ddi_cb_unregister(ixgbe->cb_hdl); 801 802 /* 803 * Remove driver properties 804 */ 805 if (ixgbe->attach_progress & ATTACH_PROGRESS_PROPS) { 806 (void) ddi_prop_remove_all(devinfo); 807 } 808 809 /* 810 * Stop the chipset 811 */ 812 if (ixgbe->attach_progress & ATTACH_PROGRESS_INIT) { 813 mutex_enter(&ixgbe->gen_lock); 814 ixgbe_chip_stop(ixgbe); 815 mutex_exit(&ixgbe->gen_lock); 816 } 817 818 /* 819 * Free register handle 820 */ 821 if (ixgbe->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 822 if (ixgbe->osdep.reg_handle != NULL) 823 ddi_regs_map_free(&ixgbe->osdep.reg_handle); 824 } 825 826 /* 827 * Free PCI config handle 828 */ 829 if (ixgbe->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 830 if (ixgbe->osdep.cfg_handle != NULL) 831 pci_config_teardown(&ixgbe->osdep.cfg_handle); 832 } 833 834 /* 835 * Free locks 836 */ 837 if (ixgbe->attach_progress & ATTACH_PROGRESS_LOCKS) { 838 ixgbe_destroy_locks(ixgbe); 839 } 840 841 /* 842 * Free the rx/tx rings 843 */ 844 if (ixgbe->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 845 ixgbe_free_rings(ixgbe); 846 } 847 848 /* 849 * Unregister FMA capabilities 850 */ 851 if (ixgbe->attach_progress & ATTACH_PROGRESS_FM_INIT) { 852 ixgbe_fm_fini(ixgbe); 853 } 854 855 /* 856 * Free the driver data structure 857 */ 858 kmem_free(ixgbe, sizeof (ixgbe_t)); 859 860 ddi_set_driver_private(devinfo, NULL); 861 } 862 863 /* 864 * ixgbe_register_mac - Register the driver and its function pointers with 865 * the GLD interface. 866 */ 867 static int 868 ixgbe_register_mac(ixgbe_t *ixgbe) 869 { 870 struct ixgbe_hw *hw = &ixgbe->hw; 871 mac_register_t *mac; 872 int status; 873 874 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 875 return (IXGBE_FAILURE); 876 877 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 878 mac->m_driver = ixgbe; 879 mac->m_dip = ixgbe->dip; 880 mac->m_src_addr = hw->mac.addr; 881 mac->m_callbacks = &ixgbe_m_callbacks; 882 mac->m_min_sdu = 0; 883 mac->m_max_sdu = ixgbe->default_mtu; 884 mac->m_margin = VLAN_TAGSZ; 885 mac->m_priv_props = ixgbe_priv_props; 886 mac->m_v12n = MAC_VIRT_LEVEL1; 887 888 status = mac_register(mac, &ixgbe->mac_hdl); 889 890 mac_free(mac); 891 892 return ((status == 0) ? IXGBE_SUCCESS : IXGBE_FAILURE); 893 } 894 895 /* 896 * ixgbe_identify_hardware - Identify the type of the chipset. 897 */ 898 static int 899 ixgbe_identify_hardware(ixgbe_t *ixgbe) 900 { 901 struct ixgbe_hw *hw = &ixgbe->hw; 902 struct ixgbe_osdep *osdep = &ixgbe->osdep; 903 904 /* 905 * Get the device id 906 */ 907 hw->vendor_id = 908 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 909 hw->device_id = 910 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 911 hw->revision_id = 912 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 913 hw->subsystem_device_id = 914 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 915 hw->subsystem_vendor_id = 916 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 917 918 /* 919 * Set the mac type of the adapter based on the device id 920 */ 921 if (ixgbe_set_mac_type(hw) != IXGBE_SUCCESS) { 922 return (IXGBE_FAILURE); 923 } 924 925 /* 926 * Install adapter capabilities 927 */ 928 switch (hw->mac.type) { 929 case ixgbe_mac_82598EB: 930 IXGBE_DEBUGLOG_0(ixgbe, "identify 82598 adapter\n"); 931 ixgbe->capab = &ixgbe_82598eb_cap; 932 933 if (ixgbe_get_media_type(hw) == ixgbe_media_type_copper) { 934 ixgbe->capab->flags |= IXGBE_FLAG_FAN_FAIL_CAPABLE; 935 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP1; 936 ixgbe->capab->other_gpie |= IXGBE_SDP1_GPIEN; 937 } 938 break; 939 940 case ixgbe_mac_82599EB: 941 IXGBE_DEBUGLOG_0(ixgbe, "identify 82599 adapter\n"); 942 ixgbe->capab = &ixgbe_82599eb_cap; 943 944 if (hw->device_id == IXGBE_DEV_ID_82599_T3_LOM) { 945 ixgbe->capab->flags |= IXGBE_FLAG_TEMP_SENSOR_CAPABLE; 946 ixgbe->capab->other_intr |= IXGBE_EICR_GPI_SDP0; 947 ixgbe->capab->other_gpie |= IXGBE_SDP0_GPIEN; 948 } 949 break; 950 951 case ixgbe_mac_X540: 952 IXGBE_DEBUGLOG_0(ixgbe, "identify X540 adapter\n"); 953 ixgbe->capab = &ixgbe_X540_cap; 954 /* 955 * For now, X540 is all set in its capab structure. 956 * As other X540 variants show up, things can change here. 957 */ 958 break; 959 960 default: 961 IXGBE_DEBUGLOG_1(ixgbe, 962 "adapter not supported in ixgbe_identify_hardware(): %d\n", 963 hw->mac.type); 964 return (IXGBE_FAILURE); 965 } 966 967 return (IXGBE_SUCCESS); 968 } 969 970 /* 971 * ixgbe_regs_map - Map the device registers. 972 * 973 */ 974 static int 975 ixgbe_regs_map(ixgbe_t *ixgbe) 976 { 977 dev_info_t *devinfo = ixgbe->dip; 978 struct ixgbe_hw *hw = &ixgbe->hw; 979 struct ixgbe_osdep *osdep = &ixgbe->osdep; 980 off_t mem_size; 981 982 /* 983 * First get the size of device registers to be mapped. 984 */ 985 if (ddi_dev_regsize(devinfo, IXGBE_ADAPTER_REGSET, &mem_size) 986 != DDI_SUCCESS) { 987 return (IXGBE_FAILURE); 988 } 989 990 /* 991 * Call ddi_regs_map_setup() to map registers 992 */ 993 if ((ddi_regs_map_setup(devinfo, IXGBE_ADAPTER_REGSET, 994 (caddr_t *)&hw->hw_addr, 0, 995 mem_size, &ixgbe_regs_acc_attr, 996 &osdep->reg_handle)) != DDI_SUCCESS) { 997 return (IXGBE_FAILURE); 998 } 999 1000 return (IXGBE_SUCCESS); 1001 } 1002 1003 /* 1004 * ixgbe_init_properties - Initialize driver properties. 1005 */ 1006 static void 1007 ixgbe_init_properties(ixgbe_t *ixgbe) 1008 { 1009 /* 1010 * Get conf file properties, including link settings 1011 * jumbo frames, ring number, descriptor number, etc. 1012 */ 1013 ixgbe_get_conf(ixgbe); 1014 1015 ixgbe_init_params(ixgbe); 1016 } 1017 1018 /* 1019 * ixgbe_init_driver_settings - Initialize driver settings. 1020 * 1021 * The settings include hardware function pointers, bus information, 1022 * rx/tx rings settings, link state, and any other parameters that 1023 * need to be setup during driver initialization. 1024 */ 1025 static int 1026 ixgbe_init_driver_settings(ixgbe_t *ixgbe) 1027 { 1028 struct ixgbe_hw *hw = &ixgbe->hw; 1029 dev_info_t *devinfo = ixgbe->dip; 1030 ixgbe_rx_ring_t *rx_ring; 1031 ixgbe_rx_group_t *rx_group; 1032 ixgbe_tx_ring_t *tx_ring; 1033 uint32_t rx_size; 1034 uint32_t tx_size; 1035 uint32_t ring_per_group; 1036 int i; 1037 1038 /* 1039 * Initialize chipset specific hardware function pointers 1040 */ 1041 if (ixgbe_init_shared_code(hw) != IXGBE_SUCCESS) { 1042 return (IXGBE_FAILURE); 1043 } 1044 1045 /* 1046 * Get the system page size 1047 */ 1048 ixgbe->sys_page_size = ddi_ptob(devinfo, (ulong_t)1); 1049 1050 /* 1051 * Set rx buffer size 1052 * 1053 * The IP header alignment room is counted in the calculation. 1054 * The rx buffer size is in unit of 1K that is required by the 1055 * chipset hardware. 1056 */ 1057 rx_size = ixgbe->max_frame_size + IPHDR_ALIGN_ROOM; 1058 ixgbe->rx_buf_size = ((rx_size >> 10) + 1059 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1060 1061 /* 1062 * Set tx buffer size 1063 */ 1064 tx_size = ixgbe->max_frame_size; 1065 ixgbe->tx_buf_size = ((tx_size >> 10) + 1066 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 1067 1068 /* 1069 * Initialize rx/tx rings/groups parameters 1070 */ 1071 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 1072 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1073 rx_ring = &ixgbe->rx_rings[i]; 1074 rx_ring->index = i; 1075 rx_ring->ixgbe = ixgbe; 1076 rx_ring->group_index = i / ring_per_group; 1077 rx_ring->hw_index = ixgbe_get_hw_rx_index(ixgbe, i); 1078 } 1079 1080 for (i = 0; i < ixgbe->num_rx_groups; i++) { 1081 rx_group = &ixgbe->rx_groups[i]; 1082 rx_group->index = i; 1083 rx_group->ixgbe = ixgbe; 1084 } 1085 1086 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1087 tx_ring = &ixgbe->tx_rings[i]; 1088 tx_ring->index = i; 1089 tx_ring->ixgbe = ixgbe; 1090 if (ixgbe->tx_head_wb_enable) 1091 tx_ring->tx_recycle = ixgbe_tx_recycle_head_wb; 1092 else 1093 tx_ring->tx_recycle = ixgbe_tx_recycle_legacy; 1094 1095 tx_ring->ring_size = ixgbe->tx_ring_size; 1096 tx_ring->free_list_size = ixgbe->tx_ring_size + 1097 (ixgbe->tx_ring_size >> 1); 1098 } 1099 1100 /* 1101 * Initialize values of interrupt throttling rate 1102 */ 1103 for (i = 1; i < MAX_INTR_VECTOR; i++) 1104 ixgbe->intr_throttling[i] = ixgbe->intr_throttling[0]; 1105 1106 /* 1107 * The initial link state should be "unknown" 1108 */ 1109 ixgbe->link_state = LINK_STATE_UNKNOWN; 1110 1111 return (IXGBE_SUCCESS); 1112 } 1113 1114 /* 1115 * ixgbe_init_locks - Initialize locks. 1116 */ 1117 static void 1118 ixgbe_init_locks(ixgbe_t *ixgbe) 1119 { 1120 ixgbe_rx_ring_t *rx_ring; 1121 ixgbe_tx_ring_t *tx_ring; 1122 int i; 1123 1124 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1125 rx_ring = &ixgbe->rx_rings[i]; 1126 mutex_init(&rx_ring->rx_lock, NULL, 1127 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1128 } 1129 1130 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1131 tx_ring = &ixgbe->tx_rings[i]; 1132 mutex_init(&tx_ring->tx_lock, NULL, 1133 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1134 mutex_init(&tx_ring->recycle_lock, NULL, 1135 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1136 mutex_init(&tx_ring->tcb_head_lock, NULL, 1137 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1138 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1139 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1140 } 1141 1142 mutex_init(&ixgbe->gen_lock, NULL, 1143 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1144 1145 mutex_init(&ixgbe->watchdog_lock, NULL, 1146 MUTEX_DRIVER, DDI_INTR_PRI(ixgbe->intr_pri)); 1147 } 1148 1149 /* 1150 * ixgbe_destroy_locks - Destroy locks. 1151 */ 1152 static void 1153 ixgbe_destroy_locks(ixgbe_t *ixgbe) 1154 { 1155 ixgbe_rx_ring_t *rx_ring; 1156 ixgbe_tx_ring_t *tx_ring; 1157 int i; 1158 1159 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1160 rx_ring = &ixgbe->rx_rings[i]; 1161 mutex_destroy(&rx_ring->rx_lock); 1162 } 1163 1164 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1165 tx_ring = &ixgbe->tx_rings[i]; 1166 mutex_destroy(&tx_ring->tx_lock); 1167 mutex_destroy(&tx_ring->recycle_lock); 1168 mutex_destroy(&tx_ring->tcb_head_lock); 1169 mutex_destroy(&tx_ring->tcb_tail_lock); 1170 } 1171 1172 mutex_destroy(&ixgbe->gen_lock); 1173 mutex_destroy(&ixgbe->watchdog_lock); 1174 } 1175 1176 static int 1177 ixgbe_resume(dev_info_t *devinfo) 1178 { 1179 ixgbe_t *ixgbe; 1180 int i; 1181 1182 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1183 if (ixgbe == NULL) 1184 return (DDI_FAILURE); 1185 1186 mutex_enter(&ixgbe->gen_lock); 1187 1188 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1189 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1190 mutex_exit(&ixgbe->gen_lock); 1191 return (DDI_FAILURE); 1192 } 1193 1194 /* 1195 * Enable and start the watchdog timer 1196 */ 1197 ixgbe_enable_watchdog_timer(ixgbe); 1198 } 1199 1200 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_SUSPENDED); 1201 1202 if (ixgbe->ixgbe_state & IXGBE_STARTED) { 1203 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1204 mac_tx_ring_update(ixgbe->mac_hdl, 1205 ixgbe->tx_rings[i].ring_handle); 1206 } 1207 } 1208 1209 mutex_exit(&ixgbe->gen_lock); 1210 1211 return (DDI_SUCCESS); 1212 } 1213 1214 static int 1215 ixgbe_suspend(dev_info_t *devinfo) 1216 { 1217 ixgbe_t *ixgbe; 1218 1219 ixgbe = (ixgbe_t *)ddi_get_driver_private(devinfo); 1220 if (ixgbe == NULL) 1221 return (DDI_FAILURE); 1222 1223 mutex_enter(&ixgbe->gen_lock); 1224 1225 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_SUSPENDED); 1226 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1227 mutex_exit(&ixgbe->gen_lock); 1228 return (DDI_SUCCESS); 1229 } 1230 ixgbe_stop(ixgbe, B_FALSE); 1231 1232 mutex_exit(&ixgbe->gen_lock); 1233 1234 /* 1235 * Disable and stop the watchdog timer 1236 */ 1237 ixgbe_disable_watchdog_timer(ixgbe); 1238 1239 return (DDI_SUCCESS); 1240 } 1241 1242 /* 1243 * ixgbe_init - Initialize the device. 1244 */ 1245 static int 1246 ixgbe_init(ixgbe_t *ixgbe) 1247 { 1248 struct ixgbe_hw *hw = &ixgbe->hw; 1249 1250 mutex_enter(&ixgbe->gen_lock); 1251 1252 /* 1253 * Reset chipset to put the hardware in a known state 1254 * before we try to do anything with the eeprom. 1255 */ 1256 if (ixgbe_reset_hw(hw) != IXGBE_SUCCESS) { 1257 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1258 goto init_fail; 1259 } 1260 1261 /* 1262 * Need to init eeprom before validating the checksum. 1263 */ 1264 if (ixgbe_init_eeprom_params(hw) < 0) { 1265 ixgbe_error(ixgbe, 1266 "Unable to intitialize the eeprom interface."); 1267 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1268 goto init_fail; 1269 } 1270 1271 /* 1272 * NVM validation 1273 */ 1274 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1275 /* 1276 * Some PCI-E parts fail the first check due to 1277 * the link being in sleep state. Call it again, 1278 * if it fails a second time it's a real issue. 1279 */ 1280 if (ixgbe_validate_eeprom_checksum(hw, NULL) < 0) { 1281 ixgbe_error(ixgbe, 1282 "Invalid NVM checksum. Please contact " 1283 "the vendor to update the NVM."); 1284 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1285 goto init_fail; 1286 } 1287 } 1288 1289 /* 1290 * Setup default flow control thresholds - enable/disable 1291 * & flow control type is controlled by ixgbe.conf 1292 */ 1293 hw->fc.high_water[0] = DEFAULT_FCRTH; 1294 hw->fc.low_water[0] = DEFAULT_FCRTL; 1295 hw->fc.pause_time = DEFAULT_FCPAUSE; 1296 hw->fc.send_xon = B_TRUE; 1297 1298 /* 1299 * Initialize link settings 1300 */ 1301 (void) ixgbe_driver_setup_link(ixgbe, B_FALSE); 1302 1303 /* 1304 * Initialize the chipset hardware 1305 */ 1306 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1307 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1308 goto init_fail; 1309 } 1310 1311 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1312 goto init_fail; 1313 } 1314 1315 mutex_exit(&ixgbe->gen_lock); 1316 return (IXGBE_SUCCESS); 1317 1318 init_fail: 1319 /* 1320 * Reset PHY 1321 */ 1322 (void) ixgbe_reset_phy(hw); 1323 1324 mutex_exit(&ixgbe->gen_lock); 1325 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1326 return (IXGBE_FAILURE); 1327 } 1328 1329 /* 1330 * ixgbe_chip_start - Initialize and start the chipset hardware. 1331 */ 1332 static int 1333 ixgbe_chip_start(ixgbe_t *ixgbe) 1334 { 1335 struct ixgbe_hw *hw = &ixgbe->hw; 1336 int ret_val, i; 1337 1338 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1339 1340 /* 1341 * Get the mac address 1342 * This function should handle SPARC case correctly. 1343 */ 1344 if (!ixgbe_find_mac_address(ixgbe)) { 1345 ixgbe_error(ixgbe, "Failed to get the mac address"); 1346 return (IXGBE_FAILURE); 1347 } 1348 1349 /* 1350 * Validate the mac address 1351 */ 1352 (void) ixgbe_init_rx_addrs(hw); 1353 if (!is_valid_mac_addr(hw->mac.addr)) { 1354 ixgbe_error(ixgbe, "Invalid mac address"); 1355 return (IXGBE_FAILURE); 1356 } 1357 1358 /* 1359 * Configure/Initialize hardware 1360 */ 1361 ret_val = ixgbe_init_hw(hw); 1362 if (ret_val != IXGBE_SUCCESS) { 1363 if (ret_val == IXGBE_ERR_EEPROM_VERSION) { 1364 ixgbe_error(ixgbe, 1365 "This 82599 device is pre-release and contains" 1366 " outdated firmware, please contact your hardware" 1367 " vendor for a replacement."); 1368 } else { 1369 ixgbe_error(ixgbe, "Failed to initialize hardware"); 1370 return (IXGBE_FAILURE); 1371 } 1372 } 1373 1374 /* 1375 * Re-enable relaxed ordering for performance. It is disabled 1376 * by default in the hardware init. 1377 */ 1378 if (ixgbe->relax_order_enable == B_TRUE) 1379 ixgbe_enable_relaxed_ordering(hw); 1380 1381 /* 1382 * Setup adapter interrupt vectors 1383 */ 1384 ixgbe_setup_adapter_vector(ixgbe); 1385 1386 /* 1387 * Initialize unicast addresses. 1388 */ 1389 ixgbe_init_unicst(ixgbe); 1390 1391 /* 1392 * Setup and initialize the mctable structures. 1393 */ 1394 ixgbe_setup_multicst(ixgbe); 1395 1396 /* 1397 * Set interrupt throttling rate 1398 */ 1399 for (i = 0; i < ixgbe->intr_cnt; i++) { 1400 IXGBE_WRITE_REG(hw, IXGBE_EITR(i), ixgbe->intr_throttling[i]); 1401 } 1402 1403 /* 1404 * Save the state of the phy 1405 */ 1406 ixgbe_get_hw_state(ixgbe); 1407 1408 /* 1409 * Make sure driver has control 1410 */ 1411 ixgbe_get_driver_control(hw); 1412 1413 return (IXGBE_SUCCESS); 1414 } 1415 1416 /* 1417 * ixgbe_chip_stop - Stop the chipset hardware 1418 */ 1419 static void 1420 ixgbe_chip_stop(ixgbe_t *ixgbe) 1421 { 1422 struct ixgbe_hw *hw = &ixgbe->hw; 1423 1424 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1425 1426 /* 1427 * Tell firmware driver is no longer in control 1428 */ 1429 ixgbe_release_driver_control(hw); 1430 1431 /* 1432 * Reset the chipset 1433 */ 1434 (void) ixgbe_reset_hw(hw); 1435 1436 /* 1437 * Reset PHY 1438 */ 1439 (void) ixgbe_reset_phy(hw); 1440 } 1441 1442 /* 1443 * ixgbe_reset - Reset the chipset and re-start the driver. 1444 * 1445 * It involves stopping and re-starting the chipset, 1446 * and re-configuring the rx/tx rings. 1447 */ 1448 static int 1449 ixgbe_reset(ixgbe_t *ixgbe) 1450 { 1451 int i; 1452 1453 /* 1454 * Disable and stop the watchdog timer 1455 */ 1456 ixgbe_disable_watchdog_timer(ixgbe); 1457 1458 mutex_enter(&ixgbe->gen_lock); 1459 1460 ASSERT(ixgbe->ixgbe_state & IXGBE_STARTED); 1461 atomic_and_32(&ixgbe->ixgbe_state, ~IXGBE_STARTED); 1462 1463 ixgbe_stop(ixgbe, B_FALSE); 1464 1465 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1466 mutex_exit(&ixgbe->gen_lock); 1467 return (IXGBE_FAILURE); 1468 } 1469 1470 /* 1471 * After resetting, need to recheck the link status. 1472 */ 1473 ixgbe->link_check_complete = B_FALSE; 1474 ixgbe->link_check_hrtime = gethrtime() + 1475 (IXGBE_LINK_UP_TIME * 100000000ULL); 1476 1477 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STARTED); 1478 1479 if (!(ixgbe->ixgbe_state & IXGBE_SUSPENDED)) { 1480 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1481 mac_tx_ring_update(ixgbe->mac_hdl, 1482 ixgbe->tx_rings[i].ring_handle); 1483 } 1484 } 1485 1486 mutex_exit(&ixgbe->gen_lock); 1487 1488 /* 1489 * Enable and start the watchdog timer 1490 */ 1491 ixgbe_enable_watchdog_timer(ixgbe); 1492 1493 return (IXGBE_SUCCESS); 1494 } 1495 1496 /* 1497 * ixgbe_tx_clean - Clean the pending transmit packets and DMA resources. 1498 */ 1499 static void 1500 ixgbe_tx_clean(ixgbe_t *ixgbe) 1501 { 1502 ixgbe_tx_ring_t *tx_ring; 1503 tx_control_block_t *tcb; 1504 link_list_t pending_list; 1505 uint32_t desc_num; 1506 int i, j; 1507 1508 LINK_LIST_INIT(&pending_list); 1509 1510 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1511 tx_ring = &ixgbe->tx_rings[i]; 1512 1513 mutex_enter(&tx_ring->recycle_lock); 1514 1515 /* 1516 * Clean the pending tx data - the pending packets in the 1517 * work_list that have no chances to be transmitted again. 1518 * 1519 * We must ensure the chipset is stopped or the link is down 1520 * before cleaning the transmit packets. 1521 */ 1522 desc_num = 0; 1523 for (j = 0; j < tx_ring->ring_size; j++) { 1524 tcb = tx_ring->work_list[j]; 1525 if (tcb != NULL) { 1526 desc_num += tcb->desc_num; 1527 1528 tx_ring->work_list[j] = NULL; 1529 1530 ixgbe_free_tcb(tcb); 1531 1532 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1533 } 1534 } 1535 1536 if (desc_num > 0) { 1537 atomic_add_32(&tx_ring->tbd_free, desc_num); 1538 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1539 1540 /* 1541 * Reset the head and tail pointers of the tbd ring; 1542 * Reset the writeback head if it's enable. 1543 */ 1544 tx_ring->tbd_head = 0; 1545 tx_ring->tbd_tail = 0; 1546 if (ixgbe->tx_head_wb_enable) 1547 *tx_ring->tbd_head_wb = 0; 1548 1549 IXGBE_WRITE_REG(&ixgbe->hw, 1550 IXGBE_TDH(tx_ring->index), 0); 1551 IXGBE_WRITE_REG(&ixgbe->hw, 1552 IXGBE_TDT(tx_ring->index), 0); 1553 } 1554 1555 mutex_exit(&tx_ring->recycle_lock); 1556 1557 /* 1558 * Add the tx control blocks in the pending list to 1559 * the free list. 1560 */ 1561 ixgbe_put_free_list(tx_ring, &pending_list); 1562 } 1563 } 1564 1565 /* 1566 * ixgbe_tx_drain - Drain the tx rings to allow pending packets to be 1567 * transmitted. 1568 */ 1569 static boolean_t 1570 ixgbe_tx_drain(ixgbe_t *ixgbe) 1571 { 1572 ixgbe_tx_ring_t *tx_ring; 1573 boolean_t done; 1574 int i, j; 1575 1576 /* 1577 * Wait for a specific time to allow pending tx packets 1578 * to be transmitted. 1579 * 1580 * Check the counter tbd_free to see if transmission is done. 1581 * No lock protection is needed here. 1582 * 1583 * Return B_TRUE if all pending packets have been transmitted; 1584 * Otherwise return B_FALSE; 1585 */ 1586 for (i = 0; i < TX_DRAIN_TIME; i++) { 1587 1588 done = B_TRUE; 1589 for (j = 0; j < ixgbe->num_tx_rings; j++) { 1590 tx_ring = &ixgbe->tx_rings[j]; 1591 done = done && 1592 (tx_ring->tbd_free == tx_ring->ring_size); 1593 } 1594 1595 if (done) 1596 break; 1597 1598 msec_delay(1); 1599 } 1600 1601 return (done); 1602 } 1603 1604 /* 1605 * ixgbe_rx_drain - Wait for all rx buffers to be released by upper layer. 1606 */ 1607 static boolean_t 1608 ixgbe_rx_drain(ixgbe_t *ixgbe) 1609 { 1610 boolean_t done = B_TRUE; 1611 int i; 1612 1613 /* 1614 * Polling the rx free list to check if those rx buffers held by 1615 * the upper layer are released. 1616 * 1617 * Check the counter rcb_free to see if all pending buffers are 1618 * released. No lock protection is needed here. 1619 * 1620 * Return B_TRUE if all pending buffers have been released; 1621 * Otherwise return B_FALSE; 1622 */ 1623 for (i = 0; i < RX_DRAIN_TIME; i++) { 1624 done = (ixgbe->rcb_pending == 0); 1625 1626 if (done) 1627 break; 1628 1629 msec_delay(1); 1630 } 1631 1632 return (done); 1633 } 1634 1635 /* 1636 * ixgbe_start - Start the driver/chipset. 1637 */ 1638 int 1639 ixgbe_start(ixgbe_t *ixgbe, boolean_t alloc_buffer) 1640 { 1641 int i; 1642 1643 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1644 1645 if (alloc_buffer) { 1646 if (ixgbe_alloc_rx_data(ixgbe) != IXGBE_SUCCESS) { 1647 ixgbe_error(ixgbe, 1648 "Failed to allocate software receive rings"); 1649 return (IXGBE_FAILURE); 1650 } 1651 1652 /* Allocate buffers for all the rx/tx rings */ 1653 if (ixgbe_alloc_dma(ixgbe) != IXGBE_SUCCESS) { 1654 ixgbe_error(ixgbe, "Failed to allocate DMA resource"); 1655 return (IXGBE_FAILURE); 1656 } 1657 1658 ixgbe->tx_ring_init = B_TRUE; 1659 } else { 1660 ixgbe->tx_ring_init = B_FALSE; 1661 } 1662 1663 for (i = 0; i < ixgbe->num_rx_rings; i++) 1664 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1665 for (i = 0; i < ixgbe->num_tx_rings; i++) 1666 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1667 1668 /* 1669 * Start the chipset hardware 1670 */ 1671 if (ixgbe_chip_start(ixgbe) != IXGBE_SUCCESS) { 1672 ixgbe_fm_ereport(ixgbe, DDI_FM_DEVICE_INVAL_STATE); 1673 goto start_failure; 1674 } 1675 1676 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1677 goto start_failure; 1678 } 1679 1680 /* 1681 * Setup the rx/tx rings 1682 */ 1683 ixgbe_setup_rings(ixgbe); 1684 1685 /* 1686 * ixgbe_start() will be called when resetting, however if reset 1687 * happens, we need to clear the ERROR, STALL and OVERTEMP flags 1688 * before enabling the interrupts. 1689 */ 1690 atomic_and_32(&ixgbe->ixgbe_state, ~(IXGBE_ERROR 1691 | IXGBE_STALL| IXGBE_OVERTEMP)); 1692 1693 /* 1694 * Enable adapter interrupts 1695 * The interrupts must be enabled after the driver state is START 1696 */ 1697 ixgbe_enable_adapter_interrupts(ixgbe); 1698 1699 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1700 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1701 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1702 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1703 1704 return (IXGBE_SUCCESS); 1705 1706 start_failure: 1707 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1708 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1709 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1710 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1711 1712 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1713 1714 return (IXGBE_FAILURE); 1715 } 1716 1717 /* 1718 * ixgbe_stop - Stop the driver/chipset. 1719 */ 1720 void 1721 ixgbe_stop(ixgbe_t *ixgbe, boolean_t free_buffer) 1722 { 1723 int i; 1724 1725 ASSERT(mutex_owned(&ixgbe->gen_lock)); 1726 1727 /* 1728 * Disable the adapter interrupts 1729 */ 1730 ixgbe_disable_adapter_interrupts(ixgbe); 1731 1732 /* 1733 * Drain the pending tx packets 1734 */ 1735 (void) ixgbe_tx_drain(ixgbe); 1736 1737 for (i = 0; i < ixgbe->num_rx_rings; i++) 1738 mutex_enter(&ixgbe->rx_rings[i].rx_lock); 1739 for (i = 0; i < ixgbe->num_tx_rings; i++) 1740 mutex_enter(&ixgbe->tx_rings[i].tx_lock); 1741 1742 /* 1743 * Stop the chipset hardware 1744 */ 1745 ixgbe_chip_stop(ixgbe); 1746 1747 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 1748 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1749 } 1750 1751 /* 1752 * Clean the pending tx data/resources 1753 */ 1754 ixgbe_tx_clean(ixgbe); 1755 1756 for (i = ixgbe->num_tx_rings - 1; i >= 0; i--) 1757 mutex_exit(&ixgbe->tx_rings[i].tx_lock); 1758 for (i = ixgbe->num_rx_rings - 1; i >= 0; i--) 1759 mutex_exit(&ixgbe->rx_rings[i].rx_lock); 1760 1761 if (ixgbe->link_state == LINK_STATE_UP) { 1762 ixgbe->link_state = LINK_STATE_UNKNOWN; 1763 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 1764 } 1765 1766 if (free_buffer) { 1767 /* 1768 * Release the DMA/memory resources of rx/tx rings 1769 */ 1770 ixgbe_free_dma(ixgbe); 1771 ixgbe_free_rx_data(ixgbe); 1772 } 1773 } 1774 1775 /* 1776 * ixgbe_cbfunc - Driver interface for generic DDI callbacks 1777 */ 1778 /* ARGSUSED */ 1779 static int 1780 ixgbe_cbfunc(dev_info_t *dip, ddi_cb_action_t cbaction, void *cbarg, 1781 void *arg1, void *arg2) 1782 { 1783 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 1784 1785 switch (cbaction) { 1786 /* IRM callback */ 1787 int count; 1788 case DDI_CB_INTR_ADD: 1789 case DDI_CB_INTR_REMOVE: 1790 count = (int)(uintptr_t)cbarg; 1791 ASSERT(ixgbe->intr_type == DDI_INTR_TYPE_MSIX); 1792 DTRACE_PROBE2(ixgbe__irm__callback, int, count, 1793 int, ixgbe->intr_cnt); 1794 if (ixgbe_intr_adjust(ixgbe, cbaction, count) != 1795 DDI_SUCCESS) { 1796 ixgbe_error(ixgbe, 1797 "IRM CB: Failed to adjust interrupts"); 1798 goto cb_fail; 1799 } 1800 break; 1801 default: 1802 IXGBE_DEBUGLOG_1(ixgbe, "DDI CB: action 0x%x NOT supported", 1803 cbaction); 1804 return (DDI_ENOTSUP); 1805 } 1806 return (DDI_SUCCESS); 1807 cb_fail: 1808 return (DDI_FAILURE); 1809 } 1810 1811 /* 1812 * ixgbe_intr_adjust - Adjust interrupt to respond to IRM request. 1813 */ 1814 static int 1815 ixgbe_intr_adjust(ixgbe_t *ixgbe, ddi_cb_action_t cbaction, int count) 1816 { 1817 int i, rc, actual; 1818 1819 if (count == 0) 1820 return (DDI_SUCCESS); 1821 1822 if ((cbaction == DDI_CB_INTR_ADD && 1823 ixgbe->intr_cnt + count > ixgbe->intr_cnt_max) || 1824 (cbaction == DDI_CB_INTR_REMOVE && 1825 ixgbe->intr_cnt - count < ixgbe->intr_cnt_min)) 1826 return (DDI_FAILURE); 1827 1828 if (!(ixgbe->ixgbe_state & IXGBE_STARTED)) { 1829 return (DDI_FAILURE); 1830 } 1831 1832 for (i = 0; i < ixgbe->num_rx_rings; i++) 1833 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, NULL); 1834 for (i = 0; i < ixgbe->num_tx_rings; i++) 1835 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, NULL); 1836 1837 mutex_enter(&ixgbe->gen_lock); 1838 ixgbe->ixgbe_state &= ~IXGBE_STARTED; 1839 ixgbe->ixgbe_state |= IXGBE_INTR_ADJUST; 1840 ixgbe->ixgbe_state |= IXGBE_SUSPENDED; 1841 mac_link_update(ixgbe->mac_hdl, LINK_STATE_UNKNOWN); 1842 1843 ixgbe_stop(ixgbe, B_FALSE); 1844 /* 1845 * Disable interrupts 1846 */ 1847 if (ixgbe->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 1848 rc = ixgbe_disable_intrs(ixgbe); 1849 ASSERT(rc == IXGBE_SUCCESS); 1850 } 1851 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ENABLE_INTR; 1852 1853 /* 1854 * Remove interrupt handlers 1855 */ 1856 if (ixgbe->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 1857 ixgbe_rem_intr_handlers(ixgbe); 1858 } 1859 ixgbe->attach_progress &= ~ATTACH_PROGRESS_ADD_INTR; 1860 1861 /* 1862 * Clear vect_map 1863 */ 1864 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 1865 switch (cbaction) { 1866 case DDI_CB_INTR_ADD: 1867 rc = ddi_intr_alloc(ixgbe->dip, ixgbe->htable, 1868 DDI_INTR_TYPE_MSIX, ixgbe->intr_cnt, count, &actual, 1869 DDI_INTR_ALLOC_NORMAL); 1870 if (rc != DDI_SUCCESS || actual != count) { 1871 ixgbe_log(ixgbe, "Adjust interrupts failed." 1872 "return: %d, irm cb size: %d, actual: %d", 1873 rc, count, actual); 1874 goto intr_adjust_fail; 1875 } 1876 ixgbe->intr_cnt += count; 1877 break; 1878 1879 case DDI_CB_INTR_REMOVE: 1880 for (i = ixgbe->intr_cnt - count; 1881 i < ixgbe->intr_cnt; i ++) { 1882 rc = ddi_intr_free(ixgbe->htable[i]); 1883 ixgbe->htable[i] = NULL; 1884 if (rc != DDI_SUCCESS) { 1885 ixgbe_log(ixgbe, "Adjust interrupts failed." 1886 "return: %d, irm cb size: %d, actual: %d", 1887 rc, count, actual); 1888 goto intr_adjust_fail; 1889 } 1890 } 1891 ixgbe->intr_cnt -= count; 1892 break; 1893 } 1894 1895 /* 1896 * Get priority for first vector, assume remaining are all the same 1897 */ 1898 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 1899 if (rc != DDI_SUCCESS) { 1900 ixgbe_log(ixgbe, 1901 "Get interrupt priority failed: %d", rc); 1902 goto intr_adjust_fail; 1903 } 1904 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 1905 if (rc != DDI_SUCCESS) { 1906 ixgbe_log(ixgbe, "Get interrupt cap failed: %d", rc); 1907 goto intr_adjust_fail; 1908 } 1909 ixgbe->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 1910 1911 /* 1912 * Map rings to interrupt vectors 1913 */ 1914 if (ixgbe_map_intrs_to_vectors(ixgbe) != IXGBE_SUCCESS) { 1915 ixgbe_error(ixgbe, 1916 "IRM CB: Failed to map interrupts to vectors"); 1917 goto intr_adjust_fail; 1918 } 1919 1920 /* 1921 * Add interrupt handlers 1922 */ 1923 if (ixgbe_add_intr_handlers(ixgbe) != IXGBE_SUCCESS) { 1924 ixgbe_error(ixgbe, "IRM CB: Failed to add interrupt handlers"); 1925 goto intr_adjust_fail; 1926 } 1927 ixgbe->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 1928 1929 /* 1930 * Now that mutex locks are initialized, and the chip is also 1931 * initialized, enable interrupts. 1932 */ 1933 if (ixgbe_enable_intrs(ixgbe) != IXGBE_SUCCESS) { 1934 ixgbe_error(ixgbe, "IRM CB: Failed to enable DDI interrupts"); 1935 goto intr_adjust_fail; 1936 } 1937 ixgbe->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 1938 if (ixgbe_start(ixgbe, B_FALSE) != IXGBE_SUCCESS) { 1939 ixgbe_error(ixgbe, "IRM CB: Failed to start"); 1940 goto intr_adjust_fail; 1941 } 1942 ixgbe->ixgbe_state &= ~IXGBE_INTR_ADJUST; 1943 ixgbe->ixgbe_state &= ~IXGBE_SUSPENDED; 1944 ixgbe->ixgbe_state |= IXGBE_STARTED; 1945 mutex_exit(&ixgbe->gen_lock); 1946 1947 for (i = 0; i < ixgbe->num_rx_rings; i++) { 1948 mac_ring_intr_set(ixgbe->rx_rings[i].ring_handle, 1949 ixgbe->htable[ixgbe->rx_rings[i].intr_vector]); 1950 } 1951 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1952 mac_ring_intr_set(ixgbe->tx_rings[i].ring_handle, 1953 ixgbe->htable[ixgbe->tx_rings[i].intr_vector]); 1954 } 1955 1956 /* Wakeup all Tx rings */ 1957 for (i = 0; i < ixgbe->num_tx_rings; i++) { 1958 mac_tx_ring_update(ixgbe->mac_hdl, 1959 ixgbe->tx_rings[i].ring_handle); 1960 } 1961 1962 IXGBE_DEBUGLOG_3(ixgbe, 1963 "IRM CB: interrupts new value: 0x%x(0x%x:0x%x).", 1964 ixgbe->intr_cnt, ixgbe->intr_cnt_min, ixgbe->intr_cnt_max); 1965 return (DDI_SUCCESS); 1966 1967 intr_adjust_fail: 1968 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 1969 mutex_exit(&ixgbe->gen_lock); 1970 return (DDI_FAILURE); 1971 } 1972 1973 /* 1974 * ixgbe_intr_cb_register - Register interrupt callback function. 1975 */ 1976 static int 1977 ixgbe_intr_cb_register(ixgbe_t *ixgbe) 1978 { 1979 if (ddi_cb_register(ixgbe->dip, DDI_CB_FLAG_INTR, ixgbe_cbfunc, 1980 ixgbe, NULL, &ixgbe->cb_hdl) != DDI_SUCCESS) { 1981 return (IXGBE_FAILURE); 1982 } 1983 IXGBE_DEBUGLOG_0(ixgbe, "Interrupt callback function registered."); 1984 return (IXGBE_SUCCESS); 1985 } 1986 1987 /* 1988 * ixgbe_alloc_rings - Allocate memory space for rx/tx rings. 1989 */ 1990 static int 1991 ixgbe_alloc_rings(ixgbe_t *ixgbe) 1992 { 1993 /* 1994 * Allocate memory space for rx rings 1995 */ 1996 ixgbe->rx_rings = kmem_zalloc( 1997 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings, 1998 KM_NOSLEEP); 1999 2000 if (ixgbe->rx_rings == NULL) { 2001 return (IXGBE_FAILURE); 2002 } 2003 2004 /* 2005 * Allocate memory space for tx rings 2006 */ 2007 ixgbe->tx_rings = kmem_zalloc( 2008 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings, 2009 KM_NOSLEEP); 2010 2011 if (ixgbe->tx_rings == NULL) { 2012 kmem_free(ixgbe->rx_rings, 2013 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2014 ixgbe->rx_rings = NULL; 2015 return (IXGBE_FAILURE); 2016 } 2017 2018 /* 2019 * Allocate memory space for rx ring groups 2020 */ 2021 ixgbe->rx_groups = kmem_zalloc( 2022 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups, 2023 KM_NOSLEEP); 2024 2025 if (ixgbe->rx_groups == NULL) { 2026 kmem_free(ixgbe->rx_rings, 2027 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2028 kmem_free(ixgbe->tx_rings, 2029 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2030 ixgbe->rx_rings = NULL; 2031 ixgbe->tx_rings = NULL; 2032 return (IXGBE_FAILURE); 2033 } 2034 2035 return (IXGBE_SUCCESS); 2036 } 2037 2038 /* 2039 * ixgbe_free_rings - Free the memory space of rx/tx rings. 2040 */ 2041 static void 2042 ixgbe_free_rings(ixgbe_t *ixgbe) 2043 { 2044 if (ixgbe->rx_rings != NULL) { 2045 kmem_free(ixgbe->rx_rings, 2046 sizeof (ixgbe_rx_ring_t) * ixgbe->num_rx_rings); 2047 ixgbe->rx_rings = NULL; 2048 } 2049 2050 if (ixgbe->tx_rings != NULL) { 2051 kmem_free(ixgbe->tx_rings, 2052 sizeof (ixgbe_tx_ring_t) * ixgbe->num_tx_rings); 2053 ixgbe->tx_rings = NULL; 2054 } 2055 2056 if (ixgbe->rx_groups != NULL) { 2057 kmem_free(ixgbe->rx_groups, 2058 sizeof (ixgbe_rx_group_t) * ixgbe->num_rx_groups); 2059 ixgbe->rx_groups = NULL; 2060 } 2061 } 2062 2063 static int 2064 ixgbe_alloc_rx_data(ixgbe_t *ixgbe) 2065 { 2066 ixgbe_rx_ring_t *rx_ring; 2067 int i; 2068 2069 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2070 rx_ring = &ixgbe->rx_rings[i]; 2071 if (ixgbe_alloc_rx_ring_data(rx_ring) != IXGBE_SUCCESS) 2072 goto alloc_rx_rings_failure; 2073 } 2074 return (IXGBE_SUCCESS); 2075 2076 alloc_rx_rings_failure: 2077 ixgbe_free_rx_data(ixgbe); 2078 return (IXGBE_FAILURE); 2079 } 2080 2081 static void 2082 ixgbe_free_rx_data(ixgbe_t *ixgbe) 2083 { 2084 ixgbe_rx_ring_t *rx_ring; 2085 ixgbe_rx_data_t *rx_data; 2086 int i; 2087 2088 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2089 rx_ring = &ixgbe->rx_rings[i]; 2090 2091 mutex_enter(&ixgbe->rx_pending_lock); 2092 rx_data = rx_ring->rx_data; 2093 2094 if (rx_data != NULL) { 2095 rx_data->flag |= IXGBE_RX_STOPPED; 2096 2097 if (rx_data->rcb_pending == 0) { 2098 ixgbe_free_rx_ring_data(rx_data); 2099 rx_ring->rx_data = NULL; 2100 } 2101 } 2102 2103 mutex_exit(&ixgbe->rx_pending_lock); 2104 } 2105 } 2106 2107 /* 2108 * ixgbe_setup_rings - Setup rx/tx rings. 2109 */ 2110 static void 2111 ixgbe_setup_rings(ixgbe_t *ixgbe) 2112 { 2113 /* 2114 * Setup the rx/tx rings, including the following: 2115 * 2116 * 1. Setup the descriptor ring and the control block buffers; 2117 * 2. Initialize necessary registers for receive/transmit; 2118 * 3. Initialize software pointers/parameters for receive/transmit; 2119 */ 2120 ixgbe_setup_rx(ixgbe); 2121 2122 ixgbe_setup_tx(ixgbe); 2123 } 2124 2125 static void 2126 ixgbe_setup_rx_ring(ixgbe_rx_ring_t *rx_ring) 2127 { 2128 ixgbe_t *ixgbe = rx_ring->ixgbe; 2129 ixgbe_rx_data_t *rx_data = rx_ring->rx_data; 2130 struct ixgbe_hw *hw = &ixgbe->hw; 2131 rx_control_block_t *rcb; 2132 union ixgbe_adv_rx_desc *rbd; 2133 uint32_t size; 2134 uint32_t buf_low; 2135 uint32_t buf_high; 2136 uint32_t reg_val; 2137 int i; 2138 2139 ASSERT(mutex_owned(&rx_ring->rx_lock)); 2140 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2141 2142 for (i = 0; i < ixgbe->rx_ring_size; i++) { 2143 rcb = rx_data->work_list[i]; 2144 rbd = &rx_data->rbd_ring[i]; 2145 2146 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 2147 rbd->read.hdr_addr = NULL; 2148 } 2149 2150 /* 2151 * Initialize the length register 2152 */ 2153 size = rx_data->ring_size * sizeof (union ixgbe_adv_rx_desc); 2154 IXGBE_WRITE_REG(hw, IXGBE_RDLEN(rx_ring->hw_index), size); 2155 2156 /* 2157 * Initialize the base address registers 2158 */ 2159 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 2160 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 2161 IXGBE_WRITE_REG(hw, IXGBE_RDBAH(rx_ring->hw_index), buf_high); 2162 IXGBE_WRITE_REG(hw, IXGBE_RDBAL(rx_ring->hw_index), buf_low); 2163 2164 /* 2165 * Setup head & tail pointers 2166 */ 2167 IXGBE_WRITE_REG(hw, IXGBE_RDT(rx_ring->hw_index), 2168 rx_data->ring_size - 1); 2169 IXGBE_WRITE_REG(hw, IXGBE_RDH(rx_ring->hw_index), 0); 2170 2171 rx_data->rbd_next = 0; 2172 rx_data->lro_first = 0; 2173 2174 /* 2175 * Setup the Receive Descriptor Control Register (RXDCTL) 2176 * PTHRESH=32 descriptors (half the internal cache) 2177 * HTHRESH=0 descriptors (to minimize latency on fetch) 2178 * WTHRESH defaults to 1 (writeback each descriptor) 2179 */ 2180 reg_val = IXGBE_READ_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index)); 2181 reg_val |= IXGBE_RXDCTL_ENABLE; /* enable queue */ 2182 2183 /* Not a valid value for 82599 or X540 */ 2184 if (hw->mac.type == ixgbe_mac_82598EB) { 2185 reg_val |= 0x0020; /* pthresh */ 2186 } 2187 IXGBE_WRITE_REG(hw, IXGBE_RXDCTL(rx_ring->hw_index), reg_val); 2188 2189 if (hw->mac.type == ixgbe_mac_82599EB || 2190 hw->mac.type == ixgbe_mac_X540) { 2191 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2192 reg_val |= (IXGBE_RDRXCTL_CRCSTRIP | IXGBE_RDRXCTL_AGGDIS); 2193 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2194 } 2195 2196 /* 2197 * Setup the Split and Replication Receive Control Register. 2198 * Set the rx buffer size and the advanced descriptor type. 2199 */ 2200 reg_val = (ixgbe->rx_buf_size >> IXGBE_SRRCTL_BSIZEPKT_SHIFT) | 2201 IXGBE_SRRCTL_DESCTYPE_ADV_ONEBUF; 2202 reg_val |= IXGBE_SRRCTL_DROP_EN; 2203 IXGBE_WRITE_REG(hw, IXGBE_SRRCTL(rx_ring->hw_index), reg_val); 2204 } 2205 2206 static void 2207 ixgbe_setup_rx(ixgbe_t *ixgbe) 2208 { 2209 ixgbe_rx_ring_t *rx_ring; 2210 struct ixgbe_hw *hw = &ixgbe->hw; 2211 uint32_t reg_val; 2212 uint32_t ring_mapping; 2213 uint32_t i, index; 2214 uint32_t psrtype_rss_bit; 2215 2216 /* PSRTYPE must be configured for 82599 */ 2217 if (ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ && 2218 ixgbe->classify_mode != IXGBE_CLASSIFY_VMDQ_RSS) { 2219 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2220 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2221 reg_val |= IXGBE_PSRTYPE_L2HDR; 2222 reg_val |= 0x80000000; 2223 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(0), reg_val); 2224 } else { 2225 if (ixgbe->num_rx_groups > 32) { 2226 psrtype_rss_bit = 0x20000000; 2227 } else { 2228 psrtype_rss_bit = 0x40000000; 2229 } 2230 for (i = 0; i < ixgbe->capab->max_rx_grp_num; i++) { 2231 reg_val = IXGBE_PSRTYPE_TCPHDR | IXGBE_PSRTYPE_UDPHDR | 2232 IXGBE_PSRTYPE_IPV4HDR | IXGBE_PSRTYPE_IPV6HDR; 2233 reg_val |= IXGBE_PSRTYPE_L2HDR; 2234 reg_val |= psrtype_rss_bit; 2235 IXGBE_WRITE_REG(hw, IXGBE_PSRTYPE(i), reg_val); 2236 } 2237 } 2238 2239 /* 2240 * Set filter control in FCTRL to accept broadcast packets and do 2241 * not pass pause frames to host. Flow control settings are already 2242 * in this register, so preserve them. 2243 */ 2244 reg_val = IXGBE_READ_REG(hw, IXGBE_FCTRL); 2245 reg_val |= IXGBE_FCTRL_BAM; /* broadcast accept mode */ 2246 reg_val |= IXGBE_FCTRL_DPF; /* discard pause frames */ 2247 IXGBE_WRITE_REG(hw, IXGBE_FCTRL, reg_val); 2248 2249 /* 2250 * Hardware checksum settings 2251 */ 2252 if (ixgbe->rx_hcksum_enable) { 2253 reg_val = IXGBE_RXCSUM_IPPCSE; /* IP checksum */ 2254 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, reg_val); 2255 } 2256 2257 /* 2258 * Setup VMDq and RSS for multiple receive queues 2259 */ 2260 switch (ixgbe->classify_mode) { 2261 case IXGBE_CLASSIFY_RSS: 2262 /* 2263 * One group, only RSS is needed when more than 2264 * one ring enabled. 2265 */ 2266 ixgbe_setup_rss(ixgbe); 2267 break; 2268 2269 case IXGBE_CLASSIFY_VMDQ: 2270 /* 2271 * Multiple groups, each group has one ring, 2272 * only VMDq is needed. 2273 */ 2274 ixgbe_setup_vmdq(ixgbe); 2275 break; 2276 2277 case IXGBE_CLASSIFY_VMDQ_RSS: 2278 /* 2279 * Multiple groups and multiple rings, both 2280 * VMDq and RSS are needed. 2281 */ 2282 ixgbe_setup_vmdq_rss(ixgbe); 2283 break; 2284 2285 default: 2286 break; 2287 } 2288 2289 /* 2290 * Enable the receive unit. This must be done after filter 2291 * control is set in FCTRL. 2292 */ 2293 reg_val = (IXGBE_RXCTRL_RXEN /* Enable Receive Unit */ 2294 | IXGBE_RXCTRL_DMBYPS); /* descriptor monitor bypass */ 2295 IXGBE_WRITE_REG(hw, IXGBE_RXCTRL, reg_val); 2296 2297 /* 2298 * ixgbe_setup_rx_ring must be called after configuring RXCTRL 2299 */ 2300 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2301 rx_ring = &ixgbe->rx_rings[i]; 2302 ixgbe_setup_rx_ring(rx_ring); 2303 } 2304 2305 /* 2306 * Setup the per-ring statistics mapping. 2307 */ 2308 ring_mapping = 0; 2309 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2310 index = ixgbe->rx_rings[i].hw_index; 2311 ring_mapping = IXGBE_READ_REG(hw, IXGBE_RQSMR(index >> 2)); 2312 ring_mapping |= (i & 0xF) << (8 * (index & 0x3)); 2313 IXGBE_WRITE_REG(hw, IXGBE_RQSMR(index >> 2), ring_mapping); 2314 } 2315 2316 /* 2317 * The Max Frame Size in MHADD/MAXFRS will be internally increased 2318 * by four bytes if the packet has a VLAN field, so includes MTU, 2319 * ethernet header and frame check sequence. 2320 * Register is MAXFRS in 82599. 2321 */ 2322 reg_val = (ixgbe->default_mtu + sizeof (struct ether_header) 2323 + ETHERFCSL) << IXGBE_MHADD_MFS_SHIFT; 2324 IXGBE_WRITE_REG(hw, IXGBE_MHADD, reg_val); 2325 2326 /* 2327 * Setup Jumbo Frame enable bit 2328 */ 2329 if (ixgbe->default_mtu > ETHERMTU) { 2330 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2331 reg_val |= IXGBE_HLREG0_JUMBOEN; 2332 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2333 } 2334 2335 /* 2336 * Setup RSC for multiple receive queues. 2337 */ 2338 if (ixgbe->lro_enable) { 2339 for (i = 0; i < ixgbe->num_rx_rings; i++) { 2340 /* 2341 * Make sure rx_buf_size * MAXDESC not greater 2342 * than 65535. 2343 * Intel recommends 4 for MAXDESC field value. 2344 */ 2345 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCCTL(i)); 2346 reg_val |= IXGBE_RSCCTL_RSCEN; 2347 if (ixgbe->rx_buf_size == IXGBE_PKG_BUF_16k) 2348 reg_val |= IXGBE_RSCCTL_MAXDESC_1; 2349 else 2350 reg_val |= IXGBE_RSCCTL_MAXDESC_4; 2351 IXGBE_WRITE_REG(hw, IXGBE_RSCCTL(i), reg_val); 2352 } 2353 2354 reg_val = IXGBE_READ_REG(hw, IXGBE_RSCDBU); 2355 reg_val |= IXGBE_RSCDBU_RSCACKDIS; 2356 IXGBE_WRITE_REG(hw, IXGBE_RSCDBU, reg_val); 2357 2358 reg_val = IXGBE_READ_REG(hw, IXGBE_RDRXCTL); 2359 reg_val |= IXGBE_RDRXCTL_RSCACKC; 2360 reg_val |= IXGBE_RDRXCTL_FCOE_WRFIX; 2361 reg_val &= ~IXGBE_RDRXCTL_RSCFRSTSIZE; 2362 2363 IXGBE_WRITE_REG(hw, IXGBE_RDRXCTL, reg_val); 2364 } 2365 } 2366 2367 static void 2368 ixgbe_setup_tx_ring(ixgbe_tx_ring_t *tx_ring) 2369 { 2370 ixgbe_t *ixgbe = tx_ring->ixgbe; 2371 struct ixgbe_hw *hw = &ixgbe->hw; 2372 uint32_t size; 2373 uint32_t buf_low; 2374 uint32_t buf_high; 2375 uint32_t reg_val; 2376 2377 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2378 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2379 2380 /* 2381 * Initialize the length register 2382 */ 2383 size = tx_ring->ring_size * sizeof (union ixgbe_adv_tx_desc); 2384 IXGBE_WRITE_REG(hw, IXGBE_TDLEN(tx_ring->index), size); 2385 2386 /* 2387 * Initialize the base address registers 2388 */ 2389 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2390 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2391 IXGBE_WRITE_REG(hw, IXGBE_TDBAL(tx_ring->index), buf_low); 2392 IXGBE_WRITE_REG(hw, IXGBE_TDBAH(tx_ring->index), buf_high); 2393 2394 /* 2395 * Setup head & tail pointers 2396 */ 2397 IXGBE_WRITE_REG(hw, IXGBE_TDH(tx_ring->index), 0); 2398 IXGBE_WRITE_REG(hw, IXGBE_TDT(tx_ring->index), 0); 2399 2400 /* 2401 * Setup head write-back 2402 */ 2403 if (ixgbe->tx_head_wb_enable) { 2404 /* 2405 * The memory of the head write-back is allocated using 2406 * the extra tbd beyond the tail of the tbd ring. 2407 */ 2408 tx_ring->tbd_head_wb = (uint32_t *) 2409 ((uintptr_t)tx_ring->tbd_area.address + size); 2410 *tx_ring->tbd_head_wb = 0; 2411 2412 buf_low = (uint32_t) 2413 (tx_ring->tbd_area.dma_address + size); 2414 buf_high = (uint32_t) 2415 ((tx_ring->tbd_area.dma_address + size) >> 32); 2416 2417 /* Set the head write-back enable bit */ 2418 buf_low |= IXGBE_TDWBAL_HEAD_WB_ENABLE; 2419 2420 IXGBE_WRITE_REG(hw, IXGBE_TDWBAL(tx_ring->index), buf_low); 2421 IXGBE_WRITE_REG(hw, IXGBE_TDWBAH(tx_ring->index), buf_high); 2422 2423 /* 2424 * Turn off relaxed ordering for head write back or it will 2425 * cause problems with the tx recycling 2426 */ 2427 2428 reg_val = (hw->mac.type == ixgbe_mac_82598EB) ? 2429 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL(tx_ring->index)) : 2430 IXGBE_READ_REG(hw, IXGBE_DCA_TXCTRL_82599(tx_ring->index)); 2431 reg_val &= ~IXGBE_DCA_TXCTRL_DESC_WRO_EN; 2432 if (hw->mac.type == ixgbe_mac_82598EB) { 2433 IXGBE_WRITE_REG(hw, 2434 IXGBE_DCA_TXCTRL(tx_ring->index), reg_val); 2435 } else { 2436 IXGBE_WRITE_REG(hw, 2437 IXGBE_DCA_TXCTRL_82599(tx_ring->index), reg_val); 2438 } 2439 } else { 2440 tx_ring->tbd_head_wb = NULL; 2441 } 2442 2443 tx_ring->tbd_head = 0; 2444 tx_ring->tbd_tail = 0; 2445 tx_ring->tbd_free = tx_ring->ring_size; 2446 2447 if (ixgbe->tx_ring_init == B_TRUE) { 2448 tx_ring->tcb_head = 0; 2449 tx_ring->tcb_tail = 0; 2450 tx_ring->tcb_free = tx_ring->free_list_size; 2451 } 2452 2453 /* 2454 * Initialize the s/w context structure 2455 */ 2456 bzero(&tx_ring->tx_context, sizeof (ixgbe_tx_context_t)); 2457 } 2458 2459 static void 2460 ixgbe_setup_tx(ixgbe_t *ixgbe) 2461 { 2462 struct ixgbe_hw *hw = &ixgbe->hw; 2463 ixgbe_tx_ring_t *tx_ring; 2464 uint32_t reg_val; 2465 uint32_t ring_mapping; 2466 int i; 2467 2468 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2469 tx_ring = &ixgbe->tx_rings[i]; 2470 ixgbe_setup_tx_ring(tx_ring); 2471 } 2472 2473 /* 2474 * Setup the per-ring statistics mapping. 2475 */ 2476 ring_mapping = 0; 2477 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2478 ring_mapping |= (i & 0xF) << (8 * (i & 0x3)); 2479 if ((i & 0x3) == 0x3) { 2480 switch (hw->mac.type) { 2481 case ixgbe_mac_82598EB: 2482 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), 2483 ring_mapping); 2484 break; 2485 2486 case ixgbe_mac_82599EB: 2487 case ixgbe_mac_X540: 2488 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), 2489 ring_mapping); 2490 break; 2491 2492 default: 2493 break; 2494 } 2495 2496 ring_mapping = 0; 2497 } 2498 } 2499 if (i & 0x3) { 2500 switch (hw->mac.type) { 2501 case ixgbe_mac_82598EB: 2502 IXGBE_WRITE_REG(hw, IXGBE_TQSMR(i >> 2), ring_mapping); 2503 break; 2504 2505 case ixgbe_mac_82599EB: 2506 case ixgbe_mac_X540: 2507 IXGBE_WRITE_REG(hw, IXGBE_TQSM(i >> 2), ring_mapping); 2508 break; 2509 2510 default: 2511 break; 2512 } 2513 } 2514 2515 /* 2516 * Enable CRC appending and TX padding (for short tx frames) 2517 */ 2518 reg_val = IXGBE_READ_REG(hw, IXGBE_HLREG0); 2519 reg_val |= IXGBE_HLREG0_TXCRCEN | IXGBE_HLREG0_TXPADEN; 2520 IXGBE_WRITE_REG(hw, IXGBE_HLREG0, reg_val); 2521 2522 /* 2523 * enable DMA for 82599 and X540 parts 2524 */ 2525 if (hw->mac.type == ixgbe_mac_82599EB || 2526 hw->mac.type == ixgbe_mac_X540) { 2527 /* DMATXCTL.TE must be set after all Tx config is complete */ 2528 reg_val = IXGBE_READ_REG(hw, IXGBE_DMATXCTL); 2529 reg_val |= IXGBE_DMATXCTL_TE; 2530 IXGBE_WRITE_REG(hw, IXGBE_DMATXCTL, reg_val); 2531 2532 /* Disable arbiter to set MTQC */ 2533 reg_val = IXGBE_READ_REG(hw, IXGBE_RTTDCS); 2534 reg_val |= IXGBE_RTTDCS_ARBDIS; 2535 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2536 IXGBE_WRITE_REG(hw, IXGBE_MTQC, IXGBE_MTQC_64Q_1PB); 2537 reg_val &= ~IXGBE_RTTDCS_ARBDIS; 2538 IXGBE_WRITE_REG(hw, IXGBE_RTTDCS, reg_val); 2539 } 2540 2541 /* 2542 * Enabling tx queues .. 2543 * For 82599 must be done after DMATXCTL.TE is set 2544 */ 2545 for (i = 0; i < ixgbe->num_tx_rings; i++) { 2546 tx_ring = &ixgbe->tx_rings[i]; 2547 reg_val = IXGBE_READ_REG(hw, IXGBE_TXDCTL(tx_ring->index)); 2548 reg_val |= IXGBE_TXDCTL_ENABLE; 2549 IXGBE_WRITE_REG(hw, IXGBE_TXDCTL(tx_ring->index), reg_val); 2550 } 2551 } 2552 2553 /* 2554 * ixgbe_setup_rss - Setup receive-side scaling feature. 2555 */ 2556 static void 2557 ixgbe_setup_rss(ixgbe_t *ixgbe) 2558 { 2559 struct ixgbe_hw *hw = &ixgbe->hw; 2560 uint32_t i, mrqc, rxcsum; 2561 uint32_t random; 2562 uint32_t reta; 2563 uint32_t ring_per_group; 2564 2565 /* 2566 * Fill out redirection table 2567 */ 2568 reta = 0; 2569 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2570 2571 for (i = 0; i < 128; i++) { 2572 reta = (reta << 8) | (i % ring_per_group) | 2573 ((i % ring_per_group) << 4); 2574 if ((i & 3) == 3) 2575 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2576 } 2577 2578 /* 2579 * Fill out hash function seeds with a random constant 2580 */ 2581 for (i = 0; i < 10; i++) { 2582 (void) random_get_pseudo_bytes((uint8_t *)&random, 2583 sizeof (uint32_t)); 2584 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2585 } 2586 2587 /* 2588 * Enable RSS & perform hash on these packet types 2589 */ 2590 mrqc = IXGBE_MRQC_RSSEN | 2591 IXGBE_MRQC_RSS_FIELD_IPV4 | 2592 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2593 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2594 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2595 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2596 IXGBE_MRQC_RSS_FIELD_IPV6 | 2597 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2598 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2599 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2600 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2601 2602 /* 2603 * Disable Packet Checksum to enable RSS for multiple receive queues. 2604 * It is an adapter hardware limitation that Packet Checksum is 2605 * mutually exclusive with RSS. 2606 */ 2607 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2608 rxcsum |= IXGBE_RXCSUM_PCSD; 2609 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2610 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2611 } 2612 2613 /* 2614 * ixgbe_setup_vmdq - Setup MAC classification feature 2615 */ 2616 static void 2617 ixgbe_setup_vmdq(ixgbe_t *ixgbe) 2618 { 2619 struct ixgbe_hw *hw = &ixgbe->hw; 2620 uint32_t vmdctl, i, vtctl; 2621 2622 /* 2623 * Setup the VMDq Control register, enable VMDq based on 2624 * packet destination MAC address: 2625 */ 2626 switch (hw->mac.type) { 2627 case ixgbe_mac_82598EB: 2628 /* 2629 * VMDq Enable = 1; 2630 * VMDq Filter = 0; MAC filtering 2631 * Default VMDq output index = 0; 2632 */ 2633 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2634 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2635 break; 2636 2637 case ixgbe_mac_82599EB: 2638 case ixgbe_mac_X540: 2639 /* 2640 * Enable VMDq-only. 2641 */ 2642 vmdctl = IXGBE_MRQC_VMDQEN; 2643 IXGBE_WRITE_REG(hw, IXGBE_MRQC, vmdctl); 2644 2645 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2646 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2647 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2648 } 2649 2650 /* 2651 * Enable Virtualization and Replication. 2652 */ 2653 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2654 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2655 2656 /* 2657 * Enable receiving packets to all VFs 2658 */ 2659 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2660 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2661 break; 2662 2663 default: 2664 break; 2665 } 2666 } 2667 2668 /* 2669 * ixgbe_setup_vmdq_rss - Setup both vmdq feature and rss feature. 2670 */ 2671 static void 2672 ixgbe_setup_vmdq_rss(ixgbe_t *ixgbe) 2673 { 2674 struct ixgbe_hw *hw = &ixgbe->hw; 2675 uint32_t i, mrqc, rxcsum; 2676 uint32_t random; 2677 uint32_t reta; 2678 uint32_t ring_per_group; 2679 uint32_t vmdctl, vtctl; 2680 2681 /* 2682 * Fill out redirection table 2683 */ 2684 reta = 0; 2685 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2686 for (i = 0; i < 128; i++) { 2687 reta = (reta << 8) | (i % ring_per_group) | 2688 ((i % ring_per_group) << 4); 2689 if ((i & 3) == 3) 2690 IXGBE_WRITE_REG(hw, IXGBE_RETA(i >> 2), reta); 2691 } 2692 2693 /* 2694 * Fill out hash function seeds with a random constant 2695 */ 2696 for (i = 0; i < 10; i++) { 2697 (void) random_get_pseudo_bytes((uint8_t *)&random, 2698 sizeof (uint32_t)); 2699 IXGBE_WRITE_REG(hw, IXGBE_RSSRK(i), random); 2700 } 2701 2702 /* 2703 * Enable and setup RSS and VMDq 2704 */ 2705 switch (hw->mac.type) { 2706 case ixgbe_mac_82598EB: 2707 /* 2708 * Enable RSS & Setup RSS Hash functions 2709 */ 2710 mrqc = IXGBE_MRQC_RSSEN | 2711 IXGBE_MRQC_RSS_FIELD_IPV4 | 2712 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2713 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2714 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2715 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2716 IXGBE_MRQC_RSS_FIELD_IPV6 | 2717 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2718 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2719 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2720 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2721 2722 /* 2723 * Enable and Setup VMDq 2724 * VMDq Filter = 0; MAC filtering 2725 * Default VMDq output index = 0; 2726 */ 2727 vmdctl = IXGBE_VMD_CTL_VMDQ_EN; 2728 IXGBE_WRITE_REG(hw, IXGBE_VMD_CTL, vmdctl); 2729 break; 2730 2731 case ixgbe_mac_82599EB: 2732 case ixgbe_mac_X540: 2733 /* 2734 * Enable RSS & Setup RSS Hash functions 2735 */ 2736 mrqc = IXGBE_MRQC_RSS_FIELD_IPV4 | 2737 IXGBE_MRQC_RSS_FIELD_IPV4_TCP | 2738 IXGBE_MRQC_RSS_FIELD_IPV4_UDP | 2739 IXGBE_MRQC_RSS_FIELD_IPV6_EX_TCP | 2740 IXGBE_MRQC_RSS_FIELD_IPV6_EX | 2741 IXGBE_MRQC_RSS_FIELD_IPV6 | 2742 IXGBE_MRQC_RSS_FIELD_IPV6_TCP | 2743 IXGBE_MRQC_RSS_FIELD_IPV6_UDP | 2744 IXGBE_MRQC_RSS_FIELD_IPV6_EX_UDP; 2745 2746 /* 2747 * Enable VMDq+RSS. 2748 */ 2749 if (ixgbe->num_rx_groups > 32) { 2750 mrqc = mrqc | IXGBE_MRQC_VMDQRSS64EN; 2751 } else { 2752 mrqc = mrqc | IXGBE_MRQC_VMDQRSS32EN; 2753 } 2754 2755 IXGBE_WRITE_REG(hw, IXGBE_MRQC, mrqc); 2756 2757 for (i = 0; i < hw->mac.num_rar_entries; i++) { 2758 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_LO(i), 0); 2759 IXGBE_WRITE_REG(hw, IXGBE_MPSAR_HI(i), 0); 2760 } 2761 break; 2762 2763 default: 2764 break; 2765 2766 } 2767 2768 /* 2769 * Disable Packet Checksum to enable RSS for multiple receive queues. 2770 * It is an adapter hardware limitation that Packet Checksum is 2771 * mutually exclusive with RSS. 2772 */ 2773 rxcsum = IXGBE_READ_REG(hw, IXGBE_RXCSUM); 2774 rxcsum |= IXGBE_RXCSUM_PCSD; 2775 rxcsum &= ~IXGBE_RXCSUM_IPPCSE; 2776 IXGBE_WRITE_REG(hw, IXGBE_RXCSUM, rxcsum); 2777 2778 if (hw->mac.type == ixgbe_mac_82599EB || 2779 hw->mac.type == ixgbe_mac_X540) { 2780 /* 2781 * Enable Virtualization and Replication. 2782 */ 2783 vtctl = IXGBE_VT_CTL_VT_ENABLE | IXGBE_VT_CTL_REPLEN; 2784 IXGBE_WRITE_REG(hw, IXGBE_VT_CTL, vtctl); 2785 2786 /* 2787 * Enable receiving packets to all VFs 2788 */ 2789 IXGBE_WRITE_REG(hw, IXGBE_VFRE(0), IXGBE_VFRE_ENABLE_ALL); 2790 IXGBE_WRITE_REG(hw, IXGBE_VFRE(1), IXGBE_VFRE_ENABLE_ALL); 2791 } 2792 } 2793 2794 /* 2795 * ixgbe_init_unicst - Initialize the unicast addresses. 2796 */ 2797 static void 2798 ixgbe_init_unicst(ixgbe_t *ixgbe) 2799 { 2800 struct ixgbe_hw *hw = &ixgbe->hw; 2801 uint8_t *mac_addr; 2802 int slot; 2803 /* 2804 * Here we should consider two situations: 2805 * 2806 * 1. Chipset is initialized at the first time, 2807 * Clear all the multiple unicast addresses. 2808 * 2809 * 2. Chipset is reset 2810 * Recover the multiple unicast addresses from the 2811 * software data structure to the RAR registers. 2812 */ 2813 if (!ixgbe->unicst_init) { 2814 /* 2815 * Initialize the multiple unicast addresses 2816 */ 2817 ixgbe->unicst_total = hw->mac.num_rar_entries; 2818 ixgbe->unicst_avail = ixgbe->unicst_total; 2819 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2820 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2821 bzero(mac_addr, ETHERADDRL); 2822 (void) ixgbe_set_rar(hw, slot, mac_addr, NULL, NULL); 2823 ixgbe->unicst_addr[slot].mac.set = 0; 2824 } 2825 ixgbe->unicst_init = B_TRUE; 2826 } else { 2827 /* Re-configure the RAR registers */ 2828 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2829 mac_addr = ixgbe->unicst_addr[slot].mac.addr; 2830 if (ixgbe->unicst_addr[slot].mac.set == 1) { 2831 (void) ixgbe_set_rar(hw, slot, mac_addr, 2832 ixgbe->unicst_addr[slot].mac.group_index, 2833 IXGBE_RAH_AV); 2834 } else { 2835 bzero(mac_addr, ETHERADDRL); 2836 (void) ixgbe_set_rar(hw, slot, mac_addr, 2837 NULL, NULL); 2838 } 2839 } 2840 } 2841 } 2842 2843 /* 2844 * ixgbe_unicst_find - Find the slot for the specified unicast address 2845 */ 2846 int 2847 ixgbe_unicst_find(ixgbe_t *ixgbe, const uint8_t *mac_addr) 2848 { 2849 int slot; 2850 2851 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2852 2853 for (slot = 0; slot < ixgbe->unicst_total; slot++) { 2854 if (bcmp(ixgbe->unicst_addr[slot].mac.addr, 2855 mac_addr, ETHERADDRL) == 0) 2856 return (slot); 2857 } 2858 2859 return (-1); 2860 } 2861 2862 /* 2863 * ixgbe_multicst_add - Add a multicst address. 2864 */ 2865 int 2866 ixgbe_multicst_add(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2867 { 2868 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2869 2870 if ((multiaddr[0] & 01) == 0) { 2871 return (EINVAL); 2872 } 2873 2874 if (ixgbe->mcast_count >= MAX_NUM_MULTICAST_ADDRESSES) { 2875 return (ENOENT); 2876 } 2877 2878 bcopy(multiaddr, 2879 &ixgbe->mcast_table[ixgbe->mcast_count], ETHERADDRL); 2880 ixgbe->mcast_count++; 2881 2882 /* 2883 * Update the multicast table in the hardware 2884 */ 2885 ixgbe_setup_multicst(ixgbe); 2886 2887 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2888 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2889 return (EIO); 2890 } 2891 2892 return (0); 2893 } 2894 2895 /* 2896 * ixgbe_multicst_remove - Remove a multicst address. 2897 */ 2898 int 2899 ixgbe_multicst_remove(ixgbe_t *ixgbe, const uint8_t *multiaddr) 2900 { 2901 int i; 2902 2903 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2904 2905 for (i = 0; i < ixgbe->mcast_count; i++) { 2906 if (bcmp(multiaddr, &ixgbe->mcast_table[i], 2907 ETHERADDRL) == 0) { 2908 for (i++; i < ixgbe->mcast_count; i++) { 2909 ixgbe->mcast_table[i - 1] = 2910 ixgbe->mcast_table[i]; 2911 } 2912 ixgbe->mcast_count--; 2913 break; 2914 } 2915 } 2916 2917 /* 2918 * Update the multicast table in the hardware 2919 */ 2920 ixgbe_setup_multicst(ixgbe); 2921 2922 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 2923 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 2924 return (EIO); 2925 } 2926 2927 return (0); 2928 } 2929 2930 /* 2931 * ixgbe_setup_multicast - Setup multicast data structures. 2932 * 2933 * This routine initializes all of the multicast related structures 2934 * and save them in the hardware registers. 2935 */ 2936 static void 2937 ixgbe_setup_multicst(ixgbe_t *ixgbe) 2938 { 2939 uint8_t *mc_addr_list; 2940 uint32_t mc_addr_count; 2941 struct ixgbe_hw *hw = &ixgbe->hw; 2942 2943 ASSERT(mutex_owned(&ixgbe->gen_lock)); 2944 2945 ASSERT(ixgbe->mcast_count <= MAX_NUM_MULTICAST_ADDRESSES); 2946 2947 mc_addr_list = (uint8_t *)ixgbe->mcast_table; 2948 mc_addr_count = ixgbe->mcast_count; 2949 2950 /* 2951 * Update the multicast addresses to the MTA registers 2952 */ 2953 (void) ixgbe_update_mc_addr_list(hw, mc_addr_list, mc_addr_count, 2954 ixgbe_mc_table_itr, TRUE); 2955 } 2956 2957 /* 2958 * ixgbe_setup_vmdq_rss_conf - Configure vmdq and rss (number and mode). 2959 * 2960 * Configure the rx classification mode (vmdq & rss) and vmdq & rss numbers. 2961 * Different chipsets may have different allowed configuration of vmdq and rss. 2962 */ 2963 static void 2964 ixgbe_setup_vmdq_rss_conf(ixgbe_t *ixgbe) 2965 { 2966 struct ixgbe_hw *hw = &ixgbe->hw; 2967 uint32_t ring_per_group; 2968 2969 switch (hw->mac.type) { 2970 case ixgbe_mac_82598EB: 2971 /* 2972 * 82598 supports the following combination: 2973 * vmdq no. x rss no. 2974 * [5..16] x 1 2975 * [1..4] x [1..16] 2976 * However 8 rss queue per pool (vmdq) is sufficient for 2977 * most cases. 2978 */ 2979 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 2980 if (ixgbe->num_rx_groups > 4) { 2981 ixgbe->num_rx_rings = ixgbe->num_rx_groups; 2982 } else { 2983 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 2984 min(8, ring_per_group); 2985 } 2986 2987 break; 2988 2989 case ixgbe_mac_82599EB: 2990 case ixgbe_mac_X540: 2991 /* 2992 * 82599 supports the following combination: 2993 * vmdq no. x rss no. 2994 * [33..64] x [1..2] 2995 * [2..32] x [1..4] 2996 * 1 x [1..16] 2997 * However 8 rss queue per pool (vmdq) is sufficient for 2998 * most cases. 2999 * 3000 * For now, treat X540 like the 82599. 3001 */ 3002 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3003 if (ixgbe->num_rx_groups == 1) { 3004 ixgbe->num_rx_rings = min(8, ring_per_group); 3005 } else if (ixgbe->num_rx_groups <= 32) { 3006 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3007 min(4, ring_per_group); 3008 } else if (ixgbe->num_rx_groups <= 64) { 3009 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3010 min(2, ring_per_group); 3011 } 3012 break; 3013 3014 default: 3015 break; 3016 } 3017 3018 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 3019 3020 if (ixgbe->num_rx_groups == 1 && ring_per_group == 1) { 3021 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3022 } else if (ixgbe->num_rx_groups != 1 && ring_per_group == 1) { 3023 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ; 3024 } else if (ixgbe->num_rx_groups != 1 && ring_per_group != 1) { 3025 ixgbe->classify_mode = IXGBE_CLASSIFY_VMDQ_RSS; 3026 } else { 3027 ixgbe->classify_mode = IXGBE_CLASSIFY_RSS; 3028 } 3029 3030 IXGBE_DEBUGLOG_2(ixgbe, "rx group number:%d, rx ring number:%d", 3031 ixgbe->num_rx_groups, ixgbe->num_rx_rings); 3032 } 3033 3034 /* 3035 * ixgbe_get_conf - Get driver configurations set in driver.conf. 3036 * 3037 * This routine gets user-configured values out of the configuration 3038 * file ixgbe.conf. 3039 * 3040 * For each configurable value, there is a minimum, a maximum, and a 3041 * default. 3042 * If user does not configure a value, use the default. 3043 * If user configures below the minimum, use the minumum. 3044 * If user configures above the maximum, use the maxumum. 3045 */ 3046 static void 3047 ixgbe_get_conf(ixgbe_t *ixgbe) 3048 { 3049 struct ixgbe_hw *hw = &ixgbe->hw; 3050 uint32_t flow_control; 3051 3052 /* 3053 * ixgbe driver supports the following user configurations: 3054 * 3055 * Jumbo frame configuration: 3056 * default_mtu 3057 * 3058 * Ethernet flow control configuration: 3059 * flow_control 3060 * 3061 * Multiple rings configurations: 3062 * tx_queue_number 3063 * tx_ring_size 3064 * rx_queue_number 3065 * rx_ring_size 3066 * 3067 * Call ixgbe_get_prop() to get the value for a specific 3068 * configuration parameter. 3069 */ 3070 3071 /* 3072 * Jumbo frame configuration - max_frame_size controls host buffer 3073 * allocation, so includes MTU, ethernet header, vlan tag and 3074 * frame check sequence. 3075 */ 3076 ixgbe->default_mtu = ixgbe_get_prop(ixgbe, PROP_DEFAULT_MTU, 3077 MIN_MTU, ixgbe->capab->max_mtu, DEFAULT_MTU); 3078 3079 ixgbe->max_frame_size = ixgbe->default_mtu + 3080 sizeof (struct ether_vlan_header) + ETHERFCSL; 3081 3082 /* 3083 * Ethernet flow control configuration 3084 */ 3085 flow_control = ixgbe_get_prop(ixgbe, PROP_FLOW_CONTROL, 3086 ixgbe_fc_none, 3, ixgbe_fc_none); 3087 if (flow_control == 3) 3088 flow_control = ixgbe_fc_default; 3089 3090 /* 3091 * fc.requested mode is what the user requests. After autoneg, 3092 * fc.current_mode will be the flow_control mode that was negotiated. 3093 */ 3094 hw->fc.requested_mode = flow_control; 3095 3096 /* 3097 * Multiple rings configurations 3098 */ 3099 ixgbe->num_tx_rings = ixgbe_get_prop(ixgbe, PROP_TX_QUEUE_NUM, 3100 ixgbe->capab->min_tx_que_num, 3101 ixgbe->capab->max_tx_que_num, 3102 ixgbe->capab->def_tx_que_num); 3103 ixgbe->tx_ring_size = ixgbe_get_prop(ixgbe, PROP_TX_RING_SIZE, 3104 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 3105 3106 ixgbe->num_rx_rings = ixgbe_get_prop(ixgbe, PROP_RX_QUEUE_NUM, 3107 ixgbe->capab->min_rx_que_num, 3108 ixgbe->capab->max_rx_que_num, 3109 ixgbe->capab->def_rx_que_num); 3110 ixgbe->rx_ring_size = ixgbe_get_prop(ixgbe, PROP_RX_RING_SIZE, 3111 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 3112 3113 /* 3114 * Multiple groups configuration 3115 */ 3116 ixgbe->num_rx_groups = ixgbe_get_prop(ixgbe, PROP_RX_GROUP_NUM, 3117 ixgbe->capab->min_rx_grp_num, ixgbe->capab->max_rx_grp_num, 3118 ixgbe->capab->def_rx_grp_num); 3119 3120 ixgbe->mr_enable = ixgbe_get_prop(ixgbe, PROP_MR_ENABLE, 3121 0, 1, DEFAULT_MR_ENABLE); 3122 3123 if (ixgbe->mr_enable == B_FALSE) { 3124 ixgbe->num_tx_rings = 1; 3125 ixgbe->num_rx_rings = 1; 3126 ixgbe->num_rx_groups = 1; 3127 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 3128 } else { 3129 ixgbe->num_rx_rings = ixgbe->num_rx_groups * 3130 max(ixgbe->num_rx_rings / ixgbe->num_rx_groups, 1); 3131 /* 3132 * The combination of num_rx_rings and num_rx_groups 3133 * may be not supported by h/w. We need to adjust 3134 * them to appropriate values. 3135 */ 3136 ixgbe_setup_vmdq_rss_conf(ixgbe); 3137 } 3138 3139 /* 3140 * Tunable used to force an interrupt type. The only use is 3141 * for testing of the lesser interrupt types. 3142 * 0 = don't force interrupt type 3143 * 1 = force interrupt type MSI-X 3144 * 2 = force interrupt type MSI 3145 * 3 = force interrupt type Legacy 3146 */ 3147 ixgbe->intr_force = ixgbe_get_prop(ixgbe, PROP_INTR_FORCE, 3148 IXGBE_INTR_NONE, IXGBE_INTR_LEGACY, IXGBE_INTR_NONE); 3149 3150 ixgbe->tx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_TX_HCKSUM_ENABLE, 3151 0, 1, DEFAULT_TX_HCKSUM_ENABLE); 3152 ixgbe->rx_hcksum_enable = ixgbe_get_prop(ixgbe, PROP_RX_HCKSUM_ENABLE, 3153 0, 1, DEFAULT_RX_HCKSUM_ENABLE); 3154 ixgbe->lso_enable = ixgbe_get_prop(ixgbe, PROP_LSO_ENABLE, 3155 0, 1, DEFAULT_LSO_ENABLE); 3156 ixgbe->lro_enable = ixgbe_get_prop(ixgbe, PROP_LRO_ENABLE, 3157 0, 1, DEFAULT_LRO_ENABLE); 3158 ixgbe->tx_head_wb_enable = ixgbe_get_prop(ixgbe, PROP_TX_HEAD_WB_ENABLE, 3159 0, 1, DEFAULT_TX_HEAD_WB_ENABLE); 3160 ixgbe->relax_order_enable = ixgbe_get_prop(ixgbe, 3161 PROP_RELAX_ORDER_ENABLE, 0, 1, DEFAULT_RELAX_ORDER_ENABLE); 3162 3163 /* Head Write Back not recommended for 82599 and X540 */ 3164 if (hw->mac.type == ixgbe_mac_82599EB || 3165 hw->mac.type == ixgbe_mac_X540) { 3166 ixgbe->tx_head_wb_enable = B_FALSE; 3167 } 3168 3169 /* 3170 * ixgbe LSO needs the tx h/w checksum support. 3171 * LSO will be disabled if tx h/w checksum is not 3172 * enabled. 3173 */ 3174 if (ixgbe->tx_hcksum_enable == B_FALSE) { 3175 ixgbe->lso_enable = B_FALSE; 3176 } 3177 3178 /* 3179 * ixgbe LRO needs the rx h/w checksum support. 3180 * LRO will be disabled if rx h/w checksum is not 3181 * enabled. 3182 */ 3183 if (ixgbe->rx_hcksum_enable == B_FALSE) { 3184 ixgbe->lro_enable = B_FALSE; 3185 } 3186 3187 /* 3188 * ixgbe LRO only been supported by 82599 and X540 now 3189 */ 3190 if (hw->mac.type == ixgbe_mac_82598EB) { 3191 ixgbe->lro_enable = B_FALSE; 3192 } 3193 ixgbe->tx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_TX_COPY_THRESHOLD, 3194 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 3195 DEFAULT_TX_COPY_THRESHOLD); 3196 ixgbe->tx_recycle_thresh = ixgbe_get_prop(ixgbe, 3197 PROP_TX_RECYCLE_THRESHOLD, MIN_TX_RECYCLE_THRESHOLD, 3198 MAX_TX_RECYCLE_THRESHOLD, DEFAULT_TX_RECYCLE_THRESHOLD); 3199 ixgbe->tx_overload_thresh = ixgbe_get_prop(ixgbe, 3200 PROP_TX_OVERLOAD_THRESHOLD, MIN_TX_OVERLOAD_THRESHOLD, 3201 MAX_TX_OVERLOAD_THRESHOLD, DEFAULT_TX_OVERLOAD_THRESHOLD); 3202 ixgbe->tx_resched_thresh = ixgbe_get_prop(ixgbe, 3203 PROP_TX_RESCHED_THRESHOLD, MIN_TX_RESCHED_THRESHOLD, 3204 MAX_TX_RESCHED_THRESHOLD, DEFAULT_TX_RESCHED_THRESHOLD); 3205 3206 ixgbe->rx_copy_thresh = ixgbe_get_prop(ixgbe, PROP_RX_COPY_THRESHOLD, 3207 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 3208 DEFAULT_RX_COPY_THRESHOLD); 3209 ixgbe->rx_limit_per_intr = ixgbe_get_prop(ixgbe, PROP_RX_LIMIT_PER_INTR, 3210 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 3211 DEFAULT_RX_LIMIT_PER_INTR); 3212 3213 ixgbe->intr_throttling[0] = ixgbe_get_prop(ixgbe, PROP_INTR_THROTTLING, 3214 ixgbe->capab->min_intr_throttle, 3215 ixgbe->capab->max_intr_throttle, 3216 ixgbe->capab->def_intr_throttle); 3217 /* 3218 * 82599 and X540 require the interrupt throttling rate is 3219 * a multiple of 8. This is enforced by the register 3220 * definiton. 3221 */ 3222 if (hw->mac.type == ixgbe_mac_82599EB || hw->mac.type == ixgbe_mac_X540) 3223 ixgbe->intr_throttling[0] = ixgbe->intr_throttling[0] & 0xFF8; 3224 3225 hw->allow_unsupported_sfp = ixgbe_get_prop(ixgbe, 3226 PROP_ALLOW_UNSUPPORTED_SFP, 0, 1, DEFAULT_ALLOW_UNSUPPORTED_SFP); 3227 } 3228 3229 static void 3230 ixgbe_init_params(ixgbe_t *ixgbe) 3231 { 3232 ixgbe->param_en_10000fdx_cap = 1; 3233 ixgbe->param_en_1000fdx_cap = 1; 3234 ixgbe->param_en_100fdx_cap = 1; 3235 ixgbe->param_adv_10000fdx_cap = 1; 3236 ixgbe->param_adv_1000fdx_cap = 1; 3237 ixgbe->param_adv_100fdx_cap = 1; 3238 3239 ixgbe->param_pause_cap = 1; 3240 ixgbe->param_asym_pause_cap = 1; 3241 ixgbe->param_rem_fault = 0; 3242 3243 ixgbe->param_adv_autoneg_cap = 1; 3244 ixgbe->param_adv_pause_cap = 1; 3245 ixgbe->param_adv_asym_pause_cap = 1; 3246 ixgbe->param_adv_rem_fault = 0; 3247 3248 ixgbe->param_lp_10000fdx_cap = 0; 3249 ixgbe->param_lp_1000fdx_cap = 0; 3250 ixgbe->param_lp_100fdx_cap = 0; 3251 ixgbe->param_lp_autoneg_cap = 0; 3252 ixgbe->param_lp_pause_cap = 0; 3253 ixgbe->param_lp_asym_pause_cap = 0; 3254 ixgbe->param_lp_rem_fault = 0; 3255 } 3256 3257 /* 3258 * ixgbe_get_prop - Get a property value out of the configuration file 3259 * ixgbe.conf. 3260 * 3261 * Caller provides the name of the property, a default value, a minimum 3262 * value, and a maximum value. 3263 * 3264 * Return configured value of the property, with default, minimum and 3265 * maximum properly applied. 3266 */ 3267 static int 3268 ixgbe_get_prop(ixgbe_t *ixgbe, 3269 char *propname, /* name of the property */ 3270 int minval, /* minimum acceptable value */ 3271 int maxval, /* maximim acceptable value */ 3272 int defval) /* default value */ 3273 { 3274 int value; 3275 3276 /* 3277 * Call ddi_prop_get_int() to read the conf settings 3278 */ 3279 value = ddi_prop_get_int(DDI_DEV_T_ANY, ixgbe->dip, 3280 DDI_PROP_DONTPASS, propname, defval); 3281 if (value > maxval) 3282 value = maxval; 3283 3284 if (value < minval) 3285 value = minval; 3286 3287 return (value); 3288 } 3289 3290 /* 3291 * ixgbe_driver_setup_link - Using the link properties to setup the link. 3292 */ 3293 int 3294 ixgbe_driver_setup_link(ixgbe_t *ixgbe, boolean_t setup_hw) 3295 { 3296 u32 autoneg_advertised = 0; 3297 3298 /* 3299 * No half duplex support with 10Gb parts 3300 */ 3301 if (ixgbe->param_adv_10000fdx_cap == 1) 3302 autoneg_advertised |= IXGBE_LINK_SPEED_10GB_FULL; 3303 3304 if (ixgbe->param_adv_1000fdx_cap == 1) 3305 autoneg_advertised |= IXGBE_LINK_SPEED_1GB_FULL; 3306 3307 if (ixgbe->param_adv_100fdx_cap == 1) 3308 autoneg_advertised |= IXGBE_LINK_SPEED_100_FULL; 3309 3310 if (ixgbe->param_adv_autoneg_cap == 1 && autoneg_advertised == 0) { 3311 ixgbe_notice(ixgbe, "Invalid link settings. Setup link " 3312 "to autonegotiation with full link capabilities."); 3313 3314 autoneg_advertised = IXGBE_LINK_SPEED_10GB_FULL | 3315 IXGBE_LINK_SPEED_1GB_FULL | 3316 IXGBE_LINK_SPEED_100_FULL; 3317 } 3318 3319 if (setup_hw) { 3320 if (ixgbe_setup_link(&ixgbe->hw, autoneg_advertised, 3321 ixgbe->param_adv_autoneg_cap, B_TRUE) != IXGBE_SUCCESS) { 3322 ixgbe_notice(ixgbe, "Setup link failed on this " 3323 "device."); 3324 return (IXGBE_FAILURE); 3325 } 3326 } 3327 3328 return (IXGBE_SUCCESS); 3329 } 3330 3331 /* 3332 * ixgbe_driver_link_check - Link status processing. 3333 * 3334 * This function can be called in both kernel context and interrupt context 3335 */ 3336 static void 3337 ixgbe_driver_link_check(ixgbe_t *ixgbe) 3338 { 3339 struct ixgbe_hw *hw = &ixgbe->hw; 3340 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 3341 boolean_t link_up = B_FALSE; 3342 boolean_t link_changed = B_FALSE; 3343 3344 ASSERT(mutex_owned(&ixgbe->gen_lock)); 3345 3346 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3347 if (link_up) { 3348 ixgbe->link_check_complete = B_TRUE; 3349 3350 /* Link is up, enable flow control settings */ 3351 (void) ixgbe_fc_enable(hw); 3352 3353 /* 3354 * The Link is up, check whether it was marked as down earlier 3355 */ 3356 if (ixgbe->link_state != LINK_STATE_UP) { 3357 switch (speed) { 3358 case IXGBE_LINK_SPEED_10GB_FULL: 3359 ixgbe->link_speed = SPEED_10GB; 3360 break; 3361 case IXGBE_LINK_SPEED_1GB_FULL: 3362 ixgbe->link_speed = SPEED_1GB; 3363 break; 3364 case IXGBE_LINK_SPEED_100_FULL: 3365 ixgbe->link_speed = SPEED_100; 3366 } 3367 ixgbe->link_duplex = LINK_DUPLEX_FULL; 3368 ixgbe->link_state = LINK_STATE_UP; 3369 link_changed = B_TRUE; 3370 } 3371 } else { 3372 if (ixgbe->link_check_complete == B_TRUE || 3373 (ixgbe->link_check_complete == B_FALSE && 3374 gethrtime() >= ixgbe->link_check_hrtime)) { 3375 /* 3376 * The link is really down 3377 */ 3378 ixgbe->link_check_complete = B_TRUE; 3379 3380 if (ixgbe->link_state != LINK_STATE_DOWN) { 3381 ixgbe->link_speed = 0; 3382 ixgbe->link_duplex = LINK_DUPLEX_UNKNOWN; 3383 ixgbe->link_state = LINK_STATE_DOWN; 3384 link_changed = B_TRUE; 3385 } 3386 } 3387 } 3388 3389 /* 3390 * If we are in an interrupt context, need to re-enable the 3391 * interrupt, which was automasked 3392 */ 3393 if (servicing_interrupt() != 0) { 3394 ixgbe->eims |= IXGBE_EICR_LSC; 3395 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3396 } 3397 3398 if (link_changed) { 3399 mac_link_update(ixgbe->mac_hdl, ixgbe->link_state); 3400 } 3401 } 3402 3403 /* 3404 * ixgbe_sfp_check - sfp module processing done in taskq only for 82599. 3405 */ 3406 static void 3407 ixgbe_sfp_check(void *arg) 3408 { 3409 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3410 uint32_t eicr = ixgbe->eicr; 3411 struct ixgbe_hw *hw = &ixgbe->hw; 3412 3413 mutex_enter(&ixgbe->gen_lock); 3414 if (eicr & IXGBE_EICR_GPI_SDP1) { 3415 /* clear the interrupt */ 3416 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP1); 3417 3418 /* if link up, do multispeed fiber setup */ 3419 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3420 B_TRUE, B_TRUE); 3421 ixgbe_driver_link_check(ixgbe); 3422 ixgbe_get_hw_state(ixgbe); 3423 } else if (eicr & IXGBE_EICR_GPI_SDP2) { 3424 /* clear the interrupt */ 3425 IXGBE_WRITE_REG(hw, IXGBE_EICR, IXGBE_EICR_GPI_SDP2); 3426 3427 /* if link up, do sfp module setup */ 3428 (void) hw->mac.ops.setup_sfp(hw); 3429 3430 /* do multispeed fiber setup */ 3431 (void) ixgbe_setup_link(hw, IXGBE_LINK_SPEED_82599_AUTONEG, 3432 B_TRUE, B_TRUE); 3433 ixgbe_driver_link_check(ixgbe); 3434 ixgbe_get_hw_state(ixgbe); 3435 } 3436 mutex_exit(&ixgbe->gen_lock); 3437 3438 /* 3439 * We need to fully re-check the link later. 3440 */ 3441 ixgbe->link_check_complete = B_FALSE; 3442 ixgbe->link_check_hrtime = gethrtime() + 3443 (IXGBE_LINK_UP_TIME * 100000000ULL); 3444 } 3445 3446 /* 3447 * ixgbe_overtemp_check - overtemp module processing done in taskq 3448 * 3449 * This routine will only be called on adapters with temperature sensor. 3450 * The indication of over-temperature can be either SDP0 interrupt or the link 3451 * status change interrupt. 3452 */ 3453 static void 3454 ixgbe_overtemp_check(void *arg) 3455 { 3456 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3457 struct ixgbe_hw *hw = &ixgbe->hw; 3458 uint32_t eicr = ixgbe->eicr; 3459 ixgbe_link_speed speed; 3460 boolean_t link_up; 3461 3462 mutex_enter(&ixgbe->gen_lock); 3463 3464 /* make sure we know current state of link */ 3465 (void) ixgbe_check_link(hw, &speed, &link_up, false); 3466 3467 /* check over-temp condition */ 3468 if (((eicr & IXGBE_EICR_GPI_SDP0) && (!link_up)) || 3469 (eicr & IXGBE_EICR_LSC)) { 3470 if (hw->phy.ops.check_overtemp(hw) == IXGBE_ERR_OVERTEMP) { 3471 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 3472 3473 /* 3474 * Disable the adapter interrupts 3475 */ 3476 ixgbe_disable_adapter_interrupts(ixgbe); 3477 3478 /* 3479 * Disable Rx/Tx units 3480 */ 3481 (void) ixgbe_stop_adapter(hw); 3482 3483 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 3484 ixgbe_error(ixgbe, 3485 "Problem: Network adapter has been stopped " 3486 "because it has overheated"); 3487 ixgbe_error(ixgbe, 3488 "Action: Restart the computer. " 3489 "If the problem persists, power off the system " 3490 "and replace the adapter"); 3491 } 3492 } 3493 3494 /* write to clear the interrupt */ 3495 IXGBE_WRITE_REG(hw, IXGBE_EICR, eicr); 3496 3497 mutex_exit(&ixgbe->gen_lock); 3498 } 3499 3500 /* 3501 * ixgbe_link_timer - timer for link status detection 3502 */ 3503 static void 3504 ixgbe_link_timer(void *arg) 3505 { 3506 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3507 3508 mutex_enter(&ixgbe->gen_lock); 3509 ixgbe_driver_link_check(ixgbe); 3510 mutex_exit(&ixgbe->gen_lock); 3511 } 3512 3513 /* 3514 * ixgbe_local_timer - Driver watchdog function. 3515 * 3516 * This function will handle the transmit stall check and other routines. 3517 */ 3518 static void 3519 ixgbe_local_timer(void *arg) 3520 { 3521 ixgbe_t *ixgbe = (ixgbe_t *)arg; 3522 3523 if (ixgbe->ixgbe_state & IXGBE_OVERTEMP) 3524 goto out; 3525 3526 if (ixgbe->ixgbe_state & IXGBE_ERROR) { 3527 ixgbe->reset_count++; 3528 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3529 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3530 goto out; 3531 } 3532 3533 if (ixgbe_stall_check(ixgbe)) { 3534 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_STALL); 3535 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3536 3537 ixgbe->reset_count++; 3538 if (ixgbe_reset(ixgbe) == IXGBE_SUCCESS) 3539 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_RESTORED); 3540 } 3541 3542 out: 3543 ixgbe_restart_watchdog_timer(ixgbe); 3544 } 3545 3546 /* 3547 * ixgbe_stall_check - Check for transmit stall. 3548 * 3549 * This function checks if the adapter is stalled (in transmit). 3550 * 3551 * It is called each time the watchdog timeout is invoked. 3552 * If the transmit descriptor reclaim continuously fails, 3553 * the watchdog value will increment by 1. If the watchdog 3554 * value exceeds the threshold, the ixgbe is assumed to 3555 * have stalled and need to be reset. 3556 */ 3557 static boolean_t 3558 ixgbe_stall_check(ixgbe_t *ixgbe) 3559 { 3560 ixgbe_tx_ring_t *tx_ring; 3561 boolean_t result; 3562 int i; 3563 3564 if (ixgbe->link_state != LINK_STATE_UP) 3565 return (B_FALSE); 3566 3567 /* 3568 * If any tx ring is stalled, we'll reset the chipset 3569 */ 3570 result = B_FALSE; 3571 for (i = 0; i < ixgbe->num_tx_rings; i++) { 3572 tx_ring = &ixgbe->tx_rings[i]; 3573 if (tx_ring->tbd_free <= ixgbe->tx_recycle_thresh) { 3574 tx_ring->tx_recycle(tx_ring); 3575 } 3576 3577 if (tx_ring->recycle_fail > 0) 3578 tx_ring->stall_watchdog++; 3579 else 3580 tx_ring->stall_watchdog = 0; 3581 3582 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3583 result = B_TRUE; 3584 break; 3585 } 3586 } 3587 3588 if (result) { 3589 tx_ring->stall_watchdog = 0; 3590 tx_ring->recycle_fail = 0; 3591 } 3592 3593 return (result); 3594 } 3595 3596 3597 /* 3598 * is_valid_mac_addr - Check if the mac address is valid. 3599 */ 3600 static boolean_t 3601 is_valid_mac_addr(uint8_t *mac_addr) 3602 { 3603 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3604 const uint8_t addr_test2[6] = 3605 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3606 3607 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3608 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3609 return (B_FALSE); 3610 3611 return (B_TRUE); 3612 } 3613 3614 static boolean_t 3615 ixgbe_find_mac_address(ixgbe_t *ixgbe) 3616 { 3617 #ifdef __sparc 3618 struct ixgbe_hw *hw = &ixgbe->hw; 3619 uchar_t *bytes; 3620 struct ether_addr sysaddr; 3621 uint_t nelts; 3622 int err; 3623 boolean_t found = B_FALSE; 3624 3625 /* 3626 * The "vendor's factory-set address" may already have 3627 * been extracted from the chip, but if the property 3628 * "local-mac-address" is set we use that instead. 3629 * 3630 * We check whether it looks like an array of 6 3631 * bytes (which it should, if OBP set it). If we can't 3632 * make sense of it this way, we'll ignore it. 3633 */ 3634 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3635 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3636 if (err == DDI_PROP_SUCCESS) { 3637 if (nelts == ETHERADDRL) { 3638 while (nelts--) 3639 hw->mac.addr[nelts] = bytes[nelts]; 3640 found = B_TRUE; 3641 } 3642 ddi_prop_free(bytes); 3643 } 3644 3645 /* 3646 * Look up the OBP property "local-mac-address?". If the user has set 3647 * 'local-mac-address? = false', use "the system address" instead. 3648 */ 3649 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 0, 3650 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3651 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3652 if (localetheraddr(NULL, &sysaddr) != 0) { 3653 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3654 found = B_TRUE; 3655 } 3656 } 3657 ddi_prop_free(bytes); 3658 } 3659 3660 /* 3661 * Finally(!), if there's a valid "mac-address" property (created 3662 * if we netbooted from this interface), we must use this instead 3663 * of any of the above to ensure that the NFS/install server doesn't 3664 * get confused by the address changing as Solaris takes over! 3665 */ 3666 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, ixgbe->dip, 3667 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3668 if (err == DDI_PROP_SUCCESS) { 3669 if (nelts == ETHERADDRL) { 3670 while (nelts--) 3671 hw->mac.addr[nelts] = bytes[nelts]; 3672 found = B_TRUE; 3673 } 3674 ddi_prop_free(bytes); 3675 } 3676 3677 if (found) { 3678 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3679 return (B_TRUE); 3680 } 3681 #else 3682 _NOTE(ARGUNUSED(ixgbe)); 3683 #endif 3684 3685 return (B_TRUE); 3686 } 3687 3688 #pragma inline(ixgbe_arm_watchdog_timer) 3689 static void 3690 ixgbe_arm_watchdog_timer(ixgbe_t *ixgbe) 3691 { 3692 /* 3693 * Fire a watchdog timer 3694 */ 3695 ixgbe->watchdog_tid = 3696 timeout(ixgbe_local_timer, 3697 (void *)ixgbe, 1 * drv_usectohz(1000000)); 3698 3699 } 3700 3701 /* 3702 * ixgbe_enable_watchdog_timer - Enable and start the driver watchdog timer. 3703 */ 3704 void 3705 ixgbe_enable_watchdog_timer(ixgbe_t *ixgbe) 3706 { 3707 mutex_enter(&ixgbe->watchdog_lock); 3708 3709 if (!ixgbe->watchdog_enable) { 3710 ixgbe->watchdog_enable = B_TRUE; 3711 ixgbe->watchdog_start = B_TRUE; 3712 ixgbe_arm_watchdog_timer(ixgbe); 3713 } 3714 3715 mutex_exit(&ixgbe->watchdog_lock); 3716 } 3717 3718 /* 3719 * ixgbe_disable_watchdog_timer - Disable and stop the driver watchdog timer. 3720 */ 3721 void 3722 ixgbe_disable_watchdog_timer(ixgbe_t *ixgbe) 3723 { 3724 timeout_id_t tid; 3725 3726 mutex_enter(&ixgbe->watchdog_lock); 3727 3728 ixgbe->watchdog_enable = B_FALSE; 3729 ixgbe->watchdog_start = B_FALSE; 3730 tid = ixgbe->watchdog_tid; 3731 ixgbe->watchdog_tid = 0; 3732 3733 mutex_exit(&ixgbe->watchdog_lock); 3734 3735 if (tid != 0) 3736 (void) untimeout(tid); 3737 } 3738 3739 /* 3740 * ixgbe_start_watchdog_timer - Start the driver watchdog timer. 3741 */ 3742 void 3743 ixgbe_start_watchdog_timer(ixgbe_t *ixgbe) 3744 { 3745 mutex_enter(&ixgbe->watchdog_lock); 3746 3747 if (ixgbe->watchdog_enable) { 3748 if (!ixgbe->watchdog_start) { 3749 ixgbe->watchdog_start = B_TRUE; 3750 ixgbe_arm_watchdog_timer(ixgbe); 3751 } 3752 } 3753 3754 mutex_exit(&ixgbe->watchdog_lock); 3755 } 3756 3757 /* 3758 * ixgbe_restart_watchdog_timer - Restart the driver watchdog timer. 3759 */ 3760 static void 3761 ixgbe_restart_watchdog_timer(ixgbe_t *ixgbe) 3762 { 3763 mutex_enter(&ixgbe->watchdog_lock); 3764 3765 if (ixgbe->watchdog_start) 3766 ixgbe_arm_watchdog_timer(ixgbe); 3767 3768 mutex_exit(&ixgbe->watchdog_lock); 3769 } 3770 3771 /* 3772 * ixgbe_stop_watchdog_timer - Stop the driver watchdog timer. 3773 */ 3774 void 3775 ixgbe_stop_watchdog_timer(ixgbe_t *ixgbe) 3776 { 3777 timeout_id_t tid; 3778 3779 mutex_enter(&ixgbe->watchdog_lock); 3780 3781 ixgbe->watchdog_start = B_FALSE; 3782 tid = ixgbe->watchdog_tid; 3783 ixgbe->watchdog_tid = 0; 3784 3785 mutex_exit(&ixgbe->watchdog_lock); 3786 3787 if (tid != 0) 3788 (void) untimeout(tid); 3789 } 3790 3791 /* 3792 * ixgbe_disable_adapter_interrupts - Disable all adapter interrupts. 3793 */ 3794 static void 3795 ixgbe_disable_adapter_interrupts(ixgbe_t *ixgbe) 3796 { 3797 struct ixgbe_hw *hw = &ixgbe->hw; 3798 3799 /* 3800 * mask all interrupts off 3801 */ 3802 IXGBE_WRITE_REG(hw, IXGBE_EIMC, 0xffffffff); 3803 3804 /* 3805 * for MSI-X, also disable autoclear 3806 */ 3807 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3808 IXGBE_WRITE_REG(hw, IXGBE_EIAC, 0x0); 3809 } 3810 3811 IXGBE_WRITE_FLUSH(hw); 3812 } 3813 3814 /* 3815 * ixgbe_enable_adapter_interrupts - Enable all hardware interrupts. 3816 */ 3817 static void 3818 ixgbe_enable_adapter_interrupts(ixgbe_t *ixgbe) 3819 { 3820 struct ixgbe_hw *hw = &ixgbe->hw; 3821 uint32_t eiac, eiam; 3822 uint32_t gpie = IXGBE_READ_REG(hw, IXGBE_GPIE); 3823 3824 /* interrupt types to enable */ 3825 ixgbe->eims = IXGBE_EIMS_ENABLE_MASK; /* shared code default */ 3826 ixgbe->eims &= ~IXGBE_EIMS_TCP_TIMER; /* minus tcp timer */ 3827 ixgbe->eims |= ixgbe->capab->other_intr; /* "other" interrupt types */ 3828 3829 /* enable automask on "other" causes that this adapter can generate */ 3830 eiam = ixgbe->capab->other_intr; 3831 3832 /* 3833 * msi-x mode 3834 */ 3835 if (ixgbe->intr_type == DDI_INTR_TYPE_MSIX) { 3836 /* enable autoclear but not on bits 29:20 */ 3837 eiac = (ixgbe->eims & ~IXGBE_OTHER_INTR); 3838 3839 /* general purpose interrupt enable */ 3840 gpie |= (IXGBE_GPIE_MSIX_MODE 3841 | IXGBE_GPIE_PBA_SUPPORT 3842 | IXGBE_GPIE_OCD 3843 | IXGBE_GPIE_EIAME); 3844 /* 3845 * non-msi-x mode 3846 */ 3847 } else { 3848 3849 /* disable autoclear, leave gpie at default */ 3850 eiac = 0; 3851 3852 /* 3853 * General purpose interrupt enable. 3854 * For 82599 or X540, extended interrupt automask enable 3855 * only in MSI or MSI-X mode 3856 */ 3857 if ((hw->mac.type == ixgbe_mac_82598EB) || 3858 (ixgbe->intr_type == DDI_INTR_TYPE_MSI)) { 3859 gpie |= IXGBE_GPIE_EIAME; 3860 } 3861 } 3862 3863 /* Enable specific "other" interrupt types */ 3864 switch (hw->mac.type) { 3865 case ixgbe_mac_82598EB: 3866 gpie |= ixgbe->capab->other_gpie; 3867 break; 3868 3869 case ixgbe_mac_82599EB: 3870 case ixgbe_mac_X540: 3871 gpie |= ixgbe->capab->other_gpie; 3872 3873 /* Enable RSC Delay 8us when LRO enabled */ 3874 if (ixgbe->lro_enable) { 3875 gpie |= (1 << IXGBE_GPIE_RSC_DELAY_SHIFT); 3876 } 3877 break; 3878 3879 default: 3880 break; 3881 } 3882 3883 /* write to interrupt control registers */ 3884 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 3885 IXGBE_WRITE_REG(hw, IXGBE_EIAC, eiac); 3886 IXGBE_WRITE_REG(hw, IXGBE_EIAM, eiam); 3887 IXGBE_WRITE_REG(hw, IXGBE_GPIE, gpie); 3888 IXGBE_WRITE_FLUSH(hw); 3889 } 3890 3891 /* 3892 * ixgbe_loopback_ioctl - Loopback support. 3893 */ 3894 enum ioc_reply 3895 ixgbe_loopback_ioctl(ixgbe_t *ixgbe, struct iocblk *iocp, mblk_t *mp) 3896 { 3897 lb_info_sz_t *lbsp; 3898 lb_property_t *lbpp; 3899 uint32_t *lbmp; 3900 uint32_t size; 3901 uint32_t value; 3902 3903 if (mp->b_cont == NULL) 3904 return (IOC_INVAL); 3905 3906 switch (iocp->ioc_cmd) { 3907 default: 3908 return (IOC_INVAL); 3909 3910 case LB_GET_INFO_SIZE: 3911 size = sizeof (lb_info_sz_t); 3912 if (iocp->ioc_count != size) 3913 return (IOC_INVAL); 3914 3915 value = sizeof (lb_normal); 3916 value += sizeof (lb_mac); 3917 value += sizeof (lb_external); 3918 3919 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3920 *lbsp = value; 3921 break; 3922 3923 case LB_GET_INFO: 3924 value = sizeof (lb_normal); 3925 value += sizeof (lb_mac); 3926 value += sizeof (lb_external); 3927 3928 size = value; 3929 if (iocp->ioc_count != size) 3930 return (IOC_INVAL); 3931 3932 value = 0; 3933 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3934 3935 lbpp[value++] = lb_normal; 3936 lbpp[value++] = lb_mac; 3937 lbpp[value++] = lb_external; 3938 break; 3939 3940 case LB_GET_MODE: 3941 size = sizeof (uint32_t); 3942 if (iocp->ioc_count != size) 3943 return (IOC_INVAL); 3944 3945 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3946 *lbmp = ixgbe->loopback_mode; 3947 break; 3948 3949 case LB_SET_MODE: 3950 size = 0; 3951 if (iocp->ioc_count != sizeof (uint32_t)) 3952 return (IOC_INVAL); 3953 3954 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3955 if (!ixgbe_set_loopback_mode(ixgbe, *lbmp)) 3956 return (IOC_INVAL); 3957 break; 3958 } 3959 3960 iocp->ioc_count = size; 3961 iocp->ioc_error = 0; 3962 3963 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 3964 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 3965 return (IOC_INVAL); 3966 } 3967 3968 return (IOC_REPLY); 3969 } 3970 3971 /* 3972 * ixgbe_set_loopback_mode - Setup loopback based on the loopback mode. 3973 */ 3974 static boolean_t 3975 ixgbe_set_loopback_mode(ixgbe_t *ixgbe, uint32_t mode) 3976 { 3977 if (mode == ixgbe->loopback_mode) 3978 return (B_TRUE); 3979 3980 ixgbe->loopback_mode = mode; 3981 3982 if (mode == IXGBE_LB_NONE) { 3983 /* 3984 * Reset the chip 3985 */ 3986 (void) ixgbe_reset(ixgbe); 3987 return (B_TRUE); 3988 } 3989 3990 mutex_enter(&ixgbe->gen_lock); 3991 3992 switch (mode) { 3993 default: 3994 mutex_exit(&ixgbe->gen_lock); 3995 return (B_FALSE); 3996 3997 case IXGBE_LB_EXTERNAL: 3998 break; 3999 4000 case IXGBE_LB_INTERNAL_MAC: 4001 ixgbe_set_internal_mac_loopback(ixgbe); 4002 break; 4003 } 4004 4005 mutex_exit(&ixgbe->gen_lock); 4006 4007 return (B_TRUE); 4008 } 4009 4010 /* 4011 * ixgbe_set_internal_mac_loopback - Set the internal MAC loopback mode. 4012 */ 4013 static void 4014 ixgbe_set_internal_mac_loopback(ixgbe_t *ixgbe) 4015 { 4016 struct ixgbe_hw *hw; 4017 uint32_t reg; 4018 uint8_t atlas; 4019 4020 hw = &ixgbe->hw; 4021 4022 /* 4023 * Setup MAC loopback 4024 */ 4025 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_HLREG0); 4026 reg |= IXGBE_HLREG0_LPBK; 4027 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_HLREG0, reg); 4028 4029 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4030 reg &= ~IXGBE_AUTOC_LMS_MASK; 4031 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4032 4033 /* 4034 * Disable Atlas Tx lanes to keep packets in loopback and not on wire 4035 */ 4036 switch (hw->mac.type) { 4037 case ixgbe_mac_82598EB: 4038 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4039 &atlas); 4040 atlas |= IXGBE_ATLAS_PDN_TX_REG_EN; 4041 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_LPBK, 4042 atlas); 4043 4044 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4045 &atlas); 4046 atlas |= IXGBE_ATLAS_PDN_TX_10G_QL_ALL; 4047 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_10G, 4048 atlas); 4049 4050 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4051 &atlas); 4052 atlas |= IXGBE_ATLAS_PDN_TX_1G_QL_ALL; 4053 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_1G, 4054 atlas); 4055 4056 (void) ixgbe_read_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4057 &atlas); 4058 atlas |= IXGBE_ATLAS_PDN_TX_AN_QL_ALL; 4059 (void) ixgbe_write_analog_reg8(&ixgbe->hw, IXGBE_ATLAS_PDN_AN, 4060 atlas); 4061 break; 4062 4063 case ixgbe_mac_82599EB: 4064 case ixgbe_mac_X540: 4065 reg = IXGBE_READ_REG(&ixgbe->hw, IXGBE_AUTOC); 4066 reg |= (IXGBE_AUTOC_FLU | 4067 IXGBE_AUTOC_10G_KX4); 4068 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_AUTOC, reg); 4069 4070 (void) ixgbe_setup_link(&ixgbe->hw, IXGBE_LINK_SPEED_10GB_FULL, 4071 B_FALSE, B_TRUE); 4072 break; 4073 4074 default: 4075 break; 4076 } 4077 } 4078 4079 #pragma inline(ixgbe_intr_rx_work) 4080 /* 4081 * ixgbe_intr_rx_work - RX processing of ISR. 4082 */ 4083 static void 4084 ixgbe_intr_rx_work(ixgbe_rx_ring_t *rx_ring) 4085 { 4086 mblk_t *mp; 4087 4088 mutex_enter(&rx_ring->rx_lock); 4089 4090 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4091 mutex_exit(&rx_ring->rx_lock); 4092 4093 if (mp != NULL) 4094 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4095 rx_ring->ring_gen_num); 4096 } 4097 4098 #pragma inline(ixgbe_intr_tx_work) 4099 /* 4100 * ixgbe_intr_tx_work - TX processing of ISR. 4101 */ 4102 static void 4103 ixgbe_intr_tx_work(ixgbe_tx_ring_t *tx_ring) 4104 { 4105 ixgbe_t *ixgbe = tx_ring->ixgbe; 4106 4107 /* 4108 * Recycle the tx descriptors 4109 */ 4110 tx_ring->tx_recycle(tx_ring); 4111 4112 /* 4113 * Schedule the re-transmit 4114 */ 4115 if (tx_ring->reschedule && 4116 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)) { 4117 tx_ring->reschedule = B_FALSE; 4118 mac_tx_ring_update(tx_ring->ixgbe->mac_hdl, 4119 tx_ring->ring_handle); 4120 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4121 } 4122 } 4123 4124 #pragma inline(ixgbe_intr_other_work) 4125 /* 4126 * ixgbe_intr_other_work - Process interrupt types other than tx/rx 4127 */ 4128 static void 4129 ixgbe_intr_other_work(ixgbe_t *ixgbe, uint32_t eicr) 4130 { 4131 ASSERT(mutex_owned(&ixgbe->gen_lock)); 4132 4133 /* 4134 * handle link status change 4135 */ 4136 if (eicr & IXGBE_EICR_LSC) { 4137 ixgbe_driver_link_check(ixgbe); 4138 ixgbe_get_hw_state(ixgbe); 4139 } 4140 4141 /* 4142 * check for fan failure on adapters with fans 4143 */ 4144 if ((ixgbe->capab->flags & IXGBE_FLAG_FAN_FAIL_CAPABLE) && 4145 (eicr & IXGBE_EICR_GPI_SDP1)) { 4146 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_OVERTEMP); 4147 4148 /* 4149 * Disable the adapter interrupts 4150 */ 4151 ixgbe_disable_adapter_interrupts(ixgbe); 4152 4153 /* 4154 * Disable Rx/Tx units 4155 */ 4156 (void) ixgbe_stop_adapter(&ixgbe->hw); 4157 4158 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_LOST); 4159 ixgbe_error(ixgbe, 4160 "Problem: Network adapter has been stopped " 4161 "because the fan has stopped.\n"); 4162 ixgbe_error(ixgbe, 4163 "Action: Replace the adapter.\n"); 4164 4165 /* re-enable the interrupt, which was automasked */ 4166 ixgbe->eims |= IXGBE_EICR_GPI_SDP1; 4167 } 4168 4169 /* 4170 * Do SFP check for adapters with hot-plug capability 4171 */ 4172 if ((ixgbe->capab->flags & IXGBE_FLAG_SFP_PLUG_CAPABLE) && 4173 ((eicr & IXGBE_EICR_GPI_SDP1) || (eicr & IXGBE_EICR_GPI_SDP2))) { 4174 ixgbe->eicr = eicr; 4175 if ((ddi_taskq_dispatch(ixgbe->sfp_taskq, 4176 ixgbe_sfp_check, (void *)ixgbe, 4177 DDI_NOSLEEP)) != DDI_SUCCESS) { 4178 ixgbe_log(ixgbe, "No memory available to dispatch " 4179 "taskq for SFP check"); 4180 } 4181 } 4182 4183 /* 4184 * Do over-temperature check for adapters with temp sensor 4185 */ 4186 if ((ixgbe->capab->flags & IXGBE_FLAG_TEMP_SENSOR_CAPABLE) && 4187 ((eicr & IXGBE_EICR_GPI_SDP0) || (eicr & IXGBE_EICR_LSC))) { 4188 ixgbe->eicr = eicr; 4189 if ((ddi_taskq_dispatch(ixgbe->overtemp_taskq, 4190 ixgbe_overtemp_check, (void *)ixgbe, 4191 DDI_NOSLEEP)) != DDI_SUCCESS) { 4192 ixgbe_log(ixgbe, "No memory available to dispatch " 4193 "taskq for overtemp check"); 4194 } 4195 } 4196 } 4197 4198 /* 4199 * ixgbe_intr_legacy - Interrupt handler for legacy interrupts. 4200 */ 4201 static uint_t 4202 ixgbe_intr_legacy(void *arg1, void *arg2) 4203 { 4204 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4205 struct ixgbe_hw *hw = &ixgbe->hw; 4206 ixgbe_tx_ring_t *tx_ring; 4207 ixgbe_rx_ring_t *rx_ring; 4208 uint32_t eicr; 4209 mblk_t *mp; 4210 boolean_t tx_reschedule; 4211 uint_t result; 4212 4213 _NOTE(ARGUNUSED(arg2)); 4214 4215 mutex_enter(&ixgbe->gen_lock); 4216 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 4217 mutex_exit(&ixgbe->gen_lock); 4218 return (DDI_INTR_UNCLAIMED); 4219 } 4220 4221 mp = NULL; 4222 tx_reschedule = B_FALSE; 4223 4224 /* 4225 * Any bit set in eicr: claim this interrupt 4226 */ 4227 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4228 4229 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4230 mutex_exit(&ixgbe->gen_lock); 4231 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4232 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4233 return (DDI_INTR_CLAIMED); 4234 } 4235 4236 if (eicr) { 4237 /* 4238 * For legacy interrupt, we have only one interrupt, 4239 * so we have only one rx ring and one tx ring enabled. 4240 */ 4241 ASSERT(ixgbe->num_rx_rings == 1); 4242 ASSERT(ixgbe->num_tx_rings == 1); 4243 4244 /* 4245 * For legacy interrupt, rx rings[0] will use RTxQ[0]. 4246 */ 4247 if (eicr & 0x1) { 4248 ixgbe->eimc |= IXGBE_EICR_RTX_QUEUE; 4249 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4250 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4251 /* 4252 * Clean the rx descriptors 4253 */ 4254 rx_ring = &ixgbe->rx_rings[0]; 4255 mp = ixgbe_ring_rx(rx_ring, IXGBE_POLL_NULL); 4256 } 4257 4258 /* 4259 * For legacy interrupt, tx rings[0] will use RTxQ[1]. 4260 */ 4261 if (eicr & 0x2) { 4262 /* 4263 * Recycle the tx descriptors 4264 */ 4265 tx_ring = &ixgbe->tx_rings[0]; 4266 tx_ring->tx_recycle(tx_ring); 4267 4268 /* 4269 * Schedule the re-transmit 4270 */ 4271 tx_reschedule = (tx_ring->reschedule && 4272 (tx_ring->tbd_free >= ixgbe->tx_resched_thresh)); 4273 } 4274 4275 /* any interrupt type other than tx/rx */ 4276 if (eicr & ixgbe->capab->other_intr) { 4277 switch (hw->mac.type) { 4278 case ixgbe_mac_82598EB: 4279 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4280 break; 4281 4282 case ixgbe_mac_82599EB: 4283 case ixgbe_mac_X540: 4284 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4285 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4286 break; 4287 4288 default: 4289 break; 4290 } 4291 ixgbe_intr_other_work(ixgbe, eicr); 4292 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4293 } 4294 4295 mutex_exit(&ixgbe->gen_lock); 4296 4297 result = DDI_INTR_CLAIMED; 4298 } else { 4299 mutex_exit(&ixgbe->gen_lock); 4300 4301 /* 4302 * No interrupt cause bits set: don't claim this interrupt. 4303 */ 4304 result = DDI_INTR_UNCLAIMED; 4305 } 4306 4307 /* re-enable the interrupts which were automasked */ 4308 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4309 4310 /* 4311 * Do the following work outside of the gen_lock 4312 */ 4313 if (mp != NULL) { 4314 mac_rx_ring(rx_ring->ixgbe->mac_hdl, rx_ring->ring_handle, mp, 4315 rx_ring->ring_gen_num); 4316 } 4317 4318 if (tx_reschedule) { 4319 tx_ring->reschedule = B_FALSE; 4320 mac_tx_ring_update(ixgbe->mac_hdl, tx_ring->ring_handle); 4321 IXGBE_DEBUG_STAT(tx_ring->stat_reschedule); 4322 } 4323 4324 return (result); 4325 } 4326 4327 /* 4328 * ixgbe_intr_msi - Interrupt handler for MSI. 4329 */ 4330 static uint_t 4331 ixgbe_intr_msi(void *arg1, void *arg2) 4332 { 4333 ixgbe_t *ixgbe = (ixgbe_t *)arg1; 4334 struct ixgbe_hw *hw = &ixgbe->hw; 4335 uint32_t eicr; 4336 4337 _NOTE(ARGUNUSED(arg2)); 4338 4339 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4340 4341 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != DDI_FM_OK) { 4342 ddi_fm_service_impact(ixgbe->dip, DDI_SERVICE_DEGRADED); 4343 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4344 return (DDI_INTR_CLAIMED); 4345 } 4346 4347 /* 4348 * For MSI interrupt, we have only one vector, 4349 * so we have only one rx ring and one tx ring enabled. 4350 */ 4351 ASSERT(ixgbe->num_rx_rings == 1); 4352 ASSERT(ixgbe->num_tx_rings == 1); 4353 4354 /* 4355 * For MSI interrupt, rx rings[0] will use RTxQ[0]. 4356 */ 4357 if (eicr & 0x1) { 4358 ixgbe_intr_rx_work(&ixgbe->rx_rings[0]); 4359 } 4360 4361 /* 4362 * For MSI interrupt, tx rings[0] will use RTxQ[1]. 4363 */ 4364 if (eicr & 0x2) { 4365 ixgbe_intr_tx_work(&ixgbe->tx_rings[0]); 4366 } 4367 4368 /* any interrupt type other than tx/rx */ 4369 if (eicr & ixgbe->capab->other_intr) { 4370 mutex_enter(&ixgbe->gen_lock); 4371 switch (hw->mac.type) { 4372 case ixgbe_mac_82598EB: 4373 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4374 break; 4375 4376 case ixgbe_mac_82599EB: 4377 case ixgbe_mac_X540: 4378 ixgbe->eimc = IXGBE_82599_OTHER_INTR; 4379 IXGBE_WRITE_REG(hw, IXGBE_EIMC, ixgbe->eimc); 4380 break; 4381 4382 default: 4383 break; 4384 } 4385 ixgbe_intr_other_work(ixgbe, eicr); 4386 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4387 mutex_exit(&ixgbe->gen_lock); 4388 } 4389 4390 /* re-enable the interrupts which were automasked */ 4391 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4392 4393 return (DDI_INTR_CLAIMED); 4394 } 4395 4396 /* 4397 * ixgbe_intr_msix - Interrupt handler for MSI-X. 4398 */ 4399 static uint_t 4400 ixgbe_intr_msix(void *arg1, void *arg2) 4401 { 4402 ixgbe_intr_vector_t *vect = (ixgbe_intr_vector_t *)arg1; 4403 ixgbe_t *ixgbe = vect->ixgbe; 4404 struct ixgbe_hw *hw = &ixgbe->hw; 4405 uint32_t eicr; 4406 int r_idx = 0; 4407 4408 _NOTE(ARGUNUSED(arg2)); 4409 4410 /* 4411 * Clean each rx ring that has its bit set in the map 4412 */ 4413 r_idx = bt_getlowbit(vect->rx_map, 0, (ixgbe->num_rx_rings - 1)); 4414 while (r_idx >= 0) { 4415 ixgbe_intr_rx_work(&ixgbe->rx_rings[r_idx]); 4416 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 4417 (ixgbe->num_rx_rings - 1)); 4418 } 4419 4420 /* 4421 * Clean each tx ring that has its bit set in the map 4422 */ 4423 r_idx = bt_getlowbit(vect->tx_map, 0, (ixgbe->num_tx_rings - 1)); 4424 while (r_idx >= 0) { 4425 ixgbe_intr_tx_work(&ixgbe->tx_rings[r_idx]); 4426 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 4427 (ixgbe->num_tx_rings - 1)); 4428 } 4429 4430 4431 /* 4432 * Clean other interrupt (link change) that has its bit set in the map 4433 */ 4434 if (BT_TEST(vect->other_map, 0) == 1) { 4435 eicr = IXGBE_READ_REG(hw, IXGBE_EICR); 4436 4437 if (ixgbe_check_acc_handle(ixgbe->osdep.reg_handle) != 4438 DDI_FM_OK) { 4439 ddi_fm_service_impact(ixgbe->dip, 4440 DDI_SERVICE_DEGRADED); 4441 atomic_or_32(&ixgbe->ixgbe_state, IXGBE_ERROR); 4442 return (DDI_INTR_CLAIMED); 4443 } 4444 4445 /* 4446 * Check "other" cause bits: any interrupt type other than tx/rx 4447 */ 4448 if (eicr & ixgbe->capab->other_intr) { 4449 mutex_enter(&ixgbe->gen_lock); 4450 switch (hw->mac.type) { 4451 case ixgbe_mac_82598EB: 4452 ixgbe->eims &= ~(eicr & IXGBE_OTHER_INTR); 4453 ixgbe_intr_other_work(ixgbe, eicr); 4454 break; 4455 4456 case ixgbe_mac_82599EB: 4457 case ixgbe_mac_X540: 4458 ixgbe->eims |= IXGBE_EICR_RTX_QUEUE; 4459 ixgbe_intr_other_work(ixgbe, eicr); 4460 break; 4461 4462 default: 4463 break; 4464 } 4465 mutex_exit(&ixgbe->gen_lock); 4466 } 4467 4468 /* re-enable the interrupts which were automasked */ 4469 IXGBE_WRITE_REG(hw, IXGBE_EIMS, ixgbe->eims); 4470 } 4471 4472 return (DDI_INTR_CLAIMED); 4473 } 4474 4475 /* 4476 * ixgbe_alloc_intrs - Allocate interrupts for the driver. 4477 * 4478 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4479 * if not successful, try Legacy. 4480 * ixgbe->intr_force can be used to force sequence to start with 4481 * any of the 3 types. 4482 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4483 */ 4484 static int 4485 ixgbe_alloc_intrs(ixgbe_t *ixgbe) 4486 { 4487 dev_info_t *devinfo; 4488 int intr_types; 4489 int rc; 4490 4491 devinfo = ixgbe->dip; 4492 4493 /* 4494 * Get supported interrupt types 4495 */ 4496 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4497 4498 if (rc != DDI_SUCCESS) { 4499 ixgbe_log(ixgbe, 4500 "Get supported interrupt types failed: %d", rc); 4501 return (IXGBE_FAILURE); 4502 } 4503 IXGBE_DEBUGLOG_1(ixgbe, "Supported interrupt types: %x", intr_types); 4504 4505 ixgbe->intr_type = 0; 4506 4507 /* 4508 * Install MSI-X interrupts 4509 */ 4510 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4511 (ixgbe->intr_force <= IXGBE_INTR_MSIX)) { 4512 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSIX); 4513 if (rc == IXGBE_SUCCESS) 4514 return (IXGBE_SUCCESS); 4515 4516 ixgbe_log(ixgbe, 4517 "Allocate MSI-X failed, trying MSI interrupts..."); 4518 } 4519 4520 /* 4521 * MSI-X not used, force rings and groups to 1 4522 */ 4523 ixgbe->num_rx_rings = 1; 4524 ixgbe->num_rx_groups = 1; 4525 ixgbe->num_tx_rings = 1; 4526 ixgbe->classify_mode = IXGBE_CLASSIFY_NONE; 4527 ixgbe_log(ixgbe, 4528 "MSI-X not used, force rings and groups number to 1"); 4529 4530 /* 4531 * Install MSI interrupts 4532 */ 4533 if ((intr_types & DDI_INTR_TYPE_MSI) && 4534 (ixgbe->intr_force <= IXGBE_INTR_MSI)) { 4535 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_MSI); 4536 if (rc == IXGBE_SUCCESS) 4537 return (IXGBE_SUCCESS); 4538 4539 ixgbe_log(ixgbe, 4540 "Allocate MSI failed, trying Legacy interrupts..."); 4541 } 4542 4543 /* 4544 * Install legacy interrupts 4545 */ 4546 if (intr_types & DDI_INTR_TYPE_FIXED) { 4547 rc = ixgbe_alloc_intr_handles(ixgbe, DDI_INTR_TYPE_FIXED); 4548 if (rc == IXGBE_SUCCESS) 4549 return (IXGBE_SUCCESS); 4550 4551 ixgbe_log(ixgbe, 4552 "Allocate Legacy interrupts failed"); 4553 } 4554 4555 /* 4556 * If none of the 3 types succeeded, return failure 4557 */ 4558 return (IXGBE_FAILURE); 4559 } 4560 4561 /* 4562 * ixgbe_alloc_intr_handles - Allocate interrupt handles. 4563 * 4564 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4565 * if fewer than 2 handles are available, return failure. 4566 * Upon success, this maps the vectors to rx and tx rings for 4567 * interrupts. 4568 */ 4569 static int 4570 ixgbe_alloc_intr_handles(ixgbe_t *ixgbe, int intr_type) 4571 { 4572 dev_info_t *devinfo; 4573 int request, count, actual; 4574 int minimum; 4575 int rc; 4576 uint32_t ring_per_group; 4577 4578 devinfo = ixgbe->dip; 4579 4580 switch (intr_type) { 4581 case DDI_INTR_TYPE_FIXED: 4582 request = 1; /* Request 1 legacy interrupt handle */ 4583 minimum = 1; 4584 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: legacy"); 4585 break; 4586 4587 case DDI_INTR_TYPE_MSI: 4588 request = 1; /* Request 1 MSI interrupt handle */ 4589 minimum = 1; 4590 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI"); 4591 break; 4592 4593 case DDI_INTR_TYPE_MSIX: 4594 /* 4595 * Best number of vectors for the adapter is 4596 * (# rx rings + # tx rings), however we will 4597 * limit the request number. 4598 */ 4599 request = min(16, ixgbe->num_rx_rings + ixgbe->num_tx_rings); 4600 if (request > ixgbe->capab->max_ring_vect) 4601 request = ixgbe->capab->max_ring_vect; 4602 minimum = 1; 4603 IXGBE_DEBUGLOG_0(ixgbe, "interrupt type: MSI-X"); 4604 break; 4605 4606 default: 4607 ixgbe_log(ixgbe, 4608 "invalid call to ixgbe_alloc_intr_handles(): %d\n", 4609 intr_type); 4610 return (IXGBE_FAILURE); 4611 } 4612 IXGBE_DEBUGLOG_2(ixgbe, "interrupt handles requested: %d minimum: %d", 4613 request, minimum); 4614 4615 /* 4616 * Get number of supported interrupts 4617 */ 4618 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4619 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4620 ixgbe_log(ixgbe, 4621 "Get interrupt number failed. Return: %d, count: %d", 4622 rc, count); 4623 return (IXGBE_FAILURE); 4624 } 4625 IXGBE_DEBUGLOG_1(ixgbe, "interrupts supported: %d", count); 4626 4627 actual = 0; 4628 ixgbe->intr_cnt = 0; 4629 ixgbe->intr_cnt_max = 0; 4630 ixgbe->intr_cnt_min = 0; 4631 4632 /* 4633 * Allocate an array of interrupt handles 4634 */ 4635 ixgbe->intr_size = request * sizeof (ddi_intr_handle_t); 4636 ixgbe->htable = kmem_alloc(ixgbe->intr_size, KM_SLEEP); 4637 4638 rc = ddi_intr_alloc(devinfo, ixgbe->htable, intr_type, 0, 4639 request, &actual, DDI_INTR_ALLOC_NORMAL); 4640 if (rc != DDI_SUCCESS) { 4641 ixgbe_log(ixgbe, "Allocate interrupts failed. " 4642 "return: %d, request: %d, actual: %d", 4643 rc, request, actual); 4644 goto alloc_handle_fail; 4645 } 4646 IXGBE_DEBUGLOG_1(ixgbe, "interrupts actually allocated: %d", actual); 4647 4648 /* 4649 * upper/lower limit of interrupts 4650 */ 4651 ixgbe->intr_cnt = actual; 4652 ixgbe->intr_cnt_max = request; 4653 ixgbe->intr_cnt_min = minimum; 4654 4655 /* 4656 * rss number per group should not exceed the rx interrupt number, 4657 * else need to adjust rx ring number. 4658 */ 4659 ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 4660 ASSERT((ixgbe->num_rx_rings % ixgbe->num_rx_groups) == 0); 4661 if (actual < ring_per_group) { 4662 ixgbe->num_rx_rings = ixgbe->num_rx_groups * actual; 4663 ixgbe_setup_vmdq_rss_conf(ixgbe); 4664 } 4665 4666 /* 4667 * Now we know the actual number of vectors. Here we map the vector 4668 * to other, rx rings and tx ring. 4669 */ 4670 if (actual < minimum) { 4671 ixgbe_log(ixgbe, "Insufficient interrupt handles available: %d", 4672 actual); 4673 goto alloc_handle_fail; 4674 } 4675 4676 /* 4677 * Get priority for first vector, assume remaining are all the same 4678 */ 4679 rc = ddi_intr_get_pri(ixgbe->htable[0], &ixgbe->intr_pri); 4680 if (rc != DDI_SUCCESS) { 4681 ixgbe_log(ixgbe, 4682 "Get interrupt priority failed: %d", rc); 4683 goto alloc_handle_fail; 4684 } 4685 4686 rc = ddi_intr_get_cap(ixgbe->htable[0], &ixgbe->intr_cap); 4687 if (rc != DDI_SUCCESS) { 4688 ixgbe_log(ixgbe, 4689 "Get interrupt cap failed: %d", rc); 4690 goto alloc_handle_fail; 4691 } 4692 4693 ixgbe->intr_type = intr_type; 4694 4695 return (IXGBE_SUCCESS); 4696 4697 alloc_handle_fail: 4698 ixgbe_rem_intrs(ixgbe); 4699 4700 return (IXGBE_FAILURE); 4701 } 4702 4703 /* 4704 * ixgbe_add_intr_handlers - Add interrupt handlers based on the interrupt type. 4705 * 4706 * Before adding the interrupt handlers, the interrupt vectors have 4707 * been allocated, and the rx/tx rings have also been allocated. 4708 */ 4709 static int 4710 ixgbe_add_intr_handlers(ixgbe_t *ixgbe) 4711 { 4712 int vector = 0; 4713 int rc; 4714 4715 switch (ixgbe->intr_type) { 4716 case DDI_INTR_TYPE_MSIX: 4717 /* 4718 * Add interrupt handler for all vectors 4719 */ 4720 for (vector = 0; vector < ixgbe->intr_cnt; vector++) { 4721 /* 4722 * install pointer to vect_map[vector] 4723 */ 4724 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4725 (ddi_intr_handler_t *)ixgbe_intr_msix, 4726 (void *)&ixgbe->vect_map[vector], NULL); 4727 4728 if (rc != DDI_SUCCESS) { 4729 ixgbe_log(ixgbe, 4730 "Add interrupt handler failed. " 4731 "return: %d, vector: %d", rc, vector); 4732 for (vector--; vector >= 0; vector--) { 4733 (void) ddi_intr_remove_handler( 4734 ixgbe->htable[vector]); 4735 } 4736 return (IXGBE_FAILURE); 4737 } 4738 } 4739 4740 break; 4741 4742 case DDI_INTR_TYPE_MSI: 4743 /* 4744 * Add interrupt handlers for the only vector 4745 */ 4746 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4747 (ddi_intr_handler_t *)ixgbe_intr_msi, 4748 (void *)ixgbe, NULL); 4749 4750 if (rc != DDI_SUCCESS) { 4751 ixgbe_log(ixgbe, 4752 "Add MSI interrupt handler failed: %d", rc); 4753 return (IXGBE_FAILURE); 4754 } 4755 4756 break; 4757 4758 case DDI_INTR_TYPE_FIXED: 4759 /* 4760 * Add interrupt handlers for the only vector 4761 */ 4762 rc = ddi_intr_add_handler(ixgbe->htable[vector], 4763 (ddi_intr_handler_t *)ixgbe_intr_legacy, 4764 (void *)ixgbe, NULL); 4765 4766 if (rc != DDI_SUCCESS) { 4767 ixgbe_log(ixgbe, 4768 "Add legacy interrupt handler failed: %d", rc); 4769 return (IXGBE_FAILURE); 4770 } 4771 4772 break; 4773 4774 default: 4775 return (IXGBE_FAILURE); 4776 } 4777 4778 return (IXGBE_SUCCESS); 4779 } 4780 4781 #pragma inline(ixgbe_map_rxring_to_vector) 4782 /* 4783 * ixgbe_map_rxring_to_vector - Map given rx ring to given interrupt vector. 4784 */ 4785 static void 4786 ixgbe_map_rxring_to_vector(ixgbe_t *ixgbe, int r_idx, int v_idx) 4787 { 4788 /* 4789 * Set bit in map 4790 */ 4791 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 4792 4793 /* 4794 * Count bits set 4795 */ 4796 ixgbe->vect_map[v_idx].rxr_cnt++; 4797 4798 /* 4799 * Remember bit position 4800 */ 4801 ixgbe->rx_rings[r_idx].intr_vector = v_idx; 4802 ixgbe->rx_rings[r_idx].vect_bit = 1 << v_idx; 4803 } 4804 4805 #pragma inline(ixgbe_map_txring_to_vector) 4806 /* 4807 * ixgbe_map_txring_to_vector - Map given tx ring to given interrupt vector. 4808 */ 4809 static void 4810 ixgbe_map_txring_to_vector(ixgbe_t *ixgbe, int t_idx, int v_idx) 4811 { 4812 /* 4813 * Set bit in map 4814 */ 4815 BT_SET(ixgbe->vect_map[v_idx].tx_map, t_idx); 4816 4817 /* 4818 * Count bits set 4819 */ 4820 ixgbe->vect_map[v_idx].txr_cnt++; 4821 4822 /* 4823 * Remember bit position 4824 */ 4825 ixgbe->tx_rings[t_idx].intr_vector = v_idx; 4826 ixgbe->tx_rings[t_idx].vect_bit = 1 << v_idx; 4827 } 4828 4829 /* 4830 * ixgbe_setup_ivar - Set the given entry in the given interrupt vector 4831 * allocation register (IVAR). 4832 * cause: 4833 * -1 : other cause 4834 * 0 : rx 4835 * 1 : tx 4836 */ 4837 static void 4838 ixgbe_setup_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, uint8_t msix_vector, 4839 int8_t cause) 4840 { 4841 struct ixgbe_hw *hw = &ixgbe->hw; 4842 u32 ivar, index; 4843 4844 switch (hw->mac.type) { 4845 case ixgbe_mac_82598EB: 4846 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4847 if (cause == -1) { 4848 cause = 0; 4849 } 4850 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4851 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4852 ivar &= ~(0xFF << (8 * (intr_alloc_entry & 0x3))); 4853 ivar |= (msix_vector << (8 * (intr_alloc_entry & 0x3))); 4854 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4855 break; 4856 4857 case ixgbe_mac_82599EB: 4858 case ixgbe_mac_X540: 4859 if (cause == -1) { 4860 /* other causes */ 4861 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4862 index = (intr_alloc_entry & 1) * 8; 4863 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4864 ivar &= ~(0xFF << index); 4865 ivar |= (msix_vector << index); 4866 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4867 } else { 4868 /* tx or rx causes */ 4869 msix_vector |= IXGBE_IVAR_ALLOC_VAL; 4870 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4871 ivar = IXGBE_READ_REG(hw, 4872 IXGBE_IVAR(intr_alloc_entry >> 1)); 4873 ivar &= ~(0xFF << index); 4874 ivar |= (msix_vector << index); 4875 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4876 ivar); 4877 } 4878 break; 4879 4880 default: 4881 break; 4882 } 4883 } 4884 4885 /* 4886 * ixgbe_enable_ivar - Enable the given entry by setting the VAL bit of 4887 * given interrupt vector allocation register (IVAR). 4888 * cause: 4889 * -1 : other cause 4890 * 0 : rx 4891 * 1 : tx 4892 */ 4893 static void 4894 ixgbe_enable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4895 { 4896 struct ixgbe_hw *hw = &ixgbe->hw; 4897 u32 ivar, index; 4898 4899 switch (hw->mac.type) { 4900 case ixgbe_mac_82598EB: 4901 if (cause == -1) { 4902 cause = 0; 4903 } 4904 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4905 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4906 ivar |= (IXGBE_IVAR_ALLOC_VAL << (8 * 4907 (intr_alloc_entry & 0x3))); 4908 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4909 break; 4910 4911 case ixgbe_mac_82599EB: 4912 case ixgbe_mac_X540: 4913 if (cause == -1) { 4914 /* other causes */ 4915 index = (intr_alloc_entry & 1) * 8; 4916 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4917 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4918 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4919 } else { 4920 /* tx or rx causes */ 4921 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4922 ivar = IXGBE_READ_REG(hw, 4923 IXGBE_IVAR(intr_alloc_entry >> 1)); 4924 ivar |= (IXGBE_IVAR_ALLOC_VAL << index); 4925 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4926 ivar); 4927 } 4928 break; 4929 4930 default: 4931 break; 4932 } 4933 } 4934 4935 /* 4936 * ixgbe_disable_ivar - Disble the given entry by clearing the VAL bit of 4937 * given interrupt vector allocation register (IVAR). 4938 * cause: 4939 * -1 : other cause 4940 * 0 : rx 4941 * 1 : tx 4942 */ 4943 static void 4944 ixgbe_disable_ivar(ixgbe_t *ixgbe, uint16_t intr_alloc_entry, int8_t cause) 4945 { 4946 struct ixgbe_hw *hw = &ixgbe->hw; 4947 u32 ivar, index; 4948 4949 switch (hw->mac.type) { 4950 case ixgbe_mac_82598EB: 4951 if (cause == -1) { 4952 cause = 0; 4953 } 4954 index = (((cause * 64) + intr_alloc_entry) >> 2) & 0x1F; 4955 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR(index)); 4956 ivar &= ~(IXGBE_IVAR_ALLOC_VAL<< (8 * 4957 (intr_alloc_entry & 0x3))); 4958 IXGBE_WRITE_REG(hw, IXGBE_IVAR(index), ivar); 4959 break; 4960 4961 case ixgbe_mac_82599EB: 4962 case ixgbe_mac_X540: 4963 if (cause == -1) { 4964 /* other causes */ 4965 index = (intr_alloc_entry & 1) * 8; 4966 ivar = IXGBE_READ_REG(hw, IXGBE_IVAR_MISC); 4967 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4968 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, ivar); 4969 } else { 4970 /* tx or rx causes */ 4971 index = ((16 * (intr_alloc_entry & 1)) + (8 * cause)); 4972 ivar = IXGBE_READ_REG(hw, 4973 IXGBE_IVAR(intr_alloc_entry >> 1)); 4974 ivar &= ~(IXGBE_IVAR_ALLOC_VAL << index); 4975 IXGBE_WRITE_REG(hw, IXGBE_IVAR(intr_alloc_entry >> 1), 4976 ivar); 4977 } 4978 break; 4979 4980 default: 4981 break; 4982 } 4983 } 4984 4985 /* 4986 * Convert the rx ring index driver maintained to the rx ring index 4987 * in h/w. 4988 */ 4989 static uint32_t 4990 ixgbe_get_hw_rx_index(ixgbe_t *ixgbe, uint32_t sw_rx_index) 4991 { 4992 4993 struct ixgbe_hw *hw = &ixgbe->hw; 4994 uint32_t rx_ring_per_group, hw_rx_index; 4995 4996 if (ixgbe->classify_mode == IXGBE_CLASSIFY_RSS || 4997 ixgbe->classify_mode == IXGBE_CLASSIFY_NONE) { 4998 return (sw_rx_index); 4999 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ) { 5000 switch (hw->mac.type) { 5001 case ixgbe_mac_82598EB: 5002 return (sw_rx_index); 5003 5004 case ixgbe_mac_82599EB: 5005 case ixgbe_mac_X540: 5006 return (sw_rx_index * 2); 5007 5008 default: 5009 break; 5010 } 5011 } else if (ixgbe->classify_mode == IXGBE_CLASSIFY_VMDQ_RSS) { 5012 rx_ring_per_group = ixgbe->num_rx_rings / ixgbe->num_rx_groups; 5013 5014 switch (hw->mac.type) { 5015 case ixgbe_mac_82598EB: 5016 hw_rx_index = (sw_rx_index / rx_ring_per_group) * 5017 16 + (sw_rx_index % rx_ring_per_group); 5018 return (hw_rx_index); 5019 5020 case ixgbe_mac_82599EB: 5021 case ixgbe_mac_X540: 5022 if (ixgbe->num_rx_groups > 32) { 5023 hw_rx_index = (sw_rx_index / 5024 rx_ring_per_group) * 2 + 5025 (sw_rx_index % rx_ring_per_group); 5026 } else { 5027 hw_rx_index = (sw_rx_index / 5028 rx_ring_per_group) * 4 + 5029 (sw_rx_index % rx_ring_per_group); 5030 } 5031 return (hw_rx_index); 5032 5033 default: 5034 break; 5035 } 5036 } 5037 5038 /* 5039 * Should never reach. Just to make compiler happy. 5040 */ 5041 return (sw_rx_index); 5042 } 5043 5044 /* 5045 * ixgbe_map_intrs_to_vectors - Map different interrupts to MSI-X vectors. 5046 * 5047 * For MSI-X, here will map rx interrupt, tx interrupt and other interrupt 5048 * to vector[0 - (intr_cnt -1)]. 5049 */ 5050 static int 5051 ixgbe_map_intrs_to_vectors(ixgbe_t *ixgbe) 5052 { 5053 int i, vector = 0; 5054 5055 /* initialize vector map */ 5056 bzero(&ixgbe->vect_map, sizeof (ixgbe->vect_map)); 5057 for (i = 0; i < ixgbe->intr_cnt; i++) { 5058 ixgbe->vect_map[i].ixgbe = ixgbe; 5059 } 5060 5061 /* 5062 * non-MSI-X case is very simple: rx rings[0] on RTxQ[0], 5063 * tx rings[0] on RTxQ[1]. 5064 */ 5065 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5066 ixgbe_map_rxring_to_vector(ixgbe, 0, 0); 5067 ixgbe_map_txring_to_vector(ixgbe, 0, 1); 5068 return (IXGBE_SUCCESS); 5069 } 5070 5071 /* 5072 * Interrupts/vectors mapping for MSI-X 5073 */ 5074 5075 /* 5076 * Map other interrupt to vector 0, 5077 * Set bit in map and count the bits set. 5078 */ 5079 BT_SET(ixgbe->vect_map[vector].other_map, 0); 5080 ixgbe->vect_map[vector].other_cnt++; 5081 5082 /* 5083 * Map rx ring interrupts to vectors 5084 */ 5085 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5086 ixgbe_map_rxring_to_vector(ixgbe, i, vector); 5087 vector = (vector +1) % ixgbe->intr_cnt; 5088 } 5089 5090 /* 5091 * Map tx ring interrupts to vectors 5092 */ 5093 for (i = 0; i < ixgbe->num_tx_rings; i++) { 5094 ixgbe_map_txring_to_vector(ixgbe, i, vector); 5095 vector = (vector +1) % ixgbe->intr_cnt; 5096 } 5097 5098 return (IXGBE_SUCCESS); 5099 } 5100 5101 /* 5102 * ixgbe_setup_adapter_vector - Setup the adapter interrupt vector(s). 5103 * 5104 * This relies on ring/vector mapping already set up in the 5105 * vect_map[] structures 5106 */ 5107 static void 5108 ixgbe_setup_adapter_vector(ixgbe_t *ixgbe) 5109 { 5110 struct ixgbe_hw *hw = &ixgbe->hw; 5111 ixgbe_intr_vector_t *vect; /* vector bitmap */ 5112 int r_idx; /* ring index */ 5113 int v_idx; /* vector index */ 5114 uint32_t hw_index; 5115 5116 /* 5117 * Clear any previous entries 5118 */ 5119 switch (hw->mac.type) { 5120 case ixgbe_mac_82598EB: 5121 for (v_idx = 0; v_idx < 25; v_idx++) 5122 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5123 break; 5124 5125 case ixgbe_mac_82599EB: 5126 case ixgbe_mac_X540: 5127 for (v_idx = 0; v_idx < 64; v_idx++) 5128 IXGBE_WRITE_REG(hw, IXGBE_IVAR(v_idx), 0); 5129 IXGBE_WRITE_REG(hw, IXGBE_IVAR_MISC, 0); 5130 break; 5131 5132 default: 5133 break; 5134 } 5135 5136 /* 5137 * For non MSI-X interrupt, rx rings[0] will use RTxQ[0], and 5138 * tx rings[0] will use RTxQ[1]. 5139 */ 5140 if (ixgbe->intr_type != DDI_INTR_TYPE_MSIX) { 5141 ixgbe_setup_ivar(ixgbe, 0, 0, 0); 5142 ixgbe_setup_ivar(ixgbe, 0, 1, 1); 5143 return; 5144 } 5145 5146 /* 5147 * For MSI-X interrupt, "Other" is always on vector[0]. 5148 */ 5149 ixgbe_setup_ivar(ixgbe, IXGBE_IVAR_OTHER_CAUSES_INDEX, 0, -1); 5150 5151 /* 5152 * For each interrupt vector, populate the IVAR table 5153 */ 5154 for (v_idx = 0; v_idx < ixgbe->intr_cnt; v_idx++) { 5155 vect = &ixgbe->vect_map[v_idx]; 5156 5157 /* 5158 * For each rx ring bit set 5159 */ 5160 r_idx = bt_getlowbit(vect->rx_map, 0, 5161 (ixgbe->num_rx_rings - 1)); 5162 5163 while (r_idx >= 0) { 5164 hw_index = ixgbe->rx_rings[r_idx].hw_index; 5165 ixgbe_setup_ivar(ixgbe, hw_index, v_idx, 0); 5166 r_idx = bt_getlowbit(vect->rx_map, (r_idx + 1), 5167 (ixgbe->num_rx_rings - 1)); 5168 } 5169 5170 /* 5171 * For each tx ring bit set 5172 */ 5173 r_idx = bt_getlowbit(vect->tx_map, 0, 5174 (ixgbe->num_tx_rings - 1)); 5175 5176 while (r_idx >= 0) { 5177 ixgbe_setup_ivar(ixgbe, r_idx, v_idx, 1); 5178 r_idx = bt_getlowbit(vect->tx_map, (r_idx + 1), 5179 (ixgbe->num_tx_rings - 1)); 5180 } 5181 } 5182 } 5183 5184 /* 5185 * ixgbe_rem_intr_handlers - Remove the interrupt handlers. 5186 */ 5187 static void 5188 ixgbe_rem_intr_handlers(ixgbe_t *ixgbe) 5189 { 5190 int i; 5191 int rc; 5192 5193 for (i = 0; i < ixgbe->intr_cnt; i++) { 5194 rc = ddi_intr_remove_handler(ixgbe->htable[i]); 5195 if (rc != DDI_SUCCESS) { 5196 IXGBE_DEBUGLOG_1(ixgbe, 5197 "Remove intr handler failed: %d", rc); 5198 } 5199 } 5200 } 5201 5202 /* 5203 * ixgbe_rem_intrs - Remove the allocated interrupts. 5204 */ 5205 static void 5206 ixgbe_rem_intrs(ixgbe_t *ixgbe) 5207 { 5208 int i; 5209 int rc; 5210 5211 for (i = 0; i < ixgbe->intr_cnt; i++) { 5212 rc = ddi_intr_free(ixgbe->htable[i]); 5213 if (rc != DDI_SUCCESS) { 5214 IXGBE_DEBUGLOG_1(ixgbe, 5215 "Free intr failed: %d", rc); 5216 } 5217 } 5218 5219 kmem_free(ixgbe->htable, ixgbe->intr_size); 5220 ixgbe->htable = NULL; 5221 } 5222 5223 /* 5224 * ixgbe_enable_intrs - Enable all the ddi interrupts. 5225 */ 5226 static int 5227 ixgbe_enable_intrs(ixgbe_t *ixgbe) 5228 { 5229 int i; 5230 int rc; 5231 5232 /* 5233 * Enable interrupts 5234 */ 5235 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5236 /* 5237 * Call ddi_intr_block_enable() for MSI 5238 */ 5239 rc = ddi_intr_block_enable(ixgbe->htable, ixgbe->intr_cnt); 5240 if (rc != DDI_SUCCESS) { 5241 ixgbe_log(ixgbe, 5242 "Enable block intr failed: %d", rc); 5243 return (IXGBE_FAILURE); 5244 } 5245 } else { 5246 /* 5247 * Call ddi_intr_enable() for Legacy/MSI non block enable 5248 */ 5249 for (i = 0; i < ixgbe->intr_cnt; i++) { 5250 rc = ddi_intr_enable(ixgbe->htable[i]); 5251 if (rc != DDI_SUCCESS) { 5252 ixgbe_log(ixgbe, 5253 "Enable intr failed: %d", rc); 5254 return (IXGBE_FAILURE); 5255 } 5256 } 5257 } 5258 5259 return (IXGBE_SUCCESS); 5260 } 5261 5262 /* 5263 * ixgbe_disable_intrs - Disable all the interrupts. 5264 */ 5265 static int 5266 ixgbe_disable_intrs(ixgbe_t *ixgbe) 5267 { 5268 int i; 5269 int rc; 5270 5271 /* 5272 * Disable all interrupts 5273 */ 5274 if (ixgbe->intr_cap & DDI_INTR_FLAG_BLOCK) { 5275 rc = ddi_intr_block_disable(ixgbe->htable, ixgbe->intr_cnt); 5276 if (rc != DDI_SUCCESS) { 5277 ixgbe_log(ixgbe, 5278 "Disable block intr failed: %d", rc); 5279 return (IXGBE_FAILURE); 5280 } 5281 } else { 5282 for (i = 0; i < ixgbe->intr_cnt; i++) { 5283 rc = ddi_intr_disable(ixgbe->htable[i]); 5284 if (rc != DDI_SUCCESS) { 5285 ixgbe_log(ixgbe, 5286 "Disable intr failed: %d", rc); 5287 return (IXGBE_FAILURE); 5288 } 5289 } 5290 } 5291 5292 return (IXGBE_SUCCESS); 5293 } 5294 5295 /* 5296 * ixgbe_get_hw_state - Get and save parameters related to adapter hardware. 5297 */ 5298 static void 5299 ixgbe_get_hw_state(ixgbe_t *ixgbe) 5300 { 5301 struct ixgbe_hw *hw = &ixgbe->hw; 5302 ixgbe_link_speed speed = IXGBE_LINK_SPEED_UNKNOWN; 5303 boolean_t link_up = B_FALSE; 5304 uint32_t pcs1g_anlp = 0; 5305 uint32_t pcs1g_ana = 0; 5306 boolean_t autoneg = B_FALSE; 5307 5308 ASSERT(mutex_owned(&ixgbe->gen_lock)); 5309 ixgbe->param_lp_1000fdx_cap = 0; 5310 ixgbe->param_lp_100fdx_cap = 0; 5311 5312 /* check for link, don't wait */ 5313 (void) ixgbe_check_link(hw, &speed, &link_up, false); 5314 pcs1g_ana = IXGBE_READ_REG(hw, IXGBE_PCS1GANA); 5315 5316 if (link_up) { 5317 pcs1g_anlp = IXGBE_READ_REG(hw, IXGBE_PCS1GANLP); 5318 5319 ixgbe->param_lp_1000fdx_cap = 5320 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5321 ixgbe->param_lp_100fdx_cap = 5322 (pcs1g_anlp & IXGBE_PCS1GANLP_LPFD) ? 1 : 0; 5323 } 5324 5325 (void) ixgbe_get_link_capabilities(hw, &speed, &autoneg); 5326 5327 ixgbe->param_adv_1000fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5328 (speed & IXGBE_LINK_SPEED_1GB_FULL)) ? 1 : 0; 5329 ixgbe->param_adv_100fdx_cap = ((pcs1g_ana & IXGBE_PCS1GANA_FDC) && 5330 (speed & IXGBE_LINK_SPEED_100_FULL)) ? 1 : 0; 5331 } 5332 5333 /* 5334 * ixgbe_get_driver_control - Notify that driver is in control of device. 5335 */ 5336 static void 5337 ixgbe_get_driver_control(struct ixgbe_hw *hw) 5338 { 5339 uint32_t ctrl_ext; 5340 5341 /* 5342 * Notify firmware that driver is in control of device 5343 */ 5344 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5345 ctrl_ext |= IXGBE_CTRL_EXT_DRV_LOAD; 5346 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5347 } 5348 5349 /* 5350 * ixgbe_release_driver_control - Notify that driver is no longer in control 5351 * of device. 5352 */ 5353 static void 5354 ixgbe_release_driver_control(struct ixgbe_hw *hw) 5355 { 5356 uint32_t ctrl_ext; 5357 5358 /* 5359 * Notify firmware that driver is no longer in control of device 5360 */ 5361 ctrl_ext = IXGBE_READ_REG(hw, IXGBE_CTRL_EXT); 5362 ctrl_ext &= ~IXGBE_CTRL_EXT_DRV_LOAD; 5363 IXGBE_WRITE_REG(hw, IXGBE_CTRL_EXT, ctrl_ext); 5364 } 5365 5366 /* 5367 * ixgbe_atomic_reserve - Atomic decrease operation. 5368 */ 5369 int 5370 ixgbe_atomic_reserve(uint32_t *count_p, uint32_t n) 5371 { 5372 uint32_t oldval; 5373 uint32_t newval; 5374 5375 /* 5376 * ATOMICALLY 5377 */ 5378 do { 5379 oldval = *count_p; 5380 if (oldval < n) 5381 return (-1); 5382 newval = oldval - n; 5383 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5384 5385 return (newval); 5386 } 5387 5388 /* 5389 * ixgbe_mc_table_itr - Traverse the entries in the multicast table. 5390 */ 5391 static uint8_t * 5392 ixgbe_mc_table_itr(struct ixgbe_hw *hw, uint8_t **upd_ptr, uint32_t *vmdq) 5393 { 5394 uint8_t *addr = *upd_ptr; 5395 uint8_t *new_ptr; 5396 5397 _NOTE(ARGUNUSED(hw)); 5398 _NOTE(ARGUNUSED(vmdq)); 5399 5400 new_ptr = addr + IXGBE_ETH_LENGTH_OF_ADDRESS; 5401 *upd_ptr = new_ptr; 5402 return (addr); 5403 } 5404 5405 /* 5406 * FMA support 5407 */ 5408 int 5409 ixgbe_check_acc_handle(ddi_acc_handle_t handle) 5410 { 5411 ddi_fm_error_t de; 5412 5413 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5414 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5415 return (de.fme_status); 5416 } 5417 5418 int 5419 ixgbe_check_dma_handle(ddi_dma_handle_t handle) 5420 { 5421 ddi_fm_error_t de; 5422 5423 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5424 return (de.fme_status); 5425 } 5426 5427 /* 5428 * ixgbe_fm_error_cb - The IO fault service error handling callback function. 5429 */ 5430 static int 5431 ixgbe_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5432 { 5433 _NOTE(ARGUNUSED(impl_data)); 5434 /* 5435 * as the driver can always deal with an error in any dma or 5436 * access handle, we can just return the fme_status value. 5437 */ 5438 pci_ereport_post(dip, err, NULL); 5439 return (err->fme_status); 5440 } 5441 5442 static void 5443 ixgbe_fm_init(ixgbe_t *ixgbe) 5444 { 5445 ddi_iblock_cookie_t iblk; 5446 int fma_dma_flag; 5447 5448 /* 5449 * Only register with IO Fault Services if we have some capability 5450 */ 5451 if (ixgbe->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5452 ixgbe_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5453 } else { 5454 ixgbe_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5455 } 5456 5457 if (ixgbe->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5458 fma_dma_flag = 1; 5459 } else { 5460 fma_dma_flag = 0; 5461 } 5462 5463 ixgbe_set_fma_flags(fma_dma_flag); 5464 5465 if (ixgbe->fm_capabilities) { 5466 5467 /* 5468 * Register capabilities with IO Fault Services 5469 */ 5470 ddi_fm_init(ixgbe->dip, &ixgbe->fm_capabilities, &iblk); 5471 5472 /* 5473 * Initialize pci ereport capabilities if ereport capable 5474 */ 5475 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5476 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5477 pci_ereport_setup(ixgbe->dip); 5478 5479 /* 5480 * Register error callback if error callback capable 5481 */ 5482 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5483 ddi_fm_handler_register(ixgbe->dip, 5484 ixgbe_fm_error_cb, (void*) ixgbe); 5485 } 5486 } 5487 5488 static void 5489 ixgbe_fm_fini(ixgbe_t *ixgbe) 5490 { 5491 /* 5492 * Only unregister FMA capabilities if they are registered 5493 */ 5494 if (ixgbe->fm_capabilities) { 5495 5496 /* 5497 * Release any resources allocated by pci_ereport_setup() 5498 */ 5499 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities) || 5500 DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5501 pci_ereport_teardown(ixgbe->dip); 5502 5503 /* 5504 * Un-register error callback if error callback capable 5505 */ 5506 if (DDI_FM_ERRCB_CAP(ixgbe->fm_capabilities)) 5507 ddi_fm_handler_unregister(ixgbe->dip); 5508 5509 /* 5510 * Unregister from IO Fault Service 5511 */ 5512 ddi_fm_fini(ixgbe->dip); 5513 } 5514 } 5515 5516 void 5517 ixgbe_fm_ereport(ixgbe_t *ixgbe, char *detail) 5518 { 5519 uint64_t ena; 5520 char buf[FM_MAX_CLASS]; 5521 5522 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5523 ena = fm_ena_generate(0, FM_ENA_FMT1); 5524 if (DDI_FM_EREPORT_CAP(ixgbe->fm_capabilities)) { 5525 ddi_fm_ereport_post(ixgbe->dip, buf, ena, DDI_NOSLEEP, 5526 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5527 } 5528 } 5529 5530 static int 5531 ixgbe_ring_start(mac_ring_driver_t rh, uint64_t mr_gen_num) 5532 { 5533 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)rh; 5534 5535 mutex_enter(&rx_ring->rx_lock); 5536 rx_ring->ring_gen_num = mr_gen_num; 5537 mutex_exit(&rx_ring->rx_lock); 5538 return (0); 5539 } 5540 5541 /* 5542 * Get the global ring index by a ring index within a group. 5543 */ 5544 static int 5545 ixgbe_get_rx_ring_index(ixgbe_t *ixgbe, int gindex, int rindex) 5546 { 5547 ixgbe_rx_ring_t *rx_ring; 5548 int i; 5549 5550 for (i = 0; i < ixgbe->num_rx_rings; i++) { 5551 rx_ring = &ixgbe->rx_rings[i]; 5552 if (rx_ring->group_index == gindex) 5553 rindex--; 5554 if (rindex < 0) 5555 return (i); 5556 } 5557 5558 return (-1); 5559 } 5560 5561 /* 5562 * Callback funtion for MAC layer to register all rings. 5563 */ 5564 /* ARGSUSED */ 5565 void 5566 ixgbe_fill_ring(void *arg, mac_ring_type_t rtype, const int group_index, 5567 const int ring_index, mac_ring_info_t *infop, mac_ring_handle_t rh) 5568 { 5569 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5570 mac_intr_t *mintr = &infop->mri_intr; 5571 5572 switch (rtype) { 5573 case MAC_RING_TYPE_RX: { 5574 /* 5575 * 'index' is the ring index within the group. 5576 * Need to get the global ring index by searching in groups. 5577 */ 5578 int global_ring_index = ixgbe_get_rx_ring_index( 5579 ixgbe, group_index, ring_index); 5580 5581 ASSERT(global_ring_index >= 0); 5582 5583 ixgbe_rx_ring_t *rx_ring = &ixgbe->rx_rings[global_ring_index]; 5584 rx_ring->ring_handle = rh; 5585 5586 infop->mri_driver = (mac_ring_driver_t)rx_ring; 5587 infop->mri_start = ixgbe_ring_start; 5588 infop->mri_stop = NULL; 5589 infop->mri_poll = ixgbe_ring_rx_poll; 5590 infop->mri_stat = ixgbe_rx_ring_stat; 5591 5592 mintr->mi_handle = (mac_intr_handle_t)rx_ring; 5593 mintr->mi_enable = ixgbe_rx_ring_intr_enable; 5594 mintr->mi_disable = ixgbe_rx_ring_intr_disable; 5595 if (ixgbe->intr_type & 5596 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5597 mintr->mi_ddi_handle = 5598 ixgbe->htable[rx_ring->intr_vector]; 5599 } 5600 5601 break; 5602 } 5603 case MAC_RING_TYPE_TX: { 5604 ASSERT(group_index == -1); 5605 ASSERT(ring_index < ixgbe->num_tx_rings); 5606 5607 ixgbe_tx_ring_t *tx_ring = &ixgbe->tx_rings[ring_index]; 5608 tx_ring->ring_handle = rh; 5609 5610 infop->mri_driver = (mac_ring_driver_t)tx_ring; 5611 infop->mri_start = NULL; 5612 infop->mri_stop = NULL; 5613 infop->mri_tx = ixgbe_ring_tx; 5614 infop->mri_stat = ixgbe_tx_ring_stat; 5615 if (ixgbe->intr_type & 5616 (DDI_INTR_TYPE_MSIX | DDI_INTR_TYPE_MSI)) { 5617 mintr->mi_ddi_handle = 5618 ixgbe->htable[tx_ring->intr_vector]; 5619 } 5620 break; 5621 } 5622 default: 5623 break; 5624 } 5625 } 5626 5627 /* 5628 * Callback funtion for MAC layer to register all groups. 5629 */ 5630 void 5631 ixgbe_fill_group(void *arg, mac_ring_type_t rtype, const int index, 5632 mac_group_info_t *infop, mac_group_handle_t gh) 5633 { 5634 ixgbe_t *ixgbe = (ixgbe_t *)arg; 5635 5636 switch (rtype) { 5637 case MAC_RING_TYPE_RX: { 5638 ixgbe_rx_group_t *rx_group; 5639 5640 rx_group = &ixgbe->rx_groups[index]; 5641 rx_group->group_handle = gh; 5642 5643 infop->mgi_driver = (mac_group_driver_t)rx_group; 5644 infop->mgi_start = NULL; 5645 infop->mgi_stop = NULL; 5646 infop->mgi_addmac = ixgbe_addmac; 5647 infop->mgi_remmac = ixgbe_remmac; 5648 infop->mgi_count = (ixgbe->num_rx_rings / ixgbe->num_rx_groups); 5649 5650 break; 5651 } 5652 case MAC_RING_TYPE_TX: 5653 break; 5654 default: 5655 break; 5656 } 5657 } 5658 5659 /* 5660 * Enable interrupt on the specificed rx ring. 5661 */ 5662 int 5663 ixgbe_rx_ring_intr_enable(mac_intr_handle_t intrh) 5664 { 5665 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5666 ixgbe_t *ixgbe = rx_ring->ixgbe; 5667 int r_idx = rx_ring->index; 5668 int hw_r_idx = rx_ring->hw_index; 5669 int v_idx = rx_ring->intr_vector; 5670 5671 mutex_enter(&ixgbe->gen_lock); 5672 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5673 mutex_exit(&ixgbe->gen_lock); 5674 /* 5675 * Simply return 0. 5676 * Interrupts are being adjusted. ixgbe_intr_adjust() 5677 * will eventually re-enable the interrupt when it's 5678 * done with the adjustment. 5679 */ 5680 return (0); 5681 } 5682 5683 /* 5684 * To enable interrupt by setting the VAL bit of given interrupt 5685 * vector allocation register (IVAR). 5686 */ 5687 ixgbe_enable_ivar(ixgbe, hw_r_idx, 0); 5688 5689 BT_SET(ixgbe->vect_map[v_idx].rx_map, r_idx); 5690 5691 /* 5692 * Trigger a Rx interrupt on this ring 5693 */ 5694 IXGBE_WRITE_REG(&ixgbe->hw, IXGBE_EICS, (1 << v_idx)); 5695 IXGBE_WRITE_FLUSH(&ixgbe->hw); 5696 5697 mutex_exit(&ixgbe->gen_lock); 5698 5699 return (0); 5700 } 5701 5702 /* 5703 * Disable interrupt on the specificed rx ring. 5704 */ 5705 int 5706 ixgbe_rx_ring_intr_disable(mac_intr_handle_t intrh) 5707 { 5708 ixgbe_rx_ring_t *rx_ring = (ixgbe_rx_ring_t *)intrh; 5709 ixgbe_t *ixgbe = rx_ring->ixgbe; 5710 int r_idx = rx_ring->index; 5711 int hw_r_idx = rx_ring->hw_index; 5712 int v_idx = rx_ring->intr_vector; 5713 5714 mutex_enter(&ixgbe->gen_lock); 5715 if (ixgbe->ixgbe_state & IXGBE_INTR_ADJUST) { 5716 mutex_exit(&ixgbe->gen_lock); 5717 /* 5718 * Simply return 0. 5719 * In the rare case where an interrupt is being 5720 * disabled while interrupts are being adjusted, 5721 * we don't fail the operation. No interrupts will 5722 * be generated while they are adjusted, and 5723 * ixgbe_intr_adjust() will cause the interrupts 5724 * to be re-enabled once it completes. Note that 5725 * in this case, packets may be delivered to the 5726 * stack via interrupts before xgbe_rx_ring_intr_enable() 5727 * is called again. This is acceptable since interrupt 5728 * adjustment is infrequent, and the stack will be 5729 * able to handle these packets. 5730 */ 5731 return (0); 5732 } 5733 5734 /* 5735 * To disable interrupt by clearing the VAL bit of given interrupt 5736 * vector allocation register (IVAR). 5737 */ 5738 ixgbe_disable_ivar(ixgbe, hw_r_idx, 0); 5739 5740 BT_CLEAR(ixgbe->vect_map[v_idx].rx_map, r_idx); 5741 5742 mutex_exit(&ixgbe->gen_lock); 5743 5744 return (0); 5745 } 5746 5747 /* 5748 * Add a mac address. 5749 */ 5750 static int 5751 ixgbe_addmac(void *arg, const uint8_t *mac_addr) 5752 { 5753 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5754 ixgbe_t *ixgbe = rx_group->ixgbe; 5755 struct ixgbe_hw *hw = &ixgbe->hw; 5756 int slot, i; 5757 5758 mutex_enter(&ixgbe->gen_lock); 5759 5760 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5761 mutex_exit(&ixgbe->gen_lock); 5762 return (ECANCELED); 5763 } 5764 5765 if (ixgbe->unicst_avail == 0) { 5766 /* no slots available */ 5767 mutex_exit(&ixgbe->gen_lock); 5768 return (ENOSPC); 5769 } 5770 5771 /* 5772 * The first ixgbe->num_rx_groups slots are reserved for each respective 5773 * group. The rest slots are shared by all groups. While adding a 5774 * MAC address, reserved slots are firstly checked then the shared 5775 * slots are searched. 5776 */ 5777 slot = -1; 5778 if (ixgbe->unicst_addr[rx_group->index].mac.set == 1) { 5779 for (i = ixgbe->num_rx_groups; i < ixgbe->unicst_total; i++) { 5780 if (ixgbe->unicst_addr[i].mac.set == 0) { 5781 slot = i; 5782 break; 5783 } 5784 } 5785 } else { 5786 slot = rx_group->index; 5787 } 5788 5789 if (slot == -1) { 5790 /* no slots available */ 5791 mutex_exit(&ixgbe->gen_lock); 5792 return (ENOSPC); 5793 } 5794 5795 bcopy(mac_addr, ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5796 (void) ixgbe_set_rar(hw, slot, ixgbe->unicst_addr[slot].mac.addr, 5797 rx_group->index, IXGBE_RAH_AV); 5798 ixgbe->unicst_addr[slot].mac.set = 1; 5799 ixgbe->unicst_addr[slot].mac.group_index = rx_group->index; 5800 ixgbe->unicst_avail--; 5801 5802 mutex_exit(&ixgbe->gen_lock); 5803 5804 return (0); 5805 } 5806 5807 /* 5808 * Remove a mac address. 5809 */ 5810 static int 5811 ixgbe_remmac(void *arg, const uint8_t *mac_addr) 5812 { 5813 ixgbe_rx_group_t *rx_group = (ixgbe_rx_group_t *)arg; 5814 ixgbe_t *ixgbe = rx_group->ixgbe; 5815 struct ixgbe_hw *hw = &ixgbe->hw; 5816 int slot; 5817 5818 mutex_enter(&ixgbe->gen_lock); 5819 5820 if (ixgbe->ixgbe_state & IXGBE_SUSPENDED) { 5821 mutex_exit(&ixgbe->gen_lock); 5822 return (ECANCELED); 5823 } 5824 5825 slot = ixgbe_unicst_find(ixgbe, mac_addr); 5826 if (slot == -1) { 5827 mutex_exit(&ixgbe->gen_lock); 5828 return (EINVAL); 5829 } 5830 5831 if (ixgbe->unicst_addr[slot].mac.set == 0) { 5832 mutex_exit(&ixgbe->gen_lock); 5833 return (EINVAL); 5834 } 5835 5836 bzero(ixgbe->unicst_addr[slot].mac.addr, ETHERADDRL); 5837 (void) ixgbe_clear_rar(hw, slot); 5838 ixgbe->unicst_addr[slot].mac.set = 0; 5839 ixgbe->unicst_avail++; 5840 5841 mutex_exit(&ixgbe->gen_lock); 5842 5843 return (0); 5844 } 5845