1 /* 2 * CDDL HEADER START 3 * 4 * Copyright(c) 2007-2009 Intel Corporation. All rights reserved. 5 * The contents of this file are subject to the terms of the 6 * Common Development and Distribution License (the "License"). 7 * You may not use this file except in compliance with the License. 8 * 9 * You can obtain a copy of the license at: 10 * http://www.opensolaris.org/os/licensing. 11 * See the License for the specific language governing permissions 12 * and limitations under the License. 13 * 14 * When using or redistributing this file, you may do so under the 15 * License only. No other modification of this header is permitted. 16 * 17 * If applicable, add the following below this CDDL HEADER, with the 18 * fields enclosed by brackets "[]" replaced with your own identifying 19 * information: Portions Copyright [yyyy] [name of copyright owner] 20 * 21 * CDDL HEADER END 22 */ 23 24 /* 25 * Copyright 2010 Sun Microsystems, Inc. All rights reserved. 26 * Use is subject to license terms. 27 */ 28 29 #include "igb_sw.h" 30 31 static char ident[] = "Intel 1Gb Ethernet"; 32 static char igb_version[] = "igb 1.1.11"; 33 34 /* 35 * Local function protoypes 36 */ 37 static int igb_register_mac(igb_t *); 38 static int igb_identify_hardware(igb_t *); 39 static int igb_regs_map(igb_t *); 40 static void igb_init_properties(igb_t *); 41 static int igb_init_driver_settings(igb_t *); 42 static void igb_init_locks(igb_t *); 43 static void igb_destroy_locks(igb_t *); 44 static int igb_init_mac_address(igb_t *); 45 static int igb_init(igb_t *); 46 static int igb_init_adapter(igb_t *); 47 static void igb_stop_adapter(igb_t *); 48 static int igb_reset(igb_t *); 49 static void igb_tx_clean(igb_t *); 50 static boolean_t igb_tx_drain(igb_t *); 51 static boolean_t igb_rx_drain(igb_t *); 52 static int igb_alloc_rings(igb_t *); 53 static int igb_alloc_rx_data(igb_t *); 54 static void igb_free_rx_data(igb_t *); 55 static void igb_free_rings(igb_t *); 56 static void igb_setup_rings(igb_t *); 57 static void igb_setup_rx(igb_t *); 58 static void igb_setup_tx(igb_t *); 59 static void igb_setup_rx_ring(igb_rx_ring_t *); 60 static void igb_setup_tx_ring(igb_tx_ring_t *); 61 static void igb_setup_rss(igb_t *); 62 static void igb_setup_mac_rss_classify(igb_t *); 63 static void igb_setup_mac_classify(igb_t *); 64 static void igb_init_unicst(igb_t *); 65 static void igb_setup_multicst(igb_t *); 66 static void igb_get_phy_state(igb_t *); 67 static void igb_param_sync(igb_t *); 68 static void igb_get_conf(igb_t *); 69 static int igb_get_prop(igb_t *, char *, int, int, int); 70 static boolean_t igb_is_link_up(igb_t *); 71 static boolean_t igb_link_check(igb_t *); 72 static void igb_local_timer(void *); 73 static void igb_link_timer(void *); 74 static void igb_arm_watchdog_timer(igb_t *); 75 static void igb_start_watchdog_timer(igb_t *); 76 static void igb_restart_watchdog_timer(igb_t *); 77 static void igb_stop_watchdog_timer(igb_t *); 78 static void igb_start_link_timer(igb_t *); 79 static void igb_stop_link_timer(igb_t *); 80 static void igb_disable_adapter_interrupts(igb_t *); 81 static void igb_enable_adapter_interrupts_82575(igb_t *); 82 static void igb_enable_adapter_interrupts_82576(igb_t *); 83 static void igb_enable_adapter_interrupts_82580(igb_t *); 84 static boolean_t is_valid_mac_addr(uint8_t *); 85 static boolean_t igb_stall_check(igb_t *); 86 static boolean_t igb_set_loopback_mode(igb_t *, uint32_t); 87 static void igb_set_external_loopback(igb_t *); 88 static void igb_set_internal_mac_loopback(igb_t *); 89 static void igb_set_internal_phy_loopback(igb_t *); 90 static void igb_set_internal_serdes_loopback(igb_t *); 91 static boolean_t igb_find_mac_address(igb_t *); 92 static int igb_alloc_intrs(igb_t *); 93 static int igb_alloc_intr_handles(igb_t *, int); 94 static int igb_add_intr_handlers(igb_t *); 95 static void igb_rem_intr_handlers(igb_t *); 96 static void igb_rem_intrs(igb_t *); 97 static int igb_enable_intrs(igb_t *); 98 static int igb_disable_intrs(igb_t *); 99 static void igb_setup_msix_82575(igb_t *); 100 static void igb_setup_msix_82576(igb_t *); 101 static void igb_setup_msix_82580(igb_t *); 102 static uint_t igb_intr_legacy(void *, void *); 103 static uint_t igb_intr_msi(void *, void *); 104 static uint_t igb_intr_rx(void *, void *); 105 static uint_t igb_intr_tx(void *, void *); 106 static uint_t igb_intr_tx_other(void *, void *); 107 static void igb_intr_rx_work(igb_rx_ring_t *); 108 static void igb_intr_tx_work(igb_tx_ring_t *); 109 static void igb_intr_link_work(igb_t *); 110 static void igb_get_driver_control(struct e1000_hw *); 111 static void igb_release_driver_control(struct e1000_hw *); 112 113 static int igb_attach(dev_info_t *, ddi_attach_cmd_t); 114 static int igb_detach(dev_info_t *, ddi_detach_cmd_t); 115 static int igb_resume(dev_info_t *); 116 static int igb_suspend(dev_info_t *); 117 static int igb_quiesce(dev_info_t *); 118 static void igb_unconfigure(dev_info_t *, igb_t *); 119 static int igb_fm_error_cb(dev_info_t *, ddi_fm_error_t *, 120 const void *); 121 static void igb_fm_init(igb_t *); 122 static void igb_fm_fini(igb_t *); 123 static void igb_release_multicast(igb_t *); 124 125 mac_priv_prop_t igb_priv_props[] = { 126 {"_tx_copy_thresh", MAC_PROP_PERM_RW}, 127 {"_tx_recycle_thresh", MAC_PROP_PERM_RW}, 128 {"_tx_overload_thresh", MAC_PROP_PERM_RW}, 129 {"_tx_resched_thresh", MAC_PROP_PERM_RW}, 130 {"_rx_copy_thresh", MAC_PROP_PERM_RW}, 131 {"_rx_limit_per_intr", MAC_PROP_PERM_RW}, 132 {"_intr_throttling", MAC_PROP_PERM_RW}, 133 {"_adv_pause_cap", MAC_PROP_PERM_READ}, 134 {"_adv_asym_pause_cap", MAC_PROP_PERM_READ} 135 }; 136 137 #define IGB_MAX_PRIV_PROPS \ 138 (sizeof (igb_priv_props) / sizeof (mac_priv_prop_t)) 139 140 static struct cb_ops igb_cb_ops = { 141 nulldev, /* cb_open */ 142 nulldev, /* cb_close */ 143 nodev, /* cb_strategy */ 144 nodev, /* cb_print */ 145 nodev, /* cb_dump */ 146 nodev, /* cb_read */ 147 nodev, /* cb_write */ 148 nodev, /* cb_ioctl */ 149 nodev, /* cb_devmap */ 150 nodev, /* cb_mmap */ 151 nodev, /* cb_segmap */ 152 nochpoll, /* cb_chpoll */ 153 ddi_prop_op, /* cb_prop_op */ 154 NULL, /* cb_stream */ 155 D_MP | D_HOTPLUG, /* cb_flag */ 156 CB_REV, /* cb_rev */ 157 nodev, /* cb_aread */ 158 nodev /* cb_awrite */ 159 }; 160 161 static struct dev_ops igb_dev_ops = { 162 DEVO_REV, /* devo_rev */ 163 0, /* devo_refcnt */ 164 NULL, /* devo_getinfo */ 165 nulldev, /* devo_identify */ 166 nulldev, /* devo_probe */ 167 igb_attach, /* devo_attach */ 168 igb_detach, /* devo_detach */ 169 nodev, /* devo_reset */ 170 &igb_cb_ops, /* devo_cb_ops */ 171 NULL, /* devo_bus_ops */ 172 ddi_power, /* devo_power */ 173 igb_quiesce, /* devo_quiesce */ 174 }; 175 176 static struct modldrv igb_modldrv = { 177 &mod_driverops, /* Type of module. This one is a driver */ 178 ident, /* Discription string */ 179 &igb_dev_ops, /* driver ops */ 180 }; 181 182 static struct modlinkage igb_modlinkage = { 183 MODREV_1, &igb_modldrv, NULL 184 }; 185 186 /* Access attributes for register mapping */ 187 ddi_device_acc_attr_t igb_regs_acc_attr = { 188 DDI_DEVICE_ATTR_V1, 189 DDI_STRUCTURE_LE_ACC, 190 DDI_STRICTORDER_ACC, 191 DDI_FLAGERR_ACC 192 }; 193 194 #define IGB_M_CALLBACK_FLAGS \ 195 (MC_IOCTL | MC_GETCAPAB | MC_SETPROP | MC_GETPROP) 196 197 static mac_callbacks_t igb_m_callbacks = { 198 IGB_M_CALLBACK_FLAGS, 199 igb_m_stat, 200 igb_m_start, 201 igb_m_stop, 202 igb_m_promisc, 203 igb_m_multicst, 204 NULL, 205 NULL, 206 igb_m_ioctl, 207 igb_m_getcapab, 208 NULL, 209 NULL, 210 igb_m_setprop, 211 igb_m_getprop 212 }; 213 214 /* 215 * Initialize capabilities of each supported adapter type 216 */ 217 static adapter_info_t igb_82575_cap = { 218 /* limits */ 219 4, /* maximum number of rx queues */ 220 1, /* minimum number of rx queues */ 221 4, /* default number of rx queues */ 222 4, /* maximum number of tx queues */ 223 1, /* minimum number of tx queues */ 224 4, /* default number of tx queues */ 225 65535, /* maximum interrupt throttle rate */ 226 0, /* minimum interrupt throttle rate */ 227 200, /* default interrupt throttle rate */ 228 229 /* function pointers */ 230 igb_enable_adapter_interrupts_82575, 231 igb_setup_msix_82575, 232 233 /* capabilities */ 234 (IGB_FLAG_HAS_DCA | /* capability flags */ 235 IGB_FLAG_VMDQ_POOL), 236 237 0xffc00000 /* mask for RXDCTL register */ 238 }; 239 240 static adapter_info_t igb_82576_cap = { 241 /* limits */ 242 16, /* maximum number of rx queues */ 243 1, /* minimum number of rx queues */ 244 4, /* default number of rx queues */ 245 16, /* maximum number of tx queues */ 246 1, /* minimum number of tx queues */ 247 4, /* default number of tx queues */ 248 65535, /* maximum interrupt throttle rate */ 249 0, /* minimum interrupt throttle rate */ 250 200, /* default interrupt throttle rate */ 251 252 /* function pointers */ 253 igb_enable_adapter_interrupts_82576, 254 igb_setup_msix_82576, 255 256 /* capabilities */ 257 (IGB_FLAG_HAS_DCA | /* capability flags */ 258 IGB_FLAG_VMDQ_POOL | 259 IGB_FLAG_NEED_CTX_IDX), 260 261 0xffe00000 /* mask for RXDCTL register */ 262 }; 263 264 static adapter_info_t igb_82580_cap = { 265 /* limits */ 266 8, /* maximum number of rx queues */ 267 1, /* minimum number of rx queues */ 268 4, /* default number of rx queues */ 269 8, /* maximum number of tx queues */ 270 1, /* minimum number of tx queues */ 271 4, /* default number of tx queues */ 272 65535, /* maximum interrupt throttle rate */ 273 0, /* minimum interrupt throttle rate */ 274 200, /* default interrupt throttle rate */ 275 276 /* function pointers */ 277 igb_enable_adapter_interrupts_82580, 278 igb_setup_msix_82580, 279 280 /* capabilities */ 281 (IGB_FLAG_HAS_DCA | /* capability flags */ 282 IGB_FLAG_VMDQ_POOL | 283 IGB_FLAG_NEED_CTX_IDX), 284 285 0xffe00000 /* mask for RXDCTL register */ 286 }; 287 288 /* 289 * Module Initialization Functions 290 */ 291 292 int 293 _init(void) 294 { 295 int status; 296 297 mac_init_ops(&igb_dev_ops, MODULE_NAME); 298 299 status = mod_install(&igb_modlinkage); 300 301 if (status != DDI_SUCCESS) { 302 mac_fini_ops(&igb_dev_ops); 303 } 304 305 return (status); 306 } 307 308 int 309 _fini(void) 310 { 311 int status; 312 313 status = mod_remove(&igb_modlinkage); 314 315 if (status == DDI_SUCCESS) { 316 mac_fini_ops(&igb_dev_ops); 317 } 318 319 return (status); 320 321 } 322 323 int 324 _info(struct modinfo *modinfop) 325 { 326 int status; 327 328 status = mod_info(&igb_modlinkage, modinfop); 329 330 return (status); 331 } 332 333 /* 334 * igb_attach - driver attach 335 * 336 * This function is the device specific initialization entry 337 * point. This entry point is required and must be written. 338 * The DDI_ATTACH command must be provided in the attach entry 339 * point. When attach() is called with cmd set to DDI_ATTACH, 340 * all normal kernel services (such as kmem_alloc(9F)) are 341 * available for use by the driver. 342 * 343 * The attach() function will be called once for each instance 344 * of the device on the system with cmd set to DDI_ATTACH. 345 * Until attach() succeeds, the only driver entry points which 346 * may be called are open(9E) and getinfo(9E). 347 */ 348 static int 349 igb_attach(dev_info_t *devinfo, ddi_attach_cmd_t cmd) 350 { 351 igb_t *igb; 352 struct igb_osdep *osdep; 353 struct e1000_hw *hw; 354 int instance; 355 356 /* 357 * Check the command and perform corresponding operations 358 */ 359 switch (cmd) { 360 default: 361 return (DDI_FAILURE); 362 363 case DDI_RESUME: 364 return (igb_resume(devinfo)); 365 366 case DDI_ATTACH: 367 break; 368 } 369 370 /* Get the device instance */ 371 instance = ddi_get_instance(devinfo); 372 373 /* Allocate memory for the instance data structure */ 374 igb = kmem_zalloc(sizeof (igb_t), KM_SLEEP); 375 376 igb->dip = devinfo; 377 igb->instance = instance; 378 379 hw = &igb->hw; 380 osdep = &igb->osdep; 381 hw->back = osdep; 382 osdep->igb = igb; 383 384 /* Attach the instance pointer to the dev_info data structure */ 385 ddi_set_driver_private(devinfo, igb); 386 387 388 /* Initialize for fma support */ 389 igb->fm_capabilities = igb_get_prop(igb, "fm-capable", 390 0, 0x0f, 391 DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE | 392 DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE); 393 igb_fm_init(igb); 394 igb->attach_progress |= ATTACH_PROGRESS_FMINIT; 395 396 /* 397 * Map PCI config space registers 398 */ 399 if (pci_config_setup(devinfo, &osdep->cfg_handle) != DDI_SUCCESS) { 400 igb_error(igb, "Failed to map PCI configurations"); 401 goto attach_fail; 402 } 403 igb->attach_progress |= ATTACH_PROGRESS_PCI_CONFIG; 404 405 /* 406 * Identify the chipset family 407 */ 408 if (igb_identify_hardware(igb) != IGB_SUCCESS) { 409 igb_error(igb, "Failed to identify hardware"); 410 goto attach_fail; 411 } 412 413 /* 414 * Map device registers 415 */ 416 if (igb_regs_map(igb) != IGB_SUCCESS) { 417 igb_error(igb, "Failed to map device registers"); 418 goto attach_fail; 419 } 420 igb->attach_progress |= ATTACH_PROGRESS_REGS_MAP; 421 422 /* 423 * Initialize driver parameters 424 */ 425 igb_init_properties(igb); 426 igb->attach_progress |= ATTACH_PROGRESS_PROPS; 427 428 /* 429 * Allocate interrupts 430 */ 431 if (igb_alloc_intrs(igb) != IGB_SUCCESS) { 432 igb_error(igb, "Failed to allocate interrupts"); 433 goto attach_fail; 434 } 435 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_INTR; 436 437 /* 438 * Allocate rx/tx rings based on the ring numbers. 439 * The actual numbers of rx/tx rings are decided by the number of 440 * allocated interrupt vectors, so we should allocate the rings after 441 * interrupts are allocated. 442 */ 443 if (igb_alloc_rings(igb) != IGB_SUCCESS) { 444 igb_error(igb, "Failed to allocate rx/tx rings or groups"); 445 goto attach_fail; 446 } 447 igb->attach_progress |= ATTACH_PROGRESS_ALLOC_RINGS; 448 449 /* 450 * Add interrupt handlers 451 */ 452 if (igb_add_intr_handlers(igb) != IGB_SUCCESS) { 453 igb_error(igb, "Failed to add interrupt handlers"); 454 goto attach_fail; 455 } 456 igb->attach_progress |= ATTACH_PROGRESS_ADD_INTR; 457 458 /* 459 * Initialize driver parameters 460 */ 461 if (igb_init_driver_settings(igb) != IGB_SUCCESS) { 462 igb_error(igb, "Failed to initialize driver settings"); 463 goto attach_fail; 464 } 465 466 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) { 467 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 468 goto attach_fail; 469 } 470 471 /* 472 * Initialize mutexes for this device. 473 * Do this before enabling the interrupt handler and 474 * register the softint to avoid the condition where 475 * interrupt handler can try using uninitialized mutex 476 */ 477 igb_init_locks(igb); 478 igb->attach_progress |= ATTACH_PROGRESS_LOCKS; 479 480 /* 481 * Initialize the adapter 482 */ 483 if (igb_init(igb) != IGB_SUCCESS) { 484 igb_error(igb, "Failed to initialize adapter"); 485 goto attach_fail; 486 } 487 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 488 489 /* 490 * Initialize statistics 491 */ 492 if (igb_init_stats(igb) != IGB_SUCCESS) { 493 igb_error(igb, "Failed to initialize statistics"); 494 goto attach_fail; 495 } 496 igb->attach_progress |= ATTACH_PROGRESS_STATS; 497 498 /* 499 * Register the driver to the MAC 500 */ 501 if (igb_register_mac(igb) != IGB_SUCCESS) { 502 igb_error(igb, "Failed to register MAC"); 503 goto attach_fail; 504 } 505 igb->attach_progress |= ATTACH_PROGRESS_MAC; 506 507 /* 508 * Now that mutex locks are initialized, and the chip is also 509 * initialized, enable interrupts. 510 */ 511 if (igb_enable_intrs(igb) != IGB_SUCCESS) { 512 igb_error(igb, "Failed to enable DDI interrupts"); 513 goto attach_fail; 514 } 515 igb->attach_progress |= ATTACH_PROGRESS_ENABLE_INTR; 516 517 igb_log(igb, "%s", igb_version); 518 atomic_or_32(&igb->igb_state, IGB_INITIALIZED); 519 520 return (DDI_SUCCESS); 521 522 attach_fail: 523 igb_unconfigure(devinfo, igb); 524 return (DDI_FAILURE); 525 } 526 527 /* 528 * igb_detach - driver detach 529 * 530 * The detach() function is the complement of the attach routine. 531 * If cmd is set to DDI_DETACH, detach() is used to remove the 532 * state associated with a given instance of a device node 533 * prior to the removal of that instance from the system. 534 * 535 * The detach() function will be called once for each instance 536 * of the device for which there has been a successful attach() 537 * once there are no longer any opens on the device. 538 * 539 * Interrupts routine are disabled, All memory allocated by this 540 * driver are freed. 541 */ 542 static int 543 igb_detach(dev_info_t *devinfo, ddi_detach_cmd_t cmd) 544 { 545 igb_t *igb; 546 547 /* 548 * Check detach command 549 */ 550 switch (cmd) { 551 default: 552 return (DDI_FAILURE); 553 554 case DDI_SUSPEND: 555 return (igb_suspend(devinfo)); 556 557 case DDI_DETACH: 558 break; 559 } 560 561 562 /* 563 * Get the pointer to the driver private data structure 564 */ 565 igb = (igb_t *)ddi_get_driver_private(devinfo); 566 if (igb == NULL) 567 return (DDI_FAILURE); 568 569 /* 570 * Unregister MAC. If failed, we have to fail the detach 571 */ 572 if (mac_unregister(igb->mac_hdl) != 0) { 573 igb_error(igb, "Failed to unregister MAC"); 574 return (DDI_FAILURE); 575 } 576 igb->attach_progress &= ~ATTACH_PROGRESS_MAC; 577 578 /* 579 * If the device is still running, it needs to be stopped first. 580 * This check is necessary because under some specific circumstances, 581 * the detach routine can be called without stopping the interface 582 * first. 583 */ 584 mutex_enter(&igb->gen_lock); 585 if (igb->igb_state & IGB_STARTED) { 586 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 587 igb_stop(igb, B_TRUE); 588 mutex_exit(&igb->gen_lock); 589 /* Disable and stop the watchdog timer */ 590 igb_disable_watchdog_timer(igb); 591 } else 592 mutex_exit(&igb->gen_lock); 593 594 /* 595 * Check if there are still rx buffers held by the upper layer. 596 * If so, fail the detach. 597 */ 598 if (!igb_rx_drain(igb)) 599 return (DDI_FAILURE); 600 601 /* 602 * Do the remaining unconfigure routines 603 */ 604 igb_unconfigure(devinfo, igb); 605 606 return (DDI_SUCCESS); 607 } 608 609 /* 610 * quiesce(9E) entry point. 611 * 612 * This function is called when the system is single-threaded at high 613 * PIL with preemption disabled. Therefore, this function must not be 614 * blocked. 615 * 616 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure. 617 * DDI_FAILURE indicates an error condition and should almost never happen. 618 */ 619 static int 620 igb_quiesce(dev_info_t *devinfo) 621 { 622 igb_t *igb; 623 struct e1000_hw *hw; 624 625 igb = (igb_t *)ddi_get_driver_private(devinfo); 626 627 if (igb == NULL) 628 return (DDI_FAILURE); 629 630 hw = &igb->hw; 631 632 /* 633 * Disable the adapter interrupts 634 */ 635 igb_disable_adapter_interrupts(igb); 636 637 /* Tell firmware driver is no longer in control */ 638 igb_release_driver_control(hw); 639 640 /* 641 * Reset the chipset 642 */ 643 (void) e1000_reset_hw(hw); 644 645 /* 646 * Reset PHY if possible 647 */ 648 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 649 (void) e1000_phy_hw_reset(hw); 650 651 return (DDI_SUCCESS); 652 } 653 654 /* 655 * igb_unconfigure - release all resources held by this instance 656 */ 657 static void 658 igb_unconfigure(dev_info_t *devinfo, igb_t *igb) 659 { 660 /* 661 * Disable interrupt 662 */ 663 if (igb->attach_progress & ATTACH_PROGRESS_ENABLE_INTR) { 664 (void) igb_disable_intrs(igb); 665 } 666 667 /* 668 * Unregister MAC 669 */ 670 if (igb->attach_progress & ATTACH_PROGRESS_MAC) { 671 (void) mac_unregister(igb->mac_hdl); 672 } 673 674 /* 675 * Free statistics 676 */ 677 if (igb->attach_progress & ATTACH_PROGRESS_STATS) { 678 kstat_delete((kstat_t *)igb->igb_ks); 679 } 680 681 /* 682 * Remove interrupt handlers 683 */ 684 if (igb->attach_progress & ATTACH_PROGRESS_ADD_INTR) { 685 igb_rem_intr_handlers(igb); 686 } 687 688 /* 689 * Remove interrupts 690 */ 691 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_INTR) { 692 igb_rem_intrs(igb); 693 } 694 695 /* 696 * Remove driver properties 697 */ 698 if (igb->attach_progress & ATTACH_PROGRESS_PROPS) { 699 (void) ddi_prop_remove_all(devinfo); 700 } 701 702 /* 703 * Stop the adapter 704 */ 705 if (igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) { 706 mutex_enter(&igb->gen_lock); 707 igb_stop_adapter(igb); 708 mutex_exit(&igb->gen_lock); 709 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 710 ddi_fm_service_impact(igb->dip, DDI_SERVICE_UNAFFECTED); 711 } 712 713 /* 714 * Free multicast table 715 */ 716 igb_release_multicast(igb); 717 718 /* 719 * Free register handle 720 */ 721 if (igb->attach_progress & ATTACH_PROGRESS_REGS_MAP) { 722 if (igb->osdep.reg_handle != NULL) 723 ddi_regs_map_free(&igb->osdep.reg_handle); 724 } 725 726 /* 727 * Free PCI config handle 728 */ 729 if (igb->attach_progress & ATTACH_PROGRESS_PCI_CONFIG) { 730 if (igb->osdep.cfg_handle != NULL) 731 pci_config_teardown(&igb->osdep.cfg_handle); 732 } 733 734 /* 735 * Free locks 736 */ 737 if (igb->attach_progress & ATTACH_PROGRESS_LOCKS) { 738 igb_destroy_locks(igb); 739 } 740 741 /* 742 * Free the rx/tx rings 743 */ 744 if (igb->attach_progress & ATTACH_PROGRESS_ALLOC_RINGS) { 745 igb_free_rings(igb); 746 } 747 748 /* 749 * Remove FMA 750 */ 751 if (igb->attach_progress & ATTACH_PROGRESS_FMINIT) { 752 igb_fm_fini(igb); 753 } 754 755 /* 756 * Free the driver data structure 757 */ 758 kmem_free(igb, sizeof (igb_t)); 759 760 ddi_set_driver_private(devinfo, NULL); 761 } 762 763 /* 764 * igb_register_mac - Register the driver and its function pointers with 765 * the GLD interface 766 */ 767 static int 768 igb_register_mac(igb_t *igb) 769 { 770 struct e1000_hw *hw = &igb->hw; 771 mac_register_t *mac; 772 int status; 773 774 if ((mac = mac_alloc(MAC_VERSION)) == NULL) 775 return (IGB_FAILURE); 776 777 mac->m_type_ident = MAC_PLUGIN_IDENT_ETHER; 778 mac->m_driver = igb; 779 mac->m_dip = igb->dip; 780 mac->m_src_addr = hw->mac.addr; 781 mac->m_callbacks = &igb_m_callbacks; 782 mac->m_min_sdu = 0; 783 mac->m_max_sdu = igb->max_frame_size - 784 sizeof (struct ether_vlan_header) - ETHERFCSL; 785 mac->m_margin = VLAN_TAGSZ; 786 mac->m_priv_props = igb_priv_props; 787 mac->m_priv_prop_count = IGB_MAX_PRIV_PROPS; 788 mac->m_v12n = MAC_VIRT_LEVEL1; 789 790 status = mac_register(mac, &igb->mac_hdl); 791 792 mac_free(mac); 793 794 return ((status == 0) ? IGB_SUCCESS : IGB_FAILURE); 795 } 796 797 /* 798 * igb_identify_hardware - Identify the type of the chipset 799 */ 800 static int 801 igb_identify_hardware(igb_t *igb) 802 { 803 struct e1000_hw *hw = &igb->hw; 804 struct igb_osdep *osdep = &igb->osdep; 805 806 /* 807 * Get the device id 808 */ 809 hw->vendor_id = 810 pci_config_get16(osdep->cfg_handle, PCI_CONF_VENID); 811 hw->device_id = 812 pci_config_get16(osdep->cfg_handle, PCI_CONF_DEVID); 813 hw->revision_id = 814 pci_config_get8(osdep->cfg_handle, PCI_CONF_REVID); 815 hw->subsystem_device_id = 816 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBSYSID); 817 hw->subsystem_vendor_id = 818 pci_config_get16(osdep->cfg_handle, PCI_CONF_SUBVENID); 819 820 /* 821 * Set the mac type of the adapter based on the device id 822 */ 823 if (e1000_set_mac_type(hw) != E1000_SUCCESS) { 824 return (IGB_FAILURE); 825 } 826 827 /* 828 * Install adapter capabilities based on mac type 829 */ 830 switch (hw->mac.type) { 831 case e1000_82575: 832 igb->capab = &igb_82575_cap; 833 break; 834 case e1000_82576: 835 igb->capab = &igb_82576_cap; 836 break; 837 case e1000_82580: 838 igb->capab = &igb_82580_cap; 839 break; 840 default: 841 return (IGB_FAILURE); 842 } 843 844 return (IGB_SUCCESS); 845 } 846 847 /* 848 * igb_regs_map - Map the device registers 849 */ 850 static int 851 igb_regs_map(igb_t *igb) 852 { 853 dev_info_t *devinfo = igb->dip; 854 struct e1000_hw *hw = &igb->hw; 855 struct igb_osdep *osdep = &igb->osdep; 856 off_t mem_size; 857 858 /* 859 * First get the size of device registers to be mapped. 860 */ 861 if (ddi_dev_regsize(devinfo, IGB_ADAPTER_REGSET, &mem_size) != 862 DDI_SUCCESS) { 863 return (IGB_FAILURE); 864 } 865 866 /* 867 * Call ddi_regs_map_setup() to map registers 868 */ 869 if ((ddi_regs_map_setup(devinfo, IGB_ADAPTER_REGSET, 870 (caddr_t *)&hw->hw_addr, 0, 871 mem_size, &igb_regs_acc_attr, 872 &osdep->reg_handle)) != DDI_SUCCESS) { 873 return (IGB_FAILURE); 874 } 875 876 return (IGB_SUCCESS); 877 } 878 879 /* 880 * igb_init_properties - Initialize driver properties 881 */ 882 static void 883 igb_init_properties(igb_t *igb) 884 { 885 /* 886 * Get conf file properties, including link settings 887 * jumbo frames, ring number, descriptor number, etc. 888 */ 889 igb_get_conf(igb); 890 } 891 892 /* 893 * igb_init_driver_settings - Initialize driver settings 894 * 895 * The settings include hardware function pointers, bus information, 896 * rx/tx rings settings, link state, and any other parameters that 897 * need to be setup during driver initialization. 898 */ 899 static int 900 igb_init_driver_settings(igb_t *igb) 901 { 902 struct e1000_hw *hw = &igb->hw; 903 igb_rx_ring_t *rx_ring; 904 igb_tx_ring_t *tx_ring; 905 uint32_t rx_size; 906 uint32_t tx_size; 907 int i; 908 909 /* 910 * Initialize chipset specific hardware function pointers 911 */ 912 if (e1000_setup_init_funcs(hw, B_TRUE) != E1000_SUCCESS) { 913 return (IGB_FAILURE); 914 } 915 916 /* 917 * Get bus information 918 */ 919 if (e1000_get_bus_info(hw) != E1000_SUCCESS) { 920 return (IGB_FAILURE); 921 } 922 923 /* 924 * Get the system page size 925 */ 926 igb->page_size = ddi_ptob(igb->dip, (ulong_t)1); 927 928 /* 929 * Set rx buffer size 930 * The IP header alignment room is counted in the calculation. 931 * The rx buffer size is in unit of 1K that is required by the 932 * chipset hardware. 933 */ 934 rx_size = igb->max_frame_size + IPHDR_ALIGN_ROOM; 935 igb->rx_buf_size = ((rx_size >> 10) + 936 ((rx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 937 938 /* 939 * Set tx buffer size 940 */ 941 tx_size = igb->max_frame_size; 942 igb->tx_buf_size = ((tx_size >> 10) + 943 ((tx_size & (((uint32_t)1 << 10) - 1)) > 0 ? 1 : 0)) << 10; 944 945 /* 946 * Initialize rx/tx rings parameters 947 */ 948 for (i = 0; i < igb->num_rx_rings; i++) { 949 rx_ring = &igb->rx_rings[i]; 950 rx_ring->index = i; 951 rx_ring->igb = igb; 952 } 953 954 for (i = 0; i < igb->num_tx_rings; i++) { 955 tx_ring = &igb->tx_rings[i]; 956 tx_ring->index = i; 957 tx_ring->igb = igb; 958 if (igb->tx_head_wb_enable) 959 tx_ring->tx_recycle = igb_tx_recycle_head_wb; 960 else 961 tx_ring->tx_recycle = igb_tx_recycle_legacy; 962 963 tx_ring->ring_size = igb->tx_ring_size; 964 tx_ring->free_list_size = igb->tx_ring_size + 965 (igb->tx_ring_size >> 1); 966 } 967 968 /* 969 * Initialize values of interrupt throttling rates 970 */ 971 for (i = 1; i < MAX_NUM_EITR; i++) 972 igb->intr_throttling[i] = igb->intr_throttling[0]; 973 974 /* 975 * The initial link state should be "unknown" 976 */ 977 igb->link_state = LINK_STATE_UNKNOWN; 978 979 return (IGB_SUCCESS); 980 } 981 982 /* 983 * igb_init_locks - Initialize locks 984 */ 985 static void 986 igb_init_locks(igb_t *igb) 987 { 988 igb_rx_ring_t *rx_ring; 989 igb_tx_ring_t *tx_ring; 990 int i; 991 992 for (i = 0; i < igb->num_rx_rings; i++) { 993 rx_ring = &igb->rx_rings[i]; 994 mutex_init(&rx_ring->rx_lock, NULL, 995 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 996 } 997 998 for (i = 0; i < igb->num_tx_rings; i++) { 999 tx_ring = &igb->tx_rings[i]; 1000 mutex_init(&tx_ring->tx_lock, NULL, 1001 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1002 mutex_init(&tx_ring->recycle_lock, NULL, 1003 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1004 mutex_init(&tx_ring->tcb_head_lock, NULL, 1005 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1006 mutex_init(&tx_ring->tcb_tail_lock, NULL, 1007 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1008 } 1009 1010 mutex_init(&igb->gen_lock, NULL, 1011 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1012 1013 mutex_init(&igb->watchdog_lock, NULL, 1014 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1015 1016 mutex_init(&igb->link_lock, NULL, 1017 MUTEX_DRIVER, DDI_INTR_PRI(igb->intr_pri)); 1018 } 1019 1020 /* 1021 * igb_destroy_locks - Destroy locks 1022 */ 1023 static void 1024 igb_destroy_locks(igb_t *igb) 1025 { 1026 igb_rx_ring_t *rx_ring; 1027 igb_tx_ring_t *tx_ring; 1028 int i; 1029 1030 for (i = 0; i < igb->num_rx_rings; i++) { 1031 rx_ring = &igb->rx_rings[i]; 1032 mutex_destroy(&rx_ring->rx_lock); 1033 } 1034 1035 for (i = 0; i < igb->num_tx_rings; i++) { 1036 tx_ring = &igb->tx_rings[i]; 1037 mutex_destroy(&tx_ring->tx_lock); 1038 mutex_destroy(&tx_ring->recycle_lock); 1039 mutex_destroy(&tx_ring->tcb_head_lock); 1040 mutex_destroy(&tx_ring->tcb_tail_lock); 1041 } 1042 1043 mutex_destroy(&igb->gen_lock); 1044 mutex_destroy(&igb->watchdog_lock); 1045 mutex_destroy(&igb->link_lock); 1046 } 1047 1048 static int 1049 igb_resume(dev_info_t *devinfo) 1050 { 1051 igb_t *igb; 1052 1053 igb = (igb_t *)ddi_get_driver_private(devinfo); 1054 if (igb == NULL) 1055 return (DDI_FAILURE); 1056 1057 mutex_enter(&igb->gen_lock); 1058 1059 if (igb->igb_state & IGB_STARTED) { 1060 if (igb_start(igb, B_FALSE) != IGB_SUCCESS) { 1061 mutex_exit(&igb->gen_lock); 1062 return (DDI_FAILURE); 1063 } 1064 1065 /* 1066 * Enable and start the watchdog timer 1067 */ 1068 igb_enable_watchdog_timer(igb); 1069 } 1070 1071 atomic_and_32(&igb->igb_state, ~IGB_SUSPENDED); 1072 1073 mutex_exit(&igb->gen_lock); 1074 1075 return (DDI_SUCCESS); 1076 } 1077 1078 static int 1079 igb_suspend(dev_info_t *devinfo) 1080 { 1081 igb_t *igb; 1082 1083 igb = (igb_t *)ddi_get_driver_private(devinfo); 1084 if (igb == NULL) 1085 return (DDI_FAILURE); 1086 1087 mutex_enter(&igb->gen_lock); 1088 1089 atomic_or_32(&igb->igb_state, IGB_SUSPENDED); 1090 1091 if (!(igb->igb_state & IGB_STARTED)) { 1092 mutex_exit(&igb->gen_lock); 1093 return (DDI_SUCCESS); 1094 } 1095 1096 igb_stop(igb, B_FALSE); 1097 1098 mutex_exit(&igb->gen_lock); 1099 1100 /* 1101 * Disable and stop the watchdog timer 1102 */ 1103 igb_disable_watchdog_timer(igb); 1104 1105 return (DDI_SUCCESS); 1106 } 1107 1108 static int 1109 igb_init(igb_t *igb) 1110 { 1111 mutex_enter(&igb->gen_lock); 1112 1113 /* 1114 * Initilize the adapter 1115 */ 1116 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1117 mutex_exit(&igb->gen_lock); 1118 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1119 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1120 return (IGB_FAILURE); 1121 } 1122 1123 mutex_exit(&igb->gen_lock); 1124 1125 return (IGB_SUCCESS); 1126 } 1127 1128 /* 1129 * igb_init_mac_address - Initialize the default MAC address 1130 * 1131 * On success, the MAC address is entered in the igb->hw.mac.addr 1132 * and hw->mac.perm_addr fields and the adapter's RAR(0) receive 1133 * address register. 1134 * 1135 * Important side effects: 1136 * 1. adapter is reset - this is required to put it in a known state. 1137 * 2. all of non-volatile memory (NVM) is read & checksummed - NVM is where 1138 * MAC address and all default settings are stored, so a valid checksum 1139 * is required. 1140 */ 1141 static int 1142 igb_init_mac_address(igb_t *igb) 1143 { 1144 struct e1000_hw *hw = &igb->hw; 1145 1146 ASSERT(mutex_owned(&igb->gen_lock)); 1147 1148 /* 1149 * Reset chipset to put the hardware in a known state 1150 * before we try to get MAC address from NVM. 1151 */ 1152 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1153 igb_error(igb, "Adapter reset failed."); 1154 goto init_mac_fail; 1155 } 1156 1157 /* 1158 * NVM validation 1159 */ 1160 if (e1000_validate_nvm_checksum(hw) < 0) { 1161 /* 1162 * Some PCI-E parts fail the first check due to 1163 * the link being in sleep state. Call it again, 1164 * if it fails a second time its a real issue. 1165 */ 1166 if (e1000_validate_nvm_checksum(hw) < 0) { 1167 igb_error(igb, 1168 "Invalid NVM checksum. Please contact " 1169 "the vendor to update the NVM."); 1170 goto init_mac_fail; 1171 } 1172 } 1173 1174 /* 1175 * Get the mac address 1176 * This function should handle SPARC case correctly. 1177 */ 1178 if (!igb_find_mac_address(igb)) { 1179 igb_error(igb, "Failed to get the mac address"); 1180 goto init_mac_fail; 1181 } 1182 1183 /* Validate mac address */ 1184 if (!is_valid_mac_addr(hw->mac.addr)) { 1185 igb_error(igb, "Invalid mac address"); 1186 goto init_mac_fail; 1187 } 1188 1189 return (IGB_SUCCESS); 1190 1191 init_mac_fail: 1192 return (IGB_FAILURE); 1193 } 1194 1195 /* 1196 * igb_init_adapter - Initialize the adapter 1197 */ 1198 static int 1199 igb_init_adapter(igb_t *igb) 1200 { 1201 struct e1000_hw *hw = &igb->hw; 1202 uint32_t pba; 1203 uint32_t high_water; 1204 int i; 1205 1206 ASSERT(mutex_owned(&igb->gen_lock)); 1207 1208 /* 1209 * In order to obtain the default MAC address, this will reset the 1210 * adapter and validate the NVM that the address and many other 1211 * default settings come from. 1212 */ 1213 if (igb_init_mac_address(igb) != IGB_SUCCESS) { 1214 igb_error(igb, "Failed to initialize MAC address"); 1215 goto init_adapter_fail; 1216 } 1217 1218 /* 1219 * Setup flow control 1220 * 1221 * These parameters set thresholds for the adapter's generation(Tx) 1222 * and response(Rx) to Ethernet PAUSE frames. These are just threshold 1223 * settings. Flow control is enabled or disabled in the configuration 1224 * file. 1225 * High-water mark is set down from the top of the rx fifo (not 1226 * sensitive to max_frame_size) and low-water is set just below 1227 * high-water mark. 1228 * The high water mark must be low enough to fit one full frame above 1229 * it in the rx FIFO. Should be the lower of: 1230 * 90% of the Rx FIFO size, or the full Rx FIFO size minus one full 1231 * frame. 1232 */ 1233 /* 1234 * The default setting of PBA is correct for 82575 and other supported 1235 * adapters do not have the E1000_PBA register, so PBA value is only 1236 * used for calculation here and is never written to the adapter. 1237 */ 1238 if (hw->mac.type == e1000_82575) { 1239 pba = E1000_PBA_34K; 1240 } else { 1241 pba = E1000_PBA_64K; 1242 } 1243 1244 high_water = min(((pba << 10) * 9 / 10), 1245 ((pba << 10) - igb->max_frame_size)); 1246 1247 if (hw->mac.type == e1000_82575) { 1248 /* 8-byte granularity */ 1249 hw->fc.high_water = high_water & 0xFFF8; 1250 hw->fc.low_water = hw->fc.high_water - 8; 1251 } else { 1252 /* 16-byte granularity */ 1253 hw->fc.high_water = high_water & 0xFFF0; 1254 hw->fc.low_water = hw->fc.high_water - 16; 1255 } 1256 1257 hw->fc.pause_time = E1000_FC_PAUSE_TIME; 1258 hw->fc.send_xon = B_TRUE; 1259 1260 (void) e1000_validate_mdi_setting(hw); 1261 1262 /* 1263 * Reset the chipset hardware the second time to put PBA settings 1264 * into effect. 1265 */ 1266 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1267 igb_error(igb, "Second reset failed"); 1268 goto init_adapter_fail; 1269 } 1270 1271 /* 1272 * Don't wait for auto-negotiation to complete 1273 */ 1274 hw->phy.autoneg_wait_to_complete = B_FALSE; 1275 1276 /* 1277 * Copper options 1278 */ 1279 if (hw->phy.media_type == e1000_media_type_copper) { 1280 hw->phy.mdix = 0; /* AUTO_ALL_MODES */ 1281 hw->phy.disable_polarity_correction = B_FALSE; 1282 hw->phy.ms_type = e1000_ms_hw_default; /* E1000_MASTER_SLAVE */ 1283 } 1284 1285 /* 1286 * Initialize link settings 1287 */ 1288 (void) igb_setup_link(igb, B_FALSE); 1289 1290 /* 1291 * Configure/Initialize hardware 1292 */ 1293 if (e1000_init_hw(hw) != E1000_SUCCESS) { 1294 igb_error(igb, "Failed to initialize hardware"); 1295 goto init_adapter_fail; 1296 } 1297 1298 /* 1299 * Start the link setup timer 1300 */ 1301 igb_start_link_timer(igb); 1302 1303 /* 1304 * Disable wakeup control by default 1305 */ 1306 E1000_WRITE_REG(hw, E1000_WUC, 0); 1307 1308 /* 1309 * Record phy info in hw struct 1310 */ 1311 (void) e1000_get_phy_info(hw); 1312 1313 /* 1314 * Make sure driver has control 1315 */ 1316 igb_get_driver_control(hw); 1317 1318 /* 1319 * Restore LED settings to the default from EEPROM 1320 * to meet the standard for Sun platforms. 1321 */ 1322 (void) e1000_cleanup_led(hw); 1323 1324 /* 1325 * Setup MSI-X interrupts 1326 */ 1327 if (igb->intr_type == DDI_INTR_TYPE_MSIX) 1328 igb->capab->setup_msix(igb); 1329 1330 /* 1331 * Initialize unicast addresses. 1332 */ 1333 igb_init_unicst(igb); 1334 1335 /* 1336 * Setup and initialize the mctable structures. 1337 */ 1338 igb_setup_multicst(igb); 1339 1340 /* 1341 * Set interrupt throttling rate 1342 */ 1343 for (i = 0; i < igb->intr_cnt; i++) 1344 E1000_WRITE_REG(hw, E1000_EITR(i), igb->intr_throttling[i]); 1345 1346 /* 1347 * Save the state of the phy 1348 */ 1349 igb_get_phy_state(igb); 1350 1351 igb_param_sync(igb); 1352 1353 return (IGB_SUCCESS); 1354 1355 init_adapter_fail: 1356 /* 1357 * Reset PHY if possible 1358 */ 1359 if (e1000_check_reset_block(hw) == E1000_SUCCESS) 1360 (void) e1000_phy_hw_reset(hw); 1361 1362 return (IGB_FAILURE); 1363 } 1364 1365 /* 1366 * igb_stop_adapter - Stop the adapter 1367 */ 1368 static void 1369 igb_stop_adapter(igb_t *igb) 1370 { 1371 struct e1000_hw *hw = &igb->hw; 1372 1373 ASSERT(mutex_owned(&igb->gen_lock)); 1374 1375 /* Stop the link setup timer */ 1376 igb_stop_link_timer(igb); 1377 1378 /* Tell firmware driver is no longer in control */ 1379 igb_release_driver_control(hw); 1380 1381 /* 1382 * Reset the chipset 1383 */ 1384 if (e1000_reset_hw(hw) != E1000_SUCCESS) { 1385 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1386 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1387 } 1388 1389 /* 1390 * e1000_phy_hw_reset is not needed here, MAC reset above is sufficient 1391 */ 1392 } 1393 1394 /* 1395 * igb_reset - Reset the chipset and restart the driver. 1396 * 1397 * It involves stopping and re-starting the chipset, 1398 * and re-configuring the rx/tx rings. 1399 */ 1400 static int 1401 igb_reset(igb_t *igb) 1402 { 1403 int i; 1404 1405 mutex_enter(&igb->gen_lock); 1406 1407 ASSERT(igb->igb_state & IGB_STARTED); 1408 atomic_and_32(&igb->igb_state, ~IGB_STARTED); 1409 1410 /* 1411 * Disable the adapter interrupts to stop any rx/tx activities 1412 * before draining pending data and resetting hardware. 1413 */ 1414 igb_disable_adapter_interrupts(igb); 1415 1416 /* 1417 * Drain the pending transmit packets 1418 */ 1419 (void) igb_tx_drain(igb); 1420 1421 for (i = 0; i < igb->num_rx_rings; i++) 1422 mutex_enter(&igb->rx_rings[i].rx_lock); 1423 for (i = 0; i < igb->num_tx_rings; i++) 1424 mutex_enter(&igb->tx_rings[i].tx_lock); 1425 1426 /* 1427 * Stop the adapter 1428 */ 1429 igb_stop_adapter(igb); 1430 1431 /* 1432 * Clean the pending tx data/resources 1433 */ 1434 igb_tx_clean(igb); 1435 1436 /* 1437 * Start the adapter 1438 */ 1439 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1440 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1441 goto reset_failure; 1442 } 1443 1444 /* 1445 * Setup the rx/tx rings 1446 */ 1447 igb->tx_ring_init = B_FALSE; 1448 igb_setup_rings(igb); 1449 1450 atomic_and_32(&igb->igb_state, ~(IGB_ERROR | IGB_STALL)); 1451 1452 /* 1453 * Enable adapter interrupts 1454 * The interrupts must be enabled after the driver state is START 1455 */ 1456 igb->capab->enable_intr(igb); 1457 1458 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1459 goto reset_failure; 1460 1461 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1462 goto reset_failure; 1463 1464 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1465 mutex_exit(&igb->tx_rings[i].tx_lock); 1466 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1467 mutex_exit(&igb->rx_rings[i].rx_lock); 1468 1469 atomic_or_32(&igb->igb_state, IGB_STARTED); 1470 1471 mutex_exit(&igb->gen_lock); 1472 1473 return (IGB_SUCCESS); 1474 1475 reset_failure: 1476 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1477 mutex_exit(&igb->tx_rings[i].tx_lock); 1478 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1479 mutex_exit(&igb->rx_rings[i].rx_lock); 1480 1481 mutex_exit(&igb->gen_lock); 1482 1483 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1484 1485 return (IGB_FAILURE); 1486 } 1487 1488 /* 1489 * igb_tx_clean - Clean the pending transmit packets and DMA resources 1490 */ 1491 static void 1492 igb_tx_clean(igb_t *igb) 1493 { 1494 igb_tx_ring_t *tx_ring; 1495 tx_control_block_t *tcb; 1496 link_list_t pending_list; 1497 uint32_t desc_num; 1498 int i, j; 1499 1500 LINK_LIST_INIT(&pending_list); 1501 1502 for (i = 0; i < igb->num_tx_rings; i++) { 1503 tx_ring = &igb->tx_rings[i]; 1504 1505 mutex_enter(&tx_ring->recycle_lock); 1506 1507 /* 1508 * Clean the pending tx data - the pending packets in the 1509 * work_list that have no chances to be transmitted again. 1510 * 1511 * We must ensure the chipset is stopped or the link is down 1512 * before cleaning the transmit packets. 1513 */ 1514 desc_num = 0; 1515 for (j = 0; j < tx_ring->ring_size; j++) { 1516 tcb = tx_ring->work_list[j]; 1517 if (tcb != NULL) { 1518 desc_num += tcb->desc_num; 1519 1520 tx_ring->work_list[j] = NULL; 1521 1522 igb_free_tcb(tcb); 1523 1524 LIST_PUSH_TAIL(&pending_list, &tcb->link); 1525 } 1526 } 1527 1528 if (desc_num > 0) { 1529 atomic_add_32(&tx_ring->tbd_free, desc_num); 1530 ASSERT(tx_ring->tbd_free == tx_ring->ring_size); 1531 1532 /* 1533 * Reset the head and tail pointers of the tbd ring; 1534 * Reset the head write-back if it is enabled. 1535 */ 1536 tx_ring->tbd_head = 0; 1537 tx_ring->tbd_tail = 0; 1538 if (igb->tx_head_wb_enable) 1539 *tx_ring->tbd_head_wb = 0; 1540 1541 E1000_WRITE_REG(&igb->hw, E1000_TDH(tx_ring->index), 0); 1542 E1000_WRITE_REG(&igb->hw, E1000_TDT(tx_ring->index), 0); 1543 } 1544 1545 mutex_exit(&tx_ring->recycle_lock); 1546 1547 /* 1548 * Add the tx control blocks in the pending list to 1549 * the free list. 1550 */ 1551 igb_put_free_list(tx_ring, &pending_list); 1552 } 1553 } 1554 1555 /* 1556 * igb_tx_drain - Drain the tx rings to allow pending packets to be transmitted 1557 */ 1558 static boolean_t 1559 igb_tx_drain(igb_t *igb) 1560 { 1561 igb_tx_ring_t *tx_ring; 1562 boolean_t done; 1563 int i, j; 1564 1565 /* 1566 * Wait for a specific time to allow pending tx packets 1567 * to be transmitted. 1568 * 1569 * Check the counter tbd_free to see if transmission is done. 1570 * No lock protection is needed here. 1571 * 1572 * Return B_TRUE if all pending packets have been transmitted; 1573 * Otherwise return B_FALSE; 1574 */ 1575 for (i = 0; i < TX_DRAIN_TIME; i++) { 1576 1577 done = B_TRUE; 1578 for (j = 0; j < igb->num_tx_rings; j++) { 1579 tx_ring = &igb->tx_rings[j]; 1580 done = done && 1581 (tx_ring->tbd_free == tx_ring->ring_size); 1582 } 1583 1584 if (done) 1585 break; 1586 1587 msec_delay(1); 1588 } 1589 1590 return (done); 1591 } 1592 1593 /* 1594 * igb_rx_drain - Wait for all rx buffers to be released by upper layer 1595 */ 1596 static boolean_t 1597 igb_rx_drain(igb_t *igb) 1598 { 1599 boolean_t done; 1600 int i; 1601 1602 /* 1603 * Polling the rx free list to check if those rx buffers held by 1604 * the upper layer are released. 1605 * 1606 * Check the counter rcb_free to see if all pending buffers are 1607 * released. No lock protection is needed here. 1608 * 1609 * Return B_TRUE if all pending buffers have been released; 1610 * Otherwise return B_FALSE; 1611 */ 1612 for (i = 0; i < RX_DRAIN_TIME; i++) { 1613 done = (igb->rcb_pending == 0); 1614 1615 if (done) 1616 break; 1617 1618 msec_delay(1); 1619 } 1620 1621 return (done); 1622 } 1623 1624 /* 1625 * igb_start - Start the driver/chipset 1626 */ 1627 int 1628 igb_start(igb_t *igb, boolean_t alloc_buffer) 1629 { 1630 int i; 1631 1632 ASSERT(mutex_owned(&igb->gen_lock)); 1633 1634 if (alloc_buffer) { 1635 if (igb_alloc_rx_data(igb) != IGB_SUCCESS) { 1636 igb_error(igb, 1637 "Failed to allocate software receive rings"); 1638 return (IGB_FAILURE); 1639 } 1640 1641 /* Allocate buffers for all the rx/tx rings */ 1642 if (igb_alloc_dma(igb) != IGB_SUCCESS) { 1643 igb_error(igb, "Failed to allocate DMA resource"); 1644 return (IGB_FAILURE); 1645 } 1646 1647 igb->tx_ring_init = B_TRUE; 1648 } else { 1649 igb->tx_ring_init = B_FALSE; 1650 } 1651 1652 for (i = 0; i < igb->num_rx_rings; i++) 1653 mutex_enter(&igb->rx_rings[i].rx_lock); 1654 for (i = 0; i < igb->num_tx_rings; i++) 1655 mutex_enter(&igb->tx_rings[i].tx_lock); 1656 1657 /* 1658 * Start the adapter 1659 */ 1660 if ((igb->attach_progress & ATTACH_PROGRESS_INIT_ADAPTER) == 0) { 1661 if (igb_init_adapter(igb) != IGB_SUCCESS) { 1662 igb_fm_ereport(igb, DDI_FM_DEVICE_INVAL_STATE); 1663 goto start_failure; 1664 } 1665 igb->attach_progress |= ATTACH_PROGRESS_INIT_ADAPTER; 1666 } 1667 1668 /* 1669 * Setup the rx/tx rings 1670 */ 1671 igb_setup_rings(igb); 1672 1673 /* 1674 * Enable adapter interrupts 1675 * The interrupts must be enabled after the driver state is START 1676 */ 1677 igb->capab->enable_intr(igb); 1678 1679 if (igb_check_acc_handle(igb->osdep.cfg_handle) != DDI_FM_OK) 1680 goto start_failure; 1681 1682 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1683 goto start_failure; 1684 1685 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1686 mutex_exit(&igb->tx_rings[i].tx_lock); 1687 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1688 mutex_exit(&igb->rx_rings[i].rx_lock); 1689 1690 return (IGB_SUCCESS); 1691 1692 start_failure: 1693 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1694 mutex_exit(&igb->tx_rings[i].tx_lock); 1695 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1696 mutex_exit(&igb->rx_rings[i].rx_lock); 1697 1698 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1699 1700 return (IGB_FAILURE); 1701 } 1702 1703 /* 1704 * igb_stop - Stop the driver/chipset 1705 */ 1706 void 1707 igb_stop(igb_t *igb, boolean_t free_buffer) 1708 { 1709 int i; 1710 1711 ASSERT(mutex_owned(&igb->gen_lock)); 1712 1713 igb->attach_progress &= ~ATTACH_PROGRESS_INIT_ADAPTER; 1714 1715 /* 1716 * Disable the adapter interrupts 1717 */ 1718 igb_disable_adapter_interrupts(igb); 1719 1720 /* 1721 * Drain the pending tx packets 1722 */ 1723 (void) igb_tx_drain(igb); 1724 1725 for (i = 0; i < igb->num_rx_rings; i++) 1726 mutex_enter(&igb->rx_rings[i].rx_lock); 1727 for (i = 0; i < igb->num_tx_rings; i++) 1728 mutex_enter(&igb->tx_rings[i].tx_lock); 1729 1730 /* 1731 * Stop the adapter 1732 */ 1733 igb_stop_adapter(igb); 1734 1735 /* 1736 * Clean the pending tx data/resources 1737 */ 1738 igb_tx_clean(igb); 1739 1740 for (i = igb->num_tx_rings - 1; i >= 0; i--) 1741 mutex_exit(&igb->tx_rings[i].tx_lock); 1742 for (i = igb->num_rx_rings - 1; i >= 0; i--) 1743 mutex_exit(&igb->rx_rings[i].rx_lock); 1744 1745 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) 1746 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 1747 1748 if (igb->link_state == LINK_STATE_UP) { 1749 igb->link_state = LINK_STATE_UNKNOWN; 1750 mac_link_update(igb->mac_hdl, igb->link_state); 1751 } 1752 1753 if (free_buffer) { 1754 /* 1755 * Release the DMA/memory resources of rx/tx rings 1756 */ 1757 igb_free_dma(igb); 1758 igb_free_rx_data(igb); 1759 } 1760 } 1761 1762 /* 1763 * igb_alloc_rings - Allocate memory space for rx/tx rings 1764 */ 1765 static int 1766 igb_alloc_rings(igb_t *igb) 1767 { 1768 /* 1769 * Allocate memory space for rx rings 1770 */ 1771 igb->rx_rings = kmem_zalloc( 1772 sizeof (igb_rx_ring_t) * igb->num_rx_rings, 1773 KM_NOSLEEP); 1774 1775 if (igb->rx_rings == NULL) { 1776 return (IGB_FAILURE); 1777 } 1778 1779 /* 1780 * Allocate memory space for tx rings 1781 */ 1782 igb->tx_rings = kmem_zalloc( 1783 sizeof (igb_tx_ring_t) * igb->num_tx_rings, 1784 KM_NOSLEEP); 1785 1786 if (igb->tx_rings == NULL) { 1787 kmem_free(igb->rx_rings, 1788 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1789 igb->rx_rings = NULL; 1790 return (IGB_FAILURE); 1791 } 1792 1793 /* 1794 * Allocate memory space for rx ring groups 1795 */ 1796 igb->rx_groups = kmem_zalloc( 1797 sizeof (igb_rx_group_t) * igb->num_rx_groups, 1798 KM_NOSLEEP); 1799 1800 if (igb->rx_groups == NULL) { 1801 kmem_free(igb->rx_rings, 1802 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1803 kmem_free(igb->tx_rings, 1804 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1805 igb->rx_rings = NULL; 1806 igb->tx_rings = NULL; 1807 return (IGB_FAILURE); 1808 } 1809 1810 return (IGB_SUCCESS); 1811 } 1812 1813 /* 1814 * igb_free_rings - Free the memory space of rx/tx rings. 1815 */ 1816 static void 1817 igb_free_rings(igb_t *igb) 1818 { 1819 if (igb->rx_rings != NULL) { 1820 kmem_free(igb->rx_rings, 1821 sizeof (igb_rx_ring_t) * igb->num_rx_rings); 1822 igb->rx_rings = NULL; 1823 } 1824 1825 if (igb->tx_rings != NULL) { 1826 kmem_free(igb->tx_rings, 1827 sizeof (igb_tx_ring_t) * igb->num_tx_rings); 1828 igb->tx_rings = NULL; 1829 } 1830 1831 if (igb->rx_groups != NULL) { 1832 kmem_free(igb->rx_groups, 1833 sizeof (igb_rx_group_t) * igb->num_rx_groups); 1834 igb->rx_groups = NULL; 1835 } 1836 } 1837 1838 static int 1839 igb_alloc_rx_data(igb_t *igb) 1840 { 1841 igb_rx_ring_t *rx_ring; 1842 int i; 1843 1844 for (i = 0; i < igb->num_rx_rings; i++) { 1845 rx_ring = &igb->rx_rings[i]; 1846 if (igb_alloc_rx_ring_data(rx_ring) != IGB_SUCCESS) 1847 goto alloc_rx_rings_failure; 1848 } 1849 return (IGB_SUCCESS); 1850 1851 alloc_rx_rings_failure: 1852 igb_free_rx_data(igb); 1853 return (IGB_FAILURE); 1854 } 1855 1856 static void 1857 igb_free_rx_data(igb_t *igb) 1858 { 1859 igb_rx_ring_t *rx_ring; 1860 igb_rx_data_t *rx_data; 1861 int i; 1862 1863 for (i = 0; i < igb->num_rx_rings; i++) { 1864 rx_ring = &igb->rx_rings[i]; 1865 1866 mutex_enter(&igb->rx_pending_lock); 1867 rx_data = rx_ring->rx_data; 1868 1869 if (rx_data != NULL) { 1870 rx_data->flag |= IGB_RX_STOPPED; 1871 1872 if (rx_data->rcb_pending == 0) { 1873 igb_free_rx_ring_data(rx_data); 1874 rx_ring->rx_data = NULL; 1875 } 1876 } 1877 1878 mutex_exit(&igb->rx_pending_lock); 1879 } 1880 } 1881 1882 /* 1883 * igb_setup_rings - Setup rx/tx rings 1884 */ 1885 static void 1886 igb_setup_rings(igb_t *igb) 1887 { 1888 /* 1889 * Setup the rx/tx rings, including the following: 1890 * 1891 * 1. Setup the descriptor ring and the control block buffers; 1892 * 2. Initialize necessary registers for receive/transmit; 1893 * 3. Initialize software pointers/parameters for receive/transmit; 1894 */ 1895 igb_setup_rx(igb); 1896 1897 igb_setup_tx(igb); 1898 } 1899 1900 static void 1901 igb_setup_rx_ring(igb_rx_ring_t *rx_ring) 1902 { 1903 igb_t *igb = rx_ring->igb; 1904 igb_rx_data_t *rx_data = rx_ring->rx_data; 1905 struct e1000_hw *hw = &igb->hw; 1906 rx_control_block_t *rcb; 1907 union e1000_adv_rx_desc *rbd; 1908 uint32_t size; 1909 uint32_t buf_low; 1910 uint32_t buf_high; 1911 uint32_t rxdctl; 1912 int i; 1913 1914 ASSERT(mutex_owned(&rx_ring->rx_lock)); 1915 ASSERT(mutex_owned(&igb->gen_lock)); 1916 1917 /* 1918 * Initialize descriptor ring with buffer addresses 1919 */ 1920 for (i = 0; i < igb->rx_ring_size; i++) { 1921 rcb = rx_data->work_list[i]; 1922 rbd = &rx_data->rbd_ring[i]; 1923 1924 rbd->read.pkt_addr = rcb->rx_buf.dma_address; 1925 rbd->read.hdr_addr = NULL; 1926 } 1927 1928 /* 1929 * Initialize the base address registers 1930 */ 1931 buf_low = (uint32_t)rx_data->rbd_area.dma_address; 1932 buf_high = (uint32_t)(rx_data->rbd_area.dma_address >> 32); 1933 E1000_WRITE_REG(hw, E1000_RDBAH(rx_ring->index), buf_high); 1934 E1000_WRITE_REG(hw, E1000_RDBAL(rx_ring->index), buf_low); 1935 1936 /* 1937 * Initialize the length register 1938 */ 1939 size = rx_data->ring_size * sizeof (union e1000_adv_rx_desc); 1940 E1000_WRITE_REG(hw, E1000_RDLEN(rx_ring->index), size); 1941 1942 /* 1943 * Initialize buffer size & descriptor type 1944 */ 1945 E1000_WRITE_REG(hw, E1000_SRRCTL(rx_ring->index), 1946 ((igb->rx_buf_size >> E1000_SRRCTL_BSIZEPKT_SHIFT) | 1947 E1000_SRRCTL_DESCTYPE_ADV_ONEBUF)); 1948 1949 /* 1950 * Setup the Receive Descriptor Control Register (RXDCTL) 1951 */ 1952 rxdctl = E1000_READ_REG(hw, E1000_RXDCTL(rx_ring->index)); 1953 rxdctl &= igb->capab->rxdctl_mask; 1954 rxdctl |= E1000_RXDCTL_QUEUE_ENABLE; 1955 rxdctl |= 16; /* pthresh */ 1956 rxdctl |= 8 << 8; /* hthresh */ 1957 rxdctl |= 1 << 16; /* wthresh */ 1958 E1000_WRITE_REG(hw, E1000_RXDCTL(rx_ring->index), rxdctl); 1959 1960 rx_data->rbd_next = 0; 1961 } 1962 1963 static void 1964 igb_setup_rx(igb_t *igb) 1965 { 1966 igb_rx_ring_t *rx_ring; 1967 igb_rx_data_t *rx_data; 1968 igb_rx_group_t *rx_group; 1969 struct e1000_hw *hw = &igb->hw; 1970 uint32_t rctl, rxcsum; 1971 uint32_t ring_per_group; 1972 int i; 1973 1974 /* 1975 * Setup the Receive Control Register (RCTL), and enable the 1976 * receiver. The initial configuration is to: enable the receiver, 1977 * accept broadcasts, discard bad packets, accept long packets, 1978 * disable VLAN filter checking, and set receive buffer size to 1979 * 2k. For 82575, also set the receive descriptor minimum 1980 * threshold size to 1/2 the ring. 1981 */ 1982 rctl = E1000_READ_REG(hw, E1000_RCTL); 1983 1984 /* 1985 * Clear the field used for wakeup control. This driver doesn't do 1986 * wakeup but leave this here for completeness. 1987 */ 1988 rctl &= ~(3 << E1000_RCTL_MO_SHIFT); 1989 rctl &= ~(E1000_RCTL_LBM_TCVR | E1000_RCTL_LBM_MAC); 1990 1991 rctl |= (E1000_RCTL_EN | /* Enable Receive Unit */ 1992 E1000_RCTL_BAM | /* Accept Broadcast Packets */ 1993 E1000_RCTL_LPE | /* Large Packet Enable */ 1994 /* Multicast filter offset */ 1995 (hw->mac.mc_filter_type << E1000_RCTL_MO_SHIFT) | 1996 E1000_RCTL_RDMTS_HALF | /* rx descriptor threshold */ 1997 E1000_RCTL_SECRC); /* Strip Ethernet CRC */ 1998 1999 for (i = 0; i < igb->num_rx_groups; i++) { 2000 rx_group = &igb->rx_groups[i]; 2001 rx_group->index = i; 2002 rx_group->igb = igb; 2003 } 2004 2005 /* 2006 * Set up all rx descriptor rings - must be called before receive unit 2007 * enabled. 2008 */ 2009 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2010 for (i = 0; i < igb->num_rx_rings; i++) { 2011 rx_ring = &igb->rx_rings[i]; 2012 igb_setup_rx_ring(rx_ring); 2013 2014 /* 2015 * Map a ring to a group by assigning a group index 2016 */ 2017 rx_ring->group_index = i / ring_per_group; 2018 } 2019 2020 /* 2021 * Setup the Rx Long Packet Max Length register 2022 */ 2023 E1000_WRITE_REG(hw, E1000_RLPML, igb->max_frame_size); 2024 2025 /* 2026 * Hardware checksum settings 2027 */ 2028 if (igb->rx_hcksum_enable) { 2029 rxcsum = 2030 E1000_RXCSUM_TUOFL | /* TCP/UDP checksum */ 2031 E1000_RXCSUM_IPOFL; /* IP checksum */ 2032 2033 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2034 } 2035 2036 /* 2037 * Setup classify and RSS for multiple receive queues 2038 */ 2039 switch (igb->vmdq_mode) { 2040 case E1000_VMDQ_OFF: 2041 /* 2042 * One ring group, only RSS is needed when more than 2043 * one ring enabled. 2044 */ 2045 if (igb->num_rx_rings > 1) 2046 igb_setup_rss(igb); 2047 break; 2048 case E1000_VMDQ_MAC: 2049 /* 2050 * Multiple groups, each group has one ring, 2051 * only the MAC classification is needed. 2052 */ 2053 igb_setup_mac_classify(igb); 2054 break; 2055 case E1000_VMDQ_MAC_RSS: 2056 /* 2057 * Multiple groups and multiple rings, both 2058 * MAC classification and RSS are needed. 2059 */ 2060 igb_setup_mac_rss_classify(igb); 2061 break; 2062 } 2063 2064 /* 2065 * Enable the receive unit - must be done after all 2066 * the rx setup above. 2067 */ 2068 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 2069 2070 /* 2071 * Initialize all adapter ring head & tail pointers - must 2072 * be done after receive unit is enabled 2073 */ 2074 for (i = 0; i < igb->num_rx_rings; i++) { 2075 rx_ring = &igb->rx_rings[i]; 2076 rx_data = rx_ring->rx_data; 2077 E1000_WRITE_REG(hw, E1000_RDH(i), 0); 2078 E1000_WRITE_REG(hw, E1000_RDT(i), rx_data->ring_size - 1); 2079 } 2080 2081 /* 2082 * 82575 with manageability enabled needs a special flush to make 2083 * sure the fifos start clean. 2084 */ 2085 if ((hw->mac.type == e1000_82575) && 2086 (E1000_READ_REG(hw, E1000_MANC) & E1000_MANC_RCV_TCO_EN)) { 2087 e1000_rx_fifo_flush_82575(hw); 2088 } 2089 } 2090 2091 static void 2092 igb_setup_tx_ring(igb_tx_ring_t *tx_ring) 2093 { 2094 igb_t *igb = tx_ring->igb; 2095 struct e1000_hw *hw = &igb->hw; 2096 uint32_t size; 2097 uint32_t buf_low; 2098 uint32_t buf_high; 2099 uint32_t reg_val; 2100 2101 ASSERT(mutex_owned(&tx_ring->tx_lock)); 2102 ASSERT(mutex_owned(&igb->gen_lock)); 2103 2104 2105 /* 2106 * Initialize the length register 2107 */ 2108 size = tx_ring->ring_size * sizeof (union e1000_adv_tx_desc); 2109 E1000_WRITE_REG(hw, E1000_TDLEN(tx_ring->index), size); 2110 2111 /* 2112 * Initialize the base address registers 2113 */ 2114 buf_low = (uint32_t)tx_ring->tbd_area.dma_address; 2115 buf_high = (uint32_t)(tx_ring->tbd_area.dma_address >> 32); 2116 E1000_WRITE_REG(hw, E1000_TDBAL(tx_ring->index), buf_low); 2117 E1000_WRITE_REG(hw, E1000_TDBAH(tx_ring->index), buf_high); 2118 2119 /* 2120 * Setup head & tail pointers 2121 */ 2122 E1000_WRITE_REG(hw, E1000_TDH(tx_ring->index), 0); 2123 E1000_WRITE_REG(hw, E1000_TDT(tx_ring->index), 0); 2124 2125 /* 2126 * Setup head write-back 2127 */ 2128 if (igb->tx_head_wb_enable) { 2129 /* 2130 * The memory of the head write-back is allocated using 2131 * the extra tbd beyond the tail of the tbd ring. 2132 */ 2133 tx_ring->tbd_head_wb = (uint32_t *) 2134 ((uintptr_t)tx_ring->tbd_area.address + size); 2135 *tx_ring->tbd_head_wb = 0; 2136 2137 buf_low = (uint32_t) 2138 (tx_ring->tbd_area.dma_address + size); 2139 buf_high = (uint32_t) 2140 ((tx_ring->tbd_area.dma_address + size) >> 32); 2141 2142 /* Set the head write-back enable bit */ 2143 buf_low |= E1000_TX_HEAD_WB_ENABLE; 2144 2145 E1000_WRITE_REG(hw, E1000_TDWBAL(tx_ring->index), buf_low); 2146 E1000_WRITE_REG(hw, E1000_TDWBAH(tx_ring->index), buf_high); 2147 2148 /* 2149 * Turn off relaxed ordering for head write back or it will 2150 * cause problems with the tx recycling 2151 */ 2152 reg_val = E1000_READ_REG(hw, 2153 E1000_DCA_TXCTRL(tx_ring->index)); 2154 reg_val &= ~E1000_DCA_TXCTRL_TX_WB_RO_EN; 2155 E1000_WRITE_REG(hw, 2156 E1000_DCA_TXCTRL(tx_ring->index), reg_val); 2157 } else { 2158 tx_ring->tbd_head_wb = NULL; 2159 } 2160 2161 tx_ring->tbd_head = 0; 2162 tx_ring->tbd_tail = 0; 2163 tx_ring->tbd_free = tx_ring->ring_size; 2164 2165 if (igb->tx_ring_init == B_TRUE) { 2166 tx_ring->tcb_head = 0; 2167 tx_ring->tcb_tail = 0; 2168 tx_ring->tcb_free = tx_ring->free_list_size; 2169 } 2170 2171 /* 2172 * Enable TXDCTL per queue 2173 */ 2174 reg_val = E1000_READ_REG(hw, E1000_TXDCTL(tx_ring->index)); 2175 reg_val |= E1000_TXDCTL_QUEUE_ENABLE; 2176 E1000_WRITE_REG(hw, E1000_TXDCTL(tx_ring->index), reg_val); 2177 2178 /* 2179 * Initialize hardware checksum offload settings 2180 */ 2181 bzero(&tx_ring->tx_context, sizeof (tx_context_t)); 2182 } 2183 2184 static void 2185 igb_setup_tx(igb_t *igb) 2186 { 2187 igb_tx_ring_t *tx_ring; 2188 struct e1000_hw *hw = &igb->hw; 2189 uint32_t reg_val; 2190 int i; 2191 2192 for (i = 0; i < igb->num_tx_rings; i++) { 2193 tx_ring = &igb->tx_rings[i]; 2194 igb_setup_tx_ring(tx_ring); 2195 } 2196 2197 /* 2198 * Setup the Transmit Control Register (TCTL) 2199 */ 2200 reg_val = E1000_READ_REG(hw, E1000_TCTL); 2201 reg_val &= ~E1000_TCTL_CT; 2202 reg_val |= E1000_TCTL_PSP | E1000_TCTL_RTLC | 2203 (E1000_COLLISION_THRESHOLD << E1000_CT_SHIFT); 2204 2205 /* Enable transmits */ 2206 reg_val |= E1000_TCTL_EN; 2207 2208 E1000_WRITE_REG(hw, E1000_TCTL, reg_val); 2209 } 2210 2211 /* 2212 * igb_setup_rss - Setup receive-side scaling feature 2213 */ 2214 static void 2215 igb_setup_rss(igb_t *igb) 2216 { 2217 struct e1000_hw *hw = &igb->hw; 2218 uint32_t i, mrqc, rxcsum; 2219 int shift = 0; 2220 uint32_t random; 2221 union e1000_reta { 2222 uint32_t dword; 2223 uint8_t bytes[4]; 2224 } reta; 2225 2226 /* Setup the Redirection Table */ 2227 if (hw->mac.type == e1000_82576) { 2228 shift = 3; 2229 } else if (hw->mac.type == e1000_82575) { 2230 shift = 6; 2231 } 2232 for (i = 0; i < (32 * 4); i++) { 2233 reta.bytes[i & 3] = (i % igb->num_rx_rings) << shift; 2234 if ((i & 3) == 3) { 2235 E1000_WRITE_REG(hw, 2236 (E1000_RETA(0) + (i & ~3)), reta.dword); 2237 } 2238 } 2239 2240 /* Fill out hash function seeds */ 2241 for (i = 0; i < 10; i++) { 2242 (void) random_get_pseudo_bytes((uint8_t *)&random, 2243 sizeof (uint32_t)); 2244 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2245 } 2246 2247 /* Setup the Multiple Receive Queue Control register */ 2248 mrqc = E1000_MRQC_ENABLE_RSS_4Q; 2249 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2250 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2251 E1000_MRQC_RSS_FIELD_IPV6 | 2252 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2253 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2254 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2255 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2256 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2257 2258 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2259 2260 /* 2261 * Disable Packet Checksum to enable RSS for multiple receive queues. 2262 * 2263 * The Packet Checksum is not ethernet CRC. It is another kind of 2264 * checksum offloading provided by the 82575 chipset besides the IP 2265 * header checksum offloading and the TCP/UDP checksum offloading. 2266 * The Packet Checksum is by default computed over the entire packet 2267 * from the first byte of the DA through the last byte of the CRC, 2268 * including the Ethernet and IP headers. 2269 * 2270 * It is a hardware limitation that Packet Checksum is mutually 2271 * exclusive with RSS. 2272 */ 2273 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2274 rxcsum |= E1000_RXCSUM_PCSD; 2275 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2276 } 2277 2278 /* 2279 * igb_setup_mac_rss_classify - Setup MAC classification and rss 2280 */ 2281 static void 2282 igb_setup_mac_rss_classify(igb_t *igb) 2283 { 2284 struct e1000_hw *hw = &igb->hw; 2285 uint32_t i, mrqc, vmdctl, rxcsum; 2286 uint32_t ring_per_group; 2287 int shift_group0, shift_group1; 2288 uint32_t random; 2289 union e1000_reta { 2290 uint32_t dword; 2291 uint8_t bytes[4]; 2292 } reta; 2293 2294 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2295 2296 /* Setup the Redirection Table, it is shared between two groups */ 2297 shift_group0 = 2; 2298 shift_group1 = 6; 2299 for (i = 0; i < (32 * 4); i++) { 2300 reta.bytes[i & 3] = ((i % ring_per_group) << shift_group0) | 2301 ((ring_per_group + (i % ring_per_group)) << shift_group1); 2302 if ((i & 3) == 3) { 2303 E1000_WRITE_REG(hw, 2304 (E1000_RETA(0) + (i & ~3)), reta.dword); 2305 } 2306 } 2307 2308 /* Fill out hash function seeds */ 2309 for (i = 0; i < 10; i++) { 2310 (void) random_get_pseudo_bytes((uint8_t *)&random, 2311 sizeof (uint32_t)); 2312 E1000_WRITE_REG(hw, E1000_RSSRK(i), random); 2313 } 2314 2315 /* 2316 * Setup the Multiple Receive Queue Control register, 2317 * enable VMDq based on packet destination MAC address and RSS. 2318 */ 2319 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_RSS_GROUP; 2320 mrqc |= (E1000_MRQC_RSS_FIELD_IPV4 | 2321 E1000_MRQC_RSS_FIELD_IPV4_TCP | 2322 E1000_MRQC_RSS_FIELD_IPV6 | 2323 E1000_MRQC_RSS_FIELD_IPV6_TCP | 2324 E1000_MRQC_RSS_FIELD_IPV4_UDP | 2325 E1000_MRQC_RSS_FIELD_IPV6_UDP | 2326 E1000_MRQC_RSS_FIELD_IPV6_UDP_EX | 2327 E1000_MRQC_RSS_FIELD_IPV6_TCP_EX); 2328 2329 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2330 2331 2332 /* Define the default group and default queues */ 2333 vmdctl = E1000_VMDQ_MAC_GROUP_DEFAULT_QUEUE; 2334 E1000_WRITE_REG(hw, E1000_VT_CTL, vmdctl); 2335 2336 /* 2337 * Disable Packet Checksum to enable RSS for multiple receive queues. 2338 * 2339 * The Packet Checksum is not ethernet CRC. It is another kind of 2340 * checksum offloading provided by the 82575 chipset besides the IP 2341 * header checksum offloading and the TCP/UDP checksum offloading. 2342 * The Packet Checksum is by default computed over the entire packet 2343 * from the first byte of the DA through the last byte of the CRC, 2344 * including the Ethernet and IP headers. 2345 * 2346 * It is a hardware limitation that Packet Checksum is mutually 2347 * exclusive with RSS. 2348 */ 2349 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2350 rxcsum |= E1000_RXCSUM_PCSD; 2351 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2352 } 2353 2354 /* 2355 * igb_setup_mac_classify - Setup MAC classification feature 2356 */ 2357 static void 2358 igb_setup_mac_classify(igb_t *igb) 2359 { 2360 struct e1000_hw *hw = &igb->hw; 2361 uint32_t mrqc, rxcsum; 2362 2363 /* 2364 * Setup the Multiple Receive Queue Control register, 2365 * enable VMDq based on packet destination MAC address. 2366 */ 2367 mrqc = E1000_MRQC_ENABLE_VMDQ_MAC_GROUP; 2368 E1000_WRITE_REG(hw, E1000_MRQC, mrqc); 2369 2370 /* 2371 * Disable Packet Checksum to enable RSS for multiple receive queues. 2372 * 2373 * The Packet Checksum is not ethernet CRC. It is another kind of 2374 * checksum offloading provided by the 82575 chipset besides the IP 2375 * header checksum offloading and the TCP/UDP checksum offloading. 2376 * The Packet Checksum is by default computed over the entire packet 2377 * from the first byte of the DA through the last byte of the CRC, 2378 * including the Ethernet and IP headers. 2379 * 2380 * It is a hardware limitation that Packet Checksum is mutually 2381 * exclusive with RSS. 2382 */ 2383 rxcsum = E1000_READ_REG(hw, E1000_RXCSUM); 2384 rxcsum |= E1000_RXCSUM_PCSD; 2385 E1000_WRITE_REG(hw, E1000_RXCSUM, rxcsum); 2386 2387 } 2388 2389 /* 2390 * igb_init_unicst - Initialize the unicast addresses 2391 */ 2392 static void 2393 igb_init_unicst(igb_t *igb) 2394 { 2395 struct e1000_hw *hw = &igb->hw; 2396 int slot; 2397 2398 /* 2399 * Here we should consider two situations: 2400 * 2401 * 1. Chipset is initialized the first time 2402 * Initialize the multiple unicast addresses, and 2403 * save the default MAC address. 2404 * 2405 * 2. Chipset is reset 2406 * Recover the multiple unicast addresses from the 2407 * software data structure to the RAR registers. 2408 */ 2409 2410 /* 2411 * Clear the default MAC address in the RAR0 rgister, 2412 * which is loaded from EEPROM when system boot or chipreset, 2413 * this will cause the conficts with add_mac/rem_mac entry 2414 * points when VMDq is enabled. For this reason, the RAR0 2415 * must be cleared for both cases mentioned above. 2416 */ 2417 e1000_rar_clear(hw, 0); 2418 2419 if (!igb->unicst_init) { 2420 2421 /* Initialize the multiple unicast addresses */ 2422 igb->unicst_total = MAX_NUM_UNICAST_ADDRESSES; 2423 igb->unicst_avail = igb->unicst_total; 2424 2425 for (slot = 0; slot < igb->unicst_total; slot++) 2426 igb->unicst_addr[slot].mac.set = 0; 2427 2428 igb->unicst_init = B_TRUE; 2429 } else { 2430 /* Re-configure the RAR registers */ 2431 for (slot = 0; slot < igb->unicst_total; slot++) { 2432 e1000_rar_set_vmdq(hw, igb->unicst_addr[slot].mac.addr, 2433 slot, igb->vmdq_mode, 2434 igb->unicst_addr[slot].mac.group_index); 2435 } 2436 } 2437 } 2438 2439 /* 2440 * igb_unicst_find - Find the slot for the specified unicast address 2441 */ 2442 int 2443 igb_unicst_find(igb_t *igb, const uint8_t *mac_addr) 2444 { 2445 int slot; 2446 2447 ASSERT(mutex_owned(&igb->gen_lock)); 2448 2449 for (slot = 0; slot < igb->unicst_total; slot++) { 2450 if (bcmp(igb->unicst_addr[slot].mac.addr, 2451 mac_addr, ETHERADDRL) == 0) 2452 return (slot); 2453 } 2454 2455 return (-1); 2456 } 2457 2458 /* 2459 * igb_unicst_set - Set the unicast address to the specified slot 2460 */ 2461 int 2462 igb_unicst_set(igb_t *igb, const uint8_t *mac_addr, 2463 int slot) 2464 { 2465 struct e1000_hw *hw = &igb->hw; 2466 2467 ASSERT(mutex_owned(&igb->gen_lock)); 2468 2469 /* 2470 * Save the unicast address in the software data structure 2471 */ 2472 bcopy(mac_addr, igb->unicst_addr[slot].mac.addr, ETHERADDRL); 2473 2474 /* 2475 * Set the unicast address to the RAR register 2476 */ 2477 e1000_rar_set(hw, (uint8_t *)mac_addr, slot); 2478 2479 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2480 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2481 return (EIO); 2482 } 2483 2484 return (0); 2485 } 2486 2487 /* 2488 * igb_multicst_add - Add a multicst address 2489 */ 2490 int 2491 igb_multicst_add(igb_t *igb, const uint8_t *multiaddr) 2492 { 2493 struct ether_addr *new_table; 2494 size_t new_len; 2495 size_t old_len; 2496 2497 ASSERT(mutex_owned(&igb->gen_lock)); 2498 2499 if ((multiaddr[0] & 01) == 0) { 2500 igb_error(igb, "Illegal multicast address"); 2501 return (EINVAL); 2502 } 2503 2504 if (igb->mcast_count >= igb->mcast_max_num) { 2505 igb_error(igb, "Adapter requested more than %d mcast addresses", 2506 igb->mcast_max_num); 2507 return (ENOENT); 2508 } 2509 2510 if (igb->mcast_count == igb->mcast_alloc_count) { 2511 old_len = igb->mcast_alloc_count * 2512 sizeof (struct ether_addr); 2513 new_len = (igb->mcast_alloc_count + MCAST_ALLOC_COUNT) * 2514 sizeof (struct ether_addr); 2515 2516 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2517 if (new_table == NULL) { 2518 igb_error(igb, 2519 "Not enough memory to alloc mcast table"); 2520 return (ENOMEM); 2521 } 2522 2523 if (igb->mcast_table != NULL) { 2524 bcopy(igb->mcast_table, new_table, old_len); 2525 kmem_free(igb->mcast_table, old_len); 2526 } 2527 igb->mcast_alloc_count += MCAST_ALLOC_COUNT; 2528 igb->mcast_table = new_table; 2529 } 2530 2531 bcopy(multiaddr, 2532 &igb->mcast_table[igb->mcast_count], ETHERADDRL); 2533 igb->mcast_count++; 2534 2535 /* 2536 * Update the multicast table in the hardware 2537 */ 2538 igb_setup_multicst(igb); 2539 2540 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2541 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2542 return (EIO); 2543 } 2544 2545 return (0); 2546 } 2547 2548 /* 2549 * igb_multicst_remove - Remove a multicst address 2550 */ 2551 int 2552 igb_multicst_remove(igb_t *igb, const uint8_t *multiaddr) 2553 { 2554 struct ether_addr *new_table; 2555 size_t new_len; 2556 size_t old_len; 2557 int i; 2558 2559 ASSERT(mutex_owned(&igb->gen_lock)); 2560 2561 for (i = 0; i < igb->mcast_count; i++) { 2562 if (bcmp(multiaddr, &igb->mcast_table[i], 2563 ETHERADDRL) == 0) { 2564 for (i++; i < igb->mcast_count; i++) { 2565 igb->mcast_table[i - 1] = 2566 igb->mcast_table[i]; 2567 } 2568 igb->mcast_count--; 2569 break; 2570 } 2571 } 2572 2573 if ((igb->mcast_alloc_count - igb->mcast_count) > 2574 MCAST_ALLOC_COUNT) { 2575 old_len = igb->mcast_alloc_count * 2576 sizeof (struct ether_addr); 2577 new_len = (igb->mcast_alloc_count - MCAST_ALLOC_COUNT) * 2578 sizeof (struct ether_addr); 2579 2580 new_table = kmem_alloc(new_len, KM_NOSLEEP); 2581 if (new_table != NULL) { 2582 bcopy(igb->mcast_table, new_table, new_len); 2583 kmem_free(igb->mcast_table, old_len); 2584 igb->mcast_alloc_count -= MCAST_ALLOC_COUNT; 2585 igb->mcast_table = new_table; 2586 } 2587 } 2588 2589 /* 2590 * Update the multicast table in the hardware 2591 */ 2592 igb_setup_multicst(igb); 2593 2594 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 2595 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 2596 return (EIO); 2597 } 2598 2599 return (0); 2600 } 2601 2602 static void 2603 igb_release_multicast(igb_t *igb) 2604 { 2605 if (igb->mcast_table != NULL) { 2606 kmem_free(igb->mcast_table, 2607 igb->mcast_alloc_count * sizeof (struct ether_addr)); 2608 igb->mcast_table = NULL; 2609 } 2610 } 2611 2612 /* 2613 * igb_setup_multicast - setup multicast data structures 2614 * 2615 * This routine initializes all of the multicast related structures 2616 * and save them in the hardware registers. 2617 */ 2618 static void 2619 igb_setup_multicst(igb_t *igb) 2620 { 2621 uint8_t *mc_addr_list; 2622 uint32_t mc_addr_count; 2623 struct e1000_hw *hw = &igb->hw; 2624 2625 ASSERT(mutex_owned(&igb->gen_lock)); 2626 ASSERT(igb->mcast_count <= igb->mcast_max_num); 2627 2628 mc_addr_list = (uint8_t *)igb->mcast_table; 2629 mc_addr_count = igb->mcast_count; 2630 2631 /* 2632 * Update the multicase addresses to the MTA registers 2633 */ 2634 e1000_update_mc_addr_list(hw, mc_addr_list, mc_addr_count); 2635 } 2636 2637 /* 2638 * igb_get_conf - Get driver configurations set in driver.conf 2639 * 2640 * This routine gets user-configured values out of the configuration 2641 * file igb.conf. 2642 * 2643 * For each configurable value, there is a minimum, a maximum, and a 2644 * default. 2645 * If user does not configure a value, use the default. 2646 * If user configures below the minimum, use the minumum. 2647 * If user configures above the maximum, use the maxumum. 2648 */ 2649 static void 2650 igb_get_conf(igb_t *igb) 2651 { 2652 struct e1000_hw *hw = &igb->hw; 2653 uint32_t default_mtu; 2654 uint32_t flow_control; 2655 uint32_t ring_per_group; 2656 int i; 2657 2658 /* 2659 * igb driver supports the following user configurations: 2660 * 2661 * Link configurations: 2662 * adv_autoneg_cap 2663 * adv_1000fdx_cap 2664 * adv_100fdx_cap 2665 * adv_100hdx_cap 2666 * adv_10fdx_cap 2667 * adv_10hdx_cap 2668 * Note: 1000hdx is not supported. 2669 * 2670 * Jumbo frame configuration: 2671 * default_mtu 2672 * 2673 * Ethernet flow control configuration: 2674 * flow_control 2675 * 2676 * Multiple rings configurations: 2677 * tx_queue_number 2678 * tx_ring_size 2679 * rx_queue_number 2680 * rx_ring_size 2681 * 2682 * Call igb_get_prop() to get the value for a specific 2683 * configuration parameter. 2684 */ 2685 2686 /* 2687 * Link configurations 2688 */ 2689 igb->param_adv_autoneg_cap = igb_get_prop(igb, 2690 PROP_ADV_AUTONEG_CAP, 0, 1, 1); 2691 igb->param_adv_1000fdx_cap = igb_get_prop(igb, 2692 PROP_ADV_1000FDX_CAP, 0, 1, 1); 2693 igb->param_adv_100fdx_cap = igb_get_prop(igb, 2694 PROP_ADV_100FDX_CAP, 0, 1, 1); 2695 igb->param_adv_100hdx_cap = igb_get_prop(igb, 2696 PROP_ADV_100HDX_CAP, 0, 1, 1); 2697 igb->param_adv_10fdx_cap = igb_get_prop(igb, 2698 PROP_ADV_10FDX_CAP, 0, 1, 1); 2699 igb->param_adv_10hdx_cap = igb_get_prop(igb, 2700 PROP_ADV_10HDX_CAP, 0, 1, 1); 2701 2702 /* 2703 * Jumbo frame configurations 2704 */ 2705 default_mtu = igb_get_prop(igb, PROP_DEFAULT_MTU, 2706 MIN_MTU, MAX_MTU, DEFAULT_MTU); 2707 2708 igb->max_frame_size = default_mtu + 2709 sizeof (struct ether_vlan_header) + ETHERFCSL; 2710 2711 /* 2712 * Ethernet flow control configuration 2713 */ 2714 flow_control = igb_get_prop(igb, PROP_FLOW_CONTROL, 2715 e1000_fc_none, 4, e1000_fc_full); 2716 if (flow_control == 4) 2717 flow_control = e1000_fc_default; 2718 2719 hw->fc.requested_mode = flow_control; 2720 2721 /* 2722 * Multiple rings configurations 2723 */ 2724 igb->tx_ring_size = igb_get_prop(igb, PROP_TX_RING_SIZE, 2725 MIN_TX_RING_SIZE, MAX_TX_RING_SIZE, DEFAULT_TX_RING_SIZE); 2726 igb->rx_ring_size = igb_get_prop(igb, PROP_RX_RING_SIZE, 2727 MIN_RX_RING_SIZE, MAX_RX_RING_SIZE, DEFAULT_RX_RING_SIZE); 2728 2729 igb->mr_enable = igb_get_prop(igb, PROP_MR_ENABLE, 0, 1, 0); 2730 igb->num_rx_groups = igb_get_prop(igb, PROP_RX_GROUP_NUM, 2731 MIN_RX_GROUP_NUM, MAX_RX_GROUP_NUM, DEFAULT_RX_GROUP_NUM); 2732 /* 2733 * Currently we do not support VMDq for 82576 and 82580. 2734 * If it is e1000_82576, set num_rx_groups to 1. 2735 */ 2736 if (hw->mac.type >= e1000_82576) 2737 igb->num_rx_groups = 1; 2738 2739 if (igb->mr_enable) { 2740 igb->num_tx_rings = igb->capab->def_tx_que_num; 2741 igb->num_rx_rings = igb->capab->def_rx_que_num; 2742 } else { 2743 igb->num_tx_rings = 1; 2744 igb->num_rx_rings = 1; 2745 2746 if (igb->num_rx_groups > 1) { 2747 igb_error(igb, 2748 "Invalid rx groups number. Please enable multiple " 2749 "rings first"); 2750 igb->num_rx_groups = 1; 2751 } 2752 } 2753 2754 /* 2755 * Check the divisibility between rx rings and rx groups. 2756 */ 2757 for (i = igb->num_rx_groups; i > 0; i--) { 2758 if ((igb->num_rx_rings % i) == 0) 2759 break; 2760 } 2761 if (i != igb->num_rx_groups) { 2762 igb_error(igb, 2763 "Invalid rx groups number. Downgrade the rx group " 2764 "number to %d.", i); 2765 igb->num_rx_groups = i; 2766 } 2767 2768 /* 2769 * Get the ring number per group. 2770 */ 2771 ring_per_group = igb->num_rx_rings / igb->num_rx_groups; 2772 2773 if (igb->num_rx_groups == 1) { 2774 /* 2775 * One rx ring group, the rx ring number is num_rx_rings. 2776 */ 2777 igb->vmdq_mode = E1000_VMDQ_OFF; 2778 } else if (ring_per_group == 1) { 2779 /* 2780 * Multiple rx groups, each group has one rx ring. 2781 */ 2782 igb->vmdq_mode = E1000_VMDQ_MAC; 2783 } else { 2784 /* 2785 * Multiple groups and multiple rings. 2786 */ 2787 igb->vmdq_mode = E1000_VMDQ_MAC_RSS; 2788 } 2789 2790 /* 2791 * Tunable used to force an interrupt type. The only use is 2792 * for testing of the lesser interrupt types. 2793 * 0 = don't force interrupt type 2794 * 1 = force interrupt type MSIX 2795 * 2 = force interrupt type MSI 2796 * 3 = force interrupt type Legacy 2797 */ 2798 igb->intr_force = igb_get_prop(igb, PROP_INTR_FORCE, 2799 IGB_INTR_NONE, IGB_INTR_LEGACY, IGB_INTR_NONE); 2800 2801 igb->tx_hcksum_enable = igb_get_prop(igb, PROP_TX_HCKSUM_ENABLE, 2802 0, 1, 1); 2803 igb->rx_hcksum_enable = igb_get_prop(igb, PROP_RX_HCKSUM_ENABLE, 2804 0, 1, 1); 2805 igb->lso_enable = igb_get_prop(igb, PROP_LSO_ENABLE, 2806 0, 1, 1); 2807 igb->tx_head_wb_enable = igb_get_prop(igb, PROP_TX_HEAD_WB_ENABLE, 2808 0, 1, 1); 2809 2810 /* 2811 * igb LSO needs the tx h/w checksum support. 2812 * Here LSO will be disabled if tx h/w checksum has been disabled. 2813 */ 2814 if (igb->tx_hcksum_enable == B_FALSE) 2815 igb->lso_enable = B_FALSE; 2816 2817 igb->tx_copy_thresh = igb_get_prop(igb, PROP_TX_COPY_THRESHOLD, 2818 MIN_TX_COPY_THRESHOLD, MAX_TX_COPY_THRESHOLD, 2819 DEFAULT_TX_COPY_THRESHOLD); 2820 igb->tx_recycle_thresh = igb_get_prop(igb, PROP_TX_RECYCLE_THRESHOLD, 2821 MIN_TX_RECYCLE_THRESHOLD, MAX_TX_RECYCLE_THRESHOLD, 2822 DEFAULT_TX_RECYCLE_THRESHOLD); 2823 igb->tx_overload_thresh = igb_get_prop(igb, PROP_TX_OVERLOAD_THRESHOLD, 2824 MIN_TX_OVERLOAD_THRESHOLD, MAX_TX_OVERLOAD_THRESHOLD, 2825 DEFAULT_TX_OVERLOAD_THRESHOLD); 2826 igb->tx_resched_thresh = igb_get_prop(igb, PROP_TX_RESCHED_THRESHOLD, 2827 MIN_TX_RESCHED_THRESHOLD, MAX_TX_RESCHED_THRESHOLD, 2828 DEFAULT_TX_RESCHED_THRESHOLD); 2829 2830 igb->rx_copy_thresh = igb_get_prop(igb, PROP_RX_COPY_THRESHOLD, 2831 MIN_RX_COPY_THRESHOLD, MAX_RX_COPY_THRESHOLD, 2832 DEFAULT_RX_COPY_THRESHOLD); 2833 igb->rx_limit_per_intr = igb_get_prop(igb, PROP_RX_LIMIT_PER_INTR, 2834 MIN_RX_LIMIT_PER_INTR, MAX_RX_LIMIT_PER_INTR, 2835 DEFAULT_RX_LIMIT_PER_INTR); 2836 2837 igb->intr_throttling[0] = igb_get_prop(igb, PROP_INTR_THROTTLING, 2838 igb->capab->min_intr_throttle, 2839 igb->capab->max_intr_throttle, 2840 igb->capab->def_intr_throttle); 2841 2842 /* 2843 * Max number of multicast addresses 2844 */ 2845 igb->mcast_max_num = 2846 igb_get_prop(igb, PROP_MCAST_MAX_NUM, 2847 MIN_MCAST_NUM, MAX_MCAST_NUM, DEFAULT_MCAST_NUM); 2848 } 2849 2850 /* 2851 * igb_get_prop - Get a property value out of the configuration file igb.conf 2852 * 2853 * Caller provides the name of the property, a default value, a minimum 2854 * value, and a maximum value. 2855 * 2856 * Return configured value of the property, with default, minimum and 2857 * maximum properly applied. 2858 */ 2859 static int 2860 igb_get_prop(igb_t *igb, 2861 char *propname, /* name of the property */ 2862 int minval, /* minimum acceptable value */ 2863 int maxval, /* maximim acceptable value */ 2864 int defval) /* default value */ 2865 { 2866 int value; 2867 2868 /* 2869 * Call ddi_prop_get_int() to read the conf settings 2870 */ 2871 value = ddi_prop_get_int(DDI_DEV_T_ANY, igb->dip, 2872 DDI_PROP_DONTPASS, propname, defval); 2873 2874 if (value > maxval) 2875 value = maxval; 2876 2877 if (value < minval) 2878 value = minval; 2879 2880 return (value); 2881 } 2882 2883 /* 2884 * igb_setup_link - Using the link properties to setup the link 2885 */ 2886 int 2887 igb_setup_link(igb_t *igb, boolean_t setup_hw) 2888 { 2889 struct e1000_mac_info *mac; 2890 struct e1000_phy_info *phy; 2891 boolean_t invalid; 2892 2893 mac = &igb->hw.mac; 2894 phy = &igb->hw.phy; 2895 invalid = B_FALSE; 2896 2897 if (igb->param_adv_autoneg_cap == 1) { 2898 mac->autoneg = B_TRUE; 2899 phy->autoneg_advertised = 0; 2900 2901 /* 2902 * 1000hdx is not supported for autonegotiation 2903 */ 2904 if (igb->param_adv_1000fdx_cap == 1) 2905 phy->autoneg_advertised |= ADVERTISE_1000_FULL; 2906 2907 if (igb->param_adv_100fdx_cap == 1) 2908 phy->autoneg_advertised |= ADVERTISE_100_FULL; 2909 2910 if (igb->param_adv_100hdx_cap == 1) 2911 phy->autoneg_advertised |= ADVERTISE_100_HALF; 2912 2913 if (igb->param_adv_10fdx_cap == 1) 2914 phy->autoneg_advertised |= ADVERTISE_10_FULL; 2915 2916 if (igb->param_adv_10hdx_cap == 1) 2917 phy->autoneg_advertised |= ADVERTISE_10_HALF; 2918 2919 if (phy->autoneg_advertised == 0) 2920 invalid = B_TRUE; 2921 } else { 2922 mac->autoneg = B_FALSE; 2923 2924 /* 2925 * 1000fdx and 1000hdx are not supported for forced link 2926 */ 2927 if (igb->param_adv_100fdx_cap == 1) 2928 mac->forced_speed_duplex = ADVERTISE_100_FULL; 2929 else if (igb->param_adv_100hdx_cap == 1) 2930 mac->forced_speed_duplex = ADVERTISE_100_HALF; 2931 else if (igb->param_adv_10fdx_cap == 1) 2932 mac->forced_speed_duplex = ADVERTISE_10_FULL; 2933 else if (igb->param_adv_10hdx_cap == 1) 2934 mac->forced_speed_duplex = ADVERTISE_10_HALF; 2935 else 2936 invalid = B_TRUE; 2937 } 2938 2939 if (invalid) { 2940 igb_notice(igb, "Invalid link settings. Setup link to " 2941 "autonegotiation with full link capabilities."); 2942 mac->autoneg = B_TRUE; 2943 phy->autoneg_advertised = ADVERTISE_1000_FULL | 2944 ADVERTISE_100_FULL | ADVERTISE_100_HALF | 2945 ADVERTISE_10_FULL | ADVERTISE_10_HALF; 2946 } 2947 2948 if (setup_hw) { 2949 if (e1000_setup_link(&igb->hw) != E1000_SUCCESS) 2950 return (IGB_FAILURE); 2951 } 2952 2953 return (IGB_SUCCESS); 2954 } 2955 2956 2957 /* 2958 * igb_is_link_up - Check if the link is up 2959 */ 2960 static boolean_t 2961 igb_is_link_up(igb_t *igb) 2962 { 2963 struct e1000_hw *hw = &igb->hw; 2964 boolean_t link_up = B_FALSE; 2965 2966 ASSERT(mutex_owned(&igb->gen_lock)); 2967 2968 /* 2969 * get_link_status is set in the interrupt handler on link-status-change 2970 * or rx sequence error interrupt. get_link_status will stay 2971 * false until the e1000_check_for_link establishes link only 2972 * for copper adapters. 2973 */ 2974 switch (hw->phy.media_type) { 2975 case e1000_media_type_copper: 2976 if (hw->mac.get_link_status) { 2977 (void) e1000_check_for_link(hw); 2978 link_up = !hw->mac.get_link_status; 2979 } else { 2980 link_up = B_TRUE; 2981 } 2982 break; 2983 case e1000_media_type_fiber: 2984 (void) e1000_check_for_link(hw); 2985 link_up = (E1000_READ_REG(hw, E1000_STATUS) & E1000_STATUS_LU); 2986 break; 2987 case e1000_media_type_internal_serdes: 2988 (void) e1000_check_for_link(hw); 2989 link_up = hw->mac.serdes_has_link; 2990 break; 2991 } 2992 2993 return (link_up); 2994 } 2995 2996 /* 2997 * igb_link_check - Link status processing 2998 */ 2999 static boolean_t 3000 igb_link_check(igb_t *igb) 3001 { 3002 struct e1000_hw *hw = &igb->hw; 3003 uint16_t speed = 0, duplex = 0; 3004 boolean_t link_changed = B_FALSE; 3005 3006 ASSERT(mutex_owned(&igb->gen_lock)); 3007 3008 if (igb_is_link_up(igb)) { 3009 /* 3010 * The Link is up, check whether it was marked as down earlier 3011 */ 3012 if (igb->link_state != LINK_STATE_UP) { 3013 (void) e1000_get_speed_and_duplex(hw, &speed, &duplex); 3014 igb->link_speed = speed; 3015 igb->link_duplex = duplex; 3016 igb->link_state = LINK_STATE_UP; 3017 igb->link_down_timeout = 0; 3018 link_changed = B_TRUE; 3019 if (!igb->link_complete) 3020 igb_stop_link_timer(igb); 3021 } 3022 } else if (igb->link_complete) { 3023 if (igb->link_state != LINK_STATE_DOWN) { 3024 igb->link_speed = 0; 3025 igb->link_duplex = 0; 3026 igb->link_state = LINK_STATE_DOWN; 3027 link_changed = B_TRUE; 3028 } 3029 3030 if (igb->igb_state & IGB_STARTED) { 3031 if (igb->link_down_timeout < MAX_LINK_DOWN_TIMEOUT) { 3032 igb->link_down_timeout++; 3033 } else if (igb->link_down_timeout == 3034 MAX_LINK_DOWN_TIMEOUT) { 3035 igb_tx_clean(igb); 3036 igb->link_down_timeout++; 3037 } 3038 } 3039 } 3040 3041 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3042 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3043 return (B_FALSE); 3044 } 3045 3046 return (link_changed); 3047 } 3048 3049 /* 3050 * igb_local_timer - driver watchdog function 3051 * 3052 * This function will handle the hardware stall check, link status 3053 * check and other routines. 3054 */ 3055 static void 3056 igb_local_timer(void *arg) 3057 { 3058 igb_t *igb = (igb_t *)arg; 3059 boolean_t link_changed = B_FALSE; 3060 3061 if (igb->igb_state & IGB_ERROR) { 3062 igb->reset_count++; 3063 if (igb_reset(igb) == IGB_SUCCESS) 3064 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3065 3066 igb_restart_watchdog_timer(igb); 3067 return; 3068 } 3069 3070 if (igb_stall_check(igb) || (igb->igb_state & IGB_STALL)) { 3071 igb_fm_ereport(igb, DDI_FM_DEVICE_STALL); 3072 ddi_fm_service_impact(igb->dip, DDI_SERVICE_LOST); 3073 igb->reset_count++; 3074 if (igb_reset(igb) == IGB_SUCCESS) 3075 ddi_fm_service_impact(igb->dip, DDI_SERVICE_RESTORED); 3076 3077 igb_restart_watchdog_timer(igb); 3078 return; 3079 } 3080 3081 mutex_enter(&igb->gen_lock); 3082 if (!(igb->igb_state & IGB_SUSPENDED) && (igb->igb_state & IGB_STARTED)) 3083 link_changed = igb_link_check(igb); 3084 mutex_exit(&igb->gen_lock); 3085 3086 if (link_changed) 3087 mac_link_update(igb->mac_hdl, igb->link_state); 3088 3089 igb_restart_watchdog_timer(igb); 3090 } 3091 3092 /* 3093 * igb_link_timer - link setup timer function 3094 * 3095 * It is called when the timer for link setup is expired, which indicates 3096 * the completion of the link setup. The link state will not be updated 3097 * until the link setup is completed. And the link state will not be sent 3098 * to the upper layer through mac_link_update() in this function. It will 3099 * be updated in the local timer routine or the interrupts service routine 3100 * after the interface is started (plumbed). 3101 */ 3102 static void 3103 igb_link_timer(void *arg) 3104 { 3105 igb_t *igb = (igb_t *)arg; 3106 3107 mutex_enter(&igb->link_lock); 3108 igb->link_complete = B_TRUE; 3109 igb->link_tid = 0; 3110 mutex_exit(&igb->link_lock); 3111 } 3112 /* 3113 * igb_stall_check - check for transmit stall 3114 * 3115 * This function checks if the adapter is stalled (in transmit). 3116 * 3117 * It is called each time the watchdog timeout is invoked. 3118 * If the transmit descriptor reclaim continuously fails, 3119 * the watchdog value will increment by 1. If the watchdog 3120 * value exceeds the threshold, the igb is assumed to 3121 * have stalled and need to be reset. 3122 */ 3123 static boolean_t 3124 igb_stall_check(igb_t *igb) 3125 { 3126 igb_tx_ring_t *tx_ring; 3127 struct e1000_hw *hw = &igb->hw; 3128 boolean_t result; 3129 int i; 3130 3131 if (igb->link_state != LINK_STATE_UP) 3132 return (B_FALSE); 3133 3134 /* 3135 * If any tx ring is stalled, we'll reset the chipset 3136 */ 3137 result = B_FALSE; 3138 for (i = 0; i < igb->num_tx_rings; i++) { 3139 tx_ring = &igb->tx_rings[i]; 3140 3141 if (tx_ring->recycle_fail > 0) 3142 tx_ring->stall_watchdog++; 3143 else 3144 tx_ring->stall_watchdog = 0; 3145 3146 if (tx_ring->stall_watchdog >= STALL_WATCHDOG_TIMEOUT) { 3147 result = B_TRUE; 3148 if (hw->mac.type == e1000_82580) { 3149 hw->dev_spec._82575.global_device_reset 3150 = B_TRUE; 3151 } 3152 break; 3153 } 3154 } 3155 3156 if (result) { 3157 tx_ring->stall_watchdog = 0; 3158 tx_ring->recycle_fail = 0; 3159 } 3160 3161 return (result); 3162 } 3163 3164 3165 /* 3166 * is_valid_mac_addr - Check if the mac address is valid 3167 */ 3168 static boolean_t 3169 is_valid_mac_addr(uint8_t *mac_addr) 3170 { 3171 const uint8_t addr_test1[6] = { 0, 0, 0, 0, 0, 0 }; 3172 const uint8_t addr_test2[6] = 3173 { 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF }; 3174 3175 if (!(bcmp(addr_test1, mac_addr, ETHERADDRL)) || 3176 !(bcmp(addr_test2, mac_addr, ETHERADDRL))) 3177 return (B_FALSE); 3178 3179 return (B_TRUE); 3180 } 3181 3182 static boolean_t 3183 igb_find_mac_address(igb_t *igb) 3184 { 3185 struct e1000_hw *hw = &igb->hw; 3186 #ifdef __sparc 3187 uchar_t *bytes; 3188 struct ether_addr sysaddr; 3189 uint_t nelts; 3190 int err; 3191 boolean_t found = B_FALSE; 3192 3193 /* 3194 * The "vendor's factory-set address" may already have 3195 * been extracted from the chip, but if the property 3196 * "local-mac-address" is set we use that instead. 3197 * 3198 * We check whether it looks like an array of 6 3199 * bytes (which it should, if OBP set it). If we can't 3200 * make sense of it this way, we'll ignore it. 3201 */ 3202 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3203 DDI_PROP_DONTPASS, "local-mac-address", &bytes, &nelts); 3204 if (err == DDI_PROP_SUCCESS) { 3205 if (nelts == ETHERADDRL) { 3206 while (nelts--) 3207 hw->mac.addr[nelts] = bytes[nelts]; 3208 found = B_TRUE; 3209 } 3210 ddi_prop_free(bytes); 3211 } 3212 3213 /* 3214 * Look up the OBP property "local-mac-address?". If the user has set 3215 * 'local-mac-address? = false', use "the system address" instead. 3216 */ 3217 if (ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 0, 3218 "local-mac-address?", &bytes, &nelts) == DDI_PROP_SUCCESS) { 3219 if (strncmp("false", (caddr_t)bytes, (size_t)nelts) == 0) { 3220 if (localetheraddr(NULL, &sysaddr) != 0) { 3221 bcopy(&sysaddr, hw->mac.addr, ETHERADDRL); 3222 found = B_TRUE; 3223 } 3224 } 3225 ddi_prop_free(bytes); 3226 } 3227 3228 /* 3229 * Finally(!), if there's a valid "mac-address" property (created 3230 * if we netbooted from this interface), we must use this instead 3231 * of any of the above to ensure that the NFS/install server doesn't 3232 * get confused by the address changing as Solaris takes over! 3233 */ 3234 err = ddi_prop_lookup_byte_array(DDI_DEV_T_ANY, igb->dip, 3235 DDI_PROP_DONTPASS, "mac-address", &bytes, &nelts); 3236 if (err == DDI_PROP_SUCCESS) { 3237 if (nelts == ETHERADDRL) { 3238 while (nelts--) 3239 hw->mac.addr[nelts] = bytes[nelts]; 3240 found = B_TRUE; 3241 } 3242 ddi_prop_free(bytes); 3243 } 3244 3245 if (found) { 3246 bcopy(hw->mac.addr, hw->mac.perm_addr, ETHERADDRL); 3247 return (B_TRUE); 3248 } 3249 #endif 3250 3251 /* 3252 * Read the device MAC address from the EEPROM 3253 */ 3254 if (e1000_read_mac_addr(hw) != E1000_SUCCESS) 3255 return (B_FALSE); 3256 3257 return (B_TRUE); 3258 } 3259 3260 #pragma inline(igb_arm_watchdog_timer) 3261 3262 static void 3263 igb_arm_watchdog_timer(igb_t *igb) 3264 { 3265 /* 3266 * Fire a watchdog timer 3267 */ 3268 igb->watchdog_tid = 3269 timeout(igb_local_timer, 3270 (void *)igb, 1 * drv_usectohz(1000000)); 3271 3272 } 3273 3274 /* 3275 * igb_enable_watchdog_timer - Enable and start the driver watchdog timer 3276 */ 3277 void 3278 igb_enable_watchdog_timer(igb_t *igb) 3279 { 3280 mutex_enter(&igb->watchdog_lock); 3281 3282 if (!igb->watchdog_enable) { 3283 igb->watchdog_enable = B_TRUE; 3284 igb->watchdog_start = B_TRUE; 3285 igb_arm_watchdog_timer(igb); 3286 } 3287 3288 mutex_exit(&igb->watchdog_lock); 3289 3290 } 3291 3292 /* 3293 * igb_disable_watchdog_timer - Disable and stop the driver watchdog timer 3294 */ 3295 void 3296 igb_disable_watchdog_timer(igb_t *igb) 3297 { 3298 timeout_id_t tid; 3299 3300 mutex_enter(&igb->watchdog_lock); 3301 3302 igb->watchdog_enable = B_FALSE; 3303 igb->watchdog_start = B_FALSE; 3304 tid = igb->watchdog_tid; 3305 igb->watchdog_tid = 0; 3306 3307 mutex_exit(&igb->watchdog_lock); 3308 3309 if (tid != 0) 3310 (void) untimeout(tid); 3311 3312 } 3313 3314 /* 3315 * igb_start_watchdog_timer - Start the driver watchdog timer 3316 */ 3317 static void 3318 igb_start_watchdog_timer(igb_t *igb) 3319 { 3320 mutex_enter(&igb->watchdog_lock); 3321 3322 if (igb->watchdog_enable) { 3323 if (!igb->watchdog_start) { 3324 igb->watchdog_start = B_TRUE; 3325 igb_arm_watchdog_timer(igb); 3326 } 3327 } 3328 3329 mutex_exit(&igb->watchdog_lock); 3330 } 3331 3332 /* 3333 * igb_restart_watchdog_timer - Restart the driver watchdog timer 3334 */ 3335 static void 3336 igb_restart_watchdog_timer(igb_t *igb) 3337 { 3338 mutex_enter(&igb->watchdog_lock); 3339 3340 if (igb->watchdog_start) 3341 igb_arm_watchdog_timer(igb); 3342 3343 mutex_exit(&igb->watchdog_lock); 3344 } 3345 3346 /* 3347 * igb_stop_watchdog_timer - Stop the driver watchdog timer 3348 */ 3349 static void 3350 igb_stop_watchdog_timer(igb_t *igb) 3351 { 3352 timeout_id_t tid; 3353 3354 mutex_enter(&igb->watchdog_lock); 3355 3356 igb->watchdog_start = B_FALSE; 3357 tid = igb->watchdog_tid; 3358 igb->watchdog_tid = 0; 3359 3360 mutex_exit(&igb->watchdog_lock); 3361 3362 if (tid != 0) 3363 (void) untimeout(tid); 3364 } 3365 3366 /* 3367 * igb_start_link_timer - Start the link setup timer 3368 */ 3369 static void 3370 igb_start_link_timer(struct igb *igb) 3371 { 3372 struct e1000_hw *hw = &igb->hw; 3373 clock_t link_timeout; 3374 3375 if (hw->mac.autoneg) 3376 link_timeout = PHY_AUTO_NEG_LIMIT * 3377 drv_usectohz(100000); 3378 else 3379 link_timeout = PHY_FORCE_LIMIT * drv_usectohz(100000); 3380 3381 mutex_enter(&igb->link_lock); 3382 if (hw->phy.autoneg_wait_to_complete) { 3383 igb->link_complete = B_TRUE; 3384 } else { 3385 igb->link_complete = B_FALSE; 3386 igb->link_tid = timeout(igb_link_timer, (void *)igb, 3387 link_timeout); 3388 } 3389 mutex_exit(&igb->link_lock); 3390 } 3391 3392 /* 3393 * igb_stop_link_timer - Stop the link setup timer 3394 */ 3395 static void 3396 igb_stop_link_timer(struct igb *igb) 3397 { 3398 timeout_id_t tid; 3399 3400 mutex_enter(&igb->link_lock); 3401 igb->link_complete = B_TRUE; 3402 tid = igb->link_tid; 3403 igb->link_tid = 0; 3404 mutex_exit(&igb->link_lock); 3405 3406 if (tid != 0) 3407 (void) untimeout(tid); 3408 } 3409 3410 /* 3411 * igb_disable_adapter_interrupts - Clear/disable all hardware interrupts 3412 */ 3413 static void 3414 igb_disable_adapter_interrupts(igb_t *igb) 3415 { 3416 struct e1000_hw *hw = &igb->hw; 3417 3418 /* 3419 * Set the IMC register to mask all the interrupts, 3420 * including the tx interrupts. 3421 */ 3422 E1000_WRITE_REG(hw, E1000_IMC, ~0); 3423 E1000_WRITE_REG(hw, E1000_IAM, 0); 3424 3425 /* 3426 * Additional disabling for MSI-X 3427 */ 3428 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3429 E1000_WRITE_REG(hw, E1000_EIMC, ~0); 3430 E1000_WRITE_REG(hw, E1000_EIAC, 0); 3431 E1000_WRITE_REG(hw, E1000_EIAM, 0); 3432 } 3433 3434 E1000_WRITE_FLUSH(hw); 3435 } 3436 3437 /* 3438 * igb_enable_adapter_interrupts_82580 - Enable NIC interrupts for 82580 3439 */ 3440 static void 3441 igb_enable_adapter_interrupts_82580(igb_t *igb) 3442 { 3443 struct e1000_hw *hw = &igb->hw; 3444 3445 /* Clear any pending interrupts */ 3446 (void) E1000_READ_REG(hw, E1000_ICR); 3447 igb->ims_mask |= E1000_IMS_DRSTA; 3448 3449 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3450 3451 /* Interrupt enabling for MSI-X */ 3452 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3453 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3454 igb->ims_mask = (E1000_IMS_LSC | E1000_IMS_DRSTA); 3455 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3456 } else { /* Interrupt enabling for MSI and legacy */ 3457 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3458 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3459 igb->ims_mask |= E1000_IMS_DRSTA; 3460 E1000_WRITE_REG(hw, E1000_IMS, igb->ims_mask); 3461 } 3462 3463 /* Disable auto-mask for ICR interrupt bits */ 3464 E1000_WRITE_REG(hw, E1000_IAM, 0); 3465 3466 E1000_WRITE_FLUSH(hw); 3467 } 3468 3469 /* 3470 * igb_enable_adapter_interrupts_82576 - Enable NIC interrupts for 82576 3471 */ 3472 static void 3473 igb_enable_adapter_interrupts_82576(igb_t *igb) 3474 { 3475 struct e1000_hw *hw = &igb->hw; 3476 3477 /* Clear any pending interrupts */ 3478 (void) E1000_READ_REG(hw, E1000_ICR); 3479 3480 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3481 3482 /* Interrupt enabling for MSI-X */ 3483 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3484 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3485 igb->ims_mask = E1000_IMS_LSC; 3486 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3487 } else { 3488 /* Interrupt enabling for MSI and legacy */ 3489 E1000_WRITE_REG(hw, E1000_IVAR0, E1000_IVAR_VALID); 3490 igb->ims_mask = IMS_ENABLE_MASK | E1000_IMS_TXQE; 3491 E1000_WRITE_REG(hw, E1000_IMS, 3492 (IMS_ENABLE_MASK | E1000_IMS_TXQE)); 3493 } 3494 3495 /* Disable auto-mask for ICR interrupt bits */ 3496 E1000_WRITE_REG(hw, E1000_IAM, 0); 3497 3498 E1000_WRITE_FLUSH(hw); 3499 } 3500 3501 /* 3502 * igb_enable_adapter_interrupts_82575 - Enable NIC interrupts for 82575 3503 */ 3504 static void 3505 igb_enable_adapter_interrupts_82575(igb_t *igb) 3506 { 3507 struct e1000_hw *hw = &igb->hw; 3508 uint32_t reg; 3509 3510 /* Clear any pending interrupts */ 3511 (void) E1000_READ_REG(hw, E1000_ICR); 3512 3513 if (igb->intr_type == DDI_INTR_TYPE_MSIX) { 3514 /* Interrupt enabling for MSI-X */ 3515 E1000_WRITE_REG(hw, E1000_EIMS, igb->eims_mask); 3516 E1000_WRITE_REG(hw, E1000_EIAC, igb->eims_mask); 3517 igb->ims_mask = E1000_IMS_LSC; 3518 E1000_WRITE_REG(hw, E1000_IMS, E1000_IMS_LSC); 3519 3520 /* Enable MSI-X PBA support */ 3521 reg = E1000_READ_REG(hw, E1000_CTRL_EXT); 3522 reg |= E1000_CTRL_EXT_PBA_CLR; 3523 3524 /* Non-selective interrupt clear-on-read */ 3525 reg |= E1000_CTRL_EXT_IRCA; /* Called NSICR in the EAS */ 3526 3527 E1000_WRITE_REG(hw, E1000_CTRL_EXT, reg); 3528 } else { 3529 /* Interrupt enabling for MSI and legacy */ 3530 igb->ims_mask = IMS_ENABLE_MASK; 3531 E1000_WRITE_REG(hw, E1000_IMS, IMS_ENABLE_MASK); 3532 } 3533 3534 E1000_WRITE_FLUSH(hw); 3535 } 3536 3537 /* 3538 * Loopback Support 3539 */ 3540 static lb_property_t lb_normal = 3541 { normal, "normal", IGB_LB_NONE }; 3542 static lb_property_t lb_external = 3543 { external, "External", IGB_LB_EXTERNAL }; 3544 static lb_property_t lb_mac = 3545 { internal, "MAC", IGB_LB_INTERNAL_MAC }; 3546 static lb_property_t lb_phy = 3547 { internal, "PHY", IGB_LB_INTERNAL_PHY }; 3548 static lb_property_t lb_serdes = 3549 { internal, "SerDes", IGB_LB_INTERNAL_SERDES }; 3550 3551 enum ioc_reply 3552 igb_loopback_ioctl(igb_t *igb, struct iocblk *iocp, mblk_t *mp) 3553 { 3554 lb_info_sz_t *lbsp; 3555 lb_property_t *lbpp; 3556 struct e1000_hw *hw; 3557 uint32_t *lbmp; 3558 uint32_t size; 3559 uint32_t value; 3560 3561 hw = &igb->hw; 3562 3563 if (mp->b_cont == NULL) 3564 return (IOC_INVAL); 3565 3566 switch (iocp->ioc_cmd) { 3567 default: 3568 return (IOC_INVAL); 3569 3570 case LB_GET_INFO_SIZE: 3571 size = sizeof (lb_info_sz_t); 3572 if (iocp->ioc_count != size) 3573 return (IOC_INVAL); 3574 3575 value = sizeof (lb_normal); 3576 value += sizeof (lb_mac); 3577 if (hw->phy.media_type == e1000_media_type_copper) 3578 value += sizeof (lb_phy); 3579 else 3580 value += sizeof (lb_serdes); 3581 value += sizeof (lb_external); 3582 3583 lbsp = (lb_info_sz_t *)(uintptr_t)mp->b_cont->b_rptr; 3584 *lbsp = value; 3585 break; 3586 3587 case LB_GET_INFO: 3588 value = sizeof (lb_normal); 3589 value += sizeof (lb_mac); 3590 if (hw->phy.media_type == e1000_media_type_copper) 3591 value += sizeof (lb_phy); 3592 else 3593 value += sizeof (lb_serdes); 3594 value += sizeof (lb_external); 3595 3596 size = value; 3597 if (iocp->ioc_count != size) 3598 return (IOC_INVAL); 3599 3600 value = 0; 3601 lbpp = (lb_property_t *)(uintptr_t)mp->b_cont->b_rptr; 3602 3603 lbpp[value++] = lb_normal; 3604 lbpp[value++] = lb_mac; 3605 if (hw->phy.media_type == e1000_media_type_copper) 3606 lbpp[value++] = lb_phy; 3607 else 3608 lbpp[value++] = lb_serdes; 3609 lbpp[value++] = lb_external; 3610 break; 3611 3612 case LB_GET_MODE: 3613 size = sizeof (uint32_t); 3614 if (iocp->ioc_count != size) 3615 return (IOC_INVAL); 3616 3617 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3618 *lbmp = igb->loopback_mode; 3619 break; 3620 3621 case LB_SET_MODE: 3622 size = 0; 3623 if (iocp->ioc_count != sizeof (uint32_t)) 3624 return (IOC_INVAL); 3625 3626 lbmp = (uint32_t *)(uintptr_t)mp->b_cont->b_rptr; 3627 if (!igb_set_loopback_mode(igb, *lbmp)) 3628 return (IOC_INVAL); 3629 break; 3630 } 3631 3632 iocp->ioc_count = size; 3633 iocp->ioc_error = 0; 3634 3635 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3636 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3637 return (IOC_INVAL); 3638 } 3639 3640 return (IOC_REPLY); 3641 } 3642 3643 /* 3644 * igb_set_loopback_mode - Setup loopback based on the loopback mode 3645 */ 3646 static boolean_t 3647 igb_set_loopback_mode(igb_t *igb, uint32_t mode) 3648 { 3649 struct e1000_hw *hw; 3650 int i; 3651 3652 if (mode == igb->loopback_mode) 3653 return (B_TRUE); 3654 3655 hw = &igb->hw; 3656 3657 igb->loopback_mode = mode; 3658 3659 if (mode == IGB_LB_NONE) { 3660 /* Reset the chip */ 3661 hw->phy.autoneg_wait_to_complete = B_TRUE; 3662 (void) igb_reset(igb); 3663 hw->phy.autoneg_wait_to_complete = B_FALSE; 3664 return (B_TRUE); 3665 } 3666 3667 mutex_enter(&igb->gen_lock); 3668 3669 switch (mode) { 3670 default: 3671 mutex_exit(&igb->gen_lock); 3672 return (B_FALSE); 3673 3674 case IGB_LB_EXTERNAL: 3675 igb_set_external_loopback(igb); 3676 break; 3677 3678 case IGB_LB_INTERNAL_MAC: 3679 igb_set_internal_mac_loopback(igb); 3680 break; 3681 3682 case IGB_LB_INTERNAL_PHY: 3683 igb_set_internal_phy_loopback(igb); 3684 break; 3685 3686 case IGB_LB_INTERNAL_SERDES: 3687 igb_set_internal_serdes_loopback(igb); 3688 break; 3689 } 3690 3691 mutex_exit(&igb->gen_lock); 3692 3693 /* 3694 * When external loopback is set, wait up to 1000ms to get the link up. 3695 * According to test, 1000ms can work and it's an experimental value. 3696 */ 3697 if (mode == IGB_LB_EXTERNAL) { 3698 for (i = 0; i <= 10; i++) { 3699 mutex_enter(&igb->gen_lock); 3700 (void) igb_link_check(igb); 3701 mutex_exit(&igb->gen_lock); 3702 3703 if (igb->link_state == LINK_STATE_UP) 3704 break; 3705 3706 msec_delay(100); 3707 } 3708 3709 if (igb->link_state != LINK_STATE_UP) { 3710 /* 3711 * Does not support external loopback. 3712 * Reset driver to loopback none. 3713 */ 3714 igb->loopback_mode = IGB_LB_NONE; 3715 3716 /* Reset the chip */ 3717 hw->phy.autoneg_wait_to_complete = B_TRUE; 3718 (void) igb_reset(igb); 3719 hw->phy.autoneg_wait_to_complete = B_FALSE; 3720 3721 IGB_DEBUGLOG_0(igb, "Set external loopback failed, " 3722 "reset to loopback none."); 3723 3724 return (B_FALSE); 3725 } 3726 } 3727 3728 return (B_TRUE); 3729 } 3730 3731 /* 3732 * igb_set_external_loopback - Set the external loopback mode 3733 */ 3734 static void 3735 igb_set_external_loopback(igb_t *igb) 3736 { 3737 struct e1000_hw *hw; 3738 3739 hw = &igb->hw; 3740 3741 /* Set phy to known state */ 3742 (void) e1000_phy_hw_reset(hw); 3743 3744 (void) e1000_write_phy_reg(hw, 0x0, 0x0140); 3745 (void) e1000_write_phy_reg(hw, 0x9, 0x1b00); 3746 (void) e1000_write_phy_reg(hw, 0x12, 0x1610); 3747 (void) e1000_write_phy_reg(hw, 0x1f37, 0x3f1c); 3748 } 3749 3750 /* 3751 * igb_set_internal_mac_loopback - Set the internal MAC loopback mode 3752 */ 3753 static void 3754 igb_set_internal_mac_loopback(igb_t *igb) 3755 { 3756 struct e1000_hw *hw; 3757 uint32_t ctrl; 3758 uint32_t rctl; 3759 uint32_t ctrl_ext; 3760 uint16_t phy_ctrl; 3761 uint16_t phy_status; 3762 3763 hw = &igb->hw; 3764 3765 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 3766 phy_ctrl &= ~MII_CR_AUTO_NEG_EN; 3767 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3768 3769 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 3770 3771 /* Set link mode to PHY (00b) in the Extended Control register */ 3772 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3773 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3774 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3775 3776 /* Set the Device Control register */ 3777 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3778 if (!(phy_status & MII_SR_LINK_STATUS)) 3779 ctrl |= E1000_CTRL_ILOS; /* Set ILOS when the link is down */ 3780 ctrl &= ~E1000_CTRL_SPD_SEL; /* Clear the speed sel bits */ 3781 ctrl |= (E1000_CTRL_SLU | /* Force link up */ 3782 E1000_CTRL_FRCSPD | /* Force speed */ 3783 E1000_CTRL_FRCDPX | /* Force duplex */ 3784 E1000_CTRL_SPD_1000 | /* Force speed to 1000 */ 3785 E1000_CTRL_FD); /* Force full duplex */ 3786 3787 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3788 3789 /* Set the Receive Control register */ 3790 rctl = E1000_READ_REG(hw, E1000_RCTL); 3791 rctl &= ~E1000_RCTL_LBM_TCVR; 3792 rctl |= E1000_RCTL_LBM_MAC; 3793 E1000_WRITE_REG(hw, E1000_RCTL, rctl); 3794 } 3795 3796 /* 3797 * igb_set_internal_phy_loopback - Set the internal PHY loopback mode 3798 */ 3799 static void 3800 igb_set_internal_phy_loopback(igb_t *igb) 3801 { 3802 struct e1000_hw *hw; 3803 uint32_t ctrl_ext; 3804 uint16_t phy_ctrl; 3805 uint16_t phy_pconf; 3806 3807 hw = &igb->hw; 3808 3809 /* Set link mode to PHY (00b) in the Extended Control register */ 3810 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3811 ctrl_ext &= ~E1000_CTRL_EXT_LINK_MODE_MASK; 3812 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3813 3814 /* 3815 * Set PHY control register (0x4140): 3816 * Set full duplex mode 3817 * Set loopback bit 3818 * Clear auto-neg enable bit 3819 * Set PHY speed 3820 */ 3821 phy_ctrl = MII_CR_FULL_DUPLEX | MII_CR_SPEED_1000 | MII_CR_LOOPBACK; 3822 (void) e1000_write_phy_reg(hw, PHY_CONTROL, phy_ctrl); 3823 3824 /* Set the link disable bit in the Port Configuration register */ 3825 (void) e1000_read_phy_reg(hw, 0x10, &phy_pconf); 3826 phy_pconf |= (uint16_t)1 << 14; 3827 (void) e1000_write_phy_reg(hw, 0x10, phy_pconf); 3828 } 3829 3830 /* 3831 * igb_set_internal_serdes_loopback - Set the internal SerDes loopback mode 3832 */ 3833 static void 3834 igb_set_internal_serdes_loopback(igb_t *igb) 3835 { 3836 struct e1000_hw *hw; 3837 uint32_t ctrl_ext; 3838 uint32_t ctrl; 3839 uint32_t pcs_lctl; 3840 uint32_t connsw; 3841 3842 hw = &igb->hw; 3843 3844 /* Set link mode to SerDes (11b) in the Extended Control register */ 3845 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 3846 ctrl_ext |= E1000_CTRL_EXT_LINK_MODE_PCIE_SERDES; 3847 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 3848 3849 /* Configure the SerDes to loopback */ 3850 E1000_WRITE_REG(hw, E1000_SCTL, 0x410); 3851 3852 /* Set Device Control register */ 3853 ctrl = E1000_READ_REG(hw, E1000_CTRL); 3854 ctrl |= (E1000_CTRL_FD | /* Force full duplex */ 3855 E1000_CTRL_SLU); /* Force link up */ 3856 ctrl &= ~(E1000_CTRL_RFCE | /* Disable receive flow control */ 3857 E1000_CTRL_TFCE | /* Disable transmit flow control */ 3858 E1000_CTRL_LRST); /* Clear link reset */ 3859 E1000_WRITE_REG(hw, E1000_CTRL, ctrl); 3860 3861 /* Set PCS Link Control register */ 3862 pcs_lctl = E1000_READ_REG(hw, E1000_PCS_LCTL); 3863 pcs_lctl |= (E1000_PCS_LCTL_FORCE_LINK | 3864 E1000_PCS_LCTL_FSD | 3865 E1000_PCS_LCTL_FDV_FULL | 3866 E1000_PCS_LCTL_FLV_LINK_UP); 3867 pcs_lctl &= ~E1000_PCS_LCTL_AN_ENABLE; 3868 E1000_WRITE_REG(hw, E1000_PCS_LCTL, pcs_lctl); 3869 3870 /* Set the Copper/Fiber Switch Control - CONNSW register */ 3871 connsw = E1000_READ_REG(hw, E1000_CONNSW); 3872 connsw &= ~E1000_CONNSW_ENRGSRC; 3873 E1000_WRITE_REG(hw, E1000_CONNSW, connsw); 3874 } 3875 3876 #pragma inline(igb_intr_rx_work) 3877 /* 3878 * igb_intr_rx_work - rx processing of ISR 3879 */ 3880 static void 3881 igb_intr_rx_work(igb_rx_ring_t *rx_ring) 3882 { 3883 mblk_t *mp; 3884 3885 mutex_enter(&rx_ring->rx_lock); 3886 mp = igb_rx(rx_ring, IGB_NO_POLL); 3887 mutex_exit(&rx_ring->rx_lock); 3888 3889 if (mp != NULL) 3890 mac_rx_ring(rx_ring->igb->mac_hdl, rx_ring->ring_handle, mp, 3891 rx_ring->ring_gen_num); 3892 } 3893 3894 #pragma inline(igb_intr_tx_work) 3895 /* 3896 * igb_intr_tx_work - tx processing of ISR 3897 */ 3898 static void 3899 igb_intr_tx_work(igb_tx_ring_t *tx_ring) 3900 { 3901 igb_t *igb = tx_ring->igb; 3902 3903 /* Recycle the tx descriptors */ 3904 tx_ring->tx_recycle(tx_ring); 3905 3906 /* Schedule the re-transmit */ 3907 if (tx_ring->reschedule && 3908 (tx_ring->tbd_free >= igb->tx_resched_thresh)) { 3909 tx_ring->reschedule = B_FALSE; 3910 mac_tx_ring_update(tx_ring->igb->mac_hdl, tx_ring->ring_handle); 3911 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 3912 } 3913 } 3914 3915 #pragma inline(igb_intr_link_work) 3916 /* 3917 * igb_intr_link_work - link-status-change processing of ISR 3918 */ 3919 static void 3920 igb_intr_link_work(igb_t *igb) 3921 { 3922 boolean_t link_changed; 3923 3924 igb_stop_watchdog_timer(igb); 3925 3926 mutex_enter(&igb->gen_lock); 3927 3928 /* 3929 * Because we got a link-status-change interrupt, force 3930 * e1000_check_for_link() to look at phy 3931 */ 3932 igb->hw.mac.get_link_status = B_TRUE; 3933 3934 /* igb_link_check takes care of link status change */ 3935 link_changed = igb_link_check(igb); 3936 3937 /* Get new phy state */ 3938 igb_get_phy_state(igb); 3939 3940 mutex_exit(&igb->gen_lock); 3941 3942 if (link_changed) 3943 mac_link_update(igb->mac_hdl, igb->link_state); 3944 3945 igb_start_watchdog_timer(igb); 3946 } 3947 3948 /* 3949 * igb_intr_legacy - Interrupt handler for legacy interrupts 3950 */ 3951 static uint_t 3952 igb_intr_legacy(void *arg1, void *arg2) 3953 { 3954 igb_t *igb = (igb_t *)arg1; 3955 igb_tx_ring_t *tx_ring; 3956 uint32_t icr; 3957 mblk_t *mp; 3958 boolean_t tx_reschedule; 3959 boolean_t link_changed; 3960 uint_t result; 3961 3962 _NOTE(ARGUNUSED(arg2)); 3963 3964 mutex_enter(&igb->gen_lock); 3965 3966 if (igb->igb_state & IGB_SUSPENDED) { 3967 mutex_exit(&igb->gen_lock); 3968 return (DDI_INTR_UNCLAIMED); 3969 } 3970 3971 mp = NULL; 3972 tx_reschedule = B_FALSE; 3973 link_changed = B_FALSE; 3974 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 3975 3976 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 3977 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 3978 atomic_or_32(&igb->igb_state, IGB_ERROR); 3979 return (DDI_INTR_UNCLAIMED); 3980 } 3981 3982 if (icr & E1000_ICR_INT_ASSERTED) { 3983 /* 3984 * E1000_ICR_INT_ASSERTED bit was set: 3985 * Read(Clear) the ICR, claim this interrupt, 3986 * look for work to do. 3987 */ 3988 ASSERT(igb->num_rx_rings == 1); 3989 ASSERT(igb->num_tx_rings == 1); 3990 3991 /* Make sure all interrupt causes cleared */ 3992 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 3993 3994 if (icr & E1000_ICR_RXT0) { 3995 mp = igb_rx(&igb->rx_rings[0], IGB_NO_POLL); 3996 } 3997 3998 if (icr & E1000_ICR_TXDW) { 3999 tx_ring = &igb->tx_rings[0]; 4000 4001 /* Recycle the tx descriptors */ 4002 tx_ring->tx_recycle(tx_ring); 4003 4004 /* Schedule the re-transmit */ 4005 tx_reschedule = (tx_ring->reschedule && 4006 (tx_ring->tbd_free >= igb->tx_resched_thresh)); 4007 } 4008 4009 if (icr & E1000_ICR_LSC) { 4010 /* 4011 * Because we got a link-status-change interrupt, force 4012 * e1000_check_for_link() to look at phy 4013 */ 4014 igb->hw.mac.get_link_status = B_TRUE; 4015 4016 /* igb_link_check takes care of link status change */ 4017 link_changed = igb_link_check(igb); 4018 4019 /* Get new phy state */ 4020 igb_get_phy_state(igb); 4021 } 4022 4023 if (icr & E1000_ICR_DRSTA) { 4024 /* 82580 Full Device Reset needed */ 4025 atomic_or_32(&igb->igb_state, IGB_STALL); 4026 } 4027 4028 result = DDI_INTR_CLAIMED; 4029 } else { 4030 /* 4031 * E1000_ICR_INT_ASSERTED bit was not set: 4032 * Don't claim this interrupt. 4033 */ 4034 result = DDI_INTR_UNCLAIMED; 4035 } 4036 4037 mutex_exit(&igb->gen_lock); 4038 4039 /* 4040 * Do the following work outside of the gen_lock 4041 */ 4042 if (mp != NULL) 4043 mac_rx(igb->mac_hdl, NULL, mp); 4044 4045 if (tx_reschedule) { 4046 tx_ring->reschedule = B_FALSE; 4047 mac_tx_ring_update(igb->mac_hdl, tx_ring->ring_handle); 4048 IGB_DEBUG_STAT(tx_ring->stat_reschedule); 4049 } 4050 4051 if (link_changed) 4052 mac_link_update(igb->mac_hdl, igb->link_state); 4053 4054 return (result); 4055 } 4056 4057 /* 4058 * igb_intr_msi - Interrupt handler for MSI 4059 */ 4060 static uint_t 4061 igb_intr_msi(void *arg1, void *arg2) 4062 { 4063 igb_t *igb = (igb_t *)arg1; 4064 uint32_t icr; 4065 4066 _NOTE(ARGUNUSED(arg2)); 4067 4068 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4069 4070 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4071 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4072 atomic_or_32(&igb->igb_state, IGB_ERROR); 4073 return (DDI_INTR_CLAIMED); 4074 } 4075 4076 /* Make sure all interrupt causes cleared */ 4077 (void) E1000_READ_REG(&igb->hw, E1000_EICR); 4078 4079 /* 4080 * For MSI interrupt, we have only one vector, 4081 * so we have only one rx ring and one tx ring enabled. 4082 */ 4083 ASSERT(igb->num_rx_rings == 1); 4084 ASSERT(igb->num_tx_rings == 1); 4085 4086 if (icr & E1000_ICR_RXT0) { 4087 igb_intr_rx_work(&igb->rx_rings[0]); 4088 } 4089 4090 if (icr & E1000_ICR_TXDW) { 4091 igb_intr_tx_work(&igb->tx_rings[0]); 4092 } 4093 4094 if (icr & E1000_ICR_LSC) { 4095 igb_intr_link_work(igb); 4096 } 4097 4098 if (icr & E1000_ICR_DRSTA) { 4099 /* 82580 Full Device Reset needed */ 4100 atomic_or_32(&igb->igb_state, IGB_STALL); 4101 } 4102 4103 return (DDI_INTR_CLAIMED); 4104 } 4105 4106 /* 4107 * igb_intr_rx - Interrupt handler for rx 4108 */ 4109 static uint_t 4110 igb_intr_rx(void *arg1, void *arg2) 4111 { 4112 igb_rx_ring_t *rx_ring = (igb_rx_ring_t *)arg1; 4113 4114 _NOTE(ARGUNUSED(arg2)); 4115 4116 /* 4117 * Only used via MSI-X vector so don't check cause bits 4118 * and only clean the given ring. 4119 */ 4120 igb_intr_rx_work(rx_ring); 4121 4122 return (DDI_INTR_CLAIMED); 4123 } 4124 4125 /* 4126 * igb_intr_tx - Interrupt handler for tx 4127 */ 4128 static uint_t 4129 igb_intr_tx(void *arg1, void *arg2) 4130 { 4131 igb_tx_ring_t *tx_ring = (igb_tx_ring_t *)arg1; 4132 4133 _NOTE(ARGUNUSED(arg2)); 4134 4135 /* 4136 * Only used via MSI-X vector so don't check cause bits 4137 * and only clean the given ring. 4138 */ 4139 igb_intr_tx_work(tx_ring); 4140 4141 return (DDI_INTR_CLAIMED); 4142 } 4143 4144 /* 4145 * igb_intr_tx_other - Interrupt handler for both tx and other 4146 * 4147 */ 4148 static uint_t 4149 igb_intr_tx_other(void *arg1, void *arg2) 4150 { 4151 igb_t *igb = (igb_t *)arg1; 4152 uint32_t icr; 4153 4154 _NOTE(ARGUNUSED(arg2)); 4155 4156 icr = E1000_READ_REG(&igb->hw, E1000_ICR); 4157 4158 if (igb_check_acc_handle(igb->osdep.reg_handle) != DDI_FM_OK) { 4159 ddi_fm_service_impact(igb->dip, DDI_SERVICE_DEGRADED); 4160 atomic_or_32(&igb->igb_state, IGB_ERROR); 4161 return (DDI_INTR_CLAIMED); 4162 } 4163 4164 /* 4165 * Look for tx reclaiming work first. Remember, in the 4166 * case of only interrupt sharing, only one tx ring is 4167 * used 4168 */ 4169 igb_intr_tx_work(&igb->tx_rings[0]); 4170 4171 /* 4172 * Check for "other" causes. 4173 */ 4174 if (icr & E1000_ICR_LSC) { 4175 igb_intr_link_work(igb); 4176 } 4177 4178 /* 4179 * The DOUTSYNC bit indicates a tx packet dropped because 4180 * DMA engine gets "out of sync". There isn't a real fix 4181 * for this. The Intel recommendation is to count the number 4182 * of occurrences so user can detect when it is happening. 4183 * The issue is non-fatal and there's no recovery action 4184 * available. 4185 */ 4186 if (icr & E1000_ICR_DOUTSYNC) { 4187 IGB_STAT(igb->dout_sync); 4188 } 4189 4190 if (icr & E1000_ICR_DRSTA) { 4191 /* 82580 Full Device Reset needed */ 4192 atomic_or_32(&igb->igb_state, IGB_STALL); 4193 } 4194 4195 return (DDI_INTR_CLAIMED); 4196 } 4197 4198 /* 4199 * igb_alloc_intrs - Allocate interrupts for the driver 4200 * 4201 * Normal sequence is to try MSI-X; if not sucessful, try MSI; 4202 * if not successful, try Legacy. 4203 * igb->intr_force can be used to force sequence to start with 4204 * any of the 3 types. 4205 * If MSI-X is not used, number of tx/rx rings is forced to 1. 4206 */ 4207 static int 4208 igb_alloc_intrs(igb_t *igb) 4209 { 4210 dev_info_t *devinfo; 4211 int intr_types; 4212 int rc; 4213 4214 devinfo = igb->dip; 4215 4216 /* Get supported interrupt types */ 4217 rc = ddi_intr_get_supported_types(devinfo, &intr_types); 4218 4219 if (rc != DDI_SUCCESS) { 4220 igb_log(igb, 4221 "Get supported interrupt types failed: %d", rc); 4222 return (IGB_FAILURE); 4223 } 4224 IGB_DEBUGLOG_1(igb, "Supported interrupt types: %x", intr_types); 4225 4226 igb->intr_type = 0; 4227 4228 /* Install MSI-X interrupts */ 4229 if ((intr_types & DDI_INTR_TYPE_MSIX) && 4230 (igb->intr_force <= IGB_INTR_MSIX)) { 4231 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSIX); 4232 4233 if (rc == IGB_SUCCESS) 4234 return (IGB_SUCCESS); 4235 4236 igb_log(igb, 4237 "Allocate MSI-X failed, trying MSI interrupts..."); 4238 } 4239 4240 /* MSI-X not used, force rings to 1 */ 4241 igb->num_rx_rings = 1; 4242 igb->num_tx_rings = 1; 4243 igb_log(igb, 4244 "MSI-X not used, force rx and tx queue number to 1"); 4245 4246 /* Install MSI interrupts */ 4247 if ((intr_types & DDI_INTR_TYPE_MSI) && 4248 (igb->intr_force <= IGB_INTR_MSI)) { 4249 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_MSI); 4250 4251 if (rc == IGB_SUCCESS) 4252 return (IGB_SUCCESS); 4253 4254 igb_log(igb, 4255 "Allocate MSI failed, trying Legacy interrupts..."); 4256 } 4257 4258 /* Install legacy interrupts */ 4259 if (intr_types & DDI_INTR_TYPE_FIXED) { 4260 rc = igb_alloc_intr_handles(igb, DDI_INTR_TYPE_FIXED); 4261 4262 if (rc == IGB_SUCCESS) 4263 return (IGB_SUCCESS); 4264 4265 igb_log(igb, 4266 "Allocate Legacy interrupts failed"); 4267 } 4268 4269 /* If none of the 3 types succeeded, return failure */ 4270 return (IGB_FAILURE); 4271 } 4272 4273 /* 4274 * igb_alloc_intr_handles - Allocate interrupt handles. 4275 * 4276 * For legacy and MSI, only 1 handle is needed. For MSI-X, 4277 * if fewer than 2 handles are available, return failure. 4278 * Upon success, this sets the number of Rx rings to a number that 4279 * matches the handles available for Rx interrupts. 4280 */ 4281 static int 4282 igb_alloc_intr_handles(igb_t *igb, int intr_type) 4283 { 4284 dev_info_t *devinfo; 4285 int orig, request, count, avail, actual; 4286 int diff, minimum; 4287 int rc; 4288 4289 devinfo = igb->dip; 4290 4291 switch (intr_type) { 4292 case DDI_INTR_TYPE_FIXED: 4293 request = 1; /* Request 1 legacy interrupt handle */ 4294 minimum = 1; 4295 IGB_DEBUGLOG_0(igb, "interrupt type: legacy"); 4296 break; 4297 4298 case DDI_INTR_TYPE_MSI: 4299 request = 1; /* Request 1 MSI interrupt handle */ 4300 minimum = 1; 4301 IGB_DEBUGLOG_0(igb, "interrupt type: MSI"); 4302 break; 4303 4304 case DDI_INTR_TYPE_MSIX: 4305 /* 4306 * Number of vectors for the adapter is 4307 * # rx rings + # tx rings 4308 * One of tx vectors is for tx & other 4309 */ 4310 request = igb->num_rx_rings + igb->num_tx_rings; 4311 orig = request; 4312 minimum = 2; 4313 IGB_DEBUGLOG_0(igb, "interrupt type: MSI-X"); 4314 break; 4315 4316 default: 4317 igb_log(igb, 4318 "invalid call to igb_alloc_intr_handles(): %d\n", 4319 intr_type); 4320 return (IGB_FAILURE); 4321 } 4322 IGB_DEBUGLOG_2(igb, "interrupt handles requested: %d minimum: %d", 4323 request, minimum); 4324 4325 /* 4326 * Get number of supported interrupts 4327 */ 4328 rc = ddi_intr_get_nintrs(devinfo, intr_type, &count); 4329 if ((rc != DDI_SUCCESS) || (count < minimum)) { 4330 igb_log(igb, 4331 "Get supported interrupt number failed. " 4332 "Return: %d, count: %d", rc, count); 4333 return (IGB_FAILURE); 4334 } 4335 IGB_DEBUGLOG_1(igb, "interrupts supported: %d", count); 4336 4337 /* 4338 * Get number of available interrupts 4339 */ 4340 rc = ddi_intr_get_navail(devinfo, intr_type, &avail); 4341 if ((rc != DDI_SUCCESS) || (avail < minimum)) { 4342 igb_log(igb, 4343 "Get available interrupt number failed. " 4344 "Return: %d, available: %d", rc, avail); 4345 return (IGB_FAILURE); 4346 } 4347 IGB_DEBUGLOG_1(igb, "interrupts available: %d", avail); 4348 4349 if (avail < request) { 4350 igb_log(igb, "Request %d handles, %d available", 4351 request, avail); 4352 request = avail; 4353 } 4354 4355 actual = 0; 4356 igb->intr_cnt = 0; 4357 4358 /* 4359 * Allocate an array of interrupt handles 4360 */ 4361 igb->intr_size = request * sizeof (ddi_intr_handle_t); 4362 igb->htable = kmem_alloc(igb->intr_size, KM_SLEEP); 4363 4364 rc = ddi_intr_alloc(devinfo, igb->htable, intr_type, 0, 4365 request, &actual, DDI_INTR_ALLOC_NORMAL); 4366 if (rc != DDI_SUCCESS) { 4367 igb_log(igb, "Allocate interrupts failed. " 4368 "return: %d, request: %d, actual: %d", 4369 rc, request, actual); 4370 goto alloc_handle_fail; 4371 } 4372 IGB_DEBUGLOG_1(igb, "interrupts actually allocated: %d", actual); 4373 4374 igb->intr_cnt = actual; 4375 4376 if (actual < minimum) { 4377 igb_log(igb, "Insufficient interrupt handles allocated: %d", 4378 actual); 4379 goto alloc_handle_fail; 4380 } 4381 4382 /* 4383 * For MSI-X, actual might force us to reduce number of tx & rx rings 4384 */ 4385 if ((intr_type == DDI_INTR_TYPE_MSIX) && (orig > actual)) { 4386 diff = orig - actual; 4387 if (diff < igb->num_tx_rings) { 4388 igb_log(igb, 4389 "MSI-X vectors force Tx queue number to %d", 4390 igb->num_tx_rings - diff); 4391 igb->num_tx_rings -= diff; 4392 } else { 4393 igb_log(igb, 4394 "MSI-X vectors force Tx queue number to 1"); 4395 igb->num_tx_rings = 1; 4396 4397 igb_log(igb, 4398 "MSI-X vectors force Rx queue number to %d", 4399 actual - 1); 4400 igb->num_rx_rings = actual - 1; 4401 } 4402 } 4403 4404 /* 4405 * Get priority for first vector, assume remaining are all the same 4406 */ 4407 rc = ddi_intr_get_pri(igb->htable[0], &igb->intr_pri); 4408 if (rc != DDI_SUCCESS) { 4409 igb_log(igb, 4410 "Get interrupt priority failed: %d", rc); 4411 goto alloc_handle_fail; 4412 } 4413 4414 rc = ddi_intr_get_cap(igb->htable[0], &igb->intr_cap); 4415 if (rc != DDI_SUCCESS) { 4416 igb_log(igb, 4417 "Get interrupt cap failed: %d", rc); 4418 goto alloc_handle_fail; 4419 } 4420 4421 igb->intr_type = intr_type; 4422 4423 return (IGB_SUCCESS); 4424 4425 alloc_handle_fail: 4426 igb_rem_intrs(igb); 4427 4428 return (IGB_FAILURE); 4429 } 4430 4431 /* 4432 * igb_add_intr_handlers - Add interrupt handlers based on the interrupt type 4433 * 4434 * Before adding the interrupt handlers, the interrupt vectors have 4435 * been allocated, and the rx/tx rings have also been allocated. 4436 */ 4437 static int 4438 igb_add_intr_handlers(igb_t *igb) 4439 { 4440 igb_rx_ring_t *rx_ring; 4441 igb_tx_ring_t *tx_ring; 4442 int vector; 4443 int rc; 4444 int i; 4445 4446 vector = 0; 4447 4448 switch (igb->intr_type) { 4449 case DDI_INTR_TYPE_MSIX: 4450 /* Add interrupt handler for tx + other */ 4451 tx_ring = &igb->tx_rings[0]; 4452 rc = ddi_intr_add_handler(igb->htable[vector], 4453 (ddi_intr_handler_t *)igb_intr_tx_other, 4454 (void *)igb, NULL); 4455 4456 if (rc != DDI_SUCCESS) { 4457 igb_log(igb, 4458 "Add tx/other interrupt handler failed: %d", rc); 4459 return (IGB_FAILURE); 4460 } 4461 tx_ring->intr_vector = vector; 4462 vector++; 4463 4464 /* Add interrupt handler for each rx ring */ 4465 for (i = 0; i < igb->num_rx_rings; i++) { 4466 rx_ring = &igb->rx_rings[i]; 4467 4468 rc = ddi_intr_add_handler(igb->htable[vector], 4469 (ddi_intr_handler_t *)igb_intr_rx, 4470 (void *)rx_ring, NULL); 4471 4472 if (rc != DDI_SUCCESS) { 4473 igb_log(igb, 4474 "Add rx interrupt handler failed. " 4475 "return: %d, rx ring: %d", rc, i); 4476 for (vector--; vector >= 0; vector--) { 4477 (void) ddi_intr_remove_handler( 4478 igb->htable[vector]); 4479 } 4480 return (IGB_FAILURE); 4481 } 4482 4483 rx_ring->intr_vector = vector; 4484 4485 vector++; 4486 } 4487 4488 /* Add interrupt handler for each tx ring from 2nd ring */ 4489 for (i = 1; i < igb->num_tx_rings; i++) { 4490 tx_ring = &igb->tx_rings[i]; 4491 4492 rc = ddi_intr_add_handler(igb->htable[vector], 4493 (ddi_intr_handler_t *)igb_intr_tx, 4494 (void *)tx_ring, NULL); 4495 4496 if (rc != DDI_SUCCESS) { 4497 igb_log(igb, 4498 "Add tx interrupt handler failed. " 4499 "return: %d, tx ring: %d", rc, i); 4500 for (vector--; vector >= 0; vector--) { 4501 (void) ddi_intr_remove_handler( 4502 igb->htable[vector]); 4503 } 4504 return (IGB_FAILURE); 4505 } 4506 4507 tx_ring->intr_vector = vector; 4508 4509 vector++; 4510 } 4511 4512 break; 4513 4514 case DDI_INTR_TYPE_MSI: 4515 /* Add interrupt handlers for the only vector */ 4516 rc = ddi_intr_add_handler(igb->htable[vector], 4517 (ddi_intr_handler_t *)igb_intr_msi, 4518 (void *)igb, NULL); 4519 4520 if (rc != DDI_SUCCESS) { 4521 igb_log(igb, 4522 "Add MSI interrupt handler failed: %d", rc); 4523 return (IGB_FAILURE); 4524 } 4525 4526 rx_ring = &igb->rx_rings[0]; 4527 rx_ring->intr_vector = vector; 4528 4529 vector++; 4530 break; 4531 4532 case DDI_INTR_TYPE_FIXED: 4533 /* Add interrupt handlers for the only vector */ 4534 rc = ddi_intr_add_handler(igb->htable[vector], 4535 (ddi_intr_handler_t *)igb_intr_legacy, 4536 (void *)igb, NULL); 4537 4538 if (rc != DDI_SUCCESS) { 4539 igb_log(igb, 4540 "Add legacy interrupt handler failed: %d", rc); 4541 return (IGB_FAILURE); 4542 } 4543 4544 rx_ring = &igb->rx_rings[0]; 4545 rx_ring->intr_vector = vector; 4546 4547 vector++; 4548 break; 4549 4550 default: 4551 return (IGB_FAILURE); 4552 } 4553 4554 ASSERT(vector == igb->intr_cnt); 4555 4556 return (IGB_SUCCESS); 4557 } 4558 4559 /* 4560 * igb_setup_msix_82575 - setup 82575 adapter to use MSI-X interrupts 4561 * 4562 * For each vector enabled on the adapter, Set the MSIXBM register accordingly 4563 */ 4564 static void 4565 igb_setup_msix_82575(igb_t *igb) 4566 { 4567 uint32_t eims = 0; 4568 int i, vector; 4569 struct e1000_hw *hw = &igb->hw; 4570 4571 /* 4572 * Set vector for tx ring 0 and other causes. 4573 * NOTE assumption that it is vector 0. 4574 */ 4575 vector = 0; 4576 4577 igb->eims_mask = E1000_EICR_TX_QUEUE0 | E1000_EICR_OTHER; 4578 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), igb->eims_mask); 4579 vector++; 4580 4581 for (i = 0; i < igb->num_rx_rings; i++) { 4582 /* 4583 * Set vector for each rx ring 4584 */ 4585 eims = (E1000_EICR_RX_QUEUE0 << i); 4586 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4587 4588 /* 4589 * Accumulate bits to enable in 4590 * igb_enable_adapter_interrupts_82575() 4591 */ 4592 igb->eims_mask |= eims; 4593 4594 vector++; 4595 } 4596 4597 for (i = 1; i < igb->num_tx_rings; i++) { 4598 /* 4599 * Set vector for each tx ring from 2nd tx ring 4600 */ 4601 eims = (E1000_EICR_TX_QUEUE0 << i); 4602 E1000_WRITE_REG(hw, E1000_MSIXBM(vector), eims); 4603 4604 /* 4605 * Accumulate bits to enable in 4606 * igb_enable_adapter_interrupts_82575() 4607 */ 4608 igb->eims_mask |= eims; 4609 4610 vector++; 4611 } 4612 4613 ASSERT(vector == igb->intr_cnt); 4614 4615 /* 4616 * Disable IAM for ICR interrupt bits 4617 */ 4618 E1000_WRITE_REG(hw, E1000_IAM, 0); 4619 E1000_WRITE_FLUSH(hw); 4620 } 4621 4622 /* 4623 * igb_setup_msix_82576 - setup 82576 adapter to use MSI-X interrupts 4624 * 4625 * 82576 uses a table based method for assigning vectors. Each queue has a 4626 * single entry in the table to which we write a vector number along with a 4627 * "valid" bit. The entry is a single byte in a 4-byte register. Vectors 4628 * take a different position in the 4-byte register depending on whether 4629 * they are numbered above or below 8. 4630 */ 4631 static void 4632 igb_setup_msix_82576(igb_t *igb) 4633 { 4634 struct e1000_hw *hw = &igb->hw; 4635 uint32_t ivar, index, vector; 4636 int i; 4637 4638 /* must enable msi-x capability before IVAR settings */ 4639 E1000_WRITE_REG(hw, E1000_GPIE, 4640 (E1000_GPIE_MSIX_MODE | E1000_GPIE_PBA | E1000_GPIE_NSICR)); 4641 4642 /* 4643 * Set vector for tx ring 0 and other causes. 4644 * NOTE assumption that it is vector 0. 4645 * This is also interdependent with installation of interrupt service 4646 * routines in igb_add_intr_handlers(). 4647 */ 4648 4649 /* assign "other" causes to vector 0 */ 4650 vector = 0; 4651 ivar = ((vector | E1000_IVAR_VALID) << 8); 4652 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4653 4654 /* assign tx ring 0 to vector 0 */ 4655 ivar = ((vector | E1000_IVAR_VALID) << 8); 4656 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4657 4658 /* prepare to enable tx & other interrupt causes */ 4659 igb->eims_mask = (1 << vector); 4660 4661 vector ++; 4662 for (i = 0; i < igb->num_rx_rings; i++) { 4663 /* 4664 * Set vector for each rx ring 4665 */ 4666 index = (i & 0x7); 4667 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4668 4669 if (i < 8) { 4670 /* vector goes into low byte of register */ 4671 ivar = ivar & 0xFFFFFF00; 4672 ivar |= (vector | E1000_IVAR_VALID); 4673 } else { 4674 /* vector goes into third byte of register */ 4675 ivar = ivar & 0xFF00FFFF; 4676 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4677 } 4678 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4679 4680 /* Accumulate interrupt-cause bits to enable */ 4681 igb->eims_mask |= (1 << vector); 4682 4683 vector ++; 4684 } 4685 4686 for (i = 1; i < igb->num_tx_rings; i++) { 4687 /* 4688 * Set vector for each tx ring from 2nd tx ring. 4689 * Note assumption that tx vectors numericall follow rx vectors. 4690 */ 4691 index = (i & 0x7); 4692 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4693 4694 if (i < 8) { 4695 /* vector goes into second byte of register */ 4696 ivar = ivar & 0xFFFF00FF; 4697 ivar |= ((vector | E1000_IVAR_VALID) << 8); 4698 } else { 4699 /* vector goes into fourth byte of register */ 4700 ivar = ivar & 0x00FFFFFF; 4701 ivar |= (vector | E1000_IVAR_VALID) << 24; 4702 } 4703 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4704 4705 /* Accumulate interrupt-cause bits to enable */ 4706 igb->eims_mask |= (1 << vector); 4707 4708 vector ++; 4709 } 4710 4711 ASSERT(vector == igb->intr_cnt); 4712 } 4713 4714 /* 4715 * igb_setup_msix_82580 - setup 82580 adapter to use MSI-X interrupts 4716 * 4717 * 82580 uses same table approach at 82576 but has fewer entries. Each 4718 * queue has a single entry in the table to which we write a vector number 4719 * along with a "valid" bit. Vectors take a different position in the 4720 * register depending on * whether * they are numbered above or below 4. 4721 */ 4722 static void 4723 igb_setup_msix_82580(igb_t *igb) 4724 { 4725 struct e1000_hw *hw = &igb->hw; 4726 uint32_t ivar, index, vector; 4727 int i; 4728 4729 /* must enable msi-x capability before IVAR settings */ 4730 E1000_WRITE_REG(hw, E1000_GPIE, (E1000_GPIE_MSIX_MODE | 4731 E1000_GPIE_PBA | E1000_GPIE_NSICR | E1000_GPIE_EIAME)); 4732 /* 4733 * Set vector for tx ring 0 and other causes. 4734 * NOTE assumption that it is vector 0. 4735 * This is also interdependent with installation of interrupt service 4736 * routines in igb_add_intr_handlers(). 4737 */ 4738 4739 /* assign "other" causes to vector 0 */ 4740 vector = 0; 4741 ivar = ((vector | E1000_IVAR_VALID) << 8); 4742 E1000_WRITE_REG(hw, E1000_IVAR_MISC, ivar); 4743 4744 /* assign tx ring 0 to vector 0 */ 4745 ivar = ((vector | E1000_IVAR_VALID) << 8); 4746 E1000_WRITE_REG(hw, E1000_IVAR0, ivar); 4747 4748 /* prepare to enable tx & other interrupt causes */ 4749 igb->eims_mask = (1 << vector); 4750 4751 vector ++; 4752 4753 for (i = 0; i < igb->num_rx_rings; i++) { 4754 /* 4755 * Set vector for each rx ring 4756 */ 4757 index = (i >> 1); 4758 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4759 4760 if (i & 1) { 4761 /* vector goes into third byte of register */ 4762 ivar = ivar & 0xFF00FFFF; 4763 ivar |= ((vector | E1000_IVAR_VALID) << 16); 4764 } else { 4765 /* vector goes into low byte of register */ 4766 ivar = ivar & 0xFFFFFF00; 4767 ivar |= (vector | E1000_IVAR_VALID); 4768 } 4769 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4770 4771 /* Accumulate interrupt-cause bits to enable */ 4772 igb->eims_mask |= (1 << vector); 4773 4774 vector ++; 4775 } 4776 4777 for (i = 1; i < igb->num_tx_rings; i++) { 4778 /* 4779 * Set vector for each tx ring from 2nd tx ring. 4780 * Note assumption that tx vectors numericall follow rx vectors. 4781 */ 4782 index = (i >> 1); 4783 ivar = E1000_READ_REG_ARRAY(hw, E1000_IVAR0, index); 4784 4785 if (i & 1) { 4786 /* vector goes into high byte of register */ 4787 ivar = ivar & 0x00FFFFFF; 4788 ivar |= ((vector | E1000_IVAR_VALID) << 24); 4789 } else { 4790 /* vector goes into second byte of register */ 4791 ivar = ivar & 0xFFFF00FF; 4792 ivar |= (vector | E1000_IVAR_VALID) << 8; 4793 } 4794 E1000_WRITE_REG_ARRAY(hw, E1000_IVAR0, index, ivar); 4795 4796 /* Accumulate interrupt-cause bits to enable */ 4797 igb->eims_mask |= (1 << vector); 4798 4799 vector ++; 4800 } 4801 ASSERT(vector == igb->intr_cnt); 4802 } 4803 4804 /* 4805 * igb_rem_intr_handlers - remove the interrupt handlers 4806 */ 4807 static void 4808 igb_rem_intr_handlers(igb_t *igb) 4809 { 4810 int i; 4811 int rc; 4812 4813 for (i = 0; i < igb->intr_cnt; i++) { 4814 rc = ddi_intr_remove_handler(igb->htable[i]); 4815 if (rc != DDI_SUCCESS) { 4816 IGB_DEBUGLOG_1(igb, 4817 "Remove intr handler failed: %d", rc); 4818 } 4819 } 4820 } 4821 4822 /* 4823 * igb_rem_intrs - remove the allocated interrupts 4824 */ 4825 static void 4826 igb_rem_intrs(igb_t *igb) 4827 { 4828 int i; 4829 int rc; 4830 4831 for (i = 0; i < igb->intr_cnt; i++) { 4832 rc = ddi_intr_free(igb->htable[i]); 4833 if (rc != DDI_SUCCESS) { 4834 IGB_DEBUGLOG_1(igb, 4835 "Free intr failed: %d", rc); 4836 } 4837 } 4838 4839 kmem_free(igb->htable, igb->intr_size); 4840 igb->htable = NULL; 4841 } 4842 4843 /* 4844 * igb_enable_intrs - enable all the ddi interrupts 4845 */ 4846 static int 4847 igb_enable_intrs(igb_t *igb) 4848 { 4849 int i; 4850 int rc; 4851 4852 /* Enable interrupts */ 4853 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4854 /* Call ddi_intr_block_enable() for MSI */ 4855 rc = ddi_intr_block_enable(igb->htable, igb->intr_cnt); 4856 if (rc != DDI_SUCCESS) { 4857 igb_log(igb, 4858 "Enable block intr failed: %d", rc); 4859 return (IGB_FAILURE); 4860 } 4861 } else { 4862 /* Call ddi_intr_enable() for Legacy/MSI non block enable */ 4863 for (i = 0; i < igb->intr_cnt; i++) { 4864 rc = ddi_intr_enable(igb->htable[i]); 4865 if (rc != DDI_SUCCESS) { 4866 igb_log(igb, 4867 "Enable intr failed: %d", rc); 4868 return (IGB_FAILURE); 4869 } 4870 } 4871 } 4872 4873 return (IGB_SUCCESS); 4874 } 4875 4876 /* 4877 * igb_disable_intrs - disable all the ddi interrupts 4878 */ 4879 static int 4880 igb_disable_intrs(igb_t *igb) 4881 { 4882 int i; 4883 int rc; 4884 4885 /* Disable all interrupts */ 4886 if (igb->intr_cap & DDI_INTR_FLAG_BLOCK) { 4887 rc = ddi_intr_block_disable(igb->htable, igb->intr_cnt); 4888 if (rc != DDI_SUCCESS) { 4889 igb_log(igb, 4890 "Disable block intr failed: %d", rc); 4891 return (IGB_FAILURE); 4892 } 4893 } else { 4894 for (i = 0; i < igb->intr_cnt; i++) { 4895 rc = ddi_intr_disable(igb->htable[i]); 4896 if (rc != DDI_SUCCESS) { 4897 igb_log(igb, 4898 "Disable intr failed: %d", rc); 4899 return (IGB_FAILURE); 4900 } 4901 } 4902 } 4903 4904 return (IGB_SUCCESS); 4905 } 4906 4907 /* 4908 * igb_get_phy_state - Get and save the parameters read from PHY registers 4909 */ 4910 static void 4911 igb_get_phy_state(igb_t *igb) 4912 { 4913 struct e1000_hw *hw = &igb->hw; 4914 uint16_t phy_ctrl; 4915 uint16_t phy_status; 4916 uint16_t phy_an_adv; 4917 uint16_t phy_an_exp; 4918 uint16_t phy_ext_status; 4919 uint16_t phy_1000t_ctrl; 4920 uint16_t phy_1000t_status; 4921 uint16_t phy_lp_able; 4922 4923 ASSERT(mutex_owned(&igb->gen_lock)); 4924 4925 if (hw->phy.media_type == e1000_media_type_copper) { 4926 (void) e1000_read_phy_reg(hw, PHY_CONTROL, &phy_ctrl); 4927 (void) e1000_read_phy_reg(hw, PHY_STATUS, &phy_status); 4928 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_ADV, &phy_an_adv); 4929 (void) e1000_read_phy_reg(hw, PHY_AUTONEG_EXP, &phy_an_exp); 4930 (void) e1000_read_phy_reg(hw, PHY_EXT_STATUS, &phy_ext_status); 4931 (void) e1000_read_phy_reg(hw, PHY_1000T_CTRL, &phy_1000t_ctrl); 4932 (void) e1000_read_phy_reg(hw, 4933 PHY_1000T_STATUS, &phy_1000t_status); 4934 (void) e1000_read_phy_reg(hw, PHY_LP_ABILITY, &phy_lp_able); 4935 4936 igb->param_autoneg_cap = 4937 (phy_status & MII_SR_AUTONEG_CAPS) ? 1 : 0; 4938 igb->param_pause_cap = 4939 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4940 igb->param_asym_pause_cap = 4941 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4942 igb->param_1000fdx_cap = 4943 ((phy_ext_status & IEEE_ESR_1000T_FD_CAPS) || 4944 (phy_ext_status & IEEE_ESR_1000X_FD_CAPS)) ? 1 : 0; 4945 igb->param_1000hdx_cap = 4946 ((phy_ext_status & IEEE_ESR_1000T_HD_CAPS) || 4947 (phy_ext_status & IEEE_ESR_1000X_HD_CAPS)) ? 1 : 0; 4948 igb->param_100t4_cap = 4949 (phy_status & MII_SR_100T4_CAPS) ? 1 : 0; 4950 igb->param_100fdx_cap = ((phy_status & MII_SR_100X_FD_CAPS) || 4951 (phy_status & MII_SR_100T2_FD_CAPS)) ? 1 : 0; 4952 igb->param_100hdx_cap = ((phy_status & MII_SR_100X_HD_CAPS) || 4953 (phy_status & MII_SR_100T2_HD_CAPS)) ? 1 : 0; 4954 igb->param_10fdx_cap = 4955 (phy_status & MII_SR_10T_FD_CAPS) ? 1 : 0; 4956 igb->param_10hdx_cap = 4957 (phy_status & MII_SR_10T_HD_CAPS) ? 1 : 0; 4958 igb->param_rem_fault = 4959 (phy_status & MII_SR_REMOTE_FAULT) ? 1 : 0; 4960 4961 igb->param_adv_autoneg_cap = hw->mac.autoneg; 4962 igb->param_adv_pause_cap = 4963 (phy_an_adv & NWAY_AR_PAUSE) ? 1 : 0; 4964 igb->param_adv_asym_pause_cap = 4965 (phy_an_adv & NWAY_AR_ASM_DIR) ? 1 : 0; 4966 igb->param_adv_1000hdx_cap = 4967 (phy_1000t_ctrl & CR_1000T_HD_CAPS) ? 1 : 0; 4968 igb->param_adv_100t4_cap = 4969 (phy_an_adv & NWAY_AR_100T4_CAPS) ? 1 : 0; 4970 igb->param_adv_rem_fault = 4971 (phy_an_adv & NWAY_AR_REMOTE_FAULT) ? 1 : 0; 4972 if (igb->param_adv_autoneg_cap == 1) { 4973 igb->param_adv_1000fdx_cap = 4974 (phy_1000t_ctrl & CR_1000T_FD_CAPS) ? 1 : 0; 4975 igb->param_adv_100fdx_cap = 4976 (phy_an_adv & NWAY_AR_100TX_FD_CAPS) ? 1 : 0; 4977 igb->param_adv_100hdx_cap = 4978 (phy_an_adv & NWAY_AR_100TX_HD_CAPS) ? 1 : 0; 4979 igb->param_adv_10fdx_cap = 4980 (phy_an_adv & NWAY_AR_10T_FD_CAPS) ? 1 : 0; 4981 igb->param_adv_10hdx_cap = 4982 (phy_an_adv & NWAY_AR_10T_HD_CAPS) ? 1 : 0; 4983 } 4984 4985 igb->param_lp_autoneg_cap = 4986 (phy_an_exp & NWAY_ER_LP_NWAY_CAPS) ? 1 : 0; 4987 igb->param_lp_pause_cap = 4988 (phy_lp_able & NWAY_LPAR_PAUSE) ? 1 : 0; 4989 igb->param_lp_asym_pause_cap = 4990 (phy_lp_able & NWAY_LPAR_ASM_DIR) ? 1 : 0; 4991 igb->param_lp_1000fdx_cap = 4992 (phy_1000t_status & SR_1000T_LP_FD_CAPS) ? 1 : 0; 4993 igb->param_lp_1000hdx_cap = 4994 (phy_1000t_status & SR_1000T_LP_HD_CAPS) ? 1 : 0; 4995 igb->param_lp_100t4_cap = 4996 (phy_lp_able & NWAY_LPAR_100T4_CAPS) ? 1 : 0; 4997 igb->param_lp_100fdx_cap = 4998 (phy_lp_able & NWAY_LPAR_100TX_FD_CAPS) ? 1 : 0; 4999 igb->param_lp_100hdx_cap = 5000 (phy_lp_able & NWAY_LPAR_100TX_HD_CAPS) ? 1 : 0; 5001 igb->param_lp_10fdx_cap = 5002 (phy_lp_able & NWAY_LPAR_10T_FD_CAPS) ? 1 : 0; 5003 igb->param_lp_10hdx_cap = 5004 (phy_lp_able & NWAY_LPAR_10T_HD_CAPS) ? 1 : 0; 5005 igb->param_lp_rem_fault = 5006 (phy_lp_able & NWAY_LPAR_REMOTE_FAULT) ? 1 : 0; 5007 } else { 5008 /* 5009 * 1Gig Fiber adapter only offers 1Gig Full Duplex. 5010 */ 5011 igb->param_autoneg_cap = 0; 5012 igb->param_pause_cap = 1; 5013 igb->param_asym_pause_cap = 1; 5014 igb->param_1000fdx_cap = 1; 5015 igb->param_1000hdx_cap = 0; 5016 igb->param_100t4_cap = 0; 5017 igb->param_100fdx_cap = 0; 5018 igb->param_100hdx_cap = 0; 5019 igb->param_10fdx_cap = 0; 5020 igb->param_10hdx_cap = 0; 5021 5022 igb->param_adv_autoneg_cap = 0; 5023 igb->param_adv_pause_cap = 1; 5024 igb->param_adv_asym_pause_cap = 1; 5025 igb->param_adv_1000fdx_cap = 1; 5026 igb->param_adv_1000hdx_cap = 0; 5027 igb->param_adv_100t4_cap = 0; 5028 igb->param_adv_100fdx_cap = 0; 5029 igb->param_adv_100hdx_cap = 0; 5030 igb->param_adv_10fdx_cap = 0; 5031 igb->param_adv_10hdx_cap = 0; 5032 5033 igb->param_lp_autoneg_cap = 0; 5034 igb->param_lp_pause_cap = 0; 5035 igb->param_lp_asym_pause_cap = 0; 5036 igb->param_lp_1000fdx_cap = 0; 5037 igb->param_lp_1000hdx_cap = 0; 5038 igb->param_lp_100t4_cap = 0; 5039 igb->param_lp_100fdx_cap = 0; 5040 igb->param_lp_100hdx_cap = 0; 5041 igb->param_lp_10fdx_cap = 0; 5042 igb->param_lp_10hdx_cap = 0; 5043 igb->param_lp_rem_fault = 0; 5044 } 5045 } 5046 5047 /* 5048 * synchronize the adv* and en* parameters. 5049 * 5050 * See comments in <sys/dld.h> for details of the *_en_* 5051 * parameters. The usage of ndd for setting adv parameters will 5052 * synchronize all the en parameters with the e1000g parameters, 5053 * implicitly disabling any settings made via dladm. 5054 */ 5055 static void 5056 igb_param_sync(igb_t *igb) 5057 { 5058 igb->param_en_1000fdx_cap = igb->param_adv_1000fdx_cap; 5059 igb->param_en_1000hdx_cap = igb->param_adv_1000hdx_cap; 5060 igb->param_en_100t4_cap = igb->param_adv_100t4_cap; 5061 igb->param_en_100fdx_cap = igb->param_adv_100fdx_cap; 5062 igb->param_en_100hdx_cap = igb->param_adv_100hdx_cap; 5063 igb->param_en_10fdx_cap = igb->param_adv_10fdx_cap; 5064 igb->param_en_10hdx_cap = igb->param_adv_10hdx_cap; 5065 } 5066 5067 /* 5068 * igb_get_driver_control 5069 */ 5070 static void 5071 igb_get_driver_control(struct e1000_hw *hw) 5072 { 5073 uint32_t ctrl_ext; 5074 5075 /* Notify firmware that driver is in control of device */ 5076 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 5077 ctrl_ext |= E1000_CTRL_EXT_DRV_LOAD; 5078 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 5079 } 5080 5081 /* 5082 * igb_release_driver_control 5083 */ 5084 static void 5085 igb_release_driver_control(struct e1000_hw *hw) 5086 { 5087 uint32_t ctrl_ext; 5088 5089 /* Notify firmware that driver is no longer in control of device */ 5090 ctrl_ext = E1000_READ_REG(hw, E1000_CTRL_EXT); 5091 ctrl_ext &= ~E1000_CTRL_EXT_DRV_LOAD; 5092 E1000_WRITE_REG(hw, E1000_CTRL_EXT, ctrl_ext); 5093 } 5094 5095 /* 5096 * igb_atomic_reserve - Atomic decrease operation 5097 */ 5098 int 5099 igb_atomic_reserve(uint32_t *count_p, uint32_t n) 5100 { 5101 uint32_t oldval; 5102 uint32_t newval; 5103 5104 /* ATOMICALLY */ 5105 do { 5106 oldval = *count_p; 5107 if (oldval < n) 5108 return (-1); 5109 newval = oldval - n; 5110 } while (atomic_cas_32(count_p, oldval, newval) != oldval); 5111 5112 return (newval); 5113 } 5114 5115 /* 5116 * FMA support 5117 */ 5118 5119 int 5120 igb_check_acc_handle(ddi_acc_handle_t handle) 5121 { 5122 ddi_fm_error_t de; 5123 5124 ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION); 5125 ddi_fm_acc_err_clear(handle, DDI_FME_VERSION); 5126 return (de.fme_status); 5127 } 5128 5129 int 5130 igb_check_dma_handle(ddi_dma_handle_t handle) 5131 { 5132 ddi_fm_error_t de; 5133 5134 ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION); 5135 return (de.fme_status); 5136 } 5137 5138 /* 5139 * The IO fault service error handling callback function 5140 */ 5141 /*ARGSUSED*/ 5142 static int 5143 igb_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data) 5144 { 5145 /* 5146 * as the driver can always deal with an error in any dma or 5147 * access handle, we can just return the fme_status value. 5148 */ 5149 pci_ereport_post(dip, err, NULL); 5150 return (err->fme_status); 5151 } 5152 5153 static void 5154 igb_fm_init(igb_t *igb) 5155 { 5156 ddi_iblock_cookie_t iblk; 5157 int fma_dma_flag; 5158 5159 /* Only register with IO Fault Services if we have some capability */ 5160 if (igb->fm_capabilities & DDI_FM_ACCCHK_CAPABLE) { 5161 igb_regs_acc_attr.devacc_attr_access = DDI_FLAGERR_ACC; 5162 } else { 5163 igb_regs_acc_attr.devacc_attr_access = DDI_DEFAULT_ACC; 5164 } 5165 5166 if (igb->fm_capabilities & DDI_FM_DMACHK_CAPABLE) { 5167 fma_dma_flag = 1; 5168 } else { 5169 fma_dma_flag = 0; 5170 } 5171 5172 (void) igb_set_fma_flags(fma_dma_flag); 5173 5174 if (igb->fm_capabilities) { 5175 5176 /* Register capabilities with IO Fault Services */ 5177 ddi_fm_init(igb->dip, &igb->fm_capabilities, &iblk); 5178 5179 /* 5180 * Initialize pci ereport capabilities if ereport capable 5181 */ 5182 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5183 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5184 pci_ereport_setup(igb->dip); 5185 5186 /* 5187 * Register error callback if error callback capable 5188 */ 5189 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5190 ddi_fm_handler_register(igb->dip, 5191 igb_fm_error_cb, (void*) igb); 5192 } 5193 } 5194 5195 static void 5196 igb_fm_fini(igb_t *igb) 5197 { 5198 /* Only unregister FMA capabilities if we registered some */ 5199 if (igb->fm_capabilities) { 5200 5201 /* 5202 * Release any resources allocated by pci_ereport_setup() 5203 */ 5204 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities) || 5205 DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5206 pci_ereport_teardown(igb->dip); 5207 5208 /* 5209 * Un-register error callback if error callback capable 5210 */ 5211 if (DDI_FM_ERRCB_CAP(igb->fm_capabilities)) 5212 ddi_fm_handler_unregister(igb->dip); 5213 5214 /* Unregister from IO Fault Services */ 5215 ddi_fm_fini(igb->dip); 5216 } 5217 } 5218 5219 void 5220 igb_fm_ereport(igb_t *igb, char *detail) 5221 { 5222 uint64_t ena; 5223 char buf[FM_MAX_CLASS]; 5224 5225 (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail); 5226 ena = fm_ena_generate(0, FM_ENA_FMT1); 5227 if (DDI_FM_EREPORT_CAP(igb->fm_capabilities)) { 5228 ddi_fm_ereport_post(igb->dip, buf, ena, DDI_NOSLEEP, 5229 FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERS0, NULL); 5230 } 5231 } 5232