xref: /illumos-gate/usr/src/uts/common/io/igc/igc.c (revision 6bbbd442)

/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2024 Oxide Computer Company
 */

/*
 * Intel I225/226 Ethernet Driver
 * ------------------------------
 *
 * This driver implements support for the Intel I225 and I226 Ethernet
 * controllers which support up to 2.5 GbE and generally only supports BASE-T
 * copper phys. This device is yet another variant on the venerable Intel 1 GbE
 * devices that are found in e1000g(4D) and igb(4D). This is its own driver in
 * part because that's how Intel did things and refactored their common code
 * which we import and is found in the 'core' directory.
 *
 * There is not a good datasheet for the MAC that we've been able to find for
 * this part. It's not clear that Intel even has a doc for this in their
 * Resource and Design Center. The primary datasheet documents the NVM and other
 * parts of it, but not the software interface. Based on observations from the
 * common code we describe this as somewhat of an evolution of the I217 and
 * I210, with less features than the I210, which comes from the server world
 * (which ws itself a more stripped down I350).
 *
 * The result of all this is us trying to focus on what we know about this part
 * and making some assumptions along the way. This includes things like:
 *
 * 1) We believe that the device only supports up to 4 RX and TX queues.
 * 2) There is only one TX context for each TX queue and it is mapped to the
 * queue.
 * 3) There is no support for the head writeback modes that we've found.
 * 4) This does otherwise support both the MSI-X and MSI/INTx interrupt
 * management which are shaped very differently in the device.
 * 5) The 2500BASE-T PHY support is unique, but the other PHY settings are
 * roughly the same as far as we can tell.
 *
 * There are certainly more differences than the points up above, but the above
 * are ones that generally influence our design.
 *
 * ------------
 * Organization
 * ------------
 *
 * This driver is first broken into two different pieces. There is the 'core'
 * code which we import from Intel via FreeBSD. All of these sources are in the
 * 'uts/common/io/igc/core' directory and we try our hardest to avoid modifying
 * them (hence the smatch gags). The core code can be thought of as abstracting
 * the MAC, NVM, and PHY across different chipsets (right now it's all the I225)
 * and providing us with a series of library calls that we can do to manage the
 * chip.
 *
 * The remaining files that sit alongside this one implement different portions
 * of functionality related to the device. In particular:
 *
 *  igc.[ch]:		This is the main entry point for the driver and the
 *			source of this block comment. It implements all of the
 *			basic DDI entry points: attach and detach, interrupts,
 *			PCI config space and register set up and tear down.
 *
 *			The header file contains all of the structure
 *			definitions that we use throughout this and the basic
 *			constants we use for sizing.
 *
 *  igc_gld.c		This file implements all of the major GLDv3 required
 *			entry points that are found in mac(9E). The guts of the
 *			actual I/O are in igc_ring.c, but getting and setting
 *			all of the various MAC properties and other bits is
 *			here.
 *
 *  igc_osdep.[ch]	The osdep (OS dependent) files, are used to define and
 *			implement the functionality required by the common code.
 *			igc_osdep.h is included in the build of each file.
 *
 *			We have a second use for igc_osdep.h which is where we
 *			put missing hardware definitions that apply. This is for
 *			cases where the core code doesn't have it and it should
 *			really live in igc_defines.h or igc_regs.h, but we keep
 *			it here to avoid modifying those.
 *
 *  igc_ring.c		This implements the core I/O routines of the device
 *			driver, starting with the descriptor ring setup and tear
 *			down as well as DMA, descriptor ring, and per-frame
 *			memory. It also implements all of the primary logic for
 *			transmitting and receiving frames.
 *
 *  igc_stat.c		This file deals with kstat creation and destruction as
 *			well as reading and fetching all of the registers that
 *			exist in hardware.
 *
 * There are a few primary data structures to be aware of. Their relationships
 * are shown in the following image and then described. Note, each structure has
 * many more fields than those pictured:
 *
 * +---------------+
 * | dev_info_t *  |
 * |              -|-+
 * | private data  | |
 * +---------------+ v
 *   +------------------------------+        +---------------------+
 *   | igc_t                        |        | igc_addr_t          |
 *   | per-instance primary         |  +---->|                     |
 *   | structure                    |  |+--->| Notes a MAC address | ...
 *   |                              |  ||    | stored in hardware  |
 *   | igc_addr_t    *igc_ucast    -|--+|    +---------------------+
 *   | igc_addr_t    *igc_mcast    -|---+      +---------------------------+
 *   | struct igc_hw *igc_hw       -|--------->| struct igc_hw (core code) |
 *   | igc_tx_ring_t *igc_tx_rings -|--+       |                           |
 *   | igc_rx_ring_t *igc_rx_rings -|--|---+   | igc_mac_info mac          |
 *   +------------------------------+  |   |   | igc_fc_info  fc           |
 *                                     |   |   | igc_phy_info phy          |
 *  +----------------------------------+   |   | igc_nvm_info nvm          |
 *  |                                      v   +---------------------------+
 *  |  +--------------------------------------+
 *  |  | igc_rx_ring_t                        |
 *  |  |                                      |
 *  |  | igc_adv_rx_desc *irr_ring         ---|--> rx hw descriptor ring
 *  |  | uint32_t        irr_next          ---|--> next entry to look for data
 *  |  | igc_rx_buffer_t **irr_work_list   ---|--> corresponds to ring entries
 *  |  | uint32_t        irr_nfree         ---|--> number of free list entries
 *  |  | igc_rx_buffer_t **irr_free_list   ---|--> set of buffers free for bind
 *  |  | igc_rx_buffer_t *irr_arena        ---|-+> array of all rx buffers
 *  |  +--------------------------------------+ |
 *  |                                           |
 *  |          +----------------------------+   |
 *  |          | igc_rx_buffer_t            |<--+
 *  |          |                            |
 *  |          | mblk_t            *igb_mp -|---> mblk_t for rx buffer
 *  |          | igc_dma_buffer_t  irb_dma -|---> DMA memory for rx buffer
 *  |          +----------------------------+
 *  |
 *  |   +------------------------------------+
 *  +-->| igc_tx_ring_t                      |
 *      |                                    |
 *      | icc_adv_tx_desc   *itr_ring      --|--> tx hw descriptor ring
 *      | uin32_t           itr_ring_head  --|--> next descriptor to recycle
 *      | uin32_t           itr_ring_fail  --|--> next descriptor to place
 *      | uin32_t           itr_ring_free  --|--> free descriptors in ring
 *      | igc_tx_buffer_t   **itr_work_list  |--> corresponds to ring entries
 *      | list_t            itr_free_list  --|--> available tx buffers
 *      | igc_tx_buffer_t   *itr_arena     --|-+> array of all tx buffers
 *      +------------------------------------+ |
 *                                             |
 *        +---------------------------------+  |
 *        | igc_tx_buffer_t                 |<-+
 *        |                                 |
 *        | mblk_t           *itb_mp      --|--> mblk to tx (only in first)
 *        | igc_dma_buffer_t itb_dma      --|--> tx DMA buffer for copy
 *        | ddi_dma_handle_t itb_bind_hdl --|--> DMA handle for bind
 *        +---------------------------------+
 *
 * igc_t		This is the primary data structure that exists for each
 *			instance of the driver. There is generally a 1:1
 *			relationship between a physical port, an instance of the
 *			driver, and a PCI function. This structure provides
 *			access to the device's registers and it embeds the
 *			common code's struct igc_hw.
 *
 * struct igc_hw	This structure is used by the core code and it contains
 *			information related to the MAC, PHY, NVM, and related
 *			information that the device uses. In general, this
 *			structure is used when performing API calls to the
 *			common code. The common code calls back into us in the
 *			igc_osdep.c interfaces.
 *
 * igc_tx_ring_t	This structure represents a single transmit ring in
 *			hardware, its associated software state, and
 *			miscellaneous data like statistics, MAC handles, etc.
 *			See the 'TX Data Path Design' section for more
 *			information.
 *
 * igc_rx_ring_t	This is the receive variant of a ring. It represents and
 *			tracks the hardware state along with all our metadata.
 *			One of these exists for each receive ring that we've
 *			enabled (currently one). See the 'RX Data Path Design'
 *			section for more information.
 *
 * igc_tx_buffer_t	This represents a single tx buffer in the driver. A tx
 *			buffer contains DMA based storage that it can use to
 *			transmit a packet and contains a second DMA handle that
 *			can be used to bind a specific mblk_t to it. tx buffers
 *			are capped at the current page size and can be smaller
 *			if the maximum packet size is smaller. A 1500 byte MTU
 *			will end up with a 2 KiB buffer due to the device's
 *			internal alignment requirements.
 *
 * igc_rx_buffer_t	This represents a single rx buffer in the driver. These
 *			buffers may be loaned up to MAC and then returned to us
 *			later. They contain a single DMA buffer which right now
 *			is a single contiguous buffer that fits the maximum
 *			packet size. Each buffer has a corresponding mblk_t that
 *			it is mapped to.
 *
 * igc_dma_buffer_t	This represent a DMA buffer in the system. DMA buffers
 *			are used for transmit buffers, receive buffers, or
 *			various ring descriptor entries. The DMA buffer
 *			structure is not inherently limited to a specific number
 *			of cookies. It is always mapped in our virtual address
 *			space and encapsulates the various DDI functions. In
 *			general, one expects to interface with the idb_va member
 *			when needing to access the memory, the idb_size member
 *			when wanting to understand how much memory is in the
 *			buffer, and the idb_hdl member when needing to access
 *			the DMA cookies.
 *
 * igc_addr_t		This represents a 48-bit Ethernet MAC address slot in
 *			the hardware that may or may not be used at any given
 *			point in time.
 *
 * --------------------
 * Rings and Interrupts
 * --------------------
 *
 * The I225/226 controller like the I210 supports up to 4 rx and tx rings. Due
 * to the long history of this controller and its tradition from the e1000g/igb
 * days and much older parts like the 8254x series, it has two entirely
 * different sets of interrupt modes. One where MSI-X is used and a mode where
 * a single MSI or INTx interrupt is used. Currently the driver only supports
 * the MSI-X mode as that gives us more flexibility and due to the fact that the
 * interrupt modes and register handling are different, reduces the complexity
 * in the driver.
 *
 * The hardware uses its IVAR registers to map specific queues to interrupts.
 * Each rx queue and tx queue is mapped to a specific bit position in the IVAR
 * and there is an additional IVAR register for miscellaneous causes like link
 * state changes. While the IVAR register allows for several bits for MSI-X
 * entries, for the most part, it appears that there is only support for values
 * in the range [0, 4] based on the I210 which we believe extends to the I225/6.
 *
 * MSI-X mode causes the device's various interrupt registers to be split into
 * two groups the 'legacy' and 'extended' (sometimes called advanced) ones. The
 * extended ones all start with 'E'. When in MSI-X mode, the EICR (cause), EICS
 * (cause set), EIAC (auto-clear), EIMS (mask set) registers all operate with
 * indexes that refer to the MSI-X. The primary way to temporarily disable
 * interrupts for polling is to remove the given MSI-X from the auto-clear set
 * and to clear it from the enabled mask set.
 *
 * The implication of all of this is that we can only really disable interrupts
 * for polling on a per-MSI-X basis. This generally means that the design for
 * interrupts and rings is that all the tx rings and the link state change
 * events share interrupt 0, while rx rings use interrupts 1-4. Because the x86
 * 'apix' modules end up defaulting to two interrupts to a driver, we end up
 * only supporting a single rx and tx ring for the time being, though the driver
 * is phrased in terms of a variable number of such rings.
 *
 * -------------------
 * RX Data Path Design
 * -------------------
 *
 * The rx data path is based around allocating a fixed number of receive buffers
 * for each ring. We have two goals in the allocation buffer and ring design:
 *
 * 1) We want to make sure that the ring is always full of valid descriptors for
 *    rx to prevent stalls. One implication of this is that we will always
 *    refill a received buffer with a new one and notify the hardware that the
 *    buffer is usable again.
 *
 * 2) We would prefer to not have to copy received memory and instead bind the
 *    DMA memory directly into an mblk_t.
 *
 * To satisfy (1) we need to allocate at least as many rx buffers as there are
 * ring entries. The ring is sized by default to 512 entries, which is a
 * somewhat arbitrary, but common, size. We then say that we want to be able to
 * loan half of our entries up the stack at any given time. This leads to us
 * allocating 1.5x the ring size rx buffers.
 *
 * All of the rx buffers are stored in the irr_arena array. They are then split
 * between the free list and the ring's work list. The work list is an array
 * that is a 1:1 mapping to a location in the descriptor ring. That is index 4
 * of the work list (irr_work_list[4]) corresponds to index 4 of the descriptor
 * ring (irr_ring[4]). However, this may refer to any of the rx descriptors that
 * is in the irr_arena. When we start up the ring, the first ring size entries
 * are all inserted into the work list and then the remaining entries are
 * inserted into the free list.
 *
 * Entries that are in the work list are always given to hardware. We track the
 * next place for us to scan for received packets through the 'irr_next' index
 * into the descriptor ring. When an interrupt fires, we start at irr_next and
 * iterate through the descriptor ring continuing while we find valid, received
 * packets. When we process a packet, we look at two things to consider whether
 * we bind it or copy it to a new mblk_t. The first piece is the received
 * packet's length. If the packet is small, there is not much value in binding
 * it and instead we just allocate and copy a new buffer for the packet.
 *
 * The second is if there are free rx descriptors. To keep goal (1) valid, we
 * only will loan a packet up if there is an entry on the free list that can
 * replace the rx buffer, as otherwise we'd want to make sure we don't stall the
 * ring. If an rx buffer is loaned, the entry on the free list takes its place
 * in the descriptor ring and when the networking stack is finally done with the
 * mblk_t, it'll be returned to us as part of the freemsg()/freeb() destructor.
 * This lifetime is illustrated in the following diagram:
 *
 *
 *    +-------------+                        +-----------+
 *    | Work List   |<---*-------------------| Free List |
 *    | Owned by HW |    . . Used to replace |   Idle    |
 *    +-------------+        loaned buffers  +-----------+
 *      |     | ^                                  ^
 *      |     | . . . Reused if a                  |
 *      |     +-+     copy is done                 . . . Returned to driver via
 *      |                                          |     freemsg() which calls
 *      |                                          |     igc_rx_recycle().
 *      v                                          |
 *    +-------------------+                        |
 *    | Loaned            |------------------------+
 *    | Owned by netstack |
 *    +-------------------+
 *
 * Currently the rx data path uses rx buffers that are equal to the maximum size
 * of a packet (rounded up based on hardware's 1 KiB alignment requirement).
 * This was mostly done for initial simplicity, though it comes at a memory
 * cost. It is possible to design this to be more like the tx subsystem where we
 * use fixed page size buffers and just cons up an mblk_t chain with b_cont
 * pointers.
 *
 * -------------------
 * TX Data Path Design
 * -------------------
 *
 * The tx data path is a bit different in design from the rx data path. When the
 * system wants to tx data there are two fundamental building blocks that we
 * use, both of which leverage the igc_tx_buffer_t:
 *
 * 1) We use the DMA memory that is allocated with the buffer and copy the
 *    mblk_t data into it. This is used when we have small mblk_t's.
 *
 * 2) We utilize the DMA handle that is in the tx buffer (but not the buffer's
 *    DMA memory) to perform DMA binding. This can result in multiple cookies
 *    and therefore descriptors mapping to the single buffer.
 *
 * Because a given tx buffer may end up using more than one descriptor and we
 * have to account for transmit context descriptors, which are used for
 * indicating checksum and segmentation offloads, we end up only allocating a
 * number of transmit buffers equal to the ring size. In addition, the tx data
 * buffer's maximum size is capped at the size of a single page. This is done
 * because we often aren't going to be copying and if we are, we don't need that
 * much more memory. The actual size may be smaller depending on the MTU.
 *
 * The tx descriptor ring is used in a bit of a different way. While part of the
 * reason for this is that we are filling it based on the stack's demands and
 * therefore only need to fill in descriptors when there's a need, the second
 * reason is because of how the hardware reports back events. There are two
 * major kinds of descriptors that can be entered into the ring. There are the
 * aforementioned context descriptors and then data descriptors. While data
 * descriptors support an interrupt on completion, context descriptors do not.
 *
 * When an mblk_t comes in to be transmitted, we walk all of the mblk_t's
 * associated with it via the b_cont pointer. For each one, we look at the size
 * of the data and determine whether or not to perform DMA binding or to copy it
 * into the current tx buffer. A given tx buffer can be used to copy multiple
 * different mblk_t's. Imagine a pathological case where we had a 500 byte
 * packet split into 125 byte chunks, this would end up using a single tx data
 * buffer.  However, if you imagine a large chunk of TCP data, this may be
 * spread across several mblk_t's so we may end up leveraging multiple tx data
 * buffers.
 *
 * The transmit buffers that are available are stored on a free list. This is
 * managed as a list_t as we end up needing to often track groups of descriptors
 * to allocate and free across packet transmit and recycling. We don't count the
 * number of transmit buffers that are free per se, but it generally tracks the
 * number of free descriptors which do track as in the worst case there is a 1:1
 * relationship between buffers and descriptors and more generally it's 1:n,
 * that is there are multiple descriptors used for a single buffer.
 *
 * The transmit ring is managed through a combination of three integers, the
 * itr_ring_head, the itr_ring_tail, and the itr_ring_free. The ring's tail
 * represents the place where the driver will place new data to transmit. The
 * ring's head represents the first place that we should check for a packet's
 * completion when we're performing recycling (the act of acknowledging what
 * hardware has processed internal to the driver) due to a tx interrupt or
 * manual recycling in the transmit path.
 *
 * When placing a packet as a series of descriptor rings we'll end up doing the
 * following:
 *
 * 1) First we determine how to map each mblk_t as mentioned above.
 * 2) This will then be turned into descriptors in the ring. Each tx data buffer
 *    that is used is placed in the itr_work_list at the corresponding index
 *    that they are used in the ring. There is one special case here, if a
 *    context descriptor is used, the first transmit buffer will refer to the
 *    context descriptor's entry (which always comes before data).
 * 3) We'll ensure that there are enough descriptors for this packet to fit into
 *    the ring or if it would exceed our mandatory gap threshold. If so, then
 *    we'll undo all the work we just did and return the mblk_t to MAC and
 *    indicate that the ring is blocked. MAC will be notified later when we free
 *    up transmit descriptors.
 * 4) In the first transmit data buffer we'll store both the mblk_t and then
 *    we'll store what the index of the last descriptor that's used is. This is
 *    important for recycling. We also indicate that the last descriptor should
 *    be the one that reports its status on interrupt completion.
 * 5) We'll notify hardware that there is data for it to transmit by writing to
 *    the ring's tail pointer.
 *
 * This all works reasonably okay, except for the small problem of the bill,
 * which we pay off in the form of recycling. Recycling is going through the
 * ring and seeing which descriptors are free. While the transmit path described
 * above is the only path that is allowed to move the tail, the recycling path
 * is the only one that's allowed to adjust the head.
 *
 * When we perform recycling we look at the current head and its corresponding
 * tx buffer. There will always be a tx buffer in the same index in the
 * itr_work_list[] unless a serious programmer error has occurred. This buffer
 * will tell us what the index to check for completion is via its itb_last_desc
 * member (only valid when itb_first is set to true). If this index indicates
 * that it has been processed by hardware, then we process all entries between
 * here and there.
 *
 * When we process descriptors, we bunch up the transmit descriptors and
 * mblk_t's. We'll reset the transmit descriptor (freeing any DMA binding if
 * used) and append the mblk_t if it exists to be freed in one large
 * freemsgchain() at the end. The fact that we won't free any tx buffers
 * associated with a packet until they're all done is important. This makes
 * sure that any memory that we have bound from the mblk_t remains valid the
 * entire time.
 *
 * If we have freed enough descriptors as part of this to allow mac to send data
 * again, then once we have finished all processing and dropped the lock, we
 * will notify MAC.
 *
 * When we are processing descriptors here we try to avoid holding the itr_lock
 * except for the start and end of the process. This is an important way to
 * ensure that we don't block transmits. Because of this, there can only be one
 * thread performing a recycle at any given time between the interrupt path and
 * the transmit path trying to clean up. This is maintained using the
 * 'itr_recycle' boolean. If a recycle is already in progress then there's
 * generally not much reason to perform one simultaneously and so the caller
 * will just return. This is why the head (and thus returning descriptors) is
 * only used by the recycle path.
 *
 * -------
 * Locking
 * -------
 *
 * Mutexes exist on three different structures in the driver:
 *
 * 1) igc_t (igc_lock)
 * 2) igc_rx_ring_t (irr_lock, irr_free_lock)
 * 3) igc_tx_ring_t (itr_lock)
 *
 * The following rules hold for locking in the driver:
 *
 * 1) One should not hold locks for both the rx rings and tx rings at the same
 *    time. If this is required, please determine if it is absolutely necessary.
 * 2) You should always take the controller's lock ahead of any ring's locks.
 * 3) The general rx ring lock (irr_lock) should be taken ahead of the free list
 *    lock (irr_free_lock) if both are required.
 *
 * -------------------
 * Future Improvements
 * -------------------
 *
 * This driver was initially written with an eye towards getting something that
 * had broad use for folks with this hardware and not towards enabling every
 * feature immediately. Here are some areas that can be improved upon in the
 * driver.
 *
 *  - Multiple ring, RSS support: As the OS changes towards offering more
 *    interrupts or opting to participate in IRM, then you can more easily
 *    offer RSS and related features. This should likely show up as a single
 *    rx group with multiple rings and leverage the tx pseudo-group support.
 *
 *  - TCP segmentation offload support: Right now the driver does not support
 *    TSO. It'd potentially be a useful addition and help out folks. Fetching
 *    information for TSO is in the tx data path right now.
 *
 *  - FMA Support: Currently the driver does not rig up support for FMA.
 *    Participating in that and more generally being able to reset the device
 *    while it is operating in the face of fatal errors would be good.
 *
 *  - TX stall detection: Related to the above, carefully designing a tx stall
 *    detection and resetting the device when that happens would probably be
 *    useful.
 *
 *  - UFM support: Exposing the NVM and PBA (printed board assembly) through the
 *    UFM subsystem would be a good thing to do.
 *
 *  - Dynamic MTU changing: Right now the driver takes advantage of the
 *    simplification of not allowing the MTU to change once the device has been
 *    started. This isn't great, but it is far from the first (igb, e1000g,
 *    ixgbe, etc.) to do this. It would be nice if this was lifted.
 */

#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/conf.h>
#include <sys/devops.h>
#include <sys/modctl.h>
#include <sys/cmn_err.h>
#include <sys/pci.h>
#include <sys/sysmacros.h>
#include <sys/debug.h>
#include <sys/bitext.h>

#include "igc.h"

/*
 * The core code expects the igc_mcast_raw to be a uint8_t packed array. We use
 * the ether_addr_t to make this a little more explicit and easy to reason
 * about, but that means we are relying on this size.
 */
CTASSERT(sizeof (ether_addr_t) == 6);

uint32_t
igc_read32(igc_t *igc, uint32_t reg)
{
	uint32_t *addr;
	ASSERT3U(reg, <, igc->igc_regs_size);
	addr = (uint32_t *)(igc->igc_regs_base + reg);
	return (ddi_get32(igc->igc_regs_hdl, addr));
}

void
igc_write32(igc_t *igc, uint32_t reg, uint32_t val)
{
	uint32_t *addr;
	ASSERT3U(reg, <, igc->igc_regs_size);
	addr = (uint32_t *)(igc->igc_regs_base + reg);
	ddi_put32(igc->igc_regs_hdl, addr, val);
}

/*
 * Ask hardware if the link is up and ready. Note, this assumes that we're on a
 * copper phy and short circuits a few things. See igb_is_link_up() for what
 * this looks like for non-copper PHYs if that ever becomes relevant.
 */
static bool
igc_link_up(igc_t *igc)
{
	ASSERT(MUTEX_HELD(&igc->igc_lock));

	/*
	 * When the link is up, then the core code will clear the value below.
	 * Otherwise we likely need to assume it's down.
	 */
	(void) igc_check_for_link(&igc->igc_hw);
	return (!igc->igc_hw.mac.get_link_status);
}

static void
igc_intr_lsc(igc_t *igc)
{
	link_state_t orig_state, new_state;
	uint32_t mmd_base;

	mutex_enter(&igc->igc_lock);
	orig_state = igc->igc_link_state;

	/*
	 * Always force a check of the link.
	 */
	igc->igc_hw.mac.get_link_status = true;
	if (igc_link_up(igc)) {
		uint16_t duplex = 0;

		(void) igc_get_speed_and_duplex(&igc->igc_hw,
		    &igc->igc_link_speed, &duplex);

		switch (duplex) {
		case HALF_DUPLEX:
			igc->igc_link_duplex = LINK_DUPLEX_HALF;
			break;
		case FULL_DUPLEX:
			igc->igc_link_duplex = LINK_DUPLEX_FULL;
			break;
		default:
			igc->igc_link_duplex = LINK_DUPLEX_UNKNOWN;
			break;
		}
		igc->igc_link_state = LINK_STATE_UP;
	} else {
		igc->igc_link_state = LINK_STATE_DOWN;
		igc->igc_link_speed = 0;
		igc->igc_link_duplex = LINK_DUPLEX_UNKNOWN;
	}
	new_state = igc->igc_link_state;

	/*
	 * Next, grab a bunch of information from the PHY for future us.
	 */
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_CONTROL, &igc->igc_phy_ctrl);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_STATUS, &igc->igc_phy_status);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_AUTONEG_ADV,
	    &igc->igc_phy_an_adv);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_LP_ABILITY,
	    &igc->igc_phy_lp);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_AUTONEG_EXP,
	    &igc->igc_phy_an_exp);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_1000T_CTRL,
	    &igc->igc_phy_1000t_ctrl);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_1000T_STATUS,
	    &igc->igc_phy_1000t_status);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_EXT_STATUS,
	    &igc->igc_phy_ext_status);
	(void) igc_read_phy_reg(&igc->igc_hw, PHY_EXT_STATUS,
	    &igc->igc_phy_ext_status);

	mmd_base = STANDARD_AN_REG_MASK << MMD_DEVADDR_SHIFT;
	(void) igc_read_phy_reg(&igc->igc_hw, mmd_base | ANEG_MULTIGBT_AN_CTRL,
	    &igc->igc_phy_mmd_ctrl);
	(void) igc_read_phy_reg(&igc->igc_hw, mmd_base | ANEG_MULTIGBT_AN_STS1,
	    &igc->igc_phy_mmd_sts);
	mutex_exit(&igc->igc_lock);

	if (orig_state != new_state) {
		mac_link_update(igc->igc_mac_hdl, new_state);
	}
}

static uint_t
igc_intr_rx_queue(caddr_t arg1, caddr_t arg2)
{
	igc_t *igc = (igc_t *)arg1;
	uintptr_t queue = (uintptr_t)arg2;
	igc_rx_ring_t *ring;
	mblk_t *mp = NULL;

	ASSERT3U(queue, <, igc->igc_nrx_rings);
	ring = &igc->igc_rx_rings[queue];

	mutex_enter(&ring->irr_lock);
	if ((ring->irr_flags & IGC_RXR_F_POLL) == 0) {
		mp = igc_ring_rx(ring, IGC_RX_POLL_INTR);
	}
	mutex_exit(&ring->irr_lock);

	if (mp != NULL) {
		mac_rx_ring(igc->igc_mac_hdl, ring->irr_rh, mp, ring->irr_gen);
	}

	return (DDI_INTR_CLAIMED);
}

static uint_t
igc_intr_tx_other(caddr_t arg1, caddr_t arg2)
{
	igc_t *igc = (igc_t *)arg1;
	uint32_t icr = igc_read32(igc, IGC_ICR);

	igc_tx_recycle(igc, &igc->igc_tx_rings[0]);

	if ((icr & IGC_ICR_LSC) != 0) {
		igc_intr_lsc(igc);
	}

	return (DDI_INTR_CLAIMED);
}

static bool
igc_setup_regs(igc_t *igc)
{
	int ret;
	ddi_device_acc_attr_t da;

	if (pci_config_setup(igc->igc_dip, &igc->igc_cfgspace) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to map config space");
		return (false);
	}

	if (ddi_dev_regsize(igc->igc_dip, IGC_PCI_BAR, &igc->igc_regs_size) !=
	    DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to get BAR %u size",
		    IGC_PCI_BAR - 1);
		return (false);
	}

	bzero(&da, sizeof (ddi_device_acc_attr_t));
	da.devacc_attr_version = DDI_DEVICE_ATTR_V1;
	da.devacc_attr_endian_flags = DDI_STRUCTURE_LE_ACC;
	da.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
	da.devacc_attr_access = DDI_DEFAULT_ACC;

	if ((ret = ddi_regs_map_setup(igc->igc_dip, IGC_PCI_BAR,
	    &igc->igc_regs_base, 0, igc->igc_regs_size, &da,
	    &igc->igc_regs_hdl)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to map registers: %d",
		    ret);
		return (false);
	}

	return (true);
}

/*
 * Go through the process of initializing the igc core code. First we have to
 * fill in the information that the common code requires to identify the
 * hardware and set the mac type. After that we can go through and set up all of
 * the function initialization.
 */
static bool
igc_core_code_init(igc_t *igc)
{
	int ret;
	int *regs;
	uint_t nprop;

	igc->igc_hw.back = igc;
	igc->igc_hw.vendor_id = pci_config_get16(igc->igc_cfgspace,
	    PCI_CONF_VENID);
	igc->igc_hw.device_id = pci_config_get16(igc->igc_cfgspace,
	    PCI_CONF_DEVID);
	igc->igc_hw.revision_id = pci_config_get8(igc->igc_cfgspace,
	    PCI_CONF_REVID);
	igc->igc_hw.subsystem_vendor_id = pci_config_get16(igc->igc_cfgspace,
	    PCI_CONF_SUBVENID);
	igc->igc_hw.subsystem_device_id = pci_config_get16(igc->igc_cfgspace,
	    PCI_CONF_SUBSYSID);

	if ((ret = ddi_prop_lookup_int_array(DDI_DEV_T_ANY, igc->igc_dip,
	    DDI_PROP_DONTPASS, "reg", &regs, &nprop)) != DDI_PROP_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to look up 'reg' "
		    "property: %d", ret);
		return (false);
	}

	/*
	 * We fill out the function and command word. We currently don't fill
	 * out the bus type, speed, and width as it's not used by the common
	 * code, leaving it all at unknown. We can grab that information when it
	 * needs it. We do fill out the function and command word as the former
	 * is important and the latter is easy to grab.
	 */
	igc->igc_hw.bus.func = PCI_REG_FUNC_G(regs[0]);
	igc->igc_hw.bus.pci_cmd_word = pci_config_get16(igc->igc_cfgspace,
	    PCI_CONF_COMM);
	ddi_prop_free(regs);

	/*
	 * The common code asks for the memory mapped address to be set in its
	 * structure. Though in theory it promises not to use it.
	 */
	igc->igc_hw.hw_addr = (uint8_t *)igc->igc_regs_base;

	if ((ret = igc_set_mac_type(&igc->igc_hw)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to set mac type: %d",
		    ret);
		return (false);
	}

	if ((ret = igc_setup_init_funcs(&igc->igc_hw, true)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to setup core code "
		    "function pointers: %d", ret);
		return (false);
	}

	/*
	 * Go ahead and attempt to get the bus information even though this
	 * doesn't actually do anything right now.
	 */
	if ((ret = igc_get_bus_info(&igc->igc_hw)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "core code failed to get bus "
		    "info: %d", ret);
		return (false);
	}

	return (true);
}

static bool
igc_limits_init(igc_t *igc)
{
	switch (igc->igc_hw.mac.type) {
	case igc_i225:
		igc->igc_limits.il_max_rx_rings = IGC_MAX_RX_RINGS_I225;
		igc->igc_limits.il_max_tx_rings = IGC_MAX_RX_RINGS_I225;
		igc->igc_limits.il_max_mtu = IGC_MAX_MTU_I225;
		break;
	default:
		dev_err(igc->igc_dip, CE_WARN, "unknown MAC type: %u",
		    igc->igc_hw.mac.type);
		return (false);
	}

	return (true);
}

/*
 * Determine the hardware buffer sizes that are required for the given MTU.
 * There are a few different constraints that we try to enforce here that come
 * from the hardware and others that come from us:
 *
 * 1) The hardware requires that the rx and tx sizes all be 1 KiB (0x400) byte
 * aligned.
 * 2) Our tx engine can handle copying across multiple descriptors, so we cap
 * the maximum tx buffer size at one page.
 * 3) Right now our rx engine does not handle scanning multiple buffers for rx
 * (see the theory statement), so we end up making the rx buffer have to fix the
 * maximum frame size.
 * 4) rx buffers need to also account for IP alignment, so we make sure to
 * allocate extra bytes for that.
 */
void
igc_hw_buf_update(igc_t *igc)
{
	unsigned long pagesize = ddi_ptob(igc->igc_dip, 1);
	uint32_t tx_mtu;

	igc->igc_max_frame = igc->igc_mtu + sizeof (struct ether_vlan_header) +
	    ETHERFCSL;
	igc->igc_rx_buf_size = P2ROUNDUP_TYPED(igc->igc_max_frame +
	    IGC_RX_BUF_IP_ALIGN, IGC_BUF_ALIGN, uint32_t);
	tx_mtu = P2ROUNDUP_TYPED(igc->igc_max_frame, IGC_BUF_ALIGN, uint32_t);
	igc->igc_tx_buf_size = MIN(tx_mtu, pagesize);
}

static bool
igc_intr_init(igc_t *igc)
{
	int ret, types, nintrs, navail, req;
	const int min_nintrs = 2;

	if ((ret = ddi_intr_get_supported_types(igc->igc_dip, &types)) !=
	    DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to get supported "
		    "interrupts: %d", ret);
		return (false);
	}

	/*
	 * For now, we simplify our lives and device support by only supporting
	 * MSI-X interrupts. When we find versions of this without MSI-X
	 * support, we can go and add what we need.
	 */
	if ((types & DDI_INTR_TYPE_MSIX) == 0) {
		dev_err(igc->igc_dip, CE_WARN, "device does not support MSI-X, "
		    "found %d", types);
		return (false);
	}

	if ((ret = ddi_intr_get_nintrs(igc->igc_dip, DDI_INTR_TYPE_MSIX,
	    &nintrs)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to get number of "
		    "supported MSI-X interrupts: %d", ret);
		return (false);
	}

	if (nintrs < min_nintrs) {
		dev_err(igc->igc_dip, CE_WARN, "igc driver currently requires "
		    "%d MSI-X interrupts be supported, found %d", min_nintrs,
		    nintrs);
		return (false);
	}

	if ((ret = ddi_intr_get_navail(igc->igc_dip, DDI_INTR_TYPE_MSIX,
	    &navail)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to get number of "
		    "available MSI-X interrupts: %d", ret);
		return (false);
	}

	if (navail < min_nintrs) {
		dev_err(igc->igc_dip, CE_WARN, "igc driver currently requires "
		    "%d MSI-X interrupts be available, found %d", min_nintrs,
		    navail);
		return (false);
	}

	/*
	 * In the future this could be based upon the multiple queues that the
	 * device supports, but for now it's limited to two. See 'Rings and
	 * Interrupts' in the theory statement for more background.
	 */
	req = min_nintrs;
	req = MIN(req, navail);
	igc->igc_intr_size = req * sizeof (ddi_intr_handle_t);
	igc->igc_intr_handles = kmem_alloc(igc->igc_intr_size, KM_SLEEP);

	if ((ret = ddi_intr_alloc(igc->igc_dip, igc->igc_intr_handles,
	    DDI_INTR_TYPE_MSIX, 0, req, &igc->igc_nintrs,
	    DDI_INTR_ALLOC_NORMAL)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to allocate interrupts: "
		    "%d", ret);
		return (false);
	}

	igc->igc_intr_type = DDI_INTR_TYPE_MSIX;
	igc->igc_attach |= IGC_ATTACH_INTR_ALLOC;
	if (igc->igc_nintrs < min_nintrs) {
		dev_err(igc->igc_dip, CE_WARN, "received %d interrupts, but "
		    "needed at least %d", igc->igc_nintrs, min_nintrs);
		return (false);
	}

	if ((ret = ddi_intr_get_pri(igc->igc_intr_handles[0],
	    &igc->igc_intr_pri)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to get interrupt "
		    "priority: %d", ret);
		return (false);
	}

	if ((ret = ddi_intr_get_cap(igc->igc_intr_handles[0],
	    &igc->igc_intr_cap)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to get interrupt "
		    "capabilities: %d", ret);
		return (false);
	}

	return (true);
}

/*
 * As part of allocating our rings we make the following assumptions about
 * interrupt assignments. All tx rings share interrupt 0. All rx rings have
 * separate interrupts starting from interrupt 1. This design may likely change
 * in the face of actual multi-ring support
 */
static bool
igc_rings_alloc(igc_t *igc)
{
	uint32_t intr = 0;
	igc->igc_tx_rings = kmem_zalloc(sizeof (igc_tx_ring_t) *
	    igc->igc_ntx_rings, KM_SLEEP);

	for (uint32_t i = 0; i < igc->igc_ntx_rings; i++) {
		igc->igc_tx_rings[i].itr_igc = igc;
		igc->igc_tx_rings[i].itr_idx = i;
		igc->igc_tx_rings[i].itr_intr_idx = intr;
		mutex_init(&igc->igc_tx_rings[i].itr_lock, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(igc->igc_intr_pri));
		if (!igc_tx_ring_stats_init(igc, &igc->igc_tx_rings[i])) {
			return (false);
		}
	}

	igc->igc_rx_rings = kmem_zalloc(sizeof (igc_rx_ring_t) *
	    igc->igc_nrx_rings, KM_SLEEP);
	intr = 1;

	for (uint32_t i = 0; i < igc->igc_nrx_rings; i++, intr++) {
		igc->igc_rx_rings[i].irr_igc = igc;
		igc->igc_rx_rings[i].irr_idx = i;
		igc->igc_rx_rings[i].irr_intr_idx = intr;
		mutex_init(&igc->igc_rx_rings[i].irr_lock, NULL, MUTEX_DRIVER,
		    DDI_INTR_PRI(igc->igc_intr_pri));
		mutex_init(&igc->igc_rx_rings[i].irr_free_lock, NULL,
		    MUTEX_DRIVER, DDI_INTR_PRI(igc->igc_intr_pri));
		cv_init(&igc->igc_rx_rings[i].irr_free_cv, NULL, CV_DRIVER,
		    NULL);
		if (!igc_rx_ring_stats_init(igc, &igc->igc_rx_rings[i])) {
			return (false);
		}
	}

	ASSERT3U(intr, ==, igc->igc_nintrs);

	return (true);
}

/*
 * Allocate our interrupts. Note, we have more or less constrained the device
 * right now to only request two interrupts which we use in a fixed way. If we
 * end up with more varied queue support then this should be changed around.
 */
static bool
igc_intr_hdlr_init(igc_t *igc)
{
	int ret;

	if ((ret = ddi_intr_add_handler(igc->igc_intr_handles[0],
	    igc_intr_tx_other, igc, NULL)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to add tx/other "
		    "interrupt handler: %d", ret);
		return (false);
	}

	if ((ret = ddi_intr_add_handler(igc->igc_intr_handles[1],
	    igc_intr_rx_queue, igc, (uintptr_t)0)) != DDI_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to add rx interrupt "
		    "handler: %d", ret);
		if ((ret = ddi_intr_remove_handler(igc->igc_intr_handles[0])) !=
		    DDI_SUCCESS) {
			dev_err(igc->igc_dip, CE_WARN, "failed to remove "
			    "tx/other interrupt handler");
		}
		return (false);
	}

	return (true);
}

static void
igc_hw_control(igc_t *igc, bool take)
{
	uint32_t ctrl = igc_read32(igc, IGC_CTRL_EXT);

	if (take) {
		ctrl |= IGC_CTRL_EXT_DRV_LOAD;
	} else {
		ctrl &= ~IGC_CTRL_EXT_DRV_LOAD;
	}

	igc_write32(igc, IGC_CTRL_EXT, ctrl);
}

/*
 * Basic device initialization and sanity check. This covers that we can
 * properly reset the device, validate its checksum, and get a valid MAC
 * address.
 */
static bool
igc_hw_init(igc_t *igc)
{
	int ret;
	uint32_t eecd;

	if ((ret = igc_reset_hw(&igc->igc_hw)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to reset device: %d",
		    ret);
		return (false);
	}

	/*
	 * Goodbye firmware.
	 */
	igc_hw_control(igc, true);

	/*
	 * Check the NVM validiity if a device is present.
	 */
	eecd = igc_read32(igc, IGC_EECD);
	if ((eecd & IGC_EECD_EE_DET) != 0) {
		if ((ret = igc_validate_nvm_checksum(&igc->igc_hw)) !=
		    IGC_SUCCESS) {
			dev_err(igc->igc_dip, CE_WARN, "failed to validate "
			    "igc NVM checksum: %d", ret);
			return (false);
		}
	}

	if ((ret = igc_read_mac_addr(&igc->igc_hw)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to read MAC address: %d",
		    ret);
		return (false);
	}

	if ((ret = igc_get_phy_id(&igc->igc_hw)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to get PHY id: %d", ret);
		return (false);
	}

	return (true);
}

/*
 * In case the user has modified the LED state through MAC_CAPAB_LED, restore
 * that back to the defaults we got when we started up the device.
 */
static void
igc_led_fini(igc_t *igc)
{
	igc_write32(igc, IGC_LEDCTL, igc->igc_ledctl);
}

/*
 * Traditionally the Intel NIC drivers avoid touching activity pins as part of
 * their behavior for what we use. We also don't touch a pin if it's in SDP mode
 * and not being used to drive an LED as it means it's likely not for us.
 */
static bool
igc_led_ignore(i225_led_mode_t mode)
{
	switch (mode) {
	case I225_LED_M_FILTER_ACT:
	case I225_LED_M_LINK_ACT:
	case I225_LED_M_SDP:
	case I225_LED_M_PAUSE:
	case I225_LED_M_ACT:
		return (true);
	default:
		return (false);
	}
}

static inline uint32_t
igc_led_bitoff(uint32_t led)
{
	VERIFY3U(led, <, 3);
	return (led * 8);
}

static inline uint32_t
igc_led_get_mode(uint32_t led, uint32_t reg)
{
	uint32_t off = igc_led_bitoff(led);
	return (bitx32(reg, 3 + off, off));
}

static inline uint32_t
igc_led_set_mode(uint32_t led, uint32_t reg, i225_led_mode_t mode)
{
	uint32_t off = igc_led_bitoff(led);
	return (bitset32(reg, 3 + off, off, mode));
}

static inline uint32_t
igc_led_get_ivrt(uint32_t led, uint32_t reg)
{
	uint32_t off = igc_led_bitoff(led) + 6;
	return (bitx32(reg, off, off));
}

static inline uint32_t
igc_led_set_blink(uint32_t led, uint32_t reg, bool en)
{
	uint32_t off = igc_led_bitoff(led) + 7;
	return (bitset32(reg, off, off, en));
}

/*
 * There are three LEDs on the chip. The reference defines LED0 for 1 GbE link
 * up, LED1 for a 2.5GbE link up, and LED 2 for activity. However, this is all
 * controllable in the NVM so we shouldn't assume that these have any of their
 * default values. We instead read the LEDCTL register to see how it was set up
 * by default (though the NVM would likely be better). We then create pre-canned
 * LEDCTL register values for on, off, and default. See igc_osdep.h for some of
 * the caveats in definitions here. Note, we only tweak the non-activity LEDs
 * and if an LED has been indicated that it's being used for SDP, we don't touch
 * it.
 */
static void
igc_led_init(igc_t *igc)
{
	uint32_t led = igc_read32(igc, IGC_LEDCTL);

	igc->igc_ledctl = led;
	igc->igc_ledctl_on = led;
	igc->igc_ledctl_off = led;
	igc->igc_ledctl_blink = led;

	for (uint32_t i = 0; i < IGC_I225_NLEDS; i++) {
		i225_led_mode_t mode = igc_led_get_mode(i, led);
		if (!igc_led_ignore(mode)) {
			/*
			 * If the inversion logic is on, that changes what the
			 * on and off modes mean, so we need to change how we
			 * set that appropriately.
			 */
			if (igc_led_get_ivrt(i, led) != 0) {
				igc->igc_ledctl_on = igc_led_set_mode(i,
				    igc->igc_ledctl_on, I225_LED_M_OFF);
				igc->igc_ledctl_off = igc_led_set_mode(i,
				    igc->igc_ledctl_off, I225_LED_M_ON);
				igc->igc_ledctl_blink = igc_led_set_mode(i,
				    igc->igc_ledctl_blink, I225_LED_M_OFF);
			} else {
				igc->igc_ledctl_on = igc_led_set_mode(i,
				    igc->igc_ledctl_on, I225_LED_M_ON);
				igc->igc_ledctl_off = igc_led_set_mode(i,
				    igc->igc_ledctl_off, I225_LED_M_OFF);
				igc->igc_ledctl_blink = igc_led_set_mode(i,
				    igc->igc_ledctl_blink, I225_LED_M_ON);
			}
		}

		igc->igc_ledctl_blink = igc_led_set_blink(i,
		    igc->igc_ledctl_blink, true);
	}

	igc->igc_led_mode = MAC_LED_DEFAULT;
}

static void
igc_write_ivar(igc_t *igc, uint32_t queue, bool rx, uint32_t msix)
{
	const uint32_t ivarno = queue >> 1;
	const uint32_t reg = IGC_IVAR0 + ivarno * 4;
	const uint32_t val = msix | IGC_IVAR_VALID;
	uint32_t bitoff, bitend, ivar;

	if (rx) {
		if ((queue % 2) == 0) {
			bitoff = IGC_IVAR_RX0_START;
		} else {
			bitoff = IGC_IVAR_RX1_START;
		}
	} else {
		if ((queue % 2) == 0) {
			bitoff = IGC_IVAR_TX0_START;
		} else {
			bitoff = IGC_IVAR_TX1_START;
		}
	}
	bitend = bitoff + IGC_IVAR_ENT_LEN - 1;

	ivar = igc_read32(igc, reg);
	ivar = bitset32(ivar, bitend, bitoff, val);
	igc_write32(igc, reg, ivar);
	igc->igc_eims |= 1 << msix;
}

/*
 * Here we need to go through and initialize the hardware's notion of how
 * interrupts are mapped to causes. The device must be specifically enabled for
 * MSI-X and then this is also where we go ensure that all of our interrupt
 * coalescing is properly enabled. Note, we must first touch the GPIE register
 * to enable MSI-X settings otherwise later settings won't do anything.
 */
static void
igc_hw_intr_init(igc_t *igc)
{
	uint32_t gpie, ivar;

	gpie = IGC_GPIE_NSICR | IGC_GPIE_MSIX_MODE | IGC_GPIE_EIAME |
	    IGC_GPIE_PBA;
	igc_write32(igc, IGC_GPIE, gpie);

	/*
	 * Other causes are always explicitly mapped to cause 0. Each ring then
	 * has its own mapping. In the MISC IVAR, these start at bit 8. We leave
	 * the '0 |' out below just to avoid a compiler complaining. We also
	 * must unamsk this interrupt cause, which is in bit 0.
	 */
	ivar = IGC_IVAR_VALID << 8;
	igc_write32(igc, IGC_IVAR_MISC, ivar);
	igc->igc_eims = 1;

	/*
	 * There are a few IVAR registers available in hardware. Each IVAR
	 * register handles mapping a given queue to an MSI-X. Each IVAR handles
	 * two queues.
	 */
	for (uint32_t i = 0; i < igc->igc_ntx_rings; i++) {
		igc_write_ivar(igc, i, false,
		    igc->igc_tx_rings[i].itr_intr_idx);
	}

	for (uint32_t i = 0; i < igc->igc_nrx_rings; i++) {
		igc_write_ivar(igc, i, true, igc->igc_rx_rings[i].irr_intr_idx);
	}

	for (uint32_t i = 0; i < igc->igc_nintrs; i++) {
		igc_write32(igc, IGC_EITR(i), igc->igc_eitr);
	}
}

/*
 * Synchronize our sense of the unicast table over to the device. If this is the
 * first time that we're here due to attach, we need to go through and allocate
 * the tracking table.
 */
static void
igc_unicast_sync(igc_t *igc)
{
	ASSERT(MUTEX_HELD(&igc->igc_lock));

	if (igc->igc_ucast == NULL) {
		igc->igc_nucast = igc->igc_hw.mac.rar_entry_count;
		igc->igc_ucast = kmem_zalloc(sizeof (igc_addr_t) *
		    igc->igc_nucast, KM_SLEEP);
	}

	for (uint16_t i = 0; i < igc->igc_nucast; i++) {
		int ret = igc_rar_set(&igc->igc_hw, igc->igc_ucast[i].ia_mac,
		    i);
		/*
		 * Common code today guarantees this can't fail. Put this here
		 * to ensure to guard against future updates.
		 */
		VERIFY3S(ret, ==, IGC_SUCCESS);
	}

}

/*
 * The core code interface to the multicast table requires us to give them a
 * packed uint8_t array that they manually walk through in ETHERADDRL (6 byte)
 * chunks. This must be packed. To deal with this we opt to preserve a normal
 * list of multicast addresses and then a secondary version that's serialized as
 * the core code wants it. We allocate the memory for this secondary version at
 * the start.
 */
void
igc_multicast_sync(igc_t *igc)
{
	uint16_t nvalid;

	ASSERT(MUTEX_HELD(&igc->igc_lock));

	if (igc->igc_mcast == NULL) {
		igc->igc_nmcast = igc->igc_hw.mac.mta_reg_count;
		igc->igc_mcast = kmem_zalloc(sizeof (igc_addr_t) *
		    igc->igc_nmcast, KM_SLEEP);
		igc->igc_mcast_raw = kmem_alloc(sizeof (ether_addr_t) *
		    igc->igc_nmcast, KM_SLEEP);
	}

	bzero(igc->igc_mcast_raw, sizeof (ether_addr_t) * igc->igc_nmcast);
	nvalid = 0;
	for (uint16_t i = 0; i < igc->igc_nmcast; i++) {
		ether_addr_t *targ = &igc->igc_mcast_raw[nvalid];

		if (!igc->igc_mcast[i].ia_valid)
			continue;
		bcopy(igc->igc_mcast[i].ia_mac, targ, sizeof (ether_addr_t));
		nvalid++;
	}

	igc_update_mc_addr_list(&igc->igc_hw, (uint8_t *)igc->igc_mcast_raw,
	    nvalid);
}

/*
 * This function is used to reinitialize the PBA, our various flow control
 * settings, reset hardware, ensure that the EEE, DPLU, and related power modes
 * are in the correct state.
 */
bool
igc_hw_common_init(igc_t *igc)
{
	int ret;
	uint32_t pba, hwm, hwmp, hwm2x;
	struct igc_hw *hw = &igc->igc_hw;

	/*
	 * The PBA register determines which portion is used for the receive
	 * buffers and which is used for the transmit buffers. This follows from
	 * the I210 and reference drivers which use 34K as the default. We
	 * currently leave the RXPBS and TXPBS at their power-on-reset defaults.
	 *
	 * We set the watermark based settings similar to igb, ensuring that we
	 * have 16-byte granularity. The general guidelines from there was that
	 * when it comes to automatic Ethernet PAUSE frame generation we should:
	 *
	 * - After an XOFF, you want to receive at least two frames. We use
	 *   whichever is smaller of 9/10ths and two frames.
	 * - The low water mark apparently wants to be closer to the high water
	 *   mark.
	 *
	 * See igb_init_adapter() for more information. We basically use the
	 * same calculation it did, given that the MAC is basically the same.
	 */
	pba = IGC_PBA_34K;
	hwmp = (pba << 10) * 9 / 10;
	hwm2x = (pba << 10) - 2 * igc->igc_max_frame;
	hwm = MIN(hwmp, hwm2x);

	hw->fc.high_water = hwm & 0xfffffff0;
	hw->fc.low_water = igc->igc_hw.fc.high_water - 16;

	/*
	 * Use the suggested default pause time.
	 */
	hw->fc.pause_time = IGC_FC_PAUSE_TIME;
	hw->fc.send_xon = true;

	if ((ret = igc_reset_hw(hw)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to reset device: %d",
		    ret);
		return (false);
	}

	if ((ret = igc_init_hw(hw)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to init hardware: %d",
		    ret);
		return (false);
	}

	/*
	 * Clear wake on LAN and set other power states. In addition, disable
	 * EEE for now.
	 */
	igc_write32(igc, IGC_WUC, 0);

	if ((ret = igc_set_d0_lplu_state(hw, false)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to set D0 LPLU mode: %d",
		    ret);
		return (false);
	}

	/*
	 * There have been reports that enabling EEE for some 2.5G devices has
	 * led to issues with the I225/226. It's not entirely clear, but we
	 * default to disabling this like in igb/e1000g for now.
	 */
	if ((ret = igc_set_eee_i225(hw, false, false, false)) != IGC_SUCCESS) {
		dev_err(igc->igc_dip, CE_WARN, "failed to set EEE mode: %d",
		    ret);
		return (false);
	}

	igc_hw_intr_init(igc);

	mutex_enter(&igc->igc_lock);
	igc_unicast_sync(igc);
	igc_multicast_sync(igc);

	igc->igc_hw.mac.get_link_status = true;
	(void) igc_get_phy_info(hw);
	(void) igc_check_for_link(hw);
	mutex_exit(&igc->igc_lock);

	return (true);
}

static bool
igc_intr_en(igc_t *igc)
{
	int ret;

	if ((igc->igc_intr_cap & DDI_INTR_FLAG_BLOCK) != 0) {
		ret = ddi_intr_block_enable(igc->igc_intr_handles,
		    igc->igc_nintrs);
		if (ret != DDI_SUCCESS) {
			dev_err(igc->igc_dip, CE_WARN, "failed to block "
			    "enable interrupts: %d", ret);
			return (false);
		}
	} else {
		for (int i = 0; i < igc->igc_nintrs; i++) {
			ret = ddi_intr_enable(igc->igc_intr_handles[i]);
			if (ret != DDI_SUCCESS) {
				dev_err(igc->igc_dip, CE_WARN, "failed to "
				    "enable interrupt %d: %d", i, ret);
				for (int clean = 0; clean < i; clean++) {
					ret = ddi_intr_disable(
					    igc->igc_intr_handles[clean]);
					if (ret != DDI_SUCCESS) {
						dev_err(igc->igc_dip, CE_WARN,
						    "failed to disable "
						    "interrupt %d while "
						    "unwinding: %d", i, ret);
					}
				}
				return (false);
			}
		}
	}

	/*
	 * Now that we've enabled interrupts here, clear any pending interrupts
	 * and make sure hardware interrupts are enabled.
	 */
	(void) igc_read32(igc, IGC_ICR);

	return (true);
}

/*
 * Undo interrupt enablement.
 */
void
igc_hw_intr_disable(igc_t *igc)
{
	igc_write32(igc, IGC_EIMC, UINT32_MAX);
	igc_write32(igc, IGC_EIAC, 0);
	igc_write32(igc, IGC_IMC, UINT32_MAX);
}

/*
 * This is used during the GLDv3 mc_start(9E) entry point to enable interrupts
 * on the device itself.
 */
void
igc_hw_intr_enable(igc_t *igc)
{
	uint32_t ims;

	/*
	 * First we clear pending interrupts.
	 */
	(void) igc_read32(igc, IGC_ICR);

	/*
	 * The hardware has extended and non-extended interrupt masks and
	 * auto-clear registers. We always disable auto-clear for the
	 * non-extended portions. See the I210 datasheet 'Setting Interrupt
	 * Registers' for a better sense of what's going on here.
	 *
	 * In the IMS register we always register link status change events and
	 * device reset assertions.
	 */
	ims = IGC_IMS_LSC | IGC_IMS_DRSTA;

	igc_write32(igc, IGC_EIAC, igc->igc_eims);
	igc_write32(igc, IGC_EIMS, igc->igc_eims);
	igc_write32(igc, IGC_IMS, ims);
	igc_write32(igc, IGC_IAM, 0);
}

static void
igc_cleanup(igc_t *igc)
{
	if (igc->igc_mcast != NULL) {
		ASSERT3U(igc->igc_nmcast, !=, 0);
		kmem_free(igc->igc_mcast_raw, sizeof (ether_addr_t) *
		    igc->igc_nmcast);
		kmem_free(igc->igc_mcast, sizeof (igc_addr_t) *
		    igc->igc_nmcast);
		igc->igc_nmcast = 0;
		igc->igc_mcast = NULL;
	}

	if (igc->igc_ucast != NULL) {
		ASSERT3U(igc->igc_nucast, !=, 0);
		kmem_free(igc->igc_ucast, sizeof (igc_addr_t) *
		    igc->igc_nucast);
		igc->igc_nucast = 0;
		igc->igc_ucast = NULL;
	}

	if ((igc->igc_attach & IGC_ATTACH_INTR_EN) != 0) {
		int ret;
		if ((igc->igc_intr_cap & DDI_INTR_FLAG_BLOCK) != 0) {
			ret = ddi_intr_block_disable(igc->igc_intr_handles,
			    igc->igc_nintrs);
			if (ret != DDI_SUCCESS) {
				dev_err(igc->igc_dip, CE_WARN, "failed to "
				    "block disable interrupts: %d", ret);
			}
		} else {
			for (int i = 0; i < igc->igc_nintrs; i++) {
				ret = ddi_intr_disable(
				    igc->igc_intr_handles[i]);
				if (ret != DDI_SUCCESS) {
					dev_err(igc->igc_dip, CE_WARN, "failed "
					    "to disable interrupt %d: %d", i,
					    ret);
				}
			}
		}
		igc->igc_attach &= ~IGC_ATTACH_INTR_EN;
	}

	if ((igc->igc_attach & IGC_ATTACH_MAC) != 0) {
		int ret = mac_unregister(igc->igc_mac_hdl);
		if (ret != 0) {
			dev_err(igc->igc_dip, CE_WARN, "failed to unregister "
			    "MAC handle: %d", ret);
		}
		igc->igc_attach &= ~IGC_ATTACH_MAC;
	}

	if ((igc->igc_attach & IGC_ATTACH_STATS) != 0) {
		igc_stats_fini(igc);
		igc->igc_attach &= ~IGC_ATTACH_STATS;
	}

	if ((igc->igc_attach & IGC_ATTACH_LED) != 0) {
		igc_led_fini(igc);
		igc->igc_attach &= ~IGC_ATTACH_LED;
	}

	if ((igc->igc_attach & IGC_ATTACH_INTR_HANDLER) != 0) {
		for (int i = 0; i < igc->igc_nintrs; i++) {
			int ret =
			    ddi_intr_remove_handler(igc->igc_intr_handles[i]);
			if (ret != 0) {
				dev_err(igc->igc_dip, CE_WARN, "failed to "
				    "remove interrupt %d handler: %d", i, ret);
			}
		}
		igc->igc_attach &= ~IGC_ATTACH_INTR_HANDLER;
	}

	if (igc->igc_tx_rings != NULL) {
		for (uint32_t i = 0; i < igc->igc_ntx_rings; i++) {
			igc_tx_ring_stats_fini(&igc->igc_tx_rings[i]);
			mutex_destroy(&igc->igc_tx_rings[i].itr_lock);
		}
		kmem_free(igc->igc_tx_rings, sizeof (igc_tx_ring_t) *
		    igc->igc_ntx_rings);
		igc->igc_tx_rings = NULL;
	}

	if (igc->igc_rx_rings != NULL) {
		for (uint32_t i = 0; i < igc->igc_nrx_rings; i++) {
			igc_rx_ring_stats_fini(&igc->igc_rx_rings[i]);
			cv_destroy(&igc->igc_rx_rings[i].irr_free_cv);
			mutex_destroy(&igc->igc_rx_rings[i].irr_free_lock);
			mutex_destroy(&igc->igc_rx_rings[i].irr_lock);
		}
		kmem_free(igc->igc_rx_rings, sizeof (igc_rx_ring_t) *
		    igc->igc_nrx_rings);
		igc->igc_rx_rings = NULL;
	}

	if ((igc->igc_attach & IGC_ATTACH_MUTEX) != 0) {
		mutex_destroy(&igc->igc_lock);
		igc->igc_attach &= ~IGC_ATTACH_MUTEX;
	}

	if ((igc->igc_attach & IGC_ATTACH_INTR_ALLOC) != 0) {
		for (int i = 0; i < igc->igc_nintrs; i++) {
			int ret = ddi_intr_free(igc->igc_intr_handles[i]);
			if (ret != DDI_SUCCESS) {
				dev_err(igc->igc_dip, CE_WARN, "unexpected "
				    "failure freeing interrupt %d: %d", i, ret);
			}
		}
		igc->igc_attach &= ~IGC_ATTACH_INTR_ALLOC;
	}

	if (igc->igc_intr_handles != NULL) {
		ASSERT3U(igc->igc_intr_size, !=, 0);
		kmem_free(igc->igc_intr_handles, igc->igc_intr_size);
	}

	/*
	 * Now that we're almost done, begrudgingly let firmware know we're
	 * done.
	 */
	igc_hw_control(igc, false);

	if (igc->igc_regs_hdl != NULL) {
		ddi_regs_map_free(&igc->igc_regs_hdl);
		igc->igc_regs_base = NULL;
	}

	if (igc->igc_cfgspace != NULL) {
		pci_config_teardown(&igc->igc_cfgspace);
	}
	igc->igc_attach &= ~IGC_ATTACH_REGS;

	ddi_set_driver_private(igc->igc_dip, NULL);
	igc->igc_dip = NULL;

	VERIFY0(igc->igc_attach);

	kmem_free(igc, sizeof (igc_t));
}

static int
igc_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
	igc_t *igc;

	if (cmd != DDI_ATTACH) {
		return (DDI_FAILURE);
	}

	igc = kmem_zalloc(sizeof (igc_t), KM_SLEEP);
	ddi_set_driver_private(dip, igc);
	igc->igc_dip = dip;

	/*
	 * Initialize a few members that are not zero-based.
	 */
	igc->igc_link_duplex = LINK_DUPLEX_UNKNOWN;
	igc->igc_link_state = LINK_STATE_UNKNOWN;

	/*
	 * Set up all the register spaces that hardware requires.
	 */
	if (!igc_setup_regs(igc)) {
		goto err;
	}
	igc->igc_attach |= IGC_ATTACH_REGS;

	/*
	 * Setup the common code.
	 */
	if (!igc_core_code_init(igc)) {
		goto err;
	}

	if (!igc_limits_init(igc)) {
		goto err;
	}

	/*
	 * Go allocate and set up all of our interrupts.
	 */
	if (!igc_intr_init(igc)) {
		goto err;
	}

	/*
	 * Initialize our main mutex for the device now that we have an
	 * interrupt priority.
	 */
	mutex_init(&igc->igc_lock, NULL, MUTEX_DRIVER,
	    DDI_INTR_PRI(igc->igc_intr_pri));
	igc->igc_attach |= IGC_ATTACH_MUTEX;

	/*
	 * We now want to determine the total number of rx and tx rings that we
	 * have based on our interrupt allocation so we can go through and
	 * perform the rest of the device setup that is required. The various
	 * queues that we have are mapped to a given MSI-X through the IVAR
	 * registers in the device. There is also an IVAR_MISC register that
	 * maps link state change events and other issues up to two vectors.
	 *
	 * There isn't strictly per-queue interrupt generation control. Instead,
	 * when in MSI-X mode, the device has an extended interrupt cause and
	 * mask register. The mask register allows us to mask the five bits
	 * described above.
	 *
	 * Because of all this we end up limiting the number of queues that we
	 * use to 2 for now: 1 for tx and 1 for rx. Interrupt 0 is for tx/other
	 * and 1 for rx.
	 */
	igc->igc_nrx_rings = 1;
	igc->igc_ntx_rings = 1;

	/*
	 * Default to a 1500 byte MTU.
	 */
	igc->igc_mtu = ETHERMTU;
	igc_hw_buf_update(igc);

	/*
	 * Initialize default descriptor limits and thresholds. We allocate 1.5
	 * times the number of rx descriptors so that way we can loan up to
	 * 1/3rd of them. We allocate an even number of tx descriptors.
	 */
	igc->igc_rx_ndesc = IGC_DEF_RX_RING_SIZE;
	igc->igc_tx_ndesc = IGC_DEF_TX_RING_SIZE;
	igc->igc_rx_nbuf = igc->igc_rx_ndesc + (igc->igc_rx_ndesc >> 1);
	igc->igc_tx_nbuf = igc->igc_tx_ndesc;
	igc->igc_rx_nfree = igc->igc_rx_nbuf - igc->igc_rx_ndesc;
	igc->igc_rx_intr_nframes = IGC_DEF_RX_RING_INTR_LIMIT;
	igc->igc_rx_bind_thresh = IGC_DEF_RX_BIND;
	igc->igc_tx_bind_thresh = IGC_DEF_TX_BIND;
	igc->igc_tx_notify_thresh = IGC_DEF_TX_NOTIFY_MIN;
	igc->igc_tx_recycle_thresh = IGC_DEF_TX_RECYCLE_MIN;
	igc->igc_tx_gap = IGC_DEF_TX_GAP;
	igc->igc_eitr = IGC_DEF_EITR;

	if (!igc_rings_alloc(igc)) {
		goto err;
	}

	if (!igc_intr_hdlr_init(igc)) {
		goto err;
	}
	igc->igc_attach |= IGC_ATTACH_INTR_HANDLER;

	/*
	 * Next reset the device before we begin initializing anything else. As
	 * part of this, validate the flash checksum if present. This is all
	 * initialization that we would only do once per device. Other
	 * initialization that we want to do after any reset is done is
	 * igc_hw_common_init().
	 */
	if (!igc_hw_init(igc)) {
		goto err;
	}

	igc_led_init(igc);
	igc->igc_attach |= IGC_ATTACH_LED;

	/*
	 * Snapshot our basic settings that users can eventually control in the
	 * device. We start with always enabling auto-negotiation and
	 * advertising the basic supported speeds. The I225v1 does have
	 * substantial problems with enabling 2.5G due to the fact that it
	 * doesn't maintain a proper inter-packet gap. Despite that, we default
	 * to enabling 2.5G for now as its supposedly not broken with all link
	 * partners and the NVM. We also don't have a way of actually
	 * identifying and mapping that to something in the driver today,
	 * unfortunately.
	 */
	igc->igc_hw.mac.autoneg = true;
	igc->igc_hw.phy.autoneg_wait_to_complete = false;
	igc->igc_hw.phy.autoneg_advertised = IGC_DEFAULT_ADV;
	igc->igc_hw.fc.requested_mode = igc_fc_default;
	igc->igc_hw.fc.current_mode = igc_fc_default;

	if (!igc_hw_common_init(igc)) {
		goto err;
	}

	if (!igc_stats_init(igc)) {
		goto err;
	}
	igc->igc_attach |= IGC_ATTACH_STATS;

	/*
	 * Register with MAC
	 */
	if (!igc_mac_register(igc)) {
		goto err;
	}
	igc->igc_attach |= IGC_ATTACH_MAC;

	/*
	 * Enable interrupts and get going.
	 */
	if (!igc_intr_en(igc)) {
		goto err;
	}
	igc->igc_attach |= IGC_ATTACH_INTR_EN;

	return (DDI_SUCCESS);

err:
	igc_cleanup(igc);
	return (DDI_FAILURE);
}

static int
igc_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
	igc_t *igc;

	if (cmd != DDI_DETACH) {
		return (DDI_FAILURE);
	}

	igc = ddi_get_driver_private(dip);
	if (igc == NULL) {
		dev_err(dip, CE_WARN, "asked to detach, but missing igc_t");
		return (DDI_FAILURE);
	}

	igc_cleanup(igc);
	return (DDI_SUCCESS);
}

static struct cb_ops igc_cb_ops = {
	.cb_open = nulldev,
	.cb_close = nulldev,
	.cb_strategy = nodev,
	.cb_print = nodev,
	.cb_dump = nodev,
	.cb_read = nodev,
	.cb_write = nodev,
	.cb_ioctl = nodev,
	.cb_devmap = nodev,
	.cb_mmap = nodev,
	.cb_segmap = nodev,
	.cb_chpoll = nochpoll,
	.cb_prop_op = ddi_prop_op,
	.cb_flag = D_MP,
	.cb_rev = CB_REV,
	.cb_aread = nodev,
	.cb_awrite = nodev
};

static struct dev_ops igc_dev_ops = {
	.devo_rev = DEVO_REV,
	.devo_refcnt = 0,
	.devo_getinfo = NULL,
	.devo_identify = nulldev,
	.devo_probe = nulldev,
	.devo_attach = igc_attach,
	.devo_detach = igc_detach,
	.devo_reset = nodev,
	.devo_quiesce = ddi_quiesce_not_supported,
	.devo_cb_ops = &igc_cb_ops
};

static struct modldrv igc_modldrv = {
	.drv_modops = &mod_driverops,
	.drv_linkinfo = "Intel I226/226 Ethernet Controller",
	.drv_dev_ops = &igc_dev_ops
};

static struct modlinkage igc_modlinkage = {
	.ml_rev = MODREV_1,
	.ml_linkage = { &igc_modldrv, NULL }
};

int
_init(void)
{
	int ret;

	mac_init_ops(&igc_dev_ops, IGC_MOD_NAME);

	if ((ret = mod_install(&igc_modlinkage)) != 0) {
		mac_fini_ops(&igc_dev_ops);
	}

	return (ret);
}

int
_info(struct modinfo *modinfop)
{
	return (mod_info(&igc_modlinkage, modinfop));
}

int
_fini(void)
{
	int ret;

	if ((ret = mod_remove(&igc_modlinkage)) == 0) {
		mac_fini_ops(&igc_dev_ops);
	}

	return (ret);
}