1d37cece2SSean Bruno /*- 296fc97c8SStephen Hurd * Copyright (c) 2016 Matthew Macy <mmacy@mattmacy.io> 3d37cece2SSean Bruno * All rights reserved. 4d37cece2SSean Bruno * 5d37cece2SSean Bruno * Redistribution and use in source and binary forms, with or without 6d37cece2SSean Bruno * modification, are permitted provided that the following conditions 7d37cece2SSean Bruno * are met: 8d37cece2SSean Bruno * 1. Redistributions of source code must retain the above copyright 9d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer. 10d37cece2SSean Bruno * 2. Redistributions in binary form must reproduce the above copyright 11d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer in the 12d37cece2SSean Bruno * documentation and/or other materials provided with the distribution. 13d37cece2SSean Bruno * 14d37cece2SSean Bruno * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15d37cece2SSean Bruno * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16d37cece2SSean Bruno * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17d37cece2SSean Bruno * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18d37cece2SSean Bruno * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19d37cece2SSean Bruno * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20d37cece2SSean Bruno * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21d37cece2SSean Bruno * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22d37cece2SSean Bruno * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23d37cece2SSean Bruno * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24d37cece2SSean Bruno * SUCH DAMAGE. 25d37cece2SSean Bruno */ 26d37cece2SSean Bruno 27f2d6ace4SSean Bruno /* $FreeBSD$ */ 28f2d6ace4SSean Bruno #include "if_em.h" 29f2d6ace4SSean Bruno 30f2d6ace4SSean Bruno #ifdef RSS 31f2d6ace4SSean Bruno #include <net/rss_config.h> 32f2d6ace4SSean Bruno #include <netinet/in_rss.h> 33f2d6ace4SSean Bruno #endif 34f2d6ace4SSean Bruno 35f2d6ace4SSean Bruno #ifdef VERBOSE_DEBUG 36f2d6ace4SSean Bruno #define DPRINTF device_printf 37f2d6ace4SSean Bruno #else 38f2d6ace4SSean Bruno #define DPRINTF(...) 39f2d6ace4SSean Bruno #endif 40f2d6ace4SSean Bruno 41f2d6ace4SSean Bruno /********************************************************************* 42f2d6ace4SSean Bruno * Local Function prototypes 43f2d6ace4SSean Bruno *********************************************************************/ 44f2d6ace4SSean Bruno static int igb_isc_txd_encap(void *arg, if_pkt_info_t pi); 4595246abbSSean Bruno static void igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 4695246abbSSean Bruno static int igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); 47f2d6ace4SSean Bruno 4895246abbSSean Bruno static void igb_isc_rxd_refill(void *arg, if_rxd_update_t iru); 4995246abbSSean Bruno 5095246abbSSean Bruno static void igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx); 5195246abbSSean Bruno static int igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget); 5295246abbSSean Bruno 53f2d6ace4SSean Bruno static int igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 54f2d6ace4SSean Bruno 55f2d6ace4SSean Bruno static int igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); 56f2d6ace4SSean Bruno static int igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status); 57f2d6ace4SSean Bruno 58f2d6ace4SSean Bruno static void igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype); 59f2d6ace4SSean Bruno static int igb_determine_rsstype(u16 pkt_info); 60f2d6ace4SSean Bruno 61f2d6ace4SSean Bruno extern void igb_if_enable_intr(if_ctx_t ctx); 62f2d6ace4SSean Bruno extern int em_intr(void *arg); 63f2d6ace4SSean Bruno 64f2d6ace4SSean Bruno struct if_txrx igb_txrx = { 65f2d6ace4SSean Bruno igb_isc_txd_encap, 66f2d6ace4SSean Bruno igb_isc_txd_flush, 67f2d6ace4SSean Bruno igb_isc_txd_credits_update, 68f2d6ace4SSean Bruno igb_isc_rxd_available, 69f2d6ace4SSean Bruno igb_isc_rxd_pkt_get, 70f2d6ace4SSean Bruno igb_isc_rxd_refill, 71f2d6ace4SSean Bruno igb_isc_rxd_flush, 72f2d6ace4SSean Bruno em_intr 73f2d6ace4SSean Bruno }; 74f2d6ace4SSean Bruno 75f2d6ace4SSean Bruno extern if_shared_ctx_t em_sctx; 76f2d6ace4SSean Bruno 77f2d6ace4SSean Bruno /********************************************************************** 78f2d6ace4SSean Bruno * 79f2d6ace4SSean Bruno * Setup work for hardware segmentation offload (TSO) on 80f2d6ace4SSean Bruno * adapters using advanced tx descriptors 81f2d6ace4SSean Bruno * 82f2d6ace4SSean Bruno **********************************************************************/ 83f2d6ace4SSean Bruno static int 84f2d6ace4SSean Bruno igb_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) 85f2d6ace4SSean Bruno { 86f2d6ace4SSean Bruno struct e1000_adv_tx_context_desc *TXD; 87f2d6ace4SSean Bruno struct adapter *adapter = txr->adapter; 88f2d6ace4SSean Bruno u32 type_tucmd_mlhl = 0, vlan_macip_lens = 0; 89f2d6ace4SSean Bruno u32 mss_l4len_idx = 0; 90f2d6ace4SSean Bruno u32 paylen; 91f2d6ace4SSean Bruno 92f2d6ace4SSean Bruno switch(pi->ipi_etype) { 93f2d6ace4SSean Bruno case ETHERTYPE_IPV6: 94f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 95f2d6ace4SSean Bruno break; 96f2d6ace4SSean Bruno case ETHERTYPE_IP: 97f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 98f2d6ace4SSean Bruno /* Tell transmit desc to also do IPv4 checksum. */ 99f2d6ace4SSean Bruno *olinfo_status |= E1000_TXD_POPTS_IXSM << 8; 100f2d6ace4SSean Bruno break; 101f2d6ace4SSean Bruno default: 102f2d6ace4SSean Bruno panic("%s: CSUM_TSO but no supported IP version (0x%04x)", 103f2d6ace4SSean Bruno __func__, ntohs(pi->ipi_etype)); 104f2d6ace4SSean Bruno break; 105f2d6ace4SSean Bruno } 106f2d6ace4SSean Bruno 107f2d6ace4SSean Bruno TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; 108f2d6ace4SSean Bruno 109f2d6ace4SSean Bruno /* This is used in the transmit desc in encap */ 110f2d6ace4SSean Bruno paylen = pi->ipi_len - pi->ipi_ehdrlen - pi->ipi_ip_hlen - pi->ipi_tcp_hlen; 111f2d6ace4SSean Bruno 112f2d6ace4SSean Bruno /* VLAN MACLEN IPLEN */ 113f2d6ace4SSean Bruno if (pi->ipi_mflags & M_VLANTAG) { 114f2d6ace4SSean Bruno vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); 115f2d6ace4SSean Bruno } 116f2d6ace4SSean Bruno 117f2d6ace4SSean Bruno vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 118f2d6ace4SSean Bruno vlan_macip_lens |= pi->ipi_ip_hlen; 119f2d6ace4SSean Bruno TXD->vlan_macip_lens = htole32(vlan_macip_lens); 120f2d6ace4SSean Bruno 121f2d6ace4SSean Bruno /* ADV DTYPE TUCMD */ 122f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 123f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 124f2d6ace4SSean Bruno TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 125f2d6ace4SSean Bruno 126f2d6ace4SSean Bruno /* MSS L4LEN IDX */ 127f2d6ace4SSean Bruno mss_l4len_idx |= (pi->ipi_tso_segsz << E1000_ADVTXD_MSS_SHIFT); 128f2d6ace4SSean Bruno mss_l4len_idx |= (pi->ipi_tcp_hlen << E1000_ADVTXD_L4LEN_SHIFT); 129f2d6ace4SSean Bruno /* 82575 needs the queue index added */ 130f2d6ace4SSean Bruno if (adapter->hw.mac.type == e1000_82575) 131f2d6ace4SSean Bruno mss_l4len_idx |= txr->me << 4; 132f2d6ace4SSean Bruno TXD->mss_l4len_idx = htole32(mss_l4len_idx); 133f2d6ace4SSean Bruno 134f2d6ace4SSean Bruno TXD->seqnum_seed = htole32(0); 135f2d6ace4SSean Bruno *cmd_type_len |= E1000_ADVTXD_DCMD_TSE; 136f2d6ace4SSean Bruno *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 137f2d6ace4SSean Bruno *olinfo_status |= paylen << E1000_ADVTXD_PAYLEN_SHIFT; 138f2d6ace4SSean Bruno 139f2d6ace4SSean Bruno return (1); 140f2d6ace4SSean Bruno } 141f2d6ace4SSean Bruno 142f2d6ace4SSean Bruno /********************************************************************* 143f2d6ace4SSean Bruno * 144f2d6ace4SSean Bruno * Advanced Context Descriptor setup for VLAN, CSUM or TSO 145f2d6ace4SSean Bruno * 146f2d6ace4SSean Bruno **********************************************************************/ 147f2d6ace4SSean Bruno static int 148f2d6ace4SSean Bruno igb_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, u32 *cmd_type_len, u32 *olinfo_status) 149f2d6ace4SSean Bruno { 150f2d6ace4SSean Bruno struct e1000_adv_tx_context_desc *TXD; 151f2d6ace4SSean Bruno struct adapter *adapter = txr->adapter; 152f2d6ace4SSean Bruno u32 vlan_macip_lens, type_tucmd_mlhl; 153f2d6ace4SSean Bruno u32 mss_l4len_idx; 154f2d6ace4SSean Bruno mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0; 155f2d6ace4SSean Bruno int offload = TRUE; 156f2d6ace4SSean Bruno 157f2d6ace4SSean Bruno /* First check if TSO is to be used */ 158f2d6ace4SSean Bruno if (pi->ipi_csum_flags & CSUM_TSO) 159f2d6ace4SSean Bruno return (igb_tso_setup(txr, pi, cmd_type_len, olinfo_status)); 160f2d6ace4SSean Bruno 161f2d6ace4SSean Bruno /* Indicate the whole packet as payload when not doing TSO */ 162f2d6ace4SSean Bruno *olinfo_status |= pi->ipi_len << E1000_ADVTXD_PAYLEN_SHIFT; 163f2d6ace4SSean Bruno 164f2d6ace4SSean Bruno /* Now ready a context descriptor */ 165f2d6ace4SSean Bruno TXD = (struct e1000_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx]; 166f2d6ace4SSean Bruno 167f2d6ace4SSean Bruno /* 168f2d6ace4SSean Bruno ** In advanced descriptors the vlan tag must 169f2d6ace4SSean Bruno ** be placed into the context descriptor. Hence 170f2d6ace4SSean Bruno ** we need to make one even if not doing offloads. 171f2d6ace4SSean Bruno */ 172f2d6ace4SSean Bruno if (pi->ipi_mflags & M_VLANTAG) { 173f2d6ace4SSean Bruno vlan_macip_lens |= (pi->ipi_vtag << E1000_ADVTXD_VLAN_SHIFT); 17482379056SSean Bruno } else if ((pi->ipi_csum_flags & IGB_CSUM_OFFLOAD) == 0) { 175f2d6ace4SSean Bruno return (0); 176f2d6ace4SSean Bruno } 177f2d6ace4SSean Bruno 178f2d6ace4SSean Bruno /* Set the ether header length */ 179f2d6ace4SSean Bruno vlan_macip_lens |= pi->ipi_ehdrlen << E1000_ADVTXD_MACLEN_SHIFT; 180f2d6ace4SSean Bruno 181f2d6ace4SSean Bruno switch(pi->ipi_etype) { 182f2d6ace4SSean Bruno case ETHERTYPE_IP: 183f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV4; 184f2d6ace4SSean Bruno break; 185f2d6ace4SSean Bruno case ETHERTYPE_IPV6: 186f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_IPV6; 187f2d6ace4SSean Bruno break; 188f2d6ace4SSean Bruno default: 189f2d6ace4SSean Bruno offload = FALSE; 190f2d6ace4SSean Bruno break; 191f2d6ace4SSean Bruno } 192f2d6ace4SSean Bruno 193f2d6ace4SSean Bruno vlan_macip_lens |= pi->ipi_ip_hlen; 194f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_DCMD_DEXT | E1000_ADVTXD_DTYP_CTXT; 195f2d6ace4SSean Bruno 196f2d6ace4SSean Bruno switch (pi->ipi_ipproto) { 197f2d6ace4SSean Bruno case IPPROTO_TCP: 198f2d6ace4SSean Bruno if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) 199f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_TCP; 200f2d6ace4SSean Bruno break; 201f2d6ace4SSean Bruno case IPPROTO_UDP: 202f2d6ace4SSean Bruno if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) 203f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_UDP; 204f2d6ace4SSean Bruno break; 205f2d6ace4SSean Bruno case IPPROTO_SCTP: 206f2d6ace4SSean Bruno if (pi->ipi_csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) 207f2d6ace4SSean Bruno type_tucmd_mlhl |= E1000_ADVTXD_TUCMD_L4T_SCTP; 208f2d6ace4SSean Bruno break; 209f2d6ace4SSean Bruno default: 210f2d6ace4SSean Bruno offload = FALSE; 211f2d6ace4SSean Bruno break; 212f2d6ace4SSean Bruno } 213f2d6ace4SSean Bruno 214f2d6ace4SSean Bruno if (offload) /* For the TX descriptor setup */ 215f2d6ace4SSean Bruno *olinfo_status |= E1000_TXD_POPTS_TXSM << 8; 216f2d6ace4SSean Bruno 217f2d6ace4SSean Bruno /* 82575 needs the queue index added */ 218f2d6ace4SSean Bruno if (adapter->hw.mac.type == e1000_82575) 219f2d6ace4SSean Bruno mss_l4len_idx = txr->me << 4; 220f2d6ace4SSean Bruno 221f2d6ace4SSean Bruno /* Now copy bits into descriptor */ 222f2d6ace4SSean Bruno TXD->vlan_macip_lens = htole32(vlan_macip_lens); 223f2d6ace4SSean Bruno TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl); 224f2d6ace4SSean Bruno TXD->seqnum_seed = htole32(0); 225f2d6ace4SSean Bruno TXD->mss_l4len_idx = htole32(mss_l4len_idx); 226f2d6ace4SSean Bruno 227f2d6ace4SSean Bruno return (1); 228f2d6ace4SSean Bruno } 229f2d6ace4SSean Bruno 230f2d6ace4SSean Bruno static int 231f2d6ace4SSean Bruno igb_isc_txd_encap(void *arg, if_pkt_info_t pi) 232f2d6ace4SSean Bruno { 233f2d6ace4SSean Bruno struct adapter *sc = arg; 234f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 235f2d6ace4SSean Bruno struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 236f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 237f2d6ace4SSean Bruno int nsegs = pi->ipi_nsegs; 238f2d6ace4SSean Bruno bus_dma_segment_t *segs = pi->ipi_segs; 239f2d6ace4SSean Bruno union e1000_adv_tx_desc *txd = NULL; 240f2d6ace4SSean Bruno int i, j, first, pidx_last; 24195246abbSSean Bruno u32 olinfo_status, cmd_type_len, txd_flags; 24295246abbSSean Bruno qidx_t ntxd; 243f2d6ace4SSean Bruno 244f2d6ace4SSean Bruno pidx_last = olinfo_status = 0; 245f2d6ace4SSean Bruno /* Basic descriptor defines */ 246f2d6ace4SSean Bruno cmd_type_len = (E1000_ADVTXD_DTYP_DATA | 247f2d6ace4SSean Bruno E1000_ADVTXD_DCMD_IFCS | E1000_ADVTXD_DCMD_DEXT); 248f2d6ace4SSean Bruno 249f2d6ace4SSean Bruno if (pi->ipi_mflags & M_VLANTAG) 250f2d6ace4SSean Bruno cmd_type_len |= E1000_ADVTXD_DCMD_VLE; 251f2d6ace4SSean Bruno 252f2d6ace4SSean Bruno first = i = pi->ipi_pidx; 25395246abbSSean Bruno ntxd = scctx->isc_ntxd[0]; 25495246abbSSean Bruno txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_ADVTXD_DCMD_RS : 0; 255f2d6ace4SSean Bruno /* Consume the first descriptor */ 256f2d6ace4SSean Bruno i += igb_tx_ctx_setup(txr, pi, &cmd_type_len, &olinfo_status); 257f2d6ace4SSean Bruno if (i == scctx->isc_ntxd[0]) 258f2d6ace4SSean Bruno i = 0; 259f2d6ace4SSean Bruno 260f2d6ace4SSean Bruno /* 82575 needs the queue index added */ 261f2d6ace4SSean Bruno if (sc->hw.mac.type == e1000_82575) 262f2d6ace4SSean Bruno olinfo_status |= txr->me << 4; 263f2d6ace4SSean Bruno 264f2d6ace4SSean Bruno for (j = 0; j < nsegs; j++) { 265f2d6ace4SSean Bruno bus_size_t seglen; 266f2d6ace4SSean Bruno bus_addr_t segaddr; 267f2d6ace4SSean Bruno 268f2d6ace4SSean Bruno txd = (union e1000_adv_tx_desc *)&txr->tx_base[i]; 269f2d6ace4SSean Bruno seglen = segs[j].ds_len; 270f2d6ace4SSean Bruno segaddr = htole64(segs[j].ds_addr); 271f2d6ace4SSean Bruno 272f2d6ace4SSean Bruno txd->read.buffer_addr = segaddr; 273f2d6ace4SSean Bruno txd->read.cmd_type_len = htole32(E1000_TXD_CMD_IFCS | 274f2d6ace4SSean Bruno cmd_type_len | seglen); 275f2d6ace4SSean Bruno txd->read.olinfo_status = htole32(olinfo_status); 276f2d6ace4SSean Bruno pidx_last = i; 277f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) { 278f2d6ace4SSean Bruno i = 0; 279f2d6ace4SSean Bruno } 280f2d6ace4SSean Bruno } 28195246abbSSean Bruno if (txd_flags) { 28295246abbSSean Bruno txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 28395246abbSSean Bruno txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); 28495246abbSSean Bruno MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 28595246abbSSean Bruno } 286f2d6ace4SSean Bruno 28795246abbSSean Bruno txd->read.cmd_type_len |= htole32(E1000_TXD_CMD_EOP | txd_flags); 288f2d6ace4SSean Bruno pi->ipi_new_pidx = i; 289f2d6ace4SSean Bruno 290f2d6ace4SSean Bruno return (0); 291f2d6ace4SSean Bruno } 292f2d6ace4SSean Bruno 293f2d6ace4SSean Bruno static void 29495246abbSSean Bruno igb_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 295f2d6ace4SSean Bruno { 296f2d6ace4SSean Bruno struct adapter *adapter = arg; 297f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 298f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 299f2d6ace4SSean Bruno 300f2d6ace4SSean Bruno E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); 301f2d6ace4SSean Bruno } 302f2d6ace4SSean Bruno 303f2d6ace4SSean Bruno static int 30495246abbSSean Bruno igb_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 305f2d6ace4SSean Bruno { 306f2d6ace4SSean Bruno struct adapter *adapter = arg; 307f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 308f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 309f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 310f2d6ace4SSean Bruno 31195246abbSSean Bruno qidx_t processed = 0; 31295246abbSSean Bruno int updated; 31395246abbSSean Bruno qidx_t cur, prev, ntxd, rs_cidx; 31495246abbSSean Bruno int32_t delta; 31595246abbSSean Bruno uint8_t status; 316f2d6ace4SSean Bruno 31795246abbSSean Bruno rs_cidx = txr->tx_rs_cidx; 31895246abbSSean Bruno if (rs_cidx == txr->tx_rs_pidx) 31995246abbSSean Bruno return (0); 32095246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 32195246abbSSean Bruno status = ((union e1000_adv_tx_desc *)&txr->tx_base[cur])->wb.status; 32295246abbSSean Bruno updated = !!(status & E1000_TXD_STAT_DD); 323f2d6ace4SSean Bruno 32495246abbSSean Bruno if (!clear || !updated) 32595246abbSSean Bruno return (updated); 326f2d6ace4SSean Bruno 32795246abbSSean Bruno prev = txr->tx_cidx_processed; 328f2d6ace4SSean Bruno ntxd = scctx->isc_ntxd[0]; 329f2d6ace4SSean Bruno do { 33095246abbSSean Bruno delta = (int32_t)cur - (int32_t)prev; 33195246abbSSean Bruno MPASS(prev == 0 || delta != 0); 33295246abbSSean Bruno if (delta < 0) 33395246abbSSean Bruno delta += ntxd; 33495246abbSSean Bruno 33595246abbSSean Bruno processed += delta; 33695246abbSSean Bruno prev = cur; 33795246abbSSean Bruno rs_cidx = (rs_cidx + 1) & (ntxd-1); 33895246abbSSean Bruno if (rs_cidx == txr->tx_rs_pidx) 339f2d6ace4SSean Bruno break; 34095246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 34195246abbSSean Bruno status = ((union e1000_adv_tx_desc *)&txr->tx_base[cur])->wb.status; 34295246abbSSean Bruno } while ((status & E1000_TXD_STAT_DD)); 343f2d6ace4SSean Bruno 34495246abbSSean Bruno txr->tx_rs_cidx = rs_cidx; 34595246abbSSean Bruno txr->tx_cidx_processed = prev; 346f2d6ace4SSean Bruno return (processed); 347f2d6ace4SSean Bruno } 348f2d6ace4SSean Bruno 349f2d6ace4SSean Bruno static void 35095246abbSSean Bruno igb_isc_rxd_refill(void *arg, if_rxd_update_t iru) 351f2d6ace4SSean Bruno { 352f2d6ace4SSean Bruno struct adapter *sc = arg; 353f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 35495246abbSSean Bruno uint16_t rxqid = iru->iru_qsidx; 355f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 356f2d6ace4SSean Bruno union e1000_adv_rx_desc *rxd; 357f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 35895246abbSSean Bruno uint64_t *paddrs; 35995246abbSSean Bruno uint32_t next_pidx, pidx; 36095246abbSSean Bruno uint16_t count; 361f2d6ace4SSean Bruno int i; 36295246abbSSean Bruno 36395246abbSSean Bruno paddrs = iru->iru_paddrs; 36495246abbSSean Bruno pidx = iru->iru_pidx; 36595246abbSSean Bruno count = iru->iru_count; 366f2d6ace4SSean Bruno 367f2d6ace4SSean Bruno for (i = 0, next_pidx = pidx; i < count; i++) { 368f2d6ace4SSean Bruno rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[next_pidx]; 369f2d6ace4SSean Bruno 370f2d6ace4SSean Bruno rxd->read.pkt_addr = htole64(paddrs[i]); 371f2d6ace4SSean Bruno if (++next_pidx == scctx->isc_nrxd[0]) 372f2d6ace4SSean Bruno next_pidx = 0; 373f2d6ace4SSean Bruno } 374f2d6ace4SSean Bruno } 375f2d6ace4SSean Bruno 376f2d6ace4SSean Bruno static void 37795246abbSSean Bruno igb_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) 378f2d6ace4SSean Bruno { 379f2d6ace4SSean Bruno struct adapter *sc = arg; 380f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 381f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 382f2d6ace4SSean Bruno 383f2d6ace4SSean Bruno E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 384f2d6ace4SSean Bruno } 385f2d6ace4SSean Bruno 386f2d6ace4SSean Bruno static int 38795246abbSSean Bruno igb_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 388f2d6ace4SSean Bruno { 389f2d6ace4SSean Bruno struct adapter *sc = arg; 390f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 391f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 392f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 393f2d6ace4SSean Bruno union e1000_adv_rx_desc *rxd; 394f2d6ace4SSean Bruno u32 staterr = 0; 395f2d6ace4SSean Bruno int cnt, i, iter; 396f2d6ace4SSean Bruno 39795246abbSSean Bruno if (budget == 1) { 39895246abbSSean Bruno rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[idx]; 39995246abbSSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 40095246abbSSean Bruno return (staterr & E1000_RXD_STAT_DD); 40195246abbSSean Bruno } 40295246abbSSean Bruno 403f2d6ace4SSean Bruno for (iter = cnt = 0, i = idx; iter < scctx->isc_nrxd[0] && iter <= budget;) { 404f2d6ace4SSean Bruno rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[i]; 405f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 406f2d6ace4SSean Bruno 407f2d6ace4SSean Bruno if ((staterr & E1000_RXD_STAT_DD) == 0) 408f2d6ace4SSean Bruno break; 409f2d6ace4SSean Bruno 410f2d6ace4SSean Bruno if (++i == scctx->isc_nrxd[0]) { 411f2d6ace4SSean Bruno i = 0; 412f2d6ace4SSean Bruno } 413f2d6ace4SSean Bruno 414f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_EOP) 415f2d6ace4SSean Bruno cnt++; 416f2d6ace4SSean Bruno iter++; 417f2d6ace4SSean Bruno } 418f2d6ace4SSean Bruno return (cnt); 419f2d6ace4SSean Bruno } 420f2d6ace4SSean Bruno 421f2d6ace4SSean Bruno /**************************************************************** 422f2d6ace4SSean Bruno * Routine sends data which has been dma'ed into host memory 423f2d6ace4SSean Bruno * to upper layer. Initialize ri structure. 424f2d6ace4SSean Bruno * 425f2d6ace4SSean Bruno * Returns 0 upon success, errno on failure 426f2d6ace4SSean Bruno ***************************************************************/ 427f2d6ace4SSean Bruno 428f2d6ace4SSean Bruno static int 429f2d6ace4SSean Bruno igb_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 430f2d6ace4SSean Bruno { 431f2d6ace4SSean Bruno struct adapter *adapter = arg; 432f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 433f2d6ace4SSean Bruno struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; 434f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 435f2d6ace4SSean Bruno struct ifnet *ifp = iflib_get_ifp(adapter->ctx); 436f2d6ace4SSean Bruno union e1000_adv_rx_desc *rxd; 437f2d6ace4SSean Bruno 438f2d6ace4SSean Bruno u16 pkt_info, len; 439f2d6ace4SSean Bruno u16 vtag = 0; 440f2d6ace4SSean Bruno u32 ptype; 441f2d6ace4SSean Bruno u32 staterr = 0; 442f2d6ace4SSean Bruno bool eop; 443f2d6ace4SSean Bruno int i = 0; 444f2d6ace4SSean Bruno int cidx = ri->iri_cidx; 445f2d6ace4SSean Bruno 446f2d6ace4SSean Bruno do { 447f2d6ace4SSean Bruno rxd = (union e1000_adv_rx_desc *)&rxr->rx_base[cidx]; 448f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 449f2d6ace4SSean Bruno pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info); 450f2d6ace4SSean Bruno 451f2d6ace4SSean Bruno MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 452f2d6ace4SSean Bruno 453f2d6ace4SSean Bruno len = le16toh(rxd->wb.upper.length); 454f2d6ace4SSean Bruno ptype = le32toh(rxd->wb.lower.lo_dword.data) & IGB_PKTTYPE_MASK; 455f2d6ace4SSean Bruno 456f2d6ace4SSean Bruno ri->iri_len += len; 457f2d6ace4SSean Bruno rxr->rx_bytes += ri->iri_len; 458f2d6ace4SSean Bruno 459f2d6ace4SSean Bruno rxd->wb.upper.status_error = 0; 460f2d6ace4SSean Bruno eop = ((staterr & E1000_RXD_STAT_EOP) == E1000_RXD_STAT_EOP); 461f2d6ace4SSean Bruno 462f2d6ace4SSean Bruno if (((adapter->hw.mac.type == e1000_i350) || 463f2d6ace4SSean Bruno (adapter->hw.mac.type == e1000_i354)) && 464f2d6ace4SSean Bruno (staterr & E1000_RXDEXT_STATERR_LB)) 465f2d6ace4SSean Bruno vtag = be16toh(rxd->wb.upper.vlan); 466f2d6ace4SSean Bruno else 467f2d6ace4SSean Bruno vtag = le16toh(rxd->wb.upper.vlan); 468f2d6ace4SSean Bruno 469f2d6ace4SSean Bruno /* Make sure bad packets are discarded */ 470f2d6ace4SSean Bruno if (eop && ((staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) != 0)) { 471f2d6ace4SSean Bruno adapter->dropped_pkts++; 472f2d6ace4SSean Bruno ++rxr->rx_discarded; 473f2d6ace4SSean Bruno return (EBADMSG); 474f2d6ace4SSean Bruno } 475f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 0; 476f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 477f2d6ace4SSean Bruno ri->iri_frags[i].irf_len = len; 478f2d6ace4SSean Bruno 479f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 480f2d6ace4SSean Bruno cidx = 0; 481f2d6ace4SSean Bruno #ifdef notyet 482f2d6ace4SSean Bruno if (rxr->hdr_split == TRUE) { 483f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 1; 484f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 485f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 486f2d6ace4SSean Bruno cidx = 0; 487f2d6ace4SSean Bruno } 488f2d6ace4SSean Bruno #endif 489f2d6ace4SSean Bruno i++; 490f2d6ace4SSean Bruno } while (!eop); 491f2d6ace4SSean Bruno 492f2d6ace4SSean Bruno rxr->rx_packets++; 493f2d6ace4SSean Bruno 494f2d6ace4SSean Bruno if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) 495f2d6ace4SSean Bruno igb_rx_checksum(staterr, ri, ptype); 496f2d6ace4SSean Bruno 497f2d6ace4SSean Bruno if ((ifp->if_capenable & IFCAP_VLAN_HWTAGGING) != 0 && 498f2d6ace4SSean Bruno (staterr & E1000_RXD_STAT_VP) != 0) { 499f2d6ace4SSean Bruno ri->iri_vtag = vtag; 500f2d6ace4SSean Bruno ri->iri_flags |= M_VLANTAG; 501f2d6ace4SSean Bruno } 502f2d6ace4SSean Bruno ri->iri_flowid = 503f2d6ace4SSean Bruno le32toh(rxd->wb.lower.hi_dword.rss); 504f2d6ace4SSean Bruno ri->iri_rsstype = igb_determine_rsstype(pkt_info); 505f2d6ace4SSean Bruno ri->iri_nfrags = i; 506f2d6ace4SSean Bruno 507f2d6ace4SSean Bruno return (0); 508f2d6ace4SSean Bruno } 509f2d6ace4SSean Bruno 510f2d6ace4SSean Bruno /********************************************************************* 511f2d6ace4SSean Bruno * 512f2d6ace4SSean Bruno * Verify that the hardware indicated that the checksum is valid. 513f2d6ace4SSean Bruno * Inform the stack about the status of checksum so that stack 514f2d6ace4SSean Bruno * doesn't spend time verifying the checksum. 515f2d6ace4SSean Bruno * 516f2d6ace4SSean Bruno *********************************************************************/ 517f2d6ace4SSean Bruno static void 518f2d6ace4SSean Bruno igb_rx_checksum(u32 staterr, if_rxd_info_t ri, u32 ptype) 519f2d6ace4SSean Bruno { 520f2d6ace4SSean Bruno u16 status = (u16)staterr; 521f2d6ace4SSean Bruno u8 errors = (u8) (staterr >> 24); 522f2d6ace4SSean Bruno bool sctp = FALSE; 523f2d6ace4SSean Bruno 524f2d6ace4SSean Bruno /* Ignore Checksum bit is set */ 525f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IXSM) { 526f2d6ace4SSean Bruno ri->iri_csum_flags = 0; 527f2d6ace4SSean Bruno return; 528f2d6ace4SSean Bruno } 529f2d6ace4SSean Bruno 530f2d6ace4SSean Bruno if ((ptype & E1000_RXDADV_PKTTYPE_ETQF) == 0 && 531f2d6ace4SSean Bruno (ptype & E1000_RXDADV_PKTTYPE_SCTP) != 0) 532f2d6ace4SSean Bruno sctp = 1; 533f2d6ace4SSean Bruno else 534f2d6ace4SSean Bruno sctp = 0; 535f2d6ace4SSean Bruno 536f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IPCS) { 537f2d6ace4SSean Bruno /* Did it pass? */ 538f2d6ace4SSean Bruno if (!(errors & E1000_RXD_ERR_IPE)) { 539f2d6ace4SSean Bruno /* IP Checksum Good */ 540f2d6ace4SSean Bruno ri->iri_csum_flags = CSUM_IP_CHECKED; 541f2d6ace4SSean Bruno ri->iri_csum_flags |= CSUM_IP_VALID; 542f2d6ace4SSean Bruno } else 543f2d6ace4SSean Bruno ri->iri_csum_flags = 0; 544f2d6ace4SSean Bruno } 545f2d6ace4SSean Bruno 546f2d6ace4SSean Bruno if (status & (E1000_RXD_STAT_TCPCS | E1000_RXD_STAT_UDPCS)) { 547f2d6ace4SSean Bruno u64 type = (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 548f2d6ace4SSean Bruno if (sctp) /* reassign */ 549f2d6ace4SSean Bruno type = CSUM_SCTP_VALID; 550f2d6ace4SSean Bruno /* Did it pass? */ 551f2d6ace4SSean Bruno if (!(errors & E1000_RXD_ERR_TCPE)) { 552f2d6ace4SSean Bruno ri->iri_csum_flags |= type; 553f2d6ace4SSean Bruno if (sctp == 0) 554f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 555f2d6ace4SSean Bruno } 556f2d6ace4SSean Bruno } 557f2d6ace4SSean Bruno return; 558f2d6ace4SSean Bruno } 559f2d6ace4SSean Bruno 560f2d6ace4SSean Bruno /******************************************************************** 561f2d6ace4SSean Bruno * 562f2d6ace4SSean Bruno * Parse the packet type to determine the appropriate hash 563f2d6ace4SSean Bruno * 564f2d6ace4SSean Bruno ******************************************************************/ 565f2d6ace4SSean Bruno static int 566f2d6ace4SSean Bruno igb_determine_rsstype(u16 pkt_info) 567f2d6ace4SSean Bruno { 568f2d6ace4SSean Bruno switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { 569f2d6ace4SSean Bruno case E1000_RXDADV_RSSTYPE_IPV4_TCP: 570f2d6ace4SSean Bruno return M_HASHTYPE_RSS_TCP_IPV4; 571f2d6ace4SSean Bruno case E1000_RXDADV_RSSTYPE_IPV4: 572f2d6ace4SSean Bruno return M_HASHTYPE_RSS_IPV4; 573f2d6ace4SSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_TCP: 574f2d6ace4SSean Bruno return M_HASHTYPE_RSS_TCP_IPV6; 575f2d6ace4SSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_EX: 576f2d6ace4SSean Bruno return M_HASHTYPE_RSS_IPV6_EX; 577f2d6ace4SSean Bruno case E1000_RXDADV_RSSTYPE_IPV6: 578f2d6ace4SSean Bruno return M_HASHTYPE_RSS_IPV6; 579f2d6ace4SSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: 580f2d6ace4SSean Bruno return M_HASHTYPE_RSS_TCP_IPV6_EX; 581f2d6ace4SSean Bruno default: 582f2d6ace4SSean Bruno return M_HASHTYPE_OPAQUE; 583f2d6ace4SSean Bruno } 584f2d6ace4SSean Bruno } 585