1d37cece2SSean Bruno /*- 2d37cece2SSean Bruno * Copyright (c) 2016 Matt Macy <mmacy@nextbsd.org> 3d37cece2SSean Bruno * All rights reserved. 4d37cece2SSean Bruno * 5d37cece2SSean Bruno * Redistribution and use in source and binary forms, with or without 6d37cece2SSean Bruno * modification, are permitted provided that the following conditions 7d37cece2SSean Bruno * are met: 8d37cece2SSean Bruno * 1. Redistributions of source code must retain the above copyright 9d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer. 10d37cece2SSean Bruno * 2. Redistributions in binary form must reproduce the above copyright 11d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer in the 12d37cece2SSean Bruno * documentation and/or other materials provided with the distribution. 13d37cece2SSean Bruno * 14d37cece2SSean Bruno * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15d37cece2SSean Bruno * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16d37cece2SSean Bruno * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17d37cece2SSean Bruno * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18d37cece2SSean Bruno * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19d37cece2SSean Bruno * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20d37cece2SSean Bruno * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21d37cece2SSean Bruno * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22d37cece2SSean Bruno * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23d37cece2SSean Bruno * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24d37cece2SSean Bruno * SUCH DAMAGE. 25d37cece2SSean Bruno */ 26d37cece2SSean Bruno 27f2d6ace4SSean Bruno /* $FreeBSD$ */ 28f2d6ace4SSean Bruno #include "if_em.h" 29f2d6ace4SSean Bruno 30f2d6ace4SSean Bruno #ifdef RSS 31f2d6ace4SSean Bruno #include <net/rss_config.h> 32f2d6ace4SSean Bruno #include <netinet/in_rss.h> 33f2d6ace4SSean Bruno #endif 34f2d6ace4SSean Bruno 35f2d6ace4SSean Bruno #ifdef VERBOSE_DEBUG 36f2d6ace4SSean Bruno #define DPRINTF device_printf 37f2d6ace4SSean Bruno #else 38f2d6ace4SSean Bruno #define DPRINTF(...) 39f2d6ace4SSean Bruno #endif 40f2d6ace4SSean Bruno 41f2d6ace4SSean Bruno /********************************************************************* 42f2d6ace4SSean Bruno * Local Function prototypes 43f2d6ace4SSean Bruno *********************************************************************/ 44f2d6ace4SSean Bruno static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower); 45f2d6ace4SSean Bruno static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower); 46f2d6ace4SSean Bruno static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); 47f2d6ace4SSean Bruno static void em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx); 48f2d6ace4SSean Bruno static int em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear); 49f2d6ace4SSean Bruno static void em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, 50f2d6ace4SSean Bruno uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused); 51f2d6ace4SSean Bruno static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx); 52f2d6ace4SSean Bruno static int em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, 53f2d6ace4SSean Bruno int budget); 54f2d6ace4SSean Bruno static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 55f2d6ace4SSean Bruno 56f2d6ace4SSean Bruno static void lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, 57f2d6ace4SSean Bruno uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused); 58f2d6ace4SSean Bruno 59f2d6ace4SSean Bruno static int lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, 60f2d6ace4SSean Bruno int budget); 61f2d6ace4SSean Bruno static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 62f2d6ace4SSean Bruno 63f2d6ace4SSean Bruno static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri); 64f2d6ace4SSean Bruno static void em_receive_checksum(uint32_t status, if_rxd_info_t ri); 65f2d6ace4SSean Bruno extern int em_intr(void *arg); 66f2d6ace4SSean Bruno 67f2d6ace4SSean Bruno struct if_txrx em_txrx = { 68f2d6ace4SSean Bruno em_isc_txd_encap, 69f2d6ace4SSean Bruno em_isc_txd_flush, 70f2d6ace4SSean Bruno em_isc_txd_credits_update, 71f2d6ace4SSean Bruno em_isc_rxd_available, 72f2d6ace4SSean Bruno em_isc_rxd_pkt_get, 73f2d6ace4SSean Bruno em_isc_rxd_refill, 74f2d6ace4SSean Bruno em_isc_rxd_flush, 75f2d6ace4SSean Bruno em_intr 76f2d6ace4SSean Bruno }; 77f2d6ace4SSean Bruno 78f2d6ace4SSean Bruno struct if_txrx lem_txrx = { 79f2d6ace4SSean Bruno em_isc_txd_encap, 80f2d6ace4SSean Bruno em_isc_txd_flush, 81f2d6ace4SSean Bruno em_isc_txd_credits_update, 82f2d6ace4SSean Bruno lem_isc_rxd_available, 83f2d6ace4SSean Bruno lem_isc_rxd_pkt_get, 84f2d6ace4SSean Bruno lem_isc_rxd_refill, 85f2d6ace4SSean Bruno em_isc_rxd_flush, 86f2d6ace4SSean Bruno em_intr 87f2d6ace4SSean Bruno }; 88f2d6ace4SSean Bruno 89f2d6ace4SSean Bruno extern if_shared_ctx_t em_sctx; 90f2d6ace4SSean Bruno 91f2d6ace4SSean Bruno /********************************************************************** 92f2d6ace4SSean Bruno * 93f2d6ace4SSean Bruno * Setup work for hardware segmentation offload (TSO) on 94f2d6ace4SSean Bruno * adapters using advanced tx descriptors 95f2d6ace4SSean Bruno * 96f2d6ace4SSean Bruno **********************************************************************/ 97f2d6ace4SSean Bruno static int 98f2d6ace4SSean Bruno em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) 99f2d6ace4SSean Bruno { 100f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 101f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; 102f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 103f2d6ace4SSean Bruno struct e1000_context_desc *TXD; 104f2d6ace4SSean Bruno struct em_txbuffer *tx_buffer; 105f2d6ace4SSean Bruno int cur, hdr_len; 106f2d6ace4SSean Bruno 107f2d6ace4SSean Bruno hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 108f2d6ace4SSean Bruno *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 109f2d6ace4SSean Bruno E1000_TXD_DTYP_D | /* Data descr type */ 110f2d6ace4SSean Bruno E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 111f2d6ace4SSean Bruno 112f2d6ace4SSean Bruno /* IP and/or TCP header checksum calculation and insertion. */ 113f2d6ace4SSean Bruno *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 114f2d6ace4SSean Bruno 115f2d6ace4SSean Bruno cur = pi->ipi_pidx; 116f2d6ace4SSean Bruno TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 117f2d6ace4SSean Bruno tx_buffer = &txr->tx_buffers[cur]; 118f2d6ace4SSean Bruno 119f2d6ace4SSean Bruno /* 120f2d6ace4SSean Bruno * Start offset for header checksum calculation. 121f2d6ace4SSean Bruno * End offset for header checksum calculation. 122f2d6ace4SSean Bruno * Offset of place put the checksum. 123f2d6ace4SSean Bruno */ 124f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 125f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcse = 126f2d6ace4SSean Bruno htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); 127f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 128f2d6ace4SSean Bruno 129f2d6ace4SSean Bruno /* 130f2d6ace4SSean Bruno * Start offset for payload checksum calculation. 131f2d6ace4SSean Bruno * End offset for payload checksum calculation. 132f2d6ace4SSean Bruno * Offset of place to put the checksum. 133f2d6ace4SSean Bruno */ 134f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 135f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucse = 0; 136f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucso = 137f2d6ace4SSean Bruno pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); 138f2d6ace4SSean Bruno 139f2d6ace4SSean Bruno /* 140f2d6ace4SSean Bruno * Payload size per packet w/o any headers. 141f2d6ace4SSean Bruno * Length of all headers up to payload. 142f2d6ace4SSean Bruno */ 143f2d6ace4SSean Bruno TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); 144f2d6ace4SSean Bruno TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 145f2d6ace4SSean Bruno 146f2d6ace4SSean Bruno TXD->cmd_and_length = htole32(adapter->txd_cmd | 147f2d6ace4SSean Bruno E1000_TXD_CMD_DEXT | /* Extended descr */ 148f2d6ace4SSean Bruno E1000_TXD_CMD_TSE | /* TSE context */ 149f2d6ace4SSean Bruno E1000_TXD_CMD_IP | /* Do IP csum */ 150f2d6ace4SSean Bruno E1000_TXD_CMD_TCP | /* Do TCP checksum */ 151f2d6ace4SSean Bruno (pi->ipi_len - hdr_len)); /* Total len */ 152f2d6ace4SSean Bruno tx_buffer->eop = -1; 153f2d6ace4SSean Bruno txr->tx_tso = TRUE; 154f2d6ace4SSean Bruno 155f2d6ace4SSean Bruno if (++cur == scctx->isc_ntxd[0]) { 156f2d6ace4SSean Bruno cur = 0; 157f2d6ace4SSean Bruno } 158f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur); 159f2d6ace4SSean Bruno return (cur); 160f2d6ace4SSean Bruno } 161f2d6ace4SSean Bruno 162f2d6ace4SSean Bruno #define TSO_WORKAROUND 4 163f2d6ace4SSean Bruno #define DONT_FORCE_CTX 1 164f2d6ace4SSean Bruno 165f2d6ace4SSean Bruno 166f2d6ace4SSean Bruno /********************************************************************* 167f2d6ace4SSean Bruno * The offload context is protocol specific (TCP/UDP) and thus 168f2d6ace4SSean Bruno * only needs to be set when the protocol changes. The occasion 169f2d6ace4SSean Bruno * of a context change can be a performance detriment, and 170f2d6ace4SSean Bruno * might be better just disabled. The reason arises in the way 171f2d6ace4SSean Bruno * in which the controller supports pipelined requests from the 172f2d6ace4SSean Bruno * Tx data DMA. Up to four requests can be pipelined, and they may 173f2d6ace4SSean Bruno * belong to the same packet or to multiple packets. However all 174f2d6ace4SSean Bruno * requests for one packet are issued before a request is issued 175f2d6ace4SSean Bruno * for a subsequent packet and if a request for the next packet 176f2d6ace4SSean Bruno * requires a context change, that request will be stalled 177f2d6ace4SSean Bruno * until the previous request completes. This means setting up 178f2d6ace4SSean Bruno * a new context effectively disables pipelined Tx data DMA which 179f2d6ace4SSean Bruno * in turn greatly slow down performance to send small sized 180f2d6ace4SSean Bruno * frames. 181f2d6ace4SSean Bruno **********************************************************************/ 182f2d6ace4SSean Bruno 183f2d6ace4SSean Bruno static int 184f2d6ace4SSean Bruno em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) 185f2d6ace4SSean Bruno { 186f2d6ace4SSean Bruno struct e1000_context_desc *TXD = NULL; 187f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 188f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; 189f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 190f2d6ace4SSean Bruno struct em_txbuffer *tx_buffer; 191f2d6ace4SSean Bruno int csum_flags = pi->ipi_csum_flags; 192f2d6ace4SSean Bruno int cur, hdr_len; 193f2d6ace4SSean Bruno u32 cmd; 194f2d6ace4SSean Bruno 195f2d6ace4SSean Bruno cur = pi->ipi_pidx; 196f2d6ace4SSean Bruno hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 197f2d6ace4SSean Bruno cmd = adapter->txd_cmd; 198f2d6ace4SSean Bruno 199f2d6ace4SSean Bruno /* 200f2d6ace4SSean Bruno * The 82574L can only remember the *last* context used 201f2d6ace4SSean Bruno * regardless of queue that it was use for. We cannot reuse 202f2d6ace4SSean Bruno * contexts on this hardware platform and must generate a new 203f2d6ace4SSean Bruno * context every time. 82574L hardware spec, section 7.2.6, 204f2d6ace4SSean Bruno * second note. 205f2d6ace4SSean Bruno */ 206f2d6ace4SSean Bruno if (DONT_FORCE_CTX && 207f2d6ace4SSean Bruno adapter->tx_num_queues == 1 && 208f2d6ace4SSean Bruno txr->csum_lhlen == pi->ipi_ehdrlen && 209f2d6ace4SSean Bruno txr->csum_iphlen == pi->ipi_ip_hlen && 210f2d6ace4SSean Bruno txr->csum_flags == csum_flags) { 211f2d6ace4SSean Bruno /* 212f2d6ace4SSean Bruno * Same csum offload context as the previous packets; 213f2d6ace4SSean Bruno * just return. 214f2d6ace4SSean Bruno */ 215f2d6ace4SSean Bruno *txd_upper = txr->csum_txd_upper; 216f2d6ace4SSean Bruno *txd_lower = txr->csum_txd_lower; 217f2d6ace4SSean Bruno return (cur); 218f2d6ace4SSean Bruno } 219f2d6ace4SSean Bruno 220f2d6ace4SSean Bruno TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 221f2d6ace4SSean Bruno if (csum_flags & CSUM_IP) { 222f2d6ace4SSean Bruno *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 223f2d6ace4SSean Bruno /* 224f2d6ace4SSean Bruno * Start offset for header checksum calculation. 225f2d6ace4SSean Bruno * End offset for header checksum calculation. 226f2d6ace4SSean Bruno * Offset of place to put the checksum. 227f2d6ace4SSean Bruno */ 228f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 229f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); 230f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 231f2d6ace4SSean Bruno cmd |= E1000_TXD_CMD_IP; 232f2d6ace4SSean Bruno } 233f2d6ace4SSean Bruno 234f2d6ace4SSean Bruno if (csum_flags & (CSUM_TCP|CSUM_UDP)) { 235f2d6ace4SSean Bruno uint8_t tucso; 236f2d6ace4SSean Bruno 237f2d6ace4SSean Bruno *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 238f2d6ace4SSean Bruno *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 239f2d6ace4SSean Bruno 240f2d6ace4SSean Bruno if (csum_flags & CSUM_TCP) { 241f2d6ace4SSean Bruno tucso = hdr_len + offsetof(struct tcphdr, th_sum); 242f2d6ace4SSean Bruno cmd |= E1000_TXD_CMD_TCP; 243f2d6ace4SSean Bruno } else 244f2d6ace4SSean Bruno tucso = hdr_len + offsetof(struct udphdr, uh_sum); 245f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucss = hdr_len; 246f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucse = htole16(0); 247f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucso = tucso; 248f2d6ace4SSean Bruno } 249f2d6ace4SSean Bruno 250f2d6ace4SSean Bruno txr->csum_lhlen = pi->ipi_ehdrlen; 251f2d6ace4SSean Bruno txr->csum_iphlen = pi->ipi_ip_hlen; 252f2d6ace4SSean Bruno txr->csum_flags = csum_flags; 253f2d6ace4SSean Bruno txr->csum_txd_upper = *txd_upper; 254f2d6ace4SSean Bruno txr->csum_txd_lower = *txd_lower; 255f2d6ace4SSean Bruno 256f2d6ace4SSean Bruno TXD->tcp_seg_setup.data = htole32(0); 257f2d6ace4SSean Bruno TXD->cmd_and_length = 258f2d6ace4SSean Bruno htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 259f2d6ace4SSean Bruno 260f2d6ace4SSean Bruno tx_buffer = &txr->tx_buffers[cur]; 261f2d6ace4SSean Bruno tx_buffer->eop = -1; 262f2d6ace4SSean Bruno 263f2d6ace4SSean Bruno if (++cur == scctx->isc_ntxd[0]) { 264f2d6ace4SSean Bruno cur = 0; 265f2d6ace4SSean Bruno } 266f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n", 267f2d6ace4SSean Bruno csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); 268f2d6ace4SSean Bruno return (cur); 269f2d6ace4SSean Bruno } 270f2d6ace4SSean Bruno 271f2d6ace4SSean Bruno static int 272f2d6ace4SSean Bruno em_isc_txd_encap(void *arg, if_pkt_info_t pi) 273f2d6ace4SSean Bruno { 274f2d6ace4SSean Bruno struct adapter *sc = arg; 275f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 276f2d6ace4SSean Bruno struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 277f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 278f2d6ace4SSean Bruno bus_dma_segment_t *segs = pi->ipi_segs; 279f2d6ace4SSean Bruno int nsegs = pi->ipi_nsegs; 280f2d6ace4SSean Bruno int csum_flags = pi->ipi_csum_flags; 281f2d6ace4SSean Bruno int i, j, first, pidx_last; 282f2d6ace4SSean Bruno u32 txd_upper = 0, txd_lower = 0; 283f2d6ace4SSean Bruno 284f2d6ace4SSean Bruno struct em_txbuffer *tx_buffer; 285f2d6ace4SSean Bruno struct e1000_tx_desc *ctxd = NULL; 286f2d6ace4SSean Bruno bool do_tso, tso_desc; 287f2d6ace4SSean Bruno 288f2d6ace4SSean Bruno i = first = pi->ipi_pidx; 289f2d6ace4SSean Bruno do_tso = (csum_flags & CSUM_TSO); 290f2d6ace4SSean Bruno tso_desc = FALSE; 291f2d6ace4SSean Bruno /* 292f2d6ace4SSean Bruno * TSO Hardware workaround, if this packet is not 293f2d6ace4SSean Bruno * TSO, and is only a single descriptor long, and 294f2d6ace4SSean Bruno * it follows a TSO burst, then we need to add a 295f2d6ace4SSean Bruno * sentinel descriptor to prevent premature writeback. 296f2d6ace4SSean Bruno */ 297f2d6ace4SSean Bruno if ((!do_tso) && (txr->tx_tso == TRUE)) { 298f2d6ace4SSean Bruno if (nsegs == 1) 299f2d6ace4SSean Bruno tso_desc = TRUE; 300f2d6ace4SSean Bruno txr->tx_tso = FALSE; 301f2d6ace4SSean Bruno } 302f2d6ace4SSean Bruno 303f2d6ace4SSean Bruno /* Do hardware assists */ 304f2d6ace4SSean Bruno if (do_tso) { 305f2d6ace4SSean Bruno i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); 306f2d6ace4SSean Bruno tso_desc = TRUE; 30782379056SSean Bruno } else if (csum_flags & EM_CSUM_OFFLOAD) { 308f2d6ace4SSean Bruno i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); 309f2d6ace4SSean Bruno } 310f2d6ace4SSean Bruno 311f2d6ace4SSean Bruno if (pi->ipi_mflags & M_VLANTAG) { 312f2d6ace4SSean Bruno /* Set the vlan id. */ 313f2d6ace4SSean Bruno txd_upper |= htole16(pi->ipi_vtag) << 16; 314f2d6ace4SSean Bruno /* Tell hardware to add tag */ 315f2d6ace4SSean Bruno txd_lower |= htole32(E1000_TXD_CMD_VLE); 316f2d6ace4SSean Bruno } 317f2d6ace4SSean Bruno 318f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); 319f2d6ace4SSean Bruno /* XXX adapter->pcix_82544 -- lem_fill_descriptors */ 320f2d6ace4SSean Bruno 321f2d6ace4SSean Bruno /* Set up our transmit descriptors */ 322f2d6ace4SSean Bruno for (j = 0; j < nsegs; j++) { 323f2d6ace4SSean Bruno bus_size_t seg_len; 324f2d6ace4SSean Bruno bus_addr_t seg_addr; 325f2d6ace4SSean Bruno uint32_t cmd; 326f2d6ace4SSean Bruno 327f2d6ace4SSean Bruno ctxd = &txr->tx_base[i]; 328f2d6ace4SSean Bruno tx_buffer = &txr->tx_buffers[i]; 329f2d6ace4SSean Bruno seg_addr = segs[j].ds_addr; 330f2d6ace4SSean Bruno seg_len = segs[j].ds_len; 331f2d6ace4SSean Bruno cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; 332f2d6ace4SSean Bruno 333f2d6ace4SSean Bruno /* 334f2d6ace4SSean Bruno ** TSO Workaround: 335f2d6ace4SSean Bruno ** If this is the last descriptor, we want to 336f2d6ace4SSean Bruno ** split it so we have a small final sentinel 337f2d6ace4SSean Bruno */ 338f2d6ace4SSean Bruno if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { 339f2d6ace4SSean Bruno seg_len -= TSO_WORKAROUND; 340f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr); 341f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 342f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 343f2d6ace4SSean Bruno 344f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 345f2d6ace4SSean Bruno i = 0; 346f2d6ace4SSean Bruno 347f2d6ace4SSean Bruno /* Now make the sentinel */ 348f2d6ace4SSean Bruno ctxd = &txr->tx_base[i]; 349f2d6ace4SSean Bruno tx_buffer = &txr->tx_buffers[i]; 350f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr + seg_len); 351f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); 352f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 353f2d6ace4SSean Bruno pidx_last = i; 354f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 355f2d6ace4SSean Bruno i = 0; 356f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); 357f2d6ace4SSean Bruno } else { 358f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr); 359f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 360f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 361f2d6ace4SSean Bruno pidx_last = i; 362f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 363f2d6ace4SSean Bruno i = 0; 364f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); 365f2d6ace4SSean Bruno } 366f2d6ace4SSean Bruno tx_buffer->eop = -1; 367f2d6ace4SSean Bruno } 368f2d6ace4SSean Bruno 369f2d6ace4SSean Bruno /* 370f2d6ace4SSean Bruno * Last Descriptor of Packet 371f2d6ace4SSean Bruno * needs End Of Packet (EOP) 372f2d6ace4SSean Bruno * and Report Status (RS) 373f2d6ace4SSean Bruno */ 374f2d6ace4SSean Bruno ctxd->lower.data |= 375f2d6ace4SSean Bruno htole32(E1000_TXD_CMD_EOP | E1000_TXD_CMD_RS); 376f2d6ace4SSean Bruno 377f2d6ace4SSean Bruno tx_buffer = &txr->tx_buffers[first]; 378f2d6ace4SSean Bruno tx_buffer->eop = pidx_last; 379f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); 380f2d6ace4SSean Bruno pi->ipi_new_pidx = i; 381f2d6ace4SSean Bruno 382f2d6ace4SSean Bruno return (0); 383f2d6ace4SSean Bruno } 384f2d6ace4SSean Bruno 385f2d6ace4SSean Bruno static void 386f2d6ace4SSean Bruno em_isc_txd_flush(void *arg, uint16_t txqid, uint32_t pidx) 387f2d6ace4SSean Bruno { 388f2d6ace4SSean Bruno struct adapter *adapter = arg; 389f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 390f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 391f2d6ace4SSean Bruno 392f2d6ace4SSean Bruno E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); 393f2d6ace4SSean Bruno } 394f2d6ace4SSean Bruno 395f2d6ace4SSean Bruno static int 396f2d6ace4SSean Bruno em_isc_txd_credits_update(void *arg, uint16_t txqid, uint32_t cidx_init, bool clear) 397f2d6ace4SSean Bruno { 398f2d6ace4SSean Bruno struct adapter *adapter = arg; 399f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 400f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 401f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 402f2d6ace4SSean Bruno 403f2d6ace4SSean Bruno u32 cidx, processed = 0; 404f2d6ace4SSean Bruno int last, done; 405f2d6ace4SSean Bruno struct em_txbuffer *buf; 406f2d6ace4SSean Bruno struct e1000_tx_desc *tx_desc, *eop_desc; 407f2d6ace4SSean Bruno 408f2d6ace4SSean Bruno cidx = cidx_init; 409f2d6ace4SSean Bruno buf = &txr->tx_buffers[cidx]; 410f2d6ace4SSean Bruno tx_desc = &txr->tx_base[cidx]; 411f2d6ace4SSean Bruno last = buf->eop; 412f2d6ace4SSean Bruno eop_desc = &txr->tx_base[last]; 413f2d6ace4SSean Bruno 414f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "credits_update: cidx_init=%d clear=%d last=%d\n", 415f2d6ace4SSean Bruno cidx_init, clear, last); 416f2d6ace4SSean Bruno /* 417f2d6ace4SSean Bruno * What this does is get the index of the 418f2d6ace4SSean Bruno * first descriptor AFTER the EOP of the 419f2d6ace4SSean Bruno * first packet, that way we can do the 420f2d6ace4SSean Bruno * simple comparison on the inner while loop. 421f2d6ace4SSean Bruno */ 422f2d6ace4SSean Bruno if (++last == scctx->isc_ntxd[0]) 423f2d6ace4SSean Bruno last = 0; 424f2d6ace4SSean Bruno done = last; 425f2d6ace4SSean Bruno 426f2d6ace4SSean Bruno 427f2d6ace4SSean Bruno while (eop_desc->upper.fields.status & E1000_TXD_STAT_DD) { 428f2d6ace4SSean Bruno /* We clean the range of the packet */ 429f2d6ace4SSean Bruno while (cidx != done) { 430f2d6ace4SSean Bruno if (clear) { 431f2d6ace4SSean Bruno tx_desc->upper.data = 0; 432f2d6ace4SSean Bruno tx_desc->lower.data = 0; 433f2d6ace4SSean Bruno tx_desc->buffer_addr = 0; 434f2d6ace4SSean Bruno buf->eop = -1; 435f2d6ace4SSean Bruno } 436f2d6ace4SSean Bruno tx_desc++; 437f2d6ace4SSean Bruno buf++; 438f2d6ace4SSean Bruno processed++; 439f2d6ace4SSean Bruno 440f2d6ace4SSean Bruno /* wrap the ring ? */ 441f2d6ace4SSean Bruno if (++cidx == scctx->isc_ntxd[0]) { 442f2d6ace4SSean Bruno cidx = 0; 443f2d6ace4SSean Bruno } 444f2d6ace4SSean Bruno buf = &txr->tx_buffers[cidx]; 445f2d6ace4SSean Bruno tx_desc = &txr->tx_base[cidx]; 446f2d6ace4SSean Bruno } 447f2d6ace4SSean Bruno /* See if we can continue to the next packet */ 448f2d6ace4SSean Bruno last = buf->eop; 449f2d6ace4SSean Bruno if (last == -1) 450f2d6ace4SSean Bruno break; 451f2d6ace4SSean Bruno eop_desc = &txr->tx_base[last]; 452f2d6ace4SSean Bruno /* Get new done point */ 453f2d6ace4SSean Bruno if (++last == scctx->isc_ntxd[0]) 454f2d6ace4SSean Bruno last = 0; 455f2d6ace4SSean Bruno done = last; 456f2d6ace4SSean Bruno } 457f2d6ace4SSean Bruno 458f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "Processed %d credits update\n", processed); 459f2d6ace4SSean Bruno return(processed); 460f2d6ace4SSean Bruno } 461f2d6ace4SSean Bruno 462f2d6ace4SSean Bruno static void 463f2d6ace4SSean Bruno lem_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, 464f2d6ace4SSean Bruno uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused) 465f2d6ace4SSean Bruno { 466f2d6ace4SSean Bruno struct adapter *sc = arg; 467f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 468f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 469f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 470f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 471f2d6ace4SSean Bruno int i; 472f2d6ace4SSean Bruno uint32_t next_pidx; 473f2d6ace4SSean Bruno 474f2d6ace4SSean Bruno for (i = 0, next_pidx = pidx; i < count; i++) { 475f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; 476f2d6ace4SSean Bruno rxd->buffer_addr = htole64(paddrs[i]); 477f2d6ace4SSean Bruno /* status bits must be cleared */ 478f2d6ace4SSean Bruno rxd->status = 0; 479f2d6ace4SSean Bruno 480f2d6ace4SSean Bruno if (++next_pidx == scctx->isc_nrxd[0]) 481f2d6ace4SSean Bruno next_pidx = 0; 482f2d6ace4SSean Bruno } 483f2d6ace4SSean Bruno } 484f2d6ace4SSean Bruno 485f2d6ace4SSean Bruno static void 486f2d6ace4SSean Bruno em_isc_rxd_refill(void *arg, uint16_t rxqid, uint8_t flid __unused, 487f2d6ace4SSean Bruno uint32_t pidx, uint64_t *paddrs, caddr_t *vaddrs __unused, uint16_t count, uint16_t buflen __unused) 488f2d6ace4SSean Bruno { 489f2d6ace4SSean Bruno struct adapter *sc = arg; 490f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 491f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 492f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 493f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 494f2d6ace4SSean Bruno int i; 495f2d6ace4SSean Bruno uint32_t next_pidx; 496f2d6ace4SSean Bruno 497f2d6ace4SSean Bruno for (i = 0, next_pidx = pidx; i < count; i++) { 498f2d6ace4SSean Bruno rxd = &rxr->rx_base[next_pidx]; 499f2d6ace4SSean Bruno rxd->read.buffer_addr = htole64(paddrs[i]); 500f2d6ace4SSean Bruno /* DD bits must be cleared */ 501f2d6ace4SSean Bruno rxd->wb.upper.status_error = 0; 502f2d6ace4SSean Bruno 503f2d6ace4SSean Bruno if (++next_pidx == scctx->isc_nrxd[0]) 504f2d6ace4SSean Bruno next_pidx = 0; 505f2d6ace4SSean Bruno } 506f2d6ace4SSean Bruno } 507f2d6ace4SSean Bruno 508f2d6ace4SSean Bruno static void 509f2d6ace4SSean Bruno em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, uint32_t pidx) 510f2d6ace4SSean Bruno { 511f2d6ace4SSean Bruno struct adapter *sc = arg; 512f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 513f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 514f2d6ace4SSean Bruno 515f2d6ace4SSean Bruno E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 516f2d6ace4SSean Bruno } 517f2d6ace4SSean Bruno 518f2d6ace4SSean Bruno static int 519f2d6ace4SSean Bruno lem_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget) 520f2d6ace4SSean Bruno { 521f2d6ace4SSean Bruno struct adapter *sc = arg; 522f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 523f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 524f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 525f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 526f2d6ace4SSean Bruno u32 staterr = 0; 527f2d6ace4SSean Bruno int cnt, i; 528f2d6ace4SSean Bruno 529f2d6ace4SSean Bruno for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 530f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; 531f2d6ace4SSean Bruno staterr = rxd->status; 532f2d6ace4SSean Bruno 533f2d6ace4SSean Bruno if ((staterr & E1000_RXD_STAT_DD) == 0) 534f2d6ace4SSean Bruno break; 535f2d6ace4SSean Bruno 536f2d6ace4SSean Bruno if (++i == scctx->isc_nrxd[0]) 537f2d6ace4SSean Bruno i = 0; 538f2d6ace4SSean Bruno 539f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_EOP) 540f2d6ace4SSean Bruno cnt++; 541f2d6ace4SSean Bruno } 542f2d6ace4SSean Bruno return (cnt); 543f2d6ace4SSean Bruno } 544f2d6ace4SSean Bruno 545f2d6ace4SSean Bruno static int 546f2d6ace4SSean Bruno em_isc_rxd_available(void *arg, uint16_t rxqid, uint32_t idx, int budget) 547f2d6ace4SSean Bruno { 548f2d6ace4SSean Bruno struct adapter *sc = arg; 549f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 550f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 551f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 552f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 553f2d6ace4SSean Bruno u32 staterr = 0; 554f2d6ace4SSean Bruno int cnt, i; 555f2d6ace4SSean Bruno 556f2d6ace4SSean Bruno for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 557f2d6ace4SSean Bruno rxd = &rxr->rx_base[i]; 558f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 559f2d6ace4SSean Bruno 560f2d6ace4SSean Bruno if ((staterr & E1000_RXD_STAT_DD) == 0) 561f2d6ace4SSean Bruno break; 562f2d6ace4SSean Bruno 563f2d6ace4SSean Bruno if (++i == scctx->isc_nrxd[0]) { 564f2d6ace4SSean Bruno i = 0; 565f2d6ace4SSean Bruno } 566f2d6ace4SSean Bruno 567f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_EOP) 568f2d6ace4SSean Bruno cnt++; 569f2d6ace4SSean Bruno 570f2d6ace4SSean Bruno } 571f2d6ace4SSean Bruno return (cnt); 572f2d6ace4SSean Bruno } 573f2d6ace4SSean Bruno 574f2d6ace4SSean Bruno static int 575f2d6ace4SSean Bruno lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 576f2d6ace4SSean Bruno { 577f2d6ace4SSean Bruno struct adapter *adapter = arg; 578f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 579f2d6ace4SSean Bruno struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; 580f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 581f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 582f2d6ace4SSean Bruno u16 len; 583f2d6ace4SSean Bruno u32 status, errors; 584f2d6ace4SSean Bruno bool eop; 585f2d6ace4SSean Bruno int i, cidx; 586f2d6ace4SSean Bruno 587f2d6ace4SSean Bruno status = errors = i = 0; 588f2d6ace4SSean Bruno cidx = ri->iri_cidx; 589f2d6ace4SSean Bruno 590f2d6ace4SSean Bruno do { 591f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; 592f2d6ace4SSean Bruno status = rxd->status; 593f2d6ace4SSean Bruno errors = rxd->errors; 594f2d6ace4SSean Bruno 595f2d6ace4SSean Bruno /* Error Checking then decrement count */ 596f2d6ace4SSean Bruno MPASS ((status & E1000_RXD_STAT_DD) != 0); 597f2d6ace4SSean Bruno 598f2d6ace4SSean Bruno len = le16toh(rxd->length); 599f2d6ace4SSean Bruno ri->iri_len += len; 600f2d6ace4SSean Bruno 601f2d6ace4SSean Bruno eop = (status & E1000_RXD_STAT_EOP) != 0; 602f2d6ace4SSean Bruno 603f2d6ace4SSean Bruno /* Make sure bad packets are discarded */ 604f2d6ace4SSean Bruno if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 605f2d6ace4SSean Bruno adapter->dropped_pkts++; 606f2d6ace4SSean Bruno /* XXX fixup if common */ 607f2d6ace4SSean Bruno return (EBADMSG); 608f2d6ace4SSean Bruno } 609f2d6ace4SSean Bruno 610f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 0; 611f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 612f2d6ace4SSean Bruno ri->iri_frags[i].irf_len = len; 613f2d6ace4SSean Bruno /* Zero out the receive descriptors status. */ 614f2d6ace4SSean Bruno rxd->status = 0; 615f2d6ace4SSean Bruno 616f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 617f2d6ace4SSean Bruno cidx = 0; 618f2d6ace4SSean Bruno i++; 619f2d6ace4SSean Bruno } while (!eop); 620f2d6ace4SSean Bruno 621f2d6ace4SSean Bruno /* XXX add a faster way to look this up */ 622f2d6ace4SSean Bruno if (adapter->hw.mac.type >= e1000_82543 && !(status & E1000_RXD_STAT_IXSM)) 623f2d6ace4SSean Bruno lem_receive_checksum(status, errors, ri); 624f2d6ace4SSean Bruno 625f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_VP) { 626f2d6ace4SSean Bruno ri->iri_vtag = le16toh(rxd->special); 627f2d6ace4SSean Bruno ri->iri_flags |= M_VLANTAG; 628f2d6ace4SSean Bruno } 629f2d6ace4SSean Bruno 630f2d6ace4SSean Bruno ri->iri_nfrags = i; 631f2d6ace4SSean Bruno 632f2d6ace4SSean Bruno return (0); 633f2d6ace4SSean Bruno } 634f2d6ace4SSean Bruno 635f2d6ace4SSean Bruno static int 636f2d6ace4SSean Bruno em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 637f2d6ace4SSean Bruno { 638f2d6ace4SSean Bruno struct adapter *adapter = arg; 639f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 640f2d6ace4SSean Bruno struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; 641f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 642f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 643f2d6ace4SSean Bruno 644f2d6ace4SSean Bruno u16 len; 645f2d6ace4SSean Bruno u32 staterr = 0; 646f2d6ace4SSean Bruno bool eop; 647f2d6ace4SSean Bruno int i, cidx, vtag; 648f2d6ace4SSean Bruno 649f2d6ace4SSean Bruno i = vtag = 0; 650f2d6ace4SSean Bruno cidx = ri->iri_cidx; 651f2d6ace4SSean Bruno 652f2d6ace4SSean Bruno do { 653f2d6ace4SSean Bruno rxd = &rxr->rx_base[cidx]; 654f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 655f2d6ace4SSean Bruno 656f2d6ace4SSean Bruno /* Error Checking then decrement count */ 657f2d6ace4SSean Bruno MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 658f2d6ace4SSean Bruno 659f2d6ace4SSean Bruno len = le16toh(rxd->wb.upper.length); 660f2d6ace4SSean Bruno ri->iri_len += len; 661f2d6ace4SSean Bruno 662f2d6ace4SSean Bruno eop = (staterr & E1000_RXD_STAT_EOP) != 0; 663f2d6ace4SSean Bruno 664f2d6ace4SSean Bruno /* Make sure bad packets are discarded */ 665f2d6ace4SSean Bruno if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 666f2d6ace4SSean Bruno adapter->dropped_pkts++; 667f2d6ace4SSean Bruno return EBADMSG; 668f2d6ace4SSean Bruno } 669f2d6ace4SSean Bruno 670f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 0; 671f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 672f2d6ace4SSean Bruno ri->iri_frags[i].irf_len = len; 673f2d6ace4SSean Bruno /* Zero out the receive descriptors status. */ 674f2d6ace4SSean Bruno rxd->wb.upper.status_error &= htole32(~0xFF); 675f2d6ace4SSean Bruno 676f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 677f2d6ace4SSean Bruno cidx = 0; 678f2d6ace4SSean Bruno i++; 679f2d6ace4SSean Bruno } while (!eop); 680f2d6ace4SSean Bruno 681f2d6ace4SSean Bruno /* XXX add a faster way to look this up */ 682f2d6ace4SSean Bruno if (adapter->hw.mac.type >= e1000_82543) 683f2d6ace4SSean Bruno em_receive_checksum(staterr, ri); 684f2d6ace4SSean Bruno 685f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_VP) { 686f2d6ace4SSean Bruno vtag = le16toh(rxd->wb.upper.vlan); 687f2d6ace4SSean Bruno } 688f2d6ace4SSean Bruno 689f2d6ace4SSean Bruno ri->iri_vtag = vtag; 690f2d6ace4SSean Bruno ri->iri_nfrags = i; 691f2d6ace4SSean Bruno if (vtag) 692f2d6ace4SSean Bruno ri->iri_flags |= M_VLANTAG; 693f2d6ace4SSean Bruno 694f2d6ace4SSean Bruno return (0); 695f2d6ace4SSean Bruno } 696f2d6ace4SSean Bruno 697f2d6ace4SSean Bruno /********************************************************************* 698f2d6ace4SSean Bruno * 699f2d6ace4SSean Bruno * Verify that the hardware indicated that the checksum is valid. 700f2d6ace4SSean Bruno * Inform the stack about the status of checksum so that stack 701f2d6ace4SSean Bruno * doesn't spend time verifying the checksum. 702f2d6ace4SSean Bruno * 703f2d6ace4SSean Bruno *********************************************************************/ 704f2d6ace4SSean Bruno static void 705f2d6ace4SSean Bruno lem_receive_checksum(int status, int errors, if_rxd_info_t ri) 706f2d6ace4SSean Bruno { 707f2d6ace4SSean Bruno /* Did it pass? */ 708f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IPCS && !(errors & E1000_RXD_ERR_IPE)) 709f2d6ace4SSean Bruno ri->iri_csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); 710f2d6ace4SSean Bruno 711f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_TCPCS) { 712f2d6ace4SSean Bruno /* Did it pass? */ 713f2d6ace4SSean Bruno if (!(errors & E1000_RXD_ERR_TCPE)) { 714f2d6ace4SSean Bruno ri->iri_csum_flags |= 715f2d6ace4SSean Bruno (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 716f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 717f2d6ace4SSean Bruno } 718f2d6ace4SSean Bruno } 719f2d6ace4SSean Bruno } 720f2d6ace4SSean Bruno 721f2d6ace4SSean Bruno static void 722f2d6ace4SSean Bruno em_receive_checksum(uint32_t status, if_rxd_info_t ri) 723f2d6ace4SSean Bruno { 724f2d6ace4SSean Bruno ri->iri_csum_flags = 0; 725f2d6ace4SSean Bruno 726f2d6ace4SSean Bruno /* Ignore Checksum bit is set */ 727f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IXSM) 728f2d6ace4SSean Bruno return; 729f2d6ace4SSean Bruno 730f2d6ace4SSean Bruno /* If the IP checksum exists and there is no IP Checksum error */ 731f2d6ace4SSean Bruno if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == 732f2d6ace4SSean Bruno E1000_RXD_STAT_IPCS) { 733f2d6ace4SSean Bruno ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); 734f2d6ace4SSean Bruno } 735f2d6ace4SSean Bruno 736f2d6ace4SSean Bruno /* TCP or UDP checksum */ 737f2d6ace4SSean Bruno if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 738f2d6ace4SSean Bruno E1000_RXD_STAT_TCPCS) { 739f2d6ace4SSean Bruno ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 740f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 741f2d6ace4SSean Bruno } 742f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_UDPCS) { 743f2d6ace4SSean Bruno ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 744f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 745f2d6ace4SSean Bruno } 746f2d6ace4SSean Bruno } 747