1d37cece2SSean Bruno /*- 27021bf05SStephen Hurd * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 37021bf05SStephen Hurd * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> 4d37cece2SSean Bruno * All rights reserved. 5d37cece2SSean Bruno * 6d37cece2SSean Bruno * Redistribution and use in source and binary forms, with or without 7d37cece2SSean Bruno * modification, are permitted provided that the following conditions 8d37cece2SSean Bruno * are met: 9d37cece2SSean Bruno * 1. Redistributions of source code must retain the above copyright 10d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer. 11d37cece2SSean Bruno * 2. Redistributions in binary form must reproduce the above copyright 12d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer in the 13d37cece2SSean Bruno * documentation and/or other materials provided with the distribution. 14d37cece2SSean Bruno * 15d37cece2SSean Bruno * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16d37cece2SSean Bruno * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17d37cece2SSean Bruno * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18d37cece2SSean Bruno * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19d37cece2SSean Bruno * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20d37cece2SSean Bruno * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21d37cece2SSean Bruno * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22d37cece2SSean Bruno * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23d37cece2SSean Bruno * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24d37cece2SSean Bruno * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25d37cece2SSean Bruno * SUCH DAMAGE. 26d37cece2SSean Bruno */ 27d37cece2SSean Bruno 28f2d6ace4SSean Bruno /* $FreeBSD$ */ 29f2d6ace4SSean Bruno #include "if_em.h" 30f2d6ace4SSean Bruno 31f2d6ace4SSean Bruno #ifdef RSS 32f2d6ace4SSean Bruno #include <net/rss_config.h> 33f2d6ace4SSean Bruno #include <netinet/in_rss.h> 34f2d6ace4SSean Bruno #endif 35f2d6ace4SSean Bruno 36f2d6ace4SSean Bruno #ifdef VERBOSE_DEBUG 37f2d6ace4SSean Bruno #define DPRINTF device_printf 38f2d6ace4SSean Bruno #else 39f2d6ace4SSean Bruno #define DPRINTF(...) 40f2d6ace4SSean Bruno #endif 41f2d6ace4SSean Bruno 42f2d6ace4SSean Bruno /********************************************************************* 43f2d6ace4SSean Bruno * Local Function prototypes 44f2d6ace4SSean Bruno *********************************************************************/ 4595246abbSSean Bruno static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, 4695246abbSSean Bruno u32 *txd_lower); 4795246abbSSean Bruno static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, 4895246abbSSean Bruno u32 *txd_upper, u32 *txd_lower); 49f2d6ace4SSean Bruno static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); 5095246abbSSean Bruno static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 5195246abbSSean Bruno static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); 5295246abbSSean Bruno static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); 5395246abbSSean Bruno static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 5495246abbSSean Bruno qidx_t pidx); 5595246abbSSean Bruno static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 5695246abbSSean Bruno qidx_t budget); 57f2d6ace4SSean Bruno static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 58f2d6ace4SSean Bruno 5995246abbSSean Bruno static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); 60f2d6ace4SSean Bruno 6195246abbSSean Bruno static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 6295246abbSSean Bruno qidx_t budget); 63f2d6ace4SSean Bruno static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 64f2d6ace4SSean Bruno 65f2d6ace4SSean Bruno static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri); 66f2d6ace4SSean Bruno static void em_receive_checksum(uint32_t status, if_rxd_info_t ri); 67cbf1505dSSean Bruno static int em_determine_rsstype(u32 pkt_info); 68f2d6ace4SSean Bruno extern int em_intr(void *arg); 69f2d6ace4SSean Bruno 70f2d6ace4SSean Bruno struct if_txrx em_txrx = { 71fbf8b74cSMark Johnston .ift_txd_encap = em_isc_txd_encap, 72fbf8b74cSMark Johnston .ift_txd_flush = em_isc_txd_flush, 73fbf8b74cSMark Johnston .ift_txd_credits_update = em_isc_txd_credits_update, 74fbf8b74cSMark Johnston .ift_rxd_available = em_isc_rxd_available, 75fbf8b74cSMark Johnston .ift_rxd_pkt_get = em_isc_rxd_pkt_get, 76fbf8b74cSMark Johnston .ift_rxd_refill = em_isc_rxd_refill, 77fbf8b74cSMark Johnston .ift_rxd_flush = em_isc_rxd_flush, 78fbf8b74cSMark Johnston .ift_legacy_intr = em_intr 79f2d6ace4SSean Bruno }; 80f2d6ace4SSean Bruno 81f2d6ace4SSean Bruno struct if_txrx lem_txrx = { 82fbf8b74cSMark Johnston .ift_txd_encap = em_isc_txd_encap, 83fbf8b74cSMark Johnston .ift_txd_flush = em_isc_txd_flush, 84fbf8b74cSMark Johnston .ift_txd_credits_update = em_isc_txd_credits_update, 85fbf8b74cSMark Johnston .ift_rxd_available = lem_isc_rxd_available, 86fbf8b74cSMark Johnston .ift_rxd_pkt_get = lem_isc_rxd_pkt_get, 87fbf8b74cSMark Johnston .ift_rxd_refill = lem_isc_rxd_refill, 88fbf8b74cSMark Johnston .ift_rxd_flush = em_isc_rxd_flush, 89fbf8b74cSMark Johnston .ift_legacy_intr = em_intr 90f2d6ace4SSean Bruno }; 91f2d6ace4SSean Bruno 92f2d6ace4SSean Bruno extern if_shared_ctx_t em_sctx; 93f2d6ace4SSean Bruno 9495246abbSSean Bruno void 9595246abbSSean Bruno em_dump_rs(struct adapter *adapter) 9695246abbSSean Bruno { 9795246abbSSean Bruno if_softc_ctx_t scctx = adapter->shared; 9895246abbSSean Bruno struct em_tx_queue *que; 9995246abbSSean Bruno struct tx_ring *txr; 10095246abbSSean Bruno qidx_t i, ntxd, qid, cur; 10195246abbSSean Bruno int16_t rs_cidx; 10295246abbSSean Bruno uint8_t status; 10395246abbSSean Bruno 10495246abbSSean Bruno printf("\n"); 10595246abbSSean Bruno ntxd = scctx->isc_ntxd[0]; 10695246abbSSean Bruno for (qid = 0; qid < adapter->tx_num_queues; qid++) { 10795246abbSSean Bruno que = &adapter->tx_queues[qid]; 10895246abbSSean Bruno txr = &que->txr; 10995246abbSSean Bruno rs_cidx = txr->tx_rs_cidx; 11095246abbSSean Bruno if (rs_cidx != txr->tx_rs_pidx) { 11195246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 11295246abbSSean Bruno status = txr->tx_base[cur].upper.fields.status; 11395246abbSSean Bruno if (!(status & E1000_TXD_STAT_DD)) 11495246abbSSean Bruno printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur); 11595246abbSSean Bruno } else { 11695246abbSSean Bruno rs_cidx = (rs_cidx-1)&(ntxd-1); 11795246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 11895246abbSSean Bruno printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur); 11995246abbSSean Bruno } 12095246abbSSean Bruno printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, txr->tx_rs_pidx); 12195246abbSSean Bruno for (i = 0; i < ntxd; i++) { 12295246abbSSean Bruno if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD) 12395246abbSSean Bruno printf("%d set ", i); 12495246abbSSean Bruno } 12595246abbSSean Bruno printf("\n"); 12695246abbSSean Bruno } 12795246abbSSean Bruno } 12895246abbSSean Bruno 129f2d6ace4SSean Bruno /********************************************************************** 130f2d6ace4SSean Bruno * 131f2d6ace4SSean Bruno * Setup work for hardware segmentation offload (TSO) on 132f2d6ace4SSean Bruno * adapters using advanced tx descriptors 133f2d6ace4SSean Bruno * 134f2d6ace4SSean Bruno **********************************************************************/ 135f2d6ace4SSean Bruno static int 136f2d6ace4SSean Bruno em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) 137f2d6ace4SSean Bruno { 138f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 139f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; 140f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 141f2d6ace4SSean Bruno struct e1000_context_desc *TXD; 142f2d6ace4SSean Bruno int cur, hdr_len; 143f2d6ace4SSean Bruno 144f2d6ace4SSean Bruno hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 145f2d6ace4SSean Bruno *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 146f2d6ace4SSean Bruno E1000_TXD_DTYP_D | /* Data descr type */ 147f2d6ace4SSean Bruno E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 148f2d6ace4SSean Bruno 149f2d6ace4SSean Bruno /* IP and/or TCP header checksum calculation and insertion. */ 150f2d6ace4SSean Bruno *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 151f2d6ace4SSean Bruno 152f2d6ace4SSean Bruno cur = pi->ipi_pidx; 153f2d6ace4SSean Bruno TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 154f2d6ace4SSean Bruno 155f2d6ace4SSean Bruno /* 156f2d6ace4SSean Bruno * Start offset for header checksum calculation. 157f2d6ace4SSean Bruno * End offset for header checksum calculation. 158f2d6ace4SSean Bruno * Offset of place put the checksum. 159f2d6ace4SSean Bruno */ 160f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 161f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcse = 162f2d6ace4SSean Bruno htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); 163f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 164f2d6ace4SSean Bruno 165f2d6ace4SSean Bruno /* 166f2d6ace4SSean Bruno * Start offset for payload checksum calculation. 167f2d6ace4SSean Bruno * End offset for payload checksum calculation. 168f2d6ace4SSean Bruno * Offset of place to put the checksum. 169f2d6ace4SSean Bruno */ 170f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 171f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucse = 0; 172f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucso = 173f2d6ace4SSean Bruno pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); 174f2d6ace4SSean Bruno 175f2d6ace4SSean Bruno /* 176f2d6ace4SSean Bruno * Payload size per packet w/o any headers. 177f2d6ace4SSean Bruno * Length of all headers up to payload. 178f2d6ace4SSean Bruno */ 179f2d6ace4SSean Bruno TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); 180f2d6ace4SSean Bruno TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 181f2d6ace4SSean Bruno 182f2d6ace4SSean Bruno TXD->cmd_and_length = htole32(adapter->txd_cmd | 183f2d6ace4SSean Bruno E1000_TXD_CMD_DEXT | /* Extended descr */ 184f2d6ace4SSean Bruno E1000_TXD_CMD_TSE | /* TSE context */ 185f2d6ace4SSean Bruno E1000_TXD_CMD_IP | /* Do IP csum */ 186f2d6ace4SSean Bruno E1000_TXD_CMD_TCP | /* Do TCP checksum */ 187f2d6ace4SSean Bruno (pi->ipi_len - hdr_len)); /* Total len */ 188f2d6ace4SSean Bruno txr->tx_tso = TRUE; 189f2d6ace4SSean Bruno 190f2d6ace4SSean Bruno if (++cur == scctx->isc_ntxd[0]) { 191f2d6ace4SSean Bruno cur = 0; 192f2d6ace4SSean Bruno } 193f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur); 194f2d6ace4SSean Bruno return (cur); 195f2d6ace4SSean Bruno } 196f2d6ace4SSean Bruno 197f2d6ace4SSean Bruno #define TSO_WORKAROUND 4 198f2d6ace4SSean Bruno #define DONT_FORCE_CTX 1 199f2d6ace4SSean Bruno 200f2d6ace4SSean Bruno 201f2d6ace4SSean Bruno /********************************************************************* 202f2d6ace4SSean Bruno * The offload context is protocol specific (TCP/UDP) and thus 203f2d6ace4SSean Bruno * only needs to be set when the protocol changes. The occasion 204f2d6ace4SSean Bruno * of a context change can be a performance detriment, and 205f2d6ace4SSean Bruno * might be better just disabled. The reason arises in the way 206f2d6ace4SSean Bruno * in which the controller supports pipelined requests from the 207f2d6ace4SSean Bruno * Tx data DMA. Up to four requests can be pipelined, and they may 208f2d6ace4SSean Bruno * belong to the same packet or to multiple packets. However all 209f2d6ace4SSean Bruno * requests for one packet are issued before a request is issued 210f2d6ace4SSean Bruno * for a subsequent packet and if a request for the next packet 211f2d6ace4SSean Bruno * requires a context change, that request will be stalled 212f2d6ace4SSean Bruno * until the previous request completes. This means setting up 213f2d6ace4SSean Bruno * a new context effectively disables pipelined Tx data DMA which 214f2d6ace4SSean Bruno * in turn greatly slow down performance to send small sized 215f2d6ace4SSean Bruno * frames. 216f2d6ace4SSean Bruno **********************************************************************/ 217f2d6ace4SSean Bruno 218f2d6ace4SSean Bruno static int 219f2d6ace4SSean Bruno em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) 220f2d6ace4SSean Bruno { 221f2d6ace4SSean Bruno struct e1000_context_desc *TXD = NULL; 222f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 223f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; 224f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 225f2d6ace4SSean Bruno int csum_flags = pi->ipi_csum_flags; 226f2d6ace4SSean Bruno int cur, hdr_len; 227f2d6ace4SSean Bruno u32 cmd; 228f2d6ace4SSean Bruno 229f2d6ace4SSean Bruno cur = pi->ipi_pidx; 230f2d6ace4SSean Bruno hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 231f2d6ace4SSean Bruno cmd = adapter->txd_cmd; 232f2d6ace4SSean Bruno 233f2d6ace4SSean Bruno /* 234f2d6ace4SSean Bruno * The 82574L can only remember the *last* context used 235f2d6ace4SSean Bruno * regardless of queue that it was use for. We cannot reuse 236f2d6ace4SSean Bruno * contexts on this hardware platform and must generate a new 237f2d6ace4SSean Bruno * context every time. 82574L hardware spec, section 7.2.6, 238f2d6ace4SSean Bruno * second note. 239f2d6ace4SSean Bruno */ 240f2d6ace4SSean Bruno if (DONT_FORCE_CTX && 241f2d6ace4SSean Bruno adapter->tx_num_queues == 1 && 242f2d6ace4SSean Bruno txr->csum_lhlen == pi->ipi_ehdrlen && 243f2d6ace4SSean Bruno txr->csum_iphlen == pi->ipi_ip_hlen && 244f2d6ace4SSean Bruno txr->csum_flags == csum_flags) { 245f2d6ace4SSean Bruno /* 246f2d6ace4SSean Bruno * Same csum offload context as the previous packets; 247f2d6ace4SSean Bruno * just return. 248f2d6ace4SSean Bruno */ 249f2d6ace4SSean Bruno *txd_upper = txr->csum_txd_upper; 250f2d6ace4SSean Bruno *txd_lower = txr->csum_txd_lower; 251f2d6ace4SSean Bruno return (cur); 252f2d6ace4SSean Bruno } 253f2d6ace4SSean Bruno 254f2d6ace4SSean Bruno TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 255f2d6ace4SSean Bruno if (csum_flags & CSUM_IP) { 256f2d6ace4SSean Bruno *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 257f2d6ace4SSean Bruno /* 258f2d6ace4SSean Bruno * Start offset for header checksum calculation. 259f2d6ace4SSean Bruno * End offset for header checksum calculation. 260f2d6ace4SSean Bruno * Offset of place to put the checksum. 261f2d6ace4SSean Bruno */ 262f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 263f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); 264f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 265f2d6ace4SSean Bruno cmd |= E1000_TXD_CMD_IP; 266f2d6ace4SSean Bruno } 267f2d6ace4SSean Bruno 268f2d6ace4SSean Bruno if (csum_flags & (CSUM_TCP|CSUM_UDP)) { 269f2d6ace4SSean Bruno uint8_t tucso; 270f2d6ace4SSean Bruno 271f2d6ace4SSean Bruno *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 272f2d6ace4SSean Bruno *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 273f2d6ace4SSean Bruno 274f2d6ace4SSean Bruno if (csum_flags & CSUM_TCP) { 275f2d6ace4SSean Bruno tucso = hdr_len + offsetof(struct tcphdr, th_sum); 276f2d6ace4SSean Bruno cmd |= E1000_TXD_CMD_TCP; 277f2d6ace4SSean Bruno } else 278f2d6ace4SSean Bruno tucso = hdr_len + offsetof(struct udphdr, uh_sum); 279f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucss = hdr_len; 280f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucse = htole16(0); 281f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucso = tucso; 282f2d6ace4SSean Bruno } 283f2d6ace4SSean Bruno 284f2d6ace4SSean Bruno txr->csum_lhlen = pi->ipi_ehdrlen; 285f2d6ace4SSean Bruno txr->csum_iphlen = pi->ipi_ip_hlen; 286f2d6ace4SSean Bruno txr->csum_flags = csum_flags; 287f2d6ace4SSean Bruno txr->csum_txd_upper = *txd_upper; 288f2d6ace4SSean Bruno txr->csum_txd_lower = *txd_lower; 289f2d6ace4SSean Bruno 290f2d6ace4SSean Bruno TXD->tcp_seg_setup.data = htole32(0); 291f2d6ace4SSean Bruno TXD->cmd_and_length = 292f2d6ace4SSean Bruno htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 293f2d6ace4SSean Bruno 294f2d6ace4SSean Bruno if (++cur == scctx->isc_ntxd[0]) { 295f2d6ace4SSean Bruno cur = 0; 296f2d6ace4SSean Bruno } 297f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n", 298f2d6ace4SSean Bruno csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); 299f2d6ace4SSean Bruno return (cur); 300f2d6ace4SSean Bruno } 301f2d6ace4SSean Bruno 302f2d6ace4SSean Bruno static int 303f2d6ace4SSean Bruno em_isc_txd_encap(void *arg, if_pkt_info_t pi) 304f2d6ace4SSean Bruno { 305f2d6ace4SSean Bruno struct adapter *sc = arg; 306f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 307f2d6ace4SSean Bruno struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 308f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 309f2d6ace4SSean Bruno bus_dma_segment_t *segs = pi->ipi_segs; 310f2d6ace4SSean Bruno int nsegs = pi->ipi_nsegs; 311f2d6ace4SSean Bruno int csum_flags = pi->ipi_csum_flags; 312f2d6ace4SSean Bruno int i, j, first, pidx_last; 31395246abbSSean Bruno u32 txd_flags, txd_upper = 0, txd_lower = 0; 314f2d6ace4SSean Bruno 315f2d6ace4SSean Bruno struct e1000_tx_desc *ctxd = NULL; 316f2d6ace4SSean Bruno bool do_tso, tso_desc; 31795246abbSSean Bruno qidx_t ntxd; 318f2d6ace4SSean Bruno 31995246abbSSean Bruno txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; 320f2d6ace4SSean Bruno i = first = pi->ipi_pidx; 321f2d6ace4SSean Bruno do_tso = (csum_flags & CSUM_TSO); 322f2d6ace4SSean Bruno tso_desc = FALSE; 32395246abbSSean Bruno ntxd = scctx->isc_ntxd[0]; 324f2d6ace4SSean Bruno /* 325f2d6ace4SSean Bruno * TSO Hardware workaround, if this packet is not 326f2d6ace4SSean Bruno * TSO, and is only a single descriptor long, and 327f2d6ace4SSean Bruno * it follows a TSO burst, then we need to add a 328f2d6ace4SSean Bruno * sentinel descriptor to prevent premature writeback. 329f2d6ace4SSean Bruno */ 330f2d6ace4SSean Bruno if ((!do_tso) && (txr->tx_tso == TRUE)) { 331f2d6ace4SSean Bruno if (nsegs == 1) 332f2d6ace4SSean Bruno tso_desc = TRUE; 333f2d6ace4SSean Bruno txr->tx_tso = FALSE; 334f2d6ace4SSean Bruno } 335f2d6ace4SSean Bruno 336f2d6ace4SSean Bruno /* Do hardware assists */ 337f2d6ace4SSean Bruno if (do_tso) { 338f2d6ace4SSean Bruno i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); 339f2d6ace4SSean Bruno tso_desc = TRUE; 34082379056SSean Bruno } else if (csum_flags & EM_CSUM_OFFLOAD) { 341f2d6ace4SSean Bruno i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); 342f2d6ace4SSean Bruno } 343f2d6ace4SSean Bruno 344f2d6ace4SSean Bruno if (pi->ipi_mflags & M_VLANTAG) { 345f2d6ace4SSean Bruno /* Set the vlan id. */ 346f2d6ace4SSean Bruno txd_upper |= htole16(pi->ipi_vtag) << 16; 347f2d6ace4SSean Bruno /* Tell hardware to add tag */ 348f2d6ace4SSean Bruno txd_lower |= htole32(E1000_TXD_CMD_VLE); 349f2d6ace4SSean Bruno } 350f2d6ace4SSean Bruno 351f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); 352f2d6ace4SSean Bruno /* XXX adapter->pcix_82544 -- lem_fill_descriptors */ 353f2d6ace4SSean Bruno 354f2d6ace4SSean Bruno /* Set up our transmit descriptors */ 355f2d6ace4SSean Bruno for (j = 0; j < nsegs; j++) { 356f2d6ace4SSean Bruno bus_size_t seg_len; 357f2d6ace4SSean Bruno bus_addr_t seg_addr; 358f2d6ace4SSean Bruno uint32_t cmd; 359f2d6ace4SSean Bruno 360f2d6ace4SSean Bruno ctxd = &txr->tx_base[i]; 361f2d6ace4SSean Bruno seg_addr = segs[j].ds_addr; 362f2d6ace4SSean Bruno seg_len = segs[j].ds_len; 363f2d6ace4SSean Bruno cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; 364f2d6ace4SSean Bruno 365f2d6ace4SSean Bruno /* 36695246abbSSean Bruno * TSO Workaround: 36795246abbSSean Bruno * If this is the last descriptor, we want to 36895246abbSSean Bruno * split it so we have a small final sentinel 369f2d6ace4SSean Bruno */ 370f2d6ace4SSean Bruno if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { 371f2d6ace4SSean Bruno seg_len -= TSO_WORKAROUND; 372f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr); 373f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 374f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 375f2d6ace4SSean Bruno 376f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 377f2d6ace4SSean Bruno i = 0; 378f2d6ace4SSean Bruno 379f2d6ace4SSean Bruno /* Now make the sentinel */ 380f2d6ace4SSean Bruno ctxd = &txr->tx_base[i]; 381f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr + seg_len); 382f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); 383f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 384f2d6ace4SSean Bruno pidx_last = i; 385f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 386f2d6ace4SSean Bruno i = 0; 387f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); 388f2d6ace4SSean Bruno } else { 389f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr); 390f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 391f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 392f2d6ace4SSean Bruno pidx_last = i; 393f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 394f2d6ace4SSean Bruno i = 0; 395f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); 396f2d6ace4SSean Bruno } 397f2d6ace4SSean Bruno } 398f2d6ace4SSean Bruno 399f2d6ace4SSean Bruno /* 400f2d6ace4SSean Bruno * Last Descriptor of Packet 401f2d6ace4SSean Bruno * needs End Of Packet (EOP) 402f2d6ace4SSean Bruno * and Report Status (RS) 403f2d6ace4SSean Bruno */ 4048fd222ebSMatt Macy if (txd_flags && nsegs) { 40595246abbSSean Bruno txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 40695246abbSSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "setting to RS on %d rs_pidx %d first: %d\n", pidx_last, txr->tx_rs_pidx, first); 40795246abbSSean Bruno txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); 40895246abbSSean Bruno MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 40995246abbSSean Bruno } 41095246abbSSean Bruno ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); 411f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); 412f2d6ace4SSean Bruno pi->ipi_new_pidx = i; 413f2d6ace4SSean Bruno 414f2d6ace4SSean Bruno return (0); 415f2d6ace4SSean Bruno } 416f2d6ace4SSean Bruno 417f2d6ace4SSean Bruno static void 41895246abbSSean Bruno em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 419f2d6ace4SSean Bruno { 420f2d6ace4SSean Bruno struct adapter *adapter = arg; 421f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 422f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 423f2d6ace4SSean Bruno 424f2d6ace4SSean Bruno E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); 425f2d6ace4SSean Bruno } 426f2d6ace4SSean Bruno 427f2d6ace4SSean Bruno static int 42895246abbSSean Bruno em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 429f2d6ace4SSean Bruno { 430f2d6ace4SSean Bruno struct adapter *adapter = arg; 431f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 432f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 433f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 434f2d6ace4SSean Bruno 43595246abbSSean Bruno qidx_t processed = 0; 43695246abbSSean Bruno int updated; 43795246abbSSean Bruno qidx_t cur, prev, ntxd, rs_cidx; 43895246abbSSean Bruno int32_t delta; 43995246abbSSean Bruno uint8_t status; 440f2d6ace4SSean Bruno 44195246abbSSean Bruno rs_cidx = txr->tx_rs_cidx; 44295246abbSSean Bruno if (rs_cidx == txr->tx_rs_pidx) 44395246abbSSean Bruno return (0); 44495246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 44595246abbSSean Bruno MPASS(cur != QIDX_INVALID); 44695246abbSSean Bruno status = txr->tx_base[cur].upper.fields.status; 44795246abbSSean Bruno updated = !!(status & E1000_TXD_STAT_DD); 448f2d6ace4SSean Bruno 449adf93b56SEric Joyner if (!updated) 450adf93b56SEric Joyner return (0); 451adf93b56SEric Joyner 452adf93b56SEric Joyner /* If clear is false just let caller know that there 453adf93b56SEric Joyner * are descriptors to reclaim */ 454adf93b56SEric Joyner if (!clear) 455adf93b56SEric Joyner return (1); 45695246abbSSean Bruno 45795246abbSSean Bruno prev = txr->tx_cidx_processed; 45895246abbSSean Bruno ntxd = scctx->isc_ntxd[0]; 45995246abbSSean Bruno do { 46095246abbSSean Bruno delta = (int32_t)cur - (int32_t)prev; 46195246abbSSean Bruno MPASS(prev == 0 || delta != 0); 46295246abbSSean Bruno if (delta < 0) 46395246abbSSean Bruno delta += ntxd; 464bd84f700SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), 46595246abbSSean Bruno "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", 46695246abbSSean Bruno __FUNCTION__, prev, cur, clear, delta); 467f2d6ace4SSean Bruno 46895246abbSSean Bruno processed += delta; 46995246abbSSean Bruno prev = cur; 47095246abbSSean Bruno rs_cidx = (rs_cidx + 1) & (ntxd-1); 47195246abbSSean Bruno if (rs_cidx == txr->tx_rs_pidx) 472f2d6ace4SSean Bruno break; 47395246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 47495246abbSSean Bruno MPASS(cur != QIDX_INVALID); 47595246abbSSean Bruno status = txr->tx_base[cur].upper.fields.status; 47695246abbSSean Bruno } while ((status & E1000_TXD_STAT_DD)); 477f2d6ace4SSean Bruno 47895246abbSSean Bruno txr->tx_rs_cidx = rs_cidx; 47995246abbSSean Bruno txr->tx_cidx_processed = prev; 480f2d6ace4SSean Bruno return(processed); 481f2d6ace4SSean Bruno } 482f2d6ace4SSean Bruno 483f2d6ace4SSean Bruno static void 48495246abbSSean Bruno lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) 485f2d6ace4SSean Bruno { 486f2d6ace4SSean Bruno struct adapter *sc = arg; 487f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 48895246abbSSean Bruno struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; 489f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 490f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 49195246abbSSean Bruno uint64_t *paddrs; 49295246abbSSean Bruno uint32_t next_pidx, pidx; 49395246abbSSean Bruno uint16_t count; 494f2d6ace4SSean Bruno int i; 49595246abbSSean Bruno 49695246abbSSean Bruno paddrs = iru->iru_paddrs; 49795246abbSSean Bruno pidx = iru->iru_pidx; 49895246abbSSean Bruno count = iru->iru_count; 499f2d6ace4SSean Bruno 500f2d6ace4SSean Bruno for (i = 0, next_pidx = pidx; i < count; i++) { 501f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; 502f2d6ace4SSean Bruno rxd->buffer_addr = htole64(paddrs[i]); 503f2d6ace4SSean Bruno /* status bits must be cleared */ 504f2d6ace4SSean Bruno rxd->status = 0; 505f2d6ace4SSean Bruno 506f2d6ace4SSean Bruno if (++next_pidx == scctx->isc_nrxd[0]) 507f2d6ace4SSean Bruno next_pidx = 0; 508f2d6ace4SSean Bruno } 509f2d6ace4SSean Bruno } 510f2d6ace4SSean Bruno 511f2d6ace4SSean Bruno static void 51295246abbSSean Bruno em_isc_rxd_refill(void *arg, if_rxd_update_t iru) 513f2d6ace4SSean Bruno { 514f2d6ace4SSean Bruno struct adapter *sc = arg; 515f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 51695246abbSSean Bruno uint16_t rxqid = iru->iru_qsidx; 517f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 518f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 519f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 52095246abbSSean Bruno uint64_t *paddrs; 52195246abbSSean Bruno uint32_t next_pidx, pidx; 52295246abbSSean Bruno uint16_t count; 523f2d6ace4SSean Bruno int i; 52495246abbSSean Bruno 52595246abbSSean Bruno paddrs = iru->iru_paddrs; 52695246abbSSean Bruno pidx = iru->iru_pidx; 52795246abbSSean Bruno count = iru->iru_count; 528f2d6ace4SSean Bruno 529f2d6ace4SSean Bruno for (i = 0, next_pidx = pidx; i < count; i++) { 530f2d6ace4SSean Bruno rxd = &rxr->rx_base[next_pidx]; 531f2d6ace4SSean Bruno rxd->read.buffer_addr = htole64(paddrs[i]); 532ab2e3f79SStephen Hurd /* DD bits must be cleared */ 533ab2e3f79SStephen Hurd rxd->wb.upper.status_error = 0; 534f2d6ace4SSean Bruno 535f2d6ace4SSean Bruno if (++next_pidx == scctx->isc_nrxd[0]) 536f2d6ace4SSean Bruno next_pidx = 0; 537f2d6ace4SSean Bruno } 538f2d6ace4SSean Bruno } 539f2d6ace4SSean Bruno 540f2d6ace4SSean Bruno static void 54195246abbSSean Bruno em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) 542f2d6ace4SSean Bruno { 543f2d6ace4SSean Bruno struct adapter *sc = arg; 544f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 545f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 546f2d6ace4SSean Bruno 547f2d6ace4SSean Bruno E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 548f2d6ace4SSean Bruno } 549f2d6ace4SSean Bruno 550f2d6ace4SSean Bruno static int 55195246abbSSean Bruno lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 552f2d6ace4SSean Bruno { 553f2d6ace4SSean Bruno struct adapter *sc = arg; 554f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 555f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 556f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 557f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 558f2d6ace4SSean Bruno u32 staterr = 0; 559f2d6ace4SSean Bruno int cnt, i; 560f2d6ace4SSean Bruno 561ab2e3f79SStephen Hurd for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 562f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; 563f2d6ace4SSean Bruno staterr = rxd->status; 564f2d6ace4SSean Bruno 565f2d6ace4SSean Bruno if ((staterr & E1000_RXD_STAT_DD) == 0) 566f2d6ace4SSean Bruno break; 567f2d6ace4SSean Bruno if (++i == scctx->isc_nrxd[0]) 568f2d6ace4SSean Bruno i = 0; 569f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_EOP) 570f2d6ace4SSean Bruno cnt++; 571f2d6ace4SSean Bruno } 572f2d6ace4SSean Bruno return (cnt); 573f2d6ace4SSean Bruno } 574f2d6ace4SSean Bruno 575f2d6ace4SSean Bruno static int 57695246abbSSean Bruno em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 577f2d6ace4SSean Bruno { 578f2d6ace4SSean Bruno struct adapter *sc = arg; 579f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 580f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 581f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 582f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 583f2d6ace4SSean Bruno u32 staterr = 0; 584f2d6ace4SSean Bruno int cnt, i; 585f2d6ace4SSean Bruno 586ab2e3f79SStephen Hurd for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 587f2d6ace4SSean Bruno rxd = &rxr->rx_base[i]; 588f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 589f2d6ace4SSean Bruno 590f2d6ace4SSean Bruno if ((staterr & E1000_RXD_STAT_DD) == 0) 591f2d6ace4SSean Bruno break; 592adf93b56SEric Joyner if (++i == scctx->isc_nrxd[0]) 593f2d6ace4SSean Bruno i = 0; 594f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_EOP) 595f2d6ace4SSean Bruno cnt++; 596f2d6ace4SSean Bruno } 597f2d6ace4SSean Bruno return (cnt); 598f2d6ace4SSean Bruno } 599f2d6ace4SSean Bruno 600f2d6ace4SSean Bruno static int 601f2d6ace4SSean Bruno lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 602f2d6ace4SSean Bruno { 603f2d6ace4SSean Bruno struct adapter *adapter = arg; 604f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 605f2d6ace4SSean Bruno struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; 606f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 607f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 608f2d6ace4SSean Bruno u16 len; 609f2d6ace4SSean Bruno u32 status, errors; 610f2d6ace4SSean Bruno bool eop; 611f2d6ace4SSean Bruno int i, cidx; 612f2d6ace4SSean Bruno 613f2d6ace4SSean Bruno status = errors = i = 0; 614f2d6ace4SSean Bruno cidx = ri->iri_cidx; 615f2d6ace4SSean Bruno 616f2d6ace4SSean Bruno do { 617f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; 618f2d6ace4SSean Bruno status = rxd->status; 619f2d6ace4SSean Bruno errors = rxd->errors; 620f2d6ace4SSean Bruno 621f2d6ace4SSean Bruno /* Error Checking then decrement count */ 622f2d6ace4SSean Bruno MPASS ((status & E1000_RXD_STAT_DD) != 0); 623f2d6ace4SSean Bruno 624f2d6ace4SSean Bruno len = le16toh(rxd->length); 625f2d6ace4SSean Bruno ri->iri_len += len; 626f2d6ace4SSean Bruno 627f2d6ace4SSean Bruno eop = (status & E1000_RXD_STAT_EOP) != 0; 628f2d6ace4SSean Bruno 629f2d6ace4SSean Bruno /* Make sure bad packets are discarded */ 630f2d6ace4SSean Bruno if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 631f2d6ace4SSean Bruno adapter->dropped_pkts++; 632f2d6ace4SSean Bruno /* XXX fixup if common */ 633f2d6ace4SSean Bruno return (EBADMSG); 634f2d6ace4SSean Bruno } 635f2d6ace4SSean Bruno 636f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 0; 637f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 638f2d6ace4SSean Bruno ri->iri_frags[i].irf_len = len; 639f2d6ace4SSean Bruno /* Zero out the receive descriptors status. */ 640f2d6ace4SSean Bruno rxd->status = 0; 641f2d6ace4SSean Bruno 642f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 643f2d6ace4SSean Bruno cidx = 0; 644f2d6ace4SSean Bruno i++; 645f2d6ace4SSean Bruno } while (!eop); 646f2d6ace4SSean Bruno 647f2d6ace4SSean Bruno /* XXX add a faster way to look this up */ 648f2d6ace4SSean Bruno if (adapter->hw.mac.type >= e1000_82543 && !(status & E1000_RXD_STAT_IXSM)) 649f2d6ace4SSean Bruno lem_receive_checksum(status, errors, ri); 650f2d6ace4SSean Bruno 651f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_VP) { 652f2d6ace4SSean Bruno ri->iri_vtag = le16toh(rxd->special); 653f2d6ace4SSean Bruno ri->iri_flags |= M_VLANTAG; 654f2d6ace4SSean Bruno } 655f2d6ace4SSean Bruno 656f2d6ace4SSean Bruno ri->iri_nfrags = i; 657f2d6ace4SSean Bruno 658f2d6ace4SSean Bruno return (0); 659f2d6ace4SSean Bruno } 660f2d6ace4SSean Bruno 661f2d6ace4SSean Bruno static int 662f2d6ace4SSean Bruno em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 663f2d6ace4SSean Bruno { 664f2d6ace4SSean Bruno struct adapter *adapter = arg; 665f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 666f2d6ace4SSean Bruno struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; 667f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 668f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 669f2d6ace4SSean Bruno 670f2d6ace4SSean Bruno u16 len; 671cbf1505dSSean Bruno u32 pkt_info; 672f2d6ace4SSean Bruno u32 staterr = 0; 673f2d6ace4SSean Bruno bool eop; 674f2d6ace4SSean Bruno int i, cidx, vtag; 675f2d6ace4SSean Bruno 676f2d6ace4SSean Bruno i = vtag = 0; 677f2d6ace4SSean Bruno cidx = ri->iri_cidx; 678f2d6ace4SSean Bruno 679f2d6ace4SSean Bruno do { 680f2d6ace4SSean Bruno rxd = &rxr->rx_base[cidx]; 681f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 682cbf1505dSSean Bruno pkt_info = le32toh(rxd->wb.lower.mrq); 683f2d6ace4SSean Bruno 684f2d6ace4SSean Bruno /* Error Checking then decrement count */ 685ab2e3f79SStephen Hurd MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 686f2d6ace4SSean Bruno 687f2d6ace4SSean Bruno len = le16toh(rxd->wb.upper.length); 688f2d6ace4SSean Bruno ri->iri_len += len; 689f2d6ace4SSean Bruno 690f2d6ace4SSean Bruno eop = (staterr & E1000_RXD_STAT_EOP) != 0; 691f2d6ace4SSean Bruno 692f2d6ace4SSean Bruno /* Make sure bad packets are discarded */ 693f2d6ace4SSean Bruno if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 694f2d6ace4SSean Bruno adapter->dropped_pkts++; 695f2d6ace4SSean Bruno return EBADMSG; 696f2d6ace4SSean Bruno } 697f2d6ace4SSean Bruno 698f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 0; 699f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 700f2d6ace4SSean Bruno ri->iri_frags[i].irf_len = len; 701f2d6ace4SSean Bruno /* Zero out the receive descriptors status. */ 702f2d6ace4SSean Bruno rxd->wb.upper.status_error &= htole32(~0xFF); 703f2d6ace4SSean Bruno 704f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 705f2d6ace4SSean Bruno cidx = 0; 706f2d6ace4SSean Bruno i++; 707f2d6ace4SSean Bruno } while (!eop); 708f2d6ace4SSean Bruno 709f2d6ace4SSean Bruno /* XXX add a faster way to look this up */ 710f2d6ace4SSean Bruno if (adapter->hw.mac.type >= e1000_82543) 711f2d6ace4SSean Bruno em_receive_checksum(staterr, ri); 712f2d6ace4SSean Bruno 713f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_VP) { 714f2d6ace4SSean Bruno vtag = le16toh(rxd->wb.upper.vlan); 715f2d6ace4SSean Bruno } 716f2d6ace4SSean Bruno 717f2d6ace4SSean Bruno ri->iri_vtag = vtag; 718f2d6ace4SSean Bruno if (vtag) 719f2d6ace4SSean Bruno ri->iri_flags |= M_VLANTAG; 720f2d6ace4SSean Bruno 72195246abbSSean Bruno ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); 722cbf1505dSSean Bruno ri->iri_rsstype = em_determine_rsstype(pkt_info); 723cbf1505dSSean Bruno 724cbf1505dSSean Bruno ri->iri_nfrags = i; 725f2d6ace4SSean Bruno return (0); 726f2d6ace4SSean Bruno } 727f2d6ace4SSean Bruno 728f2d6ace4SSean Bruno /********************************************************************* 729f2d6ace4SSean Bruno * 730f2d6ace4SSean Bruno * Verify that the hardware indicated that the checksum is valid. 731f2d6ace4SSean Bruno * Inform the stack about the status of checksum so that stack 732f2d6ace4SSean Bruno * doesn't spend time verifying the checksum. 733f2d6ace4SSean Bruno * 734f2d6ace4SSean Bruno *********************************************************************/ 735f2d6ace4SSean Bruno static void 736f2d6ace4SSean Bruno lem_receive_checksum(int status, int errors, if_rxd_info_t ri) 737f2d6ace4SSean Bruno { 738f2d6ace4SSean Bruno /* Did it pass? */ 739f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IPCS && !(errors & E1000_RXD_ERR_IPE)) 740f2d6ace4SSean Bruno ri->iri_csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); 741f2d6ace4SSean Bruno 742f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_TCPCS) { 743f2d6ace4SSean Bruno /* Did it pass? */ 744f2d6ace4SSean Bruno if (!(errors & E1000_RXD_ERR_TCPE)) { 745f2d6ace4SSean Bruno ri->iri_csum_flags |= 746f2d6ace4SSean Bruno (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 747f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 748f2d6ace4SSean Bruno } 749f2d6ace4SSean Bruno } 750f2d6ace4SSean Bruno } 751f2d6ace4SSean Bruno 752cbf1505dSSean Bruno /******************************************************************** 753cbf1505dSSean Bruno * 754cbf1505dSSean Bruno * Parse the packet type to determine the appropriate hash 755cbf1505dSSean Bruno * 756cbf1505dSSean Bruno ******************************************************************/ 757cbf1505dSSean Bruno static int 758cbf1505dSSean Bruno em_determine_rsstype(u32 pkt_info) 759cbf1505dSSean Bruno { 760cbf1505dSSean Bruno switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { 761cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV4_TCP: 762cbf1505dSSean Bruno return M_HASHTYPE_RSS_TCP_IPV4; 763cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV4: 764cbf1505dSSean Bruno return M_HASHTYPE_RSS_IPV4; 765cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_TCP: 766cbf1505dSSean Bruno return M_HASHTYPE_RSS_TCP_IPV6; 767cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_EX: 768cbf1505dSSean Bruno return M_HASHTYPE_RSS_IPV6_EX; 769cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6: 770cbf1505dSSean Bruno return M_HASHTYPE_RSS_IPV6; 771cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: 772cbf1505dSSean Bruno return M_HASHTYPE_RSS_TCP_IPV6_EX; 773cbf1505dSSean Bruno default: 774cbf1505dSSean Bruno return M_HASHTYPE_OPAQUE; 775cbf1505dSSean Bruno } 776cbf1505dSSean Bruno } 77795246abbSSean Bruno 778f2d6ace4SSean Bruno static void 779f2d6ace4SSean Bruno em_receive_checksum(uint32_t status, if_rxd_info_t ri) 780f2d6ace4SSean Bruno { 781f2d6ace4SSean Bruno ri->iri_csum_flags = 0; 782f2d6ace4SSean Bruno 783f2d6ace4SSean Bruno /* Ignore Checksum bit is set */ 784f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IXSM) 785f2d6ace4SSean Bruno return; 786f2d6ace4SSean Bruno 787f2d6ace4SSean Bruno /* If the IP checksum exists and there is no IP Checksum error */ 788f2d6ace4SSean Bruno if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == 789f2d6ace4SSean Bruno E1000_RXD_STAT_IPCS) { 790f2d6ace4SSean Bruno ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); 791f2d6ace4SSean Bruno } 792f2d6ace4SSean Bruno 793f2d6ace4SSean Bruno /* TCP or UDP checksum */ 794f2d6ace4SSean Bruno if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 795f2d6ace4SSean Bruno E1000_RXD_STAT_TCPCS) { 796f2d6ace4SSean Bruno ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 797f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 798f2d6ace4SSean Bruno } 799f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_UDPCS) { 800f2d6ace4SSean Bruno ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 801f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 802f2d6ace4SSean Bruno } 803f2d6ace4SSean Bruno } 804