1d37cece2SSean Bruno /*- 27021bf05SStephen Hurd * Copyright (c) 2016 Nicole Graziano <nicole@nextbsd.org> 37021bf05SStephen Hurd * Copyright (c) 2017 Matthew Macy <mmacy@mattmacy.io> 4d37cece2SSean Bruno * All rights reserved. 5d37cece2SSean Bruno * 6d37cece2SSean Bruno * Redistribution and use in source and binary forms, with or without 7d37cece2SSean Bruno * modification, are permitted provided that the following conditions 8d37cece2SSean Bruno * are met: 9d37cece2SSean Bruno * 1. Redistributions of source code must retain the above copyright 10d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer. 11d37cece2SSean Bruno * 2. Redistributions in binary form must reproduce the above copyright 12d37cece2SSean Bruno * notice, this list of conditions and the following disclaimer in the 13d37cece2SSean Bruno * documentation and/or other materials provided with the distribution. 14d37cece2SSean Bruno * 15d37cece2SSean Bruno * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16d37cece2SSean Bruno * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17d37cece2SSean Bruno * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18d37cece2SSean Bruno * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19d37cece2SSean Bruno * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20d37cece2SSean Bruno * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21d37cece2SSean Bruno * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22d37cece2SSean Bruno * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23d37cece2SSean Bruno * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24d37cece2SSean Bruno * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25d37cece2SSean Bruno * SUCH DAMAGE. 26d37cece2SSean Bruno */ 27d37cece2SSean Bruno 28f2d6ace4SSean Bruno /* $FreeBSD$ */ 29f2d6ace4SSean Bruno #include "if_em.h" 30f2d6ace4SSean Bruno 31f2d6ace4SSean Bruno #ifdef RSS 32f2d6ace4SSean Bruno #include <net/rss_config.h> 33f2d6ace4SSean Bruno #include <netinet/in_rss.h> 34f2d6ace4SSean Bruno #endif 35f2d6ace4SSean Bruno 36f2d6ace4SSean Bruno #ifdef VERBOSE_DEBUG 37f2d6ace4SSean Bruno #define DPRINTF device_printf 38f2d6ace4SSean Bruno #else 39f2d6ace4SSean Bruno #define DPRINTF(...) 40f2d6ace4SSean Bruno #endif 41f2d6ace4SSean Bruno 42f2d6ace4SSean Bruno /********************************************************************* 43f2d6ace4SSean Bruno * Local Function prototypes 44f2d6ace4SSean Bruno *********************************************************************/ 4595246abbSSean Bruno static int em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, 4695246abbSSean Bruno u32 *txd_lower); 4795246abbSSean Bruno static int em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, 4895246abbSSean Bruno u32 *txd_upper, u32 *txd_lower); 49f2d6ace4SSean Bruno static int em_isc_txd_encap(void *arg, if_pkt_info_t pi); 5095246abbSSean Bruno static void em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx); 5195246abbSSean Bruno static int em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear); 5295246abbSSean Bruno static void em_isc_rxd_refill(void *arg, if_rxd_update_t iru); 5395246abbSSean Bruno static void em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, 5495246abbSSean Bruno qidx_t pidx); 5595246abbSSean Bruno static int em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 5695246abbSSean Bruno qidx_t budget); 57f2d6ace4SSean Bruno static int em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 58f2d6ace4SSean Bruno 5995246abbSSean Bruno static void lem_isc_rxd_refill(void *arg, if_rxd_update_t iru); 60f2d6ace4SSean Bruno 6195246abbSSean Bruno static int lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, 6295246abbSSean Bruno qidx_t budget); 63f2d6ace4SSean Bruno static int lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri); 64f2d6ace4SSean Bruno 65f2d6ace4SSean Bruno static void lem_receive_checksum(int status, int errors, if_rxd_info_t ri); 66f2d6ace4SSean Bruno static void em_receive_checksum(uint32_t status, if_rxd_info_t ri); 67cbf1505dSSean Bruno static int em_determine_rsstype(u32 pkt_info); 68f2d6ace4SSean Bruno extern int em_intr(void *arg); 69f2d6ace4SSean Bruno 70f2d6ace4SSean Bruno struct if_txrx em_txrx = { 71fbf8b74cSMark Johnston .ift_txd_encap = em_isc_txd_encap, 72fbf8b74cSMark Johnston .ift_txd_flush = em_isc_txd_flush, 73fbf8b74cSMark Johnston .ift_txd_credits_update = em_isc_txd_credits_update, 74fbf8b74cSMark Johnston .ift_rxd_available = em_isc_rxd_available, 75fbf8b74cSMark Johnston .ift_rxd_pkt_get = em_isc_rxd_pkt_get, 76fbf8b74cSMark Johnston .ift_rxd_refill = em_isc_rxd_refill, 77fbf8b74cSMark Johnston .ift_rxd_flush = em_isc_rxd_flush, 78fbf8b74cSMark Johnston .ift_legacy_intr = em_intr 79f2d6ace4SSean Bruno }; 80f2d6ace4SSean Bruno 81f2d6ace4SSean Bruno struct if_txrx lem_txrx = { 82fbf8b74cSMark Johnston .ift_txd_encap = em_isc_txd_encap, 83fbf8b74cSMark Johnston .ift_txd_flush = em_isc_txd_flush, 84fbf8b74cSMark Johnston .ift_txd_credits_update = em_isc_txd_credits_update, 85fbf8b74cSMark Johnston .ift_rxd_available = lem_isc_rxd_available, 86fbf8b74cSMark Johnston .ift_rxd_pkt_get = lem_isc_rxd_pkt_get, 87fbf8b74cSMark Johnston .ift_rxd_refill = lem_isc_rxd_refill, 88fbf8b74cSMark Johnston .ift_rxd_flush = em_isc_rxd_flush, 89fbf8b74cSMark Johnston .ift_legacy_intr = em_intr 90f2d6ace4SSean Bruno }; 91f2d6ace4SSean Bruno 92f2d6ace4SSean Bruno extern if_shared_ctx_t em_sctx; 93f2d6ace4SSean Bruno 9495246abbSSean Bruno void 9595246abbSSean Bruno em_dump_rs(struct adapter *adapter) 9695246abbSSean Bruno { 9795246abbSSean Bruno if_softc_ctx_t scctx = adapter->shared; 9895246abbSSean Bruno struct em_tx_queue *que; 9995246abbSSean Bruno struct tx_ring *txr; 10095246abbSSean Bruno qidx_t i, ntxd, qid, cur; 10195246abbSSean Bruno int16_t rs_cidx; 10295246abbSSean Bruno uint8_t status; 10395246abbSSean Bruno 10495246abbSSean Bruno printf("\n"); 10595246abbSSean Bruno ntxd = scctx->isc_ntxd[0]; 10695246abbSSean Bruno for (qid = 0; qid < adapter->tx_num_queues; qid++) { 10795246abbSSean Bruno que = &adapter->tx_queues[qid]; 10895246abbSSean Bruno txr = &que->txr; 10995246abbSSean Bruno rs_cidx = txr->tx_rs_cidx; 11095246abbSSean Bruno if (rs_cidx != txr->tx_rs_pidx) { 11195246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 11295246abbSSean Bruno status = txr->tx_base[cur].upper.fields.status; 11395246abbSSean Bruno if (!(status & E1000_TXD_STAT_DD)) 11495246abbSSean Bruno printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur); 11595246abbSSean Bruno } else { 11695246abbSSean Bruno rs_cidx = (rs_cidx-1)&(ntxd-1); 11795246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 11895246abbSSean Bruno printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur); 11995246abbSSean Bruno } 12095246abbSSean Bruno printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, txr->tx_rs_pidx); 12195246abbSSean Bruno for (i = 0; i < ntxd; i++) { 12295246abbSSean Bruno if (txr->tx_base[i].upper.fields.status & E1000_TXD_STAT_DD) 12395246abbSSean Bruno printf("%d set ", i); 12495246abbSSean Bruno } 12595246abbSSean Bruno printf("\n"); 12695246abbSSean Bruno } 12795246abbSSean Bruno } 12895246abbSSean Bruno 129f2d6ace4SSean Bruno /********************************************************************** 130f2d6ace4SSean Bruno * 131f2d6ace4SSean Bruno * Setup work for hardware segmentation offload (TSO) on 132f2d6ace4SSean Bruno * adapters using advanced tx descriptors 133f2d6ace4SSean Bruno * 134f2d6ace4SSean Bruno **********************************************************************/ 135f2d6ace4SSean Bruno static int 136f2d6ace4SSean Bruno em_tso_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) 137f2d6ace4SSean Bruno { 138f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 139f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; 140f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 141f2d6ace4SSean Bruno struct e1000_context_desc *TXD; 142f2d6ace4SSean Bruno int cur, hdr_len; 143f2d6ace4SSean Bruno 144f2d6ace4SSean Bruno hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen + pi->ipi_tcp_hlen; 145f2d6ace4SSean Bruno *txd_lower = (E1000_TXD_CMD_DEXT | /* Extended descr type */ 146f2d6ace4SSean Bruno E1000_TXD_DTYP_D | /* Data descr type */ 147f2d6ace4SSean Bruno E1000_TXD_CMD_TSE); /* Do TSE on this packet */ 148f2d6ace4SSean Bruno 149f2d6ace4SSean Bruno /* IP and/or TCP header checksum calculation and insertion. */ 150f2d6ace4SSean Bruno *txd_upper = (E1000_TXD_POPTS_IXSM | E1000_TXD_POPTS_TXSM) << 8; 151f2d6ace4SSean Bruno 152f2d6ace4SSean Bruno cur = pi->ipi_pidx; 153f2d6ace4SSean Bruno TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 154f2d6ace4SSean Bruno 155f2d6ace4SSean Bruno /* 156f2d6ace4SSean Bruno * Start offset for header checksum calculation. 157f2d6ace4SSean Bruno * End offset for header checksum calculation. 158f2d6ace4SSean Bruno * Offset of place put the checksum. 159f2d6ace4SSean Bruno */ 160f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 161f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcse = 162f2d6ace4SSean Bruno htole16(pi->ipi_ehdrlen + pi->ipi_ip_hlen - 1); 163f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 164f2d6ace4SSean Bruno 165f2d6ace4SSean Bruno /* 166f2d6ace4SSean Bruno * Start offset for payload checksum calculation. 167f2d6ace4SSean Bruno * End offset for payload checksum calculation. 168f2d6ace4SSean Bruno * Offset of place to put the checksum. 169f2d6ace4SSean Bruno */ 170f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucss = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 171f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucse = 0; 172f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucso = 173f2d6ace4SSean Bruno pi->ipi_ehdrlen + pi->ipi_ip_hlen + offsetof(struct tcphdr, th_sum); 174f2d6ace4SSean Bruno 175f2d6ace4SSean Bruno /* 176f2d6ace4SSean Bruno * Payload size per packet w/o any headers. 177f2d6ace4SSean Bruno * Length of all headers up to payload. 178f2d6ace4SSean Bruno */ 179f2d6ace4SSean Bruno TXD->tcp_seg_setup.fields.mss = htole16(pi->ipi_tso_segsz); 180f2d6ace4SSean Bruno TXD->tcp_seg_setup.fields.hdr_len = hdr_len; 181f2d6ace4SSean Bruno 182f2d6ace4SSean Bruno TXD->cmd_and_length = htole32(adapter->txd_cmd | 183f2d6ace4SSean Bruno E1000_TXD_CMD_DEXT | /* Extended descr */ 184f2d6ace4SSean Bruno E1000_TXD_CMD_TSE | /* TSE context */ 185f2d6ace4SSean Bruno E1000_TXD_CMD_IP | /* Do IP csum */ 186f2d6ace4SSean Bruno E1000_TXD_CMD_TCP | /* Do TCP checksum */ 187f2d6ace4SSean Bruno (pi->ipi_len - hdr_len)); /* Total len */ 188f2d6ace4SSean Bruno txr->tx_tso = TRUE; 189f2d6ace4SSean Bruno 190f2d6ace4SSean Bruno if (++cur == scctx->isc_ntxd[0]) { 191f2d6ace4SSean Bruno cur = 0; 192f2d6ace4SSean Bruno } 193f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "%s: pidx: %d cur: %d\n", __FUNCTION__, pi->ipi_pidx, cur); 194f2d6ace4SSean Bruno return (cur); 195f2d6ace4SSean Bruno } 196f2d6ace4SSean Bruno 197f2d6ace4SSean Bruno #define TSO_WORKAROUND 4 198f2d6ace4SSean Bruno #define DONT_FORCE_CTX 1 199f2d6ace4SSean Bruno 200f2d6ace4SSean Bruno 201f2d6ace4SSean Bruno /********************************************************************* 202f2d6ace4SSean Bruno * The offload context is protocol specific (TCP/UDP) and thus 203f2d6ace4SSean Bruno * only needs to be set when the protocol changes. The occasion 204f2d6ace4SSean Bruno * of a context change can be a performance detriment, and 205f2d6ace4SSean Bruno * might be better just disabled. The reason arises in the way 206f2d6ace4SSean Bruno * in which the controller supports pipelined requests from the 207f2d6ace4SSean Bruno * Tx data DMA. Up to four requests can be pipelined, and they may 208f2d6ace4SSean Bruno * belong to the same packet or to multiple packets. However all 209f2d6ace4SSean Bruno * requests for one packet are issued before a request is issued 210f2d6ace4SSean Bruno * for a subsequent packet and if a request for the next packet 211f2d6ace4SSean Bruno * requires a context change, that request will be stalled 212f2d6ace4SSean Bruno * until the previous request completes. This means setting up 213f2d6ace4SSean Bruno * a new context effectively disables pipelined Tx data DMA which 214f2d6ace4SSean Bruno * in turn greatly slow down performance to send small sized 215f2d6ace4SSean Bruno * frames. 216f2d6ace4SSean Bruno **********************************************************************/ 217f2d6ace4SSean Bruno 218f2d6ace4SSean Bruno static int 219f2d6ace4SSean Bruno em_transmit_checksum_setup(struct adapter *adapter, if_pkt_info_t pi, u32 *txd_upper, u32 *txd_lower) 220f2d6ace4SSean Bruno { 221f2d6ace4SSean Bruno struct e1000_context_desc *TXD = NULL; 222f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 223f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[pi->ipi_qsidx]; 224f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 225f2d6ace4SSean Bruno int csum_flags = pi->ipi_csum_flags; 226f2d6ace4SSean Bruno int cur, hdr_len; 227f2d6ace4SSean Bruno u32 cmd; 228f2d6ace4SSean Bruno 229f2d6ace4SSean Bruno cur = pi->ipi_pidx; 230f2d6ace4SSean Bruno hdr_len = pi->ipi_ehdrlen + pi->ipi_ip_hlen; 231f2d6ace4SSean Bruno cmd = adapter->txd_cmd; 232f2d6ace4SSean Bruno 233f2d6ace4SSean Bruno /* 234f2d6ace4SSean Bruno * The 82574L can only remember the *last* context used 235f2d6ace4SSean Bruno * regardless of queue that it was use for. We cannot reuse 236f2d6ace4SSean Bruno * contexts on this hardware platform and must generate a new 237f2d6ace4SSean Bruno * context every time. 82574L hardware spec, section 7.2.6, 238f2d6ace4SSean Bruno * second note. 239f2d6ace4SSean Bruno */ 240f2d6ace4SSean Bruno if (DONT_FORCE_CTX && 241f2d6ace4SSean Bruno adapter->tx_num_queues == 1 && 242f2d6ace4SSean Bruno txr->csum_lhlen == pi->ipi_ehdrlen && 243f2d6ace4SSean Bruno txr->csum_iphlen == pi->ipi_ip_hlen && 244f2d6ace4SSean Bruno txr->csum_flags == csum_flags) { 245f2d6ace4SSean Bruno /* 246f2d6ace4SSean Bruno * Same csum offload context as the previous packets; 247f2d6ace4SSean Bruno * just return. 248f2d6ace4SSean Bruno */ 249f2d6ace4SSean Bruno *txd_upper = txr->csum_txd_upper; 250f2d6ace4SSean Bruno *txd_lower = txr->csum_txd_lower; 251f2d6ace4SSean Bruno return (cur); 252f2d6ace4SSean Bruno } 253f2d6ace4SSean Bruno 254f2d6ace4SSean Bruno TXD = (struct e1000_context_desc *)&txr->tx_base[cur]; 255f2d6ace4SSean Bruno if (csum_flags & CSUM_IP) { 256f2d6ace4SSean Bruno *txd_upper |= E1000_TXD_POPTS_IXSM << 8; 257f2d6ace4SSean Bruno /* 258f2d6ace4SSean Bruno * Start offset for header checksum calculation. 259f2d6ace4SSean Bruno * End offset for header checksum calculation. 260f2d6ace4SSean Bruno * Offset of place to put the checksum. 261f2d6ace4SSean Bruno */ 262f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcss = pi->ipi_ehdrlen; 263f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcse = htole16(hdr_len); 264f2d6ace4SSean Bruno TXD->lower_setup.ip_fields.ipcso = pi->ipi_ehdrlen + offsetof(struct ip, ip_sum); 265f2d6ace4SSean Bruno cmd |= E1000_TXD_CMD_IP; 266f2d6ace4SSean Bruno } 267f2d6ace4SSean Bruno 268f2d6ace4SSean Bruno if (csum_flags & (CSUM_TCP|CSUM_UDP)) { 269f2d6ace4SSean Bruno uint8_t tucso; 270f2d6ace4SSean Bruno 271f2d6ace4SSean Bruno *txd_upper |= E1000_TXD_POPTS_TXSM << 8; 272f2d6ace4SSean Bruno *txd_lower = E1000_TXD_CMD_DEXT | E1000_TXD_DTYP_D; 273f2d6ace4SSean Bruno 274f2d6ace4SSean Bruno if (csum_flags & CSUM_TCP) { 275f2d6ace4SSean Bruno tucso = hdr_len + offsetof(struct tcphdr, th_sum); 276f2d6ace4SSean Bruno cmd |= E1000_TXD_CMD_TCP; 277f2d6ace4SSean Bruno } else 278f2d6ace4SSean Bruno tucso = hdr_len + offsetof(struct udphdr, uh_sum); 279f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucss = hdr_len; 280f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucse = htole16(0); 281f2d6ace4SSean Bruno TXD->upper_setup.tcp_fields.tucso = tucso; 282f2d6ace4SSean Bruno } 283f2d6ace4SSean Bruno 284f2d6ace4SSean Bruno txr->csum_lhlen = pi->ipi_ehdrlen; 285f2d6ace4SSean Bruno txr->csum_iphlen = pi->ipi_ip_hlen; 286f2d6ace4SSean Bruno txr->csum_flags = csum_flags; 287f2d6ace4SSean Bruno txr->csum_txd_upper = *txd_upper; 288f2d6ace4SSean Bruno txr->csum_txd_lower = *txd_lower; 289f2d6ace4SSean Bruno 290f2d6ace4SSean Bruno TXD->tcp_seg_setup.data = htole32(0); 291f2d6ace4SSean Bruno TXD->cmd_and_length = 292f2d6ace4SSean Bruno htole32(E1000_TXD_CMD_IFCS | E1000_TXD_CMD_DEXT | cmd); 293f2d6ace4SSean Bruno 294f2d6ace4SSean Bruno if (++cur == scctx->isc_ntxd[0]) { 295f2d6ace4SSean Bruno cur = 0; 296f2d6ace4SSean Bruno } 297f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), "checksum_setup csum_flags=%x txd_upper=%x txd_lower=%x hdr_len=%d cmd=%x\n", 298f2d6ace4SSean Bruno csum_flags, *txd_upper, *txd_lower, hdr_len, cmd); 299f2d6ace4SSean Bruno return (cur); 300f2d6ace4SSean Bruno } 301f2d6ace4SSean Bruno 302f2d6ace4SSean Bruno static int 303f2d6ace4SSean Bruno em_isc_txd_encap(void *arg, if_pkt_info_t pi) 304f2d6ace4SSean Bruno { 305f2d6ace4SSean Bruno struct adapter *sc = arg; 306f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 307f2d6ace4SSean Bruno struct em_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx]; 308f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 309f2d6ace4SSean Bruno bus_dma_segment_t *segs = pi->ipi_segs; 310f2d6ace4SSean Bruno int nsegs = pi->ipi_nsegs; 311f2d6ace4SSean Bruno int csum_flags = pi->ipi_csum_flags; 312f2d6ace4SSean Bruno int i, j, first, pidx_last; 31395246abbSSean Bruno u32 txd_flags, txd_upper = 0, txd_lower = 0; 314f2d6ace4SSean Bruno 315f2d6ace4SSean Bruno struct e1000_tx_desc *ctxd = NULL; 316f2d6ace4SSean Bruno bool do_tso, tso_desc; 31795246abbSSean Bruno qidx_t ntxd; 318f2d6ace4SSean Bruno 31995246abbSSean Bruno txd_flags = pi->ipi_flags & IPI_TX_INTR ? E1000_TXD_CMD_RS : 0; 320f2d6ace4SSean Bruno i = first = pi->ipi_pidx; 321f2d6ace4SSean Bruno do_tso = (csum_flags & CSUM_TSO); 322f2d6ace4SSean Bruno tso_desc = FALSE; 32395246abbSSean Bruno ntxd = scctx->isc_ntxd[0]; 324f2d6ace4SSean Bruno /* 325f2d6ace4SSean Bruno * TSO Hardware workaround, if this packet is not 326f2d6ace4SSean Bruno * TSO, and is only a single descriptor long, and 327f2d6ace4SSean Bruno * it follows a TSO burst, then we need to add a 328f2d6ace4SSean Bruno * sentinel descriptor to prevent premature writeback. 329f2d6ace4SSean Bruno */ 330f2d6ace4SSean Bruno if ((!do_tso) && (txr->tx_tso == TRUE)) { 331f2d6ace4SSean Bruno if (nsegs == 1) 332f2d6ace4SSean Bruno tso_desc = TRUE; 333f2d6ace4SSean Bruno txr->tx_tso = FALSE; 334f2d6ace4SSean Bruno } 335f2d6ace4SSean Bruno 336f2d6ace4SSean Bruno /* Do hardware assists */ 337f2d6ace4SSean Bruno if (do_tso) { 338f2d6ace4SSean Bruno i = em_tso_setup(sc, pi, &txd_upper, &txd_lower); 339f2d6ace4SSean Bruno tso_desc = TRUE; 34082379056SSean Bruno } else if (csum_flags & EM_CSUM_OFFLOAD) { 341f2d6ace4SSean Bruno i = em_transmit_checksum_setup(sc, pi, &txd_upper, &txd_lower); 342f2d6ace4SSean Bruno } 343f2d6ace4SSean Bruno 344f2d6ace4SSean Bruno if (pi->ipi_mflags & M_VLANTAG) { 345f2d6ace4SSean Bruno /* Set the vlan id. */ 346f2d6ace4SSean Bruno txd_upper |= htole16(pi->ipi_vtag) << 16; 347f2d6ace4SSean Bruno /* Tell hardware to add tag */ 348f2d6ace4SSean Bruno txd_lower |= htole32(E1000_TXD_CMD_VLE); 349f2d6ace4SSean Bruno } 350f2d6ace4SSean Bruno 351f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "encap: set up tx: nsegs=%d first=%d i=%d\n", nsegs, first, i); 352f2d6ace4SSean Bruno /* XXX adapter->pcix_82544 -- lem_fill_descriptors */ 353f2d6ace4SSean Bruno 354f2d6ace4SSean Bruno /* Set up our transmit descriptors */ 355f2d6ace4SSean Bruno for (j = 0; j < nsegs; j++) { 356f2d6ace4SSean Bruno bus_size_t seg_len; 357f2d6ace4SSean Bruno bus_addr_t seg_addr; 358f2d6ace4SSean Bruno uint32_t cmd; 359f2d6ace4SSean Bruno 360f2d6ace4SSean Bruno ctxd = &txr->tx_base[i]; 361f2d6ace4SSean Bruno seg_addr = segs[j].ds_addr; 362f2d6ace4SSean Bruno seg_len = segs[j].ds_len; 363f2d6ace4SSean Bruno cmd = E1000_TXD_CMD_IFCS | sc->txd_cmd; 364f2d6ace4SSean Bruno 365f2d6ace4SSean Bruno /* 36695246abbSSean Bruno * TSO Workaround: 36795246abbSSean Bruno * If this is the last descriptor, we want to 36895246abbSSean Bruno * split it so we have a small final sentinel 369f2d6ace4SSean Bruno */ 370f2d6ace4SSean Bruno if (tso_desc && (j == (nsegs - 1)) && (seg_len > 8)) { 371f2d6ace4SSean Bruno seg_len -= TSO_WORKAROUND; 372f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr); 373f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 374f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 375f2d6ace4SSean Bruno 376f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 377f2d6ace4SSean Bruno i = 0; 378f2d6ace4SSean Bruno 379f2d6ace4SSean Bruno /* Now make the sentinel */ 380f2d6ace4SSean Bruno ctxd = &txr->tx_base[i]; 381f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr + seg_len); 382f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | TSO_WORKAROUND); 383f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 384f2d6ace4SSean Bruno pidx_last = i; 385f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 386f2d6ace4SSean Bruno i = 0; 387f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "TSO path pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); 388f2d6ace4SSean Bruno } else { 389f2d6ace4SSean Bruno ctxd->buffer_addr = htole64(seg_addr); 390f2d6ace4SSean Bruno ctxd->lower.data = htole32(cmd | txd_lower | seg_len); 391f2d6ace4SSean Bruno ctxd->upper.data = htole32(txd_upper); 392f2d6ace4SSean Bruno pidx_last = i; 393f2d6ace4SSean Bruno if (++i == scctx->isc_ntxd[0]) 394f2d6ace4SSean Bruno i = 0; 395f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "pidx_last=%d i=%d ntxd[0]=%d\n", pidx_last, i, scctx->isc_ntxd[0]); 396f2d6ace4SSean Bruno } 397f2d6ace4SSean Bruno } 398f2d6ace4SSean Bruno 399f2d6ace4SSean Bruno /* 400f2d6ace4SSean Bruno * Last Descriptor of Packet 401f2d6ace4SSean Bruno * needs End Of Packet (EOP) 402f2d6ace4SSean Bruno * and Report Status (RS) 403f2d6ace4SSean Bruno */ 4048fd222ebSMatt Macy if (txd_flags && nsegs) { 40595246abbSSean Bruno txr->tx_rsq[txr->tx_rs_pidx] = pidx_last; 40695246abbSSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "setting to RS on %d rs_pidx %d first: %d\n", pidx_last, txr->tx_rs_pidx, first); 40795246abbSSean Bruno txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1); 40895246abbSSean Bruno MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx); 40995246abbSSean Bruno } 41095246abbSSean Bruno ctxd->lower.data |= htole32(E1000_TXD_CMD_EOP | txd_flags); 411f2d6ace4SSean Bruno DPRINTF(iflib_get_dev(sc->ctx), "tx_buffers[%d]->eop = %d ipi_new_pidx=%d\n", first, pidx_last, i); 412f2d6ace4SSean Bruno pi->ipi_new_pidx = i; 413f2d6ace4SSean Bruno 414f2d6ace4SSean Bruno return (0); 415f2d6ace4SSean Bruno } 416f2d6ace4SSean Bruno 417f2d6ace4SSean Bruno static void 41895246abbSSean Bruno em_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx) 419f2d6ace4SSean Bruno { 420f2d6ace4SSean Bruno struct adapter *adapter = arg; 421f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 422f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 423f2d6ace4SSean Bruno 424f2d6ace4SSean Bruno E1000_WRITE_REG(&adapter->hw, E1000_TDT(txr->me), pidx); 425f2d6ace4SSean Bruno } 426f2d6ace4SSean Bruno 427f2d6ace4SSean Bruno static int 42895246abbSSean Bruno em_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear) 429f2d6ace4SSean Bruno { 430f2d6ace4SSean Bruno struct adapter *adapter = arg; 431f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 432f2d6ace4SSean Bruno struct em_tx_queue *que = &adapter->tx_queues[txqid]; 433f2d6ace4SSean Bruno struct tx_ring *txr = &que->txr; 434f2d6ace4SSean Bruno 43595246abbSSean Bruno qidx_t processed = 0; 43695246abbSSean Bruno int updated; 43795246abbSSean Bruno qidx_t cur, prev, ntxd, rs_cidx; 43895246abbSSean Bruno int32_t delta; 43995246abbSSean Bruno uint8_t status; 440f2d6ace4SSean Bruno 44195246abbSSean Bruno rs_cidx = txr->tx_rs_cidx; 44295246abbSSean Bruno if (rs_cidx == txr->tx_rs_pidx) 44395246abbSSean Bruno return (0); 44495246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 44595246abbSSean Bruno MPASS(cur != QIDX_INVALID); 44695246abbSSean Bruno status = txr->tx_base[cur].upper.fields.status; 44795246abbSSean Bruno updated = !!(status & E1000_TXD_STAT_DD); 448f2d6ace4SSean Bruno 44995246abbSSean Bruno if (clear == false || updated == 0) 45095246abbSSean Bruno return (updated); 45195246abbSSean Bruno 45295246abbSSean Bruno prev = txr->tx_cidx_processed; 45395246abbSSean Bruno ntxd = scctx->isc_ntxd[0]; 45495246abbSSean Bruno do { 45595246abbSSean Bruno delta = (int32_t)cur - (int32_t)prev; 45695246abbSSean Bruno MPASS(prev == 0 || delta != 0); 45795246abbSSean Bruno if (delta < 0) 45895246abbSSean Bruno delta += ntxd; 459bd84f700SSean Bruno DPRINTF(iflib_get_dev(adapter->ctx), 46095246abbSSean Bruno "%s: cidx_processed=%u cur=%u clear=%d delta=%d\n", 46195246abbSSean Bruno __FUNCTION__, prev, cur, clear, delta); 462f2d6ace4SSean Bruno 46395246abbSSean Bruno processed += delta; 46495246abbSSean Bruno prev = cur; 46595246abbSSean Bruno rs_cidx = (rs_cidx + 1) & (ntxd-1); 46695246abbSSean Bruno if (rs_cidx == txr->tx_rs_pidx) 467f2d6ace4SSean Bruno break; 46895246abbSSean Bruno cur = txr->tx_rsq[rs_cidx]; 46995246abbSSean Bruno MPASS(cur != QIDX_INVALID); 47095246abbSSean Bruno status = txr->tx_base[cur].upper.fields.status; 47195246abbSSean Bruno } while ((status & E1000_TXD_STAT_DD)); 472f2d6ace4SSean Bruno 47395246abbSSean Bruno txr->tx_rs_cidx = rs_cidx; 47495246abbSSean Bruno txr->tx_cidx_processed = prev; 475f2d6ace4SSean Bruno return(processed); 476f2d6ace4SSean Bruno } 477f2d6ace4SSean Bruno 478f2d6ace4SSean Bruno static void 47995246abbSSean Bruno lem_isc_rxd_refill(void *arg, if_rxd_update_t iru) 480f2d6ace4SSean Bruno { 481f2d6ace4SSean Bruno struct adapter *sc = arg; 482f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 48395246abbSSean Bruno struct em_rx_queue *que = &sc->rx_queues[iru->iru_qsidx]; 484f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 485f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 48695246abbSSean Bruno uint64_t *paddrs; 48795246abbSSean Bruno uint32_t next_pidx, pidx; 48895246abbSSean Bruno uint16_t count; 489f2d6ace4SSean Bruno int i; 49095246abbSSean Bruno 49195246abbSSean Bruno paddrs = iru->iru_paddrs; 49295246abbSSean Bruno pidx = iru->iru_pidx; 49395246abbSSean Bruno count = iru->iru_count; 494f2d6ace4SSean Bruno 495f2d6ace4SSean Bruno for (i = 0, next_pidx = pidx; i < count; i++) { 496f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[next_pidx]; 497f2d6ace4SSean Bruno rxd->buffer_addr = htole64(paddrs[i]); 498f2d6ace4SSean Bruno /* status bits must be cleared */ 499f2d6ace4SSean Bruno rxd->status = 0; 500f2d6ace4SSean Bruno 501f2d6ace4SSean Bruno if (++next_pidx == scctx->isc_nrxd[0]) 502f2d6ace4SSean Bruno next_pidx = 0; 503f2d6ace4SSean Bruno } 504f2d6ace4SSean Bruno } 505f2d6ace4SSean Bruno 506f2d6ace4SSean Bruno static void 50795246abbSSean Bruno em_isc_rxd_refill(void *arg, if_rxd_update_t iru) 508f2d6ace4SSean Bruno { 509f2d6ace4SSean Bruno struct adapter *sc = arg; 510f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 51195246abbSSean Bruno uint16_t rxqid = iru->iru_qsidx; 512f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 513f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 514f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 51595246abbSSean Bruno uint64_t *paddrs; 51695246abbSSean Bruno uint32_t next_pidx, pidx; 51795246abbSSean Bruno uint16_t count; 518f2d6ace4SSean Bruno int i; 51995246abbSSean Bruno 52095246abbSSean Bruno paddrs = iru->iru_paddrs; 52195246abbSSean Bruno pidx = iru->iru_pidx; 52295246abbSSean Bruno count = iru->iru_count; 523f2d6ace4SSean Bruno 524f2d6ace4SSean Bruno for (i = 0, next_pidx = pidx; i < count; i++) { 525f2d6ace4SSean Bruno rxd = &rxr->rx_base[next_pidx]; 526f2d6ace4SSean Bruno rxd->read.buffer_addr = htole64(paddrs[i]); 527ab2e3f79SStephen Hurd /* DD bits must be cleared */ 528ab2e3f79SStephen Hurd rxd->wb.upper.status_error = 0; 529f2d6ace4SSean Bruno 530f2d6ace4SSean Bruno if (++next_pidx == scctx->isc_nrxd[0]) 531f2d6ace4SSean Bruno next_pidx = 0; 532f2d6ace4SSean Bruno } 533f2d6ace4SSean Bruno } 534f2d6ace4SSean Bruno 535f2d6ace4SSean Bruno static void 53695246abbSSean Bruno em_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx) 537f2d6ace4SSean Bruno { 538f2d6ace4SSean Bruno struct adapter *sc = arg; 539f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 540f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 541f2d6ace4SSean Bruno 542f2d6ace4SSean Bruno E1000_WRITE_REG(&sc->hw, E1000_RDT(rxr->me), pidx); 543f2d6ace4SSean Bruno } 544f2d6ace4SSean Bruno 545f2d6ace4SSean Bruno static int 54695246abbSSean Bruno lem_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 547f2d6ace4SSean Bruno { 548f2d6ace4SSean Bruno struct adapter *sc = arg; 549f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 550f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 551f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 552f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 553f2d6ace4SSean Bruno u32 staterr = 0; 554f2d6ace4SSean Bruno int cnt, i; 555f2d6ace4SSean Bruno 556ab2e3f79SStephen Hurd if (budget == 1) { 557ab2e3f79SStephen Hurd rxd = (struct e1000_rx_desc *)&rxr->rx_base[idx]; 558ab2e3f79SStephen Hurd staterr = rxd->status; 559ab2e3f79SStephen Hurd return (staterr & E1000_RXD_STAT_DD); 560ab2e3f79SStephen Hurd } 561ab2e3f79SStephen Hurd 562ab2e3f79SStephen Hurd for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 563f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[i]; 564f2d6ace4SSean Bruno staterr = rxd->status; 565f2d6ace4SSean Bruno 566f2d6ace4SSean Bruno if ((staterr & E1000_RXD_STAT_DD) == 0) 567f2d6ace4SSean Bruno break; 568f2d6ace4SSean Bruno 569f2d6ace4SSean Bruno if (++i == scctx->isc_nrxd[0]) 570f2d6ace4SSean Bruno i = 0; 571f2d6ace4SSean Bruno 572f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_EOP) 573f2d6ace4SSean Bruno cnt++; 574f2d6ace4SSean Bruno } 575f2d6ace4SSean Bruno return (cnt); 576f2d6ace4SSean Bruno } 577f2d6ace4SSean Bruno 578f2d6ace4SSean Bruno static int 57995246abbSSean Bruno em_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget) 580f2d6ace4SSean Bruno { 581f2d6ace4SSean Bruno struct adapter *sc = arg; 582f2d6ace4SSean Bruno if_softc_ctx_t scctx = sc->shared; 583f2d6ace4SSean Bruno struct em_rx_queue *que = &sc->rx_queues[rxqid]; 584f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 585f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 586f2d6ace4SSean Bruno u32 staterr = 0; 587f2d6ace4SSean Bruno int cnt, i; 588f2d6ace4SSean Bruno 589ab2e3f79SStephen Hurd if (budget == 1) { 590ab2e3f79SStephen Hurd rxd = &rxr->rx_base[idx]; 591ab2e3f79SStephen Hurd staterr = le32toh(rxd->wb.upper.status_error); 592ab2e3f79SStephen Hurd return (staterr & E1000_RXD_STAT_DD); 593ab2e3f79SStephen Hurd } 594ab2e3f79SStephen Hurd 595ab2e3f79SStephen Hurd for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) { 596f2d6ace4SSean Bruno rxd = &rxr->rx_base[i]; 597f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 598f2d6ace4SSean Bruno 599f2d6ace4SSean Bruno if ((staterr & E1000_RXD_STAT_DD) == 0) 600f2d6ace4SSean Bruno break; 601f2d6ace4SSean Bruno 602f2d6ace4SSean Bruno if (++i == scctx->isc_nrxd[0]) { 603f2d6ace4SSean Bruno i = 0; 604f2d6ace4SSean Bruno } 605f2d6ace4SSean Bruno 606f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_EOP) 607f2d6ace4SSean Bruno cnt++; 608f2d6ace4SSean Bruno 609f2d6ace4SSean Bruno } 610f2d6ace4SSean Bruno return (cnt); 611f2d6ace4SSean Bruno } 612f2d6ace4SSean Bruno 613f2d6ace4SSean Bruno static int 614f2d6ace4SSean Bruno lem_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 615f2d6ace4SSean Bruno { 616f2d6ace4SSean Bruno struct adapter *adapter = arg; 617f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 618f2d6ace4SSean Bruno struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; 619f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 620f2d6ace4SSean Bruno struct e1000_rx_desc *rxd; 621f2d6ace4SSean Bruno u16 len; 622f2d6ace4SSean Bruno u32 status, errors; 623f2d6ace4SSean Bruno bool eop; 624f2d6ace4SSean Bruno int i, cidx; 625f2d6ace4SSean Bruno 626f2d6ace4SSean Bruno status = errors = i = 0; 627f2d6ace4SSean Bruno cidx = ri->iri_cidx; 628f2d6ace4SSean Bruno 629f2d6ace4SSean Bruno do { 630f2d6ace4SSean Bruno rxd = (struct e1000_rx_desc *)&rxr->rx_base[cidx]; 631f2d6ace4SSean Bruno status = rxd->status; 632f2d6ace4SSean Bruno errors = rxd->errors; 633f2d6ace4SSean Bruno 634f2d6ace4SSean Bruno /* Error Checking then decrement count */ 635f2d6ace4SSean Bruno MPASS ((status & E1000_RXD_STAT_DD) != 0); 636f2d6ace4SSean Bruno 637f2d6ace4SSean Bruno len = le16toh(rxd->length); 638f2d6ace4SSean Bruno ri->iri_len += len; 639f2d6ace4SSean Bruno 640f2d6ace4SSean Bruno eop = (status & E1000_RXD_STAT_EOP) != 0; 641f2d6ace4SSean Bruno 642f2d6ace4SSean Bruno /* Make sure bad packets are discarded */ 643f2d6ace4SSean Bruno if (errors & E1000_RXD_ERR_FRAME_ERR_MASK) { 644f2d6ace4SSean Bruno adapter->dropped_pkts++; 645f2d6ace4SSean Bruno /* XXX fixup if common */ 646f2d6ace4SSean Bruno return (EBADMSG); 647f2d6ace4SSean Bruno } 648f2d6ace4SSean Bruno 649f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 0; 650f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 651f2d6ace4SSean Bruno ri->iri_frags[i].irf_len = len; 652f2d6ace4SSean Bruno /* Zero out the receive descriptors status. */ 653f2d6ace4SSean Bruno rxd->status = 0; 654f2d6ace4SSean Bruno 655f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 656f2d6ace4SSean Bruno cidx = 0; 657f2d6ace4SSean Bruno i++; 658f2d6ace4SSean Bruno } while (!eop); 659f2d6ace4SSean Bruno 660f2d6ace4SSean Bruno /* XXX add a faster way to look this up */ 661f2d6ace4SSean Bruno if (adapter->hw.mac.type >= e1000_82543 && !(status & E1000_RXD_STAT_IXSM)) 662f2d6ace4SSean Bruno lem_receive_checksum(status, errors, ri); 663f2d6ace4SSean Bruno 664f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_VP) { 665f2d6ace4SSean Bruno ri->iri_vtag = le16toh(rxd->special); 666f2d6ace4SSean Bruno ri->iri_flags |= M_VLANTAG; 667f2d6ace4SSean Bruno } 668f2d6ace4SSean Bruno 669f2d6ace4SSean Bruno ri->iri_nfrags = i; 670f2d6ace4SSean Bruno 671f2d6ace4SSean Bruno return (0); 672f2d6ace4SSean Bruno } 673f2d6ace4SSean Bruno 674f2d6ace4SSean Bruno static int 675f2d6ace4SSean Bruno em_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri) 676f2d6ace4SSean Bruno { 677f2d6ace4SSean Bruno struct adapter *adapter = arg; 678f2d6ace4SSean Bruno if_softc_ctx_t scctx = adapter->shared; 679f2d6ace4SSean Bruno struct em_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx]; 680f2d6ace4SSean Bruno struct rx_ring *rxr = &que->rxr; 681f2d6ace4SSean Bruno union e1000_rx_desc_extended *rxd; 682f2d6ace4SSean Bruno 683f2d6ace4SSean Bruno u16 len; 684cbf1505dSSean Bruno u32 pkt_info; 685f2d6ace4SSean Bruno u32 staterr = 0; 686f2d6ace4SSean Bruno bool eop; 687f2d6ace4SSean Bruno int i, cidx, vtag; 688f2d6ace4SSean Bruno 689f2d6ace4SSean Bruno i = vtag = 0; 690f2d6ace4SSean Bruno cidx = ri->iri_cidx; 691f2d6ace4SSean Bruno 692f2d6ace4SSean Bruno do { 693f2d6ace4SSean Bruno rxd = &rxr->rx_base[cidx]; 694f2d6ace4SSean Bruno staterr = le32toh(rxd->wb.upper.status_error); 695cbf1505dSSean Bruno pkt_info = le32toh(rxd->wb.lower.mrq); 696f2d6ace4SSean Bruno 697f2d6ace4SSean Bruno /* Error Checking then decrement count */ 698ab2e3f79SStephen Hurd MPASS ((staterr & E1000_RXD_STAT_DD) != 0); 699f2d6ace4SSean Bruno 700f2d6ace4SSean Bruno len = le16toh(rxd->wb.upper.length); 701f2d6ace4SSean Bruno ri->iri_len += len; 702f2d6ace4SSean Bruno 703f2d6ace4SSean Bruno eop = (staterr & E1000_RXD_STAT_EOP) != 0; 704f2d6ace4SSean Bruno 705f2d6ace4SSean Bruno /* Make sure bad packets are discarded */ 706f2d6ace4SSean Bruno if (staterr & E1000_RXDEXT_ERR_FRAME_ERR_MASK) { 707f2d6ace4SSean Bruno adapter->dropped_pkts++; 708f2d6ace4SSean Bruno return EBADMSG; 709f2d6ace4SSean Bruno } 710f2d6ace4SSean Bruno 711f2d6ace4SSean Bruno ri->iri_frags[i].irf_flid = 0; 712f2d6ace4SSean Bruno ri->iri_frags[i].irf_idx = cidx; 713f2d6ace4SSean Bruno ri->iri_frags[i].irf_len = len; 714f2d6ace4SSean Bruno /* Zero out the receive descriptors status. */ 715f2d6ace4SSean Bruno rxd->wb.upper.status_error &= htole32(~0xFF); 716f2d6ace4SSean Bruno 717f2d6ace4SSean Bruno if (++cidx == scctx->isc_nrxd[0]) 718f2d6ace4SSean Bruno cidx = 0; 719f2d6ace4SSean Bruno i++; 720f2d6ace4SSean Bruno } while (!eop); 721f2d6ace4SSean Bruno 722f2d6ace4SSean Bruno /* XXX add a faster way to look this up */ 723f2d6ace4SSean Bruno if (adapter->hw.mac.type >= e1000_82543) 724f2d6ace4SSean Bruno em_receive_checksum(staterr, ri); 725f2d6ace4SSean Bruno 726f2d6ace4SSean Bruno if (staterr & E1000_RXD_STAT_VP) { 727f2d6ace4SSean Bruno vtag = le16toh(rxd->wb.upper.vlan); 728f2d6ace4SSean Bruno } 729f2d6ace4SSean Bruno 730f2d6ace4SSean Bruno ri->iri_vtag = vtag; 731f2d6ace4SSean Bruno if (vtag) 732f2d6ace4SSean Bruno ri->iri_flags |= M_VLANTAG; 733f2d6ace4SSean Bruno 73495246abbSSean Bruno ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss); 735cbf1505dSSean Bruno ri->iri_rsstype = em_determine_rsstype(pkt_info); 736cbf1505dSSean Bruno 737cbf1505dSSean Bruno ri->iri_nfrags = i; 738f2d6ace4SSean Bruno return (0); 739f2d6ace4SSean Bruno } 740f2d6ace4SSean Bruno 741f2d6ace4SSean Bruno /********************************************************************* 742f2d6ace4SSean Bruno * 743f2d6ace4SSean Bruno * Verify that the hardware indicated that the checksum is valid. 744f2d6ace4SSean Bruno * Inform the stack about the status of checksum so that stack 745f2d6ace4SSean Bruno * doesn't spend time verifying the checksum. 746f2d6ace4SSean Bruno * 747f2d6ace4SSean Bruno *********************************************************************/ 748f2d6ace4SSean Bruno static void 749f2d6ace4SSean Bruno lem_receive_checksum(int status, int errors, if_rxd_info_t ri) 750f2d6ace4SSean Bruno { 751f2d6ace4SSean Bruno /* Did it pass? */ 752f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IPCS && !(errors & E1000_RXD_ERR_IPE)) 753f2d6ace4SSean Bruno ri->iri_csum_flags = (CSUM_IP_CHECKED|CSUM_IP_VALID); 754f2d6ace4SSean Bruno 755f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_TCPCS) { 756f2d6ace4SSean Bruno /* Did it pass? */ 757f2d6ace4SSean Bruno if (!(errors & E1000_RXD_ERR_TCPE)) { 758f2d6ace4SSean Bruno ri->iri_csum_flags |= 759f2d6ace4SSean Bruno (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 760f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 761f2d6ace4SSean Bruno } 762f2d6ace4SSean Bruno } 763f2d6ace4SSean Bruno } 764f2d6ace4SSean Bruno 765cbf1505dSSean Bruno /******************************************************************** 766cbf1505dSSean Bruno * 767cbf1505dSSean Bruno * Parse the packet type to determine the appropriate hash 768cbf1505dSSean Bruno * 769cbf1505dSSean Bruno ******************************************************************/ 770cbf1505dSSean Bruno static int 771cbf1505dSSean Bruno em_determine_rsstype(u32 pkt_info) 772cbf1505dSSean Bruno { 773cbf1505dSSean Bruno switch (pkt_info & E1000_RXDADV_RSSTYPE_MASK) { 774cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV4_TCP: 775cbf1505dSSean Bruno return M_HASHTYPE_RSS_TCP_IPV4; 776cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV4: 777cbf1505dSSean Bruno return M_HASHTYPE_RSS_IPV4; 778cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_TCP: 779cbf1505dSSean Bruno return M_HASHTYPE_RSS_TCP_IPV6; 780cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_EX: 781cbf1505dSSean Bruno return M_HASHTYPE_RSS_IPV6_EX; 782cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6: 783cbf1505dSSean Bruno return M_HASHTYPE_RSS_IPV6; 784cbf1505dSSean Bruno case E1000_RXDADV_RSSTYPE_IPV6_TCP_EX: 785cbf1505dSSean Bruno return M_HASHTYPE_RSS_TCP_IPV6_EX; 786cbf1505dSSean Bruno default: 787cbf1505dSSean Bruno return M_HASHTYPE_OPAQUE; 788cbf1505dSSean Bruno } 789cbf1505dSSean Bruno } 79095246abbSSean Bruno 791f2d6ace4SSean Bruno static void 792f2d6ace4SSean Bruno em_receive_checksum(uint32_t status, if_rxd_info_t ri) 793f2d6ace4SSean Bruno { 794f2d6ace4SSean Bruno ri->iri_csum_flags = 0; 795f2d6ace4SSean Bruno 796f2d6ace4SSean Bruno /* Ignore Checksum bit is set */ 797f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_IXSM) 798f2d6ace4SSean Bruno return; 799f2d6ace4SSean Bruno 800f2d6ace4SSean Bruno /* If the IP checksum exists and there is no IP Checksum error */ 801f2d6ace4SSean Bruno if ((status & (E1000_RXD_STAT_IPCS | E1000_RXDEXT_STATERR_IPE)) == 802f2d6ace4SSean Bruno E1000_RXD_STAT_IPCS) { 803f2d6ace4SSean Bruno ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID); 804f2d6ace4SSean Bruno } 805f2d6ace4SSean Bruno 806f2d6ace4SSean Bruno /* TCP or UDP checksum */ 807f2d6ace4SSean Bruno if ((status & (E1000_RXD_STAT_TCPCS | E1000_RXDEXT_STATERR_TCPE)) == 808f2d6ace4SSean Bruno E1000_RXD_STAT_TCPCS) { 809f2d6ace4SSean Bruno ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 810f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 811f2d6ace4SSean Bruno } 812f2d6ace4SSean Bruno if (status & E1000_RXD_STAT_UDPCS) { 813f2d6ace4SSean Bruno ri->iri_csum_flags |= (CSUM_DATA_VALID | CSUM_PSEUDO_HDR); 814f2d6ace4SSean Bruno ri->iri_csum_data = htons(0xffff); 815f2d6ace4SSean Bruno } 816f2d6ace4SSean Bruno } 817