1758cc3dcSJack F Vogel /******************************************************************************
2758cc3dcSJack F Vogel
38eb6488eSEric Joyner Copyright (c) 2001-2017, Intel Corporation
4758cc3dcSJack F Vogel All rights reserved.
5758cc3dcSJack F Vogel
6758cc3dcSJack F Vogel Redistribution and use in source and binary forms, with or without
7758cc3dcSJack F Vogel modification, are permitted provided that the following conditions are met:
8758cc3dcSJack F Vogel
9758cc3dcSJack F Vogel 1. Redistributions of source code must retain the above copyright notice,
10758cc3dcSJack F Vogel this list of conditions and the following disclaimer.
11758cc3dcSJack F Vogel
12758cc3dcSJack F Vogel 2. Redistributions in binary form must reproduce the above copyright
13758cc3dcSJack F Vogel notice, this list of conditions and the following disclaimer in the
14758cc3dcSJack F Vogel documentation and/or other materials provided with the distribution.
15758cc3dcSJack F Vogel
16758cc3dcSJack F Vogel 3. Neither the name of the Intel Corporation nor the names of its
17758cc3dcSJack F Vogel contributors may be used to endorse or promote products derived from
18758cc3dcSJack F Vogel this software without specific prior written permission.
19758cc3dcSJack F Vogel
20758cc3dcSJack F Vogel THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
21758cc3dcSJack F Vogel AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22758cc3dcSJack F Vogel IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23758cc3dcSJack F Vogel ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
24758cc3dcSJack F Vogel LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25758cc3dcSJack F Vogel CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26758cc3dcSJack F Vogel SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27758cc3dcSJack F Vogel INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28758cc3dcSJack F Vogel CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29758cc3dcSJack F Vogel ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30758cc3dcSJack F Vogel POSSIBILITY OF SUCH DAMAGE.
31758cc3dcSJack F Vogel
32758cc3dcSJack F Vogel ******************************************************************************/
33758cc3dcSJack F Vogel
34758cc3dcSJack F Vogel #ifndef IXGBE_STANDALONE_BUILD
35758cc3dcSJack F Vogel #include "opt_inet.h"
36758cc3dcSJack F Vogel #include "opt_inet6.h"
37758cc3dcSJack F Vogel #include "opt_rss.h"
38758cc3dcSJack F Vogel #endif
39758cc3dcSJack F Vogel
40758cc3dcSJack F Vogel #include "ixgbe.h"
41758cc3dcSJack F Vogel
428eb6488eSEric Joyner /************************************************************************
43758cc3dcSJack F Vogel * Local Function prototypes
448eb6488eSEric Joyner ************************************************************************/
45b1d5caf3SKevin Bowling static int ixgbe_isc_txd_encap(void *, if_pkt_info_t);
46b1d5caf3SKevin Bowling static void ixgbe_isc_txd_flush(void *, uint16_t, qidx_t);
47b1d5caf3SKevin Bowling static int ixgbe_isc_txd_credits_update(void *, uint16_t, bool);
48758cc3dcSJack F Vogel
49b1d5caf3SKevin Bowling static void ixgbe_isc_rxd_refill(void *, if_rxd_update_t);
50b1d5caf3SKevin Bowling static void ixgbe_isc_rxd_flush(void *, uint16_t, uint8_t, qidx_t);
51b1d5caf3SKevin Bowling static int ixgbe_isc_rxd_available(void *, uint16_t, qidx_t, qidx_t);
52b1d5caf3SKevin Bowling static int ixgbe_isc_rxd_pkt_get(void *, if_rxd_info_t);
53758cc3dcSJack F Vogel
54b1d5caf3SKevin Bowling static void ixgbe_rx_checksum(uint32_t, if_rxd_info_t, uint32_t);
5551e46835SKevin Bowling static int ixgbe_tx_ctx_setup(struct ixgbe_adv_tx_context_desc *,
5651e46835SKevin Bowling if_pkt_info_t);
57758cc3dcSJack F Vogel
58c19c7afeSEric Joyner extern void ixgbe_if_enable_intr(if_ctx_t ctx);
5951e46835SKevin Bowling static int ixgbe_determine_rsstype(uint16_t pkt_info);
60758cc3dcSJack F Vogel
61c19c7afeSEric Joyner struct if_txrx ixgbe_txrx = {
62fbf8b74cSMark Johnston .ift_txd_encap = ixgbe_isc_txd_encap,
63fbf8b74cSMark Johnston .ift_txd_flush = ixgbe_isc_txd_flush,
64fbf8b74cSMark Johnston .ift_txd_credits_update = ixgbe_isc_txd_credits_update,
65fbf8b74cSMark Johnston .ift_rxd_available = ixgbe_isc_rxd_available,
66fbf8b74cSMark Johnston .ift_rxd_pkt_get = ixgbe_isc_rxd_pkt_get,
67fbf8b74cSMark Johnston .ift_rxd_refill = ixgbe_isc_rxd_refill,
68fbf8b74cSMark Johnston .ift_rxd_flush = ixgbe_isc_rxd_flush,
69fbf8b74cSMark Johnston .ift_legacy_intr = NULL
70c19c7afeSEric Joyner };
71758cc3dcSJack F Vogel
728eb6488eSEric Joyner /************************************************************************
738eb6488eSEric Joyner * ixgbe_tx_ctx_setup
74758cc3dcSJack F Vogel *
75758cc3dcSJack F Vogel * Advanced Context Descriptor setup for VLAN, CSUM or TSO
76c19c7afeSEric Joyner *
778eb6488eSEric Joyner ************************************************************************/
78758cc3dcSJack F Vogel static int
ixgbe_tx_ctx_setup(struct ixgbe_adv_tx_context_desc * TXD,if_pkt_info_t pi)79c19c7afeSEric Joyner ixgbe_tx_ctx_setup(struct ixgbe_adv_tx_context_desc *TXD, if_pkt_info_t pi)
80758cc3dcSJack F Vogel {
8151e46835SKevin Bowling uint32_t vlan_macip_lens, type_tucmd_mlhl;
8251e46835SKevin Bowling uint32_t olinfo_status, mss_l4len_idx, pktlen, offload;
83c19c7afeSEric Joyner u8 ehdrlen;
84a9ca1c79SSean Bruno
8579b36ec9SKevin Bowling offload = true;
86c19c7afeSEric Joyner olinfo_status = mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0;
87c19c7afeSEric Joyner /* VLAN MACLEN IPLEN */
88c19c7afeSEric Joyner vlan_macip_lens |= (htole16(pi->ipi_vtag) << IXGBE_ADVTXD_VLAN_SHIFT);
89758cc3dcSJack F Vogel
90758cc3dcSJack F Vogel /*
91c19c7afeSEric Joyner * Some of our VF devices need a context descriptor for every
92c19c7afeSEric Joyner * packet. That means the ehdrlen needs to be non-zero in order
93c19c7afeSEric Joyner * for the host driver not to flag a malicious event. The stack
94c19c7afeSEric Joyner * will most likely populate this for all other reasons of why
95c19c7afeSEric Joyner * this function was called.
96758cc3dcSJack F Vogel */
97c19c7afeSEric Joyner if (pi->ipi_ehdrlen == 0) {
98758cc3dcSJack F Vogel ehdrlen = ETHER_HDR_LEN;
99c19c7afeSEric Joyner ehdrlen += (pi->ipi_vtag != 0) ? ETHER_VLAN_ENCAP_LEN : 0;
100c19c7afeSEric Joyner } else
101c19c7afeSEric Joyner ehdrlen = pi->ipi_ehdrlen;
102758cc3dcSJack F Vogel vlan_macip_lens |= ehdrlen << IXGBE_ADVTXD_MACLEN_SHIFT;
103758cc3dcSJack F Vogel
104c19c7afeSEric Joyner pktlen = pi->ipi_len;
105c19c7afeSEric Joyner /* First check if TSO is to be used */
106c19c7afeSEric Joyner if (pi->ipi_csum_flags & CSUM_TSO) {
107c19c7afeSEric Joyner /* This is used in the transmit desc in encap */
108c19c7afeSEric Joyner pktlen = pi->ipi_len - ehdrlen - pi->ipi_ip_hlen - pi->ipi_tcp_hlen;
109c19c7afeSEric Joyner mss_l4len_idx |= (pi->ipi_tso_segsz << IXGBE_ADVTXD_MSS_SHIFT);
110c19c7afeSEric Joyner mss_l4len_idx |= (pi->ipi_tcp_hlen << IXGBE_ADVTXD_L4LEN_SHIFT);
111a9ca1c79SSean Bruno }
112c19c7afeSEric Joyner
113c19c7afeSEric Joyner olinfo_status |= pktlen << IXGBE_ADVTXD_PAYLEN_SHIFT;
114c19c7afeSEric Joyner
115c19c7afeSEric Joyner if (pi->ipi_flags & IPI_TX_IPV4) {
116c19c7afeSEric Joyner type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV4;
117c19c7afeSEric Joyner /* Tell transmit desc to also do IPv4 checksum. */
118c19c7afeSEric Joyner if (pi->ipi_csum_flags & (CSUM_IP|CSUM_TSO))
119c19c7afeSEric Joyner olinfo_status |= IXGBE_TXD_POPTS_IXSM << 8;
120c19c7afeSEric Joyner } else if (pi->ipi_flags & IPI_TX_IPV6)
121758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_IPV6;
122c19c7afeSEric Joyner else
12379b36ec9SKevin Bowling offload = false;
124c19c7afeSEric Joyner
125c19c7afeSEric Joyner vlan_macip_lens |= pi->ipi_ip_hlen;
126c19c7afeSEric Joyner
127c19c7afeSEric Joyner switch (pi->ipi_ipproto) {
128c19c7afeSEric Joyner case IPPROTO_TCP:
1297aad1f4eSEric Joyner if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP | CSUM_TSO))
130c19c7afeSEric Joyner type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_TCP;
131c19c7afeSEric Joyner else
13279b36ec9SKevin Bowling offload = false;
133758cc3dcSJack F Vogel break;
134c19c7afeSEric Joyner case IPPROTO_UDP:
135c19c7afeSEric Joyner if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP))
136c19c7afeSEric Joyner type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_UDP;
137c19c7afeSEric Joyner else
13879b36ec9SKevin Bowling offload = false;
139c19c7afeSEric Joyner break;
140c19c7afeSEric Joyner case IPPROTO_SCTP:
141c19c7afeSEric Joyner if (pi->ipi_csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP))
142c19c7afeSEric Joyner type_tucmd_mlhl |= IXGBE_ADVTXD_TUCMD_L4T_SCTP;
143c19c7afeSEric Joyner else
14479b36ec9SKevin Bowling offload = false;
145c19c7afeSEric Joyner break;
146758cc3dcSJack F Vogel default:
14779b36ec9SKevin Bowling offload = false;
148758cc3dcSJack F Vogel break;
149758cc3dcSJack F Vogel }
150c19c7afeSEric Joyner /* Insert L4 checksum into data descriptors */
151c19c7afeSEric Joyner if (offload)
152c19c7afeSEric Joyner olinfo_status |= IXGBE_TXD_POPTS_TXSM << 8;
153758cc3dcSJack F Vogel
154758cc3dcSJack F Vogel type_tucmd_mlhl |= IXGBE_ADVTXD_DCMD_DEXT | IXGBE_ADVTXD_DTYP_CTXT;
155758cc3dcSJack F Vogel
156758cc3dcSJack F Vogel /* Now copy bits into descriptor */
157758cc3dcSJack F Vogel TXD->vlan_macip_lens = htole32(vlan_macip_lens);
158758cc3dcSJack F Vogel TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
159758cc3dcSJack F Vogel TXD->seqnum_seed = htole32(0);
160c19c7afeSEric Joyner TXD->mss_l4len_idx = htole32(mss_l4len_idx);
161758cc3dcSJack F Vogel
162c19c7afeSEric Joyner return (olinfo_status);
1638eb6488eSEric Joyner } /* ixgbe_tx_ctx_setup */
164758cc3dcSJack F Vogel
1658eb6488eSEric Joyner /************************************************************************
166c19c7afeSEric Joyner * ixgbe_isc_txd_encap
1678eb6488eSEric Joyner ************************************************************************/
168758cc3dcSJack F Vogel static int
ixgbe_isc_txd_encap(void * arg,if_pkt_info_t pi)169c19c7afeSEric Joyner ixgbe_isc_txd_encap(void *arg, if_pkt_info_t pi)
170758cc3dcSJack F Vogel {
171b1d5caf3SKevin Bowling struct ixgbe_softc *sc = arg;
172c19c7afeSEric Joyner if_softc_ctx_t scctx = sc->shared;
173c19c7afeSEric Joyner struct ix_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
174c19c7afeSEric Joyner struct tx_ring *txr = &que->txr;
175c19c7afeSEric Joyner int nsegs = pi->ipi_nsegs;
176c19c7afeSEric Joyner bus_dma_segment_t *segs = pi->ipi_segs;
177c19c7afeSEric Joyner union ixgbe_adv_tx_desc *txd = NULL;
178758cc3dcSJack F Vogel struct ixgbe_adv_tx_context_desc *TXD;
179c19c7afeSEric Joyner int i, j, first, pidx_last;
18051e46835SKevin Bowling uint32_t olinfo_status, cmd, flags;
181c19c7afeSEric Joyner qidx_t ntxd;
182758cc3dcSJack F Vogel
183c19c7afeSEric Joyner cmd = (IXGBE_ADVTXD_DTYP_DATA |
184c19c7afeSEric Joyner IXGBE_ADVTXD_DCMD_IFCS | IXGBE_ADVTXD_DCMD_DEXT);
185758cc3dcSJack F Vogel
186c19c7afeSEric Joyner if (pi->ipi_mflags & M_VLANTAG)
187c19c7afeSEric Joyner cmd |= IXGBE_ADVTXD_DCMD_VLE;
188758cc3dcSJack F Vogel
189c19c7afeSEric Joyner i = first = pi->ipi_pidx;
190c19c7afeSEric Joyner flags = (pi->ipi_flags & IPI_TX_INTR) ? IXGBE_TXD_CMD_RS : 0;
191c19c7afeSEric Joyner ntxd = scctx->isc_ntxd[0];
192758cc3dcSJack F Vogel
193c19c7afeSEric Joyner TXD = (struct ixgbe_adv_tx_context_desc *) &txr->tx_base[first];
194c19c7afeSEric Joyner if ((pi->ipi_csum_flags & CSUM_OFFLOAD) ||
195c19c7afeSEric Joyner (sc->feat_en & IXGBE_FEATURE_NEEDS_CTXD) ||
196c19c7afeSEric Joyner pi->ipi_vtag) {
197c19c7afeSEric Joyner /*********************************************
198c19c7afeSEric Joyner * Set up the appropriate offload context
199c19c7afeSEric Joyner * this will consume the first descriptor
200c19c7afeSEric Joyner *********************************************/
201c19c7afeSEric Joyner olinfo_status = ixgbe_tx_ctx_setup(TXD, pi);
202c19c7afeSEric Joyner if (pi->ipi_csum_flags & CSUM_TSO) {
203c19c7afeSEric Joyner cmd |= IXGBE_ADVTXD_DCMD_TSE;
204758cc3dcSJack F Vogel ++txr->tso_tx;
205c19c7afeSEric Joyner }
206c19c7afeSEric Joyner
207c19c7afeSEric Joyner if (++i == scctx->isc_ntxd[0])
208c19c7afeSEric Joyner i = 0;
209c19c7afeSEric Joyner } else {
210c19c7afeSEric Joyner /* Indicate the whole packet as payload when not doing TSO */
211c19c7afeSEric Joyner olinfo_status = pi->ipi_len << IXGBE_ADVTXD_PAYLEN_SHIFT;
212c19c7afeSEric Joyner }
213c19c7afeSEric Joyner
214c19c7afeSEric Joyner olinfo_status |= IXGBE_ADVTXD_CC;
21524a7d6d3SMatt Macy pidx_last = 0;
216c19c7afeSEric Joyner for (j = 0; j < nsegs; j++) {
217c19c7afeSEric Joyner bus_size_t seglen;
218c19c7afeSEric Joyner
219c19c7afeSEric Joyner txd = &txr->tx_base[i];
220c19c7afeSEric Joyner seglen = segs[j].ds_len;
221c19c7afeSEric Joyner
222c19c7afeSEric Joyner txd->read.buffer_addr = htole64(segs[j].ds_addr);
223c19c7afeSEric Joyner txd->read.cmd_type_len = htole32(cmd | seglen);
224c19c7afeSEric Joyner txd->read.olinfo_status = htole32(olinfo_status);
225c19c7afeSEric Joyner
226c19c7afeSEric Joyner pidx_last = i;
227c19c7afeSEric Joyner if (++i == scctx->isc_ntxd[0]) {
228c19c7afeSEric Joyner i = 0;
229c19c7afeSEric Joyner }
230c19c7afeSEric Joyner }
231c19c7afeSEric Joyner
232c19c7afeSEric Joyner if (flags) {
233c19c7afeSEric Joyner txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
234c19c7afeSEric Joyner txr->tx_rs_pidx = (txr->tx_rs_pidx + 1) & (ntxd - 1);
235c19c7afeSEric Joyner }
236c19c7afeSEric Joyner txd->read.cmd_type_len |= htole32(IXGBE_TXD_CMD_EOP | flags);
237c19c7afeSEric Joyner
238c19c7afeSEric Joyner txr->bytes += pi->ipi_len;
239c19c7afeSEric Joyner pi->ipi_new_pidx = i;
240c19c7afeSEric Joyner
241c19c7afeSEric Joyner ++txr->total_packets;
2428eb6488eSEric Joyner
243758cc3dcSJack F Vogel return (0);
244c19c7afeSEric Joyner } /* ixgbe_isc_txd_encap */
245758cc3dcSJack F Vogel
2468eb6488eSEric Joyner /************************************************************************
247c19c7afeSEric Joyner * ixgbe_isc_txd_flush
2488eb6488eSEric Joyner ************************************************************************/
249c19c7afeSEric Joyner static void
ixgbe_isc_txd_flush(void * arg,uint16_t txqid,qidx_t pidx)250c19c7afeSEric Joyner ixgbe_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
251758cc3dcSJack F Vogel {
252b1d5caf3SKevin Bowling struct ixgbe_softc *sc = arg;
253c19c7afeSEric Joyner struct ix_tx_queue *que = &sc->tx_queues[txqid];
254c19c7afeSEric Joyner struct tx_ring *txr = &que->txr;
255758cc3dcSJack F Vogel
256c19c7afeSEric Joyner IXGBE_WRITE_REG(&sc->hw, txr->tail, pidx);
257c19c7afeSEric Joyner } /* ixgbe_isc_txd_flush */
258758cc3dcSJack F Vogel
259c19c7afeSEric Joyner /************************************************************************
260c19c7afeSEric Joyner * ixgbe_isc_txd_credits_update
261c19c7afeSEric Joyner ************************************************************************/
262c19c7afeSEric Joyner static int
ixgbe_isc_txd_credits_update(void * arg,uint16_t txqid,bool clear)263c19c7afeSEric Joyner ixgbe_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
264c19c7afeSEric Joyner {
265b1d5caf3SKevin Bowling struct ixgbe_softc *sc = arg;
266c19c7afeSEric Joyner if_softc_ctx_t scctx = sc->shared;
267c19c7afeSEric Joyner struct ix_tx_queue *que = &sc->tx_queues[txqid];
268c19c7afeSEric Joyner struct tx_ring *txr = &que->txr;
269c19c7afeSEric Joyner qidx_t processed = 0;
270c19c7afeSEric Joyner int updated;
271c19c7afeSEric Joyner qidx_t cur, prev, ntxd, rs_cidx;
272c19c7afeSEric Joyner int32_t delta;
273c19c7afeSEric Joyner uint8_t status;
274758cc3dcSJack F Vogel
275c19c7afeSEric Joyner rs_cidx = txr->tx_rs_cidx;
276c19c7afeSEric Joyner if (rs_cidx == txr->tx_rs_pidx)
277c19c7afeSEric Joyner return (0);
278758cc3dcSJack F Vogel
279c19c7afeSEric Joyner cur = txr->tx_rsq[rs_cidx];
280c19c7afeSEric Joyner status = txr->tx_base[cur].wb.status;
281c19c7afeSEric Joyner updated = !!(status & IXGBE_TXD_STAT_DD);
282758cc3dcSJack F Vogel
283adf93b56SEric Joyner if (!updated)
284adf93b56SEric Joyner return (0);
285adf93b56SEric Joyner
286adf93b56SEric Joyner /* If clear is false just let caller know that there
287adf93b56SEric Joyner * are descriptors to reclaim */
288adf93b56SEric Joyner if (!clear)
289adf93b56SEric Joyner return (1);
290c19c7afeSEric Joyner
291c19c7afeSEric Joyner prev = txr->tx_cidx_processed;
292c19c7afeSEric Joyner ntxd = scctx->isc_ntxd[0];
293758cc3dcSJack F Vogel do {
294088a0b27SEric Joyner MPASS(prev != cur);
295c19c7afeSEric Joyner delta = (int32_t)cur - (int32_t)prev;
296c19c7afeSEric Joyner if (delta < 0)
297c19c7afeSEric Joyner delta += ntxd;
298088a0b27SEric Joyner MPASS(delta > 0);
299c19c7afeSEric Joyner
300c19c7afeSEric Joyner processed += delta;
301c19c7afeSEric Joyner prev = cur;
302c19c7afeSEric Joyner rs_cidx = (rs_cidx + 1) & (ntxd - 1);
303c19c7afeSEric Joyner if (rs_cidx == txr->tx_rs_pidx)
304758cc3dcSJack F Vogel break;
305758cc3dcSJack F Vogel
306c19c7afeSEric Joyner cur = txr->tx_rsq[rs_cidx];
307c19c7afeSEric Joyner status = txr->tx_base[cur].wb.status;
308c19c7afeSEric Joyner } while ((status & IXGBE_TXD_STAT_DD));
309758cc3dcSJack F Vogel
310c19c7afeSEric Joyner txr->tx_rs_cidx = rs_cidx;
311c19c7afeSEric Joyner txr->tx_cidx_processed = prev;
312758cc3dcSJack F Vogel
313c19c7afeSEric Joyner return (processed);
314c19c7afeSEric Joyner } /* ixgbe_isc_txd_credits_update */
315758cc3dcSJack F Vogel
3168eb6488eSEric Joyner /************************************************************************
317c19c7afeSEric Joyner * ixgbe_isc_rxd_refill
3188eb6488eSEric Joyner ************************************************************************/
319758cc3dcSJack F Vogel static void
ixgbe_isc_rxd_refill(void * arg,if_rxd_update_t iru)320c19c7afeSEric Joyner ixgbe_isc_rxd_refill(void *arg, if_rxd_update_t iru)
321758cc3dcSJack F Vogel {
322b1d5caf3SKevin Bowling struct ixgbe_softc *sc = arg;
323c19c7afeSEric Joyner struct ix_rx_queue *que = &sc->rx_queues[iru->iru_qsidx];
324c19c7afeSEric Joyner struct rx_ring *rxr = &que->rxr;
325c19c7afeSEric Joyner uint64_t *paddrs;
326c19c7afeSEric Joyner int i;
327c19c7afeSEric Joyner uint32_t next_pidx, pidx;
328c19c7afeSEric Joyner uint16_t count;
329758cc3dcSJack F Vogel
330c19c7afeSEric Joyner paddrs = iru->iru_paddrs;
331c19c7afeSEric Joyner pidx = iru->iru_pidx;
332c19c7afeSEric Joyner count = iru->iru_count;
333c19c7afeSEric Joyner
334c19c7afeSEric Joyner for (i = 0, next_pidx = pidx; i < count; i++) {
335c19c7afeSEric Joyner rxr->rx_base[next_pidx].read.pkt_addr = htole64(paddrs[i]);
336c19c7afeSEric Joyner if (++next_pidx == sc->shared->isc_nrxd[0])
337c19c7afeSEric Joyner next_pidx = 0;
338758cc3dcSJack F Vogel }
339c19c7afeSEric Joyner } /* ixgbe_isc_rxd_refill */
340a9ca1c79SSean Bruno
3418eb6488eSEric Joyner /************************************************************************
342c19c7afeSEric Joyner * ixgbe_isc_rxd_flush
3438eb6488eSEric Joyner ************************************************************************/
344758cc3dcSJack F Vogel static void
ixgbe_isc_rxd_flush(void * arg,uint16_t qsidx,uint8_t flidx __unused,qidx_t pidx)345c19c7afeSEric Joyner ixgbe_isc_rxd_flush(void *arg, uint16_t qsidx, uint8_t flidx __unused, qidx_t pidx)
346758cc3dcSJack F Vogel {
347b1d5caf3SKevin Bowling struct ixgbe_softc *sc = arg;
348c19c7afeSEric Joyner struct ix_rx_queue *que = &sc->rx_queues[qsidx];
349c19c7afeSEric Joyner struct rx_ring *rxr = &que->rxr;
350758cc3dcSJack F Vogel
351c19c7afeSEric Joyner IXGBE_WRITE_REG(&sc->hw, rxr->tail, pidx);
352c19c7afeSEric Joyner } /* ixgbe_isc_rxd_flush */
3538eb6488eSEric Joyner
3548eb6488eSEric Joyner /************************************************************************
355c19c7afeSEric Joyner * ixgbe_isc_rxd_available
3568eb6488eSEric Joyner ************************************************************************/
3578eb6488eSEric Joyner static int
ixgbe_isc_rxd_available(void * arg,uint16_t qsidx,qidx_t pidx,qidx_t budget)358c19c7afeSEric Joyner ixgbe_isc_rxd_available(void *arg, uint16_t qsidx, qidx_t pidx, qidx_t budget)
359758cc3dcSJack F Vogel {
360b1d5caf3SKevin Bowling struct ixgbe_softc *sc = arg;
361c19c7afeSEric Joyner struct ix_rx_queue *que = &sc->rx_queues[qsidx];
362c19c7afeSEric Joyner struct rx_ring *rxr = &que->rxr;
363c19c7afeSEric Joyner union ixgbe_adv_rx_desc *rxd;
36451e46835SKevin Bowling uint32_t staterr;
365c19c7afeSEric Joyner int cnt, i, nrxd;
366758cc3dcSJack F Vogel
367c19c7afeSEric Joyner nrxd = sc->shared->isc_nrxd[0];
368adf93b56SEric Joyner for (cnt = 0, i = pidx; cnt < nrxd && cnt <= budget;) {
369c19c7afeSEric Joyner rxd = &rxr->rx_base[i];
370c19c7afeSEric Joyner staterr = le32toh(rxd->wb.upper.status_error);
371758cc3dcSJack F Vogel
372758cc3dcSJack F Vogel if ((staterr & IXGBE_RXD_STAT_DD) == 0)
373758cc3dcSJack F Vogel break;
374c19c7afeSEric Joyner if (++i == nrxd)
375c19c7afeSEric Joyner i = 0;
376c19c7afeSEric Joyner if (staterr & IXGBE_RXD_STAT_EOP)
377c19c7afeSEric Joyner cnt++;
378c19c7afeSEric Joyner }
379c19c7afeSEric Joyner return (cnt);
380c19c7afeSEric Joyner } /* ixgbe_isc_rxd_available */
381758cc3dcSJack F Vogel
382c19c7afeSEric Joyner /************************************************************************
383c19c7afeSEric Joyner * ixgbe_isc_rxd_pkt_get
384c19c7afeSEric Joyner *
385c19c7afeSEric Joyner * Routine sends data which has been dma'ed into host memory
386c19c7afeSEric Joyner * to upper layer. Initialize ri structure.
387c19c7afeSEric Joyner *
388c19c7afeSEric Joyner * Returns 0 upon success, errno on failure
389c19c7afeSEric Joyner ************************************************************************/
390c19c7afeSEric Joyner
391c19c7afeSEric Joyner static int
ixgbe_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)392c19c7afeSEric Joyner ixgbe_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
393c19c7afeSEric Joyner {
394b1d5caf3SKevin Bowling struct ixgbe_softc *sc = arg;
39552f45d8aSVincenzo Maffione if_softc_ctx_t scctx = sc->shared;
396b1d5caf3SKevin Bowling struct ix_rx_queue *que = &sc->rx_queues[ri->iri_qsidx];
397c19c7afeSEric Joyner struct rx_ring *rxr = &que->rxr;
398c19c7afeSEric Joyner union ixgbe_adv_rx_desc *rxd;
399c19c7afeSEric Joyner
40051e46835SKevin Bowling uint16_t pkt_info, len, cidx, i;
40151e46835SKevin Bowling uint32_t ptype;
40251e46835SKevin Bowling uint32_t staterr = 0;
403c19c7afeSEric Joyner bool eop;
404c19c7afeSEric Joyner
405c19c7afeSEric Joyner i = 0;
406c19c7afeSEric Joyner cidx = ri->iri_cidx;
407c19c7afeSEric Joyner do {
408c19c7afeSEric Joyner rxd = &rxr->rx_base[cidx];
409c19c7afeSEric Joyner staterr = le32toh(rxd->wb.upper.status_error);
410c19c7afeSEric Joyner pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info);
411c19c7afeSEric Joyner
412c19c7afeSEric Joyner /* Error Checking then decrement count */
413c19c7afeSEric Joyner MPASS ((staterr & IXGBE_RXD_STAT_DD) != 0);
414c19c7afeSEric Joyner
415c19c7afeSEric Joyner len = le16toh(rxd->wb.upper.length);
416c19c7afeSEric Joyner ptype = le32toh(rxd->wb.lower.lo_dword.data) &
417758cc3dcSJack F Vogel IXGBE_RXDADV_PKTTYPE_MASK;
418c19c7afeSEric Joyner
419c19c7afeSEric Joyner ri->iri_len += len;
420c19c7afeSEric Joyner rxr->bytes += len;
421c19c7afeSEric Joyner
422c19c7afeSEric Joyner rxd->wb.upper.status_error = 0;
423758cc3dcSJack F Vogel eop = ((staterr & IXGBE_RXD_STAT_EOP) != 0);
424de35521aSEric Joyner
425758cc3dcSJack F Vogel /* Make sure bad packets are discarded */
426758cc3dcSJack F Vogel if (eop && (staterr & IXGBE_RXDADV_ERR_FRAME_ERR_MASK) != 0) {
427b1d5caf3SKevin Bowling if (sc->feat_en & IXGBE_FEATURE_VF)
42852f45d8aSVincenzo Maffione if_inc_counter(ri->iri_ifp, IFCOUNTER_IERRORS, 1);
429c19c7afeSEric Joyner
430758cc3dcSJack F Vogel rxr->rx_discarded++;
431c19c7afeSEric Joyner return (EBADMSG);
432758cc3dcSJack F Vogel }
433c19c7afeSEric Joyner ri->iri_frags[i].irf_flid = 0;
434c19c7afeSEric Joyner ri->iri_frags[i].irf_idx = cidx;
435c19c7afeSEric Joyner ri->iri_frags[i].irf_len = len;
436b1d5caf3SKevin Bowling if (++cidx == sc->shared->isc_nrxd[0])
437c19c7afeSEric Joyner cidx = 0;
438c19c7afeSEric Joyner i++;
439c19c7afeSEric Joyner /* even a 16K packet shouldn't consume more than 8 clusters */
440c19c7afeSEric Joyner MPASS(i < 9);
441c19c7afeSEric Joyner } while (!eop);
442758cc3dcSJack F Vogel
443758cc3dcSJack F Vogel rxr->rx_packets++;
444c19c7afeSEric Joyner rxr->packets++;
445c19c7afeSEric Joyner rxr->rx_bytes += ri->iri_len;
446c19c7afeSEric Joyner
44752f45d8aSVincenzo Maffione if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
448c19c7afeSEric Joyner ixgbe_rx_checksum(staterr, ri, ptype);
449df7b11faSHiren Panchasara
450c19c7afeSEric Joyner ri->iri_flowid = le32toh(rxd->wb.lower.hi_dword.rss);
451c19c7afeSEric Joyner ri->iri_rsstype = ixgbe_determine_rsstype(pkt_info);
452b1d5caf3SKevin Bowling if ((sc->feat_en & IXGBE_FEATURE_RSS) == 0) {
453236204eeSAndriy Gapon if (ri->iri_rsstype == M_HASHTYPE_OPAQUE)
454236204eeSAndriy Gapon ri->iri_rsstype = M_HASHTYPE_NONE;
455236204eeSAndriy Gapon else
456236204eeSAndriy Gapon ri->iri_rsstype = M_HASHTYPE_OPAQUE_HASH;
457236204eeSAndriy Gapon }
458f7926a6dSVincenzo Maffione if ((rxr->vtag_strip) && (staterr & IXGBE_RXD_STAT_VP)) {
459f7926a6dSVincenzo Maffione ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
460c19c7afeSEric Joyner ri->iri_flags |= M_VLANTAG;
461f7926a6dSVincenzo Maffione }
462f7926a6dSVincenzo Maffione
463f7926a6dSVincenzo Maffione ri->iri_nfrags = i;
464c19c7afeSEric Joyner return (0);
465c19c7afeSEric Joyner } /* ixgbe_isc_rxd_pkt_get */
466758cc3dcSJack F Vogel
4678eb6488eSEric Joyner /************************************************************************
4688eb6488eSEric Joyner * ixgbe_rx_checksum
469758cc3dcSJack F Vogel *
470758cc3dcSJack F Vogel * Verify that the hardware indicated that the checksum is valid.
471758cc3dcSJack F Vogel * Inform the stack about the status of checksum so that stack
472758cc3dcSJack F Vogel * doesn't spend time verifying the checksum.
4738eb6488eSEric Joyner ************************************************************************/
474758cc3dcSJack F Vogel static void
ixgbe_rx_checksum(uint32_t staterr,if_rxd_info_t ri,uint32_t ptype)47551e46835SKevin Bowling ixgbe_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype)
476758cc3dcSJack F Vogel {
47751e46835SKevin Bowling uint16_t status = (uint16_t)staterr;
47851e46835SKevin Bowling uint8_t errors = (uint8_t)(staterr >> 24);
479758cc3dcSJack F Vogel
48051e46835SKevin Bowling /* If there is a layer 3 or 4 error we are done */
48151e46835SKevin Bowling if (__predict_false(errors & (IXGBE_RXD_ERR_IPE | IXGBE_RXD_ERR_TCPE)))
48251e46835SKevin Bowling return;
483758cc3dcSJack F Vogel
484758cc3dcSJack F Vogel /* IP Checksum Good */
48551e46835SKevin Bowling if (status & IXGBE_RXD_STAT_IPCS)
48651e46835SKevin Bowling ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
48751e46835SKevin Bowling
48851e46835SKevin Bowling /* Valid L4E checksum */
48951e46835SKevin Bowling if (__predict_true(status & IXGBE_RXD_STAT_L4CS)) {
49051e46835SKevin Bowling /* SCTP header present. */
49151e46835SKevin Bowling if (__predict_false((ptype & IXGBE_RXDADV_PKTTYPE_ETQF) == 0 &&
49251e46835SKevin Bowling (ptype & IXGBE_RXDADV_PKTTYPE_SCTP) != 0)) {
49351e46835SKevin Bowling ri->iri_csum_flags |= CSUM_SCTP_VALID;
49451e46835SKevin Bowling } else {
49551e46835SKevin Bowling ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
496c19c7afeSEric Joyner ri->iri_csum_data = htons(0xffff);
497758cc3dcSJack F Vogel }
498758cc3dcSJack F Vogel }
4998eb6488eSEric Joyner } /* ixgbe_rx_checksum */
500758cc3dcSJack F Vogel
5018eb6488eSEric Joyner /************************************************************************
502c19c7afeSEric Joyner * ixgbe_determine_rsstype
503c19c7afeSEric Joyner *
504c19c7afeSEric Joyner * Parse the packet type to determine the appropriate hash
5058eb6488eSEric Joyner ************************************************************************/
5068eb6488eSEric Joyner static int
ixgbe_determine_rsstype(uint16_t pkt_info)50751e46835SKevin Bowling ixgbe_determine_rsstype(uint16_t pkt_info)
508758cc3dcSJack F Vogel {
509c19c7afeSEric Joyner switch (pkt_info & IXGBE_RXDADV_RSSTYPE_MASK) {
510c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV4_TCP:
511c19c7afeSEric Joyner return M_HASHTYPE_RSS_TCP_IPV4;
512c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV4:
513c19c7afeSEric Joyner return M_HASHTYPE_RSS_IPV4;
514c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV6_TCP:
515c19c7afeSEric Joyner return M_HASHTYPE_RSS_TCP_IPV6;
516c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV6_EX:
517c19c7afeSEric Joyner return M_HASHTYPE_RSS_IPV6_EX;
518c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV6:
519c19c7afeSEric Joyner return M_HASHTYPE_RSS_IPV6;
520c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV6_TCP_EX:
521c19c7afeSEric Joyner return M_HASHTYPE_RSS_TCP_IPV6_EX;
522c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV4_UDP:
523c19c7afeSEric Joyner return M_HASHTYPE_RSS_UDP_IPV4;
524c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV6_UDP:
525c19c7afeSEric Joyner return M_HASHTYPE_RSS_UDP_IPV6;
526c19c7afeSEric Joyner case IXGBE_RXDADV_RSSTYPE_IPV6_UDP_EX:
527c19c7afeSEric Joyner return M_HASHTYPE_RSS_UDP_IPV6_EX;
528c19c7afeSEric Joyner default:
529c19c7afeSEric Joyner return M_HASHTYPE_OPAQUE;
530758cc3dcSJack F Vogel }
531c19c7afeSEric Joyner } /* ixgbe_determine_rsstype */
532