1517904deSPeter Grehan /*-
2517904deSPeter Grehan * SPDX-License-Identifier: BSD-2-Clause
3517904deSPeter Grehan *
4517904deSPeter Grehan * Copyright (c) 2016 Matthew Macy <mmacy@mattmacy.io>
5517904deSPeter Grehan * All rights reserved.
6517904deSPeter Grehan * Copyright (c) 2021 Rubicon Communications, LLC (Netgate)
7517904deSPeter Grehan *
8517904deSPeter Grehan * Redistribution and use in source and binary forms, with or without
9517904deSPeter Grehan * modification, are permitted provided that the following conditions
10517904deSPeter Grehan * are met:
11517904deSPeter Grehan * 1. Redistributions of source code must retain the above copyright
12517904deSPeter Grehan * notice, this list of conditions and the following disclaimer.
13517904deSPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright
14517904deSPeter Grehan * notice, this list of conditions and the following disclaimer in the
15517904deSPeter Grehan * documentation and/or other materials provided with the distribution.
16517904deSPeter Grehan *
17517904deSPeter Grehan * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
18517904deSPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19517904deSPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20517904deSPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
21517904deSPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22517904deSPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23517904deSPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24517904deSPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25517904deSPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26517904deSPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27517904deSPeter Grehan * SUCH DAMAGE.
28517904deSPeter Grehan */
29517904deSPeter Grehan
30517904deSPeter Grehan #include <sys/cdefs.h>
31517904deSPeter Grehan #include "if_igc.h"
32517904deSPeter Grehan
33517904deSPeter Grehan #ifdef RSS
34517904deSPeter Grehan #include <net/rss_config.h>
35517904deSPeter Grehan #include <netinet/in_rss.h>
36517904deSPeter Grehan #endif
37517904deSPeter Grehan
38517904deSPeter Grehan #ifdef VERBOSE_DEBUG
39517904deSPeter Grehan #define DPRINTF device_printf
40517904deSPeter Grehan #else
41517904deSPeter Grehan #define DPRINTF(...)
42517904deSPeter Grehan #endif
43517904deSPeter Grehan
44517904deSPeter Grehan /*********************************************************************
45517904deSPeter Grehan * Local Function prototypes
46517904deSPeter Grehan *********************************************************************/
47517904deSPeter Grehan static int igc_isc_txd_encap(void *arg, if_pkt_info_t pi);
48517904deSPeter Grehan static void igc_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx);
49517904deSPeter Grehan static int igc_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear);
50517904deSPeter Grehan
51517904deSPeter Grehan static void igc_isc_rxd_refill(void *arg, if_rxd_update_t iru);
52517904deSPeter Grehan
53d02e4363SKevin Bowling static void igc_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused,
54d02e4363SKevin Bowling qidx_t pidx);
55d02e4363SKevin Bowling static int igc_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx,
56d02e4363SKevin Bowling qidx_t budget);
57517904deSPeter Grehan
58517904deSPeter Grehan static int igc_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri);
59517904deSPeter Grehan
60d02e4363SKevin Bowling static int igc_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi,
61d02e4363SKevin Bowling uint32_t *cmd_type_len, uint32_t *olinfo_status);
62d02e4363SKevin Bowling static int igc_tso_setup(struct tx_ring *txr, if_pkt_info_t pi,
63d02e4363SKevin Bowling uint32_t *cmd_type_len, uint32_t *olinfo_status);
64517904deSPeter Grehan
65d02e4363SKevin Bowling static void igc_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype);
66d02e4363SKevin Bowling static int igc_determine_rsstype(uint16_t pkt_info);
67517904deSPeter Grehan
68517904deSPeter Grehan extern void igc_if_enable_intr(if_ctx_t ctx);
69517904deSPeter Grehan extern int igc_intr(void *arg);
70517904deSPeter Grehan
71517904deSPeter Grehan struct if_txrx igc_txrx = {
72517904deSPeter Grehan .ift_txd_encap = igc_isc_txd_encap,
73517904deSPeter Grehan .ift_txd_flush = igc_isc_txd_flush,
74517904deSPeter Grehan .ift_txd_credits_update = igc_isc_txd_credits_update,
75517904deSPeter Grehan .ift_rxd_available = igc_isc_rxd_available,
76517904deSPeter Grehan .ift_rxd_pkt_get = igc_isc_rxd_pkt_get,
77517904deSPeter Grehan .ift_rxd_refill = igc_isc_rxd_refill,
78517904deSPeter Grehan .ift_rxd_flush = igc_isc_rxd_flush,
79517904deSPeter Grehan .ift_legacy_intr = igc_intr
80517904deSPeter Grehan };
81517904deSPeter Grehan
82517904deSPeter Grehan void
igc_dump_rs(struct igc_adapter * adapter)83517904deSPeter Grehan igc_dump_rs(struct igc_adapter *adapter)
84517904deSPeter Grehan {
85517904deSPeter Grehan if_softc_ctx_t scctx = adapter->shared;
86517904deSPeter Grehan struct igc_tx_queue *que;
87517904deSPeter Grehan struct tx_ring *txr;
88517904deSPeter Grehan qidx_t i, ntxd, qid, cur;
89517904deSPeter Grehan int16_t rs_cidx;
90517904deSPeter Grehan uint8_t status;
91517904deSPeter Grehan
92517904deSPeter Grehan printf("\n");
93517904deSPeter Grehan ntxd = scctx->isc_ntxd[0];
94517904deSPeter Grehan for (qid = 0; qid < adapter->tx_num_queues; qid++) {
95517904deSPeter Grehan que = &adapter->tx_queues[qid];
96517904deSPeter Grehan txr = &que->txr;
97517904deSPeter Grehan rs_cidx = txr->tx_rs_cidx;
98517904deSPeter Grehan if (rs_cidx != txr->tx_rs_pidx) {
99517904deSPeter Grehan cur = txr->tx_rsq[rs_cidx];
100517904deSPeter Grehan status = txr->tx_base[cur].upper.fields.status;
101517904deSPeter Grehan if (!(status & IGC_TXD_STAT_DD))
102517904deSPeter Grehan printf("qid[%d]->tx_rsq[%d]: %d clear ", qid, rs_cidx, cur);
103517904deSPeter Grehan } else {
104517904deSPeter Grehan rs_cidx = (rs_cidx-1)&(ntxd-1);
105517904deSPeter Grehan cur = txr->tx_rsq[rs_cidx];
106517904deSPeter Grehan printf("qid[%d]->tx_rsq[rs_cidx-1=%d]: %d ", qid, rs_cidx, cur);
107517904deSPeter Grehan }
108517904deSPeter Grehan printf("cidx_prev=%d rs_pidx=%d ",txr->tx_cidx_processed, txr->tx_rs_pidx);
109517904deSPeter Grehan for (i = 0; i < ntxd; i++) {
110517904deSPeter Grehan if (txr->tx_base[i].upper.fields.status & IGC_TXD_STAT_DD)
111517904deSPeter Grehan printf("%d set ", i);
112517904deSPeter Grehan }
113517904deSPeter Grehan printf("\n");
114517904deSPeter Grehan }
115517904deSPeter Grehan }
116517904deSPeter Grehan
117517904deSPeter Grehan /**********************************************************************
118517904deSPeter Grehan *
119517904deSPeter Grehan * Setup work for hardware segmentation offload (TSO) on
120517904deSPeter Grehan * adapters using advanced tx descriptors
121517904deSPeter Grehan *
122517904deSPeter Grehan **********************************************************************/
123517904deSPeter Grehan static int
igc_tso_setup(struct tx_ring * txr,if_pkt_info_t pi,uint32_t * cmd_type_len,uint32_t * olinfo_status)124d02e4363SKevin Bowling igc_tso_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len,
125d02e4363SKevin Bowling uint32_t *olinfo_status)
126517904deSPeter Grehan {
127517904deSPeter Grehan struct igc_adv_tx_context_desc *TXD;
128d02e4363SKevin Bowling uint32_t type_tucmd_mlhl = 0, vlan_macip_lens = 0;
129d02e4363SKevin Bowling uint32_t mss_l4len_idx = 0;
130d02e4363SKevin Bowling uint32_t paylen;
131517904deSPeter Grehan
132517904deSPeter Grehan switch(pi->ipi_etype) {
133517904deSPeter Grehan case ETHERTYPE_IPV6:
134517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6;
135517904deSPeter Grehan break;
136517904deSPeter Grehan case ETHERTYPE_IP:
137517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4;
138517904deSPeter Grehan /* Tell transmit desc to also do IPv4 checksum. */
139517904deSPeter Grehan *olinfo_status |= IGC_TXD_POPTS_IXSM << 8;
140517904deSPeter Grehan break;
141517904deSPeter Grehan default:
142517904deSPeter Grehan panic("%s: CSUM_TSO but no supported IP version (0x%04x)",
143517904deSPeter Grehan __func__, ntohs(pi->ipi_etype));
144517904deSPeter Grehan break;
145517904deSPeter Grehan }
146517904deSPeter Grehan
147517904deSPeter Grehan TXD = (struct igc_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx];
148517904deSPeter Grehan
149517904deSPeter Grehan /* This is used in the transmit desc in encap */
150517904deSPeter Grehan paylen = pi->ipi_len - pi->ipi_ehdrlen - pi->ipi_ip_hlen - pi->ipi_tcp_hlen;
151517904deSPeter Grehan
152517904deSPeter Grehan /* VLAN MACLEN IPLEN */
153517904deSPeter Grehan if (pi->ipi_mflags & M_VLANTAG) {
154517904deSPeter Grehan vlan_macip_lens |= (pi->ipi_vtag << IGC_ADVTXD_VLAN_SHIFT);
155517904deSPeter Grehan }
156517904deSPeter Grehan
157517904deSPeter Grehan vlan_macip_lens |= pi->ipi_ehdrlen << IGC_ADVTXD_MACLEN_SHIFT;
158517904deSPeter Grehan vlan_macip_lens |= pi->ipi_ip_hlen;
159517904deSPeter Grehan TXD->vlan_macip_lens = htole32(vlan_macip_lens);
160517904deSPeter Grehan
161517904deSPeter Grehan /* ADV DTYPE TUCMD */
162517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
163517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP;
164517904deSPeter Grehan TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
165517904deSPeter Grehan
166517904deSPeter Grehan /* MSS L4LEN IDX */
167517904deSPeter Grehan mss_l4len_idx |= (pi->ipi_tso_segsz << IGC_ADVTXD_MSS_SHIFT);
168517904deSPeter Grehan mss_l4len_idx |= (pi->ipi_tcp_hlen << IGC_ADVTXD_L4LEN_SHIFT);
169517904deSPeter Grehan TXD->mss_l4len_idx = htole32(mss_l4len_idx);
170517904deSPeter Grehan
171517904deSPeter Grehan TXD->seqnum_seed = htole32(0);
172517904deSPeter Grehan *cmd_type_len |= IGC_ADVTXD_DCMD_TSE;
173517904deSPeter Grehan *olinfo_status |= IGC_TXD_POPTS_TXSM << 8;
174517904deSPeter Grehan *olinfo_status |= paylen << IGC_ADVTXD_PAYLEN_SHIFT;
175517904deSPeter Grehan
176517904deSPeter Grehan return (1);
177517904deSPeter Grehan }
178517904deSPeter Grehan
179517904deSPeter Grehan /*********************************************************************
180517904deSPeter Grehan *
181517904deSPeter Grehan * Advanced Context Descriptor setup for VLAN, CSUM or TSO
182517904deSPeter Grehan *
183517904deSPeter Grehan **********************************************************************/
184517904deSPeter Grehan static int
igc_tx_ctx_setup(struct tx_ring * txr,if_pkt_info_t pi,uint32_t * cmd_type_len,uint32_t * olinfo_status)185d02e4363SKevin Bowling igc_tx_ctx_setup(struct tx_ring *txr, if_pkt_info_t pi, uint32_t *cmd_type_len,
186d02e4363SKevin Bowling uint32_t *olinfo_status)
187517904deSPeter Grehan {
188517904deSPeter Grehan struct igc_adv_tx_context_desc *TXD;
189d02e4363SKevin Bowling uint32_t vlan_macip_lens, type_tucmd_mlhl;
190d02e4363SKevin Bowling uint32_t mss_l4len_idx;
191517904deSPeter Grehan mss_l4len_idx = vlan_macip_lens = type_tucmd_mlhl = 0;
192517904deSPeter Grehan
193517904deSPeter Grehan /* First check if TSO is to be used */
194517904deSPeter Grehan if (pi->ipi_csum_flags & CSUM_TSO)
195517904deSPeter Grehan return (igc_tso_setup(txr, pi, cmd_type_len, olinfo_status));
196517904deSPeter Grehan
197517904deSPeter Grehan /* Indicate the whole packet as payload when not doing TSO */
198517904deSPeter Grehan *olinfo_status |= pi->ipi_len << IGC_ADVTXD_PAYLEN_SHIFT;
199517904deSPeter Grehan
200517904deSPeter Grehan /* Now ready a context descriptor */
201517904deSPeter Grehan TXD = (struct igc_adv_tx_context_desc *) &txr->tx_base[pi->ipi_pidx];
202517904deSPeter Grehan
203517904deSPeter Grehan /*
204517904deSPeter Grehan ** In advanced descriptors the vlan tag must
205517904deSPeter Grehan ** be placed into the context descriptor. Hence
206517904deSPeter Grehan ** we need to make one even if not doing offloads.
207517904deSPeter Grehan */
208517904deSPeter Grehan if (pi->ipi_mflags & M_VLANTAG) {
209517904deSPeter Grehan vlan_macip_lens |= (pi->ipi_vtag << IGC_ADVTXD_VLAN_SHIFT);
210517904deSPeter Grehan } else if ((pi->ipi_csum_flags & IGC_CSUM_OFFLOAD) == 0) {
211517904deSPeter Grehan return (0);
212517904deSPeter Grehan }
213517904deSPeter Grehan
214517904deSPeter Grehan /* Set the ether header length */
215517904deSPeter Grehan vlan_macip_lens |= pi->ipi_ehdrlen << IGC_ADVTXD_MACLEN_SHIFT;
216517904deSPeter Grehan
217517904deSPeter Grehan switch(pi->ipi_etype) {
218517904deSPeter Grehan case ETHERTYPE_IP:
219517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV4;
220517904deSPeter Grehan break;
221517904deSPeter Grehan case ETHERTYPE_IPV6:
222517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_IPV6;
223517904deSPeter Grehan break;
224517904deSPeter Grehan default:
225517904deSPeter Grehan break;
226517904deSPeter Grehan }
227517904deSPeter Grehan
228517904deSPeter Grehan vlan_macip_lens |= pi->ipi_ip_hlen;
229517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_DCMD_DEXT | IGC_ADVTXD_DTYP_CTXT;
230517904deSPeter Grehan
231517904deSPeter Grehan switch (pi->ipi_ipproto) {
232517904deSPeter Grehan case IPPROTO_TCP:
233517904deSPeter Grehan if (pi->ipi_csum_flags & (CSUM_IP_TCP | CSUM_IP6_TCP)) {
234517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_TCP;
235517904deSPeter Grehan *olinfo_status |= IGC_TXD_POPTS_TXSM << 8;
236517904deSPeter Grehan }
237517904deSPeter Grehan break;
238517904deSPeter Grehan case IPPROTO_UDP:
239517904deSPeter Grehan if (pi->ipi_csum_flags & (CSUM_IP_UDP | CSUM_IP6_UDP)) {
240517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_UDP;
241517904deSPeter Grehan *olinfo_status |= IGC_TXD_POPTS_TXSM << 8;
242517904deSPeter Grehan }
243517904deSPeter Grehan break;
244517904deSPeter Grehan case IPPROTO_SCTP:
245517904deSPeter Grehan if (pi->ipi_csum_flags & (CSUM_IP_SCTP | CSUM_IP6_SCTP)) {
246517904deSPeter Grehan type_tucmd_mlhl |= IGC_ADVTXD_TUCMD_L4T_SCTP;
247517904deSPeter Grehan *olinfo_status |= IGC_TXD_POPTS_TXSM << 8;
248517904deSPeter Grehan }
249517904deSPeter Grehan break;
250517904deSPeter Grehan default:
251517904deSPeter Grehan break;
252517904deSPeter Grehan }
253517904deSPeter Grehan
254517904deSPeter Grehan /* Now copy bits into descriptor */
255517904deSPeter Grehan TXD->vlan_macip_lens = htole32(vlan_macip_lens);
256517904deSPeter Grehan TXD->type_tucmd_mlhl = htole32(type_tucmd_mlhl);
257517904deSPeter Grehan TXD->seqnum_seed = htole32(0);
258517904deSPeter Grehan TXD->mss_l4len_idx = htole32(mss_l4len_idx);
259517904deSPeter Grehan
260517904deSPeter Grehan return (1);
261517904deSPeter Grehan }
262517904deSPeter Grehan
263517904deSPeter Grehan static int
igc_isc_txd_encap(void * arg,if_pkt_info_t pi)264517904deSPeter Grehan igc_isc_txd_encap(void *arg, if_pkt_info_t pi)
265517904deSPeter Grehan {
266517904deSPeter Grehan struct igc_adapter *sc = arg;
267517904deSPeter Grehan if_softc_ctx_t scctx = sc->shared;
268517904deSPeter Grehan struct igc_tx_queue *que = &sc->tx_queues[pi->ipi_qsidx];
269517904deSPeter Grehan struct tx_ring *txr = &que->txr;
270517904deSPeter Grehan int nsegs = pi->ipi_nsegs;
271517904deSPeter Grehan bus_dma_segment_t *segs = pi->ipi_segs;
272517904deSPeter Grehan union igc_adv_tx_desc *txd = NULL;
273517904deSPeter Grehan int i, j, pidx_last;
274d02e4363SKevin Bowling uint32_t olinfo_status, cmd_type_len, txd_flags;
275517904deSPeter Grehan qidx_t ntxd;
276517904deSPeter Grehan
277517904deSPeter Grehan pidx_last = olinfo_status = 0;
278517904deSPeter Grehan /* Basic descriptor defines */
279517904deSPeter Grehan cmd_type_len = (IGC_ADVTXD_DTYP_DATA |
280517904deSPeter Grehan IGC_ADVTXD_DCMD_IFCS | IGC_ADVTXD_DCMD_DEXT);
281517904deSPeter Grehan
282517904deSPeter Grehan if (pi->ipi_mflags & M_VLANTAG)
283517904deSPeter Grehan cmd_type_len |= IGC_ADVTXD_DCMD_VLE;
284517904deSPeter Grehan
285517904deSPeter Grehan i = pi->ipi_pidx;
286517904deSPeter Grehan ntxd = scctx->isc_ntxd[0];
287517904deSPeter Grehan txd_flags = pi->ipi_flags & IPI_TX_INTR ? IGC_ADVTXD_DCMD_RS : 0;
288517904deSPeter Grehan /* Consume the first descriptor */
289517904deSPeter Grehan i += igc_tx_ctx_setup(txr, pi, &cmd_type_len, &olinfo_status);
290517904deSPeter Grehan if (i == scctx->isc_ntxd[0])
291517904deSPeter Grehan i = 0;
292517904deSPeter Grehan
293517904deSPeter Grehan for (j = 0; j < nsegs; j++) {
294517904deSPeter Grehan bus_size_t seglen;
295517904deSPeter Grehan bus_addr_t segaddr;
296517904deSPeter Grehan
297517904deSPeter Grehan txd = (union igc_adv_tx_desc *)&txr->tx_base[i];
298517904deSPeter Grehan seglen = segs[j].ds_len;
299517904deSPeter Grehan segaddr = htole64(segs[j].ds_addr);
300517904deSPeter Grehan
301517904deSPeter Grehan txd->read.buffer_addr = segaddr;
302517904deSPeter Grehan txd->read.cmd_type_len = htole32(IGC_ADVTXD_DCMD_IFCS |
303517904deSPeter Grehan cmd_type_len | seglen);
304517904deSPeter Grehan txd->read.olinfo_status = htole32(olinfo_status);
305517904deSPeter Grehan pidx_last = i;
306517904deSPeter Grehan if (++i == scctx->isc_ntxd[0]) {
307517904deSPeter Grehan i = 0;
308517904deSPeter Grehan }
309517904deSPeter Grehan }
310517904deSPeter Grehan if (txd_flags) {
311517904deSPeter Grehan txr->tx_rsq[txr->tx_rs_pidx] = pidx_last;
312517904deSPeter Grehan txr->tx_rs_pidx = (txr->tx_rs_pidx+1) & (ntxd-1);
313517904deSPeter Grehan MPASS(txr->tx_rs_pidx != txr->tx_rs_cidx);
314517904deSPeter Grehan }
315517904deSPeter Grehan
316517904deSPeter Grehan txd->read.cmd_type_len |= htole32(IGC_ADVTXD_DCMD_EOP | txd_flags);
317517904deSPeter Grehan pi->ipi_new_pidx = i;
318517904deSPeter Grehan
319517904deSPeter Grehan return (0);
320517904deSPeter Grehan }
321517904deSPeter Grehan
322517904deSPeter Grehan static void
igc_isc_txd_flush(void * arg,uint16_t txqid,qidx_t pidx)323517904deSPeter Grehan igc_isc_txd_flush(void *arg, uint16_t txqid, qidx_t pidx)
324517904deSPeter Grehan {
325517904deSPeter Grehan struct igc_adapter *adapter = arg;
326517904deSPeter Grehan struct igc_tx_queue *que = &adapter->tx_queues[txqid];
327517904deSPeter Grehan struct tx_ring *txr = &que->txr;
328517904deSPeter Grehan
329517904deSPeter Grehan IGC_WRITE_REG(&adapter->hw, IGC_TDT(txr->me), pidx);
330517904deSPeter Grehan }
331517904deSPeter Grehan
332517904deSPeter Grehan static int
igc_isc_txd_credits_update(void * arg,uint16_t txqid,bool clear)333517904deSPeter Grehan igc_isc_txd_credits_update(void *arg, uint16_t txqid, bool clear)
334517904deSPeter Grehan {
335517904deSPeter Grehan struct igc_adapter *adapter = arg;
336517904deSPeter Grehan if_softc_ctx_t scctx = adapter->shared;
337517904deSPeter Grehan struct igc_tx_queue *que = &adapter->tx_queues[txqid];
338517904deSPeter Grehan struct tx_ring *txr = &que->txr;
339517904deSPeter Grehan
340517904deSPeter Grehan qidx_t processed = 0;
341517904deSPeter Grehan int updated;
342517904deSPeter Grehan qidx_t cur, prev, ntxd, rs_cidx;
343517904deSPeter Grehan int32_t delta;
344517904deSPeter Grehan uint8_t status;
345517904deSPeter Grehan
346517904deSPeter Grehan rs_cidx = txr->tx_rs_cidx;
347517904deSPeter Grehan if (rs_cidx == txr->tx_rs_pidx)
348517904deSPeter Grehan return (0);
349517904deSPeter Grehan cur = txr->tx_rsq[rs_cidx];
350517904deSPeter Grehan status = ((union igc_adv_tx_desc *)&txr->tx_base[cur])->wb.status;
351517904deSPeter Grehan updated = !!(status & IGC_TXD_STAT_DD);
352517904deSPeter Grehan
353517904deSPeter Grehan if (!updated)
354517904deSPeter Grehan return (0);
355517904deSPeter Grehan
356517904deSPeter Grehan /* If clear is false just let caller know that there
357517904deSPeter Grehan * are descriptors to reclaim */
358517904deSPeter Grehan if (!clear)
359517904deSPeter Grehan return (1);
360517904deSPeter Grehan
361517904deSPeter Grehan prev = txr->tx_cidx_processed;
362517904deSPeter Grehan ntxd = scctx->isc_ntxd[0];
363517904deSPeter Grehan do {
364517904deSPeter Grehan MPASS(prev != cur);
365517904deSPeter Grehan delta = (int32_t)cur - (int32_t)prev;
366517904deSPeter Grehan if (delta < 0)
367517904deSPeter Grehan delta += ntxd;
368517904deSPeter Grehan MPASS(delta > 0);
369517904deSPeter Grehan
370517904deSPeter Grehan processed += delta;
371517904deSPeter Grehan prev = cur;
372517904deSPeter Grehan rs_cidx = (rs_cidx + 1) & (ntxd-1);
373517904deSPeter Grehan if (rs_cidx == txr->tx_rs_pidx)
374517904deSPeter Grehan break;
375517904deSPeter Grehan cur = txr->tx_rsq[rs_cidx];
376517904deSPeter Grehan status = ((union igc_adv_tx_desc *)&txr->tx_base[cur])->wb.status;
377517904deSPeter Grehan } while ((status & IGC_TXD_STAT_DD));
378517904deSPeter Grehan
379517904deSPeter Grehan txr->tx_rs_cidx = rs_cidx;
380517904deSPeter Grehan txr->tx_cidx_processed = prev;
381517904deSPeter Grehan return (processed);
382517904deSPeter Grehan }
383517904deSPeter Grehan
384517904deSPeter Grehan static void
igc_isc_rxd_refill(void * arg,if_rxd_update_t iru)385517904deSPeter Grehan igc_isc_rxd_refill(void *arg, if_rxd_update_t iru)
386517904deSPeter Grehan {
387517904deSPeter Grehan struct igc_adapter *sc = arg;
388517904deSPeter Grehan if_softc_ctx_t scctx = sc->shared;
389517904deSPeter Grehan uint16_t rxqid = iru->iru_qsidx;
390517904deSPeter Grehan struct igc_rx_queue *que = &sc->rx_queues[rxqid];
391517904deSPeter Grehan union igc_adv_rx_desc *rxd;
392517904deSPeter Grehan struct rx_ring *rxr = &que->rxr;
393517904deSPeter Grehan uint64_t *paddrs;
394517904deSPeter Grehan uint32_t next_pidx, pidx;
395517904deSPeter Grehan uint16_t count;
396517904deSPeter Grehan int i;
397517904deSPeter Grehan
398517904deSPeter Grehan paddrs = iru->iru_paddrs;
399517904deSPeter Grehan pidx = iru->iru_pidx;
400517904deSPeter Grehan count = iru->iru_count;
401517904deSPeter Grehan
402517904deSPeter Grehan for (i = 0, next_pidx = pidx; i < count; i++) {
403517904deSPeter Grehan rxd = (union igc_adv_rx_desc *)&rxr->rx_base[next_pidx];
404517904deSPeter Grehan
405517904deSPeter Grehan rxd->read.pkt_addr = htole64(paddrs[i]);
406517904deSPeter Grehan if (++next_pidx == scctx->isc_nrxd[0])
407517904deSPeter Grehan next_pidx = 0;
408517904deSPeter Grehan }
409517904deSPeter Grehan }
410517904deSPeter Grehan
411517904deSPeter Grehan static void
igc_isc_rxd_flush(void * arg,uint16_t rxqid,uint8_t flid __unused,qidx_t pidx)412517904deSPeter Grehan igc_isc_rxd_flush(void *arg, uint16_t rxqid, uint8_t flid __unused, qidx_t pidx)
413517904deSPeter Grehan {
414517904deSPeter Grehan struct igc_adapter *sc = arg;
415517904deSPeter Grehan struct igc_rx_queue *que = &sc->rx_queues[rxqid];
416517904deSPeter Grehan struct rx_ring *rxr = &que->rxr;
417517904deSPeter Grehan
418517904deSPeter Grehan IGC_WRITE_REG(&sc->hw, IGC_RDT(rxr->me), pidx);
419517904deSPeter Grehan }
420517904deSPeter Grehan
421517904deSPeter Grehan static int
igc_isc_rxd_available(void * arg,uint16_t rxqid,qidx_t idx,qidx_t budget)422517904deSPeter Grehan igc_isc_rxd_available(void *arg, uint16_t rxqid, qidx_t idx, qidx_t budget)
423517904deSPeter Grehan {
424517904deSPeter Grehan struct igc_adapter *sc = arg;
425517904deSPeter Grehan if_softc_ctx_t scctx = sc->shared;
426517904deSPeter Grehan struct igc_rx_queue *que = &sc->rx_queues[rxqid];
427517904deSPeter Grehan struct rx_ring *rxr = &que->rxr;
428517904deSPeter Grehan union igc_adv_rx_desc *rxd;
429d02e4363SKevin Bowling uint32_t staterr = 0;
430517904deSPeter Grehan int cnt, i;
431517904deSPeter Grehan
432517904deSPeter Grehan for (cnt = 0, i = idx; cnt < scctx->isc_nrxd[0] && cnt <= budget;) {
433517904deSPeter Grehan rxd = (union igc_adv_rx_desc *)&rxr->rx_base[i];
434517904deSPeter Grehan staterr = le32toh(rxd->wb.upper.status_error);
435517904deSPeter Grehan
436517904deSPeter Grehan if ((staterr & IGC_RXD_STAT_DD) == 0)
437517904deSPeter Grehan break;
438517904deSPeter Grehan if (++i == scctx->isc_nrxd[0])
439517904deSPeter Grehan i = 0;
440517904deSPeter Grehan if (staterr & IGC_RXD_STAT_EOP)
441517904deSPeter Grehan cnt++;
442517904deSPeter Grehan }
443517904deSPeter Grehan return (cnt);
444517904deSPeter Grehan }
445517904deSPeter Grehan
446517904deSPeter Grehan /****************************************************************
447517904deSPeter Grehan * Routine sends data which has been dma'ed into host memory
448517904deSPeter Grehan * to upper layer. Initialize ri structure.
449517904deSPeter Grehan *
450517904deSPeter Grehan * Returns 0 upon success, errno on failure
451517904deSPeter Grehan ***************************************************************/
452517904deSPeter Grehan
453517904deSPeter Grehan static int
igc_isc_rxd_pkt_get(void * arg,if_rxd_info_t ri)454517904deSPeter Grehan igc_isc_rxd_pkt_get(void *arg, if_rxd_info_t ri)
455517904deSPeter Grehan {
456517904deSPeter Grehan struct igc_adapter *adapter = arg;
457517904deSPeter Grehan if_softc_ctx_t scctx = adapter->shared;
458517904deSPeter Grehan struct igc_rx_queue *que = &adapter->rx_queues[ri->iri_qsidx];
459517904deSPeter Grehan struct rx_ring *rxr = &que->rxr;
460517904deSPeter Grehan union igc_adv_rx_desc *rxd;
461517904deSPeter Grehan
462f7926a6dSVincenzo Maffione uint16_t pkt_info, len;
463d02e4363SKevin Bowling uint32_t ptype, staterr;
464d02e4363SKevin Bowling int i, cidx;
465517904deSPeter Grehan bool eop;
466d02e4363SKevin Bowling
467f7926a6dSVincenzo Maffione staterr = i = 0;
468d02e4363SKevin Bowling cidx = ri->iri_cidx;
469517904deSPeter Grehan
470517904deSPeter Grehan do {
471517904deSPeter Grehan rxd = (union igc_adv_rx_desc *)&rxr->rx_base[cidx];
472517904deSPeter Grehan staterr = le32toh(rxd->wb.upper.status_error);
473517904deSPeter Grehan pkt_info = le16toh(rxd->wb.lower.lo_dword.hs_rss.pkt_info);
474517904deSPeter Grehan
475517904deSPeter Grehan MPASS ((staterr & IGC_RXD_STAT_DD) != 0);
476517904deSPeter Grehan
477517904deSPeter Grehan len = le16toh(rxd->wb.upper.length);
478517904deSPeter Grehan ptype = le32toh(rxd->wb.lower.lo_dword.data) & IGC_PKTTYPE_MASK;
479517904deSPeter Grehan
480517904deSPeter Grehan ri->iri_len += len;
481517904deSPeter Grehan rxr->rx_bytes += ri->iri_len;
482517904deSPeter Grehan
483517904deSPeter Grehan rxd->wb.upper.status_error = 0;
484517904deSPeter Grehan eop = ((staterr & IGC_RXD_STAT_EOP) == IGC_RXD_STAT_EOP);
485517904deSPeter Grehan
486517904deSPeter Grehan /* Make sure bad packets are discarded */
487517904deSPeter Grehan if (eop && ((staterr & IGC_RXDEXT_STATERR_RXE) != 0)) {
488517904deSPeter Grehan adapter->dropped_pkts++;
489517904deSPeter Grehan ++rxr->rx_discarded;
490517904deSPeter Grehan return (EBADMSG);
491517904deSPeter Grehan }
492517904deSPeter Grehan ri->iri_frags[i].irf_flid = 0;
493517904deSPeter Grehan ri->iri_frags[i].irf_idx = cidx;
494517904deSPeter Grehan ri->iri_frags[i].irf_len = len;
495517904deSPeter Grehan
496517904deSPeter Grehan if (++cidx == scctx->isc_nrxd[0])
497517904deSPeter Grehan cidx = 0;
498517904deSPeter Grehan #ifdef notyet
499517904deSPeter Grehan if (rxr->hdr_split == true) {
500517904deSPeter Grehan ri->iri_frags[i].irf_flid = 1;
501517904deSPeter Grehan ri->iri_frags[i].irf_idx = cidx;
502517904deSPeter Grehan if (++cidx == scctx->isc_nrxd[0])
503517904deSPeter Grehan cidx = 0;
504517904deSPeter Grehan }
505517904deSPeter Grehan #endif
506517904deSPeter Grehan i++;
507517904deSPeter Grehan } while (!eop);
508517904deSPeter Grehan
509517904deSPeter Grehan rxr->rx_packets++;
510517904deSPeter Grehan
511d02e4363SKevin Bowling if ((scctx->isc_capenable & IFCAP_RXCSUM) != 0)
512517904deSPeter Grehan igc_rx_checksum(staterr, ri, ptype);
513517904deSPeter Grehan
514b4a58b3dSKevin Bowling if (staterr & IGC_RXD_STAT_VP) {
515f7926a6dSVincenzo Maffione ri->iri_vtag = le16toh(rxd->wb.upper.vlan);
516517904deSPeter Grehan ri->iri_flags |= M_VLANTAG;
517517904deSPeter Grehan }
518d02e4363SKevin Bowling
519517904deSPeter Grehan ri->iri_flowid =
520517904deSPeter Grehan le32toh(rxd->wb.lower.hi_dword.rss);
521517904deSPeter Grehan ri->iri_rsstype = igc_determine_rsstype(pkt_info);
522517904deSPeter Grehan ri->iri_nfrags = i;
523517904deSPeter Grehan
524517904deSPeter Grehan return (0);
525517904deSPeter Grehan }
526517904deSPeter Grehan
527517904deSPeter Grehan /*********************************************************************
528517904deSPeter Grehan *
529517904deSPeter Grehan * Verify that the hardware indicated that the checksum is valid.
530517904deSPeter Grehan * Inform the stack about the status of checksum so that stack
531517904deSPeter Grehan * doesn't spend time verifying the checksum.
532517904deSPeter Grehan *
533517904deSPeter Grehan *********************************************************************/
534517904deSPeter Grehan static void
igc_rx_checksum(uint32_t staterr,if_rxd_info_t ri,uint32_t ptype)535d02e4363SKevin Bowling igc_rx_checksum(uint32_t staterr, if_rxd_info_t ri, uint32_t ptype)
536517904deSPeter Grehan {
537d02e4363SKevin Bowling uint16_t status = (uint16_t)staterr;
538d02e4363SKevin Bowling uint8_t errors = (uint8_t)(staterr >> 24);
539517904deSPeter Grehan
540d02e4363SKevin Bowling if (__predict_false(status & IGC_RXD_STAT_IXSM))
541517904deSPeter Grehan return;
542517904deSPeter Grehan
543d02e4363SKevin Bowling /* If there is a layer 3 or 4 error we are done */
544d02e4363SKevin Bowling if (__predict_false(errors & (IGC_RXD_ERR_IPE | IGC_RXD_ERR_TCPE)))
545d02e4363SKevin Bowling return;
546d02e4363SKevin Bowling
547d02e4363SKevin Bowling /* IP Checksum Good */
548d02e4363SKevin Bowling if (status & IGC_RXD_STAT_IPCS)
549d02e4363SKevin Bowling ri->iri_csum_flags = (CSUM_IP_CHECKED | CSUM_IP_VALID);
550d02e4363SKevin Bowling
551d02e4363SKevin Bowling /* Valid L4E checksum */
552d02e4363SKevin Bowling if (__predict_true(status &
553d02e4363SKevin Bowling (IGC_RXD_STAT_TCPCS | IGC_RXD_STAT_UDPCS))) {
554d02e4363SKevin Bowling /* SCTP header present */
555d02e4363SKevin Bowling if (__predict_false((ptype & IGC_RXDADV_PKTTYPE_ETQF) == 0 &&
556d02e4363SKevin Bowling (ptype & IGC_RXDADV_PKTTYPE_SCTP) != 0)) {
557d02e4363SKevin Bowling ri->iri_csum_flags |= CSUM_SCTP_VALID;
558d02e4363SKevin Bowling } else {
559d02e4363SKevin Bowling ri->iri_csum_flags |= CSUM_DATA_VALID | CSUM_PSEUDO_HDR;
560517904deSPeter Grehan ri->iri_csum_data = htons(0xffff);
561517904deSPeter Grehan }
562517904deSPeter Grehan }
563517904deSPeter Grehan }
564517904deSPeter Grehan
565517904deSPeter Grehan /********************************************************************
566517904deSPeter Grehan *
567517904deSPeter Grehan * Parse the packet type to determine the appropriate hash
568517904deSPeter Grehan *
569517904deSPeter Grehan ******************************************************************/
570517904deSPeter Grehan static int
igc_determine_rsstype(uint16_t pkt_info)571d02e4363SKevin Bowling igc_determine_rsstype(uint16_t pkt_info)
572517904deSPeter Grehan {
573517904deSPeter Grehan switch (pkt_info & IGC_RXDADV_RSSTYPE_MASK) {
574517904deSPeter Grehan case IGC_RXDADV_RSSTYPE_IPV4_TCP:
575517904deSPeter Grehan return M_HASHTYPE_RSS_TCP_IPV4;
576517904deSPeter Grehan case IGC_RXDADV_RSSTYPE_IPV4:
577517904deSPeter Grehan return M_HASHTYPE_RSS_IPV4;
578517904deSPeter Grehan case IGC_RXDADV_RSSTYPE_IPV6_TCP:
579517904deSPeter Grehan return M_HASHTYPE_RSS_TCP_IPV6;
580517904deSPeter Grehan case IGC_RXDADV_RSSTYPE_IPV6_EX:
581517904deSPeter Grehan return M_HASHTYPE_RSS_IPV6_EX;
582517904deSPeter Grehan case IGC_RXDADV_RSSTYPE_IPV6:
583517904deSPeter Grehan return M_HASHTYPE_RSS_IPV6;
584517904deSPeter Grehan case IGC_RXDADV_RSSTYPE_IPV6_TCP_EX:
585517904deSPeter Grehan return M_HASHTYPE_RSS_TCP_IPV6_EX;
586517904deSPeter Grehan default:
587517904deSPeter Grehan return M_HASHTYPE_OPAQUE;
588517904deSPeter Grehan }
589517904deSPeter Grehan }
590