1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (C) 2013 Emulex
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are met:
9 *
10 * 1. Redistributions of source code must retain the above copyright notice,
11 * this list of conditions and the following disclaimer.
12 *
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 *
17 * 3. Neither the name of the Emulex Corporation nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
22 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
24 * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
25 * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Contact Information:
34 * freebsd-drivers@emulex.com
35 *
36 * Emulex
37 * 3333 Susan Street
38 * Costa Mesa, CA 92626
39 */
40
41
42 #include "opt_inet6.h"
43 #include "opt_inet.h"
44
45 #include "oce_if.h"
46 #include "oce_user.h"
47
48 #define is_tso_pkt(m) (m->m_pkthdr.csum_flags & CSUM_TSO)
49
50 /* UE Status Low CSR */
51 static char *ue_status_low_desc[] = {
52 "CEV",
53 "CTX",
54 "DBUF",
55 "ERX",
56 "Host",
57 "MPU",
58 "NDMA",
59 "PTC ",
60 "RDMA ",
61 "RXF ",
62 "RXIPS ",
63 "RXULP0 ",
64 "RXULP1 ",
65 "RXULP2 ",
66 "TIM ",
67 "TPOST ",
68 "TPRE ",
69 "TXIPS ",
70 "TXULP0 ",
71 "TXULP1 ",
72 "UC ",
73 "WDMA ",
74 "TXULP2 ",
75 "HOST1 ",
76 "P0_OB_LINK ",
77 "P1_OB_LINK ",
78 "HOST_GPIO ",
79 "MBOX ",
80 "AXGMAC0",
81 "AXGMAC1",
82 "JTAG",
83 "MPU_INTPEND"
84 };
85
86 /* UE Status High CSR */
87 static char *ue_status_hi_desc[] = {
88 "LPCMEMHOST",
89 "MGMT_MAC",
90 "PCS0ONLINE",
91 "MPU_IRAM",
92 "PCS1ONLINE",
93 "PCTL0",
94 "PCTL1",
95 "PMEM",
96 "RR",
97 "TXPB",
98 "RXPP",
99 "XAUI",
100 "TXP",
101 "ARM",
102 "IPC",
103 "HOST2",
104 "HOST3",
105 "HOST4",
106 "HOST5",
107 "HOST6",
108 "HOST7",
109 "HOST8",
110 "HOST9",
111 "NETC",
112 "Unknown",
113 "Unknown",
114 "Unknown",
115 "Unknown",
116 "Unknown",
117 "Unknown",
118 "Unknown",
119 "Unknown"
120 };
121
122 struct oce_common_cqe_info{
123 uint8_t vtp:1;
124 uint8_t l4_cksum_pass:1;
125 uint8_t ip_cksum_pass:1;
126 uint8_t ipv6_frame:1;
127 uint8_t qnq:1;
128 uint8_t rsvd:3;
129 uint8_t num_frags;
130 uint16_t pkt_size;
131 uint16_t vtag;
132 };
133
134 /* Driver entry points prototypes */
135 static int oce_probe(device_t dev);
136 static int oce_attach(device_t dev);
137 static int oce_detach(device_t dev);
138 static int oce_shutdown(device_t dev);
139 static int oce_ioctl(if_t ifp, u_long command, caddr_t data);
140 static void oce_init(void *xsc);
141 static int oce_multiq_start(if_t ifp, struct mbuf *m);
142 static void oce_multiq_flush(if_t ifp);
143
144 /* Driver interrupt routines protypes */
145 static void oce_intr(void *arg, int pending);
146 static int oce_setup_intr(POCE_SOFTC sc);
147 static int oce_fast_isr(void *arg);
148 static int oce_alloc_intr(POCE_SOFTC sc, int vector,
149 void (*isr) (void *arg, int pending));
150
151 /* Media callbacks prototypes */
152 static void oce_media_status(if_t ifp, struct ifmediareq *req);
153 static int oce_media_change(if_t ifp);
154
155 /* Transmit routines prototypes */
156 static int oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index);
157 static void oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq);
158 static void oce_process_tx_completion(struct oce_wq *wq);
159 static int oce_multiq_transmit(if_t ifp, struct mbuf *m,
160 struct oce_wq *wq);
161
162 /* Receive routines prototypes */
163 static int oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
164 static int oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe);
165 static void oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe);
166 static void oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq);
167 static uint16_t oce_rq_handler_lro(void *arg);
168 static void oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2);
169 static void oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2);
170 static void oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m);
171
172 /* Helper function prototypes in this file */
173 static int oce_attach_ifp(POCE_SOFTC sc);
174 static void oce_add_vlan(void *arg, if_t ifp, uint16_t vtag);
175 static void oce_del_vlan(void *arg, if_t ifp, uint16_t vtag);
176 static int oce_vid_config(POCE_SOFTC sc);
177 static void oce_mac_addr_set(POCE_SOFTC sc);
178 static int oce_handle_passthrough(if_t ifp, caddr_t data);
179 static void oce_local_timer(void *arg);
180 static void oce_if_deactivate(POCE_SOFTC sc);
181 static void oce_if_activate(POCE_SOFTC sc);
182 static void setup_max_queues_want(POCE_SOFTC sc);
183 static void update_queues_got(POCE_SOFTC sc);
184 static void process_link_state(POCE_SOFTC sc,
185 struct oce_async_cqe_link_state *acqe);
186 static int oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m);
187 static void oce_get_config(POCE_SOFTC sc);
188 static struct mbuf *oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete);
189 static void oce_read_env_variables(POCE_SOFTC sc);
190
191 /* IP specific */
192 #if defined(INET6) || defined(INET)
193 static int oce_init_lro(POCE_SOFTC sc);
194 static struct mbuf * oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp);
195 #endif
196
197 static device_method_t oce_dispatch[] = {
198 DEVMETHOD(device_probe, oce_probe),
199 DEVMETHOD(device_attach, oce_attach),
200 DEVMETHOD(device_detach, oce_detach),
201 DEVMETHOD(device_shutdown, oce_shutdown),
202
203 DEVMETHOD_END
204 };
205
206 static driver_t oce_driver = {
207 "oce",
208 oce_dispatch,
209 sizeof(OCE_SOFTC)
210 };
211
212 /* global vars */
213 const char component_revision[32] = {"///" COMPONENT_REVISION "///"};
214
215 /* Module capabilites and parameters */
216 uint32_t oce_max_rsp_handled = OCE_MAX_RSP_HANDLED;
217 uint32_t oce_enable_rss = OCE_MODCAP_RSS;
218 uint32_t oce_rq_buf_size = 2048;
219
220 TUNABLE_INT("hw.oce.max_rsp_handled", &oce_max_rsp_handled);
221 TUNABLE_INT("hw.oce.enable_rss", &oce_enable_rss);
222
223 /* Supported devices table */
224 static uint32_t supportedDevices[] = {
225 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE2,
226 (PCI_VENDOR_SERVERENGINES << 16) | PCI_PRODUCT_BE3,
227 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_BE3,
228 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201,
229 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_XE201_VF,
230 (PCI_VENDOR_EMULEX << 16) | PCI_PRODUCT_SH
231 };
232
233 DRIVER_MODULE(oce, pci, oce_driver, 0, 0);
234 MODULE_PNP_INFO("W32:vendor/device", pci, oce, supportedDevices,
235 nitems(supportedDevices));
236 MODULE_DEPEND(oce, pci, 1, 1, 1);
237 MODULE_DEPEND(oce, ether, 1, 1, 1);
238 MODULE_VERSION(oce, 1);
239
240 POCE_SOFTC softc_head = NULL;
241 POCE_SOFTC softc_tail = NULL;
242
243 struct oce_rdma_if *oce_rdma_if = NULL;
244
245 /*****************************************************************************
246 * Driver entry points functions *
247 *****************************************************************************/
248
249 static int
oce_probe(device_t dev)250 oce_probe(device_t dev)
251 {
252 uint16_t vendor = 0;
253 uint16_t device = 0;
254 int i = 0;
255 POCE_SOFTC sc;
256
257 sc = device_get_softc(dev);
258 bzero(sc, sizeof(OCE_SOFTC));
259 sc->dev = dev;
260
261 vendor = pci_get_vendor(dev);
262 device = pci_get_device(dev);
263
264 for (i = 0; i < (sizeof(supportedDevices) / sizeof(uint32_t)); i++) {
265 if (vendor == ((supportedDevices[i] >> 16) & 0xffff)) {
266 if (device == (supportedDevices[i] & 0xffff)) {
267 device_set_descf(dev,
268 "%s:%s", "Emulex CNA NIC function",
269 component_revision);
270
271 switch (device) {
272 case PCI_PRODUCT_BE2:
273 sc->flags |= OCE_FLAGS_BE2;
274 break;
275 case PCI_PRODUCT_BE3:
276 sc->flags |= OCE_FLAGS_BE3;
277 break;
278 case PCI_PRODUCT_XE201:
279 case PCI_PRODUCT_XE201_VF:
280 sc->flags |= OCE_FLAGS_XE201;
281 break;
282 case PCI_PRODUCT_SH:
283 sc->flags |= OCE_FLAGS_SH;
284 break;
285 default:
286 return ENXIO;
287 }
288 return BUS_PROBE_DEFAULT;
289 }
290 }
291 }
292
293 return ENXIO;
294 }
295
296 static int
oce_attach(device_t dev)297 oce_attach(device_t dev)
298 {
299 POCE_SOFTC sc;
300 int rc = 0;
301
302 sc = device_get_softc(dev);
303
304 rc = oce_hw_pci_alloc(sc);
305 if (rc)
306 return rc;
307
308 sc->tx_ring_size = OCE_TX_RING_SIZE;
309 sc->rx_ring_size = OCE_RX_RING_SIZE;
310 /* receive fragment size should be multiple of 2K */
311 sc->rq_frag_size = ((oce_rq_buf_size / 2048) * 2048);
312 sc->flow_control = OCE_DEFAULT_FLOW_CONTROL;
313 sc->promisc = OCE_DEFAULT_PROMISCUOUS;
314
315 LOCK_CREATE(&sc->bmbx_lock, "Mailbox_lock");
316 LOCK_CREATE(&sc->dev_lock, "Device_lock");
317
318 /* initialise the hardware */
319 rc = oce_hw_init(sc);
320 if (rc)
321 goto pci_res_free;
322
323 oce_read_env_variables(sc);
324
325 oce_get_config(sc);
326
327 setup_max_queues_want(sc);
328
329 rc = oce_setup_intr(sc);
330 if (rc)
331 goto mbox_free;
332
333 rc = oce_queue_init_all(sc);
334 if (rc)
335 goto intr_free;
336
337 rc = oce_attach_ifp(sc);
338 if (rc)
339 goto queues_free;
340
341 #if defined(INET6) || defined(INET)
342 rc = oce_init_lro(sc);
343 if (rc)
344 goto ifp_free;
345 #endif
346
347 rc = oce_hw_start(sc);
348 if (rc)
349 goto lro_free;
350
351 sc->vlan_attach = EVENTHANDLER_REGISTER(vlan_config,
352 oce_add_vlan, sc, EVENTHANDLER_PRI_FIRST);
353 sc->vlan_detach = EVENTHANDLER_REGISTER(vlan_unconfig,
354 oce_del_vlan, sc, EVENTHANDLER_PRI_FIRST);
355
356 rc = oce_stats_init(sc);
357 if (rc)
358 goto vlan_free;
359
360 oce_add_sysctls(sc);
361
362 callout_init(&sc->timer, CALLOUT_MPSAFE);
363 rc = callout_reset(&sc->timer, 2 * hz, oce_local_timer, sc);
364 if (rc)
365 goto stats_free;
366
367 sc->next =NULL;
368 if (softc_tail != NULL) {
369 softc_tail->next = sc;
370 } else {
371 softc_head = sc;
372 }
373 softc_tail = sc;
374
375 gone_in_dev(dev, 15, "relatively uncommon 10GbE NIC");
376
377 return 0;
378
379 stats_free:
380 callout_drain(&sc->timer);
381 oce_stats_free(sc);
382 vlan_free:
383 if (sc->vlan_attach)
384 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
385 if (sc->vlan_detach)
386 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
387 oce_hw_intr_disable(sc);
388 lro_free:
389 #if defined(INET6) || defined(INET)
390 oce_free_lro(sc);
391 ifp_free:
392 #endif
393 ether_ifdetach(sc->ifp);
394 if_free(sc->ifp);
395 queues_free:
396 oce_queue_release_all(sc);
397 intr_free:
398 oce_intr_free(sc);
399 mbox_free:
400 oce_dma_free(sc, &sc->bsmbx);
401 pci_res_free:
402 oce_hw_pci_free(sc);
403 LOCK_DESTROY(&sc->dev_lock);
404 LOCK_DESTROY(&sc->bmbx_lock);
405 return rc;
406
407 }
408
409 static int
oce_detach(device_t dev)410 oce_detach(device_t dev)
411 {
412 POCE_SOFTC sc = device_get_softc(dev);
413 POCE_SOFTC poce_sc_tmp, *ppoce_sc_tmp1, poce_sc_tmp2 = NULL;
414
415 poce_sc_tmp = softc_head;
416 ppoce_sc_tmp1 = &softc_head;
417 while (poce_sc_tmp != NULL) {
418 if (poce_sc_tmp == sc) {
419 *ppoce_sc_tmp1 = sc->next;
420 if (sc->next == NULL) {
421 softc_tail = poce_sc_tmp2;
422 }
423 break;
424 }
425 poce_sc_tmp2 = poce_sc_tmp;
426 ppoce_sc_tmp1 = &poce_sc_tmp->next;
427 poce_sc_tmp = poce_sc_tmp->next;
428 }
429
430 LOCK(&sc->dev_lock);
431 oce_if_deactivate(sc);
432 UNLOCK(&sc->dev_lock);
433
434 callout_drain(&sc->timer);
435
436 if (sc->vlan_attach != NULL)
437 EVENTHANDLER_DEREGISTER(vlan_config, sc->vlan_attach);
438 if (sc->vlan_detach != NULL)
439 EVENTHANDLER_DEREGISTER(vlan_unconfig, sc->vlan_detach);
440
441 ether_ifdetach(sc->ifp);
442
443 if_free(sc->ifp);
444
445 oce_hw_shutdown(sc);
446
447 bus_generic_detach(dev);
448
449 return 0;
450 }
451
452 static int
oce_shutdown(device_t dev)453 oce_shutdown(device_t dev)
454 {
455 int rc;
456
457 rc = oce_detach(dev);
458
459 return rc;
460 }
461
462 static int
oce_ioctl(if_t ifp,u_long command,caddr_t data)463 oce_ioctl(if_t ifp, u_long command, caddr_t data)
464 {
465 struct ifreq *ifr = (struct ifreq *)data;
466 POCE_SOFTC sc = if_getsoftc(ifp);
467 struct ifi2creq i2c;
468 uint8_t offset = 0;
469 int rc = 0;
470 uint32_t u;
471
472 switch (command) {
473 case SIOCGIFMEDIA:
474 rc = ifmedia_ioctl(ifp, ifr, &sc->media, command);
475 break;
476
477 case SIOCSIFMTU:
478 if (ifr->ifr_mtu > OCE_MAX_MTU)
479 rc = EINVAL;
480 else
481 if_setmtu(ifp, ifr->ifr_mtu);
482 break;
483
484 case SIOCSIFFLAGS:
485 if (if_getflags(ifp) & IFF_UP) {
486 if (!(if_getdrvflags(ifp) & IFF_DRV_RUNNING)) {
487 if_setdrvflagbits(ifp, IFF_DRV_RUNNING, 0);
488 oce_init(sc);
489 }
490 device_printf(sc->dev, "Interface Up\n");
491 } else {
492 LOCK(&sc->dev_lock);
493
494 if_setdrvflagbits(sc->ifp, 0,
495 IFF_DRV_RUNNING | IFF_DRV_OACTIVE);
496 oce_if_deactivate(sc);
497
498 UNLOCK(&sc->dev_lock);
499
500 device_printf(sc->dev, "Interface Down\n");
501 }
502
503 if ((if_getflags(ifp) & IFF_PROMISC) && !sc->promisc) {
504 if (!oce_rxf_set_promiscuous(sc, (1 | (1 << 1))))
505 sc->promisc = TRUE;
506 } else if (!(if_getflags(ifp) & IFF_PROMISC) && sc->promisc) {
507 if (!oce_rxf_set_promiscuous(sc, 0))
508 sc->promisc = FALSE;
509 }
510
511 break;
512
513 case SIOCADDMULTI:
514 case SIOCDELMULTI:
515 rc = oce_hw_update_multicast(sc);
516 if (rc)
517 device_printf(sc->dev,
518 "Update multicast address failed\n");
519 break;
520
521 case SIOCSIFCAP:
522 u = ifr->ifr_reqcap ^ if_getcapenable(ifp);
523
524 if (u & IFCAP_TXCSUM) {
525 if_togglecapenable(ifp, IFCAP_TXCSUM);
526 if_togglehwassist(ifp, (CSUM_TCP | CSUM_UDP | CSUM_IP));
527
528 if (IFCAP_TSO & if_getcapenable(ifp) &&
529 !(IFCAP_TXCSUM & if_getcapenable(ifp))) {
530 u &= ~IFCAP_TSO;
531 if_setcapenablebit(ifp, 0, IFCAP_TSO);
532 if_sethwassistbits(ifp, 0, CSUM_TSO);
533 if_printf(ifp,
534 "TSO disabled due to -txcsum.\n");
535 }
536 }
537
538 if (u & IFCAP_RXCSUM)
539 if_togglecapenable(ifp, IFCAP_RXCSUM);
540
541 if (u & IFCAP_TSO4) {
542 if_togglecapenable(ifp, IFCAP_TSO4);
543
544 if (IFCAP_TSO & if_getcapenable(ifp)) {
545 if (IFCAP_TXCSUM & if_getcapenable(ifp))
546 if_sethwassistbits(ifp, CSUM_TSO, 0);
547 else {
548 if_setcapenablebit(ifp, 0, IFCAP_TSO);
549 if_sethwassistbits(ifp, 0, CSUM_TSO);
550 if_printf(ifp,
551 "Enable txcsum first.\n");
552 rc = EAGAIN;
553 }
554 } else
555 if_sethwassistbits(ifp, 0, CSUM_TSO);
556 }
557
558 if (u & IFCAP_VLAN_HWTAGGING)
559 if_togglecapenable(ifp, IFCAP_VLAN_HWTAGGING);
560
561 if (u & IFCAP_VLAN_HWFILTER) {
562 if_togglecapenable(ifp, IFCAP_VLAN_HWFILTER);
563 oce_vid_config(sc);
564 }
565 #if defined(INET6) || defined(INET)
566 if (u & IFCAP_LRO) {
567 if_togglecapenable(ifp, IFCAP_LRO);
568 if(sc->enable_hwlro) {
569 if(if_getcapenable(ifp) & IFCAP_LRO) {
570 rc = oce_mbox_nic_set_iface_lro_config(sc, 1);
571 }else {
572 rc = oce_mbox_nic_set_iface_lro_config(sc, 0);
573 }
574 }
575 }
576 #endif
577
578 break;
579
580 case SIOCGI2C:
581 rc = copyin(ifr_data_get_ptr(ifr), &i2c, sizeof(i2c));
582 if (rc)
583 break;
584
585 if (i2c.dev_addr == PAGE_NUM_A0) {
586 offset = i2c.offset;
587 } else if (i2c.dev_addr == PAGE_NUM_A2) {
588 offset = TRANSCEIVER_A0_SIZE + i2c.offset;
589 } else {
590 rc = EINVAL;
591 break;
592 }
593
594 if (i2c.len > sizeof(i2c.data) ||
595 i2c.len + offset > sizeof(sfp_vpd_dump_buffer)) {
596 rc = EINVAL;
597 break;
598 }
599
600 rc = oce_mbox_read_transrecv_data(sc, i2c.dev_addr);
601 if (rc) {
602 rc = -rc;
603 break;
604 }
605
606 memcpy(&i2c.data[0], &sfp_vpd_dump_buffer[offset], i2c.len);
607
608 rc = copyout(&i2c, ifr_data_get_ptr(ifr), sizeof(i2c));
609 break;
610
611 case SIOCGPRIVATE_0:
612 rc = priv_check(curthread, PRIV_DRIVER);
613 if (rc != 0)
614 break;
615 rc = oce_handle_passthrough(ifp, data);
616 break;
617 default:
618 rc = ether_ioctl(ifp, command, data);
619 break;
620 }
621
622 return rc;
623 }
624
625 static void
oce_init(void * arg)626 oce_init(void *arg)
627 {
628 POCE_SOFTC sc = arg;
629
630 LOCK(&sc->dev_lock);
631
632 if (if_getflags(sc->ifp) & IFF_UP) {
633 oce_if_deactivate(sc);
634 oce_if_activate(sc);
635 }
636
637 UNLOCK(&sc->dev_lock);
638
639 }
640
641 static int
oce_multiq_start(if_t ifp,struct mbuf * m)642 oce_multiq_start(if_t ifp, struct mbuf *m)
643 {
644 POCE_SOFTC sc = if_getsoftc(ifp);
645 struct oce_wq *wq = NULL;
646 int queue_index = 0;
647 int status = 0;
648
649 if (!sc->link_status)
650 return ENXIO;
651
652 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE)
653 queue_index = m->m_pkthdr.flowid % sc->nwqs;
654
655 wq = sc->wq[queue_index];
656
657 LOCK(&wq->tx_lock);
658 status = oce_multiq_transmit(ifp, m, wq);
659 UNLOCK(&wq->tx_lock);
660
661 return status;
662
663 }
664
665 static void
oce_multiq_flush(if_t ifp)666 oce_multiq_flush(if_t ifp)
667 {
668 POCE_SOFTC sc = if_getsoftc(ifp);
669 struct mbuf *m;
670 int i = 0;
671
672 for (i = 0; i < sc->nwqs; i++) {
673 while ((m = buf_ring_dequeue_sc(sc->wq[i]->br)) != NULL)
674 m_freem(m);
675 }
676 if_qflush(ifp);
677 }
678
679 /*****************************************************************************
680 * Driver interrupt routines functions *
681 *****************************************************************************/
682
683 static void
oce_intr(void * arg,int pending)684 oce_intr(void *arg, int pending)
685 {
686
687 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
688 POCE_SOFTC sc = ii->sc;
689 struct oce_eq *eq = ii->eq;
690 struct oce_eqe *eqe;
691 struct oce_cq *cq = NULL;
692 int i, num_eqes = 0;
693
694 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
695 BUS_DMASYNC_POSTWRITE);
696 do {
697 eqe = RING_GET_CONSUMER_ITEM_VA(eq->ring, struct oce_eqe);
698 if (eqe->evnt == 0)
699 break;
700 eqe->evnt = 0;
701 bus_dmamap_sync(eq->ring->dma.tag, eq->ring->dma.map,
702 BUS_DMASYNC_POSTWRITE);
703 RING_GET(eq->ring, 1);
704 num_eqes++;
705
706 } while (TRUE);
707
708 if (!num_eqes)
709 goto eq_arm; /* Spurious */
710
711 /* Clear EQ entries, but dont arm */
712 oce_arm_eq(sc, eq->eq_id, num_eqes, FALSE, FALSE);
713
714 /* Process TX, RX and MCC. But dont arm CQ*/
715 for (i = 0; i < eq->cq_valid; i++) {
716 cq = eq->cq[i];
717 (*cq->cq_handler)(cq->cb_arg);
718 }
719
720 /* Arm all cqs connected to this EQ */
721 for (i = 0; i < eq->cq_valid; i++) {
722 cq = eq->cq[i];
723 oce_arm_cq(sc, cq->cq_id, 0, TRUE);
724 }
725
726 eq_arm:
727 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
728
729 return;
730 }
731
732 static int
oce_setup_intr(POCE_SOFTC sc)733 oce_setup_intr(POCE_SOFTC sc)
734 {
735 int rc = 0, use_intx = 0;
736 int vector = 0, req_vectors = 0;
737 int tot_req_vectors, tot_vectors;
738
739 if (is_rss_enabled(sc))
740 req_vectors = MAX((sc->nrqs - 1), sc->nwqs);
741 else
742 req_vectors = 1;
743
744 tot_req_vectors = req_vectors;
745 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
746 if (req_vectors > 1) {
747 tot_req_vectors += OCE_RDMA_VECTORS;
748 sc->roce_intr_count = OCE_RDMA_VECTORS;
749 }
750 }
751
752 if (sc->flags & OCE_FLAGS_MSIX_CAPABLE) {
753 sc->intr_count = req_vectors;
754 tot_vectors = tot_req_vectors;
755 rc = pci_alloc_msix(sc->dev, &tot_vectors);
756 if (rc != 0) {
757 use_intx = 1;
758 pci_release_msi(sc->dev);
759 } else {
760 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
761 if (tot_vectors < tot_req_vectors) {
762 if (sc->intr_count < (2 * OCE_RDMA_VECTORS)) {
763 sc->roce_intr_count = (tot_vectors / 2);
764 }
765 sc->intr_count = tot_vectors - sc->roce_intr_count;
766 }
767 } else {
768 sc->intr_count = tot_vectors;
769 }
770 sc->flags |= OCE_FLAGS_USING_MSIX;
771 }
772 } else
773 use_intx = 1;
774
775 if (use_intx)
776 sc->intr_count = 1;
777
778 /* Scale number of queues based on intr we got */
779 update_queues_got(sc);
780
781 if (use_intx) {
782 device_printf(sc->dev, "Using legacy interrupt\n");
783 rc = oce_alloc_intr(sc, vector, oce_intr);
784 if (rc)
785 goto error;
786 } else {
787 for (; vector < sc->intr_count; vector++) {
788 rc = oce_alloc_intr(sc, vector, oce_intr);
789 if (rc)
790 goto error;
791 }
792 }
793
794 return 0;
795 error:
796 oce_intr_free(sc);
797 return rc;
798 }
799
800 static int
oce_fast_isr(void * arg)801 oce_fast_isr(void *arg)
802 {
803 POCE_INTR_INFO ii = (POCE_INTR_INFO) arg;
804 POCE_SOFTC sc = ii->sc;
805
806 if (ii->eq == NULL)
807 return FILTER_STRAY;
808
809 oce_arm_eq(sc, ii->eq->eq_id, 0, FALSE, TRUE);
810
811 taskqueue_enqueue(ii->tq, &ii->task);
812
813 ii->eq->intr++;
814
815 return FILTER_HANDLED;
816 }
817
818 static int
oce_alloc_intr(POCE_SOFTC sc,int vector,void (* isr)(void * arg,int pending))819 oce_alloc_intr(POCE_SOFTC sc, int vector, void (*isr) (void *arg, int pending))
820 {
821 POCE_INTR_INFO ii;
822 int rc = 0, rr;
823
824 if (vector >= OCE_MAX_EQ)
825 return (EINVAL);
826
827 ii = &sc->intrs[vector];
828
829 /* Set the resource id for the interrupt.
830 * MSIx is vector + 1 for the resource id,
831 * INTx is 0 for the resource id.
832 */
833 if (sc->flags & OCE_FLAGS_USING_MSIX)
834 rr = vector + 1;
835 else
836 rr = 0;
837 ii->intr_res = bus_alloc_resource_any(sc->dev,
838 SYS_RES_IRQ,
839 &rr, RF_ACTIVE|RF_SHAREABLE);
840 ii->irq_rr = rr;
841 if (ii->intr_res == NULL) {
842 device_printf(sc->dev,
843 "Could not allocate interrupt\n");
844 rc = ENXIO;
845 return rc;
846 }
847
848 TASK_INIT(&ii->task, 0, isr, ii);
849 ii->vector = vector;
850 sprintf(ii->task_name, "oce_task[%d]", ii->vector);
851 ii->tq = taskqueue_create_fast(ii->task_name,
852 M_NOWAIT,
853 taskqueue_thread_enqueue,
854 &ii->tq);
855 taskqueue_start_threads(&ii->tq, 1, PI_NET, "%s taskq",
856 device_get_nameunit(sc->dev));
857
858 ii->sc = sc;
859 rc = bus_setup_intr(sc->dev,
860 ii->intr_res,
861 INTR_TYPE_NET,
862 oce_fast_isr, NULL, ii, &ii->tag);
863 return rc;
864
865 }
866
867 void
oce_intr_free(POCE_SOFTC sc)868 oce_intr_free(POCE_SOFTC sc)
869 {
870 int i = 0;
871
872 for (i = 0; i < sc->intr_count; i++) {
873
874 if (sc->intrs[i].tag != NULL)
875 bus_teardown_intr(sc->dev, sc->intrs[i].intr_res,
876 sc->intrs[i].tag);
877 if (sc->intrs[i].tq != NULL)
878 taskqueue_free(sc->intrs[i].tq);
879
880 if (sc->intrs[i].intr_res != NULL)
881 bus_release_resource(sc->dev, SYS_RES_IRQ,
882 sc->intrs[i].irq_rr,
883 sc->intrs[i].intr_res);
884 sc->intrs[i].tag = NULL;
885 sc->intrs[i].intr_res = NULL;
886 }
887
888 if (sc->flags & OCE_FLAGS_USING_MSIX)
889 pci_release_msi(sc->dev);
890
891 }
892
893 /******************************************************************************
894 * Media callbacks functions *
895 ******************************************************************************/
896
897 static void
oce_media_status(if_t ifp,struct ifmediareq * req)898 oce_media_status(if_t ifp, struct ifmediareq *req)
899 {
900 POCE_SOFTC sc = (POCE_SOFTC) if_getsoftc(ifp);
901
902 req->ifm_status = IFM_AVALID;
903 req->ifm_active = IFM_ETHER;
904
905 if (sc->link_status == 1)
906 req->ifm_status |= IFM_ACTIVE;
907 else
908 return;
909
910 switch (sc->link_speed) {
911 case 1: /* 10 Mbps */
912 req->ifm_active |= IFM_10_T | IFM_FDX;
913 sc->speed = 10;
914 break;
915 case 2: /* 100 Mbps */
916 req->ifm_active |= IFM_100_TX | IFM_FDX;
917 sc->speed = 100;
918 break;
919 case 3: /* 1 Gbps */
920 req->ifm_active |= IFM_1000_T | IFM_FDX;
921 sc->speed = 1000;
922 break;
923 case 4: /* 10 Gbps */
924 req->ifm_active |= IFM_10G_SR | IFM_FDX;
925 sc->speed = 10000;
926 break;
927 case 5: /* 20 Gbps */
928 req->ifm_active |= IFM_10G_SR | IFM_FDX;
929 sc->speed = 20000;
930 break;
931 case 6: /* 25 Gbps */
932 req->ifm_active |= IFM_10G_SR | IFM_FDX;
933 sc->speed = 25000;
934 break;
935 case 7: /* 40 Gbps */
936 req->ifm_active |= IFM_40G_SR4 | IFM_FDX;
937 sc->speed = 40000;
938 break;
939 default:
940 sc->speed = 0;
941 break;
942 }
943
944 return;
945 }
946
947 int
oce_media_change(if_t ifp)948 oce_media_change(if_t ifp)
949 {
950 return 0;
951 }
952
oce_is_pkt_dest_bmc(POCE_SOFTC sc,struct mbuf * m,boolean_t * os2bmc,struct mbuf ** m_new)953 static void oce_is_pkt_dest_bmc(POCE_SOFTC sc,
954 struct mbuf *m, boolean_t *os2bmc,
955 struct mbuf **m_new)
956 {
957 struct ether_header *eh = NULL;
958
959 eh = mtod(m, struct ether_header *);
960
961 if (!is_os2bmc_enabled(sc) || *os2bmc) {
962 *os2bmc = FALSE;
963 goto done;
964 }
965 if (!ETHER_IS_MULTICAST(eh->ether_dhost))
966 goto done;
967
968 if (is_mc_allowed_on_bmc(sc, eh) ||
969 is_bc_allowed_on_bmc(sc, eh) ||
970 is_arp_allowed_on_bmc(sc, ntohs(eh->ether_type))) {
971 *os2bmc = TRUE;
972 goto done;
973 }
974
975 if (mtod(m, struct ip *)->ip_p == IPPROTO_IPV6) {
976 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
977 uint8_t nexthdr = ip6->ip6_nxt;
978 if (nexthdr == IPPROTO_ICMPV6) {
979 struct icmp6_hdr *icmp6 = (struct icmp6_hdr *)(ip6 + 1);
980 switch (icmp6->icmp6_type) {
981 case ND_ROUTER_ADVERT:
982 *os2bmc = is_ipv6_ra_filt_enabled(sc);
983 goto done;
984 case ND_NEIGHBOR_ADVERT:
985 *os2bmc = is_ipv6_na_filt_enabled(sc);
986 goto done;
987 default:
988 break;
989 }
990 }
991 }
992
993 if (mtod(m, struct ip *)->ip_p == IPPROTO_UDP) {
994 struct ip *ip = mtod(m, struct ip *);
995 int iphlen = ip->ip_hl << 2;
996 struct udphdr *uh = (struct udphdr *)((caddr_t)ip + iphlen);
997 switch (uh->uh_dport) {
998 case DHCP_CLIENT_PORT:
999 *os2bmc = is_dhcp_client_filt_enabled(sc);
1000 goto done;
1001 case DHCP_SERVER_PORT:
1002 *os2bmc = is_dhcp_srvr_filt_enabled(sc);
1003 goto done;
1004 case NET_BIOS_PORT1:
1005 case NET_BIOS_PORT2:
1006 *os2bmc = is_nbios_filt_enabled(sc);
1007 goto done;
1008 case DHCPV6_RAS_PORT:
1009 *os2bmc = is_ipv6_ras_filt_enabled(sc);
1010 goto done;
1011 default:
1012 break;
1013 }
1014 }
1015 done:
1016 if (*os2bmc) {
1017 *m_new = m_dup(m, M_NOWAIT);
1018 if (!*m_new) {
1019 *os2bmc = FALSE;
1020 return;
1021 }
1022 *m_new = oce_insert_vlan_tag(sc, *m_new, NULL);
1023 }
1024 }
1025
1026 /*****************************************************************************
1027 * Transmit routines functions *
1028 *****************************************************************************/
1029
1030 static int
oce_tx(POCE_SOFTC sc,struct mbuf ** mpp,int wq_index)1031 oce_tx(POCE_SOFTC sc, struct mbuf **mpp, int wq_index)
1032 {
1033 int rc = 0, i, retry_cnt = 0;
1034 bus_dma_segment_t segs[OCE_MAX_TX_ELEMENTS];
1035 struct mbuf *m, *m_temp, *m_new = NULL;
1036 struct oce_wq *wq = sc->wq[wq_index];
1037 struct oce_packet_desc *pd;
1038 struct oce_nic_hdr_wqe *nichdr;
1039 struct oce_nic_frag_wqe *nicfrag;
1040 struct ether_header *eh = NULL;
1041 int num_wqes;
1042 uint32_t reg_value;
1043 boolean_t complete = TRUE;
1044 boolean_t os2bmc = FALSE;
1045
1046 m = *mpp;
1047 if (!m)
1048 return EINVAL;
1049
1050 if (!(m->m_flags & M_PKTHDR)) {
1051 rc = ENXIO;
1052 goto free_ret;
1053 }
1054
1055 /* Don't allow non-TSO packets longer than MTU */
1056 if (!is_tso_pkt(m)) {
1057 eh = mtod(m, struct ether_header *);
1058 if(m->m_pkthdr.len > ETHER_MAX_FRAME(sc->ifp, eh->ether_type, FALSE))
1059 goto free_ret;
1060 }
1061
1062 if(oce_tx_asic_stall_verify(sc, m)) {
1063 m = oce_insert_vlan_tag(sc, m, &complete);
1064 if(!m) {
1065 device_printf(sc->dev, "Insertion unsuccessful\n");
1066 return 0;
1067 }
1068 }
1069
1070 /* Lancer, SH ASIC has a bug wherein Packets that are 32 bytes or less
1071 * may cause a transmit stall on that port. So the work-around is to
1072 * pad short packets (<= 32 bytes) to a 36-byte length.
1073 */
1074 if(IS_SH(sc) || IS_XE201(sc) ) {
1075 if(m->m_pkthdr.len <= 32) {
1076 char buf[36];
1077 bzero((void *)buf, 36);
1078 m_append(m, (36 - m->m_pkthdr.len), buf);
1079 }
1080 }
1081
1082 tx_start:
1083 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1084 /* consolidate packet buffers for TSO/LSO segment offload */
1085 #if defined(INET6) || defined(INET)
1086 m = oce_tso_setup(sc, mpp);
1087 #else
1088 m = NULL;
1089 #endif
1090 if (m == NULL) {
1091 rc = ENXIO;
1092 goto free_ret;
1093 }
1094 }
1095
1096 pd = &wq->pckts[wq->pkt_desc_head];
1097
1098 retry:
1099 rc = bus_dmamap_load_mbuf_sg(wq->tag,
1100 pd->map,
1101 m, segs, &pd->nsegs, BUS_DMA_NOWAIT);
1102 if (rc == 0) {
1103 num_wqes = pd->nsegs + 1;
1104 if (IS_BE(sc) || IS_SH(sc)) {
1105 /*Dummy required only for BE3.*/
1106 if (num_wqes & 1)
1107 num_wqes++;
1108 }
1109 if (num_wqes >= RING_NUM_FREE(wq->ring)) {
1110 bus_dmamap_unload(wq->tag, pd->map);
1111 return EBUSY;
1112 }
1113 atomic_store_rel_int(&wq->pkt_desc_head,
1114 (wq->pkt_desc_head + 1) % \
1115 OCE_WQ_PACKET_ARRAY_SIZE);
1116 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_PREWRITE);
1117 pd->mbuf = m;
1118
1119 nichdr =
1120 RING_GET_PRODUCER_ITEM_VA(wq->ring, struct oce_nic_hdr_wqe);
1121 nichdr->u0.dw[0] = 0;
1122 nichdr->u0.dw[1] = 0;
1123 nichdr->u0.dw[2] = 0;
1124 nichdr->u0.dw[3] = 0;
1125
1126 nichdr->u0.s.complete = complete;
1127 nichdr->u0.s.mgmt = os2bmc;
1128 nichdr->u0.s.event = 1;
1129 nichdr->u0.s.crc = 1;
1130 nichdr->u0.s.forward = 0;
1131 nichdr->u0.s.ipcs = (m->m_pkthdr.csum_flags & CSUM_IP) ? 1 : 0;
1132 nichdr->u0.s.udpcs =
1133 (m->m_pkthdr.csum_flags & CSUM_UDP) ? 1 : 0;
1134 nichdr->u0.s.tcpcs =
1135 (m->m_pkthdr.csum_flags & CSUM_TCP) ? 1 : 0;
1136 nichdr->u0.s.num_wqe = num_wqes;
1137 nichdr->u0.s.total_length = m->m_pkthdr.len;
1138
1139 if (m->m_flags & M_VLANTAG) {
1140 nichdr->u0.s.vlan = 1; /*Vlan present*/
1141 nichdr->u0.s.vlan_tag = m->m_pkthdr.ether_vtag;
1142 }
1143
1144 if (m->m_pkthdr.csum_flags & CSUM_TSO) {
1145 if (m->m_pkthdr.tso_segsz) {
1146 nichdr->u0.s.lso = 1;
1147 nichdr->u0.s.lso_mss = m->m_pkthdr.tso_segsz;
1148 }
1149 if (!IS_BE(sc) || !IS_SH(sc))
1150 nichdr->u0.s.ipcs = 1;
1151 }
1152
1153 RING_PUT(wq->ring, 1);
1154 atomic_add_int(&wq->ring->num_used, 1);
1155
1156 for (i = 0; i < pd->nsegs; i++) {
1157 nicfrag =
1158 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1159 struct oce_nic_frag_wqe);
1160 nicfrag->u0.s.rsvd0 = 0;
1161 nicfrag->u0.s.frag_pa_hi = ADDR_HI(segs[i].ds_addr);
1162 nicfrag->u0.s.frag_pa_lo = ADDR_LO(segs[i].ds_addr);
1163 nicfrag->u0.s.frag_len = segs[i].ds_len;
1164 pd->wqe_idx = wq->ring->pidx;
1165 RING_PUT(wq->ring, 1);
1166 atomic_add_int(&wq->ring->num_used, 1);
1167 }
1168 if (num_wqes > (pd->nsegs + 1)) {
1169 nicfrag =
1170 RING_GET_PRODUCER_ITEM_VA(wq->ring,
1171 struct oce_nic_frag_wqe);
1172 nicfrag->u0.dw[0] = 0;
1173 nicfrag->u0.dw[1] = 0;
1174 nicfrag->u0.dw[2] = 0;
1175 nicfrag->u0.dw[3] = 0;
1176 pd->wqe_idx = wq->ring->pidx;
1177 RING_PUT(wq->ring, 1);
1178 atomic_add_int(&wq->ring->num_used, 1);
1179 pd->nsegs++;
1180 }
1181
1182 if_inc_counter(sc->ifp, IFCOUNTER_OPACKETS, 1);
1183 wq->tx_stats.tx_reqs++;
1184 wq->tx_stats.tx_wrbs += num_wqes;
1185 wq->tx_stats.tx_bytes += m->m_pkthdr.len;
1186 wq->tx_stats.tx_pkts++;
1187
1188 bus_dmamap_sync(wq->ring->dma.tag, wq->ring->dma.map,
1189 BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1190 reg_value = (num_wqes << 16) | wq->wq_id;
1191
1192 /* if os2bmc is not enabled or if the pkt is already tagged as
1193 bmc, do nothing
1194 */
1195 oce_is_pkt_dest_bmc(sc, m, &os2bmc, &m_new);
1196
1197 if_inc_counter(sc->ifp, IFCOUNTER_OBYTES, m->m_pkthdr.len);
1198 if (m->m_flags & M_MCAST)
1199 if_inc_counter(sc->ifp, IFCOUNTER_OMCASTS, 1);
1200 ETHER_BPF_MTAP(sc->ifp, m);
1201
1202 OCE_WRITE_REG32(sc, db, wq->db_offset, reg_value);
1203
1204 } else if (rc == EFBIG) {
1205 if (retry_cnt == 0) {
1206 m_temp = m_defrag(m, M_NOWAIT);
1207 if (m_temp == NULL)
1208 goto free_ret;
1209 m = m_temp;
1210 *mpp = m_temp;
1211 retry_cnt = retry_cnt + 1;
1212 goto retry;
1213 } else
1214 goto free_ret;
1215 } else if (rc == ENOMEM)
1216 return rc;
1217 else
1218 goto free_ret;
1219
1220 if (os2bmc) {
1221 m = m_new;
1222 goto tx_start;
1223 }
1224
1225 return 0;
1226
1227 free_ret:
1228 m_freem(*mpp);
1229 *mpp = NULL;
1230 return rc;
1231 }
1232
1233 static void
oce_process_tx_completion(struct oce_wq * wq)1234 oce_process_tx_completion(struct oce_wq *wq)
1235 {
1236 struct oce_packet_desc *pd;
1237 POCE_SOFTC sc = (POCE_SOFTC) wq->parent;
1238 struct mbuf *m;
1239
1240 pd = &wq->pckts[wq->pkt_desc_tail];
1241 atomic_store_rel_int(&wq->pkt_desc_tail,
1242 (wq->pkt_desc_tail + 1) % OCE_WQ_PACKET_ARRAY_SIZE);
1243 atomic_subtract_int(&wq->ring->num_used, pd->nsegs + 1);
1244 bus_dmamap_sync(wq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1245 bus_dmamap_unload(wq->tag, pd->map);
1246
1247 m = pd->mbuf;
1248 m_freem(m);
1249 pd->mbuf = NULL;
1250
1251 if (if_getdrvflags(sc->ifp) & IFF_DRV_OACTIVE) {
1252 if (wq->ring->num_used < (wq->ring->num_items / 2)) {
1253 if_setdrvflagbits(sc->ifp, 0, (IFF_DRV_OACTIVE));
1254 oce_tx_restart(sc, wq);
1255 }
1256 }
1257 }
1258
1259 static void
oce_tx_restart(POCE_SOFTC sc,struct oce_wq * wq)1260 oce_tx_restart(POCE_SOFTC sc, struct oce_wq *wq)
1261 {
1262
1263 if ((if_getdrvflags(sc->ifp) & IFF_DRV_RUNNING) != IFF_DRV_RUNNING)
1264 return;
1265
1266 if (!drbr_empty(sc->ifp, wq->br))
1267 taskqueue_enqueue(taskqueue_swi, &wq->txtask);
1268
1269 }
1270
1271 #if defined(INET6) || defined(INET)
1272 static struct mbuf *
oce_tso_setup(POCE_SOFTC sc,struct mbuf ** mpp)1273 oce_tso_setup(POCE_SOFTC sc, struct mbuf **mpp)
1274 {
1275 struct mbuf *m;
1276 #ifdef INET
1277 struct ip *ip;
1278 #endif
1279 #ifdef INET6
1280 struct ip6_hdr *ip6;
1281 #endif
1282 struct ether_vlan_header *eh;
1283 struct tcphdr *th;
1284 uint16_t etype;
1285 int total_len = 0, ehdrlen = 0;
1286
1287 m = *mpp;
1288
1289 if (M_WRITABLE(m) == 0) {
1290 m = m_dup(*mpp, M_NOWAIT);
1291 if (!m)
1292 return NULL;
1293 m_freem(*mpp);
1294 *mpp = m;
1295 }
1296
1297 eh = mtod(m, struct ether_vlan_header *);
1298 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
1299 etype = ntohs(eh->evl_proto);
1300 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
1301 } else {
1302 etype = ntohs(eh->evl_encap_proto);
1303 ehdrlen = ETHER_HDR_LEN;
1304 }
1305
1306 switch (etype) {
1307 #ifdef INET
1308 case ETHERTYPE_IP:
1309 ip = (struct ip *)(m->m_data + ehdrlen);
1310 if (ip->ip_p != IPPROTO_TCP)
1311 return NULL;
1312 th = (struct tcphdr *)((caddr_t)ip + (ip->ip_hl << 2));
1313
1314 total_len = ehdrlen + (ip->ip_hl << 2) + (th->th_off << 2);
1315 break;
1316 #endif
1317 #ifdef INET6
1318 case ETHERTYPE_IPV6:
1319 ip6 = (struct ip6_hdr *)(m->m_data + ehdrlen);
1320 if (ip6->ip6_nxt != IPPROTO_TCP)
1321 return NULL;
1322 th = (struct tcphdr *)((caddr_t)ip6 + sizeof(struct ip6_hdr));
1323
1324 total_len = ehdrlen + sizeof(struct ip6_hdr) + (th->th_off << 2);
1325 break;
1326 #endif
1327 default:
1328 return NULL;
1329 }
1330
1331 m = m_pullup(m, total_len);
1332 *mpp = m;
1333 return m;
1334 }
1335 #endif /* INET6 || INET */
1336
1337 void
oce_tx_task(void * arg,int npending)1338 oce_tx_task(void *arg, int npending)
1339 {
1340 struct oce_wq *wq = arg;
1341 POCE_SOFTC sc = wq->parent;
1342 if_t ifp = sc->ifp;
1343 int rc = 0;
1344
1345 LOCK(&wq->tx_lock);
1346 rc = oce_multiq_transmit(ifp, NULL, wq);
1347 if (rc) {
1348 device_printf(sc->dev,
1349 "TX[%d] restart failed\n", wq->queue_index);
1350 }
1351 UNLOCK(&wq->tx_lock);
1352 }
1353
1354 void
oce_start(if_t ifp)1355 oce_start(if_t ifp)
1356 {
1357 POCE_SOFTC sc = if_getsoftc(ifp);
1358 struct mbuf *m;
1359 int rc = 0;
1360 int def_q = 0; /* Defualt tx queue is 0*/
1361
1362 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1363 IFF_DRV_RUNNING)
1364 return;
1365
1366 if (!sc->link_status)
1367 return;
1368
1369 while (true) {
1370 m = if_dequeue(sc->ifp);
1371 if (m == NULL)
1372 break;
1373
1374 LOCK(&sc->wq[def_q]->tx_lock);
1375 rc = oce_tx(sc, &m, def_q);
1376 UNLOCK(&sc->wq[def_q]->tx_lock);
1377 if (rc) {
1378 if (m != NULL) {
1379 sc->wq[def_q]->tx_stats.tx_stops ++;
1380 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
1381 if_sendq_prepend(ifp, m);
1382 m = NULL;
1383 }
1384 break;
1385 }
1386 }
1387 }
1388
1389 /* Handle the Completion Queue for transmit */
1390 uint16_t
oce_wq_handler(void * arg)1391 oce_wq_handler(void *arg)
1392 {
1393 struct oce_wq *wq = (struct oce_wq *)arg;
1394 POCE_SOFTC sc = wq->parent;
1395 struct oce_cq *cq = wq->cq;
1396 struct oce_nic_tx_cqe *cqe;
1397 int num_cqes = 0;
1398
1399 LOCK(&wq->tx_compl_lock);
1400 bus_dmamap_sync(cq->ring->dma.tag,
1401 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1402 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1403 while (cqe->u0.dw[3]) {
1404 DW_SWAP((uint32_t *) cqe, sizeof(oce_wq_cqe));
1405
1406 wq->ring->cidx = cqe->u0.s.wqe_index + 1;
1407 if (wq->ring->cidx >= wq->ring->num_items)
1408 wq->ring->cidx -= wq->ring->num_items;
1409
1410 oce_process_tx_completion(wq);
1411 wq->tx_stats.tx_compl++;
1412 cqe->u0.dw[3] = 0;
1413 RING_GET(cq->ring, 1);
1414 bus_dmamap_sync(cq->ring->dma.tag,
1415 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1416 cqe =
1417 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_tx_cqe);
1418 num_cqes++;
1419 }
1420
1421 if (num_cqes)
1422 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
1423
1424 UNLOCK(&wq->tx_compl_lock);
1425 return num_cqes;
1426 }
1427
1428 static int
oce_multiq_transmit(if_t ifp,struct mbuf * m,struct oce_wq * wq)1429 oce_multiq_transmit(if_t ifp, struct mbuf *m, struct oce_wq *wq)
1430 {
1431 POCE_SOFTC sc = if_getsoftc(ifp);
1432 int status = 0, queue_index = 0;
1433 struct mbuf *next = NULL;
1434 struct buf_ring *br = NULL;
1435
1436 br = wq->br;
1437 queue_index = wq->queue_index;
1438
1439 if ((if_getdrvflags(ifp) & (IFF_DRV_RUNNING | IFF_DRV_OACTIVE)) !=
1440 IFF_DRV_RUNNING) {
1441 if (m != NULL)
1442 status = drbr_enqueue(ifp, br, m);
1443 return status;
1444 }
1445
1446 if (m != NULL) {
1447 if ((status = drbr_enqueue(ifp, br, m)) != 0)
1448 return status;
1449 }
1450 while ((next = drbr_peek(ifp, br)) != NULL) {
1451 if (oce_tx(sc, &next, queue_index)) {
1452 if (next == NULL) {
1453 drbr_advance(ifp, br);
1454 } else {
1455 drbr_putback(ifp, br, next);
1456 wq->tx_stats.tx_stops ++;
1457 if_setdrvflagbits(ifp, IFF_DRV_OACTIVE, 0);
1458 }
1459 break;
1460 }
1461 drbr_advance(ifp, br);
1462 }
1463
1464 return 0;
1465 }
1466
1467 /*****************************************************************************
1468 * Receive routines functions *
1469 *****************************************************************************/
1470
1471 static void
oce_correct_header(struct mbuf * m,struct nic_hwlro_cqe_part1 * cqe1,struct nic_hwlro_cqe_part2 * cqe2)1472 oce_correct_header(struct mbuf *m, struct nic_hwlro_cqe_part1 *cqe1, struct nic_hwlro_cqe_part2 *cqe2)
1473 {
1474 uint32_t *p;
1475 struct ether_header *eh = NULL;
1476 struct tcphdr *tcp_hdr = NULL;
1477 struct ip *ip4_hdr = NULL;
1478 struct ip6_hdr *ip6 = NULL;
1479 uint32_t payload_len = 0;
1480
1481 eh = mtod(m, struct ether_header *);
1482 /* correct IP header */
1483 if(!cqe2->ipv6_frame) {
1484 ip4_hdr = (struct ip *)((char*)eh + sizeof(struct ether_header));
1485 ip4_hdr->ip_ttl = cqe2->frame_lifespan;
1486 ip4_hdr->ip_len = htons(cqe2->coalesced_size - sizeof(struct ether_header));
1487 tcp_hdr = (struct tcphdr *)((char*)ip4_hdr + sizeof(struct ip));
1488 }else {
1489 ip6 = (struct ip6_hdr *)((char*)eh + sizeof(struct ether_header));
1490 ip6->ip6_ctlun.ip6_un1.ip6_un1_hlim = cqe2->frame_lifespan;
1491 payload_len = cqe2->coalesced_size - sizeof(struct ether_header)
1492 - sizeof(struct ip6_hdr);
1493 ip6->ip6_ctlun.ip6_un1.ip6_un1_plen = htons(payload_len);
1494 tcp_hdr = (struct tcphdr *)((char*)ip6 + sizeof(struct ip6_hdr));
1495 }
1496
1497 /* correct tcp header */
1498 tcp_hdr->th_ack = htonl(cqe2->tcp_ack_num);
1499 if(cqe2->push) {
1500 tcp_hdr->th_flags |= TH_PUSH;
1501 }
1502 tcp_hdr->th_win = htons(cqe2->tcp_window);
1503 tcp_hdr->th_sum = 0xffff;
1504 if(cqe2->ts_opt) {
1505 p = (uint32_t *)((char*)tcp_hdr + sizeof(struct tcphdr) + 2);
1506 *p = cqe1->tcp_timestamp_val;
1507 *(p+1) = cqe1->tcp_timestamp_ecr;
1508 }
1509
1510 return;
1511 }
1512
1513 static void
oce_rx_mbuf_chain(struct oce_rq * rq,struct oce_common_cqe_info * cqe_info,struct mbuf ** m)1514 oce_rx_mbuf_chain(struct oce_rq *rq, struct oce_common_cqe_info *cqe_info, struct mbuf **m)
1515 {
1516 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1517 uint32_t i = 0, frag_len = 0;
1518 uint32_t len = cqe_info->pkt_size;
1519 struct oce_packet_desc *pd;
1520 struct mbuf *tail = NULL;
1521
1522 for (i = 0; i < cqe_info->num_frags; i++) {
1523 if (rq->ring->cidx == rq->ring->pidx) {
1524 device_printf(sc->dev,
1525 "oce_rx_mbuf_chain: Invalid RX completion - Queue is empty\n");
1526 return;
1527 }
1528 pd = &rq->pckts[rq->ring->cidx];
1529
1530 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1531 bus_dmamap_unload(rq->tag, pd->map);
1532 RING_GET(rq->ring, 1);
1533 rq->pending--;
1534
1535 frag_len = (len > rq->cfg.frag_size) ? rq->cfg.frag_size : len;
1536 pd->mbuf->m_len = frag_len;
1537
1538 if (tail != NULL) {
1539 /* additional fragments */
1540 pd->mbuf->m_flags &= ~M_PKTHDR;
1541 tail->m_next = pd->mbuf;
1542 if(rq->islro)
1543 tail->m_nextpkt = NULL;
1544 tail = pd->mbuf;
1545 } else {
1546 /* first fragment, fill out much of the packet header */
1547 pd->mbuf->m_pkthdr.len = len;
1548 if(rq->islro)
1549 pd->mbuf->m_nextpkt = NULL;
1550 pd->mbuf->m_pkthdr.csum_flags = 0;
1551 if (IF_CSUM_ENABLED(sc)) {
1552 if (cqe_info->l4_cksum_pass) {
1553 if(!cqe_info->ipv6_frame) { /* IPV4 */
1554 pd->mbuf->m_pkthdr.csum_flags |=
1555 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1556 }else { /* IPV6 frame */
1557 if(rq->islro) {
1558 pd->mbuf->m_pkthdr.csum_flags |=
1559 (CSUM_DATA_VALID | CSUM_PSEUDO_HDR);
1560 }
1561 }
1562 pd->mbuf->m_pkthdr.csum_data = 0xffff;
1563 }
1564 if (cqe_info->ip_cksum_pass) {
1565 pd->mbuf->m_pkthdr.csum_flags |=
1566 (CSUM_IP_CHECKED|CSUM_IP_VALID);
1567 }
1568 }
1569 *m = tail = pd->mbuf;
1570 }
1571 pd->mbuf = NULL;
1572 len -= frag_len;
1573 }
1574
1575 return;
1576 }
1577
1578 static void
oce_rx_lro(struct oce_rq * rq,struct nic_hwlro_singleton_cqe * cqe,struct nic_hwlro_cqe_part2 * cqe2)1579 oce_rx_lro(struct oce_rq *rq, struct nic_hwlro_singleton_cqe *cqe, struct nic_hwlro_cqe_part2 *cqe2)
1580 {
1581 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1582 struct nic_hwlro_cqe_part1 *cqe1 = NULL;
1583 struct mbuf *m = NULL;
1584 struct oce_common_cqe_info cq_info;
1585
1586 /* parse cqe */
1587 if(cqe2 == NULL) {
1588 cq_info.pkt_size = cqe->pkt_size;
1589 cq_info.vtag = cqe->vlan_tag;
1590 cq_info.l4_cksum_pass = cqe->l4_cksum_pass;
1591 cq_info.ip_cksum_pass = cqe->ip_cksum_pass;
1592 cq_info.ipv6_frame = cqe->ipv6_frame;
1593 cq_info.vtp = cqe->vtp;
1594 cq_info.qnq = cqe->qnq;
1595 }else {
1596 cqe1 = (struct nic_hwlro_cqe_part1 *)cqe;
1597 cq_info.pkt_size = cqe2->coalesced_size;
1598 cq_info.vtag = cqe2->vlan_tag;
1599 cq_info.l4_cksum_pass = cqe2->l4_cksum_pass;
1600 cq_info.ip_cksum_pass = cqe2->ip_cksum_pass;
1601 cq_info.ipv6_frame = cqe2->ipv6_frame;
1602 cq_info.vtp = cqe2->vtp;
1603 cq_info.qnq = cqe1->qnq;
1604 }
1605
1606 cq_info.vtag = BSWAP_16(cq_info.vtag);
1607
1608 cq_info.num_frags = cq_info.pkt_size / rq->cfg.frag_size;
1609 if(cq_info.pkt_size % rq->cfg.frag_size)
1610 cq_info.num_frags++;
1611
1612 oce_rx_mbuf_chain(rq, &cq_info, &m);
1613
1614 if (m) {
1615 if(cqe2) {
1616 //assert(cqe2->valid != 0);
1617
1618 //assert(cqe2->cqe_type != 2);
1619 oce_correct_header(m, cqe1, cqe2);
1620 }
1621
1622 m->m_pkthdr.rcvif = sc->ifp;
1623 if (rq->queue_index)
1624 m->m_pkthdr.flowid = (rq->queue_index - 1);
1625 else
1626 m->m_pkthdr.flowid = rq->queue_index;
1627 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1628
1629 /* This deternies if vlan tag is Valid */
1630 if (cq_info.vtp) {
1631 if (sc->function_mode & FNM_FLEX10_MODE) {
1632 /* FLEX10. If QnQ is not set, neglect VLAN */
1633 if (cq_info.qnq) {
1634 m->m_pkthdr.ether_vtag = cq_info.vtag;
1635 m->m_flags |= M_VLANTAG;
1636 }
1637 } else if (sc->pvid != (cq_info.vtag & VLAN_VID_MASK)) {
1638 /* In UMC mode generally pvid will be striped by
1639 hw. But in some cases we have seen it comes
1640 with pvid. So if pvid == vlan, neglect vlan.
1641 */
1642 m->m_pkthdr.ether_vtag = cq_info.vtag;
1643 m->m_flags |= M_VLANTAG;
1644 }
1645 }
1646 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1647
1648 if_input(sc->ifp, m);
1649
1650 /* Update rx stats per queue */
1651 rq->rx_stats.rx_pkts++;
1652 rq->rx_stats.rx_bytes += cq_info.pkt_size;
1653 rq->rx_stats.rx_frags += cq_info.num_frags;
1654 rq->rx_stats.rx_ucast_pkts++;
1655 }
1656 return;
1657 }
1658
1659 static void
oce_rx(struct oce_rq * rq,struct oce_nic_rx_cqe * cqe)1660 oce_rx(struct oce_rq *rq, struct oce_nic_rx_cqe *cqe)
1661 {
1662 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1663 int len;
1664 struct mbuf *m = NULL;
1665 struct oce_common_cqe_info cq_info;
1666 uint16_t vtag = 0;
1667
1668 /* Is it a flush compl that has no data */
1669 if(!cqe->u0.s.num_fragments)
1670 goto exit;
1671
1672 len = cqe->u0.s.pkt_size;
1673 if (!len) {
1674 /*partial DMA workaround for Lancer*/
1675 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1676 goto exit;
1677 }
1678
1679 if (!oce_cqe_portid_valid(sc, cqe)) {
1680 oce_discard_rx_comp(rq, cqe->u0.s.num_fragments);
1681 goto exit;
1682 }
1683
1684 /* Get vlan_tag value */
1685 if(IS_BE(sc) || IS_SH(sc))
1686 vtag = BSWAP_16(cqe->u0.s.vlan_tag);
1687 else
1688 vtag = cqe->u0.s.vlan_tag;
1689
1690 cq_info.l4_cksum_pass = cqe->u0.s.l4_cksum_pass;
1691 cq_info.ip_cksum_pass = cqe->u0.s.ip_cksum_pass;
1692 cq_info.ipv6_frame = cqe->u0.s.ip_ver;
1693 cq_info.num_frags = cqe->u0.s.num_fragments;
1694 cq_info.pkt_size = cqe->u0.s.pkt_size;
1695
1696 oce_rx_mbuf_chain(rq, &cq_info, &m);
1697
1698 if (m) {
1699 m->m_pkthdr.rcvif = sc->ifp;
1700 if (rq->queue_index)
1701 m->m_pkthdr.flowid = (rq->queue_index - 1);
1702 else
1703 m->m_pkthdr.flowid = rq->queue_index;
1704 M_HASHTYPE_SET(m, M_HASHTYPE_OPAQUE);
1705
1706 /* This deternies if vlan tag is Valid */
1707 if (oce_cqe_vtp_valid(sc, cqe)) {
1708 if (sc->function_mode & FNM_FLEX10_MODE) {
1709 /* FLEX10. If QnQ is not set, neglect VLAN */
1710 if (cqe->u0.s.qnq) {
1711 m->m_pkthdr.ether_vtag = vtag;
1712 m->m_flags |= M_VLANTAG;
1713 }
1714 } else if (sc->pvid != (vtag & VLAN_VID_MASK)) {
1715 /* In UMC mode generally pvid will be striped by
1716 hw. But in some cases we have seen it comes
1717 with pvid. So if pvid == vlan, neglect vlan.
1718 */
1719 m->m_pkthdr.ether_vtag = vtag;
1720 m->m_flags |= M_VLANTAG;
1721 }
1722 }
1723
1724 if_inc_counter(sc->ifp, IFCOUNTER_IPACKETS, 1);
1725 #if defined(INET6) || defined(INET)
1726 /* Try to queue to LRO */
1727 if (IF_LRO_ENABLED(sc) &&
1728 (cqe->u0.s.ip_cksum_pass) &&
1729 (cqe->u0.s.l4_cksum_pass) &&
1730 (!cqe->u0.s.ip_ver) &&
1731 (rq->lro.lro_cnt != 0)) {
1732 if (tcp_lro_rx(&rq->lro, m, 0) == 0) {
1733 rq->lro_pkts_queued ++;
1734 goto post_done;
1735 }
1736 /* If LRO posting fails then try to post to STACK */
1737 }
1738 #endif
1739
1740 if_input(sc->ifp, m);
1741 #if defined(INET6) || defined(INET)
1742 post_done:
1743 #endif
1744 /* Update rx stats per queue */
1745 rq->rx_stats.rx_pkts++;
1746 rq->rx_stats.rx_bytes += cqe->u0.s.pkt_size;
1747 rq->rx_stats.rx_frags += cqe->u0.s.num_fragments;
1748 if (cqe->u0.s.pkt_type == OCE_MULTICAST_PACKET)
1749 rq->rx_stats.rx_mcast_pkts++;
1750 if (cqe->u0.s.pkt_type == OCE_UNICAST_PACKET)
1751 rq->rx_stats.rx_ucast_pkts++;
1752 }
1753 exit:
1754 return;
1755 }
1756
1757 void
oce_discard_rx_comp(struct oce_rq * rq,int num_frags)1758 oce_discard_rx_comp(struct oce_rq *rq, int num_frags)
1759 {
1760 uint32_t i = 0;
1761 struct oce_packet_desc *pd;
1762 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1763
1764 for (i = 0; i < num_frags; i++) {
1765 if (rq->ring->cidx == rq->ring->pidx) {
1766 device_printf(sc->dev,
1767 "oce_discard_rx_comp: Invalid RX completion - Queue is empty\n");
1768 return;
1769 }
1770 pd = &rq->pckts[rq->ring->cidx];
1771 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_POSTWRITE);
1772 bus_dmamap_unload(rq->tag, pd->map);
1773 if (pd->mbuf != NULL) {
1774 m_freem(pd->mbuf);
1775 pd->mbuf = NULL;
1776 }
1777
1778 RING_GET(rq->ring, 1);
1779 rq->pending--;
1780 }
1781 }
1782
1783 static int
oce_cqe_vtp_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1784 oce_cqe_vtp_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1785 {
1786 struct oce_nic_rx_cqe_v1 *cqe_v1;
1787 int vtp = 0;
1788
1789 if (sc->be3_native) {
1790 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1791 vtp = cqe_v1->u0.s.vlan_tag_present;
1792 } else
1793 vtp = cqe->u0.s.vlan_tag_present;
1794
1795 return vtp;
1796
1797 }
1798
1799 static int
oce_cqe_portid_valid(POCE_SOFTC sc,struct oce_nic_rx_cqe * cqe)1800 oce_cqe_portid_valid(POCE_SOFTC sc, struct oce_nic_rx_cqe *cqe)
1801 {
1802 struct oce_nic_rx_cqe_v1 *cqe_v1;
1803 int port_id = 0;
1804
1805 if (sc->be3_native && (IS_BE(sc) || IS_SH(sc))) {
1806 cqe_v1 = (struct oce_nic_rx_cqe_v1 *)cqe;
1807 port_id = cqe_v1->u0.s.port;
1808 if (sc->port_id != port_id)
1809 return 0;
1810 } else
1811 ;/* For BE3 legacy and Lancer this is dummy */
1812
1813 return 1;
1814
1815 }
1816
1817 #if defined(INET6) || defined(INET)
1818 void
oce_rx_flush_lro(struct oce_rq * rq)1819 oce_rx_flush_lro(struct oce_rq *rq)
1820 {
1821 struct lro_ctrl *lro = &rq->lro;
1822 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1823
1824 if (!IF_LRO_ENABLED(sc))
1825 return;
1826
1827 tcp_lro_flush_all(lro);
1828 rq->lro_pkts_queued = 0;
1829
1830 return;
1831 }
1832
1833 static int
oce_init_lro(POCE_SOFTC sc)1834 oce_init_lro(POCE_SOFTC sc)
1835 {
1836 struct lro_ctrl *lro = NULL;
1837 int i = 0, rc = 0;
1838
1839 for (i = 0; i < sc->nrqs; i++) {
1840 lro = &sc->rq[i]->lro;
1841 rc = tcp_lro_init(lro);
1842 if (rc != 0) {
1843 device_printf(sc->dev, "LRO init failed\n");
1844 return rc;
1845 }
1846 lro->ifp = sc->ifp;
1847 }
1848
1849 return rc;
1850 }
1851
1852 void
oce_free_lro(POCE_SOFTC sc)1853 oce_free_lro(POCE_SOFTC sc)
1854 {
1855 struct lro_ctrl *lro = NULL;
1856 int i = 0;
1857
1858 for (i = 0; i < sc->nrqs; i++) {
1859 lro = &sc->rq[i]->lro;
1860 if (lro)
1861 tcp_lro_free(lro);
1862 }
1863 }
1864 #endif
1865
1866 int
oce_alloc_rx_bufs(struct oce_rq * rq,int count)1867 oce_alloc_rx_bufs(struct oce_rq *rq, int count)
1868 {
1869 POCE_SOFTC sc = (POCE_SOFTC) rq->parent;
1870 int i, rc;
1871 struct oce_packet_desc *pd;
1872 bus_dma_segment_t segs[6];
1873 int nsegs, added = 0;
1874 struct oce_nic_rqe *rqe;
1875 pd_rxulp_db_t rxdb_reg;
1876 uint32_t val = 0;
1877 uint32_t oce_max_rq_posts = 64;
1878
1879 bzero(&rxdb_reg, sizeof(pd_rxulp_db_t));
1880 for (i = 0; i < count; i++) {
1881 pd = &rq->pckts[rq->ring->pidx];
1882 pd->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, oce_rq_buf_size);
1883 if (pd->mbuf == NULL) {
1884 device_printf(sc->dev, "mbuf allocation failed, size = %d\n",oce_rq_buf_size);
1885 break;
1886 }
1887 pd->mbuf->m_nextpkt = NULL;
1888
1889 pd->mbuf->m_len = pd->mbuf->m_pkthdr.len = rq->cfg.frag_size;
1890
1891 rc = bus_dmamap_load_mbuf_sg(rq->tag,
1892 pd->map,
1893 pd->mbuf,
1894 segs, &nsegs, BUS_DMA_NOWAIT);
1895 if (rc) {
1896 m_free(pd->mbuf);
1897 device_printf(sc->dev, "bus_dmamap_load_mbuf_sg failed rc = %d\n", rc);
1898 break;
1899 }
1900
1901 if (nsegs != 1) {
1902 i--;
1903 continue;
1904 }
1905
1906 bus_dmamap_sync(rq->tag, pd->map, BUS_DMASYNC_PREREAD);
1907
1908 rqe = RING_GET_PRODUCER_ITEM_VA(rq->ring, struct oce_nic_rqe);
1909 rqe->u0.s.frag_pa_hi = ADDR_HI(segs[0].ds_addr);
1910 rqe->u0.s.frag_pa_lo = ADDR_LO(segs[0].ds_addr);
1911 DW_SWAP(u32ptr(rqe), sizeof(struct oce_nic_rqe));
1912 RING_PUT(rq->ring, 1);
1913 added++;
1914 rq->pending++;
1915 }
1916 oce_max_rq_posts = sc->enable_hwlro ? OCE_HWLRO_MAX_RQ_POSTS : OCE_MAX_RQ_POSTS;
1917 if (added != 0) {
1918 for (i = added / oce_max_rq_posts; i > 0; i--) {
1919 rxdb_reg.bits.num_posted = oce_max_rq_posts;
1920 rxdb_reg.bits.qid = rq->rq_id;
1921 if(rq->islro) {
1922 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1923 val |= oce_max_rq_posts << 16;
1924 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1925 }else {
1926 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1927 }
1928 added -= oce_max_rq_posts;
1929 }
1930 if (added > 0) {
1931 rxdb_reg.bits.qid = rq->rq_id;
1932 rxdb_reg.bits.num_posted = added;
1933 if(rq->islro) {
1934 val |= rq->rq_id & DB_LRO_RQ_ID_MASK;
1935 val |= added << 16;
1936 OCE_WRITE_REG32(sc, db, DB_OFFSET, val);
1937 }else {
1938 OCE_WRITE_REG32(sc, db, PD_RXULP_DB, rxdb_reg.dw0);
1939 }
1940 }
1941 }
1942
1943 return 0;
1944 }
1945
1946 static void
oce_check_rx_bufs(POCE_SOFTC sc,uint32_t num_cqes,struct oce_rq * rq)1947 oce_check_rx_bufs(POCE_SOFTC sc, uint32_t num_cqes, struct oce_rq *rq)
1948 {
1949 if (num_cqes) {
1950 oce_arm_cq(sc, rq->cq->cq_id, num_cqes, FALSE);
1951 if(!sc->enable_hwlro) {
1952 if((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) > 1)
1953 oce_alloc_rx_bufs(rq, ((OCE_RQ_PACKET_ARRAY_SIZE - rq->pending) - 1));
1954 }else {
1955 if ((OCE_RQ_PACKET_ARRAY_SIZE -1 - rq->pending) > 64)
1956 oce_alloc_rx_bufs(rq, 64);
1957 }
1958 }
1959
1960 return;
1961 }
1962
1963 uint16_t
oce_rq_handler_lro(void * arg)1964 oce_rq_handler_lro(void *arg)
1965 {
1966 struct oce_rq *rq = (struct oce_rq *)arg;
1967 struct oce_cq *cq = rq->cq;
1968 POCE_SOFTC sc = rq->parent;
1969 struct nic_hwlro_singleton_cqe *cqe;
1970 struct nic_hwlro_cqe_part2 *cqe2;
1971 int num_cqes = 0;
1972
1973 LOCK(&rq->rx_lock);
1974 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
1975 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
1976 while (cqe->valid) {
1977 if(cqe->cqe_type == 0) { /* singleton cqe */
1978 /* we should not get singleton cqe after cqe1 on same rq */
1979 if(rq->cqe_firstpart != NULL) {
1980 device_printf(sc->dev, "Got singleton cqe after cqe1 \n");
1981 goto exit_rq_handler_lro;
1982 }
1983 if(cqe->error != 0) {
1984 rq->rx_stats.rxcp_err++;
1985 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
1986 }
1987 oce_rx_lro(rq, cqe, NULL);
1988 rq->rx_stats.rx_compl++;
1989 cqe->valid = 0;
1990 RING_GET(cq->ring, 1);
1991 num_cqes++;
1992 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
1993 break;
1994 }else if(cqe->cqe_type == 0x1) { /* first part */
1995 /* we should not get cqe1 after cqe1 on same rq */
1996 if(rq->cqe_firstpart != NULL) {
1997 device_printf(sc->dev, "Got cqe1 after cqe1 \n");
1998 goto exit_rq_handler_lro;
1999 }
2000 rq->cqe_firstpart = (struct nic_hwlro_cqe_part1 *)cqe;
2001 RING_GET(cq->ring, 1);
2002 }else if(cqe->cqe_type == 0x2) { /* second part */
2003 cqe2 = (struct nic_hwlro_cqe_part2 *)cqe;
2004 if(cqe2->error != 0) {
2005 rq->rx_stats.rxcp_err++;
2006 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2007 }
2008 /* We should not get cqe2 without cqe1 */
2009 if(rq->cqe_firstpart == NULL) {
2010 device_printf(sc->dev, "Got cqe2 without cqe1 \n");
2011 goto exit_rq_handler_lro;
2012 }
2013 oce_rx_lro(rq, (struct nic_hwlro_singleton_cqe *)rq->cqe_firstpart, cqe2);
2014
2015 rq->rx_stats.rx_compl++;
2016 rq->cqe_firstpart->valid = 0;
2017 cqe2->valid = 0;
2018 rq->cqe_firstpart = NULL;
2019
2020 RING_GET(cq->ring, 1);
2021 num_cqes += 2;
2022 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2023 break;
2024 }
2025
2026 bus_dmamap_sync(cq->ring->dma.tag,cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2027 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct nic_hwlro_singleton_cqe);
2028 }
2029 oce_check_rx_bufs(sc, num_cqes, rq);
2030 exit_rq_handler_lro:
2031 UNLOCK(&rq->rx_lock);
2032 return 0;
2033 }
2034
2035 /* Handle the Completion Queue for receive */
2036 uint16_t
oce_rq_handler(void * arg)2037 oce_rq_handler(void *arg)
2038 {
2039 struct epoch_tracker et;
2040 struct oce_rq *rq = (struct oce_rq *)arg;
2041 struct oce_cq *cq = rq->cq;
2042 POCE_SOFTC sc = rq->parent;
2043 struct oce_nic_rx_cqe *cqe;
2044 int num_cqes = 0;
2045
2046 NET_EPOCH_ENTER(et);
2047 if(rq->islro) {
2048 oce_rq_handler_lro(arg);
2049 NET_EPOCH_EXIT(et);
2050 return 0;
2051 }
2052 LOCK(&rq->rx_lock);
2053 bus_dmamap_sync(cq->ring->dma.tag,
2054 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2055 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2056 while (cqe->u0.dw[2]) {
2057 DW_SWAP((uint32_t *) cqe, sizeof(oce_rq_cqe));
2058
2059 if (cqe->u0.s.error == 0) {
2060 oce_rx(rq, cqe);
2061 } else {
2062 rq->rx_stats.rxcp_err++;
2063 if_inc_counter(sc->ifp, IFCOUNTER_IERRORS, 1);
2064 /* Post L3/L4 errors to stack.*/
2065 oce_rx(rq, cqe);
2066 }
2067 rq->rx_stats.rx_compl++;
2068 cqe->u0.dw[2] = 0;
2069
2070 #if defined(INET6) || defined(INET)
2071 if (IF_LRO_ENABLED(sc) && rq->lro_pkts_queued >= 16) {
2072 oce_rx_flush_lro(rq);
2073 }
2074 #endif
2075
2076 RING_GET(cq->ring, 1);
2077 bus_dmamap_sync(cq->ring->dma.tag,
2078 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2079 cqe =
2080 RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_nic_rx_cqe);
2081 num_cqes++;
2082 if (num_cqes >= (IS_XE201(sc) ? 8 : oce_max_rsp_handled))
2083 break;
2084 }
2085
2086 #if defined(INET6) || defined(INET)
2087 if (IF_LRO_ENABLED(sc))
2088 oce_rx_flush_lro(rq);
2089 #endif
2090
2091 oce_check_rx_bufs(sc, num_cqes, rq);
2092 UNLOCK(&rq->rx_lock);
2093 NET_EPOCH_EXIT(et);
2094 return 0;
2095
2096 }
2097
2098 /*****************************************************************************
2099 * Helper function prototypes in this file *
2100 *****************************************************************************/
2101
2102 static int
oce_attach_ifp(POCE_SOFTC sc)2103 oce_attach_ifp(POCE_SOFTC sc)
2104 {
2105
2106 sc->ifp = if_alloc(IFT_ETHER);
2107 if (!sc->ifp)
2108 return ENOMEM;
2109
2110 ifmedia_init(&sc->media, IFM_IMASK, oce_media_change, oce_media_status);
2111 ifmedia_add(&sc->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2112 ifmedia_set(&sc->media, IFM_ETHER | IFM_AUTO);
2113
2114 if_setflags(sc->ifp, IFF_BROADCAST | IFF_MULTICAST);
2115 if_setioctlfn(sc->ifp, oce_ioctl);
2116 if_setstartfn(sc->ifp, oce_start);
2117 if_setinitfn(sc->ifp, oce_init);
2118 if_setmtu(sc->ifp, ETHERMTU);
2119 if_setsoftc(sc->ifp, sc);
2120 if_settransmitfn(sc->ifp, oce_multiq_start);
2121 if_setqflushfn(sc->ifp, oce_multiq_flush);
2122
2123 if_initname(sc->ifp,
2124 device_get_name(sc->dev), device_get_unit(sc->dev));
2125
2126 if_setsendqlen(sc->ifp, OCE_MAX_TX_DESC - 1);
2127 if_setsendqready(sc->ifp);
2128
2129 if_sethwassist(sc->ifp, OCE_IF_HWASSIST);
2130 if_sethwassistbits(sc->ifp, CSUM_TSO, 0);
2131 if_sethwassistbits(sc->ifp, (CSUM_IP | CSUM_TCP | CSUM_UDP), 0);
2132
2133 if_setcapabilities(sc->ifp, OCE_IF_CAPABILITIES);
2134 if_setcapabilitiesbit(sc->ifp, IFCAP_HWCSUM, 0);
2135 if_setcapabilitiesbit(sc->ifp, IFCAP_VLAN_HWFILTER, 0);
2136
2137 #if defined(INET6) || defined(INET)
2138 if_setcapabilitiesbit(sc->ifp, IFCAP_TSO, 0);
2139 if_setcapabilitiesbit(sc->ifp, IFCAP_LRO, 0);
2140 if_setcapabilitiesbit(sc->ifp, IFCAP_VLAN_HWTSO, 0);
2141 #endif
2142
2143 if_setcapenable(sc->ifp, if_getcapabilities(sc->ifp));
2144 if_setbaudrate(sc->ifp, IF_Gbps(10));
2145
2146 if_sethwtsomax(sc->ifp, 65536 - (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN));
2147 if_sethwtsomaxsegcount(sc->ifp, OCE_MAX_TX_ELEMENTS);
2148 if_sethwtsomaxsegsize(sc->ifp, 4096);
2149
2150 ether_ifattach(sc->ifp, sc->macaddr.mac_addr);
2151
2152 return 0;
2153 }
2154
2155 static void
oce_add_vlan(void * arg,if_t ifp,uint16_t vtag)2156 oce_add_vlan(void *arg, if_t ifp, uint16_t vtag)
2157 {
2158 POCE_SOFTC sc = if_getsoftc(ifp);
2159
2160 if (if_getsoftc(ifp) != arg)
2161 return;
2162 if ((vtag == 0) || (vtag > 4095))
2163 return;
2164
2165 sc->vlan_tag[vtag] = 1;
2166 sc->vlans_added++;
2167 if (sc->vlans_added <= (sc->max_vlans + 1))
2168 oce_vid_config(sc);
2169 }
2170
2171 static void
oce_del_vlan(void * arg,if_t ifp,uint16_t vtag)2172 oce_del_vlan(void *arg, if_t ifp, uint16_t vtag)
2173 {
2174 POCE_SOFTC sc = if_getsoftc(ifp);
2175
2176 if (if_getsoftc(ifp) != arg)
2177 return;
2178 if ((vtag == 0) || (vtag > 4095))
2179 return;
2180
2181 sc->vlan_tag[vtag] = 0;
2182 sc->vlans_added--;
2183 oce_vid_config(sc);
2184 }
2185
2186 /*
2187 * A max of 64 vlans can be configured in BE. If the user configures
2188 * more, place the card in vlan promiscuous mode.
2189 */
2190 static int
oce_vid_config(POCE_SOFTC sc)2191 oce_vid_config(POCE_SOFTC sc)
2192 {
2193 struct normal_vlan vtags[MAX_VLANFILTER_SIZE];
2194 uint16_t ntags = 0, i;
2195 int status = 0;
2196
2197 if ((sc->vlans_added <= MAX_VLANFILTER_SIZE) &&
2198 (if_getcapenable(sc->ifp) & IFCAP_VLAN_HWFILTER)) {
2199 for (i = 0; i < MAX_VLANS; i++) {
2200 if (sc->vlan_tag[i]) {
2201 vtags[ntags].vtag = i;
2202 ntags++;
2203 }
2204 }
2205 if (ntags)
2206 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2207 vtags, ntags, 1, 0);
2208 } else
2209 status = oce_config_vlan(sc, (uint8_t) sc->if_id,
2210 NULL, 0, 1, 1);
2211 return status;
2212 }
2213
2214 static void
oce_mac_addr_set(POCE_SOFTC sc)2215 oce_mac_addr_set(POCE_SOFTC sc)
2216 {
2217 uint32_t old_pmac_id = sc->pmac_id;
2218 int status = 0;
2219
2220 status = bcmp((if_getlladdr(sc->ifp)), sc->macaddr.mac_addr,
2221 sc->macaddr.size_of_struct);
2222 if (!status)
2223 return;
2224
2225 status = oce_mbox_macaddr_add(sc, (uint8_t *)(if_getlladdr(sc->ifp)),
2226 sc->if_id, &sc->pmac_id);
2227 if (!status) {
2228 status = oce_mbox_macaddr_del(sc, sc->if_id, old_pmac_id);
2229 bcopy((if_getlladdr(sc->ifp)), sc->macaddr.mac_addr,
2230 sc->macaddr.size_of_struct);
2231 }
2232 if (status)
2233 device_printf(sc->dev, "Failed update macaddress\n");
2234
2235 }
2236
2237 static int
oce_handle_passthrough(if_t ifp,caddr_t data)2238 oce_handle_passthrough(if_t ifp, caddr_t data)
2239 {
2240 POCE_SOFTC sc = if_getsoftc(ifp);
2241 struct ifreq *ifr = (struct ifreq *)data;
2242 int rc = ENXIO;
2243 char cookie[32] = {0};
2244 void *priv_data = ifr_data_get_ptr(ifr);
2245 void *ioctl_ptr;
2246 uint32_t req_size;
2247 struct mbx_hdr req;
2248 OCE_DMA_MEM dma_mem;
2249
2250 if (copyin(priv_data, cookie, strlen(IOCTL_COOKIE)))
2251 return EFAULT;
2252
2253 if (memcmp(cookie, IOCTL_COOKIE, strlen(IOCTL_COOKIE)))
2254 return EINVAL;
2255
2256 ioctl_ptr = (char *)priv_data + strlen(IOCTL_COOKIE);
2257 if (copyin(ioctl_ptr, &req, sizeof(struct mbx_hdr)))
2258 return EFAULT;
2259
2260 req_size = le32toh(req.u0.req.request_length);
2261 if (req_size > 65536)
2262 return EINVAL;
2263
2264 req_size += sizeof(struct mbx_hdr);
2265 rc = oce_dma_alloc(sc, req_size, &dma_mem, 0);
2266 if (rc)
2267 return ENOMEM;
2268
2269 if (copyin(ioctl_ptr, OCE_DMAPTR(&dma_mem,char), req_size)) {
2270 rc = EFAULT;
2271 goto dma_free;
2272 }
2273
2274 rc = oce_pass_through_mbox(sc, &dma_mem, req_size);
2275 if (rc) {
2276 rc = EIO;
2277 goto dma_free;
2278 }
2279
2280 if (copyout(OCE_DMAPTR(&dma_mem,char), ioctl_ptr, req_size)) {
2281 rc = EFAULT;
2282 goto dma_free;
2283 }
2284
2285 /*
2286 firmware is filling all the attributes for this ioctl except
2287 the driver version..so fill it
2288 */
2289 if(req.u0.rsp.opcode == OPCODE_COMMON_GET_CNTL_ATTRIBUTES) {
2290 struct mbx_common_get_cntl_attr *fw_cmd =
2291 (struct mbx_common_get_cntl_attr *)ioctl_ptr;
2292 _Static_assert(sizeof(COMPONENT_REVISION) <=
2293 sizeof(fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str),
2294 "driver version string too long");
2295
2296 rc = copyout(COMPONENT_REVISION,
2297 fw_cmd->params.rsp.cntl_attr_info.hba_attr.drv_ver_str,
2298 sizeof(COMPONENT_REVISION));
2299 }
2300
2301 dma_free:
2302 oce_dma_free(sc, &dma_mem);
2303 return rc;
2304
2305 }
2306
2307 static void
oce_eqd_set_periodic(POCE_SOFTC sc)2308 oce_eqd_set_periodic(POCE_SOFTC sc)
2309 {
2310 struct oce_set_eqd set_eqd[OCE_MAX_EQ];
2311 struct oce_aic_obj *aic;
2312 struct oce_eq *eqo;
2313 uint64_t now = 0, delta;
2314 int eqd, i, num = 0;
2315 uint32_t tx_reqs = 0, rxpkts = 0, pps;
2316 struct oce_wq *wq;
2317 struct oce_rq *rq;
2318
2319 #define ticks_to_msecs(t) (1000 * (t) / hz)
2320
2321 for (i = 0 ; i < sc->neqs; i++) {
2322 eqo = sc->eq[i];
2323 aic = &sc->aic_obj[i];
2324 /* When setting the static eq delay from the user space */
2325 if (!aic->enable) {
2326 if (aic->ticks)
2327 aic->ticks = 0;
2328 eqd = aic->et_eqd;
2329 goto modify_eqd;
2330 }
2331
2332 if (i == 0) {
2333 rq = sc->rq[0];
2334 rxpkts = rq->rx_stats.rx_pkts;
2335 } else
2336 rxpkts = 0;
2337 if (i + 1 < sc->nrqs) {
2338 rq = sc->rq[i + 1];
2339 rxpkts += rq->rx_stats.rx_pkts;
2340 }
2341 if (i < sc->nwqs) {
2342 wq = sc->wq[i];
2343 tx_reqs = wq->tx_stats.tx_reqs;
2344 } else
2345 tx_reqs = 0;
2346 now = ticks;
2347
2348 if (!aic->ticks || now < aic->ticks ||
2349 rxpkts < aic->prev_rxpkts || tx_reqs < aic->prev_txreqs) {
2350 aic->prev_rxpkts = rxpkts;
2351 aic->prev_txreqs = tx_reqs;
2352 aic->ticks = now;
2353 continue;
2354 }
2355
2356 delta = ticks_to_msecs(now - aic->ticks);
2357
2358 pps = (((uint32_t)(rxpkts - aic->prev_rxpkts) * 1000) / delta) +
2359 (((uint32_t)(tx_reqs - aic->prev_txreqs) * 1000) / delta);
2360 eqd = (pps / 15000) << 2;
2361 if (eqd < 8)
2362 eqd = 0;
2363
2364 /* Make sure that the eq delay is in the known range */
2365 eqd = min(eqd, aic->max_eqd);
2366 eqd = max(eqd, aic->min_eqd);
2367
2368 aic->prev_rxpkts = rxpkts;
2369 aic->prev_txreqs = tx_reqs;
2370 aic->ticks = now;
2371
2372 modify_eqd:
2373 if (eqd != aic->cur_eqd) {
2374 set_eqd[num].delay_multiplier = (eqd * 65)/100;
2375 set_eqd[num].eq_id = eqo->eq_id;
2376 aic->cur_eqd = eqd;
2377 num++;
2378 }
2379 }
2380
2381 /* Is there atleast one eq that needs to be modified? */
2382 for(i = 0; i < num; i += 8) {
2383 if((num - i) >=8 )
2384 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], 8);
2385 else
2386 oce_mbox_eqd_modify_periodic(sc, &set_eqd[i], (num - i));
2387 }
2388
2389 }
2390
oce_detect_hw_error(POCE_SOFTC sc)2391 static void oce_detect_hw_error(POCE_SOFTC sc)
2392 {
2393
2394 uint32_t ue_low = 0, ue_high = 0, ue_low_mask = 0, ue_high_mask = 0;
2395 uint32_t sliport_status = 0, sliport_err1 = 0, sliport_err2 = 0;
2396 uint32_t i;
2397
2398 if (sc->hw_error)
2399 return;
2400
2401 if (IS_XE201(sc)) {
2402 sliport_status = OCE_READ_REG32(sc, db, SLIPORT_STATUS_OFFSET);
2403 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2404 sliport_err1 = OCE_READ_REG32(sc, db, SLIPORT_ERROR1_OFFSET);
2405 sliport_err2 = OCE_READ_REG32(sc, db, SLIPORT_ERROR2_OFFSET);
2406 }
2407 } else {
2408 ue_low = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW);
2409 ue_high = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HIGH);
2410 ue_low_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_LOW_MASK);
2411 ue_high_mask = OCE_READ_REG32(sc, devcfg, PCICFG_UE_STATUS_HI_MASK);
2412
2413 ue_low = (ue_low & ~ue_low_mask);
2414 ue_high = (ue_high & ~ue_high_mask);
2415 }
2416
2417 /* On certain platforms BE hardware can indicate spurious UEs.
2418 * Allow the h/w to stop working completely in case of a real UE.
2419 * Hence not setting the hw_error for UE detection.
2420 */
2421 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2422 sc->hw_error = TRUE;
2423 device_printf(sc->dev, "Error detected in the card\n");
2424 }
2425
2426 if (sliport_status & SLIPORT_STATUS_ERR_MASK) {
2427 device_printf(sc->dev,
2428 "ERR: sliport status 0x%x\n", sliport_status);
2429 device_printf(sc->dev,
2430 "ERR: sliport error1 0x%x\n", sliport_err1);
2431 device_printf(sc->dev,
2432 "ERR: sliport error2 0x%x\n", sliport_err2);
2433 }
2434
2435 if (ue_low) {
2436 for (i = 0; ue_low; ue_low >>= 1, i++) {
2437 if (ue_low & 1)
2438 device_printf(sc->dev, "UE: %s bit set\n",
2439 ue_status_low_desc[i]);
2440 }
2441 }
2442
2443 if (ue_high) {
2444 for (i = 0; ue_high; ue_high >>= 1, i++) {
2445 if (ue_high & 1)
2446 device_printf(sc->dev, "UE: %s bit set\n",
2447 ue_status_hi_desc[i]);
2448 }
2449 }
2450
2451 }
2452
2453 static void
oce_local_timer(void * arg)2454 oce_local_timer(void *arg)
2455 {
2456 POCE_SOFTC sc = arg;
2457 int i = 0;
2458
2459 oce_detect_hw_error(sc);
2460 oce_refresh_nic_stats(sc);
2461 oce_refresh_queue_stats(sc);
2462 oce_mac_addr_set(sc);
2463
2464 /* TX Watch Dog*/
2465 for (i = 0; i < sc->nwqs; i++)
2466 oce_tx_restart(sc, sc->wq[i]);
2467
2468 /* calculate and set the eq delay for optimal interrupt rate */
2469 if (IS_BE(sc) || IS_SH(sc))
2470 oce_eqd_set_periodic(sc);
2471
2472 callout_reset(&sc->timer, hz, oce_local_timer, sc);
2473 }
2474
2475 static void
oce_tx_compl_clean(POCE_SOFTC sc)2476 oce_tx_compl_clean(POCE_SOFTC sc)
2477 {
2478 struct oce_wq *wq;
2479 int i = 0, timeo = 0, num_wqes = 0;
2480 int pending_txqs = sc->nwqs;
2481
2482 /* Stop polling for compls when HW has been silent for 10ms or
2483 * hw_error or no outstanding completions expected
2484 */
2485 do {
2486 pending_txqs = sc->nwqs;
2487
2488 for_all_wq_queues(sc, wq, i) {
2489 num_wqes = oce_wq_handler(wq);
2490
2491 if(num_wqes)
2492 timeo = 0;
2493
2494 if(!wq->ring->num_used)
2495 pending_txqs--;
2496 }
2497
2498 if (pending_txqs == 0 || ++timeo > 10 || sc->hw_error)
2499 break;
2500
2501 DELAY(1000);
2502 } while (TRUE);
2503
2504 for_all_wq_queues(sc, wq, i) {
2505 while(wq->ring->num_used) {
2506 LOCK(&wq->tx_compl_lock);
2507 oce_process_tx_completion(wq);
2508 UNLOCK(&wq->tx_compl_lock);
2509 }
2510 }
2511
2512 }
2513
2514 /* NOTE : This should only be called holding
2515 * DEVICE_LOCK.
2516 */
2517 static void
oce_if_deactivate(POCE_SOFTC sc)2518 oce_if_deactivate(POCE_SOFTC sc)
2519 {
2520 int i;
2521 struct oce_rq *rq;
2522 struct oce_wq *wq;
2523 struct oce_eq *eq;
2524
2525 if_setdrvflagbits(sc->ifp, 0, (IFF_DRV_RUNNING | IFF_DRV_OACTIVE));
2526
2527 oce_tx_compl_clean(sc);
2528
2529 /* Stop intrs and finish any bottom halves pending */
2530 oce_hw_intr_disable(sc);
2531
2532 /* Since taskqueue_drain takes a Gaint Lock, We should not acquire
2533 any other lock. So unlock device lock and require after
2534 completing taskqueue_drain.
2535 */
2536 UNLOCK(&sc->dev_lock);
2537 for (i = 0; i < sc->intr_count; i++) {
2538 if (sc->intrs[i].tq != NULL) {
2539 taskqueue_drain(sc->intrs[i].tq, &sc->intrs[i].task);
2540 }
2541 }
2542 LOCK(&sc->dev_lock);
2543
2544 /* Delete RX queue in card with flush param */
2545 oce_stop_rx(sc);
2546
2547 /* Invalidate any pending cq and eq entries*/
2548 for_all_evnt_queues(sc, eq, i)
2549 oce_drain_eq(eq);
2550 for_all_rq_queues(sc, rq, i)
2551 oce_drain_rq_cq(rq);
2552 for_all_wq_queues(sc, wq, i)
2553 oce_drain_wq_cq(wq);
2554
2555 /* But still we need to get MCC aync events.
2556 So enable intrs and also arm first EQ
2557 */
2558 oce_hw_intr_enable(sc);
2559 oce_arm_eq(sc, sc->eq[0]->eq_id, 0, TRUE, FALSE);
2560
2561 DELAY(10);
2562 }
2563
2564 static void
oce_if_activate(POCE_SOFTC sc)2565 oce_if_activate(POCE_SOFTC sc)
2566 {
2567 struct oce_eq *eq;
2568 struct oce_rq *rq;
2569 struct oce_wq *wq;
2570 int i, rc = 0;
2571
2572 if_setdrvflagbits(sc->ifp, IFF_DRV_RUNNING , 0);
2573
2574 oce_hw_intr_disable(sc);
2575
2576 oce_start_rx(sc);
2577
2578 for_all_rq_queues(sc, rq, i) {
2579 rc = oce_start_rq(rq);
2580 if (rc)
2581 device_printf(sc->dev, "Unable to start RX\n");
2582 }
2583
2584 for_all_wq_queues(sc, wq, i) {
2585 rc = oce_start_wq(wq);
2586 if (rc)
2587 device_printf(sc->dev, "Unable to start TX\n");
2588 }
2589
2590 for_all_evnt_queues(sc, eq, i)
2591 oce_arm_eq(sc, eq->eq_id, 0, TRUE, FALSE);
2592
2593 oce_hw_intr_enable(sc);
2594
2595 }
2596
2597 static void
process_link_state(POCE_SOFTC sc,struct oce_async_cqe_link_state * acqe)2598 process_link_state(POCE_SOFTC sc, struct oce_async_cqe_link_state *acqe)
2599 {
2600 /* Update Link status */
2601 if ((acqe->u0.s.link_status & ~ASYNC_EVENT_LOGICAL) ==
2602 ASYNC_EVENT_LINK_UP) {
2603 sc->link_status = ASYNC_EVENT_LINK_UP;
2604 if_link_state_change(sc->ifp, LINK_STATE_UP);
2605 } else {
2606 sc->link_status = ASYNC_EVENT_LINK_DOWN;
2607 if_link_state_change(sc->ifp, LINK_STATE_DOWN);
2608 }
2609 }
2610
oce_async_grp5_osbmc_process(POCE_SOFTC sc,struct oce_async_evt_grp5_os2bmc * evt)2611 static void oce_async_grp5_osbmc_process(POCE_SOFTC sc,
2612 struct oce_async_evt_grp5_os2bmc *evt)
2613 {
2614 DW_SWAP(evt, sizeof(struct oce_async_evt_grp5_os2bmc));
2615 if (evt->u.s.mgmt_enable)
2616 sc->flags |= OCE_FLAGS_OS2BMC;
2617 else
2618 return;
2619
2620 sc->bmc_filt_mask = evt->u.s.arp_filter;
2621 sc->bmc_filt_mask |= (evt->u.s.dhcp_client_filt << 1);
2622 sc->bmc_filt_mask |= (evt->u.s.dhcp_server_filt << 2);
2623 sc->bmc_filt_mask |= (evt->u.s.net_bios_filt << 3);
2624 sc->bmc_filt_mask |= (evt->u.s.bcast_filt << 4);
2625 sc->bmc_filt_mask |= (evt->u.s.ipv6_nbr_filt << 5);
2626 sc->bmc_filt_mask |= (evt->u.s.ipv6_ra_filt << 6);
2627 sc->bmc_filt_mask |= (evt->u.s.ipv6_ras_filt << 7);
2628 sc->bmc_filt_mask |= (evt->u.s.mcast_filt << 8);
2629 }
2630
oce_process_grp5_events(POCE_SOFTC sc,struct oce_mq_cqe * cqe)2631 static void oce_process_grp5_events(POCE_SOFTC sc, struct oce_mq_cqe *cqe)
2632 {
2633 struct oce_async_event_grp5_pvid_state *gcqe;
2634 struct oce_async_evt_grp5_os2bmc *bmccqe;
2635
2636 switch (cqe->u0.s.async_type) {
2637 case ASYNC_EVENT_PVID_STATE:
2638 /* GRP5 PVID */
2639 gcqe = (struct oce_async_event_grp5_pvid_state *)cqe;
2640 if (gcqe->enabled)
2641 sc->pvid = gcqe->tag & VLAN_VID_MASK;
2642 else
2643 sc->pvid = 0;
2644 break;
2645 case ASYNC_EVENT_OS2BMC:
2646 bmccqe = (struct oce_async_evt_grp5_os2bmc *)cqe;
2647 oce_async_grp5_osbmc_process(sc, bmccqe);
2648 break;
2649 default:
2650 break;
2651 }
2652 }
2653
2654 /* Handle the Completion Queue for the Mailbox/Async notifications */
2655 uint16_t
oce_mq_handler(void * arg)2656 oce_mq_handler(void *arg)
2657 {
2658 struct oce_mq *mq = (struct oce_mq *)arg;
2659 POCE_SOFTC sc = mq->parent;
2660 struct oce_cq *cq = mq->cq;
2661 int num_cqes = 0, evt_type = 0, optype = 0;
2662 struct oce_mq_cqe *cqe;
2663 struct oce_async_cqe_link_state *acqe;
2664 struct oce_async_event_qnq *dbgcqe;
2665
2666 bus_dmamap_sync(cq->ring->dma.tag,
2667 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2668 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2669
2670 while (cqe->u0.dw[3]) {
2671 DW_SWAP((uint32_t *) cqe, sizeof(oce_mq_cqe));
2672 if (cqe->u0.s.async_event) {
2673 evt_type = cqe->u0.s.event_type;
2674 optype = cqe->u0.s.async_type;
2675 if (evt_type == ASYNC_EVENT_CODE_LINK_STATE) {
2676 /* Link status evt */
2677 acqe = (struct oce_async_cqe_link_state *)cqe;
2678 process_link_state(sc, acqe);
2679 } else if (evt_type == ASYNC_EVENT_GRP5) {
2680 oce_process_grp5_events(sc, cqe);
2681 } else if (evt_type == ASYNC_EVENT_CODE_DEBUG &&
2682 optype == ASYNC_EVENT_DEBUG_QNQ) {
2683 dbgcqe = (struct oce_async_event_qnq *)cqe;
2684 if(dbgcqe->valid)
2685 sc->qnqid = dbgcqe->vlan_tag;
2686 sc->qnq_debug_event = TRUE;
2687 }
2688 }
2689 cqe->u0.dw[3] = 0;
2690 RING_GET(cq->ring, 1);
2691 bus_dmamap_sync(cq->ring->dma.tag,
2692 cq->ring->dma.map, BUS_DMASYNC_POSTWRITE);
2693 cqe = RING_GET_CONSUMER_ITEM_VA(cq->ring, struct oce_mq_cqe);
2694 num_cqes++;
2695 }
2696
2697 if (num_cqes)
2698 oce_arm_cq(sc, cq->cq_id, num_cqes, FALSE);
2699
2700 return 0;
2701 }
2702
2703 static void
setup_max_queues_want(POCE_SOFTC sc)2704 setup_max_queues_want(POCE_SOFTC sc)
2705 {
2706 /* Check if it is FLEX machine. Is so dont use RSS */
2707 if ((sc->function_mode & FNM_FLEX10_MODE) ||
2708 (sc->function_mode & FNM_UMC_MODE) ||
2709 (sc->function_mode & FNM_VNIC_MODE) ||
2710 (!is_rss_enabled(sc)) ||
2711 IS_BE2(sc)) {
2712 sc->nrqs = 1;
2713 sc->nwqs = 1;
2714 } else {
2715 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2716 sc->nwqs = MIN(OCE_NCPUS, sc->nrssqs);
2717 }
2718
2719 if (IS_BE2(sc) && is_rss_enabled(sc))
2720 sc->nrqs = MIN(OCE_NCPUS, sc->nrssqs) + 1;
2721 }
2722
2723 static void
update_queues_got(POCE_SOFTC sc)2724 update_queues_got(POCE_SOFTC sc)
2725 {
2726 if (is_rss_enabled(sc)) {
2727 sc->nrqs = sc->intr_count + 1;
2728 sc->nwqs = sc->intr_count;
2729 } else {
2730 sc->nrqs = 1;
2731 sc->nwqs = 1;
2732 }
2733
2734 if (IS_BE2(sc))
2735 sc->nwqs = 1;
2736 }
2737
2738 static int
oce_check_ipv6_ext_hdr(struct mbuf * m)2739 oce_check_ipv6_ext_hdr(struct mbuf *m)
2740 {
2741 struct ether_header *eh = mtod(m, struct ether_header *);
2742 caddr_t m_datatemp = m->m_data;
2743
2744 if (eh->ether_type == htons(ETHERTYPE_IPV6)) {
2745 m->m_data += sizeof(struct ether_header);
2746 struct ip6_hdr *ip6 = mtod(m, struct ip6_hdr *);
2747
2748 if((ip6->ip6_nxt != IPPROTO_TCP) && \
2749 (ip6->ip6_nxt != IPPROTO_UDP)){
2750 struct ip6_ext *ip6e = NULL;
2751 m->m_data += sizeof(struct ip6_hdr);
2752
2753 ip6e = (struct ip6_ext *) mtod(m, struct ip6_ext *);
2754 if(ip6e->ip6e_len == 0xff) {
2755 m->m_data = m_datatemp;
2756 return TRUE;
2757 }
2758 }
2759 m->m_data = m_datatemp;
2760 }
2761 return FALSE;
2762 }
2763
2764 static int
is_be3_a1(POCE_SOFTC sc)2765 is_be3_a1(POCE_SOFTC sc)
2766 {
2767 if((sc->flags & OCE_FLAGS_BE3) && ((sc->asic_revision & 0xFF) < 2)) {
2768 return TRUE;
2769 }
2770 return FALSE;
2771 }
2772
2773 static struct mbuf *
oce_insert_vlan_tag(POCE_SOFTC sc,struct mbuf * m,boolean_t * complete)2774 oce_insert_vlan_tag(POCE_SOFTC sc, struct mbuf *m, boolean_t *complete)
2775 {
2776 uint16_t vlan_tag = 0;
2777
2778 if(!M_WRITABLE(m))
2779 return NULL;
2780
2781 /* Embed vlan tag in the packet if it is not part of it */
2782 if(m->m_flags & M_VLANTAG) {
2783 vlan_tag = EVL_VLANOFTAG(m->m_pkthdr.ether_vtag);
2784 m->m_flags &= ~M_VLANTAG;
2785 }
2786
2787 /* if UMC, ignore vlan tag insertion and instead insert pvid */
2788 if(sc->pvid) {
2789 if(!vlan_tag)
2790 vlan_tag = sc->pvid;
2791 if (complete)
2792 *complete = FALSE;
2793 }
2794
2795 if(vlan_tag) {
2796 m = ether_vlanencap(m, vlan_tag);
2797 }
2798
2799 if(sc->qnqid) {
2800 m = ether_vlanencap(m, sc->qnqid);
2801
2802 if (complete)
2803 *complete = FALSE;
2804 }
2805 return m;
2806 }
2807
2808 static int
oce_tx_asic_stall_verify(POCE_SOFTC sc,struct mbuf * m)2809 oce_tx_asic_stall_verify(POCE_SOFTC sc, struct mbuf *m)
2810 {
2811 if(is_be3_a1(sc) && IS_QNQ_OR_UMC(sc) && \
2812 oce_check_ipv6_ext_hdr(m)) {
2813 return TRUE;
2814 }
2815 return FALSE;
2816 }
2817
2818 static void
oce_get_config(POCE_SOFTC sc)2819 oce_get_config(POCE_SOFTC sc)
2820 {
2821 int rc = 0;
2822 uint32_t max_rss = 0;
2823
2824 if ((IS_BE(sc) || IS_SH(sc)) && (!sc->be3_native))
2825 max_rss = OCE_LEGACY_MODE_RSS;
2826 else
2827 max_rss = OCE_MAX_RSS;
2828
2829 if (!IS_BE(sc)) {
2830 rc = oce_get_profile_config(sc, max_rss);
2831 if (rc) {
2832 sc->nwqs = OCE_MAX_WQ;
2833 sc->nrssqs = max_rss;
2834 sc->nrqs = sc->nrssqs + 1;
2835 }
2836 }
2837 else { /* For BE3 don't rely on fw for determining the resources */
2838 sc->nrssqs = max_rss;
2839 sc->nrqs = sc->nrssqs + 1;
2840 sc->nwqs = OCE_MAX_WQ;
2841 sc->max_vlans = MAX_VLANFILTER_SIZE;
2842 }
2843 }
2844
2845 static void
oce_rdma_close(void)2846 oce_rdma_close(void)
2847 {
2848 if (oce_rdma_if != NULL) {
2849 oce_rdma_if = NULL;
2850 }
2851 }
2852
2853 static void
oce_get_mac_addr(POCE_SOFTC sc,uint8_t * macaddr)2854 oce_get_mac_addr(POCE_SOFTC sc, uint8_t *macaddr)
2855 {
2856 memcpy(macaddr, sc->macaddr.mac_addr, 6);
2857 }
2858
2859 int
oce_register_rdma(POCE_RDMA_INFO rdma_info,POCE_RDMA_IF rdma_if)2860 oce_register_rdma(POCE_RDMA_INFO rdma_info, POCE_RDMA_IF rdma_if)
2861 {
2862 POCE_SOFTC sc;
2863 struct oce_dev_info di;
2864 int i;
2865
2866 if ((rdma_info == NULL) || (rdma_if == NULL)) {
2867 return -EINVAL;
2868 }
2869
2870 if ((rdma_info->size != OCE_RDMA_INFO_SIZE) ||
2871 (rdma_if->size != OCE_RDMA_IF_SIZE)) {
2872 return -ENXIO;
2873 }
2874
2875 rdma_info->close = oce_rdma_close;
2876 rdma_info->mbox_post = oce_mbox_post;
2877 rdma_info->common_req_hdr_init = mbx_common_req_hdr_init;
2878 rdma_info->get_mac_addr = oce_get_mac_addr;
2879
2880 oce_rdma_if = rdma_if;
2881
2882 sc = softc_head;
2883 while (sc != NULL) {
2884 if (oce_rdma_if->announce != NULL) {
2885 memset(&di, 0, sizeof(di));
2886 di.dev = sc->dev;
2887 di.softc = sc;
2888 di.ifp = sc->ifp;
2889 di.db_bhandle = sc->db_bhandle;
2890 di.db_btag = sc->db_btag;
2891 di.db_page_size = 4096;
2892 if (sc->flags & OCE_FLAGS_USING_MSIX) {
2893 di.intr_mode = OCE_INTERRUPT_MODE_MSIX;
2894 } else if (sc->flags & OCE_FLAGS_USING_MSI) {
2895 di.intr_mode = OCE_INTERRUPT_MODE_MSI;
2896 } else {
2897 di.intr_mode = OCE_INTERRUPT_MODE_INTX;
2898 }
2899 di.dev_family = OCE_GEN2_FAMILY; // fixme: must detect skyhawk
2900 if (di.intr_mode != OCE_INTERRUPT_MODE_INTX) {
2901 di.msix.num_vectors = sc->intr_count + sc->roce_intr_count;
2902 di.msix.start_vector = sc->intr_count;
2903 for (i=0; i<di.msix.num_vectors; i++) {
2904 di.msix.vector_list[i] = sc->intrs[i].vector;
2905 }
2906 } else {
2907 }
2908 memcpy(di.mac_addr, sc->macaddr.mac_addr, 6);
2909 di.vendor_id = pci_get_vendor(sc->dev);
2910 di.dev_id = pci_get_device(sc->dev);
2911
2912 if (sc->rdma_flags & OCE_RDMA_FLAG_SUPPORTED) {
2913 di.flags |= OCE_RDMA_INFO_RDMA_SUPPORTED;
2914 }
2915
2916 rdma_if->announce(&di);
2917 sc = sc->next;
2918 }
2919 }
2920
2921 return 0;
2922 }
2923
2924 static void
oce_read_env_variables(POCE_SOFTC sc)2925 oce_read_env_variables( POCE_SOFTC sc )
2926 {
2927 char *value = NULL;
2928 int rc = 0;
2929
2930 /* read if user wants to enable hwlro or swlro */
2931 //value = getenv("oce_enable_hwlro");
2932 if(value && IS_SH(sc)) {
2933 sc->enable_hwlro = strtol(value, NULL, 10);
2934 if(sc->enable_hwlro) {
2935 rc = oce_mbox_nic_query_lro_capabilities(sc, NULL, NULL);
2936 if(rc) {
2937 device_printf(sc->dev, "no hardware lro support\n");
2938 device_printf(sc->dev, "software lro enabled\n");
2939 sc->enable_hwlro = 0;
2940 }else {
2941 device_printf(sc->dev, "hardware lro enabled\n");
2942 oce_max_rsp_handled = 32;
2943 }
2944 }else {
2945 device_printf(sc->dev, "software lro enabled\n");
2946 }
2947 }else {
2948 sc->enable_hwlro = 0;
2949 }
2950
2951 /* read mbuf size */
2952 //value = getenv("oce_rq_buf_size");
2953 if(value && IS_SH(sc)) {
2954 oce_rq_buf_size = strtol(value, NULL, 10);
2955 switch(oce_rq_buf_size) {
2956 case 2048:
2957 case 4096:
2958 case 9216:
2959 case 16384:
2960 break;
2961
2962 default:
2963 device_printf(sc->dev, " Supported oce_rq_buf_size values are 2K, 4K, 9K, 16K \n");
2964 oce_rq_buf_size = 2048;
2965 }
2966 }
2967
2968 return;
2969 }
2970