1394324d3SSascha Wildner /*-
2394324d3SSascha Wildner * BSD LICENSE
3394324d3SSascha Wildner *
4394324d3SSascha Wildner * Copyright (c) 2015-2017 Amazon.com, Inc. or its affiliates.
5394324d3SSascha Wildner * All rights reserved.
6394324d3SSascha Wildner *
7394324d3SSascha Wildner * Redistribution and use in source and binary forms, with or without
8394324d3SSascha Wildner * modification, are permitted provided that the following conditions
9394324d3SSascha Wildner * are met:
10394324d3SSascha Wildner *
11394324d3SSascha Wildner * 1. Redistributions of source code must retain the above copyright
12394324d3SSascha Wildner * notice, this list of conditions and the following disclaimer.
13394324d3SSascha Wildner *
14394324d3SSascha Wildner * 2. Redistributions in binary form must reproduce the above copyright
15394324d3SSascha Wildner * notice, this list of conditions and the following disclaimer in the
16394324d3SSascha Wildner * documentation and/or other materials provided with the distribution.
17394324d3SSascha Wildner *
18394324d3SSascha Wildner * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19394324d3SSascha Wildner * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20394324d3SSascha Wildner * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21394324d3SSascha Wildner * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22394324d3SSascha Wildner * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23394324d3SSascha Wildner * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24394324d3SSascha Wildner * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25394324d3SSascha Wildner * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26394324d3SSascha Wildner * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27394324d3SSascha Wildner * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28394324d3SSascha Wildner * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
2982a3fa28SBrad Hoffman *
3082a3fa28SBrad Hoffman * $FreeBSD: head/sys/dev/ena/ena.c 325593 2017-11-09 13:38:17Z mw $
31394324d3SSascha Wildner */
32394324d3SSascha Wildner
33394324d3SSascha Wildner #include <sys/param.h>
34394324d3SSascha Wildner #include <sys/systm.h>
35394324d3SSascha Wildner #include <sys/bus.h>
36394324d3SSascha Wildner #include <sys/endian.h>
37394324d3SSascha Wildner #include <sys/kernel.h>
38394324d3SSascha Wildner #include <sys/kthread.h>
39394324d3SSascha Wildner #include <sys/malloc.h>
40394324d3SSascha Wildner #include <sys/mbuf.h>
41394324d3SSascha Wildner #include <sys/module.h>
42394324d3SSascha Wildner #include <sys/rman.h>
43394324d3SSascha Wildner #include <sys/socket.h>
44394324d3SSascha Wildner #include <sys/sockio.h>
45394324d3SSascha Wildner #include <sys/sysctl.h>
46394324d3SSascha Wildner #include <sys/taskqueue.h>
47394324d3SSascha Wildner #include <sys/time.h>
48394324d3SSascha Wildner #include <sys/eventhandler.h>
49394324d3SSascha Wildner
50394324d3SSascha Wildner #include <net/bpf.h>
51394324d3SSascha Wildner #include <net/if.h>
52394324d3SSascha Wildner #include <net/if_var.h>
53394324d3SSascha Wildner #include <net/if_arp.h>
54394324d3SSascha Wildner #include <net/if_dl.h>
55394324d3SSascha Wildner #include <net/if_media.h>
56394324d3SSascha Wildner #include <net/if_types.h>
5782a3fa28SBrad Hoffman #include <net/ifq_var.h>
5882a3fa28SBrad Hoffman #include <net/vlan/if_vlan_var.h>
59394324d3SSascha Wildner
60394324d3SSascha Wildner #include <netinet/in_systm.h>
61394324d3SSascha Wildner #include <netinet/in.h>
62394324d3SSascha Wildner #include <netinet/if_ether.h>
63394324d3SSascha Wildner #include <netinet/ip.h>
64394324d3SSascha Wildner #include <netinet/ip6.h>
65394324d3SSascha Wildner #include <netinet/tcp.h>
66394324d3SSascha Wildner #include <netinet/udp.h>
67394324d3SSascha Wildner
6882a3fa28SBrad Hoffman #include <bus/pci/pcivar.h>
6982a3fa28SBrad Hoffman #include <bus/pci/pcireg.h>
70394324d3SSascha Wildner
71394324d3SSascha Wildner #include "ena.h"
72394324d3SSascha Wildner #include "ena_sysctl.h"
73394324d3SSascha Wildner
74394324d3SSascha Wildner /*********************************************************
75394324d3SSascha Wildner * Function prototypes
76394324d3SSascha Wildner *********************************************************/
77394324d3SSascha Wildner static int ena_probe(device_t);
78394324d3SSascha Wildner static void ena_intr_msix_mgmnt(void *);
79394324d3SSascha Wildner static int ena_allocate_pci_resources(struct ena_adapter*);
80394324d3SSascha Wildner static void ena_free_pci_resources(struct ena_adapter *);
81394324d3SSascha Wildner static int ena_change_mtu(if_t, int);
8282a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
83394324d3SSascha Wildner static inline void ena_alloc_counters(counter_u64_t *, int);
84394324d3SSascha Wildner static inline void ena_free_counters(counter_u64_t *, int);
85394324d3SSascha Wildner static inline void ena_reset_counters(counter_u64_t *, int);
8682a3fa28SBrad Hoffman #endif
87394324d3SSascha Wildner static void ena_init_io_rings_common(struct ena_adapter *,
88394324d3SSascha Wildner struct ena_ring *, uint16_t);
89394324d3SSascha Wildner static void ena_init_io_rings(struct ena_adapter *);
90394324d3SSascha Wildner static void ena_free_io_ring_resources(struct ena_adapter *, unsigned int);
91394324d3SSascha Wildner static void ena_free_all_io_rings_resources(struct ena_adapter *);
92394324d3SSascha Wildner static int ena_setup_tx_dma_tag(struct ena_adapter *);
93394324d3SSascha Wildner static int ena_free_tx_dma_tag(struct ena_adapter *);
94394324d3SSascha Wildner static int ena_setup_rx_dma_tag(struct ena_adapter *);
95394324d3SSascha Wildner static int ena_free_rx_dma_tag(struct ena_adapter *);
96394324d3SSascha Wildner static int ena_setup_tx_resources(struct ena_adapter *, int);
97394324d3SSascha Wildner static void ena_free_tx_resources(struct ena_adapter *, int);
98394324d3SSascha Wildner static int ena_setup_all_tx_resources(struct ena_adapter *);
99394324d3SSascha Wildner static void ena_free_all_tx_resources(struct ena_adapter *);
100394324d3SSascha Wildner static inline int validate_rx_req_id(struct ena_ring *, uint16_t);
101394324d3SSascha Wildner static int ena_setup_rx_resources(struct ena_adapter *, unsigned int);
102394324d3SSascha Wildner static void ena_free_rx_resources(struct ena_adapter *, unsigned int);
103394324d3SSascha Wildner static int ena_setup_all_rx_resources(struct ena_adapter *);
104394324d3SSascha Wildner static void ena_free_all_rx_resources(struct ena_adapter *);
105394324d3SSascha Wildner static inline int ena_alloc_rx_mbuf(struct ena_adapter *, struct ena_ring *,
106394324d3SSascha Wildner struct ena_rx_buffer *);
107394324d3SSascha Wildner static void ena_free_rx_mbuf(struct ena_adapter *, struct ena_ring *,
108394324d3SSascha Wildner struct ena_rx_buffer *);
109394324d3SSascha Wildner static int ena_refill_rx_bufs(struct ena_ring *, uint32_t);
110394324d3SSascha Wildner static void ena_free_rx_bufs(struct ena_adapter *, unsigned int);
111394324d3SSascha Wildner static void ena_refill_all_rx_bufs(struct ena_adapter *);
112394324d3SSascha Wildner static void ena_free_all_rx_bufs(struct ena_adapter *);
113394324d3SSascha Wildner static void ena_free_tx_bufs(struct ena_adapter *, unsigned int);
114394324d3SSascha Wildner static void ena_free_all_tx_bufs(struct ena_adapter *);
115394324d3SSascha Wildner static void ena_destroy_all_tx_queues(struct ena_adapter *);
116394324d3SSascha Wildner static void ena_destroy_all_rx_queues(struct ena_adapter *);
117394324d3SSascha Wildner static void ena_destroy_all_io_queues(struct ena_adapter *);
118394324d3SSascha Wildner static int ena_create_io_queues(struct ena_adapter *);
119394324d3SSascha Wildner static int ena_tx_cleanup(struct ena_ring *);
120394324d3SSascha Wildner static void ena_deferred_rx_cleanup(void *, int);
121394324d3SSascha Wildner static int ena_rx_cleanup(struct ena_ring *);
122394324d3SSascha Wildner static inline int validate_tx_req_id(struct ena_ring *, uint16_t);
123394324d3SSascha Wildner static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
124394324d3SSascha Wildner struct mbuf *);
125394324d3SSascha Wildner static struct mbuf* ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
126394324d3SSascha Wildner struct ena_com_rx_ctx *, uint16_t *);
127394324d3SSascha Wildner static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
128394324d3SSascha Wildner struct mbuf *);
129394324d3SSascha Wildner static void ena_handle_msix(void *);
130394324d3SSascha Wildner static int ena_enable_msix(struct ena_adapter *);
131394324d3SSascha Wildner static void ena_setup_mgmnt_intr(struct ena_adapter *);
132394324d3SSascha Wildner static void ena_setup_io_intr(struct ena_adapter *);
133394324d3SSascha Wildner static int ena_request_mgmnt_irq(struct ena_adapter *);
134394324d3SSascha Wildner static int ena_request_io_irq(struct ena_adapter *);
135394324d3SSascha Wildner static void ena_free_mgmnt_irq(struct ena_adapter *);
136394324d3SSascha Wildner static void ena_free_io_irq(struct ena_adapter *);
137394324d3SSascha Wildner static void ena_free_irqs(struct ena_adapter*);
138394324d3SSascha Wildner static void ena_disable_msix(struct ena_adapter *);
139394324d3SSascha Wildner static void ena_unmask_all_io_irqs(struct ena_adapter *);
140394324d3SSascha Wildner static int ena_rss_configure(struct ena_adapter *);
141394324d3SSascha Wildner static int ena_up_complete(struct ena_adapter *);
142394324d3SSascha Wildner static int ena_up(struct ena_adapter *);
143394324d3SSascha Wildner static void ena_down(struct ena_adapter *);
14482a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
145394324d3SSascha Wildner static uint64_t ena_get_counter(if_t, ift_counter);
14682a3fa28SBrad Hoffman #endif
147394324d3SSascha Wildner static int ena_media_change(if_t);
148394324d3SSascha Wildner static void ena_media_status(if_t, struct ifmediareq *);
149394324d3SSascha Wildner static void ena_init(void *);
15082a3fa28SBrad Hoffman static int ena_ioctl(if_t, u_long, caddr_t, struct ucred *);
151394324d3SSascha Wildner static int ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *);
152394324d3SSascha Wildner static void ena_update_host_info(struct ena_admin_host_info *, if_t);
153394324d3SSascha Wildner static void ena_update_hwassist(struct ena_adapter *);
154394324d3SSascha Wildner static int ena_setup_ifnet(device_t, struct ena_adapter *,
155394324d3SSascha Wildner struct ena_com_dev_get_features_ctx *);
156394324d3SSascha Wildner static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *);
157394324d3SSascha Wildner static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
158394324d3SSascha Wildner struct mbuf **mbuf);
159394324d3SSascha Wildner static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
16082a3fa28SBrad Hoffman static void ena_start_xmit(struct ifnet *, struct ifaltq_subque *);
161394324d3SSascha Wildner static int ena_calc_io_queue_num(struct ena_adapter *,
162394324d3SSascha Wildner struct ena_com_dev_get_features_ctx *);
163394324d3SSascha Wildner static int ena_calc_queue_size(struct ena_adapter *, uint16_t *,
164394324d3SSascha Wildner uint16_t *, struct ena_com_dev_get_features_ctx *);
165394324d3SSascha Wildner static int ena_rss_init_default(struct ena_adapter *);
166394324d3SSascha Wildner static void ena_rss_init_default_deferred(void *);
167394324d3SSascha Wildner static void ena_config_host_info(struct ena_com_dev *);
168394324d3SSascha Wildner static int ena_attach(device_t);
169394324d3SSascha Wildner static int ena_detach(device_t);
170394324d3SSascha Wildner static int ena_device_init(struct ena_adapter *, device_t,
171394324d3SSascha Wildner struct ena_com_dev_get_features_ctx *, int *);
172394324d3SSascha Wildner static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *,
173394324d3SSascha Wildner int);
174394324d3SSascha Wildner static void ena_update_on_link_change(void *, struct ena_admin_aenq_entry *);
175394324d3SSascha Wildner static void unimplemented_aenq_handler(void *,
176394324d3SSascha Wildner struct ena_admin_aenq_entry *);
177394324d3SSascha Wildner static void ena_timer_service(void *);
178394324d3SSascha Wildner
179394324d3SSascha Wildner static char ena_version[] = DEVICE_NAME DRV_MODULE_NAME " v" DRV_MODULE_VERSION;
180394324d3SSascha Wildner
181394324d3SSascha Wildner static SYSCTL_NODE(_hw, OID_AUTO, ena, CTLFLAG_RD, 0, "ENA driver parameters");
182394324d3SSascha Wildner
183394324d3SSascha Wildner /*
184394324d3SSascha Wildner * Logging level for changing verbosity of the output
185394324d3SSascha Wildner */
186394324d3SSascha Wildner int ena_log_level = ENA_ALERT | ENA_WARNING;
18782a3fa28SBrad Hoffman TUNABLE_INT("hw.ena.ena_log_level", &ena_log_level);
18882a3fa28SBrad Hoffman SYSCTL_INT(_hw_ena, OID_AUTO, log_level, CTLFLAG_RW,
189394324d3SSascha Wildner &ena_log_level, 0, "Logging level indicating verbosity of the logs");
190394324d3SSascha Wildner
191394324d3SSascha Wildner static ena_vendor_info_t ena_vendor_info_array[] = {
192394324d3SSascha Wildner { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_PF, 0},
193394324d3SSascha Wildner { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_PF, 0},
194394324d3SSascha Wildner { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_VF, 0},
195394324d3SSascha Wildner { PCI_VENDOR_ID_AMAZON, PCI_DEV_ID_ENA_LLQ_VF, 0},
196394324d3SSascha Wildner /* Last entry */
197394324d3SSascha Wildner { 0, 0, 0 }
198394324d3SSascha Wildner };
199394324d3SSascha Wildner
200394324d3SSascha Wildner /*
201394324d3SSascha Wildner * Contains pointers to event handlers, e.g. link state chage.
202394324d3SSascha Wildner */
203394324d3SSascha Wildner static struct ena_aenq_handlers aenq_handlers;
204394324d3SSascha Wildner
205394324d3SSascha Wildner void
ena_dmamap_callback(void * arg,bus_dma_segment_t * segs,int nseg,int error)206394324d3SSascha Wildner ena_dmamap_callback(void *arg, bus_dma_segment_t *segs, int nseg, int error)
207394324d3SSascha Wildner {
208394324d3SSascha Wildner if (error != 0)
209394324d3SSascha Wildner return;
210394324d3SSascha Wildner *(bus_addr_t *) arg = segs[0].ds_addr;
211394324d3SSascha Wildner }
212394324d3SSascha Wildner
213394324d3SSascha Wildner int
ena_dma_alloc(device_t dmadev,bus_size_t size,ena_mem_handle_t * dma,int mapflags)214394324d3SSascha Wildner ena_dma_alloc(device_t dmadev, bus_size_t size,
215394324d3SSascha Wildner ena_mem_handle_t *dma , int mapflags)
216394324d3SSascha Wildner {
217394324d3SSascha Wildner struct ena_adapter* adapter = device_get_softc(dmadev);
218394324d3SSascha Wildner uint32_t maxsize;
219394324d3SSascha Wildner uint64_t dma_space_addr;
220394324d3SSascha Wildner int error;
221394324d3SSascha Wildner
222394324d3SSascha Wildner maxsize = ((size - 1) / PAGE_SIZE + 1) * PAGE_SIZE;
223394324d3SSascha Wildner
224394324d3SSascha Wildner dma_space_addr = ENA_DMA_BIT_MASK(adapter->dma_width);
225394324d3SSascha Wildner if (unlikely(dma_space_addr == 0))
226394324d3SSascha Wildner dma_space_addr = BUS_SPACE_MAXADDR;
227394324d3SSascha Wildner
228394324d3SSascha Wildner error = bus_dma_tag_create(bus_get_dma_tag(dmadev), /* parent */
229394324d3SSascha Wildner 8, 0, /* alignment, bounds */
230394324d3SSascha Wildner dma_space_addr, /* lowaddr of exclusion window */
231394324d3SSascha Wildner BUS_SPACE_MAXADDR,/* highaddr of exclusion window */
232394324d3SSascha Wildner maxsize, /* maxsize */
233394324d3SSascha Wildner 1, /* nsegments */
234394324d3SSascha Wildner maxsize, /* maxsegsize */
235394324d3SSascha Wildner BUS_DMA_ALLOCNOW, /* flags */
236394324d3SSascha Wildner &dma->tag);
237394324d3SSascha Wildner if (unlikely(error != 0)) {
238394324d3SSascha Wildner ena_trace(ENA_ALERT, "bus_dma_tag_create failed: %d\n", error);
239394324d3SSascha Wildner goto fail_tag;
240394324d3SSascha Wildner }
241394324d3SSascha Wildner
242394324d3SSascha Wildner error = bus_dmamem_alloc(dma->tag, (void**) &dma->vaddr,
243394324d3SSascha Wildner BUS_DMA_COHERENT | BUS_DMA_ZERO, &dma->map);
244394324d3SSascha Wildner if (unlikely(error != 0)) {
245394324d3SSascha Wildner ena_trace(ENA_ALERT, "bus_dmamem_alloc(%ju) failed: %d\n",
246394324d3SSascha Wildner (uintmax_t)size, error);
247394324d3SSascha Wildner goto fail_map_create;
248394324d3SSascha Wildner }
249394324d3SSascha Wildner
250394324d3SSascha Wildner dma->paddr = 0;
251394324d3SSascha Wildner error = bus_dmamap_load(dma->tag, dma->map, dma->vaddr,
252394324d3SSascha Wildner size, ena_dmamap_callback, &dma->paddr, mapflags);
253394324d3SSascha Wildner if (unlikely((error != 0) || (dma->paddr == 0))) {
254394324d3SSascha Wildner ena_trace(ENA_ALERT, ": bus_dmamap_load failed: %d\n", error);
255394324d3SSascha Wildner goto fail_map_load;
256394324d3SSascha Wildner }
257394324d3SSascha Wildner
258394324d3SSascha Wildner return (0);
259394324d3SSascha Wildner
260394324d3SSascha Wildner fail_map_load:
261394324d3SSascha Wildner bus_dmamem_free(dma->tag, dma->vaddr, dma->map);
262394324d3SSascha Wildner fail_map_create:
263394324d3SSascha Wildner bus_dma_tag_destroy(dma->tag);
264394324d3SSascha Wildner fail_tag:
265394324d3SSascha Wildner dma->tag = NULL;
266394324d3SSascha Wildner
267394324d3SSascha Wildner return (error);
268394324d3SSascha Wildner }
269394324d3SSascha Wildner
270394324d3SSascha Wildner static int
ena_allocate_pci_resources(struct ena_adapter * adapter)271394324d3SSascha Wildner ena_allocate_pci_resources(struct ena_adapter* adapter)
272394324d3SSascha Wildner {
273394324d3SSascha Wildner device_t pdev = adapter->pdev;
274394324d3SSascha Wildner int rid;
275394324d3SSascha Wildner
276394324d3SSascha Wildner rid = PCIR_BAR(ENA_REG_BAR);
277394324d3SSascha Wildner adapter->memory = NULL;
278394324d3SSascha Wildner adapter->registers = bus_alloc_resource_any(pdev, SYS_RES_MEMORY,
279394324d3SSascha Wildner &rid, RF_ACTIVE);
280394324d3SSascha Wildner if (unlikely(adapter->registers == NULL)) {
281394324d3SSascha Wildner device_printf(pdev, "Unable to allocate bus resource: "
282394324d3SSascha Wildner "registers\n");
283394324d3SSascha Wildner return (ENXIO);
284394324d3SSascha Wildner }
285394324d3SSascha Wildner
286394324d3SSascha Wildner return (0);
287394324d3SSascha Wildner }
288394324d3SSascha Wildner
289394324d3SSascha Wildner static void
ena_free_pci_resources(struct ena_adapter * adapter)290394324d3SSascha Wildner ena_free_pci_resources(struct ena_adapter *adapter)
291394324d3SSascha Wildner {
292394324d3SSascha Wildner device_t pdev = adapter->pdev;
293394324d3SSascha Wildner
294394324d3SSascha Wildner if (adapter->memory != NULL) {
295394324d3SSascha Wildner bus_release_resource(pdev, SYS_RES_MEMORY,
296394324d3SSascha Wildner PCIR_BAR(ENA_MEM_BAR), adapter->memory);
297394324d3SSascha Wildner }
298394324d3SSascha Wildner
299394324d3SSascha Wildner if (adapter->registers != NULL) {
300394324d3SSascha Wildner bus_release_resource(pdev, SYS_RES_MEMORY,
301394324d3SSascha Wildner PCIR_BAR(ENA_REG_BAR), adapter->registers);
302394324d3SSascha Wildner }
303394324d3SSascha Wildner }
304394324d3SSascha Wildner
305394324d3SSascha Wildner static int
ena_probe(device_t dev)306394324d3SSascha Wildner ena_probe(device_t dev)
307394324d3SSascha Wildner {
308394324d3SSascha Wildner ena_vendor_info_t *ent;
309394324d3SSascha Wildner char adapter_name[60];
310394324d3SSascha Wildner uint16_t pci_vendor_id = 0;
311394324d3SSascha Wildner uint16_t pci_device_id = 0;
312394324d3SSascha Wildner
313394324d3SSascha Wildner pci_vendor_id = pci_get_vendor(dev);
314394324d3SSascha Wildner pci_device_id = pci_get_device(dev);
315394324d3SSascha Wildner
316394324d3SSascha Wildner ent = ena_vendor_info_array;
317394324d3SSascha Wildner while (ent->vendor_id != 0) {
318394324d3SSascha Wildner if ((pci_vendor_id == ent->vendor_id) &&
319394324d3SSascha Wildner (pci_device_id == ent->device_id)) {
320394324d3SSascha Wildner ena_trace(ENA_DBG, "vendor=%x device=%x ",
321394324d3SSascha Wildner pci_vendor_id, pci_device_id);
322394324d3SSascha Wildner
32382a3fa28SBrad Hoffman ksprintf(adapter_name, DEVICE_DESC);
324394324d3SSascha Wildner device_set_desc_copy(dev, adapter_name);
325394324d3SSascha Wildner return (BUS_PROBE_DEFAULT);
326394324d3SSascha Wildner }
327394324d3SSascha Wildner
328394324d3SSascha Wildner ent++;
329394324d3SSascha Wildner
330394324d3SSascha Wildner }
331394324d3SSascha Wildner
332394324d3SSascha Wildner return (ENXIO);
333394324d3SSascha Wildner }
334394324d3SSascha Wildner
335394324d3SSascha Wildner static int
ena_change_mtu(if_t ifp,int new_mtu)336394324d3SSascha Wildner ena_change_mtu(if_t ifp, int new_mtu)
337394324d3SSascha Wildner {
33882a3fa28SBrad Hoffman struct ena_adapter *adapter = ifp->if_softc;
339394324d3SSascha Wildner int rc;
340394324d3SSascha Wildner
341394324d3SSascha Wildner if ((new_mtu > adapter->max_mtu) || (new_mtu < ENA_MIN_MTU)) {
342394324d3SSascha Wildner device_printf(adapter->pdev, "Invalid MTU setting. "
343394324d3SSascha Wildner "new_mtu: %d max mtu: %d min mtu: %d\n",
344394324d3SSascha Wildner new_mtu, adapter->max_mtu, ENA_MIN_MTU);
345394324d3SSascha Wildner return (EINVAL);
346394324d3SSascha Wildner }
347394324d3SSascha Wildner
348394324d3SSascha Wildner rc = ena_com_set_dev_mtu(adapter->ena_dev, new_mtu);
349394324d3SSascha Wildner if (likely(rc == 0)) {
350394324d3SSascha Wildner ena_trace(ENA_DBG, "set MTU to %d\n", new_mtu);
35182a3fa28SBrad Hoffman ifp->if_mtu = new_mtu;
352394324d3SSascha Wildner } else {
353394324d3SSascha Wildner device_printf(adapter->pdev, "Failed to set MTU to %d\n",
354394324d3SSascha Wildner new_mtu);
355394324d3SSascha Wildner }
356394324d3SSascha Wildner
357394324d3SSascha Wildner return (rc);
358394324d3SSascha Wildner }
359394324d3SSascha Wildner
36082a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
361394324d3SSascha Wildner static inline void
362394324d3SSascha Wildner ena_alloc_counters(counter_u64_t *begin, int size)
363394324d3SSascha Wildner {
364394324d3SSascha Wildner counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
365394324d3SSascha Wildner
366394324d3SSascha Wildner for (; begin < end; ++begin)
367394324d3SSascha Wildner *begin = counter_u64_alloc(M_WAITOK);
368394324d3SSascha Wildner }
369394324d3SSascha Wildner
370394324d3SSascha Wildner static inline void
371394324d3SSascha Wildner ena_free_counters(counter_u64_t *begin, int size)
372394324d3SSascha Wildner {
373394324d3SSascha Wildner counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
374394324d3SSascha Wildner
375394324d3SSascha Wildner for (; begin < end; ++begin)
376394324d3SSascha Wildner counter_u64_free(*begin);
377394324d3SSascha Wildner }
378394324d3SSascha Wildner
379394324d3SSascha Wildner static inline void
380394324d3SSascha Wildner ena_reset_counters(counter_u64_t *begin, int size)
381394324d3SSascha Wildner {
382394324d3SSascha Wildner counter_u64_t *end = (counter_u64_t *)((char *)begin + size);
383394324d3SSascha Wildner
384394324d3SSascha Wildner for (; begin < end; ++begin)
385394324d3SSascha Wildner counter_u64_zero(*begin);
386394324d3SSascha Wildner }
38782a3fa28SBrad Hoffman #endif
388394324d3SSascha Wildner
389394324d3SSascha Wildner static void
ena_init_io_rings_common(struct ena_adapter * adapter,struct ena_ring * ring,uint16_t qid)390394324d3SSascha Wildner ena_init_io_rings_common(struct ena_adapter *adapter, struct ena_ring *ring,
391394324d3SSascha Wildner uint16_t qid)
392394324d3SSascha Wildner {
393394324d3SSascha Wildner
394394324d3SSascha Wildner ring->qid = qid;
395394324d3SSascha Wildner ring->adapter = adapter;
396394324d3SSascha Wildner ring->ena_dev = adapter->ena_dev;
397394324d3SSascha Wildner }
398394324d3SSascha Wildner
399394324d3SSascha Wildner static void
ena_init_io_rings(struct ena_adapter * adapter)400394324d3SSascha Wildner ena_init_io_rings(struct ena_adapter *adapter)
401394324d3SSascha Wildner {
402394324d3SSascha Wildner struct ena_com_dev *ena_dev;
403394324d3SSascha Wildner struct ena_ring *txr, *rxr;
404394324d3SSascha Wildner struct ena_que *que;
405394324d3SSascha Wildner int i;
406394324d3SSascha Wildner
407394324d3SSascha Wildner ena_dev = adapter->ena_dev;
408394324d3SSascha Wildner
409394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
410394324d3SSascha Wildner txr = &adapter->tx_ring[i];
411394324d3SSascha Wildner rxr = &adapter->rx_ring[i];
412394324d3SSascha Wildner
413394324d3SSascha Wildner /* TX/RX common ring state */
414394324d3SSascha Wildner ena_init_io_rings_common(adapter, txr, i);
415394324d3SSascha Wildner ena_init_io_rings_common(adapter, rxr, i);
416394324d3SSascha Wildner
417394324d3SSascha Wildner /* TX specific ring state */
418394324d3SSascha Wildner txr->ring_size = adapter->tx_ring_size;
419394324d3SSascha Wildner txr->tx_max_header_size = ena_dev->tx_max_header_size;
420394324d3SSascha Wildner txr->tx_mem_queue_type = ena_dev->tx_mem_queue_type;
421394324d3SSascha Wildner txr->smoothed_interval =
422394324d3SSascha Wildner ena_com_get_nonadaptive_moderation_interval_tx(ena_dev);
423394324d3SSascha Wildner
42482a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
425394324d3SSascha Wildner /* Alloc TX statistics. */
426394324d3SSascha Wildner ena_alloc_counters((counter_u64_t *)&txr->tx_stats,
427394324d3SSascha Wildner sizeof(txr->tx_stats));
42882a3fa28SBrad Hoffman #endif
429394324d3SSascha Wildner
430394324d3SSascha Wildner /* RX specific ring state */
431394324d3SSascha Wildner rxr->ring_size = adapter->rx_ring_size;
432394324d3SSascha Wildner rxr->smoothed_interval =
433394324d3SSascha Wildner ena_com_get_nonadaptive_moderation_interval_rx(ena_dev);
434394324d3SSascha Wildner
43582a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
436394324d3SSascha Wildner /* Alloc RX statistics. */
437394324d3SSascha Wildner ena_alloc_counters((counter_u64_t *)&rxr->rx_stats,
438394324d3SSascha Wildner sizeof(rxr->rx_stats));
43982a3fa28SBrad Hoffman #endif
440394324d3SSascha Wildner
441394324d3SSascha Wildner /* Initialize locks */
44282a3fa28SBrad Hoffman ksnprintf(txr->lock_name, nitems(txr->lock_name), "%s:tx(%d)",
443394324d3SSascha Wildner device_get_nameunit(adapter->pdev), i);
44482a3fa28SBrad Hoffman ksnprintf(rxr->lock_name, nitems(rxr->lock_name), "%s:rx(%d)",
445394324d3SSascha Wildner device_get_nameunit(adapter->pdev), i);
446394324d3SSascha Wildner
44782a3fa28SBrad Hoffman lockinit(&txr->ring_lock, txr->lock_name, 0, LK_CANRECURSE);
44882a3fa28SBrad Hoffman lockinit(&rxr->ring_lock, rxr->lock_name, 0, LK_CANRECURSE);
449394324d3SSascha Wildner
450394324d3SSascha Wildner que = &adapter->que[i];
451394324d3SSascha Wildner que->adapter = adapter;
452394324d3SSascha Wildner que->id = i;
453394324d3SSascha Wildner que->tx_ring = txr;
454394324d3SSascha Wildner que->rx_ring = rxr;
455394324d3SSascha Wildner
456394324d3SSascha Wildner txr->que = que;
457394324d3SSascha Wildner rxr->que = que;
458394324d3SSascha Wildner
459394324d3SSascha Wildner rxr->empty_rx_queue = 0;
460394324d3SSascha Wildner }
461394324d3SSascha Wildner }
462394324d3SSascha Wildner
463394324d3SSascha Wildner static void
ena_free_io_ring_resources(struct ena_adapter * adapter,unsigned int qid)464394324d3SSascha Wildner ena_free_io_ring_resources(struct ena_adapter *adapter, unsigned int qid)
465394324d3SSascha Wildner {
466394324d3SSascha Wildner struct ena_ring *txr = &adapter->tx_ring[qid];
467394324d3SSascha Wildner struct ena_ring *rxr = &adapter->rx_ring[qid];
468394324d3SSascha Wildner
46982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
470394324d3SSascha Wildner ena_free_counters((counter_u64_t *)&txr->tx_stats,
471394324d3SSascha Wildner sizeof(txr->tx_stats));
472394324d3SSascha Wildner ena_free_counters((counter_u64_t *)&rxr->rx_stats,
473394324d3SSascha Wildner sizeof(rxr->rx_stats));
47482a3fa28SBrad Hoffman #endif
475394324d3SSascha Wildner
47682a3fa28SBrad Hoffman lockuninit(&txr->ring_lock);
47782a3fa28SBrad Hoffman lockuninit(&rxr->ring_lock);
478394324d3SSascha Wildner }
479394324d3SSascha Wildner
480394324d3SSascha Wildner static void
ena_free_all_io_rings_resources(struct ena_adapter * adapter)481394324d3SSascha Wildner ena_free_all_io_rings_resources(struct ena_adapter *adapter)
482394324d3SSascha Wildner {
483394324d3SSascha Wildner int i;
484394324d3SSascha Wildner
485394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++)
486394324d3SSascha Wildner ena_free_io_ring_resources(adapter, i);
487394324d3SSascha Wildner
488394324d3SSascha Wildner }
489394324d3SSascha Wildner
490394324d3SSascha Wildner static int
ena_setup_tx_dma_tag(struct ena_adapter * adapter)491394324d3SSascha Wildner ena_setup_tx_dma_tag(struct ena_adapter *adapter)
492394324d3SSascha Wildner {
493394324d3SSascha Wildner int ret;
494394324d3SSascha Wildner
495394324d3SSascha Wildner /* Create DMA tag for Tx buffers */
496394324d3SSascha Wildner ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev),
497394324d3SSascha Wildner 1, 0, /* alignment, bounds */
498394324d3SSascha Wildner ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
499394324d3SSascha Wildner BUS_SPACE_MAXADDR, /* highaddr of excl window */
500394324d3SSascha Wildner ENA_TSO_MAXSIZE, /* maxsize */
50182a3fa28SBrad Hoffman ENA_BUS_DMA_SEGS, /* nsegments */
502394324d3SSascha Wildner ENA_TSO_MAXSIZE, /* maxsegsize */
503394324d3SSascha Wildner 0, /* flags */
504394324d3SSascha Wildner &adapter->tx_buf_tag);
505394324d3SSascha Wildner
506394324d3SSascha Wildner return (ret);
507394324d3SSascha Wildner }
508394324d3SSascha Wildner
509394324d3SSascha Wildner static int
ena_free_tx_dma_tag(struct ena_adapter * adapter)510394324d3SSascha Wildner ena_free_tx_dma_tag(struct ena_adapter *adapter)
511394324d3SSascha Wildner {
512394324d3SSascha Wildner int ret;
513394324d3SSascha Wildner
514394324d3SSascha Wildner ret = bus_dma_tag_destroy(adapter->tx_buf_tag);
515394324d3SSascha Wildner
516394324d3SSascha Wildner if (likely(ret == 0))
517394324d3SSascha Wildner adapter->tx_buf_tag = NULL;
518394324d3SSascha Wildner
519394324d3SSascha Wildner return (ret);
520394324d3SSascha Wildner }
521394324d3SSascha Wildner
522394324d3SSascha Wildner static int
ena_setup_rx_dma_tag(struct ena_adapter * adapter)523394324d3SSascha Wildner ena_setup_rx_dma_tag(struct ena_adapter *adapter)
524394324d3SSascha Wildner {
525394324d3SSascha Wildner int ret;
526394324d3SSascha Wildner
527394324d3SSascha Wildner /* Create DMA tag for Rx buffers*/
528394324d3SSascha Wildner ret = bus_dma_tag_create(bus_get_dma_tag(adapter->pdev), /* parent */
529394324d3SSascha Wildner 1, 0, /* alignment, bounds */
530394324d3SSascha Wildner ENA_DMA_BIT_MASK(adapter->dma_width), /* lowaddr of excl window */
531394324d3SSascha Wildner BUS_SPACE_MAXADDR, /* highaddr of excl window */
532394324d3SSascha Wildner MJUM16BYTES, /* maxsize */
533394324d3SSascha Wildner adapter->max_rx_sgl_size, /* nsegments */
534394324d3SSascha Wildner MJUM16BYTES, /* maxsegsize */
535394324d3SSascha Wildner 0, /* flags */
536394324d3SSascha Wildner &adapter->rx_buf_tag);
537394324d3SSascha Wildner
538394324d3SSascha Wildner return (ret);
539394324d3SSascha Wildner }
540394324d3SSascha Wildner
541394324d3SSascha Wildner static int
ena_free_rx_dma_tag(struct ena_adapter * adapter)542394324d3SSascha Wildner ena_free_rx_dma_tag(struct ena_adapter *adapter)
543394324d3SSascha Wildner {
544394324d3SSascha Wildner int ret;
545394324d3SSascha Wildner
546394324d3SSascha Wildner ret = bus_dma_tag_destroy(adapter->rx_buf_tag);
547394324d3SSascha Wildner
548394324d3SSascha Wildner if (likely(ret == 0))
549394324d3SSascha Wildner adapter->rx_buf_tag = NULL;
550394324d3SSascha Wildner
551394324d3SSascha Wildner return (ret);
552394324d3SSascha Wildner }
553394324d3SSascha Wildner
554394324d3SSascha Wildner /**
555394324d3SSascha Wildner * ena_setup_tx_resources - allocate Tx resources (Descriptors)
556394324d3SSascha Wildner * @adapter: network interface device structure
557394324d3SSascha Wildner * @qid: queue index
558394324d3SSascha Wildner *
559394324d3SSascha Wildner * Returns 0 on success, otherwise on failure.
560394324d3SSascha Wildner **/
561394324d3SSascha Wildner static int
ena_setup_tx_resources(struct ena_adapter * adapter,int qid)562394324d3SSascha Wildner ena_setup_tx_resources(struct ena_adapter *adapter, int qid)
563394324d3SSascha Wildner {
564394324d3SSascha Wildner struct ena_que *que = &adapter->que[qid];
565394324d3SSascha Wildner struct ena_ring *tx_ring = que->tx_ring;
566394324d3SSascha Wildner int size, i, err;
567394324d3SSascha Wildner #ifdef RSS
568394324d3SSascha Wildner cpuset_t cpu_mask;
569394324d3SSascha Wildner #endif
570394324d3SSascha Wildner
571394324d3SSascha Wildner size = sizeof(struct ena_tx_buffer) * tx_ring->ring_size;
572394324d3SSascha Wildner
57382a3fa28SBrad Hoffman tx_ring->tx_buffer_info = kmalloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
574394324d3SSascha Wildner if (unlikely(tx_ring->tx_buffer_info == NULL))
575394324d3SSascha Wildner return (ENOMEM);
576394324d3SSascha Wildner
577394324d3SSascha Wildner size = sizeof(uint16_t) * tx_ring->ring_size;
57882a3fa28SBrad Hoffman tx_ring->free_tx_ids = kmalloc(size, M_DEVBUF, M_NOWAIT | M_ZERO);
579394324d3SSascha Wildner if (unlikely(tx_ring->free_tx_ids == NULL))
580394324d3SSascha Wildner goto err_buf_info_free;
581394324d3SSascha Wildner
582394324d3SSascha Wildner /* Req id stack for TX OOO completions */
583394324d3SSascha Wildner for (i = 0; i < tx_ring->ring_size; i++)
584394324d3SSascha Wildner tx_ring->free_tx_ids[i] = i;
585394324d3SSascha Wildner
58682a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
587394324d3SSascha Wildner /* Reset TX statistics. */
588394324d3SSascha Wildner ena_reset_counters((counter_u64_t *)&tx_ring->tx_stats,
589394324d3SSascha Wildner sizeof(tx_ring->tx_stats));
59082a3fa28SBrad Hoffman #endif
591394324d3SSascha Wildner
592394324d3SSascha Wildner tx_ring->next_to_use = 0;
593394324d3SSascha Wildner tx_ring->next_to_clean = 0;
594394324d3SSascha Wildner
595394324d3SSascha Wildner /* ... and create the buffer DMA maps */
596394324d3SSascha Wildner for (i = 0; i < tx_ring->ring_size; i++) {
597394324d3SSascha Wildner err = bus_dmamap_create(adapter->tx_buf_tag, 0,
598394324d3SSascha Wildner &tx_ring->tx_buffer_info[i].map);
599394324d3SSascha Wildner if (unlikely(err != 0)) {
600394324d3SSascha Wildner ena_trace(ENA_ALERT,
601394324d3SSascha Wildner "Unable to create Tx DMA map for buffer %d\n", i);
602394324d3SSascha Wildner goto err_buf_info_unmap;
603394324d3SSascha Wildner }
604394324d3SSascha Wildner }
605394324d3SSascha Wildner
606394324d3SSascha Wildner return (0);
607394324d3SSascha Wildner
608394324d3SSascha Wildner err_buf_info_unmap:
609394324d3SSascha Wildner while (i--) {
610394324d3SSascha Wildner bus_dmamap_destroy(adapter->tx_buf_tag,
611394324d3SSascha Wildner tx_ring->tx_buffer_info[i].map);
612394324d3SSascha Wildner }
61382a3fa28SBrad Hoffman kfree(tx_ring->free_tx_ids, M_DEVBUF);
614394324d3SSascha Wildner tx_ring->free_tx_ids = NULL;
615394324d3SSascha Wildner err_buf_info_free:
61682a3fa28SBrad Hoffman kfree(tx_ring->tx_buffer_info, M_DEVBUF);
617394324d3SSascha Wildner tx_ring->tx_buffer_info = NULL;
618394324d3SSascha Wildner
619394324d3SSascha Wildner return (ENOMEM);
620394324d3SSascha Wildner }
621394324d3SSascha Wildner
622394324d3SSascha Wildner /**
623394324d3SSascha Wildner * ena_free_tx_resources - Free Tx Resources per Queue
624394324d3SSascha Wildner * @adapter: network interface device structure
625394324d3SSascha Wildner * @qid: queue index
626394324d3SSascha Wildner *
627394324d3SSascha Wildner * Free all transmit software resources
628394324d3SSascha Wildner **/
629394324d3SSascha Wildner static void
ena_free_tx_resources(struct ena_adapter * adapter,int qid)630394324d3SSascha Wildner ena_free_tx_resources(struct ena_adapter *adapter, int qid)
631394324d3SSascha Wildner {
632394324d3SSascha Wildner struct ena_ring *tx_ring = &adapter->tx_ring[qid];
633394324d3SSascha Wildner
634394324d3SSascha Wildner ENA_RING_MTX_LOCK(tx_ring);
635394324d3SSascha Wildner
636394324d3SSascha Wildner /* Free buffer DMA maps, */
637394324d3SSascha Wildner for (int i = 0; i < tx_ring->ring_size; i++) {
638394324d3SSascha Wildner m_freem(tx_ring->tx_buffer_info[i].mbuf);
639394324d3SSascha Wildner tx_ring->tx_buffer_info[i].mbuf = NULL;
640394324d3SSascha Wildner bus_dmamap_unload(adapter->tx_buf_tag,
641394324d3SSascha Wildner tx_ring->tx_buffer_info[i].map);
642394324d3SSascha Wildner bus_dmamap_destroy(adapter->tx_buf_tag,
643394324d3SSascha Wildner tx_ring->tx_buffer_info[i].map);
644394324d3SSascha Wildner }
645394324d3SSascha Wildner ENA_RING_MTX_UNLOCK(tx_ring);
646394324d3SSascha Wildner
647394324d3SSascha Wildner /* And free allocated memory. */
64882a3fa28SBrad Hoffman kfree(tx_ring->tx_buffer_info, M_DEVBUF);
649394324d3SSascha Wildner tx_ring->tx_buffer_info = NULL;
650394324d3SSascha Wildner
65182a3fa28SBrad Hoffman kfree(tx_ring->free_tx_ids, M_DEVBUF);
652394324d3SSascha Wildner tx_ring->free_tx_ids = NULL;
653394324d3SSascha Wildner }
654394324d3SSascha Wildner
655394324d3SSascha Wildner /**
656394324d3SSascha Wildner * ena_setup_all_tx_resources - allocate all queues Tx resources
657394324d3SSascha Wildner * @adapter: network interface device structure
658394324d3SSascha Wildner *
659394324d3SSascha Wildner * Returns 0 on success, otherwise on failure.
660394324d3SSascha Wildner **/
661394324d3SSascha Wildner static int
ena_setup_all_tx_resources(struct ena_adapter * adapter)662394324d3SSascha Wildner ena_setup_all_tx_resources(struct ena_adapter *adapter)
663394324d3SSascha Wildner {
664394324d3SSascha Wildner int i, rc;
665394324d3SSascha Wildner
666394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
667394324d3SSascha Wildner rc = ena_setup_tx_resources(adapter, i);
668394324d3SSascha Wildner if (rc != 0) {
669394324d3SSascha Wildner device_printf(adapter->pdev,
670394324d3SSascha Wildner "Allocation for Tx Queue %u failed\n", i);
671394324d3SSascha Wildner goto err_setup_tx;
672394324d3SSascha Wildner }
673394324d3SSascha Wildner }
674394324d3SSascha Wildner
675394324d3SSascha Wildner return (0);
676394324d3SSascha Wildner
677394324d3SSascha Wildner err_setup_tx:
678394324d3SSascha Wildner /* Rewind the index freeing the rings as we go */
679394324d3SSascha Wildner while (i--)
680394324d3SSascha Wildner ena_free_tx_resources(adapter, i);
681394324d3SSascha Wildner return (rc);
682394324d3SSascha Wildner }
683394324d3SSascha Wildner
684394324d3SSascha Wildner /**
685394324d3SSascha Wildner * ena_free_all_tx_resources - Free Tx Resources for All Queues
686394324d3SSascha Wildner * @adapter: network interface device structure
687394324d3SSascha Wildner *
688394324d3SSascha Wildner * Free all transmit software resources
689394324d3SSascha Wildner **/
690394324d3SSascha Wildner static void
ena_free_all_tx_resources(struct ena_adapter * adapter)691394324d3SSascha Wildner ena_free_all_tx_resources(struct ena_adapter *adapter)
692394324d3SSascha Wildner {
693394324d3SSascha Wildner int i;
694394324d3SSascha Wildner
695394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++)
696394324d3SSascha Wildner ena_free_tx_resources(adapter, i);
697394324d3SSascha Wildner }
698394324d3SSascha Wildner
699394324d3SSascha Wildner static inline int
validate_rx_req_id(struct ena_ring * rx_ring,uint16_t req_id)700394324d3SSascha Wildner validate_rx_req_id(struct ena_ring *rx_ring, uint16_t req_id)
701394324d3SSascha Wildner {
702394324d3SSascha Wildner if (likely(req_id < rx_ring->ring_size))
703394324d3SSascha Wildner return (0);
704394324d3SSascha Wildner
705394324d3SSascha Wildner device_printf(rx_ring->adapter->pdev, "Invalid rx req_id: %hu\n",
706394324d3SSascha Wildner req_id);
70782a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
70882a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
709394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.bad_req_id, 1);
71082a3fa28SBrad Hoffman #endif
711394324d3SSascha Wildner
712394324d3SSascha Wildner /* Trigger device reset */
713394324d3SSascha Wildner rx_ring->adapter->reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
714394324d3SSascha Wildner rx_ring->adapter->trigger_reset = true;
715394324d3SSascha Wildner
716394324d3SSascha Wildner return (EFAULT);
717394324d3SSascha Wildner }
718394324d3SSascha Wildner
719394324d3SSascha Wildner /**
720394324d3SSascha Wildner * ena_setup_rx_resources - allocate Rx resources (Descriptors)
721394324d3SSascha Wildner * @adapter: network interface device structure
722394324d3SSascha Wildner * @qid: queue index
723394324d3SSascha Wildner *
724394324d3SSascha Wildner * Returns 0 on success, otherwise on failure.
725394324d3SSascha Wildner **/
726394324d3SSascha Wildner static int
ena_setup_rx_resources(struct ena_adapter * adapter,unsigned int qid)727394324d3SSascha Wildner ena_setup_rx_resources(struct ena_adapter *adapter, unsigned int qid)
728394324d3SSascha Wildner {
729394324d3SSascha Wildner struct ena_que *que = &adapter->que[qid];
730394324d3SSascha Wildner struct ena_ring *rx_ring = que->rx_ring;
731394324d3SSascha Wildner int size, err, i;
732394324d3SSascha Wildner #ifdef RSS
733394324d3SSascha Wildner cpuset_t cpu_mask;
734394324d3SSascha Wildner #endif
735394324d3SSascha Wildner
736394324d3SSascha Wildner size = sizeof(struct ena_rx_buffer) * rx_ring->ring_size;
737394324d3SSascha Wildner
738394324d3SSascha Wildner /*
739394324d3SSascha Wildner * Alloc extra element so in rx path
740394324d3SSascha Wildner * we can always prefetch rx_info + 1
741394324d3SSascha Wildner */
742394324d3SSascha Wildner size += sizeof(struct ena_rx_buffer);
743394324d3SSascha Wildner
74482a3fa28SBrad Hoffman rx_ring->rx_buffer_info = kmalloc(size, M_DEVBUF, M_WAITOK | M_ZERO);
745394324d3SSascha Wildner
746394324d3SSascha Wildner size = sizeof(uint16_t) * rx_ring->ring_size;
74782a3fa28SBrad Hoffman rx_ring->free_rx_ids = kmalloc(size, M_DEVBUF, M_WAITOK);
748394324d3SSascha Wildner
749394324d3SSascha Wildner for (i = 0; i < rx_ring->ring_size; i++)
750394324d3SSascha Wildner rx_ring->free_rx_ids[i] = i;
751394324d3SSascha Wildner
75282a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
753394324d3SSascha Wildner /* Reset RX statistics. */
754394324d3SSascha Wildner ena_reset_counters((counter_u64_t *)&rx_ring->rx_stats,
755394324d3SSascha Wildner sizeof(rx_ring->rx_stats));
75682a3fa28SBrad Hoffman #endif
757394324d3SSascha Wildner
758394324d3SSascha Wildner rx_ring->next_to_clean = 0;
759394324d3SSascha Wildner rx_ring->next_to_use = 0;
760394324d3SSascha Wildner
761394324d3SSascha Wildner /* ... and create the buffer DMA maps */
762394324d3SSascha Wildner for (i = 0; i < rx_ring->ring_size; i++) {
763394324d3SSascha Wildner err = bus_dmamap_create(adapter->rx_buf_tag, 0,
764394324d3SSascha Wildner &(rx_ring->rx_buffer_info[i].map));
765394324d3SSascha Wildner if (err != 0) {
766394324d3SSascha Wildner ena_trace(ENA_ALERT,
767394324d3SSascha Wildner "Unable to create Rx DMA map for buffer %d\n", i);
768394324d3SSascha Wildner goto err_buf_info_unmap;
769394324d3SSascha Wildner }
770394324d3SSascha Wildner }
771394324d3SSascha Wildner
77282a3fa28SBrad Hoffman #if 0 /* XXX LRO */
773394324d3SSascha Wildner /* Create LRO for the ring */
774394324d3SSascha Wildner if ((adapter->ifp->if_capenable & IFCAP_LRO) != 0) {
775394324d3SSascha Wildner int err = tcp_lro_init(&rx_ring->lro);
776394324d3SSascha Wildner if (err != 0) {
777394324d3SSascha Wildner device_printf(adapter->pdev,
778394324d3SSascha Wildner "LRO[%d] Initialization failed!\n", qid);
779394324d3SSascha Wildner } else {
780394324d3SSascha Wildner ena_trace(ENA_INFO,
781394324d3SSascha Wildner "RX Soft LRO[%d] Initialized\n", qid);
782394324d3SSascha Wildner rx_ring->lro.ifp = adapter->ifp;
783394324d3SSascha Wildner }
784394324d3SSascha Wildner }
78582a3fa28SBrad Hoffman #endif
786394324d3SSascha Wildner
787394324d3SSascha Wildner /* Allocate taskqueues */
788394324d3SSascha Wildner TASK_INIT(&rx_ring->cmpl_task, 0, ena_deferred_rx_cleanup, rx_ring);
78982a3fa28SBrad Hoffman rx_ring->cmpl_tq = taskqueue_create("ena RX completion", M_WAITOK,
790394324d3SSascha Wildner taskqueue_thread_enqueue, &rx_ring->cmpl_tq);
791394324d3SSascha Wildner
792394324d3SSascha Wildner /* RSS set cpu for thread */
793394324d3SSascha Wildner #ifdef RSS
794394324d3SSascha Wildner CPU_SETOF(que->cpu, &cpu_mask);
795394324d3SSascha Wildner taskqueue_start_threads_cpuset(&rx_ring->cmpl_tq, 1, PI_NET, &cpu_mask,
796394324d3SSascha Wildner "%s rx_ring cmpl (bucket %d)",
797394324d3SSascha Wildner device_get_nameunit(adapter->pdev), que->cpu);
798394324d3SSascha Wildner #else
79982a3fa28SBrad Hoffman taskqueue_start_threads(&rx_ring->cmpl_tq, 1, TDPRI_KERN_DAEMON, -1,
80082a3fa28SBrad Hoffman "%s rx_ring cmpl", device_get_nameunit(adapter->pdev));
801394324d3SSascha Wildner #endif
802394324d3SSascha Wildner
803394324d3SSascha Wildner return (0);
804394324d3SSascha Wildner
805394324d3SSascha Wildner err_buf_info_unmap:
806394324d3SSascha Wildner while (i--) {
807394324d3SSascha Wildner bus_dmamap_destroy(adapter->rx_buf_tag,
808394324d3SSascha Wildner rx_ring->rx_buffer_info[i].map);
809394324d3SSascha Wildner }
810394324d3SSascha Wildner
81182a3fa28SBrad Hoffman kfree(rx_ring->free_rx_ids, M_DEVBUF);
812394324d3SSascha Wildner rx_ring->free_rx_ids = NULL;
81382a3fa28SBrad Hoffman kfree(rx_ring->rx_buffer_info, M_DEVBUF);
814394324d3SSascha Wildner rx_ring->rx_buffer_info = NULL;
815394324d3SSascha Wildner return (ENOMEM);
816394324d3SSascha Wildner }
817394324d3SSascha Wildner
818394324d3SSascha Wildner /**
819394324d3SSascha Wildner * ena_free_rx_resources - Free Rx Resources
820394324d3SSascha Wildner * @adapter: network interface device structure
821394324d3SSascha Wildner * @qid: queue index
822394324d3SSascha Wildner *
823394324d3SSascha Wildner * Free all receive software resources
824394324d3SSascha Wildner **/
825394324d3SSascha Wildner static void
ena_free_rx_resources(struct ena_adapter * adapter,unsigned int qid)826394324d3SSascha Wildner ena_free_rx_resources(struct ena_adapter *adapter, unsigned int qid)
827394324d3SSascha Wildner {
828394324d3SSascha Wildner struct ena_ring *rx_ring = &adapter->rx_ring[qid];
829394324d3SSascha Wildner
830394324d3SSascha Wildner while (taskqueue_cancel(rx_ring->cmpl_tq, &rx_ring->cmpl_task, NULL) != 0)
831394324d3SSascha Wildner taskqueue_drain(rx_ring->cmpl_tq, &rx_ring->cmpl_task);
832394324d3SSascha Wildner
833394324d3SSascha Wildner taskqueue_free(rx_ring->cmpl_tq);
834394324d3SSascha Wildner
835394324d3SSascha Wildner /* Free buffer DMA maps, */
836394324d3SSascha Wildner for (int i = 0; i < rx_ring->ring_size; i++) {
837394324d3SSascha Wildner m_freem(rx_ring->rx_buffer_info[i].mbuf);
838394324d3SSascha Wildner rx_ring->rx_buffer_info[i].mbuf = NULL;
839394324d3SSascha Wildner bus_dmamap_unload(adapter->rx_buf_tag,
840394324d3SSascha Wildner rx_ring->rx_buffer_info[i].map);
841394324d3SSascha Wildner bus_dmamap_destroy(adapter->rx_buf_tag,
842394324d3SSascha Wildner rx_ring->rx_buffer_info[i].map);
843394324d3SSascha Wildner }
844394324d3SSascha Wildner
84582a3fa28SBrad Hoffman #if 0 /* XXX LRO */
846394324d3SSascha Wildner /* free LRO resources, */
847394324d3SSascha Wildner tcp_lro_free(&rx_ring->lro);
84882a3fa28SBrad Hoffman #endif
849394324d3SSascha Wildner
850394324d3SSascha Wildner /* free allocated memory */
85182a3fa28SBrad Hoffman kfree(rx_ring->rx_buffer_info, M_DEVBUF);
852394324d3SSascha Wildner rx_ring->rx_buffer_info = NULL;
853394324d3SSascha Wildner
85482a3fa28SBrad Hoffman kfree(rx_ring->free_rx_ids, M_DEVBUF);
855394324d3SSascha Wildner rx_ring->free_rx_ids = NULL;
856394324d3SSascha Wildner }
857394324d3SSascha Wildner
858394324d3SSascha Wildner /**
859394324d3SSascha Wildner * ena_setup_all_rx_resources - allocate all queues Rx resources
860394324d3SSascha Wildner * @adapter: network interface device structure
861394324d3SSascha Wildner *
862394324d3SSascha Wildner * Returns 0 on success, otherwise on failure.
863394324d3SSascha Wildner **/
864394324d3SSascha Wildner static int
ena_setup_all_rx_resources(struct ena_adapter * adapter)865394324d3SSascha Wildner ena_setup_all_rx_resources(struct ena_adapter *adapter)
866394324d3SSascha Wildner {
867394324d3SSascha Wildner int i, rc = 0;
868394324d3SSascha Wildner
869394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
870394324d3SSascha Wildner rc = ena_setup_rx_resources(adapter, i);
871394324d3SSascha Wildner if (rc != 0) {
872394324d3SSascha Wildner device_printf(adapter->pdev,
873394324d3SSascha Wildner "Allocation for Rx Queue %u failed\n", i);
874394324d3SSascha Wildner goto err_setup_rx;
875394324d3SSascha Wildner }
876394324d3SSascha Wildner }
877394324d3SSascha Wildner return (0);
878394324d3SSascha Wildner
879394324d3SSascha Wildner err_setup_rx:
880394324d3SSascha Wildner /* rewind the index freeing the rings as we go */
881394324d3SSascha Wildner while (i--)
882394324d3SSascha Wildner ena_free_rx_resources(adapter, i);
883394324d3SSascha Wildner return (rc);
884394324d3SSascha Wildner }
885394324d3SSascha Wildner
886394324d3SSascha Wildner /**
887394324d3SSascha Wildner * ena_free_all_rx_resources - Free Rx resources for all queues
888394324d3SSascha Wildner * @adapter: network interface device structure
889394324d3SSascha Wildner *
890394324d3SSascha Wildner * Free all receive software resources
891394324d3SSascha Wildner **/
892394324d3SSascha Wildner static void
ena_free_all_rx_resources(struct ena_adapter * adapter)893394324d3SSascha Wildner ena_free_all_rx_resources(struct ena_adapter *adapter)
894394324d3SSascha Wildner {
895394324d3SSascha Wildner int i;
896394324d3SSascha Wildner
897394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++)
898394324d3SSascha Wildner ena_free_rx_resources(adapter, i);
899394324d3SSascha Wildner }
900394324d3SSascha Wildner
901394324d3SSascha Wildner static inline int
ena_alloc_rx_mbuf(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)902394324d3SSascha Wildner ena_alloc_rx_mbuf(struct ena_adapter *adapter,
903394324d3SSascha Wildner struct ena_ring *rx_ring, struct ena_rx_buffer *rx_info)
904394324d3SSascha Wildner {
905394324d3SSascha Wildner struct ena_com_buf *ena_buf;
906394324d3SSascha Wildner bus_dma_segment_t segs[1];
907394324d3SSascha Wildner int nsegs, error;
908394324d3SSascha Wildner int mlen;
909394324d3SSascha Wildner
910394324d3SSascha Wildner /* if previous allocated frag is not used */
911394324d3SSascha Wildner if (unlikely(rx_info->mbuf != NULL))
912394324d3SSascha Wildner return (0);
913394324d3SSascha Wildner
914394324d3SSascha Wildner /* Get mbuf using UMA allocator */
91582a3fa28SBrad Hoffman rx_info->mbuf = m_getjcl(M_NOWAIT, MT_DATA, M_PKTHDR, MJUMPAGESIZE);
916394324d3SSascha Wildner
917394324d3SSascha Wildner if (unlikely(rx_info->mbuf == NULL)) {
91882a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
919394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.mjum_alloc_fail, 1);
92082a3fa28SBrad Hoffman #endif
921394324d3SSascha Wildner rx_info->mbuf = m_getcl(M_NOWAIT, MT_DATA, M_PKTHDR);
922394324d3SSascha Wildner if (unlikely(rx_info->mbuf == NULL)) {
92382a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
92482a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
925394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
92682a3fa28SBrad Hoffman #endif
927394324d3SSascha Wildner return (ENOMEM);
928394324d3SSascha Wildner }
929394324d3SSascha Wildner mlen = MCLBYTES;
930394324d3SSascha Wildner } else {
93182a3fa28SBrad Hoffman mlen = MJUMPAGESIZE;
932394324d3SSascha Wildner }
933394324d3SSascha Wildner /* Set mbuf length*/
934394324d3SSascha Wildner rx_info->mbuf->m_pkthdr.len = rx_info->mbuf->m_len = mlen;
935394324d3SSascha Wildner
936394324d3SSascha Wildner /* Map packets for DMA */
937394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
938394324d3SSascha Wildner "Using tag %p for buffers' DMA mapping, mbuf %p len: %d",
939394324d3SSascha Wildner adapter->rx_buf_tag,rx_info->mbuf, rx_info->mbuf->m_len);
94082a3fa28SBrad Hoffman error = bus_dmamap_load_mbuf_segment(adapter->rx_buf_tag, rx_info->map,
94182a3fa28SBrad Hoffman rx_info->mbuf, segs, 1, &nsegs, BUS_DMA_NOWAIT);
942394324d3SSascha Wildner if (unlikely((error != 0) || (nsegs != 1))) {
943394324d3SSascha Wildner ena_trace(ENA_WARNING, "failed to map mbuf, error: %d, "
944394324d3SSascha Wildner "nsegs: %d\n", error, nsegs);
94582a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
94682a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
947394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.dma_mapping_err, 1);
94882a3fa28SBrad Hoffman #endif
949394324d3SSascha Wildner goto exit;
950394324d3SSascha Wildner
951394324d3SSascha Wildner }
952394324d3SSascha Wildner
953394324d3SSascha Wildner bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map, BUS_DMASYNC_PREREAD);
954394324d3SSascha Wildner
955394324d3SSascha Wildner ena_buf = &rx_info->ena_buf;
956394324d3SSascha Wildner ena_buf->paddr = segs[0].ds_addr;
957394324d3SSascha Wildner ena_buf->len = mlen;
958394324d3SSascha Wildner
959394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RSC | ENA_RXPTH,
960394324d3SSascha Wildner "ALLOC RX BUF: mbuf %p, rx_info %p, len %d, paddr %#jx\n",
961394324d3SSascha Wildner rx_info->mbuf, rx_info,ena_buf->len, (uintmax_t)ena_buf->paddr);
962394324d3SSascha Wildner
963394324d3SSascha Wildner return (0);
964394324d3SSascha Wildner
965394324d3SSascha Wildner exit:
966394324d3SSascha Wildner m_freem(rx_info->mbuf);
967394324d3SSascha Wildner rx_info->mbuf = NULL;
968394324d3SSascha Wildner return (EFAULT);
969394324d3SSascha Wildner }
970394324d3SSascha Wildner
971394324d3SSascha Wildner static void
ena_free_rx_mbuf(struct ena_adapter * adapter,struct ena_ring * rx_ring,struct ena_rx_buffer * rx_info)972394324d3SSascha Wildner ena_free_rx_mbuf(struct ena_adapter *adapter, struct ena_ring *rx_ring,
973394324d3SSascha Wildner struct ena_rx_buffer *rx_info)
974394324d3SSascha Wildner {
975394324d3SSascha Wildner
976394324d3SSascha Wildner if (rx_info->mbuf == NULL) {
977394324d3SSascha Wildner ena_trace(ENA_WARNING, "Trying to free unallocated buffer\n");
978394324d3SSascha Wildner return;
979394324d3SSascha Wildner }
980394324d3SSascha Wildner
981394324d3SSascha Wildner bus_dmamap_unload(adapter->rx_buf_tag, rx_info->map);
982394324d3SSascha Wildner m_freem(rx_info->mbuf);
983394324d3SSascha Wildner rx_info->mbuf = NULL;
984394324d3SSascha Wildner }
985394324d3SSascha Wildner
986394324d3SSascha Wildner /**
987394324d3SSascha Wildner * ena_refill_rx_bufs - Refills ring with descriptors
988394324d3SSascha Wildner * @rx_ring: the ring which we want to feed with free descriptors
989394324d3SSascha Wildner * @num: number of descriptors to refill
990394324d3SSascha Wildner * Refills the ring with newly allocated DMA-mapped mbufs for receiving
991394324d3SSascha Wildner **/
992394324d3SSascha Wildner static int
ena_refill_rx_bufs(struct ena_ring * rx_ring,uint32_t num)993394324d3SSascha Wildner ena_refill_rx_bufs(struct ena_ring *rx_ring, uint32_t num)
994394324d3SSascha Wildner {
995394324d3SSascha Wildner struct ena_adapter *adapter = rx_ring->adapter;
996394324d3SSascha Wildner uint16_t next_to_use, req_id;
997394324d3SSascha Wildner uint32_t i;
998394324d3SSascha Wildner int rc;
999394324d3SSascha Wildner
1000394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC, "refill qid: %d",
1001394324d3SSascha Wildner rx_ring->qid);
1002394324d3SSascha Wildner
1003394324d3SSascha Wildner next_to_use = rx_ring->next_to_use;
1004394324d3SSascha Wildner
1005394324d3SSascha Wildner for (i = 0; i < num; i++) {
1006394324d3SSascha Wildner struct ena_rx_buffer *rx_info;
1007394324d3SSascha Wildner
1008394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RXPTH | ENA_RSC,
1009394324d3SSascha Wildner "RX buffer - next to use: %d", next_to_use);
1010394324d3SSascha Wildner
1011394324d3SSascha Wildner req_id = rx_ring->free_rx_ids[next_to_use];
1012394324d3SSascha Wildner rc = validate_rx_req_id(rx_ring, req_id);
1013394324d3SSascha Wildner if (unlikely(rc != 0))
1014394324d3SSascha Wildner break;
1015394324d3SSascha Wildner
1016394324d3SSascha Wildner rx_info = &rx_ring->rx_buffer_info[req_id];
1017394324d3SSascha Wildner
1018394324d3SSascha Wildner rc = ena_alloc_rx_mbuf(adapter, rx_ring, rx_info);
1019394324d3SSascha Wildner if (unlikely(rc != 0)) {
1020394324d3SSascha Wildner ena_trace(ENA_WARNING,
1021394324d3SSascha Wildner "failed to alloc buffer for rx queue %d\n",
1022394324d3SSascha Wildner rx_ring->qid);
1023394324d3SSascha Wildner break;
1024394324d3SSascha Wildner }
1025394324d3SSascha Wildner rc = ena_com_add_single_rx_desc(rx_ring->ena_com_io_sq,
1026394324d3SSascha Wildner &rx_info->ena_buf, req_id);
1027394324d3SSascha Wildner if (unlikely(rc != 0)) {
1028394324d3SSascha Wildner ena_trace(ENA_WARNING,
1029394324d3SSascha Wildner "failed to add buffer for rx queue %d\n",
1030394324d3SSascha Wildner rx_ring->qid);
1031394324d3SSascha Wildner break;
1032394324d3SSascha Wildner }
1033394324d3SSascha Wildner next_to_use = ENA_RX_RING_IDX_NEXT(next_to_use,
1034394324d3SSascha Wildner rx_ring->ring_size);
1035394324d3SSascha Wildner }
1036394324d3SSascha Wildner
1037394324d3SSascha Wildner if (unlikely(i < num)) {
103882a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
103982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1040394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.refil_partial, 1);
104182a3fa28SBrad Hoffman #endif
1042394324d3SSascha Wildner ena_trace(ENA_WARNING,
1043394324d3SSascha Wildner "refilled rx qid %d with only %d mbufs (from %d)\n",
1044394324d3SSascha Wildner rx_ring->qid, i, num);
1045394324d3SSascha Wildner }
1046394324d3SSascha Wildner
1047394324d3SSascha Wildner if (likely(i != 0)) {
1048394324d3SSascha Wildner wmb();
1049394324d3SSascha Wildner ena_com_write_sq_doorbell(rx_ring->ena_com_io_sq);
1050394324d3SSascha Wildner }
1051394324d3SSascha Wildner rx_ring->next_to_use = next_to_use;
1052394324d3SSascha Wildner return (i);
1053394324d3SSascha Wildner }
1054394324d3SSascha Wildner
1055394324d3SSascha Wildner static void
ena_free_rx_bufs(struct ena_adapter * adapter,unsigned int qid)1056394324d3SSascha Wildner ena_free_rx_bufs(struct ena_adapter *adapter, unsigned int qid)
1057394324d3SSascha Wildner {
1058394324d3SSascha Wildner struct ena_ring *rx_ring = &adapter->rx_ring[qid];
1059394324d3SSascha Wildner unsigned int i;
1060394324d3SSascha Wildner
1061394324d3SSascha Wildner for (i = 0; i < rx_ring->ring_size; i++) {
1062394324d3SSascha Wildner struct ena_rx_buffer *rx_info = &rx_ring->rx_buffer_info[i];
1063394324d3SSascha Wildner
1064394324d3SSascha Wildner if (rx_info->mbuf != NULL)
1065394324d3SSascha Wildner ena_free_rx_mbuf(adapter, rx_ring, rx_info);
1066394324d3SSascha Wildner }
1067394324d3SSascha Wildner }
1068394324d3SSascha Wildner
1069394324d3SSascha Wildner /**
1070394324d3SSascha Wildner * ena_refill_all_rx_bufs - allocate all queues Rx buffers
1071394324d3SSascha Wildner * @adapter: network interface device structure
1072394324d3SSascha Wildner *
1073394324d3SSascha Wildner */
1074394324d3SSascha Wildner static void
ena_refill_all_rx_bufs(struct ena_adapter * adapter)1075394324d3SSascha Wildner ena_refill_all_rx_bufs(struct ena_adapter *adapter)
1076394324d3SSascha Wildner {
1077394324d3SSascha Wildner struct ena_ring *rx_ring;
1078394324d3SSascha Wildner int i, rc, bufs_num;
1079394324d3SSascha Wildner
1080394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
1081394324d3SSascha Wildner rx_ring = &adapter->rx_ring[i];
1082394324d3SSascha Wildner bufs_num = rx_ring->ring_size - 1;
1083394324d3SSascha Wildner rc = ena_refill_rx_bufs(rx_ring, bufs_num);
1084394324d3SSascha Wildner
1085394324d3SSascha Wildner if (unlikely(rc != bufs_num))
1086394324d3SSascha Wildner ena_trace(ENA_WARNING, "refilling Queue %d failed. "
1087394324d3SSascha Wildner "Allocated %d buffers from: %d\n", i, rc, bufs_num);
1088394324d3SSascha Wildner }
1089394324d3SSascha Wildner }
1090394324d3SSascha Wildner
1091394324d3SSascha Wildner static void
ena_free_all_rx_bufs(struct ena_adapter * adapter)1092394324d3SSascha Wildner ena_free_all_rx_bufs(struct ena_adapter *adapter)
1093394324d3SSascha Wildner {
1094394324d3SSascha Wildner int i;
1095394324d3SSascha Wildner
1096394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++)
1097394324d3SSascha Wildner ena_free_rx_bufs(adapter, i);
1098394324d3SSascha Wildner }
1099394324d3SSascha Wildner
1100394324d3SSascha Wildner /**
1101394324d3SSascha Wildner * ena_free_tx_bufs - Free Tx Buffers per Queue
1102394324d3SSascha Wildner * @adapter: network interface device structure
1103394324d3SSascha Wildner * @qid: queue index
1104394324d3SSascha Wildner **/
1105394324d3SSascha Wildner static void
ena_free_tx_bufs(struct ena_adapter * adapter,unsigned int qid)1106394324d3SSascha Wildner ena_free_tx_bufs(struct ena_adapter *adapter, unsigned int qid)
1107394324d3SSascha Wildner {
1108394324d3SSascha Wildner bool print_once = true;
1109394324d3SSascha Wildner struct ena_ring *tx_ring = &adapter->tx_ring[qid];
1110394324d3SSascha Wildner
1111394324d3SSascha Wildner ENA_RING_MTX_LOCK(tx_ring);
1112394324d3SSascha Wildner for (int i = 0; i < tx_ring->ring_size; i++) {
1113394324d3SSascha Wildner struct ena_tx_buffer *tx_info = &tx_ring->tx_buffer_info[i];
1114394324d3SSascha Wildner
1115394324d3SSascha Wildner if (tx_info->mbuf == NULL)
1116394324d3SSascha Wildner continue;
1117394324d3SSascha Wildner
1118394324d3SSascha Wildner if (print_once) {
1119394324d3SSascha Wildner device_printf(adapter->pdev,
1120394324d3SSascha Wildner "free uncompleted tx mbuf qid %d idx 0x%x",
1121394324d3SSascha Wildner qid, i);
1122394324d3SSascha Wildner print_once = false;
1123394324d3SSascha Wildner } else {
1124394324d3SSascha Wildner ena_trace(ENA_DBG,
1125394324d3SSascha Wildner "free uncompleted tx mbuf qid %d idx 0x%x",
1126394324d3SSascha Wildner qid, i);
1127394324d3SSascha Wildner }
1128394324d3SSascha Wildner
1129394324d3SSascha Wildner bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map);
1130394324d3SSascha Wildner m_free(tx_info->mbuf);
1131394324d3SSascha Wildner tx_info->mbuf = NULL;
1132394324d3SSascha Wildner }
1133394324d3SSascha Wildner ENA_RING_MTX_UNLOCK(tx_ring);
1134394324d3SSascha Wildner }
1135394324d3SSascha Wildner
1136394324d3SSascha Wildner static void
ena_free_all_tx_bufs(struct ena_adapter * adapter)1137394324d3SSascha Wildner ena_free_all_tx_bufs(struct ena_adapter *adapter)
1138394324d3SSascha Wildner {
1139394324d3SSascha Wildner
1140394324d3SSascha Wildner for (int i = 0; i < adapter->num_queues; i++)
1141394324d3SSascha Wildner ena_free_tx_bufs(adapter, i);
1142394324d3SSascha Wildner }
1143394324d3SSascha Wildner
1144394324d3SSascha Wildner static void
ena_destroy_all_tx_queues(struct ena_adapter * adapter)1145394324d3SSascha Wildner ena_destroy_all_tx_queues(struct ena_adapter *adapter)
1146394324d3SSascha Wildner {
1147394324d3SSascha Wildner uint16_t ena_qid;
1148394324d3SSascha Wildner int i;
1149394324d3SSascha Wildner
1150394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
1151394324d3SSascha Wildner ena_qid = ENA_IO_TXQ_IDX(i);
1152394324d3SSascha Wildner ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1153394324d3SSascha Wildner }
1154394324d3SSascha Wildner }
1155394324d3SSascha Wildner
1156394324d3SSascha Wildner static void
ena_destroy_all_rx_queues(struct ena_adapter * adapter)1157394324d3SSascha Wildner ena_destroy_all_rx_queues(struct ena_adapter *adapter)
1158394324d3SSascha Wildner {
1159394324d3SSascha Wildner uint16_t ena_qid;
1160394324d3SSascha Wildner int i;
1161394324d3SSascha Wildner
1162394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
1163394324d3SSascha Wildner ena_qid = ENA_IO_RXQ_IDX(i);
1164394324d3SSascha Wildner ena_com_destroy_io_queue(adapter->ena_dev, ena_qid);
1165394324d3SSascha Wildner }
1166394324d3SSascha Wildner }
1167394324d3SSascha Wildner
1168394324d3SSascha Wildner static void
ena_destroy_all_io_queues(struct ena_adapter * adapter)1169394324d3SSascha Wildner ena_destroy_all_io_queues(struct ena_adapter *adapter)
1170394324d3SSascha Wildner {
1171394324d3SSascha Wildner ena_destroy_all_tx_queues(adapter);
1172394324d3SSascha Wildner ena_destroy_all_rx_queues(adapter);
1173394324d3SSascha Wildner }
1174394324d3SSascha Wildner
1175394324d3SSascha Wildner static inline int
validate_tx_req_id(struct ena_ring * tx_ring,uint16_t req_id)1176394324d3SSascha Wildner validate_tx_req_id(struct ena_ring *tx_ring, uint16_t req_id)
1177394324d3SSascha Wildner {
1178394324d3SSascha Wildner struct ena_adapter *adapter = tx_ring->adapter;
1179394324d3SSascha Wildner struct ena_tx_buffer *tx_info = NULL;
1180394324d3SSascha Wildner
1181394324d3SSascha Wildner if (likely(req_id < tx_ring->ring_size)) {
1182394324d3SSascha Wildner tx_info = &tx_ring->tx_buffer_info[req_id];
1183394324d3SSascha Wildner if (tx_info->mbuf != NULL)
1184394324d3SSascha Wildner return (0);
1185394324d3SSascha Wildner }
1186394324d3SSascha Wildner
1187394324d3SSascha Wildner if (tx_info->mbuf == NULL)
1188394324d3SSascha Wildner device_printf(adapter->pdev,
1189394324d3SSascha Wildner "tx_info doesn't have valid mbuf\n");
1190394324d3SSascha Wildner else
1191394324d3SSascha Wildner device_printf(adapter->pdev, "Invalid req_id: %hu\n", req_id);
1192394324d3SSascha Wildner
119382a3fa28SBrad Hoffman IFNET_STAT_INC(tx_ring->adapter->ifp, oerrors, 1);
119482a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1195394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.bad_req_id, 1);
119682a3fa28SBrad Hoffman #endif
1197394324d3SSascha Wildner
1198394324d3SSascha Wildner return (EFAULT);
1199394324d3SSascha Wildner }
1200394324d3SSascha Wildner
1201394324d3SSascha Wildner static int
ena_create_io_queues(struct ena_adapter * adapter)1202394324d3SSascha Wildner ena_create_io_queues(struct ena_adapter *adapter)
1203394324d3SSascha Wildner {
1204394324d3SSascha Wildner struct ena_com_dev *ena_dev = adapter->ena_dev;
1205394324d3SSascha Wildner struct ena_com_create_io_ctx ctx;
1206394324d3SSascha Wildner struct ena_ring *ring;
1207394324d3SSascha Wildner uint16_t ena_qid;
1208394324d3SSascha Wildner uint32_t msix_vector;
1209394324d3SSascha Wildner int rc, i;
1210394324d3SSascha Wildner
1211394324d3SSascha Wildner /* Create TX queues */
1212394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
1213394324d3SSascha Wildner msix_vector = ENA_IO_IRQ_IDX(i);
1214394324d3SSascha Wildner ena_qid = ENA_IO_TXQ_IDX(i);
1215394324d3SSascha Wildner ctx.mem_queue_type = ena_dev->tx_mem_queue_type;
1216394324d3SSascha Wildner ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_TX;
1217394324d3SSascha Wildner ctx.queue_size = adapter->tx_ring_size;
1218394324d3SSascha Wildner ctx.msix_vector = msix_vector;
1219394324d3SSascha Wildner ctx.qid = ena_qid;
1220394324d3SSascha Wildner rc = ena_com_create_io_queue(ena_dev, &ctx);
1221394324d3SSascha Wildner if (rc != 0) {
1222394324d3SSascha Wildner device_printf(adapter->pdev,
1223394324d3SSascha Wildner "Failed to create io TX queue #%d rc: %d\n", i, rc);
1224394324d3SSascha Wildner goto err_tx;
1225394324d3SSascha Wildner }
1226394324d3SSascha Wildner ring = &adapter->tx_ring[i];
1227394324d3SSascha Wildner rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1228394324d3SSascha Wildner &ring->ena_com_io_sq,
1229394324d3SSascha Wildner &ring->ena_com_io_cq);
1230394324d3SSascha Wildner if (rc != 0) {
1231394324d3SSascha Wildner device_printf(adapter->pdev,
1232394324d3SSascha Wildner "Failed to get TX queue handlers. TX queue num"
1233394324d3SSascha Wildner " %d rc: %d\n", i, rc);
1234394324d3SSascha Wildner ena_com_destroy_io_queue(ena_dev, ena_qid);
1235394324d3SSascha Wildner goto err_tx;
1236394324d3SSascha Wildner }
1237394324d3SSascha Wildner }
1238394324d3SSascha Wildner
1239394324d3SSascha Wildner /* Create RX queues */
1240394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
1241394324d3SSascha Wildner msix_vector = ENA_IO_IRQ_IDX(i);
1242394324d3SSascha Wildner ena_qid = ENA_IO_RXQ_IDX(i);
1243394324d3SSascha Wildner ctx.mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
1244394324d3SSascha Wildner ctx.direction = ENA_COM_IO_QUEUE_DIRECTION_RX;
1245394324d3SSascha Wildner ctx.queue_size = adapter->rx_ring_size;
1246394324d3SSascha Wildner ctx.msix_vector = msix_vector;
1247394324d3SSascha Wildner ctx.qid = ena_qid;
1248394324d3SSascha Wildner rc = ena_com_create_io_queue(ena_dev, &ctx);
1249394324d3SSascha Wildner if (unlikely(rc != 0)) {
1250394324d3SSascha Wildner device_printf(adapter->pdev,
1251394324d3SSascha Wildner "Failed to create io RX queue[%d] rc: %d\n", i, rc);
1252394324d3SSascha Wildner goto err_rx;
1253394324d3SSascha Wildner }
1254394324d3SSascha Wildner
1255394324d3SSascha Wildner ring = &adapter->rx_ring[i];
1256394324d3SSascha Wildner rc = ena_com_get_io_handlers(ena_dev, ena_qid,
1257394324d3SSascha Wildner &ring->ena_com_io_sq,
1258394324d3SSascha Wildner &ring->ena_com_io_cq);
1259394324d3SSascha Wildner if (unlikely(rc != 0)) {
1260394324d3SSascha Wildner device_printf(adapter->pdev,
1261394324d3SSascha Wildner "Failed to get RX queue handlers. RX queue num"
1262394324d3SSascha Wildner " %d rc: %d\n", i, rc);
1263394324d3SSascha Wildner ena_com_destroy_io_queue(ena_dev, ena_qid);
1264394324d3SSascha Wildner goto err_rx;
1265394324d3SSascha Wildner }
1266394324d3SSascha Wildner }
1267394324d3SSascha Wildner
1268394324d3SSascha Wildner return (0);
1269394324d3SSascha Wildner
1270394324d3SSascha Wildner err_rx:
1271394324d3SSascha Wildner while (i--)
1272394324d3SSascha Wildner ena_com_destroy_io_queue(ena_dev, ENA_IO_RXQ_IDX(i));
1273394324d3SSascha Wildner i = adapter->num_queues;
1274394324d3SSascha Wildner err_tx:
1275394324d3SSascha Wildner while (i--)
1276394324d3SSascha Wildner ena_com_destroy_io_queue(ena_dev, ENA_IO_TXQ_IDX(i));
1277394324d3SSascha Wildner
1278394324d3SSascha Wildner return (ENXIO);
1279394324d3SSascha Wildner }
1280394324d3SSascha Wildner
1281394324d3SSascha Wildner /**
1282394324d3SSascha Wildner * ena_tx_cleanup - clear sent packets and corresponding descriptors
1283394324d3SSascha Wildner * @tx_ring: ring for which we want to clean packets
1284394324d3SSascha Wildner *
1285394324d3SSascha Wildner * Once packets are sent, we ask the device in a loop for no longer used
1286394324d3SSascha Wildner * descriptors. We find the related mbuf chain in a map (index in an array)
1287394324d3SSascha Wildner * and free it, then update ring state.
1288394324d3SSascha Wildner * This is performed in "endless" loop, updating ring pointers every
1289394324d3SSascha Wildner * TX_COMMIT. The first check of free descriptor is performed before the actual
1290394324d3SSascha Wildner * loop, then repeated at the loop end.
1291394324d3SSascha Wildner **/
1292394324d3SSascha Wildner static int
ena_tx_cleanup(struct ena_ring * tx_ring)1293394324d3SSascha Wildner ena_tx_cleanup(struct ena_ring *tx_ring)
1294394324d3SSascha Wildner {
1295394324d3SSascha Wildner struct ena_adapter *adapter;
1296394324d3SSascha Wildner struct ena_com_io_cq* io_cq;
1297394324d3SSascha Wildner uint16_t next_to_clean;
1298394324d3SSascha Wildner uint16_t req_id;
1299394324d3SSascha Wildner uint16_t ena_qid;
1300394324d3SSascha Wildner unsigned int total_done = 0;
1301394324d3SSascha Wildner int rc;
1302394324d3SSascha Wildner int commit = TX_COMMIT;
1303394324d3SSascha Wildner int budget = TX_BUDGET;
1304394324d3SSascha Wildner int work_done;
1305394324d3SSascha Wildner
1306394324d3SSascha Wildner adapter = tx_ring->que->adapter;
1307394324d3SSascha Wildner ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
1308394324d3SSascha Wildner io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1309394324d3SSascha Wildner next_to_clean = tx_ring->next_to_clean;
1310394324d3SSascha Wildner
1311394324d3SSascha Wildner do {
1312394324d3SSascha Wildner struct ena_tx_buffer *tx_info;
1313394324d3SSascha Wildner struct mbuf *mbuf;
1314394324d3SSascha Wildner
1315394324d3SSascha Wildner rc = ena_com_tx_comp_req_id_get(io_cq, &req_id);
1316394324d3SSascha Wildner if (unlikely(rc != 0))
1317394324d3SSascha Wildner break;
1318394324d3SSascha Wildner
1319394324d3SSascha Wildner rc = validate_tx_req_id(tx_ring, req_id);
1320394324d3SSascha Wildner if (unlikely(rc != 0))
1321394324d3SSascha Wildner break;
1322394324d3SSascha Wildner
1323394324d3SSascha Wildner tx_info = &tx_ring->tx_buffer_info[req_id];
1324394324d3SSascha Wildner
1325394324d3SSascha Wildner mbuf = tx_info->mbuf;
1326394324d3SSascha Wildner
1327394324d3SSascha Wildner tx_info->mbuf = NULL;
132882a3fa28SBrad Hoffman timevalclear(&tx_info->timestamp);
1329394324d3SSascha Wildner
1330394324d3SSascha Wildner if (likely(tx_info->num_of_bufs != 0)) {
1331394324d3SSascha Wildner /* Map is no longer required */
1332394324d3SSascha Wildner bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map);
1333394324d3SSascha Wildner }
1334394324d3SSascha Wildner
1335394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d mbuf %p completed",
1336394324d3SSascha Wildner tx_ring->qid, mbuf);
1337394324d3SSascha Wildner
1338394324d3SSascha Wildner m_freem(mbuf);
1339394324d3SSascha Wildner
1340394324d3SSascha Wildner total_done += tx_info->tx_descs;
1341394324d3SSascha Wildner
1342394324d3SSascha Wildner tx_ring->free_tx_ids[next_to_clean] = req_id;
1343394324d3SSascha Wildner next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
1344394324d3SSascha Wildner tx_ring->ring_size);
1345394324d3SSascha Wildner
1346394324d3SSascha Wildner if (unlikely(--commit == 0)) {
1347394324d3SSascha Wildner commit = TX_COMMIT;
1348394324d3SSascha Wildner /* update ring state every TX_COMMIT descriptor */
1349394324d3SSascha Wildner tx_ring->next_to_clean = next_to_clean;
1350394324d3SSascha Wildner ena_com_comp_ack(
1351394324d3SSascha Wildner &adapter->ena_dev->io_sq_queues[ena_qid],
1352394324d3SSascha Wildner total_done);
1353394324d3SSascha Wildner ena_com_update_dev_comp_head(io_cq);
1354394324d3SSascha Wildner total_done = 0;
1355394324d3SSascha Wildner }
1356394324d3SSascha Wildner } while (likely(--budget));
1357394324d3SSascha Wildner
1358394324d3SSascha Wildner work_done = TX_BUDGET - budget;
1359394324d3SSascha Wildner
1360394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_TXPTH, "tx: q %d done. total pkts: %d",
1361394324d3SSascha Wildner tx_ring->qid, work_done);
1362394324d3SSascha Wildner
1363394324d3SSascha Wildner /* If there is still something to commit update ring state */
1364394324d3SSascha Wildner if (likely(commit != TX_COMMIT)) {
1365394324d3SSascha Wildner tx_ring->next_to_clean = next_to_clean;
1366394324d3SSascha Wildner ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid],
1367394324d3SSascha Wildner total_done);
1368394324d3SSascha Wildner ena_com_update_dev_comp_head(io_cq);
1369394324d3SSascha Wildner }
1370394324d3SSascha Wildner
1371394324d3SSascha Wildner return (work_done);
1372394324d3SSascha Wildner }
1373394324d3SSascha Wildner
1374394324d3SSascha Wildner static void
ena_rx_hash_mbuf(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct mbuf * mbuf)1375394324d3SSascha Wildner ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
1376394324d3SSascha Wildner struct mbuf *mbuf)
1377394324d3SSascha Wildner {
1378394324d3SSascha Wildner struct ena_adapter *adapter = rx_ring->adapter;
1379394324d3SSascha Wildner
1380394324d3SSascha Wildner if (likely(adapter->rss_support)) {
138182a3fa28SBrad Hoffman //mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
138282a3fa28SBrad Hoffman m_sethash(mbuf, ena_rx_ctx->hash);
1383394324d3SSascha Wildner
138482a3fa28SBrad Hoffman #if 0 /* XXX rsstype doesn't seem to be needed by the network stack, we will only supply the hash. */
1385394324d3SSascha Wildner if (ena_rx_ctx->frag &&
1386394324d3SSascha Wildner (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) {
1387394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
1388394324d3SSascha Wildner return;
1389394324d3SSascha Wildner }
1390394324d3SSascha Wildner
1391394324d3SSascha Wildner switch (ena_rx_ctx->l3_proto) {
1392394324d3SSascha Wildner case ENA_ETH_IO_L3_PROTO_IPV4:
1393394324d3SSascha Wildner switch (ena_rx_ctx->l4_proto) {
1394394324d3SSascha Wildner case ENA_ETH_IO_L4_PROTO_TCP:
1395394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
1396394324d3SSascha Wildner break;
1397394324d3SSascha Wildner case ENA_ETH_IO_L4_PROTO_UDP:
1398394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
1399394324d3SSascha Wildner break;
1400394324d3SSascha Wildner default:
1401394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
1402394324d3SSascha Wildner }
1403394324d3SSascha Wildner break;
1404394324d3SSascha Wildner case ENA_ETH_IO_L3_PROTO_IPV6:
1405394324d3SSascha Wildner switch (ena_rx_ctx->l4_proto) {
1406394324d3SSascha Wildner case ENA_ETH_IO_L4_PROTO_TCP:
1407394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
1408394324d3SSascha Wildner break;
1409394324d3SSascha Wildner case ENA_ETH_IO_L4_PROTO_UDP:
1410394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
1411394324d3SSascha Wildner break;
1412394324d3SSascha Wildner default:
1413394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
1414394324d3SSascha Wildner }
1415394324d3SSascha Wildner break;
1416394324d3SSascha Wildner case ENA_ETH_IO_L3_PROTO_UNKNOWN:
1417394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
1418394324d3SSascha Wildner break;
1419394324d3SSascha Wildner default:
1420394324d3SSascha Wildner M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
1421394324d3SSascha Wildner }
142282a3fa28SBrad Hoffman #endif
1423394324d3SSascha Wildner } else {
142482a3fa28SBrad Hoffman //mbuf->m_pkthdr.flowid = rx_ring->qid;
142582a3fa28SBrad Hoffman //M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
142682a3fa28SBrad Hoffman m_sethash(mbuf, rx_ring->qid);
1427394324d3SSascha Wildner }
1428394324d3SSascha Wildner }
1429394324d3SSascha Wildner
1430394324d3SSascha Wildner /**
1431394324d3SSascha Wildner * ena_rx_mbuf - assemble mbuf from descriptors
1432394324d3SSascha Wildner * @rx_ring: ring for which we want to clean packets
1433394324d3SSascha Wildner * @ena_bufs: buffer info
1434394324d3SSascha Wildner * @ena_rx_ctx: metadata for this packet(s)
1435394324d3SSascha Wildner * @next_to_clean: ring pointer, will be updated only upon success
1436394324d3SSascha Wildner *
1437394324d3SSascha Wildner **/
1438394324d3SSascha Wildner static struct mbuf*
ena_rx_mbuf(struct ena_ring * rx_ring,struct ena_com_rx_buf_info * ena_bufs,struct ena_com_rx_ctx * ena_rx_ctx,uint16_t * next_to_clean)1439394324d3SSascha Wildner ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
1440394324d3SSascha Wildner struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
1441394324d3SSascha Wildner {
1442394324d3SSascha Wildner struct mbuf *mbuf;
1443394324d3SSascha Wildner struct ena_rx_buffer *rx_info;
1444394324d3SSascha Wildner struct ena_adapter *adapter;
1445394324d3SSascha Wildner unsigned int descs = ena_rx_ctx->descs;
1446394324d3SSascha Wildner uint16_t ntc, len, req_id, buf = 0;
1447394324d3SSascha Wildner
1448394324d3SSascha Wildner ntc = *next_to_clean;
1449394324d3SSascha Wildner adapter = rx_ring->adapter;
1450394324d3SSascha Wildner rx_info = &rx_ring->rx_buffer_info[ntc];
1451394324d3SSascha Wildner
1452394324d3SSascha Wildner if (unlikely(rx_info->mbuf == NULL)) {
1453394324d3SSascha Wildner device_printf(adapter->pdev, "NULL mbuf in rx_info");
1454394324d3SSascha Wildner return (NULL);
1455394324d3SSascha Wildner }
1456394324d3SSascha Wildner
1457394324d3SSascha Wildner len = ena_bufs[buf].len;
1458394324d3SSascha Wildner req_id = ena_bufs[buf].req_id;
1459394324d3SSascha Wildner rx_info = &rx_ring->rx_buffer_info[req_id];
1460394324d3SSascha Wildner
1461394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RXPTH, "rx_info %p, mbuf %p, paddr %jx",
1462394324d3SSascha Wildner rx_info, rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
1463394324d3SSascha Wildner
1464394324d3SSascha Wildner mbuf = rx_info->mbuf;
1465394324d3SSascha Wildner mbuf->m_flags |= M_PKTHDR;
1466394324d3SSascha Wildner mbuf->m_pkthdr.len = len;
1467394324d3SSascha Wildner mbuf->m_len = len;
1468394324d3SSascha Wildner mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
1469394324d3SSascha Wildner
1470394324d3SSascha Wildner /* Fill mbuf with hash key and it's interpretation for optimization */
1471394324d3SSascha Wildner ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
1472394324d3SSascha Wildner
1473394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RXPTH, "rx mbuf 0x%p, flags=0x%x, len: %d",
1474394324d3SSascha Wildner mbuf, mbuf->m_flags, mbuf->m_pkthdr.len);
1475394324d3SSascha Wildner
1476394324d3SSascha Wildner /* DMA address is not needed anymore, unmap it */
1477394324d3SSascha Wildner bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1478394324d3SSascha Wildner
1479394324d3SSascha Wildner rx_info->mbuf = NULL;
1480394324d3SSascha Wildner rx_ring->free_rx_ids[ntc] = req_id;
1481394324d3SSascha Wildner ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
1482394324d3SSascha Wildner
1483394324d3SSascha Wildner /*
1484394324d3SSascha Wildner * While we have more than 1 descriptors for one rcvd packet, append
1485394324d3SSascha Wildner * other mbufs to the main one
1486394324d3SSascha Wildner */
1487394324d3SSascha Wildner while (--descs) {
1488394324d3SSascha Wildner ++buf;
1489394324d3SSascha Wildner len = ena_bufs[buf].len;
1490394324d3SSascha Wildner req_id = ena_bufs[buf].req_id;
1491394324d3SSascha Wildner rx_info = &rx_ring->rx_buffer_info[req_id];
1492394324d3SSascha Wildner
1493394324d3SSascha Wildner if (unlikely(rx_info->mbuf == NULL)) {
1494394324d3SSascha Wildner device_printf(adapter->pdev, "NULL mbuf in rx_info");
1495394324d3SSascha Wildner /*
1496394324d3SSascha Wildner * If one of the required mbufs was not allocated yet,
1497394324d3SSascha Wildner * we can break there.
1498394324d3SSascha Wildner * All earlier used descriptors will be reallocated
1499394324d3SSascha Wildner * later and not used mbufs can be reused.
1500394324d3SSascha Wildner * The next_to_clean pointer will not be updated in case
1501394324d3SSascha Wildner * of an error, so caller should advance it manually
1502394324d3SSascha Wildner * in error handling routine to keep it up to date
1503394324d3SSascha Wildner * with hw ring.
1504394324d3SSascha Wildner */
1505394324d3SSascha Wildner m_freem(mbuf);
1506394324d3SSascha Wildner return (NULL);
1507394324d3SSascha Wildner }
1508394324d3SSascha Wildner
1509394324d3SSascha Wildner if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
151082a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
151182a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1512394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
151382a3fa28SBrad Hoffman #endif
1514394324d3SSascha Wildner ena_trace(ENA_WARNING, "Failed to append Rx mbuf %p",
1515394324d3SSascha Wildner mbuf);
1516394324d3SSascha Wildner }
1517394324d3SSascha Wildner
1518394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RXPTH,
1519394324d3SSascha Wildner "rx mbuf updated. len %d", mbuf->m_pkthdr.len);
1520394324d3SSascha Wildner
1521394324d3SSascha Wildner /* Free already appended mbuf, it won't be useful anymore */
1522394324d3SSascha Wildner bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
1523394324d3SSascha Wildner m_freem(rx_info->mbuf);
1524394324d3SSascha Wildner rx_info->mbuf = NULL;
1525394324d3SSascha Wildner
1526394324d3SSascha Wildner rx_ring->free_rx_ids[ntc] = req_id;
1527394324d3SSascha Wildner ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
1528394324d3SSascha Wildner }
1529394324d3SSascha Wildner
1530394324d3SSascha Wildner *next_to_clean = ntc;
1531394324d3SSascha Wildner
1532394324d3SSascha Wildner return (mbuf);
1533394324d3SSascha Wildner }
1534394324d3SSascha Wildner
1535394324d3SSascha Wildner /**
1536394324d3SSascha Wildner * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
1537394324d3SSascha Wildner **/
1538394324d3SSascha Wildner static inline void
ena_rx_checksum(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct mbuf * mbuf)1539394324d3SSascha Wildner ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
1540394324d3SSascha Wildner struct mbuf *mbuf)
1541394324d3SSascha Wildner {
1542394324d3SSascha Wildner
1543394324d3SSascha Wildner /* if IP and error */
1544394324d3SSascha Wildner if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
1545394324d3SSascha Wildner ena_rx_ctx->l3_csum_err)) {
1546394324d3SSascha Wildner /* ipv4 checksum error */
1547394324d3SSascha Wildner mbuf->m_pkthdr.csum_flags = 0;
154882a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
154982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1550394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
155182a3fa28SBrad Hoffman #endif
1552394324d3SSascha Wildner ena_trace(ENA_DBG, "RX IPv4 header checksum error");
1553394324d3SSascha Wildner return;
1554394324d3SSascha Wildner }
1555394324d3SSascha Wildner
1556394324d3SSascha Wildner /* if TCP/UDP */
1557394324d3SSascha Wildner if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
1558394324d3SSascha Wildner (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
1559394324d3SSascha Wildner if (ena_rx_ctx->l4_csum_err) {
1560394324d3SSascha Wildner /* TCP/UDP checksum error */
1561394324d3SSascha Wildner mbuf->m_pkthdr.csum_flags = 0;
156282a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
156382a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1564394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.bad_csum, 1);
156582a3fa28SBrad Hoffman #endif
1566394324d3SSascha Wildner ena_trace(ENA_DBG, "RX L4 checksum error");
1567394324d3SSascha Wildner } else {
1568394324d3SSascha Wildner mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
1569394324d3SSascha Wildner mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
1570394324d3SSascha Wildner }
1571394324d3SSascha Wildner }
1572394324d3SSascha Wildner }
1573394324d3SSascha Wildner
1574394324d3SSascha Wildner static void
ena_deferred_rx_cleanup(void * arg,int pending)1575394324d3SSascha Wildner ena_deferred_rx_cleanup(void *arg, int pending)
1576394324d3SSascha Wildner {
1577394324d3SSascha Wildner struct ena_ring *rx_ring = arg;
1578394324d3SSascha Wildner int budget = CLEAN_BUDGET;
1579394324d3SSascha Wildner
1580394324d3SSascha Wildner ENA_RING_MTX_LOCK(rx_ring);
1581394324d3SSascha Wildner /*
1582394324d3SSascha Wildner * If deferred task was executed, perform cleanup of all awaiting
1583394324d3SSascha Wildner * descs (or until given budget is depleted to avoid infinite loop).
1584394324d3SSascha Wildner */
1585394324d3SSascha Wildner while (likely(budget--)) {
1586394324d3SSascha Wildner if (ena_rx_cleanup(rx_ring) == 0)
1587394324d3SSascha Wildner break;
1588394324d3SSascha Wildner }
1589394324d3SSascha Wildner ENA_RING_MTX_UNLOCK(rx_ring);
1590394324d3SSascha Wildner }
1591394324d3SSascha Wildner
1592394324d3SSascha Wildner /**
1593394324d3SSascha Wildner * ena_rx_cleanup - handle rx irq
1594394324d3SSascha Wildner * @arg: ring for which irq is being handled
1595394324d3SSascha Wildner **/
1596394324d3SSascha Wildner static int
ena_rx_cleanup(struct ena_ring * rx_ring)1597394324d3SSascha Wildner ena_rx_cleanup(struct ena_ring *rx_ring)
1598394324d3SSascha Wildner {
1599394324d3SSascha Wildner struct ena_adapter *adapter;
1600394324d3SSascha Wildner struct mbuf *mbuf;
1601394324d3SSascha Wildner struct ena_com_rx_ctx ena_rx_ctx;
1602394324d3SSascha Wildner struct ena_com_io_cq* io_cq;
1603394324d3SSascha Wildner struct ena_com_io_sq* io_sq;
1604394324d3SSascha Wildner if_t ifp;
1605394324d3SSascha Wildner uint16_t ena_qid;
1606394324d3SSascha Wildner uint16_t next_to_clean;
1607394324d3SSascha Wildner uint32_t refill_required;
1608394324d3SSascha Wildner uint32_t refill_threshold;
1609394324d3SSascha Wildner uint32_t do_if_input = 0;
1610394324d3SSascha Wildner unsigned int qid;
1611394324d3SSascha Wildner int rc, i;
1612394324d3SSascha Wildner int budget = RX_BUDGET;
1613394324d3SSascha Wildner
1614394324d3SSascha Wildner adapter = rx_ring->que->adapter;
1615394324d3SSascha Wildner ifp = adapter->ifp;
1616394324d3SSascha Wildner qid = rx_ring->que->id;
1617394324d3SSascha Wildner ena_qid = ENA_IO_RXQ_IDX(qid);
1618394324d3SSascha Wildner io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1619394324d3SSascha Wildner io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
1620394324d3SSascha Wildner next_to_clean = rx_ring->next_to_clean;
1621394324d3SSascha Wildner
1622394324d3SSascha Wildner ena_trace(ENA_DBG, "rx: qid %d", qid);
1623394324d3SSascha Wildner
1624394324d3SSascha Wildner do {
1625394324d3SSascha Wildner ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
1626394324d3SSascha Wildner ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
1627394324d3SSascha Wildner ena_rx_ctx.descs = 0;
1628394324d3SSascha Wildner rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
1629394324d3SSascha Wildner
1630394324d3SSascha Wildner if (unlikely(rc != 0))
1631394324d3SSascha Wildner goto error;
1632394324d3SSascha Wildner
1633394324d3SSascha Wildner if (unlikely(ena_rx_ctx.descs == 0))
1634394324d3SSascha Wildner break;
1635394324d3SSascha Wildner
1636394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RXPTH, "rx: q %d got packet from ena. "
1637394324d3SSascha Wildner "descs #: %d l3 proto %d l4 proto %d hash: %x",
1638394324d3SSascha Wildner rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
1639394324d3SSascha Wildner ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
1640394324d3SSascha Wildner
1641394324d3SSascha Wildner /* Receive mbuf from the ring */
1642394324d3SSascha Wildner mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs,
1643394324d3SSascha Wildner &ena_rx_ctx, &next_to_clean);
1644394324d3SSascha Wildner
1645394324d3SSascha Wildner /* Exit if we failed to retrieve a buffer */
1646394324d3SSascha Wildner if (unlikely(mbuf == NULL)) {
1647394324d3SSascha Wildner for (i = 0; i < ena_rx_ctx.descs; ++i) {
1648394324d3SSascha Wildner rx_ring->free_rx_ids[next_to_clean] =
1649394324d3SSascha Wildner rx_ring->ena_bufs[i].req_id;
1650394324d3SSascha Wildner next_to_clean =
1651394324d3SSascha Wildner ENA_RX_RING_IDX_NEXT(next_to_clean,
1652394324d3SSascha Wildner rx_ring->ring_size);
1653394324d3SSascha Wildner
1654394324d3SSascha Wildner }
1655394324d3SSascha Wildner break;
1656394324d3SSascha Wildner }
1657394324d3SSascha Wildner
165882a3fa28SBrad Hoffman /*
165982a3fa28SBrad Hoffman * XXX Removed IFCAP_RXCSUM_IPV6 check because DragonFly
166082a3fa28SBrad Hoffman * does not seem to support it
166182a3fa28SBrad Hoffman */
166282a3fa28SBrad Hoffman if ((ifp->if_capenable & IFCAP_RXCSUM) != 0) {
1663394324d3SSascha Wildner ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
1664394324d3SSascha Wildner }
1665394324d3SSascha Wildner
166682a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1667394324d3SSascha Wildner counter_enter();
1668394324d3SSascha Wildner counter_u64_add_protected(rx_ring->rx_stats.bytes,
1669394324d3SSascha Wildner mbuf->m_pkthdr.len);
1670394324d3SSascha Wildner counter_u64_add_protected(adapter->hw_stats.rx_bytes,
1671394324d3SSascha Wildner mbuf->m_pkthdr.len);
1672394324d3SSascha Wildner counter_exit();
167382a3fa28SBrad Hoffman #endif
1674394324d3SSascha Wildner /*
1675394324d3SSascha Wildner * LRO is only for IP/TCP packets and TCP checksum of the packet
1676394324d3SSascha Wildner * should be computed by hardware.
1677394324d3SSascha Wildner */
1678394324d3SSascha Wildner do_if_input = 1;
167982a3fa28SBrad Hoffman #if 0 /* XXX LRO */
1680394324d3SSascha Wildner if (((ifp->if_capenable & IFCAP_LRO) != 0) &&
1681394324d3SSascha Wildner ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
1682394324d3SSascha Wildner (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
1683394324d3SSascha Wildner /*
1684394324d3SSascha Wildner * Send to the stack if:
1685394324d3SSascha Wildner * - LRO not enabled, or
1686394324d3SSascha Wildner * - no LRO resources, or
1687394324d3SSascha Wildner * - lro enqueue fails
1688394324d3SSascha Wildner */
1689394324d3SSascha Wildner if ((rx_ring->lro.lro_cnt != 0) &&
1690394324d3SSascha Wildner (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
1691394324d3SSascha Wildner do_if_input = 0;
1692394324d3SSascha Wildner }
169382a3fa28SBrad Hoffman #endif
1694394324d3SSascha Wildner if (do_if_input != 0) {
1695394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_RXPTH,
1696394324d3SSascha Wildner "calling if_input() with mbuf %p", mbuf);
169782a3fa28SBrad Hoffman ENA_RING_MTX_UNLOCK(rx_ring);
169882a3fa28SBrad Hoffman (*ifp->if_input)(ifp, mbuf, NULL, -1);
169982a3fa28SBrad Hoffman ENA_RING_MTX_LOCK(rx_ring);
1700394324d3SSascha Wildner }
1701394324d3SSascha Wildner
170282a3fa28SBrad Hoffman IFNET_STAT_INC(ifp, ipackets, 1);
170382a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1704394324d3SSascha Wildner counter_enter();
1705394324d3SSascha Wildner counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
1706394324d3SSascha Wildner counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
1707394324d3SSascha Wildner counter_exit();
170882a3fa28SBrad Hoffman #endif
1709394324d3SSascha Wildner } while (--budget);
1710394324d3SSascha Wildner
1711394324d3SSascha Wildner rx_ring->next_to_clean = next_to_clean;
1712394324d3SSascha Wildner
1713394324d3SSascha Wildner refill_required = ena_com_free_desc(io_sq);
1714394324d3SSascha Wildner refill_threshold = rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER;
1715394324d3SSascha Wildner
1716394324d3SSascha Wildner if (refill_required > refill_threshold) {
1717394324d3SSascha Wildner ena_com_update_dev_comp_head(rx_ring->ena_com_io_cq);
1718394324d3SSascha Wildner ena_refill_rx_bufs(rx_ring, refill_required);
1719394324d3SSascha Wildner }
1720394324d3SSascha Wildner
172182a3fa28SBrad Hoffman #if 0 /* XXX LRO */
1722394324d3SSascha Wildner tcp_lro_flush_all(&rx_ring->lro);
172382a3fa28SBrad Hoffman #endif
1724394324d3SSascha Wildner
1725394324d3SSascha Wildner return (RX_BUDGET - budget);
1726394324d3SSascha Wildner
1727394324d3SSascha Wildner error:
172882a3fa28SBrad Hoffman IFNET_STAT_INC(rx_ring->adapter->ifp, ierrors, 1);
172982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
1730394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.bad_desc_num, 1);
173182a3fa28SBrad Hoffman #endif
1732394324d3SSascha Wildner return (RX_BUDGET - budget);
1733394324d3SSascha Wildner }
1734394324d3SSascha Wildner
1735394324d3SSascha Wildner /*********************************************************************
1736394324d3SSascha Wildner *
1737394324d3SSascha Wildner * MSIX & Interrupt Service routine
1738394324d3SSascha Wildner *
1739394324d3SSascha Wildner **********************************************************************/
1740394324d3SSascha Wildner
1741394324d3SSascha Wildner /**
1742394324d3SSascha Wildner * ena_handle_msix - MSIX Interrupt Handler for admin/async queue
1743394324d3SSascha Wildner * @arg: interrupt number
1744394324d3SSascha Wildner **/
1745394324d3SSascha Wildner static void
ena_intr_msix_mgmnt(void * arg)1746394324d3SSascha Wildner ena_intr_msix_mgmnt(void *arg)
1747394324d3SSascha Wildner {
1748394324d3SSascha Wildner struct ena_adapter *adapter = (struct ena_adapter *)arg;
1749394324d3SSascha Wildner
1750394324d3SSascha Wildner ena_com_admin_q_comp_intr_handler(adapter->ena_dev);
1751394324d3SSascha Wildner if (likely(adapter->running))
1752394324d3SSascha Wildner ena_com_aenq_intr_handler(adapter->ena_dev, arg);
1753394324d3SSascha Wildner }
1754394324d3SSascha Wildner
1755394324d3SSascha Wildner /**
1756394324d3SSascha Wildner * ena_handle_msix - MSIX Interrupt Handler for Tx/Rx
1757394324d3SSascha Wildner * @arg: interrupt number
1758394324d3SSascha Wildner **/
1759394324d3SSascha Wildner static void
ena_handle_msix(void * arg)1760394324d3SSascha Wildner ena_handle_msix(void *arg)
1761394324d3SSascha Wildner {
1762394324d3SSascha Wildner struct ena_que *que = arg;
1763394324d3SSascha Wildner struct ena_adapter *adapter = que->adapter;
1764394324d3SSascha Wildner if_t ifp = adapter->ifp;
1765394324d3SSascha Wildner struct ena_ring *tx_ring;
1766394324d3SSascha Wildner struct ena_ring *rx_ring;
1767394324d3SSascha Wildner struct ena_com_io_cq* io_cq;
1768394324d3SSascha Wildner struct ena_eth_io_intr_reg intr_reg;
1769394324d3SSascha Wildner int qid, ena_qid;
1770394324d3SSascha Wildner int txc, rxc, i;
1771394324d3SSascha Wildner
177282a3fa28SBrad Hoffman if (unlikely((ifp->if_flags & IFF_RUNNING) == 0))
1773394324d3SSascha Wildner return;
1774394324d3SSascha Wildner
1775394324d3SSascha Wildner ena_trace(ENA_DBG, "MSI-X TX/RX routine");
1776394324d3SSascha Wildner
1777394324d3SSascha Wildner tx_ring = que->tx_ring;
1778394324d3SSascha Wildner rx_ring = que->rx_ring;
1779394324d3SSascha Wildner qid = que->id;
1780394324d3SSascha Wildner ena_qid = ENA_IO_TXQ_IDX(qid);
1781394324d3SSascha Wildner io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
1782394324d3SSascha Wildner
1783394324d3SSascha Wildner for (i = 0; i < CLEAN_BUDGET; ++i) {
1784394324d3SSascha Wildner /*
1785394324d3SSascha Wildner * If lock cannot be acquired, then deferred cleanup task was
1786394324d3SSascha Wildner * being executed and rx ring is being cleaned up in
1787394324d3SSascha Wildner * another thread.
1788394324d3SSascha Wildner */
1789394324d3SSascha Wildner if (likely(ENA_RING_MTX_TRYLOCK(rx_ring) != 0)) {
1790394324d3SSascha Wildner rxc = ena_rx_cleanup(rx_ring);
1791394324d3SSascha Wildner ENA_RING_MTX_UNLOCK(rx_ring);
1792394324d3SSascha Wildner } else {
1793394324d3SSascha Wildner rxc = 0;
1794394324d3SSascha Wildner }
1795394324d3SSascha Wildner
1796394324d3SSascha Wildner /* Protection from calling ena_tx_cleanup from ena_start_xmit */
1797394324d3SSascha Wildner ENA_RING_MTX_LOCK(tx_ring);
1798394324d3SSascha Wildner txc = ena_tx_cleanup(tx_ring);
1799394324d3SSascha Wildner ENA_RING_MTX_UNLOCK(tx_ring);
1800394324d3SSascha Wildner
180182a3fa28SBrad Hoffman if (unlikely((ifp->if_flags & IFF_RUNNING) == 0))
1802394324d3SSascha Wildner return;
1803394324d3SSascha Wildner
1804394324d3SSascha Wildner if ((txc != TX_BUDGET) && (rxc != RX_BUDGET))
1805394324d3SSascha Wildner break;
1806394324d3SSascha Wildner }
1807394324d3SSascha Wildner
1808394324d3SSascha Wildner /* Signal that work is done and unmask interrupt */
1809394324d3SSascha Wildner ena_com_update_intr_reg(&intr_reg,
1810394324d3SSascha Wildner RX_IRQ_INTERVAL,
1811394324d3SSascha Wildner TX_IRQ_INTERVAL,
1812394324d3SSascha Wildner true);
1813394324d3SSascha Wildner ena_com_unmask_intr(io_cq, &intr_reg);
1814394324d3SSascha Wildner }
1815394324d3SSascha Wildner
1816394324d3SSascha Wildner static int
ena_enable_msix(struct ena_adapter * adapter)1817394324d3SSascha Wildner ena_enable_msix(struct ena_adapter *adapter)
1818394324d3SSascha Wildner {
1819394324d3SSascha Wildner device_t dev = adapter->pdev;
182082a3fa28SBrad Hoffman int msix_vecs;
182182a3fa28SBrad Hoffman int error, i, rc = 0;
1822394324d3SSascha Wildner
1823394324d3SSascha Wildner /* Reserved the max msix vectors we might need */
1824394324d3SSascha Wildner msix_vecs = ENA_MAX_MSIX_VEC(adapter->num_queues);
1825394324d3SSascha Wildner
182682a3fa28SBrad Hoffman adapter->msix_entries = kmalloc(msix_vecs * sizeof(struct msix_entry),
1827394324d3SSascha Wildner M_DEVBUF, M_WAITOK | M_ZERO);
1828394324d3SSascha Wildner
1829394324d3SSascha Wildner ena_trace(ENA_DBG, "trying to enable MSI-X, vectors: %d", msix_vecs);
1830394324d3SSascha Wildner
1831394324d3SSascha Wildner for (i = 0; i < msix_vecs; i++) {
1832394324d3SSascha Wildner adapter->msix_entries[i].entry = i;
1833394324d3SSascha Wildner /* Vectors must start from 1 */
1834394324d3SSascha Wildner adapter->msix_entries[i].vector = i + 1;
1835394324d3SSascha Wildner }
1836394324d3SSascha Wildner
183782a3fa28SBrad Hoffman error = pci_setup_msix(dev);
183882a3fa28SBrad Hoffman if (error) {
183982a3fa28SBrad Hoffman device_printf(dev, "pci_setup_msix() failed\n");
1840394324d3SSascha Wildner goto err_msix_free;
1841394324d3SSascha Wildner }
1842394324d3SSascha Wildner
1843394324d3SSascha Wildner adapter->msix_vecs = msix_vecs;
1844394324d3SSascha Wildner adapter->msix_enabled = true;
1845394324d3SSascha Wildner
1846394324d3SSascha Wildner return (0);
1847394324d3SSascha Wildner
1848394324d3SSascha Wildner err_msix_free:
184982a3fa28SBrad Hoffman kfree(adapter->msix_entries, M_DEVBUF);
1850394324d3SSascha Wildner adapter->msix_entries = NULL;
1851394324d3SSascha Wildner
1852394324d3SSascha Wildner return (rc);
1853394324d3SSascha Wildner }
1854394324d3SSascha Wildner
1855394324d3SSascha Wildner static void
ena_setup_mgmnt_intr(struct ena_adapter * adapter)1856394324d3SSascha Wildner ena_setup_mgmnt_intr(struct ena_adapter *adapter)
1857394324d3SSascha Wildner {
1858394324d3SSascha Wildner
185982a3fa28SBrad Hoffman ksnprintf(adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].name,
1860394324d3SSascha Wildner ENA_IRQNAME_SIZE, "ena-mgmnt@pci:%s",
1861394324d3SSascha Wildner device_get_nameunit(adapter->pdev));
1862394324d3SSascha Wildner /*
1863394324d3SSascha Wildner * Handler is NULL on purpose, it will be set
1864394324d3SSascha Wildner * when mgmnt interrupt is acquired
1865394324d3SSascha Wildner */
1866394324d3SSascha Wildner adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].handler = NULL;
1867394324d3SSascha Wildner adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].data = adapter;
1868394324d3SSascha Wildner adapter->irq_tbl[ENA_MGMNT_IRQ_IDX].vector =
1869394324d3SSascha Wildner adapter->msix_entries[ENA_MGMNT_IRQ_IDX].vector;
1870394324d3SSascha Wildner }
1871394324d3SSascha Wildner
1872394324d3SSascha Wildner static void
ena_setup_io_intr(struct ena_adapter * adapter)1873394324d3SSascha Wildner ena_setup_io_intr(struct ena_adapter *adapter)
1874394324d3SSascha Wildner {
1875394324d3SSascha Wildner static int last_bind_cpu = -1;
1876394324d3SSascha Wildner int irq_idx;
1877394324d3SSascha Wildner
1878394324d3SSascha Wildner for (int i = 0; i < adapter->num_queues; i++) {
1879394324d3SSascha Wildner irq_idx = ENA_IO_IRQ_IDX(i);
1880394324d3SSascha Wildner
188182a3fa28SBrad Hoffman ksnprintf(adapter->irq_tbl[irq_idx].name, ENA_IRQNAME_SIZE,
1882394324d3SSascha Wildner "%s-TxRx-%d", device_get_nameunit(adapter->pdev), i);
1883394324d3SSascha Wildner adapter->irq_tbl[irq_idx].handler = ena_handle_msix;
1884394324d3SSascha Wildner adapter->irq_tbl[irq_idx].data = &adapter->que[i];
1885394324d3SSascha Wildner adapter->irq_tbl[irq_idx].vector =
1886394324d3SSascha Wildner adapter->msix_entries[irq_idx].vector;
1887394324d3SSascha Wildner ena_trace(ENA_INFO | ENA_IOQ, "ena_setup_io_intr vector: %d\n",
1888394324d3SSascha Wildner adapter->msix_entries[irq_idx].vector);
1889394324d3SSascha Wildner #ifdef RSS
1890394324d3SSascha Wildner adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1891394324d3SSascha Wildner rss_getcpu(i % rss_getnumbuckets());
1892394324d3SSascha Wildner #else
1893394324d3SSascha Wildner /*
1894394324d3SSascha Wildner * We still want to bind rings to the corresponding cpu
1895394324d3SSascha Wildner * using something similar to the RSS round-robin technique.
189682a3fa28SBrad Hoffman *
189782a3fa28SBrad Hoffman * XXX It seems that this can be removed since DragonFly has
189882a3fa28SBrad Hoffman * native support for RSS. DragonFly also does not have
189982a3fa28SBrad Hoffman * support for CPU_FIRST or CPU_NEXT.
1900394324d3SSascha Wildner */
190182a3fa28SBrad Hoffman
190282a3fa28SBrad Hoffman if (last_bind_cpu < 0)
190382a3fa28SBrad Hoffman last_bind_cpu = (last_bind_cpu + 1) % ncpus;
1904394324d3SSascha Wildner adapter->que[i].cpu = adapter->irq_tbl[irq_idx].cpu =
1905394324d3SSascha Wildner last_bind_cpu;
190682a3fa28SBrad Hoffman last_bind_cpu = (last_bind_cpu + 1) % ncpus;
1907394324d3SSascha Wildner #endif
1908394324d3SSascha Wildner }
1909394324d3SSascha Wildner }
1910394324d3SSascha Wildner
1911394324d3SSascha Wildner static int
ena_request_mgmnt_irq(struct ena_adapter * adapter)1912394324d3SSascha Wildner ena_request_mgmnt_irq(struct ena_adapter *adapter)
1913394324d3SSascha Wildner {
1914394324d3SSascha Wildner struct ena_irq *irq;
1915394324d3SSascha Wildner unsigned long flags;
191682a3fa28SBrad Hoffman int error, rc, rcc;
1917394324d3SSascha Wildner
1918394324d3SSascha Wildner flags = RF_ACTIVE | RF_SHAREABLE;
1919394324d3SSascha Wildner
1920394324d3SSascha Wildner irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
192182a3fa28SBrad Hoffman
192282a3fa28SBrad Hoffman error = pci_alloc_msix_vector(adapter->pdev, 0, &irq->vector, 0);
192382a3fa28SBrad Hoffman if (error) {
192482a3fa28SBrad Hoffman device_printf(adapter->pdev, "Could not initialize MGMNT MSI-X Vector on cpu0\n");
1925e4196227SSascha Wildner return (ENXIO);
192682a3fa28SBrad Hoffman }
192782a3fa28SBrad Hoffman
1928394324d3SSascha Wildner irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
1929394324d3SSascha Wildner &irq->vector, flags);
1930394324d3SSascha Wildner
1931394324d3SSascha Wildner if (unlikely(irq->res == NULL)) {
1932394324d3SSascha Wildner device_printf(adapter->pdev, "could not allocate "
1933394324d3SSascha Wildner "irq vector: %d\n", irq->vector);
19347c5020f6SSascha Wildner pci_release_msix_vector(adapter->pdev, irq->vector);
1935394324d3SSascha Wildner return (ENXIO);
1936394324d3SSascha Wildner }
1937394324d3SSascha Wildner
1938394324d3SSascha Wildner rc = bus_activate_resource(adapter->pdev, SYS_RES_IRQ,
1939394324d3SSascha Wildner irq->vector, irq->res);
1940394324d3SSascha Wildner if (unlikely(rc != 0)) {
1941394324d3SSascha Wildner device_printf(adapter->pdev, "could not activate "
1942394324d3SSascha Wildner "irq vector: %d\n", irq->vector);
1943394324d3SSascha Wildner goto err_res_free;
1944394324d3SSascha Wildner }
1945394324d3SSascha Wildner
1946394324d3SSascha Wildner rc = bus_setup_intr(adapter->pdev, irq->res,
194782a3fa28SBrad Hoffman INTR_MPSAFE, ena_intr_msix_mgmnt,
194882a3fa28SBrad Hoffman irq->data, &irq->cookie, NULL);
1949394324d3SSascha Wildner if (unlikely(rc != 0)) {
1950394324d3SSascha Wildner device_printf(adapter->pdev, "failed to register "
1951394324d3SSascha Wildner "interrupt handler for irq %ju: %d\n",
1952394324d3SSascha Wildner rman_get_start(irq->res), rc);
1953394324d3SSascha Wildner goto err_res_free;
1954394324d3SSascha Wildner }
1955394324d3SSascha Wildner irq->requested = true;
1956394324d3SSascha Wildner
1957394324d3SSascha Wildner return (rc);
1958394324d3SSascha Wildner
1959394324d3SSascha Wildner err_res_free:
1960394324d3SSascha Wildner ena_trace(ENA_INFO | ENA_ADMQ, "releasing resource for irq %d\n",
1961394324d3SSascha Wildner irq->vector);
1962394324d3SSascha Wildner rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
1963394324d3SSascha Wildner irq->vector, irq->res);
196482a3fa28SBrad Hoffman pci_release_msix_vector(adapter->pdev, irq->vector);
1965394324d3SSascha Wildner if (unlikely(rcc != 0))
1966394324d3SSascha Wildner device_printf(adapter->pdev, "dev has no parent while "
1967394324d3SSascha Wildner "releasing res for irq: %d\n", irq->vector);
1968394324d3SSascha Wildner irq->res = NULL;
1969394324d3SSascha Wildner
1970394324d3SSascha Wildner return (rc);
1971394324d3SSascha Wildner }
1972394324d3SSascha Wildner
1973394324d3SSascha Wildner static int
ena_request_io_irq(struct ena_adapter * adapter)1974394324d3SSascha Wildner ena_request_io_irq(struct ena_adapter *adapter)
1975394324d3SSascha Wildner {
1976394324d3SSascha Wildner struct ena_irq *irq;
1977394324d3SSascha Wildner unsigned long flags = 0;
197882a3fa28SBrad Hoffman int rc = 0, i, rcc, error;
1979394324d3SSascha Wildner
1980394324d3SSascha Wildner if (unlikely(adapter->msix_enabled == 0)) {
1981394324d3SSascha Wildner device_printf(adapter->pdev,
1982394324d3SSascha Wildner "failed to request I/O IRQ: MSI-X is not enabled\n");
1983394324d3SSascha Wildner return (EINVAL);
1984394324d3SSascha Wildner } else {
1985394324d3SSascha Wildner flags = RF_ACTIVE | RF_SHAREABLE;
1986394324d3SSascha Wildner }
1987394324d3SSascha Wildner
1988394324d3SSascha Wildner for (i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
1989394324d3SSascha Wildner irq = &adapter->irq_tbl[i];
1990394324d3SSascha Wildner
1991394324d3SSascha Wildner if (unlikely(irq->requested))
1992394324d3SSascha Wildner continue;
1993394324d3SSascha Wildner
199482a3fa28SBrad Hoffman error = pci_alloc_msix_vector(adapter->pdev, i, &irq->vector, irq->cpu);
199582a3fa28SBrad Hoffman if (error) {
199682a3fa28SBrad Hoffman device_printf(adapter->pdev, "Unable to allocated MSI-X %d on cpu%d\n", i, irq->cpu);
199782a3fa28SBrad Hoffman goto err;
199882a3fa28SBrad Hoffman }
199982a3fa28SBrad Hoffman
2000394324d3SSascha Wildner irq->res = bus_alloc_resource_any(adapter->pdev, SYS_RES_IRQ,
2001394324d3SSascha Wildner &irq->vector, flags);
2002394324d3SSascha Wildner if (unlikely(irq->res == NULL)) {
2003394324d3SSascha Wildner device_printf(adapter->pdev, "could not allocate "
2004394324d3SSascha Wildner "irq vector: %d\n", irq->vector);
2005394324d3SSascha Wildner goto err;
2006394324d3SSascha Wildner }
2007394324d3SSascha Wildner
200882a3fa28SBrad Hoffman
200982a3fa28SBrad Hoffman /*
201082a3fa28SBrad Hoffman * TODO: Might need to setup desc and use irq->name as the
201182a3fa28SBrad Hoffman * value
201282a3fa28SBrad Hoffman */
2013394324d3SSascha Wildner rc = bus_setup_intr(adapter->pdev, irq->res,
201482a3fa28SBrad Hoffman INTR_MPSAFE,
201582a3fa28SBrad Hoffman irq->handler, irq->data, &irq->cookie, NULL);
2016394324d3SSascha Wildner if (unlikely(rc != 0)) {
2017394324d3SSascha Wildner device_printf(adapter->pdev, "failed to register "
2018394324d3SSascha Wildner "interrupt handler for irq %ju: %d\n",
2019394324d3SSascha Wildner rman_get_start(irq->res), rc);
2020394324d3SSascha Wildner goto err;
2021394324d3SSascha Wildner }
2022394324d3SSascha Wildner irq->requested = true;
2023394324d3SSascha Wildner
2024394324d3SSascha Wildner #ifdef RSS
2025394324d3SSascha Wildner ena_trace(ENA_INFO, "queue %d - RSS bucket %d\n",
2026394324d3SSascha Wildner i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
2027394324d3SSascha Wildner #else
2028394324d3SSascha Wildner ena_trace(ENA_INFO, "queue %d - cpu %d\n",
2029394324d3SSascha Wildner i - ENA_IO_IRQ_FIRST_IDX, irq->cpu);
2030394324d3SSascha Wildner #endif
2031394324d3SSascha Wildner }
2032394324d3SSascha Wildner
2033394324d3SSascha Wildner return (rc);
2034394324d3SSascha Wildner
2035394324d3SSascha Wildner err:
2036394324d3SSascha Wildner
2037394324d3SSascha Wildner for (; i >= ENA_IO_IRQ_FIRST_IDX; i--) {
2038394324d3SSascha Wildner irq = &adapter->irq_tbl[i];
2039394324d3SSascha Wildner rcc = 0;
2040394324d3SSascha Wildner
2041394324d3SSascha Wildner /* Once we entered err: section and irq->requested is true we
2042394324d3SSascha Wildner free both intr and resources */
2043394324d3SSascha Wildner if (irq->requested)
2044394324d3SSascha Wildner rcc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
2045394324d3SSascha Wildner if (unlikely(rcc != 0))
2046394324d3SSascha Wildner device_printf(adapter->pdev, "could not release"
2047394324d3SSascha Wildner " irq: %d, error: %d\n", irq->vector, rcc);
2048394324d3SSascha Wildner
2049394324d3SSascha Wildner /* If we entred err: section without irq->requested set we know
2050394324d3SSascha Wildner it was bus_alloc_resource_any() that needs cleanup, provided
2051394324d3SSascha Wildner res is not NULL. In case res is NULL no work in needed in
2052394324d3SSascha Wildner this iteration */
2053394324d3SSascha Wildner rcc = 0;
2054394324d3SSascha Wildner if (irq->res != NULL) {
2055394324d3SSascha Wildner rcc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2056394324d3SSascha Wildner irq->vector, irq->res);
205782a3fa28SBrad Hoffman pci_release_msix_vector(adapter->pdev, irq->vector);
2058394324d3SSascha Wildner }
2059394324d3SSascha Wildner if (unlikely(rcc != 0))
2060394324d3SSascha Wildner device_printf(adapter->pdev, "dev has no parent while "
2061394324d3SSascha Wildner "releasing res for irq: %d\n", irq->vector);
2062394324d3SSascha Wildner irq->requested = false;
2063394324d3SSascha Wildner irq->res = NULL;
2064394324d3SSascha Wildner }
2065394324d3SSascha Wildner
2066394324d3SSascha Wildner return (rc);
2067394324d3SSascha Wildner }
2068394324d3SSascha Wildner
2069394324d3SSascha Wildner static void
ena_free_mgmnt_irq(struct ena_adapter * adapter)2070394324d3SSascha Wildner ena_free_mgmnt_irq(struct ena_adapter *adapter)
2071394324d3SSascha Wildner {
2072394324d3SSascha Wildner struct ena_irq *irq;
2073394324d3SSascha Wildner int rc;
2074394324d3SSascha Wildner
2075394324d3SSascha Wildner irq = &adapter->irq_tbl[ENA_MGMNT_IRQ_IDX];
2076394324d3SSascha Wildner if (irq->requested) {
2077394324d3SSascha Wildner ena_trace(ENA_INFO | ENA_ADMQ, "tear down irq: %d\n",
2078394324d3SSascha Wildner irq->vector);
2079394324d3SSascha Wildner rc = bus_teardown_intr(adapter->pdev, irq->res, irq->cookie);
2080394324d3SSascha Wildner if (unlikely(rc != 0))
2081394324d3SSascha Wildner device_printf(adapter->pdev, "failed to tear "
2082394324d3SSascha Wildner "down irq: %d\n", irq->vector);
2083394324d3SSascha Wildner irq->requested = 0;
2084394324d3SSascha Wildner }
2085394324d3SSascha Wildner
2086394324d3SSascha Wildner if (irq->res != NULL) {
2087394324d3SSascha Wildner ena_trace(ENA_INFO | ENA_ADMQ, "release resource irq: %d\n",
2088394324d3SSascha Wildner irq->vector);
2089394324d3SSascha Wildner rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2090394324d3SSascha Wildner irq->vector, irq->res);
209182a3fa28SBrad Hoffman pci_release_msix_vector(adapter->pdev, irq->vector);
2092394324d3SSascha Wildner irq->res = NULL;
2093394324d3SSascha Wildner if (unlikely(rc != 0))
2094394324d3SSascha Wildner device_printf(adapter->pdev, "dev has no parent while "
2095394324d3SSascha Wildner "releasing res for irq: %d\n", irq->vector);
2096394324d3SSascha Wildner }
2097394324d3SSascha Wildner }
2098394324d3SSascha Wildner
2099394324d3SSascha Wildner static void
ena_free_io_irq(struct ena_adapter * adapter)2100394324d3SSascha Wildner ena_free_io_irq(struct ena_adapter *adapter)
2101394324d3SSascha Wildner {
2102394324d3SSascha Wildner struct ena_irq *irq;
2103394324d3SSascha Wildner int rc;
2104394324d3SSascha Wildner
2105394324d3SSascha Wildner for (int i = ENA_IO_IRQ_FIRST_IDX; i < adapter->msix_vecs; i++) {
2106394324d3SSascha Wildner irq = &adapter->irq_tbl[i];
2107394324d3SSascha Wildner if (irq->requested) {
2108394324d3SSascha Wildner ena_trace(ENA_INFO | ENA_IOQ, "tear down irq: %d\n",
2109394324d3SSascha Wildner irq->vector);
2110394324d3SSascha Wildner rc = bus_teardown_intr(adapter->pdev, irq->res,
2111394324d3SSascha Wildner irq->cookie);
2112394324d3SSascha Wildner if (unlikely(rc != 0)) {
2113394324d3SSascha Wildner device_printf(adapter->pdev, "failed to tear "
2114394324d3SSascha Wildner "down irq: %d\n", irq->vector);
2115394324d3SSascha Wildner }
2116394324d3SSascha Wildner irq->requested = 0;
2117394324d3SSascha Wildner }
2118394324d3SSascha Wildner
2119394324d3SSascha Wildner if (irq->res != NULL) {
2120394324d3SSascha Wildner ena_trace(ENA_INFO | ENA_IOQ, "release resource irq: %d\n",
2121394324d3SSascha Wildner irq->vector);
2122394324d3SSascha Wildner rc = bus_release_resource(adapter->pdev, SYS_RES_IRQ,
2123394324d3SSascha Wildner irq->vector, irq->res);
212482a3fa28SBrad Hoffman pci_release_msix_vector(adapter->pdev, irq->vector);
2125394324d3SSascha Wildner irq->res = NULL;
2126394324d3SSascha Wildner if (unlikely(rc != 0)) {
2127394324d3SSascha Wildner device_printf(adapter->pdev, "dev has no parent"
2128394324d3SSascha Wildner " while releasing res for irq: %d\n",
2129394324d3SSascha Wildner irq->vector);
2130394324d3SSascha Wildner }
2131394324d3SSascha Wildner }
2132394324d3SSascha Wildner }
2133394324d3SSascha Wildner }
2134394324d3SSascha Wildner
2135394324d3SSascha Wildner static void
ena_free_irqs(struct ena_adapter * adapter)2136394324d3SSascha Wildner ena_free_irqs(struct ena_adapter* adapter)
2137394324d3SSascha Wildner {
2138394324d3SSascha Wildner
2139394324d3SSascha Wildner ena_free_io_irq(adapter);
2140394324d3SSascha Wildner ena_free_mgmnt_irq(adapter);
2141394324d3SSascha Wildner ena_disable_msix(adapter);
2142394324d3SSascha Wildner }
2143394324d3SSascha Wildner
2144394324d3SSascha Wildner static void
ena_disable_msix(struct ena_adapter * adapter)2145394324d3SSascha Wildner ena_disable_msix(struct ena_adapter *adapter)
2146394324d3SSascha Wildner {
2147394324d3SSascha Wildner
2148394324d3SSascha Wildner pci_release_msi(adapter->pdev);
2149394324d3SSascha Wildner
2150394324d3SSascha Wildner adapter->msix_vecs = 0;
215182a3fa28SBrad Hoffman kfree(adapter->msix_entries, M_DEVBUF);
2152394324d3SSascha Wildner adapter->msix_entries = NULL;
2153394324d3SSascha Wildner }
2154394324d3SSascha Wildner
2155394324d3SSascha Wildner static void
ena_unmask_all_io_irqs(struct ena_adapter * adapter)2156394324d3SSascha Wildner ena_unmask_all_io_irqs(struct ena_adapter *adapter)
2157394324d3SSascha Wildner {
2158394324d3SSascha Wildner struct ena_com_io_cq* io_cq;
2159394324d3SSascha Wildner struct ena_eth_io_intr_reg intr_reg;
2160394324d3SSascha Wildner uint16_t ena_qid;
2161394324d3SSascha Wildner int i;
2162394324d3SSascha Wildner
2163394324d3SSascha Wildner /* Unmask interrupts for all queues */
2164394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
2165394324d3SSascha Wildner ena_qid = ENA_IO_TXQ_IDX(i);
2166394324d3SSascha Wildner io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
2167394324d3SSascha Wildner ena_com_update_intr_reg(&intr_reg, 0, 0, true);
2168394324d3SSascha Wildner ena_com_unmask_intr(io_cq, &intr_reg);
2169394324d3SSascha Wildner }
2170394324d3SSascha Wildner }
2171394324d3SSascha Wildner
2172394324d3SSascha Wildner /* Configure the Rx forwarding */
2173394324d3SSascha Wildner static int
ena_rss_configure(struct ena_adapter * adapter)2174394324d3SSascha Wildner ena_rss_configure(struct ena_adapter *adapter)
2175394324d3SSascha Wildner {
2176394324d3SSascha Wildner struct ena_com_dev *ena_dev = adapter->ena_dev;
2177394324d3SSascha Wildner int rc;
2178394324d3SSascha Wildner
2179394324d3SSascha Wildner /* Set indirect table */
2180394324d3SSascha Wildner rc = ena_com_indirect_table_set(ena_dev);
2181394324d3SSascha Wildner if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
2182394324d3SSascha Wildner return (rc);
2183394324d3SSascha Wildner
2184394324d3SSascha Wildner /* Configure hash function (if supported) */
2185394324d3SSascha Wildner rc = ena_com_set_hash_function(ena_dev);
2186394324d3SSascha Wildner if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
2187394324d3SSascha Wildner return (rc);
2188394324d3SSascha Wildner
2189394324d3SSascha Wildner /* Configure hash inputs (if supported) */
2190394324d3SSascha Wildner rc = ena_com_set_hash_ctrl(ena_dev);
2191394324d3SSascha Wildner if (unlikely((rc != 0) && (rc != EOPNOTSUPP)))
2192394324d3SSascha Wildner return (rc);
2193394324d3SSascha Wildner
2194394324d3SSascha Wildner return (0);
2195394324d3SSascha Wildner }
2196394324d3SSascha Wildner
2197394324d3SSascha Wildner static int
ena_up_complete(struct ena_adapter * adapter)2198394324d3SSascha Wildner ena_up_complete(struct ena_adapter *adapter)
2199394324d3SSascha Wildner {
2200394324d3SSascha Wildner int rc;
2201394324d3SSascha Wildner
2202394324d3SSascha Wildner if (likely(adapter->rss_support)) {
2203394324d3SSascha Wildner rc = ena_rss_configure(adapter);
2204394324d3SSascha Wildner if (rc != 0)
2205394324d3SSascha Wildner return (rc);
2206394324d3SSascha Wildner }
2207394324d3SSascha Wildner
2208394324d3SSascha Wildner rc = ena_change_mtu(adapter->ifp, adapter->ifp->if_mtu);
2209394324d3SSascha Wildner if (unlikely(rc != 0))
2210394324d3SSascha Wildner return (rc);
2211394324d3SSascha Wildner
2212394324d3SSascha Wildner ena_refill_all_rx_bufs(adapter);
221382a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2214394324d3SSascha Wildner ena_reset_counters((counter_u64_t *)&adapter->hw_stats,
2215394324d3SSascha Wildner sizeof(adapter->hw_stats));
221682a3fa28SBrad Hoffman #endif
2217394324d3SSascha Wildner
2218394324d3SSascha Wildner return (0);
2219394324d3SSascha Wildner }
2220394324d3SSascha Wildner
2221394324d3SSascha Wildner static int
ena_up(struct ena_adapter * adapter)2222394324d3SSascha Wildner ena_up(struct ena_adapter *adapter)
2223394324d3SSascha Wildner {
2224394324d3SSascha Wildner int rc = 0;
2225394324d3SSascha Wildner
2226394324d3SSascha Wildner if (unlikely(device_is_attached(adapter->pdev) == 0)) {
2227394324d3SSascha Wildner device_printf(adapter->pdev, "device is not attached!\n");
2228394324d3SSascha Wildner return (ENXIO);
2229394324d3SSascha Wildner }
2230394324d3SSascha Wildner
2231394324d3SSascha Wildner if (unlikely(!adapter->running)) {
2232394324d3SSascha Wildner device_printf(adapter->pdev, "device is not running!\n");
2233394324d3SSascha Wildner return (ENXIO);
2234394324d3SSascha Wildner }
2235394324d3SSascha Wildner
2236394324d3SSascha Wildner if (!adapter->up) {
2237394324d3SSascha Wildner device_printf(adapter->pdev, "device is going UP\n");
2238394324d3SSascha Wildner
2239394324d3SSascha Wildner /* setup interrupts for IO queues */
2240394324d3SSascha Wildner ena_setup_io_intr(adapter);
2241394324d3SSascha Wildner rc = ena_request_io_irq(adapter);
2242394324d3SSascha Wildner if (unlikely(rc != 0)) {
2243394324d3SSascha Wildner ena_trace(ENA_ALERT, "err_req_irq");
2244394324d3SSascha Wildner goto err_req_irq;
2245394324d3SSascha Wildner }
2246394324d3SSascha Wildner
2247394324d3SSascha Wildner /* allocate transmit descriptors */
2248394324d3SSascha Wildner rc = ena_setup_all_tx_resources(adapter);
2249394324d3SSascha Wildner if (unlikely(rc != 0)) {
2250394324d3SSascha Wildner ena_trace(ENA_ALERT, "err_setup_tx");
2251394324d3SSascha Wildner goto err_setup_tx;
2252394324d3SSascha Wildner }
2253394324d3SSascha Wildner
2254394324d3SSascha Wildner /* allocate receive descriptors */
2255394324d3SSascha Wildner rc = ena_setup_all_rx_resources(adapter);
2256394324d3SSascha Wildner if (unlikely(rc != 0)) {
2257394324d3SSascha Wildner ena_trace(ENA_ALERT, "err_setup_rx");
2258394324d3SSascha Wildner goto err_setup_rx;
2259394324d3SSascha Wildner }
2260394324d3SSascha Wildner
2261394324d3SSascha Wildner /* create IO queues for Rx & Tx */
2262394324d3SSascha Wildner rc = ena_create_io_queues(adapter);
2263394324d3SSascha Wildner if (unlikely(rc != 0)) {
2264394324d3SSascha Wildner ena_trace(ENA_ALERT,
2265394324d3SSascha Wildner "create IO queues failed");
2266394324d3SSascha Wildner goto err_io_que;
2267394324d3SSascha Wildner }
2268394324d3SSascha Wildner
226982a3fa28SBrad Hoffman if (unlikely(adapter->link_status)) {
227082a3fa28SBrad Hoffman adapter->ifp->if_link_state = LINK_STATE_UP;
227182a3fa28SBrad Hoffman if_link_state_change(adapter->ifp);
227282a3fa28SBrad Hoffman }
2273394324d3SSascha Wildner
2274394324d3SSascha Wildner rc = ena_up_complete(adapter);
2275394324d3SSascha Wildner if (unlikely(rc != 0))
2276394324d3SSascha Wildner goto err_up_complete;
2277394324d3SSascha Wildner
227882a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2279394324d3SSascha Wildner counter_u64_add(adapter->dev_stats.interface_up, 1);
228082a3fa28SBrad Hoffman #endif
2281394324d3SSascha Wildner
2282394324d3SSascha Wildner ena_update_hwassist(adapter);
2283394324d3SSascha Wildner
228482a3fa28SBrad Hoffman adapter->ifp->if_flags |= IFF_RUNNING;
228582a3fa28SBrad Hoffman ifq_clr_oactive(&adapter->ifp->if_snd);
2286394324d3SSascha Wildner
228782a3fa28SBrad Hoffman callout_reset(&adapter->timer_service, hz,
228882a3fa28SBrad Hoffman ena_timer_service, (void *)adapter);
2289394324d3SSascha Wildner
2290394324d3SSascha Wildner adapter->up = true;
2291394324d3SSascha Wildner
2292394324d3SSascha Wildner ena_unmask_all_io_irqs(adapter);
2293394324d3SSascha Wildner }
2294394324d3SSascha Wildner
2295394324d3SSascha Wildner return (0);
2296394324d3SSascha Wildner
2297394324d3SSascha Wildner err_up_complete:
2298394324d3SSascha Wildner ena_destroy_all_io_queues(adapter);
2299394324d3SSascha Wildner err_io_que:
2300394324d3SSascha Wildner ena_free_all_rx_resources(adapter);
2301394324d3SSascha Wildner err_setup_rx:
2302394324d3SSascha Wildner ena_free_all_tx_resources(adapter);
2303394324d3SSascha Wildner err_setup_tx:
2304394324d3SSascha Wildner ena_free_io_irq(adapter);
2305394324d3SSascha Wildner err_req_irq:
2306394324d3SSascha Wildner return (rc);
2307394324d3SSascha Wildner }
2308394324d3SSascha Wildner
230982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2310394324d3SSascha Wildner static uint64_t
2311394324d3SSascha Wildner ena_get_counter(if_t ifp, ift_counter cnt)
2312394324d3SSascha Wildner {
2313394324d3SSascha Wildner struct ena_adapter *adapter;
2314394324d3SSascha Wildner struct ena_hw_stats *stats;
2315394324d3SSascha Wildner
231682a3fa28SBrad Hoffman adapter = ifp->if_softc;
2317394324d3SSascha Wildner stats = &adapter->hw_stats;
2318394324d3SSascha Wildner
2319394324d3SSascha Wildner switch (cnt) {
2320394324d3SSascha Wildner case IFCOUNTER_IPACKETS:
2321394324d3SSascha Wildner return (counter_u64_fetch(stats->rx_packets));
2322394324d3SSascha Wildner case IFCOUNTER_OPACKETS:
2323394324d3SSascha Wildner return (counter_u64_fetch(stats->tx_packets));
2324394324d3SSascha Wildner case IFCOUNTER_IBYTES:
2325394324d3SSascha Wildner return (counter_u64_fetch(stats->rx_bytes));
2326394324d3SSascha Wildner case IFCOUNTER_OBYTES:
2327394324d3SSascha Wildner return (counter_u64_fetch(stats->tx_bytes));
2328394324d3SSascha Wildner case IFCOUNTER_IQDROPS:
2329394324d3SSascha Wildner return (counter_u64_fetch(stats->rx_drops));
2330394324d3SSascha Wildner default:
2331394324d3SSascha Wildner return (if_get_counter_default(ifp, cnt));
2332394324d3SSascha Wildner }
2333394324d3SSascha Wildner }
233482a3fa28SBrad Hoffman #endif
2335394324d3SSascha Wildner
2336394324d3SSascha Wildner static int
ena_media_change(if_t ifp)2337394324d3SSascha Wildner ena_media_change(if_t ifp)
2338394324d3SSascha Wildner {
2339394324d3SSascha Wildner /* Media Change is not supported by firmware */
2340394324d3SSascha Wildner return (0);
2341394324d3SSascha Wildner }
2342394324d3SSascha Wildner
2343394324d3SSascha Wildner static void
ena_media_status(if_t ifp,struct ifmediareq * ifmr)2344394324d3SSascha Wildner ena_media_status(if_t ifp, struct ifmediareq *ifmr)
2345394324d3SSascha Wildner {
234682a3fa28SBrad Hoffman struct ena_adapter *adapter = ifp->if_softc;
2347394324d3SSascha Wildner ena_trace(ENA_DBG, "enter");
2348394324d3SSascha Wildner
234982a3fa28SBrad Hoffman lockmgr(&adapter->global_lock, LK_EXCLUSIVE);
2350394324d3SSascha Wildner
2351394324d3SSascha Wildner ifmr->ifm_status = IFM_AVALID;
2352394324d3SSascha Wildner ifmr->ifm_active = IFM_ETHER;
2353394324d3SSascha Wildner
2354394324d3SSascha Wildner if (!adapter->link_status) {
235582a3fa28SBrad Hoffman lockmgr(&adapter->global_lock, LK_RELEASE);
2356394324d3SSascha Wildner ena_trace(ENA_INFO, "link_status = false");
2357394324d3SSascha Wildner return;
2358394324d3SSascha Wildner }
2359394324d3SSascha Wildner
2360394324d3SSascha Wildner ifmr->ifm_status |= IFM_ACTIVE;
2361394324d3SSascha Wildner ifmr->ifm_active |= IFM_10G_T | IFM_FDX;
2362394324d3SSascha Wildner
236382a3fa28SBrad Hoffman lockmgr(&adapter->global_lock, LK_RELEASE);
2364394324d3SSascha Wildner }
2365394324d3SSascha Wildner
2366394324d3SSascha Wildner static void
ena_init(void * arg)2367394324d3SSascha Wildner ena_init(void *arg)
2368394324d3SSascha Wildner {
2369394324d3SSascha Wildner struct ena_adapter *adapter = (struct ena_adapter *)arg;
2370394324d3SSascha Wildner
2371394324d3SSascha Wildner if (!adapter->up) {
237282a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_EXCLUSIVE);
2373394324d3SSascha Wildner ena_up(adapter);
237482a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
2375394324d3SSascha Wildner }
2376394324d3SSascha Wildner }
2377394324d3SSascha Wildner
2378394324d3SSascha Wildner static int
ena_ioctl(if_t ifp,u_long command,caddr_t data,struct ucred * cred)237982a3fa28SBrad Hoffman ena_ioctl(if_t ifp, u_long command, caddr_t data, struct ucred *cred)
2380394324d3SSascha Wildner {
2381394324d3SSascha Wildner struct ena_adapter *adapter;
2382394324d3SSascha Wildner struct ifreq *ifr;
2383394324d3SSascha Wildner int rc;
2384394324d3SSascha Wildner
2385394324d3SSascha Wildner adapter = ifp->if_softc;
2386394324d3SSascha Wildner ifr = (struct ifreq *)data;
2387394324d3SSascha Wildner
2388394324d3SSascha Wildner /*
2389394324d3SSascha Wildner * Acquiring lock to prevent from running up and down routines parallel.
2390394324d3SSascha Wildner */
2391394324d3SSascha Wildner rc = 0;
2392394324d3SSascha Wildner switch (command) {
2393394324d3SSascha Wildner case SIOCSIFMTU:
239482a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_EXCLUSIVE);
2395394324d3SSascha Wildner ena_down(adapter);
2396394324d3SSascha Wildner
2397394324d3SSascha Wildner ena_change_mtu(ifp, ifr->ifr_mtu);
2398394324d3SSascha Wildner
2399394324d3SSascha Wildner rc = ena_up(adapter);
240082a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
2401394324d3SSascha Wildner break;
2402394324d3SSascha Wildner
2403394324d3SSascha Wildner case SIOCSIFFLAGS:
2404394324d3SSascha Wildner if ((ifp->if_flags & IFF_UP) != 0) {
240582a3fa28SBrad Hoffman if ((ifp->if_flags & IFF_RUNNING) != 0) {
2406394324d3SSascha Wildner if ((ifp->if_flags & (IFF_PROMISC |
2407394324d3SSascha Wildner IFF_ALLMULTI)) != 0) {
2408394324d3SSascha Wildner device_printf(adapter->pdev,
2409394324d3SSascha Wildner "ioctl promisc/allmulti\n");
2410394324d3SSascha Wildner }
2411394324d3SSascha Wildner } else {
241282a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_EXCLUSIVE);
2413394324d3SSascha Wildner rc = ena_up(adapter);
241482a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
2415394324d3SSascha Wildner }
2416394324d3SSascha Wildner } else {
241782a3fa28SBrad Hoffman if ((ifp->if_flags & IFF_RUNNING) != 0) {
241882a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_EXCLUSIVE);
2419394324d3SSascha Wildner ena_down(adapter);
242082a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
2421394324d3SSascha Wildner }
2422394324d3SSascha Wildner }
2423394324d3SSascha Wildner break;
2424394324d3SSascha Wildner
2425394324d3SSascha Wildner case SIOCADDMULTI:
2426394324d3SSascha Wildner case SIOCDELMULTI:
2427394324d3SSascha Wildner break;
2428394324d3SSascha Wildner
2429394324d3SSascha Wildner case SIOCSIFMEDIA:
2430394324d3SSascha Wildner case SIOCGIFMEDIA:
2431394324d3SSascha Wildner rc = ifmedia_ioctl(ifp, ifr, &adapter->media, command);
2432394324d3SSascha Wildner break;
2433394324d3SSascha Wildner
2434394324d3SSascha Wildner case SIOCSIFCAP:
2435394324d3SSascha Wildner {
2436394324d3SSascha Wildner int reinit = 0;
2437394324d3SSascha Wildner
2438394324d3SSascha Wildner if (ifr->ifr_reqcap != ifp->if_capenable) {
2439394324d3SSascha Wildner ifp->if_capenable = ifr->ifr_reqcap;
2440394324d3SSascha Wildner reinit = 1;
2441394324d3SSascha Wildner }
2442394324d3SSascha Wildner
2443394324d3SSascha Wildner if ((reinit != 0) &&
244482a3fa28SBrad Hoffman ((ifp->if_flags & IFF_RUNNING) != 0)) {
244582a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_EXCLUSIVE);
2446394324d3SSascha Wildner ena_down(adapter);
2447394324d3SSascha Wildner rc = ena_up(adapter);
244882a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
2449394324d3SSascha Wildner }
2450394324d3SSascha Wildner }
2451394324d3SSascha Wildner
2452394324d3SSascha Wildner break;
2453394324d3SSascha Wildner default:
2454394324d3SSascha Wildner rc = ether_ioctl(ifp, command, data);
2455394324d3SSascha Wildner break;
2456394324d3SSascha Wildner }
2457394324d3SSascha Wildner
2458394324d3SSascha Wildner return (rc);
2459394324d3SSascha Wildner }
2460394324d3SSascha Wildner
2461394324d3SSascha Wildner static int
ena_get_dev_offloads(struct ena_com_dev_get_features_ctx * feat)2462394324d3SSascha Wildner ena_get_dev_offloads(struct ena_com_dev_get_features_ctx *feat)
2463394324d3SSascha Wildner {
2464394324d3SSascha Wildner int caps = 0;
2465394324d3SSascha Wildner
2466394324d3SSascha Wildner if ((feat->offload.tx &
2467394324d3SSascha Wildner (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2468394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK |
2469394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK)) != 0)
2470394324d3SSascha Wildner caps |= IFCAP_TXCSUM;
2471394324d3SSascha Wildner
2472394324d3SSascha Wildner if ((feat->offload.tx &
2473394324d3SSascha Wildner (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_FULL_MASK |
2474394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV6_CSUM_PART_MASK)) != 0)
247582a3fa28SBrad Hoffman caps |= IFCAP_TXCSUM;
2476394324d3SSascha Wildner
2477394324d3SSascha Wildner if ((feat->offload.tx &
2478394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV4_MASK) != 0)
2479394324d3SSascha Wildner caps |= IFCAP_TSO4;
2480394324d3SSascha Wildner
2481394324d3SSascha Wildner if ((feat->offload.tx &
2482394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_TSO_IPV6_MASK) != 0)
2483394324d3SSascha Wildner caps |= IFCAP_TSO6;
2484394324d3SSascha Wildner
2485394324d3SSascha Wildner if ((feat->offload.rx_supported &
2486394324d3SSascha Wildner (ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV4_CSUM_MASK |
2487394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L3_CSUM_IPV4_MASK)) != 0)
2488394324d3SSascha Wildner caps |= IFCAP_RXCSUM;
2489394324d3SSascha Wildner
249082a3fa28SBrad Hoffman #if 0
2491394324d3SSascha Wildner if ((feat->offload.rx_supported &
2492394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_RX_L4_IPV6_CSUM_MASK) != 0)
2493394324d3SSascha Wildner caps |= IFCAP_RXCSUM_IPV6;
249482a3fa28SBrad Hoffman #endif
249582a3fa28SBrad Hoffman #if 0 /* XXX LRO */
249682a3fa28SBrad Hoffman caps |= IFCAP_LRO;
249782a3fa28SBrad Hoffman #endif
249882a3fa28SBrad Hoffman caps |= IFCAP_JUMBO_MTU;
2499394324d3SSascha Wildner
2500394324d3SSascha Wildner return (caps);
2501394324d3SSascha Wildner }
2502394324d3SSascha Wildner
2503394324d3SSascha Wildner static void
ena_update_host_info(struct ena_admin_host_info * host_info,if_t ifp)2504394324d3SSascha Wildner ena_update_host_info(struct ena_admin_host_info *host_info, if_t ifp)
2505394324d3SSascha Wildner {
2506394324d3SSascha Wildner
2507394324d3SSascha Wildner host_info->supported_network_features[0] =
250882a3fa28SBrad Hoffman (uint32_t)ifp->if_capabilities;
2509394324d3SSascha Wildner }
2510394324d3SSascha Wildner
2511394324d3SSascha Wildner static void
ena_update_hwassist(struct ena_adapter * adapter)2512394324d3SSascha Wildner ena_update_hwassist(struct ena_adapter *adapter)
2513394324d3SSascha Wildner {
2514394324d3SSascha Wildner if_t ifp = adapter->ifp;
2515394324d3SSascha Wildner uint32_t feat = adapter->tx_offload_cap;
251682a3fa28SBrad Hoffman int cap = ifp->if_capenable;
2517394324d3SSascha Wildner int flags = 0;
2518394324d3SSascha Wildner
251982a3fa28SBrad Hoffman ifp->if_hwassist = 0;
2520394324d3SSascha Wildner
2521394324d3SSascha Wildner if ((cap & IFCAP_TXCSUM) != 0) {
2522394324d3SSascha Wildner if ((feat &
2523394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L3_CSUM_IPV4_MASK) != 0)
2524394324d3SSascha Wildner flags |= CSUM_IP;
2525394324d3SSascha Wildner if ((feat &
2526394324d3SSascha Wildner (ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_FULL_MASK |
2527394324d3SSascha Wildner ENA_ADMIN_FEATURE_OFFLOAD_DESC_TX_L4_IPV4_CSUM_PART_MASK)) != 0)
252882a3fa28SBrad Hoffman flags |= CSUM_UDP | CSUM_TCP;
2529394324d3SSascha Wildner }
2530394324d3SSascha Wildner
253182a3fa28SBrad Hoffman #if 0
2532394324d3SSascha Wildner if ((cap & IFCAP_TXCSUM_IPV6) != 0)
2533394324d3SSascha Wildner flags |= CSUM_IP6_UDP | CSUM_IP6_TCP;
253482a3fa28SBrad Hoffman #endif
2535394324d3SSascha Wildner
253682a3fa28SBrad Hoffman if ((cap & IFCAP_TSO4) != 0 || (cap & IFCAP_TSO6) != 0)
253782a3fa28SBrad Hoffman flags |= CSUM_TSO;
2538394324d3SSascha Wildner
253982a3fa28SBrad Hoffman ifp->if_hwassist |= flags;
2540394324d3SSascha Wildner }
2541394324d3SSascha Wildner
2542394324d3SSascha Wildner static int
ena_setup_ifnet(device_t pdev,struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * feat)2543394324d3SSascha Wildner ena_setup_ifnet(device_t pdev, struct ena_adapter *adapter,
2544394324d3SSascha Wildner struct ena_com_dev_get_features_ctx *feat)
2545394324d3SSascha Wildner {
2546394324d3SSascha Wildner if_t ifp;
2547394324d3SSascha Wildner int caps = 0;
2548394324d3SSascha Wildner
254982a3fa28SBrad Hoffman ifp = adapter->ifp = if_alloc(IFT_ETHER);
2550394324d3SSascha Wildner if (unlikely(ifp == NULL)) {
2551394324d3SSascha Wildner ena_trace(ENA_ALERT, "can not allocate ifnet structure\n");
2552394324d3SSascha Wildner return (ENXIO);
2553394324d3SSascha Wildner }
2554394324d3SSascha Wildner if_initname(ifp, device_get_name(pdev), device_get_unit(pdev));
255582a3fa28SBrad Hoffman ifp->if_softc = adapter;
2556394324d3SSascha Wildner
255782a3fa28SBrad Hoffman ifp->if_flags = IFF_BROADCAST | IFF_SIMPLEX | IFF_MULTICAST;
255882a3fa28SBrad Hoffman
255982a3fa28SBrad Hoffman ifp->if_init = ena_init;
256082a3fa28SBrad Hoffman ifp->if_start = ena_start_xmit;
256182a3fa28SBrad Hoffman ifp->if_ioctl = ena_ioctl;
256282a3fa28SBrad Hoffman #if 0 /* XXX swildner counter */
2563394324d3SSascha Wildner if_setgetcounterfn(ifp, ena_get_counter);
256482a3fa28SBrad Hoffman #endif
2565394324d3SSascha Wildner
256682a3fa28SBrad Hoffman ifq_set_maxlen(&ifp->if_snd, adapter->tx_ring_size);
256782a3fa28SBrad Hoffman ifq_set_ready(&ifp->if_snd);
256882a3fa28SBrad Hoffman ifp->if_mtu = ETHERMTU;
256982a3fa28SBrad Hoffman ifp->if_baudrate = 0;
2570394324d3SSascha Wildner /* Zeroize capabilities... */
257182a3fa28SBrad Hoffman ifp->if_capabilities = 0;
257282a3fa28SBrad Hoffman ifp->if_capenable = 0;
2573394324d3SSascha Wildner /* check hardware support */
2574394324d3SSascha Wildner caps = ena_get_dev_offloads(feat);
2575394324d3SSascha Wildner /* ... and set them */
257682a3fa28SBrad Hoffman //if_setcapabilitiesbit(ifp, caps, 0);
257782a3fa28SBrad Hoffman ((struct ifnet *)ifp)->if_capabilities |= caps;
257882a3fa28SBrad Hoffman ((struct ifnet *)ifp)->if_capabilities &= ~0;
2579394324d3SSascha Wildner
2580394324d3SSascha Wildner /* TSO parameters */
258182a3fa28SBrad Hoffman //ifp->if_hw_tsomax = ENA_TSO_MAXSIZE -
258282a3fa28SBrad Hoffman // (ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN);
258382a3fa28SBrad Hoffman ifp->if_tsolen = adapter->max_tx_sgl_size - 1;
258482a3fa28SBrad Hoffman //ifp->if_hw_tsomaxsegsize = ENA_TSO_MAXSIZE;
2585394324d3SSascha Wildner
258682a3fa28SBrad Hoffman ifp->if_hdrlen = sizeof(struct ether_vlan_header);
258782a3fa28SBrad Hoffman ifp->if_capenable= ifp->if_capabilities;
2588394324d3SSascha Wildner
2589394324d3SSascha Wildner /*
2590394324d3SSascha Wildner * Specify the media types supported by this adapter and register
2591394324d3SSascha Wildner * callbacks to update media and link information
2592394324d3SSascha Wildner */
2593394324d3SSascha Wildner ifmedia_init(&adapter->media, IFM_IMASK,
2594394324d3SSascha Wildner ena_media_change, ena_media_status);
2595394324d3SSascha Wildner ifmedia_add(&adapter->media, IFM_ETHER | IFM_AUTO, 0, NULL);
2596394324d3SSascha Wildner ifmedia_set(&adapter->media, IFM_ETHER | IFM_AUTO);
2597394324d3SSascha Wildner
259882a3fa28SBrad Hoffman ether_ifattach(ifp, adapter->mac_addr, NULL);
2599394324d3SSascha Wildner
2600394324d3SSascha Wildner return (0);
2601394324d3SSascha Wildner }
2602394324d3SSascha Wildner
2603394324d3SSascha Wildner static void
ena_down(struct ena_adapter * adapter)2604394324d3SSascha Wildner ena_down(struct ena_adapter *adapter)
2605394324d3SSascha Wildner {
2606394324d3SSascha Wildner int rc;
2607394324d3SSascha Wildner
2608394324d3SSascha Wildner if (adapter->up) {
2609394324d3SSascha Wildner device_printf(adapter->pdev, "device is going DOWN\n");
2610394324d3SSascha Wildner
2611394324d3SSascha Wildner callout_drain(&adapter->timer_service);
2612394324d3SSascha Wildner
2613394324d3SSascha Wildner adapter->up = false;
261482a3fa28SBrad Hoffman ifq_set_oactive(&adapter->ifp->if_snd);
261582a3fa28SBrad Hoffman adapter->ifp->if_flags &= ~IFF_RUNNING;
2616394324d3SSascha Wildner
2617394324d3SSascha Wildner ena_free_io_irq(adapter);
2618394324d3SSascha Wildner
2619394324d3SSascha Wildner if (adapter->trigger_reset) {
2620394324d3SSascha Wildner rc = ena_com_dev_reset(adapter->ena_dev,
2621394324d3SSascha Wildner adapter->reset_reason);
2622394324d3SSascha Wildner if (unlikely(rc != 0))
2623394324d3SSascha Wildner device_printf(adapter->pdev,
2624394324d3SSascha Wildner "Device reset failed\n");
2625394324d3SSascha Wildner }
2626394324d3SSascha Wildner
2627394324d3SSascha Wildner ena_destroy_all_io_queues(adapter);
2628394324d3SSascha Wildner
2629394324d3SSascha Wildner ena_free_all_tx_bufs(adapter);
2630394324d3SSascha Wildner ena_free_all_rx_bufs(adapter);
2631394324d3SSascha Wildner ena_free_all_tx_resources(adapter);
2632394324d3SSascha Wildner ena_free_all_rx_resources(adapter);
2633394324d3SSascha Wildner
263482a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2635394324d3SSascha Wildner counter_u64_add(adapter->dev_stats.interface_down, 1);
263682a3fa28SBrad Hoffman #endif
2637394324d3SSascha Wildner }
2638394324d3SSascha Wildner }
2639394324d3SSascha Wildner
2640394324d3SSascha Wildner static void
ena_tx_csum(struct ena_com_tx_ctx * ena_tx_ctx,struct mbuf * mbuf)2641394324d3SSascha Wildner ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf)
2642394324d3SSascha Wildner {
2643394324d3SSascha Wildner struct ena_com_tx_meta *ena_meta;
2644394324d3SSascha Wildner struct ether_vlan_header *eh;
2645394324d3SSascha Wildner u32 mss;
2646394324d3SSascha Wildner bool offload;
2647394324d3SSascha Wildner uint16_t etype;
2648394324d3SSascha Wildner int ehdrlen;
2649394324d3SSascha Wildner struct ip *ip;
2650394324d3SSascha Wildner int iphlen;
2651394324d3SSascha Wildner struct tcphdr *th;
2652394324d3SSascha Wildner
2653394324d3SSascha Wildner offload = false;
2654394324d3SSascha Wildner ena_meta = &ena_tx_ctx->ena_meta;
2655394324d3SSascha Wildner mss = mbuf->m_pkthdr.tso_segsz;
2656394324d3SSascha Wildner
2657394324d3SSascha Wildner if (mss != 0)
2658394324d3SSascha Wildner offload = true;
2659394324d3SSascha Wildner
2660394324d3SSascha Wildner if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)
2661394324d3SSascha Wildner offload = true;
2662394324d3SSascha Wildner
2663394324d3SSascha Wildner if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)
2664394324d3SSascha Wildner offload = true;
2665394324d3SSascha Wildner
2666394324d3SSascha Wildner if (!offload) {
2667394324d3SSascha Wildner ena_tx_ctx->meta_valid = 0;
2668394324d3SSascha Wildner return;
2669394324d3SSascha Wildner }
2670394324d3SSascha Wildner
2671394324d3SSascha Wildner /* Determine where frame payload starts. */
2672394324d3SSascha Wildner eh = mtod(mbuf, struct ether_vlan_header *);
2673394324d3SSascha Wildner if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
2674394324d3SSascha Wildner etype = ntohs(eh->evl_proto);
2675394324d3SSascha Wildner ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
2676394324d3SSascha Wildner } else {
2677394324d3SSascha Wildner etype = ntohs(eh->evl_encap_proto);
2678394324d3SSascha Wildner ehdrlen = ETHER_HDR_LEN;
2679394324d3SSascha Wildner }
2680394324d3SSascha Wildner
2681394324d3SSascha Wildner ip = (struct ip *)(mbuf->m_data + ehdrlen);
2682394324d3SSascha Wildner iphlen = ip->ip_hl << 2;
2683394324d3SSascha Wildner th = (struct tcphdr *)((caddr_t)ip + iphlen);
2684394324d3SSascha Wildner
2685394324d3SSascha Wildner if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) {
2686394324d3SSascha Wildner ena_tx_ctx->l3_csum_enable = 1;
2687394324d3SSascha Wildner }
2688394324d3SSascha Wildner if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
2689394324d3SSascha Wildner ena_tx_ctx->tso_enable = 1;
2690394324d3SSascha Wildner ena_meta->l4_hdr_len = (th->th_off);
2691394324d3SSascha Wildner }
2692394324d3SSascha Wildner
2693394324d3SSascha Wildner switch (etype) {
2694394324d3SSascha Wildner case ETHERTYPE_IP:
2695394324d3SSascha Wildner ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
2696394324d3SSascha Wildner if ((ip->ip_off & htons(IP_DF)) != 0)
2697394324d3SSascha Wildner ena_tx_ctx->df = 1;
2698394324d3SSascha Wildner break;
2699394324d3SSascha Wildner case ETHERTYPE_IPV6:
2700394324d3SSascha Wildner ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
2701394324d3SSascha Wildner
2702394324d3SSascha Wildner default:
2703394324d3SSascha Wildner break;
2704394324d3SSascha Wildner }
2705394324d3SSascha Wildner
2706394324d3SSascha Wildner if (ip->ip_p == IPPROTO_TCP) {
2707394324d3SSascha Wildner ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
2708394324d3SSascha Wildner if ((mbuf->m_pkthdr.csum_flags &
270982a3fa28SBrad Hoffman CSUM_TCP) != 0)
2710394324d3SSascha Wildner ena_tx_ctx->l4_csum_enable = 1;
2711394324d3SSascha Wildner else
2712394324d3SSascha Wildner ena_tx_ctx->l4_csum_enable = 0;
2713394324d3SSascha Wildner } else if (ip->ip_p == IPPROTO_UDP) {
2714394324d3SSascha Wildner ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
2715394324d3SSascha Wildner if ((mbuf->m_pkthdr.csum_flags &
271682a3fa28SBrad Hoffman CSUM_UDP) != 0)
2717394324d3SSascha Wildner ena_tx_ctx->l4_csum_enable = 1;
2718394324d3SSascha Wildner else
2719394324d3SSascha Wildner ena_tx_ctx->l4_csum_enable = 0;
2720394324d3SSascha Wildner } else {
2721394324d3SSascha Wildner ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
2722394324d3SSascha Wildner ena_tx_ctx->l4_csum_enable = 0;
2723394324d3SSascha Wildner }
2724394324d3SSascha Wildner
2725394324d3SSascha Wildner ena_meta->mss = mss;
2726394324d3SSascha Wildner ena_meta->l3_hdr_len = iphlen;
2727394324d3SSascha Wildner ena_meta->l3_hdr_offset = ehdrlen;
2728394324d3SSascha Wildner ena_tx_ctx->meta_valid = 1;
2729394324d3SSascha Wildner }
2730394324d3SSascha Wildner
2731394324d3SSascha Wildner static int
ena_check_and_collapse_mbuf(struct ena_ring * tx_ring,struct mbuf ** mbuf)2732394324d3SSascha Wildner ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
2733394324d3SSascha Wildner {
2734394324d3SSascha Wildner struct ena_adapter *adapter;
2735394324d3SSascha Wildner struct mbuf *collapsed_mbuf;
2736394324d3SSascha Wildner int num_frags;
2737394324d3SSascha Wildner
2738394324d3SSascha Wildner adapter = tx_ring->adapter;
2739394324d3SSascha Wildner num_frags = ena_mbuf_count(*mbuf);
2740394324d3SSascha Wildner
2741394324d3SSascha Wildner /* One segment must be reserved for configuration descriptor. */
2742394324d3SSascha Wildner if (num_frags < adapter->max_tx_sgl_size)
2743394324d3SSascha Wildner return (0);
274482a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2745394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.collapse, 1);
274682a3fa28SBrad Hoffman #endif
2747394324d3SSascha Wildner
274882a3fa28SBrad Hoffman collapsed_mbuf = m_defrag(*mbuf, M_NOWAIT);
2749394324d3SSascha Wildner if (unlikely(collapsed_mbuf == NULL)) {
275082a3fa28SBrad Hoffman IFNET_STAT_INC(tx_ring->adapter->ifp, oerrors, 1);
275182a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2752394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.collapse_err, 1);
275382a3fa28SBrad Hoffman #endif
2754394324d3SSascha Wildner return (ENOMEM);
2755394324d3SSascha Wildner }
2756394324d3SSascha Wildner
2757394324d3SSascha Wildner /* If mbuf was collapsed succesfully, original mbuf is released. */
2758394324d3SSascha Wildner *mbuf = collapsed_mbuf;
2759394324d3SSascha Wildner
2760394324d3SSascha Wildner return (0);
2761394324d3SSascha Wildner }
2762394324d3SSascha Wildner
2763394324d3SSascha Wildner static int
ena_xmit_mbuf(struct ena_ring * tx_ring,struct mbuf ** mbuf)2764394324d3SSascha Wildner ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
2765394324d3SSascha Wildner {
2766394324d3SSascha Wildner struct ena_adapter *adapter;
2767394324d3SSascha Wildner struct ena_tx_buffer *tx_info;
2768394324d3SSascha Wildner struct ena_com_tx_ctx ena_tx_ctx;
2769394324d3SSascha Wildner struct ena_com_dev *ena_dev;
2770394324d3SSascha Wildner struct ena_com_buf *ena_buf;
2771394324d3SSascha Wildner struct ena_com_io_sq* io_sq;
2772394324d3SSascha Wildner bus_dma_segment_t segs[ENA_BUS_DMA_SEGS];
2773394324d3SSascha Wildner void *push_hdr;
2774394324d3SSascha Wildner uint16_t next_to_use;
2775394324d3SSascha Wildner uint16_t req_id;
2776394324d3SSascha Wildner uint16_t push_len;
2777394324d3SSascha Wildner uint16_t ena_qid;
2778394324d3SSascha Wildner uint32_t len, nsegs, header_len;
2779394324d3SSascha Wildner int i, rc;
2780394324d3SSascha Wildner int nb_hw_desc;
2781394324d3SSascha Wildner
2782394324d3SSascha Wildner ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
2783394324d3SSascha Wildner adapter = tx_ring->que->adapter;
2784394324d3SSascha Wildner ena_dev = adapter->ena_dev;
2785394324d3SSascha Wildner io_sq = &ena_dev->io_sq_queues[ena_qid];
2786394324d3SSascha Wildner
278782a3fa28SBrad Hoffman //tx_ring is just used to grab the adapter
2788394324d3SSascha Wildner rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);
2789394324d3SSascha Wildner if (unlikely(rc != 0)) {
2790394324d3SSascha Wildner ena_trace(ENA_WARNING,
2791394324d3SSascha Wildner "Failed to collapse mbuf! err: %d", rc);
2792394324d3SSascha Wildner return (rc);
2793394324d3SSascha Wildner }
2794394324d3SSascha Wildner
2795394324d3SSascha Wildner next_to_use = tx_ring->next_to_use;
2796394324d3SSascha Wildner req_id = tx_ring->free_tx_ids[next_to_use];
2797394324d3SSascha Wildner tx_info = &tx_ring->tx_buffer_info[req_id];
2798394324d3SSascha Wildner
2799394324d3SSascha Wildner tx_info->mbuf = *mbuf;
2800394324d3SSascha Wildner tx_info->num_of_bufs = 0;
2801394324d3SSascha Wildner
2802394324d3SSascha Wildner ena_buf = tx_info->bufs;
2803394324d3SSascha Wildner len = (*mbuf)->m_len;
2804394324d3SSascha Wildner
2805394324d3SSascha Wildner ena_trace(ENA_DBG | ENA_TXPTH, "Tx: %d bytes", (*mbuf)->m_pkthdr.len);
2806394324d3SSascha Wildner
2807394324d3SSascha Wildner push_len = 0;
2808394324d3SSascha Wildner header_len = min_t(uint32_t, len, tx_ring->tx_max_header_size);
2809394324d3SSascha Wildner push_hdr = NULL;
2810394324d3SSascha Wildner
281182a3fa28SBrad Hoffman rc = bus_dmamap_load_mbuf_segment(adapter->tx_buf_tag, tx_info->map,
281282a3fa28SBrad Hoffman *mbuf, segs, adapter->max_tx_sgl_size, &nsegs, BUS_DMA_NOWAIT);
2813394324d3SSascha Wildner
2814394324d3SSascha Wildner if (unlikely((rc != 0) || (nsegs == 0))) {
2815394324d3SSascha Wildner ena_trace(ENA_WARNING,
2816394324d3SSascha Wildner "dmamap load failed! err: %d nsegs: %d", rc, nsegs);
281782a3fa28SBrad Hoffman IFNET_STAT_INC(tx_ring->adapter->ifp, oerrors, 1);
281882a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2819394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1);
282082a3fa28SBrad Hoffman #endif
2821394324d3SSascha Wildner tx_info->mbuf = NULL;
2822394324d3SSascha Wildner if (rc == ENOMEM)
2823394324d3SSascha Wildner return (ENA_COM_NO_MEM);
2824394324d3SSascha Wildner else
2825394324d3SSascha Wildner return (ENA_COM_INVAL);
2826394324d3SSascha Wildner }
2827394324d3SSascha Wildner
2828394324d3SSascha Wildner for (i = 0; i < nsegs; i++) {
2829394324d3SSascha Wildner ena_buf->len = segs[i].ds_len;
2830394324d3SSascha Wildner ena_buf->paddr = segs[i].ds_addr;
2831394324d3SSascha Wildner ena_buf++;
2832394324d3SSascha Wildner }
2833394324d3SSascha Wildner tx_info->num_of_bufs = nsegs;
2834394324d3SSascha Wildner
2835394324d3SSascha Wildner memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
2836394324d3SSascha Wildner ena_tx_ctx.ena_bufs = tx_info->bufs;
2837394324d3SSascha Wildner ena_tx_ctx.push_header = push_hdr;
2838394324d3SSascha Wildner ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
2839394324d3SSascha Wildner ena_tx_ctx.req_id = req_id;
2840394324d3SSascha Wildner ena_tx_ctx.header_len = header_len;
2841394324d3SSascha Wildner
2842394324d3SSascha Wildner /* Set flags and meta data */
2843394324d3SSascha Wildner ena_tx_csum(&ena_tx_ctx, *mbuf);
2844394324d3SSascha Wildner /* Prepare the packet's descriptors and send them to device */
2845394324d3SSascha Wildner rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc);
2846394324d3SSascha Wildner if (unlikely(rc != 0)) {
2847394324d3SSascha Wildner device_printf(adapter->pdev, "failed to prepare tx bufs\n");
284882a3fa28SBrad Hoffman IFNET_STAT_INC(tx_ring->adapter->ifp, oerrors, 1);
284982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2850394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
285182a3fa28SBrad Hoffman #endif
2852394324d3SSascha Wildner goto dma_error;
2853394324d3SSascha Wildner }
2854394324d3SSascha Wildner
285582a3fa28SBrad Hoffman IFNET_STAT_INC(tx_ring->adapter->ifp, opackets, 1);
285682a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2857394324d3SSascha Wildner counter_enter();
2858394324d3SSascha Wildner counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
2859394324d3SSascha Wildner counter_u64_add_protected(tx_ring->tx_stats.bytes,
2860394324d3SSascha Wildner (*mbuf)->m_pkthdr.len);
2861394324d3SSascha Wildner
2862394324d3SSascha Wildner counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
2863394324d3SSascha Wildner counter_u64_add_protected(adapter->hw_stats.tx_bytes,
2864394324d3SSascha Wildner (*mbuf)->m_pkthdr.len);
2865394324d3SSascha Wildner counter_exit();
286682a3fa28SBrad Hoffman #endif
2867394324d3SSascha Wildner
2868394324d3SSascha Wildner tx_info->tx_descs = nb_hw_desc;
286982a3fa28SBrad Hoffman getmicrouptime(&tx_info->timestamp);
2870394324d3SSascha Wildner tx_info->print_once = true;
2871394324d3SSascha Wildner
2872394324d3SSascha Wildner tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
2873394324d3SSascha Wildner tx_ring->ring_size);
2874394324d3SSascha Wildner
2875394324d3SSascha Wildner bus_dmamap_sync(adapter->tx_buf_tag, tx_info->map,
2876394324d3SSascha Wildner BUS_DMASYNC_PREWRITE);
2877394324d3SSascha Wildner
2878394324d3SSascha Wildner return (0);
2879394324d3SSascha Wildner
2880394324d3SSascha Wildner dma_error:
2881394324d3SSascha Wildner tx_info->mbuf = NULL;
2882394324d3SSascha Wildner bus_dmamap_unload(adapter->tx_buf_tag, tx_info->map);
2883394324d3SSascha Wildner
2884394324d3SSascha Wildner return (rc);
2885394324d3SSascha Wildner }
2886394324d3SSascha Wildner
2887394324d3SSascha Wildner static void
ena_start_xmit(struct ifnet * ifp,struct ifaltq_subque * ifsq)288882a3fa28SBrad Hoffman ena_start_xmit(struct ifnet *ifp, struct ifaltq_subque *ifsq)
2889394324d3SSascha Wildner {
289082a3fa28SBrad Hoffman /*
289182a3fa28SBrad Hoffman * TODO: Might need to initialize an ena_ring with the
289282a3fa28SBrad Hoffman * ifaltq_subque in it
289382a3fa28SBrad Hoffman */
289482a3fa28SBrad Hoffman struct ena_adapter *adapter = ifp->if_softc;
2895394324d3SSascha Wildner struct ena_com_io_sq *io_sq;
289682a3fa28SBrad Hoffman struct ena_ring *tx_ring;
2897394324d3SSascha Wildner int ena_qid;
2898394324d3SSascha Wildner int acum_pkts = 0;
2899394324d3SSascha Wildner int ret = 0;
2900394324d3SSascha Wildner
290182a3fa28SBrad Hoffman if (unlikely((adapter->ifp->if_flags & IFF_RUNNING) == 0) ||
290282a3fa28SBrad Hoffman ifsq_is_oactive(ifsq)) {
2903394324d3SSascha Wildner return;
290482a3fa28SBrad Hoffman }
2905394324d3SSascha Wildner
290682a3fa28SBrad Hoffman /* Check is link_active and some other shit. If it is, purge. */
290782a3fa28SBrad Hoffman
290882a3fa28SBrad Hoffman #if 0
2909394324d3SSascha Wildner if (unlikely(!adapter->link_status))
2910394324d3SSascha Wildner return;
291182a3fa28SBrad Hoffman #endif
2912394324d3SSascha Wildner
291382a3fa28SBrad Hoffman io_sq = NULL;
291482a3fa28SBrad Hoffman tx_ring = NULL;
291582a3fa28SBrad Hoffman
291682a3fa28SBrad Hoffman while (!ifsq_is_empty(ifsq)) {
291782a3fa28SBrad Hoffman struct mbuf *m_head;
291882a3fa28SBrad Hoffman int i;
291982a3fa28SBrad Hoffman
292082a3fa28SBrad Hoffman //Grab head from mbuf list
292182a3fa28SBrad Hoffman m_head = ifsq_dequeue(ifsq);
292282a3fa28SBrad Hoffman if (m_head == NULL)
292382a3fa28SBrad Hoffman break;
292482a3fa28SBrad Hoffman
292582a3fa28SBrad Hoffman //pick the associated tx_ring based on hash
292682a3fa28SBrad Hoffman i = m_head->m_pkthdr.hash % adapter->num_queues;
292782a3fa28SBrad Hoffman
292882a3fa28SBrad Hoffman tx_ring = &adapter->tx_ring[i];
292982a3fa28SBrad Hoffman ENA_RING_MTX_LOCK(tx_ring);
2930394324d3SSascha Wildner ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
2931394324d3SSascha Wildner io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
2932394324d3SSascha Wildner
293382a3fa28SBrad Hoffman if (unlikely(!ena_com_sq_have_enough_space(io_sq, ENA_TX_CLEANUP_THRESHOLD)))
2934394324d3SSascha Wildner ena_tx_cleanup(tx_ring);
2935394324d3SSascha Wildner
293682a3fa28SBrad Hoffman if (unlikely((ret = ena_xmit_mbuf(tx_ring, &m_head)) != 0)) {
2937394324d3SSascha Wildner if (ret == ENA_COM_NO_MEM) {
293882a3fa28SBrad Hoffman /* XXX put mbuf back on queue */
2939394324d3SSascha Wildner } else if (ret == ENA_COM_NO_SPACE) {
294082a3fa28SBrad Hoffman /* XXX put mbuf back on queue */
2941394324d3SSascha Wildner } else {
294282a3fa28SBrad Hoffman m_freem(m_head);
294382a3fa28SBrad Hoffman /* XXX advance mbuf queue aka move it forward? */
2944394324d3SSascha Wildner }
294582a3fa28SBrad Hoffman ENA_RING_MTX_UNLOCK(tx_ring);
2946394324d3SSascha Wildner break;
2947394324d3SSascha Wildner }
2948394324d3SSascha Wildner
294982a3fa28SBrad Hoffman //advance mbuf queue, might already be handled by dequeue
2950394324d3SSascha Wildner
295182a3fa28SBrad Hoffman #if 0
295282a3fa28SBrad Hoffman // dillon - wtf is this doing here?
295382a3fa28SBrad Hoffman // NOT SURE WHAT TO DO WITH THIS CODE
295482a3fa28SBrad Hoffman if (unlikely((adapter->ifp->if_flags & IFF_RUNNING) == 0))
295582a3fa28SBrad Hoffman return; // break here, not return. tx_ring locked
295682a3fa28SBrad Hoffman #endif
2957394324d3SSascha Wildner
2958394324d3SSascha Wildner acum_pkts++;
2959394324d3SSascha Wildner
296082a3fa28SBrad Hoffman ENA_RING_MTX_UNLOCK(tx_ring);
296182a3fa28SBrad Hoffman BPF_MTAP(adapter->ifp, m_head);
2962394324d3SSascha Wildner
2963394324d3SSascha Wildner if (unlikely(acum_pkts == DB_THRESHOLD)) {
2964394324d3SSascha Wildner acum_pkts = 0;
2965394324d3SSascha Wildner wmb();
2966394324d3SSascha Wildner /* Trigger the dma engine */
2967394324d3SSascha Wildner ena_com_write_sq_doorbell(io_sq);
296882a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2969394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.doorbells, 1);
297082a3fa28SBrad Hoffman #endif
2971394324d3SSascha Wildner }
2972394324d3SSascha Wildner
2973394324d3SSascha Wildner }
2974394324d3SSascha Wildner
2975394324d3SSascha Wildner if (likely(acum_pkts != 0)) {
2976394324d3SSascha Wildner wmb();
2977394324d3SSascha Wildner /* Trigger the dma engine */
2978394324d3SSascha Wildner ena_com_write_sq_doorbell(io_sq);
297982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
2980394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.doorbells, 1);
2981394324d3SSascha Wildner #endif
2982394324d3SSascha Wildner }
2983394324d3SSascha Wildner
298482a3fa28SBrad Hoffman if (io_sq &&
298582a3fa28SBrad Hoffman !ena_com_sq_have_enough_space(io_sq, ENA_TX_CLEANUP_THRESHOLD)) {
2986394324d3SSascha Wildner ENA_RING_MTX_LOCK(tx_ring);
298782a3fa28SBrad Hoffman ena_tx_cleanup(tx_ring);
2988394324d3SSascha Wildner ENA_RING_MTX_UNLOCK(tx_ring);
2989394324d3SSascha Wildner }
2990394324d3SSascha Wildner }
2991394324d3SSascha Wildner
2992394324d3SSascha Wildner static int
ena_calc_io_queue_num(struct ena_adapter * adapter,struct ena_com_dev_get_features_ctx * get_feat_ctx)2993394324d3SSascha Wildner ena_calc_io_queue_num(struct ena_adapter *adapter,
2994394324d3SSascha Wildner struct ena_com_dev_get_features_ctx *get_feat_ctx)
2995394324d3SSascha Wildner {
2996394324d3SSascha Wildner int io_sq_num, io_cq_num, io_queue_num;
2997394324d3SSascha Wildner
2998394324d3SSascha Wildner io_sq_num = get_feat_ctx->max_queues.max_sq_num;
2999394324d3SSascha Wildner io_cq_num = get_feat_ctx->max_queues.max_cq_num;
3000394324d3SSascha Wildner
300182a3fa28SBrad Hoffman io_queue_num = min_t(int, ncpus, ENA_MAX_NUM_IO_QUEUES);
3002394324d3SSascha Wildner io_queue_num = min_t(int, io_queue_num, io_sq_num);
3003394324d3SSascha Wildner io_queue_num = min_t(int, io_queue_num, io_cq_num);
3004394324d3SSascha Wildner /* 1 IRQ for for mgmnt and 1 IRQ for each TX/RX pair */
3005394324d3SSascha Wildner io_queue_num = min_t(int, io_queue_num,
3006394324d3SSascha Wildner pci_msix_count(adapter->pdev) - 1);
3007394324d3SSascha Wildner #ifdef RSS
3008394324d3SSascha Wildner io_queue_num = min_t(int, io_queue_num, rss_getnumbuckets());
3009394324d3SSascha Wildner #endif
3010394324d3SSascha Wildner
3011394324d3SSascha Wildner return (io_queue_num);
3012394324d3SSascha Wildner }
3013394324d3SSascha Wildner
3014394324d3SSascha Wildner static int
ena_calc_queue_size(struct ena_adapter * adapter,uint16_t * max_tx_sgl_size,uint16_t * max_rx_sgl_size,struct ena_com_dev_get_features_ctx * feat)3015394324d3SSascha Wildner ena_calc_queue_size(struct ena_adapter *adapter, uint16_t *max_tx_sgl_size,
3016394324d3SSascha Wildner uint16_t *max_rx_sgl_size, struct ena_com_dev_get_features_ctx *feat)
3017394324d3SSascha Wildner {
3018394324d3SSascha Wildner uint32_t queue_size = ENA_DEFAULT_RING_SIZE;
3019394324d3SSascha Wildner uint32_t v;
3020394324d3SSascha Wildner uint32_t q;
3021394324d3SSascha Wildner
3022394324d3SSascha Wildner queue_size = min_t(uint32_t, queue_size,
3023394324d3SSascha Wildner feat->max_queues.max_cq_depth);
3024394324d3SSascha Wildner queue_size = min_t(uint32_t, queue_size,
3025394324d3SSascha Wildner feat->max_queues.max_sq_depth);
3026394324d3SSascha Wildner
3027394324d3SSascha Wildner /* round down to the nearest power of 2 */
3028394324d3SSascha Wildner v = queue_size;
3029394324d3SSascha Wildner while (v != 0) {
3030394324d3SSascha Wildner if (powerof2(queue_size) != 0)
3031394324d3SSascha Wildner break;
3032394324d3SSascha Wildner v /= 2;
3033394324d3SSascha Wildner q = rounddown2(queue_size, v);
3034394324d3SSascha Wildner if (q != 0) {
3035394324d3SSascha Wildner queue_size = q;
3036394324d3SSascha Wildner break;
3037394324d3SSascha Wildner }
3038394324d3SSascha Wildner }
3039394324d3SSascha Wildner
3040394324d3SSascha Wildner if (unlikely(queue_size == 0)) {
3041394324d3SSascha Wildner device_printf(adapter->pdev, "Invalid queue size\n");
3042394324d3SSascha Wildner return (ENA_COM_FAULT);
3043394324d3SSascha Wildner }
3044394324d3SSascha Wildner
3045394324d3SSascha Wildner *max_tx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
3046394324d3SSascha Wildner feat->max_queues.max_packet_tx_descs);
3047394324d3SSascha Wildner *max_rx_sgl_size = min_t(uint16_t, ENA_PKT_MAX_BUFS,
3048394324d3SSascha Wildner feat->max_queues.max_packet_rx_descs);
3049394324d3SSascha Wildner
3050394324d3SSascha Wildner return (queue_size);
3051394324d3SSascha Wildner }
3052394324d3SSascha Wildner
3053394324d3SSascha Wildner static int
ena_rss_init_default(struct ena_adapter * adapter)3054394324d3SSascha Wildner ena_rss_init_default(struct ena_adapter *adapter)
3055394324d3SSascha Wildner {
3056394324d3SSascha Wildner struct ena_com_dev *ena_dev = adapter->ena_dev;
3057394324d3SSascha Wildner device_t dev = adapter->pdev;
3058394324d3SSascha Wildner int qid, rc, i;
3059394324d3SSascha Wildner
3060394324d3SSascha Wildner rc = ena_com_rss_init(ena_dev, ENA_RX_RSS_TABLE_LOG_SIZE);
3061394324d3SSascha Wildner if (unlikely(rc != 0)) {
3062394324d3SSascha Wildner device_printf(dev, "Cannot init indirect table\n");
3063394324d3SSascha Wildner return (rc);
3064394324d3SSascha Wildner }
3065394324d3SSascha Wildner
3066394324d3SSascha Wildner for (i = 0; i < ENA_RX_RSS_TABLE_SIZE; i++) {
3067394324d3SSascha Wildner #ifdef RSS
3068394324d3SSascha Wildner qid = rss_get_indirection_to_bucket(i);
3069394324d3SSascha Wildner qid = qid % adapter->num_queues;
3070394324d3SSascha Wildner #else
3071394324d3SSascha Wildner qid = i % adapter->num_queues;
3072394324d3SSascha Wildner #endif
3073394324d3SSascha Wildner rc = ena_com_indirect_table_fill_entry(ena_dev, i,
3074394324d3SSascha Wildner ENA_IO_RXQ_IDX(qid));
3075394324d3SSascha Wildner if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
3076394324d3SSascha Wildner device_printf(dev, "Cannot fill indirect table\n");
3077394324d3SSascha Wildner goto err_rss_destroy;
3078394324d3SSascha Wildner }
3079394324d3SSascha Wildner }
3080394324d3SSascha Wildner
3081394324d3SSascha Wildner rc = ena_com_fill_hash_function(ena_dev, ENA_ADMIN_CRC32, NULL,
3082394324d3SSascha Wildner ENA_HASH_KEY_SIZE, 0xFFFFFFFF);
3083394324d3SSascha Wildner if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
3084394324d3SSascha Wildner device_printf(dev, "Cannot fill hash function\n");
3085394324d3SSascha Wildner goto err_rss_destroy;
3086394324d3SSascha Wildner }
3087394324d3SSascha Wildner
3088394324d3SSascha Wildner rc = ena_com_set_default_hash_ctrl(ena_dev);
3089394324d3SSascha Wildner if (unlikely((rc != 0) && (rc != EOPNOTSUPP))) {
3090394324d3SSascha Wildner device_printf(dev, "Cannot fill hash control\n");
3091394324d3SSascha Wildner goto err_rss_destroy;
3092394324d3SSascha Wildner }
3093394324d3SSascha Wildner
3094394324d3SSascha Wildner return (0);
3095394324d3SSascha Wildner
3096394324d3SSascha Wildner err_rss_destroy:
3097394324d3SSascha Wildner ena_com_rss_destroy(ena_dev);
3098394324d3SSascha Wildner return (rc);
3099394324d3SSascha Wildner }
3100394324d3SSascha Wildner
3101394324d3SSascha Wildner static void
ena_rss_init_default_deferred(void * arg)3102394324d3SSascha Wildner ena_rss_init_default_deferred(void *arg)
3103394324d3SSascha Wildner {
3104394324d3SSascha Wildner struct ena_adapter *adapter;
3105394324d3SSascha Wildner devclass_t dc;
3106394324d3SSascha Wildner int max;
3107394324d3SSascha Wildner int rc;
3108394324d3SSascha Wildner
3109394324d3SSascha Wildner dc = devclass_find("ena");
3110394324d3SSascha Wildner if (unlikely(dc == NULL)) {
3111394324d3SSascha Wildner ena_trace(ENA_ALERT, "No devclass ena\n");
3112394324d3SSascha Wildner return;
3113394324d3SSascha Wildner }
3114394324d3SSascha Wildner
3115394324d3SSascha Wildner max = devclass_get_maxunit(dc);
3116394324d3SSascha Wildner while (max-- >= 0) {
3117394324d3SSascha Wildner adapter = devclass_get_softc(dc, max);
3118394324d3SSascha Wildner if (adapter != NULL) {
3119394324d3SSascha Wildner rc = ena_rss_init_default(adapter);
3120394324d3SSascha Wildner adapter->rss_support = true;
3121394324d3SSascha Wildner if (unlikely(rc != 0)) {
3122394324d3SSascha Wildner device_printf(adapter->pdev,
3123394324d3SSascha Wildner "WARNING: RSS was not properly initialized,"
3124394324d3SSascha Wildner " it will affect bandwidth\n");
3125394324d3SSascha Wildner adapter->rss_support = false;
3126394324d3SSascha Wildner }
3127394324d3SSascha Wildner }
3128394324d3SSascha Wildner }
3129394324d3SSascha Wildner }
3130394324d3SSascha Wildner SYSINIT(ena_rss_init, SI_SUB_KICK_SCHEDULER, SI_ORDER_SECOND, ena_rss_init_default_deferred, NULL);
3131394324d3SSascha Wildner
3132394324d3SSascha Wildner static void
ena_config_host_info(struct ena_com_dev * ena_dev)3133394324d3SSascha Wildner ena_config_host_info(struct ena_com_dev *ena_dev)
3134394324d3SSascha Wildner {
3135394324d3SSascha Wildner struct ena_admin_host_info *host_info;
3136394324d3SSascha Wildner int rc;
3137394324d3SSascha Wildner
3138394324d3SSascha Wildner /* Allocate only the host info */
3139394324d3SSascha Wildner rc = ena_com_allocate_host_info(ena_dev);
3140394324d3SSascha Wildner if (unlikely(rc != 0)) {
3141394324d3SSascha Wildner ena_trace(ENA_ALERT, "Cannot allocate host info\n");
3142394324d3SSascha Wildner return;
3143394324d3SSascha Wildner }
3144394324d3SSascha Wildner
3145394324d3SSascha Wildner host_info = ena_dev->host_attr.host_info;
3146394324d3SSascha Wildner
3147394324d3SSascha Wildner host_info->os_type = ENA_ADMIN_OS_FREEBSD;
3148394324d3SSascha Wildner host_info->kernel_ver = osreldate;
3149394324d3SSascha Wildner
315082a3fa28SBrad Hoffman ksprintf(host_info->kernel_ver_str, "%d", osreldate);
3151394324d3SSascha Wildner host_info->os_dist = 0;
3152394324d3SSascha Wildner strncpy(host_info->os_dist_str, osrelease,
3153394324d3SSascha Wildner sizeof(host_info->os_dist_str) - 1);
3154394324d3SSascha Wildner
3155394324d3SSascha Wildner host_info->driver_version =
3156394324d3SSascha Wildner (DRV_MODULE_VER_MAJOR) |
3157394324d3SSascha Wildner (DRV_MODULE_VER_MINOR << ENA_ADMIN_HOST_INFO_MINOR_SHIFT) |
3158394324d3SSascha Wildner (DRV_MODULE_VER_SUBMINOR << ENA_ADMIN_HOST_INFO_SUB_MINOR_SHIFT);
3159394324d3SSascha Wildner
3160394324d3SSascha Wildner rc = ena_com_set_host_attributes(ena_dev);
3161394324d3SSascha Wildner if (unlikely(rc != 0)) {
3162394324d3SSascha Wildner if (rc == EOPNOTSUPP)
3163394324d3SSascha Wildner ena_trace(ENA_WARNING, "Cannot set host attributes\n");
3164394324d3SSascha Wildner else
3165394324d3SSascha Wildner ena_trace(ENA_ALERT, "Cannot set host attributes\n");
3166394324d3SSascha Wildner
3167394324d3SSascha Wildner goto err;
3168394324d3SSascha Wildner }
3169394324d3SSascha Wildner
3170394324d3SSascha Wildner return;
3171394324d3SSascha Wildner
3172394324d3SSascha Wildner err:
3173394324d3SSascha Wildner ena_com_delete_host_info(ena_dev);
3174394324d3SSascha Wildner }
3175394324d3SSascha Wildner
3176394324d3SSascha Wildner static int
ena_device_init(struct ena_adapter * adapter,device_t pdev,struct ena_com_dev_get_features_ctx * get_feat_ctx,int * wd_active)3177394324d3SSascha Wildner ena_device_init(struct ena_adapter *adapter, device_t pdev,
3178394324d3SSascha Wildner struct ena_com_dev_get_features_ctx *get_feat_ctx, int *wd_active)
3179394324d3SSascha Wildner {
3180394324d3SSascha Wildner struct ena_com_dev* ena_dev = adapter->ena_dev;
3181394324d3SSascha Wildner bool readless_supported;
3182394324d3SSascha Wildner uint32_t aenq_groups;
3183394324d3SSascha Wildner int dma_width;
3184394324d3SSascha Wildner int rc;
3185394324d3SSascha Wildner
3186394324d3SSascha Wildner rc = ena_com_mmio_reg_read_request_init(ena_dev);
3187394324d3SSascha Wildner if (unlikely(rc != 0)) {
3188394324d3SSascha Wildner device_printf(pdev, "failed to init mmio read less\n");
3189394324d3SSascha Wildner return (rc);
3190394324d3SSascha Wildner }
3191394324d3SSascha Wildner
3192394324d3SSascha Wildner /*
3193394324d3SSascha Wildner * The PCIe configuration space revision id indicate if mmio reg
3194394324d3SSascha Wildner * read is disabled
3195394324d3SSascha Wildner */
3196394324d3SSascha Wildner readless_supported = !(pci_get_revid(pdev) & ENA_MMIO_DISABLE_REG_READ);
3197394324d3SSascha Wildner ena_com_set_mmio_read_mode(ena_dev, readless_supported);
3198394324d3SSascha Wildner
3199394324d3SSascha Wildner rc = ena_com_dev_reset(ena_dev, ENA_REGS_RESET_NORMAL);
3200394324d3SSascha Wildner if (unlikely(rc != 0)) {
3201394324d3SSascha Wildner device_printf(pdev, "Can not reset device\n");
3202394324d3SSascha Wildner goto err_mmio_read_less;
3203394324d3SSascha Wildner }
3204394324d3SSascha Wildner
3205394324d3SSascha Wildner rc = ena_com_validate_version(ena_dev);
3206394324d3SSascha Wildner if (unlikely(rc != 0)) {
3207394324d3SSascha Wildner device_printf(pdev, "device version is too low\n");
3208394324d3SSascha Wildner goto err_mmio_read_less;
3209394324d3SSascha Wildner }
3210394324d3SSascha Wildner
3211394324d3SSascha Wildner dma_width = ena_com_get_dma_width(ena_dev);
3212394324d3SSascha Wildner if (unlikely(dma_width < 0)) {
3213394324d3SSascha Wildner device_printf(pdev, "Invalid dma width value %d", dma_width);
3214394324d3SSascha Wildner rc = dma_width;
3215394324d3SSascha Wildner goto err_mmio_read_less;
3216394324d3SSascha Wildner }
3217394324d3SSascha Wildner adapter->dma_width = dma_width;
3218394324d3SSascha Wildner
3219394324d3SSascha Wildner /* ENA admin level init */
3220394324d3SSascha Wildner rc = ena_com_admin_init(ena_dev, &aenq_handlers, true);
3221394324d3SSascha Wildner if (unlikely(rc != 0)) {
3222394324d3SSascha Wildner device_printf(pdev,
3223394324d3SSascha Wildner "Can not initialize ena admin queue with device\n");
3224394324d3SSascha Wildner goto err_mmio_read_less;
3225394324d3SSascha Wildner }
3226394324d3SSascha Wildner
3227394324d3SSascha Wildner /*
3228394324d3SSascha Wildner * To enable the msix interrupts the driver needs to know the number
3229394324d3SSascha Wildner * of queues. So the driver uses polling mode to retrieve this
3230394324d3SSascha Wildner * information
3231394324d3SSascha Wildner */
3232394324d3SSascha Wildner ena_com_set_admin_polling_mode(ena_dev, true);
3233394324d3SSascha Wildner
3234394324d3SSascha Wildner ena_config_host_info(ena_dev);
3235394324d3SSascha Wildner
3236394324d3SSascha Wildner /* Get Device Attributes */
3237394324d3SSascha Wildner rc = ena_com_get_dev_attr_feat(ena_dev, get_feat_ctx);
3238394324d3SSascha Wildner if (unlikely(rc != 0)) {
3239394324d3SSascha Wildner device_printf(pdev,
3240394324d3SSascha Wildner "Cannot get attribute for ena device rc: %d\n", rc);
3241394324d3SSascha Wildner goto err_admin_init;
3242394324d3SSascha Wildner }
3243394324d3SSascha Wildner
3244394324d3SSascha Wildner aenq_groups = BIT(ENA_ADMIN_LINK_CHANGE) | BIT(ENA_ADMIN_KEEP_ALIVE);
3245394324d3SSascha Wildner
3246394324d3SSascha Wildner aenq_groups &= get_feat_ctx->aenq.supported_groups;
3247394324d3SSascha Wildner rc = ena_com_set_aenq_config(ena_dev, aenq_groups);
3248394324d3SSascha Wildner if (unlikely(rc != 0)) {
3249394324d3SSascha Wildner device_printf(pdev, "Cannot configure aenq groups rc: %d\n", rc);
3250394324d3SSascha Wildner goto err_admin_init;
3251394324d3SSascha Wildner }
3252394324d3SSascha Wildner
3253394324d3SSascha Wildner *wd_active = !!(aenq_groups & BIT(ENA_ADMIN_KEEP_ALIVE));
3254394324d3SSascha Wildner
3255394324d3SSascha Wildner return (0);
3256394324d3SSascha Wildner
3257394324d3SSascha Wildner err_admin_init:
3258394324d3SSascha Wildner ena_com_delete_host_info(ena_dev);
3259394324d3SSascha Wildner ena_com_admin_destroy(ena_dev);
3260394324d3SSascha Wildner err_mmio_read_less:
3261394324d3SSascha Wildner ena_com_mmio_reg_read_request_destroy(ena_dev);
3262394324d3SSascha Wildner
3263394324d3SSascha Wildner return (rc);
3264394324d3SSascha Wildner }
3265394324d3SSascha Wildner
ena_enable_msix_and_set_admin_interrupts(struct ena_adapter * adapter,int io_vectors)3266394324d3SSascha Wildner static int ena_enable_msix_and_set_admin_interrupts(struct ena_adapter *adapter,
3267394324d3SSascha Wildner int io_vectors)
3268394324d3SSascha Wildner {
3269394324d3SSascha Wildner struct ena_com_dev *ena_dev = adapter->ena_dev;
3270394324d3SSascha Wildner int rc;
3271394324d3SSascha Wildner
3272394324d3SSascha Wildner rc = ena_enable_msix(adapter);
3273394324d3SSascha Wildner if (unlikely(rc != 0)) {
3274394324d3SSascha Wildner device_printf(adapter->pdev, "Error with MSI-X enablement\n");
3275394324d3SSascha Wildner return (rc);
3276394324d3SSascha Wildner }
3277394324d3SSascha Wildner
3278394324d3SSascha Wildner ena_setup_mgmnt_intr(adapter);
3279394324d3SSascha Wildner
3280394324d3SSascha Wildner rc = ena_request_mgmnt_irq(adapter);
3281394324d3SSascha Wildner if (unlikely(rc != 0)) {
3282394324d3SSascha Wildner device_printf(adapter->pdev, "Cannot setup mgmnt queue intr\n");
3283394324d3SSascha Wildner goto err_disable_msix;
3284394324d3SSascha Wildner }
3285394324d3SSascha Wildner
328682a3fa28SBrad Hoffman pci_enable_msix(adapter->pdev);
328782a3fa28SBrad Hoffman
3288394324d3SSascha Wildner ena_com_set_admin_polling_mode(ena_dev, false);
3289394324d3SSascha Wildner
3290394324d3SSascha Wildner ena_com_admin_aenq_enable(ena_dev);
3291394324d3SSascha Wildner
3292394324d3SSascha Wildner return (0);
3293394324d3SSascha Wildner
3294394324d3SSascha Wildner err_disable_msix:
3295394324d3SSascha Wildner ena_disable_msix(adapter);
3296394324d3SSascha Wildner
3297394324d3SSascha Wildner return (rc);
3298394324d3SSascha Wildner }
3299394324d3SSascha Wildner
3300394324d3SSascha Wildner /* Function called on ENA_ADMIN_KEEP_ALIVE event */
ena_keep_alive_wd(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)3301394324d3SSascha Wildner static void ena_keep_alive_wd(void *adapter_data,
3302394324d3SSascha Wildner struct ena_admin_aenq_entry *aenq_e)
3303394324d3SSascha Wildner {
3304394324d3SSascha Wildner struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3305394324d3SSascha Wildner struct ena_admin_aenq_keep_alive_desc *desc;
330682a3fa28SBrad Hoffman struct timeval time;
3307394324d3SSascha Wildner uint64_t rx_drops;
3308394324d3SSascha Wildner
3309394324d3SSascha Wildner desc = (struct ena_admin_aenq_keep_alive_desc *)aenq_e;
3310394324d3SSascha Wildner
3311394324d3SSascha Wildner rx_drops = ((uint64_t)desc->rx_drops_high << 32) | desc->rx_drops_low;
331282a3fa28SBrad Hoffman IFNET_STAT_INC(adapter->ifp, iqdrops, 1);
331382a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
3314394324d3SSascha Wildner counter_u64_zero(adapter->hw_stats.rx_drops);
3315394324d3SSascha Wildner counter_u64_add(adapter->hw_stats.rx_drops, rx_drops);
331682a3fa28SBrad Hoffman #endif
3317394324d3SSascha Wildner
331882a3fa28SBrad Hoffman getmicrouptime(&time);
331982a3fa28SBrad Hoffman atomic_store_rel_64(&adapter->keep_alive_timestamp.tv_sec, time.tv_sec);
3320394324d3SSascha Wildner }
3321394324d3SSascha Wildner
3322394324d3SSascha Wildner /* Check for keep alive expiration */
check_for_missing_keep_alive(struct ena_adapter * adapter)3323394324d3SSascha Wildner static void check_for_missing_keep_alive(struct ena_adapter *adapter)
3324394324d3SSascha Wildner {
332582a3fa28SBrad Hoffman struct timeval timestamp, time;
3326394324d3SSascha Wildner
3327394324d3SSascha Wildner if (adapter->wd_active == 0)
3328394324d3SSascha Wildner return;
3329394324d3SSascha Wildner
3330394324d3SSascha Wildner if (likely(adapter->keep_alive_timeout == 0))
3331394324d3SSascha Wildner return;
3332394324d3SSascha Wildner
333382a3fa28SBrad Hoffman timestamp.tv_sec = atomic_load_acq_64(&adapter->keep_alive_timestamp.tv_sec);
333482a3fa28SBrad Hoffman getmicrouptime(&time);
333582a3fa28SBrad Hoffman timevalsub(&time, ×tamp);
333682a3fa28SBrad Hoffman if (unlikely(time.tv_sec > adapter->keep_alive_timeout)) {
3337394324d3SSascha Wildner device_printf(adapter->pdev,
3338394324d3SSascha Wildner "Keep alive watchdog timeout.\n");
333982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
3340394324d3SSascha Wildner counter_u64_add(adapter->dev_stats.wd_expired, 1);
334182a3fa28SBrad Hoffman #endif
3342394324d3SSascha Wildner adapter->reset_reason = ENA_REGS_RESET_KEEP_ALIVE_TO;
3343394324d3SSascha Wildner adapter->trigger_reset = true;
3344394324d3SSascha Wildner }
3345394324d3SSascha Wildner }
3346394324d3SSascha Wildner
3347394324d3SSascha Wildner /* Check if admin queue is enabled */
check_for_admin_com_state(struct ena_adapter * adapter)3348394324d3SSascha Wildner static void check_for_admin_com_state(struct ena_adapter *adapter)
3349394324d3SSascha Wildner {
3350394324d3SSascha Wildner if (unlikely(ena_com_get_admin_running_state(adapter->ena_dev) ==
3351394324d3SSascha Wildner false)) {
3352394324d3SSascha Wildner device_printf(adapter->pdev,
3353394324d3SSascha Wildner "ENA admin queue is not in running state!\n");
335482a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
3355394324d3SSascha Wildner counter_u64_add(adapter->dev_stats.admin_q_pause, 1);
335682a3fa28SBrad Hoffman #endif
3357394324d3SSascha Wildner adapter->reset_reason = ENA_REGS_RESET_ADMIN_TO;
3358394324d3SSascha Wildner adapter->trigger_reset = true;
3359394324d3SSascha Wildner }
3360394324d3SSascha Wildner }
3361394324d3SSascha Wildner
3362394324d3SSascha Wildner static int
check_missing_comp_in_queue(struct ena_adapter * adapter,struct ena_ring * tx_ring)3363394324d3SSascha Wildner check_missing_comp_in_queue(struct ena_adapter *adapter,
3364394324d3SSascha Wildner struct ena_ring *tx_ring)
3365394324d3SSascha Wildner {
336682a3fa28SBrad Hoffman struct timeval curtime, time;
3367394324d3SSascha Wildner struct ena_tx_buffer *tx_buf;
3368394324d3SSascha Wildner uint32_t missed_tx = 0;
3369394324d3SSascha Wildner int i;
3370394324d3SSascha Wildner
337182a3fa28SBrad Hoffman getmicrouptime(&curtime);
3372394324d3SSascha Wildner
3373394324d3SSascha Wildner for (i = 0; i < tx_ring->ring_size; i++) {
3374394324d3SSascha Wildner tx_buf = &tx_ring->tx_buffer_info[i];
3375394324d3SSascha Wildner
337682a3fa28SBrad Hoffman if (timevalisset(&tx_buf->timestamp) == 0)
3377394324d3SSascha Wildner continue;
3378394324d3SSascha Wildner
3379394324d3SSascha Wildner time = curtime;
338082a3fa28SBrad Hoffman timevalsub(&time, &tx_buf->timestamp);
3381394324d3SSascha Wildner
3382394324d3SSascha Wildner /* Check again if packet is still waiting */
338382a3fa28SBrad Hoffman //WATCH: Might not be exactly comparable
338482a3fa28SBrad Hoffman if (unlikely(time.tv_sec > adapter->missing_tx_timeout)) {
3385394324d3SSascha Wildner
3386394324d3SSascha Wildner if (!tx_buf->print_once)
3387394324d3SSascha Wildner ena_trace(ENA_WARNING, "Found a Tx that wasn't "
3388394324d3SSascha Wildner "completed on time, qid %d, index %d.\n",
3389394324d3SSascha Wildner tx_ring->qid, i);
3390394324d3SSascha Wildner
3391394324d3SSascha Wildner tx_buf->print_once = true;
3392394324d3SSascha Wildner missed_tx++;
339382a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
3394394324d3SSascha Wildner counter_u64_add(tx_ring->tx_stats.missing_tx_comp, 1);
339582a3fa28SBrad Hoffman #endif
3396394324d3SSascha Wildner
3397394324d3SSascha Wildner if (unlikely(missed_tx >
3398394324d3SSascha Wildner adapter->missing_tx_threshold)) {
3399394324d3SSascha Wildner device_printf(adapter->pdev,
3400394324d3SSascha Wildner "The number of lost tx completion "
3401394324d3SSascha Wildner "is above the threshold (%d > %d). "
3402394324d3SSascha Wildner "Reset the device\n",
3403394324d3SSascha Wildner missed_tx, adapter->missing_tx_threshold);
3404394324d3SSascha Wildner adapter->reset_reason =
3405394324d3SSascha Wildner ENA_REGS_RESET_MISS_TX_CMPL;
3406394324d3SSascha Wildner adapter->trigger_reset = true;
3407394324d3SSascha Wildner return (EIO);
3408394324d3SSascha Wildner }
3409394324d3SSascha Wildner }
3410394324d3SSascha Wildner }
3411394324d3SSascha Wildner
3412394324d3SSascha Wildner return (0);
3413394324d3SSascha Wildner }
3414394324d3SSascha Wildner
3415394324d3SSascha Wildner /*
3416394324d3SSascha Wildner * Check for TX which were not completed on time.
3417394324d3SSascha Wildner * Timeout is defined by "missing_tx_timeout".
3418394324d3SSascha Wildner * Reset will be performed if number of incompleted
3419394324d3SSascha Wildner * transactions exceeds "missing_tx_threshold".
3420394324d3SSascha Wildner */
3421394324d3SSascha Wildner static void
check_for_missing_tx_completions(struct ena_adapter * adapter)3422394324d3SSascha Wildner check_for_missing_tx_completions(struct ena_adapter *adapter)
3423394324d3SSascha Wildner {
3424394324d3SSascha Wildner struct ena_ring *tx_ring;
3425394324d3SSascha Wildner int i, budget, rc;
3426394324d3SSascha Wildner
3427394324d3SSascha Wildner /* Make sure the driver doesn't turn the device in other process */
3428394324d3SSascha Wildner rmb();
3429394324d3SSascha Wildner
3430394324d3SSascha Wildner if (!adapter->up)
3431394324d3SSascha Wildner return;
3432394324d3SSascha Wildner
3433394324d3SSascha Wildner if (adapter->trigger_reset)
3434394324d3SSascha Wildner return;
3435394324d3SSascha Wildner
3436394324d3SSascha Wildner if (adapter->missing_tx_timeout == 0)
3437394324d3SSascha Wildner return;
3438394324d3SSascha Wildner
3439394324d3SSascha Wildner budget = adapter->missing_tx_max_queues;
3440394324d3SSascha Wildner
3441394324d3SSascha Wildner for (i = adapter->next_monitored_tx_qid; i < adapter->num_queues; i++) {
3442394324d3SSascha Wildner tx_ring = &adapter->tx_ring[i];
3443394324d3SSascha Wildner
3444394324d3SSascha Wildner rc = check_missing_comp_in_queue(adapter, tx_ring);
3445394324d3SSascha Wildner if (unlikely(rc != 0))
3446394324d3SSascha Wildner return;
3447394324d3SSascha Wildner
3448394324d3SSascha Wildner budget--;
3449394324d3SSascha Wildner if (budget == 0) {
3450394324d3SSascha Wildner i++;
3451394324d3SSascha Wildner break;
3452394324d3SSascha Wildner }
3453394324d3SSascha Wildner }
3454394324d3SSascha Wildner
3455394324d3SSascha Wildner adapter->next_monitored_tx_qid = i % adapter->num_queues;
3456394324d3SSascha Wildner }
3457394324d3SSascha Wildner
3458394324d3SSascha Wildner /* trigger deferred rx cleanup after 2 consecutive detections */
3459394324d3SSascha Wildner #define EMPTY_RX_REFILL 2
3460394324d3SSascha Wildner /* For the rare case where the device runs out of Rx descriptors and the
3461394324d3SSascha Wildner * msix handler failed to refill new Rx descriptors (due to a lack of memory
3462394324d3SSascha Wildner * for example).
3463394324d3SSascha Wildner * This case will lead to a deadlock:
3464394324d3SSascha Wildner * The device won't send interrupts since all the new Rx packets will be dropped
3465394324d3SSascha Wildner * The msix handler won't allocate new Rx descriptors so the device won't be
3466394324d3SSascha Wildner * able to send new packets.
3467394324d3SSascha Wildner *
3468394324d3SSascha Wildner * When such a situation is detected - execute rx cleanup task in another thread
3469394324d3SSascha Wildner */
3470394324d3SSascha Wildner static void
check_for_empty_rx_ring(struct ena_adapter * adapter)3471394324d3SSascha Wildner check_for_empty_rx_ring(struct ena_adapter *adapter)
3472394324d3SSascha Wildner {
3473394324d3SSascha Wildner struct ena_ring *rx_ring;
3474394324d3SSascha Wildner int i, refill_required;
3475394324d3SSascha Wildner
3476394324d3SSascha Wildner if (!adapter->up)
3477394324d3SSascha Wildner return;
3478394324d3SSascha Wildner
3479394324d3SSascha Wildner if (adapter->trigger_reset)
3480394324d3SSascha Wildner return;
3481394324d3SSascha Wildner
3482394324d3SSascha Wildner for (i = 0; i < adapter->num_queues; i++) {
3483394324d3SSascha Wildner rx_ring = &adapter->rx_ring[i];
3484394324d3SSascha Wildner
3485394324d3SSascha Wildner refill_required = ena_com_free_desc(rx_ring->ena_com_io_sq);
3486394324d3SSascha Wildner if (unlikely(refill_required == (rx_ring->ring_size - 1))) {
3487394324d3SSascha Wildner rx_ring->empty_rx_queue++;
3488394324d3SSascha Wildner
3489394324d3SSascha Wildner if (rx_ring->empty_rx_queue >= EMPTY_RX_REFILL) {
349082a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
3491394324d3SSascha Wildner counter_u64_add(rx_ring->rx_stats.empty_rx_ring,
3492394324d3SSascha Wildner 1);
349382a3fa28SBrad Hoffman #endif
3494394324d3SSascha Wildner
3495394324d3SSascha Wildner device_printf(adapter->pdev,
3496394324d3SSascha Wildner "trigger refill for ring %d\n", i);
3497394324d3SSascha Wildner
3498394324d3SSascha Wildner taskqueue_enqueue(rx_ring->cmpl_tq,
3499394324d3SSascha Wildner &rx_ring->cmpl_task);
3500394324d3SSascha Wildner rx_ring->empty_rx_queue = 0;
3501394324d3SSascha Wildner }
3502394324d3SSascha Wildner } else {
3503394324d3SSascha Wildner rx_ring->empty_rx_queue = 0;
3504394324d3SSascha Wildner }
3505394324d3SSascha Wildner }
3506394324d3SSascha Wildner }
3507394324d3SSascha Wildner
3508394324d3SSascha Wildner static void
ena_timer_service(void * data)3509394324d3SSascha Wildner ena_timer_service(void *data)
3510394324d3SSascha Wildner {
3511394324d3SSascha Wildner struct ena_adapter *adapter = (struct ena_adapter *)data;
3512394324d3SSascha Wildner struct ena_admin_host_info *host_info =
3513394324d3SSascha Wildner adapter->ena_dev->host_attr.host_info;
3514394324d3SSascha Wildner
3515394324d3SSascha Wildner check_for_missing_keep_alive(adapter);
3516394324d3SSascha Wildner
3517394324d3SSascha Wildner check_for_admin_com_state(adapter);
3518394324d3SSascha Wildner
3519394324d3SSascha Wildner check_for_missing_tx_completions(adapter);
3520394324d3SSascha Wildner
3521394324d3SSascha Wildner check_for_empty_rx_ring(adapter);
3522394324d3SSascha Wildner
3523394324d3SSascha Wildner if (host_info != NULL)
3524394324d3SSascha Wildner ena_update_host_info(host_info, adapter->ifp);
3525394324d3SSascha Wildner
3526394324d3SSascha Wildner if (unlikely(adapter->trigger_reset)) {
3527394324d3SSascha Wildner device_printf(adapter->pdev, "Trigger reset is on\n");
3528394324d3SSascha Wildner taskqueue_enqueue(adapter->reset_tq, &adapter->reset_task);
3529394324d3SSascha Wildner return;
3530394324d3SSascha Wildner }
3531394324d3SSascha Wildner
3532394324d3SSascha Wildner /*
3533394324d3SSascha Wildner * Schedule another timeout one second from now.
3534394324d3SSascha Wildner */
353582a3fa28SBrad Hoffman /* XXX swildner callout_schedule_sbt(&adapter->timer_service, SBT_1S, SBT_1S, 0); */
353682a3fa28SBrad Hoffman callout_reset(&adapter->timer_service, hz, ena_timer_service,
353782a3fa28SBrad Hoffman (void *)adapter);
3538394324d3SSascha Wildner }
3539394324d3SSascha Wildner
3540394324d3SSascha Wildner static void
ena_reset_task(void * arg,int pending)3541394324d3SSascha Wildner ena_reset_task(void *arg, int pending)
3542394324d3SSascha Wildner {
3543394324d3SSascha Wildner struct ena_com_dev_get_features_ctx get_feat_ctx;
3544394324d3SSascha Wildner struct ena_adapter *adapter = (struct ena_adapter *)arg;
3545394324d3SSascha Wildner struct ena_com_dev *ena_dev = adapter->ena_dev;
3546394324d3SSascha Wildner bool dev_up;
3547394324d3SSascha Wildner int rc;
3548394324d3SSascha Wildner
3549394324d3SSascha Wildner if (unlikely(!adapter->trigger_reset)) {
3550394324d3SSascha Wildner device_printf(adapter->pdev,
3551394324d3SSascha Wildner "device reset scheduled but trigger_reset is off\n");
3552394324d3SSascha Wildner return;
3553394324d3SSascha Wildner }
3554394324d3SSascha Wildner
355582a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_EXCLUSIVE);
3556394324d3SSascha Wildner
3557394324d3SSascha Wildner callout_drain(&adapter->timer_service);
3558394324d3SSascha Wildner
3559394324d3SSascha Wildner dev_up = adapter->up;
3560394324d3SSascha Wildner
3561394324d3SSascha Wildner ena_com_set_admin_running_state(ena_dev, false);
3562394324d3SSascha Wildner ena_down(adapter);
3563394324d3SSascha Wildner ena_free_mgmnt_irq(adapter);
3564394324d3SSascha Wildner ena_disable_msix(adapter);
3565394324d3SSascha Wildner ena_com_abort_admin_commands(ena_dev);
3566394324d3SSascha Wildner ena_com_wait_for_abort_completion(ena_dev);
3567394324d3SSascha Wildner ena_com_admin_destroy(ena_dev);
3568394324d3SSascha Wildner ena_com_mmio_reg_read_request_destroy(ena_dev);
3569394324d3SSascha Wildner
3570394324d3SSascha Wildner adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3571394324d3SSascha Wildner adapter->trigger_reset = false;
3572394324d3SSascha Wildner
3573394324d3SSascha Wildner /* Finished destroy part. Restart the device */
3574394324d3SSascha Wildner rc = ena_device_init(adapter, adapter->pdev, &get_feat_ctx,
3575394324d3SSascha Wildner &adapter->wd_active);
3576394324d3SSascha Wildner if (unlikely(rc != 0)) {
3577394324d3SSascha Wildner device_printf(adapter->pdev,
3578394324d3SSascha Wildner "ENA device init failed! (err: %d)\n", rc);
3579394324d3SSascha Wildner goto err_dev_free;
3580394324d3SSascha Wildner }
3581394324d3SSascha Wildner
3582394324d3SSascha Wildner rc = ena_enable_msix_and_set_admin_interrupts(adapter,
3583394324d3SSascha Wildner adapter->num_queues);
3584394324d3SSascha Wildner if (unlikely(rc != 0)) {
3585394324d3SSascha Wildner device_printf(adapter->pdev, "Enable MSI-X failed\n");
3586394324d3SSascha Wildner goto err_com_free;
3587394324d3SSascha Wildner }
3588394324d3SSascha Wildner
3589394324d3SSascha Wildner /* If the interface was up before the reset bring it up */
3590394324d3SSascha Wildner if (dev_up) {
3591394324d3SSascha Wildner rc = ena_up(adapter);
3592394324d3SSascha Wildner if (unlikely(rc != 0)) {
3593394324d3SSascha Wildner device_printf(adapter->pdev,
3594394324d3SSascha Wildner "Failed to create I/O queues\n");
3595394324d3SSascha Wildner goto err_msix_free;
3596394324d3SSascha Wildner }
3597394324d3SSascha Wildner }
3598394324d3SSascha Wildner
359982a3fa28SBrad Hoffman callout_reset(&adapter->timer_service, hz,
360082a3fa28SBrad Hoffman ena_timer_service, (void *)adapter);
3601394324d3SSascha Wildner
360282a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
3603394324d3SSascha Wildner
3604394324d3SSascha Wildner return;
3605394324d3SSascha Wildner
3606394324d3SSascha Wildner err_msix_free:
3607394324d3SSascha Wildner ena_free_mgmnt_irq(adapter);
3608394324d3SSascha Wildner ena_disable_msix(adapter);
3609394324d3SSascha Wildner err_com_free:
3610394324d3SSascha Wildner ena_com_admin_destroy(ena_dev);
3611394324d3SSascha Wildner err_dev_free:
3612394324d3SSascha Wildner device_printf(adapter->pdev, "ENA reset failed!\n");
3613394324d3SSascha Wildner adapter->running = false;
361482a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
3615394324d3SSascha Wildner }
3616394324d3SSascha Wildner
3617394324d3SSascha Wildner /**
3618394324d3SSascha Wildner * ena_attach - Device Initialization Routine
3619394324d3SSascha Wildner * @pdev: device information struct
3620394324d3SSascha Wildner *
3621394324d3SSascha Wildner * Returns 0 on success, otherwise on failure.
3622394324d3SSascha Wildner *
3623394324d3SSascha Wildner * ena_attach initializes an adapter identified by a device structure.
3624394324d3SSascha Wildner * The OS initialization, configuring of the adapter private structure,
3625394324d3SSascha Wildner * and a hardware reset occur.
3626394324d3SSascha Wildner **/
3627394324d3SSascha Wildner static int
ena_attach(device_t pdev)3628394324d3SSascha Wildner ena_attach(device_t pdev)
3629394324d3SSascha Wildner {
3630394324d3SSascha Wildner struct ena_com_dev_get_features_ctx get_feat_ctx;
3631394324d3SSascha Wildner static int version_printed;
3632394324d3SSascha Wildner struct ena_adapter *adapter;
3633394324d3SSascha Wildner struct ena_com_dev *ena_dev = NULL;
3634394324d3SSascha Wildner uint16_t tx_sgl_size = 0;
3635394324d3SSascha Wildner uint16_t rx_sgl_size = 0;
3636394324d3SSascha Wildner int io_queue_num;
3637394324d3SSascha Wildner int queue_size;
3638394324d3SSascha Wildner int rc;
3639394324d3SSascha Wildner adapter = device_get_softc(pdev);
3640394324d3SSascha Wildner adapter->pdev = pdev;
3641394324d3SSascha Wildner
364282a3fa28SBrad Hoffman lockinit(&adapter->global_lock, "ENA global mtx", 0, LK_CANRECURSE);
364382a3fa28SBrad Hoffman lockinit(&adapter->ioctl_lock, "ENA ioctl sx", 0, LK_CANRECURSE);
3644394324d3SSascha Wildner
3645394324d3SSascha Wildner /* Set up the timer service */
364682a3fa28SBrad Hoffman callout_init_lk(&adapter->timer_service, &adapter->global_lock);
3647394324d3SSascha Wildner adapter->keep_alive_timeout = DEFAULT_KEEP_ALIVE_TO;
3648394324d3SSascha Wildner adapter->missing_tx_timeout = DEFAULT_TX_CMP_TO;
3649394324d3SSascha Wildner adapter->missing_tx_max_queues = DEFAULT_TX_MONITORED_QUEUES;
3650394324d3SSascha Wildner adapter->missing_tx_threshold = DEFAULT_TX_CMP_THRESHOLD;
3651394324d3SSascha Wildner
3652394324d3SSascha Wildner if (version_printed++ == 0)
3653394324d3SSascha Wildner device_printf(pdev, "%s\n", ena_version);
3654394324d3SSascha Wildner
3655394324d3SSascha Wildner rc = ena_allocate_pci_resources(adapter);
3656394324d3SSascha Wildner if (unlikely(rc != 0)) {
3657394324d3SSascha Wildner device_printf(pdev, "PCI resource allocation failed!\n");
3658394324d3SSascha Wildner ena_free_pci_resources(adapter);
3659394324d3SSascha Wildner return (rc);
3660394324d3SSascha Wildner }
3661394324d3SSascha Wildner
3662394324d3SSascha Wildner /* Allocate memory for ena_dev structure */
366382a3fa28SBrad Hoffman ena_dev = kmalloc(sizeof(struct ena_com_dev), M_DEVBUF,
3664394324d3SSascha Wildner M_WAITOK | M_ZERO);
3665394324d3SSascha Wildner
3666394324d3SSascha Wildner adapter->ena_dev = ena_dev;
3667394324d3SSascha Wildner ena_dev->dmadev = pdev;
366882a3fa28SBrad Hoffman ena_dev->bus = kmalloc(sizeof(struct ena_bus), M_DEVBUF,
3669394324d3SSascha Wildner M_WAITOK | M_ZERO);
3670394324d3SSascha Wildner
3671394324d3SSascha Wildner /* Store register resources */
3672394324d3SSascha Wildner ((struct ena_bus*)(ena_dev->bus))->reg_bar_t =
3673394324d3SSascha Wildner rman_get_bustag(adapter->registers);
3674394324d3SSascha Wildner ((struct ena_bus*)(ena_dev->bus))->reg_bar_h =
3675394324d3SSascha Wildner rman_get_bushandle(adapter->registers);
3676394324d3SSascha Wildner
3677394324d3SSascha Wildner if (unlikely(((struct ena_bus*)(ena_dev->bus))->reg_bar_h == 0)) {
3678394324d3SSascha Wildner device_printf(pdev, "failed to pmap registers bar\n");
3679394324d3SSascha Wildner rc = ENXIO;
3680394324d3SSascha Wildner goto err_bus_free;
3681394324d3SSascha Wildner }
3682394324d3SSascha Wildner
3683394324d3SSascha Wildner ena_dev->tx_mem_queue_type = ENA_ADMIN_PLACEMENT_POLICY_HOST;
3684394324d3SSascha Wildner
3685394324d3SSascha Wildner /* Device initialization */
3686394324d3SSascha Wildner rc = ena_device_init(adapter, pdev, &get_feat_ctx, &adapter->wd_active);
3687394324d3SSascha Wildner if (unlikely(rc != 0)) {
3688394324d3SSascha Wildner device_printf(pdev, "ENA device init failed! (err: %d)\n", rc);
3689394324d3SSascha Wildner rc = ENXIO;
3690394324d3SSascha Wildner goto err_bus_free;
3691394324d3SSascha Wildner }
3692394324d3SSascha Wildner
369382a3fa28SBrad Hoffman getmicrouptime(&adapter->keep_alive_timestamp);
3694394324d3SSascha Wildner
3695394324d3SSascha Wildner adapter->tx_offload_cap = get_feat_ctx.offload.tx;
3696394324d3SSascha Wildner
3697394324d3SSascha Wildner /* Set for sure that interface is not up */
3698394324d3SSascha Wildner adapter->up = false;
3699394324d3SSascha Wildner
3700394324d3SSascha Wildner memcpy(adapter->mac_addr, get_feat_ctx.dev_attr.mac_addr,
3701394324d3SSascha Wildner ETHER_ADDR_LEN);
3702394324d3SSascha Wildner
3703394324d3SSascha Wildner /* calculate IO queue number to create */
3704394324d3SSascha Wildner io_queue_num = ena_calc_io_queue_num(adapter, &get_feat_ctx);
3705394324d3SSascha Wildner
3706394324d3SSascha Wildner ENA_ASSERT(io_queue_num > 0, "Invalid queue number: %d\n",
3707394324d3SSascha Wildner io_queue_num);
3708394324d3SSascha Wildner adapter->num_queues = io_queue_num;
3709394324d3SSascha Wildner
3710394324d3SSascha Wildner adapter->max_mtu = get_feat_ctx.dev_attr.max_mtu;
3711394324d3SSascha Wildner
3712394324d3SSascha Wildner /* calculatre ring sizes */
3713394324d3SSascha Wildner queue_size = ena_calc_queue_size(adapter,&tx_sgl_size,
3714394324d3SSascha Wildner &rx_sgl_size, &get_feat_ctx);
3715394324d3SSascha Wildner if (unlikely((queue_size <= 0) || (io_queue_num <= 0))) {
3716394324d3SSascha Wildner rc = ENA_COM_FAULT;
3717394324d3SSascha Wildner goto err_com_free;
3718394324d3SSascha Wildner }
3719394324d3SSascha Wildner
3720394324d3SSascha Wildner adapter->reset_reason = ENA_REGS_RESET_NORMAL;
3721394324d3SSascha Wildner
3722394324d3SSascha Wildner adapter->tx_ring_size = queue_size;
3723394324d3SSascha Wildner adapter->rx_ring_size = queue_size;
3724394324d3SSascha Wildner
3725394324d3SSascha Wildner adapter->max_tx_sgl_size = tx_sgl_size;
3726394324d3SSascha Wildner adapter->max_rx_sgl_size = rx_sgl_size;
3727394324d3SSascha Wildner
3728394324d3SSascha Wildner /* set up dma tags for rx and tx buffers */
3729394324d3SSascha Wildner rc = ena_setup_tx_dma_tag(adapter);
3730394324d3SSascha Wildner if (unlikely(rc != 0)) {
3731394324d3SSascha Wildner device_printf(pdev, "Failed to create TX DMA tag\n");
3732394324d3SSascha Wildner goto err_com_free;
3733394324d3SSascha Wildner }
3734394324d3SSascha Wildner
3735394324d3SSascha Wildner rc = ena_setup_rx_dma_tag(adapter);
3736394324d3SSascha Wildner if (unlikely(rc != 0)) {
3737394324d3SSascha Wildner device_printf(pdev, "Failed to create RX DMA tag\n");
3738394324d3SSascha Wildner goto err_tx_tag_free;
3739394324d3SSascha Wildner }
3740394324d3SSascha Wildner
3741394324d3SSascha Wildner /* initialize rings basic information */
3742*1dac82a7SSascha Wildner device_printf(pdev, "initialize %d io queues\n", io_queue_num);
3743394324d3SSascha Wildner ena_init_io_rings(adapter);
3744394324d3SSascha Wildner
3745394324d3SSascha Wildner /* setup network interface */
3746394324d3SSascha Wildner rc = ena_setup_ifnet(pdev, adapter, &get_feat_ctx);
3747394324d3SSascha Wildner if (unlikely(rc != 0)) {
3748394324d3SSascha Wildner device_printf(pdev, "Error with network interface setup\n");
3749394324d3SSascha Wildner goto err_io_free;
3750394324d3SSascha Wildner }
3751394324d3SSascha Wildner
3752394324d3SSascha Wildner rc = ena_enable_msix_and_set_admin_interrupts(adapter, io_queue_num);
3753394324d3SSascha Wildner if (unlikely(rc != 0)) {
3754394324d3SSascha Wildner device_printf(pdev,
3755394324d3SSascha Wildner "Failed to enable and set the admin interrupts\n");
3756394324d3SSascha Wildner goto err_ifp_free;
3757394324d3SSascha Wildner }
3758394324d3SSascha Wildner
3759394324d3SSascha Wildner /* Initialize reset task queue */
3760394324d3SSascha Wildner TASK_INIT(&adapter->reset_task, 0, ena_reset_task, adapter);
3761394324d3SSascha Wildner adapter->reset_tq = taskqueue_create("ena_reset_enqueue",
3762394324d3SSascha Wildner M_WAITOK | M_ZERO, taskqueue_thread_enqueue, &adapter->reset_tq);
376382a3fa28SBrad Hoffman taskqueue_start_threads(&adapter->reset_tq, 1, TDPRI_KERN_DAEMON, -1,
3764394324d3SSascha Wildner "%s rstq", device_get_nameunit(adapter->pdev));
3765394324d3SSascha Wildner
3766394324d3SSascha Wildner /* Initialize statistics */
376782a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
3768394324d3SSascha Wildner ena_alloc_counters((counter_u64_t *)&adapter->dev_stats,
3769394324d3SSascha Wildner sizeof(struct ena_stats_dev));
3770394324d3SSascha Wildner ena_alloc_counters((counter_u64_t *)&adapter->hw_stats,
3771394324d3SSascha Wildner sizeof(struct ena_hw_stats));
377282a3fa28SBrad Hoffman #endif
3773394324d3SSascha Wildner ena_sysctl_add_nodes(adapter);
3774394324d3SSascha Wildner
3775394324d3SSascha Wildner /* Tell the stack that the interface is not active */
377682a3fa28SBrad Hoffman ifq_set_oactive(&adapter->ifp->if_snd);
377782a3fa28SBrad Hoffman adapter->ifp->if_flags &= ~IFF_RUNNING;
3778394324d3SSascha Wildner
3779394324d3SSascha Wildner adapter->running = true;
3780394324d3SSascha Wildner return (0);
3781394324d3SSascha Wildner
3782394324d3SSascha Wildner err_ifp_free:
3783394324d3SSascha Wildner if_detach(adapter->ifp);
3784394324d3SSascha Wildner if_free(adapter->ifp);
3785394324d3SSascha Wildner err_io_free:
3786394324d3SSascha Wildner ena_free_all_io_rings_resources(adapter);
3787394324d3SSascha Wildner ena_free_rx_dma_tag(adapter);
3788394324d3SSascha Wildner err_tx_tag_free:
3789394324d3SSascha Wildner ena_free_tx_dma_tag(adapter);
3790394324d3SSascha Wildner err_com_free:
3791394324d3SSascha Wildner ena_com_admin_destroy(ena_dev);
3792394324d3SSascha Wildner ena_com_delete_host_info(ena_dev);
3793394324d3SSascha Wildner ena_com_mmio_reg_read_request_destroy(ena_dev);
3794394324d3SSascha Wildner err_bus_free:
379582a3fa28SBrad Hoffman kfree(ena_dev->bus, M_DEVBUF);
379682a3fa28SBrad Hoffman kfree(ena_dev, M_DEVBUF);
3797394324d3SSascha Wildner ena_free_pci_resources(adapter);
3798394324d3SSascha Wildner
3799394324d3SSascha Wildner return (rc);
3800394324d3SSascha Wildner }
3801394324d3SSascha Wildner
3802394324d3SSascha Wildner /**
3803394324d3SSascha Wildner * ena_detach - Device Removal Routine
3804394324d3SSascha Wildner * @pdev: device information struct
3805394324d3SSascha Wildner *
3806394324d3SSascha Wildner * ena_detach is called by the device subsystem to alert the driver
3807394324d3SSascha Wildner * that it should release a PCI device.
3808394324d3SSascha Wildner **/
3809394324d3SSascha Wildner static int
ena_detach(device_t pdev)3810394324d3SSascha Wildner ena_detach(device_t pdev)
3811394324d3SSascha Wildner {
3812394324d3SSascha Wildner struct ena_adapter *adapter = device_get_softc(pdev);
3813394324d3SSascha Wildner struct ena_com_dev *ena_dev = adapter->ena_dev;
3814394324d3SSascha Wildner int rc;
3815394324d3SSascha Wildner
3816394324d3SSascha Wildner /* Make sure VLANS are not using driver */
381782a3fa28SBrad Hoffman if (adapter->ifp->if_vlantrunks != NULL) {
3818394324d3SSascha Wildner device_printf(adapter->pdev ,"VLAN is in use, detach first\n");
3819394324d3SSascha Wildner return (EBUSY);
3820394324d3SSascha Wildner }
3821394324d3SSascha Wildner
3822394324d3SSascha Wildner /* Free reset task and callout */
3823394324d3SSascha Wildner callout_drain(&adapter->timer_service);
3824394324d3SSascha Wildner while (taskqueue_cancel(adapter->reset_tq, &adapter->reset_task, NULL))
3825394324d3SSascha Wildner taskqueue_drain(adapter->reset_tq, &adapter->reset_task);
3826394324d3SSascha Wildner taskqueue_free(adapter->reset_tq);
3827394324d3SSascha Wildner
382882a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_EXCLUSIVE);
3829394324d3SSascha Wildner ena_down(adapter);
383082a3fa28SBrad Hoffman lockmgr(&adapter->ioctl_lock, LK_RELEASE);
3831394324d3SSascha Wildner
3832394324d3SSascha Wildner if (adapter->ifp != NULL) {
3833394324d3SSascha Wildner ether_ifdetach(adapter->ifp);
3834394324d3SSascha Wildner if_free(adapter->ifp);
3835394324d3SSascha Wildner }
3836394324d3SSascha Wildner
3837394324d3SSascha Wildner ena_free_all_io_rings_resources(adapter);
3838394324d3SSascha Wildner
383982a3fa28SBrad Hoffman #if 0 /* XXX swildner counters */
3840394324d3SSascha Wildner ena_free_counters((counter_u64_t *)&adapter->hw_stats,
3841394324d3SSascha Wildner sizeof(struct ena_hw_stats));
3842394324d3SSascha Wildner ena_free_counters((counter_u64_t *)&adapter->dev_stats,
3843394324d3SSascha Wildner sizeof(struct ena_stats_dev));
384482a3fa28SBrad Hoffman #endif
3845394324d3SSascha Wildner
3846394324d3SSascha Wildner if (likely(adapter->rss_support))
3847394324d3SSascha Wildner ena_com_rss_destroy(ena_dev);
3848394324d3SSascha Wildner
3849394324d3SSascha Wildner rc = ena_free_rx_dma_tag(adapter);
3850394324d3SSascha Wildner if (unlikely(rc != 0))
3851394324d3SSascha Wildner device_printf(adapter->pdev,
3852394324d3SSascha Wildner "Unmapped RX DMA tag associations\n");
3853394324d3SSascha Wildner
3854394324d3SSascha Wildner rc = ena_free_tx_dma_tag(adapter);
3855394324d3SSascha Wildner if (unlikely(rc != 0))
3856394324d3SSascha Wildner device_printf(adapter->pdev,
3857394324d3SSascha Wildner "Unmapped TX DMA tag associations\n");
3858394324d3SSascha Wildner
3859394324d3SSascha Wildner /* Reset the device only if the device is running. */
3860394324d3SSascha Wildner if (adapter->running)
3861394324d3SSascha Wildner ena_com_dev_reset(ena_dev, adapter->reset_reason);
3862394324d3SSascha Wildner
3863394324d3SSascha Wildner ena_com_delete_host_info(ena_dev);
3864394324d3SSascha Wildner
3865394324d3SSascha Wildner ena_free_irqs(adapter);
3866394324d3SSascha Wildner
3867394324d3SSascha Wildner ena_com_abort_admin_commands(ena_dev);
3868394324d3SSascha Wildner
3869394324d3SSascha Wildner ena_com_wait_for_abort_completion(ena_dev);
3870394324d3SSascha Wildner
3871394324d3SSascha Wildner ena_com_admin_destroy(ena_dev);
3872394324d3SSascha Wildner
3873394324d3SSascha Wildner ena_com_mmio_reg_read_request_destroy(ena_dev);
3874394324d3SSascha Wildner
3875394324d3SSascha Wildner ena_free_pci_resources(adapter);
3876394324d3SSascha Wildner
387782a3fa28SBrad Hoffman lockuninit(&adapter->global_lock);
387882a3fa28SBrad Hoffman lockuninit(&adapter->ioctl_lock);
3879394324d3SSascha Wildner
3880394324d3SSascha Wildner if (ena_dev->bus != NULL)
388182a3fa28SBrad Hoffman kfree(ena_dev->bus, M_DEVBUF);
3882394324d3SSascha Wildner
3883394324d3SSascha Wildner if (ena_dev != NULL)
388482a3fa28SBrad Hoffman kfree(ena_dev, M_DEVBUF);
3885394324d3SSascha Wildner
3886394324d3SSascha Wildner return (bus_generic_detach(pdev));
3887394324d3SSascha Wildner }
3888394324d3SSascha Wildner
3889394324d3SSascha Wildner /******************************************************************************
3890394324d3SSascha Wildner ******************************** AENQ Handlers *******************************
3891394324d3SSascha Wildner *****************************************************************************/
3892394324d3SSascha Wildner /**
3893394324d3SSascha Wildner * ena_update_on_link_change:
3894394324d3SSascha Wildner * Notify the network interface about the change in link status
3895394324d3SSascha Wildner **/
3896394324d3SSascha Wildner static void
ena_update_on_link_change(void * adapter_data,struct ena_admin_aenq_entry * aenq_e)3897394324d3SSascha Wildner ena_update_on_link_change(void *adapter_data,
3898394324d3SSascha Wildner struct ena_admin_aenq_entry *aenq_e)
3899394324d3SSascha Wildner {
3900394324d3SSascha Wildner struct ena_adapter *adapter = (struct ena_adapter *)adapter_data;
3901394324d3SSascha Wildner struct ena_admin_aenq_link_change_desc *aenq_desc;
3902394324d3SSascha Wildner int status;
3903394324d3SSascha Wildner if_t ifp;
3904394324d3SSascha Wildner
3905394324d3SSascha Wildner aenq_desc = (struct ena_admin_aenq_link_change_desc *)aenq_e;
3906394324d3SSascha Wildner ifp = adapter->ifp;
3907394324d3SSascha Wildner status = aenq_desc->flags &
3908394324d3SSascha Wildner ENA_ADMIN_AENQ_LINK_CHANGE_DESC_LINK_STATUS_MASK;
3909394324d3SSascha Wildner
3910394324d3SSascha Wildner if (status != 0) {
3911394324d3SSascha Wildner device_printf(adapter->pdev, "link is UP\n");
391282a3fa28SBrad Hoffman ifp->if_link_state = LINK_STATE_UP;
391382a3fa28SBrad Hoffman if_link_state_change(ifp);
3914394324d3SSascha Wildner } else if (status == 0) {
3915394324d3SSascha Wildner device_printf(adapter->pdev, "link is DOWN\n");
391682a3fa28SBrad Hoffman ifp->if_link_state = LINK_STATE_DOWN;
391782a3fa28SBrad Hoffman if_link_state_change(ifp);
3918394324d3SSascha Wildner } else {
3919394324d3SSascha Wildner device_printf(adapter->pdev, "invalid value recvd\n");
3920394324d3SSascha Wildner BUG();
3921394324d3SSascha Wildner }
3922394324d3SSascha Wildner
3923394324d3SSascha Wildner adapter->link_status = status;
3924394324d3SSascha Wildner }
3925394324d3SSascha Wildner
3926394324d3SSascha Wildner /**
3927394324d3SSascha Wildner * This handler will called for unknown event group or unimplemented handlers
3928394324d3SSascha Wildner **/
3929394324d3SSascha Wildner static void
unimplemented_aenq_handler(void * data,struct ena_admin_aenq_entry * aenq_e)3930394324d3SSascha Wildner unimplemented_aenq_handler(void *data,
3931394324d3SSascha Wildner struct ena_admin_aenq_entry *aenq_e)
3932394324d3SSascha Wildner {
3933394324d3SSascha Wildner return;
3934394324d3SSascha Wildner }
3935394324d3SSascha Wildner
3936394324d3SSascha Wildner static struct ena_aenq_handlers aenq_handlers = {
3937394324d3SSascha Wildner .handlers = {
3938394324d3SSascha Wildner [ENA_ADMIN_LINK_CHANGE] = ena_update_on_link_change,
3939394324d3SSascha Wildner [ENA_ADMIN_KEEP_ALIVE] = ena_keep_alive_wd,
3940394324d3SSascha Wildner },
3941394324d3SSascha Wildner .unimplemented_handler = unimplemented_aenq_handler
3942394324d3SSascha Wildner };
3943394324d3SSascha Wildner
3944394324d3SSascha Wildner /*********************************************************************
3945394324d3SSascha Wildner * FreeBSD Device Interface Entry Points
3946394324d3SSascha Wildner *********************************************************************/
3947394324d3SSascha Wildner
3948394324d3SSascha Wildner static device_method_t ena_methods[] = {
3949394324d3SSascha Wildner /* Device interface */
3950394324d3SSascha Wildner DEVMETHOD(device_probe, ena_probe),
3951394324d3SSascha Wildner DEVMETHOD(device_attach, ena_attach),
3952394324d3SSascha Wildner DEVMETHOD(device_detach, ena_detach),
3953394324d3SSascha Wildner DEVMETHOD_END
3954394324d3SSascha Wildner };
3955394324d3SSascha Wildner
3956394324d3SSascha Wildner static driver_t ena_driver = {
3957394324d3SSascha Wildner "ena", ena_methods, sizeof(struct ena_adapter),
3958394324d3SSascha Wildner };
3959394324d3SSascha Wildner
3960394324d3SSascha Wildner devclass_t ena_devclass;
396117975de1SSascha Wildner DRIVER_MODULE(ena, pci, ena_driver, ena_devclass, NULL, NULL);
3962394324d3SSascha Wildner MODULE_DEPEND(ena, pci, 1, 1, 1);
3963394324d3SSascha Wildner MODULE_DEPEND(ena, ether, 1, 1, 1);
3964394324d3SSascha Wildner
3965394324d3SSascha Wildner /*********************************************************************/
3966