1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2015-2024 Amazon.com, Inc. or its affiliates.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 *
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 *
14 * 2. Redistributions in binary form must reproduce the above copyright
15 * notice, this list of conditions and the following disclaimer in the
16 * documentation and/or other materials provided with the distribution.
17 *
18 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 */
30 #include <sys/cdefs.h>
31 #include "opt_rss.h"
32 #include "ena.h"
33 #include "ena_datapath.h"
34 #ifdef DEV_NETMAP
35 #include "ena_netmap.h"
36 #endif /* DEV_NETMAP */
37 #ifdef RSS
38 #include <net/rss_config.h>
39 #endif /* RSS */
40
41 #include <netinet6/ip6_var.h>
42
43 /*********************************************************************
44 * Static functions prototypes
45 *********************************************************************/
46
47 static int ena_tx_cleanup(struct ena_ring *);
48 static int ena_rx_cleanup(struct ena_ring *);
49 static inline int ena_get_tx_req_id(struct ena_ring *tx_ring,
50 struct ena_com_io_cq *io_cq, uint16_t *req_id);
51 static void ena_rx_hash_mbuf(struct ena_ring *, struct ena_com_rx_ctx *,
52 struct mbuf *);
53 static struct mbuf *ena_rx_mbuf(struct ena_ring *, struct ena_com_rx_buf_info *,
54 struct ena_com_rx_ctx *, uint16_t *);
55 static inline void ena_rx_checksum(struct ena_ring *, struct ena_com_rx_ctx *,
56 struct mbuf *);
57 static void ena_tx_csum(struct ena_com_tx_ctx *, struct mbuf *, bool);
58 static int ena_check_and_collapse_mbuf(struct ena_ring *tx_ring,
59 struct mbuf **mbuf);
60 static int ena_xmit_mbuf(struct ena_ring *, struct mbuf **);
61 static void ena_start_xmit(struct ena_ring *);
62
63 /*********************************************************************
64 * Global functions
65 *********************************************************************/
66
67 void
ena_cleanup(void * arg,int pending)68 ena_cleanup(void *arg, int pending)
69 {
70 struct ena_que *que = arg;
71 struct ena_adapter *adapter = que->adapter;
72 if_t ifp = adapter->ifp;
73 struct ena_ring *tx_ring;
74 struct ena_ring *rx_ring;
75 struct ena_com_io_cq *io_cq;
76 struct ena_eth_io_intr_reg intr_reg;
77 int qid, ena_qid;
78 int txc, rxc, i;
79
80 tx_ring = que->tx_ring;
81 rx_ring = que->rx_ring;
82 qid = que->id;
83 ena_qid = ENA_IO_TXQ_IDX(qid);
84 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
85
86 atomic_store_8(&tx_ring->cleanup_running, 1);
87 /* Need to make sure that ENA_FLAG_TRIGGER_RESET is visible to ena_cleanup() and
88 * that cleanup_running is visible to check_missing_comp_in_tx_queue() to
89 * prevent the case of accessing CQ concurrently with check_cdesc_in_tx_cq()
90 */
91 mb();
92 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
93 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))))
94 return;
95
96 ena_log_io(adapter->pdev, DBG, "MSI-X TX/RX routine\n");
97
98 atomic_store_8(&tx_ring->first_interrupt, 1);
99 atomic_store_8(&rx_ring->first_interrupt, 1);
100
101 for (i = 0; i < ENA_CLEAN_BUDGET; ++i) {
102 rxc = ena_rx_cleanup(rx_ring);
103 txc = ena_tx_cleanup(tx_ring);
104
105 if (unlikely(((if_getdrvflags(ifp) & IFF_DRV_RUNNING) == 0) ||
106 (ENA_FLAG_ISSET(ENA_FLAG_TRIGGER_RESET, adapter))))
107 return;
108
109 if ((txc != ENA_TX_BUDGET) && (rxc != ENA_RX_BUDGET))
110 break;
111 }
112
113 /* Signal that work is done and unmask interrupt */
114 ena_com_update_intr_reg(&intr_reg, ENA_RX_IRQ_INTERVAL,
115 ENA_TX_IRQ_INTERVAL, true, false);
116 counter_u64_add(tx_ring->tx_stats.unmask_interrupt_num, 1);
117 ena_com_unmask_intr(io_cq, &intr_reg);
118 atomic_store_8(&tx_ring->cleanup_running, 0);
119 }
120
121 void
ena_deferred_mq_start(void * arg,int pending)122 ena_deferred_mq_start(void *arg, int pending)
123 {
124 struct ena_ring *tx_ring = (struct ena_ring *)arg;
125 if_t ifp = tx_ring->adapter->ifp;
126
127 while (!drbr_empty(ifp, tx_ring->br) && tx_ring->running &&
128 (if_getdrvflags(ifp) & IFF_DRV_RUNNING) != 0) {
129 ENA_RING_MTX_LOCK(tx_ring);
130 ena_start_xmit(tx_ring);
131 ENA_RING_MTX_UNLOCK(tx_ring);
132 }
133 }
134
135 int
ena_mq_start(if_t ifp,struct mbuf * m)136 ena_mq_start(if_t ifp, struct mbuf *m)
137 {
138 struct ena_adapter *adapter = if_getsoftc(ifp);
139 struct ena_ring *tx_ring;
140 int ret, is_drbr_empty;
141 uint32_t i;
142 #ifdef RSS
143 uint32_t bucket_id;
144 #endif
145
146 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
147 return (ENODEV);
148
149 /* Which queue to use */
150 /*
151 * If everything is setup correctly, it should be the
152 * same bucket that the current CPU we're on is.
153 * It should improve performance.
154 */
155 if (M_HASHTYPE_GET(m) != M_HASHTYPE_NONE) {
156 #ifdef RSS
157 if (rss_hash2bucket(m->m_pkthdr.flowid, M_HASHTYPE_GET(m),
158 &bucket_id) == 0)
159 i = bucket_id % adapter->num_io_queues;
160 else
161 #endif
162 i = m->m_pkthdr.flowid % adapter->num_io_queues;
163 } else {
164 i = curcpu % adapter->num_io_queues;
165 }
166 tx_ring = &adapter->tx_ring[i];
167
168 /* Check if drbr is empty before putting packet */
169 is_drbr_empty = drbr_empty(ifp, tx_ring->br);
170 ret = drbr_enqueue(ifp, tx_ring->br, m);
171 if (unlikely(ret != 0)) {
172 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
173 return (ret);
174 }
175
176 if (is_drbr_empty && (ENA_RING_MTX_TRYLOCK(tx_ring) != 0)) {
177 ena_start_xmit(tx_ring);
178 ENA_RING_MTX_UNLOCK(tx_ring);
179 } else {
180 taskqueue_enqueue(tx_ring->enqueue_tq, &tx_ring->enqueue_task);
181 }
182
183 return (0);
184 }
185
186 void
ena_qflush(if_t ifp)187 ena_qflush(if_t ifp)
188 {
189 struct ena_adapter *adapter = if_getsoftc(ifp);
190 struct ena_ring *tx_ring = adapter->tx_ring;
191 int i;
192
193 for (i = 0; i < adapter->num_io_queues; ++i, ++tx_ring)
194 if (!drbr_empty(ifp, tx_ring->br)) {
195 ENA_RING_MTX_LOCK(tx_ring);
196 drbr_flush(ifp, tx_ring->br);
197 ENA_RING_MTX_UNLOCK(tx_ring);
198 }
199
200 if_qflush(ifp);
201 }
202
203 /*********************************************************************
204 * Static functions
205 *********************************************************************/
206
207 static inline int
ena_get_tx_req_id(struct ena_ring * tx_ring,struct ena_com_io_cq * io_cq,uint16_t * req_id)208 ena_get_tx_req_id(struct ena_ring *tx_ring, struct ena_com_io_cq *io_cq,
209 uint16_t *req_id)
210 {
211 struct ena_adapter *adapter = tx_ring->adapter;
212 int rc = ena_com_tx_comp_req_id_get(io_cq, req_id);
213
214 if (unlikely(rc == ENA_COM_TRY_AGAIN))
215 return (EAGAIN);
216
217 rc = validate_tx_req_id(tx_ring, *req_id, rc);
218
219 if (unlikely(tx_ring->tx_buffer_info[*req_id].mbuf == NULL)) {
220 ena_log(adapter->pdev, ERR,
221 "tx_info doesn't have valid mbuf. req_id %hu qid %hu\n",
222 *req_id, tx_ring->qid);
223 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_TX_REQ_ID);
224 rc = EFAULT;
225 }
226
227 return (rc);
228 }
229
230 /**
231 * ena_tx_cleanup - clear sent packets and corresponding descriptors
232 * @tx_ring: ring for which we want to clean packets
233 *
234 * Once packets are sent, we ask the device in a loop for no longer used
235 * descriptors. We find the related mbuf chain in a map (index in an array)
236 * and free it, then update ring state.
237 * This is performed in "endless" loop, updating ring pointers every
238 * TX_COMMIT. The first check of free descriptor is performed before the actual
239 * loop, then repeated at the loop end.
240 **/
241 static int
ena_tx_cleanup(struct ena_ring * tx_ring)242 ena_tx_cleanup(struct ena_ring *tx_ring)
243 {
244 struct ena_adapter *adapter;
245 struct ena_com_io_cq *io_cq;
246 uint16_t next_to_clean;
247 uint16_t req_id;
248 uint16_t ena_qid;
249 unsigned int total_done = 0;
250 int rc;
251 int commit = ENA_TX_COMMIT;
252 int budget = ENA_TX_BUDGET;
253 int work_done;
254 bool above_thresh;
255
256 adapter = tx_ring->que->adapter;
257 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
258 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
259 next_to_clean = tx_ring->next_to_clean;
260
261 #ifdef DEV_NETMAP
262 if (netmap_tx_irq(adapter->ifp, tx_ring->qid) != NM_IRQ_PASS)
263 return (0);
264 #endif /* DEV_NETMAP */
265
266 do {
267 struct ena_tx_buffer *tx_info;
268 struct mbuf *mbuf;
269
270 rc = ena_get_tx_req_id(tx_ring, io_cq, &req_id);
271 if (unlikely(rc != 0))
272 break;
273
274 tx_info = &tx_ring->tx_buffer_info[req_id];
275
276 mbuf = tx_info->mbuf;
277
278 tx_info->mbuf = NULL;
279 bintime_clear(&tx_info->timestamp);
280
281 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
282 BUS_DMASYNC_POSTWRITE);
283 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
284
285 ena_log_io(adapter->pdev, DBG, "tx: q %d mbuf %p completed\n",
286 tx_ring->qid, mbuf);
287
288 m_freem(mbuf);
289
290 total_done += tx_info->tx_descs;
291
292 tx_ring->free_tx_ids[next_to_clean] = req_id;
293 next_to_clean = ENA_TX_RING_IDX_NEXT(next_to_clean,
294 tx_ring->ring_size);
295
296 if (unlikely(--commit == 0)) {
297 commit = ENA_TX_COMMIT;
298 /* update ring state every ENA_TX_COMMIT descriptor */
299 tx_ring->next_to_clean = next_to_clean;
300 ena_com_comp_ack(
301 &adapter->ena_dev->io_sq_queues[ena_qid],
302 total_done);
303 total_done = 0;
304 }
305 } while (likely(--budget));
306
307 work_done = ENA_TX_BUDGET - budget;
308
309 ena_log_io(adapter->pdev, DBG, "tx: q %d done. total pkts: %d\n",
310 tx_ring->qid, work_done);
311
312 /* If there is still something to commit update ring state */
313 if (likely(commit != ENA_TX_COMMIT)) {
314 tx_ring->next_to_clean = next_to_clean;
315 ena_com_comp_ack(&adapter->ena_dev->io_sq_queues[ena_qid],
316 total_done);
317 }
318
319 /*
320 * Need to make the rings circular update visible to
321 * ena_xmit_mbuf() before checking for tx_ring->running.
322 */
323 mb();
324
325 above_thresh = ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
326 ENA_TX_RESUME_THRESH);
327 if (unlikely(!tx_ring->running && above_thresh)) {
328 ENA_RING_MTX_LOCK(tx_ring);
329 above_thresh = ena_com_sq_have_enough_space(
330 tx_ring->ena_com_io_sq, ENA_TX_RESUME_THRESH);
331 if (!tx_ring->running && above_thresh) {
332 tx_ring->running = true;
333 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
334 taskqueue_enqueue(tx_ring->enqueue_tq,
335 &tx_ring->enqueue_task);
336 }
337 ENA_RING_MTX_UNLOCK(tx_ring);
338 }
339
340 tx_ring->tx_last_cleanup_ticks = ticks;
341
342 return (work_done);
343 }
344
345 static void
ena_rx_hash_mbuf(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct mbuf * mbuf)346 ena_rx_hash_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
347 struct mbuf *mbuf)
348 {
349 struct ena_adapter *adapter = rx_ring->adapter;
350
351 if (likely(ENA_FLAG_ISSET(ENA_FLAG_RSS_ACTIVE, adapter))) {
352 mbuf->m_pkthdr.flowid = ena_rx_ctx->hash;
353
354 #ifdef RSS
355 /*
356 * Hardware and software RSS are in agreement only when both are
357 * configured to Toeplitz algorithm. This driver configures
358 * that algorithm only when software RSS is enabled and uses it.
359 */
360 if (adapter->ena_dev->rss.hash_func != ENA_ADMIN_TOEPLITZ &&
361 ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN) {
362 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
363 return;
364 }
365 #endif
366
367 if (ena_rx_ctx->frag &&
368 (ena_rx_ctx->l3_proto != ENA_ETH_IO_L3_PROTO_UNKNOWN)) {
369 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
370 return;
371 }
372
373 switch (ena_rx_ctx->l3_proto) {
374 case ENA_ETH_IO_L3_PROTO_IPV4:
375 switch (ena_rx_ctx->l4_proto) {
376 case ENA_ETH_IO_L4_PROTO_TCP:
377 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV4);
378 break;
379 case ENA_ETH_IO_L4_PROTO_UDP:
380 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV4);
381 break;
382 default:
383 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV4);
384 }
385 break;
386 case ENA_ETH_IO_L3_PROTO_IPV6:
387 switch (ena_rx_ctx->l4_proto) {
388 case ENA_ETH_IO_L4_PROTO_TCP:
389 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_TCP_IPV6);
390 break;
391 case ENA_ETH_IO_L4_PROTO_UDP:
392 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_UDP_IPV6);
393 break;
394 default:
395 M_HASHTYPE_SET(mbuf, M_HASHTYPE_RSS_IPV6);
396 }
397 break;
398 case ENA_ETH_IO_L3_PROTO_UNKNOWN:
399 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
400 break;
401 default:
402 M_HASHTYPE_SET(mbuf, M_HASHTYPE_OPAQUE_HASH);
403 }
404 } else {
405 mbuf->m_pkthdr.flowid = rx_ring->qid;
406 M_HASHTYPE_SET(mbuf, M_HASHTYPE_NONE);
407 }
408 }
409
410 /**
411 * ena_rx_mbuf - assemble mbuf from descriptors
412 * @rx_ring: ring for which we want to clean packets
413 * @ena_bufs: buffer info
414 * @ena_rx_ctx: metadata for this packet(s)
415 * @next_to_clean: ring pointer, will be updated only upon success
416 *
417 **/
418 static struct mbuf *
ena_rx_mbuf(struct ena_ring * rx_ring,struct ena_com_rx_buf_info * ena_bufs,struct ena_com_rx_ctx * ena_rx_ctx,uint16_t * next_to_clean)419 ena_rx_mbuf(struct ena_ring *rx_ring, struct ena_com_rx_buf_info *ena_bufs,
420 struct ena_com_rx_ctx *ena_rx_ctx, uint16_t *next_to_clean)
421 {
422 struct mbuf *mbuf;
423 struct ena_rx_buffer *rx_info;
424 struct ena_adapter *adapter;
425 device_t pdev;
426 unsigned int descs = ena_rx_ctx->descs;
427 uint16_t ntc, len, req_id, buf = 0;
428
429 ntc = *next_to_clean;
430 adapter = rx_ring->adapter;
431 pdev = adapter->pdev;
432
433 len = ena_bufs[buf].len;
434 req_id = ena_bufs[buf].req_id;
435 rx_info = &rx_ring->rx_buffer_info[req_id];
436 if (unlikely(rx_info->mbuf == NULL)) {
437 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n",
438 rx_ring->qid, req_id);
439 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
440 return (NULL);
441 }
442
443 ena_log_io(pdev, DBG, "rx_info %p, mbuf %p, paddr %jx\n", rx_info,
444 rx_info->mbuf, (uintmax_t)rx_info->ena_buf.paddr);
445
446 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
447 BUS_DMASYNC_POSTREAD);
448 mbuf = rx_info->mbuf;
449 mbuf->m_flags |= M_PKTHDR;
450 mbuf->m_pkthdr.len = len;
451 mbuf->m_len = len;
452 /* Only for the first segment the data starts at specific offset */
453 mbuf->m_data = mtodo(mbuf, ena_rx_ctx->pkt_offset);
454 ena_log_io(pdev, DBG, "Mbuf data offset=%u\n", ena_rx_ctx->pkt_offset);
455 mbuf->m_pkthdr.rcvif = rx_ring->que->adapter->ifp;
456
457 /* Fill mbuf with hash key and it's interpretation for optimization */
458 ena_rx_hash_mbuf(rx_ring, ena_rx_ctx, mbuf);
459
460 ena_log_io(pdev, DBG, "rx mbuf 0x%p, flags=0x%x, len: %d\n", mbuf,
461 mbuf->m_flags, mbuf->m_pkthdr.len);
462
463 /* DMA address is not needed anymore, unmap it */
464 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
465
466 rx_info->mbuf = NULL;
467 rx_ring->free_rx_ids[ntc] = req_id;
468 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
469
470 /*
471 * While we have more than 1 descriptors for one rcvd packet, append
472 * other mbufs to the main one
473 */
474 while (--descs) {
475 ++buf;
476 len = ena_bufs[buf].len;
477 req_id = ena_bufs[buf].req_id;
478 rx_info = &rx_ring->rx_buffer_info[req_id];
479
480 if (unlikely(rx_info->mbuf == NULL)) {
481 ena_log(pdev, ERR, "NULL mbuf in rx_info. qid %u req_id %u\n",
482 rx_ring->qid, req_id);
483 /*
484 * If one of the required mbufs was not allocated yet,
485 * we can break there.
486 * All earlier used descriptors will be reallocated
487 * later and not used mbufs can be reused.
488 * The next_to_clean pointer will not be updated in case
489 * of an error, so caller should advance it manually
490 * in error handling routine to keep it up to date
491 * with hw ring.
492 */
493 m_freem(mbuf);
494 ena_trigger_reset(adapter, ENA_REGS_RESET_INV_RX_REQ_ID);
495 return (NULL);
496 }
497
498 bus_dmamap_sync(adapter->rx_buf_tag, rx_info->map,
499 BUS_DMASYNC_POSTREAD);
500 if (unlikely(m_append(mbuf, len, rx_info->mbuf->m_data) == 0)) {
501 counter_u64_add(rx_ring->rx_stats.mbuf_alloc_fail, 1);
502 ena_log_io(pdev, WARN, "Failed to append Rx mbuf %p\n",
503 mbuf);
504 }
505
506 ena_log_io(pdev, DBG, "rx mbuf updated. len %d\n",
507 mbuf->m_pkthdr.len);
508
509 /* Free already appended mbuf, it won't be useful anymore */
510 bus_dmamap_unload(rx_ring->adapter->rx_buf_tag, rx_info->map);
511 m_freem(rx_info->mbuf);
512 rx_info->mbuf = NULL;
513
514 rx_ring->free_rx_ids[ntc] = req_id;
515 ntc = ENA_RX_RING_IDX_NEXT(ntc, rx_ring->ring_size);
516 }
517
518 *next_to_clean = ntc;
519
520 return (mbuf);
521 }
522
523 /**
524 * ena_rx_checksum - indicate in mbuf if hw indicated a good cksum
525 **/
526 static inline void
ena_rx_checksum(struct ena_ring * rx_ring,struct ena_com_rx_ctx * ena_rx_ctx,struct mbuf * mbuf)527 ena_rx_checksum(struct ena_ring *rx_ring, struct ena_com_rx_ctx *ena_rx_ctx,
528 struct mbuf *mbuf)
529 {
530 device_t pdev = rx_ring->adapter->pdev;
531
532 /* if IP and error */
533 if (unlikely((ena_rx_ctx->l3_proto == ENA_ETH_IO_L3_PROTO_IPV4) &&
534 ena_rx_ctx->l3_csum_err)) {
535 /* ipv4 checksum error */
536 mbuf->m_pkthdr.csum_flags = 0;
537 counter_u64_add(rx_ring->rx_stats.csum_bad, 1);
538 ena_log_io(pdev, DBG, "RX IPv4 header checksum error\n");
539 return;
540 }
541
542 /* if TCP/UDP */
543 if ((ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_TCP) ||
544 (ena_rx_ctx->l4_proto == ENA_ETH_IO_L4_PROTO_UDP)) {
545 if (ena_rx_ctx->l4_csum_err) {
546 /* TCP/UDP checksum error */
547 mbuf->m_pkthdr.csum_flags = 0;
548 counter_u64_add(rx_ring->rx_stats.csum_bad, 1);
549 ena_log_io(pdev, DBG, "RX L4 checksum error\n");
550 } else {
551 mbuf->m_pkthdr.csum_flags = CSUM_IP_CHECKED;
552 mbuf->m_pkthdr.csum_flags |= CSUM_IP_VALID;
553 counter_u64_add(rx_ring->rx_stats.csum_good, 1);
554 }
555 }
556 }
557
558 /**
559 * ena_rx_cleanup - handle rx irq
560 * @arg: ring for which irq is being handled
561 **/
562 static int
ena_rx_cleanup(struct ena_ring * rx_ring)563 ena_rx_cleanup(struct ena_ring *rx_ring)
564 {
565 struct ena_adapter *adapter;
566 device_t pdev;
567 struct mbuf *mbuf;
568 struct ena_com_rx_ctx ena_rx_ctx;
569 struct ena_com_io_cq *io_cq;
570 struct ena_com_io_sq *io_sq;
571 enum ena_regs_reset_reason_types reset_reason;
572 if_t ifp;
573 uint16_t ena_qid;
574 uint16_t next_to_clean;
575 uint32_t refill_required;
576 uint32_t refill_threshold;
577 uint32_t do_if_input = 0;
578 unsigned int qid;
579 int rc, i;
580 int budget = ENA_RX_BUDGET;
581 #ifdef DEV_NETMAP
582 int done;
583 #endif /* DEV_NETMAP */
584
585 adapter = rx_ring->que->adapter;
586 pdev = adapter->pdev;
587 ifp = adapter->ifp;
588 qid = rx_ring->que->id;
589 ena_qid = ENA_IO_RXQ_IDX(qid);
590 io_cq = &adapter->ena_dev->io_cq_queues[ena_qid];
591 io_sq = &adapter->ena_dev->io_sq_queues[ena_qid];
592 next_to_clean = rx_ring->next_to_clean;
593
594 #ifdef DEV_NETMAP
595 if (netmap_rx_irq(adapter->ifp, rx_ring->qid, &done) != NM_IRQ_PASS)
596 return (0);
597 #endif /* DEV_NETMAP */
598
599 ena_log_io(pdev, DBG, "rx: qid %d\n", qid);
600
601 do {
602 ena_rx_ctx.ena_bufs = rx_ring->ena_bufs;
603 ena_rx_ctx.max_bufs = adapter->max_rx_sgl_size;
604 ena_rx_ctx.descs = 0;
605 ena_rx_ctx.pkt_offset = 0;
606
607 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
608 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_POSTREAD);
609 rc = ena_com_rx_pkt(io_cq, io_sq, &ena_rx_ctx);
610 if (unlikely(rc != 0)) {
611 if (rc == ENA_COM_NO_SPACE) {
612 counter_u64_add(rx_ring->rx_stats.bad_desc_num,
613 1);
614 reset_reason = ENA_REGS_RESET_TOO_MANY_RX_DESCS;
615 } else if (rc == ENA_COM_FAULT) {
616 reset_reason = ENA_REGS_RESET_RX_DESCRIPTOR_MALFORMED;
617 } else {
618 counter_u64_add(rx_ring->rx_stats.bad_req_id,
619 1);
620 reset_reason = ENA_REGS_RESET_INV_RX_REQ_ID;
621 }
622 ena_trigger_reset(adapter, reset_reason);
623 return (0);
624 }
625
626 if (unlikely(ena_rx_ctx.descs == 0))
627 break;
628
629 ena_log_io(pdev, DBG,
630 "rx: q %d got packet from ena. descs #: %d l3 proto %d l4 proto %d hash: %x\n",
631 rx_ring->qid, ena_rx_ctx.descs, ena_rx_ctx.l3_proto,
632 ena_rx_ctx.l4_proto, ena_rx_ctx.hash);
633
634 /* Receive mbuf from the ring */
635 mbuf = ena_rx_mbuf(rx_ring, rx_ring->ena_bufs, &ena_rx_ctx,
636 &next_to_clean);
637 bus_dmamap_sync(io_cq->cdesc_addr.mem_handle.tag,
638 io_cq->cdesc_addr.mem_handle.map, BUS_DMASYNC_PREREAD);
639 /* Exit if we failed to retrieve a buffer */
640 if (unlikely(mbuf == NULL)) {
641 for (i = 0; i < ena_rx_ctx.descs; ++i) {
642 rx_ring->free_rx_ids[next_to_clean] =
643 rx_ring->ena_bufs[i].req_id;
644 next_to_clean = ENA_RX_RING_IDX_NEXT(
645 next_to_clean, rx_ring->ring_size);
646 }
647 break;
648 }
649
650 if (((if_getcapenable(ifp) & IFCAP_RXCSUM) != 0) ||
651 ((if_getcapenable(ifp) & IFCAP_RXCSUM_IPV6) != 0)) {
652 ena_rx_checksum(rx_ring, &ena_rx_ctx, mbuf);
653 }
654
655 counter_enter();
656 counter_u64_add_protected(rx_ring->rx_stats.bytes,
657 mbuf->m_pkthdr.len);
658 counter_u64_add_protected(adapter->hw_stats.rx_bytes,
659 mbuf->m_pkthdr.len);
660 counter_exit();
661 /*
662 * LRO is only for IP/TCP packets and TCP checksum of the packet
663 * should be computed by hardware.
664 */
665 do_if_input = 1;
666 if (((if_getcapenable(ifp) & IFCAP_LRO) != 0) &&
667 ((mbuf->m_pkthdr.csum_flags & CSUM_IP_VALID) != 0) &&
668 (ena_rx_ctx.l4_proto == ENA_ETH_IO_L4_PROTO_TCP)) {
669 /*
670 * Send to the stack if:
671 * - LRO not enabled, or
672 * - no LRO resources, or
673 * - lro enqueue fails
674 */
675 if ((rx_ring->lro.lro_cnt != 0) &&
676 (tcp_lro_rx(&rx_ring->lro, mbuf, 0) == 0))
677 do_if_input = 0;
678 }
679 if (do_if_input != 0) {
680 ena_log_io(pdev, DBG,
681 "calling if_input() with mbuf %p\n", mbuf);
682 if_input(ifp, mbuf);
683 }
684
685 counter_enter();
686 counter_u64_add_protected(rx_ring->rx_stats.cnt, 1);
687 counter_u64_add_protected(adapter->hw_stats.rx_packets, 1);
688 counter_exit();
689 } while (--budget);
690
691 rx_ring->next_to_clean = next_to_clean;
692
693 refill_required = ena_com_free_q_entries(io_sq);
694 refill_threshold = min_t(int,
695 rx_ring->ring_size / ENA_RX_REFILL_THRESH_DIVIDER,
696 ENA_RX_REFILL_THRESH_PACKET);
697
698 if (refill_required > refill_threshold) {
699 ena_refill_rx_bufs(rx_ring, refill_required);
700 }
701
702 tcp_lro_flush_all(&rx_ring->lro);
703
704 return (ENA_RX_BUDGET - budget);
705 }
706
707 static void
ena_tx_csum(struct ena_com_tx_ctx * ena_tx_ctx,struct mbuf * mbuf,bool disable_meta_caching)708 ena_tx_csum(struct ena_com_tx_ctx *ena_tx_ctx, struct mbuf *mbuf,
709 bool disable_meta_caching)
710 {
711 struct ena_com_tx_meta *ena_meta;
712 struct ether_vlan_header *eh;
713 struct mbuf *mbuf_next;
714 u32 mss;
715 bool offload;
716 uint16_t etype;
717 int ehdrlen;
718 struct ip *ip;
719 int ipproto;
720 int iphlen;
721 struct tcphdr *th;
722 int offset;
723
724 offload = false;
725 ena_meta = &ena_tx_ctx->ena_meta;
726 mss = mbuf->m_pkthdr.tso_segsz;
727
728 if (mss != 0)
729 offload = true;
730
731 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0)
732 offload = true;
733
734 if ((mbuf->m_pkthdr.csum_flags & CSUM_OFFLOAD) != 0)
735 offload = true;
736
737 if ((mbuf->m_pkthdr.csum_flags & CSUM6_OFFLOAD) != 0)
738 offload = true;
739
740 if (!offload) {
741 if (disable_meta_caching) {
742 memset(ena_meta, 0, sizeof(*ena_meta));
743 ena_tx_ctx->meta_valid = 1;
744 } else {
745 ena_tx_ctx->meta_valid = 0;
746 }
747 return;
748 }
749
750 /* Determine where frame payload starts. */
751 eh = mtod(mbuf, struct ether_vlan_header *);
752 if (eh->evl_encap_proto == htons(ETHERTYPE_VLAN)) {
753 etype = ntohs(eh->evl_proto);
754 ehdrlen = ETHER_HDR_LEN + ETHER_VLAN_ENCAP_LEN;
755 } else {
756 etype = ntohs(eh->evl_encap_proto);
757 ehdrlen = ETHER_HDR_LEN;
758 }
759
760 mbuf_next = m_getptr(mbuf, ehdrlen, &offset);
761
762 switch (etype) {
763 case ETHERTYPE_IP:
764 ip = (struct ip *)(mtodo(mbuf_next, offset));
765 iphlen = ip->ip_hl << 2;
766 ipproto = ip->ip_p;
767 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV4;
768 if ((ip->ip_off & htons(IP_DF)) != 0)
769 ena_tx_ctx->df = 1;
770 break;
771 case ETHERTYPE_IPV6:
772 ena_tx_ctx->l3_proto = ENA_ETH_IO_L3_PROTO_IPV6;
773 iphlen = ip6_lasthdr(mbuf, ehdrlen, IPPROTO_IPV6, &ipproto);
774 iphlen -= ehdrlen;
775 ena_tx_ctx->df = 1;
776 break;
777 default:
778 iphlen = 0;
779 ipproto = 0;
780 break;
781 }
782
783 mbuf_next = m_getptr(mbuf, iphlen + ehdrlen, &offset);
784 th = (struct tcphdr *)(mtodo(mbuf_next, offset));
785
786 if ((mbuf->m_pkthdr.csum_flags & CSUM_IP) != 0) {
787 ena_tx_ctx->l3_csum_enable = 1;
788 }
789 if ((mbuf->m_pkthdr.csum_flags & CSUM_TSO) != 0) {
790 ena_tx_ctx->tso_enable = 1;
791 ena_meta->l4_hdr_len = (th->th_off);
792 }
793
794 if (ipproto == IPPROTO_TCP) {
795 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_TCP;
796 if ((mbuf->m_pkthdr.csum_flags &
797 (CSUM_IP_TCP | CSUM_IP6_TCP)) != 0)
798 ena_tx_ctx->l4_csum_enable = 1;
799 else
800 ena_tx_ctx->l4_csum_enable = 0;
801 } else if (ipproto == IPPROTO_UDP) {
802 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UDP;
803 if ((mbuf->m_pkthdr.csum_flags &
804 (CSUM_IP_UDP | CSUM_IP6_UDP)) != 0)
805 ena_tx_ctx->l4_csum_enable = 1;
806 else
807 ena_tx_ctx->l4_csum_enable = 0;
808 } else {
809 ena_tx_ctx->l4_proto = ENA_ETH_IO_L4_PROTO_UNKNOWN;
810 ena_tx_ctx->l4_csum_enable = 0;
811 }
812
813 ena_meta->mss = mss;
814 ena_meta->l3_hdr_len = iphlen;
815 ena_meta->l3_hdr_offset = ehdrlen;
816 ena_tx_ctx->meta_valid = 1;
817 }
818
819 static int
ena_check_and_collapse_mbuf(struct ena_ring * tx_ring,struct mbuf ** mbuf)820 ena_check_and_collapse_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
821 {
822 struct ena_adapter *adapter;
823 struct mbuf *collapsed_mbuf;
824 int num_frags;
825
826 adapter = tx_ring->adapter;
827 num_frags = ena_mbuf_count(*mbuf);
828
829 /* One segment must be reserved for configuration descriptor. */
830 if (num_frags < adapter->max_tx_sgl_size)
831 return (0);
832
833 if ((num_frags == adapter->max_tx_sgl_size) &&
834 ((*mbuf)->m_pkthdr.len < tx_ring->tx_max_header_size))
835 return (0);
836
837 counter_u64_add(tx_ring->tx_stats.collapse, 1);
838
839 collapsed_mbuf = m_collapse(*mbuf, M_NOWAIT,
840 adapter->max_tx_sgl_size - 1);
841 if (unlikely(collapsed_mbuf == NULL)) {
842 counter_u64_add(tx_ring->tx_stats.collapse_err, 1);
843 return (ENOMEM);
844 }
845
846 /* If mbuf was collapsed succesfully, original mbuf is released. */
847 *mbuf = collapsed_mbuf;
848
849 return (0);
850 }
851
852 static int
ena_tx_map_mbuf(struct ena_ring * tx_ring,struct ena_tx_buffer * tx_info,struct mbuf * mbuf,void ** push_hdr,u16 * header_len)853 ena_tx_map_mbuf(struct ena_ring *tx_ring, struct ena_tx_buffer *tx_info,
854 struct mbuf *mbuf, void **push_hdr, u16 *header_len)
855 {
856 struct ena_adapter *adapter = tx_ring->adapter;
857 struct ena_com_buf *ena_buf;
858 bus_dma_segment_t segs[ENA_BUS_DMA_SEGS];
859 size_t iseg = 0;
860 uint32_t mbuf_head_len;
861 uint16_t offset;
862 int rc, nsegs;
863
864 mbuf_head_len = mbuf->m_len;
865 tx_info->mbuf = mbuf;
866 ena_buf = tx_info->bufs;
867
868 /*
869 * For easier maintaining of the DMA map, map the whole mbuf even if
870 * the LLQ is used. The descriptors will be filled using the segments.
871 */
872 rc = bus_dmamap_load_mbuf_sg(adapter->tx_buf_tag,
873 tx_info->dmamap, mbuf, segs, &nsegs, BUS_DMA_NOWAIT);
874 if (unlikely((rc != 0) || (nsegs == 0))) {
875 ena_log_io(adapter->pdev, WARN,
876 "dmamap load failed! err: %d nsegs: %d\n", rc, nsegs);
877 goto dma_error;
878 }
879
880 if (tx_ring->tx_mem_queue_type == ENA_ADMIN_PLACEMENT_POLICY_DEV) {
881 /*
882 * When the device is LLQ mode, the driver will copy
883 * the header into the device memory space.
884 * the ena_com layer assumes the header is in a linear
885 * memory space.
886 * This assumption might be wrong since part of the header
887 * can be in the fragmented buffers.
888 * First check if header fits in the mbuf. If not, copy it to
889 * separate buffer that will be holding linearized data.
890 */
891 *header_len = min_t(uint32_t, mbuf->m_pkthdr.len,
892 tx_ring->tx_max_header_size);
893
894 /* If header is in linear space, just point into mbuf's data. */
895 if (likely(*header_len <= mbuf_head_len)) {
896 *push_hdr = mbuf->m_data;
897 /*
898 * Otherwise, copy whole portion of header from multiple
899 * mbufs to intermediate buffer.
900 */
901 } else {
902 m_copydata(mbuf, 0, *header_len,
903 tx_ring->push_buf_intermediate_buf);
904 *push_hdr = tx_ring->push_buf_intermediate_buf;
905
906 counter_u64_add(tx_ring->tx_stats.llq_buffer_copy, 1);
907 }
908
909 ena_log_io(adapter->pdev, DBG,
910 "mbuf: %p header_buf->vaddr: %p push_len: %d\n",
911 mbuf, *push_hdr, *header_len);
912
913 /* If packet is fitted in LLQ header, no need for DMA segments. */
914 if (mbuf->m_pkthdr.len <= tx_ring->tx_max_header_size) {
915 return (0);
916 } else {
917 offset = tx_ring->tx_max_header_size;
918 /*
919 * As Header part is mapped to LLQ header, we can skip
920 * it and just map the residuum of the mbuf to DMA
921 * Segments.
922 */
923 while (offset > 0) {
924 if (offset >= segs[iseg].ds_len) {
925 offset -= segs[iseg].ds_len;
926 } else {
927 ena_buf->paddr = segs[iseg].ds_addr +
928 offset;
929 ena_buf->len = segs[iseg].ds_len -
930 offset;
931 ena_buf++;
932 tx_info->num_of_bufs++;
933 offset = 0;
934 }
935 iseg++;
936 }
937 }
938 } else {
939 *push_hdr = NULL;
940 /*
941 * header_len is just a hint for the device. Because FreeBSD is
942 * not giving us information about packet header length and it
943 * is not guaranteed that all packet headers will be in the 1st
944 * mbuf, setting header_len to 0 is making the device ignore
945 * this value and resolve header on it's own.
946 */
947 *header_len = 0;
948 }
949
950 /* Map rest of the mbuf */
951 while (iseg < nsegs) {
952 ena_buf->paddr = segs[iseg].ds_addr;
953 ena_buf->len = segs[iseg].ds_len;
954 ena_buf++;
955 iseg++;
956 tx_info->num_of_bufs++;
957 }
958
959 return (0);
960
961 dma_error:
962 counter_u64_add(tx_ring->tx_stats.dma_mapping_err, 1);
963 tx_info->mbuf = NULL;
964 return (rc);
965 }
966
967 static int
ena_xmit_mbuf(struct ena_ring * tx_ring,struct mbuf ** mbuf)968 ena_xmit_mbuf(struct ena_ring *tx_ring, struct mbuf **mbuf)
969 {
970 struct ena_adapter *adapter;
971 device_t pdev;
972 struct ena_tx_buffer *tx_info;
973 struct ena_com_tx_ctx ena_tx_ctx;
974 struct ena_com_dev *ena_dev;
975 struct ena_com_io_sq *io_sq;
976 void *push_hdr;
977 uint16_t next_to_use;
978 uint16_t req_id;
979 uint16_t ena_qid;
980 uint16_t header_len;
981 int rc;
982 int nb_hw_desc;
983
984 ena_qid = ENA_IO_TXQ_IDX(tx_ring->que->id);
985 adapter = tx_ring->que->adapter;
986 pdev = adapter->pdev;
987 ena_dev = adapter->ena_dev;
988 io_sq = &ena_dev->io_sq_queues[ena_qid];
989
990 rc = ena_check_and_collapse_mbuf(tx_ring, mbuf);
991 if (unlikely(rc != 0)) {
992 ena_log_io(pdev, WARN, "Failed to collapse mbuf! err: %d\n",
993 rc);
994 return (rc);
995 }
996
997 ena_log_io(pdev, DBG, "Tx: %d bytes\n", (*mbuf)->m_pkthdr.len);
998
999 next_to_use = tx_ring->next_to_use;
1000 req_id = tx_ring->free_tx_ids[next_to_use];
1001 tx_info = &tx_ring->tx_buffer_info[req_id];
1002 tx_info->num_of_bufs = 0;
1003
1004 ENA_WARN(tx_info->mbuf != NULL, adapter->ena_dev,
1005 "mbuf isn't NULL for req_id %d\n", req_id);
1006
1007 rc = ena_tx_map_mbuf(tx_ring, tx_info, *mbuf, &push_hdr, &header_len);
1008 if (unlikely(rc != 0)) {
1009 ena_log_io(pdev, WARN, "Failed to map TX mbuf\n");
1010 return (rc);
1011 }
1012 memset(&ena_tx_ctx, 0x0, sizeof(struct ena_com_tx_ctx));
1013 ena_tx_ctx.ena_bufs = tx_info->bufs;
1014 ena_tx_ctx.push_header = push_hdr;
1015 ena_tx_ctx.num_bufs = tx_info->num_of_bufs;
1016 ena_tx_ctx.req_id = req_id;
1017 ena_tx_ctx.header_len = header_len;
1018
1019 /* Set flags and meta data */
1020 ena_tx_csum(&ena_tx_ctx, *mbuf, adapter->disable_meta_caching);
1021
1022 if (tx_ring->acum_pkts == ENA_DB_THRESHOLD ||
1023 ena_com_is_doorbell_needed(tx_ring->ena_com_io_sq, &ena_tx_ctx)) {
1024 ena_log_io(pdev, DBG,
1025 "llq tx max burst size of queue %d achieved, writing doorbell to send burst\n",
1026 tx_ring->que->id);
1027 ena_ring_tx_doorbell(tx_ring);
1028 }
1029
1030 /* Prepare the packet's descriptors and send them to device */
1031 rc = ena_com_prepare_tx(io_sq, &ena_tx_ctx, &nb_hw_desc);
1032 if (unlikely(rc != 0)) {
1033 if (likely(rc == ENA_COM_NO_MEM)) {
1034 ena_log_io(pdev, DBG, "tx ring[%d] is out of space\n",
1035 tx_ring->que->id);
1036 } else {
1037 ena_log(pdev, ERR, "failed to prepare tx bufs\n");
1038 ena_trigger_reset(adapter,
1039 ENA_REGS_RESET_DRIVER_INVALID_STATE);
1040 }
1041 counter_u64_add(tx_ring->tx_stats.prepare_ctx_err, 1);
1042 goto dma_error;
1043 }
1044
1045 counter_enter();
1046 counter_u64_add_protected(tx_ring->tx_stats.cnt, 1);
1047 counter_u64_add_protected(tx_ring->tx_stats.bytes,
1048 (*mbuf)->m_pkthdr.len);
1049
1050 counter_u64_add_protected(adapter->hw_stats.tx_packets, 1);
1051 counter_u64_add_protected(adapter->hw_stats.tx_bytes,
1052 (*mbuf)->m_pkthdr.len);
1053 counter_exit();
1054
1055 tx_info->tx_descs = nb_hw_desc;
1056 getbinuptime(&tx_info->timestamp);
1057 tx_info->print_once = true;
1058
1059 tx_ring->next_to_use = ENA_TX_RING_IDX_NEXT(next_to_use,
1060 tx_ring->ring_size);
1061
1062 /* stop the queue when no more space available, the packet can have up
1063 * to sgl_size + 2. one for the meta descriptor and one for header
1064 * (if the header is larger than tx_max_header_size).
1065 */
1066 if (unlikely(!ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1067 adapter->max_tx_sgl_size + 2))) {
1068 ena_log_io(pdev, DBG, "Stop queue %d\n", tx_ring->que->id);
1069
1070 tx_ring->running = false;
1071 counter_u64_add(tx_ring->tx_stats.queue_stop, 1);
1072
1073 /* There is a rare condition where this function decides to
1074 * stop the queue but meanwhile tx_cleanup() updates
1075 * next_to_completion and terminates.
1076 * The queue will remain stopped forever.
1077 * To solve this issue this function performs mb(), checks
1078 * the wakeup condition and wakes up the queue if needed.
1079 */
1080 mb();
1081
1082 if (ena_com_sq_have_enough_space(tx_ring->ena_com_io_sq,
1083 ENA_TX_RESUME_THRESH)) {
1084 tx_ring->running = true;
1085 counter_u64_add(tx_ring->tx_stats.queue_wakeup, 1);
1086 }
1087 }
1088
1089 bus_dmamap_sync(adapter->tx_buf_tag, tx_info->dmamap,
1090 BUS_DMASYNC_PREWRITE);
1091
1092 return (0);
1093
1094 dma_error:
1095 tx_info->mbuf = NULL;
1096 bus_dmamap_unload(adapter->tx_buf_tag, tx_info->dmamap);
1097
1098 return (rc);
1099 }
1100
1101 static void
ena_start_xmit(struct ena_ring * tx_ring)1102 ena_start_xmit(struct ena_ring *tx_ring)
1103 {
1104 struct mbuf *mbuf;
1105 struct ena_adapter *adapter = tx_ring->adapter;
1106 int ret = 0;
1107
1108 ENA_RING_MTX_ASSERT(tx_ring);
1109
1110 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
1111 return;
1112
1113 if (unlikely(!ENA_FLAG_ISSET(ENA_FLAG_LINK_UP, adapter)))
1114 return;
1115
1116 while ((mbuf = drbr_peek(adapter->ifp, tx_ring->br)) != NULL) {
1117 ena_log_io(adapter->pdev, DBG,
1118 "\ndequeued mbuf %p with flags %#x and header csum flags %#jx\n",
1119 mbuf, mbuf->m_flags, (uint64_t)mbuf->m_pkthdr.csum_flags);
1120
1121 if (unlikely(!tx_ring->running)) {
1122 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1123 break;
1124 }
1125
1126 if (unlikely((ret = ena_xmit_mbuf(tx_ring, &mbuf)) != 0)) {
1127 if (ret == ENA_COM_NO_MEM) {
1128 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1129 } else if (ret == ENA_COM_NO_SPACE) {
1130 drbr_putback(adapter->ifp, tx_ring->br, mbuf);
1131 } else {
1132 m_freem(mbuf);
1133 drbr_advance(adapter->ifp, tx_ring->br);
1134 }
1135
1136 break;
1137 }
1138
1139 drbr_advance(adapter->ifp, tx_ring->br);
1140
1141 if (unlikely((if_getdrvflags(adapter->ifp) & IFF_DRV_RUNNING) == 0))
1142 return;
1143
1144 tx_ring->acum_pkts++;
1145
1146 BPF_MTAP(adapter->ifp, mbuf);
1147 }
1148
1149 if (likely(tx_ring->acum_pkts != 0)) {
1150 /* Trigger the dma engine */
1151 ena_ring_tx_doorbell(tx_ring);
1152 }
1153
1154 if (unlikely(!tx_ring->running))
1155 taskqueue_enqueue(tx_ring->que->cleanup_tq,
1156 &tx_ring->que->cleanup_task);
1157 }
1158