1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "tx.h" 5 #include "pool.h" 6 #include "en/xdp.h" 7 #include "en/params.h" 8 #include <net/xdp_sock_drv.h> 9 10 int mlx5e_xsk_wakeup(struct net_device *dev, u32 qid, u32 flags) 11 { 12 struct mlx5e_priv *priv = netdev_priv(dev); 13 struct mlx5e_params *params = &priv->channels.params; 14 struct mlx5e_channel *c; 15 u16 ix; 16 17 if (unlikely(!mlx5e_xdp_is_active(priv))) 18 return -ENETDOWN; 19 20 if (unlikely(!mlx5e_qid_get_ch_if_in_group(params, qid, MLX5E_RQ_GROUP_XSK, &ix))) 21 return -EINVAL; 22 23 c = priv->channels.c[ix]; 24 25 if (unlikely(!test_bit(MLX5E_CHANNEL_STATE_XSK, c->state))) 26 return -ENXIO; 27 28 if (!napi_if_scheduled_mark_missed(&c->napi)) { 29 /* To avoid WQE overrun, don't post a NOP if async_icosq is not 30 * active and not polled by NAPI. Return 0, because the upcoming 31 * activate will trigger the IRQ for us. 32 */ 33 if (unlikely(!test_bit(MLX5E_SQ_STATE_ENABLED, &c->async_icosq.state))) 34 return 0; 35 36 if (test_and_set_bit(MLX5E_SQ_STATE_PENDING_XSK_TX, &c->async_icosq.state)) 37 return 0; 38 39 spin_lock_bh(&c->async_icosq_lock); 40 mlx5e_trigger_irq(&c->async_icosq); 41 spin_unlock_bh(&c->async_icosq_lock); 42 } 43 44 return 0; 45 } 46 47 /* When TX fails (because of the size of the packet), we need to get completions 48 * in order, so post a NOP to get a CQE. Since AF_XDP doesn't distinguish 49 * between successful TX and errors, handling in mlx5e_poll_xdpsq_cq is the 50 * same. 51 */ 52 static void mlx5e_xsk_tx_post_err(struct mlx5e_xdpsq *sq, 53 struct mlx5e_xdp_info *xdpi) 54 { 55 u16 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 56 struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[pi]; 57 struct mlx5e_tx_wqe *nopwqe; 58 59 wi->num_wqebbs = 1; 60 wi->num_pkts = 1; 61 62 nopwqe = mlx5e_post_nop(&sq->wq, sq->sqn, &sq->pc); 63 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, xdpi); 64 sq->doorbell_cseg = &nopwqe->ctrl; 65 } 66 67 bool mlx5e_xsk_tx(struct mlx5e_xdpsq *sq, unsigned int budget) 68 { 69 struct xsk_buff_pool *pool = sq->xsk_pool; 70 struct mlx5e_xmit_data xdptxd; 71 struct mlx5e_xdp_info xdpi; 72 bool work_done = true; 73 bool flush = false; 74 75 xdpi.mode = MLX5E_XDP_XMIT_MODE_XSK; 76 77 for (; budget; budget--) { 78 int check_result = INDIRECT_CALL_2(sq->xmit_xdp_frame_check, 79 mlx5e_xmit_xdp_frame_check_mpwqe, 80 mlx5e_xmit_xdp_frame_check, 81 sq); 82 struct xdp_desc desc; 83 bool ret; 84 85 if (unlikely(check_result < 0)) { 86 work_done = false; 87 break; 88 } 89 90 if (!xsk_tx_peek_desc(pool, &desc)) { 91 /* TX will get stuck until something wakes it up by 92 * triggering NAPI. Currently it's expected that the 93 * application calls sendto() if there are consumed, but 94 * not completed frames. 95 */ 96 break; 97 } 98 99 xdptxd.dma_addr = xsk_buff_raw_get_dma(pool, desc.addr); 100 xdptxd.data = xsk_buff_raw_get_data(pool, desc.addr); 101 xdptxd.len = desc.len; 102 103 xsk_buff_raw_dma_sync_for_device(pool, xdptxd.dma_addr, xdptxd.len); 104 105 ret = INDIRECT_CALL_2(sq->xmit_xdp_frame, mlx5e_xmit_xdp_frame_mpwqe, 106 mlx5e_xmit_xdp_frame, sq, &xdptxd, NULL, 107 check_result); 108 if (unlikely(!ret)) { 109 if (sq->mpwqe.wqe) 110 mlx5e_xdp_mpwqe_complete(sq); 111 112 mlx5e_xsk_tx_post_err(sq, &xdpi); 113 } else { 114 mlx5e_xdpi_fifo_push(&sq->db.xdpi_fifo, &xdpi); 115 } 116 117 flush = true; 118 } 119 120 if (flush) { 121 if (sq->mpwqe.wqe) 122 mlx5e_xdp_mpwqe_complete(sq); 123 mlx5e_xmit_xdp_doorbell(sq); 124 125 xsk_tx_release(pool); 126 } 127 128 return !(budget && work_done); 129 } 130