1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include "setup.h" 5 #include "en/params.h" 6 #include "en/txrx.h" 7 #include "en/health.h" 8 #include <net/xdp_sock_drv.h> 9 10 static int mlx5e_legacy_rq_validate_xsk(struct mlx5_core_dev *mdev, 11 struct mlx5e_params *params, 12 struct mlx5e_xsk_param *xsk) 13 { 14 if (!mlx5e_rx_is_linear_skb(mdev, params, xsk)) { 15 mlx5_core_err(mdev, "Legacy RQ linear mode for XSK can't be activated with current params\n"); 16 return -EINVAL; 17 } 18 19 return 0; 20 } 21 22 /* The limitation of 2048 can be altered, but shouldn't go beyond the minimal 23 * stride size of striding RQ. 24 */ 25 #define MLX5E_MIN_XSK_CHUNK_SIZE max(2048, XDP_UMEM_MIN_CHUNK_SIZE) 26 27 bool mlx5e_validate_xsk_param(struct mlx5e_params *params, 28 struct mlx5e_xsk_param *xsk, 29 struct mlx5_core_dev *mdev) 30 { 31 /* AF_XDP doesn't support frames larger than PAGE_SIZE. */ 32 if (xsk->chunk_size > PAGE_SIZE || xsk->chunk_size < MLX5E_MIN_XSK_CHUNK_SIZE) { 33 mlx5_core_err(mdev, "XSK chunk size %u out of bounds [%u, %lu]\n", xsk->chunk_size, 34 MLX5E_MIN_XSK_CHUNK_SIZE, PAGE_SIZE); 35 return false; 36 } 37 38 /* frag_sz is different for regular and XSK RQs, so ensure that linear 39 * SKB mode is possible. 40 */ 41 switch (params->rq_wq_type) { 42 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 43 return !mlx5e_mpwrq_validate_xsk(mdev, params, xsk); 44 default: /* MLX5_WQ_TYPE_CYCLIC */ 45 return !mlx5e_legacy_rq_validate_xsk(mdev, params, xsk); 46 } 47 } 48 49 static void mlx5e_build_xsk_cparam(struct mlx5_core_dev *mdev, 50 struct mlx5e_params *params, 51 struct mlx5e_xsk_param *xsk, 52 struct mlx5e_channel_param *cparam) 53 { 54 mlx5e_build_rq_param(mdev, params, xsk, &cparam->rq); 55 mlx5e_build_xdpsq_param(mdev, params, xsk, &cparam->xdp_sq); 56 } 57 58 static int mlx5e_init_xsk_rq(struct mlx5e_channel *c, 59 struct mlx5e_params *params, 60 struct xsk_buff_pool *pool, 61 struct mlx5e_xsk_param *xsk, 62 struct mlx5e_rq *rq) 63 { 64 struct mlx5_core_dev *mdev = c->mdev; 65 int rq_xdp_ix; 66 int err; 67 68 rq->wq_type = params->rq_wq_type; 69 rq->pdev = c->pdev; 70 rq->netdev = c->netdev; 71 rq->priv = c->priv; 72 rq->tstamp = c->tstamp; 73 rq->clock = &mdev->clock; 74 rq->icosq = &c->icosq; 75 rq->ix = c->ix; 76 rq->channel = c; 77 rq->mdev = mdev; 78 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 79 rq->xdpsq = &c->rq_xdpsq; 80 rq->xsk_pool = pool; 81 rq->stats = &c->priv->channel_stats[c->ix]->xskrq; 82 rq->ptp_cyc2time = mlx5_rq_ts_translator(mdev); 83 rq_xdp_ix = c->ix; 84 err = mlx5e_rq_set_handlers(rq, params, xsk); 85 if (err) 86 return err; 87 88 return xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix, c->napi.napi_id); 89 } 90 91 static int mlx5e_open_xsk_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 92 struct mlx5e_rq_param *rq_params, struct xsk_buff_pool *pool, 93 struct mlx5e_xsk_param *xsk) 94 { 95 u16 q_counter = c->priv->q_counter[c->sd_ix]; 96 struct mlx5e_rq *xskrq = &c->xskrq; 97 int err; 98 99 err = mlx5e_init_xsk_rq(c, params, pool, xsk, xskrq); 100 if (err) 101 return err; 102 103 err = mlx5e_open_rq(params, rq_params, xsk, cpu_to_node(c->cpu), q_counter, xskrq); 104 if (err) 105 return err; 106 107 __set_bit(MLX5E_RQ_STATE_XSK, &xskrq->state); 108 return 0; 109 } 110 111 int mlx5e_open_xsk(struct mlx5e_priv *priv, struct mlx5e_params *params, 112 struct mlx5e_xsk_param *xsk, struct xsk_buff_pool *pool, 113 struct mlx5e_channel *c) 114 { 115 struct mlx5e_channel_param *cparam; 116 struct mlx5e_create_cq_param ccp; 117 int err; 118 119 mlx5e_build_create_cq_param(&ccp, c); 120 121 if (!mlx5e_validate_xsk_param(params, xsk, priv->mdev)) 122 return -EINVAL; 123 124 cparam = kvzalloc(sizeof(*cparam), GFP_KERNEL); 125 if (!cparam) 126 return -ENOMEM; 127 128 mlx5e_build_xsk_cparam(priv->mdev, params, xsk, cparam); 129 130 err = mlx5e_open_cq(c->mdev, params->rx_cq_moderation, &cparam->rq.cqp, &ccp, 131 &c->xskrq.cq); 132 if (unlikely(err)) 133 goto err_free_cparam; 134 135 err = mlx5e_open_xsk_rq(c, params, &cparam->rq, pool, xsk); 136 if (unlikely(err)) 137 goto err_close_rx_cq; 138 139 err = mlx5e_open_cq(c->mdev, params->tx_cq_moderation, &cparam->xdp_sq.cqp, &ccp, 140 &c->xsksq.cq); 141 if (unlikely(err)) 142 goto err_close_rq; 143 144 /* Create a separate SQ, so that when the buff pool is disabled, we could 145 * close this SQ safely and stop receiving CQEs. In other case, e.g., if 146 * the XDPSQ was used instead, we might run into trouble when the buff pool 147 * is disabled and then re-enabled, but the SQ continues receiving CQEs 148 * from the old buff pool. 149 */ 150 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, pool, &c->xsksq, true); 151 if (unlikely(err)) 152 goto err_close_tx_cq; 153 154 kvfree(cparam); 155 156 set_bit(MLX5E_CHANNEL_STATE_XSK, c->state); 157 158 return 0; 159 160 err_close_tx_cq: 161 mlx5e_close_cq(&c->xsksq.cq); 162 163 err_close_rq: 164 mlx5e_close_rq(&c->xskrq); 165 166 err_close_rx_cq: 167 mlx5e_close_cq(&c->xskrq.cq); 168 169 err_free_cparam: 170 kvfree(cparam); 171 172 return err; 173 } 174 175 void mlx5e_close_xsk(struct mlx5e_channel *c) 176 { 177 clear_bit(MLX5E_CHANNEL_STATE_XSK, c->state); 178 synchronize_net(); /* Sync with NAPI. */ 179 180 mlx5e_close_rq(&c->xskrq); 181 mlx5e_close_cq(&c->xskrq.cq); 182 mlx5e_close_xdpsq(&c->xsksq); 183 mlx5e_close_cq(&c->xsksq.cq); 184 185 memset(&c->xskrq, 0, sizeof(c->xskrq)); 186 memset(&c->xsksq, 0, sizeof(c->xsksq)); 187 } 188 189 void mlx5e_activate_xsk(struct mlx5e_channel *c) 190 { 191 /* ICOSQ recovery deactivates RQs. Suspend the recovery to avoid 192 * activating XSKRQ in the middle of recovery. 193 */ 194 mlx5e_reporter_icosq_suspend_recovery(c); 195 set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); 196 mlx5e_reporter_icosq_resume_recovery(c); 197 198 /* TX queue is created active. */ 199 } 200 201 void mlx5e_deactivate_xsk(struct mlx5e_channel *c) 202 { 203 /* ICOSQ recovery may reactivate XSKRQ if clear_bit is called in the 204 * middle of recovery. Suspend the recovery to avoid it. 205 */ 206 mlx5e_reporter_icosq_suspend_recovery(c); 207 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); 208 mlx5e_reporter_icosq_resume_recovery(c); 209 synchronize_net(); /* Sync with NAPI to prevent mlx5e_post_rx_wqes. */ 210 211 /* TX queue is disabled on close. */ 212 } 213