1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <net/tc_act/tc_gact.h> 34 #include <net/pkt_cls.h> 35 #include <linux/mlx5/fs.h> 36 #include <net/vxlan.h> 37 #include <net/geneve.h> 38 #include <linux/bpf.h> 39 #include <linux/if_bridge.h> 40 #include <net/page_pool.h> 41 #include <net/xdp_sock.h> 42 #include "eswitch.h" 43 #include "en.h" 44 #include "en/txrx.h" 45 #include "en_tc.h" 46 #include "en_rep.h" 47 #include "en_accel/ipsec.h" 48 #include "en_accel/ipsec_rxtx.h" 49 #include "en_accel/en_accel.h" 50 #include "en_accel/tls.h" 51 #include "accel/ipsec.h" 52 #include "accel/tls.h" 53 #include "lib/vxlan.h" 54 #include "lib/clock.h" 55 #include "en/port.h" 56 #include "en/xdp.h" 57 #include "lib/eq.h" 58 #include "en/monitor_stats.h" 59 #include "en/health.h" 60 #include "en/params.h" 61 #include "en/xsk/umem.h" 62 #include "en/xsk/setup.h" 63 #include "en/xsk/rx.h" 64 #include "en/xsk/tx.h" 65 #include "en/hv_vhca_stats.h" 66 #include "lib/mlx5.h" 67 68 69 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) 70 { 71 bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && 72 MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 73 MLX5_CAP_ETH(mdev, reg_umr_sq); 74 u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq); 75 bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap; 76 77 if (!striding_rq_umr) 78 return false; 79 if (!inline_umr) { 80 mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n", 81 (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap); 82 return false; 83 } 84 return true; 85 } 86 87 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, 88 struct mlx5e_params *params) 89 { 90 params->log_rq_mtu_frames = is_kdump_kernel() ? 91 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : 92 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; 93 94 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", 95 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, 96 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? 97 BIT(mlx5e_mpwqe_get_log_rq_size(params, NULL)) : 98 BIT(params->log_rq_mtu_frames), 99 BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params, NULL)), 100 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); 101 } 102 103 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, 104 struct mlx5e_params *params) 105 { 106 if (!mlx5e_check_fragmented_striding_rq_cap(mdev)) 107 return false; 108 109 if (MLX5_IPSEC_DEV(mdev)) 110 return false; 111 112 if (params->xdp_prog) { 113 /* XSK params are not considered here. If striding RQ is in use, 114 * and an XSK is being opened, mlx5e_rx_mpwqe_is_linear_skb will 115 * be called with the known XSK params. 116 */ 117 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) 118 return false; 119 } 120 121 return true; 122 } 123 124 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 125 { 126 params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) && 127 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ? 128 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ : 129 MLX5_WQ_TYPE_CYCLIC; 130 } 131 132 void mlx5e_update_carrier(struct mlx5e_priv *priv) 133 { 134 struct mlx5_core_dev *mdev = priv->mdev; 135 u8 port_state; 136 137 port_state = mlx5_query_vport_state(mdev, 138 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 139 0); 140 141 if (port_state == VPORT_STATE_UP) { 142 netdev_info(priv->netdev, "Link up\n"); 143 netif_carrier_on(priv->netdev); 144 } else { 145 netdev_info(priv->netdev, "Link down\n"); 146 netif_carrier_off(priv->netdev); 147 } 148 } 149 150 static void mlx5e_update_carrier_work(struct work_struct *work) 151 { 152 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 153 update_carrier_work); 154 155 mutex_lock(&priv->state_lock); 156 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 157 if (priv->profile->update_carrier) 158 priv->profile->update_carrier(priv); 159 mutex_unlock(&priv->state_lock); 160 } 161 162 void mlx5e_update_stats(struct mlx5e_priv *priv) 163 { 164 int i; 165 166 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--) 167 if (mlx5e_stats_grps[i].update_stats) 168 mlx5e_stats_grps[i].update_stats(priv); 169 } 170 171 void mlx5e_update_ndo_stats(struct mlx5e_priv *priv) 172 { 173 int i; 174 175 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--) 176 if (mlx5e_stats_grps[i].update_stats_mask & 177 MLX5E_NDO_UPDATE_STATS) 178 mlx5e_stats_grps[i].update_stats(priv); 179 } 180 181 static void mlx5e_update_stats_work(struct work_struct *work) 182 { 183 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 184 update_stats_work); 185 186 mutex_lock(&priv->state_lock); 187 priv->profile->update_stats(priv); 188 mutex_unlock(&priv->state_lock); 189 } 190 191 void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 192 { 193 if (!priv->profile->update_stats) 194 return; 195 196 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 197 return; 198 199 queue_work(priv->wq, &priv->update_stats_work); 200 } 201 202 static int async_event(struct notifier_block *nb, unsigned long event, void *data) 203 { 204 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 205 struct mlx5_eqe *eqe = data; 206 207 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 208 return NOTIFY_DONE; 209 210 switch (eqe->sub_type) { 211 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 212 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 213 queue_work(priv->wq, &priv->update_carrier_work); 214 break; 215 default: 216 return NOTIFY_DONE; 217 } 218 219 return NOTIFY_OK; 220 } 221 222 static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 223 { 224 priv->events_nb.notifier_call = async_event; 225 mlx5_notifier_register(priv->mdev, &priv->events_nb); 226 } 227 228 static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 229 { 230 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 231 } 232 233 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 234 struct mlx5e_icosq *sq, 235 struct mlx5e_umr_wqe *wqe) 236 { 237 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 238 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; 239 u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS); 240 241 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 242 ds_cnt); 243 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 244 cseg->imm = rq->mkey_be; 245 246 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 247 ucseg->xlt_octowords = 248 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE)); 249 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 250 } 251 252 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, 253 struct mlx5e_channel *c) 254 { 255 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 256 257 rq->mpwqe.info = kvzalloc_node(array_size(wq_sz, 258 sizeof(*rq->mpwqe.info)), 259 GFP_KERNEL, cpu_to_node(c->cpu)); 260 if (!rq->mpwqe.info) 261 return -ENOMEM; 262 263 mlx5e_build_umr_wqe(rq, &c->icosq, &rq->mpwqe.umr_wqe); 264 265 return 0; 266 } 267 268 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 269 u64 npages, u8 page_shift, 270 struct mlx5_core_mkey *umr_mkey) 271 { 272 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 273 void *mkc; 274 u32 *in; 275 int err; 276 277 in = kvzalloc(inlen, GFP_KERNEL); 278 if (!in) 279 return -ENOMEM; 280 281 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 282 283 MLX5_SET(mkc, mkc, free, 1); 284 MLX5_SET(mkc, mkc, umr_en, 1); 285 MLX5_SET(mkc, mkc, lw, 1); 286 MLX5_SET(mkc, mkc, lr, 1); 287 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 288 289 MLX5_SET(mkc, mkc, qpn, 0xffffff); 290 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn); 291 MLX5_SET64(mkc, mkc, len, npages << page_shift); 292 MLX5_SET(mkc, mkc, translations_octword_size, 293 MLX5_MTT_OCTW(npages)); 294 MLX5_SET(mkc, mkc, log_page_size, page_shift); 295 296 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 297 298 kvfree(in); 299 return err; 300 } 301 302 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 303 { 304 u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq)); 305 306 return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey); 307 } 308 309 static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix) 310 { 311 return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT; 312 } 313 314 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 315 { 316 struct mlx5e_wqe_frag_info next_frag = {}; 317 struct mlx5e_wqe_frag_info *prev = NULL; 318 int i; 319 320 next_frag.di = &rq->wqe.di[0]; 321 322 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 323 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 324 struct mlx5e_wqe_frag_info *frag = 325 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 326 int f; 327 328 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 329 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 330 next_frag.di++; 331 next_frag.offset = 0; 332 if (prev) 333 prev->last_in_page = true; 334 } 335 *frag = next_frag; 336 337 /* prepare next */ 338 next_frag.offset += frag_info[f].frag_stride; 339 prev = frag; 340 } 341 } 342 343 if (prev) 344 prev->last_in_page = true; 345 } 346 347 static int mlx5e_init_di_list(struct mlx5e_rq *rq, 348 int wq_sz, int cpu) 349 { 350 int len = wq_sz << rq->wqe.info.log_num_frags; 351 352 rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), 353 GFP_KERNEL, cpu_to_node(cpu)); 354 if (!rq->wqe.di) 355 return -ENOMEM; 356 357 mlx5e_init_frags_partition(rq); 358 359 return 0; 360 } 361 362 static void mlx5e_free_di_list(struct mlx5e_rq *rq) 363 { 364 kvfree(rq->wqe.di); 365 } 366 367 static void mlx5e_rq_err_cqe_work(struct work_struct *recover_work) 368 { 369 struct mlx5e_rq *rq = container_of(recover_work, struct mlx5e_rq, recover_work); 370 371 mlx5e_reporter_rq_cqe_err(rq); 372 } 373 374 static int mlx5e_alloc_rq(struct mlx5e_channel *c, 375 struct mlx5e_params *params, 376 struct mlx5e_xsk_param *xsk, 377 struct xdp_umem *umem, 378 struct mlx5e_rq_param *rqp, 379 struct mlx5e_rq *rq) 380 { 381 struct page_pool_params pp_params = { 0 }; 382 struct mlx5_core_dev *mdev = c->mdev; 383 void *rqc = rqp->rqc; 384 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 385 u32 num_xsk_frames = 0; 386 u32 rq_xdp_ix; 387 u32 pool_size; 388 int wq_sz; 389 int err; 390 int i; 391 392 rqp->wq.db_numa_node = cpu_to_node(c->cpu); 393 394 rq->wq_type = params->rq_wq_type; 395 rq->pdev = c->pdev; 396 rq->netdev = c->netdev; 397 rq->tstamp = c->tstamp; 398 rq->clock = &mdev->clock; 399 rq->channel = c; 400 rq->ix = c->ix; 401 rq->mdev = mdev; 402 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 403 rq->xdpsq = &c->rq_xdpsq; 404 rq->umem = umem; 405 406 if (rq->umem) 407 rq->stats = &c->priv->channel_stats[c->ix].xskrq; 408 else 409 rq->stats = &c->priv->channel_stats[c->ix].rq; 410 INIT_WORK(&rq->recover_work, mlx5e_rq_err_cqe_work); 411 412 if (params->xdp_prog) 413 bpf_prog_inc(params->xdp_prog); 414 rq->xdp_prog = params->xdp_prog; 415 416 rq_xdp_ix = rq->ix; 417 if (xsk) 418 rq_xdp_ix += params->num_channels * MLX5E_RQ_GROUP_XSK; 419 err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq_xdp_ix); 420 if (err < 0) 421 goto err_rq_wq_destroy; 422 423 rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 424 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params, xsk); 425 rq->buff.umem_headroom = xsk ? xsk->headroom : 0; 426 pool_size = 1 << params->log_rq_mtu_frames; 427 428 switch (rq->wq_type) { 429 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 430 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, 431 &rq->wq_ctrl); 432 if (err) 433 return err; 434 435 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 436 437 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 438 439 if (xsk) 440 num_xsk_frames = wq_sz << 441 mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); 442 443 pool_size = MLX5_MPWRQ_PAGES_PER_WQE << 444 mlx5e_mpwqe_get_log_rq_size(params, xsk); 445 446 rq->post_wqes = mlx5e_post_rx_mpwqes; 447 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe; 448 449 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe; 450 #ifdef CONFIG_MLX5_EN_IPSEC 451 if (MLX5_IPSEC_DEV(mdev)) { 452 err = -EINVAL; 453 netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n"); 454 goto err_rq_wq_destroy; 455 } 456 #endif 457 if (!rq->handle_rx_cqe) { 458 err = -EINVAL; 459 netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err); 460 goto err_rq_wq_destroy; 461 } 462 463 rq->mpwqe.skb_from_cqe_mpwrq = xsk ? 464 mlx5e_xsk_skb_from_cqe_mpwrq_linear : 465 mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) ? 466 mlx5e_skb_from_cqe_mpwrq_linear : 467 mlx5e_skb_from_cqe_mpwrq_nonlinear; 468 469 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk); 470 rq->mpwqe.num_strides = 471 BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk)); 472 473 err = mlx5e_create_rq_umr_mkey(mdev, rq); 474 if (err) 475 goto err_rq_wq_destroy; 476 rq->mkey_be = cpu_to_be32(rq->umr_mkey.key); 477 478 err = mlx5e_rq_alloc_mpwqe_info(rq, c); 479 if (err) 480 goto err_free; 481 break; 482 default: /* MLX5_WQ_TYPE_CYCLIC */ 483 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, 484 &rq->wq_ctrl); 485 if (err) 486 return err; 487 488 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 489 490 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 491 492 if (xsk) 493 num_xsk_frames = wq_sz << rq->wqe.info.log_num_frags; 494 495 rq->wqe.info = rqp->frags_info; 496 rq->wqe.frags = 497 kvzalloc_node(array_size(sizeof(*rq->wqe.frags), 498 (wq_sz << rq->wqe.info.log_num_frags)), 499 GFP_KERNEL, cpu_to_node(c->cpu)); 500 if (!rq->wqe.frags) { 501 err = -ENOMEM; 502 goto err_free; 503 } 504 505 err = mlx5e_init_di_list(rq, wq_sz, c->cpu); 506 if (err) 507 goto err_free; 508 509 rq->post_wqes = mlx5e_post_rx_wqes; 510 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe; 511 512 #ifdef CONFIG_MLX5_EN_IPSEC 513 if (c->priv->ipsec) 514 rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe; 515 else 516 #endif 517 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe; 518 if (!rq->handle_rx_cqe) { 519 err = -EINVAL; 520 netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err); 521 goto err_free; 522 } 523 524 rq->wqe.skb_from_cqe = xsk ? 525 mlx5e_xsk_skb_from_cqe_linear : 526 mlx5e_rx_is_linear_skb(params, NULL) ? 527 mlx5e_skb_from_cqe_linear : 528 mlx5e_skb_from_cqe_nonlinear; 529 rq->mkey_be = c->mkey_be; 530 } 531 532 if (xsk) { 533 err = mlx5e_xsk_resize_reuseq(umem, num_xsk_frames); 534 if (unlikely(err)) { 535 mlx5_core_err(mdev, "Unable to allocate the Reuse Ring for %u frames\n", 536 num_xsk_frames); 537 goto err_free; 538 } 539 540 rq->zca.free = mlx5e_xsk_zca_free; 541 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 542 MEM_TYPE_ZERO_COPY, 543 &rq->zca); 544 } else { 545 /* Create a page_pool and register it with rxq */ 546 pp_params.order = 0; 547 pp_params.flags = 0; /* No-internal DMA mapping in page_pool */ 548 pp_params.pool_size = pool_size; 549 pp_params.nid = cpu_to_node(c->cpu); 550 pp_params.dev = c->pdev; 551 pp_params.dma_dir = rq->buff.map_dir; 552 553 /* page_pool can be used even when there is no rq->xdp_prog, 554 * given page_pool does not handle DMA mapping there is no 555 * required state to clear. And page_pool gracefully handle 556 * elevated refcnt. 557 */ 558 rq->page_pool = page_pool_create(&pp_params); 559 if (IS_ERR(rq->page_pool)) { 560 err = PTR_ERR(rq->page_pool); 561 rq->page_pool = NULL; 562 goto err_free; 563 } 564 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 565 MEM_TYPE_PAGE_POOL, rq->page_pool); 566 } 567 if (err) 568 goto err_free; 569 570 for (i = 0; i < wq_sz; i++) { 571 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 572 struct mlx5e_rx_wqe_ll *wqe = 573 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 574 u32 byte_count = 575 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 576 u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i); 577 578 wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom); 579 wqe->data[0].byte_count = cpu_to_be32(byte_count); 580 wqe->data[0].lkey = rq->mkey_be; 581 } else { 582 struct mlx5e_rx_wqe_cyc *wqe = 583 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 584 int f; 585 586 for (f = 0; f < rq->wqe.info.num_frags; f++) { 587 u32 frag_size = rq->wqe.info.arr[f].frag_size | 588 MLX5_HW_START_PADDING; 589 590 wqe->data[f].byte_count = cpu_to_be32(frag_size); 591 wqe->data[f].lkey = rq->mkey_be; 592 } 593 /* check if num_frags is not a pow of two */ 594 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 595 wqe->data[f].byte_count = 0; 596 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 597 wqe->data[f].addr = 0; 598 } 599 } 600 } 601 602 INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work); 603 604 switch (params->rx_cq_moderation.cq_period_mode) { 605 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: 606 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_CQE; 607 break; 608 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: 609 default: 610 rq->dim.mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 611 } 612 613 rq->page_cache.head = 0; 614 rq->page_cache.tail = 0; 615 616 return 0; 617 618 err_free: 619 switch (rq->wq_type) { 620 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 621 kvfree(rq->mpwqe.info); 622 mlx5_core_destroy_mkey(mdev, &rq->umr_mkey); 623 break; 624 default: /* MLX5_WQ_TYPE_CYCLIC */ 625 kvfree(rq->wqe.frags); 626 mlx5e_free_di_list(rq); 627 } 628 629 err_rq_wq_destroy: 630 if (rq->xdp_prog) 631 bpf_prog_put(rq->xdp_prog); 632 xdp_rxq_info_unreg(&rq->xdp_rxq); 633 page_pool_destroy(rq->page_pool); 634 mlx5_wq_destroy(&rq->wq_ctrl); 635 636 return err; 637 } 638 639 static void mlx5e_free_rq(struct mlx5e_rq *rq) 640 { 641 int i; 642 643 if (rq->xdp_prog) 644 bpf_prog_put(rq->xdp_prog); 645 646 switch (rq->wq_type) { 647 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 648 kvfree(rq->mpwqe.info); 649 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey); 650 break; 651 default: /* MLX5_WQ_TYPE_CYCLIC */ 652 kvfree(rq->wqe.frags); 653 mlx5e_free_di_list(rq); 654 } 655 656 for (i = rq->page_cache.head; i != rq->page_cache.tail; 657 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) { 658 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i]; 659 660 /* With AF_XDP, page_cache is not used, so this loop is not 661 * entered, and it's safe to call mlx5e_page_release_dynamic 662 * directly. 663 */ 664 mlx5e_page_release_dynamic(rq, dma_info, false); 665 } 666 667 xdp_rxq_info_unreg(&rq->xdp_rxq); 668 page_pool_destroy(rq->page_pool); 669 mlx5_wq_destroy(&rq->wq_ctrl); 670 } 671 672 static int mlx5e_create_rq(struct mlx5e_rq *rq, 673 struct mlx5e_rq_param *param) 674 { 675 struct mlx5_core_dev *mdev = rq->mdev; 676 677 void *in; 678 void *rqc; 679 void *wq; 680 int inlen; 681 int err; 682 683 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 684 sizeof(u64) * rq->wq_ctrl.buf.npages; 685 in = kvzalloc(inlen, GFP_KERNEL); 686 if (!in) 687 return -ENOMEM; 688 689 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 690 wq = MLX5_ADDR_OF(rqc, rqc, wq); 691 692 memcpy(rqc, param->rqc, sizeof(param->rqc)); 693 694 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 695 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 696 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 697 MLX5_ADAPTER_PAGE_SHIFT); 698 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 699 700 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 701 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 702 703 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 704 705 kvfree(in); 706 707 return err; 708 } 709 710 int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, int next_state) 711 { 712 struct mlx5_core_dev *mdev = rq->mdev; 713 714 void *in; 715 void *rqc; 716 int inlen; 717 int err; 718 719 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 720 in = kvzalloc(inlen, GFP_KERNEL); 721 if (!in) 722 return -ENOMEM; 723 724 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 725 726 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 727 MLX5_SET(rqc, rqc, state, next_state); 728 729 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen); 730 731 kvfree(in); 732 733 return err; 734 } 735 736 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable) 737 { 738 struct mlx5e_channel *c = rq->channel; 739 struct mlx5e_priv *priv = c->priv; 740 struct mlx5_core_dev *mdev = priv->mdev; 741 742 void *in; 743 void *rqc; 744 int inlen; 745 int err; 746 747 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 748 in = kvzalloc(inlen, GFP_KERNEL); 749 if (!in) 750 return -ENOMEM; 751 752 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 753 754 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 755 MLX5_SET64(modify_rq_in, in, modify_bitmask, 756 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS); 757 MLX5_SET(rqc, rqc, scatter_fcs, enable); 758 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 759 760 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen); 761 762 kvfree(in); 763 764 return err; 765 } 766 767 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 768 { 769 struct mlx5e_channel *c = rq->channel; 770 struct mlx5_core_dev *mdev = c->mdev; 771 void *in; 772 void *rqc; 773 int inlen; 774 int err; 775 776 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 777 in = kvzalloc(inlen, GFP_KERNEL); 778 if (!in) 779 return -ENOMEM; 780 781 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 782 783 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 784 MLX5_SET64(modify_rq_in, in, modify_bitmask, 785 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 786 MLX5_SET(rqc, rqc, vsd, vsd); 787 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 788 789 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen); 790 791 kvfree(in); 792 793 return err; 794 } 795 796 static void mlx5e_destroy_rq(struct mlx5e_rq *rq) 797 { 798 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 799 } 800 801 int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 802 { 803 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 804 struct mlx5e_channel *c = rq->channel; 805 806 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 807 808 do { 809 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 810 return 0; 811 812 msleep(20); 813 } while (time_before(jiffies, exp_time)); 814 815 netdev_warn(c->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 816 c->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 817 818 mlx5e_reporter_rx_timeout(rq); 819 return -ETIMEDOUT; 820 } 821 822 void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 823 { 824 __be16 wqe_ix_be; 825 u16 wqe_ix; 826 827 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 828 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 829 u16 head = wq->head; 830 int i; 831 832 /* Outstanding UMR WQEs (in progress) start at wq->head */ 833 for (i = 0; i < rq->mpwqe.umr_in_progress; i++) { 834 rq->dealloc_wqe(rq, head); 835 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 836 } 837 838 while (!mlx5_wq_ll_is_empty(wq)) { 839 struct mlx5e_rx_wqe_ll *wqe; 840 841 wqe_ix_be = *wq->tail_next; 842 wqe_ix = be16_to_cpu(wqe_ix_be); 843 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 844 rq->dealloc_wqe(rq, wqe_ix); 845 mlx5_wq_ll_pop(wq, wqe_ix_be, 846 &wqe->next.next_wqe_index); 847 } 848 } else { 849 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 850 851 while (!mlx5_wq_cyc_is_empty(wq)) { 852 wqe_ix = mlx5_wq_cyc_get_tail(wq); 853 rq->dealloc_wqe(rq, wqe_ix); 854 mlx5_wq_cyc_pop(wq); 855 } 856 } 857 858 } 859 860 int mlx5e_open_rq(struct mlx5e_channel *c, struct mlx5e_params *params, 861 struct mlx5e_rq_param *param, struct mlx5e_xsk_param *xsk, 862 struct xdp_umem *umem, struct mlx5e_rq *rq) 863 { 864 int err; 865 866 err = mlx5e_alloc_rq(c, params, xsk, umem, param, rq); 867 if (err) 868 return err; 869 870 err = mlx5e_create_rq(rq, param); 871 if (err) 872 goto err_free_rq; 873 874 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 875 if (err) 876 goto err_destroy_rq; 877 878 if (MLX5_CAP_ETH(c->mdev, cqe_checksum_full)) 879 __set_bit(MLX5E_RQ_STATE_CSUM_FULL, &c->rq.state); 880 881 if (params->rx_dim_enabled) 882 __set_bit(MLX5E_RQ_STATE_AM, &c->rq.state); 883 884 /* We disable csum_complete when XDP is enabled since 885 * XDP programs might manipulate packets which will render 886 * skb->checksum incorrect. 887 */ 888 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || c->xdp) 889 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state); 890 891 return 0; 892 893 err_destroy_rq: 894 mlx5e_destroy_rq(rq); 895 err_free_rq: 896 mlx5e_free_rq(rq); 897 898 return err; 899 } 900 901 void mlx5e_activate_rq(struct mlx5e_rq *rq) 902 { 903 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 904 mlx5e_trigger_irq(&rq->channel->icosq); 905 } 906 907 void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 908 { 909 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 910 napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */ 911 } 912 913 void mlx5e_close_rq(struct mlx5e_rq *rq) 914 { 915 cancel_work_sync(&rq->dim.work); 916 cancel_work_sync(&rq->channel->icosq.recover_work); 917 cancel_work_sync(&rq->recover_work); 918 mlx5e_destroy_rq(rq); 919 mlx5e_free_rx_descs(rq); 920 mlx5e_free_rq(rq); 921 } 922 923 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 924 { 925 kvfree(sq->db.xdpi_fifo.xi); 926 kvfree(sq->db.wqe_info); 927 } 928 929 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 930 { 931 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 932 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 933 int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 934 935 xdpi_fifo->xi = kvzalloc_node(sizeof(*xdpi_fifo->xi) * dsegs_per_wq, 936 GFP_KERNEL, numa); 937 if (!xdpi_fifo->xi) 938 return -ENOMEM; 939 940 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 941 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 942 xdpi_fifo->mask = dsegs_per_wq - 1; 943 944 return 0; 945 } 946 947 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 948 { 949 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 950 int err; 951 952 sq->db.wqe_info = kvzalloc_node(sizeof(*sq->db.wqe_info) * wq_sz, 953 GFP_KERNEL, numa); 954 if (!sq->db.wqe_info) 955 return -ENOMEM; 956 957 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 958 if (err) { 959 mlx5e_free_xdpsq_db(sq); 960 return err; 961 } 962 963 return 0; 964 } 965 966 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 967 struct mlx5e_params *params, 968 struct xdp_umem *umem, 969 struct mlx5e_sq_param *param, 970 struct mlx5e_xdpsq *sq, 971 bool is_redirect) 972 { 973 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 974 struct mlx5_core_dev *mdev = c->mdev; 975 struct mlx5_wq_cyc *wq = &sq->wq; 976 int err; 977 978 sq->pdev = c->pdev; 979 sq->mkey_be = c->mkey_be; 980 sq->channel = c; 981 sq->uar_map = mdev->mlx5e_res.bfreg.map; 982 sq->min_inline_mode = params->tx_min_inline_mode; 983 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 984 sq->umem = umem; 985 986 sq->stats = sq->umem ? 987 &c->priv->channel_stats[c->ix].xsksq : 988 is_redirect ? 989 &c->priv->channel_stats[c->ix].xdpsq : 990 &c->priv->channel_stats[c->ix].rq_xdpsq; 991 992 param->wq.db_numa_node = cpu_to_node(c->cpu); 993 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 994 if (err) 995 return err; 996 wq->db = &wq->db[MLX5_SND_DBR]; 997 998 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 999 if (err) 1000 goto err_sq_wq_destroy; 1001 1002 return 0; 1003 1004 err_sq_wq_destroy: 1005 mlx5_wq_destroy(&sq->wq_ctrl); 1006 1007 return err; 1008 } 1009 1010 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 1011 { 1012 mlx5e_free_xdpsq_db(sq); 1013 mlx5_wq_destroy(&sq->wq_ctrl); 1014 } 1015 1016 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 1017 { 1018 kvfree(sq->db.ico_wqe); 1019 } 1020 1021 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 1022 { 1023 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1024 1025 sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz, 1026 sizeof(*sq->db.ico_wqe)), 1027 GFP_KERNEL, numa); 1028 if (!sq->db.ico_wqe) 1029 return -ENOMEM; 1030 1031 return 0; 1032 } 1033 1034 static void mlx5e_icosq_err_cqe_work(struct work_struct *recover_work) 1035 { 1036 struct mlx5e_icosq *sq = container_of(recover_work, struct mlx5e_icosq, 1037 recover_work); 1038 1039 mlx5e_reporter_icosq_cqe_err(sq); 1040 } 1041 1042 static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1043 struct mlx5e_sq_param *param, 1044 struct mlx5e_icosq *sq) 1045 { 1046 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1047 struct mlx5_core_dev *mdev = c->mdev; 1048 struct mlx5_wq_cyc *wq = &sq->wq; 1049 int err; 1050 1051 sq->channel = c; 1052 sq->uar_map = mdev->mlx5e_res.bfreg.map; 1053 1054 param->wq.db_numa_node = cpu_to_node(c->cpu); 1055 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1056 if (err) 1057 return err; 1058 wq->db = &wq->db[MLX5_SND_DBR]; 1059 1060 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1061 if (err) 1062 goto err_sq_wq_destroy; 1063 1064 INIT_WORK(&sq->recover_work, mlx5e_icosq_err_cqe_work); 1065 1066 return 0; 1067 1068 err_sq_wq_destroy: 1069 mlx5_wq_destroy(&sq->wq_ctrl); 1070 1071 return err; 1072 } 1073 1074 static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1075 { 1076 mlx5e_free_icosq_db(sq); 1077 mlx5_wq_destroy(&sq->wq_ctrl); 1078 } 1079 1080 static void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1081 { 1082 kvfree(sq->db.wqe_info); 1083 kvfree(sq->db.dma_fifo); 1084 } 1085 1086 static int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1087 { 1088 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1089 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1090 1091 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1092 sizeof(*sq->db.dma_fifo)), 1093 GFP_KERNEL, numa); 1094 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1095 sizeof(*sq->db.wqe_info)), 1096 GFP_KERNEL, numa); 1097 if (!sq->db.dma_fifo || !sq->db.wqe_info) { 1098 mlx5e_free_txqsq_db(sq); 1099 return -ENOMEM; 1100 } 1101 1102 sq->dma_fifo_mask = df_sz - 1; 1103 1104 return 0; 1105 } 1106 1107 static void mlx5e_tx_err_cqe_work(struct work_struct *recover_work); 1108 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1109 int txq_ix, 1110 struct mlx5e_params *params, 1111 struct mlx5e_sq_param *param, 1112 struct mlx5e_txqsq *sq, 1113 int tc) 1114 { 1115 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1116 struct mlx5_core_dev *mdev = c->mdev; 1117 struct mlx5_wq_cyc *wq = &sq->wq; 1118 int err; 1119 1120 sq->pdev = c->pdev; 1121 sq->tstamp = c->tstamp; 1122 sq->clock = &mdev->clock; 1123 sq->mkey_be = c->mkey_be; 1124 sq->channel = c; 1125 sq->ch_ix = c->ix; 1126 sq->txq_ix = txq_ix; 1127 sq->uar_map = mdev->mlx5e_res.bfreg.map; 1128 sq->min_inline_mode = params->tx_min_inline_mode; 1129 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1130 sq->stats = &c->priv->channel_stats[c->ix].sq[tc]; 1131 sq->stop_room = MLX5E_SQ_STOP_ROOM; 1132 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1133 if (!MLX5_CAP_ETH(mdev, wqe_vlan_insert)) 1134 set_bit(MLX5E_SQ_STATE_VLAN_NEED_L2_INLINE, &sq->state); 1135 if (MLX5_IPSEC_DEV(c->priv->mdev)) 1136 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1137 #ifdef CONFIG_MLX5_EN_TLS 1138 if (mlx5_accel_is_tls_device(c->priv->mdev)) { 1139 set_bit(MLX5E_SQ_STATE_TLS, &sq->state); 1140 sq->stop_room += MLX5E_SQ_TLS_ROOM + 1141 mlx5e_ktls_dumps_num_wqebbs(sq, MAX_SKB_FRAGS, 1142 TLS_MAX_PAYLOAD_SIZE); 1143 } 1144 #endif 1145 1146 param->wq.db_numa_node = cpu_to_node(c->cpu); 1147 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1148 if (err) 1149 return err; 1150 wq->db = &wq->db[MLX5_SND_DBR]; 1151 1152 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1153 if (err) 1154 goto err_sq_wq_destroy; 1155 1156 INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work); 1157 sq->dim.mode = params->tx_cq_moderation.cq_period_mode; 1158 1159 return 0; 1160 1161 err_sq_wq_destroy: 1162 mlx5_wq_destroy(&sq->wq_ctrl); 1163 1164 return err; 1165 } 1166 1167 static void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1168 { 1169 mlx5e_free_txqsq_db(sq); 1170 mlx5_wq_destroy(&sq->wq_ctrl); 1171 } 1172 1173 struct mlx5e_create_sq_param { 1174 struct mlx5_wq_ctrl *wq_ctrl; 1175 u32 cqn; 1176 u32 tisn; 1177 u8 tis_lst_sz; 1178 u8 min_inline_mode; 1179 }; 1180 1181 static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1182 struct mlx5e_sq_param *param, 1183 struct mlx5e_create_sq_param *csp, 1184 u32 *sqn) 1185 { 1186 void *in; 1187 void *sqc; 1188 void *wq; 1189 int inlen; 1190 int err; 1191 1192 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1193 sizeof(u64) * csp->wq_ctrl->buf.npages; 1194 in = kvzalloc(inlen, GFP_KERNEL); 1195 if (!in) 1196 return -ENOMEM; 1197 1198 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1199 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1200 1201 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1202 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1203 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1204 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1205 1206 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1207 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1208 1209 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1210 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1211 1212 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1213 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.bfreg.index); 1214 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1215 MLX5_ADAPTER_PAGE_SHIFT); 1216 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1217 1218 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1219 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1220 1221 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1222 1223 kvfree(in); 1224 1225 return err; 1226 } 1227 1228 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1229 struct mlx5e_modify_sq_param *p) 1230 { 1231 void *in; 1232 void *sqc; 1233 int inlen; 1234 int err; 1235 1236 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1237 in = kvzalloc(inlen, GFP_KERNEL); 1238 if (!in) 1239 return -ENOMEM; 1240 1241 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1242 1243 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1244 MLX5_SET(sqc, sqc, state, p->next_state); 1245 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1246 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); 1247 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1248 } 1249 1250 err = mlx5_core_modify_sq(mdev, sqn, in, inlen); 1251 1252 kvfree(in); 1253 1254 return err; 1255 } 1256 1257 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1258 { 1259 mlx5_core_destroy_sq(mdev, sqn); 1260 } 1261 1262 static int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1263 struct mlx5e_sq_param *param, 1264 struct mlx5e_create_sq_param *csp, 1265 u32 *sqn) 1266 { 1267 struct mlx5e_modify_sq_param msp = {0}; 1268 int err; 1269 1270 err = mlx5e_create_sq(mdev, param, csp, sqn); 1271 if (err) 1272 return err; 1273 1274 msp.curr_state = MLX5_SQC_STATE_RST; 1275 msp.next_state = MLX5_SQC_STATE_RDY; 1276 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1277 if (err) 1278 mlx5e_destroy_sq(mdev, *sqn); 1279 1280 return err; 1281 } 1282 1283 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1284 struct mlx5e_txqsq *sq, u32 rate); 1285 1286 static int mlx5e_open_txqsq(struct mlx5e_channel *c, 1287 u32 tisn, 1288 int txq_ix, 1289 struct mlx5e_params *params, 1290 struct mlx5e_sq_param *param, 1291 struct mlx5e_txqsq *sq, 1292 int tc) 1293 { 1294 struct mlx5e_create_sq_param csp = {}; 1295 u32 tx_rate; 1296 int err; 1297 1298 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1299 if (err) 1300 return err; 1301 1302 csp.tisn = tisn; 1303 csp.tis_lst_sz = 1; 1304 csp.cqn = sq->cq.mcq.cqn; 1305 csp.wq_ctrl = &sq->wq_ctrl; 1306 csp.min_inline_mode = sq->min_inline_mode; 1307 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1308 if (err) 1309 goto err_free_txqsq; 1310 1311 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1312 if (tx_rate) 1313 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1314 1315 if (params->tx_dim_enabled) 1316 sq->state |= BIT(MLX5E_SQ_STATE_AM); 1317 1318 return 0; 1319 1320 err_free_txqsq: 1321 mlx5e_free_txqsq(sq); 1322 1323 return err; 1324 } 1325 1326 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1327 { 1328 sq->txq = netdev_get_tx_queue(sq->channel->netdev, sq->txq_ix); 1329 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1330 netdev_tx_reset_queue(sq->txq); 1331 netif_tx_start_queue(sq->txq); 1332 } 1333 1334 void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1335 { 1336 __netif_tx_lock_bh(txq); 1337 netif_tx_stop_queue(txq); 1338 __netif_tx_unlock_bh(txq); 1339 } 1340 1341 static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1342 { 1343 struct mlx5e_channel *c = sq->channel; 1344 struct mlx5_wq_cyc *wq = &sq->wq; 1345 1346 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1347 /* prevent netif_tx_wake_queue */ 1348 napi_synchronize(&c->napi); 1349 1350 mlx5e_tx_disable_queue(sq->txq); 1351 1352 /* last doorbell out, godspeed .. */ 1353 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1354 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1355 struct mlx5e_tx_wqe_info *wi; 1356 struct mlx5e_tx_wqe *nop; 1357 1358 wi = &sq->db.wqe_info[pi]; 1359 1360 memset(wi, 0, sizeof(*wi)); 1361 wi->num_wqebbs = 1; 1362 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1363 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1364 } 1365 } 1366 1367 static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1368 { 1369 struct mlx5e_channel *c = sq->channel; 1370 struct mlx5_core_dev *mdev = c->mdev; 1371 struct mlx5_rate_limit rl = {0}; 1372 1373 cancel_work_sync(&sq->dim.work); 1374 cancel_work_sync(&sq->recover_work); 1375 mlx5e_destroy_sq(mdev, sq->sqn); 1376 if (sq->rate_limit) { 1377 rl.rate = sq->rate_limit; 1378 mlx5_rl_remove_rate(mdev, &rl); 1379 } 1380 mlx5e_free_txqsq_descs(sq); 1381 mlx5e_free_txqsq(sq); 1382 } 1383 1384 static void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1385 { 1386 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 1387 recover_work); 1388 1389 mlx5e_reporter_tx_err_cqe(sq); 1390 } 1391 1392 int mlx5e_open_icosq(struct mlx5e_channel *c, struct mlx5e_params *params, 1393 struct mlx5e_sq_param *param, struct mlx5e_icosq *sq) 1394 { 1395 struct mlx5e_create_sq_param csp = {}; 1396 int err; 1397 1398 err = mlx5e_alloc_icosq(c, param, sq); 1399 if (err) 1400 return err; 1401 1402 csp.cqn = sq->cq.mcq.cqn; 1403 csp.wq_ctrl = &sq->wq_ctrl; 1404 csp.min_inline_mode = params->tx_min_inline_mode; 1405 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1406 if (err) 1407 goto err_free_icosq; 1408 1409 return 0; 1410 1411 err_free_icosq: 1412 mlx5e_free_icosq(sq); 1413 1414 return err; 1415 } 1416 1417 void mlx5e_activate_icosq(struct mlx5e_icosq *icosq) 1418 { 1419 set_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1420 } 1421 1422 void mlx5e_deactivate_icosq(struct mlx5e_icosq *icosq) 1423 { 1424 struct mlx5e_channel *c = icosq->channel; 1425 1426 clear_bit(MLX5E_SQ_STATE_ENABLED, &icosq->state); 1427 napi_synchronize(&c->napi); 1428 } 1429 1430 void mlx5e_close_icosq(struct mlx5e_icosq *sq) 1431 { 1432 struct mlx5e_channel *c = sq->channel; 1433 1434 mlx5e_destroy_sq(c->mdev, sq->sqn); 1435 mlx5e_free_icosq(sq); 1436 } 1437 1438 int mlx5e_open_xdpsq(struct mlx5e_channel *c, struct mlx5e_params *params, 1439 struct mlx5e_sq_param *param, struct xdp_umem *umem, 1440 struct mlx5e_xdpsq *sq, bool is_redirect) 1441 { 1442 struct mlx5e_create_sq_param csp = {}; 1443 int err; 1444 1445 err = mlx5e_alloc_xdpsq(c, params, umem, param, sq, is_redirect); 1446 if (err) 1447 return err; 1448 1449 csp.tis_lst_sz = 1; 1450 csp.tisn = c->priv->tisn[c->lag_port][0]; /* tc = 0 */ 1451 csp.cqn = sq->cq.mcq.cqn; 1452 csp.wq_ctrl = &sq->wq_ctrl; 1453 csp.min_inline_mode = sq->min_inline_mode; 1454 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1455 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1456 if (err) 1457 goto err_free_xdpsq; 1458 1459 mlx5e_set_xmit_fp(sq, param->is_mpw); 1460 1461 if (!param->is_mpw) { 1462 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT; 1463 unsigned int inline_hdr_sz = 0; 1464 int i; 1465 1466 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) { 1467 inline_hdr_sz = MLX5E_XDP_MIN_INLINE; 1468 ds_cnt++; 1469 } 1470 1471 /* Pre initialize fixed WQE fields */ 1472 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) { 1473 struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[i]; 1474 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i); 1475 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 1476 struct mlx5_wqe_eth_seg *eseg = &wqe->eth; 1477 struct mlx5_wqe_data_seg *dseg; 1478 1479 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 1480 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz); 1481 1482 dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1); 1483 dseg->lkey = sq->mkey_be; 1484 1485 wi->num_wqebbs = 1; 1486 wi->num_pkts = 1; 1487 } 1488 } 1489 1490 return 0; 1491 1492 err_free_xdpsq: 1493 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1494 mlx5e_free_xdpsq(sq); 1495 1496 return err; 1497 } 1498 1499 void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq) 1500 { 1501 struct mlx5e_channel *c = sq->channel; 1502 1503 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1504 napi_synchronize(&c->napi); 1505 1506 mlx5e_destroy_sq(c->mdev, sq->sqn); 1507 mlx5e_free_xdpsq_descs(sq); 1508 mlx5e_free_xdpsq(sq); 1509 } 1510 1511 static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, 1512 struct mlx5e_cq_param *param, 1513 struct mlx5e_cq *cq) 1514 { 1515 struct mlx5_core_cq *mcq = &cq->mcq; 1516 int eqn_not_used; 1517 unsigned int irqn; 1518 int err; 1519 u32 i; 1520 1521 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn); 1522 if (err) 1523 return err; 1524 1525 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 1526 &cq->wq_ctrl); 1527 if (err) 1528 return err; 1529 1530 mcq->cqe_sz = 64; 1531 mcq->set_ci_db = cq->wq_ctrl.db.db; 1532 mcq->arm_db = cq->wq_ctrl.db.db + 1; 1533 *mcq->set_ci_db = 0; 1534 *mcq->arm_db = 0; 1535 mcq->vector = param->eq_ix; 1536 mcq->comp = mlx5e_completion_event; 1537 mcq->event = mlx5e_cq_error_event; 1538 mcq->irqn = irqn; 1539 1540 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 1541 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 1542 1543 cqe->op_own = 0xf1; 1544 } 1545 1546 cq->mdev = mdev; 1547 1548 return 0; 1549 } 1550 1551 static int mlx5e_alloc_cq(struct mlx5e_channel *c, 1552 struct mlx5e_cq_param *param, 1553 struct mlx5e_cq *cq) 1554 { 1555 struct mlx5_core_dev *mdev = c->priv->mdev; 1556 int err; 1557 1558 param->wq.buf_numa_node = cpu_to_node(c->cpu); 1559 param->wq.db_numa_node = cpu_to_node(c->cpu); 1560 param->eq_ix = c->ix; 1561 1562 err = mlx5e_alloc_cq_common(mdev, param, cq); 1563 1564 cq->napi = &c->napi; 1565 cq->channel = c; 1566 1567 return err; 1568 } 1569 1570 static void mlx5e_free_cq(struct mlx5e_cq *cq) 1571 { 1572 mlx5_wq_destroy(&cq->wq_ctrl); 1573 } 1574 1575 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 1576 { 1577 u32 out[MLX5_ST_SZ_DW(create_cq_out)]; 1578 struct mlx5_core_dev *mdev = cq->mdev; 1579 struct mlx5_core_cq *mcq = &cq->mcq; 1580 1581 void *in; 1582 void *cqc; 1583 int inlen; 1584 unsigned int irqn_not_used; 1585 int eqn; 1586 int err; 1587 1588 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used); 1589 if (err) 1590 return err; 1591 1592 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 1593 sizeof(u64) * cq->wq_ctrl.buf.npages; 1594 in = kvzalloc(inlen, GFP_KERNEL); 1595 if (!in) 1596 return -ENOMEM; 1597 1598 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 1599 1600 memcpy(cqc, param->cqc, sizeof(param->cqc)); 1601 1602 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 1603 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 1604 1605 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode); 1606 MLX5_SET(cqc, cqc, c_eqn, eqn); 1607 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); 1608 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 1609 MLX5_ADAPTER_PAGE_SHIFT); 1610 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 1611 1612 err = mlx5_core_create_cq(mdev, mcq, in, inlen, out, sizeof(out)); 1613 1614 kvfree(in); 1615 1616 if (err) 1617 return err; 1618 1619 mlx5e_cq_arm(cq); 1620 1621 return 0; 1622 } 1623 1624 static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 1625 { 1626 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 1627 } 1628 1629 int mlx5e_open_cq(struct mlx5e_channel *c, struct dim_cq_moder moder, 1630 struct mlx5e_cq_param *param, struct mlx5e_cq *cq) 1631 { 1632 struct mlx5_core_dev *mdev = c->mdev; 1633 int err; 1634 1635 err = mlx5e_alloc_cq(c, param, cq); 1636 if (err) 1637 return err; 1638 1639 err = mlx5e_create_cq(cq, param); 1640 if (err) 1641 goto err_free_cq; 1642 1643 if (MLX5_CAP_GEN(mdev, cq_moderation)) 1644 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts); 1645 return 0; 1646 1647 err_free_cq: 1648 mlx5e_free_cq(cq); 1649 1650 return err; 1651 } 1652 1653 void mlx5e_close_cq(struct mlx5e_cq *cq) 1654 { 1655 mlx5e_destroy_cq(cq); 1656 mlx5e_free_cq(cq); 1657 } 1658 1659 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 1660 struct mlx5e_params *params, 1661 struct mlx5e_channel_param *cparam) 1662 { 1663 int err; 1664 int tc; 1665 1666 for (tc = 0; tc < c->num_tc; tc++) { 1667 err = mlx5e_open_cq(c, params->tx_cq_moderation, 1668 &cparam->tx_cq, &c->sq[tc].cq); 1669 if (err) 1670 goto err_close_tx_cqs; 1671 } 1672 1673 return 0; 1674 1675 err_close_tx_cqs: 1676 for (tc--; tc >= 0; tc--) 1677 mlx5e_close_cq(&c->sq[tc].cq); 1678 1679 return err; 1680 } 1681 1682 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 1683 { 1684 int tc; 1685 1686 for (tc = 0; tc < c->num_tc; tc++) 1687 mlx5e_close_cq(&c->sq[tc].cq); 1688 } 1689 1690 static int mlx5e_open_sqs(struct mlx5e_channel *c, 1691 struct mlx5e_params *params, 1692 struct mlx5e_channel_param *cparam) 1693 { 1694 int err, tc; 1695 1696 for (tc = 0; tc < params->num_tc; tc++) { 1697 int txq_ix = c->ix + tc * params->num_channels; 1698 1699 err = mlx5e_open_txqsq(c, c->priv->tisn[c->lag_port][tc], txq_ix, 1700 params, &cparam->sq, &c->sq[tc], tc); 1701 if (err) 1702 goto err_close_sqs; 1703 } 1704 1705 return 0; 1706 1707 err_close_sqs: 1708 for (tc--; tc >= 0; tc--) 1709 mlx5e_close_txqsq(&c->sq[tc]); 1710 1711 return err; 1712 } 1713 1714 static void mlx5e_close_sqs(struct mlx5e_channel *c) 1715 { 1716 int tc; 1717 1718 for (tc = 0; tc < c->num_tc; tc++) 1719 mlx5e_close_txqsq(&c->sq[tc]); 1720 } 1721 1722 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1723 struct mlx5e_txqsq *sq, u32 rate) 1724 { 1725 struct mlx5e_priv *priv = netdev_priv(dev); 1726 struct mlx5_core_dev *mdev = priv->mdev; 1727 struct mlx5e_modify_sq_param msp = {0}; 1728 struct mlx5_rate_limit rl = {0}; 1729 u16 rl_index = 0; 1730 int err; 1731 1732 if (rate == sq->rate_limit) 1733 /* nothing to do */ 1734 return 0; 1735 1736 if (sq->rate_limit) { 1737 rl.rate = sq->rate_limit; 1738 /* remove current rl index to free space to next ones */ 1739 mlx5_rl_remove_rate(mdev, &rl); 1740 } 1741 1742 sq->rate_limit = 0; 1743 1744 if (rate) { 1745 rl.rate = rate; 1746 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 1747 if (err) { 1748 netdev_err(dev, "Failed configuring rate %u: %d\n", 1749 rate, err); 1750 return err; 1751 } 1752 } 1753 1754 msp.curr_state = MLX5_SQC_STATE_RDY; 1755 msp.next_state = MLX5_SQC_STATE_RDY; 1756 msp.rl_index = rl_index; 1757 msp.rl_update = true; 1758 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 1759 if (err) { 1760 netdev_err(dev, "Failed configuring rate %u: %d\n", 1761 rate, err); 1762 /* remove the rate from the table */ 1763 if (rate) 1764 mlx5_rl_remove_rate(mdev, &rl); 1765 return err; 1766 } 1767 1768 sq->rate_limit = rate; 1769 return 0; 1770 } 1771 1772 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 1773 { 1774 struct mlx5e_priv *priv = netdev_priv(dev); 1775 struct mlx5_core_dev *mdev = priv->mdev; 1776 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 1777 int err = 0; 1778 1779 if (!mlx5_rl_is_supported(mdev)) { 1780 netdev_err(dev, "Rate limiting is not supported on this device\n"); 1781 return -EINVAL; 1782 } 1783 1784 /* rate is given in Mb/sec, HW config is in Kb/sec */ 1785 rate = rate << 10; 1786 1787 /* Check whether rate in valid range, 0 is always valid */ 1788 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 1789 netdev_err(dev, "TX rate %u, is not in range\n", rate); 1790 return -ERANGE; 1791 } 1792 1793 mutex_lock(&priv->state_lock); 1794 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 1795 err = mlx5e_set_sq_maxrate(dev, sq, rate); 1796 if (!err) 1797 priv->tx_rates[index] = rate; 1798 mutex_unlock(&priv->state_lock); 1799 1800 return err; 1801 } 1802 1803 static int mlx5e_alloc_xps_cpumask(struct mlx5e_channel *c, 1804 struct mlx5e_params *params) 1805 { 1806 int num_comp_vectors = mlx5_comp_vectors_count(c->mdev); 1807 int irq; 1808 1809 if (!zalloc_cpumask_var(&c->xps_cpumask, GFP_KERNEL)) 1810 return -ENOMEM; 1811 1812 for (irq = c->ix; irq < num_comp_vectors; irq += params->num_channels) { 1813 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(c->mdev, irq)); 1814 1815 cpumask_set_cpu(cpu, c->xps_cpumask); 1816 } 1817 1818 return 0; 1819 } 1820 1821 static void mlx5e_free_xps_cpumask(struct mlx5e_channel *c) 1822 { 1823 free_cpumask_var(c->xps_cpumask); 1824 } 1825 1826 static int mlx5e_open_queues(struct mlx5e_channel *c, 1827 struct mlx5e_params *params, 1828 struct mlx5e_channel_param *cparam) 1829 { 1830 struct dim_cq_moder icocq_moder = {0, 0}; 1831 int err; 1832 1833 err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); 1834 if (err) 1835 return err; 1836 1837 err = mlx5e_open_tx_cqs(c, params, cparam); 1838 if (err) 1839 goto err_close_icosq_cq; 1840 1841 err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq); 1842 if (err) 1843 goto err_close_tx_cqs; 1844 1845 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq); 1846 if (err) 1847 goto err_close_xdp_tx_cqs; 1848 1849 /* XDP SQ CQ params are same as normal TXQ sq CQ params */ 1850 err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation, 1851 &cparam->tx_cq, &c->rq_xdpsq.cq) : 0; 1852 if (err) 1853 goto err_close_rx_cq; 1854 1855 napi_enable(&c->napi); 1856 1857 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq); 1858 if (err) 1859 goto err_disable_napi; 1860 1861 err = mlx5e_open_sqs(c, params, cparam); 1862 if (err) 1863 goto err_close_icosq; 1864 1865 if (c->xdp) { 1866 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, 1867 &c->rq_xdpsq, false); 1868 if (err) 1869 goto err_close_sqs; 1870 } 1871 1872 err = mlx5e_open_rq(c, params, &cparam->rq, NULL, NULL, &c->rq); 1873 if (err) 1874 goto err_close_xdp_sq; 1875 1876 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, NULL, &c->xdpsq, true); 1877 if (err) 1878 goto err_close_rq; 1879 1880 return 0; 1881 1882 err_close_rq: 1883 mlx5e_close_rq(&c->rq); 1884 1885 err_close_xdp_sq: 1886 if (c->xdp) 1887 mlx5e_close_xdpsq(&c->rq_xdpsq); 1888 1889 err_close_sqs: 1890 mlx5e_close_sqs(c); 1891 1892 err_close_icosq: 1893 mlx5e_close_icosq(&c->icosq); 1894 1895 err_disable_napi: 1896 napi_disable(&c->napi); 1897 1898 if (c->xdp) 1899 mlx5e_close_cq(&c->rq_xdpsq.cq); 1900 1901 err_close_rx_cq: 1902 mlx5e_close_cq(&c->rq.cq); 1903 1904 err_close_xdp_tx_cqs: 1905 mlx5e_close_cq(&c->xdpsq.cq); 1906 1907 err_close_tx_cqs: 1908 mlx5e_close_tx_cqs(c); 1909 1910 err_close_icosq_cq: 1911 mlx5e_close_cq(&c->icosq.cq); 1912 1913 return err; 1914 } 1915 1916 static void mlx5e_close_queues(struct mlx5e_channel *c) 1917 { 1918 mlx5e_close_xdpsq(&c->xdpsq); 1919 mlx5e_close_rq(&c->rq); 1920 if (c->xdp) 1921 mlx5e_close_xdpsq(&c->rq_xdpsq); 1922 mlx5e_close_sqs(c); 1923 mlx5e_close_icosq(&c->icosq); 1924 napi_disable(&c->napi); 1925 if (c->xdp) 1926 mlx5e_close_cq(&c->rq_xdpsq.cq); 1927 mlx5e_close_cq(&c->rq.cq); 1928 mlx5e_close_cq(&c->xdpsq.cq); 1929 mlx5e_close_tx_cqs(c); 1930 mlx5e_close_cq(&c->icosq.cq); 1931 } 1932 1933 static u8 mlx5e_enumerate_lag_port(struct mlx5_core_dev *mdev, int ix) 1934 { 1935 u16 port_aff_bias = mlx5_core_is_pf(mdev) ? 0 : MLX5_CAP_GEN(mdev, vhca_id); 1936 1937 return (ix + port_aff_bias) % mlx5e_get_num_lag_ports(mdev); 1938 } 1939 1940 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 1941 struct mlx5e_params *params, 1942 struct mlx5e_channel_param *cparam, 1943 struct xdp_umem *umem, 1944 struct mlx5e_channel **cp) 1945 { 1946 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix)); 1947 struct net_device *netdev = priv->netdev; 1948 struct mlx5e_xsk_param xsk; 1949 struct mlx5e_channel *c; 1950 unsigned int irq; 1951 int err; 1952 int eqn; 1953 1954 err = mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq); 1955 if (err) 1956 return err; 1957 1958 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 1959 if (!c) 1960 return -ENOMEM; 1961 1962 c->priv = priv; 1963 c->mdev = priv->mdev; 1964 c->tstamp = &priv->tstamp; 1965 c->ix = ix; 1966 c->cpu = cpu; 1967 c->pdev = priv->mdev->device; 1968 c->netdev = priv->netdev; 1969 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key); 1970 c->num_tc = params->num_tc; 1971 c->xdp = !!params->xdp_prog; 1972 c->stats = &priv->channel_stats[ix].ch; 1973 c->irq_desc = irq_to_desc(irq); 1974 c->lag_port = mlx5e_enumerate_lag_port(priv->mdev, ix); 1975 1976 err = mlx5e_alloc_xps_cpumask(c, params); 1977 if (err) 1978 goto err_free_channel; 1979 1980 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64); 1981 1982 err = mlx5e_open_queues(c, params, cparam); 1983 if (unlikely(err)) 1984 goto err_napi_del; 1985 1986 if (umem) { 1987 mlx5e_build_xsk_param(umem, &xsk); 1988 err = mlx5e_open_xsk(priv, params, &xsk, umem, c); 1989 if (unlikely(err)) 1990 goto err_close_queues; 1991 } 1992 1993 *cp = c; 1994 1995 return 0; 1996 1997 err_close_queues: 1998 mlx5e_close_queues(c); 1999 2000 err_napi_del: 2001 netif_napi_del(&c->napi); 2002 mlx5e_free_xps_cpumask(c); 2003 2004 err_free_channel: 2005 kvfree(c); 2006 2007 return err; 2008 } 2009 2010 static void mlx5e_activate_channel(struct mlx5e_channel *c) 2011 { 2012 int tc; 2013 2014 for (tc = 0; tc < c->num_tc; tc++) 2015 mlx5e_activate_txqsq(&c->sq[tc]); 2016 mlx5e_activate_icosq(&c->icosq); 2017 mlx5e_activate_rq(&c->rq); 2018 netif_set_xps_queue(c->netdev, c->xps_cpumask, c->ix); 2019 2020 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2021 mlx5e_activate_xsk(c); 2022 } 2023 2024 static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 2025 { 2026 int tc; 2027 2028 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2029 mlx5e_deactivate_xsk(c); 2030 2031 mlx5e_deactivate_rq(&c->rq); 2032 mlx5e_deactivate_icosq(&c->icosq); 2033 for (tc = 0; tc < c->num_tc; tc++) 2034 mlx5e_deactivate_txqsq(&c->sq[tc]); 2035 } 2036 2037 static void mlx5e_close_channel(struct mlx5e_channel *c) 2038 { 2039 if (test_bit(MLX5E_CHANNEL_STATE_XSK, c->state)) 2040 mlx5e_close_xsk(c); 2041 mlx5e_close_queues(c); 2042 netif_napi_del(&c->napi); 2043 mlx5e_free_xps_cpumask(c); 2044 2045 kvfree(c); 2046 } 2047 2048 #define DEFAULT_FRAG_SIZE (2048) 2049 2050 static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, 2051 struct mlx5e_params *params, 2052 struct mlx5e_xsk_param *xsk, 2053 struct mlx5e_rq_frags_info *info) 2054 { 2055 u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu); 2056 int frag_size_max = DEFAULT_FRAG_SIZE; 2057 u32 buf_size = 0; 2058 int i; 2059 2060 #ifdef CONFIG_MLX5_EN_IPSEC 2061 if (MLX5_IPSEC_DEV(mdev)) 2062 byte_count += MLX5E_METADATA_ETHER_LEN; 2063 #endif 2064 2065 if (mlx5e_rx_is_linear_skb(params, xsk)) { 2066 int frag_stride; 2067 2068 frag_stride = mlx5e_rx_get_linear_frag_sz(params, xsk); 2069 frag_stride = roundup_pow_of_two(frag_stride); 2070 2071 info->arr[0].frag_size = byte_count; 2072 info->arr[0].frag_stride = frag_stride; 2073 info->num_frags = 1; 2074 info->wqe_bulk = PAGE_SIZE / frag_stride; 2075 goto out; 2076 } 2077 2078 if (byte_count > PAGE_SIZE + 2079 (MLX5E_MAX_RX_FRAGS - 1) * frag_size_max) 2080 frag_size_max = PAGE_SIZE; 2081 2082 i = 0; 2083 while (buf_size < byte_count) { 2084 int frag_size = byte_count - buf_size; 2085 2086 if (i < MLX5E_MAX_RX_FRAGS - 1) 2087 frag_size = min(frag_size, frag_size_max); 2088 2089 info->arr[i].frag_size = frag_size; 2090 info->arr[i].frag_stride = roundup_pow_of_two(frag_size); 2091 2092 buf_size += frag_size; 2093 i++; 2094 } 2095 info->num_frags = i; 2096 /* number of different wqes sharing a page */ 2097 info->wqe_bulk = 1 + (info->num_frags % 2); 2098 2099 out: 2100 info->wqe_bulk = max_t(u8, info->wqe_bulk, 8); 2101 info->log_num_frags = order_base_2(info->num_frags); 2102 } 2103 2104 static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs) 2105 { 2106 int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs; 2107 2108 switch (wq_type) { 2109 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2110 sz += sizeof(struct mlx5e_rx_wqe_ll); 2111 break; 2112 default: /* MLX5_WQ_TYPE_CYCLIC */ 2113 sz += sizeof(struct mlx5e_rx_wqe_cyc); 2114 } 2115 2116 return order_base_2(sz); 2117 } 2118 2119 static u8 mlx5e_get_rq_log_wq_sz(void *rqc) 2120 { 2121 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2122 2123 return MLX5_GET(wq, wq, log_wq_sz); 2124 } 2125 2126 void mlx5e_build_rq_param(struct mlx5e_priv *priv, 2127 struct mlx5e_params *params, 2128 struct mlx5e_xsk_param *xsk, 2129 struct mlx5e_rq_param *param) 2130 { 2131 struct mlx5_core_dev *mdev = priv->mdev; 2132 void *rqc = param->rqc; 2133 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2134 int ndsegs = 1; 2135 2136 switch (params->rq_wq_type) { 2137 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2138 MLX5_SET(wq, wq, log_wqe_num_of_strides, 2139 mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk) - 2140 MLX5_MPWQE_LOG_NUM_STRIDES_BASE); 2141 MLX5_SET(wq, wq, log_wqe_stride_size, 2142 mlx5e_mpwqe_get_log_stride_size(mdev, params, xsk) - 2143 MLX5_MPWQE_LOG_STRIDE_SZ_BASE); 2144 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params, xsk)); 2145 break; 2146 default: /* MLX5_WQ_TYPE_CYCLIC */ 2147 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames); 2148 mlx5e_build_rq_frags_info(mdev, params, xsk, ¶m->frags_info); 2149 ndsegs = param->frags_info.num_frags; 2150 } 2151 2152 MLX5_SET(wq, wq, wq_type, params->rq_wq_type); 2153 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN); 2154 MLX5_SET(wq, wq, log_wq_stride, 2155 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs)); 2156 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.pdn); 2157 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter); 2158 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); 2159 MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); 2160 2161 param->wq.buf_numa_node = dev_to_node(mdev->device); 2162 } 2163 2164 static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, 2165 struct mlx5e_rq_param *param) 2166 { 2167 struct mlx5_core_dev *mdev = priv->mdev; 2168 void *rqc = param->rqc; 2169 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2170 2171 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 2172 MLX5_SET(wq, wq, log_wq_stride, 2173 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); 2174 MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter); 2175 2176 param->wq.buf_numa_node = dev_to_node(mdev->device); 2177 } 2178 2179 void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, 2180 struct mlx5e_sq_param *param) 2181 { 2182 void *sqc = param->sqc; 2183 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2184 2185 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); 2186 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn); 2187 2188 param->wq.buf_numa_node = dev_to_node(priv->mdev->device); 2189 } 2190 2191 static void mlx5e_build_sq_param(struct mlx5e_priv *priv, 2192 struct mlx5e_params *params, 2193 struct mlx5e_sq_param *param) 2194 { 2195 void *sqc = param->sqc; 2196 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2197 bool allow_swp; 2198 2199 allow_swp = mlx5_geneve_tx_allowed(priv->mdev) || 2200 !!MLX5_IPSEC_DEV(priv->mdev); 2201 mlx5e_build_sq_param_common(priv, param); 2202 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 2203 MLX5_SET(sqc, sqc, allow_swp, allow_swp); 2204 } 2205 2206 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, 2207 struct mlx5e_cq_param *param) 2208 { 2209 void *cqc = param->cqc; 2210 2211 MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index); 2212 if (MLX5_CAP_GEN(priv->mdev, cqe_128_always) && cache_line_size() >= 128) 2213 MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD); 2214 } 2215 2216 void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, 2217 struct mlx5e_params *params, 2218 struct mlx5e_xsk_param *xsk, 2219 struct mlx5e_cq_param *param) 2220 { 2221 struct mlx5_core_dev *mdev = priv->mdev; 2222 void *cqc = param->cqc; 2223 u8 log_cq_size; 2224 2225 switch (params->rq_wq_type) { 2226 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2227 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params, xsk) + 2228 mlx5e_mpwqe_get_log_num_strides(mdev, params, xsk); 2229 break; 2230 default: /* MLX5_WQ_TYPE_CYCLIC */ 2231 log_cq_size = params->log_rq_mtu_frames; 2232 } 2233 2234 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size); 2235 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 2236 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM); 2237 MLX5_SET(cqc, cqc, cqe_comp_en, 1); 2238 } 2239 2240 mlx5e_build_common_cq_param(priv, param); 2241 param->cq_period_mode = params->rx_cq_moderation.cq_period_mode; 2242 } 2243 2244 void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv, 2245 struct mlx5e_params *params, 2246 struct mlx5e_cq_param *param) 2247 { 2248 void *cqc = param->cqc; 2249 2250 MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size); 2251 2252 mlx5e_build_common_cq_param(priv, param); 2253 param->cq_period_mode = params->tx_cq_moderation.cq_period_mode; 2254 } 2255 2256 void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv, 2257 u8 log_wq_size, 2258 struct mlx5e_cq_param *param) 2259 { 2260 void *cqc = param->cqc; 2261 2262 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size); 2263 2264 mlx5e_build_common_cq_param(priv, param); 2265 2266 param->cq_period_mode = DIM_CQ_PERIOD_MODE_START_FROM_EQE; 2267 } 2268 2269 void mlx5e_build_icosq_param(struct mlx5e_priv *priv, 2270 u8 log_wq_size, 2271 struct mlx5e_sq_param *param) 2272 { 2273 void *sqc = param->sqc; 2274 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2275 2276 mlx5e_build_sq_param_common(priv, param); 2277 2278 MLX5_SET(wq, wq, log_wq_sz, log_wq_size); 2279 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq)); 2280 } 2281 2282 void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, 2283 struct mlx5e_params *params, 2284 struct mlx5e_sq_param *param) 2285 { 2286 void *sqc = param->sqc; 2287 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2288 2289 mlx5e_build_sq_param_common(priv, param); 2290 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 2291 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); 2292 } 2293 2294 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, 2295 struct mlx5e_rq_param *rqp) 2296 { 2297 switch (params->rq_wq_type) { 2298 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2299 return order_base_2(MLX5E_UMR_WQEBBS) + 2300 mlx5e_get_rq_log_wq_sz(rqp->rqc); 2301 default: /* MLX5_WQ_TYPE_CYCLIC */ 2302 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; 2303 } 2304 } 2305 2306 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, 2307 struct mlx5e_params *params, 2308 struct mlx5e_channel_param *cparam) 2309 { 2310 u8 icosq_log_wq_sz; 2311 2312 mlx5e_build_rq_param(priv, params, NULL, &cparam->rq); 2313 2314 icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq); 2315 2316 mlx5e_build_sq_param(priv, params, &cparam->sq); 2317 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); 2318 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq); 2319 mlx5e_build_rx_cq_param(priv, params, NULL, &cparam->rx_cq); 2320 mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); 2321 mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq); 2322 } 2323 2324 int mlx5e_open_channels(struct mlx5e_priv *priv, 2325 struct mlx5e_channels *chs) 2326 { 2327 struct mlx5e_channel_param *cparam; 2328 int err = -ENOMEM; 2329 int i; 2330 2331 chs->num = chs->params.num_channels; 2332 2333 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL); 2334 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL); 2335 if (!chs->c || !cparam) 2336 goto err_free; 2337 2338 mlx5e_build_channel_param(priv, &chs->params, cparam); 2339 for (i = 0; i < chs->num; i++) { 2340 struct xdp_umem *umem = NULL; 2341 2342 if (chs->params.xdp_prog) 2343 umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, i); 2344 2345 err = mlx5e_open_channel(priv, i, &chs->params, cparam, umem, &chs->c[i]); 2346 if (err) 2347 goto err_close_channels; 2348 } 2349 2350 mlx5e_health_channels_update(priv); 2351 kvfree(cparam); 2352 return 0; 2353 2354 err_close_channels: 2355 for (i--; i >= 0; i--) 2356 mlx5e_close_channel(chs->c[i]); 2357 2358 err_free: 2359 kfree(chs->c); 2360 kvfree(cparam); 2361 chs->num = 0; 2362 return err; 2363 } 2364 2365 static void mlx5e_activate_channels(struct mlx5e_channels *chs) 2366 { 2367 int i; 2368 2369 for (i = 0; i < chs->num; i++) 2370 mlx5e_activate_channel(chs->c[i]); 2371 } 2372 2373 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ 2374 2375 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 2376 { 2377 int err = 0; 2378 int i; 2379 2380 for (i = 0; i < chs->num; i++) { 2381 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 2382 2383 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout); 2384 2385 /* Don't wait on the XSK RQ, because the newer xdpsock sample 2386 * doesn't provide any Fill Ring entries at the setup stage. 2387 */ 2388 } 2389 2390 return err ? -ETIMEDOUT : 0; 2391 } 2392 2393 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 2394 { 2395 int i; 2396 2397 for (i = 0; i < chs->num; i++) 2398 mlx5e_deactivate_channel(chs->c[i]); 2399 } 2400 2401 void mlx5e_close_channels(struct mlx5e_channels *chs) 2402 { 2403 int i; 2404 2405 for (i = 0; i < chs->num; i++) 2406 mlx5e_close_channel(chs->c[i]); 2407 2408 kfree(chs->c); 2409 chs->num = 0; 2410 } 2411 2412 static int 2413 mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt) 2414 { 2415 struct mlx5_core_dev *mdev = priv->mdev; 2416 void *rqtc; 2417 int inlen; 2418 int err; 2419 u32 *in; 2420 int i; 2421 2422 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; 2423 in = kvzalloc(inlen, GFP_KERNEL); 2424 if (!in) 2425 return -ENOMEM; 2426 2427 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 2428 2429 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 2430 MLX5_SET(rqtc, rqtc, rqt_max_size, sz); 2431 2432 for (i = 0; i < sz; i++) 2433 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn); 2434 2435 err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn); 2436 if (!err) 2437 rqt->enabled = true; 2438 2439 kvfree(in); 2440 return err; 2441 } 2442 2443 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt) 2444 { 2445 rqt->enabled = false; 2446 mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn); 2447 } 2448 2449 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv) 2450 { 2451 struct mlx5e_rqt *rqt = &priv->indir_rqt; 2452 int err; 2453 2454 err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt); 2455 if (err) 2456 mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err); 2457 return err; 2458 } 2459 2460 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 2461 { 2462 int err; 2463 int ix; 2464 2465 for (ix = 0; ix < priv->max_nch; ix++) { 2466 err = mlx5e_create_rqt(priv, 1 /*size */, &tirs[ix].rqt); 2467 if (unlikely(err)) 2468 goto err_destroy_rqts; 2469 } 2470 2471 return 0; 2472 2473 err_destroy_rqts: 2474 mlx5_core_warn(priv->mdev, "create rqts failed, %d\n", err); 2475 for (ix--; ix >= 0; ix--) 2476 mlx5e_destroy_rqt(priv, &tirs[ix].rqt); 2477 2478 return err; 2479 } 2480 2481 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 2482 { 2483 int i; 2484 2485 for (i = 0; i < priv->max_nch; i++) 2486 mlx5e_destroy_rqt(priv, &tirs[i].rqt); 2487 } 2488 2489 static int mlx5e_rx_hash_fn(int hfunc) 2490 { 2491 return (hfunc == ETH_RSS_HASH_TOP) ? 2492 MLX5_RX_HASH_FN_TOEPLITZ : 2493 MLX5_RX_HASH_FN_INVERTED_XOR8; 2494 } 2495 2496 int mlx5e_bits_invert(unsigned long a, int size) 2497 { 2498 int inv = 0; 2499 int i; 2500 2501 for (i = 0; i < size; i++) 2502 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i; 2503 2504 return inv; 2505 } 2506 2507 static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz, 2508 struct mlx5e_redirect_rqt_param rrp, void *rqtc) 2509 { 2510 int i; 2511 2512 for (i = 0; i < sz; i++) { 2513 u32 rqn; 2514 2515 if (rrp.is_rss) { 2516 int ix = i; 2517 2518 if (rrp.rss.hfunc == ETH_RSS_HASH_XOR) 2519 ix = mlx5e_bits_invert(i, ilog2(sz)); 2520 2521 ix = priv->rss_params.indirection_rqt[ix]; 2522 rqn = rrp.rss.channels->c[ix]->rq.rqn; 2523 } else { 2524 rqn = rrp.rqn; 2525 } 2526 MLX5_SET(rqtc, rqtc, rq_num[i], rqn); 2527 } 2528 } 2529 2530 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, 2531 struct mlx5e_redirect_rqt_param rrp) 2532 { 2533 struct mlx5_core_dev *mdev = priv->mdev; 2534 void *rqtc; 2535 int inlen; 2536 u32 *in; 2537 int err; 2538 2539 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz; 2540 in = kvzalloc(inlen, GFP_KERNEL); 2541 if (!in) 2542 return -ENOMEM; 2543 2544 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 2545 2546 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 2547 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); 2548 mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc); 2549 err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen); 2550 2551 kvfree(in); 2552 return err; 2553 } 2554 2555 static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix, 2556 struct mlx5e_redirect_rqt_param rrp) 2557 { 2558 if (!rrp.is_rss) 2559 return rrp.rqn; 2560 2561 if (ix >= rrp.rss.channels->num) 2562 return priv->drop_rq.rqn; 2563 2564 return rrp.rss.channels->c[ix]->rq.rqn; 2565 } 2566 2567 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv, 2568 struct mlx5e_redirect_rqt_param rrp) 2569 { 2570 u32 rqtn; 2571 int ix; 2572 2573 if (priv->indir_rqt.enabled) { 2574 /* RSS RQ table */ 2575 rqtn = priv->indir_rqt.rqtn; 2576 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp); 2577 } 2578 2579 for (ix = 0; ix < priv->max_nch; ix++) { 2580 struct mlx5e_redirect_rqt_param direct_rrp = { 2581 .is_rss = false, 2582 { 2583 .rqn = mlx5e_get_direct_rqn(priv, ix, rrp) 2584 }, 2585 }; 2586 2587 /* Direct RQ Tables */ 2588 if (!priv->direct_tir[ix].rqt.enabled) 2589 continue; 2590 2591 rqtn = priv->direct_tir[ix].rqt.rqtn; 2592 mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp); 2593 } 2594 } 2595 2596 static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv, 2597 struct mlx5e_channels *chs) 2598 { 2599 struct mlx5e_redirect_rqt_param rrp = { 2600 .is_rss = true, 2601 { 2602 .rss = { 2603 .channels = chs, 2604 .hfunc = priv->rss_params.hfunc, 2605 } 2606 }, 2607 }; 2608 2609 mlx5e_redirect_rqts(priv, rrp); 2610 } 2611 2612 static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv) 2613 { 2614 struct mlx5e_redirect_rqt_param drop_rrp = { 2615 .is_rss = false, 2616 { 2617 .rqn = priv->drop_rq.rqn, 2618 }, 2619 }; 2620 2621 mlx5e_redirect_rqts(priv, drop_rrp); 2622 } 2623 2624 static const struct mlx5e_tirc_config tirc_default_config[MLX5E_NUM_INDIR_TIRS] = { 2625 [MLX5E_TT_IPV4_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2626 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 2627 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2628 }, 2629 [MLX5E_TT_IPV6_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2630 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 2631 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2632 }, 2633 [MLX5E_TT_IPV4_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2634 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 2635 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2636 }, 2637 [MLX5E_TT_IPV6_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2638 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 2639 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2640 }, 2641 [MLX5E_TT_IPV4_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2642 .l4_prot_type = 0, 2643 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2644 }, 2645 [MLX5E_TT_IPV6_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2646 .l4_prot_type = 0, 2647 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2648 }, 2649 [MLX5E_TT_IPV4_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2650 .l4_prot_type = 0, 2651 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2652 }, 2653 [MLX5E_TT_IPV6_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2654 .l4_prot_type = 0, 2655 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2656 }, 2657 [MLX5E_TT_IPV4] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2658 .l4_prot_type = 0, 2659 .rx_hash_fields = MLX5_HASH_IP, 2660 }, 2661 [MLX5E_TT_IPV6] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2662 .l4_prot_type = 0, 2663 .rx_hash_fields = MLX5_HASH_IP, 2664 }, 2665 }; 2666 2667 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt) 2668 { 2669 return tirc_default_config[tt]; 2670 } 2671 2672 static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc) 2673 { 2674 if (!params->lro_en) 2675 return; 2676 2677 #define ROUGH_MAX_L2_L3_HDR_SZ 256 2678 2679 MLX5_SET(tirc, tirc, lro_enable_mask, 2680 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 2681 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO); 2682 MLX5_SET(tirc, tirc, lro_max_ip_payload_size, 2683 (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - ROUGH_MAX_L2_L3_HDR_SZ) >> 8); 2684 MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout); 2685 } 2686 2687 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params, 2688 const struct mlx5e_tirc_config *ttconfig, 2689 void *tirc, bool inner) 2690 { 2691 void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) : 2692 MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 2693 2694 MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(rss_params->hfunc)); 2695 if (rss_params->hfunc == ETH_RSS_HASH_TOP) { 2696 void *rss_key = MLX5_ADDR_OF(tirc, tirc, 2697 rx_hash_toeplitz_key); 2698 size_t len = MLX5_FLD_SZ_BYTES(tirc, 2699 rx_hash_toeplitz_key); 2700 2701 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 2702 memcpy(rss_key, rss_params->toeplitz_hash_key, len); 2703 } 2704 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, 2705 ttconfig->l3_prot_type); 2706 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, 2707 ttconfig->l4_prot_type); 2708 MLX5_SET(rx_hash_field_select, hfso, selected_fields, 2709 ttconfig->rx_hash_fields); 2710 } 2711 2712 static void mlx5e_update_rx_hash_fields(struct mlx5e_tirc_config *ttconfig, 2713 enum mlx5e_traffic_types tt, 2714 u32 rx_hash_fields) 2715 { 2716 *ttconfig = tirc_default_config[tt]; 2717 ttconfig->rx_hash_fields = rx_hash_fields; 2718 } 2719 2720 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in, int inlen) 2721 { 2722 void *tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx); 2723 struct mlx5e_rss_params *rss = &priv->rss_params; 2724 struct mlx5_core_dev *mdev = priv->mdev; 2725 int ctxlen = MLX5_ST_SZ_BYTES(tirc); 2726 struct mlx5e_tirc_config ttconfig; 2727 int tt; 2728 2729 MLX5_SET(modify_tir_in, in, bitmask.hash, 1); 2730 2731 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2732 memset(tirc, 0, ctxlen); 2733 mlx5e_update_rx_hash_fields(&ttconfig, tt, 2734 rss->rx_hash_fields[tt]); 2735 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, false); 2736 mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in, inlen); 2737 } 2738 2739 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev)) 2740 return; 2741 2742 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2743 memset(tirc, 0, ctxlen); 2744 mlx5e_update_rx_hash_fields(&ttconfig, tt, 2745 rss->rx_hash_fields[tt]); 2746 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, true); 2747 mlx5_core_modify_tir(mdev, priv->inner_indir_tir[tt].tirn, in, 2748 inlen); 2749 } 2750 } 2751 2752 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv) 2753 { 2754 struct mlx5_core_dev *mdev = priv->mdev; 2755 2756 void *in; 2757 void *tirc; 2758 int inlen; 2759 int err; 2760 int tt; 2761 int ix; 2762 2763 inlen = MLX5_ST_SZ_BYTES(modify_tir_in); 2764 in = kvzalloc(inlen, GFP_KERNEL); 2765 if (!in) 2766 return -ENOMEM; 2767 2768 MLX5_SET(modify_tir_in, in, bitmask.lro, 1); 2769 tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx); 2770 2771 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); 2772 2773 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2774 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in, 2775 inlen); 2776 if (err) 2777 goto free_in; 2778 } 2779 2780 for (ix = 0; ix < priv->max_nch; ix++) { 2781 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn, 2782 in, inlen); 2783 if (err) 2784 goto free_in; 2785 } 2786 2787 free_in: 2788 kvfree(in); 2789 2790 return err; 2791 } 2792 2793 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 2794 struct mlx5e_params *params, u16 mtu) 2795 { 2796 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 2797 int err; 2798 2799 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 2800 if (err) 2801 return err; 2802 2803 /* Update vport context MTU */ 2804 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 2805 return 0; 2806 } 2807 2808 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 2809 struct mlx5e_params *params, u16 *mtu) 2810 { 2811 u16 hw_mtu = 0; 2812 int err; 2813 2814 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2815 if (err || !hw_mtu) /* fallback to port oper mtu */ 2816 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 2817 2818 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 2819 } 2820 2821 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 2822 { 2823 struct mlx5e_params *params = &priv->channels.params; 2824 struct net_device *netdev = priv->netdev; 2825 struct mlx5_core_dev *mdev = priv->mdev; 2826 u16 mtu; 2827 int err; 2828 2829 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 2830 if (err) 2831 return err; 2832 2833 mlx5e_query_mtu(mdev, params, &mtu); 2834 if (mtu != params->sw_mtu) 2835 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 2836 __func__, mtu, params->sw_mtu); 2837 2838 params->sw_mtu = mtu; 2839 return 0; 2840 } 2841 2842 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 2843 { 2844 struct mlx5e_params *params = &priv->channels.params; 2845 struct net_device *netdev = priv->netdev; 2846 struct mlx5_core_dev *mdev = priv->mdev; 2847 u16 max_mtu; 2848 2849 /* MTU range: 68 - hw-specific max */ 2850 netdev->min_mtu = ETH_MIN_MTU; 2851 2852 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 2853 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 2854 ETH_MAX_MTU); 2855 } 2856 2857 static void mlx5e_netdev_set_tcs(struct net_device *netdev) 2858 { 2859 struct mlx5e_priv *priv = netdev_priv(netdev); 2860 int nch = priv->channels.params.num_channels; 2861 int ntc = priv->channels.params.num_tc; 2862 int tc; 2863 2864 netdev_reset_tc(netdev); 2865 2866 if (ntc == 1) 2867 return; 2868 2869 netdev_set_num_tc(netdev, ntc); 2870 2871 /* Map netdev TCs to offset 0 2872 * We have our own UP to TXQ mapping for QoS 2873 */ 2874 for (tc = 0; tc < ntc; tc++) 2875 netdev_set_tc_queue(netdev, tc, nch, 0); 2876 } 2877 2878 static void mlx5e_build_txq_maps(struct mlx5e_priv *priv) 2879 { 2880 int i, ch; 2881 2882 ch = priv->channels.num; 2883 2884 for (i = 0; i < ch; i++) { 2885 int tc; 2886 2887 for (tc = 0; tc < priv->channels.params.num_tc; tc++) { 2888 struct mlx5e_channel *c = priv->channels.c[i]; 2889 struct mlx5e_txqsq *sq = &c->sq[tc]; 2890 2891 priv->txq2sq[sq->txq_ix] = sq; 2892 priv->channel_tc2realtxq[i][tc] = i + tc * ch; 2893 } 2894 } 2895 } 2896 2897 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 2898 { 2899 int num_txqs = priv->channels.num * priv->channels.params.num_tc; 2900 int num_rxqs = priv->channels.num * priv->profile->rq_groups; 2901 struct net_device *netdev = priv->netdev; 2902 2903 mlx5e_netdev_set_tcs(netdev); 2904 netif_set_real_num_tx_queues(netdev, num_txqs); 2905 netif_set_real_num_rx_queues(netdev, num_rxqs); 2906 2907 mlx5e_build_txq_maps(priv); 2908 mlx5e_activate_channels(&priv->channels); 2909 mlx5e_xdp_tx_enable(priv); 2910 netif_tx_start_all_queues(priv->netdev); 2911 2912 if (mlx5e_is_vport_rep(priv)) 2913 mlx5e_add_sqs_fwd_rules(priv); 2914 2915 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 2916 mlx5e_redirect_rqts_to_channels(priv, &priv->channels); 2917 2918 mlx5e_xsk_redirect_rqts_to_channels(priv, &priv->channels); 2919 } 2920 2921 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 2922 { 2923 mlx5e_xsk_redirect_rqts_to_drop(priv, &priv->channels); 2924 2925 mlx5e_redirect_rqts_to_drop(priv); 2926 2927 if (mlx5e_is_vport_rep(priv)) 2928 mlx5e_remove_sqs_fwd_rules(priv); 2929 2930 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when 2931 * polling for inactive tx queues. 2932 */ 2933 netif_tx_stop_all_queues(priv->netdev); 2934 netif_tx_disable(priv->netdev); 2935 mlx5e_xdp_tx_disable(priv); 2936 mlx5e_deactivate_channels(&priv->channels); 2937 } 2938 2939 static void mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 2940 struct mlx5e_channels *new_chs, 2941 mlx5e_fp_hw_modify hw_modify) 2942 { 2943 struct net_device *netdev = priv->netdev; 2944 int new_num_txqs; 2945 int carrier_ok; 2946 2947 new_num_txqs = new_chs->num * new_chs->params.num_tc; 2948 2949 carrier_ok = netif_carrier_ok(netdev); 2950 netif_carrier_off(netdev); 2951 2952 if (new_num_txqs < netdev->real_num_tx_queues) 2953 netif_set_real_num_tx_queues(netdev, new_num_txqs); 2954 2955 mlx5e_deactivate_priv_channels(priv); 2956 mlx5e_close_channels(&priv->channels); 2957 2958 priv->channels = *new_chs; 2959 2960 /* New channels are ready to roll, modify HW settings if needed */ 2961 if (hw_modify) 2962 hw_modify(priv); 2963 2964 priv->profile->update_rx(priv); 2965 mlx5e_activate_priv_channels(priv); 2966 2967 /* return carrier back if needed */ 2968 if (carrier_ok) 2969 netif_carrier_on(netdev); 2970 } 2971 2972 int mlx5e_safe_switch_channels(struct mlx5e_priv *priv, 2973 struct mlx5e_channels *new_chs, 2974 mlx5e_fp_hw_modify hw_modify) 2975 { 2976 int err; 2977 2978 err = mlx5e_open_channels(priv, new_chs); 2979 if (err) 2980 return err; 2981 2982 mlx5e_switch_priv_channels(priv, new_chs, hw_modify); 2983 return 0; 2984 } 2985 2986 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 2987 { 2988 struct mlx5e_channels new_channels = {}; 2989 2990 new_channels.params = priv->channels.params; 2991 return mlx5e_safe_switch_channels(priv, &new_channels, NULL); 2992 } 2993 2994 void mlx5e_timestamp_init(struct mlx5e_priv *priv) 2995 { 2996 priv->tstamp.tx_type = HWTSTAMP_TX_OFF; 2997 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE; 2998 } 2999 3000 int mlx5e_open_locked(struct net_device *netdev) 3001 { 3002 struct mlx5e_priv *priv = netdev_priv(netdev); 3003 int err; 3004 3005 set_bit(MLX5E_STATE_OPENED, &priv->state); 3006 3007 err = mlx5e_open_channels(priv, &priv->channels); 3008 if (err) 3009 goto err_clear_state_opened_flag; 3010 3011 priv->profile->update_rx(priv); 3012 mlx5e_activate_priv_channels(priv); 3013 if (priv->profile->update_carrier) 3014 priv->profile->update_carrier(priv); 3015 3016 mlx5e_queue_update_stats(priv); 3017 return 0; 3018 3019 err_clear_state_opened_flag: 3020 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3021 return err; 3022 } 3023 3024 int mlx5e_open(struct net_device *netdev) 3025 { 3026 struct mlx5e_priv *priv = netdev_priv(netdev); 3027 int err; 3028 3029 mutex_lock(&priv->state_lock); 3030 err = mlx5e_open_locked(netdev); 3031 if (!err) 3032 mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_UP); 3033 mutex_unlock(&priv->state_lock); 3034 3035 if (mlx5_vxlan_allowed(priv->mdev->vxlan)) 3036 udp_tunnel_get_rx_info(netdev); 3037 3038 return err; 3039 } 3040 3041 int mlx5e_close_locked(struct net_device *netdev) 3042 { 3043 struct mlx5e_priv *priv = netdev_priv(netdev); 3044 3045 /* May already be CLOSED in case a previous configuration operation 3046 * (e.g RX/TX queue size change) that involves close&open failed. 3047 */ 3048 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3049 return 0; 3050 3051 clear_bit(MLX5E_STATE_OPENED, &priv->state); 3052 3053 netif_carrier_off(priv->netdev); 3054 mlx5e_deactivate_priv_channels(priv); 3055 mlx5e_close_channels(&priv->channels); 3056 3057 return 0; 3058 } 3059 3060 int mlx5e_close(struct net_device *netdev) 3061 { 3062 struct mlx5e_priv *priv = netdev_priv(netdev); 3063 int err; 3064 3065 if (!netif_device_present(netdev)) 3066 return -ENODEV; 3067 3068 mutex_lock(&priv->state_lock); 3069 mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_DOWN); 3070 err = mlx5e_close_locked(netdev); 3071 mutex_unlock(&priv->state_lock); 3072 3073 return err; 3074 } 3075 3076 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 3077 struct mlx5e_rq *rq, 3078 struct mlx5e_rq_param *param) 3079 { 3080 void *rqc = param->rqc; 3081 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 3082 int err; 3083 3084 param->wq.db_numa_node = param->wq.buf_numa_node; 3085 3086 err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, 3087 &rq->wq_ctrl); 3088 if (err) 3089 return err; 3090 3091 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 3092 xdp_rxq_info_unused(&rq->xdp_rxq); 3093 3094 rq->mdev = mdev; 3095 3096 return 0; 3097 } 3098 3099 static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev, 3100 struct mlx5e_cq *cq, 3101 struct mlx5e_cq_param *param) 3102 { 3103 param->wq.buf_numa_node = dev_to_node(mdev->device); 3104 param->wq.db_numa_node = dev_to_node(mdev->device); 3105 3106 return mlx5e_alloc_cq_common(mdev, param, cq); 3107 } 3108 3109 int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 3110 struct mlx5e_rq *drop_rq) 3111 { 3112 struct mlx5_core_dev *mdev = priv->mdev; 3113 struct mlx5e_cq_param cq_param = {}; 3114 struct mlx5e_rq_param rq_param = {}; 3115 struct mlx5e_cq *cq = &drop_rq->cq; 3116 int err; 3117 3118 mlx5e_build_drop_rq_param(priv, &rq_param); 3119 3120 err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param); 3121 if (err) 3122 return err; 3123 3124 err = mlx5e_create_cq(cq, &cq_param); 3125 if (err) 3126 goto err_free_cq; 3127 3128 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3129 if (err) 3130 goto err_destroy_cq; 3131 3132 err = mlx5e_create_rq(drop_rq, &rq_param); 3133 if (err) 3134 goto err_free_rq; 3135 3136 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3137 if (err) 3138 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3139 3140 return 0; 3141 3142 err_free_rq: 3143 mlx5e_free_rq(drop_rq); 3144 3145 err_destroy_cq: 3146 mlx5e_destroy_cq(cq); 3147 3148 err_free_cq: 3149 mlx5e_free_cq(cq); 3150 3151 return err; 3152 } 3153 3154 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3155 { 3156 mlx5e_destroy_rq(drop_rq); 3157 mlx5e_free_rq(drop_rq); 3158 mlx5e_destroy_cq(&drop_rq->cq); 3159 mlx5e_free_cq(&drop_rq->cq); 3160 } 3161 3162 int mlx5e_create_tis(struct mlx5_core_dev *mdev, void *in, u32 *tisn) 3163 { 3164 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3165 3166 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn); 3167 3168 if (MLX5_GET(tisc, tisc, tls_en)) 3169 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.pdn); 3170 3171 if (mlx5_lag_is_lacp_owner(mdev)) 3172 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); 3173 3174 return mlx5_core_create_tis(mdev, in, MLX5_ST_SZ_BYTES(create_tis_in), tisn); 3175 } 3176 3177 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) 3178 { 3179 mlx5_core_destroy_tis(mdev, tisn); 3180 } 3181 3182 void mlx5e_destroy_tises(struct mlx5e_priv *priv) 3183 { 3184 int tc, i; 3185 3186 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) 3187 for (tc = 0; tc < priv->profile->max_tc; tc++) 3188 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); 3189 } 3190 3191 static bool mlx5e_lag_should_assign_affinity(struct mlx5_core_dev *mdev) 3192 { 3193 return MLX5_CAP_GEN(mdev, lag_tx_port_affinity) && mlx5e_get_num_lag_ports(mdev) > 1; 3194 } 3195 3196 int mlx5e_create_tises(struct mlx5e_priv *priv) 3197 { 3198 int tc, i; 3199 int err; 3200 3201 for (i = 0; i < mlx5e_get_num_lag_ports(priv->mdev); i++) { 3202 for (tc = 0; tc < priv->profile->max_tc; tc++) { 3203 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 3204 void *tisc; 3205 3206 tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3207 3208 MLX5_SET(tisc, tisc, prio, tc << 1); 3209 3210 if (mlx5e_lag_should_assign_affinity(priv->mdev)) 3211 MLX5_SET(tisc, tisc, lag_tx_port_affinity, i + 1); 3212 3213 err = mlx5e_create_tis(priv->mdev, in, &priv->tisn[i][tc]); 3214 if (err) 3215 goto err_close_tises; 3216 } 3217 } 3218 3219 return 0; 3220 3221 err_close_tises: 3222 for (; i >= 0; i--) { 3223 for (tc--; tc >= 0; tc--) 3224 mlx5e_destroy_tis(priv->mdev, priv->tisn[i][tc]); 3225 tc = priv->profile->max_tc; 3226 } 3227 3228 return err; 3229 } 3230 3231 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3232 { 3233 mlx5e_destroy_tises(priv); 3234 } 3235 3236 static void mlx5e_build_indir_tir_ctx_common(struct mlx5e_priv *priv, 3237 u32 rqtn, u32 *tirc) 3238 { 3239 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn); 3240 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 3241 MLX5_SET(tirc, tirc, indirect_table, rqtn); 3242 MLX5_SET(tirc, tirc, tunneled_offload_en, 3243 priv->channels.params.tunneled_offload_en); 3244 3245 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); 3246 } 3247 3248 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, 3249 enum mlx5e_traffic_types tt, 3250 u32 *tirc) 3251 { 3252 mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); 3253 mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, 3254 &tirc_default_config[tt], tirc, false); 3255 } 3256 3257 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc) 3258 { 3259 mlx5e_build_indir_tir_ctx_common(priv, rqtn, tirc); 3260 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); 3261 } 3262 3263 static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv, 3264 enum mlx5e_traffic_types tt, 3265 u32 *tirc) 3266 { 3267 mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); 3268 mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, 3269 &tirc_default_config[tt], tirc, true); 3270 } 3271 3272 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc) 3273 { 3274 struct mlx5e_tir *tir; 3275 void *tirc; 3276 int inlen; 3277 int i = 0; 3278 int err; 3279 u32 *in; 3280 int tt; 3281 3282 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 3283 in = kvzalloc(inlen, GFP_KERNEL); 3284 if (!in) 3285 return -ENOMEM; 3286 3287 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 3288 memset(in, 0, inlen); 3289 tir = &priv->indir_tir[tt]; 3290 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3291 mlx5e_build_indir_tir_ctx(priv, tt, tirc); 3292 err = mlx5e_create_tir(priv->mdev, tir, in, inlen); 3293 if (err) { 3294 mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err); 3295 goto err_destroy_inner_tirs; 3296 } 3297 } 3298 3299 if (!inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev)) 3300 goto out; 3301 3302 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) { 3303 memset(in, 0, inlen); 3304 tir = &priv->inner_indir_tir[i]; 3305 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3306 mlx5e_build_inner_indir_tir_ctx(priv, i, tirc); 3307 err = mlx5e_create_tir(priv->mdev, tir, in, inlen); 3308 if (err) { 3309 mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err); 3310 goto err_destroy_inner_tirs; 3311 } 3312 } 3313 3314 out: 3315 kvfree(in); 3316 3317 return 0; 3318 3319 err_destroy_inner_tirs: 3320 for (i--; i >= 0; i--) 3321 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]); 3322 3323 for (tt--; tt >= 0; tt--) 3324 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]); 3325 3326 kvfree(in); 3327 3328 return err; 3329 } 3330 3331 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 3332 { 3333 struct mlx5e_tir *tir; 3334 void *tirc; 3335 int inlen; 3336 int err = 0; 3337 u32 *in; 3338 int ix; 3339 3340 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 3341 in = kvzalloc(inlen, GFP_KERNEL); 3342 if (!in) 3343 return -ENOMEM; 3344 3345 for (ix = 0; ix < priv->max_nch; ix++) { 3346 memset(in, 0, inlen); 3347 tir = &tirs[ix]; 3348 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3349 mlx5e_build_direct_tir_ctx(priv, tir->rqt.rqtn, tirc); 3350 err = mlx5e_create_tir(priv->mdev, tir, in, inlen); 3351 if (unlikely(err)) 3352 goto err_destroy_ch_tirs; 3353 } 3354 3355 goto out; 3356 3357 err_destroy_ch_tirs: 3358 mlx5_core_warn(priv->mdev, "create tirs failed, %d\n", err); 3359 for (ix--; ix >= 0; ix--) 3360 mlx5e_destroy_tir(priv->mdev, &tirs[ix]); 3361 3362 out: 3363 kvfree(in); 3364 3365 return err; 3366 } 3367 3368 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc) 3369 { 3370 int i; 3371 3372 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) 3373 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]); 3374 3375 if (!inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev)) 3376 return; 3377 3378 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) 3379 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]); 3380 } 3381 3382 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv, struct mlx5e_tir *tirs) 3383 { 3384 int i; 3385 3386 for (i = 0; i < priv->max_nch; i++) 3387 mlx5e_destroy_tir(priv->mdev, &tirs[i]); 3388 } 3389 3390 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable) 3391 { 3392 int err = 0; 3393 int i; 3394 3395 for (i = 0; i < chs->num; i++) { 3396 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable); 3397 if (err) 3398 return err; 3399 } 3400 3401 return 0; 3402 } 3403 3404 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3405 { 3406 int err = 0; 3407 int i; 3408 3409 for (i = 0; i < chs->num; i++) { 3410 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3411 if (err) 3412 return err; 3413 } 3414 3415 return 0; 3416 } 3417 3418 static int mlx5e_setup_tc_mqprio(struct mlx5e_priv *priv, 3419 struct tc_mqprio_qopt *mqprio) 3420 { 3421 struct mlx5e_channels new_channels = {}; 3422 u8 tc = mqprio->num_tc; 3423 int err = 0; 3424 3425 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3426 3427 if (tc && tc != MLX5E_MAX_NUM_TC) 3428 return -EINVAL; 3429 3430 mutex_lock(&priv->state_lock); 3431 3432 new_channels.params = priv->channels.params; 3433 new_channels.params.num_tc = tc ? tc : 1; 3434 3435 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 3436 priv->channels.params = new_channels.params; 3437 goto out; 3438 } 3439 3440 err = mlx5e_safe_switch_channels(priv, &new_channels, NULL); 3441 if (err) 3442 goto out; 3443 3444 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3445 new_channels.params.num_tc); 3446 out: 3447 mutex_unlock(&priv->state_lock); 3448 return err; 3449 } 3450 3451 #ifdef CONFIG_MLX5_ESWITCH 3452 static int mlx5e_setup_tc_cls_flower(struct mlx5e_priv *priv, 3453 struct flow_cls_offload *cls_flower, 3454 unsigned long flags) 3455 { 3456 switch (cls_flower->command) { 3457 case FLOW_CLS_REPLACE: 3458 return mlx5e_configure_flower(priv->netdev, priv, cls_flower, 3459 flags); 3460 case FLOW_CLS_DESTROY: 3461 return mlx5e_delete_flower(priv->netdev, priv, cls_flower, 3462 flags); 3463 case FLOW_CLS_STATS: 3464 return mlx5e_stats_flower(priv->netdev, priv, cls_flower, 3465 flags); 3466 default: 3467 return -EOPNOTSUPP; 3468 } 3469 } 3470 3471 static int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 3472 void *cb_priv) 3473 { 3474 unsigned long flags = MLX5_TC_FLAG(INGRESS) | MLX5_TC_FLAG(NIC_OFFLOAD); 3475 struct mlx5e_priv *priv = cb_priv; 3476 3477 switch (type) { 3478 case TC_SETUP_CLSFLOWER: 3479 return mlx5e_setup_tc_cls_flower(priv, type_data, flags); 3480 default: 3481 return -EOPNOTSUPP; 3482 } 3483 } 3484 #endif 3485 3486 static LIST_HEAD(mlx5e_block_cb_list); 3487 3488 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 3489 void *type_data) 3490 { 3491 struct mlx5e_priv *priv = netdev_priv(dev); 3492 3493 switch (type) { 3494 #ifdef CONFIG_MLX5_ESWITCH 3495 case TC_SETUP_BLOCK: { 3496 struct flow_block_offload *f = type_data; 3497 3498 f->unlocked_driver_cb = true; 3499 return flow_block_cb_setup_simple(type_data, 3500 &mlx5e_block_cb_list, 3501 mlx5e_setup_tc_block_cb, 3502 priv, priv, true); 3503 } 3504 #endif 3505 case TC_SETUP_QDISC_MQPRIO: 3506 return mlx5e_setup_tc_mqprio(priv, type_data); 3507 default: 3508 return -EOPNOTSUPP; 3509 } 3510 } 3511 3512 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 3513 { 3514 int i; 3515 3516 for (i = 0; i < priv->max_nch; i++) { 3517 struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i]; 3518 struct mlx5e_rq_stats *xskrq_stats = &channel_stats->xskrq; 3519 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 3520 int j; 3521 3522 s->rx_packets += rq_stats->packets + xskrq_stats->packets; 3523 s->rx_bytes += rq_stats->bytes + xskrq_stats->bytes; 3524 3525 for (j = 0; j < priv->max_opened_tc; j++) { 3526 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 3527 3528 s->tx_packets += sq_stats->packets; 3529 s->tx_bytes += sq_stats->bytes; 3530 s->tx_dropped += sq_stats->dropped; 3531 } 3532 } 3533 } 3534 3535 void 3536 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 3537 { 3538 struct mlx5e_priv *priv = netdev_priv(dev); 3539 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 3540 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 3541 3542 if (!mlx5e_monitor_counter_supported(priv)) { 3543 /* update HW stats in background for next time */ 3544 mlx5e_queue_update_stats(priv); 3545 } 3546 3547 if (mlx5e_is_uplink_rep(priv)) { 3548 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 3549 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 3550 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 3551 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 3552 } else { 3553 mlx5e_fold_sw_stats64(priv, stats); 3554 } 3555 3556 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer; 3557 3558 stats->rx_length_errors = 3559 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 3560 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 3561 PPORT_802_3_GET(pstats, a_frame_too_long_errors); 3562 stats->rx_crc_errors = 3563 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 3564 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors); 3565 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards); 3566 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors + 3567 stats->rx_frame_errors; 3568 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; 3569 3570 /* vport multicast also counts packets that are dropped due to steering 3571 * or rx out of buffer 3572 */ 3573 stats->multicast = 3574 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 3575 } 3576 3577 static void mlx5e_set_rx_mode(struct net_device *dev) 3578 { 3579 struct mlx5e_priv *priv = netdev_priv(dev); 3580 3581 queue_work(priv->wq, &priv->set_rx_mode_work); 3582 } 3583 3584 static int mlx5e_set_mac(struct net_device *netdev, void *addr) 3585 { 3586 struct mlx5e_priv *priv = netdev_priv(netdev); 3587 struct sockaddr *saddr = addr; 3588 3589 if (!is_valid_ether_addr(saddr->sa_data)) 3590 return -EADDRNOTAVAIL; 3591 3592 netif_addr_lock_bh(netdev); 3593 ether_addr_copy(netdev->dev_addr, saddr->sa_data); 3594 netif_addr_unlock_bh(netdev); 3595 3596 queue_work(priv->wq, &priv->set_rx_mode_work); 3597 3598 return 0; 3599 } 3600 3601 #define MLX5E_SET_FEATURE(features, feature, enable) \ 3602 do { \ 3603 if (enable) \ 3604 *features |= feature; \ 3605 else \ 3606 *features &= ~feature; \ 3607 } while (0) 3608 3609 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 3610 3611 static int set_feature_lro(struct net_device *netdev, bool enable) 3612 { 3613 struct mlx5e_priv *priv = netdev_priv(netdev); 3614 struct mlx5_core_dev *mdev = priv->mdev; 3615 struct mlx5e_channels new_channels = {}; 3616 struct mlx5e_params *old_params; 3617 int err = 0; 3618 bool reset; 3619 3620 mutex_lock(&priv->state_lock); 3621 3622 if (enable && priv->xsk.refcnt) { 3623 netdev_warn(netdev, "LRO is incompatible with AF_XDP (%hu XSKs are active)\n", 3624 priv->xsk.refcnt); 3625 err = -EINVAL; 3626 goto out; 3627 } 3628 3629 old_params = &priv->channels.params; 3630 if (enable && !MLX5E_GET_PFLAG(old_params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 3631 netdev_warn(netdev, "can't set LRO with legacy RQ\n"); 3632 err = -EINVAL; 3633 goto out; 3634 } 3635 3636 reset = test_bit(MLX5E_STATE_OPENED, &priv->state); 3637 3638 new_channels.params = *old_params; 3639 new_channels.params.lro_en = enable; 3640 3641 if (old_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) { 3642 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params, NULL) == 3643 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params, NULL)) 3644 reset = false; 3645 } 3646 3647 if (!reset) { 3648 *old_params = new_channels.params; 3649 err = mlx5e_modify_tirs_lro(priv); 3650 goto out; 3651 } 3652 3653 err = mlx5e_safe_switch_channels(priv, &new_channels, mlx5e_modify_tirs_lro); 3654 out: 3655 mutex_unlock(&priv->state_lock); 3656 return err; 3657 } 3658 3659 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 3660 { 3661 struct mlx5e_priv *priv = netdev_priv(netdev); 3662 3663 if (enable) 3664 mlx5e_enable_cvlan_filter(priv); 3665 else 3666 mlx5e_disable_cvlan_filter(priv); 3667 3668 return 0; 3669 } 3670 3671 #ifdef CONFIG_MLX5_ESWITCH 3672 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable) 3673 { 3674 struct mlx5e_priv *priv = netdev_priv(netdev); 3675 3676 if (!enable && mlx5e_tc_num_filters(priv, MLX5_TC_FLAG(NIC_OFFLOAD))) { 3677 netdev_err(netdev, 3678 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 3679 return -EINVAL; 3680 } 3681 3682 return 0; 3683 } 3684 #endif 3685 3686 static int set_feature_rx_all(struct net_device *netdev, bool enable) 3687 { 3688 struct mlx5e_priv *priv = netdev_priv(netdev); 3689 struct mlx5_core_dev *mdev = priv->mdev; 3690 3691 return mlx5_set_port_fcs(mdev, !enable); 3692 } 3693 3694 static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 3695 { 3696 struct mlx5e_priv *priv = netdev_priv(netdev); 3697 int err; 3698 3699 mutex_lock(&priv->state_lock); 3700 3701 priv->channels.params.scatter_fcs_en = enable; 3702 err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable); 3703 if (err) 3704 priv->channels.params.scatter_fcs_en = !enable; 3705 3706 mutex_unlock(&priv->state_lock); 3707 3708 return err; 3709 } 3710 3711 static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 3712 { 3713 struct mlx5e_priv *priv = netdev_priv(netdev); 3714 int err = 0; 3715 3716 mutex_lock(&priv->state_lock); 3717 3718 priv->channels.params.vlan_strip_disable = !enable; 3719 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3720 goto unlock; 3721 3722 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 3723 if (err) 3724 priv->channels.params.vlan_strip_disable = enable; 3725 3726 unlock: 3727 mutex_unlock(&priv->state_lock); 3728 3729 return err; 3730 } 3731 3732 #ifdef CONFIG_MLX5_EN_ARFS 3733 static int set_feature_arfs(struct net_device *netdev, bool enable) 3734 { 3735 struct mlx5e_priv *priv = netdev_priv(netdev); 3736 int err; 3737 3738 if (enable) 3739 err = mlx5e_arfs_enable(priv); 3740 else 3741 err = mlx5e_arfs_disable(priv); 3742 3743 return err; 3744 } 3745 #endif 3746 3747 static int mlx5e_handle_feature(struct net_device *netdev, 3748 netdev_features_t *features, 3749 netdev_features_t wanted_features, 3750 netdev_features_t feature, 3751 mlx5e_feature_handler feature_handler) 3752 { 3753 netdev_features_t changes = wanted_features ^ netdev->features; 3754 bool enable = !!(wanted_features & feature); 3755 int err; 3756 3757 if (!(changes & feature)) 3758 return 0; 3759 3760 err = feature_handler(netdev, enable); 3761 if (err) { 3762 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 3763 enable ? "Enable" : "Disable", &feature, err); 3764 return err; 3765 } 3766 3767 MLX5E_SET_FEATURE(features, feature, enable); 3768 return 0; 3769 } 3770 3771 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 3772 { 3773 netdev_features_t oper_features = netdev->features; 3774 int err = 0; 3775 3776 #define MLX5E_HANDLE_FEATURE(feature, handler) \ 3777 mlx5e_handle_feature(netdev, &oper_features, features, feature, handler) 3778 3779 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 3780 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 3781 set_feature_cvlan_filter); 3782 #ifdef CONFIG_MLX5_ESWITCH 3783 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_tc_num_filters); 3784 #endif 3785 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 3786 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 3787 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 3788 #ifdef CONFIG_MLX5_EN_ARFS 3789 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 3790 #endif 3791 3792 if (err) { 3793 netdev->features = oper_features; 3794 return -EINVAL; 3795 } 3796 3797 return 0; 3798 } 3799 3800 static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 3801 netdev_features_t features) 3802 { 3803 struct mlx5e_priv *priv = netdev_priv(netdev); 3804 struct mlx5e_params *params; 3805 3806 mutex_lock(&priv->state_lock); 3807 params = &priv->channels.params; 3808 if (!bitmap_empty(priv->fs.vlan.active_svlans, VLAN_N_VID)) { 3809 /* HW strips the outer C-tag header, this is a problem 3810 * for S-tag traffic. 3811 */ 3812 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3813 if (!params->vlan_strip_disable) 3814 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 3815 } 3816 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 3817 if (features & NETIF_F_LRO) { 3818 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 3819 features &= ~NETIF_F_LRO; 3820 } 3821 } 3822 3823 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 3824 features &= ~NETIF_F_RXHASH; 3825 if (netdev->features & NETIF_F_RXHASH) 3826 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 3827 } 3828 3829 mutex_unlock(&priv->state_lock); 3830 3831 return features; 3832 } 3833 3834 static bool mlx5e_xsk_validate_mtu(struct net_device *netdev, 3835 struct mlx5e_channels *chs, 3836 struct mlx5e_params *new_params, 3837 struct mlx5_core_dev *mdev) 3838 { 3839 u16 ix; 3840 3841 for (ix = 0; ix < chs->params.num_channels; ix++) { 3842 struct xdp_umem *umem = mlx5e_xsk_get_umem(&chs->params, chs->params.xsk, ix); 3843 struct mlx5e_xsk_param xsk; 3844 3845 if (!umem) 3846 continue; 3847 3848 mlx5e_build_xsk_param(umem, &xsk); 3849 3850 if (!mlx5e_validate_xsk_param(new_params, &xsk, mdev)) { 3851 u32 hr = mlx5e_get_linear_rq_headroom(new_params, &xsk); 3852 int max_mtu_frame, max_mtu_page, max_mtu; 3853 3854 /* Two criteria must be met: 3855 * 1. HW MTU + all headrooms <= XSK frame size. 3856 * 2. Size of SKBs allocated on XDP_PASS <= PAGE_SIZE. 3857 */ 3858 max_mtu_frame = MLX5E_HW2SW_MTU(new_params, xsk.chunk_size - hr); 3859 max_mtu_page = mlx5e_xdp_max_mtu(new_params, &xsk); 3860 max_mtu = min(max_mtu_frame, max_mtu_page); 3861 3862 netdev_err(netdev, "MTU %d is too big for an XSK running on channel %hu. Try MTU <= %d\n", 3863 new_params->sw_mtu, ix, max_mtu); 3864 return false; 3865 } 3866 } 3867 3868 return true; 3869 } 3870 3871 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 3872 change_hw_mtu_cb set_mtu_cb) 3873 { 3874 struct mlx5e_priv *priv = netdev_priv(netdev); 3875 struct mlx5e_channels new_channels = {}; 3876 struct mlx5e_params *params; 3877 int err = 0; 3878 bool reset; 3879 3880 mutex_lock(&priv->state_lock); 3881 3882 params = &priv->channels.params; 3883 3884 reset = !params->lro_en; 3885 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state); 3886 3887 new_channels.params = *params; 3888 new_channels.params.sw_mtu = new_mtu; 3889 3890 if (params->xdp_prog && 3891 !mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) { 3892 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n", 3893 new_mtu, mlx5e_xdp_max_mtu(params, NULL)); 3894 err = -EINVAL; 3895 goto out; 3896 } 3897 3898 if (priv->xsk.refcnt && 3899 !mlx5e_xsk_validate_mtu(netdev, &priv->channels, 3900 &new_channels.params, priv->mdev)) { 3901 err = -EINVAL; 3902 goto out; 3903 } 3904 3905 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 3906 bool is_linear = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, 3907 &new_channels.params, 3908 NULL); 3909 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params, NULL); 3910 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params, NULL); 3911 3912 /* If XSK is active, XSK RQs are linear. */ 3913 is_linear |= priv->xsk.refcnt; 3914 3915 /* Always reset in linear mode - hw_mtu is used in data path. */ 3916 reset = reset && (is_linear || (ppw_old != ppw_new)); 3917 } 3918 3919 if (!reset) { 3920 params->sw_mtu = new_mtu; 3921 if (set_mtu_cb) 3922 set_mtu_cb(priv); 3923 netdev->mtu = params->sw_mtu; 3924 goto out; 3925 } 3926 3927 err = mlx5e_safe_switch_channels(priv, &new_channels, set_mtu_cb); 3928 if (err) 3929 goto out; 3930 3931 netdev->mtu = new_channels.params.sw_mtu; 3932 3933 out: 3934 mutex_unlock(&priv->state_lock); 3935 return err; 3936 } 3937 3938 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 3939 { 3940 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu); 3941 } 3942 3943 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) 3944 { 3945 struct hwtstamp_config config; 3946 int err; 3947 3948 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 3949 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) 3950 return -EOPNOTSUPP; 3951 3952 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 3953 return -EFAULT; 3954 3955 /* TX HW timestamp */ 3956 switch (config.tx_type) { 3957 case HWTSTAMP_TX_OFF: 3958 case HWTSTAMP_TX_ON: 3959 break; 3960 default: 3961 return -ERANGE; 3962 } 3963 3964 mutex_lock(&priv->state_lock); 3965 /* RX HW timestamp */ 3966 switch (config.rx_filter) { 3967 case HWTSTAMP_FILTER_NONE: 3968 /* Reset CQE compression to Admin default */ 3969 mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def); 3970 break; 3971 case HWTSTAMP_FILTER_ALL: 3972 case HWTSTAMP_FILTER_SOME: 3973 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3974 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3975 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3976 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3977 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3978 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3979 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 3980 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 3981 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 3982 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3983 case HWTSTAMP_FILTER_PTP_V2_SYNC: 3984 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 3985 case HWTSTAMP_FILTER_NTP_ALL: 3986 /* Disable CQE compression */ 3987 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 3988 netdev_warn(priv->netdev, "Disabling RX cqe compression\n"); 3989 err = mlx5e_modify_rx_cqe_compression_locked(priv, false); 3990 if (err) { 3991 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 3992 mutex_unlock(&priv->state_lock); 3993 return err; 3994 } 3995 config.rx_filter = HWTSTAMP_FILTER_ALL; 3996 break; 3997 default: 3998 mutex_unlock(&priv->state_lock); 3999 return -ERANGE; 4000 } 4001 4002 memcpy(&priv->tstamp, &config, sizeof(config)); 4003 mutex_unlock(&priv->state_lock); 4004 4005 /* might need to fix some features */ 4006 netdev_update_features(priv->netdev); 4007 4008 return copy_to_user(ifr->ifr_data, &config, 4009 sizeof(config)) ? -EFAULT : 0; 4010 } 4011 4012 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr) 4013 { 4014 struct hwtstamp_config *cfg = &priv->tstamp; 4015 4016 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 4017 return -EOPNOTSUPP; 4018 4019 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0; 4020 } 4021 4022 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 4023 { 4024 struct mlx5e_priv *priv = netdev_priv(dev); 4025 4026 switch (cmd) { 4027 case SIOCSHWTSTAMP: 4028 return mlx5e_hwstamp_set(priv, ifr); 4029 case SIOCGHWTSTAMP: 4030 return mlx5e_hwstamp_get(priv, ifr); 4031 default: 4032 return -EOPNOTSUPP; 4033 } 4034 } 4035 4036 #ifdef CONFIG_MLX5_ESWITCH 4037 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 4038 { 4039 struct mlx5e_priv *priv = netdev_priv(dev); 4040 struct mlx5_core_dev *mdev = priv->mdev; 4041 4042 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 4043 } 4044 4045 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 4046 __be16 vlan_proto) 4047 { 4048 struct mlx5e_priv *priv = netdev_priv(dev); 4049 struct mlx5_core_dev *mdev = priv->mdev; 4050 4051 if (vlan_proto != htons(ETH_P_8021Q)) 4052 return -EPROTONOSUPPORT; 4053 4054 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 4055 vlan, qos); 4056 } 4057 4058 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 4059 { 4060 struct mlx5e_priv *priv = netdev_priv(dev); 4061 struct mlx5_core_dev *mdev = priv->mdev; 4062 4063 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 4064 } 4065 4066 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 4067 { 4068 struct mlx5e_priv *priv = netdev_priv(dev); 4069 struct mlx5_core_dev *mdev = priv->mdev; 4070 4071 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 4072 } 4073 4074 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 4075 int max_tx_rate) 4076 { 4077 struct mlx5e_priv *priv = netdev_priv(dev); 4078 struct mlx5_core_dev *mdev = priv->mdev; 4079 4080 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 4081 max_tx_rate, min_tx_rate); 4082 } 4083 4084 static int mlx5_vport_link2ifla(u8 esw_link) 4085 { 4086 switch (esw_link) { 4087 case MLX5_VPORT_ADMIN_STATE_DOWN: 4088 return IFLA_VF_LINK_STATE_DISABLE; 4089 case MLX5_VPORT_ADMIN_STATE_UP: 4090 return IFLA_VF_LINK_STATE_ENABLE; 4091 } 4092 return IFLA_VF_LINK_STATE_AUTO; 4093 } 4094 4095 static int mlx5_ifla_link2vport(u8 ifla_link) 4096 { 4097 switch (ifla_link) { 4098 case IFLA_VF_LINK_STATE_DISABLE: 4099 return MLX5_VPORT_ADMIN_STATE_DOWN; 4100 case IFLA_VF_LINK_STATE_ENABLE: 4101 return MLX5_VPORT_ADMIN_STATE_UP; 4102 } 4103 return MLX5_VPORT_ADMIN_STATE_AUTO; 4104 } 4105 4106 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 4107 int link_state) 4108 { 4109 struct mlx5e_priv *priv = netdev_priv(dev); 4110 struct mlx5_core_dev *mdev = priv->mdev; 4111 4112 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 4113 mlx5_ifla_link2vport(link_state)); 4114 } 4115 4116 int mlx5e_get_vf_config(struct net_device *dev, 4117 int vf, struct ifla_vf_info *ivi) 4118 { 4119 struct mlx5e_priv *priv = netdev_priv(dev); 4120 struct mlx5_core_dev *mdev = priv->mdev; 4121 int err; 4122 4123 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 4124 if (err) 4125 return err; 4126 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 4127 return 0; 4128 } 4129 4130 int mlx5e_get_vf_stats(struct net_device *dev, 4131 int vf, struct ifla_vf_stats *vf_stats) 4132 { 4133 struct mlx5e_priv *priv = netdev_priv(dev); 4134 struct mlx5_core_dev *mdev = priv->mdev; 4135 4136 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 4137 vf_stats); 4138 } 4139 #endif 4140 4141 struct mlx5e_vxlan_work { 4142 struct work_struct work; 4143 struct mlx5e_priv *priv; 4144 u16 port; 4145 }; 4146 4147 static void mlx5e_vxlan_add_work(struct work_struct *work) 4148 { 4149 struct mlx5e_vxlan_work *vxlan_work = 4150 container_of(work, struct mlx5e_vxlan_work, work); 4151 struct mlx5e_priv *priv = vxlan_work->priv; 4152 u16 port = vxlan_work->port; 4153 4154 mutex_lock(&priv->state_lock); 4155 mlx5_vxlan_add_port(priv->mdev->vxlan, port); 4156 mutex_unlock(&priv->state_lock); 4157 4158 kfree(vxlan_work); 4159 } 4160 4161 static void mlx5e_vxlan_del_work(struct work_struct *work) 4162 { 4163 struct mlx5e_vxlan_work *vxlan_work = 4164 container_of(work, struct mlx5e_vxlan_work, work); 4165 struct mlx5e_priv *priv = vxlan_work->priv; 4166 u16 port = vxlan_work->port; 4167 4168 mutex_lock(&priv->state_lock); 4169 mlx5_vxlan_del_port(priv->mdev->vxlan, port); 4170 mutex_unlock(&priv->state_lock); 4171 kfree(vxlan_work); 4172 } 4173 4174 static void mlx5e_vxlan_queue_work(struct mlx5e_priv *priv, u16 port, int add) 4175 { 4176 struct mlx5e_vxlan_work *vxlan_work; 4177 4178 vxlan_work = kmalloc(sizeof(*vxlan_work), GFP_ATOMIC); 4179 if (!vxlan_work) 4180 return; 4181 4182 if (add) 4183 INIT_WORK(&vxlan_work->work, mlx5e_vxlan_add_work); 4184 else 4185 INIT_WORK(&vxlan_work->work, mlx5e_vxlan_del_work); 4186 4187 vxlan_work->priv = priv; 4188 vxlan_work->port = port; 4189 queue_work(priv->wq, &vxlan_work->work); 4190 } 4191 4192 void mlx5e_add_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) 4193 { 4194 struct mlx5e_priv *priv = netdev_priv(netdev); 4195 4196 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 4197 return; 4198 4199 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 4200 return; 4201 4202 mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 1); 4203 } 4204 4205 void mlx5e_del_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) 4206 { 4207 struct mlx5e_priv *priv = netdev_priv(netdev); 4208 4209 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 4210 return; 4211 4212 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 4213 return; 4214 4215 mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 0); 4216 } 4217 4218 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 4219 struct sk_buff *skb, 4220 netdev_features_t features) 4221 { 4222 unsigned int offset = 0; 4223 struct udphdr *udph; 4224 u8 proto; 4225 u16 port; 4226 4227 switch (vlan_get_protocol(skb)) { 4228 case htons(ETH_P_IP): 4229 proto = ip_hdr(skb)->protocol; 4230 break; 4231 case htons(ETH_P_IPV6): 4232 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 4233 break; 4234 default: 4235 goto out; 4236 } 4237 4238 switch (proto) { 4239 case IPPROTO_GRE: 4240 return features; 4241 case IPPROTO_IPIP: 4242 case IPPROTO_IPV6: 4243 if (mlx5e_tunnel_proto_supported(priv->mdev, IPPROTO_IPIP)) 4244 return features; 4245 break; 4246 case IPPROTO_UDP: 4247 udph = udp_hdr(skb); 4248 port = be16_to_cpu(udph->dest); 4249 4250 /* Verify if UDP port is being offloaded by HW */ 4251 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 4252 return features; 4253 4254 #if IS_ENABLED(CONFIG_GENEVE) 4255 /* Support Geneve offload for default UDP port */ 4256 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 4257 return features; 4258 #endif 4259 } 4260 4261 out: 4262 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */ 4263 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4264 } 4265 4266 netdev_features_t mlx5e_features_check(struct sk_buff *skb, 4267 struct net_device *netdev, 4268 netdev_features_t features) 4269 { 4270 struct mlx5e_priv *priv = netdev_priv(netdev); 4271 4272 features = vlan_features_check(skb, features); 4273 features = vxlan_features_check(skb, features); 4274 4275 #ifdef CONFIG_MLX5_EN_IPSEC 4276 if (mlx5e_ipsec_feature_check(skb, netdev, features)) 4277 return features; 4278 #endif 4279 4280 /* Validate if the tunneled packet is being offloaded by HW */ 4281 if (skb->encapsulation && 4282 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 4283 return mlx5e_tunnel_features_check(priv, skb, features); 4284 4285 return features; 4286 } 4287 4288 static void mlx5e_tx_timeout_work(struct work_struct *work) 4289 { 4290 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 4291 tx_timeout_work); 4292 bool report_failed = false; 4293 int err; 4294 int i; 4295 4296 rtnl_lock(); 4297 mutex_lock(&priv->state_lock); 4298 4299 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4300 goto unlock; 4301 4302 for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) { 4303 struct netdev_queue *dev_queue = 4304 netdev_get_tx_queue(priv->netdev, i); 4305 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 4306 4307 if (!netif_xmit_stopped(dev_queue)) 4308 continue; 4309 4310 if (mlx5e_reporter_tx_timeout(sq)) 4311 report_failed = true; 4312 } 4313 4314 if (!report_failed) 4315 goto unlock; 4316 4317 err = mlx5e_safe_reopen_channels(priv); 4318 if (err) 4319 netdev_err(priv->netdev, 4320 "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", 4321 err); 4322 4323 unlock: 4324 mutex_unlock(&priv->state_lock); 4325 rtnl_unlock(); 4326 } 4327 4328 static void mlx5e_tx_timeout(struct net_device *dev) 4329 { 4330 struct mlx5e_priv *priv = netdev_priv(dev); 4331 4332 netdev_err(dev, "TX timeout detected\n"); 4333 queue_work(priv->wq, &priv->tx_timeout_work); 4334 } 4335 4336 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) 4337 { 4338 struct net_device *netdev = priv->netdev; 4339 struct mlx5e_channels new_channels = {}; 4340 4341 if (priv->channels.params.lro_en) { 4342 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n"); 4343 return -EINVAL; 4344 } 4345 4346 if (MLX5_IPSEC_DEV(priv->mdev)) { 4347 netdev_warn(netdev, "can't set XDP with IPSec offload\n"); 4348 return -EINVAL; 4349 } 4350 4351 new_channels.params = priv->channels.params; 4352 new_channels.params.xdp_prog = prog; 4353 4354 /* No XSK params: AF_XDP can't be enabled yet at the point of setting 4355 * the XDP program. 4356 */ 4357 if (!mlx5e_rx_is_linear_skb(&new_channels.params, NULL)) { 4358 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n", 4359 new_channels.params.sw_mtu, 4360 mlx5e_xdp_max_mtu(&new_channels.params, NULL)); 4361 return -EINVAL; 4362 } 4363 4364 return 0; 4365 } 4366 4367 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 4368 { 4369 struct mlx5e_priv *priv = netdev_priv(netdev); 4370 struct bpf_prog *old_prog; 4371 bool reset, was_opened; 4372 int err = 0; 4373 int i; 4374 4375 mutex_lock(&priv->state_lock); 4376 4377 if (prog) { 4378 err = mlx5e_xdp_allowed(priv, prog); 4379 if (err) 4380 goto unlock; 4381 } 4382 4383 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); 4384 /* no need for full reset when exchanging programs */ 4385 reset = (!priv->channels.params.xdp_prog || !prog); 4386 4387 if (was_opened && !reset) 4388 /* num_channels is invariant here, so we can take the 4389 * batched reference right upfront. 4390 */ 4391 bpf_prog_add(prog, priv->channels.num); 4392 4393 if (was_opened && reset) { 4394 struct mlx5e_channels new_channels = {}; 4395 4396 new_channels.params = priv->channels.params; 4397 new_channels.params.xdp_prog = prog; 4398 mlx5e_set_rq_type(priv->mdev, &new_channels.params); 4399 old_prog = priv->channels.params.xdp_prog; 4400 4401 err = mlx5e_safe_switch_channels(priv, &new_channels, NULL); 4402 if (err) 4403 goto unlock; 4404 } else { 4405 /* exchange programs, extra prog reference we got from caller 4406 * as long as we don't fail from this point onwards. 4407 */ 4408 old_prog = xchg(&priv->channels.params.xdp_prog, prog); 4409 } 4410 4411 if (old_prog) 4412 bpf_prog_put(old_prog); 4413 4414 if (!was_opened && reset) /* change RQ type according to priv->xdp_prog */ 4415 mlx5e_set_rq_type(priv->mdev, &priv->channels.params); 4416 4417 if (!was_opened || reset) 4418 goto unlock; 4419 4420 /* exchanging programs w/o reset, we update ref counts on behalf 4421 * of the channels RQs here. 4422 */ 4423 for (i = 0; i < priv->channels.num; i++) { 4424 struct mlx5e_channel *c = priv->channels.c[i]; 4425 bool xsk_open = test_bit(MLX5E_CHANNEL_STATE_XSK, c->state); 4426 4427 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state); 4428 if (xsk_open) 4429 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); 4430 napi_synchronize(&c->napi); 4431 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */ 4432 4433 old_prog = xchg(&c->rq.xdp_prog, prog); 4434 if (old_prog) 4435 bpf_prog_put(old_prog); 4436 4437 if (xsk_open) { 4438 old_prog = xchg(&c->xskrq.xdp_prog, prog); 4439 if (old_prog) 4440 bpf_prog_put(old_prog); 4441 } 4442 4443 set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state); 4444 if (xsk_open) 4445 set_bit(MLX5E_RQ_STATE_ENABLED, &c->xskrq.state); 4446 /* napi_schedule in case we have missed anything */ 4447 napi_schedule(&c->napi); 4448 } 4449 4450 unlock: 4451 mutex_unlock(&priv->state_lock); 4452 return err; 4453 } 4454 4455 static u32 mlx5e_xdp_query(struct net_device *dev) 4456 { 4457 struct mlx5e_priv *priv = netdev_priv(dev); 4458 const struct bpf_prog *xdp_prog; 4459 u32 prog_id = 0; 4460 4461 mutex_lock(&priv->state_lock); 4462 xdp_prog = priv->channels.params.xdp_prog; 4463 if (xdp_prog) 4464 prog_id = xdp_prog->aux->id; 4465 mutex_unlock(&priv->state_lock); 4466 4467 return prog_id; 4468 } 4469 4470 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4471 { 4472 switch (xdp->command) { 4473 case XDP_SETUP_PROG: 4474 return mlx5e_xdp_set(dev, xdp->prog); 4475 case XDP_QUERY_PROG: 4476 xdp->prog_id = mlx5e_xdp_query(dev); 4477 return 0; 4478 case XDP_SETUP_XSK_UMEM: 4479 return mlx5e_xsk_setup_umem(dev, xdp->xsk.umem, 4480 xdp->xsk.queue_id); 4481 default: 4482 return -EINVAL; 4483 } 4484 } 4485 4486 #ifdef CONFIG_MLX5_ESWITCH 4487 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4488 struct net_device *dev, u32 filter_mask, 4489 int nlflags) 4490 { 4491 struct mlx5e_priv *priv = netdev_priv(dev); 4492 struct mlx5_core_dev *mdev = priv->mdev; 4493 u8 mode, setting; 4494 int err; 4495 4496 err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting); 4497 if (err) 4498 return err; 4499 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 4500 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4501 mode, 4502 0, 0, nlflags, filter_mask, NULL); 4503 } 4504 4505 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4506 u16 flags, struct netlink_ext_ack *extack) 4507 { 4508 struct mlx5e_priv *priv = netdev_priv(dev); 4509 struct mlx5_core_dev *mdev = priv->mdev; 4510 struct nlattr *attr, *br_spec; 4511 u16 mode = BRIDGE_MODE_UNDEF; 4512 u8 setting; 4513 int rem; 4514 4515 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4516 if (!br_spec) 4517 return -EINVAL; 4518 4519 nla_for_each_nested(attr, br_spec, rem) { 4520 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4521 continue; 4522 4523 if (nla_len(attr) < sizeof(mode)) 4524 return -EINVAL; 4525 4526 mode = nla_get_u16(attr); 4527 if (mode > BRIDGE_MODE_VEPA) 4528 return -EINVAL; 4529 4530 break; 4531 } 4532 4533 if (mode == BRIDGE_MODE_UNDEF) 4534 return -EINVAL; 4535 4536 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 4537 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 4538 } 4539 #endif 4540 4541 const struct net_device_ops mlx5e_netdev_ops = { 4542 .ndo_open = mlx5e_open, 4543 .ndo_stop = mlx5e_close, 4544 .ndo_start_xmit = mlx5e_xmit, 4545 .ndo_setup_tc = mlx5e_setup_tc, 4546 .ndo_select_queue = mlx5e_select_queue, 4547 .ndo_get_stats64 = mlx5e_get_stats, 4548 .ndo_set_rx_mode = mlx5e_set_rx_mode, 4549 .ndo_set_mac_address = mlx5e_set_mac, 4550 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 4551 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 4552 .ndo_set_features = mlx5e_set_features, 4553 .ndo_fix_features = mlx5e_fix_features, 4554 .ndo_change_mtu = mlx5e_change_nic_mtu, 4555 .ndo_do_ioctl = mlx5e_ioctl, 4556 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 4557 .ndo_udp_tunnel_add = mlx5e_add_vxlan_port, 4558 .ndo_udp_tunnel_del = mlx5e_del_vxlan_port, 4559 .ndo_features_check = mlx5e_features_check, 4560 .ndo_tx_timeout = mlx5e_tx_timeout, 4561 .ndo_bpf = mlx5e_xdp, 4562 .ndo_xdp_xmit = mlx5e_xdp_xmit, 4563 .ndo_xsk_wakeup = mlx5e_xsk_wakeup, 4564 #ifdef CONFIG_MLX5_EN_ARFS 4565 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 4566 #endif 4567 #ifdef CONFIG_MLX5_ESWITCH 4568 .ndo_bridge_setlink = mlx5e_bridge_setlink, 4569 .ndo_bridge_getlink = mlx5e_bridge_getlink, 4570 4571 /* SRIOV E-Switch NDOs */ 4572 .ndo_set_vf_mac = mlx5e_set_vf_mac, 4573 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 4574 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 4575 .ndo_set_vf_trust = mlx5e_set_vf_trust, 4576 .ndo_set_vf_rate = mlx5e_set_vf_rate, 4577 .ndo_get_vf_config = mlx5e_get_vf_config, 4578 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 4579 .ndo_get_vf_stats = mlx5e_get_vf_stats, 4580 #endif 4581 }; 4582 4583 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev) 4584 { 4585 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) 4586 return -EOPNOTSUPP; 4587 if (!MLX5_CAP_GEN(mdev, eth_net_offloads) || 4588 !MLX5_CAP_GEN(mdev, nic_flow_table) || 4589 !MLX5_CAP_ETH(mdev, csum_cap) || 4590 !MLX5_CAP_ETH(mdev, max_lso_cap) || 4591 !MLX5_CAP_ETH(mdev, vlan_cap) || 4592 !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) || 4593 MLX5_CAP_FLOWTABLE(mdev, 4594 flow_table_properties_nic_receive.max_ft_level) 4595 < 3) { 4596 mlx5_core_warn(mdev, 4597 "Not creating net device, some required device capabilities are missing\n"); 4598 return -EOPNOTSUPP; 4599 } 4600 if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable)) 4601 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n"); 4602 if (!MLX5_CAP_GEN(mdev, cq_moderation)) 4603 mlx5_core_warn(mdev, "CQ moderation is not supported\n"); 4604 4605 return 0; 4606 } 4607 4608 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len, 4609 int num_channels) 4610 { 4611 int i; 4612 4613 for (i = 0; i < len; i++) 4614 indirection_rqt[i] = i % num_channels; 4615 } 4616 4617 static bool slow_pci_heuristic(struct mlx5_core_dev *mdev) 4618 { 4619 u32 link_speed = 0; 4620 u32 pci_bw = 0; 4621 4622 mlx5e_port_max_linkspeed(mdev, &link_speed); 4623 pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL); 4624 mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n", 4625 link_speed, pci_bw); 4626 4627 #define MLX5E_SLOW_PCI_RATIO (2) 4628 4629 return link_speed && pci_bw && 4630 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw; 4631 } 4632 4633 static struct dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) 4634 { 4635 struct dim_cq_moder moder; 4636 4637 moder.cq_period_mode = cq_period_mode; 4638 moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS; 4639 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC; 4640 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 4641 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE; 4642 4643 return moder; 4644 } 4645 4646 static struct dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) 4647 { 4648 struct dim_cq_moder moder; 4649 4650 moder.cq_period_mode = cq_period_mode; 4651 moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS; 4652 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC; 4653 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 4654 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE; 4655 4656 return moder; 4657 } 4658 4659 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode) 4660 { 4661 return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ? 4662 DIM_CQ_PERIOD_MODE_START_FROM_CQE : 4663 DIM_CQ_PERIOD_MODE_START_FROM_EQE; 4664 } 4665 4666 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 4667 { 4668 if (params->tx_dim_enabled) { 4669 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 4670 4671 params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode); 4672 } else { 4673 params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode); 4674 } 4675 4676 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER, 4677 params->tx_cq_moderation.cq_period_mode == 4678 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 4679 } 4680 4681 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 4682 { 4683 if (params->rx_dim_enabled) { 4684 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 4685 4686 params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode); 4687 } else { 4688 params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode); 4689 } 4690 4691 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER, 4692 params->rx_cq_moderation.cq_period_mode == 4693 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 4694 } 4695 4696 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) 4697 { 4698 int i; 4699 4700 /* The supported periods are organized in ascending order */ 4701 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++) 4702 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout) 4703 break; 4704 4705 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]); 4706 } 4707 4708 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, 4709 struct mlx5e_params *params) 4710 { 4711 /* Prefer Striding RQ, unless any of the following holds: 4712 * - Striding RQ configuration is not possible/supported. 4713 * - Slow PCI heuristic. 4714 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear. 4715 * 4716 * No XSK params: checking the availability of striding RQ in general. 4717 */ 4718 if (!slow_pci_heuristic(mdev) && 4719 mlx5e_striding_rq_possible(mdev, params) && 4720 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL) || 4721 !mlx5e_rx_is_linear_skb(params, NULL))) 4722 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true); 4723 mlx5e_set_rq_type(mdev, params); 4724 mlx5e_init_rq_type_params(mdev, params); 4725 } 4726 4727 void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params, 4728 u16 num_channels) 4729 { 4730 enum mlx5e_traffic_types tt; 4731 4732 rss_params->hfunc = ETH_RSS_HASH_TOP; 4733 netdev_rss_key_fill(rss_params->toeplitz_hash_key, 4734 sizeof(rss_params->toeplitz_hash_key)); 4735 mlx5e_build_default_indir_rqt(rss_params->indirection_rqt, 4736 MLX5E_INDIR_RQT_SIZE, num_channels); 4737 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 4738 rss_params->rx_hash_fields[tt] = 4739 tirc_default_config[tt].rx_hash_fields; 4740 } 4741 4742 void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, 4743 struct mlx5e_xsk *xsk, 4744 struct mlx5e_rss_params *rss_params, 4745 struct mlx5e_params *params, 4746 u16 max_channels, u16 mtu) 4747 { 4748 u8 rx_cq_period_mode; 4749 4750 params->sw_mtu = mtu; 4751 params->hard_mtu = MLX5E_ETH_HARD_MTU; 4752 params->num_channels = max_channels; 4753 params->num_tc = 1; 4754 4755 /* SQ */ 4756 params->log_sq_size = is_kdump_kernel() ? 4757 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 4758 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 4759 4760 /* XDP SQ */ 4761 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, 4762 MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe)); 4763 4764 /* set CQE compression */ 4765 params->rx_cqe_compress_def = false; 4766 if (MLX5_CAP_GEN(mdev, cqe_compression) && 4767 MLX5_CAP_GEN(mdev, vport_group_manager)) 4768 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 4769 4770 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 4771 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 4772 4773 /* RQ */ 4774 mlx5e_build_rq_params(mdev, params); 4775 4776 /* HW LRO */ 4777 4778 /* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */ 4779 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 4780 /* No XSK params: checking the availability of striding RQ in general. */ 4781 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params, NULL)) 4782 params->lro_en = !slow_pci_heuristic(mdev); 4783 } 4784 params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 4785 4786 /* CQ moderation params */ 4787 rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 4788 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 4789 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 4790 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4791 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4792 mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode); 4793 mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); 4794 4795 /* TX inline */ 4796 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 4797 4798 /* RSS */ 4799 mlx5e_build_rss_params(rss_params, params->num_channels); 4800 params->tunneled_offload_en = 4801 mlx5e_tunnel_inner_ft_supported(mdev); 4802 4803 /* AF_XDP */ 4804 params->xsk = xsk; 4805 } 4806 4807 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 4808 { 4809 struct mlx5e_priv *priv = netdev_priv(netdev); 4810 4811 mlx5_query_mac_address(priv->mdev, netdev->dev_addr); 4812 if (is_zero_ether_addr(netdev->dev_addr) && 4813 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) { 4814 eth_hw_addr_random(netdev); 4815 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 4816 } 4817 } 4818 4819 static void mlx5e_build_nic_netdev(struct net_device *netdev) 4820 { 4821 struct mlx5e_priv *priv = netdev_priv(netdev); 4822 struct mlx5_core_dev *mdev = priv->mdev; 4823 bool fcs_supported; 4824 bool fcs_enabled; 4825 4826 SET_NETDEV_DEV(netdev, mdev->device); 4827 4828 netdev->netdev_ops = &mlx5e_netdev_ops; 4829 4830 #ifdef CONFIG_MLX5_CORE_EN_DCB 4831 if (MLX5_CAP_GEN(mdev, vport_group_manager) && MLX5_CAP_GEN(mdev, qos)) 4832 netdev->dcbnl_ops = &mlx5e_dcbnl_ops; 4833 #endif 4834 4835 netdev->watchdog_timeo = 15 * HZ; 4836 4837 netdev->ethtool_ops = &mlx5e_ethtool_ops; 4838 4839 netdev->vlan_features |= NETIF_F_SG; 4840 netdev->vlan_features |= NETIF_F_HW_CSUM; 4841 netdev->vlan_features |= NETIF_F_GRO; 4842 netdev->vlan_features |= NETIF_F_TSO; 4843 netdev->vlan_features |= NETIF_F_TSO6; 4844 netdev->vlan_features |= NETIF_F_RXCSUM; 4845 netdev->vlan_features |= NETIF_F_RXHASH; 4846 4847 netdev->mpls_features |= NETIF_F_SG; 4848 netdev->mpls_features |= NETIF_F_HW_CSUM; 4849 netdev->mpls_features |= NETIF_F_TSO; 4850 netdev->mpls_features |= NETIF_F_TSO6; 4851 4852 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 4853 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 4854 4855 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 4856 mlx5e_check_fragmented_striding_rq_cap(mdev)) 4857 netdev->vlan_features |= NETIF_F_LRO; 4858 4859 netdev->hw_features = netdev->vlan_features; 4860 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4861 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4862 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4863 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 4864 4865 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev) || 4866 mlx5e_any_tunnel_proto_supported(mdev)) { 4867 netdev->hw_enc_features |= NETIF_F_HW_CSUM; 4868 netdev->hw_enc_features |= NETIF_F_TSO; 4869 netdev->hw_enc_features |= NETIF_F_TSO6; 4870 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 4871 } 4872 4873 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 4874 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 4875 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4876 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 4877 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4878 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 4879 } 4880 4881 if (mlx5e_tunnel_proto_supported(mdev, IPPROTO_GRE)) { 4882 netdev->hw_features |= NETIF_F_GSO_GRE | 4883 NETIF_F_GSO_GRE_CSUM; 4884 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 4885 NETIF_F_GSO_GRE_CSUM; 4886 netdev->gso_partial_features |= NETIF_F_GSO_GRE | 4887 NETIF_F_GSO_GRE_CSUM; 4888 } 4889 4890 if (mlx5e_tunnel_proto_supported(mdev, IPPROTO_IPIP)) { 4891 netdev->hw_features |= NETIF_F_GSO_IPXIP4 | 4892 NETIF_F_GSO_IPXIP6; 4893 netdev->hw_enc_features |= NETIF_F_GSO_IPXIP4 | 4894 NETIF_F_GSO_IPXIP6; 4895 netdev->gso_partial_features |= NETIF_F_GSO_IPXIP4 | 4896 NETIF_F_GSO_IPXIP6; 4897 } 4898 4899 netdev->hw_features |= NETIF_F_GSO_PARTIAL; 4900 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 4901 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 4902 netdev->features |= NETIF_F_GSO_UDP_L4; 4903 4904 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 4905 4906 if (fcs_supported) 4907 netdev->hw_features |= NETIF_F_RXALL; 4908 4909 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 4910 netdev->hw_features |= NETIF_F_RXFCS; 4911 4912 netdev->features = netdev->hw_features; 4913 if (!priv->channels.params.lro_en) 4914 netdev->features &= ~NETIF_F_LRO; 4915 4916 if (fcs_enabled) 4917 netdev->features &= ~NETIF_F_RXALL; 4918 4919 if (!priv->channels.params.scatter_fcs_en) 4920 netdev->features &= ~NETIF_F_RXFCS; 4921 4922 /* prefere CQE compression over rxhash */ 4923 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4924 netdev->features &= ~NETIF_F_RXHASH; 4925 4926 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 4927 if (FT_CAP(flow_modify_en) && 4928 FT_CAP(modify_root) && 4929 FT_CAP(identified_miss_table_mode) && 4930 FT_CAP(flow_table_modify)) { 4931 #ifdef CONFIG_MLX5_ESWITCH 4932 netdev->hw_features |= NETIF_F_HW_TC; 4933 #endif 4934 #ifdef CONFIG_MLX5_EN_ARFS 4935 netdev->hw_features |= NETIF_F_NTUPLE; 4936 #endif 4937 } 4938 4939 netdev->features |= NETIF_F_HIGHDMA; 4940 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 4941 4942 netdev->priv_flags |= IFF_UNICAST_FLT; 4943 4944 mlx5e_set_netdev_dev_addr(netdev); 4945 mlx5e_ipsec_build_netdev(priv); 4946 mlx5e_tls_build_netdev(priv); 4947 } 4948 4949 void mlx5e_create_q_counters(struct mlx5e_priv *priv) 4950 { 4951 struct mlx5_core_dev *mdev = priv->mdev; 4952 int err; 4953 4954 err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter); 4955 if (err) { 4956 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err); 4957 priv->q_counter = 0; 4958 } 4959 4960 err = mlx5_core_alloc_q_counter(mdev, &priv->drop_rq_q_counter); 4961 if (err) { 4962 mlx5_core_warn(mdev, "alloc drop RQ counter failed, %d\n", err); 4963 priv->drop_rq_q_counter = 0; 4964 } 4965 } 4966 4967 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 4968 { 4969 if (priv->q_counter) 4970 mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter); 4971 4972 if (priv->drop_rq_q_counter) 4973 mlx5_core_dealloc_q_counter(priv->mdev, priv->drop_rq_q_counter); 4974 } 4975 4976 static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 4977 struct net_device *netdev, 4978 const struct mlx5e_profile *profile, 4979 void *ppriv) 4980 { 4981 struct mlx5e_priv *priv = netdev_priv(netdev); 4982 struct mlx5e_rss_params *rss = &priv->rss_params; 4983 int err; 4984 4985 err = mlx5e_netdev_init(netdev, priv, mdev, profile, ppriv); 4986 if (err) 4987 return err; 4988 4989 mlx5e_build_nic_params(mdev, &priv->xsk, rss, &priv->channels.params, 4990 priv->max_nch, netdev->mtu); 4991 4992 mlx5e_timestamp_init(priv); 4993 4994 err = mlx5e_ipsec_init(priv); 4995 if (err) 4996 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err); 4997 err = mlx5e_tls_init(priv); 4998 if (err) 4999 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 5000 mlx5e_build_nic_netdev(netdev); 5001 mlx5e_health_create_reporters(priv); 5002 5003 return 0; 5004 } 5005 5006 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 5007 { 5008 mlx5e_health_destroy_reporters(priv); 5009 mlx5e_tls_cleanup(priv); 5010 mlx5e_ipsec_cleanup(priv); 5011 mlx5e_netdev_cleanup(priv->netdev, priv); 5012 } 5013 5014 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 5015 { 5016 struct mlx5_core_dev *mdev = priv->mdev; 5017 int err; 5018 5019 mlx5e_create_q_counters(priv); 5020 5021 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 5022 if (err) { 5023 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 5024 goto err_destroy_q_counters; 5025 } 5026 5027 err = mlx5e_create_indirect_rqt(priv); 5028 if (err) 5029 goto err_close_drop_rq; 5030 5031 err = mlx5e_create_direct_rqts(priv, priv->direct_tir); 5032 if (err) 5033 goto err_destroy_indirect_rqts; 5034 5035 err = mlx5e_create_indirect_tirs(priv, true); 5036 if (err) 5037 goto err_destroy_direct_rqts; 5038 5039 err = mlx5e_create_direct_tirs(priv, priv->direct_tir); 5040 if (err) 5041 goto err_destroy_indirect_tirs; 5042 5043 err = mlx5e_create_direct_rqts(priv, priv->xsk_tir); 5044 if (unlikely(err)) 5045 goto err_destroy_direct_tirs; 5046 5047 err = mlx5e_create_direct_tirs(priv, priv->xsk_tir); 5048 if (unlikely(err)) 5049 goto err_destroy_xsk_rqts; 5050 5051 err = mlx5e_create_flow_steering(priv); 5052 if (err) { 5053 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 5054 goto err_destroy_xsk_tirs; 5055 } 5056 5057 err = mlx5e_tc_nic_init(priv); 5058 if (err) 5059 goto err_destroy_flow_steering; 5060 5061 return 0; 5062 5063 err_destroy_flow_steering: 5064 mlx5e_destroy_flow_steering(priv); 5065 err_destroy_xsk_tirs: 5066 mlx5e_destroy_direct_tirs(priv, priv->xsk_tir); 5067 err_destroy_xsk_rqts: 5068 mlx5e_destroy_direct_rqts(priv, priv->xsk_tir); 5069 err_destroy_direct_tirs: 5070 mlx5e_destroy_direct_tirs(priv, priv->direct_tir); 5071 err_destroy_indirect_tirs: 5072 mlx5e_destroy_indirect_tirs(priv, true); 5073 err_destroy_direct_rqts: 5074 mlx5e_destroy_direct_rqts(priv, priv->direct_tir); 5075 err_destroy_indirect_rqts: 5076 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 5077 err_close_drop_rq: 5078 mlx5e_close_drop_rq(&priv->drop_rq); 5079 err_destroy_q_counters: 5080 mlx5e_destroy_q_counters(priv); 5081 return err; 5082 } 5083 5084 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 5085 { 5086 mlx5e_tc_nic_cleanup(priv); 5087 mlx5e_destroy_flow_steering(priv); 5088 mlx5e_destroy_direct_tirs(priv, priv->xsk_tir); 5089 mlx5e_destroy_direct_rqts(priv, priv->xsk_tir); 5090 mlx5e_destroy_direct_tirs(priv, priv->direct_tir); 5091 mlx5e_destroy_indirect_tirs(priv, true); 5092 mlx5e_destroy_direct_rqts(priv, priv->direct_tir); 5093 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 5094 mlx5e_close_drop_rq(&priv->drop_rq); 5095 mlx5e_destroy_q_counters(priv); 5096 } 5097 5098 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 5099 { 5100 int err; 5101 5102 err = mlx5e_create_tises(priv); 5103 if (err) { 5104 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); 5105 return err; 5106 } 5107 5108 #ifdef CONFIG_MLX5_CORE_EN_DCB 5109 mlx5e_dcbnl_initialize(priv); 5110 #endif 5111 return 0; 5112 } 5113 5114 static void mlx5e_nic_enable(struct mlx5e_priv *priv) 5115 { 5116 struct net_device *netdev = priv->netdev; 5117 struct mlx5_core_dev *mdev = priv->mdev; 5118 5119 mlx5e_init_l2_addr(priv); 5120 5121 /* Marking the link as currently not needed by the Driver */ 5122 if (!netif_running(netdev)) 5123 mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN); 5124 5125 mlx5e_set_netdev_mtu_boundaries(priv); 5126 mlx5e_set_dev_port_mtu(priv); 5127 5128 mlx5_lag_add(mdev, netdev); 5129 5130 mlx5e_enable_async_events(priv); 5131 if (mlx5e_monitor_counter_supported(priv)) 5132 mlx5e_monitor_counter_init(priv); 5133 5134 mlx5e_hv_vhca_stats_create(priv); 5135 if (netdev->reg_state != NETREG_REGISTERED) 5136 return; 5137 #ifdef CONFIG_MLX5_CORE_EN_DCB 5138 mlx5e_dcbnl_init_app(priv); 5139 #endif 5140 5141 queue_work(priv->wq, &priv->set_rx_mode_work); 5142 5143 rtnl_lock(); 5144 if (netif_running(netdev)) 5145 mlx5e_open(netdev); 5146 netif_device_attach(netdev); 5147 rtnl_unlock(); 5148 } 5149 5150 static void mlx5e_nic_disable(struct mlx5e_priv *priv) 5151 { 5152 struct mlx5_core_dev *mdev = priv->mdev; 5153 5154 #ifdef CONFIG_MLX5_CORE_EN_DCB 5155 if (priv->netdev->reg_state == NETREG_REGISTERED) 5156 mlx5e_dcbnl_delete_app(priv); 5157 #endif 5158 5159 rtnl_lock(); 5160 if (netif_running(priv->netdev)) 5161 mlx5e_close(priv->netdev); 5162 netif_device_detach(priv->netdev); 5163 rtnl_unlock(); 5164 5165 queue_work(priv->wq, &priv->set_rx_mode_work); 5166 5167 mlx5e_hv_vhca_stats_destroy(priv); 5168 if (mlx5e_monitor_counter_supported(priv)) 5169 mlx5e_monitor_counter_cleanup(priv); 5170 5171 mlx5e_disable_async_events(priv); 5172 mlx5_lag_remove(mdev); 5173 } 5174 5175 int mlx5e_update_nic_rx(struct mlx5e_priv *priv) 5176 { 5177 return mlx5e_refresh_tirs(priv, false); 5178 } 5179 5180 static const struct mlx5e_profile mlx5e_nic_profile = { 5181 .init = mlx5e_nic_init, 5182 .cleanup = mlx5e_nic_cleanup, 5183 .init_rx = mlx5e_init_nic_rx, 5184 .cleanup_rx = mlx5e_cleanup_nic_rx, 5185 .init_tx = mlx5e_init_nic_tx, 5186 .cleanup_tx = mlx5e_cleanup_nic_tx, 5187 .enable = mlx5e_nic_enable, 5188 .disable = mlx5e_nic_disable, 5189 .update_rx = mlx5e_update_nic_rx, 5190 .update_stats = mlx5e_update_ndo_stats, 5191 .update_carrier = mlx5e_update_carrier, 5192 .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe, 5193 .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, 5194 .max_tc = MLX5E_MAX_NUM_TC, 5195 .rq_groups = MLX5E_NUM_RQ_GROUPS(XSK), 5196 }; 5197 5198 /* mlx5e generic netdev management API (move to en_common.c) */ 5199 5200 /* mlx5e_netdev_init/cleanup must be called from profile->init/cleanup callbacks */ 5201 int mlx5e_netdev_init(struct net_device *netdev, 5202 struct mlx5e_priv *priv, 5203 struct mlx5_core_dev *mdev, 5204 const struct mlx5e_profile *profile, 5205 void *ppriv) 5206 { 5207 /* priv init */ 5208 priv->mdev = mdev; 5209 priv->netdev = netdev; 5210 priv->profile = profile; 5211 priv->ppriv = ppriv; 5212 priv->msglevel = MLX5E_MSG_LEVEL; 5213 priv->max_nch = netdev->num_rx_queues / max_t(u8, profile->rq_groups, 1); 5214 priv->max_opened_tc = 1; 5215 5216 mutex_init(&priv->state_lock); 5217 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 5218 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 5219 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 5220 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 5221 5222 priv->wq = create_singlethread_workqueue("mlx5e"); 5223 if (!priv->wq) 5224 return -ENOMEM; 5225 5226 /* netdev init */ 5227 netif_carrier_off(netdev); 5228 5229 #ifdef CONFIG_MLX5_EN_ARFS 5230 netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(mdev); 5231 #endif 5232 5233 return 0; 5234 } 5235 5236 void mlx5e_netdev_cleanup(struct net_device *netdev, struct mlx5e_priv *priv) 5237 { 5238 destroy_workqueue(priv->wq); 5239 } 5240 5241 struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev, 5242 const struct mlx5e_profile *profile, 5243 int nch, 5244 void *ppriv) 5245 { 5246 struct net_device *netdev; 5247 int err; 5248 5249 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), 5250 nch * profile->max_tc, 5251 nch * profile->rq_groups); 5252 if (!netdev) { 5253 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 5254 return NULL; 5255 } 5256 5257 err = profile->init(mdev, netdev, profile, ppriv); 5258 if (err) { 5259 mlx5_core_err(mdev, "failed to init mlx5e profile %d\n", err); 5260 goto err_free_netdev; 5261 } 5262 5263 return netdev; 5264 5265 err_free_netdev: 5266 free_netdev(netdev); 5267 5268 return NULL; 5269 } 5270 5271 int mlx5e_attach_netdev(struct mlx5e_priv *priv) 5272 { 5273 const struct mlx5e_profile *profile; 5274 int max_nch; 5275 int err; 5276 5277 profile = priv->profile; 5278 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 5279 5280 /* max number of channels may have changed */ 5281 max_nch = mlx5e_get_max_num_channels(priv->mdev); 5282 if (priv->channels.params.num_channels > max_nch) { 5283 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 5284 priv->channels.params.num_channels = max_nch; 5285 mlx5e_build_default_indir_rqt(priv->rss_params.indirection_rqt, 5286 MLX5E_INDIR_RQT_SIZE, max_nch); 5287 } 5288 5289 err = profile->init_tx(priv); 5290 if (err) 5291 goto out; 5292 5293 err = profile->init_rx(priv); 5294 if (err) 5295 goto err_cleanup_tx; 5296 5297 if (profile->enable) 5298 profile->enable(priv); 5299 5300 return 0; 5301 5302 err_cleanup_tx: 5303 profile->cleanup_tx(priv); 5304 5305 out: 5306 return err; 5307 } 5308 5309 void mlx5e_detach_netdev(struct mlx5e_priv *priv) 5310 { 5311 const struct mlx5e_profile *profile = priv->profile; 5312 5313 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5314 5315 if (profile->disable) 5316 profile->disable(priv); 5317 flush_workqueue(priv->wq); 5318 5319 profile->cleanup_rx(priv); 5320 profile->cleanup_tx(priv); 5321 cancel_work_sync(&priv->update_stats_work); 5322 } 5323 5324 void mlx5e_destroy_netdev(struct mlx5e_priv *priv) 5325 { 5326 const struct mlx5e_profile *profile = priv->profile; 5327 struct net_device *netdev = priv->netdev; 5328 5329 if (profile->cleanup) 5330 profile->cleanup(priv); 5331 free_netdev(netdev); 5332 } 5333 5334 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying 5335 * hardware contexts and to connect it to the current netdev. 5336 */ 5337 static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv) 5338 { 5339 struct mlx5e_priv *priv = vpriv; 5340 struct net_device *netdev = priv->netdev; 5341 int err; 5342 5343 if (netif_device_present(netdev)) 5344 return 0; 5345 5346 err = mlx5e_create_mdev_resources(mdev); 5347 if (err) 5348 return err; 5349 5350 err = mlx5e_attach_netdev(priv); 5351 if (err) { 5352 mlx5e_destroy_mdev_resources(mdev); 5353 return err; 5354 } 5355 5356 return 0; 5357 } 5358 5359 static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv) 5360 { 5361 struct mlx5e_priv *priv = vpriv; 5362 struct net_device *netdev = priv->netdev; 5363 5364 #ifdef CONFIG_MLX5_ESWITCH 5365 if (MLX5_ESWITCH_MANAGER(mdev) && vpriv == mdev) 5366 return; 5367 #endif 5368 5369 if (!netif_device_present(netdev)) 5370 return; 5371 5372 mlx5e_detach_netdev(priv); 5373 mlx5e_destroy_mdev_resources(mdev); 5374 } 5375 5376 static void *mlx5e_add(struct mlx5_core_dev *mdev) 5377 { 5378 struct net_device *netdev; 5379 void *priv; 5380 int err; 5381 int nch; 5382 5383 err = mlx5e_check_required_hca_cap(mdev); 5384 if (err) 5385 return NULL; 5386 5387 #ifdef CONFIG_MLX5_ESWITCH 5388 if (MLX5_ESWITCH_MANAGER(mdev) && 5389 mlx5_eswitch_mode(mdev->priv.eswitch) == MLX5_ESWITCH_OFFLOADS) { 5390 mlx5e_rep_register_vport_reps(mdev); 5391 return mdev; 5392 } 5393 #endif 5394 5395 nch = mlx5e_get_max_num_channels(mdev); 5396 netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, nch, NULL); 5397 if (!netdev) { 5398 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 5399 return NULL; 5400 } 5401 5402 dev_net_set(netdev, mlx5_core_net(mdev)); 5403 priv = netdev_priv(netdev); 5404 5405 err = mlx5e_attach(mdev, priv); 5406 if (err) { 5407 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err); 5408 goto err_destroy_netdev; 5409 } 5410 5411 err = register_netdev(netdev); 5412 if (err) { 5413 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 5414 goto err_detach; 5415 } 5416 5417 #ifdef CONFIG_MLX5_CORE_EN_DCB 5418 mlx5e_dcbnl_init_app(priv); 5419 #endif 5420 return priv; 5421 5422 err_detach: 5423 mlx5e_detach(mdev, priv); 5424 err_destroy_netdev: 5425 mlx5e_destroy_netdev(priv); 5426 return NULL; 5427 } 5428 5429 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv) 5430 { 5431 struct mlx5e_priv *priv; 5432 5433 #ifdef CONFIG_MLX5_ESWITCH 5434 if (MLX5_ESWITCH_MANAGER(mdev) && vpriv == mdev) { 5435 mlx5e_rep_unregister_vport_reps(mdev); 5436 return; 5437 } 5438 #endif 5439 priv = vpriv; 5440 #ifdef CONFIG_MLX5_CORE_EN_DCB 5441 mlx5e_dcbnl_delete_app(priv); 5442 #endif 5443 unregister_netdev(priv->netdev); 5444 mlx5e_detach(mdev, vpriv); 5445 mlx5e_destroy_netdev(priv); 5446 } 5447 5448 static struct mlx5_interface mlx5e_interface = { 5449 .add = mlx5e_add, 5450 .remove = mlx5e_remove, 5451 .attach = mlx5e_attach, 5452 .detach = mlx5e_detach, 5453 .protocol = MLX5_INTERFACE_PROTOCOL_ETH, 5454 }; 5455 5456 void mlx5e_init(void) 5457 { 5458 mlx5e_ipsec_build_inverse_table(); 5459 mlx5e_build_ptys2ethtool_map(); 5460 mlx5_register_interface(&mlx5e_interface); 5461 } 5462 5463 void mlx5e_cleanup(void) 5464 { 5465 mlx5_unregister_interface(&mlx5e_interface); 5466 } 5467