1 /* 2 * Copyright (c) 2015-2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <net/tc_act/tc_gact.h> 34 #include <net/pkt_cls.h> 35 #include <linux/mlx5/fs.h> 36 #include <net/vxlan.h> 37 #include <net/geneve.h> 38 #include <linux/bpf.h> 39 #include <linux/if_bridge.h> 40 #include <net/page_pool.h> 41 #include "eswitch.h" 42 #include "en.h" 43 #include "en_tc.h" 44 #include "en_rep.h" 45 #include "en_accel/ipsec.h" 46 #include "en_accel/ipsec_rxtx.h" 47 #include "en_accel/en_accel.h" 48 #include "en_accel/tls.h" 49 #include "accel/ipsec.h" 50 #include "accel/tls.h" 51 #include "lib/vxlan.h" 52 #include "lib/clock.h" 53 #include "en/port.h" 54 #include "en/xdp.h" 55 #include "lib/eq.h" 56 #include "en/monitor_stats.h" 57 #include "en/reporter.h" 58 #include "en/params.h" 59 60 struct mlx5e_rq_param { 61 u32 rqc[MLX5_ST_SZ_DW(rqc)]; 62 struct mlx5_wq_param wq; 63 struct mlx5e_rq_frags_info frags_info; 64 }; 65 66 struct mlx5e_sq_param { 67 u32 sqc[MLX5_ST_SZ_DW(sqc)]; 68 struct mlx5_wq_param wq; 69 bool is_mpw; 70 }; 71 72 struct mlx5e_cq_param { 73 u32 cqc[MLX5_ST_SZ_DW(cqc)]; 74 struct mlx5_wq_param wq; 75 u16 eq_ix; 76 u8 cq_period_mode; 77 }; 78 79 struct mlx5e_channel_param { 80 struct mlx5e_rq_param rq; 81 struct mlx5e_sq_param sq; 82 struct mlx5e_sq_param xdp_sq; 83 struct mlx5e_sq_param icosq; 84 struct mlx5e_cq_param rx_cq; 85 struct mlx5e_cq_param tx_cq; 86 struct mlx5e_cq_param icosq_cq; 87 }; 88 89 bool mlx5e_check_fragmented_striding_rq_cap(struct mlx5_core_dev *mdev) 90 { 91 bool striding_rq_umr = MLX5_CAP_GEN(mdev, striding_rq) && 92 MLX5_CAP_GEN(mdev, umr_ptr_rlky) && 93 MLX5_CAP_ETH(mdev, reg_umr_sq); 94 u16 max_wqe_sz_cap = MLX5_CAP_GEN(mdev, max_wqe_sz_sq); 95 bool inline_umr = MLX5E_UMR_WQE_INLINE_SZ <= max_wqe_sz_cap; 96 97 if (!striding_rq_umr) 98 return false; 99 if (!inline_umr) { 100 mlx5_core_warn(mdev, "Cannot support Striding RQ: UMR WQE size (%d) exceeds maximum supported (%d).\n", 101 (int)MLX5E_UMR_WQE_INLINE_SZ, max_wqe_sz_cap); 102 return false; 103 } 104 return true; 105 } 106 107 void mlx5e_init_rq_type_params(struct mlx5_core_dev *mdev, 108 struct mlx5e_params *params) 109 { 110 params->log_rq_mtu_frames = is_kdump_kernel() ? 111 MLX5E_PARAMS_MINIMUM_LOG_RQ_SIZE : 112 MLX5E_PARAMS_DEFAULT_LOG_RQ_SIZE; 113 114 mlx5_core_info(mdev, "MLX5E: StrdRq(%d) RqSz(%ld) StrdSz(%ld) RxCqeCmprss(%d)\n", 115 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ, 116 params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ ? 117 BIT(mlx5e_mpwqe_get_log_rq_size(params)) : 118 BIT(params->log_rq_mtu_frames), 119 BIT(mlx5e_mpwqe_get_log_stride_size(mdev, params)), 120 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)); 121 } 122 123 bool mlx5e_striding_rq_possible(struct mlx5_core_dev *mdev, 124 struct mlx5e_params *params) 125 { 126 return mlx5e_check_fragmented_striding_rq_cap(mdev) && 127 !MLX5_IPSEC_DEV(mdev) && 128 !(params->xdp_prog && !mlx5e_rx_mpwqe_is_linear_skb(mdev, params)); 129 } 130 131 void mlx5e_set_rq_type(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 132 { 133 params->rq_wq_type = mlx5e_striding_rq_possible(mdev, params) && 134 MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ) ? 135 MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ : 136 MLX5_WQ_TYPE_CYCLIC; 137 } 138 139 void mlx5e_update_carrier(struct mlx5e_priv *priv) 140 { 141 struct mlx5_core_dev *mdev = priv->mdev; 142 u8 port_state; 143 144 port_state = mlx5_query_vport_state(mdev, 145 MLX5_VPORT_STATE_OP_MOD_VNIC_VPORT, 146 0); 147 148 if (port_state == VPORT_STATE_UP) { 149 netdev_info(priv->netdev, "Link up\n"); 150 netif_carrier_on(priv->netdev); 151 } else { 152 netdev_info(priv->netdev, "Link down\n"); 153 netif_carrier_off(priv->netdev); 154 } 155 } 156 157 static void mlx5e_update_carrier_work(struct work_struct *work) 158 { 159 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 160 update_carrier_work); 161 162 mutex_lock(&priv->state_lock); 163 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 164 if (priv->profile->update_carrier) 165 priv->profile->update_carrier(priv); 166 mutex_unlock(&priv->state_lock); 167 } 168 169 void mlx5e_update_stats(struct mlx5e_priv *priv) 170 { 171 int i; 172 173 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--) 174 if (mlx5e_stats_grps[i].update_stats) 175 mlx5e_stats_grps[i].update_stats(priv); 176 } 177 178 void mlx5e_update_ndo_stats(struct mlx5e_priv *priv) 179 { 180 int i; 181 182 for (i = mlx5e_num_stats_grps - 1; i >= 0; i--) 183 if (mlx5e_stats_grps[i].update_stats_mask & 184 MLX5E_NDO_UPDATE_STATS) 185 mlx5e_stats_grps[i].update_stats(priv); 186 } 187 188 static void mlx5e_update_stats_work(struct work_struct *work) 189 { 190 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 191 update_stats_work); 192 193 mutex_lock(&priv->state_lock); 194 priv->profile->update_stats(priv); 195 mutex_unlock(&priv->state_lock); 196 } 197 198 void mlx5e_queue_update_stats(struct mlx5e_priv *priv) 199 { 200 if (!priv->profile->update_stats) 201 return; 202 203 if (unlikely(test_bit(MLX5E_STATE_DESTROYING, &priv->state))) 204 return; 205 206 queue_work(priv->wq, &priv->update_stats_work); 207 } 208 209 static int async_event(struct notifier_block *nb, unsigned long event, void *data) 210 { 211 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 212 struct mlx5_eqe *eqe = data; 213 214 if (event != MLX5_EVENT_TYPE_PORT_CHANGE) 215 return NOTIFY_DONE; 216 217 switch (eqe->sub_type) { 218 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 219 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 220 queue_work(priv->wq, &priv->update_carrier_work); 221 break; 222 default: 223 return NOTIFY_DONE; 224 } 225 226 return NOTIFY_OK; 227 } 228 229 static void mlx5e_enable_async_events(struct mlx5e_priv *priv) 230 { 231 priv->events_nb.notifier_call = async_event; 232 mlx5_notifier_register(priv->mdev, &priv->events_nb); 233 } 234 235 static void mlx5e_disable_async_events(struct mlx5e_priv *priv) 236 { 237 mlx5_notifier_unregister(priv->mdev, &priv->events_nb); 238 } 239 240 static inline void mlx5e_build_umr_wqe(struct mlx5e_rq *rq, 241 struct mlx5e_icosq *sq, 242 struct mlx5e_umr_wqe *wqe) 243 { 244 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 245 struct mlx5_wqe_umr_ctrl_seg *ucseg = &wqe->uctrl; 246 u8 ds_cnt = DIV_ROUND_UP(MLX5E_UMR_WQE_INLINE_SZ, MLX5_SEND_WQE_DS); 247 248 cseg->qpn_ds = cpu_to_be32((sq->sqn << MLX5_WQE_CTRL_QPN_SHIFT) | 249 ds_cnt); 250 cseg->fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE; 251 cseg->imm = rq->mkey_be; 252 253 ucseg->flags = MLX5_UMR_TRANSLATION_OFFSET_EN | MLX5_UMR_INLINE; 254 ucseg->xlt_octowords = 255 cpu_to_be16(MLX5_MTT_OCTW(MLX5_MPWRQ_PAGES_PER_WQE)); 256 ucseg->mkey_mask = cpu_to_be64(MLX5_MKEY_MASK_FREE); 257 } 258 259 static u32 mlx5e_rqwq_get_size(struct mlx5e_rq *rq) 260 { 261 switch (rq->wq_type) { 262 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 263 return mlx5_wq_ll_get_size(&rq->mpwqe.wq); 264 default: 265 return mlx5_wq_cyc_get_size(&rq->wqe.wq); 266 } 267 } 268 269 static u32 mlx5e_rqwq_get_cur_sz(struct mlx5e_rq *rq) 270 { 271 switch (rq->wq_type) { 272 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 273 return rq->mpwqe.wq.cur_sz; 274 default: 275 return rq->wqe.wq.cur_sz; 276 } 277 } 278 279 static int mlx5e_rq_alloc_mpwqe_info(struct mlx5e_rq *rq, 280 struct mlx5e_channel *c) 281 { 282 int wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 283 284 rq->mpwqe.info = kvzalloc_node(array_size(wq_sz, 285 sizeof(*rq->mpwqe.info)), 286 GFP_KERNEL, cpu_to_node(c->cpu)); 287 if (!rq->mpwqe.info) 288 return -ENOMEM; 289 290 mlx5e_build_umr_wqe(rq, &c->icosq, &rq->mpwqe.umr_wqe); 291 292 return 0; 293 } 294 295 static int mlx5e_create_umr_mkey(struct mlx5_core_dev *mdev, 296 u64 npages, u8 page_shift, 297 struct mlx5_core_mkey *umr_mkey) 298 { 299 int inlen = MLX5_ST_SZ_BYTES(create_mkey_in); 300 void *mkc; 301 u32 *in; 302 int err; 303 304 in = kvzalloc(inlen, GFP_KERNEL); 305 if (!in) 306 return -ENOMEM; 307 308 mkc = MLX5_ADDR_OF(create_mkey_in, in, memory_key_mkey_entry); 309 310 MLX5_SET(mkc, mkc, free, 1); 311 MLX5_SET(mkc, mkc, umr_en, 1); 312 MLX5_SET(mkc, mkc, lw, 1); 313 MLX5_SET(mkc, mkc, lr, 1); 314 MLX5_SET(mkc, mkc, access_mode_1_0, MLX5_MKC_ACCESS_MODE_MTT); 315 316 MLX5_SET(mkc, mkc, qpn, 0xffffff); 317 MLX5_SET(mkc, mkc, pd, mdev->mlx5e_res.pdn); 318 MLX5_SET64(mkc, mkc, len, npages << page_shift); 319 MLX5_SET(mkc, mkc, translations_octword_size, 320 MLX5_MTT_OCTW(npages)); 321 MLX5_SET(mkc, mkc, log_page_size, page_shift); 322 323 err = mlx5_core_create_mkey(mdev, umr_mkey, in, inlen); 324 325 kvfree(in); 326 return err; 327 } 328 329 static int mlx5e_create_rq_umr_mkey(struct mlx5_core_dev *mdev, struct mlx5e_rq *rq) 330 { 331 u64 num_mtts = MLX5E_REQUIRED_MTTS(mlx5_wq_ll_get_size(&rq->mpwqe.wq)); 332 333 return mlx5e_create_umr_mkey(mdev, num_mtts, PAGE_SHIFT, &rq->umr_mkey); 334 } 335 336 static inline u64 mlx5e_get_mpwqe_offset(struct mlx5e_rq *rq, u16 wqe_ix) 337 { 338 return (wqe_ix << MLX5E_LOG_ALIGNED_MPWQE_PPW) << PAGE_SHIFT; 339 } 340 341 static void mlx5e_init_frags_partition(struct mlx5e_rq *rq) 342 { 343 struct mlx5e_wqe_frag_info next_frag, *prev; 344 int i; 345 346 next_frag.di = &rq->wqe.di[0]; 347 next_frag.offset = 0; 348 prev = NULL; 349 350 for (i = 0; i < mlx5_wq_cyc_get_size(&rq->wqe.wq); i++) { 351 struct mlx5e_rq_frag_info *frag_info = &rq->wqe.info.arr[0]; 352 struct mlx5e_wqe_frag_info *frag = 353 &rq->wqe.frags[i << rq->wqe.info.log_num_frags]; 354 int f; 355 356 for (f = 0; f < rq->wqe.info.num_frags; f++, frag++) { 357 if (next_frag.offset + frag_info[f].frag_stride > PAGE_SIZE) { 358 next_frag.di++; 359 next_frag.offset = 0; 360 if (prev) 361 prev->last_in_page = true; 362 } 363 *frag = next_frag; 364 365 /* prepare next */ 366 next_frag.offset += frag_info[f].frag_stride; 367 prev = frag; 368 } 369 } 370 371 if (prev) 372 prev->last_in_page = true; 373 } 374 375 static int mlx5e_init_di_list(struct mlx5e_rq *rq, 376 int wq_sz, int cpu) 377 { 378 int len = wq_sz << rq->wqe.info.log_num_frags; 379 380 rq->wqe.di = kvzalloc_node(array_size(len, sizeof(*rq->wqe.di)), 381 GFP_KERNEL, cpu_to_node(cpu)); 382 if (!rq->wqe.di) 383 return -ENOMEM; 384 385 mlx5e_init_frags_partition(rq); 386 387 return 0; 388 } 389 390 static void mlx5e_free_di_list(struct mlx5e_rq *rq) 391 { 392 kvfree(rq->wqe.di); 393 } 394 395 static int mlx5e_alloc_rq(struct mlx5e_channel *c, 396 struct mlx5e_params *params, 397 struct mlx5e_rq_param *rqp, 398 struct mlx5e_rq *rq) 399 { 400 struct page_pool_params pp_params = { 0 }; 401 struct mlx5_core_dev *mdev = c->mdev; 402 void *rqc = rqp->rqc; 403 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 404 u32 pool_size; 405 int wq_sz; 406 int err; 407 int i; 408 409 rqp->wq.db_numa_node = cpu_to_node(c->cpu); 410 411 rq->wq_type = params->rq_wq_type; 412 rq->pdev = c->pdev; 413 rq->netdev = c->netdev; 414 rq->tstamp = c->tstamp; 415 rq->clock = &mdev->clock; 416 rq->channel = c; 417 rq->ix = c->ix; 418 rq->mdev = mdev; 419 rq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 420 rq->stats = &c->priv->channel_stats[c->ix].rq; 421 422 rq->xdp_prog = params->xdp_prog ? bpf_prog_inc(params->xdp_prog) : NULL; 423 if (IS_ERR(rq->xdp_prog)) { 424 err = PTR_ERR(rq->xdp_prog); 425 rq->xdp_prog = NULL; 426 goto err_rq_wq_destroy; 427 } 428 429 err = xdp_rxq_info_reg(&rq->xdp_rxq, rq->netdev, rq->ix); 430 if (err < 0) 431 goto err_rq_wq_destroy; 432 433 rq->buff.map_dir = rq->xdp_prog ? DMA_BIDIRECTIONAL : DMA_FROM_DEVICE; 434 rq->buff.headroom = mlx5e_get_rq_headroom(mdev, params); 435 pool_size = 1 << params->log_rq_mtu_frames; 436 437 switch (rq->wq_type) { 438 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 439 err = mlx5_wq_ll_create(mdev, &rqp->wq, rqc_wq, &rq->mpwqe.wq, 440 &rq->wq_ctrl); 441 if (err) 442 return err; 443 444 rq->mpwqe.wq.db = &rq->mpwqe.wq.db[MLX5_RCV_DBR]; 445 446 wq_sz = mlx5_wq_ll_get_size(&rq->mpwqe.wq); 447 448 pool_size = MLX5_MPWRQ_PAGES_PER_WQE << mlx5e_mpwqe_get_log_rq_size(params); 449 450 rq->post_wqes = mlx5e_post_rx_mpwqes; 451 rq->dealloc_wqe = mlx5e_dealloc_rx_mpwqe; 452 453 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe_mpwqe; 454 #ifdef CONFIG_MLX5_EN_IPSEC 455 if (MLX5_IPSEC_DEV(mdev)) { 456 err = -EINVAL; 457 netdev_err(c->netdev, "MPWQE RQ with IPSec offload not supported\n"); 458 goto err_rq_wq_destroy; 459 } 460 #endif 461 if (!rq->handle_rx_cqe) { 462 err = -EINVAL; 463 netdev_err(c->netdev, "RX handler of MPWQE RQ is not set, err %d\n", err); 464 goto err_rq_wq_destroy; 465 } 466 467 rq->mpwqe.skb_from_cqe_mpwrq = 468 mlx5e_rx_mpwqe_is_linear_skb(mdev, params) ? 469 mlx5e_skb_from_cqe_mpwrq_linear : 470 mlx5e_skb_from_cqe_mpwrq_nonlinear; 471 rq->mpwqe.log_stride_sz = mlx5e_mpwqe_get_log_stride_size(mdev, params); 472 rq->mpwqe.num_strides = BIT(mlx5e_mpwqe_get_log_num_strides(mdev, params)); 473 474 err = mlx5e_create_rq_umr_mkey(mdev, rq); 475 if (err) 476 goto err_rq_wq_destroy; 477 rq->mkey_be = cpu_to_be32(rq->umr_mkey.key); 478 479 err = mlx5e_rq_alloc_mpwqe_info(rq, c); 480 if (err) 481 goto err_free; 482 break; 483 default: /* MLX5_WQ_TYPE_CYCLIC */ 484 err = mlx5_wq_cyc_create(mdev, &rqp->wq, rqc_wq, &rq->wqe.wq, 485 &rq->wq_ctrl); 486 if (err) 487 return err; 488 489 rq->wqe.wq.db = &rq->wqe.wq.db[MLX5_RCV_DBR]; 490 491 wq_sz = mlx5_wq_cyc_get_size(&rq->wqe.wq); 492 493 rq->wqe.info = rqp->frags_info; 494 rq->wqe.frags = 495 kvzalloc_node(array_size(sizeof(*rq->wqe.frags), 496 (wq_sz << rq->wqe.info.log_num_frags)), 497 GFP_KERNEL, cpu_to_node(c->cpu)); 498 if (!rq->wqe.frags) { 499 err = -ENOMEM; 500 goto err_free; 501 } 502 503 err = mlx5e_init_di_list(rq, wq_sz, c->cpu); 504 if (err) 505 goto err_free; 506 rq->post_wqes = mlx5e_post_rx_wqes; 507 rq->dealloc_wqe = mlx5e_dealloc_rx_wqe; 508 509 #ifdef CONFIG_MLX5_EN_IPSEC 510 if (c->priv->ipsec) 511 rq->handle_rx_cqe = mlx5e_ipsec_handle_rx_cqe; 512 else 513 #endif 514 rq->handle_rx_cqe = c->priv->profile->rx_handlers.handle_rx_cqe; 515 if (!rq->handle_rx_cqe) { 516 err = -EINVAL; 517 netdev_err(c->netdev, "RX handler of RQ is not set, err %d\n", err); 518 goto err_free; 519 } 520 521 rq->wqe.skb_from_cqe = mlx5e_rx_is_linear_skb(params) ? 522 mlx5e_skb_from_cqe_linear : 523 mlx5e_skb_from_cqe_nonlinear; 524 rq->mkey_be = c->mkey_be; 525 } 526 527 /* Create a page_pool and register it with rxq */ 528 pp_params.order = 0; 529 pp_params.flags = 0; /* No-internal DMA mapping in page_pool */ 530 pp_params.pool_size = pool_size; 531 pp_params.nid = cpu_to_node(c->cpu); 532 pp_params.dev = c->pdev; 533 pp_params.dma_dir = rq->buff.map_dir; 534 535 /* page_pool can be used even when there is no rq->xdp_prog, 536 * given page_pool does not handle DMA mapping there is no 537 * required state to clear. And page_pool gracefully handle 538 * elevated refcnt. 539 */ 540 rq->page_pool = page_pool_create(&pp_params); 541 if (IS_ERR(rq->page_pool)) { 542 err = PTR_ERR(rq->page_pool); 543 rq->page_pool = NULL; 544 goto err_free; 545 } 546 err = xdp_rxq_info_reg_mem_model(&rq->xdp_rxq, 547 MEM_TYPE_PAGE_POOL, rq->page_pool); 548 if (err) 549 goto err_free; 550 551 for (i = 0; i < wq_sz; i++) { 552 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 553 struct mlx5e_rx_wqe_ll *wqe = 554 mlx5_wq_ll_get_wqe(&rq->mpwqe.wq, i); 555 u32 byte_count = 556 rq->mpwqe.num_strides << rq->mpwqe.log_stride_sz; 557 u64 dma_offset = mlx5e_get_mpwqe_offset(rq, i); 558 559 wqe->data[0].addr = cpu_to_be64(dma_offset + rq->buff.headroom); 560 wqe->data[0].byte_count = cpu_to_be32(byte_count); 561 wqe->data[0].lkey = rq->mkey_be; 562 } else { 563 struct mlx5e_rx_wqe_cyc *wqe = 564 mlx5_wq_cyc_get_wqe(&rq->wqe.wq, i); 565 int f; 566 567 for (f = 0; f < rq->wqe.info.num_frags; f++) { 568 u32 frag_size = rq->wqe.info.arr[f].frag_size | 569 MLX5_HW_START_PADDING; 570 571 wqe->data[f].byte_count = cpu_to_be32(frag_size); 572 wqe->data[f].lkey = rq->mkey_be; 573 } 574 /* check if num_frags is not a pow of two */ 575 if (rq->wqe.info.num_frags < (1 << rq->wqe.info.log_num_frags)) { 576 wqe->data[f].byte_count = 0; 577 wqe->data[f].lkey = cpu_to_be32(MLX5_INVALID_LKEY); 578 wqe->data[f].addr = 0; 579 } 580 } 581 } 582 583 INIT_WORK(&rq->dim.work, mlx5e_rx_dim_work); 584 585 switch (params->rx_cq_moderation.cq_period_mode) { 586 case MLX5_CQ_PERIOD_MODE_START_FROM_CQE: 587 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE; 588 break; 589 case MLX5_CQ_PERIOD_MODE_START_FROM_EQE: 590 default: 591 rq->dim.mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; 592 } 593 594 rq->page_cache.head = 0; 595 rq->page_cache.tail = 0; 596 597 return 0; 598 599 err_free: 600 switch (rq->wq_type) { 601 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 602 kvfree(rq->mpwqe.info); 603 mlx5_core_destroy_mkey(mdev, &rq->umr_mkey); 604 break; 605 default: /* MLX5_WQ_TYPE_CYCLIC */ 606 kvfree(rq->wqe.frags); 607 mlx5e_free_di_list(rq); 608 } 609 610 err_rq_wq_destroy: 611 if (rq->xdp_prog) 612 bpf_prog_put(rq->xdp_prog); 613 xdp_rxq_info_unreg(&rq->xdp_rxq); 614 if (rq->page_pool) 615 page_pool_destroy(rq->page_pool); 616 mlx5_wq_destroy(&rq->wq_ctrl); 617 618 return err; 619 } 620 621 static void mlx5e_free_rq(struct mlx5e_rq *rq) 622 { 623 int i; 624 625 if (rq->xdp_prog) 626 bpf_prog_put(rq->xdp_prog); 627 628 xdp_rxq_info_unreg(&rq->xdp_rxq); 629 if (rq->page_pool) 630 page_pool_destroy(rq->page_pool); 631 632 switch (rq->wq_type) { 633 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 634 kvfree(rq->mpwqe.info); 635 mlx5_core_destroy_mkey(rq->mdev, &rq->umr_mkey); 636 break; 637 default: /* MLX5_WQ_TYPE_CYCLIC */ 638 kvfree(rq->wqe.frags); 639 mlx5e_free_di_list(rq); 640 } 641 642 for (i = rq->page_cache.head; i != rq->page_cache.tail; 643 i = (i + 1) & (MLX5E_CACHE_SIZE - 1)) { 644 struct mlx5e_dma_info *dma_info = &rq->page_cache.page_cache[i]; 645 646 mlx5e_page_release(rq, dma_info, false); 647 } 648 mlx5_wq_destroy(&rq->wq_ctrl); 649 } 650 651 static int mlx5e_create_rq(struct mlx5e_rq *rq, 652 struct mlx5e_rq_param *param) 653 { 654 struct mlx5_core_dev *mdev = rq->mdev; 655 656 void *in; 657 void *rqc; 658 void *wq; 659 int inlen; 660 int err; 661 662 inlen = MLX5_ST_SZ_BYTES(create_rq_in) + 663 sizeof(u64) * rq->wq_ctrl.buf.npages; 664 in = kvzalloc(inlen, GFP_KERNEL); 665 if (!in) 666 return -ENOMEM; 667 668 rqc = MLX5_ADDR_OF(create_rq_in, in, ctx); 669 wq = MLX5_ADDR_OF(rqc, rqc, wq); 670 671 memcpy(rqc, param->rqc, sizeof(param->rqc)); 672 673 MLX5_SET(rqc, rqc, cqn, rq->cq.mcq.cqn); 674 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RST); 675 MLX5_SET(wq, wq, log_wq_pg_sz, rq->wq_ctrl.buf.page_shift - 676 MLX5_ADAPTER_PAGE_SHIFT); 677 MLX5_SET64(wq, wq, dbr_addr, rq->wq_ctrl.db.dma); 678 679 mlx5_fill_page_frag_array(&rq->wq_ctrl.buf, 680 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 681 682 err = mlx5_core_create_rq(mdev, in, inlen, &rq->rqn); 683 684 kvfree(in); 685 686 return err; 687 } 688 689 static int mlx5e_modify_rq_state(struct mlx5e_rq *rq, int curr_state, 690 int next_state) 691 { 692 struct mlx5_core_dev *mdev = rq->mdev; 693 694 void *in; 695 void *rqc; 696 int inlen; 697 int err; 698 699 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 700 in = kvzalloc(inlen, GFP_KERNEL); 701 if (!in) 702 return -ENOMEM; 703 704 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 705 706 MLX5_SET(modify_rq_in, in, rq_state, curr_state); 707 MLX5_SET(rqc, rqc, state, next_state); 708 709 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen); 710 711 kvfree(in); 712 713 return err; 714 } 715 716 static int mlx5e_modify_rq_scatter_fcs(struct mlx5e_rq *rq, bool enable) 717 { 718 struct mlx5e_channel *c = rq->channel; 719 struct mlx5e_priv *priv = c->priv; 720 struct mlx5_core_dev *mdev = priv->mdev; 721 722 void *in; 723 void *rqc; 724 int inlen; 725 int err; 726 727 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 728 in = kvzalloc(inlen, GFP_KERNEL); 729 if (!in) 730 return -ENOMEM; 731 732 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 733 734 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 735 MLX5_SET64(modify_rq_in, in, modify_bitmask, 736 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_SCATTER_FCS); 737 MLX5_SET(rqc, rqc, scatter_fcs, enable); 738 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 739 740 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen); 741 742 kvfree(in); 743 744 return err; 745 } 746 747 static int mlx5e_modify_rq_vsd(struct mlx5e_rq *rq, bool vsd) 748 { 749 struct mlx5e_channel *c = rq->channel; 750 struct mlx5_core_dev *mdev = c->mdev; 751 void *in; 752 void *rqc; 753 int inlen; 754 int err; 755 756 inlen = MLX5_ST_SZ_BYTES(modify_rq_in); 757 in = kvzalloc(inlen, GFP_KERNEL); 758 if (!in) 759 return -ENOMEM; 760 761 rqc = MLX5_ADDR_OF(modify_rq_in, in, ctx); 762 763 MLX5_SET(modify_rq_in, in, rq_state, MLX5_RQC_STATE_RDY); 764 MLX5_SET64(modify_rq_in, in, modify_bitmask, 765 MLX5_MODIFY_RQ_IN_MODIFY_BITMASK_VSD); 766 MLX5_SET(rqc, rqc, vsd, vsd); 767 MLX5_SET(rqc, rqc, state, MLX5_RQC_STATE_RDY); 768 769 err = mlx5_core_modify_rq(mdev, rq->rqn, in, inlen); 770 771 kvfree(in); 772 773 return err; 774 } 775 776 static void mlx5e_destroy_rq(struct mlx5e_rq *rq) 777 { 778 mlx5_core_destroy_rq(rq->mdev, rq->rqn); 779 } 780 781 static int mlx5e_wait_for_min_rx_wqes(struct mlx5e_rq *rq, int wait_time) 782 { 783 unsigned long exp_time = jiffies + msecs_to_jiffies(wait_time); 784 struct mlx5e_channel *c = rq->channel; 785 786 u16 min_wqes = mlx5_min_rx_wqes(rq->wq_type, mlx5e_rqwq_get_size(rq)); 787 788 do { 789 if (mlx5e_rqwq_get_cur_sz(rq) >= min_wqes) 790 return 0; 791 792 msleep(20); 793 } while (time_before(jiffies, exp_time)); 794 795 netdev_warn(c->netdev, "Failed to get min RX wqes on Channel[%d] RQN[0x%x] wq cur_sz(%d) min_rx_wqes(%d)\n", 796 c->ix, rq->rqn, mlx5e_rqwq_get_cur_sz(rq), min_wqes); 797 798 return -ETIMEDOUT; 799 } 800 801 static void mlx5e_free_rx_descs(struct mlx5e_rq *rq) 802 { 803 __be16 wqe_ix_be; 804 u16 wqe_ix; 805 806 if (rq->wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 807 struct mlx5_wq_ll *wq = &rq->mpwqe.wq; 808 u16 head = wq->head; 809 int i; 810 811 /* Outstanding UMR WQEs (in progress) start at wq->head */ 812 for (i = 0; i < rq->mpwqe.umr_in_progress; i++) { 813 rq->dealloc_wqe(rq, head); 814 head = mlx5_wq_ll_get_wqe_next_ix(wq, head); 815 } 816 817 while (!mlx5_wq_ll_is_empty(wq)) { 818 struct mlx5e_rx_wqe_ll *wqe; 819 820 wqe_ix_be = *wq->tail_next; 821 wqe_ix = be16_to_cpu(wqe_ix_be); 822 wqe = mlx5_wq_ll_get_wqe(wq, wqe_ix); 823 rq->dealloc_wqe(rq, wqe_ix); 824 mlx5_wq_ll_pop(wq, wqe_ix_be, 825 &wqe->next.next_wqe_index); 826 } 827 } else { 828 struct mlx5_wq_cyc *wq = &rq->wqe.wq; 829 830 while (!mlx5_wq_cyc_is_empty(wq)) { 831 wqe_ix = mlx5_wq_cyc_get_tail(wq); 832 rq->dealloc_wqe(rq, wqe_ix); 833 mlx5_wq_cyc_pop(wq); 834 } 835 } 836 837 } 838 839 static int mlx5e_open_rq(struct mlx5e_channel *c, 840 struct mlx5e_params *params, 841 struct mlx5e_rq_param *param, 842 struct mlx5e_rq *rq) 843 { 844 int err; 845 846 err = mlx5e_alloc_rq(c, params, param, rq); 847 if (err) 848 return err; 849 850 err = mlx5e_create_rq(rq, param); 851 if (err) 852 goto err_free_rq; 853 854 err = mlx5e_modify_rq_state(rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 855 if (err) 856 goto err_destroy_rq; 857 858 if (params->rx_dim_enabled) 859 __set_bit(MLX5E_RQ_STATE_AM, &c->rq.state); 860 861 /* We disable csum_complete when XDP is enabled since 862 * XDP programs might manipulate packets which will render 863 * skb->checksum incorrect. 864 */ 865 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE) || c->xdp) 866 __set_bit(MLX5E_RQ_STATE_NO_CSUM_COMPLETE, &c->rq.state); 867 868 return 0; 869 870 err_destroy_rq: 871 mlx5e_destroy_rq(rq); 872 err_free_rq: 873 mlx5e_free_rq(rq); 874 875 return err; 876 } 877 878 static void mlx5e_activate_rq(struct mlx5e_rq *rq) 879 { 880 set_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 881 mlx5e_trigger_irq(&rq->channel->icosq); 882 } 883 884 static void mlx5e_deactivate_rq(struct mlx5e_rq *rq) 885 { 886 clear_bit(MLX5E_RQ_STATE_ENABLED, &rq->state); 887 napi_synchronize(&rq->channel->napi); /* prevent mlx5e_post_rx_wqes */ 888 } 889 890 static void mlx5e_close_rq(struct mlx5e_rq *rq) 891 { 892 cancel_work_sync(&rq->dim.work); 893 mlx5e_destroy_rq(rq); 894 mlx5e_free_rx_descs(rq); 895 mlx5e_free_rq(rq); 896 } 897 898 static void mlx5e_free_xdpsq_db(struct mlx5e_xdpsq *sq) 899 { 900 kvfree(sq->db.xdpi_fifo.xi); 901 kvfree(sq->db.wqe_info); 902 } 903 904 static int mlx5e_alloc_xdpsq_fifo(struct mlx5e_xdpsq *sq, int numa) 905 { 906 struct mlx5e_xdp_info_fifo *xdpi_fifo = &sq->db.xdpi_fifo; 907 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 908 int dsegs_per_wq = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 909 910 xdpi_fifo->xi = kvzalloc_node(sizeof(*xdpi_fifo->xi) * dsegs_per_wq, 911 GFP_KERNEL, numa); 912 if (!xdpi_fifo->xi) 913 return -ENOMEM; 914 915 xdpi_fifo->pc = &sq->xdpi_fifo_pc; 916 xdpi_fifo->cc = &sq->xdpi_fifo_cc; 917 xdpi_fifo->mask = dsegs_per_wq - 1; 918 919 return 0; 920 } 921 922 static int mlx5e_alloc_xdpsq_db(struct mlx5e_xdpsq *sq, int numa) 923 { 924 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 925 int err; 926 927 sq->db.wqe_info = kvzalloc_node(sizeof(*sq->db.wqe_info) * wq_sz, 928 GFP_KERNEL, numa); 929 if (!sq->db.wqe_info) 930 return -ENOMEM; 931 932 err = mlx5e_alloc_xdpsq_fifo(sq, numa); 933 if (err) { 934 mlx5e_free_xdpsq_db(sq); 935 return err; 936 } 937 938 return 0; 939 } 940 941 static int mlx5e_alloc_xdpsq(struct mlx5e_channel *c, 942 struct mlx5e_params *params, 943 struct mlx5e_sq_param *param, 944 struct mlx5e_xdpsq *sq, 945 bool is_redirect) 946 { 947 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 948 struct mlx5_core_dev *mdev = c->mdev; 949 struct mlx5_wq_cyc *wq = &sq->wq; 950 int err; 951 952 sq->pdev = c->pdev; 953 sq->mkey_be = c->mkey_be; 954 sq->channel = c; 955 sq->uar_map = mdev->mlx5e_res.bfreg.map; 956 sq->min_inline_mode = params->tx_min_inline_mode; 957 sq->hw_mtu = MLX5E_SW2HW_MTU(params, params->sw_mtu); 958 sq->stats = is_redirect ? 959 &c->priv->channel_stats[c->ix].xdpsq : 960 &c->priv->channel_stats[c->ix].rq_xdpsq; 961 962 param->wq.db_numa_node = cpu_to_node(c->cpu); 963 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 964 if (err) 965 return err; 966 wq->db = &wq->db[MLX5_SND_DBR]; 967 968 err = mlx5e_alloc_xdpsq_db(sq, cpu_to_node(c->cpu)); 969 if (err) 970 goto err_sq_wq_destroy; 971 972 return 0; 973 974 err_sq_wq_destroy: 975 mlx5_wq_destroy(&sq->wq_ctrl); 976 977 return err; 978 } 979 980 static void mlx5e_free_xdpsq(struct mlx5e_xdpsq *sq) 981 { 982 mlx5e_free_xdpsq_db(sq); 983 mlx5_wq_destroy(&sq->wq_ctrl); 984 } 985 986 static void mlx5e_free_icosq_db(struct mlx5e_icosq *sq) 987 { 988 kvfree(sq->db.ico_wqe); 989 } 990 991 static int mlx5e_alloc_icosq_db(struct mlx5e_icosq *sq, int numa) 992 { 993 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 994 995 sq->db.ico_wqe = kvzalloc_node(array_size(wq_sz, 996 sizeof(*sq->db.ico_wqe)), 997 GFP_KERNEL, numa); 998 if (!sq->db.ico_wqe) 999 return -ENOMEM; 1000 1001 return 0; 1002 } 1003 1004 static int mlx5e_alloc_icosq(struct mlx5e_channel *c, 1005 struct mlx5e_sq_param *param, 1006 struct mlx5e_icosq *sq) 1007 { 1008 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1009 struct mlx5_core_dev *mdev = c->mdev; 1010 struct mlx5_wq_cyc *wq = &sq->wq; 1011 int err; 1012 1013 sq->channel = c; 1014 sq->uar_map = mdev->mlx5e_res.bfreg.map; 1015 1016 param->wq.db_numa_node = cpu_to_node(c->cpu); 1017 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1018 if (err) 1019 return err; 1020 wq->db = &wq->db[MLX5_SND_DBR]; 1021 1022 err = mlx5e_alloc_icosq_db(sq, cpu_to_node(c->cpu)); 1023 if (err) 1024 goto err_sq_wq_destroy; 1025 1026 return 0; 1027 1028 err_sq_wq_destroy: 1029 mlx5_wq_destroy(&sq->wq_ctrl); 1030 1031 return err; 1032 } 1033 1034 static void mlx5e_free_icosq(struct mlx5e_icosq *sq) 1035 { 1036 mlx5e_free_icosq_db(sq); 1037 mlx5_wq_destroy(&sq->wq_ctrl); 1038 } 1039 1040 static void mlx5e_free_txqsq_db(struct mlx5e_txqsq *sq) 1041 { 1042 kvfree(sq->db.wqe_info); 1043 kvfree(sq->db.dma_fifo); 1044 } 1045 1046 static int mlx5e_alloc_txqsq_db(struct mlx5e_txqsq *sq, int numa) 1047 { 1048 int wq_sz = mlx5_wq_cyc_get_size(&sq->wq); 1049 int df_sz = wq_sz * MLX5_SEND_WQEBB_NUM_DS; 1050 1051 sq->db.dma_fifo = kvzalloc_node(array_size(df_sz, 1052 sizeof(*sq->db.dma_fifo)), 1053 GFP_KERNEL, numa); 1054 sq->db.wqe_info = kvzalloc_node(array_size(wq_sz, 1055 sizeof(*sq->db.wqe_info)), 1056 GFP_KERNEL, numa); 1057 if (!sq->db.dma_fifo || !sq->db.wqe_info) { 1058 mlx5e_free_txqsq_db(sq); 1059 return -ENOMEM; 1060 } 1061 1062 sq->dma_fifo_mask = df_sz - 1; 1063 1064 return 0; 1065 } 1066 1067 static void mlx5e_tx_err_cqe_work(struct work_struct *recover_work); 1068 static int mlx5e_alloc_txqsq(struct mlx5e_channel *c, 1069 int txq_ix, 1070 struct mlx5e_params *params, 1071 struct mlx5e_sq_param *param, 1072 struct mlx5e_txqsq *sq, 1073 int tc) 1074 { 1075 void *sqc_wq = MLX5_ADDR_OF(sqc, param->sqc, wq); 1076 struct mlx5_core_dev *mdev = c->mdev; 1077 struct mlx5_wq_cyc *wq = &sq->wq; 1078 int err; 1079 1080 sq->pdev = c->pdev; 1081 sq->tstamp = c->tstamp; 1082 sq->clock = &mdev->clock; 1083 sq->mkey_be = c->mkey_be; 1084 sq->channel = c; 1085 sq->ch_ix = c->ix; 1086 sq->txq_ix = txq_ix; 1087 sq->uar_map = mdev->mlx5e_res.bfreg.map; 1088 sq->min_inline_mode = params->tx_min_inline_mode; 1089 sq->stats = &c->priv->channel_stats[c->ix].sq[tc]; 1090 INIT_WORK(&sq->recover_work, mlx5e_tx_err_cqe_work); 1091 if (MLX5_IPSEC_DEV(c->priv->mdev)) 1092 set_bit(MLX5E_SQ_STATE_IPSEC, &sq->state); 1093 if (mlx5_accel_is_tls_device(c->priv->mdev)) 1094 set_bit(MLX5E_SQ_STATE_TLS, &sq->state); 1095 1096 param->wq.db_numa_node = cpu_to_node(c->cpu); 1097 err = mlx5_wq_cyc_create(mdev, ¶m->wq, sqc_wq, wq, &sq->wq_ctrl); 1098 if (err) 1099 return err; 1100 wq->db = &wq->db[MLX5_SND_DBR]; 1101 1102 err = mlx5e_alloc_txqsq_db(sq, cpu_to_node(c->cpu)); 1103 if (err) 1104 goto err_sq_wq_destroy; 1105 1106 INIT_WORK(&sq->dim.work, mlx5e_tx_dim_work); 1107 sq->dim.mode = params->tx_cq_moderation.cq_period_mode; 1108 1109 return 0; 1110 1111 err_sq_wq_destroy: 1112 mlx5_wq_destroy(&sq->wq_ctrl); 1113 1114 return err; 1115 } 1116 1117 static void mlx5e_free_txqsq(struct mlx5e_txqsq *sq) 1118 { 1119 mlx5e_free_txqsq_db(sq); 1120 mlx5_wq_destroy(&sq->wq_ctrl); 1121 } 1122 1123 struct mlx5e_create_sq_param { 1124 struct mlx5_wq_ctrl *wq_ctrl; 1125 u32 cqn; 1126 u32 tisn; 1127 u8 tis_lst_sz; 1128 u8 min_inline_mode; 1129 }; 1130 1131 static int mlx5e_create_sq(struct mlx5_core_dev *mdev, 1132 struct mlx5e_sq_param *param, 1133 struct mlx5e_create_sq_param *csp, 1134 u32 *sqn) 1135 { 1136 void *in; 1137 void *sqc; 1138 void *wq; 1139 int inlen; 1140 int err; 1141 1142 inlen = MLX5_ST_SZ_BYTES(create_sq_in) + 1143 sizeof(u64) * csp->wq_ctrl->buf.npages; 1144 in = kvzalloc(inlen, GFP_KERNEL); 1145 if (!in) 1146 return -ENOMEM; 1147 1148 sqc = MLX5_ADDR_OF(create_sq_in, in, ctx); 1149 wq = MLX5_ADDR_OF(sqc, sqc, wq); 1150 1151 memcpy(sqc, param->sqc, sizeof(param->sqc)); 1152 MLX5_SET(sqc, sqc, tis_lst_sz, csp->tis_lst_sz); 1153 MLX5_SET(sqc, sqc, tis_num_0, csp->tisn); 1154 MLX5_SET(sqc, sqc, cqn, csp->cqn); 1155 1156 if (MLX5_CAP_ETH(mdev, wqe_inline_mode) == MLX5_CAP_INLINE_MODE_VPORT_CONTEXT) 1157 MLX5_SET(sqc, sqc, min_wqe_inline_mode, csp->min_inline_mode); 1158 1159 MLX5_SET(sqc, sqc, state, MLX5_SQC_STATE_RST); 1160 MLX5_SET(sqc, sqc, flush_in_error_en, 1); 1161 1162 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 1163 MLX5_SET(wq, wq, uar_page, mdev->mlx5e_res.bfreg.index); 1164 MLX5_SET(wq, wq, log_wq_pg_sz, csp->wq_ctrl->buf.page_shift - 1165 MLX5_ADAPTER_PAGE_SHIFT); 1166 MLX5_SET64(wq, wq, dbr_addr, csp->wq_ctrl->db.dma); 1167 1168 mlx5_fill_page_frag_array(&csp->wq_ctrl->buf, 1169 (__be64 *)MLX5_ADDR_OF(wq, wq, pas)); 1170 1171 err = mlx5_core_create_sq(mdev, in, inlen, sqn); 1172 1173 kvfree(in); 1174 1175 return err; 1176 } 1177 1178 int mlx5e_modify_sq(struct mlx5_core_dev *mdev, u32 sqn, 1179 struct mlx5e_modify_sq_param *p) 1180 { 1181 void *in; 1182 void *sqc; 1183 int inlen; 1184 int err; 1185 1186 inlen = MLX5_ST_SZ_BYTES(modify_sq_in); 1187 in = kvzalloc(inlen, GFP_KERNEL); 1188 if (!in) 1189 return -ENOMEM; 1190 1191 sqc = MLX5_ADDR_OF(modify_sq_in, in, ctx); 1192 1193 MLX5_SET(modify_sq_in, in, sq_state, p->curr_state); 1194 MLX5_SET(sqc, sqc, state, p->next_state); 1195 if (p->rl_update && p->next_state == MLX5_SQC_STATE_RDY) { 1196 MLX5_SET64(modify_sq_in, in, modify_bitmask, 1); 1197 MLX5_SET(sqc, sqc, packet_pacing_rate_limit_index, p->rl_index); 1198 } 1199 1200 err = mlx5_core_modify_sq(mdev, sqn, in, inlen); 1201 1202 kvfree(in); 1203 1204 return err; 1205 } 1206 1207 static void mlx5e_destroy_sq(struct mlx5_core_dev *mdev, u32 sqn) 1208 { 1209 mlx5_core_destroy_sq(mdev, sqn); 1210 } 1211 1212 static int mlx5e_create_sq_rdy(struct mlx5_core_dev *mdev, 1213 struct mlx5e_sq_param *param, 1214 struct mlx5e_create_sq_param *csp, 1215 u32 *sqn) 1216 { 1217 struct mlx5e_modify_sq_param msp = {0}; 1218 int err; 1219 1220 err = mlx5e_create_sq(mdev, param, csp, sqn); 1221 if (err) 1222 return err; 1223 1224 msp.curr_state = MLX5_SQC_STATE_RST; 1225 msp.next_state = MLX5_SQC_STATE_RDY; 1226 err = mlx5e_modify_sq(mdev, *sqn, &msp); 1227 if (err) 1228 mlx5e_destroy_sq(mdev, *sqn); 1229 1230 return err; 1231 } 1232 1233 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1234 struct mlx5e_txqsq *sq, u32 rate); 1235 1236 static int mlx5e_open_txqsq(struct mlx5e_channel *c, 1237 u32 tisn, 1238 int txq_ix, 1239 struct mlx5e_params *params, 1240 struct mlx5e_sq_param *param, 1241 struct mlx5e_txqsq *sq, 1242 int tc) 1243 { 1244 struct mlx5e_create_sq_param csp = {}; 1245 u32 tx_rate; 1246 int err; 1247 1248 err = mlx5e_alloc_txqsq(c, txq_ix, params, param, sq, tc); 1249 if (err) 1250 return err; 1251 1252 csp.tisn = tisn; 1253 csp.tis_lst_sz = 1; 1254 csp.cqn = sq->cq.mcq.cqn; 1255 csp.wq_ctrl = &sq->wq_ctrl; 1256 csp.min_inline_mode = sq->min_inline_mode; 1257 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1258 if (err) 1259 goto err_free_txqsq; 1260 1261 tx_rate = c->priv->tx_rates[sq->txq_ix]; 1262 if (tx_rate) 1263 mlx5e_set_sq_maxrate(c->netdev, sq, tx_rate); 1264 1265 if (params->tx_dim_enabled) 1266 sq->state |= BIT(MLX5E_SQ_STATE_AM); 1267 1268 return 0; 1269 1270 err_free_txqsq: 1271 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1272 mlx5e_free_txqsq(sq); 1273 1274 return err; 1275 } 1276 1277 void mlx5e_activate_txqsq(struct mlx5e_txqsq *sq) 1278 { 1279 sq->txq = netdev_get_tx_queue(sq->channel->netdev, sq->txq_ix); 1280 clear_bit(MLX5E_SQ_STATE_RECOVERING, &sq->state); 1281 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1282 netdev_tx_reset_queue(sq->txq); 1283 netif_tx_start_queue(sq->txq); 1284 } 1285 1286 void mlx5e_tx_disable_queue(struct netdev_queue *txq) 1287 { 1288 __netif_tx_lock_bh(txq); 1289 netif_tx_stop_queue(txq); 1290 __netif_tx_unlock_bh(txq); 1291 } 1292 1293 static void mlx5e_deactivate_txqsq(struct mlx5e_txqsq *sq) 1294 { 1295 struct mlx5e_channel *c = sq->channel; 1296 struct mlx5_wq_cyc *wq = &sq->wq; 1297 1298 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1299 /* prevent netif_tx_wake_queue */ 1300 napi_synchronize(&c->napi); 1301 1302 mlx5e_tx_disable_queue(sq->txq); 1303 1304 /* last doorbell out, godspeed .. */ 1305 if (mlx5e_wqc_has_room_for(wq, sq->cc, sq->pc, 1)) { 1306 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 1307 struct mlx5e_tx_wqe *nop; 1308 1309 sq->db.wqe_info[pi].skb = NULL; 1310 nop = mlx5e_post_nop(wq, sq->sqn, &sq->pc); 1311 mlx5e_notify_hw(wq, sq->pc, sq->uar_map, &nop->ctrl); 1312 } 1313 } 1314 1315 static void mlx5e_close_txqsq(struct mlx5e_txqsq *sq) 1316 { 1317 struct mlx5e_channel *c = sq->channel; 1318 struct mlx5_core_dev *mdev = c->mdev; 1319 struct mlx5_rate_limit rl = {0}; 1320 1321 cancel_work_sync(&sq->dim.work); 1322 cancel_work_sync(&sq->recover_work); 1323 mlx5e_destroy_sq(mdev, sq->sqn); 1324 if (sq->rate_limit) { 1325 rl.rate = sq->rate_limit; 1326 mlx5_rl_remove_rate(mdev, &rl); 1327 } 1328 mlx5e_free_txqsq_descs(sq); 1329 mlx5e_free_txqsq(sq); 1330 } 1331 1332 static void mlx5e_tx_err_cqe_work(struct work_struct *recover_work) 1333 { 1334 struct mlx5e_txqsq *sq = container_of(recover_work, struct mlx5e_txqsq, 1335 recover_work); 1336 1337 mlx5e_tx_reporter_err_cqe(sq); 1338 } 1339 1340 static int mlx5e_open_icosq(struct mlx5e_channel *c, 1341 struct mlx5e_params *params, 1342 struct mlx5e_sq_param *param, 1343 struct mlx5e_icosq *sq) 1344 { 1345 struct mlx5e_create_sq_param csp = {}; 1346 int err; 1347 1348 err = mlx5e_alloc_icosq(c, param, sq); 1349 if (err) 1350 return err; 1351 1352 csp.cqn = sq->cq.mcq.cqn; 1353 csp.wq_ctrl = &sq->wq_ctrl; 1354 csp.min_inline_mode = params->tx_min_inline_mode; 1355 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1356 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1357 if (err) 1358 goto err_free_icosq; 1359 1360 return 0; 1361 1362 err_free_icosq: 1363 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1364 mlx5e_free_icosq(sq); 1365 1366 return err; 1367 } 1368 1369 static void mlx5e_close_icosq(struct mlx5e_icosq *sq) 1370 { 1371 struct mlx5e_channel *c = sq->channel; 1372 1373 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1374 napi_synchronize(&c->napi); 1375 1376 mlx5e_destroy_sq(c->mdev, sq->sqn); 1377 mlx5e_free_icosq(sq); 1378 } 1379 1380 static int mlx5e_open_xdpsq(struct mlx5e_channel *c, 1381 struct mlx5e_params *params, 1382 struct mlx5e_sq_param *param, 1383 struct mlx5e_xdpsq *sq, 1384 bool is_redirect) 1385 { 1386 struct mlx5e_create_sq_param csp = {}; 1387 int err; 1388 1389 err = mlx5e_alloc_xdpsq(c, params, param, sq, is_redirect); 1390 if (err) 1391 return err; 1392 1393 csp.tis_lst_sz = 1; 1394 csp.tisn = c->priv->tisn[0]; /* tc = 0 */ 1395 csp.cqn = sq->cq.mcq.cqn; 1396 csp.wq_ctrl = &sq->wq_ctrl; 1397 csp.min_inline_mode = sq->min_inline_mode; 1398 set_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1399 err = mlx5e_create_sq_rdy(c->mdev, param, &csp, &sq->sqn); 1400 if (err) 1401 goto err_free_xdpsq; 1402 1403 mlx5e_set_xmit_fp(sq, param->is_mpw); 1404 1405 if (!param->is_mpw) { 1406 unsigned int ds_cnt = MLX5E_XDP_TX_DS_COUNT; 1407 unsigned int inline_hdr_sz = 0; 1408 int i; 1409 1410 if (sq->min_inline_mode != MLX5_INLINE_MODE_NONE) { 1411 inline_hdr_sz = MLX5E_XDP_MIN_INLINE; 1412 ds_cnt++; 1413 } 1414 1415 /* Pre initialize fixed WQE fields */ 1416 for (i = 0; i < mlx5_wq_cyc_get_size(&sq->wq); i++) { 1417 struct mlx5e_xdp_wqe_info *wi = &sq->db.wqe_info[i]; 1418 struct mlx5e_tx_wqe *wqe = mlx5_wq_cyc_get_wqe(&sq->wq, i); 1419 struct mlx5_wqe_ctrl_seg *cseg = &wqe->ctrl; 1420 struct mlx5_wqe_eth_seg *eseg = &wqe->eth; 1421 struct mlx5_wqe_data_seg *dseg; 1422 1423 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 1424 eseg->inline_hdr.sz = cpu_to_be16(inline_hdr_sz); 1425 1426 dseg = (struct mlx5_wqe_data_seg *)cseg + (ds_cnt - 1); 1427 dseg->lkey = sq->mkey_be; 1428 1429 wi->num_wqebbs = 1; 1430 wi->num_pkts = 1; 1431 } 1432 } 1433 1434 return 0; 1435 1436 err_free_xdpsq: 1437 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1438 mlx5e_free_xdpsq(sq); 1439 1440 return err; 1441 } 1442 1443 static void mlx5e_close_xdpsq(struct mlx5e_xdpsq *sq, struct mlx5e_rq *rq) 1444 { 1445 struct mlx5e_channel *c = sq->channel; 1446 1447 clear_bit(MLX5E_SQ_STATE_ENABLED, &sq->state); 1448 napi_synchronize(&c->napi); 1449 1450 mlx5e_destroy_sq(c->mdev, sq->sqn); 1451 mlx5e_free_xdpsq_descs(sq, rq); 1452 mlx5e_free_xdpsq(sq); 1453 } 1454 1455 static int mlx5e_alloc_cq_common(struct mlx5_core_dev *mdev, 1456 struct mlx5e_cq_param *param, 1457 struct mlx5e_cq *cq) 1458 { 1459 struct mlx5_core_cq *mcq = &cq->mcq; 1460 int eqn_not_used; 1461 unsigned int irqn; 1462 int err; 1463 u32 i; 1464 1465 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn_not_used, &irqn); 1466 if (err) 1467 return err; 1468 1469 err = mlx5_cqwq_create(mdev, ¶m->wq, param->cqc, &cq->wq, 1470 &cq->wq_ctrl); 1471 if (err) 1472 return err; 1473 1474 mcq->cqe_sz = 64; 1475 mcq->set_ci_db = cq->wq_ctrl.db.db; 1476 mcq->arm_db = cq->wq_ctrl.db.db + 1; 1477 *mcq->set_ci_db = 0; 1478 *mcq->arm_db = 0; 1479 mcq->vector = param->eq_ix; 1480 mcq->comp = mlx5e_completion_event; 1481 mcq->event = mlx5e_cq_error_event; 1482 mcq->irqn = irqn; 1483 1484 for (i = 0; i < mlx5_cqwq_get_size(&cq->wq); i++) { 1485 struct mlx5_cqe64 *cqe = mlx5_cqwq_get_wqe(&cq->wq, i); 1486 1487 cqe->op_own = 0xf1; 1488 } 1489 1490 cq->mdev = mdev; 1491 1492 return 0; 1493 } 1494 1495 static int mlx5e_alloc_cq(struct mlx5e_channel *c, 1496 struct mlx5e_cq_param *param, 1497 struct mlx5e_cq *cq) 1498 { 1499 struct mlx5_core_dev *mdev = c->priv->mdev; 1500 int err; 1501 1502 param->wq.buf_numa_node = cpu_to_node(c->cpu); 1503 param->wq.db_numa_node = cpu_to_node(c->cpu); 1504 param->eq_ix = c->ix; 1505 1506 err = mlx5e_alloc_cq_common(mdev, param, cq); 1507 1508 cq->napi = &c->napi; 1509 cq->channel = c; 1510 1511 return err; 1512 } 1513 1514 static void mlx5e_free_cq(struct mlx5e_cq *cq) 1515 { 1516 mlx5_wq_destroy(&cq->wq_ctrl); 1517 } 1518 1519 static int mlx5e_create_cq(struct mlx5e_cq *cq, struct mlx5e_cq_param *param) 1520 { 1521 struct mlx5_core_dev *mdev = cq->mdev; 1522 struct mlx5_core_cq *mcq = &cq->mcq; 1523 1524 void *in; 1525 void *cqc; 1526 int inlen; 1527 unsigned int irqn_not_used; 1528 int eqn; 1529 int err; 1530 1531 err = mlx5_vector2eqn(mdev, param->eq_ix, &eqn, &irqn_not_used); 1532 if (err) 1533 return err; 1534 1535 inlen = MLX5_ST_SZ_BYTES(create_cq_in) + 1536 sizeof(u64) * cq->wq_ctrl.buf.npages; 1537 in = kvzalloc(inlen, GFP_KERNEL); 1538 if (!in) 1539 return -ENOMEM; 1540 1541 cqc = MLX5_ADDR_OF(create_cq_in, in, cq_context); 1542 1543 memcpy(cqc, param->cqc, sizeof(param->cqc)); 1544 1545 mlx5_fill_page_frag_array(&cq->wq_ctrl.buf, 1546 (__be64 *)MLX5_ADDR_OF(create_cq_in, in, pas)); 1547 1548 MLX5_SET(cqc, cqc, cq_period_mode, param->cq_period_mode); 1549 MLX5_SET(cqc, cqc, c_eqn, eqn); 1550 MLX5_SET(cqc, cqc, uar_page, mdev->priv.uar->index); 1551 MLX5_SET(cqc, cqc, log_page_size, cq->wq_ctrl.buf.page_shift - 1552 MLX5_ADAPTER_PAGE_SHIFT); 1553 MLX5_SET64(cqc, cqc, dbr_addr, cq->wq_ctrl.db.dma); 1554 1555 err = mlx5_core_create_cq(mdev, mcq, in, inlen); 1556 1557 kvfree(in); 1558 1559 if (err) 1560 return err; 1561 1562 mlx5e_cq_arm(cq); 1563 1564 return 0; 1565 } 1566 1567 static void mlx5e_destroy_cq(struct mlx5e_cq *cq) 1568 { 1569 mlx5_core_destroy_cq(cq->mdev, &cq->mcq); 1570 } 1571 1572 static int mlx5e_open_cq(struct mlx5e_channel *c, 1573 struct net_dim_cq_moder moder, 1574 struct mlx5e_cq_param *param, 1575 struct mlx5e_cq *cq) 1576 { 1577 struct mlx5_core_dev *mdev = c->mdev; 1578 int err; 1579 1580 err = mlx5e_alloc_cq(c, param, cq); 1581 if (err) 1582 return err; 1583 1584 err = mlx5e_create_cq(cq, param); 1585 if (err) 1586 goto err_free_cq; 1587 1588 if (MLX5_CAP_GEN(mdev, cq_moderation)) 1589 mlx5_core_modify_cq_moderation(mdev, &cq->mcq, moder.usec, moder.pkts); 1590 return 0; 1591 1592 err_free_cq: 1593 mlx5e_free_cq(cq); 1594 1595 return err; 1596 } 1597 1598 static void mlx5e_close_cq(struct mlx5e_cq *cq) 1599 { 1600 mlx5e_destroy_cq(cq); 1601 mlx5e_free_cq(cq); 1602 } 1603 1604 static int mlx5e_open_tx_cqs(struct mlx5e_channel *c, 1605 struct mlx5e_params *params, 1606 struct mlx5e_channel_param *cparam) 1607 { 1608 int err; 1609 int tc; 1610 1611 for (tc = 0; tc < c->num_tc; tc++) { 1612 err = mlx5e_open_cq(c, params->tx_cq_moderation, 1613 &cparam->tx_cq, &c->sq[tc].cq); 1614 if (err) 1615 goto err_close_tx_cqs; 1616 } 1617 1618 return 0; 1619 1620 err_close_tx_cqs: 1621 for (tc--; tc >= 0; tc--) 1622 mlx5e_close_cq(&c->sq[tc].cq); 1623 1624 return err; 1625 } 1626 1627 static void mlx5e_close_tx_cqs(struct mlx5e_channel *c) 1628 { 1629 int tc; 1630 1631 for (tc = 0; tc < c->num_tc; tc++) 1632 mlx5e_close_cq(&c->sq[tc].cq); 1633 } 1634 1635 static int mlx5e_open_sqs(struct mlx5e_channel *c, 1636 struct mlx5e_params *params, 1637 struct mlx5e_channel_param *cparam) 1638 { 1639 struct mlx5e_priv *priv = c->priv; 1640 int err, tc, max_nch = mlx5e_get_netdev_max_channels(priv->netdev); 1641 1642 for (tc = 0; tc < params->num_tc; tc++) { 1643 int txq_ix = c->ix + tc * max_nch; 1644 1645 err = mlx5e_open_txqsq(c, c->priv->tisn[tc], txq_ix, 1646 params, &cparam->sq, &c->sq[tc], tc); 1647 if (err) 1648 goto err_close_sqs; 1649 } 1650 1651 return 0; 1652 1653 err_close_sqs: 1654 for (tc--; tc >= 0; tc--) 1655 mlx5e_close_txqsq(&c->sq[tc]); 1656 1657 return err; 1658 } 1659 1660 static void mlx5e_close_sqs(struct mlx5e_channel *c) 1661 { 1662 int tc; 1663 1664 for (tc = 0; tc < c->num_tc; tc++) 1665 mlx5e_close_txqsq(&c->sq[tc]); 1666 } 1667 1668 static int mlx5e_set_sq_maxrate(struct net_device *dev, 1669 struct mlx5e_txqsq *sq, u32 rate) 1670 { 1671 struct mlx5e_priv *priv = netdev_priv(dev); 1672 struct mlx5_core_dev *mdev = priv->mdev; 1673 struct mlx5e_modify_sq_param msp = {0}; 1674 struct mlx5_rate_limit rl = {0}; 1675 u16 rl_index = 0; 1676 int err; 1677 1678 if (rate == sq->rate_limit) 1679 /* nothing to do */ 1680 return 0; 1681 1682 if (sq->rate_limit) { 1683 rl.rate = sq->rate_limit; 1684 /* remove current rl index to free space to next ones */ 1685 mlx5_rl_remove_rate(mdev, &rl); 1686 } 1687 1688 sq->rate_limit = 0; 1689 1690 if (rate) { 1691 rl.rate = rate; 1692 err = mlx5_rl_add_rate(mdev, &rl_index, &rl); 1693 if (err) { 1694 netdev_err(dev, "Failed configuring rate %u: %d\n", 1695 rate, err); 1696 return err; 1697 } 1698 } 1699 1700 msp.curr_state = MLX5_SQC_STATE_RDY; 1701 msp.next_state = MLX5_SQC_STATE_RDY; 1702 msp.rl_index = rl_index; 1703 msp.rl_update = true; 1704 err = mlx5e_modify_sq(mdev, sq->sqn, &msp); 1705 if (err) { 1706 netdev_err(dev, "Failed configuring rate %u: %d\n", 1707 rate, err); 1708 /* remove the rate from the table */ 1709 if (rate) 1710 mlx5_rl_remove_rate(mdev, &rl); 1711 return err; 1712 } 1713 1714 sq->rate_limit = rate; 1715 return 0; 1716 } 1717 1718 static int mlx5e_set_tx_maxrate(struct net_device *dev, int index, u32 rate) 1719 { 1720 struct mlx5e_priv *priv = netdev_priv(dev); 1721 struct mlx5_core_dev *mdev = priv->mdev; 1722 struct mlx5e_txqsq *sq = priv->txq2sq[index]; 1723 int err = 0; 1724 1725 if (!mlx5_rl_is_supported(mdev)) { 1726 netdev_err(dev, "Rate limiting is not supported on this device\n"); 1727 return -EINVAL; 1728 } 1729 1730 /* rate is given in Mb/sec, HW config is in Kb/sec */ 1731 rate = rate << 10; 1732 1733 /* Check whether rate in valid range, 0 is always valid */ 1734 if (rate && !mlx5_rl_is_in_range(mdev, rate)) { 1735 netdev_err(dev, "TX rate %u, is not in range\n", rate); 1736 return -ERANGE; 1737 } 1738 1739 mutex_lock(&priv->state_lock); 1740 if (test_bit(MLX5E_STATE_OPENED, &priv->state)) 1741 err = mlx5e_set_sq_maxrate(dev, sq, rate); 1742 if (!err) 1743 priv->tx_rates[index] = rate; 1744 mutex_unlock(&priv->state_lock); 1745 1746 return err; 1747 } 1748 1749 static int mlx5e_alloc_xps_cpumask(struct mlx5e_channel *c, 1750 struct mlx5e_params *params) 1751 { 1752 int num_comp_vectors = mlx5_comp_vectors_count(c->mdev); 1753 int irq; 1754 1755 if (!zalloc_cpumask_var(&c->xps_cpumask, GFP_KERNEL)) 1756 return -ENOMEM; 1757 1758 for (irq = c->ix; irq < num_comp_vectors; irq += params->num_channels) { 1759 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(c->mdev, irq)); 1760 1761 cpumask_set_cpu(cpu, c->xps_cpumask); 1762 } 1763 1764 return 0; 1765 } 1766 1767 static void mlx5e_free_xps_cpumask(struct mlx5e_channel *c) 1768 { 1769 free_cpumask_var(c->xps_cpumask); 1770 } 1771 1772 static int mlx5e_open_channel(struct mlx5e_priv *priv, int ix, 1773 struct mlx5e_params *params, 1774 struct mlx5e_channel_param *cparam, 1775 struct mlx5e_channel **cp) 1776 { 1777 int cpu = cpumask_first(mlx5_comp_irq_get_affinity_mask(priv->mdev, ix)); 1778 struct net_dim_cq_moder icocq_moder = {0, 0}; 1779 struct net_device *netdev = priv->netdev; 1780 struct mlx5e_channel *c; 1781 unsigned int irq; 1782 int err; 1783 int eqn; 1784 1785 err = mlx5_vector2eqn(priv->mdev, ix, &eqn, &irq); 1786 if (err) 1787 return err; 1788 1789 c = kvzalloc_node(sizeof(*c), GFP_KERNEL, cpu_to_node(cpu)); 1790 if (!c) 1791 return -ENOMEM; 1792 1793 c->priv = priv; 1794 c->mdev = priv->mdev; 1795 c->tstamp = &priv->tstamp; 1796 c->ix = ix; 1797 c->cpu = cpu; 1798 c->pdev = priv->mdev->device; 1799 c->netdev = priv->netdev; 1800 c->mkey_be = cpu_to_be32(priv->mdev->mlx5e_res.mkey.key); 1801 c->num_tc = params->num_tc; 1802 c->xdp = !!params->xdp_prog; 1803 c->stats = &priv->channel_stats[ix].ch; 1804 c->irq_desc = irq_to_desc(irq); 1805 1806 err = mlx5e_alloc_xps_cpumask(c, params); 1807 if (err) 1808 goto err_free_channel; 1809 1810 netif_napi_add(netdev, &c->napi, mlx5e_napi_poll, 64); 1811 1812 err = mlx5e_open_cq(c, icocq_moder, &cparam->icosq_cq, &c->icosq.cq); 1813 if (err) 1814 goto err_napi_del; 1815 1816 err = mlx5e_open_tx_cqs(c, params, cparam); 1817 if (err) 1818 goto err_close_icosq_cq; 1819 1820 err = mlx5e_open_cq(c, params->tx_cq_moderation, &cparam->tx_cq, &c->xdpsq.cq); 1821 if (err) 1822 goto err_close_tx_cqs; 1823 1824 err = mlx5e_open_cq(c, params->rx_cq_moderation, &cparam->rx_cq, &c->rq.cq); 1825 if (err) 1826 goto err_close_xdp_tx_cqs; 1827 1828 /* XDP SQ CQ params are same as normal TXQ sq CQ params */ 1829 err = c->xdp ? mlx5e_open_cq(c, params->tx_cq_moderation, 1830 &cparam->tx_cq, &c->rq.xdpsq.cq) : 0; 1831 if (err) 1832 goto err_close_rx_cq; 1833 1834 napi_enable(&c->napi); 1835 1836 err = mlx5e_open_icosq(c, params, &cparam->icosq, &c->icosq); 1837 if (err) 1838 goto err_disable_napi; 1839 1840 err = mlx5e_open_sqs(c, params, cparam); 1841 if (err) 1842 goto err_close_icosq; 1843 1844 err = c->xdp ? mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->rq.xdpsq, false) : 0; 1845 if (err) 1846 goto err_close_sqs; 1847 1848 err = mlx5e_open_rq(c, params, &cparam->rq, &c->rq); 1849 if (err) 1850 goto err_close_xdp_sq; 1851 1852 err = mlx5e_open_xdpsq(c, params, &cparam->xdp_sq, &c->xdpsq, true); 1853 if (err) 1854 goto err_close_rq; 1855 1856 *cp = c; 1857 1858 return 0; 1859 1860 err_close_rq: 1861 mlx5e_close_rq(&c->rq); 1862 1863 err_close_xdp_sq: 1864 if (c->xdp) 1865 mlx5e_close_xdpsq(&c->rq.xdpsq, &c->rq); 1866 1867 err_close_sqs: 1868 mlx5e_close_sqs(c); 1869 1870 err_close_icosq: 1871 mlx5e_close_icosq(&c->icosq); 1872 1873 err_disable_napi: 1874 napi_disable(&c->napi); 1875 if (c->xdp) 1876 mlx5e_close_cq(&c->rq.xdpsq.cq); 1877 1878 err_close_rx_cq: 1879 mlx5e_close_cq(&c->rq.cq); 1880 1881 err_close_xdp_tx_cqs: 1882 mlx5e_close_cq(&c->xdpsq.cq); 1883 1884 err_close_tx_cqs: 1885 mlx5e_close_tx_cqs(c); 1886 1887 err_close_icosq_cq: 1888 mlx5e_close_cq(&c->icosq.cq); 1889 1890 err_napi_del: 1891 netif_napi_del(&c->napi); 1892 mlx5e_free_xps_cpumask(c); 1893 1894 err_free_channel: 1895 kvfree(c); 1896 1897 return err; 1898 } 1899 1900 static void mlx5e_activate_channel(struct mlx5e_channel *c) 1901 { 1902 int tc; 1903 1904 for (tc = 0; tc < c->num_tc; tc++) 1905 mlx5e_activate_txqsq(&c->sq[tc]); 1906 mlx5e_activate_rq(&c->rq); 1907 netif_set_xps_queue(c->netdev, c->xps_cpumask, c->ix); 1908 } 1909 1910 static void mlx5e_deactivate_channel(struct mlx5e_channel *c) 1911 { 1912 int tc; 1913 1914 mlx5e_deactivate_rq(&c->rq); 1915 for (tc = 0; tc < c->num_tc; tc++) 1916 mlx5e_deactivate_txqsq(&c->sq[tc]); 1917 } 1918 1919 static void mlx5e_close_channel(struct mlx5e_channel *c) 1920 { 1921 mlx5e_close_xdpsq(&c->xdpsq, NULL); 1922 mlx5e_close_rq(&c->rq); 1923 if (c->xdp) 1924 mlx5e_close_xdpsq(&c->rq.xdpsq, &c->rq); 1925 mlx5e_close_sqs(c); 1926 mlx5e_close_icosq(&c->icosq); 1927 napi_disable(&c->napi); 1928 if (c->xdp) 1929 mlx5e_close_cq(&c->rq.xdpsq.cq); 1930 mlx5e_close_cq(&c->rq.cq); 1931 mlx5e_close_cq(&c->xdpsq.cq); 1932 mlx5e_close_tx_cqs(c); 1933 mlx5e_close_cq(&c->icosq.cq); 1934 netif_napi_del(&c->napi); 1935 mlx5e_free_xps_cpumask(c); 1936 1937 kvfree(c); 1938 } 1939 1940 #define DEFAULT_FRAG_SIZE (2048) 1941 1942 static void mlx5e_build_rq_frags_info(struct mlx5_core_dev *mdev, 1943 struct mlx5e_params *params, 1944 struct mlx5e_rq_frags_info *info) 1945 { 1946 u32 byte_count = MLX5E_SW2HW_MTU(params, params->sw_mtu); 1947 int frag_size_max = DEFAULT_FRAG_SIZE; 1948 u32 buf_size = 0; 1949 int i; 1950 1951 #ifdef CONFIG_MLX5_EN_IPSEC 1952 if (MLX5_IPSEC_DEV(mdev)) 1953 byte_count += MLX5E_METADATA_ETHER_LEN; 1954 #endif 1955 1956 if (mlx5e_rx_is_linear_skb(params)) { 1957 int frag_stride; 1958 1959 frag_stride = mlx5e_rx_get_linear_frag_sz(params); 1960 frag_stride = roundup_pow_of_two(frag_stride); 1961 1962 info->arr[0].frag_size = byte_count; 1963 info->arr[0].frag_stride = frag_stride; 1964 info->num_frags = 1; 1965 info->wqe_bulk = PAGE_SIZE / frag_stride; 1966 goto out; 1967 } 1968 1969 if (byte_count > PAGE_SIZE + 1970 (MLX5E_MAX_RX_FRAGS - 1) * frag_size_max) 1971 frag_size_max = PAGE_SIZE; 1972 1973 i = 0; 1974 while (buf_size < byte_count) { 1975 int frag_size = byte_count - buf_size; 1976 1977 if (i < MLX5E_MAX_RX_FRAGS - 1) 1978 frag_size = min(frag_size, frag_size_max); 1979 1980 info->arr[i].frag_size = frag_size; 1981 info->arr[i].frag_stride = roundup_pow_of_two(frag_size); 1982 1983 buf_size += frag_size; 1984 i++; 1985 } 1986 info->num_frags = i; 1987 /* number of different wqes sharing a page */ 1988 info->wqe_bulk = 1 + (info->num_frags % 2); 1989 1990 out: 1991 info->wqe_bulk = max_t(u8, info->wqe_bulk, 8); 1992 info->log_num_frags = order_base_2(info->num_frags); 1993 } 1994 1995 static inline u8 mlx5e_get_rqwq_log_stride(u8 wq_type, int ndsegs) 1996 { 1997 int sz = sizeof(struct mlx5_wqe_data_seg) * ndsegs; 1998 1999 switch (wq_type) { 2000 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2001 sz += sizeof(struct mlx5e_rx_wqe_ll); 2002 break; 2003 default: /* MLX5_WQ_TYPE_CYCLIC */ 2004 sz += sizeof(struct mlx5e_rx_wqe_cyc); 2005 } 2006 2007 return order_base_2(sz); 2008 } 2009 2010 static u8 mlx5e_get_rq_log_wq_sz(void *rqc) 2011 { 2012 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2013 2014 return MLX5_GET(wq, wq, log_wq_sz); 2015 } 2016 2017 static void mlx5e_build_rq_param(struct mlx5e_priv *priv, 2018 struct mlx5e_params *params, 2019 struct mlx5e_rq_param *param) 2020 { 2021 struct mlx5_core_dev *mdev = priv->mdev; 2022 void *rqc = param->rqc; 2023 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2024 int ndsegs = 1; 2025 2026 switch (params->rq_wq_type) { 2027 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2028 MLX5_SET(wq, wq, log_wqe_num_of_strides, 2029 mlx5e_mpwqe_get_log_num_strides(mdev, params) - 2030 MLX5_MPWQE_LOG_NUM_STRIDES_BASE); 2031 MLX5_SET(wq, wq, log_wqe_stride_size, 2032 mlx5e_mpwqe_get_log_stride_size(mdev, params) - 2033 MLX5_MPWQE_LOG_STRIDE_SZ_BASE); 2034 MLX5_SET(wq, wq, log_wq_sz, mlx5e_mpwqe_get_log_rq_size(params)); 2035 break; 2036 default: /* MLX5_WQ_TYPE_CYCLIC */ 2037 MLX5_SET(wq, wq, log_wq_sz, params->log_rq_mtu_frames); 2038 mlx5e_build_rq_frags_info(mdev, params, ¶m->frags_info); 2039 ndsegs = param->frags_info.num_frags; 2040 } 2041 2042 MLX5_SET(wq, wq, wq_type, params->rq_wq_type); 2043 MLX5_SET(wq, wq, end_padding_mode, MLX5_WQ_END_PAD_MODE_ALIGN); 2044 MLX5_SET(wq, wq, log_wq_stride, 2045 mlx5e_get_rqwq_log_stride(params->rq_wq_type, ndsegs)); 2046 MLX5_SET(wq, wq, pd, mdev->mlx5e_res.pdn); 2047 MLX5_SET(rqc, rqc, counter_set_id, priv->q_counter); 2048 MLX5_SET(rqc, rqc, vsd, params->vlan_strip_disable); 2049 MLX5_SET(rqc, rqc, scatter_fcs, params->scatter_fcs_en); 2050 2051 param->wq.buf_numa_node = dev_to_node(mdev->device); 2052 } 2053 2054 static void mlx5e_build_drop_rq_param(struct mlx5e_priv *priv, 2055 struct mlx5e_rq_param *param) 2056 { 2057 struct mlx5_core_dev *mdev = priv->mdev; 2058 void *rqc = param->rqc; 2059 void *wq = MLX5_ADDR_OF(rqc, rqc, wq); 2060 2061 MLX5_SET(wq, wq, wq_type, MLX5_WQ_TYPE_CYCLIC); 2062 MLX5_SET(wq, wq, log_wq_stride, 2063 mlx5e_get_rqwq_log_stride(MLX5_WQ_TYPE_CYCLIC, 1)); 2064 MLX5_SET(rqc, rqc, counter_set_id, priv->drop_rq_q_counter); 2065 2066 param->wq.buf_numa_node = dev_to_node(mdev->device); 2067 } 2068 2069 static void mlx5e_build_sq_param_common(struct mlx5e_priv *priv, 2070 struct mlx5e_sq_param *param) 2071 { 2072 void *sqc = param->sqc; 2073 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2074 2075 MLX5_SET(wq, wq, log_wq_stride, ilog2(MLX5_SEND_WQE_BB)); 2076 MLX5_SET(wq, wq, pd, priv->mdev->mlx5e_res.pdn); 2077 2078 param->wq.buf_numa_node = dev_to_node(priv->mdev->device); 2079 } 2080 2081 static void mlx5e_build_sq_param(struct mlx5e_priv *priv, 2082 struct mlx5e_params *params, 2083 struct mlx5e_sq_param *param) 2084 { 2085 void *sqc = param->sqc; 2086 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2087 bool allow_swp; 2088 2089 allow_swp = mlx5_geneve_tx_allowed(priv->mdev) || 2090 !!MLX5_IPSEC_DEV(priv->mdev); 2091 mlx5e_build_sq_param_common(priv, param); 2092 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 2093 MLX5_SET(sqc, sqc, allow_swp, allow_swp); 2094 } 2095 2096 static void mlx5e_build_common_cq_param(struct mlx5e_priv *priv, 2097 struct mlx5e_cq_param *param) 2098 { 2099 void *cqc = param->cqc; 2100 2101 MLX5_SET(cqc, cqc, uar_page, priv->mdev->priv.uar->index); 2102 if (MLX5_CAP_GEN(priv->mdev, cqe_128_always) && cache_line_size() >= 128) 2103 MLX5_SET(cqc, cqc, cqe_sz, CQE_STRIDE_128_PAD); 2104 } 2105 2106 static void mlx5e_build_rx_cq_param(struct mlx5e_priv *priv, 2107 struct mlx5e_params *params, 2108 struct mlx5e_cq_param *param) 2109 { 2110 struct mlx5_core_dev *mdev = priv->mdev; 2111 void *cqc = param->cqc; 2112 u8 log_cq_size; 2113 2114 switch (params->rq_wq_type) { 2115 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2116 log_cq_size = mlx5e_mpwqe_get_log_rq_size(params) + 2117 mlx5e_mpwqe_get_log_num_strides(mdev, params); 2118 break; 2119 default: /* MLX5_WQ_TYPE_CYCLIC */ 2120 log_cq_size = params->log_rq_mtu_frames; 2121 } 2122 2123 MLX5_SET(cqc, cqc, log_cq_size, log_cq_size); 2124 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 2125 MLX5_SET(cqc, cqc, mini_cqe_res_format, MLX5_CQE_FORMAT_CSUM); 2126 MLX5_SET(cqc, cqc, cqe_comp_en, 1); 2127 } 2128 2129 mlx5e_build_common_cq_param(priv, param); 2130 param->cq_period_mode = params->rx_cq_moderation.cq_period_mode; 2131 } 2132 2133 static void mlx5e_build_tx_cq_param(struct mlx5e_priv *priv, 2134 struct mlx5e_params *params, 2135 struct mlx5e_cq_param *param) 2136 { 2137 void *cqc = param->cqc; 2138 2139 MLX5_SET(cqc, cqc, log_cq_size, params->log_sq_size); 2140 2141 mlx5e_build_common_cq_param(priv, param); 2142 param->cq_period_mode = params->tx_cq_moderation.cq_period_mode; 2143 } 2144 2145 static void mlx5e_build_ico_cq_param(struct mlx5e_priv *priv, 2146 u8 log_wq_size, 2147 struct mlx5e_cq_param *param) 2148 { 2149 void *cqc = param->cqc; 2150 2151 MLX5_SET(cqc, cqc, log_cq_size, log_wq_size); 2152 2153 mlx5e_build_common_cq_param(priv, param); 2154 2155 param->cq_period_mode = NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; 2156 } 2157 2158 static void mlx5e_build_icosq_param(struct mlx5e_priv *priv, 2159 u8 log_wq_size, 2160 struct mlx5e_sq_param *param) 2161 { 2162 void *sqc = param->sqc; 2163 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2164 2165 mlx5e_build_sq_param_common(priv, param); 2166 2167 MLX5_SET(wq, wq, log_wq_sz, log_wq_size); 2168 MLX5_SET(sqc, sqc, reg_umr, MLX5_CAP_ETH(priv->mdev, reg_umr_sq)); 2169 } 2170 2171 static void mlx5e_build_xdpsq_param(struct mlx5e_priv *priv, 2172 struct mlx5e_params *params, 2173 struct mlx5e_sq_param *param) 2174 { 2175 void *sqc = param->sqc; 2176 void *wq = MLX5_ADDR_OF(sqc, sqc, wq); 2177 2178 mlx5e_build_sq_param_common(priv, param); 2179 MLX5_SET(wq, wq, log_wq_sz, params->log_sq_size); 2180 param->is_mpw = MLX5E_GET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE); 2181 } 2182 2183 static u8 mlx5e_build_icosq_log_wq_sz(struct mlx5e_params *params, 2184 struct mlx5e_rq_param *rqp) 2185 { 2186 switch (params->rq_wq_type) { 2187 case MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ: 2188 return order_base_2(MLX5E_UMR_WQEBBS) + 2189 mlx5e_get_rq_log_wq_sz(rqp->rqc); 2190 default: /* MLX5_WQ_TYPE_CYCLIC */ 2191 return MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE; 2192 } 2193 } 2194 2195 static void mlx5e_build_channel_param(struct mlx5e_priv *priv, 2196 struct mlx5e_params *params, 2197 struct mlx5e_channel_param *cparam) 2198 { 2199 u8 icosq_log_wq_sz; 2200 2201 mlx5e_build_rq_param(priv, params, &cparam->rq); 2202 2203 icosq_log_wq_sz = mlx5e_build_icosq_log_wq_sz(params, &cparam->rq); 2204 2205 mlx5e_build_sq_param(priv, params, &cparam->sq); 2206 mlx5e_build_xdpsq_param(priv, params, &cparam->xdp_sq); 2207 mlx5e_build_icosq_param(priv, icosq_log_wq_sz, &cparam->icosq); 2208 mlx5e_build_rx_cq_param(priv, params, &cparam->rx_cq); 2209 mlx5e_build_tx_cq_param(priv, params, &cparam->tx_cq); 2210 mlx5e_build_ico_cq_param(priv, icosq_log_wq_sz, &cparam->icosq_cq); 2211 } 2212 2213 int mlx5e_open_channels(struct mlx5e_priv *priv, 2214 struct mlx5e_channels *chs) 2215 { 2216 struct mlx5e_channel_param *cparam; 2217 int err = -ENOMEM; 2218 int i; 2219 2220 chs->num = chs->params.num_channels; 2221 2222 chs->c = kcalloc(chs->num, sizeof(struct mlx5e_channel *), GFP_KERNEL); 2223 cparam = kvzalloc(sizeof(struct mlx5e_channel_param), GFP_KERNEL); 2224 if (!chs->c || !cparam) 2225 goto err_free; 2226 2227 mlx5e_build_channel_param(priv, &chs->params, cparam); 2228 for (i = 0; i < chs->num; i++) { 2229 err = mlx5e_open_channel(priv, i, &chs->params, cparam, &chs->c[i]); 2230 if (err) 2231 goto err_close_channels; 2232 } 2233 2234 if (!IS_ERR_OR_NULL(priv->tx_reporter)) 2235 devlink_health_reporter_state_update(priv->tx_reporter, 2236 DEVLINK_HEALTH_REPORTER_STATE_HEALTHY); 2237 2238 kvfree(cparam); 2239 return 0; 2240 2241 err_close_channels: 2242 for (i--; i >= 0; i--) 2243 mlx5e_close_channel(chs->c[i]); 2244 2245 err_free: 2246 kfree(chs->c); 2247 kvfree(cparam); 2248 chs->num = 0; 2249 return err; 2250 } 2251 2252 static void mlx5e_activate_channels(struct mlx5e_channels *chs) 2253 { 2254 int i; 2255 2256 for (i = 0; i < chs->num; i++) 2257 mlx5e_activate_channel(chs->c[i]); 2258 } 2259 2260 #define MLX5E_RQ_WQES_TIMEOUT 20000 /* msecs */ 2261 2262 static int mlx5e_wait_channels_min_rx_wqes(struct mlx5e_channels *chs) 2263 { 2264 int err = 0; 2265 int i; 2266 2267 for (i = 0; i < chs->num; i++) { 2268 int timeout = err ? 0 : MLX5E_RQ_WQES_TIMEOUT; 2269 2270 err |= mlx5e_wait_for_min_rx_wqes(&chs->c[i]->rq, timeout); 2271 } 2272 2273 return err ? -ETIMEDOUT : 0; 2274 } 2275 2276 static void mlx5e_deactivate_channels(struct mlx5e_channels *chs) 2277 { 2278 int i; 2279 2280 for (i = 0; i < chs->num; i++) 2281 mlx5e_deactivate_channel(chs->c[i]); 2282 } 2283 2284 void mlx5e_close_channels(struct mlx5e_channels *chs) 2285 { 2286 int i; 2287 2288 for (i = 0; i < chs->num; i++) 2289 mlx5e_close_channel(chs->c[i]); 2290 2291 kfree(chs->c); 2292 chs->num = 0; 2293 } 2294 2295 static int 2296 mlx5e_create_rqt(struct mlx5e_priv *priv, int sz, struct mlx5e_rqt *rqt) 2297 { 2298 struct mlx5_core_dev *mdev = priv->mdev; 2299 void *rqtc; 2300 int inlen; 2301 int err; 2302 u32 *in; 2303 int i; 2304 2305 inlen = MLX5_ST_SZ_BYTES(create_rqt_in) + sizeof(u32) * sz; 2306 in = kvzalloc(inlen, GFP_KERNEL); 2307 if (!in) 2308 return -ENOMEM; 2309 2310 rqtc = MLX5_ADDR_OF(create_rqt_in, in, rqt_context); 2311 2312 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 2313 MLX5_SET(rqtc, rqtc, rqt_max_size, sz); 2314 2315 for (i = 0; i < sz; i++) 2316 MLX5_SET(rqtc, rqtc, rq_num[i], priv->drop_rq.rqn); 2317 2318 err = mlx5_core_create_rqt(mdev, in, inlen, &rqt->rqtn); 2319 if (!err) 2320 rqt->enabled = true; 2321 2322 kvfree(in); 2323 return err; 2324 } 2325 2326 void mlx5e_destroy_rqt(struct mlx5e_priv *priv, struct mlx5e_rqt *rqt) 2327 { 2328 rqt->enabled = false; 2329 mlx5_core_destroy_rqt(priv->mdev, rqt->rqtn); 2330 } 2331 2332 int mlx5e_create_indirect_rqt(struct mlx5e_priv *priv) 2333 { 2334 struct mlx5e_rqt *rqt = &priv->indir_rqt; 2335 int err; 2336 2337 err = mlx5e_create_rqt(priv, MLX5E_INDIR_RQT_SIZE, rqt); 2338 if (err) 2339 mlx5_core_warn(priv->mdev, "create indirect rqts failed, %d\n", err); 2340 return err; 2341 } 2342 2343 int mlx5e_create_direct_rqts(struct mlx5e_priv *priv) 2344 { 2345 struct mlx5e_rqt *rqt; 2346 int err; 2347 int ix; 2348 2349 for (ix = 0; ix < mlx5e_get_netdev_max_channels(priv->netdev); ix++) { 2350 rqt = &priv->direct_tir[ix].rqt; 2351 err = mlx5e_create_rqt(priv, 1 /*size */, rqt); 2352 if (err) 2353 goto err_destroy_rqts; 2354 } 2355 2356 return 0; 2357 2358 err_destroy_rqts: 2359 mlx5_core_warn(priv->mdev, "create direct rqts failed, %d\n", err); 2360 for (ix--; ix >= 0; ix--) 2361 mlx5e_destroy_rqt(priv, &priv->direct_tir[ix].rqt); 2362 2363 return err; 2364 } 2365 2366 void mlx5e_destroy_direct_rqts(struct mlx5e_priv *priv) 2367 { 2368 int i; 2369 2370 for (i = 0; i < mlx5e_get_netdev_max_channels(priv->netdev); i++) 2371 mlx5e_destroy_rqt(priv, &priv->direct_tir[i].rqt); 2372 } 2373 2374 static int mlx5e_rx_hash_fn(int hfunc) 2375 { 2376 return (hfunc == ETH_RSS_HASH_TOP) ? 2377 MLX5_RX_HASH_FN_TOEPLITZ : 2378 MLX5_RX_HASH_FN_INVERTED_XOR8; 2379 } 2380 2381 int mlx5e_bits_invert(unsigned long a, int size) 2382 { 2383 int inv = 0; 2384 int i; 2385 2386 for (i = 0; i < size; i++) 2387 inv |= (test_bit(size - i - 1, &a) ? 1 : 0) << i; 2388 2389 return inv; 2390 } 2391 2392 static void mlx5e_fill_rqt_rqns(struct mlx5e_priv *priv, int sz, 2393 struct mlx5e_redirect_rqt_param rrp, void *rqtc) 2394 { 2395 int i; 2396 2397 for (i = 0; i < sz; i++) { 2398 u32 rqn; 2399 2400 if (rrp.is_rss) { 2401 int ix = i; 2402 2403 if (rrp.rss.hfunc == ETH_RSS_HASH_XOR) 2404 ix = mlx5e_bits_invert(i, ilog2(sz)); 2405 2406 ix = priv->rss_params.indirection_rqt[ix]; 2407 rqn = rrp.rss.channels->c[ix]->rq.rqn; 2408 } else { 2409 rqn = rrp.rqn; 2410 } 2411 MLX5_SET(rqtc, rqtc, rq_num[i], rqn); 2412 } 2413 } 2414 2415 int mlx5e_redirect_rqt(struct mlx5e_priv *priv, u32 rqtn, int sz, 2416 struct mlx5e_redirect_rqt_param rrp) 2417 { 2418 struct mlx5_core_dev *mdev = priv->mdev; 2419 void *rqtc; 2420 int inlen; 2421 u32 *in; 2422 int err; 2423 2424 inlen = MLX5_ST_SZ_BYTES(modify_rqt_in) + sizeof(u32) * sz; 2425 in = kvzalloc(inlen, GFP_KERNEL); 2426 if (!in) 2427 return -ENOMEM; 2428 2429 rqtc = MLX5_ADDR_OF(modify_rqt_in, in, ctx); 2430 2431 MLX5_SET(rqtc, rqtc, rqt_actual_size, sz); 2432 MLX5_SET(modify_rqt_in, in, bitmask.rqn_list, 1); 2433 mlx5e_fill_rqt_rqns(priv, sz, rrp, rqtc); 2434 err = mlx5_core_modify_rqt(mdev, rqtn, in, inlen); 2435 2436 kvfree(in); 2437 return err; 2438 } 2439 2440 static u32 mlx5e_get_direct_rqn(struct mlx5e_priv *priv, int ix, 2441 struct mlx5e_redirect_rqt_param rrp) 2442 { 2443 if (!rrp.is_rss) 2444 return rrp.rqn; 2445 2446 if (ix >= rrp.rss.channels->num) 2447 return priv->drop_rq.rqn; 2448 2449 return rrp.rss.channels->c[ix]->rq.rqn; 2450 } 2451 2452 static void mlx5e_redirect_rqts(struct mlx5e_priv *priv, 2453 struct mlx5e_redirect_rqt_param rrp) 2454 { 2455 u32 rqtn; 2456 int ix; 2457 2458 if (priv->indir_rqt.enabled) { 2459 /* RSS RQ table */ 2460 rqtn = priv->indir_rqt.rqtn; 2461 mlx5e_redirect_rqt(priv, rqtn, MLX5E_INDIR_RQT_SIZE, rrp); 2462 } 2463 2464 for (ix = 0; ix < mlx5e_get_netdev_max_channels(priv->netdev); ix++) { 2465 struct mlx5e_redirect_rqt_param direct_rrp = { 2466 .is_rss = false, 2467 { 2468 .rqn = mlx5e_get_direct_rqn(priv, ix, rrp) 2469 }, 2470 }; 2471 2472 /* Direct RQ Tables */ 2473 if (!priv->direct_tir[ix].rqt.enabled) 2474 continue; 2475 2476 rqtn = priv->direct_tir[ix].rqt.rqtn; 2477 mlx5e_redirect_rqt(priv, rqtn, 1, direct_rrp); 2478 } 2479 } 2480 2481 static void mlx5e_redirect_rqts_to_channels(struct mlx5e_priv *priv, 2482 struct mlx5e_channels *chs) 2483 { 2484 struct mlx5e_redirect_rqt_param rrp = { 2485 .is_rss = true, 2486 { 2487 .rss = { 2488 .channels = chs, 2489 .hfunc = priv->rss_params.hfunc, 2490 } 2491 }, 2492 }; 2493 2494 mlx5e_redirect_rqts(priv, rrp); 2495 } 2496 2497 static void mlx5e_redirect_rqts_to_drop(struct mlx5e_priv *priv) 2498 { 2499 struct mlx5e_redirect_rqt_param drop_rrp = { 2500 .is_rss = false, 2501 { 2502 .rqn = priv->drop_rq.rqn, 2503 }, 2504 }; 2505 2506 mlx5e_redirect_rqts(priv, drop_rrp); 2507 } 2508 2509 static const struct mlx5e_tirc_config tirc_default_config[MLX5E_NUM_INDIR_TIRS] = { 2510 [MLX5E_TT_IPV4_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2511 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 2512 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2513 }, 2514 [MLX5E_TT_IPV6_TCP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2515 .l4_prot_type = MLX5_L4_PROT_TYPE_TCP, 2516 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2517 }, 2518 [MLX5E_TT_IPV4_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2519 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 2520 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2521 }, 2522 [MLX5E_TT_IPV6_UDP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2523 .l4_prot_type = MLX5_L4_PROT_TYPE_UDP, 2524 .rx_hash_fields = MLX5_HASH_IP_L4PORTS, 2525 }, 2526 [MLX5E_TT_IPV4_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2527 .l4_prot_type = 0, 2528 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2529 }, 2530 [MLX5E_TT_IPV6_IPSEC_AH] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2531 .l4_prot_type = 0, 2532 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2533 }, 2534 [MLX5E_TT_IPV4_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2535 .l4_prot_type = 0, 2536 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2537 }, 2538 [MLX5E_TT_IPV6_IPSEC_ESP] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2539 .l4_prot_type = 0, 2540 .rx_hash_fields = MLX5_HASH_IP_IPSEC_SPI, 2541 }, 2542 [MLX5E_TT_IPV4] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV4, 2543 .l4_prot_type = 0, 2544 .rx_hash_fields = MLX5_HASH_IP, 2545 }, 2546 [MLX5E_TT_IPV6] = { .l3_prot_type = MLX5_L3_PROT_TYPE_IPV6, 2547 .l4_prot_type = 0, 2548 .rx_hash_fields = MLX5_HASH_IP, 2549 }, 2550 }; 2551 2552 struct mlx5e_tirc_config mlx5e_tirc_get_default_config(enum mlx5e_traffic_types tt) 2553 { 2554 return tirc_default_config[tt]; 2555 } 2556 2557 static void mlx5e_build_tir_ctx_lro(struct mlx5e_params *params, void *tirc) 2558 { 2559 if (!params->lro_en) 2560 return; 2561 2562 #define ROUGH_MAX_L2_L3_HDR_SZ 256 2563 2564 MLX5_SET(tirc, tirc, lro_enable_mask, 2565 MLX5_TIRC_LRO_ENABLE_MASK_IPV4_LRO | 2566 MLX5_TIRC_LRO_ENABLE_MASK_IPV6_LRO); 2567 MLX5_SET(tirc, tirc, lro_max_ip_payload_size, 2568 (MLX5E_PARAMS_DEFAULT_LRO_WQE_SZ - ROUGH_MAX_L2_L3_HDR_SZ) >> 8); 2569 MLX5_SET(tirc, tirc, lro_timeout_period_usecs, params->lro_timeout); 2570 } 2571 2572 void mlx5e_build_indir_tir_ctx_hash(struct mlx5e_rss_params *rss_params, 2573 const struct mlx5e_tirc_config *ttconfig, 2574 void *tirc, bool inner) 2575 { 2576 void *hfso = inner ? MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_inner) : 2577 MLX5_ADDR_OF(tirc, tirc, rx_hash_field_selector_outer); 2578 2579 MLX5_SET(tirc, tirc, rx_hash_fn, mlx5e_rx_hash_fn(rss_params->hfunc)); 2580 if (rss_params->hfunc == ETH_RSS_HASH_TOP) { 2581 void *rss_key = MLX5_ADDR_OF(tirc, tirc, 2582 rx_hash_toeplitz_key); 2583 size_t len = MLX5_FLD_SZ_BYTES(tirc, 2584 rx_hash_toeplitz_key); 2585 2586 MLX5_SET(tirc, tirc, rx_hash_symmetric, 1); 2587 memcpy(rss_key, rss_params->toeplitz_hash_key, len); 2588 } 2589 MLX5_SET(rx_hash_field_select, hfso, l3_prot_type, 2590 ttconfig->l3_prot_type); 2591 MLX5_SET(rx_hash_field_select, hfso, l4_prot_type, 2592 ttconfig->l4_prot_type); 2593 MLX5_SET(rx_hash_field_select, hfso, selected_fields, 2594 ttconfig->rx_hash_fields); 2595 } 2596 2597 static void mlx5e_update_rx_hash_fields(struct mlx5e_tirc_config *ttconfig, 2598 enum mlx5e_traffic_types tt, 2599 u32 rx_hash_fields) 2600 { 2601 *ttconfig = tirc_default_config[tt]; 2602 ttconfig->rx_hash_fields = rx_hash_fields; 2603 } 2604 2605 void mlx5e_modify_tirs_hash(struct mlx5e_priv *priv, void *in, int inlen) 2606 { 2607 void *tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx); 2608 struct mlx5e_rss_params *rss = &priv->rss_params; 2609 struct mlx5_core_dev *mdev = priv->mdev; 2610 int ctxlen = MLX5_ST_SZ_BYTES(tirc); 2611 struct mlx5e_tirc_config ttconfig; 2612 int tt; 2613 2614 MLX5_SET(modify_tir_in, in, bitmask.hash, 1); 2615 2616 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2617 memset(tirc, 0, ctxlen); 2618 mlx5e_update_rx_hash_fields(&ttconfig, tt, 2619 rss->rx_hash_fields[tt]); 2620 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, false); 2621 mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in, inlen); 2622 } 2623 2624 if (!mlx5e_tunnel_inner_ft_supported(priv->mdev)) 2625 return; 2626 2627 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2628 memset(tirc, 0, ctxlen); 2629 mlx5e_update_rx_hash_fields(&ttconfig, tt, 2630 rss->rx_hash_fields[tt]); 2631 mlx5e_build_indir_tir_ctx_hash(rss, &ttconfig, tirc, true); 2632 mlx5_core_modify_tir(mdev, priv->inner_indir_tir[tt].tirn, in, 2633 inlen); 2634 } 2635 } 2636 2637 static int mlx5e_modify_tirs_lro(struct mlx5e_priv *priv) 2638 { 2639 struct mlx5_core_dev *mdev = priv->mdev; 2640 2641 void *in; 2642 void *tirc; 2643 int inlen; 2644 int err; 2645 int tt; 2646 int ix; 2647 2648 inlen = MLX5_ST_SZ_BYTES(modify_tir_in); 2649 in = kvzalloc(inlen, GFP_KERNEL); 2650 if (!in) 2651 return -ENOMEM; 2652 2653 MLX5_SET(modify_tir_in, in, bitmask.lro, 1); 2654 tirc = MLX5_ADDR_OF(modify_tir_in, in, ctx); 2655 2656 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); 2657 2658 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 2659 err = mlx5_core_modify_tir(mdev, priv->indir_tir[tt].tirn, in, 2660 inlen); 2661 if (err) 2662 goto free_in; 2663 } 2664 2665 for (ix = 0; ix < mlx5e_get_netdev_max_channels(priv->netdev); ix++) { 2666 err = mlx5_core_modify_tir(mdev, priv->direct_tir[ix].tirn, 2667 in, inlen); 2668 if (err) 2669 goto free_in; 2670 } 2671 2672 free_in: 2673 kvfree(in); 2674 2675 return err; 2676 } 2677 2678 static int mlx5e_set_mtu(struct mlx5_core_dev *mdev, 2679 struct mlx5e_params *params, u16 mtu) 2680 { 2681 u16 hw_mtu = MLX5E_SW2HW_MTU(params, mtu); 2682 int err; 2683 2684 err = mlx5_set_port_mtu(mdev, hw_mtu, 1); 2685 if (err) 2686 return err; 2687 2688 /* Update vport context MTU */ 2689 mlx5_modify_nic_vport_mtu(mdev, hw_mtu); 2690 return 0; 2691 } 2692 2693 static void mlx5e_query_mtu(struct mlx5_core_dev *mdev, 2694 struct mlx5e_params *params, u16 *mtu) 2695 { 2696 u16 hw_mtu = 0; 2697 int err; 2698 2699 err = mlx5_query_nic_vport_mtu(mdev, &hw_mtu); 2700 if (err || !hw_mtu) /* fallback to port oper mtu */ 2701 mlx5_query_port_oper_mtu(mdev, &hw_mtu, 1); 2702 2703 *mtu = MLX5E_HW2SW_MTU(params, hw_mtu); 2704 } 2705 2706 int mlx5e_set_dev_port_mtu(struct mlx5e_priv *priv) 2707 { 2708 struct mlx5e_params *params = &priv->channels.params; 2709 struct net_device *netdev = priv->netdev; 2710 struct mlx5_core_dev *mdev = priv->mdev; 2711 u16 mtu; 2712 int err; 2713 2714 err = mlx5e_set_mtu(mdev, params, params->sw_mtu); 2715 if (err) 2716 return err; 2717 2718 mlx5e_query_mtu(mdev, params, &mtu); 2719 if (mtu != params->sw_mtu) 2720 netdev_warn(netdev, "%s: VPort MTU %d is different than netdev mtu %d\n", 2721 __func__, mtu, params->sw_mtu); 2722 2723 params->sw_mtu = mtu; 2724 return 0; 2725 } 2726 2727 void mlx5e_set_netdev_mtu_boundaries(struct mlx5e_priv *priv) 2728 { 2729 struct mlx5e_params *params = &priv->channels.params; 2730 struct net_device *netdev = priv->netdev; 2731 struct mlx5_core_dev *mdev = priv->mdev; 2732 u16 max_mtu; 2733 2734 /* MTU range: 68 - hw-specific max */ 2735 netdev->min_mtu = ETH_MIN_MTU; 2736 2737 mlx5_query_port_max_mtu(mdev, &max_mtu, 1); 2738 netdev->max_mtu = min_t(unsigned int, MLX5E_HW2SW_MTU(params, max_mtu), 2739 ETH_MAX_MTU); 2740 } 2741 2742 static void mlx5e_netdev_set_tcs(struct net_device *netdev) 2743 { 2744 struct mlx5e_priv *priv = netdev_priv(netdev); 2745 int nch = priv->channels.params.num_channels; 2746 int ntc = priv->channels.params.num_tc; 2747 int tc; 2748 2749 netdev_reset_tc(netdev); 2750 2751 if (ntc == 1) 2752 return; 2753 2754 netdev_set_num_tc(netdev, ntc); 2755 2756 /* Map netdev TCs to offset 0 2757 * We have our own UP to TXQ mapping for QoS 2758 */ 2759 for (tc = 0; tc < ntc; tc++) 2760 netdev_set_tc_queue(netdev, tc, nch, 0); 2761 } 2762 2763 static void mlx5e_build_tc2txq_maps(struct mlx5e_priv *priv) 2764 { 2765 int max_nch = mlx5e_get_netdev_max_channels(priv->netdev); 2766 int i, tc; 2767 2768 for (i = 0; i < max_nch; i++) 2769 for (tc = 0; tc < priv->profile->max_tc; tc++) 2770 priv->channel_tc2txq[i][tc] = i + tc * max_nch; 2771 } 2772 2773 static void mlx5e_build_tx2sq_maps(struct mlx5e_priv *priv) 2774 { 2775 struct mlx5e_channel *c; 2776 struct mlx5e_txqsq *sq; 2777 int i, tc; 2778 2779 for (i = 0; i < priv->channels.num; i++) { 2780 c = priv->channels.c[i]; 2781 for (tc = 0; tc < c->num_tc; tc++) { 2782 sq = &c->sq[tc]; 2783 priv->txq2sq[sq->txq_ix] = sq; 2784 } 2785 } 2786 } 2787 2788 void mlx5e_activate_priv_channels(struct mlx5e_priv *priv) 2789 { 2790 int num_txqs = priv->channels.num * priv->channels.params.num_tc; 2791 struct net_device *netdev = priv->netdev; 2792 2793 mlx5e_netdev_set_tcs(netdev); 2794 netif_set_real_num_tx_queues(netdev, num_txqs); 2795 netif_set_real_num_rx_queues(netdev, priv->channels.num); 2796 2797 mlx5e_build_tx2sq_maps(priv); 2798 mlx5e_activate_channels(&priv->channels); 2799 mlx5e_xdp_tx_enable(priv); 2800 netif_tx_start_all_queues(priv->netdev); 2801 2802 if (mlx5e_is_vport_rep(priv)) 2803 mlx5e_add_sqs_fwd_rules(priv); 2804 2805 mlx5e_wait_channels_min_rx_wqes(&priv->channels); 2806 mlx5e_redirect_rqts_to_channels(priv, &priv->channels); 2807 } 2808 2809 void mlx5e_deactivate_priv_channels(struct mlx5e_priv *priv) 2810 { 2811 mlx5e_redirect_rqts_to_drop(priv); 2812 2813 if (mlx5e_is_vport_rep(priv)) 2814 mlx5e_remove_sqs_fwd_rules(priv); 2815 2816 /* FIXME: This is a W/A only for tx timeout watch dog false alarm when 2817 * polling for inactive tx queues. 2818 */ 2819 netif_tx_stop_all_queues(priv->netdev); 2820 netif_tx_disable(priv->netdev); 2821 mlx5e_xdp_tx_disable(priv); 2822 mlx5e_deactivate_channels(&priv->channels); 2823 } 2824 2825 static void mlx5e_switch_priv_channels(struct mlx5e_priv *priv, 2826 struct mlx5e_channels *new_chs, 2827 mlx5e_fp_hw_modify hw_modify) 2828 { 2829 struct net_device *netdev = priv->netdev; 2830 int new_num_txqs; 2831 int carrier_ok; 2832 2833 new_num_txqs = new_chs->num * new_chs->params.num_tc; 2834 2835 carrier_ok = netif_carrier_ok(netdev); 2836 netif_carrier_off(netdev); 2837 2838 if (new_num_txqs < netdev->real_num_tx_queues) 2839 netif_set_real_num_tx_queues(netdev, new_num_txqs); 2840 2841 mlx5e_deactivate_priv_channels(priv); 2842 mlx5e_close_channels(&priv->channels); 2843 2844 priv->channels = *new_chs; 2845 2846 /* New channels are ready to roll, modify HW settings if needed */ 2847 if (hw_modify) 2848 hw_modify(priv); 2849 2850 mlx5e_refresh_tirs(priv, false); 2851 mlx5e_activate_priv_channels(priv); 2852 2853 /* return carrier back if needed */ 2854 if (carrier_ok) 2855 netif_carrier_on(netdev); 2856 } 2857 2858 int mlx5e_safe_switch_channels(struct mlx5e_priv *priv, 2859 struct mlx5e_channels *new_chs, 2860 mlx5e_fp_hw_modify hw_modify) 2861 { 2862 int err; 2863 2864 err = mlx5e_open_channels(priv, new_chs); 2865 if (err) 2866 return err; 2867 2868 mlx5e_switch_priv_channels(priv, new_chs, hw_modify); 2869 return 0; 2870 } 2871 2872 int mlx5e_safe_reopen_channels(struct mlx5e_priv *priv) 2873 { 2874 struct mlx5e_channels new_channels = {}; 2875 2876 new_channels.params = priv->channels.params; 2877 return mlx5e_safe_switch_channels(priv, &new_channels, NULL); 2878 } 2879 2880 void mlx5e_timestamp_init(struct mlx5e_priv *priv) 2881 { 2882 priv->tstamp.tx_type = HWTSTAMP_TX_OFF; 2883 priv->tstamp.rx_filter = HWTSTAMP_FILTER_NONE; 2884 } 2885 2886 int mlx5e_open_locked(struct net_device *netdev) 2887 { 2888 struct mlx5e_priv *priv = netdev_priv(netdev); 2889 int err; 2890 2891 set_bit(MLX5E_STATE_OPENED, &priv->state); 2892 2893 err = mlx5e_open_channels(priv, &priv->channels); 2894 if (err) 2895 goto err_clear_state_opened_flag; 2896 2897 mlx5e_refresh_tirs(priv, false); 2898 mlx5e_activate_priv_channels(priv); 2899 if (priv->profile->update_carrier) 2900 priv->profile->update_carrier(priv); 2901 2902 mlx5e_queue_update_stats(priv); 2903 return 0; 2904 2905 err_clear_state_opened_flag: 2906 clear_bit(MLX5E_STATE_OPENED, &priv->state); 2907 return err; 2908 } 2909 2910 int mlx5e_open(struct net_device *netdev) 2911 { 2912 struct mlx5e_priv *priv = netdev_priv(netdev); 2913 int err; 2914 2915 mutex_lock(&priv->state_lock); 2916 err = mlx5e_open_locked(netdev); 2917 if (!err) 2918 mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_UP); 2919 mutex_unlock(&priv->state_lock); 2920 2921 if (mlx5_vxlan_allowed(priv->mdev->vxlan)) 2922 udp_tunnel_get_rx_info(netdev); 2923 2924 return err; 2925 } 2926 2927 int mlx5e_close_locked(struct net_device *netdev) 2928 { 2929 struct mlx5e_priv *priv = netdev_priv(netdev); 2930 2931 /* May already be CLOSED in case a previous configuration operation 2932 * (e.g RX/TX queue size change) that involves close&open failed. 2933 */ 2934 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 2935 return 0; 2936 2937 clear_bit(MLX5E_STATE_OPENED, &priv->state); 2938 2939 netif_carrier_off(priv->netdev); 2940 mlx5e_deactivate_priv_channels(priv); 2941 mlx5e_close_channels(&priv->channels); 2942 2943 return 0; 2944 } 2945 2946 int mlx5e_close(struct net_device *netdev) 2947 { 2948 struct mlx5e_priv *priv = netdev_priv(netdev); 2949 int err; 2950 2951 if (!netif_device_present(netdev)) 2952 return -ENODEV; 2953 2954 mutex_lock(&priv->state_lock); 2955 mlx5_set_port_admin_status(priv->mdev, MLX5_PORT_DOWN); 2956 err = mlx5e_close_locked(netdev); 2957 mutex_unlock(&priv->state_lock); 2958 2959 return err; 2960 } 2961 2962 static int mlx5e_alloc_drop_rq(struct mlx5_core_dev *mdev, 2963 struct mlx5e_rq *rq, 2964 struct mlx5e_rq_param *param) 2965 { 2966 void *rqc = param->rqc; 2967 void *rqc_wq = MLX5_ADDR_OF(rqc, rqc, wq); 2968 int err; 2969 2970 param->wq.db_numa_node = param->wq.buf_numa_node; 2971 2972 err = mlx5_wq_cyc_create(mdev, ¶m->wq, rqc_wq, &rq->wqe.wq, 2973 &rq->wq_ctrl); 2974 if (err) 2975 return err; 2976 2977 /* Mark as unused given "Drop-RQ" packets never reach XDP */ 2978 xdp_rxq_info_unused(&rq->xdp_rxq); 2979 2980 rq->mdev = mdev; 2981 2982 return 0; 2983 } 2984 2985 static int mlx5e_alloc_drop_cq(struct mlx5_core_dev *mdev, 2986 struct mlx5e_cq *cq, 2987 struct mlx5e_cq_param *param) 2988 { 2989 param->wq.buf_numa_node = dev_to_node(mdev->device); 2990 param->wq.db_numa_node = dev_to_node(mdev->device); 2991 2992 return mlx5e_alloc_cq_common(mdev, param, cq); 2993 } 2994 2995 int mlx5e_open_drop_rq(struct mlx5e_priv *priv, 2996 struct mlx5e_rq *drop_rq) 2997 { 2998 struct mlx5_core_dev *mdev = priv->mdev; 2999 struct mlx5e_cq_param cq_param = {}; 3000 struct mlx5e_rq_param rq_param = {}; 3001 struct mlx5e_cq *cq = &drop_rq->cq; 3002 int err; 3003 3004 mlx5e_build_drop_rq_param(priv, &rq_param); 3005 3006 err = mlx5e_alloc_drop_cq(mdev, cq, &cq_param); 3007 if (err) 3008 return err; 3009 3010 err = mlx5e_create_cq(cq, &cq_param); 3011 if (err) 3012 goto err_free_cq; 3013 3014 err = mlx5e_alloc_drop_rq(mdev, drop_rq, &rq_param); 3015 if (err) 3016 goto err_destroy_cq; 3017 3018 err = mlx5e_create_rq(drop_rq, &rq_param); 3019 if (err) 3020 goto err_free_rq; 3021 3022 err = mlx5e_modify_rq_state(drop_rq, MLX5_RQC_STATE_RST, MLX5_RQC_STATE_RDY); 3023 if (err) 3024 mlx5_core_warn(priv->mdev, "modify_rq_state failed, rx_if_down_packets won't be counted %d\n", err); 3025 3026 return 0; 3027 3028 err_free_rq: 3029 mlx5e_free_rq(drop_rq); 3030 3031 err_destroy_cq: 3032 mlx5e_destroy_cq(cq); 3033 3034 err_free_cq: 3035 mlx5e_free_cq(cq); 3036 3037 return err; 3038 } 3039 3040 void mlx5e_close_drop_rq(struct mlx5e_rq *drop_rq) 3041 { 3042 mlx5e_destroy_rq(drop_rq); 3043 mlx5e_free_rq(drop_rq); 3044 mlx5e_destroy_cq(&drop_rq->cq); 3045 mlx5e_free_cq(&drop_rq->cq); 3046 } 3047 3048 int mlx5e_create_tis(struct mlx5_core_dev *mdev, int tc, 3049 u32 underlay_qpn, u32 *tisn) 3050 { 3051 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {0}; 3052 void *tisc = MLX5_ADDR_OF(create_tis_in, in, ctx); 3053 3054 MLX5_SET(tisc, tisc, prio, tc << 1); 3055 MLX5_SET(tisc, tisc, underlay_qpn, underlay_qpn); 3056 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.td.tdn); 3057 3058 if (mlx5_lag_is_lacp_owner(mdev)) 3059 MLX5_SET(tisc, tisc, strict_lag_tx_port_affinity, 1); 3060 3061 return mlx5_core_create_tis(mdev, in, sizeof(in), tisn); 3062 } 3063 3064 void mlx5e_destroy_tis(struct mlx5_core_dev *mdev, u32 tisn) 3065 { 3066 mlx5_core_destroy_tis(mdev, tisn); 3067 } 3068 3069 int mlx5e_create_tises(struct mlx5e_priv *priv) 3070 { 3071 int err; 3072 int tc; 3073 3074 for (tc = 0; tc < priv->profile->max_tc; tc++) { 3075 err = mlx5e_create_tis(priv->mdev, tc, 0, &priv->tisn[tc]); 3076 if (err) 3077 goto err_close_tises; 3078 } 3079 3080 return 0; 3081 3082 err_close_tises: 3083 for (tc--; tc >= 0; tc--) 3084 mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]); 3085 3086 return err; 3087 } 3088 3089 static void mlx5e_cleanup_nic_tx(struct mlx5e_priv *priv) 3090 { 3091 int tc; 3092 3093 mlx5e_tx_reporter_destroy(priv); 3094 for (tc = 0; tc < priv->profile->max_tc; tc++) 3095 mlx5e_destroy_tis(priv->mdev, priv->tisn[tc]); 3096 } 3097 3098 static void mlx5e_build_indir_tir_ctx_common(struct mlx5e_priv *priv, 3099 u32 rqtn, u32 *tirc) 3100 { 3101 MLX5_SET(tirc, tirc, transport_domain, priv->mdev->mlx5e_res.td.tdn); 3102 MLX5_SET(tirc, tirc, disp_type, MLX5_TIRC_DISP_TYPE_INDIRECT); 3103 MLX5_SET(tirc, tirc, indirect_table, rqtn); 3104 MLX5_SET(tirc, tirc, tunneled_offload_en, 3105 priv->channels.params.tunneled_offload_en); 3106 3107 mlx5e_build_tir_ctx_lro(&priv->channels.params, tirc); 3108 } 3109 3110 static void mlx5e_build_indir_tir_ctx(struct mlx5e_priv *priv, 3111 enum mlx5e_traffic_types tt, 3112 u32 *tirc) 3113 { 3114 mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); 3115 mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, 3116 &tirc_default_config[tt], tirc, false); 3117 } 3118 3119 static void mlx5e_build_direct_tir_ctx(struct mlx5e_priv *priv, u32 rqtn, u32 *tirc) 3120 { 3121 mlx5e_build_indir_tir_ctx_common(priv, rqtn, tirc); 3122 MLX5_SET(tirc, tirc, rx_hash_fn, MLX5_RX_HASH_FN_INVERTED_XOR8); 3123 } 3124 3125 static void mlx5e_build_inner_indir_tir_ctx(struct mlx5e_priv *priv, 3126 enum mlx5e_traffic_types tt, 3127 u32 *tirc) 3128 { 3129 mlx5e_build_indir_tir_ctx_common(priv, priv->indir_rqt.rqtn, tirc); 3130 mlx5e_build_indir_tir_ctx_hash(&priv->rss_params, 3131 &tirc_default_config[tt], tirc, true); 3132 } 3133 3134 int mlx5e_create_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc) 3135 { 3136 struct mlx5e_tir *tir; 3137 void *tirc; 3138 int inlen; 3139 int i = 0; 3140 int err; 3141 u32 *in; 3142 int tt; 3143 3144 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 3145 in = kvzalloc(inlen, GFP_KERNEL); 3146 if (!in) 3147 return -ENOMEM; 3148 3149 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) { 3150 memset(in, 0, inlen); 3151 tir = &priv->indir_tir[tt]; 3152 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3153 mlx5e_build_indir_tir_ctx(priv, tt, tirc); 3154 err = mlx5e_create_tir(priv->mdev, tir, in, inlen); 3155 if (err) { 3156 mlx5_core_warn(priv->mdev, "create indirect tirs failed, %d\n", err); 3157 goto err_destroy_inner_tirs; 3158 } 3159 } 3160 3161 if (!inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev)) 3162 goto out; 3163 3164 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) { 3165 memset(in, 0, inlen); 3166 tir = &priv->inner_indir_tir[i]; 3167 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3168 mlx5e_build_inner_indir_tir_ctx(priv, i, tirc); 3169 err = mlx5e_create_tir(priv->mdev, tir, in, inlen); 3170 if (err) { 3171 mlx5_core_warn(priv->mdev, "create inner indirect tirs failed, %d\n", err); 3172 goto err_destroy_inner_tirs; 3173 } 3174 } 3175 3176 out: 3177 kvfree(in); 3178 3179 return 0; 3180 3181 err_destroy_inner_tirs: 3182 for (i--; i >= 0; i--) 3183 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]); 3184 3185 for (tt--; tt >= 0; tt--) 3186 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[tt]); 3187 3188 kvfree(in); 3189 3190 return err; 3191 } 3192 3193 int mlx5e_create_direct_tirs(struct mlx5e_priv *priv) 3194 { 3195 int nch = mlx5e_get_netdev_max_channels(priv->netdev); 3196 struct mlx5e_tir *tir; 3197 void *tirc; 3198 int inlen; 3199 int err; 3200 u32 *in; 3201 int ix; 3202 3203 inlen = MLX5_ST_SZ_BYTES(create_tir_in); 3204 in = kvzalloc(inlen, GFP_KERNEL); 3205 if (!in) 3206 return -ENOMEM; 3207 3208 for (ix = 0; ix < nch; ix++) { 3209 memset(in, 0, inlen); 3210 tir = &priv->direct_tir[ix]; 3211 tirc = MLX5_ADDR_OF(create_tir_in, in, ctx); 3212 mlx5e_build_direct_tir_ctx(priv, priv->direct_tir[ix].rqt.rqtn, tirc); 3213 err = mlx5e_create_tir(priv->mdev, tir, in, inlen); 3214 if (err) 3215 goto err_destroy_ch_tirs; 3216 } 3217 3218 kvfree(in); 3219 3220 return 0; 3221 3222 err_destroy_ch_tirs: 3223 mlx5_core_warn(priv->mdev, "create direct tirs failed, %d\n", err); 3224 for (ix--; ix >= 0; ix--) 3225 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[ix]); 3226 3227 kvfree(in); 3228 3229 return err; 3230 } 3231 3232 void mlx5e_destroy_indirect_tirs(struct mlx5e_priv *priv, bool inner_ttc) 3233 { 3234 int i; 3235 3236 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) 3237 mlx5e_destroy_tir(priv->mdev, &priv->indir_tir[i]); 3238 3239 if (!inner_ttc || !mlx5e_tunnel_inner_ft_supported(priv->mdev)) 3240 return; 3241 3242 for (i = 0; i < MLX5E_NUM_INDIR_TIRS; i++) 3243 mlx5e_destroy_tir(priv->mdev, &priv->inner_indir_tir[i]); 3244 } 3245 3246 void mlx5e_destroy_direct_tirs(struct mlx5e_priv *priv) 3247 { 3248 int nch = mlx5e_get_netdev_max_channels(priv->netdev); 3249 int i; 3250 3251 for (i = 0; i < nch; i++) 3252 mlx5e_destroy_tir(priv->mdev, &priv->direct_tir[i]); 3253 } 3254 3255 static int mlx5e_modify_channels_scatter_fcs(struct mlx5e_channels *chs, bool enable) 3256 { 3257 int err = 0; 3258 int i; 3259 3260 for (i = 0; i < chs->num; i++) { 3261 err = mlx5e_modify_rq_scatter_fcs(&chs->c[i]->rq, enable); 3262 if (err) 3263 return err; 3264 } 3265 3266 return 0; 3267 } 3268 3269 static int mlx5e_modify_channels_vsd(struct mlx5e_channels *chs, bool vsd) 3270 { 3271 int err = 0; 3272 int i; 3273 3274 for (i = 0; i < chs->num; i++) { 3275 err = mlx5e_modify_rq_vsd(&chs->c[i]->rq, vsd); 3276 if (err) 3277 return err; 3278 } 3279 3280 return 0; 3281 } 3282 3283 static int mlx5e_setup_tc_mqprio(struct net_device *netdev, 3284 struct tc_mqprio_qopt *mqprio) 3285 { 3286 struct mlx5e_priv *priv = netdev_priv(netdev); 3287 struct mlx5e_channels new_channels = {}; 3288 u8 tc = mqprio->num_tc; 3289 int err = 0; 3290 3291 mqprio->hw = TC_MQPRIO_HW_OFFLOAD_TCS; 3292 3293 if (tc && tc != MLX5E_MAX_NUM_TC) 3294 return -EINVAL; 3295 3296 mutex_lock(&priv->state_lock); 3297 3298 new_channels.params = priv->channels.params; 3299 new_channels.params.num_tc = tc ? tc : 1; 3300 3301 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) { 3302 priv->channels.params = new_channels.params; 3303 goto out; 3304 } 3305 3306 err = mlx5e_safe_switch_channels(priv, &new_channels, NULL); 3307 if (err) 3308 goto out; 3309 3310 priv->max_opened_tc = max_t(u8, priv->max_opened_tc, 3311 new_channels.params.num_tc); 3312 out: 3313 mutex_unlock(&priv->state_lock); 3314 return err; 3315 } 3316 3317 #ifdef CONFIG_MLX5_ESWITCH 3318 static int mlx5e_setup_tc_cls_flower(struct mlx5e_priv *priv, 3319 struct tc_cls_flower_offload *cls_flower, 3320 int flags) 3321 { 3322 switch (cls_flower->command) { 3323 case TC_CLSFLOWER_REPLACE: 3324 return mlx5e_configure_flower(priv->netdev, priv, cls_flower, 3325 flags); 3326 case TC_CLSFLOWER_DESTROY: 3327 return mlx5e_delete_flower(priv->netdev, priv, cls_flower, 3328 flags); 3329 case TC_CLSFLOWER_STATS: 3330 return mlx5e_stats_flower(priv->netdev, priv, cls_flower, 3331 flags); 3332 default: 3333 return -EOPNOTSUPP; 3334 } 3335 } 3336 3337 static int mlx5e_setup_tc_block_cb(enum tc_setup_type type, void *type_data, 3338 void *cb_priv) 3339 { 3340 struct mlx5e_priv *priv = cb_priv; 3341 3342 switch (type) { 3343 case TC_SETUP_CLSFLOWER: 3344 return mlx5e_setup_tc_cls_flower(priv, type_data, MLX5E_TC_INGRESS | 3345 MLX5E_TC_NIC_OFFLOAD); 3346 default: 3347 return -EOPNOTSUPP; 3348 } 3349 } 3350 3351 static int mlx5e_setup_tc_block(struct net_device *dev, 3352 struct tc_block_offload *f) 3353 { 3354 struct mlx5e_priv *priv = netdev_priv(dev); 3355 3356 if (f->binder_type != TCF_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 3357 return -EOPNOTSUPP; 3358 3359 switch (f->command) { 3360 case TC_BLOCK_BIND: 3361 return tcf_block_cb_register(f->block, mlx5e_setup_tc_block_cb, 3362 priv, priv, f->extack); 3363 case TC_BLOCK_UNBIND: 3364 tcf_block_cb_unregister(f->block, mlx5e_setup_tc_block_cb, 3365 priv); 3366 return 0; 3367 default: 3368 return -EOPNOTSUPP; 3369 } 3370 } 3371 #endif 3372 3373 static int mlx5e_setup_tc(struct net_device *dev, enum tc_setup_type type, 3374 void *type_data) 3375 { 3376 switch (type) { 3377 #ifdef CONFIG_MLX5_ESWITCH 3378 case TC_SETUP_BLOCK: 3379 return mlx5e_setup_tc_block(dev, type_data); 3380 #endif 3381 case TC_SETUP_QDISC_MQPRIO: 3382 return mlx5e_setup_tc_mqprio(dev, type_data); 3383 default: 3384 return -EOPNOTSUPP; 3385 } 3386 } 3387 3388 void mlx5e_fold_sw_stats64(struct mlx5e_priv *priv, struct rtnl_link_stats64 *s) 3389 { 3390 int i; 3391 3392 for (i = 0; i < mlx5e_get_netdev_max_channels(priv->netdev); i++) { 3393 struct mlx5e_channel_stats *channel_stats = &priv->channel_stats[i]; 3394 struct mlx5e_rq_stats *rq_stats = &channel_stats->rq; 3395 int j; 3396 3397 s->rx_packets += rq_stats->packets; 3398 s->rx_bytes += rq_stats->bytes; 3399 3400 for (j = 0; j < priv->max_opened_tc; j++) { 3401 struct mlx5e_sq_stats *sq_stats = &channel_stats->sq[j]; 3402 3403 s->tx_packets += sq_stats->packets; 3404 s->tx_bytes += sq_stats->bytes; 3405 s->tx_dropped += sq_stats->dropped; 3406 } 3407 } 3408 } 3409 3410 void 3411 mlx5e_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 3412 { 3413 struct mlx5e_priv *priv = netdev_priv(dev); 3414 struct mlx5e_vport_stats *vstats = &priv->stats.vport; 3415 struct mlx5e_pport_stats *pstats = &priv->stats.pport; 3416 3417 if (!mlx5e_monitor_counter_supported(priv)) { 3418 /* update HW stats in background for next time */ 3419 mlx5e_queue_update_stats(priv); 3420 } 3421 3422 if (mlx5e_is_uplink_rep(priv)) { 3423 stats->rx_packets = PPORT_802_3_GET(pstats, a_frames_received_ok); 3424 stats->rx_bytes = PPORT_802_3_GET(pstats, a_octets_received_ok); 3425 stats->tx_packets = PPORT_802_3_GET(pstats, a_frames_transmitted_ok); 3426 stats->tx_bytes = PPORT_802_3_GET(pstats, a_octets_transmitted_ok); 3427 } else { 3428 mlx5e_fold_sw_stats64(priv, stats); 3429 } 3430 3431 stats->rx_dropped = priv->stats.qcnt.rx_out_of_buffer; 3432 3433 stats->rx_length_errors = 3434 PPORT_802_3_GET(pstats, a_in_range_length_errors) + 3435 PPORT_802_3_GET(pstats, a_out_of_range_length_field) + 3436 PPORT_802_3_GET(pstats, a_frame_too_long_errors); 3437 stats->rx_crc_errors = 3438 PPORT_802_3_GET(pstats, a_frame_check_sequence_errors); 3439 stats->rx_frame_errors = PPORT_802_3_GET(pstats, a_alignment_errors); 3440 stats->tx_aborted_errors = PPORT_2863_GET(pstats, if_out_discards); 3441 stats->rx_errors = stats->rx_length_errors + stats->rx_crc_errors + 3442 stats->rx_frame_errors; 3443 stats->tx_errors = stats->tx_aborted_errors + stats->tx_carrier_errors; 3444 3445 /* vport multicast also counts packets that are dropped due to steering 3446 * or rx out of buffer 3447 */ 3448 stats->multicast = 3449 VPORT_COUNTER_GET(vstats, received_eth_multicast.packets); 3450 } 3451 3452 static void mlx5e_set_rx_mode(struct net_device *dev) 3453 { 3454 struct mlx5e_priv *priv = netdev_priv(dev); 3455 3456 queue_work(priv->wq, &priv->set_rx_mode_work); 3457 } 3458 3459 static int mlx5e_set_mac(struct net_device *netdev, void *addr) 3460 { 3461 struct mlx5e_priv *priv = netdev_priv(netdev); 3462 struct sockaddr *saddr = addr; 3463 3464 if (!is_valid_ether_addr(saddr->sa_data)) 3465 return -EADDRNOTAVAIL; 3466 3467 netif_addr_lock_bh(netdev); 3468 ether_addr_copy(netdev->dev_addr, saddr->sa_data); 3469 netif_addr_unlock_bh(netdev); 3470 3471 queue_work(priv->wq, &priv->set_rx_mode_work); 3472 3473 return 0; 3474 } 3475 3476 #define MLX5E_SET_FEATURE(features, feature, enable) \ 3477 do { \ 3478 if (enable) \ 3479 *features |= feature; \ 3480 else \ 3481 *features &= ~feature; \ 3482 } while (0) 3483 3484 typedef int (*mlx5e_feature_handler)(struct net_device *netdev, bool enable); 3485 3486 static int set_feature_lro(struct net_device *netdev, bool enable) 3487 { 3488 struct mlx5e_priv *priv = netdev_priv(netdev); 3489 struct mlx5_core_dev *mdev = priv->mdev; 3490 struct mlx5e_channels new_channels = {}; 3491 struct mlx5e_params *old_params; 3492 int err = 0; 3493 bool reset; 3494 3495 mutex_lock(&priv->state_lock); 3496 3497 old_params = &priv->channels.params; 3498 if (enable && !MLX5E_GET_PFLAG(old_params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 3499 netdev_warn(netdev, "can't set LRO with legacy RQ\n"); 3500 err = -EINVAL; 3501 goto out; 3502 } 3503 3504 reset = test_bit(MLX5E_STATE_OPENED, &priv->state); 3505 3506 new_channels.params = *old_params; 3507 new_channels.params.lro_en = enable; 3508 3509 if (old_params->rq_wq_type != MLX5_WQ_TYPE_CYCLIC) { 3510 if (mlx5e_rx_mpwqe_is_linear_skb(mdev, old_params) == 3511 mlx5e_rx_mpwqe_is_linear_skb(mdev, &new_channels.params)) 3512 reset = false; 3513 } 3514 3515 if (!reset) { 3516 *old_params = new_channels.params; 3517 err = mlx5e_modify_tirs_lro(priv); 3518 goto out; 3519 } 3520 3521 err = mlx5e_safe_switch_channels(priv, &new_channels, mlx5e_modify_tirs_lro); 3522 out: 3523 mutex_unlock(&priv->state_lock); 3524 return err; 3525 } 3526 3527 static int set_feature_cvlan_filter(struct net_device *netdev, bool enable) 3528 { 3529 struct mlx5e_priv *priv = netdev_priv(netdev); 3530 3531 if (enable) 3532 mlx5e_enable_cvlan_filter(priv); 3533 else 3534 mlx5e_disable_cvlan_filter(priv); 3535 3536 return 0; 3537 } 3538 3539 #ifdef CONFIG_MLX5_ESWITCH 3540 static int set_feature_tc_num_filters(struct net_device *netdev, bool enable) 3541 { 3542 struct mlx5e_priv *priv = netdev_priv(netdev); 3543 3544 if (!enable && mlx5e_tc_num_filters(priv, MLX5E_TC_NIC_OFFLOAD)) { 3545 netdev_err(netdev, 3546 "Active offloaded tc filters, can't turn hw_tc_offload off\n"); 3547 return -EINVAL; 3548 } 3549 3550 return 0; 3551 } 3552 #endif 3553 3554 static int set_feature_rx_all(struct net_device *netdev, bool enable) 3555 { 3556 struct mlx5e_priv *priv = netdev_priv(netdev); 3557 struct mlx5_core_dev *mdev = priv->mdev; 3558 3559 return mlx5_set_port_fcs(mdev, !enable); 3560 } 3561 3562 static int set_feature_rx_fcs(struct net_device *netdev, bool enable) 3563 { 3564 struct mlx5e_priv *priv = netdev_priv(netdev); 3565 int err; 3566 3567 mutex_lock(&priv->state_lock); 3568 3569 priv->channels.params.scatter_fcs_en = enable; 3570 err = mlx5e_modify_channels_scatter_fcs(&priv->channels, enable); 3571 if (err) 3572 priv->channels.params.scatter_fcs_en = !enable; 3573 3574 mutex_unlock(&priv->state_lock); 3575 3576 return err; 3577 } 3578 3579 static int set_feature_rx_vlan(struct net_device *netdev, bool enable) 3580 { 3581 struct mlx5e_priv *priv = netdev_priv(netdev); 3582 int err = 0; 3583 3584 mutex_lock(&priv->state_lock); 3585 3586 priv->channels.params.vlan_strip_disable = !enable; 3587 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 3588 goto unlock; 3589 3590 err = mlx5e_modify_channels_vsd(&priv->channels, !enable); 3591 if (err) 3592 priv->channels.params.vlan_strip_disable = enable; 3593 3594 unlock: 3595 mutex_unlock(&priv->state_lock); 3596 3597 return err; 3598 } 3599 3600 #ifdef CONFIG_MLX5_EN_ARFS 3601 static int set_feature_arfs(struct net_device *netdev, bool enable) 3602 { 3603 struct mlx5e_priv *priv = netdev_priv(netdev); 3604 int err; 3605 3606 if (enable) 3607 err = mlx5e_arfs_enable(priv); 3608 else 3609 err = mlx5e_arfs_disable(priv); 3610 3611 return err; 3612 } 3613 #endif 3614 3615 static int mlx5e_handle_feature(struct net_device *netdev, 3616 netdev_features_t *features, 3617 netdev_features_t wanted_features, 3618 netdev_features_t feature, 3619 mlx5e_feature_handler feature_handler) 3620 { 3621 netdev_features_t changes = wanted_features ^ netdev->features; 3622 bool enable = !!(wanted_features & feature); 3623 int err; 3624 3625 if (!(changes & feature)) 3626 return 0; 3627 3628 err = feature_handler(netdev, enable); 3629 if (err) { 3630 netdev_err(netdev, "%s feature %pNF failed, err %d\n", 3631 enable ? "Enable" : "Disable", &feature, err); 3632 return err; 3633 } 3634 3635 MLX5E_SET_FEATURE(features, feature, enable); 3636 return 0; 3637 } 3638 3639 int mlx5e_set_features(struct net_device *netdev, netdev_features_t features) 3640 { 3641 netdev_features_t oper_features = netdev->features; 3642 int err = 0; 3643 3644 #define MLX5E_HANDLE_FEATURE(feature, handler) \ 3645 mlx5e_handle_feature(netdev, &oper_features, features, feature, handler) 3646 3647 err |= MLX5E_HANDLE_FEATURE(NETIF_F_LRO, set_feature_lro); 3648 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_FILTER, 3649 set_feature_cvlan_filter); 3650 #ifdef CONFIG_MLX5_ESWITCH 3651 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_TC, set_feature_tc_num_filters); 3652 #endif 3653 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXALL, set_feature_rx_all); 3654 err |= MLX5E_HANDLE_FEATURE(NETIF_F_RXFCS, set_feature_rx_fcs); 3655 err |= MLX5E_HANDLE_FEATURE(NETIF_F_HW_VLAN_CTAG_RX, set_feature_rx_vlan); 3656 #ifdef CONFIG_MLX5_EN_ARFS 3657 err |= MLX5E_HANDLE_FEATURE(NETIF_F_NTUPLE, set_feature_arfs); 3658 #endif 3659 3660 if (err) { 3661 netdev->features = oper_features; 3662 return -EINVAL; 3663 } 3664 3665 return 0; 3666 } 3667 3668 static netdev_features_t mlx5e_fix_features(struct net_device *netdev, 3669 netdev_features_t features) 3670 { 3671 struct mlx5e_priv *priv = netdev_priv(netdev); 3672 struct mlx5e_params *params; 3673 3674 mutex_lock(&priv->state_lock); 3675 params = &priv->channels.params; 3676 if (!bitmap_empty(priv->fs.vlan.active_svlans, VLAN_N_VID)) { 3677 /* HW strips the outer C-tag header, this is a problem 3678 * for S-tag traffic. 3679 */ 3680 features &= ~NETIF_F_HW_VLAN_CTAG_RX; 3681 if (!params->vlan_strip_disable) 3682 netdev_warn(netdev, "Dropping C-tag vlan stripping offload due to S-tag vlan\n"); 3683 } 3684 if (!MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ)) { 3685 features &= ~NETIF_F_LRO; 3686 if (params->lro_en) 3687 netdev_warn(netdev, "Disabling LRO, not supported in legacy RQ\n"); 3688 } 3689 3690 if (MLX5E_GET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS)) { 3691 features &= ~NETIF_F_RXHASH; 3692 if (netdev->features & NETIF_F_RXHASH) 3693 netdev_warn(netdev, "Disabling rxhash, not supported when CQE compress is active\n"); 3694 } 3695 3696 mutex_unlock(&priv->state_lock); 3697 3698 return features; 3699 } 3700 3701 int mlx5e_change_mtu(struct net_device *netdev, int new_mtu, 3702 change_hw_mtu_cb set_mtu_cb) 3703 { 3704 struct mlx5e_priv *priv = netdev_priv(netdev); 3705 struct mlx5e_channels new_channels = {}; 3706 struct mlx5e_params *params; 3707 int err = 0; 3708 bool reset; 3709 3710 mutex_lock(&priv->state_lock); 3711 3712 params = &priv->channels.params; 3713 3714 reset = !params->lro_en; 3715 reset = reset && test_bit(MLX5E_STATE_OPENED, &priv->state); 3716 3717 new_channels.params = *params; 3718 new_channels.params.sw_mtu = new_mtu; 3719 3720 if (params->xdp_prog && 3721 !mlx5e_rx_is_linear_skb(&new_channels.params)) { 3722 netdev_err(netdev, "MTU(%d) > %d is not allowed while XDP enabled\n", 3723 new_mtu, mlx5e_xdp_max_mtu(params)); 3724 err = -EINVAL; 3725 goto out; 3726 } 3727 3728 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) { 3729 bool is_linear = mlx5e_rx_mpwqe_is_linear_skb(priv->mdev, &new_channels.params); 3730 u8 ppw_old = mlx5e_mpwqe_log_pkts_per_wqe(params); 3731 u8 ppw_new = mlx5e_mpwqe_log_pkts_per_wqe(&new_channels.params); 3732 3733 reset = reset && (is_linear || (ppw_old != ppw_new)); 3734 } 3735 3736 if (!reset) { 3737 params->sw_mtu = new_mtu; 3738 if (set_mtu_cb) 3739 set_mtu_cb(priv); 3740 netdev->mtu = params->sw_mtu; 3741 goto out; 3742 } 3743 3744 err = mlx5e_safe_switch_channels(priv, &new_channels, set_mtu_cb); 3745 if (err) 3746 goto out; 3747 3748 netdev->mtu = new_channels.params.sw_mtu; 3749 3750 out: 3751 mutex_unlock(&priv->state_lock); 3752 return err; 3753 } 3754 3755 static int mlx5e_change_nic_mtu(struct net_device *netdev, int new_mtu) 3756 { 3757 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu); 3758 } 3759 3760 int mlx5e_hwstamp_set(struct mlx5e_priv *priv, struct ifreq *ifr) 3761 { 3762 struct hwtstamp_config config; 3763 int err; 3764 3765 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz) || 3766 (mlx5_clock_get_ptp_index(priv->mdev) == -1)) 3767 return -EOPNOTSUPP; 3768 3769 if (copy_from_user(&config, ifr->ifr_data, sizeof(config))) 3770 return -EFAULT; 3771 3772 /* TX HW timestamp */ 3773 switch (config.tx_type) { 3774 case HWTSTAMP_TX_OFF: 3775 case HWTSTAMP_TX_ON: 3776 break; 3777 default: 3778 return -ERANGE; 3779 } 3780 3781 mutex_lock(&priv->state_lock); 3782 /* RX HW timestamp */ 3783 switch (config.rx_filter) { 3784 case HWTSTAMP_FILTER_NONE: 3785 /* Reset CQE compression to Admin default */ 3786 mlx5e_modify_rx_cqe_compression_locked(priv, priv->channels.params.rx_cqe_compress_def); 3787 break; 3788 case HWTSTAMP_FILTER_ALL: 3789 case HWTSTAMP_FILTER_SOME: 3790 case HWTSTAMP_FILTER_PTP_V1_L4_EVENT: 3791 case HWTSTAMP_FILTER_PTP_V1_L4_SYNC: 3792 case HWTSTAMP_FILTER_PTP_V1_L4_DELAY_REQ: 3793 case HWTSTAMP_FILTER_PTP_V2_L4_EVENT: 3794 case HWTSTAMP_FILTER_PTP_V2_L4_SYNC: 3795 case HWTSTAMP_FILTER_PTP_V2_L4_DELAY_REQ: 3796 case HWTSTAMP_FILTER_PTP_V2_L2_EVENT: 3797 case HWTSTAMP_FILTER_PTP_V2_L2_SYNC: 3798 case HWTSTAMP_FILTER_PTP_V2_L2_DELAY_REQ: 3799 case HWTSTAMP_FILTER_PTP_V2_EVENT: 3800 case HWTSTAMP_FILTER_PTP_V2_SYNC: 3801 case HWTSTAMP_FILTER_PTP_V2_DELAY_REQ: 3802 case HWTSTAMP_FILTER_NTP_ALL: 3803 /* Disable CQE compression */ 3804 netdev_warn(priv->netdev, "Disabling cqe compression"); 3805 err = mlx5e_modify_rx_cqe_compression_locked(priv, false); 3806 if (err) { 3807 netdev_err(priv->netdev, "Failed disabling cqe compression err=%d\n", err); 3808 mutex_unlock(&priv->state_lock); 3809 return err; 3810 } 3811 config.rx_filter = HWTSTAMP_FILTER_ALL; 3812 break; 3813 default: 3814 mutex_unlock(&priv->state_lock); 3815 return -ERANGE; 3816 } 3817 3818 memcpy(&priv->tstamp, &config, sizeof(config)); 3819 mutex_unlock(&priv->state_lock); 3820 3821 /* might need to fix some features */ 3822 netdev_update_features(priv->netdev); 3823 3824 return copy_to_user(ifr->ifr_data, &config, 3825 sizeof(config)) ? -EFAULT : 0; 3826 } 3827 3828 int mlx5e_hwstamp_get(struct mlx5e_priv *priv, struct ifreq *ifr) 3829 { 3830 struct hwtstamp_config *cfg = &priv->tstamp; 3831 3832 if (!MLX5_CAP_GEN(priv->mdev, device_frequency_khz)) 3833 return -EOPNOTSUPP; 3834 3835 return copy_to_user(ifr->ifr_data, cfg, sizeof(*cfg)) ? -EFAULT : 0; 3836 } 3837 3838 static int mlx5e_ioctl(struct net_device *dev, struct ifreq *ifr, int cmd) 3839 { 3840 struct mlx5e_priv *priv = netdev_priv(dev); 3841 3842 switch (cmd) { 3843 case SIOCSHWTSTAMP: 3844 return mlx5e_hwstamp_set(priv, ifr); 3845 case SIOCGHWTSTAMP: 3846 return mlx5e_hwstamp_get(priv, ifr); 3847 default: 3848 return -EOPNOTSUPP; 3849 } 3850 } 3851 3852 #ifdef CONFIG_MLX5_ESWITCH 3853 int mlx5e_set_vf_mac(struct net_device *dev, int vf, u8 *mac) 3854 { 3855 struct mlx5e_priv *priv = netdev_priv(dev); 3856 struct mlx5_core_dev *mdev = priv->mdev; 3857 3858 return mlx5_eswitch_set_vport_mac(mdev->priv.eswitch, vf + 1, mac); 3859 } 3860 3861 static int mlx5e_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 3862 __be16 vlan_proto) 3863 { 3864 struct mlx5e_priv *priv = netdev_priv(dev); 3865 struct mlx5_core_dev *mdev = priv->mdev; 3866 3867 if (vlan_proto != htons(ETH_P_8021Q)) 3868 return -EPROTONOSUPPORT; 3869 3870 return mlx5_eswitch_set_vport_vlan(mdev->priv.eswitch, vf + 1, 3871 vlan, qos); 3872 } 3873 3874 static int mlx5e_set_vf_spoofchk(struct net_device *dev, int vf, bool setting) 3875 { 3876 struct mlx5e_priv *priv = netdev_priv(dev); 3877 struct mlx5_core_dev *mdev = priv->mdev; 3878 3879 return mlx5_eswitch_set_vport_spoofchk(mdev->priv.eswitch, vf + 1, setting); 3880 } 3881 3882 static int mlx5e_set_vf_trust(struct net_device *dev, int vf, bool setting) 3883 { 3884 struct mlx5e_priv *priv = netdev_priv(dev); 3885 struct mlx5_core_dev *mdev = priv->mdev; 3886 3887 return mlx5_eswitch_set_vport_trust(mdev->priv.eswitch, vf + 1, setting); 3888 } 3889 3890 int mlx5e_set_vf_rate(struct net_device *dev, int vf, int min_tx_rate, 3891 int max_tx_rate) 3892 { 3893 struct mlx5e_priv *priv = netdev_priv(dev); 3894 struct mlx5_core_dev *mdev = priv->mdev; 3895 3896 return mlx5_eswitch_set_vport_rate(mdev->priv.eswitch, vf + 1, 3897 max_tx_rate, min_tx_rate); 3898 } 3899 3900 static int mlx5_vport_link2ifla(u8 esw_link) 3901 { 3902 switch (esw_link) { 3903 case MLX5_VPORT_ADMIN_STATE_DOWN: 3904 return IFLA_VF_LINK_STATE_DISABLE; 3905 case MLX5_VPORT_ADMIN_STATE_UP: 3906 return IFLA_VF_LINK_STATE_ENABLE; 3907 } 3908 return IFLA_VF_LINK_STATE_AUTO; 3909 } 3910 3911 static int mlx5_ifla_link2vport(u8 ifla_link) 3912 { 3913 switch (ifla_link) { 3914 case IFLA_VF_LINK_STATE_DISABLE: 3915 return MLX5_VPORT_ADMIN_STATE_DOWN; 3916 case IFLA_VF_LINK_STATE_ENABLE: 3917 return MLX5_VPORT_ADMIN_STATE_UP; 3918 } 3919 return MLX5_VPORT_ADMIN_STATE_AUTO; 3920 } 3921 3922 static int mlx5e_set_vf_link_state(struct net_device *dev, int vf, 3923 int link_state) 3924 { 3925 struct mlx5e_priv *priv = netdev_priv(dev); 3926 struct mlx5_core_dev *mdev = priv->mdev; 3927 3928 return mlx5_eswitch_set_vport_state(mdev->priv.eswitch, vf + 1, 3929 mlx5_ifla_link2vport(link_state)); 3930 } 3931 3932 int mlx5e_get_vf_config(struct net_device *dev, 3933 int vf, struct ifla_vf_info *ivi) 3934 { 3935 struct mlx5e_priv *priv = netdev_priv(dev); 3936 struct mlx5_core_dev *mdev = priv->mdev; 3937 int err; 3938 3939 err = mlx5_eswitch_get_vport_config(mdev->priv.eswitch, vf + 1, ivi); 3940 if (err) 3941 return err; 3942 ivi->linkstate = mlx5_vport_link2ifla(ivi->linkstate); 3943 return 0; 3944 } 3945 3946 int mlx5e_get_vf_stats(struct net_device *dev, 3947 int vf, struct ifla_vf_stats *vf_stats) 3948 { 3949 struct mlx5e_priv *priv = netdev_priv(dev); 3950 struct mlx5_core_dev *mdev = priv->mdev; 3951 3952 return mlx5_eswitch_get_vport_stats(mdev->priv.eswitch, vf + 1, 3953 vf_stats); 3954 } 3955 #endif 3956 3957 struct mlx5e_vxlan_work { 3958 struct work_struct work; 3959 struct mlx5e_priv *priv; 3960 u16 port; 3961 }; 3962 3963 static void mlx5e_vxlan_add_work(struct work_struct *work) 3964 { 3965 struct mlx5e_vxlan_work *vxlan_work = 3966 container_of(work, struct mlx5e_vxlan_work, work); 3967 struct mlx5e_priv *priv = vxlan_work->priv; 3968 u16 port = vxlan_work->port; 3969 3970 mutex_lock(&priv->state_lock); 3971 mlx5_vxlan_add_port(priv->mdev->vxlan, port); 3972 mutex_unlock(&priv->state_lock); 3973 3974 kfree(vxlan_work); 3975 } 3976 3977 static void mlx5e_vxlan_del_work(struct work_struct *work) 3978 { 3979 struct mlx5e_vxlan_work *vxlan_work = 3980 container_of(work, struct mlx5e_vxlan_work, work); 3981 struct mlx5e_priv *priv = vxlan_work->priv; 3982 u16 port = vxlan_work->port; 3983 3984 mutex_lock(&priv->state_lock); 3985 mlx5_vxlan_del_port(priv->mdev->vxlan, port); 3986 mutex_unlock(&priv->state_lock); 3987 kfree(vxlan_work); 3988 } 3989 3990 static void mlx5e_vxlan_queue_work(struct mlx5e_priv *priv, u16 port, int add) 3991 { 3992 struct mlx5e_vxlan_work *vxlan_work; 3993 3994 vxlan_work = kmalloc(sizeof(*vxlan_work), GFP_ATOMIC); 3995 if (!vxlan_work) 3996 return; 3997 3998 if (add) 3999 INIT_WORK(&vxlan_work->work, mlx5e_vxlan_add_work); 4000 else 4001 INIT_WORK(&vxlan_work->work, mlx5e_vxlan_del_work); 4002 4003 vxlan_work->priv = priv; 4004 vxlan_work->port = port; 4005 queue_work(priv->wq, &vxlan_work->work); 4006 } 4007 4008 void mlx5e_add_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) 4009 { 4010 struct mlx5e_priv *priv = netdev_priv(netdev); 4011 4012 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 4013 return; 4014 4015 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 4016 return; 4017 4018 mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 1); 4019 } 4020 4021 void mlx5e_del_vxlan_port(struct net_device *netdev, struct udp_tunnel_info *ti) 4022 { 4023 struct mlx5e_priv *priv = netdev_priv(netdev); 4024 4025 if (ti->type != UDP_TUNNEL_TYPE_VXLAN) 4026 return; 4027 4028 if (!mlx5_vxlan_allowed(priv->mdev->vxlan)) 4029 return; 4030 4031 mlx5e_vxlan_queue_work(priv, be16_to_cpu(ti->port), 0); 4032 } 4033 4034 static netdev_features_t mlx5e_tunnel_features_check(struct mlx5e_priv *priv, 4035 struct sk_buff *skb, 4036 netdev_features_t features) 4037 { 4038 unsigned int offset = 0; 4039 struct udphdr *udph; 4040 u8 proto; 4041 u16 port; 4042 4043 switch (vlan_get_protocol(skb)) { 4044 case htons(ETH_P_IP): 4045 proto = ip_hdr(skb)->protocol; 4046 break; 4047 case htons(ETH_P_IPV6): 4048 proto = ipv6_find_hdr(skb, &offset, -1, NULL, NULL); 4049 break; 4050 default: 4051 goto out; 4052 } 4053 4054 switch (proto) { 4055 case IPPROTO_GRE: 4056 return features; 4057 case IPPROTO_UDP: 4058 udph = udp_hdr(skb); 4059 port = be16_to_cpu(udph->dest); 4060 4061 /* Verify if UDP port is being offloaded by HW */ 4062 if (mlx5_vxlan_lookup_port(priv->mdev->vxlan, port)) 4063 return features; 4064 4065 #if IS_ENABLED(CONFIG_GENEVE) 4066 /* Support Geneve offload for default UDP port */ 4067 if (port == GENEVE_UDP_PORT && mlx5_geneve_tx_allowed(priv->mdev)) 4068 return features; 4069 #endif 4070 } 4071 4072 out: 4073 /* Disable CSUM and GSO if the udp dport is not offloaded by HW */ 4074 return features & ~(NETIF_F_CSUM_MASK | NETIF_F_GSO_MASK); 4075 } 4076 4077 netdev_features_t mlx5e_features_check(struct sk_buff *skb, 4078 struct net_device *netdev, 4079 netdev_features_t features) 4080 { 4081 struct mlx5e_priv *priv = netdev_priv(netdev); 4082 4083 features = vlan_features_check(skb, features); 4084 features = vxlan_features_check(skb, features); 4085 4086 #ifdef CONFIG_MLX5_EN_IPSEC 4087 if (mlx5e_ipsec_feature_check(skb, netdev, features)) 4088 return features; 4089 #endif 4090 4091 /* Validate if the tunneled packet is being offloaded by HW */ 4092 if (skb->encapsulation && 4093 (features & NETIF_F_CSUM_MASK || features & NETIF_F_GSO_MASK)) 4094 return mlx5e_tunnel_features_check(priv, skb, features); 4095 4096 return features; 4097 } 4098 4099 static void mlx5e_tx_timeout_work(struct work_struct *work) 4100 { 4101 struct mlx5e_priv *priv = container_of(work, struct mlx5e_priv, 4102 tx_timeout_work); 4103 bool report_failed = false; 4104 int err; 4105 int i; 4106 4107 rtnl_lock(); 4108 mutex_lock(&priv->state_lock); 4109 4110 if (!test_bit(MLX5E_STATE_OPENED, &priv->state)) 4111 goto unlock; 4112 4113 for (i = 0; i < priv->channels.num * priv->channels.params.num_tc; i++) { 4114 struct netdev_queue *dev_queue = 4115 netdev_get_tx_queue(priv->netdev, i); 4116 struct mlx5e_txqsq *sq = priv->txq2sq[i]; 4117 4118 if (!netif_xmit_stopped(dev_queue)) 4119 continue; 4120 4121 if (mlx5e_tx_reporter_timeout(sq)) 4122 report_failed = true; 4123 } 4124 4125 if (!report_failed) 4126 goto unlock; 4127 4128 err = mlx5e_safe_reopen_channels(priv); 4129 if (err) 4130 netdev_err(priv->netdev, 4131 "mlx5e_safe_reopen_channels failed recovering from a tx_timeout, err(%d).\n", 4132 err); 4133 4134 unlock: 4135 mutex_unlock(&priv->state_lock); 4136 rtnl_unlock(); 4137 } 4138 4139 static void mlx5e_tx_timeout(struct net_device *dev) 4140 { 4141 struct mlx5e_priv *priv = netdev_priv(dev); 4142 4143 netdev_err(dev, "TX timeout detected\n"); 4144 queue_work(priv->wq, &priv->tx_timeout_work); 4145 } 4146 4147 static int mlx5e_xdp_allowed(struct mlx5e_priv *priv, struct bpf_prog *prog) 4148 { 4149 struct net_device *netdev = priv->netdev; 4150 struct mlx5e_channels new_channels = {}; 4151 4152 if (priv->channels.params.lro_en) { 4153 netdev_warn(netdev, "can't set XDP while LRO is on, disable LRO first\n"); 4154 return -EINVAL; 4155 } 4156 4157 if (MLX5_IPSEC_DEV(priv->mdev)) { 4158 netdev_warn(netdev, "can't set XDP with IPSec offload\n"); 4159 return -EINVAL; 4160 } 4161 4162 new_channels.params = priv->channels.params; 4163 new_channels.params.xdp_prog = prog; 4164 4165 if (!mlx5e_rx_is_linear_skb(&new_channels.params)) { 4166 netdev_warn(netdev, "XDP is not allowed with MTU(%d) > %d\n", 4167 new_channels.params.sw_mtu, 4168 mlx5e_xdp_max_mtu(&new_channels.params)); 4169 return -EINVAL; 4170 } 4171 4172 return 0; 4173 } 4174 4175 static int mlx5e_xdp_set(struct net_device *netdev, struct bpf_prog *prog) 4176 { 4177 struct mlx5e_priv *priv = netdev_priv(netdev); 4178 struct bpf_prog *old_prog; 4179 bool reset, was_opened; 4180 int err = 0; 4181 int i; 4182 4183 mutex_lock(&priv->state_lock); 4184 4185 if (prog) { 4186 err = mlx5e_xdp_allowed(priv, prog); 4187 if (err) 4188 goto unlock; 4189 } 4190 4191 was_opened = test_bit(MLX5E_STATE_OPENED, &priv->state); 4192 /* no need for full reset when exchanging programs */ 4193 reset = (!priv->channels.params.xdp_prog || !prog); 4194 4195 if (was_opened && reset) 4196 mlx5e_close_locked(netdev); 4197 if (was_opened && !reset) { 4198 /* num_channels is invariant here, so we can take the 4199 * batched reference right upfront. 4200 */ 4201 prog = bpf_prog_add(prog, priv->channels.num); 4202 if (IS_ERR(prog)) { 4203 err = PTR_ERR(prog); 4204 goto unlock; 4205 } 4206 } 4207 4208 /* exchange programs, extra prog reference we got from caller 4209 * as long as we don't fail from this point onwards. 4210 */ 4211 old_prog = xchg(&priv->channels.params.xdp_prog, prog); 4212 if (old_prog) 4213 bpf_prog_put(old_prog); 4214 4215 if (reset) /* change RQ type according to priv->xdp_prog */ 4216 mlx5e_set_rq_type(priv->mdev, &priv->channels.params); 4217 4218 if (was_opened && reset) 4219 err = mlx5e_open_locked(netdev); 4220 4221 if (!test_bit(MLX5E_STATE_OPENED, &priv->state) || reset) 4222 goto unlock; 4223 4224 /* exchanging programs w/o reset, we update ref counts on behalf 4225 * of the channels RQs here. 4226 */ 4227 for (i = 0; i < priv->channels.num; i++) { 4228 struct mlx5e_channel *c = priv->channels.c[i]; 4229 4230 clear_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state); 4231 napi_synchronize(&c->napi); 4232 /* prevent mlx5e_poll_rx_cq from accessing rq->xdp_prog */ 4233 4234 old_prog = xchg(&c->rq.xdp_prog, prog); 4235 4236 set_bit(MLX5E_RQ_STATE_ENABLED, &c->rq.state); 4237 /* napi_schedule in case we have missed anything */ 4238 napi_schedule(&c->napi); 4239 4240 if (old_prog) 4241 bpf_prog_put(old_prog); 4242 } 4243 4244 unlock: 4245 mutex_unlock(&priv->state_lock); 4246 return err; 4247 } 4248 4249 static u32 mlx5e_xdp_query(struct net_device *dev) 4250 { 4251 struct mlx5e_priv *priv = netdev_priv(dev); 4252 const struct bpf_prog *xdp_prog; 4253 u32 prog_id = 0; 4254 4255 mutex_lock(&priv->state_lock); 4256 xdp_prog = priv->channels.params.xdp_prog; 4257 if (xdp_prog) 4258 prog_id = xdp_prog->aux->id; 4259 mutex_unlock(&priv->state_lock); 4260 4261 return prog_id; 4262 } 4263 4264 static int mlx5e_xdp(struct net_device *dev, struct netdev_bpf *xdp) 4265 { 4266 switch (xdp->command) { 4267 case XDP_SETUP_PROG: 4268 return mlx5e_xdp_set(dev, xdp->prog); 4269 case XDP_QUERY_PROG: 4270 xdp->prog_id = mlx5e_xdp_query(dev); 4271 return 0; 4272 default: 4273 return -EINVAL; 4274 } 4275 } 4276 4277 #ifdef CONFIG_MLX5_ESWITCH 4278 static int mlx5e_bridge_getlink(struct sk_buff *skb, u32 pid, u32 seq, 4279 struct net_device *dev, u32 filter_mask, 4280 int nlflags) 4281 { 4282 struct mlx5e_priv *priv = netdev_priv(dev); 4283 struct mlx5_core_dev *mdev = priv->mdev; 4284 u8 mode, setting; 4285 int err; 4286 4287 err = mlx5_eswitch_get_vepa(mdev->priv.eswitch, &setting); 4288 if (err) 4289 return err; 4290 mode = setting ? BRIDGE_MODE_VEPA : BRIDGE_MODE_VEB; 4291 return ndo_dflt_bridge_getlink(skb, pid, seq, dev, 4292 mode, 4293 0, 0, nlflags, filter_mask, NULL); 4294 } 4295 4296 static int mlx5e_bridge_setlink(struct net_device *dev, struct nlmsghdr *nlh, 4297 u16 flags, struct netlink_ext_ack *extack) 4298 { 4299 struct mlx5e_priv *priv = netdev_priv(dev); 4300 struct mlx5_core_dev *mdev = priv->mdev; 4301 struct nlattr *attr, *br_spec; 4302 u16 mode = BRIDGE_MODE_UNDEF; 4303 u8 setting; 4304 int rem; 4305 4306 br_spec = nlmsg_find_attr(nlh, sizeof(struct ifinfomsg), IFLA_AF_SPEC); 4307 if (!br_spec) 4308 return -EINVAL; 4309 4310 nla_for_each_nested(attr, br_spec, rem) { 4311 if (nla_type(attr) != IFLA_BRIDGE_MODE) 4312 continue; 4313 4314 if (nla_len(attr) < sizeof(mode)) 4315 return -EINVAL; 4316 4317 mode = nla_get_u16(attr); 4318 if (mode > BRIDGE_MODE_VEPA) 4319 return -EINVAL; 4320 4321 break; 4322 } 4323 4324 if (mode == BRIDGE_MODE_UNDEF) 4325 return -EINVAL; 4326 4327 setting = (mode == BRIDGE_MODE_VEPA) ? 1 : 0; 4328 return mlx5_eswitch_set_vepa(mdev->priv.eswitch, setting); 4329 } 4330 #endif 4331 4332 const struct net_device_ops mlx5e_netdev_ops = { 4333 .ndo_open = mlx5e_open, 4334 .ndo_stop = mlx5e_close, 4335 .ndo_start_xmit = mlx5e_xmit, 4336 .ndo_setup_tc = mlx5e_setup_tc, 4337 .ndo_select_queue = mlx5e_select_queue, 4338 .ndo_get_stats64 = mlx5e_get_stats, 4339 .ndo_set_rx_mode = mlx5e_set_rx_mode, 4340 .ndo_set_mac_address = mlx5e_set_mac, 4341 .ndo_vlan_rx_add_vid = mlx5e_vlan_rx_add_vid, 4342 .ndo_vlan_rx_kill_vid = mlx5e_vlan_rx_kill_vid, 4343 .ndo_set_features = mlx5e_set_features, 4344 .ndo_fix_features = mlx5e_fix_features, 4345 .ndo_change_mtu = mlx5e_change_nic_mtu, 4346 .ndo_do_ioctl = mlx5e_ioctl, 4347 .ndo_set_tx_maxrate = mlx5e_set_tx_maxrate, 4348 .ndo_udp_tunnel_add = mlx5e_add_vxlan_port, 4349 .ndo_udp_tunnel_del = mlx5e_del_vxlan_port, 4350 .ndo_features_check = mlx5e_features_check, 4351 .ndo_tx_timeout = mlx5e_tx_timeout, 4352 .ndo_bpf = mlx5e_xdp, 4353 .ndo_xdp_xmit = mlx5e_xdp_xmit, 4354 #ifdef CONFIG_MLX5_EN_ARFS 4355 .ndo_rx_flow_steer = mlx5e_rx_flow_steer, 4356 #endif 4357 #ifdef CONFIG_MLX5_ESWITCH 4358 .ndo_bridge_setlink = mlx5e_bridge_setlink, 4359 .ndo_bridge_getlink = mlx5e_bridge_getlink, 4360 4361 /* SRIOV E-Switch NDOs */ 4362 .ndo_set_vf_mac = mlx5e_set_vf_mac, 4363 .ndo_set_vf_vlan = mlx5e_set_vf_vlan, 4364 .ndo_set_vf_spoofchk = mlx5e_set_vf_spoofchk, 4365 .ndo_set_vf_trust = mlx5e_set_vf_trust, 4366 .ndo_set_vf_rate = mlx5e_set_vf_rate, 4367 .ndo_get_vf_config = mlx5e_get_vf_config, 4368 .ndo_set_vf_link_state = mlx5e_set_vf_link_state, 4369 .ndo_get_vf_stats = mlx5e_get_vf_stats, 4370 #endif 4371 }; 4372 4373 static int mlx5e_check_required_hca_cap(struct mlx5_core_dev *mdev) 4374 { 4375 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) 4376 return -EOPNOTSUPP; 4377 if (!MLX5_CAP_GEN(mdev, eth_net_offloads) || 4378 !MLX5_CAP_GEN(mdev, nic_flow_table) || 4379 !MLX5_CAP_ETH(mdev, csum_cap) || 4380 !MLX5_CAP_ETH(mdev, max_lso_cap) || 4381 !MLX5_CAP_ETH(mdev, vlan_cap) || 4382 !MLX5_CAP_ETH(mdev, rss_ind_tbl_cap) || 4383 MLX5_CAP_FLOWTABLE(mdev, 4384 flow_table_properties_nic_receive.max_ft_level) 4385 < 3) { 4386 mlx5_core_warn(mdev, 4387 "Not creating net device, some required device capabilities are missing\n"); 4388 return -EOPNOTSUPP; 4389 } 4390 if (!MLX5_CAP_ETH(mdev, self_lb_en_modifiable)) 4391 mlx5_core_warn(mdev, "Self loop back prevention is not supported\n"); 4392 if (!MLX5_CAP_GEN(mdev, cq_moderation)) 4393 mlx5_core_warn(mdev, "CQ moderation is not supported\n"); 4394 4395 return 0; 4396 } 4397 4398 void mlx5e_build_default_indir_rqt(u32 *indirection_rqt, int len, 4399 int num_channels) 4400 { 4401 int i; 4402 4403 for (i = 0; i < len; i++) 4404 indirection_rqt[i] = i % num_channels; 4405 } 4406 4407 static bool slow_pci_heuristic(struct mlx5_core_dev *mdev) 4408 { 4409 u32 link_speed = 0; 4410 u32 pci_bw = 0; 4411 4412 mlx5e_port_max_linkspeed(mdev, &link_speed); 4413 pci_bw = pcie_bandwidth_available(mdev->pdev, NULL, NULL, NULL); 4414 mlx5_core_dbg_once(mdev, "Max link speed = %d, PCI BW = %d\n", 4415 link_speed, pci_bw); 4416 4417 #define MLX5E_SLOW_PCI_RATIO (2) 4418 4419 return link_speed && pci_bw && 4420 link_speed > MLX5E_SLOW_PCI_RATIO * pci_bw; 4421 } 4422 4423 static struct net_dim_cq_moder mlx5e_get_def_tx_moderation(u8 cq_period_mode) 4424 { 4425 struct net_dim_cq_moder moder; 4426 4427 moder.cq_period_mode = cq_period_mode; 4428 moder.pkts = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_PKTS; 4429 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC; 4430 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 4431 moder.usec = MLX5E_PARAMS_DEFAULT_TX_CQ_MODERATION_USEC_FROM_CQE; 4432 4433 return moder; 4434 } 4435 4436 static struct net_dim_cq_moder mlx5e_get_def_rx_moderation(u8 cq_period_mode) 4437 { 4438 struct net_dim_cq_moder moder; 4439 4440 moder.cq_period_mode = cq_period_mode; 4441 moder.pkts = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_PKTS; 4442 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC; 4443 if (cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE) 4444 moder.usec = MLX5E_PARAMS_DEFAULT_RX_CQ_MODERATION_USEC_FROM_CQE; 4445 4446 return moder; 4447 } 4448 4449 static u8 mlx5_to_net_dim_cq_period_mode(u8 cq_period_mode) 4450 { 4451 return cq_period_mode == MLX5_CQ_PERIOD_MODE_START_FROM_CQE ? 4452 NET_DIM_CQ_PERIOD_MODE_START_FROM_CQE : 4453 NET_DIM_CQ_PERIOD_MODE_START_FROM_EQE; 4454 } 4455 4456 void mlx5e_set_tx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 4457 { 4458 if (params->tx_dim_enabled) { 4459 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 4460 4461 params->tx_cq_moderation = net_dim_get_def_tx_moderation(dim_period_mode); 4462 } else { 4463 params->tx_cq_moderation = mlx5e_get_def_tx_moderation(cq_period_mode); 4464 } 4465 4466 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_TX_CQE_BASED_MODER, 4467 params->tx_cq_moderation.cq_period_mode == 4468 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 4469 } 4470 4471 void mlx5e_set_rx_cq_mode_params(struct mlx5e_params *params, u8 cq_period_mode) 4472 { 4473 if (params->rx_dim_enabled) { 4474 u8 dim_period_mode = mlx5_to_net_dim_cq_period_mode(cq_period_mode); 4475 4476 params->rx_cq_moderation = net_dim_get_def_rx_moderation(dim_period_mode); 4477 } else { 4478 params->rx_cq_moderation = mlx5e_get_def_rx_moderation(cq_period_mode); 4479 } 4480 4481 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_BASED_MODER, 4482 params->rx_cq_moderation.cq_period_mode == 4483 MLX5_CQ_PERIOD_MODE_START_FROM_CQE); 4484 } 4485 4486 static u32 mlx5e_choose_lro_timeout(struct mlx5_core_dev *mdev, u32 wanted_timeout) 4487 { 4488 int i; 4489 4490 /* The supported periods are organized in ascending order */ 4491 for (i = 0; i < MLX5E_LRO_TIMEOUT_ARR_SIZE - 1; i++) 4492 if (MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]) >= wanted_timeout) 4493 break; 4494 4495 return MLX5_CAP_ETH(mdev, lro_timer_supported_periods[i]); 4496 } 4497 4498 void mlx5e_build_rq_params(struct mlx5_core_dev *mdev, 4499 struct mlx5e_params *params) 4500 { 4501 /* Prefer Striding RQ, unless any of the following holds: 4502 * - Striding RQ configuration is not possible/supported. 4503 * - Slow PCI heuristic. 4504 * - Legacy RQ would use linear SKB while Striding RQ would use non-linear. 4505 */ 4506 if (!slow_pci_heuristic(mdev) && 4507 mlx5e_striding_rq_possible(mdev, params) && 4508 (mlx5e_rx_mpwqe_is_linear_skb(mdev, params) || 4509 !mlx5e_rx_is_linear_skb(params))) 4510 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_STRIDING_RQ, true); 4511 mlx5e_set_rq_type(mdev, params); 4512 mlx5e_init_rq_type_params(mdev, params); 4513 } 4514 4515 void mlx5e_build_rss_params(struct mlx5e_rss_params *rss_params, 4516 u16 num_channels) 4517 { 4518 enum mlx5e_traffic_types tt; 4519 4520 rss_params->hfunc = ETH_RSS_HASH_TOP; 4521 netdev_rss_key_fill(rss_params->toeplitz_hash_key, 4522 sizeof(rss_params->toeplitz_hash_key)); 4523 mlx5e_build_default_indir_rqt(rss_params->indirection_rqt, 4524 MLX5E_INDIR_RQT_SIZE, num_channels); 4525 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 4526 rss_params->rx_hash_fields[tt] = 4527 tirc_default_config[tt].rx_hash_fields; 4528 } 4529 4530 void mlx5e_build_nic_params(struct mlx5_core_dev *mdev, 4531 struct mlx5e_rss_params *rss_params, 4532 struct mlx5e_params *params, 4533 u16 max_channels, u16 mtu) 4534 { 4535 u8 rx_cq_period_mode; 4536 4537 params->sw_mtu = mtu; 4538 params->hard_mtu = MLX5E_ETH_HARD_MTU; 4539 params->num_channels = max_channels; 4540 params->num_tc = 1; 4541 4542 /* SQ */ 4543 params->log_sq_size = is_kdump_kernel() ? 4544 MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE : 4545 MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 4546 4547 /* XDP SQ */ 4548 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_XDP_TX_MPWQE, 4549 MLX5_CAP_ETH(mdev, enhanced_multi_pkt_send_wqe)); 4550 4551 /* set CQE compression */ 4552 params->rx_cqe_compress_def = false; 4553 if (MLX5_CAP_GEN(mdev, cqe_compression) && 4554 MLX5_CAP_GEN(mdev, vport_group_manager)) 4555 params->rx_cqe_compress_def = slow_pci_heuristic(mdev); 4556 4557 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_CQE_COMPRESS, params->rx_cqe_compress_def); 4558 MLX5E_SET_PFLAG(params, MLX5E_PFLAG_RX_NO_CSUM_COMPLETE, false); 4559 4560 /* RQ */ 4561 mlx5e_build_rq_params(mdev, params); 4562 4563 /* HW LRO */ 4564 4565 /* TODO: && MLX5_CAP_ETH(mdev, lro_cap) */ 4566 if (params->rq_wq_type == MLX5_WQ_TYPE_LINKED_LIST_STRIDING_RQ) 4567 if (!mlx5e_rx_mpwqe_is_linear_skb(mdev, params)) 4568 params->lro_en = !slow_pci_heuristic(mdev); 4569 params->lro_timeout = mlx5e_choose_lro_timeout(mdev, MLX5E_DEFAULT_LRO_TIMEOUT); 4570 4571 /* CQ moderation params */ 4572 rx_cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 4573 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 4574 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 4575 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4576 params->tx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 4577 mlx5e_set_rx_cq_mode_params(params, rx_cq_period_mode); 4578 mlx5e_set_tx_cq_mode_params(params, MLX5_CQ_PERIOD_MODE_START_FROM_EQE); 4579 4580 /* TX inline */ 4581 params->tx_min_inline_mode = mlx5e_params_calculate_tx_min_inline(mdev); 4582 4583 /* RSS */ 4584 mlx5e_build_rss_params(rss_params, params->num_channels); 4585 params->tunneled_offload_en = 4586 mlx5e_tunnel_inner_ft_supported(mdev); 4587 } 4588 4589 static void mlx5e_set_netdev_dev_addr(struct net_device *netdev) 4590 { 4591 struct mlx5e_priv *priv = netdev_priv(netdev); 4592 4593 mlx5_query_nic_vport_mac_address(priv->mdev, 0, netdev->dev_addr); 4594 if (is_zero_ether_addr(netdev->dev_addr) && 4595 !MLX5_CAP_GEN(priv->mdev, vport_group_manager)) { 4596 eth_hw_addr_random(netdev); 4597 mlx5_core_info(priv->mdev, "Assigned random MAC address %pM\n", netdev->dev_addr); 4598 } 4599 } 4600 4601 static void mlx5e_build_nic_netdev(struct net_device *netdev) 4602 { 4603 struct mlx5e_priv *priv = netdev_priv(netdev); 4604 struct mlx5_core_dev *mdev = priv->mdev; 4605 bool fcs_supported; 4606 bool fcs_enabled; 4607 4608 SET_NETDEV_DEV(netdev, mdev->device); 4609 4610 netdev->netdev_ops = &mlx5e_netdev_ops; 4611 4612 #ifdef CONFIG_MLX5_CORE_EN_DCB 4613 if (MLX5_CAP_GEN(mdev, vport_group_manager) && MLX5_CAP_GEN(mdev, qos)) 4614 netdev->dcbnl_ops = &mlx5e_dcbnl_ops; 4615 #endif 4616 4617 netdev->watchdog_timeo = 15 * HZ; 4618 4619 netdev->ethtool_ops = &mlx5e_ethtool_ops; 4620 4621 netdev->vlan_features |= NETIF_F_SG; 4622 netdev->vlan_features |= NETIF_F_IP_CSUM; 4623 netdev->vlan_features |= NETIF_F_IPV6_CSUM; 4624 netdev->vlan_features |= NETIF_F_GRO; 4625 netdev->vlan_features |= NETIF_F_TSO; 4626 netdev->vlan_features |= NETIF_F_TSO6; 4627 netdev->vlan_features |= NETIF_F_RXCSUM; 4628 netdev->vlan_features |= NETIF_F_RXHASH; 4629 4630 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_TX; 4631 netdev->hw_enc_features |= NETIF_F_HW_VLAN_CTAG_RX; 4632 4633 if (!!MLX5_CAP_ETH(mdev, lro_cap) && 4634 mlx5e_check_fragmented_striding_rq_cap(mdev)) 4635 netdev->vlan_features |= NETIF_F_LRO; 4636 4637 netdev->hw_features = netdev->vlan_features; 4638 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_TX; 4639 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 4640 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_FILTER; 4641 netdev->hw_features |= NETIF_F_HW_VLAN_STAG_TX; 4642 4643 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev) || 4644 MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) { 4645 netdev->hw_enc_features |= NETIF_F_IP_CSUM; 4646 netdev->hw_enc_features |= NETIF_F_IPV6_CSUM; 4647 netdev->hw_enc_features |= NETIF_F_TSO; 4648 netdev->hw_enc_features |= NETIF_F_TSO6; 4649 netdev->hw_enc_features |= NETIF_F_GSO_PARTIAL; 4650 } 4651 4652 if (mlx5_vxlan_allowed(mdev->vxlan) || mlx5_geneve_tx_allowed(mdev)) { 4653 netdev->hw_features |= NETIF_F_GSO_UDP_TUNNEL | 4654 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4655 netdev->hw_enc_features |= NETIF_F_GSO_UDP_TUNNEL | 4656 NETIF_F_GSO_UDP_TUNNEL_CSUM; 4657 netdev->gso_partial_features = NETIF_F_GSO_UDP_TUNNEL_CSUM; 4658 } 4659 4660 if (MLX5_CAP_ETH(mdev, tunnel_stateless_gre)) { 4661 netdev->hw_features |= NETIF_F_GSO_GRE | 4662 NETIF_F_GSO_GRE_CSUM; 4663 netdev->hw_enc_features |= NETIF_F_GSO_GRE | 4664 NETIF_F_GSO_GRE_CSUM; 4665 netdev->gso_partial_features |= NETIF_F_GSO_GRE | 4666 NETIF_F_GSO_GRE_CSUM; 4667 } 4668 4669 netdev->hw_features |= NETIF_F_GSO_PARTIAL; 4670 netdev->gso_partial_features |= NETIF_F_GSO_UDP_L4; 4671 netdev->hw_features |= NETIF_F_GSO_UDP_L4; 4672 netdev->features |= NETIF_F_GSO_UDP_L4; 4673 4674 mlx5_query_port_fcs(mdev, &fcs_supported, &fcs_enabled); 4675 4676 if (fcs_supported) 4677 netdev->hw_features |= NETIF_F_RXALL; 4678 4679 if (MLX5_CAP_ETH(mdev, scatter_fcs)) 4680 netdev->hw_features |= NETIF_F_RXFCS; 4681 4682 netdev->features = netdev->hw_features; 4683 if (!priv->channels.params.lro_en) 4684 netdev->features &= ~NETIF_F_LRO; 4685 4686 if (fcs_enabled) 4687 netdev->features &= ~NETIF_F_RXALL; 4688 4689 if (!priv->channels.params.scatter_fcs_en) 4690 netdev->features &= ~NETIF_F_RXFCS; 4691 4692 /* prefere CQE compression over rxhash */ 4693 if (MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_RX_CQE_COMPRESS)) 4694 netdev->features &= ~NETIF_F_RXHASH; 4695 4696 #define FT_CAP(f) MLX5_CAP_FLOWTABLE(mdev, flow_table_properties_nic_receive.f) 4697 if (FT_CAP(flow_modify_en) && 4698 FT_CAP(modify_root) && 4699 FT_CAP(identified_miss_table_mode) && 4700 FT_CAP(flow_table_modify)) { 4701 #ifdef CONFIG_MLX5_ESWITCH 4702 netdev->hw_features |= NETIF_F_HW_TC; 4703 #endif 4704 #ifdef CONFIG_MLX5_EN_ARFS 4705 netdev->hw_features |= NETIF_F_NTUPLE; 4706 #endif 4707 } 4708 4709 netdev->features |= NETIF_F_HIGHDMA; 4710 netdev->features |= NETIF_F_HW_VLAN_STAG_FILTER; 4711 4712 netdev->priv_flags |= IFF_UNICAST_FLT; 4713 4714 mlx5e_set_netdev_dev_addr(netdev); 4715 mlx5e_ipsec_build_netdev(priv); 4716 mlx5e_tls_build_netdev(priv); 4717 } 4718 4719 void mlx5e_create_q_counters(struct mlx5e_priv *priv) 4720 { 4721 struct mlx5_core_dev *mdev = priv->mdev; 4722 int err; 4723 4724 err = mlx5_core_alloc_q_counter(mdev, &priv->q_counter); 4725 if (err) { 4726 mlx5_core_warn(mdev, "alloc queue counter failed, %d\n", err); 4727 priv->q_counter = 0; 4728 } 4729 4730 err = mlx5_core_alloc_q_counter(mdev, &priv->drop_rq_q_counter); 4731 if (err) { 4732 mlx5_core_warn(mdev, "alloc drop RQ counter failed, %d\n", err); 4733 priv->drop_rq_q_counter = 0; 4734 } 4735 } 4736 4737 void mlx5e_destroy_q_counters(struct mlx5e_priv *priv) 4738 { 4739 if (priv->q_counter) 4740 mlx5_core_dealloc_q_counter(priv->mdev, priv->q_counter); 4741 4742 if (priv->drop_rq_q_counter) 4743 mlx5_core_dealloc_q_counter(priv->mdev, priv->drop_rq_q_counter); 4744 } 4745 4746 static int mlx5e_nic_init(struct mlx5_core_dev *mdev, 4747 struct net_device *netdev, 4748 const struct mlx5e_profile *profile, 4749 void *ppriv) 4750 { 4751 struct mlx5e_priv *priv = netdev_priv(netdev); 4752 struct mlx5e_rss_params *rss = &priv->rss_params; 4753 int err; 4754 4755 err = mlx5e_netdev_init(netdev, priv, mdev, profile, ppriv); 4756 if (err) 4757 return err; 4758 4759 mlx5e_build_nic_params(mdev, rss, &priv->channels.params, 4760 mlx5e_get_netdev_max_channels(netdev), 4761 netdev->mtu); 4762 4763 mlx5e_timestamp_init(priv); 4764 4765 err = mlx5e_ipsec_init(priv); 4766 if (err) 4767 mlx5_core_err(mdev, "IPSec initialization failed, %d\n", err); 4768 err = mlx5e_tls_init(priv); 4769 if (err) 4770 mlx5_core_err(mdev, "TLS initialization failed, %d\n", err); 4771 mlx5e_build_nic_netdev(netdev); 4772 mlx5e_build_tc2txq_maps(priv); 4773 4774 return 0; 4775 } 4776 4777 static void mlx5e_nic_cleanup(struct mlx5e_priv *priv) 4778 { 4779 mlx5e_tls_cleanup(priv); 4780 mlx5e_ipsec_cleanup(priv); 4781 mlx5e_netdev_cleanup(priv->netdev, priv); 4782 } 4783 4784 static int mlx5e_init_nic_rx(struct mlx5e_priv *priv) 4785 { 4786 struct mlx5_core_dev *mdev = priv->mdev; 4787 int err; 4788 4789 mlx5e_create_q_counters(priv); 4790 4791 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 4792 if (err) { 4793 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 4794 goto err_destroy_q_counters; 4795 } 4796 4797 err = mlx5e_create_indirect_rqt(priv); 4798 if (err) 4799 goto err_close_drop_rq; 4800 4801 err = mlx5e_create_direct_rqts(priv); 4802 if (err) 4803 goto err_destroy_indirect_rqts; 4804 4805 err = mlx5e_create_indirect_tirs(priv, true); 4806 if (err) 4807 goto err_destroy_direct_rqts; 4808 4809 err = mlx5e_create_direct_tirs(priv); 4810 if (err) 4811 goto err_destroy_indirect_tirs; 4812 4813 err = mlx5e_create_flow_steering(priv); 4814 if (err) { 4815 mlx5_core_warn(mdev, "create flow steering failed, %d\n", err); 4816 goto err_destroy_direct_tirs; 4817 } 4818 4819 err = mlx5e_tc_nic_init(priv); 4820 if (err) 4821 goto err_destroy_flow_steering; 4822 4823 return 0; 4824 4825 err_destroy_flow_steering: 4826 mlx5e_destroy_flow_steering(priv); 4827 err_destroy_direct_tirs: 4828 mlx5e_destroy_direct_tirs(priv); 4829 err_destroy_indirect_tirs: 4830 mlx5e_destroy_indirect_tirs(priv, true); 4831 err_destroy_direct_rqts: 4832 mlx5e_destroy_direct_rqts(priv); 4833 err_destroy_indirect_rqts: 4834 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 4835 err_close_drop_rq: 4836 mlx5e_close_drop_rq(&priv->drop_rq); 4837 err_destroy_q_counters: 4838 mlx5e_destroy_q_counters(priv); 4839 return err; 4840 } 4841 4842 static void mlx5e_cleanup_nic_rx(struct mlx5e_priv *priv) 4843 { 4844 mlx5e_tc_nic_cleanup(priv); 4845 mlx5e_destroy_flow_steering(priv); 4846 mlx5e_destroy_direct_tirs(priv); 4847 mlx5e_destroy_indirect_tirs(priv, true); 4848 mlx5e_destroy_direct_rqts(priv); 4849 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 4850 mlx5e_close_drop_rq(&priv->drop_rq); 4851 mlx5e_destroy_q_counters(priv); 4852 } 4853 4854 static int mlx5e_init_nic_tx(struct mlx5e_priv *priv) 4855 { 4856 int err; 4857 4858 err = mlx5e_create_tises(priv); 4859 if (err) { 4860 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); 4861 return err; 4862 } 4863 4864 #ifdef CONFIG_MLX5_CORE_EN_DCB 4865 mlx5e_dcbnl_initialize(priv); 4866 #endif 4867 mlx5e_tx_reporter_create(priv); 4868 return 0; 4869 } 4870 4871 static void mlx5e_nic_enable(struct mlx5e_priv *priv) 4872 { 4873 struct net_device *netdev = priv->netdev; 4874 struct mlx5_core_dev *mdev = priv->mdev; 4875 4876 mlx5e_init_l2_addr(priv); 4877 4878 /* Marking the link as currently not needed by the Driver */ 4879 if (!netif_running(netdev)) 4880 mlx5_set_port_admin_status(mdev, MLX5_PORT_DOWN); 4881 4882 mlx5e_set_netdev_mtu_boundaries(priv); 4883 mlx5e_set_dev_port_mtu(priv); 4884 4885 mlx5_lag_add(mdev, netdev); 4886 4887 mlx5e_enable_async_events(priv); 4888 if (mlx5e_monitor_counter_supported(priv)) 4889 mlx5e_monitor_counter_init(priv); 4890 4891 if (netdev->reg_state != NETREG_REGISTERED) 4892 return; 4893 #ifdef CONFIG_MLX5_CORE_EN_DCB 4894 mlx5e_dcbnl_init_app(priv); 4895 #endif 4896 4897 queue_work(priv->wq, &priv->set_rx_mode_work); 4898 4899 rtnl_lock(); 4900 if (netif_running(netdev)) 4901 mlx5e_open(netdev); 4902 netif_device_attach(netdev); 4903 rtnl_unlock(); 4904 } 4905 4906 static void mlx5e_nic_disable(struct mlx5e_priv *priv) 4907 { 4908 struct mlx5_core_dev *mdev = priv->mdev; 4909 4910 #ifdef CONFIG_MLX5_CORE_EN_DCB 4911 if (priv->netdev->reg_state == NETREG_REGISTERED) 4912 mlx5e_dcbnl_delete_app(priv); 4913 #endif 4914 4915 rtnl_lock(); 4916 if (netif_running(priv->netdev)) 4917 mlx5e_close(priv->netdev); 4918 netif_device_detach(priv->netdev); 4919 rtnl_unlock(); 4920 4921 queue_work(priv->wq, &priv->set_rx_mode_work); 4922 4923 if (mlx5e_monitor_counter_supported(priv)) 4924 mlx5e_monitor_counter_cleanup(priv); 4925 4926 mlx5e_disable_async_events(priv); 4927 mlx5_lag_remove(mdev); 4928 } 4929 4930 static const struct mlx5e_profile mlx5e_nic_profile = { 4931 .init = mlx5e_nic_init, 4932 .cleanup = mlx5e_nic_cleanup, 4933 .init_rx = mlx5e_init_nic_rx, 4934 .cleanup_rx = mlx5e_cleanup_nic_rx, 4935 .init_tx = mlx5e_init_nic_tx, 4936 .cleanup_tx = mlx5e_cleanup_nic_tx, 4937 .enable = mlx5e_nic_enable, 4938 .disable = mlx5e_nic_disable, 4939 .update_stats = mlx5e_update_ndo_stats, 4940 .update_carrier = mlx5e_update_carrier, 4941 .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe, 4942 .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, 4943 .max_tc = MLX5E_MAX_NUM_TC, 4944 }; 4945 4946 /* mlx5e generic netdev management API (move to en_common.c) */ 4947 4948 /* mlx5e_netdev_init/cleanup must be called from profile->init/cleanup callbacks */ 4949 int mlx5e_netdev_init(struct net_device *netdev, 4950 struct mlx5e_priv *priv, 4951 struct mlx5_core_dev *mdev, 4952 const struct mlx5e_profile *profile, 4953 void *ppriv) 4954 { 4955 /* priv init */ 4956 priv->mdev = mdev; 4957 priv->netdev = netdev; 4958 priv->profile = profile; 4959 priv->ppriv = ppriv; 4960 priv->msglevel = MLX5E_MSG_LEVEL; 4961 priv->max_opened_tc = 1; 4962 4963 mutex_init(&priv->state_lock); 4964 INIT_WORK(&priv->update_carrier_work, mlx5e_update_carrier_work); 4965 INIT_WORK(&priv->set_rx_mode_work, mlx5e_set_rx_mode_work); 4966 INIT_WORK(&priv->tx_timeout_work, mlx5e_tx_timeout_work); 4967 INIT_WORK(&priv->update_stats_work, mlx5e_update_stats_work); 4968 4969 priv->wq = create_singlethread_workqueue("mlx5e"); 4970 if (!priv->wq) 4971 return -ENOMEM; 4972 4973 /* netdev init */ 4974 netif_carrier_off(netdev); 4975 4976 #ifdef CONFIG_MLX5_EN_ARFS 4977 netdev->rx_cpu_rmap = mlx5_eq_table_get_rmap(mdev); 4978 #endif 4979 4980 return 0; 4981 } 4982 4983 void mlx5e_netdev_cleanup(struct net_device *netdev, struct mlx5e_priv *priv) 4984 { 4985 destroy_workqueue(priv->wq); 4986 } 4987 4988 struct net_device *mlx5e_create_netdev(struct mlx5_core_dev *mdev, 4989 const struct mlx5e_profile *profile, 4990 int nch, 4991 void *ppriv) 4992 { 4993 struct net_device *netdev; 4994 int err; 4995 4996 netdev = alloc_etherdev_mqs(sizeof(struct mlx5e_priv), 4997 nch * profile->max_tc, 4998 nch); 4999 if (!netdev) { 5000 mlx5_core_err(mdev, "alloc_etherdev_mqs() failed\n"); 5001 return NULL; 5002 } 5003 5004 err = profile->init(mdev, netdev, profile, ppriv); 5005 if (err) { 5006 mlx5_core_err(mdev, "failed to init mlx5e profile %d\n", err); 5007 goto err_free_netdev; 5008 } 5009 5010 return netdev; 5011 5012 err_free_netdev: 5013 free_netdev(netdev); 5014 5015 return NULL; 5016 } 5017 5018 int mlx5e_attach_netdev(struct mlx5e_priv *priv) 5019 { 5020 const struct mlx5e_profile *profile; 5021 int max_nch; 5022 int err; 5023 5024 profile = priv->profile; 5025 clear_bit(MLX5E_STATE_DESTROYING, &priv->state); 5026 5027 /* max number of channels may have changed */ 5028 max_nch = mlx5e_get_max_num_channels(priv->mdev); 5029 if (priv->channels.params.num_channels > max_nch) { 5030 mlx5_core_warn(priv->mdev, "MLX5E: Reducing number of channels to %d\n", max_nch); 5031 priv->channels.params.num_channels = max_nch; 5032 mlx5e_build_default_indir_rqt(priv->rss_params.indirection_rqt, 5033 MLX5E_INDIR_RQT_SIZE, max_nch); 5034 } 5035 5036 err = profile->init_tx(priv); 5037 if (err) 5038 goto out; 5039 5040 err = profile->init_rx(priv); 5041 if (err) 5042 goto err_cleanup_tx; 5043 5044 if (profile->enable) 5045 profile->enable(priv); 5046 5047 return 0; 5048 5049 err_cleanup_tx: 5050 profile->cleanup_tx(priv); 5051 5052 out: 5053 return err; 5054 } 5055 5056 void mlx5e_detach_netdev(struct mlx5e_priv *priv) 5057 { 5058 const struct mlx5e_profile *profile = priv->profile; 5059 5060 set_bit(MLX5E_STATE_DESTROYING, &priv->state); 5061 5062 if (profile->disable) 5063 profile->disable(priv); 5064 flush_workqueue(priv->wq); 5065 5066 profile->cleanup_rx(priv); 5067 profile->cleanup_tx(priv); 5068 cancel_work_sync(&priv->update_stats_work); 5069 } 5070 5071 void mlx5e_destroy_netdev(struct mlx5e_priv *priv) 5072 { 5073 const struct mlx5e_profile *profile = priv->profile; 5074 struct net_device *netdev = priv->netdev; 5075 5076 if (profile->cleanup) 5077 profile->cleanup(priv); 5078 free_netdev(netdev); 5079 } 5080 5081 /* mlx5e_attach and mlx5e_detach scope should be only creating/destroying 5082 * hardware contexts and to connect it to the current netdev. 5083 */ 5084 static int mlx5e_attach(struct mlx5_core_dev *mdev, void *vpriv) 5085 { 5086 struct mlx5e_priv *priv = vpriv; 5087 struct net_device *netdev = priv->netdev; 5088 int err; 5089 5090 if (netif_device_present(netdev)) 5091 return 0; 5092 5093 err = mlx5e_create_mdev_resources(mdev); 5094 if (err) 5095 return err; 5096 5097 err = mlx5e_attach_netdev(priv); 5098 if (err) { 5099 mlx5e_destroy_mdev_resources(mdev); 5100 return err; 5101 } 5102 5103 return 0; 5104 } 5105 5106 static void mlx5e_detach(struct mlx5_core_dev *mdev, void *vpriv) 5107 { 5108 struct mlx5e_priv *priv = vpriv; 5109 struct net_device *netdev = priv->netdev; 5110 5111 #ifdef CONFIG_MLX5_ESWITCH 5112 if (MLX5_ESWITCH_MANAGER(mdev) && vpriv == mdev) 5113 return; 5114 #endif 5115 5116 if (!netif_device_present(netdev)) 5117 return; 5118 5119 mlx5e_detach_netdev(priv); 5120 mlx5e_destroy_mdev_resources(mdev); 5121 } 5122 5123 static void *mlx5e_add(struct mlx5_core_dev *mdev) 5124 { 5125 struct net_device *netdev; 5126 void *priv; 5127 int err; 5128 int nch; 5129 5130 err = mlx5e_check_required_hca_cap(mdev); 5131 if (err) 5132 return NULL; 5133 5134 #ifdef CONFIG_MLX5_ESWITCH 5135 if (MLX5_ESWITCH_MANAGER(mdev) && 5136 mlx5_eswitch_mode(mdev->priv.eswitch) == SRIOV_OFFLOADS) { 5137 mlx5e_rep_register_vport_reps(mdev); 5138 return mdev; 5139 } 5140 #endif 5141 5142 nch = mlx5e_get_max_num_channels(mdev); 5143 netdev = mlx5e_create_netdev(mdev, &mlx5e_nic_profile, nch, NULL); 5144 if (!netdev) { 5145 mlx5_core_err(mdev, "mlx5e_create_netdev failed\n"); 5146 return NULL; 5147 } 5148 5149 priv = netdev_priv(netdev); 5150 5151 err = mlx5e_attach(mdev, priv); 5152 if (err) { 5153 mlx5_core_err(mdev, "mlx5e_attach failed, %d\n", err); 5154 goto err_destroy_netdev; 5155 } 5156 5157 err = register_netdev(netdev); 5158 if (err) { 5159 mlx5_core_err(mdev, "register_netdev failed, %d\n", err); 5160 goto err_detach; 5161 } 5162 5163 #ifdef CONFIG_MLX5_CORE_EN_DCB 5164 mlx5e_dcbnl_init_app(priv); 5165 #endif 5166 return priv; 5167 5168 err_detach: 5169 mlx5e_detach(mdev, priv); 5170 err_destroy_netdev: 5171 mlx5e_destroy_netdev(priv); 5172 return NULL; 5173 } 5174 5175 static void mlx5e_remove(struct mlx5_core_dev *mdev, void *vpriv) 5176 { 5177 struct mlx5e_priv *priv; 5178 5179 #ifdef CONFIG_MLX5_ESWITCH 5180 if (MLX5_ESWITCH_MANAGER(mdev) && vpriv == mdev) { 5181 mlx5e_rep_unregister_vport_reps(mdev); 5182 return; 5183 } 5184 #endif 5185 priv = vpriv; 5186 #ifdef CONFIG_MLX5_CORE_EN_DCB 5187 mlx5e_dcbnl_delete_app(priv); 5188 #endif 5189 unregister_netdev(priv->netdev); 5190 mlx5e_detach(mdev, vpriv); 5191 mlx5e_destroy_netdev(priv); 5192 } 5193 5194 static struct mlx5_interface mlx5e_interface = { 5195 .add = mlx5e_add, 5196 .remove = mlx5e_remove, 5197 .attach = mlx5e_attach, 5198 .detach = mlx5e_detach, 5199 .protocol = MLX5_INTERFACE_PROTOCOL_ETH, 5200 }; 5201 5202 void mlx5e_init(void) 5203 { 5204 mlx5e_ipsec_build_inverse_table(); 5205 mlx5e_build_ptys2ethtool_map(); 5206 mlx5_register_interface(&mlx5e_interface); 5207 } 5208 5209 void mlx5e_cleanup(void) 5210 { 5211 mlx5_unregister_interface(&mlx5e_interface); 5212 } 5213