1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <generated/utsrelease.h> 34 #include <linux/mlx5/fs.h> 35 #include <net/switchdev.h> 36 #include <net/pkt_cls.h> 37 #include <net/act_api.h> 38 #include <net/netevent.h> 39 #include <net/arp.h> 40 #include <net/devlink.h> 41 #include <net/ipv6_stubs.h> 42 43 #include "eswitch.h" 44 #include "eswitch_offloads_chains.h" 45 #include "en.h" 46 #include "en_rep.h" 47 #include "en_tc.h" 48 #include "en/tc_tun.h" 49 #include "fs_core.h" 50 #include "lib/port_tun.h" 51 #include "lib/mlx5.h" 52 #define CREATE_TRACE_POINTS 53 #include "diag/en_rep_tracepoint.h" 54 55 #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \ 56 max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE) 57 #define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1 58 59 static const char mlx5e_rep_driver_name[] = "mlx5e_rep"; 60 61 struct mlx5e_rep_indr_block_priv { 62 struct net_device *netdev; 63 struct mlx5e_rep_priv *rpriv; 64 65 struct list_head list; 66 }; 67 68 static void mlx5e_rep_indr_unregister_block(struct mlx5e_rep_priv *rpriv, 69 struct net_device *netdev); 70 71 static void mlx5e_rep_get_drvinfo(struct net_device *dev, 72 struct ethtool_drvinfo *drvinfo) 73 { 74 struct mlx5e_priv *priv = netdev_priv(dev); 75 struct mlx5_core_dev *mdev = priv->mdev; 76 77 strlcpy(drvinfo->driver, mlx5e_rep_driver_name, 78 sizeof(drvinfo->driver)); 79 strlcpy(drvinfo->version, UTS_RELEASE, sizeof(drvinfo->version)); 80 snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), 81 "%d.%d.%04d (%.16s)", 82 fw_rev_maj(mdev), fw_rev_min(mdev), 83 fw_rev_sub(mdev), mdev->board_id); 84 } 85 86 static void mlx5e_uplink_rep_get_drvinfo(struct net_device *dev, 87 struct ethtool_drvinfo *drvinfo) 88 { 89 struct mlx5e_priv *priv = netdev_priv(dev); 90 91 mlx5e_rep_get_drvinfo(dev, drvinfo); 92 strlcpy(drvinfo->bus_info, pci_name(priv->mdev->pdev), 93 sizeof(drvinfo->bus_info)); 94 } 95 96 static const struct counter_desc sw_rep_stats_desc[] = { 97 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) }, 98 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_bytes) }, 99 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_packets) }, 100 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) }, 101 }; 102 103 struct vport_stats { 104 u64 vport_rx_packets; 105 u64 vport_tx_packets; 106 u64 vport_rx_bytes; 107 u64 vport_tx_bytes; 108 }; 109 110 static const struct counter_desc vport_rep_stats_desc[] = { 111 { MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_packets) }, 112 { MLX5E_DECLARE_STAT(struct vport_stats, vport_rx_bytes) }, 113 { MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_packets) }, 114 { MLX5E_DECLARE_STAT(struct vport_stats, vport_tx_bytes) }, 115 }; 116 117 #define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc) 118 #define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc) 119 120 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep) 121 { 122 return NUM_VPORT_REP_SW_COUNTERS; 123 } 124 125 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw_rep) 126 { 127 int i; 128 129 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++) 130 strcpy(data + (idx++) * ETH_GSTRING_LEN, 131 sw_rep_stats_desc[i].format); 132 return idx; 133 } 134 135 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw_rep) 136 { 137 int i; 138 139 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++) 140 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.sw, 141 sw_rep_stats_desc, i); 142 return idx; 143 } 144 145 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep) 146 { 147 struct mlx5e_sw_stats *s = &priv->stats.sw; 148 struct rtnl_link_stats64 stats64 = {}; 149 150 memset(s, 0, sizeof(*s)); 151 mlx5e_fold_sw_stats64(priv, &stats64); 152 153 s->rx_packets = stats64.rx_packets; 154 s->rx_bytes = stats64.rx_bytes; 155 s->tx_packets = stats64.tx_packets; 156 s->tx_bytes = stats64.tx_bytes; 157 s->tx_queue_dropped = stats64.tx_dropped; 158 } 159 160 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep) 161 { 162 return NUM_VPORT_REP_HW_COUNTERS; 163 } 164 165 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep) 166 { 167 int i; 168 169 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++) 170 strcpy(data + (idx++) * ETH_GSTRING_LEN, vport_rep_stats_desc[i].format); 171 return idx; 172 } 173 174 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep) 175 { 176 int i; 177 178 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++) 179 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.vf_vport, 180 vport_rep_stats_desc, i); 181 return idx; 182 } 183 184 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep) 185 { 186 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 187 struct mlx5e_rep_priv *rpriv = priv->ppriv; 188 struct mlx5_eswitch_rep *rep = rpriv->rep; 189 struct rtnl_link_stats64 *vport_stats; 190 struct ifla_vf_stats vf_stats; 191 int err; 192 193 err = mlx5_eswitch_get_vport_stats(esw, rep->vport, &vf_stats); 194 if (err) { 195 pr_warn("vport %d error %d reading stats\n", rep->vport, err); 196 return; 197 } 198 199 vport_stats = &priv->stats.vf_vport; 200 /* flip tx/rx as we are reporting the counters for the switch vport */ 201 vport_stats->rx_packets = vf_stats.tx_packets; 202 vport_stats->rx_bytes = vf_stats.tx_bytes; 203 vport_stats->tx_packets = vf_stats.rx_packets; 204 vport_stats->tx_bytes = vf_stats.rx_bytes; 205 } 206 207 static void mlx5e_rep_get_strings(struct net_device *dev, 208 u32 stringset, uint8_t *data) 209 { 210 struct mlx5e_priv *priv = netdev_priv(dev); 211 212 switch (stringset) { 213 case ETH_SS_STATS: 214 mlx5e_stats_fill_strings(priv, data); 215 break; 216 } 217 } 218 219 static void mlx5e_rep_get_ethtool_stats(struct net_device *dev, 220 struct ethtool_stats *stats, u64 *data) 221 { 222 struct mlx5e_priv *priv = netdev_priv(dev); 223 224 mlx5e_ethtool_get_ethtool_stats(priv, stats, data); 225 } 226 227 static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset) 228 { 229 struct mlx5e_priv *priv = netdev_priv(dev); 230 231 switch (sset) { 232 case ETH_SS_STATS: 233 return mlx5e_stats_total_num(priv); 234 default: 235 return -EOPNOTSUPP; 236 } 237 } 238 239 static void mlx5e_rep_get_ringparam(struct net_device *dev, 240 struct ethtool_ringparam *param) 241 { 242 struct mlx5e_priv *priv = netdev_priv(dev); 243 244 mlx5e_ethtool_get_ringparam(priv, param); 245 } 246 247 static int mlx5e_rep_set_ringparam(struct net_device *dev, 248 struct ethtool_ringparam *param) 249 { 250 struct mlx5e_priv *priv = netdev_priv(dev); 251 252 return mlx5e_ethtool_set_ringparam(priv, param); 253 } 254 255 static int mlx5e_replace_rep_vport_rx_rule(struct mlx5e_priv *priv, 256 struct mlx5_flow_destination *dest) 257 { 258 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 259 struct mlx5e_rep_priv *rpriv = priv->ppriv; 260 struct mlx5_eswitch_rep *rep = rpriv->rep; 261 struct mlx5_flow_handle *flow_rule; 262 263 flow_rule = mlx5_eswitch_create_vport_rx_rule(esw, 264 rep->vport, 265 dest); 266 if (IS_ERR(flow_rule)) 267 return PTR_ERR(flow_rule); 268 269 mlx5_del_flow_rules(rpriv->vport_rx_rule); 270 rpriv->vport_rx_rule = flow_rule; 271 return 0; 272 } 273 274 static void mlx5e_rep_get_channels(struct net_device *dev, 275 struct ethtool_channels *ch) 276 { 277 struct mlx5e_priv *priv = netdev_priv(dev); 278 279 mlx5e_ethtool_get_channels(priv, ch); 280 } 281 282 static int mlx5e_rep_set_channels(struct net_device *dev, 283 struct ethtool_channels *ch) 284 { 285 struct mlx5e_priv *priv = netdev_priv(dev); 286 u16 curr_channels_amount = priv->channels.params.num_channels; 287 u32 new_channels_amount = ch->combined_count; 288 struct mlx5_flow_destination new_dest; 289 int err = 0; 290 291 err = mlx5e_ethtool_set_channels(priv, ch); 292 if (err) 293 return err; 294 295 if (curr_channels_amount == 1 && new_channels_amount > 1) { 296 new_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 297 new_dest.ft = priv->fs.ttc.ft.t; 298 } else if (new_channels_amount == 1 && curr_channels_amount > 1) { 299 new_dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 300 new_dest.tir_num = priv->direct_tir[0].tirn; 301 } else { 302 return 0; 303 } 304 305 err = mlx5e_replace_rep_vport_rx_rule(priv, &new_dest); 306 if (err) { 307 netdev_warn(priv->netdev, "Failed to update vport rx rule, when going from (%d) channels to (%d) channels\n", 308 curr_channels_amount, new_channels_amount); 309 return err; 310 } 311 312 return 0; 313 } 314 315 static int mlx5e_rep_get_coalesce(struct net_device *netdev, 316 struct ethtool_coalesce *coal) 317 { 318 struct mlx5e_priv *priv = netdev_priv(netdev); 319 320 return mlx5e_ethtool_get_coalesce(priv, coal); 321 } 322 323 static int mlx5e_rep_set_coalesce(struct net_device *netdev, 324 struct ethtool_coalesce *coal) 325 { 326 struct mlx5e_priv *priv = netdev_priv(netdev); 327 328 return mlx5e_ethtool_set_coalesce(priv, coal); 329 } 330 331 static u32 mlx5e_rep_get_rxfh_key_size(struct net_device *netdev) 332 { 333 struct mlx5e_priv *priv = netdev_priv(netdev); 334 335 return mlx5e_ethtool_get_rxfh_key_size(priv); 336 } 337 338 static u32 mlx5e_rep_get_rxfh_indir_size(struct net_device *netdev) 339 { 340 struct mlx5e_priv *priv = netdev_priv(netdev); 341 342 return mlx5e_ethtool_get_rxfh_indir_size(priv); 343 } 344 345 static void mlx5e_uplink_rep_get_pauseparam(struct net_device *netdev, 346 struct ethtool_pauseparam *pauseparam) 347 { 348 struct mlx5e_priv *priv = netdev_priv(netdev); 349 350 mlx5e_ethtool_get_pauseparam(priv, pauseparam); 351 } 352 353 static int mlx5e_uplink_rep_set_pauseparam(struct net_device *netdev, 354 struct ethtool_pauseparam *pauseparam) 355 { 356 struct mlx5e_priv *priv = netdev_priv(netdev); 357 358 return mlx5e_ethtool_set_pauseparam(priv, pauseparam); 359 } 360 361 static int mlx5e_uplink_rep_get_link_ksettings(struct net_device *netdev, 362 struct ethtool_link_ksettings *link_ksettings) 363 { 364 struct mlx5e_priv *priv = netdev_priv(netdev); 365 366 return mlx5e_ethtool_get_link_ksettings(priv, link_ksettings); 367 } 368 369 static int mlx5e_uplink_rep_set_link_ksettings(struct net_device *netdev, 370 const struct ethtool_link_ksettings *link_ksettings) 371 { 372 struct mlx5e_priv *priv = netdev_priv(netdev); 373 374 return mlx5e_ethtool_set_link_ksettings(priv, link_ksettings); 375 } 376 377 static const struct ethtool_ops mlx5e_rep_ethtool_ops = { 378 .get_drvinfo = mlx5e_rep_get_drvinfo, 379 .get_link = ethtool_op_get_link, 380 .get_strings = mlx5e_rep_get_strings, 381 .get_sset_count = mlx5e_rep_get_sset_count, 382 .get_ethtool_stats = mlx5e_rep_get_ethtool_stats, 383 .get_ringparam = mlx5e_rep_get_ringparam, 384 .set_ringparam = mlx5e_rep_set_ringparam, 385 .get_channels = mlx5e_rep_get_channels, 386 .set_channels = mlx5e_rep_set_channels, 387 .get_coalesce = mlx5e_rep_get_coalesce, 388 .set_coalesce = mlx5e_rep_set_coalesce, 389 .get_rxfh_key_size = mlx5e_rep_get_rxfh_key_size, 390 .get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size, 391 }; 392 393 static const struct ethtool_ops mlx5e_uplink_rep_ethtool_ops = { 394 .get_drvinfo = mlx5e_uplink_rep_get_drvinfo, 395 .get_link = ethtool_op_get_link, 396 .get_strings = mlx5e_rep_get_strings, 397 .get_sset_count = mlx5e_rep_get_sset_count, 398 .get_ethtool_stats = mlx5e_rep_get_ethtool_stats, 399 .get_ringparam = mlx5e_rep_get_ringparam, 400 .set_ringparam = mlx5e_rep_set_ringparam, 401 .get_channels = mlx5e_rep_get_channels, 402 .set_channels = mlx5e_rep_set_channels, 403 .get_coalesce = mlx5e_rep_get_coalesce, 404 .set_coalesce = mlx5e_rep_set_coalesce, 405 .get_link_ksettings = mlx5e_uplink_rep_get_link_ksettings, 406 .set_link_ksettings = mlx5e_uplink_rep_set_link_ksettings, 407 .get_rxfh_key_size = mlx5e_rep_get_rxfh_key_size, 408 .get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size, 409 .get_pauseparam = mlx5e_uplink_rep_get_pauseparam, 410 .set_pauseparam = mlx5e_uplink_rep_set_pauseparam, 411 }; 412 413 static void mlx5e_rep_get_port_parent_id(struct net_device *dev, 414 struct netdev_phys_item_id *ppid) 415 { 416 struct mlx5e_priv *priv; 417 u64 parent_id; 418 419 priv = netdev_priv(dev); 420 421 parent_id = mlx5_query_nic_system_image_guid(priv->mdev); 422 ppid->id_len = sizeof(parent_id); 423 memcpy(ppid->id, &parent_id, sizeof(parent_id)); 424 } 425 426 static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw, 427 struct mlx5_eswitch_rep *rep) 428 { 429 struct mlx5e_rep_sq *rep_sq, *tmp; 430 struct mlx5e_rep_priv *rpriv; 431 432 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 433 return; 434 435 rpriv = mlx5e_rep_to_rep_priv(rep); 436 list_for_each_entry_safe(rep_sq, tmp, &rpriv->vport_sqs_list, list) { 437 mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule); 438 list_del(&rep_sq->list); 439 kfree(rep_sq); 440 } 441 } 442 443 static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, 444 struct mlx5_eswitch_rep *rep, 445 u32 *sqns_array, int sqns_num) 446 { 447 struct mlx5_flow_handle *flow_rule; 448 struct mlx5e_rep_priv *rpriv; 449 struct mlx5e_rep_sq *rep_sq; 450 int err; 451 int i; 452 453 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 454 return 0; 455 456 rpriv = mlx5e_rep_to_rep_priv(rep); 457 for (i = 0; i < sqns_num; i++) { 458 rep_sq = kzalloc(sizeof(*rep_sq), GFP_KERNEL); 459 if (!rep_sq) { 460 err = -ENOMEM; 461 goto out_err; 462 } 463 464 /* Add re-inject rule to the PF/representor sqs */ 465 flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw, 466 rep->vport, 467 sqns_array[i]); 468 if (IS_ERR(flow_rule)) { 469 err = PTR_ERR(flow_rule); 470 kfree(rep_sq); 471 goto out_err; 472 } 473 rep_sq->send_to_vport_rule = flow_rule; 474 list_add(&rep_sq->list, &rpriv->vport_sqs_list); 475 } 476 return 0; 477 478 out_err: 479 mlx5e_sqs2vport_stop(esw, rep); 480 return err; 481 } 482 483 int mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) 484 { 485 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 486 struct mlx5e_rep_priv *rpriv = priv->ppriv; 487 struct mlx5_eswitch_rep *rep = rpriv->rep; 488 struct mlx5e_channel *c; 489 int n, tc, num_sqs = 0; 490 int err = -ENOMEM; 491 u32 *sqs; 492 493 sqs = kcalloc(priv->channels.num * priv->channels.params.num_tc, sizeof(*sqs), GFP_KERNEL); 494 if (!sqs) 495 goto out; 496 497 for (n = 0; n < priv->channels.num; n++) { 498 c = priv->channels.c[n]; 499 for (tc = 0; tc < c->num_tc; tc++) 500 sqs[num_sqs++] = c->sq[tc].sqn; 501 } 502 503 err = mlx5e_sqs2vport_start(esw, rep, sqs, num_sqs); 504 kfree(sqs); 505 506 out: 507 if (err) 508 netdev_warn(priv->netdev, "Failed to add SQs FWD rules %d\n", err); 509 return err; 510 } 511 512 void mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) 513 { 514 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 515 struct mlx5e_rep_priv *rpriv = priv->ppriv; 516 struct mlx5_eswitch_rep *rep = rpriv->rep; 517 518 mlx5e_sqs2vport_stop(esw, rep); 519 } 520 521 static unsigned long mlx5e_rep_ipv6_interval(void) 522 { 523 if (IS_ENABLED(CONFIG_IPV6) && ipv6_stub->nd_tbl) 524 return NEIGH_VAR(&ipv6_stub->nd_tbl->parms, DELAY_PROBE_TIME); 525 526 return ~0UL; 527 } 528 529 static void mlx5e_rep_neigh_update_init_interval(struct mlx5e_rep_priv *rpriv) 530 { 531 unsigned long ipv4_interval = NEIGH_VAR(&arp_tbl.parms, DELAY_PROBE_TIME); 532 unsigned long ipv6_interval = mlx5e_rep_ipv6_interval(); 533 struct net_device *netdev = rpriv->netdev; 534 struct mlx5e_priv *priv = netdev_priv(netdev); 535 536 rpriv->neigh_update.min_interval = min_t(unsigned long, ipv6_interval, ipv4_interval); 537 mlx5_fc_update_sampling_interval(priv->mdev, rpriv->neigh_update.min_interval); 538 } 539 540 void mlx5e_rep_queue_neigh_stats_work(struct mlx5e_priv *priv) 541 { 542 struct mlx5e_rep_priv *rpriv = priv->ppriv; 543 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update; 544 545 mlx5_fc_queue_stats_work(priv->mdev, 546 &neigh_update->neigh_stats_work, 547 neigh_update->min_interval); 548 } 549 550 static bool mlx5e_rep_neigh_entry_hold(struct mlx5e_neigh_hash_entry *nhe) 551 { 552 return refcount_inc_not_zero(&nhe->refcnt); 553 } 554 555 static void mlx5e_rep_neigh_entry_remove(struct mlx5e_neigh_hash_entry *nhe); 556 557 static void mlx5e_rep_neigh_entry_release(struct mlx5e_neigh_hash_entry *nhe) 558 { 559 if (refcount_dec_and_test(&nhe->refcnt)) { 560 mlx5e_rep_neigh_entry_remove(nhe); 561 kfree_rcu(nhe, rcu); 562 } 563 } 564 565 static struct mlx5e_neigh_hash_entry * 566 mlx5e_get_next_nhe(struct mlx5e_rep_priv *rpriv, 567 struct mlx5e_neigh_hash_entry *nhe) 568 { 569 struct mlx5e_neigh_hash_entry *next = NULL; 570 571 rcu_read_lock(); 572 573 for (next = nhe ? 574 list_next_or_null_rcu(&rpriv->neigh_update.neigh_list, 575 &nhe->neigh_list, 576 struct mlx5e_neigh_hash_entry, 577 neigh_list) : 578 list_first_or_null_rcu(&rpriv->neigh_update.neigh_list, 579 struct mlx5e_neigh_hash_entry, 580 neigh_list); 581 next; 582 next = list_next_or_null_rcu(&rpriv->neigh_update.neigh_list, 583 &next->neigh_list, 584 struct mlx5e_neigh_hash_entry, 585 neigh_list)) 586 if (mlx5e_rep_neigh_entry_hold(next)) 587 break; 588 589 rcu_read_unlock(); 590 591 if (nhe) 592 mlx5e_rep_neigh_entry_release(nhe); 593 594 return next; 595 } 596 597 static void mlx5e_rep_neigh_stats_work(struct work_struct *work) 598 { 599 struct mlx5e_rep_priv *rpriv = container_of(work, struct mlx5e_rep_priv, 600 neigh_update.neigh_stats_work.work); 601 struct net_device *netdev = rpriv->netdev; 602 struct mlx5e_priv *priv = netdev_priv(netdev); 603 struct mlx5e_neigh_hash_entry *nhe = NULL; 604 605 rtnl_lock(); 606 if (!list_empty(&rpriv->neigh_update.neigh_list)) 607 mlx5e_rep_queue_neigh_stats_work(priv); 608 609 while ((nhe = mlx5e_get_next_nhe(rpriv, nhe)) != NULL) 610 mlx5e_tc_update_neigh_used_value(nhe); 611 612 rtnl_unlock(); 613 } 614 615 static void mlx5e_rep_update_flows(struct mlx5e_priv *priv, 616 struct mlx5e_encap_entry *e, 617 bool neigh_connected, 618 unsigned char ha[ETH_ALEN]) 619 { 620 struct ethhdr *eth = (struct ethhdr *)e->encap_header; 621 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 622 bool encap_connected; 623 LIST_HEAD(flow_list); 624 625 ASSERT_RTNL(); 626 627 /* wait for encap to be fully initialized */ 628 wait_for_completion(&e->res_ready); 629 630 mutex_lock(&esw->offloads.encap_tbl_lock); 631 encap_connected = !!(e->flags & MLX5_ENCAP_ENTRY_VALID); 632 if (e->compl_result < 0 || (encap_connected == neigh_connected && 633 ether_addr_equal(e->h_dest, ha))) 634 goto unlock; 635 636 mlx5e_take_all_encap_flows(e, &flow_list); 637 638 if ((e->flags & MLX5_ENCAP_ENTRY_VALID) && 639 (!neigh_connected || !ether_addr_equal(e->h_dest, ha))) 640 mlx5e_tc_encap_flows_del(priv, e, &flow_list); 641 642 if (neigh_connected && !(e->flags & MLX5_ENCAP_ENTRY_VALID)) { 643 ether_addr_copy(e->h_dest, ha); 644 ether_addr_copy(eth->h_dest, ha); 645 /* Update the encap source mac, in case that we delete 646 * the flows when encap source mac changed. 647 */ 648 ether_addr_copy(eth->h_source, e->route_dev->dev_addr); 649 650 mlx5e_tc_encap_flows_add(priv, e, &flow_list); 651 } 652 unlock: 653 mutex_unlock(&esw->offloads.encap_tbl_lock); 654 mlx5e_put_encap_flow_list(priv, &flow_list); 655 } 656 657 static void mlx5e_rep_neigh_update(struct work_struct *work) 658 { 659 struct mlx5e_neigh_hash_entry *nhe = 660 container_of(work, struct mlx5e_neigh_hash_entry, neigh_update_work); 661 struct neighbour *n = nhe->n; 662 struct mlx5e_encap_entry *e; 663 unsigned char ha[ETH_ALEN]; 664 struct mlx5e_priv *priv; 665 bool neigh_connected; 666 u8 nud_state, dead; 667 668 rtnl_lock(); 669 670 /* If these parameters are changed after we release the lock, 671 * we'll receive another event letting us know about it. 672 * We use this lock to avoid inconsistency between the neigh validity 673 * and it's hw address. 674 */ 675 read_lock_bh(&n->lock); 676 memcpy(ha, n->ha, ETH_ALEN); 677 nud_state = n->nud_state; 678 dead = n->dead; 679 read_unlock_bh(&n->lock); 680 681 neigh_connected = (nud_state & NUD_VALID) && !dead; 682 683 trace_mlx5e_rep_neigh_update(nhe, ha, neigh_connected); 684 685 list_for_each_entry(e, &nhe->encap_list, encap_list) { 686 if (!mlx5e_encap_take(e)) 687 continue; 688 689 priv = netdev_priv(e->out_dev); 690 mlx5e_rep_update_flows(priv, e, neigh_connected, ha); 691 mlx5e_encap_put(priv, e); 692 } 693 mlx5e_rep_neigh_entry_release(nhe); 694 rtnl_unlock(); 695 neigh_release(n); 696 } 697 698 static struct mlx5e_rep_indr_block_priv * 699 mlx5e_rep_indr_block_priv_lookup(struct mlx5e_rep_priv *rpriv, 700 struct net_device *netdev) 701 { 702 struct mlx5e_rep_indr_block_priv *cb_priv; 703 704 /* All callback list access should be protected by RTNL. */ 705 ASSERT_RTNL(); 706 707 list_for_each_entry(cb_priv, 708 &rpriv->uplink_priv.tc_indr_block_priv_list, 709 list) 710 if (cb_priv->netdev == netdev) 711 return cb_priv; 712 713 return NULL; 714 } 715 716 static void mlx5e_rep_indr_clean_block_privs(struct mlx5e_rep_priv *rpriv) 717 { 718 struct mlx5e_rep_indr_block_priv *cb_priv, *temp; 719 struct list_head *head = &rpriv->uplink_priv.tc_indr_block_priv_list; 720 721 list_for_each_entry_safe(cb_priv, temp, head, list) { 722 mlx5e_rep_indr_unregister_block(rpriv, cb_priv->netdev); 723 kfree(cb_priv); 724 } 725 } 726 727 static int 728 mlx5e_rep_indr_offload(struct net_device *netdev, 729 struct flow_cls_offload *flower, 730 struct mlx5e_rep_indr_block_priv *indr_priv) 731 { 732 unsigned long flags = MLX5_TC_FLAG(EGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD); 733 struct mlx5e_priv *priv = netdev_priv(indr_priv->rpriv->netdev); 734 int err = 0; 735 736 switch (flower->command) { 737 case FLOW_CLS_REPLACE: 738 err = mlx5e_configure_flower(netdev, priv, flower, flags); 739 break; 740 case FLOW_CLS_DESTROY: 741 err = mlx5e_delete_flower(netdev, priv, flower, flags); 742 break; 743 case FLOW_CLS_STATS: 744 err = mlx5e_stats_flower(netdev, priv, flower, flags); 745 break; 746 default: 747 err = -EOPNOTSUPP; 748 } 749 750 return err; 751 } 752 753 static int mlx5e_rep_indr_setup_block_cb(enum tc_setup_type type, 754 void *type_data, void *indr_priv) 755 { 756 struct mlx5e_rep_indr_block_priv *priv = indr_priv; 757 758 switch (type) { 759 case TC_SETUP_CLSFLOWER: 760 return mlx5e_rep_indr_offload(priv->netdev, type_data, priv); 761 default: 762 return -EOPNOTSUPP; 763 } 764 } 765 766 static void mlx5e_rep_indr_tc_block_unbind(void *cb_priv) 767 { 768 struct mlx5e_rep_indr_block_priv *indr_priv = cb_priv; 769 770 list_del(&indr_priv->list); 771 kfree(indr_priv); 772 } 773 774 static LIST_HEAD(mlx5e_block_cb_list); 775 776 static int 777 mlx5e_rep_indr_setup_tc_block(struct net_device *netdev, 778 struct mlx5e_rep_priv *rpriv, 779 struct flow_block_offload *f) 780 { 781 struct mlx5e_rep_indr_block_priv *indr_priv; 782 struct flow_block_cb *block_cb; 783 784 if (f->binder_type != FLOW_BLOCK_BINDER_TYPE_CLSACT_INGRESS) 785 return -EOPNOTSUPP; 786 787 f->unlocked_driver_cb = true; 788 f->driver_block_list = &mlx5e_block_cb_list; 789 790 switch (f->command) { 791 case FLOW_BLOCK_BIND: 792 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev); 793 if (indr_priv) 794 return -EEXIST; 795 796 indr_priv = kmalloc(sizeof(*indr_priv), GFP_KERNEL); 797 if (!indr_priv) 798 return -ENOMEM; 799 800 indr_priv->netdev = netdev; 801 indr_priv->rpriv = rpriv; 802 list_add(&indr_priv->list, 803 &rpriv->uplink_priv.tc_indr_block_priv_list); 804 805 block_cb = flow_block_cb_alloc(mlx5e_rep_indr_setup_block_cb, 806 indr_priv, indr_priv, 807 mlx5e_rep_indr_tc_block_unbind); 808 if (IS_ERR(block_cb)) { 809 list_del(&indr_priv->list); 810 kfree(indr_priv); 811 return PTR_ERR(block_cb); 812 } 813 flow_block_cb_add(block_cb, f); 814 list_add_tail(&block_cb->driver_list, &mlx5e_block_cb_list); 815 816 return 0; 817 case FLOW_BLOCK_UNBIND: 818 indr_priv = mlx5e_rep_indr_block_priv_lookup(rpriv, netdev); 819 if (!indr_priv) 820 return -ENOENT; 821 822 block_cb = flow_block_cb_lookup(f->block, 823 mlx5e_rep_indr_setup_block_cb, 824 indr_priv); 825 if (!block_cb) 826 return -ENOENT; 827 828 flow_block_cb_remove(block_cb, f); 829 list_del(&block_cb->driver_list); 830 return 0; 831 default: 832 return -EOPNOTSUPP; 833 } 834 return 0; 835 } 836 837 static 838 int mlx5e_rep_indr_setup_tc_cb(struct net_device *netdev, void *cb_priv, 839 enum tc_setup_type type, void *type_data) 840 { 841 switch (type) { 842 case TC_SETUP_BLOCK: 843 return mlx5e_rep_indr_setup_tc_block(netdev, cb_priv, 844 type_data); 845 default: 846 return -EOPNOTSUPP; 847 } 848 } 849 850 static int mlx5e_rep_indr_register_block(struct mlx5e_rep_priv *rpriv, 851 struct net_device *netdev) 852 { 853 int err; 854 855 err = __flow_indr_block_cb_register(netdev, rpriv, 856 mlx5e_rep_indr_setup_tc_cb, 857 rpriv); 858 if (err) { 859 struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); 860 861 mlx5_core_err(priv->mdev, "Failed to register remote block notifier for %s err=%d\n", 862 netdev_name(netdev), err); 863 } 864 return err; 865 } 866 867 static void mlx5e_rep_indr_unregister_block(struct mlx5e_rep_priv *rpriv, 868 struct net_device *netdev) 869 { 870 __flow_indr_block_cb_unregister(netdev, mlx5e_rep_indr_setup_tc_cb, 871 rpriv); 872 } 873 874 static int mlx5e_nic_rep_netdevice_event(struct notifier_block *nb, 875 unsigned long event, void *ptr) 876 { 877 struct mlx5e_rep_priv *rpriv = container_of(nb, struct mlx5e_rep_priv, 878 uplink_priv.netdevice_nb); 879 struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); 880 struct net_device *netdev = netdev_notifier_info_to_dev(ptr); 881 882 if (!mlx5e_tc_tun_device_to_offload(priv, netdev) && 883 !(is_vlan_dev(netdev) && vlan_dev_real_dev(netdev) == rpriv->netdev)) 884 return NOTIFY_OK; 885 886 switch (event) { 887 case NETDEV_REGISTER: 888 mlx5e_rep_indr_register_block(rpriv, netdev); 889 break; 890 case NETDEV_UNREGISTER: 891 mlx5e_rep_indr_unregister_block(rpriv, netdev); 892 break; 893 } 894 return NOTIFY_OK; 895 } 896 897 static void 898 mlx5e_rep_queue_neigh_update_work(struct mlx5e_priv *priv, 899 struct mlx5e_neigh_hash_entry *nhe, 900 struct neighbour *n) 901 { 902 /* Take a reference to ensure the neighbour and mlx5 encap 903 * entry won't be destructed until we drop the reference in 904 * delayed work. 905 */ 906 neigh_hold(n); 907 908 /* This assignment is valid as long as the the neigh reference 909 * is taken 910 */ 911 nhe->n = n; 912 913 if (!queue_work(priv->wq, &nhe->neigh_update_work)) { 914 mlx5e_rep_neigh_entry_release(nhe); 915 neigh_release(n); 916 } 917 } 918 919 static struct mlx5e_neigh_hash_entry * 920 mlx5e_rep_neigh_entry_lookup(struct mlx5e_priv *priv, 921 struct mlx5e_neigh *m_neigh); 922 923 static int mlx5e_rep_netevent_event(struct notifier_block *nb, 924 unsigned long event, void *ptr) 925 { 926 struct mlx5e_rep_priv *rpriv = container_of(nb, struct mlx5e_rep_priv, 927 neigh_update.netevent_nb); 928 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update; 929 struct net_device *netdev = rpriv->netdev; 930 struct mlx5e_priv *priv = netdev_priv(netdev); 931 struct mlx5e_neigh_hash_entry *nhe = NULL; 932 struct mlx5e_neigh m_neigh = {}; 933 struct neigh_parms *p; 934 struct neighbour *n; 935 bool found = false; 936 937 switch (event) { 938 case NETEVENT_NEIGH_UPDATE: 939 n = ptr; 940 #if IS_ENABLED(CONFIG_IPV6) 941 if (n->tbl != ipv6_stub->nd_tbl && n->tbl != &arp_tbl) 942 #else 943 if (n->tbl != &arp_tbl) 944 #endif 945 return NOTIFY_DONE; 946 947 m_neigh.dev = n->dev; 948 m_neigh.family = n->ops->family; 949 memcpy(&m_neigh.dst_ip, n->primary_key, n->tbl->key_len); 950 951 rcu_read_lock(); 952 nhe = mlx5e_rep_neigh_entry_lookup(priv, &m_neigh); 953 rcu_read_unlock(); 954 if (!nhe) 955 return NOTIFY_DONE; 956 957 mlx5e_rep_queue_neigh_update_work(priv, nhe, n); 958 break; 959 960 case NETEVENT_DELAY_PROBE_TIME_UPDATE: 961 p = ptr; 962 963 /* We check the device is present since we don't care about 964 * changes in the default table, we only care about changes 965 * done per device delay prob time parameter. 966 */ 967 #if IS_ENABLED(CONFIG_IPV6) 968 if (!p->dev || (p->tbl != ipv6_stub->nd_tbl && p->tbl != &arp_tbl)) 969 #else 970 if (!p->dev || p->tbl != &arp_tbl) 971 #endif 972 return NOTIFY_DONE; 973 974 rcu_read_lock(); 975 list_for_each_entry_rcu(nhe, &neigh_update->neigh_list, 976 neigh_list) { 977 if (p->dev == nhe->m_neigh.dev) { 978 found = true; 979 break; 980 } 981 } 982 rcu_read_unlock(); 983 if (!found) 984 return NOTIFY_DONE; 985 986 neigh_update->min_interval = min_t(unsigned long, 987 NEIGH_VAR(p, DELAY_PROBE_TIME), 988 neigh_update->min_interval); 989 mlx5_fc_update_sampling_interval(priv->mdev, 990 neigh_update->min_interval); 991 break; 992 } 993 return NOTIFY_DONE; 994 } 995 996 static const struct rhashtable_params mlx5e_neigh_ht_params = { 997 .head_offset = offsetof(struct mlx5e_neigh_hash_entry, rhash_node), 998 .key_offset = offsetof(struct mlx5e_neigh_hash_entry, m_neigh), 999 .key_len = sizeof(struct mlx5e_neigh), 1000 .automatic_shrinking = true, 1001 }; 1002 1003 static int mlx5e_rep_neigh_init(struct mlx5e_rep_priv *rpriv) 1004 { 1005 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update; 1006 int err; 1007 1008 err = rhashtable_init(&neigh_update->neigh_ht, &mlx5e_neigh_ht_params); 1009 if (err) 1010 return err; 1011 1012 INIT_LIST_HEAD(&neigh_update->neigh_list); 1013 mutex_init(&neigh_update->encap_lock); 1014 INIT_DELAYED_WORK(&neigh_update->neigh_stats_work, 1015 mlx5e_rep_neigh_stats_work); 1016 mlx5e_rep_neigh_update_init_interval(rpriv); 1017 1018 rpriv->neigh_update.netevent_nb.notifier_call = mlx5e_rep_netevent_event; 1019 err = register_netevent_notifier(&rpriv->neigh_update.netevent_nb); 1020 if (err) 1021 goto out_err; 1022 return 0; 1023 1024 out_err: 1025 rhashtable_destroy(&neigh_update->neigh_ht); 1026 return err; 1027 } 1028 1029 static void mlx5e_rep_neigh_cleanup(struct mlx5e_rep_priv *rpriv) 1030 { 1031 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update; 1032 struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); 1033 1034 unregister_netevent_notifier(&neigh_update->netevent_nb); 1035 1036 flush_workqueue(priv->wq); /* flush neigh update works */ 1037 1038 cancel_delayed_work_sync(&rpriv->neigh_update.neigh_stats_work); 1039 1040 mutex_destroy(&neigh_update->encap_lock); 1041 rhashtable_destroy(&neigh_update->neigh_ht); 1042 } 1043 1044 static int mlx5e_rep_neigh_entry_insert(struct mlx5e_priv *priv, 1045 struct mlx5e_neigh_hash_entry *nhe) 1046 { 1047 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1048 int err; 1049 1050 err = rhashtable_insert_fast(&rpriv->neigh_update.neigh_ht, 1051 &nhe->rhash_node, 1052 mlx5e_neigh_ht_params); 1053 if (err) 1054 return err; 1055 1056 list_add_rcu(&nhe->neigh_list, &rpriv->neigh_update.neigh_list); 1057 1058 return err; 1059 } 1060 1061 static void mlx5e_rep_neigh_entry_remove(struct mlx5e_neigh_hash_entry *nhe) 1062 { 1063 struct mlx5e_rep_priv *rpriv = nhe->priv->ppriv; 1064 1065 mutex_lock(&rpriv->neigh_update.encap_lock); 1066 1067 list_del_rcu(&nhe->neigh_list); 1068 1069 rhashtable_remove_fast(&rpriv->neigh_update.neigh_ht, 1070 &nhe->rhash_node, 1071 mlx5e_neigh_ht_params); 1072 mutex_unlock(&rpriv->neigh_update.encap_lock); 1073 } 1074 1075 /* This function must only be called under the representor's encap_lock or 1076 * inside rcu read lock section. 1077 */ 1078 static struct mlx5e_neigh_hash_entry * 1079 mlx5e_rep_neigh_entry_lookup(struct mlx5e_priv *priv, 1080 struct mlx5e_neigh *m_neigh) 1081 { 1082 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1083 struct mlx5e_neigh_update_table *neigh_update = &rpriv->neigh_update; 1084 struct mlx5e_neigh_hash_entry *nhe; 1085 1086 nhe = rhashtable_lookup_fast(&neigh_update->neigh_ht, m_neigh, 1087 mlx5e_neigh_ht_params); 1088 return nhe && mlx5e_rep_neigh_entry_hold(nhe) ? nhe : NULL; 1089 } 1090 1091 static int mlx5e_rep_neigh_entry_create(struct mlx5e_priv *priv, 1092 struct mlx5e_encap_entry *e, 1093 struct mlx5e_neigh_hash_entry **nhe) 1094 { 1095 int err; 1096 1097 *nhe = kzalloc(sizeof(**nhe), GFP_KERNEL); 1098 if (!*nhe) 1099 return -ENOMEM; 1100 1101 (*nhe)->priv = priv; 1102 memcpy(&(*nhe)->m_neigh, &e->m_neigh, sizeof(e->m_neigh)); 1103 INIT_WORK(&(*nhe)->neigh_update_work, mlx5e_rep_neigh_update); 1104 spin_lock_init(&(*nhe)->encap_list_lock); 1105 INIT_LIST_HEAD(&(*nhe)->encap_list); 1106 refcount_set(&(*nhe)->refcnt, 1); 1107 1108 err = mlx5e_rep_neigh_entry_insert(priv, *nhe); 1109 if (err) 1110 goto out_free; 1111 return 0; 1112 1113 out_free: 1114 kfree(*nhe); 1115 return err; 1116 } 1117 1118 int mlx5e_rep_encap_entry_attach(struct mlx5e_priv *priv, 1119 struct mlx5e_encap_entry *e) 1120 { 1121 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1122 struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv; 1123 struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy; 1124 struct mlx5e_neigh_hash_entry *nhe; 1125 int err; 1126 1127 err = mlx5_tun_entropy_refcount_inc(tun_entropy, e->reformat_type); 1128 if (err) 1129 return err; 1130 1131 mutex_lock(&rpriv->neigh_update.encap_lock); 1132 nhe = mlx5e_rep_neigh_entry_lookup(priv, &e->m_neigh); 1133 if (!nhe) { 1134 err = mlx5e_rep_neigh_entry_create(priv, e, &nhe); 1135 if (err) { 1136 mutex_unlock(&rpriv->neigh_update.encap_lock); 1137 mlx5_tun_entropy_refcount_dec(tun_entropy, 1138 e->reformat_type); 1139 return err; 1140 } 1141 } 1142 1143 e->nhe = nhe; 1144 spin_lock(&nhe->encap_list_lock); 1145 list_add_rcu(&e->encap_list, &nhe->encap_list); 1146 spin_unlock(&nhe->encap_list_lock); 1147 1148 mutex_unlock(&rpriv->neigh_update.encap_lock); 1149 1150 return 0; 1151 } 1152 1153 void mlx5e_rep_encap_entry_detach(struct mlx5e_priv *priv, 1154 struct mlx5e_encap_entry *e) 1155 { 1156 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1157 struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv; 1158 struct mlx5_tun_entropy *tun_entropy = &uplink_priv->tun_entropy; 1159 1160 if (!e->nhe) 1161 return; 1162 1163 spin_lock(&e->nhe->encap_list_lock); 1164 list_del_rcu(&e->encap_list); 1165 spin_unlock(&e->nhe->encap_list_lock); 1166 1167 mlx5e_rep_neigh_entry_release(e->nhe); 1168 e->nhe = NULL; 1169 mlx5_tun_entropy_refcount_dec(tun_entropy, e->reformat_type); 1170 } 1171 1172 static int mlx5e_rep_open(struct net_device *dev) 1173 { 1174 struct mlx5e_priv *priv = netdev_priv(dev); 1175 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1176 struct mlx5_eswitch_rep *rep = rpriv->rep; 1177 int err; 1178 1179 mutex_lock(&priv->state_lock); 1180 err = mlx5e_open_locked(dev); 1181 if (err) 1182 goto unlock; 1183 1184 if (!mlx5_modify_vport_admin_state(priv->mdev, 1185 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 1186 rep->vport, 1, 1187 MLX5_VPORT_ADMIN_STATE_UP)) 1188 netif_carrier_on(dev); 1189 1190 unlock: 1191 mutex_unlock(&priv->state_lock); 1192 return err; 1193 } 1194 1195 static int mlx5e_rep_close(struct net_device *dev) 1196 { 1197 struct mlx5e_priv *priv = netdev_priv(dev); 1198 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1199 struct mlx5_eswitch_rep *rep = rpriv->rep; 1200 int ret; 1201 1202 mutex_lock(&priv->state_lock); 1203 mlx5_modify_vport_admin_state(priv->mdev, 1204 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 1205 rep->vport, 1, 1206 MLX5_VPORT_ADMIN_STATE_DOWN); 1207 ret = mlx5e_close_locked(dev); 1208 mutex_unlock(&priv->state_lock); 1209 return ret; 1210 } 1211 1212 static int 1213 mlx5e_rep_setup_tc_cls_flower(struct mlx5e_priv *priv, 1214 struct flow_cls_offload *cls_flower, int flags) 1215 { 1216 switch (cls_flower->command) { 1217 case FLOW_CLS_REPLACE: 1218 return mlx5e_configure_flower(priv->netdev, priv, cls_flower, 1219 flags); 1220 case FLOW_CLS_DESTROY: 1221 return mlx5e_delete_flower(priv->netdev, priv, cls_flower, 1222 flags); 1223 case FLOW_CLS_STATS: 1224 return mlx5e_stats_flower(priv->netdev, priv, cls_flower, 1225 flags); 1226 default: 1227 return -EOPNOTSUPP; 1228 } 1229 } 1230 1231 static 1232 int mlx5e_rep_setup_tc_cls_matchall(struct mlx5e_priv *priv, 1233 struct tc_cls_matchall_offload *ma) 1234 { 1235 switch (ma->command) { 1236 case TC_CLSMATCHALL_REPLACE: 1237 return mlx5e_tc_configure_matchall(priv, ma); 1238 case TC_CLSMATCHALL_DESTROY: 1239 return mlx5e_tc_delete_matchall(priv, ma); 1240 case TC_CLSMATCHALL_STATS: 1241 mlx5e_tc_stats_matchall(priv, ma); 1242 return 0; 1243 default: 1244 return -EOPNOTSUPP; 1245 } 1246 } 1247 1248 static int mlx5e_rep_setup_tc_cb(enum tc_setup_type type, void *type_data, 1249 void *cb_priv) 1250 { 1251 unsigned long flags = MLX5_TC_FLAG(INGRESS) | MLX5_TC_FLAG(ESW_OFFLOAD); 1252 struct mlx5e_priv *priv = cb_priv; 1253 1254 switch (type) { 1255 case TC_SETUP_CLSFLOWER: 1256 return mlx5e_rep_setup_tc_cls_flower(priv, type_data, flags); 1257 case TC_SETUP_CLSMATCHALL: 1258 return mlx5e_rep_setup_tc_cls_matchall(priv, type_data); 1259 default: 1260 return -EOPNOTSUPP; 1261 } 1262 } 1263 1264 static int mlx5e_rep_setup_ft_cb(enum tc_setup_type type, void *type_data, 1265 void *cb_priv) 1266 { 1267 struct flow_cls_offload tmp, *f = type_data; 1268 struct mlx5e_priv *priv = cb_priv; 1269 struct mlx5_eswitch *esw; 1270 unsigned long flags; 1271 int err; 1272 1273 flags = MLX5_TC_FLAG(INGRESS) | 1274 MLX5_TC_FLAG(ESW_OFFLOAD) | 1275 MLX5_TC_FLAG(FT_OFFLOAD); 1276 esw = priv->mdev->priv.eswitch; 1277 1278 switch (type) { 1279 case TC_SETUP_CLSFLOWER: 1280 memcpy(&tmp, f, sizeof(*f)); 1281 1282 if (!mlx5_esw_chains_prios_supported(esw) || 1283 tmp.common.chain_index) 1284 return -EOPNOTSUPP; 1285 1286 /* Re-use tc offload path by moving the ft flow to the 1287 * reserved ft chain. 1288 * 1289 * FT offload can use prio range [0, INT_MAX], so we normalize 1290 * it to range [1, mlx5_esw_chains_get_prio_range(esw)] 1291 * as with tc, where prio 0 isn't supported. 1292 * 1293 * We only support chain 0 of FT offload. 1294 */ 1295 if (tmp.common.prio >= mlx5_esw_chains_get_prio_range(esw)) 1296 return -EOPNOTSUPP; 1297 if (tmp.common.chain_index != 0) 1298 return -EOPNOTSUPP; 1299 1300 tmp.common.chain_index = mlx5_esw_chains_get_ft_chain(esw); 1301 tmp.common.prio++; 1302 err = mlx5e_rep_setup_tc_cls_flower(priv, &tmp, flags); 1303 memcpy(&f->stats, &tmp.stats, sizeof(f->stats)); 1304 return err; 1305 default: 1306 return -EOPNOTSUPP; 1307 } 1308 } 1309 1310 static LIST_HEAD(mlx5e_rep_block_tc_cb_list); 1311 static LIST_HEAD(mlx5e_rep_block_ft_cb_list); 1312 static int mlx5e_rep_setup_tc(struct net_device *dev, enum tc_setup_type type, 1313 void *type_data) 1314 { 1315 struct mlx5e_priv *priv = netdev_priv(dev); 1316 struct flow_block_offload *f = type_data; 1317 1318 f->unlocked_driver_cb = true; 1319 1320 switch (type) { 1321 case TC_SETUP_BLOCK: 1322 return flow_block_cb_setup_simple(type_data, 1323 &mlx5e_rep_block_tc_cb_list, 1324 mlx5e_rep_setup_tc_cb, 1325 priv, priv, true); 1326 case TC_SETUP_FT: 1327 return flow_block_cb_setup_simple(type_data, 1328 &mlx5e_rep_block_ft_cb_list, 1329 mlx5e_rep_setup_ft_cb, 1330 priv, priv, true); 1331 default: 1332 return -EOPNOTSUPP; 1333 } 1334 } 1335 1336 bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv) 1337 { 1338 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1339 struct mlx5_eswitch_rep *rep; 1340 1341 if (!MLX5_ESWITCH_MANAGER(priv->mdev)) 1342 return false; 1343 1344 if (!rpriv) /* non vport rep mlx5e instances don't use this field */ 1345 return false; 1346 1347 rep = rpriv->rep; 1348 return (rep->vport == MLX5_VPORT_UPLINK); 1349 } 1350 1351 static bool mlx5e_rep_has_offload_stats(const struct net_device *dev, int attr_id) 1352 { 1353 switch (attr_id) { 1354 case IFLA_OFFLOAD_XSTATS_CPU_HIT: 1355 return true; 1356 } 1357 1358 return false; 1359 } 1360 1361 static int 1362 mlx5e_get_sw_stats64(const struct net_device *dev, 1363 struct rtnl_link_stats64 *stats) 1364 { 1365 struct mlx5e_priv *priv = netdev_priv(dev); 1366 1367 mlx5e_fold_sw_stats64(priv, stats); 1368 return 0; 1369 } 1370 1371 static int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev, 1372 void *sp) 1373 { 1374 switch (attr_id) { 1375 case IFLA_OFFLOAD_XSTATS_CPU_HIT: 1376 return mlx5e_get_sw_stats64(dev, sp); 1377 } 1378 1379 return -EINVAL; 1380 } 1381 1382 static void 1383 mlx5e_rep_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 1384 { 1385 struct mlx5e_priv *priv = netdev_priv(dev); 1386 1387 /* update HW stats in background for next time */ 1388 mlx5e_queue_update_stats(priv); 1389 memcpy(stats, &priv->stats.vf_vport, sizeof(*stats)); 1390 } 1391 1392 static int mlx5e_rep_change_mtu(struct net_device *netdev, int new_mtu) 1393 { 1394 return mlx5e_change_mtu(netdev, new_mtu, NULL); 1395 } 1396 1397 static int mlx5e_uplink_rep_change_mtu(struct net_device *netdev, int new_mtu) 1398 { 1399 return mlx5e_change_mtu(netdev, new_mtu, mlx5e_set_dev_port_mtu); 1400 } 1401 1402 static int mlx5e_uplink_rep_set_mac(struct net_device *netdev, void *addr) 1403 { 1404 struct sockaddr *saddr = addr; 1405 1406 if (!is_valid_ether_addr(saddr->sa_data)) 1407 return -EADDRNOTAVAIL; 1408 1409 ether_addr_copy(netdev->dev_addr, saddr->sa_data); 1410 return 0; 1411 } 1412 1413 static int mlx5e_uplink_rep_set_vf_vlan(struct net_device *dev, int vf, u16 vlan, u8 qos, 1414 __be16 vlan_proto) 1415 { 1416 netdev_warn_once(dev, "legacy vf vlan setting isn't supported in switchdev mode\n"); 1417 1418 if (vlan != 0) 1419 return -EOPNOTSUPP; 1420 1421 /* allow setting 0-vid for compatibility with libvirt */ 1422 return 0; 1423 } 1424 1425 static struct devlink_port *mlx5e_get_devlink_port(struct net_device *dev) 1426 { 1427 struct mlx5e_priv *priv = netdev_priv(dev); 1428 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1429 1430 return &rpriv->dl_port; 1431 } 1432 1433 static const struct net_device_ops mlx5e_netdev_ops_rep = { 1434 .ndo_open = mlx5e_rep_open, 1435 .ndo_stop = mlx5e_rep_close, 1436 .ndo_start_xmit = mlx5e_xmit, 1437 .ndo_setup_tc = mlx5e_rep_setup_tc, 1438 .ndo_get_devlink_port = mlx5e_get_devlink_port, 1439 .ndo_get_stats64 = mlx5e_rep_get_stats, 1440 .ndo_has_offload_stats = mlx5e_rep_has_offload_stats, 1441 .ndo_get_offload_stats = mlx5e_rep_get_offload_stats, 1442 .ndo_change_mtu = mlx5e_rep_change_mtu, 1443 }; 1444 1445 static const struct net_device_ops mlx5e_netdev_ops_uplink_rep = { 1446 .ndo_open = mlx5e_open, 1447 .ndo_stop = mlx5e_close, 1448 .ndo_start_xmit = mlx5e_xmit, 1449 .ndo_set_mac_address = mlx5e_uplink_rep_set_mac, 1450 .ndo_setup_tc = mlx5e_rep_setup_tc, 1451 .ndo_get_devlink_port = mlx5e_get_devlink_port, 1452 .ndo_get_stats64 = mlx5e_get_stats, 1453 .ndo_has_offload_stats = mlx5e_rep_has_offload_stats, 1454 .ndo_get_offload_stats = mlx5e_rep_get_offload_stats, 1455 .ndo_change_mtu = mlx5e_uplink_rep_change_mtu, 1456 .ndo_udp_tunnel_add = mlx5e_add_vxlan_port, 1457 .ndo_udp_tunnel_del = mlx5e_del_vxlan_port, 1458 .ndo_features_check = mlx5e_features_check, 1459 .ndo_set_vf_mac = mlx5e_set_vf_mac, 1460 .ndo_set_vf_rate = mlx5e_set_vf_rate, 1461 .ndo_get_vf_config = mlx5e_get_vf_config, 1462 .ndo_get_vf_stats = mlx5e_get_vf_stats, 1463 .ndo_set_vf_vlan = mlx5e_uplink_rep_set_vf_vlan, 1464 .ndo_set_features = mlx5e_set_features, 1465 }; 1466 1467 bool mlx5e_eswitch_rep(struct net_device *netdev) 1468 { 1469 if (netdev->netdev_ops == &mlx5e_netdev_ops_rep || 1470 netdev->netdev_ops == &mlx5e_netdev_ops_uplink_rep) 1471 return true; 1472 1473 return false; 1474 } 1475 1476 static void mlx5e_build_rep_params(struct net_device *netdev) 1477 { 1478 struct mlx5e_priv *priv = netdev_priv(netdev); 1479 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1480 struct mlx5_eswitch_rep *rep = rpriv->rep; 1481 struct mlx5_core_dev *mdev = priv->mdev; 1482 struct mlx5e_params *params; 1483 1484 u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 1485 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 1486 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 1487 1488 params = &priv->channels.params; 1489 params->hard_mtu = MLX5E_ETH_HARD_MTU; 1490 params->sw_mtu = netdev->mtu; 1491 1492 /* SQ */ 1493 if (rep->vport == MLX5_VPORT_UPLINK) 1494 params->log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 1495 else 1496 params->log_sq_size = MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE; 1497 1498 /* RQ */ 1499 mlx5e_build_rq_params(mdev, params); 1500 1501 /* CQ moderation params */ 1502 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 1503 mlx5e_set_rx_cq_mode_params(params, cq_period_mode); 1504 1505 params->num_tc = 1; 1506 params->tunneled_offload_en = false; 1507 1508 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 1509 1510 /* RSS */ 1511 mlx5e_build_rss_params(&priv->rss_params, params->num_channels); 1512 } 1513 1514 static void mlx5e_build_rep_netdev(struct net_device *netdev) 1515 { 1516 struct mlx5e_priv *priv = netdev_priv(netdev); 1517 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1518 struct mlx5_eswitch_rep *rep = rpriv->rep; 1519 struct mlx5_core_dev *mdev = priv->mdev; 1520 1521 if (rep->vport == MLX5_VPORT_UPLINK) { 1522 SET_NETDEV_DEV(netdev, mdev->device); 1523 netdev->netdev_ops = &mlx5e_netdev_ops_uplink_rep; 1524 /* we want a persistent mac for the uplink rep */ 1525 mlx5_query_mac_address(mdev, netdev->dev_addr); 1526 netdev->ethtool_ops = &mlx5e_uplink_rep_ethtool_ops; 1527 #ifdef CONFIG_MLX5_CORE_EN_DCB 1528 if (MLX5_CAP_GEN(mdev, qos)) 1529 netdev->dcbnl_ops = &mlx5e_dcbnl_ops; 1530 #endif 1531 } else { 1532 netdev->netdev_ops = &mlx5e_netdev_ops_rep; 1533 eth_hw_addr_random(netdev); 1534 netdev->ethtool_ops = &mlx5e_rep_ethtool_ops; 1535 } 1536 1537 netdev->watchdog_timeo = 15 * HZ; 1538 1539 netdev->features |= NETIF_F_NETNS_LOCAL; 1540 1541 netdev->hw_features |= NETIF_F_HW_TC; 1542 netdev->hw_features |= NETIF_F_SG; 1543 netdev->hw_features |= NETIF_F_IP_CSUM; 1544 netdev->hw_features |= NETIF_F_IPV6_CSUM; 1545 netdev->hw_features |= NETIF_F_GRO; 1546 netdev->hw_features |= NETIF_F_TSO; 1547 netdev->hw_features |= NETIF_F_TSO6; 1548 netdev->hw_features |= NETIF_F_RXCSUM; 1549 1550 if (rep->vport == MLX5_VPORT_UPLINK) 1551 netdev->hw_features |= NETIF_F_HW_VLAN_CTAG_RX; 1552 else 1553 netdev->features |= NETIF_F_VLAN_CHALLENGED; 1554 1555 netdev->features |= netdev->hw_features; 1556 } 1557 1558 static int mlx5e_init_rep(struct mlx5_core_dev *mdev, 1559 struct net_device *netdev, 1560 const struct mlx5e_profile *profile, 1561 void *ppriv) 1562 { 1563 struct mlx5e_priv *priv = netdev_priv(netdev); 1564 int err; 1565 1566 err = mlx5e_netdev_init(netdev, priv, mdev, profile, ppriv); 1567 if (err) 1568 return err; 1569 1570 priv->channels.params.num_channels = MLX5E_REP_PARAMS_DEF_NUM_CHANNELS; 1571 1572 mlx5e_build_rep_params(netdev); 1573 mlx5e_build_rep_netdev(netdev); 1574 1575 mlx5e_timestamp_init(priv); 1576 1577 return 0; 1578 } 1579 1580 static void mlx5e_cleanup_rep(struct mlx5e_priv *priv) 1581 { 1582 mlx5e_netdev_cleanup(priv->netdev, priv); 1583 } 1584 1585 static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) 1586 { 1587 struct ttc_params ttc_params = {}; 1588 int tt, err; 1589 1590 priv->fs.ns = mlx5_get_flow_namespace(priv->mdev, 1591 MLX5_FLOW_NAMESPACE_KERNEL); 1592 1593 /* The inner_ttc in the ttc params is intentionally not set */ 1594 ttc_params.any_tt_tirn = priv->direct_tir[0].tirn; 1595 mlx5e_set_ttc_ft_params(&ttc_params); 1596 for (tt = 0; tt < MLX5E_NUM_INDIR_TIRS; tt++) 1597 ttc_params.indir_tirn[tt] = priv->indir_tir[tt].tirn; 1598 1599 err = mlx5e_create_ttc_table(priv, &ttc_params, &priv->fs.ttc); 1600 if (err) { 1601 netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n", err); 1602 return err; 1603 } 1604 return 0; 1605 } 1606 1607 static int mlx5e_create_rep_vport_rx_rule(struct mlx5e_priv *priv) 1608 { 1609 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 1610 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1611 struct mlx5_eswitch_rep *rep = rpriv->rep; 1612 struct mlx5_flow_handle *flow_rule; 1613 struct mlx5_flow_destination dest; 1614 1615 dest.type = MLX5_FLOW_DESTINATION_TYPE_TIR; 1616 dest.tir_num = priv->direct_tir[0].tirn; 1617 flow_rule = mlx5_eswitch_create_vport_rx_rule(esw, 1618 rep->vport, 1619 &dest); 1620 if (IS_ERR(flow_rule)) 1621 return PTR_ERR(flow_rule); 1622 rpriv->vport_rx_rule = flow_rule; 1623 return 0; 1624 } 1625 1626 static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) 1627 { 1628 struct mlx5_core_dev *mdev = priv->mdev; 1629 int err; 1630 1631 mlx5e_init_l2_addr(priv); 1632 1633 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 1634 if (err) { 1635 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 1636 return err; 1637 } 1638 1639 err = mlx5e_create_indirect_rqt(priv); 1640 if (err) 1641 goto err_close_drop_rq; 1642 1643 err = mlx5e_create_direct_rqts(priv, priv->direct_tir); 1644 if (err) 1645 goto err_destroy_indirect_rqts; 1646 1647 err = mlx5e_create_indirect_tirs(priv, false); 1648 if (err) 1649 goto err_destroy_direct_rqts; 1650 1651 err = mlx5e_create_direct_tirs(priv, priv->direct_tir); 1652 if (err) 1653 goto err_destroy_indirect_tirs; 1654 1655 err = mlx5e_create_rep_ttc_table(priv); 1656 if (err) 1657 goto err_destroy_direct_tirs; 1658 1659 err = mlx5e_create_rep_vport_rx_rule(priv); 1660 if (err) 1661 goto err_destroy_ttc_table; 1662 1663 return 0; 1664 1665 err_destroy_ttc_table: 1666 mlx5e_destroy_ttc_table(priv, &priv->fs.ttc); 1667 err_destroy_direct_tirs: 1668 mlx5e_destroy_direct_tirs(priv, priv->direct_tir); 1669 err_destroy_indirect_tirs: 1670 mlx5e_destroy_indirect_tirs(priv, false); 1671 err_destroy_direct_rqts: 1672 mlx5e_destroy_direct_rqts(priv, priv->direct_tir); 1673 err_destroy_indirect_rqts: 1674 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 1675 err_close_drop_rq: 1676 mlx5e_close_drop_rq(&priv->drop_rq); 1677 return err; 1678 } 1679 1680 static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv) 1681 { 1682 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1683 1684 mlx5_del_flow_rules(rpriv->vport_rx_rule); 1685 mlx5e_destroy_ttc_table(priv, &priv->fs.ttc); 1686 mlx5e_destroy_direct_tirs(priv, priv->direct_tir); 1687 mlx5e_destroy_indirect_tirs(priv, false); 1688 mlx5e_destroy_direct_rqts(priv, priv->direct_tir); 1689 mlx5e_destroy_rqt(priv, &priv->indir_rqt); 1690 mlx5e_close_drop_rq(&priv->drop_rq); 1691 } 1692 1693 static int mlx5e_init_ul_rep_rx(struct mlx5e_priv *priv) 1694 { 1695 int err = mlx5e_init_rep_rx(priv); 1696 1697 if (err) 1698 return err; 1699 1700 mlx5e_create_q_counters(priv); 1701 return 0; 1702 } 1703 1704 static void mlx5e_cleanup_ul_rep_rx(struct mlx5e_priv *priv) 1705 { 1706 mlx5e_destroy_q_counters(priv); 1707 mlx5e_cleanup_rep_rx(priv); 1708 } 1709 1710 static int mlx5e_init_uplink_rep_tx(struct mlx5e_rep_priv *rpriv) 1711 { 1712 struct mlx5_rep_uplink_priv *uplink_priv; 1713 struct net_device *netdev; 1714 struct mlx5e_priv *priv; 1715 int err; 1716 1717 netdev = rpriv->netdev; 1718 priv = netdev_priv(netdev); 1719 uplink_priv = &rpriv->uplink_priv; 1720 1721 mutex_init(&uplink_priv->unready_flows_lock); 1722 INIT_LIST_HEAD(&uplink_priv->unready_flows); 1723 1724 /* init shared tc flow table */ 1725 err = mlx5e_tc_esw_init(&uplink_priv->tc_ht); 1726 if (err) 1727 return err; 1728 1729 mlx5_init_port_tun_entropy(&uplink_priv->tun_entropy, priv->mdev); 1730 1731 /* init indirect block notifications */ 1732 INIT_LIST_HEAD(&uplink_priv->tc_indr_block_priv_list); 1733 uplink_priv->netdevice_nb.notifier_call = mlx5e_nic_rep_netdevice_event; 1734 err = register_netdevice_notifier_dev_net(rpriv->netdev, 1735 &uplink_priv->netdevice_nb, 1736 &uplink_priv->netdevice_nn); 1737 if (err) { 1738 mlx5_core_err(priv->mdev, "Failed to register netdev notifier\n"); 1739 goto tc_esw_cleanup; 1740 } 1741 1742 return 0; 1743 1744 tc_esw_cleanup: 1745 mlx5e_tc_esw_cleanup(&uplink_priv->tc_ht); 1746 return err; 1747 } 1748 1749 static int mlx5e_init_rep_tx(struct mlx5e_priv *priv) 1750 { 1751 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1752 int err; 1753 1754 err = mlx5e_create_tises(priv); 1755 if (err) { 1756 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); 1757 return err; 1758 } 1759 1760 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) { 1761 err = mlx5e_init_uplink_rep_tx(rpriv); 1762 if (err) 1763 goto destroy_tises; 1764 } 1765 1766 return 0; 1767 1768 destroy_tises: 1769 mlx5e_destroy_tises(priv); 1770 return err; 1771 } 1772 1773 static void mlx5e_cleanup_uplink_rep_tx(struct mlx5e_rep_priv *rpriv) 1774 { 1775 struct mlx5_rep_uplink_priv *uplink_priv = &rpriv->uplink_priv; 1776 1777 /* clean indirect TC block notifications */ 1778 unregister_netdevice_notifier_dev_net(rpriv->netdev, 1779 &uplink_priv->netdevice_nb, 1780 &uplink_priv->netdevice_nn); 1781 mlx5e_rep_indr_clean_block_privs(rpriv); 1782 1783 /* delete shared tc flow table */ 1784 mlx5e_tc_esw_cleanup(&rpriv->uplink_priv.tc_ht); 1785 mutex_destroy(&rpriv->uplink_priv.unready_flows_lock); 1786 } 1787 1788 static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv) 1789 { 1790 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1791 1792 mlx5e_destroy_tises(priv); 1793 1794 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) 1795 mlx5e_cleanup_uplink_rep_tx(rpriv); 1796 } 1797 1798 static void mlx5e_rep_enable(struct mlx5e_priv *priv) 1799 { 1800 mlx5e_set_netdev_mtu_boundaries(priv); 1801 } 1802 1803 static int mlx5e_update_rep_rx(struct mlx5e_priv *priv) 1804 { 1805 return 0; 1806 } 1807 1808 static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event, void *data) 1809 { 1810 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 1811 1812 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 1813 struct mlx5_eqe *eqe = data; 1814 1815 switch (eqe->sub_type) { 1816 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 1817 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 1818 queue_work(priv->wq, &priv->update_carrier_work); 1819 break; 1820 default: 1821 return NOTIFY_DONE; 1822 } 1823 1824 return NOTIFY_OK; 1825 } 1826 1827 if (event == MLX5_DEV_EVENT_PORT_AFFINITY) { 1828 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1829 1830 queue_work(priv->wq, &rpriv->uplink_priv.reoffload_flows_work); 1831 1832 return NOTIFY_OK; 1833 } 1834 1835 return NOTIFY_DONE; 1836 } 1837 1838 static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) 1839 { 1840 struct net_device *netdev = priv->netdev; 1841 struct mlx5_core_dev *mdev = priv->mdev; 1842 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1843 u16 max_mtu; 1844 1845 netdev->min_mtu = ETH_MIN_MTU; 1846 mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1); 1847 netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu); 1848 mlx5e_set_dev_port_mtu(priv); 1849 1850 INIT_WORK(&rpriv->uplink_priv.reoffload_flows_work, 1851 mlx5e_tc_reoffload_flows_work); 1852 1853 mlx5_lag_add(mdev, netdev); 1854 priv->events_nb.notifier_call = uplink_rep_async_event; 1855 mlx5_notifier_register(mdev, &priv->events_nb); 1856 #ifdef CONFIG_MLX5_CORE_EN_DCB 1857 mlx5e_dcbnl_initialize(priv); 1858 mlx5e_dcbnl_init_app(priv); 1859 #endif 1860 } 1861 1862 static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv) 1863 { 1864 struct net_device *netdev = priv->netdev; 1865 struct mlx5_core_dev *mdev = priv->mdev; 1866 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1867 1868 #ifdef CONFIG_MLX5_CORE_EN_DCB 1869 mlx5e_dcbnl_delete_app(priv); 1870 #endif 1871 mlx5_notifier_unregister(mdev, &priv->events_nb); 1872 cancel_work_sync(&rpriv->uplink_priv.reoffload_flows_work); 1873 mlx5_lag_remove(mdev, netdev); 1874 } 1875 1876 static MLX5E_DEFINE_STATS_GRP(sw_rep, 0); 1877 static MLX5E_DEFINE_STATS_GRP(vport_rep, MLX5E_NDO_UPDATE_STATS); 1878 1879 /* The stats groups order is opposite to the update_stats() order calls */ 1880 static mlx5e_stats_grp_t mlx5e_rep_stats_grps[] = { 1881 &MLX5E_STATS_GRP(sw_rep), 1882 &MLX5E_STATS_GRP(vport_rep), 1883 }; 1884 1885 static unsigned int mlx5e_rep_stats_grps_num(struct mlx5e_priv *priv) 1886 { 1887 return ARRAY_SIZE(mlx5e_rep_stats_grps); 1888 } 1889 1890 /* The stats groups order is opposite to the update_stats() order calls */ 1891 static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = { 1892 &MLX5E_STATS_GRP(sw), 1893 &MLX5E_STATS_GRP(qcnt), 1894 &MLX5E_STATS_GRP(vnic_env), 1895 &MLX5E_STATS_GRP(vport), 1896 &MLX5E_STATS_GRP(802_3), 1897 &MLX5E_STATS_GRP(2863), 1898 &MLX5E_STATS_GRP(2819), 1899 &MLX5E_STATS_GRP(phy), 1900 &MLX5E_STATS_GRP(eth_ext), 1901 &MLX5E_STATS_GRP(pcie), 1902 &MLX5E_STATS_GRP(per_prio), 1903 &MLX5E_STATS_GRP(pme), 1904 &MLX5E_STATS_GRP(channels), 1905 &MLX5E_STATS_GRP(per_port_buff_congest), 1906 }; 1907 1908 static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv) 1909 { 1910 return ARRAY_SIZE(mlx5e_ul_rep_stats_grps); 1911 } 1912 1913 static const struct mlx5e_profile mlx5e_rep_profile = { 1914 .init = mlx5e_init_rep, 1915 .cleanup = mlx5e_cleanup_rep, 1916 .init_rx = mlx5e_init_rep_rx, 1917 .cleanup_rx = mlx5e_cleanup_rep_rx, 1918 .init_tx = mlx5e_init_rep_tx, 1919 .cleanup_tx = mlx5e_cleanup_rep_tx, 1920 .enable = mlx5e_rep_enable, 1921 .update_rx = mlx5e_update_rep_rx, 1922 .update_stats = mlx5e_update_ndo_stats, 1923 .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe_rep, 1924 .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, 1925 .max_tc = 1, 1926 .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), 1927 .stats_grps = mlx5e_rep_stats_grps, 1928 .stats_grps_num = mlx5e_rep_stats_grps_num, 1929 }; 1930 1931 static const struct mlx5e_profile mlx5e_uplink_rep_profile = { 1932 .init = mlx5e_init_rep, 1933 .cleanup = mlx5e_cleanup_rep, 1934 .init_rx = mlx5e_init_ul_rep_rx, 1935 .cleanup_rx = mlx5e_cleanup_ul_rep_rx, 1936 .init_tx = mlx5e_init_rep_tx, 1937 .cleanup_tx = mlx5e_cleanup_rep_tx, 1938 .enable = mlx5e_uplink_rep_enable, 1939 .disable = mlx5e_uplink_rep_disable, 1940 .update_rx = mlx5e_update_rep_rx, 1941 .update_stats = mlx5e_update_ndo_stats, 1942 .update_carrier = mlx5e_update_carrier, 1943 .rx_handlers.handle_rx_cqe = mlx5e_handle_rx_cqe_rep, 1944 .rx_handlers.handle_rx_cqe_mpwqe = mlx5e_handle_rx_cqe_mpwrq, 1945 .max_tc = MLX5E_MAX_NUM_TC, 1946 .rq_groups = MLX5E_NUM_RQ_GROUPS(REGULAR), 1947 .stats_grps = mlx5e_ul_rep_stats_grps, 1948 .stats_grps_num = mlx5e_ul_rep_stats_grps_num, 1949 }; 1950 1951 static bool 1952 is_devlink_port_supported(const struct mlx5_core_dev *dev, 1953 const struct mlx5e_rep_priv *rpriv) 1954 { 1955 return rpriv->rep->vport == MLX5_VPORT_UPLINK || 1956 rpriv->rep->vport == MLX5_VPORT_PF || 1957 mlx5_eswitch_is_vf_vport(dev->priv.eswitch, rpriv->rep->vport); 1958 } 1959 1960 static unsigned int 1961 vport_to_devlink_port_index(const struct mlx5_core_dev *dev, u16 vport_num) 1962 { 1963 return (MLX5_CAP_GEN(dev, vhca_id) << 16) | vport_num; 1964 } 1965 1966 static int register_devlink_port(struct mlx5_core_dev *dev, 1967 struct mlx5e_rep_priv *rpriv) 1968 { 1969 struct devlink *devlink = priv_to_devlink(dev); 1970 struct mlx5_eswitch_rep *rep = rpriv->rep; 1971 struct netdev_phys_item_id ppid = {}; 1972 unsigned int dl_port_index = 0; 1973 1974 if (!is_devlink_port_supported(dev, rpriv)) 1975 return 0; 1976 1977 mlx5e_rep_get_port_parent_id(rpriv->netdev, &ppid); 1978 1979 if (rep->vport == MLX5_VPORT_UPLINK) { 1980 devlink_port_attrs_set(&rpriv->dl_port, 1981 DEVLINK_PORT_FLAVOUR_PHYSICAL, 1982 PCI_FUNC(dev->pdev->devfn), false, 0, 1983 &ppid.id[0], ppid.id_len); 1984 dl_port_index = vport_to_devlink_port_index(dev, rep->vport); 1985 } else if (rep->vport == MLX5_VPORT_PF) { 1986 devlink_port_attrs_pci_pf_set(&rpriv->dl_port, 1987 &ppid.id[0], ppid.id_len, 1988 dev->pdev->devfn); 1989 dl_port_index = rep->vport; 1990 } else if (mlx5_eswitch_is_vf_vport(dev->priv.eswitch, 1991 rpriv->rep->vport)) { 1992 devlink_port_attrs_pci_vf_set(&rpriv->dl_port, 1993 &ppid.id[0], ppid.id_len, 1994 dev->pdev->devfn, 1995 rep->vport - 1); 1996 dl_port_index = vport_to_devlink_port_index(dev, rep->vport); 1997 } 1998 1999 return devlink_port_register(devlink, &rpriv->dl_port, dl_port_index); 2000 } 2001 2002 static void unregister_devlink_port(struct mlx5_core_dev *dev, 2003 struct mlx5e_rep_priv *rpriv) 2004 { 2005 if (is_devlink_port_supported(dev, rpriv)) 2006 devlink_port_unregister(&rpriv->dl_port); 2007 } 2008 2009 /* e-Switch vport representors */ 2010 static int 2011 mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) 2012 { 2013 const struct mlx5e_profile *profile; 2014 struct mlx5e_rep_priv *rpriv; 2015 struct net_device *netdev; 2016 int nch, err; 2017 2018 rpriv = kzalloc(sizeof(*rpriv), GFP_KERNEL); 2019 if (!rpriv) 2020 return -ENOMEM; 2021 2022 /* rpriv->rep to be looked up when profile->init() is called */ 2023 rpriv->rep = rep; 2024 2025 nch = mlx5e_get_max_num_channels(dev); 2026 profile = (rep->vport == MLX5_VPORT_UPLINK) ? 2027 &mlx5e_uplink_rep_profile : &mlx5e_rep_profile; 2028 netdev = mlx5e_create_netdev(dev, profile, nch, rpriv); 2029 if (!netdev) { 2030 pr_warn("Failed to create representor netdev for vport %d\n", 2031 rep->vport); 2032 kfree(rpriv); 2033 return -EINVAL; 2034 } 2035 2036 dev_net_set(netdev, mlx5_core_net(dev)); 2037 rpriv->netdev = netdev; 2038 rep->rep_data[REP_ETH].priv = rpriv; 2039 INIT_LIST_HEAD(&rpriv->vport_sqs_list); 2040 2041 if (rep->vport == MLX5_VPORT_UPLINK) { 2042 err = mlx5e_create_mdev_resources(dev); 2043 if (err) 2044 goto err_destroy_netdev; 2045 } 2046 2047 err = mlx5e_attach_netdev(netdev_priv(netdev)); 2048 if (err) { 2049 pr_warn("Failed to attach representor netdev for vport %d\n", 2050 rep->vport); 2051 goto err_destroy_mdev_resources; 2052 } 2053 2054 err = mlx5e_rep_neigh_init(rpriv); 2055 if (err) { 2056 pr_warn("Failed to initialized neighbours handling for vport %d\n", 2057 rep->vport); 2058 goto err_detach_netdev; 2059 } 2060 2061 err = register_devlink_port(dev, rpriv); 2062 if (err) { 2063 esw_warn(dev, "Failed to register devlink port %d\n", 2064 rep->vport); 2065 goto err_neigh_cleanup; 2066 } 2067 2068 err = register_netdev(netdev); 2069 if (err) { 2070 pr_warn("Failed to register representor netdev for vport %d\n", 2071 rep->vport); 2072 goto err_devlink_cleanup; 2073 } 2074 2075 if (is_devlink_port_supported(dev, rpriv)) 2076 devlink_port_type_eth_set(&rpriv->dl_port, netdev); 2077 return 0; 2078 2079 err_devlink_cleanup: 2080 unregister_devlink_port(dev, rpriv); 2081 2082 err_neigh_cleanup: 2083 mlx5e_rep_neigh_cleanup(rpriv); 2084 2085 err_detach_netdev: 2086 mlx5e_detach_netdev(netdev_priv(netdev)); 2087 2088 err_destroy_mdev_resources: 2089 if (rep->vport == MLX5_VPORT_UPLINK) 2090 mlx5e_destroy_mdev_resources(dev); 2091 2092 err_destroy_netdev: 2093 mlx5e_destroy_netdev(netdev_priv(netdev)); 2094 kfree(rpriv); 2095 return err; 2096 } 2097 2098 static void 2099 mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) 2100 { 2101 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); 2102 struct net_device *netdev = rpriv->netdev; 2103 struct mlx5e_priv *priv = netdev_priv(netdev); 2104 struct mlx5_core_dev *dev = priv->mdev; 2105 void *ppriv = priv->ppriv; 2106 2107 if (is_devlink_port_supported(dev, rpriv)) 2108 devlink_port_type_clear(&rpriv->dl_port); 2109 unregister_netdev(netdev); 2110 unregister_devlink_port(dev, rpriv); 2111 mlx5e_rep_neigh_cleanup(rpriv); 2112 mlx5e_detach_netdev(priv); 2113 if (rep->vport == MLX5_VPORT_UPLINK) 2114 mlx5e_destroy_mdev_resources(priv->mdev); 2115 mlx5e_destroy_netdev(priv); 2116 kfree(ppriv); /* mlx5e_rep_priv */ 2117 } 2118 2119 static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep *rep) 2120 { 2121 struct mlx5e_rep_priv *rpriv; 2122 2123 rpriv = mlx5e_rep_to_rep_priv(rep); 2124 2125 return rpriv->netdev; 2126 } 2127 2128 static const struct mlx5_eswitch_rep_ops rep_ops = { 2129 .load = mlx5e_vport_rep_load, 2130 .unload = mlx5e_vport_rep_unload, 2131 .get_proto_dev = mlx5e_vport_rep_get_proto_dev 2132 }; 2133 2134 void mlx5e_rep_register_vport_reps(struct mlx5_core_dev *mdev) 2135 { 2136 struct mlx5_eswitch *esw = mdev->priv.eswitch; 2137 2138 mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_ETH); 2139 } 2140 2141 void mlx5e_rep_unregister_vport_reps(struct mlx5_core_dev *mdev) 2142 { 2143 struct mlx5_eswitch *esw = mdev->priv.eswitch; 2144 2145 mlx5_eswitch_unregister_vport_reps(esw, REP_ETH); 2146 } 2147