1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/debugfs.h> 34 #include <linux/mlx5/fs.h> 35 #include <net/switchdev.h> 36 #include <net/pkt_cls.h> 37 #include <net/act_api.h> 38 #include <net/devlink.h> 39 #include <net/ipv6_stubs.h> 40 41 #include "eswitch.h" 42 #include "en.h" 43 #include "en_rep.h" 44 #include "en/params.h" 45 #include "en/txrx.h" 46 #include "en_tc.h" 47 #include "en/rep/tc.h" 48 #include "en/rep/neigh.h" 49 #include "en/rep/bridge.h" 50 #include "en/devlink.h" 51 #include "fs_core.h" 52 #include "lib/mlx5.h" 53 #include "lib/devcom.h" 54 #include "lib/vxlan.h" 55 #define CREATE_TRACE_POINTS 56 #include "diag/en_rep_tracepoint.h" 57 #include "diag/reporter_vnic.h" 58 #include "en_accel/ipsec.h" 59 #include "en/tc/int_port.h" 60 #include "en/ptp.h" 61 #include "en/fs_ethtool.h" 62 63 #define MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE \ 64 max(0x7, MLX5E_PARAMS_MINIMUM_LOG_SQ_SIZE) 65 #define MLX5E_REP_PARAMS_DEF_NUM_CHANNELS 1 66 67 static const char mlx5e_rep_driver_name[] = "mlx5e_rep"; 68 69 static void mlx5e_rep_get_drvinfo(struct net_device *dev, 70 struct ethtool_drvinfo *drvinfo) 71 { 72 struct mlx5e_priv *priv = netdev_priv(dev); 73 struct mlx5_core_dev *mdev = priv->mdev; 74 75 strscpy(drvinfo->driver, mlx5e_rep_driver_name, 76 sizeof(drvinfo->driver)); 77 snprintf(drvinfo->fw_version, sizeof(drvinfo->fw_version), 78 "%d.%d.%04d (%.16s)", 79 fw_rev_maj(mdev), fw_rev_min(mdev), 80 fw_rev_sub(mdev), mdev->board_id); 81 } 82 83 static const struct counter_desc sw_rep_stats_desc[] = { 84 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_packets) }, 85 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, rx_bytes) }, 86 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_packets) }, 87 { MLX5E_DECLARE_STAT(struct mlx5e_sw_stats, tx_bytes) }, 88 }; 89 90 static const struct counter_desc vport_rep_stats_desc[] = { 91 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_packets) }, 92 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_rx_bytes) }, 93 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_packets) }, 94 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, vport_tx_bytes) }, 95 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 96 rx_vport_rdma_unicast_packets) }, 97 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, rx_vport_rdma_unicast_bytes) }, 98 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 99 tx_vport_rdma_unicast_packets) }, 100 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, tx_vport_rdma_unicast_bytes) }, 101 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 102 rx_vport_rdma_multicast_packets) }, 103 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 104 rx_vport_rdma_multicast_bytes) }, 105 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 106 tx_vport_rdma_multicast_packets) }, 107 { MLX5E_DECLARE_STAT(struct mlx5e_rep_stats, 108 tx_vport_rdma_multicast_bytes) }, 109 }; 110 111 #define NUM_VPORT_REP_SW_COUNTERS ARRAY_SIZE(sw_rep_stats_desc) 112 #define NUM_VPORT_REP_HW_COUNTERS ARRAY_SIZE(vport_rep_stats_desc) 113 114 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(sw_rep) 115 { 116 return NUM_VPORT_REP_SW_COUNTERS; 117 } 118 119 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(sw_rep) 120 { 121 int i; 122 123 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++) 124 strcpy(data + (idx++) * ETH_GSTRING_LEN, 125 sw_rep_stats_desc[i].format); 126 return idx; 127 } 128 129 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(sw_rep) 130 { 131 int i; 132 133 for (i = 0; i < NUM_VPORT_REP_SW_COUNTERS; i++) 134 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.sw, 135 sw_rep_stats_desc, i); 136 return idx; 137 } 138 139 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(sw_rep) 140 { 141 struct mlx5e_sw_stats *s = &priv->stats.sw; 142 struct rtnl_link_stats64 stats64 = {}; 143 144 memset(s, 0, sizeof(*s)); 145 mlx5e_fold_sw_stats64(priv, &stats64); 146 147 s->rx_packets = stats64.rx_packets; 148 s->rx_bytes = stats64.rx_bytes; 149 s->tx_packets = stats64.tx_packets; 150 s->tx_bytes = stats64.tx_bytes; 151 s->tx_queue_dropped = stats64.tx_dropped; 152 } 153 154 static MLX5E_DECLARE_STATS_GRP_OP_NUM_STATS(vport_rep) 155 { 156 return NUM_VPORT_REP_HW_COUNTERS; 157 } 158 159 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STRS(vport_rep) 160 { 161 int i; 162 163 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++) 164 strcpy(data + (idx++) * ETH_GSTRING_LEN, vport_rep_stats_desc[i].format); 165 return idx; 166 } 167 168 static MLX5E_DECLARE_STATS_GRP_OP_FILL_STATS(vport_rep) 169 { 170 int i; 171 172 for (i = 0; i < NUM_VPORT_REP_HW_COUNTERS; i++) 173 data[idx++] = MLX5E_READ_CTR64_CPU(&priv->stats.rep_stats, 174 vport_rep_stats_desc, i); 175 return idx; 176 } 177 178 static MLX5E_DECLARE_STATS_GRP_OP_UPDATE_STATS(vport_rep) 179 { 180 struct mlx5e_rep_stats *rep_stats = &priv->stats.rep_stats; 181 int outlen = MLX5_ST_SZ_BYTES(query_vport_counter_out); 182 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 183 struct mlx5e_rep_priv *rpriv = priv->ppriv; 184 struct mlx5_eswitch_rep *rep = rpriv->rep; 185 u32 *out; 186 int err; 187 188 out = kvzalloc(outlen, GFP_KERNEL); 189 if (!out) 190 return; 191 192 err = mlx5_core_query_vport_counter(esw->dev, 1, rep->vport - 1, 0, out); 193 if (err) { 194 netdev_warn(priv->netdev, "vport %d error %d reading stats\n", 195 rep->vport, err); 196 goto out; 197 } 198 199 #define MLX5_GET_CTR(p, x) \ 200 MLX5_GET64(query_vport_counter_out, p, x) 201 /* flip tx/rx as we are reporting the counters for the switch vport */ 202 rep_stats->vport_rx_packets = 203 MLX5_GET_CTR(out, transmitted_ib_unicast.packets) + 204 MLX5_GET_CTR(out, transmitted_eth_unicast.packets) + 205 MLX5_GET_CTR(out, transmitted_ib_multicast.packets) + 206 MLX5_GET_CTR(out, transmitted_eth_multicast.packets) + 207 MLX5_GET_CTR(out, transmitted_eth_broadcast.packets); 208 209 rep_stats->vport_tx_packets = 210 MLX5_GET_CTR(out, received_ib_unicast.packets) + 211 MLX5_GET_CTR(out, received_eth_unicast.packets) + 212 MLX5_GET_CTR(out, received_ib_multicast.packets) + 213 MLX5_GET_CTR(out, received_eth_multicast.packets) + 214 MLX5_GET_CTR(out, received_eth_broadcast.packets); 215 216 rep_stats->vport_rx_bytes = 217 MLX5_GET_CTR(out, transmitted_ib_unicast.octets) + 218 MLX5_GET_CTR(out, transmitted_eth_unicast.octets) + 219 MLX5_GET_CTR(out, transmitted_ib_multicast.octets) + 220 MLX5_GET_CTR(out, transmitted_eth_broadcast.octets); 221 222 rep_stats->vport_tx_bytes = 223 MLX5_GET_CTR(out, received_ib_unicast.octets) + 224 MLX5_GET_CTR(out, received_eth_unicast.octets) + 225 MLX5_GET_CTR(out, received_ib_multicast.octets) + 226 MLX5_GET_CTR(out, received_eth_multicast.octets) + 227 MLX5_GET_CTR(out, received_eth_broadcast.octets); 228 229 rep_stats->rx_vport_rdma_unicast_packets = 230 MLX5_GET_CTR(out, transmitted_ib_unicast.packets); 231 rep_stats->tx_vport_rdma_unicast_packets = 232 MLX5_GET_CTR(out, received_ib_unicast.packets); 233 rep_stats->rx_vport_rdma_unicast_bytes = 234 MLX5_GET_CTR(out, transmitted_ib_unicast.octets); 235 rep_stats->tx_vport_rdma_unicast_bytes = 236 MLX5_GET_CTR(out, received_ib_unicast.octets); 237 rep_stats->rx_vport_rdma_multicast_packets = 238 MLX5_GET_CTR(out, transmitted_ib_multicast.packets); 239 rep_stats->tx_vport_rdma_multicast_packets = 240 MLX5_GET_CTR(out, received_ib_multicast.packets); 241 rep_stats->rx_vport_rdma_multicast_bytes = 242 MLX5_GET_CTR(out, transmitted_ib_multicast.octets); 243 rep_stats->tx_vport_rdma_multicast_bytes = 244 MLX5_GET_CTR(out, received_ib_multicast.octets); 245 246 out: 247 kvfree(out); 248 } 249 250 static void mlx5e_rep_get_strings(struct net_device *dev, 251 u32 stringset, uint8_t *data) 252 { 253 struct mlx5e_priv *priv = netdev_priv(dev); 254 255 switch (stringset) { 256 case ETH_SS_STATS: 257 mlx5e_stats_fill_strings(priv, data); 258 break; 259 } 260 } 261 262 static void mlx5e_rep_get_ethtool_stats(struct net_device *dev, 263 struct ethtool_stats *stats, u64 *data) 264 { 265 struct mlx5e_priv *priv = netdev_priv(dev); 266 267 mlx5e_ethtool_get_ethtool_stats(priv, stats, data); 268 } 269 270 static int mlx5e_rep_get_sset_count(struct net_device *dev, int sset) 271 { 272 struct mlx5e_priv *priv = netdev_priv(dev); 273 274 switch (sset) { 275 case ETH_SS_STATS: 276 return mlx5e_stats_total_num(priv); 277 default: 278 return -EOPNOTSUPP; 279 } 280 } 281 282 static void 283 mlx5e_rep_get_ringparam(struct net_device *dev, 284 struct ethtool_ringparam *param, 285 struct kernel_ethtool_ringparam *kernel_param, 286 struct netlink_ext_ack *extack) 287 { 288 struct mlx5e_priv *priv = netdev_priv(dev); 289 290 mlx5e_ethtool_get_ringparam(priv, param, kernel_param); 291 } 292 293 static int 294 mlx5e_rep_set_ringparam(struct net_device *dev, 295 struct ethtool_ringparam *param, 296 struct kernel_ethtool_ringparam *kernel_param, 297 struct netlink_ext_ack *extack) 298 { 299 struct mlx5e_priv *priv = netdev_priv(dev); 300 301 return mlx5e_ethtool_set_ringparam(priv, param); 302 } 303 304 static void mlx5e_rep_get_channels(struct net_device *dev, 305 struct ethtool_channels *ch) 306 { 307 struct mlx5e_priv *priv = netdev_priv(dev); 308 309 mlx5e_ethtool_get_channels(priv, ch); 310 } 311 312 static int mlx5e_rep_set_channels(struct net_device *dev, 313 struct ethtool_channels *ch) 314 { 315 struct mlx5e_priv *priv = netdev_priv(dev); 316 317 return mlx5e_ethtool_set_channels(priv, ch); 318 } 319 320 static int mlx5e_rep_get_coalesce(struct net_device *netdev, 321 struct ethtool_coalesce *coal, 322 struct kernel_ethtool_coalesce *kernel_coal, 323 struct netlink_ext_ack *extack) 324 { 325 struct mlx5e_priv *priv = netdev_priv(netdev); 326 327 return mlx5e_ethtool_get_coalesce(priv, coal, kernel_coal); 328 } 329 330 static int mlx5e_rep_set_coalesce(struct net_device *netdev, 331 struct ethtool_coalesce *coal, 332 struct kernel_ethtool_coalesce *kernel_coal, 333 struct netlink_ext_ack *extack) 334 { 335 struct mlx5e_priv *priv = netdev_priv(netdev); 336 337 return mlx5e_ethtool_set_coalesce(priv, coal, kernel_coal, extack); 338 } 339 340 static u32 mlx5e_rep_get_rxfh_key_size(struct net_device *netdev) 341 { 342 struct mlx5e_priv *priv = netdev_priv(netdev); 343 344 return mlx5e_ethtool_get_rxfh_key_size(priv); 345 } 346 347 static u32 mlx5e_rep_get_rxfh_indir_size(struct net_device *netdev) 348 { 349 struct mlx5e_priv *priv = netdev_priv(netdev); 350 351 return mlx5e_ethtool_get_rxfh_indir_size(priv); 352 } 353 354 static const struct ethtool_ops mlx5e_rep_ethtool_ops = { 355 .supported_coalesce_params = ETHTOOL_COALESCE_USECS | 356 ETHTOOL_COALESCE_MAX_FRAMES | 357 ETHTOOL_COALESCE_USE_ADAPTIVE, 358 .get_drvinfo = mlx5e_rep_get_drvinfo, 359 .get_link = ethtool_op_get_link, 360 .get_strings = mlx5e_rep_get_strings, 361 .get_sset_count = mlx5e_rep_get_sset_count, 362 .get_ethtool_stats = mlx5e_rep_get_ethtool_stats, 363 .get_ringparam = mlx5e_rep_get_ringparam, 364 .set_ringparam = mlx5e_rep_set_ringparam, 365 .get_channels = mlx5e_rep_get_channels, 366 .set_channels = mlx5e_rep_set_channels, 367 .get_coalesce = mlx5e_rep_get_coalesce, 368 .set_coalesce = mlx5e_rep_set_coalesce, 369 .get_rxfh_key_size = mlx5e_rep_get_rxfh_key_size, 370 .get_rxfh_indir_size = mlx5e_rep_get_rxfh_indir_size, 371 }; 372 373 static void mlx5e_sqs2vport_stop(struct mlx5_eswitch *esw, 374 struct mlx5_eswitch_rep *rep) 375 { 376 struct mlx5e_rep_sq *rep_sq, *tmp; 377 struct mlx5e_rep_sq_peer *sq_peer; 378 struct mlx5e_rep_priv *rpriv; 379 unsigned long i; 380 381 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 382 return; 383 384 rpriv = mlx5e_rep_to_rep_priv(rep); 385 list_for_each_entry_safe(rep_sq, tmp, &rpriv->vport_sqs_list, list) { 386 mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule); 387 xa_for_each(&rep_sq->sq_peer, i, sq_peer) { 388 if (sq_peer->rule) 389 mlx5_eswitch_del_send_to_vport_rule(sq_peer->rule); 390 391 xa_erase(&rep_sq->sq_peer, i); 392 kfree(sq_peer); 393 } 394 395 xa_destroy(&rep_sq->sq_peer); 396 list_del(&rep_sq->list); 397 kfree(rep_sq); 398 } 399 } 400 401 static int mlx5e_sqs2vport_add_peers_rules(struct mlx5_eswitch *esw, struct mlx5_eswitch_rep *rep, 402 struct mlx5e_rep_sq *rep_sq, int i) 403 { 404 struct mlx5_flow_handle *flow_rule; 405 struct mlx5_devcom_comp_dev *tmp; 406 struct mlx5_eswitch *peer_esw; 407 408 mlx5_devcom_for_each_peer_entry(esw->devcom, peer_esw, tmp) { 409 u16 peer_rule_idx = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 410 struct mlx5e_rep_sq_peer *sq_peer; 411 int err; 412 413 sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL); 414 if (!sq_peer) 415 return -ENOMEM; 416 417 flow_rule = mlx5_eswitch_add_send_to_vport_rule(peer_esw, esw, 418 rep, rep_sq->sqn); 419 if (IS_ERR(flow_rule)) { 420 kfree(sq_peer); 421 return PTR_ERR(flow_rule); 422 } 423 424 sq_peer->rule = flow_rule; 425 sq_peer->peer = peer_esw; 426 err = xa_insert(&rep_sq->sq_peer, peer_rule_idx, sq_peer, GFP_KERNEL); 427 if (err) { 428 kfree(sq_peer); 429 mlx5_eswitch_del_send_to_vport_rule(flow_rule); 430 return err; 431 } 432 } 433 434 return 0; 435 } 436 437 static int mlx5e_sqs2vport_start(struct mlx5_eswitch *esw, 438 struct mlx5_eswitch_rep *rep, 439 u32 *sqns_array, int sqns_num) 440 { 441 struct mlx5_flow_handle *flow_rule; 442 struct mlx5e_rep_priv *rpriv; 443 struct mlx5e_rep_sq *rep_sq; 444 bool devcom_locked = false; 445 int err; 446 int i; 447 448 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 449 return 0; 450 451 rpriv = mlx5e_rep_to_rep_priv(rep); 452 453 if (mlx5_devcom_comp_is_ready(esw->devcom) && 454 mlx5_devcom_for_each_peer_begin(esw->devcom)) 455 devcom_locked = true; 456 457 for (i = 0; i < sqns_num; i++) { 458 rep_sq = kzalloc(sizeof(*rep_sq), GFP_KERNEL); 459 if (!rep_sq) { 460 err = -ENOMEM; 461 goto out_err; 462 } 463 464 /* Add re-inject rule to the PF/representor sqs */ 465 flow_rule = mlx5_eswitch_add_send_to_vport_rule(esw, esw, rep, 466 sqns_array[i]); 467 if (IS_ERR(flow_rule)) { 468 err = PTR_ERR(flow_rule); 469 kfree(rep_sq); 470 goto out_err; 471 } 472 rep_sq->send_to_vport_rule = flow_rule; 473 rep_sq->sqn = sqns_array[i]; 474 475 xa_init(&rep_sq->sq_peer); 476 if (devcom_locked) { 477 err = mlx5e_sqs2vport_add_peers_rules(esw, rep, rep_sq, i); 478 if (err) { 479 mlx5_eswitch_del_send_to_vport_rule(rep_sq->send_to_vport_rule); 480 xa_destroy(&rep_sq->sq_peer); 481 kfree(rep_sq); 482 goto out_err; 483 } 484 } 485 486 list_add(&rep_sq->list, &rpriv->vport_sqs_list); 487 } 488 489 if (devcom_locked) 490 mlx5_devcom_for_each_peer_end(esw->devcom); 491 492 return 0; 493 494 out_err: 495 mlx5e_sqs2vport_stop(esw, rep); 496 497 if (devcom_locked) 498 mlx5_devcom_for_each_peer_end(esw->devcom); 499 500 return err; 501 } 502 503 static int 504 mlx5e_add_sqs_fwd_rules(struct mlx5e_priv *priv) 505 { 506 int sqs_per_channel = mlx5e_get_dcb_num_tc(&priv->channels.params); 507 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 508 bool is_uplink_rep = mlx5e_is_uplink_rep(priv); 509 struct mlx5e_rep_priv *rpriv = priv->ppriv; 510 struct mlx5_eswitch_rep *rep = rpriv->rep; 511 int n, tc, nch, num_sqs = 0; 512 struct mlx5e_channel *c; 513 int err = -ENOMEM; 514 bool ptp_sq; 515 u32 *sqs; 516 517 ptp_sq = !!(priv->channels.ptp && 518 MLX5E_GET_PFLAG(&priv->channels.params, MLX5E_PFLAG_TX_PORT_TS)); 519 nch = priv->channels.num + ptp_sq; 520 /* +2 for xdpsqs, they don't exist on the ptp channel but will not be 521 * counted for by num_sqs. 522 */ 523 if (is_uplink_rep) 524 sqs_per_channel += 2; 525 526 sqs = kvcalloc(nch * sqs_per_channel, sizeof(*sqs), GFP_KERNEL); 527 if (!sqs) 528 goto out; 529 530 for (n = 0; n < priv->channels.num; n++) { 531 c = priv->channels.c[n]; 532 for (tc = 0; tc < c->num_tc; tc++) 533 sqs[num_sqs++] = c->sq[tc].sqn; 534 535 if (is_uplink_rep) { 536 if (c->xdp) 537 sqs[num_sqs++] = c->rq_xdpsq.sqn; 538 539 sqs[num_sqs++] = c->xdpsq.sqn; 540 } 541 } 542 if (ptp_sq) { 543 struct mlx5e_ptp *ptp_ch = priv->channels.ptp; 544 545 for (tc = 0; tc < ptp_ch->num_tc; tc++) 546 sqs[num_sqs++] = ptp_ch->ptpsq[tc].txqsq.sqn; 547 } 548 549 err = mlx5e_sqs2vport_start(esw, rep, sqs, num_sqs); 550 kvfree(sqs); 551 552 out: 553 if (err) 554 netdev_warn(priv->netdev, "Failed to add SQs FWD rules %d\n", err); 555 return err; 556 } 557 558 static void 559 mlx5e_remove_sqs_fwd_rules(struct mlx5e_priv *priv) 560 { 561 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 562 struct mlx5e_rep_priv *rpriv = priv->ppriv; 563 struct mlx5_eswitch_rep *rep = rpriv->rep; 564 565 mlx5e_sqs2vport_stop(esw, rep); 566 } 567 568 static int 569 mlx5e_rep_add_meta_tunnel_rule(struct mlx5e_priv *priv) 570 { 571 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 572 struct mlx5e_rep_priv *rpriv = priv->ppriv; 573 struct mlx5_eswitch_rep *rep = rpriv->rep; 574 struct mlx5_flow_handle *flow_rule; 575 struct mlx5_flow_group *g; 576 577 g = esw->fdb_table.offloads.send_to_vport_meta_grp; 578 if (!g) 579 return 0; 580 581 flow_rule = mlx5_eswitch_add_send_to_vport_meta_rule(esw, rep->vport); 582 if (IS_ERR(flow_rule)) 583 return PTR_ERR(flow_rule); 584 585 rpriv->send_to_vport_meta_rule = flow_rule; 586 587 return 0; 588 } 589 590 static void 591 mlx5e_rep_del_meta_tunnel_rule(struct mlx5e_priv *priv) 592 { 593 struct mlx5e_rep_priv *rpriv = priv->ppriv; 594 595 if (rpriv->send_to_vport_meta_rule) 596 mlx5_eswitch_del_send_to_vport_meta_rule(rpriv->send_to_vport_meta_rule); 597 } 598 599 void mlx5e_rep_activate_channels(struct mlx5e_priv *priv) 600 { 601 mlx5e_add_sqs_fwd_rules(priv); 602 mlx5e_rep_add_meta_tunnel_rule(priv); 603 } 604 605 void mlx5e_rep_deactivate_channels(struct mlx5e_priv *priv) 606 { 607 mlx5e_rep_del_meta_tunnel_rule(priv); 608 mlx5e_remove_sqs_fwd_rules(priv); 609 } 610 611 static int mlx5e_rep_open(struct net_device *dev) 612 { 613 struct mlx5e_priv *priv = netdev_priv(dev); 614 struct mlx5e_rep_priv *rpriv = priv->ppriv; 615 struct mlx5_eswitch_rep *rep = rpriv->rep; 616 int err; 617 618 mutex_lock(&priv->state_lock); 619 err = mlx5e_open_locked(dev); 620 if (err) 621 goto unlock; 622 623 if (!mlx5_modify_vport_admin_state(priv->mdev, 624 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 625 rep->vport, 1, 626 MLX5_VPORT_ADMIN_STATE_UP)) 627 netif_carrier_on(dev); 628 629 unlock: 630 mutex_unlock(&priv->state_lock); 631 return err; 632 } 633 634 static int mlx5e_rep_close(struct net_device *dev) 635 { 636 struct mlx5e_priv *priv = netdev_priv(dev); 637 struct mlx5e_rep_priv *rpriv = priv->ppriv; 638 struct mlx5_eswitch_rep *rep = rpriv->rep; 639 int ret; 640 641 mutex_lock(&priv->state_lock); 642 mlx5_modify_vport_admin_state(priv->mdev, 643 MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 644 rep->vport, 1, 645 MLX5_VPORT_ADMIN_STATE_DOWN); 646 ret = mlx5e_close_locked(dev); 647 mutex_unlock(&priv->state_lock); 648 return ret; 649 } 650 651 bool mlx5e_is_uplink_rep(struct mlx5e_priv *priv) 652 { 653 struct mlx5e_rep_priv *rpriv = priv->ppriv; 654 struct mlx5_eswitch_rep *rep; 655 656 if (!MLX5_ESWITCH_MANAGER(priv->mdev)) 657 return false; 658 659 if (!rpriv) /* non vport rep mlx5e instances don't use this field */ 660 return false; 661 662 rep = rpriv->rep; 663 return (rep->vport == MLX5_VPORT_UPLINK); 664 } 665 666 bool mlx5e_rep_has_offload_stats(const struct net_device *dev, int attr_id) 667 { 668 switch (attr_id) { 669 case IFLA_OFFLOAD_XSTATS_CPU_HIT: 670 return true; 671 } 672 673 return false; 674 } 675 676 static int 677 mlx5e_get_sw_stats64(const struct net_device *dev, 678 struct rtnl_link_stats64 *stats) 679 { 680 struct mlx5e_priv *priv = netdev_priv(dev); 681 682 mlx5e_fold_sw_stats64(priv, stats); 683 return 0; 684 } 685 686 int mlx5e_rep_get_offload_stats(int attr_id, const struct net_device *dev, 687 void *sp) 688 { 689 switch (attr_id) { 690 case IFLA_OFFLOAD_XSTATS_CPU_HIT: 691 return mlx5e_get_sw_stats64(dev, sp); 692 } 693 694 return -EINVAL; 695 } 696 697 static void 698 mlx5e_rep_get_stats(struct net_device *dev, struct rtnl_link_stats64 *stats) 699 { 700 struct mlx5e_priv *priv = netdev_priv(dev); 701 702 /* update HW stats in background for next time */ 703 mlx5e_queue_update_stats(priv); 704 memcpy(stats, &priv->stats.vf_vport, sizeof(*stats)); 705 } 706 707 static int mlx5e_rep_change_mtu(struct net_device *netdev, int new_mtu) 708 { 709 return mlx5e_change_mtu(netdev, new_mtu, NULL); 710 } 711 712 static int mlx5e_rep_change_carrier(struct net_device *dev, bool new_carrier) 713 { 714 struct mlx5e_priv *priv = netdev_priv(dev); 715 struct mlx5e_rep_priv *rpriv = priv->ppriv; 716 struct mlx5_eswitch_rep *rep = rpriv->rep; 717 int err; 718 719 if (new_carrier) { 720 err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 721 rep->vport, 1, MLX5_VPORT_ADMIN_STATE_UP); 722 if (err) 723 return err; 724 netif_carrier_on(dev); 725 } else { 726 err = mlx5_modify_vport_admin_state(priv->mdev, MLX5_VPORT_STATE_OP_MOD_ESW_VPORT, 727 rep->vport, 1, MLX5_VPORT_ADMIN_STATE_DOWN); 728 if (err) 729 return err; 730 netif_carrier_off(dev); 731 } 732 return 0; 733 } 734 735 static const struct net_device_ops mlx5e_netdev_ops_rep = { 736 .ndo_open = mlx5e_rep_open, 737 .ndo_stop = mlx5e_rep_close, 738 .ndo_start_xmit = mlx5e_xmit, 739 .ndo_setup_tc = mlx5e_rep_setup_tc, 740 .ndo_get_stats64 = mlx5e_rep_get_stats, 741 .ndo_has_offload_stats = mlx5e_rep_has_offload_stats, 742 .ndo_get_offload_stats = mlx5e_rep_get_offload_stats, 743 .ndo_change_mtu = mlx5e_rep_change_mtu, 744 .ndo_change_carrier = mlx5e_rep_change_carrier, 745 }; 746 747 bool mlx5e_eswitch_uplink_rep(const struct net_device *netdev) 748 { 749 return netdev->netdev_ops == &mlx5e_netdev_ops && 750 mlx5e_is_uplink_rep(netdev_priv(netdev)); 751 } 752 753 bool mlx5e_eswitch_vf_rep(const struct net_device *netdev) 754 { 755 return netdev->netdev_ops == &mlx5e_netdev_ops_rep; 756 } 757 758 /* One indirect TIR set for outer. Inner not supported in reps. */ 759 #define REP_NUM_INDIR_TIRS MLX5E_NUM_INDIR_TIRS 760 761 static int mlx5e_rep_max_nch_limit(struct mlx5_core_dev *mdev) 762 { 763 int max_tir_num = 1 << MLX5_CAP_GEN(mdev, log_max_tir); 764 int num_vports = mlx5_eswitch_get_total_vports(mdev); 765 766 return (max_tir_num - mlx5e_get_pf_num_tirs(mdev) 767 - (num_vports * REP_NUM_INDIR_TIRS)) / num_vports; 768 } 769 770 static void mlx5e_build_rep_params(struct net_device *netdev) 771 { 772 struct mlx5e_priv *priv = netdev_priv(netdev); 773 struct mlx5e_rep_priv *rpriv = priv->ppriv; 774 struct mlx5_eswitch_rep *rep = rpriv->rep; 775 struct mlx5_core_dev *mdev = priv->mdev; 776 struct mlx5e_params *params; 777 778 u8 cq_period_mode = MLX5_CAP_GEN(mdev, cq_period_start_from_cqe) ? 779 MLX5_CQ_PERIOD_MODE_START_FROM_CQE : 780 MLX5_CQ_PERIOD_MODE_START_FROM_EQE; 781 782 params = &priv->channels.params; 783 784 params->num_channels = MLX5E_REP_PARAMS_DEF_NUM_CHANNELS; 785 params->hard_mtu = MLX5E_ETH_HARD_MTU; 786 params->sw_mtu = netdev->mtu; 787 788 /* SQ */ 789 if (rep->vport == MLX5_VPORT_UPLINK) 790 params->log_sq_size = MLX5E_PARAMS_DEFAULT_LOG_SQ_SIZE; 791 else 792 params->log_sq_size = MLX5E_REP_PARAMS_DEF_LOG_SQ_SIZE; 793 794 /* RQ */ 795 mlx5e_build_rq_params(mdev, params); 796 797 /* update XDP supported features */ 798 mlx5e_set_xdp_feature(netdev); 799 800 /* CQ moderation params */ 801 params->rx_dim_enabled = MLX5_CAP_GEN(mdev, cq_moderation); 802 mlx5e_set_rx_cq_mode_params(params, cq_period_mode); 803 804 params->mqprio.num_tc = 1; 805 if (rep->vport != MLX5_VPORT_UPLINK) 806 params->vlan_strip_disable = true; 807 808 mlx5_query_min_inline(mdev, ¶ms->tx_min_inline_mode); 809 } 810 811 static void mlx5e_build_rep_netdev(struct net_device *netdev, 812 struct mlx5_core_dev *mdev) 813 { 814 SET_NETDEV_DEV(netdev, mdev->device); 815 netdev->netdev_ops = &mlx5e_netdev_ops_rep; 816 eth_hw_addr_random(netdev); 817 netdev->ethtool_ops = &mlx5e_rep_ethtool_ops; 818 819 netdev->watchdog_timeo = 15 * HZ; 820 821 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 822 netdev->hw_features |= NETIF_F_HW_TC; 823 #endif 824 netdev->hw_features |= NETIF_F_SG; 825 netdev->hw_features |= NETIF_F_IP_CSUM; 826 netdev->hw_features |= NETIF_F_IPV6_CSUM; 827 netdev->hw_features |= NETIF_F_GRO; 828 netdev->hw_features |= NETIF_F_TSO; 829 netdev->hw_features |= NETIF_F_TSO6; 830 netdev->hw_features |= NETIF_F_RXCSUM; 831 832 netdev->features |= netdev->hw_features; 833 netdev->features |= NETIF_F_NETNS_LOCAL; 834 } 835 836 static int mlx5e_init_rep(struct mlx5_core_dev *mdev, 837 struct net_device *netdev) 838 { 839 struct mlx5e_priv *priv = netdev_priv(netdev); 840 841 priv->fs = 842 mlx5e_fs_init(priv->profile, mdev, 843 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 844 priv->dfs_root); 845 if (!priv->fs) { 846 netdev_err(priv->netdev, "FS allocation failed\n"); 847 return -ENOMEM; 848 } 849 850 mlx5e_build_rep_params(netdev); 851 mlx5e_timestamp_init(priv); 852 853 return 0; 854 } 855 856 static int mlx5e_init_ul_rep(struct mlx5_core_dev *mdev, 857 struct net_device *netdev) 858 { 859 struct mlx5e_priv *priv = netdev_priv(netdev); 860 861 priv->dfs_root = debugfs_create_dir("nic", 862 mlx5_debugfs_get_dev_root(mdev)); 863 864 priv->fs = mlx5e_fs_init(priv->profile, mdev, 865 !test_bit(MLX5E_STATE_DESTROYING, &priv->state), 866 priv->dfs_root); 867 if (!priv->fs) { 868 netdev_err(priv->netdev, "FS allocation failed\n"); 869 debugfs_remove_recursive(priv->dfs_root); 870 return -ENOMEM; 871 } 872 873 mlx5e_vxlan_set_netdev_info(priv); 874 mlx5e_build_rep_params(netdev); 875 mlx5e_timestamp_init(priv); 876 return 0; 877 } 878 879 static void mlx5e_cleanup_rep(struct mlx5e_priv *priv) 880 { 881 mlx5e_fs_cleanup(priv->fs); 882 debugfs_remove_recursive(priv->dfs_root); 883 priv->fs = NULL; 884 } 885 886 static int mlx5e_create_rep_ttc_table(struct mlx5e_priv *priv) 887 { 888 struct mlx5e_rep_priv *rpriv = priv->ppriv; 889 struct mlx5_eswitch_rep *rep = rpriv->rep; 890 struct ttc_params ttc_params = {}; 891 int err; 892 893 mlx5e_fs_set_ns(priv->fs, 894 mlx5_get_flow_namespace(priv->mdev, 895 MLX5_FLOW_NAMESPACE_KERNEL), false); 896 897 /* The inner_ttc in the ttc params is intentionally not set */ 898 mlx5e_set_ttc_params(priv->fs, priv->rx_res, &ttc_params, false); 899 900 if (rep->vport != MLX5_VPORT_UPLINK) 901 /* To give uplik rep TTC a lower level for chaining from root ft */ 902 ttc_params.ft_attr.level = MLX5E_TTC_FT_LEVEL + 1; 903 904 mlx5e_fs_set_ttc(priv->fs, mlx5_create_ttc_table(priv->mdev, &ttc_params), false); 905 if (IS_ERR(mlx5e_fs_get_ttc(priv->fs, false))) { 906 err = PTR_ERR(mlx5e_fs_get_ttc(priv->fs, false)); 907 netdev_err(priv->netdev, "Failed to create rep ttc table, err=%d\n", 908 err); 909 return err; 910 } 911 return 0; 912 } 913 914 static int mlx5e_create_rep_root_ft(struct mlx5e_priv *priv) 915 { 916 struct mlx5e_rep_priv *rpriv = priv->ppriv; 917 struct mlx5_eswitch_rep *rep = rpriv->rep; 918 struct mlx5_flow_table_attr ft_attr = {}; 919 struct mlx5_flow_namespace *ns; 920 int err = 0; 921 922 if (rep->vport != MLX5_VPORT_UPLINK) { 923 /* non uplik reps will skip any bypass tables and go directly to 924 * their own ttc 925 */ 926 rpriv->root_ft = mlx5_get_ttc_flow_table(mlx5e_fs_get_ttc(priv->fs, false)); 927 return 0; 928 } 929 930 /* uplink root ft will be used to auto chain, to ethtool or ttc tables */ 931 ns = mlx5_get_flow_namespace(priv->mdev, MLX5_FLOW_NAMESPACE_OFFLOADS); 932 if (!ns) { 933 netdev_err(priv->netdev, "Failed to get reps offloads namespace\n"); 934 return -EOPNOTSUPP; 935 } 936 937 ft_attr.max_fte = 0; /* Empty table, miss rule will always point to next table */ 938 ft_attr.prio = 1; 939 ft_attr.level = 1; 940 941 rpriv->root_ft = mlx5_create_flow_table(ns, &ft_attr); 942 if (IS_ERR(rpriv->root_ft)) { 943 err = PTR_ERR(rpriv->root_ft); 944 rpriv->root_ft = NULL; 945 } 946 947 return err; 948 } 949 950 static void mlx5e_destroy_rep_root_ft(struct mlx5e_priv *priv) 951 { 952 struct mlx5e_rep_priv *rpriv = priv->ppriv; 953 struct mlx5_eswitch_rep *rep = rpriv->rep; 954 955 if (rep->vport != MLX5_VPORT_UPLINK) 956 return; 957 mlx5_destroy_flow_table(rpriv->root_ft); 958 } 959 960 static int mlx5e_create_rep_vport_rx_rule(struct mlx5e_priv *priv) 961 { 962 struct mlx5_eswitch *esw = priv->mdev->priv.eswitch; 963 struct mlx5e_rep_priv *rpriv = priv->ppriv; 964 struct mlx5_eswitch_rep *rep = rpriv->rep; 965 struct mlx5_flow_handle *flow_rule; 966 struct mlx5_flow_destination dest; 967 968 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 969 dest.ft = rpriv->root_ft; 970 971 flow_rule = mlx5_eswitch_create_vport_rx_rule(esw, rep->vport, &dest); 972 if (IS_ERR(flow_rule)) 973 return PTR_ERR(flow_rule); 974 rpriv->vport_rx_rule = flow_rule; 975 return 0; 976 } 977 978 static void rep_vport_rx_rule_destroy(struct mlx5e_priv *priv) 979 { 980 struct mlx5e_rep_priv *rpriv = priv->ppriv; 981 982 if (!rpriv->vport_rx_rule) 983 return; 984 985 mlx5_del_flow_rules(rpriv->vport_rx_rule); 986 rpriv->vport_rx_rule = NULL; 987 } 988 989 int mlx5e_rep_bond_update(struct mlx5e_priv *priv, bool cleanup) 990 { 991 rep_vport_rx_rule_destroy(priv); 992 993 return cleanup ? 0 : mlx5e_create_rep_vport_rx_rule(priv); 994 } 995 996 static int mlx5e_init_rep_rx(struct mlx5e_priv *priv) 997 { 998 struct mlx5_core_dev *mdev = priv->mdev; 999 int err; 1000 1001 priv->rx_res = mlx5e_rx_res_alloc(); 1002 if (!priv->rx_res) { 1003 err = -ENOMEM; 1004 goto err_free_fs; 1005 } 1006 1007 mlx5e_fs_init_l2_addr(priv->fs, priv->netdev); 1008 1009 err = mlx5e_open_drop_rq(priv, &priv->drop_rq); 1010 if (err) { 1011 mlx5_core_err(mdev, "open drop rq failed, %d\n", err); 1012 goto err_rx_res_free; 1013 } 1014 1015 err = mlx5e_rx_res_init(priv->rx_res, priv->mdev, 0, 1016 priv->max_nch, priv->drop_rq.rqn, 1017 &priv->channels.params.packet_merge, 1018 priv->channels.params.num_channels); 1019 if (err) 1020 goto err_close_drop_rq; 1021 1022 err = mlx5e_create_rep_ttc_table(priv); 1023 if (err) 1024 goto err_destroy_rx_res; 1025 1026 err = mlx5e_create_rep_root_ft(priv); 1027 if (err) 1028 goto err_destroy_ttc_table; 1029 1030 err = mlx5e_create_rep_vport_rx_rule(priv); 1031 if (err) 1032 goto err_destroy_root_ft; 1033 1034 mlx5e_ethtool_init_steering(priv->fs); 1035 1036 return 0; 1037 1038 err_destroy_root_ft: 1039 mlx5e_destroy_rep_root_ft(priv); 1040 err_destroy_ttc_table: 1041 mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false)); 1042 err_destroy_rx_res: 1043 mlx5e_rx_res_destroy(priv->rx_res); 1044 err_close_drop_rq: 1045 mlx5e_close_drop_rq(&priv->drop_rq); 1046 err_rx_res_free: 1047 mlx5e_rx_res_free(priv->rx_res); 1048 priv->rx_res = NULL; 1049 err_free_fs: 1050 mlx5e_fs_cleanup(priv->fs); 1051 priv->fs = NULL; 1052 return err; 1053 } 1054 1055 static void mlx5e_cleanup_rep_rx(struct mlx5e_priv *priv) 1056 { 1057 mlx5e_ethtool_cleanup_steering(priv->fs); 1058 rep_vport_rx_rule_destroy(priv); 1059 mlx5e_destroy_rep_root_ft(priv); 1060 mlx5_destroy_ttc_table(mlx5e_fs_get_ttc(priv->fs, false)); 1061 mlx5e_rx_res_destroy(priv->rx_res); 1062 mlx5e_close_drop_rq(&priv->drop_rq); 1063 mlx5e_rx_res_free(priv->rx_res); 1064 priv->rx_res = NULL; 1065 } 1066 1067 static void mlx5e_rep_mpesw_work(struct work_struct *work) 1068 { 1069 struct mlx5_rep_uplink_priv *uplink_priv = 1070 container_of(work, struct mlx5_rep_uplink_priv, 1071 mpesw_work); 1072 struct mlx5e_rep_priv *rpriv = 1073 container_of(uplink_priv, struct mlx5e_rep_priv, 1074 uplink_priv); 1075 struct mlx5e_priv *priv = netdev_priv(rpriv->netdev); 1076 1077 rep_vport_rx_rule_destroy(priv); 1078 mlx5e_create_rep_vport_rx_rule(priv); 1079 } 1080 1081 static int mlx5e_init_ul_rep_rx(struct mlx5e_priv *priv) 1082 { 1083 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1084 int err; 1085 1086 mlx5e_create_q_counters(priv); 1087 err = mlx5e_init_rep_rx(priv); 1088 if (err) 1089 goto out; 1090 1091 mlx5e_tc_int_port_init_rep_rx(priv); 1092 1093 INIT_WORK(&rpriv->uplink_priv.mpesw_work, mlx5e_rep_mpesw_work); 1094 1095 out: 1096 return err; 1097 } 1098 1099 static void mlx5e_cleanup_ul_rep_rx(struct mlx5e_priv *priv) 1100 { 1101 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1102 1103 cancel_work_sync(&rpriv->uplink_priv.mpesw_work); 1104 mlx5e_tc_int_port_cleanup_rep_rx(priv); 1105 mlx5e_cleanup_rep_rx(priv); 1106 mlx5e_destroy_q_counters(priv); 1107 } 1108 1109 static int mlx5e_init_uplink_rep_tx(struct mlx5e_rep_priv *rpriv) 1110 { 1111 struct mlx5_rep_uplink_priv *uplink_priv; 1112 struct net_device *netdev; 1113 struct mlx5e_priv *priv; 1114 int err; 1115 1116 netdev = rpriv->netdev; 1117 priv = netdev_priv(netdev); 1118 uplink_priv = &rpriv->uplink_priv; 1119 1120 err = mlx5e_rep_tc_init(rpriv); 1121 if (err) 1122 return err; 1123 1124 mlx5_init_port_tun_entropy(&uplink_priv->tun_entropy, priv->mdev); 1125 1126 mlx5e_rep_bond_init(rpriv); 1127 err = mlx5e_rep_tc_netdevice_event_register(rpriv); 1128 if (err) { 1129 mlx5_core_err(priv->mdev, "Failed to register netdev notifier, err: %d\n", 1130 err); 1131 goto err_event_reg; 1132 } 1133 1134 return 0; 1135 1136 err_event_reg: 1137 mlx5e_rep_bond_cleanup(rpriv); 1138 mlx5e_rep_tc_cleanup(rpriv); 1139 return err; 1140 } 1141 1142 static void mlx5e_cleanup_uplink_rep_tx(struct mlx5e_rep_priv *rpriv) 1143 { 1144 mlx5e_rep_tc_netdevice_event_unregister(rpriv); 1145 mlx5e_rep_bond_cleanup(rpriv); 1146 mlx5e_rep_tc_cleanup(rpriv); 1147 } 1148 1149 static int mlx5e_init_rep_tx(struct mlx5e_priv *priv) 1150 { 1151 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1152 int err; 1153 1154 err = mlx5e_create_tises(priv); 1155 if (err) { 1156 mlx5_core_warn(priv->mdev, "create tises failed, %d\n", err); 1157 return err; 1158 } 1159 1160 err = mlx5e_rep_neigh_init(rpriv); 1161 if (err) 1162 goto err_neigh_init; 1163 1164 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) { 1165 err = mlx5e_init_uplink_rep_tx(rpriv); 1166 if (err) 1167 goto err_init_tx; 1168 } 1169 1170 err = mlx5e_tc_ht_init(&rpriv->tc_ht); 1171 if (err) 1172 goto err_ht_init; 1173 1174 return 0; 1175 1176 err_ht_init: 1177 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) 1178 mlx5e_cleanup_uplink_rep_tx(rpriv); 1179 err_init_tx: 1180 mlx5e_rep_neigh_cleanup(rpriv); 1181 err_neigh_init: 1182 mlx5e_destroy_tises(priv); 1183 return err; 1184 } 1185 1186 static void mlx5e_cleanup_rep_tx(struct mlx5e_priv *priv) 1187 { 1188 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1189 1190 mlx5e_tc_ht_cleanup(&rpriv->tc_ht); 1191 1192 if (rpriv->rep->vport == MLX5_VPORT_UPLINK) 1193 mlx5e_cleanup_uplink_rep_tx(rpriv); 1194 1195 mlx5e_rep_neigh_cleanup(rpriv); 1196 mlx5e_destroy_tises(priv); 1197 } 1198 1199 static void mlx5e_rep_enable(struct mlx5e_priv *priv) 1200 { 1201 mlx5e_set_netdev_mtu_boundaries(priv); 1202 } 1203 1204 static void mlx5e_rep_disable(struct mlx5e_priv *priv) 1205 { 1206 } 1207 1208 static int mlx5e_update_rep_rx(struct mlx5e_priv *priv) 1209 { 1210 return 0; 1211 } 1212 1213 static int mlx5e_rep_event_mpesw(struct mlx5e_priv *priv) 1214 { 1215 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1216 struct mlx5_eswitch_rep *rep = rpriv->rep; 1217 1218 if (rep->vport != MLX5_VPORT_UPLINK) 1219 return NOTIFY_DONE; 1220 1221 queue_work(priv->wq, &rpriv->uplink_priv.mpesw_work); 1222 1223 return NOTIFY_OK; 1224 } 1225 1226 static int uplink_rep_async_event(struct notifier_block *nb, unsigned long event, void *data) 1227 { 1228 struct mlx5e_priv *priv = container_of(nb, struct mlx5e_priv, events_nb); 1229 1230 if (event == MLX5_EVENT_TYPE_PORT_CHANGE) { 1231 struct mlx5_eqe *eqe = data; 1232 1233 switch (eqe->sub_type) { 1234 case MLX5_PORT_CHANGE_SUBTYPE_DOWN: 1235 case MLX5_PORT_CHANGE_SUBTYPE_ACTIVE: 1236 queue_work(priv->wq, &priv->update_carrier_work); 1237 break; 1238 default: 1239 return NOTIFY_DONE; 1240 } 1241 1242 return NOTIFY_OK; 1243 } 1244 1245 if (event == MLX5_DEV_EVENT_PORT_AFFINITY) 1246 return mlx5e_rep_tc_event_port_affinity(priv); 1247 else if (event == MLX5_DEV_EVENT_MULTIPORT_ESW) 1248 return mlx5e_rep_event_mpesw(priv); 1249 1250 return NOTIFY_DONE; 1251 } 1252 1253 static void mlx5e_uplink_rep_enable(struct mlx5e_priv *priv) 1254 { 1255 struct net_device *netdev = priv->netdev; 1256 struct mlx5_core_dev *mdev = priv->mdev; 1257 u16 max_mtu; 1258 1259 mlx5e_ipsec_init(priv); 1260 1261 netdev->min_mtu = ETH_MIN_MTU; 1262 mlx5_query_port_max_mtu(priv->mdev, &max_mtu, 1); 1263 netdev->max_mtu = MLX5E_HW2SW_MTU(&priv->channels.params, max_mtu); 1264 mlx5e_set_dev_port_mtu(priv); 1265 1266 mlx5e_rep_tc_enable(priv); 1267 1268 if (MLX5_CAP_GEN(mdev, uplink_follow)) 1269 mlx5_modify_vport_admin_state(mdev, MLX5_VPORT_STATE_OP_MOD_UPLINK, 1270 0, 0, MLX5_VPORT_ADMIN_STATE_AUTO); 1271 mlx5_lag_add_netdev(mdev, netdev); 1272 priv->events_nb.notifier_call = uplink_rep_async_event; 1273 mlx5_notifier_register(mdev, &priv->events_nb); 1274 mlx5e_dcbnl_initialize(priv); 1275 mlx5e_dcbnl_init_app(priv); 1276 mlx5e_rep_bridge_init(priv); 1277 1278 netdev->wanted_features |= NETIF_F_HW_TC; 1279 1280 rtnl_lock(); 1281 if (netif_running(netdev)) 1282 mlx5e_open(netdev); 1283 udp_tunnel_nic_reset_ntf(priv->netdev); 1284 netif_device_attach(netdev); 1285 rtnl_unlock(); 1286 } 1287 1288 static void mlx5e_uplink_rep_disable(struct mlx5e_priv *priv) 1289 { 1290 struct mlx5_core_dev *mdev = priv->mdev; 1291 1292 rtnl_lock(); 1293 if (netif_running(priv->netdev)) 1294 mlx5e_close(priv->netdev); 1295 netif_device_detach(priv->netdev); 1296 rtnl_unlock(); 1297 1298 mlx5e_rep_bridge_cleanup(priv); 1299 mlx5e_dcbnl_delete_app(priv); 1300 mlx5_notifier_unregister(mdev, &priv->events_nb); 1301 mlx5e_rep_tc_disable(priv); 1302 mlx5_lag_remove_netdev(mdev, priv->netdev); 1303 mlx5_vxlan_reset_to_default(mdev->vxlan); 1304 1305 mlx5e_ipsec_cleanup(priv); 1306 } 1307 1308 static MLX5E_DEFINE_STATS_GRP(sw_rep, 0); 1309 static MLX5E_DEFINE_STATS_GRP(vport_rep, MLX5E_NDO_UPDATE_STATS); 1310 1311 /* The stats groups order is opposite to the update_stats() order calls */ 1312 static mlx5e_stats_grp_t mlx5e_rep_stats_grps[] = { 1313 &MLX5E_STATS_GRP(sw_rep), 1314 &MLX5E_STATS_GRP(vport_rep), 1315 }; 1316 1317 static unsigned int mlx5e_rep_stats_grps_num(struct mlx5e_priv *priv) 1318 { 1319 return ARRAY_SIZE(mlx5e_rep_stats_grps); 1320 } 1321 1322 /* The stats groups order is opposite to the update_stats() order calls */ 1323 static mlx5e_stats_grp_t mlx5e_ul_rep_stats_grps[] = { 1324 &MLX5E_STATS_GRP(sw), 1325 &MLX5E_STATS_GRP(qcnt), 1326 &MLX5E_STATS_GRP(vnic_env), 1327 &MLX5E_STATS_GRP(vport), 1328 &MLX5E_STATS_GRP(802_3), 1329 &MLX5E_STATS_GRP(2863), 1330 &MLX5E_STATS_GRP(2819), 1331 &MLX5E_STATS_GRP(phy), 1332 &MLX5E_STATS_GRP(eth_ext), 1333 &MLX5E_STATS_GRP(pcie), 1334 &MLX5E_STATS_GRP(per_prio), 1335 &MLX5E_STATS_GRP(pme), 1336 &MLX5E_STATS_GRP(channels), 1337 &MLX5E_STATS_GRP(per_port_buff_congest), 1338 #ifdef CONFIG_MLX5_EN_IPSEC 1339 &MLX5E_STATS_GRP(ipsec_hw), 1340 &MLX5E_STATS_GRP(ipsec_sw), 1341 #endif 1342 &MLX5E_STATS_GRP(ptp), 1343 }; 1344 1345 static unsigned int mlx5e_ul_rep_stats_grps_num(struct mlx5e_priv *priv) 1346 { 1347 return ARRAY_SIZE(mlx5e_ul_rep_stats_grps); 1348 } 1349 1350 static int 1351 mlx5e_rep_vnic_reporter_diagnose(struct devlink_health_reporter *reporter, 1352 struct devlink_fmsg *fmsg, 1353 struct netlink_ext_ack *extack) 1354 { 1355 struct mlx5e_rep_priv *rpriv = devlink_health_reporter_priv(reporter); 1356 struct mlx5_eswitch_rep *rep = rpriv->rep; 1357 1358 return mlx5_reporter_vnic_diagnose_counters(rep->esw->dev, fmsg, 1359 rep->vport, true); 1360 } 1361 1362 static const struct devlink_health_reporter_ops mlx5_rep_vnic_reporter_ops = { 1363 .name = "vnic", 1364 .diagnose = mlx5e_rep_vnic_reporter_diagnose, 1365 }; 1366 1367 static void mlx5e_rep_vnic_reporter_create(struct mlx5e_priv *priv, 1368 struct devlink_port *dl_port) 1369 { 1370 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1371 struct devlink_health_reporter *reporter; 1372 1373 reporter = devl_port_health_reporter_create(dl_port, 1374 &mlx5_rep_vnic_reporter_ops, 1375 0, rpriv); 1376 if (IS_ERR(reporter)) { 1377 mlx5_core_err(priv->mdev, 1378 "Failed to create representor vnic reporter, err = %ld\n", 1379 PTR_ERR(reporter)); 1380 return; 1381 } 1382 1383 rpriv->rep_vnic_reporter = reporter; 1384 } 1385 1386 static void mlx5e_rep_vnic_reporter_destroy(struct mlx5e_priv *priv) 1387 { 1388 struct mlx5e_rep_priv *rpriv = priv->ppriv; 1389 1390 if (!IS_ERR_OR_NULL(rpriv->rep_vnic_reporter)) 1391 devl_health_reporter_destroy(rpriv->rep_vnic_reporter); 1392 } 1393 1394 static const struct mlx5e_profile mlx5e_rep_profile = { 1395 .init = mlx5e_init_rep, 1396 .cleanup = mlx5e_cleanup_rep, 1397 .init_rx = mlx5e_init_rep_rx, 1398 .cleanup_rx = mlx5e_cleanup_rep_rx, 1399 .init_tx = mlx5e_init_rep_tx, 1400 .cleanup_tx = mlx5e_cleanup_rep_tx, 1401 .enable = mlx5e_rep_enable, 1402 .disable = mlx5e_rep_disable, 1403 .update_rx = mlx5e_update_rep_rx, 1404 .update_stats = mlx5e_stats_update_ndo_stats, 1405 .rx_handlers = &mlx5e_rx_handlers_rep, 1406 .max_tc = 1, 1407 .stats_grps = mlx5e_rep_stats_grps, 1408 .stats_grps_num = mlx5e_rep_stats_grps_num, 1409 .max_nch_limit = mlx5e_rep_max_nch_limit, 1410 }; 1411 1412 static const struct mlx5e_profile mlx5e_uplink_rep_profile = { 1413 .init = mlx5e_init_ul_rep, 1414 .cleanup = mlx5e_cleanup_rep, 1415 .init_rx = mlx5e_init_ul_rep_rx, 1416 .cleanup_rx = mlx5e_cleanup_ul_rep_rx, 1417 .init_tx = mlx5e_init_rep_tx, 1418 .cleanup_tx = mlx5e_cleanup_rep_tx, 1419 .enable = mlx5e_uplink_rep_enable, 1420 .disable = mlx5e_uplink_rep_disable, 1421 .update_rx = mlx5e_update_rep_rx, 1422 .update_stats = mlx5e_stats_update_ndo_stats, 1423 .update_carrier = mlx5e_update_carrier, 1424 .rx_handlers = &mlx5e_rx_handlers_rep, 1425 .max_tc = MLX5E_MAX_NUM_TC, 1426 .stats_grps = mlx5e_ul_rep_stats_grps, 1427 .stats_grps_num = mlx5e_ul_rep_stats_grps_num, 1428 }; 1429 1430 /* e-Switch vport representors */ 1431 static int 1432 mlx5e_vport_uplink_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) 1433 { 1434 struct mlx5e_priv *priv = netdev_priv(mlx5_uplink_netdev_get(dev)); 1435 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); 1436 1437 rpriv->netdev = priv->netdev; 1438 return mlx5e_netdev_change_profile(priv, &mlx5e_uplink_rep_profile, 1439 rpriv); 1440 } 1441 1442 static void 1443 mlx5e_vport_uplink_rep_unload(struct mlx5e_rep_priv *rpriv) 1444 { 1445 struct net_device *netdev = rpriv->netdev; 1446 struct mlx5e_priv *priv; 1447 1448 priv = netdev_priv(netdev); 1449 1450 mlx5e_netdev_attach_nic_profile(priv); 1451 } 1452 1453 static int 1454 mlx5e_vport_vf_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) 1455 { 1456 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); 1457 const struct mlx5e_profile *profile; 1458 struct devlink_port *dl_port; 1459 struct net_device *netdev; 1460 struct mlx5e_priv *priv; 1461 int err; 1462 1463 profile = &mlx5e_rep_profile; 1464 netdev = mlx5e_create_netdev(dev, profile); 1465 if (!netdev) { 1466 mlx5_core_warn(dev, 1467 "Failed to create representor netdev for vport %d\n", 1468 rep->vport); 1469 return -EINVAL; 1470 } 1471 1472 mlx5e_build_rep_netdev(netdev, dev); 1473 rpriv->netdev = netdev; 1474 1475 priv = netdev_priv(netdev); 1476 priv->profile = profile; 1477 priv->ppriv = rpriv; 1478 err = profile->init(dev, netdev); 1479 if (err) { 1480 netdev_warn(netdev, "rep profile init failed, %d\n", err); 1481 goto err_destroy_netdev; 1482 } 1483 1484 err = mlx5e_attach_netdev(netdev_priv(netdev)); 1485 if (err) { 1486 netdev_warn(netdev, 1487 "Failed to attach representor netdev for vport %d\n", 1488 rep->vport); 1489 goto err_cleanup_profile; 1490 } 1491 1492 dl_port = mlx5_esw_offloads_devlink_port(dev->priv.eswitch, 1493 rpriv->rep->vport); 1494 if (dl_port) { 1495 SET_NETDEV_DEVLINK_PORT(netdev, dl_port); 1496 mlx5e_rep_vnic_reporter_create(priv, dl_port); 1497 } 1498 1499 err = register_netdev(netdev); 1500 if (err) { 1501 netdev_warn(netdev, 1502 "Failed to register representor netdev for vport %d\n", 1503 rep->vport); 1504 goto err_detach_netdev; 1505 } 1506 1507 return 0; 1508 1509 err_detach_netdev: 1510 mlx5e_rep_vnic_reporter_destroy(priv); 1511 mlx5e_detach_netdev(netdev_priv(netdev)); 1512 err_cleanup_profile: 1513 priv->profile->cleanup(priv); 1514 1515 err_destroy_netdev: 1516 mlx5e_destroy_netdev(netdev_priv(netdev)); 1517 return err; 1518 } 1519 1520 static int 1521 mlx5e_vport_rep_load(struct mlx5_core_dev *dev, struct mlx5_eswitch_rep *rep) 1522 { 1523 struct mlx5e_rep_priv *rpriv; 1524 int err; 1525 1526 rpriv = kvzalloc(sizeof(*rpriv), GFP_KERNEL); 1527 if (!rpriv) 1528 return -ENOMEM; 1529 1530 /* rpriv->rep to be looked up when profile->init() is called */ 1531 rpriv->rep = rep; 1532 rep->rep_data[REP_ETH].priv = rpriv; 1533 INIT_LIST_HEAD(&rpriv->vport_sqs_list); 1534 1535 if (rep->vport == MLX5_VPORT_UPLINK) 1536 err = mlx5e_vport_uplink_rep_load(dev, rep); 1537 else 1538 err = mlx5e_vport_vf_rep_load(dev, rep); 1539 1540 if (err) 1541 kvfree(rpriv); 1542 1543 return err; 1544 } 1545 1546 static void 1547 mlx5e_vport_rep_unload(struct mlx5_eswitch_rep *rep) 1548 { 1549 struct mlx5e_rep_priv *rpriv = mlx5e_rep_to_rep_priv(rep); 1550 struct net_device *netdev = rpriv->netdev; 1551 struct mlx5e_priv *priv = netdev_priv(netdev); 1552 void *ppriv = priv->ppriv; 1553 1554 if (rep->vport == MLX5_VPORT_UPLINK) { 1555 mlx5e_vport_uplink_rep_unload(rpriv); 1556 goto free_ppriv; 1557 } 1558 1559 unregister_netdev(netdev); 1560 mlx5e_rep_vnic_reporter_destroy(priv); 1561 mlx5e_detach_netdev(priv); 1562 priv->profile->cleanup(priv); 1563 mlx5e_destroy_netdev(priv); 1564 free_ppriv: 1565 kvfree(ppriv); /* mlx5e_rep_priv */ 1566 } 1567 1568 static void *mlx5e_vport_rep_get_proto_dev(struct mlx5_eswitch_rep *rep) 1569 { 1570 struct mlx5e_rep_priv *rpriv; 1571 1572 rpriv = mlx5e_rep_to_rep_priv(rep); 1573 1574 return rpriv->netdev; 1575 } 1576 1577 static void mlx5e_vport_rep_event_unpair(struct mlx5_eswitch_rep *rep, 1578 struct mlx5_eswitch *peer_esw) 1579 { 1580 u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 1581 struct mlx5e_rep_priv *rpriv; 1582 struct mlx5e_rep_sq *rep_sq; 1583 1584 WARN_ON_ONCE(!peer_esw); 1585 rpriv = mlx5e_rep_to_rep_priv(rep); 1586 list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) { 1587 struct mlx5e_rep_sq_peer *sq_peer = xa_load(&rep_sq->sq_peer, i); 1588 1589 if (!sq_peer || sq_peer->peer != peer_esw) 1590 continue; 1591 1592 mlx5_eswitch_del_send_to_vport_rule(sq_peer->rule); 1593 xa_erase(&rep_sq->sq_peer, i); 1594 kfree(sq_peer); 1595 } 1596 } 1597 1598 static int mlx5e_vport_rep_event_pair(struct mlx5_eswitch *esw, 1599 struct mlx5_eswitch_rep *rep, 1600 struct mlx5_eswitch *peer_esw) 1601 { 1602 u16 i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 1603 struct mlx5_flow_handle *flow_rule; 1604 struct mlx5e_rep_sq_peer *sq_peer; 1605 struct mlx5e_rep_priv *rpriv; 1606 struct mlx5e_rep_sq *rep_sq; 1607 int err; 1608 1609 rpriv = mlx5e_rep_to_rep_priv(rep); 1610 list_for_each_entry(rep_sq, &rpriv->vport_sqs_list, list) { 1611 sq_peer = xa_load(&rep_sq->sq_peer, i); 1612 1613 if (sq_peer && sq_peer->peer) 1614 continue; 1615 1616 flow_rule = mlx5_eswitch_add_send_to_vport_rule(peer_esw, esw, rep, 1617 rep_sq->sqn); 1618 if (IS_ERR(flow_rule)) { 1619 err = PTR_ERR(flow_rule); 1620 goto err_out; 1621 } 1622 1623 if (sq_peer) { 1624 sq_peer->rule = flow_rule; 1625 sq_peer->peer = peer_esw; 1626 continue; 1627 } 1628 sq_peer = kzalloc(sizeof(*sq_peer), GFP_KERNEL); 1629 if (!sq_peer) { 1630 err = -ENOMEM; 1631 goto err_sq_alloc; 1632 } 1633 err = xa_insert(&rep_sq->sq_peer, i, sq_peer, GFP_KERNEL); 1634 if (err) 1635 goto err_xa; 1636 sq_peer->rule = flow_rule; 1637 sq_peer->peer = peer_esw; 1638 } 1639 1640 return 0; 1641 err_xa: 1642 kfree(sq_peer); 1643 err_sq_alloc: 1644 mlx5_eswitch_del_send_to_vport_rule(flow_rule); 1645 err_out: 1646 mlx5e_vport_rep_event_unpair(rep, peer_esw); 1647 return err; 1648 } 1649 1650 static int mlx5e_vport_rep_event(struct mlx5_eswitch *esw, 1651 struct mlx5_eswitch_rep *rep, 1652 enum mlx5_switchdev_event event, 1653 void *data) 1654 { 1655 int err = 0; 1656 1657 if (event == MLX5_SWITCHDEV_EVENT_PAIR) 1658 err = mlx5e_vport_rep_event_pair(esw, rep, data); 1659 else if (event == MLX5_SWITCHDEV_EVENT_UNPAIR) 1660 mlx5e_vport_rep_event_unpair(rep, data); 1661 1662 return err; 1663 } 1664 1665 static const struct mlx5_eswitch_rep_ops rep_ops = { 1666 .load = mlx5e_vport_rep_load, 1667 .unload = mlx5e_vport_rep_unload, 1668 .get_proto_dev = mlx5e_vport_rep_get_proto_dev, 1669 .event = mlx5e_vport_rep_event, 1670 }; 1671 1672 static int mlx5e_rep_probe(struct auxiliary_device *adev, 1673 const struct auxiliary_device_id *id) 1674 { 1675 struct mlx5_adev *edev = container_of(adev, struct mlx5_adev, adev); 1676 struct mlx5_core_dev *mdev = edev->mdev; 1677 struct mlx5_eswitch *esw; 1678 1679 esw = mdev->priv.eswitch; 1680 mlx5_eswitch_register_vport_reps(esw, &rep_ops, REP_ETH); 1681 return 0; 1682 } 1683 1684 static void mlx5e_rep_remove(struct auxiliary_device *adev) 1685 { 1686 struct mlx5_adev *vdev = container_of(adev, struct mlx5_adev, adev); 1687 struct mlx5_core_dev *mdev = vdev->mdev; 1688 struct mlx5_eswitch *esw; 1689 1690 esw = mdev->priv.eswitch; 1691 mlx5_eswitch_unregister_vport_reps(esw, REP_ETH); 1692 } 1693 1694 static const struct auxiliary_device_id mlx5e_rep_id_table[] = { 1695 { .name = MLX5_ADEV_NAME ".eth-rep", }, 1696 {}, 1697 }; 1698 1699 MODULE_DEVICE_TABLE(auxiliary, mlx5e_rep_id_table); 1700 1701 static struct auxiliary_driver mlx5e_rep_driver = { 1702 .name = "eth-rep", 1703 .probe = mlx5e_rep_probe, 1704 .remove = mlx5e_rep_remove, 1705 .id_table = mlx5e_rep_id_table, 1706 }; 1707 1708 int mlx5e_rep_init(void) 1709 { 1710 return auxiliary_driver_register(&mlx5e_rep_driver); 1711 } 1712 1713 void mlx5e_rep_cleanup(void) 1714 { 1715 auxiliary_driver_unregister(&mlx5e_rep_driver); 1716 } 1717