1 /* 2 * Copyright (c) 2016, Mellanox Technologies. All rights reserved. 3 * 4 * This software is available to you under a choice of one of two 5 * licenses. You may choose to be licensed under the terms of the GNU 6 * General Public License (GPL) Version 2, available from the file 7 * COPYING in the main directory of this source tree, or the 8 * OpenIB.org BSD license below: 9 * 10 * Redistribution and use in source and binary forms, with or 11 * without modification, are permitted provided that the following 12 * conditions are met: 13 * 14 * - Redistributions of source code must retain the above 15 * copyright notice, this list of conditions and the following 16 * disclaimer. 17 * 18 * - Redistributions in binary form must reproduce the above 19 * copyright notice, this list of conditions and the following 20 * disclaimer in the documentation and/or other materials 21 * provided with the distribution. 22 * 23 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 24 * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 25 * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 26 * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS 27 * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN 28 * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN 29 * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 30 * SOFTWARE. 31 */ 32 33 #include <linux/etherdevice.h> 34 #include <linux/idr.h> 35 #include <linux/mlx5/driver.h> 36 #include <linux/mlx5/mlx5_ifc.h> 37 #include <linux/mlx5/vport.h> 38 #include <linux/mlx5/fs.h> 39 #include "mlx5_core.h" 40 #include "eswitch.h" 41 #include "esw/indir_table.h" 42 #include "esw/acl/ofld.h" 43 #include "rdma.h" 44 #include "en.h" 45 #include "fs_core.h" 46 #include "lib/devcom.h" 47 #include "lib/eq.h" 48 #include "lib/fs_chains.h" 49 #include "en_tc.h" 50 #include "en/mapping.h" 51 #include "devlink.h" 52 #include "lag/lag.h" 53 #include "en/tc/post_meter.h" 54 55 #define mlx5_esw_for_each_rep(esw, i, rep) \ 56 xa_for_each(&((esw)->offloads.vport_reps), i, rep) 57 58 /* There are two match-all miss flows, one for unicast dst mac and 59 * one for multicast. 60 */ 61 #define MLX5_ESW_MISS_FLOWS (2) 62 #define UPLINK_REP_INDEX 0 63 64 #define MLX5_ESW_VPORT_TBL_SIZE 128 65 #define MLX5_ESW_VPORT_TBL_NUM_GROUPS 4 66 67 #define MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 68 69 static struct esw_vport_tbl_namespace mlx5_esw_vport_tbl_mirror_ns = { 70 .max_fte = MLX5_ESW_VPORT_TBL_SIZE, 71 .max_num_groups = MLX5_ESW_VPORT_TBL_NUM_GROUPS, 72 .flags = 0, 73 }; 74 75 static struct mlx5_eswitch_rep *mlx5_eswitch_get_rep(struct mlx5_eswitch *esw, 76 u16 vport_num) 77 { 78 return xa_load(&esw->offloads.vport_reps, vport_num); 79 } 80 81 static void 82 mlx5_eswitch_set_rule_flow_source(struct mlx5_eswitch *esw, 83 struct mlx5_flow_spec *spec, 84 struct mlx5_esw_flow_attr *attr) 85 { 86 if (!MLX5_CAP_ESW_FLOWTABLE(esw->dev, flow_source) || !attr || !attr->in_rep) 87 return; 88 89 if (attr->int_port) { 90 spec->flow_context.flow_source = mlx5e_tc_int_port_get_flow_source(attr->int_port); 91 92 return; 93 } 94 95 spec->flow_context.flow_source = (attr->in_rep->vport == MLX5_VPORT_UPLINK) ? 96 MLX5_FLOW_CONTEXT_FLOW_SOURCE_UPLINK : 97 MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 98 } 99 100 /* Actually only the upper 16 bits of reg c0 need to be cleared, but the lower 16 bits 101 * are not needed as well in the following process. So clear them all for simplicity. 102 */ 103 void 104 mlx5_eswitch_clear_rule_source_port(struct mlx5_eswitch *esw, struct mlx5_flow_spec *spec) 105 { 106 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 107 void *misc2; 108 109 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 110 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 111 112 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 113 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 0); 114 115 if (!memchr_inv(misc2, 0, MLX5_ST_SZ_BYTES(fte_match_set_misc2))) 116 spec->match_criteria_enable &= ~MLX5_MATCH_MISC_PARAMETERS_2; 117 } 118 } 119 120 static void 121 mlx5_eswitch_set_rule_source_port(struct mlx5_eswitch *esw, 122 struct mlx5_flow_spec *spec, 123 struct mlx5_flow_attr *attr, 124 struct mlx5_eswitch *src_esw, 125 u16 vport) 126 { 127 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 128 u32 metadata; 129 void *misc2; 130 void *misc; 131 132 /* Use metadata matching because vport is not represented by single 133 * VHCA in dual-port RoCE mode, and matching on source vport may fail. 134 */ 135 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 136 if (mlx5_esw_indir_table_decap_vport(attr)) 137 vport = mlx5_esw_indir_table_decap_vport(attr); 138 139 if (!attr->chain && esw_attr && esw_attr->int_port) 140 metadata = 141 mlx5e_tc_int_port_get_metadata_for_match(esw_attr->int_port); 142 else 143 metadata = 144 mlx5_eswitch_get_vport_metadata_for_match(src_esw, vport); 145 146 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 147 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, metadata); 148 149 misc2 = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 150 MLX5_SET(fte_match_set_misc2, misc2, metadata_reg_c_0, 151 mlx5_eswitch_get_vport_metadata_mask()); 152 153 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 154 } else { 155 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 156 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 157 158 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 159 MLX5_SET(fte_match_set_misc, misc, 160 source_eswitch_owner_vhca_id, 161 MLX5_CAP_GEN(src_esw->dev, vhca_id)); 162 163 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 164 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 165 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 166 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 167 source_eswitch_owner_vhca_id); 168 169 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 170 } 171 } 172 173 static int 174 esw_setup_decap_indir(struct mlx5_eswitch *esw, 175 struct mlx5_flow_attr *attr) 176 { 177 struct mlx5_flow_table *ft; 178 179 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 180 return -EOPNOTSUPP; 181 182 ft = mlx5_esw_indir_table_get(esw, attr, 183 mlx5_esw_indir_table_decap_vport(attr), true); 184 return PTR_ERR_OR_ZERO(ft); 185 } 186 187 static void 188 esw_cleanup_decap_indir(struct mlx5_eswitch *esw, 189 struct mlx5_flow_attr *attr) 190 { 191 if (mlx5_esw_indir_table_decap_vport(attr)) 192 mlx5_esw_indir_table_put(esw, 193 mlx5_esw_indir_table_decap_vport(attr), 194 true); 195 } 196 197 static int 198 esw_setup_mtu_dest(struct mlx5_flow_destination *dest, 199 struct mlx5e_meter_attr *meter, 200 int i) 201 { 202 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_RANGE; 203 dest[i].range.field = MLX5_FLOW_DEST_RANGE_FIELD_PKT_LEN; 204 dest[i].range.min = 0; 205 dest[i].range.max = meter->params.mtu; 206 dest[i].range.hit_ft = mlx5e_post_meter_get_mtu_true_ft(meter->post_meter); 207 dest[i].range.miss_ft = mlx5e_post_meter_get_mtu_false_ft(meter->post_meter); 208 209 return 0; 210 } 211 212 static int 213 esw_setup_sampler_dest(struct mlx5_flow_destination *dest, 214 struct mlx5_flow_act *flow_act, 215 u32 sampler_id, 216 int i) 217 { 218 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 219 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_SAMPLER; 220 dest[i].sampler_id = sampler_id; 221 222 return 0; 223 } 224 225 static int 226 esw_setup_ft_dest(struct mlx5_flow_destination *dest, 227 struct mlx5_flow_act *flow_act, 228 struct mlx5_eswitch *esw, 229 struct mlx5_flow_attr *attr, 230 int i) 231 { 232 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 233 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 234 dest[i].ft = attr->dest_ft; 235 236 if (mlx5_esw_indir_table_decap_vport(attr)) 237 return esw_setup_decap_indir(esw, attr); 238 return 0; 239 } 240 241 static void 242 esw_setup_accept_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 243 struct mlx5_fs_chains *chains, int i) 244 { 245 if (mlx5_chains_ignore_flow_level_supported(chains)) 246 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 247 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 248 dest[i].ft = mlx5_chains_get_tc_end_ft(chains); 249 } 250 251 static void 252 esw_setup_slow_path_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 253 struct mlx5_eswitch *esw, int i) 254 { 255 if (MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level)) 256 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 257 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 258 dest[i].ft = mlx5_eswitch_get_slow_fdb(esw); 259 } 260 261 static int 262 esw_setup_chain_dest(struct mlx5_flow_destination *dest, 263 struct mlx5_flow_act *flow_act, 264 struct mlx5_fs_chains *chains, 265 u32 chain, u32 prio, u32 level, 266 int i) 267 { 268 struct mlx5_flow_table *ft; 269 270 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 271 ft = mlx5_chains_get_table(chains, chain, prio, level); 272 if (IS_ERR(ft)) 273 return PTR_ERR(ft); 274 275 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 276 dest[i].ft = ft; 277 return 0; 278 } 279 280 static void esw_put_dest_tables_loop(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr, 281 int from, int to) 282 { 283 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 284 struct mlx5_fs_chains *chains = esw_chains(esw); 285 int i; 286 287 for (i = from; i < to; i++) 288 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 289 mlx5_chains_put_table(chains, 0, 1, 0); 290 else if (mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 291 esw_attr->dests[i].mdev)) 292 mlx5_esw_indir_table_put(esw, esw_attr->dests[i].vport, false); 293 } 294 295 static bool 296 esw_is_chain_src_port_rewrite(struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr) 297 { 298 int i; 299 300 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 301 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 302 return true; 303 return false; 304 } 305 306 static int 307 esw_setup_chain_src_port_rewrite(struct mlx5_flow_destination *dest, 308 struct mlx5_flow_act *flow_act, 309 struct mlx5_eswitch *esw, 310 struct mlx5_fs_chains *chains, 311 struct mlx5_flow_attr *attr, 312 int *i) 313 { 314 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 315 int err; 316 317 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 318 return -EOPNOTSUPP; 319 320 /* flow steering cannot handle more than one dest with the same ft 321 * in a single flow 322 */ 323 if (esw_attr->out_count - esw_attr->split_count > 1) 324 return -EOPNOTSUPP; 325 326 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 1, 0, *i); 327 if (err) 328 return err; 329 330 if (esw_attr->dests[esw_attr->split_count].pkt_reformat) { 331 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 332 flow_act->pkt_reformat = esw_attr->dests[esw_attr->split_count].pkt_reformat; 333 } 334 (*i)++; 335 336 return 0; 337 } 338 339 static void esw_cleanup_chain_src_port_rewrite(struct mlx5_eswitch *esw, 340 struct mlx5_flow_attr *attr) 341 { 342 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 343 344 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 345 } 346 347 static bool 348 esw_is_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 349 { 350 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 351 bool result = false; 352 int i; 353 354 /* Indirect table is supported only for flows with in_port uplink 355 * and the destination is vport on the same eswitch as the uplink, 356 * return false in case at least one of destinations doesn't meet 357 * this criteria. 358 */ 359 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) { 360 if (esw_attr->dests[i].vport_valid && 361 mlx5_esw_indir_table_needed(esw, attr, esw_attr->dests[i].vport, 362 esw_attr->dests[i].mdev)) { 363 result = true; 364 } else { 365 result = false; 366 break; 367 } 368 } 369 return result; 370 } 371 372 static int 373 esw_setup_indir_table(struct mlx5_flow_destination *dest, 374 struct mlx5_flow_act *flow_act, 375 struct mlx5_eswitch *esw, 376 struct mlx5_flow_attr *attr, 377 int *i) 378 { 379 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 380 int j, err; 381 382 if (!(attr->flags & MLX5_ATTR_FLAG_SRC_REWRITE)) 383 return -EOPNOTSUPP; 384 385 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, (*i)++) { 386 flow_act->flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 387 dest[*i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 388 389 dest[*i].ft = mlx5_esw_indir_table_get(esw, attr, 390 esw_attr->dests[j].vport, false); 391 if (IS_ERR(dest[*i].ft)) { 392 err = PTR_ERR(dest[*i].ft); 393 goto err_indir_tbl_get; 394 } 395 } 396 397 if (mlx5_esw_indir_table_decap_vport(attr)) { 398 err = esw_setup_decap_indir(esw, attr); 399 if (err) 400 goto err_indir_tbl_get; 401 } 402 403 return 0; 404 405 err_indir_tbl_get: 406 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, j); 407 return err; 408 } 409 410 static void esw_cleanup_indir_table(struct mlx5_eswitch *esw, struct mlx5_flow_attr *attr) 411 { 412 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 413 414 esw_put_dest_tables_loop(esw, attr, esw_attr->split_count, esw_attr->out_count); 415 esw_cleanup_decap_indir(esw, attr); 416 } 417 418 static void 419 esw_cleanup_chain_dest(struct mlx5_fs_chains *chains, u32 chain, u32 prio, u32 level) 420 { 421 mlx5_chains_put_table(chains, chain, prio, level); 422 } 423 424 static bool esw_same_vhca_id(struct mlx5_core_dev *mdev1, struct mlx5_core_dev *mdev2) 425 { 426 return MLX5_CAP_GEN(mdev1, vhca_id) == MLX5_CAP_GEN(mdev2, vhca_id); 427 } 428 429 static bool esw_setup_uplink_fwd_ipsec_needed(struct mlx5_eswitch *esw, 430 struct mlx5_esw_flow_attr *esw_attr, 431 int attr_idx) 432 { 433 if (esw->offloads.ft_ipsec_tx_pol && 434 esw_attr->dests[attr_idx].vport_valid && 435 esw_attr->dests[attr_idx].vport == MLX5_VPORT_UPLINK && 436 /* To be aligned with software, encryption is needed only for tunnel device */ 437 (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) && 438 esw_attr->dests[attr_idx].vport != esw_attr->in_rep->vport && 439 esw_same_vhca_id(esw_attr->dests[attr_idx].mdev, esw->dev)) 440 return true; 441 442 return false; 443 } 444 445 static bool esw_flow_dests_fwd_ipsec_check(struct mlx5_eswitch *esw, 446 struct mlx5_esw_flow_attr *esw_attr) 447 { 448 int i; 449 450 if (!esw->offloads.ft_ipsec_tx_pol) 451 return true; 452 453 for (i = 0; i < esw_attr->split_count; i++) 454 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i)) 455 return false; 456 457 for (i = esw_attr->split_count; i < esw_attr->out_count; i++) 458 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, i) && 459 (esw_attr->out_count - esw_attr->split_count > 1)) 460 return false; 461 462 return true; 463 } 464 465 static void 466 esw_setup_dest_fwd_vport(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 467 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 468 int attr_idx, int dest_idx, bool pkt_reformat) 469 { 470 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 471 dest[dest_idx].vport.num = esw_attr->dests[attr_idx].vport; 472 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 473 dest[dest_idx].vport.vhca_id = 474 MLX5_CAP_GEN(esw_attr->dests[attr_idx].mdev, vhca_id); 475 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 476 if (dest[dest_idx].vport.num == MLX5_VPORT_UPLINK && 477 mlx5_lag_is_mpesw(esw->dev)) 478 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_UPLINK; 479 } 480 if (esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 481 if (pkt_reformat) { 482 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 483 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 484 } 485 dest[dest_idx].vport.flags |= MLX5_FLOW_DEST_VPORT_REFORMAT_ID; 486 dest[dest_idx].vport.pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 487 } 488 } 489 490 static void 491 esw_setup_dest_fwd_ipsec(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 492 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 493 int attr_idx, int dest_idx, bool pkt_reformat) 494 { 495 dest[dest_idx].ft = esw->offloads.ft_ipsec_tx_pol; 496 dest[dest_idx].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 497 if (pkt_reformat && 498 esw_attr->dests[attr_idx].flags & MLX5_ESW_DEST_ENCAP_VALID) { 499 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 500 flow_act->pkt_reformat = esw_attr->dests[attr_idx].pkt_reformat; 501 } 502 } 503 504 static void 505 esw_setup_vport_dest(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 506 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 507 int attr_idx, int dest_idx, bool pkt_reformat) 508 { 509 if (esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 510 esw_setup_dest_fwd_ipsec(dest, flow_act, esw, esw_attr, 511 attr_idx, dest_idx, pkt_reformat); 512 else 513 esw_setup_dest_fwd_vport(dest, flow_act, esw, esw_attr, 514 attr_idx, dest_idx, pkt_reformat); 515 } 516 517 static int 518 esw_setup_vport_dests(struct mlx5_flow_destination *dest, struct mlx5_flow_act *flow_act, 519 struct mlx5_eswitch *esw, struct mlx5_esw_flow_attr *esw_attr, 520 int i) 521 { 522 int j; 523 524 for (j = esw_attr->split_count; j < esw_attr->out_count; j++, i++) 525 esw_setup_vport_dest(dest, flow_act, esw, esw_attr, j, i, true); 526 return i; 527 } 528 529 static bool 530 esw_src_port_rewrite_supported(struct mlx5_eswitch *esw) 531 { 532 return MLX5_CAP_GEN(esw->dev, reg_c_preserve) && 533 mlx5_eswitch_vport_match_metadata_enabled(esw) && 534 MLX5_CAP_ESW_FLOWTABLE_FDB(esw->dev, ignore_flow_level); 535 } 536 537 static bool 538 esw_dests_to_vf_pf_vports(struct mlx5_flow_destination *dests, int max_dest) 539 { 540 bool vf_dest = false, pf_dest = false; 541 int i; 542 543 for (i = 0; i < max_dest; i++) { 544 if (dests[i].type != MLX5_FLOW_DESTINATION_TYPE_VPORT) 545 continue; 546 547 if (dests[i].vport.num == MLX5_VPORT_UPLINK) 548 pf_dest = true; 549 else 550 vf_dest = true; 551 552 if (vf_dest && pf_dest) 553 return true; 554 } 555 556 return false; 557 } 558 559 static int 560 esw_setup_dests(struct mlx5_flow_destination *dest, 561 struct mlx5_flow_act *flow_act, 562 struct mlx5_eswitch *esw, 563 struct mlx5_flow_attr *attr, 564 struct mlx5_flow_spec *spec, 565 int *i) 566 { 567 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 568 struct mlx5_fs_chains *chains = esw_chains(esw); 569 int err = 0; 570 571 if (!mlx5_eswitch_termtbl_required(esw, attr, flow_act, spec) && 572 esw_src_port_rewrite_supported(esw)) 573 attr->flags |= MLX5_ATTR_FLAG_SRC_REWRITE; 574 575 if (attr->flags & MLX5_ATTR_FLAG_SLOW_PATH) { 576 esw_setup_slow_path_dest(dest, flow_act, esw, *i); 577 (*i)++; 578 goto out; 579 } 580 581 if (attr->flags & MLX5_ATTR_FLAG_SAMPLE) { 582 esw_setup_sampler_dest(dest, flow_act, attr->sample_attr.sampler_id, *i); 583 (*i)++; 584 } else if (attr->flags & MLX5_ATTR_FLAG_ACCEPT) { 585 esw_setup_accept_dest(dest, flow_act, chains, *i); 586 (*i)++; 587 } else if (attr->flags & MLX5_ATTR_FLAG_MTU) { 588 err = esw_setup_mtu_dest(dest, &attr->meter_attr, *i); 589 (*i)++; 590 } else if (esw_is_indir_table(esw, attr)) { 591 err = esw_setup_indir_table(dest, flow_act, esw, attr, i); 592 } else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) { 593 err = esw_setup_chain_src_port_rewrite(dest, flow_act, esw, chains, attr, i); 594 } else { 595 *i = esw_setup_vport_dests(dest, flow_act, esw, esw_attr, *i); 596 597 if (attr->dest_ft) { 598 err = esw_setup_ft_dest(dest, flow_act, esw, attr, *i); 599 (*i)++; 600 } else if (attr->dest_chain) { 601 err = esw_setup_chain_dest(dest, flow_act, chains, attr->dest_chain, 602 1, 0, *i); 603 (*i)++; 604 } 605 } 606 607 out: 608 return err; 609 } 610 611 static void 612 esw_cleanup_dests(struct mlx5_eswitch *esw, 613 struct mlx5_flow_attr *attr) 614 { 615 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 616 struct mlx5_fs_chains *chains = esw_chains(esw); 617 618 if (attr->dest_ft) { 619 esw_cleanup_decap_indir(esw, attr); 620 } else if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 621 if (attr->dest_chain) 622 esw_cleanup_chain_dest(chains, attr->dest_chain, 1, 0); 623 else if (esw_is_indir_table(esw, attr)) 624 esw_cleanup_indir_table(esw, attr); 625 else if (esw_is_chain_src_port_rewrite(esw, esw_attr)) 626 esw_cleanup_chain_src_port_rewrite(esw, attr); 627 } 628 } 629 630 static void 631 esw_setup_meter(struct mlx5_flow_attr *attr, struct mlx5_flow_act *flow_act) 632 { 633 struct mlx5e_flow_meter_handle *meter; 634 635 meter = attr->meter_attr.meter; 636 flow_act->exe_aso.type = attr->exe_aso_type; 637 flow_act->exe_aso.object_id = meter->obj_id; 638 flow_act->exe_aso.flow_meter.meter_idx = meter->idx; 639 flow_act->exe_aso.flow_meter.init_color = MLX5_FLOW_METER_COLOR_GREEN; 640 /* use metadata reg 5 for packet color */ 641 flow_act->exe_aso.return_reg_id = 5; 642 } 643 644 struct mlx5_flow_handle * 645 mlx5_eswitch_add_offloaded_rule(struct mlx5_eswitch *esw, 646 struct mlx5_flow_spec *spec, 647 struct mlx5_flow_attr *attr) 648 { 649 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 650 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 651 struct mlx5_fs_chains *chains = esw_chains(esw); 652 bool split = !!(esw_attr->split_count); 653 struct mlx5_vport_tbl_attr fwd_attr; 654 struct mlx5_flow_destination *dest; 655 struct mlx5_flow_handle *rule; 656 struct mlx5_flow_table *fdb; 657 int i = 0; 658 659 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 660 return ERR_PTR(-EOPNOTSUPP); 661 662 if (!mlx5_eswitch_vlan_actions_supported(esw->dev, 1)) 663 return ERR_PTR(-EOPNOTSUPP); 664 665 if (!esw_flow_dests_fwd_ipsec_check(esw, esw_attr)) 666 return ERR_PTR(-EOPNOTSUPP); 667 668 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 669 if (!dest) 670 return ERR_PTR(-ENOMEM); 671 672 flow_act.action = attr->action; 673 674 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH) { 675 flow_act.vlan[0].ethtype = ntohs(esw_attr->vlan_proto[0]); 676 flow_act.vlan[0].vid = esw_attr->vlan_vid[0]; 677 flow_act.vlan[0].prio = esw_attr->vlan_prio[0]; 678 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_VLAN_PUSH_2) { 679 flow_act.vlan[1].ethtype = ntohs(esw_attr->vlan_proto[1]); 680 flow_act.vlan[1].vid = esw_attr->vlan_vid[1]; 681 flow_act.vlan[1].prio = esw_attr->vlan_prio[1]; 682 } 683 } 684 685 mlx5_eswitch_set_rule_flow_source(esw, spec, esw_attr); 686 687 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_FWD_DEST) { 688 int err; 689 690 err = esw_setup_dests(dest, &flow_act, esw, attr, spec, &i); 691 if (err) { 692 rule = ERR_PTR(err); 693 goto err_create_goto_table; 694 } 695 696 /* Header rewrite with combined wire+loopback in FDB is not allowed */ 697 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) && 698 esw_dests_to_vf_pf_vports(dest, i)) { 699 esw_warn(esw->dev, 700 "FDB: Header rewrite with forwarding to both PF and VF is not allowed\n"); 701 rule = ERR_PTR(-EINVAL); 702 goto err_esw_get; 703 } 704 } 705 706 if (esw_attr->decap_pkt_reformat) 707 flow_act.pkt_reformat = esw_attr->decap_pkt_reformat; 708 709 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_COUNT) { 710 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 711 dest[i].counter_id = mlx5_fc_id(attr->counter); 712 i++; 713 } 714 715 if (attr->outer_match_level != MLX5_MATCH_NONE) 716 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 717 if (attr->inner_match_level != MLX5_MATCH_NONE) 718 spec->match_criteria_enable |= MLX5_MATCH_INNER_HEADERS; 719 720 if (flow_act.action & MLX5_FLOW_CONTEXT_ACTION_MOD_HDR) 721 flow_act.modify_hdr = attr->modify_hdr; 722 723 if ((flow_act.action & MLX5_FLOW_CONTEXT_ACTION_EXECUTE_ASO) && 724 attr->exe_aso_type == MLX5_EXE_ASO_FLOW_METER) 725 esw_setup_meter(attr, &flow_act); 726 727 if (split) { 728 fwd_attr.chain = attr->chain; 729 fwd_attr.prio = attr->prio; 730 fwd_attr.vport = esw_attr->in_rep->vport; 731 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 732 733 fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 734 } else { 735 if (attr->chain || attr->prio) 736 fdb = mlx5_chains_get_table(chains, attr->chain, 737 attr->prio, 0); 738 else 739 fdb = attr->ft; 740 741 if (!(attr->flags & MLX5_ATTR_FLAG_NO_IN_PORT)) 742 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 743 esw_attr->in_mdev->priv.eswitch, 744 esw_attr->in_rep->vport); 745 } 746 if (IS_ERR(fdb)) { 747 rule = ERR_CAST(fdb); 748 goto err_esw_get; 749 } 750 751 if (!i) { 752 kfree(dest); 753 dest = NULL; 754 } 755 756 if (mlx5_eswitch_termtbl_required(esw, attr, &flow_act, spec)) 757 rule = mlx5_eswitch_add_termtbl_rule(esw, fdb, spec, esw_attr, 758 &flow_act, dest, i); 759 else 760 rule = mlx5_add_flow_rules(fdb, spec, &flow_act, dest, i); 761 if (IS_ERR(rule)) 762 goto err_add_rule; 763 else 764 atomic64_inc(&esw->offloads.num_flows); 765 766 kfree(dest); 767 return rule; 768 769 err_add_rule: 770 if (split) 771 mlx5_esw_vporttbl_put(esw, &fwd_attr); 772 else if (attr->chain || attr->prio) 773 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 774 err_esw_get: 775 esw_cleanup_dests(esw, attr); 776 err_create_goto_table: 777 kfree(dest); 778 return rule; 779 } 780 781 struct mlx5_flow_handle * 782 mlx5_eswitch_add_fwd_rule(struct mlx5_eswitch *esw, 783 struct mlx5_flow_spec *spec, 784 struct mlx5_flow_attr *attr) 785 { 786 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 787 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 788 struct mlx5_fs_chains *chains = esw_chains(esw); 789 struct mlx5_vport_tbl_attr fwd_attr; 790 struct mlx5_flow_destination *dest; 791 struct mlx5_flow_table *fast_fdb; 792 struct mlx5_flow_table *fwd_fdb; 793 struct mlx5_flow_handle *rule; 794 int i, err = 0; 795 796 dest = kcalloc(MLX5_MAX_FLOW_FWD_VPORTS + 1, sizeof(*dest), GFP_KERNEL); 797 if (!dest) 798 return ERR_PTR(-ENOMEM); 799 800 fast_fdb = mlx5_chains_get_table(chains, attr->chain, attr->prio, 0); 801 if (IS_ERR(fast_fdb)) { 802 rule = ERR_CAST(fast_fdb); 803 goto err_get_fast; 804 } 805 806 fwd_attr.chain = attr->chain; 807 fwd_attr.prio = attr->prio; 808 fwd_attr.vport = esw_attr->in_rep->vport; 809 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 810 fwd_fdb = mlx5_esw_vporttbl_get(esw, &fwd_attr); 811 if (IS_ERR(fwd_fdb)) { 812 rule = ERR_CAST(fwd_fdb); 813 goto err_get_fwd; 814 } 815 816 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 817 for (i = 0; i < esw_attr->split_count; i++) { 818 if (esw_attr->dests[i].flags & MLX5_ESW_DEST_CHAIN_WITH_SRC_PORT_CHANGE) 819 /* Source port rewrite (forward to ovs internal port or statck device) isn't 820 * supported in the rule of split action. 821 */ 822 err = -EOPNOTSUPP; 823 else 824 esw_setup_vport_dest(dest, &flow_act, esw, esw_attr, i, i, false); 825 826 if (err) { 827 rule = ERR_PTR(err); 828 goto err_chain_src_rewrite; 829 } 830 } 831 dest[i].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 832 dest[i].ft = fwd_fdb; 833 i++; 834 835 mlx5_eswitch_set_rule_source_port(esw, spec, attr, 836 esw_attr->in_mdev->priv.eswitch, 837 esw_attr->in_rep->vport); 838 839 if (attr->outer_match_level != MLX5_MATCH_NONE) 840 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 841 842 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 843 rule = mlx5_add_flow_rules(fast_fdb, spec, &flow_act, dest, i); 844 845 if (IS_ERR(rule)) { 846 i = esw_attr->split_count; 847 goto err_chain_src_rewrite; 848 } 849 850 atomic64_inc(&esw->offloads.num_flows); 851 852 kfree(dest); 853 return rule; 854 err_chain_src_rewrite: 855 mlx5_esw_vporttbl_put(esw, &fwd_attr); 856 err_get_fwd: 857 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 858 err_get_fast: 859 kfree(dest); 860 return rule; 861 } 862 863 static void 864 __mlx5_eswitch_del_rule(struct mlx5_eswitch *esw, 865 struct mlx5_flow_handle *rule, 866 struct mlx5_flow_attr *attr, 867 bool fwd_rule) 868 { 869 struct mlx5_esw_flow_attr *esw_attr = attr->esw_attr; 870 struct mlx5_fs_chains *chains = esw_chains(esw); 871 bool split = (esw_attr->split_count > 0); 872 struct mlx5_vport_tbl_attr fwd_attr; 873 int i; 874 875 mlx5_del_flow_rules(rule); 876 877 if (!mlx5e_tc_attr_flags_skip(attr->flags)) { 878 /* unref the term table */ 879 for (i = 0; i < MLX5_MAX_FLOW_FWD_VPORTS; i++) { 880 if (esw_attr->dests[i].termtbl) 881 mlx5_eswitch_termtbl_put(esw, esw_attr->dests[i].termtbl); 882 } 883 } 884 885 atomic64_dec(&esw->offloads.num_flows); 886 887 if (fwd_rule || split) { 888 fwd_attr.chain = attr->chain; 889 fwd_attr.prio = attr->prio; 890 fwd_attr.vport = esw_attr->in_rep->vport; 891 fwd_attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 892 } 893 894 if (fwd_rule) { 895 mlx5_esw_vporttbl_put(esw, &fwd_attr); 896 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 897 } else { 898 if (split) 899 mlx5_esw_vporttbl_put(esw, &fwd_attr); 900 else if (attr->chain || attr->prio) 901 mlx5_chains_put_table(chains, attr->chain, attr->prio, 0); 902 esw_cleanup_dests(esw, attr); 903 } 904 } 905 906 void 907 mlx5_eswitch_del_offloaded_rule(struct mlx5_eswitch *esw, 908 struct mlx5_flow_handle *rule, 909 struct mlx5_flow_attr *attr) 910 { 911 __mlx5_eswitch_del_rule(esw, rule, attr, false); 912 } 913 914 void 915 mlx5_eswitch_del_fwd_rule(struct mlx5_eswitch *esw, 916 struct mlx5_flow_handle *rule, 917 struct mlx5_flow_attr *attr) 918 { 919 __mlx5_eswitch_del_rule(esw, rule, attr, true); 920 } 921 922 struct mlx5_flow_handle * 923 mlx5_eswitch_add_send_to_vport_rule(struct mlx5_eswitch *on_esw, 924 struct mlx5_eswitch *from_esw, 925 struct mlx5_eswitch_rep *rep, 926 u32 sqn) 927 { 928 struct mlx5_flow_act flow_act = {0}; 929 struct mlx5_flow_destination dest = {}; 930 struct mlx5_flow_handle *flow_rule; 931 struct mlx5_flow_spec *spec; 932 void *misc; 933 u16 vport; 934 935 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 936 if (!spec) { 937 flow_rule = ERR_PTR(-ENOMEM); 938 goto out; 939 } 940 941 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 942 MLX5_SET(fte_match_set_misc, misc, source_sqn, sqn); 943 944 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 945 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_sqn); 946 947 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 948 949 /* source vport is the esw manager */ 950 vport = from_esw->manager_vport; 951 952 if (mlx5_eswitch_vport_match_metadata_enabled(on_esw)) { 953 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 954 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 955 mlx5_eswitch_get_vport_metadata_for_match(from_esw, vport)); 956 957 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 958 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 959 mlx5_eswitch_get_vport_metadata_mask()); 960 961 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 962 } else { 963 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 964 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 965 966 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 967 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 968 MLX5_CAP_GEN(from_esw->dev, vhca_id)); 969 970 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 971 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 972 973 if (MLX5_CAP_ESW(on_esw->dev, merged_eswitch)) 974 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 975 source_eswitch_owner_vhca_id); 976 977 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 978 } 979 980 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 981 dest.vport.num = rep->vport; 982 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 983 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 984 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 985 986 if (rep->vport == MLX5_VPORT_UPLINK && 987 on_esw == from_esw && on_esw->offloads.ft_ipsec_tx_pol) { 988 dest.ft = on_esw->offloads.ft_ipsec_tx_pol; 989 flow_act.flags = FLOW_ACT_IGNORE_FLOW_LEVEL; 990 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 991 } else { 992 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 993 dest.vport.num = rep->vport; 994 dest.vport.vhca_id = MLX5_CAP_GEN(rep->esw->dev, vhca_id); 995 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 996 } 997 998 if (MLX5_CAP_ESW_FLOWTABLE(on_esw->dev, flow_source) && 999 rep->vport == MLX5_VPORT_UPLINK) 1000 spec->flow_context.flow_source = MLX5_FLOW_CONTEXT_FLOW_SOURCE_LOCAL_VPORT; 1001 1002 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(on_esw), 1003 spec, &flow_act, &dest, 1); 1004 if (IS_ERR(flow_rule)) 1005 esw_warn(on_esw->dev, "FDB: Failed to add send to vport rule err %ld\n", 1006 PTR_ERR(flow_rule)); 1007 out: 1008 kvfree(spec); 1009 return flow_rule; 1010 } 1011 EXPORT_SYMBOL(mlx5_eswitch_add_send_to_vport_rule); 1012 1013 void mlx5_eswitch_del_send_to_vport_rule(struct mlx5_flow_handle *rule) 1014 { 1015 mlx5_del_flow_rules(rule); 1016 } 1017 1018 void mlx5_eswitch_del_send_to_vport_meta_rule(struct mlx5_flow_handle *rule) 1019 { 1020 if (rule) 1021 mlx5_del_flow_rules(rule); 1022 } 1023 1024 struct mlx5_flow_handle * 1025 mlx5_eswitch_add_send_to_vport_meta_rule(struct mlx5_eswitch *esw, u16 vport_num) 1026 { 1027 struct mlx5_flow_destination dest = {}; 1028 struct mlx5_flow_act flow_act = {0}; 1029 struct mlx5_flow_handle *flow_rule; 1030 struct mlx5_flow_spec *spec; 1031 1032 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1033 if (!spec) 1034 return ERR_PTR(-ENOMEM); 1035 1036 MLX5_SET(fte_match_param, spec->match_criteria, 1037 misc_parameters_2.metadata_reg_c_0, mlx5_eswitch_get_vport_metadata_mask()); 1038 MLX5_SET(fte_match_param, spec->match_criteria, 1039 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1040 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_1, 1041 ESW_TUN_SLOW_TABLE_GOTO_VPORT_MARK); 1042 1043 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1044 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1045 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1046 1047 MLX5_SET(fte_match_param, spec->match_value, misc_parameters_2.metadata_reg_c_0, 1048 mlx5_eswitch_get_vport_metadata_for_match(esw, vport_num)); 1049 dest.vport.num = vport_num; 1050 1051 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1052 spec, &flow_act, &dest, 1); 1053 if (IS_ERR(flow_rule)) 1054 esw_warn(esw->dev, "FDB: Failed to add send to vport meta rule vport %d, err %ld\n", 1055 vport_num, PTR_ERR(flow_rule)); 1056 1057 kvfree(spec); 1058 return flow_rule; 1059 } 1060 1061 static bool mlx5_eswitch_reg_c1_loopback_supported(struct mlx5_eswitch *esw) 1062 { 1063 return MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 1064 MLX5_FDB_TO_VPORT_REG_C_1; 1065 } 1066 1067 static int esw_set_passing_vport_metadata(struct mlx5_eswitch *esw, bool enable) 1068 { 1069 u32 out[MLX5_ST_SZ_DW(query_esw_vport_context_out)] = {}; 1070 u32 min[MLX5_ST_SZ_DW(modify_esw_vport_context_in)] = {}; 1071 u32 in[MLX5_ST_SZ_DW(query_esw_vport_context_in)] = {}; 1072 u8 curr, wanted; 1073 int err; 1074 1075 if (!mlx5_eswitch_reg_c1_loopback_supported(esw) && 1076 !mlx5_eswitch_vport_match_metadata_enabled(esw)) 1077 return 0; 1078 1079 MLX5_SET(query_esw_vport_context_in, in, opcode, 1080 MLX5_CMD_OP_QUERY_ESW_VPORT_CONTEXT); 1081 err = mlx5_cmd_exec_inout(esw->dev, query_esw_vport_context, in, out); 1082 if (err) 1083 return err; 1084 1085 curr = MLX5_GET(query_esw_vport_context_out, out, 1086 esw_vport_context.fdb_to_vport_reg_c_id); 1087 wanted = MLX5_FDB_TO_VPORT_REG_C_0; 1088 if (mlx5_eswitch_reg_c1_loopback_supported(esw)) 1089 wanted |= MLX5_FDB_TO_VPORT_REG_C_1; 1090 1091 if (enable) 1092 curr |= wanted; 1093 else 1094 curr &= ~wanted; 1095 1096 MLX5_SET(modify_esw_vport_context_in, min, 1097 esw_vport_context.fdb_to_vport_reg_c_id, curr); 1098 MLX5_SET(modify_esw_vport_context_in, min, 1099 field_select.fdb_to_vport_reg_c_id, 1); 1100 1101 err = mlx5_eswitch_modify_esw_vport_context(esw->dev, 0, false, min); 1102 if (!err) { 1103 if (enable && (curr & MLX5_FDB_TO_VPORT_REG_C_1)) 1104 esw->flags |= MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1105 else 1106 esw->flags &= ~MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED; 1107 } 1108 1109 return err; 1110 } 1111 1112 static void peer_miss_rules_setup(struct mlx5_eswitch *esw, 1113 struct mlx5_core_dev *peer_dev, 1114 struct mlx5_flow_spec *spec, 1115 struct mlx5_flow_destination *dest) 1116 { 1117 void *misc; 1118 1119 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1120 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1121 misc_parameters_2); 1122 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1123 mlx5_eswitch_get_vport_metadata_mask()); 1124 1125 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1126 } else { 1127 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1128 misc_parameters); 1129 1130 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, 1131 MLX5_CAP_GEN(peer_dev, vhca_id)); 1132 1133 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 1134 1135 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1136 misc_parameters); 1137 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 1138 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 1139 source_eswitch_owner_vhca_id); 1140 } 1141 1142 dest->type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1143 dest->vport.num = peer_dev->priv.eswitch->manager_vport; 1144 dest->vport.vhca_id = MLX5_CAP_GEN(peer_dev, vhca_id); 1145 dest->vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 1146 } 1147 1148 static void esw_set_peer_miss_rule_source_port(struct mlx5_eswitch *esw, 1149 struct mlx5_eswitch *peer_esw, 1150 struct mlx5_flow_spec *spec, 1151 u16 vport) 1152 { 1153 void *misc; 1154 1155 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1156 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1157 misc_parameters_2); 1158 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1159 mlx5_eswitch_get_vport_metadata_for_match(peer_esw, 1160 vport)); 1161 } else { 1162 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1163 misc_parameters); 1164 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 1165 } 1166 } 1167 1168 static int esw_add_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1169 struct mlx5_core_dev *peer_dev) 1170 { 1171 struct mlx5_flow_destination dest = {}; 1172 struct mlx5_flow_act flow_act = {0}; 1173 struct mlx5_flow_handle **flows; 1174 /* total vports is the same for both e-switches */ 1175 int nvports = esw->total_vports; 1176 struct mlx5_flow_handle *flow; 1177 struct mlx5_flow_spec *spec; 1178 struct mlx5_vport *vport; 1179 int err, pfindex; 1180 unsigned long i; 1181 void *misc; 1182 1183 if (!MLX5_VPORT_MANAGER(esw->dev) && !mlx5_core_is_ecpf_esw_manager(esw->dev)) 1184 return 0; 1185 1186 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1187 if (!spec) 1188 return -ENOMEM; 1189 1190 peer_miss_rules_setup(esw, peer_dev, spec, &dest); 1191 1192 flows = kvcalloc(nvports, sizeof(*flows), GFP_KERNEL); 1193 if (!flows) { 1194 err = -ENOMEM; 1195 goto alloc_flows_err; 1196 } 1197 1198 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1199 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1200 misc_parameters); 1201 1202 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1203 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1204 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1205 spec, MLX5_VPORT_PF); 1206 1207 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1208 spec, &flow_act, &dest, 1); 1209 if (IS_ERR(flow)) { 1210 err = PTR_ERR(flow); 1211 goto add_pf_flow_err; 1212 } 1213 flows[vport->index] = flow; 1214 } 1215 1216 if (mlx5_ecpf_vport_exists(esw->dev)) { 1217 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1218 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_ECPF); 1219 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1220 spec, &flow_act, &dest, 1); 1221 if (IS_ERR(flow)) { 1222 err = PTR_ERR(flow); 1223 goto add_ecpf_flow_err; 1224 } 1225 flows[vport->index] = flow; 1226 } 1227 1228 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1229 esw_set_peer_miss_rule_source_port(esw, 1230 peer_dev->priv.eswitch, 1231 spec, vport->vport); 1232 1233 flow = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1234 spec, &flow_act, &dest, 1); 1235 if (IS_ERR(flow)) { 1236 err = PTR_ERR(flow); 1237 goto add_vf_flow_err; 1238 } 1239 flows[vport->index] = flow; 1240 } 1241 1242 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 1243 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1244 if (i >= mlx5_core_max_ec_vfs(peer_dev)) 1245 break; 1246 esw_set_peer_miss_rule_source_port(esw, peer_dev->priv.eswitch, 1247 spec, vport->vport); 1248 flow = mlx5_add_flow_rules(esw->fdb_table.offloads.slow_fdb, 1249 spec, &flow_act, &dest, 1); 1250 if (IS_ERR(flow)) { 1251 err = PTR_ERR(flow); 1252 goto add_ec_vf_flow_err; 1253 } 1254 flows[vport->index] = flow; 1255 } 1256 } 1257 1258 pfindex = mlx5_get_dev_index(peer_dev); 1259 if (pfindex >= MLX5_MAX_PORTS) { 1260 esw_warn(esw->dev, "Peer dev index(%d) is over the max num defined(%d)\n", 1261 pfindex, MLX5_MAX_PORTS); 1262 err = -EINVAL; 1263 goto add_ec_vf_flow_err; 1264 } 1265 esw->fdb_table.offloads.peer_miss_rules[pfindex] = flows; 1266 1267 kvfree(spec); 1268 return 0; 1269 1270 add_ec_vf_flow_err: 1271 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1272 if (!flows[vport->index]) 1273 continue; 1274 mlx5_del_flow_rules(flows[vport->index]); 1275 } 1276 add_vf_flow_err: 1277 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) { 1278 if (!flows[vport->index]) 1279 continue; 1280 mlx5_del_flow_rules(flows[vport->index]); 1281 } 1282 if (mlx5_ecpf_vport_exists(esw->dev)) { 1283 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1284 mlx5_del_flow_rules(flows[vport->index]); 1285 } 1286 add_ecpf_flow_err: 1287 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1288 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1289 mlx5_del_flow_rules(flows[vport->index]); 1290 } 1291 add_pf_flow_err: 1292 esw_warn(esw->dev, "FDB: Failed to add peer miss flow rule err %d\n", err); 1293 kvfree(flows); 1294 alloc_flows_err: 1295 kvfree(spec); 1296 return err; 1297 } 1298 1299 static void esw_del_fdb_peer_miss_rules(struct mlx5_eswitch *esw, 1300 struct mlx5_core_dev *peer_dev) 1301 { 1302 u16 peer_index = mlx5_get_dev_index(peer_dev); 1303 struct mlx5_flow_handle **flows; 1304 struct mlx5_vport *vport; 1305 unsigned long i; 1306 1307 flows = esw->fdb_table.offloads.peer_miss_rules[peer_index]; 1308 if (!flows) 1309 return; 1310 1311 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 1312 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, mlx5_core_max_ec_vfs(esw->dev)) { 1313 /* The flow for a particular vport could be NULL if the other ECPF 1314 * has fewer or no VFs enabled 1315 */ 1316 if (!flows[vport->index]) 1317 continue; 1318 mlx5_del_flow_rules(flows[vport->index]); 1319 } 1320 } 1321 1322 mlx5_esw_for_each_vf_vport(esw, i, vport, mlx5_core_max_vfs(esw->dev)) 1323 mlx5_del_flow_rules(flows[vport->index]); 1324 1325 if (mlx5_ecpf_vport_exists(esw->dev)) { 1326 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_ECPF); 1327 mlx5_del_flow_rules(flows[vport->index]); 1328 } 1329 1330 if (mlx5_core_is_ecpf_esw_manager(esw->dev)) { 1331 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_PF); 1332 mlx5_del_flow_rules(flows[vport->index]); 1333 } 1334 1335 kvfree(flows); 1336 esw->fdb_table.offloads.peer_miss_rules[peer_index] = NULL; 1337 } 1338 1339 static int esw_add_fdb_miss_rule(struct mlx5_eswitch *esw) 1340 { 1341 struct mlx5_flow_act flow_act = {0}; 1342 struct mlx5_flow_destination dest = {}; 1343 struct mlx5_flow_handle *flow_rule = NULL; 1344 struct mlx5_flow_spec *spec; 1345 void *headers_c; 1346 void *headers_v; 1347 int err = 0; 1348 u8 *dmac_c; 1349 u8 *dmac_v; 1350 1351 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1352 if (!spec) { 1353 err = -ENOMEM; 1354 goto out; 1355 } 1356 1357 spec->match_criteria_enable = MLX5_MATCH_OUTER_HEADERS; 1358 headers_c = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1359 outer_headers); 1360 dmac_c = MLX5_ADDR_OF(fte_match_param, headers_c, 1361 outer_headers.dmac_47_16); 1362 dmac_c[0] = 0x01; 1363 1364 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 1365 dest.vport.num = esw->manager_vport; 1366 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1367 1368 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1369 spec, &flow_act, &dest, 1); 1370 if (IS_ERR(flow_rule)) { 1371 err = PTR_ERR(flow_rule); 1372 esw_warn(esw->dev, "FDB: Failed to add unicast miss flow rule err %d\n", err); 1373 goto out; 1374 } 1375 1376 esw->fdb_table.offloads.miss_rule_uni = flow_rule; 1377 1378 headers_v = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1379 outer_headers); 1380 dmac_v = MLX5_ADDR_OF(fte_match_param, headers_v, 1381 outer_headers.dmac_47_16); 1382 dmac_v[0] = 0x01; 1383 flow_rule = mlx5_add_flow_rules(mlx5_eswitch_get_slow_fdb(esw), 1384 spec, &flow_act, &dest, 1); 1385 if (IS_ERR(flow_rule)) { 1386 err = PTR_ERR(flow_rule); 1387 esw_warn(esw->dev, "FDB: Failed to add multicast miss flow rule err %d\n", err); 1388 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1389 goto out; 1390 } 1391 1392 esw->fdb_table.offloads.miss_rule_multi = flow_rule; 1393 1394 out: 1395 kvfree(spec); 1396 return err; 1397 } 1398 1399 struct mlx5_flow_handle * 1400 esw_add_restore_rule(struct mlx5_eswitch *esw, u32 tag) 1401 { 1402 struct mlx5_flow_act flow_act = { .flags = FLOW_ACT_NO_APPEND, }; 1403 struct mlx5_flow_table *ft = esw->offloads.ft_offloads_restore; 1404 struct mlx5_flow_context *flow_context; 1405 struct mlx5_flow_handle *flow_rule; 1406 struct mlx5_flow_destination dest; 1407 struct mlx5_flow_spec *spec; 1408 void *misc; 1409 1410 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 1411 return ERR_PTR(-EOPNOTSUPP); 1412 1413 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1414 if (!spec) 1415 return ERR_PTR(-ENOMEM); 1416 1417 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 1418 misc_parameters_2); 1419 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 1420 ESW_REG_C0_USER_DATA_METADATA_MASK); 1421 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 1422 misc_parameters_2); 1423 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, tag); 1424 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 1425 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 1426 MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1427 flow_act.modify_hdr = esw->offloads.restore_copy_hdr_id; 1428 1429 flow_context = &spec->flow_context; 1430 flow_context->flags |= FLOW_CONTEXT_HAS_TAG; 1431 flow_context->flow_tag = tag; 1432 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1433 dest.ft = esw->offloads.ft_offloads; 1434 1435 flow_rule = mlx5_add_flow_rules(ft, spec, &flow_act, &dest, 1); 1436 kvfree(spec); 1437 1438 if (IS_ERR(flow_rule)) 1439 esw_warn(esw->dev, 1440 "Failed to create restore rule for tag: %d, err(%d)\n", 1441 tag, (int)PTR_ERR(flow_rule)); 1442 1443 return flow_rule; 1444 } 1445 1446 #define MAX_PF_SQ 256 1447 #define MAX_SQ_NVPORTS 32 1448 1449 void 1450 mlx5_esw_set_flow_group_source_port(struct mlx5_eswitch *esw, 1451 u32 *flow_group_in, 1452 int match_params) 1453 { 1454 void *match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1455 flow_group_in, 1456 match_criteria); 1457 1458 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1459 MLX5_SET(create_flow_group_in, flow_group_in, 1460 match_criteria_enable, 1461 MLX5_MATCH_MISC_PARAMETERS_2 | match_params); 1462 1463 MLX5_SET(fte_match_param, match_criteria, 1464 misc_parameters_2.metadata_reg_c_0, 1465 mlx5_eswitch_get_vport_metadata_mask()); 1466 } else { 1467 MLX5_SET(create_flow_group_in, flow_group_in, 1468 match_criteria_enable, 1469 MLX5_MATCH_MISC_PARAMETERS | match_params); 1470 1471 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1472 misc_parameters.source_port); 1473 } 1474 } 1475 1476 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 1477 static void esw_vport_tbl_put(struct mlx5_eswitch *esw) 1478 { 1479 struct mlx5_vport_tbl_attr attr; 1480 struct mlx5_vport *vport; 1481 unsigned long i; 1482 1483 attr.chain = 0; 1484 attr.prio = 1; 1485 mlx5_esw_for_each_vport(esw, i, vport) { 1486 attr.vport = vport->vport; 1487 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1488 mlx5_esw_vporttbl_put(esw, &attr); 1489 } 1490 } 1491 1492 static int esw_vport_tbl_get(struct mlx5_eswitch *esw) 1493 { 1494 struct mlx5_vport_tbl_attr attr; 1495 struct mlx5_flow_table *fdb; 1496 struct mlx5_vport *vport; 1497 unsigned long i; 1498 1499 attr.chain = 0; 1500 attr.prio = 1; 1501 mlx5_esw_for_each_vport(esw, i, vport) { 1502 attr.vport = vport->vport; 1503 attr.vport_ns = &mlx5_esw_vport_tbl_mirror_ns; 1504 fdb = mlx5_esw_vporttbl_get(esw, &attr); 1505 if (IS_ERR(fdb)) 1506 goto out; 1507 } 1508 return 0; 1509 1510 out: 1511 esw_vport_tbl_put(esw); 1512 return PTR_ERR(fdb); 1513 } 1514 1515 #define fdb_modify_header_fwd_to_table_supported(esw) \ 1516 (MLX5_CAP_ESW_FLOWTABLE((esw)->dev, fdb_modify_header_fwd_to_table)) 1517 static void esw_init_chains_offload_flags(struct mlx5_eswitch *esw, u32 *flags) 1518 { 1519 struct mlx5_core_dev *dev = esw->dev; 1520 1521 if (MLX5_CAP_ESW_FLOWTABLE_FDB(dev, ignore_flow_level)) 1522 *flags |= MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 1523 1524 if (!MLX5_CAP_ESW_FLOWTABLE(dev, multi_fdb_encap) && 1525 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 1526 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1527 esw_warn(dev, "Tc chains and priorities offload aren't supported, update firmware if needed\n"); 1528 } else if (!mlx5_eswitch_reg_c1_loopback_enabled(esw)) { 1529 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1530 esw_warn(dev, "Tc chains and priorities offload aren't supported\n"); 1531 } else if (!fdb_modify_header_fwd_to_table_supported(esw)) { 1532 /* Disabled when ttl workaround is needed, e.g 1533 * when ESWITCH_IPV4_TTL_MODIFY_ENABLE = true in mlxconfig 1534 */ 1535 esw_warn(dev, 1536 "Tc chains and priorities offload aren't supported, check firmware version, or mlxconfig settings\n"); 1537 *flags &= ~MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1538 } else { 1539 *flags |= MLX5_CHAINS_AND_PRIOS_SUPPORTED; 1540 esw_info(dev, "Supported tc chains and prios offload\n"); 1541 } 1542 1543 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1544 *flags |= MLX5_CHAINS_FT_TUNNEL_SUPPORTED; 1545 } 1546 1547 static int 1548 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1549 { 1550 struct mlx5_core_dev *dev = esw->dev; 1551 struct mlx5_flow_table *nf_ft, *ft; 1552 struct mlx5_chains_attr attr = {}; 1553 struct mlx5_fs_chains *chains; 1554 int err; 1555 1556 esw_init_chains_offload_flags(esw, &attr.flags); 1557 attr.ns = MLX5_FLOW_NAMESPACE_FDB; 1558 attr.max_grp_num = esw->params.large_group_num; 1559 attr.default_ft = miss_fdb; 1560 attr.mapping = esw->offloads.reg_c0_obj_pool; 1561 1562 chains = mlx5_chains_create(dev, &attr); 1563 if (IS_ERR(chains)) { 1564 err = PTR_ERR(chains); 1565 esw_warn(dev, "Failed to create fdb chains err(%d)\n", err); 1566 return err; 1567 } 1568 mlx5_chains_print_info(chains); 1569 1570 esw->fdb_table.offloads.esw_chains_priv = chains; 1571 1572 /* Create tc_end_ft which is the always created ft chain */ 1573 nf_ft = mlx5_chains_get_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1574 1, 0); 1575 if (IS_ERR(nf_ft)) { 1576 err = PTR_ERR(nf_ft); 1577 goto nf_ft_err; 1578 } 1579 1580 /* Always open the root for fast path */ 1581 ft = mlx5_chains_get_table(chains, 0, 1, 0); 1582 if (IS_ERR(ft)) { 1583 err = PTR_ERR(ft); 1584 goto level_0_err; 1585 } 1586 1587 /* Open level 1 for split fdb rules now if prios isn't supported */ 1588 if (!mlx5_chains_prios_supported(chains)) { 1589 err = esw_vport_tbl_get(esw); 1590 if (err) 1591 goto level_1_err; 1592 } 1593 1594 mlx5_chains_set_end_ft(chains, nf_ft); 1595 1596 return 0; 1597 1598 level_1_err: 1599 mlx5_chains_put_table(chains, 0, 1, 0); 1600 level_0_err: 1601 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1602 nf_ft_err: 1603 mlx5_chains_destroy(chains); 1604 esw->fdb_table.offloads.esw_chains_priv = NULL; 1605 1606 return err; 1607 } 1608 1609 static void 1610 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1611 { 1612 if (!mlx5_chains_prios_supported(chains)) 1613 esw_vport_tbl_put(esw); 1614 mlx5_chains_put_table(chains, 0, 1, 0); 1615 mlx5_chains_put_table(chains, mlx5_chains_get_nf_ft_chain(chains), 1, 0); 1616 mlx5_chains_destroy(chains); 1617 } 1618 1619 #else /* CONFIG_MLX5_CLS_ACT */ 1620 1621 static int 1622 esw_chains_create(struct mlx5_eswitch *esw, struct mlx5_flow_table *miss_fdb) 1623 { return 0; } 1624 1625 static void 1626 esw_chains_destroy(struct mlx5_eswitch *esw, struct mlx5_fs_chains *chains) 1627 {} 1628 1629 #endif 1630 1631 static int 1632 esw_create_send_to_vport_group(struct mlx5_eswitch *esw, 1633 struct mlx5_flow_table *fdb, 1634 u32 *flow_group_in, 1635 int *ix) 1636 { 1637 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1638 struct mlx5_flow_group *g; 1639 void *match_criteria; 1640 int count, err = 0; 1641 1642 memset(flow_group_in, 0, inlen); 1643 1644 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, MLX5_MATCH_MISC_PARAMETERS); 1645 1646 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1647 MLX5_SET_TO_ONES(fte_match_param, match_criteria, misc_parameters.source_sqn); 1648 1649 if (!mlx5_eswitch_vport_match_metadata_enabled(esw) && 1650 MLX5_CAP_ESW(esw->dev, merged_eswitch)) { 1651 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1652 misc_parameters.source_eswitch_owner_vhca_id); 1653 MLX5_SET(create_flow_group_in, flow_group_in, 1654 source_eswitch_owner_vhca_id_valid, 1); 1655 } 1656 1657 /* See comment at table_size calculation */ 1658 count = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ); 1659 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 1660 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, *ix + count - 1); 1661 *ix += count; 1662 1663 g = mlx5_create_flow_group(fdb, flow_group_in); 1664 if (IS_ERR(g)) { 1665 err = PTR_ERR(g); 1666 esw_warn(esw->dev, "Failed to create send-to-vport flow group err(%d)\n", err); 1667 goto out; 1668 } 1669 esw->fdb_table.offloads.send_to_vport_grp = g; 1670 1671 out: 1672 return err; 1673 } 1674 1675 static int 1676 esw_create_meta_send_to_vport_group(struct mlx5_eswitch *esw, 1677 struct mlx5_flow_table *fdb, 1678 u32 *flow_group_in, 1679 int *ix) 1680 { 1681 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1682 struct mlx5_flow_group *g; 1683 void *match_criteria; 1684 int err = 0; 1685 1686 if (!esw_src_port_rewrite_supported(esw)) 1687 return 0; 1688 1689 memset(flow_group_in, 0, inlen); 1690 1691 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1692 MLX5_MATCH_MISC_PARAMETERS_2); 1693 1694 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, match_criteria); 1695 1696 MLX5_SET(fte_match_param, match_criteria, 1697 misc_parameters_2.metadata_reg_c_0, 1698 mlx5_eswitch_get_vport_metadata_mask()); 1699 MLX5_SET(fte_match_param, match_criteria, 1700 misc_parameters_2.metadata_reg_c_1, ESW_TUN_MASK); 1701 1702 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1703 MLX5_SET(create_flow_group_in, flow_group_in, 1704 end_flow_index, *ix + esw->total_vports - 1); 1705 *ix += esw->total_vports; 1706 1707 g = mlx5_create_flow_group(fdb, flow_group_in); 1708 if (IS_ERR(g)) { 1709 err = PTR_ERR(g); 1710 esw_warn(esw->dev, 1711 "Failed to create send-to-vport meta flow group err(%d)\n", err); 1712 goto send_vport_meta_err; 1713 } 1714 esw->fdb_table.offloads.send_to_vport_meta_grp = g; 1715 1716 return 0; 1717 1718 send_vport_meta_err: 1719 return err; 1720 } 1721 1722 static int 1723 esw_create_peer_esw_miss_group(struct mlx5_eswitch *esw, 1724 struct mlx5_flow_table *fdb, 1725 u32 *flow_group_in, 1726 int *ix) 1727 { 1728 int max_peer_ports = (esw->total_vports - 1) * (MLX5_MAX_PORTS - 1); 1729 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1730 struct mlx5_flow_group *g; 1731 void *match_criteria; 1732 int err = 0; 1733 1734 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1735 return 0; 1736 1737 memset(flow_group_in, 0, inlen); 1738 1739 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 1740 1741 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) { 1742 match_criteria = MLX5_ADDR_OF(create_flow_group_in, 1743 flow_group_in, 1744 match_criteria); 1745 1746 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 1747 misc_parameters.source_eswitch_owner_vhca_id); 1748 1749 MLX5_SET(create_flow_group_in, flow_group_in, 1750 source_eswitch_owner_vhca_id_valid, 1); 1751 } 1752 1753 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1754 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1755 *ix + max_peer_ports); 1756 *ix += max_peer_ports + 1; 1757 1758 g = mlx5_create_flow_group(fdb, flow_group_in); 1759 if (IS_ERR(g)) { 1760 err = PTR_ERR(g); 1761 esw_warn(esw->dev, "Failed to create peer miss flow group err(%d)\n", err); 1762 goto out; 1763 } 1764 esw->fdb_table.offloads.peer_miss_grp = g; 1765 1766 out: 1767 return err; 1768 } 1769 1770 static int 1771 esw_create_miss_group(struct mlx5_eswitch *esw, 1772 struct mlx5_flow_table *fdb, 1773 u32 *flow_group_in, 1774 int *ix) 1775 { 1776 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1777 struct mlx5_flow_group *g; 1778 void *match_criteria; 1779 int err = 0; 1780 u8 *dmac; 1781 1782 memset(flow_group_in, 0, inlen); 1783 1784 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 1785 MLX5_MATCH_OUTER_HEADERS); 1786 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 1787 match_criteria); 1788 dmac = MLX5_ADDR_OF(fte_match_param, match_criteria, 1789 outer_headers.dmac_47_16); 1790 dmac[0] = 0x01; 1791 1792 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, *ix); 1793 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 1794 *ix + MLX5_ESW_MISS_FLOWS); 1795 1796 g = mlx5_create_flow_group(fdb, flow_group_in); 1797 if (IS_ERR(g)) { 1798 err = PTR_ERR(g); 1799 esw_warn(esw->dev, "Failed to create miss flow group err(%d)\n", err); 1800 goto miss_err; 1801 } 1802 esw->fdb_table.offloads.miss_grp = g; 1803 1804 err = esw_add_fdb_miss_rule(esw); 1805 if (err) 1806 goto miss_rule_err; 1807 1808 return 0; 1809 1810 miss_rule_err: 1811 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1812 miss_err: 1813 return err; 1814 } 1815 1816 static int esw_create_offloads_fdb_tables(struct mlx5_eswitch *esw) 1817 { 1818 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 1819 struct mlx5_flow_table_attr ft_attr = {}; 1820 struct mlx5_core_dev *dev = esw->dev; 1821 struct mlx5_flow_namespace *root_ns; 1822 struct mlx5_flow_table *fdb = NULL; 1823 int table_size, ix = 0, err = 0; 1824 u32 flags = 0, *flow_group_in; 1825 1826 esw_debug(esw->dev, "Create offloads FDB Tables\n"); 1827 1828 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 1829 if (!flow_group_in) 1830 return -ENOMEM; 1831 1832 root_ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_FDB); 1833 if (!root_ns) { 1834 esw_warn(dev, "Failed to get FDB flow namespace\n"); 1835 err = -EOPNOTSUPP; 1836 goto ns_err; 1837 } 1838 esw->fdb_table.offloads.ns = root_ns; 1839 err = mlx5_flow_namespace_set_mode(root_ns, 1840 esw->dev->priv.steering->mode); 1841 if (err) { 1842 esw_warn(dev, "Failed to set FDB namespace steering mode\n"); 1843 goto ns_err; 1844 } 1845 1846 /* To be strictly correct: 1847 * MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) 1848 * should be: 1849 * esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ + 1850 * peer_esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ 1851 * but as the peer device might not be in switchdev mode it's not 1852 * possible. We use the fact that by default FW sets max vfs and max sfs 1853 * to the same value on both devices. If it needs to be changed in the future note 1854 * the peer miss group should also be created based on the number of 1855 * total vports of the peer (currently is also uses esw->total_vports). 1856 */ 1857 table_size = MLX5_MAX_PORTS * (esw->total_vports * MAX_SQ_NVPORTS + MAX_PF_SQ) + 1858 esw->total_vports * MLX5_MAX_PORTS + MLX5_ESW_MISS_FLOWS; 1859 1860 /* create the slow path fdb with encap set, so further table instances 1861 * can be created at run time while VFs are probed if the FW allows that. 1862 */ 1863 if (esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) 1864 flags |= (MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT | 1865 MLX5_FLOW_TABLE_TUNNEL_EN_DECAP); 1866 1867 ft_attr.flags = flags; 1868 ft_attr.max_fte = table_size; 1869 ft_attr.prio = FDB_SLOW_PATH; 1870 1871 fdb = mlx5_create_flow_table(root_ns, &ft_attr); 1872 if (IS_ERR(fdb)) { 1873 err = PTR_ERR(fdb); 1874 esw_warn(dev, "Failed to create slow path FDB Table err %d\n", err); 1875 goto slow_fdb_err; 1876 } 1877 esw->fdb_table.offloads.slow_fdb = fdb; 1878 1879 /* Create empty TC-miss managed table. This allows plugging in following 1880 * priorities without directly exposing their level 0 table to 1881 * eswitch_offloads and passing it as miss_fdb to following call to 1882 * esw_chains_create(). 1883 */ 1884 memset(&ft_attr, 0, sizeof(ft_attr)); 1885 ft_attr.prio = FDB_TC_MISS; 1886 esw->fdb_table.offloads.tc_miss_table = mlx5_create_flow_table(root_ns, &ft_attr); 1887 if (IS_ERR(esw->fdb_table.offloads.tc_miss_table)) { 1888 err = PTR_ERR(esw->fdb_table.offloads.tc_miss_table); 1889 esw_warn(dev, "Failed to create TC miss FDB Table err %d\n", err); 1890 goto tc_miss_table_err; 1891 } 1892 1893 err = esw_chains_create(esw, esw->fdb_table.offloads.tc_miss_table); 1894 if (err) { 1895 esw_warn(dev, "Failed to open fdb chains err(%d)\n", err); 1896 goto fdb_chains_err; 1897 } 1898 1899 err = esw_create_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1900 if (err) 1901 goto send_vport_err; 1902 1903 err = esw_create_meta_send_to_vport_group(esw, fdb, flow_group_in, &ix); 1904 if (err) 1905 goto send_vport_meta_err; 1906 1907 err = esw_create_peer_esw_miss_group(esw, fdb, flow_group_in, &ix); 1908 if (err) 1909 goto peer_miss_err; 1910 1911 err = esw_create_miss_group(esw, fdb, flow_group_in, &ix); 1912 if (err) 1913 goto miss_err; 1914 1915 kvfree(flow_group_in); 1916 return 0; 1917 1918 miss_err: 1919 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1920 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1921 peer_miss_err: 1922 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1923 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1924 send_vport_meta_err: 1925 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1926 send_vport_err: 1927 esw_chains_destroy(esw, esw_chains(esw)); 1928 fdb_chains_err: 1929 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1930 tc_miss_table_err: 1931 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1932 slow_fdb_err: 1933 /* Holds true only as long as DMFS is the default */ 1934 mlx5_flow_namespace_set_mode(root_ns, MLX5_FLOW_STEERING_MODE_DMFS); 1935 ns_err: 1936 kvfree(flow_group_in); 1937 return err; 1938 } 1939 1940 static void esw_destroy_offloads_fdb_tables(struct mlx5_eswitch *esw) 1941 { 1942 if (!mlx5_eswitch_get_slow_fdb(esw)) 1943 return; 1944 1945 esw_debug(esw->dev, "Destroy offloads FDB Tables\n"); 1946 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_multi); 1947 mlx5_del_flow_rules(esw->fdb_table.offloads.miss_rule_uni); 1948 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_grp); 1949 if (esw->fdb_table.offloads.send_to_vport_meta_grp) 1950 mlx5_destroy_flow_group(esw->fdb_table.offloads.send_to_vport_meta_grp); 1951 if (MLX5_CAP_ESW(esw->dev, merged_eswitch)) 1952 mlx5_destroy_flow_group(esw->fdb_table.offloads.peer_miss_grp); 1953 mlx5_destroy_flow_group(esw->fdb_table.offloads.miss_grp); 1954 1955 esw_chains_destroy(esw, esw_chains(esw)); 1956 1957 mlx5_destroy_flow_table(esw->fdb_table.offloads.tc_miss_table); 1958 mlx5_destroy_flow_table(mlx5_eswitch_get_slow_fdb(esw)); 1959 /* Holds true only as long as DMFS is the default */ 1960 mlx5_flow_namespace_set_mode(esw->fdb_table.offloads.ns, 1961 MLX5_FLOW_STEERING_MODE_DMFS); 1962 atomic64_set(&esw->user_count, 0); 1963 } 1964 1965 static int esw_get_nr_ft_offloads_steering_src_ports(struct mlx5_eswitch *esw) 1966 { 1967 int nvports; 1968 1969 nvports = esw->total_vports + MLX5_ESW_MISS_FLOWS; 1970 if (mlx5e_tc_int_port_supported(esw)) 1971 nvports += MLX5E_TC_MAX_INT_PORT_NUM; 1972 1973 return nvports; 1974 } 1975 1976 static int esw_create_offloads_table(struct mlx5_eswitch *esw) 1977 { 1978 struct mlx5_flow_table_attr ft_attr = {}; 1979 struct mlx5_core_dev *dev = esw->dev; 1980 struct mlx5_flow_table *ft_offloads; 1981 struct mlx5_flow_namespace *ns; 1982 int err = 0; 1983 1984 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 1985 if (!ns) { 1986 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 1987 return -EOPNOTSUPP; 1988 } 1989 1990 ft_attr.max_fte = esw_get_nr_ft_offloads_steering_src_ports(esw) + 1991 MLX5_ESW_FT_OFFLOADS_DROP_RULE; 1992 ft_attr.prio = 1; 1993 1994 ft_offloads = mlx5_create_flow_table(ns, &ft_attr); 1995 if (IS_ERR(ft_offloads)) { 1996 err = PTR_ERR(ft_offloads); 1997 esw_warn(esw->dev, "Failed to create offloads table, err %d\n", err); 1998 return err; 1999 } 2000 2001 esw->offloads.ft_offloads = ft_offloads; 2002 return 0; 2003 } 2004 2005 static void esw_destroy_offloads_table(struct mlx5_eswitch *esw) 2006 { 2007 struct mlx5_esw_offload *offloads = &esw->offloads; 2008 2009 mlx5_destroy_flow_table(offloads->ft_offloads); 2010 } 2011 2012 static int esw_create_vport_rx_group(struct mlx5_eswitch *esw) 2013 { 2014 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2015 struct mlx5_flow_group *g; 2016 u32 *flow_group_in; 2017 int nvports; 2018 int err = 0; 2019 2020 nvports = esw_get_nr_ft_offloads_steering_src_ports(esw); 2021 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2022 if (!flow_group_in) 2023 return -ENOMEM; 2024 2025 mlx5_esw_set_flow_group_source_port(esw, flow_group_in, 0); 2026 2027 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2028 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, nvports - 1); 2029 2030 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2031 2032 if (IS_ERR(g)) { 2033 err = PTR_ERR(g); 2034 mlx5_core_warn(esw->dev, "Failed to create vport rx group err %d\n", err); 2035 goto out; 2036 } 2037 2038 esw->offloads.vport_rx_group = g; 2039 out: 2040 kvfree(flow_group_in); 2041 return err; 2042 } 2043 2044 static void esw_destroy_vport_rx_group(struct mlx5_eswitch *esw) 2045 { 2046 mlx5_destroy_flow_group(esw->offloads.vport_rx_group); 2047 } 2048 2049 static int esw_create_vport_rx_drop_rule_index(struct mlx5_eswitch *esw) 2050 { 2051 /* ft_offloads table is enlarged by MLX5_ESW_FT_OFFLOADS_DROP_RULE (1) 2052 * for the drop rule, which is placed at the end of the table. 2053 * So return the total of vport and int_port as rule index. 2054 */ 2055 return esw_get_nr_ft_offloads_steering_src_ports(esw); 2056 } 2057 2058 static int esw_create_vport_rx_drop_group(struct mlx5_eswitch *esw) 2059 { 2060 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2061 struct mlx5_flow_group *g; 2062 u32 *flow_group_in; 2063 int flow_index; 2064 int err = 0; 2065 2066 flow_index = esw_create_vport_rx_drop_rule_index(esw); 2067 2068 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2069 if (!flow_group_in) 2070 return -ENOMEM; 2071 2072 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, flow_index); 2073 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, flow_index); 2074 2075 g = mlx5_create_flow_group(esw->offloads.ft_offloads, flow_group_in); 2076 2077 if (IS_ERR(g)) { 2078 err = PTR_ERR(g); 2079 mlx5_core_warn(esw->dev, "Failed to create vport rx drop group err %d\n", err); 2080 goto out; 2081 } 2082 2083 esw->offloads.vport_rx_drop_group = g; 2084 out: 2085 kvfree(flow_group_in); 2086 return err; 2087 } 2088 2089 static void esw_destroy_vport_rx_drop_group(struct mlx5_eswitch *esw) 2090 { 2091 if (esw->offloads.vport_rx_drop_group) 2092 mlx5_destroy_flow_group(esw->offloads.vport_rx_drop_group); 2093 } 2094 2095 void 2096 mlx5_esw_set_spec_source_port(struct mlx5_eswitch *esw, 2097 u16 vport, 2098 struct mlx5_flow_spec *spec) 2099 { 2100 void *misc; 2101 2102 if (mlx5_eswitch_vport_match_metadata_enabled(esw)) { 2103 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters_2); 2104 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2105 mlx5_eswitch_get_vport_metadata_for_match(esw, vport)); 2106 2107 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters_2); 2108 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2109 mlx5_eswitch_get_vport_metadata_mask()); 2110 2111 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS_2; 2112 } else { 2113 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, misc_parameters); 2114 MLX5_SET(fte_match_set_misc, misc, source_port, vport); 2115 2116 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2117 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2118 2119 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2120 } 2121 } 2122 2123 struct mlx5_flow_handle * 2124 mlx5_eswitch_create_vport_rx_rule(struct mlx5_eswitch *esw, u16 vport, 2125 struct mlx5_flow_destination *dest) 2126 { 2127 struct mlx5_flow_act flow_act = {0}; 2128 struct mlx5_flow_handle *flow_rule; 2129 struct mlx5_flow_spec *spec; 2130 2131 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2132 if (!spec) { 2133 flow_rule = ERR_PTR(-ENOMEM); 2134 goto out; 2135 } 2136 2137 mlx5_esw_set_spec_source_port(esw, vport, spec); 2138 2139 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2140 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, spec, 2141 &flow_act, dest, 1); 2142 if (IS_ERR(flow_rule)) { 2143 esw_warn(esw->dev, "fs offloads: Failed to add vport rx rule err %ld\n", PTR_ERR(flow_rule)); 2144 goto out; 2145 } 2146 2147 out: 2148 kvfree(spec); 2149 return flow_rule; 2150 } 2151 2152 static int esw_create_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2153 { 2154 struct mlx5_flow_act flow_act = {}; 2155 struct mlx5_flow_handle *flow_rule; 2156 2157 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_DROP; 2158 flow_rule = mlx5_add_flow_rules(esw->offloads.ft_offloads, NULL, 2159 &flow_act, NULL, 0); 2160 if (IS_ERR(flow_rule)) { 2161 esw_warn(esw->dev, 2162 "fs offloads: Failed to add vport rx drop rule err %ld\n", 2163 PTR_ERR(flow_rule)); 2164 return PTR_ERR(flow_rule); 2165 } 2166 2167 esw->offloads.vport_rx_drop_rule = flow_rule; 2168 2169 return 0; 2170 } 2171 2172 static void esw_destroy_vport_rx_drop_rule(struct mlx5_eswitch *esw) 2173 { 2174 if (esw->offloads.vport_rx_drop_rule) 2175 mlx5_del_flow_rules(esw->offloads.vport_rx_drop_rule); 2176 } 2177 2178 static int mlx5_eswitch_inline_mode_get(struct mlx5_eswitch *esw, u8 *mode) 2179 { 2180 u8 prev_mlx5_mode, mlx5_mode = MLX5_INLINE_MODE_L2; 2181 struct mlx5_core_dev *dev = esw->dev; 2182 struct mlx5_vport *vport; 2183 unsigned long i; 2184 2185 if (!MLX5_CAP_GEN(dev, vport_group_manager)) 2186 return -EOPNOTSUPP; 2187 2188 if (!mlx5_esw_is_fdb_created(esw)) 2189 return -EOPNOTSUPP; 2190 2191 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 2192 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 2193 mlx5_mode = MLX5_INLINE_MODE_NONE; 2194 goto out; 2195 case MLX5_CAP_INLINE_MODE_L2: 2196 mlx5_mode = MLX5_INLINE_MODE_L2; 2197 goto out; 2198 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 2199 goto query_vports; 2200 } 2201 2202 query_vports: 2203 mlx5_query_nic_vport_min_inline(dev, esw->first_host_vport, &prev_mlx5_mode); 2204 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 2205 mlx5_query_nic_vport_min_inline(dev, vport->vport, &mlx5_mode); 2206 if (prev_mlx5_mode != mlx5_mode) 2207 return -EINVAL; 2208 prev_mlx5_mode = mlx5_mode; 2209 } 2210 2211 out: 2212 *mode = mlx5_mode; 2213 return 0; 2214 } 2215 2216 static void esw_destroy_restore_table(struct mlx5_eswitch *esw) 2217 { 2218 struct mlx5_esw_offload *offloads = &esw->offloads; 2219 2220 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2221 return; 2222 2223 mlx5_modify_header_dealloc(esw->dev, offloads->restore_copy_hdr_id); 2224 mlx5_destroy_flow_group(offloads->restore_group); 2225 mlx5_destroy_flow_table(offloads->ft_offloads_restore); 2226 } 2227 2228 static int esw_create_restore_table(struct mlx5_eswitch *esw) 2229 { 2230 u8 modact[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 2231 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2232 struct mlx5_flow_table_attr ft_attr = {}; 2233 struct mlx5_core_dev *dev = esw->dev; 2234 struct mlx5_flow_namespace *ns; 2235 struct mlx5_modify_hdr *mod_hdr; 2236 void *match_criteria, *misc; 2237 struct mlx5_flow_table *ft; 2238 struct mlx5_flow_group *g; 2239 u32 *flow_group_in; 2240 int err = 0; 2241 2242 if (!mlx5_eswitch_reg_c1_loopback_supported(esw)) 2243 return 0; 2244 2245 ns = mlx5_get_flow_namespace(dev, MLX5_FLOW_NAMESPACE_OFFLOADS); 2246 if (!ns) { 2247 esw_warn(esw->dev, "Failed to get offloads flow namespace\n"); 2248 return -EOPNOTSUPP; 2249 } 2250 2251 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2252 if (!flow_group_in) { 2253 err = -ENOMEM; 2254 goto out_free; 2255 } 2256 2257 ft_attr.max_fte = 1 << ESW_REG_C0_USER_DATA_METADATA_BITS; 2258 ft = mlx5_create_flow_table(ns, &ft_attr); 2259 if (IS_ERR(ft)) { 2260 err = PTR_ERR(ft); 2261 esw_warn(esw->dev, "Failed to create restore table, err %d\n", 2262 err); 2263 goto out_free; 2264 } 2265 2266 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2267 match_criteria); 2268 misc = MLX5_ADDR_OF(fte_match_param, match_criteria, 2269 misc_parameters_2); 2270 2271 MLX5_SET(fte_match_set_misc2, misc, metadata_reg_c_0, 2272 ESW_REG_C0_USER_DATA_METADATA_MASK); 2273 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2274 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, 2275 ft_attr.max_fte - 1); 2276 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2277 MLX5_MATCH_MISC_PARAMETERS_2); 2278 g = mlx5_create_flow_group(ft, flow_group_in); 2279 if (IS_ERR(g)) { 2280 err = PTR_ERR(g); 2281 esw_warn(dev, "Failed to create restore flow group, err: %d\n", 2282 err); 2283 goto err_group; 2284 } 2285 2286 MLX5_SET(copy_action_in, modact, action_type, MLX5_ACTION_TYPE_COPY); 2287 MLX5_SET(copy_action_in, modact, src_field, 2288 MLX5_ACTION_IN_FIELD_METADATA_REG_C_1); 2289 MLX5_SET(copy_action_in, modact, dst_field, 2290 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 2291 mod_hdr = mlx5_modify_header_alloc(esw->dev, 2292 MLX5_FLOW_NAMESPACE_KERNEL, 1, 2293 modact); 2294 if (IS_ERR(mod_hdr)) { 2295 err = PTR_ERR(mod_hdr); 2296 esw_warn(dev, "Failed to create restore mod header, err: %d\n", 2297 err); 2298 goto err_mod_hdr; 2299 } 2300 2301 esw->offloads.ft_offloads_restore = ft; 2302 esw->offloads.restore_group = g; 2303 esw->offloads.restore_copy_hdr_id = mod_hdr; 2304 2305 kvfree(flow_group_in); 2306 2307 return 0; 2308 2309 err_mod_hdr: 2310 mlx5_destroy_flow_group(g); 2311 err_group: 2312 mlx5_destroy_flow_table(ft); 2313 out_free: 2314 kvfree(flow_group_in); 2315 2316 return err; 2317 } 2318 2319 static int esw_offloads_start(struct mlx5_eswitch *esw, 2320 struct netlink_ext_ack *extack) 2321 { 2322 int err; 2323 2324 esw->mode = MLX5_ESWITCH_OFFLOADS; 2325 err = mlx5_eswitch_enable_locked(esw, esw->dev->priv.sriov.num_vfs); 2326 if (err) { 2327 NL_SET_ERR_MSG_MOD(extack, 2328 "Failed setting eswitch to offloads"); 2329 esw->mode = MLX5_ESWITCH_LEGACY; 2330 mlx5_rescan_drivers(esw->dev); 2331 return err; 2332 } 2333 if (esw->offloads.inline_mode == MLX5_INLINE_MODE_NONE) { 2334 if (mlx5_eswitch_inline_mode_get(esw, 2335 &esw->offloads.inline_mode)) { 2336 esw->offloads.inline_mode = MLX5_INLINE_MODE_L2; 2337 NL_SET_ERR_MSG_MOD(extack, 2338 "Inline mode is different between vports"); 2339 } 2340 } 2341 return 0; 2342 } 2343 2344 static int mlx5_esw_offloads_rep_init(struct mlx5_eswitch *esw, const struct mlx5_vport *vport) 2345 { 2346 struct mlx5_eswitch_rep *rep; 2347 int rep_type; 2348 int err; 2349 2350 rep = kzalloc(sizeof(*rep), GFP_KERNEL); 2351 if (!rep) 2352 return -ENOMEM; 2353 2354 rep->vport = vport->vport; 2355 rep->vport_index = vport->index; 2356 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2357 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 2358 2359 err = xa_insert(&esw->offloads.vport_reps, rep->vport, rep, GFP_KERNEL); 2360 if (err) 2361 goto insert_err; 2362 2363 return 0; 2364 2365 insert_err: 2366 kfree(rep); 2367 return err; 2368 } 2369 2370 static void mlx5_esw_offloads_rep_cleanup(struct mlx5_eswitch *esw, 2371 struct mlx5_eswitch_rep *rep) 2372 { 2373 xa_erase(&esw->offloads.vport_reps, rep->vport); 2374 kfree(rep); 2375 } 2376 2377 static void esw_offloads_cleanup_reps(struct mlx5_eswitch *esw) 2378 { 2379 struct mlx5_eswitch_rep *rep; 2380 unsigned long i; 2381 2382 mlx5_esw_for_each_rep(esw, i, rep) 2383 mlx5_esw_offloads_rep_cleanup(esw, rep); 2384 xa_destroy(&esw->offloads.vport_reps); 2385 } 2386 2387 static int esw_offloads_init_reps(struct mlx5_eswitch *esw) 2388 { 2389 struct mlx5_vport *vport; 2390 unsigned long i; 2391 int err; 2392 2393 xa_init(&esw->offloads.vport_reps); 2394 2395 mlx5_esw_for_each_vport(esw, i, vport) { 2396 err = mlx5_esw_offloads_rep_init(esw, vport); 2397 if (err) 2398 goto err; 2399 } 2400 return 0; 2401 2402 err: 2403 esw_offloads_cleanup_reps(esw); 2404 return err; 2405 } 2406 2407 static int esw_port_metadata_set(struct devlink *devlink, u32 id, 2408 struct devlink_param_gset_ctx *ctx) 2409 { 2410 struct mlx5_core_dev *dev = devlink_priv(devlink); 2411 struct mlx5_eswitch *esw = dev->priv.eswitch; 2412 int err = 0; 2413 2414 down_write(&esw->mode_lock); 2415 if (mlx5_esw_is_fdb_created(esw)) { 2416 err = -EBUSY; 2417 goto done; 2418 } 2419 if (!mlx5_esw_vport_match_metadata_supported(esw)) { 2420 err = -EOPNOTSUPP; 2421 goto done; 2422 } 2423 if (ctx->val.vbool) 2424 esw->flags |= MLX5_ESWITCH_VPORT_MATCH_METADATA; 2425 else 2426 esw->flags &= ~MLX5_ESWITCH_VPORT_MATCH_METADATA; 2427 done: 2428 up_write(&esw->mode_lock); 2429 return err; 2430 } 2431 2432 static int esw_port_metadata_get(struct devlink *devlink, u32 id, 2433 struct devlink_param_gset_ctx *ctx) 2434 { 2435 struct mlx5_core_dev *dev = devlink_priv(devlink); 2436 2437 ctx->val.vbool = mlx5_eswitch_vport_match_metadata_enabled(dev->priv.eswitch); 2438 return 0; 2439 } 2440 2441 static int esw_port_metadata_validate(struct devlink *devlink, u32 id, 2442 union devlink_param_value val, 2443 struct netlink_ext_ack *extack) 2444 { 2445 struct mlx5_core_dev *dev = devlink_priv(devlink); 2446 u8 esw_mode; 2447 2448 esw_mode = mlx5_eswitch_mode(dev); 2449 if (esw_mode == MLX5_ESWITCH_OFFLOADS) { 2450 NL_SET_ERR_MSG_MOD(extack, 2451 "E-Switch must either disabled or non switchdev mode"); 2452 return -EBUSY; 2453 } 2454 return 0; 2455 } 2456 2457 static const struct devlink_param esw_devlink_params[] = { 2458 DEVLINK_PARAM_DRIVER(MLX5_DEVLINK_PARAM_ID_ESW_PORT_METADATA, 2459 "esw_port_metadata", DEVLINK_PARAM_TYPE_BOOL, 2460 BIT(DEVLINK_PARAM_CMODE_RUNTIME), 2461 esw_port_metadata_get, 2462 esw_port_metadata_set, 2463 esw_port_metadata_validate), 2464 }; 2465 2466 int esw_offloads_init(struct mlx5_eswitch *esw) 2467 { 2468 int err; 2469 2470 err = esw_offloads_init_reps(esw); 2471 if (err) 2472 return err; 2473 2474 err = devl_params_register(priv_to_devlink(esw->dev), 2475 esw_devlink_params, 2476 ARRAY_SIZE(esw_devlink_params)); 2477 if (err) 2478 goto err_params; 2479 2480 return 0; 2481 2482 err_params: 2483 esw_offloads_cleanup_reps(esw); 2484 return err; 2485 } 2486 2487 void esw_offloads_cleanup(struct mlx5_eswitch *esw) 2488 { 2489 devl_params_unregister(priv_to_devlink(esw->dev), 2490 esw_devlink_params, 2491 ARRAY_SIZE(esw_devlink_params)); 2492 esw_offloads_cleanup_reps(esw); 2493 } 2494 2495 static void __esw_offloads_unload_rep(struct mlx5_eswitch *esw, 2496 struct mlx5_eswitch_rep *rep, u8 rep_type) 2497 { 2498 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2499 REP_LOADED, REP_REGISTERED) == REP_LOADED) 2500 esw->offloads.rep_ops[rep_type]->unload(rep); 2501 } 2502 2503 static void __unload_reps_all_vport(struct mlx5_eswitch *esw, u8 rep_type) 2504 { 2505 struct mlx5_eswitch_rep *rep; 2506 unsigned long i; 2507 2508 mlx5_esw_for_each_rep(esw, i, rep) 2509 __esw_offloads_unload_rep(esw, rep, rep_type); 2510 } 2511 2512 static int mlx5_esw_offloads_rep_load(struct mlx5_eswitch *esw, u16 vport_num) 2513 { 2514 struct mlx5_eswitch_rep *rep; 2515 int rep_type; 2516 int err; 2517 2518 rep = mlx5_eswitch_get_rep(esw, vport_num); 2519 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) 2520 if (atomic_cmpxchg(&rep->rep_data[rep_type].state, 2521 REP_REGISTERED, REP_LOADED) == REP_REGISTERED) { 2522 err = esw->offloads.rep_ops[rep_type]->load(esw->dev, rep); 2523 if (err) 2524 goto err_reps; 2525 } 2526 2527 return 0; 2528 2529 err_reps: 2530 atomic_set(&rep->rep_data[rep_type].state, REP_REGISTERED); 2531 for (--rep_type; rep_type >= 0; rep_type--) 2532 __esw_offloads_unload_rep(esw, rep, rep_type); 2533 return err; 2534 } 2535 2536 static void mlx5_esw_offloads_rep_unload(struct mlx5_eswitch *esw, u16 vport_num) 2537 { 2538 struct mlx5_eswitch_rep *rep; 2539 int rep_type; 2540 2541 rep = mlx5_eswitch_get_rep(esw, vport_num); 2542 for (rep_type = NUM_REP_TYPES - 1; rep_type >= 0; rep_type--) 2543 __esw_offloads_unload_rep(esw, rep, rep_type); 2544 } 2545 2546 int mlx5_esw_offloads_init_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2547 { 2548 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2549 return 0; 2550 2551 return mlx5_esw_offloads_pf_vf_devlink_port_init(esw, vport); 2552 } 2553 2554 void mlx5_esw_offloads_cleanup_pf_vf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2555 { 2556 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2557 return; 2558 2559 mlx5_esw_offloads_pf_vf_devlink_port_cleanup(esw, vport); 2560 } 2561 2562 int mlx5_esw_offloads_init_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport, 2563 struct mlx5_devlink_port *dl_port, 2564 u32 controller, u32 sfnum) 2565 { 2566 return mlx5_esw_offloads_sf_devlink_port_init(esw, vport, dl_port, controller, sfnum); 2567 } 2568 2569 void mlx5_esw_offloads_cleanup_sf_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2570 { 2571 mlx5_esw_offloads_sf_devlink_port_cleanup(esw, vport); 2572 } 2573 2574 int mlx5_esw_offloads_load_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2575 { 2576 int err; 2577 2578 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2579 return 0; 2580 2581 err = mlx5_esw_offloads_devlink_port_register(esw, vport); 2582 if (err) 2583 return err; 2584 2585 err = mlx5_esw_offloads_rep_load(esw, vport->vport); 2586 if (err) 2587 goto load_err; 2588 return err; 2589 2590 load_err: 2591 mlx5_esw_offloads_devlink_port_unregister(esw, vport); 2592 return err; 2593 } 2594 2595 void mlx5_esw_offloads_unload_rep(struct mlx5_eswitch *esw, struct mlx5_vport *vport) 2596 { 2597 if (esw->mode != MLX5_ESWITCH_OFFLOADS) 2598 return; 2599 2600 mlx5_esw_offloads_rep_unload(esw, vport->vport); 2601 2602 mlx5_esw_offloads_devlink_port_unregister(esw, vport); 2603 } 2604 2605 static int esw_set_slave_root_fdb(struct mlx5_core_dev *master, 2606 struct mlx5_core_dev *slave) 2607 { 2608 u32 in[MLX5_ST_SZ_DW(set_flow_table_root_in)] = {}; 2609 u32 out[MLX5_ST_SZ_DW(set_flow_table_root_out)] = {}; 2610 struct mlx5_flow_root_namespace *root; 2611 struct mlx5_flow_namespace *ns; 2612 int err; 2613 2614 MLX5_SET(set_flow_table_root_in, in, opcode, 2615 MLX5_CMD_OP_SET_FLOW_TABLE_ROOT); 2616 MLX5_SET(set_flow_table_root_in, in, table_type, 2617 FS_FT_FDB); 2618 2619 if (master) { 2620 ns = mlx5_get_flow_namespace(master, 2621 MLX5_FLOW_NAMESPACE_FDB); 2622 root = find_root(&ns->node); 2623 mutex_lock(&root->chain_lock); 2624 MLX5_SET(set_flow_table_root_in, in, 2625 table_eswitch_owner_vhca_id_valid, 1); 2626 MLX5_SET(set_flow_table_root_in, in, 2627 table_eswitch_owner_vhca_id, 2628 MLX5_CAP_GEN(master, vhca_id)); 2629 MLX5_SET(set_flow_table_root_in, in, table_id, 2630 root->root_ft->id); 2631 } else { 2632 ns = mlx5_get_flow_namespace(slave, 2633 MLX5_FLOW_NAMESPACE_FDB); 2634 root = find_root(&ns->node); 2635 mutex_lock(&root->chain_lock); 2636 MLX5_SET(set_flow_table_root_in, in, table_id, 2637 root->root_ft->id); 2638 } 2639 2640 err = mlx5_cmd_exec(slave, in, sizeof(in), out, sizeof(out)); 2641 mutex_unlock(&root->chain_lock); 2642 2643 return err; 2644 } 2645 2646 static int __esw_set_master_egress_rule(struct mlx5_core_dev *master, 2647 struct mlx5_core_dev *slave, 2648 struct mlx5_vport *vport, 2649 struct mlx5_flow_table *acl) 2650 { 2651 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 2652 struct mlx5_flow_handle *flow_rule = NULL; 2653 struct mlx5_flow_destination dest = {}; 2654 struct mlx5_flow_act flow_act = {}; 2655 struct mlx5_flow_spec *spec; 2656 int err = 0; 2657 void *misc; 2658 2659 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 2660 if (!spec) 2661 return -ENOMEM; 2662 2663 spec->match_criteria_enable = MLX5_MATCH_MISC_PARAMETERS; 2664 misc = MLX5_ADDR_OF(fte_match_param, spec->match_value, 2665 misc_parameters); 2666 MLX5_SET(fte_match_set_misc, misc, source_port, MLX5_VPORT_UPLINK); 2667 MLX5_SET(fte_match_set_misc, misc, source_eswitch_owner_vhca_id, slave_index); 2668 2669 misc = MLX5_ADDR_OF(fte_match_param, spec->match_criteria, misc_parameters); 2670 MLX5_SET_TO_ONES(fte_match_set_misc, misc, source_port); 2671 MLX5_SET_TO_ONES(fte_match_set_misc, misc, 2672 source_eswitch_owner_vhca_id); 2673 2674 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 2675 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 2676 dest.vport.num = slave->priv.eswitch->manager_vport; 2677 dest.vport.vhca_id = MLX5_CAP_GEN(slave, vhca_id); 2678 dest.vport.flags |= MLX5_FLOW_DEST_VPORT_VHCA_ID; 2679 2680 flow_rule = mlx5_add_flow_rules(acl, spec, &flow_act, 2681 &dest, 1); 2682 if (IS_ERR(flow_rule)) { 2683 err = PTR_ERR(flow_rule); 2684 } else { 2685 err = xa_insert(&vport->egress.offloads.bounce_rules, 2686 slave_index, flow_rule, GFP_KERNEL); 2687 if (err) 2688 mlx5_del_flow_rules(flow_rule); 2689 } 2690 2691 kvfree(spec); 2692 return err; 2693 } 2694 2695 static int esw_master_egress_create_resources(struct mlx5_eswitch *esw, 2696 struct mlx5_flow_namespace *egress_ns, 2697 struct mlx5_vport *vport, size_t count) 2698 { 2699 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 2700 struct mlx5_flow_table_attr ft_attr = { 2701 .max_fte = count, .prio = 0, .level = 0, 2702 }; 2703 struct mlx5_flow_table *acl; 2704 struct mlx5_flow_group *g; 2705 void *match_criteria; 2706 u32 *flow_group_in; 2707 int err; 2708 2709 if (vport->egress.acl) 2710 return 0; 2711 2712 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 2713 if (!flow_group_in) 2714 return -ENOMEM; 2715 2716 if (vport->vport || mlx5_core_is_ecpf(esw->dev)) 2717 ft_attr.flags = MLX5_FLOW_TABLE_OTHER_VPORT; 2718 2719 acl = mlx5_create_vport_flow_table(egress_ns, &ft_attr, vport->vport); 2720 if (IS_ERR(acl)) { 2721 err = PTR_ERR(acl); 2722 goto out; 2723 } 2724 2725 match_criteria = MLX5_ADDR_OF(create_flow_group_in, flow_group_in, 2726 match_criteria); 2727 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2728 misc_parameters.source_port); 2729 MLX5_SET_TO_ONES(fte_match_param, match_criteria, 2730 misc_parameters.source_eswitch_owner_vhca_id); 2731 MLX5_SET(create_flow_group_in, flow_group_in, match_criteria_enable, 2732 MLX5_MATCH_MISC_PARAMETERS); 2733 2734 MLX5_SET(create_flow_group_in, flow_group_in, 2735 source_eswitch_owner_vhca_id_valid, 1); 2736 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, 0); 2737 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, count); 2738 2739 g = mlx5_create_flow_group(acl, flow_group_in); 2740 if (IS_ERR(g)) { 2741 err = PTR_ERR(g); 2742 goto err_group; 2743 } 2744 2745 vport->egress.acl = acl; 2746 vport->egress.offloads.bounce_grp = g; 2747 vport->egress.type = VPORT_EGRESS_ACL_TYPE_SHARED_FDB; 2748 xa_init_flags(&vport->egress.offloads.bounce_rules, XA_FLAGS_ALLOC); 2749 2750 kvfree(flow_group_in); 2751 2752 return 0; 2753 2754 err_group: 2755 mlx5_destroy_flow_table(acl); 2756 out: 2757 kvfree(flow_group_in); 2758 return err; 2759 } 2760 2761 static void esw_master_egress_destroy_resources(struct mlx5_vport *vport) 2762 { 2763 if (!xa_empty(&vport->egress.offloads.bounce_rules)) 2764 return; 2765 mlx5_destroy_flow_group(vport->egress.offloads.bounce_grp); 2766 vport->egress.offloads.bounce_grp = NULL; 2767 mlx5_destroy_flow_table(vport->egress.acl); 2768 vport->egress.acl = NULL; 2769 } 2770 2771 static int esw_set_master_egress_rule(struct mlx5_core_dev *master, 2772 struct mlx5_core_dev *slave, size_t count) 2773 { 2774 struct mlx5_eswitch *esw = master->priv.eswitch; 2775 u16 slave_index = MLX5_CAP_GEN(slave, vhca_id); 2776 struct mlx5_flow_namespace *egress_ns; 2777 struct mlx5_vport *vport; 2778 int err; 2779 2780 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 2781 if (IS_ERR(vport)) 2782 return PTR_ERR(vport); 2783 2784 egress_ns = mlx5_get_flow_vport_acl_namespace(master, 2785 MLX5_FLOW_NAMESPACE_ESW_EGRESS, 2786 vport->index); 2787 if (!egress_ns) 2788 return -EINVAL; 2789 2790 if (vport->egress.acl && vport->egress.type != VPORT_EGRESS_ACL_TYPE_SHARED_FDB) 2791 return 0; 2792 2793 err = esw_master_egress_create_resources(esw, egress_ns, vport, count); 2794 if (err) 2795 return err; 2796 2797 if (xa_load(&vport->egress.offloads.bounce_rules, slave_index)) 2798 return -EINVAL; 2799 2800 err = __esw_set_master_egress_rule(master, slave, vport, vport->egress.acl); 2801 if (err) 2802 goto err_rule; 2803 2804 return 0; 2805 2806 err_rule: 2807 esw_master_egress_destroy_resources(vport); 2808 return err; 2809 } 2810 2811 static void esw_unset_master_egress_rule(struct mlx5_core_dev *dev, 2812 struct mlx5_core_dev *slave_dev) 2813 { 2814 struct mlx5_vport *vport; 2815 2816 vport = mlx5_eswitch_get_vport(dev->priv.eswitch, 2817 dev->priv.eswitch->manager_vport); 2818 2819 esw_acl_egress_ofld_bounce_rule_destroy(vport, MLX5_CAP_GEN(slave_dev, vhca_id)); 2820 2821 if (xa_empty(&vport->egress.offloads.bounce_rules)) { 2822 esw_acl_egress_ofld_cleanup(vport); 2823 xa_destroy(&vport->egress.offloads.bounce_rules); 2824 } 2825 } 2826 2827 int mlx5_eswitch_offloads_single_fdb_add_one(struct mlx5_eswitch *master_esw, 2828 struct mlx5_eswitch *slave_esw, int max_slaves) 2829 { 2830 int err; 2831 2832 err = esw_set_slave_root_fdb(master_esw->dev, 2833 slave_esw->dev); 2834 if (err) 2835 return err; 2836 2837 err = esw_set_master_egress_rule(master_esw->dev, 2838 slave_esw->dev, max_slaves); 2839 if (err) 2840 goto err_acl; 2841 2842 return err; 2843 2844 err_acl: 2845 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2846 return err; 2847 } 2848 2849 void mlx5_eswitch_offloads_single_fdb_del_one(struct mlx5_eswitch *master_esw, 2850 struct mlx5_eswitch *slave_esw) 2851 { 2852 esw_set_slave_root_fdb(NULL, slave_esw->dev); 2853 esw_unset_master_egress_rule(master_esw->dev, slave_esw->dev); 2854 } 2855 2856 #define ESW_OFFLOADS_DEVCOM_PAIR (0) 2857 #define ESW_OFFLOADS_DEVCOM_UNPAIR (1) 2858 2859 static void mlx5_esw_offloads_rep_event_unpair(struct mlx5_eswitch *esw, 2860 struct mlx5_eswitch *peer_esw) 2861 { 2862 const struct mlx5_eswitch_rep_ops *ops; 2863 struct mlx5_eswitch_rep *rep; 2864 unsigned long i; 2865 u8 rep_type; 2866 2867 mlx5_esw_for_each_rep(esw, i, rep) { 2868 rep_type = NUM_REP_TYPES; 2869 while (rep_type--) { 2870 ops = esw->offloads.rep_ops[rep_type]; 2871 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2872 ops->event) 2873 ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_UNPAIR, peer_esw); 2874 } 2875 } 2876 } 2877 2878 static void mlx5_esw_offloads_unpair(struct mlx5_eswitch *esw, 2879 struct mlx5_eswitch *peer_esw) 2880 { 2881 #if IS_ENABLED(CONFIG_MLX5_CLS_ACT) 2882 mlx5e_tc_clean_fdb_peer_flows(esw); 2883 #endif 2884 mlx5_esw_offloads_rep_event_unpair(esw, peer_esw); 2885 esw_del_fdb_peer_miss_rules(esw, peer_esw->dev); 2886 } 2887 2888 static int mlx5_esw_offloads_pair(struct mlx5_eswitch *esw, 2889 struct mlx5_eswitch *peer_esw) 2890 { 2891 const struct mlx5_eswitch_rep_ops *ops; 2892 struct mlx5_eswitch_rep *rep; 2893 unsigned long i; 2894 u8 rep_type; 2895 int err; 2896 2897 err = esw_add_fdb_peer_miss_rules(esw, peer_esw->dev); 2898 if (err) 2899 return err; 2900 2901 mlx5_esw_for_each_rep(esw, i, rep) { 2902 for (rep_type = 0; rep_type < NUM_REP_TYPES; rep_type++) { 2903 ops = esw->offloads.rep_ops[rep_type]; 2904 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 2905 ops->event) { 2906 err = ops->event(esw, rep, MLX5_SWITCHDEV_EVENT_PAIR, peer_esw); 2907 if (err) 2908 goto err_out; 2909 } 2910 } 2911 } 2912 2913 return 0; 2914 2915 err_out: 2916 mlx5_esw_offloads_unpair(esw, peer_esw); 2917 return err; 2918 } 2919 2920 static int mlx5_esw_offloads_set_ns_peer(struct mlx5_eswitch *esw, 2921 struct mlx5_eswitch *peer_esw, 2922 bool pair) 2923 { 2924 u16 peer_vhca_id = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 2925 u16 vhca_id = MLX5_CAP_GEN(esw->dev, vhca_id); 2926 struct mlx5_flow_root_namespace *peer_ns; 2927 struct mlx5_flow_root_namespace *ns; 2928 int err; 2929 2930 peer_ns = peer_esw->dev->priv.steering->fdb_root_ns; 2931 ns = esw->dev->priv.steering->fdb_root_ns; 2932 2933 if (pair) { 2934 err = mlx5_flow_namespace_set_peer(ns, peer_ns, peer_vhca_id); 2935 if (err) 2936 return err; 2937 2938 err = mlx5_flow_namespace_set_peer(peer_ns, ns, vhca_id); 2939 if (err) { 2940 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 2941 return err; 2942 } 2943 } else { 2944 mlx5_flow_namespace_set_peer(ns, NULL, peer_vhca_id); 2945 mlx5_flow_namespace_set_peer(peer_ns, NULL, vhca_id); 2946 } 2947 2948 return 0; 2949 } 2950 2951 static int mlx5_esw_offloads_devcom_event(int event, 2952 void *my_data, 2953 void *event_data) 2954 { 2955 struct mlx5_eswitch *esw = my_data; 2956 struct mlx5_eswitch *peer_esw = event_data; 2957 u16 esw_i, peer_esw_i; 2958 bool esw_paired; 2959 int err; 2960 2961 peer_esw_i = MLX5_CAP_GEN(peer_esw->dev, vhca_id); 2962 esw_i = MLX5_CAP_GEN(esw->dev, vhca_id); 2963 esw_paired = !!xa_load(&esw->paired, peer_esw_i); 2964 2965 switch (event) { 2966 case ESW_OFFLOADS_DEVCOM_PAIR: 2967 if (mlx5_eswitch_vport_match_metadata_enabled(esw) != 2968 mlx5_eswitch_vport_match_metadata_enabled(peer_esw)) 2969 break; 2970 2971 if (esw_paired) 2972 break; 2973 2974 err = mlx5_esw_offloads_set_ns_peer(esw, peer_esw, true); 2975 if (err) 2976 goto err_out; 2977 2978 err = mlx5_esw_offloads_pair(esw, peer_esw); 2979 if (err) 2980 goto err_peer; 2981 2982 err = mlx5_esw_offloads_pair(peer_esw, esw); 2983 if (err) 2984 goto err_pair; 2985 2986 err = xa_insert(&esw->paired, peer_esw_i, peer_esw, GFP_KERNEL); 2987 if (err) 2988 goto err_xa; 2989 2990 err = xa_insert(&peer_esw->paired, esw_i, esw, GFP_KERNEL); 2991 if (err) 2992 goto err_peer_xa; 2993 2994 esw->num_peers++; 2995 peer_esw->num_peers++; 2996 mlx5_devcom_comp_set_ready(esw->devcom, true); 2997 break; 2998 2999 case ESW_OFFLOADS_DEVCOM_UNPAIR: 3000 if (!esw_paired) 3001 break; 3002 3003 peer_esw->num_peers--; 3004 esw->num_peers--; 3005 if (!esw->num_peers && !peer_esw->num_peers) 3006 mlx5_devcom_comp_set_ready(esw->devcom, false); 3007 xa_erase(&peer_esw->paired, esw_i); 3008 xa_erase(&esw->paired, peer_esw_i); 3009 mlx5_esw_offloads_unpair(peer_esw, esw); 3010 mlx5_esw_offloads_unpair(esw, peer_esw); 3011 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3012 break; 3013 } 3014 3015 return 0; 3016 3017 err_peer_xa: 3018 xa_erase(&esw->paired, peer_esw_i); 3019 err_xa: 3020 mlx5_esw_offloads_unpair(peer_esw, esw); 3021 err_pair: 3022 mlx5_esw_offloads_unpair(esw, peer_esw); 3023 err_peer: 3024 mlx5_esw_offloads_set_ns_peer(esw, peer_esw, false); 3025 err_out: 3026 mlx5_core_err(esw->dev, "esw offloads devcom event failure, event %u err %d", 3027 event, err); 3028 return err; 3029 } 3030 3031 void mlx5_esw_offloads_devcom_init(struct mlx5_eswitch *esw, u64 key) 3032 { 3033 int i; 3034 3035 for (i = 0; i < MLX5_MAX_PORTS; i++) 3036 INIT_LIST_HEAD(&esw->offloads.peer_flows[i]); 3037 mutex_init(&esw->offloads.peer_mutex); 3038 3039 if (!MLX5_CAP_ESW(esw->dev, merged_eswitch)) 3040 return; 3041 3042 if ((MLX5_VPORT_MANAGER(esw->dev) || mlx5_core_is_ecpf_esw_manager(esw->dev)) && 3043 !mlx5_lag_is_supported(esw->dev)) 3044 return; 3045 3046 xa_init(&esw->paired); 3047 esw->num_peers = 0; 3048 esw->devcom = mlx5_devcom_register_component(esw->dev->priv.devc, 3049 MLX5_DEVCOM_ESW_OFFLOADS, 3050 key, 3051 mlx5_esw_offloads_devcom_event, 3052 esw); 3053 if (IS_ERR_OR_NULL(esw->devcom)) 3054 return; 3055 3056 mlx5_devcom_send_event(esw->devcom, 3057 ESW_OFFLOADS_DEVCOM_PAIR, 3058 ESW_OFFLOADS_DEVCOM_UNPAIR, 3059 esw); 3060 } 3061 3062 void mlx5_esw_offloads_devcom_cleanup(struct mlx5_eswitch *esw) 3063 { 3064 if (IS_ERR_OR_NULL(esw->devcom)) 3065 return; 3066 3067 mlx5_devcom_send_event(esw->devcom, 3068 ESW_OFFLOADS_DEVCOM_UNPAIR, 3069 ESW_OFFLOADS_DEVCOM_UNPAIR, 3070 esw); 3071 3072 mlx5_devcom_unregister_component(esw->devcom); 3073 xa_destroy(&esw->paired); 3074 esw->devcom = NULL; 3075 } 3076 3077 bool mlx5_esw_offloads_devcom_is_ready(struct mlx5_eswitch *esw) 3078 { 3079 return mlx5_devcom_comp_is_ready(esw->devcom); 3080 } 3081 3082 bool mlx5_esw_vport_match_metadata_supported(const struct mlx5_eswitch *esw) 3083 { 3084 if (!MLX5_CAP_ESW(esw->dev, esw_uplink_ingress_acl)) 3085 return false; 3086 3087 if (!(MLX5_CAP_ESW_FLOWTABLE(esw->dev, fdb_to_vport_reg_c_id) & 3088 MLX5_FDB_TO_VPORT_REG_C_0)) 3089 return false; 3090 3091 return true; 3092 } 3093 3094 #define MLX5_ESW_METADATA_RSVD_UPLINK 1 3095 3096 /* Share the same metadata for uplink's. This is fine because: 3097 * (a) In shared FDB mode (LAG) both uplink's are treated the 3098 * same and tagged with the same metadata. 3099 * (b) In non shared FDB mode, packets from physical port0 3100 * cannot hit eswitch of PF1 and vice versa. 3101 */ 3102 static u32 mlx5_esw_match_metadata_reserved(struct mlx5_eswitch *esw) 3103 { 3104 return MLX5_ESW_METADATA_RSVD_UPLINK; 3105 } 3106 3107 u32 mlx5_esw_match_metadata_alloc(struct mlx5_eswitch *esw) 3108 { 3109 u32 vport_end_ida = (1 << ESW_VPORT_BITS) - 1; 3110 /* Reserve 0xf for internal port offload */ 3111 u32 max_pf_num = (1 << ESW_PFNUM_BITS) - 2; 3112 u32 pf_num; 3113 int id; 3114 3115 /* Only 4 bits of pf_num */ 3116 pf_num = mlx5_get_dev_index(esw->dev); 3117 if (pf_num > max_pf_num) 3118 return 0; 3119 3120 /* Metadata is 4 bits of PFNUM and 12 bits of unique id */ 3121 /* Use only non-zero vport_id (2-4095) for all PF's */ 3122 id = ida_alloc_range(&esw->offloads.vport_metadata_ida, 3123 MLX5_ESW_METADATA_RSVD_UPLINK + 1, 3124 vport_end_ida, GFP_KERNEL); 3125 if (id < 0) 3126 return 0; 3127 id = (pf_num << ESW_VPORT_BITS) | id; 3128 return id; 3129 } 3130 3131 void mlx5_esw_match_metadata_free(struct mlx5_eswitch *esw, u32 metadata) 3132 { 3133 u32 vport_bit_mask = (1 << ESW_VPORT_BITS) - 1; 3134 3135 /* Metadata contains only 12 bits of actual ida id */ 3136 ida_free(&esw->offloads.vport_metadata_ida, metadata & vport_bit_mask); 3137 } 3138 3139 static int esw_offloads_vport_metadata_setup(struct mlx5_eswitch *esw, 3140 struct mlx5_vport *vport) 3141 { 3142 if (vport->vport == MLX5_VPORT_UPLINK) 3143 vport->default_metadata = mlx5_esw_match_metadata_reserved(esw); 3144 else 3145 vport->default_metadata = mlx5_esw_match_metadata_alloc(esw); 3146 3147 vport->metadata = vport->default_metadata; 3148 return vport->metadata ? 0 : -ENOSPC; 3149 } 3150 3151 static void esw_offloads_vport_metadata_cleanup(struct mlx5_eswitch *esw, 3152 struct mlx5_vport *vport) 3153 { 3154 if (!vport->default_metadata) 3155 return; 3156 3157 if (vport->vport == MLX5_VPORT_UPLINK) 3158 return; 3159 3160 WARN_ON(vport->metadata != vport->default_metadata); 3161 mlx5_esw_match_metadata_free(esw, vport->default_metadata); 3162 } 3163 3164 static void esw_offloads_metadata_uninit(struct mlx5_eswitch *esw) 3165 { 3166 struct mlx5_vport *vport; 3167 unsigned long i; 3168 3169 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3170 return; 3171 3172 mlx5_esw_for_each_vport(esw, i, vport) 3173 esw_offloads_vport_metadata_cleanup(esw, vport); 3174 } 3175 3176 static int esw_offloads_metadata_init(struct mlx5_eswitch *esw) 3177 { 3178 struct mlx5_vport *vport; 3179 unsigned long i; 3180 int err; 3181 3182 if (!mlx5_eswitch_vport_match_metadata_enabled(esw)) 3183 return 0; 3184 3185 mlx5_esw_for_each_vport(esw, i, vport) { 3186 err = esw_offloads_vport_metadata_setup(esw, vport); 3187 if (err) 3188 goto metadata_err; 3189 } 3190 3191 return 0; 3192 3193 metadata_err: 3194 esw_offloads_metadata_uninit(esw); 3195 return err; 3196 } 3197 3198 int 3199 esw_vport_create_offloads_acl_tables(struct mlx5_eswitch *esw, 3200 struct mlx5_vport *vport) 3201 { 3202 int err; 3203 3204 err = esw_acl_ingress_ofld_setup(esw, vport); 3205 if (err) 3206 return err; 3207 3208 err = esw_acl_egress_ofld_setup(esw, vport); 3209 if (err) 3210 goto egress_err; 3211 3212 return 0; 3213 3214 egress_err: 3215 esw_acl_ingress_ofld_cleanup(esw, vport); 3216 return err; 3217 } 3218 3219 void 3220 esw_vport_destroy_offloads_acl_tables(struct mlx5_eswitch *esw, 3221 struct mlx5_vport *vport) 3222 { 3223 esw_acl_egress_ofld_cleanup(vport); 3224 esw_acl_ingress_ofld_cleanup(esw, vport); 3225 } 3226 3227 static int esw_create_offloads_acl_tables(struct mlx5_eswitch *esw) 3228 { 3229 struct mlx5_vport *uplink, *manager; 3230 int ret; 3231 3232 uplink = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3233 if (IS_ERR(uplink)) 3234 return PTR_ERR(uplink); 3235 3236 ret = esw_vport_create_offloads_acl_tables(esw, uplink); 3237 if (ret) 3238 return ret; 3239 3240 manager = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3241 if (IS_ERR(manager)) { 3242 ret = PTR_ERR(manager); 3243 goto err_manager; 3244 } 3245 3246 ret = esw_vport_create_offloads_acl_tables(esw, manager); 3247 if (ret) 3248 goto err_manager; 3249 3250 return 0; 3251 3252 err_manager: 3253 esw_vport_destroy_offloads_acl_tables(esw, uplink); 3254 return ret; 3255 } 3256 3257 static void esw_destroy_offloads_acl_tables(struct mlx5_eswitch *esw) 3258 { 3259 struct mlx5_vport *vport; 3260 3261 vport = mlx5_eswitch_get_vport(esw, esw->manager_vport); 3262 if (!IS_ERR(vport)) 3263 esw_vport_destroy_offloads_acl_tables(esw, vport); 3264 3265 vport = mlx5_eswitch_get_vport(esw, MLX5_VPORT_UPLINK); 3266 if (!IS_ERR(vport)) 3267 esw_vport_destroy_offloads_acl_tables(esw, vport); 3268 } 3269 3270 int mlx5_eswitch_reload_reps(struct mlx5_eswitch *esw) 3271 { 3272 struct mlx5_eswitch_rep *rep; 3273 unsigned long i; 3274 int ret; 3275 3276 if (!esw || esw->mode != MLX5_ESWITCH_OFFLOADS) 3277 return 0; 3278 3279 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 3280 if (atomic_read(&rep->rep_data[REP_ETH].state) != REP_LOADED) 3281 return 0; 3282 3283 ret = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3284 if (ret) 3285 return ret; 3286 3287 mlx5_esw_for_each_rep(esw, i, rep) { 3288 if (atomic_read(&rep->rep_data[REP_ETH].state) == REP_LOADED) 3289 mlx5_esw_offloads_rep_load(esw, rep->vport); 3290 } 3291 3292 return 0; 3293 } 3294 3295 static int esw_offloads_steering_init(struct mlx5_eswitch *esw) 3296 { 3297 struct mlx5_esw_indir_table *indir; 3298 int err; 3299 3300 memset(&esw->fdb_table.offloads, 0, sizeof(struct offloads_fdb)); 3301 mutex_init(&esw->fdb_table.offloads.vports.lock); 3302 hash_init(esw->fdb_table.offloads.vports.table); 3303 atomic64_set(&esw->user_count, 0); 3304 3305 indir = mlx5_esw_indir_table_init(); 3306 if (IS_ERR(indir)) { 3307 err = PTR_ERR(indir); 3308 goto create_indir_err; 3309 } 3310 esw->fdb_table.offloads.indir = indir; 3311 3312 err = esw_create_offloads_acl_tables(esw); 3313 if (err) 3314 goto create_acl_err; 3315 3316 err = esw_create_offloads_table(esw); 3317 if (err) 3318 goto create_offloads_err; 3319 3320 err = esw_create_restore_table(esw); 3321 if (err) 3322 goto create_restore_err; 3323 3324 err = esw_create_offloads_fdb_tables(esw); 3325 if (err) 3326 goto create_fdb_err; 3327 3328 err = esw_create_vport_rx_group(esw); 3329 if (err) 3330 goto create_fg_err; 3331 3332 err = esw_create_vport_rx_drop_group(esw); 3333 if (err) 3334 goto create_rx_drop_fg_err; 3335 3336 err = esw_create_vport_rx_drop_rule(esw); 3337 if (err) 3338 goto create_rx_drop_rule_err; 3339 3340 return 0; 3341 3342 create_rx_drop_rule_err: 3343 esw_destroy_vport_rx_drop_group(esw); 3344 create_rx_drop_fg_err: 3345 esw_destroy_vport_rx_group(esw); 3346 create_fg_err: 3347 esw_destroy_offloads_fdb_tables(esw); 3348 create_fdb_err: 3349 esw_destroy_restore_table(esw); 3350 create_restore_err: 3351 esw_destroy_offloads_table(esw); 3352 create_offloads_err: 3353 esw_destroy_offloads_acl_tables(esw); 3354 create_acl_err: 3355 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3356 create_indir_err: 3357 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3358 return err; 3359 } 3360 3361 static void esw_offloads_steering_cleanup(struct mlx5_eswitch *esw) 3362 { 3363 esw_destroy_vport_rx_drop_rule(esw); 3364 esw_destroy_vport_rx_drop_group(esw); 3365 esw_destroy_vport_rx_group(esw); 3366 esw_destroy_offloads_fdb_tables(esw); 3367 esw_destroy_restore_table(esw); 3368 esw_destroy_offloads_table(esw); 3369 esw_destroy_offloads_acl_tables(esw); 3370 mlx5_esw_indir_table_destroy(esw->fdb_table.offloads.indir); 3371 mutex_destroy(&esw->fdb_table.offloads.vports.lock); 3372 } 3373 3374 static void 3375 esw_vfs_changed_event_handler(struct mlx5_eswitch *esw, const u32 *out) 3376 { 3377 struct devlink *devlink; 3378 bool host_pf_disabled; 3379 u16 new_num_vfs; 3380 3381 new_num_vfs = MLX5_GET(query_esw_functions_out, out, 3382 host_params_context.host_num_of_vfs); 3383 host_pf_disabled = MLX5_GET(query_esw_functions_out, out, 3384 host_params_context.host_pf_disabled); 3385 3386 if (new_num_vfs == esw->esw_funcs.num_vfs || host_pf_disabled) 3387 return; 3388 3389 devlink = priv_to_devlink(esw->dev); 3390 devl_lock(devlink); 3391 /* Number of VFs can only change from "0 to x" or "x to 0". */ 3392 if (esw->esw_funcs.num_vfs > 0) { 3393 mlx5_eswitch_unload_vf_vports(esw, esw->esw_funcs.num_vfs); 3394 } else { 3395 int err; 3396 3397 err = mlx5_eswitch_load_vf_vports(esw, new_num_vfs, 3398 MLX5_VPORT_UC_ADDR_CHANGE); 3399 if (err) { 3400 devl_unlock(devlink); 3401 return; 3402 } 3403 } 3404 esw->esw_funcs.num_vfs = new_num_vfs; 3405 devl_unlock(devlink); 3406 } 3407 3408 static void esw_functions_changed_event_handler(struct work_struct *work) 3409 { 3410 struct mlx5_host_work *host_work; 3411 struct mlx5_eswitch *esw; 3412 const u32 *out; 3413 3414 host_work = container_of(work, struct mlx5_host_work, work); 3415 esw = host_work->esw; 3416 3417 out = mlx5_esw_query_functions(esw->dev); 3418 if (IS_ERR(out)) 3419 goto out; 3420 3421 esw_vfs_changed_event_handler(esw, out); 3422 kvfree(out); 3423 out: 3424 kfree(host_work); 3425 } 3426 3427 int mlx5_esw_funcs_changed_handler(struct notifier_block *nb, unsigned long type, void *data) 3428 { 3429 struct mlx5_esw_functions *esw_funcs; 3430 struct mlx5_host_work *host_work; 3431 struct mlx5_eswitch *esw; 3432 3433 host_work = kzalloc(sizeof(*host_work), GFP_ATOMIC); 3434 if (!host_work) 3435 return NOTIFY_DONE; 3436 3437 esw_funcs = mlx5_nb_cof(nb, struct mlx5_esw_functions, nb); 3438 esw = container_of(esw_funcs, struct mlx5_eswitch, esw_funcs); 3439 3440 host_work->esw = esw; 3441 3442 INIT_WORK(&host_work->work, esw_functions_changed_event_handler); 3443 queue_work(esw->work_queue, &host_work->work); 3444 3445 return NOTIFY_OK; 3446 } 3447 3448 static int mlx5_esw_host_number_init(struct mlx5_eswitch *esw) 3449 { 3450 const u32 *query_host_out; 3451 3452 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3453 return 0; 3454 3455 query_host_out = mlx5_esw_query_functions(esw->dev); 3456 if (IS_ERR(query_host_out)) 3457 return PTR_ERR(query_host_out); 3458 3459 /* Mark non local controller with non zero controller number. */ 3460 esw->offloads.host_number = MLX5_GET(query_esw_functions_out, query_host_out, 3461 host_params_context.host_number); 3462 kvfree(query_host_out); 3463 return 0; 3464 } 3465 3466 bool mlx5_esw_offloads_controller_valid(const struct mlx5_eswitch *esw, u32 controller) 3467 { 3468 /* Local controller is always valid */ 3469 if (controller == 0) 3470 return true; 3471 3472 if (!mlx5_core_is_ecpf_esw_manager(esw->dev)) 3473 return false; 3474 3475 /* External host number starts with zero in device */ 3476 return (controller == esw->offloads.host_number + 1); 3477 } 3478 3479 int esw_offloads_enable(struct mlx5_eswitch *esw) 3480 { 3481 struct mapping_ctx *reg_c0_obj_pool; 3482 struct mlx5_vport *vport; 3483 unsigned long i; 3484 u64 mapping_id; 3485 int err; 3486 3487 mutex_init(&esw->offloads.termtbl_mutex); 3488 mlx5_rdma_enable_roce(esw->dev); 3489 3490 err = mlx5_esw_host_number_init(esw); 3491 if (err) 3492 goto err_metadata; 3493 3494 err = esw_offloads_metadata_init(esw); 3495 if (err) 3496 goto err_metadata; 3497 3498 err = esw_set_passing_vport_metadata(esw, true); 3499 if (err) 3500 goto err_vport_metadata; 3501 3502 mapping_id = mlx5_query_nic_system_image_guid(esw->dev); 3503 3504 reg_c0_obj_pool = mapping_create_for_id(mapping_id, MAPPING_TYPE_CHAIN, 3505 sizeof(struct mlx5_mapped_obj), 3506 ESW_REG_C0_USER_DATA_METADATA_MASK, 3507 true); 3508 3509 if (IS_ERR(reg_c0_obj_pool)) { 3510 err = PTR_ERR(reg_c0_obj_pool); 3511 goto err_pool; 3512 } 3513 esw->offloads.reg_c0_obj_pool = reg_c0_obj_pool; 3514 3515 err = esw_offloads_steering_init(esw); 3516 if (err) 3517 goto err_steering_init; 3518 3519 /* Representor will control the vport link state */ 3520 mlx5_esw_for_each_vf_vport(esw, i, vport, esw->esw_funcs.num_vfs) 3521 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3522 if (mlx5_core_ec_sriov_enabled(esw->dev)) 3523 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) 3524 vport->info.link_state = MLX5_VPORT_ADMIN_STATE_DOWN; 3525 3526 /* Uplink vport rep must load first. */ 3527 err = mlx5_esw_offloads_rep_load(esw, MLX5_VPORT_UPLINK); 3528 if (err) 3529 goto err_uplink; 3530 3531 err = mlx5_eswitch_enable_pf_vf_vports(esw, MLX5_VPORT_UC_ADDR_CHANGE); 3532 if (err) 3533 goto err_vports; 3534 3535 return 0; 3536 3537 err_vports: 3538 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3539 err_uplink: 3540 esw_offloads_steering_cleanup(esw); 3541 err_steering_init: 3542 mapping_destroy(reg_c0_obj_pool); 3543 err_pool: 3544 esw_set_passing_vport_metadata(esw, false); 3545 err_vport_metadata: 3546 esw_offloads_metadata_uninit(esw); 3547 err_metadata: 3548 mlx5_rdma_disable_roce(esw->dev); 3549 mutex_destroy(&esw->offloads.termtbl_mutex); 3550 return err; 3551 } 3552 3553 static int esw_offloads_stop(struct mlx5_eswitch *esw, 3554 struct netlink_ext_ack *extack) 3555 { 3556 int err; 3557 3558 esw->mode = MLX5_ESWITCH_LEGACY; 3559 3560 /* If changing from switchdev to legacy mode without sriov enabled, 3561 * no need to create legacy fdb. 3562 */ 3563 if (!mlx5_core_is_pf(esw->dev) || !mlx5_sriov_is_enabled(esw->dev)) 3564 return 0; 3565 3566 err = mlx5_eswitch_enable_locked(esw, MLX5_ESWITCH_IGNORE_NUM_VFS); 3567 if (err) 3568 NL_SET_ERR_MSG_MOD(extack, "Failed setting eswitch to legacy"); 3569 3570 return err; 3571 } 3572 3573 void esw_offloads_disable(struct mlx5_eswitch *esw) 3574 { 3575 mlx5_eswitch_disable_pf_vf_vports(esw); 3576 mlx5_esw_offloads_rep_unload(esw, MLX5_VPORT_UPLINK); 3577 esw_set_passing_vport_metadata(esw, false); 3578 esw_offloads_steering_cleanup(esw); 3579 mapping_destroy(esw->offloads.reg_c0_obj_pool); 3580 esw_offloads_metadata_uninit(esw); 3581 mlx5_rdma_disable_roce(esw->dev); 3582 mutex_destroy(&esw->offloads.termtbl_mutex); 3583 } 3584 3585 static int esw_mode_from_devlink(u16 mode, u16 *mlx5_mode) 3586 { 3587 switch (mode) { 3588 case DEVLINK_ESWITCH_MODE_LEGACY: 3589 *mlx5_mode = MLX5_ESWITCH_LEGACY; 3590 break; 3591 case DEVLINK_ESWITCH_MODE_SWITCHDEV: 3592 *mlx5_mode = MLX5_ESWITCH_OFFLOADS; 3593 break; 3594 default: 3595 return -EINVAL; 3596 } 3597 3598 return 0; 3599 } 3600 3601 static int esw_mode_to_devlink(u16 mlx5_mode, u16 *mode) 3602 { 3603 switch (mlx5_mode) { 3604 case MLX5_ESWITCH_LEGACY: 3605 *mode = DEVLINK_ESWITCH_MODE_LEGACY; 3606 break; 3607 case MLX5_ESWITCH_OFFLOADS: 3608 *mode = DEVLINK_ESWITCH_MODE_SWITCHDEV; 3609 break; 3610 default: 3611 return -EINVAL; 3612 } 3613 3614 return 0; 3615 } 3616 3617 static int esw_inline_mode_from_devlink(u8 mode, u8 *mlx5_mode) 3618 { 3619 switch (mode) { 3620 case DEVLINK_ESWITCH_INLINE_MODE_NONE: 3621 *mlx5_mode = MLX5_INLINE_MODE_NONE; 3622 break; 3623 case DEVLINK_ESWITCH_INLINE_MODE_LINK: 3624 *mlx5_mode = MLX5_INLINE_MODE_L2; 3625 break; 3626 case DEVLINK_ESWITCH_INLINE_MODE_NETWORK: 3627 *mlx5_mode = MLX5_INLINE_MODE_IP; 3628 break; 3629 case DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT: 3630 *mlx5_mode = MLX5_INLINE_MODE_TCP_UDP; 3631 break; 3632 default: 3633 return -EINVAL; 3634 } 3635 3636 return 0; 3637 } 3638 3639 static int esw_inline_mode_to_devlink(u8 mlx5_mode, u8 *mode) 3640 { 3641 switch (mlx5_mode) { 3642 case MLX5_INLINE_MODE_NONE: 3643 *mode = DEVLINK_ESWITCH_INLINE_MODE_NONE; 3644 break; 3645 case MLX5_INLINE_MODE_L2: 3646 *mode = DEVLINK_ESWITCH_INLINE_MODE_LINK; 3647 break; 3648 case MLX5_INLINE_MODE_IP: 3649 *mode = DEVLINK_ESWITCH_INLINE_MODE_NETWORK; 3650 break; 3651 case MLX5_INLINE_MODE_TCP_UDP: 3652 *mode = DEVLINK_ESWITCH_INLINE_MODE_TRANSPORT; 3653 break; 3654 default: 3655 return -EINVAL; 3656 } 3657 3658 return 0; 3659 } 3660 3661 static bool esw_offloads_devlink_ns_eq_netdev_ns(struct devlink *devlink) 3662 { 3663 struct mlx5_core_dev *dev = devlink_priv(devlink); 3664 struct net *devl_net, *netdev_net; 3665 bool ret = false; 3666 3667 mutex_lock(&dev->mlx5e_res.uplink_netdev_lock); 3668 if (dev->mlx5e_res.uplink_netdev) { 3669 netdev_net = dev_net(dev->mlx5e_res.uplink_netdev); 3670 devl_net = devlink_net(devlink); 3671 ret = net_eq(devl_net, netdev_net); 3672 } 3673 mutex_unlock(&dev->mlx5e_res.uplink_netdev_lock); 3674 return ret; 3675 } 3676 3677 int mlx5_eswitch_block_mode(struct mlx5_core_dev *dev) 3678 { 3679 struct mlx5_eswitch *esw = dev->priv.eswitch; 3680 int err; 3681 3682 if (!mlx5_esw_allowed(esw)) 3683 return 0; 3684 3685 /* Take TC into account */ 3686 err = mlx5_esw_try_lock(esw); 3687 if (err < 0) 3688 return err; 3689 3690 esw->offloads.num_block_mode++; 3691 mlx5_esw_unlock(esw); 3692 return 0; 3693 } 3694 3695 void mlx5_eswitch_unblock_mode(struct mlx5_core_dev *dev) 3696 { 3697 struct mlx5_eswitch *esw = dev->priv.eswitch; 3698 3699 if (!mlx5_esw_allowed(esw)) 3700 return; 3701 3702 down_write(&esw->mode_lock); 3703 esw->offloads.num_block_mode--; 3704 up_write(&esw->mode_lock); 3705 } 3706 3707 int mlx5_devlink_eswitch_mode_set(struct devlink *devlink, u16 mode, 3708 struct netlink_ext_ack *extack) 3709 { 3710 u16 cur_mlx5_mode, mlx5_mode = 0; 3711 struct mlx5_eswitch *esw; 3712 int err = 0; 3713 3714 esw = mlx5_devlink_eswitch_get(devlink); 3715 if (IS_ERR(esw)) 3716 return PTR_ERR(esw); 3717 3718 if (esw_mode_from_devlink(mode, &mlx5_mode)) 3719 return -EINVAL; 3720 3721 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV && 3722 !esw_offloads_devlink_ns_eq_netdev_ns(devlink)) { 3723 NL_SET_ERR_MSG_MOD(extack, 3724 "Can't change E-Switch mode to switchdev when netdev net namespace has diverged from the devlink's."); 3725 return -EPERM; 3726 } 3727 3728 mlx5_lag_disable_change(esw->dev); 3729 err = mlx5_esw_try_lock(esw); 3730 if (err < 0) { 3731 NL_SET_ERR_MSG_MOD(extack, "Can't change mode, E-Switch is busy"); 3732 goto enable_lag; 3733 } 3734 cur_mlx5_mode = err; 3735 err = 0; 3736 3737 if (cur_mlx5_mode == mlx5_mode) 3738 goto unlock; 3739 3740 if (esw->offloads.num_block_mode) { 3741 NL_SET_ERR_MSG_MOD(extack, 3742 "Can't change eswitch mode when IPsec SA and/or policies are configured"); 3743 err = -EOPNOTSUPP; 3744 goto unlock; 3745 } 3746 3747 esw->eswitch_operation_in_progress = true; 3748 up_write(&esw->mode_lock); 3749 3750 mlx5_eswitch_disable_locked(esw); 3751 if (mode == DEVLINK_ESWITCH_MODE_SWITCHDEV) { 3752 if (mlx5_devlink_trap_get_num_active(esw->dev)) { 3753 NL_SET_ERR_MSG_MOD(extack, 3754 "Can't change mode while devlink traps are active"); 3755 err = -EOPNOTSUPP; 3756 goto skip; 3757 } 3758 err = esw_offloads_start(esw, extack); 3759 } else if (mode == DEVLINK_ESWITCH_MODE_LEGACY) { 3760 err = esw_offloads_stop(esw, extack); 3761 mlx5_rescan_drivers(esw->dev); 3762 } else { 3763 err = -EINVAL; 3764 } 3765 3766 skip: 3767 down_write(&esw->mode_lock); 3768 esw->eswitch_operation_in_progress = false; 3769 unlock: 3770 mlx5_esw_unlock(esw); 3771 enable_lag: 3772 mlx5_lag_enable_change(esw->dev); 3773 return err; 3774 } 3775 3776 int mlx5_devlink_eswitch_mode_get(struct devlink *devlink, u16 *mode) 3777 { 3778 struct mlx5_eswitch *esw; 3779 3780 esw = mlx5_devlink_eswitch_get(devlink); 3781 if (IS_ERR(esw)) 3782 return PTR_ERR(esw); 3783 3784 return esw_mode_to_devlink(esw->mode, mode); 3785 } 3786 3787 static int mlx5_esw_vports_inline_set(struct mlx5_eswitch *esw, u8 mlx5_mode, 3788 struct netlink_ext_ack *extack) 3789 { 3790 struct mlx5_core_dev *dev = esw->dev; 3791 struct mlx5_vport *vport; 3792 u16 err_vport_num = 0; 3793 unsigned long i; 3794 int err = 0; 3795 3796 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3797 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3798 if (err) { 3799 err_vport_num = vport->vport; 3800 NL_SET_ERR_MSG_MOD(extack, 3801 "Failed to set min inline on vport"); 3802 goto revert_inline_mode; 3803 } 3804 } 3805 if (mlx5_core_ec_sriov_enabled(esw->dev)) { 3806 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 3807 err = mlx5_modify_nic_vport_min_inline(dev, vport->vport, mlx5_mode); 3808 if (err) { 3809 err_vport_num = vport->vport; 3810 NL_SET_ERR_MSG_MOD(extack, 3811 "Failed to set min inline on vport"); 3812 goto revert_ec_vf_inline_mode; 3813 } 3814 } 3815 } 3816 return 0; 3817 3818 revert_ec_vf_inline_mode: 3819 mlx5_esw_for_each_ec_vf_vport(esw, i, vport, esw->esw_funcs.num_ec_vfs) { 3820 if (vport->vport == err_vport_num) 3821 break; 3822 mlx5_modify_nic_vport_min_inline(dev, 3823 vport->vport, 3824 esw->offloads.inline_mode); 3825 } 3826 revert_inline_mode: 3827 mlx5_esw_for_each_host_func_vport(esw, i, vport, esw->esw_funcs.num_vfs) { 3828 if (vport->vport == err_vport_num) 3829 break; 3830 mlx5_modify_nic_vport_min_inline(dev, 3831 vport->vport, 3832 esw->offloads.inline_mode); 3833 } 3834 return err; 3835 } 3836 3837 int mlx5_devlink_eswitch_inline_mode_set(struct devlink *devlink, u8 mode, 3838 struct netlink_ext_ack *extack) 3839 { 3840 struct mlx5_core_dev *dev = devlink_priv(devlink); 3841 struct mlx5_eswitch *esw; 3842 u8 mlx5_mode; 3843 int err; 3844 3845 esw = mlx5_devlink_eswitch_get(devlink); 3846 if (IS_ERR(esw)) 3847 return PTR_ERR(esw); 3848 3849 down_write(&esw->mode_lock); 3850 3851 switch (MLX5_CAP_ETH(dev, wqe_inline_mode)) { 3852 case MLX5_CAP_INLINE_MODE_NOT_REQUIRED: 3853 if (mode == DEVLINK_ESWITCH_INLINE_MODE_NONE) { 3854 err = 0; 3855 goto out; 3856 } 3857 3858 fallthrough; 3859 case MLX5_CAP_INLINE_MODE_L2: 3860 NL_SET_ERR_MSG_MOD(extack, "Inline mode can't be set"); 3861 err = -EOPNOTSUPP; 3862 goto out; 3863 case MLX5_CAP_INLINE_MODE_VPORT_CONTEXT: 3864 break; 3865 } 3866 3867 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3868 NL_SET_ERR_MSG_MOD(extack, 3869 "Can't set inline mode when flows are configured"); 3870 err = -EOPNOTSUPP; 3871 goto out; 3872 } 3873 3874 err = esw_inline_mode_from_devlink(mode, &mlx5_mode); 3875 if (err) 3876 goto out; 3877 3878 esw->eswitch_operation_in_progress = true; 3879 up_write(&esw->mode_lock); 3880 3881 err = mlx5_esw_vports_inline_set(esw, mlx5_mode, extack); 3882 if (!err) 3883 esw->offloads.inline_mode = mlx5_mode; 3884 3885 down_write(&esw->mode_lock); 3886 esw->eswitch_operation_in_progress = false; 3887 up_write(&esw->mode_lock); 3888 return 0; 3889 3890 out: 3891 up_write(&esw->mode_lock); 3892 return err; 3893 } 3894 3895 int mlx5_devlink_eswitch_inline_mode_get(struct devlink *devlink, u8 *mode) 3896 { 3897 struct mlx5_eswitch *esw; 3898 3899 esw = mlx5_devlink_eswitch_get(devlink); 3900 if (IS_ERR(esw)) 3901 return PTR_ERR(esw); 3902 3903 return esw_inline_mode_to_devlink(esw->offloads.inline_mode, mode); 3904 } 3905 3906 bool mlx5_eswitch_block_encap(struct mlx5_core_dev *dev) 3907 { 3908 struct mlx5_eswitch *esw = dev->priv.eswitch; 3909 3910 if (!mlx5_esw_allowed(esw)) 3911 return true; 3912 3913 down_write(&esw->mode_lock); 3914 if (esw->mode != MLX5_ESWITCH_LEGACY && 3915 esw->offloads.encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE) { 3916 up_write(&esw->mode_lock); 3917 return false; 3918 } 3919 3920 esw->offloads.num_block_encap++; 3921 up_write(&esw->mode_lock); 3922 return true; 3923 } 3924 3925 void mlx5_eswitch_unblock_encap(struct mlx5_core_dev *dev) 3926 { 3927 struct mlx5_eswitch *esw = dev->priv.eswitch; 3928 3929 if (!mlx5_esw_allowed(esw)) 3930 return; 3931 3932 down_write(&esw->mode_lock); 3933 esw->offloads.num_block_encap--; 3934 up_write(&esw->mode_lock); 3935 } 3936 3937 int mlx5_devlink_eswitch_encap_mode_set(struct devlink *devlink, 3938 enum devlink_eswitch_encap_mode encap, 3939 struct netlink_ext_ack *extack) 3940 { 3941 struct mlx5_core_dev *dev = devlink_priv(devlink); 3942 struct mlx5_eswitch *esw; 3943 int err = 0; 3944 3945 esw = mlx5_devlink_eswitch_get(devlink); 3946 if (IS_ERR(esw)) 3947 return PTR_ERR(esw); 3948 3949 down_write(&esw->mode_lock); 3950 3951 if (encap != DEVLINK_ESWITCH_ENCAP_MODE_NONE && 3952 (!MLX5_CAP_ESW_FLOWTABLE_FDB(dev, reformat) || 3953 !MLX5_CAP_ESW_FLOWTABLE_FDB(dev, decap))) { 3954 err = -EOPNOTSUPP; 3955 goto unlock; 3956 } 3957 3958 if (encap && encap != DEVLINK_ESWITCH_ENCAP_MODE_BASIC) { 3959 err = -EOPNOTSUPP; 3960 goto unlock; 3961 } 3962 3963 if (esw->mode == MLX5_ESWITCH_LEGACY) { 3964 esw->offloads.encap = encap; 3965 goto unlock; 3966 } 3967 3968 if (esw->offloads.encap == encap) 3969 goto unlock; 3970 3971 if (atomic64_read(&esw->offloads.num_flows) > 0) { 3972 NL_SET_ERR_MSG_MOD(extack, 3973 "Can't set encapsulation when flows are configured"); 3974 err = -EOPNOTSUPP; 3975 goto unlock; 3976 } 3977 3978 if (esw->offloads.num_block_encap) { 3979 NL_SET_ERR_MSG_MOD(extack, 3980 "Can't set encapsulation when IPsec SA and/or policies are configured"); 3981 err = -EOPNOTSUPP; 3982 goto unlock; 3983 } 3984 3985 esw->eswitch_operation_in_progress = true; 3986 up_write(&esw->mode_lock); 3987 3988 esw_destroy_offloads_fdb_tables(esw); 3989 3990 esw->offloads.encap = encap; 3991 3992 err = esw_create_offloads_fdb_tables(esw); 3993 3994 if (err) { 3995 NL_SET_ERR_MSG_MOD(extack, 3996 "Failed re-creating fast FDB table"); 3997 esw->offloads.encap = !encap; 3998 (void)esw_create_offloads_fdb_tables(esw); 3999 } 4000 4001 down_write(&esw->mode_lock); 4002 esw->eswitch_operation_in_progress = false; 4003 4004 unlock: 4005 up_write(&esw->mode_lock); 4006 return err; 4007 } 4008 4009 int mlx5_devlink_eswitch_encap_mode_get(struct devlink *devlink, 4010 enum devlink_eswitch_encap_mode *encap) 4011 { 4012 struct mlx5_eswitch *esw; 4013 4014 esw = mlx5_devlink_eswitch_get(devlink); 4015 if (IS_ERR(esw)) 4016 return PTR_ERR(esw); 4017 4018 *encap = esw->offloads.encap; 4019 return 0; 4020 } 4021 4022 static bool 4023 mlx5_eswitch_vport_has_rep(const struct mlx5_eswitch *esw, u16 vport_num) 4024 { 4025 /* Currently, only ECPF based device has representor for host PF. */ 4026 if (vport_num == MLX5_VPORT_PF && 4027 !mlx5_core_is_ecpf_esw_manager(esw->dev)) 4028 return false; 4029 4030 if (vport_num == MLX5_VPORT_ECPF && 4031 !mlx5_ecpf_vport_exists(esw->dev)) 4032 return false; 4033 4034 return true; 4035 } 4036 4037 void mlx5_eswitch_register_vport_reps(struct mlx5_eswitch *esw, 4038 const struct mlx5_eswitch_rep_ops *ops, 4039 u8 rep_type) 4040 { 4041 struct mlx5_eswitch_rep_data *rep_data; 4042 struct mlx5_eswitch_rep *rep; 4043 unsigned long i; 4044 4045 esw->offloads.rep_ops[rep_type] = ops; 4046 mlx5_esw_for_each_rep(esw, i, rep) { 4047 if (likely(mlx5_eswitch_vport_has_rep(esw, rep->vport))) { 4048 rep->esw = esw; 4049 rep_data = &rep->rep_data[rep_type]; 4050 atomic_set(&rep_data->state, REP_REGISTERED); 4051 } 4052 } 4053 } 4054 EXPORT_SYMBOL(mlx5_eswitch_register_vport_reps); 4055 4056 void mlx5_eswitch_unregister_vport_reps(struct mlx5_eswitch *esw, u8 rep_type) 4057 { 4058 struct mlx5_eswitch_rep *rep; 4059 unsigned long i; 4060 4061 if (esw->mode == MLX5_ESWITCH_OFFLOADS) 4062 __unload_reps_all_vport(esw, rep_type); 4063 4064 mlx5_esw_for_each_rep(esw, i, rep) 4065 atomic_set(&rep->rep_data[rep_type].state, REP_UNREGISTERED); 4066 } 4067 EXPORT_SYMBOL(mlx5_eswitch_unregister_vport_reps); 4068 4069 void *mlx5_eswitch_get_uplink_priv(struct mlx5_eswitch *esw, u8 rep_type) 4070 { 4071 struct mlx5_eswitch_rep *rep; 4072 4073 rep = mlx5_eswitch_get_rep(esw, MLX5_VPORT_UPLINK); 4074 return rep->rep_data[rep_type].priv; 4075 } 4076 4077 void *mlx5_eswitch_get_proto_dev(struct mlx5_eswitch *esw, 4078 u16 vport, 4079 u8 rep_type) 4080 { 4081 struct mlx5_eswitch_rep *rep; 4082 4083 rep = mlx5_eswitch_get_rep(esw, vport); 4084 4085 if (atomic_read(&rep->rep_data[rep_type].state) == REP_LOADED && 4086 esw->offloads.rep_ops[rep_type]->get_proto_dev) 4087 return esw->offloads.rep_ops[rep_type]->get_proto_dev(rep); 4088 return NULL; 4089 } 4090 EXPORT_SYMBOL(mlx5_eswitch_get_proto_dev); 4091 4092 void *mlx5_eswitch_uplink_get_proto_dev(struct mlx5_eswitch *esw, u8 rep_type) 4093 { 4094 return mlx5_eswitch_get_proto_dev(esw, MLX5_VPORT_UPLINK, rep_type); 4095 } 4096 EXPORT_SYMBOL(mlx5_eswitch_uplink_get_proto_dev); 4097 4098 struct mlx5_eswitch_rep *mlx5_eswitch_vport_rep(struct mlx5_eswitch *esw, 4099 u16 vport) 4100 { 4101 return mlx5_eswitch_get_rep(esw, vport); 4102 } 4103 EXPORT_SYMBOL(mlx5_eswitch_vport_rep); 4104 4105 bool mlx5_eswitch_reg_c1_loopback_enabled(const struct mlx5_eswitch *esw) 4106 { 4107 return !!(esw->flags & MLX5_ESWITCH_REG_C1_LOOPBACK_ENABLED); 4108 } 4109 EXPORT_SYMBOL(mlx5_eswitch_reg_c1_loopback_enabled); 4110 4111 bool mlx5_eswitch_vport_match_metadata_enabled(const struct mlx5_eswitch *esw) 4112 { 4113 return !!(esw->flags & MLX5_ESWITCH_VPORT_MATCH_METADATA); 4114 } 4115 EXPORT_SYMBOL(mlx5_eswitch_vport_match_metadata_enabled); 4116 4117 u32 mlx5_eswitch_get_vport_metadata_for_match(struct mlx5_eswitch *esw, 4118 u16 vport_num) 4119 { 4120 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4121 4122 if (WARN_ON_ONCE(IS_ERR(vport))) 4123 return 0; 4124 4125 return vport->metadata << (32 - ESW_SOURCE_PORT_METADATA_BITS); 4126 } 4127 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_match); 4128 4129 static int mlx5_esw_query_vport_vhca_id(struct mlx5_eswitch *esw, u16 vport_num, u16 *vhca_id) 4130 { 4131 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4132 void *query_ctx; 4133 void *hca_caps; 4134 int err; 4135 4136 *vhca_id = 0; 4137 4138 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4139 if (!query_ctx) 4140 return -ENOMEM; 4141 4142 err = mlx5_vport_get_other_func_general_cap(esw->dev, vport_num, query_ctx); 4143 if (err) 4144 goto out_free; 4145 4146 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4147 *vhca_id = MLX5_GET(cmd_hca_cap, hca_caps, vhca_id); 4148 4149 out_free: 4150 kfree(query_ctx); 4151 return err; 4152 } 4153 4154 int mlx5_esw_vport_vhca_id_set(struct mlx5_eswitch *esw, u16 vport_num) 4155 { 4156 u16 *old_entry, *vhca_map_entry, vhca_id; 4157 int err; 4158 4159 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 4160 if (err) { 4161 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%u,err=%d)\n", 4162 vport_num, err); 4163 return err; 4164 } 4165 4166 vhca_map_entry = kmalloc(sizeof(*vhca_map_entry), GFP_KERNEL); 4167 if (!vhca_map_entry) 4168 return -ENOMEM; 4169 4170 *vhca_map_entry = vport_num; 4171 old_entry = xa_store(&esw->offloads.vhca_map, vhca_id, vhca_map_entry, GFP_KERNEL); 4172 if (xa_is_err(old_entry)) { 4173 kfree(vhca_map_entry); 4174 return xa_err(old_entry); 4175 } 4176 kfree(old_entry); 4177 return 0; 4178 } 4179 4180 void mlx5_esw_vport_vhca_id_clear(struct mlx5_eswitch *esw, u16 vport_num) 4181 { 4182 u16 *vhca_map_entry, vhca_id; 4183 int err; 4184 4185 err = mlx5_esw_query_vport_vhca_id(esw, vport_num, &vhca_id); 4186 if (err) 4187 esw_warn(esw->dev, "Getting vhca_id for vport failed (vport=%hu,err=%d)\n", 4188 vport_num, err); 4189 4190 vhca_map_entry = xa_erase(&esw->offloads.vhca_map, vhca_id); 4191 kfree(vhca_map_entry); 4192 } 4193 4194 int mlx5_eswitch_vhca_id_to_vport(struct mlx5_eswitch *esw, u16 vhca_id, u16 *vport_num) 4195 { 4196 u16 *res = xa_load(&esw->offloads.vhca_map, vhca_id); 4197 4198 if (!res) 4199 return -ENOENT; 4200 4201 *vport_num = *res; 4202 return 0; 4203 } 4204 4205 u32 mlx5_eswitch_get_vport_metadata_for_set(struct mlx5_eswitch *esw, 4206 u16 vport_num) 4207 { 4208 struct mlx5_vport *vport = mlx5_eswitch_get_vport(esw, vport_num); 4209 4210 if (WARN_ON_ONCE(IS_ERR(vport))) 4211 return 0; 4212 4213 return vport->metadata; 4214 } 4215 EXPORT_SYMBOL(mlx5_eswitch_get_vport_metadata_for_set); 4216 4217 int mlx5_devlink_port_fn_hw_addr_get(struct devlink_port *port, 4218 u8 *hw_addr, int *hw_addr_len, 4219 struct netlink_ext_ack *extack) 4220 { 4221 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4222 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4223 4224 mutex_lock(&esw->state_lock); 4225 ether_addr_copy(hw_addr, vport->info.mac); 4226 *hw_addr_len = ETH_ALEN; 4227 mutex_unlock(&esw->state_lock); 4228 return 0; 4229 } 4230 4231 int mlx5_devlink_port_fn_hw_addr_set(struct devlink_port *port, 4232 const u8 *hw_addr, int hw_addr_len, 4233 struct netlink_ext_ack *extack) 4234 { 4235 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4236 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4237 4238 return mlx5_eswitch_set_vport_mac(esw, vport->vport, hw_addr); 4239 } 4240 4241 int mlx5_devlink_port_fn_migratable_get(struct devlink_port *port, bool *is_enabled, 4242 struct netlink_ext_ack *extack) 4243 { 4244 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4245 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4246 4247 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4248 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4249 return -EOPNOTSUPP; 4250 } 4251 4252 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4253 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4254 return -EOPNOTSUPP; 4255 } 4256 4257 mutex_lock(&esw->state_lock); 4258 *is_enabled = vport->info.mig_enabled; 4259 mutex_unlock(&esw->state_lock); 4260 return 0; 4261 } 4262 4263 int mlx5_devlink_port_fn_migratable_set(struct devlink_port *port, bool enable, 4264 struct netlink_ext_ack *extack) 4265 { 4266 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4267 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4268 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4269 void *query_ctx; 4270 void *hca_caps; 4271 int err; 4272 4273 if (!MLX5_CAP_GEN(esw->dev, migration)) { 4274 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support migration"); 4275 return -EOPNOTSUPP; 4276 } 4277 4278 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4279 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4280 return -EOPNOTSUPP; 4281 } 4282 4283 mutex_lock(&esw->state_lock); 4284 4285 if (vport->info.mig_enabled == enable) { 4286 err = 0; 4287 goto out; 4288 } 4289 4290 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4291 if (!query_ctx) { 4292 err = -ENOMEM; 4293 goto out; 4294 } 4295 4296 err = mlx5_vport_get_other_func_cap(esw->dev, vport->vport, query_ctx, 4297 MLX5_CAP_GENERAL_2); 4298 if (err) { 4299 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4300 goto out_free; 4301 } 4302 4303 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4304 MLX5_SET(cmd_hca_cap_2, hca_caps, migratable, enable); 4305 4306 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport->vport, 4307 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE2); 4308 if (err) { 4309 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA migratable cap"); 4310 goto out_free; 4311 } 4312 4313 vport->info.mig_enabled = enable; 4314 4315 out_free: 4316 kfree(query_ctx); 4317 out: 4318 mutex_unlock(&esw->state_lock); 4319 return err; 4320 } 4321 4322 int mlx5_devlink_port_fn_roce_get(struct devlink_port *port, bool *is_enabled, 4323 struct netlink_ext_ack *extack) 4324 { 4325 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4326 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4327 4328 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4329 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4330 return -EOPNOTSUPP; 4331 } 4332 4333 mutex_lock(&esw->state_lock); 4334 *is_enabled = vport->info.roce_enabled; 4335 mutex_unlock(&esw->state_lock); 4336 return 0; 4337 } 4338 4339 int mlx5_devlink_port_fn_roce_set(struct devlink_port *port, bool enable, 4340 struct netlink_ext_ack *extack) 4341 { 4342 struct mlx5_eswitch *esw = mlx5_devlink_eswitch_nocheck_get(port->devlink); 4343 struct mlx5_vport *vport = mlx5_devlink_port_vport_get(port); 4344 int query_out_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out); 4345 u16 vport_num = vport->vport; 4346 void *query_ctx; 4347 void *hca_caps; 4348 int err; 4349 4350 if (!MLX5_CAP_GEN(esw->dev, vhca_resource_manager)) { 4351 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support VHCA management"); 4352 return -EOPNOTSUPP; 4353 } 4354 4355 mutex_lock(&esw->state_lock); 4356 4357 if (vport->info.roce_enabled == enable) { 4358 err = 0; 4359 goto out; 4360 } 4361 4362 query_ctx = kzalloc(query_out_sz, GFP_KERNEL); 4363 if (!query_ctx) { 4364 err = -ENOMEM; 4365 goto out; 4366 } 4367 4368 err = mlx5_vport_get_other_func_cap(esw->dev, vport_num, query_ctx, 4369 MLX5_CAP_GENERAL); 4370 if (err) { 4371 NL_SET_ERR_MSG_MOD(extack, "Failed getting HCA caps"); 4372 goto out_free; 4373 } 4374 4375 hca_caps = MLX5_ADDR_OF(query_hca_cap_out, query_ctx, capability); 4376 MLX5_SET(cmd_hca_cap, hca_caps, roce, enable); 4377 4378 err = mlx5_vport_set_other_func_cap(esw->dev, hca_caps, vport_num, 4379 MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE); 4380 if (err) { 4381 NL_SET_ERR_MSG_MOD(extack, "Failed setting HCA roce cap"); 4382 goto out_free; 4383 } 4384 4385 vport->info.roce_enabled = enable; 4386 4387 out_free: 4388 kfree(query_ctx); 4389 out: 4390 mutex_unlock(&esw->state_lock); 4391 return err; 4392 } 4393 4394 int 4395 mlx5_eswitch_restore_ipsec_rule(struct mlx5_eswitch *esw, struct mlx5_flow_handle *rule, 4396 struct mlx5_esw_flow_attr *esw_attr, int attr_idx) 4397 { 4398 struct mlx5_flow_destination new_dest = {}; 4399 struct mlx5_flow_destination old_dest = {}; 4400 4401 if (!esw_setup_uplink_fwd_ipsec_needed(esw, esw_attr, attr_idx)) 4402 return 0; 4403 4404 esw_setup_dest_fwd_ipsec(&old_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4405 esw_setup_dest_fwd_vport(&new_dest, NULL, esw, esw_attr, attr_idx, 0, false); 4406 4407 return mlx5_modify_rule_destination(rule, &new_dest, &old_dest); 4408 } 4409 4410 #ifdef CONFIG_XFRM_OFFLOAD 4411 int mlx5_devlink_port_fn_ipsec_crypto_get(struct devlink_port *port, bool *is_enabled, 4412 struct netlink_ext_ack *extack) 4413 { 4414 struct mlx5_eswitch *esw; 4415 struct mlx5_vport *vport; 4416 int err = 0; 4417 4418 esw = mlx5_devlink_eswitch_get(port->devlink); 4419 if (IS_ERR(esw)) 4420 return PTR_ERR(esw); 4421 4422 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4423 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPSec crypto"); 4424 return -EOPNOTSUPP; 4425 } 4426 4427 vport = mlx5_devlink_port_vport_get(port); 4428 4429 mutex_lock(&esw->state_lock); 4430 if (!vport->enabled) { 4431 err = -EOPNOTSUPP; 4432 goto unlock; 4433 } 4434 4435 *is_enabled = vport->info.ipsec_crypto_enabled; 4436 unlock: 4437 mutex_unlock(&esw->state_lock); 4438 return err; 4439 } 4440 4441 int mlx5_devlink_port_fn_ipsec_crypto_set(struct devlink_port *port, bool enable, 4442 struct netlink_ext_ack *extack) 4443 { 4444 struct mlx5_eswitch *esw; 4445 struct mlx5_vport *vport; 4446 u16 vport_num; 4447 int err; 4448 4449 esw = mlx5_devlink_eswitch_get(port->devlink); 4450 if (IS_ERR(esw)) 4451 return PTR_ERR(esw); 4452 4453 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4454 err = mlx5_esw_ipsec_vf_crypto_offload_supported(esw->dev, vport_num); 4455 if (err) { 4456 NL_SET_ERR_MSG_MOD(extack, 4457 "Device doesn't support IPsec crypto"); 4458 return err; 4459 } 4460 4461 vport = mlx5_devlink_port_vport_get(port); 4462 4463 mutex_lock(&esw->state_lock); 4464 if (!vport->enabled) { 4465 err = -EOPNOTSUPP; 4466 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4467 goto unlock; 4468 } 4469 4470 if (vport->info.ipsec_crypto_enabled == enable) 4471 goto unlock; 4472 4473 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 4474 err = -EBUSY; 4475 goto unlock; 4476 } 4477 4478 err = mlx5_esw_ipsec_vf_crypto_offload_set(esw, vport, enable); 4479 if (err) { 4480 NL_SET_ERR_MSG_MOD(extack, "Failed to set IPsec crypto"); 4481 goto unlock; 4482 } 4483 4484 vport->info.ipsec_crypto_enabled = enable; 4485 if (enable) 4486 esw->enabled_ipsec_vf_count++; 4487 else 4488 esw->enabled_ipsec_vf_count--; 4489 unlock: 4490 mutex_unlock(&esw->state_lock); 4491 return err; 4492 } 4493 4494 int mlx5_devlink_port_fn_ipsec_packet_get(struct devlink_port *port, bool *is_enabled, 4495 struct netlink_ext_ack *extack) 4496 { 4497 struct mlx5_eswitch *esw; 4498 struct mlx5_vport *vport; 4499 int err = 0; 4500 4501 esw = mlx5_devlink_eswitch_get(port->devlink); 4502 if (IS_ERR(esw)) 4503 return PTR_ERR(esw); 4504 4505 if (!mlx5_esw_ipsec_vf_offload_supported(esw->dev)) { 4506 NL_SET_ERR_MSG_MOD(extack, "Device doesn't support IPsec packet"); 4507 return -EOPNOTSUPP; 4508 } 4509 4510 vport = mlx5_devlink_port_vport_get(port); 4511 4512 mutex_lock(&esw->state_lock); 4513 if (!vport->enabled) { 4514 err = -EOPNOTSUPP; 4515 goto unlock; 4516 } 4517 4518 *is_enabled = vport->info.ipsec_packet_enabled; 4519 unlock: 4520 mutex_unlock(&esw->state_lock); 4521 return err; 4522 } 4523 4524 int mlx5_devlink_port_fn_ipsec_packet_set(struct devlink_port *port, 4525 bool enable, 4526 struct netlink_ext_ack *extack) 4527 { 4528 struct mlx5_eswitch *esw; 4529 struct mlx5_vport *vport; 4530 u16 vport_num; 4531 int err; 4532 4533 esw = mlx5_devlink_eswitch_get(port->devlink); 4534 if (IS_ERR(esw)) 4535 return PTR_ERR(esw); 4536 4537 vport_num = mlx5_esw_devlink_port_index_to_vport_num(port->index); 4538 err = mlx5_esw_ipsec_vf_packet_offload_supported(esw->dev, vport_num); 4539 if (err) { 4540 NL_SET_ERR_MSG_MOD(extack, 4541 "Device doesn't support IPsec packet mode"); 4542 return err; 4543 } 4544 4545 vport = mlx5_devlink_port_vport_get(port); 4546 mutex_lock(&esw->state_lock); 4547 if (!vport->enabled) { 4548 err = -EOPNOTSUPP; 4549 NL_SET_ERR_MSG_MOD(extack, "Eswitch vport is disabled"); 4550 goto unlock; 4551 } 4552 4553 if (vport->info.ipsec_packet_enabled == enable) 4554 goto unlock; 4555 4556 if (!esw->enabled_ipsec_vf_count && esw->dev->num_ipsec_offloads) { 4557 err = -EBUSY; 4558 goto unlock; 4559 } 4560 4561 err = mlx5_esw_ipsec_vf_packet_offload_set(esw, vport, enable); 4562 if (err) { 4563 NL_SET_ERR_MSG_MOD(extack, 4564 "Failed to set IPsec packet mode"); 4565 goto unlock; 4566 } 4567 4568 vport->info.ipsec_packet_enabled = enable; 4569 if (enable) 4570 esw->enabled_ipsec_vf_count++; 4571 else 4572 esw->enabled_ipsec_vf_count--; 4573 unlock: 4574 mutex_unlock(&esw->state_lock); 4575 return err; 4576 } 4577 #endif /* CONFIG_XFRM_OFFLOAD */ 4578