1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2021, Mellanox Technologies inc. All rights reserved. */ 3 4 #include "rx_res.h" 5 #include "channels.h" 6 #include "params.h" 7 8 #define MLX5E_MAX_NUM_RSS 16 9 10 struct mlx5e_rx_res { 11 struct mlx5_core_dev *mdev; 12 enum mlx5e_rx_res_features features; 13 unsigned int max_nch; 14 u32 drop_rqn; 15 16 struct mlx5e_packet_merge_param pkt_merge_param; 17 struct rw_semaphore pkt_merge_param_sem; 18 19 struct mlx5e_rss *rss[MLX5E_MAX_NUM_RSS]; 20 bool rss_active; 21 u32 rss_rqns[MLX5E_INDIR_RQT_SIZE]; 22 unsigned int rss_nch; 23 24 struct { 25 struct mlx5e_rqt direct_rqt; 26 struct mlx5e_tir direct_tir; 27 } *channels; 28 29 struct { 30 struct mlx5e_rqt rqt; 31 struct mlx5e_tir tir; 32 } ptp; 33 }; 34 35 /* API for rx_res_rss_* */ 36 37 static int mlx5e_rx_res_rss_init_def(struct mlx5e_rx_res *res, 38 unsigned int init_nch) 39 { 40 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 41 struct mlx5e_rss *rss; 42 int err; 43 44 if (WARN_ON(res->rss[0])) 45 return -EINVAL; 46 47 rss = mlx5e_rss_alloc(); 48 if (!rss) 49 return -ENOMEM; 50 51 err = mlx5e_rss_init(rss, res->mdev, inner_ft_support, res->drop_rqn, 52 &res->pkt_merge_param); 53 if (err) 54 goto err_rss_free; 55 56 mlx5e_rss_set_indir_uniform(rss, init_nch); 57 58 res->rss[0] = rss; 59 60 return 0; 61 62 err_rss_free: 63 mlx5e_rss_free(rss); 64 return err; 65 } 66 67 int mlx5e_rx_res_rss_init(struct mlx5e_rx_res *res, u32 *rss_idx, unsigned int init_nch) 68 { 69 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 70 struct mlx5e_rss *rss; 71 int err, i; 72 73 for (i = 1; i < MLX5E_MAX_NUM_RSS; i++) 74 if (!res->rss[i]) 75 break; 76 77 if (i == MLX5E_MAX_NUM_RSS) 78 return -ENOSPC; 79 80 rss = mlx5e_rss_alloc(); 81 if (!rss) 82 return -ENOMEM; 83 84 err = mlx5e_rss_init_no_tirs(rss, res->mdev, inner_ft_support, res->drop_rqn); 85 if (err) 86 goto err_rss_free; 87 88 mlx5e_rss_set_indir_uniform(rss, init_nch); 89 if (res->rss_active) 90 mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); 91 92 res->rss[i] = rss; 93 *rss_idx = i; 94 95 return 0; 96 97 err_rss_free: 98 mlx5e_rss_free(rss); 99 return err; 100 } 101 102 static int __mlx5e_rx_res_rss_destroy(struct mlx5e_rx_res *res, u32 rss_idx) 103 { 104 struct mlx5e_rss *rss = res->rss[rss_idx]; 105 int err; 106 107 err = mlx5e_rss_cleanup(rss); 108 if (err) 109 return err; 110 111 mlx5e_rss_free(rss); 112 res->rss[rss_idx] = NULL; 113 114 return 0; 115 } 116 117 int mlx5e_rx_res_rss_destroy(struct mlx5e_rx_res *res, u32 rss_idx) 118 { 119 struct mlx5e_rss *rss; 120 121 if (rss_idx >= MLX5E_MAX_NUM_RSS) 122 return -EINVAL; 123 124 rss = res->rss[rss_idx]; 125 if (!rss) 126 return -EINVAL; 127 128 return __mlx5e_rx_res_rss_destroy(res, rss_idx); 129 } 130 131 static void mlx5e_rx_res_rss_destroy_all(struct mlx5e_rx_res *res) 132 { 133 int i; 134 135 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { 136 struct mlx5e_rss *rss = res->rss[i]; 137 int err; 138 139 if (!rss) 140 continue; 141 142 err = __mlx5e_rx_res_rss_destroy(res, i); 143 if (err) { 144 unsigned int refcount; 145 146 refcount = mlx5e_rss_refcnt_read(rss); 147 mlx5_core_warn(res->mdev, 148 "Failed to destroy RSS context %d, refcount = %u, err = %d\n", 149 i, refcount, err); 150 } 151 } 152 } 153 154 static void mlx5e_rx_res_rss_enable(struct mlx5e_rx_res *res) 155 { 156 int i; 157 158 res->rss_active = true; 159 160 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { 161 struct mlx5e_rss *rss = res->rss[i]; 162 163 if (!rss) 164 continue; 165 mlx5e_rss_enable(rss, res->rss_rqns, res->rss_nch); 166 } 167 } 168 169 static void mlx5e_rx_res_rss_disable(struct mlx5e_rx_res *res) 170 { 171 int i; 172 173 res->rss_active = false; 174 175 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) { 176 struct mlx5e_rss *rss = res->rss[i]; 177 178 if (!rss) 179 continue; 180 mlx5e_rss_disable(rss); 181 } 182 } 183 184 /* Updates the indirection table SW shadow, does not update the HW resources yet */ 185 void mlx5e_rx_res_rss_set_indir_uniform(struct mlx5e_rx_res *res, unsigned int nch) 186 { 187 WARN_ON_ONCE(res->rss_active); 188 mlx5e_rss_set_indir_uniform(res->rss[0], nch); 189 } 190 191 int mlx5e_rx_res_rss_get_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, 192 u32 *indir, u8 *key, u8 *hfunc) 193 { 194 struct mlx5e_rss *rss; 195 196 if (rss_idx >= MLX5E_MAX_NUM_RSS) 197 return -EINVAL; 198 199 rss = res->rss[rss_idx]; 200 if (!rss) 201 return -ENOENT; 202 203 return mlx5e_rss_get_rxfh(rss, indir, key, hfunc); 204 } 205 206 int mlx5e_rx_res_rss_set_rxfh(struct mlx5e_rx_res *res, u32 rss_idx, 207 const u32 *indir, const u8 *key, const u8 *hfunc) 208 { 209 struct mlx5e_rss *rss; 210 211 if (rss_idx >= MLX5E_MAX_NUM_RSS) 212 return -EINVAL; 213 214 rss = res->rss[rss_idx]; 215 if (!rss) 216 return -ENOENT; 217 218 return mlx5e_rss_set_rxfh(rss, indir, key, hfunc, res->rss_rqns, res->rss_nch); 219 } 220 221 int mlx5e_rx_res_rss_get_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, 222 enum mlx5_traffic_types tt) 223 { 224 struct mlx5e_rss *rss; 225 226 if (rss_idx >= MLX5E_MAX_NUM_RSS) 227 return -EINVAL; 228 229 rss = res->rss[rss_idx]; 230 if (!rss) 231 return -ENOENT; 232 233 return mlx5e_rss_get_hash_fields(rss, tt); 234 } 235 236 int mlx5e_rx_res_rss_set_hash_fields(struct mlx5e_rx_res *res, u32 rss_idx, 237 enum mlx5_traffic_types tt, u8 rx_hash_fields) 238 { 239 struct mlx5e_rss *rss; 240 241 if (rss_idx >= MLX5E_MAX_NUM_RSS) 242 return -EINVAL; 243 244 rss = res->rss[rss_idx]; 245 if (!rss) 246 return -ENOENT; 247 248 return mlx5e_rss_set_hash_fields(rss, tt, rx_hash_fields); 249 } 250 251 int mlx5e_rx_res_rss_cnt(struct mlx5e_rx_res *res) 252 { 253 int i, cnt; 254 255 cnt = 0; 256 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) 257 if (res->rss[i]) 258 cnt++; 259 260 return cnt; 261 } 262 263 int mlx5e_rx_res_rss_index(struct mlx5e_rx_res *res, struct mlx5e_rss *rss) 264 { 265 int i; 266 267 if (!rss) 268 return -EINVAL; 269 270 for (i = 0; i < MLX5E_MAX_NUM_RSS; i++) 271 if (rss == res->rss[i]) 272 return i; 273 274 return -ENOENT; 275 } 276 277 struct mlx5e_rss *mlx5e_rx_res_rss_get(struct mlx5e_rx_res *res, u32 rss_idx) 278 { 279 if (rss_idx >= MLX5E_MAX_NUM_RSS) 280 return NULL; 281 282 return res->rss[rss_idx]; 283 } 284 285 /* End of API rx_res_rss_* */ 286 287 struct mlx5e_rx_res *mlx5e_rx_res_alloc(void) 288 { 289 return kvzalloc(sizeof(struct mlx5e_rx_res), GFP_KERNEL); 290 } 291 292 static int mlx5e_rx_res_channels_init(struct mlx5e_rx_res *res) 293 { 294 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 295 struct mlx5e_tir_builder *builder; 296 int err = 0; 297 int ix; 298 299 builder = mlx5e_tir_builder_alloc(false); 300 if (!builder) 301 return -ENOMEM; 302 303 res->channels = kvcalloc(res->max_nch, sizeof(*res->channels), GFP_KERNEL); 304 if (!res->channels) { 305 err = -ENOMEM; 306 goto out; 307 } 308 309 for (ix = 0; ix < res->max_nch; ix++) { 310 err = mlx5e_rqt_init_direct(&res->channels[ix].direct_rqt, 311 res->mdev, false, res->drop_rqn); 312 if (err) { 313 mlx5_core_warn(res->mdev, "Failed to create a direct RQT: err = %d, ix = %u\n", 314 err, ix); 315 goto err_destroy_direct_rqts; 316 } 317 } 318 319 for (ix = 0; ix < res->max_nch; ix++) { 320 mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, 321 mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), 322 inner_ft_support); 323 mlx5e_tir_builder_build_packet_merge(builder, &res->pkt_merge_param); 324 mlx5e_tir_builder_build_direct(builder); 325 326 err = mlx5e_tir_init(&res->channels[ix].direct_tir, builder, res->mdev, true); 327 if (err) { 328 mlx5_core_warn(res->mdev, "Failed to create a direct TIR: err = %d, ix = %u\n", 329 err, ix); 330 goto err_destroy_direct_tirs; 331 } 332 333 mlx5e_tir_builder_clear(builder); 334 } 335 336 goto out; 337 338 err_destroy_direct_tirs: 339 while (--ix >= 0) 340 mlx5e_tir_destroy(&res->channels[ix].direct_tir); 341 342 ix = res->max_nch; 343 err_destroy_direct_rqts: 344 while (--ix >= 0) 345 mlx5e_rqt_destroy(&res->channels[ix].direct_rqt); 346 347 kvfree(res->channels); 348 349 out: 350 mlx5e_tir_builder_free(builder); 351 352 return err; 353 } 354 355 static int mlx5e_rx_res_ptp_init(struct mlx5e_rx_res *res) 356 { 357 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 358 struct mlx5e_tir_builder *builder; 359 int err; 360 361 builder = mlx5e_tir_builder_alloc(false); 362 if (!builder) 363 return -ENOMEM; 364 365 err = mlx5e_rqt_init_direct(&res->ptp.rqt, res->mdev, false, res->drop_rqn); 366 if (err) 367 goto out; 368 369 /* Separated from the channels RQs, does not share pkt_merge state with them */ 370 mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, 371 mlx5e_rqt_get_rqtn(&res->ptp.rqt), 372 inner_ft_support); 373 mlx5e_tir_builder_build_direct(builder); 374 375 err = mlx5e_tir_init(&res->ptp.tir, builder, res->mdev, true); 376 if (err) 377 goto err_destroy_ptp_rqt; 378 379 goto out; 380 381 err_destroy_ptp_rqt: 382 mlx5e_rqt_destroy(&res->ptp.rqt); 383 384 out: 385 mlx5e_tir_builder_free(builder); 386 return err; 387 } 388 389 static void mlx5e_rx_res_channels_destroy(struct mlx5e_rx_res *res) 390 { 391 unsigned int ix; 392 393 for (ix = 0; ix < res->max_nch; ix++) { 394 mlx5e_tir_destroy(&res->channels[ix].direct_tir); 395 mlx5e_rqt_destroy(&res->channels[ix].direct_rqt); 396 } 397 398 kvfree(res->channels); 399 } 400 401 static void mlx5e_rx_res_ptp_destroy(struct mlx5e_rx_res *res) 402 { 403 mlx5e_tir_destroy(&res->ptp.tir); 404 mlx5e_rqt_destroy(&res->ptp.rqt); 405 } 406 407 int mlx5e_rx_res_init(struct mlx5e_rx_res *res, struct mlx5_core_dev *mdev, 408 enum mlx5e_rx_res_features features, unsigned int max_nch, 409 u32 drop_rqn, const struct mlx5e_packet_merge_param *init_pkt_merge_param, 410 unsigned int init_nch) 411 { 412 int err; 413 414 res->mdev = mdev; 415 res->features = features; 416 res->max_nch = max_nch; 417 res->drop_rqn = drop_rqn; 418 419 res->pkt_merge_param = *init_pkt_merge_param; 420 init_rwsem(&res->pkt_merge_param_sem); 421 422 err = mlx5e_rx_res_rss_init_def(res, init_nch); 423 if (err) 424 goto err_out; 425 426 err = mlx5e_rx_res_channels_init(res); 427 if (err) 428 goto err_rss_destroy; 429 430 err = mlx5e_rx_res_ptp_init(res); 431 if (err) 432 goto err_channels_destroy; 433 434 return 0; 435 436 err_channels_destroy: 437 mlx5e_rx_res_channels_destroy(res); 438 err_rss_destroy: 439 __mlx5e_rx_res_rss_destroy(res, 0); 440 err_out: 441 return err; 442 } 443 444 void mlx5e_rx_res_destroy(struct mlx5e_rx_res *res) 445 { 446 mlx5e_rx_res_ptp_destroy(res); 447 mlx5e_rx_res_channels_destroy(res); 448 mlx5e_rx_res_rss_destroy_all(res); 449 } 450 451 void mlx5e_rx_res_free(struct mlx5e_rx_res *res) 452 { 453 kvfree(res); 454 } 455 456 u32 mlx5e_rx_res_get_tirn_direct(struct mlx5e_rx_res *res, unsigned int ix) 457 { 458 return mlx5e_tir_get_tirn(&res->channels[ix].direct_tir); 459 } 460 461 u32 mlx5e_rx_res_get_tirn_rss(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt) 462 { 463 struct mlx5e_rss *rss = res->rss[0]; 464 465 return mlx5e_rss_get_tirn(rss, tt, false); 466 } 467 468 u32 mlx5e_rx_res_get_tirn_rss_inner(struct mlx5e_rx_res *res, enum mlx5_traffic_types tt) 469 { 470 struct mlx5e_rss *rss = res->rss[0]; 471 472 return mlx5e_rss_get_tirn(rss, tt, true); 473 } 474 475 u32 mlx5e_rx_res_get_tirn_ptp(struct mlx5e_rx_res *res) 476 { 477 WARN_ON(!(res->features & MLX5E_RX_RES_FEATURE_PTP)); 478 return mlx5e_tir_get_tirn(&res->ptp.tir); 479 } 480 481 static u32 mlx5e_rx_res_get_rqtn_direct(struct mlx5e_rx_res *res, unsigned int ix) 482 { 483 return mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt); 484 } 485 486 static void mlx5e_rx_res_channel_activate_direct(struct mlx5e_rx_res *res, 487 struct mlx5e_channels *chs, 488 unsigned int ix) 489 { 490 u32 rqn = res->rss_rqns[ix]; 491 int err; 492 493 err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, rqn); 494 if (err) 495 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (channel %u): err = %d\n", 496 mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), 497 rqn, ix, err); 498 } 499 500 static void mlx5e_rx_res_channel_deactivate_direct(struct mlx5e_rx_res *res, 501 unsigned int ix) 502 { 503 int err; 504 505 err = mlx5e_rqt_redirect_direct(&res->channels[ix].direct_rqt, res->drop_rqn); 506 if (err) 507 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (channel %u): err = %d\n", 508 mlx5e_rqt_get_rqtn(&res->channels[ix].direct_rqt), 509 res->drop_rqn, ix, err); 510 } 511 512 void mlx5e_rx_res_channels_activate(struct mlx5e_rx_res *res, struct mlx5e_channels *chs) 513 { 514 unsigned int nch, ix; 515 int err; 516 517 nch = mlx5e_channels_get_num(chs); 518 519 for (ix = 0; ix < chs->num; ix++) { 520 if (mlx5e_channels_is_xsk(chs, ix)) 521 mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); 522 else 523 mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); 524 } 525 res->rss_nch = chs->num; 526 527 mlx5e_rx_res_rss_enable(res); 528 529 for (ix = 0; ix < nch; ix++) 530 mlx5e_rx_res_channel_activate_direct(res, chs, ix); 531 for (ix = nch; ix < res->max_nch; ix++) 532 mlx5e_rx_res_channel_deactivate_direct(res, ix); 533 534 if (res->features & MLX5E_RX_RES_FEATURE_PTP) { 535 u32 rqn; 536 537 if (!mlx5e_channels_get_ptp_rqn(chs, &rqn)) 538 rqn = res->drop_rqn; 539 540 err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, rqn); 541 if (err) 542 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to RQ %#x (PTP): err = %d\n", 543 mlx5e_rqt_get_rqtn(&res->ptp.rqt), 544 rqn, err); 545 } 546 } 547 548 void mlx5e_rx_res_channels_deactivate(struct mlx5e_rx_res *res) 549 { 550 unsigned int ix; 551 int err; 552 553 mlx5e_rx_res_rss_disable(res); 554 555 for (ix = 0; ix < res->max_nch; ix++) 556 mlx5e_rx_res_channel_deactivate_direct(res, ix); 557 558 if (res->features & MLX5E_RX_RES_FEATURE_PTP) { 559 err = mlx5e_rqt_redirect_direct(&res->ptp.rqt, res->drop_rqn); 560 if (err) 561 mlx5_core_warn(res->mdev, "Failed to redirect direct RQT %#x to drop RQ %#x (PTP): err = %d\n", 562 mlx5e_rqt_get_rqtn(&res->ptp.rqt), 563 res->drop_rqn, err); 564 } 565 } 566 567 void mlx5e_rx_res_xsk_update(struct mlx5e_rx_res *res, struct mlx5e_channels *chs, 568 unsigned int ix, bool xsk) 569 { 570 if (xsk) 571 mlx5e_channels_get_xsk_rqn(chs, ix, &res->rss_rqns[ix]); 572 else 573 mlx5e_channels_get_regular_rqn(chs, ix, &res->rss_rqns[ix]); 574 575 mlx5e_rx_res_rss_enable(res); 576 577 mlx5e_rx_res_channel_activate_direct(res, chs, ix); 578 } 579 580 int mlx5e_rx_res_packet_merge_set_param(struct mlx5e_rx_res *res, 581 struct mlx5e_packet_merge_param *pkt_merge_param) 582 { 583 struct mlx5e_tir_builder *builder; 584 int err, final_err; 585 unsigned int ix; 586 587 builder = mlx5e_tir_builder_alloc(true); 588 if (!builder) 589 return -ENOMEM; 590 591 down_write(&res->pkt_merge_param_sem); 592 res->pkt_merge_param = *pkt_merge_param; 593 594 mlx5e_tir_builder_build_packet_merge(builder, pkt_merge_param); 595 596 final_err = 0; 597 598 for (ix = 0; ix < MLX5E_MAX_NUM_RSS; ix++) { 599 struct mlx5e_rss *rss = res->rss[ix]; 600 601 if (!rss) 602 continue; 603 604 err = mlx5e_rss_packet_merge_set_param(rss, pkt_merge_param); 605 if (err) 606 final_err = final_err ? : err; 607 } 608 609 for (ix = 0; ix < res->max_nch; ix++) { 610 err = mlx5e_tir_modify(&res->channels[ix].direct_tir, builder); 611 if (err) { 612 mlx5_core_warn(res->mdev, "Failed to update packet merge state of direct TIR %#x for channel %u: err = %d\n", 613 mlx5e_tir_get_tirn(&res->channels[ix].direct_tir), ix, err); 614 if (!final_err) 615 final_err = err; 616 } 617 } 618 619 up_write(&res->pkt_merge_param_sem); 620 mlx5e_tir_builder_free(builder); 621 return final_err; 622 } 623 624 struct mlx5e_rss_params_hash mlx5e_rx_res_get_current_hash(struct mlx5e_rx_res *res) 625 { 626 return mlx5e_rss_get_hash(res->rss[0]); 627 } 628 629 int mlx5e_rx_res_tls_tir_create(struct mlx5e_rx_res *res, unsigned int rxq, 630 struct mlx5e_tir *tir) 631 { 632 bool inner_ft_support = res->features & MLX5E_RX_RES_FEATURE_INNER_FT; 633 struct mlx5e_tir_builder *builder; 634 u32 rqtn; 635 int err; 636 637 builder = mlx5e_tir_builder_alloc(false); 638 if (!builder) 639 return -ENOMEM; 640 641 rqtn = mlx5e_rx_res_get_rqtn_direct(res, rxq); 642 643 mlx5e_tir_builder_build_rqt(builder, res->mdev->mlx5e_res.hw_objs.td.tdn, rqtn, 644 inner_ft_support); 645 mlx5e_tir_builder_build_direct(builder); 646 mlx5e_tir_builder_build_tls(builder); 647 down_read(&res->pkt_merge_param_sem); 648 mlx5e_tir_builder_build_packet_merge(builder, &res->pkt_merge_param); 649 err = mlx5e_tir_init(tir, builder, res->mdev, false); 650 up_read(&res->pkt_merge_param_sem); 651 652 mlx5e_tir_builder_free(builder); 653 654 return err; 655 } 656