1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2019 Mellanox Technologies. */ 3 4 #include <linux/mlx5/eswitch.h> 5 #include <linux/err.h> 6 #include "dr_types.h" 7 8 #define DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, dmn_type) \ 9 ((dmn)->info.caps.dmn_type##_sw_owner || \ 10 ((dmn)->info.caps.dmn_type##_sw_owner_v2 && \ 11 (dmn)->info.caps.sw_format_ver <= MLX5_STEERING_FORMAT_CONNECTX_7)) 12 13 bool mlx5dr_domain_is_support_ptrn_arg(struct mlx5dr_domain *dmn) 14 { 15 return dmn->info.caps.sw_format_ver >= MLX5_STEERING_FORMAT_CONNECTX_6DX && 16 dmn->info.caps.support_modify_argument; 17 } 18 19 static int dr_domain_init_modify_header_resources(struct mlx5dr_domain *dmn) 20 { 21 if (!mlx5dr_domain_is_support_ptrn_arg(dmn)) 22 return 0; 23 24 dmn->ptrn_mgr = mlx5dr_ptrn_mgr_create(dmn); 25 if (!dmn->ptrn_mgr) { 26 mlx5dr_err(dmn, "Couldn't create ptrn_mgr\n"); 27 return -ENOMEM; 28 } 29 30 /* create argument pool */ 31 dmn->arg_mgr = mlx5dr_arg_mgr_create(dmn); 32 if (!dmn->arg_mgr) { 33 mlx5dr_err(dmn, "Couldn't create arg_mgr\n"); 34 goto free_modify_header_pattern; 35 } 36 37 return 0; 38 39 free_modify_header_pattern: 40 mlx5dr_ptrn_mgr_destroy(dmn->ptrn_mgr); 41 return -ENOMEM; 42 } 43 44 static void dr_domain_destroy_modify_header_resources(struct mlx5dr_domain *dmn) 45 { 46 if (!mlx5dr_domain_is_support_ptrn_arg(dmn)) 47 return; 48 49 mlx5dr_arg_mgr_destroy(dmn->arg_mgr); 50 mlx5dr_ptrn_mgr_destroy(dmn->ptrn_mgr); 51 } 52 53 static void dr_domain_init_csum_recalc_fts(struct mlx5dr_domain *dmn) 54 { 55 /* Per vport cached FW FT for checksum recalculation, this 56 * recalculation is needed due to a HW bug in STEv0. 57 */ 58 xa_init(&dmn->csum_fts_xa); 59 } 60 61 static void dr_domain_uninit_csum_recalc_fts(struct mlx5dr_domain *dmn) 62 { 63 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft; 64 unsigned long i; 65 66 xa_for_each(&dmn->csum_fts_xa, i, recalc_cs_ft) { 67 if (recalc_cs_ft) 68 mlx5dr_fw_destroy_recalc_cs_ft(dmn, recalc_cs_ft); 69 } 70 71 xa_destroy(&dmn->csum_fts_xa); 72 } 73 74 int mlx5dr_domain_get_recalc_cs_ft_addr(struct mlx5dr_domain *dmn, 75 u16 vport_num, 76 u64 *rx_icm_addr) 77 { 78 struct mlx5dr_fw_recalc_cs_ft *recalc_cs_ft; 79 int ret; 80 81 recalc_cs_ft = xa_load(&dmn->csum_fts_xa, vport_num); 82 if (!recalc_cs_ft) { 83 /* Table hasn't been created yet */ 84 recalc_cs_ft = mlx5dr_fw_create_recalc_cs_ft(dmn, vport_num); 85 if (!recalc_cs_ft) 86 return -EINVAL; 87 88 ret = xa_err(xa_store(&dmn->csum_fts_xa, vport_num, 89 recalc_cs_ft, GFP_KERNEL)); 90 if (ret) 91 return ret; 92 } 93 94 *rx_icm_addr = recalc_cs_ft->rx_icm_addr; 95 96 return 0; 97 } 98 99 static int dr_domain_init_mem_resources(struct mlx5dr_domain *dmn) 100 { 101 int ret; 102 103 dmn->chunks_kmem_cache = kmem_cache_create("mlx5_dr_chunks", 104 sizeof(struct mlx5dr_icm_chunk), 0, 105 SLAB_HWCACHE_ALIGN, NULL); 106 if (!dmn->chunks_kmem_cache) { 107 mlx5dr_err(dmn, "Couldn't create chunks kmem_cache\n"); 108 return -ENOMEM; 109 } 110 111 dmn->htbls_kmem_cache = kmem_cache_create("mlx5_dr_htbls", 112 sizeof(struct mlx5dr_ste_htbl), 0, 113 SLAB_HWCACHE_ALIGN, NULL); 114 if (!dmn->htbls_kmem_cache) { 115 mlx5dr_err(dmn, "Couldn't create hash tables kmem_cache\n"); 116 ret = -ENOMEM; 117 goto free_chunks_kmem_cache; 118 } 119 120 dmn->ste_icm_pool = mlx5dr_icm_pool_create(dmn, DR_ICM_TYPE_STE); 121 if (!dmn->ste_icm_pool) { 122 mlx5dr_err(dmn, "Couldn't get icm memory\n"); 123 ret = -ENOMEM; 124 goto free_htbls_kmem_cache; 125 } 126 127 dmn->action_icm_pool = mlx5dr_icm_pool_create(dmn, DR_ICM_TYPE_MODIFY_ACTION); 128 if (!dmn->action_icm_pool) { 129 mlx5dr_err(dmn, "Couldn't get action icm memory\n"); 130 ret = -ENOMEM; 131 goto free_ste_icm_pool; 132 } 133 134 ret = mlx5dr_send_info_pool_create(dmn); 135 if (ret) { 136 mlx5dr_err(dmn, "Couldn't create send info pool\n"); 137 goto free_action_icm_pool; 138 } 139 140 return 0; 141 142 free_action_icm_pool: 143 mlx5dr_icm_pool_destroy(dmn->action_icm_pool); 144 free_ste_icm_pool: 145 mlx5dr_icm_pool_destroy(dmn->ste_icm_pool); 146 free_htbls_kmem_cache: 147 kmem_cache_destroy(dmn->htbls_kmem_cache); 148 free_chunks_kmem_cache: 149 kmem_cache_destroy(dmn->chunks_kmem_cache); 150 151 return ret; 152 } 153 154 static void dr_domain_uninit_mem_resources(struct mlx5dr_domain *dmn) 155 { 156 mlx5dr_send_info_pool_destroy(dmn); 157 mlx5dr_icm_pool_destroy(dmn->action_icm_pool); 158 mlx5dr_icm_pool_destroy(dmn->ste_icm_pool); 159 kmem_cache_destroy(dmn->htbls_kmem_cache); 160 kmem_cache_destroy(dmn->chunks_kmem_cache); 161 } 162 163 static int dr_domain_init_resources(struct mlx5dr_domain *dmn) 164 { 165 int ret; 166 167 dmn->ste_ctx = mlx5dr_ste_get_ctx(dmn->info.caps.sw_format_ver); 168 if (!dmn->ste_ctx) { 169 mlx5dr_err(dmn, "SW Steering on this device is unsupported\n"); 170 return -EOPNOTSUPP; 171 } 172 173 ret = mlx5_core_alloc_pd(dmn->mdev, &dmn->pdn); 174 if (ret) { 175 mlx5dr_err(dmn, "Couldn't allocate PD, ret: %d", ret); 176 return ret; 177 } 178 179 dmn->uar = mlx5_get_uars_page(dmn->mdev); 180 if (IS_ERR(dmn->uar)) { 181 mlx5dr_err(dmn, "Couldn't allocate UAR\n"); 182 ret = PTR_ERR(dmn->uar); 183 goto clean_pd; 184 } 185 186 ret = dr_domain_init_mem_resources(dmn); 187 if (ret) { 188 mlx5dr_err(dmn, "Couldn't create domain memory resources\n"); 189 goto clean_uar; 190 } 191 192 ret = dr_domain_init_modify_header_resources(dmn); 193 if (ret) { 194 mlx5dr_err(dmn, "Couldn't create modify-header-resources\n"); 195 goto clean_mem_resources; 196 } 197 198 ret = mlx5dr_send_ring_alloc(dmn); 199 if (ret) { 200 mlx5dr_err(dmn, "Couldn't create send-ring\n"); 201 goto clean_modify_hdr; 202 } 203 204 return 0; 205 206 clean_modify_hdr: 207 dr_domain_destroy_modify_header_resources(dmn); 208 clean_mem_resources: 209 dr_domain_uninit_mem_resources(dmn); 210 clean_uar: 211 mlx5_put_uars_page(dmn->mdev, dmn->uar); 212 clean_pd: 213 mlx5_core_dealloc_pd(dmn->mdev, dmn->pdn); 214 215 return ret; 216 } 217 218 static void dr_domain_uninit_resources(struct mlx5dr_domain *dmn) 219 { 220 mlx5dr_send_ring_free(dmn, dmn->send_ring); 221 dr_domain_destroy_modify_header_resources(dmn); 222 dr_domain_uninit_mem_resources(dmn); 223 mlx5_put_uars_page(dmn->mdev, dmn->uar); 224 mlx5_core_dealloc_pd(dmn->mdev, dmn->pdn); 225 } 226 227 static void dr_domain_fill_uplink_caps(struct mlx5dr_domain *dmn, 228 struct mlx5dr_cmd_vport_cap *uplink_vport) 229 { 230 struct mlx5dr_esw_caps *esw_caps = &dmn->info.caps.esw_caps; 231 232 uplink_vport->num = MLX5_VPORT_UPLINK; 233 uplink_vport->icm_address_rx = esw_caps->uplink_icm_address_rx; 234 uplink_vport->icm_address_tx = esw_caps->uplink_icm_address_tx; 235 uplink_vport->vport_gvmi = 0; 236 uplink_vport->vhca_gvmi = dmn->info.caps.gvmi; 237 } 238 239 static int dr_domain_query_vport(struct mlx5dr_domain *dmn, 240 u16 vport_number, 241 bool other_vport, 242 struct mlx5dr_cmd_vport_cap *vport_caps) 243 { 244 int ret; 245 246 ret = mlx5dr_cmd_query_esw_vport_context(dmn->mdev, 247 other_vport, 248 vport_number, 249 &vport_caps->icm_address_rx, 250 &vport_caps->icm_address_tx); 251 if (ret) 252 return ret; 253 254 ret = mlx5dr_cmd_query_gvmi(dmn->mdev, 255 other_vport, 256 vport_number, 257 &vport_caps->vport_gvmi); 258 if (ret) 259 return ret; 260 261 vport_caps->num = vport_number; 262 vport_caps->vhca_gvmi = dmn->info.caps.gvmi; 263 264 return 0; 265 } 266 267 static int dr_domain_query_esw_mgr(struct mlx5dr_domain *dmn) 268 { 269 return dr_domain_query_vport(dmn, 0, false, 270 &dmn->info.caps.vports.esw_manager_caps); 271 } 272 273 static void dr_domain_query_uplink(struct mlx5dr_domain *dmn) 274 { 275 dr_domain_fill_uplink_caps(dmn, &dmn->info.caps.vports.uplink_caps); 276 } 277 278 static struct mlx5dr_cmd_vport_cap * 279 dr_domain_add_vport_cap(struct mlx5dr_domain *dmn, u16 vport) 280 { 281 struct mlx5dr_cmd_caps *caps = &dmn->info.caps; 282 struct mlx5dr_cmd_vport_cap *vport_caps; 283 int ret; 284 285 vport_caps = kvzalloc(sizeof(*vport_caps), GFP_KERNEL); 286 if (!vport_caps) 287 return NULL; 288 289 ret = dr_domain_query_vport(dmn, vport, true, vport_caps); 290 if (ret) { 291 kvfree(vport_caps); 292 return NULL; 293 } 294 295 ret = xa_insert(&caps->vports.vports_caps_xa, vport, 296 vport_caps, GFP_KERNEL); 297 if (ret) { 298 mlx5dr_dbg(dmn, "Couldn't insert new vport into xarray (%d)\n", ret); 299 kvfree(vport_caps); 300 return ERR_PTR(ret); 301 } 302 303 return vport_caps; 304 } 305 306 static bool dr_domain_is_esw_mgr_vport(struct mlx5dr_domain *dmn, u16 vport) 307 { 308 struct mlx5dr_cmd_caps *caps = &dmn->info.caps; 309 310 return (caps->is_ecpf && vport == MLX5_VPORT_ECPF) || 311 (!caps->is_ecpf && vport == 0); 312 } 313 314 struct mlx5dr_cmd_vport_cap * 315 mlx5dr_domain_get_vport_cap(struct mlx5dr_domain *dmn, u16 vport) 316 { 317 struct mlx5dr_cmd_caps *caps = &dmn->info.caps; 318 struct mlx5dr_cmd_vport_cap *vport_caps; 319 320 if (dr_domain_is_esw_mgr_vport(dmn, vport)) 321 return &caps->vports.esw_manager_caps; 322 323 if (vport == MLX5_VPORT_UPLINK) 324 return &caps->vports.uplink_caps; 325 326 vport_load: 327 vport_caps = xa_load(&caps->vports.vports_caps_xa, vport); 328 if (vport_caps) 329 return vport_caps; 330 331 vport_caps = dr_domain_add_vport_cap(dmn, vport); 332 if (PTR_ERR(vport_caps) == -EBUSY) 333 /* caps were already stored by another thread */ 334 goto vport_load; 335 336 return vport_caps; 337 } 338 339 static void dr_domain_clear_vports(struct mlx5dr_domain *dmn) 340 { 341 struct mlx5dr_cmd_vport_cap *vport_caps; 342 unsigned long i; 343 344 xa_for_each(&dmn->info.caps.vports.vports_caps_xa, i, vport_caps) { 345 vport_caps = xa_erase(&dmn->info.caps.vports.vports_caps_xa, i); 346 kvfree(vport_caps); 347 } 348 } 349 350 static int dr_domain_query_fdb_caps(struct mlx5_core_dev *mdev, 351 struct mlx5dr_domain *dmn) 352 { 353 int ret; 354 355 if (!dmn->info.caps.eswitch_manager) 356 return -EOPNOTSUPP; 357 358 ret = mlx5dr_cmd_query_esw_caps(mdev, &dmn->info.caps.esw_caps); 359 if (ret) 360 return ret; 361 362 dmn->info.caps.fdb_sw_owner = dmn->info.caps.esw_caps.sw_owner; 363 dmn->info.caps.fdb_sw_owner_v2 = dmn->info.caps.esw_caps.sw_owner_v2; 364 dmn->info.caps.esw_rx_drop_address = dmn->info.caps.esw_caps.drop_icm_address_rx; 365 dmn->info.caps.esw_tx_drop_address = dmn->info.caps.esw_caps.drop_icm_address_tx; 366 367 xa_init(&dmn->info.caps.vports.vports_caps_xa); 368 369 /* Query eswitch manager and uplink vports only. Rest of the 370 * vports (vport 0, VFs and SFs) will be queried dynamically. 371 */ 372 373 ret = dr_domain_query_esw_mgr(dmn); 374 if (ret) { 375 mlx5dr_err(dmn, "Failed to query eswitch manager vport caps (err: %d)", ret); 376 goto free_vports_caps_xa; 377 } 378 379 dr_domain_query_uplink(dmn); 380 381 return 0; 382 383 free_vports_caps_xa: 384 xa_destroy(&dmn->info.caps.vports.vports_caps_xa); 385 386 return ret; 387 } 388 389 static int dr_domain_caps_init(struct mlx5_core_dev *mdev, 390 struct mlx5dr_domain *dmn) 391 { 392 struct mlx5dr_cmd_vport_cap *vport_cap; 393 int ret; 394 395 if (MLX5_CAP_GEN(mdev, port_type) != MLX5_CAP_PORT_TYPE_ETH) { 396 mlx5dr_err(dmn, "Failed to allocate domain, bad link type\n"); 397 return -EOPNOTSUPP; 398 } 399 400 ret = mlx5dr_cmd_query_device(mdev, &dmn->info.caps); 401 if (ret) 402 return ret; 403 404 ret = dr_domain_query_fdb_caps(mdev, dmn); 405 if (ret) 406 return ret; 407 408 switch (dmn->type) { 409 case MLX5DR_DOMAIN_TYPE_NIC_RX: 410 if (!DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, rx)) 411 return -ENOTSUPP; 412 413 dmn->info.supp_sw_steering = true; 414 dmn->info.rx.type = DR_DOMAIN_NIC_TYPE_RX; 415 dmn->info.rx.default_icm_addr = dmn->info.caps.nic_rx_drop_address; 416 dmn->info.rx.drop_icm_addr = dmn->info.caps.nic_rx_drop_address; 417 break; 418 case MLX5DR_DOMAIN_TYPE_NIC_TX: 419 if (!DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, tx)) 420 return -ENOTSUPP; 421 422 dmn->info.supp_sw_steering = true; 423 dmn->info.tx.type = DR_DOMAIN_NIC_TYPE_TX; 424 dmn->info.tx.default_icm_addr = dmn->info.caps.nic_tx_allow_address; 425 dmn->info.tx.drop_icm_addr = dmn->info.caps.nic_tx_drop_address; 426 break; 427 case MLX5DR_DOMAIN_TYPE_FDB: 428 if (!dmn->info.caps.eswitch_manager) 429 return -ENOTSUPP; 430 431 if (!DR_DOMAIN_SW_STEERING_SUPPORTED(dmn, fdb)) 432 return -ENOTSUPP; 433 434 dmn->info.rx.type = DR_DOMAIN_NIC_TYPE_RX; 435 dmn->info.tx.type = DR_DOMAIN_NIC_TYPE_TX; 436 vport_cap = &dmn->info.caps.vports.esw_manager_caps; 437 438 dmn->info.supp_sw_steering = true; 439 dmn->info.tx.default_icm_addr = vport_cap->icm_address_tx; 440 dmn->info.rx.default_icm_addr = vport_cap->icm_address_rx; 441 dmn->info.rx.drop_icm_addr = dmn->info.caps.esw_rx_drop_address; 442 dmn->info.tx.drop_icm_addr = dmn->info.caps.esw_tx_drop_address; 443 break; 444 default: 445 mlx5dr_err(dmn, "Invalid domain\n"); 446 ret = -EINVAL; 447 break; 448 } 449 450 return ret; 451 } 452 453 static void dr_domain_caps_uninit(struct mlx5dr_domain *dmn) 454 { 455 dr_domain_clear_vports(dmn); 456 xa_destroy(&dmn->info.caps.vports.vports_caps_xa); 457 } 458 459 struct mlx5dr_domain * 460 mlx5dr_domain_create(struct mlx5_core_dev *mdev, enum mlx5dr_domain_type type) 461 { 462 struct mlx5dr_domain *dmn; 463 int ret; 464 465 if (type > MLX5DR_DOMAIN_TYPE_FDB) 466 return NULL; 467 468 dmn = kzalloc(sizeof(*dmn), GFP_KERNEL); 469 if (!dmn) 470 return NULL; 471 472 dmn->mdev = mdev; 473 dmn->type = type; 474 refcount_set(&dmn->refcount, 1); 475 mutex_init(&dmn->info.rx.mutex); 476 mutex_init(&dmn->info.tx.mutex); 477 xa_init(&dmn->definers_xa); 478 xa_init(&dmn->peer_dmn_xa); 479 480 if (dr_domain_caps_init(mdev, dmn)) { 481 mlx5dr_err(dmn, "Failed init domain, no caps\n"); 482 goto def_xa_destroy; 483 } 484 485 dmn->info.max_log_action_icm_sz = DR_CHUNK_SIZE_4K; 486 dmn->info.max_log_sw_icm_sz = min_t(u32, DR_CHUNK_SIZE_1024K, 487 dmn->info.caps.log_icm_size); 488 dmn->info.max_log_modify_hdr_pattern_icm_sz = 489 min_t(u32, DR_CHUNK_SIZE_4K, 490 dmn->info.caps.log_modify_pattern_icm_size); 491 492 if (!dmn->info.supp_sw_steering) { 493 mlx5dr_err(dmn, "SW steering is not supported\n"); 494 goto uninit_caps; 495 } 496 497 /* Allocate resources */ 498 ret = dr_domain_init_resources(dmn); 499 if (ret) { 500 mlx5dr_err(dmn, "Failed init domain resources\n"); 501 goto uninit_caps; 502 } 503 504 dr_domain_init_csum_recalc_fts(dmn); 505 mlx5dr_dbg_init_dump(dmn); 506 return dmn; 507 508 uninit_caps: 509 dr_domain_caps_uninit(dmn); 510 def_xa_destroy: 511 xa_destroy(&dmn->peer_dmn_xa); 512 xa_destroy(&dmn->definers_xa); 513 kfree(dmn); 514 return NULL; 515 } 516 517 /* Assure synchronization of the device steering tables with updates made by SW 518 * insertion. 519 */ 520 int mlx5dr_domain_sync(struct mlx5dr_domain *dmn, u32 flags) 521 { 522 int ret = 0; 523 524 if (flags & MLX5DR_DOMAIN_SYNC_FLAGS_SW) { 525 mlx5dr_domain_lock(dmn); 526 ret = mlx5dr_send_ring_force_drain(dmn); 527 mlx5dr_domain_unlock(dmn); 528 if (ret) { 529 mlx5dr_err(dmn, "Force drain failed flags: %d, ret: %d\n", 530 flags, ret); 531 return ret; 532 } 533 } 534 535 if (flags & MLX5DR_DOMAIN_SYNC_FLAGS_HW) 536 ret = mlx5dr_cmd_sync_steering(dmn->mdev); 537 538 return ret; 539 } 540 541 int mlx5dr_domain_destroy(struct mlx5dr_domain *dmn) 542 { 543 if (WARN_ON_ONCE(refcount_read(&dmn->refcount) > 1)) 544 return -EBUSY; 545 546 /* make sure resources are not used by the hardware */ 547 mlx5dr_cmd_sync_steering(dmn->mdev); 548 mlx5dr_dbg_uninit_dump(dmn); 549 dr_domain_uninit_csum_recalc_fts(dmn); 550 dr_domain_uninit_resources(dmn); 551 dr_domain_caps_uninit(dmn); 552 xa_destroy(&dmn->peer_dmn_xa); 553 xa_destroy(&dmn->definers_xa); 554 mutex_destroy(&dmn->info.tx.mutex); 555 mutex_destroy(&dmn->info.rx.mutex); 556 kfree(dmn); 557 return 0; 558 } 559 560 void mlx5dr_domain_set_peer(struct mlx5dr_domain *dmn, 561 struct mlx5dr_domain *peer_dmn, 562 u16 peer_vhca_id) 563 { 564 struct mlx5dr_domain *peer; 565 566 mlx5dr_domain_lock(dmn); 567 568 peer = xa_load(&dmn->peer_dmn_xa, peer_vhca_id); 569 if (peer) 570 refcount_dec(&peer->refcount); 571 572 WARN_ON(xa_err(xa_store(&dmn->peer_dmn_xa, peer_vhca_id, peer_dmn, GFP_KERNEL))); 573 574 peer = xa_load(&dmn->peer_dmn_xa, peer_vhca_id); 575 if (peer) 576 refcount_inc(&peer->refcount); 577 578 mlx5dr_domain_unlock(dmn); 579 } 580