1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 /* Copyright (c) 2020, Mellanox Technologies inc. All rights reserved. */ 3 4 #include <linux/netdevice.h> 5 #include "en.h" 6 #include "en/fs.h" 7 #include "eswitch.h" 8 #include "ipsec.h" 9 #include "fs_core.h" 10 #include "lib/ipsec_fs_roce.h" 11 #include "lib/fs_chains.h" 12 #include "esw/ipsec_fs.h" 13 #include "en_rep.h" 14 15 #define NUM_IPSEC_FTE BIT(15) 16 #define MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE 16 17 #define IPSEC_TUNNEL_DEFAULT_TTL 0x40 18 19 struct mlx5e_ipsec_fc { 20 struct mlx5_fc *cnt; 21 struct mlx5_fc *drop; 22 }; 23 24 struct mlx5e_ipsec_tx { 25 struct mlx5e_ipsec_ft ft; 26 struct mlx5e_ipsec_miss pol; 27 struct mlx5e_ipsec_miss sa; 28 struct mlx5e_ipsec_rule status; 29 struct mlx5_flow_namespace *ns; 30 struct mlx5e_ipsec_fc *fc; 31 struct mlx5_fs_chains *chains; 32 u8 allow_tunnel_mode : 1; 33 }; 34 35 /* IPsec RX flow steering */ 36 static enum mlx5_traffic_types family2tt(u32 family) 37 { 38 if (family == AF_INET) 39 return MLX5_TT_IPV4_IPSEC_ESP; 40 return MLX5_TT_IPV6_IPSEC_ESP; 41 } 42 43 static struct mlx5e_ipsec_rx *ipsec_rx(struct mlx5e_ipsec *ipsec, u32 family, int type) 44 { 45 if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) 46 return ipsec->rx_esw; 47 48 if (family == AF_INET) 49 return ipsec->rx_ipv4; 50 51 return ipsec->rx_ipv6; 52 } 53 54 static struct mlx5e_ipsec_tx *ipsec_tx(struct mlx5e_ipsec *ipsec, int type) 55 { 56 if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) 57 return ipsec->tx_esw; 58 59 return ipsec->tx; 60 } 61 62 static struct mlx5_fs_chains * 63 ipsec_chains_create(struct mlx5_core_dev *mdev, struct mlx5_flow_table *miss_ft, 64 enum mlx5_flow_namespace_type ns, int base_prio, 65 int base_level, struct mlx5_flow_table **root_ft) 66 { 67 struct mlx5_chains_attr attr = {}; 68 struct mlx5_fs_chains *chains; 69 struct mlx5_flow_table *ft; 70 int err; 71 72 attr.flags = MLX5_CHAINS_AND_PRIOS_SUPPORTED | 73 MLX5_CHAINS_IGNORE_FLOW_LEVEL_SUPPORTED; 74 attr.max_grp_num = 2; 75 attr.default_ft = miss_ft; 76 attr.ns = ns; 77 attr.fs_base_prio = base_prio; 78 attr.fs_base_level = base_level; 79 chains = mlx5_chains_create(mdev, &attr); 80 if (IS_ERR(chains)) 81 return chains; 82 83 /* Create chain 0, prio 1, level 0 to connect chains to prev in fs_core */ 84 ft = mlx5_chains_get_table(chains, 0, 1, 0); 85 if (IS_ERR(ft)) { 86 err = PTR_ERR(ft); 87 goto err_chains_get; 88 } 89 90 *root_ft = ft; 91 return chains; 92 93 err_chains_get: 94 mlx5_chains_destroy(chains); 95 return ERR_PTR(err); 96 } 97 98 static void ipsec_chains_destroy(struct mlx5_fs_chains *chains) 99 { 100 mlx5_chains_put_table(chains, 0, 1, 0); 101 mlx5_chains_destroy(chains); 102 } 103 104 static struct mlx5_flow_table * 105 ipsec_chains_get_table(struct mlx5_fs_chains *chains, u32 prio) 106 { 107 return mlx5_chains_get_table(chains, 0, prio + 1, 0); 108 } 109 110 static void ipsec_chains_put_table(struct mlx5_fs_chains *chains, u32 prio) 111 { 112 mlx5_chains_put_table(chains, 0, prio + 1, 0); 113 } 114 115 static struct mlx5_flow_table *ipsec_ft_create(struct mlx5_flow_namespace *ns, 116 int level, int prio, 117 int max_num_groups, u32 flags) 118 { 119 struct mlx5_flow_table_attr ft_attr = {}; 120 121 ft_attr.autogroup.num_reserved_entries = 1; 122 ft_attr.autogroup.max_num_groups = max_num_groups; 123 ft_attr.max_fte = NUM_IPSEC_FTE; 124 ft_attr.level = level; 125 ft_attr.prio = prio; 126 ft_attr.flags = flags; 127 128 return mlx5_create_auto_grouped_flow_table(ns, &ft_attr); 129 } 130 131 static int ipsec_status_rule(struct mlx5_core_dev *mdev, 132 struct mlx5e_ipsec_rx *rx, 133 struct mlx5_flow_destination *dest) 134 { 135 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 136 struct mlx5_flow_act flow_act = {}; 137 struct mlx5_modify_hdr *modify_hdr; 138 struct mlx5_flow_handle *fte; 139 struct mlx5_flow_spec *spec; 140 int err; 141 142 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 143 if (!spec) 144 return -ENOMEM; 145 146 /* Action to copy 7 bit ipsec_syndrome to regB[24:30] */ 147 MLX5_SET(copy_action_in, action, action_type, MLX5_ACTION_TYPE_COPY); 148 MLX5_SET(copy_action_in, action, src_field, MLX5_ACTION_IN_FIELD_IPSEC_SYNDROME); 149 MLX5_SET(copy_action_in, action, src_offset, 0); 150 MLX5_SET(copy_action_in, action, length, 7); 151 MLX5_SET(copy_action_in, action, dst_field, MLX5_ACTION_IN_FIELD_METADATA_REG_B); 152 MLX5_SET(copy_action_in, action, dst_offset, 24); 153 154 modify_hdr = mlx5_modify_header_alloc(mdev, MLX5_FLOW_NAMESPACE_KERNEL, 155 1, action); 156 157 if (IS_ERR(modify_hdr)) { 158 err = PTR_ERR(modify_hdr); 159 mlx5_core_err(mdev, 160 "fail to alloc ipsec copy modify_header_id err=%d\n", err); 161 goto out_spec; 162 } 163 164 /* create fte */ 165 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_MOD_HDR | 166 MLX5_FLOW_CONTEXT_ACTION_FWD_DEST | 167 MLX5_FLOW_CONTEXT_ACTION_COUNT; 168 flow_act.modify_hdr = modify_hdr; 169 fte = mlx5_add_flow_rules(rx->ft.status, spec, &flow_act, dest, 2); 170 if (IS_ERR(fte)) { 171 err = PTR_ERR(fte); 172 mlx5_core_err(mdev, "fail to add ipsec rx err copy rule err=%d\n", err); 173 goto out; 174 } 175 176 kvfree(spec); 177 rx->status.rule = fte; 178 rx->status.modify_hdr = modify_hdr; 179 return 0; 180 181 out: 182 mlx5_modify_header_dealloc(mdev, modify_hdr); 183 out_spec: 184 kvfree(spec); 185 return err; 186 } 187 188 static int ipsec_miss_create(struct mlx5_core_dev *mdev, 189 struct mlx5_flow_table *ft, 190 struct mlx5e_ipsec_miss *miss, 191 struct mlx5_flow_destination *dest) 192 { 193 int inlen = MLX5_ST_SZ_BYTES(create_flow_group_in); 194 MLX5_DECLARE_FLOW_ACT(flow_act); 195 struct mlx5_flow_spec *spec; 196 u32 *flow_group_in; 197 int err = 0; 198 199 flow_group_in = kvzalloc(inlen, GFP_KERNEL); 200 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 201 if (!flow_group_in || !spec) { 202 err = -ENOMEM; 203 goto out; 204 } 205 206 /* Create miss_group */ 207 MLX5_SET(create_flow_group_in, flow_group_in, start_flow_index, ft->max_fte - 1); 208 MLX5_SET(create_flow_group_in, flow_group_in, end_flow_index, ft->max_fte - 1); 209 miss->group = mlx5_create_flow_group(ft, flow_group_in); 210 if (IS_ERR(miss->group)) { 211 err = PTR_ERR(miss->group); 212 mlx5_core_err(mdev, "fail to create IPsec miss_group err=%d\n", 213 err); 214 goto out; 215 } 216 217 /* Create miss rule */ 218 miss->rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, 1); 219 if (IS_ERR(miss->rule)) { 220 mlx5_destroy_flow_group(miss->group); 221 err = PTR_ERR(miss->rule); 222 mlx5_core_err(mdev, "fail to create IPsec miss_rule err=%d\n", 223 err); 224 goto out; 225 } 226 out: 227 kvfree(flow_group_in); 228 kvfree(spec); 229 return err; 230 } 231 232 static void handle_ipsec_rx_bringup(struct mlx5e_ipsec *ipsec, u32 family) 233 { 234 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); 235 struct mlx5_flow_namespace *ns = mlx5e_fs_get_ns(ipsec->fs, false); 236 struct mlx5_flow_destination old_dest, new_dest; 237 238 old_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false), 239 family2tt(family)); 240 241 mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, ns, &old_dest, family, 242 MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, MLX5E_NIC_PRIO); 243 244 new_dest.ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); 245 new_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 246 mlx5_modify_rule_destination(rx->status.rule, &new_dest, &old_dest); 247 mlx5_modify_rule_destination(rx->sa.rule, &new_dest, &old_dest); 248 } 249 250 static void handle_ipsec_rx_cleanup(struct mlx5e_ipsec *ipsec, u32 family) 251 { 252 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, XFRM_DEV_OFFLOAD_PACKET); 253 struct mlx5_flow_destination old_dest, new_dest; 254 255 old_dest.ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, family); 256 old_dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 257 new_dest = mlx5_ttc_get_default_dest(mlx5e_fs_get_ttc(ipsec->fs, false), 258 family2tt(family)); 259 mlx5_modify_rule_destination(rx->sa.rule, &new_dest, &old_dest); 260 mlx5_modify_rule_destination(rx->status.rule, &new_dest, &old_dest); 261 262 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, ipsec->mdev); 263 } 264 265 static void ipsec_mpv_work_handler(struct work_struct *_work) 266 { 267 struct mlx5e_ipsec_mpv_work *work = container_of(_work, struct mlx5e_ipsec_mpv_work, work); 268 struct mlx5e_ipsec *ipsec = work->slave_priv->ipsec; 269 270 switch (work->event) { 271 case MPV_DEVCOM_IPSEC_MASTER_UP: 272 mutex_lock(&ipsec->tx->ft.mutex); 273 if (ipsec->tx->ft.refcnt) 274 mlx5_ipsec_fs_roce_tx_create(ipsec->mdev, ipsec->roce, ipsec->tx->ft.pol, 275 true); 276 mutex_unlock(&ipsec->tx->ft.mutex); 277 278 mutex_lock(&ipsec->rx_ipv4->ft.mutex); 279 if (ipsec->rx_ipv4->ft.refcnt) 280 handle_ipsec_rx_bringup(ipsec, AF_INET); 281 mutex_unlock(&ipsec->rx_ipv4->ft.mutex); 282 283 mutex_lock(&ipsec->rx_ipv6->ft.mutex); 284 if (ipsec->rx_ipv6->ft.refcnt) 285 handle_ipsec_rx_bringup(ipsec, AF_INET6); 286 mutex_unlock(&ipsec->rx_ipv6->ft.mutex); 287 break; 288 case MPV_DEVCOM_IPSEC_MASTER_DOWN: 289 mutex_lock(&ipsec->tx->ft.mutex); 290 if (ipsec->tx->ft.refcnt) 291 mlx5_ipsec_fs_roce_tx_destroy(ipsec->roce, ipsec->mdev); 292 mutex_unlock(&ipsec->tx->ft.mutex); 293 294 mutex_lock(&ipsec->rx_ipv4->ft.mutex); 295 if (ipsec->rx_ipv4->ft.refcnt) 296 handle_ipsec_rx_cleanup(ipsec, AF_INET); 297 mutex_unlock(&ipsec->rx_ipv4->ft.mutex); 298 299 mutex_lock(&ipsec->rx_ipv6->ft.mutex); 300 if (ipsec->rx_ipv6->ft.refcnt) 301 handle_ipsec_rx_cleanup(ipsec, AF_INET6); 302 mutex_unlock(&ipsec->rx_ipv6->ft.mutex); 303 break; 304 } 305 306 complete(&work->master_priv->ipsec->comp); 307 } 308 309 static void ipsec_rx_ft_disconnect(struct mlx5e_ipsec *ipsec, u32 family) 310 { 311 struct mlx5_ttc_table *ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 312 313 mlx5_ttc_fwd_default_dest(ttc, family2tt(family)); 314 } 315 316 static void rx_destroy(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 317 struct mlx5e_ipsec_rx *rx, u32 family) 318 { 319 /* disconnect */ 320 if (rx != ipsec->rx_esw) 321 ipsec_rx_ft_disconnect(ipsec, family); 322 323 if (rx->chains) { 324 ipsec_chains_destroy(rx->chains); 325 } else { 326 mlx5_del_flow_rules(rx->pol.rule); 327 mlx5_destroy_flow_group(rx->pol.group); 328 mlx5_destroy_flow_table(rx->ft.pol); 329 } 330 331 mlx5_del_flow_rules(rx->sa.rule); 332 mlx5_destroy_flow_group(rx->sa.group); 333 mlx5_destroy_flow_table(rx->ft.sa); 334 if (rx->allow_tunnel_mode) 335 mlx5_eswitch_unblock_encap(mdev); 336 if (rx == ipsec->rx_esw) { 337 mlx5_esw_ipsec_rx_status_destroy(ipsec, rx); 338 } else { 339 mlx5_del_flow_rules(rx->status.rule); 340 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 341 } 342 mlx5_destroy_flow_table(rx->ft.status); 343 344 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, mdev); 345 } 346 347 static void ipsec_rx_create_attr_set(struct mlx5e_ipsec *ipsec, 348 struct mlx5e_ipsec_rx *rx, 349 u32 family, 350 struct mlx5e_ipsec_rx_create_attr *attr) 351 { 352 if (rx == ipsec->rx_esw) { 353 /* For packet offload in switchdev mode, RX & TX use FDB namespace */ 354 attr->ns = ipsec->tx_esw->ns; 355 mlx5_esw_ipsec_rx_create_attr_set(ipsec, attr); 356 return; 357 } 358 359 attr->ns = mlx5e_fs_get_ns(ipsec->fs, false); 360 attr->ttc = mlx5e_fs_get_ttc(ipsec->fs, false); 361 attr->family = family; 362 attr->prio = MLX5E_NIC_PRIO; 363 attr->pol_level = MLX5E_ACCEL_FS_POL_FT_LEVEL; 364 attr->sa_level = MLX5E_ACCEL_FS_ESP_FT_LEVEL; 365 attr->status_level = MLX5E_ACCEL_FS_ESP_FT_ERR_LEVEL; 366 attr->chains_ns = MLX5_FLOW_NAMESPACE_KERNEL; 367 } 368 369 static int ipsec_rx_status_pass_dest_get(struct mlx5e_ipsec *ipsec, 370 struct mlx5e_ipsec_rx *rx, 371 struct mlx5e_ipsec_rx_create_attr *attr, 372 struct mlx5_flow_destination *dest) 373 { 374 struct mlx5_flow_table *ft; 375 int err; 376 377 if (rx == ipsec->rx_esw) 378 return mlx5_esw_ipsec_rx_status_pass_dest_get(ipsec, dest); 379 380 *dest = mlx5_ttc_get_default_dest(attr->ttc, family2tt(attr->family)); 381 err = mlx5_ipsec_fs_roce_rx_create(ipsec->mdev, ipsec->roce, attr->ns, dest, 382 attr->family, MLX5E_ACCEL_FS_ESP_FT_ROCE_LEVEL, 383 attr->prio); 384 if (err) 385 return err; 386 387 ft = mlx5_ipsec_fs_roce_ft_get(ipsec->roce, attr->family); 388 if (ft) { 389 dest->type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 390 dest->ft = ft; 391 } 392 393 return 0; 394 } 395 396 static void ipsec_rx_ft_connect(struct mlx5e_ipsec *ipsec, 397 struct mlx5e_ipsec_rx *rx, 398 struct mlx5e_ipsec_rx_create_attr *attr) 399 { 400 struct mlx5_flow_destination dest = {}; 401 402 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 403 dest.ft = rx->ft.pol; 404 mlx5_ttc_fwd_dest(attr->ttc, family2tt(attr->family), &dest); 405 } 406 407 static int rx_create(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 408 struct mlx5e_ipsec_rx *rx, u32 family) 409 { 410 struct mlx5e_ipsec_rx_create_attr attr; 411 struct mlx5_flow_destination dest[2]; 412 struct mlx5_flow_table *ft; 413 u32 flags = 0; 414 int err; 415 416 ipsec_rx_create_attr_set(ipsec, rx, family, &attr); 417 418 err = ipsec_rx_status_pass_dest_get(ipsec, rx, &attr, &dest[0]); 419 if (err) 420 return err; 421 422 ft = ipsec_ft_create(attr.ns, attr.status_level, attr.prio, 1, 0); 423 if (IS_ERR(ft)) { 424 err = PTR_ERR(ft); 425 goto err_fs_ft_status; 426 } 427 rx->ft.status = ft; 428 429 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 430 dest[1].counter_id = mlx5_fc_id(rx->fc->cnt); 431 if (rx == ipsec->rx_esw) 432 err = mlx5_esw_ipsec_rx_status_create(ipsec, rx, dest); 433 else 434 err = ipsec_status_rule(mdev, rx, dest); 435 if (err) 436 goto err_add; 437 438 /* Create FT */ 439 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) 440 rx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); 441 if (rx->allow_tunnel_mode) 442 flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 443 ft = ipsec_ft_create(attr.ns, attr.sa_level, attr.prio, 2, flags); 444 if (IS_ERR(ft)) { 445 err = PTR_ERR(ft); 446 goto err_fs_ft; 447 } 448 rx->ft.sa = ft; 449 450 err = ipsec_miss_create(mdev, rx->ft.sa, &rx->sa, dest); 451 if (err) 452 goto err_fs; 453 454 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 455 rx->chains = ipsec_chains_create(mdev, rx->ft.sa, 456 attr.chains_ns, 457 attr.prio, 458 attr.pol_level, 459 &rx->ft.pol); 460 if (IS_ERR(rx->chains)) { 461 err = PTR_ERR(rx->chains); 462 goto err_pol_ft; 463 } 464 465 goto connect; 466 } 467 468 ft = ipsec_ft_create(attr.ns, attr.pol_level, attr.prio, 2, 0); 469 if (IS_ERR(ft)) { 470 err = PTR_ERR(ft); 471 goto err_pol_ft; 472 } 473 rx->ft.pol = ft; 474 memset(dest, 0x00, 2 * sizeof(*dest)); 475 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 476 dest[0].ft = rx->ft.sa; 477 err = ipsec_miss_create(mdev, rx->ft.pol, &rx->pol, dest); 478 if (err) 479 goto err_pol_miss; 480 481 connect: 482 /* connect */ 483 if (rx != ipsec->rx_esw) 484 ipsec_rx_ft_connect(ipsec, rx, &attr); 485 return 0; 486 487 err_pol_miss: 488 mlx5_destroy_flow_table(rx->ft.pol); 489 err_pol_ft: 490 mlx5_del_flow_rules(rx->sa.rule); 491 mlx5_destroy_flow_group(rx->sa.group); 492 err_fs: 493 mlx5_destroy_flow_table(rx->ft.sa); 494 err_fs_ft: 495 if (rx->allow_tunnel_mode) 496 mlx5_eswitch_unblock_encap(mdev); 497 mlx5_del_flow_rules(rx->status.rule); 498 mlx5_modify_header_dealloc(mdev, rx->status.modify_hdr); 499 err_add: 500 mlx5_destroy_flow_table(rx->ft.status); 501 err_fs_ft_status: 502 mlx5_ipsec_fs_roce_rx_destroy(ipsec->roce, family, mdev); 503 return err; 504 } 505 506 static int rx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 507 struct mlx5e_ipsec_rx *rx, u32 family) 508 { 509 int err; 510 511 if (rx->ft.refcnt) 512 goto skip; 513 514 err = mlx5_eswitch_block_mode(mdev); 515 if (err) 516 return err; 517 518 err = rx_create(mdev, ipsec, rx, family); 519 if (err) { 520 mlx5_eswitch_unblock_mode(mdev); 521 return err; 522 } 523 524 skip: 525 rx->ft.refcnt++; 526 return 0; 527 } 528 529 static void rx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_rx *rx, 530 u32 family) 531 { 532 if (--rx->ft.refcnt) 533 return; 534 535 rx_destroy(ipsec->mdev, ipsec, rx, family); 536 mlx5_eswitch_unblock_mode(ipsec->mdev); 537 } 538 539 static struct mlx5e_ipsec_rx *rx_ft_get(struct mlx5_core_dev *mdev, 540 struct mlx5e_ipsec *ipsec, u32 family, 541 int type) 542 { 543 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 544 int err; 545 546 mutex_lock(&rx->ft.mutex); 547 err = rx_get(mdev, ipsec, rx, family); 548 mutex_unlock(&rx->ft.mutex); 549 if (err) 550 return ERR_PTR(err); 551 552 return rx; 553 } 554 555 static struct mlx5_flow_table *rx_ft_get_policy(struct mlx5_core_dev *mdev, 556 struct mlx5e_ipsec *ipsec, 557 u32 family, u32 prio, int type) 558 { 559 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 560 struct mlx5_flow_table *ft; 561 int err; 562 563 mutex_lock(&rx->ft.mutex); 564 err = rx_get(mdev, ipsec, rx, family); 565 if (err) 566 goto err_get; 567 568 ft = rx->chains ? ipsec_chains_get_table(rx->chains, prio) : rx->ft.pol; 569 if (IS_ERR(ft)) { 570 err = PTR_ERR(ft); 571 goto err_get_ft; 572 } 573 574 mutex_unlock(&rx->ft.mutex); 575 return ft; 576 577 err_get_ft: 578 rx_put(ipsec, rx, family); 579 err_get: 580 mutex_unlock(&rx->ft.mutex); 581 return ERR_PTR(err); 582 } 583 584 static void rx_ft_put(struct mlx5e_ipsec *ipsec, u32 family, int type) 585 { 586 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 587 588 mutex_lock(&rx->ft.mutex); 589 rx_put(ipsec, rx, family); 590 mutex_unlock(&rx->ft.mutex); 591 } 592 593 static void rx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 family, u32 prio, int type) 594 { 595 struct mlx5e_ipsec_rx *rx = ipsec_rx(ipsec, family, type); 596 597 mutex_lock(&rx->ft.mutex); 598 if (rx->chains) 599 ipsec_chains_put_table(rx->chains, prio); 600 601 rx_put(ipsec, rx, family); 602 mutex_unlock(&rx->ft.mutex); 603 } 604 605 static int ipsec_counter_rule_tx(struct mlx5_core_dev *mdev, struct mlx5e_ipsec_tx *tx) 606 { 607 struct mlx5_flow_destination dest = {}; 608 struct mlx5_flow_act flow_act = {}; 609 struct mlx5_flow_handle *fte; 610 struct mlx5_flow_spec *spec; 611 int err; 612 613 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 614 if (!spec) 615 return -ENOMEM; 616 617 /* create fte */ 618 flow_act.action = MLX5_FLOW_CONTEXT_ACTION_ALLOW | 619 MLX5_FLOW_CONTEXT_ACTION_COUNT; 620 dest.type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 621 dest.counter_id = mlx5_fc_id(tx->fc->cnt); 622 fte = mlx5_add_flow_rules(tx->ft.status, spec, &flow_act, &dest, 1); 623 if (IS_ERR(fte)) { 624 err = PTR_ERR(fte); 625 mlx5_core_err(mdev, "Fail to add ipsec tx counter rule err=%d\n", err); 626 goto err_rule; 627 } 628 629 kvfree(spec); 630 tx->status.rule = fte; 631 return 0; 632 633 err_rule: 634 kvfree(spec); 635 return err; 636 } 637 638 /* IPsec TX flow steering */ 639 static void tx_destroy(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx, 640 struct mlx5_ipsec_fs *roce) 641 { 642 mlx5_ipsec_fs_roce_tx_destroy(roce, ipsec->mdev); 643 if (tx->chains) { 644 ipsec_chains_destroy(tx->chains); 645 } else { 646 mlx5_del_flow_rules(tx->pol.rule); 647 mlx5_destroy_flow_group(tx->pol.group); 648 mlx5_destroy_flow_table(tx->ft.pol); 649 } 650 651 if (tx == ipsec->tx_esw) { 652 mlx5_del_flow_rules(tx->sa.rule); 653 mlx5_destroy_flow_group(tx->sa.group); 654 } 655 mlx5_destroy_flow_table(tx->ft.sa); 656 if (tx->allow_tunnel_mode) 657 mlx5_eswitch_unblock_encap(ipsec->mdev); 658 mlx5_del_flow_rules(tx->status.rule); 659 mlx5_destroy_flow_table(tx->ft.status); 660 } 661 662 static void ipsec_tx_create_attr_set(struct mlx5e_ipsec *ipsec, 663 struct mlx5e_ipsec_tx *tx, 664 struct mlx5e_ipsec_tx_create_attr *attr) 665 { 666 if (tx == ipsec->tx_esw) { 667 mlx5_esw_ipsec_tx_create_attr_set(ipsec, attr); 668 return; 669 } 670 671 attr->prio = 0; 672 attr->pol_level = 0; 673 attr->sa_level = 1; 674 attr->cnt_level = 2; 675 attr->chains_ns = MLX5_FLOW_NAMESPACE_EGRESS_IPSEC; 676 } 677 678 static int tx_create(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx, 679 struct mlx5_ipsec_fs *roce) 680 { 681 struct mlx5_core_dev *mdev = ipsec->mdev; 682 struct mlx5e_ipsec_tx_create_attr attr; 683 struct mlx5_flow_destination dest = {}; 684 struct mlx5_flow_table *ft; 685 u32 flags = 0; 686 int err; 687 688 ipsec_tx_create_attr_set(ipsec, tx, &attr); 689 ft = ipsec_ft_create(tx->ns, attr.cnt_level, attr.prio, 1, 0); 690 if (IS_ERR(ft)) 691 return PTR_ERR(ft); 692 tx->ft.status = ft; 693 694 err = ipsec_counter_rule_tx(mdev, tx); 695 if (err) 696 goto err_status_rule; 697 698 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_TUNNEL) 699 tx->allow_tunnel_mode = mlx5_eswitch_block_encap(mdev); 700 if (tx->allow_tunnel_mode) 701 flags = MLX5_FLOW_TABLE_TUNNEL_EN_REFORMAT; 702 ft = ipsec_ft_create(tx->ns, attr.sa_level, attr.prio, 4, flags); 703 if (IS_ERR(ft)) { 704 err = PTR_ERR(ft); 705 goto err_sa_ft; 706 } 707 tx->ft.sa = ft; 708 709 if (tx == ipsec->tx_esw) { 710 dest.type = MLX5_FLOW_DESTINATION_TYPE_VPORT; 711 dest.vport.num = MLX5_VPORT_UPLINK; 712 err = ipsec_miss_create(mdev, tx->ft.sa, &tx->sa, &dest); 713 if (err) 714 goto err_sa_miss; 715 memset(&dest, 0, sizeof(dest)); 716 } 717 718 if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_PRIO) { 719 tx->chains = ipsec_chains_create( 720 mdev, tx->ft.sa, attr.chains_ns, attr.prio, attr.pol_level, 721 &tx->ft.pol); 722 if (IS_ERR(tx->chains)) { 723 err = PTR_ERR(tx->chains); 724 goto err_pol_ft; 725 } 726 727 goto connect_roce; 728 } 729 730 ft = ipsec_ft_create(tx->ns, attr.pol_level, attr.prio, 2, 0); 731 if (IS_ERR(ft)) { 732 err = PTR_ERR(ft); 733 goto err_pol_ft; 734 } 735 tx->ft.pol = ft; 736 dest.type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 737 dest.ft = tx->ft.sa; 738 err = ipsec_miss_create(mdev, tx->ft.pol, &tx->pol, &dest); 739 if (err) { 740 mlx5_destroy_flow_table(tx->ft.pol); 741 goto err_pol_ft; 742 } 743 744 connect_roce: 745 err = mlx5_ipsec_fs_roce_tx_create(mdev, roce, tx->ft.pol, false); 746 if (err) 747 goto err_roce; 748 return 0; 749 750 err_roce: 751 if (tx->chains) { 752 ipsec_chains_destroy(tx->chains); 753 } else { 754 mlx5_del_flow_rules(tx->pol.rule); 755 mlx5_destroy_flow_group(tx->pol.group); 756 mlx5_destroy_flow_table(tx->ft.pol); 757 } 758 err_pol_ft: 759 if (tx == ipsec->tx_esw) { 760 mlx5_del_flow_rules(tx->sa.rule); 761 mlx5_destroy_flow_group(tx->sa.group); 762 } 763 err_sa_miss: 764 mlx5_destroy_flow_table(tx->ft.sa); 765 err_sa_ft: 766 if (tx->allow_tunnel_mode) 767 mlx5_eswitch_unblock_encap(mdev); 768 mlx5_del_flow_rules(tx->status.rule); 769 err_status_rule: 770 mlx5_destroy_flow_table(tx->ft.status); 771 return err; 772 } 773 774 static void ipsec_esw_tx_ft_policy_set(struct mlx5_core_dev *mdev, 775 struct mlx5_flow_table *ft) 776 { 777 #ifdef CONFIG_MLX5_ESWITCH 778 struct mlx5_eswitch *esw = mdev->priv.eswitch; 779 struct mlx5e_rep_priv *uplink_rpriv; 780 struct mlx5e_priv *priv; 781 782 esw->offloads.ft_ipsec_tx_pol = ft; 783 uplink_rpriv = mlx5_eswitch_get_uplink_priv(esw, REP_ETH); 784 priv = netdev_priv(uplink_rpriv->netdev); 785 if (!priv->channels.num) 786 return; 787 788 mlx5e_rep_deactivate_channels(priv); 789 mlx5e_rep_activate_channels(priv); 790 #endif 791 } 792 793 static int tx_get(struct mlx5_core_dev *mdev, struct mlx5e_ipsec *ipsec, 794 struct mlx5e_ipsec_tx *tx) 795 { 796 int err; 797 798 if (tx->ft.refcnt) 799 goto skip; 800 801 err = mlx5_eswitch_block_mode(mdev); 802 if (err) 803 return err; 804 805 err = tx_create(ipsec, tx, ipsec->roce); 806 if (err) { 807 mlx5_eswitch_unblock_mode(mdev); 808 return err; 809 } 810 811 if (tx == ipsec->tx_esw) 812 ipsec_esw_tx_ft_policy_set(mdev, tx->ft.pol); 813 814 skip: 815 tx->ft.refcnt++; 816 return 0; 817 } 818 819 static void tx_put(struct mlx5e_ipsec *ipsec, struct mlx5e_ipsec_tx *tx) 820 { 821 if (--tx->ft.refcnt) 822 return; 823 824 if (tx == ipsec->tx_esw) { 825 mlx5_esw_ipsec_restore_dest_uplink(ipsec->mdev); 826 ipsec_esw_tx_ft_policy_set(ipsec->mdev, NULL); 827 } 828 829 tx_destroy(ipsec, tx, ipsec->roce); 830 mlx5_eswitch_unblock_mode(ipsec->mdev); 831 } 832 833 static struct mlx5_flow_table *tx_ft_get_policy(struct mlx5_core_dev *mdev, 834 struct mlx5e_ipsec *ipsec, 835 u32 prio, int type) 836 { 837 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 838 struct mlx5_flow_table *ft; 839 int err; 840 841 mutex_lock(&tx->ft.mutex); 842 err = tx_get(mdev, ipsec, tx); 843 if (err) 844 goto err_get; 845 846 ft = tx->chains ? ipsec_chains_get_table(tx->chains, prio) : tx->ft.pol; 847 if (IS_ERR(ft)) { 848 err = PTR_ERR(ft); 849 goto err_get_ft; 850 } 851 852 mutex_unlock(&tx->ft.mutex); 853 return ft; 854 855 err_get_ft: 856 tx_put(ipsec, tx); 857 err_get: 858 mutex_unlock(&tx->ft.mutex); 859 return ERR_PTR(err); 860 } 861 862 static struct mlx5e_ipsec_tx *tx_ft_get(struct mlx5_core_dev *mdev, 863 struct mlx5e_ipsec *ipsec, int type) 864 { 865 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 866 int err; 867 868 mutex_lock(&tx->ft.mutex); 869 err = tx_get(mdev, ipsec, tx); 870 mutex_unlock(&tx->ft.mutex); 871 if (err) 872 return ERR_PTR(err); 873 874 return tx; 875 } 876 877 static void tx_ft_put(struct mlx5e_ipsec *ipsec, int type) 878 { 879 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 880 881 mutex_lock(&tx->ft.mutex); 882 tx_put(ipsec, tx); 883 mutex_unlock(&tx->ft.mutex); 884 } 885 886 static void tx_ft_put_policy(struct mlx5e_ipsec *ipsec, u32 prio, int type) 887 { 888 struct mlx5e_ipsec_tx *tx = ipsec_tx(ipsec, type); 889 890 mutex_lock(&tx->ft.mutex); 891 if (tx->chains) 892 ipsec_chains_put_table(tx->chains, prio); 893 894 tx_put(ipsec, tx); 895 mutex_unlock(&tx->ft.mutex); 896 } 897 898 static void setup_fte_addr4(struct mlx5_flow_spec *spec, __be32 *saddr, 899 __be32 *daddr) 900 { 901 if (!*saddr && !*daddr) 902 return; 903 904 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 905 906 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 907 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 4); 908 909 if (*saddr) { 910 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 911 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4), saddr, 4); 912 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 913 outer_headers.src_ipv4_src_ipv6.ipv4_layout.ipv4); 914 } 915 916 if (*daddr) { 917 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 918 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4), daddr, 4); 919 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 920 outer_headers.dst_ipv4_dst_ipv6.ipv4_layout.ipv4); 921 } 922 } 923 924 static void setup_fte_addr6(struct mlx5_flow_spec *spec, __be32 *saddr, 925 __be32 *daddr) 926 { 927 if (addr6_all_zero(saddr) && addr6_all_zero(daddr)) 928 return; 929 930 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 931 932 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_version); 933 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_version, 6); 934 935 if (!addr6_all_zero(saddr)) { 936 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 937 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), saddr, 16); 938 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 939 outer_headers.src_ipv4_src_ipv6.ipv6_layout.ipv6), 0xff, 16); 940 } 941 942 if (!addr6_all_zero(daddr)) { 943 memcpy(MLX5_ADDR_OF(fte_match_param, spec->match_value, 944 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), daddr, 16); 945 memset(MLX5_ADDR_OF(fte_match_param, spec->match_criteria, 946 outer_headers.dst_ipv4_dst_ipv6.ipv6_layout.ipv6), 0xff, 16); 947 } 948 } 949 950 static void setup_fte_esp(struct mlx5_flow_spec *spec) 951 { 952 /* ESP header */ 953 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 954 955 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.ip_protocol); 956 MLX5_SET(fte_match_param, spec->match_value, outer_headers.ip_protocol, IPPROTO_ESP); 957 } 958 959 static void setup_fte_spi(struct mlx5_flow_spec *spec, u32 spi) 960 { 961 /* SPI number */ 962 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS; 963 964 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, misc_parameters.outer_esp_spi); 965 MLX5_SET(fte_match_param, spec->match_value, misc_parameters.outer_esp_spi, spi); 966 } 967 968 static void setup_fte_no_frags(struct mlx5_flow_spec *spec) 969 { 970 /* Non fragmented */ 971 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 972 973 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, outer_headers.frag); 974 MLX5_SET(fte_match_param, spec->match_value, outer_headers.frag, 0); 975 } 976 977 static void setup_fte_reg_a(struct mlx5_flow_spec *spec) 978 { 979 /* Add IPsec indicator in metadata_reg_a */ 980 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 981 982 MLX5_SET(fte_match_param, spec->match_criteria, 983 misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); 984 MLX5_SET(fte_match_param, spec->match_value, 985 misc_parameters_2.metadata_reg_a, MLX5_ETH_WQE_FT_META_IPSEC); 986 } 987 988 static void setup_fte_reg_c4(struct mlx5_flow_spec *spec, u32 reqid) 989 { 990 /* Pass policy check before choosing this SA */ 991 spec->match_criteria_enable |= MLX5_MATCH_MISC_PARAMETERS_2; 992 993 MLX5_SET_TO_ONES(fte_match_param, spec->match_criteria, 994 misc_parameters_2.metadata_reg_c_4); 995 MLX5_SET(fte_match_param, spec->match_value, 996 misc_parameters_2.metadata_reg_c_4, reqid); 997 } 998 999 static void setup_fte_upper_proto_match(struct mlx5_flow_spec *spec, struct upspec *upspec) 1000 { 1001 switch (upspec->proto) { 1002 case IPPROTO_UDP: 1003 if (upspec->dport) { 1004 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1005 udp_dport, upspec->dport_mask); 1006 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1007 udp_dport, upspec->dport); 1008 } 1009 if (upspec->sport) { 1010 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1011 udp_sport, upspec->sport_mask); 1012 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1013 udp_sport, upspec->sport); 1014 } 1015 break; 1016 case IPPROTO_TCP: 1017 if (upspec->dport) { 1018 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1019 tcp_dport, upspec->dport_mask); 1020 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1021 tcp_dport, upspec->dport); 1022 } 1023 if (upspec->sport) { 1024 MLX5_SET(fte_match_set_lyr_2_4, spec->match_criteria, 1025 tcp_sport, upspec->sport_mask); 1026 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, 1027 tcp_sport, upspec->sport); 1028 } 1029 break; 1030 default: 1031 return; 1032 } 1033 1034 spec->match_criteria_enable |= MLX5_MATCH_OUTER_HEADERS; 1035 MLX5_SET_TO_ONES(fte_match_set_lyr_2_4, spec->match_criteria, ip_protocol); 1036 MLX5_SET(fte_match_set_lyr_2_4, spec->match_value, ip_protocol, upspec->proto); 1037 } 1038 1039 static enum mlx5_flow_namespace_type ipsec_fs_get_ns(struct mlx5e_ipsec *ipsec, 1040 int type, u8 dir) 1041 { 1042 if (ipsec->is_uplink_rep && type == XFRM_DEV_OFFLOAD_PACKET) 1043 return MLX5_FLOW_NAMESPACE_FDB; 1044 1045 if (dir == XFRM_DEV_OFFLOAD_IN) 1046 return MLX5_FLOW_NAMESPACE_KERNEL; 1047 1048 return MLX5_FLOW_NAMESPACE_EGRESS; 1049 } 1050 1051 static int setup_modify_header(struct mlx5e_ipsec *ipsec, int type, u32 val, u8 dir, 1052 struct mlx5_flow_act *flow_act) 1053 { 1054 enum mlx5_flow_namespace_type ns_type = ipsec_fs_get_ns(ipsec, type, dir); 1055 u8 action[MLX5_UN_SZ_BYTES(set_add_copy_action_in_auto)] = {}; 1056 struct mlx5_core_dev *mdev = ipsec->mdev; 1057 struct mlx5_modify_hdr *modify_hdr; 1058 1059 MLX5_SET(set_action_in, action, action_type, MLX5_ACTION_TYPE_SET); 1060 switch (dir) { 1061 case XFRM_DEV_OFFLOAD_IN: 1062 MLX5_SET(set_action_in, action, field, 1063 MLX5_ACTION_IN_FIELD_METADATA_REG_B); 1064 break; 1065 case XFRM_DEV_OFFLOAD_OUT: 1066 MLX5_SET(set_action_in, action, field, 1067 MLX5_ACTION_IN_FIELD_METADATA_REG_C_4); 1068 break; 1069 default: 1070 return -EINVAL; 1071 } 1072 1073 MLX5_SET(set_action_in, action, data, val); 1074 MLX5_SET(set_action_in, action, offset, 0); 1075 MLX5_SET(set_action_in, action, length, 32); 1076 1077 modify_hdr = mlx5_modify_header_alloc(mdev, ns_type, 1, action); 1078 if (IS_ERR(modify_hdr)) { 1079 mlx5_core_err(mdev, "Failed to allocate modify_header %ld\n", 1080 PTR_ERR(modify_hdr)); 1081 return PTR_ERR(modify_hdr); 1082 } 1083 1084 flow_act->modify_hdr = modify_hdr; 1085 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_MOD_HDR; 1086 return 0; 1087 } 1088 1089 static int 1090 setup_pkt_tunnel_reformat(struct mlx5_core_dev *mdev, 1091 struct mlx5_accel_esp_xfrm_attrs *attrs, 1092 struct mlx5_pkt_reformat_params *reformat_params) 1093 { 1094 struct ip_esp_hdr *esp_hdr; 1095 struct ipv6hdr *ipv6hdr; 1096 struct ethhdr *eth_hdr; 1097 struct iphdr *iphdr; 1098 char *reformatbf; 1099 size_t bfflen; 1100 void *hdr; 1101 1102 bfflen = sizeof(*eth_hdr); 1103 1104 if (attrs->dir == XFRM_DEV_OFFLOAD_OUT) { 1105 bfflen += sizeof(*esp_hdr) + 8; 1106 1107 switch (attrs->family) { 1108 case AF_INET: 1109 bfflen += sizeof(*iphdr); 1110 break; 1111 case AF_INET6: 1112 bfflen += sizeof(*ipv6hdr); 1113 break; 1114 default: 1115 return -EINVAL; 1116 } 1117 } 1118 1119 reformatbf = kzalloc(bfflen, GFP_KERNEL); 1120 if (!reformatbf) 1121 return -ENOMEM; 1122 1123 eth_hdr = (struct ethhdr *)reformatbf; 1124 switch (attrs->family) { 1125 case AF_INET: 1126 eth_hdr->h_proto = htons(ETH_P_IP); 1127 break; 1128 case AF_INET6: 1129 eth_hdr->h_proto = htons(ETH_P_IPV6); 1130 break; 1131 default: 1132 goto free_reformatbf; 1133 } 1134 1135 ether_addr_copy(eth_hdr->h_dest, attrs->dmac); 1136 ether_addr_copy(eth_hdr->h_source, attrs->smac); 1137 1138 switch (attrs->dir) { 1139 case XFRM_DEV_OFFLOAD_IN: 1140 reformat_params->type = MLX5_REFORMAT_TYPE_L3_ESP_TUNNEL_TO_L2; 1141 break; 1142 case XFRM_DEV_OFFLOAD_OUT: 1143 reformat_params->type = MLX5_REFORMAT_TYPE_L2_TO_L3_ESP_TUNNEL; 1144 reformat_params->param_0 = attrs->authsize; 1145 1146 hdr = reformatbf + sizeof(*eth_hdr); 1147 switch (attrs->family) { 1148 case AF_INET: 1149 iphdr = (struct iphdr *)hdr; 1150 memcpy(&iphdr->saddr, &attrs->saddr.a4, 4); 1151 memcpy(&iphdr->daddr, &attrs->daddr.a4, 4); 1152 iphdr->version = 4; 1153 iphdr->ihl = 5; 1154 iphdr->ttl = IPSEC_TUNNEL_DEFAULT_TTL; 1155 iphdr->protocol = IPPROTO_ESP; 1156 hdr += sizeof(*iphdr); 1157 break; 1158 case AF_INET6: 1159 ipv6hdr = (struct ipv6hdr *)hdr; 1160 memcpy(&ipv6hdr->saddr, &attrs->saddr.a6, 16); 1161 memcpy(&ipv6hdr->daddr, &attrs->daddr.a6, 16); 1162 ipv6hdr->nexthdr = IPPROTO_ESP; 1163 ipv6hdr->version = 6; 1164 ipv6hdr->hop_limit = IPSEC_TUNNEL_DEFAULT_TTL; 1165 hdr += sizeof(*ipv6hdr); 1166 break; 1167 default: 1168 goto free_reformatbf; 1169 } 1170 1171 esp_hdr = (struct ip_esp_hdr *)hdr; 1172 esp_hdr->spi = htonl(attrs->spi); 1173 break; 1174 default: 1175 goto free_reformatbf; 1176 } 1177 1178 reformat_params->size = bfflen; 1179 reformat_params->data = reformatbf; 1180 return 0; 1181 1182 free_reformatbf: 1183 kfree(reformatbf); 1184 return -EINVAL; 1185 } 1186 1187 static int get_reformat_type(struct mlx5_accel_esp_xfrm_attrs *attrs) 1188 { 1189 switch (attrs->dir) { 1190 case XFRM_DEV_OFFLOAD_IN: 1191 if (attrs->encap) 1192 return MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT_OVER_UDP; 1193 return MLX5_REFORMAT_TYPE_DEL_ESP_TRANSPORT; 1194 case XFRM_DEV_OFFLOAD_OUT: 1195 if (attrs->family == AF_INET) { 1196 if (attrs->encap) 1197 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV4; 1198 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV4; 1199 } 1200 1201 if (attrs->encap) 1202 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_UDPV6; 1203 return MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_OVER_IPV6; 1204 default: 1205 WARN_ON(true); 1206 } 1207 1208 return -EINVAL; 1209 } 1210 1211 static int 1212 setup_pkt_transport_reformat(struct mlx5_accel_esp_xfrm_attrs *attrs, 1213 struct mlx5_pkt_reformat_params *reformat_params) 1214 { 1215 struct udphdr *udphdr; 1216 char *reformatbf; 1217 size_t bfflen; 1218 __be32 spi; 1219 void *hdr; 1220 1221 reformat_params->type = get_reformat_type(attrs); 1222 if (reformat_params->type < 0) 1223 return reformat_params->type; 1224 1225 switch (attrs->dir) { 1226 case XFRM_DEV_OFFLOAD_IN: 1227 break; 1228 case XFRM_DEV_OFFLOAD_OUT: 1229 bfflen = MLX5_REFORMAT_TYPE_ADD_ESP_TRANSPORT_SIZE; 1230 if (attrs->encap) 1231 bfflen += sizeof(*udphdr); 1232 1233 reformatbf = kzalloc(bfflen, GFP_KERNEL); 1234 if (!reformatbf) 1235 return -ENOMEM; 1236 1237 hdr = reformatbf; 1238 if (attrs->encap) { 1239 udphdr = (struct udphdr *)reformatbf; 1240 udphdr->source = attrs->sport; 1241 udphdr->dest = attrs->dport; 1242 hdr += sizeof(*udphdr); 1243 } 1244 1245 /* convert to network format */ 1246 spi = htonl(attrs->spi); 1247 memcpy(hdr, &spi, sizeof(spi)); 1248 1249 reformat_params->param_0 = attrs->authsize; 1250 reformat_params->size = bfflen; 1251 reformat_params->data = reformatbf; 1252 break; 1253 default: 1254 return -EINVAL; 1255 } 1256 1257 return 0; 1258 } 1259 1260 static int setup_pkt_reformat(struct mlx5e_ipsec *ipsec, 1261 struct mlx5_accel_esp_xfrm_attrs *attrs, 1262 struct mlx5_flow_act *flow_act) 1263 { 1264 enum mlx5_flow_namespace_type ns_type = ipsec_fs_get_ns(ipsec, attrs->type, 1265 attrs->dir); 1266 struct mlx5_pkt_reformat_params reformat_params = {}; 1267 struct mlx5_core_dev *mdev = ipsec->mdev; 1268 struct mlx5_pkt_reformat *pkt_reformat; 1269 int ret; 1270 1271 switch (attrs->mode) { 1272 case XFRM_MODE_TRANSPORT: 1273 ret = setup_pkt_transport_reformat(attrs, &reformat_params); 1274 break; 1275 case XFRM_MODE_TUNNEL: 1276 ret = setup_pkt_tunnel_reformat(mdev, attrs, &reformat_params); 1277 break; 1278 default: 1279 ret = -EINVAL; 1280 } 1281 1282 if (ret) 1283 return ret; 1284 1285 pkt_reformat = 1286 mlx5_packet_reformat_alloc(mdev, &reformat_params, ns_type); 1287 kfree(reformat_params.data); 1288 if (IS_ERR(pkt_reformat)) 1289 return PTR_ERR(pkt_reformat); 1290 1291 flow_act->pkt_reformat = pkt_reformat; 1292 flow_act->action |= MLX5_FLOW_CONTEXT_ACTION_PACKET_REFORMAT; 1293 return 0; 1294 } 1295 1296 static int rx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1297 { 1298 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1299 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1300 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 1301 struct mlx5_flow_destination dest[2]; 1302 struct mlx5_flow_act flow_act = {}; 1303 struct mlx5_flow_handle *rule; 1304 struct mlx5_flow_spec *spec; 1305 struct mlx5e_ipsec_rx *rx; 1306 struct mlx5_fc *counter; 1307 int err = 0; 1308 1309 rx = rx_ft_get(mdev, ipsec, attrs->family, attrs->type); 1310 if (IS_ERR(rx)) 1311 return PTR_ERR(rx); 1312 1313 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1314 if (!spec) { 1315 err = -ENOMEM; 1316 goto err_alloc; 1317 } 1318 1319 if (attrs->family == AF_INET) 1320 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1321 else 1322 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1323 1324 setup_fte_spi(spec, attrs->spi); 1325 setup_fte_esp(spec); 1326 setup_fte_no_frags(spec); 1327 setup_fte_upper_proto_match(spec, &attrs->upspec); 1328 1329 if (!attrs->drop) { 1330 if (rx != ipsec->rx_esw) 1331 err = setup_modify_header(ipsec, attrs->type, 1332 sa_entry->ipsec_obj_id | BIT(31), 1333 XFRM_DEV_OFFLOAD_IN, &flow_act); 1334 else 1335 err = mlx5_esw_ipsec_rx_setup_modify_header(sa_entry, &flow_act); 1336 1337 if (err) 1338 goto err_mod_header; 1339 } 1340 1341 switch (attrs->type) { 1342 case XFRM_DEV_OFFLOAD_PACKET: 1343 err = setup_pkt_reformat(ipsec, attrs, &flow_act); 1344 if (err) 1345 goto err_pkt_reformat; 1346 break; 1347 default: 1348 break; 1349 } 1350 1351 counter = mlx5_fc_create(mdev, true); 1352 if (IS_ERR(counter)) { 1353 err = PTR_ERR(counter); 1354 goto err_add_cnt; 1355 } 1356 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 1357 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 1358 flow_act.flags |= FLOW_ACT_NO_APPEND; 1359 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_DECRYPT | 1360 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1361 if (attrs->drop) 1362 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1363 else 1364 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1365 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1366 dest[0].ft = rx->ft.status; 1367 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1368 dest[1].counter_id = mlx5_fc_id(counter); 1369 rule = mlx5_add_flow_rules(rx->ft.sa, spec, &flow_act, dest, 2); 1370 if (IS_ERR(rule)) { 1371 err = PTR_ERR(rule); 1372 mlx5_core_err(mdev, "fail to add RX ipsec rule err=%d\n", err); 1373 goto err_add_flow; 1374 } 1375 kvfree(spec); 1376 1377 sa_entry->ipsec_rule.rule = rule; 1378 sa_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 1379 sa_entry->ipsec_rule.fc = counter; 1380 sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; 1381 return 0; 1382 1383 err_add_flow: 1384 mlx5_fc_destroy(mdev, counter); 1385 err_add_cnt: 1386 if (flow_act.pkt_reformat) 1387 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 1388 err_pkt_reformat: 1389 if (flow_act.modify_hdr) 1390 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 1391 err_mod_header: 1392 kvfree(spec); 1393 err_alloc: 1394 rx_ft_put(ipsec, attrs->family, attrs->type); 1395 return err; 1396 } 1397 1398 static int tx_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1399 { 1400 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 1401 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1402 struct mlx5e_ipsec *ipsec = sa_entry->ipsec; 1403 struct mlx5_flow_destination dest[2]; 1404 struct mlx5_flow_act flow_act = {}; 1405 struct mlx5_flow_handle *rule; 1406 struct mlx5_flow_spec *spec; 1407 struct mlx5e_ipsec_tx *tx; 1408 struct mlx5_fc *counter; 1409 int err; 1410 1411 tx = tx_ft_get(mdev, ipsec, attrs->type); 1412 if (IS_ERR(tx)) 1413 return PTR_ERR(tx); 1414 1415 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1416 if (!spec) { 1417 err = -ENOMEM; 1418 goto err_alloc; 1419 } 1420 1421 if (attrs->family == AF_INET) 1422 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1423 else 1424 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1425 1426 setup_fte_no_frags(spec); 1427 setup_fte_upper_proto_match(spec, &attrs->upspec); 1428 1429 switch (attrs->type) { 1430 case XFRM_DEV_OFFLOAD_CRYPTO: 1431 setup_fte_spi(spec, attrs->spi); 1432 setup_fte_esp(spec); 1433 setup_fte_reg_a(spec); 1434 break; 1435 case XFRM_DEV_OFFLOAD_PACKET: 1436 if (attrs->reqid) 1437 setup_fte_reg_c4(spec, attrs->reqid); 1438 err = setup_pkt_reformat(ipsec, attrs, &flow_act); 1439 if (err) 1440 goto err_pkt_reformat; 1441 break; 1442 default: 1443 break; 1444 } 1445 1446 counter = mlx5_fc_create(mdev, true); 1447 if (IS_ERR(counter)) { 1448 err = PTR_ERR(counter); 1449 goto err_add_cnt; 1450 } 1451 1452 flow_act.crypto.type = MLX5_FLOW_CONTEXT_ENCRYPT_DECRYPT_TYPE_IPSEC; 1453 flow_act.crypto.obj_id = sa_entry->ipsec_obj_id; 1454 flow_act.flags |= FLOW_ACT_NO_APPEND; 1455 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_CRYPTO_ENCRYPT | 1456 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1457 if (attrs->drop) 1458 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP; 1459 else 1460 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1461 1462 dest[0].ft = tx->ft.status; 1463 dest[0].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1464 dest[1].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1465 dest[1].counter_id = mlx5_fc_id(counter); 1466 rule = mlx5_add_flow_rules(tx->ft.sa, spec, &flow_act, dest, 2); 1467 if (IS_ERR(rule)) { 1468 err = PTR_ERR(rule); 1469 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 1470 goto err_add_flow; 1471 } 1472 1473 kvfree(spec); 1474 sa_entry->ipsec_rule.rule = rule; 1475 sa_entry->ipsec_rule.fc = counter; 1476 sa_entry->ipsec_rule.pkt_reformat = flow_act.pkt_reformat; 1477 return 0; 1478 1479 err_add_flow: 1480 mlx5_fc_destroy(mdev, counter); 1481 err_add_cnt: 1482 if (flow_act.pkt_reformat) 1483 mlx5_packet_reformat_dealloc(mdev, flow_act.pkt_reformat); 1484 err_pkt_reformat: 1485 kvfree(spec); 1486 err_alloc: 1487 tx_ft_put(ipsec, attrs->type); 1488 return err; 1489 } 1490 1491 static int tx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1492 { 1493 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1494 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1495 struct mlx5e_ipsec *ipsec = pol_entry->ipsec; 1496 struct mlx5_flow_destination dest[2] = {}; 1497 struct mlx5_flow_act flow_act = {}; 1498 struct mlx5_flow_handle *rule; 1499 struct mlx5_flow_spec *spec; 1500 struct mlx5_flow_table *ft; 1501 struct mlx5e_ipsec_tx *tx; 1502 int err, dstn = 0; 1503 1504 ft = tx_ft_get_policy(mdev, ipsec, attrs->prio, attrs->type); 1505 if (IS_ERR(ft)) 1506 return PTR_ERR(ft); 1507 1508 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1509 if (!spec) { 1510 err = -ENOMEM; 1511 goto err_alloc; 1512 } 1513 1514 tx = ipsec_tx(ipsec, attrs->type); 1515 if (attrs->family == AF_INET) 1516 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1517 else 1518 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1519 1520 setup_fte_no_frags(spec); 1521 setup_fte_upper_proto_match(spec, &attrs->upspec); 1522 1523 switch (attrs->action) { 1524 case XFRM_POLICY_ALLOW: 1525 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1526 if (!attrs->reqid) 1527 break; 1528 1529 err = setup_modify_header(ipsec, attrs->type, attrs->reqid, 1530 XFRM_DEV_OFFLOAD_OUT, &flow_act); 1531 if (err) 1532 goto err_mod_header; 1533 break; 1534 case XFRM_POLICY_BLOCK: 1535 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | 1536 MLX5_FLOW_CONTEXT_ACTION_COUNT; 1537 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1538 dest[dstn].counter_id = mlx5_fc_id(tx->fc->drop); 1539 dstn++; 1540 break; 1541 default: 1542 WARN_ON(true); 1543 err = -EINVAL; 1544 goto err_mod_header; 1545 } 1546 1547 flow_act.flags |= FLOW_ACT_NO_APPEND; 1548 if (tx == ipsec->tx_esw && tx->chains) 1549 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 1550 dest[dstn].ft = tx->ft.sa; 1551 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1552 dstn++; 1553 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1554 if (IS_ERR(rule)) { 1555 err = PTR_ERR(rule); 1556 mlx5_core_err(mdev, "fail to add TX ipsec rule err=%d\n", err); 1557 goto err_action; 1558 } 1559 1560 kvfree(spec); 1561 pol_entry->ipsec_rule.rule = rule; 1562 pol_entry->ipsec_rule.modify_hdr = flow_act.modify_hdr; 1563 return 0; 1564 1565 err_action: 1566 if (flow_act.modify_hdr) 1567 mlx5_modify_header_dealloc(mdev, flow_act.modify_hdr); 1568 err_mod_header: 1569 kvfree(spec); 1570 err_alloc: 1571 tx_ft_put_policy(ipsec, attrs->prio, attrs->type); 1572 return err; 1573 } 1574 1575 static int rx_add_policy(struct mlx5e_ipsec_pol_entry *pol_entry) 1576 { 1577 struct mlx5_accel_pol_xfrm_attrs *attrs = &pol_entry->attrs; 1578 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1579 struct mlx5e_ipsec *ipsec = pol_entry->ipsec; 1580 struct mlx5_flow_destination dest[2]; 1581 struct mlx5_flow_act flow_act = {}; 1582 struct mlx5_flow_handle *rule; 1583 struct mlx5_flow_spec *spec; 1584 struct mlx5_flow_table *ft; 1585 struct mlx5e_ipsec_rx *rx; 1586 int err, dstn = 0; 1587 1588 ft = rx_ft_get_policy(mdev, pol_entry->ipsec, attrs->family, attrs->prio, 1589 attrs->type); 1590 if (IS_ERR(ft)) 1591 return PTR_ERR(ft); 1592 1593 rx = ipsec_rx(pol_entry->ipsec, attrs->family, attrs->type); 1594 1595 spec = kvzalloc(sizeof(*spec), GFP_KERNEL); 1596 if (!spec) { 1597 err = -ENOMEM; 1598 goto err_alloc; 1599 } 1600 1601 if (attrs->family == AF_INET) 1602 setup_fte_addr4(spec, &attrs->saddr.a4, &attrs->daddr.a4); 1603 else 1604 setup_fte_addr6(spec, attrs->saddr.a6, attrs->daddr.a6); 1605 1606 setup_fte_no_frags(spec); 1607 setup_fte_upper_proto_match(spec, &attrs->upspec); 1608 1609 switch (attrs->action) { 1610 case XFRM_POLICY_ALLOW: 1611 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_FWD_DEST; 1612 break; 1613 case XFRM_POLICY_BLOCK: 1614 flow_act.action |= MLX5_FLOW_CONTEXT_ACTION_DROP | MLX5_FLOW_CONTEXT_ACTION_COUNT; 1615 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_COUNTER; 1616 dest[dstn].counter_id = mlx5_fc_id(rx->fc->drop); 1617 dstn++; 1618 break; 1619 default: 1620 WARN_ON(true); 1621 err = -EINVAL; 1622 goto err_action; 1623 } 1624 1625 flow_act.flags |= FLOW_ACT_NO_APPEND; 1626 if (rx == ipsec->rx_esw && rx->chains) 1627 flow_act.flags |= FLOW_ACT_IGNORE_FLOW_LEVEL; 1628 dest[dstn].type = MLX5_FLOW_DESTINATION_TYPE_FLOW_TABLE; 1629 dest[dstn].ft = rx->ft.sa; 1630 dstn++; 1631 rule = mlx5_add_flow_rules(ft, spec, &flow_act, dest, dstn); 1632 if (IS_ERR(rule)) { 1633 err = PTR_ERR(rule); 1634 mlx5_core_err(mdev, "Fail to add RX IPsec policy rule err=%d\n", err); 1635 goto err_action; 1636 } 1637 1638 kvfree(spec); 1639 pol_entry->ipsec_rule.rule = rule; 1640 return 0; 1641 1642 err_action: 1643 kvfree(spec); 1644 err_alloc: 1645 rx_ft_put_policy(pol_entry->ipsec, attrs->family, attrs->prio, attrs->type); 1646 return err; 1647 } 1648 1649 static void ipsec_fs_destroy_single_counter(struct mlx5_core_dev *mdev, 1650 struct mlx5e_ipsec_fc *fc) 1651 { 1652 mlx5_fc_destroy(mdev, fc->drop); 1653 mlx5_fc_destroy(mdev, fc->cnt); 1654 kfree(fc); 1655 } 1656 1657 static void ipsec_fs_destroy_counters(struct mlx5e_ipsec *ipsec) 1658 { 1659 struct mlx5_core_dev *mdev = ipsec->mdev; 1660 1661 ipsec_fs_destroy_single_counter(mdev, ipsec->tx->fc); 1662 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_ipv4->fc); 1663 if (ipsec->is_uplink_rep) { 1664 ipsec_fs_destroy_single_counter(mdev, ipsec->tx_esw->fc); 1665 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_esw->fc); 1666 } 1667 } 1668 1669 static struct mlx5e_ipsec_fc *ipsec_fs_init_single_counter(struct mlx5_core_dev *mdev) 1670 { 1671 struct mlx5e_ipsec_fc *fc; 1672 struct mlx5_fc *counter; 1673 int err; 1674 1675 fc = kzalloc(sizeof(*fc), GFP_KERNEL); 1676 if (!fc) 1677 return ERR_PTR(-ENOMEM); 1678 1679 counter = mlx5_fc_create(mdev, false); 1680 if (IS_ERR(counter)) { 1681 err = PTR_ERR(counter); 1682 goto err_cnt; 1683 } 1684 fc->cnt = counter; 1685 1686 counter = mlx5_fc_create(mdev, false); 1687 if (IS_ERR(counter)) { 1688 err = PTR_ERR(counter); 1689 goto err_drop; 1690 } 1691 fc->drop = counter; 1692 1693 return fc; 1694 1695 err_drop: 1696 mlx5_fc_destroy(mdev, fc->cnt); 1697 err_cnt: 1698 kfree(fc); 1699 return ERR_PTR(err); 1700 } 1701 1702 static int ipsec_fs_init_counters(struct mlx5e_ipsec *ipsec) 1703 { 1704 struct mlx5_core_dev *mdev = ipsec->mdev; 1705 struct mlx5e_ipsec_fc *fc; 1706 int err; 1707 1708 fc = ipsec_fs_init_single_counter(mdev); 1709 if (IS_ERR(fc)) { 1710 err = PTR_ERR(fc); 1711 goto err_rx_cnt; 1712 } 1713 ipsec->rx_ipv4->fc = fc; 1714 1715 fc = ipsec_fs_init_single_counter(mdev); 1716 if (IS_ERR(fc)) { 1717 err = PTR_ERR(fc); 1718 goto err_tx_cnt; 1719 } 1720 ipsec->tx->fc = fc; 1721 1722 if (ipsec->is_uplink_rep) { 1723 fc = ipsec_fs_init_single_counter(mdev); 1724 if (IS_ERR(fc)) { 1725 err = PTR_ERR(fc); 1726 goto err_rx_esw_cnt; 1727 } 1728 ipsec->rx_esw->fc = fc; 1729 1730 fc = ipsec_fs_init_single_counter(mdev); 1731 if (IS_ERR(fc)) { 1732 err = PTR_ERR(fc); 1733 goto err_tx_esw_cnt; 1734 } 1735 ipsec->tx_esw->fc = fc; 1736 } 1737 1738 /* Both IPv4 and IPv6 point to same flow counters struct. */ 1739 ipsec->rx_ipv6->fc = ipsec->rx_ipv4->fc; 1740 return 0; 1741 1742 err_tx_esw_cnt: 1743 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_esw->fc); 1744 err_rx_esw_cnt: 1745 ipsec_fs_destroy_single_counter(mdev, ipsec->tx->fc); 1746 err_tx_cnt: 1747 ipsec_fs_destroy_single_counter(mdev, ipsec->rx_ipv4->fc); 1748 err_rx_cnt: 1749 return err; 1750 } 1751 1752 void mlx5e_accel_ipsec_fs_read_stats(struct mlx5e_priv *priv, void *ipsec_stats) 1753 { 1754 struct mlx5_core_dev *mdev = priv->mdev; 1755 struct mlx5e_ipsec *ipsec = priv->ipsec; 1756 struct mlx5e_ipsec_hw_stats *stats; 1757 struct mlx5e_ipsec_fc *fc; 1758 u64 packets, bytes; 1759 1760 stats = (struct mlx5e_ipsec_hw_stats *)ipsec_stats; 1761 1762 stats->ipsec_rx_pkts = 0; 1763 stats->ipsec_rx_bytes = 0; 1764 stats->ipsec_rx_drop_pkts = 0; 1765 stats->ipsec_rx_drop_bytes = 0; 1766 stats->ipsec_tx_pkts = 0; 1767 stats->ipsec_tx_bytes = 0; 1768 stats->ipsec_tx_drop_pkts = 0; 1769 stats->ipsec_tx_drop_bytes = 0; 1770 1771 fc = ipsec->rx_ipv4->fc; 1772 mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_rx_pkts, &stats->ipsec_rx_bytes); 1773 mlx5_fc_query(mdev, fc->drop, &stats->ipsec_rx_drop_pkts, 1774 &stats->ipsec_rx_drop_bytes); 1775 1776 fc = ipsec->tx->fc; 1777 mlx5_fc_query(mdev, fc->cnt, &stats->ipsec_tx_pkts, &stats->ipsec_tx_bytes); 1778 mlx5_fc_query(mdev, fc->drop, &stats->ipsec_tx_drop_pkts, 1779 &stats->ipsec_tx_drop_bytes); 1780 1781 if (ipsec->is_uplink_rep) { 1782 fc = ipsec->rx_esw->fc; 1783 if (!mlx5_fc_query(mdev, fc->cnt, &packets, &bytes)) { 1784 stats->ipsec_rx_pkts += packets; 1785 stats->ipsec_rx_bytes += bytes; 1786 } 1787 1788 if (!mlx5_fc_query(mdev, fc->drop, &packets, &bytes)) { 1789 stats->ipsec_rx_drop_pkts += packets; 1790 stats->ipsec_rx_drop_bytes += bytes; 1791 } 1792 1793 fc = ipsec->tx_esw->fc; 1794 if (!mlx5_fc_query(mdev, fc->cnt, &packets, &bytes)) { 1795 stats->ipsec_tx_pkts += packets; 1796 stats->ipsec_tx_bytes += bytes; 1797 } 1798 1799 if (!mlx5_fc_query(mdev, fc->drop, &packets, &bytes)) { 1800 stats->ipsec_tx_drop_pkts += packets; 1801 stats->ipsec_tx_drop_bytes += bytes; 1802 } 1803 } 1804 } 1805 1806 #ifdef CONFIG_MLX5_ESWITCH 1807 static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) 1808 { 1809 struct mlx5_eswitch *esw = mdev->priv.eswitch; 1810 int err = 0; 1811 1812 if (esw) 1813 down_write(&esw->mode_lock); 1814 1815 if (mdev->num_block_ipsec) { 1816 err = -EBUSY; 1817 goto unlock; 1818 } 1819 1820 mdev->num_block_tc++; 1821 1822 unlock: 1823 if (esw) 1824 up_write(&esw->mode_lock); 1825 1826 return err; 1827 } 1828 #else 1829 static int mlx5e_ipsec_block_tc_offload(struct mlx5_core_dev *mdev) 1830 { 1831 if (mdev->num_block_ipsec) 1832 return -EBUSY; 1833 1834 mdev->num_block_tc++; 1835 return 0; 1836 } 1837 #endif 1838 1839 static void mlx5e_ipsec_unblock_tc_offload(struct mlx5_core_dev *mdev) 1840 { 1841 mdev->num_block_tc++; 1842 } 1843 1844 int mlx5e_accel_ipsec_fs_add_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1845 { 1846 int err; 1847 1848 if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) { 1849 err = mlx5e_ipsec_block_tc_offload(sa_entry->ipsec->mdev); 1850 if (err) 1851 return err; 1852 } 1853 1854 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 1855 err = tx_add_rule(sa_entry); 1856 else 1857 err = rx_add_rule(sa_entry); 1858 1859 if (err) 1860 goto err_out; 1861 1862 return 0; 1863 1864 err_out: 1865 if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) 1866 mlx5e_ipsec_unblock_tc_offload(sa_entry->ipsec->mdev); 1867 return err; 1868 } 1869 1870 void mlx5e_accel_ipsec_fs_del_rule(struct mlx5e_ipsec_sa_entry *sa_entry) 1871 { 1872 struct mlx5e_ipsec_rule *ipsec_rule = &sa_entry->ipsec_rule; 1873 struct mlx5_core_dev *mdev = mlx5e_ipsec_sa2dev(sa_entry); 1874 1875 mlx5_del_flow_rules(ipsec_rule->rule); 1876 mlx5_fc_destroy(mdev, ipsec_rule->fc); 1877 if (ipsec_rule->pkt_reformat) 1878 mlx5_packet_reformat_dealloc(mdev, ipsec_rule->pkt_reformat); 1879 1880 if (sa_entry->attrs.type == XFRM_DEV_OFFLOAD_PACKET) 1881 mlx5e_ipsec_unblock_tc_offload(mdev); 1882 1883 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) { 1884 tx_ft_put(sa_entry->ipsec, sa_entry->attrs.type); 1885 return; 1886 } 1887 1888 if (ipsec_rule->modify_hdr) 1889 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1890 mlx5_esw_ipsec_rx_id_mapping_remove(sa_entry); 1891 rx_ft_put(sa_entry->ipsec, sa_entry->attrs.family, sa_entry->attrs.type); 1892 } 1893 1894 int mlx5e_accel_ipsec_fs_add_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1895 { 1896 int err; 1897 1898 err = mlx5e_ipsec_block_tc_offload(pol_entry->ipsec->mdev); 1899 if (err) 1900 return err; 1901 1902 if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 1903 err = tx_add_policy(pol_entry); 1904 else 1905 err = rx_add_policy(pol_entry); 1906 1907 if (err) 1908 goto err_out; 1909 1910 return 0; 1911 1912 err_out: 1913 mlx5e_ipsec_unblock_tc_offload(pol_entry->ipsec->mdev); 1914 return err; 1915 } 1916 1917 void mlx5e_accel_ipsec_fs_del_pol(struct mlx5e_ipsec_pol_entry *pol_entry) 1918 { 1919 struct mlx5e_ipsec_rule *ipsec_rule = &pol_entry->ipsec_rule; 1920 struct mlx5_core_dev *mdev = mlx5e_ipsec_pol2dev(pol_entry); 1921 1922 mlx5_del_flow_rules(ipsec_rule->rule); 1923 1924 mlx5e_ipsec_unblock_tc_offload(pol_entry->ipsec->mdev); 1925 1926 if (pol_entry->attrs.dir == XFRM_DEV_OFFLOAD_IN) { 1927 rx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.family, 1928 pol_entry->attrs.prio, pol_entry->attrs.type); 1929 return; 1930 } 1931 1932 if (ipsec_rule->modify_hdr) 1933 mlx5_modify_header_dealloc(mdev, ipsec_rule->modify_hdr); 1934 1935 tx_ft_put_policy(pol_entry->ipsec, pol_entry->attrs.prio, pol_entry->attrs.type); 1936 } 1937 1938 void mlx5e_accel_ipsec_fs_cleanup(struct mlx5e_ipsec *ipsec) 1939 { 1940 if (!ipsec->tx) 1941 return; 1942 1943 if (ipsec->roce) 1944 mlx5_ipsec_fs_roce_cleanup(ipsec->roce); 1945 1946 ipsec_fs_destroy_counters(ipsec); 1947 mutex_destroy(&ipsec->tx->ft.mutex); 1948 WARN_ON(ipsec->tx->ft.refcnt); 1949 kfree(ipsec->tx); 1950 1951 mutex_destroy(&ipsec->rx_ipv4->ft.mutex); 1952 WARN_ON(ipsec->rx_ipv4->ft.refcnt); 1953 kfree(ipsec->rx_ipv4); 1954 1955 mutex_destroy(&ipsec->rx_ipv6->ft.mutex); 1956 WARN_ON(ipsec->rx_ipv6->ft.refcnt); 1957 kfree(ipsec->rx_ipv6); 1958 1959 if (ipsec->is_uplink_rep) { 1960 xa_destroy(&ipsec->rx_esw->ipsec_obj_id_map); 1961 1962 mutex_destroy(&ipsec->tx_esw->ft.mutex); 1963 WARN_ON(ipsec->tx_esw->ft.refcnt); 1964 kfree(ipsec->tx_esw); 1965 1966 mutex_destroy(&ipsec->rx_esw->ft.mutex); 1967 WARN_ON(ipsec->rx_esw->ft.refcnt); 1968 kfree(ipsec->rx_esw); 1969 } 1970 } 1971 1972 int mlx5e_accel_ipsec_fs_init(struct mlx5e_ipsec *ipsec, 1973 struct mlx5_devcom_comp_dev **devcom) 1974 { 1975 struct mlx5_core_dev *mdev = ipsec->mdev; 1976 struct mlx5_flow_namespace *ns, *ns_esw; 1977 int err = -ENOMEM; 1978 1979 ns = mlx5_get_flow_namespace(ipsec->mdev, 1980 MLX5_FLOW_NAMESPACE_EGRESS_IPSEC); 1981 if (!ns) 1982 return -EOPNOTSUPP; 1983 1984 if (ipsec->is_uplink_rep) { 1985 ns_esw = mlx5_get_flow_namespace(mdev, MLX5_FLOW_NAMESPACE_FDB); 1986 if (!ns_esw) 1987 return -EOPNOTSUPP; 1988 1989 ipsec->tx_esw = kzalloc(sizeof(*ipsec->tx_esw), GFP_KERNEL); 1990 if (!ipsec->tx_esw) 1991 return -ENOMEM; 1992 1993 ipsec->rx_esw = kzalloc(sizeof(*ipsec->rx_esw), GFP_KERNEL); 1994 if (!ipsec->rx_esw) 1995 goto err_rx_esw; 1996 } 1997 1998 ipsec->tx = kzalloc(sizeof(*ipsec->tx), GFP_KERNEL); 1999 if (!ipsec->tx) 2000 goto err_tx; 2001 2002 ipsec->rx_ipv4 = kzalloc(sizeof(*ipsec->rx_ipv4), GFP_KERNEL); 2003 if (!ipsec->rx_ipv4) 2004 goto err_rx_ipv4; 2005 2006 ipsec->rx_ipv6 = kzalloc(sizeof(*ipsec->rx_ipv6), GFP_KERNEL); 2007 if (!ipsec->rx_ipv6) 2008 goto err_rx_ipv6; 2009 2010 err = ipsec_fs_init_counters(ipsec); 2011 if (err) 2012 goto err_counters; 2013 2014 mutex_init(&ipsec->tx->ft.mutex); 2015 mutex_init(&ipsec->rx_ipv4->ft.mutex); 2016 mutex_init(&ipsec->rx_ipv6->ft.mutex); 2017 ipsec->tx->ns = ns; 2018 2019 if (ipsec->is_uplink_rep) { 2020 mutex_init(&ipsec->tx_esw->ft.mutex); 2021 mutex_init(&ipsec->rx_esw->ft.mutex); 2022 ipsec->tx_esw->ns = ns_esw; 2023 xa_init_flags(&ipsec->rx_esw->ipsec_obj_id_map, XA_FLAGS_ALLOC1); 2024 } else if (mlx5_ipsec_device_caps(mdev) & MLX5_IPSEC_CAP_ROCE) { 2025 ipsec->roce = mlx5_ipsec_fs_roce_init(mdev, devcom); 2026 } else { 2027 mlx5_core_warn(mdev, "IPsec was initialized without RoCE support\n"); 2028 } 2029 2030 return 0; 2031 2032 err_counters: 2033 kfree(ipsec->rx_ipv6); 2034 err_rx_ipv6: 2035 kfree(ipsec->rx_ipv4); 2036 err_rx_ipv4: 2037 kfree(ipsec->tx); 2038 err_tx: 2039 kfree(ipsec->rx_esw); 2040 err_rx_esw: 2041 kfree(ipsec->tx_esw); 2042 return err; 2043 } 2044 2045 void mlx5e_accel_ipsec_fs_modify(struct mlx5e_ipsec_sa_entry *sa_entry) 2046 { 2047 struct mlx5e_ipsec_sa_entry sa_entry_shadow = {}; 2048 int err; 2049 2050 memcpy(&sa_entry_shadow, sa_entry, sizeof(*sa_entry)); 2051 memset(&sa_entry_shadow.ipsec_rule, 0x00, sizeof(sa_entry->ipsec_rule)); 2052 2053 err = mlx5e_accel_ipsec_fs_add_rule(&sa_entry_shadow); 2054 if (err) 2055 return; 2056 2057 mlx5e_accel_ipsec_fs_del_rule(sa_entry); 2058 memcpy(sa_entry, &sa_entry_shadow, sizeof(*sa_entry)); 2059 } 2060 2061 bool mlx5e_ipsec_fs_tunnel_enabled(struct mlx5e_ipsec_sa_entry *sa_entry) 2062 { 2063 struct mlx5_accel_esp_xfrm_attrs *attrs = &sa_entry->attrs; 2064 struct mlx5e_ipsec_rx *rx; 2065 struct mlx5e_ipsec_tx *tx; 2066 2067 rx = ipsec_rx(sa_entry->ipsec, attrs->family, attrs->type); 2068 tx = ipsec_tx(sa_entry->ipsec, attrs->type); 2069 if (sa_entry->attrs.dir == XFRM_DEV_OFFLOAD_OUT) 2070 return tx->allow_tunnel_mode; 2071 2072 return rx->allow_tunnel_mode; 2073 } 2074 2075 void mlx5e_ipsec_handle_mpv_event(int event, struct mlx5e_priv *slave_priv, 2076 struct mlx5e_priv *master_priv) 2077 { 2078 struct mlx5e_ipsec_mpv_work *work; 2079 2080 reinit_completion(&master_priv->ipsec->comp); 2081 2082 if (!slave_priv->ipsec) { 2083 complete(&master_priv->ipsec->comp); 2084 return; 2085 } 2086 2087 work = &slave_priv->ipsec->mpv_work; 2088 2089 INIT_WORK(&work->work, ipsec_mpv_work_handler); 2090 work->event = event; 2091 work->slave_priv = slave_priv; 2092 work->master_priv = master_priv; 2093 queue_work(slave_priv->ipsec->wq, &work->work); 2094 } 2095 2096 void mlx5e_ipsec_send_event(struct mlx5e_priv *priv, int event) 2097 { 2098 if (!priv->ipsec) 2099 return; /* IPsec not supported */ 2100 2101 mlx5_devcom_send_event(priv->devcom, event, event, priv); 2102 wait_for_completion(&priv->ipsec->comp); 2103 } 2104