1 // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB 2 // Copyright (c) 2019 Mellanox Technologies. 3 4 #include "en_accel/ktls.h" 5 #include "en_accel/ktls_txrx.h" 6 #include "en_accel/ktls_utils.h" 7 8 struct mlx5e_dump_wqe { 9 struct mlx5_wqe_ctrl_seg ctrl; 10 struct mlx5_wqe_data_seg data; 11 }; 12 13 #define MLX5E_KTLS_DUMP_WQEBBS \ 14 (DIV_ROUND_UP(sizeof(struct mlx5e_dump_wqe), MLX5_SEND_WQE_BB)) 15 16 static u8 17 mlx5e_ktls_dumps_num_wqes(struct mlx5e_params *params, unsigned int nfrags, 18 unsigned int sync_len) 19 { 20 /* Given the MTU and sync_len, calculates an upper bound for the 21 * number of DUMP WQEs needed for the TX resync of a record. 22 */ 23 return nfrags + DIV_ROUND_UP(sync_len, MLX5E_SW2HW_MTU(params, params->sw_mtu)); 24 } 25 26 u16 mlx5e_ktls_get_stop_room(struct mlx5_core_dev *mdev, struct mlx5e_params *params) 27 { 28 u16 num_dumps, stop_room = 0; 29 30 if (!mlx5e_is_ktls_tx(mdev)) 31 return 0; 32 33 num_dumps = mlx5e_ktls_dumps_num_wqes(params, MAX_SKB_FRAGS, TLS_MAX_PAYLOAD_SIZE); 34 35 stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS); 36 stop_room += mlx5e_stop_room_for_wqe(mdev, MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS); 37 stop_room += num_dumps * mlx5e_stop_room_for_wqe(mdev, MLX5E_KTLS_DUMP_WQEBBS); 38 stop_room += 1; /* fence nop */ 39 40 return stop_room; 41 } 42 43 static void mlx5e_ktls_set_tisc(struct mlx5_core_dev *mdev, void *tisc) 44 { 45 MLX5_SET(tisc, tisc, tls_en, 1); 46 MLX5_SET(tisc, tisc, pd, mdev->mlx5e_res.hw_objs.pdn); 47 MLX5_SET(tisc, tisc, transport_domain, mdev->mlx5e_res.hw_objs.td.tdn); 48 } 49 50 static int mlx5e_ktls_create_tis(struct mlx5_core_dev *mdev, u32 *tisn) 51 { 52 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 53 54 mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx)); 55 56 return mlx5_core_create_tis(mdev, in, tisn); 57 } 58 59 static int mlx5e_ktls_create_tis_cb(struct mlx5_core_dev *mdev, 60 struct mlx5_async_ctx *async_ctx, 61 u32 *out, int outlen, 62 mlx5_async_cbk_t callback, 63 struct mlx5_async_work *context) 64 { 65 u32 in[MLX5_ST_SZ_DW(create_tis_in)] = {}; 66 67 mlx5e_ktls_set_tisc(mdev, MLX5_ADDR_OF(create_tis_in, in, ctx)); 68 MLX5_SET(create_tis_in, in, opcode, MLX5_CMD_OP_CREATE_TIS); 69 70 return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in), 71 out, outlen, callback, context); 72 } 73 74 static int mlx5e_ktls_destroy_tis_cb(struct mlx5_core_dev *mdev, u32 tisn, 75 struct mlx5_async_ctx *async_ctx, 76 u32 *out, int outlen, 77 mlx5_async_cbk_t callback, 78 struct mlx5_async_work *context) 79 { 80 u32 in[MLX5_ST_SZ_DW(destroy_tis_in)] = {}; 81 82 MLX5_SET(destroy_tis_in, in, opcode, MLX5_CMD_OP_DESTROY_TIS); 83 MLX5_SET(destroy_tis_in, in, tisn, tisn); 84 85 return mlx5_cmd_exec_cb(async_ctx, in, sizeof(in), 86 out, outlen, callback, context); 87 } 88 89 struct mlx5e_ktls_offload_context_tx { 90 /* fast path */ 91 u32 expected_seq; 92 u32 tisn; 93 bool ctx_post_pending; 94 /* control / resync */ 95 struct list_head list_node; /* member of the pool */ 96 union mlx5e_crypto_info crypto_info; 97 struct tls_offload_context_tx *tx_ctx; 98 struct mlx5_core_dev *mdev; 99 struct mlx5e_tls_sw_stats *sw_stats; 100 u32 key_id; 101 u8 create_err : 1; 102 }; 103 104 static void 105 mlx5e_set_ktls_tx_priv_ctx(struct tls_context *tls_ctx, 106 struct mlx5e_ktls_offload_context_tx *priv_tx) 107 { 108 struct mlx5e_ktls_offload_context_tx **ctx = 109 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); 110 111 BUILD_BUG_ON(sizeof(priv_tx) > TLS_DRIVER_STATE_SIZE_TX); 112 113 *ctx = priv_tx; 114 } 115 116 static struct mlx5e_ktls_offload_context_tx * 117 mlx5e_get_ktls_tx_priv_ctx(struct tls_context *tls_ctx) 118 { 119 struct mlx5e_ktls_offload_context_tx **ctx = 120 __tls_driver_ctx(tls_ctx, TLS_OFFLOAD_CTX_DIR_TX); 121 122 return *ctx; 123 } 124 125 /* struct for callback API management */ 126 struct mlx5e_async_ctx { 127 struct mlx5_async_work context; 128 struct mlx5_async_ctx async_ctx; 129 struct work_struct work; 130 struct mlx5e_ktls_offload_context_tx *priv_tx; 131 struct completion complete; 132 int err; 133 union { 134 u32 out_create[MLX5_ST_SZ_DW(create_tis_out)]; 135 u32 out_destroy[MLX5_ST_SZ_DW(destroy_tis_out)]; 136 }; 137 }; 138 139 static struct mlx5e_async_ctx *mlx5e_bulk_async_init(struct mlx5_core_dev *mdev, int n) 140 { 141 struct mlx5e_async_ctx *bulk_async; 142 int i; 143 144 bulk_async = kvcalloc(n, sizeof(struct mlx5e_async_ctx), GFP_KERNEL); 145 if (!bulk_async) 146 return NULL; 147 148 for (i = 0; i < n; i++) { 149 struct mlx5e_async_ctx *async = &bulk_async[i]; 150 151 mlx5_cmd_init_async_ctx(mdev, &async->async_ctx); 152 init_completion(&async->complete); 153 } 154 155 return bulk_async; 156 } 157 158 static void mlx5e_bulk_async_cleanup(struct mlx5e_async_ctx *bulk_async, int n) 159 { 160 int i; 161 162 for (i = 0; i < n; i++) { 163 struct mlx5e_async_ctx *async = &bulk_async[i]; 164 165 mlx5_cmd_cleanup_async_ctx(&async->async_ctx); 166 } 167 kvfree(bulk_async); 168 } 169 170 static void create_tis_callback(int status, struct mlx5_async_work *context) 171 { 172 struct mlx5e_async_ctx *async = 173 container_of(context, struct mlx5e_async_ctx, context); 174 struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx; 175 176 if (status) { 177 async->err = status; 178 priv_tx->create_err = 1; 179 goto out; 180 } 181 182 priv_tx->tisn = MLX5_GET(create_tis_out, async->out_create, tisn); 183 out: 184 complete(&async->complete); 185 } 186 187 static void destroy_tis_callback(int status, struct mlx5_async_work *context) 188 { 189 struct mlx5e_async_ctx *async = 190 container_of(context, struct mlx5e_async_ctx, context); 191 struct mlx5e_ktls_offload_context_tx *priv_tx = async->priv_tx; 192 193 complete(&async->complete); 194 kfree(priv_tx); 195 } 196 197 static struct mlx5e_ktls_offload_context_tx * 198 mlx5e_tls_priv_tx_init(struct mlx5_core_dev *mdev, struct mlx5e_tls_sw_stats *sw_stats, 199 struct mlx5e_async_ctx *async) 200 { 201 struct mlx5e_ktls_offload_context_tx *priv_tx; 202 int err; 203 204 priv_tx = kzalloc(sizeof(*priv_tx), GFP_KERNEL); 205 if (!priv_tx) 206 return ERR_PTR(-ENOMEM); 207 208 priv_tx->mdev = mdev; 209 priv_tx->sw_stats = sw_stats; 210 211 if (!async) { 212 err = mlx5e_ktls_create_tis(mdev, &priv_tx->tisn); 213 if (err) 214 goto err_out; 215 } else { 216 async->priv_tx = priv_tx; 217 err = mlx5e_ktls_create_tis_cb(mdev, &async->async_ctx, 218 async->out_create, sizeof(async->out_create), 219 create_tis_callback, &async->context); 220 if (err) 221 goto err_out; 222 } 223 224 return priv_tx; 225 226 err_out: 227 kfree(priv_tx); 228 return ERR_PTR(err); 229 } 230 231 static void mlx5e_tls_priv_tx_cleanup(struct mlx5e_ktls_offload_context_tx *priv_tx, 232 struct mlx5e_async_ctx *async) 233 { 234 if (priv_tx->create_err) { 235 complete(&async->complete); 236 kfree(priv_tx); 237 return; 238 } 239 async->priv_tx = priv_tx; 240 mlx5e_ktls_destroy_tis_cb(priv_tx->mdev, priv_tx->tisn, 241 &async->async_ctx, 242 async->out_destroy, sizeof(async->out_destroy), 243 destroy_tis_callback, &async->context); 244 } 245 246 static void mlx5e_tls_priv_tx_list_cleanup(struct mlx5_core_dev *mdev, 247 struct list_head *list, int size) 248 { 249 struct mlx5e_ktls_offload_context_tx *obj, *n; 250 struct mlx5e_async_ctx *bulk_async; 251 int i; 252 253 bulk_async = mlx5e_bulk_async_init(mdev, size); 254 if (!bulk_async) 255 return; 256 257 i = 0; 258 list_for_each_entry_safe(obj, n, list, list_node) { 259 mlx5e_tls_priv_tx_cleanup(obj, &bulk_async[i]); 260 i++; 261 } 262 263 for (i = 0; i < size; i++) { 264 struct mlx5e_async_ctx *async = &bulk_async[i]; 265 266 wait_for_completion(&async->complete); 267 } 268 mlx5e_bulk_async_cleanup(bulk_async, size); 269 } 270 271 /* Recycling pool API */ 272 273 #define MLX5E_TLS_TX_POOL_BULK (16) 274 #define MLX5E_TLS_TX_POOL_HIGH (4 * 1024) 275 #define MLX5E_TLS_TX_POOL_LOW (MLX5E_TLS_TX_POOL_HIGH / 4) 276 277 struct mlx5e_tls_tx_pool { 278 struct mlx5_core_dev *mdev; 279 struct mlx5e_tls_sw_stats *sw_stats; 280 struct mutex lock; /* Protects access to the pool */ 281 struct list_head list; 282 size_t size; 283 284 struct workqueue_struct *wq; 285 struct work_struct create_work; 286 struct work_struct destroy_work; 287 }; 288 289 static void create_work(struct work_struct *work) 290 { 291 struct mlx5e_tls_tx_pool *pool = 292 container_of(work, struct mlx5e_tls_tx_pool, create_work); 293 struct mlx5e_ktls_offload_context_tx *obj; 294 struct mlx5e_async_ctx *bulk_async; 295 LIST_HEAD(local_list); 296 int i, j, err = 0; 297 298 bulk_async = mlx5e_bulk_async_init(pool->mdev, MLX5E_TLS_TX_POOL_BULK); 299 if (!bulk_async) 300 return; 301 302 for (i = 0; i < MLX5E_TLS_TX_POOL_BULK; i++) { 303 obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, &bulk_async[i]); 304 if (IS_ERR(obj)) { 305 err = PTR_ERR(obj); 306 break; 307 } 308 list_add(&obj->list_node, &local_list); 309 } 310 311 for (j = 0; j < i; j++) { 312 struct mlx5e_async_ctx *async = &bulk_async[j]; 313 314 wait_for_completion(&async->complete); 315 if (!err && async->err) 316 err = async->err; 317 } 318 atomic64_add(i, &pool->sw_stats->tx_tls_pool_alloc); 319 mlx5e_bulk_async_cleanup(bulk_async, MLX5E_TLS_TX_POOL_BULK); 320 if (err) 321 goto err_out; 322 323 mutex_lock(&pool->lock); 324 if (pool->size + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH) { 325 mutex_unlock(&pool->lock); 326 goto err_out; 327 } 328 list_splice(&local_list, &pool->list); 329 pool->size += MLX5E_TLS_TX_POOL_BULK; 330 if (pool->size <= MLX5E_TLS_TX_POOL_LOW) 331 queue_work(pool->wq, work); 332 mutex_unlock(&pool->lock); 333 return; 334 335 err_out: 336 mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, i); 337 atomic64_add(i, &pool->sw_stats->tx_tls_pool_free); 338 } 339 340 static void destroy_work(struct work_struct *work) 341 { 342 struct mlx5e_tls_tx_pool *pool = 343 container_of(work, struct mlx5e_tls_tx_pool, destroy_work); 344 struct mlx5e_ktls_offload_context_tx *obj; 345 LIST_HEAD(local_list); 346 int i = 0; 347 348 mutex_lock(&pool->lock); 349 if (pool->size < MLX5E_TLS_TX_POOL_HIGH) { 350 mutex_unlock(&pool->lock); 351 return; 352 } 353 354 list_for_each_entry(obj, &pool->list, list_node) 355 if (++i == MLX5E_TLS_TX_POOL_BULK) 356 break; 357 358 list_cut_position(&local_list, &pool->list, &obj->list_node); 359 pool->size -= MLX5E_TLS_TX_POOL_BULK; 360 if (pool->size >= MLX5E_TLS_TX_POOL_HIGH) 361 queue_work(pool->wq, work); 362 mutex_unlock(&pool->lock); 363 364 mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK); 365 atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free); 366 } 367 368 static struct mlx5e_tls_tx_pool *mlx5e_tls_tx_pool_init(struct mlx5_core_dev *mdev, 369 struct mlx5e_tls_sw_stats *sw_stats) 370 { 371 struct mlx5e_tls_tx_pool *pool; 372 373 BUILD_BUG_ON(MLX5E_TLS_TX_POOL_LOW + MLX5E_TLS_TX_POOL_BULK >= MLX5E_TLS_TX_POOL_HIGH); 374 375 pool = kvzalloc(sizeof(*pool), GFP_KERNEL); 376 if (!pool) 377 return NULL; 378 379 pool->wq = create_singlethread_workqueue("mlx5e_tls_tx_pool"); 380 if (!pool->wq) 381 goto err_free; 382 383 INIT_LIST_HEAD(&pool->list); 384 mutex_init(&pool->lock); 385 386 INIT_WORK(&pool->create_work, create_work); 387 INIT_WORK(&pool->destroy_work, destroy_work); 388 389 pool->mdev = mdev; 390 pool->sw_stats = sw_stats; 391 392 return pool; 393 394 err_free: 395 kvfree(pool); 396 return NULL; 397 } 398 399 static void mlx5e_tls_tx_pool_list_cleanup(struct mlx5e_tls_tx_pool *pool) 400 { 401 while (pool->size > MLX5E_TLS_TX_POOL_BULK) { 402 struct mlx5e_ktls_offload_context_tx *obj; 403 LIST_HEAD(local_list); 404 int i = 0; 405 406 list_for_each_entry(obj, &pool->list, list_node) 407 if (++i == MLX5E_TLS_TX_POOL_BULK) 408 break; 409 410 list_cut_position(&local_list, &pool->list, &obj->list_node); 411 mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &local_list, MLX5E_TLS_TX_POOL_BULK); 412 atomic64_add(MLX5E_TLS_TX_POOL_BULK, &pool->sw_stats->tx_tls_pool_free); 413 pool->size -= MLX5E_TLS_TX_POOL_BULK; 414 } 415 if (pool->size) { 416 mlx5e_tls_priv_tx_list_cleanup(pool->mdev, &pool->list, pool->size); 417 atomic64_add(pool->size, &pool->sw_stats->tx_tls_pool_free); 418 } 419 } 420 421 static void mlx5e_tls_tx_pool_cleanup(struct mlx5e_tls_tx_pool *pool) 422 { 423 mlx5e_tls_tx_pool_list_cleanup(pool); 424 destroy_workqueue(pool->wq); 425 kvfree(pool); 426 } 427 428 static void pool_push(struct mlx5e_tls_tx_pool *pool, struct mlx5e_ktls_offload_context_tx *obj) 429 { 430 mutex_lock(&pool->lock); 431 list_add(&obj->list_node, &pool->list); 432 if (++pool->size == MLX5E_TLS_TX_POOL_HIGH) 433 queue_work(pool->wq, &pool->destroy_work); 434 mutex_unlock(&pool->lock); 435 } 436 437 static struct mlx5e_ktls_offload_context_tx *pool_pop(struct mlx5e_tls_tx_pool *pool) 438 { 439 struct mlx5e_ktls_offload_context_tx *obj; 440 441 mutex_lock(&pool->lock); 442 if (unlikely(pool->size == 0)) { 443 /* pool is empty: 444 * - trigger the populating work, and 445 * - serve the current context via the regular blocking api. 446 */ 447 queue_work(pool->wq, &pool->create_work); 448 mutex_unlock(&pool->lock); 449 obj = mlx5e_tls_priv_tx_init(pool->mdev, pool->sw_stats, NULL); 450 if (!IS_ERR(obj)) 451 atomic64_inc(&pool->sw_stats->tx_tls_pool_alloc); 452 return obj; 453 } 454 455 obj = list_first_entry(&pool->list, struct mlx5e_ktls_offload_context_tx, 456 list_node); 457 list_del(&obj->list_node); 458 if (--pool->size == MLX5E_TLS_TX_POOL_LOW) 459 queue_work(pool->wq, &pool->create_work); 460 mutex_unlock(&pool->lock); 461 return obj; 462 } 463 464 /* End of pool API */ 465 466 int mlx5e_ktls_add_tx(struct net_device *netdev, struct sock *sk, 467 struct tls_crypto_info *crypto_info, u32 start_offload_tcp_sn) 468 { 469 struct mlx5e_ktls_offload_context_tx *priv_tx; 470 struct mlx5e_tls_tx_pool *pool; 471 struct tls_context *tls_ctx; 472 struct mlx5e_priv *priv; 473 int err; 474 475 tls_ctx = tls_get_ctx(sk); 476 priv = netdev_priv(netdev); 477 pool = priv->tls->tx_pool; 478 479 priv_tx = pool_pop(pool); 480 if (IS_ERR(priv_tx)) 481 return PTR_ERR(priv_tx); 482 483 err = mlx5_ktls_create_key(pool->mdev, crypto_info, &priv_tx->key_id); 484 if (err) 485 goto err_create_key; 486 487 priv_tx->expected_seq = start_offload_tcp_sn; 488 switch (crypto_info->cipher_type) { 489 case TLS_CIPHER_AES_GCM_128: 490 priv_tx->crypto_info.crypto_info_128 = 491 *(struct tls12_crypto_info_aes_gcm_128 *)crypto_info; 492 break; 493 case TLS_CIPHER_AES_GCM_256: 494 priv_tx->crypto_info.crypto_info_256 = 495 *(struct tls12_crypto_info_aes_gcm_256 *)crypto_info; 496 break; 497 default: 498 WARN_ONCE(1, "Unsupported cipher type %u\n", 499 crypto_info->cipher_type); 500 return -EOPNOTSUPP; 501 } 502 priv_tx->tx_ctx = tls_offload_ctx_tx(tls_ctx); 503 504 mlx5e_set_ktls_tx_priv_ctx(tls_ctx, priv_tx); 505 506 priv_tx->ctx_post_pending = true; 507 atomic64_inc(&priv_tx->sw_stats->tx_tls_ctx); 508 509 return 0; 510 511 err_create_key: 512 pool_push(pool, priv_tx); 513 return err; 514 } 515 516 void mlx5e_ktls_del_tx(struct net_device *netdev, struct tls_context *tls_ctx) 517 { 518 struct mlx5e_ktls_offload_context_tx *priv_tx; 519 struct mlx5e_tls_tx_pool *pool; 520 struct mlx5e_priv *priv; 521 522 priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); 523 priv = netdev_priv(netdev); 524 pool = priv->tls->tx_pool; 525 526 atomic64_inc(&priv_tx->sw_stats->tx_tls_del); 527 mlx5_ktls_destroy_key(priv_tx->mdev, priv_tx->key_id); 528 pool_push(pool, priv_tx); 529 } 530 531 static void tx_fill_wi(struct mlx5e_txqsq *sq, 532 u16 pi, u8 num_wqebbs, u32 num_bytes, 533 struct page *page) 534 { 535 struct mlx5e_tx_wqe_info *wi = &sq->db.wqe_info[pi]; 536 537 *wi = (struct mlx5e_tx_wqe_info) { 538 .num_wqebbs = num_wqebbs, 539 .num_bytes = num_bytes, 540 .resync_dump_frag_page = page, 541 }; 542 } 543 544 static bool 545 mlx5e_ktls_tx_offload_test_and_clear_pending(struct mlx5e_ktls_offload_context_tx *priv_tx) 546 { 547 bool ret = priv_tx->ctx_post_pending; 548 549 priv_tx->ctx_post_pending = false; 550 551 return ret; 552 } 553 554 static void 555 post_static_params(struct mlx5e_txqsq *sq, 556 struct mlx5e_ktls_offload_context_tx *priv_tx, 557 bool fence) 558 { 559 struct mlx5e_set_tls_static_params_wqe *wqe; 560 u16 pi, num_wqebbs; 561 562 num_wqebbs = MLX5E_TLS_SET_STATIC_PARAMS_WQEBBS; 563 pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); 564 wqe = MLX5E_TLS_FETCH_SET_STATIC_PARAMS_WQE(sq, pi); 565 mlx5e_ktls_build_static_params(wqe, sq->pc, sq->sqn, &priv_tx->crypto_info, 566 priv_tx->tisn, priv_tx->key_id, 0, fence, 567 TLS_OFFLOAD_CTX_DIR_TX); 568 tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); 569 sq->pc += num_wqebbs; 570 } 571 572 static void 573 post_progress_params(struct mlx5e_txqsq *sq, 574 struct mlx5e_ktls_offload_context_tx *priv_tx, 575 bool fence) 576 { 577 struct mlx5e_set_tls_progress_params_wqe *wqe; 578 u16 pi, num_wqebbs; 579 580 num_wqebbs = MLX5E_TLS_SET_PROGRESS_PARAMS_WQEBBS; 581 pi = mlx5e_txqsq_get_next_pi(sq, num_wqebbs); 582 wqe = MLX5E_TLS_FETCH_SET_PROGRESS_PARAMS_WQE(sq, pi); 583 mlx5e_ktls_build_progress_params(wqe, sq->pc, sq->sqn, priv_tx->tisn, fence, 0, 584 TLS_OFFLOAD_CTX_DIR_TX); 585 tx_fill_wi(sq, pi, num_wqebbs, 0, NULL); 586 sq->pc += num_wqebbs; 587 } 588 589 static void tx_post_fence_nop(struct mlx5e_txqsq *sq) 590 { 591 struct mlx5_wq_cyc *wq = &sq->wq; 592 u16 pi = mlx5_wq_cyc_ctr2ix(wq, sq->pc); 593 594 tx_fill_wi(sq, pi, 1, 0, NULL); 595 596 mlx5e_post_nop_fence(wq, sq->sqn, &sq->pc); 597 } 598 599 static void 600 mlx5e_ktls_tx_post_param_wqes(struct mlx5e_txqsq *sq, 601 struct mlx5e_ktls_offload_context_tx *priv_tx, 602 bool skip_static_post, bool fence_first_post) 603 { 604 bool progress_fence = skip_static_post || !fence_first_post; 605 606 if (!skip_static_post) 607 post_static_params(sq, priv_tx, fence_first_post); 608 609 post_progress_params(sq, priv_tx, progress_fence); 610 tx_post_fence_nop(sq); 611 } 612 613 struct tx_sync_info { 614 u64 rcd_sn; 615 u32 sync_len; 616 int nr_frags; 617 skb_frag_t frags[MAX_SKB_FRAGS]; 618 }; 619 620 enum mlx5e_ktls_sync_retval { 621 MLX5E_KTLS_SYNC_DONE, 622 MLX5E_KTLS_SYNC_FAIL, 623 MLX5E_KTLS_SYNC_SKIP_NO_DATA, 624 }; 625 626 static enum mlx5e_ktls_sync_retval 627 tx_sync_info_get(struct mlx5e_ktls_offload_context_tx *priv_tx, 628 u32 tcp_seq, int datalen, struct tx_sync_info *info) 629 { 630 struct tls_offload_context_tx *tx_ctx = priv_tx->tx_ctx; 631 enum mlx5e_ktls_sync_retval ret = MLX5E_KTLS_SYNC_DONE; 632 struct tls_record_info *record; 633 int remaining, i = 0; 634 unsigned long flags; 635 bool ends_before; 636 637 spin_lock_irqsave(&tx_ctx->lock, flags); 638 record = tls_get_record(tx_ctx, tcp_seq, &info->rcd_sn); 639 640 if (unlikely(!record)) { 641 ret = MLX5E_KTLS_SYNC_FAIL; 642 goto out; 643 } 644 645 /* There are the following cases: 646 * 1. packet ends before start marker: bypass offload. 647 * 2. packet starts before start marker and ends after it: drop, 648 * not supported, breaks contract with kernel. 649 * 3. packet ends before tls record info starts: drop, 650 * this packet was already acknowledged and its record info 651 * was released. 652 */ 653 ends_before = before(tcp_seq + datalen - 1, tls_record_start_seq(record)); 654 655 if (unlikely(tls_record_is_start_marker(record))) { 656 ret = ends_before ? MLX5E_KTLS_SYNC_SKIP_NO_DATA : MLX5E_KTLS_SYNC_FAIL; 657 goto out; 658 } else if (ends_before) { 659 ret = MLX5E_KTLS_SYNC_FAIL; 660 goto out; 661 } 662 663 info->sync_len = tcp_seq - tls_record_start_seq(record); 664 remaining = info->sync_len; 665 while (remaining > 0) { 666 skb_frag_t *frag = &record->frags[i]; 667 668 get_page(skb_frag_page(frag)); 669 remaining -= skb_frag_size(frag); 670 info->frags[i++] = *frag; 671 } 672 /* reduce the part which will be sent with the original SKB */ 673 if (remaining < 0) 674 skb_frag_size_add(&info->frags[i - 1], remaining); 675 info->nr_frags = i; 676 out: 677 spin_unlock_irqrestore(&tx_ctx->lock, flags); 678 return ret; 679 } 680 681 static void 682 tx_post_resync_params(struct mlx5e_txqsq *sq, 683 struct mlx5e_ktls_offload_context_tx *priv_tx, 684 u64 rcd_sn) 685 { 686 __be64 rn_be = cpu_to_be64(rcd_sn); 687 bool skip_static_post; 688 u16 rec_seq_sz; 689 char *rec_seq; 690 691 switch (priv_tx->crypto_info.crypto_info.cipher_type) { 692 case TLS_CIPHER_AES_GCM_128: { 693 struct tls12_crypto_info_aes_gcm_128 *info = &priv_tx->crypto_info.crypto_info_128; 694 695 rec_seq = info->rec_seq; 696 rec_seq_sz = sizeof(info->rec_seq); 697 break; 698 } 699 case TLS_CIPHER_AES_GCM_256: { 700 struct tls12_crypto_info_aes_gcm_256 *info = &priv_tx->crypto_info.crypto_info_256; 701 702 rec_seq = info->rec_seq; 703 rec_seq_sz = sizeof(info->rec_seq); 704 break; 705 } 706 default: 707 WARN_ONCE(1, "Unsupported cipher type %u\n", 708 priv_tx->crypto_info.crypto_info.cipher_type); 709 return; 710 } 711 712 skip_static_post = !memcmp(rec_seq, &rn_be, rec_seq_sz); 713 if (!skip_static_post) 714 memcpy(rec_seq, &rn_be, rec_seq_sz); 715 716 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, skip_static_post, true); 717 } 718 719 static int 720 tx_post_resync_dump(struct mlx5e_txqsq *sq, skb_frag_t *frag, u32 tisn) 721 { 722 struct mlx5_wqe_ctrl_seg *cseg; 723 struct mlx5_wqe_data_seg *dseg; 724 struct mlx5e_dump_wqe *wqe; 725 dma_addr_t dma_addr = 0; 726 u16 ds_cnt; 727 int fsz; 728 u16 pi; 729 730 BUILD_BUG_ON(MLX5E_KTLS_DUMP_WQEBBS != 1); 731 pi = mlx5_wq_cyc_ctr2ix(&sq->wq, sq->pc); 732 wqe = MLX5E_TLS_FETCH_DUMP_WQE(sq, pi); 733 734 ds_cnt = sizeof(*wqe) / MLX5_SEND_WQE_DS; 735 736 cseg = &wqe->ctrl; 737 dseg = &wqe->data; 738 739 cseg->opmod_idx_opcode = cpu_to_be32((sq->pc << 8) | MLX5_OPCODE_DUMP); 740 cseg->qpn_ds = cpu_to_be32((sq->sqn << 8) | ds_cnt); 741 cseg->tis_tir_num = cpu_to_be32(tisn << 8); 742 743 fsz = skb_frag_size(frag); 744 dma_addr = skb_frag_dma_map(sq->pdev, frag, 0, fsz, 745 DMA_TO_DEVICE); 746 if (unlikely(dma_mapping_error(sq->pdev, dma_addr))) 747 return -ENOMEM; 748 749 dseg->addr = cpu_to_be64(dma_addr); 750 dseg->lkey = sq->mkey_be; 751 dseg->byte_count = cpu_to_be32(fsz); 752 mlx5e_dma_push(sq, dma_addr, fsz, MLX5E_DMA_MAP_PAGE); 753 754 tx_fill_wi(sq, pi, MLX5E_KTLS_DUMP_WQEBBS, fsz, skb_frag_page(frag)); 755 sq->pc += MLX5E_KTLS_DUMP_WQEBBS; 756 757 return 0; 758 } 759 760 void mlx5e_ktls_tx_handle_resync_dump_comp(struct mlx5e_txqsq *sq, 761 struct mlx5e_tx_wqe_info *wi, 762 u32 *dma_fifo_cc) 763 { 764 struct mlx5e_sq_stats *stats; 765 struct mlx5e_sq_dma *dma; 766 767 dma = mlx5e_dma_get(sq, (*dma_fifo_cc)++); 768 stats = sq->stats; 769 770 mlx5e_tx_dma_unmap(sq->pdev, dma); 771 put_page(wi->resync_dump_frag_page); 772 stats->tls_dump_packets++; 773 stats->tls_dump_bytes += wi->num_bytes; 774 } 775 776 static enum mlx5e_ktls_sync_retval 777 mlx5e_ktls_tx_handle_ooo(struct mlx5e_ktls_offload_context_tx *priv_tx, 778 struct mlx5e_txqsq *sq, 779 int datalen, 780 u32 seq) 781 { 782 enum mlx5e_ktls_sync_retval ret; 783 struct tx_sync_info info = {}; 784 int i; 785 786 ret = tx_sync_info_get(priv_tx, seq, datalen, &info); 787 if (unlikely(ret != MLX5E_KTLS_SYNC_DONE)) 788 /* We might get here with ret == FAIL if a retransmission 789 * reaches the driver after the relevant record is acked. 790 * It should be safe to drop the packet in this case 791 */ 792 return ret; 793 794 tx_post_resync_params(sq, priv_tx, info.rcd_sn); 795 796 for (i = 0; i < info.nr_frags; i++) { 797 unsigned int orig_fsz, frag_offset = 0, n = 0; 798 skb_frag_t *f = &info.frags[i]; 799 800 orig_fsz = skb_frag_size(f); 801 802 do { 803 unsigned int fsz; 804 805 n++; 806 fsz = min_t(unsigned int, sq->hw_mtu, orig_fsz - frag_offset); 807 skb_frag_size_set(f, fsz); 808 if (tx_post_resync_dump(sq, f, priv_tx->tisn)) { 809 page_ref_add(skb_frag_page(f), n - 1); 810 goto err_out; 811 } 812 813 skb_frag_off_add(f, fsz); 814 frag_offset += fsz; 815 } while (frag_offset < orig_fsz); 816 817 page_ref_add(skb_frag_page(f), n - 1); 818 } 819 820 return MLX5E_KTLS_SYNC_DONE; 821 822 err_out: 823 for (; i < info.nr_frags; i++) 824 /* The put_page() here undoes the page ref obtained in tx_sync_info_get(). 825 * Page refs obtained for the DUMP WQEs above (by page_ref_add) will be 826 * released only upon their completions (or in mlx5e_free_txqsq_descs, 827 * if channel closes). 828 */ 829 put_page(skb_frag_page(&info.frags[i])); 830 831 return MLX5E_KTLS_SYNC_FAIL; 832 } 833 834 bool mlx5e_ktls_handle_tx_skb(struct net_device *netdev, struct mlx5e_txqsq *sq, 835 struct sk_buff *skb, 836 struct mlx5e_accel_tx_tls_state *state) 837 { 838 struct mlx5e_ktls_offload_context_tx *priv_tx; 839 struct mlx5e_sq_stats *stats = sq->stats; 840 struct net_device *tls_netdev; 841 struct tls_context *tls_ctx; 842 int datalen; 843 u32 seq; 844 845 datalen = skb->len - skb_tcp_all_headers(skb); 846 if (!datalen) 847 return true; 848 849 mlx5e_tx_mpwqe_ensure_complete(sq); 850 851 tls_ctx = tls_get_ctx(skb->sk); 852 tls_netdev = rcu_dereference_bh(tls_ctx->netdev); 853 /* Don't WARN on NULL: if tls_device_down is running in parallel, 854 * netdev might become NULL, even if tls_is_sk_tx_device_offloaded was 855 * true. Rather continue processing this packet. 856 */ 857 if (WARN_ON_ONCE(tls_netdev && tls_netdev != netdev)) 858 goto err_out; 859 860 priv_tx = mlx5e_get_ktls_tx_priv_ctx(tls_ctx); 861 862 if (unlikely(mlx5e_ktls_tx_offload_test_and_clear_pending(priv_tx))) 863 mlx5e_ktls_tx_post_param_wqes(sq, priv_tx, false, false); 864 865 seq = ntohl(tcp_hdr(skb)->seq); 866 if (unlikely(priv_tx->expected_seq != seq)) { 867 enum mlx5e_ktls_sync_retval ret = 868 mlx5e_ktls_tx_handle_ooo(priv_tx, sq, datalen, seq); 869 870 stats->tls_ooo++; 871 872 switch (ret) { 873 case MLX5E_KTLS_SYNC_DONE: 874 break; 875 case MLX5E_KTLS_SYNC_SKIP_NO_DATA: 876 stats->tls_skip_no_sync_data++; 877 if (likely(!skb->decrypted)) 878 goto out; 879 WARN_ON_ONCE(1); 880 goto err_out; 881 case MLX5E_KTLS_SYNC_FAIL: 882 stats->tls_drop_no_sync_data++; 883 goto err_out; 884 } 885 } 886 887 priv_tx->expected_seq = seq + datalen; 888 889 state->tls_tisn = priv_tx->tisn; 890 891 stats->tls_encrypted_packets += skb_is_gso(skb) ? skb_shinfo(skb)->gso_segs : 1; 892 stats->tls_encrypted_bytes += datalen; 893 894 out: 895 return true; 896 897 err_out: 898 dev_kfree_skb_any(skb); 899 return false; 900 } 901 902 int mlx5e_ktls_init_tx(struct mlx5e_priv *priv) 903 { 904 if (!mlx5e_is_ktls_tx(priv->mdev)) 905 return 0; 906 907 priv->tls->tx_pool = mlx5e_tls_tx_pool_init(priv->mdev, &priv->tls->sw_stats); 908 if (!priv->tls->tx_pool) 909 return -ENOMEM; 910 911 return 0; 912 } 913 914 void mlx5e_ktls_cleanup_tx(struct mlx5e_priv *priv) 915 { 916 if (!mlx5e_is_ktls_tx(priv->mdev)) 917 return; 918 919 mlx5e_tls_tx_pool_cleanup(priv->tls->tx_pool); 920 priv->tls->tx_pool = NULL; 921 } 922