1 /*-
2 * Copyright (c) 2021-2022 NVIDIA corporation & affiliates.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_kern_tls.h"
27 #include "opt_rss.h"
28 #include "opt_ratelimit.h"
29
30 #include <dev/mlx5/mlx5_en/en.h>
31
32 #include <dev/mlx5/crypto.h>
33 #include <dev/mlx5/tls.h>
34
35 #include <dev/mlx5/fs.h>
36 #include <dev/mlx5/mlx5_core/fs_tcp.h>
37
38 #include <sys/ktls.h>
39 #include <opencrypto/cryptodev.h>
40
41 #ifdef KERN_TLS
42
43 static if_snd_tag_free_t mlx5e_tls_rx_snd_tag_free;
44 static if_snd_tag_modify_t mlx5e_tls_rx_snd_tag_modify;
45
46 static const struct if_snd_tag_sw mlx5e_tls_rx_snd_tag_sw = {
47 .snd_tag_modify = mlx5e_tls_rx_snd_tag_modify,
48 .snd_tag_free = mlx5e_tls_rx_snd_tag_free,
49 .type = IF_SND_TAG_TYPE_TLS_RX
50 };
51
52 MALLOC_DEFINE(M_MLX5E_TLS_RX, "MLX5E_TLS_RX", "MLX5 ethernet HW TLS RX");
53
54 /* software TLS RX context */
55 struct mlx5_ifc_sw_tls_rx_cntx_bits {
56 struct mlx5_ifc_tls_static_params_bits param;
57 struct mlx5_ifc_tls_progress_params_bits progress;
58 struct {
59 uint8_t key_data[8][0x20];
60 uint8_t key_len[0x20];
61 } key;
62 };
63
64 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_rx_cntx) <= sizeof(((struct mlx5e_tls_rx_tag *)NULL)->crypto_params));
65 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)NULL)->mkc));
66
67 static const char *mlx5e_tls_rx_stats_desc[] = {
68 MLX5E_TLS_RX_STATS(MLX5E_STATS_DESC)
69 };
70
71 static void mlx5e_tls_rx_work(struct work_struct *);
72 static bool mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *,
73 uint32_t, uint32_t *, uint64_t *);
74
75 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) % 16) == 0);
76
77 static uint32_t
mlx5e_tls_rx_get_ch(struct mlx5e_priv * priv,uint32_t flowid,uint32_t flowtype)78 mlx5e_tls_rx_get_ch(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype)
79 {
80 u32 ch;
81 #ifdef RSS
82 u32 temp;
83 #endif
84
85 /* keep this code synced with mlx5e_select_queue() */
86 ch = priv->params.num_channels;
87 #ifdef RSS
88 if (rss_hash2bucket(flowid, flowtype, &temp) == 0)
89 ch = temp % ch;
90 else
91 #endif
92 ch = (flowid % 128) % ch;
93 return (ch);
94 }
95
96 /*
97 * This function gets a pointer to an internal queue, IQ, based on the
98 * provided "flowid" and "flowtype". The IQ returned may in some rare
99 * cases not be activated or running, but this is all handled by the
100 * "mlx5e_iq_get_producer_index()" function.
101 *
102 * The idea behind this function is to spread the IQ traffic as much
103 * as possible and to avoid congestion on the same IQ when processing
104 * RX traffic.
105 */
106 static struct mlx5e_iq *
mlx5e_tls_rx_get_iq(struct mlx5e_priv * priv,uint32_t flowid,uint32_t flowtype)107 mlx5e_tls_rx_get_iq(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype)
108 {
109 /*
110 * NOTE: The channels array is only freed at detach
111 * and it safe to return a pointer to the send tag
112 * inside the channels structure as long as we
113 * reference the priv.
114 */
115 return (&priv->channel[mlx5e_tls_rx_get_ch(priv, flowid, flowtype)].iq);
116 }
117
118 static void
mlx5e_tls_rx_send_static_parameters_cb(void * arg)119 mlx5e_tls_rx_send_static_parameters_cb(void *arg)
120 {
121 struct mlx5e_tls_rx_tag *ptag;
122
123 ptag = (struct mlx5e_tls_rx_tag *)arg;
124
125 m_snd_tag_rele(&ptag->tag);
126 }
127
128 /*
129 * This function sends the so-called TLS RX static parameters to the
130 * hardware. These parameters are temporarily stored in the
131 * "crypto_params" field of the TLS RX tag. Most importantly this
132 * function sets the TCP sequence number (32-bit) and TLS record
133 * number (64-bit) where the decryption can resume.
134 *
135 * Zero is returned upon success. Else some error happend.
136 */
137 static int
mlx5e_tls_rx_send_static_parameters(struct mlx5e_iq * iq,struct mlx5e_tls_rx_tag * ptag)138 mlx5e_tls_rx_send_static_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag)
139 {
140 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
141 MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param), MLX5_SEND_WQE_DS);
142 struct mlx5e_tx_umr_wqe *wqe;
143 int pi;
144
145 mtx_lock(&iq->lock);
146 pi = mlx5e_iq_get_producer_index(iq);
147 if (pi < 0) {
148 mtx_unlock(&iq->lock);
149 return (-ENOMEM);
150 }
151 wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
152
153 memset(wqe, 0, sizeof(*wqe));
154
155 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
156 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIR_STATIC_PARAMS << 24));
157 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
158 wqe->ctrl.imm = cpu_to_be32(ptag->tirn << 8);
159 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
160
161 /* fill out UMR control segment */
162 wqe->umr.flags = 0x80; /* inline data */
163 wqe->umr.bsf_octowords =
164 cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) / 16);
165
166 /* copy in the static crypto parameters */
167 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, param),
168 MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param));
169
170 /* copy data for doorbell */
171 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
172
173 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
174 iq->data[pi].callback = &mlx5e_tls_rx_send_static_parameters_cb;
175 iq->data[pi].arg = ptag;
176
177 m_snd_tag_ref(&ptag->tag);
178
179 iq->pc += iq->data[pi].num_wqebbs;
180
181 mlx5e_iq_notify_hw(iq);
182
183 mtx_unlock(&iq->lock);
184
185 return (0); /* success */
186 }
187
188 static void
mlx5e_tls_rx_send_progress_parameters_cb(void * arg)189 mlx5e_tls_rx_send_progress_parameters_cb(void *arg)
190 {
191 struct mlx5e_tls_rx_tag *ptag;
192
193 ptag = (struct mlx5e_tls_rx_tag *)arg;
194
195 complete(&ptag->progress_complete);
196 }
197
198 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, progress) ==
199 sizeof(((struct mlx5e_tx_psv_wqe *)NULL)->psv));
200
201 /*
202 * This function resets the state of the TIR context to start
203 * searching for a valid TLS header and is used only when allocating
204 * the TLS RX tag.
205 *
206 * Zero is returned upon success, else some error happened.
207 */
208 static int
mlx5e_tls_rx_send_progress_parameters_sync(struct mlx5e_iq * iq,struct mlx5e_tls_rx_tag * ptag)209 mlx5e_tls_rx_send_progress_parameters_sync(struct mlx5e_iq *iq,
210 struct mlx5e_tls_rx_tag *ptag)
211 {
212 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
213 MLX5_SEND_WQE_DS);
214 struct mlx5e_priv *priv;
215 struct mlx5e_tx_psv_wqe *wqe;
216 int pi;
217
218 mtx_lock(&iq->lock);
219 pi = mlx5e_iq_get_producer_index(iq);
220 if (pi < 0) {
221 mtx_unlock(&iq->lock);
222 return (-ENOMEM);
223 }
224 wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
225
226 memset(wqe, 0, sizeof(*wqe));
227
228 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
229 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24));
230 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
231 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
232
233 /* copy in the PSV control segment */
234 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, progress),
235 sizeof(wqe->psv));
236
237 /* copy data for doorbell */
238 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
239
240 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
241 iq->data[pi].callback = &mlx5e_tls_rx_send_progress_parameters_cb;
242 iq->data[pi].arg = ptag;
243
244 iq->pc += iq->data[pi].num_wqebbs;
245
246 init_completion(&ptag->progress_complete);
247
248 mlx5e_iq_notify_hw(iq);
249
250 mtx_unlock(&iq->lock);
251
252 while (1) {
253 if (wait_for_completion_timeout(&ptag->progress_complete, hz) != 0)
254 break;
255 priv = container_of(iq, struct mlx5e_channel, iq)->priv;
256 if (priv->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR ||
257 pci_channel_offline(priv->mdev->pdev) != 0)
258 return (-EWOULDBLOCK);
259 }
260
261 return (0); /* success */
262 }
263
264 CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE >= MLX5_ST_SZ_BYTES(tls_progress_params));
265 CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE <= PAGE_SIZE);
266
267 struct mlx5e_get_tls_progress_params_wqe {
268 struct mlx5_wqe_ctrl_seg ctrl;
269 struct mlx5_seg_get_psv psv;
270 };
271
272 static void
mlx5e_tls_rx_receive_progress_parameters_cb(void * arg)273 mlx5e_tls_rx_receive_progress_parameters_cb(void *arg)
274 {
275 struct mlx5e_tls_rx_tag *ptag;
276 struct mlx5e_iq *iq;
277 uint32_t tcp_curr_sn_he;
278 uint32_t tcp_next_sn_he;
279 uint64_t tls_rcd_num;
280 void *buffer;
281
282 ptag = (struct mlx5e_tls_rx_tag *)arg;
283 buffer = mlx5e_tls_rx_get_progress_buffer(ptag);
284
285 MLX5E_TLS_RX_TAG_LOCK(ptag);
286
287 ptag->tcp_resync_pending = 0;
288
289 switch (MLX5_GET(tls_progress_params, buffer, record_tracker_state)) {
290 case MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING:
291 break;
292 default:
293 goto done;
294 }
295
296 switch (MLX5_GET(tls_progress_params, buffer, auth_state)) {
297 case MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD:
298 break;
299 default:
300 goto done;
301 }
302
303 tcp_curr_sn_he = MLX5_GET(tls_progress_params, buffer, hw_resync_tcp_sn);
304
305 if (mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(ptag, tcp_curr_sn_he,
306 &tcp_next_sn_he, &tls_rcd_num)) {
307
308 MLX5_SET64(sw_tls_rx_cntx, ptag->crypto_params,
309 param.initial_record_number, tls_rcd_num);
310 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params,
311 param.resync_tcp_sn, tcp_curr_sn_he);
312
313 iq = mlx5e_tls_rx_get_iq(
314 container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx),
315 ptag->flowid, ptag->flowtype);
316
317 if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0)
318 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
319 }
320 done:
321 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
322
323 m_snd_tag_rele(&ptag->tag);
324 }
325
326 /*
327 * This function queries the hardware for the current state of the TIR
328 * in question. It is typically called when encrypted data is received
329 * to re-establish hardware decryption of received TLS data.
330 *
331 * Zero is returned upon success, else some error happened.
332 */
333 static int
mlx5e_tls_rx_receive_progress_parameters(struct mlx5e_iq * iq,struct mlx5e_tls_rx_tag * ptag)334 mlx5e_tls_rx_receive_progress_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag)
335 {
336 struct mlx5e_get_tls_progress_params_wqe *wqe;
337 const u32 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
338 u64 dma_address;
339 int pi;
340
341 mtx_lock(&iq->lock);
342 pi = mlx5e_iq_get_producer_index(iq);
343 if (pi < 0) {
344 mtx_unlock(&iq->lock);
345 return (-ENOMEM);
346 }
347
348 mlx5e_iq_load_memory_single(iq, pi,
349 mlx5e_tls_rx_get_progress_buffer(ptag),
350 MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE,
351 &dma_address, BUS_DMASYNC_PREREAD);
352
353 wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
354
355 memset(wqe, 0, sizeof(*wqe));
356
357 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
358 MLX5_OPCODE_GET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24));
359 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
360 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
361 wqe->psv.num_psv = 1 << 4;
362 wqe->psv.l_key = iq->mkey_be;
363 wqe->psv.psv_index[0] = cpu_to_be32(ptag->tirn);
364 wqe->psv.va = cpu_to_be64(dma_address);
365
366 /* copy data for doorbell */
367 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
368
369 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
370 iq->data[pi].callback = &mlx5e_tls_rx_receive_progress_parameters_cb;
371 iq->data[pi].arg = ptag;
372
373 m_snd_tag_ref(&ptag->tag);
374
375 iq->pc += iq->data[pi].num_wqebbs;
376
377 mlx5e_iq_notify_hw(iq);
378
379 mtx_unlock(&iq->lock);
380
381 return (0); /* success */
382 }
383
384 /*
385 * This is the import function for TLS RX tags.
386 */
387 static int
mlx5e_tls_rx_tag_import(void * arg,void ** store,int cnt,int domain,int flags)388 mlx5e_tls_rx_tag_import(void *arg, void **store, int cnt, int domain, int flags)
389 {
390 struct mlx5e_tls_rx_tag *ptag;
391 int i;
392
393 for (i = 0; i != cnt; i++) {
394 ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS_RX,
395 mlx5_dev_domainset(arg), flags | M_ZERO);
396 mtx_init(&ptag->mtx, "mlx5-tls-rx-tag-mtx", NULL, MTX_DEF);
397 INIT_WORK(&ptag->work, mlx5e_tls_rx_work);
398 store[i] = ptag;
399 }
400 return (i);
401 }
402
403 /*
404 * This is the release function for TLS RX tags.
405 */
406 static void
mlx5e_tls_rx_tag_release(void * arg,void ** store,int cnt)407 mlx5e_tls_rx_tag_release(void *arg, void **store, int cnt)
408 {
409 struct mlx5e_tls_rx_tag *ptag;
410 int i;
411
412 for (i = 0; i != cnt; i++) {
413 ptag = store[i];
414
415 flush_work(&ptag->work);
416 mtx_destroy(&ptag->mtx);
417 free(ptag, M_MLX5E_TLS_RX);
418 }
419 }
420
421 /*
422 * This is a convenience function to free TLS RX tags. It resets some
423 * selected fields, updates the number of resources and returns the
424 * TLS RX tag to the UMA pool of free tags.
425 */
426 static void
mlx5e_tls_rx_tag_zfree(struct mlx5e_tls_rx_tag * ptag)427 mlx5e_tls_rx_tag_zfree(struct mlx5e_tls_rx_tag *ptag)
428 {
429 /* make sure any unhandled taskqueue events are ignored */
430 ptag->state = MLX5E_TLS_RX_ST_FREED;
431
432 /* reset some variables */
433 ptag->dek_index = 0;
434 ptag->dek_index_ok = 0;
435 ptag->tirn = 0;
436 ptag->flow_rule = NULL;
437 ptag->tcp_resync_active = 0;
438 ptag->tcp_resync_pending = 0;
439
440 /* avoid leaking keys */
441 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
442
443 /* update number of resources in use */
444 atomic_add_32(&ptag->tls_rx->num_resources, -1U);
445
446 /* return tag to UMA */
447 uma_zfree(ptag->tls_rx->zone, ptag);
448 }
449
450 /*
451 * This function enables TLS RX support for the given NIC, if all
452 * needed firmware capabilites are present.
453 */
454 int
mlx5e_tls_rx_init(struct mlx5e_priv * priv)455 mlx5e_tls_rx_init(struct mlx5e_priv *priv)
456 {
457 struct mlx5e_tls_rx *ptls = &priv->tls_rx;
458 struct sysctl_oid *node;
459 uint32_t x;
460
461 if (MLX5_CAP_GEN(priv->mdev, tls_rx) == 0 ||
462 MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0 ||
463 MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version) == 0)
464 return (0);
465
466 ptls->wq = create_singlethread_workqueue("mlx5-tls-rx-wq");
467 if (ptls->wq == NULL)
468 return (ENOMEM);
469
470 sysctl_ctx_init(&ptls->ctx);
471
472 snprintf(ptls->zname, sizeof(ptls->zname),
473 "mlx5_%u_tls_rx", device_get_unit(priv->mdev->pdev->dev.bsddev));
474
475 ptls->zone = uma_zcache_create(ptls->zname,
476 sizeof(struct mlx5e_tls_rx_tag), NULL, NULL, NULL, NULL,
477 mlx5e_tls_rx_tag_import, mlx5e_tls_rx_tag_release, priv->mdev,
478 UMA_ZONE_UNMANAGED);
479
480 /* shared between RX and TX TLS */
481 ptls->max_resources = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1);
482
483 for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++)
484 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
485
486 ptls->init = 1;
487
488 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
489 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
490 "tls_rx", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS receive offload");
491 if (node == NULL)
492 return (0);
493
494 mlx5e_create_counter_stats(&ptls->ctx,
495 SYSCTL_CHILDREN(node), "stats",
496 mlx5e_tls_rx_stats_desc, MLX5E_TLS_RX_STATS_NUM,
497 ptls->stats.arg);
498
499 return (0);
500 }
501
502 /*
503 * This function disables TLS RX support for the given NIC.
504 */
505 void
mlx5e_tls_rx_cleanup(struct mlx5e_priv * priv)506 mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv)
507 {
508 struct mlx5e_tls_rx *ptls = &priv->tls_rx;
509 uint32_t x;
510
511 if (ptls->init == 0)
512 return;
513
514 ptls->init = 0;
515 flush_workqueue(ptls->wq);
516 sysctl_ctx_free(&ptls->ctx);
517 uma_zdestroy(ptls->zone);
518 destroy_workqueue(ptls->wq);
519
520 /* check if all resources are freed */
521 MPASS(priv->tls_rx.num_resources == 0);
522
523 for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++)
524 counter_u64_free(ptls->stats.arg[x]);
525 }
526
527 /*
528 * This function is used to serialize sleeping firmware operations
529 * needed in order to establish and destroy a TLS RX tag.
530 */
531 static void
mlx5e_tls_rx_work(struct work_struct * work)532 mlx5e_tls_rx_work(struct work_struct *work)
533 {
534 struct mlx5e_tls_rx_tag *ptag;
535 struct mlx5e_priv *priv;
536 int err;
537
538 ptag = container_of(work, struct mlx5e_tls_rx_tag, work);
539 priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx);
540
541 switch (ptag->state) {
542 case MLX5E_TLS_RX_ST_INIT:
543 /* try to allocate new TIR context */
544 err = mlx5_tls_open_tir(priv->mdev, priv->tdn,
545 priv->channel[mlx5e_tls_rx_get_ch(priv, ptag->flowid, ptag->flowtype)].rqtn,
546 &ptag->tirn);
547 if (err) {
548 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
549 break;
550 }
551 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.pd, ptag->tirn);
552
553 /* try to allocate a DEK context ID */
554 err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
555 MLX5_GENERAL_OBJECT_TYPE_ENCRYPTION_KEY_TYPE_TLS,
556 MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, key.key_data),
557 MLX5_GET(sw_tls_rx_cntx, ptag->crypto_params, key.key_len),
558 &ptag->dek_index);
559 if (err) {
560 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
561 break;
562 }
563
564 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
565
566 ptag->dek_index_ok = 1;
567
568 MLX5E_TLS_RX_TAG_LOCK(ptag);
569 if (ptag->state == MLX5E_TLS_RX_ST_INIT)
570 ptag->state = MLX5E_TLS_RX_ST_SETUP;
571 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
572 break;
573
574 case MLX5E_TLS_RX_ST_RELEASE:
575 /* remove flow rule for incoming traffic, if any */
576 if (ptag->flow_rule != NULL)
577 mlx5e_accel_fs_del_inpcb(ptag->flow_rule);
578
579 /* try to destroy DEK context by ID */
580 if (ptag->dek_index_ok)
581 mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
582
583 /* try to destroy TIR context by ID */
584 if (ptag->tirn != 0)
585 mlx5_tls_close_tir(priv->mdev, ptag->tirn);
586
587 /* free tag */
588 mlx5e_tls_rx_tag_zfree(ptag);
589 break;
590
591 default:
592 break;
593 }
594 }
595
596 /*
597 * This function translates the crypto parameters into the format used
598 * by the firmware and hardware. Currently only AES-128 and AES-256 is
599 * supported for TLS v1.2 and TLS v1.3.
600 *
601 * Returns zero on success, else an error happened.
602 */
603 static int
mlx5e_tls_rx_set_params(void * ctx,struct inpcb * inp,const struct tls_session_params * en)604 mlx5e_tls_rx_set_params(void *ctx, struct inpcb *inp, const struct tls_session_params *en)
605 {
606 uint32_t tcp_sn_he;
607 uint64_t tls_sn_he;
608
609 MLX5_SET(sw_tls_rx_cntx, ctx, param.const_2, 2);
610 if (en->tls_vminor == TLS_MINOR_VER_TWO)
611 MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 2); /* v1.2 */
612 else
613 MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 3); /* v1.3 */
614 MLX5_SET(sw_tls_rx_cntx, ctx, param.const_1, 1);
615 MLX5_SET(sw_tls_rx_cntx, ctx, param.encryption_standard, 1); /* TLS */
616
617 /* copy the initial vector in place */
618 switch (en->iv_len) {
619 case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv):
620 case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv) +
621 MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.implicit_iv):
622 memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, param.gcm_iv),
623 en->iv, en->iv_len);
624 break;
625 default:
626 return (EINVAL);
627 }
628
629 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, key.key_data)) {
630 memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, key.key_data),
631 en->cipher_key, en->cipher_key_len);
632 MLX5_SET(sw_tls_rx_cntx, ctx, key.key_len, en->cipher_key_len);
633 } else {
634 return (EINVAL);
635 }
636
637 if (__predict_false(inp == NULL ||
638 ktls_get_rx_sequence(inp, &tcp_sn_he, &tls_sn_he) != 0))
639 return (EINVAL);
640
641 MLX5_SET64(sw_tls_rx_cntx, ctx, param.initial_record_number, tls_sn_he);
642 MLX5_SET(sw_tls_rx_cntx, ctx, param.resync_tcp_sn, tcp_sn_he);
643
644 return (0);
645 }
646
647 /* Verify zero default */
648 CTASSERT(MLX5E_TLS_RX_ST_INIT == 0);
649
650 /*
651 * This function is responsible for allocating a TLS RX tag. It is a
652 * callback function invoked by the network stack.
653 *
654 * Returns zero on success else an error happened.
655 */
656 int
mlx5e_tls_rx_snd_tag_alloc(if_t ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)657 mlx5e_tls_rx_snd_tag_alloc(if_t ifp,
658 union if_snd_tag_alloc_params *params,
659 struct m_snd_tag **ppmt)
660 {
661 struct mlx5e_iq *iq;
662 struct mlx5e_priv *priv;
663 struct mlx5e_tls_rx_tag *ptag;
664 struct mlx5_flow_handle *flow_rule;
665 const struct tls_session_params *en;
666 uint32_t value;
667 int error;
668
669 priv = if_getsoftc(ifp);
670
671 if (unlikely(priv->gone != 0 || priv->tls_rx.init == 0 ||
672 params->hdr.flowtype == M_HASHTYPE_NONE))
673 return (EOPNOTSUPP);
674
675 /* allocate new tag from zone, if any */
676 ptag = uma_zalloc(priv->tls_rx.zone, M_NOWAIT);
677 if (ptag == NULL)
678 return (ENOMEM);
679
680 /* sanity check default values */
681 MPASS(ptag->dek_index == 0);
682 MPASS(ptag->dek_index_ok == 0);
683
684 /* setup TLS RX tag */
685 ptag->tls_rx = &priv->tls_rx;
686 ptag->flowtype = params->hdr.flowtype;
687 ptag->flowid = params->hdr.flowid;
688
689 value = atomic_fetchadd_32(&priv->tls_rx.num_resources, 1U);
690
691 /* check resource limits */
692 if (value >= priv->tls_rx.max_resources) {
693 error = ENOMEM;
694 goto failure;
695 }
696
697 en = ¶ms->tls_rx.tls->params;
698
699 /* only TLS v1.2 and v1.3 is currently supported */
700 if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
701 (en->tls_vminor != TLS_MINOR_VER_TWO
702 #ifdef TLS_MINOR_VER_THREE
703 && en->tls_vminor != TLS_MINOR_VER_THREE
704 #endif
705 )) {
706 error = EPROTONOSUPPORT;
707 goto failure;
708 }
709
710 switch (en->cipher_algorithm) {
711 case CRYPTO_AES_NIST_GCM_16:
712 switch (en->cipher_key_len) {
713 case 128 / 8:
714 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
715 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
716 error = EPROTONOSUPPORT;
717 goto failure;
718 }
719 } else {
720 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
721 error = EPROTONOSUPPORT;
722 goto failure;
723 }
724 }
725 error = mlx5e_tls_rx_set_params(
726 ptag->crypto_params, params->tls_rx.inp, en);
727 if (error)
728 goto failure;
729 break;
730
731 case 256 / 8:
732 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
733 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
734 error = EPROTONOSUPPORT;
735 goto failure;
736 }
737 } else {
738 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
739 error = EPROTONOSUPPORT;
740 goto failure;
741 }
742 }
743 error = mlx5e_tls_rx_set_params(
744 ptag->crypto_params, params->tls_rx.inp, en);
745 if (error)
746 goto failure;
747 break;
748
749 default:
750 error = EINVAL;
751 goto failure;
752 }
753 break;
754 default:
755 error = EPROTONOSUPPORT;
756 goto failure;
757 }
758
759 /* store pointer to mbuf tag */
760 MPASS(ptag->tag.refcount == 0);
761 m_snd_tag_init(&ptag->tag, ifp, &mlx5e_tls_rx_snd_tag_sw);
762 *ppmt = &ptag->tag;
763
764 /* reset state */
765 ptag->state = MLX5E_TLS_RX_ST_INIT;
766
767 queue_work(priv->tls_rx.wq, &ptag->work);
768 flush_work(&ptag->work);
769
770 /* check that worker task completed successfully */
771 MLX5E_TLS_RX_TAG_LOCK(ptag);
772 if (ptag->state == MLX5E_TLS_RX_ST_SETUP) {
773 ptag->state = MLX5E_TLS_RX_ST_READY;
774 error = 0;
775 } else {
776 error = ENOMEM;
777 }
778 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
779
780 if (unlikely(error))
781 goto cleanup;
782
783 iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype);
784
785 /* establish connection between DEK and TIR */
786 if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0) {
787 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
788 error = ENOMEM;
789 goto cleanup;
790 }
791
792 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.auth_state,
793 MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
794 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.record_tracker_state,
795 MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
796
797 /* reset state to all zeros */
798 if (mlx5e_tls_rx_send_progress_parameters_sync(iq, ptag) != 0) {
799 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
800 error = ENOMEM;
801 goto cleanup;
802 }
803
804 if (if_getpcp(ifp) != IFNET_PCP_NONE || params->tls_rx.vlan_id != 0) {
805 /* create flow rule for TLS RX traffic (tagged) */
806 flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp,
807 ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, params->tls_rx.vlan_id);
808 } else {
809 /* create flow rule for TLS RX traffic (untagged) */
810 flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp,
811 ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN);
812 }
813
814 if (IS_ERR_OR_NULL(flow_rule)) {
815 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
816 error = ENOMEM;
817 goto cleanup;
818 }
819
820 ptag->flow_rule = flow_rule;
821
822 return (0);
823
824 cleanup:
825 m_snd_tag_rele(&ptag->tag);
826 return (error);
827
828 failure:
829 mlx5e_tls_rx_tag_zfree(ptag);
830 return (error);
831 }
832
833
834 /*
835 * This function adds the TCP sequence number and TLS record number in
836 * host endian format to a small database. When TLS records have the
837 * same length, they are simply accumulated by counting instead of
838 * separated entries in the TLS database. The dimension of the
839 * database is such that it cannot store more than 1GByte of
840 * continuous TCP data to avoid issues with TCP sequence number wrap
841 * around. A record length of zero bytes has special meaning and means
842 * that resync completed and all data in the database can be
843 * discarded. This function is called after the TCP stack has
844 * re-assembled all TCP fragments due to out of order packet reception
845 * and all TCP sequence numbers should be sequential.
846 *
847 * This function returns true if a so-called TLS RX resync operation
848 * is in progress. Else no such operation is in progress.
849 */
850 static bool
mlx5e_tls_rx_snd_tag_add_tcp_sequence(struct mlx5e_tls_rx_tag * ptag,uint32_t tcp_sn_he,uint32_t len,uint64_t tls_rcd)851 mlx5e_tls_rx_snd_tag_add_tcp_sequence(struct mlx5e_tls_rx_tag *ptag,
852 uint32_t tcp_sn_he, uint32_t len, uint64_t tls_rcd)
853 {
854 uint16_t i, j, n;
855
856 if (ptag->tcp_resync_active == 0 ||
857 ptag->tcp_resync_next != tcp_sn_he ||
858 len == 0) {
859 /* start over again or terminate */
860 ptag->tcp_resync_active = (len != 0);
861 ptag->tcp_resync_len[0] = len;
862 ptag->tcp_resync_num[0] = 1;
863 ptag->tcp_resync_pc = (len != 0);
864 ptag->tcp_resync_cc = 0;
865 ptag->tcp_resync_start = tcp_sn_he;
866 ptag->rcd_resync_start = tls_rcd;
867 } else {
868 i = (ptag->tcp_resync_pc - 1) & (MLX5E_TLS_RX_RESYNC_MAX - 1);
869 n = ptag->tcp_resync_pc - ptag->tcp_resync_cc;
870
871 /* check if same length like last time */
872 if (ptag->tcp_resync_len[i] == len &&
873 ptag->tcp_resync_num[i] != MLX5E_TLS_RX_NUM_MAX) {
874 /* use existing entry */
875 ptag->tcp_resync_num[i]++;
876 } else if (n == MLX5E_TLS_RX_RESYNC_MAX) {
877 j = ptag->tcp_resync_cc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
878 /* adjust starting TCP sequence number */
879 ptag->rcd_resync_start += ptag->tcp_resync_num[j];
880 ptag->tcp_resync_start += ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j];
881 i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
882 /* store new entry */
883 ptag->tcp_resync_len[i] = len;
884 ptag->tcp_resync_num[i] = 1;
885 } else {
886 i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
887 /* add new entry */
888 ptag->tcp_resync_len[i] = len;
889 ptag->tcp_resync_num[i] = 1;
890 }
891 }
892
893 /* store next TCP SN in host endian format */
894 ptag->tcp_resync_next = tcp_sn_he + len;
895
896 return (ptag->tcp_resync_active);
897 }
898
899 /*
900 * This function checks if the given TCP sequence number points to the
901 * beginning of a valid TLS header.
902 *
903 * Returns true if a match is found. Else false.
904 */
905 static bool
mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag * ptag,uint32_t tcp_sn_he,uint32_t * p_next_tcp_sn_he,uint64_t * p_tls_rcd)906 mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *ptag,
907 uint32_t tcp_sn_he, uint32_t *p_next_tcp_sn_he, uint64_t *p_tls_rcd)
908 {
909 uint16_t i, j;
910 uint32_t off = 0;
911 uint32_t rcd = 0;
912 uint32_t delta;
913 uint32_t leap;
914
915 for (i = ptag->tcp_resync_cc; i != ptag->tcp_resync_pc; i++) {
916 delta = tcp_sn_he - off - ptag->tcp_resync_start;
917
918 /* check if subtraction went negative */
919 if ((int32_t)delta < 0)
920 break;
921
922 j = i & (MLX5E_TLS_RX_RESYNC_MAX - 1);
923 leap = ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j];
924 if (delta < leap) {
925 if ((delta % ptag->tcp_resync_len[j]) == 0) {
926 *p_next_tcp_sn_he = tcp_sn_he +
927 ptag->tcp_resync_len[j];
928 *p_tls_rcd = ptag->rcd_resync_start +
929 (uint64_t)rcd +
930 (uint64_t)(delta / ptag->tcp_resync_len[j]);
931 return (true); /* success */
932 }
933 break; /* invalid offset */
934 }
935 rcd += ptag->tcp_resync_num[j];
936 off += leap;
937 }
938 return (false); /* not found */
939 }
940
941 /*
942 * This is a callback function from the network stack to keep track of
943 * TLS RX TCP sequence numbers.
944 *
945 * Returns zero on success else an error happened.
946 */
947 static int
mlx5e_tls_rx_snd_tag_modify(struct m_snd_tag * pmt,union if_snd_tag_modify_params * params)948 mlx5e_tls_rx_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
949 {
950 struct mlx5e_tls_rx_tag *ptag;
951 struct mlx5e_priv *priv;
952 struct mlx5e_iq *iq;
953 int err;
954
955 ptag = container_of(pmt, struct mlx5e_tls_rx_tag, tag);
956 priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx);
957
958 if (unlikely(priv->gone != 0))
959 return (ENXIO);
960
961 iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype);
962
963 MLX5E_TLS_RX_TAG_LOCK(ptag);
964
965 if (mlx5e_tls_rx_snd_tag_add_tcp_sequence(ptag,
966 params->tls_rx.tls_hdr_tcp_sn,
967 params->tls_rx.tls_rec_length,
968 params->tls_rx.tls_seq_number) &&
969 ptag->tcp_resync_pending == 0) {
970 err = mlx5e_tls_rx_receive_progress_parameters(iq, ptag);
971 if (err != 0) {
972 MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_err, 1);
973 } else {
974 ptag->tcp_resync_pending = 1;
975 MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_ok, 1);
976 }
977 } else {
978 err = 0;
979 }
980 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
981
982 return (-err);
983 }
984
985 /*
986 * This function frees a TLS RX tag in a non-blocking way.
987 */
988 static void
mlx5e_tls_rx_snd_tag_free(struct m_snd_tag * pmt)989 mlx5e_tls_rx_snd_tag_free(struct m_snd_tag *pmt)
990 {
991 struct mlx5e_tls_rx_tag *ptag =
992 container_of(pmt, struct mlx5e_tls_rx_tag, tag);
993 struct mlx5e_priv *priv;
994
995 MLX5E_TLS_RX_TAG_LOCK(ptag);
996 ptag->state = MLX5E_TLS_RX_ST_RELEASE;
997 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
998
999 priv = if_getsoftc(ptag->tag.ifp);
1000 queue_work(priv->tls_rx.wq, &ptag->work);
1001 }
1002
1003 #else
1004
1005 int
mlx5e_tls_rx_init(struct mlx5e_priv * priv)1006 mlx5e_tls_rx_init(struct mlx5e_priv *priv)
1007 {
1008
1009 return (0);
1010 }
1011
1012 void
mlx5e_tls_rx_cleanup(struct mlx5e_priv * priv)1013 mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv)
1014 {
1015 /* NOP */
1016 }
1017
1018 #endif
1019