1 /*-
2 * Copyright (c) 2021-2022 NVIDIA corporation & affiliates.
3 *
4 * Redistribution and use in source and binary forms, with or without
5 * modification, are permitted provided that the following conditions
6 * are met:
7 * 1. Redistributions of source code must retain the above copyright
8 * notice, this list of conditions and the following disclaimer.
9 * 2. Redistributions in binary form must reproduce the above copyright
10 * notice, this list of conditions and the following disclaimer in the
11 * documentation and/or other materials provided with the distribution.
12 *
13 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS `AS IS' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 */
25
26 #include "opt_kern_tls.h"
27 #include "opt_rss.h"
28 #include "opt_ratelimit.h"
29
30 #include <dev/mlx5/mlx5_en/en.h>
31
32 #include <dev/mlx5/tls.h>
33
34 #include <dev/mlx5/fs.h>
35 #include <dev/mlx5/mlx5_core/fs_tcp.h>
36
37 #include <sys/ktls.h>
38 #include <opencrypto/cryptodev.h>
39
40 #ifdef KERN_TLS
41
42 static if_snd_tag_free_t mlx5e_tls_rx_snd_tag_free;
43 static if_snd_tag_modify_t mlx5e_tls_rx_snd_tag_modify;
44
45 static const struct if_snd_tag_sw mlx5e_tls_rx_snd_tag_sw = {
46 .snd_tag_modify = mlx5e_tls_rx_snd_tag_modify,
47 .snd_tag_free = mlx5e_tls_rx_snd_tag_free,
48 .type = IF_SND_TAG_TYPE_TLS_RX
49 };
50
51 MALLOC_DEFINE(M_MLX5E_TLS_RX, "MLX5E_TLS_RX", "MLX5 ethernet HW TLS RX");
52
53 /* software TLS RX context */
54 struct mlx5_ifc_sw_tls_rx_cntx_bits {
55 struct mlx5_ifc_tls_static_params_bits param;
56 struct mlx5_ifc_tls_progress_params_bits progress;
57 struct {
58 uint8_t key_data[8][0x20];
59 uint8_t key_len[0x20];
60 } key;
61 };
62
63 CTASSERT(MLX5_ST_SZ_BYTES(sw_tls_rx_cntx) <= sizeof(((struct mlx5e_tls_rx_tag *)NULL)->crypto_params));
64 CTASSERT(MLX5_ST_SZ_BYTES(mkc) == sizeof(((struct mlx5e_tx_umr_wqe *)NULL)->mkc));
65
66 static const char *mlx5e_tls_rx_stats_desc[] = {
67 MLX5E_TLS_RX_STATS(MLX5E_STATS_DESC)
68 };
69
70 static void mlx5e_tls_rx_work(struct work_struct *);
71 static bool mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *,
72 uint32_t, uint32_t *, uint64_t *);
73
74 CTASSERT((MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) % 16) == 0);
75
76 static uint32_t
mlx5e_tls_rx_get_ch(struct mlx5e_priv * priv,uint32_t flowid,uint32_t flowtype)77 mlx5e_tls_rx_get_ch(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype)
78 {
79 u32 ch;
80 #ifdef RSS
81 u32 temp;
82 #endif
83
84 /* keep this code synced with mlx5e_select_queue() */
85 ch = priv->params.num_channels;
86 #ifdef RSS
87 if (rss_hash2bucket(flowid, flowtype, &temp) == 0)
88 ch = temp % ch;
89 else
90 #endif
91 ch = (flowid % 128) % ch;
92 return (ch);
93 }
94
95 /*
96 * This function gets a pointer to an internal queue, IQ, based on the
97 * provided "flowid" and "flowtype". The IQ returned may in some rare
98 * cases not be activated or running, but this is all handled by the
99 * "mlx5e_iq_get_producer_index()" function.
100 *
101 * The idea behind this function is to spread the IQ traffic as much
102 * as possible and to avoid congestion on the same IQ when processing
103 * RX traffic.
104 */
105 static struct mlx5e_iq *
mlx5e_tls_rx_get_iq(struct mlx5e_priv * priv,uint32_t flowid,uint32_t flowtype)106 mlx5e_tls_rx_get_iq(struct mlx5e_priv *priv, uint32_t flowid, uint32_t flowtype)
107 {
108 /*
109 * NOTE: The channels array is only freed at detach
110 * and it safe to return a pointer to the send tag
111 * inside the channels structure as long as we
112 * reference the priv.
113 */
114 return (&priv->channel[mlx5e_tls_rx_get_ch(priv, flowid, flowtype)].iq);
115 }
116
117 static void
mlx5e_tls_rx_send_static_parameters_cb(void * arg)118 mlx5e_tls_rx_send_static_parameters_cb(void *arg)
119 {
120 struct mlx5e_tls_rx_tag *ptag;
121
122 ptag = (struct mlx5e_tls_rx_tag *)arg;
123
124 m_snd_tag_rele(&ptag->tag);
125 }
126
127 /*
128 * This function sends the so-called TLS RX static parameters to the
129 * hardware. These parameters are temporarily stored in the
130 * "crypto_params" field of the TLS RX tag. Most importantly this
131 * function sets the TCP sequence number (32-bit) and TLS record
132 * number (64-bit) where the decryption can resume.
133 *
134 * Zero is returned upon success. Else some error happend.
135 */
136 static int
mlx5e_tls_rx_send_static_parameters(struct mlx5e_iq * iq,struct mlx5e_tls_rx_tag * ptag)137 mlx5e_tls_rx_send_static_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag)
138 {
139 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_umr_wqe) +
140 MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param), MLX5_SEND_WQE_DS);
141 struct mlx5e_tx_umr_wqe *wqe;
142 int pi;
143
144 mtx_lock(&iq->lock);
145 pi = mlx5e_iq_get_producer_index(iq);
146 if (pi < 0) {
147 mtx_unlock(&iq->lock);
148 return (-ENOMEM);
149 }
150 wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
151
152 memset(wqe, 0, sizeof(*wqe));
153
154 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
155 MLX5_OPCODE_UMR | (MLX5_OPCODE_MOD_UMR_TLS_TIR_STATIC_PARAMS << 24));
156 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
157 wqe->ctrl.imm = cpu_to_be32(ptag->tirn << 8);
158 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE | MLX5_FENCE_MODE_INITIATOR_SMALL;
159
160 /* fill out UMR control segment */
161 wqe->umr.flags = 0x80; /* inline data */
162 wqe->umr.bsf_octowords =
163 cpu_to_be16(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param) / 16);
164
165 /* copy in the static crypto parameters */
166 memcpy(wqe + 1, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, param),
167 MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param));
168
169 /* copy data for doorbell */
170 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
171
172 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
173 iq->data[pi].callback = &mlx5e_tls_rx_send_static_parameters_cb;
174 iq->data[pi].arg = ptag;
175
176 m_snd_tag_ref(&ptag->tag);
177
178 iq->pc += iq->data[pi].num_wqebbs;
179
180 mlx5e_iq_notify_hw(iq);
181
182 mtx_unlock(&iq->lock);
183
184 return (0); /* success */
185 }
186
187 static void
mlx5e_tls_rx_send_progress_parameters_cb(void * arg)188 mlx5e_tls_rx_send_progress_parameters_cb(void *arg)
189 {
190 struct mlx5e_tls_rx_tag *ptag;
191
192 ptag = (struct mlx5e_tls_rx_tag *)arg;
193
194 complete(&ptag->progress_complete);
195 }
196
197 CTASSERT(MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, progress) ==
198 sizeof(((struct mlx5e_tx_psv_wqe *)NULL)->psv));
199
200 /*
201 * This function resets the state of the TIR context to start
202 * searching for a valid TLS header and is used only when allocating
203 * the TLS RX tag.
204 *
205 * Zero is returned upon success, else some error happened.
206 */
207 static int
mlx5e_tls_rx_send_progress_parameters_sync(struct mlx5e_iq * iq,struct mlx5e_tls_rx_tag * ptag)208 mlx5e_tls_rx_send_progress_parameters_sync(struct mlx5e_iq *iq,
209 struct mlx5e_tls_rx_tag *ptag)
210 {
211 const u32 ds_cnt = DIV_ROUND_UP(sizeof(struct mlx5e_tx_psv_wqe),
212 MLX5_SEND_WQE_DS);
213 struct mlx5e_priv *priv;
214 struct mlx5e_tx_psv_wqe *wqe;
215 int pi;
216
217 mtx_lock(&iq->lock);
218 pi = mlx5e_iq_get_producer_index(iq);
219 if (pi < 0) {
220 mtx_unlock(&iq->lock);
221 return (-ENOMEM);
222 }
223 wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
224
225 memset(wqe, 0, sizeof(*wqe));
226
227 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
228 MLX5_OPCODE_SET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24));
229 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
230 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
231
232 /* copy in the PSV control segment */
233 memcpy(&wqe->psv, MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, progress),
234 sizeof(wqe->psv));
235
236 /* copy data for doorbell */
237 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
238
239 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
240 iq->data[pi].callback = &mlx5e_tls_rx_send_progress_parameters_cb;
241 iq->data[pi].arg = ptag;
242
243 iq->pc += iq->data[pi].num_wqebbs;
244
245 init_completion(&ptag->progress_complete);
246
247 mlx5e_iq_notify_hw(iq);
248
249 mtx_unlock(&iq->lock);
250
251 while (1) {
252 if (wait_for_completion_timeout(&ptag->progress_complete, hz) != 0)
253 break;
254 priv = container_of(iq, struct mlx5e_channel, iq)->priv;
255 if (priv->mdev->state == MLX5_DEVICE_STATE_INTERNAL_ERROR ||
256 pci_channel_offline(priv->mdev->pdev) != 0)
257 return (-EWOULDBLOCK);
258 }
259
260 return (0); /* success */
261 }
262
263 CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE >= MLX5_ST_SZ_BYTES(tls_progress_params));
264 CTASSERT(MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE <= PAGE_SIZE);
265
266 struct mlx5e_get_tls_progress_params_wqe {
267 struct mlx5_wqe_ctrl_seg ctrl;
268 struct mlx5_seg_get_psv psv;
269 };
270
271 static void
mlx5e_tls_rx_receive_progress_parameters_cb(void * arg)272 mlx5e_tls_rx_receive_progress_parameters_cb(void *arg)
273 {
274 struct mlx5e_tls_rx_tag *ptag;
275 struct mlx5e_iq *iq;
276 uint32_t tcp_curr_sn_he;
277 uint32_t tcp_next_sn_he;
278 uint64_t tls_rcd_num;
279 void *buffer;
280
281 ptag = (struct mlx5e_tls_rx_tag *)arg;
282 buffer = mlx5e_tls_rx_get_progress_buffer(ptag);
283
284 MLX5E_TLS_RX_TAG_LOCK(ptag);
285
286 ptag->tcp_resync_pending = 0;
287
288 switch (MLX5_GET(tls_progress_params, buffer, record_tracker_state)) {
289 case MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_TRACKING:
290 break;
291 default:
292 goto done;
293 }
294
295 switch (MLX5_GET(tls_progress_params, buffer, auth_state)) {
296 case MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD:
297 break;
298 default:
299 goto done;
300 }
301
302 tcp_curr_sn_he = MLX5_GET(tls_progress_params, buffer, hw_resync_tcp_sn);
303
304 if (mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(ptag, tcp_curr_sn_he,
305 &tcp_next_sn_he, &tls_rcd_num)) {
306
307 MLX5_SET64(sw_tls_rx_cntx, ptag->crypto_params,
308 param.initial_record_number, tls_rcd_num);
309 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params,
310 param.resync_tcp_sn, tcp_curr_sn_he);
311
312 iq = mlx5e_tls_rx_get_iq(
313 container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx),
314 ptag->flowid, ptag->flowtype);
315
316 if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0)
317 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
318 }
319 done:
320 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
321
322 m_snd_tag_rele(&ptag->tag);
323 }
324
325 /*
326 * This function queries the hardware for the current state of the TIR
327 * in question. It is typically called when encrypted data is received
328 * to re-establish hardware decryption of received TLS data.
329 *
330 * Zero is returned upon success, else some error happened.
331 */
332 static int
mlx5e_tls_rx_receive_progress_parameters(struct mlx5e_iq * iq,struct mlx5e_tls_rx_tag * ptag)333 mlx5e_tls_rx_receive_progress_parameters(struct mlx5e_iq *iq, struct mlx5e_tls_rx_tag *ptag)
334 {
335 struct mlx5e_get_tls_progress_params_wqe *wqe;
336 const u32 ds_cnt = DIV_ROUND_UP(sizeof(*wqe), MLX5_SEND_WQE_DS);
337 u64 dma_address;
338 int pi;
339
340 mtx_lock(&iq->lock);
341 pi = mlx5e_iq_get_producer_index(iq);
342 if (pi < 0) {
343 mtx_unlock(&iq->lock);
344 return (-ENOMEM);
345 }
346
347 mlx5e_iq_load_memory_single(iq, pi,
348 mlx5e_tls_rx_get_progress_buffer(ptag),
349 MLX5E_TLS_RX_PROGRESS_BUFFER_SIZE,
350 &dma_address, BUS_DMASYNC_PREREAD);
351
352 wqe = mlx5_wq_cyc_get_wqe(&iq->wq, pi);
353
354 memset(wqe, 0, sizeof(*wqe));
355
356 wqe->ctrl.opmod_idx_opcode = cpu_to_be32((iq->pc << 8) |
357 MLX5_OPCODE_GET_PSV | (MLX5_OPCODE_MOD_PSV_TLS_TIR_PROGRESS_PARAMS << 24));
358 wqe->ctrl.qpn_ds = cpu_to_be32((iq->sqn << 8) | ds_cnt);
359 wqe->ctrl.fm_ce_se = MLX5_WQE_CTRL_CQ_UPDATE;
360 wqe->psv.num_psv = 1 << 4;
361 wqe->psv.l_key = iq->mkey_be;
362 wqe->psv.psv_index[0] = cpu_to_be32(ptag->tirn);
363 wqe->psv.va = cpu_to_be64(dma_address);
364
365 /* copy data for doorbell */
366 memcpy(iq->doorbell.d32, &wqe->ctrl, sizeof(iq->doorbell.d32));
367
368 iq->data[pi].num_wqebbs = DIV_ROUND_UP(ds_cnt, MLX5_SEND_WQEBB_NUM_DS);
369 iq->data[pi].callback = &mlx5e_tls_rx_receive_progress_parameters_cb;
370 iq->data[pi].arg = ptag;
371
372 m_snd_tag_ref(&ptag->tag);
373
374 iq->pc += iq->data[pi].num_wqebbs;
375
376 mlx5e_iq_notify_hw(iq);
377
378 mtx_unlock(&iq->lock);
379
380 return (0); /* success */
381 }
382
383 /*
384 * This is the import function for TLS RX tags.
385 */
386 static int
mlx5e_tls_rx_tag_import(void * arg,void ** store,int cnt,int domain,int flags)387 mlx5e_tls_rx_tag_import(void *arg, void **store, int cnt, int domain, int flags)
388 {
389 struct mlx5e_tls_rx_tag *ptag;
390 int i;
391
392 for (i = 0; i != cnt; i++) {
393 ptag = malloc_domainset(sizeof(*ptag), M_MLX5E_TLS_RX,
394 mlx5_dev_domainset(arg), flags | M_ZERO);
395 mtx_init(&ptag->mtx, "mlx5-tls-rx-tag-mtx", NULL, MTX_DEF);
396 INIT_WORK(&ptag->work, mlx5e_tls_rx_work);
397 store[i] = ptag;
398 }
399 return (i);
400 }
401
402 /*
403 * This is the release function for TLS RX tags.
404 */
405 static void
mlx5e_tls_rx_tag_release(void * arg,void ** store,int cnt)406 mlx5e_tls_rx_tag_release(void *arg, void **store, int cnt)
407 {
408 struct mlx5e_tls_rx_tag *ptag;
409 int i;
410
411 for (i = 0; i != cnt; i++) {
412 ptag = store[i];
413
414 flush_work(&ptag->work);
415 mtx_destroy(&ptag->mtx);
416 free(ptag, M_MLX5E_TLS_RX);
417 }
418 }
419
420 /*
421 * This is a convenience function to free TLS RX tags. It resets some
422 * selected fields, updates the number of resources and returns the
423 * TLS RX tag to the UMA pool of free tags.
424 */
425 static void
mlx5e_tls_rx_tag_zfree(struct mlx5e_tls_rx_tag * ptag)426 mlx5e_tls_rx_tag_zfree(struct mlx5e_tls_rx_tag *ptag)
427 {
428 /* make sure any unhandled taskqueue events are ignored */
429 ptag->state = MLX5E_TLS_RX_ST_FREED;
430
431 /* reset some variables */
432 ptag->dek_index = 0;
433 ptag->dek_index_ok = 0;
434 ptag->tirn = 0;
435 ptag->flow_rule = NULL;
436 ptag->tcp_resync_active = 0;
437 ptag->tcp_resync_pending = 0;
438
439 /* avoid leaking keys */
440 memset(ptag->crypto_params, 0, sizeof(ptag->crypto_params));
441
442 /* update number of resources in use */
443 atomic_add_32(&ptag->tls_rx->num_resources, -1U);
444
445 /* return tag to UMA */
446 uma_zfree(ptag->tls_rx->zone, ptag);
447 }
448
449 /*
450 * This function enables TLS RX support for the given NIC, if all
451 * needed firmware capabilites are present.
452 */
453 int
mlx5e_tls_rx_init(struct mlx5e_priv * priv)454 mlx5e_tls_rx_init(struct mlx5e_priv *priv)
455 {
456 struct mlx5e_tls_rx *ptls = &priv->tls_rx;
457 struct sysctl_oid *node;
458 uint32_t x;
459
460 if (MLX5_CAP_GEN(priv->mdev, tls_rx) == 0 ||
461 MLX5_CAP_GEN(priv->mdev, log_max_dek) == 0 ||
462 MLX5_CAP_FLOWTABLE_NIC_RX(priv->mdev, ft_field_support.outer_ip_version) == 0)
463 return (0);
464
465 ptls->wq = create_singlethread_workqueue("mlx5-tls-rx-wq");
466 if (ptls->wq == NULL)
467 return (ENOMEM);
468
469 sysctl_ctx_init(&ptls->ctx);
470
471 snprintf(ptls->zname, sizeof(ptls->zname),
472 "mlx5_%u_tls_rx", device_get_unit(priv->mdev->pdev->dev.bsddev));
473
474 ptls->zone = uma_zcache_create(ptls->zname,
475 sizeof(struct mlx5e_tls_rx_tag), NULL, NULL, NULL, NULL,
476 mlx5e_tls_rx_tag_import, mlx5e_tls_rx_tag_release, priv->mdev,
477 UMA_ZONE_UNMANAGED);
478
479 /* shared between RX and TX TLS */
480 ptls->max_resources = 1U << (MLX5_CAP_GEN(priv->mdev, log_max_dek) - 1);
481
482 for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++)
483 ptls->stats.arg[x] = counter_u64_alloc(M_WAITOK);
484
485 ptls->init = 1;
486
487 node = SYSCTL_ADD_NODE(&priv->sysctl_ctx,
488 SYSCTL_CHILDREN(priv->sysctl_ifnet), OID_AUTO,
489 "tls_rx", CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, "Hardware TLS receive offload");
490 if (node == NULL)
491 return (0);
492
493 mlx5e_create_counter_stats(&ptls->ctx,
494 SYSCTL_CHILDREN(node), "stats",
495 mlx5e_tls_rx_stats_desc, MLX5E_TLS_RX_STATS_NUM,
496 ptls->stats.arg);
497
498 return (0);
499 }
500
501 /*
502 * This function disables TLS RX support for the given NIC.
503 */
504 void
mlx5e_tls_rx_cleanup(struct mlx5e_priv * priv)505 mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv)
506 {
507 struct mlx5e_tls_rx *ptls = &priv->tls_rx;
508 uint32_t x;
509
510 if (ptls->init == 0)
511 return;
512
513 ptls->init = 0;
514 flush_workqueue(ptls->wq);
515 sysctl_ctx_free(&ptls->ctx);
516 uma_zdestroy(ptls->zone);
517 destroy_workqueue(ptls->wq);
518
519 /* check if all resources are freed */
520 MPASS(priv->tls_rx.num_resources == 0);
521
522 for (x = 0; x != MLX5E_TLS_RX_STATS_NUM; x++)
523 counter_u64_free(ptls->stats.arg[x]);
524 }
525
526 /*
527 * This function is used to serialize sleeping firmware operations
528 * needed in order to establish and destroy a TLS RX tag.
529 */
530 static void
mlx5e_tls_rx_work(struct work_struct * work)531 mlx5e_tls_rx_work(struct work_struct *work)
532 {
533 struct mlx5e_tls_rx_tag *ptag;
534 struct mlx5e_priv *priv;
535 int err;
536
537 ptag = container_of(work, struct mlx5e_tls_rx_tag, work);
538 priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx);
539
540 switch (ptag->state) {
541 case MLX5E_TLS_RX_ST_INIT:
542 /* try to allocate new TIR context */
543 err = mlx5_tls_open_tir(priv->mdev, priv->tdn,
544 priv->channel[mlx5e_tls_rx_get_ch(priv, ptag->flowid, ptag->flowtype)].rqtn,
545 &ptag->tirn);
546 if (err) {
547 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
548 break;
549 }
550 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.pd, ptag->tirn);
551
552 /* try to allocate a DEK context ID */
553 err = mlx5_encryption_key_create(priv->mdev, priv->pdn,
554 MLX5_ADDR_OF(sw_tls_rx_cntx, ptag->crypto_params, key.key_data),
555 MLX5_GET(sw_tls_rx_cntx, ptag->crypto_params, key.key_len),
556 &ptag->dek_index);
557 if (err) {
558 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
559 break;
560 }
561
562 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, param.dek_index, ptag->dek_index);
563
564 ptag->dek_index_ok = 1;
565
566 MLX5E_TLS_RX_TAG_LOCK(ptag);
567 if (ptag->state == MLX5E_TLS_RX_ST_INIT)
568 ptag->state = MLX5E_TLS_RX_ST_SETUP;
569 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
570 break;
571
572 case MLX5E_TLS_RX_ST_RELEASE:
573 /* remove flow rule for incoming traffic, if any */
574 if (ptag->flow_rule != NULL)
575 mlx5e_accel_fs_del_inpcb(ptag->flow_rule);
576
577 /* try to destroy DEK context by ID */
578 if (ptag->dek_index_ok)
579 mlx5_encryption_key_destroy(priv->mdev, ptag->dek_index);
580
581 /* try to destroy TIR context by ID */
582 if (ptag->tirn != 0)
583 mlx5_tls_close_tir(priv->mdev, ptag->tirn);
584
585 /* free tag */
586 mlx5e_tls_rx_tag_zfree(ptag);
587 break;
588
589 default:
590 break;
591 }
592 }
593
594 /*
595 * This function translates the crypto parameters into the format used
596 * by the firmware and hardware. Currently only AES-128 and AES-256 is
597 * supported for TLS v1.2 and TLS v1.3.
598 *
599 * Returns zero on success, else an error happened.
600 */
601 static int
mlx5e_tls_rx_set_params(void * ctx,struct inpcb * inp,const struct tls_session_params * en)602 mlx5e_tls_rx_set_params(void *ctx, struct inpcb *inp, const struct tls_session_params *en)
603 {
604 uint32_t tcp_sn_he;
605 uint64_t tls_sn_he;
606
607 MLX5_SET(sw_tls_rx_cntx, ctx, param.const_2, 2);
608 if (en->tls_vminor == TLS_MINOR_VER_TWO)
609 MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 2); /* v1.2 */
610 else
611 MLX5_SET(sw_tls_rx_cntx, ctx, param.tls_version, 3); /* v1.3 */
612 MLX5_SET(sw_tls_rx_cntx, ctx, param.const_1, 1);
613 MLX5_SET(sw_tls_rx_cntx, ctx, param.encryption_standard, 1); /* TLS */
614
615 /* copy the initial vector in place */
616 switch (en->iv_len) {
617 case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv):
618 case MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.gcm_iv) +
619 MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, param.implicit_iv):
620 memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, param.gcm_iv),
621 en->iv, en->iv_len);
622 break;
623 default:
624 return (EINVAL);
625 }
626
627 if (en->cipher_key_len <= MLX5_FLD_SZ_BYTES(sw_tls_rx_cntx, key.key_data)) {
628 memcpy(MLX5_ADDR_OF(sw_tls_rx_cntx, ctx, key.key_data),
629 en->cipher_key, en->cipher_key_len);
630 MLX5_SET(sw_tls_rx_cntx, ctx, key.key_len, en->cipher_key_len);
631 } else {
632 return (EINVAL);
633 }
634
635 if (__predict_false(inp == NULL ||
636 ktls_get_rx_sequence(inp, &tcp_sn_he, &tls_sn_he) != 0))
637 return (EINVAL);
638
639 MLX5_SET64(sw_tls_rx_cntx, ctx, param.initial_record_number, tls_sn_he);
640 MLX5_SET(sw_tls_rx_cntx, ctx, param.resync_tcp_sn, tcp_sn_he);
641
642 return (0);
643 }
644
645 /* Verify zero default */
646 CTASSERT(MLX5E_TLS_RX_ST_INIT == 0);
647
648 /*
649 * This function is responsible for allocating a TLS RX tag. It is a
650 * callback function invoked by the network stack.
651 *
652 * Returns zero on success else an error happened.
653 */
654 int
mlx5e_tls_rx_snd_tag_alloc(if_t ifp,union if_snd_tag_alloc_params * params,struct m_snd_tag ** ppmt)655 mlx5e_tls_rx_snd_tag_alloc(if_t ifp,
656 union if_snd_tag_alloc_params *params,
657 struct m_snd_tag **ppmt)
658 {
659 struct mlx5e_iq *iq;
660 struct mlx5e_priv *priv;
661 struct mlx5e_tls_rx_tag *ptag;
662 struct mlx5_flow_rule *flow_rule;
663 const struct tls_session_params *en;
664 uint32_t value;
665 int error;
666
667 priv = if_getsoftc(ifp);
668
669 if (unlikely(priv->gone != 0 || priv->tls_rx.init == 0 ||
670 params->hdr.flowtype == M_HASHTYPE_NONE))
671 return (EOPNOTSUPP);
672
673 /* allocate new tag from zone, if any */
674 ptag = uma_zalloc(priv->tls_rx.zone, M_NOWAIT);
675 if (ptag == NULL)
676 return (ENOMEM);
677
678 /* sanity check default values */
679 MPASS(ptag->dek_index == 0);
680 MPASS(ptag->dek_index_ok == 0);
681
682 /* setup TLS RX tag */
683 ptag->tls_rx = &priv->tls_rx;
684 ptag->flowtype = params->hdr.flowtype;
685 ptag->flowid = params->hdr.flowid;
686
687 value = atomic_fetchadd_32(&priv->tls_rx.num_resources, 1U);
688
689 /* check resource limits */
690 if (value >= priv->tls_rx.max_resources) {
691 error = ENOMEM;
692 goto failure;
693 }
694
695 en = ¶ms->tls_rx.tls->params;
696
697 /* only TLS v1.2 and v1.3 is currently supported */
698 if (en->tls_vmajor != TLS_MAJOR_VER_ONE ||
699 (en->tls_vminor != TLS_MINOR_VER_TWO
700 #ifdef TLS_MINOR_VER_THREE
701 && en->tls_vminor != TLS_MINOR_VER_THREE
702 #endif
703 )) {
704 error = EPROTONOSUPPORT;
705 goto failure;
706 }
707
708 switch (en->cipher_algorithm) {
709 case CRYPTO_AES_NIST_GCM_16:
710 switch (en->cipher_key_len) {
711 case 128 / 8:
712 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
713 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_128) == 0) {
714 error = EPROTONOSUPPORT;
715 goto failure;
716 }
717 } else {
718 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_128) == 0) {
719 error = EPROTONOSUPPORT;
720 goto failure;
721 }
722 }
723 error = mlx5e_tls_rx_set_params(
724 ptag->crypto_params, params->tls_rx.inp, en);
725 if (error)
726 goto failure;
727 break;
728
729 case 256 / 8:
730 if (en->tls_vminor == TLS_MINOR_VER_TWO) {
731 if (MLX5_CAP_TLS(priv->mdev, tls_1_2_aes_gcm_256) == 0) {
732 error = EPROTONOSUPPORT;
733 goto failure;
734 }
735 } else {
736 if (MLX5_CAP_TLS(priv->mdev, tls_1_3_aes_gcm_256) == 0) {
737 error = EPROTONOSUPPORT;
738 goto failure;
739 }
740 }
741 error = mlx5e_tls_rx_set_params(
742 ptag->crypto_params, params->tls_rx.inp, en);
743 if (error)
744 goto failure;
745 break;
746
747 default:
748 error = EINVAL;
749 goto failure;
750 }
751 break;
752 default:
753 error = EPROTONOSUPPORT;
754 goto failure;
755 }
756
757 /* store pointer to mbuf tag */
758 MPASS(ptag->tag.refcount == 0);
759 m_snd_tag_init(&ptag->tag, ifp, &mlx5e_tls_rx_snd_tag_sw);
760 *ppmt = &ptag->tag;
761
762 /* reset state */
763 ptag->state = MLX5E_TLS_RX_ST_INIT;
764
765 queue_work(priv->tls_rx.wq, &ptag->work);
766 flush_work(&ptag->work);
767
768 /* check that worker task completed successfully */
769 MLX5E_TLS_RX_TAG_LOCK(ptag);
770 if (ptag->state == MLX5E_TLS_RX_ST_SETUP) {
771 ptag->state = MLX5E_TLS_RX_ST_READY;
772 error = 0;
773 } else {
774 error = ENOMEM;
775 }
776 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
777
778 if (unlikely(error))
779 goto cleanup;
780
781 iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype);
782
783 /* establish connection between DEK and TIR */
784 if (mlx5e_tls_rx_send_static_parameters(iq, ptag) != 0) {
785 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
786 error = ENOMEM;
787 goto cleanup;
788 }
789
790 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.auth_state,
791 MLX5E_TLS_RX_PROGRESS_PARAMS_AUTH_STATE_NO_OFFLOAD);
792 MLX5_SET(sw_tls_rx_cntx, ptag->crypto_params, progress.record_tracker_state,
793 MLX5E_TLS_RX_PROGRESS_PARAMS_RECORD_TRACKER_STATE_START);
794
795 /* reset state to all zeros */
796 if (mlx5e_tls_rx_send_progress_parameters_sync(iq, ptag) != 0) {
797 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
798 error = ENOMEM;
799 goto cleanup;
800 }
801
802 if (if_getpcp(ifp) != IFNET_PCP_NONE || params->tls_rx.vlan_id != 0) {
803 /* create flow rule for TLS RX traffic (tagged) */
804 flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp,
805 ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, params->tls_rx.vlan_id);
806 } else {
807 /* create flow rule for TLS RX traffic (untagged) */
808 flow_rule = mlx5e_accel_fs_add_inpcb(priv, params->tls_rx.inp,
809 ptag->tirn, MLX5_FS_DEFAULT_FLOW_TAG, MLX5E_ACCEL_FS_ADD_INPCB_NO_VLAN);
810 }
811
812 if (IS_ERR_OR_NULL(flow_rule)) {
813 MLX5E_TLS_RX_STAT_INC(ptag, rx_error, 1);
814 error = ENOMEM;
815 goto cleanup;
816 }
817
818 ptag->flow_rule = flow_rule;
819
820 return (0);
821
822 cleanup:
823 m_snd_tag_rele(&ptag->tag);
824 return (error);
825
826 failure:
827 mlx5e_tls_rx_tag_zfree(ptag);
828 return (error);
829 }
830
831
832 /*
833 * This function adds the TCP sequence number and TLS record number in
834 * host endian format to a small database. When TLS records have the
835 * same length, they are simply accumulated by counting instead of
836 * separated entries in the TLS database. The dimension of the
837 * database is such that it cannot store more than 1GByte of
838 * continuous TCP data to avoid issues with TCP sequence number wrap
839 * around. A record length of zero bytes has special meaning and means
840 * that resync completed and all data in the database can be
841 * discarded. This function is called after the TCP stack has
842 * re-assembled all TCP fragments due to out of order packet reception
843 * and all TCP sequence numbers should be sequential.
844 *
845 * This function returns true if a so-called TLS RX resync operation
846 * is in progress. Else no such operation is in progress.
847 */
848 static bool
mlx5e_tls_rx_snd_tag_add_tcp_sequence(struct mlx5e_tls_rx_tag * ptag,uint32_t tcp_sn_he,uint32_t len,uint64_t tls_rcd)849 mlx5e_tls_rx_snd_tag_add_tcp_sequence(struct mlx5e_tls_rx_tag *ptag,
850 uint32_t tcp_sn_he, uint32_t len, uint64_t tls_rcd)
851 {
852 uint16_t i, j, n;
853
854 if (ptag->tcp_resync_active == 0 ||
855 ptag->tcp_resync_next != tcp_sn_he ||
856 len == 0) {
857 /* start over again or terminate */
858 ptag->tcp_resync_active = (len != 0);
859 ptag->tcp_resync_len[0] = len;
860 ptag->tcp_resync_num[0] = 1;
861 ptag->tcp_resync_pc = (len != 0);
862 ptag->tcp_resync_cc = 0;
863 ptag->tcp_resync_start = tcp_sn_he;
864 ptag->rcd_resync_start = tls_rcd;
865 } else {
866 i = (ptag->tcp_resync_pc - 1) & (MLX5E_TLS_RX_RESYNC_MAX - 1);
867 n = ptag->tcp_resync_pc - ptag->tcp_resync_cc;
868
869 /* check if same length like last time */
870 if (ptag->tcp_resync_len[i] == len &&
871 ptag->tcp_resync_num[i] != MLX5E_TLS_RX_NUM_MAX) {
872 /* use existing entry */
873 ptag->tcp_resync_num[i]++;
874 } else if (n == MLX5E_TLS_RX_RESYNC_MAX) {
875 j = ptag->tcp_resync_cc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
876 /* adjust starting TCP sequence number */
877 ptag->rcd_resync_start += ptag->tcp_resync_num[j];
878 ptag->tcp_resync_start += ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j];
879 i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
880 /* store new entry */
881 ptag->tcp_resync_len[i] = len;
882 ptag->tcp_resync_num[i] = 1;
883 } else {
884 i = ptag->tcp_resync_pc++ & (MLX5E_TLS_RX_RESYNC_MAX - 1);
885 /* add new entry */
886 ptag->tcp_resync_len[i] = len;
887 ptag->tcp_resync_num[i] = 1;
888 }
889 }
890
891 /* store next TCP SN in host endian format */
892 ptag->tcp_resync_next = tcp_sn_he + len;
893
894 return (ptag->tcp_resync_active);
895 }
896
897 /*
898 * This function checks if the given TCP sequence number points to the
899 * beginning of a valid TLS header.
900 *
901 * Returns true if a match is found. Else false.
902 */
903 static bool
mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag * ptag,uint32_t tcp_sn_he,uint32_t * p_next_tcp_sn_he,uint64_t * p_tls_rcd)904 mlx5e_tls_rx_snd_tag_find_tcp_sn_and_tls_rcd(struct mlx5e_tls_rx_tag *ptag,
905 uint32_t tcp_sn_he, uint32_t *p_next_tcp_sn_he, uint64_t *p_tls_rcd)
906 {
907 uint16_t i, j;
908 uint32_t off = 0;
909 uint32_t rcd = 0;
910 uint32_t delta;
911 uint32_t leap;
912
913 for (i = ptag->tcp_resync_cc; i != ptag->tcp_resync_pc; i++) {
914 delta = tcp_sn_he - off - ptag->tcp_resync_start;
915
916 /* check if subtraction went negative */
917 if ((int32_t)delta < 0)
918 break;
919
920 j = i & (MLX5E_TLS_RX_RESYNC_MAX - 1);
921 leap = ptag->tcp_resync_len[j] * ptag->tcp_resync_num[j];
922 if (delta < leap) {
923 if ((delta % ptag->tcp_resync_len[j]) == 0) {
924 *p_next_tcp_sn_he = tcp_sn_he +
925 ptag->tcp_resync_len[j];
926 *p_tls_rcd = ptag->rcd_resync_start +
927 (uint64_t)rcd +
928 (uint64_t)(delta / ptag->tcp_resync_len[j]);
929 return (true); /* success */
930 }
931 break; /* invalid offset */
932 }
933 rcd += ptag->tcp_resync_num[j];
934 off += leap;
935 }
936 return (false); /* not found */
937 }
938
939 /*
940 * This is a callback function from the network stack to keep track of
941 * TLS RX TCP sequence numbers.
942 *
943 * Returns zero on success else an error happened.
944 */
945 static int
mlx5e_tls_rx_snd_tag_modify(struct m_snd_tag * pmt,union if_snd_tag_modify_params * params)946 mlx5e_tls_rx_snd_tag_modify(struct m_snd_tag *pmt, union if_snd_tag_modify_params *params)
947 {
948 struct mlx5e_tls_rx_tag *ptag;
949 struct mlx5e_priv *priv;
950 struct mlx5e_iq *iq;
951 int err;
952
953 ptag = container_of(pmt, struct mlx5e_tls_rx_tag, tag);
954 priv = container_of(ptag->tls_rx, struct mlx5e_priv, tls_rx);
955
956 if (unlikely(priv->gone != 0))
957 return (ENXIO);
958
959 iq = mlx5e_tls_rx_get_iq(priv, ptag->flowid, ptag->flowtype);
960
961 MLX5E_TLS_RX_TAG_LOCK(ptag);
962
963 if (mlx5e_tls_rx_snd_tag_add_tcp_sequence(ptag,
964 params->tls_rx.tls_hdr_tcp_sn,
965 params->tls_rx.tls_rec_length,
966 params->tls_rx.tls_seq_number) &&
967 ptag->tcp_resync_pending == 0) {
968 err = mlx5e_tls_rx_receive_progress_parameters(iq, ptag);
969 if (err != 0) {
970 MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_err, 1);
971 } else {
972 ptag->tcp_resync_pending = 1;
973 MLX5E_TLS_RX_STAT_INC(ptag, rx_resync_ok, 1);
974 }
975 } else {
976 err = 0;
977 }
978 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
979
980 return (-err);
981 }
982
983 /*
984 * This function frees a TLS RX tag in a non-blocking way.
985 */
986 static void
mlx5e_tls_rx_snd_tag_free(struct m_snd_tag * pmt)987 mlx5e_tls_rx_snd_tag_free(struct m_snd_tag *pmt)
988 {
989 struct mlx5e_tls_rx_tag *ptag =
990 container_of(pmt, struct mlx5e_tls_rx_tag, tag);
991 struct mlx5e_priv *priv;
992
993 MLX5E_TLS_RX_TAG_LOCK(ptag);
994 ptag->state = MLX5E_TLS_RX_ST_RELEASE;
995 MLX5E_TLS_RX_TAG_UNLOCK(ptag);
996
997 priv = if_getsoftc(ptag->tag.ifp);
998 queue_work(priv->tls_rx.wq, &ptag->work);
999 }
1000
1001 #else
1002
1003 int
mlx5e_tls_rx_init(struct mlx5e_priv * priv)1004 mlx5e_tls_rx_init(struct mlx5e_priv *priv)
1005 {
1006
1007 return (0);
1008 }
1009
1010 void
mlx5e_tls_rx_cleanup(struct mlx5e_priv * priv)1011 mlx5e_tls_rx_cleanup(struct mlx5e_priv *priv)
1012 {
1013 /* NOP */
1014 }
1015
1016 #endif
1017