1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Copyright (C) 2020 Chelsio Communications. All rights reserved. */
3
4 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
5
6 #include <linux/skbuff.h>
7 #include <linux/module.h>
8 #include <linux/highmem.h>
9 #include <linux/ip.h>
10 #include <net/ipv6.h>
11 #include <linux/netdevice.h>
12 #include <crypto/aes.h>
13 #include "chcr_ktls.h"
14
15 static LIST_HEAD(uld_ctx_list);
16 static DEFINE_MUTEX(dev_mutex);
17
18 /* chcr_get_nfrags_to_send: get the remaining nfrags after start offset
19 * @skb: skb
20 * @start: start offset.
21 * @len: how much data to send after @start
22 */
chcr_get_nfrags_to_send(struct sk_buff * skb,u32 start,u32 len)23 static int chcr_get_nfrags_to_send(struct sk_buff *skb, u32 start, u32 len)
24 {
25 struct skb_shared_info *si = skb_shinfo(skb);
26 u32 frag_size, skb_linear_data_len = skb_headlen(skb);
27 u8 nfrags = 0, frag_idx = 0;
28 skb_frag_t *frag;
29
30 /* if its a linear skb then return 1 */
31 if (!skb_is_nonlinear(skb))
32 return 1;
33
34 if (unlikely(start < skb_linear_data_len)) {
35 frag_size = min(len, skb_linear_data_len - start);
36 } else {
37 start -= skb_linear_data_len;
38
39 frag = &si->frags[frag_idx];
40 frag_size = skb_frag_size(frag);
41 while (start >= frag_size) {
42 start -= frag_size;
43 frag_idx++;
44 frag = &si->frags[frag_idx];
45 frag_size = skb_frag_size(frag);
46 }
47 frag_size = min(len, skb_frag_size(frag) - start);
48 }
49 len -= frag_size;
50 nfrags++;
51
52 while (len) {
53 frag_size = min(len, skb_frag_size(&si->frags[frag_idx]));
54 len -= frag_size;
55 nfrags++;
56 frag_idx++;
57 }
58 return nfrags;
59 }
60
61 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info);
62 /*
63 * chcr_ktls_save_keys: calculate and save crypto keys.
64 * @tx_info - driver specific tls info.
65 * @crypto_info - tls crypto information.
66 * @direction - TX/RX direction.
67 * return - SUCCESS/FAILURE.
68 */
chcr_ktls_save_keys(struct chcr_ktls_info * tx_info,struct tls_crypto_info * crypto_info,enum tls_offload_ctx_dir direction)69 static int chcr_ktls_save_keys(struct chcr_ktls_info *tx_info,
70 struct tls_crypto_info *crypto_info,
71 enum tls_offload_ctx_dir direction)
72 {
73 int ck_size, key_ctx_size, mac_key_size, keylen, ghash_size, ret;
74 unsigned char ghash_h[TLS_CIPHER_AES_GCM_256_TAG_SIZE];
75 struct tls12_crypto_info_aes_gcm_128 *info_128_gcm;
76 struct ktls_key_ctx *kctx = &tx_info->key_ctx;
77 struct crypto_aes_ctx aes_ctx;
78 unsigned char *key, *salt;
79
80 switch (crypto_info->cipher_type) {
81 case TLS_CIPHER_AES_GCM_128:
82 info_128_gcm =
83 (struct tls12_crypto_info_aes_gcm_128 *)crypto_info;
84 keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
85 ck_size = CHCR_KEYCTX_CIPHER_KEY_SIZE_128;
86 tx_info->salt_size = TLS_CIPHER_AES_GCM_128_SALT_SIZE;
87 mac_key_size = CHCR_KEYCTX_MAC_KEY_SIZE_128;
88 tx_info->iv_size = TLS_CIPHER_AES_GCM_128_IV_SIZE;
89 tx_info->iv = be64_to_cpu(*(__be64 *)info_128_gcm->iv);
90
91 ghash_size = TLS_CIPHER_AES_GCM_128_TAG_SIZE;
92 key = info_128_gcm->key;
93 salt = info_128_gcm->salt;
94 tx_info->record_no = *(u64 *)info_128_gcm->rec_seq;
95
96 /* The SCMD fields used when encrypting a full TLS
97 * record. Its a one time calculation till the
98 * connection exists.
99 */
100 tx_info->scmd0_seqno_numivs =
101 SCMD_SEQ_NO_CTRL_V(CHCR_SCMD_SEQ_NO_CTRL_64BIT) |
102 SCMD_CIPH_AUTH_SEQ_CTRL_F |
103 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_TLS) |
104 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_GCM) |
105 SCMD_AUTH_MODE_V(CHCR_SCMD_AUTH_MODE_GHASH) |
106 SCMD_IV_SIZE_V(TLS_CIPHER_AES_GCM_128_IV_SIZE >> 1) |
107 SCMD_NUM_IVS_V(1);
108
109 /* keys will be sent inline. */
110 tx_info->scmd0_ivgen_hdrlen = SCMD_KEY_CTX_INLINE_F;
111
112 /* The SCMD fields used when encrypting a partial TLS
113 * record (no trailer and possibly a truncated payload).
114 */
115 tx_info->scmd0_short_seqno_numivs =
116 SCMD_CIPH_AUTH_SEQ_CTRL_F |
117 SCMD_PROTO_VERSION_V(CHCR_SCMD_PROTO_VERSION_GENERIC) |
118 SCMD_CIPH_MODE_V(CHCR_SCMD_CIPHER_MODE_AES_CTR) |
119 SCMD_IV_SIZE_V(AES_BLOCK_LEN >> 1);
120
121 tx_info->scmd0_short_ivgen_hdrlen =
122 tx_info->scmd0_ivgen_hdrlen | SCMD_AADIVDROP_F;
123
124 break;
125
126 default:
127 pr_err("GCM: cipher type 0x%x not supported\n",
128 crypto_info->cipher_type);
129 ret = -EINVAL;
130 goto out;
131 }
132
133 key_ctx_size = CHCR_KTLS_KEY_CTX_LEN +
134 roundup(keylen, 16) + ghash_size;
135 /* Calculate the H = CIPH(K, 0 repeated 16 times).
136 * It will go in key context
137 */
138
139 ret = aes_expandkey(&aes_ctx, key, keylen);
140 if (ret)
141 goto out;
142
143 memset(ghash_h, 0, ghash_size);
144 aes_encrypt(&aes_ctx, ghash_h, ghash_h);
145 memzero_explicit(&aes_ctx, sizeof(aes_ctx));
146
147 /* fill the Key context */
148 if (direction == TLS_OFFLOAD_CTX_DIR_TX) {
149 kctx->ctx_hdr = FILL_KEY_CTX_HDR(ck_size,
150 mac_key_size,
151 key_ctx_size >> 4);
152 } else {
153 ret = -EINVAL;
154 goto out;
155 }
156
157 memcpy(kctx->salt, salt, tx_info->salt_size);
158 memcpy(kctx->key, key, keylen);
159 memcpy(kctx->key + keylen, ghash_h, ghash_size);
160 tx_info->key_ctx_len = key_ctx_size;
161
162 out:
163 return ret;
164 }
165
166 /*
167 * chcr_ktls_act_open_req: creates TCB entry for ipv4 connection.
168 * @sk - tcp socket.
169 * @tx_info - driver specific tls info.
170 * @atid - connection active tid.
171 * return - send success/failure.
172 */
chcr_ktls_act_open_req(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)173 static int chcr_ktls_act_open_req(struct sock *sk,
174 struct chcr_ktls_info *tx_info,
175 int atid)
176 {
177 struct inet_sock *inet = inet_sk(sk);
178 struct cpl_t6_act_open_req *cpl6;
179 struct cpl_act_open_req *cpl;
180 struct sk_buff *skb;
181 unsigned int len;
182 int qid_atid;
183 u64 options;
184
185 len = sizeof(*cpl6);
186 skb = alloc_skb(len, GFP_KERNEL);
187 if (unlikely(!skb))
188 return -ENOMEM;
189 /* mark it a control pkt */
190 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
191
192 cpl6 = __skb_put_zero(skb, len);
193 cpl = (struct cpl_act_open_req *)cpl6;
194 INIT_TP_WR(cpl6, 0);
195 qid_atid = TID_QID_V(tx_info->rx_qid) |
196 TID_TID_V(atid);
197 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ, qid_atid));
198 cpl->local_port = inet->inet_sport;
199 cpl->peer_port = inet->inet_dport;
200 cpl->local_ip = inet->inet_rcv_saddr;
201 cpl->peer_ip = inet->inet_daddr;
202
203 /* fill first 64 bit option field. */
204 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
205 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
206 cpl->opt0 = cpu_to_be64(options);
207
208 /* next 64 bit option field. */
209 options =
210 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
211 cpl->opt2 = htonl(options);
212
213 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
214 }
215
216 #if IS_ENABLED(CONFIG_IPV6)
217 /*
218 * chcr_ktls_act_open_req6: creates TCB entry for ipv6 connection.
219 * @sk - tcp socket.
220 * @tx_info - driver specific tls info.
221 * @atid - connection active tid.
222 * return - send success/failure.
223 */
chcr_ktls_act_open_req6(struct sock * sk,struct chcr_ktls_info * tx_info,int atid)224 static int chcr_ktls_act_open_req6(struct sock *sk,
225 struct chcr_ktls_info *tx_info,
226 int atid)
227 {
228 struct inet_sock *inet = inet_sk(sk);
229 struct cpl_t6_act_open_req6 *cpl6;
230 struct cpl_act_open_req6 *cpl;
231 struct sk_buff *skb;
232 unsigned int len;
233 int qid_atid;
234 u64 options;
235
236 len = sizeof(*cpl6);
237 skb = alloc_skb(len, GFP_KERNEL);
238 if (unlikely(!skb))
239 return -ENOMEM;
240 /* mark it a control pkt */
241 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
242
243 cpl6 = __skb_put_zero(skb, len);
244 cpl = (struct cpl_act_open_req6 *)cpl6;
245 INIT_TP_WR(cpl6, 0);
246 qid_atid = TID_QID_V(tx_info->rx_qid) | TID_TID_V(atid);
247 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_ACT_OPEN_REQ6, qid_atid));
248 cpl->local_port = inet->inet_sport;
249 cpl->peer_port = inet->inet_dport;
250 cpl->local_ip_hi = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[0];
251 cpl->local_ip_lo = *(__be64 *)&sk->sk_v6_rcv_saddr.in6_u.u6_addr8[8];
252 cpl->peer_ip_hi = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[0];
253 cpl->peer_ip_lo = *(__be64 *)&sk->sk_v6_daddr.in6_u.u6_addr8[8];
254
255 /* first 64 bit option field. */
256 options = TCAM_BYPASS_F | ULP_MODE_V(ULP_MODE_NONE) | NON_OFFLOAD_F |
257 SMAC_SEL_V(tx_info->smt_idx) | TX_CHAN_V(tx_info->tx_chan);
258 cpl->opt0 = cpu_to_be64(options);
259 /* next 64 bit option field. */
260 options =
261 TX_QUEUE_V(tx_info->adap->params.tp.tx_modq[tx_info->tx_chan]);
262 cpl->opt2 = htonl(options);
263
264 return cxgb4_l2t_send(tx_info->netdev, skb, tx_info->l2te);
265 }
266 #endif /* #if IS_ENABLED(CONFIG_IPV6) */
267
268 /*
269 * chcr_setup_connection: create a TCB entry so that TP will form tcp packets.
270 * @sk - tcp socket.
271 * @tx_info - driver specific tls info.
272 * return: NET_TX_OK/NET_XMIT_DROP
273 */
chcr_setup_connection(struct sock * sk,struct chcr_ktls_info * tx_info)274 static int chcr_setup_connection(struct sock *sk,
275 struct chcr_ktls_info *tx_info)
276 {
277 struct tid_info *t = &tx_info->adap->tids;
278 int atid, ret = 0;
279
280 atid = cxgb4_alloc_atid(t, tx_info);
281 if (atid == -1)
282 return -EINVAL;
283
284 tx_info->atid = atid;
285
286 if (tx_info->ip_family == AF_INET) {
287 ret = chcr_ktls_act_open_req(sk, tx_info, atid);
288 #if IS_ENABLED(CONFIG_IPV6)
289 } else {
290 ret = cxgb4_clip_get(tx_info->netdev, (const u32 *)
291 &sk->sk_v6_rcv_saddr,
292 1);
293 if (ret)
294 return ret;
295 ret = chcr_ktls_act_open_req6(sk, tx_info, atid);
296 #endif
297 }
298
299 /* if return type is NET_XMIT_CN, msg will be sent but delayed, mark ret
300 * success, if any other return type clear atid and return that failure.
301 */
302 if (ret) {
303 if (ret == NET_XMIT_CN) {
304 ret = 0;
305 } else {
306 #if IS_ENABLED(CONFIG_IPV6)
307 /* clear clip entry */
308 if (tx_info->ip_family == AF_INET6)
309 cxgb4_clip_release(tx_info->netdev,
310 (const u32 *)
311 &sk->sk_v6_rcv_saddr,
312 1);
313 #endif
314 cxgb4_free_atid(t, atid);
315 }
316 }
317
318 return ret;
319 }
320
321 /*
322 * chcr_set_tcb_field: update tcb fields.
323 * @tx_info - driver specific tls info.
324 * @word - TCB word.
325 * @mask - TCB word related mask.
326 * @val - TCB word related value.
327 * @no_reply - set 1 if not looking for TP response.
328 */
chcr_set_tcb_field(struct chcr_ktls_info * tx_info,u16 word,u64 mask,u64 val,int no_reply)329 static int chcr_set_tcb_field(struct chcr_ktls_info *tx_info, u16 word,
330 u64 mask, u64 val, int no_reply)
331 {
332 struct cpl_set_tcb_field *req;
333 struct sk_buff *skb;
334
335 skb = alloc_skb(sizeof(struct cpl_set_tcb_field), GFP_ATOMIC);
336 if (!skb)
337 return -ENOMEM;
338
339 req = (struct cpl_set_tcb_field *)__skb_put_zero(skb, sizeof(*req));
340 INIT_TP_WR_CPL(req, CPL_SET_TCB_FIELD, tx_info->tid);
341 req->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
342 NO_REPLY_V(no_reply));
343 req->word_cookie = htons(TCB_WORD_V(word));
344 req->mask = cpu_to_be64(mask);
345 req->val = cpu_to_be64(val);
346
347 set_wr_txq(skb, CPL_PRIORITY_CONTROL, tx_info->port_id);
348 return cxgb4_ofld_send(tx_info->netdev, skb);
349 }
350
351 /*
352 * chcr_ktls_dev_del: call back for tls_dev_del.
353 * Remove the tid and l2t entry and close the connection.
354 * it per connection basis.
355 * @netdev - net device.
356 * @tls_cts - tls context.
357 * @direction - TX/RX crypto direction
358 */
chcr_ktls_dev_del(struct net_device * netdev,struct tls_context * tls_ctx,enum tls_offload_ctx_dir direction)359 static void chcr_ktls_dev_del(struct net_device *netdev,
360 struct tls_context *tls_ctx,
361 enum tls_offload_ctx_dir direction)
362 {
363 struct chcr_ktls_ofld_ctx_tx *tx_ctx =
364 chcr_get_ktls_tx_context(tls_ctx);
365 struct chcr_ktls_info *tx_info = tx_ctx->chcr_info;
366 struct ch_ktls_port_stats_debug *port_stats;
367
368 if (!tx_info)
369 return;
370
371 /* clear l2t entry */
372 if (tx_info->l2te)
373 cxgb4_l2t_release(tx_info->l2te);
374
375 #if IS_ENABLED(CONFIG_IPV6)
376 /* clear clip entry */
377 if (tx_info->ip_family == AF_INET6)
378 cxgb4_clip_release(netdev, (const u32 *)
379 &tx_info->sk->sk_v6_rcv_saddr,
380 1);
381 #endif
382
383 /* clear tid */
384 if (tx_info->tid != -1) {
385 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
386 tx_info->tid, tx_info->ip_family);
387 }
388
389 port_stats = &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
390 atomic64_inc(&port_stats->ktls_tx_connection_close);
391 kvfree(tx_info);
392 tx_ctx->chcr_info = NULL;
393 /* release module refcount */
394 module_put(THIS_MODULE);
395 }
396
397 /*
398 * chcr_ktls_dev_add: call back for tls_dev_add.
399 * Create a tcb entry for TP. Also add l2t entry for the connection. And
400 * generate keys & save those keys locally.
401 * @netdev - net device.
402 * @tls_cts - tls context.
403 * @direction - TX/RX crypto direction
404 * return: SUCCESS/FAILURE.
405 */
chcr_ktls_dev_add(struct net_device * netdev,struct sock * sk,enum tls_offload_ctx_dir direction,struct tls_crypto_info * crypto_info,u32 start_offload_tcp_sn)406 static int chcr_ktls_dev_add(struct net_device *netdev, struct sock *sk,
407 enum tls_offload_ctx_dir direction,
408 struct tls_crypto_info *crypto_info,
409 u32 start_offload_tcp_sn)
410 {
411 struct tls_context *tls_ctx = tls_get_ctx(sk);
412 struct ch_ktls_port_stats_debug *port_stats;
413 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
414 struct chcr_ktls_info *tx_info;
415 struct dst_entry *dst;
416 struct adapter *adap;
417 struct port_info *pi;
418 struct neighbour *n;
419 u8 daaddr[16];
420 int ret = -1;
421
422 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
423
424 pi = netdev_priv(netdev);
425 adap = pi->adapter;
426 port_stats = &adap->ch_ktls_stats.ktls_port[pi->port_id];
427 atomic64_inc(&port_stats->ktls_tx_connection_open);
428
429 if (direction == TLS_OFFLOAD_CTX_DIR_RX) {
430 pr_err("not expecting for RX direction\n");
431 goto out;
432 }
433
434 if (tx_ctx->chcr_info)
435 goto out;
436
437 tx_info = kvzalloc(sizeof(*tx_info), GFP_KERNEL);
438 if (!tx_info)
439 goto out;
440
441 tx_info->sk = sk;
442 spin_lock_init(&tx_info->lock);
443 /* initialize tid and atid to -1, 0 is a also a valid id. */
444 tx_info->tid = -1;
445 tx_info->atid = -1;
446
447 tx_info->adap = adap;
448 tx_info->netdev = netdev;
449 tx_info->first_qset = pi->first_qset;
450 tx_info->tx_chan = pi->tx_chan;
451 tx_info->smt_idx = pi->smt_idx;
452 tx_info->port_id = pi->port_id;
453 tx_info->prev_ack = 0;
454 tx_info->prev_win = 0;
455
456 tx_info->rx_qid = chcr_get_first_rx_qid(adap);
457 if (unlikely(tx_info->rx_qid < 0))
458 goto free_tx_info;
459
460 tx_info->prev_seq = start_offload_tcp_sn;
461 tx_info->tcp_start_seq_number = start_offload_tcp_sn;
462
463 /* save crypto keys */
464 ret = chcr_ktls_save_keys(tx_info, crypto_info, direction);
465 if (ret < 0)
466 goto free_tx_info;
467
468 /* get peer ip */
469 if (sk->sk_family == AF_INET) {
470 memcpy(daaddr, &sk->sk_daddr, 4);
471 tx_info->ip_family = AF_INET;
472 #if IS_ENABLED(CONFIG_IPV6)
473 } else {
474 if (!sk->sk_ipv6only &&
475 ipv6_addr_type(&sk->sk_v6_daddr) == IPV6_ADDR_MAPPED) {
476 memcpy(daaddr, &sk->sk_daddr, 4);
477 tx_info->ip_family = AF_INET;
478 } else {
479 memcpy(daaddr, sk->sk_v6_daddr.in6_u.u6_addr8, 16);
480 tx_info->ip_family = AF_INET6;
481 }
482 #endif
483 }
484
485 /* get the l2t index */
486 dst = sk_dst_get(sk);
487 if (!dst) {
488 pr_err("DST entry not found\n");
489 goto free_tx_info;
490 }
491 n = dst_neigh_lookup(dst, daaddr);
492 if (!n || !n->dev) {
493 pr_err("neighbour not found\n");
494 dst_release(dst);
495 goto free_tx_info;
496 }
497 tx_info->l2te = cxgb4_l2t_get(adap->l2t, n, n->dev, 0);
498
499 neigh_release(n);
500 dst_release(dst);
501
502 if (!tx_info->l2te) {
503 pr_err("l2t entry not found\n");
504 goto free_tx_info;
505 }
506
507 /* Driver shouldn't be removed until any single connection exists */
508 if (!try_module_get(THIS_MODULE))
509 goto free_l2t;
510
511 init_completion(&tx_info->completion);
512 /* create a filter and call cxgb4_l2t_send to send the packet out, which
513 * will take care of updating l2t entry in hw if not already done.
514 */
515 tx_info->open_state = CH_KTLS_OPEN_PENDING;
516
517 if (chcr_setup_connection(sk, tx_info))
518 goto put_module;
519
520 /* Wait for reply */
521 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
522 spin_lock_bh(&tx_info->lock);
523 if (tx_info->open_state) {
524 /* need to wait for hw response, can't free tx_info yet. */
525 if (tx_info->open_state == CH_KTLS_OPEN_PENDING)
526 tx_info->pending_close = true;
527 else
528 spin_unlock_bh(&tx_info->lock);
529 /* if in pending close, free the lock after the cleanup */
530 goto put_module;
531 }
532 spin_unlock_bh(&tx_info->lock);
533
534 /* initialize tcb */
535 reinit_completion(&tx_info->completion);
536 /* mark it pending for hw response */
537 tx_info->open_state = CH_KTLS_OPEN_PENDING;
538
539 if (chcr_init_tcb_fields(tx_info))
540 goto free_tid;
541
542 /* Wait for reply */
543 wait_for_completion_timeout(&tx_info->completion, 30 * HZ);
544 spin_lock_bh(&tx_info->lock);
545 if (tx_info->open_state) {
546 /* need to wait for hw response, can't free tx_info yet. */
547 tx_info->pending_close = true;
548 /* free the lock after cleanup */
549 goto free_tid;
550 }
551 spin_unlock_bh(&tx_info->lock);
552
553 if (!cxgb4_check_l2t_valid(tx_info->l2te))
554 goto free_tid;
555
556 atomic64_inc(&port_stats->ktls_tx_ctx);
557 tx_ctx->chcr_info = tx_info;
558
559 return 0;
560
561 free_tid:
562 #if IS_ENABLED(CONFIG_IPV6)
563 /* clear clip entry */
564 if (tx_info->ip_family == AF_INET6)
565 cxgb4_clip_release(netdev, (const u32 *)
566 &sk->sk_v6_rcv_saddr,
567 1);
568 #endif
569 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
570 tx_info->tid, tx_info->ip_family);
571
572 put_module:
573 /* release module refcount */
574 module_put(THIS_MODULE);
575 free_l2t:
576 cxgb4_l2t_release(tx_info->l2te);
577 free_tx_info:
578 if (tx_info->pending_close)
579 spin_unlock_bh(&tx_info->lock);
580 else
581 kvfree(tx_info);
582 out:
583 atomic64_inc(&port_stats->ktls_tx_connection_fail);
584 return -1;
585 }
586
587 /*
588 * chcr_init_tcb_fields: Initialize tcb fields to handle TCP seq number
589 * handling.
590 * @tx_info - driver specific tls info.
591 * return: NET_TX_OK/NET_XMIT_DROP
592 */
chcr_init_tcb_fields(struct chcr_ktls_info * tx_info)593 static int chcr_init_tcb_fields(struct chcr_ktls_info *tx_info)
594 {
595 int ret = 0;
596
597 /* set tcb in offload and bypass */
598 ret =
599 chcr_set_tcb_field(tx_info, TCB_T_FLAGS_W,
600 TCB_T_FLAGS_V(TF_CORE_BYPASS_F | TF_NON_OFFLOAD_F),
601 TCB_T_FLAGS_V(TF_CORE_BYPASS_F), 1);
602 if (ret)
603 return ret;
604 /* reset snd_una and snd_next fields in tcb */
605 ret = chcr_set_tcb_field(tx_info, TCB_SND_UNA_RAW_W,
606 TCB_SND_NXT_RAW_V(TCB_SND_NXT_RAW_M) |
607 TCB_SND_UNA_RAW_V(TCB_SND_UNA_RAW_M),
608 0, 1);
609 if (ret)
610 return ret;
611
612 /* reset send max */
613 ret = chcr_set_tcb_field(tx_info, TCB_SND_MAX_RAW_W,
614 TCB_SND_MAX_RAW_V(TCB_SND_MAX_RAW_M),
615 0, 1);
616 if (ret)
617 return ret;
618
619 /* update l2t index and request for tp reply to confirm tcb is
620 * initialised to handle tx traffic.
621 */
622 ret = chcr_set_tcb_field(tx_info, TCB_L2T_IX_W,
623 TCB_L2T_IX_V(TCB_L2T_IX_M),
624 TCB_L2T_IX_V(tx_info->l2te->idx), 0);
625 return ret;
626 }
627
628 /*
629 * chcr_ktls_cpl_act_open_rpl: connection reply received from TP.
630 */
chcr_ktls_cpl_act_open_rpl(struct adapter * adap,unsigned char * input)631 static int chcr_ktls_cpl_act_open_rpl(struct adapter *adap,
632 unsigned char *input)
633 {
634 const struct cpl_act_open_rpl *p = (void *)input;
635 struct chcr_ktls_info *tx_info = NULL;
636 unsigned int atid, tid, status;
637 struct tid_info *t;
638
639 tid = GET_TID(p);
640 status = AOPEN_STATUS_G(ntohl(p->atid_status));
641 atid = TID_TID_G(AOPEN_ATID_G(ntohl(p->atid_status)));
642
643 t = &adap->tids;
644 tx_info = lookup_atid(t, atid);
645
646 if (!tx_info || tx_info->atid != atid) {
647 pr_err("%s: incorrect tx_info or atid\n", __func__);
648 return -1;
649 }
650
651 cxgb4_free_atid(t, atid);
652 tx_info->atid = -1;
653
654 spin_lock(&tx_info->lock);
655 /* HW response is very close, finish pending cleanup */
656 if (tx_info->pending_close) {
657 spin_unlock(&tx_info->lock);
658 if (!status) {
659 cxgb4_remove_tid(&tx_info->adap->tids, tx_info->tx_chan,
660 tid, tx_info->ip_family);
661 }
662 kvfree(tx_info);
663 return 0;
664 }
665
666 if (!status) {
667 tx_info->tid = tid;
668 cxgb4_insert_tid(t, tx_info, tx_info->tid, tx_info->ip_family);
669 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
670 } else {
671 tx_info->open_state = CH_KTLS_OPEN_FAILURE;
672 }
673 spin_unlock(&tx_info->lock);
674
675 complete(&tx_info->completion);
676 return 0;
677 }
678
679 /*
680 * chcr_ktls_cpl_set_tcb_rpl: TCB reply received from TP.
681 */
chcr_ktls_cpl_set_tcb_rpl(struct adapter * adap,unsigned char * input)682 static int chcr_ktls_cpl_set_tcb_rpl(struct adapter *adap, unsigned char *input)
683 {
684 const struct cpl_set_tcb_rpl *p = (void *)input;
685 struct chcr_ktls_info *tx_info = NULL;
686 struct tid_info *t;
687 u32 tid;
688
689 tid = GET_TID(p);
690
691 t = &adap->tids;
692 tx_info = lookup_tid(t, tid);
693
694 if (!tx_info || tx_info->tid != tid) {
695 pr_err("%s: incorrect tx_info or tid\n", __func__);
696 return -1;
697 }
698
699 spin_lock(&tx_info->lock);
700 if (tx_info->pending_close) {
701 spin_unlock(&tx_info->lock);
702 kvfree(tx_info);
703 return 0;
704 }
705 tx_info->open_state = CH_KTLS_OPEN_SUCCESS;
706 spin_unlock(&tx_info->lock);
707
708 complete(&tx_info->completion);
709 return 0;
710 }
711
__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,u32 tid,void * pos,u16 word,struct sge_eth_txq * q,u64 mask,u64 val,u32 reply)712 static void *__chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
713 u32 tid, void *pos, u16 word,
714 struct sge_eth_txq *q, u64 mask,
715 u64 val, u32 reply)
716 {
717 struct cpl_set_tcb_field_core *cpl;
718 struct ulptx_idata *idata;
719 struct ulp_txpkt *txpkt;
720
721 /* ULP_TXPKT */
722 txpkt = pos;
723 txpkt->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
724 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
725 ULP_TXPKT_FID_V(q->q.cntxt_id) |
726 ULP_TXPKT_RO_F);
727 txpkt->len = htonl(DIV_ROUND_UP(CHCR_SET_TCB_FIELD_LEN, 16));
728
729 /* ULPTX_IDATA sub-command */
730 idata = (struct ulptx_idata *)(txpkt + 1);
731 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM));
732 idata->len = htonl(sizeof(*cpl));
733 pos = idata + 1;
734
735 cpl = pos;
736 /* CPL_SET_TCB_FIELD */
737 OPCODE_TID(cpl) = htonl(MK_OPCODE_TID(CPL_SET_TCB_FIELD, tid));
738 cpl->reply_ctrl = htons(QUEUENO_V(tx_info->rx_qid) |
739 NO_REPLY_V(!reply));
740 cpl->word_cookie = htons(TCB_WORD_V(word));
741 cpl->mask = cpu_to_be64(mask);
742 cpl->val = cpu_to_be64(val);
743
744 /* ULPTX_NOOP */
745 idata = (struct ulptx_idata *)(cpl + 1);
746 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_NOOP));
747 idata->len = htonl(0);
748 pos = idata + 1;
749
750 return pos;
751 }
752
753
754 /*
755 * chcr_write_cpl_set_tcb_ulp: update tcb values.
756 * TCB is responsible to create tcp headers, so all the related values
757 * should be correctly updated.
758 * @tx_info - driver specific tls info.
759 * @q - tx queue on which packet is going out.
760 * @tid - TCB identifier.
761 * @pos - current index where should we start writing.
762 * @word - TCB word.
763 * @mask - TCB word related mask.
764 * @val - TCB word related value.
765 * @reply - set 1 if looking for TP response.
766 * return - next position to write.
767 */
chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tid,void * pos,u16 word,u64 mask,u64 val,u32 reply)768 static void *chcr_write_cpl_set_tcb_ulp(struct chcr_ktls_info *tx_info,
769 struct sge_eth_txq *q, u32 tid,
770 void *pos, u16 word, u64 mask,
771 u64 val, u32 reply)
772 {
773 int left = (void *)q->q.stat - pos;
774
775 if (unlikely(left < CHCR_SET_TCB_FIELD_LEN)) {
776 if (!left) {
777 pos = q->q.desc;
778 } else {
779 u8 buf[48] = {0};
780
781 __chcr_write_cpl_set_tcb_ulp(tx_info, tid, buf, word, q,
782 mask, val, reply);
783
784 return chcr_copy_to_txd(buf, &q->q, pos,
785 CHCR_SET_TCB_FIELD_LEN);
786 }
787 }
788
789 pos = __chcr_write_cpl_set_tcb_ulp(tx_info, tid, pos, word, q,
790 mask, val, reply);
791
792 /* check again if we are at the end of the queue */
793 if (left == CHCR_SET_TCB_FIELD_LEN)
794 pos = q->q.desc;
795
796 return pos;
797 }
798
799 /*
800 * chcr_ktls_xmit_tcb_cpls: update tcb entry so that TP will create the header
801 * with updated values like tcp seq, ack, window etc.
802 * @tx_info - driver specific tls info.
803 * @q - TX queue.
804 * @tcp_seq
805 * @tcp_ack
806 * @tcp_win
807 * return: NETDEV_TX_BUSY/NET_TX_OK.
808 */
chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u64 tcp_seq,u64 tcp_ack,u64 tcp_win,bool offset)809 static int chcr_ktls_xmit_tcb_cpls(struct chcr_ktls_info *tx_info,
810 struct sge_eth_txq *q, u64 tcp_seq,
811 u64 tcp_ack, u64 tcp_win, bool offset)
812 {
813 bool first_wr = ((tx_info->prev_ack == 0) && (tx_info->prev_win == 0));
814 struct ch_ktls_port_stats_debug *port_stats;
815 u32 len, cpl = 0, ndesc, wr_len, wr_mid = 0;
816 struct fw_ulptx_wr *wr;
817 int credits;
818 void *pos;
819
820 wr_len = sizeof(*wr);
821 /* there can be max 4 cpls, check if we have enough credits */
822 len = wr_len + 4 * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
823 ndesc = DIV_ROUND_UP(len, 64);
824
825 credits = chcr_txq_avail(&q->q) - ndesc;
826 if (unlikely(credits < 0)) {
827 chcr_eth_txq_stop(q);
828 return NETDEV_TX_BUSY;
829 }
830
831 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
832 chcr_eth_txq_stop(q);
833 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
834 }
835
836 pos = &q->q.desc[q->q.pidx];
837 /* make space for WR, we'll fill it later when we know all the cpls
838 * being sent out and have complete length.
839 */
840 wr = pos;
841 pos += wr_len;
842 /* update tx_max if its a re-transmit or the first wr */
843 if (first_wr || tcp_seq != tx_info->prev_seq) {
844 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
845 TCB_TX_MAX_W,
846 TCB_TX_MAX_V(TCB_TX_MAX_M),
847 TCB_TX_MAX_V(tcp_seq), 0);
848 cpl++;
849 }
850 /* reset snd una if it's a re-transmit pkt */
851 if (tcp_seq != tx_info->prev_seq || offset) {
852 /* reset snd_una */
853 port_stats =
854 &tx_info->adap->ch_ktls_stats.ktls_port[tx_info->port_id];
855 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
856 TCB_SND_UNA_RAW_W,
857 TCB_SND_UNA_RAW_V
858 (TCB_SND_UNA_RAW_M),
859 TCB_SND_UNA_RAW_V(0), 0);
860 if (tcp_seq != tx_info->prev_seq)
861 atomic64_inc(&port_stats->ktls_tx_ooo);
862 cpl++;
863 }
864 /* update ack */
865 if (first_wr || tx_info->prev_ack != tcp_ack) {
866 pos = chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
867 TCB_RCV_NXT_W,
868 TCB_RCV_NXT_V(TCB_RCV_NXT_M),
869 TCB_RCV_NXT_V(tcp_ack), 0);
870 tx_info->prev_ack = tcp_ack;
871 cpl++;
872 }
873 /* update receive window */
874 if (first_wr || tx_info->prev_win != tcp_win) {
875 chcr_write_cpl_set_tcb_ulp(tx_info, q, tx_info->tid, pos,
876 TCB_RCV_WND_W,
877 TCB_RCV_WND_V(TCB_RCV_WND_M),
878 TCB_RCV_WND_V(tcp_win), 0);
879 tx_info->prev_win = tcp_win;
880 cpl++;
881 }
882
883 if (cpl) {
884 /* get the actual length */
885 len = wr_len + cpl * roundup(CHCR_SET_TCB_FIELD_LEN, 16);
886 /* ULPTX wr */
887 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
888 wr->cookie = 0;
889 /* fill len in wr field */
890 wr->flowid_len16 = htonl(wr_mid |
891 FW_WR_LEN16_V(DIV_ROUND_UP(len, 16)));
892
893 ndesc = DIV_ROUND_UP(len, 64);
894 chcr_txq_advance(&q->q, ndesc);
895 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
896 }
897 return 0;
898 }
899
900 /*
901 * chcr_ktls_get_tx_flits
902 * returns number of flits to be sent out, it includes key context length, WR
903 * size and skb fragments.
904 */
905 static unsigned int
chcr_ktls_get_tx_flits(u32 nr_frags,unsigned int key_ctx_len)906 chcr_ktls_get_tx_flits(u32 nr_frags, unsigned int key_ctx_len)
907 {
908 return chcr_sgl_len(nr_frags) +
909 DIV_ROUND_UP(key_ctx_len + CHCR_KTLS_WR_SIZE, 8);
910 }
911
912 /*
913 * chcr_ktls_check_tcp_options: To check if there is any TCP option available
914 * other than timestamp.
915 * @skb - skb contains partial record..
916 * return: 1 / 0
917 */
918 static int
chcr_ktls_check_tcp_options(struct tcphdr * tcp)919 chcr_ktls_check_tcp_options(struct tcphdr *tcp)
920 {
921 int cnt, opt, optlen;
922 u_char *cp;
923
924 cp = (u_char *)(tcp + 1);
925 cnt = (tcp->doff << 2) - sizeof(struct tcphdr);
926 for (; cnt > 0; cnt -= optlen, cp += optlen) {
927 opt = cp[0];
928 if (opt == TCPOPT_EOL)
929 break;
930 if (opt == TCPOPT_NOP) {
931 optlen = 1;
932 } else {
933 if (cnt < 2)
934 break;
935 optlen = cp[1];
936 if (optlen < 2 || optlen > cnt)
937 break;
938 }
939 switch (opt) {
940 case TCPOPT_NOP:
941 break;
942 default:
943 return 1;
944 }
945 }
946 return 0;
947 }
948
949 /*
950 * chcr_ktls_write_tcp_options : TP can't send out all the options, we need to
951 * send out separately.
952 * @tx_info - driver specific tls info.
953 * @skb - skb contains partial record..
954 * @q - TX queue.
955 * @tx_chan - channel number.
956 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
957 */
958 static int
chcr_ktls_write_tcp_options(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q,uint32_t tx_chan)959 chcr_ktls_write_tcp_options(struct chcr_ktls_info *tx_info, struct sk_buff *skb,
960 struct sge_eth_txq *q, uint32_t tx_chan)
961 {
962 struct fw_eth_tx_pkt_wr *wr;
963 struct cpl_tx_pkt_core *cpl;
964 u32 ctrl, iplen, maclen;
965 struct ipv6hdr *ip6;
966 unsigned int ndesc;
967 struct tcphdr *tcp;
968 int len16, pktlen;
969 struct iphdr *ip;
970 u32 wr_mid = 0;
971 int credits;
972 u8 buf[150];
973 u64 cntrl1;
974 void *pos;
975
976 iplen = skb_network_header_len(skb);
977 maclen = skb_mac_header_len(skb);
978
979 /* packet length = eth hdr len + ip hdr len + tcp hdr len
980 * (including options).
981 */
982 pktlen = skb_transport_offset(skb) + tcp_hdrlen(skb);
983
984 ctrl = sizeof(*cpl) + pktlen;
985 len16 = DIV_ROUND_UP(sizeof(*wr) + ctrl, 16);
986 /* check how many descriptors needed */
987 ndesc = DIV_ROUND_UP(len16, 4);
988
989 credits = chcr_txq_avail(&q->q) - ndesc;
990 if (unlikely(credits < 0)) {
991 chcr_eth_txq_stop(q);
992 return NETDEV_TX_BUSY;
993 }
994
995 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
996 chcr_eth_txq_stop(q);
997 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
998 }
999
1000 pos = &q->q.desc[q->q.pidx];
1001 wr = pos;
1002
1003 /* Firmware work request header */
1004 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1005 FW_WR_IMMDLEN_V(ctrl));
1006
1007 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1008 wr->r3 = 0;
1009
1010 cpl = (void *)(wr + 1);
1011
1012 /* CPL header */
1013 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) | TXPKT_INTF_V(tx_chan) |
1014 TXPKT_PF_V(tx_info->adap->pf));
1015 cpl->pack = 0;
1016 cpl->len = htons(pktlen);
1017
1018 memcpy(buf, skb->data, pktlen);
1019 if (!IS_ENABLED(CONFIG_IPV6) || tx_info->ip_family == AF_INET) {
1020 /* we need to correct ip header len */
1021 ip = (struct iphdr *)(buf + maclen);
1022 ip->tot_len = htons(pktlen - maclen);
1023 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP);
1024 } else {
1025 ip6 = (struct ipv6hdr *)(buf + maclen);
1026 ip6->payload_len = htons(pktlen - maclen - iplen);
1027 cntrl1 = TXPKT_CSUM_TYPE_V(TX_CSUM_TCPIP6);
1028 }
1029
1030 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1031 TXPKT_IPHDR_LEN_V(iplen);
1032 /* checksum offload */
1033 cpl->ctrl1 = cpu_to_be64(cntrl1);
1034
1035 pos = cpl + 1;
1036
1037 /* now take care of the tcp header, if fin is not set then clear push
1038 * bit as well, and if fin is set, it will be sent at the last so we
1039 * need to update the tcp sequence number as per the last packet.
1040 */
1041 tcp = (struct tcphdr *)(buf + maclen + iplen);
1042
1043 if (!tcp->fin)
1044 tcp->psh = 0;
1045 else
1046 tcp->seq = htonl(tx_info->prev_seq);
1047
1048 chcr_copy_to_txd(buf, &q->q, pos, pktlen);
1049
1050 chcr_txq_advance(&q->q, ndesc);
1051 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1052 return 0;
1053 }
1054
1055 /*
1056 * chcr_ktls_xmit_wr_complete: This sends out the complete record. If an skb
1057 * received has partial end part of the record, send out the complete record, so
1058 * that crypto block will be able to generate TAG/HASH.
1059 * @skb - segment which has complete or partial end part.
1060 * @tx_info - driver specific tls info.
1061 * @q - TX queue.
1062 * @tcp_seq
1063 * @tcp_push - tcp push bit.
1064 * @mss - segment size.
1065 * return: NETDEV_TX_BUSY/NET_TX_OK.
1066 */
chcr_ktls_xmit_wr_complete(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool is_last_wr,u32 data_len,u32 skb_offset,u32 nfrags,bool tcp_push,u32 mss)1067 static int chcr_ktls_xmit_wr_complete(struct sk_buff *skb,
1068 struct chcr_ktls_info *tx_info,
1069 struct sge_eth_txq *q, u32 tcp_seq,
1070 bool is_last_wr, u32 data_len,
1071 u32 skb_offset, u32 nfrags,
1072 bool tcp_push, u32 mss)
1073 {
1074 u32 len16, wr_mid = 0, flits = 0, ndesc, cipher_start;
1075 struct adapter *adap = tx_info->adap;
1076 int credits, left, last_desc;
1077 struct tx_sw_desc *sgl_sdesc;
1078 struct cpl_tx_data *tx_data;
1079 struct cpl_tx_sec_pdu *cpl;
1080 struct ulptx_idata *idata;
1081 struct ulp_txpkt *ulptx;
1082 struct fw_ulptx_wr *wr;
1083 void *pos;
1084 u64 *end;
1085
1086 /* get the number of flits required */
1087 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len);
1088 /* number of descriptors */
1089 ndesc = chcr_flits_to_desc(flits);
1090 /* check if enough credits available */
1091 credits = chcr_txq_avail(&q->q) - ndesc;
1092 if (unlikely(credits < 0)) {
1093 chcr_eth_txq_stop(q);
1094 return NETDEV_TX_BUSY;
1095 }
1096
1097 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1098 /* Credits are below the threshold values, stop the queue after
1099 * injecting the Work Request for this packet.
1100 */
1101 chcr_eth_txq_stop(q);
1102 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1103 }
1104
1105 last_desc = q->q.pidx + ndesc - 1;
1106 if (last_desc >= q->q.size)
1107 last_desc -= q->q.size;
1108 sgl_sdesc = &q->q.sdesc[last_desc];
1109
1110 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1111 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1112 q->mapping_err++;
1113 return NETDEV_TX_BUSY;
1114 }
1115
1116 if (!is_last_wr)
1117 skb_get(skb);
1118
1119 pos = &q->q.desc[q->q.pidx];
1120 end = (u64 *)pos + flits;
1121 /* FW_ULPTX_WR */
1122 wr = pos;
1123 /* WR will need len16 */
1124 len16 = DIV_ROUND_UP(flits, 2);
1125 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1126 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1127 wr->cookie = 0;
1128 pos += sizeof(*wr);
1129 /* ULP_TXPKT */
1130 ulptx = pos;
1131 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1132 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1133 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1134 ULP_TXPKT_RO_F);
1135 ulptx->len = htonl(len16 - 1);
1136 /* ULPTX_IDATA sub-command */
1137 idata = (struct ulptx_idata *)(ulptx + 1);
1138 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1139 /* idata length will include cpl_tx_sec_pdu + key context size +
1140 * cpl_tx_data header.
1141 */
1142 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1143 sizeof(*tx_data));
1144 /* SEC CPL */
1145 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1146 cpl->op_ivinsrtofst =
1147 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1148 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1149 CPL_TX_SEC_PDU_PLACEHOLDER_V(1) |
1150 CPL_TX_SEC_PDU_IVINSRTOFST_V(TLS_HEADER_SIZE + 1));
1151 cpl->pldlen = htonl(data_len);
1152
1153 /* encryption should start after tls header size + iv size */
1154 cipher_start = TLS_HEADER_SIZE + tx_info->iv_size + 1;
1155
1156 cpl->aadstart_cipherstop_hi =
1157 htonl(CPL_TX_SEC_PDU_AADSTART_V(1) |
1158 CPL_TX_SEC_PDU_AADSTOP_V(TLS_HEADER_SIZE) |
1159 CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1160
1161 /* authentication will also start after tls header + iv size */
1162 cpl->cipherstop_lo_authinsert =
1163 htonl(CPL_TX_SEC_PDU_AUTHSTART_V(cipher_start) |
1164 CPL_TX_SEC_PDU_AUTHSTOP_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE) |
1165 CPL_TX_SEC_PDU_AUTHINSERT_V(TLS_CIPHER_AES_GCM_128_TAG_SIZE));
1166
1167 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1168 cpl->seqno_numivs = htonl(tx_info->scmd0_seqno_numivs);
1169 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_ivgen_hdrlen);
1170 cpl->scmd1 = cpu_to_be64(tx_info->record_no);
1171
1172 pos = cpl + 1;
1173 /* check if space left to fill the keys */
1174 left = (void *)q->q.stat - pos;
1175 if (!left) {
1176 left = (void *)end - (void *)q->q.stat;
1177 pos = q->q.desc;
1178 end = pos + left;
1179 }
1180
1181 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1182 tx_info->key_ctx_len);
1183 left = (void *)q->q.stat - pos;
1184
1185 if (!left) {
1186 left = (void *)end - (void *)q->q.stat;
1187 pos = q->q.desc;
1188 end = pos + left;
1189 }
1190 /* CPL_TX_DATA */
1191 tx_data = (void *)pos;
1192 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1193 tx_data->len = htonl(TX_DATA_MSS_V(mss) | TX_LENGTH_V(data_len));
1194
1195 tx_data->rsvd = htonl(tcp_seq);
1196
1197 tx_data->flags = htonl(TX_BYPASS_F);
1198 if (tcp_push)
1199 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1200
1201 /* check left again, it might go beyond queue limit */
1202 pos = tx_data + 1;
1203 left = (void *)q->q.stat - pos;
1204
1205 /* check the position again */
1206 if (!left) {
1207 left = (void *)end - (void *)q->q.stat;
1208 pos = q->q.desc;
1209 end = pos + left;
1210 }
1211
1212 /* send the complete packet except the header */
1213 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1214 skb_offset, data_len);
1215 sgl_sdesc->skb = skb;
1216
1217 chcr_txq_advance(&q->q, ndesc);
1218 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1219 atomic64_inc(&adap->ch_ktls_stats.ktls_tx_send_records);
1220
1221 return 0;
1222 }
1223
1224 /*
1225 * chcr_ktls_xmit_wr_short: This is to send out partial records. If its
1226 * a middle part of a record, fetch the prior data to make it 16 byte aligned
1227 * and then only send it out.
1228 *
1229 * @skb - skb contains partial record..
1230 * @tx_info - driver specific tls info.
1231 * @q - TX queue.
1232 * @tcp_seq
1233 * @tcp_push - tcp push bit.
1234 * @mss - segment size.
1235 * @tls_rec_offset - offset from start of the tls record.
1236 * @perior_data - data before the current segment, required to make this record
1237 * 16 byte aligned.
1238 * @prior_data_len - prior_data length (less than 16)
1239 * return: NETDEV_TX_BUSY/NET_TX_OK.
1240 */
chcr_ktls_xmit_wr_short(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q,u32 tcp_seq,bool tcp_push,u32 mss,u32 tls_rec_offset,u8 * prior_data,u32 prior_data_len,u32 data_len,u32 skb_offset)1241 static int chcr_ktls_xmit_wr_short(struct sk_buff *skb,
1242 struct chcr_ktls_info *tx_info,
1243 struct sge_eth_txq *q,
1244 u32 tcp_seq, bool tcp_push, u32 mss,
1245 u32 tls_rec_offset, u8 *prior_data,
1246 u32 prior_data_len, u32 data_len,
1247 u32 skb_offset)
1248 {
1249 u32 len16, wr_mid = 0, cipher_start, nfrags;
1250 struct adapter *adap = tx_info->adap;
1251 unsigned int flits = 0, ndesc;
1252 int credits, left, last_desc;
1253 struct tx_sw_desc *sgl_sdesc;
1254 struct cpl_tx_data *tx_data;
1255 struct cpl_tx_sec_pdu *cpl;
1256 struct ulptx_idata *idata;
1257 struct ulp_txpkt *ulptx;
1258 struct fw_ulptx_wr *wr;
1259 __be64 iv_record;
1260 void *pos;
1261 u64 *end;
1262
1263 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1264 /* get the number of flits required, it's a partial record so 2 flits
1265 * (AES_BLOCK_SIZE) will be added.
1266 */
1267 flits = chcr_ktls_get_tx_flits(nfrags, tx_info->key_ctx_len) + 2;
1268 /* get the correct 8 byte IV of this record */
1269 iv_record = cpu_to_be64(tx_info->iv + tx_info->record_no);
1270 /* If it's a middle record and not 16 byte aligned to run AES CTR, need
1271 * to make it 16 byte aligned. So atleadt 2 extra flits of immediate
1272 * data will be added.
1273 */
1274 if (prior_data_len)
1275 flits += 2;
1276 /* number of descriptors */
1277 ndesc = chcr_flits_to_desc(flits);
1278 /* check if enough credits available */
1279 credits = chcr_txq_avail(&q->q) - ndesc;
1280 if (unlikely(credits < 0)) {
1281 chcr_eth_txq_stop(q);
1282 return NETDEV_TX_BUSY;
1283 }
1284
1285 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1286 chcr_eth_txq_stop(q);
1287 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1288 }
1289
1290 last_desc = q->q.pidx + ndesc - 1;
1291 if (last_desc >= q->q.size)
1292 last_desc -= q->q.size;
1293 sgl_sdesc = &q->q.sdesc[last_desc];
1294
1295 if (unlikely(cxgb4_map_skb(adap->pdev_dev, skb, sgl_sdesc->addr) < 0)) {
1296 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1297 q->mapping_err++;
1298 return NETDEV_TX_BUSY;
1299 }
1300
1301 pos = &q->q.desc[q->q.pidx];
1302 end = (u64 *)pos + flits;
1303 /* FW_ULPTX_WR */
1304 wr = pos;
1305 /* WR will need len16 */
1306 len16 = DIV_ROUND_UP(flits, 2);
1307 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1308 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1309 wr->cookie = 0;
1310 pos += sizeof(*wr);
1311 /* ULP_TXPKT */
1312 ulptx = pos;
1313 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1314 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1315 ULP_TXPKT_FID_V(q->q.cntxt_id) |
1316 ULP_TXPKT_RO_F);
1317 ulptx->len = htonl(len16 - 1);
1318 /* ULPTX_IDATA sub-command */
1319 idata = (struct ulptx_idata *)(ulptx + 1);
1320 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1321 /* idata length will include cpl_tx_sec_pdu + key context size +
1322 * cpl_tx_data header.
1323 */
1324 idata->len = htonl(sizeof(*cpl) + tx_info->key_ctx_len +
1325 sizeof(*tx_data) + AES_BLOCK_LEN + prior_data_len);
1326 /* SEC CPL */
1327 cpl = (struct cpl_tx_sec_pdu *)(idata + 1);
1328 /* cipher start will have tls header + iv size extra if its a header
1329 * part of tls record. else only 16 byte IV will be added.
1330 */
1331 cipher_start =
1332 AES_BLOCK_LEN + 1 +
1333 (!tls_rec_offset ? TLS_HEADER_SIZE + tx_info->iv_size : 0);
1334
1335 cpl->op_ivinsrtofst =
1336 htonl(CPL_TX_SEC_PDU_OPCODE_V(CPL_TX_SEC_PDU) |
1337 CPL_TX_SEC_PDU_CPLLEN_V(CHCR_CPL_TX_SEC_PDU_LEN_64BIT) |
1338 CPL_TX_SEC_PDU_IVINSRTOFST_V(1));
1339 cpl->pldlen = htonl(data_len + AES_BLOCK_LEN + prior_data_len);
1340 cpl->aadstart_cipherstop_hi =
1341 htonl(CPL_TX_SEC_PDU_CIPHERSTART_V(cipher_start));
1342 cpl->cipherstop_lo_authinsert = 0;
1343 /* These two flits are actually a CPL_TLS_TX_SCMD_FMT. */
1344 cpl->seqno_numivs = htonl(tx_info->scmd0_short_seqno_numivs);
1345 cpl->ivgen_hdrlen = htonl(tx_info->scmd0_short_ivgen_hdrlen);
1346 cpl->scmd1 = 0;
1347
1348 pos = cpl + 1;
1349 /* check if space left to fill the keys */
1350 left = (void *)q->q.stat - pos;
1351 if (!left) {
1352 left = (void *)end - (void *)q->q.stat;
1353 pos = q->q.desc;
1354 end = pos + left;
1355 }
1356
1357 pos = chcr_copy_to_txd(&tx_info->key_ctx, &q->q, pos,
1358 tx_info->key_ctx_len);
1359 left = (void *)q->q.stat - pos;
1360
1361 if (!left) {
1362 left = (void *)end - (void *)q->q.stat;
1363 pos = q->q.desc;
1364 end = pos + left;
1365 }
1366 /* CPL_TX_DATA */
1367 tx_data = (void *)pos;
1368 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1369 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1370 TX_LENGTH_V(data_len + prior_data_len));
1371 tx_data->rsvd = htonl(tcp_seq);
1372 tx_data->flags = htonl(TX_BYPASS_F);
1373 if (tcp_push)
1374 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1375
1376 /* check left again, it might go beyond queue limit */
1377 pos = tx_data + 1;
1378 left = (void *)q->q.stat - pos;
1379
1380 /* check the position again */
1381 if (!left) {
1382 left = (void *)end - (void *)q->q.stat;
1383 pos = q->q.desc;
1384 end = pos + left;
1385 }
1386 /* copy the 16 byte IV for AES-CTR, which includes 4 bytes of salt, 8
1387 * bytes of actual IV and 4 bytes of 16 byte-sequence.
1388 */
1389 memcpy(pos, tx_info->key_ctx.salt, tx_info->salt_size);
1390 memcpy(pos + tx_info->salt_size, &iv_record, tx_info->iv_size);
1391 *(__be32 *)(pos + tx_info->salt_size + tx_info->iv_size) =
1392 htonl(2 + (tls_rec_offset ? ((tls_rec_offset -
1393 (TLS_HEADER_SIZE + tx_info->iv_size)) / AES_BLOCK_LEN) : 0));
1394
1395 pos += 16;
1396 /* Prior_data_len will always be less than 16 bytes, fill the
1397 * prio_data_len after AES_CTRL_BLOCK and clear the remaining length
1398 * to 0.
1399 */
1400 if (prior_data_len)
1401 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1402 /* send the complete packet except the header */
1403 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1404 skb_offset, data_len);
1405 sgl_sdesc->skb = skb;
1406
1407 chcr_txq_advance(&q->q, ndesc);
1408 cxgb4_ring_tx_db(adap, &q->q, ndesc);
1409
1410 return 0;
1411 }
1412
1413 /*
1414 * chcr_ktls_tx_plaintxt: This handler will take care of the records which has
1415 * only plain text (only tls header and iv)
1416 * @tx_info - driver specific tls info.
1417 * @skb - skb contains partial record..
1418 * @tcp_seq
1419 * @mss - segment size.
1420 * @tcp_push - tcp push bit.
1421 * @q - TX queue.
1422 * @port_id : port number
1423 * @perior_data - data before the current segment, required to make this record
1424 * 16 byte aligned.
1425 * @prior_data_len - prior_data length (less than 16)
1426 * return: NETDEV_TX_BUSY/NET_TX_OK.
1427 */
chcr_ktls_tx_plaintxt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,u32 tcp_seq,u32 mss,bool tcp_push,struct sge_eth_txq * q,u32 port_id,u8 * prior_data,u32 data_len,u32 skb_offset,u32 prior_data_len)1428 static int chcr_ktls_tx_plaintxt(struct chcr_ktls_info *tx_info,
1429 struct sk_buff *skb, u32 tcp_seq, u32 mss,
1430 bool tcp_push, struct sge_eth_txq *q,
1431 u32 port_id, u8 *prior_data,
1432 u32 data_len, u32 skb_offset,
1433 u32 prior_data_len)
1434 {
1435 int credits, left, len16, last_desc;
1436 unsigned int flits = 0, ndesc;
1437 struct tx_sw_desc *sgl_sdesc;
1438 struct cpl_tx_data *tx_data;
1439 struct ulptx_idata *idata;
1440 struct ulp_txpkt *ulptx;
1441 struct fw_ulptx_wr *wr;
1442 u32 wr_mid = 0, nfrags;
1443 void *pos;
1444 u64 *end;
1445
1446 flits = DIV_ROUND_UP(CHCR_PLAIN_TX_DATA_LEN, 8);
1447 nfrags = chcr_get_nfrags_to_send(skb, skb_offset, data_len);
1448 flits += chcr_sgl_len(nfrags);
1449 if (prior_data_len)
1450 flits += 2;
1451
1452 /* WR will need len16 */
1453 len16 = DIV_ROUND_UP(flits, 2);
1454 /* check how many descriptors needed */
1455 ndesc = DIV_ROUND_UP(flits, 8);
1456
1457 credits = chcr_txq_avail(&q->q) - ndesc;
1458 if (unlikely(credits < 0)) {
1459 chcr_eth_txq_stop(q);
1460 return NETDEV_TX_BUSY;
1461 }
1462
1463 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1464 chcr_eth_txq_stop(q);
1465 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1466 }
1467
1468 last_desc = q->q.pidx + ndesc - 1;
1469 if (last_desc >= q->q.size)
1470 last_desc -= q->q.size;
1471 sgl_sdesc = &q->q.sdesc[last_desc];
1472
1473 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1474 sgl_sdesc->addr) < 0)) {
1475 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1476 q->mapping_err++;
1477 return NETDEV_TX_BUSY;
1478 }
1479
1480 pos = &q->q.desc[q->q.pidx];
1481 end = (u64 *)pos + flits;
1482 /* FW_ULPTX_WR */
1483 wr = pos;
1484 wr->op_to_compl = htonl(FW_WR_OP_V(FW_ULPTX_WR));
1485 wr->flowid_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1486 wr->cookie = 0;
1487 /* ULP_TXPKT */
1488 ulptx = (struct ulp_txpkt *)(wr + 1);
1489 ulptx->cmd_dest = htonl(ULPTX_CMD_V(ULP_TX_PKT) |
1490 ULP_TXPKT_DATAMODIFY_V(0) |
1491 ULP_TXPKT_CHANNELID_V(tx_info->port_id) |
1492 ULP_TXPKT_DEST_V(0) |
1493 ULP_TXPKT_FID_V(q->q.cntxt_id) | ULP_TXPKT_RO_V(1));
1494 ulptx->len = htonl(len16 - 1);
1495 /* ULPTX_IDATA sub-command */
1496 idata = (struct ulptx_idata *)(ulptx + 1);
1497 idata->cmd_more = htonl(ULPTX_CMD_V(ULP_TX_SC_IMM) | ULP_TX_SC_MORE_F);
1498 idata->len = htonl(sizeof(*tx_data) + prior_data_len);
1499 /* CPL_TX_DATA */
1500 tx_data = (struct cpl_tx_data *)(idata + 1);
1501 OPCODE_TID(tx_data) = htonl(MK_OPCODE_TID(CPL_TX_DATA, tx_info->tid));
1502 tx_data->len = htonl(TX_DATA_MSS_V(mss) |
1503 TX_LENGTH_V(data_len + prior_data_len));
1504 /* set tcp seq number */
1505 tx_data->rsvd = htonl(tcp_seq);
1506 tx_data->flags = htonl(TX_BYPASS_F);
1507 if (tcp_push)
1508 tx_data->flags |= htonl(TX_PUSH_F | TX_SHOVE_F);
1509
1510 pos = tx_data + 1;
1511 /* apart from prior_data_len, we should set remaining part of 16 bytes
1512 * to be zero.
1513 */
1514 if (prior_data_len)
1515 pos = chcr_copy_to_txd(prior_data, &q->q, pos, 16);
1516
1517 /* check left again, it might go beyond queue limit */
1518 left = (void *)q->q.stat - pos;
1519
1520 /* check the position again */
1521 if (!left) {
1522 left = (void *)end - (void *)q->q.stat;
1523 pos = q->q.desc;
1524 end = pos + left;
1525 }
1526 /* send the complete packet including the header */
1527 cxgb4_write_partial_sgl(skb, &q->q, pos, end, sgl_sdesc->addr,
1528 skb_offset, data_len);
1529 sgl_sdesc->skb = skb;
1530
1531 chcr_txq_advance(&q->q, ndesc);
1532 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1533 return 0;
1534 }
1535
chcr_ktls_tunnel_pkt(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct sge_eth_txq * q)1536 static int chcr_ktls_tunnel_pkt(struct chcr_ktls_info *tx_info,
1537 struct sk_buff *skb,
1538 struct sge_eth_txq *q)
1539 {
1540 u32 ctrl, iplen, maclen, wr_mid = 0, len16;
1541 struct tx_sw_desc *sgl_sdesc;
1542 struct fw_eth_tx_pkt_wr *wr;
1543 struct cpl_tx_pkt_core *cpl;
1544 unsigned int flits, ndesc;
1545 int credits, last_desc;
1546 u64 cntrl1, *end;
1547 void *pos;
1548
1549 ctrl = sizeof(*cpl);
1550 flits = DIV_ROUND_UP(sizeof(*wr) + ctrl, 8);
1551
1552 flits += chcr_sgl_len(skb_shinfo(skb)->nr_frags + 1);
1553 len16 = DIV_ROUND_UP(flits, 2);
1554 /* check how many descriptors needed */
1555 ndesc = DIV_ROUND_UP(flits, 8);
1556
1557 credits = chcr_txq_avail(&q->q) - ndesc;
1558 if (unlikely(credits < 0)) {
1559 chcr_eth_txq_stop(q);
1560 return -ENOMEM;
1561 }
1562
1563 if (unlikely(credits < ETHTXQ_STOP_THRES)) {
1564 chcr_eth_txq_stop(q);
1565 wr_mid |= FW_WR_EQUEQ_F | FW_WR_EQUIQ_F;
1566 }
1567
1568 last_desc = q->q.pidx + ndesc - 1;
1569 if (last_desc >= q->q.size)
1570 last_desc -= q->q.size;
1571 sgl_sdesc = &q->q.sdesc[last_desc];
1572
1573 if (unlikely(cxgb4_map_skb(tx_info->adap->pdev_dev, skb,
1574 sgl_sdesc->addr) < 0)) {
1575 memset(sgl_sdesc->addr, 0, sizeof(sgl_sdesc->addr));
1576 q->mapping_err++;
1577 return -ENOMEM;
1578 }
1579
1580 iplen = skb_network_header_len(skb);
1581 maclen = skb_mac_header_len(skb);
1582
1583 pos = &q->q.desc[q->q.pidx];
1584 end = (u64 *)pos + flits;
1585 wr = pos;
1586
1587 /* Firmware work request header */
1588 wr->op_immdlen = htonl(FW_WR_OP_V(FW_ETH_TX_PKT_WR) |
1589 FW_WR_IMMDLEN_V(ctrl));
1590
1591 wr->equiq_to_len16 = htonl(wr_mid | FW_WR_LEN16_V(len16));
1592 wr->r3 = 0;
1593
1594 cpl = (void *)(wr + 1);
1595
1596 /* CPL header */
1597 cpl->ctrl0 = htonl(TXPKT_OPCODE_V(CPL_TX_PKT) |
1598 TXPKT_INTF_V(tx_info->tx_chan) |
1599 TXPKT_PF_V(tx_info->adap->pf));
1600 cpl->pack = 0;
1601 cntrl1 = TXPKT_CSUM_TYPE_V(tx_info->ip_family == AF_INET ?
1602 TX_CSUM_TCPIP : TX_CSUM_TCPIP6);
1603 cntrl1 |= T6_TXPKT_ETHHDR_LEN_V(maclen - ETH_HLEN) |
1604 TXPKT_IPHDR_LEN_V(iplen);
1605 /* checksum offload */
1606 cpl->ctrl1 = cpu_to_be64(cntrl1);
1607 cpl->len = htons(skb->len);
1608
1609 pos = cpl + 1;
1610
1611 cxgb4_write_sgl(skb, &q->q, pos, end, 0, sgl_sdesc->addr);
1612 sgl_sdesc->skb = skb;
1613 chcr_txq_advance(&q->q, ndesc);
1614 cxgb4_ring_tx_db(tx_info->adap, &q->q, ndesc);
1615 return 0;
1616 }
1617
1618 /*
1619 * chcr_ktls_copy_record_in_skb
1620 * @nskb - new skb where the frags to be added.
1621 * @skb - old skb, to copy socket and destructor details.
1622 * @record - specific record which has complete 16k record in frags.
1623 */
chcr_ktls_copy_record_in_skb(struct sk_buff * nskb,struct sk_buff * skb,struct tls_record_info * record)1624 static void chcr_ktls_copy_record_in_skb(struct sk_buff *nskb,
1625 struct sk_buff *skb,
1626 struct tls_record_info *record)
1627 {
1628 int i = 0;
1629
1630 for (i = 0; i < record->num_frags; i++) {
1631 skb_shinfo(nskb)->frags[i] = record->frags[i];
1632 /* increase the frag ref count */
1633 __skb_frag_ref(&skb_shinfo(nskb)->frags[i]);
1634 }
1635
1636 skb_shinfo(nskb)->nr_frags = record->num_frags;
1637 nskb->data_len = record->len;
1638 nskb->len += record->len;
1639 nskb->truesize += record->len;
1640 nskb->sk = skb->sk;
1641 nskb->destructor = skb->destructor;
1642 refcount_add(nskb->truesize, &nskb->sk->sk_wmem_alloc);
1643 }
1644
1645 /*
1646 * chcr_end_part_handler: This handler will handle the record which
1647 * is complete or if record's end part is received. T6 adapter has a issue that
1648 * it can't send out TAG with partial record so if its an end part then we have
1649 * to send TAG as well and for which we need to fetch the complete record and
1650 * send it to crypto module.
1651 * @tx_info - driver specific tls info.
1652 * @skb - skb contains partial record.
1653 * @record - complete record of 16K size.
1654 * @tcp_seq
1655 * @mss - segment size in which TP needs to chop a packet.
1656 * @tcp_push_no_fin - tcp push if fin is not set.
1657 * @q - TX queue.
1658 * @tls_end_offset - offset from end of the record.
1659 * @last wr : check if this is the last part of the skb going out.
1660 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1661 */
chcr_end_part_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,struct sge_eth_txq * q,u32 skb_offset,u32 tls_end_offset,bool last_wr)1662 static int chcr_end_part_handler(struct chcr_ktls_info *tx_info,
1663 struct sk_buff *skb,
1664 struct tls_record_info *record,
1665 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1666 struct sge_eth_txq *q, u32 skb_offset,
1667 u32 tls_end_offset, bool last_wr)
1668 {
1669 bool free_skb_if_tx_fails = false;
1670 struct sk_buff *nskb = NULL;
1671
1672 /* check if it is a complete record */
1673 if (tls_end_offset == record->len) {
1674 nskb = skb;
1675 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_complete_pkts);
1676 } else {
1677 nskb = alloc_skb(0, GFP_ATOMIC);
1678 if (!nskb) {
1679 dev_kfree_skb_any(skb);
1680 return NETDEV_TX_BUSY;
1681 }
1682
1683 /* copy complete record in skb */
1684 chcr_ktls_copy_record_in_skb(nskb, skb, record);
1685 /* packet is being sent from the beginning, update the tcp_seq
1686 * accordingly.
1687 */
1688 tcp_seq = tls_record_start_seq(record);
1689 /* reset skb offset */
1690 skb_offset = 0;
1691
1692 if (last_wr)
1693 dev_kfree_skb_any(skb);
1694 else
1695 free_skb_if_tx_fails = true;
1696
1697 last_wr = true;
1698
1699 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_end_pkts);
1700 }
1701
1702 if (chcr_ktls_xmit_wr_complete(nskb, tx_info, q, tcp_seq,
1703 last_wr, record->len, skb_offset,
1704 record->num_frags,
1705 (last_wr && tcp_push_no_fin),
1706 mss)) {
1707 if (free_skb_if_tx_fails)
1708 dev_kfree_skb_any(skb);
1709 goto out;
1710 }
1711 tx_info->prev_seq = record->end_seq;
1712 return 0;
1713 out:
1714 dev_kfree_skb_any(nskb);
1715 return NETDEV_TX_BUSY;
1716 }
1717
1718 /*
1719 * chcr_short_record_handler: This handler will take care of the records which
1720 * doesn't have end part (1st part or the middle part(/s) of a record). In such
1721 * cases, AES CTR will be used in place of AES GCM to send out partial packet.
1722 * This partial record might be the first part of the record, or the middle
1723 * part. In case of middle record we should fetch the prior data to make it 16
1724 * byte aligned. If it has a partial tls header or iv then get to the start of
1725 * tls header. And if it has partial TAG, then remove the complete TAG and send
1726 * only the payload.
1727 * There is one more possibility that it gets a partial header, send that
1728 * portion as a plaintext.
1729 * @tx_info - driver specific tls info.
1730 * @skb - skb contains partial record..
1731 * @record - complete record of 16K size.
1732 * @tcp_seq
1733 * @mss - segment size in which TP needs to chop a packet.
1734 * @tcp_push_no_fin - tcp push if fin is not set.
1735 * @q - TX queue.
1736 * @tls_end_offset - offset from end of the record.
1737 * return: NETDEV_TX_OK/NETDEV_TX_BUSY.
1738 */
chcr_short_record_handler(struct chcr_ktls_info * tx_info,struct sk_buff * skb,struct tls_record_info * record,u32 tcp_seq,int mss,bool tcp_push_no_fin,u32 data_len,u32 skb_offset,struct sge_eth_txq * q,u32 tls_end_offset)1739 static int chcr_short_record_handler(struct chcr_ktls_info *tx_info,
1740 struct sk_buff *skb,
1741 struct tls_record_info *record,
1742 u32 tcp_seq, int mss, bool tcp_push_no_fin,
1743 u32 data_len, u32 skb_offset,
1744 struct sge_eth_txq *q, u32 tls_end_offset)
1745 {
1746 u32 tls_rec_offset = tcp_seq - tls_record_start_seq(record);
1747 u8 prior_data[16] = {0};
1748 u32 prior_data_len = 0;
1749
1750 /* check if the skb is ending in middle of tag/HASH, its a big
1751 * trouble, send the packet before the HASH.
1752 */
1753 int remaining_record = tls_end_offset - data_len;
1754
1755 if (remaining_record > 0 &&
1756 remaining_record < TLS_CIPHER_AES_GCM_128_TAG_SIZE) {
1757 int trimmed_len = 0;
1758
1759 if (tls_end_offset > TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1760 trimmed_len = data_len -
1761 (TLS_CIPHER_AES_GCM_128_TAG_SIZE -
1762 remaining_record);
1763 if (!trimmed_len)
1764 return FALLBACK;
1765
1766 WARN_ON(trimmed_len > data_len);
1767
1768 data_len = trimmed_len;
1769 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_trimmed_pkts);
1770 }
1771
1772 /* check if it is only the header part. */
1773 if (tls_rec_offset + data_len <= (TLS_HEADER_SIZE + tx_info->iv_size)) {
1774 if (chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
1775 tcp_push_no_fin, q,
1776 tx_info->port_id, prior_data,
1777 data_len, skb_offset, prior_data_len))
1778 goto out;
1779
1780 tx_info->prev_seq = tcp_seq + data_len;
1781 return 0;
1782 }
1783
1784 /* check if the middle record's start point is 16 byte aligned. CTR
1785 * needs 16 byte aligned start point to start encryption.
1786 */
1787 if (tls_rec_offset) {
1788 /* there is an offset from start, means its a middle record */
1789 int remaining = 0;
1790
1791 if (tls_rec_offset < (TLS_HEADER_SIZE + tx_info->iv_size)) {
1792 prior_data_len = tls_rec_offset;
1793 tls_rec_offset = 0;
1794 remaining = 0;
1795 } else {
1796 prior_data_len =
1797 (tls_rec_offset -
1798 (TLS_HEADER_SIZE + tx_info->iv_size))
1799 % AES_BLOCK_LEN;
1800 remaining = tls_rec_offset - prior_data_len;
1801 }
1802
1803 /* if prior_data_len is not zero, means we need to fetch prior
1804 * data to make this record 16 byte aligned, or we need to reach
1805 * to start offset.
1806 */
1807 if (prior_data_len) {
1808 int i = 0;
1809 u8 *data = NULL;
1810 skb_frag_t *f;
1811 u8 *vaddr;
1812 int frag_size = 0, frag_delta = 0;
1813
1814 while (remaining > 0) {
1815 frag_size = skb_frag_size(&record->frags[i]);
1816 if (remaining < frag_size)
1817 break;
1818
1819 remaining -= frag_size;
1820 i++;
1821 }
1822 f = &record->frags[i];
1823 vaddr = kmap_atomic(skb_frag_page(f));
1824
1825 data = vaddr + skb_frag_off(f) + remaining;
1826 frag_delta = skb_frag_size(f) - remaining;
1827
1828 if (frag_delta >= prior_data_len) {
1829 memcpy(prior_data, data, prior_data_len);
1830 kunmap_atomic(vaddr);
1831 } else {
1832 memcpy(prior_data, data, frag_delta);
1833 kunmap_atomic(vaddr);
1834 /* get the next page */
1835 f = &record->frags[i + 1];
1836 vaddr = kmap_atomic(skb_frag_page(f));
1837 data = vaddr + skb_frag_off(f);
1838 memcpy(prior_data + frag_delta,
1839 data, (prior_data_len - frag_delta));
1840 kunmap_atomic(vaddr);
1841 }
1842 /* reset tcp_seq as per the prior_data_required len */
1843 tcp_seq -= prior_data_len;
1844 }
1845 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_middle_pkts);
1846 } else {
1847 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_start_pkts);
1848 }
1849
1850 if (chcr_ktls_xmit_wr_short(skb, tx_info, q, tcp_seq, tcp_push_no_fin,
1851 mss, tls_rec_offset, prior_data,
1852 prior_data_len, data_len, skb_offset)) {
1853 goto out;
1854 }
1855
1856 tx_info->prev_seq = tcp_seq + data_len + prior_data_len;
1857 return 0;
1858 out:
1859 dev_kfree_skb_any(skb);
1860 return NETDEV_TX_BUSY;
1861 }
1862
chcr_ktls_sw_fallback(struct sk_buff * skb,struct chcr_ktls_info * tx_info,struct sge_eth_txq * q)1863 static int chcr_ktls_sw_fallback(struct sk_buff *skb,
1864 struct chcr_ktls_info *tx_info,
1865 struct sge_eth_txq *q)
1866 {
1867 u32 data_len, skb_offset;
1868 struct sk_buff *nskb;
1869 struct tcphdr *th;
1870
1871 nskb = tls_encrypt_skb(skb);
1872
1873 if (!nskb)
1874 return 0;
1875
1876 th = tcp_hdr(nskb);
1877 skb_offset = skb_transport_offset(nskb) + tcp_hdrlen(nskb);
1878 data_len = nskb->len - skb_offset;
1879 skb_tx_timestamp(nskb);
1880
1881 if (chcr_ktls_tunnel_pkt(tx_info, nskb, q))
1882 goto out;
1883
1884 tx_info->prev_seq = ntohl(th->seq) + data_len;
1885 atomic64_inc(&tx_info->adap->ch_ktls_stats.ktls_tx_fallback);
1886 return 0;
1887 out:
1888 dev_kfree_skb_any(nskb);
1889 return 0;
1890 }
1891 /* nic tls TX handler */
chcr_ktls_xmit(struct sk_buff * skb,struct net_device * dev)1892 static int chcr_ktls_xmit(struct sk_buff *skb, struct net_device *dev)
1893 {
1894 u32 tls_end_offset, tcp_seq, skb_data_len, skb_offset;
1895 struct ch_ktls_port_stats_debug *port_stats;
1896 struct chcr_ktls_ofld_ctx_tx *tx_ctx;
1897 struct ch_ktls_stats_debug *stats;
1898 struct tcphdr *th = tcp_hdr(skb);
1899 int data_len, qidx, ret = 0, mss;
1900 struct tls_record_info *record;
1901 struct chcr_ktls_info *tx_info;
1902 struct tls_context *tls_ctx;
1903 struct sge_eth_txq *q;
1904 struct adapter *adap;
1905 unsigned long flags;
1906
1907 tcp_seq = ntohl(th->seq);
1908 skb_offset = skb_transport_offset(skb) + tcp_hdrlen(skb);
1909 skb_data_len = skb->len - skb_offset;
1910 data_len = skb_data_len;
1911
1912 mss = skb_is_gso(skb) ? skb_shinfo(skb)->gso_size : data_len;
1913
1914 tls_ctx = tls_get_ctx(skb->sk);
1915 if (unlikely(tls_ctx->netdev != dev))
1916 goto out;
1917
1918 tx_ctx = chcr_get_ktls_tx_context(tls_ctx);
1919 tx_info = tx_ctx->chcr_info;
1920
1921 if (unlikely(!tx_info))
1922 goto out;
1923
1924 adap = tx_info->adap;
1925 stats = &adap->ch_ktls_stats;
1926 port_stats = &stats->ktls_port[tx_info->port_id];
1927
1928 qidx = skb->queue_mapping;
1929 q = &adap->sge.ethtxq[qidx + tx_info->first_qset];
1930 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1931 /* if tcp options are set but finish is not send the options first */
1932 if (!th->fin && chcr_ktls_check_tcp_options(th)) {
1933 ret = chcr_ktls_write_tcp_options(tx_info, skb, q,
1934 tx_info->tx_chan);
1935 if (ret)
1936 return NETDEV_TX_BUSY;
1937 }
1938
1939 /* TCP segments can be in received either complete or partial.
1940 * chcr_end_part_handler will handle cases if complete record or end
1941 * part of the record is received. In case of partial end part of record,
1942 * we will send the complete record again.
1943 */
1944
1945 spin_lock_irqsave(&tx_ctx->base.lock, flags);
1946
1947 do {
1948
1949 cxgb4_reclaim_completed_tx(adap, &q->q, true);
1950 /* fetch the tls record */
1951 record = tls_get_record(&tx_ctx->base, tcp_seq,
1952 &tx_info->record_no);
1953 /* By the time packet reached to us, ACK is received, and record
1954 * won't be found in that case, handle it gracefully.
1955 */
1956 if (unlikely(!record)) {
1957 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
1958 atomic64_inc(&port_stats->ktls_tx_drop_no_sync_data);
1959 goto out;
1960 }
1961
1962 tls_end_offset = record->end_seq - tcp_seq;
1963
1964 pr_debug("seq 0x%x, end_seq 0x%x prev_seq 0x%x, datalen 0x%x\n",
1965 tcp_seq, record->end_seq, tx_info->prev_seq, data_len);
1966 /* update tcb for the skb */
1967 if (skb_data_len == data_len) {
1968 u32 tx_max = tcp_seq;
1969
1970 if (!tls_record_is_start_marker(record) &&
1971 tls_end_offset < TLS_CIPHER_AES_GCM_128_TAG_SIZE)
1972 tx_max = record->end_seq -
1973 TLS_CIPHER_AES_GCM_128_TAG_SIZE;
1974
1975 ret = chcr_ktls_xmit_tcb_cpls(tx_info, q, tx_max,
1976 ntohl(th->ack_seq),
1977 ntohs(th->window),
1978 tls_end_offset !=
1979 record->len);
1980 if (ret) {
1981 spin_unlock_irqrestore(&tx_ctx->base.lock,
1982 flags);
1983 goto out;
1984 }
1985
1986 if (th->fin)
1987 skb_get(skb);
1988 }
1989
1990 if (unlikely(tls_record_is_start_marker(record))) {
1991 atomic64_inc(&port_stats->ktls_tx_skip_no_sync_data);
1992 /* If tls_end_offset < data_len, means there is some
1993 * data after start marker, which needs encryption, send
1994 * plaintext first and take skb refcount. else send out
1995 * complete pkt as plaintext.
1996 */
1997 if (tls_end_offset < data_len)
1998 skb_get(skb);
1999 else
2000 tls_end_offset = data_len;
2001
2002 ret = chcr_ktls_tx_plaintxt(tx_info, skb, tcp_seq, mss,
2003 (!th->fin && th->psh), q,
2004 tx_info->port_id, NULL,
2005 tls_end_offset, skb_offset,
2006 0);
2007
2008 if (ret) {
2009 /* free the refcount taken earlier */
2010 if (tls_end_offset < data_len)
2011 dev_kfree_skb_any(skb);
2012 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2013 goto out;
2014 }
2015
2016 data_len -= tls_end_offset;
2017 tcp_seq = record->end_seq;
2018 skb_offset += tls_end_offset;
2019 continue;
2020 }
2021
2022 /* if a tls record is finishing in this SKB */
2023 if (tls_end_offset <= data_len) {
2024 ret = chcr_end_part_handler(tx_info, skb, record,
2025 tcp_seq, mss,
2026 (!th->fin && th->psh), q,
2027 skb_offset,
2028 tls_end_offset,
2029 skb_offset +
2030 tls_end_offset == skb->len);
2031
2032 data_len -= tls_end_offset;
2033 /* tcp_seq increment is required to handle next record.
2034 */
2035 tcp_seq += tls_end_offset;
2036 skb_offset += tls_end_offset;
2037 } else {
2038 ret = chcr_short_record_handler(tx_info, skb,
2039 record, tcp_seq, mss,
2040 (!th->fin && th->psh),
2041 data_len, skb_offset,
2042 q, tls_end_offset);
2043 data_len = 0;
2044 }
2045
2046 /* if any failure, come out from the loop. */
2047 if (ret) {
2048 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2049 if (th->fin)
2050 dev_kfree_skb_any(skb);
2051
2052 if (ret == FALLBACK)
2053 return chcr_ktls_sw_fallback(skb, tx_info, q);
2054
2055 return NETDEV_TX_OK;
2056 }
2057
2058 /* length should never be less than 0 */
2059 WARN_ON(data_len < 0);
2060
2061 } while (data_len > 0);
2062
2063 spin_unlock_irqrestore(&tx_ctx->base.lock, flags);
2064 atomic64_inc(&port_stats->ktls_tx_encrypted_packets);
2065 atomic64_add(skb_data_len, &port_stats->ktls_tx_encrypted_bytes);
2066
2067 /* tcp finish is set, send a separate tcp msg including all the options
2068 * as well.
2069 */
2070 if (th->fin) {
2071 chcr_ktls_write_tcp_options(tx_info, skb, q, tx_info->tx_chan);
2072 dev_kfree_skb_any(skb);
2073 }
2074
2075 return NETDEV_TX_OK;
2076 out:
2077 dev_kfree_skb_any(skb);
2078 return NETDEV_TX_OK;
2079 }
2080
chcr_ktls_uld_add(const struct cxgb4_lld_info * lldi)2081 static void *chcr_ktls_uld_add(const struct cxgb4_lld_info *lldi)
2082 {
2083 struct chcr_ktls_uld_ctx *u_ctx;
2084
2085 pr_info_once("%s - version %s\n", CHCR_KTLS_DRV_DESC,
2086 CHCR_KTLS_DRV_VERSION);
2087 u_ctx = kzalloc(sizeof(*u_ctx), GFP_KERNEL);
2088 if (!u_ctx) {
2089 u_ctx = ERR_PTR(-ENOMEM);
2090 goto out;
2091 }
2092 u_ctx->lldi = *lldi;
2093 out:
2094 return u_ctx;
2095 }
2096
2097 static const struct tlsdev_ops chcr_ktls_ops = {
2098 .tls_dev_add = chcr_ktls_dev_add,
2099 .tls_dev_del = chcr_ktls_dev_del,
2100 };
2101
2102 static chcr_handler_func work_handlers[NUM_CPL_CMDS] = {
2103 [CPL_ACT_OPEN_RPL] = chcr_ktls_cpl_act_open_rpl,
2104 [CPL_SET_TCB_RPL] = chcr_ktls_cpl_set_tcb_rpl,
2105 };
2106
chcr_ktls_uld_rx_handler(void * handle,const __be64 * rsp,const struct pkt_gl * pgl)2107 static int chcr_ktls_uld_rx_handler(void *handle, const __be64 *rsp,
2108 const struct pkt_gl *pgl)
2109 {
2110 const struct cpl_act_open_rpl *rpl = (struct cpl_act_open_rpl *)rsp;
2111 struct chcr_ktls_uld_ctx *u_ctx = handle;
2112 u8 opcode = rpl->ot.opcode;
2113 struct adapter *adap;
2114
2115 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2116
2117 if (!work_handlers[opcode]) {
2118 pr_err("Unsupported opcode %d received\n", opcode);
2119 return 0;
2120 }
2121
2122 work_handlers[opcode](adap, (unsigned char *)&rsp[1]);
2123 return 0;
2124 }
2125
chcr_ktls_uld_state_change(void * handle,enum cxgb4_state new_state)2126 static int chcr_ktls_uld_state_change(void *handle, enum cxgb4_state new_state)
2127 {
2128 struct chcr_ktls_uld_ctx *u_ctx = handle;
2129
2130 switch (new_state) {
2131 case CXGB4_STATE_UP:
2132 pr_info("%s: Up\n", pci_name(u_ctx->lldi.pdev));
2133 mutex_lock(&dev_mutex);
2134 list_add_tail(&u_ctx->entry, &uld_ctx_list);
2135 mutex_unlock(&dev_mutex);
2136 break;
2137 case CXGB4_STATE_START_RECOVERY:
2138 case CXGB4_STATE_DOWN:
2139 case CXGB4_STATE_DETACH:
2140 pr_info("%s: Down\n", pci_name(u_ctx->lldi.pdev));
2141 mutex_lock(&dev_mutex);
2142 list_del(&u_ctx->entry);
2143 mutex_unlock(&dev_mutex);
2144 break;
2145 default:
2146 break;
2147 }
2148
2149 return 0;
2150 }
2151
2152 static struct cxgb4_uld_info chcr_ktls_uld_info = {
2153 .name = CHCR_KTLS_DRV_MODULE_NAME,
2154 .nrxq = 1,
2155 .rxq_size = 1024,
2156 .add = chcr_ktls_uld_add,
2157 .tx_handler = chcr_ktls_xmit,
2158 .rx_handler = chcr_ktls_uld_rx_handler,
2159 .state_change = chcr_ktls_uld_state_change,
2160 .tlsdev_ops = &chcr_ktls_ops,
2161 };
2162
chcr_ktls_init(void)2163 static int __init chcr_ktls_init(void)
2164 {
2165 cxgb4_register_uld(CXGB4_ULD_KTLS, &chcr_ktls_uld_info);
2166 return 0;
2167 }
2168
chcr_ktls_exit(void)2169 static void __exit chcr_ktls_exit(void)
2170 {
2171 struct chcr_ktls_uld_ctx *u_ctx, *tmp;
2172 struct adapter *adap;
2173
2174 pr_info("driver unloaded\n");
2175
2176 mutex_lock(&dev_mutex);
2177 list_for_each_entry_safe(u_ctx, tmp, &uld_ctx_list, entry) {
2178 adap = pci_get_drvdata(u_ctx->lldi.pdev);
2179 memset(&adap->ch_ktls_stats, 0, sizeof(adap->ch_ktls_stats));
2180 list_del(&u_ctx->entry);
2181 kfree(u_ctx);
2182 }
2183 mutex_unlock(&dev_mutex);
2184 cxgb4_unregister_uld(CXGB4_ULD_KTLS);
2185 }
2186
2187 module_init(chcr_ktls_init);
2188 module_exit(chcr_ktls_exit);
2189
2190 MODULE_DESCRIPTION("Chelsio NIC TLS ULD driver");
2191 MODULE_LICENSE("GPL");
2192 MODULE_AUTHOR("Chelsio Communications");
2193 MODULE_VERSION(CHCR_KTLS_DRV_VERSION);
2194