1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (c) 2018 Chelsio Communications, Inc.
4  *
5  * Written by: Atul Gupta (atul.gupta@chelsio.com)
6  */
7 #include <linux/kernel.h>
8 #include <linux/module.h>
9 #include <linux/skbuff.h>
10 #include <linux/socket.h>
11 #include <linux/hash.h>
12 #include <linux/in.h>
13 #include <linux/net.h>
14 #include <linux/ip.h>
15 #include <linux/tcp.h>
16 #include <net/ipv6.h>
17 #include <net/transp_v6.h>
18 #include <net/tcp.h>
19 #include <net/tls.h>
20 
21 #include "chtls.h"
22 #include "chtls_cm.h"
23 
24 #define DRV_NAME "chtls"
25 
26 /*
27  * chtls device management
28  * maintains a list of the chtls devices
29  */
30 static LIST_HEAD(cdev_list);
31 static DEFINE_MUTEX(cdev_mutex);
32 
33 static DEFINE_MUTEX(notify_mutex);
34 static RAW_NOTIFIER_HEAD(listen_notify_list);
35 static struct proto chtls_cpl_prot, chtls_cpl_protv6;
36 struct request_sock_ops chtls_rsk_ops, chtls_rsk_opsv6;
37 static uint send_page_order = (14 - PAGE_SHIFT < 0) ? 0 : 14 - PAGE_SHIFT;
38 
39 static void register_listen_notifier(struct notifier_block *nb)
40 {
41 	mutex_lock(&notify_mutex);
42 	raw_notifier_chain_register(&listen_notify_list, nb);
43 	mutex_unlock(&notify_mutex);
44 }
45 
46 static void unregister_listen_notifier(struct notifier_block *nb)
47 {
48 	mutex_lock(&notify_mutex);
49 	raw_notifier_chain_unregister(&listen_notify_list, nb);
50 	mutex_unlock(&notify_mutex);
51 }
52 
53 static int listen_notify_handler(struct notifier_block *this,
54 				 unsigned long event, void *data)
55 {
56 	struct chtls_listen *clisten;
57 	int ret = NOTIFY_DONE;
58 
59 	clisten = (struct chtls_listen *)data;
60 
61 	switch (event) {
62 	case CHTLS_LISTEN_START:
63 		ret = chtls_listen_start(clisten->cdev, clisten->sk);
64 		kfree(clisten);
65 		break;
66 	case CHTLS_LISTEN_STOP:
67 		chtls_listen_stop(clisten->cdev, clisten->sk);
68 		kfree(clisten);
69 		break;
70 	}
71 	return ret;
72 }
73 
74 static struct notifier_block listen_notifier = {
75 	.notifier_call = listen_notify_handler
76 };
77 
78 static int listen_backlog_rcv(struct sock *sk, struct sk_buff *skb)
79 {
80 	if (likely(skb_transport_header(skb) != skb_network_header(skb)))
81 		return tcp_v4_do_rcv(sk, skb);
82 	BLOG_SKB_CB(skb)->backlog_rcv(sk, skb);
83 	return 0;
84 }
85 
86 static int chtls_start_listen(struct chtls_dev *cdev, struct sock *sk)
87 {
88 	struct chtls_listen *clisten;
89 
90 	if (sk->sk_protocol != IPPROTO_TCP)
91 		return -EPROTONOSUPPORT;
92 
93 	if (sk->sk_family == PF_INET &&
94 	    LOOPBACK(inet_sk(sk)->inet_rcv_saddr))
95 		return -EADDRNOTAVAIL;
96 
97 	sk->sk_backlog_rcv = listen_backlog_rcv;
98 	clisten = kmalloc(sizeof(*clisten), GFP_KERNEL);
99 	if (!clisten)
100 		return -ENOMEM;
101 	clisten->cdev = cdev;
102 	clisten->sk = sk;
103 	mutex_lock(&notify_mutex);
104 	raw_notifier_call_chain(&listen_notify_list,
105 				      CHTLS_LISTEN_START, clisten);
106 	mutex_unlock(&notify_mutex);
107 	return 0;
108 }
109 
110 static void chtls_stop_listen(struct chtls_dev *cdev, struct sock *sk)
111 {
112 	struct chtls_listen *clisten;
113 
114 	if (sk->sk_protocol != IPPROTO_TCP)
115 		return;
116 
117 	clisten = kmalloc(sizeof(*clisten), GFP_KERNEL);
118 	if (!clisten)
119 		return;
120 	clisten->cdev = cdev;
121 	clisten->sk = sk;
122 	mutex_lock(&notify_mutex);
123 	raw_notifier_call_chain(&listen_notify_list,
124 				CHTLS_LISTEN_STOP, clisten);
125 	mutex_unlock(&notify_mutex);
126 }
127 
128 static int chtls_inline_feature(struct tls_toe_device *dev)
129 {
130 	struct net_device *netdev;
131 	struct chtls_dev *cdev;
132 	int i;
133 
134 	cdev = to_chtls_dev(dev);
135 
136 	for (i = 0; i < cdev->lldi->nports; i++) {
137 		netdev = cdev->ports[i];
138 		if (netdev->features & NETIF_F_HW_TLS_RECORD)
139 			return 1;
140 	}
141 	return 0;
142 }
143 
144 static int chtls_create_hash(struct tls_toe_device *dev, struct sock *sk)
145 {
146 	struct chtls_dev *cdev = to_chtls_dev(dev);
147 
148 	if (sk->sk_state == TCP_LISTEN)
149 		return chtls_start_listen(cdev, sk);
150 	return 0;
151 }
152 
153 static void chtls_destroy_hash(struct tls_toe_device *dev, struct sock *sk)
154 {
155 	struct chtls_dev *cdev = to_chtls_dev(dev);
156 
157 	if (sk->sk_state == TCP_LISTEN)
158 		chtls_stop_listen(cdev, sk);
159 }
160 
161 static void chtls_free_uld(struct chtls_dev *cdev)
162 {
163 	int i;
164 
165 	tls_toe_unregister_device(&cdev->tlsdev);
166 	kvfree(cdev->kmap.addr);
167 	idr_destroy(&cdev->hwtid_idr);
168 	for (i = 0; i < (1 << RSPQ_HASH_BITS); i++)
169 		kfree_skb(cdev->rspq_skb_cache[i]);
170 	kfree(cdev->lldi);
171 	kfree_skb(cdev->askb);
172 	kfree(cdev);
173 }
174 
175 static inline void chtls_dev_release(struct kref *kref)
176 {
177 	struct tls_toe_device *dev;
178 	struct chtls_dev *cdev;
179 	struct adapter *adap;
180 
181 	dev = container_of(kref, struct tls_toe_device, kref);
182 	cdev = to_chtls_dev(dev);
183 
184 	/* Reset tls rx/tx stats */
185 	adap = pci_get_drvdata(cdev->pdev);
186 	atomic_set(&adap->chcr_stats.tls_pdu_tx, 0);
187 	atomic_set(&adap->chcr_stats.tls_pdu_rx, 0);
188 
189 	chtls_free_uld(cdev);
190 }
191 
192 static void chtls_register_dev(struct chtls_dev *cdev)
193 {
194 	struct tls_toe_device *tlsdev = &cdev->tlsdev;
195 
196 	strlcpy(tlsdev->name, "chtls", TLS_TOE_DEVICE_NAME_MAX);
197 	strlcat(tlsdev->name, cdev->lldi->ports[0]->name,
198 		TLS_TOE_DEVICE_NAME_MAX);
199 	tlsdev->feature = chtls_inline_feature;
200 	tlsdev->hash = chtls_create_hash;
201 	tlsdev->unhash = chtls_destroy_hash;
202 	tlsdev->release = chtls_dev_release;
203 	kref_init(&tlsdev->kref);
204 	tls_toe_register_device(tlsdev);
205 	cdev->cdev_state = CHTLS_CDEV_STATE_UP;
206 }
207 
208 static void process_deferq(struct work_struct *task_param)
209 {
210 	struct chtls_dev *cdev = container_of(task_param,
211 				struct chtls_dev, deferq_task);
212 	struct sk_buff *skb;
213 
214 	spin_lock_bh(&cdev->deferq.lock);
215 	while ((skb = __skb_dequeue(&cdev->deferq)) != NULL) {
216 		spin_unlock_bh(&cdev->deferq.lock);
217 		DEFERRED_SKB_CB(skb)->handler(cdev, skb);
218 		spin_lock_bh(&cdev->deferq.lock);
219 	}
220 	spin_unlock_bh(&cdev->deferq.lock);
221 }
222 
223 static int chtls_get_skb(struct chtls_dev *cdev)
224 {
225 	cdev->askb = alloc_skb(sizeof(struct tcphdr), GFP_KERNEL);
226 	if (!cdev->askb)
227 		return -ENOMEM;
228 
229 	skb_put(cdev->askb, sizeof(struct tcphdr));
230 	skb_reset_transport_header(cdev->askb);
231 	memset(cdev->askb->data, 0, cdev->askb->len);
232 	return 0;
233 }
234 
235 static void *chtls_uld_add(const struct cxgb4_lld_info *info)
236 {
237 	struct cxgb4_lld_info *lldi;
238 	struct chtls_dev *cdev;
239 	int i, j;
240 
241 	cdev = kzalloc(sizeof(*cdev), GFP_KERNEL);
242 	if (!cdev)
243 		goto out;
244 
245 	lldi = kzalloc(sizeof(*lldi), GFP_KERNEL);
246 	if (!lldi)
247 		goto out_lldi;
248 
249 	if (chtls_get_skb(cdev))
250 		goto out_skb;
251 
252 	*lldi = *info;
253 	cdev->lldi = lldi;
254 	cdev->pdev = lldi->pdev;
255 	cdev->tids = lldi->tids;
256 	cdev->ports = lldi->ports;
257 	cdev->mtus = lldi->mtus;
258 	cdev->tids = lldi->tids;
259 	cdev->pfvf = FW_VIID_PFN_G(cxgb4_port_viid(lldi->ports[0]))
260 			<< FW_VIID_PFN_S;
261 
262 	for (i = 0; i < (1 << RSPQ_HASH_BITS); i++) {
263 		unsigned int size = 64 - sizeof(struct rsp_ctrl) - 8;
264 
265 		cdev->rspq_skb_cache[i] = __alloc_skb(size,
266 						      gfp_any(), 0,
267 						      lldi->nodeid);
268 		if (unlikely(!cdev->rspq_skb_cache[i]))
269 			goto out_rspq_skb;
270 	}
271 
272 	idr_init(&cdev->hwtid_idr);
273 	INIT_WORK(&cdev->deferq_task, process_deferq);
274 	spin_lock_init(&cdev->listen_lock);
275 	spin_lock_init(&cdev->idr_lock);
276 	cdev->send_page_order = min_t(uint, get_order(32768),
277 				      send_page_order);
278 	cdev->max_host_sndbuf = 48 * 1024;
279 
280 	if (lldi->vr->key.size)
281 		if (chtls_init_kmap(cdev, lldi))
282 			goto out_rspq_skb;
283 
284 	mutex_lock(&cdev_mutex);
285 	list_add_tail(&cdev->list, &cdev_list);
286 	mutex_unlock(&cdev_mutex);
287 
288 	return cdev;
289 out_rspq_skb:
290 	for (j = 0; j < i; j++)
291 		kfree_skb(cdev->rspq_skb_cache[j]);
292 	kfree_skb(cdev->askb);
293 out_skb:
294 	kfree(lldi);
295 out_lldi:
296 	kfree(cdev);
297 out:
298 	return NULL;
299 }
300 
301 static void chtls_free_all_uld(void)
302 {
303 	struct chtls_dev *cdev, *tmp;
304 
305 	mutex_lock(&cdev_mutex);
306 	list_for_each_entry_safe(cdev, tmp, &cdev_list, list) {
307 		if (cdev->cdev_state == CHTLS_CDEV_STATE_UP) {
308 			list_del(&cdev->list);
309 			kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
310 		}
311 	}
312 	mutex_unlock(&cdev_mutex);
313 }
314 
315 static int chtls_uld_state_change(void *handle, enum cxgb4_state new_state)
316 {
317 	struct chtls_dev *cdev = handle;
318 
319 	switch (new_state) {
320 	case CXGB4_STATE_UP:
321 		chtls_register_dev(cdev);
322 		break;
323 	case CXGB4_STATE_DOWN:
324 		break;
325 	case CXGB4_STATE_START_RECOVERY:
326 		break;
327 	case CXGB4_STATE_DETACH:
328 		mutex_lock(&cdev_mutex);
329 		list_del(&cdev->list);
330 		mutex_unlock(&cdev_mutex);
331 		kref_put(&cdev->tlsdev.kref, cdev->tlsdev.release);
332 		break;
333 	default:
334 		break;
335 	}
336 	return 0;
337 }
338 
339 static struct sk_buff *copy_gl_to_skb_pkt(const struct pkt_gl *gl,
340 					  const __be64 *rsp,
341 					  u32 pktshift)
342 {
343 	struct sk_buff *skb;
344 
345 	/* Allocate space for cpl_pass_accpet_req which will be synthesized by
346 	 * driver. Once driver synthesizes cpl_pass_accpet_req the skb will go
347 	 * through the regular cpl_pass_accept_req processing in TOM.
348 	 */
349 	skb = alloc_skb(gl->tot_len + sizeof(struct cpl_pass_accept_req)
350 			- pktshift, GFP_ATOMIC);
351 	if (unlikely(!skb))
352 		return NULL;
353 	__skb_put(skb, gl->tot_len + sizeof(struct cpl_pass_accept_req)
354 		   - pktshift);
355 	/* For now we will copy  cpl_rx_pkt in the skb */
356 	skb_copy_to_linear_data(skb, rsp, sizeof(struct cpl_rx_pkt));
357 	skb_copy_to_linear_data_offset(skb, sizeof(struct cpl_pass_accept_req)
358 				       , gl->va + pktshift,
359 				       gl->tot_len - pktshift);
360 
361 	return skb;
362 }
363 
364 static int chtls_recv_packet(struct chtls_dev *cdev,
365 			     const struct pkt_gl *gl, const __be64 *rsp)
366 {
367 	unsigned int opcode = *(u8 *)rsp;
368 	struct sk_buff *skb;
369 	int ret;
370 
371 	skb = copy_gl_to_skb_pkt(gl, rsp, cdev->lldi->sge_pktshift);
372 	if (!skb)
373 		return -ENOMEM;
374 
375 	ret = chtls_handlers[opcode](cdev, skb);
376 	if (ret & CPL_RET_BUF_DONE)
377 		kfree_skb(skb);
378 
379 	return 0;
380 }
381 
382 static int chtls_recv_rsp(struct chtls_dev *cdev, const __be64 *rsp)
383 {
384 	unsigned long rspq_bin;
385 	unsigned int opcode;
386 	struct sk_buff *skb;
387 	unsigned int len;
388 	int ret;
389 
390 	len = 64 - sizeof(struct rsp_ctrl) - 8;
391 	opcode = *(u8 *)rsp;
392 
393 	rspq_bin = hash_ptr((void *)rsp, RSPQ_HASH_BITS);
394 	skb = cdev->rspq_skb_cache[rspq_bin];
395 	if (skb && !skb_is_nonlinear(skb) &&
396 	    !skb_shared(skb) && !skb_cloned(skb)) {
397 		refcount_inc(&skb->users);
398 		if (refcount_read(&skb->users) == 2) {
399 			__skb_trim(skb, 0);
400 			if (skb_tailroom(skb) >= len)
401 				goto copy_out;
402 		}
403 		refcount_dec(&skb->users);
404 	}
405 	skb = alloc_skb(len, GFP_ATOMIC);
406 	if (unlikely(!skb))
407 		return -ENOMEM;
408 
409 copy_out:
410 	__skb_put(skb, len);
411 	skb_copy_to_linear_data(skb, rsp, len);
412 	skb_reset_network_header(skb);
413 	skb_reset_transport_header(skb);
414 	ret = chtls_handlers[opcode](cdev, skb);
415 
416 	if (ret & CPL_RET_BUF_DONE)
417 		kfree_skb(skb);
418 	return 0;
419 }
420 
421 static void chtls_recv(struct chtls_dev *cdev,
422 		       struct sk_buff **skbs, const __be64 *rsp)
423 {
424 	struct sk_buff *skb = *skbs;
425 	unsigned int opcode;
426 	int ret;
427 
428 	opcode = *(u8 *)rsp;
429 
430 	__skb_push(skb, sizeof(struct rss_header));
431 	skb_copy_to_linear_data(skb, rsp, sizeof(struct rss_header));
432 
433 	ret = chtls_handlers[opcode](cdev, skb);
434 	if (ret & CPL_RET_BUF_DONE)
435 		kfree_skb(skb);
436 }
437 
438 static int chtls_uld_rx_handler(void *handle, const __be64 *rsp,
439 				const struct pkt_gl *gl)
440 {
441 	struct chtls_dev *cdev = handle;
442 	unsigned int opcode;
443 	struct sk_buff *skb;
444 
445 	opcode = *(u8 *)rsp;
446 
447 	if (unlikely(opcode == CPL_RX_PKT)) {
448 		if (chtls_recv_packet(cdev, gl, rsp) < 0)
449 			goto nomem;
450 		return 0;
451 	}
452 
453 	if (!gl)
454 		return chtls_recv_rsp(cdev, rsp);
455 
456 #define RX_PULL_LEN 128
457 	skb = cxgb4_pktgl_to_skb(gl, RX_PULL_LEN, RX_PULL_LEN);
458 	if (unlikely(!skb))
459 		goto nomem;
460 	chtls_recv(cdev, &skb, rsp);
461 	return 0;
462 
463 nomem:
464 	return -ENOMEM;
465 }
466 
467 static int do_chtls_getsockopt(struct sock *sk, char __user *optval,
468 			       int __user *optlen)
469 {
470 	struct tls_crypto_info crypto_info = { 0 };
471 
472 	crypto_info.version = TLS_1_2_VERSION;
473 	if (copy_to_user(optval, &crypto_info, sizeof(struct tls_crypto_info)))
474 		return -EFAULT;
475 	return 0;
476 }
477 
478 static int chtls_getsockopt(struct sock *sk, int level, int optname,
479 			    char __user *optval, int __user *optlen)
480 {
481 	struct tls_context *ctx = tls_get_ctx(sk);
482 
483 	if (level != SOL_TLS)
484 		return ctx->sk_proto->getsockopt(sk, level,
485 						 optname, optval, optlen);
486 
487 	return do_chtls_getsockopt(sk, optval, optlen);
488 }
489 
490 static int do_chtls_setsockopt(struct sock *sk, int optname,
491 			       sockptr_t optval, unsigned int optlen)
492 {
493 	struct tls_crypto_info *crypto_info, tmp_crypto_info;
494 	struct chtls_sock *csk;
495 	int keylen;
496 	int cipher_type;
497 	int rc = 0;
498 
499 	csk = rcu_dereference_sk_user_data(sk);
500 
501 	if (sockptr_is_null(optval) || optlen < sizeof(*crypto_info)) {
502 		rc = -EINVAL;
503 		goto out;
504 	}
505 
506 	rc = copy_from_sockptr(&tmp_crypto_info, optval, sizeof(*crypto_info));
507 	if (rc) {
508 		rc = -EFAULT;
509 		goto out;
510 	}
511 
512 	/* check version */
513 	if (tmp_crypto_info.version != TLS_1_2_VERSION) {
514 		rc = -ENOTSUPP;
515 		goto out;
516 	}
517 
518 	crypto_info = (struct tls_crypto_info *)&csk->tlshws.crypto_info;
519 
520 	/* GCM mode of AES supports 128 and 256 bit encryption, so
521 	 * copy keys from user based on GCM cipher type.
522 	 */
523 	switch (tmp_crypto_info.cipher_type) {
524 	case TLS_CIPHER_AES_GCM_128: {
525 		/* Obtain version and type from previous copy */
526 		crypto_info[0] = tmp_crypto_info;
527 		/* Now copy the following data */
528 		rc = copy_from_sockptr_offset((char *)crypto_info +
529 				sizeof(*crypto_info),
530 				optval, sizeof(*crypto_info),
531 				sizeof(struct tls12_crypto_info_aes_gcm_128)
532 				- sizeof(*crypto_info));
533 
534 		if (rc) {
535 			rc = -EFAULT;
536 			goto out;
537 		}
538 
539 		keylen = TLS_CIPHER_AES_GCM_128_KEY_SIZE;
540 		cipher_type = TLS_CIPHER_AES_GCM_128;
541 		break;
542 	}
543 	case TLS_CIPHER_AES_GCM_256: {
544 		crypto_info[0] = tmp_crypto_info;
545 		rc = copy_from_sockptr_offset((char *)crypto_info +
546 				sizeof(*crypto_info),
547 				optval, sizeof(*crypto_info),
548 				sizeof(struct tls12_crypto_info_aes_gcm_256)
549 				- sizeof(*crypto_info));
550 
551 		if (rc) {
552 			rc = -EFAULT;
553 			goto out;
554 		}
555 
556 		keylen = TLS_CIPHER_AES_GCM_256_KEY_SIZE;
557 		cipher_type = TLS_CIPHER_AES_GCM_256;
558 		break;
559 	}
560 	default:
561 		rc = -EINVAL;
562 		goto out;
563 	}
564 	rc = chtls_setkey(csk, keylen, optname, cipher_type);
565 out:
566 	return rc;
567 }
568 
569 static int chtls_setsockopt(struct sock *sk, int level, int optname,
570 			    sockptr_t optval, unsigned int optlen)
571 {
572 	struct tls_context *ctx = tls_get_ctx(sk);
573 
574 	if (level != SOL_TLS)
575 		return ctx->sk_proto->setsockopt(sk, level,
576 						 optname, optval, optlen);
577 
578 	return do_chtls_setsockopt(sk, optname, optval, optlen);
579 }
580 
581 static struct cxgb4_uld_info chtls_uld_info = {
582 	.name = DRV_NAME,
583 	.nrxq = MAX_ULD_QSETS,
584 	.ntxq = MAX_ULD_QSETS,
585 	.rxq_size = 1024,
586 	.add = chtls_uld_add,
587 	.state_change = chtls_uld_state_change,
588 	.rx_handler = chtls_uld_rx_handler,
589 };
590 
591 void chtls_install_cpl_ops(struct sock *sk)
592 {
593 	if (sk->sk_family == AF_INET)
594 		sk->sk_prot = &chtls_cpl_prot;
595 	else
596 		sk->sk_prot = &chtls_cpl_protv6;
597 }
598 
599 static void __init chtls_init_ulp_ops(void)
600 {
601 	chtls_cpl_prot			= tcp_prot;
602 	chtls_init_rsk_ops(&chtls_cpl_prot, &chtls_rsk_ops,
603 			   &tcp_prot, PF_INET);
604 	chtls_cpl_prot.close		= chtls_close;
605 	chtls_cpl_prot.disconnect	= chtls_disconnect;
606 	chtls_cpl_prot.destroy		= chtls_destroy_sock;
607 	chtls_cpl_prot.shutdown		= chtls_shutdown;
608 	chtls_cpl_prot.sendmsg		= chtls_sendmsg;
609 	chtls_cpl_prot.sendpage		= chtls_sendpage;
610 	chtls_cpl_prot.recvmsg		= chtls_recvmsg;
611 	chtls_cpl_prot.setsockopt	= chtls_setsockopt;
612 	chtls_cpl_prot.getsockopt	= chtls_getsockopt;
613 #if IS_ENABLED(CONFIG_IPV6)
614 	chtls_cpl_protv6		= chtls_cpl_prot;
615 	chtls_init_rsk_ops(&chtls_cpl_protv6, &chtls_rsk_opsv6,
616 			   &tcpv6_prot, PF_INET6);
617 #endif
618 }
619 
620 static int __init chtls_register(void)
621 {
622 	chtls_init_ulp_ops();
623 	register_listen_notifier(&listen_notifier);
624 	cxgb4_register_uld(CXGB4_ULD_TLS, &chtls_uld_info);
625 	return 0;
626 }
627 
628 static void __exit chtls_unregister(void)
629 {
630 	unregister_listen_notifier(&listen_notifier);
631 	chtls_free_all_uld();
632 	cxgb4_unregister_uld(CXGB4_ULD_TLS);
633 }
634 
635 module_init(chtls_register);
636 module_exit(chtls_unregister);
637 
638 MODULE_DESCRIPTION("Chelsio TLS Inline driver");
639 MODULE_LICENSE("GPL");
640 MODULE_AUTHOR("Chelsio Communications");
641 MODULE_VERSION(CHTLS_DRV_VERSION);
642