1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /*
3 * INET An implementation of the TCP/IP protocol suite for the LINUX
4 * operating system. INET is implemented using the BSD Socket
5 * interface as the means of communication with the user level.
6 *
7 * Implementation of the Transmission Control Protocol(TCP).
8 *
9 * Authors: Ross Biro
10 * Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
11 * Mark Evans, <evansmp@uhura.aston.ac.uk>
12 * Corey Minyard <wf-rch!minyard@relay.EU.net>
13 * Florian La Roche, <flla@stud.uni-sb.de>
14 * Charles Hedrick, <hedrick@klinzhai.rutgers.edu>
15 * Linus Torvalds, <torvalds@cs.helsinki.fi>
16 * Alan Cox, <gw4pts@gw4pts.ampr.org>
17 * Matthew Dillon, <dillon@apollo.west.oic.com>
18 * Arnt Gulbrandsen, <agulbra@nvg.unit.no>
19 * Jorge Cwik, <jorge@laser.satlink.net>
20 *
21 * Fixes:
22 * Alan Cox : Numerous verify_area() calls
23 * Alan Cox : Set the ACK bit on a reset
24 * Alan Cox : Stopped it crashing if it closed while
25 * sk->inuse=1 and was trying to connect
26 * (tcp_err()).
27 * Alan Cox : All icmp error handling was broken
28 * pointers passed where wrong and the
29 * socket was looked up backwards. Nobody
30 * tested any icmp error code obviously.
31 * Alan Cox : tcp_err() now handled properly. It
32 * wakes people on errors. poll
33 * behaves and the icmp error race
34 * has gone by moving it into sock.c
35 * Alan Cox : tcp_send_reset() fixed to work for
36 * everything not just packets for
37 * unknown sockets.
38 * Alan Cox : tcp option processing.
39 * Alan Cox : Reset tweaked (still not 100%) [Had
40 * syn rule wrong]
41 * Herp Rosmanith : More reset fixes
42 * Alan Cox : No longer acks invalid rst frames.
43 * Acking any kind of RST is right out.
44 * Alan Cox : Sets an ignore me flag on an rst
45 * receive otherwise odd bits of prattle
46 * escape still
47 * Alan Cox : Fixed another acking RST frame bug.
48 * Should stop LAN workplace lockups.
49 * Alan Cox : Some tidyups using the new skb list
50 * facilities
51 * Alan Cox : sk->keepopen now seems to work
52 * Alan Cox : Pulls options out correctly on accepts
53 * Alan Cox : Fixed assorted sk->rqueue->next errors
54 * Alan Cox : PSH doesn't end a TCP read. Switched a
55 * bit to skb ops.
56 * Alan Cox : Tidied tcp_data to avoid a potential
57 * nasty.
58 * Alan Cox : Added some better commenting, as the
59 * tcp is hard to follow
60 * Alan Cox : Removed incorrect check for 20 * psh
61 * Michael O'Reilly : ack < copied bug fix.
62 * Johannes Stille : Misc tcp fixes (not all in yet).
63 * Alan Cox : FIN with no memory -> CRASH
64 * Alan Cox : Added socket option proto entries.
65 * Also added awareness of them to accept.
66 * Alan Cox : Added TCP options (SOL_TCP)
67 * Alan Cox : Switched wakeup calls to callbacks,
68 * so the kernel can layer network
69 * sockets.
70 * Alan Cox : Use ip_tos/ip_ttl settings.
71 * Alan Cox : Handle FIN (more) properly (we hope).
72 * Alan Cox : RST frames sent on unsynchronised
73 * state ack error.
74 * Alan Cox : Put in missing check for SYN bit.
75 * Alan Cox : Added tcp_select_window() aka NET2E
76 * window non shrink trick.
77 * Alan Cox : Added a couple of small NET2E timer
78 * fixes
79 * Charles Hedrick : TCP fixes
80 * Toomas Tamm : TCP window fixes
81 * Alan Cox : Small URG fix to rlogin ^C ack fight
82 * Charles Hedrick : Rewrote most of it to actually work
83 * Linus : Rewrote tcp_read() and URG handling
84 * completely
85 * Gerhard Koerting: Fixed some missing timer handling
86 * Matthew Dillon : Reworked TCP machine states as per RFC
87 * Gerhard Koerting: PC/TCP workarounds
88 * Adam Caldwell : Assorted timer/timing errors
89 * Matthew Dillon : Fixed another RST bug
90 * Alan Cox : Move to kernel side addressing changes.
91 * Alan Cox : Beginning work on TCP fastpathing
92 * (not yet usable)
93 * Arnt Gulbrandsen: Turbocharged tcp_check() routine.
94 * Alan Cox : TCP fast path debugging
95 * Alan Cox : Window clamping
96 * Michael Riepe : Bug in tcp_check()
97 * Matt Dillon : More TCP improvements and RST bug fixes
98 * Matt Dillon : Yet more small nasties remove from the
99 * TCP code (Be very nice to this man if
100 * tcp finally works 100%) 8)
101 * Alan Cox : BSD accept semantics.
102 * Alan Cox : Reset on closedown bug.
103 * Peter De Schrijver : ENOTCONN check missing in tcp_sendto().
104 * Michael Pall : Handle poll() after URG properly in
105 * all cases.
106 * Michael Pall : Undo the last fix in tcp_read_urg()
107 * (multi URG PUSH broke rlogin).
108 * Michael Pall : Fix the multi URG PUSH problem in
109 * tcp_readable(), poll() after URG
110 * works now.
111 * Michael Pall : recv(...,MSG_OOB) never blocks in the
112 * BSD api.
113 * Alan Cox : Changed the semantics of sk->socket to
114 * fix a race and a signal problem with
115 * accept() and async I/O.
116 * Alan Cox : Relaxed the rules on tcp_sendto().
117 * Yury Shevchuk : Really fixed accept() blocking problem.
118 * Craig I. Hagan : Allow for BSD compatible TIME_WAIT for
119 * clients/servers which listen in on
120 * fixed ports.
121 * Alan Cox : Cleaned the above up and shrank it to
122 * a sensible code size.
123 * Alan Cox : Self connect lockup fix.
124 * Alan Cox : No connect to multicast.
125 * Ross Biro : Close unaccepted children on master
126 * socket close.
127 * Alan Cox : Reset tracing code.
128 * Alan Cox : Spurious resets on shutdown.
129 * Alan Cox : Giant 15 minute/60 second timer error
130 * Alan Cox : Small whoops in polling before an
131 * accept.
132 * Alan Cox : Kept the state trace facility since
133 * it's handy for debugging.
134 * Alan Cox : More reset handler fixes.
135 * Alan Cox : Started rewriting the code based on
136 * the RFC's for other useful protocol
137 * references see: Comer, KA9Q NOS, and
138 * for a reference on the difference
139 * between specifications and how BSD
140 * works see the 4.4lite source.
141 * A.N.Kuznetsov : Don't time wait on completion of tidy
142 * close.
143 * Linus Torvalds : Fin/Shutdown & copied_seq changes.
144 * Linus Torvalds : Fixed BSD port reuse to work first syn
145 * Alan Cox : Reimplemented timers as per the RFC
146 * and using multiple timers for sanity.
147 * Alan Cox : Small bug fixes, and a lot of new
148 * comments.
149 * Alan Cox : Fixed dual reader crash by locking
150 * the buffers (much like datagram.c)
151 * Alan Cox : Fixed stuck sockets in probe. A probe
152 * now gets fed up of retrying without
153 * (even a no space) answer.
154 * Alan Cox : Extracted closing code better
155 * Alan Cox : Fixed the closing state machine to
156 * resemble the RFC.
157 * Alan Cox : More 'per spec' fixes.
158 * Jorge Cwik : Even faster checksumming.
159 * Alan Cox : tcp_data() doesn't ack illegal PSH
160 * only frames. At least one pc tcp stack
161 * generates them.
162 * Alan Cox : Cache last socket.
163 * Alan Cox : Per route irtt.
164 * Matt Day : poll()->select() match BSD precisely on error
165 * Alan Cox : New buffers
166 * Marc Tamsky : Various sk->prot->retransmits and
167 * sk->retransmits misupdating fixed.
168 * Fixed tcp_write_timeout: stuck close,
169 * and TCP syn retries gets used now.
170 * Mark Yarvis : In tcp_read_wakeup(), don't send an
171 * ack if state is TCP_CLOSED.
172 * Alan Cox : Look up device on a retransmit - routes may
173 * change. Doesn't yet cope with MSS shrink right
174 * but it's a start!
175 * Marc Tamsky : Closing in closing fixes.
176 * Mike Shaver : RFC1122 verifications.
177 * Alan Cox : rcv_saddr errors.
178 * Alan Cox : Block double connect().
179 * Alan Cox : Small hooks for enSKIP.
180 * Alexey Kuznetsov: Path MTU discovery.
181 * Alan Cox : Support soft errors.
182 * Alan Cox : Fix MTU discovery pathological case
183 * when the remote claims no mtu!
184 * Marc Tamsky : TCP_CLOSE fix.
185 * Colin (G3TNE) : Send a reset on syn ack replies in
186 * window but wrong (fixes NT lpd problems)
187 * Pedro Roque : Better TCP window handling, delayed ack.
188 * Joerg Reuter : No modification of locked buffers in
189 * tcp_do_retransmit()
190 * Eric Schenk : Changed receiver side silly window
191 * avoidance algorithm to BSD style
192 * algorithm. This doubles throughput
193 * against machines running Solaris,
194 * and seems to result in general
195 * improvement.
196 * Stefan Magdalinski : adjusted tcp_readable() to fix FIONREAD
197 * Willy Konynenberg : Transparent proxying support.
198 * Mike McLagan : Routing by source
199 * Keith Owens : Do proper merging with partial SKB's in
200 * tcp_do_sendmsg to avoid burstiness.
201 * Eric Schenk : Fix fast close down bug with
202 * shutdown() followed by close().
203 * Andi Kleen : Make poll agree with SIGIO
204 * Salvatore Sanfilippo : Support SO_LINGER with linger == 1 and
205 * lingertime == 0 (RFC 793 ABORT Call)
206 * Hirokazu Takahashi : Use copy_from_user() instead of
207 * csum_and_copy_from_user() if possible.
208 *
209 * Description of States:
210 *
211 * TCP_SYN_SENT sent a connection request, waiting for ack
212 *
213 * TCP_SYN_RECV received a connection request, sent ack,
214 * waiting for final ack in three-way handshake.
215 *
216 * TCP_ESTABLISHED connection established
217 *
218 * TCP_FIN_WAIT1 our side has shutdown, waiting to complete
219 * transmission of remaining buffered data
220 *
221 * TCP_FIN_WAIT2 all buffered data sent, waiting for remote
222 * to shutdown
223 *
224 * TCP_CLOSING both sides have shutdown but we still have
225 * data we have to finish sending
226 *
227 * TCP_TIME_WAIT timeout to catch resent junk before entering
228 * closed, can only be entered from FIN_WAIT2
229 * or CLOSING. Required because the other end
230 * may not have gotten our last ACK causing it
231 * to retransmit the data packet (which we ignore)
232 *
233 * TCP_CLOSE_WAIT remote side has shutdown and is waiting for
234 * us to finish writing our data and to shutdown
235 * (we have to close() to move on to LAST_ACK)
236 *
237 * TCP_LAST_ACK out side has shutdown after remote has
238 * shutdown. There may still be data in our
239 * buffer that we have to finish sending
240 *
241 * TCP_CLOSE socket is finished
242 */
243
244 #define pr_fmt(fmt) "TCP: " fmt
245
246 #include <crypto/hash.h>
247 #include <linux/kernel.h>
248 #include <linux/module.h>
249 #include <linux/types.h>
250 #include <linux/fcntl.h>
251 #include <linux/poll.h>
252 #include <linux/inet_diag.h>
253 #include <linux/init.h>
254 #include <linux/fs.h>
255 #include <linux/skbuff.h>
256 #include <linux/scatterlist.h>
257 #include <linux/splice.h>
258 #include <linux/net.h>
259 #include <linux/socket.h>
260 #include <linux/random.h>
261 #include <linux/memblock.h>
262 #include <linux/highmem.h>
263 #include <linux/cache.h>
264 #include <linux/err.h>
265 #include <linux/time.h>
266 #include <linux/slab.h>
267 #include <linux/errqueue.h>
268 #include <linux/static_key.h>
269 #include <linux/btf.h>
270
271 #include <net/icmp.h>
272 #include <net/inet_common.h>
273 #include <net/tcp.h>
274 #include <net/mptcp.h>
275 #include <net/proto_memory.h>
276 #include <net/xfrm.h>
277 #include <net/ip.h>
278 #include <net/sock.h>
279 #include <net/rstreason.h>
280
281 #include <linux/uaccess.h>
282 #include <asm/ioctls.h>
283 #include <net/busy_poll.h>
284 #include <net/hotdata.h>
285 #include <net/rps.h>
286
287 /* Track pending CMSGs. */
288 enum {
289 TCP_CMSG_INQ = 1,
290 TCP_CMSG_TS = 2
291 };
292
293 DEFINE_PER_CPU(unsigned int, tcp_orphan_count);
294 EXPORT_PER_CPU_SYMBOL_GPL(tcp_orphan_count);
295
296 DEFINE_PER_CPU(u32, tcp_tw_isn);
297 EXPORT_PER_CPU_SYMBOL_GPL(tcp_tw_isn);
298
299 long sysctl_tcp_mem[3] __read_mostly;
300 EXPORT_SYMBOL(sysctl_tcp_mem);
301
302 atomic_long_t tcp_memory_allocated ____cacheline_aligned_in_smp; /* Current allocated memory. */
303 EXPORT_SYMBOL(tcp_memory_allocated);
304 DEFINE_PER_CPU(int, tcp_memory_per_cpu_fw_alloc);
305 EXPORT_PER_CPU_SYMBOL_GPL(tcp_memory_per_cpu_fw_alloc);
306
307 #if IS_ENABLED(CONFIG_SMC)
308 DEFINE_STATIC_KEY_FALSE(tcp_have_smc);
309 EXPORT_SYMBOL(tcp_have_smc);
310 #endif
311
312 /*
313 * Current number of TCP sockets.
314 */
315 struct percpu_counter tcp_sockets_allocated ____cacheline_aligned_in_smp;
316 EXPORT_SYMBOL(tcp_sockets_allocated);
317
318 /*
319 * TCP splice context
320 */
321 struct tcp_splice_state {
322 struct pipe_inode_info *pipe;
323 size_t len;
324 unsigned int flags;
325 };
326
327 /*
328 * Pressure flag: try to collapse.
329 * Technical note: it is used by multiple contexts non atomically.
330 * All the __sk_mem_schedule() is of this nature: accounting
331 * is strict, actions are advisory and have some latency.
332 */
333 unsigned long tcp_memory_pressure __read_mostly;
334 EXPORT_SYMBOL_GPL(tcp_memory_pressure);
335
tcp_enter_memory_pressure(struct sock * sk)336 void tcp_enter_memory_pressure(struct sock *sk)
337 {
338 unsigned long val;
339
340 if (READ_ONCE(tcp_memory_pressure))
341 return;
342 val = jiffies;
343
344 if (!val)
345 val--;
346 if (!cmpxchg(&tcp_memory_pressure, 0, val))
347 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURES);
348 }
349 EXPORT_SYMBOL_GPL(tcp_enter_memory_pressure);
350
tcp_leave_memory_pressure(struct sock * sk)351 void tcp_leave_memory_pressure(struct sock *sk)
352 {
353 unsigned long val;
354
355 if (!READ_ONCE(tcp_memory_pressure))
356 return;
357 val = xchg(&tcp_memory_pressure, 0);
358 if (val)
359 NET_ADD_STATS(sock_net(sk), LINUX_MIB_TCPMEMORYPRESSURESCHRONO,
360 jiffies_to_msecs(jiffies - val));
361 }
362 EXPORT_SYMBOL_GPL(tcp_leave_memory_pressure);
363
364 /* Convert seconds to retransmits based on initial and max timeout */
secs_to_retrans(int seconds,int timeout,int rto_max)365 static u8 secs_to_retrans(int seconds, int timeout, int rto_max)
366 {
367 u8 res = 0;
368
369 if (seconds > 0) {
370 int period = timeout;
371
372 res = 1;
373 while (seconds > period && res < 255) {
374 res++;
375 timeout <<= 1;
376 if (timeout > rto_max)
377 timeout = rto_max;
378 period += timeout;
379 }
380 }
381 return res;
382 }
383
384 /* Convert retransmits to seconds based on initial and max timeout */
retrans_to_secs(u8 retrans,int timeout,int rto_max)385 static int retrans_to_secs(u8 retrans, int timeout, int rto_max)
386 {
387 int period = 0;
388
389 if (retrans > 0) {
390 period = timeout;
391 while (--retrans) {
392 timeout <<= 1;
393 if (timeout > rto_max)
394 timeout = rto_max;
395 period += timeout;
396 }
397 }
398 return period;
399 }
400
tcp_compute_delivery_rate(const struct tcp_sock * tp)401 static u64 tcp_compute_delivery_rate(const struct tcp_sock *tp)
402 {
403 u32 rate = READ_ONCE(tp->rate_delivered);
404 u32 intv = READ_ONCE(tp->rate_interval_us);
405 u64 rate64 = 0;
406
407 if (rate && intv) {
408 rate64 = (u64)rate * tp->mss_cache * USEC_PER_SEC;
409 do_div(rate64, intv);
410 }
411 return rate64;
412 }
413
414 /* Address-family independent initialization for a tcp_sock.
415 *
416 * NOTE: A lot of things set to zero explicitly by call to
417 * sk_alloc() so need not be done here.
418 */
tcp_init_sock(struct sock * sk)419 void tcp_init_sock(struct sock *sk)
420 {
421 struct inet_connection_sock *icsk = inet_csk(sk);
422 struct tcp_sock *tp = tcp_sk(sk);
423
424 tp->out_of_order_queue = RB_ROOT;
425 sk->tcp_rtx_queue = RB_ROOT;
426 tcp_init_xmit_timers(sk);
427 INIT_LIST_HEAD(&tp->tsq_node);
428 INIT_LIST_HEAD(&tp->tsorted_sent_queue);
429
430 icsk->icsk_rto = TCP_TIMEOUT_INIT;
431 icsk->icsk_rto_min = TCP_RTO_MIN;
432 icsk->icsk_delack_max = TCP_DELACK_MAX;
433 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
434 minmax_reset(&tp->rtt_min, tcp_jiffies32, ~0U);
435
436 /* So many TCP implementations out there (incorrectly) count the
437 * initial SYN frame in their delayed-ACK and congestion control
438 * algorithms that we must have the following bandaid to talk
439 * efficiently to them. -DaveM
440 */
441 tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
442
443 /* There's a bubble in the pipe until at least the first ACK. */
444 tp->app_limited = ~0U;
445 tp->rate_app_limited = 1;
446
447 /* See draft-stevens-tcpca-spec-01 for discussion of the
448 * initialization of these values.
449 */
450 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
451 tp->snd_cwnd_clamp = ~0;
452 tp->mss_cache = TCP_MSS_DEFAULT;
453
454 tp->reordering = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_reordering);
455 tcp_assign_congestion_control(sk);
456
457 tp->tsoffset = 0;
458 tp->rack.reo_wnd_steps = 1;
459
460 sk->sk_write_space = sk_stream_write_space;
461 sock_set_flag(sk, SOCK_USE_WRITE_QUEUE);
462
463 icsk->icsk_sync_mss = tcp_sync_mss;
464
465 WRITE_ONCE(sk->sk_sndbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[1]));
466 WRITE_ONCE(sk->sk_rcvbuf, READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[1]));
467 tcp_scaling_ratio_init(sk);
468
469 set_bit(SOCK_SUPPORT_ZC, &sk->sk_socket->flags);
470 sk_sockets_allocated_inc(sk);
471 }
472 EXPORT_SYMBOL(tcp_init_sock);
473
tcp_tx_timestamp(struct sock * sk,u16 tsflags)474 static void tcp_tx_timestamp(struct sock *sk, u16 tsflags)
475 {
476 struct sk_buff *skb = tcp_write_queue_tail(sk);
477
478 if (tsflags && skb) {
479 struct skb_shared_info *shinfo = skb_shinfo(skb);
480 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
481
482 sock_tx_timestamp(sk, tsflags, &shinfo->tx_flags);
483 if (tsflags & SOF_TIMESTAMPING_TX_ACK)
484 tcb->txstamp_ack = 1;
485 if (tsflags & SOF_TIMESTAMPING_TX_RECORD_MASK)
486 shinfo->tskey = TCP_SKB_CB(skb)->seq + skb->len - 1;
487 }
488 }
489
tcp_stream_is_readable(struct sock * sk,int target)490 static bool tcp_stream_is_readable(struct sock *sk, int target)
491 {
492 if (tcp_epollin_ready(sk, target))
493 return true;
494 return sk_is_readable(sk);
495 }
496
497 /*
498 * Wait for a TCP event.
499 *
500 * Note that we don't need to lock the socket, as the upper poll layers
501 * take care of normal races (between the test and the event) and we don't
502 * go look at any of the socket buffers directly.
503 */
tcp_poll(struct file * file,struct socket * sock,poll_table * wait)504 __poll_t tcp_poll(struct file *file, struct socket *sock, poll_table *wait)
505 {
506 __poll_t mask;
507 struct sock *sk = sock->sk;
508 const struct tcp_sock *tp = tcp_sk(sk);
509 u8 shutdown;
510 int state;
511
512 sock_poll_wait(file, sock, wait);
513
514 state = inet_sk_state_load(sk);
515 if (state == TCP_LISTEN)
516 return inet_csk_listen_poll(sk);
517
518 /* Socket is not locked. We are protected from async events
519 * by poll logic and correct handling of state changes
520 * made by other threads is impossible in any case.
521 */
522
523 mask = 0;
524
525 /*
526 * EPOLLHUP is certainly not done right. But poll() doesn't
527 * have a notion of HUP in just one direction, and for a
528 * socket the read side is more interesting.
529 *
530 * Some poll() documentation says that EPOLLHUP is incompatible
531 * with the EPOLLOUT/POLLWR flags, so somebody should check this
532 * all. But careful, it tends to be safer to return too many
533 * bits than too few, and you can easily break real applications
534 * if you don't tell them that something has hung up!
535 *
536 * Check-me.
537 *
538 * Check number 1. EPOLLHUP is _UNMASKABLE_ event (see UNIX98 and
539 * our fs/select.c). It means that after we received EOF,
540 * poll always returns immediately, making impossible poll() on write()
541 * in state CLOSE_WAIT. One solution is evident --- to set EPOLLHUP
542 * if and only if shutdown has been made in both directions.
543 * Actually, it is interesting to look how Solaris and DUX
544 * solve this dilemma. I would prefer, if EPOLLHUP were maskable,
545 * then we could set it on SND_SHUTDOWN. BTW examples given
546 * in Stevens' books assume exactly this behaviour, it explains
547 * why EPOLLHUP is incompatible with EPOLLOUT. --ANK
548 *
549 * NOTE. Check for TCP_CLOSE is added. The goal is to prevent
550 * blocking on fresh not-connected or disconnected socket. --ANK
551 */
552 shutdown = READ_ONCE(sk->sk_shutdown);
553 if (shutdown == SHUTDOWN_MASK || state == TCP_CLOSE)
554 mask |= EPOLLHUP;
555 if (shutdown & RCV_SHUTDOWN)
556 mask |= EPOLLIN | EPOLLRDNORM | EPOLLRDHUP;
557
558 /* Connected or passive Fast Open socket? */
559 if (state != TCP_SYN_SENT &&
560 (state != TCP_SYN_RECV || rcu_access_pointer(tp->fastopen_rsk))) {
561 int target = sock_rcvlowat(sk, 0, INT_MAX);
562 u16 urg_data = READ_ONCE(tp->urg_data);
563
564 if (unlikely(urg_data) &&
565 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq) &&
566 !sock_flag(sk, SOCK_URGINLINE))
567 target++;
568
569 if (tcp_stream_is_readable(sk, target))
570 mask |= EPOLLIN | EPOLLRDNORM;
571
572 if (!(shutdown & SEND_SHUTDOWN)) {
573 if (__sk_stream_is_writeable(sk, 1)) {
574 mask |= EPOLLOUT | EPOLLWRNORM;
575 } else { /* send SIGIO later */
576 sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
577 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
578
579 /* Race breaker. If space is freed after
580 * wspace test but before the flags are set,
581 * IO signal will be lost. Memory barrier
582 * pairs with the input side.
583 */
584 smp_mb__after_atomic();
585 if (__sk_stream_is_writeable(sk, 1))
586 mask |= EPOLLOUT | EPOLLWRNORM;
587 }
588 } else
589 mask |= EPOLLOUT | EPOLLWRNORM;
590
591 if (urg_data & TCP_URG_VALID)
592 mask |= EPOLLPRI;
593 } else if (state == TCP_SYN_SENT &&
594 inet_test_bit(DEFER_CONNECT, sk)) {
595 /* Active TCP fastopen socket with defer_connect
596 * Return EPOLLOUT so application can call write()
597 * in order for kernel to generate SYN+data
598 */
599 mask |= EPOLLOUT | EPOLLWRNORM;
600 }
601 /* This barrier is coupled with smp_wmb() in tcp_reset() */
602 smp_rmb();
603 if (READ_ONCE(sk->sk_err) ||
604 !skb_queue_empty_lockless(&sk->sk_error_queue))
605 mask |= EPOLLERR;
606
607 return mask;
608 }
609 EXPORT_SYMBOL(tcp_poll);
610
tcp_ioctl(struct sock * sk,int cmd,int * karg)611 int tcp_ioctl(struct sock *sk, int cmd, int *karg)
612 {
613 struct tcp_sock *tp = tcp_sk(sk);
614 int answ;
615 bool slow;
616
617 switch (cmd) {
618 case SIOCINQ:
619 if (sk->sk_state == TCP_LISTEN)
620 return -EINVAL;
621
622 slow = lock_sock_fast(sk);
623 answ = tcp_inq(sk);
624 unlock_sock_fast(sk, slow);
625 break;
626 case SIOCATMARK:
627 answ = READ_ONCE(tp->urg_data) &&
628 READ_ONCE(tp->urg_seq) == READ_ONCE(tp->copied_seq);
629 break;
630 case SIOCOUTQ:
631 if (sk->sk_state == TCP_LISTEN)
632 return -EINVAL;
633
634 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
635 answ = 0;
636 else
637 answ = READ_ONCE(tp->write_seq) - tp->snd_una;
638 break;
639 case SIOCOUTQNSD:
640 if (sk->sk_state == TCP_LISTEN)
641 return -EINVAL;
642
643 if ((1 << sk->sk_state) & (TCPF_SYN_SENT | TCPF_SYN_RECV))
644 answ = 0;
645 else
646 answ = READ_ONCE(tp->write_seq) -
647 READ_ONCE(tp->snd_nxt);
648 break;
649 default:
650 return -ENOIOCTLCMD;
651 }
652
653 *karg = answ;
654 return 0;
655 }
656 EXPORT_SYMBOL(tcp_ioctl);
657
tcp_mark_push(struct tcp_sock * tp,struct sk_buff * skb)658 void tcp_mark_push(struct tcp_sock *tp, struct sk_buff *skb)
659 {
660 TCP_SKB_CB(skb)->tcp_flags |= TCPHDR_PSH;
661 tp->pushed_seq = tp->write_seq;
662 }
663
forced_push(const struct tcp_sock * tp)664 static inline bool forced_push(const struct tcp_sock *tp)
665 {
666 return after(tp->write_seq, tp->pushed_seq + (tp->max_window >> 1));
667 }
668
tcp_skb_entail(struct sock * sk,struct sk_buff * skb)669 void tcp_skb_entail(struct sock *sk, struct sk_buff *skb)
670 {
671 struct tcp_sock *tp = tcp_sk(sk);
672 struct tcp_skb_cb *tcb = TCP_SKB_CB(skb);
673
674 tcb->seq = tcb->end_seq = tp->write_seq;
675 tcb->tcp_flags = TCPHDR_ACK;
676 __skb_header_release(skb);
677 tcp_add_write_queue_tail(sk, skb);
678 sk_wmem_queued_add(sk, skb->truesize);
679 sk_mem_charge(sk, skb->truesize);
680 if (tp->nonagle & TCP_NAGLE_PUSH)
681 tp->nonagle &= ~TCP_NAGLE_PUSH;
682
683 tcp_slow_start_after_idle_check(sk);
684 }
685
tcp_mark_urg(struct tcp_sock * tp,int flags)686 static inline void tcp_mark_urg(struct tcp_sock *tp, int flags)
687 {
688 if (flags & MSG_OOB)
689 tp->snd_up = tp->write_seq;
690 }
691
692 /* If a not yet filled skb is pushed, do not send it if
693 * we have data packets in Qdisc or NIC queues :
694 * Because TX completion will happen shortly, it gives a chance
695 * to coalesce future sendmsg() payload into this skb, without
696 * need for a timer, and with no latency trade off.
697 * As packets containing data payload have a bigger truesize
698 * than pure acks (dataless) packets, the last checks prevent
699 * autocorking if we only have an ACK in Qdisc/NIC queues,
700 * or if TX completion was delayed after we processed ACK packet.
701 */
tcp_should_autocork(struct sock * sk,struct sk_buff * skb,int size_goal)702 static bool tcp_should_autocork(struct sock *sk, struct sk_buff *skb,
703 int size_goal)
704 {
705 return skb->len < size_goal &&
706 READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_autocorking) &&
707 !tcp_rtx_queue_empty(sk) &&
708 refcount_read(&sk->sk_wmem_alloc) > skb->truesize &&
709 tcp_skb_can_collapse_to(skb);
710 }
711
tcp_push(struct sock * sk,int flags,int mss_now,int nonagle,int size_goal)712 void tcp_push(struct sock *sk, int flags, int mss_now,
713 int nonagle, int size_goal)
714 {
715 struct tcp_sock *tp = tcp_sk(sk);
716 struct sk_buff *skb;
717
718 skb = tcp_write_queue_tail(sk);
719 if (!skb)
720 return;
721 if (!(flags & MSG_MORE) || forced_push(tp))
722 tcp_mark_push(tp, skb);
723
724 tcp_mark_urg(tp, flags);
725
726 if (tcp_should_autocork(sk, skb, size_goal)) {
727
728 /* avoid atomic op if TSQ_THROTTLED bit is already set */
729 if (!test_bit(TSQ_THROTTLED, &sk->sk_tsq_flags)) {
730 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPAUTOCORKING);
731 set_bit(TSQ_THROTTLED, &sk->sk_tsq_flags);
732 smp_mb__after_atomic();
733 }
734 /* It is possible TX completion already happened
735 * before we set TSQ_THROTTLED.
736 */
737 if (refcount_read(&sk->sk_wmem_alloc) > skb->truesize)
738 return;
739 }
740
741 if (flags & MSG_MORE)
742 nonagle = TCP_NAGLE_CORK;
743
744 __tcp_push_pending_frames(sk, mss_now, nonagle);
745 }
746
tcp_splice_data_recv(read_descriptor_t * rd_desc,struct sk_buff * skb,unsigned int offset,size_t len)747 static int tcp_splice_data_recv(read_descriptor_t *rd_desc, struct sk_buff *skb,
748 unsigned int offset, size_t len)
749 {
750 struct tcp_splice_state *tss = rd_desc->arg.data;
751 int ret;
752
753 ret = skb_splice_bits(skb, skb->sk, offset, tss->pipe,
754 min(rd_desc->count, len), tss->flags);
755 if (ret > 0)
756 rd_desc->count -= ret;
757 return ret;
758 }
759
__tcp_splice_read(struct sock * sk,struct tcp_splice_state * tss)760 static int __tcp_splice_read(struct sock *sk, struct tcp_splice_state *tss)
761 {
762 /* Store TCP splice context information in read_descriptor_t. */
763 read_descriptor_t rd_desc = {
764 .arg.data = tss,
765 .count = tss->len,
766 };
767
768 return tcp_read_sock(sk, &rd_desc, tcp_splice_data_recv);
769 }
770
771 /**
772 * tcp_splice_read - splice data from TCP socket to a pipe
773 * @sock: socket to splice from
774 * @ppos: position (not valid)
775 * @pipe: pipe to splice to
776 * @len: number of bytes to splice
777 * @flags: splice modifier flags
778 *
779 * Description:
780 * Will read pages from given socket and fill them into a pipe.
781 *
782 **/
tcp_splice_read(struct socket * sock,loff_t * ppos,struct pipe_inode_info * pipe,size_t len,unsigned int flags)783 ssize_t tcp_splice_read(struct socket *sock, loff_t *ppos,
784 struct pipe_inode_info *pipe, size_t len,
785 unsigned int flags)
786 {
787 struct sock *sk = sock->sk;
788 struct tcp_splice_state tss = {
789 .pipe = pipe,
790 .len = len,
791 .flags = flags,
792 };
793 long timeo;
794 ssize_t spliced;
795 int ret;
796
797 sock_rps_record_flow(sk);
798 /*
799 * We can't seek on a socket input
800 */
801 if (unlikely(*ppos))
802 return -ESPIPE;
803
804 ret = spliced = 0;
805
806 lock_sock(sk);
807
808 timeo = sock_rcvtimeo(sk, sock->file->f_flags & O_NONBLOCK);
809 while (tss.len) {
810 ret = __tcp_splice_read(sk, &tss);
811 if (ret < 0)
812 break;
813 else if (!ret) {
814 if (spliced)
815 break;
816 if (sock_flag(sk, SOCK_DONE))
817 break;
818 if (sk->sk_err) {
819 ret = sock_error(sk);
820 break;
821 }
822 if (sk->sk_shutdown & RCV_SHUTDOWN)
823 break;
824 if (sk->sk_state == TCP_CLOSE) {
825 /*
826 * This occurs when user tries to read
827 * from never connected socket.
828 */
829 ret = -ENOTCONN;
830 break;
831 }
832 if (!timeo) {
833 ret = -EAGAIN;
834 break;
835 }
836 /* if __tcp_splice_read() got nothing while we have
837 * an skb in receive queue, we do not want to loop.
838 * This might happen with URG data.
839 */
840 if (!skb_queue_empty(&sk->sk_receive_queue))
841 break;
842 ret = sk_wait_data(sk, &timeo, NULL);
843 if (ret < 0)
844 break;
845 if (signal_pending(current)) {
846 ret = sock_intr_errno(timeo);
847 break;
848 }
849 continue;
850 }
851 tss.len -= ret;
852 spliced += ret;
853
854 if (!tss.len || !timeo)
855 break;
856 release_sock(sk);
857 lock_sock(sk);
858
859 if (sk->sk_err || sk->sk_state == TCP_CLOSE ||
860 (sk->sk_shutdown & RCV_SHUTDOWN) ||
861 signal_pending(current))
862 break;
863 }
864
865 release_sock(sk);
866
867 if (spliced)
868 return spliced;
869
870 return ret;
871 }
872 EXPORT_SYMBOL(tcp_splice_read);
873
tcp_stream_alloc_skb(struct sock * sk,gfp_t gfp,bool force_schedule)874 struct sk_buff *tcp_stream_alloc_skb(struct sock *sk, gfp_t gfp,
875 bool force_schedule)
876 {
877 struct sk_buff *skb;
878
879 skb = alloc_skb_fclone(MAX_TCP_HEADER, gfp);
880 if (likely(skb)) {
881 bool mem_scheduled;
882
883 skb->truesize = SKB_TRUESIZE(skb_end_offset(skb));
884 if (force_schedule) {
885 mem_scheduled = true;
886 sk_forced_mem_schedule(sk, skb->truesize);
887 } else {
888 mem_scheduled = sk_wmem_schedule(sk, skb->truesize);
889 }
890 if (likely(mem_scheduled)) {
891 skb_reserve(skb, MAX_TCP_HEADER);
892 skb->ip_summed = CHECKSUM_PARTIAL;
893 INIT_LIST_HEAD(&skb->tcp_tsorted_anchor);
894 return skb;
895 }
896 __kfree_skb(skb);
897 } else {
898 sk->sk_prot->enter_memory_pressure(sk);
899 sk_stream_moderate_sndbuf(sk);
900 }
901 return NULL;
902 }
903
tcp_xmit_size_goal(struct sock * sk,u32 mss_now,int large_allowed)904 static unsigned int tcp_xmit_size_goal(struct sock *sk, u32 mss_now,
905 int large_allowed)
906 {
907 struct tcp_sock *tp = tcp_sk(sk);
908 u32 new_size_goal, size_goal;
909
910 if (!large_allowed)
911 return mss_now;
912
913 /* Note : tcp_tso_autosize() will eventually split this later */
914 new_size_goal = tcp_bound_to_half_wnd(tp, sk->sk_gso_max_size);
915
916 /* We try hard to avoid divides here */
917 size_goal = tp->gso_segs * mss_now;
918 if (unlikely(new_size_goal < size_goal ||
919 new_size_goal >= size_goal + mss_now)) {
920 tp->gso_segs = min_t(u16, new_size_goal / mss_now,
921 sk->sk_gso_max_segs);
922 size_goal = tp->gso_segs * mss_now;
923 }
924
925 return max(size_goal, mss_now);
926 }
927
tcp_send_mss(struct sock * sk,int * size_goal,int flags)928 int tcp_send_mss(struct sock *sk, int *size_goal, int flags)
929 {
930 int mss_now;
931
932 mss_now = tcp_current_mss(sk);
933 *size_goal = tcp_xmit_size_goal(sk, mss_now, !(flags & MSG_OOB));
934
935 return mss_now;
936 }
937
938 /* In some cases, sendmsg() could have added an skb to the write queue,
939 * but failed adding payload on it. We need to remove it to consume less
940 * memory, but more importantly be able to generate EPOLLOUT for Edge Trigger
941 * epoll() users. Another reason is that tcp_write_xmit() does not like
942 * finding an empty skb in the write queue.
943 */
tcp_remove_empty_skb(struct sock * sk)944 void tcp_remove_empty_skb(struct sock *sk)
945 {
946 struct sk_buff *skb = tcp_write_queue_tail(sk);
947
948 if (skb && TCP_SKB_CB(skb)->seq == TCP_SKB_CB(skb)->end_seq) {
949 tcp_unlink_write_queue(skb, sk);
950 if (tcp_write_queue_empty(sk))
951 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
952 tcp_wmem_free_skb(sk, skb);
953 }
954 }
955
956 /* skb changing from pure zc to mixed, must charge zc */
tcp_downgrade_zcopy_pure(struct sock * sk,struct sk_buff * skb)957 static int tcp_downgrade_zcopy_pure(struct sock *sk, struct sk_buff *skb)
958 {
959 if (unlikely(skb_zcopy_pure(skb))) {
960 u32 extra = skb->truesize -
961 SKB_TRUESIZE(skb_end_offset(skb));
962
963 if (!sk_wmem_schedule(sk, extra))
964 return -ENOMEM;
965
966 sk_mem_charge(sk, extra);
967 skb_shinfo(skb)->flags &= ~SKBFL_PURE_ZEROCOPY;
968 }
969 return 0;
970 }
971
972
tcp_wmem_schedule(struct sock * sk,int copy)973 int tcp_wmem_schedule(struct sock *sk, int copy)
974 {
975 int left;
976
977 if (likely(sk_wmem_schedule(sk, copy)))
978 return copy;
979
980 /* We could be in trouble if we have nothing queued.
981 * Use whatever is left in sk->sk_forward_alloc and tcp_wmem[0]
982 * to guarantee some progress.
983 */
984 left = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_wmem[0]) - sk->sk_wmem_queued;
985 if (left > 0)
986 sk_forced_mem_schedule(sk, min(left, copy));
987 return min(copy, sk->sk_forward_alloc);
988 }
989
tcp_free_fastopen_req(struct tcp_sock * tp)990 void tcp_free_fastopen_req(struct tcp_sock *tp)
991 {
992 if (tp->fastopen_req) {
993 kfree(tp->fastopen_req);
994 tp->fastopen_req = NULL;
995 }
996 }
997
tcp_sendmsg_fastopen(struct sock * sk,struct msghdr * msg,int * copied,size_t size,struct ubuf_info * uarg)998 int tcp_sendmsg_fastopen(struct sock *sk, struct msghdr *msg, int *copied,
999 size_t size, struct ubuf_info *uarg)
1000 {
1001 struct tcp_sock *tp = tcp_sk(sk);
1002 struct inet_sock *inet = inet_sk(sk);
1003 struct sockaddr *uaddr = msg->msg_name;
1004 int err, flags;
1005
1006 if (!(READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_fastopen) &
1007 TFO_CLIENT_ENABLE) ||
1008 (uaddr && msg->msg_namelen >= sizeof(uaddr->sa_family) &&
1009 uaddr->sa_family == AF_UNSPEC))
1010 return -EOPNOTSUPP;
1011 if (tp->fastopen_req)
1012 return -EALREADY; /* Another Fast Open is in progress */
1013
1014 tp->fastopen_req = kzalloc(sizeof(struct tcp_fastopen_request),
1015 sk->sk_allocation);
1016 if (unlikely(!tp->fastopen_req))
1017 return -ENOBUFS;
1018 tp->fastopen_req->data = msg;
1019 tp->fastopen_req->size = size;
1020 tp->fastopen_req->uarg = uarg;
1021
1022 if (inet_test_bit(DEFER_CONNECT, sk)) {
1023 err = tcp_connect(sk);
1024 /* Same failure procedure as in tcp_v4/6_connect */
1025 if (err) {
1026 tcp_set_state(sk, TCP_CLOSE);
1027 inet->inet_dport = 0;
1028 sk->sk_route_caps = 0;
1029 }
1030 }
1031 flags = (msg->msg_flags & MSG_DONTWAIT) ? O_NONBLOCK : 0;
1032 err = __inet_stream_connect(sk->sk_socket, uaddr,
1033 msg->msg_namelen, flags, 1);
1034 /* fastopen_req could already be freed in __inet_stream_connect
1035 * if the connection times out or gets rst
1036 */
1037 if (tp->fastopen_req) {
1038 *copied = tp->fastopen_req->copied;
1039 tcp_free_fastopen_req(tp);
1040 inet_clear_bit(DEFER_CONNECT, sk);
1041 }
1042 return err;
1043 }
1044
tcp_sendmsg_locked(struct sock * sk,struct msghdr * msg,size_t size)1045 int tcp_sendmsg_locked(struct sock *sk, struct msghdr *msg, size_t size)
1046 {
1047 struct tcp_sock *tp = tcp_sk(sk);
1048 struct ubuf_info *uarg = NULL;
1049 struct sk_buff *skb;
1050 struct sockcm_cookie sockc;
1051 int flags, err, copied = 0;
1052 int mss_now = 0, size_goal, copied_syn = 0;
1053 int process_backlog = 0;
1054 int zc = 0;
1055 long timeo;
1056
1057 flags = msg->msg_flags;
1058
1059 if ((flags & MSG_ZEROCOPY) && size) {
1060 if (msg->msg_ubuf) {
1061 uarg = msg->msg_ubuf;
1062 if (sk->sk_route_caps & NETIF_F_SG)
1063 zc = MSG_ZEROCOPY;
1064 } else if (sock_flag(sk, SOCK_ZEROCOPY)) {
1065 skb = tcp_write_queue_tail(sk);
1066 uarg = msg_zerocopy_realloc(sk, size, skb_zcopy(skb));
1067 if (!uarg) {
1068 err = -ENOBUFS;
1069 goto out_err;
1070 }
1071 if (sk->sk_route_caps & NETIF_F_SG)
1072 zc = MSG_ZEROCOPY;
1073 else
1074 uarg_to_msgzc(uarg)->zerocopy = 0;
1075 }
1076 } else if (unlikely(msg->msg_flags & MSG_SPLICE_PAGES) && size) {
1077 if (sk->sk_route_caps & NETIF_F_SG)
1078 zc = MSG_SPLICE_PAGES;
1079 }
1080
1081 if (unlikely(flags & MSG_FASTOPEN ||
1082 inet_test_bit(DEFER_CONNECT, sk)) &&
1083 !tp->repair) {
1084 err = tcp_sendmsg_fastopen(sk, msg, &copied_syn, size, uarg);
1085 if (err == -EINPROGRESS && copied_syn > 0)
1086 goto out;
1087 else if (err)
1088 goto out_err;
1089 }
1090
1091 timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
1092
1093 tcp_rate_check_app_limited(sk); /* is sending application-limited? */
1094
1095 /* Wait for a connection to finish. One exception is TCP Fast Open
1096 * (passive side) where data is allowed to be sent before a connection
1097 * is fully established.
1098 */
1099 if (((1 << sk->sk_state) & ~(TCPF_ESTABLISHED | TCPF_CLOSE_WAIT)) &&
1100 !tcp_passive_fastopen(sk)) {
1101 err = sk_stream_wait_connect(sk, &timeo);
1102 if (err != 0)
1103 goto do_error;
1104 }
1105
1106 if (unlikely(tp->repair)) {
1107 if (tp->repair_queue == TCP_RECV_QUEUE) {
1108 copied = tcp_send_rcvq(sk, msg, size);
1109 goto out_nopush;
1110 }
1111
1112 err = -EINVAL;
1113 if (tp->repair_queue == TCP_NO_QUEUE)
1114 goto out_err;
1115
1116 /* 'common' sending to sendq */
1117 }
1118
1119 sockcm_init(&sockc, sk);
1120 if (msg->msg_controllen) {
1121 err = sock_cmsg_send(sk, msg, &sockc);
1122 if (unlikely(err)) {
1123 err = -EINVAL;
1124 goto out_err;
1125 }
1126 }
1127
1128 /* This should be in poll */
1129 sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
1130
1131 /* Ok commence sending. */
1132 copied = 0;
1133
1134 restart:
1135 mss_now = tcp_send_mss(sk, &size_goal, flags);
1136
1137 err = -EPIPE;
1138 if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN))
1139 goto do_error;
1140
1141 while (msg_data_left(msg)) {
1142 ssize_t copy = 0;
1143
1144 skb = tcp_write_queue_tail(sk);
1145 if (skb)
1146 copy = size_goal - skb->len;
1147
1148 if (copy <= 0 || !tcp_skb_can_collapse_to(skb)) {
1149 bool first_skb;
1150
1151 new_segment:
1152 if (!sk_stream_memory_free(sk))
1153 goto wait_for_space;
1154
1155 if (unlikely(process_backlog >= 16)) {
1156 process_backlog = 0;
1157 if (sk_flush_backlog(sk))
1158 goto restart;
1159 }
1160 first_skb = tcp_rtx_and_write_queues_empty(sk);
1161 skb = tcp_stream_alloc_skb(sk, sk->sk_allocation,
1162 first_skb);
1163 if (!skb)
1164 goto wait_for_space;
1165
1166 process_backlog++;
1167
1168 tcp_skb_entail(sk, skb);
1169 copy = size_goal;
1170
1171 /* All packets are restored as if they have
1172 * already been sent. skb_mstamp_ns isn't set to
1173 * avoid wrong rtt estimation.
1174 */
1175 if (tp->repair)
1176 TCP_SKB_CB(skb)->sacked |= TCPCB_REPAIRED;
1177 }
1178
1179 /* Try to append data to the end of skb. */
1180 if (copy > msg_data_left(msg))
1181 copy = msg_data_left(msg);
1182
1183 if (zc == 0) {
1184 bool merge = true;
1185 int i = skb_shinfo(skb)->nr_frags;
1186 struct page_frag *pfrag = sk_page_frag(sk);
1187
1188 if (!sk_page_frag_refill(sk, pfrag))
1189 goto wait_for_space;
1190
1191 if (!skb_can_coalesce(skb, i, pfrag->page,
1192 pfrag->offset)) {
1193 if (i >= READ_ONCE(net_hotdata.sysctl_max_skb_frags)) {
1194 tcp_mark_push(tp, skb);
1195 goto new_segment;
1196 }
1197 merge = false;
1198 }
1199
1200 copy = min_t(int, copy, pfrag->size - pfrag->offset);
1201
1202 if (unlikely(skb_zcopy_pure(skb) || skb_zcopy_managed(skb))) {
1203 if (tcp_downgrade_zcopy_pure(sk, skb))
1204 goto wait_for_space;
1205 skb_zcopy_downgrade_managed(skb);
1206 }
1207
1208 copy = tcp_wmem_schedule(sk, copy);
1209 if (!copy)
1210 goto wait_for_space;
1211
1212 err = skb_copy_to_page_nocache(sk, &msg->msg_iter, skb,
1213 pfrag->page,
1214 pfrag->offset,
1215 copy);
1216 if (err)
1217 goto do_error;
1218
1219 /* Update the skb. */
1220 if (merge) {
1221 skb_frag_size_add(&skb_shinfo(skb)->frags[i - 1], copy);
1222 } else {
1223 skb_fill_page_desc(skb, i, pfrag->page,
1224 pfrag->offset, copy);
1225 page_ref_inc(pfrag->page);
1226 }
1227 pfrag->offset += copy;
1228 } else if (zc == MSG_ZEROCOPY) {
1229 /* First append to a fragless skb builds initial
1230 * pure zerocopy skb
1231 */
1232 if (!skb->len)
1233 skb_shinfo(skb)->flags |= SKBFL_PURE_ZEROCOPY;
1234
1235 if (!skb_zcopy_pure(skb)) {
1236 copy = tcp_wmem_schedule(sk, copy);
1237 if (!copy)
1238 goto wait_for_space;
1239 }
1240
1241 err = skb_zerocopy_iter_stream(sk, skb, msg, copy, uarg);
1242 if (err == -EMSGSIZE || err == -EEXIST) {
1243 tcp_mark_push(tp, skb);
1244 goto new_segment;
1245 }
1246 if (err < 0)
1247 goto do_error;
1248 copy = err;
1249 } else if (zc == MSG_SPLICE_PAGES) {
1250 /* Splice in data if we can; copy if we can't. */
1251 if (tcp_downgrade_zcopy_pure(sk, skb))
1252 goto wait_for_space;
1253 copy = tcp_wmem_schedule(sk, copy);
1254 if (!copy)
1255 goto wait_for_space;
1256
1257 err = skb_splice_from_iter(skb, &msg->msg_iter, copy,
1258 sk->sk_allocation);
1259 if (err < 0) {
1260 if (err == -EMSGSIZE) {
1261 tcp_mark_push(tp, skb);
1262 goto new_segment;
1263 }
1264 goto do_error;
1265 }
1266 copy = err;
1267
1268 if (!(flags & MSG_NO_SHARED_FRAGS))
1269 skb_shinfo(skb)->flags |= SKBFL_SHARED_FRAG;
1270
1271 sk_wmem_queued_add(sk, copy);
1272 sk_mem_charge(sk, copy);
1273 }
1274
1275 if (!copied)
1276 TCP_SKB_CB(skb)->tcp_flags &= ~TCPHDR_PSH;
1277
1278 WRITE_ONCE(tp->write_seq, tp->write_seq + copy);
1279 TCP_SKB_CB(skb)->end_seq += copy;
1280 tcp_skb_pcount_set(skb, 0);
1281
1282 copied += copy;
1283 if (!msg_data_left(msg)) {
1284 if (unlikely(flags & MSG_EOR))
1285 TCP_SKB_CB(skb)->eor = 1;
1286 goto out;
1287 }
1288
1289 if (skb->len < size_goal || (flags & MSG_OOB) || unlikely(tp->repair))
1290 continue;
1291
1292 if (forced_push(tp)) {
1293 tcp_mark_push(tp, skb);
1294 __tcp_push_pending_frames(sk, mss_now, TCP_NAGLE_PUSH);
1295 } else if (skb == tcp_send_head(sk))
1296 tcp_push_one(sk, mss_now);
1297 continue;
1298
1299 wait_for_space:
1300 set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
1301 tcp_remove_empty_skb(sk);
1302 if (copied)
1303 tcp_push(sk, flags & ~MSG_MORE, mss_now,
1304 TCP_NAGLE_PUSH, size_goal);
1305
1306 err = sk_stream_wait_memory(sk, &timeo);
1307 if (err != 0)
1308 goto do_error;
1309
1310 mss_now = tcp_send_mss(sk, &size_goal, flags);
1311 }
1312
1313 out:
1314 if (copied) {
1315 tcp_tx_timestamp(sk, sockc.tsflags);
1316 tcp_push(sk, flags, mss_now, tp->nonagle, size_goal);
1317 }
1318 out_nopush:
1319 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1320 if (uarg && !msg->msg_ubuf)
1321 net_zcopy_put(uarg);
1322 return copied + copied_syn;
1323
1324 do_error:
1325 tcp_remove_empty_skb(sk);
1326
1327 if (copied + copied_syn)
1328 goto out;
1329 out_err:
1330 /* msg->msg_ubuf is pinned by the caller so we don't take extra refs */
1331 if (uarg && !msg->msg_ubuf)
1332 net_zcopy_put_abort(uarg, true);
1333 err = sk_stream_error(sk, flags, err);
1334 /* make sure we wake any epoll edge trigger waiter */
1335 if (unlikely(tcp_rtx_and_write_queues_empty(sk) && err == -EAGAIN)) {
1336 sk->sk_write_space(sk);
1337 tcp_chrono_stop(sk, TCP_CHRONO_SNDBUF_LIMITED);
1338 }
1339 return err;
1340 }
1341 EXPORT_SYMBOL_GPL(tcp_sendmsg_locked);
1342
tcp_sendmsg(struct sock * sk,struct msghdr * msg,size_t size)1343 int tcp_sendmsg(struct sock *sk, struct msghdr *msg, size_t size)
1344 {
1345 int ret;
1346
1347 lock_sock(sk);
1348 ret = tcp_sendmsg_locked(sk, msg, size);
1349 release_sock(sk);
1350
1351 return ret;
1352 }
1353 EXPORT_SYMBOL(tcp_sendmsg);
1354
tcp_splice_eof(struct socket * sock)1355 void tcp_splice_eof(struct socket *sock)
1356 {
1357 struct sock *sk = sock->sk;
1358 struct tcp_sock *tp = tcp_sk(sk);
1359 int mss_now, size_goal;
1360
1361 if (!tcp_write_queue_tail(sk))
1362 return;
1363
1364 lock_sock(sk);
1365 mss_now = tcp_send_mss(sk, &size_goal, 0);
1366 tcp_push(sk, 0, mss_now, tp->nonagle, size_goal);
1367 release_sock(sk);
1368 }
1369 EXPORT_SYMBOL_GPL(tcp_splice_eof);
1370
1371 /*
1372 * Handle reading urgent data. BSD has very simple semantics for
1373 * this, no blocking and very strange errors 8)
1374 */
1375
tcp_recv_urg(struct sock * sk,struct msghdr * msg,int len,int flags)1376 static int tcp_recv_urg(struct sock *sk, struct msghdr *msg, int len, int flags)
1377 {
1378 struct tcp_sock *tp = tcp_sk(sk);
1379
1380 /* No URG data to read. */
1381 if (sock_flag(sk, SOCK_URGINLINE) || !tp->urg_data ||
1382 tp->urg_data == TCP_URG_READ)
1383 return -EINVAL; /* Yes this is right ! */
1384
1385 if (sk->sk_state == TCP_CLOSE && !sock_flag(sk, SOCK_DONE))
1386 return -ENOTCONN;
1387
1388 if (tp->urg_data & TCP_URG_VALID) {
1389 int err = 0;
1390 char c = tp->urg_data;
1391
1392 if (!(flags & MSG_PEEK))
1393 WRITE_ONCE(tp->urg_data, TCP_URG_READ);
1394
1395 /* Read urgent data. */
1396 msg->msg_flags |= MSG_OOB;
1397
1398 if (len > 0) {
1399 if (!(flags & MSG_TRUNC))
1400 err = memcpy_to_msg(msg, &c, 1);
1401 len = 1;
1402 } else
1403 msg->msg_flags |= MSG_TRUNC;
1404
1405 return err ? -EFAULT : len;
1406 }
1407
1408 if (sk->sk_state == TCP_CLOSE || (sk->sk_shutdown & RCV_SHUTDOWN))
1409 return 0;
1410
1411 /* Fixed the recv(..., MSG_OOB) behaviour. BSD docs and
1412 * the available implementations agree in this case:
1413 * this call should never block, independent of the
1414 * blocking state of the socket.
1415 * Mike <pall@rz.uni-karlsruhe.de>
1416 */
1417 return -EAGAIN;
1418 }
1419
tcp_peek_sndq(struct sock * sk,struct msghdr * msg,int len)1420 static int tcp_peek_sndq(struct sock *sk, struct msghdr *msg, int len)
1421 {
1422 struct sk_buff *skb;
1423 int copied = 0, err = 0;
1424
1425 skb_rbtree_walk(skb, &sk->tcp_rtx_queue) {
1426 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1427 if (err)
1428 return err;
1429 copied += skb->len;
1430 }
1431
1432 skb_queue_walk(&sk->sk_write_queue, skb) {
1433 err = skb_copy_datagram_msg(skb, 0, msg, skb->len);
1434 if (err)
1435 break;
1436
1437 copied += skb->len;
1438 }
1439
1440 return err ?: copied;
1441 }
1442
1443 /* Clean up the receive buffer for full frames taken by the user,
1444 * then send an ACK if necessary. COPIED is the number of bytes
1445 * tcp_recvmsg has given to the user so far, it speeds up the
1446 * calculation of whether or not we must ACK for the sake of
1447 * a window update.
1448 */
__tcp_cleanup_rbuf(struct sock * sk,int copied)1449 void __tcp_cleanup_rbuf(struct sock *sk, int copied)
1450 {
1451 struct tcp_sock *tp = tcp_sk(sk);
1452 bool time_to_ack = false;
1453
1454 if (inet_csk_ack_scheduled(sk)) {
1455 const struct inet_connection_sock *icsk = inet_csk(sk);
1456
1457 if (/* Once-per-two-segments ACK was not sent by tcp_input.c */
1458 tp->rcv_nxt - tp->rcv_wup > icsk->icsk_ack.rcv_mss ||
1459 /*
1460 * If this read emptied read buffer, we send ACK, if
1461 * connection is not bidirectional, user drained
1462 * receive buffer and there was a small segment
1463 * in queue.
1464 */
1465 (copied > 0 &&
1466 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED2) ||
1467 ((icsk->icsk_ack.pending & ICSK_ACK_PUSHED) &&
1468 !inet_csk_in_pingpong_mode(sk))) &&
1469 !atomic_read(&sk->sk_rmem_alloc)))
1470 time_to_ack = true;
1471 }
1472
1473 /* We send an ACK if we can now advertise a non-zero window
1474 * which has been raised "significantly".
1475 *
1476 * Even if window raised up to infinity, do not send window open ACK
1477 * in states, where we will not receive more. It is useless.
1478 */
1479 if (copied > 0 && !time_to_ack && !(sk->sk_shutdown & RCV_SHUTDOWN)) {
1480 __u32 rcv_window_now = tcp_receive_window(tp);
1481
1482 /* Optimize, __tcp_select_window() is not cheap. */
1483 if (2*rcv_window_now <= tp->window_clamp) {
1484 __u32 new_window = __tcp_select_window(sk);
1485
1486 /* Send ACK now, if this read freed lots of space
1487 * in our buffer. Certainly, new_window is new window.
1488 * We can advertise it now, if it is not less than current one.
1489 * "Lots" means "at least twice" here.
1490 */
1491 if (new_window && new_window >= 2 * rcv_window_now)
1492 time_to_ack = true;
1493 }
1494 }
1495 if (time_to_ack)
1496 tcp_send_ack(sk);
1497 }
1498
tcp_cleanup_rbuf(struct sock * sk,int copied)1499 void tcp_cleanup_rbuf(struct sock *sk, int copied)
1500 {
1501 struct sk_buff *skb = skb_peek(&sk->sk_receive_queue);
1502 struct tcp_sock *tp = tcp_sk(sk);
1503
1504 WARN(skb && !before(tp->copied_seq, TCP_SKB_CB(skb)->end_seq),
1505 "cleanup rbuf bug: copied %X seq %X rcvnxt %X\n",
1506 tp->copied_seq, TCP_SKB_CB(skb)->end_seq, tp->rcv_nxt);
1507 __tcp_cleanup_rbuf(sk, copied);
1508 }
1509
tcp_eat_recv_skb(struct sock * sk,struct sk_buff * skb)1510 static void tcp_eat_recv_skb(struct sock *sk, struct sk_buff *skb)
1511 {
1512 __skb_unlink(skb, &sk->sk_receive_queue);
1513 if (likely(skb->destructor == sock_rfree)) {
1514 sock_rfree(skb);
1515 skb->destructor = NULL;
1516 skb->sk = NULL;
1517 return skb_attempt_defer_free(skb);
1518 }
1519 __kfree_skb(skb);
1520 }
1521
tcp_recv_skb(struct sock * sk,u32 seq,u32 * off)1522 struct sk_buff *tcp_recv_skb(struct sock *sk, u32 seq, u32 *off)
1523 {
1524 struct sk_buff *skb;
1525 u32 offset;
1526
1527 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1528 offset = seq - TCP_SKB_CB(skb)->seq;
1529 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
1530 pr_err_once("%s: found a SYN, please report !\n", __func__);
1531 offset--;
1532 }
1533 if (offset < skb->len || (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)) {
1534 *off = offset;
1535 return skb;
1536 }
1537 /* This looks weird, but this can happen if TCP collapsing
1538 * splitted a fat GRO packet, while we released socket lock
1539 * in skb_splice_bits()
1540 */
1541 tcp_eat_recv_skb(sk, skb);
1542 }
1543 return NULL;
1544 }
1545 EXPORT_SYMBOL(tcp_recv_skb);
1546
1547 /*
1548 * This routine provides an alternative to tcp_recvmsg() for routines
1549 * that would like to handle copying from skbuffs directly in 'sendfile'
1550 * fashion.
1551 * Note:
1552 * - It is assumed that the socket was locked by the caller.
1553 * - The routine does not block.
1554 * - At present, there is no support for reading OOB data
1555 * or for 'peeking' the socket using this routine
1556 * (although both would be easy to implement).
1557 */
tcp_read_sock(struct sock * sk,read_descriptor_t * desc,sk_read_actor_t recv_actor)1558 int tcp_read_sock(struct sock *sk, read_descriptor_t *desc,
1559 sk_read_actor_t recv_actor)
1560 {
1561 struct sk_buff *skb;
1562 struct tcp_sock *tp = tcp_sk(sk);
1563 u32 seq = tp->copied_seq;
1564 u32 offset;
1565 int copied = 0;
1566
1567 if (sk->sk_state == TCP_LISTEN)
1568 return -ENOTCONN;
1569 while ((skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1570 if (offset < skb->len) {
1571 int used;
1572 size_t len;
1573
1574 len = skb->len - offset;
1575 /* Stop reading if we hit a patch of urgent data */
1576 if (unlikely(tp->urg_data)) {
1577 u32 urg_offset = tp->urg_seq - seq;
1578 if (urg_offset < len)
1579 len = urg_offset;
1580 if (!len)
1581 break;
1582 }
1583 used = recv_actor(desc, skb, offset, len);
1584 if (used <= 0) {
1585 if (!copied)
1586 copied = used;
1587 break;
1588 }
1589 if (WARN_ON_ONCE(used > len))
1590 used = len;
1591 seq += used;
1592 copied += used;
1593 offset += used;
1594
1595 /* If recv_actor drops the lock (e.g. TCP splice
1596 * receive) the skb pointer might be invalid when
1597 * getting here: tcp_collapse might have deleted it
1598 * while aggregating skbs from the socket queue.
1599 */
1600 skb = tcp_recv_skb(sk, seq - 1, &offset);
1601 if (!skb)
1602 break;
1603 /* TCP coalescing might have appended data to the skb.
1604 * Try to splice more frags
1605 */
1606 if (offset + 1 != skb->len)
1607 continue;
1608 }
1609 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1610 tcp_eat_recv_skb(sk, skb);
1611 ++seq;
1612 break;
1613 }
1614 tcp_eat_recv_skb(sk, skb);
1615 if (!desc->count)
1616 break;
1617 WRITE_ONCE(tp->copied_seq, seq);
1618 }
1619 WRITE_ONCE(tp->copied_seq, seq);
1620
1621 tcp_rcv_space_adjust(sk);
1622
1623 /* Clean up data we have read: This will do ACK frames. */
1624 if (copied > 0) {
1625 tcp_recv_skb(sk, seq, &offset);
1626 tcp_cleanup_rbuf(sk, copied);
1627 }
1628 return copied;
1629 }
1630 EXPORT_SYMBOL(tcp_read_sock);
1631
tcp_read_skb(struct sock * sk,skb_read_actor_t recv_actor)1632 int tcp_read_skb(struct sock *sk, skb_read_actor_t recv_actor)
1633 {
1634 struct sk_buff *skb;
1635 int copied = 0;
1636
1637 if (sk->sk_state == TCP_LISTEN)
1638 return -ENOTCONN;
1639
1640 while ((skb = skb_peek(&sk->sk_receive_queue)) != NULL) {
1641 u8 tcp_flags;
1642 int used;
1643
1644 __skb_unlink(skb, &sk->sk_receive_queue);
1645 WARN_ON_ONCE(!skb_set_owner_sk_safe(skb, sk));
1646 tcp_flags = TCP_SKB_CB(skb)->tcp_flags;
1647 used = recv_actor(sk, skb);
1648 if (used < 0) {
1649 if (!copied)
1650 copied = used;
1651 break;
1652 }
1653 copied += used;
1654
1655 if (tcp_flags & TCPHDR_FIN)
1656 break;
1657 }
1658 return copied;
1659 }
1660 EXPORT_SYMBOL(tcp_read_skb);
1661
tcp_read_done(struct sock * sk,size_t len)1662 void tcp_read_done(struct sock *sk, size_t len)
1663 {
1664 struct tcp_sock *tp = tcp_sk(sk);
1665 u32 seq = tp->copied_seq;
1666 struct sk_buff *skb;
1667 size_t left;
1668 u32 offset;
1669
1670 if (sk->sk_state == TCP_LISTEN)
1671 return;
1672
1673 left = len;
1674 while (left && (skb = tcp_recv_skb(sk, seq, &offset)) != NULL) {
1675 int used;
1676
1677 used = min_t(size_t, skb->len - offset, left);
1678 seq += used;
1679 left -= used;
1680
1681 if (skb->len > offset + used)
1682 break;
1683
1684 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN) {
1685 tcp_eat_recv_skb(sk, skb);
1686 ++seq;
1687 break;
1688 }
1689 tcp_eat_recv_skb(sk, skb);
1690 }
1691 WRITE_ONCE(tp->copied_seq, seq);
1692
1693 tcp_rcv_space_adjust(sk);
1694
1695 /* Clean up data we have read: This will do ACK frames. */
1696 if (left != len)
1697 tcp_cleanup_rbuf(sk, len - left);
1698 }
1699 EXPORT_SYMBOL(tcp_read_done);
1700
tcp_peek_len(struct socket * sock)1701 int tcp_peek_len(struct socket *sock)
1702 {
1703 return tcp_inq(sock->sk);
1704 }
1705 EXPORT_SYMBOL(tcp_peek_len);
1706
1707 /* Make sure sk_rcvbuf is big enough to satisfy SO_RCVLOWAT hint */
tcp_set_rcvlowat(struct sock * sk,int val)1708 int tcp_set_rcvlowat(struct sock *sk, int val)
1709 {
1710 int space, cap;
1711
1712 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1713 cap = sk->sk_rcvbuf >> 1;
1714 else
1715 cap = READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_rmem[2]) >> 1;
1716 val = min(val, cap);
1717 WRITE_ONCE(sk->sk_rcvlowat, val ? : 1);
1718
1719 /* Check if we need to signal EPOLLIN right now */
1720 tcp_data_ready(sk);
1721
1722 if (sk->sk_userlocks & SOCK_RCVBUF_LOCK)
1723 return 0;
1724
1725 space = tcp_space_from_win(sk, val);
1726 if (space > sk->sk_rcvbuf) {
1727 WRITE_ONCE(sk->sk_rcvbuf, space);
1728 WRITE_ONCE(tcp_sk(sk)->window_clamp, val);
1729 }
1730 return 0;
1731 }
1732 EXPORT_SYMBOL(tcp_set_rcvlowat);
1733
tcp_update_recv_tstamps(struct sk_buff * skb,struct scm_timestamping_internal * tss)1734 void tcp_update_recv_tstamps(struct sk_buff *skb,
1735 struct scm_timestamping_internal *tss)
1736 {
1737 if (skb->tstamp)
1738 tss->ts[0] = ktime_to_timespec64(skb->tstamp);
1739 else
1740 tss->ts[0] = (struct timespec64) {0};
1741
1742 if (skb_hwtstamps(skb)->hwtstamp)
1743 tss->ts[2] = ktime_to_timespec64(skb_hwtstamps(skb)->hwtstamp);
1744 else
1745 tss->ts[2] = (struct timespec64) {0};
1746 }
1747
1748 #ifdef CONFIG_MMU
1749 static const struct vm_operations_struct tcp_vm_ops = {
1750 };
1751
tcp_mmap(struct file * file,struct socket * sock,struct vm_area_struct * vma)1752 int tcp_mmap(struct file *file, struct socket *sock,
1753 struct vm_area_struct *vma)
1754 {
1755 if (vma->vm_flags & (VM_WRITE | VM_EXEC))
1756 return -EPERM;
1757 vm_flags_clear(vma, VM_MAYWRITE | VM_MAYEXEC);
1758
1759 /* Instruct vm_insert_page() to not mmap_read_lock(mm) */
1760 vm_flags_set(vma, VM_MIXEDMAP);
1761
1762 vma->vm_ops = &tcp_vm_ops;
1763 return 0;
1764 }
1765 EXPORT_SYMBOL(tcp_mmap);
1766
skb_advance_to_frag(struct sk_buff * skb,u32 offset_skb,u32 * offset_frag)1767 static skb_frag_t *skb_advance_to_frag(struct sk_buff *skb, u32 offset_skb,
1768 u32 *offset_frag)
1769 {
1770 skb_frag_t *frag;
1771
1772 if (unlikely(offset_skb >= skb->len))
1773 return NULL;
1774
1775 offset_skb -= skb_headlen(skb);
1776 if ((int)offset_skb < 0 || skb_has_frag_list(skb))
1777 return NULL;
1778
1779 frag = skb_shinfo(skb)->frags;
1780 while (offset_skb) {
1781 if (skb_frag_size(frag) > offset_skb) {
1782 *offset_frag = offset_skb;
1783 return frag;
1784 }
1785 offset_skb -= skb_frag_size(frag);
1786 ++frag;
1787 }
1788 *offset_frag = 0;
1789 return frag;
1790 }
1791
can_map_frag(const skb_frag_t * frag)1792 static bool can_map_frag(const skb_frag_t *frag)
1793 {
1794 struct page *page;
1795
1796 if (skb_frag_size(frag) != PAGE_SIZE || skb_frag_off(frag))
1797 return false;
1798
1799 page = skb_frag_page(frag);
1800
1801 if (PageCompound(page) || page->mapping)
1802 return false;
1803
1804 return true;
1805 }
1806
find_next_mappable_frag(const skb_frag_t * frag,int remaining_in_skb)1807 static int find_next_mappable_frag(const skb_frag_t *frag,
1808 int remaining_in_skb)
1809 {
1810 int offset = 0;
1811
1812 if (likely(can_map_frag(frag)))
1813 return 0;
1814
1815 while (offset < remaining_in_skb && !can_map_frag(frag)) {
1816 offset += skb_frag_size(frag);
1817 ++frag;
1818 }
1819 return offset;
1820 }
1821
tcp_zerocopy_set_hint_for_skb(struct sock * sk,struct tcp_zerocopy_receive * zc,struct sk_buff * skb,u32 offset)1822 static void tcp_zerocopy_set_hint_for_skb(struct sock *sk,
1823 struct tcp_zerocopy_receive *zc,
1824 struct sk_buff *skb, u32 offset)
1825 {
1826 u32 frag_offset, partial_frag_remainder = 0;
1827 int mappable_offset;
1828 skb_frag_t *frag;
1829
1830 /* worst case: skip to next skb. try to improve on this case below */
1831 zc->recv_skip_hint = skb->len - offset;
1832
1833 /* Find the frag containing this offset (and how far into that frag) */
1834 frag = skb_advance_to_frag(skb, offset, &frag_offset);
1835 if (!frag)
1836 return;
1837
1838 if (frag_offset) {
1839 struct skb_shared_info *info = skb_shinfo(skb);
1840
1841 /* We read part of the last frag, must recvmsg() rest of skb. */
1842 if (frag == &info->frags[info->nr_frags - 1])
1843 return;
1844
1845 /* Else, we must at least read the remainder in this frag. */
1846 partial_frag_remainder = skb_frag_size(frag) - frag_offset;
1847 zc->recv_skip_hint -= partial_frag_remainder;
1848 ++frag;
1849 }
1850
1851 /* partial_frag_remainder: If part way through a frag, must read rest.
1852 * mappable_offset: Bytes till next mappable frag, *not* counting bytes
1853 * in partial_frag_remainder.
1854 */
1855 mappable_offset = find_next_mappable_frag(frag, zc->recv_skip_hint);
1856 zc->recv_skip_hint = mappable_offset + partial_frag_remainder;
1857 }
1858
1859 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
1860 int flags, struct scm_timestamping_internal *tss,
1861 int *cmsg_flags);
receive_fallback_to_copy(struct sock * sk,struct tcp_zerocopy_receive * zc,int inq,struct scm_timestamping_internal * tss)1862 static int receive_fallback_to_copy(struct sock *sk,
1863 struct tcp_zerocopy_receive *zc, int inq,
1864 struct scm_timestamping_internal *tss)
1865 {
1866 unsigned long copy_address = (unsigned long)zc->copybuf_address;
1867 struct msghdr msg = {};
1868 int err;
1869
1870 zc->length = 0;
1871 zc->recv_skip_hint = 0;
1872
1873 if (copy_address != zc->copybuf_address)
1874 return -EINVAL;
1875
1876 err = import_ubuf(ITER_DEST, (void __user *)copy_address, inq,
1877 &msg.msg_iter);
1878 if (err)
1879 return err;
1880
1881 err = tcp_recvmsg_locked(sk, &msg, inq, MSG_DONTWAIT,
1882 tss, &zc->msg_flags);
1883 if (err < 0)
1884 return err;
1885
1886 zc->copybuf_len = err;
1887 if (likely(zc->copybuf_len)) {
1888 struct sk_buff *skb;
1889 u32 offset;
1890
1891 skb = tcp_recv_skb(sk, tcp_sk(sk)->copied_seq, &offset);
1892 if (skb)
1893 tcp_zerocopy_set_hint_for_skb(sk, zc, skb, offset);
1894 }
1895 return 0;
1896 }
1897
tcp_copy_straggler_data(struct tcp_zerocopy_receive * zc,struct sk_buff * skb,u32 copylen,u32 * offset,u32 * seq)1898 static int tcp_copy_straggler_data(struct tcp_zerocopy_receive *zc,
1899 struct sk_buff *skb, u32 copylen,
1900 u32 *offset, u32 *seq)
1901 {
1902 unsigned long copy_address = (unsigned long)zc->copybuf_address;
1903 struct msghdr msg = {};
1904 int err;
1905
1906 if (copy_address != zc->copybuf_address)
1907 return -EINVAL;
1908
1909 err = import_ubuf(ITER_DEST, (void __user *)copy_address, copylen,
1910 &msg.msg_iter);
1911 if (err)
1912 return err;
1913 err = skb_copy_datagram_msg(skb, *offset, &msg, copylen);
1914 if (err)
1915 return err;
1916 zc->recv_skip_hint -= copylen;
1917 *offset += copylen;
1918 *seq += copylen;
1919 return (__s32)copylen;
1920 }
1921
tcp_zc_handle_leftover(struct tcp_zerocopy_receive * zc,struct sock * sk,struct sk_buff * skb,u32 * seq,s32 copybuf_len,struct scm_timestamping_internal * tss)1922 static int tcp_zc_handle_leftover(struct tcp_zerocopy_receive *zc,
1923 struct sock *sk,
1924 struct sk_buff *skb,
1925 u32 *seq,
1926 s32 copybuf_len,
1927 struct scm_timestamping_internal *tss)
1928 {
1929 u32 offset, copylen = min_t(u32, copybuf_len, zc->recv_skip_hint);
1930
1931 if (!copylen)
1932 return 0;
1933 /* skb is null if inq < PAGE_SIZE. */
1934 if (skb) {
1935 offset = *seq - TCP_SKB_CB(skb)->seq;
1936 } else {
1937 skb = tcp_recv_skb(sk, *seq, &offset);
1938 if (TCP_SKB_CB(skb)->has_rxtstamp) {
1939 tcp_update_recv_tstamps(skb, tss);
1940 zc->msg_flags |= TCP_CMSG_TS;
1941 }
1942 }
1943
1944 zc->copybuf_len = tcp_copy_straggler_data(zc, skb, copylen, &offset,
1945 seq);
1946 return zc->copybuf_len < 0 ? 0 : copylen;
1947 }
1948
tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct * vma,struct page ** pending_pages,unsigned long pages_remaining,unsigned long * address,u32 * length,u32 * seq,struct tcp_zerocopy_receive * zc,u32 total_bytes_to_map,int err)1949 static int tcp_zerocopy_vm_insert_batch_error(struct vm_area_struct *vma,
1950 struct page **pending_pages,
1951 unsigned long pages_remaining,
1952 unsigned long *address,
1953 u32 *length,
1954 u32 *seq,
1955 struct tcp_zerocopy_receive *zc,
1956 u32 total_bytes_to_map,
1957 int err)
1958 {
1959 /* At least one page did not map. Try zapping if we skipped earlier. */
1960 if (err == -EBUSY &&
1961 zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT) {
1962 u32 maybe_zap_len;
1963
1964 maybe_zap_len = total_bytes_to_map - /* All bytes to map */
1965 *length + /* Mapped or pending */
1966 (pages_remaining * PAGE_SIZE); /* Failed map. */
1967 zap_page_range_single(vma, *address, maybe_zap_len, NULL);
1968 err = 0;
1969 }
1970
1971 if (!err) {
1972 unsigned long leftover_pages = pages_remaining;
1973 int bytes_mapped;
1974
1975 /* We called zap_page_range_single, try to reinsert. */
1976 err = vm_insert_pages(vma, *address,
1977 pending_pages,
1978 &pages_remaining);
1979 bytes_mapped = PAGE_SIZE * (leftover_pages - pages_remaining);
1980 *seq += bytes_mapped;
1981 *address += bytes_mapped;
1982 }
1983 if (err) {
1984 /* Either we were unable to zap, OR we zapped, retried an
1985 * insert, and still had an issue. Either ways, pages_remaining
1986 * is the number of pages we were unable to map, and we unroll
1987 * some state we speculatively touched before.
1988 */
1989 const int bytes_not_mapped = PAGE_SIZE * pages_remaining;
1990
1991 *length -= bytes_not_mapped;
1992 zc->recv_skip_hint += bytes_not_mapped;
1993 }
1994 return err;
1995 }
1996
tcp_zerocopy_vm_insert_batch(struct vm_area_struct * vma,struct page ** pages,unsigned int pages_to_map,unsigned long * address,u32 * length,u32 * seq,struct tcp_zerocopy_receive * zc,u32 total_bytes_to_map)1997 static int tcp_zerocopy_vm_insert_batch(struct vm_area_struct *vma,
1998 struct page **pages,
1999 unsigned int pages_to_map,
2000 unsigned long *address,
2001 u32 *length,
2002 u32 *seq,
2003 struct tcp_zerocopy_receive *zc,
2004 u32 total_bytes_to_map)
2005 {
2006 unsigned long pages_remaining = pages_to_map;
2007 unsigned int pages_mapped;
2008 unsigned int bytes_mapped;
2009 int err;
2010
2011 err = vm_insert_pages(vma, *address, pages, &pages_remaining);
2012 pages_mapped = pages_to_map - (unsigned int)pages_remaining;
2013 bytes_mapped = PAGE_SIZE * pages_mapped;
2014 /* Even if vm_insert_pages fails, it may have partially succeeded in
2015 * mapping (some but not all of the pages).
2016 */
2017 *seq += bytes_mapped;
2018 *address += bytes_mapped;
2019
2020 if (likely(!err))
2021 return 0;
2022
2023 /* Error: maybe zap and retry + rollback state for failed inserts. */
2024 return tcp_zerocopy_vm_insert_batch_error(vma, pages + pages_mapped,
2025 pages_remaining, address, length, seq, zc, total_bytes_to_map,
2026 err);
2027 }
2028
2029 #define TCP_VALID_ZC_MSG_FLAGS (TCP_CMSG_TS)
tcp_zc_finalize_rx_tstamp(struct sock * sk,struct tcp_zerocopy_receive * zc,struct scm_timestamping_internal * tss)2030 static void tcp_zc_finalize_rx_tstamp(struct sock *sk,
2031 struct tcp_zerocopy_receive *zc,
2032 struct scm_timestamping_internal *tss)
2033 {
2034 unsigned long msg_control_addr;
2035 struct msghdr cmsg_dummy;
2036
2037 msg_control_addr = (unsigned long)zc->msg_control;
2038 cmsg_dummy.msg_control_user = (void __user *)msg_control_addr;
2039 cmsg_dummy.msg_controllen =
2040 (__kernel_size_t)zc->msg_controllen;
2041 cmsg_dummy.msg_flags = in_compat_syscall()
2042 ? MSG_CMSG_COMPAT : 0;
2043 cmsg_dummy.msg_control_is_user = true;
2044 zc->msg_flags = 0;
2045 if (zc->msg_control == msg_control_addr &&
2046 zc->msg_controllen == cmsg_dummy.msg_controllen) {
2047 tcp_recv_timestamp(&cmsg_dummy, sk, tss);
2048 zc->msg_control = (__u64)
2049 ((uintptr_t)cmsg_dummy.msg_control_user);
2050 zc->msg_controllen =
2051 (__u64)cmsg_dummy.msg_controllen;
2052 zc->msg_flags = (__u32)cmsg_dummy.msg_flags;
2053 }
2054 }
2055
find_tcp_vma(struct mm_struct * mm,unsigned long address,bool * mmap_locked)2056 static struct vm_area_struct *find_tcp_vma(struct mm_struct *mm,
2057 unsigned long address,
2058 bool *mmap_locked)
2059 {
2060 struct vm_area_struct *vma = lock_vma_under_rcu(mm, address);
2061
2062 if (vma) {
2063 if (vma->vm_ops != &tcp_vm_ops) {
2064 vma_end_read(vma);
2065 return NULL;
2066 }
2067 *mmap_locked = false;
2068 return vma;
2069 }
2070
2071 mmap_read_lock(mm);
2072 vma = vma_lookup(mm, address);
2073 if (!vma || vma->vm_ops != &tcp_vm_ops) {
2074 mmap_read_unlock(mm);
2075 return NULL;
2076 }
2077 *mmap_locked = true;
2078 return vma;
2079 }
2080
2081 #define TCP_ZEROCOPY_PAGE_BATCH_SIZE 32
tcp_zerocopy_receive(struct sock * sk,struct tcp_zerocopy_receive * zc,struct scm_timestamping_internal * tss)2082 static int tcp_zerocopy_receive(struct sock *sk,
2083 struct tcp_zerocopy_receive *zc,
2084 struct scm_timestamping_internal *tss)
2085 {
2086 u32 length = 0, offset, vma_len, avail_len, copylen = 0;
2087 unsigned long address = (unsigned long)zc->address;
2088 struct page *pages[TCP_ZEROCOPY_PAGE_BATCH_SIZE];
2089 s32 copybuf_len = zc->copybuf_len;
2090 struct tcp_sock *tp = tcp_sk(sk);
2091 const skb_frag_t *frags = NULL;
2092 unsigned int pages_to_map = 0;
2093 struct vm_area_struct *vma;
2094 struct sk_buff *skb = NULL;
2095 u32 seq = tp->copied_seq;
2096 u32 total_bytes_to_map;
2097 int inq = tcp_inq(sk);
2098 bool mmap_locked;
2099 int ret;
2100
2101 zc->copybuf_len = 0;
2102 zc->msg_flags = 0;
2103
2104 if (address & (PAGE_SIZE - 1) || address != zc->address)
2105 return -EINVAL;
2106
2107 if (sk->sk_state == TCP_LISTEN)
2108 return -ENOTCONN;
2109
2110 sock_rps_record_flow(sk);
2111
2112 if (inq && inq <= copybuf_len)
2113 return receive_fallback_to_copy(sk, zc, inq, tss);
2114
2115 if (inq < PAGE_SIZE) {
2116 zc->length = 0;
2117 zc->recv_skip_hint = inq;
2118 if (!inq && sock_flag(sk, SOCK_DONE))
2119 return -EIO;
2120 return 0;
2121 }
2122
2123 vma = find_tcp_vma(current->mm, address, &mmap_locked);
2124 if (!vma)
2125 return -EINVAL;
2126
2127 vma_len = min_t(unsigned long, zc->length, vma->vm_end - address);
2128 avail_len = min_t(u32, vma_len, inq);
2129 total_bytes_to_map = avail_len & ~(PAGE_SIZE - 1);
2130 if (total_bytes_to_map) {
2131 if (!(zc->flags & TCP_RECEIVE_ZEROCOPY_FLAG_TLB_CLEAN_HINT))
2132 zap_page_range_single(vma, address, total_bytes_to_map,
2133 NULL);
2134 zc->length = total_bytes_to_map;
2135 zc->recv_skip_hint = 0;
2136 } else {
2137 zc->length = avail_len;
2138 zc->recv_skip_hint = avail_len;
2139 }
2140 ret = 0;
2141 while (length + PAGE_SIZE <= zc->length) {
2142 int mappable_offset;
2143 struct page *page;
2144
2145 if (zc->recv_skip_hint < PAGE_SIZE) {
2146 u32 offset_frag;
2147
2148 if (skb) {
2149 if (zc->recv_skip_hint > 0)
2150 break;
2151 skb = skb->next;
2152 offset = seq - TCP_SKB_CB(skb)->seq;
2153 } else {
2154 skb = tcp_recv_skb(sk, seq, &offset);
2155 }
2156
2157 if (TCP_SKB_CB(skb)->has_rxtstamp) {
2158 tcp_update_recv_tstamps(skb, tss);
2159 zc->msg_flags |= TCP_CMSG_TS;
2160 }
2161 zc->recv_skip_hint = skb->len - offset;
2162 frags = skb_advance_to_frag(skb, offset, &offset_frag);
2163 if (!frags || offset_frag)
2164 break;
2165 }
2166
2167 mappable_offset = find_next_mappable_frag(frags,
2168 zc->recv_skip_hint);
2169 if (mappable_offset) {
2170 zc->recv_skip_hint = mappable_offset;
2171 break;
2172 }
2173 page = skb_frag_page(frags);
2174 prefetchw(page);
2175 pages[pages_to_map++] = page;
2176 length += PAGE_SIZE;
2177 zc->recv_skip_hint -= PAGE_SIZE;
2178 frags++;
2179 if (pages_to_map == TCP_ZEROCOPY_PAGE_BATCH_SIZE ||
2180 zc->recv_skip_hint < PAGE_SIZE) {
2181 /* Either full batch, or we're about to go to next skb
2182 * (and we cannot unroll failed ops across skbs).
2183 */
2184 ret = tcp_zerocopy_vm_insert_batch(vma, pages,
2185 pages_to_map,
2186 &address, &length,
2187 &seq, zc,
2188 total_bytes_to_map);
2189 if (ret)
2190 goto out;
2191 pages_to_map = 0;
2192 }
2193 }
2194 if (pages_to_map) {
2195 ret = tcp_zerocopy_vm_insert_batch(vma, pages, pages_to_map,
2196 &address, &length, &seq,
2197 zc, total_bytes_to_map);
2198 }
2199 out:
2200 if (mmap_locked)
2201 mmap_read_unlock(current->mm);
2202 else
2203 vma_end_read(vma);
2204 /* Try to copy straggler data. */
2205 if (!ret)
2206 copylen = tcp_zc_handle_leftover(zc, sk, skb, &seq, copybuf_len, tss);
2207
2208 if (length + copylen) {
2209 WRITE_ONCE(tp->copied_seq, seq);
2210 tcp_rcv_space_adjust(sk);
2211
2212 /* Clean up data we have read: This will do ACK frames. */
2213 tcp_recv_skb(sk, seq, &offset);
2214 tcp_cleanup_rbuf(sk, length + copylen);
2215 ret = 0;
2216 if (length == zc->length)
2217 zc->recv_skip_hint = 0;
2218 } else {
2219 if (!zc->recv_skip_hint && sock_flag(sk, SOCK_DONE))
2220 ret = -EIO;
2221 }
2222 zc->length = length;
2223 return ret;
2224 }
2225 #endif
2226
2227 /* Similar to __sock_recv_timestamp, but does not require an skb */
tcp_recv_timestamp(struct msghdr * msg,const struct sock * sk,struct scm_timestamping_internal * tss)2228 void tcp_recv_timestamp(struct msghdr *msg, const struct sock *sk,
2229 struct scm_timestamping_internal *tss)
2230 {
2231 int new_tstamp = sock_flag(sk, SOCK_TSTAMP_NEW);
2232 bool has_timestamping = false;
2233
2234 if (tss->ts[0].tv_sec || tss->ts[0].tv_nsec) {
2235 if (sock_flag(sk, SOCK_RCVTSTAMP)) {
2236 if (sock_flag(sk, SOCK_RCVTSTAMPNS)) {
2237 if (new_tstamp) {
2238 struct __kernel_timespec kts = {
2239 .tv_sec = tss->ts[0].tv_sec,
2240 .tv_nsec = tss->ts[0].tv_nsec,
2241 };
2242 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_NEW,
2243 sizeof(kts), &kts);
2244 } else {
2245 struct __kernel_old_timespec ts_old = {
2246 .tv_sec = tss->ts[0].tv_sec,
2247 .tv_nsec = tss->ts[0].tv_nsec,
2248 };
2249 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMPNS_OLD,
2250 sizeof(ts_old), &ts_old);
2251 }
2252 } else {
2253 if (new_tstamp) {
2254 struct __kernel_sock_timeval stv = {
2255 .tv_sec = tss->ts[0].tv_sec,
2256 .tv_usec = tss->ts[0].tv_nsec / 1000,
2257 };
2258 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_NEW,
2259 sizeof(stv), &stv);
2260 } else {
2261 struct __kernel_old_timeval tv = {
2262 .tv_sec = tss->ts[0].tv_sec,
2263 .tv_usec = tss->ts[0].tv_nsec / 1000,
2264 };
2265 put_cmsg(msg, SOL_SOCKET, SO_TIMESTAMP_OLD,
2266 sizeof(tv), &tv);
2267 }
2268 }
2269 }
2270
2271 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_SOFTWARE)
2272 has_timestamping = true;
2273 else
2274 tss->ts[0] = (struct timespec64) {0};
2275 }
2276
2277 if (tss->ts[2].tv_sec || tss->ts[2].tv_nsec) {
2278 if (READ_ONCE(sk->sk_tsflags) & SOF_TIMESTAMPING_RAW_HARDWARE)
2279 has_timestamping = true;
2280 else
2281 tss->ts[2] = (struct timespec64) {0};
2282 }
2283
2284 if (has_timestamping) {
2285 tss->ts[1] = (struct timespec64) {0};
2286 if (sock_flag(sk, SOCK_TSTAMP_NEW))
2287 put_cmsg_scm_timestamping64(msg, tss);
2288 else
2289 put_cmsg_scm_timestamping(msg, tss);
2290 }
2291 }
2292
tcp_inq_hint(struct sock * sk)2293 static int tcp_inq_hint(struct sock *sk)
2294 {
2295 const struct tcp_sock *tp = tcp_sk(sk);
2296 u32 copied_seq = READ_ONCE(tp->copied_seq);
2297 u32 rcv_nxt = READ_ONCE(tp->rcv_nxt);
2298 int inq;
2299
2300 inq = rcv_nxt - copied_seq;
2301 if (unlikely(inq < 0 || copied_seq != READ_ONCE(tp->copied_seq))) {
2302 lock_sock(sk);
2303 inq = tp->rcv_nxt - tp->copied_seq;
2304 release_sock(sk);
2305 }
2306 /* After receiving a FIN, tell the user-space to continue reading
2307 * by returning a non-zero inq.
2308 */
2309 if (inq == 0 && sock_flag(sk, SOCK_DONE))
2310 inq = 1;
2311 return inq;
2312 }
2313
2314 /*
2315 * This routine copies from a sock struct into the user buffer.
2316 *
2317 * Technical note: in 2.3 we work on _locked_ socket, so that
2318 * tricks with *seq access order and skb->users are not required.
2319 * Probably, code can be easily improved even more.
2320 */
2321
tcp_recvmsg_locked(struct sock * sk,struct msghdr * msg,size_t len,int flags,struct scm_timestamping_internal * tss,int * cmsg_flags)2322 static int tcp_recvmsg_locked(struct sock *sk, struct msghdr *msg, size_t len,
2323 int flags, struct scm_timestamping_internal *tss,
2324 int *cmsg_flags)
2325 {
2326 struct tcp_sock *tp = tcp_sk(sk);
2327 int copied = 0;
2328 u32 peek_seq;
2329 u32 *seq;
2330 unsigned long used;
2331 int err;
2332 int target; /* Read at least this many bytes */
2333 long timeo;
2334 struct sk_buff *skb, *last;
2335 u32 peek_offset = 0;
2336 u32 urg_hole = 0;
2337
2338 err = -ENOTCONN;
2339 if (sk->sk_state == TCP_LISTEN)
2340 goto out;
2341
2342 if (tp->recvmsg_inq) {
2343 *cmsg_flags = TCP_CMSG_INQ;
2344 msg->msg_get_inq = 1;
2345 }
2346 timeo = sock_rcvtimeo(sk, flags & MSG_DONTWAIT);
2347
2348 /* Urgent data needs to be handled specially. */
2349 if (flags & MSG_OOB)
2350 goto recv_urg;
2351
2352 if (unlikely(tp->repair)) {
2353 err = -EPERM;
2354 if (!(flags & MSG_PEEK))
2355 goto out;
2356
2357 if (tp->repair_queue == TCP_SEND_QUEUE)
2358 goto recv_sndq;
2359
2360 err = -EINVAL;
2361 if (tp->repair_queue == TCP_NO_QUEUE)
2362 goto out;
2363
2364 /* 'common' recv queue MSG_PEEK-ing */
2365 }
2366
2367 seq = &tp->copied_seq;
2368 if (flags & MSG_PEEK) {
2369 peek_offset = max(sk_peek_offset(sk, flags), 0);
2370 peek_seq = tp->copied_seq + peek_offset;
2371 seq = &peek_seq;
2372 }
2373
2374 target = sock_rcvlowat(sk, flags & MSG_WAITALL, len);
2375
2376 do {
2377 u32 offset;
2378
2379 /* Are we at urgent data? Stop if we have read anything or have SIGURG pending. */
2380 if (unlikely(tp->urg_data) && tp->urg_seq == *seq) {
2381 if (copied)
2382 break;
2383 if (signal_pending(current)) {
2384 copied = timeo ? sock_intr_errno(timeo) : -EAGAIN;
2385 break;
2386 }
2387 }
2388
2389 /* Next get a buffer. */
2390
2391 last = skb_peek_tail(&sk->sk_receive_queue);
2392 skb_queue_walk(&sk->sk_receive_queue, skb) {
2393 last = skb;
2394 /* Now that we have two receive queues this
2395 * shouldn't happen.
2396 */
2397 if (WARN(before(*seq, TCP_SKB_CB(skb)->seq),
2398 "TCP recvmsg seq # bug: copied %X, seq %X, rcvnxt %X, fl %X\n",
2399 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt,
2400 flags))
2401 break;
2402
2403 offset = *seq - TCP_SKB_CB(skb)->seq;
2404 if (unlikely(TCP_SKB_CB(skb)->tcp_flags & TCPHDR_SYN)) {
2405 pr_err_once("%s: found a SYN, please report !\n", __func__);
2406 offset--;
2407 }
2408 if (offset < skb->len)
2409 goto found_ok_skb;
2410 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2411 goto found_fin_ok;
2412 WARN(!(flags & MSG_PEEK),
2413 "TCP recvmsg seq # bug 2: copied %X, seq %X, rcvnxt %X, fl %X\n",
2414 *seq, TCP_SKB_CB(skb)->seq, tp->rcv_nxt, flags);
2415 }
2416
2417 /* Well, if we have backlog, try to process it now yet. */
2418
2419 if (copied >= target && !READ_ONCE(sk->sk_backlog.tail))
2420 break;
2421
2422 if (copied) {
2423 if (!timeo ||
2424 sk->sk_err ||
2425 sk->sk_state == TCP_CLOSE ||
2426 (sk->sk_shutdown & RCV_SHUTDOWN) ||
2427 signal_pending(current))
2428 break;
2429 } else {
2430 if (sock_flag(sk, SOCK_DONE))
2431 break;
2432
2433 if (sk->sk_err) {
2434 copied = sock_error(sk);
2435 break;
2436 }
2437
2438 if (sk->sk_shutdown & RCV_SHUTDOWN)
2439 break;
2440
2441 if (sk->sk_state == TCP_CLOSE) {
2442 /* This occurs when user tries to read
2443 * from never connected socket.
2444 */
2445 copied = -ENOTCONN;
2446 break;
2447 }
2448
2449 if (!timeo) {
2450 copied = -EAGAIN;
2451 break;
2452 }
2453
2454 if (signal_pending(current)) {
2455 copied = sock_intr_errno(timeo);
2456 break;
2457 }
2458 }
2459
2460 if (copied >= target) {
2461 /* Do not sleep, just process backlog. */
2462 __sk_flush_backlog(sk);
2463 } else {
2464 tcp_cleanup_rbuf(sk, copied);
2465 err = sk_wait_data(sk, &timeo, last);
2466 if (err < 0) {
2467 err = copied ? : err;
2468 goto out;
2469 }
2470 }
2471
2472 if ((flags & MSG_PEEK) &&
2473 (peek_seq - peek_offset - copied - urg_hole != tp->copied_seq)) {
2474 net_dbg_ratelimited("TCP(%s:%d): Application bug, race in MSG_PEEK\n",
2475 current->comm,
2476 task_pid_nr(current));
2477 peek_seq = tp->copied_seq + peek_offset;
2478 }
2479 continue;
2480
2481 found_ok_skb:
2482 /* Ok so how much can we use? */
2483 used = skb->len - offset;
2484 if (len < used)
2485 used = len;
2486
2487 /* Do we have urgent data here? */
2488 if (unlikely(tp->urg_data)) {
2489 u32 urg_offset = tp->urg_seq - *seq;
2490 if (urg_offset < used) {
2491 if (!urg_offset) {
2492 if (!sock_flag(sk, SOCK_URGINLINE)) {
2493 WRITE_ONCE(*seq, *seq + 1);
2494 urg_hole++;
2495 offset++;
2496 used--;
2497 if (!used)
2498 goto skip_copy;
2499 }
2500 } else
2501 used = urg_offset;
2502 }
2503 }
2504
2505 if (!(flags & MSG_TRUNC)) {
2506 err = skb_copy_datagram_msg(skb, offset, msg, used);
2507 if (err) {
2508 /* Exception. Bailout! */
2509 if (!copied)
2510 copied = -EFAULT;
2511 break;
2512 }
2513 }
2514
2515 WRITE_ONCE(*seq, *seq + used);
2516 copied += used;
2517 len -= used;
2518 if (flags & MSG_PEEK)
2519 sk_peek_offset_fwd(sk, used);
2520 else
2521 sk_peek_offset_bwd(sk, used);
2522 tcp_rcv_space_adjust(sk);
2523
2524 skip_copy:
2525 if (unlikely(tp->urg_data) && after(tp->copied_seq, tp->urg_seq)) {
2526 WRITE_ONCE(tp->urg_data, 0);
2527 tcp_fast_path_check(sk);
2528 }
2529
2530 if (TCP_SKB_CB(skb)->has_rxtstamp) {
2531 tcp_update_recv_tstamps(skb, tss);
2532 *cmsg_flags |= TCP_CMSG_TS;
2533 }
2534
2535 if (used + offset < skb->len)
2536 continue;
2537
2538 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2539 goto found_fin_ok;
2540 if (!(flags & MSG_PEEK))
2541 tcp_eat_recv_skb(sk, skb);
2542 continue;
2543
2544 found_fin_ok:
2545 /* Process the FIN. */
2546 WRITE_ONCE(*seq, *seq + 1);
2547 if (!(flags & MSG_PEEK))
2548 tcp_eat_recv_skb(sk, skb);
2549 break;
2550 } while (len > 0);
2551
2552 /* According to UNIX98, msg_name/msg_namelen are ignored
2553 * on connected socket. I was just happy when found this 8) --ANK
2554 */
2555
2556 /* Clean up data we have read: This will do ACK frames. */
2557 tcp_cleanup_rbuf(sk, copied);
2558 return copied;
2559
2560 out:
2561 return err;
2562
2563 recv_urg:
2564 err = tcp_recv_urg(sk, msg, len, flags);
2565 goto out;
2566
2567 recv_sndq:
2568 err = tcp_peek_sndq(sk, msg, len);
2569 goto out;
2570 }
2571
tcp_recvmsg(struct sock * sk,struct msghdr * msg,size_t len,int flags,int * addr_len)2572 int tcp_recvmsg(struct sock *sk, struct msghdr *msg, size_t len, int flags,
2573 int *addr_len)
2574 {
2575 int cmsg_flags = 0, ret;
2576 struct scm_timestamping_internal tss;
2577
2578 if (unlikely(flags & MSG_ERRQUEUE))
2579 return inet_recv_error(sk, msg, len, addr_len);
2580
2581 if (sk_can_busy_loop(sk) &&
2582 skb_queue_empty_lockless(&sk->sk_receive_queue) &&
2583 sk->sk_state == TCP_ESTABLISHED)
2584 sk_busy_loop(sk, flags & MSG_DONTWAIT);
2585
2586 lock_sock(sk);
2587 ret = tcp_recvmsg_locked(sk, msg, len, flags, &tss, &cmsg_flags);
2588 release_sock(sk);
2589
2590 if ((cmsg_flags || msg->msg_get_inq) && ret >= 0) {
2591 if (cmsg_flags & TCP_CMSG_TS)
2592 tcp_recv_timestamp(msg, sk, &tss);
2593 if (msg->msg_get_inq) {
2594 msg->msg_inq = tcp_inq_hint(sk);
2595 if (cmsg_flags & TCP_CMSG_INQ)
2596 put_cmsg(msg, SOL_TCP, TCP_CM_INQ,
2597 sizeof(msg->msg_inq), &msg->msg_inq);
2598 }
2599 }
2600 return ret;
2601 }
2602 EXPORT_SYMBOL(tcp_recvmsg);
2603
tcp_set_state(struct sock * sk,int state)2604 void tcp_set_state(struct sock *sk, int state)
2605 {
2606 int oldstate = sk->sk_state;
2607
2608 /* We defined a new enum for TCP states that are exported in BPF
2609 * so as not force the internal TCP states to be frozen. The
2610 * following checks will detect if an internal state value ever
2611 * differs from the BPF value. If this ever happens, then we will
2612 * need to remap the internal value to the BPF value before calling
2613 * tcp_call_bpf_2arg.
2614 */
2615 BUILD_BUG_ON((int)BPF_TCP_ESTABLISHED != (int)TCP_ESTABLISHED);
2616 BUILD_BUG_ON((int)BPF_TCP_SYN_SENT != (int)TCP_SYN_SENT);
2617 BUILD_BUG_ON((int)BPF_TCP_SYN_RECV != (int)TCP_SYN_RECV);
2618 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT1 != (int)TCP_FIN_WAIT1);
2619 BUILD_BUG_ON((int)BPF_TCP_FIN_WAIT2 != (int)TCP_FIN_WAIT2);
2620 BUILD_BUG_ON((int)BPF_TCP_TIME_WAIT != (int)TCP_TIME_WAIT);
2621 BUILD_BUG_ON((int)BPF_TCP_CLOSE != (int)TCP_CLOSE);
2622 BUILD_BUG_ON((int)BPF_TCP_CLOSE_WAIT != (int)TCP_CLOSE_WAIT);
2623 BUILD_BUG_ON((int)BPF_TCP_LAST_ACK != (int)TCP_LAST_ACK);
2624 BUILD_BUG_ON((int)BPF_TCP_LISTEN != (int)TCP_LISTEN);
2625 BUILD_BUG_ON((int)BPF_TCP_CLOSING != (int)TCP_CLOSING);
2626 BUILD_BUG_ON((int)BPF_TCP_NEW_SYN_RECV != (int)TCP_NEW_SYN_RECV);
2627 BUILD_BUG_ON((int)BPF_TCP_BOUND_INACTIVE != (int)TCP_BOUND_INACTIVE);
2628 BUILD_BUG_ON((int)BPF_TCP_MAX_STATES != (int)TCP_MAX_STATES);
2629
2630 /* bpf uapi header bpf.h defines an anonymous enum with values
2631 * BPF_TCP_* used by bpf programs. Currently gcc built vmlinux
2632 * is able to emit this enum in DWARF due to the above BUILD_BUG_ON.
2633 * But clang built vmlinux does not have this enum in DWARF
2634 * since clang removes the above code before generating IR/debuginfo.
2635 * Let us explicitly emit the type debuginfo to ensure the
2636 * above-mentioned anonymous enum in the vmlinux DWARF and hence BTF
2637 * regardless of which compiler is used.
2638 */
2639 BTF_TYPE_EMIT_ENUM(BPF_TCP_ESTABLISHED);
2640
2641 if (BPF_SOCK_OPS_TEST_FLAG(tcp_sk(sk), BPF_SOCK_OPS_STATE_CB_FLAG))
2642 tcp_call_bpf_2arg(sk, BPF_SOCK_OPS_STATE_CB, oldstate, state);
2643
2644 switch (state) {
2645 case TCP_ESTABLISHED:
2646 if (oldstate != TCP_ESTABLISHED)
2647 TCP_INC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2648 break;
2649
2650 case TCP_CLOSE:
2651 if (oldstate == TCP_CLOSE_WAIT || oldstate == TCP_ESTABLISHED)
2652 TCP_INC_STATS(sock_net(sk), TCP_MIB_ESTABRESETS);
2653
2654 sk->sk_prot->unhash(sk);
2655 if (inet_csk(sk)->icsk_bind_hash &&
2656 !(sk->sk_userlocks & SOCK_BINDPORT_LOCK))
2657 inet_put_port(sk);
2658 fallthrough;
2659 default:
2660 if (oldstate == TCP_ESTABLISHED)
2661 TCP_DEC_STATS(sock_net(sk), TCP_MIB_CURRESTAB);
2662 }
2663
2664 /* Change state AFTER socket is unhashed to avoid closed
2665 * socket sitting in hash tables.
2666 */
2667 inet_sk_state_store(sk, state);
2668 }
2669 EXPORT_SYMBOL_GPL(tcp_set_state);
2670
2671 /*
2672 * State processing on a close. This implements the state shift for
2673 * sending our FIN frame. Note that we only send a FIN for some
2674 * states. A shutdown() may have already sent the FIN, or we may be
2675 * closed.
2676 */
2677
2678 static const unsigned char new_state[16] = {
2679 /* current state: new state: action: */
2680 [0 /* (Invalid) */] = TCP_CLOSE,
2681 [TCP_ESTABLISHED] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2682 [TCP_SYN_SENT] = TCP_CLOSE,
2683 [TCP_SYN_RECV] = TCP_FIN_WAIT1 | TCP_ACTION_FIN,
2684 [TCP_FIN_WAIT1] = TCP_FIN_WAIT1,
2685 [TCP_FIN_WAIT2] = TCP_FIN_WAIT2,
2686 [TCP_TIME_WAIT] = TCP_CLOSE,
2687 [TCP_CLOSE] = TCP_CLOSE,
2688 [TCP_CLOSE_WAIT] = TCP_LAST_ACK | TCP_ACTION_FIN,
2689 [TCP_LAST_ACK] = TCP_LAST_ACK,
2690 [TCP_LISTEN] = TCP_CLOSE,
2691 [TCP_CLOSING] = TCP_CLOSING,
2692 [TCP_NEW_SYN_RECV] = TCP_CLOSE, /* should not happen ! */
2693 };
2694
tcp_close_state(struct sock * sk)2695 static int tcp_close_state(struct sock *sk)
2696 {
2697 int next = (int)new_state[sk->sk_state];
2698 int ns = next & TCP_STATE_MASK;
2699
2700 tcp_set_state(sk, ns);
2701
2702 return next & TCP_ACTION_FIN;
2703 }
2704
2705 /*
2706 * Shutdown the sending side of a connection. Much like close except
2707 * that we don't receive shut down or sock_set_flag(sk, SOCK_DEAD).
2708 */
2709
tcp_shutdown(struct sock * sk,int how)2710 void tcp_shutdown(struct sock *sk, int how)
2711 {
2712 /* We need to grab some memory, and put together a FIN,
2713 * and then put it into the queue to be sent.
2714 * Tim MacKenzie(tym@dibbler.cs.monash.edu.au) 4 Dec '92.
2715 */
2716 if (!(how & SEND_SHUTDOWN))
2717 return;
2718
2719 /* If we've already sent a FIN, or it's a closed state, skip this. */
2720 if ((1 << sk->sk_state) &
2721 (TCPF_ESTABLISHED | TCPF_SYN_SENT |
2722 TCPF_CLOSE_WAIT)) {
2723 /* Clear out any half completed packets. FIN if needed. */
2724 if (tcp_close_state(sk))
2725 tcp_send_fin(sk);
2726 }
2727 }
2728 EXPORT_SYMBOL(tcp_shutdown);
2729
tcp_orphan_count_sum(void)2730 int tcp_orphan_count_sum(void)
2731 {
2732 int i, total = 0;
2733
2734 for_each_possible_cpu(i)
2735 total += per_cpu(tcp_orphan_count, i);
2736
2737 return max(total, 0);
2738 }
2739
2740 static int tcp_orphan_cache;
2741 static struct timer_list tcp_orphan_timer;
2742 #define TCP_ORPHAN_TIMER_PERIOD msecs_to_jiffies(100)
2743
tcp_orphan_update(struct timer_list * unused)2744 static void tcp_orphan_update(struct timer_list *unused)
2745 {
2746 WRITE_ONCE(tcp_orphan_cache, tcp_orphan_count_sum());
2747 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
2748 }
2749
tcp_too_many_orphans(int shift)2750 static bool tcp_too_many_orphans(int shift)
2751 {
2752 return READ_ONCE(tcp_orphan_cache) << shift >
2753 READ_ONCE(sysctl_tcp_max_orphans);
2754 }
2755
tcp_out_of_memory(const struct sock * sk)2756 static bool tcp_out_of_memory(const struct sock *sk)
2757 {
2758 if (sk->sk_wmem_queued > SOCK_MIN_SNDBUF &&
2759 sk_memory_allocated(sk) > sk_prot_mem_limits(sk, 2))
2760 return true;
2761 return false;
2762 }
2763
tcp_check_oom(const struct sock * sk,int shift)2764 bool tcp_check_oom(const struct sock *sk, int shift)
2765 {
2766 bool too_many_orphans, out_of_socket_memory;
2767
2768 too_many_orphans = tcp_too_many_orphans(shift);
2769 out_of_socket_memory = tcp_out_of_memory(sk);
2770
2771 if (too_many_orphans)
2772 net_info_ratelimited("too many orphaned sockets\n");
2773 if (out_of_socket_memory)
2774 net_info_ratelimited("out of memory -- consider tuning tcp_mem\n");
2775 return too_many_orphans || out_of_socket_memory;
2776 }
2777
__tcp_close(struct sock * sk,long timeout)2778 void __tcp_close(struct sock *sk, long timeout)
2779 {
2780 struct sk_buff *skb;
2781 int data_was_unread = 0;
2782 int state;
2783
2784 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
2785
2786 if (sk->sk_state == TCP_LISTEN) {
2787 tcp_set_state(sk, TCP_CLOSE);
2788
2789 /* Special case. */
2790 inet_csk_listen_stop(sk);
2791
2792 goto adjudge_to_death;
2793 }
2794
2795 /* We need to flush the recv. buffs. We do this only on the
2796 * descriptor close, not protocol-sourced closes, because the
2797 * reader process may not have drained the data yet!
2798 */
2799 while ((skb = __skb_dequeue(&sk->sk_receive_queue)) != NULL) {
2800 u32 len = TCP_SKB_CB(skb)->end_seq - TCP_SKB_CB(skb)->seq;
2801
2802 if (TCP_SKB_CB(skb)->tcp_flags & TCPHDR_FIN)
2803 len--;
2804 data_was_unread += len;
2805 __kfree_skb(skb);
2806 }
2807
2808 /* If socket has been already reset (e.g. in tcp_reset()) - kill it. */
2809 if (sk->sk_state == TCP_CLOSE)
2810 goto adjudge_to_death;
2811
2812 /* As outlined in RFC 2525, section 2.17, we send a RST here because
2813 * data was lost. To witness the awful effects of the old behavior of
2814 * always doing a FIN, run an older 2.1.x kernel or 2.0.x, start a bulk
2815 * GET in an FTP client, suspend the process, wait for the client to
2816 * advertise a zero window, then kill -9 the FTP client, wheee...
2817 * Note: timeout is always zero in such a case.
2818 */
2819 if (unlikely(tcp_sk(sk)->repair)) {
2820 sk->sk_prot->disconnect(sk, 0);
2821 } else if (data_was_unread) {
2822 /* Unread data was tossed, zap the connection. */
2823 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONCLOSE);
2824 tcp_set_state(sk, TCP_CLOSE);
2825 tcp_send_active_reset(sk, sk->sk_allocation,
2826 SK_RST_REASON_NOT_SPECIFIED);
2827 } else if (sock_flag(sk, SOCK_LINGER) && !sk->sk_lingertime) {
2828 /* Check zero linger _after_ checking for unread data. */
2829 sk->sk_prot->disconnect(sk, 0);
2830 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPABORTONDATA);
2831 } else if (tcp_close_state(sk)) {
2832 /* We FIN if the application ate all the data before
2833 * zapping the connection.
2834 */
2835
2836 /* RED-PEN. Formally speaking, we have broken TCP state
2837 * machine. State transitions:
2838 *
2839 * TCP_ESTABLISHED -> TCP_FIN_WAIT1
2840 * TCP_SYN_RECV -> TCP_FIN_WAIT1 (it is difficult)
2841 * TCP_CLOSE_WAIT -> TCP_LAST_ACK
2842 *
2843 * are legal only when FIN has been sent (i.e. in window),
2844 * rather than queued out of window. Purists blame.
2845 *
2846 * F.e. "RFC state" is ESTABLISHED,
2847 * if Linux state is FIN-WAIT-1, but FIN is still not sent.
2848 *
2849 * The visible declinations are that sometimes
2850 * we enter time-wait state, when it is not required really
2851 * (harmless), do not send active resets, when they are
2852 * required by specs (TCP_ESTABLISHED, TCP_CLOSE_WAIT, when
2853 * they look as CLOSING or LAST_ACK for Linux)
2854 * Probably, I missed some more holelets.
2855 * --ANK
2856 * XXX (TFO) - To start off we don't support SYN+ACK+FIN
2857 * in a single packet! (May consider it later but will
2858 * probably need API support or TCP_CORK SYN-ACK until
2859 * data is written and socket is closed.)
2860 */
2861 tcp_send_fin(sk);
2862 }
2863
2864 sk_stream_wait_close(sk, timeout);
2865
2866 adjudge_to_death:
2867 state = sk->sk_state;
2868 sock_hold(sk);
2869 sock_orphan(sk);
2870
2871 local_bh_disable();
2872 bh_lock_sock(sk);
2873 /* remove backlog if any, without releasing ownership. */
2874 __release_sock(sk);
2875
2876 this_cpu_inc(tcp_orphan_count);
2877
2878 /* Have we already been destroyed by a softirq or backlog? */
2879 if (state != TCP_CLOSE && sk->sk_state == TCP_CLOSE)
2880 goto out;
2881
2882 /* This is a (useful) BSD violating of the RFC. There is a
2883 * problem with TCP as specified in that the other end could
2884 * keep a socket open forever with no application left this end.
2885 * We use a 1 minute timeout (about the same as BSD) then kill
2886 * our end. If they send after that then tough - BUT: long enough
2887 * that we won't make the old 4*rto = almost no time - whoops
2888 * reset mistake.
2889 *
2890 * Nope, it was not mistake. It is really desired behaviour
2891 * f.e. on http servers, when such sockets are useless, but
2892 * consume significant resources. Let's do it with special
2893 * linger2 option. --ANK
2894 */
2895
2896 if (sk->sk_state == TCP_FIN_WAIT2) {
2897 struct tcp_sock *tp = tcp_sk(sk);
2898 if (READ_ONCE(tp->linger2) < 0) {
2899 tcp_set_state(sk, TCP_CLOSE);
2900 tcp_send_active_reset(sk, GFP_ATOMIC,
2901 SK_RST_REASON_NOT_SPECIFIED);
2902 __NET_INC_STATS(sock_net(sk),
2903 LINUX_MIB_TCPABORTONLINGER);
2904 } else {
2905 const int tmo = tcp_fin_time(sk);
2906
2907 if (tmo > TCP_TIMEWAIT_LEN) {
2908 inet_csk_reset_keepalive_timer(sk,
2909 tmo - TCP_TIMEWAIT_LEN);
2910 } else {
2911 tcp_time_wait(sk, TCP_FIN_WAIT2, tmo);
2912 goto out;
2913 }
2914 }
2915 }
2916 if (sk->sk_state != TCP_CLOSE) {
2917 if (tcp_check_oom(sk, 0)) {
2918 tcp_set_state(sk, TCP_CLOSE);
2919 tcp_send_active_reset(sk, GFP_ATOMIC,
2920 SK_RST_REASON_NOT_SPECIFIED);
2921 __NET_INC_STATS(sock_net(sk),
2922 LINUX_MIB_TCPABORTONMEMORY);
2923 } else if (!check_net(sock_net(sk))) {
2924 /* Not possible to send reset; just close */
2925 tcp_set_state(sk, TCP_CLOSE);
2926 }
2927 }
2928
2929 if (sk->sk_state == TCP_CLOSE) {
2930 struct request_sock *req;
2931
2932 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk,
2933 lockdep_sock_is_held(sk));
2934 /* We could get here with a non-NULL req if the socket is
2935 * aborted (e.g., closed with unread data) before 3WHS
2936 * finishes.
2937 */
2938 if (req)
2939 reqsk_fastopen_remove(sk, req, false);
2940 inet_csk_destroy_sock(sk);
2941 }
2942 /* Otherwise, socket is reprieved until protocol close. */
2943
2944 out:
2945 bh_unlock_sock(sk);
2946 local_bh_enable();
2947 }
2948
tcp_close(struct sock * sk,long timeout)2949 void tcp_close(struct sock *sk, long timeout)
2950 {
2951 lock_sock(sk);
2952 __tcp_close(sk, timeout);
2953 release_sock(sk);
2954 if (!sk->sk_net_refcnt)
2955 inet_csk_clear_xmit_timers_sync(sk);
2956 sock_put(sk);
2957 }
2958 EXPORT_SYMBOL(tcp_close);
2959
2960 /* These states need RST on ABORT according to RFC793 */
2961
tcp_need_reset(int state)2962 static inline bool tcp_need_reset(int state)
2963 {
2964 return (1 << state) &
2965 (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT | TCPF_FIN_WAIT1 |
2966 TCPF_FIN_WAIT2 | TCPF_SYN_RECV);
2967 }
2968
tcp_rtx_queue_purge(struct sock * sk)2969 static void tcp_rtx_queue_purge(struct sock *sk)
2970 {
2971 struct rb_node *p = rb_first(&sk->tcp_rtx_queue);
2972
2973 tcp_sk(sk)->highest_sack = NULL;
2974 while (p) {
2975 struct sk_buff *skb = rb_to_skb(p);
2976
2977 p = rb_next(p);
2978 /* Since we are deleting whole queue, no need to
2979 * list_del(&skb->tcp_tsorted_anchor)
2980 */
2981 tcp_rtx_queue_unlink(skb, sk);
2982 tcp_wmem_free_skb(sk, skb);
2983 }
2984 }
2985
tcp_write_queue_purge(struct sock * sk)2986 void tcp_write_queue_purge(struct sock *sk)
2987 {
2988 struct sk_buff *skb;
2989
2990 tcp_chrono_stop(sk, TCP_CHRONO_BUSY);
2991 while ((skb = __skb_dequeue(&sk->sk_write_queue)) != NULL) {
2992 tcp_skb_tsorted_anchor_cleanup(skb);
2993 tcp_wmem_free_skb(sk, skb);
2994 }
2995 tcp_rtx_queue_purge(sk);
2996 INIT_LIST_HEAD(&tcp_sk(sk)->tsorted_sent_queue);
2997 tcp_clear_all_retrans_hints(tcp_sk(sk));
2998 tcp_sk(sk)->packets_out = 0;
2999 inet_csk(sk)->icsk_backoff = 0;
3000 }
3001
tcp_disconnect(struct sock * sk,int flags)3002 int tcp_disconnect(struct sock *sk, int flags)
3003 {
3004 struct inet_sock *inet = inet_sk(sk);
3005 struct inet_connection_sock *icsk = inet_csk(sk);
3006 struct tcp_sock *tp = tcp_sk(sk);
3007 int old_state = sk->sk_state;
3008 u32 seq;
3009
3010 if (old_state != TCP_CLOSE)
3011 tcp_set_state(sk, TCP_CLOSE);
3012
3013 /* ABORT function of RFC793 */
3014 if (old_state == TCP_LISTEN) {
3015 inet_csk_listen_stop(sk);
3016 } else if (unlikely(tp->repair)) {
3017 WRITE_ONCE(sk->sk_err, ECONNABORTED);
3018 } else if (tcp_need_reset(old_state) ||
3019 (tp->snd_nxt != tp->write_seq &&
3020 (1 << old_state) & (TCPF_CLOSING | TCPF_LAST_ACK))) {
3021 /* The last check adjusts for discrepancy of Linux wrt. RFC
3022 * states
3023 */
3024 tcp_send_active_reset(sk, gfp_any(), SK_RST_REASON_NOT_SPECIFIED);
3025 WRITE_ONCE(sk->sk_err, ECONNRESET);
3026 } else if (old_state == TCP_SYN_SENT)
3027 WRITE_ONCE(sk->sk_err, ECONNRESET);
3028
3029 tcp_clear_xmit_timers(sk);
3030 __skb_queue_purge(&sk->sk_receive_queue);
3031 WRITE_ONCE(tp->copied_seq, tp->rcv_nxt);
3032 WRITE_ONCE(tp->urg_data, 0);
3033 sk_set_peek_off(sk, -1);
3034 tcp_write_queue_purge(sk);
3035 tcp_fastopen_active_disable_ofo_check(sk);
3036 skb_rbtree_purge(&tp->out_of_order_queue);
3037
3038 inet->inet_dport = 0;
3039
3040 inet_bhash2_reset_saddr(sk);
3041
3042 WRITE_ONCE(sk->sk_shutdown, 0);
3043 sock_reset_flag(sk, SOCK_DONE);
3044 tp->srtt_us = 0;
3045 tp->mdev_us = jiffies_to_usecs(TCP_TIMEOUT_INIT);
3046 tp->rcv_rtt_last_tsecr = 0;
3047
3048 seq = tp->write_seq + tp->max_window + 2;
3049 if (!seq)
3050 seq = 1;
3051 WRITE_ONCE(tp->write_seq, seq);
3052
3053 icsk->icsk_backoff = 0;
3054 icsk->icsk_probes_out = 0;
3055 icsk->icsk_probes_tstamp = 0;
3056 icsk->icsk_rto = TCP_TIMEOUT_INIT;
3057 icsk->icsk_rto_min = TCP_RTO_MIN;
3058 icsk->icsk_delack_max = TCP_DELACK_MAX;
3059 tp->snd_ssthresh = TCP_INFINITE_SSTHRESH;
3060 tcp_snd_cwnd_set(tp, TCP_INIT_CWND);
3061 tp->snd_cwnd_cnt = 0;
3062 tp->is_cwnd_limited = 0;
3063 tp->max_packets_out = 0;
3064 tp->window_clamp = 0;
3065 tp->delivered = 0;
3066 tp->delivered_ce = 0;
3067 if (icsk->icsk_ca_ops->release)
3068 icsk->icsk_ca_ops->release(sk);
3069 memset(icsk->icsk_ca_priv, 0, sizeof(icsk->icsk_ca_priv));
3070 icsk->icsk_ca_initialized = 0;
3071 tcp_set_ca_state(sk, TCP_CA_Open);
3072 tp->is_sack_reneg = 0;
3073 tcp_clear_retrans(tp);
3074 tp->total_retrans = 0;
3075 inet_csk_delack_init(sk);
3076 /* Initialize rcv_mss to TCP_MIN_MSS to avoid division by 0
3077 * issue in __tcp_select_window()
3078 */
3079 icsk->icsk_ack.rcv_mss = TCP_MIN_MSS;
3080 memset(&tp->rx_opt, 0, sizeof(tp->rx_opt));
3081 __sk_dst_reset(sk);
3082 dst_release(xchg((__force struct dst_entry **)&sk->sk_rx_dst, NULL));
3083 tcp_saved_syn_free(tp);
3084 tp->compressed_ack = 0;
3085 tp->segs_in = 0;
3086 tp->segs_out = 0;
3087 tp->bytes_sent = 0;
3088 tp->bytes_acked = 0;
3089 tp->bytes_received = 0;
3090 tp->bytes_retrans = 0;
3091 tp->data_segs_in = 0;
3092 tp->data_segs_out = 0;
3093 tp->duplicate_sack[0].start_seq = 0;
3094 tp->duplicate_sack[0].end_seq = 0;
3095 tp->dsack_dups = 0;
3096 tp->reord_seen = 0;
3097 tp->retrans_out = 0;
3098 tp->sacked_out = 0;
3099 tp->tlp_high_seq = 0;
3100 tp->last_oow_ack_time = 0;
3101 tp->plb_rehash = 0;
3102 /* There's a bubble in the pipe until at least the first ACK. */
3103 tp->app_limited = ~0U;
3104 tp->rate_app_limited = 1;
3105 tp->rack.mstamp = 0;
3106 tp->rack.advanced = 0;
3107 tp->rack.reo_wnd_steps = 1;
3108 tp->rack.last_delivered = 0;
3109 tp->rack.reo_wnd_persist = 0;
3110 tp->rack.dsack_seen = 0;
3111 tp->syn_data_acked = 0;
3112 tp->rx_opt.saw_tstamp = 0;
3113 tp->rx_opt.dsack = 0;
3114 tp->rx_opt.num_sacks = 0;
3115 tp->rcv_ooopack = 0;
3116
3117
3118 /* Clean up fastopen related fields */
3119 tcp_free_fastopen_req(tp);
3120 inet_clear_bit(DEFER_CONNECT, sk);
3121 tp->fastopen_client_fail = 0;
3122
3123 WARN_ON(inet->inet_num && !icsk->icsk_bind_hash);
3124
3125 if (sk->sk_frag.page) {
3126 put_page(sk->sk_frag.page);
3127 sk->sk_frag.page = NULL;
3128 sk->sk_frag.offset = 0;
3129 }
3130 sk_error_report(sk);
3131 return 0;
3132 }
3133 EXPORT_SYMBOL(tcp_disconnect);
3134
tcp_can_repair_sock(const struct sock * sk)3135 static inline bool tcp_can_repair_sock(const struct sock *sk)
3136 {
3137 return sockopt_ns_capable(sock_net(sk)->user_ns, CAP_NET_ADMIN) &&
3138 (sk->sk_state != TCP_LISTEN);
3139 }
3140
tcp_repair_set_window(struct tcp_sock * tp,sockptr_t optbuf,int len)3141 static int tcp_repair_set_window(struct tcp_sock *tp, sockptr_t optbuf, int len)
3142 {
3143 struct tcp_repair_window opt;
3144
3145 if (!tp->repair)
3146 return -EPERM;
3147
3148 if (len != sizeof(opt))
3149 return -EINVAL;
3150
3151 if (copy_from_sockptr(&opt, optbuf, sizeof(opt)))
3152 return -EFAULT;
3153
3154 if (opt.max_window < opt.snd_wnd)
3155 return -EINVAL;
3156
3157 if (after(opt.snd_wl1, tp->rcv_nxt + opt.rcv_wnd))
3158 return -EINVAL;
3159
3160 if (after(opt.rcv_wup, tp->rcv_nxt))
3161 return -EINVAL;
3162
3163 tp->snd_wl1 = opt.snd_wl1;
3164 tp->snd_wnd = opt.snd_wnd;
3165 tp->max_window = opt.max_window;
3166
3167 tp->rcv_wnd = opt.rcv_wnd;
3168 tp->rcv_wup = opt.rcv_wup;
3169
3170 return 0;
3171 }
3172
tcp_repair_options_est(struct sock * sk,sockptr_t optbuf,unsigned int len)3173 static int tcp_repair_options_est(struct sock *sk, sockptr_t optbuf,
3174 unsigned int len)
3175 {
3176 struct tcp_sock *tp = tcp_sk(sk);
3177 struct tcp_repair_opt opt;
3178 size_t offset = 0;
3179
3180 while (len >= sizeof(opt)) {
3181 if (copy_from_sockptr_offset(&opt, optbuf, offset, sizeof(opt)))
3182 return -EFAULT;
3183
3184 offset += sizeof(opt);
3185 len -= sizeof(opt);
3186
3187 switch (opt.opt_code) {
3188 case TCPOPT_MSS:
3189 tp->rx_opt.mss_clamp = opt.opt_val;
3190 tcp_mtup_init(sk);
3191 break;
3192 case TCPOPT_WINDOW:
3193 {
3194 u16 snd_wscale = opt.opt_val & 0xFFFF;
3195 u16 rcv_wscale = opt.opt_val >> 16;
3196
3197 if (snd_wscale > TCP_MAX_WSCALE || rcv_wscale > TCP_MAX_WSCALE)
3198 return -EFBIG;
3199
3200 tp->rx_opt.snd_wscale = snd_wscale;
3201 tp->rx_opt.rcv_wscale = rcv_wscale;
3202 tp->rx_opt.wscale_ok = 1;
3203 }
3204 break;
3205 case TCPOPT_SACK_PERM:
3206 if (opt.opt_val != 0)
3207 return -EINVAL;
3208
3209 tp->rx_opt.sack_ok |= TCP_SACK_SEEN;
3210 break;
3211 case TCPOPT_TIMESTAMP:
3212 if (opt.opt_val != 0)
3213 return -EINVAL;
3214
3215 tp->rx_opt.tstamp_ok = 1;
3216 break;
3217 }
3218 }
3219
3220 return 0;
3221 }
3222
3223 DEFINE_STATIC_KEY_FALSE(tcp_tx_delay_enabled);
3224 EXPORT_SYMBOL(tcp_tx_delay_enabled);
3225
tcp_enable_tx_delay(void)3226 static void tcp_enable_tx_delay(void)
3227 {
3228 if (!static_branch_unlikely(&tcp_tx_delay_enabled)) {
3229 static int __tcp_tx_delay_enabled = 0;
3230
3231 if (cmpxchg(&__tcp_tx_delay_enabled, 0, 1) == 0) {
3232 static_branch_enable(&tcp_tx_delay_enabled);
3233 pr_info("TCP_TX_DELAY enabled\n");
3234 }
3235 }
3236 }
3237
3238 /* When set indicates to always queue non-full frames. Later the user clears
3239 * this option and we transmit any pending partial frames in the queue. This is
3240 * meant to be used alongside sendfile() to get properly filled frames when the
3241 * user (for example) must write out headers with a write() call first and then
3242 * use sendfile to send out the data parts.
3243 *
3244 * TCP_CORK can be set together with TCP_NODELAY and it is stronger than
3245 * TCP_NODELAY.
3246 */
__tcp_sock_set_cork(struct sock * sk,bool on)3247 void __tcp_sock_set_cork(struct sock *sk, bool on)
3248 {
3249 struct tcp_sock *tp = tcp_sk(sk);
3250
3251 if (on) {
3252 tp->nonagle |= TCP_NAGLE_CORK;
3253 } else {
3254 tp->nonagle &= ~TCP_NAGLE_CORK;
3255 if (tp->nonagle & TCP_NAGLE_OFF)
3256 tp->nonagle |= TCP_NAGLE_PUSH;
3257 tcp_push_pending_frames(sk);
3258 }
3259 }
3260
tcp_sock_set_cork(struct sock * sk,bool on)3261 void tcp_sock_set_cork(struct sock *sk, bool on)
3262 {
3263 lock_sock(sk);
3264 __tcp_sock_set_cork(sk, on);
3265 release_sock(sk);
3266 }
3267 EXPORT_SYMBOL(tcp_sock_set_cork);
3268
3269 /* TCP_NODELAY is weaker than TCP_CORK, so that this option on corked socket is
3270 * remembered, but it is not activated until cork is cleared.
3271 *
3272 * However, when TCP_NODELAY is set we make an explicit push, which overrides
3273 * even TCP_CORK for currently queued segments.
3274 */
__tcp_sock_set_nodelay(struct sock * sk,bool on)3275 void __tcp_sock_set_nodelay(struct sock *sk, bool on)
3276 {
3277 if (on) {
3278 tcp_sk(sk)->nonagle |= TCP_NAGLE_OFF|TCP_NAGLE_PUSH;
3279 tcp_push_pending_frames(sk);
3280 } else {
3281 tcp_sk(sk)->nonagle &= ~TCP_NAGLE_OFF;
3282 }
3283 }
3284
tcp_sock_set_nodelay(struct sock * sk)3285 void tcp_sock_set_nodelay(struct sock *sk)
3286 {
3287 lock_sock(sk);
3288 __tcp_sock_set_nodelay(sk, true);
3289 release_sock(sk);
3290 }
3291 EXPORT_SYMBOL(tcp_sock_set_nodelay);
3292
__tcp_sock_set_quickack(struct sock * sk,int val)3293 static void __tcp_sock_set_quickack(struct sock *sk, int val)
3294 {
3295 if (!val) {
3296 inet_csk_enter_pingpong_mode(sk);
3297 return;
3298 }
3299
3300 inet_csk_exit_pingpong_mode(sk);
3301 if ((1 << sk->sk_state) & (TCPF_ESTABLISHED | TCPF_CLOSE_WAIT) &&
3302 inet_csk_ack_scheduled(sk)) {
3303 inet_csk(sk)->icsk_ack.pending |= ICSK_ACK_PUSHED;
3304 tcp_cleanup_rbuf(sk, 1);
3305 if (!(val & 1))
3306 inet_csk_enter_pingpong_mode(sk);
3307 }
3308 }
3309
tcp_sock_set_quickack(struct sock * sk,int val)3310 void tcp_sock_set_quickack(struct sock *sk, int val)
3311 {
3312 lock_sock(sk);
3313 __tcp_sock_set_quickack(sk, val);
3314 release_sock(sk);
3315 }
3316 EXPORT_SYMBOL(tcp_sock_set_quickack);
3317
tcp_sock_set_syncnt(struct sock * sk,int val)3318 int tcp_sock_set_syncnt(struct sock *sk, int val)
3319 {
3320 if (val < 1 || val > MAX_TCP_SYNCNT)
3321 return -EINVAL;
3322
3323 WRITE_ONCE(inet_csk(sk)->icsk_syn_retries, val);
3324 return 0;
3325 }
3326 EXPORT_SYMBOL(tcp_sock_set_syncnt);
3327
tcp_sock_set_user_timeout(struct sock * sk,int val)3328 int tcp_sock_set_user_timeout(struct sock *sk, int val)
3329 {
3330 /* Cap the max time in ms TCP will retry or probe the window
3331 * before giving up and aborting (ETIMEDOUT) a connection.
3332 */
3333 if (val < 0)
3334 return -EINVAL;
3335
3336 WRITE_ONCE(inet_csk(sk)->icsk_user_timeout, val);
3337 return 0;
3338 }
3339 EXPORT_SYMBOL(tcp_sock_set_user_timeout);
3340
tcp_sock_set_keepidle_locked(struct sock * sk,int val)3341 int tcp_sock_set_keepidle_locked(struct sock *sk, int val)
3342 {
3343 struct tcp_sock *tp = tcp_sk(sk);
3344
3345 if (val < 1 || val > MAX_TCP_KEEPIDLE)
3346 return -EINVAL;
3347
3348 /* Paired with WRITE_ONCE() in keepalive_time_when() */
3349 WRITE_ONCE(tp->keepalive_time, val * HZ);
3350 if (sock_flag(sk, SOCK_KEEPOPEN) &&
3351 !((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN))) {
3352 u32 elapsed = keepalive_time_elapsed(tp);
3353
3354 if (tp->keepalive_time > elapsed)
3355 elapsed = tp->keepalive_time - elapsed;
3356 else
3357 elapsed = 0;
3358 inet_csk_reset_keepalive_timer(sk, elapsed);
3359 }
3360
3361 return 0;
3362 }
3363
tcp_sock_set_keepidle(struct sock * sk,int val)3364 int tcp_sock_set_keepidle(struct sock *sk, int val)
3365 {
3366 int err;
3367
3368 lock_sock(sk);
3369 err = tcp_sock_set_keepidle_locked(sk, val);
3370 release_sock(sk);
3371 return err;
3372 }
3373 EXPORT_SYMBOL(tcp_sock_set_keepidle);
3374
tcp_sock_set_keepintvl(struct sock * sk,int val)3375 int tcp_sock_set_keepintvl(struct sock *sk, int val)
3376 {
3377 if (val < 1 || val > MAX_TCP_KEEPINTVL)
3378 return -EINVAL;
3379
3380 WRITE_ONCE(tcp_sk(sk)->keepalive_intvl, val * HZ);
3381 return 0;
3382 }
3383 EXPORT_SYMBOL(tcp_sock_set_keepintvl);
3384
tcp_sock_set_keepcnt(struct sock * sk,int val)3385 int tcp_sock_set_keepcnt(struct sock *sk, int val)
3386 {
3387 if (val < 1 || val > MAX_TCP_KEEPCNT)
3388 return -EINVAL;
3389
3390 /* Paired with READ_ONCE() in keepalive_probes() */
3391 WRITE_ONCE(tcp_sk(sk)->keepalive_probes, val);
3392 return 0;
3393 }
3394 EXPORT_SYMBOL(tcp_sock_set_keepcnt);
3395
tcp_set_window_clamp(struct sock * sk,int val)3396 int tcp_set_window_clamp(struct sock *sk, int val)
3397 {
3398 struct tcp_sock *tp = tcp_sk(sk);
3399
3400 if (!val) {
3401 if (sk->sk_state != TCP_CLOSE)
3402 return -EINVAL;
3403 WRITE_ONCE(tp->window_clamp, 0);
3404 } else {
3405 u32 new_rcv_ssthresh, old_window_clamp = tp->window_clamp;
3406 u32 new_window_clamp = val < SOCK_MIN_RCVBUF / 2 ?
3407 SOCK_MIN_RCVBUF / 2 : val;
3408
3409 if (new_window_clamp == old_window_clamp)
3410 return 0;
3411
3412 WRITE_ONCE(tp->window_clamp, new_window_clamp);
3413 if (new_window_clamp < old_window_clamp) {
3414 /* need to apply the reserved mem provisioning only
3415 * when shrinking the window clamp
3416 */
3417 __tcp_adjust_rcv_ssthresh(sk, tp->window_clamp);
3418
3419 } else {
3420 new_rcv_ssthresh = min(tp->rcv_wnd, tp->window_clamp);
3421 tp->rcv_ssthresh = max(new_rcv_ssthresh,
3422 tp->rcv_ssthresh);
3423 }
3424 }
3425 return 0;
3426 }
3427
3428 /*
3429 * Socket option code for TCP.
3430 */
do_tcp_setsockopt(struct sock * sk,int level,int optname,sockptr_t optval,unsigned int optlen)3431 int do_tcp_setsockopt(struct sock *sk, int level, int optname,
3432 sockptr_t optval, unsigned int optlen)
3433 {
3434 struct tcp_sock *tp = tcp_sk(sk);
3435 struct inet_connection_sock *icsk = inet_csk(sk);
3436 struct net *net = sock_net(sk);
3437 int val;
3438 int err = 0;
3439
3440 /* These are data/string values, all the others are ints */
3441 switch (optname) {
3442 case TCP_CONGESTION: {
3443 char name[TCP_CA_NAME_MAX];
3444
3445 if (optlen < 1)
3446 return -EINVAL;
3447
3448 val = strncpy_from_sockptr(name, optval,
3449 min_t(long, TCP_CA_NAME_MAX-1, optlen));
3450 if (val < 0)
3451 return -EFAULT;
3452 name[val] = 0;
3453
3454 sockopt_lock_sock(sk);
3455 err = tcp_set_congestion_control(sk, name, !has_current_bpf_ctx(),
3456 sockopt_ns_capable(sock_net(sk)->user_ns,
3457 CAP_NET_ADMIN));
3458 sockopt_release_sock(sk);
3459 return err;
3460 }
3461 case TCP_ULP: {
3462 char name[TCP_ULP_NAME_MAX];
3463
3464 if (optlen < 1)
3465 return -EINVAL;
3466
3467 val = strncpy_from_sockptr(name, optval,
3468 min_t(long, TCP_ULP_NAME_MAX - 1,
3469 optlen));
3470 if (val < 0)
3471 return -EFAULT;
3472 name[val] = 0;
3473
3474 sockopt_lock_sock(sk);
3475 err = tcp_set_ulp(sk, name);
3476 sockopt_release_sock(sk);
3477 return err;
3478 }
3479 case TCP_FASTOPEN_KEY: {
3480 __u8 key[TCP_FASTOPEN_KEY_BUF_LENGTH];
3481 __u8 *backup_key = NULL;
3482
3483 /* Allow a backup key as well to facilitate key rotation
3484 * First key is the active one.
3485 */
3486 if (optlen != TCP_FASTOPEN_KEY_LENGTH &&
3487 optlen != TCP_FASTOPEN_KEY_BUF_LENGTH)
3488 return -EINVAL;
3489
3490 if (copy_from_sockptr(key, optval, optlen))
3491 return -EFAULT;
3492
3493 if (optlen == TCP_FASTOPEN_KEY_BUF_LENGTH)
3494 backup_key = key + TCP_FASTOPEN_KEY_LENGTH;
3495
3496 return tcp_fastopen_reset_cipher(net, sk, key, backup_key);
3497 }
3498 default:
3499 /* fallthru */
3500 break;
3501 }
3502
3503 if (optlen < sizeof(int))
3504 return -EINVAL;
3505
3506 if (copy_from_sockptr(&val, optval, sizeof(val)))
3507 return -EFAULT;
3508
3509 /* Handle options that can be set without locking the socket. */
3510 switch (optname) {
3511 case TCP_SYNCNT:
3512 return tcp_sock_set_syncnt(sk, val);
3513 case TCP_USER_TIMEOUT:
3514 return tcp_sock_set_user_timeout(sk, val);
3515 case TCP_KEEPINTVL:
3516 return tcp_sock_set_keepintvl(sk, val);
3517 case TCP_KEEPCNT:
3518 return tcp_sock_set_keepcnt(sk, val);
3519 case TCP_LINGER2:
3520 if (val < 0)
3521 WRITE_ONCE(tp->linger2, -1);
3522 else if (val > TCP_FIN_TIMEOUT_MAX / HZ)
3523 WRITE_ONCE(tp->linger2, TCP_FIN_TIMEOUT_MAX);
3524 else
3525 WRITE_ONCE(tp->linger2, val * HZ);
3526 return 0;
3527 case TCP_DEFER_ACCEPT:
3528 /* Translate value in seconds to number of retransmits */
3529 WRITE_ONCE(icsk->icsk_accept_queue.rskq_defer_accept,
3530 secs_to_retrans(val, TCP_TIMEOUT_INIT / HZ,
3531 TCP_RTO_MAX / HZ));
3532 return 0;
3533 }
3534
3535 sockopt_lock_sock(sk);
3536
3537 switch (optname) {
3538 case TCP_MAXSEG:
3539 /* Values greater than interface MTU won't take effect. However
3540 * at the point when this call is done we typically don't yet
3541 * know which interface is going to be used
3542 */
3543 if (val && (val < TCP_MIN_MSS || val > MAX_TCP_WINDOW)) {
3544 err = -EINVAL;
3545 break;
3546 }
3547 tp->rx_opt.user_mss = val;
3548 break;
3549
3550 case TCP_NODELAY:
3551 __tcp_sock_set_nodelay(sk, val);
3552 break;
3553
3554 case TCP_THIN_LINEAR_TIMEOUTS:
3555 if (val < 0 || val > 1)
3556 err = -EINVAL;
3557 else
3558 tp->thin_lto = val;
3559 break;
3560
3561 case TCP_THIN_DUPACK:
3562 if (val < 0 || val > 1)
3563 err = -EINVAL;
3564 break;
3565
3566 case TCP_REPAIR:
3567 if (!tcp_can_repair_sock(sk))
3568 err = -EPERM;
3569 else if (val == TCP_REPAIR_ON) {
3570 tp->repair = 1;
3571 sk->sk_reuse = SK_FORCE_REUSE;
3572 tp->repair_queue = TCP_NO_QUEUE;
3573 } else if (val == TCP_REPAIR_OFF) {
3574 tp->repair = 0;
3575 sk->sk_reuse = SK_NO_REUSE;
3576 tcp_send_window_probe(sk);
3577 } else if (val == TCP_REPAIR_OFF_NO_WP) {
3578 tp->repair = 0;
3579 sk->sk_reuse = SK_NO_REUSE;
3580 } else
3581 err = -EINVAL;
3582
3583 break;
3584
3585 case TCP_REPAIR_QUEUE:
3586 if (!tp->repair)
3587 err = -EPERM;
3588 else if ((unsigned int)val < TCP_QUEUES_NR)
3589 tp->repair_queue = val;
3590 else
3591 err = -EINVAL;
3592 break;
3593
3594 case TCP_QUEUE_SEQ:
3595 if (sk->sk_state != TCP_CLOSE) {
3596 err = -EPERM;
3597 } else if (tp->repair_queue == TCP_SEND_QUEUE) {
3598 if (!tcp_rtx_queue_empty(sk))
3599 err = -EPERM;
3600 else
3601 WRITE_ONCE(tp->write_seq, val);
3602 } else if (tp->repair_queue == TCP_RECV_QUEUE) {
3603 if (tp->rcv_nxt != tp->copied_seq) {
3604 err = -EPERM;
3605 } else {
3606 WRITE_ONCE(tp->rcv_nxt, val);
3607 WRITE_ONCE(tp->copied_seq, val);
3608 }
3609 } else {
3610 err = -EINVAL;
3611 }
3612 break;
3613
3614 case TCP_REPAIR_OPTIONS:
3615 if (!tp->repair)
3616 err = -EINVAL;
3617 else if (sk->sk_state == TCP_ESTABLISHED && !tp->bytes_sent)
3618 err = tcp_repair_options_est(sk, optval, optlen);
3619 else
3620 err = -EPERM;
3621 break;
3622
3623 case TCP_CORK:
3624 __tcp_sock_set_cork(sk, val);
3625 break;
3626
3627 case TCP_KEEPIDLE:
3628 err = tcp_sock_set_keepidle_locked(sk, val);
3629 break;
3630 case TCP_SAVE_SYN:
3631 /* 0: disable, 1: enable, 2: start from ether_header */
3632 if (val < 0 || val > 2)
3633 err = -EINVAL;
3634 else
3635 tp->save_syn = val;
3636 break;
3637
3638 case TCP_WINDOW_CLAMP:
3639 err = tcp_set_window_clamp(sk, val);
3640 break;
3641
3642 case TCP_QUICKACK:
3643 __tcp_sock_set_quickack(sk, val);
3644 break;
3645
3646 case TCP_AO_REPAIR:
3647 if (!tcp_can_repair_sock(sk)) {
3648 err = -EPERM;
3649 break;
3650 }
3651 err = tcp_ao_set_repair(sk, optval, optlen);
3652 break;
3653 #ifdef CONFIG_TCP_AO
3654 case TCP_AO_ADD_KEY:
3655 case TCP_AO_DEL_KEY:
3656 case TCP_AO_INFO: {
3657 /* If this is the first TCP-AO setsockopt() on the socket,
3658 * sk_state has to be LISTEN or CLOSE. Allow TCP_REPAIR
3659 * in any state.
3660 */
3661 if ((1 << sk->sk_state) & (TCPF_LISTEN | TCPF_CLOSE))
3662 goto ao_parse;
3663 if (rcu_dereference_protected(tcp_sk(sk)->ao_info,
3664 lockdep_sock_is_held(sk)))
3665 goto ao_parse;
3666 if (tp->repair)
3667 goto ao_parse;
3668 err = -EISCONN;
3669 break;
3670 ao_parse:
3671 err = tp->af_specific->ao_parse(sk, optname, optval, optlen);
3672 break;
3673 }
3674 #endif
3675 #ifdef CONFIG_TCP_MD5SIG
3676 case TCP_MD5SIG:
3677 case TCP_MD5SIG_EXT:
3678 err = tp->af_specific->md5_parse(sk, optname, optval, optlen);
3679 break;
3680 #endif
3681 case TCP_FASTOPEN:
3682 if (val >= 0 && ((1 << sk->sk_state) & (TCPF_CLOSE |
3683 TCPF_LISTEN))) {
3684 tcp_fastopen_init_key_once(net);
3685
3686 fastopen_queue_tune(sk, val);
3687 } else {
3688 err = -EINVAL;
3689 }
3690 break;
3691 case TCP_FASTOPEN_CONNECT:
3692 if (val > 1 || val < 0) {
3693 err = -EINVAL;
3694 } else if (READ_ONCE(net->ipv4.sysctl_tcp_fastopen) &
3695 TFO_CLIENT_ENABLE) {
3696 if (sk->sk_state == TCP_CLOSE)
3697 tp->fastopen_connect = val;
3698 else
3699 err = -EINVAL;
3700 } else {
3701 err = -EOPNOTSUPP;
3702 }
3703 break;
3704 case TCP_FASTOPEN_NO_COOKIE:
3705 if (val > 1 || val < 0)
3706 err = -EINVAL;
3707 else if (!((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
3708 err = -EINVAL;
3709 else
3710 tp->fastopen_no_cookie = val;
3711 break;
3712 case TCP_TIMESTAMP:
3713 if (!tp->repair) {
3714 err = -EPERM;
3715 break;
3716 }
3717 /* val is an opaque field,
3718 * and low order bit contains usec_ts enable bit.
3719 * Its a best effort, and we do not care if user makes an error.
3720 */
3721 tp->tcp_usec_ts = val & 1;
3722 WRITE_ONCE(tp->tsoffset, val - tcp_clock_ts(tp->tcp_usec_ts));
3723 break;
3724 case TCP_REPAIR_WINDOW:
3725 err = tcp_repair_set_window(tp, optval, optlen);
3726 break;
3727 case TCP_NOTSENT_LOWAT:
3728 WRITE_ONCE(tp->notsent_lowat, val);
3729 sk->sk_write_space(sk);
3730 break;
3731 case TCP_INQ:
3732 if (val > 1 || val < 0)
3733 err = -EINVAL;
3734 else
3735 tp->recvmsg_inq = val;
3736 break;
3737 case TCP_TX_DELAY:
3738 if (val)
3739 tcp_enable_tx_delay();
3740 WRITE_ONCE(tp->tcp_tx_delay, val);
3741 break;
3742 default:
3743 err = -ENOPROTOOPT;
3744 break;
3745 }
3746
3747 sockopt_release_sock(sk);
3748 return err;
3749 }
3750
tcp_setsockopt(struct sock * sk,int level,int optname,sockptr_t optval,unsigned int optlen)3751 int tcp_setsockopt(struct sock *sk, int level, int optname, sockptr_t optval,
3752 unsigned int optlen)
3753 {
3754 const struct inet_connection_sock *icsk = inet_csk(sk);
3755
3756 if (level != SOL_TCP)
3757 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
3758 return READ_ONCE(icsk->icsk_af_ops)->setsockopt(sk, level, optname,
3759 optval, optlen);
3760 return do_tcp_setsockopt(sk, level, optname, optval, optlen);
3761 }
3762 EXPORT_SYMBOL(tcp_setsockopt);
3763
tcp_get_info_chrono_stats(const struct tcp_sock * tp,struct tcp_info * info)3764 static void tcp_get_info_chrono_stats(const struct tcp_sock *tp,
3765 struct tcp_info *info)
3766 {
3767 u64 stats[__TCP_CHRONO_MAX], total = 0;
3768 enum tcp_chrono i;
3769
3770 for (i = TCP_CHRONO_BUSY; i < __TCP_CHRONO_MAX; ++i) {
3771 stats[i] = tp->chrono_stat[i - 1];
3772 if (i == tp->chrono_type)
3773 stats[i] += tcp_jiffies32 - tp->chrono_start;
3774 stats[i] *= USEC_PER_SEC / HZ;
3775 total += stats[i];
3776 }
3777
3778 info->tcpi_busy_time = total;
3779 info->tcpi_rwnd_limited = stats[TCP_CHRONO_RWND_LIMITED];
3780 info->tcpi_sndbuf_limited = stats[TCP_CHRONO_SNDBUF_LIMITED];
3781 }
3782
3783 /* Return information about state of tcp endpoint in API format. */
tcp_get_info(struct sock * sk,struct tcp_info * info)3784 void tcp_get_info(struct sock *sk, struct tcp_info *info)
3785 {
3786 const struct tcp_sock *tp = tcp_sk(sk); /* iff sk_type == SOCK_STREAM */
3787 const struct inet_connection_sock *icsk = inet_csk(sk);
3788 unsigned long rate;
3789 u32 now;
3790 u64 rate64;
3791 bool slow;
3792
3793 memset(info, 0, sizeof(*info));
3794 if (sk->sk_type != SOCK_STREAM)
3795 return;
3796
3797 info->tcpi_state = inet_sk_state_load(sk);
3798
3799 /* Report meaningful fields for all TCP states, including listeners */
3800 rate = READ_ONCE(sk->sk_pacing_rate);
3801 rate64 = (rate != ~0UL) ? rate : ~0ULL;
3802 info->tcpi_pacing_rate = rate64;
3803
3804 rate = READ_ONCE(sk->sk_max_pacing_rate);
3805 rate64 = (rate != ~0UL) ? rate : ~0ULL;
3806 info->tcpi_max_pacing_rate = rate64;
3807
3808 info->tcpi_reordering = tp->reordering;
3809 info->tcpi_snd_cwnd = tcp_snd_cwnd(tp);
3810
3811 if (info->tcpi_state == TCP_LISTEN) {
3812 /* listeners aliased fields :
3813 * tcpi_unacked -> Number of children ready for accept()
3814 * tcpi_sacked -> max backlog
3815 */
3816 info->tcpi_unacked = READ_ONCE(sk->sk_ack_backlog);
3817 info->tcpi_sacked = READ_ONCE(sk->sk_max_ack_backlog);
3818 return;
3819 }
3820
3821 slow = lock_sock_fast(sk);
3822
3823 info->tcpi_ca_state = icsk->icsk_ca_state;
3824 info->tcpi_retransmits = icsk->icsk_retransmits;
3825 info->tcpi_probes = icsk->icsk_probes_out;
3826 info->tcpi_backoff = icsk->icsk_backoff;
3827
3828 if (tp->rx_opt.tstamp_ok)
3829 info->tcpi_options |= TCPI_OPT_TIMESTAMPS;
3830 if (tcp_is_sack(tp))
3831 info->tcpi_options |= TCPI_OPT_SACK;
3832 if (tp->rx_opt.wscale_ok) {
3833 info->tcpi_options |= TCPI_OPT_WSCALE;
3834 info->tcpi_snd_wscale = tp->rx_opt.snd_wscale;
3835 info->tcpi_rcv_wscale = tp->rx_opt.rcv_wscale;
3836 }
3837
3838 if (tp->ecn_flags & TCP_ECN_OK)
3839 info->tcpi_options |= TCPI_OPT_ECN;
3840 if (tp->ecn_flags & TCP_ECN_SEEN)
3841 info->tcpi_options |= TCPI_OPT_ECN_SEEN;
3842 if (tp->syn_data_acked)
3843 info->tcpi_options |= TCPI_OPT_SYN_DATA;
3844 if (tp->tcp_usec_ts)
3845 info->tcpi_options |= TCPI_OPT_USEC_TS;
3846
3847 info->tcpi_rto = jiffies_to_usecs(icsk->icsk_rto);
3848 info->tcpi_ato = jiffies_to_usecs(min_t(u32, icsk->icsk_ack.ato,
3849 tcp_delack_max(sk)));
3850 info->tcpi_snd_mss = tp->mss_cache;
3851 info->tcpi_rcv_mss = icsk->icsk_ack.rcv_mss;
3852
3853 info->tcpi_unacked = tp->packets_out;
3854 info->tcpi_sacked = tp->sacked_out;
3855
3856 info->tcpi_lost = tp->lost_out;
3857 info->tcpi_retrans = tp->retrans_out;
3858
3859 now = tcp_jiffies32;
3860 info->tcpi_last_data_sent = jiffies_to_msecs(now - tp->lsndtime);
3861 info->tcpi_last_data_recv = jiffies_to_msecs(now - icsk->icsk_ack.lrcvtime);
3862 info->tcpi_last_ack_recv = jiffies_to_msecs(now - tp->rcv_tstamp);
3863
3864 info->tcpi_pmtu = icsk->icsk_pmtu_cookie;
3865 info->tcpi_rcv_ssthresh = tp->rcv_ssthresh;
3866 info->tcpi_rtt = tp->srtt_us >> 3;
3867 info->tcpi_rttvar = tp->mdev_us >> 2;
3868 info->tcpi_snd_ssthresh = tp->snd_ssthresh;
3869 info->tcpi_advmss = tp->advmss;
3870
3871 info->tcpi_rcv_rtt = tp->rcv_rtt_est.rtt_us >> 3;
3872 info->tcpi_rcv_space = tp->rcvq_space.space;
3873
3874 info->tcpi_total_retrans = tp->total_retrans;
3875
3876 info->tcpi_bytes_acked = tp->bytes_acked;
3877 info->tcpi_bytes_received = tp->bytes_received;
3878 info->tcpi_notsent_bytes = max_t(int, 0, tp->write_seq - tp->snd_nxt);
3879 tcp_get_info_chrono_stats(tp, info);
3880
3881 info->tcpi_segs_out = tp->segs_out;
3882
3883 /* segs_in and data_segs_in can be updated from tcp_segs_in() from BH */
3884 info->tcpi_segs_in = READ_ONCE(tp->segs_in);
3885 info->tcpi_data_segs_in = READ_ONCE(tp->data_segs_in);
3886
3887 info->tcpi_min_rtt = tcp_min_rtt(tp);
3888 info->tcpi_data_segs_out = tp->data_segs_out;
3889
3890 info->tcpi_delivery_rate_app_limited = tp->rate_app_limited ? 1 : 0;
3891 rate64 = tcp_compute_delivery_rate(tp);
3892 if (rate64)
3893 info->tcpi_delivery_rate = rate64;
3894 info->tcpi_delivered = tp->delivered;
3895 info->tcpi_delivered_ce = tp->delivered_ce;
3896 info->tcpi_bytes_sent = tp->bytes_sent;
3897 info->tcpi_bytes_retrans = tp->bytes_retrans;
3898 info->tcpi_dsack_dups = tp->dsack_dups;
3899 info->tcpi_reord_seen = tp->reord_seen;
3900 info->tcpi_rcv_ooopack = tp->rcv_ooopack;
3901 info->tcpi_snd_wnd = tp->snd_wnd;
3902 info->tcpi_rcv_wnd = tp->rcv_wnd;
3903 info->tcpi_rehash = tp->plb_rehash + tp->timeout_rehash;
3904 info->tcpi_fastopen_client_fail = tp->fastopen_client_fail;
3905
3906 info->tcpi_total_rto = tp->total_rto;
3907 info->tcpi_total_rto_recoveries = tp->total_rto_recoveries;
3908 info->tcpi_total_rto_time = tp->total_rto_time;
3909 if (tp->rto_stamp)
3910 info->tcpi_total_rto_time += tcp_clock_ms() - tp->rto_stamp;
3911
3912 unlock_sock_fast(sk, slow);
3913 }
3914 EXPORT_SYMBOL_GPL(tcp_get_info);
3915
tcp_opt_stats_get_size(void)3916 static size_t tcp_opt_stats_get_size(void)
3917 {
3918 return
3919 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BUSY */
3920 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_RWND_LIMITED */
3921 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_SNDBUF_LIMITED */
3922 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DATA_SEGS_OUT */
3923 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_TOTAL_RETRANS */
3924 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_PACING_RATE */
3925 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_DELIVERY_RATE */
3926 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_CWND */
3927 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORDERING */
3928 nla_total_size(sizeof(u32)) + /* TCP_NLA_MIN_RTT */
3929 nla_total_size(sizeof(u8)) + /* TCP_NLA_RECUR_RETRANS */
3930 nla_total_size(sizeof(u8)) + /* TCP_NLA_DELIVERY_RATE_APP_LMT */
3931 nla_total_size(sizeof(u32)) + /* TCP_NLA_SNDQ_SIZE */
3932 nla_total_size(sizeof(u8)) + /* TCP_NLA_CA_STATE */
3933 nla_total_size(sizeof(u32)) + /* TCP_NLA_SND_SSTHRESH */
3934 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED */
3935 nla_total_size(sizeof(u32)) + /* TCP_NLA_DELIVERED_CE */
3936 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_SENT */
3937 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_BYTES_RETRANS */
3938 nla_total_size(sizeof(u32)) + /* TCP_NLA_DSACK_DUPS */
3939 nla_total_size(sizeof(u32)) + /* TCP_NLA_REORD_SEEN */
3940 nla_total_size(sizeof(u32)) + /* TCP_NLA_SRTT */
3941 nla_total_size(sizeof(u16)) + /* TCP_NLA_TIMEOUT_REHASH */
3942 nla_total_size(sizeof(u32)) + /* TCP_NLA_BYTES_NOTSENT */
3943 nla_total_size_64bit(sizeof(u64)) + /* TCP_NLA_EDT */
3944 nla_total_size(sizeof(u8)) + /* TCP_NLA_TTL */
3945 nla_total_size(sizeof(u32)) + /* TCP_NLA_REHASH */
3946 0;
3947 }
3948
3949 /* Returns TTL or hop limit of an incoming packet from skb. */
tcp_skb_ttl_or_hop_limit(const struct sk_buff * skb)3950 static u8 tcp_skb_ttl_or_hop_limit(const struct sk_buff *skb)
3951 {
3952 if (skb->protocol == htons(ETH_P_IP))
3953 return ip_hdr(skb)->ttl;
3954 else if (skb->protocol == htons(ETH_P_IPV6))
3955 return ipv6_hdr(skb)->hop_limit;
3956 else
3957 return 0;
3958 }
3959
tcp_get_timestamping_opt_stats(const struct sock * sk,const struct sk_buff * orig_skb,const struct sk_buff * ack_skb)3960 struct sk_buff *tcp_get_timestamping_opt_stats(const struct sock *sk,
3961 const struct sk_buff *orig_skb,
3962 const struct sk_buff *ack_skb)
3963 {
3964 const struct tcp_sock *tp = tcp_sk(sk);
3965 struct sk_buff *stats;
3966 struct tcp_info info;
3967 unsigned long rate;
3968 u64 rate64;
3969
3970 stats = alloc_skb(tcp_opt_stats_get_size(), GFP_ATOMIC);
3971 if (!stats)
3972 return NULL;
3973
3974 tcp_get_info_chrono_stats(tp, &info);
3975 nla_put_u64_64bit(stats, TCP_NLA_BUSY,
3976 info.tcpi_busy_time, TCP_NLA_PAD);
3977 nla_put_u64_64bit(stats, TCP_NLA_RWND_LIMITED,
3978 info.tcpi_rwnd_limited, TCP_NLA_PAD);
3979 nla_put_u64_64bit(stats, TCP_NLA_SNDBUF_LIMITED,
3980 info.tcpi_sndbuf_limited, TCP_NLA_PAD);
3981 nla_put_u64_64bit(stats, TCP_NLA_DATA_SEGS_OUT,
3982 tp->data_segs_out, TCP_NLA_PAD);
3983 nla_put_u64_64bit(stats, TCP_NLA_TOTAL_RETRANS,
3984 tp->total_retrans, TCP_NLA_PAD);
3985
3986 rate = READ_ONCE(sk->sk_pacing_rate);
3987 rate64 = (rate != ~0UL) ? rate : ~0ULL;
3988 nla_put_u64_64bit(stats, TCP_NLA_PACING_RATE, rate64, TCP_NLA_PAD);
3989
3990 rate64 = tcp_compute_delivery_rate(tp);
3991 nla_put_u64_64bit(stats, TCP_NLA_DELIVERY_RATE, rate64, TCP_NLA_PAD);
3992
3993 nla_put_u32(stats, TCP_NLA_SND_CWND, tcp_snd_cwnd(tp));
3994 nla_put_u32(stats, TCP_NLA_REORDERING, tp->reordering);
3995 nla_put_u32(stats, TCP_NLA_MIN_RTT, tcp_min_rtt(tp));
3996
3997 nla_put_u8(stats, TCP_NLA_RECUR_RETRANS, inet_csk(sk)->icsk_retransmits);
3998 nla_put_u8(stats, TCP_NLA_DELIVERY_RATE_APP_LMT, !!tp->rate_app_limited);
3999 nla_put_u32(stats, TCP_NLA_SND_SSTHRESH, tp->snd_ssthresh);
4000 nla_put_u32(stats, TCP_NLA_DELIVERED, tp->delivered);
4001 nla_put_u32(stats, TCP_NLA_DELIVERED_CE, tp->delivered_ce);
4002
4003 nla_put_u32(stats, TCP_NLA_SNDQ_SIZE, tp->write_seq - tp->snd_una);
4004 nla_put_u8(stats, TCP_NLA_CA_STATE, inet_csk(sk)->icsk_ca_state);
4005
4006 nla_put_u64_64bit(stats, TCP_NLA_BYTES_SENT, tp->bytes_sent,
4007 TCP_NLA_PAD);
4008 nla_put_u64_64bit(stats, TCP_NLA_BYTES_RETRANS, tp->bytes_retrans,
4009 TCP_NLA_PAD);
4010 nla_put_u32(stats, TCP_NLA_DSACK_DUPS, tp->dsack_dups);
4011 nla_put_u32(stats, TCP_NLA_REORD_SEEN, tp->reord_seen);
4012 nla_put_u32(stats, TCP_NLA_SRTT, tp->srtt_us >> 3);
4013 nla_put_u16(stats, TCP_NLA_TIMEOUT_REHASH, tp->timeout_rehash);
4014 nla_put_u32(stats, TCP_NLA_BYTES_NOTSENT,
4015 max_t(int, 0, tp->write_seq - tp->snd_nxt));
4016 nla_put_u64_64bit(stats, TCP_NLA_EDT, orig_skb->skb_mstamp_ns,
4017 TCP_NLA_PAD);
4018 if (ack_skb)
4019 nla_put_u8(stats, TCP_NLA_TTL,
4020 tcp_skb_ttl_or_hop_limit(ack_skb));
4021
4022 nla_put_u32(stats, TCP_NLA_REHASH, tp->plb_rehash + tp->timeout_rehash);
4023 return stats;
4024 }
4025
do_tcp_getsockopt(struct sock * sk,int level,int optname,sockptr_t optval,sockptr_t optlen)4026 int do_tcp_getsockopt(struct sock *sk, int level,
4027 int optname, sockptr_t optval, sockptr_t optlen)
4028 {
4029 struct inet_connection_sock *icsk = inet_csk(sk);
4030 struct tcp_sock *tp = tcp_sk(sk);
4031 struct net *net = sock_net(sk);
4032 int val, len;
4033
4034 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4035 return -EFAULT;
4036
4037 if (len < 0)
4038 return -EINVAL;
4039
4040 len = min_t(unsigned int, len, sizeof(int));
4041
4042 switch (optname) {
4043 case TCP_MAXSEG:
4044 val = tp->mss_cache;
4045 if (tp->rx_opt.user_mss &&
4046 ((1 << sk->sk_state) & (TCPF_CLOSE | TCPF_LISTEN)))
4047 val = tp->rx_opt.user_mss;
4048 if (tp->repair)
4049 val = tp->rx_opt.mss_clamp;
4050 break;
4051 case TCP_NODELAY:
4052 val = !!(tp->nonagle&TCP_NAGLE_OFF);
4053 break;
4054 case TCP_CORK:
4055 val = !!(tp->nonagle&TCP_NAGLE_CORK);
4056 break;
4057 case TCP_KEEPIDLE:
4058 val = keepalive_time_when(tp) / HZ;
4059 break;
4060 case TCP_KEEPINTVL:
4061 val = keepalive_intvl_when(tp) / HZ;
4062 break;
4063 case TCP_KEEPCNT:
4064 val = keepalive_probes(tp);
4065 break;
4066 case TCP_SYNCNT:
4067 val = READ_ONCE(icsk->icsk_syn_retries) ? :
4068 READ_ONCE(net->ipv4.sysctl_tcp_syn_retries);
4069 break;
4070 case TCP_LINGER2:
4071 val = READ_ONCE(tp->linger2);
4072 if (val >= 0)
4073 val = (val ? : READ_ONCE(net->ipv4.sysctl_tcp_fin_timeout)) / HZ;
4074 break;
4075 case TCP_DEFER_ACCEPT:
4076 val = READ_ONCE(icsk->icsk_accept_queue.rskq_defer_accept);
4077 val = retrans_to_secs(val, TCP_TIMEOUT_INIT / HZ,
4078 TCP_RTO_MAX / HZ);
4079 break;
4080 case TCP_WINDOW_CLAMP:
4081 val = READ_ONCE(tp->window_clamp);
4082 break;
4083 case TCP_INFO: {
4084 struct tcp_info info;
4085
4086 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4087 return -EFAULT;
4088
4089 tcp_get_info(sk, &info);
4090
4091 len = min_t(unsigned int, len, sizeof(info));
4092 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4093 return -EFAULT;
4094 if (copy_to_sockptr(optval, &info, len))
4095 return -EFAULT;
4096 return 0;
4097 }
4098 case TCP_CC_INFO: {
4099 const struct tcp_congestion_ops *ca_ops;
4100 union tcp_cc_info info;
4101 size_t sz = 0;
4102 int attr;
4103
4104 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4105 return -EFAULT;
4106
4107 ca_ops = icsk->icsk_ca_ops;
4108 if (ca_ops && ca_ops->get_info)
4109 sz = ca_ops->get_info(sk, ~0U, &attr, &info);
4110
4111 len = min_t(unsigned int, len, sz);
4112 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4113 return -EFAULT;
4114 if (copy_to_sockptr(optval, &info, len))
4115 return -EFAULT;
4116 return 0;
4117 }
4118 case TCP_QUICKACK:
4119 val = !inet_csk_in_pingpong_mode(sk);
4120 break;
4121
4122 case TCP_CONGESTION:
4123 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4124 return -EFAULT;
4125 len = min_t(unsigned int, len, TCP_CA_NAME_MAX);
4126 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4127 return -EFAULT;
4128 if (copy_to_sockptr(optval, icsk->icsk_ca_ops->name, len))
4129 return -EFAULT;
4130 return 0;
4131
4132 case TCP_ULP:
4133 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4134 return -EFAULT;
4135 len = min_t(unsigned int, len, TCP_ULP_NAME_MAX);
4136 if (!icsk->icsk_ulp_ops) {
4137 len = 0;
4138 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4139 return -EFAULT;
4140 return 0;
4141 }
4142 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4143 return -EFAULT;
4144 if (copy_to_sockptr(optval, icsk->icsk_ulp_ops->name, len))
4145 return -EFAULT;
4146 return 0;
4147
4148 case TCP_FASTOPEN_KEY: {
4149 u64 key[TCP_FASTOPEN_KEY_BUF_LENGTH / sizeof(u64)];
4150 unsigned int key_len;
4151
4152 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4153 return -EFAULT;
4154
4155 key_len = tcp_fastopen_get_cipher(net, icsk, key) *
4156 TCP_FASTOPEN_KEY_LENGTH;
4157 len = min_t(unsigned int, len, key_len);
4158 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4159 return -EFAULT;
4160 if (copy_to_sockptr(optval, key, len))
4161 return -EFAULT;
4162 return 0;
4163 }
4164 case TCP_THIN_LINEAR_TIMEOUTS:
4165 val = tp->thin_lto;
4166 break;
4167
4168 case TCP_THIN_DUPACK:
4169 val = 0;
4170 break;
4171
4172 case TCP_REPAIR:
4173 val = tp->repair;
4174 break;
4175
4176 case TCP_REPAIR_QUEUE:
4177 if (tp->repair)
4178 val = tp->repair_queue;
4179 else
4180 return -EINVAL;
4181 break;
4182
4183 case TCP_REPAIR_WINDOW: {
4184 struct tcp_repair_window opt;
4185
4186 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4187 return -EFAULT;
4188
4189 if (len != sizeof(opt))
4190 return -EINVAL;
4191
4192 if (!tp->repair)
4193 return -EPERM;
4194
4195 opt.snd_wl1 = tp->snd_wl1;
4196 opt.snd_wnd = tp->snd_wnd;
4197 opt.max_window = tp->max_window;
4198 opt.rcv_wnd = tp->rcv_wnd;
4199 opt.rcv_wup = tp->rcv_wup;
4200
4201 if (copy_to_sockptr(optval, &opt, len))
4202 return -EFAULT;
4203 return 0;
4204 }
4205 case TCP_QUEUE_SEQ:
4206 if (tp->repair_queue == TCP_SEND_QUEUE)
4207 val = tp->write_seq;
4208 else if (tp->repair_queue == TCP_RECV_QUEUE)
4209 val = tp->rcv_nxt;
4210 else
4211 return -EINVAL;
4212 break;
4213
4214 case TCP_USER_TIMEOUT:
4215 val = READ_ONCE(icsk->icsk_user_timeout);
4216 break;
4217
4218 case TCP_FASTOPEN:
4219 val = READ_ONCE(icsk->icsk_accept_queue.fastopenq.max_qlen);
4220 break;
4221
4222 case TCP_FASTOPEN_CONNECT:
4223 val = tp->fastopen_connect;
4224 break;
4225
4226 case TCP_FASTOPEN_NO_COOKIE:
4227 val = tp->fastopen_no_cookie;
4228 break;
4229
4230 case TCP_TX_DELAY:
4231 val = READ_ONCE(tp->tcp_tx_delay);
4232 break;
4233
4234 case TCP_TIMESTAMP:
4235 val = tcp_clock_ts(tp->tcp_usec_ts) + READ_ONCE(tp->tsoffset);
4236 if (tp->tcp_usec_ts)
4237 val |= 1;
4238 else
4239 val &= ~1;
4240 break;
4241 case TCP_NOTSENT_LOWAT:
4242 val = READ_ONCE(tp->notsent_lowat);
4243 break;
4244 case TCP_INQ:
4245 val = tp->recvmsg_inq;
4246 break;
4247 case TCP_SAVE_SYN:
4248 val = tp->save_syn;
4249 break;
4250 case TCP_SAVED_SYN: {
4251 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4252 return -EFAULT;
4253
4254 sockopt_lock_sock(sk);
4255 if (tp->saved_syn) {
4256 if (len < tcp_saved_syn_len(tp->saved_syn)) {
4257 len = tcp_saved_syn_len(tp->saved_syn);
4258 if (copy_to_sockptr(optlen, &len, sizeof(int))) {
4259 sockopt_release_sock(sk);
4260 return -EFAULT;
4261 }
4262 sockopt_release_sock(sk);
4263 return -EINVAL;
4264 }
4265 len = tcp_saved_syn_len(tp->saved_syn);
4266 if (copy_to_sockptr(optlen, &len, sizeof(int))) {
4267 sockopt_release_sock(sk);
4268 return -EFAULT;
4269 }
4270 if (copy_to_sockptr(optval, tp->saved_syn->data, len)) {
4271 sockopt_release_sock(sk);
4272 return -EFAULT;
4273 }
4274 tcp_saved_syn_free(tp);
4275 sockopt_release_sock(sk);
4276 } else {
4277 sockopt_release_sock(sk);
4278 len = 0;
4279 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4280 return -EFAULT;
4281 }
4282 return 0;
4283 }
4284 #ifdef CONFIG_MMU
4285 case TCP_ZEROCOPY_RECEIVE: {
4286 struct scm_timestamping_internal tss;
4287 struct tcp_zerocopy_receive zc = {};
4288 int err;
4289
4290 if (copy_from_sockptr(&len, optlen, sizeof(int)))
4291 return -EFAULT;
4292 if (len < 0 ||
4293 len < offsetofend(struct tcp_zerocopy_receive, length))
4294 return -EINVAL;
4295 if (unlikely(len > sizeof(zc))) {
4296 err = check_zeroed_sockptr(optval, sizeof(zc),
4297 len - sizeof(zc));
4298 if (err < 1)
4299 return err == 0 ? -EINVAL : err;
4300 len = sizeof(zc);
4301 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4302 return -EFAULT;
4303 }
4304 if (copy_from_sockptr(&zc, optval, len))
4305 return -EFAULT;
4306 if (zc.reserved)
4307 return -EINVAL;
4308 if (zc.msg_flags & ~(TCP_VALID_ZC_MSG_FLAGS))
4309 return -EINVAL;
4310 sockopt_lock_sock(sk);
4311 err = tcp_zerocopy_receive(sk, &zc, &tss);
4312 err = BPF_CGROUP_RUN_PROG_GETSOCKOPT_KERN(sk, level, optname,
4313 &zc, &len, err);
4314 sockopt_release_sock(sk);
4315 if (len >= offsetofend(struct tcp_zerocopy_receive, msg_flags))
4316 goto zerocopy_rcv_cmsg;
4317 switch (len) {
4318 case offsetofend(struct tcp_zerocopy_receive, msg_flags):
4319 goto zerocopy_rcv_cmsg;
4320 case offsetofend(struct tcp_zerocopy_receive, msg_controllen):
4321 case offsetofend(struct tcp_zerocopy_receive, msg_control):
4322 case offsetofend(struct tcp_zerocopy_receive, flags):
4323 case offsetofend(struct tcp_zerocopy_receive, copybuf_len):
4324 case offsetofend(struct tcp_zerocopy_receive, copybuf_address):
4325 case offsetofend(struct tcp_zerocopy_receive, err):
4326 goto zerocopy_rcv_sk_err;
4327 case offsetofend(struct tcp_zerocopy_receive, inq):
4328 goto zerocopy_rcv_inq;
4329 case offsetofend(struct tcp_zerocopy_receive, length):
4330 default:
4331 goto zerocopy_rcv_out;
4332 }
4333 zerocopy_rcv_cmsg:
4334 if (zc.msg_flags & TCP_CMSG_TS)
4335 tcp_zc_finalize_rx_tstamp(sk, &zc, &tss);
4336 else
4337 zc.msg_flags = 0;
4338 zerocopy_rcv_sk_err:
4339 if (!err)
4340 zc.err = sock_error(sk);
4341 zerocopy_rcv_inq:
4342 zc.inq = tcp_inq_hint(sk);
4343 zerocopy_rcv_out:
4344 if (!err && copy_to_sockptr(optval, &zc, len))
4345 err = -EFAULT;
4346 return err;
4347 }
4348 #endif
4349 case TCP_AO_REPAIR:
4350 if (!tcp_can_repair_sock(sk))
4351 return -EPERM;
4352 return tcp_ao_get_repair(sk, optval, optlen);
4353 case TCP_AO_GET_KEYS:
4354 case TCP_AO_INFO: {
4355 int err;
4356
4357 sockopt_lock_sock(sk);
4358 if (optname == TCP_AO_GET_KEYS)
4359 err = tcp_ao_get_mkts(sk, optval, optlen);
4360 else
4361 err = tcp_ao_get_sock_info(sk, optval, optlen);
4362 sockopt_release_sock(sk);
4363
4364 return err;
4365 }
4366 case TCP_IS_MPTCP:
4367 val = 0;
4368 break;
4369 default:
4370 return -ENOPROTOOPT;
4371 }
4372
4373 if (copy_to_sockptr(optlen, &len, sizeof(int)))
4374 return -EFAULT;
4375 if (copy_to_sockptr(optval, &val, len))
4376 return -EFAULT;
4377 return 0;
4378 }
4379
tcp_bpf_bypass_getsockopt(int level,int optname)4380 bool tcp_bpf_bypass_getsockopt(int level, int optname)
4381 {
4382 /* TCP do_tcp_getsockopt has optimized getsockopt implementation
4383 * to avoid extra socket lock for TCP_ZEROCOPY_RECEIVE.
4384 */
4385 if (level == SOL_TCP && optname == TCP_ZEROCOPY_RECEIVE)
4386 return true;
4387
4388 return false;
4389 }
4390 EXPORT_SYMBOL(tcp_bpf_bypass_getsockopt);
4391
tcp_getsockopt(struct sock * sk,int level,int optname,char __user * optval,int __user * optlen)4392 int tcp_getsockopt(struct sock *sk, int level, int optname, char __user *optval,
4393 int __user *optlen)
4394 {
4395 struct inet_connection_sock *icsk = inet_csk(sk);
4396
4397 if (level != SOL_TCP)
4398 /* Paired with WRITE_ONCE() in do_ipv6_setsockopt() and tcp_v6_connect() */
4399 return READ_ONCE(icsk->icsk_af_ops)->getsockopt(sk, level, optname,
4400 optval, optlen);
4401 return do_tcp_getsockopt(sk, level, optname, USER_SOCKPTR(optval),
4402 USER_SOCKPTR(optlen));
4403 }
4404 EXPORT_SYMBOL(tcp_getsockopt);
4405
4406 #ifdef CONFIG_TCP_MD5SIG
4407 int tcp_md5_sigpool_id = -1;
4408 EXPORT_SYMBOL_GPL(tcp_md5_sigpool_id);
4409
tcp_md5_alloc_sigpool(void)4410 int tcp_md5_alloc_sigpool(void)
4411 {
4412 size_t scratch_size;
4413 int ret;
4414
4415 scratch_size = sizeof(union tcp_md5sum_block) + sizeof(struct tcphdr);
4416 ret = tcp_sigpool_alloc_ahash("md5", scratch_size);
4417 if (ret >= 0) {
4418 /* As long as any md5 sigpool was allocated, the return
4419 * id would stay the same. Re-write the id only for the case
4420 * when previously all MD5 keys were deleted and this call
4421 * allocates the first MD5 key, which may return a different
4422 * sigpool id than was used previously.
4423 */
4424 WRITE_ONCE(tcp_md5_sigpool_id, ret); /* Avoids the compiler potentially being smart here */
4425 return 0;
4426 }
4427 return ret;
4428 }
4429
tcp_md5_release_sigpool(void)4430 void tcp_md5_release_sigpool(void)
4431 {
4432 tcp_sigpool_release(READ_ONCE(tcp_md5_sigpool_id));
4433 }
4434
tcp_md5_add_sigpool(void)4435 void tcp_md5_add_sigpool(void)
4436 {
4437 tcp_sigpool_get(READ_ONCE(tcp_md5_sigpool_id));
4438 }
4439
tcp_md5_hash_key(struct tcp_sigpool * hp,const struct tcp_md5sig_key * key)4440 int tcp_md5_hash_key(struct tcp_sigpool *hp,
4441 const struct tcp_md5sig_key *key)
4442 {
4443 u8 keylen = READ_ONCE(key->keylen); /* paired with WRITE_ONCE() in tcp_md5_do_add */
4444 struct scatterlist sg;
4445
4446 sg_init_one(&sg, key->key, keylen);
4447 ahash_request_set_crypt(hp->req, &sg, NULL, keylen);
4448
4449 /* We use data_race() because tcp_md5_do_add() might change
4450 * key->key under us
4451 */
4452 return data_race(crypto_ahash_update(hp->req));
4453 }
4454 EXPORT_SYMBOL(tcp_md5_hash_key);
4455
4456 /* Called with rcu_read_lock() */
4457 enum skb_drop_reason
tcp_inbound_md5_hash(const struct sock * sk,const struct sk_buff * skb,const void * saddr,const void * daddr,int family,int l3index,const __u8 * hash_location)4458 tcp_inbound_md5_hash(const struct sock *sk, const struct sk_buff *skb,
4459 const void *saddr, const void *daddr,
4460 int family, int l3index, const __u8 *hash_location)
4461 {
4462 /* This gets called for each TCP segment that has TCP-MD5 option.
4463 * We have 3 drop cases:
4464 * o No MD5 hash and one expected.
4465 * o MD5 hash and we're not expecting one.
4466 * o MD5 hash and its wrong.
4467 */
4468 const struct tcp_sock *tp = tcp_sk(sk);
4469 struct tcp_md5sig_key *key;
4470 u8 newhash[16];
4471 int genhash;
4472
4473 key = tcp_md5_do_lookup(sk, l3index, saddr, family);
4474
4475 if (!key && hash_location) {
4476 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5UNEXPECTED);
4477 tcp_hash_fail("Unexpected MD5 Hash found", family, skb, "");
4478 return SKB_DROP_REASON_TCP_MD5UNEXPECTED;
4479 }
4480
4481 /* Check the signature.
4482 * To support dual stack listeners, we need to handle
4483 * IPv4-mapped case.
4484 */
4485 if (family == AF_INET)
4486 genhash = tcp_v4_md5_hash_skb(newhash, key, NULL, skb);
4487 else
4488 genhash = tp->af_specific->calc_md5_hash(newhash, key,
4489 NULL, skb);
4490 if (genhash || memcmp(hash_location, newhash, 16) != 0) {
4491 NET_INC_STATS(sock_net(sk), LINUX_MIB_TCPMD5FAILURE);
4492 if (family == AF_INET) {
4493 tcp_hash_fail("MD5 Hash failed", AF_INET, skb, "%s L3 index %d",
4494 genhash ? "tcp_v4_calc_md5_hash failed"
4495 : "", l3index);
4496 } else {
4497 if (genhash) {
4498 tcp_hash_fail("MD5 Hash failed",
4499 AF_INET6, skb, "L3 index %d",
4500 l3index);
4501 } else {
4502 tcp_hash_fail("MD5 Hash mismatch",
4503 AF_INET6, skb, "L3 index %d",
4504 l3index);
4505 }
4506 }
4507 return SKB_DROP_REASON_TCP_MD5FAILURE;
4508 }
4509 return SKB_NOT_DROPPED_YET;
4510 }
4511 EXPORT_SYMBOL(tcp_inbound_md5_hash);
4512
4513 #endif
4514
tcp_done(struct sock * sk)4515 void tcp_done(struct sock *sk)
4516 {
4517 struct request_sock *req;
4518
4519 /* We might be called with a new socket, after
4520 * inet_csk_prepare_forced_close() has been called
4521 * so we can not use lockdep_sock_is_held(sk)
4522 */
4523 req = rcu_dereference_protected(tcp_sk(sk)->fastopen_rsk, 1);
4524
4525 if (sk->sk_state == TCP_SYN_SENT || sk->sk_state == TCP_SYN_RECV)
4526 TCP_INC_STATS(sock_net(sk), TCP_MIB_ATTEMPTFAILS);
4527
4528 tcp_set_state(sk, TCP_CLOSE);
4529 tcp_clear_xmit_timers(sk);
4530 if (req)
4531 reqsk_fastopen_remove(sk, req, false);
4532
4533 WRITE_ONCE(sk->sk_shutdown, SHUTDOWN_MASK);
4534
4535 if (!sock_flag(sk, SOCK_DEAD))
4536 sk->sk_state_change(sk);
4537 else
4538 inet_csk_destroy_sock(sk);
4539 }
4540 EXPORT_SYMBOL_GPL(tcp_done);
4541
tcp_abort(struct sock * sk,int err)4542 int tcp_abort(struct sock *sk, int err)
4543 {
4544 int state = inet_sk_state_load(sk);
4545
4546 if (state == TCP_NEW_SYN_RECV) {
4547 struct request_sock *req = inet_reqsk(sk);
4548
4549 local_bh_disable();
4550 inet_csk_reqsk_queue_drop(req->rsk_listener, req);
4551 local_bh_enable();
4552 return 0;
4553 }
4554 if (state == TCP_TIME_WAIT) {
4555 struct inet_timewait_sock *tw = inet_twsk(sk);
4556
4557 refcount_inc(&tw->tw_refcnt);
4558 local_bh_disable();
4559 inet_twsk_deschedule_put(tw);
4560 local_bh_enable();
4561 return 0;
4562 }
4563
4564 /* BPF context ensures sock locking. */
4565 if (!has_current_bpf_ctx())
4566 /* Don't race with userspace socket closes such as tcp_close. */
4567 lock_sock(sk);
4568
4569 if (sk->sk_state == TCP_LISTEN) {
4570 tcp_set_state(sk, TCP_CLOSE);
4571 inet_csk_listen_stop(sk);
4572 }
4573
4574 /* Don't race with BH socket closes such as inet_csk_listen_stop. */
4575 local_bh_disable();
4576 bh_lock_sock(sk);
4577
4578 if (!sock_flag(sk, SOCK_DEAD)) {
4579 WRITE_ONCE(sk->sk_err, err);
4580 /* This barrier is coupled with smp_rmb() in tcp_poll() */
4581 smp_wmb();
4582 sk_error_report(sk);
4583 if (tcp_need_reset(sk->sk_state))
4584 tcp_send_active_reset(sk, GFP_ATOMIC,
4585 SK_RST_REASON_NOT_SPECIFIED);
4586 tcp_done(sk);
4587 }
4588
4589 bh_unlock_sock(sk);
4590 local_bh_enable();
4591 tcp_write_queue_purge(sk);
4592 if (!has_current_bpf_ctx())
4593 release_sock(sk);
4594 return 0;
4595 }
4596 EXPORT_SYMBOL_GPL(tcp_abort);
4597
4598 extern struct tcp_congestion_ops tcp_reno;
4599
4600 static __initdata unsigned long thash_entries;
set_thash_entries(char * str)4601 static int __init set_thash_entries(char *str)
4602 {
4603 ssize_t ret;
4604
4605 if (!str)
4606 return 0;
4607
4608 ret = kstrtoul(str, 0, &thash_entries);
4609 if (ret)
4610 return 0;
4611
4612 return 1;
4613 }
4614 __setup("thash_entries=", set_thash_entries);
4615
tcp_init_mem(void)4616 static void __init tcp_init_mem(void)
4617 {
4618 unsigned long limit = nr_free_buffer_pages() / 16;
4619
4620 limit = max(limit, 128UL);
4621 sysctl_tcp_mem[0] = limit / 4 * 3; /* 4.68 % */
4622 sysctl_tcp_mem[1] = limit; /* 6.25 % */
4623 sysctl_tcp_mem[2] = sysctl_tcp_mem[0] * 2; /* 9.37 % */
4624 }
4625
tcp_struct_check(void)4626 static void __init tcp_struct_check(void)
4627 {
4628 /* TX read-mostly hotpath cache lines */
4629 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, max_window);
4630 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, rcv_ssthresh);
4631 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, reordering);
4632 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, notsent_lowat);
4633 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, gso_segs);
4634 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, lost_skb_hint);
4635 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_tx, retransmit_skb_hint);
4636 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_tx, 40);
4637
4638 /* TXRX read-mostly hotpath cache lines */
4639 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, tsoffset);
4640 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_wnd);
4641 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, mss_cache);
4642 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, snd_cwnd);
4643 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, prr_out);
4644 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, lost_out);
4645 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, sacked_out);
4646 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_txrx, scaling_ratio);
4647 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_txrx, 32);
4648
4649 /* RX read-mostly hotpath cache lines */
4650 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, copied_seq);
4651 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rcv_tstamp);
4652 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_wl1);
4653 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, tlp_high_seq);
4654 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rttvar_us);
4655 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, retrans_out);
4656 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, advmss);
4657 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, urg_data);
4658 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, lost);
4659 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, rtt_min);
4660 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, out_of_order_queue);
4661 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_read_rx, snd_ssthresh);
4662 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_read_rx, 69);
4663
4664 /* TX read-write hotpath cache lines */
4665 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, segs_out);
4666 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, data_segs_out);
4667 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, bytes_sent);
4668 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, snd_sml);
4669 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_start);
4670 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, chrono_stat);
4671 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, write_seq);
4672 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, pushed_seq);
4673 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, lsndtime);
4674 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, mdev_us);
4675 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tcp_wstamp_ns);
4676 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, rtt_seq);
4677 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, tsorted_sent_queue);
4678 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, highest_sack);
4679 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_tx, ecn_flags);
4680 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_tx, 89);
4681
4682 /* TXRX read-write hotpath cache lines */
4683 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, pred_flags);
4684 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_clock_cache);
4685 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, tcp_mstamp);
4686 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_nxt);
4687 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_nxt);
4688 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_una);
4689 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, window_clamp);
4690 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, srtt_us);
4691 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, packets_out);
4692 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, snd_up);
4693 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered);
4694 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, delivered_ce);
4695 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, app_limited);
4696 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rcv_wnd);
4697 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_txrx, rx_opt);
4698
4699 /* 32bit arches with 8byte alignment on u64 fields might need padding
4700 * before tcp_clock_cache.
4701 */
4702 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_txrx, 92 + 4);
4703
4704 /* RX read-write hotpath cache lines */
4705 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_received);
4706 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, segs_in);
4707 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, data_segs_in);
4708 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_wup);
4709 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, max_packets_out);
4710 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, cwnd_usage_seq);
4711 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_delivered);
4712 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rate_interval_us);
4713 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_last_tsecr);
4714 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, first_tx_mstamp);
4715 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, delivered_mstamp);
4716 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, bytes_acked);
4717 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcv_rtt_est);
4718 CACHELINE_ASSERT_GROUP_MEMBER(struct tcp_sock, tcp_sock_write_rx, rcvq_space);
4719 CACHELINE_ASSERT_GROUP_SIZE(struct tcp_sock, tcp_sock_write_rx, 99);
4720 }
4721
tcp_init(void)4722 void __init tcp_init(void)
4723 {
4724 int max_rshare, max_wshare, cnt;
4725 unsigned long limit;
4726 unsigned int i;
4727
4728 BUILD_BUG_ON(TCP_MIN_SND_MSS <= MAX_TCP_OPTION_SPACE);
4729 BUILD_BUG_ON(sizeof(struct tcp_skb_cb) >
4730 sizeof_field(struct sk_buff, cb));
4731
4732 tcp_struct_check();
4733
4734 percpu_counter_init(&tcp_sockets_allocated, 0, GFP_KERNEL);
4735
4736 timer_setup(&tcp_orphan_timer, tcp_orphan_update, TIMER_DEFERRABLE);
4737 mod_timer(&tcp_orphan_timer, jiffies + TCP_ORPHAN_TIMER_PERIOD);
4738
4739 inet_hashinfo2_init(&tcp_hashinfo, "tcp_listen_portaddr_hash",
4740 thash_entries, 21, /* one slot per 2 MB*/
4741 0, 64 * 1024);
4742 tcp_hashinfo.bind_bucket_cachep =
4743 kmem_cache_create("tcp_bind_bucket",
4744 sizeof(struct inet_bind_bucket), 0,
4745 SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4746 SLAB_ACCOUNT,
4747 NULL);
4748 tcp_hashinfo.bind2_bucket_cachep =
4749 kmem_cache_create("tcp_bind2_bucket",
4750 sizeof(struct inet_bind2_bucket), 0,
4751 SLAB_HWCACHE_ALIGN | SLAB_PANIC |
4752 SLAB_ACCOUNT,
4753 NULL);
4754
4755 /* Size and allocate the main established and bind bucket
4756 * hash tables.
4757 *
4758 * The methodology is similar to that of the buffer cache.
4759 */
4760 tcp_hashinfo.ehash =
4761 alloc_large_system_hash("TCP established",
4762 sizeof(struct inet_ehash_bucket),
4763 thash_entries,
4764 17, /* one slot per 128 KB of memory */
4765 0,
4766 NULL,
4767 &tcp_hashinfo.ehash_mask,
4768 0,
4769 thash_entries ? 0 : 512 * 1024);
4770 for (i = 0; i <= tcp_hashinfo.ehash_mask; i++)
4771 INIT_HLIST_NULLS_HEAD(&tcp_hashinfo.ehash[i].chain, i);
4772
4773 if (inet_ehash_locks_alloc(&tcp_hashinfo))
4774 panic("TCP: failed to alloc ehash_locks");
4775 tcp_hashinfo.bhash =
4776 alloc_large_system_hash("TCP bind",
4777 2 * sizeof(struct inet_bind_hashbucket),
4778 tcp_hashinfo.ehash_mask + 1,
4779 17, /* one slot per 128 KB of memory */
4780 0,
4781 &tcp_hashinfo.bhash_size,
4782 NULL,
4783 0,
4784 64 * 1024);
4785 tcp_hashinfo.bhash_size = 1U << tcp_hashinfo.bhash_size;
4786 tcp_hashinfo.bhash2 = tcp_hashinfo.bhash + tcp_hashinfo.bhash_size;
4787 for (i = 0; i < tcp_hashinfo.bhash_size; i++) {
4788 spin_lock_init(&tcp_hashinfo.bhash[i].lock);
4789 INIT_HLIST_HEAD(&tcp_hashinfo.bhash[i].chain);
4790 spin_lock_init(&tcp_hashinfo.bhash2[i].lock);
4791 INIT_HLIST_HEAD(&tcp_hashinfo.bhash2[i].chain);
4792 }
4793
4794 tcp_hashinfo.pernet = false;
4795
4796 cnt = tcp_hashinfo.ehash_mask + 1;
4797 sysctl_tcp_max_orphans = cnt / 2;
4798
4799 tcp_init_mem();
4800 /* Set per-socket limits to no more than 1/128 the pressure threshold */
4801 limit = nr_free_buffer_pages() << (PAGE_SHIFT - 7);
4802 max_wshare = min(4UL*1024*1024, limit);
4803 max_rshare = min(6UL*1024*1024, limit);
4804
4805 init_net.ipv4.sysctl_tcp_wmem[0] = PAGE_SIZE;
4806 init_net.ipv4.sysctl_tcp_wmem[1] = 16*1024;
4807 init_net.ipv4.sysctl_tcp_wmem[2] = max(64*1024, max_wshare);
4808
4809 init_net.ipv4.sysctl_tcp_rmem[0] = PAGE_SIZE;
4810 init_net.ipv4.sysctl_tcp_rmem[1] = 131072;
4811 init_net.ipv4.sysctl_tcp_rmem[2] = max(131072, max_rshare);
4812
4813 pr_info("Hash tables configured (established %u bind %u)\n",
4814 tcp_hashinfo.ehash_mask + 1, tcp_hashinfo.bhash_size);
4815
4816 tcp_v4_init();
4817 tcp_metrics_init();
4818 BUG_ON(tcp_register_congestion_control(&tcp_reno) != 0);
4819 tcp_tasklet_init();
4820 mptcp_init();
4821 }
4822