1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 1982, 1986, 1988, 1990, 1993
5 * The Regents of the University of California.
6 * Copyright (c) 2004 The FreeBSD Foundation
7 * Copyright (c) 2004-2008 Robert N. M. Watson
8 * All rights reserved.
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 * 3. Neither the name of the University nor the names of its contributors
19 * may be used to endorse or promote products derived from this software
20 * without specific prior written permission.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 /*
36 * Comments on the socket life cycle:
37 *
38 * soalloc() sets of socket layer state for a socket, called only by
39 * socreate() and sonewconn(). Socket layer private.
40 *
41 * sodealloc() tears down socket layer state for a socket, called only by
42 * sofree() and sonewconn(). Socket layer private.
43 *
44 * pru_attach() associates protocol layer state with an allocated socket;
45 * called only once, may fail, aborting socket allocation. This is called
46 * from socreate() and sonewconn(). Socket layer private.
47 *
48 * pru_detach() disassociates protocol layer state from an attached socket,
49 * and will be called exactly once for sockets in which pru_attach() has
50 * been successfully called. If pru_attach() returned an error,
51 * pru_detach() will not be called. Socket layer private.
52 *
53 * pru_abort() and pru_close() notify the protocol layer that the last
54 * consumer of a socket is starting to tear down the socket, and that the
55 * protocol should terminate the connection. Historically, pru_abort() also
56 * detached protocol state from the socket state, but this is no longer the
57 * case.
58 *
59 * socreate() creates a socket and attaches protocol state. This is a public
60 * interface that may be used by socket layer consumers to create new
61 * sockets.
62 *
63 * sonewconn() creates a socket and attaches protocol state. This is a
64 * public interface that may be used by protocols to create new sockets when
65 * a new connection is received and will be available for accept() on a
66 * listen socket.
67 *
68 * soclose() destroys a socket after possibly waiting for it to disconnect.
69 * This is a public interface that socket consumers should use to close and
70 * release a socket when done with it.
71 *
72 * soabort() destroys a socket without waiting for it to disconnect (used
73 * only for incoming connections that are already partially or fully
74 * connected). This is used internally by the socket layer when clearing
75 * listen socket queues (due to overflow or close on the listen socket), but
76 * is also a public interface protocols may use to abort connections in
77 * their incomplete listen queues should they no longer be required. Sockets
78 * placed in completed connection listen queues should not be aborted for
79 * reasons described in the comment above the soclose() implementation. This
80 * is not a general purpose close routine, and except in the specific
81 * circumstances described here, should not be used.
82 *
83 * sofree() will free a socket and its protocol state if all references on
84 * the socket have been released, and is the public interface to attempt to
85 * free a socket when a reference is removed. This is a socket layer private
86 * interface.
87 *
88 * NOTE: In addition to socreate() and soclose(), which provide a single
89 * socket reference to the consumer to be managed as required, there are two
90 * calls to explicitly manage socket references, soref(), and sorele().
91 * Currently, these are generally required only when transitioning a socket
92 * from a listen queue to a file descriptor, in order to prevent garbage
93 * collection of the socket at an untimely moment. For a number of reasons,
94 * these interfaces are not preferred, and should be avoided.
95 *
96 * NOTE: With regard to VNETs the general rule is that callers do not set
97 * curvnet. Exceptions to this rule include soabort(), sodisconnect(),
98 * sofree(), sorele(), sonewconn() and sorflush(), which are usually called
99 * from a pre-set VNET context. sopoll() currently does not need a VNET
100 * context to be set.
101 */
102
103 #include <sys/cdefs.h>
104 #include "opt_inet.h"
105 #include "opt_inet6.h"
106 #include "opt_kern_tls.h"
107 #include "opt_ktrace.h"
108 #include "opt_sctp.h"
109
110 #include <sys/param.h>
111 #include <sys/systm.h>
112 #include <sys/capsicum.h>
113 #include <sys/fcntl.h>
114 #include <sys/limits.h>
115 #include <sys/lock.h>
116 #include <sys/mac.h>
117 #include <sys/malloc.h>
118 #include <sys/mbuf.h>
119 #include <sys/mutex.h>
120 #include <sys/domain.h>
121 #include <sys/file.h> /* for struct knote */
122 #include <sys/hhook.h>
123 #include <sys/kernel.h>
124 #include <sys/khelp.h>
125 #include <sys/ktls.h>
126 #include <sys/event.h>
127 #include <sys/eventhandler.h>
128 #include <sys/poll.h>
129 #include <sys/proc.h>
130 #include <sys/protosw.h>
131 #include <sys/sbuf.h>
132 #include <sys/socket.h>
133 #include <sys/socketvar.h>
134 #include <sys/resourcevar.h>
135 #include <net/route.h>
136 #include <sys/signalvar.h>
137 #include <sys/stat.h>
138 #include <sys/sx.h>
139 #include <sys/sysctl.h>
140 #include <sys/taskqueue.h>
141 #include <sys/uio.h>
142 #include <sys/un.h>
143 #include <sys/unpcb.h>
144 #include <sys/jail.h>
145 #include <sys/syslog.h>
146 #include <netinet/in.h>
147 #include <netinet/in_pcb.h>
148 #include <netinet/tcp.h>
149
150 #include <net/vnet.h>
151
152 #include <security/mac/mac_framework.h>
153
154 #include <vm/uma.h>
155
156 #ifdef COMPAT_FREEBSD32
157 #include <sys/mount.h>
158 #include <sys/sysent.h>
159 #include <compat/freebsd32/freebsd32.h>
160 #endif
161
162 static int soreceive_rcvoob(struct socket *so, struct uio *uio,
163 int flags);
164 static void so_rdknl_lock(void *);
165 static void so_rdknl_unlock(void *);
166 static void so_rdknl_assert_lock(void *, int);
167 static void so_wrknl_lock(void *);
168 static void so_wrknl_unlock(void *);
169 static void so_wrknl_assert_lock(void *, int);
170
171 static void filt_sordetach(struct knote *kn);
172 static int filt_soread(struct knote *kn, long hint);
173 static void filt_sowdetach(struct knote *kn);
174 static int filt_sowrite(struct knote *kn, long hint);
175 static int filt_soempty(struct knote *kn, long hint);
176 fo_kqfilter_t soo_kqfilter;
177
178 static struct filterops soread_filtops = {
179 .f_isfd = 1,
180 .f_detach = filt_sordetach,
181 .f_event = filt_soread,
182 };
183 static struct filterops sowrite_filtops = {
184 .f_isfd = 1,
185 .f_detach = filt_sowdetach,
186 .f_event = filt_sowrite,
187 };
188 static struct filterops soempty_filtops = {
189 .f_isfd = 1,
190 .f_detach = filt_sowdetach,
191 .f_event = filt_soempty,
192 };
193
194 so_gen_t so_gencnt; /* generation count for sockets */
195
196 MALLOC_DEFINE(M_SONAME, "soname", "socket name");
197 MALLOC_DEFINE(M_PCB, "pcb", "protocol control block");
198
199 #define VNET_SO_ASSERT(so) \
200 VNET_ASSERT(curvnet != NULL, \
201 ("%s:%d curvnet is NULL, so=%p", __func__, __LINE__, (so)));
202
203 #ifdef SOCKET_HHOOK
204 VNET_DEFINE(struct hhook_head *, socket_hhh[HHOOK_SOCKET_LAST + 1]);
205 #define V_socket_hhh VNET(socket_hhh)
206 static inline int hhook_run_socket(struct socket *, void *, int32_t);
207 #endif
208
209 /*
210 * Limit on the number of connections in the listen queue waiting
211 * for accept(2).
212 * NB: The original sysctl somaxconn is still available but hidden
213 * to prevent confusion about the actual purpose of this number.
214 */
215 static u_int somaxconn = SOMAXCONN;
216
217 static int
sysctl_somaxconn(SYSCTL_HANDLER_ARGS)218 sysctl_somaxconn(SYSCTL_HANDLER_ARGS)
219 {
220 int error;
221 int val;
222
223 val = somaxconn;
224 error = sysctl_handle_int(oidp, &val, 0, req);
225 if (error || !req->newptr )
226 return (error);
227
228 /*
229 * The purpose of the UINT_MAX / 3 limit, is so that the formula
230 * 3 * so_qlimit / 2
231 * below, will not overflow.
232 */
233
234 if (val < 1 || val > UINT_MAX / 3)
235 return (EINVAL);
236
237 somaxconn = val;
238 return (0);
239 }
240 SYSCTL_PROC(_kern_ipc, OID_AUTO, soacceptqueue,
241 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_MPSAFE, 0, sizeof(int),
242 sysctl_somaxconn, "I",
243 "Maximum listen socket pending connection accept queue size");
244 SYSCTL_PROC(_kern_ipc, KIPC_SOMAXCONN, somaxconn,
245 CTLTYPE_UINT | CTLFLAG_RW | CTLFLAG_SKIP | CTLFLAG_MPSAFE, 0,
246 sizeof(int), sysctl_somaxconn, "I",
247 "Maximum listen socket pending connection accept queue size (compat)");
248
249 static int numopensockets;
250 SYSCTL_INT(_kern_ipc, OID_AUTO, numopensockets, CTLFLAG_RD,
251 &numopensockets, 0, "Number of open sockets");
252
253 /*
254 * so_global_mtx protects so_gencnt, numopensockets, and the per-socket
255 * so_gencnt field.
256 */
257 static struct mtx so_global_mtx;
258 MTX_SYSINIT(so_global_mtx, &so_global_mtx, "so_glabel", MTX_DEF);
259
260 /*
261 * General IPC sysctl name space, used by sockets and a variety of other IPC
262 * types.
263 */
264 SYSCTL_NODE(_kern, KERN_IPC, ipc, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
265 "IPC");
266
267 /*
268 * Initialize the socket subsystem and set up the socket
269 * memory allocator.
270 */
271 static uma_zone_t socket_zone;
272 int maxsockets;
273
274 static void
socket_zone_change(void * tag)275 socket_zone_change(void *tag)
276 {
277
278 maxsockets = uma_zone_set_max(socket_zone, maxsockets);
279 }
280
281 static void
socket_init(void * tag)282 socket_init(void *tag)
283 {
284
285 socket_zone = uma_zcreate("socket", sizeof(struct socket), NULL, NULL,
286 NULL, NULL, UMA_ALIGN_PTR, 0);
287 maxsockets = uma_zone_set_max(socket_zone, maxsockets);
288 uma_zone_set_warning(socket_zone, "kern.ipc.maxsockets limit reached");
289 EVENTHANDLER_REGISTER(maxsockets_change, socket_zone_change, NULL,
290 EVENTHANDLER_PRI_FIRST);
291 }
292 SYSINIT(socket, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY, socket_init, NULL);
293
294 #ifdef SOCKET_HHOOK
295 static void
socket_hhook_register(int subtype)296 socket_hhook_register(int subtype)
297 {
298
299 if (hhook_head_register(HHOOK_TYPE_SOCKET, subtype,
300 &V_socket_hhh[subtype],
301 HHOOK_NOWAIT|HHOOK_HEADISINVNET) != 0)
302 printf("%s: WARNING: unable to register hook\n", __func__);
303 }
304
305 static void
socket_hhook_deregister(int subtype)306 socket_hhook_deregister(int subtype)
307 {
308
309 if (hhook_head_deregister(V_socket_hhh[subtype]) != 0)
310 printf("%s: WARNING: unable to deregister hook\n", __func__);
311 }
312
313 static void
socket_vnet_init(const void * unused __unused)314 socket_vnet_init(const void *unused __unused)
315 {
316 int i;
317
318 /* We expect a contiguous range */
319 for (i = 0; i <= HHOOK_SOCKET_LAST; i++)
320 socket_hhook_register(i);
321 }
322 VNET_SYSINIT(socket_vnet_init, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY,
323 socket_vnet_init, NULL);
324
325 static void
socket_vnet_uninit(const void * unused __unused)326 socket_vnet_uninit(const void *unused __unused)
327 {
328 int i;
329
330 for (i = 0; i <= HHOOK_SOCKET_LAST; i++)
331 socket_hhook_deregister(i);
332 }
333 VNET_SYSUNINIT(socket_vnet_uninit, SI_SUB_PROTO_DOMAININIT, SI_ORDER_ANY,
334 socket_vnet_uninit, NULL);
335 #endif /* SOCKET_HHOOK */
336
337 /*
338 * Initialise maxsockets. This SYSINIT must be run after
339 * tunable_mbinit().
340 */
341 static void
init_maxsockets(void * ignored)342 init_maxsockets(void *ignored)
343 {
344
345 TUNABLE_INT_FETCH("kern.ipc.maxsockets", &maxsockets);
346 maxsockets = imax(maxsockets, maxfiles);
347 }
348 SYSINIT(param, SI_SUB_TUNABLES, SI_ORDER_ANY, init_maxsockets, NULL);
349
350 /*
351 * Sysctl to get and set the maximum global sockets limit. Notify protocols
352 * of the change so that they can update their dependent limits as required.
353 */
354 static int
sysctl_maxsockets(SYSCTL_HANDLER_ARGS)355 sysctl_maxsockets(SYSCTL_HANDLER_ARGS)
356 {
357 int error, newmaxsockets;
358
359 newmaxsockets = maxsockets;
360 error = sysctl_handle_int(oidp, &newmaxsockets, 0, req);
361 if (error == 0 && req->newptr && newmaxsockets != maxsockets) {
362 if (newmaxsockets > maxsockets &&
363 newmaxsockets <= maxfiles) {
364 maxsockets = newmaxsockets;
365 EVENTHANDLER_INVOKE(maxsockets_change);
366 } else
367 error = EINVAL;
368 }
369 return (error);
370 }
371 SYSCTL_PROC(_kern_ipc, OID_AUTO, maxsockets,
372 CTLTYPE_INT | CTLFLAG_RWTUN | CTLFLAG_NOFETCH | CTLFLAG_MPSAFE,
373 &maxsockets, 0, sysctl_maxsockets, "IU",
374 "Maximum number of sockets available");
375
376 /*
377 * Socket operation routines. These routines are called by the routines in
378 * sys_socket.c or from a system process, and implement the semantics of
379 * socket operations by switching out to the protocol specific routines.
380 */
381
382 /*
383 * Get a socket structure from our zone, and initialize it. Note that it
384 * would probably be better to allocate socket and PCB at the same time, but
385 * I'm not convinced that all the protocols can be easily modified to do
386 * this.
387 *
388 * soalloc() returns a socket with a ref count of 0.
389 */
390 static struct socket *
soalloc(struct vnet * vnet)391 soalloc(struct vnet *vnet)
392 {
393 struct socket *so;
394
395 so = uma_zalloc(socket_zone, M_NOWAIT | M_ZERO);
396 if (so == NULL)
397 return (NULL);
398 #ifdef MAC
399 if (mac_socket_init(so, M_NOWAIT) != 0) {
400 uma_zfree(socket_zone, so);
401 return (NULL);
402 }
403 #endif
404 if (khelp_init_osd(HELPER_CLASS_SOCKET, &so->osd)) {
405 uma_zfree(socket_zone, so);
406 return (NULL);
407 }
408
409 /*
410 * The socket locking protocol allows to lock 2 sockets at a time,
411 * however, the first one must be a listening socket. WITNESS lacks
412 * a feature to change class of an existing lock, so we use DUPOK.
413 */
414 mtx_init(&so->so_lock, "socket", NULL, MTX_DEF | MTX_DUPOK);
415 mtx_init(&so->so_snd_mtx, "so_snd", NULL, MTX_DEF);
416 mtx_init(&so->so_rcv_mtx, "so_rcv", NULL, MTX_DEF);
417 so->so_rcv.sb_sel = &so->so_rdsel;
418 so->so_snd.sb_sel = &so->so_wrsel;
419 sx_init(&so->so_snd_sx, "so_snd_sx");
420 sx_init(&so->so_rcv_sx, "so_rcv_sx");
421 TAILQ_INIT(&so->so_snd.sb_aiojobq);
422 TAILQ_INIT(&so->so_rcv.sb_aiojobq);
423 TASK_INIT(&so->so_snd.sb_aiotask, 0, soaio_snd, so);
424 TASK_INIT(&so->so_rcv.sb_aiotask, 0, soaio_rcv, so);
425 #ifdef VIMAGE
426 VNET_ASSERT(vnet != NULL, ("%s:%d vnet is NULL, so=%p",
427 __func__, __LINE__, so));
428 so->so_vnet = vnet;
429 #endif
430 #ifdef SOCKET_HHOOK
431 /* We shouldn't need the so_global_mtx */
432 if (hhook_run_socket(so, NULL, HHOOK_SOCKET_CREATE)) {
433 /* Do we need more comprehensive error returns? */
434 uma_zfree(socket_zone, so);
435 return (NULL);
436 }
437 #endif
438 mtx_lock(&so_global_mtx);
439 so->so_gencnt = ++so_gencnt;
440 ++numopensockets;
441 #ifdef VIMAGE
442 vnet->vnet_sockcnt++;
443 #endif
444 mtx_unlock(&so_global_mtx);
445
446 return (so);
447 }
448
449 /*
450 * Free the storage associated with a socket at the socket layer, tear down
451 * locks, labels, etc. All protocol state is assumed already to have been
452 * torn down (and possibly never set up) by the caller.
453 */
454 void
sodealloc(struct socket * so)455 sodealloc(struct socket *so)
456 {
457
458 KASSERT(so->so_count == 0, ("sodealloc(): so_count %d", so->so_count));
459 KASSERT(so->so_pcb == NULL, ("sodealloc(): so_pcb != NULL"));
460
461 mtx_lock(&so_global_mtx);
462 so->so_gencnt = ++so_gencnt;
463 --numopensockets; /* Could be below, but faster here. */
464 #ifdef VIMAGE
465 VNET_ASSERT(so->so_vnet != NULL, ("%s:%d so_vnet is NULL, so=%p",
466 __func__, __LINE__, so));
467 so->so_vnet->vnet_sockcnt--;
468 #endif
469 mtx_unlock(&so_global_mtx);
470 #ifdef MAC
471 mac_socket_destroy(so);
472 #endif
473 #ifdef SOCKET_HHOOK
474 hhook_run_socket(so, NULL, HHOOK_SOCKET_CLOSE);
475 #endif
476
477 khelp_destroy_osd(&so->osd);
478 if (SOLISTENING(so)) {
479 if (so->sol_accept_filter != NULL)
480 accept_filt_setopt(so, NULL);
481 } else {
482 if (so->so_rcv.sb_hiwat)
483 (void)chgsbsize(so->so_cred->cr_uidinfo,
484 &so->so_rcv.sb_hiwat, 0, RLIM_INFINITY);
485 if (so->so_snd.sb_hiwat)
486 (void)chgsbsize(so->so_cred->cr_uidinfo,
487 &so->so_snd.sb_hiwat, 0, RLIM_INFINITY);
488 sx_destroy(&so->so_snd_sx);
489 sx_destroy(&so->so_rcv_sx);
490 mtx_destroy(&so->so_snd_mtx);
491 mtx_destroy(&so->so_rcv_mtx);
492 }
493 crfree(so->so_cred);
494 mtx_destroy(&so->so_lock);
495 uma_zfree(socket_zone, so);
496 }
497
498 /*
499 * socreate returns a socket with a ref count of 1 and a file descriptor
500 * reference. The socket should be closed with soclose().
501 */
502 int
socreate(int dom,struct socket ** aso,int type,int proto,struct ucred * cred,struct thread * td)503 socreate(int dom, struct socket **aso, int type, int proto,
504 struct ucred *cred, struct thread *td)
505 {
506 struct protosw *prp;
507 struct socket *so;
508 int error;
509
510 /*
511 * XXX: divert(4) historically abused PF_INET. Keep this compatibility
512 * shim until all applications have been updated.
513 */
514 if (__predict_false(dom == PF_INET && type == SOCK_RAW &&
515 proto == IPPROTO_DIVERT)) {
516 dom = PF_DIVERT;
517 printf("%s uses obsolete way to create divert(4) socket\n",
518 td->td_proc->p_comm);
519 }
520
521 prp = pffindproto(dom, type, proto);
522 if (prp == NULL) {
523 /* No support for domain. */
524 if (pffinddomain(dom) == NULL)
525 return (EAFNOSUPPORT);
526 /* No support for socket type. */
527 if (proto == 0 && type != 0)
528 return (EPROTOTYPE);
529 return (EPROTONOSUPPORT);
530 }
531
532 MPASS(prp->pr_attach);
533
534 if ((prp->pr_flags & PR_CAPATTACH) == 0) {
535 if (CAP_TRACING(td))
536 ktrcapfail(CAPFAIL_PROTO, &proto);
537 if (IN_CAPABILITY_MODE(td))
538 return (ECAPMODE);
539 }
540
541 if (prison_check_af(cred, prp->pr_domain->dom_family) != 0)
542 return (EPROTONOSUPPORT);
543
544 so = soalloc(CRED_TO_VNET(cred));
545 if (so == NULL)
546 return (ENOBUFS);
547
548 so->so_type = type;
549 so->so_cred = crhold(cred);
550 if ((prp->pr_domain->dom_family == PF_INET) ||
551 (prp->pr_domain->dom_family == PF_INET6) ||
552 (prp->pr_domain->dom_family == PF_ROUTE))
553 so->so_fibnum = td->td_proc->p_fibnum;
554 else
555 so->so_fibnum = 0;
556 so->so_proto = prp;
557 #ifdef MAC
558 mac_socket_create(cred, so);
559 #endif
560 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
561 so_rdknl_assert_lock);
562 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
563 so_wrknl_assert_lock);
564 if ((prp->pr_flags & PR_SOCKBUF) == 0) {
565 so->so_snd.sb_mtx = &so->so_snd_mtx;
566 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
567 }
568 /*
569 * Auto-sizing of socket buffers is managed by the protocols and
570 * the appropriate flags must be set in the pru_attach function.
571 */
572 CURVNET_SET(so->so_vnet);
573 error = prp->pr_attach(so, proto, td);
574 CURVNET_RESTORE();
575 if (error) {
576 sodealloc(so);
577 return (error);
578 }
579 soref(so);
580 *aso = so;
581 return (0);
582 }
583
584 #ifdef REGRESSION
585 static int regression_sonewconn_earlytest = 1;
586 SYSCTL_INT(_regression, OID_AUTO, sonewconn_earlytest, CTLFLAG_RW,
587 ®ression_sonewconn_earlytest, 0, "Perform early sonewconn limit test");
588 #endif
589
590 static int sooverprio = LOG_DEBUG;
591 SYSCTL_INT(_kern_ipc, OID_AUTO, sooverprio, CTLFLAG_RW,
592 &sooverprio, 0, "Log priority for listen socket overflows: 0..7 or -1 to disable");
593
594 static struct timeval overinterval = { 60, 0 };
595 SYSCTL_TIMEVAL_SEC(_kern_ipc, OID_AUTO, sooverinterval, CTLFLAG_RW,
596 &overinterval,
597 "Delay in seconds between warnings for listen socket overflows");
598
599 /*
600 * When an attempt at a new connection is noted on a socket which supports
601 * accept(2), the protocol has two options:
602 * 1) Call legacy sonewconn() function, which would call protocol attach
603 * method, same as used for socket(2).
604 * 2) Call solisten_clone(), do attach that is specific to a cloned connection,
605 * and then call solisten_enqueue().
606 *
607 * Note: the ref count on the socket is 0 on return.
608 */
609 struct socket *
solisten_clone(struct socket * head)610 solisten_clone(struct socket *head)
611 {
612 struct sbuf descrsb;
613 struct socket *so;
614 int len, overcount;
615 u_int qlen;
616 const char localprefix[] = "local:";
617 char descrbuf[SUNPATHLEN + sizeof(localprefix)];
618 #if defined(INET6)
619 char addrbuf[INET6_ADDRSTRLEN];
620 #elif defined(INET)
621 char addrbuf[INET_ADDRSTRLEN];
622 #endif
623 bool dolog, over;
624
625 SOLISTEN_LOCK(head);
626 over = (head->sol_qlen > 3 * head->sol_qlimit / 2);
627 #ifdef REGRESSION
628 if (regression_sonewconn_earlytest && over) {
629 #else
630 if (over) {
631 #endif
632 head->sol_overcount++;
633 dolog = (sooverprio >= 0) &&
634 !!ratecheck(&head->sol_lastover, &overinterval);
635
636 /*
637 * If we're going to log, copy the overflow count and queue
638 * length from the listen socket before dropping the lock.
639 * Also, reset the overflow count.
640 */
641 if (dolog) {
642 overcount = head->sol_overcount;
643 head->sol_overcount = 0;
644 qlen = head->sol_qlen;
645 }
646 SOLISTEN_UNLOCK(head);
647
648 if (dolog) {
649 /*
650 * Try to print something descriptive about the
651 * socket for the error message.
652 */
653 sbuf_new(&descrsb, descrbuf, sizeof(descrbuf),
654 SBUF_FIXEDLEN);
655 switch (head->so_proto->pr_domain->dom_family) {
656 #if defined(INET) || defined(INET6)
657 #ifdef INET
658 case AF_INET:
659 #endif
660 #ifdef INET6
661 case AF_INET6:
662 if (head->so_proto->pr_domain->dom_family ==
663 AF_INET6 ||
664 (sotoinpcb(head)->inp_inc.inc_flags &
665 INC_ISIPV6)) {
666 ip6_sprintf(addrbuf,
667 &sotoinpcb(head)->inp_inc.inc6_laddr);
668 sbuf_printf(&descrsb, "[%s]", addrbuf);
669 } else
670 #endif
671 {
672 #ifdef INET
673 inet_ntoa_r(
674 sotoinpcb(head)->inp_inc.inc_laddr,
675 addrbuf);
676 sbuf_cat(&descrsb, addrbuf);
677 #endif
678 }
679 sbuf_printf(&descrsb, ":%hu (proto %u)",
680 ntohs(sotoinpcb(head)->inp_inc.inc_lport),
681 head->so_proto->pr_protocol);
682 break;
683 #endif /* INET || INET6 */
684 case AF_UNIX:
685 sbuf_cat(&descrsb, localprefix);
686 if (sotounpcb(head)->unp_addr != NULL)
687 len =
688 sotounpcb(head)->unp_addr->sun_len -
689 offsetof(struct sockaddr_un,
690 sun_path);
691 else
692 len = 0;
693 if (len > 0)
694 sbuf_bcat(&descrsb,
695 sotounpcb(head)->unp_addr->sun_path,
696 len);
697 else
698 sbuf_cat(&descrsb, "(unknown)");
699 break;
700 }
701
702 /*
703 * If we can't print something more specific, at least
704 * print the domain name.
705 */
706 if (sbuf_finish(&descrsb) != 0 ||
707 sbuf_len(&descrsb) <= 0) {
708 sbuf_clear(&descrsb);
709 sbuf_cat(&descrsb,
710 head->so_proto->pr_domain->dom_name ?:
711 "unknown");
712 sbuf_finish(&descrsb);
713 }
714 KASSERT(sbuf_len(&descrsb) > 0,
715 ("%s: sbuf creation failed", __func__));
716 /*
717 * Preserve the historic listen queue overflow log
718 * message, that starts with "sonewconn:". It has
719 * been known to sysadmins for years and also test
720 * sys/kern/sonewconn_overflow checks for it.
721 */
722 if (head->so_cred == 0) {
723 log(LOG_PRI(sooverprio),
724 "sonewconn: pcb %p (%s): "
725 "Listen queue overflow: %i already in "
726 "queue awaiting acceptance (%d "
727 "occurrences)\n", head->so_pcb,
728 sbuf_data(&descrsb),
729 qlen, overcount);
730 } else {
731 log(LOG_PRI(sooverprio),
732 "sonewconn: pcb %p (%s): "
733 "Listen queue overflow: "
734 "%i already in queue awaiting acceptance "
735 "(%d occurrences), euid %d, rgid %d, jail %s\n",
736 head->so_pcb, sbuf_data(&descrsb), qlen,
737 overcount, head->so_cred->cr_uid,
738 head->so_cred->cr_rgid,
739 head->so_cred->cr_prison ?
740 head->so_cred->cr_prison->pr_name :
741 "not_jailed");
742 }
743 sbuf_delete(&descrsb);
744
745 overcount = 0;
746 }
747
748 return (NULL);
749 }
750 SOLISTEN_UNLOCK(head);
751 VNET_ASSERT(head->so_vnet != NULL, ("%s: so %p vnet is NULL",
752 __func__, head));
753 so = soalloc(head->so_vnet);
754 if (so == NULL) {
755 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: "
756 "limit reached or out of memory\n",
757 __func__, head->so_pcb);
758 return (NULL);
759 }
760 so->so_listen = head;
761 so->so_type = head->so_type;
762 /*
763 * POSIX is ambiguous on what options an accept(2)ed socket should
764 * inherit from the listener. Words "create a new socket" may be
765 * interpreted as not inheriting anything. Best programming practice
766 * for application developers is to not rely on such inheritance.
767 * FreeBSD had historically inherited all so_options excluding
768 * SO_ACCEPTCONN, which virtually means all SOL_SOCKET level options,
769 * including those completely irrelevant to a new born socket. For
770 * compatibility with older versions we will inherit a list of
771 * meaningful options.
772 * The crucial bit to inherit is SO_ACCEPTFILTER. We need it present
773 * in the child socket for soisconnected() promoting socket from the
774 * incomplete queue to complete. It will be cleared before the child
775 * gets available to accept(2).
776 */
777 so->so_options = head->so_options & (SO_ACCEPTFILTER | SO_KEEPALIVE |
778 SO_DONTROUTE | SO_LINGER | SO_OOBINLINE | SO_NOSIGPIPE);
779 so->so_linger = head->so_linger;
780 so->so_state = head->so_state;
781 so->so_fibnum = head->so_fibnum;
782 so->so_proto = head->so_proto;
783 so->so_cred = crhold(head->so_cred);
784 #ifdef SOCKET_HHOOK
785 if (V_socket_hhh[HHOOK_SOCKET_NEWCONN]->hhh_nhooks > 0) {
786 if (hhook_run_socket(so, head, HHOOK_SOCKET_NEWCONN)) {
787 sodealloc(so);
788 log(LOG_DEBUG, "%s: hhook run failed\n", __func__);
789 return (NULL);
790 }
791 }
792 #endif
793 #ifdef MAC
794 mac_socket_newconn(head, so);
795 #endif
796 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
797 so_rdknl_assert_lock);
798 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
799 so_wrknl_assert_lock);
800 VNET_SO_ASSERT(head);
801 if (soreserve(so, head->sol_sbsnd_hiwat, head->sol_sbrcv_hiwat)) {
802 sodealloc(so);
803 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n",
804 __func__, head->so_pcb);
805 return (NULL);
806 }
807 so->so_rcv.sb_lowat = head->sol_sbrcv_lowat;
808 so->so_snd.sb_lowat = head->sol_sbsnd_lowat;
809 so->so_rcv.sb_timeo = head->sol_sbrcv_timeo;
810 so->so_snd.sb_timeo = head->sol_sbsnd_timeo;
811 so->so_rcv.sb_flags = head->sol_sbrcv_flags & SB_AUTOSIZE;
812 so->so_snd.sb_flags = head->sol_sbsnd_flags & SB_AUTOSIZE;
813 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) {
814 so->so_snd.sb_mtx = &so->so_snd_mtx;
815 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
816 }
817
818 return (so);
819 }
820
821 /* Connstatus may be 0 or SS_ISCONNECTED. */
822 struct socket *
823 sonewconn(struct socket *head, int connstatus)
824 {
825 struct socket *so;
826
827 if ((so = solisten_clone(head)) == NULL)
828 return (NULL);
829
830 if (so->so_proto->pr_attach(so, 0, NULL) != 0) {
831 sodealloc(so);
832 log(LOG_DEBUG, "%s: pcb %p: pr_attach() failed\n",
833 __func__, head->so_pcb);
834 return (NULL);
835 }
836
837 (void)solisten_enqueue(so, connstatus);
838
839 return (so);
840 }
841
842 /*
843 * Enqueue socket cloned by solisten_clone() to the listen queue of the
844 * listener it has been cloned from.
845 *
846 * Return 'true' if socket landed on complete queue, otherwise 'false'.
847 */
848 bool
849 solisten_enqueue(struct socket *so, int connstatus)
850 {
851 struct socket *head = so->so_listen;
852
853 MPASS(refcount_load(&so->so_count) == 0);
854 refcount_init(&so->so_count, 1);
855
856 SOLISTEN_LOCK(head);
857 if (head->sol_accept_filter != NULL)
858 connstatus = 0;
859 so->so_state |= connstatus;
860 soref(head); /* A socket on (in)complete queue refs head. */
861 if (connstatus) {
862 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
863 so->so_qstate = SQ_COMP;
864 head->sol_qlen++;
865 solisten_wakeup(head); /* unlocks */
866 return (true);
867 } else {
868 /*
869 * Keep removing sockets from the head until there's room for
870 * us to insert on the tail. In pre-locking revisions, this
871 * was a simple if(), but as we could be racing with other
872 * threads and soabort() requires dropping locks, we must
873 * loop waiting for the condition to be true.
874 */
875 while (head->sol_incqlen > head->sol_qlimit) {
876 struct socket *sp;
877
878 sp = TAILQ_FIRST(&head->sol_incomp);
879 TAILQ_REMOVE(&head->sol_incomp, sp, so_list);
880 head->sol_incqlen--;
881 SOCK_LOCK(sp);
882 sp->so_qstate = SQ_NONE;
883 sp->so_listen = NULL;
884 SOCK_UNLOCK(sp);
885 sorele_locked(head); /* does SOLISTEN_UNLOCK, head stays */
886 soabort(sp);
887 SOLISTEN_LOCK(head);
888 }
889 TAILQ_INSERT_TAIL(&head->sol_incomp, so, so_list);
890 so->so_qstate = SQ_INCOMP;
891 head->sol_incqlen++;
892 SOLISTEN_UNLOCK(head);
893 return (false);
894 }
895 }
896
897 #if defined(SCTP) || defined(SCTP_SUPPORT)
898 /*
899 * Socket part of sctp_peeloff(). Detach a new socket from an
900 * association. The new socket is returned with a reference.
901 *
902 * XXXGL: reduce copy-paste with solisten_clone().
903 */
904 struct socket *
905 sopeeloff(struct socket *head)
906 {
907 struct socket *so;
908
909 VNET_ASSERT(head->so_vnet != NULL, ("%s:%d so_vnet is NULL, head=%p",
910 __func__, __LINE__, head));
911 so = soalloc(head->so_vnet);
912 if (so == NULL) {
913 log(LOG_DEBUG, "%s: pcb %p: New socket allocation failure: "
914 "limit reached or out of memory\n",
915 __func__, head->so_pcb);
916 return (NULL);
917 }
918 so->so_type = head->so_type;
919 so->so_options = head->so_options;
920 so->so_linger = head->so_linger;
921 so->so_state = (head->so_state & SS_NBIO) | SS_ISCONNECTED;
922 so->so_fibnum = head->so_fibnum;
923 so->so_proto = head->so_proto;
924 so->so_cred = crhold(head->so_cred);
925 #ifdef MAC
926 mac_socket_newconn(head, so);
927 #endif
928 knlist_init(&so->so_rdsel.si_note, so, so_rdknl_lock, so_rdknl_unlock,
929 so_rdknl_assert_lock);
930 knlist_init(&so->so_wrsel.si_note, so, so_wrknl_lock, so_wrknl_unlock,
931 so_wrknl_assert_lock);
932 VNET_SO_ASSERT(head);
933 if (soreserve(so, head->so_snd.sb_hiwat, head->so_rcv.sb_hiwat)) {
934 sodealloc(so);
935 log(LOG_DEBUG, "%s: pcb %p: soreserve() failed\n",
936 __func__, head->so_pcb);
937 return (NULL);
938 }
939 if ((*so->so_proto->pr_attach)(so, 0, NULL)) {
940 sodealloc(so);
941 log(LOG_DEBUG, "%s: pcb %p: pru_attach() failed\n",
942 __func__, head->so_pcb);
943 return (NULL);
944 }
945 so->so_rcv.sb_lowat = head->so_rcv.sb_lowat;
946 so->so_snd.sb_lowat = head->so_snd.sb_lowat;
947 so->so_rcv.sb_timeo = head->so_rcv.sb_timeo;
948 so->so_snd.sb_timeo = head->so_snd.sb_timeo;
949 so->so_rcv.sb_flags |= head->so_rcv.sb_flags & SB_AUTOSIZE;
950 so->so_snd.sb_flags |= head->so_snd.sb_flags & SB_AUTOSIZE;
951 if ((so->so_proto->pr_flags & PR_SOCKBUF) == 0) {
952 so->so_snd.sb_mtx = &so->so_snd_mtx;
953 so->so_rcv.sb_mtx = &so->so_rcv_mtx;
954 }
955
956 soref(so);
957
958 return (so);
959 }
960 #endif /* SCTP */
961
962 int
963 sobind(struct socket *so, struct sockaddr *nam, struct thread *td)
964 {
965 int error;
966
967 CURVNET_SET(so->so_vnet);
968 error = so->so_proto->pr_bind(so, nam, td);
969 CURVNET_RESTORE();
970 return (error);
971 }
972
973 int
974 sobindat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td)
975 {
976 int error;
977
978 CURVNET_SET(so->so_vnet);
979 error = so->so_proto->pr_bindat(fd, so, nam, td);
980 CURVNET_RESTORE();
981 return (error);
982 }
983
984 /*
985 * solisten() transitions a socket from a non-listening state to a listening
986 * state, but can also be used to update the listen queue depth on an
987 * existing listen socket. The protocol will call back into the sockets
988 * layer using solisten_proto_check() and solisten_proto() to check and set
989 * socket-layer listen state. Call backs are used so that the protocol can
990 * acquire both protocol and socket layer locks in whatever order is required
991 * by the protocol.
992 *
993 * Protocol implementors are advised to hold the socket lock across the
994 * socket-layer test and set to avoid races at the socket layer.
995 */
996 int
997 solisten(struct socket *so, int backlog, struct thread *td)
998 {
999 int error;
1000
1001 CURVNET_SET(so->so_vnet);
1002 error = so->so_proto->pr_listen(so, backlog, td);
1003 CURVNET_RESTORE();
1004 return (error);
1005 }
1006
1007 /*
1008 * Prepare for a call to solisten_proto(). Acquire all socket buffer locks in
1009 * order to interlock with socket I/O.
1010 */
1011 int
1012 solisten_proto_check(struct socket *so)
1013 {
1014 SOCK_LOCK_ASSERT(so);
1015
1016 if ((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
1017 SS_ISDISCONNECTING)) != 0)
1018 return (EINVAL);
1019
1020 /*
1021 * Sleeping is not permitted here, so simply fail if userspace is
1022 * attempting to transmit or receive on the socket. This kind of
1023 * transient failure is not ideal, but it should occur only if userspace
1024 * is misusing the socket interfaces.
1025 */
1026 if (!sx_try_xlock(&so->so_snd_sx))
1027 return (EAGAIN);
1028 if (!sx_try_xlock(&so->so_rcv_sx)) {
1029 sx_xunlock(&so->so_snd_sx);
1030 return (EAGAIN);
1031 }
1032 mtx_lock(&so->so_snd_mtx);
1033 mtx_lock(&so->so_rcv_mtx);
1034
1035 /* Interlock with soo_aio_queue() and KTLS. */
1036 if (!SOLISTENING(so)) {
1037 bool ktls;
1038
1039 #ifdef KERN_TLS
1040 ktls = so->so_snd.sb_tls_info != NULL ||
1041 so->so_rcv.sb_tls_info != NULL;
1042 #else
1043 ktls = false;
1044 #endif
1045 if (ktls ||
1046 (so->so_snd.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0 ||
1047 (so->so_rcv.sb_flags & (SB_AIO | SB_AIO_RUNNING)) != 0) {
1048 solisten_proto_abort(so);
1049 return (EINVAL);
1050 }
1051 }
1052
1053 return (0);
1054 }
1055
1056 /*
1057 * Undo the setup done by solisten_proto_check().
1058 */
1059 void
1060 solisten_proto_abort(struct socket *so)
1061 {
1062 mtx_unlock(&so->so_snd_mtx);
1063 mtx_unlock(&so->so_rcv_mtx);
1064 sx_xunlock(&so->so_snd_sx);
1065 sx_xunlock(&so->so_rcv_sx);
1066 }
1067
1068 void
1069 solisten_proto(struct socket *so, int backlog)
1070 {
1071 int sbrcv_lowat, sbsnd_lowat;
1072 u_int sbrcv_hiwat, sbsnd_hiwat;
1073 short sbrcv_flags, sbsnd_flags;
1074 sbintime_t sbrcv_timeo, sbsnd_timeo;
1075
1076 SOCK_LOCK_ASSERT(so);
1077 KASSERT((so->so_state & (SS_ISCONNECTED | SS_ISCONNECTING |
1078 SS_ISDISCONNECTING)) == 0,
1079 ("%s: bad socket state %p", __func__, so));
1080
1081 if (SOLISTENING(so))
1082 goto listening;
1083
1084 /*
1085 * Change this socket to listening state.
1086 */
1087 sbrcv_lowat = so->so_rcv.sb_lowat;
1088 sbsnd_lowat = so->so_snd.sb_lowat;
1089 sbrcv_hiwat = so->so_rcv.sb_hiwat;
1090 sbsnd_hiwat = so->so_snd.sb_hiwat;
1091 sbrcv_flags = so->so_rcv.sb_flags;
1092 sbsnd_flags = so->so_snd.sb_flags;
1093 sbrcv_timeo = so->so_rcv.sb_timeo;
1094 sbsnd_timeo = so->so_snd.sb_timeo;
1095
1096 if (!(so->so_proto->pr_flags & PR_SOCKBUF)) {
1097 sbdestroy(so, SO_SND);
1098 sbdestroy(so, SO_RCV);
1099 }
1100
1101 #ifdef INVARIANTS
1102 bzero(&so->so_rcv,
1103 sizeof(struct socket) - offsetof(struct socket, so_rcv));
1104 #endif
1105
1106 so->sol_sbrcv_lowat = sbrcv_lowat;
1107 so->sol_sbsnd_lowat = sbsnd_lowat;
1108 so->sol_sbrcv_hiwat = sbrcv_hiwat;
1109 so->sol_sbsnd_hiwat = sbsnd_hiwat;
1110 so->sol_sbrcv_flags = sbrcv_flags;
1111 so->sol_sbsnd_flags = sbsnd_flags;
1112 so->sol_sbrcv_timeo = sbrcv_timeo;
1113 so->sol_sbsnd_timeo = sbsnd_timeo;
1114
1115 so->sol_qlen = so->sol_incqlen = 0;
1116 TAILQ_INIT(&so->sol_incomp);
1117 TAILQ_INIT(&so->sol_comp);
1118
1119 so->sol_accept_filter = NULL;
1120 so->sol_accept_filter_arg = NULL;
1121 so->sol_accept_filter_str = NULL;
1122
1123 so->sol_upcall = NULL;
1124 so->sol_upcallarg = NULL;
1125
1126 so->so_options |= SO_ACCEPTCONN;
1127
1128 listening:
1129 if (backlog < 0 || backlog > somaxconn)
1130 backlog = somaxconn;
1131 so->sol_qlimit = backlog;
1132
1133 mtx_unlock(&so->so_snd_mtx);
1134 mtx_unlock(&so->so_rcv_mtx);
1135 sx_xunlock(&so->so_snd_sx);
1136 sx_xunlock(&so->so_rcv_sx);
1137 }
1138
1139 /*
1140 * Wakeup listeners/subsystems once we have a complete connection.
1141 * Enters with lock, returns unlocked.
1142 */
1143 void
1144 solisten_wakeup(struct socket *sol)
1145 {
1146
1147 if (sol->sol_upcall != NULL)
1148 (void )sol->sol_upcall(sol, sol->sol_upcallarg, M_NOWAIT);
1149 else {
1150 selwakeuppri(&sol->so_rdsel, PSOCK);
1151 KNOTE_LOCKED(&sol->so_rdsel.si_note, 0);
1152 }
1153 SOLISTEN_UNLOCK(sol);
1154 wakeup_one(&sol->sol_comp);
1155 if ((sol->so_state & SS_ASYNC) && sol->so_sigio != NULL)
1156 pgsigio(&sol->so_sigio, SIGIO, 0);
1157 }
1158
1159 /*
1160 * Return single connection off a listening socket queue. Main consumer of
1161 * the function is kern_accept4(). Some modules, that do their own accept
1162 * management also use the function. The socket reference held by the
1163 * listen queue is handed to the caller.
1164 *
1165 * Listening socket must be locked on entry and is returned unlocked on
1166 * return.
1167 * The flags argument is set of accept4(2) flags and ACCEPT4_INHERIT.
1168 */
1169 int
1170 solisten_dequeue(struct socket *head, struct socket **ret, int flags)
1171 {
1172 struct socket *so;
1173 int error;
1174
1175 SOLISTEN_LOCK_ASSERT(head);
1176
1177 while (!(head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp) &&
1178 head->so_error == 0) {
1179 error = msleep(&head->sol_comp, SOCK_MTX(head), PSOCK | PCATCH,
1180 "accept", 0);
1181 if (error != 0) {
1182 SOLISTEN_UNLOCK(head);
1183 return (error);
1184 }
1185 }
1186 if (head->so_error) {
1187 error = head->so_error;
1188 head->so_error = 0;
1189 } else if ((head->so_state & SS_NBIO) && TAILQ_EMPTY(&head->sol_comp))
1190 error = EWOULDBLOCK;
1191 else
1192 error = 0;
1193 if (error) {
1194 SOLISTEN_UNLOCK(head);
1195 return (error);
1196 }
1197 so = TAILQ_FIRST(&head->sol_comp);
1198 SOCK_LOCK(so);
1199 KASSERT(so->so_qstate == SQ_COMP,
1200 ("%s: so %p not SQ_COMP", __func__, so));
1201 head->sol_qlen--;
1202 so->so_qstate = SQ_NONE;
1203 so->so_listen = NULL;
1204 TAILQ_REMOVE(&head->sol_comp, so, so_list);
1205 if (flags & ACCEPT4_INHERIT)
1206 so->so_state |= (head->so_state & SS_NBIO);
1207 else
1208 so->so_state |= (flags & SOCK_NONBLOCK) ? SS_NBIO : 0;
1209 SOCK_UNLOCK(so);
1210 sorele_locked(head);
1211
1212 *ret = so;
1213 return (0);
1214 }
1215
1216 /*
1217 * Free socket upon release of the very last reference.
1218 */
1219 static void
1220 sofree(struct socket *so)
1221 {
1222 struct protosw *pr = so->so_proto;
1223
1224 SOCK_LOCK_ASSERT(so);
1225 KASSERT(refcount_load(&so->so_count) == 0,
1226 ("%s: so %p has references", __func__, so));
1227 KASSERT(SOLISTENING(so) || so->so_qstate == SQ_NONE,
1228 ("%s: so %p is on listen queue", __func__, so));
1229
1230 SOCK_UNLOCK(so);
1231
1232 if (so->so_dtor != NULL)
1233 so->so_dtor(so);
1234
1235 VNET_SO_ASSERT(so);
1236 if (pr->pr_detach != NULL)
1237 pr->pr_detach(so);
1238
1239 /*
1240 * From this point on, we assume that no other references to this
1241 * socket exist anywhere else in the stack. Therefore, no locks need
1242 * to be acquired or held.
1243 */
1244 if (!(pr->pr_flags & PR_SOCKBUF) && !SOLISTENING(so)) {
1245 sbdestroy(so, SO_SND);
1246 sbdestroy(so, SO_RCV);
1247 }
1248 seldrain(&so->so_rdsel);
1249 seldrain(&so->so_wrsel);
1250 knlist_destroy(&so->so_rdsel.si_note);
1251 knlist_destroy(&so->so_wrsel.si_note);
1252 sodealloc(so);
1253 }
1254
1255 /*
1256 * Release a reference on a socket while holding the socket lock.
1257 * Unlocks the socket lock before returning.
1258 */
1259 void
1260 sorele_locked(struct socket *so)
1261 {
1262 SOCK_LOCK_ASSERT(so);
1263 if (refcount_release(&so->so_count))
1264 sofree(so);
1265 else
1266 SOCK_UNLOCK(so);
1267 }
1268
1269 /*
1270 * Close a socket on last file table reference removal. Initiate disconnect
1271 * if connected. Free socket when disconnect complete.
1272 *
1273 * This function will sorele() the socket. Note that soclose() may be called
1274 * prior to the ref count reaching zero. The actual socket structure will
1275 * not be freed until the ref count reaches zero.
1276 */
1277 int
1278 soclose(struct socket *so)
1279 {
1280 struct accept_queue lqueue;
1281 int error = 0;
1282 bool listening, last __diagused;
1283
1284 CURVNET_SET(so->so_vnet);
1285 funsetown(&so->so_sigio);
1286 if (so->so_state & SS_ISCONNECTED) {
1287 if ((so->so_state & SS_ISDISCONNECTING) == 0) {
1288 error = sodisconnect(so);
1289 if (error) {
1290 if (error == ENOTCONN)
1291 error = 0;
1292 goto drop;
1293 }
1294 }
1295
1296 if ((so->so_options & SO_LINGER) != 0 && so->so_linger != 0) {
1297 if ((so->so_state & SS_ISDISCONNECTING) &&
1298 (so->so_state & SS_NBIO))
1299 goto drop;
1300 while (so->so_state & SS_ISCONNECTED) {
1301 error = tsleep(&so->so_timeo,
1302 PSOCK | PCATCH, "soclos",
1303 so->so_linger * hz);
1304 if (error)
1305 break;
1306 }
1307 }
1308 }
1309
1310 drop:
1311 if (so->so_proto->pr_close != NULL)
1312 so->so_proto->pr_close(so);
1313
1314 SOCK_LOCK(so);
1315 if ((listening = SOLISTENING(so))) {
1316 struct socket *sp;
1317
1318 TAILQ_INIT(&lqueue);
1319 TAILQ_SWAP(&lqueue, &so->sol_incomp, socket, so_list);
1320 TAILQ_CONCAT(&lqueue, &so->sol_comp, so_list);
1321
1322 so->sol_qlen = so->sol_incqlen = 0;
1323
1324 TAILQ_FOREACH(sp, &lqueue, so_list) {
1325 SOCK_LOCK(sp);
1326 sp->so_qstate = SQ_NONE;
1327 sp->so_listen = NULL;
1328 SOCK_UNLOCK(sp);
1329 last = refcount_release(&so->so_count);
1330 KASSERT(!last, ("%s: released last reference for %p",
1331 __func__, so));
1332 }
1333 }
1334 sorele_locked(so);
1335 if (listening) {
1336 struct socket *sp, *tsp;
1337
1338 TAILQ_FOREACH_SAFE(sp, &lqueue, so_list, tsp)
1339 soabort(sp);
1340 }
1341 CURVNET_RESTORE();
1342 return (error);
1343 }
1344
1345 /*
1346 * soabort() is used to abruptly tear down a connection, such as when a
1347 * resource limit is reached (listen queue depth exceeded), or if a listen
1348 * socket is closed while there are sockets waiting to be accepted.
1349 *
1350 * This interface is tricky, because it is called on an unreferenced socket,
1351 * and must be called only by a thread that has actually removed the socket
1352 * from the listen queue it was on. Likely this thread holds the last
1353 * reference on the socket and soabort() will proceed with sofree(). But
1354 * it might be not the last, as the sockets on the listen queues are seen
1355 * from the protocol side.
1356 *
1357 * This interface will call into the protocol code, so must not be called
1358 * with any socket locks held. Protocols do call it while holding their own
1359 * recursible protocol mutexes, but this is something that should be subject
1360 * to review in the future.
1361 *
1362 * Usually socket should have a single reference left, but this is not a
1363 * requirement. In the past, when we have had named references for file
1364 * descriptor and protocol, we asserted that none of them are being held.
1365 */
1366 void
1367 soabort(struct socket *so)
1368 {
1369
1370 VNET_SO_ASSERT(so);
1371
1372 if (so->so_proto->pr_abort != NULL)
1373 so->so_proto->pr_abort(so);
1374 SOCK_LOCK(so);
1375 sorele_locked(so);
1376 }
1377
1378 int
1379 soaccept(struct socket *so, struct sockaddr *sa)
1380 {
1381 #ifdef INVARIANTS
1382 u_char len = sa->sa_len;
1383 #endif
1384 int error;
1385
1386 CURVNET_SET(so->so_vnet);
1387 error = so->so_proto->pr_accept(so, sa);
1388 KASSERT(sa->sa_len <= len,
1389 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto));
1390 CURVNET_RESTORE();
1391 return (error);
1392 }
1393
1394 int
1395 sopeeraddr(struct socket *so, struct sockaddr *sa)
1396 {
1397 #ifdef INVARIANTS
1398 u_char len = sa->sa_len;
1399 #endif
1400 int error;
1401
1402 CURVNET_SET(so->so_vnet);
1403 error = so->so_proto->pr_peeraddr(so, sa);
1404 KASSERT(sa->sa_len <= len,
1405 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto));
1406 CURVNET_RESTORE();
1407
1408 return (error);
1409 }
1410
1411 int
1412 sosockaddr(struct socket *so, struct sockaddr *sa)
1413 {
1414 #ifdef INVARIANTS
1415 u_char len = sa->sa_len;
1416 #endif
1417 int error;
1418
1419 CURVNET_SET(so->so_vnet);
1420 error = so->so_proto->pr_sockaddr(so, sa);
1421 KASSERT(sa->sa_len <= len,
1422 ("%s: protocol %p sockaddr overflow", __func__, so->so_proto));
1423 CURVNET_RESTORE();
1424
1425 return (error);
1426 }
1427
1428 int
1429 soconnect(struct socket *so, struct sockaddr *nam, struct thread *td)
1430 {
1431
1432 return (soconnectat(AT_FDCWD, so, nam, td));
1433 }
1434
1435 int
1436 soconnectat(int fd, struct socket *so, struct sockaddr *nam, struct thread *td)
1437 {
1438 int error;
1439
1440 CURVNET_SET(so->so_vnet);
1441
1442 /*
1443 * If protocol is connection-based, can only connect once.
1444 * Otherwise, if connected, try to disconnect first. This allows
1445 * user to disconnect by connecting to, e.g., a null address.
1446 *
1447 * Note, this check is racy and may need to be re-evaluated at the
1448 * protocol layer.
1449 */
1450 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
1451 ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
1452 (error = sodisconnect(so)))) {
1453 error = EISCONN;
1454 } else {
1455 /*
1456 * Prevent accumulated error from previous connection from
1457 * biting us.
1458 */
1459 so->so_error = 0;
1460 if (fd == AT_FDCWD) {
1461 error = so->so_proto->pr_connect(so, nam, td);
1462 } else {
1463 error = so->so_proto->pr_connectat(fd, so, nam, td);
1464 }
1465 }
1466 CURVNET_RESTORE();
1467
1468 return (error);
1469 }
1470
1471 int
1472 soconnect2(struct socket *so1, struct socket *so2)
1473 {
1474 int error;
1475
1476 CURVNET_SET(so1->so_vnet);
1477 error = so1->so_proto->pr_connect2(so1, so2);
1478 CURVNET_RESTORE();
1479 return (error);
1480 }
1481
1482 int
1483 sodisconnect(struct socket *so)
1484 {
1485 int error;
1486
1487 if ((so->so_state & SS_ISCONNECTED) == 0)
1488 return (ENOTCONN);
1489 if (so->so_state & SS_ISDISCONNECTING)
1490 return (EALREADY);
1491 VNET_SO_ASSERT(so);
1492 error = so->so_proto->pr_disconnect(so);
1493 return (error);
1494 }
1495
1496 int
1497 sosend_dgram(struct socket *so, struct sockaddr *addr, struct uio *uio,
1498 struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
1499 {
1500 long space;
1501 ssize_t resid;
1502 int clen = 0, error, dontroute;
1503
1504 KASSERT(so->so_type == SOCK_DGRAM, ("sosend_dgram: !SOCK_DGRAM"));
1505 KASSERT(so->so_proto->pr_flags & PR_ATOMIC,
1506 ("sosend_dgram: !PR_ATOMIC"));
1507
1508 if (uio != NULL)
1509 resid = uio->uio_resid;
1510 else
1511 resid = top->m_pkthdr.len;
1512 /*
1513 * In theory resid should be unsigned. However, space must be
1514 * signed, as it might be less than 0 if we over-committed, and we
1515 * must use a signed comparison of space and resid. On the other
1516 * hand, a negative resid causes us to loop sending 0-length
1517 * segments to the protocol.
1518 */
1519 if (resid < 0) {
1520 error = EINVAL;
1521 goto out;
1522 }
1523
1524 dontroute =
1525 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0;
1526 if (td != NULL)
1527 td->td_ru.ru_msgsnd++;
1528 if (control != NULL)
1529 clen = control->m_len;
1530
1531 SOCKBUF_LOCK(&so->so_snd);
1532 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
1533 SOCKBUF_UNLOCK(&so->so_snd);
1534 error = EPIPE;
1535 goto out;
1536 }
1537 if (so->so_error) {
1538 error = so->so_error;
1539 so->so_error = 0;
1540 SOCKBUF_UNLOCK(&so->so_snd);
1541 goto out;
1542 }
1543 if ((so->so_state & SS_ISCONNECTED) == 0) {
1544 /*
1545 * `sendto' and `sendmsg' is allowed on a connection-based
1546 * socket if it supports implied connect. Return ENOTCONN if
1547 * not connected and no address is supplied.
1548 */
1549 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
1550 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
1551 if (!(resid == 0 && clen != 0)) {
1552 SOCKBUF_UNLOCK(&so->so_snd);
1553 error = ENOTCONN;
1554 goto out;
1555 }
1556 } else if (addr == NULL) {
1557 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
1558 error = ENOTCONN;
1559 else
1560 error = EDESTADDRREQ;
1561 SOCKBUF_UNLOCK(&so->so_snd);
1562 goto out;
1563 }
1564 }
1565
1566 /*
1567 * Do we need MSG_OOB support in SOCK_DGRAM? Signs here may be a
1568 * problem and need fixing.
1569 */
1570 space = sbspace(&so->so_snd);
1571 if (flags & MSG_OOB)
1572 space += 1024;
1573 space -= clen;
1574 SOCKBUF_UNLOCK(&so->so_snd);
1575 if (resid > space) {
1576 error = EMSGSIZE;
1577 goto out;
1578 }
1579 if (uio == NULL) {
1580 resid = 0;
1581 if (flags & MSG_EOR)
1582 top->m_flags |= M_EOR;
1583 } else {
1584 /*
1585 * Copy the data from userland into a mbuf chain.
1586 * If no data is to be copied in, a single empty mbuf
1587 * is returned.
1588 */
1589 top = m_uiotombuf(uio, M_WAITOK, space, max_hdr,
1590 (M_PKTHDR | ((flags & MSG_EOR) ? M_EOR : 0)));
1591 if (top == NULL) {
1592 error = EFAULT; /* only possible error */
1593 goto out;
1594 }
1595 space -= resid - uio->uio_resid;
1596 resid = uio->uio_resid;
1597 }
1598 KASSERT(resid == 0, ("sosend_dgram: resid != 0"));
1599 /*
1600 * XXXRW: Frobbing SO_DONTROUTE here is even worse without sblock
1601 * than with.
1602 */
1603 if (dontroute) {
1604 SOCK_LOCK(so);
1605 so->so_options |= SO_DONTROUTE;
1606 SOCK_UNLOCK(so);
1607 }
1608 /*
1609 * XXX all the SBS_CANTSENDMORE checks previously done could be out
1610 * of date. We could have received a reset packet in an interrupt or
1611 * maybe we slept while doing page faults in uiomove() etc. We could
1612 * probably recheck again inside the locking protection here, but
1613 * there are probably other places that this also happens. We must
1614 * rethink this.
1615 */
1616 VNET_SO_ASSERT(so);
1617 error = so->so_proto->pr_send(so, (flags & MSG_OOB) ? PRUS_OOB :
1618 /*
1619 * If the user set MSG_EOF, the protocol understands this flag and
1620 * nothing left to send then use PRU_SEND_EOF instead of PRU_SEND.
1621 */
1622 ((flags & MSG_EOF) &&
1623 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
1624 (resid <= 0)) ?
1625 PRUS_EOF :
1626 /* If there is more to send set PRUS_MORETOCOME */
1627 (flags & MSG_MORETOCOME) ||
1628 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0,
1629 top, addr, control, td);
1630 if (dontroute) {
1631 SOCK_LOCK(so);
1632 so->so_options &= ~SO_DONTROUTE;
1633 SOCK_UNLOCK(so);
1634 }
1635 clen = 0;
1636 control = NULL;
1637 top = NULL;
1638 out:
1639 if (top != NULL)
1640 m_freem(top);
1641 if (control != NULL)
1642 m_freem(control);
1643 return (error);
1644 }
1645
1646 /*
1647 * Send on a socket. If send must go all at once and message is larger than
1648 * send buffering, then hard error. Lock against other senders. If must go
1649 * all at once and not enough room now, then inform user that this would
1650 * block and do nothing. Otherwise, if nonblocking, send as much as
1651 * possible. The data to be sent is described by "uio" if nonzero, otherwise
1652 * by the mbuf chain "top" (which must be null if uio is not). Data provided
1653 * in mbuf chain must be small enough to send all at once.
1654 *
1655 * Returns nonzero on error, timeout or signal; callers must check for short
1656 * counts if EINTR/ERESTART are returned. Data and control buffers are freed
1657 * on return.
1658 */
1659 int
1660 sosend_generic(struct socket *so, struct sockaddr *addr, struct uio *uio,
1661 struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
1662 {
1663 long space;
1664 ssize_t resid;
1665 int clen = 0, error, dontroute;
1666 int atomic = sosendallatonce(so) || top;
1667 int pr_send_flag;
1668 #ifdef KERN_TLS
1669 struct ktls_session *tls;
1670 int tls_enq_cnt, tls_send_flag;
1671 uint8_t tls_rtype;
1672
1673 tls = NULL;
1674 tls_rtype = TLS_RLTYPE_APP;
1675 #endif
1676 if (uio != NULL)
1677 resid = uio->uio_resid;
1678 else if ((top->m_flags & M_PKTHDR) != 0)
1679 resid = top->m_pkthdr.len;
1680 else
1681 resid = m_length(top, NULL);
1682 /*
1683 * In theory resid should be unsigned. However, space must be
1684 * signed, as it might be less than 0 if we over-committed, and we
1685 * must use a signed comparison of space and resid. On the other
1686 * hand, a negative resid causes us to loop sending 0-length
1687 * segments to the protocol.
1688 *
1689 * Also check to make sure that MSG_EOR isn't used on SOCK_STREAM
1690 * type sockets since that's an error.
1691 */
1692 if (resid < 0 || (so->so_type == SOCK_STREAM && (flags & MSG_EOR))) {
1693 error = EINVAL;
1694 goto out;
1695 }
1696
1697 dontroute =
1698 (flags & MSG_DONTROUTE) && (so->so_options & SO_DONTROUTE) == 0 &&
1699 (so->so_proto->pr_flags & PR_ATOMIC);
1700 if (td != NULL)
1701 td->td_ru.ru_msgsnd++;
1702 if (control != NULL)
1703 clen = control->m_len;
1704
1705 error = SOCK_IO_SEND_LOCK(so, SBLOCKWAIT(flags));
1706 if (error)
1707 goto out;
1708
1709 #ifdef KERN_TLS
1710 tls_send_flag = 0;
1711 tls = ktls_hold(so->so_snd.sb_tls_info);
1712 if (tls != NULL) {
1713 if (tls->mode == TCP_TLS_MODE_SW)
1714 tls_send_flag = PRUS_NOTREADY;
1715
1716 if (control != NULL) {
1717 struct cmsghdr *cm = mtod(control, struct cmsghdr *);
1718
1719 if (clen >= sizeof(*cm) &&
1720 cm->cmsg_type == TLS_SET_RECORD_TYPE) {
1721 tls_rtype = *((uint8_t *)CMSG_DATA(cm));
1722 clen = 0;
1723 m_freem(control);
1724 control = NULL;
1725 atomic = 1;
1726 }
1727 }
1728
1729 if (resid == 0 && !ktls_permit_empty_frames(tls)) {
1730 error = EINVAL;
1731 goto release;
1732 }
1733 }
1734 #endif
1735
1736 restart:
1737 do {
1738 SOCKBUF_LOCK(&so->so_snd);
1739 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
1740 SOCKBUF_UNLOCK(&so->so_snd);
1741 error = EPIPE;
1742 goto release;
1743 }
1744 if (so->so_error) {
1745 error = so->so_error;
1746 so->so_error = 0;
1747 SOCKBUF_UNLOCK(&so->so_snd);
1748 goto release;
1749 }
1750 if ((so->so_state & SS_ISCONNECTED) == 0) {
1751 /*
1752 * `sendto' and `sendmsg' is allowed on a connection-
1753 * based socket if it supports implied connect.
1754 * Return ENOTCONN if not connected and no address is
1755 * supplied.
1756 */
1757 if ((so->so_proto->pr_flags & PR_CONNREQUIRED) &&
1758 (so->so_proto->pr_flags & PR_IMPLOPCL) == 0) {
1759 if (!(resid == 0 && clen != 0)) {
1760 SOCKBUF_UNLOCK(&so->so_snd);
1761 error = ENOTCONN;
1762 goto release;
1763 }
1764 } else if (addr == NULL) {
1765 SOCKBUF_UNLOCK(&so->so_snd);
1766 if (so->so_proto->pr_flags & PR_CONNREQUIRED)
1767 error = ENOTCONN;
1768 else
1769 error = EDESTADDRREQ;
1770 goto release;
1771 }
1772 }
1773 space = sbspace(&so->so_snd);
1774 if (flags & MSG_OOB)
1775 space += 1024;
1776 if ((atomic && resid > so->so_snd.sb_hiwat) ||
1777 clen > so->so_snd.sb_hiwat) {
1778 SOCKBUF_UNLOCK(&so->so_snd);
1779 error = EMSGSIZE;
1780 goto release;
1781 }
1782 if (space < resid + clen &&
1783 (atomic || space < so->so_snd.sb_lowat || space < clen)) {
1784 if ((so->so_state & SS_NBIO) ||
1785 (flags & (MSG_NBIO | MSG_DONTWAIT)) != 0) {
1786 SOCKBUF_UNLOCK(&so->so_snd);
1787 error = EWOULDBLOCK;
1788 goto release;
1789 }
1790 error = sbwait(so, SO_SND);
1791 SOCKBUF_UNLOCK(&so->so_snd);
1792 if (error)
1793 goto release;
1794 goto restart;
1795 }
1796 SOCKBUF_UNLOCK(&so->so_snd);
1797 space -= clen;
1798 do {
1799 if (uio == NULL) {
1800 resid = 0;
1801 if (flags & MSG_EOR)
1802 top->m_flags |= M_EOR;
1803 #ifdef KERN_TLS
1804 if (tls != NULL) {
1805 ktls_frame(top, tls, &tls_enq_cnt,
1806 tls_rtype);
1807 tls_rtype = TLS_RLTYPE_APP;
1808 }
1809 #endif
1810 } else {
1811 /*
1812 * Copy the data from userland into a mbuf
1813 * chain. If resid is 0, which can happen
1814 * only if we have control to send, then
1815 * a single empty mbuf is returned. This
1816 * is a workaround to prevent protocol send
1817 * methods to panic.
1818 */
1819 #ifdef KERN_TLS
1820 if (tls != NULL) {
1821 top = m_uiotombuf(uio, M_WAITOK, space,
1822 tls->params.max_frame_len,
1823 M_EXTPG |
1824 ((flags & MSG_EOR) ? M_EOR : 0));
1825 if (top != NULL) {
1826 ktls_frame(top, tls,
1827 &tls_enq_cnt, tls_rtype);
1828 }
1829 tls_rtype = TLS_RLTYPE_APP;
1830 } else
1831 #endif
1832 top = m_uiotombuf(uio, M_WAITOK, space,
1833 (atomic ? max_hdr : 0),
1834 (atomic ? M_PKTHDR : 0) |
1835 ((flags & MSG_EOR) ? M_EOR : 0));
1836 if (top == NULL) {
1837 error = EFAULT; /* only possible error */
1838 goto release;
1839 }
1840 space -= resid - uio->uio_resid;
1841 resid = uio->uio_resid;
1842 }
1843 if (dontroute) {
1844 SOCK_LOCK(so);
1845 so->so_options |= SO_DONTROUTE;
1846 SOCK_UNLOCK(so);
1847 }
1848 /*
1849 * XXX all the SBS_CANTSENDMORE checks previously
1850 * done could be out of date. We could have received
1851 * a reset packet in an interrupt or maybe we slept
1852 * while doing page faults in uiomove() etc. We
1853 * could probably recheck again inside the locking
1854 * protection here, but there are probably other
1855 * places that this also happens. We must rethink
1856 * this.
1857 */
1858 VNET_SO_ASSERT(so);
1859
1860 pr_send_flag = (flags & MSG_OOB) ? PRUS_OOB :
1861 /*
1862 * If the user set MSG_EOF, the protocol understands
1863 * this flag and nothing left to send then use
1864 * PRU_SEND_EOF instead of PRU_SEND.
1865 */
1866 ((flags & MSG_EOF) &&
1867 (so->so_proto->pr_flags & PR_IMPLOPCL) &&
1868 (resid <= 0)) ?
1869 PRUS_EOF :
1870 /* If there is more to send set PRUS_MORETOCOME. */
1871 (flags & MSG_MORETOCOME) ||
1872 (resid > 0 && space > 0) ? PRUS_MORETOCOME : 0;
1873
1874 #ifdef KERN_TLS
1875 pr_send_flag |= tls_send_flag;
1876 #endif
1877
1878 error = so->so_proto->pr_send(so, pr_send_flag, top,
1879 addr, control, td);
1880
1881 if (dontroute) {
1882 SOCK_LOCK(so);
1883 so->so_options &= ~SO_DONTROUTE;
1884 SOCK_UNLOCK(so);
1885 }
1886
1887 #ifdef KERN_TLS
1888 if (tls != NULL && tls->mode == TCP_TLS_MODE_SW) {
1889 if (error != 0) {
1890 m_freem(top);
1891 top = NULL;
1892 } else {
1893 soref(so);
1894 ktls_enqueue(top, so, tls_enq_cnt);
1895 }
1896 }
1897 #endif
1898 clen = 0;
1899 control = NULL;
1900 top = NULL;
1901 if (error)
1902 goto release;
1903 } while (resid && space > 0);
1904 } while (resid);
1905
1906 release:
1907 SOCK_IO_SEND_UNLOCK(so);
1908 out:
1909 #ifdef KERN_TLS
1910 if (tls != NULL)
1911 ktls_free(tls);
1912 #endif
1913 if (top != NULL)
1914 m_freem(top);
1915 if (control != NULL)
1916 m_freem(control);
1917 return (error);
1918 }
1919
1920 /*
1921 * Send to a socket from a kernel thread.
1922 *
1923 * XXXGL: in almost all cases uio is NULL and the mbuf is supplied.
1924 * Exception is nfs/bootp_subr.c. It is arguable that the VNET context needs
1925 * to be set at all. This function should just boil down to a static inline
1926 * calling the protocol method.
1927 */
1928 int
1929 sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
1930 struct mbuf *top, struct mbuf *control, int flags, struct thread *td)
1931 {
1932 int error;
1933
1934 CURVNET_SET(so->so_vnet);
1935 error = so->so_proto->pr_sosend(so, addr, uio,
1936 top, control, flags, td);
1937 CURVNET_RESTORE();
1938 return (error);
1939 }
1940
1941 /*
1942 * send(2), write(2) or aio_write(2) on a socket.
1943 */
1944 int
1945 sousrsend(struct socket *so, struct sockaddr *addr, struct uio *uio,
1946 struct mbuf *control, int flags, struct proc *userproc)
1947 {
1948 struct thread *td;
1949 ssize_t len;
1950 int error;
1951
1952 td = uio->uio_td;
1953 len = uio->uio_resid;
1954 CURVNET_SET(so->so_vnet);
1955 error = so->so_proto->pr_sosend(so, addr, uio, NULL, control, flags,
1956 td);
1957 CURVNET_RESTORE();
1958 if (error != 0) {
1959 /*
1960 * Clear transient errors for stream protocols if they made
1961 * some progress. Make exclusion for aio(4) that would
1962 * schedule a new write in case of EWOULDBLOCK and clear
1963 * error itself. See soaio_process_job().
1964 */
1965 if (uio->uio_resid != len &&
1966 (so->so_proto->pr_flags & PR_ATOMIC) == 0 &&
1967 userproc == NULL &&
1968 (error == ERESTART || error == EINTR ||
1969 error == EWOULDBLOCK))
1970 error = 0;
1971 /* Generation of SIGPIPE can be controlled per socket. */
1972 if (error == EPIPE && (so->so_options & SO_NOSIGPIPE) == 0 &&
1973 (flags & MSG_NOSIGNAL) == 0) {
1974 if (userproc != NULL) {
1975 /* aio(4) job */
1976 PROC_LOCK(userproc);
1977 kern_psignal(userproc, SIGPIPE);
1978 PROC_UNLOCK(userproc);
1979 } else {
1980 PROC_LOCK(td->td_proc);
1981 tdsignal(td, SIGPIPE);
1982 PROC_UNLOCK(td->td_proc);
1983 }
1984 }
1985 }
1986 return (error);
1987 }
1988
1989 /*
1990 * The part of soreceive() that implements reading non-inline out-of-band
1991 * data from a socket. For more complete comments, see soreceive(), from
1992 * which this code originated.
1993 *
1994 * Note that soreceive_rcvoob(), unlike the remainder of soreceive(), is
1995 * unable to return an mbuf chain to the caller.
1996 */
1997 static int
1998 soreceive_rcvoob(struct socket *so, struct uio *uio, int flags)
1999 {
2000 struct protosw *pr = so->so_proto;
2001 struct mbuf *m;
2002 int error;
2003
2004 KASSERT(flags & MSG_OOB, ("soreceive_rcvoob: (flags & MSG_OOB) == 0"));
2005 VNET_SO_ASSERT(so);
2006
2007 m = m_get(M_WAITOK, MT_DATA);
2008 error = pr->pr_rcvoob(so, m, flags & MSG_PEEK);
2009 if (error)
2010 goto bad;
2011 do {
2012 error = uiomove(mtod(m, void *),
2013 (int) min(uio->uio_resid, m->m_len), uio);
2014 m = m_free(m);
2015 } while (uio->uio_resid && error == 0 && m);
2016 bad:
2017 if (m != NULL)
2018 m_freem(m);
2019 return (error);
2020 }
2021
2022 /*
2023 * Following replacement or removal of the first mbuf on the first mbuf chain
2024 * of a socket buffer, push necessary state changes back into the socket
2025 * buffer so that other consumers see the values consistently. 'nextrecord'
2026 * is the callers locally stored value of the original value of
2027 * sb->sb_mb->m_nextpkt which must be restored when the lead mbuf changes.
2028 * NOTE: 'nextrecord' may be NULL.
2029 */
2030 static __inline void
2031 sockbuf_pushsync(struct sockbuf *sb, struct mbuf *nextrecord)
2032 {
2033
2034 SOCKBUF_LOCK_ASSERT(sb);
2035 /*
2036 * First, update for the new value of nextrecord. If necessary, make
2037 * it the first record.
2038 */
2039 if (sb->sb_mb != NULL)
2040 sb->sb_mb->m_nextpkt = nextrecord;
2041 else
2042 sb->sb_mb = nextrecord;
2043
2044 /*
2045 * Now update any dependent socket buffer fields to reflect the new
2046 * state. This is an expanded inline of SB_EMPTY_FIXUP(), with the
2047 * addition of a second clause that takes care of the case where
2048 * sb_mb has been updated, but remains the last record.
2049 */
2050 if (sb->sb_mb == NULL) {
2051 sb->sb_mbtail = NULL;
2052 sb->sb_lastrecord = NULL;
2053 } else if (sb->sb_mb->m_nextpkt == NULL)
2054 sb->sb_lastrecord = sb->sb_mb;
2055 }
2056
2057 /*
2058 * Implement receive operations on a socket. We depend on the way that
2059 * records are added to the sockbuf by sbappend. In particular, each record
2060 * (mbufs linked through m_next) must begin with an address if the protocol
2061 * so specifies, followed by an optional mbuf or mbufs containing ancillary
2062 * data, and then zero or more mbufs of data. In order to allow parallelism
2063 * between network receive and copying to user space, as well as avoid
2064 * sleeping with a mutex held, we release the socket buffer mutex during the
2065 * user space copy. Although the sockbuf is locked, new data may still be
2066 * appended, and thus we must maintain consistency of the sockbuf during that
2067 * time.
2068 *
2069 * The caller may receive the data as a single mbuf chain by supplying an
2070 * mbuf **mp0 for use in returning the chain. The uio is then used only for
2071 * the count in uio_resid.
2072 */
2073 int
2074 soreceive_generic(struct socket *so, struct sockaddr **psa, struct uio *uio,
2075 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2076 {
2077 struct mbuf *m, **mp;
2078 int flags, error, offset;
2079 ssize_t len;
2080 struct protosw *pr = so->so_proto;
2081 struct mbuf *nextrecord;
2082 int moff, type = 0;
2083 ssize_t orig_resid = uio->uio_resid;
2084 bool report_real_len = false;
2085
2086 mp = mp0;
2087 if (psa != NULL)
2088 *psa = NULL;
2089 if (controlp != NULL)
2090 *controlp = NULL;
2091 if (flagsp != NULL) {
2092 report_real_len = *flagsp & MSG_TRUNC;
2093 *flagsp &= ~MSG_TRUNC;
2094 flags = *flagsp &~ MSG_EOR;
2095 } else
2096 flags = 0;
2097 if (flags & MSG_OOB)
2098 return (soreceive_rcvoob(so, uio, flags));
2099 if (mp != NULL)
2100 *mp = NULL;
2101
2102 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
2103 if (error)
2104 return (error);
2105
2106 restart:
2107 SOCKBUF_LOCK(&so->so_rcv);
2108 m = so->so_rcv.sb_mb;
2109 /*
2110 * If we have less data than requested, block awaiting more (subject
2111 * to any timeout) if:
2112 * 1. the current count is less than the low water mark, or
2113 * 2. MSG_DONTWAIT is not set
2114 */
2115 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 &&
2116 sbavail(&so->so_rcv) < uio->uio_resid) &&
2117 sbavail(&so->so_rcv) < so->so_rcv.sb_lowat &&
2118 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
2119 KASSERT(m != NULL || !sbavail(&so->so_rcv),
2120 ("receive: m == %p sbavail == %u",
2121 m, sbavail(&so->so_rcv)));
2122 if (so->so_error || so->so_rerror) {
2123 if (m != NULL)
2124 goto dontblock;
2125 if (so->so_error)
2126 error = so->so_error;
2127 else
2128 error = so->so_rerror;
2129 if ((flags & MSG_PEEK) == 0) {
2130 if (so->so_error)
2131 so->so_error = 0;
2132 else
2133 so->so_rerror = 0;
2134 }
2135 SOCKBUF_UNLOCK(&so->so_rcv);
2136 goto release;
2137 }
2138 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2139 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
2140 if (m != NULL)
2141 goto dontblock;
2142 #ifdef KERN_TLS
2143 else if (so->so_rcv.sb_tlsdcc == 0 &&
2144 so->so_rcv.sb_tlscc == 0) {
2145 #else
2146 else {
2147 #endif
2148 SOCKBUF_UNLOCK(&so->so_rcv);
2149 goto release;
2150 }
2151 }
2152 for (; m != NULL; m = m->m_next)
2153 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
2154 m = so->so_rcv.sb_mb;
2155 goto dontblock;
2156 }
2157 if ((so->so_state & (SS_ISCONNECTING | SS_ISCONNECTED |
2158 SS_ISDISCONNECTING | SS_ISDISCONNECTED)) == 0 &&
2159 (so->so_proto->pr_flags & PR_CONNREQUIRED) != 0) {
2160 SOCKBUF_UNLOCK(&so->so_rcv);
2161 error = ENOTCONN;
2162 goto release;
2163 }
2164 if (uio->uio_resid == 0 && !report_real_len) {
2165 SOCKBUF_UNLOCK(&so->so_rcv);
2166 goto release;
2167 }
2168 if ((so->so_state & SS_NBIO) ||
2169 (flags & (MSG_DONTWAIT|MSG_NBIO))) {
2170 SOCKBUF_UNLOCK(&so->so_rcv);
2171 error = EWOULDBLOCK;
2172 goto release;
2173 }
2174 SBLASTRECORDCHK(&so->so_rcv);
2175 SBLASTMBUFCHK(&so->so_rcv);
2176 error = sbwait(so, SO_RCV);
2177 SOCKBUF_UNLOCK(&so->so_rcv);
2178 if (error)
2179 goto release;
2180 goto restart;
2181 }
2182 dontblock:
2183 /*
2184 * From this point onward, we maintain 'nextrecord' as a cache of the
2185 * pointer to the next record in the socket buffer. We must keep the
2186 * various socket buffer pointers and local stack versions of the
2187 * pointers in sync, pushing out modifications before dropping the
2188 * socket buffer mutex, and re-reading them when picking it up.
2189 *
2190 * Otherwise, we will race with the network stack appending new data
2191 * or records onto the socket buffer by using inconsistent/stale
2192 * versions of the field, possibly resulting in socket buffer
2193 * corruption.
2194 *
2195 * By holding the high-level sblock(), we prevent simultaneous
2196 * readers from pulling off the front of the socket buffer.
2197 */
2198 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2199 if (uio->uio_td)
2200 uio->uio_td->td_ru.ru_msgrcv++;
2201 KASSERT(m == so->so_rcv.sb_mb, ("soreceive: m != so->so_rcv.sb_mb"));
2202 SBLASTRECORDCHK(&so->so_rcv);
2203 SBLASTMBUFCHK(&so->so_rcv);
2204 nextrecord = m->m_nextpkt;
2205 if (pr->pr_flags & PR_ADDR) {
2206 KASSERT(m->m_type == MT_SONAME,
2207 ("m->m_type == %d", m->m_type));
2208 orig_resid = 0;
2209 if (psa != NULL)
2210 *psa = sodupsockaddr(mtod(m, struct sockaddr *),
2211 M_NOWAIT);
2212 if (flags & MSG_PEEK) {
2213 m = m->m_next;
2214 } else {
2215 sbfree(&so->so_rcv, m);
2216 so->so_rcv.sb_mb = m_free(m);
2217 m = so->so_rcv.sb_mb;
2218 sockbuf_pushsync(&so->so_rcv, nextrecord);
2219 }
2220 }
2221
2222 /*
2223 * Process one or more MT_CONTROL mbufs present before any data mbufs
2224 * in the first mbuf chain on the socket buffer. If MSG_PEEK, we
2225 * just copy the data; if !MSG_PEEK, we call into the protocol to
2226 * perform externalization (or freeing if controlp == NULL).
2227 */
2228 if (m != NULL && m->m_type == MT_CONTROL) {
2229 struct mbuf *cm = NULL, *cmn;
2230 struct mbuf **cme = &cm;
2231 #ifdef KERN_TLS
2232 struct cmsghdr *cmsg;
2233 struct tls_get_record tgr;
2234
2235 /*
2236 * For MSG_TLSAPPDATA, check for an alert record.
2237 * If found, return ENXIO without removing
2238 * it from the receive queue. This allows a subsequent
2239 * call without MSG_TLSAPPDATA to receive it.
2240 * Note that, for TLS, there should only be a single
2241 * control mbuf with the TLS_GET_RECORD message in it.
2242 */
2243 if (flags & MSG_TLSAPPDATA) {
2244 cmsg = mtod(m, struct cmsghdr *);
2245 if (cmsg->cmsg_type == TLS_GET_RECORD &&
2246 cmsg->cmsg_len == CMSG_LEN(sizeof(tgr))) {
2247 memcpy(&tgr, CMSG_DATA(cmsg), sizeof(tgr));
2248 if (__predict_false(tgr.tls_type ==
2249 TLS_RLTYPE_ALERT)) {
2250 SOCKBUF_UNLOCK(&so->so_rcv);
2251 error = ENXIO;
2252 goto release;
2253 }
2254 }
2255 }
2256 #endif
2257
2258 do {
2259 if (flags & MSG_PEEK) {
2260 if (controlp != NULL) {
2261 *controlp = m_copym(m, 0, m->m_len,
2262 M_NOWAIT);
2263 controlp = &(*controlp)->m_next;
2264 }
2265 m = m->m_next;
2266 } else {
2267 sbfree(&so->so_rcv, m);
2268 so->so_rcv.sb_mb = m->m_next;
2269 m->m_next = NULL;
2270 *cme = m;
2271 cme = &(*cme)->m_next;
2272 m = so->so_rcv.sb_mb;
2273 }
2274 } while (m != NULL && m->m_type == MT_CONTROL);
2275 if ((flags & MSG_PEEK) == 0)
2276 sockbuf_pushsync(&so->so_rcv, nextrecord);
2277 while (cm != NULL) {
2278 cmn = cm->m_next;
2279 cm->m_next = NULL;
2280 if (pr->pr_domain->dom_externalize != NULL) {
2281 SOCKBUF_UNLOCK(&so->so_rcv);
2282 VNET_SO_ASSERT(so);
2283 error = (*pr->pr_domain->dom_externalize)
2284 (cm, controlp, flags);
2285 SOCKBUF_LOCK(&so->so_rcv);
2286 } else if (controlp != NULL)
2287 *controlp = cm;
2288 else
2289 m_freem(cm);
2290 if (controlp != NULL) {
2291 while (*controlp != NULL)
2292 controlp = &(*controlp)->m_next;
2293 }
2294 cm = cmn;
2295 }
2296 if (m != NULL)
2297 nextrecord = so->so_rcv.sb_mb->m_nextpkt;
2298 else
2299 nextrecord = so->so_rcv.sb_mb;
2300 orig_resid = 0;
2301 }
2302 if (m != NULL) {
2303 if ((flags & MSG_PEEK) == 0) {
2304 KASSERT(m->m_nextpkt == nextrecord,
2305 ("soreceive: post-control, nextrecord !sync"));
2306 if (nextrecord == NULL) {
2307 KASSERT(so->so_rcv.sb_mb == m,
2308 ("soreceive: post-control, sb_mb!=m"));
2309 KASSERT(so->so_rcv.sb_lastrecord == m,
2310 ("soreceive: post-control, lastrecord!=m"));
2311 }
2312 }
2313 type = m->m_type;
2314 if (type == MT_OOBDATA)
2315 flags |= MSG_OOB;
2316 } else {
2317 if ((flags & MSG_PEEK) == 0) {
2318 KASSERT(so->so_rcv.sb_mb == nextrecord,
2319 ("soreceive: sb_mb != nextrecord"));
2320 if (so->so_rcv.sb_mb == NULL) {
2321 KASSERT(so->so_rcv.sb_lastrecord == NULL,
2322 ("soreceive: sb_lastercord != NULL"));
2323 }
2324 }
2325 }
2326 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2327 SBLASTRECORDCHK(&so->so_rcv);
2328 SBLASTMBUFCHK(&so->so_rcv);
2329
2330 /*
2331 * Now continue to read any data mbufs off of the head of the socket
2332 * buffer until the read request is satisfied. Note that 'type' is
2333 * used to store the type of any mbuf reads that have happened so far
2334 * such that soreceive() can stop reading if the type changes, which
2335 * causes soreceive() to return only one of regular data and inline
2336 * out-of-band data in a single socket receive operation.
2337 */
2338 moff = 0;
2339 offset = 0;
2340 while (m != NULL && !(m->m_flags & M_NOTAVAIL) && uio->uio_resid > 0
2341 && error == 0) {
2342 /*
2343 * If the type of mbuf has changed since the last mbuf
2344 * examined ('type'), end the receive operation.
2345 */
2346 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2347 if (m->m_type == MT_OOBDATA || m->m_type == MT_CONTROL) {
2348 if (type != m->m_type)
2349 break;
2350 } else if (type == MT_OOBDATA)
2351 break;
2352 else
2353 KASSERT(m->m_type == MT_DATA,
2354 ("m->m_type == %d", m->m_type));
2355 so->so_rcv.sb_state &= ~SBS_RCVATMARK;
2356 len = uio->uio_resid;
2357 if (so->so_oobmark && len > so->so_oobmark - offset)
2358 len = so->so_oobmark - offset;
2359 if (len > m->m_len - moff)
2360 len = m->m_len - moff;
2361 /*
2362 * If mp is set, just pass back the mbufs. Otherwise copy
2363 * them out via the uio, then free. Sockbuf must be
2364 * consistent here (points to current mbuf, it points to next
2365 * record) when we drop priority; we must note any additions
2366 * to the sockbuf when we block interrupts again.
2367 */
2368 if (mp == NULL) {
2369 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2370 SBLASTRECORDCHK(&so->so_rcv);
2371 SBLASTMBUFCHK(&so->so_rcv);
2372 SOCKBUF_UNLOCK(&so->so_rcv);
2373 if ((m->m_flags & M_EXTPG) != 0)
2374 error = m_unmapped_uiomove(m, moff, uio,
2375 (int)len);
2376 else
2377 error = uiomove(mtod(m, char *) + moff,
2378 (int)len, uio);
2379 SOCKBUF_LOCK(&so->so_rcv);
2380 if (error) {
2381 /*
2382 * The MT_SONAME mbuf has already been removed
2383 * from the record, so it is necessary to
2384 * remove the data mbufs, if any, to preserve
2385 * the invariant in the case of PR_ADDR that
2386 * requires MT_SONAME mbufs at the head of
2387 * each record.
2388 */
2389 if (pr->pr_flags & PR_ATOMIC &&
2390 ((flags & MSG_PEEK) == 0))
2391 (void)sbdroprecord_locked(&so->so_rcv);
2392 SOCKBUF_UNLOCK(&so->so_rcv);
2393 goto release;
2394 }
2395 } else
2396 uio->uio_resid -= len;
2397 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2398 if (len == m->m_len - moff) {
2399 if (m->m_flags & M_EOR)
2400 flags |= MSG_EOR;
2401 if (flags & MSG_PEEK) {
2402 m = m->m_next;
2403 moff = 0;
2404 } else {
2405 nextrecord = m->m_nextpkt;
2406 sbfree(&so->so_rcv, m);
2407 if (mp != NULL) {
2408 m->m_nextpkt = NULL;
2409 *mp = m;
2410 mp = &m->m_next;
2411 so->so_rcv.sb_mb = m = m->m_next;
2412 *mp = NULL;
2413 } else {
2414 so->so_rcv.sb_mb = m_free(m);
2415 m = so->so_rcv.sb_mb;
2416 }
2417 sockbuf_pushsync(&so->so_rcv, nextrecord);
2418 SBLASTRECORDCHK(&so->so_rcv);
2419 SBLASTMBUFCHK(&so->so_rcv);
2420 }
2421 } else {
2422 if (flags & MSG_PEEK)
2423 moff += len;
2424 else {
2425 if (mp != NULL) {
2426 if (flags & MSG_DONTWAIT) {
2427 *mp = m_copym(m, 0, len,
2428 M_NOWAIT);
2429 if (*mp == NULL) {
2430 /*
2431 * m_copym() couldn't
2432 * allocate an mbuf.
2433 * Adjust uio_resid back
2434 * (it was adjusted
2435 * down by len bytes,
2436 * which we didn't end
2437 * up "copying" over).
2438 */
2439 uio->uio_resid += len;
2440 break;
2441 }
2442 } else {
2443 SOCKBUF_UNLOCK(&so->so_rcv);
2444 *mp = m_copym(m, 0, len,
2445 M_WAITOK);
2446 SOCKBUF_LOCK(&so->so_rcv);
2447 }
2448 }
2449 sbcut_locked(&so->so_rcv, len);
2450 }
2451 }
2452 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2453 if (so->so_oobmark) {
2454 if ((flags & MSG_PEEK) == 0) {
2455 so->so_oobmark -= len;
2456 if (so->so_oobmark == 0) {
2457 so->so_rcv.sb_state |= SBS_RCVATMARK;
2458 break;
2459 }
2460 } else {
2461 offset += len;
2462 if (offset == so->so_oobmark)
2463 break;
2464 }
2465 }
2466 if (flags & MSG_EOR)
2467 break;
2468 /*
2469 * If the MSG_WAITALL flag is set (for non-atomic socket), we
2470 * must not quit until "uio->uio_resid == 0" or an error
2471 * termination. If a signal/timeout occurs, return with a
2472 * short count but without error. Keep sockbuf locked
2473 * against other readers.
2474 */
2475 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
2476 !sosendallatonce(so) && nextrecord == NULL) {
2477 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2478 if (so->so_error || so->so_rerror ||
2479 so->so_rcv.sb_state & SBS_CANTRCVMORE)
2480 break;
2481 /*
2482 * Notify the protocol that some data has been
2483 * drained before blocking.
2484 */
2485 if (pr->pr_flags & PR_WANTRCVD) {
2486 SOCKBUF_UNLOCK(&so->so_rcv);
2487 VNET_SO_ASSERT(so);
2488 pr->pr_rcvd(so, flags);
2489 SOCKBUF_LOCK(&so->so_rcv);
2490 if (__predict_false(so->so_rcv.sb_mb == NULL &&
2491 (so->so_error || so->so_rerror ||
2492 so->so_rcv.sb_state & SBS_CANTRCVMORE)))
2493 break;
2494 }
2495 SBLASTRECORDCHK(&so->so_rcv);
2496 SBLASTMBUFCHK(&so->so_rcv);
2497 /*
2498 * We could receive some data while was notifying
2499 * the protocol. Skip blocking in this case.
2500 */
2501 if (so->so_rcv.sb_mb == NULL) {
2502 error = sbwait(so, SO_RCV);
2503 if (error) {
2504 SOCKBUF_UNLOCK(&so->so_rcv);
2505 goto release;
2506 }
2507 }
2508 m = so->so_rcv.sb_mb;
2509 if (m != NULL)
2510 nextrecord = m->m_nextpkt;
2511 }
2512 }
2513
2514 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2515 if (m != NULL && pr->pr_flags & PR_ATOMIC) {
2516 if (report_real_len)
2517 uio->uio_resid -= m_length(m, NULL) - moff;
2518 flags |= MSG_TRUNC;
2519 if ((flags & MSG_PEEK) == 0)
2520 (void) sbdroprecord_locked(&so->so_rcv);
2521 }
2522 if ((flags & MSG_PEEK) == 0) {
2523 if (m == NULL) {
2524 /*
2525 * First part is an inline SB_EMPTY_FIXUP(). Second
2526 * part makes sure sb_lastrecord is up-to-date if
2527 * there is still data in the socket buffer.
2528 */
2529 so->so_rcv.sb_mb = nextrecord;
2530 if (so->so_rcv.sb_mb == NULL) {
2531 so->so_rcv.sb_mbtail = NULL;
2532 so->so_rcv.sb_lastrecord = NULL;
2533 } else if (nextrecord->m_nextpkt == NULL)
2534 so->so_rcv.sb_lastrecord = nextrecord;
2535 }
2536 SBLASTRECORDCHK(&so->so_rcv);
2537 SBLASTMBUFCHK(&so->so_rcv);
2538 /*
2539 * If soreceive() is being done from the socket callback,
2540 * then don't need to generate ACK to peer to update window,
2541 * since ACK will be generated on return to TCP.
2542 */
2543 if (!(flags & MSG_SOCALLBCK) &&
2544 (pr->pr_flags & PR_WANTRCVD)) {
2545 SOCKBUF_UNLOCK(&so->so_rcv);
2546 VNET_SO_ASSERT(so);
2547 pr->pr_rcvd(so, flags);
2548 SOCKBUF_LOCK(&so->so_rcv);
2549 }
2550 }
2551 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2552 if (orig_resid == uio->uio_resid && orig_resid &&
2553 (flags & MSG_EOR) == 0 && (so->so_rcv.sb_state & SBS_CANTRCVMORE) == 0) {
2554 SOCKBUF_UNLOCK(&so->so_rcv);
2555 goto restart;
2556 }
2557 SOCKBUF_UNLOCK(&so->so_rcv);
2558
2559 if (flagsp != NULL)
2560 *flagsp |= flags;
2561 release:
2562 SOCK_IO_RECV_UNLOCK(so);
2563 return (error);
2564 }
2565
2566 /*
2567 * Optimized version of soreceive() for stream (TCP) sockets.
2568 */
2569 int
2570 soreceive_stream(struct socket *so, struct sockaddr **psa, struct uio *uio,
2571 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2572 {
2573 int len = 0, error = 0, flags, oresid;
2574 struct sockbuf *sb;
2575 struct mbuf *m, *n = NULL;
2576
2577 /* We only do stream sockets. */
2578 if (so->so_type != SOCK_STREAM)
2579 return (EINVAL);
2580 if (psa != NULL)
2581 *psa = NULL;
2582 if (flagsp != NULL)
2583 flags = *flagsp &~ MSG_EOR;
2584 else
2585 flags = 0;
2586 if (controlp != NULL)
2587 *controlp = NULL;
2588 if (flags & MSG_OOB)
2589 return (soreceive_rcvoob(so, uio, flags));
2590 if (mp0 != NULL)
2591 *mp0 = NULL;
2592
2593 sb = &so->so_rcv;
2594
2595 #ifdef KERN_TLS
2596 /*
2597 * KTLS store TLS records as records with a control message to
2598 * describe the framing.
2599 *
2600 * We check once here before acquiring locks to optimize the
2601 * common case.
2602 */
2603 if (sb->sb_tls_info != NULL)
2604 return (soreceive_generic(so, psa, uio, mp0, controlp,
2605 flagsp));
2606 #endif
2607
2608 /* Prevent other readers from entering the socket. */
2609 error = SOCK_IO_RECV_LOCK(so, SBLOCKWAIT(flags));
2610 if (error)
2611 return (error);
2612 SOCKBUF_LOCK(sb);
2613
2614 #ifdef KERN_TLS
2615 if (sb->sb_tls_info != NULL) {
2616 SOCKBUF_UNLOCK(sb);
2617 SOCK_IO_RECV_UNLOCK(so);
2618 return (soreceive_generic(so, psa, uio, mp0, controlp,
2619 flagsp));
2620 }
2621 #endif
2622
2623 /* Easy one, no space to copyout anything. */
2624 if (uio->uio_resid == 0) {
2625 error = EINVAL;
2626 goto out;
2627 }
2628 oresid = uio->uio_resid;
2629
2630 /* We will never ever get anything unless we are or were connected. */
2631 if (!(so->so_state & (SS_ISCONNECTED|SS_ISDISCONNECTED))) {
2632 error = ENOTCONN;
2633 goto out;
2634 }
2635
2636 restart:
2637 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2638
2639 /* Abort if socket has reported problems. */
2640 if (so->so_error) {
2641 if (sbavail(sb) > 0)
2642 goto deliver;
2643 if (oresid > uio->uio_resid)
2644 goto out;
2645 error = so->so_error;
2646 if (!(flags & MSG_PEEK))
2647 so->so_error = 0;
2648 goto out;
2649 }
2650
2651 /* Door is closed. Deliver what is left, if any. */
2652 if (sb->sb_state & SBS_CANTRCVMORE) {
2653 if (sbavail(sb) > 0)
2654 goto deliver;
2655 else
2656 goto out;
2657 }
2658
2659 /* Socket buffer is empty and we shall not block. */
2660 if (sbavail(sb) == 0 &&
2661 ((so->so_state & SS_NBIO) || (flags & (MSG_DONTWAIT|MSG_NBIO)))) {
2662 error = EAGAIN;
2663 goto out;
2664 }
2665
2666 /* Socket buffer got some data that we shall deliver now. */
2667 if (sbavail(sb) > 0 && !(flags & MSG_WAITALL) &&
2668 ((so->so_state & SS_NBIO) ||
2669 (flags & (MSG_DONTWAIT|MSG_NBIO)) ||
2670 sbavail(sb) >= sb->sb_lowat ||
2671 sbavail(sb) >= uio->uio_resid ||
2672 sbavail(sb) >= sb->sb_hiwat) ) {
2673 goto deliver;
2674 }
2675
2676 /* On MSG_WAITALL we must wait until all data or error arrives. */
2677 if ((flags & MSG_WAITALL) &&
2678 (sbavail(sb) >= uio->uio_resid || sbavail(sb) >= sb->sb_hiwat))
2679 goto deliver;
2680
2681 /*
2682 * Wait and block until (more) data comes in.
2683 * NB: Drops the sockbuf lock during wait.
2684 */
2685 error = sbwait(so, SO_RCV);
2686 if (error)
2687 goto out;
2688 goto restart;
2689
2690 deliver:
2691 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2692 KASSERT(sbavail(sb) > 0, ("%s: sockbuf empty", __func__));
2693 KASSERT(sb->sb_mb != NULL, ("%s: sb_mb == NULL", __func__));
2694
2695 /* Statistics. */
2696 if (uio->uio_td)
2697 uio->uio_td->td_ru.ru_msgrcv++;
2698
2699 /* Fill uio until full or current end of socket buffer is reached. */
2700 len = min(uio->uio_resid, sbavail(sb));
2701 if (mp0 != NULL) {
2702 /* Dequeue as many mbufs as possible. */
2703 if (!(flags & MSG_PEEK) && len >= sb->sb_mb->m_len) {
2704 if (*mp0 == NULL)
2705 *mp0 = sb->sb_mb;
2706 else
2707 m_cat(*mp0, sb->sb_mb);
2708 for (m = sb->sb_mb;
2709 m != NULL && m->m_len <= len;
2710 m = m->m_next) {
2711 KASSERT(!(m->m_flags & M_NOTAVAIL),
2712 ("%s: m %p not available", __func__, m));
2713 len -= m->m_len;
2714 uio->uio_resid -= m->m_len;
2715 sbfree(sb, m);
2716 n = m;
2717 }
2718 n->m_next = NULL;
2719 sb->sb_mb = m;
2720 sb->sb_lastrecord = sb->sb_mb;
2721 if (sb->sb_mb == NULL)
2722 SB_EMPTY_FIXUP(sb);
2723 }
2724 /* Copy the remainder. */
2725 if (len > 0) {
2726 KASSERT(sb->sb_mb != NULL,
2727 ("%s: len > 0 && sb->sb_mb empty", __func__));
2728
2729 m = m_copym(sb->sb_mb, 0, len, M_NOWAIT);
2730 if (m == NULL)
2731 len = 0; /* Don't flush data from sockbuf. */
2732 else
2733 uio->uio_resid -= len;
2734 if (*mp0 != NULL)
2735 m_cat(*mp0, m);
2736 else
2737 *mp0 = m;
2738 if (*mp0 == NULL) {
2739 error = ENOBUFS;
2740 goto out;
2741 }
2742 }
2743 } else {
2744 /* NB: Must unlock socket buffer as uiomove may sleep. */
2745 SOCKBUF_UNLOCK(sb);
2746 error = m_mbuftouio(uio, sb->sb_mb, len);
2747 SOCKBUF_LOCK(sb);
2748 if (error)
2749 goto out;
2750 }
2751 SBLASTRECORDCHK(sb);
2752 SBLASTMBUFCHK(sb);
2753
2754 /*
2755 * Remove the delivered data from the socket buffer unless we
2756 * were only peeking.
2757 */
2758 if (!(flags & MSG_PEEK)) {
2759 if (len > 0)
2760 sbdrop_locked(sb, len);
2761
2762 /* Notify protocol that we drained some data. */
2763 if ((so->so_proto->pr_flags & PR_WANTRCVD) &&
2764 (((flags & MSG_WAITALL) && uio->uio_resid > 0) ||
2765 !(flags & MSG_SOCALLBCK))) {
2766 SOCKBUF_UNLOCK(sb);
2767 VNET_SO_ASSERT(so);
2768 so->so_proto->pr_rcvd(so, flags);
2769 SOCKBUF_LOCK(sb);
2770 }
2771 }
2772
2773 /*
2774 * For MSG_WAITALL we may have to loop again and wait for
2775 * more data to come in.
2776 */
2777 if ((flags & MSG_WAITALL) && uio->uio_resid > 0)
2778 goto restart;
2779 out:
2780 SBLASTRECORDCHK(sb);
2781 SBLASTMBUFCHK(sb);
2782 SOCKBUF_UNLOCK(sb);
2783 SOCK_IO_RECV_UNLOCK(so);
2784 return (error);
2785 }
2786
2787 /*
2788 * Optimized version of soreceive() for simple datagram cases from userspace.
2789 * Unlike in the stream case, we're able to drop a datagram if copyout()
2790 * fails, and because we handle datagrams atomically, we don't need to use a
2791 * sleep lock to prevent I/O interlacing.
2792 */
2793 int
2794 soreceive_dgram(struct socket *so, struct sockaddr **psa, struct uio *uio,
2795 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2796 {
2797 struct mbuf *m, *m2;
2798 int flags, error;
2799 ssize_t len;
2800 struct protosw *pr = so->so_proto;
2801 struct mbuf *nextrecord;
2802
2803 if (psa != NULL)
2804 *psa = NULL;
2805 if (controlp != NULL)
2806 *controlp = NULL;
2807 if (flagsp != NULL)
2808 flags = *flagsp &~ MSG_EOR;
2809 else
2810 flags = 0;
2811
2812 /*
2813 * For any complicated cases, fall back to the full
2814 * soreceive_generic().
2815 */
2816 if (mp0 != NULL || (flags & (MSG_PEEK | MSG_OOB | MSG_TRUNC)))
2817 return (soreceive_generic(so, psa, uio, mp0, controlp,
2818 flagsp));
2819
2820 /*
2821 * Enforce restrictions on use.
2822 */
2823 KASSERT((pr->pr_flags & PR_WANTRCVD) == 0,
2824 ("soreceive_dgram: wantrcvd"));
2825 KASSERT(pr->pr_flags & PR_ATOMIC, ("soreceive_dgram: !atomic"));
2826 KASSERT((so->so_rcv.sb_state & SBS_RCVATMARK) == 0,
2827 ("soreceive_dgram: SBS_RCVATMARK"));
2828 KASSERT((so->so_proto->pr_flags & PR_CONNREQUIRED) == 0,
2829 ("soreceive_dgram: P_CONNREQUIRED"));
2830
2831 /*
2832 * Loop blocking while waiting for a datagram.
2833 */
2834 SOCKBUF_LOCK(&so->so_rcv);
2835 while ((m = so->so_rcv.sb_mb) == NULL) {
2836 KASSERT(sbavail(&so->so_rcv) == 0,
2837 ("soreceive_dgram: sb_mb NULL but sbavail %u",
2838 sbavail(&so->so_rcv)));
2839 if (so->so_error) {
2840 error = so->so_error;
2841 so->so_error = 0;
2842 SOCKBUF_UNLOCK(&so->so_rcv);
2843 return (error);
2844 }
2845 if (so->so_rcv.sb_state & SBS_CANTRCVMORE ||
2846 uio->uio_resid == 0) {
2847 SOCKBUF_UNLOCK(&so->so_rcv);
2848 return (0);
2849 }
2850 if ((so->so_state & SS_NBIO) ||
2851 (flags & (MSG_DONTWAIT|MSG_NBIO))) {
2852 SOCKBUF_UNLOCK(&so->so_rcv);
2853 return (EWOULDBLOCK);
2854 }
2855 SBLASTRECORDCHK(&so->so_rcv);
2856 SBLASTMBUFCHK(&so->so_rcv);
2857 error = sbwait(so, SO_RCV);
2858 if (error) {
2859 SOCKBUF_UNLOCK(&so->so_rcv);
2860 return (error);
2861 }
2862 }
2863 SOCKBUF_LOCK_ASSERT(&so->so_rcv);
2864
2865 if (uio->uio_td)
2866 uio->uio_td->td_ru.ru_msgrcv++;
2867 SBLASTRECORDCHK(&so->so_rcv);
2868 SBLASTMBUFCHK(&so->so_rcv);
2869 nextrecord = m->m_nextpkt;
2870 if (nextrecord == NULL) {
2871 KASSERT(so->so_rcv.sb_lastrecord == m,
2872 ("soreceive_dgram: lastrecord != m"));
2873 }
2874
2875 KASSERT(so->so_rcv.sb_mb->m_nextpkt == nextrecord,
2876 ("soreceive_dgram: m_nextpkt != nextrecord"));
2877
2878 /*
2879 * Pull 'm' and its chain off the front of the packet queue.
2880 */
2881 so->so_rcv.sb_mb = NULL;
2882 sockbuf_pushsync(&so->so_rcv, nextrecord);
2883
2884 /*
2885 * Walk 'm's chain and free that many bytes from the socket buffer.
2886 */
2887 for (m2 = m; m2 != NULL; m2 = m2->m_next)
2888 sbfree(&so->so_rcv, m2);
2889
2890 /*
2891 * Do a few last checks before we let go of the lock.
2892 */
2893 SBLASTRECORDCHK(&so->so_rcv);
2894 SBLASTMBUFCHK(&so->so_rcv);
2895 SOCKBUF_UNLOCK(&so->so_rcv);
2896
2897 if (pr->pr_flags & PR_ADDR) {
2898 KASSERT(m->m_type == MT_SONAME,
2899 ("m->m_type == %d", m->m_type));
2900 if (psa != NULL)
2901 *psa = sodupsockaddr(mtod(m, struct sockaddr *),
2902 M_WAITOK);
2903 m = m_free(m);
2904 }
2905 KASSERT(m, ("%s: no data or control after soname", __func__));
2906
2907 /*
2908 * Packet to copyout() is now in 'm' and it is disconnected from the
2909 * queue.
2910 *
2911 * Process one or more MT_CONTROL mbufs present before any data mbufs
2912 * in the first mbuf chain on the socket buffer. We call into the
2913 * protocol to perform externalization (or freeing if controlp ==
2914 * NULL). In some cases there can be only MT_CONTROL mbufs without
2915 * MT_DATA mbufs.
2916 */
2917 if (m->m_type == MT_CONTROL) {
2918 struct mbuf *cm = NULL, *cmn;
2919 struct mbuf **cme = &cm;
2920
2921 do {
2922 m2 = m->m_next;
2923 m->m_next = NULL;
2924 *cme = m;
2925 cme = &(*cme)->m_next;
2926 m = m2;
2927 } while (m != NULL && m->m_type == MT_CONTROL);
2928 while (cm != NULL) {
2929 cmn = cm->m_next;
2930 cm->m_next = NULL;
2931 if (pr->pr_domain->dom_externalize != NULL) {
2932 error = (*pr->pr_domain->dom_externalize)
2933 (cm, controlp, flags);
2934 } else if (controlp != NULL)
2935 *controlp = cm;
2936 else
2937 m_freem(cm);
2938 if (controlp != NULL) {
2939 while (*controlp != NULL)
2940 controlp = &(*controlp)->m_next;
2941 }
2942 cm = cmn;
2943 }
2944 }
2945 KASSERT(m == NULL || m->m_type == MT_DATA,
2946 ("soreceive_dgram: !data"));
2947 while (m != NULL && uio->uio_resid > 0) {
2948 len = uio->uio_resid;
2949 if (len > m->m_len)
2950 len = m->m_len;
2951 error = uiomove(mtod(m, char *), (int)len, uio);
2952 if (error) {
2953 m_freem(m);
2954 return (error);
2955 }
2956 if (len == m->m_len)
2957 m = m_free(m);
2958 else {
2959 m->m_data += len;
2960 m->m_len -= len;
2961 }
2962 }
2963 if (m != NULL) {
2964 flags |= MSG_TRUNC;
2965 m_freem(m);
2966 }
2967 if (flagsp != NULL)
2968 *flagsp |= flags;
2969 return (0);
2970 }
2971
2972 int
2973 soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
2974 struct mbuf **mp0, struct mbuf **controlp, int *flagsp)
2975 {
2976 int error;
2977
2978 CURVNET_SET(so->so_vnet);
2979 error = so->so_proto->pr_soreceive(so, psa, uio, mp0, controlp, flagsp);
2980 CURVNET_RESTORE();
2981 return (error);
2982 }
2983
2984 int
2985 soshutdown(struct socket *so, enum shutdown_how how)
2986 {
2987 int error;
2988
2989 CURVNET_SET(so->so_vnet);
2990 error = so->so_proto->pr_shutdown(so, how);
2991 CURVNET_RESTORE();
2992
2993 return (error);
2994 }
2995
2996 /*
2997 * Used by several pr_shutdown implementations that use generic socket buffers.
2998 */
2999 void
3000 sorflush(struct socket *so)
3001 {
3002 int error;
3003
3004 VNET_SO_ASSERT(so);
3005
3006 /*
3007 * Dislodge threads currently blocked in receive and wait to acquire
3008 * a lock against other simultaneous readers before clearing the
3009 * socket buffer. Don't let our acquire be interrupted by a signal
3010 * despite any existing socket disposition on interruptable waiting.
3011 *
3012 * The SOCK_IO_RECV_LOCK() is important here as there some pr_soreceive
3013 * methods that read the top of the socket buffer without acquisition
3014 * of the socket buffer mutex, assuming that top of the buffer
3015 * exclusively belongs to the read(2) syscall. This is handy when
3016 * performing MSG_PEEK.
3017 */
3018 socantrcvmore(so);
3019
3020 error = SOCK_IO_RECV_LOCK(so, SBL_WAIT | SBL_NOINTR);
3021 if (error != 0) {
3022 KASSERT(SOLISTENING(so),
3023 ("%s: soiolock(%p) failed", __func__, so));
3024 return;
3025 }
3026
3027 sbrelease(so, SO_RCV);
3028 SOCK_IO_RECV_UNLOCK(so);
3029
3030 }
3031
3032 #ifdef SOCKET_HHOOK
3033 /*
3034 * Wrapper for Socket established helper hook.
3035 * Parameters: socket, context of the hook point, hook id.
3036 */
3037 static inline int
3038 hhook_run_socket(struct socket *so, void *hctx, int32_t h_id)
3039 {
3040 struct socket_hhook_data hhook_data = {
3041 .so = so,
3042 .hctx = hctx,
3043 .m = NULL,
3044 .status = 0
3045 };
3046
3047 CURVNET_SET(so->so_vnet);
3048 HHOOKS_RUN_IF(V_socket_hhh[h_id], &hhook_data, &so->osd);
3049 CURVNET_RESTORE();
3050
3051 /* Ugly but needed, since hhooks return void for now */
3052 return (hhook_data.status);
3053 }
3054 #endif
3055
3056 /*
3057 * Perhaps this routine, and sooptcopyout(), below, ought to come in an
3058 * additional variant to handle the case where the option value needs to be
3059 * some kind of integer, but not a specific size. In addition to their use
3060 * here, these functions are also called by the protocol-level pr_ctloutput()
3061 * routines.
3062 */
3063 int
3064 sooptcopyin(struct sockopt *sopt, void *buf, size_t len, size_t minlen)
3065 {
3066 size_t valsize;
3067
3068 /*
3069 * If the user gives us more than we wanted, we ignore it, but if we
3070 * don't get the minimum length the caller wants, we return EINVAL.
3071 * On success, sopt->sopt_valsize is set to however much we actually
3072 * retrieved.
3073 */
3074 if ((valsize = sopt->sopt_valsize) < minlen)
3075 return EINVAL;
3076 if (valsize > len)
3077 sopt->sopt_valsize = valsize = len;
3078
3079 if (sopt->sopt_td != NULL)
3080 return (copyin(sopt->sopt_val, buf, valsize));
3081
3082 bcopy(sopt->sopt_val, buf, valsize);
3083 return (0);
3084 }
3085
3086 /*
3087 * Kernel version of setsockopt(2).
3088 *
3089 * XXX: optlen is size_t, not socklen_t
3090 */
3091 int
3092 so_setsockopt(struct socket *so, int level, int optname, void *optval,
3093 size_t optlen)
3094 {
3095 struct sockopt sopt;
3096
3097 sopt.sopt_level = level;
3098 sopt.sopt_name = optname;
3099 sopt.sopt_dir = SOPT_SET;
3100 sopt.sopt_val = optval;
3101 sopt.sopt_valsize = optlen;
3102 sopt.sopt_td = NULL;
3103 return (sosetopt(so, &sopt));
3104 }
3105
3106 int
3107 sosetopt(struct socket *so, struct sockopt *sopt)
3108 {
3109 int error, optval;
3110 struct linger l;
3111 struct timeval tv;
3112 sbintime_t val, *valp;
3113 uint32_t val32;
3114 #ifdef MAC
3115 struct mac extmac;
3116 #endif
3117
3118 CURVNET_SET(so->so_vnet);
3119 error = 0;
3120 if (sopt->sopt_level != SOL_SOCKET) {
3121 if (so->so_proto->pr_ctloutput != NULL)
3122 error = (*so->so_proto->pr_ctloutput)(so, sopt);
3123 else
3124 error = ENOPROTOOPT;
3125 } else {
3126 switch (sopt->sopt_name) {
3127 case SO_ACCEPTFILTER:
3128 error = accept_filt_setopt(so, sopt);
3129 if (error)
3130 goto bad;
3131 break;
3132
3133 case SO_LINGER:
3134 error = sooptcopyin(sopt, &l, sizeof l, sizeof l);
3135 if (error)
3136 goto bad;
3137 if (l.l_linger < 0 ||
3138 l.l_linger > USHRT_MAX ||
3139 l.l_linger > (INT_MAX / hz)) {
3140 error = EDOM;
3141 goto bad;
3142 }
3143 SOCK_LOCK(so);
3144 so->so_linger = l.l_linger;
3145 if (l.l_onoff)
3146 so->so_options |= SO_LINGER;
3147 else
3148 so->so_options &= ~SO_LINGER;
3149 SOCK_UNLOCK(so);
3150 break;
3151
3152 case SO_DEBUG:
3153 case SO_KEEPALIVE:
3154 case SO_DONTROUTE:
3155 case SO_USELOOPBACK:
3156 case SO_BROADCAST:
3157 case SO_REUSEADDR:
3158 case SO_REUSEPORT:
3159 case SO_REUSEPORT_LB:
3160 case SO_OOBINLINE:
3161 case SO_TIMESTAMP:
3162 case SO_BINTIME:
3163 case SO_NOSIGPIPE:
3164 case SO_NO_DDP:
3165 case SO_NO_OFFLOAD:
3166 case SO_RERROR:
3167 error = sooptcopyin(sopt, &optval, sizeof optval,
3168 sizeof optval);
3169 if (error)
3170 goto bad;
3171 SOCK_LOCK(so);
3172 if (optval)
3173 so->so_options |= sopt->sopt_name;
3174 else
3175 so->so_options &= ~sopt->sopt_name;
3176 SOCK_UNLOCK(so);
3177 break;
3178
3179 case SO_SETFIB:
3180 error = sooptcopyin(sopt, &optval, sizeof optval,
3181 sizeof optval);
3182 if (error)
3183 goto bad;
3184
3185 if (optval < 0 || optval >= rt_numfibs) {
3186 error = EINVAL;
3187 goto bad;
3188 }
3189 if (((so->so_proto->pr_domain->dom_family == PF_INET) ||
3190 (so->so_proto->pr_domain->dom_family == PF_INET6) ||
3191 (so->so_proto->pr_domain->dom_family == PF_ROUTE)))
3192 so->so_fibnum = optval;
3193 else
3194 so->so_fibnum = 0;
3195 break;
3196
3197 case SO_USER_COOKIE:
3198 error = sooptcopyin(sopt, &val32, sizeof val32,
3199 sizeof val32);
3200 if (error)
3201 goto bad;
3202 so->so_user_cookie = val32;
3203 break;
3204
3205 case SO_SNDBUF:
3206 case SO_RCVBUF:
3207 case SO_SNDLOWAT:
3208 case SO_RCVLOWAT:
3209 error = so->so_proto->pr_setsbopt(so, sopt);
3210 if (error)
3211 goto bad;
3212 break;
3213
3214 case SO_SNDTIMEO:
3215 case SO_RCVTIMEO:
3216 #ifdef COMPAT_FREEBSD32
3217 if (SV_CURPROC_FLAG(SV_ILP32)) {
3218 struct timeval32 tv32;
3219
3220 error = sooptcopyin(sopt, &tv32, sizeof tv32,
3221 sizeof tv32);
3222 CP(tv32, tv, tv_sec);
3223 CP(tv32, tv, tv_usec);
3224 } else
3225 #endif
3226 error = sooptcopyin(sopt, &tv, sizeof tv,
3227 sizeof tv);
3228 if (error)
3229 goto bad;
3230 if (tv.tv_sec < 0 || tv.tv_usec < 0 ||
3231 tv.tv_usec >= 1000000) {
3232 error = EDOM;
3233 goto bad;
3234 }
3235 if (tv.tv_sec > INT32_MAX)
3236 val = SBT_MAX;
3237 else
3238 val = tvtosbt(tv);
3239 SOCK_LOCK(so);
3240 valp = sopt->sopt_name == SO_SNDTIMEO ?
3241 (SOLISTENING(so) ? &so->sol_sbsnd_timeo :
3242 &so->so_snd.sb_timeo) :
3243 (SOLISTENING(so) ? &so->sol_sbrcv_timeo :
3244 &so->so_rcv.sb_timeo);
3245 *valp = val;
3246 SOCK_UNLOCK(so);
3247 break;
3248
3249 case SO_LABEL:
3250 #ifdef MAC
3251 error = sooptcopyin(sopt, &extmac, sizeof extmac,
3252 sizeof extmac);
3253 if (error)
3254 goto bad;
3255 error = mac_setsockopt_label(sopt->sopt_td->td_ucred,
3256 so, &extmac);
3257 #else
3258 error = EOPNOTSUPP;
3259 #endif
3260 break;
3261
3262 case SO_TS_CLOCK:
3263 error = sooptcopyin(sopt, &optval, sizeof optval,
3264 sizeof optval);
3265 if (error)
3266 goto bad;
3267 if (optval < 0 || optval > SO_TS_CLOCK_MAX) {
3268 error = EINVAL;
3269 goto bad;
3270 }
3271 so->so_ts_clock = optval;
3272 break;
3273
3274 case SO_MAX_PACING_RATE:
3275 error = sooptcopyin(sopt, &val32, sizeof(val32),
3276 sizeof(val32));
3277 if (error)
3278 goto bad;
3279 so->so_max_pacing_rate = val32;
3280 break;
3281
3282 default:
3283 #ifdef SOCKET_HHOOK
3284 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0)
3285 error = hhook_run_socket(so, sopt,
3286 HHOOK_SOCKET_OPT);
3287 else
3288 #endif
3289 error = ENOPROTOOPT;
3290 break;
3291 }
3292 if (error == 0 && so->so_proto->pr_ctloutput != NULL)
3293 (void)(*so->so_proto->pr_ctloutput)(so, sopt);
3294 }
3295 bad:
3296 CURVNET_RESTORE();
3297 return (error);
3298 }
3299
3300 /*
3301 * Helper routine for getsockopt.
3302 */
3303 int
3304 sooptcopyout(struct sockopt *sopt, const void *buf, size_t len)
3305 {
3306 int error;
3307 size_t valsize;
3308
3309 error = 0;
3310
3311 /*
3312 * Documented get behavior is that we always return a value, possibly
3313 * truncated to fit in the user's buffer. Traditional behavior is
3314 * that we always tell the user precisely how much we copied, rather
3315 * than something useful like the total amount we had available for
3316 * her. Note that this interface is not idempotent; the entire
3317 * answer must be generated ahead of time.
3318 */
3319 valsize = min(len, sopt->sopt_valsize);
3320 sopt->sopt_valsize = valsize;
3321 if (sopt->sopt_val != NULL) {
3322 if (sopt->sopt_td != NULL)
3323 error = copyout(buf, sopt->sopt_val, valsize);
3324 else
3325 bcopy(buf, sopt->sopt_val, valsize);
3326 }
3327 return (error);
3328 }
3329
3330 int
3331 sogetopt(struct socket *so, struct sockopt *sopt)
3332 {
3333 int error, optval;
3334 struct linger l;
3335 struct timeval tv;
3336 #ifdef MAC
3337 struct mac extmac;
3338 #endif
3339
3340 CURVNET_SET(so->so_vnet);
3341 error = 0;
3342 if (sopt->sopt_level != SOL_SOCKET) {
3343 if (so->so_proto->pr_ctloutput != NULL)
3344 error = (*so->so_proto->pr_ctloutput)(so, sopt);
3345 else
3346 error = ENOPROTOOPT;
3347 CURVNET_RESTORE();
3348 return (error);
3349 } else {
3350 switch (sopt->sopt_name) {
3351 case SO_ACCEPTFILTER:
3352 error = accept_filt_getopt(so, sopt);
3353 break;
3354
3355 case SO_LINGER:
3356 SOCK_LOCK(so);
3357 l.l_onoff = so->so_options & SO_LINGER;
3358 l.l_linger = so->so_linger;
3359 SOCK_UNLOCK(so);
3360 error = sooptcopyout(sopt, &l, sizeof l);
3361 break;
3362
3363 case SO_USELOOPBACK:
3364 case SO_DONTROUTE:
3365 case SO_DEBUG:
3366 case SO_KEEPALIVE:
3367 case SO_REUSEADDR:
3368 case SO_REUSEPORT:
3369 case SO_REUSEPORT_LB:
3370 case SO_BROADCAST:
3371 case SO_OOBINLINE:
3372 case SO_ACCEPTCONN:
3373 case SO_TIMESTAMP:
3374 case SO_BINTIME:
3375 case SO_NOSIGPIPE:
3376 case SO_NO_DDP:
3377 case SO_NO_OFFLOAD:
3378 case SO_RERROR:
3379 optval = so->so_options & sopt->sopt_name;
3380 integer:
3381 error = sooptcopyout(sopt, &optval, sizeof optval);
3382 break;
3383
3384 case SO_DOMAIN:
3385 optval = so->so_proto->pr_domain->dom_family;
3386 goto integer;
3387
3388 case SO_TYPE:
3389 optval = so->so_type;
3390 goto integer;
3391
3392 case SO_PROTOCOL:
3393 optval = so->so_proto->pr_protocol;
3394 goto integer;
3395
3396 case SO_ERROR:
3397 SOCK_LOCK(so);
3398 if (so->so_error) {
3399 optval = so->so_error;
3400 so->so_error = 0;
3401 } else {
3402 optval = so->so_rerror;
3403 so->so_rerror = 0;
3404 }
3405 SOCK_UNLOCK(so);
3406 goto integer;
3407
3408 case SO_SNDBUF:
3409 optval = SOLISTENING(so) ? so->sol_sbsnd_hiwat :
3410 so->so_snd.sb_hiwat;
3411 goto integer;
3412
3413 case SO_RCVBUF:
3414 optval = SOLISTENING(so) ? so->sol_sbrcv_hiwat :
3415 so->so_rcv.sb_hiwat;
3416 goto integer;
3417
3418 case SO_SNDLOWAT:
3419 optval = SOLISTENING(so) ? so->sol_sbsnd_lowat :
3420 so->so_snd.sb_lowat;
3421 goto integer;
3422
3423 case SO_RCVLOWAT:
3424 optval = SOLISTENING(so) ? so->sol_sbrcv_lowat :
3425 so->so_rcv.sb_lowat;
3426 goto integer;
3427
3428 case SO_SNDTIMEO:
3429 case SO_RCVTIMEO:
3430 SOCK_LOCK(so);
3431 tv = sbttotv(sopt->sopt_name == SO_SNDTIMEO ?
3432 (SOLISTENING(so) ? so->sol_sbsnd_timeo :
3433 so->so_snd.sb_timeo) :
3434 (SOLISTENING(so) ? so->sol_sbrcv_timeo :
3435 so->so_rcv.sb_timeo));
3436 SOCK_UNLOCK(so);
3437 #ifdef COMPAT_FREEBSD32
3438 if (SV_CURPROC_FLAG(SV_ILP32)) {
3439 struct timeval32 tv32;
3440
3441 CP(tv, tv32, tv_sec);
3442 CP(tv, tv32, tv_usec);
3443 error = sooptcopyout(sopt, &tv32, sizeof tv32);
3444 } else
3445 #endif
3446 error = sooptcopyout(sopt, &tv, sizeof tv);
3447 break;
3448
3449 case SO_LABEL:
3450 #ifdef MAC
3451 error = sooptcopyin(sopt, &extmac, sizeof(extmac),
3452 sizeof(extmac));
3453 if (error)
3454 goto bad;
3455 error = mac_getsockopt_label(sopt->sopt_td->td_ucred,
3456 so, &extmac);
3457 if (error)
3458 goto bad;
3459 /* Don't copy out extmac, it is unchanged. */
3460 #else
3461 error = EOPNOTSUPP;
3462 #endif
3463 break;
3464
3465 case SO_PEERLABEL:
3466 #ifdef MAC
3467 error = sooptcopyin(sopt, &extmac, sizeof(extmac),
3468 sizeof(extmac));
3469 if (error)
3470 goto bad;
3471 error = mac_getsockopt_peerlabel(
3472 sopt->sopt_td->td_ucred, so, &extmac);
3473 if (error)
3474 goto bad;
3475 /* Don't copy out extmac, it is unchanged. */
3476 #else
3477 error = EOPNOTSUPP;
3478 #endif
3479 break;
3480
3481 case SO_LISTENQLIMIT:
3482 optval = SOLISTENING(so) ? so->sol_qlimit : 0;
3483 goto integer;
3484
3485 case SO_LISTENQLEN:
3486 optval = SOLISTENING(so) ? so->sol_qlen : 0;
3487 goto integer;
3488
3489 case SO_LISTENINCQLEN:
3490 optval = SOLISTENING(so) ? so->sol_incqlen : 0;
3491 goto integer;
3492
3493 case SO_TS_CLOCK:
3494 optval = so->so_ts_clock;
3495 goto integer;
3496
3497 case SO_MAX_PACING_RATE:
3498 optval = so->so_max_pacing_rate;
3499 goto integer;
3500
3501 default:
3502 #ifdef SOCKET_HHOOK
3503 if (V_socket_hhh[HHOOK_SOCKET_OPT]->hhh_nhooks > 0)
3504 error = hhook_run_socket(so, sopt,
3505 HHOOK_SOCKET_OPT);
3506 else
3507 #endif
3508 error = ENOPROTOOPT;
3509 break;
3510 }
3511 }
3512 #ifdef MAC
3513 bad:
3514 #endif
3515 CURVNET_RESTORE();
3516 return (error);
3517 }
3518
3519 int
3520 soopt_getm(struct sockopt *sopt, struct mbuf **mp)
3521 {
3522 struct mbuf *m, *m_prev;
3523 int sopt_size = sopt->sopt_valsize;
3524
3525 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA);
3526 if (m == NULL)
3527 return ENOBUFS;
3528 if (sopt_size > MLEN) {
3529 MCLGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT);
3530 if ((m->m_flags & M_EXT) == 0) {
3531 m_free(m);
3532 return ENOBUFS;
3533 }
3534 m->m_len = min(MCLBYTES, sopt_size);
3535 } else {
3536 m->m_len = min(MLEN, sopt_size);
3537 }
3538 sopt_size -= m->m_len;
3539 *mp = m;
3540 m_prev = m;
3541
3542 while (sopt_size) {
3543 MGET(m, sopt->sopt_td ? M_WAITOK : M_NOWAIT, MT_DATA);
3544 if (m == NULL) {
3545 m_freem(*mp);
3546 return ENOBUFS;
3547 }
3548 if (sopt_size > MLEN) {
3549 MCLGET(m, sopt->sopt_td != NULL ? M_WAITOK :
3550 M_NOWAIT);
3551 if ((m->m_flags & M_EXT) == 0) {
3552 m_freem(m);
3553 m_freem(*mp);
3554 return ENOBUFS;
3555 }
3556 m->m_len = min(MCLBYTES, sopt_size);
3557 } else {
3558 m->m_len = min(MLEN, sopt_size);
3559 }
3560 sopt_size -= m->m_len;
3561 m_prev->m_next = m;
3562 m_prev = m;
3563 }
3564 return (0);
3565 }
3566
3567 int
3568 soopt_mcopyin(struct sockopt *sopt, struct mbuf *m)
3569 {
3570 struct mbuf *m0 = m;
3571
3572 if (sopt->sopt_val == NULL)
3573 return (0);
3574 while (m != NULL && sopt->sopt_valsize >= m->m_len) {
3575 if (sopt->sopt_td != NULL) {
3576 int error;
3577
3578 error = copyin(sopt->sopt_val, mtod(m, char *),
3579 m->m_len);
3580 if (error != 0) {
3581 m_freem(m0);
3582 return(error);
3583 }
3584 } else
3585 bcopy(sopt->sopt_val, mtod(m, char *), m->m_len);
3586 sopt->sopt_valsize -= m->m_len;
3587 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
3588 m = m->m_next;
3589 }
3590 if (m != NULL) /* should be allocated enoughly at ip6_sooptmcopyin() */
3591 panic("ip6_sooptmcopyin");
3592 return (0);
3593 }
3594
3595 int
3596 soopt_mcopyout(struct sockopt *sopt, struct mbuf *m)
3597 {
3598 struct mbuf *m0 = m;
3599 size_t valsize = 0;
3600
3601 if (sopt->sopt_val == NULL)
3602 return (0);
3603 while (m != NULL && sopt->sopt_valsize >= m->m_len) {
3604 if (sopt->sopt_td != NULL) {
3605 int error;
3606
3607 error = copyout(mtod(m, char *), sopt->sopt_val,
3608 m->m_len);
3609 if (error != 0) {
3610 m_freem(m0);
3611 return(error);
3612 }
3613 } else
3614 bcopy(mtod(m, char *), sopt->sopt_val, m->m_len);
3615 sopt->sopt_valsize -= m->m_len;
3616 sopt->sopt_val = (char *)sopt->sopt_val + m->m_len;
3617 valsize += m->m_len;
3618 m = m->m_next;
3619 }
3620 if (m != NULL) {
3621 /* enough soopt buffer should be given from user-land */
3622 m_freem(m0);
3623 return(EINVAL);
3624 }
3625 sopt->sopt_valsize = valsize;
3626 return (0);
3627 }
3628
3629 /*
3630 * sohasoutofband(): protocol notifies socket layer of the arrival of new
3631 * out-of-band data, which will then notify socket consumers.
3632 */
3633 void
3634 sohasoutofband(struct socket *so)
3635 {
3636
3637 if (so->so_sigio != NULL)
3638 pgsigio(&so->so_sigio, SIGURG, 0);
3639 selwakeuppri(&so->so_rdsel, PSOCK);
3640 }
3641
3642 int
3643 sopoll(struct socket *so, int events, struct ucred *active_cred,
3644 struct thread *td)
3645 {
3646
3647 /*
3648 * We do not need to set or assert curvnet as long as everyone uses
3649 * sopoll_generic().
3650 */
3651 return (so->so_proto->pr_sopoll(so, events, active_cred, td));
3652 }
3653
3654 int
3655 sopoll_generic(struct socket *so, int events, struct ucred *active_cred,
3656 struct thread *td)
3657 {
3658 int revents;
3659
3660 SOCK_LOCK(so);
3661 if (SOLISTENING(so)) {
3662 if (!(events & (POLLIN | POLLRDNORM)))
3663 revents = 0;
3664 else if (!TAILQ_EMPTY(&so->sol_comp))
3665 revents = events & (POLLIN | POLLRDNORM);
3666 else if ((events & POLLINIGNEOF) == 0 && so->so_error)
3667 revents = (events & (POLLIN | POLLRDNORM)) | POLLHUP;
3668 else {
3669 selrecord(td, &so->so_rdsel);
3670 revents = 0;
3671 }
3672 } else {
3673 revents = 0;
3674 SOCK_SENDBUF_LOCK(so);
3675 SOCK_RECVBUF_LOCK(so);
3676 if (events & (POLLIN | POLLRDNORM))
3677 if (soreadabledata(so))
3678 revents |= events & (POLLIN | POLLRDNORM);
3679 if (events & (POLLOUT | POLLWRNORM))
3680 if (sowriteable(so))
3681 revents |= events & (POLLOUT | POLLWRNORM);
3682 if (events & (POLLPRI | POLLRDBAND))
3683 if (so->so_oobmark ||
3684 (so->so_rcv.sb_state & SBS_RCVATMARK))
3685 revents |= events & (POLLPRI | POLLRDBAND);
3686 if ((events & POLLINIGNEOF) == 0) {
3687 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
3688 revents |= events & (POLLIN | POLLRDNORM);
3689 if (so->so_snd.sb_state & SBS_CANTSENDMORE)
3690 revents |= POLLHUP;
3691 }
3692 }
3693 if (so->so_rcv.sb_state & SBS_CANTRCVMORE)
3694 revents |= events & POLLRDHUP;
3695 if (revents == 0) {
3696 if (events &
3697 (POLLIN | POLLPRI | POLLRDNORM | POLLRDBAND | POLLRDHUP)) {
3698 selrecord(td, &so->so_rdsel);
3699 so->so_rcv.sb_flags |= SB_SEL;
3700 }
3701 if (events & (POLLOUT | POLLWRNORM)) {
3702 selrecord(td, &so->so_wrsel);
3703 so->so_snd.sb_flags |= SB_SEL;
3704 }
3705 }
3706 SOCK_RECVBUF_UNLOCK(so);
3707 SOCK_SENDBUF_UNLOCK(so);
3708 }
3709 SOCK_UNLOCK(so);
3710 return (revents);
3711 }
3712
3713 int
3714 soo_kqfilter(struct file *fp, struct knote *kn)
3715 {
3716 struct socket *so = kn->kn_fp->f_data;
3717 struct sockbuf *sb;
3718 sb_which which;
3719 struct knlist *knl;
3720
3721 switch (kn->kn_filter) {
3722 case EVFILT_READ:
3723 kn->kn_fop = &soread_filtops;
3724 knl = &so->so_rdsel.si_note;
3725 sb = &so->so_rcv;
3726 which = SO_RCV;
3727 break;
3728 case EVFILT_WRITE:
3729 kn->kn_fop = &sowrite_filtops;
3730 knl = &so->so_wrsel.si_note;
3731 sb = &so->so_snd;
3732 which = SO_SND;
3733 break;
3734 case EVFILT_EMPTY:
3735 kn->kn_fop = &soempty_filtops;
3736 knl = &so->so_wrsel.si_note;
3737 sb = &so->so_snd;
3738 which = SO_SND;
3739 break;
3740 default:
3741 return (EINVAL);
3742 }
3743
3744 SOCK_LOCK(so);
3745 if (SOLISTENING(so)) {
3746 knlist_add(knl, kn, 1);
3747 } else {
3748 SOCK_BUF_LOCK(so, which);
3749 knlist_add(knl, kn, 1);
3750 sb->sb_flags |= SB_KNOTE;
3751 SOCK_BUF_UNLOCK(so, which);
3752 }
3753 SOCK_UNLOCK(so);
3754 return (0);
3755 }
3756
3757 static void
3758 filt_sordetach(struct knote *kn)
3759 {
3760 struct socket *so = kn->kn_fp->f_data;
3761
3762 so_rdknl_lock(so);
3763 knlist_remove(&so->so_rdsel.si_note, kn, 1);
3764 if (!SOLISTENING(so) && knlist_empty(&so->so_rdsel.si_note))
3765 so->so_rcv.sb_flags &= ~SB_KNOTE;
3766 so_rdknl_unlock(so);
3767 }
3768
3769 /*ARGSUSED*/
3770 static int
3771 filt_soread(struct knote *kn, long hint)
3772 {
3773 struct socket *so;
3774
3775 so = kn->kn_fp->f_data;
3776
3777 if (SOLISTENING(so)) {
3778 SOCK_LOCK_ASSERT(so);
3779 kn->kn_data = so->sol_qlen;
3780 if (so->so_error) {
3781 kn->kn_flags |= EV_EOF;
3782 kn->kn_fflags = so->so_error;
3783 return (1);
3784 }
3785 return (!TAILQ_EMPTY(&so->sol_comp));
3786 }
3787
3788 SOCK_RECVBUF_LOCK_ASSERT(so);
3789
3790 kn->kn_data = sbavail(&so->so_rcv) - so->so_rcv.sb_ctl;
3791 if (so->so_rcv.sb_state & SBS_CANTRCVMORE) {
3792 kn->kn_flags |= EV_EOF;
3793 kn->kn_fflags = so->so_error;
3794 return (1);
3795 } else if (so->so_error || so->so_rerror)
3796 return (1);
3797
3798 if (kn->kn_sfflags & NOTE_LOWAT) {
3799 if (kn->kn_data >= kn->kn_sdata)
3800 return (1);
3801 } else if (sbavail(&so->so_rcv) >= so->so_rcv.sb_lowat)
3802 return (1);
3803
3804 #ifdef SOCKET_HHOOK
3805 /* This hook returning non-zero indicates an event, not error */
3806 return (hhook_run_socket(so, NULL, HHOOK_FILT_SOREAD));
3807 #else
3808 return (0);
3809 #endif
3810 }
3811
3812 static void
3813 filt_sowdetach(struct knote *kn)
3814 {
3815 struct socket *so = kn->kn_fp->f_data;
3816
3817 so_wrknl_lock(so);
3818 knlist_remove(&so->so_wrsel.si_note, kn, 1);
3819 if (!SOLISTENING(so) && knlist_empty(&so->so_wrsel.si_note))
3820 so->so_snd.sb_flags &= ~SB_KNOTE;
3821 so_wrknl_unlock(so);
3822 }
3823
3824 /*ARGSUSED*/
3825 static int
3826 filt_sowrite(struct knote *kn, long hint)
3827 {
3828 struct socket *so;
3829
3830 so = kn->kn_fp->f_data;
3831
3832 if (SOLISTENING(so))
3833 return (0);
3834
3835 SOCK_SENDBUF_LOCK_ASSERT(so);
3836 kn->kn_data = sbspace(&so->so_snd);
3837
3838 #ifdef SOCKET_HHOOK
3839 hhook_run_socket(so, kn, HHOOK_FILT_SOWRITE);
3840 #endif
3841
3842 if (so->so_snd.sb_state & SBS_CANTSENDMORE) {
3843 kn->kn_flags |= EV_EOF;
3844 kn->kn_fflags = so->so_error;
3845 return (1);
3846 } else if (so->so_error) /* temporary udp error */
3847 return (1);
3848 else if (((so->so_state & SS_ISCONNECTED) == 0) &&
3849 (so->so_proto->pr_flags & PR_CONNREQUIRED))
3850 return (0);
3851 else if (kn->kn_sfflags & NOTE_LOWAT)
3852 return (kn->kn_data >= kn->kn_sdata);
3853 else
3854 return (kn->kn_data >= so->so_snd.sb_lowat);
3855 }
3856
3857 static int
3858 filt_soempty(struct knote *kn, long hint)
3859 {
3860 struct socket *so;
3861
3862 so = kn->kn_fp->f_data;
3863
3864 if (SOLISTENING(so))
3865 return (1);
3866
3867 SOCK_SENDBUF_LOCK_ASSERT(so);
3868 kn->kn_data = sbused(&so->so_snd);
3869
3870 if (kn->kn_data == 0)
3871 return (1);
3872 else
3873 return (0);
3874 }
3875
3876 int
3877 socheckuid(struct socket *so, uid_t uid)
3878 {
3879
3880 if (so == NULL)
3881 return (EPERM);
3882 if (so->so_cred->cr_uid != uid)
3883 return (EPERM);
3884 return (0);
3885 }
3886
3887 /*
3888 * These functions are used by protocols to notify the socket layer (and its
3889 * consumers) of state changes in the sockets driven by protocol-side events.
3890 */
3891
3892 /*
3893 * Procedures to manipulate state flags of socket and do appropriate wakeups.
3894 *
3895 * Normal sequence from the active (originating) side is that
3896 * soisconnecting() is called during processing of connect() call, resulting
3897 * in an eventual call to soisconnected() if/when the connection is
3898 * established. When the connection is torn down soisdisconnecting() is
3899 * called during processing of disconnect() call, and soisdisconnected() is
3900 * called when the connection to the peer is totally severed. The semantics
3901 * of these routines are such that connectionless protocols can call
3902 * soisconnected() and soisdisconnected() only, bypassing the in-progress
3903 * calls when setting up a ``connection'' takes no time.
3904 *
3905 * From the passive side, a socket is created with two queues of sockets:
3906 * so_incomp for connections in progress and so_comp for connections already
3907 * made and awaiting user acceptance. As a protocol is preparing incoming
3908 * connections, it creates a socket structure queued on so_incomp by calling
3909 * sonewconn(). When the connection is established, soisconnected() is
3910 * called, and transfers the socket structure to so_comp, making it available
3911 * to accept().
3912 *
3913 * If a socket is closed with sockets on either so_incomp or so_comp, these
3914 * sockets are dropped.
3915 *
3916 * If higher-level protocols are implemented in the kernel, the wakeups done
3917 * here will sometimes cause software-interrupt process scheduling.
3918 */
3919 void
3920 soisconnecting(struct socket *so)
3921 {
3922
3923 SOCK_LOCK(so);
3924 so->so_state &= ~(SS_ISCONNECTED|SS_ISDISCONNECTING);
3925 so->so_state |= SS_ISCONNECTING;
3926 SOCK_UNLOCK(so);
3927 }
3928
3929 void
3930 soisconnected(struct socket *so)
3931 {
3932 bool last __diagused;
3933
3934 SOCK_LOCK(so);
3935 so->so_state &= ~(SS_ISCONNECTING|SS_ISDISCONNECTING);
3936 so->so_state |= SS_ISCONNECTED;
3937
3938 if (so->so_qstate == SQ_INCOMP) {
3939 struct socket *head = so->so_listen;
3940 int ret;
3941
3942 KASSERT(head, ("%s: so %p on incomp of NULL", __func__, so));
3943 /*
3944 * Promoting a socket from incomplete queue to complete, we
3945 * need to go through reverse order of locking. We first do
3946 * trylock, and if that doesn't succeed, we go the hard way
3947 * leaving a reference and rechecking consistency after proper
3948 * locking.
3949 */
3950 if (__predict_false(SOLISTEN_TRYLOCK(head) == 0)) {
3951 soref(head);
3952 SOCK_UNLOCK(so);
3953 SOLISTEN_LOCK(head);
3954 SOCK_LOCK(so);
3955 if (__predict_false(head != so->so_listen)) {
3956 /*
3957 * The socket went off the listen queue,
3958 * should be lost race to close(2) of sol.
3959 * The socket is about to soabort().
3960 */
3961 SOCK_UNLOCK(so);
3962 sorele_locked(head);
3963 return;
3964 }
3965 last = refcount_release(&head->so_count);
3966 KASSERT(!last, ("%s: released last reference for %p",
3967 __func__, head));
3968 }
3969 again:
3970 if ((so->so_options & SO_ACCEPTFILTER) == 0) {
3971 TAILQ_REMOVE(&head->sol_incomp, so, so_list);
3972 head->sol_incqlen--;
3973 TAILQ_INSERT_TAIL(&head->sol_comp, so, so_list);
3974 head->sol_qlen++;
3975 so->so_qstate = SQ_COMP;
3976 SOCK_UNLOCK(so);
3977 solisten_wakeup(head); /* unlocks */
3978 } else {
3979 SOCK_RECVBUF_LOCK(so);
3980 soupcall_set(so, SO_RCV,
3981 head->sol_accept_filter->accf_callback,
3982 head->sol_accept_filter_arg);
3983 so->so_options &= ~SO_ACCEPTFILTER;
3984 ret = head->sol_accept_filter->accf_callback(so,
3985 head->sol_accept_filter_arg, M_NOWAIT);
3986 if (ret == SU_ISCONNECTED) {
3987 soupcall_clear(so, SO_RCV);
3988 SOCK_RECVBUF_UNLOCK(so);
3989 goto again;
3990 }
3991 SOCK_RECVBUF_UNLOCK(so);
3992 SOCK_UNLOCK(so);
3993 SOLISTEN_UNLOCK(head);
3994 }
3995 return;
3996 }
3997 SOCK_UNLOCK(so);
3998 wakeup(&so->so_timeo);
3999 sorwakeup(so);
4000 sowwakeup(so);
4001 }
4002
4003 void
4004 soisdisconnecting(struct socket *so)
4005 {
4006
4007 SOCK_LOCK(so);
4008 so->so_state &= ~SS_ISCONNECTING;
4009 so->so_state |= SS_ISDISCONNECTING;
4010
4011 if (!SOLISTENING(so)) {
4012 SOCK_RECVBUF_LOCK(so);
4013 socantrcvmore_locked(so);
4014 SOCK_SENDBUF_LOCK(so);
4015 socantsendmore_locked(so);
4016 }
4017 SOCK_UNLOCK(so);
4018 wakeup(&so->so_timeo);
4019 }
4020
4021 void
4022 soisdisconnected(struct socket *so)
4023 {
4024
4025 SOCK_LOCK(so);
4026
4027 /*
4028 * There is at least one reader of so_state that does not
4029 * acquire socket lock, namely soreceive_generic(). Ensure
4030 * that it never sees all flags that track connection status
4031 * cleared, by ordering the update with a barrier semantic of
4032 * our release thread fence.
4033 */
4034 so->so_state |= SS_ISDISCONNECTED;
4035 atomic_thread_fence_rel();
4036 so->so_state &= ~(SS_ISCONNECTING|SS_ISCONNECTED|SS_ISDISCONNECTING);
4037
4038 if (!SOLISTENING(so)) {
4039 SOCK_UNLOCK(so);
4040 SOCK_RECVBUF_LOCK(so);
4041 socantrcvmore_locked(so);
4042 SOCK_SENDBUF_LOCK(so);
4043 sbdrop_locked(&so->so_snd, sbused(&so->so_snd));
4044 socantsendmore_locked(so);
4045 } else
4046 SOCK_UNLOCK(so);
4047 wakeup(&so->so_timeo);
4048 }
4049
4050 int
4051 soiolock(struct socket *so, struct sx *sx, int flags)
4052 {
4053 int error;
4054
4055 KASSERT((flags & SBL_VALID) == flags,
4056 ("soiolock: invalid flags %#x", flags));
4057
4058 if ((flags & SBL_WAIT) != 0) {
4059 if ((flags & SBL_NOINTR) != 0) {
4060 sx_xlock(sx);
4061 } else {
4062 error = sx_xlock_sig(sx);
4063 if (error != 0)
4064 return (error);
4065 }
4066 } else if (!sx_try_xlock(sx)) {
4067 return (EWOULDBLOCK);
4068 }
4069
4070 if (__predict_false(SOLISTENING(so))) {
4071 sx_xunlock(sx);
4072 return (ENOTCONN);
4073 }
4074 return (0);
4075 }
4076
4077 void
4078 soiounlock(struct sx *sx)
4079 {
4080 sx_xunlock(sx);
4081 }
4082
4083 /*
4084 * Make a copy of a sockaddr in a malloced buffer of type M_SONAME.
4085 */
4086 struct sockaddr *
4087 sodupsockaddr(const struct sockaddr *sa, int mflags)
4088 {
4089 struct sockaddr *sa2;
4090
4091 sa2 = malloc(sa->sa_len, M_SONAME, mflags);
4092 if (sa2)
4093 bcopy(sa, sa2, sa->sa_len);
4094 return sa2;
4095 }
4096
4097 /*
4098 * Register per-socket destructor.
4099 */
4100 void
4101 sodtor_set(struct socket *so, so_dtor_t *func)
4102 {
4103
4104 SOCK_LOCK_ASSERT(so);
4105 so->so_dtor = func;
4106 }
4107
4108 /*
4109 * Register per-socket buffer upcalls.
4110 */
4111 void
4112 soupcall_set(struct socket *so, sb_which which, so_upcall_t func, void *arg)
4113 {
4114 struct sockbuf *sb;
4115
4116 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4117
4118 switch (which) {
4119 case SO_RCV:
4120 sb = &so->so_rcv;
4121 break;
4122 case SO_SND:
4123 sb = &so->so_snd;
4124 break;
4125 }
4126 SOCK_BUF_LOCK_ASSERT(so, which);
4127 sb->sb_upcall = func;
4128 sb->sb_upcallarg = arg;
4129 sb->sb_flags |= SB_UPCALL;
4130 }
4131
4132 void
4133 soupcall_clear(struct socket *so, sb_which which)
4134 {
4135 struct sockbuf *sb;
4136
4137 KASSERT(!SOLISTENING(so), ("%s: so %p listening", __func__, so));
4138
4139 switch (which) {
4140 case SO_RCV:
4141 sb = &so->so_rcv;
4142 break;
4143 case SO_SND:
4144 sb = &so->so_snd;
4145 break;
4146 }
4147 SOCK_BUF_LOCK_ASSERT(so, which);
4148 KASSERT(sb->sb_upcall != NULL,
4149 ("%s: so %p no upcall to clear", __func__, so));
4150 sb->sb_upcall = NULL;
4151 sb->sb_upcallarg = NULL;
4152 sb->sb_flags &= ~SB_UPCALL;
4153 }
4154
4155 void
4156 solisten_upcall_set(struct socket *so, so_upcall_t func, void *arg)
4157 {
4158
4159 SOLISTEN_LOCK_ASSERT(so);
4160 so->sol_upcall = func;
4161 so->sol_upcallarg = arg;
4162 }
4163
4164 static void
4165 so_rdknl_lock(void *arg)
4166 {
4167 struct socket *so = arg;
4168
4169 retry:
4170 if (SOLISTENING(so)) {
4171 SOLISTEN_LOCK(so);
4172 } else {
4173 SOCK_RECVBUF_LOCK(so);
4174 if (__predict_false(SOLISTENING(so))) {
4175 SOCK_RECVBUF_UNLOCK(so);
4176 goto retry;
4177 }
4178 }
4179 }
4180
4181 static void
4182 so_rdknl_unlock(void *arg)
4183 {
4184 struct socket *so = arg;
4185
4186 if (SOLISTENING(so))
4187 SOLISTEN_UNLOCK(so);
4188 else
4189 SOCK_RECVBUF_UNLOCK(so);
4190 }
4191
4192 static void
4193 so_rdknl_assert_lock(void *arg, int what)
4194 {
4195 struct socket *so = arg;
4196
4197 if (what == LA_LOCKED) {
4198 if (SOLISTENING(so))
4199 SOLISTEN_LOCK_ASSERT(so);
4200 else
4201 SOCK_RECVBUF_LOCK_ASSERT(so);
4202 } else {
4203 if (SOLISTENING(so))
4204 SOLISTEN_UNLOCK_ASSERT(so);
4205 else
4206 SOCK_RECVBUF_UNLOCK_ASSERT(so);
4207 }
4208 }
4209
4210 static void
4211 so_wrknl_lock(void *arg)
4212 {
4213 struct socket *so = arg;
4214
4215 retry:
4216 if (SOLISTENING(so)) {
4217 SOLISTEN_LOCK(so);
4218 } else {
4219 SOCK_SENDBUF_LOCK(so);
4220 if (__predict_false(SOLISTENING(so))) {
4221 SOCK_SENDBUF_UNLOCK(so);
4222 goto retry;
4223 }
4224 }
4225 }
4226
4227 static void
4228 so_wrknl_unlock(void *arg)
4229 {
4230 struct socket *so = arg;
4231
4232 if (SOLISTENING(so))
4233 SOLISTEN_UNLOCK(so);
4234 else
4235 SOCK_SENDBUF_UNLOCK(so);
4236 }
4237
4238 static void
4239 so_wrknl_assert_lock(void *arg, int what)
4240 {
4241 struct socket *so = arg;
4242
4243 if (what == LA_LOCKED) {
4244 if (SOLISTENING(so))
4245 SOLISTEN_LOCK_ASSERT(so);
4246 else
4247 SOCK_SENDBUF_LOCK_ASSERT(so);
4248 } else {
4249 if (SOLISTENING(so))
4250 SOLISTEN_UNLOCK_ASSERT(so);
4251 else
4252 SOCK_SENDBUF_UNLOCK_ASSERT(so);
4253 }
4254 }
4255
4256 /*
4257 * Create an external-format (``xsocket'') structure using the information in
4258 * the kernel-format socket structure pointed to by so. This is done to
4259 * reduce the spew of irrelevant information over this interface, to isolate
4260 * user code from changes in the kernel structure, and potentially to provide
4261 * information-hiding if we decide that some of this information should be
4262 * hidden from users.
4263 */
4264 void
4265 sotoxsocket(struct socket *so, struct xsocket *xso)
4266 {
4267
4268 bzero(xso, sizeof(*xso));
4269 xso->xso_len = sizeof *xso;
4270 xso->xso_so = (uintptr_t)so;
4271 xso->so_type = so->so_type;
4272 xso->so_options = so->so_options;
4273 xso->so_linger = so->so_linger;
4274 xso->so_state = so->so_state;
4275 xso->so_pcb = (uintptr_t)so->so_pcb;
4276 xso->xso_protocol = so->so_proto->pr_protocol;
4277 xso->xso_family = so->so_proto->pr_domain->dom_family;
4278 xso->so_timeo = so->so_timeo;
4279 xso->so_error = so->so_error;
4280 xso->so_uid = so->so_cred->cr_uid;
4281 xso->so_pgid = so->so_sigio ? so->so_sigio->sio_pgid : 0;
4282 if (SOLISTENING(so)) {
4283 xso->so_qlen = so->sol_qlen;
4284 xso->so_incqlen = so->sol_incqlen;
4285 xso->so_qlimit = so->sol_qlimit;
4286 xso->so_oobmark = 0;
4287 } else {
4288 xso->so_state |= so->so_qstate;
4289 xso->so_qlen = xso->so_incqlen = xso->so_qlimit = 0;
4290 xso->so_oobmark = so->so_oobmark;
4291 sbtoxsockbuf(&so->so_snd, &xso->so_snd);
4292 sbtoxsockbuf(&so->so_rcv, &xso->so_rcv);
4293 }
4294 }
4295
4296 int
4297 so_options_get(const struct socket *so)
4298 {
4299
4300 return (so->so_options);
4301 }
4302
4303 void
4304 so_options_set(struct socket *so, int val)
4305 {
4306
4307 so->so_options = val;
4308 }
4309
4310 int
4311 so_error_get(const struct socket *so)
4312 {
4313
4314 return (so->so_error);
4315 }
4316
4317 void
4318 so_error_set(struct socket *so, int val)
4319 {
4320
4321 so->so_error = val;
4322 }
4323