1 /* $OpenBSD: uipc_socket.c,v 1.336 2024/06/14 08:32:22 mvs Exp $ */
2 /* $NetBSD: uipc_socket.c,v 1.21 1996/02/04 02:17:52 christos Exp $ */
3
4 /*
5 * Copyright (c) 1982, 1986, 1988, 1990, 1993
6 * The Regents of the University of California. All rights reserved.
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. Neither the name of the University nor the names of its contributors
17 * may be used to endorse or promote products derived from this software
18 * without specific prior written permission.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 *
32 * @(#)uipc_socket.c 8.3 (Berkeley) 4/15/94
33 */
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/proc.h>
38 #include <sys/file.h>
39 #include <sys/filedesc.h>
40 #include <sys/malloc.h>
41 #include <sys/mbuf.h>
42 #include <sys/domain.h>
43 #include <sys/event.h>
44 #include <sys/protosw.h>
45 #include <sys/socket.h>
46 #include <sys/unpcb.h>
47 #include <sys/socketvar.h>
48 #include <sys/signalvar.h>
49 #include <sys/pool.h>
50 #include <sys/atomic.h>
51 #include <sys/rwlock.h>
52 #include <sys/time.h>
53 #include <sys/refcnt.h>
54
55 #ifdef DDB
56 #include <machine/db_machdep.h>
57 #endif
58
59 void sbsync(struct sockbuf *, struct mbuf *);
60
61 int sosplice(struct socket *, int, off_t, struct timeval *);
62 void sounsplice(struct socket *, struct socket *, int);
63 void soidle(void *);
64 void sotask(void *);
65 void soreaper(void *);
66 void soput(void *);
67 int somove(struct socket *, int);
68 void sorflush(struct socket *);
69
70 void filt_sordetach(struct knote *kn);
71 int filt_soread(struct knote *kn, long hint);
72 void filt_sowdetach(struct knote *kn);
73 int filt_sowrite(struct knote *kn, long hint);
74 int filt_soexcept(struct knote *kn, long hint);
75
76 int filt_sowmodify(struct kevent *kev, struct knote *kn);
77 int filt_sowprocess(struct knote *kn, struct kevent *kev);
78
79 int filt_sormodify(struct kevent *kev, struct knote *kn);
80 int filt_sorprocess(struct knote *kn, struct kevent *kev);
81
82 const struct filterops soread_filtops = {
83 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
84 .f_attach = NULL,
85 .f_detach = filt_sordetach,
86 .f_event = filt_soread,
87 .f_modify = filt_sormodify,
88 .f_process = filt_sorprocess,
89 };
90
91 const struct filterops sowrite_filtops = {
92 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
93 .f_attach = NULL,
94 .f_detach = filt_sowdetach,
95 .f_event = filt_sowrite,
96 .f_modify = filt_sowmodify,
97 .f_process = filt_sowprocess,
98 };
99
100 const struct filterops soexcept_filtops = {
101 .f_flags = FILTEROP_ISFD | FILTEROP_MPSAFE,
102 .f_attach = NULL,
103 .f_detach = filt_sordetach,
104 .f_event = filt_soexcept,
105 .f_modify = filt_sormodify,
106 .f_process = filt_sorprocess,
107 };
108
109 #ifndef SOMINCONN
110 #define SOMINCONN 80
111 #endif /* SOMINCONN */
112
113 int somaxconn = SOMAXCONN;
114 int sominconn = SOMINCONN;
115
116 struct pool socket_pool;
117 #ifdef SOCKET_SPLICE
118 struct pool sosplice_pool;
119 struct taskq *sosplice_taskq;
120 struct rwlock sosplice_lock = RWLOCK_INITIALIZER("sosplicelk");
121 #endif
122
123 void
soinit(void)124 soinit(void)
125 {
126 pool_init(&socket_pool, sizeof(struct socket), 0, IPL_SOFTNET, 0,
127 "sockpl", NULL);
128 #ifdef SOCKET_SPLICE
129 pool_init(&sosplice_pool, sizeof(struct sosplice), 0, IPL_SOFTNET, 0,
130 "sosppl", NULL);
131 #endif
132 }
133
134 struct socket *
soalloc(const struct protosw * prp,int wait)135 soalloc(const struct protosw *prp, int wait)
136 {
137 const struct domain *dp = prp->pr_domain;
138 struct socket *so;
139
140 so = pool_get(&socket_pool, (wait == M_WAIT ? PR_WAITOK : PR_NOWAIT) |
141 PR_ZERO);
142 if (so == NULL)
143 return (NULL);
144 rw_init_flags(&so->so_lock, dp->dom_name, RWL_DUPOK);
145 refcnt_init(&so->so_refcnt);
146 rw_init(&so->so_rcv.sb_lock, "sbufrcv");
147 rw_init(&so->so_snd.sb_lock, "sbufsnd");
148 mtx_init_flags(&so->so_rcv.sb_mtx, IPL_MPFLOOR, "sbrcv", 0);
149 mtx_init_flags(&so->so_snd.sb_mtx, IPL_MPFLOOR, "sbsnd", 0);
150 klist_init_mutex(&so->so_rcv.sb_klist, &so->so_rcv.sb_mtx);
151 klist_init_mutex(&so->so_snd.sb_klist, &so->so_snd.sb_mtx);
152 sigio_init(&so->so_sigio);
153 TAILQ_INIT(&so->so_q0);
154 TAILQ_INIT(&so->so_q);
155
156 switch (dp->dom_family) {
157 case AF_INET:
158 case AF_INET6:
159 switch (prp->pr_type) {
160 case SOCK_RAW:
161 so->so_snd.sb_flags |= SB_MTXLOCK;
162 /* FALLTHROUGH */
163 case SOCK_DGRAM:
164 so->so_rcv.sb_flags |= SB_MTXLOCK;
165 break;
166 }
167 break;
168 case AF_KEY:
169 case AF_ROUTE:
170 case AF_UNIX:
171 so->so_snd.sb_flags |= SB_MTXLOCK;
172 so->so_rcv.sb_flags |= SB_MTXLOCK;
173 break;
174 }
175
176 return (so);
177 }
178
179 /*
180 * Socket operation routines.
181 * These routines are called by the routines in
182 * sys_socket.c or from a system process, and
183 * implement the semantics of socket operations by
184 * switching out to the protocol specific routines.
185 */
186 int
socreate(int dom,struct socket ** aso,int type,int proto)187 socreate(int dom, struct socket **aso, int type, int proto)
188 {
189 struct proc *p = curproc; /* XXX */
190 const struct protosw *prp;
191 struct socket *so;
192 int error;
193
194 if (proto)
195 prp = pffindproto(dom, proto, type);
196 else
197 prp = pffindtype(dom, type);
198 if (prp == NULL || prp->pr_usrreqs == NULL)
199 return (EPROTONOSUPPORT);
200 if (prp->pr_type != type)
201 return (EPROTOTYPE);
202 so = soalloc(prp, M_WAIT);
203 so->so_type = type;
204 if (suser(p) == 0)
205 so->so_state = SS_PRIV;
206 so->so_ruid = p->p_ucred->cr_ruid;
207 so->so_euid = p->p_ucred->cr_uid;
208 so->so_rgid = p->p_ucred->cr_rgid;
209 so->so_egid = p->p_ucred->cr_gid;
210 so->so_cpid = p->p_p->ps_pid;
211 so->so_proto = prp;
212 so->so_snd.sb_timeo_nsecs = INFSLP;
213 so->so_rcv.sb_timeo_nsecs = INFSLP;
214
215 solock(so);
216 error = pru_attach(so, proto, M_WAIT);
217 if (error) {
218 so->so_state |= SS_NOFDREF;
219 /* sofree() calls sounlock(). */
220 sofree(so, 0);
221 return (error);
222 }
223 sounlock(so);
224 *aso = so;
225 return (0);
226 }
227
228 int
sobind(struct socket * so,struct mbuf * nam,struct proc * p)229 sobind(struct socket *so, struct mbuf *nam, struct proc *p)
230 {
231 soassertlocked(so);
232 return pru_bind(so, nam, p);
233 }
234
235 int
solisten(struct socket * so,int backlog)236 solisten(struct socket *so, int backlog)
237 {
238 int somaxconn_local = READ_ONCE(somaxconn);
239 int sominconn_local = READ_ONCE(sominconn);
240 int error;
241
242 switch (so->so_type) {
243 case SOCK_STREAM:
244 case SOCK_SEQPACKET:
245 break;
246 default:
247 return (EOPNOTSUPP);
248 }
249
250 soassertlocked(so);
251
252 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING|SS_ISDISCONNECTING))
253 return (EINVAL);
254 #ifdef SOCKET_SPLICE
255 if (isspliced(so) || issplicedback(so))
256 return (EOPNOTSUPP);
257 #endif /* SOCKET_SPLICE */
258 error = pru_listen(so);
259 if (error)
260 return (error);
261 if (TAILQ_FIRST(&so->so_q) == NULL)
262 so->so_options |= SO_ACCEPTCONN;
263 if (backlog < 0 || backlog > somaxconn_local)
264 backlog = somaxconn_local;
265 if (backlog < sominconn_local)
266 backlog = sominconn_local;
267 so->so_qlimit = backlog;
268 return (0);
269 }
270
271 #define SOSP_FREEING_READ 1
272 #define SOSP_FREEING_WRITE 2
273 void
sofree(struct socket * so,int keep_lock)274 sofree(struct socket *so, int keep_lock)
275 {
276 int persocket = solock_persocket(so);
277
278 soassertlocked(so);
279
280 if (so->so_pcb || (so->so_state & SS_NOFDREF) == 0) {
281 if (!keep_lock)
282 sounlock(so);
283 return;
284 }
285 if (so->so_head) {
286 struct socket *head = so->so_head;
287
288 /*
289 * We must not decommission a socket that's on the accept(2)
290 * queue. If we do, then accept(2) may hang after select(2)
291 * indicated that the listening socket was ready.
292 */
293 if (so->so_onq == &head->so_q) {
294 if (!keep_lock)
295 sounlock(so);
296 return;
297 }
298
299 if (persocket) {
300 /*
301 * Concurrent close of `head' could
302 * abort `so' due to re-lock.
303 */
304 soref(so);
305 soref(head);
306 sounlock(so);
307 solock(head);
308 solock(so);
309
310 if (so->so_onq != &head->so_q0) {
311 sounlock(head);
312 sounlock(so);
313 sorele(head);
314 sorele(so);
315 return;
316 }
317
318 sorele(head);
319 sorele(so);
320 }
321
322 soqremque(so, 0);
323
324 if (persocket)
325 sounlock(head);
326 }
327
328 if (persocket) {
329 sounlock(so);
330 refcnt_finalize(&so->so_refcnt, "sofinal");
331 solock(so);
332 }
333
334 sigio_free(&so->so_sigio);
335 klist_free(&so->so_rcv.sb_klist);
336 klist_free(&so->so_snd.sb_klist);
337 #ifdef SOCKET_SPLICE
338 if (issplicedback(so)) {
339 int freeing = SOSP_FREEING_WRITE;
340
341 if (so->so_sp->ssp_soback == so)
342 freeing |= SOSP_FREEING_READ;
343 sounsplice(so->so_sp->ssp_soback, so, freeing);
344 }
345 if (isspliced(so)) {
346 int freeing = SOSP_FREEING_READ;
347
348 if (so == so->so_sp->ssp_socket)
349 freeing |= SOSP_FREEING_WRITE;
350 sounsplice(so, so->so_sp->ssp_socket, freeing);
351 }
352 #endif /* SOCKET_SPLICE */
353
354 mtx_enter(&so->so_snd.sb_mtx);
355 sbrelease(so, &so->so_snd);
356 mtx_leave(&so->so_snd.sb_mtx);
357
358 /*
359 * Unlocked dispose and cleanup is safe. Socket is unlinked
360 * from everywhere. Even concurrent sotask() thread will not
361 * call somove().
362 */
363 if (so->so_proto->pr_flags & PR_RIGHTS &&
364 so->so_proto->pr_domain->dom_dispose)
365 (*so->so_proto->pr_domain->dom_dispose)(so->so_rcv.sb_mb);
366 m_purge(so->so_rcv.sb_mb);
367
368 if (!keep_lock)
369 sounlock(so);
370
371 #ifdef SOCKET_SPLICE
372 if (so->so_sp) {
373 /* Reuse splice idle, sounsplice() has been called before. */
374 timeout_set_proc(&so->so_sp->ssp_idleto, soreaper, so);
375 timeout_add(&so->so_sp->ssp_idleto, 0);
376 } else
377 #endif /* SOCKET_SPLICE */
378 {
379 pool_put(&socket_pool, so);
380 }
381 }
382
383 static inline uint64_t
solinger_nsec(struct socket * so)384 solinger_nsec(struct socket *so)
385 {
386 if (so->so_linger == 0)
387 return INFSLP;
388
389 return SEC_TO_NSEC(so->so_linger);
390 }
391
392 /*
393 * Close a socket on last file table reference removal.
394 * Initiate disconnect if connected.
395 * Free socket when disconnect complete.
396 */
397 int
soclose(struct socket * so,int flags)398 soclose(struct socket *so, int flags)
399 {
400 struct socket *so2;
401 int error = 0;
402
403 solock(so);
404 /* Revoke async IO early. There is a final revocation in sofree(). */
405 sigio_free(&so->so_sigio);
406 if (so->so_state & SS_ISCONNECTED) {
407 if (so->so_pcb == NULL)
408 goto discard;
409 if ((so->so_state & SS_ISDISCONNECTING) == 0) {
410 error = sodisconnect(so);
411 if (error)
412 goto drop;
413 }
414 if (so->so_options & SO_LINGER) {
415 if ((so->so_state & SS_ISDISCONNECTING) &&
416 (flags & MSG_DONTWAIT))
417 goto drop;
418 while (so->so_state & SS_ISCONNECTED) {
419 error = sosleep_nsec(so, &so->so_timeo,
420 PSOCK | PCATCH, "netcls",
421 solinger_nsec(so));
422 if (error)
423 break;
424 }
425 }
426 }
427 drop:
428 if (so->so_pcb) {
429 int error2;
430 error2 = pru_detach(so);
431 if (error == 0)
432 error = error2;
433 }
434 if (so->so_options & SO_ACCEPTCONN) {
435 int persocket = solock_persocket(so);
436
437 while ((so2 = TAILQ_FIRST(&so->so_q0)) != NULL) {
438 if (persocket)
439 solock(so2);
440 (void) soqremque(so2, 0);
441 if (persocket)
442 sounlock(so);
443 soabort(so2);
444 if (persocket)
445 solock(so);
446 }
447 while ((so2 = TAILQ_FIRST(&so->so_q)) != NULL) {
448 if (persocket)
449 solock(so2);
450 (void) soqremque(so2, 1);
451 if (persocket)
452 sounlock(so);
453 soabort(so2);
454 if (persocket)
455 solock(so);
456 }
457 }
458 discard:
459 if (so->so_state & SS_NOFDREF)
460 panic("soclose NOFDREF: so %p, so_type %d", so, so->so_type);
461 so->so_state |= SS_NOFDREF;
462 /* sofree() calls sounlock(). */
463 sofree(so, 0);
464 return (error);
465 }
466
467 void
soabort(struct socket * so)468 soabort(struct socket *so)
469 {
470 soassertlocked(so);
471 pru_abort(so);
472 }
473
474 int
soaccept(struct socket * so,struct mbuf * nam)475 soaccept(struct socket *so, struct mbuf *nam)
476 {
477 int error = 0;
478
479 soassertlocked(so);
480
481 if ((so->so_state & SS_NOFDREF) == 0)
482 panic("soaccept !NOFDREF: so %p, so_type %d", so, so->so_type);
483 so->so_state &= ~SS_NOFDREF;
484 if ((so->so_state & SS_ISDISCONNECTED) == 0 ||
485 (so->so_proto->pr_flags & PR_ABRTACPTDIS) == 0)
486 error = pru_accept(so, nam);
487 else
488 error = ECONNABORTED;
489 return (error);
490 }
491
492 int
soconnect(struct socket * so,struct mbuf * nam)493 soconnect(struct socket *so, struct mbuf *nam)
494 {
495 int error;
496
497 soassertlocked(so);
498
499 if (so->so_options & SO_ACCEPTCONN)
500 return (EOPNOTSUPP);
501 /*
502 * If protocol is connection-based, can only connect once.
503 * Otherwise, if connected, try to disconnect first.
504 * This allows user to disconnect by connecting to, e.g.,
505 * a null address.
506 */
507 if (so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING) &&
508 ((so->so_proto->pr_flags & PR_CONNREQUIRED) ||
509 (error = sodisconnect(so))))
510 error = EISCONN;
511 else
512 error = pru_connect(so, nam);
513 return (error);
514 }
515
516 int
soconnect2(struct socket * so1,struct socket * so2)517 soconnect2(struct socket *so1, struct socket *so2)
518 {
519 int persocket, error;
520
521 if ((persocket = solock_persocket(so1)))
522 solock_pair(so1, so2);
523 else
524 solock(so1);
525
526 error = pru_connect2(so1, so2);
527
528 if (persocket)
529 sounlock(so2);
530 sounlock(so1);
531 return (error);
532 }
533
534 int
sodisconnect(struct socket * so)535 sodisconnect(struct socket *so)
536 {
537 int error;
538
539 soassertlocked(so);
540
541 if ((so->so_state & SS_ISCONNECTED) == 0)
542 return (ENOTCONN);
543 if (so->so_state & SS_ISDISCONNECTING)
544 return (EALREADY);
545 error = pru_disconnect(so);
546 return (error);
547 }
548
549 int m_getuio(struct mbuf **, int, long, struct uio *);
550
551 #define SBLOCKWAIT(f) (((f) & MSG_DONTWAIT) ? 0 : SBL_WAIT)
552 /*
553 * Send on a socket.
554 * If send must go all at once and message is larger than
555 * send buffering, then hard error.
556 * Lock against other senders.
557 * If must go all at once and not enough room now, then
558 * inform user that this would block and do nothing.
559 * Otherwise, if nonblocking, send as much as possible.
560 * The data to be sent is described by "uio" if nonzero,
561 * otherwise by the mbuf chain "top" (which must be null
562 * if uio is not). Data provided in mbuf chain must be small
563 * enough to send all at once.
564 *
565 * Returns nonzero on error, timeout or signal; callers
566 * must check for short counts if EINTR/ERESTART are returned.
567 * Data and control buffers are freed on return.
568 */
569 int
sosend(struct socket * so,struct mbuf * addr,struct uio * uio,struct mbuf * top,struct mbuf * control,int flags)570 sosend(struct socket *so, struct mbuf *addr, struct uio *uio, struct mbuf *top,
571 struct mbuf *control, int flags)
572 {
573 long space, clen = 0;
574 size_t resid;
575 int error;
576 int atomic = sosendallatonce(so) || top;
577 int dosolock = ((so->so_snd.sb_flags & SB_MTXLOCK) == 0);
578
579 if (uio)
580 resid = uio->uio_resid;
581 else
582 resid = top->m_pkthdr.len;
583 /* MSG_EOR on a SOCK_STREAM socket is invalid. */
584 if (so->so_type == SOCK_STREAM && (flags & MSG_EOR)) {
585 m_freem(top);
586 m_freem(control);
587 return (EINVAL);
588 }
589 if (uio && uio->uio_procp)
590 uio->uio_procp->p_ru.ru_msgsnd++;
591 if (control) {
592 /*
593 * In theory clen should be unsigned (since control->m_len is).
594 * However, space must be signed, as it might be less than 0
595 * if we over-committed, and we must use a signed comparison
596 * of space and clen.
597 */
598 clen = control->m_len;
599 /* reserve extra space for AF_UNIX's internalize */
600 if (so->so_proto->pr_domain->dom_family == AF_UNIX &&
601 clen >= CMSG_ALIGN(sizeof(struct cmsghdr)) &&
602 mtod(control, struct cmsghdr *)->cmsg_type == SCM_RIGHTS)
603 clen = CMSG_SPACE(
604 (clen - CMSG_ALIGN(sizeof(struct cmsghdr))) *
605 (sizeof(struct fdpass) / sizeof(int)));
606 }
607
608 #define snderr(errno) { error = errno; goto release; }
609
610 restart:
611 if ((error = sblock(&so->so_snd, SBLOCKWAIT(flags))) != 0)
612 goto out;
613 if (dosolock)
614 solock_shared(so);
615 sb_mtx_lock(&so->so_snd);
616 so->so_snd.sb_state |= SS_ISSENDING;
617 do {
618 if (so->so_snd.sb_state & SS_CANTSENDMORE)
619 snderr(EPIPE);
620 if ((error = READ_ONCE(so->so_error))) {
621 so->so_error = 0;
622 snderr(error);
623 }
624 if ((so->so_state & SS_ISCONNECTED) == 0) {
625 if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
626 if (!(resid == 0 && clen != 0))
627 snderr(ENOTCONN);
628 } else if (addr == NULL)
629 snderr(EDESTADDRREQ);
630 }
631 space = sbspace(so, &so->so_snd);
632 if (flags & MSG_OOB)
633 space += 1024;
634 if (so->so_proto->pr_domain->dom_family == AF_UNIX) {
635 if (atomic && resid > so->so_snd.sb_hiwat)
636 snderr(EMSGSIZE);
637 } else {
638 if (clen > so->so_snd.sb_hiwat ||
639 (atomic && resid > so->so_snd.sb_hiwat - clen))
640 snderr(EMSGSIZE);
641 }
642 if (space < clen ||
643 (space - clen < resid &&
644 (atomic || space < so->so_snd.sb_lowat))) {
645 if (flags & MSG_DONTWAIT)
646 snderr(EWOULDBLOCK);
647 sbunlock(&so->so_snd);
648 error = sbwait(so, &so->so_snd);
649 so->so_snd.sb_state &= ~SS_ISSENDING;
650 sb_mtx_unlock(&so->so_snd);
651 if (dosolock)
652 sounlock_shared(so);
653 if (error)
654 goto out;
655 goto restart;
656 }
657 space -= clen;
658 do {
659 if (uio == NULL) {
660 /*
661 * Data is prepackaged in "top".
662 */
663 resid = 0;
664 if (flags & MSG_EOR)
665 top->m_flags |= M_EOR;
666 } else {
667 sb_mtx_unlock(&so->so_snd);
668 if (dosolock)
669 sounlock_shared(so);
670 error = m_getuio(&top, atomic, space, uio);
671 if (dosolock)
672 solock_shared(so);
673 sb_mtx_lock(&so->so_snd);
674 if (error)
675 goto release;
676 space -= top->m_pkthdr.len;
677 resid = uio->uio_resid;
678 if (flags & MSG_EOR)
679 top->m_flags |= M_EOR;
680 }
681 if (resid == 0)
682 so->so_snd.sb_state &= ~SS_ISSENDING;
683 if (top && so->so_options & SO_ZEROIZE)
684 top->m_flags |= M_ZEROIZE;
685 sb_mtx_unlock(&so->so_snd);
686 if (!dosolock)
687 solock_shared(so);
688 if (flags & MSG_OOB)
689 error = pru_sendoob(so, top, addr, control);
690 else
691 error = pru_send(so, top, addr, control);
692 if (!dosolock)
693 sounlock_shared(so);
694 sb_mtx_lock(&so->so_snd);
695 clen = 0;
696 control = NULL;
697 top = NULL;
698 if (error)
699 goto release;
700 } while (resid && space > 0);
701 } while (resid);
702
703 release:
704 so->so_snd.sb_state &= ~SS_ISSENDING;
705 sb_mtx_unlock(&so->so_snd);
706 if (dosolock)
707 sounlock_shared(so);
708 sbunlock(&so->so_snd);
709 out:
710 m_freem(top);
711 m_freem(control);
712 return (error);
713 }
714
715 int
m_getuio(struct mbuf ** mp,int atomic,long space,struct uio * uio)716 m_getuio(struct mbuf **mp, int atomic, long space, struct uio *uio)
717 {
718 struct mbuf *m, *top = NULL;
719 struct mbuf **nextp = ⊤
720 u_long len, mlen;
721 size_t resid = uio->uio_resid;
722 int error;
723
724 do {
725 if (top == NULL) {
726 MGETHDR(m, M_WAIT, MT_DATA);
727 mlen = MHLEN;
728 m->m_pkthdr.len = 0;
729 m->m_pkthdr.ph_ifidx = 0;
730 } else {
731 MGET(m, M_WAIT, MT_DATA);
732 mlen = MLEN;
733 }
734 /* chain mbuf together */
735 *nextp = m;
736 nextp = &m->m_next;
737
738 resid = ulmin(resid, space);
739 if (resid >= MINCLSIZE) {
740 MCLGETL(m, M_NOWAIT, ulmin(resid, MAXMCLBYTES));
741 if ((m->m_flags & M_EXT) == 0)
742 MCLGETL(m, M_NOWAIT, MCLBYTES);
743 if ((m->m_flags & M_EXT) == 0)
744 goto nopages;
745 mlen = m->m_ext.ext_size;
746 len = ulmin(mlen, resid);
747 /*
748 * For datagram protocols, leave room
749 * for protocol headers in first mbuf.
750 */
751 if (atomic && m == top && len < mlen - max_hdr)
752 m->m_data += max_hdr;
753 } else {
754 nopages:
755 len = ulmin(mlen, resid);
756 /*
757 * For datagram protocols, leave room
758 * for protocol headers in first mbuf.
759 */
760 if (atomic && m == top && len < mlen - max_hdr)
761 m_align(m, len);
762 }
763
764 error = uiomove(mtod(m, caddr_t), len, uio);
765 if (error) {
766 m_freem(top);
767 return (error);
768 }
769
770 /* adjust counters */
771 resid = uio->uio_resid;
772 space -= len;
773 m->m_len = len;
774 top->m_pkthdr.len += len;
775
776 /* Is there more space and more data? */
777 } while (space > 0 && resid > 0);
778
779 *mp = top;
780 return 0;
781 }
782
783 /*
784 * Following replacement or removal of the first mbuf on the first
785 * mbuf chain of a socket buffer, push necessary state changes back
786 * into the socket buffer so that other consumers see the values
787 * consistently. 'nextrecord' is the callers locally stored value of
788 * the original value of sb->sb_mb->m_nextpkt which must be restored
789 * when the lead mbuf changes. NOTE: 'nextrecord' may be NULL.
790 */
791 void
sbsync(struct sockbuf * sb,struct mbuf * nextrecord)792 sbsync(struct sockbuf *sb, struct mbuf *nextrecord)
793 {
794
795 /*
796 * First, update for the new value of nextrecord. If necessary,
797 * make it the first record.
798 */
799 if (sb->sb_mb != NULL)
800 sb->sb_mb->m_nextpkt = nextrecord;
801 else
802 sb->sb_mb = nextrecord;
803
804 /*
805 * Now update any dependent socket buffer fields to reflect
806 * the new state. This is an inline of SB_EMPTY_FIXUP, with
807 * the addition of a second clause that takes care of the
808 * case where sb_mb has been updated, but remains the last
809 * record.
810 */
811 if (sb->sb_mb == NULL) {
812 sb->sb_mbtail = NULL;
813 sb->sb_lastrecord = NULL;
814 } else if (sb->sb_mb->m_nextpkt == NULL)
815 sb->sb_lastrecord = sb->sb_mb;
816 }
817
818 /*
819 * Implement receive operations on a socket.
820 * We depend on the way that records are added to the sockbuf
821 * by sbappend*. In particular, each record (mbufs linked through m_next)
822 * must begin with an address if the protocol so specifies,
823 * followed by an optional mbuf or mbufs containing ancillary data,
824 * and then zero or more mbufs of data.
825 * In order to avoid blocking network for the entire time here, we release
826 * the solock() while doing the actual copy to user space.
827 * Although the sockbuf is locked, new data may still be appended,
828 * and thus we must maintain consistency of the sockbuf during that time.
829 *
830 * The caller may receive the data as a single mbuf chain by supplying
831 * an mbuf **mp0 for use in returning the chain. The uio is then used
832 * only for the count in uio_resid.
833 */
834 int
soreceive(struct socket * so,struct mbuf ** paddr,struct uio * uio,struct mbuf ** mp0,struct mbuf ** controlp,int * flagsp,socklen_t controllen)835 soreceive(struct socket *so, struct mbuf **paddr, struct uio *uio,
836 struct mbuf **mp0, struct mbuf **controlp, int *flagsp,
837 socklen_t controllen)
838 {
839 struct mbuf *m, **mp;
840 struct mbuf *cm;
841 u_long len, offset, moff;
842 int flags, error, error2, type, uio_error = 0;
843 const struct protosw *pr = so->so_proto;
844 struct mbuf *nextrecord;
845 size_t resid, orig_resid = uio->uio_resid;
846 int dosolock = ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0);
847
848 mp = mp0;
849 if (paddr)
850 *paddr = NULL;
851 if (controlp)
852 *controlp = NULL;
853 if (flagsp)
854 flags = *flagsp &~ MSG_EOR;
855 else
856 flags = 0;
857 if (flags & MSG_OOB) {
858 m = m_get(M_WAIT, MT_DATA);
859 solock(so);
860 error = pru_rcvoob(so, m, flags & MSG_PEEK);
861 sounlock(so);
862 if (error)
863 goto bad;
864 do {
865 error = uiomove(mtod(m, caddr_t),
866 ulmin(uio->uio_resid, m->m_len), uio);
867 m = m_free(m);
868 } while (uio->uio_resid && error == 0 && m);
869 bad:
870 m_freem(m);
871 return (error);
872 }
873 if (mp)
874 *mp = NULL;
875
876 restart:
877 if ((error = sblock(&so->so_rcv, SBLOCKWAIT(flags))) != 0)
878 return (error);
879 if (dosolock)
880 solock_shared(so);
881 sb_mtx_lock(&so->so_rcv);
882
883 m = so->so_rcv.sb_mb;
884 #ifdef SOCKET_SPLICE
885 if (isspliced(so))
886 m = NULL;
887 #endif /* SOCKET_SPLICE */
888 /*
889 * If we have less data than requested, block awaiting more
890 * (subject to any timeout) if:
891 * 1. the current count is less than the low water mark,
892 * 2. MSG_WAITALL is set, and it is possible to do the entire
893 * receive operation at once if we block (resid <= hiwat), or
894 * 3. MSG_DONTWAIT is not set.
895 * If MSG_WAITALL is set but resid is larger than the receive buffer,
896 * we have to do the receive in sections, and thus risk returning
897 * a short count if a timeout or signal occurs after we start.
898 */
899 if (m == NULL || (((flags & MSG_DONTWAIT) == 0 &&
900 so->so_rcv.sb_cc < uio->uio_resid) &&
901 (so->so_rcv.sb_cc < so->so_rcv.sb_lowat ||
902 ((flags & MSG_WAITALL) && uio->uio_resid <= so->so_rcv.sb_hiwat)) &&
903 m->m_nextpkt == NULL && (pr->pr_flags & PR_ATOMIC) == 0)) {
904 #ifdef DIAGNOSTIC
905 if (m == NULL && so->so_rcv.sb_cc)
906 #ifdef SOCKET_SPLICE
907 if (!isspliced(so))
908 #endif /* SOCKET_SPLICE */
909 panic("receive 1: so %p, so_type %d, sb_cc %lu",
910 so, so->so_type, so->so_rcv.sb_cc);
911 #endif
912 if ((error2 = READ_ONCE(so->so_error))) {
913 if (m)
914 goto dontblock;
915 error = error2;
916 if ((flags & MSG_PEEK) == 0)
917 so->so_error = 0;
918 goto release;
919 }
920 if (so->so_rcv.sb_state & SS_CANTRCVMORE) {
921 if (m)
922 goto dontblock;
923 else if (so->so_rcv.sb_cc == 0)
924 goto release;
925 }
926 for (; m; m = m->m_next)
927 if (m->m_type == MT_OOBDATA || (m->m_flags & M_EOR)) {
928 m = so->so_rcv.sb_mb;
929 goto dontblock;
930 }
931 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
932 (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
933 error = ENOTCONN;
934 goto release;
935 }
936 if (uio->uio_resid == 0 && controlp == NULL)
937 goto release;
938 if (flags & MSG_DONTWAIT) {
939 error = EWOULDBLOCK;
940 goto release;
941 }
942 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 1");
943 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 1");
944
945 sbunlock(&so->so_rcv);
946 error = sbwait(so, &so->so_rcv);
947 sb_mtx_unlock(&so->so_rcv);
948 if (dosolock)
949 sounlock_shared(so);
950 if (error)
951 return (error);
952 goto restart;
953 }
954 dontblock:
955 /*
956 * On entry here, m points to the first record of the socket buffer.
957 * From this point onward, we maintain 'nextrecord' as a cache of the
958 * pointer to the next record in the socket buffer. We must keep the
959 * various socket buffer pointers and local stack versions of the
960 * pointers in sync, pushing out modifications before operations that
961 * may sleep, and re-reading them afterwards.
962 *
963 * Otherwise, we will race with the network stack appending new data
964 * or records onto the socket buffer by using inconsistent/stale
965 * versions of the field, possibly resulting in socket buffer
966 * corruption.
967 */
968 if (uio->uio_procp)
969 uio->uio_procp->p_ru.ru_msgrcv++;
970 KASSERT(m == so->so_rcv.sb_mb);
971 SBLASTRECORDCHK(&so->so_rcv, "soreceive 1");
972 SBLASTMBUFCHK(&so->so_rcv, "soreceive 1");
973 nextrecord = m->m_nextpkt;
974 if (pr->pr_flags & PR_ADDR) {
975 #ifdef DIAGNOSTIC
976 if (m->m_type != MT_SONAME)
977 panic("receive 1a: so %p, so_type %d, m %p, m_type %d",
978 so, so->so_type, m, m->m_type);
979 #endif
980 orig_resid = 0;
981 if (flags & MSG_PEEK) {
982 if (paddr)
983 *paddr = m_copym(m, 0, m->m_len, M_NOWAIT);
984 m = m->m_next;
985 } else {
986 sbfree(so, &so->so_rcv, m);
987 if (paddr) {
988 *paddr = m;
989 so->so_rcv.sb_mb = m->m_next;
990 m->m_next = NULL;
991 m = so->so_rcv.sb_mb;
992 } else {
993 so->so_rcv.sb_mb = m_free(m);
994 m = so->so_rcv.sb_mb;
995 }
996 sbsync(&so->so_rcv, nextrecord);
997 }
998 }
999 while (m && m->m_type == MT_CONTROL && error == 0) {
1000 int skip = 0;
1001 if (flags & MSG_PEEK) {
1002 if (mtod(m, struct cmsghdr *)->cmsg_type ==
1003 SCM_RIGHTS) {
1004 /* don't leak internalized SCM_RIGHTS msgs */
1005 skip = 1;
1006 } else if (controlp)
1007 *controlp = m_copym(m, 0, m->m_len, M_NOWAIT);
1008 m = m->m_next;
1009 } else {
1010 sbfree(so, &so->so_rcv, m);
1011 so->so_rcv.sb_mb = m->m_next;
1012 m->m_nextpkt = m->m_next = NULL;
1013 cm = m;
1014 m = so->so_rcv.sb_mb;
1015 sbsync(&so->so_rcv, nextrecord);
1016 if (controlp) {
1017 if (pr->pr_domain->dom_externalize) {
1018 sb_mtx_unlock(&so->so_rcv);
1019 if (dosolock)
1020 sounlock_shared(so);
1021 error =
1022 (*pr->pr_domain->dom_externalize)
1023 (cm, controllen, flags);
1024 if (dosolock)
1025 solock_shared(so);
1026 sb_mtx_lock(&so->so_rcv);
1027 }
1028 *controlp = cm;
1029 } else {
1030 /*
1031 * Dispose of any SCM_RIGHTS message that went
1032 * through the read path rather than recv.
1033 */
1034 if (pr->pr_domain->dom_dispose) {
1035 sb_mtx_unlock(&so->so_rcv);
1036 pr->pr_domain->dom_dispose(cm);
1037 sb_mtx_lock(&so->so_rcv);
1038 }
1039 m_free(cm);
1040 }
1041 }
1042 if (m != NULL)
1043 nextrecord = so->so_rcv.sb_mb->m_nextpkt;
1044 else
1045 nextrecord = so->so_rcv.sb_mb;
1046 if (controlp && !skip)
1047 controlp = &(*controlp)->m_next;
1048 orig_resid = 0;
1049 }
1050
1051 /* If m is non-NULL, we have some data to read. */
1052 if (m) {
1053 type = m->m_type;
1054 if (type == MT_OOBDATA)
1055 flags |= MSG_OOB;
1056 if (m->m_flags & M_BCAST)
1057 flags |= MSG_BCAST;
1058 if (m->m_flags & M_MCAST)
1059 flags |= MSG_MCAST;
1060 }
1061 SBLASTRECORDCHK(&so->so_rcv, "soreceive 2");
1062 SBLASTMBUFCHK(&so->so_rcv, "soreceive 2");
1063
1064 moff = 0;
1065 offset = 0;
1066 while (m && uio->uio_resid > 0 && error == 0) {
1067 if (m->m_type == MT_OOBDATA) {
1068 if (type != MT_OOBDATA)
1069 break;
1070 } else if (type == MT_OOBDATA) {
1071 break;
1072 } else if (m->m_type == MT_CONTROL) {
1073 /*
1074 * If there is more than one control message in the
1075 * stream, we do a short read. Next can be received
1076 * or disposed by another system call.
1077 */
1078 break;
1079 #ifdef DIAGNOSTIC
1080 } else if (m->m_type != MT_DATA && m->m_type != MT_HEADER) {
1081 panic("receive 3: so %p, so_type %d, m %p, m_type %d",
1082 so, so->so_type, m, m->m_type);
1083 #endif
1084 }
1085 so->so_rcv.sb_state &= ~SS_RCVATMARK;
1086 len = uio->uio_resid;
1087 if (so->so_oobmark && len > so->so_oobmark - offset)
1088 len = so->so_oobmark - offset;
1089 if (len > m->m_len - moff)
1090 len = m->m_len - moff;
1091 /*
1092 * If mp is set, just pass back the mbufs.
1093 * Otherwise copy them out via the uio, then free.
1094 * Sockbuf must be consistent here (points to current mbuf,
1095 * it points to next record) when we drop priority;
1096 * we must note any additions to the sockbuf when we
1097 * block interrupts again.
1098 */
1099 if (mp == NULL && uio_error == 0) {
1100 SBLASTRECORDCHK(&so->so_rcv, "soreceive uiomove");
1101 SBLASTMBUFCHK(&so->so_rcv, "soreceive uiomove");
1102 resid = uio->uio_resid;
1103 sb_mtx_unlock(&so->so_rcv);
1104 if (dosolock)
1105 sounlock_shared(so);
1106 uio_error = uiomove(mtod(m, caddr_t) + moff, len, uio);
1107 if (dosolock)
1108 solock_shared(so);
1109 sb_mtx_lock(&so->so_rcv);
1110 if (uio_error)
1111 uio->uio_resid = resid - len;
1112 } else
1113 uio->uio_resid -= len;
1114 if (len == m->m_len - moff) {
1115 if (m->m_flags & M_EOR)
1116 flags |= MSG_EOR;
1117 if (flags & MSG_PEEK) {
1118 m = m->m_next;
1119 moff = 0;
1120 orig_resid = 0;
1121 } else {
1122 nextrecord = m->m_nextpkt;
1123 sbfree(so, &so->so_rcv, m);
1124 if (mp) {
1125 *mp = m;
1126 mp = &m->m_next;
1127 so->so_rcv.sb_mb = m = m->m_next;
1128 *mp = NULL;
1129 } else {
1130 so->so_rcv.sb_mb = m_free(m);
1131 m = so->so_rcv.sb_mb;
1132 }
1133 /*
1134 * If m != NULL, we also know that
1135 * so->so_rcv.sb_mb != NULL.
1136 */
1137 KASSERT(so->so_rcv.sb_mb == m);
1138 if (m) {
1139 m->m_nextpkt = nextrecord;
1140 if (nextrecord == NULL)
1141 so->so_rcv.sb_lastrecord = m;
1142 } else {
1143 so->so_rcv.sb_mb = nextrecord;
1144 SB_EMPTY_FIXUP(&so->so_rcv);
1145 }
1146 SBLASTRECORDCHK(&so->so_rcv, "soreceive 3");
1147 SBLASTMBUFCHK(&so->so_rcv, "soreceive 3");
1148 }
1149 } else {
1150 if (flags & MSG_PEEK) {
1151 moff += len;
1152 orig_resid = 0;
1153 } else {
1154 if (mp)
1155 *mp = m_copym(m, 0, len, M_WAIT);
1156 m->m_data += len;
1157 m->m_len -= len;
1158 so->so_rcv.sb_cc -= len;
1159 so->so_rcv.sb_datacc -= len;
1160 }
1161 }
1162 if (so->so_oobmark) {
1163 if ((flags & MSG_PEEK) == 0) {
1164 so->so_oobmark -= len;
1165 if (so->so_oobmark == 0) {
1166 so->so_rcv.sb_state |= SS_RCVATMARK;
1167 break;
1168 }
1169 } else {
1170 offset += len;
1171 if (offset == so->so_oobmark)
1172 break;
1173 }
1174 }
1175 if (flags & MSG_EOR)
1176 break;
1177 /*
1178 * If the MSG_WAITALL flag is set (for non-atomic socket),
1179 * we must not quit until "uio->uio_resid == 0" or an error
1180 * termination. If a signal/timeout occurs, return
1181 * with a short count but without error.
1182 * Keep sockbuf locked against other readers.
1183 */
1184 while (flags & MSG_WAITALL && m == NULL && uio->uio_resid > 0 &&
1185 !sosendallatonce(so) && !nextrecord) {
1186 if (so->so_rcv.sb_state & SS_CANTRCVMORE ||
1187 so->so_error)
1188 break;
1189 SBLASTRECORDCHK(&so->so_rcv, "soreceive sbwait 2");
1190 SBLASTMBUFCHK(&so->so_rcv, "soreceive sbwait 2");
1191 if (sbwait(so, &so->so_rcv)) {
1192 sb_mtx_unlock(&so->so_rcv);
1193 if (dosolock)
1194 sounlock_shared(so);
1195 sbunlock(&so->so_rcv);
1196 return (0);
1197 }
1198 if ((m = so->so_rcv.sb_mb) != NULL)
1199 nextrecord = m->m_nextpkt;
1200 }
1201 }
1202
1203 if (m && pr->pr_flags & PR_ATOMIC) {
1204 flags |= MSG_TRUNC;
1205 if ((flags & MSG_PEEK) == 0)
1206 (void) sbdroprecord(so, &so->so_rcv);
1207 }
1208 if ((flags & MSG_PEEK) == 0) {
1209 if (m == NULL) {
1210 /*
1211 * First part is an inline SB_EMPTY_FIXUP(). Second
1212 * part makes sure sb_lastrecord is up-to-date if
1213 * there is still data in the socket buffer.
1214 */
1215 so->so_rcv.sb_mb = nextrecord;
1216 if (so->so_rcv.sb_mb == NULL) {
1217 so->so_rcv.sb_mbtail = NULL;
1218 so->so_rcv.sb_lastrecord = NULL;
1219 } else if (nextrecord->m_nextpkt == NULL)
1220 so->so_rcv.sb_lastrecord = nextrecord;
1221 }
1222 SBLASTRECORDCHK(&so->so_rcv, "soreceive 4");
1223 SBLASTMBUFCHK(&so->so_rcv, "soreceive 4");
1224 if (pr->pr_flags & PR_WANTRCVD) {
1225 sb_mtx_unlock(&so->so_rcv);
1226 if (!dosolock)
1227 solock_shared(so);
1228 pru_rcvd(so);
1229 if (!dosolock)
1230 sounlock_shared(so);
1231 sb_mtx_lock(&so->so_rcv);
1232 }
1233 }
1234 if (orig_resid == uio->uio_resid && orig_resid &&
1235 (flags & MSG_EOR) == 0 &&
1236 (so->so_rcv.sb_state & SS_CANTRCVMORE) == 0) {
1237 sb_mtx_unlock(&so->so_rcv);
1238 sbunlock(&so->so_rcv);
1239 goto restart;
1240 }
1241
1242 if (uio_error)
1243 error = uio_error;
1244
1245 if (flagsp)
1246 *flagsp |= flags;
1247 release:
1248 sb_mtx_unlock(&so->so_rcv);
1249 if (dosolock)
1250 sounlock_shared(so);
1251 sbunlock(&so->so_rcv);
1252 return (error);
1253 }
1254
1255 int
soshutdown(struct socket * so,int how)1256 soshutdown(struct socket *so, int how)
1257 {
1258 int error = 0;
1259
1260 switch (how) {
1261 case SHUT_RD:
1262 sorflush(so);
1263 break;
1264 case SHUT_RDWR:
1265 sorflush(so);
1266 /* FALLTHROUGH */
1267 case SHUT_WR:
1268 solock(so);
1269 error = pru_shutdown(so);
1270 sounlock(so);
1271 break;
1272 default:
1273 error = EINVAL;
1274 break;
1275 }
1276
1277 return (error);
1278 }
1279
1280 void
sorflush(struct socket * so)1281 sorflush(struct socket *so)
1282 {
1283 struct sockbuf *sb = &so->so_rcv;
1284 struct mbuf *m;
1285 const struct protosw *pr = so->so_proto;
1286 int error;
1287
1288 error = sblock(sb, SBL_WAIT | SBL_NOINTR);
1289 /* with SBL_WAIT and SLB_NOINTR sblock() must not fail */
1290 KASSERT(error == 0);
1291
1292 solock_shared(so);
1293 socantrcvmore(so);
1294 mtx_enter(&sb->sb_mtx);
1295 m = sb->sb_mb;
1296 memset(&sb->sb_startzero, 0,
1297 (caddr_t)&sb->sb_endzero - (caddr_t)&sb->sb_startzero);
1298 sb->sb_timeo_nsecs = INFSLP;
1299 mtx_leave(&sb->sb_mtx);
1300 sounlock_shared(so);
1301 sbunlock(sb);
1302
1303 if (pr->pr_flags & PR_RIGHTS && pr->pr_domain->dom_dispose)
1304 (*pr->pr_domain->dom_dispose)(m);
1305 m_purge(m);
1306 }
1307
1308 #ifdef SOCKET_SPLICE
1309
1310 #define so_splicelen so_sp->ssp_len
1311 #define so_splicemax so_sp->ssp_max
1312 #define so_idletv so_sp->ssp_idletv
1313 #define so_idleto so_sp->ssp_idleto
1314 #define so_splicetask so_sp->ssp_task
1315
1316 int
sosplice(struct socket * so,int fd,off_t max,struct timeval * tv)1317 sosplice(struct socket *so, int fd, off_t max, struct timeval *tv)
1318 {
1319 struct file *fp;
1320 struct socket *sosp;
1321 struct taskq *tq;
1322 int error = 0;
1323
1324 if ((so->so_proto->pr_flags & PR_SPLICE) == 0)
1325 return (EPROTONOSUPPORT);
1326 if (max && max < 0)
1327 return (EINVAL);
1328 if (tv && (tv->tv_sec < 0 || !timerisvalid(tv)))
1329 return (EINVAL);
1330
1331 /* If no fd is given, unsplice by removing existing link. */
1332 if (fd < 0) {
1333 if ((error = sblock(&so->so_rcv, SBL_WAIT)) != 0)
1334 return (error);
1335 solock(so);
1336 if (so->so_options & SO_ACCEPTCONN) {
1337 error = EOPNOTSUPP;
1338 goto out;
1339 }
1340 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
1341 (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
1342 error = ENOTCONN;
1343 goto out;
1344 }
1345
1346 if (so->so_sp && so->so_sp->ssp_socket)
1347 sounsplice(so, so->so_sp->ssp_socket, 0);
1348 out:
1349 sounlock(so);
1350 sbunlock(&so->so_rcv);
1351 return (error);
1352 }
1353
1354 if (sosplice_taskq == NULL) {
1355 rw_enter_write(&sosplice_lock);
1356 if (sosplice_taskq == NULL) {
1357 tq = taskq_create("sosplice", 1, IPL_SOFTNET,
1358 TASKQ_MPSAFE);
1359 if (tq == NULL) {
1360 rw_exit_write(&sosplice_lock);
1361 return (ENOMEM);
1362 }
1363 /* Ensure the taskq is fully visible to other CPUs. */
1364 membar_producer();
1365 sosplice_taskq = tq;
1366 }
1367 rw_exit_write(&sosplice_lock);
1368 } else {
1369 /* Ensure the taskq is fully visible on this CPU. */
1370 membar_consumer();
1371 }
1372
1373 /* Find sosp, the drain socket where data will be spliced into. */
1374 if ((error = getsock(curproc, fd, &fp)) != 0)
1375 return (error);
1376 sosp = fp->f_data;
1377
1378 if (sosp->so_proto->pr_usrreqs->pru_send !=
1379 so->so_proto->pr_usrreqs->pru_send) {
1380 error = EPROTONOSUPPORT;
1381 goto frele;
1382 }
1383
1384 if ((error = sblock(&so->so_rcv, SBL_WAIT)) != 0)
1385 goto frele;
1386 if ((error = sblock(&sosp->so_snd, SBL_WAIT)) != 0) {
1387 sbunlock(&so->so_rcv);
1388 goto frele;
1389 }
1390 solock(so);
1391
1392 if ((so->so_options & SO_ACCEPTCONN) ||
1393 (sosp->so_options & SO_ACCEPTCONN)) {
1394 error = EOPNOTSUPP;
1395 goto release;
1396 }
1397 if ((so->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0 &&
1398 (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
1399 error = ENOTCONN;
1400 goto release;
1401 }
1402 if ((sosp->so_state & (SS_ISCONNECTED|SS_ISCONNECTING)) == 0) {
1403 error = ENOTCONN;
1404 goto release;
1405 }
1406 if (so->so_sp == NULL)
1407 so->so_sp = pool_get(&sosplice_pool, PR_WAITOK | PR_ZERO);
1408 if (sosp->so_sp == NULL)
1409 sosp->so_sp = pool_get(&sosplice_pool, PR_WAITOK | PR_ZERO);
1410 if (so->so_sp->ssp_socket || sosp->so_sp->ssp_soback) {
1411 error = EBUSY;
1412 goto release;
1413 }
1414
1415 /* Splice so and sosp together. */
1416 mtx_enter(&so->so_rcv.sb_mtx);
1417 so->so_sp->ssp_socket = sosp;
1418 sosp->so_sp->ssp_soback = so;
1419 mtx_leave(&so->so_rcv.sb_mtx);
1420 so->so_splicelen = 0;
1421 so->so_splicemax = max;
1422 if (tv)
1423 so->so_idletv = *tv;
1424 else
1425 timerclear(&so->so_idletv);
1426 timeout_set_proc(&so->so_idleto, soidle, so);
1427 task_set(&so->so_splicetask, sotask, so);
1428
1429 /*
1430 * To prevent softnet interrupt from calling somove() while
1431 * we sleep, the socket buffers are not marked as spliced yet.
1432 */
1433 if (somove(so, M_WAIT)) {
1434 mtx_enter(&so->so_rcv.sb_mtx);
1435 so->so_rcv.sb_flags |= SB_SPLICE;
1436 mtx_leave(&so->so_rcv.sb_mtx);
1437 sosp->so_snd.sb_flags |= SB_SPLICE;
1438 }
1439
1440 release:
1441 sounlock(so);
1442 sbunlock(&sosp->so_snd);
1443 sbunlock(&so->so_rcv);
1444 frele:
1445 FRELE(fp, curproc);
1446
1447 return (error);
1448 }
1449
1450 void
sounsplice(struct socket * so,struct socket * sosp,int freeing)1451 sounsplice(struct socket *so, struct socket *sosp, int freeing)
1452 {
1453 soassertlocked(so);
1454
1455 task_del(sosplice_taskq, &so->so_splicetask);
1456 timeout_del(&so->so_idleto);
1457 sosp->so_snd.sb_flags &= ~SB_SPLICE;
1458
1459 mtx_enter(&so->so_rcv.sb_mtx);
1460 so->so_rcv.sb_flags &= ~SB_SPLICE;
1461 so->so_sp->ssp_socket = sosp->so_sp->ssp_soback = NULL;
1462 mtx_leave(&so->so_rcv.sb_mtx);
1463
1464 /* Do not wakeup a socket that is about to be freed. */
1465 if ((freeing & SOSP_FREEING_READ) == 0 && soreadable(so))
1466 sorwakeup(so);
1467 if ((freeing & SOSP_FREEING_WRITE) == 0 && sowriteable(sosp))
1468 sowwakeup(sosp);
1469 }
1470
1471 void
soidle(void * arg)1472 soidle(void *arg)
1473 {
1474 struct socket *so = arg;
1475
1476 solock(so);
1477 if (so->so_rcv.sb_flags & SB_SPLICE) {
1478 so->so_error = ETIMEDOUT;
1479 sounsplice(so, so->so_sp->ssp_socket, 0);
1480 }
1481 sounlock(so);
1482 }
1483
1484 void
sotask(void * arg)1485 sotask(void *arg)
1486 {
1487 struct socket *so = arg;
1488
1489 solock(so);
1490 if (so->so_rcv.sb_flags & SB_SPLICE) {
1491 /*
1492 * We may not sleep here as sofree() and unsplice() may be
1493 * called from softnet interrupt context. This would remove
1494 * the socket during somove().
1495 */
1496 somove(so, M_DONTWAIT);
1497 }
1498 sounlock(so);
1499
1500 /* Avoid user land starvation. */
1501 yield();
1502 }
1503
1504 /*
1505 * The socket splicing task or idle timeout may sleep while grabbing the net
1506 * lock. As sofree() can be called anytime, sotask() or soidle() could access
1507 * the socket memory of a freed socket after wakeup. So delay the pool_put()
1508 * after all pending socket splicing tasks or timeouts have finished. Do this
1509 * by scheduling it on the same threads.
1510 */
1511 void
soreaper(void * arg)1512 soreaper(void *arg)
1513 {
1514 struct socket *so = arg;
1515
1516 /* Reuse splice task, sounsplice() has been called before. */
1517 task_set(&so->so_sp->ssp_task, soput, so);
1518 task_add(sosplice_taskq, &so->so_sp->ssp_task);
1519 }
1520
1521 void
soput(void * arg)1522 soput(void *arg)
1523 {
1524 struct socket *so = arg;
1525
1526 pool_put(&sosplice_pool, so->so_sp);
1527 pool_put(&socket_pool, so);
1528 }
1529
1530 /*
1531 * Move data from receive buffer of spliced source socket to send
1532 * buffer of drain socket. Try to move as much as possible in one
1533 * big chunk. It is a TCP only implementation.
1534 * Return value 0 means splicing has been finished, 1 continue.
1535 */
1536 int
somove(struct socket * so,int wait)1537 somove(struct socket *so, int wait)
1538 {
1539 struct socket *sosp = so->so_sp->ssp_socket;
1540 struct mbuf *m, **mp, *nextrecord;
1541 u_long len, off, oobmark;
1542 long space;
1543 int error = 0, maxreached = 0;
1544 unsigned int rcvstate;
1545
1546 soassertlocked(so);
1547
1548 nextpkt:
1549 if (so->so_error) {
1550 error = so->so_error;
1551 goto release;
1552 }
1553 if (sosp->so_snd.sb_state & SS_CANTSENDMORE) {
1554 error = EPIPE;
1555 goto release;
1556 }
1557 if (sosp->so_error && sosp->so_error != ETIMEDOUT &&
1558 sosp->so_error != EFBIG && sosp->so_error != ELOOP) {
1559 error = sosp->so_error;
1560 goto release;
1561 }
1562 if ((sosp->so_state & SS_ISCONNECTED) == 0)
1563 goto release;
1564
1565 /* Calculate how many bytes can be copied now. */
1566 len = so->so_rcv.sb_datacc;
1567 if (so->so_splicemax) {
1568 KASSERT(so->so_splicelen < so->so_splicemax);
1569 if (so->so_splicemax <= so->so_splicelen + len) {
1570 len = so->so_splicemax - so->so_splicelen;
1571 maxreached = 1;
1572 }
1573 }
1574 space = sbspace(sosp, &sosp->so_snd);
1575 if (so->so_oobmark && so->so_oobmark < len &&
1576 so->so_oobmark < space + 1024)
1577 space += 1024;
1578 if (space <= 0) {
1579 maxreached = 0;
1580 goto release;
1581 }
1582 if (space < len) {
1583 maxreached = 0;
1584 if (space < sosp->so_snd.sb_lowat)
1585 goto release;
1586 len = space;
1587 }
1588 sosp->so_snd.sb_state |= SS_ISSENDING;
1589
1590 SBLASTRECORDCHK(&so->so_rcv, "somove 1");
1591 SBLASTMBUFCHK(&so->so_rcv, "somove 1");
1592 m = so->so_rcv.sb_mb;
1593 if (m == NULL)
1594 goto release;
1595 nextrecord = m->m_nextpkt;
1596
1597 /* Drop address and control information not used with splicing. */
1598 if (so->so_proto->pr_flags & PR_ADDR) {
1599 #ifdef DIAGNOSTIC
1600 if (m->m_type != MT_SONAME)
1601 panic("somove soname: so %p, so_type %d, m %p, "
1602 "m_type %d", so, so->so_type, m, m->m_type);
1603 #endif
1604 m = m->m_next;
1605 }
1606 while (m && m->m_type == MT_CONTROL)
1607 m = m->m_next;
1608 if (m == NULL) {
1609 sbdroprecord(so, &so->so_rcv);
1610 if (so->so_proto->pr_flags & PR_WANTRCVD)
1611 pru_rcvd(so);
1612 goto nextpkt;
1613 }
1614
1615 /*
1616 * By splicing sockets connected to localhost, userland might create a
1617 * loop. Dissolve splicing with error if loop is detected by counter.
1618 *
1619 * If we deal with looped broadcast/multicast packet we bail out with
1620 * no error to suppress splice termination.
1621 */
1622 if ((m->m_flags & M_PKTHDR) &&
1623 ((m->m_pkthdr.ph_loopcnt++ >= M_MAXLOOP) ||
1624 ((m->m_flags & M_LOOP) && (m->m_flags & (M_BCAST|M_MCAST))))) {
1625 error = ELOOP;
1626 goto release;
1627 }
1628
1629 if (so->so_proto->pr_flags & PR_ATOMIC) {
1630 if ((m->m_flags & M_PKTHDR) == 0)
1631 panic("somove !PKTHDR: so %p, so_type %d, m %p, "
1632 "m_type %d", so, so->so_type, m, m->m_type);
1633 if (sosp->so_snd.sb_hiwat < m->m_pkthdr.len) {
1634 error = EMSGSIZE;
1635 goto release;
1636 }
1637 if (len < m->m_pkthdr.len)
1638 goto release;
1639 if (m->m_pkthdr.len < len) {
1640 maxreached = 0;
1641 len = m->m_pkthdr.len;
1642 }
1643 /*
1644 * Throw away the name mbuf after it has been assured
1645 * that the whole first record can be processed.
1646 */
1647 m = so->so_rcv.sb_mb;
1648 sbfree(so, &so->so_rcv, m);
1649 so->so_rcv.sb_mb = m_free(m);
1650 sbsync(&so->so_rcv, nextrecord);
1651 }
1652 /*
1653 * Throw away the control mbufs after it has been assured
1654 * that the whole first record can be processed.
1655 */
1656 m = so->so_rcv.sb_mb;
1657 while (m && m->m_type == MT_CONTROL) {
1658 sbfree(so, &so->so_rcv, m);
1659 so->so_rcv.sb_mb = m_free(m);
1660 m = so->so_rcv.sb_mb;
1661 sbsync(&so->so_rcv, nextrecord);
1662 }
1663
1664 SBLASTRECORDCHK(&so->so_rcv, "somove 2");
1665 SBLASTMBUFCHK(&so->so_rcv, "somove 2");
1666
1667 /* Take at most len mbufs out of receive buffer. */
1668 for (off = 0, mp = &m; off <= len && *mp;
1669 off += (*mp)->m_len, mp = &(*mp)->m_next) {
1670 u_long size = len - off;
1671
1672 #ifdef DIAGNOSTIC
1673 if ((*mp)->m_type != MT_DATA && (*mp)->m_type != MT_HEADER)
1674 panic("somove type: so %p, so_type %d, m %p, "
1675 "m_type %d", so, so->so_type, *mp, (*mp)->m_type);
1676 #endif
1677 if ((*mp)->m_len > size) {
1678 /*
1679 * Move only a partial mbuf at maximum splice length or
1680 * if the drain buffer is too small for this large mbuf.
1681 */
1682 if (!maxreached && so->so_snd.sb_datacc > 0) {
1683 len -= size;
1684 break;
1685 }
1686 *mp = m_copym(so->so_rcv.sb_mb, 0, size, wait);
1687 if (*mp == NULL) {
1688 len -= size;
1689 break;
1690 }
1691 so->so_rcv.sb_mb->m_data += size;
1692 so->so_rcv.sb_mb->m_len -= size;
1693 so->so_rcv.sb_cc -= size;
1694 so->so_rcv.sb_datacc -= size;
1695 } else {
1696 *mp = so->so_rcv.sb_mb;
1697 sbfree(so, &so->so_rcv, *mp);
1698 so->so_rcv.sb_mb = (*mp)->m_next;
1699 sbsync(&so->so_rcv, nextrecord);
1700 }
1701 }
1702 *mp = NULL;
1703
1704 SBLASTRECORDCHK(&so->so_rcv, "somove 3");
1705 SBLASTMBUFCHK(&so->so_rcv, "somove 3");
1706 SBCHECK(so, &so->so_rcv);
1707 if (m == NULL)
1708 goto release;
1709 m->m_nextpkt = NULL;
1710 if (m->m_flags & M_PKTHDR) {
1711 m_resethdr(m);
1712 m->m_pkthdr.len = len;
1713 }
1714
1715 /* Send window update to source peer as receive buffer has changed. */
1716 if (so->so_proto->pr_flags & PR_WANTRCVD)
1717 pru_rcvd(so);
1718
1719 /* Receive buffer did shrink by len bytes, adjust oob. */
1720 mtx_enter(&so->so_rcv.sb_mtx);
1721 rcvstate = so->so_rcv.sb_state;
1722 so->so_rcv.sb_state &= ~SS_RCVATMARK;
1723 oobmark = so->so_oobmark;
1724 so->so_oobmark = oobmark > len ? oobmark - len : 0;
1725 if (oobmark) {
1726 if (oobmark == len)
1727 so->so_rcv.sb_state |= SS_RCVATMARK;
1728 if (oobmark >= len)
1729 oobmark = 0;
1730 }
1731 mtx_leave(&so->so_rcv.sb_mtx);
1732
1733 /*
1734 * Handle oob data. If any malloc fails, ignore error.
1735 * TCP urgent data is not very reliable anyway.
1736 */
1737 while (((rcvstate & SS_RCVATMARK) || oobmark) &&
1738 (so->so_options & SO_OOBINLINE)) {
1739 struct mbuf *o = NULL;
1740
1741 if (rcvstate & SS_RCVATMARK) {
1742 o = m_get(wait, MT_DATA);
1743 rcvstate &= ~SS_RCVATMARK;
1744 } else if (oobmark) {
1745 o = m_split(m, oobmark, wait);
1746 if (o) {
1747 error = pru_send(sosp, m, NULL, NULL);
1748 if (error) {
1749 if (sosp->so_snd.sb_state &
1750 SS_CANTSENDMORE)
1751 error = EPIPE;
1752 m_freem(o);
1753 goto release;
1754 }
1755 len -= oobmark;
1756 so->so_splicelen += oobmark;
1757 m = o;
1758 o = m_get(wait, MT_DATA);
1759 }
1760 oobmark = 0;
1761 }
1762 if (o) {
1763 o->m_len = 1;
1764 *mtod(o, caddr_t) = *mtod(m, caddr_t);
1765 error = pru_sendoob(sosp, o, NULL, NULL);
1766 if (error) {
1767 if (sosp->so_snd.sb_state & SS_CANTSENDMORE)
1768 error = EPIPE;
1769 m_freem(m);
1770 goto release;
1771 }
1772 len -= 1;
1773 so->so_splicelen += 1;
1774 if (oobmark) {
1775 oobmark -= 1;
1776 if (oobmark == 0)
1777 rcvstate |= SS_RCVATMARK;
1778 }
1779 m_adj(m, 1);
1780 }
1781 }
1782
1783 /* Append all remaining data to drain socket. */
1784 if (so->so_rcv.sb_cc == 0 || maxreached)
1785 sosp->so_snd.sb_state &= ~SS_ISSENDING;
1786 error = pru_send(sosp, m, NULL, NULL);
1787 if (error) {
1788 if (sosp->so_snd.sb_state & SS_CANTSENDMORE)
1789 error = EPIPE;
1790 goto release;
1791 }
1792 so->so_splicelen += len;
1793
1794 /* Move several packets if possible. */
1795 if (!maxreached && nextrecord)
1796 goto nextpkt;
1797
1798 release:
1799 sosp->so_snd.sb_state &= ~SS_ISSENDING;
1800 if (!error && maxreached && so->so_splicemax == so->so_splicelen)
1801 error = EFBIG;
1802 if (error)
1803 so->so_error = error;
1804 if (((so->so_rcv.sb_state & SS_CANTRCVMORE) &&
1805 so->so_rcv.sb_cc == 0) ||
1806 (sosp->so_snd.sb_state & SS_CANTSENDMORE) ||
1807 maxreached || error) {
1808 sounsplice(so, sosp, 0);
1809 return (0);
1810 }
1811 if (timerisset(&so->so_idletv))
1812 timeout_add_tv(&so->so_idleto, &so->so_idletv);
1813 return (1);
1814 }
1815
1816 #endif /* SOCKET_SPLICE */
1817
1818 void
sorwakeup(struct socket * so)1819 sorwakeup(struct socket *so)
1820 {
1821 if ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0)
1822 soassertlocked_readonly(so);
1823
1824 #ifdef SOCKET_SPLICE
1825 if (so->so_rcv.sb_flags & SB_SPLICE) {
1826 /*
1827 * TCP has a sendbuffer that can handle multiple packets
1828 * at once. So queue the stream a bit to accumulate data.
1829 * The sosplice thread will call somove() later and send
1830 * the packets calling tcp_output() only once.
1831 * In the UDP case, send out the packets immediately.
1832 * Using a thread would make things slower.
1833 */
1834 if (so->so_proto->pr_flags & PR_WANTRCVD)
1835 task_add(sosplice_taskq, &so->so_splicetask);
1836 else
1837 somove(so, M_DONTWAIT);
1838 }
1839 if (isspliced(so))
1840 return;
1841 #endif
1842 sowakeup(so, &so->so_rcv);
1843 if (so->so_upcall)
1844 (*(so->so_upcall))(so, so->so_upcallarg, M_DONTWAIT);
1845 }
1846
1847 void
sowwakeup(struct socket * so)1848 sowwakeup(struct socket *so)
1849 {
1850 if ((so->so_snd.sb_flags & SB_MTXLOCK) == 0)
1851 soassertlocked_readonly(so);
1852
1853 #ifdef SOCKET_SPLICE
1854 if (so->so_snd.sb_flags & SB_SPLICE)
1855 task_add(sosplice_taskq, &so->so_sp->ssp_soback->so_splicetask);
1856 if (issplicedback(so))
1857 return;
1858 #endif
1859 sowakeup(so, &so->so_snd);
1860 }
1861
1862 int
sosetopt(struct socket * so,int level,int optname,struct mbuf * m)1863 sosetopt(struct socket *so, int level, int optname, struct mbuf *m)
1864 {
1865 int error = 0;
1866
1867 if (level != SOL_SOCKET) {
1868 if (so->so_proto->pr_ctloutput) {
1869 solock(so);
1870 error = (*so->so_proto->pr_ctloutput)(PRCO_SETOPT, so,
1871 level, optname, m);
1872 sounlock(so);
1873 return (error);
1874 }
1875 error = ENOPROTOOPT;
1876 } else {
1877 switch (optname) {
1878
1879 case SO_LINGER:
1880 if (m == NULL || m->m_len != sizeof (struct linger) ||
1881 mtod(m, struct linger *)->l_linger < 0 ||
1882 mtod(m, struct linger *)->l_linger > SHRT_MAX)
1883 return (EINVAL);
1884
1885 solock(so);
1886 so->so_linger = mtod(m, struct linger *)->l_linger;
1887 if (*mtod(m, int *))
1888 so->so_options |= optname;
1889 else
1890 so->so_options &= ~optname;
1891 sounlock(so);
1892
1893 break;
1894 case SO_BINDANY:
1895 if ((error = suser(curproc)) != 0) /* XXX */
1896 return (error);
1897 /* FALLTHROUGH */
1898
1899 case SO_DEBUG:
1900 case SO_KEEPALIVE:
1901 case SO_USELOOPBACK:
1902 case SO_BROADCAST:
1903 case SO_REUSEADDR:
1904 case SO_REUSEPORT:
1905 case SO_OOBINLINE:
1906 case SO_TIMESTAMP:
1907 case SO_ZEROIZE:
1908 if (m == NULL || m->m_len < sizeof (int))
1909 return (EINVAL);
1910
1911 solock(so);
1912 if (*mtod(m, int *))
1913 so->so_options |= optname;
1914 else
1915 so->so_options &= ~optname;
1916 sounlock(so);
1917
1918 break;
1919 case SO_DONTROUTE:
1920 if (m == NULL || m->m_len < sizeof (int))
1921 return (EINVAL);
1922 if (*mtod(m, int *))
1923 error = EOPNOTSUPP;
1924 break;
1925
1926 case SO_SNDBUF:
1927 case SO_RCVBUF:
1928 case SO_SNDLOWAT:
1929 case SO_RCVLOWAT:
1930 {
1931 struct sockbuf *sb = (optname == SO_SNDBUF ||
1932 optname == SO_SNDLOWAT ?
1933 &so->so_snd : &so->so_rcv);
1934 u_long cnt;
1935
1936 if (m == NULL || m->m_len < sizeof (int))
1937 return (EINVAL);
1938 cnt = *mtod(m, int *);
1939 if ((long)cnt <= 0)
1940 cnt = 1;
1941
1942 if (((sb->sb_flags & SB_MTXLOCK) == 0))
1943 solock(so);
1944 mtx_enter(&sb->sb_mtx);
1945
1946 switch (optname) {
1947 case SO_SNDBUF:
1948 case SO_RCVBUF:
1949 if (sb->sb_state &
1950 (SS_CANTSENDMORE | SS_CANTRCVMORE)) {
1951 error = EINVAL;
1952 break;
1953 }
1954 if (sbcheckreserve(cnt, sb->sb_wat) ||
1955 sbreserve(so, sb, cnt)) {
1956 error = ENOBUFS;
1957 break;
1958 }
1959 sb->sb_wat = cnt;
1960 break;
1961 case SO_SNDLOWAT:
1962 case SO_RCVLOWAT:
1963 sb->sb_lowat = (cnt > sb->sb_hiwat) ?
1964 sb->sb_hiwat : cnt;
1965 break;
1966 }
1967
1968 mtx_leave(&sb->sb_mtx);
1969 if (((sb->sb_flags & SB_MTXLOCK) == 0))
1970 sounlock(so);
1971
1972 break;
1973 }
1974
1975 case SO_SNDTIMEO:
1976 case SO_RCVTIMEO:
1977 {
1978 struct sockbuf *sb = (optname == SO_SNDTIMEO ?
1979 &so->so_snd : &so->so_rcv);
1980 struct timeval tv;
1981 uint64_t nsecs;
1982
1983 if (m == NULL || m->m_len < sizeof (tv))
1984 return (EINVAL);
1985 memcpy(&tv, mtod(m, struct timeval *), sizeof tv);
1986 if (!timerisvalid(&tv))
1987 return (EINVAL);
1988 nsecs = TIMEVAL_TO_NSEC(&tv);
1989 if (nsecs == UINT64_MAX)
1990 return (EDOM);
1991 if (nsecs == 0)
1992 nsecs = INFSLP;
1993
1994 mtx_enter(&sb->sb_mtx);
1995 sb->sb_timeo_nsecs = nsecs;
1996 mtx_leave(&sb->sb_mtx);
1997 break;
1998 }
1999
2000 case SO_RTABLE:
2001 if (so->so_proto->pr_domain &&
2002 so->so_proto->pr_domain->dom_protosw &&
2003 so->so_proto->pr_ctloutput) {
2004 const struct domain *dom =
2005 so->so_proto->pr_domain;
2006
2007 level = dom->dom_protosw->pr_protocol;
2008 solock(so);
2009 error = (*so->so_proto->pr_ctloutput)
2010 (PRCO_SETOPT, so, level, optname, m);
2011 sounlock(so);
2012 } else
2013 error = ENOPROTOOPT;
2014 break;
2015 #ifdef SOCKET_SPLICE
2016 case SO_SPLICE:
2017 if (m == NULL) {
2018 error = sosplice(so, -1, 0, NULL);
2019 } else if (m->m_len < sizeof(int)) {
2020 error = EINVAL;
2021 } else if (m->m_len < sizeof(struct splice)) {
2022 error = sosplice(so, *mtod(m, int *), 0, NULL);
2023 } else {
2024 error = sosplice(so,
2025 mtod(m, struct splice *)->sp_fd,
2026 mtod(m, struct splice *)->sp_max,
2027 &mtod(m, struct splice *)->sp_idle);
2028 }
2029 break;
2030 #endif /* SOCKET_SPLICE */
2031
2032 default:
2033 error = ENOPROTOOPT;
2034 break;
2035 }
2036 }
2037
2038 return (error);
2039 }
2040
2041 int
sogetopt(struct socket * so,int level,int optname,struct mbuf * m)2042 sogetopt(struct socket *so, int level, int optname, struct mbuf *m)
2043 {
2044 int error = 0;
2045
2046 if (level != SOL_SOCKET) {
2047 if (so->so_proto->pr_ctloutput) {
2048 m->m_len = 0;
2049
2050 solock(so);
2051 error = (*so->so_proto->pr_ctloutput)(PRCO_GETOPT, so,
2052 level, optname, m);
2053 sounlock(so);
2054 return (error);
2055 } else
2056 return (ENOPROTOOPT);
2057 } else {
2058 m->m_len = sizeof (int);
2059
2060 switch (optname) {
2061
2062 case SO_LINGER:
2063 m->m_len = sizeof (struct linger);
2064 solock_shared(so);
2065 mtod(m, struct linger *)->l_onoff =
2066 so->so_options & SO_LINGER;
2067 mtod(m, struct linger *)->l_linger = so->so_linger;
2068 sounlock_shared(so);
2069 break;
2070
2071 case SO_BINDANY:
2072 case SO_USELOOPBACK:
2073 case SO_DEBUG:
2074 case SO_KEEPALIVE:
2075 case SO_REUSEADDR:
2076 case SO_REUSEPORT:
2077 case SO_BROADCAST:
2078 case SO_OOBINLINE:
2079 case SO_ACCEPTCONN:
2080 case SO_TIMESTAMP:
2081 case SO_ZEROIZE:
2082 *mtod(m, int *) = so->so_options & optname;
2083 break;
2084
2085 case SO_DONTROUTE:
2086 *mtod(m, int *) = 0;
2087 break;
2088
2089 case SO_TYPE:
2090 *mtod(m, int *) = so->so_type;
2091 break;
2092
2093 case SO_ERROR:
2094 solock(so);
2095 *mtod(m, int *) = so->so_error;
2096 so->so_error = 0;
2097 sounlock(so);
2098
2099 break;
2100
2101 case SO_DOMAIN:
2102 *mtod(m, int *) = so->so_proto->pr_domain->dom_family;
2103 break;
2104
2105 case SO_PROTOCOL:
2106 *mtod(m, int *) = so->so_proto->pr_protocol;
2107 break;
2108
2109 case SO_SNDBUF:
2110 *mtod(m, int *) = so->so_snd.sb_hiwat;
2111 break;
2112
2113 case SO_RCVBUF:
2114 *mtod(m, int *) = so->so_rcv.sb_hiwat;
2115 break;
2116
2117 case SO_SNDLOWAT:
2118 *mtod(m, int *) = so->so_snd.sb_lowat;
2119 break;
2120
2121 case SO_RCVLOWAT:
2122 *mtod(m, int *) = so->so_rcv.sb_lowat;
2123 break;
2124
2125 case SO_SNDTIMEO:
2126 case SO_RCVTIMEO:
2127 {
2128 struct sockbuf *sb = (optname == SO_SNDTIMEO ?
2129 &so->so_snd : &so->so_rcv);
2130 struct timeval tv;
2131 uint64_t nsecs;
2132
2133 mtx_enter(&sb->sb_mtx);
2134 nsecs = sb->sb_timeo_nsecs;
2135 mtx_leave(&sb->sb_mtx);
2136
2137 m->m_len = sizeof(struct timeval);
2138 memset(&tv, 0, sizeof(tv));
2139 if (nsecs != INFSLP)
2140 NSEC_TO_TIMEVAL(nsecs, &tv);
2141 memcpy(mtod(m, struct timeval *), &tv, sizeof tv);
2142 break;
2143 }
2144
2145 case SO_RTABLE:
2146 if (so->so_proto->pr_domain &&
2147 so->so_proto->pr_domain->dom_protosw &&
2148 so->so_proto->pr_ctloutput) {
2149 const struct domain *dom =
2150 so->so_proto->pr_domain;
2151
2152 level = dom->dom_protosw->pr_protocol;
2153 solock(so);
2154 error = (*so->so_proto->pr_ctloutput)
2155 (PRCO_GETOPT, so, level, optname, m);
2156 sounlock(so);
2157 if (error)
2158 return (error);
2159 break;
2160 }
2161 return (ENOPROTOOPT);
2162
2163 #ifdef SOCKET_SPLICE
2164 case SO_SPLICE:
2165 {
2166 off_t len;
2167
2168 m->m_len = sizeof(off_t);
2169 solock_shared(so);
2170 len = so->so_sp ? so->so_sp->ssp_len : 0;
2171 sounlock_shared(so);
2172 memcpy(mtod(m, off_t *), &len, sizeof(off_t));
2173 break;
2174 }
2175 #endif /* SOCKET_SPLICE */
2176
2177 case SO_PEERCRED:
2178 if (so->so_proto->pr_protocol == AF_UNIX) {
2179 struct unpcb *unp = sotounpcb(so);
2180
2181 solock(so);
2182 if (unp->unp_flags & UNP_FEIDS) {
2183 m->m_len = sizeof(unp->unp_connid);
2184 memcpy(mtod(m, caddr_t),
2185 &(unp->unp_connid), m->m_len);
2186 sounlock(so);
2187 break;
2188 }
2189 sounlock(so);
2190
2191 return (ENOTCONN);
2192 }
2193 return (EOPNOTSUPP);
2194
2195 default:
2196 return (ENOPROTOOPT);
2197 }
2198 return (0);
2199 }
2200 }
2201
2202 void
sohasoutofband(struct socket * so)2203 sohasoutofband(struct socket *so)
2204 {
2205 pgsigio(&so->so_sigio, SIGURG, 0);
2206 knote(&so->so_rcv.sb_klist, 0);
2207 }
2208
2209 void
sofilt_lock(struct socket * so,struct sockbuf * sb)2210 sofilt_lock(struct socket *so, struct sockbuf *sb)
2211 {
2212 switch (so->so_proto->pr_domain->dom_family) {
2213 case PF_INET:
2214 case PF_INET6:
2215 NET_LOCK_SHARED();
2216 break;
2217 default:
2218 rw_enter_write(&so->so_lock);
2219 break;
2220 }
2221
2222 mtx_enter(&sb->sb_mtx);
2223 }
2224
2225 void
sofilt_unlock(struct socket * so,struct sockbuf * sb)2226 sofilt_unlock(struct socket *so, struct sockbuf *sb)
2227 {
2228 mtx_leave(&sb->sb_mtx);
2229
2230 switch (so->so_proto->pr_domain->dom_family) {
2231 case PF_INET:
2232 case PF_INET6:
2233 NET_UNLOCK_SHARED();
2234 break;
2235 default:
2236 rw_exit_write(&so->so_lock);
2237 break;
2238 }
2239 }
2240
2241 int
soo_kqfilter(struct file * fp,struct knote * kn)2242 soo_kqfilter(struct file *fp, struct knote *kn)
2243 {
2244 struct socket *so = kn->kn_fp->f_data;
2245 struct sockbuf *sb;
2246
2247 switch (kn->kn_filter) {
2248 case EVFILT_READ:
2249 kn->kn_fop = &soread_filtops;
2250 sb = &so->so_rcv;
2251 break;
2252 case EVFILT_WRITE:
2253 kn->kn_fop = &sowrite_filtops;
2254 sb = &so->so_snd;
2255 break;
2256 case EVFILT_EXCEPT:
2257 kn->kn_fop = &soexcept_filtops;
2258 sb = &so->so_rcv;
2259 break;
2260 default:
2261 return (EINVAL);
2262 }
2263
2264 klist_insert(&sb->sb_klist, kn);
2265
2266 return (0);
2267 }
2268
2269 void
filt_sordetach(struct knote * kn)2270 filt_sordetach(struct knote *kn)
2271 {
2272 struct socket *so = kn->kn_fp->f_data;
2273
2274 klist_remove(&so->so_rcv.sb_klist, kn);
2275 }
2276
2277 int
filt_soread(struct knote * kn,long hint)2278 filt_soread(struct knote *kn, long hint)
2279 {
2280 struct socket *so = kn->kn_fp->f_data;
2281 int rv = 0;
2282
2283 MUTEX_ASSERT_LOCKED(&so->so_rcv.sb_mtx);
2284 if ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0)
2285 soassertlocked_readonly(so);
2286
2287 if (so->so_options & SO_ACCEPTCONN) {
2288 if (so->so_rcv.sb_flags & SB_MTXLOCK)
2289 soassertlocked_readonly(so);
2290
2291 kn->kn_data = so->so_qlen;
2292 rv = (kn->kn_data != 0);
2293
2294 if (kn->kn_flags & (__EV_POLL | __EV_SELECT)) {
2295 if (so->so_state & SS_ISDISCONNECTED) {
2296 kn->kn_flags |= __EV_HUP;
2297 rv = 1;
2298 } else {
2299 rv = soreadable(so);
2300 }
2301 }
2302
2303 return rv;
2304 }
2305
2306 kn->kn_data = so->so_rcv.sb_cc;
2307 #ifdef SOCKET_SPLICE
2308 if (isspliced(so)) {
2309 rv = 0;
2310 } else
2311 #endif /* SOCKET_SPLICE */
2312 if (so->so_rcv.sb_state & SS_CANTRCVMORE) {
2313 kn->kn_flags |= EV_EOF;
2314 if (kn->kn_flags & __EV_POLL) {
2315 if (so->so_state & SS_ISDISCONNECTED)
2316 kn->kn_flags |= __EV_HUP;
2317 }
2318 kn->kn_fflags = so->so_error;
2319 rv = 1;
2320 } else if (so->so_error) {
2321 rv = 1;
2322 } else if (kn->kn_sfflags & NOTE_LOWAT) {
2323 rv = (kn->kn_data >= kn->kn_sdata);
2324 } else {
2325 rv = (kn->kn_data >= so->so_rcv.sb_lowat);
2326 }
2327
2328 return rv;
2329 }
2330
2331 void
filt_sowdetach(struct knote * kn)2332 filt_sowdetach(struct knote *kn)
2333 {
2334 struct socket *so = kn->kn_fp->f_data;
2335
2336 klist_remove(&so->so_snd.sb_klist, kn);
2337 }
2338
2339 int
filt_sowrite(struct knote * kn,long hint)2340 filt_sowrite(struct knote *kn, long hint)
2341 {
2342 struct socket *so = kn->kn_fp->f_data;
2343 int rv;
2344
2345 MUTEX_ASSERT_LOCKED(&so->so_snd.sb_mtx);
2346 if ((so->so_snd.sb_flags & SB_MTXLOCK) == 0)
2347 soassertlocked_readonly(so);
2348
2349 kn->kn_data = sbspace(so, &so->so_snd);
2350 if (so->so_snd.sb_state & SS_CANTSENDMORE) {
2351 kn->kn_flags |= EV_EOF;
2352 if (kn->kn_flags & __EV_POLL) {
2353 if (so->so_state & SS_ISDISCONNECTED)
2354 kn->kn_flags |= __EV_HUP;
2355 }
2356 kn->kn_fflags = so->so_error;
2357 rv = 1;
2358 } else if (so->so_error) {
2359 rv = 1;
2360 } else if (((so->so_state & SS_ISCONNECTED) == 0) &&
2361 (so->so_proto->pr_flags & PR_CONNREQUIRED)) {
2362 rv = 0;
2363 } else if (kn->kn_sfflags & NOTE_LOWAT) {
2364 rv = (kn->kn_data >= kn->kn_sdata);
2365 } else {
2366 rv = (kn->kn_data >= so->so_snd.sb_lowat);
2367 }
2368
2369 return (rv);
2370 }
2371
2372 int
filt_soexcept(struct knote * kn,long hint)2373 filt_soexcept(struct knote *kn, long hint)
2374 {
2375 struct socket *so = kn->kn_fp->f_data;
2376 int rv = 0;
2377
2378 MUTEX_ASSERT_LOCKED(&so->so_rcv.sb_mtx);
2379 if ((so->so_rcv.sb_flags & SB_MTXLOCK) == 0)
2380 soassertlocked_readonly(so);
2381
2382 #ifdef SOCKET_SPLICE
2383 if (isspliced(so)) {
2384 rv = 0;
2385 } else
2386 #endif /* SOCKET_SPLICE */
2387 if (kn->kn_sfflags & NOTE_OOB) {
2388 if (so->so_oobmark || (so->so_rcv.sb_state & SS_RCVATMARK)) {
2389 kn->kn_fflags |= NOTE_OOB;
2390 kn->kn_data -= so->so_oobmark;
2391 rv = 1;
2392 }
2393 }
2394
2395 if (kn->kn_flags & __EV_POLL) {
2396 if (so->so_state & SS_ISDISCONNECTED) {
2397 kn->kn_flags |= __EV_HUP;
2398 rv = 1;
2399 }
2400 }
2401
2402 return rv;
2403 }
2404
2405 int
filt_sowmodify(struct kevent * kev,struct knote * kn)2406 filt_sowmodify(struct kevent *kev, struct knote *kn)
2407 {
2408 struct socket *so = kn->kn_fp->f_data;
2409 int rv;
2410
2411 sofilt_lock(so, &so->so_snd);
2412 rv = knote_modify(kev, kn);
2413 sofilt_unlock(so, &so->so_snd);
2414
2415 return (rv);
2416 }
2417
2418 int
filt_sowprocess(struct knote * kn,struct kevent * kev)2419 filt_sowprocess(struct knote *kn, struct kevent *kev)
2420 {
2421 struct socket *so = kn->kn_fp->f_data;
2422 int rv;
2423
2424 sofilt_lock(so, &so->so_snd);
2425 rv = knote_process(kn, kev);
2426 sofilt_unlock(so, &so->so_snd);
2427
2428 return (rv);
2429 }
2430
2431 int
filt_sormodify(struct kevent * kev,struct knote * kn)2432 filt_sormodify(struct kevent *kev, struct knote *kn)
2433 {
2434 struct socket *so = kn->kn_fp->f_data;
2435 int rv;
2436
2437 sofilt_lock(so, &so->so_rcv);
2438 rv = knote_modify(kev, kn);
2439 sofilt_unlock(so, &so->so_rcv);
2440
2441 return (rv);
2442 }
2443
2444 int
filt_sorprocess(struct knote * kn,struct kevent * kev)2445 filt_sorprocess(struct knote *kn, struct kevent *kev)
2446 {
2447 struct socket *so = kn->kn_fp->f_data;
2448 int rv;
2449
2450 sofilt_lock(so, &so->so_rcv);
2451 rv = knote_process(kn, kev);
2452 sofilt_unlock(so, &so->so_rcv);
2453
2454 return (rv);
2455 }
2456
2457 #ifdef DDB
2458 void
2459 sobuf_print(struct sockbuf *,
2460 int (*)(const char *, ...) __attribute__((__format__(__kprintf__,1,2))));
2461
2462 void
sobuf_print(struct sockbuf * sb,int (* pr)(const char *,...))2463 sobuf_print(struct sockbuf *sb,
2464 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2))))
2465 {
2466 (*pr)("\tsb_cc: %lu\n", sb->sb_cc);
2467 (*pr)("\tsb_datacc: %lu\n", sb->sb_datacc);
2468 (*pr)("\tsb_hiwat: %lu\n", sb->sb_hiwat);
2469 (*pr)("\tsb_wat: %lu\n", sb->sb_wat);
2470 (*pr)("\tsb_mbcnt: %lu\n", sb->sb_mbcnt);
2471 (*pr)("\tsb_mbmax: %lu\n", sb->sb_mbmax);
2472 (*pr)("\tsb_lowat: %ld\n", sb->sb_lowat);
2473 (*pr)("\tsb_mb: %p\n", sb->sb_mb);
2474 (*pr)("\tsb_mbtail: %p\n", sb->sb_mbtail);
2475 (*pr)("\tsb_lastrecord: %p\n", sb->sb_lastrecord);
2476 (*pr)("\tsb_sel: ...\n");
2477 (*pr)("\tsb_flags: %04x\n", sb->sb_flags);
2478 (*pr)("\tsb_state: %04x\n", sb->sb_state);
2479 (*pr)("\tsb_timeo_nsecs: %llu\n", sb->sb_timeo_nsecs);
2480 }
2481
2482 void
so_print(void * v,int (* pr)(const char *,...))2483 so_print(void *v,
2484 int (*pr)(const char *, ...) __attribute__((__format__(__kprintf__,1,2))))
2485 {
2486 struct socket *so = v;
2487
2488 (*pr)("socket %p\n", so);
2489 (*pr)("so_type: %i\n", so->so_type);
2490 (*pr)("so_options: 0x%04x\n", so->so_options); /* %b */
2491 (*pr)("so_linger: %i\n", so->so_linger);
2492 (*pr)("so_state: 0x%04x\n", so->so_state);
2493 (*pr)("so_pcb: %p\n", so->so_pcb);
2494 (*pr)("so_proto: %p\n", so->so_proto);
2495 (*pr)("so_sigio: %p\n", so->so_sigio.sir_sigio);
2496
2497 (*pr)("so_head: %p\n", so->so_head);
2498 (*pr)("so_onq: %p\n", so->so_onq);
2499 (*pr)("so_q0: @%p first: %p\n", &so->so_q0, TAILQ_FIRST(&so->so_q0));
2500 (*pr)("so_q: @%p first: %p\n", &so->so_q, TAILQ_FIRST(&so->so_q));
2501 (*pr)("so_eq: next: %p\n", TAILQ_NEXT(so, so_qe));
2502 (*pr)("so_q0len: %i\n", so->so_q0len);
2503 (*pr)("so_qlen: %i\n", so->so_qlen);
2504 (*pr)("so_qlimit: %i\n", so->so_qlimit);
2505 (*pr)("so_timeo: %i\n", so->so_timeo);
2506 (*pr)("so_obmark: %lu\n", so->so_oobmark);
2507
2508 (*pr)("so_sp: %p\n", so->so_sp);
2509 if (so->so_sp != NULL) {
2510 (*pr)("\tssp_socket: %p\n", so->so_sp->ssp_socket);
2511 (*pr)("\tssp_soback: %p\n", so->so_sp->ssp_soback);
2512 (*pr)("\tssp_len: %lld\n",
2513 (unsigned long long)so->so_sp->ssp_len);
2514 (*pr)("\tssp_max: %lld\n",
2515 (unsigned long long)so->so_sp->ssp_max);
2516 (*pr)("\tssp_idletv: %lld %ld\n", so->so_sp->ssp_idletv.tv_sec,
2517 so->so_sp->ssp_idletv.tv_usec);
2518 (*pr)("\tssp_idleto: %spending (@%i)\n",
2519 timeout_pending(&so->so_sp->ssp_idleto) ? "" : "not ",
2520 so->so_sp->ssp_idleto.to_time);
2521 }
2522
2523 (*pr)("so_rcv:\n");
2524 sobuf_print(&so->so_rcv, pr);
2525 (*pr)("so_snd:\n");
2526 sobuf_print(&so->so_snd, pr);
2527
2528 (*pr)("so_upcall: %p so_upcallarg: %p\n",
2529 so->so_upcall, so->so_upcallarg);
2530
2531 (*pr)("so_euid: %d so_ruid: %d\n", so->so_euid, so->so_ruid);
2532 (*pr)("so_egid: %d so_rgid: %d\n", so->so_egid, so->so_rgid);
2533 (*pr)("so_cpid: %d\n", so->so_cpid);
2534 }
2535 #endif
2536