xref: /dragonfly/sys/kern/uipc_usrreq.c (revision e4adeac1)
1 /*
2  * Copyright (c) 1982, 1986, 1989, 1991, 1993
3  *	The Regents of the University of California.  All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  * 3. Neither the name of the University nor the names of its contributors
14  *    may be used to endorse or promote products derived from this software
15  *    without specific prior written permission.
16  *
17  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
18  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
19  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
20  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
21  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
22  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
23  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
24  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
25  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
26  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27  * SUCH DAMAGE.
28  *
29  *	From: @(#)uipc_usrreq.c	8.3 (Berkeley) 1/4/94
30  * $FreeBSD: src/sys/kern/uipc_usrreq.c,v 1.54.2.10 2003/03/04 17:28:09 nectar Exp $
31  */
32 
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/domain.h>
37 #include <sys/fcntl.h>
38 #include <sys/malloc.h>		/* XXX must be before <sys/file.h> */
39 #include <sys/proc.h>
40 #include <sys/file.h>
41 #include <sys/filedesc.h>
42 #include <sys/mbuf.h>
43 #include <sys/nlookup.h>
44 #include <sys/protosw.h>
45 #include <sys/socket.h>
46 #include <sys/socketvar.h>
47 #include <sys/resourcevar.h>
48 #include <sys/stat.h>
49 #include <sys/mount.h>
50 #include <sys/sysctl.h>
51 #include <sys/un.h>
52 #include <sys/unpcb.h>
53 #include <sys/vnode.h>
54 #include <sys/kern_syscall.h>
55 #include <sys/taskqueue.h>
56 
57 #include <sys/file2.h>
58 #include <sys/spinlock2.h>
59 #include <sys/socketvar2.h>
60 #include <sys/msgport2.h>
61 
62 /*
63  * Unix communications domain.
64  *
65  * TODO:
66  *	RDM
67  *	rethink name space problems
68  *	need a proper out-of-band
69  *	lock pushdown
70  *
71  *
72  * Unix domain sockets GC.
73  *
74  * It was originally designed to address following three cases:
75  * 1) Receiving unix domain socket can not accept the rights, e.g.
76  *    when the so_rcv is full.
77  * 2) Caller of recvmsg(2) does not pass buffer to receive rights.
78  * 3) Unix domain sockets loop reference, e.g. s1 is on s2.so_rcv,
79  *    while s2 on s1.so_rcv.
80  *
81  * Code under UNP_GC_ALLFILES is intended to address all above three
82  * cases.  However, 1) was addressed a long time ago in uipc_send()
83  * (we inheritted the fix from FreeBSD when DragonFly forked).  2)
84  * was addressed in soreceive() by git-e62cfe62.  3) is the only
85  * case that needs GC.  The new code (!UNP_GC_ALLFILES) addresses
86  * case 3) in the following way:
87  * - Record the struct file in unpcb, if the Unix domain socket is
88  *   passed as one of the rights.
89  * - At GC time, only unpcbs are scanned, and only Unix domain sockets
90  *   that are still used as rights are potential GC targets.
91  */
92 
93 #define UNP_DETACHED		UNP_PRIVATE1
94 #define UNP_CONNECTING		UNP_PRIVATE2
95 #define UNP_DROPPED		UNP_PRIVATE3
96 #define UNP_MARKER		UNP_PRIVATE4
97 
98 #define UNPGC_REF		0x1	/* unpcb has external ref. */
99 #define UNPGC_DEAD		0x2	/* unpcb might be dead. */
100 #define UNPGC_SCANNED		0x4	/* Has been scanned. */
101 
102 #define UNP_GCFILE_MAX		256
103 
104 /* For unp_internalize() and unp_externalize() */
105 CTASSERT(sizeof(struct file *) >= sizeof(int));
106 
107 #define UNP_ISATTACHED(unp)	\
108     ((unp) != NULL && ((unp)->unp_flags & UNP_DETACHED) == 0)
109 
110 #ifdef INVARIANTS
111 #define UNP_ASSERT_TOKEN_HELD(unp) \
112     ASSERT_LWKT_TOKEN_HELD(lwkt_token_pool_lookup((unp)))
113 #else	/* !INVARIANTS */
114 #define UNP_ASSERT_TOKEN_HELD(unp)
115 #endif	/* INVARIANTS */
116 
117 struct unp_defdiscard {
118 	SLIST_ENTRY(unp_defdiscard) next;
119 	struct file *fp;
120 };
121 SLIST_HEAD(unp_defdiscard_list, unp_defdiscard);
122 
123 TAILQ_HEAD(unpcb_qhead, unpcb);
124 struct unp_global_head {
125 	struct unpcb_qhead	list;
126 	int			count;
127 };
128 
129 static	MALLOC_DEFINE(M_UNPCB, "unpcb", "unpcb struct");
130 static	unp_gen_t unp_gencnt;
131 
132 static struct unp_global_head unp_stream_head;
133 static struct unp_global_head unp_dgram_head;
134 static struct unp_global_head unp_seqpkt_head;
135 
136 static struct unp_global_head * const unp_heads[] =
137     { &unp_stream_head, &unp_dgram_head, &unp_seqpkt_head, NULL };
138 
139 static struct lwkt_token unp_token = LWKT_TOKEN_INITIALIZER(unp_token);
140 static struct taskqueue *unp_taskqueue;
141 
142 static struct unp_defdiscard_list unp_defdiscard_head;
143 static struct spinlock unp_defdiscard_spin;
144 static struct task unp_defdiscard_task;
145 
146 static struct	sockaddr sun_noname = { sizeof(sun_noname), AF_LOCAL };
147 
148 static int     unp_attach (struct socket *, struct pru_attach_info *);
149 static void    unp_detach (struct unpcb *);
150 static int     unp_bind (struct unpcb *,struct sockaddr *, struct thread *);
151 static int     unp_connect (struct socket *,struct sockaddr *,
152 				struct thread *);
153 static void    unp_disconnect(struct unpcb *, int);
154 static void    unp_shutdown (struct unpcb *);
155 static void    unp_gc(void *, int);
156 #ifdef UNP_GC_ALLFILES
157 static int     unp_gc_clearmarks(struct file *, void *);
158 static int     unp_gc_checkmarks(struct file *, void *);
159 static int     unp_gc_checkrefs(struct file *, void *);
160 static void    unp_mark(struct file *, void *data);
161 #endif
162 static void    unp_scan (struct mbuf *, void (*)(struct file *, void *),
163 				void *data);
164 static void    unp_discard (struct file *, void *);
165 static int     unp_internalize (struct mbuf *, struct thread *);
166 static int     unp_listen (struct unpcb *, struct thread *);
167 static void    unp_fp_externalize(struct lwp *lp, struct file *fp, int fd,
168 		   int flags);
169 static int     unp_find_lockref(struct sockaddr *nam, struct thread *td,
170 		   short type, struct unpcb **unp_ret);
171 static int     unp_connect_pair(struct unpcb *unp, struct unpcb *unp2);
172 static void    unp_drop(struct unpcb *unp, int error);
173 static void    unp_defdiscard_taskfunc(void *, int);
174 
175 static int	unp_rights;			/* file descriptors in flight */
176 static struct lwkt_token unp_rights_token =
177     LWKT_TOKEN_INITIALIZER(unp_rights_token);
178 static struct task unp_gc_task;
179 static struct unpcb *unp_gc_marker;
180 
181 SYSCTL_DECL(_net_local);
182 SYSCTL_INT(_net_local, OID_AUTO, inflight, CTLFLAG_RD, &unp_rights, 0,
183    "File descriptors in flight");
184 
185 /*
186  * SMP Considerations:
187  *
188  *	Since unp_token will be automaticly released upon execution of
189  *	blocking code, we need to reference unp_conn before any possible
190  *	blocking code to prevent it from being ripped behind our back.
191  *
192  *	Any adjustment to unp->unp_conn requires both the global unp_token
193  *	AND the per-unp token (lwkt_token_pool_lookup(unp)) to be held.
194  *
195  *	Any access to so_pcb to obtain unp requires the pool token for
196  *	unp to be held.
197  */
198 
199 static __inline void
200 unp_reference(struct unpcb *unp)
201 {
202 	/* 0->1 transition will not work */
203 	KKASSERT(unp->unp_refcnt > 0);
204 	atomic_add_int(&unp->unp_refcnt, 1);
205 }
206 
207 static __inline void
208 unp_free(struct unpcb *unp)
209 {
210 	KKASSERT(unp->unp_refcnt > 0);
211 	if (atomic_fetchadd_int(&unp->unp_refcnt, -1) == 1)
212 		unp_detach(unp);
213 }
214 
215 static __inline struct unpcb *
216 unp_getsocktoken(struct socket *so)
217 {
218 	struct unpcb *unp;
219 
220 	/*
221 	 * The unp pointer is invalid until we verify that it is
222 	 * good by re-checking so_pcb AFTER obtaining the token.
223 	 */
224 	while ((unp = so->so_pcb) != NULL) {
225 		lwkt_getpooltoken(unp);
226 		if (unp == so->so_pcb)
227 			break;
228 		lwkt_relpooltoken(unp);
229 	}
230 	return unp;
231 }
232 
233 static __inline void
234 unp_reltoken(struct unpcb *unp)
235 {
236 	if (unp != NULL)
237 		lwkt_relpooltoken(unp);
238 }
239 
240 static __inline void
241 unp_setflags(struct unpcb *unp, int flags)
242 {
243 	atomic_set_int(&unp->unp_flags, flags);
244 }
245 
246 static __inline void
247 unp_clrflags(struct unpcb *unp, int flags)
248 {
249 	atomic_clear_int(&unp->unp_flags, flags);
250 }
251 
252 static __inline struct unp_global_head *
253 unp_globalhead(short type)
254 {
255 	switch (type) {
256 	case SOCK_STREAM:
257 		return &unp_stream_head;
258 	case SOCK_DGRAM:
259 		return &unp_dgram_head;
260 	case SOCK_SEQPACKET:
261 		return &unp_seqpkt_head;
262 	default:
263 		panic("unknown socket type %d", type);
264 	}
265 }
266 
267 static __inline struct unpcb *
268 unp_fp2unpcb(struct file *fp)
269 {
270 	struct socket *so;
271 
272 	if (fp->f_type != DTYPE_SOCKET)
273 		return NULL;
274 
275 	so = fp->f_data;
276 	if (so == NULL)
277 		return NULL;
278 
279 	if (so->so_proto->pr_domain != &localdomain)
280 		return NULL;
281 
282 	return so->so_pcb;
283 }
284 
285 static __inline void
286 unp_add_right(struct file *fp)
287 {
288 	struct unpcb *unp;
289 
290 	ASSERT_LWKT_TOKEN_HELD(&unp_rights_token);
291 	KASSERT(fp->f_count > 0, ("invalid f_count %d", fp->f_count));
292 
293 	unp = unp_fp2unpcb(fp);
294 	if (unp != NULL) {
295 		unp->unp_fp = fp;
296 		unp->unp_msgcount++;
297 	}
298 	fp->f_msgcount++;
299 	unp_rights++;
300 }
301 
302 static __inline void
303 unp_del_right(struct file *fp)
304 {
305 	struct unpcb *unp;
306 
307 	ASSERT_LWKT_TOKEN_HELD(&unp_rights_token);
308 	KASSERT(fp->f_count > 0, ("invalid f_count %d", fp->f_count));
309 
310 	unp = unp_fp2unpcb(fp);
311 	if (unp != NULL) {
312 		KASSERT(unp->unp_msgcount > 0,
313 		    ("invalid unp msgcount %d", unp->unp_msgcount));
314 		unp->unp_msgcount--;
315 		if (unp->unp_msgcount == 0)
316 			unp->unp_fp = NULL;
317 	}
318 	fp->f_msgcount--;
319 	unp_rights--;
320 }
321 
322 /*
323  * NOTE: (so) is referenced from soabort*() and netmsg_pru_abort()
324  *	 will sofree() it when we return.
325  */
326 static void
327 uipc_abort(netmsg_t msg)
328 {
329 	struct unpcb *unp;
330 	int error;
331 
332 	lwkt_gettoken(&unp_token);
333 	unp = unp_getsocktoken(msg->base.nm_so);
334 
335 	if (UNP_ISATTACHED(unp)) {
336 		unp_drop(unp, ECONNABORTED);
337 		error = 0;
338 	} else {
339 		error = EINVAL;
340 	}
341 
342 	unp_reltoken(unp);
343 	lwkt_reltoken(&unp_token);
344 
345 	lwkt_replymsg(&msg->lmsg, error);
346 }
347 
348 static void
349 uipc_accept(netmsg_t msg)
350 {
351 	struct unpcb *unp;
352 	int error;
353 
354 	lwkt_gettoken(&unp_token);
355 	unp = unp_getsocktoken(msg->base.nm_so);
356 
357 	if (!UNP_ISATTACHED(unp)) {
358 		error = EINVAL;
359 	} else {
360 		struct unpcb *unp2 = unp->unp_conn;
361 
362 		/*
363 		 * Pass back name of connected socket,
364 		 * if it was bound and we are still connected
365 		 * (our peer may have closed already!).
366 		 */
367 		if (unp2 && unp2->unp_addr) {
368 			unp_reference(unp2);
369 			*msg->accept.nm_nam = dup_sockaddr(
370 				(struct sockaddr *)unp2->unp_addr);
371 			unp_free(unp2);
372 		} else {
373 			*msg->accept.nm_nam = dup_sockaddr(&sun_noname);
374 		}
375 		error = 0;
376 	}
377 
378 	unp_reltoken(unp);
379 	lwkt_reltoken(&unp_token);
380 
381 	lwkt_replymsg(&msg->lmsg, error);
382 }
383 
384 static void
385 uipc_attach(netmsg_t msg)
386 {
387 	int error;
388 
389 	lwkt_gettoken(&unp_token);
390 
391 	KASSERT(msg->base.nm_so->so_pcb == NULL, ("double unp attach"));
392 	error = unp_attach(msg->base.nm_so, msg->attach.nm_ai);
393 
394 	lwkt_reltoken(&unp_token);
395 	lwkt_replymsg(&msg->lmsg, error);
396 }
397 
398 static void
399 uipc_bind(netmsg_t msg)
400 {
401 	struct unpcb *unp;
402 	int error;
403 
404 	lwkt_gettoken(&unp_token);
405 	unp = unp_getsocktoken(msg->base.nm_so);
406 
407 	if (UNP_ISATTACHED(unp))
408 		error = unp_bind(unp, msg->bind.nm_nam, msg->bind.nm_td);
409 	else
410 		error = EINVAL;
411 
412 	unp_reltoken(unp);
413 	lwkt_reltoken(&unp_token);
414 
415 	lwkt_replymsg(&msg->lmsg, error);
416 }
417 
418 static void
419 uipc_connect(netmsg_t msg)
420 {
421 	int error;
422 
423 	error = unp_connect(msg->base.nm_so, msg->connect.nm_nam,
424 	    msg->connect.nm_td);
425 	lwkt_replymsg(&msg->lmsg, error);
426 }
427 
428 static void
429 uipc_connect2(netmsg_t msg)
430 {
431 	int error;
432 
433 	error = unp_connect2(msg->connect2.nm_so1, msg->connect2.nm_so2);
434 	lwkt_replymsg(&msg->lmsg, error);
435 }
436 
437 /* control is EOPNOTSUPP */
438 
439 static void
440 uipc_detach(netmsg_t msg)
441 {
442 	struct unpcb *unp;
443 	int error;
444 
445 	lwkt_gettoken(&unp_token);
446 	unp = unp_getsocktoken(msg->base.nm_so);
447 
448 	if (UNP_ISATTACHED(unp)) {
449 		unp_drop(unp, 0);
450 		error = 0;
451 	} else {
452 		error = EINVAL;
453 	}
454 
455 	unp_reltoken(unp);
456 	lwkt_reltoken(&unp_token);
457 
458 	lwkt_replymsg(&msg->lmsg, error);
459 }
460 
461 static void
462 uipc_disconnect(netmsg_t msg)
463 {
464 	struct unpcb *unp;
465 	int error;
466 
467 	lwkt_gettoken(&unp_token);
468 	unp = unp_getsocktoken(msg->base.nm_so);
469 
470 	if (UNP_ISATTACHED(unp)) {
471 		unp_disconnect(unp, 0);
472 		error = 0;
473 	} else {
474 		error = EINVAL;
475 	}
476 
477 	unp_reltoken(unp);
478 	lwkt_reltoken(&unp_token);
479 
480 	lwkt_replymsg(&msg->lmsg, error);
481 }
482 
483 static void
484 uipc_listen(netmsg_t msg)
485 {
486 	struct unpcb *unp;
487 	int error;
488 
489 	lwkt_gettoken(&unp_token);
490 	unp = unp_getsocktoken(msg->base.nm_so);
491 
492 	if (!UNP_ISATTACHED(unp) || unp->unp_vnode == NULL)
493 		error = EINVAL;
494 	else
495 		error = unp_listen(unp, msg->listen.nm_td);
496 
497 	unp_reltoken(unp);
498 	lwkt_reltoken(&unp_token);
499 
500 	lwkt_replymsg(&msg->lmsg, error);
501 }
502 
503 static void
504 uipc_peeraddr(netmsg_t msg)
505 {
506 	struct unpcb *unp;
507 	int error;
508 
509 	lwkt_gettoken(&unp_token);
510 	unp = unp_getsocktoken(msg->base.nm_so);
511 
512 	if (!UNP_ISATTACHED(unp)) {
513 		error = EINVAL;
514 	} else if (unp->unp_conn && unp->unp_conn->unp_addr) {
515 		struct unpcb *unp2 = unp->unp_conn;
516 
517 		unp_reference(unp2);
518 		*msg->peeraddr.nm_nam = dup_sockaddr(
519 				(struct sockaddr *)unp2->unp_addr);
520 		unp_free(unp2);
521 		error = 0;
522 	} else {
523 		/*
524 		 * XXX: It seems that this test always fails even when
525 		 * connection is established.  So, this else clause is
526 		 * added as workaround to return PF_LOCAL sockaddr.
527 		 */
528 		*msg->peeraddr.nm_nam = dup_sockaddr(&sun_noname);
529 		error = 0;
530 	}
531 
532 	unp_reltoken(unp);
533 	lwkt_reltoken(&unp_token);
534 
535 	lwkt_replymsg(&msg->lmsg, error);
536 }
537 
538 static void
539 uipc_rcvd(netmsg_t msg)
540 {
541 	struct unpcb *unp, *unp2;
542 	struct socket *so;
543 	struct socket *so2;
544 	int error;
545 
546 	/*
547 	 * so_pcb is only modified with both the global and the unp
548 	 * pool token held.
549 	 */
550 	so = msg->base.nm_so;
551 	unp = unp_getsocktoken(so);
552 
553 	if (!UNP_ISATTACHED(unp)) {
554 		error = EINVAL;
555 		goto done;
556 	}
557 
558 	switch (so->so_type) {
559 	case SOCK_DGRAM:
560 		panic("uipc_rcvd DGRAM?");
561 		/*NOTREACHED*/
562 	case SOCK_STREAM:
563 	case SOCK_SEQPACKET:
564 		if (unp->unp_conn == NULL)
565 			break;
566 		unp2 = unp->unp_conn;	/* protected by pool token */
567 
568 		/*
569 		 * Because we are transfering mbufs directly to the
570 		 * peer socket we have to use SSB_STOP on the sender
571 		 * to prevent it from building up infinite mbufs.
572 		 *
573 		 * As in several places in this module w ehave to ref unp2
574 		 * to ensure that it does not get ripped out from under us
575 		 * if we block on the so2 token or in sowwakeup().
576 		 */
577 		so2 = unp2->unp_socket;
578 		unp_reference(unp2);
579 		lwkt_gettoken(&so2->so_rcv.ssb_token);
580 		if (so->so_rcv.ssb_cc < so2->so_snd.ssb_hiwat &&
581 		    so->so_rcv.ssb_mbcnt < so2->so_snd.ssb_mbmax
582 		) {
583 			atomic_clear_int(&so2->so_snd.ssb_flags, SSB_STOP);
584 
585 			sowwakeup(so2);
586 		}
587 		lwkt_reltoken(&so2->so_rcv.ssb_token);
588 		unp_free(unp2);
589 		break;
590 	default:
591 		panic("uipc_rcvd unknown socktype");
592 		/*NOTREACHED*/
593 	}
594 	error = 0;
595 done:
596 	unp_reltoken(unp);
597 	lwkt_replymsg(&msg->lmsg, error);
598 }
599 
600 /* pru_rcvoob is EOPNOTSUPP */
601 
602 static void
603 uipc_send(netmsg_t msg)
604 {
605 	struct unpcb *unp, *unp2;
606 	struct socket *so;
607 	struct socket *so2;
608 	struct mbuf *control;
609 	struct mbuf *m;
610 	int error = 0;
611 
612 	so = msg->base.nm_so;
613 	control = msg->send.nm_control;
614 	m = msg->send.nm_m;
615 
616 	/*
617 	 * so_pcb is only modified with both the global and the unp
618 	 * pool token held.
619 	 */
620 	so = msg->base.nm_so;
621 	unp = unp_getsocktoken(so);
622 
623 	if (!UNP_ISATTACHED(unp)) {
624 		error = EINVAL;
625 		goto release;
626 	}
627 
628 	if (msg->send.nm_flags & PRUS_OOB) {
629 		error = EOPNOTSUPP;
630 		goto release;
631 	}
632 
633 	wakeup_start_delayed();
634 
635 	if (control && (error = unp_internalize(control, msg->send.nm_td)))
636 		goto release;
637 
638 	switch (so->so_type) {
639 	case SOCK_DGRAM:
640 	{
641 		struct sockaddr *from;
642 
643 		if (msg->send.nm_addr) {
644 			if (unp->unp_conn) {
645 				error = EISCONN;
646 				break;
647 			}
648 			lwkt_gettoken(&unp_token);
649 			error = unp_find_lockref(msg->send.nm_addr,
650 			    msg->send.nm_td, so->so_type, &unp2);
651 			if (error) {
652 				lwkt_reltoken(&unp_token);
653 				break;
654 			}
655 			/*
656 			 * NOTE:
657 			 * unp2 is locked and referenced.
658 			 *
659 			 * We could unlock unp2 now, since it was checked
660 			 * and referenced.
661 			 */
662 			unp_reltoken(unp2);
663 			lwkt_reltoken(&unp_token);
664 		} else {
665 			if (unp->unp_conn == NULL) {
666 				error = ENOTCONN;
667 				break;
668 			}
669 			unp2 = unp->unp_conn;
670 			unp_reference(unp2);
671 		}
672 		/* NOTE: unp2 is referenced. */
673 		so2 = unp2->unp_socket;
674 
675 		/*
676 		 * Include creds if the receive side wants them, even if
677 		 * the send side did not send them.
678 		 */
679 		if (so2->so_options & SO_PASSCRED) {
680 			struct mbuf **mp;
681 			struct cmsghdr *cm;
682 			struct cmsgcred cred;
683 			struct mbuf *ncon;
684 
685 			mp = &control;
686 			while ((ncon = *mp) != NULL) {
687 				cm = mtod(ncon, struct cmsghdr *);
688 				if (cm->cmsg_type == SCM_CREDS &&
689 				    cm->cmsg_level == SOL_SOCKET)
690 					break;
691 				mp = &ncon->m_next;
692 			}
693 			if (ncon == NULL) {
694 				ncon = sbcreatecontrol((caddr_t)&cred,
695 						       sizeof(cred),
696 						       SCM_CREDS, SOL_SOCKET);
697 				unp_internalize(ncon, msg->send.nm_td);
698 				*mp = ncon;
699 			}
700 		}
701 
702 		if (unp->unp_addr)
703 			from = (struct sockaddr *)unp->unp_addr;
704 		else
705 			from = &sun_noname;
706 
707 		lwkt_gettoken(&so2->so_rcv.ssb_token);
708 		if (ssb_appendaddr(&so2->so_rcv, from, m, control)) {
709 			sorwakeup(so2);
710 			m = NULL;
711 			control = NULL;
712 		} else {
713 			error = ENOBUFS;
714 		}
715 		lwkt_reltoken(&so2->so_rcv.ssb_token);
716 
717 		unp_free(unp2);
718 		break;
719 	}
720 
721 	case SOCK_STREAM:
722 	case SOCK_SEQPACKET:
723 		/* Connect if not connected yet. */
724 		/*
725 		 * Note: A better implementation would complain
726 		 * if not equal to the peer's address.
727 		 */
728 		if (unp->unp_conn == NULL) {
729 			if (msg->send.nm_addr) {
730 				error = unp_connect(so,
731 						    msg->send.nm_addr,
732 						    msg->send.nm_td);
733 				if (error)
734 					break;	/* XXX */
735 			}
736 			/*
737 			 * NOTE:
738 			 * unp_conn still could be NULL, even if the
739 			 * above unp_connect() succeeds; since the
740 			 * current unp's token could be released due
741 			 * to blocking operations after unp_conn is
742 			 * assigned.
743 			 */
744 			if (unp->unp_conn == NULL) {
745 				error = ENOTCONN;
746 				break;
747 			}
748 		}
749 		if (so->so_state & SS_CANTSENDMORE) {
750 			error = EPIPE;
751 			break;
752 		}
753 
754 		unp2 = unp->unp_conn;
755 		KASSERT(unp2 != NULL, ("unp is not connected"));
756 		so2 = unp2->unp_socket;
757 
758 		unp_reference(unp2);
759 
760 		/*
761 		 * Send to paired receive port, and then reduce
762 		 * send buffer hiwater marks to maintain backpressure.
763 		 * Wake up readers.
764 		 */
765 		lwkt_gettoken(&so2->so_rcv.ssb_token);
766 		if (control) {
767 			if (ssb_appendcontrol(&so2->so_rcv, m, control)) {
768 				control = NULL;
769 				m = NULL;
770 			}
771 		} else if (so->so_type == SOCK_SEQPACKET) {
772 			sbappendrecord(&so2->so_rcv.sb, m);
773 			m = NULL;
774 		} else {
775 			sbappend(&so2->so_rcv.sb, m);
776 			m = NULL;
777 		}
778 
779 		/*
780 		 * Because we are transfering mbufs directly to the
781 		 * peer socket we have to use SSB_STOP on the sender
782 		 * to prevent it from building up infinite mbufs.
783 		 */
784 		if (so2->so_rcv.ssb_cc >= so->so_snd.ssb_hiwat ||
785 		    so2->so_rcv.ssb_mbcnt >= so->so_snd.ssb_mbmax
786 		) {
787 			atomic_set_int(&so->so_snd.ssb_flags, SSB_STOP);
788 		}
789 		lwkt_reltoken(&so2->so_rcv.ssb_token);
790 		sorwakeup(so2);
791 
792 		unp_free(unp2);
793 		break;
794 
795 	default:
796 		panic("uipc_send unknown socktype");
797 	}
798 
799 	/*
800 	 * SEND_EOF is equivalent to a SEND followed by a SHUTDOWN.
801 	 */
802 	if (msg->send.nm_flags & PRUS_EOF) {
803 		socantsendmore(so);
804 		unp_shutdown(unp);
805 	}
806 
807 	if (control && error != 0)
808 		unp_dispose(control);
809 release:
810 	unp_reltoken(unp);
811 	wakeup_end_delayed();
812 
813 	if (control)
814 		m_freem(control);
815 	if (m)
816 		m_freem(m);
817 	lwkt_replymsg(&msg->lmsg, error);
818 }
819 
820 /*
821  * MPSAFE
822  */
823 static void
824 uipc_sense(netmsg_t msg)
825 {
826 	struct unpcb *unp;
827 	struct socket *so;
828 	struct stat *sb;
829 	int error;
830 
831 	so = msg->base.nm_so;
832 	sb = msg->sense.nm_stat;
833 
834 	/*
835 	 * so_pcb is only modified with both the global and the unp
836 	 * pool token held.
837 	 */
838 	unp = unp_getsocktoken(so);
839 
840 	if (!UNP_ISATTACHED(unp)) {
841 		error = EINVAL;
842 		goto done;
843 	}
844 
845 	sb->st_blksize = so->so_snd.ssb_hiwat;
846 	sb->st_dev = NOUDEV;
847 	error = 0;
848 done:
849 	unp_reltoken(unp);
850 	lwkt_replymsg(&msg->lmsg, error);
851 }
852 
853 static void
854 uipc_shutdown(netmsg_t msg)
855 {
856 	struct socket *so;
857 	struct unpcb *unp;
858 	int error;
859 
860 	/*
861 	 * so_pcb is only modified with both the global and the unp
862 	 * pool token held.
863 	 */
864 	so = msg->base.nm_so;
865 	unp = unp_getsocktoken(so);
866 
867 	if (UNP_ISATTACHED(unp)) {
868 		socantsendmore(so);
869 		unp_shutdown(unp);
870 		error = 0;
871 	} else {
872 		error = EINVAL;
873 	}
874 
875 	unp_reltoken(unp);
876 	lwkt_replymsg(&msg->lmsg, error);
877 }
878 
879 static void
880 uipc_sockaddr(netmsg_t msg)
881 {
882 	struct unpcb *unp;
883 	int error;
884 
885 	/*
886 	 * so_pcb is only modified with both the global and the unp
887 	 * pool token held.
888 	 */
889 	unp = unp_getsocktoken(msg->base.nm_so);
890 
891 	if (UNP_ISATTACHED(unp)) {
892 		if (unp->unp_addr) {
893 			*msg->sockaddr.nm_nam =
894 				dup_sockaddr((struct sockaddr *)unp->unp_addr);
895 		}
896 		error = 0;
897 	} else {
898 		error = EINVAL;
899 	}
900 
901 	unp_reltoken(unp);
902 	lwkt_replymsg(&msg->lmsg, error);
903 }
904 
905 struct pr_usrreqs uipc_usrreqs = {
906 	.pru_abort = uipc_abort,
907 	.pru_accept = uipc_accept,
908 	.pru_attach = uipc_attach,
909 	.pru_bind = uipc_bind,
910 	.pru_connect = uipc_connect,
911 	.pru_connect2 = uipc_connect2,
912 	.pru_control = pr_generic_notsupp,
913 	.pru_detach = uipc_detach,
914 	.pru_disconnect = uipc_disconnect,
915 	.pru_listen = uipc_listen,
916 	.pru_peeraddr = uipc_peeraddr,
917 	.pru_rcvd = uipc_rcvd,
918 	.pru_rcvoob = pr_generic_notsupp,
919 	.pru_send = uipc_send,
920 	.pru_sense = uipc_sense,
921 	.pru_shutdown = uipc_shutdown,
922 	.pru_sockaddr = uipc_sockaddr,
923 	.pru_sosend = sosend,
924 	.pru_soreceive = soreceive
925 };
926 
927 void
928 uipc_ctloutput(netmsg_t msg)
929 {
930 	struct socket *so;
931 	struct sockopt *sopt;
932 	struct unpcb *unp;
933 	int error = 0;
934 
935 	so = msg->base.nm_so;
936 	sopt = msg->ctloutput.nm_sopt;
937 
938 	lwkt_gettoken(&unp_token);
939 	unp = unp_getsocktoken(so);
940 
941 	if (!UNP_ISATTACHED(unp)) {
942 		error = EINVAL;
943 		goto done;
944 	}
945 
946 	switch (sopt->sopt_dir) {
947 	case SOPT_GET:
948 		switch (sopt->sopt_name) {
949 		case LOCAL_PEERCRED:
950 			if (unp->unp_flags & UNP_HAVEPC)
951 				soopt_from_kbuf(sopt, &unp->unp_peercred,
952 						sizeof(unp->unp_peercred));
953 			else {
954 				if (so->so_type == SOCK_STREAM)
955 					error = ENOTCONN;
956 				else if (so->so_type == SOCK_SEQPACKET)
957 					error = ENOTCONN;
958 				else
959 					error = EINVAL;
960 			}
961 			break;
962 		default:
963 			error = EOPNOTSUPP;
964 			break;
965 		}
966 		break;
967 	case SOPT_SET:
968 	default:
969 		error = EOPNOTSUPP;
970 		break;
971 	}
972 
973 done:
974 	unp_reltoken(unp);
975 	lwkt_reltoken(&unp_token);
976 
977 	lwkt_replymsg(&msg->lmsg, error);
978 }
979 
980 /*
981  * Both send and receive buffers are allocated PIPSIZ bytes of buffering
982  * for stream sockets, although the total for sender and receiver is
983  * actually only PIPSIZ.
984  *
985  * Datagram sockets really use the sendspace as the maximum datagram size,
986  * and don't really want to reserve the sendspace.  Their recvspace should
987  * be large enough for at least one max-size datagram plus address.
988  *
989  * We want the local send/recv space to be significant larger then lo0's
990  * mtu of 16384.
991  *
992  * We no longer need to worry about avoiding the windows scaling option.
993  * Programs which use unix domain sockets expect larger defaults these days.
994  */
995 #ifndef PIPSIZ
996 #define	PIPSIZ	65536
997 #endif
998 static u_long	unpst_sendspace = PIPSIZ;
999 static u_long	unpst_recvspace = PIPSIZ;
1000 static u_long	unpdg_sendspace = PIPSIZ;	/* really max datagram size */
1001 static u_long	unpdg_recvspace = PIPSIZ;
1002 static u_long	unpsp_sendspace = PIPSIZ;	/* really max datagram size */
1003 static u_long	unpsp_recvspace = PIPSIZ;
1004 
1005 SYSCTL_DECL(_net_local_stream);
1006 SYSCTL_DECL(_net_local_dgram);
1007 SYSCTL_DECL(_net_local_seqpacket);
1008 
1009 SYSCTL_ULONG(_net_local_stream, OID_AUTO, sendspace, CTLFLAG_RW,
1010     &unpst_sendspace, 0, "Size of stream socket send buffer");
1011 SYSCTL_ULONG(_net_local_stream, OID_AUTO, recvspace, CTLFLAG_RW,
1012     &unpst_recvspace, 0, "Size of stream socket receive buffer");
1013 
1014 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, maxdgram, CTLFLAG_RW,
1015     &unpdg_sendspace, 0, "Max datagram socket size");
1016 SYSCTL_ULONG(_net_local_dgram, OID_AUTO, recvspace, CTLFLAG_RW,
1017     &unpdg_recvspace, 0, "Size of datagram socket receive buffer");
1018 
1019 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, maxseqpacket, CTLFLAG_RW,
1020     &unpsp_sendspace, 0, "Default seqpacket send space.");
1021 SYSCTL_ULONG(_net_local_seqpacket, OID_AUTO, recvspace, CTLFLAG_RW,
1022     &unpsp_recvspace, 0, "Default seqpacket receive space.");
1023 
1024 
1025 static int
1026 unp_attach(struct socket *so, struct pru_attach_info *ai)
1027 {
1028 	struct unp_global_head *head;
1029 	struct unpcb *unp;
1030 	int error;
1031 
1032 	lwkt_gettoken(&unp_token);
1033 
1034 	if (so->so_snd.ssb_hiwat == 0 || so->so_rcv.ssb_hiwat == 0) {
1035 		switch (so->so_type) {
1036 		case SOCK_STREAM:
1037 			error = soreserve(so, unpst_sendspace, unpst_recvspace,
1038 					  ai->sb_rlimit);
1039 			break;
1040 		case SOCK_DGRAM:
1041 			error = soreserve(so, unpdg_sendspace, unpdg_recvspace,
1042 					  ai->sb_rlimit);
1043 			break;
1044 		case SOCK_SEQPACKET:
1045 			error = soreserve(so, unpsp_sendspace, unpsp_recvspace,
1046 					  ai->sb_rlimit);
1047 			break;
1048 		default:
1049 			panic("unp_attach");
1050 		}
1051 		if (error)
1052 			goto failed;
1053 	}
1054 
1055 	/*
1056 	 * In order to support sendfile we have to set either SSB_STOPSUPP
1057 	 * or SSB_PREALLOC.  Unix domain sockets use the SSB_STOP flow
1058 	 * control mechanism.
1059 	 */
1060 	if (so->so_type == SOCK_STREAM) {
1061 		atomic_set_int(&so->so_rcv.ssb_flags, SSB_STOPSUPP);
1062 		atomic_set_int(&so->so_snd.ssb_flags, SSB_STOPSUPP);
1063 	}
1064 
1065 	unp = kmalloc(sizeof(*unp), M_UNPCB, M_WAITOK | M_ZERO | M_NULLOK);
1066 	if (unp == NULL) {
1067 		error = ENOBUFS;
1068 		goto failed;
1069 	}
1070 	unp->unp_refcnt = 1;
1071 	unp->unp_gencnt = ++unp_gencnt;
1072 	LIST_INIT(&unp->unp_refs);
1073 	unp->unp_socket = so;
1074 	unp->unp_rvnode = ai->fd_rdir;		/* jail cruft XXX JH */
1075 	so->so_pcb = (caddr_t)unp;
1076 	soreference(so);
1077 
1078 	head = unp_globalhead(so->so_type);
1079 	TAILQ_INSERT_TAIL(&head->list, unp, unp_link);
1080 	head->count++;
1081 	error = 0;
1082 failed:
1083 	lwkt_reltoken(&unp_token);
1084 	return error;
1085 }
1086 
1087 static void
1088 unp_detach(struct unpcb *unp)
1089 {
1090 	struct socket *so;
1091 
1092 	lwkt_gettoken(&unp_token);
1093 	lwkt_getpooltoken(unp);
1094 
1095 	so = unp->unp_socket;
1096 
1097 	unp->unp_gencnt = ++unp_gencnt;
1098 	if (unp->unp_vnode) {
1099 		unp->unp_vnode->v_socket = NULL;
1100 		vrele(unp->unp_vnode);
1101 		unp->unp_vnode = NULL;
1102 	}
1103 	soisdisconnected(so);
1104 	KKASSERT(so->so_pcb == unp);
1105 	so->so_pcb = NULL;		/* both tokens required */
1106 	unp->unp_socket = NULL;
1107 
1108 	lwkt_relpooltoken(unp);
1109 	lwkt_reltoken(&unp_token);
1110 
1111 	sofree(so);
1112 
1113 	KASSERT(unp->unp_conn == NULL, ("unp is still connected"));
1114 	KASSERT(LIST_EMPTY(&unp->unp_refs), ("unp still has references"));
1115 
1116 	if (unp->unp_addr)
1117 		kfree(unp->unp_addr, M_SONAME);
1118 	kfree(unp, M_UNPCB);
1119 
1120 	if (unp_rights)
1121 		taskqueue_enqueue(unp_taskqueue, &unp_gc_task);
1122 }
1123 
1124 static int
1125 unp_bind(struct unpcb *unp, struct sockaddr *nam, struct thread *td)
1126 {
1127 	struct proc *p = td->td_proc;
1128 	struct sockaddr_un *soun = (struct sockaddr_un *)nam;
1129 	struct vnode *vp;
1130 	struct vattr vattr;
1131 	int error, namelen;
1132 	struct nlookupdata nd;
1133 	char buf[SOCK_MAXADDRLEN];
1134 
1135 	ASSERT_LWKT_TOKEN_HELD(&unp_token);
1136 	UNP_ASSERT_TOKEN_HELD(unp);
1137 
1138 	if (unp->unp_vnode != NULL)
1139 		return EINVAL;
1140 
1141 	namelen = soun->sun_len - offsetof(struct sockaddr_un, sun_path);
1142 	if (namelen <= 0)
1143 		return EINVAL;
1144 	strncpy(buf, soun->sun_path, namelen);
1145 	buf[namelen] = 0;	/* null-terminate the string */
1146 	error = nlookup_init(&nd, buf, UIO_SYSSPACE,
1147 			     NLC_LOCKVP | NLC_CREATE | NLC_REFDVP);
1148 	if (error == 0)
1149 		error = nlookup(&nd);
1150 	if (error == 0 && nd.nl_nch.ncp->nc_vp != NULL)
1151 		error = EADDRINUSE;
1152 	if (error)
1153 		goto done;
1154 
1155 	VATTR_NULL(&vattr);
1156 	vattr.va_type = VSOCK;
1157 	vattr.va_mode = (ACCESSPERMS & ~p->p_fd->fd_cmask);
1158 	error = VOP_NCREATE(&nd.nl_nch, nd.nl_dvp, &vp, nd.nl_cred, &vattr);
1159 	if (error == 0) {
1160 		if (unp->unp_vnode == NULL) {
1161 			vp->v_socket = unp->unp_socket;
1162 			unp->unp_vnode = vp;
1163 			unp->unp_addr = (struct sockaddr_un *)dup_sockaddr(nam);
1164 			vn_unlock(vp);
1165 		} else {
1166 			vput(vp);		/* late race */
1167 			error = EINVAL;
1168 		}
1169 	}
1170 done:
1171 	nlookup_done(&nd);
1172 	return (error);
1173 }
1174 
1175 static int
1176 unp_connect(struct socket *so, struct sockaddr *nam, struct thread *td)
1177 {
1178 	struct unpcb *unp, *unp2;
1179 	int error, flags = 0;
1180 
1181 	lwkt_gettoken(&unp_token);
1182 
1183 	unp = unp_getsocktoken(so);
1184 	if (!UNP_ISATTACHED(unp)) {
1185 		error = EINVAL;
1186 		goto failed;
1187 	}
1188 
1189 	if ((unp->unp_flags & UNP_CONNECTING) || unp->unp_conn != NULL) {
1190 		error = EISCONN;
1191 		goto failed;
1192 	}
1193 
1194 	flags = UNP_CONNECTING;
1195 	unp_setflags(unp, flags);
1196 
1197 	error = unp_find_lockref(nam, td, so->so_type, &unp2);
1198 	if (error)
1199 		goto failed;
1200 	/*
1201 	 * NOTE:
1202 	 * unp2 is locked and referenced.
1203 	 */
1204 
1205 	if (so->so_proto->pr_flags & PR_CONNREQUIRED) {
1206 		struct socket *so2, *so3;
1207 		struct unpcb *unp3;
1208 
1209 		so2 = unp2->unp_socket;
1210 		if (!(so2->so_options & SO_ACCEPTCONN) ||
1211 		    /* listen is not completed yet */
1212 		    !(unp2->unp_flags & UNP_HAVEPCCACHED) ||
1213 		    (so3 = sonewconn_faddr(so2, 0, NULL,
1214 		     TRUE /* keep ref */)) == NULL) {
1215 			error = ECONNREFUSED;
1216 			goto done;
1217 		}
1218 		/* so3 has a socket reference. */
1219 
1220 		unp3 = unp_getsocktoken(so3);
1221 		if (!UNP_ISATTACHED(unp3)) {
1222 			unp_reltoken(unp3);
1223 			/*
1224 			 * Already aborted; we only need to drop the
1225 			 * socket reference held by sonewconn_faddr().
1226 			 */
1227 			sofree(so3);
1228 			error = ECONNREFUSED;
1229 			goto done;
1230 		}
1231 		unp_reference(unp3);
1232 		/*
1233 		 * NOTE:
1234 		 * unp3 is locked and referenced.
1235 		 */
1236 
1237 		/*
1238 		 * Release so3 socket reference held by sonewconn_faddr().
1239 		 * Since we have referenced unp3, neither unp3 nor so3 will
1240 		 * be destroyed here.
1241 		 */
1242 		sofree(so3);
1243 
1244 		if (unp2->unp_addr != NULL) {
1245 			unp3->unp_addr = (struct sockaddr_un *)
1246 			    dup_sockaddr((struct sockaddr *)unp2->unp_addr);
1247 		}
1248 
1249 		/*
1250 		 * unp_peercred management:
1251 		 *
1252 		 * The connecter's (client's) credentials are copied
1253 		 * from its process structure at the time of connect()
1254 		 * (which is now).
1255 		 */
1256 		cru2x(td->td_proc->p_ucred, &unp3->unp_peercred);
1257 		unp_setflags(unp3, UNP_HAVEPC);
1258 		/*
1259 		 * The receiver's (server's) credentials are copied
1260 		 * from the unp_peercred member of socket on which the
1261 		 * former called listen(); unp_listen() cached that
1262 		 * process's credentials at that time so we can use
1263 		 * them now.
1264 		 */
1265 		KASSERT(unp2->unp_flags & UNP_HAVEPCCACHED,
1266 		    ("unp_connect: listener without cached peercred"));
1267 		memcpy(&unp->unp_peercred, &unp2->unp_peercred,
1268 		    sizeof(unp->unp_peercred));
1269 		unp_setflags(unp, UNP_HAVEPC);
1270 
1271 		error = unp_connect_pair(unp, unp3);
1272 		if (error)
1273 			soabort_direct(so3);
1274 
1275 		/* Done with unp3 */
1276 		unp_free(unp3);
1277 		unp_reltoken(unp3);
1278 	} else {
1279 		error = unp_connect_pair(unp, unp2);
1280 	}
1281 done:
1282 	unp_free(unp2);
1283 	unp_reltoken(unp2);
1284 failed:
1285 	if (flags)
1286 		unp_clrflags(unp, flags);
1287 	unp_reltoken(unp);
1288 
1289 	lwkt_reltoken(&unp_token);
1290 	return (error);
1291 }
1292 
1293 /*
1294  * Connect two unix domain sockets together.
1295  *
1296  * NOTE: Semantics for any change to unp_conn requires that the per-unp
1297  *	 pool token also be held.
1298  */
1299 int
1300 unp_connect2(struct socket *so, struct socket *so2)
1301 {
1302 	struct unpcb *unp, *unp2;
1303 	int error;
1304 
1305 	lwkt_gettoken(&unp_token);
1306 	if (so2->so_type != so->so_type) {
1307 		lwkt_reltoken(&unp_token);
1308 		return (EPROTOTYPE);
1309 	}
1310 	unp = unp_getsocktoken(so);
1311 	unp2 = unp_getsocktoken(so2);
1312 
1313 	if (!UNP_ISATTACHED(unp)) {
1314 		error = EINVAL;
1315 		goto done;
1316 	}
1317 	if (!UNP_ISATTACHED(unp2)) {
1318 		error = ECONNREFUSED;
1319 		goto done;
1320 	}
1321 
1322 	if (unp->unp_conn != NULL) {
1323 		error = EISCONN;
1324 		goto done;
1325 	}
1326 	if ((so->so_type == SOCK_STREAM || so->so_type == SOCK_SEQPACKET) &&
1327 	    unp2->unp_conn != NULL) {
1328 		error = EISCONN;
1329 		goto done;
1330 	}
1331 
1332 	error = unp_connect_pair(unp, unp2);
1333 done:
1334 	unp_reltoken(unp2);
1335 	unp_reltoken(unp);
1336 	lwkt_reltoken(&unp_token);
1337 	return (error);
1338 }
1339 
1340 /*
1341  * Disconnect a unix domain socket pair.
1342  *
1343  * NOTE: Semantics for any change to unp_conn requires that the per-unp
1344  *	 pool token also be held.
1345  */
1346 static void
1347 unp_disconnect(struct unpcb *unp, int error)
1348 {
1349 	struct socket *so = unp->unp_socket;
1350 	struct unpcb *unp2;
1351 
1352 	ASSERT_LWKT_TOKEN_HELD(&unp_token);
1353 	UNP_ASSERT_TOKEN_HELD(unp);
1354 
1355 	if (error)
1356 		so->so_error = error;
1357 
1358 	while ((unp2 = unp->unp_conn) != NULL) {
1359 		lwkt_getpooltoken(unp2);
1360 		if (unp2 == unp->unp_conn)
1361 			break;
1362 		lwkt_relpooltoken(unp2);
1363 	}
1364 	if (unp2 == NULL)
1365 		return;
1366 	/* unp2 is locked. */
1367 
1368 	KASSERT((unp2->unp_flags & UNP_DROPPED) == 0, ("unp2 was dropped"));
1369 
1370 	unp->unp_conn = NULL;
1371 
1372 	switch (so->so_type) {
1373 	case SOCK_DGRAM:
1374 		LIST_REMOVE(unp, unp_reflink);
1375 		soclrstate(so, SS_ISCONNECTED);
1376 		break;
1377 
1378 	case SOCK_STREAM:
1379 	case SOCK_SEQPACKET:
1380 		/*
1381 		 * Keep a reference before clearing the unp_conn
1382 		 * to avoid racing uipc_detach()/uipc_abort() in
1383 		 * other thread.
1384 		 */
1385 		unp_reference(unp2);
1386 		KASSERT(unp2->unp_conn == unp, ("unp_conn mismatch"));
1387 		unp2->unp_conn = NULL;
1388 
1389 		soisdisconnected(so);
1390 		soisdisconnected(unp2->unp_socket);
1391 
1392 		unp_free(unp2);
1393 		break;
1394 	}
1395 
1396 	lwkt_relpooltoken(unp2);
1397 }
1398 
1399 #ifdef notdef
1400 void
1401 unp_abort(struct unpcb *unp)
1402 {
1403 	lwkt_gettoken(&unp_token);
1404 	unp_free(unp);
1405 	lwkt_reltoken(&unp_token);
1406 }
1407 #endif
1408 
1409 static int
1410 prison_unpcb(struct thread *td, struct unpcb *unp)
1411 {
1412 	struct proc *p;
1413 
1414 	if (td == NULL)
1415 		return (0);
1416 	if ((p = td->td_proc) == NULL)
1417 		return (0);
1418 	if (!p->p_ucred->cr_prison)
1419 		return (0);
1420 	if (p->p_fd->fd_rdir == unp->unp_rvnode)
1421 		return (0);
1422 	return (1);
1423 }
1424 
1425 static int
1426 unp_pcblist(SYSCTL_HANDLER_ARGS)
1427 {
1428 	struct unp_global_head *head = arg1;
1429 	int error, i, n;
1430 	struct unpcb *unp, *marker;
1431 
1432 	KKASSERT(curproc != NULL);
1433 
1434 	/*
1435 	 * The process of preparing the PCB list is too time-consuming and
1436 	 * resource-intensive to repeat twice on every request.
1437 	 */
1438 	if (req->oldptr == NULL) {
1439 		n = head->count;
1440 		req->oldidx = (n + n/8) * sizeof(struct xunpcb);
1441 		return 0;
1442 	}
1443 
1444 	if (req->newptr != NULL)
1445 		return EPERM;
1446 
1447 	marker = kmalloc(sizeof(*marker), M_UNPCB, M_WAITOK | M_ZERO);
1448 	marker->unp_flags |= UNP_MARKER;
1449 
1450 	lwkt_gettoken(&unp_token);
1451 
1452 	n = head->count;
1453 	i = 0;
1454 	error = 0;
1455 
1456 	TAILQ_INSERT_HEAD(&head->list, marker, unp_link);
1457 	while ((unp = TAILQ_NEXT(marker, unp_link)) != NULL && i < n) {
1458 		struct xunpcb xu;
1459 
1460 		TAILQ_REMOVE(&head->list, marker, unp_link);
1461 		TAILQ_INSERT_AFTER(&head->list, unp, marker, unp_link);
1462 
1463 		if (unp->unp_flags & UNP_MARKER)
1464 			continue;
1465 		if (prison_unpcb(req->td, unp))
1466 			continue;
1467 
1468 		xu.xu_len = sizeof(xu);
1469 		xu.xu_unpp = unp;
1470 
1471 		/*
1472 		 * NOTE:
1473 		 * unp->unp_addr and unp->unp_conn are protected by
1474 		 * unp_token.  So if we want to get rid of unp_token
1475 		 * or reduce the coverage of unp_token, care must be
1476 		 * taken.
1477 		 */
1478 		if (unp->unp_addr) {
1479 			bcopy(unp->unp_addr, &xu.xu_addr,
1480 			      unp->unp_addr->sun_len);
1481 		}
1482 		if (unp->unp_conn && unp->unp_conn->unp_addr) {
1483 			bcopy(unp->unp_conn->unp_addr,
1484 			      &xu.xu_caddr,
1485 			      unp->unp_conn->unp_addr->sun_len);
1486 		}
1487 		bcopy(unp, &xu.xu_unp, sizeof(*unp));
1488 		sotoxsocket(unp->unp_socket, &xu.xu_socket);
1489 
1490 		/* NOTE: This could block and temporarily release unp_token */
1491 		error = SYSCTL_OUT(req, &xu, sizeof(xu));
1492 		if (error)
1493 			break;
1494 		++i;
1495 	}
1496 	TAILQ_REMOVE(&head->list, marker, unp_link);
1497 
1498 	lwkt_reltoken(&unp_token);
1499 
1500 	kfree(marker, M_UNPCB);
1501 	return error;
1502 }
1503 
1504 SYSCTL_PROC(_net_local_dgram, OID_AUTO, pcblist, CTLFLAG_RD,
1505 	    &unp_dgram_head, 0, unp_pcblist, "S,xunpcb",
1506 	    "List of active local datagram sockets");
1507 SYSCTL_PROC(_net_local_stream, OID_AUTO, pcblist, CTLFLAG_RD,
1508 	    &unp_stream_head, 0, unp_pcblist, "S,xunpcb",
1509 	    "List of active local stream sockets");
1510 SYSCTL_PROC(_net_local_seqpacket, OID_AUTO, pcblist, CTLFLAG_RD,
1511 	    &unp_seqpkt_head, 0, unp_pcblist, "S,xunpcb",
1512 	    "List of active local seqpacket sockets");
1513 
1514 static void
1515 unp_shutdown(struct unpcb *unp)
1516 {
1517 	struct socket *so;
1518 
1519 	if ((unp->unp_socket->so_type == SOCK_STREAM ||
1520 	     unp->unp_socket->so_type == SOCK_SEQPACKET) &&
1521 	    unp->unp_conn != NULL && (so = unp->unp_conn->unp_socket)) {
1522 		socantrcvmore(so);
1523 	}
1524 }
1525 
1526 #ifdef notdef
1527 void
1528 unp_drain(void)
1529 {
1530 	lwkt_gettoken(&unp_token);
1531 	lwkt_reltoken(&unp_token);
1532 }
1533 #endif
1534 
1535 int
1536 unp_externalize(struct mbuf *rights, int flags)
1537 {
1538 	struct thread *td = curthread;
1539 	struct proc *p = td->td_proc;		/* XXX */
1540 	struct lwp *lp = td->td_lwp;
1541 	struct cmsghdr *cm = mtod(rights, struct cmsghdr *);
1542 	int *fdp;
1543 	int i;
1544 	struct file **rp;
1545 	struct file *fp;
1546 	int newfds = (cm->cmsg_len - (CMSG_DATA(cm) - (u_char *)cm))
1547 		/ sizeof(struct file *);
1548 	int f;
1549 
1550 	lwkt_gettoken(&unp_rights_token);
1551 
1552 	/*
1553 	 * if the new FD's will not fit, then we free them all
1554 	 */
1555 	if (!fdavail(p, newfds)) {
1556 		rp = (struct file **)CMSG_DATA(cm);
1557 		for (i = 0; i < newfds; i++) {
1558 			fp = *rp;
1559 			/*
1560 			 * zero the pointer before calling unp_discard,
1561 			 * since it may end up in unp_gc()..
1562 			 */
1563 			*rp++ = NULL;
1564 			unp_discard(fp, NULL);
1565 		}
1566 		lwkt_reltoken(&unp_rights_token);
1567 		return (EMSGSIZE);
1568 	}
1569 
1570 	/*
1571 	 * now change each pointer to an fd in the global table to
1572 	 * an integer that is the index to the local fd table entry
1573 	 * that we set up to point to the global one we are transferring.
1574 	 * Since the sizeof(struct file *) is bigger than or equal to
1575 	 * the sizeof(int), we do it in forward order.  In that case,
1576 	 * an integer will always come in the same place or before its
1577 	 * corresponding struct file pointer.
1578 	 *
1579 	 * Hold revoke_token in 'shared' mode, so that we won't miss
1580 	 * the FREVOKED update on fps being externalized (fsetfd).
1581 	 */
1582 	lwkt_gettoken_shared(&revoke_token);
1583 	fdp = (int *)CMSG_DATA(cm);
1584 	rp = (struct file **)CMSG_DATA(cm);
1585 	for (i = 0; i < newfds; i++) {
1586 		if (fdalloc(p, 0, &f)) {
1587 			int j;
1588 
1589 			/*
1590 			 * Previous fdavail() can't garantee
1591 			 * fdalloc() success due to SMP race.
1592 			 * Just clean up and return the same
1593 			 * error value as if fdavail() failed.
1594 			 */
1595 			lwkt_reltoken(&revoke_token);
1596 
1597 			/* Close externalized files */
1598 			for (j = 0; j < i; j++)
1599 				kern_close(fdp[j]);
1600 			/* Discard the rest of internal files */
1601 			for (; i < newfds; i++)
1602 				unp_discard(rp[i], NULL);
1603 			/* Wipe out the control message */
1604 			for (i = 0; i < newfds; i++)
1605 				rp[i] = NULL;
1606 
1607 			lwkt_reltoken(&unp_rights_token);
1608 			return (EMSGSIZE);
1609 		}
1610 		fp = rp[i];
1611 		unp_fp_externalize(lp, fp, f, flags);
1612 		fdp[i] = f;
1613 	}
1614 	lwkt_reltoken(&revoke_token);
1615 
1616 	lwkt_reltoken(&unp_rights_token);
1617 
1618 	/*
1619 	 * Adjust length, in case sizeof(struct file *) and sizeof(int)
1620 	 * differs.
1621 	 */
1622 	cm->cmsg_len = CMSG_LEN(newfds * sizeof(int));
1623 	rights->m_len = cm->cmsg_len;
1624 
1625 	return (0);
1626 }
1627 
1628 static void
1629 unp_fp_externalize(struct lwp *lp, struct file *fp, int fd, int flags)
1630 {
1631 	if (lp) {
1632 		struct filedesc *fdp = lp->lwp_proc->p_fd;
1633 
1634 		KKASSERT(fd >= 0);
1635 		if (fp->f_flag & FREVOKED) {
1636 			struct file *fx;
1637 			int error;
1638 
1639 			kprintf("Warning: revoked fp exiting unix socket\n");
1640 			error = falloc(lp, &fx, NULL);
1641 			if (error == 0) {
1642 				if (flags & MSG_CMSG_CLOEXEC)
1643 					fdp->fd_files[fd].fileflags |= UF_EXCLOSE;
1644 				fsetfd(fdp, fx, fd);
1645 				fdrop(fx);
1646 			} else {
1647 				fsetfd(fdp, NULL, fd);
1648 			}
1649 		} else {
1650 			if (flags & MSG_CMSG_CLOEXEC)
1651 				fdp->fd_files[fd].fileflags |= UF_EXCLOSE;
1652 			fsetfd(fdp, fp, fd);
1653 		}
1654 	}
1655 	unp_del_right(fp);
1656 	fdrop(fp);
1657 }
1658 
1659 void
1660 unp_init(void)
1661 {
1662 	TAILQ_INIT(&unp_stream_head.list);
1663 	TAILQ_INIT(&unp_dgram_head.list);
1664 	TAILQ_INIT(&unp_seqpkt_head.list);
1665 
1666 	SLIST_INIT(&unp_defdiscard_head);
1667 	spin_init(&unp_defdiscard_spin, "unpdisc");
1668 	TASK_INIT(&unp_defdiscard_task, 0, unp_defdiscard_taskfunc, NULL);
1669 
1670 	/*
1671 	 * This implies that only one gc can be in-progress at any
1672 	 * given moment.
1673 	 */
1674 	TASK_INIT(&unp_gc_task, 0, unp_gc, NULL);
1675 
1676 	unp_gc_marker = kmalloc(sizeof(*unp_gc_marker), M_UNPCB,
1677 	    M_WAITOK | M_ZERO);
1678 	unp_gc_marker->unp_flags |= UNP_MARKER;
1679 
1680 	/*
1681 	 * Create taskqueue for defered discard, and stick it to
1682 	 * the last CPU.
1683 	 */
1684 	unp_taskqueue = taskqueue_create("unp_taskq", M_WAITOK,
1685 	    taskqueue_thread_enqueue, &unp_taskqueue);
1686 	taskqueue_start_threads(&unp_taskqueue, 1, TDPRI_KERN_DAEMON,
1687 	    ncpus - 1, "unp taskq");
1688 }
1689 
1690 static int
1691 unp_internalize(struct mbuf *control, struct thread *td)
1692 {
1693 	struct proc *p = td->td_proc;
1694 	struct filedesc *fdescp;
1695 	struct cmsghdr *cm = mtod(control, struct cmsghdr *);
1696 	struct file **rp;
1697 	struct file *fp;
1698 	int i, fd, *fdp;
1699 	struct cmsgcred *cmcred;
1700 	int oldfds;
1701 	u_int newlen;
1702 	int error;
1703 
1704 	KKASSERT(p);
1705 
1706 	/*
1707 	 * Make sure the message is reasonable, and either CREDS or RIGHTS.
1708 	 *
1709 	 * NOTE: overall message length does not have to be aligned, but the
1710 	 *	 data start does.
1711 	 */
1712 	if ((cm->cmsg_type != SCM_RIGHTS && cm->cmsg_type != SCM_CREDS) ||
1713 	    cm->cmsg_level != SOL_SOCKET ||
1714 	    control->m_len < sizeof(*cm) ||	/* control too small */
1715 	    cm->cmsg_len < sizeof(*cm) ||	/* cmsg_len too small */
1716 	    cm->cmsg_len > control->m_len) {	/* cmsg_len too big */
1717 		return EINVAL;
1718 	}
1719 
1720 	/*
1721 	 * Fill in credential information.
1722 	 */
1723 	if (cm->cmsg_type == SCM_CREDS) {
1724 		cmcred = (struct cmsgcred *)CMSG_DATA(cm);
1725 		cmcred->cmcred_pid = p->p_pid;
1726 		cmcred->cmcred_uid = p->p_ucred->cr_ruid;
1727 		cmcred->cmcred_gid = p->p_ucred->cr_rgid;
1728 		cmcred->cmcred_euid = p->p_ucred->cr_uid;
1729 		cmcred->cmcred_ngroups = MIN(p->p_ucred->cr_ngroups,
1730 							CMGROUP_MAX);
1731 		for (i = 0; i < cmcred->cmcred_ngroups; i++)
1732 			cmcred->cmcred_groups[i] = p->p_ucred->cr_groups[i];
1733 		return 0;
1734 	}
1735 
1736 	/*
1737 	 * cmsghdr may not be aligned, do not allow calculation(s) to
1738 	 * go negative.
1739 	 *
1740 	 * Data must be aligned but the data length does not have to be.
1741 	 *
1742 	 * If there are multiple headers (XXX not supported) then the
1743 	 * next header will be aligned after the end of the possibly
1744 	 * unaligned data.
1745 	 */
1746 	if (cm->cmsg_len < CMSG_LEN(0)) {
1747 		return EINVAL;
1748 	}
1749 
1750 	oldfds = (cm->cmsg_len - CMSG_LEN(0)) / sizeof(int);
1751 
1752 	/*
1753 	 * Now replace the integer FDs with pointers to
1754 	 * the associated global file table entry..
1755 	 * Allocate a bigger buffer as necessary. But if an cluster is not
1756 	 * enough, return E2BIG.
1757 	 */
1758 	newlen = CMSG_LEN(oldfds * sizeof(struct file *));
1759 	if (newlen > MCLBYTES)
1760 		return E2BIG;
1761 	if (newlen - control->m_len > M_TRAILINGSPACE(control)) {
1762 		if (control->m_flags & M_EXT)
1763 			return E2BIG;
1764 		MCLGET(control, M_WAITOK);
1765 		if (!(control->m_flags & M_EXT))
1766 			return ENOBUFS;
1767 
1768 		/* copy the data to the cluster */
1769 		memcpy(mtod(control, char *), cm, cm->cmsg_len);
1770 		cm = mtod(control, struct cmsghdr *);
1771 	}
1772 
1773 	lwkt_gettoken(&unp_rights_token);
1774 
1775 	fdescp = p->p_fd;
1776 	spin_lock_shared(&fdescp->fd_spin);
1777 
1778 	/*
1779 	 * check that all the FDs passed in refer to legal OPEN files
1780 	 * If not, reject the entire operation.
1781 	 */
1782 	fdp = (int *)CMSG_DATA(cm);
1783 	for (i = 0; i < oldfds; i++) {
1784 		fd = *fdp++;
1785 		if ((unsigned)fd >= fdescp->fd_nfiles ||
1786 		    fdescp->fd_files[fd].fp == NULL) {
1787 			error = EBADF;
1788 			goto done;
1789 		}
1790 		if (fdescp->fd_files[fd].fp->f_type == DTYPE_KQUEUE) {
1791 			error = EOPNOTSUPP;
1792 			goto done;
1793 		}
1794 	}
1795 
1796 	/*
1797 	 * Adjust length, in case sizeof(struct file *) and sizeof(int)
1798 	 * differs.
1799 	 */
1800 	cm->cmsg_len = newlen;
1801 	control->m_len = CMSG_ALIGN(newlen);
1802 
1803 	/*
1804 	 * Transform the file descriptors into struct file pointers.
1805 	 * Since the sizeof(struct file *) is bigger than or equal to
1806 	 * the sizeof(int), we do it in reverse order so that the int
1807 	 * won't get trashed until we're done.
1808 	 */
1809 	fdp = (int *)CMSG_DATA(cm) + oldfds - 1;
1810 	rp = (struct file **)CMSG_DATA(cm) + oldfds - 1;
1811 	for (i = 0; i < oldfds; i++) {
1812 		fp = fdescp->fd_files[*fdp--].fp;
1813 		*rp-- = fp;
1814 		fhold(fp);
1815 		unp_add_right(fp);
1816 	}
1817 	error = 0;
1818 done:
1819 	spin_unlock_shared(&fdescp->fd_spin);
1820 	lwkt_reltoken(&unp_rights_token);
1821 	return error;
1822 }
1823 
1824 #ifdef UNP_GC_ALLFILES
1825 
1826 /*
1827  * Garbage collect in-transit file descriptors that get lost due to
1828  * loops (i.e. when a socket is sent to another process over itself,
1829  * and more complex situations).
1830  *
1831  * NOT MPSAFE - TODO socket flush code and maybe fdrop.  Rest is MPSAFE.
1832  */
1833 
1834 struct unp_gc_info {
1835 	struct file **extra_ref;
1836 	struct file *locked_fp;
1837 	int defer;
1838 	int index;
1839 	int maxindex;
1840 };
1841 
1842 static void
1843 unp_gc(void *arg __unused, int pending __unused)
1844 {
1845 	struct unp_gc_info info;
1846 	struct file **fpp;
1847 	int i;
1848 
1849 	lwkt_gettoken(&unp_rights_token);
1850 
1851 	/*
1852 	 * Before going through all this, set all FDs to be NOT defered
1853 	 * and NOT externally accessible (not marked).  During the scan
1854 	 * a fd can be marked externally accessible but we may or may not
1855 	 * be able to immediately process it (controlled by FDEFER).
1856 	 *
1857 	 * If we loop sleep a bit.  The complexity of the topology can cause
1858 	 * multiple loops.  Also failure to acquire the socket's so_rcv
1859 	 * token can cause us to loop.
1860 	 */
1861 	allfiles_scan_exclusive(unp_gc_clearmarks, NULL);
1862 	do {
1863 		info.defer = 0;
1864 		allfiles_scan_exclusive(unp_gc_checkmarks, &info);
1865 		if (info.defer)
1866 			tsleep(&info, 0, "gcagain", 1);
1867 	} while (info.defer);
1868 
1869 	/*
1870 	 * We grab an extra reference to each of the file table entries
1871 	 * that are not otherwise accessible and then free the rights
1872 	 * that are stored in messages on them.
1873 	 *
1874 	 * The bug in the orginal code is a little tricky, so I'll describe
1875 	 * what's wrong with it here.
1876 	 *
1877 	 * It is incorrect to simply unp_discard each entry for f_msgcount
1878 	 * times -- consider the case of sockets A and B that contain
1879 	 * references to each other.  On a last close of some other socket,
1880 	 * we trigger a gc since the number of outstanding rights (unp_rights)
1881 	 * is non-zero.  If during the sweep phase the gc code unp_discards,
1882 	 * we end up doing a (full) fdrop on the descriptor.  A fdrop on A
1883 	 * results in the following chain.  Closef calls soo_close, which
1884 	 * calls soclose.   Soclose calls first (through the switch
1885 	 * uipc_usrreq) unp_detach, which re-invokes unp_gc.  Unp_gc simply
1886 	 * returns because the previous instance had set unp_gcing, and
1887 	 * we return all the way back to soclose, which marks the socket
1888 	 * with SS_NOFDREF, and then calls sofree.  Sofree calls sorflush
1889 	 * to free up the rights that are queued in messages on the socket A,
1890 	 * i.e., the reference on B.  The sorflush calls via the dom_dispose
1891 	 * switch unp_dispose, which unp_scans with unp_discard.  This second
1892 	 * instance of unp_discard just calls fdrop on B.
1893 	 *
1894 	 * Well, a similar chain occurs on B, resulting in a sorflush on B,
1895 	 * which results in another fdrop on A.  Unfortunately, A is already
1896 	 * being closed, and the descriptor has already been marked with
1897 	 * SS_NOFDREF, and soclose panics at this point.
1898 	 *
1899 	 * Here, we first take an extra reference to each inaccessible
1900 	 * descriptor.  Then, we call sorflush ourself, since we know
1901 	 * it is a Unix domain socket anyhow.  After we destroy all the
1902 	 * rights carried in messages, we do a last fdrop to get rid
1903 	 * of our extra reference.  This is the last close, and the
1904 	 * unp_detach etc will shut down the socket.
1905 	 *
1906 	 * 91/09/19, bsy@cs.cmu.edu
1907 	 */
1908 	info.extra_ref = kmalloc(256 * sizeof(struct file *), M_FILE, M_WAITOK);
1909 	info.maxindex = 256;
1910 
1911 	do {
1912 		/*
1913 		 * Look for matches
1914 		 */
1915 		info.index = 0;
1916 		allfiles_scan_exclusive(unp_gc_checkrefs, &info);
1917 
1918 		/*
1919 		 * For each FD on our hit list, do the following two things
1920 		 */
1921 		for (i = info.index, fpp = info.extra_ref; --i >= 0; ++fpp) {
1922 			struct file *tfp = *fpp;
1923 			if (tfp->f_type == DTYPE_SOCKET && tfp->f_data != NULL)
1924 				sorflush((struct socket *)(tfp->f_data));
1925 		}
1926 		for (i = info.index, fpp = info.extra_ref; --i >= 0; ++fpp)
1927 			fdrop(*fpp);
1928 	} while (info.index == info.maxindex);
1929 
1930 	kfree((caddr_t)info.extra_ref, M_FILE);
1931 
1932 	lwkt_reltoken(&unp_rights_token);
1933 }
1934 
1935 /*
1936  * MPSAFE - NOTE: filehead list and file pointer spinlocked on entry
1937  */
1938 static int
1939 unp_gc_checkrefs(struct file *fp, void *data)
1940 {
1941 	struct unp_gc_info *info = data;
1942 
1943 	if (fp->f_count == 0)
1944 		return(0);
1945 	if (info->index == info->maxindex)
1946 		return(-1);
1947 
1948 	/*
1949 	 * If all refs are from msgs, and it's not marked accessible
1950 	 * then it must be referenced from some unreachable cycle
1951 	 * of (shut-down) FDs, so include it in our
1952 	 * list of FDs to remove
1953 	 */
1954 	if (fp->f_count == fp->f_msgcount && !(fp->f_flag & FMARK)) {
1955 		info->extra_ref[info->index++] = fp;
1956 		fhold(fp);
1957 	}
1958 	return(0);
1959 }
1960 
1961 /*
1962  * MPSAFE - NOTE: filehead list and file pointer spinlocked on entry
1963  */
1964 static int
1965 unp_gc_clearmarks(struct file *fp, void *data __unused)
1966 {
1967 	atomic_clear_int(&fp->f_flag, FMARK | FDEFER);
1968 	return(0);
1969 }
1970 
1971 /*
1972  * MPSAFE - NOTE: filehead list and file pointer spinlocked on entry
1973  */
1974 static int
1975 unp_gc_checkmarks(struct file *fp, void *data)
1976 {
1977 	struct unp_gc_info *info = data;
1978 	struct socket *so;
1979 
1980 	/*
1981 	 * If the file is not open, skip it.  Make sure it isn't marked
1982 	 * defered or we could loop forever, in case we somehow race
1983 	 * something.
1984 	 */
1985 	if (fp->f_count == 0) {
1986 		if (fp->f_flag & FDEFER)
1987 			atomic_clear_int(&fp->f_flag, FDEFER);
1988 		return(0);
1989 	}
1990 	/*
1991 	 * If we already marked it as 'defer'  in a
1992 	 * previous pass, then try process it this time
1993 	 * and un-mark it
1994 	 */
1995 	if (fp->f_flag & FDEFER) {
1996 		atomic_clear_int(&fp->f_flag, FDEFER);
1997 	} else {
1998 		/*
1999 		 * if it's not defered, then check if it's
2000 		 * already marked.. if so skip it
2001 		 */
2002 		if (fp->f_flag & FMARK)
2003 			return(0);
2004 		/*
2005 		 * If all references are from messages
2006 		 * in transit, then skip it. it's not
2007 		 * externally accessible.
2008 		 */
2009 		if (fp->f_count == fp->f_msgcount)
2010 			return(0);
2011 		/*
2012 		 * If it got this far then it must be
2013 		 * externally accessible.
2014 		 */
2015 		atomic_set_int(&fp->f_flag, FMARK);
2016 	}
2017 
2018 	/*
2019 	 * either it was defered, or it is externally
2020 	 * accessible and not already marked so.
2021 	 * Now check if it is possibly one of OUR sockets.
2022 	 */
2023 	if (fp->f_type != DTYPE_SOCKET ||
2024 	    (so = (struct socket *)fp->f_data) == NULL) {
2025 		return(0);
2026 	}
2027 	if (so->so_proto->pr_domain != &localdomain ||
2028 	    !(so->so_proto->pr_flags & PR_RIGHTS)) {
2029 		return(0);
2030 	}
2031 
2032 	/*
2033 	 * So, Ok, it's one of our sockets and it IS externally accessible
2034 	 * (or was defered).  Now we look to see if we hold any file
2035 	 * descriptors in its message buffers.  Follow those links and mark
2036 	 * them as accessible too.
2037 	 *
2038 	 * We are holding multiple spinlocks here, if we cannot get the
2039 	 * token non-blocking defer until the next loop.
2040 	 */
2041 	info->locked_fp = fp;
2042 	if (lwkt_trytoken(&so->so_rcv.ssb_token)) {
2043 		unp_scan(so->so_rcv.ssb_mb, unp_mark, info);
2044 		lwkt_reltoken(&so->so_rcv.ssb_token);
2045 	} else {
2046 		atomic_set_int(&fp->f_flag, FDEFER);
2047 		++info->defer;
2048 	}
2049 	return (0);
2050 }
2051 
2052 /*
2053  * Mark visibility.  info->defer is recalculated on every pass.
2054  */
2055 static void
2056 unp_mark(struct file *fp, void *data)
2057 {
2058 	struct unp_gc_info *info = data;
2059 
2060 	if ((fp->f_flag & FMARK) == 0) {
2061 		++info->defer;
2062 		atomic_set_int(&fp->f_flag, FMARK | FDEFER);
2063 	} else if (fp->f_flag & FDEFER) {
2064 		++info->defer;
2065 	}
2066 }
2067 
2068 #else	/* !UNP_GC_ALLFILES */
2069 
2070 /*
2071  * They are thread local and do not require explicit synchronization.
2072  */
2073 static int	unp_marked;
2074 static int	unp_unreachable;
2075 
2076 static void
2077 unp_accessable(struct file *fp, void *data __unused)
2078 {
2079 	struct unpcb *unp;
2080 
2081 	if ((unp = unp_fp2unpcb(fp)) == NULL)
2082 		return;
2083 	if (unp->unp_gcflags & UNPGC_REF)
2084 		return;
2085 	unp->unp_gcflags &= ~UNPGC_DEAD;
2086 	unp->unp_gcflags |= UNPGC_REF;
2087 	unp_marked++;
2088 }
2089 
2090 static void
2091 unp_gc_process(struct unpcb *unp)
2092 {
2093 	struct file *fp;
2094 
2095 	/* Already processed. */
2096 	if (unp->unp_gcflags & UNPGC_SCANNED)
2097 		return;
2098 	fp = unp->unp_fp;
2099 
2100 	/*
2101 	 * Check for a socket potentially in a cycle.  It must be in a
2102 	 * queue as indicated by msgcount, and this must equal the file
2103 	 * reference count.  Note that when msgcount is 0 the file is NULL.
2104 	 */
2105 	if ((unp->unp_gcflags & UNPGC_REF) == 0 && fp &&
2106 	    unp->unp_msgcount != 0 && fp->f_count == unp->unp_msgcount) {
2107 		unp->unp_gcflags |= UNPGC_DEAD;
2108 		unp_unreachable++;
2109 		return;
2110 	}
2111 
2112 	/*
2113 	 * Mark all sockets we reference with RIGHTS.
2114 	 */
2115 	if (UNP_ISATTACHED(unp)) {
2116 		struct signalsockbuf *ssb = &unp->unp_socket->so_rcv;
2117 
2118 		unp_reference(unp);
2119 		lwkt_gettoken(&ssb->ssb_token);
2120 		/*
2121 		 * unp_token would be temporarily dropped, if getting
2122 		 * so_rcv token blocks, so we need to check unp state
2123 		 * here again.
2124 		 */
2125 		if (UNP_ISATTACHED(unp))
2126 			unp_scan(ssb->ssb_mb, unp_accessable, NULL);
2127 		lwkt_reltoken(&ssb->ssb_token);
2128 		unp->unp_gcflags |= UNPGC_SCANNED;
2129 		unp_free(unp);
2130 	} else {
2131 		unp->unp_gcflags |= UNPGC_SCANNED;
2132 	}
2133 }
2134 
2135 static void
2136 unp_gc(void *arg __unused, int pending __unused)
2137 {
2138 	struct unp_global_head *head;
2139 	int h, filemax, fileidx, filetot;
2140 	struct file **unref;
2141 	struct unpcb *unp;
2142 
2143 	lwkt_gettoken(&unp_rights_token);
2144 	lwkt_gettoken(&unp_token);
2145 
2146 	/*
2147 	 * First clear all gc flags from previous runs.
2148 	 */
2149 	for (h = 0; unp_heads[h] != NULL; ++h) {
2150 		/*
2151 		 * NOTE: This loop does not block, so it is safe
2152 		 * to use TAILQ_FOREACH here.
2153 		 */
2154 		head = unp_heads[h];
2155 		TAILQ_FOREACH(unp, &head->list, unp_link)
2156 			unp->unp_gcflags = 0;
2157 	}
2158 
2159 	/*
2160 	 * Scan marking all reachable sockets with UNPGC_REF.  Once a socket
2161 	 * is reachable all of the sockets it references are reachable.
2162 	 * Stop the scan once we do a complete loop without discovering
2163 	 * a new reachable socket.
2164 	 */
2165 	do {
2166 		unp_unreachable = 0;
2167 		unp_marked = 0;
2168 		for (h = 0; unp_heads[h] != NULL; ++h) {
2169 			head = unp_heads[h];
2170 			TAILQ_INSERT_HEAD(&head->list, unp_gc_marker, unp_link);
2171 			while ((unp = TAILQ_NEXT(unp_gc_marker, unp_link))
2172 			    != NULL) {
2173 				TAILQ_REMOVE(&head->list, unp_gc_marker,
2174 				    unp_link);
2175 				TAILQ_INSERT_AFTER(&head->list, unp,
2176 				    unp_gc_marker, unp_link);
2177 
2178 				if (unp->unp_flags & UNP_MARKER)
2179 					continue;
2180 				unp_gc_process(unp);
2181 			}
2182 			TAILQ_REMOVE(&head->list, unp_gc_marker, unp_link);
2183 		}
2184 	} while (unp_marked);
2185 
2186 	if (unp_unreachable == 0)
2187 		goto done;
2188 
2189 	/*
2190 	 * We grab an extra reference to each of the file table entries
2191 	 * that are not otherwise accessible and then free the rights
2192 	 * that are stored in messages on them.
2193 	 *
2194 	 * The bug in the orginal code is a little tricky, so I'll describe
2195 	 * what's wrong with it here.
2196 	 *
2197 	 * It is incorrect to simply unp_discard each entry for f_msgcount
2198 	 * times -- consider the case of sockets A and B that contain
2199 	 * references to each other.  On a last close of some other socket,
2200 	 * we trigger a gc since the number of outstanding rights (unp_rights)
2201 	 * is non-zero.  If during the sweep phase the gc code unp_discards,
2202 	 * we end up doing a (full) fdrop on the descriptor.  A fdrop on A
2203 	 * results in the following chain.  Closef calls soo_close, which
2204 	 * calls soclose.   Soclose calls first (through the switch
2205 	 * uipc_usrreq) unp_detach, which re-invokes unp_gc.  Unp_gc simply
2206 	 * returns because the previous instance had set unp_gcing, and
2207 	 * we return all the way back to soclose, which marks the socket
2208 	 * with SS_NOFDREF, and then calls sofree.  Sofree calls sorflush
2209 	 * to free up the rights that are queued in messages on the socket A,
2210 	 * i.e., the reference on B.  The sorflush calls via the dom_dispose
2211 	 * switch unp_dispose, which unp_scans with unp_discard.  This second
2212 	 * instance of unp_discard just calls fdrop on B.
2213 	 *
2214 	 * Well, a similar chain occurs on B, resulting in a sorflush on B,
2215 	 * which results in another fdrop on A.  Unfortunately, A is already
2216 	 * being closed, and the descriptor has already been marked with
2217 	 * SS_NOFDREF, and soclose panics at this point.
2218 	 *
2219 	 * Here, we first take an extra reference to each inaccessible
2220 	 * descriptor.  Then, we call sorflush ourself, since we know
2221 	 * it is a Unix domain socket anyhow.  After we destroy all the
2222 	 * rights carried in messages, we do a last fdrop to get rid
2223 	 * of our extra reference.  This is the last close, and the
2224 	 * unp_detach etc will shut down the socket.
2225 	 *
2226 	 * 91/09/19, bsy@cs.cmu.edu
2227 	 */
2228 
2229 	filemax = unp_unreachable;
2230 	if (filemax > UNP_GCFILE_MAX)
2231 		filemax = UNP_GCFILE_MAX;
2232 	unref = kmalloc(filemax * sizeof(struct file *), M_TEMP, M_WAITOK);
2233 
2234 	filetot = 0;
2235 	do {
2236 		int i;
2237 
2238 		/*
2239 		 * Iterate looking for sockets which have been specifically
2240 		 * marked as as unreachable and store them locally.
2241 		 */
2242 		fileidx = 0;
2243 		for (h = 0; unp_heads[h] != NULL; ++h) {
2244 			/*
2245 			 * NOTE: This loop does not block, so it is safe
2246 			 * to use TAILQ_FOREACH here.
2247 			 */
2248 			head = unp_heads[h];
2249 			TAILQ_FOREACH(unp, &head->list, unp_link) {
2250 				struct file *fp;
2251 
2252 				if ((unp->unp_gcflags & UNPGC_DEAD) == 0)
2253 					continue;
2254 				unp->unp_gcflags &= ~UNPGC_DEAD;
2255 
2256 				fp = unp->unp_fp;
2257 				if (unp->unp_msgcount == 0 || fp == NULL ||
2258 				    fp->f_count != unp->unp_msgcount)
2259 					continue;
2260 				fhold(fp);
2261 
2262 				KASSERT(fileidx < filemax,
2263 				    ("invalid fileidx %d, filemax %d",
2264 				     fileidx, filemax));
2265 				unref[fileidx++] = fp;
2266 
2267 				KASSERT(filetot < unp_unreachable,
2268 				    ("invalid filetot %d and "
2269 				     "unp_unreachable %d",
2270 				     filetot, unp_unreachable));
2271 				++filetot;
2272 
2273 				if (fileidx == filemax ||
2274 				    filetot == unp_unreachable)
2275 					goto dogc;
2276 			}
2277 		}
2278 dogc:
2279 		/*
2280 		 * For each Unix domain socket on our hit list, do the
2281 		 * following two things.
2282 		 */
2283 		for (i = 0; i < fileidx; ++i)
2284 			sorflush(unref[i]->f_data);
2285 		for (i = 0; i < fileidx; ++i)
2286 			fdrop(unref[i]);
2287 	} while (fileidx == filemax && filetot < unp_unreachable);
2288 	kfree(unref, M_TEMP);
2289 done:
2290 	lwkt_reltoken(&unp_token);
2291 	lwkt_reltoken(&unp_rights_token);
2292 }
2293 
2294 #endif	/* UNP_GC_ALLFILES */
2295 
2296 /*
2297  * Dispose of the fp's stored in a mbuf.
2298  *
2299  * The dds loop can cause additional fps to be entered onto the
2300  * list while it is running, flattening out the operation and avoiding
2301  * a deep kernel stack recursion.
2302  */
2303 void
2304 unp_dispose(struct mbuf *m)
2305 {
2306 	lwkt_gettoken(&unp_rights_token);
2307 	if (m)
2308 		unp_scan(m, unp_discard, NULL);
2309 	lwkt_reltoken(&unp_rights_token);
2310 }
2311 
2312 static int
2313 unp_listen(struct unpcb *unp, struct thread *td)
2314 {
2315 	struct proc *p = td->td_proc;
2316 
2317 	ASSERT_LWKT_TOKEN_HELD(&unp_token);
2318 	UNP_ASSERT_TOKEN_HELD(unp);
2319 
2320 	KKASSERT(p);
2321 	cru2x(p->p_ucred, &unp->unp_peercred);
2322 	unp_setflags(unp, UNP_HAVEPCCACHED);
2323 	return (0);
2324 }
2325 
2326 static void
2327 unp_scan(struct mbuf *m0, void (*op)(struct file *, void *), void *data)
2328 {
2329 	struct mbuf *m;
2330 	struct file **rp;
2331 	struct cmsghdr *cm;
2332 	int i;
2333 	int qfds;
2334 
2335 	while (m0) {
2336 		for (m = m0; m; m = m->m_next) {
2337 			if (m->m_type == MT_CONTROL &&
2338 			    m->m_len >= sizeof(*cm)) {
2339 				cm = mtod(m, struct cmsghdr *);
2340 				if (cm->cmsg_level != SOL_SOCKET ||
2341 				    cm->cmsg_type != SCM_RIGHTS)
2342 					continue;
2343 				qfds = (cm->cmsg_len - CMSG_LEN(0)) /
2344 					sizeof(void *);
2345 				rp = (struct file **)CMSG_DATA(cm);
2346 				for (i = 0; i < qfds; i++)
2347 					(*op)(*rp++, data);
2348 				break;		/* XXX, but saves time */
2349 			}
2350 		}
2351 		m0 = m0->m_nextpkt;
2352 	}
2353 }
2354 
2355 /*
2356  * Discard a fp previously held in a unix domain socket mbuf.  To
2357  * avoid blowing out the kernel stack due to contrived chain-reactions
2358  * we may have to defer the operation to a dedicated taskqueue.
2359  *
2360  * Caller holds unp_rights_token.
2361  */
2362 static void
2363 unp_discard(struct file *fp, void *data __unused)
2364 {
2365 	unp_del_right(fp);
2366 	if (unp_fp2unpcb(fp) != NULL) {
2367 		struct unp_defdiscard *d;
2368 
2369 		/*
2370 		 * This fp is a Unix domain socket itself and fdrop()
2371 		 * it here directly may cause deep unp_discard()
2372 		 * recursion, so the fdrop() is defered to the
2373 		 * dedicated taskqueue.
2374 		 */
2375 		d = kmalloc(sizeof(*d), M_UNPCB, M_WAITOK);
2376 		d->fp = fp;
2377 
2378 		spin_lock(&unp_defdiscard_spin);
2379 		SLIST_INSERT_HEAD(&unp_defdiscard_head, d, next);
2380 		spin_unlock(&unp_defdiscard_spin);
2381 
2382 		taskqueue_enqueue(unp_taskqueue, &unp_defdiscard_task);
2383 	} else {
2384 		/* This fp is not a Unix domain socket */
2385 		fdrop(fp);
2386 	}
2387 }
2388 
2389 /*
2390  * NOTE:
2391  * unp_token must be held before calling this function to avoid name
2392  * resolution and v_socket accessing races, especially racing against
2393  * the unp_detach().
2394  *
2395  * NOTE:
2396  * For anyone caring about unconnected Unix domain socket sending
2397  * performance, other approach could be taken...
2398  */
2399 static int
2400 unp_find_lockref(struct sockaddr *nam, struct thread *td, short type,
2401     struct unpcb **unp_ret)
2402 {
2403 	struct proc *p = td->td_proc;
2404 	struct sockaddr_un *soun = (struct sockaddr_un *)nam;
2405 	struct vnode *vp = NULL;
2406 	struct socket *so;
2407 	struct unpcb *unp;
2408 	int error, len;
2409 	struct nlookupdata nd;
2410 	char buf[SOCK_MAXADDRLEN];
2411 
2412 	ASSERT_LWKT_TOKEN_HELD(&unp_token);
2413 
2414 	*unp_ret = NULL;
2415 
2416 	len = nam->sa_len - offsetof(struct sockaddr_un, sun_path);
2417 	if (len <= 0) {
2418 		error = EINVAL;
2419 		goto failed;
2420 	}
2421 	strncpy(buf, soun->sun_path, len);
2422 	buf[len] = 0;
2423 
2424 	error = nlookup_init(&nd, buf, UIO_SYSSPACE, NLC_FOLLOW);
2425 	if (error == 0)
2426 		error = nlookup(&nd);
2427 	if (error == 0)
2428 		error = cache_vget(&nd.nl_nch, nd.nl_cred, LK_EXCLUSIVE, &vp);
2429 	nlookup_done(&nd);
2430 	if (error) {
2431 		vp = NULL;
2432 		goto failed;
2433 	}
2434 
2435 	if (vp->v_type != VSOCK) {
2436 		error = ENOTSOCK;
2437 		goto failed;
2438 	}
2439 	error = VOP_EACCESS(vp, VWRITE, p->p_ucred);
2440 	if (error)
2441 		goto failed;
2442 	so = vp->v_socket;
2443 	if (so == NULL) {
2444 		error = ECONNREFUSED;
2445 		goto failed;
2446 	}
2447 	if (so->so_type != type) {
2448 		error = EPROTOTYPE;
2449 		goto failed;
2450 	}
2451 
2452 	/* Lock this unp. */
2453 	unp = unp_getsocktoken(so);
2454 	if (!UNP_ISATTACHED(unp)) {
2455 		unp_reltoken(unp);
2456 		error = ECONNREFUSED;
2457 		goto failed;
2458 	}
2459 	/* And keep this unp referenced. */
2460 	unp_reference(unp);
2461 
2462 	/* Done! */
2463 	*unp_ret = unp;
2464 	error = 0;
2465 failed:
2466 	if (vp != NULL)
2467 		vput(vp);
2468 	return error;
2469 }
2470 
2471 static int
2472 unp_connect_pair(struct unpcb *unp, struct unpcb *unp2)
2473 {
2474 	struct socket *so = unp->unp_socket;
2475 	struct socket *so2 = unp2->unp_socket;
2476 
2477 	ASSERT_LWKT_TOKEN_HELD(&unp_token);
2478 	UNP_ASSERT_TOKEN_HELD(unp);
2479 	UNP_ASSERT_TOKEN_HELD(unp2);
2480 
2481 	KASSERT(so->so_type == so2->so_type,
2482 	    ("socket type mismatch, so %d, so2 %d", so->so_type, so2->so_type));
2483 
2484 	if (!UNP_ISATTACHED(unp))
2485 		return EINVAL;
2486 	if (!UNP_ISATTACHED(unp2))
2487 		return ECONNREFUSED;
2488 
2489 	KASSERT(unp->unp_conn == NULL, ("unp is already connected"));
2490 	unp->unp_conn = unp2;
2491 
2492 	switch (so->so_type) {
2493 	case SOCK_DGRAM:
2494 		LIST_INSERT_HEAD(&unp2->unp_refs, unp, unp_reflink);
2495 		soisconnected(so);
2496 		break;
2497 
2498 	case SOCK_STREAM:
2499 	case SOCK_SEQPACKET:
2500 		KASSERT(unp2->unp_conn == NULL, ("unp2 is already connected"));
2501 		unp2->unp_conn = unp;
2502 		soisconnected(so);
2503 		soisconnected(so2);
2504 		break;
2505 
2506 	default:
2507 		panic("unp_connect_pair: unknown socket type %d", so->so_type);
2508 	}
2509 	return 0;
2510 }
2511 
2512 static void
2513 unp_drop(struct unpcb *unp, int error)
2514 {
2515 	struct unp_global_head *head;
2516 	struct unpcb *unp2;
2517 
2518 	ASSERT_LWKT_TOKEN_HELD(&unp_token);
2519 	UNP_ASSERT_TOKEN_HELD(unp);
2520 
2521 	KASSERT((unp->unp_flags & (UNP_DETACHED | UNP_DROPPED)) == 0,
2522 	    ("unp is dropped"));
2523 
2524 	/* Mark this unp as detached. */
2525 	unp_setflags(unp, UNP_DETACHED);
2526 
2527 	/* Remove this unp from the global unp list. */
2528 	head = unp_globalhead(unp->unp_socket->so_type);
2529 	KASSERT(head->count > 0, ("invalid unp count"));
2530 	TAILQ_REMOVE(&head->list, unp, unp_link);
2531 	head->count--;
2532 
2533 	/* Disconnect all. */
2534 	unp_disconnect(unp, error);
2535 	while ((unp2 = LIST_FIRST(&unp->unp_refs)) != NULL) {
2536 		lwkt_getpooltoken(unp2);
2537 		unp_disconnect(unp2, ECONNRESET);
2538 		lwkt_relpooltoken(unp2);
2539 	}
2540 	unp_setflags(unp, UNP_DROPPED);
2541 
2542 	/* Try freeing this unp. */
2543 	unp_free(unp);
2544 }
2545 
2546 static void
2547 unp_defdiscard_taskfunc(void *arg __unused, int pending __unused)
2548 {
2549 	struct unp_defdiscard *d;
2550 
2551 	spin_lock(&unp_defdiscard_spin);
2552 	while ((d = SLIST_FIRST(&unp_defdiscard_head)) != NULL) {
2553 		SLIST_REMOVE_HEAD(&unp_defdiscard_head, next);
2554 		spin_unlock(&unp_defdiscard_spin);
2555 
2556 		fdrop(d->fp);
2557 		kfree(d, M_UNPCB);
2558 
2559 		spin_lock(&unp_defdiscard_spin);
2560 	}
2561 	spin_unlock(&unp_defdiscard_spin);
2562 }
2563