1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 1995, 2010, Oracle and/or its affiliates. All rights reserved.
24 * Copyright 2015, Joyent, Inc. All rights reserved.
25 * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
26 * Copyright 2015 Nexenta Systems, Inc. All rights reserved.
27 * Copyright 2020 OmniOS Community Edition (OmniOSce) Association.
28 * Copyright 2022 Garrett D'Amore
29 */
30
31 #include <sys/types.h>
32 #include <sys/t_lock.h>
33 #include <sys/param.h>
34 #include <sys/systm.h>
35 #include <sys/buf.h>
36 #include <sys/conf.h>
37 #include <sys/cred.h>
38 #include <sys/kmem.h>
39 #include <sys/sysmacros.h>
40 #include <sys/vfs.h>
41 #include <sys/vnode.h>
42 #include <sys/debug.h>
43 #include <sys/errno.h>
44 #include <sys/time.h>
45 #include <sys/file.h>
46 #include <sys/user.h>
47 #include <sys/stream.h>
48 #include <sys/strsubr.h>
49 #include <sys/strsun.h>
50 #include <sys/sunddi.h>
51 #include <sys/esunddi.h>
52 #include <sys/flock.h>
53 #include <sys/modctl.h>
54 #include <sys/cmn_err.h>
55 #include <sys/vmsystm.h>
56 #include <sys/policy.h>
57 #include <sys/limits.h>
58
59 #include <sys/socket.h>
60 #include <sys/socketvar.h>
61
62 #include <sys/isa_defs.h>
63 #include <sys/inttypes.h>
64 #include <sys/systm.h>
65 #include <sys/cpuvar.h>
66 #include <sys/filio.h>
67 #include <sys/sendfile.h>
68 #include <sys/ddi.h>
69 #include <vm/seg.h>
70 #include <vm/seg_map.h>
71 #include <vm/seg_kpm.h>
72
73 #include <fs/sockfs/sockcommon.h>
74 #include <fs/sockfs/sockfilter_impl.h>
75 #include <fs/sockfs/socktpi.h>
76
77 #ifdef SOCK_TEST
78 int do_useracc = 1; /* Controlled by setting SO_DEBUG to 4 */
79 #else
80 #define do_useracc 1
81 #endif /* SOCK_TEST */
82
83 extern int xnet_truncate_print;
84
85 /*
86 * Kernel component of socket creation.
87 *
88 * The socket library determines which version number to use.
89 * First the library calls this with a NULL devpath. If this fails
90 * to find a transport (using solookup) the library will look in /etc/netconfig
91 * for the appropriate transport. If one is found it will pass in the
92 * devpath for the kernel to use.
93 */
94 int
so_socket(int family,int type_w_flags,int protocol,char * devpath,int version)95 so_socket(int family, int type_w_flags, int protocol, char *devpath,
96 int version)
97 {
98 struct sonode *so;
99 vnode_t *vp;
100 struct file *fp;
101 int fd;
102 int error;
103 int type;
104
105 type = type_w_flags & SOCK_TYPE_MASK;
106 type_w_flags &= ~SOCK_TYPE_MASK;
107 if (type_w_flags & ~(SOCK_CLOEXEC|SOCK_NDELAY|SOCK_NONBLOCK))
108 return (set_errno(EINVAL));
109
110 if (devpath != NULL) {
111 char *buf;
112 size_t kdevpathlen = 0;
113
114 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
115 if ((error = copyinstr(devpath, buf,
116 MAXPATHLEN, &kdevpathlen)) != 0) {
117 kmem_free(buf, MAXPATHLEN);
118 return (set_errno(error));
119 }
120 so = socket_create(family, type, protocol, buf, NULL,
121 SOCKET_SLEEP, version, CRED(), &error);
122 kmem_free(buf, MAXPATHLEN);
123 } else {
124 so = socket_create(family, type, protocol, NULL, NULL,
125 SOCKET_SLEEP, version, CRED(), &error);
126 }
127 if (so == NULL)
128 return (set_errno(error));
129
130 /* Allocate a file descriptor for the socket */
131 vp = SOTOV(so);
132 error = falloc(vp, FWRITE|FREAD, &fp, &fd);
133 if (error != 0) {
134 (void) socket_close(so, 0, CRED());
135 socket_destroy(so);
136 return (set_errno(error));
137 }
138
139 /*
140 * Now fill in the entries that falloc reserved
141 */
142 if (type_w_flags & SOCK_NDELAY) {
143 so->so_state |= SS_NDELAY;
144 fp->f_flag |= FNDELAY;
145 }
146 if (type_w_flags & SOCK_NONBLOCK) {
147 so->so_state |= SS_NONBLOCK;
148 fp->f_flag |= FNONBLOCK;
149 }
150 mutex_exit(&fp->f_tlock);
151 setf(fd, fp);
152 if ((type_w_flags & SOCK_CLOEXEC) != 0) {
153 f_setfd(fd, FD_CLOEXEC);
154 }
155
156 return (fd);
157 }
158
159 /*
160 * Map from a file descriptor to a socket node.
161 * Returns with the file descriptor held i.e. the caller has to
162 * use releasef when done with the file descriptor.
163 */
164 struct sonode *
getsonode(int sock,int * errorp,file_t ** fpp)165 getsonode(int sock, int *errorp, file_t **fpp)
166 {
167 file_t *fp;
168 vnode_t *vp;
169 struct sonode *so;
170
171 if ((fp = getf(sock)) == NULL) {
172 *errorp = EBADF;
173 eprintline(*errorp);
174 return (NULL);
175 }
176 vp = fp->f_vnode;
177 /* Check if it is a socket */
178 if (vp->v_type != VSOCK) {
179 releasef(sock);
180 *errorp = ENOTSOCK;
181 eprintline(*errorp);
182 return (NULL);
183 }
184 /*
185 * Use the stream head to find the real socket vnode.
186 * This is needed when namefs sits above sockfs.
187 */
188 if (vp->v_stream) {
189 ASSERT(vp->v_stream->sd_vnode);
190 vp = vp->v_stream->sd_vnode;
191
192 so = VTOSO(vp);
193 if (so->so_version == SOV_STREAM) {
194 releasef(sock);
195 *errorp = ENOTSOCK;
196 eprintsoline(so, *errorp);
197 return (NULL);
198 }
199 } else {
200 so = VTOSO(vp);
201 }
202 if (fpp)
203 *fpp = fp;
204 return (so);
205 }
206
207 /*
208 * Allocate and copyin a sockaddr.
209 * Ensures NULL termination for AF_UNIX addresses by extending them
210 * with one NULL byte if need be. Verifies that the length is not
211 * excessive to prevent an application from consuming all of kernel
212 * memory. Returns NULL when an error occurred.
213 */
214 static struct sockaddr *
copyin_name(struct sonode * so,struct sockaddr * name,socklen_t * namelenp,int * errorp)215 copyin_name(struct sonode *so, struct sockaddr *name, socklen_t *namelenp,
216 int *errorp)
217 {
218 char *faddr;
219 size_t namelen = (size_t)*namelenp;
220
221 ASSERT(namelen != 0);
222 if (namelen > SO_MAXARGSIZE) {
223 *errorp = EINVAL;
224 eprintsoline(so, *errorp);
225 return (NULL);
226 }
227
228 faddr = (char *)kmem_alloc(namelen, KM_SLEEP);
229 if (copyin(name, faddr, namelen)) {
230 kmem_free(faddr, namelen);
231 *errorp = EFAULT;
232 eprintsoline(so, *errorp);
233 return (NULL);
234 }
235
236 /*
237 * Add space for NULL termination if needed.
238 * Do a quick check if the last byte is NUL.
239 */
240 if (so->so_family == AF_UNIX && faddr[namelen - 1] != '\0') {
241 /* Check if there is any NULL termination */
242 size_t i;
243 int foundnull = 0;
244
245 for (i = sizeof (name->sa_family); i < namelen; i++) {
246 if (faddr[i] == '\0') {
247 foundnull = 1;
248 break;
249 }
250 }
251 if (!foundnull) {
252 /* Add extra byte for NUL padding */
253 char *nfaddr;
254
255 nfaddr = (char *)kmem_alloc(namelen + 1, KM_SLEEP);
256 bcopy(faddr, nfaddr, namelen);
257 kmem_free(faddr, namelen);
258
259 /* NUL terminate */
260 nfaddr[namelen] = '\0';
261 namelen++;
262 ASSERT((socklen_t)namelen == namelen);
263 *namelenp = (socklen_t)namelen;
264 faddr = nfaddr;
265 }
266 }
267 return ((struct sockaddr *)faddr);
268 }
269
270 /*
271 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
272 */
273 static int
copyout_arg(void * uaddr,socklen_t ulen,void * ulenp,void * kaddr,socklen_t klen)274 copyout_arg(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr,
275 socklen_t klen)
276 {
277 if (uaddr != NULL) {
278 if (ulen > klen)
279 ulen = klen;
280
281 if (ulen != 0) {
282 if (copyout(kaddr, uaddr, ulen))
283 return (EFAULT);
284 }
285 } else
286 ulen = 0;
287
288 if (ulenp != NULL) {
289 if (copyout(&ulen, ulenp, sizeof (ulen)))
290 return (EFAULT);
291 }
292 return (0);
293 }
294
295 /*
296 * Copy from kaddr/klen to uaddr/ulen. Updates ulenp if non-NULL.
297 * If klen is greater than ulen it still uses the non-truncated
298 * klen to update ulenp.
299 */
300 static int
copyout_name(void * uaddr,socklen_t ulen,void * ulenp,void * kaddr,socklen_t klen)301 copyout_name(void *uaddr, socklen_t ulen, void *ulenp, void *kaddr,
302 socklen_t klen)
303 {
304 if (uaddr != NULL) {
305 if (ulen >= klen)
306 ulen = klen;
307 else if (ulen != 0 && xnet_truncate_print) {
308 printf("sockfs: truncating copyout of address using "
309 "XNET semantics for pid = %d. Lengths %d, %d\n",
310 curproc->p_pid, klen, ulen);
311 }
312
313 if (ulen != 0) {
314 if (copyout(kaddr, uaddr, ulen))
315 return (EFAULT);
316 } else
317 klen = 0;
318 } else
319 klen = 0;
320
321 if (ulenp != NULL) {
322 if (copyout(&klen, ulenp, sizeof (klen)))
323 return (EFAULT);
324 }
325 return (0);
326 }
327
328 /*
329 * The socketpair() code in libsocket creates two sockets (using
330 * the /etc/netconfig fallback if needed) before calling this routine
331 * to connect the two sockets together.
332 *
333 * For a SOCK_STREAM socketpair a listener is needed - in that case this
334 * routine will create a new file descriptor as part of accepting the
335 * connection. The library socketpair() will check if svs[2] has changed
336 * in which case it will close the changed fd.
337 *
338 * Note that this code could use the TPI feature of accepting the connection
339 * on the listening endpoint. However, that would require significant changes
340 * to soaccept.
341 */
342 int
so_socketpair(int sv[2])343 so_socketpair(int sv[2])
344 {
345 int svs[2];
346 struct sonode *so1, *so2;
347 int error;
348 int orig_flags;
349 struct sockaddr_ux *name;
350 size_t namelen;
351 sotpi_info_t *sti1;
352 sotpi_info_t *sti2;
353
354 dprint(1, ("so_socketpair(%p)\n", (void *)sv));
355
356 error = useracc(sv, sizeof (svs), B_WRITE);
357 if (error && do_useracc)
358 return (set_errno(EFAULT));
359
360 if (copyin(sv, svs, sizeof (svs)))
361 return (set_errno(EFAULT));
362
363 if ((so1 = getsonode(svs[0], &error, NULL)) == NULL)
364 return (set_errno(error));
365
366 if ((so2 = getsonode(svs[1], &error, NULL)) == NULL) {
367 releasef(svs[0]);
368 return (set_errno(error));
369 }
370
371 if (so1->so_family != AF_UNIX || so2->so_family != AF_UNIX) {
372 error = EOPNOTSUPP;
373 goto done;
374 }
375
376 sti1 = SOTOTPI(so1);
377 sti2 = SOTOTPI(so2);
378
379 /*
380 * The code below makes assumptions about the "sockfs" implementation.
381 * So make sure that the correct implementation is really used.
382 */
383 ASSERT(so1->so_ops == &sotpi_sonodeops);
384 ASSERT(so2->so_ops == &sotpi_sonodeops);
385
386 if (so1->so_type == SOCK_DGRAM) {
387 /*
388 * Bind both sockets and connect them with each other.
389 * Need to allocate name/namelen for soconnect.
390 */
391 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC, CRED());
392 if (error) {
393 eprintsoline(so1, error);
394 goto done;
395 }
396 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
397 if (error) {
398 eprintsoline(so2, error);
399 goto done;
400 }
401 namelen = sizeof (struct sockaddr_ux);
402 name = kmem_alloc(namelen, KM_SLEEP);
403 name->sou_family = AF_UNIX;
404 name->sou_addr = sti2->sti_ux_laddr;
405 error = socket_connect(so1,
406 (struct sockaddr *)name,
407 (socklen_t)namelen,
408 0, _SOCONNECT_NOXLATE, CRED());
409 if (error) {
410 kmem_free(name, namelen);
411 eprintsoline(so1, error);
412 goto done;
413 }
414 name->sou_addr = sti1->sti_ux_laddr;
415 error = socket_connect(so2,
416 (struct sockaddr *)name,
417 (socklen_t)namelen,
418 0, _SOCONNECT_NOXLATE, CRED());
419 kmem_free(name, namelen);
420 if (error) {
421 eprintsoline(so2, error);
422 goto done;
423 }
424 releasef(svs[0]);
425 releasef(svs[1]);
426 } else {
427 /*
428 * Bind both sockets, with so1 being a listener.
429 * Connect so2 to so1 - nonblocking to avoid waiting for
430 * soaccept to complete.
431 * Accept a connection on so1. Pass out the new fd as sv[0].
432 * The library will detect the changed fd and close
433 * the original one.
434 */
435 struct sonode *nso;
436 struct vnode *nvp;
437 struct file *nfp;
438 int nfd;
439
440 /*
441 * We could simply call socket_listen() here (which would do the
442 * binding automatically) if the code didn't rely on passing
443 * _SOBIND_NOXLATE to the TPI implementation of socket_bind().
444 */
445 error = socket_bind(so1, NULL, 0, _SOBIND_UNSPEC|
446 _SOBIND_NOXLATE|_SOBIND_LISTEN|_SOBIND_SOCKETPAIR,
447 CRED());
448 if (error) {
449 eprintsoline(so1, error);
450 goto done;
451 }
452 error = socket_bind(so2, NULL, 0, _SOBIND_UNSPEC, CRED());
453 if (error) {
454 eprintsoline(so2, error);
455 goto done;
456 }
457
458 namelen = sizeof (struct sockaddr_ux);
459 name = kmem_alloc(namelen, KM_SLEEP);
460 name->sou_family = AF_UNIX;
461 name->sou_addr = sti1->sti_ux_laddr;
462 error = socket_connect(so2,
463 (struct sockaddr *)name,
464 (socklen_t)namelen,
465 FNONBLOCK, _SOCONNECT_NOXLATE, CRED());
466 kmem_free(name, namelen);
467 if (error) {
468 if (error != EINPROGRESS) {
469 eprintsoline(so2, error); goto done;
470 }
471 }
472
473 error = socket_accept(so1, 0, CRED(), &nso);
474 if (error) {
475 eprintsoline(so1, error);
476 goto done;
477 }
478
479 /* wait for so2 being SS_CONNECTED ignoring signals */
480 mutex_enter(&so2->so_lock);
481 error = sowaitconnected(so2, 0, 1);
482 mutex_exit(&so2->so_lock);
483 if (error != 0) {
484 (void) socket_close(nso, 0, CRED());
485 socket_destroy(nso);
486 eprintsoline(so2, error);
487 goto done;
488 }
489
490 nvp = SOTOV(nso);
491 error = falloc(nvp, FWRITE|FREAD, &nfp, &nfd);
492 if (error != 0) {
493 (void) socket_close(nso, 0, CRED());
494 socket_destroy(nso);
495 eprintsoline(nso, error);
496 goto done;
497 }
498 /*
499 * copy over FNONBLOCK and FNDELAY flags should they exist
500 */
501 if (so1->so_state & SS_NONBLOCK)
502 nfp->f_flag |= FNONBLOCK;
503 if (so1->so_state & SS_NDELAY)
504 nfp->f_flag |= FNDELAY;
505
506 /*
507 * fill in the entries that falloc reserved
508 */
509 mutex_exit(&nfp->f_tlock);
510 setf(nfd, nfp);
511
512 /*
513 * get the original flags before we release
514 */
515 VERIFY(f_getfd_error(svs[0], &orig_flags) == 0);
516
517 releasef(svs[0]);
518 releasef(svs[1]);
519
520 /*
521 * If FD_CLOEXEC was set on the filedescriptor we're
522 * swapping out, we should set it on the new one too.
523 */
524 if (orig_flags & FD_CLOEXEC) {
525 f_setfd(nfd, FD_CLOEXEC);
526 }
527
528 /*
529 * The socketpair library routine will close the original
530 * svs[0] when this code passes out a different file
531 * descriptor.
532 */
533 svs[0] = nfd;
534
535 if (copyout(svs, sv, sizeof (svs))) {
536 (void) closeandsetf(nfd, NULL);
537 eprintline(EFAULT);
538 return (set_errno(EFAULT));
539 }
540 }
541 return (0);
542
543 done:
544 releasef(svs[0]);
545 releasef(svs[1]);
546 return (set_errno(error));
547 }
548
549 int
bind(int sock,struct sockaddr * name,socklen_t namelen,int version)550 bind(int sock, struct sockaddr *name, socklen_t namelen, int version)
551 {
552 struct sonode *so;
553 int error;
554
555 dprint(1, ("bind(%d, %p, %d)\n",
556 sock, (void *)name, namelen));
557
558 if ((so = getsonode(sock, &error, NULL)) == NULL)
559 return (set_errno(error));
560
561 /* Allocate and copyin name */
562 /*
563 * X/Open test does not expect EFAULT with NULL name and non-zero
564 * namelen.
565 */
566 if (name != NULL && namelen != 0) {
567 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
568 name = copyin_name(so, name, &namelen, &error);
569 if (name == NULL) {
570 releasef(sock);
571 return (set_errno(error));
572 }
573 } else {
574 name = NULL;
575 namelen = 0;
576 }
577
578 switch (version) {
579 default:
580 error = socket_bind(so, name, namelen, 0, CRED());
581 break;
582 case SOV_XPG4_2:
583 error = socket_bind(so, name, namelen, _SOBIND_XPG4_2, CRED());
584 break;
585 case SOV_SOCKBSD:
586 error = socket_bind(so, name, namelen, _SOBIND_SOCKBSD, CRED());
587 break;
588 }
589
590 releasef(sock);
591 if (name != NULL)
592 kmem_free(name, (size_t)namelen);
593
594 if (error)
595 return (set_errno(error));
596 return (0);
597 }
598
599 /* ARGSUSED2 */
600 int
listen(int sock,int backlog,int version)601 listen(int sock, int backlog, int version)
602 {
603 struct sonode *so;
604 int error;
605
606 dprint(1, ("listen(%d, %d)\n",
607 sock, backlog));
608
609 if ((so = getsonode(sock, &error, NULL)) == NULL)
610 return (set_errno(error));
611
612 error = socket_listen(so, backlog, CRED());
613
614 releasef(sock);
615 if (error)
616 return (set_errno(error));
617 return (0);
618 }
619
620 /*ARGSUSED3*/
621 int
accept(int sock,struct sockaddr * name,socklen_t * namelenp,int version,int flags)622 accept(int sock, struct sockaddr *name, socklen_t *namelenp, int version,
623 int flags)
624 {
625 struct sonode *so;
626 file_t *fp;
627 int error;
628 socklen_t namelen;
629 struct sonode *nso;
630 struct vnode *nvp;
631 struct file *nfp;
632 int nfd;
633 int ssflags;
634 struct sockaddr *addrp;
635 socklen_t addrlen;
636
637 dprint(1, ("accept(%d, %p, %p)\n",
638 sock, (void *)name, (void *)namelenp));
639
640 if (flags & ~(SOCK_CLOEXEC|SOCK_NONBLOCK|SOCK_NDELAY)) {
641 return (set_errno(EINVAL));
642 }
643
644 /* Translate SOCK_ flags to their SS_ variant */
645 ssflags = 0;
646 if (flags & SOCK_NONBLOCK)
647 ssflags |= SS_NONBLOCK;
648 if (flags & SOCK_NDELAY)
649 ssflags |= SS_NDELAY;
650
651 if ((so = getsonode(sock, &error, &fp)) == NULL)
652 return (set_errno(error));
653
654 if (name != NULL) {
655 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
656 if (copyin(namelenp, &namelen, sizeof (namelen))) {
657 releasef(sock);
658 return (set_errno(EFAULT));
659 }
660 if (namelen != 0) {
661 error = useracc(name, (size_t)namelen, B_WRITE);
662 if (error && do_useracc) {
663 releasef(sock);
664 return (set_errno(EFAULT));
665 }
666 } else
667 name = NULL;
668 } else {
669 namelen = 0;
670 }
671
672 /*
673 * Allocate the user fd before socket_accept() in order to
674 * catch EMFILE errors before calling socket_accept().
675 */
676 if ((nfd = ufalloc(0)) == -1) {
677 eprintsoline(so, EMFILE);
678 releasef(sock);
679 return (set_errno(EMFILE));
680 }
681 error = socket_accept(so, fp->f_flag, CRED(), &nso);
682 if (error) {
683 setf(nfd, NULL);
684 releasef(sock);
685 return (set_errno(error));
686 }
687
688 nvp = SOTOV(nso);
689
690 ASSERT(MUTEX_NOT_HELD(&nso->so_lock));
691 if (namelen != 0) {
692 addrlen = so->so_max_addr_len;
693 addrp = (struct sockaddr *)kmem_alloc(addrlen, KM_SLEEP);
694
695 if ((error = socket_getpeername(nso, (struct sockaddr *)addrp,
696 &addrlen, B_TRUE, CRED())) == 0) {
697 error = copyout_name(name, namelen, namelenp,
698 addrp, addrlen);
699 } else {
700 ASSERT(error == EINVAL || error == ENOTCONN);
701 error = ECONNABORTED;
702 }
703 kmem_free(addrp, so->so_max_addr_len);
704 }
705
706 if (error) {
707 setf(nfd, NULL);
708 (void) socket_close(nso, 0, CRED());
709 socket_destroy(nso);
710 releasef(sock);
711 return (set_errno(error));
712 }
713 error = falloc(NULL, FWRITE|FREAD, &nfp, NULL);
714 if (error != 0) {
715 setf(nfd, NULL);
716 (void) socket_close(nso, 0, CRED());
717 socket_destroy(nso);
718 eprintsoline(so, error);
719 releasef(sock);
720 return (set_errno(error));
721 }
722 /*
723 * fill in the entries that falloc reserved
724 */
725 nfp->f_vnode = nvp;
726 mutex_exit(&nfp->f_tlock);
727 setf(nfd, nfp);
728
729 /*
730 * Act on SOCK_CLOEXEC from flags
731 */
732 if (flags & SOCK_CLOEXEC) {
733 f_setfd(nfd, FD_CLOEXEC);
734 }
735
736 /*
737 * Copy FNDELAY and FNONBLOCK from listener to acceptor
738 * and from ssflags
739 */
740 if ((ssflags | so->so_state) & (SS_NDELAY|SS_NONBLOCK)) {
741 uint_t oflag = nfp->f_flag;
742 int arg = 0;
743
744 if ((ssflags | so->so_state) & SS_NONBLOCK)
745 arg |= FNONBLOCK;
746 else if ((ssflags | so->so_state) & SS_NDELAY)
747 arg |= FNDELAY;
748
749 /*
750 * This code is a simplification of the F_SETFL code in fcntl()
751 * Ignore any errors from VOP_SETFL.
752 */
753 if ((error = VOP_SETFL(nvp, oflag, arg, nfp->f_cred, NULL))
754 != 0) {
755 eprintsoline(so, error);
756 error = 0;
757 } else {
758 mutex_enter(&nfp->f_tlock);
759 nfp->f_flag &= ~FMASK | (FREAD|FWRITE);
760 nfp->f_flag |= arg;
761 mutex_exit(&nfp->f_tlock);
762 }
763 }
764 releasef(sock);
765 return (nfd);
766 }
767
768 int
connect(int sock,struct sockaddr * name,socklen_t namelen,int version)769 connect(int sock, struct sockaddr *name, socklen_t namelen, int version)
770 {
771 struct sonode *so;
772 file_t *fp;
773 int error;
774
775 dprint(1, ("connect(%d, %p, %d)\n",
776 sock, (void *)name, namelen));
777
778 if ((so = getsonode(sock, &error, &fp)) == NULL)
779 return (set_errno(error));
780
781 /* Allocate and copyin name */
782 if (namelen != 0) {
783 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
784 name = copyin_name(so, name, &namelen, &error);
785 if (name == NULL) {
786 releasef(sock);
787 return (set_errno(error));
788 }
789 } else
790 name = NULL;
791
792 error = socket_connect(so, name, namelen, fp->f_flag,
793 (version != SOV_XPG4_2) ? 0 : _SOCONNECT_XPG4_2, CRED());
794 releasef(sock);
795 if (name)
796 kmem_free(name, (size_t)namelen);
797 if (error)
798 return (set_errno(error));
799 return (0);
800 }
801
802 /*ARGSUSED2*/
803 int
shutdown(int sock,int how,int version)804 shutdown(int sock, int how, int version)
805 {
806 struct sonode *so;
807 int error;
808
809 dprint(1, ("shutdown(%d, %d)\n",
810 sock, how));
811
812 if ((so = getsonode(sock, &error, NULL)) == NULL)
813 return (set_errno(error));
814
815 error = socket_shutdown(so, how, CRED());
816
817 releasef(sock);
818 if (error)
819 return (set_errno(error));
820 return (0);
821 }
822
823 /*
824 * Common receive routine.
825 */
826 static ssize_t
recvit(int sock,struct nmsghdr * msg,struct uio * uiop,int flags,socklen_t * namelenp,socklen_t * controllenp,int * flagsp)827 recvit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags,
828 socklen_t *namelenp, socklen_t *controllenp, int *flagsp)
829 {
830 struct sonode *so;
831 file_t *fp;
832 void *name;
833 socklen_t namelen;
834 void *control;
835 socklen_t controllen, free_controllen;
836 ssize_t len;
837 int error;
838
839 if ((so = getsonode(sock, &error, &fp)) == NULL)
840 return (set_errno(error));
841
842 len = uiop->uio_resid;
843 uiop->uio_fmode = fp->f_flag;
844 uiop->uio_extflg = UIO_COPY_CACHED;
845
846 name = msg->msg_name;
847 namelen = msg->msg_namelen;
848 control = msg->msg_control;
849 controllen = msg->msg_controllen;
850
851 msg->msg_flags = flags & (MSG_OOB | MSG_PEEK | MSG_WAITALL |
852 MSG_DONTWAIT | MSG_XPG4_2);
853
854 error = socket_recvmsg(so, msg, uiop, CRED());
855 if (error) {
856 releasef(sock);
857 return (set_errno(error));
858 }
859 lwp_stat_update(LWP_STAT_MSGRCV, 1);
860 releasef(sock);
861
862 free_controllen = msg->msg_controllen;
863
864 error = copyout_name(name, namelen, namelenp,
865 msg->msg_name, msg->msg_namelen);
866 if (error)
867 goto err;
868
869 if (flagsp != NULL) {
870 /*
871 * Clear internal flag.
872 */
873 msg->msg_flags &= ~MSG_XPG4_2;
874
875 /*
876 * Determine MSG_CTRUNC. sorecvmsg sets MSG_CTRUNC only
877 * when controllen is zero and there is control data to
878 * copy out.
879 */
880 if (controllen != 0 &&
881 (msg->msg_controllen > controllen || control == NULL)) {
882 dprint(1, ("recvit: CTRUNC %d %d %p\n",
883 msg->msg_controllen, controllen, control));
884
885 msg->msg_flags |= MSG_CTRUNC;
886 }
887 if (copyout(&msg->msg_flags, flagsp,
888 sizeof (msg->msg_flags))) {
889 error = EFAULT;
890 goto err;
891 }
892 }
893
894 if (controllen != 0) {
895 if (!(flags & MSG_XPG4_2)) {
896 /*
897 * Good old msg_accrights can only return a multiple
898 * of 4 bytes.
899 */
900 controllen &= ~((int)sizeof (uint32_t) - 1);
901 }
902
903 if (msg->msg_controllen > controllen || control == NULL) {
904 /*
905 * If the truncated part contains file descriptors,
906 * then they must be closed in the kernel as they
907 * will not be included in the data returned to
908 * user space. Close them now so that the header size
909 * can be safely adjusted prior to copyout. In case of
910 * an error during copyout, the remaining file
911 * descriptors will be closed in the error handler
912 * below.
913 */
914 so_closefds(msg->msg_control, msg->msg_controllen,
915 !(flags & MSG_XPG4_2),
916 control == NULL ? 0 : controllen);
917
918 /*
919 * In the case of a truncated control message, the last
920 * cmsg header that fits into the available buffer
921 * space must be adjusted to reflect the actual amount
922 * of associated data that will be returned. This only
923 * needs to be done for XPG4 messages as non-XPG4
924 * messages are not structured (they are just a
925 * buffer and a length - msg_accrights(len)).
926 */
927 if (control != NULL && (flags & MSG_XPG4_2)) {
928 so_truncatecmsg(msg->msg_control,
929 msg->msg_controllen, controllen);
930 msg->msg_controllen = controllen;
931 }
932 }
933
934 error = copyout_arg(control, controllen, controllenp,
935 msg->msg_control, msg->msg_controllen);
936
937 if (error)
938 goto err;
939
940 }
941 if (msg->msg_namelen != 0)
942 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
943 if (free_controllen != 0)
944 kmem_free(msg->msg_control, (size_t)free_controllen);
945 return (len - uiop->uio_resid);
946
947 err:
948 /*
949 * If we fail and the control part contains file descriptors
950 * we have to close them. For a truncated control message, the
951 * descriptors which were cut off have already been closed and the
952 * length adjusted so that they will not be closed again.
953 */
954 if (msg->msg_controllen != 0)
955 so_closefds(msg->msg_control, msg->msg_controllen,
956 !(flags & MSG_XPG4_2), 0);
957 if (msg->msg_namelen != 0)
958 kmem_free(msg->msg_name, (size_t)msg->msg_namelen);
959 if (free_controllen != 0)
960 kmem_free(msg->msg_control, (size_t)free_controllen);
961 return (set_errno(error));
962 }
963
964 /*
965 * Native system call
966 */
967 ssize_t
recv(int sock,void * buffer,size_t len,int flags)968 recv(int sock, void *buffer, size_t len, int flags)
969 {
970 struct nmsghdr lmsg;
971 struct uio auio;
972 struct iovec aiov[1];
973
974 dprint(1, ("recv(%d, %p, %ld, %d)\n",
975 sock, buffer, len, flags));
976
977 if ((ssize_t)len < 0) {
978 return (set_errno(EINVAL));
979 }
980
981 aiov[0].iov_base = buffer;
982 aiov[0].iov_len = len;
983 auio.uio_loffset = 0;
984 auio.uio_iov = aiov;
985 auio.uio_iovcnt = 1;
986 auio.uio_resid = len;
987 auio.uio_segflg = UIO_USERSPACE;
988 auio.uio_limit = 0;
989
990 lmsg.msg_namelen = 0;
991 lmsg.msg_controllen = 0;
992 lmsg.msg_flags = 0;
993 return (recvit(sock, &lmsg, &auio, flags, NULL, NULL, NULL));
994 }
995
996 ssize_t
recvfrom(int sock,void * buffer,size_t len,int flags,struct sockaddr * name,socklen_t * namelenp)997 recvfrom(int sock, void *buffer, size_t len, int flags, struct sockaddr *name,
998 socklen_t *namelenp)
999 {
1000 struct nmsghdr lmsg;
1001 struct uio auio;
1002 struct iovec aiov[1];
1003
1004 dprint(1, ("recvfrom(%d, %p, %ld, %d, %p, %p)\n",
1005 sock, buffer, len, flags, (void *)name, (void *)namelenp));
1006
1007 if ((ssize_t)len < 0) {
1008 return (set_errno(EINVAL));
1009 }
1010
1011 aiov[0].iov_base = buffer;
1012 aiov[0].iov_len = len;
1013 auio.uio_loffset = 0;
1014 auio.uio_iov = aiov;
1015 auio.uio_iovcnt = 1;
1016 auio.uio_resid = len;
1017 auio.uio_segflg = UIO_USERSPACE;
1018 auio.uio_limit = 0;
1019
1020 lmsg.msg_name = (char *)name;
1021 if (namelenp != NULL) {
1022 if (copyin(namelenp, &lmsg.msg_namelen,
1023 sizeof (lmsg.msg_namelen)))
1024 return (set_errno(EFAULT));
1025 } else {
1026 lmsg.msg_namelen = 0;
1027 }
1028 lmsg.msg_controllen = 0;
1029 lmsg.msg_flags = 0;
1030
1031 return (recvit(sock, &lmsg, &auio, flags, namelenp, NULL, NULL));
1032 }
1033
1034 /*
1035 * Uses the MSG_XPG4_2 flag to determine if the caller is using
1036 * struct omsghdr or struct nmsghdr.
1037 */
1038 ssize_t
recvmsg(int sock,struct nmsghdr * msg,int flags)1039 recvmsg(int sock, struct nmsghdr *msg, int flags)
1040 {
1041 STRUCT_DECL(nmsghdr, u_lmsg);
1042 STRUCT_HANDLE(nmsghdr, umsgptr);
1043 struct nmsghdr lmsg;
1044 struct uio auio;
1045 struct iovec buf[IOV_MAX_STACK], *aiov = buf;
1046 ssize_t iovsize = 0;
1047 int iovcnt;
1048 ssize_t len, rval;
1049 int i;
1050 int *flagsp;
1051 model_t model;
1052
1053 dprint(1, ("recvmsg(%d, %p, %d)\n",
1054 sock, (void *)msg, flags));
1055
1056 model = get_udatamodel();
1057 STRUCT_INIT(u_lmsg, model);
1058 STRUCT_SET_HANDLE(umsgptr, model, msg);
1059
1060 if (flags & MSG_XPG4_2) {
1061 if (copyin(msg, STRUCT_BUF(u_lmsg), STRUCT_SIZE(u_lmsg)))
1062 return (set_errno(EFAULT));
1063 flagsp = STRUCT_FADDR(umsgptr, msg_flags);
1064 } else {
1065 /*
1066 * Assumes that nmsghdr and omsghdr are identically shaped
1067 * except for the added msg_flags field.
1068 */
1069 if (copyin(msg, STRUCT_BUF(u_lmsg),
1070 SIZEOF_STRUCT(omsghdr, model)))
1071 return (set_errno(EFAULT));
1072 STRUCT_FSET(u_lmsg, msg_flags, 0);
1073 flagsp = NULL;
1074 }
1075
1076 /*
1077 * Code below us will kmem_alloc memory and hang it
1078 * off msg_control and msg_name fields. This forces
1079 * us to copy the structure to its native form.
1080 */
1081 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1082 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1083 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1084 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1085 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1086 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1087 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1088
1089 iovcnt = lmsg.msg_iovlen;
1090
1091 if (iovcnt <= 0 || iovcnt > IOV_MAX) {
1092 return (set_errno(EMSGSIZE));
1093 }
1094
1095 if (iovcnt > IOV_MAX_STACK) {
1096 iovsize = iovcnt * sizeof (struct iovec);
1097 aiov = kmem_alloc(iovsize, KM_SLEEP);
1098 }
1099
1100 #ifdef _SYSCALL32_IMPL
1101 /*
1102 * 32-bit callers need to have their iovec expanded, while ensuring
1103 * that they can't move more than 2Gbytes of data in a single call.
1104 */
1105 if (model == DATAMODEL_ILP32) {
1106 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32;
1107 ssize_t iov32size;
1108 ssize32_t count32;
1109
1110 iov32size = iovcnt * sizeof (struct iovec32);
1111 if (iovsize != 0)
1112 aiov32 = kmem_alloc(iov32size, KM_SLEEP);
1113
1114 if (copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) {
1115 if (iovsize != 0) {
1116 kmem_free(aiov32, iov32size);
1117 kmem_free(aiov, iovsize);
1118 }
1119
1120 return (set_errno(EFAULT));
1121 }
1122
1123 count32 = 0;
1124 for (i = 0; i < iovcnt; i++) {
1125 ssize32_t iovlen32;
1126
1127 iovlen32 = aiov32[i].iov_len;
1128 count32 += iovlen32;
1129 if (iovlen32 < 0 || count32 < 0) {
1130 if (iovsize != 0) {
1131 kmem_free(aiov32, iov32size);
1132 kmem_free(aiov, iovsize);
1133 }
1134
1135 return (set_errno(EINVAL));
1136 }
1137
1138 aiov[i].iov_len = iovlen32;
1139 aiov[i].iov_base =
1140 (caddr_t)(uintptr_t)aiov32[i].iov_base;
1141 }
1142
1143 if (iovsize != 0)
1144 kmem_free(aiov32, iov32size);
1145 } else
1146 #endif /* _SYSCALL32_IMPL */
1147 if (copyin(lmsg.msg_iov, aiov, iovcnt * sizeof (struct iovec))) {
1148 if (iovsize != 0)
1149 kmem_free(aiov, iovsize);
1150
1151 return (set_errno(EFAULT));
1152 }
1153 len = 0;
1154 for (i = 0; i < iovcnt; i++) {
1155 ssize_t iovlen = aiov[i].iov_len;
1156 len += iovlen;
1157 if (iovlen < 0 || len < 0) {
1158 if (iovsize != 0)
1159 kmem_free(aiov, iovsize);
1160
1161 return (set_errno(EINVAL));
1162 }
1163 }
1164 auio.uio_loffset = 0;
1165 auio.uio_iov = aiov;
1166 auio.uio_iovcnt = iovcnt;
1167 auio.uio_resid = len;
1168 auio.uio_segflg = UIO_USERSPACE;
1169 auio.uio_limit = 0;
1170
1171 if (lmsg.msg_control != NULL &&
1172 (do_useracc == 0 ||
1173 useracc(lmsg.msg_control, lmsg.msg_controllen,
1174 B_WRITE) != 0)) {
1175 if (iovsize != 0)
1176 kmem_free(aiov, iovsize);
1177
1178 return (set_errno(EFAULT));
1179 }
1180
1181 rval = recvit(sock, &lmsg, &auio, flags,
1182 STRUCT_FADDR(umsgptr, msg_namelen),
1183 STRUCT_FADDR(umsgptr, msg_controllen), flagsp);
1184
1185 if (iovsize != 0)
1186 kmem_free(aiov, iovsize);
1187
1188 return (rval);
1189 }
1190
1191 /*
1192 * Common send function.
1193 */
1194 static ssize_t
sendit(int sock,struct nmsghdr * msg,struct uio * uiop,int flags)1195 sendit(int sock, struct nmsghdr *msg, struct uio *uiop, int flags)
1196 {
1197 struct sonode *so;
1198 file_t *fp;
1199 void *name;
1200 socklen_t namelen;
1201 void *control;
1202 socklen_t controllen;
1203 ssize_t len;
1204 int error;
1205
1206 if ((so = getsonode(sock, &error, &fp)) == NULL)
1207 return (set_errno(error));
1208
1209 uiop->uio_fmode = fp->f_flag;
1210
1211 if (so->so_family == AF_UNIX)
1212 uiop->uio_extflg = UIO_COPY_CACHED;
1213 else
1214 uiop->uio_extflg = UIO_COPY_DEFAULT;
1215
1216 len = uiop->uio_resid;
1217
1218 /* Allocate and copyin name and control */
1219 name = msg->msg_name;
1220 namelen = msg->msg_namelen;
1221 if (name != NULL && namelen != 0) {
1222 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1223 name = copyin_name(so,
1224 (struct sockaddr *)name,
1225 &namelen, &error);
1226 if (name == NULL)
1227 goto done3;
1228 /* copyin_name null terminates addresses for AF_UNIX */
1229 msg->msg_namelen = namelen;
1230 msg->msg_name = name;
1231 } else {
1232 msg->msg_name = name = NULL;
1233 msg->msg_namelen = namelen = 0;
1234 }
1235
1236 control = msg->msg_control;
1237 controllen = msg->msg_controllen;
1238 if ((control != NULL) && (controllen != 0)) {
1239 /*
1240 * Verify that the length is not excessive to prevent
1241 * an application from consuming all of kernel memory.
1242 */
1243 if (controllen > SO_MAXARGSIZE) {
1244 error = EINVAL;
1245 goto done2;
1246 }
1247 control = kmem_alloc(controllen, KM_SLEEP);
1248
1249 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1250 if (copyin(msg->msg_control, control, controllen)) {
1251 error = EFAULT;
1252 goto done1;
1253 }
1254 msg->msg_control = control;
1255 } else {
1256 msg->msg_control = control = NULL;
1257 msg->msg_controllen = controllen = 0;
1258 }
1259
1260 msg->msg_flags = flags;
1261
1262 error = socket_sendmsg(so, msg, uiop, CRED());
1263 done1:
1264 if (control != NULL)
1265 kmem_free(control, controllen);
1266 done2:
1267 if (name != NULL)
1268 kmem_free(name, namelen);
1269 done3:
1270 if (error != 0) {
1271 releasef(sock);
1272 return (set_errno(error));
1273 }
1274 lwp_stat_update(LWP_STAT_MSGSND, 1);
1275 releasef(sock);
1276 return (len - uiop->uio_resid);
1277 }
1278
1279 /*
1280 * Native system call
1281 */
1282 ssize_t
send(int sock,void * buffer,size_t len,int flags)1283 send(int sock, void *buffer, size_t len, int flags)
1284 {
1285 struct nmsghdr lmsg;
1286 struct uio auio;
1287 struct iovec aiov[1];
1288
1289 dprint(1, ("send(%d, %p, %ld, %d)\n",
1290 sock, buffer, len, flags));
1291
1292 if ((ssize_t)len < 0) {
1293 return (set_errno(EINVAL));
1294 }
1295
1296 aiov[0].iov_base = buffer;
1297 aiov[0].iov_len = len;
1298 auio.uio_loffset = 0;
1299 auio.uio_iov = aiov;
1300 auio.uio_iovcnt = 1;
1301 auio.uio_resid = len;
1302 auio.uio_segflg = UIO_USERSPACE;
1303 auio.uio_limit = 0;
1304
1305 lmsg.msg_name = NULL;
1306 lmsg.msg_control = NULL;
1307 if (!(flags & MSG_XPG4_2)) {
1308 /*
1309 * In order to be compatible with the libsocket/sockmod
1310 * implementation we set EOR for all send* calls.
1311 */
1312 flags |= MSG_EOR;
1313 }
1314 return (sendit(sock, &lmsg, &auio, flags));
1315 }
1316
1317 /*
1318 * Uses the MSG_XPG4_2 flag to determine if the caller is using
1319 * struct omsghdr or struct nmsghdr.
1320 */
1321 ssize_t
sendmsg(int sock,struct nmsghdr * msg,int flags)1322 sendmsg(int sock, struct nmsghdr *msg, int flags)
1323 {
1324 struct nmsghdr lmsg;
1325 STRUCT_DECL(nmsghdr, u_lmsg);
1326 struct uio auio;
1327 struct iovec buf[IOV_MAX_STACK], *aiov = buf;
1328 ssize_t iovsize = 0;
1329 int iovcnt;
1330 ssize_t len, rval;
1331 int i;
1332 model_t model;
1333
1334 dprint(1, ("sendmsg(%d, %p, %d)\n", sock, (void *)msg, flags));
1335
1336 model = get_udatamodel();
1337 STRUCT_INIT(u_lmsg, model);
1338
1339 if (flags & MSG_XPG4_2) {
1340 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1341 STRUCT_SIZE(u_lmsg)))
1342 return (set_errno(EFAULT));
1343 } else {
1344 /*
1345 * Assumes that nmsghdr and omsghdr are identically shaped
1346 * except for the added msg_flags field.
1347 */
1348 if (copyin(msg, (char *)STRUCT_BUF(u_lmsg),
1349 SIZEOF_STRUCT(omsghdr, model)))
1350 return (set_errno(EFAULT));
1351 /*
1352 * In order to be compatible with the libsocket/sockmod
1353 * implementation we set EOR for all send* calls.
1354 */
1355 flags |= MSG_EOR;
1356 }
1357
1358 /*
1359 * Code below us will kmem_alloc memory and hang it
1360 * off msg_control and msg_name fields. This forces
1361 * us to copy the structure to its native form.
1362 */
1363 lmsg.msg_name = STRUCT_FGETP(u_lmsg, msg_name);
1364 lmsg.msg_namelen = STRUCT_FGET(u_lmsg, msg_namelen);
1365 lmsg.msg_iov = STRUCT_FGETP(u_lmsg, msg_iov);
1366 lmsg.msg_iovlen = STRUCT_FGET(u_lmsg, msg_iovlen);
1367 lmsg.msg_control = STRUCT_FGETP(u_lmsg, msg_control);
1368 lmsg.msg_controllen = STRUCT_FGET(u_lmsg, msg_controllen);
1369 lmsg.msg_flags = STRUCT_FGET(u_lmsg, msg_flags);
1370
1371 iovcnt = lmsg.msg_iovlen;
1372
1373 if (iovcnt <= 0 || iovcnt > IOV_MAX) {
1374 /*
1375 * Unless this is XPG 4.2 we allow iovcnt == 0 to
1376 * be compatible with SunOS 4.X and 4.4BSD.
1377 */
1378 if (iovcnt != 0 || (flags & MSG_XPG4_2))
1379 return (set_errno(EMSGSIZE));
1380 }
1381
1382 if (iovcnt > IOV_MAX_STACK) {
1383 iovsize = iovcnt * sizeof (struct iovec);
1384 aiov = kmem_alloc(iovsize, KM_SLEEP);
1385 }
1386
1387 #ifdef _SYSCALL32_IMPL
1388 /*
1389 * 32-bit callers need to have their iovec expanded, while ensuring
1390 * that they can't move more than 2Gbytes of data in a single call.
1391 */
1392 if (model == DATAMODEL_ILP32) {
1393 struct iovec32 buf32[IOV_MAX_STACK], *aiov32 = buf32;
1394 ssize_t iov32size;
1395 ssize32_t count32;
1396
1397 iov32size = iovcnt * sizeof (struct iovec32);
1398 if (iovsize != 0)
1399 aiov32 = kmem_alloc(iov32size, KM_SLEEP);
1400
1401 if (iovcnt != 0 &&
1402 copyin((struct iovec32 *)lmsg.msg_iov, aiov32, iov32size)) {
1403 if (iovsize != 0) {
1404 kmem_free(aiov32, iov32size);
1405 kmem_free(aiov, iovsize);
1406 }
1407
1408 return (set_errno(EFAULT));
1409 }
1410
1411 count32 = 0;
1412 for (i = 0; i < iovcnt; i++) {
1413 ssize32_t iovlen32;
1414
1415 iovlen32 = aiov32[i].iov_len;
1416 count32 += iovlen32;
1417 if (iovlen32 < 0 || count32 < 0) {
1418 if (iovsize != 0) {
1419 kmem_free(aiov32, iov32size);
1420 kmem_free(aiov, iovsize);
1421 }
1422
1423 return (set_errno(EINVAL));
1424 }
1425
1426 aiov[i].iov_len = iovlen32;
1427 aiov[i].iov_base =
1428 (caddr_t)(uintptr_t)aiov32[i].iov_base;
1429 }
1430
1431 if (iovsize != 0)
1432 kmem_free(aiov32, iov32size);
1433 } else
1434 #endif /* _SYSCALL32_IMPL */
1435 if (iovcnt != 0 &&
1436 copyin(lmsg.msg_iov, aiov,
1437 (unsigned)iovcnt * sizeof (struct iovec))) {
1438 if (iovsize != 0)
1439 kmem_free(aiov, iovsize);
1440
1441 return (set_errno(EFAULT));
1442 }
1443 len = 0;
1444 for (i = 0; i < iovcnt; i++) {
1445 ssize_t iovlen = aiov[i].iov_len;
1446 len += iovlen;
1447 if (iovlen < 0 || len < 0) {
1448 if (iovsize != 0)
1449 kmem_free(aiov, iovsize);
1450
1451 return (set_errno(EINVAL));
1452 }
1453 }
1454 auio.uio_loffset = 0;
1455 auio.uio_iov = aiov;
1456 auio.uio_iovcnt = iovcnt;
1457 auio.uio_resid = len;
1458 auio.uio_segflg = UIO_USERSPACE;
1459 auio.uio_limit = 0;
1460
1461 rval = sendit(sock, &lmsg, &auio, flags);
1462
1463 if (iovsize != 0)
1464 kmem_free(aiov, iovsize);
1465
1466 return (rval);
1467 }
1468
1469 ssize_t
sendto(int sock,void * buffer,size_t len,int flags,struct sockaddr * name,socklen_t namelen)1470 sendto(int sock, void *buffer, size_t len, int flags,
1471 struct sockaddr *name, socklen_t namelen)
1472 {
1473 struct nmsghdr lmsg;
1474 struct uio auio;
1475 struct iovec aiov[1];
1476
1477 dprint(1, ("sendto(%d, %p, %ld, %d, %p, %d)\n",
1478 sock, buffer, len, flags, (void *)name, namelen));
1479
1480 if ((ssize_t)len < 0) {
1481 return (set_errno(EINVAL));
1482 }
1483
1484 aiov[0].iov_base = buffer;
1485 aiov[0].iov_len = len;
1486 auio.uio_loffset = 0;
1487 auio.uio_iov = aiov;
1488 auio.uio_iovcnt = 1;
1489 auio.uio_resid = len;
1490 auio.uio_segflg = UIO_USERSPACE;
1491 auio.uio_limit = 0;
1492
1493 lmsg.msg_name = (char *)name;
1494 lmsg.msg_namelen = namelen;
1495 lmsg.msg_control = NULL;
1496 if (!(flags & MSG_XPG4_2)) {
1497 /*
1498 * In order to be compatible with the libsocket/sockmod
1499 * implementation we set EOR for all send* calls.
1500 */
1501 flags |= MSG_EOR;
1502 }
1503 return (sendit(sock, &lmsg, &auio, flags));
1504 }
1505
1506 /*ARGSUSED3*/
1507 int
getpeername(int sock,struct sockaddr * name,socklen_t * namelenp,int version)1508 getpeername(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1509 {
1510 struct sonode *so;
1511 int error;
1512 socklen_t namelen;
1513 socklen_t sock_addrlen;
1514 struct sockaddr *sock_addrp;
1515
1516 dprint(1, ("getpeername(%d, %p, %p)\n",
1517 sock, (void *)name, (void *)namelenp));
1518
1519 if ((so = getsonode(sock, &error, NULL)) == NULL)
1520 goto bad;
1521
1522 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1523 if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1524 (name == NULL && namelen != 0)) {
1525 error = EFAULT;
1526 goto rel_out;
1527 }
1528 sock_addrlen = so->so_max_addr_len;
1529 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1530
1531 if ((error = socket_getpeername(so, sock_addrp, &sock_addrlen,
1532 B_FALSE, CRED())) == 0) {
1533 ASSERT(sock_addrlen <= so->so_max_addr_len);
1534 error = copyout_name(name, namelen, namelenp,
1535 (void *)sock_addrp, sock_addrlen);
1536 }
1537 kmem_free(sock_addrp, so->so_max_addr_len);
1538 rel_out:
1539 releasef(sock);
1540 bad: return (error != 0 ? set_errno(error) : 0);
1541 }
1542
1543 /*ARGSUSED3*/
1544 int
getsockname(int sock,struct sockaddr * name,socklen_t * namelenp,int version)1545 getsockname(int sock, struct sockaddr *name, socklen_t *namelenp, int version)
1546 {
1547 struct sonode *so;
1548 int error;
1549 socklen_t namelen, sock_addrlen;
1550 struct sockaddr *sock_addrp;
1551
1552 dprint(1, ("getsockname(%d, %p, %p)\n",
1553 sock, (void *)name, (void *)namelenp));
1554
1555 if ((so = getsonode(sock, &error, NULL)) == NULL)
1556 goto bad;
1557
1558 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1559 if (copyin(namelenp, &namelen, sizeof (namelen)) ||
1560 (name == NULL && namelen != 0)) {
1561 error = EFAULT;
1562 goto rel_out;
1563 }
1564
1565 sock_addrlen = so->so_max_addr_len;
1566 sock_addrp = (struct sockaddr *)kmem_alloc(sock_addrlen, KM_SLEEP);
1567 if ((error = socket_getsockname(so, sock_addrp, &sock_addrlen,
1568 CRED())) == 0) {
1569 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1570 ASSERT(sock_addrlen <= so->so_max_addr_len);
1571 error = copyout_name(name, namelen, namelenp,
1572 (void *)sock_addrp, sock_addrlen);
1573 }
1574 kmem_free(sock_addrp, so->so_max_addr_len);
1575 rel_out:
1576 releasef(sock);
1577 bad: return (error != 0 ? set_errno(error) : 0);
1578 }
1579
1580 /*ARGSUSED5*/
1581 int
getsockopt(int sock,int level,int option_name,void * option_value,socklen_t * option_lenp,int version)1582 getsockopt(int sock, int level, int option_name, void *option_value,
1583 socklen_t *option_lenp, int version)
1584 {
1585 struct sonode *so;
1586 socklen_t optlen, optlen_res;
1587 void *optval;
1588 int error;
1589
1590 dprint(1, ("getsockopt(%d, %d, %d, %p, %p)\n",
1591 sock, level, option_name, option_value, (void *)option_lenp));
1592
1593 if ((so = getsonode(sock, &error, NULL)) == NULL)
1594 return (set_errno(error));
1595
1596 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1597 if (copyin(option_lenp, &optlen, sizeof (optlen))) {
1598 releasef(sock);
1599 return (set_errno(EFAULT));
1600 }
1601 /*
1602 * Verify that the length is not excessive to prevent
1603 * an application from consuming all of kernel memory.
1604 */
1605 if (optlen > SO_MAXARGSIZE) {
1606 error = EINVAL;
1607 releasef(sock);
1608 return (set_errno(error));
1609 }
1610 optval = kmem_alloc(optlen, KM_SLEEP);
1611 optlen_res = optlen;
1612 error = socket_getsockopt(so, level, option_name, optval,
1613 &optlen_res, (version != SOV_XPG4_2) ? 0 : _SOGETSOCKOPT_XPG4_2,
1614 CRED());
1615 releasef(sock);
1616 if (error) {
1617 kmem_free(optval, optlen);
1618 return (set_errno(error));
1619 }
1620 error = copyout_arg(option_value, optlen, option_lenp,
1621 optval, optlen_res);
1622 kmem_free(optval, optlen);
1623 if (error)
1624 return (set_errno(error));
1625 return (0);
1626 }
1627
1628 /*ARGSUSED5*/
1629 int
setsockopt(int sock,int level,int option_name,void * option_value,socklen_t option_len,int version)1630 setsockopt(int sock, int level, int option_name, void *option_value,
1631 socklen_t option_len, int version)
1632 {
1633 struct sonode *so;
1634 intptr_t buffer[2];
1635 void *optval = NULL;
1636 int error;
1637
1638 dprint(1, ("setsockopt(%d, %d, %d, %p, %d)\n",
1639 sock, level, option_name, option_value, option_len));
1640
1641 if ((so = getsonode(sock, &error, NULL)) == NULL)
1642 return (set_errno(error));
1643
1644 if (option_value != NULL) {
1645 if (option_len != 0) {
1646 /*
1647 * Verify that the length is not excessive to prevent
1648 * an application from consuming all of kernel memory.
1649 */
1650 if (option_len > SO_MAXARGSIZE) {
1651 error = EINVAL;
1652 goto done2;
1653 }
1654 optval = option_len <= sizeof (buffer) ?
1655 &buffer : kmem_alloc((size_t)option_len, KM_SLEEP);
1656 ASSERT(MUTEX_NOT_HELD(&so->so_lock));
1657 if (copyin(option_value, optval, (size_t)option_len)) {
1658 error = EFAULT;
1659 goto done1;
1660 }
1661 }
1662 } else
1663 option_len = 0;
1664
1665 error = socket_setsockopt(so, level, option_name, optval,
1666 (t_uscalar_t)option_len, CRED());
1667 done1:
1668 if (optval != buffer)
1669 kmem_free(optval, (size_t)option_len);
1670 done2:
1671 releasef(sock);
1672 if (error)
1673 return (set_errno(error));
1674 return (0);
1675 }
1676
1677 static int
sockconf_add_sock(int family,int type,int protocol,char * name)1678 sockconf_add_sock(int family, int type, int protocol, char *name)
1679 {
1680 int error = 0;
1681 char *kdevpath = NULL;
1682 char *kmodule = NULL;
1683 char *buf = NULL;
1684 size_t pathlen = 0;
1685 struct sockparams *sp;
1686
1687 if (name == NULL)
1688 return (EINVAL);
1689 /*
1690 * Copyin the name.
1691 * This also makes it possible to check for too long pathnames.
1692 * Compress the space needed for the name before passing it
1693 * to soconfig - soconfig will store the string until
1694 * the configuration is removed.
1695 */
1696 buf = kmem_alloc(MAXPATHLEN, KM_SLEEP);
1697 if ((error = copyinstr(name, buf, MAXPATHLEN, &pathlen)) != 0) {
1698 kmem_free(buf, MAXPATHLEN);
1699 return (error);
1700 }
1701 if (strncmp(buf, "/dev", strlen("/dev")) == 0) {
1702 /* For device */
1703 kdevpath = kmem_alloc(pathlen, KM_SLEEP);
1704 bcopy(buf, kdevpath, pathlen);
1705 kdevpath[pathlen - 1] = '\0';
1706 } else {
1707 /* For socket module */
1708 kmodule = kmem_alloc(pathlen, KM_SLEEP);
1709 bcopy(buf, kmodule, pathlen);
1710 kmodule[pathlen - 1] = '\0';
1711 pathlen = 0;
1712 }
1713 kmem_free(buf, MAXPATHLEN);
1714
1715 /* sockparams_create frees mod name and devpath upon failure */
1716 sp = sockparams_create(family, type, protocol, kmodule,
1717 kdevpath, pathlen, 0, KM_SLEEP, &error);
1718 if (sp != NULL) {
1719 error = sockparams_add(sp);
1720 if (error != 0)
1721 sockparams_destroy(sp);
1722 }
1723
1724 return (error);
1725 }
1726
1727 static int
sockconf_remove_sock(int family,int type,int protocol)1728 sockconf_remove_sock(int family, int type, int protocol)
1729 {
1730 return (sockparams_delete(family, type, protocol));
1731 }
1732
1733 static int
sockconfig_remove_filter(const char * uname)1734 sockconfig_remove_filter(const char *uname)
1735 {
1736 char kname[SOF_MAXNAMELEN];
1737 size_t len;
1738 int error;
1739 sof_entry_t *ent;
1740
1741 if ((error = copyinstr(uname, kname, SOF_MAXNAMELEN, &len)) != 0)
1742 return (error);
1743
1744 ent = sof_entry_remove_by_name(kname);
1745 if (ent == NULL)
1746 return (ENXIO);
1747
1748 mutex_enter(&ent->sofe_lock);
1749 ASSERT(!(ent->sofe_flags & SOFEF_CONDEMED));
1750 if (ent->sofe_refcnt == 0) {
1751 mutex_exit(&ent->sofe_lock);
1752 sof_entry_free(ent);
1753 } else {
1754 /* let the last socket free the filter */
1755 ent->sofe_flags |= SOFEF_CONDEMED;
1756 mutex_exit(&ent->sofe_lock);
1757 }
1758
1759 return (0);
1760 }
1761
1762 static int
sockconfig_add_filter(const char * uname,void * ufilpropp)1763 sockconfig_add_filter(const char *uname, void *ufilpropp)
1764 {
1765 struct sockconfig_filter_props filprop;
1766 sof_entry_t *ent;
1767 int error;
1768 size_t tuplesz, len;
1769 char hintbuf[SOF_MAXNAMELEN];
1770
1771 ent = kmem_zalloc(sizeof (sof_entry_t), KM_SLEEP);
1772 mutex_init(&ent->sofe_lock, NULL, MUTEX_DEFAULT, NULL);
1773
1774 if ((error = copyinstr(uname, ent->sofe_name, SOF_MAXNAMELEN,
1775 &len)) != 0) {
1776 sof_entry_free(ent);
1777 return (error);
1778 }
1779
1780 if (get_udatamodel() == DATAMODEL_NATIVE) {
1781 if (copyin(ufilpropp, &filprop, sizeof (filprop)) != 0) {
1782 sof_entry_free(ent);
1783 return (EFAULT);
1784 }
1785 }
1786 #ifdef _SYSCALL32_IMPL
1787 else {
1788 struct sockconfig_filter_props32 filprop32;
1789
1790 if (copyin(ufilpropp, &filprop32, sizeof (filprop32)) != 0) {
1791 sof_entry_free(ent);
1792 return (EFAULT);
1793 }
1794 filprop.sfp_modname = (char *)(uintptr_t)filprop32.sfp_modname;
1795 filprop.sfp_autoattach = filprop32.sfp_autoattach;
1796 filprop.sfp_hint = filprop32.sfp_hint;
1797 filprop.sfp_hintarg = (char *)(uintptr_t)filprop32.sfp_hintarg;
1798 filprop.sfp_socktuple_cnt = filprop32.sfp_socktuple_cnt;
1799 filprop.sfp_socktuple =
1800 (sof_socktuple_t *)(uintptr_t)filprop32.sfp_socktuple;
1801 }
1802 #endif /* _SYSCALL32_IMPL */
1803
1804 if ((error = copyinstr(filprop.sfp_modname, ent->sofe_modname,
1805 sizeof (ent->sofe_modname), &len)) != 0) {
1806 sof_entry_free(ent);
1807 return (error);
1808 }
1809
1810 /*
1811 * A filter must specify at least one socket tuple.
1812 */
1813 if (filprop.sfp_socktuple_cnt == 0 ||
1814 filprop.sfp_socktuple_cnt > SOF_MAXSOCKTUPLECNT) {
1815 sof_entry_free(ent);
1816 return (EINVAL);
1817 }
1818 ent->sofe_flags = filprop.sfp_autoattach ? SOFEF_AUTO : SOFEF_PROG;
1819 ent->sofe_hint = filprop.sfp_hint;
1820
1821 /*
1822 * Verify the hint, and copy in the hint argument, if necessary.
1823 */
1824 switch (ent->sofe_hint) {
1825 case SOF_HINT_BEFORE:
1826 case SOF_HINT_AFTER:
1827 if ((error = copyinstr(filprop.sfp_hintarg, hintbuf,
1828 sizeof (hintbuf), &len)) != 0) {
1829 sof_entry_free(ent);
1830 return (error);
1831 }
1832 ent->sofe_hintarg = kmem_alloc(len, KM_SLEEP);
1833 bcopy(hintbuf, ent->sofe_hintarg, len);
1834 /* FALLTHRU */
1835 case SOF_HINT_TOP:
1836 case SOF_HINT_BOTTOM:
1837 /* hints cannot be used with programmatic filters */
1838 if (ent->sofe_flags & SOFEF_PROG) {
1839 sof_entry_free(ent);
1840 return (EINVAL);
1841 }
1842 break;
1843 case SOF_HINT_NONE:
1844 break;
1845 default:
1846 /* bad hint value */
1847 sof_entry_free(ent);
1848 return (EINVAL);
1849 }
1850
1851 ent->sofe_socktuple_cnt = filprop.sfp_socktuple_cnt;
1852 tuplesz = sizeof (sof_socktuple_t) * ent->sofe_socktuple_cnt;
1853 ent->sofe_socktuple = kmem_alloc(tuplesz, KM_SLEEP);
1854
1855 if (get_udatamodel() == DATAMODEL_NATIVE) {
1856 if (copyin(filprop.sfp_socktuple, ent->sofe_socktuple,
1857 tuplesz)) {
1858 sof_entry_free(ent);
1859 return (EFAULT);
1860 }
1861 }
1862 #ifdef _SYSCALL32_IMPL
1863 else {
1864 int i;
1865 caddr_t data = (caddr_t)filprop.sfp_socktuple;
1866 sof_socktuple_t *tup = ent->sofe_socktuple;
1867 sof_socktuple32_t tup32;
1868
1869 tup = ent->sofe_socktuple;
1870 for (i = 0; i < ent->sofe_socktuple_cnt; i++, tup++) {
1871 ASSERT(tup < ent->sofe_socktuple + tuplesz);
1872
1873 if (copyin(data, &tup32, sizeof (tup32)) != 0) {
1874 sof_entry_free(ent);
1875 return (EFAULT);
1876 }
1877 tup->sofst_family = tup32.sofst_family;
1878 tup->sofst_type = tup32.sofst_type;
1879 tup->sofst_protocol = tup32.sofst_protocol;
1880
1881 data += sizeof (tup32);
1882 }
1883 }
1884 #endif /* _SYSCALL32_IMPL */
1885
1886 /* Sockets can start using the filter as soon as the filter is added */
1887 if ((error = sof_entry_add(ent)) != 0)
1888 sof_entry_free(ent);
1889
1890 return (error);
1891 }
1892
1893 /*
1894 * Socket configuration system call. It is used to add and remove
1895 * socket types.
1896 */
1897 int
sockconfig(int cmd,void * arg1,void * arg2,void * arg3,void * arg4)1898 sockconfig(int cmd, void *arg1, void *arg2, void *arg3, void *arg4)
1899 {
1900 int error = 0;
1901
1902 if (secpolicy_net_config(CRED(), B_FALSE) != 0)
1903 return (set_errno(EPERM));
1904
1905 switch (cmd) {
1906 case SOCKCONFIG_ADD_SOCK:
1907 error = sockconf_add_sock((int)(uintptr_t)arg1,
1908 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3, arg4);
1909 break;
1910 case SOCKCONFIG_REMOVE_SOCK:
1911 error = sockconf_remove_sock((int)(uintptr_t)arg1,
1912 (int)(uintptr_t)arg2, (int)(uintptr_t)arg3);
1913 break;
1914 case SOCKCONFIG_ADD_FILTER:
1915 error = sockconfig_add_filter((const char *)arg1, arg2);
1916 break;
1917 case SOCKCONFIG_REMOVE_FILTER:
1918 error = sockconfig_remove_filter((const char *)arg1);
1919 break;
1920 case SOCKCONFIG_GET_SOCKTABLE:
1921 error = sockparams_copyout_socktable((int)(uintptr_t)arg1);
1922 break;
1923 default:
1924 #ifdef DEBUG
1925 cmn_err(CE_NOTE, "sockconfig: unkonwn subcommand %d", cmd);
1926 #endif
1927 error = EINVAL;
1928 break;
1929 }
1930
1931 if (error != 0) {
1932 eprintline(error);
1933 return (set_errno(error));
1934 }
1935 return (0);
1936 }
1937
1938
1939 /*
1940 * Sendfile is implemented through two schemes, direct I/O or by
1941 * caching in the filesystem page cache. We cache the input file by
1942 * default and use direct I/O only if sendfile_max_size is set
1943 * appropriately as explained below. Note that this logic is consistent
1944 * with other filesystems where caching is turned on by default
1945 * unless explicitly turned off by using the DIRECTIO ioctl.
1946 *
1947 * We choose a slightly different scheme here. One can turn off
1948 * caching by setting sendfile_max_size to 0. One can also enable
1949 * caching of files <= sendfile_max_size by setting sendfile_max_size
1950 * to an appropriate value. By default sendfile_max_size is set to the
1951 * maximum value so that all files are cached. In future, we may provide
1952 * better interfaces for caching the file.
1953 *
1954 * Sendfile through Direct I/O (Zero copy)
1955 * --------------------------------------
1956 *
1957 * As disks are normally slower than the network, we can't have a
1958 * single thread that reads the disk and writes to the network. We
1959 * need to have parallelism. This is done by having the sendfile
1960 * thread create another thread that reads from the filesystem
1961 * and queues it for network processing. In this scheme, the data
1962 * is never copied anywhere i.e it is zero copy unlike the other
1963 * scheme.
1964 *
1965 * We have a sendfile queue (snfq) where each sendfile
1966 * request (snf_req_t) is queued for processing by a thread. Number
1967 * of threads is dynamically allocated and they exit if they are idling
1968 * beyond a specified amount of time. When each request (snf_req_t) is
1969 * processed by a thread, it produces a number of mblk_t structures to
1970 * be consumed by the sendfile thread. snf_deque and snf_enque are
1971 * used for consuming and producing mblks. Size of the filesystem
1972 * read is determined by the tunable (sendfile_read_size). A single
1973 * mblk holds sendfile_read_size worth of data (except the last
1974 * read of the file) which is sent down as a whole to the network.
1975 * sendfile_read_size is set to 1 MB as this seems to be the optimal
1976 * value for the UFS filesystem backed by a striped storage array.
1977 *
1978 * Synchronisation between read (producer) and write (consumer) threads.
1979 * --------------------------------------------------------------------
1980 *
1981 * sr_lock protects sr_ib_head and sr_ib_tail. The lock is held while
1982 * adding and deleting items in this list. Error can happen anytime
1983 * during read or write. There could be unprocessed mblks in the
1984 * sr_ib_XXX list when a read or write error occurs. Whenever error
1985 * is encountered, we need two things to happen :
1986 *
1987 * a) One of the threads need to clean the mblks.
1988 * b) When one thread encounters an error, the other should stop.
1989 *
1990 * For (a), we don't want to penalize the reader thread as it could do
1991 * some useful work processing other requests. For (b), the error can
1992 * be detected by examining sr_read_error or sr_write_error.
1993 * sr_lock protects sr_read_error and sr_write_error. If both reader and
1994 * writer encounters error, we need to report the write error back to
1995 * the application as that's what would have happened if the operations
1996 * were done sequentially. With this in mind, following should work :
1997 *
1998 * - Check for errors before read or write.
1999 * - If the reader encounters error, set the error in sr_read_error.
2000 * Check sr_write_error, if it is set, send cv_signal as it is
2001 * waiting for reader to complete. If it is not set, the writer
2002 * is either running sinking data to the network or blocked
2003 * because of flow control. For handling the latter case, we
2004 * always send a signal. In any case, it will examine sr_read_error
2005 * and return. sr_read_error is marked with SR_READ_DONE to tell
2006 * the writer that the reader is done in all the cases.
2007 * - If the writer encounters error, set the error in sr_write_error.
2008 * The reader thread is either blocked because of flow control or
2009 * running reading data from the disk. For the former, we need to
2010 * wakeup the thread. Again to keep it simple, we always wake up
2011 * the reader thread. Then, wait for the read thread to complete
2012 * if it is not done yet. Cleanup and return.
2013 *
2014 * High and low water marks for the read thread.
2015 * --------------------------------------------
2016 *
2017 * If sendfile() is used to send data over a slow network, we need to
2018 * make sure that the read thread does not produce data at a faster
2019 * rate than the network. This can happen if the disk is faster than
2020 * the network. In such a case, we don't want to build a very large queue.
2021 * But we would still like to get all of the network throughput possible.
2022 * This implies that network should never block waiting for data.
2023 * As there are lot of disk throughput/network throughput combinations
2024 * possible, it is difficult to come up with an accurate number.
2025 * A typical 10K RPM disk has a max seek latency 17ms and rotational
2026 * latency of 3ms for reading a disk block. Thus, the total latency to
2027 * initiate a new read, transfer data from the disk and queue for
2028 * transmission would take about a max of 25ms. Todays max transfer rate
2029 * for network is 100MB/sec. If the thread is blocked because of flow
2030 * control, it would take 25ms to get new data ready for transmission.
2031 * We have to make sure that network is not idling, while we are initiating
2032 * new transfers. So, at 100MB/sec, to keep network busy we would need
2033 * 2.5MB of data. Rounding off, we keep the low water mark to be 3MB of data.
2034 * We need to pick a high water mark so that the woken up thread would
2035 * do considerable work before blocking again to prevent thrashing. Currently,
2036 * we pick this to be 10 times that of the low water mark.
2037 *
2038 * Sendfile with segmap caching (One copy from page cache to mblks).
2039 * ----------------------------------------------------------------
2040 *
2041 * We use the segmap cache for caching the file, if the size of file
2042 * is <= sendfile_max_size. In this case we don't use threads as VM
2043 * is reasonably fast enough to keep up with the network. If the underlying
2044 * transport allows, we call segmap_getmapflt() to map MAXBSIZE (8K) worth
2045 * of data into segmap space, and use the virtual address from segmap
2046 * directly through desballoc() to avoid copy. Once the transport is done
2047 * with the data, the mapping will be released through segmap_release()
2048 * called by the call-back routine.
2049 *
2050 * If zero-copy is not allowed by the transport, we simply call VOP_READ()
2051 * to copy the data from the filesystem into our temporary network buffer.
2052 *
2053 * To disable caching, set sendfile_max_size to 0.
2054 */
2055
2056 uint_t sendfile_read_size = 1024 * 1024;
2057 #define SENDFILE_REQ_LOWAT 3 * 1024 * 1024
2058 uint_t sendfile_req_lowat = SENDFILE_REQ_LOWAT;
2059 uint_t sendfile_req_hiwat = 10 * SENDFILE_REQ_LOWAT;
2060 struct sendfile_stats sf_stats;
2061 struct sendfile_queue *snfq;
2062 clock_t snfq_timeout;
2063 off64_t sendfile_max_size;
2064
2065 static void snf_enque(snf_req_t *, mblk_t *);
2066 static mblk_t *snf_deque(snf_req_t *);
2067
2068 void
sendfile_init(void)2069 sendfile_init(void)
2070 {
2071 snfq = kmem_zalloc(sizeof (struct sendfile_queue), KM_SLEEP);
2072
2073 mutex_init(&snfq->snfq_lock, NULL, MUTEX_DEFAULT, NULL);
2074 cv_init(&snfq->snfq_cv, NULL, CV_DEFAULT, NULL);
2075 snfq->snfq_max_threads = max_ncpus;
2076 snfq_timeout = SNFQ_TIMEOUT;
2077 /* Cache all files by default. */
2078 sendfile_max_size = MAXOFFSET_T;
2079 }
2080
2081 /*
2082 * Queues a mblk_t for network processing.
2083 */
2084 static void
snf_enque(snf_req_t * sr,mblk_t * mp)2085 snf_enque(snf_req_t *sr, mblk_t *mp)
2086 {
2087 mp->b_next = NULL;
2088 mutex_enter(&sr->sr_lock);
2089 if (sr->sr_mp_head == NULL) {
2090 sr->sr_mp_head = sr->sr_mp_tail = mp;
2091 cv_signal(&sr->sr_cv);
2092 } else {
2093 sr->sr_mp_tail->b_next = mp;
2094 sr->sr_mp_tail = mp;
2095 }
2096 sr->sr_qlen += MBLKL(mp);
2097 while ((sr->sr_qlen > sr->sr_hiwat) &&
2098 (sr->sr_write_error == 0)) {
2099 sf_stats.ss_full_waits++;
2100 cv_wait(&sr->sr_cv, &sr->sr_lock);
2101 }
2102 mutex_exit(&sr->sr_lock);
2103 }
2104
2105 /*
2106 * De-queues a mblk_t for network processing.
2107 */
2108 static mblk_t *
snf_deque(snf_req_t * sr)2109 snf_deque(snf_req_t *sr)
2110 {
2111 mblk_t *mp;
2112
2113 mutex_enter(&sr->sr_lock);
2114 /*
2115 * If we have encountered an error on read or read is
2116 * completed and no more mblks, return NULL.
2117 * We need to check for NULL sr_mp_head also as
2118 * the reads could have completed and there is
2119 * nothing more to come.
2120 */
2121 if (((sr->sr_read_error & ~SR_READ_DONE) != 0) ||
2122 ((sr->sr_read_error & SR_READ_DONE) &&
2123 sr->sr_mp_head == NULL)) {
2124 mutex_exit(&sr->sr_lock);
2125 return (NULL);
2126 }
2127 /*
2128 * To start with neither SR_READ_DONE is marked nor
2129 * the error is set. When we wake up from cv_wait,
2130 * following are the possibilities :
2131 *
2132 * a) sr_read_error is zero and mblks are queued.
2133 * b) sr_read_error is set to SR_READ_DONE
2134 * and mblks are queued.
2135 * c) sr_read_error is set to SR_READ_DONE
2136 * and no mblks.
2137 * d) sr_read_error is set to some error other
2138 * than SR_READ_DONE.
2139 */
2140
2141 while ((sr->sr_read_error == 0) && (sr->sr_mp_head == NULL)) {
2142 sf_stats.ss_empty_waits++;
2143 cv_wait(&sr->sr_cv, &sr->sr_lock);
2144 }
2145 /* Handle (a) and (b) first - the normal case. */
2146 if (((sr->sr_read_error & ~SR_READ_DONE) == 0) &&
2147 (sr->sr_mp_head != NULL)) {
2148 mp = sr->sr_mp_head;
2149 sr->sr_mp_head = mp->b_next;
2150 sr->sr_qlen -= MBLKL(mp);
2151 if (sr->sr_qlen < sr->sr_lowat)
2152 cv_signal(&sr->sr_cv);
2153 mutex_exit(&sr->sr_lock);
2154 mp->b_next = NULL;
2155 return (mp);
2156 }
2157 /* Handle (c) and (d). */
2158 mutex_exit(&sr->sr_lock);
2159 return (NULL);
2160 }
2161
2162 /*
2163 * Reads data from the filesystem and queues it for network processing.
2164 */
2165 void
snf_async_read(snf_req_t * sr)2166 snf_async_read(snf_req_t *sr)
2167 {
2168 size_t iosize;
2169 u_offset_t fileoff;
2170 u_offset_t size;
2171 int ret_size;
2172 int error;
2173 file_t *fp;
2174 mblk_t *mp;
2175 struct vnode *vp;
2176 int extra = 0;
2177 int maxblk = 0;
2178 int wroff = 0;
2179 struct sonode *so = NULL;
2180
2181 fp = sr->sr_fp;
2182 size = sr->sr_file_size;
2183 fileoff = sr->sr_file_off;
2184
2185 /*
2186 * Ignore the error for filesystems that doesn't support DIRECTIO.
2187 */
2188 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_ON, 0,
2189 kcred, NULL, NULL);
2190
2191 vp = sr->sr_vp;
2192 if (vp->v_type == VSOCK) {
2193 stdata_t *stp;
2194
2195 /*
2196 * Get the extra space to insert a header and a trailer.
2197 */
2198 so = VTOSO(vp);
2199 stp = vp->v_stream;
2200 if (stp == NULL) {
2201 wroff = so->so_proto_props.sopp_wroff;
2202 maxblk = so->so_proto_props.sopp_maxblk;
2203 extra = wroff + so->so_proto_props.sopp_tail;
2204 } else {
2205 wroff = (int)(stp->sd_wroff);
2206 maxblk = (int)(stp->sd_maxblk);
2207 extra = wroff + (int)(stp->sd_tail);
2208 }
2209 }
2210
2211 while ((size != 0) && (sr->sr_write_error == 0)) {
2212
2213 iosize = (int)MIN(sr->sr_maxpsz, size);
2214
2215 /*
2216 * Socket filters can limit the mblk size,
2217 * so limit reads to maxblk if there are
2218 * filters present.
2219 */
2220 if (vp->v_type == VSOCK &&
2221 so->so_filter_active > 0 && maxblk != INFPSZ)
2222 iosize = (int)MIN(iosize, maxblk);
2223
2224 if (is_system_labeled()) {
2225 mp = allocb_cred(iosize + extra, CRED(),
2226 curproc->p_pid);
2227 } else {
2228 mp = allocb(iosize + extra, BPRI_MED);
2229 }
2230 if (mp == NULL) {
2231 error = EAGAIN;
2232 break;
2233 }
2234
2235 mp->b_rptr += wroff;
2236
2237 ret_size = soreadfile(fp, mp->b_rptr, fileoff, &error, iosize);
2238
2239 /* Error or Reached EOF ? */
2240 if ((error != 0) || (ret_size == 0)) {
2241 freeb(mp);
2242 break;
2243 }
2244 mp->b_wptr = mp->b_rptr + ret_size;
2245
2246 snf_enque(sr, mp);
2247 size -= ret_size;
2248 fileoff += ret_size;
2249 }
2250 (void) VOP_IOCTL(fp->f_vnode, _FIODIRECTIO, DIRECTIO_OFF, 0,
2251 kcred, NULL, NULL);
2252 mutex_enter(&sr->sr_lock);
2253 sr->sr_read_error = error;
2254 sr->sr_read_error |= SR_READ_DONE;
2255 cv_signal(&sr->sr_cv);
2256 mutex_exit(&sr->sr_lock);
2257 }
2258
2259 void
snf_async_thread(void)2260 snf_async_thread(void)
2261 {
2262 snf_req_t *sr;
2263 callb_cpr_t cprinfo;
2264 clock_t time_left = 1;
2265
2266 CALLB_CPR_INIT(&cprinfo, &snfq->snfq_lock, callb_generic_cpr, "snfq");
2267
2268 mutex_enter(&snfq->snfq_lock);
2269 for (;;) {
2270 /*
2271 * If we didn't find a entry, then block until woken up
2272 * again and then look through the queues again.
2273 */
2274 while ((sr = snfq->snfq_req_head) == NULL) {
2275 CALLB_CPR_SAFE_BEGIN(&cprinfo);
2276 if (time_left <= 0) {
2277 snfq->snfq_svc_threads--;
2278 CALLB_CPR_EXIT(&cprinfo);
2279 thread_exit();
2280 /* NOTREACHED */
2281 }
2282 snfq->snfq_idle_cnt++;
2283
2284 time_left = cv_reltimedwait(&snfq->snfq_cv,
2285 &snfq->snfq_lock, snfq_timeout, TR_CLOCK_TICK);
2286 snfq->snfq_idle_cnt--;
2287
2288 CALLB_CPR_SAFE_END(&cprinfo, &snfq->snfq_lock);
2289 }
2290 snfq->snfq_req_head = sr->sr_next;
2291 snfq->snfq_req_cnt--;
2292 mutex_exit(&snfq->snfq_lock);
2293 snf_async_read(sr);
2294 mutex_enter(&snfq->snfq_lock);
2295 }
2296 }
2297
2298
2299 snf_req_t *
create_thread(int operation,struct vnode * vp,file_t * fp,u_offset_t fileoff,u_offset_t size)2300 create_thread(int operation, struct vnode *vp, file_t *fp,
2301 u_offset_t fileoff, u_offset_t size)
2302 {
2303 snf_req_t *sr;
2304 stdata_t *stp;
2305
2306 sr = (snf_req_t *)kmem_zalloc(sizeof (snf_req_t), KM_SLEEP);
2307
2308 sr->sr_vp = vp;
2309 sr->sr_fp = fp;
2310 stp = vp->v_stream;
2311
2312 /*
2313 * store sd_qn_maxpsz into sr_maxpsz while we have stream head.
2314 * stream might be closed before thread returns from snf_async_read.
2315 */
2316 if (stp != NULL && stp->sd_qn_maxpsz > 0) {
2317 sr->sr_maxpsz = MIN(MAXBSIZE, stp->sd_qn_maxpsz);
2318 } else {
2319 sr->sr_maxpsz = MAXBSIZE;
2320 }
2321
2322 sr->sr_operation = operation;
2323 sr->sr_file_off = fileoff;
2324 sr->sr_file_size = size;
2325 sr->sr_hiwat = sendfile_req_hiwat;
2326 sr->sr_lowat = sendfile_req_lowat;
2327 mutex_init(&sr->sr_lock, NULL, MUTEX_DEFAULT, NULL);
2328 cv_init(&sr->sr_cv, NULL, CV_DEFAULT, NULL);
2329 /*
2330 * See whether we need another thread for servicing this
2331 * request. If there are already enough requests queued
2332 * for the threads, create one if not exceeding
2333 * snfq_max_threads.
2334 */
2335 mutex_enter(&snfq->snfq_lock);
2336 if (snfq->snfq_req_cnt >= snfq->snfq_idle_cnt &&
2337 snfq->snfq_svc_threads < snfq->snfq_max_threads) {
2338 (void) thread_create(NULL, 0, &snf_async_thread, 0, 0, &p0,
2339 TS_RUN, minclsyspri);
2340 snfq->snfq_svc_threads++;
2341 }
2342 if (snfq->snfq_req_head == NULL) {
2343 snfq->snfq_req_head = snfq->snfq_req_tail = sr;
2344 cv_signal(&snfq->snfq_cv);
2345 } else {
2346 snfq->snfq_req_tail->sr_next = sr;
2347 snfq->snfq_req_tail = sr;
2348 }
2349 snfq->snfq_req_cnt++;
2350 mutex_exit(&snfq->snfq_lock);
2351 return (sr);
2352 }
2353
2354 int
snf_direct_io(file_t * fp,file_t * rfp,u_offset_t fileoff,u_offset_t size,ssize_t * count)2355 snf_direct_io(file_t *fp, file_t *rfp, u_offset_t fileoff, u_offset_t size,
2356 ssize_t *count)
2357 {
2358 snf_req_t *sr;
2359 mblk_t *mp;
2360 int iosize;
2361 int error = 0;
2362 short fflag;
2363 struct vnode *vp;
2364 int ksize;
2365 struct nmsghdr msg;
2366
2367 ksize = 0;
2368 *count = 0;
2369 bzero(&msg, sizeof (msg));
2370
2371 vp = fp->f_vnode;
2372 fflag = fp->f_flag;
2373 if ((sr = create_thread(READ_OP, vp, rfp, fileoff, size)) == NULL)
2374 return (EAGAIN);
2375
2376 /*
2377 * We check for read error in snf_deque. It has to check
2378 * for successful READ_DONE and return NULL, and we might
2379 * as well make an additional check there.
2380 */
2381 while ((mp = snf_deque(sr)) != NULL) {
2382
2383 if (ISSIG(curthread, JUSTLOOKING)) {
2384 freeb(mp);
2385 error = EINTR;
2386 break;
2387 }
2388 iosize = MBLKL(mp);
2389
2390 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2391
2392 if (error != 0) {
2393 if (mp != NULL)
2394 freeb(mp);
2395 break;
2396 }
2397 ksize += iosize;
2398 }
2399 *count = ksize;
2400
2401 mutex_enter(&sr->sr_lock);
2402 sr->sr_write_error = error;
2403 /* Look at the big comments on why we cv_signal here. */
2404 cv_signal(&sr->sr_cv);
2405
2406 /* Wait for the reader to complete always. */
2407 while (!(sr->sr_read_error & SR_READ_DONE)) {
2408 cv_wait(&sr->sr_cv, &sr->sr_lock);
2409 }
2410 /* If there is no write error, check for read error. */
2411 if (error == 0)
2412 error = (sr->sr_read_error & ~SR_READ_DONE);
2413
2414 if (error != 0) {
2415 mblk_t *next_mp;
2416
2417 mp = sr->sr_mp_head;
2418 while (mp != NULL) {
2419 next_mp = mp->b_next;
2420 mp->b_next = NULL;
2421 freeb(mp);
2422 mp = next_mp;
2423 }
2424 }
2425 mutex_exit(&sr->sr_lock);
2426 kmem_free(sr, sizeof (snf_req_t));
2427 return (error);
2428 }
2429
2430 /* Maximum no.of pages allocated by vpm for sendfile at a time */
2431 #define SNF_VPMMAXPGS (VPMMAXPGS/2)
2432
2433 /*
2434 * Maximum no.of elements in the list returned by vpm, including
2435 * NULL for the last entry
2436 */
2437 #define SNF_MAXVMAPS (SNF_VPMMAXPGS + 1)
2438
2439 typedef struct {
2440 unsigned int snfv_ref;
2441 frtn_t snfv_frtn;
2442 vnode_t *snfv_vp;
2443 struct vmap snfv_vml[SNF_MAXVMAPS];
2444 } snf_vmap_desbinfo;
2445
2446 typedef struct {
2447 frtn_t snfi_frtn;
2448 caddr_t snfi_base;
2449 uint_t snfi_mapoff;
2450 size_t snfi_len;
2451 vnode_t *snfi_vp;
2452 } snf_smap_desbinfo;
2453
2454 /*
2455 * The callback function used for vpm mapped mblks called when the last ref of
2456 * the mblk is dropped which normally occurs when TCP receives the ack. But it
2457 * can be the driver too due to lazy reclaim.
2458 */
2459 void
snf_vmap_desbfree(snf_vmap_desbinfo * snfv)2460 snf_vmap_desbfree(snf_vmap_desbinfo *snfv)
2461 {
2462 ASSERT(snfv->snfv_ref != 0);
2463 if (atomic_dec_32_nv(&snfv->snfv_ref) == 0) {
2464 vpm_unmap_pages(snfv->snfv_vml, S_READ);
2465 VN_RELE(snfv->snfv_vp);
2466 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2467 }
2468 }
2469
2470 /*
2471 * The callback function used for segmap'ped mblks called when the last ref of
2472 * the mblk is dropped which normally occurs when TCP receives the ack. But it
2473 * can be the driver too due to lazy reclaim.
2474 */
2475 void
snf_smap_desbfree(snf_smap_desbinfo * snfi)2476 snf_smap_desbfree(snf_smap_desbinfo *snfi)
2477 {
2478 if (! IS_KPM_ADDR(snfi->snfi_base)) {
2479 /*
2480 * We don't need to call segmap_fault(F_SOFTUNLOCK) for
2481 * segmap_kpm as long as the latter never falls back to
2482 * "use_segmap_range". (See segmap_getmapflt().)
2483 *
2484 * Using S_OTHER saves an redundant hat_setref() in
2485 * segmap_unlock()
2486 */
2487 (void) segmap_fault(kas.a_hat, segkmap,
2488 (caddr_t)(uintptr_t)(((uintptr_t)snfi->snfi_base +
2489 snfi->snfi_mapoff) & PAGEMASK), snfi->snfi_len,
2490 F_SOFTUNLOCK, S_OTHER);
2491 }
2492 (void) segmap_release(segkmap, snfi->snfi_base, SM_DONTNEED);
2493 VN_RELE(snfi->snfi_vp);
2494 kmem_free(snfi, sizeof (*snfi));
2495 }
2496
2497 /*
2498 * Use segmap or vpm instead of bcopy to send down a desballoca'ed, mblk.
2499 * When segmap is used, the mblk contains a segmap slot of no more
2500 * than MAXBSIZE.
2501 *
2502 * With vpm, a maximum of SNF_MAXVMAPS page-sized mappings can be obtained
2503 * in each iteration and sent by socket_sendmblk until an error occurs or
2504 * the requested size has been transferred. An mblk is esballoca'ed from
2505 * each mapped page and a chain of these mblk is sent to the transport layer.
2506 * vpm will be called to unmap the pages when all mblks have been freed by
2507 * free_func.
2508 *
2509 * At the end of the whole sendfile() operation, we wait till the data from
2510 * the last mblk is ack'ed by the transport before returning so that the
2511 * caller of sendfile() can safely modify the file content.
2512 *
2513 * The caller of this function should make sure that total_size does not exceed
2514 * the actual file size of fvp.
2515 */
2516 int
snf_segmap(file_t * fp,vnode_t * fvp,u_offset_t fileoff,u_offset_t total_size,ssize_t * count,boolean_t nowait)2517 snf_segmap(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t total_size,
2518 ssize_t *count, boolean_t nowait)
2519 {
2520 caddr_t base;
2521 int mapoff;
2522 vnode_t *vp;
2523 mblk_t *mp = NULL;
2524 int chain_size;
2525 int error;
2526 clock_t deadlk_wait;
2527 short fflag;
2528 int ksize;
2529 struct vattr va;
2530 boolean_t dowait = B_FALSE;
2531 struct nmsghdr msg;
2532
2533 vp = fp->f_vnode;
2534 fflag = fp->f_flag;
2535 ksize = 0;
2536 bzero(&msg, sizeof (msg));
2537
2538 for (;;) {
2539 if (ISSIG(curthread, JUSTLOOKING)) {
2540 error = EINTR;
2541 break;
2542 }
2543
2544 if (vpm_enable) {
2545 snf_vmap_desbinfo *snfv;
2546 mblk_t *nmp;
2547 int mblk_size;
2548 int maxsize;
2549 int i;
2550
2551 mapoff = fileoff & PAGEOFFSET;
2552 maxsize = MIN((SNF_VPMMAXPGS * PAGESIZE), total_size);
2553
2554 snfv = kmem_zalloc(sizeof (snf_vmap_desbinfo),
2555 KM_SLEEP);
2556
2557 /*
2558 * Get vpm mappings for maxsize with read access.
2559 * If the pages aren't available yet, we get
2560 * DEADLK, so wait and try again a little later using
2561 * an increasing wait. We might be here a long time.
2562 *
2563 * If delay_sig returns EINTR, be sure to exit and
2564 * pass it up to the caller.
2565 */
2566 deadlk_wait = 0;
2567 while ((error = vpm_map_pages(fvp, fileoff,
2568 (size_t)maxsize, (VPM_FETCHPAGE), snfv->snfv_vml,
2569 SNF_MAXVMAPS, NULL, S_READ)) == EDEADLK) {
2570 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2571 if ((error = delay_sig(deadlk_wait)) != 0) {
2572 break;
2573 }
2574 }
2575 if (error != 0) {
2576 kmem_free(snfv, sizeof (snf_vmap_desbinfo));
2577 error = (error == EINTR) ? EINTR : EIO;
2578 goto out;
2579 }
2580 snfv->snfv_frtn.free_func = snf_vmap_desbfree;
2581 snfv->snfv_frtn.free_arg = (caddr_t)snfv;
2582
2583 /* Construct the mblk chain from the page mappings */
2584 chain_size = 0;
2585 for (i = 0; (snfv->snfv_vml[i].vs_addr != NULL) &&
2586 total_size > 0; i++) {
2587 ASSERT(chain_size < maxsize);
2588 mblk_size = MIN(snfv->snfv_vml[i].vs_len -
2589 mapoff, total_size);
2590 nmp = esballoca(
2591 (uchar_t *)snfv->snfv_vml[i].vs_addr +
2592 mapoff, mblk_size, BPRI_HI,
2593 &snfv->snfv_frtn);
2594
2595 /*
2596 * We return EAGAIN after unmapping the pages
2597 * if we cannot allocate the the head of the
2598 * chain. Otherwise, we continue sending the
2599 * mblks constructed so far.
2600 */
2601 if (nmp == NULL) {
2602 if (i == 0) {
2603 vpm_unmap_pages(snfv->snfv_vml,
2604 S_READ);
2605 kmem_free(snfv,
2606 sizeof (snf_vmap_desbinfo));
2607 error = EAGAIN;
2608 goto out;
2609 }
2610 break;
2611 }
2612 /* Mark this dblk with the zero-copy flag */
2613 nmp->b_datap->db_struioflag |= STRUIO_ZC;
2614 nmp->b_wptr += mblk_size;
2615 chain_size += mblk_size;
2616 fileoff += mblk_size;
2617 total_size -= mblk_size;
2618 snfv->snfv_ref++;
2619 mapoff = 0;
2620 if (i > 0)
2621 linkb(mp, nmp);
2622 else
2623 mp = nmp;
2624 }
2625 VN_HOLD(fvp);
2626 snfv->snfv_vp = fvp;
2627 } else {
2628 /* vpm not supported. fallback to segmap */
2629 snf_smap_desbinfo *snfi;
2630
2631 mapoff = fileoff & MAXBOFFSET;
2632 chain_size = MAXBSIZE - mapoff;
2633 if (chain_size > total_size)
2634 chain_size = total_size;
2635 /*
2636 * we don't forcefault because we'll call
2637 * segmap_fault(F_SOFTLOCK) next.
2638 *
2639 * S_READ will get the ref bit set (by either
2640 * segmap_getmapflt() or segmap_fault()) and page
2641 * shared locked.
2642 */
2643 base = segmap_getmapflt(segkmap, fvp, fileoff,
2644 chain_size, segmap_kpm ? SM_FAULT : 0, S_READ);
2645
2646 snfi = kmem_alloc(sizeof (*snfi), KM_SLEEP);
2647 snfi->snfi_len = (size_t)roundup(mapoff+chain_size,
2648 PAGESIZE)- (mapoff & PAGEMASK);
2649 /*
2650 * We must call segmap_fault() even for segmap_kpm
2651 * because that's how error gets returned.
2652 * (segmap_getmapflt() never fails but segmap_fault()
2653 * does.)
2654 *
2655 * If the pages aren't available yet, we get
2656 * DEADLK, so wait and try again a little later using
2657 * an increasing wait. We might be here a long time.
2658 *
2659 * If delay_sig returns EINTR, be sure to exit and
2660 * pass it up to the caller.
2661 */
2662 deadlk_wait = 0;
2663 while ((error = FC_ERRNO(segmap_fault(kas.a_hat,
2664 segkmap, (caddr_t)(uintptr_t)(((uintptr_t)base +
2665 mapoff) & PAGEMASK), snfi->snfi_len, F_SOFTLOCK,
2666 S_READ))) == EDEADLK) {
2667 deadlk_wait += (deadlk_wait < 5) ? 1 : 4;
2668 if ((error = delay_sig(deadlk_wait)) != 0) {
2669 break;
2670 }
2671 }
2672 if (error != 0) {
2673 (void) segmap_release(segkmap, base, 0);
2674 kmem_free(snfi, sizeof (*snfi));
2675 error = (error == EINTR) ? EINTR : EIO;
2676 goto out;
2677 }
2678 snfi->snfi_frtn.free_func = snf_smap_desbfree;
2679 snfi->snfi_frtn.free_arg = (caddr_t)snfi;
2680 snfi->snfi_base = base;
2681 snfi->snfi_mapoff = mapoff;
2682 mp = esballoca((uchar_t *)base + mapoff, chain_size,
2683 BPRI_HI, &snfi->snfi_frtn);
2684
2685 if (mp == NULL) {
2686 (void) segmap_fault(kas.a_hat, segkmap,
2687 (caddr_t)(uintptr_t)(((uintptr_t)base +
2688 mapoff) & PAGEMASK), snfi->snfi_len,
2689 F_SOFTUNLOCK, S_OTHER);
2690 (void) segmap_release(segkmap, base, 0);
2691 kmem_free(snfi, sizeof (*snfi));
2692 freemsg(mp);
2693 error = EAGAIN;
2694 goto out;
2695 }
2696 VN_HOLD(fvp);
2697 snfi->snfi_vp = fvp;
2698 mp->b_wptr += chain_size;
2699
2700 /* Mark this dblk with the zero-copy flag */
2701 mp->b_datap->db_struioflag |= STRUIO_ZC;
2702 fileoff += chain_size;
2703 total_size -= chain_size;
2704 }
2705
2706 if (total_size == 0 && !nowait) {
2707 ASSERT(!dowait);
2708 dowait = B_TRUE;
2709 mp->b_datap->db_struioflag |= STRUIO_ZCNOTIFY;
2710 }
2711 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2712 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2713 if (error != 0) {
2714 /*
2715 * mp contains the mblks that were not sent by
2716 * socket_sendmblk. Use its size to update *count
2717 */
2718 *count = ksize + (chain_size - msgdsize(mp));
2719 if (mp != NULL)
2720 freemsg(mp);
2721 return (error);
2722 }
2723 ksize += chain_size;
2724 if (total_size == 0)
2725 goto done;
2726
2727 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2728 va.va_mask = AT_SIZE;
2729 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2730 if (error)
2731 break;
2732 /* Read as much as possible. */
2733 if (fileoff >= va.va_size)
2734 break;
2735 if (total_size + fileoff > va.va_size)
2736 total_size = va.va_size - fileoff;
2737 }
2738 out:
2739 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2740 done:
2741 *count = ksize;
2742 if (dowait) {
2743 stdata_t *stp;
2744
2745 stp = vp->v_stream;
2746 if (stp == NULL) {
2747 struct sonode *so;
2748 so = VTOSO(vp);
2749 error = so_zcopy_wait(so);
2750 } else {
2751 mutex_enter(&stp->sd_lock);
2752 while (!(stp->sd_flag & STZCNOTIFY)) {
2753 if (cv_wait_sig(&stp->sd_zcopy_wait,
2754 &stp->sd_lock) == 0) {
2755 error = EINTR;
2756 break;
2757 }
2758 }
2759 stp->sd_flag &= ~STZCNOTIFY;
2760 mutex_exit(&stp->sd_lock);
2761 }
2762 }
2763 return (error);
2764 }
2765
2766 int
snf_cache(file_t * fp,vnode_t * fvp,u_offset_t fileoff,u_offset_t size,uint_t maxpsz,ssize_t * count)2767 snf_cache(file_t *fp, vnode_t *fvp, u_offset_t fileoff, u_offset_t size,
2768 uint_t maxpsz, ssize_t *count)
2769 {
2770 struct vnode *vp;
2771 mblk_t *mp;
2772 int iosize;
2773 int extra = 0;
2774 int error;
2775 short fflag;
2776 int ksize;
2777 int ioflag;
2778 struct uio auio;
2779 struct iovec aiov;
2780 struct vattr va;
2781 int maxblk = 0;
2782 int wroff = 0;
2783 struct sonode *so = NULL;
2784 struct nmsghdr msg;
2785
2786 vp = fp->f_vnode;
2787 if (vp->v_type == VSOCK) {
2788 stdata_t *stp;
2789
2790 /*
2791 * Get the extra space to insert a header and a trailer.
2792 */
2793 so = VTOSO(vp);
2794 stp = vp->v_stream;
2795 if (stp == NULL) {
2796 wroff = so->so_proto_props.sopp_wroff;
2797 maxblk = so->so_proto_props.sopp_maxblk;
2798 extra = wroff + so->so_proto_props.sopp_tail;
2799 } else {
2800 wroff = (int)(stp->sd_wroff);
2801 maxblk = (int)(stp->sd_maxblk);
2802 extra = wroff + (int)(stp->sd_tail);
2803 }
2804 }
2805 bzero(&msg, sizeof (msg));
2806 fflag = fp->f_flag;
2807 ksize = 0;
2808 auio.uio_iov = &aiov;
2809 auio.uio_iovcnt = 1;
2810 auio.uio_segflg = UIO_SYSSPACE;
2811 auio.uio_llimit = MAXOFFSET_T;
2812 auio.uio_fmode = fflag;
2813 auio.uio_extflg = UIO_COPY_CACHED;
2814 ioflag = auio.uio_fmode & (FSYNC|FDSYNC|FRSYNC);
2815 /* If read sync is not asked for, filter sync flags */
2816 if ((ioflag & FRSYNC) == 0)
2817 ioflag &= ~(FSYNC|FDSYNC);
2818 for (;;) {
2819 if (ISSIG(curthread, JUSTLOOKING)) {
2820 error = EINTR;
2821 break;
2822 }
2823 iosize = (int)MIN(maxpsz, size);
2824
2825 /*
2826 * Socket filters can limit the mblk size,
2827 * so limit reads to maxblk if there are
2828 * filters present.
2829 */
2830 if (vp->v_type == VSOCK &&
2831 so->so_filter_active > 0 && maxblk != INFPSZ)
2832 iosize = (int)MIN(iosize, maxblk);
2833
2834 if (is_system_labeled()) {
2835 mp = allocb_cred(iosize + extra, CRED(),
2836 curproc->p_pid);
2837 } else {
2838 mp = allocb(iosize + extra, BPRI_MED);
2839 }
2840 if (mp == NULL) {
2841 error = EAGAIN;
2842 break;
2843 }
2844
2845 mp->b_rptr += wroff;
2846
2847 aiov.iov_base = (caddr_t)mp->b_rptr;
2848 aiov.iov_len = iosize;
2849 auio.uio_loffset = fileoff;
2850 auio.uio_resid = iosize;
2851
2852 error = VOP_READ(fvp, &auio, ioflag, fp->f_cred, NULL);
2853 iosize -= auio.uio_resid;
2854
2855 if (error == EINTR && iosize != 0)
2856 error = 0;
2857
2858 if (error != 0 || iosize == 0) {
2859 freeb(mp);
2860 break;
2861 }
2862 mp->b_wptr = mp->b_rptr + iosize;
2863
2864 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2865
2866 error = socket_sendmblk(VTOSO(vp), &msg, fflag, CRED(), &mp);
2867
2868 if (error != 0) {
2869 *count = ksize;
2870 if (mp != NULL)
2871 freeb(mp);
2872 return (error);
2873 }
2874 ksize += iosize;
2875 size -= iosize;
2876 if (size == 0)
2877 goto done;
2878
2879 fileoff += iosize;
2880 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2881 va.va_mask = AT_SIZE;
2882 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2883 if (error)
2884 break;
2885 /* Read as much as possible. */
2886 if (fileoff >= va.va_size)
2887 size = 0;
2888 else if (size + fileoff > va.va_size)
2889 size = va.va_size - fileoff;
2890 }
2891 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2892 done:
2893 *count = ksize;
2894 return (error);
2895 }
2896
2897 #if defined(_SYSCALL32_IMPL) || defined(_ILP32)
2898 /*
2899 * Largefile support for 32 bit applications only.
2900 */
2901 int
sosendfile64(file_t * fp,file_t * rfp,const struct ksendfilevec64 * sfv,ssize32_t * count32)2902 sosendfile64(file_t *fp, file_t *rfp, const struct ksendfilevec64 *sfv,
2903 ssize32_t *count32)
2904 {
2905 ssize32_t sfv_len;
2906 u_offset_t sfv_off, va_size;
2907 struct vnode *vp, *fvp, *realvp;
2908 struct vattr va;
2909 stdata_t *stp;
2910 ssize_t count = 0;
2911 int error = 0;
2912 boolean_t dozcopy = B_FALSE;
2913 uint_t maxpsz;
2914
2915 sfv_len = (ssize32_t)sfv->sfv_len;
2916 if (sfv_len < 0) {
2917 error = EINVAL;
2918 goto out;
2919 }
2920
2921 if (sfv_len == 0) goto out;
2922
2923 sfv_off = (u_offset_t)sfv->sfv_off;
2924
2925 /* Same checks as in pread */
2926 if (sfv_off > MAXOFFSET_T) {
2927 error = EINVAL;
2928 goto out;
2929 }
2930 if (sfv_off + sfv_len > MAXOFFSET_T)
2931 sfv_len = (ssize32_t)(MAXOFFSET_T - sfv_off);
2932
2933 /*
2934 * There are no more checks on sfv_len. So, we cast it to
2935 * u_offset_t and share the snf_direct_io/snf_cache code between
2936 * 32 bit and 64 bit.
2937 *
2938 * TODO: should do nbl_need_check() like read()?
2939 */
2940 if (sfv_len > sendfile_max_size) {
2941 sf_stats.ss_file_not_cached++;
2942 error = snf_direct_io(fp, rfp, sfv_off, (u_offset_t)sfv_len,
2943 &count);
2944 goto out;
2945 }
2946 fvp = rfp->f_vnode;
2947 if (VOP_REALVP(fvp, &realvp, NULL) == 0)
2948 fvp = realvp;
2949 /*
2950 * Grab the lock as a reader to prevent the file size
2951 * from changing underneath.
2952 */
2953 (void) VOP_RWLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2954 va.va_mask = AT_SIZE;
2955 error = VOP_GETATTR(fvp, &va, 0, kcred, NULL);
2956 va_size = va.va_size;
2957 if ((error != 0) || (va_size == 0) || (sfv_off >= va_size)) {
2958 VOP_RWUNLOCK(fvp, V_WRITELOCK_FALSE, NULL);
2959 goto out;
2960 }
2961 /* Read as much as possible. */
2962 if (sfv_off + sfv_len > va_size)
2963 sfv_len = va_size - sfv_off;
2964
2965 vp = fp->f_vnode;
2966 stp = vp->v_stream;
2967 /*
2968 * When the NOWAIT flag is not set, we enable zero-copy only if the
2969 * transfer size is large enough. This prevents performance loss
2970 * when the caller sends the file piece by piece.
2971 */
2972 if (sfv_len >= MAXBSIZE && (sfv_len >= (va_size >> 1) ||
2973 (sfv->sfv_flag & SFV_NOWAIT) || sfv_len >= 0x1000000) &&
2974 !vn_has_flocks(fvp) && !(fvp->v_flag & VNOMAP)) {
2975 uint_t copyflag;
2976 copyflag = stp != NULL ? stp->sd_copyflag :
2977 VTOSO(vp)->so_proto_props.sopp_zcopyflag;
2978 if ((copyflag & (STZCVMSAFE|STZCVMUNSAFE)) == 0) {
2979 int on = 1;
2980
2981 if (socket_setsockopt(VTOSO(vp), SOL_SOCKET,
2982 SO_SND_COPYAVOID, &on, sizeof (on), CRED()) == 0)
2983 dozcopy = B_TRUE;
2984 } else {
2985 dozcopy = copyflag & STZCVMSAFE;
2986 }
2987 }
2988 if (dozcopy) {
2989 sf_stats.ss_file_segmap++;
2990 error = snf_segmap(fp, fvp, sfv_off, (u_offset_t)sfv_len,
2991 &count, ((sfv->sfv_flag & SFV_NOWAIT) != 0));
2992 } else {
2993 if (vp->v_type == VSOCK && stp == NULL) {
2994 sonode_t *so = VTOSO(vp);
2995 maxpsz = so->so_proto_props.sopp_maxpsz;
2996 } else if (stp != NULL) {
2997 maxpsz = stp->sd_qn_maxpsz;
2998 } else {
2999 maxpsz = maxphys;
3000 }
3001
3002 if (maxpsz == INFPSZ)
3003 maxpsz = maxphys;
3004 else
3005 maxpsz = roundup(maxpsz, MAXBSIZE);
3006 sf_stats.ss_file_cached++;
3007 error = snf_cache(fp, fvp, sfv_off, (u_offset_t)sfv_len,
3008 maxpsz, &count);
3009 }
3010 out:
3011 releasef(sfv->sfv_fd);
3012 *count32 = (ssize32_t)count;
3013 return (error);
3014 }
3015 #endif
3016
3017 #ifdef _SYSCALL32_IMPL
3018 /*
3019 * recv32(), recvfrom32(), send32(), sendto32(): intentionally return a
3020 * ssize_t rather than ssize32_t; see the comments above read32 for details.
3021 */
3022
3023 ssize_t
recv32(int32_t sock,caddr32_t buffer,size32_t len,int32_t flags)3024 recv32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
3025 {
3026 return (recv(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
3027 }
3028
3029 ssize_t
recvfrom32(int32_t sock,caddr32_t buffer,size32_t len,int32_t flags,caddr32_t name,caddr32_t namelenp)3030 recvfrom32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
3031 caddr32_t name, caddr32_t namelenp)
3032 {
3033 return (recvfrom(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
3034 (void *)(uintptr_t)name, (void *)(uintptr_t)namelenp));
3035 }
3036
3037 ssize_t
send32(int32_t sock,caddr32_t buffer,size32_t len,int32_t flags)3038 send32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags)
3039 {
3040 return (send(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags));
3041 }
3042
3043 ssize_t
sendto32(int32_t sock,caddr32_t buffer,size32_t len,int32_t flags,caddr32_t name,socklen_t namelen)3044 sendto32(int32_t sock, caddr32_t buffer, size32_t len, int32_t flags,
3045 caddr32_t name, socklen_t namelen)
3046 {
3047 return (sendto(sock, (void *)(uintptr_t)buffer, (ssize32_t)len, flags,
3048 (void *)(uintptr_t)name, namelen));
3049 }
3050 #endif /* _SYSCALL32_IMPL */
3051
3052 /*
3053 * Function wrappers (mostly around the sonode switch) for
3054 * backward compatibility.
3055 */
3056
3057 int
soaccept(struct sonode * so,int fflag,struct sonode ** nsop)3058 soaccept(struct sonode *so, int fflag, struct sonode **nsop)
3059 {
3060 return (socket_accept(so, fflag, CRED(), nsop));
3061 }
3062
3063 int
sobind(struct sonode * so,struct sockaddr * name,socklen_t namelen,int backlog,int flags)3064 sobind(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3065 int backlog, int flags)
3066 {
3067 int error;
3068
3069 error = socket_bind(so, name, namelen, flags, CRED());
3070 if (error == 0 && backlog != 0)
3071 return (socket_listen(so, backlog, CRED()));
3072
3073 return (error);
3074 }
3075
3076 int
solisten(struct sonode * so,int backlog)3077 solisten(struct sonode *so, int backlog)
3078 {
3079 return (socket_listen(so, backlog, CRED()));
3080 }
3081
3082 int
soconnect(struct sonode * so,struct sockaddr * name,socklen_t namelen,int fflag,int flags)3083 soconnect(struct sonode *so, struct sockaddr *name, socklen_t namelen,
3084 int fflag, int flags)
3085 {
3086 return (socket_connect(so, name, namelen, fflag, flags, CRED()));
3087 }
3088
3089 int
sorecvmsg(struct sonode * so,struct nmsghdr * msg,struct uio * uiop)3090 sorecvmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3091 {
3092 return (socket_recvmsg(so, msg, uiop, CRED()));
3093 }
3094
3095 int
sosendmsg(struct sonode * so,struct nmsghdr * msg,struct uio * uiop)3096 sosendmsg(struct sonode *so, struct nmsghdr *msg, struct uio *uiop)
3097 {
3098 return (socket_sendmsg(so, msg, uiop, CRED()));
3099 }
3100
3101 int
soshutdown(struct sonode * so,int how)3102 soshutdown(struct sonode *so, int how)
3103 {
3104 return (socket_shutdown(so, how, CRED()));
3105 }
3106
3107 int
sogetsockopt(struct sonode * so,int level,int option_name,void * optval,socklen_t * optlenp,int flags)3108 sogetsockopt(struct sonode *so, int level, int option_name, void *optval,
3109 socklen_t *optlenp, int flags)
3110 {
3111 return (socket_getsockopt(so, level, option_name, optval, optlenp,
3112 flags, CRED()));
3113 }
3114
3115 int
sosetsockopt(struct sonode * so,int level,int option_name,const void * optval,t_uscalar_t optlen)3116 sosetsockopt(struct sonode *so, int level, int option_name, const void *optval,
3117 t_uscalar_t optlen)
3118 {
3119 return (socket_setsockopt(so, level, option_name, optval, optlen,
3120 CRED()));
3121 }
3122
3123 /*
3124 * Because this is backward compatibility interface it only needs to be
3125 * able to handle the creation of TPI sockfs sockets.
3126 */
3127 struct sonode *
socreate(struct sockparams * sp,int family,int type,int protocol,int version,int * errorp)3128 socreate(struct sockparams *sp, int family, int type, int protocol, int version,
3129 int *errorp)
3130 {
3131 struct sonode *so;
3132
3133 ASSERT(sp != NULL);
3134
3135 so = sp->sp_smod_info->smod_sock_create_func(sp, family, type, protocol,
3136 version, SOCKET_SLEEP, errorp, CRED());
3137 if (so == NULL) {
3138 SOCKPARAMS_DEC_REF(sp);
3139 } else {
3140 if ((*errorp = SOP_INIT(so, NULL, CRED(), SOCKET_SLEEP)) == 0) {
3141 /* Cannot fail, only bumps so_count */
3142 (void) VOP_OPEN(&SOTOV(so), FREAD|FWRITE, CRED(), NULL);
3143 } else {
3144 socket_destroy(so);
3145 so = NULL;
3146 }
3147 }
3148 return (so);
3149 }
3150