xref: /illumos-gate/usr/src/uts/common/syscall/fcntl.c (revision 0f24ff92)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 
22 /* ONC_PLUS EXTRACT START */
23 /*
24  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
25  * Copyright (c) 2013, OmniTI Computer Consulting, Inc. All rights reserved.
26  */
27 
28 /*	Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T	*/
29 /*	  All Rights Reserved  	*/
30 
31 /*
32  * Portions of this source code were derived from Berkeley 4.3 BSD
33  * under license from the Regents of the University of California.
34  */
35 
36 
37 /* ONC_PLUS EXTRACT END */
38 
39 #include <sys/param.h>
40 #include <sys/isa_defs.h>
41 #include <sys/types.h>
42 #include <sys/sysmacros.h>
43 #include <sys/systm.h>
44 #include <sys/errno.h>
45 #include <sys/fcntl.h>
46 /* ONC_PLUS EXTRACT START */
47 #include <sys/flock.h>
48 /* ONC_PLUS EXTRACT END */
49 #include <sys/vnode.h>
50 #include <sys/file.h>
51 #include <sys/mode.h>
52 #include <sys/proc.h>
53 #include <sys/filio.h>
54 #include <sys/share.h>
55 #include <sys/debug.h>
56 #include <sys/rctl.h>
57 #include <sys/nbmlock.h>
58 
59 #include <sys/cmn_err.h>
60 
61 /* ONC_PLUS EXTRACT START */
62 static int flock_check(vnode_t *, flock64_t *, offset_t, offset_t);
63 static int flock_get_start(vnode_t *, flock64_t *, offset_t, u_offset_t *);
64 static void fd_too_big(proc_t *);
65 
66 /*
67  * File control.
68  */
69 int
70 fcntl(int fdes, int cmd, intptr_t arg)
71 {
72 	int iarg;
73 	int error = 0;
74 	int retval;
75 	proc_t *p;
76 	file_t *fp;
77 	vnode_t *vp;
78 	u_offset_t offset;
79 	u_offset_t start;
80 	struct vattr vattr;
81 	int in_crit;
82 	int flag;
83 	struct flock sbf;
84 	struct flock64 bf;
85 	struct o_flock obf;
86 	struct flock64_32 bf64_32;
87 	struct fshare fsh;
88 	struct shrlock shr;
89 	struct shr_locowner shr_own;
90 	offset_t maxoffset;
91 	model_t datamodel;
92 	int fdres;
93 
94 #if defined(_ILP32) && !defined(lint) && defined(_SYSCALL32)
95 	ASSERT(sizeof (struct flock) == sizeof (struct flock32));
96 	ASSERT(sizeof (struct flock64) == sizeof (struct flock64_32));
97 #endif
98 #if defined(_LP64) && !defined(lint) && defined(_SYSCALL32)
99 	ASSERT(sizeof (struct flock) == sizeof (struct flock64_64));
100 	ASSERT(sizeof (struct flock64) == sizeof (struct flock64_64));
101 #endif
102 
103 	/*
104 	 * First, for speed, deal with the subset of cases
105 	 * that do not require getf() / releasef().
106 	 */
107 	switch (cmd) {
108 	case F_GETFD:
109 		if ((error = f_getfd_error(fdes, &flag)) == 0)
110 			retval = flag;
111 		goto out;
112 
113 	case F_SETFD:
114 		error = f_setfd_error(fdes, (int)arg);
115 		retval = 0;
116 		goto out;
117 
118 	case F_GETFL:
119 		if ((error = f_getfl(fdes, &flag)) == 0) {
120 			retval = (flag & (FMASK | FASYNC));
121 			if ((flag & (FSEARCH | FEXEC)) == 0)
122 				retval += FOPEN;
123 			else
124 				retval |= (flag & (FSEARCH | FEXEC));
125 		}
126 		goto out;
127 
128 	case F_GETXFL:
129 		if ((error = f_getfl(fdes, &flag)) == 0) {
130 			retval = flag;
131 			if ((flag & (FSEARCH | FEXEC)) == 0)
132 				retval += FOPEN;
133 		}
134 		goto out;
135 
136 	case F_BADFD:
137 		if ((error = f_badfd(fdes, &fdres, (int)arg)) == 0)
138 			retval = fdres;
139 		goto out;
140 	}
141 
142 	/*
143 	 * Second, for speed, deal with the subset of cases that
144 	 * require getf() / releasef() but do not require copyin.
145 	 */
146 	if ((fp = getf(fdes)) == NULL) {
147 		error = EBADF;
148 		goto out;
149 	}
150 	iarg = (int)arg;
151 
152 	switch (cmd) {
153 /* ONC_PLUS EXTRACT END */
154 
155 	case F_DUPFD:
156 	case F_DUPFD_CLOEXEC:
157 		p = curproc;
158 		if ((uint_t)iarg >= p->p_fno_ctl) {
159 			if (iarg >= 0)
160 				fd_too_big(p);
161 			error = EINVAL;
162 			goto done;
163 		}
164 		/*
165 		 * We need to increment the f_count reference counter
166 		 * before allocating a new file descriptor.
167 		 * Doing it other way round opens a window for race condition
168 		 * with closeandsetf() on the target file descriptor which can
169 		 * close the file still referenced by the original
170 		 * file descriptor.
171 		 */
172 		mutex_enter(&fp->f_tlock);
173 		fp->f_count++;
174 		mutex_exit(&fp->f_tlock);
175 		if ((retval = ufalloc_file(iarg, fp)) == -1) {
176 			/*
177 			 * New file descriptor can't be allocated.
178 			 * Revert the reference count.
179 			 */
180 			mutex_enter(&fp->f_tlock);
181 			fp->f_count--;
182 			mutex_exit(&fp->f_tlock);
183 			error = EMFILE;
184 		} else {
185 			if (cmd == F_DUPFD_CLOEXEC) {
186 				f_setfd(retval, FD_CLOEXEC);
187 			}
188 		}
189 		goto done;
190 
191 	case F_DUP2FD_CLOEXEC:
192 		if (fdes == iarg) {
193 			error = EINVAL;
194 			goto done;
195 		}
196 
197 		/*FALLTHROUGH*/
198 
199 	case F_DUP2FD:
200 		p = curproc;
201 		if (fdes == iarg) {
202 			retval = iarg;
203 		} else if ((uint_t)iarg >= p->p_fno_ctl) {
204 			if (iarg >= 0)
205 				fd_too_big(p);
206 			error = EBADF;
207 		} else {
208 			/*
209 			 * We can't hold our getf(fdes) across the call to
210 			 * closeandsetf() because it creates a window for
211 			 * deadlock: if one thread is doing dup2(a, b) while
212 			 * another is doing dup2(b, a), each one will block
213 			 * waiting for the other to call releasef().  The
214 			 * solution is to increment the file reference count
215 			 * (which we have to do anyway), then releasef(fdes),
216 			 * then closeandsetf().  Incrementing f_count ensures
217 			 * that fp won't disappear after we call releasef().
218 			 * When closeandsetf() fails, we try avoid calling
219 			 * closef() because of all the side effects.
220 			 */
221 			mutex_enter(&fp->f_tlock);
222 			fp->f_count++;
223 			mutex_exit(&fp->f_tlock);
224 			releasef(fdes);
225 			if ((error = closeandsetf(iarg, fp)) == 0) {
226 				if (cmd == F_DUP2FD_CLOEXEC) {
227 					f_setfd(iarg, FD_CLOEXEC);
228 				}
229 				retval = iarg;
230 			} else {
231 				mutex_enter(&fp->f_tlock);
232 				if (fp->f_count > 1) {
233 					fp->f_count--;
234 					mutex_exit(&fp->f_tlock);
235 				} else {
236 					mutex_exit(&fp->f_tlock);
237 					(void) closef(fp);
238 				}
239 			}
240 			goto out;
241 		}
242 		goto done;
243 
244 	case F_SETFL:
245 		vp = fp->f_vnode;
246 		flag = fp->f_flag;
247 		if ((iarg & (FNONBLOCK|FNDELAY)) == (FNONBLOCK|FNDELAY))
248 			iarg &= ~FNDELAY;
249 		if ((error = VOP_SETFL(vp, flag, iarg, fp->f_cred, NULL)) ==
250 		    0) {
251 			iarg &= FMASK;
252 			mutex_enter(&fp->f_tlock);
253 			fp->f_flag &= ~FMASK | (FREAD|FWRITE);
254 			fp->f_flag |= (iarg - FOPEN) & ~(FREAD|FWRITE);
255 			mutex_exit(&fp->f_tlock);
256 		}
257 		retval = 0;
258 		goto done;
259 	}
260 
261 	/*
262 	 * Finally, deal with the expensive cases.
263 	 */
264 	retval = 0;
265 	in_crit = 0;
266 	maxoffset = MAXOFF_T;
267 	datamodel = DATAMODEL_NATIVE;
268 #if defined(_SYSCALL32_IMPL)
269 	if ((datamodel = get_udatamodel()) == DATAMODEL_ILP32)
270 		maxoffset = MAXOFF32_T;
271 #endif
272 
273 	vp = fp->f_vnode;
274 	flag = fp->f_flag;
275 	offset = fp->f_offset;
276 
277 	switch (cmd) {
278 /* ONC_PLUS EXTRACT START */
279 	/*
280 	 * The file system and vnode layers understand and implement
281 	 * locking with flock64 structures. So here once we pass through
282 	 * the test for compatibility as defined by LFS API, (for F_SETLK,
283 	 * F_SETLKW, F_GETLK, F_GETLKW, F_FREESP) we transform
284 	 * the flock structure to a flock64 structure and send it to the
285 	 * lower layers. Similarly in case of GETLK the returned flock64
286 	 * structure is transformed to a flock structure if everything fits
287 	 * in nicely, otherwise we return EOVERFLOW.
288 	 */
289 
290 	case F_GETLK:
291 	case F_O_GETLK:
292 	case F_SETLK:
293 	case F_SETLKW:
294 	case F_SETLK_NBMAND:
295 
296 		/*
297 		 * Copy in input fields only.
298 		 */
299 
300 		if (cmd == F_O_GETLK) {
301 			if (datamodel != DATAMODEL_ILP32) {
302 				error = EINVAL;
303 				break;
304 			}
305 
306 			if (copyin((void *)arg, &obf, sizeof (obf))) {
307 				error = EFAULT;
308 				break;
309 			}
310 			bf.l_type = obf.l_type;
311 			bf.l_whence = obf.l_whence;
312 			bf.l_start = (off64_t)obf.l_start;
313 			bf.l_len = (off64_t)obf.l_len;
314 			bf.l_sysid = (int)obf.l_sysid;
315 			bf.l_pid = obf.l_pid;
316 		} else if (datamodel == DATAMODEL_NATIVE) {
317 			if (copyin((void *)arg, &sbf, sizeof (sbf))) {
318 				error = EFAULT;
319 				break;
320 			}
321 			/*
322 			 * XXX	In an LP64 kernel with an LP64 application
323 			 *	there's no need to do a structure copy here
324 			 *	struct flock == struct flock64. However,
325 			 *	we did it this way to avoid more conditional
326 			 *	compilation.
327 			 */
328 			bf.l_type = sbf.l_type;
329 			bf.l_whence = sbf.l_whence;
330 			bf.l_start = (off64_t)sbf.l_start;
331 			bf.l_len = (off64_t)sbf.l_len;
332 			bf.l_sysid = sbf.l_sysid;
333 			bf.l_pid = sbf.l_pid;
334 		}
335 #if defined(_SYSCALL32_IMPL)
336 		else {
337 			struct flock32 sbf32;
338 			if (copyin((void *)arg, &sbf32, sizeof (sbf32))) {
339 				error = EFAULT;
340 				break;
341 			}
342 			bf.l_type = sbf32.l_type;
343 			bf.l_whence = sbf32.l_whence;
344 			bf.l_start = (off64_t)sbf32.l_start;
345 			bf.l_len = (off64_t)sbf32.l_len;
346 			bf.l_sysid = sbf32.l_sysid;
347 			bf.l_pid = sbf32.l_pid;
348 		}
349 #endif /* _SYSCALL32_IMPL */
350 
351 		/*
352 		 * 64-bit support: check for overflow for 32-bit lock ops
353 		 */
354 		if ((error = flock_check(vp, &bf, offset, maxoffset)) != 0)
355 			break;
356 
357 		/*
358 		 * Not all of the filesystems understand F_O_GETLK, and
359 		 * there's no need for them to know.  Map it to F_GETLK.
360 		 */
361 		if ((error = VOP_FRLOCK(vp, (cmd == F_O_GETLK) ? F_GETLK : cmd,
362 		    &bf, flag, offset, NULL, fp->f_cred, NULL)) != 0)
363 			break;
364 
365 		/*
366 		 * If command is GETLK and no lock is found, only
367 		 * the type field is changed.
368 		 */
369 		if ((cmd == F_O_GETLK || cmd == F_GETLK) &&
370 		    bf.l_type == F_UNLCK) {
371 			/* l_type always first entry, always a short */
372 			if (copyout(&bf.l_type, &((struct flock *)arg)->l_type,
373 			    sizeof (bf.l_type)))
374 				error = EFAULT;
375 			break;
376 		}
377 
378 		if (cmd == F_O_GETLK) {
379 			/*
380 			 * Return an SVR3 flock structure to the user.
381 			 */
382 			obf.l_type = (int16_t)bf.l_type;
383 			obf.l_whence = (int16_t)bf.l_whence;
384 			obf.l_start = (int32_t)bf.l_start;
385 			obf.l_len = (int32_t)bf.l_len;
386 			if (bf.l_sysid > SHRT_MAX || bf.l_pid > SHRT_MAX) {
387 				/*
388 				 * One or both values for the above fields
389 				 * is too large to store in an SVR3 flock
390 				 * structure.
391 				 */
392 				error = EOVERFLOW;
393 				break;
394 			}
395 			obf.l_sysid = (int16_t)bf.l_sysid;
396 			obf.l_pid = (int16_t)bf.l_pid;
397 			if (copyout(&obf, (void *)arg, sizeof (obf)))
398 				error = EFAULT;
399 		} else if (cmd == F_GETLK) {
400 			/*
401 			 * Copy out SVR4 flock.
402 			 */
403 			int i;
404 
405 			if (bf.l_start > maxoffset || bf.l_len > maxoffset) {
406 				error = EOVERFLOW;
407 				break;
408 			}
409 
410 			if (datamodel == DATAMODEL_NATIVE) {
411 				for (i = 0; i < 4; i++)
412 					sbf.l_pad[i] = 0;
413 				/*
414 				 * XXX	In an LP64 kernel with an LP64
415 				 *	application there's no need to do a
416 				 *	structure copy here as currently
417 				 *	struct flock == struct flock64.
418 				 *	We did it this way to avoid more
419 				 *	conditional compilation.
420 				 */
421 				sbf.l_type = bf.l_type;
422 				sbf.l_whence = bf.l_whence;
423 				sbf.l_start = (off_t)bf.l_start;
424 				sbf.l_len = (off_t)bf.l_len;
425 				sbf.l_sysid = bf.l_sysid;
426 				sbf.l_pid = bf.l_pid;
427 				if (copyout(&sbf, (void *)arg, sizeof (sbf)))
428 					error = EFAULT;
429 			}
430 #if defined(_SYSCALL32_IMPL)
431 			else {
432 				struct flock32 sbf32;
433 				if (bf.l_start > MAXOFF32_T ||
434 				    bf.l_len > MAXOFF32_T) {
435 					error = EOVERFLOW;
436 					break;
437 				}
438 				for (i = 0; i < 4; i++)
439 					sbf32.l_pad[i] = 0;
440 				sbf32.l_type = (int16_t)bf.l_type;
441 				sbf32.l_whence = (int16_t)bf.l_whence;
442 				sbf32.l_start = (off32_t)bf.l_start;
443 				sbf32.l_len = (off32_t)bf.l_len;
444 				sbf32.l_sysid = (int32_t)bf.l_sysid;
445 				sbf32.l_pid = (pid32_t)bf.l_pid;
446 				if (copyout(&sbf32,
447 				    (void *)arg, sizeof (sbf32)))
448 					error = EFAULT;
449 			}
450 #endif
451 		}
452 		break;
453 /* ONC_PLUS EXTRACT END */
454 
455 	case F_CHKFL:
456 		/*
457 		 * This is for internal use only, to allow the vnode layer
458 		 * to validate a flags setting before applying it.  User
459 		 * programs can't issue it.
460 		 */
461 		error = EINVAL;
462 		break;
463 
464 	case F_ALLOCSP:
465 	case F_FREESP:
466 	case F_ALLOCSP64:
467 	case F_FREESP64:
468 		/*
469 		 * Test for not-a-regular-file (and returning EINVAL)
470 		 * before testing for open-for-writing (and returning EBADF).
471 		 * This is relied upon by posix_fallocate() in libc.
472 		 */
473 		if (vp->v_type != VREG) {
474 			error = EINVAL;
475 			break;
476 		}
477 
478 		if ((flag & FWRITE) == 0) {
479 			error = EBADF;
480 			break;
481 		}
482 
483 		if (datamodel != DATAMODEL_ILP32 &&
484 		    (cmd == F_ALLOCSP64 || cmd == F_FREESP64)) {
485 			error = EINVAL;
486 			break;
487 		}
488 
489 #if defined(_ILP32) || defined(_SYSCALL32_IMPL)
490 		if (datamodel == DATAMODEL_ILP32 &&
491 		    (cmd == F_ALLOCSP || cmd == F_FREESP)) {
492 			struct flock32 sbf32;
493 			/*
494 			 * For compatibility we overlay an SVR3 flock on an SVR4
495 			 * flock.  This works because the input field offsets
496 			 * in "struct flock" were preserved.
497 			 */
498 			if (copyin((void *)arg, &sbf32, sizeof (sbf32))) {
499 				error = EFAULT;
500 				break;
501 			} else {
502 				bf.l_type = sbf32.l_type;
503 				bf.l_whence = sbf32.l_whence;
504 				bf.l_start = (off64_t)sbf32.l_start;
505 				bf.l_len = (off64_t)sbf32.l_len;
506 				bf.l_sysid = sbf32.l_sysid;
507 				bf.l_pid = sbf32.l_pid;
508 			}
509 		}
510 #endif /* _ILP32 || _SYSCALL32_IMPL */
511 
512 #if defined(_LP64)
513 		if (datamodel == DATAMODEL_LP64 &&
514 		    (cmd == F_ALLOCSP || cmd == F_FREESP)) {
515 			if (copyin((void *)arg, &bf, sizeof (bf))) {
516 				error = EFAULT;
517 				break;
518 			}
519 		}
520 #endif /* defined(_LP64) */
521 
522 #if !defined(_LP64) || defined(_SYSCALL32_IMPL)
523 		if (datamodel == DATAMODEL_ILP32 &&
524 		    (cmd == F_ALLOCSP64 || cmd == F_FREESP64)) {
525 			if (copyin((void *)arg, &bf64_32, sizeof (bf64_32))) {
526 				error = EFAULT;
527 				break;
528 			} else {
529 				/*
530 				 * Note that the size of flock64 is different in
531 				 * the ILP32 and LP64 models, due to the l_pad
532 				 * field. We do not want to assume that the
533 				 * flock64 structure is laid out the same in
534 				 * ILP32 and LP64 environments, so we will
535 				 * copy in the ILP32 version of flock64
536 				 * explicitly and copy it to the native
537 				 * flock64 structure.
538 				 */
539 				bf.l_type = (short)bf64_32.l_type;
540 				bf.l_whence = (short)bf64_32.l_whence;
541 				bf.l_start = bf64_32.l_start;
542 				bf.l_len = bf64_32.l_len;
543 				bf.l_sysid = (int)bf64_32.l_sysid;
544 				bf.l_pid = (pid_t)bf64_32.l_pid;
545 			}
546 		}
547 #endif /* !defined(_LP64) || defined(_SYSCALL32_IMPL) */
548 
549 		if (cmd == F_ALLOCSP || cmd == F_FREESP)
550 			error = flock_check(vp, &bf, offset, maxoffset);
551 		else if (cmd == F_ALLOCSP64 || cmd == F_FREESP64)
552 			error = flock_check(vp, &bf, offset, MAXOFFSET_T);
553 		if (error)
554 			break;
555 
556 		if (vp->v_type == VREG && bf.l_len == 0 &&
557 		    bf.l_start > OFFSET_MAX(fp)) {
558 			error = EFBIG;
559 			break;
560 		}
561 
562 		/*
563 		 * Make sure that there are no conflicting non-blocking
564 		 * mandatory locks in the region being manipulated. If
565 		 * there are such locks then return EACCES.
566 		 */
567 		if ((error = flock_get_start(vp, &bf, offset, &start)) != 0)
568 			break;
569 
570 		if (nbl_need_check(vp)) {
571 			u_offset_t	begin;
572 			ssize_t		length;
573 
574 			nbl_start_crit(vp, RW_READER);
575 			in_crit = 1;
576 			vattr.va_mask = AT_SIZE;
577 			if ((error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))
578 			    != 0)
579 				break;
580 			begin = start > vattr.va_size ? vattr.va_size : start;
581 			length = vattr.va_size > start ? vattr.va_size - start :
582 			    start - vattr.va_size;
583 			if (nbl_conflict(vp, NBL_WRITE, begin, length, 0,
584 			    NULL)) {
585 				error = EACCES;
586 				break;
587 			}
588 		}
589 
590 		if (cmd == F_ALLOCSP64)
591 			cmd = F_ALLOCSP;
592 		else if (cmd == F_FREESP64)
593 			cmd = F_FREESP;
594 
595 		error = VOP_SPACE(vp, cmd, &bf, flag, offset, fp->f_cred, NULL);
596 
597 		break;
598 
599 #if !defined(_LP64) || defined(_SYSCALL32_IMPL)
600 /* ONC_PLUS EXTRACT START */
601 	case F_GETLK64:
602 	case F_SETLK64:
603 	case F_SETLKW64:
604 	case F_SETLK64_NBMAND:
605 		/*
606 		 * Large Files: Here we set cmd as *LK and send it to
607 		 * lower layers. *LK64 is only for the user land.
608 		 * Most of the comments described above for F_SETLK
609 		 * applies here too.
610 		 * Large File support is only needed for ILP32 apps!
611 		 */
612 		if (datamodel != DATAMODEL_ILP32) {
613 			error = EINVAL;
614 			break;
615 		}
616 
617 		if (cmd == F_GETLK64)
618 			cmd = F_GETLK;
619 		else if (cmd == F_SETLK64)
620 			cmd = F_SETLK;
621 		else if (cmd == F_SETLKW64)
622 			cmd = F_SETLKW;
623 		else if (cmd == F_SETLK64_NBMAND)
624 			cmd = F_SETLK_NBMAND;
625 
626 		/*
627 		 * Note that the size of flock64 is different in the ILP32
628 		 * and LP64 models, due to the sucking l_pad field.
629 		 * We do not want to assume that the flock64 structure is
630 		 * laid out in the same in ILP32 and LP64 environments, so
631 		 * we will copy in the ILP32 version of flock64 explicitly
632 		 * and copy it to the native flock64 structure.
633 		 */
634 
635 		if (copyin((void *)arg, &bf64_32, sizeof (bf64_32))) {
636 			error = EFAULT;
637 			break;
638 		}
639 
640 		bf.l_type = (short)bf64_32.l_type;
641 		bf.l_whence = (short)bf64_32.l_whence;
642 		bf.l_start = bf64_32.l_start;
643 		bf.l_len = bf64_32.l_len;
644 		bf.l_sysid = (int)bf64_32.l_sysid;
645 		bf.l_pid = (pid_t)bf64_32.l_pid;
646 
647 		if ((error = flock_check(vp, &bf, offset, MAXOFFSET_T)) != 0)
648 			break;
649 
650 		if ((error = VOP_FRLOCK(vp, cmd, &bf, flag, offset,
651 		    NULL, fp->f_cred, NULL)) != 0)
652 			break;
653 
654 		if ((cmd == F_GETLK) && bf.l_type == F_UNLCK) {
655 			if (copyout(&bf.l_type, &((struct flock *)arg)->l_type,
656 			    sizeof (bf.l_type)))
657 				error = EFAULT;
658 			break;
659 		}
660 
661 		if (cmd == F_GETLK) {
662 			int i;
663 
664 			/*
665 			 * We do not want to assume that the flock64 structure
666 			 * is laid out in the same in ILP32 and LP64
667 			 * environments, so we will copy out the ILP32 version
668 			 * of flock64 explicitly after copying the native
669 			 * flock64 structure to it.
670 			 */
671 			for (i = 0; i < 4; i++)
672 				bf64_32.l_pad[i] = 0;
673 			bf64_32.l_type = (int16_t)bf.l_type;
674 			bf64_32.l_whence = (int16_t)bf.l_whence;
675 			bf64_32.l_start = bf.l_start;
676 			bf64_32.l_len = bf.l_len;
677 			bf64_32.l_sysid = (int32_t)bf.l_sysid;
678 			bf64_32.l_pid = (pid32_t)bf.l_pid;
679 			if (copyout(&bf64_32, (void *)arg, sizeof (bf64_32)))
680 				error = EFAULT;
681 		}
682 		break;
683 /* ONC_PLUS EXTRACT END */
684 #endif /* !defined(_LP64) || defined(_SYSCALL32_IMPL) */
685 
686 /* ONC_PLUS EXTRACT START */
687 	case F_SHARE:
688 	case F_SHARE_NBMAND:
689 	case F_UNSHARE:
690 
691 		/*
692 		 * Copy in input fields only.
693 		 */
694 		if (copyin((void *)arg, &fsh, sizeof (fsh))) {
695 			error = EFAULT;
696 			break;
697 		}
698 
699 		/*
700 		 * Local share reservations always have this simple form
701 		 */
702 		shr.s_access = fsh.f_access;
703 		shr.s_deny = fsh.f_deny;
704 		shr.s_sysid = 0;
705 		shr.s_pid = ttoproc(curthread)->p_pid;
706 		shr_own.sl_pid = shr.s_pid;
707 		shr_own.sl_id = fsh.f_id;
708 		shr.s_own_len = sizeof (shr_own);
709 		shr.s_owner = (caddr_t)&shr_own;
710 		error = VOP_SHRLOCK(vp, cmd, &shr, flag, fp->f_cred, NULL);
711 /* ONC_PLUS EXTRACT END */
712 		break;
713 
714 	default:
715 		error = EINVAL;
716 		break;
717 	}
718 
719 	if (in_crit)
720 		nbl_end_crit(vp);
721 
722 done:
723 	releasef(fdes);
724 out:
725 	if (error)
726 		return (set_errno(error));
727 	return (retval);
728 }
729 
730 /* ONC_PLUS EXTRACT START */
731 int
732 flock_check(vnode_t *vp, flock64_t *flp, offset_t offset, offset_t max)
733 {
734 	struct vattr	vattr;
735 	int	error;
736 	u_offset_t start, end;
737 
738 	/*
739 	 * Determine the starting point of the request
740 	 */
741 	switch (flp->l_whence) {
742 	case 0:		/* SEEK_SET */
743 		start = (u_offset_t)flp->l_start;
744 		if (start > max)
745 			return (EINVAL);
746 		break;
747 	case 1:		/* SEEK_CUR */
748 		if (flp->l_start > (max - offset))
749 			return (EOVERFLOW);
750 		start = (u_offset_t)(flp->l_start + offset);
751 		if (start > max)
752 			return (EINVAL);
753 		break;
754 	case 2:		/* SEEK_END */
755 		vattr.va_mask = AT_SIZE;
756 		if (error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))
757 			return (error);
758 		if (flp->l_start > (max - (offset_t)vattr.va_size))
759 			return (EOVERFLOW);
760 		start = (u_offset_t)(flp->l_start + (offset_t)vattr.va_size);
761 		if (start > max)
762 			return (EINVAL);
763 		break;
764 	default:
765 		return (EINVAL);
766 	}
767 
768 	/*
769 	 * Determine the range covered by the request.
770 	 */
771 	if (flp->l_len == 0)
772 		end = MAXEND;
773 	else if ((offset_t)flp->l_len > 0) {
774 		if (flp->l_len > (max - start + 1))
775 			return (EOVERFLOW);
776 		end = (u_offset_t)(start + (flp->l_len - 1));
777 		ASSERT(end <= max);
778 	} else {
779 		/*
780 		 * Negative length; why do we even allow this ?
781 		 * Because this allows easy specification of
782 		 * the last n bytes of the file.
783 		 */
784 		end = start;
785 		start += (u_offset_t)flp->l_len;
786 		(start)++;
787 		if (start > max)
788 			return (EINVAL);
789 		ASSERT(end <= max);
790 	}
791 	ASSERT(start <= max);
792 	if (flp->l_type == F_UNLCK && flp->l_len > 0 &&
793 	    end == (offset_t)max) {
794 		flp->l_len = 0;
795 	}
796 	if (start  > end)
797 		return (EINVAL);
798 	return (0);
799 }
800 
801 static int
802 flock_get_start(vnode_t *vp, flock64_t *flp, offset_t offset, u_offset_t *start)
803 {
804 	struct vattr	vattr;
805 	int	error;
806 
807 	/*
808 	 * Determine the starting point of the request. Assume that it is
809 	 * a valid starting point.
810 	 */
811 	switch (flp->l_whence) {
812 	case 0:		/* SEEK_SET */
813 		*start = (u_offset_t)flp->l_start;
814 		break;
815 	case 1:		/* SEEK_CUR */
816 		*start = (u_offset_t)(flp->l_start + offset);
817 		break;
818 	case 2:		/* SEEK_END */
819 		vattr.va_mask = AT_SIZE;
820 		if (error = VOP_GETATTR(vp, &vattr, 0, CRED(), NULL))
821 			return (error);
822 		*start = (u_offset_t)(flp->l_start + (offset_t)vattr.va_size);
823 		break;
824 	default:
825 		return (EINVAL);
826 	}
827 
828 	return (0);
829 }
830 
831 /*
832  * Take rctl action when the requested file descriptor is too big.
833  */
834 static void
835 fd_too_big(proc_t *p)
836 {
837 	mutex_enter(&p->p_lock);
838 	(void) rctl_action(rctlproc_legacy[RLIMIT_NOFILE],
839 	    p->p_rctls, p, RCA_SAFE);
840 	mutex_exit(&p->p_lock);
841 }
842 /* ONC_PLUS EXTRACT END */
843