1 /* $NetBSD: sys_aio.c,v 1.48 2020/05/23 23:42:43 ad Exp $ */
2
3 /*
4 * Copyright (c) 2007 Mindaugas Rasiukevicius <rmind at NetBSD org>
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions
9 * are met:
10 * 1. Redistributions of source code must retain the above copyright
11 * notice, this list of conditions and the following disclaimer.
12 * 2. Redistributions in binary form must reproduce the above copyright
13 * notice, this list of conditions and the following disclaimer in the
14 * documentation and/or other materials provided with the distribution.
15 *
16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26 * SUCH DAMAGE.
27 */
28
29 /*
30 * Implementation of POSIX asynchronous I/O.
31 * Defined in the Base Definitions volume of IEEE Std 1003.1-2001.
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: sys_aio.c,v 1.48 2020/05/23 23:42:43 ad Exp $");
36
37 #ifdef _KERNEL_OPT
38 #include "opt_ddb.h"
39 #endif
40
41 #include <sys/param.h>
42 #include <sys/condvar.h>
43 #include <sys/file.h>
44 #include <sys/filedesc.h>
45 #include <sys/kernel.h>
46 #include <sys/kmem.h>
47 #include <sys/lwp.h>
48 #include <sys/mutex.h>
49 #include <sys/pool.h>
50 #include <sys/proc.h>
51 #include <sys/queue.h>
52 #include <sys/signal.h>
53 #include <sys/signalvar.h>
54 #include <sys/syscall.h>
55 #include <sys/syscallargs.h>
56 #include <sys/syscallvar.h>
57 #include <sys/sysctl.h>
58 #include <sys/systm.h>
59 #include <sys/types.h>
60 #include <sys/vnode.h>
61 #include <sys/atomic.h>
62 #include <sys/module.h>
63 #include <sys/buf.h>
64
65 #include <uvm/uvm_extern.h>
66
67 MODULE(MODULE_CLASS_MISC, aio, NULL);
68
69 /*
70 * System-wide limits and counter of AIO operations.
71 */
72 u_int aio_listio_max = AIO_LISTIO_MAX;
73 static u_int aio_max = AIO_MAX;
74 static u_int aio_jobs_count;
75
76 static struct pool aio_job_pool;
77 static struct pool aio_lio_pool;
78 static void * aio_ehook;
79
80 static void aio_worker(void *);
81 static void aio_process(struct aio_job *);
82 static void aio_sendsig(struct proc *, struct sigevent *);
83 static int aio_enqueue_job(int, void *, struct lio_req *);
84 static void aio_exit(proc_t *, void *);
85
86 static int sysctl_aio_listio_max(SYSCTLFN_PROTO);
87 static int sysctl_aio_max(SYSCTLFN_PROTO);
88
89 static const struct syscall_package aio_syscalls[] = {
90 { SYS_aio_cancel, 0, (sy_call_t *)sys_aio_cancel },
91 { SYS_aio_error, 0, (sy_call_t *)sys_aio_error },
92 { SYS_aio_fsync, 0, (sy_call_t *)sys_aio_fsync },
93 { SYS_aio_read, 0, (sy_call_t *)sys_aio_read },
94 { SYS_aio_return, 0, (sy_call_t *)sys_aio_return },
95 { SYS___aio_suspend50, 0, (sy_call_t *)sys___aio_suspend50 },
96 { SYS_aio_write, 0, (sy_call_t *)sys_aio_write },
97 { SYS_lio_listio, 0, (sy_call_t *)sys_lio_listio },
98 { 0, 0, NULL },
99 };
100
101 /*
102 * Tear down all AIO state.
103 */
104 static int
aio_fini(bool interface)105 aio_fini(bool interface)
106 {
107 int error;
108 proc_t *p;
109
110 if (interface) {
111 /* Stop syscall activity. */
112 error = syscall_disestablish(NULL, aio_syscalls);
113 if (error != 0)
114 return error;
115 /* Abort if any processes are using AIO. */
116 mutex_enter(&proc_lock);
117 PROCLIST_FOREACH(p, &allproc) {
118 if (p->p_aio != NULL)
119 break;
120 }
121 mutex_exit(&proc_lock);
122 if (p != NULL) {
123 error = syscall_establish(NULL, aio_syscalls);
124 KASSERT(error == 0);
125 return EBUSY;
126 }
127 }
128
129 KASSERT(aio_jobs_count == 0);
130 exithook_disestablish(aio_ehook);
131 pool_destroy(&aio_job_pool);
132 pool_destroy(&aio_lio_pool);
133 return 0;
134 }
135
136 /*
137 * Initialize global AIO state.
138 */
139 static int
aio_init(void)140 aio_init(void)
141 {
142 int error;
143
144 pool_init(&aio_job_pool, sizeof(struct aio_job), 0, 0, 0,
145 "aio_jobs_pool", &pool_allocator_nointr, IPL_NONE);
146 pool_init(&aio_lio_pool, sizeof(struct lio_req), 0, 0, 0,
147 "aio_lio_pool", &pool_allocator_nointr, IPL_NONE);
148 aio_ehook = exithook_establish(aio_exit, NULL);
149
150 error = syscall_establish(NULL, aio_syscalls);
151 if (error != 0)
152 (void)aio_fini(false);
153 return error;
154 }
155
156 /*
157 * Module interface.
158 */
159 static int
aio_modcmd(modcmd_t cmd,void * arg)160 aio_modcmd(modcmd_t cmd, void *arg)
161 {
162
163 switch (cmd) {
164 case MODULE_CMD_INIT:
165 return aio_init();
166 case MODULE_CMD_FINI:
167 return aio_fini(true);
168 default:
169 return ENOTTY;
170 }
171 }
172
173 /*
174 * Initialize Asynchronous I/O data structures for the process.
175 */
176 static int
aio_procinit(struct proc * p)177 aio_procinit(struct proc *p)
178 {
179 struct aioproc *aio;
180 struct lwp *l;
181 int error;
182 vaddr_t uaddr;
183
184 /* Allocate and initialize AIO structure */
185 aio = kmem_zalloc(sizeof(struct aioproc), KM_SLEEP);
186
187 /* Initialize queue and their synchronization structures */
188 mutex_init(&aio->aio_mtx, MUTEX_DEFAULT, IPL_NONE);
189 cv_init(&aio->aio_worker_cv, "aiowork");
190 cv_init(&aio->done_cv, "aiodone");
191 TAILQ_INIT(&aio->jobs_queue);
192
193 /*
194 * Create an AIO worker thread.
195 * XXX: Currently, AIO thread is not protected against user's actions.
196 */
197 uaddr = uvm_uarea_alloc();
198 if (uaddr == 0) {
199 aio_exit(p, aio);
200 return EAGAIN;
201 }
202 error = lwp_create(curlwp, p, uaddr, 0, NULL, 0, aio_worker,
203 NULL, &l, curlwp->l_class, &curlwp->l_sigmask, &curlwp->l_sigstk);
204 if (error != 0) {
205 uvm_uarea_free(uaddr);
206 aio_exit(p, aio);
207 return error;
208 }
209
210 /* Recheck if we are really first */
211 mutex_enter(p->p_lock);
212 if (p->p_aio) {
213 mutex_exit(p->p_lock);
214 aio_exit(p, aio);
215 lwp_exit(l);
216 return 0;
217 }
218 p->p_aio = aio;
219
220 /* Complete the initialization of thread, and run it */
221 aio->aio_worker = l;
222 lwp_lock(l);
223 lwp_changepri(l, MAXPRI_USER);
224 setrunnable(l);
225 /* LWP now unlocked */
226 mutex_exit(p->p_lock);
227
228 return 0;
229 }
230
231 /*
232 * Exit of Asynchronous I/O subsystem of process.
233 */
234 static void
aio_exit(struct proc * p,void * cookie)235 aio_exit(struct proc *p, void *cookie)
236 {
237 struct aio_job *a_job;
238 struct aioproc *aio;
239
240 if (cookie != NULL)
241 aio = cookie;
242 else if ((aio = p->p_aio) == NULL)
243 return;
244
245 /* Free AIO queue */
246 while (!TAILQ_EMPTY(&aio->jobs_queue)) {
247 a_job = TAILQ_FIRST(&aio->jobs_queue);
248 TAILQ_REMOVE(&aio->jobs_queue, a_job, list);
249 pool_put(&aio_job_pool, a_job);
250 atomic_dec_uint(&aio_jobs_count);
251 }
252
253 /* Destroy and free the entire AIO data structure */
254 cv_destroy(&aio->aio_worker_cv);
255 cv_destroy(&aio->done_cv);
256 mutex_destroy(&aio->aio_mtx);
257 kmem_free(aio, sizeof(struct aioproc));
258 }
259
260 /*
261 * AIO worker thread and processor.
262 */
263 static void
aio_worker(void * arg)264 aio_worker(void *arg)
265 {
266 struct proc *p = curlwp->l_proc;
267 struct aioproc *aio = p->p_aio;
268 struct aio_job *a_job;
269 struct lio_req *lio;
270 sigset_t oss, nss;
271 int error __diagused, refcnt;
272
273 /*
274 * Make an empty signal mask, so it
275 * handles only SIGKILL and SIGSTOP.
276 */
277 sigfillset(&nss);
278 mutex_enter(p->p_lock);
279 error = sigprocmask1(curlwp, SIG_SETMASK, &nss, &oss);
280 mutex_exit(p->p_lock);
281 KASSERT(error == 0);
282
283 for (;;) {
284 /*
285 * Loop for each job in the queue. If there
286 * are no jobs then sleep.
287 */
288 mutex_enter(&aio->aio_mtx);
289 while ((a_job = TAILQ_FIRST(&aio->jobs_queue)) == NULL) {
290 if (cv_wait_sig(&aio->aio_worker_cv, &aio->aio_mtx)) {
291 /*
292 * Thread was interrupted - check for
293 * pending exit or suspend.
294 */
295 mutex_exit(&aio->aio_mtx);
296 lwp_userret(curlwp);
297 mutex_enter(&aio->aio_mtx);
298 }
299 }
300
301 /* Take the job from the queue */
302 aio->curjob = a_job;
303 TAILQ_REMOVE(&aio->jobs_queue, a_job, list);
304
305 atomic_dec_uint(&aio_jobs_count);
306 aio->jobs_count--;
307
308 mutex_exit(&aio->aio_mtx);
309
310 /* Process an AIO operation */
311 aio_process(a_job);
312
313 /* Copy data structure back to the user-space */
314 (void)copyout(&a_job->aiocbp, a_job->aiocb_uptr,
315 sizeof(struct aiocb));
316
317 mutex_enter(&aio->aio_mtx);
318 KASSERT(aio->curjob == a_job);
319 aio->curjob = NULL;
320
321 /* Decrease a reference counter, if there is a LIO structure */
322 lio = a_job->lio;
323 refcnt = (lio != NULL ? --lio->refcnt : -1);
324
325 /* Notify all suspenders */
326 cv_broadcast(&aio->done_cv);
327 mutex_exit(&aio->aio_mtx);
328
329 /* Send a signal, if any */
330 aio_sendsig(p, &a_job->aiocbp.aio_sigevent);
331
332 /* Destroy the LIO structure */
333 if (refcnt == 0) {
334 aio_sendsig(p, &lio->sig);
335 pool_put(&aio_lio_pool, lio);
336 }
337
338 /* Destroy the job */
339 pool_put(&aio_job_pool, a_job);
340 }
341
342 /* NOTREACHED */
343 }
344
345 static void
aio_process(struct aio_job * a_job)346 aio_process(struct aio_job *a_job)
347 {
348 struct proc *p = curlwp->l_proc;
349 struct aiocb *aiocbp = &a_job->aiocbp;
350 struct file *fp;
351 int fd = aiocbp->aio_fildes;
352 int error = 0;
353
354 KASSERT(a_job->aio_op != 0);
355
356 if ((a_job->aio_op & (AIO_READ | AIO_WRITE)) != 0) {
357 struct iovec aiov;
358 struct uio auio;
359
360 if (aiocbp->aio_nbytes > SSIZE_MAX) {
361 error = EINVAL;
362 goto done;
363 }
364
365 fp = fd_getfile(fd);
366 if (fp == NULL) {
367 error = EBADF;
368 goto done;
369 }
370
371 aiov.iov_base = (void *)(uintptr_t)aiocbp->aio_buf;
372 aiov.iov_len = aiocbp->aio_nbytes;
373 auio.uio_iov = &aiov;
374 auio.uio_iovcnt = 1;
375 auio.uio_resid = aiocbp->aio_nbytes;
376 auio.uio_vmspace = p->p_vmspace;
377
378 if (a_job->aio_op & AIO_READ) {
379 /*
380 * Perform a Read operation
381 */
382 KASSERT((a_job->aio_op & AIO_WRITE) == 0);
383
384 if ((fp->f_flag & FREAD) == 0) {
385 fd_putfile(fd);
386 error = EBADF;
387 goto done;
388 }
389 auio.uio_rw = UIO_READ;
390 error = (*fp->f_ops->fo_read)(fp, &aiocbp->aio_offset,
391 &auio, fp->f_cred, FOF_UPDATE_OFFSET);
392 } else {
393 /*
394 * Perform a Write operation
395 */
396 KASSERT(a_job->aio_op & AIO_WRITE);
397
398 if ((fp->f_flag & FWRITE) == 0) {
399 fd_putfile(fd);
400 error = EBADF;
401 goto done;
402 }
403 auio.uio_rw = UIO_WRITE;
404 error = (*fp->f_ops->fo_write)(fp, &aiocbp->aio_offset,
405 &auio, fp->f_cred, FOF_UPDATE_OFFSET);
406 }
407 fd_putfile(fd);
408
409 /* Store the result value */
410 a_job->aiocbp.aio_nbytes -= auio.uio_resid;
411 a_job->aiocbp._retval = (error == 0) ?
412 a_job->aiocbp.aio_nbytes : -1;
413
414 } else if ((a_job->aio_op & (AIO_SYNC | AIO_DSYNC)) != 0) {
415 /*
416 * Perform a file Sync operation
417 */
418 struct vnode *vp;
419
420 if ((error = fd_getvnode(fd, &fp)) != 0)
421 goto done;
422
423 if ((fp->f_flag & FWRITE) == 0) {
424 fd_putfile(fd);
425 error = EBADF;
426 goto done;
427 }
428
429 vp = fp->f_vnode;
430 vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
431 if (a_job->aio_op & AIO_DSYNC) {
432 error = VOP_FSYNC(vp, fp->f_cred,
433 FSYNC_WAIT | FSYNC_DATAONLY, 0, 0);
434 } else if (a_job->aio_op & AIO_SYNC) {
435 error = VOP_FSYNC(vp, fp->f_cred,
436 FSYNC_WAIT, 0, 0);
437 }
438 VOP_UNLOCK(vp);
439 fd_putfile(fd);
440
441 /* Store the result value */
442 a_job->aiocbp._retval = (error == 0) ? 0 : -1;
443
444 } else
445 panic("aio_process: invalid operation code\n");
446
447 done:
448 /* Job is done, set the error, if any */
449 a_job->aiocbp._errno = error;
450 a_job->aiocbp._state = JOB_DONE;
451 }
452
453 /*
454 * Send AIO signal.
455 */
456 static void
aio_sendsig(struct proc * p,struct sigevent * sig)457 aio_sendsig(struct proc *p, struct sigevent *sig)
458 {
459 ksiginfo_t ksi;
460
461 if (sig->sigev_signo == 0 || sig->sigev_notify == SIGEV_NONE)
462 return;
463
464 KSI_INIT(&ksi);
465 ksi.ksi_signo = sig->sigev_signo;
466 ksi.ksi_code = SI_ASYNCIO;
467 ksi.ksi_value = sig->sigev_value;
468 mutex_enter(&proc_lock);
469 kpsignal(p, &ksi, NULL);
470 mutex_exit(&proc_lock);
471 }
472
473 /*
474 * Enqueue the job.
475 */
476 static int
aio_enqueue_job(int op,void * aiocb_uptr,struct lio_req * lio)477 aio_enqueue_job(int op, void *aiocb_uptr, struct lio_req *lio)
478 {
479 struct proc *p = curlwp->l_proc;
480 struct aioproc *aio;
481 struct aio_job *a_job;
482 struct aiocb aiocbp;
483 struct sigevent *sig;
484 int error;
485
486 /* Non-accurate check for the limit */
487 if (aio_jobs_count + 1 > aio_max)
488 return EAGAIN;
489
490 /* Get the data structure from user-space */
491 error = copyin(aiocb_uptr, &aiocbp, sizeof(struct aiocb));
492 if (error)
493 return error;
494
495 /* Check if signal is set, and validate it */
496 sig = &aiocbp.aio_sigevent;
497 if (sig->sigev_signo < 0 || sig->sigev_signo >= NSIG ||
498 sig->sigev_notify < SIGEV_NONE || sig->sigev_notify > SIGEV_SA)
499 return EINVAL;
500
501 /* Buffer and byte count */
502 if (((AIO_SYNC | AIO_DSYNC) & op) == 0)
503 if (aiocbp.aio_buf == NULL || aiocbp.aio_nbytes > SSIZE_MAX)
504 return EINVAL;
505
506 /* Check the opcode, if LIO_NOP - simply ignore */
507 if (op == AIO_LIO) {
508 KASSERT(lio != NULL);
509 if (aiocbp.aio_lio_opcode == LIO_WRITE)
510 op = AIO_WRITE;
511 else if (aiocbp.aio_lio_opcode == LIO_READ)
512 op = AIO_READ;
513 else
514 return (aiocbp.aio_lio_opcode == LIO_NOP) ? 0 : EINVAL;
515 } else {
516 KASSERT(lio == NULL);
517 }
518
519 /*
520 * Look for already existing job. If found - the job is in-progress.
521 * According to POSIX this is invalid, so return the error.
522 */
523 aio = p->p_aio;
524 if (aio) {
525 mutex_enter(&aio->aio_mtx);
526 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) {
527 if (a_job->aiocb_uptr != aiocb_uptr)
528 continue;
529 mutex_exit(&aio->aio_mtx);
530 return EINVAL;
531 }
532 mutex_exit(&aio->aio_mtx);
533 }
534
535 /*
536 * Check if AIO structure is initialized, if not - initialize it.
537 * In LIO case, we did that already. We will recheck this with
538 * the lock in aio_procinit().
539 */
540 if (lio == NULL && p->p_aio == NULL)
541 if (aio_procinit(p))
542 return EAGAIN;
543 aio = p->p_aio;
544
545 /*
546 * Set the state with errno, and copy data
547 * structure back to the user-space.
548 */
549 aiocbp._state = JOB_WIP;
550 aiocbp._errno = EINPROGRESS;
551 aiocbp._retval = -1;
552 error = copyout(&aiocbp, aiocb_uptr, sizeof(struct aiocb));
553 if (error)
554 return error;
555
556 /* Allocate and initialize a new AIO job */
557 a_job = pool_get(&aio_job_pool, PR_WAITOK | PR_ZERO);
558
559 /*
560 * Set the data.
561 * Store the user-space pointer for searching. Since we
562 * are storing only per proc pointers - it is safe.
563 */
564 memcpy(&a_job->aiocbp, &aiocbp, sizeof(struct aiocb));
565 a_job->aiocb_uptr = aiocb_uptr;
566 a_job->aio_op |= op;
567 a_job->lio = lio;
568
569 /*
570 * Add the job to the queue, update the counters, and
571 * notify the AIO worker thread to handle the job.
572 */
573 mutex_enter(&aio->aio_mtx);
574
575 /* Fail, if the limit was reached */
576 if (atomic_inc_uint_nv(&aio_jobs_count) > aio_max ||
577 aio->jobs_count >= aio_listio_max) {
578 atomic_dec_uint(&aio_jobs_count);
579 mutex_exit(&aio->aio_mtx);
580 pool_put(&aio_job_pool, a_job);
581 return EAGAIN;
582 }
583
584 TAILQ_INSERT_TAIL(&aio->jobs_queue, a_job, list);
585 aio->jobs_count++;
586 if (lio)
587 lio->refcnt++;
588 cv_signal(&aio->aio_worker_cv);
589
590 mutex_exit(&aio->aio_mtx);
591
592 /*
593 * One would handle the errors only with aio_error() function.
594 * This way is appropriate according to POSIX.
595 */
596 return 0;
597 }
598
599 /*
600 * Syscall functions.
601 */
602
603 int
sys_aio_cancel(struct lwp * l,const struct sys_aio_cancel_args * uap,register_t * retval)604 sys_aio_cancel(struct lwp *l, const struct sys_aio_cancel_args *uap,
605 register_t *retval)
606 {
607 /* {
608 syscallarg(int) fildes;
609 syscallarg(struct aiocb *) aiocbp;
610 } */
611 struct proc *p = l->l_proc;
612 struct aioproc *aio;
613 struct aio_job *a_job;
614 struct aiocb *aiocbp_ptr;
615 struct lio_req *lio;
616 struct filedesc *fdp = p->p_fd;
617 unsigned int cn, errcnt, fildes;
618 fdtab_t *dt;
619
620 TAILQ_HEAD(, aio_job) tmp_jobs_list;
621
622 /* Check for invalid file descriptor */
623 fildes = (unsigned int)SCARG(uap, fildes);
624 dt = atomic_load_consume(&fdp->fd_dt);
625 if (fildes >= dt->dt_nfiles)
626 return EBADF;
627 if (dt->dt_ff[fildes] == NULL || dt->dt_ff[fildes]->ff_file == NULL)
628 return EBADF;
629
630 /* Check if AIO structure is initialized */
631 if (p->p_aio == NULL) {
632 *retval = AIO_NOTCANCELED;
633 return 0;
634 }
635
636 aio = p->p_aio;
637 aiocbp_ptr = (struct aiocb *)SCARG(uap, aiocbp);
638
639 mutex_enter(&aio->aio_mtx);
640
641 /* Cancel the jobs, and remove them from the queue */
642 cn = 0;
643 TAILQ_INIT(&tmp_jobs_list);
644 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) {
645 if (aiocbp_ptr) {
646 if (aiocbp_ptr != a_job->aiocb_uptr)
647 continue;
648 if (fildes != a_job->aiocbp.aio_fildes) {
649 mutex_exit(&aio->aio_mtx);
650 return EBADF;
651 }
652 } else if (a_job->aiocbp.aio_fildes != fildes)
653 continue;
654
655 TAILQ_REMOVE(&aio->jobs_queue, a_job, list);
656 TAILQ_INSERT_TAIL(&tmp_jobs_list, a_job, list);
657
658 /* Decrease the counters */
659 atomic_dec_uint(&aio_jobs_count);
660 aio->jobs_count--;
661 lio = a_job->lio;
662 if (lio != NULL && --lio->refcnt != 0)
663 a_job->lio = NULL;
664
665 cn++;
666 if (aiocbp_ptr)
667 break;
668 }
669
670 /* There are canceled jobs */
671 if (cn)
672 *retval = AIO_CANCELED;
673
674 /* We cannot cancel current job */
675 a_job = aio->curjob;
676 if (a_job && ((a_job->aiocbp.aio_fildes == fildes) ||
677 (a_job->aiocb_uptr == aiocbp_ptr)))
678 *retval = AIO_NOTCANCELED;
679
680 mutex_exit(&aio->aio_mtx);
681
682 /* Free the jobs after the lock */
683 errcnt = 0;
684 while (!TAILQ_EMPTY(&tmp_jobs_list)) {
685 a_job = TAILQ_FIRST(&tmp_jobs_list);
686 TAILQ_REMOVE(&tmp_jobs_list, a_job, list);
687 /* Set the errno and copy structures back to the user-space */
688 a_job->aiocbp._errno = ECANCELED;
689 a_job->aiocbp._state = JOB_DONE;
690 if (copyout(&a_job->aiocbp, a_job->aiocb_uptr,
691 sizeof(struct aiocb)))
692 errcnt++;
693 /* Send a signal if any */
694 aio_sendsig(p, &a_job->aiocbp.aio_sigevent);
695 if (a_job->lio) {
696 lio = a_job->lio;
697 aio_sendsig(p, &lio->sig);
698 pool_put(&aio_lio_pool, lio);
699 }
700 pool_put(&aio_job_pool, a_job);
701 }
702
703 if (errcnt)
704 return EFAULT;
705
706 /* Set a correct return value */
707 if (*retval == 0)
708 *retval = AIO_ALLDONE;
709
710 return 0;
711 }
712
713 int
sys_aio_error(struct lwp * l,const struct sys_aio_error_args * uap,register_t * retval)714 sys_aio_error(struct lwp *l, const struct sys_aio_error_args *uap,
715 register_t *retval)
716 {
717 /* {
718 syscallarg(const struct aiocb *) aiocbp;
719 } */
720 struct proc *p = l->l_proc;
721 struct aioproc *aio = p->p_aio;
722 struct aiocb aiocbp;
723 int error;
724
725 if (aio == NULL)
726 return EINVAL;
727
728 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb));
729 if (error)
730 return error;
731
732 if (aiocbp._state == JOB_NONE)
733 return EINVAL;
734
735 *retval = aiocbp._errno;
736
737 return 0;
738 }
739
740 int
sys_aio_fsync(struct lwp * l,const struct sys_aio_fsync_args * uap,register_t * retval)741 sys_aio_fsync(struct lwp *l, const struct sys_aio_fsync_args *uap,
742 register_t *retval)
743 {
744 /* {
745 syscallarg(int) op;
746 syscallarg(struct aiocb *) aiocbp;
747 } */
748 int op = SCARG(uap, op);
749
750 if ((op != O_DSYNC) && (op != O_SYNC))
751 return EINVAL;
752
753 op = O_DSYNC ? AIO_DSYNC : AIO_SYNC;
754
755 return aio_enqueue_job(op, SCARG(uap, aiocbp), NULL);
756 }
757
758 int
sys_aio_read(struct lwp * l,const struct sys_aio_read_args * uap,register_t * retval)759 sys_aio_read(struct lwp *l, const struct sys_aio_read_args *uap,
760 register_t *retval)
761 {
762 /* {
763 syscallarg(struct aiocb *) aiocbp;
764 } */
765
766 return aio_enqueue_job(AIO_READ, SCARG(uap, aiocbp), NULL);
767 }
768
769 int
sys_aio_return(struct lwp * l,const struct sys_aio_return_args * uap,register_t * retval)770 sys_aio_return(struct lwp *l, const struct sys_aio_return_args *uap,
771 register_t *retval)
772 {
773 /* {
774 syscallarg(struct aiocb *) aiocbp;
775 } */
776 struct proc *p = l->l_proc;
777 struct aioproc *aio = p->p_aio;
778 struct aiocb aiocbp;
779 int error;
780
781 if (aio == NULL)
782 return EINVAL;
783
784 error = copyin(SCARG(uap, aiocbp), &aiocbp, sizeof(struct aiocb));
785 if (error)
786 return error;
787
788 if (aiocbp._errno == EINPROGRESS || aiocbp._state != JOB_DONE)
789 return EINVAL;
790
791 *retval = aiocbp._retval;
792
793 /* Reset the internal variables */
794 aiocbp._errno = 0;
795 aiocbp._retval = -1;
796 aiocbp._state = JOB_NONE;
797 error = copyout(&aiocbp, SCARG(uap, aiocbp), sizeof(struct aiocb));
798
799 return error;
800 }
801
802 int
sys___aio_suspend50(struct lwp * l,const struct sys___aio_suspend50_args * uap,register_t * retval)803 sys___aio_suspend50(struct lwp *l, const struct sys___aio_suspend50_args *uap,
804 register_t *retval)
805 {
806 /* {
807 syscallarg(const struct aiocb *const[]) list;
808 syscallarg(int) nent;
809 syscallarg(const struct timespec *) timeout;
810 } */
811 struct aiocb **list;
812 struct timespec ts;
813 int error, nent;
814
815 nent = SCARG(uap, nent);
816 if (nent <= 0 || nent > aio_listio_max)
817 return EAGAIN;
818
819 if (SCARG(uap, timeout)) {
820 /* Convert timespec to ticks */
821 error = copyin(SCARG(uap, timeout), &ts,
822 sizeof(struct timespec));
823 if (error)
824 return error;
825 }
826
827 list = kmem_alloc(nent * sizeof(*list), KM_SLEEP);
828 error = copyin(SCARG(uap, list), list, nent * sizeof(*list));
829 if (error)
830 goto out;
831 error = aio_suspend1(l, list, nent, SCARG(uap, timeout) ? &ts : NULL);
832 out:
833 kmem_free(list, nent * sizeof(*list));
834 return error;
835 }
836
837 int
aio_suspend1(struct lwp * l,struct aiocb ** aiocbp_list,int nent,struct timespec * ts)838 aio_suspend1(struct lwp *l, struct aiocb **aiocbp_list, int nent,
839 struct timespec *ts)
840 {
841 struct proc *p = l->l_proc;
842 struct aioproc *aio;
843 struct aio_job *a_job;
844 int i, error, timo;
845
846 if (p->p_aio == NULL)
847 return EAGAIN;
848 aio = p->p_aio;
849
850 if (ts) {
851 timo = mstohz((ts->tv_sec * 1000) + (ts->tv_nsec / 1000000));
852 if (timo == 0 && ts->tv_sec == 0 && ts->tv_nsec > 0)
853 timo = 1;
854 if (timo <= 0)
855 return EAGAIN;
856 } else
857 timo = 0;
858
859 mutex_enter(&aio->aio_mtx);
860 for (;;) {
861 for (i = 0; i < nent; i++) {
862
863 /* Skip NULL entries */
864 if (aiocbp_list[i] == NULL)
865 continue;
866
867 /* Skip current job */
868 if (aio->curjob) {
869 a_job = aio->curjob;
870 if (a_job->aiocb_uptr == aiocbp_list[i])
871 continue;
872 }
873
874 /* Look for a job in the queue */
875 TAILQ_FOREACH(a_job, &aio->jobs_queue, list)
876 if (a_job->aiocb_uptr == aiocbp_list[i])
877 break;
878
879 if (a_job == NULL) {
880 struct aiocb aiocbp;
881
882 mutex_exit(&aio->aio_mtx);
883
884 /* Check if the job is done. */
885 error = copyin(aiocbp_list[i], &aiocbp,
886 sizeof(struct aiocb));
887 if (error == 0 && aiocbp._state != JOB_DONE) {
888 mutex_enter(&aio->aio_mtx);
889 continue;
890 }
891 return error;
892 }
893 }
894
895 /* Wait for a signal or when timeout occurs */
896 error = cv_timedwait_sig(&aio->done_cv, &aio->aio_mtx, timo);
897 if (error) {
898 if (error == EWOULDBLOCK)
899 error = EAGAIN;
900 break;
901 }
902 }
903 mutex_exit(&aio->aio_mtx);
904 return error;
905 }
906
907 int
sys_aio_write(struct lwp * l,const struct sys_aio_write_args * uap,register_t * retval)908 sys_aio_write(struct lwp *l, const struct sys_aio_write_args *uap,
909 register_t *retval)
910 {
911 /* {
912 syscallarg(struct aiocb *) aiocbp;
913 } */
914
915 return aio_enqueue_job(AIO_WRITE, SCARG(uap, aiocbp), NULL);
916 }
917
918 int
sys_lio_listio(struct lwp * l,const struct sys_lio_listio_args * uap,register_t * retval)919 sys_lio_listio(struct lwp *l, const struct sys_lio_listio_args *uap,
920 register_t *retval)
921 {
922 /* {
923 syscallarg(int) mode;
924 syscallarg(struct aiocb *const[]) list;
925 syscallarg(int) nent;
926 syscallarg(struct sigevent *) sig;
927 } */
928 struct proc *p = l->l_proc;
929 struct aioproc *aio;
930 struct aiocb **aiocbp_list;
931 struct lio_req *lio;
932 int i, error, errcnt, mode, nent;
933
934 mode = SCARG(uap, mode);
935 nent = SCARG(uap, nent);
936
937 /* Non-accurate checks for the limit and invalid values */
938 if (nent < 1 || nent > aio_listio_max)
939 return EINVAL;
940 if (aio_jobs_count + nent > aio_max)
941 return EAGAIN;
942
943 /* Check if AIO structure is initialized, if not - initialize it */
944 if (p->p_aio == NULL)
945 if (aio_procinit(p))
946 return EAGAIN;
947 aio = p->p_aio;
948
949 /* Create a LIO structure */
950 lio = pool_get(&aio_lio_pool, PR_WAITOK);
951 lio->refcnt = 1;
952 error = 0;
953
954 switch (mode) {
955 case LIO_WAIT:
956 memset(&lio->sig, 0, sizeof(struct sigevent));
957 break;
958 case LIO_NOWAIT:
959 /* Check for signal, validate it */
960 if (SCARG(uap, sig)) {
961 struct sigevent *sig = &lio->sig;
962
963 error = copyin(SCARG(uap, sig), &lio->sig,
964 sizeof(struct sigevent));
965 if (error == 0 &&
966 (sig->sigev_signo < 0 ||
967 sig->sigev_signo >= NSIG ||
968 sig->sigev_notify < SIGEV_NONE ||
969 sig->sigev_notify > SIGEV_SA))
970 error = EINVAL;
971 } else
972 memset(&lio->sig, 0, sizeof(struct sigevent));
973 break;
974 default:
975 error = EINVAL;
976 break;
977 }
978
979 if (error != 0) {
980 pool_put(&aio_lio_pool, lio);
981 return error;
982 }
983
984 /* Get the list from user-space */
985 aiocbp_list = kmem_alloc(nent * sizeof(*aiocbp_list), KM_SLEEP);
986 error = copyin(SCARG(uap, list), aiocbp_list,
987 nent * sizeof(*aiocbp_list));
988 if (error) {
989 mutex_enter(&aio->aio_mtx);
990 goto err;
991 }
992
993 /* Enqueue all jobs */
994 errcnt = 0;
995 for (i = 0; i < nent; i++) {
996 error = aio_enqueue_job(AIO_LIO, aiocbp_list[i], lio);
997 /*
998 * According to POSIX, in such error case it may
999 * fail with other I/O operations initiated.
1000 */
1001 if (error)
1002 errcnt++;
1003 }
1004
1005 mutex_enter(&aio->aio_mtx);
1006
1007 /* Return an error, if any */
1008 if (errcnt) {
1009 error = EIO;
1010 goto err;
1011 }
1012
1013 if (mode == LIO_WAIT) {
1014 /*
1015 * Wait for AIO completion. In such case,
1016 * the LIO structure will be freed here.
1017 */
1018 while (lio->refcnt > 1 && error == 0)
1019 error = cv_wait_sig(&aio->done_cv, &aio->aio_mtx);
1020 if (error)
1021 error = EINTR;
1022 }
1023
1024 err:
1025 if (--lio->refcnt != 0)
1026 lio = NULL;
1027 mutex_exit(&aio->aio_mtx);
1028 if (lio != NULL) {
1029 aio_sendsig(p, &lio->sig);
1030 pool_put(&aio_lio_pool, lio);
1031 }
1032 kmem_free(aiocbp_list, nent * sizeof(*aiocbp_list));
1033 return error;
1034 }
1035
1036 /*
1037 * SysCtl
1038 */
1039
1040 static int
sysctl_aio_listio_max(SYSCTLFN_ARGS)1041 sysctl_aio_listio_max(SYSCTLFN_ARGS)
1042 {
1043 struct sysctlnode node;
1044 int error, newsize;
1045
1046 node = *rnode;
1047 node.sysctl_data = &newsize;
1048
1049 newsize = aio_listio_max;
1050 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1051 if (error || newp == NULL)
1052 return error;
1053
1054 if (newsize < 1 || newsize > aio_max)
1055 return EINVAL;
1056 aio_listio_max = newsize;
1057
1058 return 0;
1059 }
1060
1061 static int
sysctl_aio_max(SYSCTLFN_ARGS)1062 sysctl_aio_max(SYSCTLFN_ARGS)
1063 {
1064 struct sysctlnode node;
1065 int error, newsize;
1066
1067 node = *rnode;
1068 node.sysctl_data = &newsize;
1069
1070 newsize = aio_max;
1071 error = sysctl_lookup(SYSCTLFN_CALL(&node));
1072 if (error || newp == NULL)
1073 return error;
1074
1075 if (newsize < 1 || newsize < aio_listio_max)
1076 return EINVAL;
1077 aio_max = newsize;
1078
1079 return 0;
1080 }
1081
1082 SYSCTL_SETUP(sysctl_aio_init, "aio sysctl")
1083 {
1084 int rv;
1085
1086 rv = sysctl_createv(clog, 0, NULL, NULL,
1087 CTLFLAG_PERMANENT | CTLFLAG_IMMEDIATE,
1088 CTLTYPE_INT, "posix_aio",
1089 SYSCTL_DESCR("Version of IEEE Std 1003.1 and its "
1090 "Asynchronous I/O option to which the "
1091 "system attempts to conform"),
1092 NULL, _POSIX_ASYNCHRONOUS_IO, NULL, 0,
1093 CTL_KERN, CTL_CREATE, CTL_EOL);
1094
1095 if (rv != 0)
1096 return;
1097
1098 rv = sysctl_createv(clog, 0, NULL, NULL,
1099 CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
1100 CTLTYPE_INT, "aio_listio_max",
1101 SYSCTL_DESCR("Maximum number of asynchronous I/O "
1102 "operations in a single list I/O call"),
1103 sysctl_aio_listio_max, 0, &aio_listio_max, 0,
1104 CTL_KERN, CTL_CREATE, CTL_EOL);
1105
1106 if (rv != 0)
1107 return;
1108
1109 rv = sysctl_createv(clog, 0, NULL, NULL,
1110 CTLFLAG_PERMANENT | CTLFLAG_READWRITE,
1111 CTLTYPE_INT, "aio_max",
1112 SYSCTL_DESCR("Maximum number of asynchronous I/O "
1113 "operations"),
1114 sysctl_aio_max, 0, &aio_max, 0,
1115 CTL_KERN, CTL_CREATE, CTL_EOL);
1116
1117 return;
1118 }
1119
1120 /*
1121 * Debugging
1122 */
1123 #if defined(DDB)
1124 void
aio_print_jobs(void (* pr)(const char *,...))1125 aio_print_jobs(void (*pr)(const char *, ...))
1126 {
1127 struct proc *p = curlwp->l_proc;
1128 struct aioproc *aio;
1129 struct aio_job *a_job;
1130 struct aiocb *aiocbp;
1131
1132 if (p == NULL) {
1133 (*pr)("AIO: We are not in the processes right now.\n");
1134 return;
1135 }
1136
1137 aio = p->p_aio;
1138 if (aio == NULL) {
1139 (*pr)("AIO data is not initialized (PID = %d).\n", p->p_pid);
1140 return;
1141 }
1142
1143 (*pr)("AIO: PID = %d\n", p->p_pid);
1144 (*pr)("AIO: Global count of the jobs = %u\n", aio_jobs_count);
1145 (*pr)("AIO: Count of the jobs = %u\n", aio->jobs_count);
1146
1147 if (aio->curjob) {
1148 a_job = aio->curjob;
1149 (*pr)("\nAIO current job:\n");
1150 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n",
1151 a_job->aio_op, a_job->aiocbp._errno,
1152 a_job->aiocbp._state, a_job->aiocb_uptr);
1153 aiocbp = &a_job->aiocbp;
1154 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n",
1155 aiocbp->aio_fildes, aiocbp->aio_offset,
1156 aiocbp->aio_buf, aiocbp->aio_nbytes);
1157 }
1158
1159 (*pr)("\nAIO queue:\n");
1160 TAILQ_FOREACH(a_job, &aio->jobs_queue, list) {
1161 (*pr)(" opcode = %d, errno = %d, state = %d, aiocb_ptr = %p\n",
1162 a_job->aio_op, a_job->aiocbp._errno,
1163 a_job->aiocbp._state, a_job->aiocb_uptr);
1164 aiocbp = &a_job->aiocbp;
1165 (*pr)(" fd = %d, offset = %u, buf = %p, nbytes = %u\n",
1166 aiocbp->aio_fildes, aiocbp->aio_offset,
1167 aiocbp->aio_buf, aiocbp->aio_nbytes);
1168 }
1169 }
1170 #endif /* defined(DDB) */
1171