1 /* 2 * Copyright (c) 1997 John S. Dyson. All rights reserved. 3 * 4 * Redistribution and use in source and binary forms, with or without 5 * modification, are permitted provided that the following conditions 6 * are met: 7 * 1. Redistributions of source code must retain the above copyright 8 * notice, this list of conditions and the following disclaimer. 9 * 2. John S. Dyson's name may not be used to endorse or promote products 10 * derived from this software without specific prior written permission. 11 * 12 * DISCLAIMER: This code isn't warranted to do anything useful. Anything 13 * bad that happens because of using this software isn't the responsibility 14 * of the author. This software is distributed AS-IS. 15 * 16 * $FreeBSD: src/sys/kern/vfs_aio.c,v 1.70.2.28 2003/05/29 06:15:35 alc Exp $ 17 * $DragonFly: src/sys/kern/vfs_aio.c,v 1.10 2003/07/30 00:19:14 dillon Exp $ 18 */ 19 20 /* 21 * This file contains support for the POSIX 1003.1B AIO/LIO facility. 22 */ 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/buf.h> 27 #include <sys/sysproto.h> 28 #include <sys/filedesc.h> 29 #include <sys/kernel.h> 30 #include <sys/fcntl.h> 31 #include <sys/file.h> 32 #include <sys/lock.h> 33 #include <sys/unistd.h> 34 #include <sys/proc.h> 35 #include <sys/resourcevar.h> 36 #include <sys/signalvar.h> 37 #include <sys/protosw.h> 38 #include <sys/socketvar.h> 39 #include <sys/sysctl.h> 40 #include <sys/vnode.h> 41 #include <sys/conf.h> 42 #include <sys/event.h> 43 44 #include <vm/vm.h> 45 #include <vm/vm_extern.h> 46 #include <vm/pmap.h> 47 #include <vm/vm_map.h> 48 #include <vm/vm_zone.h> 49 #include <sys/aio.h> 50 #include <sys/file2.h> 51 #include <sys/buf2.h> 52 53 #include <machine/limits.h> 54 #include "opt_vfs_aio.h" 55 56 #ifdef VFS_AIO 57 58 /* 59 * Counter for allocating reference ids to new jobs. Wrapped to 1 on 60 * overflow. 61 */ 62 static long jobrefid; 63 64 #define JOBST_NULL 0x0 65 #define JOBST_JOBQGLOBAL 0x2 66 #define JOBST_JOBRUNNING 0x3 67 #define JOBST_JOBFINISHED 0x4 68 #define JOBST_JOBQBUF 0x5 69 #define JOBST_JOBBFINISHED 0x6 70 71 #ifndef MAX_AIO_PER_PROC 72 #define MAX_AIO_PER_PROC 32 73 #endif 74 75 #ifndef MAX_AIO_QUEUE_PER_PROC 76 #define MAX_AIO_QUEUE_PER_PROC 256 /* Bigger than AIO_LISTIO_MAX */ 77 #endif 78 79 #ifndef MAX_AIO_PROCS 80 #define MAX_AIO_PROCS 32 81 #endif 82 83 #ifndef MAX_AIO_QUEUE 84 #define MAX_AIO_QUEUE 1024 /* Bigger than AIO_LISTIO_MAX */ 85 #endif 86 87 #ifndef TARGET_AIO_PROCS 88 #define TARGET_AIO_PROCS 4 89 #endif 90 91 #ifndef MAX_BUF_AIO 92 #define MAX_BUF_AIO 16 93 #endif 94 95 #ifndef AIOD_TIMEOUT_DEFAULT 96 #define AIOD_TIMEOUT_DEFAULT (10 * hz) 97 #endif 98 99 #ifndef AIOD_LIFETIME_DEFAULT 100 #define AIOD_LIFETIME_DEFAULT (30 * hz) 101 #endif 102 103 SYSCTL_NODE(_vfs, OID_AUTO, aio, CTLFLAG_RW, 0, "Async IO management"); 104 105 static int max_aio_procs = MAX_AIO_PROCS; 106 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_procs, 107 CTLFLAG_RW, &max_aio_procs, 0, 108 "Maximum number of kernel threads to use for handling async IO"); 109 110 static int num_aio_procs = 0; 111 SYSCTL_INT(_vfs_aio, OID_AUTO, num_aio_procs, 112 CTLFLAG_RD, &num_aio_procs, 0, 113 "Number of presently active kernel threads for async IO"); 114 115 /* 116 * The code will adjust the actual number of AIO processes towards this 117 * number when it gets a chance. 118 */ 119 static int target_aio_procs = TARGET_AIO_PROCS; 120 SYSCTL_INT(_vfs_aio, OID_AUTO, target_aio_procs, CTLFLAG_RW, &target_aio_procs, 121 0, "Preferred number of ready kernel threads for async IO"); 122 123 static int max_queue_count = MAX_AIO_QUEUE; 124 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue, CTLFLAG_RW, &max_queue_count, 0, 125 "Maximum number of aio requests to queue, globally"); 126 127 static int num_queue_count = 0; 128 SYSCTL_INT(_vfs_aio, OID_AUTO, num_queue_count, CTLFLAG_RD, &num_queue_count, 0, 129 "Number of queued aio requests"); 130 131 static int num_buf_aio = 0; 132 SYSCTL_INT(_vfs_aio, OID_AUTO, num_buf_aio, CTLFLAG_RD, &num_buf_aio, 0, 133 "Number of aio requests presently handled by the buf subsystem"); 134 135 /* Number of async I/O thread in the process of being started */ 136 /* XXX This should be local to _aio_aqueue() */ 137 static int num_aio_resv_start = 0; 138 139 static int aiod_timeout; 140 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_timeout, CTLFLAG_RW, &aiod_timeout, 0, 141 "Timeout value for synchronous aio operations"); 142 143 static int aiod_lifetime; 144 SYSCTL_INT(_vfs_aio, OID_AUTO, aiod_lifetime, CTLFLAG_RW, &aiod_lifetime, 0, 145 "Maximum lifetime for idle aiod"); 146 147 static int max_aio_per_proc = MAX_AIO_PER_PROC; 148 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_per_proc, CTLFLAG_RW, &max_aio_per_proc, 149 0, "Maximum active aio requests per process (stored in the process)"); 150 151 static int max_aio_queue_per_proc = MAX_AIO_QUEUE_PER_PROC; 152 SYSCTL_INT(_vfs_aio, OID_AUTO, max_aio_queue_per_proc, CTLFLAG_RW, 153 &max_aio_queue_per_proc, 0, 154 "Maximum queued aio requests per process (stored in the process)"); 155 156 static int max_buf_aio = MAX_BUF_AIO; 157 SYSCTL_INT(_vfs_aio, OID_AUTO, max_buf_aio, CTLFLAG_RW, &max_buf_aio, 0, 158 "Maximum buf aio requests per process (stored in the process)"); 159 160 /* 161 * AIO process info 162 */ 163 #define AIOP_FREE 0x1 /* proc on free queue */ 164 #define AIOP_SCHED 0x2 /* proc explicitly scheduled */ 165 166 struct aioproclist { 167 int aioprocflags; /* AIO proc flags */ 168 TAILQ_ENTRY(aioproclist) list; /* List of processes */ 169 struct proc *aioproc; /* The AIO thread */ 170 }; 171 172 /* 173 * data-structure for lio signal management 174 */ 175 struct aio_liojob { 176 int lioj_flags; 177 int lioj_buffer_count; 178 int lioj_buffer_finished_count; 179 int lioj_queue_count; 180 int lioj_queue_finished_count; 181 struct sigevent lioj_signal; /* signal on all I/O done */ 182 TAILQ_ENTRY(aio_liojob) lioj_list; 183 struct kaioinfo *lioj_ki; 184 }; 185 #define LIOJ_SIGNAL 0x1 /* signal on all done (lio) */ 186 #define LIOJ_SIGNAL_POSTED 0x2 /* signal has been posted */ 187 188 /* 189 * per process aio data structure 190 */ 191 struct kaioinfo { 192 int kaio_flags; /* per process kaio flags */ 193 int kaio_maxactive_count; /* maximum number of AIOs */ 194 int kaio_active_count; /* number of currently used AIOs */ 195 int kaio_qallowed_count; /* maxiumu size of AIO queue */ 196 int kaio_queue_count; /* size of AIO queue */ 197 int kaio_ballowed_count; /* maximum number of buffers */ 198 int kaio_queue_finished_count; /* number of daemon jobs finished */ 199 int kaio_buffer_count; /* number of physio buffers */ 200 int kaio_buffer_finished_count; /* count of I/O done */ 201 struct proc *kaio_p; /* process that uses this kaio block */ 202 TAILQ_HEAD(,aio_liojob) kaio_liojoblist; /* list of lio jobs */ 203 TAILQ_HEAD(,aiocblist) kaio_jobqueue; /* job queue for process */ 204 TAILQ_HEAD(,aiocblist) kaio_jobdone; /* done queue for process */ 205 TAILQ_HEAD(,aiocblist) kaio_bufqueue; /* buffer job queue for process */ 206 TAILQ_HEAD(,aiocblist) kaio_bufdone; /* buffer done queue for process */ 207 TAILQ_HEAD(,aiocblist) kaio_sockqueue; /* queue for aios waiting on sockets */ 208 }; 209 210 #define KAIO_RUNDOWN 0x1 /* process is being run down */ 211 #define KAIO_WAKEUP 0x2 /* wakeup process when there is a significant event */ 212 213 static TAILQ_HEAD(,aioproclist) aio_freeproc, aio_activeproc; 214 static TAILQ_HEAD(,aiocblist) aio_jobs; /* Async job list */ 215 static TAILQ_HEAD(,aiocblist) aio_bufjobs; /* Phys I/O job list */ 216 static TAILQ_HEAD(,aiocblist) aio_freejobs; /* Pool of free jobs */ 217 218 static void aio_init_aioinfo(struct proc *p); 219 static void aio_onceonly(void *); 220 static int aio_free_entry(struct aiocblist *aiocbe); 221 static void aio_process(struct aiocblist *aiocbe); 222 static int aio_newproc(void); 223 static int aio_aqueue(struct aiocb *job, int type); 224 static void aio_physwakeup(struct buf *bp); 225 static int aio_fphysio(struct aiocblist *aiocbe); 226 static int aio_qphysio(struct proc *p, struct aiocblist *iocb); 227 static void aio_daemon(void *uproc); 228 static void process_signal(void *aioj); 229 230 SYSINIT(aio, SI_SUB_VFS, SI_ORDER_ANY, aio_onceonly, NULL); 231 232 /* 233 * Zones for: 234 * kaio Per process async io info 235 * aiop async io thread data 236 * aiocb async io jobs 237 * aiol list io job pointer - internal to aio_suspend XXX 238 * aiolio list io jobs 239 */ 240 static vm_zone_t kaio_zone, aiop_zone, aiocb_zone, aiol_zone, aiolio_zone; 241 242 /* 243 * Startup initialization 244 */ 245 static void 246 aio_onceonly(void *na) 247 { 248 TAILQ_INIT(&aio_freeproc); 249 TAILQ_INIT(&aio_activeproc); 250 TAILQ_INIT(&aio_jobs); 251 TAILQ_INIT(&aio_bufjobs); 252 TAILQ_INIT(&aio_freejobs); 253 kaio_zone = zinit("AIO", sizeof(struct kaioinfo), 0, 0, 1); 254 aiop_zone = zinit("AIOP", sizeof(struct aioproclist), 0, 0, 1); 255 aiocb_zone = zinit("AIOCB", sizeof(struct aiocblist), 0, 0, 1); 256 aiol_zone = zinit("AIOL", AIO_LISTIO_MAX*sizeof(intptr_t), 0, 0, 1); 257 aiolio_zone = zinit("AIOLIO", sizeof(struct aio_liojob), 0, 0, 1); 258 aiod_timeout = AIOD_TIMEOUT_DEFAULT; 259 aiod_lifetime = AIOD_LIFETIME_DEFAULT; 260 jobrefid = 1; 261 } 262 263 /* 264 * Init the per-process aioinfo structure. The aioinfo limits are set 265 * per-process for user limit (resource) management. 266 */ 267 static void 268 aio_init_aioinfo(struct proc *p) 269 { 270 struct kaioinfo *ki; 271 if (p->p_aioinfo == NULL) { 272 ki = zalloc(kaio_zone); 273 p->p_aioinfo = ki; 274 ki->kaio_flags = 0; 275 ki->kaio_maxactive_count = max_aio_per_proc; 276 ki->kaio_active_count = 0; 277 ki->kaio_qallowed_count = max_aio_queue_per_proc; 278 ki->kaio_queue_count = 0; 279 ki->kaio_ballowed_count = max_buf_aio; 280 ki->kaio_buffer_count = 0; 281 ki->kaio_buffer_finished_count = 0; 282 ki->kaio_p = p; 283 TAILQ_INIT(&ki->kaio_jobdone); 284 TAILQ_INIT(&ki->kaio_jobqueue); 285 TAILQ_INIT(&ki->kaio_bufdone); 286 TAILQ_INIT(&ki->kaio_bufqueue); 287 TAILQ_INIT(&ki->kaio_liojoblist); 288 TAILQ_INIT(&ki->kaio_sockqueue); 289 } 290 291 while (num_aio_procs < target_aio_procs) 292 aio_newproc(); 293 } 294 295 /* 296 * Free a job entry. Wait for completion if it is currently active, but don't 297 * delay forever. If we delay, we return a flag that says that we have to 298 * restart the queue scan. 299 */ 300 static int 301 aio_free_entry(struct aiocblist *aiocbe) 302 { 303 struct kaioinfo *ki; 304 struct aio_liojob *lj; 305 struct proc *p; 306 int error; 307 int s; 308 309 if (aiocbe->jobstate == JOBST_NULL) 310 panic("aio_free_entry: freeing already free job"); 311 312 p = aiocbe->userproc; 313 ki = p->p_aioinfo; 314 lj = aiocbe->lio; 315 if (ki == NULL) 316 panic("aio_free_entry: missing p->p_aioinfo"); 317 318 while (aiocbe->jobstate == JOBST_JOBRUNNING) { 319 aiocbe->jobflags |= AIOCBLIST_RUNDOWN; 320 tsleep(aiocbe, 0, "jobwai", 0); 321 } 322 if (aiocbe->bp == NULL) { 323 if (ki->kaio_queue_count <= 0) 324 panic("aio_free_entry: process queue size <= 0"); 325 if (num_queue_count <= 0) 326 panic("aio_free_entry: system wide queue size <= 0"); 327 328 if (lj) { 329 lj->lioj_queue_count--; 330 if (aiocbe->jobflags & AIOCBLIST_DONE) 331 lj->lioj_queue_finished_count--; 332 } 333 ki->kaio_queue_count--; 334 if (aiocbe->jobflags & AIOCBLIST_DONE) 335 ki->kaio_queue_finished_count--; 336 num_queue_count--; 337 } else { 338 if (lj) { 339 lj->lioj_buffer_count--; 340 if (aiocbe->jobflags & AIOCBLIST_DONE) 341 lj->lioj_buffer_finished_count--; 342 } 343 if (aiocbe->jobflags & AIOCBLIST_DONE) 344 ki->kaio_buffer_finished_count--; 345 ki->kaio_buffer_count--; 346 num_buf_aio--; 347 } 348 349 /* aiocbe is going away, we need to destroy any knotes */ 350 knote_remove(p->p_thread, &aiocbe->klist); 351 352 if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags & KAIO_RUNDOWN) 353 && ((ki->kaio_buffer_count == 0) && (ki->kaio_queue_count == 0)))) { 354 ki->kaio_flags &= ~KAIO_WAKEUP; 355 wakeup(p); 356 } 357 358 if (aiocbe->jobstate == JOBST_JOBQBUF) { 359 if ((error = aio_fphysio(aiocbe)) != 0) 360 return error; 361 if (aiocbe->jobstate != JOBST_JOBBFINISHED) 362 panic("aio_free_entry: invalid physio finish-up state"); 363 s = splbio(); 364 TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist); 365 splx(s); 366 } else if (aiocbe->jobstate == JOBST_JOBQGLOBAL) { 367 s = splnet(); 368 TAILQ_REMOVE(&aio_jobs, aiocbe, list); 369 TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist); 370 splx(s); 371 } else if (aiocbe->jobstate == JOBST_JOBFINISHED) 372 TAILQ_REMOVE(&ki->kaio_jobdone, aiocbe, plist); 373 else if (aiocbe->jobstate == JOBST_JOBBFINISHED) { 374 s = splbio(); 375 TAILQ_REMOVE(&ki->kaio_bufdone, aiocbe, plist); 376 splx(s); 377 if (aiocbe->bp) { 378 vunmapbuf(aiocbe->bp); 379 relpbuf(aiocbe->bp, NULL); 380 aiocbe->bp = NULL; 381 } 382 } 383 if (lj && (lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 0)) { 384 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list); 385 zfree(aiolio_zone, lj); 386 } 387 aiocbe->jobstate = JOBST_NULL; 388 untimeout(process_signal, aiocbe, aiocbe->timeouthandle); 389 fdrop(aiocbe->fd_file, curthread); 390 TAILQ_INSERT_HEAD(&aio_freejobs, aiocbe, list); 391 return 0; 392 } 393 #endif /* VFS_AIO */ 394 395 /* 396 * Rundown the jobs for a given process. 397 */ 398 void 399 aio_proc_rundown(struct proc *p) 400 { 401 #ifndef VFS_AIO 402 return; 403 #else 404 int s; 405 struct kaioinfo *ki; 406 struct aio_liojob *lj, *ljn; 407 struct aiocblist *aiocbe, *aiocbn; 408 struct file *fp; 409 struct socket *so; 410 411 ki = p->p_aioinfo; 412 if (ki == NULL) 413 return; 414 415 ki->kaio_flags |= LIOJ_SIGNAL_POSTED; 416 while ((ki->kaio_active_count > 0) || (ki->kaio_buffer_count > 417 ki->kaio_buffer_finished_count)) { 418 ki->kaio_flags |= KAIO_RUNDOWN; 419 if (tsleep(p, 0, "kaiowt", aiod_timeout)) 420 break; 421 } 422 423 /* 424 * Move any aio ops that are waiting on socket I/O to the normal job 425 * queues so they are cleaned up with any others. 426 */ 427 s = splnet(); 428 for (aiocbe = TAILQ_FIRST(&ki->kaio_sockqueue); aiocbe; aiocbe = 429 aiocbn) { 430 aiocbn = TAILQ_NEXT(aiocbe, plist); 431 fp = aiocbe->fd_file; 432 if (fp != NULL) { 433 so = (struct socket *)fp->f_data; 434 TAILQ_REMOVE(&so->so_aiojobq, aiocbe, list); 435 if (TAILQ_EMPTY(&so->so_aiojobq)) { 436 so->so_snd.sb_flags &= ~SB_AIO; 437 so->so_rcv.sb_flags &= ~SB_AIO; 438 } 439 } 440 TAILQ_REMOVE(&ki->kaio_sockqueue, aiocbe, plist); 441 TAILQ_INSERT_HEAD(&aio_jobs, aiocbe, list); 442 TAILQ_INSERT_HEAD(&ki->kaio_jobqueue, aiocbe, plist); 443 } 444 splx(s); 445 446 restart1: 447 for (aiocbe = TAILQ_FIRST(&ki->kaio_jobdone); aiocbe; aiocbe = aiocbn) { 448 aiocbn = TAILQ_NEXT(aiocbe, plist); 449 if (aio_free_entry(aiocbe)) 450 goto restart1; 451 } 452 453 restart2: 454 for (aiocbe = TAILQ_FIRST(&ki->kaio_jobqueue); aiocbe; aiocbe = 455 aiocbn) { 456 aiocbn = TAILQ_NEXT(aiocbe, plist); 457 if (aio_free_entry(aiocbe)) 458 goto restart2; 459 } 460 461 /* 462 * Note the use of lots of splbio here, trying to avoid splbio for long chains 463 * of I/O. Probably unnecessary. 464 */ 465 restart3: 466 s = splbio(); 467 while (TAILQ_FIRST(&ki->kaio_bufqueue)) { 468 ki->kaio_flags |= KAIO_WAKEUP; 469 tsleep(p, 0, "aioprn", 0); 470 splx(s); 471 goto restart3; 472 } 473 splx(s); 474 475 restart4: 476 s = splbio(); 477 for (aiocbe = TAILQ_FIRST(&ki->kaio_bufdone); aiocbe; aiocbe = aiocbn) { 478 aiocbn = TAILQ_NEXT(aiocbe, plist); 479 if (aio_free_entry(aiocbe)) { 480 splx(s); 481 goto restart4; 482 } 483 } 484 splx(s); 485 486 /* 487 * If we've slept, jobs might have moved from one queue to another. 488 * Retry rundown if we didn't manage to empty the queues. 489 */ 490 if (TAILQ_FIRST(&ki->kaio_jobdone) != NULL || 491 TAILQ_FIRST(&ki->kaio_jobqueue) != NULL || 492 TAILQ_FIRST(&ki->kaio_bufqueue) != NULL || 493 TAILQ_FIRST(&ki->kaio_bufdone) != NULL) 494 goto restart1; 495 496 for (lj = TAILQ_FIRST(&ki->kaio_liojoblist); lj; lj = ljn) { 497 ljn = TAILQ_NEXT(lj, lioj_list); 498 if ((lj->lioj_buffer_count == 0) && (lj->lioj_queue_count == 499 0)) { 500 TAILQ_REMOVE(&ki->kaio_liojoblist, lj, lioj_list); 501 zfree(aiolio_zone, lj); 502 } else { 503 #ifdef DIAGNOSTIC 504 printf("LIO job not cleaned up: B:%d, BF:%d, Q:%d, " 505 "QF:%d\n", lj->lioj_buffer_count, 506 lj->lioj_buffer_finished_count, 507 lj->lioj_queue_count, 508 lj->lioj_queue_finished_count); 509 #endif 510 } 511 } 512 513 zfree(kaio_zone, ki); 514 p->p_aioinfo = NULL; 515 #endif /* VFS_AIO */ 516 } 517 518 #ifdef VFS_AIO 519 /* 520 * Select a job to run (called by an AIO daemon). 521 */ 522 static struct aiocblist * 523 aio_selectjob(struct aioproclist *aiop) 524 { 525 int s; 526 struct aiocblist *aiocbe; 527 struct kaioinfo *ki; 528 struct proc *userp; 529 530 s = splnet(); 531 for (aiocbe = TAILQ_FIRST(&aio_jobs); aiocbe; aiocbe = 532 TAILQ_NEXT(aiocbe, list)) { 533 userp = aiocbe->userproc; 534 ki = userp->p_aioinfo; 535 536 if (ki->kaio_active_count < ki->kaio_maxactive_count) { 537 TAILQ_REMOVE(&aio_jobs, aiocbe, list); 538 splx(s); 539 return aiocbe; 540 } 541 } 542 splx(s); 543 544 return NULL; 545 } 546 547 /* 548 * The AIO processing activity. This is the code that does the I/O request for 549 * the non-physio version of the operations. The normal vn operations are used, 550 * and this code should work in all instances for every type of file, including 551 * pipes, sockets, fifos, and regular files. 552 */ 553 static void 554 aio_process(struct aiocblist *aiocbe) 555 { 556 struct thread *mytd; 557 struct aiocb *cb; 558 struct file *fp; 559 struct uio auio; 560 struct iovec aiov; 561 int cnt; 562 int error; 563 int oublock_st, oublock_end; 564 int inblock_st, inblock_end; 565 566 mytd = curthread; 567 cb = &aiocbe->uaiocb; 568 fp = aiocbe->fd_file; 569 570 aiov.iov_base = (void *)(uintptr_t)cb->aio_buf; 571 aiov.iov_len = cb->aio_nbytes; 572 573 auio.uio_iov = &aiov; 574 auio.uio_iovcnt = 1; 575 auio.uio_offset = cb->aio_offset; 576 auio.uio_resid = cb->aio_nbytes; 577 cnt = cb->aio_nbytes; 578 auio.uio_segflg = UIO_USERSPACE; 579 auio.uio_td = mytd; 580 581 inblock_st = mytd->td_proc->p_stats->p_ru.ru_inblock; 582 oublock_st = mytd->td_proc->p_stats->p_ru.ru_oublock; 583 /* 584 * _aio_aqueue() acquires a reference to the file that is 585 * released in aio_free_entry(). 586 */ 587 if (cb->aio_lio_opcode == LIO_READ) { 588 auio.uio_rw = UIO_READ; 589 error = fo_read(fp, &auio, fp->f_cred, FOF_OFFSET, mytd); 590 } else { 591 auio.uio_rw = UIO_WRITE; 592 error = fo_write(fp, &auio, fp->f_cred, FOF_OFFSET, mytd); 593 } 594 inblock_end = mytd->td_proc->p_stats->p_ru.ru_inblock; 595 oublock_end = mytd->td_proc->p_stats->p_ru.ru_oublock; 596 597 aiocbe->inputcharge = inblock_end - inblock_st; 598 aiocbe->outputcharge = oublock_end - oublock_st; 599 600 if ((error) && (auio.uio_resid != cnt)) { 601 if (error == ERESTART || error == EINTR || error == EWOULDBLOCK) 602 error = 0; 603 if ((error == EPIPE) && (cb->aio_lio_opcode == LIO_WRITE)) 604 psignal(aiocbe->userproc, SIGPIPE); 605 } 606 607 cnt -= auio.uio_resid; 608 cb->_aiocb_private.error = error; 609 cb->_aiocb_private.status = cnt; 610 } 611 612 /* 613 * The AIO daemon, most of the actual work is done in aio_process, 614 * but the setup (and address space mgmt) is done in this routine. 615 * 616 * The MP lock is held on entry. 617 */ 618 static void 619 aio_daemon(void *uproc) 620 { 621 int s; 622 struct aio_liojob *lj; 623 struct aiocb *cb; 624 struct aiocblist *aiocbe; 625 struct aioproclist *aiop; 626 struct kaioinfo *ki; 627 struct proc *curcp, *mycp, *userp; 628 struct vmspace *myvm, *tmpvm; 629 struct ucred *cr; 630 631 /* 632 * Local copies of curproc (cp) and vmspace (myvm) 633 */ 634 mycp = curproc; 635 myvm = mycp->p_vmspace; 636 637 if (mycp->p_textvp) { 638 vrele(mycp->p_textvp); 639 mycp->p_textvp = NULL; 640 } 641 642 /* 643 * Allocate and ready the aio control info. There is one aiop structure 644 * per daemon. 645 */ 646 aiop = zalloc(aiop_zone); 647 aiop->aioproc = mycp; 648 aiop->aioprocflags |= AIOP_FREE; 649 650 s = splnet(); 651 652 /* 653 * Place thread (lightweight process) onto the AIO free thread list. 654 */ 655 if (TAILQ_EMPTY(&aio_freeproc)) 656 wakeup(&aio_freeproc); 657 TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list); 658 659 splx(s); 660 661 /* Make up a name for the daemon. */ 662 strcpy(mycp->p_comm, "aiod"); 663 664 /* 665 * Get rid of our current filedescriptors. AIOD's don't need any 666 * filedescriptors, except as temporarily inherited from the client. 667 * Credentials are also cloned, and made equivalent to "root". 668 */ 669 fdfree(mycp); 670 mycp->p_fd = NULL; 671 cr = cratom(&mycp->p_ucred); 672 cr->cr_uid = 0; 673 uifree(cr->cr_uidinfo); 674 cr->cr_uidinfo = uifind(0); 675 cr->cr_ngroups = 1; 676 cr->cr_groups[0] = 1; 677 678 /* The daemon resides in its own pgrp. */ 679 enterpgrp(mycp, mycp->p_pid, 1); 680 681 /* Mark special process type. */ 682 mycp->p_flag |= P_SYSTEM | P_KTHREADP; 683 684 /* 685 * Wakeup parent process. (Parent sleeps to keep from blasting away 686 * and creating too many daemons.) 687 */ 688 wakeup(mycp); 689 690 for (;;) { 691 /* 692 * curcp is the current daemon process context. 693 * userp is the current user process context. 694 */ 695 curcp = mycp; 696 697 /* 698 * Take daemon off of free queue 699 */ 700 if (aiop->aioprocflags & AIOP_FREE) { 701 s = splnet(); 702 TAILQ_REMOVE(&aio_freeproc, aiop, list); 703 TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list); 704 aiop->aioprocflags &= ~AIOP_FREE; 705 splx(s); 706 } 707 aiop->aioprocflags &= ~AIOP_SCHED; 708 709 /* 710 * Check for jobs. 711 */ 712 while ((aiocbe = aio_selectjob(aiop)) != NULL) { 713 cb = &aiocbe->uaiocb; 714 userp = aiocbe->userproc; 715 716 aiocbe->jobstate = JOBST_JOBRUNNING; 717 718 /* 719 * Connect to process address space for user program. 720 */ 721 if (userp != curcp) { 722 /* 723 * Save the current address space that we are 724 * connected to. 725 */ 726 tmpvm = mycp->p_vmspace; 727 728 /* 729 * Point to the new user address space, and 730 * refer to it. 731 */ 732 mycp->p_vmspace = userp->p_vmspace; 733 mycp->p_vmspace->vm_refcnt++; 734 735 /* Activate the new mapping. */ 736 pmap_activate(mycp); 737 738 /* 739 * If the old address space wasn't the daemons 740 * own address space, then we need to remove the 741 * daemon's reference from the other process 742 * that it was acting on behalf of. 743 */ 744 if (tmpvm != myvm) { 745 vmspace_free(tmpvm); 746 } 747 curcp = userp; 748 } 749 750 ki = userp->p_aioinfo; 751 lj = aiocbe->lio; 752 753 /* Account for currently active jobs. */ 754 ki->kaio_active_count++; 755 756 /* Do the I/O function. */ 757 aio_process(aiocbe); 758 759 /* Decrement the active job count. */ 760 ki->kaio_active_count--; 761 762 /* 763 * Increment the completion count for wakeup/signal 764 * comparisons. 765 */ 766 aiocbe->jobflags |= AIOCBLIST_DONE; 767 ki->kaio_queue_finished_count++; 768 if (lj) 769 lj->lioj_queue_finished_count++; 770 if ((ki->kaio_flags & KAIO_WAKEUP) || ((ki->kaio_flags 771 & KAIO_RUNDOWN) && (ki->kaio_active_count == 0))) { 772 ki->kaio_flags &= ~KAIO_WAKEUP; 773 wakeup(userp); 774 } 775 776 s = splbio(); 777 if (lj && (lj->lioj_flags & 778 (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == LIOJ_SIGNAL) { 779 if ((lj->lioj_queue_finished_count == 780 lj->lioj_queue_count) && 781 (lj->lioj_buffer_finished_count == 782 lj->lioj_buffer_count)) { 783 psignal(userp, 784 lj->lioj_signal.sigev_signo); 785 lj->lioj_flags |= 786 LIOJ_SIGNAL_POSTED; 787 } 788 } 789 splx(s); 790 791 aiocbe->jobstate = JOBST_JOBFINISHED; 792 793 s = splnet(); 794 TAILQ_REMOVE(&ki->kaio_jobqueue, aiocbe, plist); 795 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, aiocbe, plist); 796 splx(s); 797 KNOTE(&aiocbe->klist, 0); 798 799 if (aiocbe->jobflags & AIOCBLIST_RUNDOWN) { 800 wakeup(aiocbe); 801 aiocbe->jobflags &= ~AIOCBLIST_RUNDOWN; 802 } 803 804 if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) { 805 psignal(userp, cb->aio_sigevent.sigev_signo); 806 } 807 } 808 809 /* 810 * Disconnect from user address space. 811 */ 812 if (curcp != mycp) { 813 /* Get the user address space to disconnect from. */ 814 tmpvm = mycp->p_vmspace; 815 816 /* Get original address space for daemon. */ 817 mycp->p_vmspace = myvm; 818 819 /* Activate the daemon's address space. */ 820 pmap_activate(mycp); 821 #ifdef DIAGNOSTIC 822 if (tmpvm == myvm) { 823 printf("AIOD: vmspace problem -- %d\n", 824 mycp->p_pid); 825 } 826 #endif 827 /* Remove our vmspace reference. */ 828 vmspace_free(tmpvm); 829 830 curcp = mycp; 831 } 832 833 /* 834 * If we are the first to be put onto the free queue, wakeup 835 * anyone waiting for a daemon. 836 */ 837 s = splnet(); 838 TAILQ_REMOVE(&aio_activeproc, aiop, list); 839 if (TAILQ_EMPTY(&aio_freeproc)) 840 wakeup(&aio_freeproc); 841 TAILQ_INSERT_HEAD(&aio_freeproc, aiop, list); 842 aiop->aioprocflags |= AIOP_FREE; 843 splx(s); 844 845 /* 846 * If daemon is inactive for a long time, allow it to exit, 847 * thereby freeing resources. 848 */ 849 if (((aiop->aioprocflags & AIOP_SCHED) == 0) && tsleep(mycp, 850 0, "aiordy", aiod_lifetime)) { 851 s = splnet(); 852 if (TAILQ_EMPTY(&aio_jobs)) { 853 if ((aiop->aioprocflags & AIOP_FREE) && 854 (num_aio_procs > target_aio_procs)) { 855 TAILQ_REMOVE(&aio_freeproc, aiop, list); 856 splx(s); 857 zfree(aiop_zone, aiop); 858 num_aio_procs--; 859 #ifdef DIAGNOSTIC 860 if (mycp->p_vmspace->vm_refcnt <= 1) { 861 printf("AIOD: bad vm refcnt for" 862 " exiting daemon: %d\n", 863 mycp->p_vmspace->vm_refcnt); 864 } 865 #endif 866 exit1(0); 867 } 868 } 869 splx(s); 870 } 871 } 872 } 873 874 /* 875 * Create a new AIO daemon. This is mostly a kernel-thread fork routine. The 876 * AIO daemon modifies its environment itself. 877 */ 878 static int 879 aio_newproc() 880 { 881 int error; 882 struct proc *p, *np; 883 884 p = &proc0; 885 error = fork1(p, RFPROC|RFMEM|RFNOWAIT, &np); 886 if (error) 887 return error; 888 cpu_set_fork_handler(np, aio_daemon, curproc); 889 start_forked_proc(p, np); 890 891 /* 892 * Wait until daemon is started, but continue on just in case to 893 * handle error conditions. 894 */ 895 error = tsleep(np, 0, "aiosta", aiod_timeout); 896 num_aio_procs++; 897 898 return error; 899 } 900 901 /* 902 * Try the high-performance, low-overhead physio method for eligible 903 * VCHR devices. This method doesn't use an aio helper thread, and 904 * thus has very low overhead. 905 * 906 * Assumes that the caller, _aio_aqueue(), has incremented the file 907 * structure's reference count, preventing its deallocation for the 908 * duration of this call. 909 */ 910 static int 911 aio_qphysio(struct proc *p, struct aiocblist *aiocbe) 912 { 913 int error; 914 struct aiocb *cb; 915 struct file *fp; 916 struct buf *bp; 917 struct vnode *vp; 918 struct kaioinfo *ki; 919 struct aio_liojob *lj; 920 int s; 921 int notify; 922 923 cb = &aiocbe->uaiocb; 924 fp = aiocbe->fd_file; 925 926 if (fp->f_type != DTYPE_VNODE) 927 return (-1); 928 929 vp = (struct vnode *)fp->f_data; 930 931 /* 932 * If its not a disk, we don't want to return a positive error. 933 * It causes the aio code to not fall through to try the thread 934 * way when you're talking to a regular file. 935 */ 936 if (!vn_isdisk(vp, &error)) { 937 if (error == ENOTBLK) 938 return (-1); 939 else 940 return (error); 941 } 942 943 if (cb->aio_nbytes % vp->v_rdev->si_bsize_phys) 944 return (-1); 945 946 if (cb->aio_nbytes > 947 MAXPHYS - (((vm_offset_t) cb->aio_buf) & PAGE_MASK)) 948 return (-1); 949 950 ki = p->p_aioinfo; 951 if (ki->kaio_buffer_count >= ki->kaio_ballowed_count) 952 return (-1); 953 954 ki->kaio_buffer_count++; 955 956 lj = aiocbe->lio; 957 if (lj) 958 lj->lioj_buffer_count++; 959 960 /* Create and build a buffer header for a transfer. */ 961 bp = (struct buf *)getpbuf(NULL); 962 BUF_KERNPROC(bp); 963 964 /* 965 * Get a copy of the kva from the physical buffer. 966 */ 967 bp->b_caller1 = p; 968 bp->b_dev = vp->v_rdev; 969 error = 0; 970 971 bp->b_bcount = cb->aio_nbytes; 972 bp->b_bufsize = cb->aio_nbytes; 973 bp->b_flags = B_PHYS | B_CALL | (cb->aio_lio_opcode == LIO_WRITE ? 974 B_WRITE : B_READ); 975 bp->b_iodone = aio_physwakeup; 976 bp->b_saveaddr = bp->b_data; 977 bp->b_data = (void *)(uintptr_t)cb->aio_buf; 978 bp->b_blkno = btodb(cb->aio_offset); 979 980 /* Bring buffer into kernel space. */ 981 if (vmapbuf(bp) < 0) { 982 error = EFAULT; 983 goto doerror; 984 } 985 986 s = splbio(); 987 aiocbe->bp = bp; 988 bp->b_spc = (void *)aiocbe; 989 TAILQ_INSERT_TAIL(&aio_bufjobs, aiocbe, list); 990 TAILQ_INSERT_TAIL(&ki->kaio_bufqueue, aiocbe, plist); 991 aiocbe->jobstate = JOBST_JOBQBUF; 992 cb->_aiocb_private.status = cb->aio_nbytes; 993 num_buf_aio++; 994 bp->b_error = 0; 995 996 splx(s); 997 998 /* Perform transfer. */ 999 BUF_STRATEGY(bp, 0); 1000 1001 notify = 0; 1002 s = splbio(); 1003 1004 /* 1005 * If we had an error invoking the request, or an error in processing 1006 * the request before we have returned, we process it as an error in 1007 * transfer. Note that such an I/O error is not indicated immediately, 1008 * but is returned using the aio_error mechanism. In this case, 1009 * aio_suspend will return immediately. 1010 */ 1011 if (bp->b_error || (bp->b_flags & B_ERROR)) { 1012 struct aiocb *job = aiocbe->uuaiocb; 1013 1014 aiocbe->uaiocb._aiocb_private.status = 0; 1015 suword(&job->_aiocb_private.status, 0); 1016 aiocbe->uaiocb._aiocb_private.error = bp->b_error; 1017 suword(&job->_aiocb_private.error, bp->b_error); 1018 1019 ki->kaio_buffer_finished_count++; 1020 1021 if (aiocbe->jobstate != JOBST_JOBBFINISHED) { 1022 aiocbe->jobstate = JOBST_JOBBFINISHED; 1023 aiocbe->jobflags |= AIOCBLIST_DONE; 1024 TAILQ_REMOVE(&aio_bufjobs, aiocbe, list); 1025 TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist); 1026 TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist); 1027 notify = 1; 1028 } 1029 } 1030 splx(s); 1031 if (notify) 1032 KNOTE(&aiocbe->klist, 0); 1033 return 0; 1034 1035 doerror: 1036 ki->kaio_buffer_count--; 1037 if (lj) 1038 lj->lioj_buffer_count--; 1039 aiocbe->bp = NULL; 1040 relpbuf(bp, NULL); 1041 return error; 1042 } 1043 1044 /* 1045 * This waits/tests physio completion. 1046 */ 1047 static int 1048 aio_fphysio(struct aiocblist *iocb) 1049 { 1050 int s; 1051 struct buf *bp; 1052 int error; 1053 1054 bp = iocb->bp; 1055 1056 s = splbio(); 1057 while ((bp->b_flags & B_DONE) == 0) { 1058 if (tsleep(bp, 0, "physstr", aiod_timeout)) { 1059 if ((bp->b_flags & B_DONE) == 0) { 1060 splx(s); 1061 return EINPROGRESS; 1062 } else 1063 break; 1064 } 1065 } 1066 splx(s); 1067 1068 /* Release mapping into kernel space. */ 1069 vunmapbuf(bp); 1070 iocb->bp = 0; 1071 1072 error = 0; 1073 1074 /* Check for an error. */ 1075 if (bp->b_flags & B_ERROR) 1076 error = bp->b_error; 1077 1078 relpbuf(bp, NULL); 1079 return (error); 1080 } 1081 #endif /* VFS_AIO */ 1082 1083 /* 1084 * Wake up aio requests that may be serviceable now. 1085 */ 1086 void 1087 aio_swake(struct socket *so, struct sockbuf *sb) 1088 { 1089 #ifndef VFS_AIO 1090 return; 1091 #else 1092 struct aiocblist *cb,*cbn; 1093 struct proc *p; 1094 struct kaioinfo *ki = NULL; 1095 int opcode, wakecount = 0; 1096 struct aioproclist *aiop; 1097 1098 if (sb == &so->so_snd) { 1099 opcode = LIO_WRITE; 1100 so->so_snd.sb_flags &= ~SB_AIO; 1101 } else { 1102 opcode = LIO_READ; 1103 so->so_rcv.sb_flags &= ~SB_AIO; 1104 } 1105 1106 for (cb = TAILQ_FIRST(&so->so_aiojobq); cb; cb = cbn) { 1107 cbn = TAILQ_NEXT(cb, list); 1108 if (opcode == cb->uaiocb.aio_lio_opcode) { 1109 p = cb->userproc; 1110 ki = p->p_aioinfo; 1111 TAILQ_REMOVE(&so->so_aiojobq, cb, list); 1112 TAILQ_REMOVE(&ki->kaio_sockqueue, cb, plist); 1113 TAILQ_INSERT_TAIL(&aio_jobs, cb, list); 1114 TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, cb, plist); 1115 wakecount++; 1116 if (cb->jobstate != JOBST_JOBQGLOBAL) 1117 panic("invalid queue value"); 1118 } 1119 } 1120 1121 while (wakecount--) { 1122 if ((aiop = TAILQ_FIRST(&aio_freeproc)) != 0) { 1123 TAILQ_REMOVE(&aio_freeproc, aiop, list); 1124 TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list); 1125 aiop->aioprocflags &= ~AIOP_FREE; 1126 wakeup(aiop->aioproc); 1127 } 1128 } 1129 #endif /* VFS_AIO */ 1130 } 1131 1132 #ifdef VFS_AIO 1133 /* 1134 * Queue a new AIO request. Choosing either the threaded or direct physio VCHR 1135 * technique is done in this code. 1136 */ 1137 static int 1138 _aio_aqueue(struct aiocb *job, struct aio_liojob *lj, int type) 1139 { 1140 struct proc *p = curproc; 1141 struct filedesc *fdp; 1142 struct file *fp; 1143 unsigned int fd; 1144 struct socket *so; 1145 int s; 1146 int error; 1147 int opcode, user_opcode; 1148 struct aiocblist *aiocbe; 1149 struct aioproclist *aiop; 1150 struct kaioinfo *ki; 1151 struct kevent kev; 1152 struct kqueue *kq; 1153 struct file *kq_fp; 1154 1155 if ((aiocbe = TAILQ_FIRST(&aio_freejobs)) != NULL) 1156 TAILQ_REMOVE(&aio_freejobs, aiocbe, list); 1157 else 1158 aiocbe = zalloc (aiocb_zone); 1159 1160 aiocbe->inputcharge = 0; 1161 aiocbe->outputcharge = 0; 1162 callout_handle_init(&aiocbe->timeouthandle); 1163 SLIST_INIT(&aiocbe->klist); 1164 1165 suword(&job->_aiocb_private.status, -1); 1166 suword(&job->_aiocb_private.error, 0); 1167 suword(&job->_aiocb_private.kernelinfo, -1); 1168 1169 error = copyin(job, &aiocbe->uaiocb, sizeof(aiocbe->uaiocb)); 1170 if (error) { 1171 suword(&job->_aiocb_private.error, error); 1172 TAILQ_INSERT_HEAD(&aio_freejobs, aiocbe, list); 1173 return error; 1174 } 1175 if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL && 1176 !_SIG_VALID(aiocbe->uaiocb.aio_sigevent.sigev_signo)) { 1177 TAILQ_INSERT_HEAD(&aio_freejobs, aiocbe, list); 1178 return EINVAL; 1179 } 1180 1181 /* Save userspace address of the job info. */ 1182 aiocbe->uuaiocb = job; 1183 1184 /* Get the opcode. */ 1185 user_opcode = aiocbe->uaiocb.aio_lio_opcode; 1186 if (type != LIO_NOP) 1187 aiocbe->uaiocb.aio_lio_opcode = type; 1188 opcode = aiocbe->uaiocb.aio_lio_opcode; 1189 1190 /* Get the fd info for process. */ 1191 fdp = p->p_fd; 1192 1193 /* 1194 * Range check file descriptor. 1195 */ 1196 fd = aiocbe->uaiocb.aio_fildes; 1197 if (fd >= fdp->fd_nfiles) { 1198 TAILQ_INSERT_HEAD(&aio_freejobs, aiocbe, list); 1199 if (type == 0) 1200 suword(&job->_aiocb_private.error, EBADF); 1201 return EBADF; 1202 } 1203 1204 fp = aiocbe->fd_file = fdp->fd_ofiles[fd]; 1205 if ((fp == NULL) || ((opcode == LIO_WRITE) && ((fp->f_flag & FWRITE) == 1206 0))) { 1207 TAILQ_INSERT_HEAD(&aio_freejobs, aiocbe, list); 1208 if (type == 0) 1209 suword(&job->_aiocb_private.error, EBADF); 1210 return EBADF; 1211 } 1212 fhold(fp); 1213 1214 if (aiocbe->uaiocb.aio_offset == -1LL) { 1215 error = EINVAL; 1216 goto aqueue_fail; 1217 } 1218 error = suword(&job->_aiocb_private.kernelinfo, jobrefid); 1219 if (error) { 1220 error = EINVAL; 1221 goto aqueue_fail; 1222 } 1223 aiocbe->uaiocb._aiocb_private.kernelinfo = (void *)(intptr_t)jobrefid; 1224 if (jobrefid == LONG_MAX) 1225 jobrefid = 1; 1226 else 1227 jobrefid++; 1228 1229 if (opcode == LIO_NOP) { 1230 fdrop(fp, p->p_thread); 1231 TAILQ_INSERT_HEAD(&aio_freejobs, aiocbe, list); 1232 if (type == 0) { 1233 suword(&job->_aiocb_private.error, 0); 1234 suword(&job->_aiocb_private.status, 0); 1235 suword(&job->_aiocb_private.kernelinfo, 0); 1236 } 1237 return 0; 1238 } 1239 if ((opcode != LIO_READ) && (opcode != LIO_WRITE)) { 1240 if (type == 0) 1241 suword(&job->_aiocb_private.status, 0); 1242 error = EINVAL; 1243 goto aqueue_fail; 1244 } 1245 1246 if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_KEVENT) { 1247 kev.ident = aiocbe->uaiocb.aio_sigevent.sigev_notify_kqueue; 1248 kev.udata = aiocbe->uaiocb.aio_sigevent.sigev_value.sigval_ptr; 1249 } 1250 else { 1251 /* 1252 * This method for requesting kevent-based notification won't 1253 * work on the alpha, since we're passing in a pointer 1254 * via aio_lio_opcode, which is an int. Use the SIGEV_KEVENT- 1255 * based method instead. 1256 */ 1257 if (user_opcode == LIO_NOP || user_opcode == LIO_READ || 1258 user_opcode == LIO_WRITE) 1259 goto no_kqueue; 1260 1261 error = copyin((struct kevent *)(uintptr_t)user_opcode, 1262 &kev, sizeof(kev)); 1263 if (error) 1264 goto aqueue_fail; 1265 } 1266 if ((u_int)kev.ident >= fdp->fd_nfiles || 1267 (kq_fp = fdp->fd_ofiles[kev.ident]) == NULL || 1268 (kq_fp->f_type != DTYPE_KQUEUE)) { 1269 error = EBADF; 1270 goto aqueue_fail; 1271 } 1272 kq = (struct kqueue *)kq_fp->f_data; 1273 kev.ident = (uintptr_t)aiocbe->uuaiocb; 1274 kev.filter = EVFILT_AIO; 1275 kev.flags = EV_ADD | EV_ENABLE | EV_FLAG1; 1276 kev.data = (intptr_t)aiocbe; 1277 error = kqueue_register(kq, &kev, p->p_thread); 1278 aqueue_fail: 1279 if (error) { 1280 fdrop(fp, p->p_thread); 1281 TAILQ_INSERT_HEAD(&aio_freejobs, aiocbe, list); 1282 if (type == 0) 1283 suword(&job->_aiocb_private.error, error); 1284 goto done; 1285 } 1286 no_kqueue: 1287 1288 suword(&job->_aiocb_private.error, EINPROGRESS); 1289 aiocbe->uaiocb._aiocb_private.error = EINPROGRESS; 1290 aiocbe->userproc = p; 1291 aiocbe->jobflags = 0; 1292 aiocbe->lio = lj; 1293 ki = p->p_aioinfo; 1294 1295 if (fp->f_type == DTYPE_SOCKET) { 1296 /* 1297 * Alternate queueing for socket ops: Reach down into the 1298 * descriptor to get the socket data. Then check to see if the 1299 * socket is ready to be read or written (based on the requested 1300 * operation). 1301 * 1302 * If it is not ready for io, then queue the aiocbe on the 1303 * socket, and set the flags so we get a call when sbnotify() 1304 * happens. 1305 */ 1306 so = (struct socket *)fp->f_data; 1307 s = splnet(); 1308 if (((opcode == LIO_READ) && (!soreadable(so))) || ((opcode == 1309 LIO_WRITE) && (!sowriteable(so)))) { 1310 TAILQ_INSERT_TAIL(&so->so_aiojobq, aiocbe, list); 1311 TAILQ_INSERT_TAIL(&ki->kaio_sockqueue, aiocbe, plist); 1312 if (opcode == LIO_READ) 1313 so->so_rcv.sb_flags |= SB_AIO; 1314 else 1315 so->so_snd.sb_flags |= SB_AIO; 1316 aiocbe->jobstate = JOBST_JOBQGLOBAL; /* XXX */ 1317 ki->kaio_queue_count++; 1318 num_queue_count++; 1319 splx(s); 1320 error = 0; 1321 goto done; 1322 } 1323 splx(s); 1324 } 1325 1326 if ((error = aio_qphysio(p, aiocbe)) == 0) 1327 goto done; 1328 if (error > 0) { 1329 suword(&job->_aiocb_private.status, 0); 1330 aiocbe->uaiocb._aiocb_private.error = error; 1331 suword(&job->_aiocb_private.error, error); 1332 goto done; 1333 } 1334 1335 /* No buffer for daemon I/O. */ 1336 aiocbe->bp = NULL; 1337 1338 ki->kaio_queue_count++; 1339 if (lj) 1340 lj->lioj_queue_count++; 1341 s = splnet(); 1342 TAILQ_INSERT_TAIL(&ki->kaio_jobqueue, aiocbe, plist); 1343 TAILQ_INSERT_TAIL(&aio_jobs, aiocbe, list); 1344 splx(s); 1345 aiocbe->jobstate = JOBST_JOBQGLOBAL; 1346 1347 num_queue_count++; 1348 error = 0; 1349 1350 /* 1351 * If we don't have a free AIO process, and we are below our quota, then 1352 * start one. Otherwise, depend on the subsequent I/O completions to 1353 * pick-up this job. If we don't sucessfully create the new process 1354 * (thread) due to resource issues, we return an error for now (EAGAIN), 1355 * which is likely not the correct thing to do. 1356 */ 1357 s = splnet(); 1358 retryproc: 1359 if ((aiop = TAILQ_FIRST(&aio_freeproc)) != NULL) { 1360 TAILQ_REMOVE(&aio_freeproc, aiop, list); 1361 TAILQ_INSERT_TAIL(&aio_activeproc, aiop, list); 1362 aiop->aioprocflags &= ~AIOP_FREE; 1363 wakeup(aiop->aioproc); 1364 } else if (((num_aio_resv_start + num_aio_procs) < max_aio_procs) && 1365 ((ki->kaio_active_count + num_aio_resv_start) < 1366 ki->kaio_maxactive_count)) { 1367 num_aio_resv_start++; 1368 if ((error = aio_newproc()) == 0) { 1369 num_aio_resv_start--; 1370 goto retryproc; 1371 } 1372 num_aio_resv_start--; 1373 } 1374 splx(s); 1375 done: 1376 return error; 1377 } 1378 1379 /* 1380 * This routine queues an AIO request, checking for quotas. 1381 */ 1382 static int 1383 aio_aqueue(struct aiocb *job, int type) 1384 { 1385 struct proc *p = curproc; 1386 struct kaioinfo *ki; 1387 1388 if (p->p_aioinfo == NULL) 1389 aio_init_aioinfo(p); 1390 1391 if (num_queue_count >= max_queue_count) 1392 return EAGAIN; 1393 1394 ki = p->p_aioinfo; 1395 if (ki->kaio_queue_count >= ki->kaio_qallowed_count) 1396 return EAGAIN; 1397 1398 return _aio_aqueue(job, NULL, type); 1399 } 1400 #endif /* VFS_AIO */ 1401 1402 /* 1403 * Support the aio_return system call, as a side-effect, kernel resources are 1404 * released. 1405 */ 1406 int 1407 aio_return(struct aio_return_args *uap) 1408 { 1409 #ifndef VFS_AIO 1410 return ENOSYS; 1411 #else 1412 struct proc *p = curproc; 1413 int s; 1414 long jobref; 1415 struct aiocblist *cb, *ncb; 1416 struct aiocb *ujob; 1417 struct kaioinfo *ki; 1418 1419 ki = p->p_aioinfo; 1420 if (ki == NULL) 1421 return EINVAL; 1422 1423 ujob = uap->aiocbp; 1424 1425 jobref = fuword(&ujob->_aiocb_private.kernelinfo); 1426 if (jobref == -1 || jobref == 0) 1427 return EINVAL; 1428 1429 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) { 1430 if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) == 1431 jobref) { 1432 if (ujob == cb->uuaiocb) { 1433 uap->sysmsg_result = 1434 cb->uaiocb._aiocb_private.status; 1435 } else 1436 uap->sysmsg_result = EFAULT; 1437 if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) { 1438 p->p_stats->p_ru.ru_oublock += 1439 cb->outputcharge; 1440 cb->outputcharge = 0; 1441 } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) { 1442 p->p_stats->p_ru.ru_inblock += cb->inputcharge; 1443 cb->inputcharge = 0; 1444 } 1445 aio_free_entry(cb); 1446 return 0; 1447 } 1448 } 1449 s = splbio(); 1450 for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = ncb) { 1451 ncb = TAILQ_NEXT(cb, plist); 1452 if (((intptr_t) cb->uaiocb._aiocb_private.kernelinfo) 1453 == jobref) { 1454 splx(s); 1455 if (ujob == cb->uuaiocb) { 1456 uap->sysmsg_result = 1457 cb->uaiocb._aiocb_private.status; 1458 } else 1459 uap->sysmsg_result = EFAULT; 1460 aio_free_entry(cb); 1461 return 0; 1462 } 1463 } 1464 splx(s); 1465 1466 return (EINVAL); 1467 #endif /* VFS_AIO */ 1468 } 1469 1470 /* 1471 * Allow a process to wakeup when any of the I/O requests are completed. 1472 */ 1473 int 1474 aio_suspend(struct aio_suspend_args *uap) 1475 { 1476 #ifndef VFS_AIO 1477 return ENOSYS; 1478 #else 1479 struct proc *p = curproc; 1480 struct timeval atv; 1481 struct timespec ts; 1482 struct aiocb *const *cbptr, *cbp; 1483 struct kaioinfo *ki; 1484 struct aiocblist *cb; 1485 int i; 1486 int njoblist; 1487 int error, s, timo; 1488 long *ijoblist; 1489 struct aiocb **ujoblist; 1490 1491 if (uap->nent > AIO_LISTIO_MAX) 1492 return EINVAL; 1493 1494 timo = 0; 1495 if (uap->timeout) { 1496 /* Get timespec struct. */ 1497 if ((error = copyin(uap->timeout, &ts, sizeof(ts))) != 0) 1498 return error; 1499 1500 if (ts.tv_nsec < 0 || ts.tv_nsec >= 1000000000) 1501 return (EINVAL); 1502 1503 TIMESPEC_TO_TIMEVAL(&atv, &ts); 1504 if (itimerfix(&atv)) 1505 return (EINVAL); 1506 timo = tvtohz(&atv); 1507 } 1508 1509 ki = p->p_aioinfo; 1510 if (ki == NULL) 1511 return EAGAIN; 1512 1513 njoblist = 0; 1514 ijoblist = zalloc(aiol_zone); 1515 ujoblist = zalloc(aiol_zone); 1516 cbptr = uap->aiocbp; 1517 1518 for (i = 0; i < uap->nent; i++) { 1519 cbp = (struct aiocb *)(intptr_t)fuword(&cbptr[i]); 1520 if (cbp == 0) 1521 continue; 1522 ujoblist[njoblist] = cbp; 1523 ijoblist[njoblist] = fuword(&cbp->_aiocb_private.kernelinfo); 1524 njoblist++; 1525 } 1526 1527 if (njoblist == 0) { 1528 zfree(aiol_zone, ijoblist); 1529 zfree(aiol_zone, ujoblist); 1530 return 0; 1531 } 1532 1533 error = 0; 1534 for (;;) { 1535 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) { 1536 for (i = 0; i < njoblist; i++) { 1537 if (((intptr_t) 1538 cb->uaiocb._aiocb_private.kernelinfo) == 1539 ijoblist[i]) { 1540 if (ujoblist[i] != cb->uuaiocb) 1541 error = EINVAL; 1542 zfree(aiol_zone, ijoblist); 1543 zfree(aiol_zone, ujoblist); 1544 return error; 1545 } 1546 } 1547 } 1548 1549 s = splbio(); 1550 for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = 1551 TAILQ_NEXT(cb, plist)) { 1552 for (i = 0; i < njoblist; i++) { 1553 if (((intptr_t) 1554 cb->uaiocb._aiocb_private.kernelinfo) == 1555 ijoblist[i]) { 1556 splx(s); 1557 if (ujoblist[i] != cb->uuaiocb) 1558 error = EINVAL; 1559 zfree(aiol_zone, ijoblist); 1560 zfree(aiol_zone, ujoblist); 1561 return error; 1562 } 1563 } 1564 } 1565 1566 ki->kaio_flags |= KAIO_WAKEUP; 1567 error = tsleep(p, PCATCH, "aiospn", timo); 1568 splx(s); 1569 1570 if (error == ERESTART || error == EINTR) { 1571 zfree(aiol_zone, ijoblist); 1572 zfree(aiol_zone, ujoblist); 1573 return EINTR; 1574 } else if (error == EWOULDBLOCK) { 1575 zfree(aiol_zone, ijoblist); 1576 zfree(aiol_zone, ujoblist); 1577 return EAGAIN; 1578 } 1579 } 1580 1581 /* NOTREACHED */ 1582 return EINVAL; 1583 #endif /* VFS_AIO */ 1584 } 1585 1586 /* 1587 * aio_cancel cancels any non-physio aio operations not currently in 1588 * progress. 1589 */ 1590 int 1591 aio_cancel(struct aio_cancel_args *uap) 1592 { 1593 #ifndef VFS_AIO 1594 return ENOSYS; 1595 #else 1596 struct proc *p = curproc; 1597 struct kaioinfo *ki; 1598 struct aiocblist *cbe, *cbn; 1599 struct file *fp; 1600 struct filedesc *fdp; 1601 struct socket *so; 1602 struct proc *po; 1603 int s,error; 1604 int cancelled=0; 1605 int notcancelled=0; 1606 struct vnode *vp; 1607 1608 fdp = p->p_fd; 1609 if ((u_int)uap->fd >= fdp->fd_nfiles || 1610 (fp = fdp->fd_ofiles[uap->fd]) == NULL) 1611 return (EBADF); 1612 1613 if (fp->f_type == DTYPE_VNODE) { 1614 vp = (struct vnode *)fp->f_data; 1615 1616 if (vn_isdisk(vp,&error)) { 1617 uap->sysmsg_result = AIO_NOTCANCELED; 1618 return 0; 1619 } 1620 } else if (fp->f_type == DTYPE_SOCKET) { 1621 so = (struct socket *)fp->f_data; 1622 1623 s = splnet(); 1624 1625 for (cbe = TAILQ_FIRST(&so->so_aiojobq); cbe; cbe = cbn) { 1626 cbn = TAILQ_NEXT(cbe, list); 1627 if ((uap->aiocbp == NULL) || 1628 (uap->aiocbp == cbe->uuaiocb) ) { 1629 po = cbe->userproc; 1630 ki = po->p_aioinfo; 1631 TAILQ_REMOVE(&so->so_aiojobq, cbe, list); 1632 TAILQ_REMOVE(&ki->kaio_sockqueue, cbe, plist); 1633 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, plist); 1634 if (ki->kaio_flags & KAIO_WAKEUP) { 1635 wakeup(po); 1636 } 1637 cbe->jobstate = JOBST_JOBFINISHED; 1638 cbe->uaiocb._aiocb_private.status=-1; 1639 cbe->uaiocb._aiocb_private.error=ECANCELED; 1640 cancelled++; 1641 /* XXX cancelled, knote? */ 1642 if (cbe->uaiocb.aio_sigevent.sigev_notify == 1643 SIGEV_SIGNAL) 1644 psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo); 1645 if (uap->aiocbp) 1646 break; 1647 } 1648 } 1649 splx(s); 1650 1651 if ((cancelled) && (uap->aiocbp)) { 1652 uap->sysmsg_result = AIO_CANCELED; 1653 return 0; 1654 } 1655 } 1656 ki=p->p_aioinfo; 1657 if (ki == NULL) 1658 goto done; 1659 s = splnet(); 1660 1661 for (cbe = TAILQ_FIRST(&ki->kaio_jobqueue); cbe; cbe = cbn) { 1662 cbn = TAILQ_NEXT(cbe, plist); 1663 1664 if ((uap->fd == cbe->uaiocb.aio_fildes) && 1665 ((uap->aiocbp == NULL ) || 1666 (uap->aiocbp == cbe->uuaiocb))) { 1667 1668 if (cbe->jobstate == JOBST_JOBQGLOBAL) { 1669 TAILQ_REMOVE(&aio_jobs, cbe, list); 1670 TAILQ_REMOVE(&ki->kaio_jobqueue, cbe, plist); 1671 TAILQ_INSERT_TAIL(&ki->kaio_jobdone, cbe, 1672 plist); 1673 cancelled++; 1674 ki->kaio_queue_finished_count++; 1675 cbe->jobstate = JOBST_JOBFINISHED; 1676 cbe->uaiocb._aiocb_private.status = -1; 1677 cbe->uaiocb._aiocb_private.error = ECANCELED; 1678 /* XXX cancelled, knote? */ 1679 if (cbe->uaiocb.aio_sigevent.sigev_notify == 1680 SIGEV_SIGNAL) 1681 psignal(cbe->userproc, cbe->uaiocb.aio_sigevent.sigev_signo); 1682 } else { 1683 notcancelled++; 1684 } 1685 } 1686 } 1687 splx(s); 1688 done: 1689 if (notcancelled) { 1690 uap->sysmsg_result = AIO_NOTCANCELED; 1691 return 0; 1692 } 1693 if (cancelled) { 1694 uap->sysmsg_result = AIO_CANCELED; 1695 return 0; 1696 } 1697 uap->sysmsg_result = AIO_ALLDONE; 1698 1699 return 0; 1700 #endif /* VFS_AIO */ 1701 } 1702 1703 /* 1704 * aio_error is implemented in the kernel level for compatibility purposes only. 1705 * For a user mode async implementation, it would be best to do it in a userland 1706 * subroutine. 1707 */ 1708 int 1709 aio_error(struct aio_error_args *uap) 1710 { 1711 #ifndef VFS_AIO 1712 return ENOSYS; 1713 #else 1714 struct proc *p = curproc; 1715 int s; 1716 struct aiocblist *cb; 1717 struct kaioinfo *ki; 1718 long jobref; 1719 1720 ki = p->p_aioinfo; 1721 if (ki == NULL) 1722 return EINVAL; 1723 1724 jobref = fuword(&uap->aiocbp->_aiocb_private.kernelinfo); 1725 if ((jobref == -1) || (jobref == 0)) 1726 return EINVAL; 1727 1728 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) { 1729 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1730 jobref) { 1731 uap->sysmsg_result = cb->uaiocb._aiocb_private.error; 1732 return 0; 1733 } 1734 } 1735 1736 s = splnet(); 1737 1738 for (cb = TAILQ_FIRST(&ki->kaio_jobqueue); cb; cb = TAILQ_NEXT(cb, 1739 plist)) { 1740 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1741 jobref) { 1742 uap->sysmsg_result = EINPROGRESS; 1743 splx(s); 1744 return 0; 1745 } 1746 } 1747 1748 for (cb = TAILQ_FIRST(&ki->kaio_sockqueue); cb; cb = TAILQ_NEXT(cb, 1749 plist)) { 1750 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1751 jobref) { 1752 uap->sysmsg_result = EINPROGRESS; 1753 splx(s); 1754 return 0; 1755 } 1756 } 1757 splx(s); 1758 1759 s = splbio(); 1760 for (cb = TAILQ_FIRST(&ki->kaio_bufdone); cb; cb = TAILQ_NEXT(cb, 1761 plist)) { 1762 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1763 jobref) { 1764 uap->sysmsg_result = cb->uaiocb._aiocb_private.error; 1765 splx(s); 1766 return 0; 1767 } 1768 } 1769 1770 for (cb = TAILQ_FIRST(&ki->kaio_bufqueue); cb; cb = TAILQ_NEXT(cb, 1771 plist)) { 1772 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) == 1773 jobref) { 1774 uap->sysmsg_result = EINPROGRESS; 1775 splx(s); 1776 return 0; 1777 } 1778 } 1779 splx(s); 1780 1781 #if (0) 1782 /* 1783 * Hack for lio. 1784 */ 1785 status = fuword(&uap->aiocbp->_aiocb_private.status); 1786 if (status == -1) 1787 return fuword(&uap->aiocbp->_aiocb_private.error); 1788 #endif 1789 return EINVAL; 1790 #endif /* VFS_AIO */ 1791 } 1792 1793 /* syscall - asynchronous read from a file (REALTIME) */ 1794 int 1795 aio_read(struct aio_read_args *uap) 1796 { 1797 #ifndef VFS_AIO 1798 return ENOSYS; 1799 #else 1800 return aio_aqueue(uap->aiocbp, LIO_READ); 1801 #endif /* VFS_AIO */ 1802 } 1803 1804 /* syscall - asynchronous write to a file (REALTIME) */ 1805 int 1806 aio_write(struct aio_write_args *uap) 1807 { 1808 #ifndef VFS_AIO 1809 return ENOSYS; 1810 #else 1811 return aio_aqueue(uap->aiocbp, LIO_WRITE); 1812 #endif /* VFS_AIO */ 1813 } 1814 1815 /* syscall - XXX undocumented */ 1816 int 1817 lio_listio(struct lio_listio_args *uap) 1818 { 1819 #ifndef VFS_AIO 1820 return ENOSYS; 1821 #else 1822 struct proc *p = curproc; 1823 int nent, nentqueued; 1824 struct aiocb *iocb, * const *cbptr; 1825 struct aiocblist *cb; 1826 struct kaioinfo *ki; 1827 struct aio_liojob *lj; 1828 int error, runningcode; 1829 int nerror; 1830 int i; 1831 int s; 1832 1833 if ((uap->mode != LIO_NOWAIT) && (uap->mode != LIO_WAIT)) 1834 return EINVAL; 1835 1836 nent = uap->nent; 1837 if (nent > AIO_LISTIO_MAX) 1838 return EINVAL; 1839 1840 if (p->p_aioinfo == NULL) 1841 aio_init_aioinfo(p); 1842 1843 if ((nent + num_queue_count) > max_queue_count) 1844 return EAGAIN; 1845 1846 ki = p->p_aioinfo; 1847 if ((nent + ki->kaio_queue_count) > ki->kaio_qallowed_count) 1848 return EAGAIN; 1849 1850 lj = zalloc(aiolio_zone); 1851 if (!lj) 1852 return EAGAIN; 1853 1854 lj->lioj_flags = 0; 1855 lj->lioj_buffer_count = 0; 1856 lj->lioj_buffer_finished_count = 0; 1857 lj->lioj_queue_count = 0; 1858 lj->lioj_queue_finished_count = 0; 1859 lj->lioj_ki = ki; 1860 1861 /* 1862 * Setup signal. 1863 */ 1864 if (uap->sig && (uap->mode == LIO_NOWAIT)) { 1865 error = copyin(uap->sig, &lj->lioj_signal, 1866 sizeof(lj->lioj_signal)); 1867 if (error) { 1868 zfree(aiolio_zone, lj); 1869 return error; 1870 } 1871 if (!_SIG_VALID(lj->lioj_signal.sigev_signo)) { 1872 zfree(aiolio_zone, lj); 1873 return EINVAL; 1874 } 1875 lj->lioj_flags |= LIOJ_SIGNAL; 1876 lj->lioj_flags &= ~LIOJ_SIGNAL_POSTED; 1877 } else 1878 lj->lioj_flags &= ~LIOJ_SIGNAL; 1879 1880 TAILQ_INSERT_TAIL(&ki->kaio_liojoblist, lj, lioj_list); 1881 /* 1882 * Get pointers to the list of I/O requests. 1883 */ 1884 nerror = 0; 1885 nentqueued = 0; 1886 cbptr = uap->acb_list; 1887 for (i = 0; i < uap->nent; i++) { 1888 iocb = (struct aiocb *)(intptr_t)fuword(&cbptr[i]); 1889 if (((intptr_t)iocb != -1) && ((intptr_t)iocb != 0)) { 1890 error = _aio_aqueue(iocb, lj, 0); 1891 if (error == 0) 1892 nentqueued++; 1893 else 1894 nerror++; 1895 } 1896 } 1897 1898 /* 1899 * If we haven't queued any, then just return error. 1900 */ 1901 if (nentqueued == 0) 1902 return 0; 1903 1904 /* 1905 * Calculate the appropriate error return. 1906 */ 1907 runningcode = 0; 1908 if (nerror) 1909 runningcode = EIO; 1910 1911 if (uap->mode == LIO_WAIT) { 1912 int command, found, jobref; 1913 1914 for (;;) { 1915 found = 0; 1916 for (i = 0; i < uap->nent; i++) { 1917 /* 1918 * Fetch address of the control buf pointer in 1919 * user space. 1920 */ 1921 iocb = (struct aiocb *) 1922 (intptr_t)fuword(&cbptr[i]); 1923 if (((intptr_t)iocb == -1) || ((intptr_t)iocb 1924 == 0)) 1925 continue; 1926 1927 /* 1928 * Fetch the associated command from user space. 1929 */ 1930 command = fuword(&iocb->aio_lio_opcode); 1931 if (command == LIO_NOP) { 1932 found++; 1933 continue; 1934 } 1935 1936 jobref = fuword(&iocb->_aiocb_private.kernelinfo); 1937 1938 TAILQ_FOREACH(cb, &ki->kaio_jobdone, plist) { 1939 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) 1940 == jobref) { 1941 if (cb->uaiocb.aio_lio_opcode 1942 == LIO_WRITE) { 1943 p->p_stats->p_ru.ru_oublock 1944 += 1945 cb->outputcharge; 1946 cb->outputcharge = 0; 1947 } else if (cb->uaiocb.aio_lio_opcode 1948 == LIO_READ) { 1949 p->p_stats->p_ru.ru_inblock 1950 += cb->inputcharge; 1951 cb->inputcharge = 0; 1952 } 1953 found++; 1954 break; 1955 } 1956 } 1957 1958 s = splbio(); 1959 TAILQ_FOREACH(cb, &ki->kaio_bufdone, plist) { 1960 if (((intptr_t)cb->uaiocb._aiocb_private.kernelinfo) 1961 == jobref) { 1962 found++; 1963 break; 1964 } 1965 } 1966 splx(s); 1967 } 1968 1969 /* 1970 * If all I/Os have been disposed of, then we can 1971 * return. 1972 */ 1973 if (found == nentqueued) 1974 return runningcode; 1975 1976 ki->kaio_flags |= KAIO_WAKEUP; 1977 error = tsleep(p, PCATCH, "aiospn", 0); 1978 1979 if (error == EINTR) 1980 return EINTR; 1981 else if (error == EWOULDBLOCK) 1982 return EAGAIN; 1983 } 1984 } 1985 1986 return runningcode; 1987 #endif /* VFS_AIO */ 1988 } 1989 1990 #ifdef VFS_AIO 1991 /* 1992 * This is a weird hack so that we can post a signal. It is safe to do so from 1993 * a timeout routine, but *not* from an interrupt routine. 1994 */ 1995 static void 1996 process_signal(void *aioj) 1997 { 1998 struct aiocblist *aiocbe = aioj; 1999 struct aio_liojob *lj = aiocbe->lio; 2000 struct aiocb *cb = &aiocbe->uaiocb; 2001 2002 if ((lj) && (lj->lioj_signal.sigev_notify == SIGEV_SIGNAL) && 2003 (lj->lioj_queue_count == lj->lioj_queue_finished_count)) { 2004 psignal(lj->lioj_ki->kaio_p, lj->lioj_signal.sigev_signo); 2005 lj->lioj_flags |= LIOJ_SIGNAL_POSTED; 2006 } 2007 2008 if (cb->aio_sigevent.sigev_notify == SIGEV_SIGNAL) 2009 psignal(aiocbe->userproc, cb->aio_sigevent.sigev_signo); 2010 } 2011 2012 /* 2013 * Interrupt handler for physio, performs the necessary process wakeups, and 2014 * signals. 2015 */ 2016 static void 2017 aio_physwakeup(struct buf *bp) 2018 { 2019 struct aiocblist *aiocbe; 2020 struct proc *p; 2021 struct kaioinfo *ki; 2022 struct aio_liojob *lj; 2023 2024 wakeup(bp); 2025 2026 aiocbe = (struct aiocblist *)bp->b_spc; 2027 if (aiocbe) { 2028 p = bp->b_caller1; 2029 2030 aiocbe->jobstate = JOBST_JOBBFINISHED; 2031 aiocbe->uaiocb._aiocb_private.status -= bp->b_resid; 2032 aiocbe->uaiocb._aiocb_private.error = 0; 2033 aiocbe->jobflags |= AIOCBLIST_DONE; 2034 2035 if (bp->b_flags & B_ERROR) 2036 aiocbe->uaiocb._aiocb_private.error = bp->b_error; 2037 2038 lj = aiocbe->lio; 2039 if (lj) { 2040 lj->lioj_buffer_finished_count++; 2041 2042 /* 2043 * wakeup/signal if all of the interrupt jobs are done. 2044 */ 2045 if (lj->lioj_buffer_finished_count == 2046 lj->lioj_buffer_count) { 2047 /* 2048 * Post a signal if it is called for. 2049 */ 2050 if ((lj->lioj_flags & 2051 (LIOJ_SIGNAL|LIOJ_SIGNAL_POSTED)) == 2052 LIOJ_SIGNAL) { 2053 lj->lioj_flags |= LIOJ_SIGNAL_POSTED; 2054 aiocbe->timeouthandle = 2055 timeout(process_signal, 2056 aiocbe, 0); 2057 } 2058 } 2059 } 2060 2061 ki = p->p_aioinfo; 2062 if (ki) { 2063 ki->kaio_buffer_finished_count++; 2064 TAILQ_REMOVE(&aio_bufjobs, aiocbe, list); 2065 TAILQ_REMOVE(&ki->kaio_bufqueue, aiocbe, plist); 2066 TAILQ_INSERT_TAIL(&ki->kaio_bufdone, aiocbe, plist); 2067 2068 KNOTE(&aiocbe->klist, 0); 2069 /* Do the wakeup. */ 2070 if (ki->kaio_flags & (KAIO_RUNDOWN|KAIO_WAKEUP)) { 2071 ki->kaio_flags &= ~KAIO_WAKEUP; 2072 wakeup(p); 2073 } 2074 } 2075 2076 if (aiocbe->uaiocb.aio_sigevent.sigev_notify == SIGEV_SIGNAL) 2077 aiocbe->timeouthandle = 2078 timeout(process_signal, aiocbe, 0); 2079 } 2080 } 2081 #endif /* VFS_AIO */ 2082 2083 /* syscall - wait for the next completion of an aio request */ 2084 int 2085 aio_waitcomplete(struct aio_waitcomplete_args *uap) 2086 { 2087 #ifndef VFS_AIO 2088 return ENOSYS; 2089 #else 2090 struct proc *p = curproc; 2091 struct timeval atv; 2092 struct timespec ts; 2093 struct kaioinfo *ki; 2094 struct aiocblist *cb = NULL; 2095 int error, s, timo; 2096 2097 suword(uap->aiocbp, (int)NULL); 2098 2099 timo = 0; 2100 if (uap->timeout) { 2101 /* Get timespec struct. */ 2102 error = copyin(uap->timeout, &ts, sizeof(ts)); 2103 if (error) 2104 return error; 2105 2106 if ((ts.tv_nsec < 0) || (ts.tv_nsec >= 1000000000)) 2107 return (EINVAL); 2108 2109 TIMESPEC_TO_TIMEVAL(&atv, &ts); 2110 if (itimerfix(&atv)) 2111 return (EINVAL); 2112 timo = tvtohz(&atv); 2113 } 2114 2115 ki = p->p_aioinfo; 2116 if (ki == NULL) 2117 return EAGAIN; 2118 2119 for (;;) { 2120 if ((cb = TAILQ_FIRST(&ki->kaio_jobdone)) != 0) { 2121 suword(uap->aiocbp, (uintptr_t)cb->uuaiocb); 2122 uap->sysmsg_result = cb->uaiocb._aiocb_private.status; 2123 if (cb->uaiocb.aio_lio_opcode == LIO_WRITE) { 2124 p->p_stats->p_ru.ru_oublock += 2125 cb->outputcharge; 2126 cb->outputcharge = 0; 2127 } else if (cb->uaiocb.aio_lio_opcode == LIO_READ) { 2128 p->p_stats->p_ru.ru_inblock += cb->inputcharge; 2129 cb->inputcharge = 0; 2130 } 2131 aio_free_entry(cb); 2132 return cb->uaiocb._aiocb_private.error; 2133 } 2134 2135 s = splbio(); 2136 if ((cb = TAILQ_FIRST(&ki->kaio_bufdone)) != 0 ) { 2137 splx(s); 2138 suword(uap->aiocbp, (uintptr_t)cb->uuaiocb); 2139 uap->sysmsg_result = cb->uaiocb._aiocb_private.status; 2140 aio_free_entry(cb); 2141 return cb->uaiocb._aiocb_private.error; 2142 } 2143 2144 ki->kaio_flags |= KAIO_WAKEUP; 2145 error = tsleep(p, PCATCH, "aiowc", timo); 2146 splx(s); 2147 2148 if (error == ERESTART) 2149 return EINTR; 2150 else if (error < 0) 2151 return error; 2152 else if (error == EINTR) 2153 return EINTR; 2154 else if (error == EWOULDBLOCK) 2155 return EAGAIN; 2156 } 2157 #endif /* VFS_AIO */ 2158 } 2159 2160 #ifndef VFS_AIO 2161 static int 2162 filt_aioattach(struct knote *kn) 2163 { 2164 2165 return (ENXIO); 2166 } 2167 2168 struct filterops aio_filtops = 2169 { 0, filt_aioattach, NULL, NULL }; 2170 2171 #else 2172 /* kqueue attach function */ 2173 static int 2174 filt_aioattach(struct knote *kn) 2175 { 2176 struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata; 2177 2178 /* 2179 * The aiocbe pointer must be validated before using it, so 2180 * registration is restricted to the kernel; the user cannot 2181 * set EV_FLAG1. 2182 */ 2183 if ((kn->kn_flags & EV_FLAG1) == 0) 2184 return (EPERM); 2185 kn->kn_flags &= ~EV_FLAG1; 2186 2187 SLIST_INSERT_HEAD(&aiocbe->klist, kn, kn_selnext); 2188 2189 return (0); 2190 } 2191 2192 /* kqueue detach function */ 2193 static void 2194 filt_aiodetach(struct knote *kn) 2195 { 2196 struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata; 2197 2198 SLIST_REMOVE(&aiocbe->klist, kn, knote, kn_selnext); 2199 } 2200 2201 /* kqueue filter function */ 2202 /*ARGSUSED*/ 2203 static int 2204 filt_aio(struct knote *kn, long hint) 2205 { 2206 struct aiocblist *aiocbe = (struct aiocblist *)kn->kn_sdata; 2207 2208 kn->kn_data = aiocbe->uaiocb._aiocb_private.error; 2209 if (aiocbe->jobstate != JOBST_JOBFINISHED && 2210 aiocbe->jobstate != JOBST_JOBBFINISHED) 2211 return (0); 2212 kn->kn_flags |= EV_EOF; 2213 return (1); 2214 } 2215 2216 struct filterops aio_filtops = 2217 { 0, filt_aioattach, filt_aiodetach, filt_aio }; 2218 #endif /* VFS_AIO */ 2219