xref: /freebsd/sys/dev/usb/usb_dev.c (revision e17f5b1d)
1 /* $FreeBSD$ */
2 /*-
3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4  *
5  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  *
29  * usb_dev.c - An abstraction layer for creating devices under /dev/...
30  */
31 
32 #ifdef USB_GLOBAL_INCLUDE_FILE
33 #include USB_GLOBAL_INCLUDE_FILE
34 #else
35 #include <sys/stdint.h>
36 #include <sys/stddef.h>
37 #include <sys/param.h>
38 #include <sys/queue.h>
39 #include <sys/types.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/bus.h>
43 #include <sys/module.h>
44 #include <sys/lock.h>
45 #include <sys/mutex.h>
46 #include <sys/condvar.h>
47 #include <sys/sysctl.h>
48 #include <sys/sx.h>
49 #include <sys/unistd.h>
50 #include <sys/callout.h>
51 #include <sys/malloc.h>
52 #include <sys/priv.h>
53 #include <sys/vnode.h>
54 #include <sys/conf.h>
55 #include <sys/fcntl.h>
56 
57 #include <dev/usb/usb.h>
58 #include <dev/usb/usb_ioctl.h>
59 #include <dev/usb/usbdi.h>
60 #include <dev/usb/usbdi_util.h>
61 
62 #define	USB_DEBUG_VAR usb_fifo_debug
63 
64 #include <dev/usb/usb_core.h>
65 #include <dev/usb/usb_dev.h>
66 #include <dev/usb/usb_mbuf.h>
67 #include <dev/usb/usb_process.h>
68 #include <dev/usb/usb_device.h>
69 #include <dev/usb/usb_debug.h>
70 #include <dev/usb/usb_busdma.h>
71 #include <dev/usb/usb_generic.h>
72 #include <dev/usb/usb_dynamic.h>
73 #include <dev/usb/usb_util.h>
74 
75 #include <dev/usb/usb_controller.h>
76 #include <dev/usb/usb_bus.h>
77 
78 #include <sys/filio.h>
79 #include <sys/ttycom.h>
80 #include <sys/syscallsubr.h>
81 
82 #include <machine/stdarg.h>
83 #endif			/* USB_GLOBAL_INCLUDE_FILE */
84 
85 #if USB_HAVE_UGEN
86 
87 #ifdef USB_DEBUG
88 static int usb_fifo_debug = 0;
89 
90 static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
91     "USB device");
92 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RWTUN,
93     &usb_fifo_debug, 0, "Debug Level");
94 #endif
95 
96 #if ((__FreeBSD_version >= 700001) || (__FreeBSD_version == 0) || \
97      ((__FreeBSD_version >= 600034) && (__FreeBSD_version < 700000)))
98 #define	USB_UCRED struct ucred *ucred,
99 #else
100 #define	USB_UCRED
101 #endif
102 
103 /* prototypes */
104 
105 static int	usb_fifo_open(struct usb_cdev_privdata *,
106 		    struct usb_fifo *, int);
107 static void	usb_fifo_close(struct usb_fifo *, int);
108 static void	usb_dev_init(void *);
109 static void	usb_dev_init_post(void *);
110 static void	usb_dev_uninit(void *);
111 static int	usb_fifo_uiomove(struct usb_fifo *, void *, int,
112 		    struct uio *);
113 static void	usb_fifo_check_methods(struct usb_fifo_methods *);
114 static struct	usb_fifo *usb_fifo_alloc(struct mtx *);
115 static struct	usb_endpoint *usb_dev_get_ep(struct usb_device *, uint8_t,
116 		    uint8_t);
117 static void	usb_loc_fill(struct usb_fs_privdata *,
118 		    struct usb_cdev_privdata *);
119 static void	usb_close(void *);
120 static usb_error_t usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *, int);
121 static usb_error_t usb_usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
122 static void	usb_unref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
123 
124 static d_open_t usb_open;
125 static d_ioctl_t usb_ioctl;
126 static d_read_t usb_read;
127 static d_write_t usb_write;
128 static d_poll_t usb_poll;
129 static d_kqfilter_t usb_kqfilter;
130 
131 static d_ioctl_t usb_static_ioctl;
132 
133 static usb_fifo_open_t usb_fifo_dummy_open;
134 static usb_fifo_close_t usb_fifo_dummy_close;
135 static usb_fifo_ioctl_t usb_fifo_dummy_ioctl;
136 static usb_fifo_cmd_t usb_fifo_dummy_cmd;
137 
138 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
139 struct cdevsw usb_devsw = {
140 	.d_version = D_VERSION,
141 	.d_open = usb_open,
142 	.d_ioctl = usb_ioctl,
143 	.d_name = "usbdev",
144 	.d_flags = D_TRACKCLOSE,
145 	.d_read = usb_read,
146 	.d_write = usb_write,
147 	.d_poll = usb_poll,
148 	.d_kqfilter = usb_kqfilter,
149 };
150 
151 static struct cdev* usb_dev = NULL;
152 
153 /* character device structure used for /dev/usb */
154 static struct cdevsw usb_static_devsw = {
155 	.d_version = D_VERSION,
156 	.d_ioctl = usb_static_ioctl,
157 	.d_name = "usb"
158 };
159 
160 static TAILQ_HEAD(, usb_symlink) usb_sym_head;
161 static struct sx usb_sym_lock;
162 
163 struct mtx usb_ref_lock;
164 
165 /*------------------------------------------------------------------------*
166  *	usb_loc_fill
167  *
168  * This is used to fill out a usb_cdev_privdata structure based on the
169  * device's address as contained in usb_fs_privdata.
170  *------------------------------------------------------------------------*/
171 static void
172 usb_loc_fill(struct usb_fs_privdata* pd, struct usb_cdev_privdata *cpd)
173 {
174 	cpd->bus_index = pd->bus_index;
175 	cpd->dev_index = pd->dev_index;
176 	cpd->ep_addr = pd->ep_addr;
177 	cpd->fifo_index = pd->fifo_index;
178 }
179 
180 /*------------------------------------------------------------------------*
181  *	usb_ref_device
182  *
183  * This function is used to atomically refer an USB device by its
184  * device location. If this function returns success the USB device
185  * will not disappear until the USB device is unreferenced.
186  *
187  * Return values:
188  *  0: Success, refcount incremented on the given USB device.
189  *  Else: Failure.
190  *------------------------------------------------------------------------*/
191 static usb_error_t
192 usb_ref_device(struct usb_cdev_privdata *cpd,
193     struct usb_cdev_refdata *crd, int need_uref)
194 {
195 	struct usb_fifo **ppf;
196 	struct usb_fifo *f;
197 
198 	DPRINTFN(2, "cpd=%p need uref=%d\n", cpd, need_uref);
199 
200 	/* clear all refs */
201 	memset(crd, 0, sizeof(*crd));
202 
203 	mtx_lock(&usb_ref_lock);
204 	cpd->bus = devclass_get_softc(usb_devclass_ptr, cpd->bus_index);
205 	if (cpd->bus == NULL) {
206 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
207 		goto error;
208 	}
209 	cpd->udev = cpd->bus->devices[cpd->dev_index];
210 	if (cpd->udev == NULL) {
211 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
212 		goto error;
213 	}
214 	if (cpd->udev->state == USB_STATE_DETACHED &&
215 	    (need_uref != 2)) {
216 		DPRINTFN(2, "device is detached\n");
217 		goto error;
218 	}
219 	if (need_uref) {
220 		DPRINTFN(2, "ref udev - needed\n");
221 
222 		if (cpd->udev->refcount == USB_DEV_REF_MAX) {
223 			DPRINTFN(2, "no dev ref\n");
224 			goto error;
225 		}
226 		cpd->udev->refcount++;
227 
228 		mtx_unlock(&usb_ref_lock);
229 
230 		/*
231 		 * We need to grab the enumeration SX-lock before
232 		 * grabbing the FIFO refs to avoid deadlock at detach!
233 		 */
234 		crd->do_unlock = usbd_enum_lock_sig(cpd->udev);
235 
236 		mtx_lock(&usb_ref_lock);
237 
238 		/*
239 		 * Set "is_uref" after grabbing the default SX lock
240 		 */
241 		crd->is_uref = 1;
242 
243 		/* check for signal */
244 		if (crd->do_unlock > 1) {
245 			crd->do_unlock = 0;
246 			goto error;
247 		}
248 	}
249 
250 	/* check if we are doing an open */
251 	if (cpd->fflags == 0) {
252 		/* use zero defaults */
253 	} else {
254 		/* check for write */
255 		if (cpd->fflags & FWRITE) {
256 			ppf = cpd->udev->fifo;
257 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
258 			crd->txfifo = f;
259 			crd->is_write = 1;	/* ref */
260 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
261 				goto error;
262 			if (f->curr_cpd != cpd)
263 				goto error;
264 			/* check if USB-FS is active */
265 			if (f->fs_ep_max != 0) {
266 				crd->is_usbfs = 1;
267 			}
268 		}
269 
270 		/* check for read */
271 		if (cpd->fflags & FREAD) {
272 			ppf = cpd->udev->fifo;
273 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
274 			crd->rxfifo = f;
275 			crd->is_read = 1;	/* ref */
276 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
277 				goto error;
278 			if (f->curr_cpd != cpd)
279 				goto error;
280 			/* check if USB-FS is active */
281 			if (f->fs_ep_max != 0) {
282 				crd->is_usbfs = 1;
283 			}
284 		}
285 	}
286 
287 	/* when everything is OK we increment the refcounts */
288 	if (crd->is_write) {
289 		DPRINTFN(2, "ref write\n");
290 		crd->txfifo->refcount++;
291 	}
292 	if (crd->is_read) {
293 		DPRINTFN(2, "ref read\n");
294 		crd->rxfifo->refcount++;
295 	}
296 	mtx_unlock(&usb_ref_lock);
297 
298 	return (0);
299 
300 error:
301 	if (crd->do_unlock)
302 		usbd_enum_unlock(cpd->udev);
303 
304 	if (crd->is_uref) {
305 		if (--(cpd->udev->refcount) == 0)
306 			cv_broadcast(&cpd->udev->ref_cv);
307 	}
308 	mtx_unlock(&usb_ref_lock);
309 	DPRINTFN(2, "fail\n");
310 
311 	/* clear all refs */
312 	memset(crd, 0, sizeof(*crd));
313 
314 	return (USB_ERR_INVAL);
315 }
316 
317 /*------------------------------------------------------------------------*
318  *	usb_usb_ref_device
319  *
320  * This function is used to upgrade an USB reference to include the
321  * USB device reference on a USB location.
322  *
323  * Return values:
324  *  0: Success, refcount incremented on the given USB device.
325  *  Else: Failure.
326  *------------------------------------------------------------------------*/
327 static usb_error_t
328 usb_usb_ref_device(struct usb_cdev_privdata *cpd,
329     struct usb_cdev_refdata *crd)
330 {
331 	/*
332 	 * Check if we already got an USB reference on this location:
333 	 */
334 	if (crd->is_uref)
335 		return (0);		/* success */
336 
337 	/*
338 	 * To avoid deadlock at detach we need to drop the FIFO ref
339 	 * and re-acquire a new ref!
340 	 */
341 	usb_unref_device(cpd, crd);
342 
343 	return (usb_ref_device(cpd, crd, 1 /* need uref */));
344 }
345 
346 /*------------------------------------------------------------------------*
347  *	usb_unref_device
348  *
349  * This function will release the reference count by one unit for the
350  * given USB device.
351  *------------------------------------------------------------------------*/
352 static void
353 usb_unref_device(struct usb_cdev_privdata *cpd,
354     struct usb_cdev_refdata *crd)
355 {
356 
357 	DPRINTFN(2, "cpd=%p is_uref=%d\n", cpd, crd->is_uref);
358 
359 	if (crd->do_unlock)
360 		usbd_enum_unlock(cpd->udev);
361 
362 	mtx_lock(&usb_ref_lock);
363 	if (crd->is_read) {
364 		if (--(crd->rxfifo->refcount) == 0) {
365 			cv_signal(&crd->rxfifo->cv_drain);
366 		}
367 		crd->is_read = 0;
368 	}
369 	if (crd->is_write) {
370 		if (--(crd->txfifo->refcount) == 0) {
371 			cv_signal(&crd->txfifo->cv_drain);
372 		}
373 		crd->is_write = 0;
374 	}
375 	if (crd->is_uref) {
376 		crd->is_uref = 0;
377 		if (--(cpd->udev->refcount) == 0)
378 			cv_broadcast(&cpd->udev->ref_cv);
379 	}
380 	mtx_unlock(&usb_ref_lock);
381 }
382 
383 static struct usb_fifo *
384 usb_fifo_alloc(struct mtx *mtx)
385 {
386 	struct usb_fifo *f;
387 
388 	f = malloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
389 	if (f != NULL) {
390 		cv_init(&f->cv_io, "FIFO-IO");
391 		cv_init(&f->cv_drain, "FIFO-DRAIN");
392 		f->priv_mtx = mtx;
393 		f->refcount = 1;
394 		knlist_init_mtx(&f->selinfo.si_note, mtx);
395 	}
396 	return (f);
397 }
398 
399 /*------------------------------------------------------------------------*
400  *	usb_fifo_create
401  *------------------------------------------------------------------------*/
402 static int
403 usb_fifo_create(struct usb_cdev_privdata *cpd,
404     struct usb_cdev_refdata *crd)
405 {
406 	struct usb_device *udev = cpd->udev;
407 	struct usb_fifo *f;
408 	struct usb_endpoint *ep;
409 	uint8_t n;
410 	uint8_t is_tx;
411 	uint8_t is_rx;
412 	uint8_t no_null;
413 	uint8_t is_busy;
414 	int e = cpd->ep_addr;
415 
416 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
417 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
418 	no_null = 1;
419 	is_busy = 0;
420 
421 	/* Preallocated FIFO */
422 	if (e < 0) {
423 		DPRINTFN(5, "Preallocated FIFO\n");
424 		if (is_tx) {
425 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
426 			if (f == NULL)
427 				return (EINVAL);
428 			crd->txfifo = f;
429 		}
430 		if (is_rx) {
431 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
432 			if (f == NULL)
433 				return (EINVAL);
434 			crd->rxfifo = f;
435 		}
436 		return (0);
437 	}
438 
439 	KASSERT(e >= 0 && e <= 15, ("endpoint %d out of range", e));
440 
441 	/* search for a free FIFO slot */
442 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", e);
443 	for (n = 0;; n += 2) {
444 
445 		if (n == USB_FIFO_MAX) {
446 			if (no_null) {
447 				no_null = 0;
448 				n = 0;
449 			} else {
450 				/* end of FIFOs reached */
451 				DPRINTFN(5, "out of FIFOs\n");
452 				return (ENOMEM);
453 			}
454 		}
455 		/* Check for TX FIFO */
456 		if (is_tx) {
457 			f = udev->fifo[n + USB_FIFO_TX];
458 			if (f != NULL) {
459 				if (f->dev_ep_index != e) {
460 					/* wrong endpoint index */
461 					continue;
462 				}
463 				if (f->curr_cpd != NULL) {
464 					/* FIFO is opened */
465 					is_busy = 1;
466 					continue;
467 				}
468 			} else if (no_null) {
469 				continue;
470 			}
471 		}
472 		/* Check for RX FIFO */
473 		if (is_rx) {
474 			f = udev->fifo[n + USB_FIFO_RX];
475 			if (f != NULL) {
476 				if (f->dev_ep_index != e) {
477 					/* wrong endpoint index */
478 					continue;
479 				}
480 				if (f->curr_cpd != NULL) {
481 					/* FIFO is opened */
482 					is_busy = 1;
483 					continue;
484 				}
485 			} else if (no_null) {
486 				continue;
487 			}
488 		}
489 		break;
490 	}
491 
492 	if (no_null == 0) {
493 		if (e >= (USB_EP_MAX / 2)) {
494 			/* we don't create any endpoints in this range */
495 			DPRINTFN(5, "ep out of range\n");
496 			return (is_busy ? EBUSY : EINVAL);
497 		}
498 	}
499 
500 	if ((e != 0) && is_busy) {
501 		/*
502 		 * Only the default control endpoint is allowed to be
503 		 * opened multiple times!
504 		 */
505 		DPRINTFN(5, "busy\n");
506 		return (EBUSY);
507 	}
508 
509 	/* Check TX FIFO */
510 	if (is_tx &&
511 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
512 		ep = usb_dev_get_ep(udev, e, USB_FIFO_TX);
513 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_TX);
514 		if (ep == NULL) {
515 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
516 			return (EINVAL);
517 		}
518 		f = usb_fifo_alloc(&udev->device_mtx);
519 		if (f == NULL) {
520 			DPRINTFN(5, "could not alloc tx fifo\n");
521 			return (ENOMEM);
522 		}
523 		/* update some fields */
524 		f->fifo_index = n + USB_FIFO_TX;
525 		f->dev_ep_index = e;
526 		f->priv_sc0 = ep;
527 		f->methods = &usb_ugen_methods;
528 		f->iface_index = ep->iface_index;
529 		f->udev = udev;
530 		mtx_lock(&usb_ref_lock);
531 		udev->fifo[n + USB_FIFO_TX] = f;
532 		mtx_unlock(&usb_ref_lock);
533 	}
534 	/* Check RX FIFO */
535 	if (is_rx &&
536 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
537 
538 		ep = usb_dev_get_ep(udev, e, USB_FIFO_RX);
539 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_RX);
540 		if (ep == NULL) {
541 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
542 			return (EINVAL);
543 		}
544 		f = usb_fifo_alloc(&udev->device_mtx);
545 		if (f == NULL) {
546 			DPRINTFN(5, "could not alloc rx fifo\n");
547 			return (ENOMEM);
548 		}
549 		/* update some fields */
550 		f->fifo_index = n + USB_FIFO_RX;
551 		f->dev_ep_index = e;
552 		f->priv_sc0 = ep;
553 		f->methods = &usb_ugen_methods;
554 		f->iface_index = ep->iface_index;
555 		f->udev = udev;
556 		mtx_lock(&usb_ref_lock);
557 		udev->fifo[n + USB_FIFO_RX] = f;
558 		mtx_unlock(&usb_ref_lock);
559 	}
560 	if (is_tx) {
561 		crd->txfifo = udev->fifo[n + USB_FIFO_TX];
562 	}
563 	if (is_rx) {
564 		crd->rxfifo = udev->fifo[n + USB_FIFO_RX];
565 	}
566 	/* fill out fifo index */
567 	DPRINTFN(5, "fifo index = %d\n", n);
568 	cpd->fifo_index = n;
569 
570 	/* complete */
571 
572 	return (0);
573 }
574 
575 void
576 usb_fifo_free(struct usb_fifo *f)
577 {
578 	uint8_t n;
579 
580 	if (f == NULL) {
581 		/* be NULL safe */
582 		return;
583 	}
584 	/* destroy symlink devices, if any */
585 	for (n = 0; n != 2; n++) {
586 		if (f->symlink[n]) {
587 			usb_free_symlink(f->symlink[n]);
588 			f->symlink[n] = NULL;
589 		}
590 	}
591 	mtx_lock(&usb_ref_lock);
592 
593 	/* delink ourselves to stop calls from userland */
594 	if ((f->fifo_index < USB_FIFO_MAX) &&
595 	    (f->udev != NULL) &&
596 	    (f->udev->fifo[f->fifo_index] == f)) {
597 		f->udev->fifo[f->fifo_index] = NULL;
598 	} else {
599 		DPRINTFN(0, "USB FIFO %p has not been linked\n", f);
600 	}
601 
602 	/* decrease refcount */
603 	f->refcount--;
604 	/* need to wait until all callers have exited */
605 	while (f->refcount != 0) {
606 		mtx_unlock(&usb_ref_lock);	/* avoid LOR */
607 		mtx_lock(f->priv_mtx);
608 		/* prevent write flush, if any */
609 		f->flag_iserror = 1;
610 		/* get I/O thread out of any sleep state */
611 		if (f->flag_sleeping) {
612 			f->flag_sleeping = 0;
613 			cv_broadcast(&f->cv_io);
614 		}
615 		mtx_unlock(f->priv_mtx);
616 		mtx_lock(&usb_ref_lock);
617 
618 		/*
619 		 * Check if the "f->refcount" variable reached zero
620 		 * during the unlocked time before entering wait:
621 		 */
622 		if (f->refcount == 0)
623 			break;
624 
625 		/* wait for sync */
626 		cv_wait(&f->cv_drain, &usb_ref_lock);
627 	}
628 	mtx_unlock(&usb_ref_lock);
629 
630 	/* take care of closing the device here, if any */
631 	usb_fifo_close(f, 0);
632 
633 	cv_destroy(&f->cv_io);
634 	cv_destroy(&f->cv_drain);
635 
636 	knlist_clear(&f->selinfo.si_note, 0);
637 	seldrain(&f->selinfo);
638 	knlist_destroy(&f->selinfo.si_note);
639 
640 	free(f, M_USBDEV);
641 }
642 
643 static struct usb_endpoint *
644 usb_dev_get_ep(struct usb_device *udev, uint8_t ep_index, uint8_t dir)
645 {
646 	struct usb_endpoint *ep;
647 	uint8_t ep_dir;
648 
649 	if (ep_index == 0) {
650 		ep = &udev->ctrl_ep;
651 	} else {
652 		if (dir == USB_FIFO_RX) {
653 			if (udev->flags.usb_mode == USB_MODE_HOST) {
654 				ep_dir = UE_DIR_IN;
655 			} else {
656 				ep_dir = UE_DIR_OUT;
657 			}
658 		} else {
659 			if (udev->flags.usb_mode == USB_MODE_HOST) {
660 				ep_dir = UE_DIR_OUT;
661 			} else {
662 				ep_dir = UE_DIR_IN;
663 			}
664 		}
665 		ep = usbd_get_ep_by_addr(udev, ep_index | ep_dir);
666 	}
667 
668 	if (ep == NULL) {
669 		/* if the endpoint does not exist then return */
670 		return (NULL);
671 	}
672 	if (ep->edesc == NULL) {
673 		/* invalid endpoint */
674 		return (NULL);
675 	}
676 	return (ep);			/* success */
677 }
678 
679 /*------------------------------------------------------------------------*
680  *	usb_fifo_open
681  *
682  * Returns:
683  * 0: Success
684  * Else: Failure
685  *------------------------------------------------------------------------*/
686 static int
687 usb_fifo_open(struct usb_cdev_privdata *cpd,
688     struct usb_fifo *f, int fflags)
689 {
690 	int err;
691 
692 	if (f == NULL) {
693 		/* no FIFO there */
694 		DPRINTFN(2, "no FIFO\n");
695 		return (ENXIO);
696 	}
697 	/* remove FWRITE and FREAD flags */
698 	fflags &= ~(FWRITE | FREAD);
699 
700 	/* set correct file flags */
701 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
702 		fflags |= FWRITE;
703 	} else {
704 		fflags |= FREAD;
705 	}
706 
707 	/* check if we are already opened */
708 	/* we don't need any locks when checking this variable */
709 	if (f->curr_cpd != NULL) {
710 		err = EBUSY;
711 		goto done;
712 	}
713 
714 	/* reset short flag before open */
715 	f->flag_short = 0;
716 
717 	/* call open method */
718 	err = (f->methods->f_open) (f, fflags);
719 	if (err) {
720 		goto done;
721 	}
722 	mtx_lock(f->priv_mtx);
723 
724 	/* reset sleep flag */
725 	f->flag_sleeping = 0;
726 
727 	/* reset error flag */
728 	f->flag_iserror = 0;
729 
730 	/* reset complete flag */
731 	f->flag_iscomplete = 0;
732 
733 	/* reset select flag */
734 	f->flag_isselect = 0;
735 
736 	/* reset flushing flag */
737 	f->flag_flushing = 0;
738 
739 	/* reset ASYNC proc flag */
740 	f->async_p = NULL;
741 
742 	mtx_lock(&usb_ref_lock);
743 	/* flag the fifo as opened to prevent others */
744 	f->curr_cpd = cpd;
745 	mtx_unlock(&usb_ref_lock);
746 
747 	/* reset queue */
748 	usb_fifo_reset(f);
749 
750 	mtx_unlock(f->priv_mtx);
751 done:
752 	return (err);
753 }
754 
755 /*------------------------------------------------------------------------*
756  *	usb_fifo_reset
757  *------------------------------------------------------------------------*/
758 void
759 usb_fifo_reset(struct usb_fifo *f)
760 {
761 	struct usb_mbuf *m;
762 
763 	if (f == NULL) {
764 		return;
765 	}
766 	while (1) {
767 		USB_IF_DEQUEUE(&f->used_q, m);
768 		if (m) {
769 			USB_IF_ENQUEUE(&f->free_q, m);
770 		} else {
771 			break;
772 		}
773 	}
774 	/* reset have fragment flag */
775 	f->flag_have_fragment = 0;
776 }
777 
778 /*------------------------------------------------------------------------*
779  *	usb_fifo_close
780  *------------------------------------------------------------------------*/
781 static void
782 usb_fifo_close(struct usb_fifo *f, int fflags)
783 {
784 	int err;
785 
786 	/* check if we are not opened */
787 	if (f->curr_cpd == NULL) {
788 		/* nothing to do - already closed */
789 		return;
790 	}
791 	mtx_lock(f->priv_mtx);
792 
793 	/* clear current cdev private data pointer */
794 	mtx_lock(&usb_ref_lock);
795 	f->curr_cpd = NULL;
796 	mtx_unlock(&usb_ref_lock);
797 
798 	/* check if we are watched by kevent */
799 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
800 
801 	/* check if we are selected */
802 	if (f->flag_isselect) {
803 		selwakeup(&f->selinfo);
804 		f->flag_isselect = 0;
805 	}
806 	/* check if a thread wants SIGIO */
807 	if (f->async_p != NULL) {
808 		PROC_LOCK(f->async_p);
809 		kern_psignal(f->async_p, SIGIO);
810 		PROC_UNLOCK(f->async_p);
811 		f->async_p = NULL;
812 	}
813 	/* remove FWRITE and FREAD flags */
814 	fflags &= ~(FWRITE | FREAD);
815 
816 	/* flush written data, if any */
817 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
818 
819 		if (!f->flag_iserror) {
820 
821 			/* set flushing flag */
822 			f->flag_flushing = 1;
823 
824 			/* get the last packet in */
825 			if (f->flag_have_fragment) {
826 				struct usb_mbuf *m;
827 				f->flag_have_fragment = 0;
828 				USB_IF_DEQUEUE(&f->free_q, m);
829 				if (m) {
830 					USB_IF_ENQUEUE(&f->used_q, m);
831 				}
832 			}
833 
834 			/* start write transfer, if not already started */
835 			(f->methods->f_start_write) (f);
836 
837 			/* check if flushed already */
838 			while (f->flag_flushing &&
839 			    (!f->flag_iserror)) {
840 				/* wait until all data has been written */
841 				f->flag_sleeping = 1;
842 				err = cv_timedwait_sig(&f->cv_io, f->priv_mtx,
843 				    USB_MS_TO_TICKS(USB_DEFAULT_TIMEOUT));
844 				if (err) {
845 					DPRINTF("signal received\n");
846 					break;
847 				}
848 			}
849 		}
850 		fflags |= FWRITE;
851 
852 		/* stop write transfer, if not already stopped */
853 		(f->methods->f_stop_write) (f);
854 	} else {
855 		fflags |= FREAD;
856 
857 		/* stop write transfer, if not already stopped */
858 		(f->methods->f_stop_read) (f);
859 	}
860 
861 	/* check if we are sleeping */
862 	if (f->flag_sleeping) {
863 		DPRINTFN(2, "Sleeping at close!\n");
864 	}
865 	mtx_unlock(f->priv_mtx);
866 
867 	/* call close method */
868 	(f->methods->f_close) (f, fflags);
869 
870 	DPRINTF("closed\n");
871 }
872 
873 /*------------------------------------------------------------------------*
874  *	usb_open - cdev callback
875  *------------------------------------------------------------------------*/
876 static int
877 usb_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
878 {
879 	struct usb_fs_privdata* pd = (struct usb_fs_privdata*)dev->si_drv1;
880 	struct usb_cdev_refdata refs;
881 	struct usb_cdev_privdata *cpd;
882 	int err;
883 
884 	DPRINTFN(2, "%s fflags=0x%08x\n", devtoname(dev), fflags);
885 
886 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
887 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
888 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
889 		DPRINTFN(2, "access mode not supported\n");
890 		return (EPERM);
891 	}
892 
893 	cpd = malloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
894 
895 	usb_loc_fill(pd, cpd);
896 	err = usb_ref_device(cpd, &refs, 1);
897 	if (err) {
898 		DPRINTFN(2, "cannot ref device\n");
899 		free(cpd, M_USBDEV);
900 		return (ENXIO);
901 	}
902 	cpd->fflags = fflags;	/* access mode for open lifetime */
903 
904 	/* create FIFOs, if any */
905 	err = usb_fifo_create(cpd, &refs);
906 	/* check for error */
907 	if (err) {
908 		DPRINTFN(2, "cannot create fifo\n");
909 		usb_unref_device(cpd, &refs);
910 		free(cpd, M_USBDEV);
911 		return (err);
912 	}
913 	if (fflags & FREAD) {
914 		err = usb_fifo_open(cpd, refs.rxfifo, fflags);
915 		if (err) {
916 			DPRINTFN(2, "read open failed\n");
917 			usb_unref_device(cpd, &refs);
918 			free(cpd, M_USBDEV);
919 			return (err);
920 		}
921 	}
922 	if (fflags & FWRITE) {
923 		err = usb_fifo_open(cpd, refs.txfifo, fflags);
924 		if (err) {
925 			DPRINTFN(2, "write open failed\n");
926 			if (fflags & FREAD) {
927 				usb_fifo_close(refs.rxfifo, fflags);
928 			}
929 			usb_unref_device(cpd, &refs);
930 			free(cpd, M_USBDEV);
931 			return (err);
932 		}
933 	}
934 	usb_unref_device(cpd, &refs);
935 	devfs_set_cdevpriv(cpd, usb_close);
936 
937 	return (0);
938 }
939 
940 /*------------------------------------------------------------------------*
941  *	usb_close - cdev callback
942  *------------------------------------------------------------------------*/
943 static void
944 usb_close(void *arg)
945 {
946 	struct usb_cdev_refdata refs;
947 	struct usb_cdev_privdata *cpd = arg;
948 	int err;
949 
950 	DPRINTFN(2, "cpd=%p\n", cpd);
951 
952 	err = usb_ref_device(cpd, &refs,
953 	    2 /* uref and allow detached state */);
954 	if (err) {
955 		DPRINTFN(2, "Cannot grab USB reference when "
956 		    "closing USB file handle\n");
957 		goto done;
958 	}
959 	if (cpd->fflags & FREAD) {
960 		usb_fifo_close(refs.rxfifo, cpd->fflags);
961 	}
962 	if (cpd->fflags & FWRITE) {
963 		usb_fifo_close(refs.txfifo, cpd->fflags);
964 	}
965 	usb_unref_device(cpd, &refs);
966 done:
967 	free(cpd, M_USBDEV);
968 }
969 
970 static void
971 usb_dev_init(void *arg)
972 {
973 	mtx_init(&usb_ref_lock, "USB ref mutex", NULL, MTX_DEF);
974 	sx_init(&usb_sym_lock, "USB sym mutex");
975 	TAILQ_INIT(&usb_sym_head);
976 
977 	/* check the UGEN methods */
978 	usb_fifo_check_methods(&usb_ugen_methods);
979 }
980 
981 SYSINIT(usb_dev_init, SI_SUB_KLD, SI_ORDER_FIRST, usb_dev_init, NULL);
982 
983 static void
984 usb_dev_init_post(void *arg)
985 {
986 	/*
987 	 * Create /dev/usb - this is needed for usbconfig(8), which
988 	 * needs a well-known device name to access.
989 	 */
990 	usb_dev = make_dev(&usb_static_devsw, 0, UID_ROOT, GID_OPERATOR,
991 	    0644, USB_DEVICE_NAME);
992 	if (usb_dev == NULL) {
993 		DPRINTFN(0, "Could not create usb bus device\n");
994 	}
995 }
996 
997 SYSINIT(usb_dev_init_post, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, usb_dev_init_post, NULL);
998 
999 static void
1000 usb_dev_uninit(void *arg)
1001 {
1002 	if (usb_dev != NULL) {
1003 		destroy_dev(usb_dev);
1004 		usb_dev = NULL;
1005 	}
1006 	mtx_destroy(&usb_ref_lock);
1007 	sx_destroy(&usb_sym_lock);
1008 }
1009 
1010 SYSUNINIT(usb_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb_dev_uninit, NULL);
1011 
1012 static int
1013 usb_ioctl_f_sub(struct usb_fifo *f, u_long cmd, void *addr,
1014     struct thread *td)
1015 {
1016 	int error = 0;
1017 
1018 	switch (cmd) {
1019 	case FIODTYPE:
1020 		*(int *)addr = 0;	/* character device */
1021 		break;
1022 
1023 	case FIONBIO:
1024 		/* handled by upper FS layer */
1025 		break;
1026 
1027 	case FIOASYNC:
1028 		if (*(int *)addr) {
1029 			if (f->async_p != NULL) {
1030 				error = EBUSY;
1031 				break;
1032 			}
1033 			f->async_p = USB_TD_GET_PROC(td);
1034 		} else {
1035 			f->async_p = NULL;
1036 		}
1037 		break;
1038 
1039 		/* XXX this is not the most general solution */
1040 	case TIOCSPGRP:
1041 		if (f->async_p == NULL) {
1042 			error = EINVAL;
1043 			break;
1044 		}
1045 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
1046 			error = EPERM;
1047 			break;
1048 		}
1049 		break;
1050 	default:
1051 		return (ENOIOCTL);
1052 	}
1053 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
1054 	return (error);
1055 }
1056 
1057 /*------------------------------------------------------------------------*
1058  *	usb_ioctl - cdev callback
1059  *------------------------------------------------------------------------*/
1060 static int
1061 usb_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread* td)
1062 {
1063 	struct usb_cdev_refdata refs;
1064 	struct usb_cdev_privdata* cpd;
1065 	struct usb_fifo *f;
1066 	int fflags;
1067 	int err;
1068 
1069 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1070 
1071 	err = devfs_get_cdevpriv((void **)&cpd);
1072 	if (err != 0)
1073 		return (err);
1074 
1075 	/*
1076 	 * Performance optimisation: We try to check for IOCTL's that
1077 	 * don't need the USB reference first. Then we grab the USB
1078 	 * reference if we need it!
1079 	 */
1080 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1081 	if (err)
1082 		return (ENXIO);
1083 
1084 	fflags = cpd->fflags;
1085 
1086 	f = NULL;			/* set default value */
1087 	err = ENOIOCTL;			/* set default value */
1088 
1089 	if (fflags & FWRITE) {
1090 		f = refs.txfifo;
1091 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1092 	}
1093 	if (fflags & FREAD) {
1094 		f = refs.rxfifo;
1095 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1096 	}
1097 	KASSERT(f != NULL, ("fifo not found"));
1098 	if (err != ENOIOCTL)
1099 		goto done;
1100 
1101 	err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1102 
1103 	DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1104 
1105 	if (err != ENOIOCTL)
1106 		goto done;
1107 
1108 	if (usb_usb_ref_device(cpd, &refs)) {
1109 		/* we lost the reference */
1110 		return (ENXIO);
1111 	}
1112 
1113 	err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1114 
1115 	DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1116 
1117 	if (err == ENOIOCTL)
1118 		err = ENOTTY;
1119 
1120 	if (err)
1121 		goto done;
1122 
1123 	/* Wait for re-enumeration, if any */
1124 
1125 	while (f->udev->re_enumerate_wait != USB_RE_ENUM_DONE) {
1126 
1127 		usb_unref_device(cpd, &refs);
1128 
1129 		usb_pause_mtx(NULL, hz / 128);
1130 
1131 		while (usb_ref_device(cpd, &refs, 1 /* need uref */)) {
1132 			if (usb_ref_device(cpd, &refs, 0)) {
1133 				/* device no longer exists */
1134 				return (ENXIO);
1135 			}
1136 			usb_unref_device(cpd, &refs);
1137 			usb_pause_mtx(NULL, hz / 128);
1138 		}
1139 	}
1140 
1141 done:
1142 	usb_unref_device(cpd, &refs);
1143 	return (err);
1144 }
1145 
1146 static void
1147 usb_filter_detach(struct knote *kn)
1148 {
1149 	struct usb_fifo *f = kn->kn_hook;
1150 	knlist_remove(&f->selinfo.si_note, kn, 0);
1151 }
1152 
1153 static int
1154 usb_filter_write(struct knote *kn, long hint)
1155 {
1156 	struct usb_cdev_privdata* cpd;
1157 	struct usb_fifo *f;
1158 	struct usb_mbuf *m;
1159 
1160 	DPRINTFN(2, "\n");
1161 
1162 	f = kn->kn_hook;
1163 
1164 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1165 
1166 	cpd = f->curr_cpd;
1167 	if (cpd == NULL) {
1168 		m = (void *)1;
1169 	} else if (f->fs_ep_max == 0) {
1170 		if (f->flag_iserror) {
1171 			/* we got an error */
1172 			m = (void *)1;
1173 		} else {
1174 			if (f->queue_data == NULL) {
1175 				/*
1176 				 * start write transfer, if not
1177 				 * already started
1178 				 */
1179 				(f->methods->f_start_write) (f);
1180 			}
1181 			/* check if any packets are available */
1182 			USB_IF_POLL(&f->free_q, m);
1183 		}
1184 	} else {
1185 		if (f->flag_iscomplete) {
1186 			m = (void *)1;
1187 		} else {
1188 			m = NULL;
1189 		}
1190 	}
1191 	return (m ? 1 : 0);
1192 }
1193 
1194 static int
1195 usb_filter_read(struct knote *kn, long hint)
1196 {
1197 	struct usb_cdev_privdata* cpd;
1198 	struct usb_fifo *f;
1199 	struct usb_mbuf *m;
1200 
1201 	DPRINTFN(2, "\n");
1202 
1203 	f = kn->kn_hook;
1204 
1205 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1206 
1207 	cpd = f->curr_cpd;
1208 	if (cpd == NULL) {
1209 		m = (void *)1;
1210 	} else if (f->fs_ep_max == 0) {
1211 		if (f->flag_iserror) {
1212 			/* we have an error */
1213 			m = (void *)1;
1214 		} else {
1215 			if (f->queue_data == NULL) {
1216 				/*
1217 				 * start read transfer, if not
1218 				 * already started
1219 				 */
1220 				(f->methods->f_start_read) (f);
1221 			}
1222 			/* check if any packets are available */
1223 			USB_IF_POLL(&f->used_q, m);
1224 
1225 			/* start reading data, if any */
1226 			if (m == NULL)
1227 				(f->methods->f_start_read) (f);
1228 		}
1229 	} else {
1230 		if (f->flag_iscomplete) {
1231 			m = (void *)1;
1232 		} else {
1233 			m = NULL;
1234 		}
1235 	}
1236 	return (m ? 1 : 0);
1237 }
1238 
1239 static struct filterops usb_filtops_write = {
1240 	.f_isfd = 1,
1241 	.f_detach = usb_filter_detach,
1242 	.f_event = usb_filter_write,
1243 };
1244 
1245 static struct filterops usb_filtops_read = {
1246 	.f_isfd = 1,
1247 	.f_detach = usb_filter_detach,
1248 	.f_event = usb_filter_read,
1249 };
1250 
1251 
1252 /* ARGSUSED */
1253 static int
1254 usb_kqfilter(struct cdev* dev, struct knote *kn)
1255 {
1256 	struct usb_cdev_refdata refs;
1257 	struct usb_cdev_privdata* cpd;
1258 	struct usb_fifo *f;
1259 	int fflags;
1260 	int err = EINVAL;
1261 
1262 	DPRINTFN(2, "\n");
1263 
1264 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1265 	    usb_ref_device(cpd, &refs, 0) != 0)
1266 		return (ENXIO);
1267 
1268 	fflags = cpd->fflags;
1269 
1270 	/* Figure out who needs service */
1271 	switch (kn->kn_filter) {
1272 	case EVFILT_WRITE:
1273 		if (fflags & FWRITE) {
1274 			f = refs.txfifo;
1275 			kn->kn_fop = &usb_filtops_write;
1276 			err = 0;
1277 		}
1278 		break;
1279 	case EVFILT_READ:
1280 		if (fflags & FREAD) {
1281 			f = refs.rxfifo;
1282 			kn->kn_fop = &usb_filtops_read;
1283 			err = 0;
1284 		}
1285 		break;
1286 	default:
1287 		err = EOPNOTSUPP;
1288 		break;
1289 	}
1290 
1291 	if (err == 0) {
1292 		kn->kn_hook = f;
1293 		mtx_lock(f->priv_mtx);
1294 		knlist_add(&f->selinfo.si_note, kn, 1);
1295 		mtx_unlock(f->priv_mtx);
1296 	}
1297 
1298 	usb_unref_device(cpd, &refs);
1299 	return (err);
1300 }
1301 
1302 /* ARGSUSED */
1303 static int
1304 usb_poll(struct cdev* dev, int events, struct thread* td)
1305 {
1306 	struct usb_cdev_refdata refs;
1307 	struct usb_cdev_privdata* cpd;
1308 	struct usb_fifo *f;
1309 	struct usb_mbuf *m;
1310 	int fflags, revents;
1311 
1312 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1313 	    usb_ref_device(cpd, &refs, 0) != 0)
1314 		return (events &
1315 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1316 
1317 	fflags = cpd->fflags;
1318 
1319 	/* Figure out who needs service */
1320 	revents = 0;
1321 	if ((events & (POLLOUT | POLLWRNORM)) &&
1322 	    (fflags & FWRITE)) {
1323 
1324 		f = refs.txfifo;
1325 
1326 		mtx_lock(f->priv_mtx);
1327 
1328 		if (!refs.is_usbfs) {
1329 			if (f->flag_iserror) {
1330 				/* we got an error */
1331 				m = (void *)1;
1332 			} else {
1333 				if (f->queue_data == NULL) {
1334 					/*
1335 					 * start write transfer, if not
1336 					 * already started
1337 					 */
1338 					(f->methods->f_start_write) (f);
1339 				}
1340 				/* check if any packets are available */
1341 				USB_IF_POLL(&f->free_q, m);
1342 			}
1343 		} else {
1344 			if (f->flag_iscomplete) {
1345 				m = (void *)1;
1346 			} else {
1347 				m = NULL;
1348 			}
1349 		}
1350 
1351 		if (m) {
1352 			revents |= events & (POLLOUT | POLLWRNORM);
1353 		} else {
1354 			f->flag_isselect = 1;
1355 			selrecord(td, &f->selinfo);
1356 		}
1357 
1358 		mtx_unlock(f->priv_mtx);
1359 	}
1360 	if ((events & (POLLIN | POLLRDNORM)) &&
1361 	    (fflags & FREAD)) {
1362 
1363 		f = refs.rxfifo;
1364 
1365 		mtx_lock(f->priv_mtx);
1366 
1367 		if (!refs.is_usbfs) {
1368 			if (f->flag_iserror) {
1369 				/* we have an error */
1370 				m = (void *)1;
1371 			} else {
1372 				if (f->queue_data == NULL) {
1373 					/*
1374 					 * start read transfer, if not
1375 					 * already started
1376 					 */
1377 					(f->methods->f_start_read) (f);
1378 				}
1379 				/* check if any packets are available */
1380 				USB_IF_POLL(&f->used_q, m);
1381 			}
1382 		} else {
1383 			if (f->flag_iscomplete) {
1384 				m = (void *)1;
1385 			} else {
1386 				m = NULL;
1387 			}
1388 		}
1389 
1390 		if (m) {
1391 			revents |= events & (POLLIN | POLLRDNORM);
1392 		} else {
1393 			f->flag_isselect = 1;
1394 			selrecord(td, &f->selinfo);
1395 
1396 			if (!refs.is_usbfs) {
1397 				/* start reading data */
1398 				(f->methods->f_start_read) (f);
1399 			}
1400 		}
1401 
1402 		mtx_unlock(f->priv_mtx);
1403 	}
1404 	usb_unref_device(cpd, &refs);
1405 	return (revents);
1406 }
1407 
1408 static int
1409 usb_read(struct cdev *dev, struct uio *uio, int ioflag)
1410 {
1411 	struct usb_cdev_refdata refs;
1412 	struct usb_cdev_privdata* cpd;
1413 	struct usb_fifo *f;
1414 	struct usb_mbuf *m;
1415 	int io_len;
1416 	int err;
1417 	uint8_t tr_data = 0;
1418 
1419 	err = devfs_get_cdevpriv((void **)&cpd);
1420 	if (err != 0)
1421 		return (err);
1422 
1423 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1424 	if (err)
1425 		return (ENXIO);
1426 
1427 	f = refs.rxfifo;
1428 	if (f == NULL) {
1429 		/* should not happen */
1430 		usb_unref_device(cpd, &refs);
1431 		return (EPERM);
1432 	}
1433 
1434 	mtx_lock(f->priv_mtx);
1435 
1436 	/* check for permanent read error */
1437 	if (f->flag_iserror) {
1438 		err = EIO;
1439 		goto done;
1440 	}
1441 	/* check if USB-FS interface is active */
1442 	if (refs.is_usbfs) {
1443 		/*
1444 		 * The queue is used for events that should be
1445 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1446 		 */
1447 		err = EINVAL;
1448 		goto done;
1449 	}
1450 	while (uio->uio_resid > 0) {
1451 
1452 		USB_IF_DEQUEUE(&f->used_q, m);
1453 
1454 		if (m == NULL) {
1455 
1456 			/* start read transfer, if not already started */
1457 
1458 			(f->methods->f_start_read) (f);
1459 
1460 			if (ioflag & IO_NDELAY) {
1461 				if (tr_data) {
1462 					/* return length before error */
1463 					break;
1464 				}
1465 				err = EWOULDBLOCK;
1466 				break;
1467 			}
1468 			DPRINTF("sleeping\n");
1469 
1470 			err = usb_fifo_wait(f);
1471 			if (err) {
1472 				break;
1473 			}
1474 			continue;
1475 		}
1476 		if (f->methods->f_filter_read) {
1477 			/*
1478 			 * Sometimes it is convenient to process data at the
1479 			 * expense of a userland process instead of a kernel
1480 			 * process.
1481 			 */
1482 			(f->methods->f_filter_read) (f, m);
1483 		}
1484 		tr_data = 1;
1485 
1486 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1487 
1488 		DPRINTFN(2, "transfer %d bytes from %p\n",
1489 		    io_len, m->cur_data_ptr);
1490 
1491 		err = usb_fifo_uiomove(f,
1492 		    m->cur_data_ptr, io_len, uio);
1493 
1494 		m->cur_data_len -= io_len;
1495 		m->cur_data_ptr += io_len;
1496 
1497 		if (m->cur_data_len == 0) {
1498 
1499 			uint8_t last_packet;
1500 
1501 			last_packet = m->last_packet;
1502 
1503 			USB_IF_ENQUEUE(&f->free_q, m);
1504 
1505 			if (last_packet) {
1506 				/* keep framing */
1507 				break;
1508 			}
1509 		} else {
1510 			USB_IF_PREPEND(&f->used_q, m);
1511 		}
1512 
1513 		if (err) {
1514 			break;
1515 		}
1516 	}
1517 done:
1518 	mtx_unlock(f->priv_mtx);
1519 
1520 	usb_unref_device(cpd, &refs);
1521 
1522 	return (err);
1523 }
1524 
1525 static int
1526 usb_write(struct cdev *dev, struct uio *uio, int ioflag)
1527 {
1528 	struct usb_cdev_refdata refs;
1529 	struct usb_cdev_privdata* cpd;
1530 	struct usb_fifo *f;
1531 	struct usb_mbuf *m;
1532 	uint8_t *pdata;
1533 	int io_len;
1534 	int err;
1535 	uint8_t tr_data = 0;
1536 
1537 	DPRINTFN(2, "\n");
1538 
1539 	err = devfs_get_cdevpriv((void **)&cpd);
1540 	if (err != 0)
1541 		return (err);
1542 
1543 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1544 	if (err)
1545 		return (ENXIO);
1546 
1547 	f = refs.txfifo;
1548 	if (f == NULL) {
1549 		/* should not happen */
1550 		usb_unref_device(cpd, &refs);
1551 		return (EPERM);
1552 	}
1553 
1554 	mtx_lock(f->priv_mtx);
1555 
1556 	/* check for permanent write error */
1557 	if (f->flag_iserror) {
1558 		err = EIO;
1559 		goto done;
1560 	}
1561 	/* check if USB-FS interface is active */
1562 	if (refs.is_usbfs) {
1563 		/*
1564 		 * The queue is used for events that should be
1565 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1566 		 */
1567 		err = EINVAL;
1568 		goto done;
1569 	}
1570 	if (f->queue_data == NULL) {
1571 		/* start write transfer, if not already started */
1572 		(f->methods->f_start_write) (f);
1573 	}
1574 	/* we allow writing zero length data */
1575 	do {
1576 		USB_IF_DEQUEUE(&f->free_q, m);
1577 
1578 		if (m == NULL) {
1579 
1580 			if (ioflag & IO_NDELAY) {
1581 				if (tr_data) {
1582 					/* return length before error */
1583 					break;
1584 				}
1585 				err = EWOULDBLOCK;
1586 				break;
1587 			}
1588 			DPRINTF("sleeping\n");
1589 
1590 			err = usb_fifo_wait(f);
1591 			if (err) {
1592 				break;
1593 			}
1594 			continue;
1595 		}
1596 		tr_data = 1;
1597 
1598 		if (f->flag_have_fragment == 0) {
1599 			USB_MBUF_RESET(m);
1600 			io_len = m->cur_data_len;
1601 			pdata = m->cur_data_ptr;
1602 			if (io_len > uio->uio_resid)
1603 				io_len = uio->uio_resid;
1604 			m->cur_data_len = io_len;
1605 		} else {
1606 			io_len = m->max_data_len - m->cur_data_len;
1607 			pdata = m->cur_data_ptr + m->cur_data_len;
1608 			if (io_len > uio->uio_resid)
1609 				io_len = uio->uio_resid;
1610 			m->cur_data_len += io_len;
1611 		}
1612 
1613 		DPRINTFN(2, "transfer %d bytes to %p\n",
1614 		    io_len, pdata);
1615 
1616 		err = usb_fifo_uiomove(f, pdata, io_len, uio);
1617 
1618 		if (err) {
1619 			f->flag_have_fragment = 0;
1620 			USB_IF_ENQUEUE(&f->free_q, m);
1621 			break;
1622 		}
1623 
1624 		/* check if the buffer is ready to be transmitted */
1625 
1626 		if ((f->flag_write_defrag == 0) ||
1627 		    (m->cur_data_len == m->max_data_len)) {
1628 			f->flag_have_fragment = 0;
1629 
1630 			/*
1631 			 * Check for write filter:
1632 			 *
1633 			 * Sometimes it is convenient to process data
1634 			 * at the expense of a userland process
1635 			 * instead of a kernel process.
1636 			 */
1637 			if (f->methods->f_filter_write) {
1638 				(f->methods->f_filter_write) (f, m);
1639 			}
1640 
1641 			/* Put USB mbuf in the used queue */
1642 			USB_IF_ENQUEUE(&f->used_q, m);
1643 
1644 			/* Start writing data, if not already started */
1645 			(f->methods->f_start_write) (f);
1646 		} else {
1647 			/* Wait for more data or close */
1648 			f->flag_have_fragment = 1;
1649 			USB_IF_PREPEND(&f->free_q, m);
1650 		}
1651 
1652 	} while (uio->uio_resid > 0);
1653 done:
1654 	mtx_unlock(f->priv_mtx);
1655 
1656 	usb_unref_device(cpd, &refs);
1657 
1658 	return (err);
1659 }
1660 
1661 int
1662 usb_static_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
1663     struct thread *td)
1664 {
1665 	union {
1666 		struct usb_read_dir *urd;
1667 		void* data;
1668 	} u;
1669 	int err;
1670 
1671 	u.data = data;
1672 	switch (cmd) {
1673 		case USB_READ_DIR:
1674 			err = usb_read_symlink(u.urd->urd_data,
1675 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1676 			break;
1677 		case USB_DEV_QUIRK_GET:
1678 		case USB_QUIRK_NAME_GET:
1679 		case USB_DEV_QUIRK_ADD:
1680 		case USB_DEV_QUIRK_REMOVE:
1681 			err = usb_quirk_ioctl_p(cmd, data, fflag, td);
1682 			break;
1683 		case USB_GET_TEMPLATE:
1684 			*(int *)data = usb_template;
1685 			err = 0;
1686 			break;
1687 		case USB_SET_TEMPLATE:
1688 			err = priv_check(curthread, PRIV_DRIVER);
1689 			if (err)
1690 				break;
1691 			usb_template = *(int *)data;
1692 			break;
1693 		default:
1694 			err = ENOTTY;
1695 			break;
1696 	}
1697 	return (err);
1698 }
1699 
1700 static int
1701 usb_fifo_uiomove(struct usb_fifo *f, void *cp,
1702     int n, struct uio *uio)
1703 {
1704 	int error;
1705 
1706 	mtx_unlock(f->priv_mtx);
1707 
1708 	/*
1709 	 * "uiomove()" can sleep so one needs to make a wrapper,
1710 	 * exiting the mutex and checking things:
1711 	 */
1712 	error = uiomove(cp, n, uio);
1713 
1714 	mtx_lock(f->priv_mtx);
1715 
1716 	return (error);
1717 }
1718 
1719 int
1720 usb_fifo_wait(struct usb_fifo *f)
1721 {
1722 	int err;
1723 
1724 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1725 
1726 	if (f->flag_iserror) {
1727 		/* we are gone */
1728 		return (EIO);
1729 	}
1730 	f->flag_sleeping = 1;
1731 
1732 	err = cv_wait_sig(&f->cv_io, f->priv_mtx);
1733 
1734 	if (f->flag_iserror) {
1735 		/* we are gone */
1736 		err = EIO;
1737 	}
1738 	return (err);
1739 }
1740 
1741 void
1742 usb_fifo_signal(struct usb_fifo *f)
1743 {
1744 	if (f->flag_sleeping) {
1745 		f->flag_sleeping = 0;
1746 		cv_broadcast(&f->cv_io);
1747 	}
1748 }
1749 
1750 void
1751 usb_fifo_wakeup(struct usb_fifo *f)
1752 {
1753 	usb_fifo_signal(f);
1754 
1755 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
1756 
1757 	if (f->flag_isselect) {
1758 		selwakeup(&f->selinfo);
1759 		f->flag_isselect = 0;
1760 	}
1761 	if (f->async_p != NULL) {
1762 		PROC_LOCK(f->async_p);
1763 		kern_psignal(f->async_p, SIGIO);
1764 		PROC_UNLOCK(f->async_p);
1765 	}
1766 }
1767 
1768 static int
1769 usb_fifo_dummy_open(struct usb_fifo *fifo, int fflags)
1770 {
1771 	return (0);
1772 }
1773 
1774 static void
1775 usb_fifo_dummy_close(struct usb_fifo *fifo, int fflags)
1776 {
1777 	return;
1778 }
1779 
1780 static int
1781 usb_fifo_dummy_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags)
1782 {
1783 	return (ENOIOCTL);
1784 }
1785 
1786 static void
1787 usb_fifo_dummy_cmd(struct usb_fifo *fifo)
1788 {
1789 	fifo->flag_flushing = 0;	/* not flushing */
1790 }
1791 
1792 static void
1793 usb_fifo_check_methods(struct usb_fifo_methods *pm)
1794 {
1795 	/* check that all callback functions are OK */
1796 
1797 	if (pm->f_open == NULL)
1798 		pm->f_open = &usb_fifo_dummy_open;
1799 
1800 	if (pm->f_close == NULL)
1801 		pm->f_close = &usb_fifo_dummy_close;
1802 
1803 	if (pm->f_ioctl == NULL)
1804 		pm->f_ioctl = &usb_fifo_dummy_ioctl;
1805 
1806 	if (pm->f_ioctl_post == NULL)
1807 		pm->f_ioctl_post = &usb_fifo_dummy_ioctl;
1808 
1809 	if (pm->f_start_read == NULL)
1810 		pm->f_start_read = &usb_fifo_dummy_cmd;
1811 
1812 	if (pm->f_stop_read == NULL)
1813 		pm->f_stop_read = &usb_fifo_dummy_cmd;
1814 
1815 	if (pm->f_start_write == NULL)
1816 		pm->f_start_write = &usb_fifo_dummy_cmd;
1817 
1818 	if (pm->f_stop_write == NULL)
1819 		pm->f_stop_write = &usb_fifo_dummy_cmd;
1820 }
1821 
1822 /*------------------------------------------------------------------------*
1823  *	usb_fifo_attach
1824  *
1825  * The following function will create a duplex FIFO.
1826  *
1827  * Return values:
1828  * 0: Success.
1829  * Else: Failure.
1830  *------------------------------------------------------------------------*/
1831 int
1832 usb_fifo_attach(struct usb_device *udev, void *priv_sc,
1833     struct mtx *priv_mtx, struct usb_fifo_methods *pm,
1834     struct usb_fifo_sc *f_sc, uint16_t unit, int16_t subunit,
1835     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1836 {
1837 	struct usb_fifo *f_tx;
1838 	struct usb_fifo *f_rx;
1839 	char devname[32];
1840 	uint8_t n;
1841 
1842 	f_sc->fp[USB_FIFO_TX] = NULL;
1843 	f_sc->fp[USB_FIFO_RX] = NULL;
1844 
1845 	if (pm == NULL)
1846 		return (EINVAL);
1847 
1848 	/* check the methods */
1849 	usb_fifo_check_methods(pm);
1850 
1851 	if (priv_mtx == NULL)
1852 		priv_mtx = &Giant;
1853 
1854 	/* search for a free FIFO slot */
1855 	for (n = 0;; n += 2) {
1856 
1857 		if (n == USB_FIFO_MAX) {
1858 			/* end of FIFOs reached */
1859 			return (ENOMEM);
1860 		}
1861 		/* Check for TX FIFO */
1862 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1863 			continue;
1864 		}
1865 		/* Check for RX FIFO */
1866 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1867 			continue;
1868 		}
1869 		break;
1870 	}
1871 
1872 	f_tx = usb_fifo_alloc(priv_mtx);
1873 	f_rx = usb_fifo_alloc(priv_mtx);
1874 
1875 	if ((f_tx == NULL) || (f_rx == NULL)) {
1876 		usb_fifo_free(f_tx);
1877 		usb_fifo_free(f_rx);
1878 		return (ENOMEM);
1879 	}
1880 	/* initialise FIFO structures */
1881 
1882 	f_tx->fifo_index = n + USB_FIFO_TX;
1883 	f_tx->dev_ep_index = -1;
1884 	f_tx->priv_sc0 = priv_sc;
1885 	f_tx->methods = pm;
1886 	f_tx->iface_index = iface_index;
1887 	f_tx->udev = udev;
1888 
1889 	f_rx->fifo_index = n + USB_FIFO_RX;
1890 	f_rx->dev_ep_index = -1;
1891 	f_rx->priv_sc0 = priv_sc;
1892 	f_rx->methods = pm;
1893 	f_rx->iface_index = iface_index;
1894 	f_rx->udev = udev;
1895 
1896 	f_sc->fp[USB_FIFO_TX] = f_tx;
1897 	f_sc->fp[USB_FIFO_RX] = f_rx;
1898 
1899 	mtx_lock(&usb_ref_lock);
1900 	udev->fifo[f_tx->fifo_index] = f_tx;
1901 	udev->fifo[f_rx->fifo_index] = f_rx;
1902 	mtx_unlock(&usb_ref_lock);
1903 
1904 	for (n = 0; n != 4; n++) {
1905 
1906 		if (pm->basename[n] == NULL) {
1907 			continue;
1908 		}
1909 		if (subunit < 0) {
1910 			if (snprintf(devname, sizeof(devname),
1911 			    "%s%u%s", pm->basename[n],
1912 			    unit, pm->postfix[n] ?
1913 			    pm->postfix[n] : "")) {
1914 				/* ignore */
1915 			}
1916 		} else {
1917 			if (snprintf(devname, sizeof(devname),
1918 			    "%s%u.%d%s", pm->basename[n],
1919 			    unit, subunit, pm->postfix[n] ?
1920 			    pm->postfix[n] : "")) {
1921 				/* ignore */
1922 			}
1923 		}
1924 
1925 		/*
1926 		 * Distribute the symbolic links into two FIFO structures:
1927 		 */
1928 		if (n & 1) {
1929 			f_rx->symlink[n / 2] =
1930 			    usb_alloc_symlink(devname);
1931 		} else {
1932 			f_tx->symlink[n / 2] =
1933 			    usb_alloc_symlink(devname);
1934 		}
1935 
1936 		/* Create the device */
1937 		f_sc->dev = usb_make_dev(udev, devname, -1,
1938 		    f_tx->fifo_index & f_rx->fifo_index,
1939 		    FREAD|FWRITE, uid, gid, mode);
1940 	}
1941 
1942 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1943 	return (0);
1944 }
1945 
1946 /*------------------------------------------------------------------------*
1947  *	usb_fifo_alloc_buffer
1948  *
1949  * Return values:
1950  * 0: Success
1951  * Else failure
1952  *------------------------------------------------------------------------*/
1953 int
1954 usb_fifo_alloc_buffer(struct usb_fifo *f, usb_size_t bufsize,
1955     uint16_t nbuf)
1956 {
1957 	usb_fifo_free_buffer(f);
1958 
1959 	/* allocate an endpoint */
1960 	f->free_q.ifq_maxlen = nbuf;
1961 	f->used_q.ifq_maxlen = nbuf;
1962 
1963 	f->queue_data = usb_alloc_mbufs(
1964 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1965 
1966 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1967 		return (ENOMEM);
1968 	}
1969 	return (0);			/* success */
1970 }
1971 
1972 /*------------------------------------------------------------------------*
1973  *	usb_fifo_free_buffer
1974  *
1975  * This function will free the buffers associated with a FIFO. This
1976  * function can be called multiple times in a row.
1977  *------------------------------------------------------------------------*/
1978 void
1979 usb_fifo_free_buffer(struct usb_fifo *f)
1980 {
1981 	if (f->queue_data) {
1982 		/* free old buffer */
1983 		free(f->queue_data, M_USBDEV);
1984 		f->queue_data = NULL;
1985 	}
1986 	/* reset queues */
1987 
1988 	memset(&f->free_q, 0, sizeof(f->free_q));
1989 	memset(&f->used_q, 0, sizeof(f->used_q));
1990 }
1991 
1992 void
1993 usb_fifo_detach(struct usb_fifo_sc *f_sc)
1994 {
1995 	if (f_sc == NULL) {
1996 		return;
1997 	}
1998 	usb_fifo_free(f_sc->fp[USB_FIFO_TX]);
1999 	usb_fifo_free(f_sc->fp[USB_FIFO_RX]);
2000 
2001 	f_sc->fp[USB_FIFO_TX] = NULL;
2002 	f_sc->fp[USB_FIFO_RX] = NULL;
2003 
2004 	usb_destroy_dev(f_sc->dev);
2005 
2006 	f_sc->dev = NULL;
2007 
2008 	DPRINTFN(2, "detached %p\n", f_sc);
2009 }
2010 
2011 usb_size_t
2012 usb_fifo_put_bytes_max(struct usb_fifo *f)
2013 {
2014 	struct usb_mbuf *m;
2015 	usb_size_t len;
2016 
2017 	USB_IF_POLL(&f->free_q, m);
2018 
2019 	if (m) {
2020 		len = m->max_data_len;
2021 	} else {
2022 		len = 0;
2023 	}
2024 	return (len);
2025 }
2026 
2027 /*------------------------------------------------------------------------*
2028  *	usb_fifo_put_data
2029  *
2030  * what:
2031  *  0 - normal operation
2032  *  1 - set last packet flag to enforce framing
2033  *------------------------------------------------------------------------*/
2034 void
2035 usb_fifo_put_data(struct usb_fifo *f, struct usb_page_cache *pc,
2036     usb_frlength_t offset, usb_frlength_t len, uint8_t what)
2037 {
2038 	struct usb_mbuf *m;
2039 	usb_frlength_t io_len;
2040 
2041 	while (len || (what == 1)) {
2042 
2043 		USB_IF_DEQUEUE(&f->free_q, m);
2044 
2045 		if (m) {
2046 			USB_MBUF_RESET(m);
2047 
2048 			io_len = MIN(len, m->cur_data_len);
2049 
2050 			usbd_copy_out(pc, offset, m->cur_data_ptr, io_len);
2051 
2052 			m->cur_data_len = io_len;
2053 			offset += io_len;
2054 			len -= io_len;
2055 
2056 			if ((len == 0) && (what == 1)) {
2057 				m->last_packet = 1;
2058 			}
2059 			USB_IF_ENQUEUE(&f->used_q, m);
2060 
2061 			usb_fifo_wakeup(f);
2062 
2063 			if ((len == 0) || (what == 1)) {
2064 				break;
2065 			}
2066 		} else {
2067 			break;
2068 		}
2069 	}
2070 }
2071 
2072 void
2073 usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
2074     usb_size_t len, uint8_t what)
2075 {
2076 	struct usb_mbuf *m;
2077 	usb_size_t io_len;
2078 
2079 	while (len || (what == 1)) {
2080 
2081 		USB_IF_DEQUEUE(&f->free_q, m);
2082 
2083 		if (m) {
2084 			USB_MBUF_RESET(m);
2085 
2086 			io_len = MIN(len, m->cur_data_len);
2087 
2088 			memcpy(m->cur_data_ptr, ptr, io_len);
2089 
2090 			m->cur_data_len = io_len;
2091 			ptr = USB_ADD_BYTES(ptr, io_len);
2092 			len -= io_len;
2093 
2094 			if ((len == 0) && (what == 1)) {
2095 				m->last_packet = 1;
2096 			}
2097 			USB_IF_ENQUEUE(&f->used_q, m);
2098 
2099 			usb_fifo_wakeup(f);
2100 
2101 			if ((len == 0) || (what == 1)) {
2102 				break;
2103 			}
2104 		} else {
2105 			break;
2106 		}
2107 	}
2108 }
2109 
2110 uint8_t
2111 usb_fifo_put_data_buffer(struct usb_fifo *f, void *ptr, usb_size_t len)
2112 {
2113 	struct usb_mbuf *m;
2114 
2115 	USB_IF_DEQUEUE(&f->free_q, m);
2116 
2117 	if (m) {
2118 		m->cur_data_len = len;
2119 		m->cur_data_ptr = ptr;
2120 		USB_IF_ENQUEUE(&f->used_q, m);
2121 		usb_fifo_wakeup(f);
2122 		return (1);
2123 	}
2124 	return (0);
2125 }
2126 
2127 void
2128 usb_fifo_put_data_error(struct usb_fifo *f)
2129 {
2130 	f->flag_iserror = 1;
2131 	usb_fifo_wakeup(f);
2132 }
2133 
2134 /*------------------------------------------------------------------------*
2135  *	usb_fifo_get_data
2136  *
2137  * what:
2138  *  0 - normal operation
2139  *  1 - only get one "usb_mbuf"
2140  *
2141  * returns:
2142  *  0 - no more data
2143  *  1 - data in buffer
2144  *------------------------------------------------------------------------*/
2145 uint8_t
2146 usb_fifo_get_data(struct usb_fifo *f, struct usb_page_cache *pc,
2147     usb_frlength_t offset, usb_frlength_t len, usb_frlength_t *actlen,
2148     uint8_t what)
2149 {
2150 	struct usb_mbuf *m;
2151 	usb_frlength_t io_len;
2152 	uint8_t tr_data = 0;
2153 
2154 	actlen[0] = 0;
2155 
2156 	while (1) {
2157 
2158 		USB_IF_DEQUEUE(&f->used_q, m);
2159 
2160 		if (m) {
2161 
2162 			tr_data = 1;
2163 
2164 			io_len = MIN(len, m->cur_data_len);
2165 
2166 			usbd_copy_in(pc, offset, m->cur_data_ptr, io_len);
2167 
2168 			len -= io_len;
2169 			offset += io_len;
2170 			actlen[0] += io_len;
2171 			m->cur_data_ptr += io_len;
2172 			m->cur_data_len -= io_len;
2173 
2174 			if ((m->cur_data_len == 0) || (what == 1)) {
2175 				USB_IF_ENQUEUE(&f->free_q, m);
2176 
2177 				usb_fifo_wakeup(f);
2178 
2179 				if (what == 1) {
2180 					break;
2181 				}
2182 			} else {
2183 				USB_IF_PREPEND(&f->used_q, m);
2184 			}
2185 		} else {
2186 
2187 			if (tr_data) {
2188 				/* wait for data to be written out */
2189 				break;
2190 			}
2191 			if (f->flag_flushing) {
2192 				/* check if we should send a short packet */
2193 				if (f->flag_short != 0) {
2194 					f->flag_short = 0;
2195 					tr_data = 1;
2196 					break;
2197 				}
2198 				/* flushing complete */
2199 				f->flag_flushing = 0;
2200 				usb_fifo_wakeup(f);
2201 			}
2202 			break;
2203 		}
2204 		if (len == 0) {
2205 			break;
2206 		}
2207 	}
2208 	return (tr_data);
2209 }
2210 
2211 uint8_t
2212 usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
2213     usb_size_t len, usb_size_t *actlen, uint8_t what)
2214 {
2215 	struct usb_mbuf *m;
2216 	usb_size_t io_len;
2217 	uint8_t tr_data = 0;
2218 
2219 	actlen[0] = 0;
2220 
2221 	while (1) {
2222 
2223 		USB_IF_DEQUEUE(&f->used_q, m);
2224 
2225 		if (m) {
2226 
2227 			tr_data = 1;
2228 
2229 			io_len = MIN(len, m->cur_data_len);
2230 
2231 			memcpy(ptr, m->cur_data_ptr, io_len);
2232 
2233 			len -= io_len;
2234 			ptr = USB_ADD_BYTES(ptr, io_len);
2235 			actlen[0] += io_len;
2236 			m->cur_data_ptr += io_len;
2237 			m->cur_data_len -= io_len;
2238 
2239 			if ((m->cur_data_len == 0) || (what == 1)) {
2240 				USB_IF_ENQUEUE(&f->free_q, m);
2241 
2242 				usb_fifo_wakeup(f);
2243 
2244 				if (what == 1) {
2245 					break;
2246 				}
2247 			} else {
2248 				USB_IF_PREPEND(&f->used_q, m);
2249 			}
2250 		} else {
2251 
2252 			if (tr_data) {
2253 				/* wait for data to be written out */
2254 				break;
2255 			}
2256 			if (f->flag_flushing) {
2257 				/* check if we should send a short packet */
2258 				if (f->flag_short != 0) {
2259 					f->flag_short = 0;
2260 					tr_data = 1;
2261 					break;
2262 				}
2263 				/* flushing complete */
2264 				f->flag_flushing = 0;
2265 				usb_fifo_wakeup(f);
2266 			}
2267 			break;
2268 		}
2269 		if (len == 0) {
2270 			break;
2271 		}
2272 	}
2273 	return (tr_data);
2274 }
2275 
2276 uint8_t
2277 usb_fifo_get_data_buffer(struct usb_fifo *f, void **pptr, usb_size_t *plen)
2278 {
2279 	struct usb_mbuf *m;
2280 
2281 	USB_IF_POLL(&f->used_q, m);
2282 
2283 	if (m) {
2284 		*plen = m->cur_data_len;
2285 		*pptr = m->cur_data_ptr;
2286 
2287 		return (1);
2288 	}
2289 	return (0);
2290 }
2291 
2292 void
2293 usb_fifo_get_data_error(struct usb_fifo *f)
2294 {
2295 	f->flag_iserror = 1;
2296 	usb_fifo_wakeup(f);
2297 }
2298 
2299 /*------------------------------------------------------------------------*
2300  *	usb_alloc_symlink
2301  *
2302  * Return values:
2303  * NULL: Failure
2304  * Else: Pointer to symlink entry
2305  *------------------------------------------------------------------------*/
2306 struct usb_symlink *
2307 usb_alloc_symlink(const char *target)
2308 {
2309 	struct usb_symlink *ps;
2310 
2311 	ps = malloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2312 	if (ps == NULL) {
2313 		return (ps);
2314 	}
2315 	/* XXX no longer needed */
2316 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2317 	ps->src_len = strlen(ps->src_path);
2318 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2319 	ps->dst_len = strlen(ps->dst_path);
2320 
2321 	sx_xlock(&usb_sym_lock);
2322 	TAILQ_INSERT_TAIL(&usb_sym_head, ps, sym_entry);
2323 	sx_unlock(&usb_sym_lock);
2324 	return (ps);
2325 }
2326 
2327 /*------------------------------------------------------------------------*
2328  *	usb_free_symlink
2329  *------------------------------------------------------------------------*/
2330 void
2331 usb_free_symlink(struct usb_symlink *ps)
2332 {
2333 	if (ps == NULL) {
2334 		return;
2335 	}
2336 	sx_xlock(&usb_sym_lock);
2337 	TAILQ_REMOVE(&usb_sym_head, ps, sym_entry);
2338 	sx_unlock(&usb_sym_lock);
2339 
2340 	free(ps, M_USBDEV);
2341 }
2342 
2343 /*------------------------------------------------------------------------*
2344  *	usb_read_symlink
2345  *
2346  * Return value:
2347  * 0: Success
2348  * Else: Failure
2349  *------------------------------------------------------------------------*/
2350 int
2351 usb_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2352 {
2353 	struct usb_symlink *ps;
2354 	uint32_t temp;
2355 	uint32_t delta = 0;
2356 	uint8_t len;
2357 	int error = 0;
2358 
2359 	sx_xlock(&usb_sym_lock);
2360 
2361 	TAILQ_FOREACH(ps, &usb_sym_head, sym_entry) {
2362 
2363 		/*
2364 		 * Compute total length of source and destination symlink
2365 		 * strings pluss one length byte and two NUL bytes:
2366 		 */
2367 		temp = ps->src_len + ps->dst_len + 3;
2368 
2369 		if (temp > 255) {
2370 			/*
2371 			 * Skip entry because this length cannot fit
2372 			 * into one byte:
2373 			 */
2374 			continue;
2375 		}
2376 		if (startentry != 0) {
2377 			/* decrement read offset */
2378 			startentry--;
2379 			continue;
2380 		}
2381 		if (temp > user_len) {
2382 			/* out of buffer space */
2383 			break;
2384 		}
2385 		len = temp;
2386 
2387 		/* copy out total length */
2388 
2389 		error = copyout(&len,
2390 		    USB_ADD_BYTES(user_ptr, delta), 1);
2391 		if (error) {
2392 			break;
2393 		}
2394 		delta += 1;
2395 
2396 		/* copy out source string */
2397 
2398 		error = copyout(ps->src_path,
2399 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2400 		if (error) {
2401 			break;
2402 		}
2403 		len = 0;
2404 		delta += ps->src_len;
2405 		error = copyout(&len,
2406 		    USB_ADD_BYTES(user_ptr, delta), 1);
2407 		if (error) {
2408 			break;
2409 		}
2410 		delta += 1;
2411 
2412 		/* copy out destination string */
2413 
2414 		error = copyout(ps->dst_path,
2415 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2416 		if (error) {
2417 			break;
2418 		}
2419 		len = 0;
2420 		delta += ps->dst_len;
2421 		error = copyout(&len,
2422 		    USB_ADD_BYTES(user_ptr, delta), 1);
2423 		if (error) {
2424 			break;
2425 		}
2426 		delta += 1;
2427 
2428 		user_len -= temp;
2429 	}
2430 
2431 	/* a zero length entry indicates the end */
2432 
2433 	if ((user_len != 0) && (error == 0)) {
2434 
2435 		len = 0;
2436 
2437 		error = copyout(&len,
2438 		    USB_ADD_BYTES(user_ptr, delta), 1);
2439 	}
2440 	sx_unlock(&usb_sym_lock);
2441 	return (error);
2442 }
2443 
2444 void
2445 usb_fifo_set_close_zlp(struct usb_fifo *f, uint8_t onoff)
2446 {
2447 	if (f == NULL)
2448 		return;
2449 
2450 	/* send a Zero Length Packet, ZLP, before close */
2451 	f->flag_short = onoff;
2452 }
2453 
2454 void
2455 usb_fifo_set_write_defrag(struct usb_fifo *f, uint8_t onoff)
2456 {
2457 	if (f == NULL)
2458 		return;
2459 
2460 	/* defrag written data */
2461 	f->flag_write_defrag = onoff;
2462 	/* reset defrag state */
2463 	f->flag_have_fragment = 0;
2464 }
2465 
2466 void *
2467 usb_fifo_softc(struct usb_fifo *f)
2468 {
2469 	return (f->priv_sc0);
2470 }
2471 #endif	/* USB_HAVE_UGEN */
2472