xref: /freebsd/sys/dev/usb/usb_dev.c (revision 7cc42f6d)
1 /* $FreeBSD$ */
2 /*-
3  * SPDX-License-Identifier: BSD-2-Clause-FreeBSD
4  *
5  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
6  *
7  * Redistribution and use in source and binary forms, with or without
8  * modification, are permitted provided that the following conditions
9  * are met:
10  * 1. Redistributions of source code must retain the above copyright
11  *    notice, this list of conditions and the following disclaimer.
12  * 2. Redistributions in binary form must reproduce the above copyright
13  *    notice, this list of conditions and the following disclaimer in the
14  *    documentation and/or other materials provided with the distribution.
15  *
16  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
17  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
18  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
19  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
20  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
21  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
22  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
23  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
24  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
25  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
26  * SUCH DAMAGE.
27  *
28  *
29  * usb_dev.c - An abstraction layer for creating devices under /dev/...
30  */
31 
32 #ifdef USB_GLOBAL_INCLUDE_FILE
33 #include USB_GLOBAL_INCLUDE_FILE
34 #else
35 #include <sys/stdint.h>
36 #include <sys/stddef.h>
37 #include <sys/param.h>
38 #include <sys/queue.h>
39 #include <sys/types.h>
40 #include <sys/systm.h>
41 #include <sys/kernel.h>
42 #include <sys/bus.h>
43 #include <sys/module.h>
44 #include <sys/lock.h>
45 #include <sys/mutex.h>
46 #include <sys/condvar.h>
47 #include <sys/sysctl.h>
48 #include <sys/sx.h>
49 #include <sys/unistd.h>
50 #include <sys/callout.h>
51 #include <sys/malloc.h>
52 #include <sys/priv.h>
53 #include <sys/vnode.h>
54 #include <sys/conf.h>
55 #include <sys/fcntl.h>
56 
57 #include <dev/usb/usb.h>
58 #include <dev/usb/usb_ioctl.h>
59 #include <dev/usb/usbdi.h>
60 #include <dev/usb/usbdi_util.h>
61 
62 #define	USB_DEBUG_VAR usb_fifo_debug
63 
64 #include <dev/usb/usb_core.h>
65 #include <dev/usb/usb_dev.h>
66 #include <dev/usb/usb_mbuf.h>
67 #include <dev/usb/usb_process.h>
68 #include <dev/usb/usb_device.h>
69 #include <dev/usb/usb_debug.h>
70 #include <dev/usb/usb_busdma.h>
71 #include <dev/usb/usb_generic.h>
72 #include <dev/usb/usb_dynamic.h>
73 #include <dev/usb/usb_util.h>
74 
75 #include <dev/usb/usb_controller.h>
76 #include <dev/usb/usb_bus.h>
77 
78 #include <sys/filio.h>
79 #include <sys/ttycom.h>
80 #include <sys/syscallsubr.h>
81 
82 #include <machine/stdarg.h>
83 #endif			/* USB_GLOBAL_INCLUDE_FILE */
84 
85 #if USB_HAVE_UGEN
86 
87 #ifdef USB_DEBUG
88 static int usb_fifo_debug = 0;
89 
90 static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW | CTLFLAG_MPSAFE, 0,
91     "USB device");
92 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RWTUN,
93     &usb_fifo_debug, 0, "Debug Level");
94 #endif
95 
96 #if ((__FreeBSD_version >= 700001) || (__FreeBSD_version == 0) || \
97      ((__FreeBSD_version >= 600034) && (__FreeBSD_version < 700000)))
98 #define	USB_UCRED struct ucred *ucred,
99 #else
100 #define	USB_UCRED
101 #endif
102 
103 /* prototypes */
104 
105 static int	usb_fifo_open(struct usb_cdev_privdata *,
106 		    struct usb_fifo *, int);
107 static void	usb_fifo_close(struct usb_fifo *, int);
108 static void	usb_dev_init(void *);
109 static void	usb_dev_init_post(void *);
110 static void	usb_dev_uninit(void *);
111 static int	usb_fifo_uiomove(struct usb_fifo *, void *, int,
112 		    struct uio *);
113 static void	usb_fifo_check_methods(struct usb_fifo_methods *);
114 static struct	usb_fifo *usb_fifo_alloc(struct mtx *);
115 static struct	usb_endpoint *usb_dev_get_ep(struct usb_device *, uint8_t,
116 		    uint8_t);
117 static void	usb_loc_fill(struct usb_fs_privdata *,
118 		    struct usb_cdev_privdata *);
119 static void	usb_close(void *);
120 static usb_error_t usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *, int);
121 static usb_error_t usb_usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
122 static void	usb_unref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
123 
124 static d_open_t usb_open;
125 static d_ioctl_t usb_ioctl;
126 static d_read_t usb_read;
127 static d_write_t usb_write;
128 static d_poll_t usb_poll;
129 static d_kqfilter_t usb_kqfilter;
130 
131 static d_ioctl_t usb_static_ioctl;
132 
133 static usb_fifo_open_t usb_fifo_dummy_open;
134 static usb_fifo_close_t usb_fifo_dummy_close;
135 static usb_fifo_ioctl_t usb_fifo_dummy_ioctl;
136 static usb_fifo_cmd_t usb_fifo_dummy_cmd;
137 
138 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
139 struct cdevsw usb_devsw = {
140 	.d_version = D_VERSION,
141 	.d_open = usb_open,
142 	.d_ioctl = usb_ioctl,
143 	.d_name = "usbdev",
144 	.d_flags = D_TRACKCLOSE,
145 	.d_read = usb_read,
146 	.d_write = usb_write,
147 	.d_poll = usb_poll,
148 	.d_kqfilter = usb_kqfilter,
149 };
150 
151 static struct cdev* usb_dev = NULL;
152 
153 /* character device structure used for /dev/usb */
154 static struct cdevsw usb_static_devsw = {
155 	.d_version = D_VERSION,
156 	.d_ioctl = usb_static_ioctl,
157 	.d_name = "usb"
158 };
159 
160 static TAILQ_HEAD(, usb_symlink) usb_sym_head;
161 static struct sx usb_sym_lock;
162 
163 struct mtx usb_ref_lock;
164 
165 /*------------------------------------------------------------------------*
166  *	usb_loc_fill
167  *
168  * This is used to fill out a usb_cdev_privdata structure based on the
169  * device's address as contained in usb_fs_privdata.
170  *------------------------------------------------------------------------*/
171 static void
172 usb_loc_fill(struct usb_fs_privdata* pd, struct usb_cdev_privdata *cpd)
173 {
174 	cpd->bus_index = pd->bus_index;
175 	cpd->dev_index = pd->dev_index;
176 	cpd->ep_addr = pd->ep_addr;
177 	cpd->fifo_index = pd->fifo_index;
178 }
179 
180 /*------------------------------------------------------------------------*
181  *	usb_ref_device
182  *
183  * This function is used to atomically refer an USB device by its
184  * device location. If this function returns success the USB device
185  * will not disappear until the USB device is unreferenced.
186  *
187  * Return values:
188  *  0: Success, refcount incremented on the given USB device.
189  *  Else: Failure.
190  *------------------------------------------------------------------------*/
191 static usb_error_t
192 usb_ref_device(struct usb_cdev_privdata *cpd,
193     struct usb_cdev_refdata *crd, int need_uref)
194 {
195 	struct usb_fifo **ppf;
196 	struct usb_fifo *f;
197 
198 	DPRINTFN(2, "cpd=%p need uref=%d\n", cpd, need_uref);
199 
200 	/* clear all refs */
201 	memset(crd, 0, sizeof(*crd));
202 
203 	mtx_lock(&usb_ref_lock);
204 	cpd->bus = devclass_get_softc(usb_devclass_ptr, cpd->bus_index);
205 	if (cpd->bus == NULL) {
206 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
207 		goto error;
208 	}
209 	cpd->udev = cpd->bus->devices[cpd->dev_index];
210 	if (cpd->udev == NULL) {
211 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
212 		goto error;
213 	}
214 	if (cpd->udev->state == USB_STATE_DETACHED &&
215 	    (need_uref != 2)) {
216 		DPRINTFN(2, "device is detached\n");
217 		goto error;
218 	}
219 	if (need_uref) {
220 		DPRINTFN(2, "ref udev - needed\n");
221 
222 		if (cpd->udev->refcount == USB_DEV_REF_MAX) {
223 			DPRINTFN(2, "no dev ref\n");
224 			goto error;
225 		}
226 		cpd->udev->refcount++;
227 
228 		mtx_unlock(&usb_ref_lock);
229 
230 		/*
231 		 * We need to grab the enumeration SX-lock before
232 		 * grabbing the FIFO refs to avoid deadlock at detach!
233 		 */
234 		crd->do_unlock = usbd_enum_lock_sig(cpd->udev);
235 
236 		mtx_lock(&usb_ref_lock);
237 
238 		/*
239 		 * Set "is_uref" after grabbing the default SX lock
240 		 */
241 		crd->is_uref = 1;
242 
243 		/* check for signal */
244 		if (crd->do_unlock > 1) {
245 			crd->do_unlock = 0;
246 			goto error;
247 		}
248 	}
249 
250 	/* check if we are doing an open */
251 	if (cpd->fflags == 0) {
252 		/* use zero defaults */
253 	} else {
254 		/* check for write */
255 		if (cpd->fflags & FWRITE) {
256 			ppf = cpd->udev->fifo;
257 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
258 			crd->txfifo = f;
259 			crd->is_write = 1;	/* ref */
260 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
261 				goto error;
262 			if (f->curr_cpd != cpd)
263 				goto error;
264 			/* check if USB-FS is active */
265 			if (f->fs_ep_max != 0) {
266 				crd->is_usbfs = 1;
267 			}
268 		}
269 
270 		/* check for read */
271 		if (cpd->fflags & FREAD) {
272 			ppf = cpd->udev->fifo;
273 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
274 			crd->rxfifo = f;
275 			crd->is_read = 1;	/* ref */
276 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
277 				goto error;
278 			if (f->curr_cpd != cpd)
279 				goto error;
280 			/* check if USB-FS is active */
281 			if (f->fs_ep_max != 0) {
282 				crd->is_usbfs = 1;
283 			}
284 		}
285 	}
286 
287 	/* when everything is OK we increment the refcounts */
288 	if (crd->is_write) {
289 		DPRINTFN(2, "ref write\n");
290 		crd->txfifo->refcount++;
291 	}
292 	if (crd->is_read) {
293 		DPRINTFN(2, "ref read\n");
294 		crd->rxfifo->refcount++;
295 	}
296 	mtx_unlock(&usb_ref_lock);
297 
298 	return (0);
299 
300 error:
301 	if (crd->do_unlock)
302 		usbd_enum_unlock(cpd->udev);
303 
304 	if (crd->is_uref) {
305 		if (--(cpd->udev->refcount) == 0)
306 			cv_broadcast(&cpd->udev->ref_cv);
307 	}
308 	mtx_unlock(&usb_ref_lock);
309 	DPRINTFN(2, "fail\n");
310 
311 	/* clear all refs */
312 	memset(crd, 0, sizeof(*crd));
313 
314 	return (USB_ERR_INVAL);
315 }
316 
317 /*------------------------------------------------------------------------*
318  *	usb_usb_ref_device
319  *
320  * This function is used to upgrade an USB reference to include the
321  * USB device reference on a USB location.
322  *
323  * Return values:
324  *  0: Success, refcount incremented on the given USB device.
325  *  Else: Failure.
326  *------------------------------------------------------------------------*/
327 static usb_error_t
328 usb_usb_ref_device(struct usb_cdev_privdata *cpd,
329     struct usb_cdev_refdata *crd)
330 {
331 	/*
332 	 * Check if we already got an USB reference on this location:
333 	 */
334 	if (crd->is_uref)
335 		return (0);		/* success */
336 
337 	/*
338 	 * To avoid deadlock at detach we need to drop the FIFO ref
339 	 * and re-acquire a new ref!
340 	 */
341 	usb_unref_device(cpd, crd);
342 
343 	return (usb_ref_device(cpd, crd, 1 /* need uref */));
344 }
345 
346 /*------------------------------------------------------------------------*
347  *	usb_unref_device
348  *
349  * This function will release the reference count by one unit for the
350  * given USB device.
351  *------------------------------------------------------------------------*/
352 static void
353 usb_unref_device(struct usb_cdev_privdata *cpd,
354     struct usb_cdev_refdata *crd)
355 {
356 
357 	DPRINTFN(2, "cpd=%p is_uref=%d\n", cpd, crd->is_uref);
358 
359 	if (crd->do_unlock)
360 		usbd_enum_unlock(cpd->udev);
361 
362 	mtx_lock(&usb_ref_lock);
363 	if (crd->is_read) {
364 		if (--(crd->rxfifo->refcount) == 0) {
365 			cv_signal(&crd->rxfifo->cv_drain);
366 		}
367 		crd->is_read = 0;
368 	}
369 	if (crd->is_write) {
370 		if (--(crd->txfifo->refcount) == 0) {
371 			cv_signal(&crd->txfifo->cv_drain);
372 		}
373 		crd->is_write = 0;
374 	}
375 	if (crd->is_uref) {
376 		crd->is_uref = 0;
377 		if (--(cpd->udev->refcount) == 0)
378 			cv_broadcast(&cpd->udev->ref_cv);
379 	}
380 	mtx_unlock(&usb_ref_lock);
381 }
382 
383 static struct usb_fifo *
384 usb_fifo_alloc(struct mtx *mtx)
385 {
386 	struct usb_fifo *f;
387 
388 	f = malloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
389 	cv_init(&f->cv_io, "FIFO-IO");
390 	cv_init(&f->cv_drain, "FIFO-DRAIN");
391 	f->priv_mtx = mtx;
392 	f->refcount = 1;
393 	knlist_init_mtx(&f->selinfo.si_note, mtx);
394 	return (f);
395 }
396 
397 /*------------------------------------------------------------------------*
398  *	usb_fifo_create
399  *------------------------------------------------------------------------*/
400 static int
401 usb_fifo_create(struct usb_cdev_privdata *cpd,
402     struct usb_cdev_refdata *crd)
403 {
404 	struct usb_device *udev = cpd->udev;
405 	struct usb_fifo *f;
406 	struct usb_endpoint *ep;
407 	uint8_t n;
408 	uint8_t is_tx;
409 	uint8_t is_rx;
410 	uint8_t no_null;
411 	uint8_t is_busy;
412 	int e = cpd->ep_addr;
413 
414 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
415 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
416 	no_null = 1;
417 	is_busy = 0;
418 
419 	/* Preallocated FIFO */
420 	if (e < 0) {
421 		DPRINTFN(5, "Preallocated FIFO\n");
422 		if (is_tx) {
423 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
424 			if (f == NULL)
425 				return (EINVAL);
426 			crd->txfifo = f;
427 		}
428 		if (is_rx) {
429 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
430 			if (f == NULL)
431 				return (EINVAL);
432 			crd->rxfifo = f;
433 		}
434 		return (0);
435 	}
436 
437 	KASSERT(e >= 0 && e <= 15, ("endpoint %d out of range", e));
438 
439 	/* search for a free FIFO slot */
440 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", e);
441 	for (n = 0;; n += 2) {
442 		if (n == USB_FIFO_MAX) {
443 			if (no_null) {
444 				no_null = 0;
445 				n = 0;
446 			} else {
447 				/* end of FIFOs reached */
448 				DPRINTFN(5, "out of FIFOs\n");
449 				return (ENOMEM);
450 			}
451 		}
452 		/* Check for TX FIFO */
453 		if (is_tx) {
454 			f = udev->fifo[n + USB_FIFO_TX];
455 			if (f != NULL) {
456 				if (f->dev_ep_index != e) {
457 					/* wrong endpoint index */
458 					continue;
459 				}
460 				if (f->curr_cpd != NULL) {
461 					/* FIFO is opened */
462 					is_busy = 1;
463 					continue;
464 				}
465 			} else if (no_null) {
466 				continue;
467 			}
468 		}
469 		/* Check for RX FIFO */
470 		if (is_rx) {
471 			f = udev->fifo[n + USB_FIFO_RX];
472 			if (f != NULL) {
473 				if (f->dev_ep_index != e) {
474 					/* wrong endpoint index */
475 					continue;
476 				}
477 				if (f->curr_cpd != NULL) {
478 					/* FIFO is opened */
479 					is_busy = 1;
480 					continue;
481 				}
482 			} else if (no_null) {
483 				continue;
484 			}
485 		}
486 		break;
487 	}
488 
489 	if (no_null == 0) {
490 		if (e >= (USB_EP_MAX / 2)) {
491 			/* we don't create any endpoints in this range */
492 			DPRINTFN(5, "ep out of range\n");
493 			return (is_busy ? EBUSY : EINVAL);
494 		}
495 	}
496 
497 	if ((e != 0) && is_busy) {
498 		/*
499 		 * Only the default control endpoint is allowed to be
500 		 * opened multiple times!
501 		 */
502 		DPRINTFN(5, "busy\n");
503 		return (EBUSY);
504 	}
505 
506 	/* Check TX FIFO */
507 	if (is_tx &&
508 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
509 		ep = usb_dev_get_ep(udev, e, USB_FIFO_TX);
510 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_TX);
511 		if (ep == NULL) {
512 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
513 			return (EINVAL);
514 		}
515 		f = usb_fifo_alloc(&udev->device_mtx);
516 		if (f == NULL) {
517 			DPRINTFN(5, "could not alloc tx fifo\n");
518 			return (ENOMEM);
519 		}
520 		/* update some fields */
521 		f->fifo_index = n + USB_FIFO_TX;
522 		f->dev_ep_index = e;
523 		f->priv_sc0 = ep;
524 		f->methods = &usb_ugen_methods;
525 		f->iface_index = ep->iface_index;
526 		f->udev = udev;
527 		mtx_lock(&usb_ref_lock);
528 		udev->fifo[n + USB_FIFO_TX] = f;
529 		mtx_unlock(&usb_ref_lock);
530 	}
531 	/* Check RX FIFO */
532 	if (is_rx &&
533 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
534 		ep = usb_dev_get_ep(udev, e, USB_FIFO_RX);
535 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_RX);
536 		if (ep == NULL) {
537 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
538 			return (EINVAL);
539 		}
540 		f = usb_fifo_alloc(&udev->device_mtx);
541 		if (f == NULL) {
542 			DPRINTFN(5, "could not alloc rx fifo\n");
543 			return (ENOMEM);
544 		}
545 		/* update some fields */
546 		f->fifo_index = n + USB_FIFO_RX;
547 		f->dev_ep_index = e;
548 		f->priv_sc0 = ep;
549 		f->methods = &usb_ugen_methods;
550 		f->iface_index = ep->iface_index;
551 		f->udev = udev;
552 		mtx_lock(&usb_ref_lock);
553 		udev->fifo[n + USB_FIFO_RX] = f;
554 		mtx_unlock(&usb_ref_lock);
555 	}
556 	if (is_tx) {
557 		crd->txfifo = udev->fifo[n + USB_FIFO_TX];
558 	}
559 	if (is_rx) {
560 		crd->rxfifo = udev->fifo[n + USB_FIFO_RX];
561 	}
562 	/* fill out fifo index */
563 	DPRINTFN(5, "fifo index = %d\n", n);
564 	cpd->fifo_index = n;
565 
566 	/* complete */
567 
568 	return (0);
569 }
570 
571 void
572 usb_fifo_free(struct usb_fifo *f)
573 {
574 	uint8_t n;
575 
576 	if (f == NULL) {
577 		/* be NULL safe */
578 		return;
579 	}
580 	/* destroy symlink devices, if any */
581 	for (n = 0; n != 2; n++) {
582 		if (f->symlink[n]) {
583 			usb_free_symlink(f->symlink[n]);
584 			f->symlink[n] = NULL;
585 		}
586 	}
587 	mtx_lock(&usb_ref_lock);
588 
589 	/* delink ourselves to stop calls from userland */
590 	if ((f->fifo_index < USB_FIFO_MAX) &&
591 	    (f->udev != NULL) &&
592 	    (f->udev->fifo[f->fifo_index] == f)) {
593 		f->udev->fifo[f->fifo_index] = NULL;
594 	} else {
595 		DPRINTFN(0, "USB FIFO %p has not been linked\n", f);
596 	}
597 
598 	/* decrease refcount */
599 	f->refcount--;
600 	/* need to wait until all callers have exited */
601 	while (f->refcount != 0) {
602 		mtx_unlock(&usb_ref_lock);	/* avoid LOR */
603 		mtx_lock(f->priv_mtx);
604 		/* prevent write flush, if any */
605 		f->flag_iserror = 1;
606 		/* get I/O thread out of any sleep state */
607 		if (f->flag_sleeping) {
608 			f->flag_sleeping = 0;
609 			cv_broadcast(&f->cv_io);
610 		}
611 		mtx_unlock(f->priv_mtx);
612 		mtx_lock(&usb_ref_lock);
613 
614 		/*
615 		 * Check if the "f->refcount" variable reached zero
616 		 * during the unlocked time before entering wait:
617 		 */
618 		if (f->refcount == 0)
619 			break;
620 
621 		/* wait for sync */
622 		cv_wait(&f->cv_drain, &usb_ref_lock);
623 	}
624 	mtx_unlock(&usb_ref_lock);
625 
626 	/* take care of closing the device here, if any */
627 	usb_fifo_close(f, 0);
628 
629 	cv_destroy(&f->cv_io);
630 	cv_destroy(&f->cv_drain);
631 
632 	knlist_clear(&f->selinfo.si_note, 0);
633 	seldrain(&f->selinfo);
634 	knlist_destroy(&f->selinfo.si_note);
635 
636 	free(f, M_USBDEV);
637 }
638 
639 static struct usb_endpoint *
640 usb_dev_get_ep(struct usb_device *udev, uint8_t ep_index, uint8_t dir)
641 {
642 	struct usb_endpoint *ep;
643 	uint8_t ep_dir;
644 
645 	if (ep_index == 0) {
646 		ep = &udev->ctrl_ep;
647 	} else {
648 		if (dir == USB_FIFO_RX) {
649 			if (udev->flags.usb_mode == USB_MODE_HOST) {
650 				ep_dir = UE_DIR_IN;
651 			} else {
652 				ep_dir = UE_DIR_OUT;
653 			}
654 		} else {
655 			if (udev->flags.usb_mode == USB_MODE_HOST) {
656 				ep_dir = UE_DIR_OUT;
657 			} else {
658 				ep_dir = UE_DIR_IN;
659 			}
660 		}
661 		ep = usbd_get_ep_by_addr(udev, ep_index | ep_dir);
662 	}
663 
664 	if (ep == NULL) {
665 		/* if the endpoint does not exist then return */
666 		return (NULL);
667 	}
668 	if (ep->edesc == NULL) {
669 		/* invalid endpoint */
670 		return (NULL);
671 	}
672 	return (ep);			/* success */
673 }
674 
675 /*------------------------------------------------------------------------*
676  *	usb_fifo_open
677  *
678  * Returns:
679  * 0: Success
680  * Else: Failure
681  *------------------------------------------------------------------------*/
682 static int
683 usb_fifo_open(struct usb_cdev_privdata *cpd,
684     struct usb_fifo *f, int fflags)
685 {
686 	int err;
687 
688 	if (f == NULL) {
689 		/* no FIFO there */
690 		DPRINTFN(2, "no FIFO\n");
691 		return (ENXIO);
692 	}
693 	/* remove FWRITE and FREAD flags */
694 	fflags &= ~(FWRITE | FREAD);
695 
696 	/* set correct file flags */
697 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
698 		fflags |= FWRITE;
699 	} else {
700 		fflags |= FREAD;
701 	}
702 
703 	/* check if we are already opened */
704 	/* we don't need any locks when checking this variable */
705 	if (f->curr_cpd != NULL) {
706 		err = EBUSY;
707 		goto done;
708 	}
709 
710 	/* reset short flag before open */
711 	f->flag_short = 0;
712 
713 	/* call open method */
714 	err = (f->methods->f_open) (f, fflags);
715 	if (err) {
716 		goto done;
717 	}
718 	mtx_lock(f->priv_mtx);
719 
720 	/* reset sleep flag */
721 	f->flag_sleeping = 0;
722 
723 	/* reset error flag */
724 	f->flag_iserror = 0;
725 
726 	/* reset complete flag */
727 	f->flag_iscomplete = 0;
728 
729 	/* reset select flag */
730 	f->flag_isselect = 0;
731 
732 	/* reset flushing flag */
733 	f->flag_flushing = 0;
734 
735 	/* reset ASYNC proc flag */
736 	f->async_p = NULL;
737 
738 	mtx_lock(&usb_ref_lock);
739 	/* flag the fifo as opened to prevent others */
740 	f->curr_cpd = cpd;
741 	mtx_unlock(&usb_ref_lock);
742 
743 	/* reset queue */
744 	usb_fifo_reset(f);
745 
746 	mtx_unlock(f->priv_mtx);
747 done:
748 	return (err);
749 }
750 
751 /*------------------------------------------------------------------------*
752  *	usb_fifo_reset
753  *------------------------------------------------------------------------*/
754 void
755 usb_fifo_reset(struct usb_fifo *f)
756 {
757 	struct usb_mbuf *m;
758 
759 	if (f == NULL) {
760 		return;
761 	}
762 	while (1) {
763 		USB_IF_DEQUEUE(&f->used_q, m);
764 		if (m) {
765 			USB_IF_ENQUEUE(&f->free_q, m);
766 		} else {
767 			break;
768 		}
769 	}
770 	/* reset have fragment flag */
771 	f->flag_have_fragment = 0;
772 }
773 
774 /*------------------------------------------------------------------------*
775  *	usb_fifo_close
776  *------------------------------------------------------------------------*/
777 static void
778 usb_fifo_close(struct usb_fifo *f, int fflags)
779 {
780 	int err;
781 
782 	/* check if we are not opened */
783 	if (f->curr_cpd == NULL) {
784 		/* nothing to do - already closed */
785 		return;
786 	}
787 	mtx_lock(f->priv_mtx);
788 
789 	/* clear current cdev private data pointer */
790 	mtx_lock(&usb_ref_lock);
791 	f->curr_cpd = NULL;
792 	mtx_unlock(&usb_ref_lock);
793 
794 	/* check if we are watched by kevent */
795 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
796 
797 	/* check if we are selected */
798 	if (f->flag_isselect) {
799 		selwakeup(&f->selinfo);
800 		f->flag_isselect = 0;
801 	}
802 	/* check if a thread wants SIGIO */
803 	if (f->async_p != NULL) {
804 		PROC_LOCK(f->async_p);
805 		kern_psignal(f->async_p, SIGIO);
806 		PROC_UNLOCK(f->async_p);
807 		f->async_p = NULL;
808 	}
809 	/* remove FWRITE and FREAD flags */
810 	fflags &= ~(FWRITE | FREAD);
811 
812 	/* flush written data, if any */
813 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
814 		if (!f->flag_iserror) {
815 			/* set flushing flag */
816 			f->flag_flushing = 1;
817 
818 			/* get the last packet in */
819 			if (f->flag_have_fragment) {
820 				struct usb_mbuf *m;
821 				f->flag_have_fragment = 0;
822 				USB_IF_DEQUEUE(&f->free_q, m);
823 				if (m) {
824 					USB_IF_ENQUEUE(&f->used_q, m);
825 				}
826 			}
827 
828 			/* start write transfer, if not already started */
829 			(f->methods->f_start_write) (f);
830 
831 			/* check if flushed already */
832 			while (f->flag_flushing &&
833 			    (!f->flag_iserror)) {
834 				/* wait until all data has been written */
835 				f->flag_sleeping = 1;
836 				err = cv_timedwait_sig(&f->cv_io, f->priv_mtx,
837 				    USB_MS_TO_TICKS(USB_DEFAULT_TIMEOUT));
838 				if (err) {
839 					DPRINTF("signal received\n");
840 					break;
841 				}
842 			}
843 		}
844 		fflags |= FWRITE;
845 
846 		/* stop write transfer, if not already stopped */
847 		(f->methods->f_stop_write) (f);
848 	} else {
849 		fflags |= FREAD;
850 
851 		/* stop write transfer, if not already stopped */
852 		(f->methods->f_stop_read) (f);
853 	}
854 
855 	/* check if we are sleeping */
856 	if (f->flag_sleeping) {
857 		DPRINTFN(2, "Sleeping at close!\n");
858 	}
859 	mtx_unlock(f->priv_mtx);
860 
861 	/* call close method */
862 	(f->methods->f_close) (f, fflags);
863 
864 	DPRINTF("closed\n");
865 }
866 
867 /*------------------------------------------------------------------------*
868  *	usb_open - cdev callback
869  *------------------------------------------------------------------------*/
870 static int
871 usb_open(struct cdev *dev, int fflags, int devtype, struct thread *td)
872 {
873 	struct usb_fs_privdata* pd = (struct usb_fs_privdata*)dev->si_drv1;
874 	struct usb_cdev_refdata refs;
875 	struct usb_cdev_privdata *cpd;
876 	int err;
877 
878 	DPRINTFN(2, "%s fflags=0x%08x\n", devtoname(dev), fflags);
879 
880 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
881 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
882 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
883 		DPRINTFN(2, "access mode not supported\n");
884 		return (EPERM);
885 	}
886 
887 	cpd = malloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
888 
889 	usb_loc_fill(pd, cpd);
890 	err = usb_ref_device(cpd, &refs, 1);
891 	if (err) {
892 		DPRINTFN(2, "cannot ref device\n");
893 		free(cpd, M_USBDEV);
894 		return (ENXIO);
895 	}
896 	cpd->fflags = fflags;	/* access mode for open lifetime */
897 
898 	/* create FIFOs, if any */
899 	err = usb_fifo_create(cpd, &refs);
900 	/* check for error */
901 	if (err) {
902 		DPRINTFN(2, "cannot create fifo\n");
903 		usb_unref_device(cpd, &refs);
904 		free(cpd, M_USBDEV);
905 		return (err);
906 	}
907 	if (fflags & FREAD) {
908 		err = usb_fifo_open(cpd, refs.rxfifo, fflags);
909 		if (err) {
910 			DPRINTFN(2, "read open failed\n");
911 			usb_unref_device(cpd, &refs);
912 			free(cpd, M_USBDEV);
913 			return (err);
914 		}
915 	}
916 	if (fflags & FWRITE) {
917 		err = usb_fifo_open(cpd, refs.txfifo, fflags);
918 		if (err) {
919 			DPRINTFN(2, "write open failed\n");
920 			if (fflags & FREAD) {
921 				usb_fifo_close(refs.rxfifo, fflags);
922 			}
923 			usb_unref_device(cpd, &refs);
924 			free(cpd, M_USBDEV);
925 			return (err);
926 		}
927 	}
928 	usb_unref_device(cpd, &refs);
929 	devfs_set_cdevpriv(cpd, usb_close);
930 
931 	return (0);
932 }
933 
934 /*------------------------------------------------------------------------*
935  *	usb_close - cdev callback
936  *------------------------------------------------------------------------*/
937 static void
938 usb_close(void *arg)
939 {
940 	struct usb_cdev_refdata refs;
941 	struct usb_cdev_privdata *cpd = arg;
942 	int err;
943 
944 	DPRINTFN(2, "cpd=%p\n", cpd);
945 
946 	err = usb_ref_device(cpd, &refs,
947 	    2 /* uref and allow detached state */);
948 	if (err) {
949 		DPRINTFN(2, "Cannot grab USB reference when "
950 		    "closing USB file handle\n");
951 		goto done;
952 	}
953 	if (cpd->fflags & FREAD) {
954 		usb_fifo_close(refs.rxfifo, cpd->fflags);
955 	}
956 	if (cpd->fflags & FWRITE) {
957 		usb_fifo_close(refs.txfifo, cpd->fflags);
958 	}
959 	usb_unref_device(cpd, &refs);
960 done:
961 	free(cpd, M_USBDEV);
962 }
963 
964 static void
965 usb_dev_init(void *arg)
966 {
967 	mtx_init(&usb_ref_lock, "USB ref mutex", NULL, MTX_DEF);
968 	sx_init(&usb_sym_lock, "USB sym mutex");
969 	TAILQ_INIT(&usb_sym_head);
970 
971 	/* check the UGEN methods */
972 	usb_fifo_check_methods(&usb_ugen_methods);
973 }
974 
975 SYSINIT(usb_dev_init, SI_SUB_KLD, SI_ORDER_FIRST, usb_dev_init, NULL);
976 
977 static void
978 usb_dev_init_post(void *arg)
979 {
980 	/*
981 	 * Create /dev/usb - this is needed for usbconfig(8), which
982 	 * needs a well-known device name to access.
983 	 */
984 	usb_dev = make_dev(&usb_static_devsw, 0, UID_ROOT, GID_OPERATOR,
985 	    0644, USB_DEVICE_NAME);
986 	if (usb_dev == NULL) {
987 		DPRINTFN(0, "Could not create usb bus device\n");
988 	}
989 }
990 
991 SYSINIT(usb_dev_init_post, SI_SUB_KICK_SCHEDULER, SI_ORDER_FIRST, usb_dev_init_post, NULL);
992 
993 static void
994 usb_dev_uninit(void *arg)
995 {
996 	if (usb_dev != NULL) {
997 		destroy_dev(usb_dev);
998 		usb_dev = NULL;
999 	}
1000 	mtx_destroy(&usb_ref_lock);
1001 	sx_destroy(&usb_sym_lock);
1002 }
1003 
1004 SYSUNINIT(usb_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb_dev_uninit, NULL);
1005 
1006 static int
1007 usb_ioctl_f_sub(struct usb_fifo *f, u_long cmd, void *addr,
1008     struct thread *td)
1009 {
1010 	int error = 0;
1011 
1012 	switch (cmd) {
1013 	case FIODTYPE:
1014 		*(int *)addr = 0;	/* character device */
1015 		break;
1016 
1017 	case FIONBIO:
1018 		/* handled by upper FS layer */
1019 		break;
1020 
1021 	case FIOASYNC:
1022 		if (*(int *)addr) {
1023 			if (f->async_p != NULL) {
1024 				error = EBUSY;
1025 				break;
1026 			}
1027 			f->async_p = USB_TD_GET_PROC(td);
1028 		} else {
1029 			f->async_p = NULL;
1030 		}
1031 		break;
1032 
1033 		/* XXX this is not the most general solution */
1034 	case TIOCSPGRP:
1035 		if (f->async_p == NULL) {
1036 			error = EINVAL;
1037 			break;
1038 		}
1039 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
1040 			error = EPERM;
1041 			break;
1042 		}
1043 		break;
1044 	default:
1045 		return (ENOIOCTL);
1046 	}
1047 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
1048 	return (error);
1049 }
1050 
1051 /*------------------------------------------------------------------------*
1052  *	usb_ioctl - cdev callback
1053  *------------------------------------------------------------------------*/
1054 static int
1055 usb_ioctl(struct cdev *dev, u_long cmd, caddr_t addr, int fflag, struct thread* td)
1056 {
1057 	struct usb_cdev_refdata refs;
1058 	struct usb_cdev_privdata* cpd;
1059 	struct usb_fifo *f;
1060 	int fflags;
1061 	int err;
1062 
1063 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1064 
1065 	err = devfs_get_cdevpriv((void **)&cpd);
1066 	if (err != 0)
1067 		return (err);
1068 
1069 	/*
1070 	 * Performance optimisation: We try to check for IOCTL's that
1071 	 * don't need the USB reference first. Then we grab the USB
1072 	 * reference if we need it!
1073 	 */
1074 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1075 	if (err)
1076 		return (ENXIO);
1077 
1078 	fflags = cpd->fflags;
1079 
1080 	f = NULL;			/* set default value */
1081 	err = ENOIOCTL;			/* set default value */
1082 
1083 	if (fflags & FWRITE) {
1084 		f = refs.txfifo;
1085 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1086 	}
1087 	if (fflags & FREAD) {
1088 		f = refs.rxfifo;
1089 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1090 	}
1091 	KASSERT(f != NULL, ("fifo not found"));
1092 	if (err != ENOIOCTL)
1093 		goto done;
1094 
1095 	err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1096 
1097 	DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1098 
1099 	if (err != ENOIOCTL)
1100 		goto done;
1101 
1102 	if (usb_usb_ref_device(cpd, &refs)) {
1103 		/* we lost the reference */
1104 		return (ENXIO);
1105 	}
1106 
1107 	err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1108 
1109 	DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1110 
1111 	if (err == ENOIOCTL)
1112 		err = ENOTTY;
1113 
1114 	if (err)
1115 		goto done;
1116 
1117 	/* Wait for re-enumeration, if any */
1118 
1119 	while (f->udev->re_enumerate_wait != USB_RE_ENUM_DONE) {
1120 		usb_unref_device(cpd, &refs);
1121 
1122 		usb_pause_mtx(NULL, hz / 128);
1123 
1124 		while (usb_ref_device(cpd, &refs, 1 /* need uref */)) {
1125 			if (usb_ref_device(cpd, &refs, 0)) {
1126 				/* device no longer exists */
1127 				return (ENXIO);
1128 			}
1129 			usb_unref_device(cpd, &refs);
1130 			usb_pause_mtx(NULL, hz / 128);
1131 		}
1132 	}
1133 
1134 done:
1135 	usb_unref_device(cpd, &refs);
1136 	return (err);
1137 }
1138 
1139 static void
1140 usb_filter_detach(struct knote *kn)
1141 {
1142 	struct usb_fifo *f = kn->kn_hook;
1143 	knlist_remove(&f->selinfo.si_note, kn, 0);
1144 }
1145 
1146 static int
1147 usb_filter_write(struct knote *kn, long hint)
1148 {
1149 	struct usb_cdev_privdata* cpd;
1150 	struct usb_fifo *f;
1151 	struct usb_mbuf *m;
1152 
1153 	DPRINTFN(2, "\n");
1154 
1155 	f = kn->kn_hook;
1156 
1157 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1158 
1159 	cpd = f->curr_cpd;
1160 	if (cpd == NULL) {
1161 		m = (void *)1;
1162 	} else if (f->fs_ep_max == 0) {
1163 		if (f->flag_iserror) {
1164 			/* we got an error */
1165 			m = (void *)1;
1166 		} else {
1167 			if (f->queue_data == NULL) {
1168 				/*
1169 				 * start write transfer, if not
1170 				 * already started
1171 				 */
1172 				(f->methods->f_start_write) (f);
1173 			}
1174 			/* check if any packets are available */
1175 			USB_IF_POLL(&f->free_q, m);
1176 		}
1177 	} else {
1178 		if (f->flag_iscomplete) {
1179 			m = (void *)1;
1180 		} else {
1181 			m = NULL;
1182 		}
1183 	}
1184 	return (m ? 1 : 0);
1185 }
1186 
1187 static int
1188 usb_filter_read(struct knote *kn, long hint)
1189 {
1190 	struct usb_cdev_privdata* cpd;
1191 	struct usb_fifo *f;
1192 	struct usb_mbuf *m;
1193 
1194 	DPRINTFN(2, "\n");
1195 
1196 	f = kn->kn_hook;
1197 
1198 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1199 
1200 	cpd = f->curr_cpd;
1201 	if (cpd == NULL) {
1202 		m = (void *)1;
1203 	} else if (f->fs_ep_max == 0) {
1204 		if (f->flag_iserror) {
1205 			/* we have an error */
1206 			m = (void *)1;
1207 		} else {
1208 			if (f->queue_data == NULL) {
1209 				/*
1210 				 * start read transfer, if not
1211 				 * already started
1212 				 */
1213 				(f->methods->f_start_read) (f);
1214 			}
1215 			/* check if any packets are available */
1216 			USB_IF_POLL(&f->used_q, m);
1217 
1218 			/* start reading data, if any */
1219 			if (m == NULL)
1220 				(f->methods->f_start_read) (f);
1221 		}
1222 	} else {
1223 		if (f->flag_iscomplete) {
1224 			m = (void *)1;
1225 		} else {
1226 			m = NULL;
1227 		}
1228 	}
1229 	return (m ? 1 : 0);
1230 }
1231 
1232 static struct filterops usb_filtops_write = {
1233 	.f_isfd = 1,
1234 	.f_detach = usb_filter_detach,
1235 	.f_event = usb_filter_write,
1236 };
1237 
1238 static struct filterops usb_filtops_read = {
1239 	.f_isfd = 1,
1240 	.f_detach = usb_filter_detach,
1241 	.f_event = usb_filter_read,
1242 };
1243 
1244 /* ARGSUSED */
1245 static int
1246 usb_kqfilter(struct cdev* dev, struct knote *kn)
1247 {
1248 	struct usb_cdev_refdata refs;
1249 	struct usb_cdev_privdata* cpd;
1250 	struct usb_fifo *f;
1251 	int fflags;
1252 	int err = EINVAL;
1253 
1254 	DPRINTFN(2, "\n");
1255 
1256 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1257 	    usb_ref_device(cpd, &refs, 0) != 0)
1258 		return (ENXIO);
1259 
1260 	fflags = cpd->fflags;
1261 
1262 	/* Figure out who needs service */
1263 	switch (kn->kn_filter) {
1264 	case EVFILT_WRITE:
1265 		if (fflags & FWRITE) {
1266 			f = refs.txfifo;
1267 			kn->kn_fop = &usb_filtops_write;
1268 			err = 0;
1269 		}
1270 		break;
1271 	case EVFILT_READ:
1272 		if (fflags & FREAD) {
1273 			f = refs.rxfifo;
1274 			kn->kn_fop = &usb_filtops_read;
1275 			err = 0;
1276 		}
1277 		break;
1278 	default:
1279 		err = EOPNOTSUPP;
1280 		break;
1281 	}
1282 
1283 	if (err == 0) {
1284 		kn->kn_hook = f;
1285 		mtx_lock(f->priv_mtx);
1286 		knlist_add(&f->selinfo.si_note, kn, 1);
1287 		mtx_unlock(f->priv_mtx);
1288 	}
1289 
1290 	usb_unref_device(cpd, &refs);
1291 	return (err);
1292 }
1293 
1294 /* ARGSUSED */
1295 static int
1296 usb_poll(struct cdev* dev, int events, struct thread* td)
1297 {
1298 	struct usb_cdev_refdata refs;
1299 	struct usb_cdev_privdata* cpd;
1300 	struct usb_fifo *f;
1301 	struct usb_mbuf *m;
1302 	int fflags, revents;
1303 
1304 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1305 	    usb_ref_device(cpd, &refs, 0) != 0)
1306 		return (events &
1307 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1308 
1309 	fflags = cpd->fflags;
1310 
1311 	/* Figure out who needs service */
1312 	revents = 0;
1313 	if ((events & (POLLOUT | POLLWRNORM)) &&
1314 	    (fflags & FWRITE)) {
1315 		f = refs.txfifo;
1316 
1317 		mtx_lock(f->priv_mtx);
1318 
1319 		if (!refs.is_usbfs) {
1320 			if (f->flag_iserror) {
1321 				/* we got an error */
1322 				m = (void *)1;
1323 			} else {
1324 				if (f->queue_data == NULL) {
1325 					/*
1326 					 * start write transfer, if not
1327 					 * already started
1328 					 */
1329 					(f->methods->f_start_write) (f);
1330 				}
1331 				/* check if any packets are available */
1332 				USB_IF_POLL(&f->free_q, m);
1333 			}
1334 		} else {
1335 			if (f->flag_iscomplete) {
1336 				m = (void *)1;
1337 			} else {
1338 				m = NULL;
1339 			}
1340 		}
1341 
1342 		if (m) {
1343 			revents |= events & (POLLOUT | POLLWRNORM);
1344 		} else {
1345 			f->flag_isselect = 1;
1346 			selrecord(td, &f->selinfo);
1347 		}
1348 
1349 		mtx_unlock(f->priv_mtx);
1350 	}
1351 	if ((events & (POLLIN | POLLRDNORM)) &&
1352 	    (fflags & FREAD)) {
1353 		f = refs.rxfifo;
1354 
1355 		mtx_lock(f->priv_mtx);
1356 
1357 		if (!refs.is_usbfs) {
1358 			if (f->flag_iserror) {
1359 				/* we have an error */
1360 				m = (void *)1;
1361 			} else {
1362 				if (f->queue_data == NULL) {
1363 					/*
1364 					 * start read transfer, if not
1365 					 * already started
1366 					 */
1367 					(f->methods->f_start_read) (f);
1368 				}
1369 				/* check if any packets are available */
1370 				USB_IF_POLL(&f->used_q, m);
1371 			}
1372 		} else {
1373 			if (f->flag_iscomplete) {
1374 				m = (void *)1;
1375 			} else {
1376 				m = NULL;
1377 			}
1378 		}
1379 
1380 		if (m) {
1381 			revents |= events & (POLLIN | POLLRDNORM);
1382 		} else {
1383 			f->flag_isselect = 1;
1384 			selrecord(td, &f->selinfo);
1385 
1386 			if (!refs.is_usbfs) {
1387 				/* start reading data */
1388 				(f->methods->f_start_read) (f);
1389 			}
1390 		}
1391 
1392 		mtx_unlock(f->priv_mtx);
1393 	}
1394 	usb_unref_device(cpd, &refs);
1395 	return (revents);
1396 }
1397 
1398 static int
1399 usb_read(struct cdev *dev, struct uio *uio, int ioflag)
1400 {
1401 	struct usb_cdev_refdata refs;
1402 	struct usb_cdev_privdata* cpd;
1403 	struct usb_fifo *f;
1404 	struct usb_mbuf *m;
1405 	int io_len;
1406 	int err;
1407 	uint8_t tr_data = 0;
1408 
1409 	err = devfs_get_cdevpriv((void **)&cpd);
1410 	if (err != 0)
1411 		return (err);
1412 
1413 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1414 	if (err)
1415 		return (ENXIO);
1416 
1417 	f = refs.rxfifo;
1418 	if (f == NULL) {
1419 		/* should not happen */
1420 		usb_unref_device(cpd, &refs);
1421 		return (EPERM);
1422 	}
1423 
1424 	mtx_lock(f->priv_mtx);
1425 
1426 	/* check for permanent read error */
1427 	if (f->flag_iserror) {
1428 		err = EIO;
1429 		goto done;
1430 	}
1431 	/* check if USB-FS interface is active */
1432 	if (refs.is_usbfs) {
1433 		/*
1434 		 * The queue is used for events that should be
1435 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1436 		 */
1437 		err = EINVAL;
1438 		goto done;
1439 	}
1440 	while (uio->uio_resid > 0) {
1441 		USB_IF_DEQUEUE(&f->used_q, m);
1442 
1443 		if (m == NULL) {
1444 			/* start read transfer, if not already started */
1445 
1446 			(f->methods->f_start_read) (f);
1447 
1448 			if (ioflag & IO_NDELAY) {
1449 				if (tr_data) {
1450 					/* return length before error */
1451 					break;
1452 				}
1453 				err = EWOULDBLOCK;
1454 				break;
1455 			}
1456 			DPRINTF("sleeping\n");
1457 
1458 			err = usb_fifo_wait(f);
1459 			if (err) {
1460 				break;
1461 			}
1462 			continue;
1463 		}
1464 		if (f->methods->f_filter_read) {
1465 			/*
1466 			 * Sometimes it is convenient to process data at the
1467 			 * expense of a userland process instead of a kernel
1468 			 * process.
1469 			 */
1470 			(f->methods->f_filter_read) (f, m);
1471 		}
1472 		tr_data = 1;
1473 
1474 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1475 
1476 		DPRINTFN(2, "transfer %d bytes from %p\n",
1477 		    io_len, m->cur_data_ptr);
1478 
1479 		err = usb_fifo_uiomove(f,
1480 		    m->cur_data_ptr, io_len, uio);
1481 
1482 		m->cur_data_len -= io_len;
1483 		m->cur_data_ptr += io_len;
1484 
1485 		if (m->cur_data_len == 0) {
1486 			uint8_t last_packet;
1487 
1488 			last_packet = m->last_packet;
1489 
1490 			USB_IF_ENQUEUE(&f->free_q, m);
1491 
1492 			if (last_packet) {
1493 				/* keep framing */
1494 				break;
1495 			}
1496 		} else {
1497 			USB_IF_PREPEND(&f->used_q, m);
1498 		}
1499 
1500 		if (err) {
1501 			break;
1502 		}
1503 	}
1504 done:
1505 	mtx_unlock(f->priv_mtx);
1506 
1507 	usb_unref_device(cpd, &refs);
1508 
1509 	return (err);
1510 }
1511 
1512 static int
1513 usb_write(struct cdev *dev, struct uio *uio, int ioflag)
1514 {
1515 	struct usb_cdev_refdata refs;
1516 	struct usb_cdev_privdata* cpd;
1517 	struct usb_fifo *f;
1518 	struct usb_mbuf *m;
1519 	uint8_t *pdata;
1520 	int io_len;
1521 	int err;
1522 	uint8_t tr_data = 0;
1523 
1524 	DPRINTFN(2, "\n");
1525 
1526 	err = devfs_get_cdevpriv((void **)&cpd);
1527 	if (err != 0)
1528 		return (err);
1529 
1530 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1531 	if (err)
1532 		return (ENXIO);
1533 
1534 	f = refs.txfifo;
1535 	if (f == NULL) {
1536 		/* should not happen */
1537 		usb_unref_device(cpd, &refs);
1538 		return (EPERM);
1539 	}
1540 
1541 	mtx_lock(f->priv_mtx);
1542 
1543 	/* check for permanent write error */
1544 	if (f->flag_iserror) {
1545 		err = EIO;
1546 		goto done;
1547 	}
1548 	/* check if USB-FS interface is active */
1549 	if (refs.is_usbfs) {
1550 		/*
1551 		 * The queue is used for events that should be
1552 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1553 		 */
1554 		err = EINVAL;
1555 		goto done;
1556 	}
1557 	if (f->queue_data == NULL) {
1558 		/* start write transfer, if not already started */
1559 		(f->methods->f_start_write) (f);
1560 	}
1561 	/* we allow writing zero length data */
1562 	do {
1563 		USB_IF_DEQUEUE(&f->free_q, m);
1564 
1565 		if (m == NULL) {
1566 			if (ioflag & IO_NDELAY) {
1567 				if (tr_data) {
1568 					/* return length before error */
1569 					break;
1570 				}
1571 				err = EWOULDBLOCK;
1572 				break;
1573 			}
1574 			DPRINTF("sleeping\n");
1575 
1576 			err = usb_fifo_wait(f);
1577 			if (err) {
1578 				break;
1579 			}
1580 			continue;
1581 		}
1582 		tr_data = 1;
1583 
1584 		if (f->flag_have_fragment == 0) {
1585 			USB_MBUF_RESET(m);
1586 			io_len = m->cur_data_len;
1587 			pdata = m->cur_data_ptr;
1588 			if (io_len > uio->uio_resid)
1589 				io_len = uio->uio_resid;
1590 			m->cur_data_len = io_len;
1591 		} else {
1592 			io_len = m->max_data_len - m->cur_data_len;
1593 			pdata = m->cur_data_ptr + m->cur_data_len;
1594 			if (io_len > uio->uio_resid)
1595 				io_len = uio->uio_resid;
1596 			m->cur_data_len += io_len;
1597 		}
1598 
1599 		DPRINTFN(2, "transfer %d bytes to %p\n",
1600 		    io_len, pdata);
1601 
1602 		err = usb_fifo_uiomove(f, pdata, io_len, uio);
1603 
1604 		if (err) {
1605 			f->flag_have_fragment = 0;
1606 			USB_IF_ENQUEUE(&f->free_q, m);
1607 			break;
1608 		}
1609 
1610 		/* check if the buffer is ready to be transmitted */
1611 
1612 		if ((f->flag_write_defrag == 0) ||
1613 		    (m->cur_data_len == m->max_data_len)) {
1614 			f->flag_have_fragment = 0;
1615 
1616 			/*
1617 			 * Check for write filter:
1618 			 *
1619 			 * Sometimes it is convenient to process data
1620 			 * at the expense of a userland process
1621 			 * instead of a kernel process.
1622 			 */
1623 			if (f->methods->f_filter_write) {
1624 				(f->methods->f_filter_write) (f, m);
1625 			}
1626 
1627 			/* Put USB mbuf in the used queue */
1628 			USB_IF_ENQUEUE(&f->used_q, m);
1629 
1630 			/* Start writing data, if not already started */
1631 			(f->methods->f_start_write) (f);
1632 		} else {
1633 			/* Wait for more data or close */
1634 			f->flag_have_fragment = 1;
1635 			USB_IF_PREPEND(&f->free_q, m);
1636 		}
1637 
1638 	} while (uio->uio_resid > 0);
1639 done:
1640 	mtx_unlock(f->priv_mtx);
1641 
1642 	usb_unref_device(cpd, &refs);
1643 
1644 	return (err);
1645 }
1646 
1647 int
1648 usb_static_ioctl(struct cdev *dev, u_long cmd, caddr_t data, int fflag,
1649     struct thread *td)
1650 {
1651 	union {
1652 		struct usb_read_dir *urd;
1653 		void* data;
1654 	} u;
1655 	int err;
1656 
1657 	u.data = data;
1658 	switch (cmd) {
1659 		case USB_READ_DIR:
1660 			err = usb_read_symlink(u.urd->urd_data,
1661 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1662 			break;
1663 		case USB_DEV_QUIRK_GET:
1664 		case USB_QUIRK_NAME_GET:
1665 		case USB_DEV_QUIRK_ADD:
1666 		case USB_DEV_QUIRK_REMOVE:
1667 			err = usb_quirk_ioctl_p(cmd, data, fflag, td);
1668 			break;
1669 		case USB_GET_TEMPLATE:
1670 			*(int *)data = usb_template;
1671 			err = 0;
1672 			break;
1673 		case USB_SET_TEMPLATE:
1674 			err = priv_check(curthread, PRIV_DRIVER);
1675 			if (err)
1676 				break;
1677 			usb_template = *(int *)data;
1678 			break;
1679 		default:
1680 			err = ENOTTY;
1681 			break;
1682 	}
1683 	return (err);
1684 }
1685 
1686 static int
1687 usb_fifo_uiomove(struct usb_fifo *f, void *cp,
1688     int n, struct uio *uio)
1689 {
1690 	int error;
1691 
1692 	mtx_unlock(f->priv_mtx);
1693 
1694 	/*
1695 	 * "uiomove()" can sleep so one needs to make a wrapper,
1696 	 * exiting the mutex and checking things:
1697 	 */
1698 	error = uiomove(cp, n, uio);
1699 
1700 	mtx_lock(f->priv_mtx);
1701 
1702 	return (error);
1703 }
1704 
1705 int
1706 usb_fifo_wait(struct usb_fifo *f)
1707 {
1708 	int err;
1709 
1710 	USB_MTX_ASSERT(f->priv_mtx, MA_OWNED);
1711 
1712 	if (f->flag_iserror) {
1713 		/* we are gone */
1714 		return (EIO);
1715 	}
1716 	f->flag_sleeping = 1;
1717 
1718 	err = cv_wait_sig(&f->cv_io, f->priv_mtx);
1719 
1720 	if (f->flag_iserror) {
1721 		/* we are gone */
1722 		err = EIO;
1723 	}
1724 	return (err);
1725 }
1726 
1727 void
1728 usb_fifo_signal(struct usb_fifo *f)
1729 {
1730 	if (f->flag_sleeping) {
1731 		f->flag_sleeping = 0;
1732 		cv_broadcast(&f->cv_io);
1733 	}
1734 }
1735 
1736 void
1737 usb_fifo_wakeup(struct usb_fifo *f)
1738 {
1739 	usb_fifo_signal(f);
1740 
1741 	KNOTE_LOCKED(&f->selinfo.si_note, 0);
1742 
1743 	if (f->flag_isselect) {
1744 		selwakeup(&f->selinfo);
1745 		f->flag_isselect = 0;
1746 	}
1747 	if (f->async_p != NULL) {
1748 		PROC_LOCK(f->async_p);
1749 		kern_psignal(f->async_p, SIGIO);
1750 		PROC_UNLOCK(f->async_p);
1751 	}
1752 }
1753 
1754 static int
1755 usb_fifo_dummy_open(struct usb_fifo *fifo, int fflags)
1756 {
1757 	return (0);
1758 }
1759 
1760 static void
1761 usb_fifo_dummy_close(struct usb_fifo *fifo, int fflags)
1762 {
1763 	return;
1764 }
1765 
1766 static int
1767 usb_fifo_dummy_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags)
1768 {
1769 	return (ENOIOCTL);
1770 }
1771 
1772 static void
1773 usb_fifo_dummy_cmd(struct usb_fifo *fifo)
1774 {
1775 	fifo->flag_flushing = 0;	/* not flushing */
1776 }
1777 
1778 static void
1779 usb_fifo_check_methods(struct usb_fifo_methods *pm)
1780 {
1781 	/* check that all callback functions are OK */
1782 
1783 	if (pm->f_open == NULL)
1784 		pm->f_open = &usb_fifo_dummy_open;
1785 
1786 	if (pm->f_close == NULL)
1787 		pm->f_close = &usb_fifo_dummy_close;
1788 
1789 	if (pm->f_ioctl == NULL)
1790 		pm->f_ioctl = &usb_fifo_dummy_ioctl;
1791 
1792 	if (pm->f_ioctl_post == NULL)
1793 		pm->f_ioctl_post = &usb_fifo_dummy_ioctl;
1794 
1795 	if (pm->f_start_read == NULL)
1796 		pm->f_start_read = &usb_fifo_dummy_cmd;
1797 
1798 	if (pm->f_stop_read == NULL)
1799 		pm->f_stop_read = &usb_fifo_dummy_cmd;
1800 
1801 	if (pm->f_start_write == NULL)
1802 		pm->f_start_write = &usb_fifo_dummy_cmd;
1803 
1804 	if (pm->f_stop_write == NULL)
1805 		pm->f_stop_write = &usb_fifo_dummy_cmd;
1806 }
1807 
1808 /*------------------------------------------------------------------------*
1809  *	usb_fifo_attach
1810  *
1811  * The following function will create a duplex FIFO.
1812  *
1813  * Return values:
1814  * 0: Success.
1815  * Else: Failure.
1816  *------------------------------------------------------------------------*/
1817 int
1818 usb_fifo_attach(struct usb_device *udev, void *priv_sc,
1819     struct mtx *priv_mtx, struct usb_fifo_methods *pm,
1820     struct usb_fifo_sc *f_sc, uint16_t unit, int16_t subunit,
1821     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1822 {
1823 	struct usb_fifo *f_tx;
1824 	struct usb_fifo *f_rx;
1825 	char devname[32];
1826 	uint8_t n;
1827 
1828 	f_sc->fp[USB_FIFO_TX] = NULL;
1829 	f_sc->fp[USB_FIFO_RX] = NULL;
1830 
1831 	if (pm == NULL)
1832 		return (EINVAL);
1833 
1834 	/* check the methods */
1835 	usb_fifo_check_methods(pm);
1836 
1837 	if (priv_mtx == NULL)
1838 		priv_mtx = &Giant;
1839 
1840 	/* search for a free FIFO slot */
1841 	for (n = 0;; n += 2) {
1842 		if (n == USB_FIFO_MAX) {
1843 			/* end of FIFOs reached */
1844 			return (ENOMEM);
1845 		}
1846 		/* Check for TX FIFO */
1847 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1848 			continue;
1849 		}
1850 		/* Check for RX FIFO */
1851 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1852 			continue;
1853 		}
1854 		break;
1855 	}
1856 
1857 	f_tx = usb_fifo_alloc(priv_mtx);
1858 	f_rx = usb_fifo_alloc(priv_mtx);
1859 
1860 	if ((f_tx == NULL) || (f_rx == NULL)) {
1861 		usb_fifo_free(f_tx);
1862 		usb_fifo_free(f_rx);
1863 		return (ENOMEM);
1864 	}
1865 	/* initialise FIFO structures */
1866 
1867 	f_tx->fifo_index = n + USB_FIFO_TX;
1868 	f_tx->dev_ep_index = -1;
1869 	f_tx->priv_sc0 = priv_sc;
1870 	f_tx->methods = pm;
1871 	f_tx->iface_index = iface_index;
1872 	f_tx->udev = udev;
1873 
1874 	f_rx->fifo_index = n + USB_FIFO_RX;
1875 	f_rx->dev_ep_index = -1;
1876 	f_rx->priv_sc0 = priv_sc;
1877 	f_rx->methods = pm;
1878 	f_rx->iface_index = iface_index;
1879 	f_rx->udev = udev;
1880 
1881 	f_sc->fp[USB_FIFO_TX] = f_tx;
1882 	f_sc->fp[USB_FIFO_RX] = f_rx;
1883 
1884 	mtx_lock(&usb_ref_lock);
1885 	udev->fifo[f_tx->fifo_index] = f_tx;
1886 	udev->fifo[f_rx->fifo_index] = f_rx;
1887 	mtx_unlock(&usb_ref_lock);
1888 
1889 	for (n = 0; n != 4; n++) {
1890 		if (pm->basename[n] == NULL) {
1891 			continue;
1892 		}
1893 		if (subunit < 0) {
1894 			if (snprintf(devname, sizeof(devname),
1895 			    "%s%u%s", pm->basename[n],
1896 			    unit, pm->postfix[n] ?
1897 			    pm->postfix[n] : "")) {
1898 				/* ignore */
1899 			}
1900 		} else {
1901 			if (snprintf(devname, sizeof(devname),
1902 			    "%s%u.%d%s", pm->basename[n],
1903 			    unit, subunit, pm->postfix[n] ?
1904 			    pm->postfix[n] : "")) {
1905 				/* ignore */
1906 			}
1907 		}
1908 
1909 		/*
1910 		 * Distribute the symbolic links into two FIFO structures:
1911 		 */
1912 		if (n & 1) {
1913 			f_rx->symlink[n / 2] =
1914 			    usb_alloc_symlink(devname);
1915 		} else {
1916 			f_tx->symlink[n / 2] =
1917 			    usb_alloc_symlink(devname);
1918 		}
1919 
1920 		/* Create the device */
1921 		f_sc->dev = usb_make_dev(udev, devname, -1,
1922 		    f_tx->fifo_index & f_rx->fifo_index,
1923 		    FREAD|FWRITE, uid, gid, mode);
1924 	}
1925 
1926 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1927 	return (0);
1928 }
1929 
1930 /*------------------------------------------------------------------------*
1931  *	usb_fifo_alloc_buffer
1932  *
1933  * Return values:
1934  * 0: Success
1935  * Else failure
1936  *------------------------------------------------------------------------*/
1937 int
1938 usb_fifo_alloc_buffer(struct usb_fifo *f, usb_size_t bufsize,
1939     uint16_t nbuf)
1940 {
1941 	usb_fifo_free_buffer(f);
1942 
1943 	/* allocate an endpoint */
1944 	f->free_q.ifq_maxlen = nbuf;
1945 	f->used_q.ifq_maxlen = nbuf;
1946 
1947 	f->queue_data = usb_alloc_mbufs(
1948 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1949 
1950 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1951 		return (ENOMEM);
1952 	}
1953 	return (0);			/* success */
1954 }
1955 
1956 /*------------------------------------------------------------------------*
1957  *	usb_fifo_free_buffer
1958  *
1959  * This function will free the buffers associated with a FIFO. This
1960  * function can be called multiple times in a row.
1961  *------------------------------------------------------------------------*/
1962 void
1963 usb_fifo_free_buffer(struct usb_fifo *f)
1964 {
1965 	if (f->queue_data) {
1966 		/* free old buffer */
1967 		free(f->queue_data, M_USBDEV);
1968 		f->queue_data = NULL;
1969 	}
1970 	/* reset queues */
1971 
1972 	memset(&f->free_q, 0, sizeof(f->free_q));
1973 	memset(&f->used_q, 0, sizeof(f->used_q));
1974 }
1975 
1976 void
1977 usb_fifo_detach(struct usb_fifo_sc *f_sc)
1978 {
1979 	if (f_sc == NULL) {
1980 		return;
1981 	}
1982 	usb_fifo_free(f_sc->fp[USB_FIFO_TX]);
1983 	usb_fifo_free(f_sc->fp[USB_FIFO_RX]);
1984 
1985 	f_sc->fp[USB_FIFO_TX] = NULL;
1986 	f_sc->fp[USB_FIFO_RX] = NULL;
1987 
1988 	usb_destroy_dev(f_sc->dev);
1989 
1990 	f_sc->dev = NULL;
1991 
1992 	DPRINTFN(2, "detached %p\n", f_sc);
1993 }
1994 
1995 usb_size_t
1996 usb_fifo_put_bytes_max(struct usb_fifo *f)
1997 {
1998 	struct usb_mbuf *m;
1999 	usb_size_t len;
2000 
2001 	USB_IF_POLL(&f->free_q, m);
2002 
2003 	if (m) {
2004 		len = m->max_data_len;
2005 	} else {
2006 		len = 0;
2007 	}
2008 	return (len);
2009 }
2010 
2011 /*------------------------------------------------------------------------*
2012  *	usb_fifo_put_data
2013  *
2014  * what:
2015  *  0 - normal operation
2016  *  1 - set last packet flag to enforce framing
2017  *------------------------------------------------------------------------*/
2018 void
2019 usb_fifo_put_data(struct usb_fifo *f, struct usb_page_cache *pc,
2020     usb_frlength_t offset, usb_frlength_t len, uint8_t what)
2021 {
2022 	struct usb_mbuf *m;
2023 	usb_frlength_t io_len;
2024 
2025 	while (len || (what == 1)) {
2026 		USB_IF_DEQUEUE(&f->free_q, m);
2027 
2028 		if (m) {
2029 			USB_MBUF_RESET(m);
2030 
2031 			io_len = MIN(len, m->cur_data_len);
2032 
2033 			usbd_copy_out(pc, offset, m->cur_data_ptr, io_len);
2034 
2035 			m->cur_data_len = io_len;
2036 			offset += io_len;
2037 			len -= io_len;
2038 
2039 			if ((len == 0) && (what == 1)) {
2040 				m->last_packet = 1;
2041 			}
2042 			USB_IF_ENQUEUE(&f->used_q, m);
2043 
2044 			usb_fifo_wakeup(f);
2045 
2046 			if ((len == 0) || (what == 1)) {
2047 				break;
2048 			}
2049 		} else {
2050 			break;
2051 		}
2052 	}
2053 }
2054 
2055 void
2056 usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
2057     usb_size_t len, uint8_t what)
2058 {
2059 	struct usb_mbuf *m;
2060 	usb_size_t io_len;
2061 
2062 	while (len || (what == 1)) {
2063 		USB_IF_DEQUEUE(&f->free_q, m);
2064 
2065 		if (m) {
2066 			USB_MBUF_RESET(m);
2067 
2068 			io_len = MIN(len, m->cur_data_len);
2069 
2070 			memcpy(m->cur_data_ptr, ptr, io_len);
2071 
2072 			m->cur_data_len = io_len;
2073 			ptr = USB_ADD_BYTES(ptr, io_len);
2074 			len -= io_len;
2075 
2076 			if ((len == 0) && (what == 1)) {
2077 				m->last_packet = 1;
2078 			}
2079 			USB_IF_ENQUEUE(&f->used_q, m);
2080 
2081 			usb_fifo_wakeup(f);
2082 
2083 			if ((len == 0) || (what == 1)) {
2084 				break;
2085 			}
2086 		} else {
2087 			break;
2088 		}
2089 	}
2090 }
2091 
2092 uint8_t
2093 usb_fifo_put_data_buffer(struct usb_fifo *f, void *ptr, usb_size_t len)
2094 {
2095 	struct usb_mbuf *m;
2096 
2097 	USB_IF_DEQUEUE(&f->free_q, m);
2098 
2099 	if (m) {
2100 		m->cur_data_len = len;
2101 		m->cur_data_ptr = ptr;
2102 		USB_IF_ENQUEUE(&f->used_q, m);
2103 		usb_fifo_wakeup(f);
2104 		return (1);
2105 	}
2106 	return (0);
2107 }
2108 
2109 void
2110 usb_fifo_put_data_error(struct usb_fifo *f)
2111 {
2112 	f->flag_iserror = 1;
2113 	usb_fifo_wakeup(f);
2114 }
2115 
2116 /*------------------------------------------------------------------------*
2117  *	usb_fifo_get_data
2118  *
2119  * what:
2120  *  0 - normal operation
2121  *  1 - only get one "usb_mbuf"
2122  *
2123  * returns:
2124  *  0 - no more data
2125  *  1 - data in buffer
2126  *------------------------------------------------------------------------*/
2127 uint8_t
2128 usb_fifo_get_data(struct usb_fifo *f, struct usb_page_cache *pc,
2129     usb_frlength_t offset, usb_frlength_t len, usb_frlength_t *actlen,
2130     uint8_t what)
2131 {
2132 	struct usb_mbuf *m;
2133 	usb_frlength_t io_len;
2134 	uint8_t tr_data = 0;
2135 
2136 	actlen[0] = 0;
2137 
2138 	while (1) {
2139 		USB_IF_DEQUEUE(&f->used_q, m);
2140 
2141 		if (m) {
2142 			tr_data = 1;
2143 
2144 			io_len = MIN(len, m->cur_data_len);
2145 
2146 			usbd_copy_in(pc, offset, m->cur_data_ptr, io_len);
2147 
2148 			len -= io_len;
2149 			offset += io_len;
2150 			actlen[0] += io_len;
2151 			m->cur_data_ptr += io_len;
2152 			m->cur_data_len -= io_len;
2153 
2154 			if ((m->cur_data_len == 0) || (what == 1)) {
2155 				USB_IF_ENQUEUE(&f->free_q, m);
2156 
2157 				usb_fifo_wakeup(f);
2158 
2159 				if (what == 1) {
2160 					break;
2161 				}
2162 			} else {
2163 				USB_IF_PREPEND(&f->used_q, m);
2164 			}
2165 		} else {
2166 			if (tr_data) {
2167 				/* wait for data to be written out */
2168 				break;
2169 			}
2170 			if (f->flag_flushing) {
2171 				/* check if we should send a short packet */
2172 				if (f->flag_short != 0) {
2173 					f->flag_short = 0;
2174 					tr_data = 1;
2175 					break;
2176 				}
2177 				/* flushing complete */
2178 				f->flag_flushing = 0;
2179 				usb_fifo_wakeup(f);
2180 			}
2181 			break;
2182 		}
2183 		if (len == 0) {
2184 			break;
2185 		}
2186 	}
2187 	return (tr_data);
2188 }
2189 
2190 uint8_t
2191 usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
2192     usb_size_t len, usb_size_t *actlen, uint8_t what)
2193 {
2194 	struct usb_mbuf *m;
2195 	usb_size_t io_len;
2196 	uint8_t tr_data = 0;
2197 
2198 	actlen[0] = 0;
2199 
2200 	while (1) {
2201 		USB_IF_DEQUEUE(&f->used_q, m);
2202 
2203 		if (m) {
2204 			tr_data = 1;
2205 
2206 			io_len = MIN(len, m->cur_data_len);
2207 
2208 			memcpy(ptr, m->cur_data_ptr, io_len);
2209 
2210 			len -= io_len;
2211 			ptr = USB_ADD_BYTES(ptr, io_len);
2212 			actlen[0] += io_len;
2213 			m->cur_data_ptr += io_len;
2214 			m->cur_data_len -= io_len;
2215 
2216 			if ((m->cur_data_len == 0) || (what == 1)) {
2217 				USB_IF_ENQUEUE(&f->free_q, m);
2218 
2219 				usb_fifo_wakeup(f);
2220 
2221 				if (what == 1) {
2222 					break;
2223 				}
2224 			} else {
2225 				USB_IF_PREPEND(&f->used_q, m);
2226 			}
2227 		} else {
2228 			if (tr_data) {
2229 				/* wait for data to be written out */
2230 				break;
2231 			}
2232 			if (f->flag_flushing) {
2233 				/* check if we should send a short packet */
2234 				if (f->flag_short != 0) {
2235 					f->flag_short = 0;
2236 					tr_data = 1;
2237 					break;
2238 				}
2239 				/* flushing complete */
2240 				f->flag_flushing = 0;
2241 				usb_fifo_wakeup(f);
2242 			}
2243 			break;
2244 		}
2245 		if (len == 0) {
2246 			break;
2247 		}
2248 	}
2249 	return (tr_data);
2250 }
2251 
2252 uint8_t
2253 usb_fifo_get_data_buffer(struct usb_fifo *f, void **pptr, usb_size_t *plen)
2254 {
2255 	struct usb_mbuf *m;
2256 
2257 	USB_IF_POLL(&f->used_q, m);
2258 
2259 	if (m) {
2260 		*plen = m->cur_data_len;
2261 		*pptr = m->cur_data_ptr;
2262 
2263 		return (1);
2264 	}
2265 	return (0);
2266 }
2267 
2268 void
2269 usb_fifo_get_data_error(struct usb_fifo *f)
2270 {
2271 	f->flag_iserror = 1;
2272 	usb_fifo_wakeup(f);
2273 }
2274 
2275 /*------------------------------------------------------------------------*
2276  *	usb_alloc_symlink
2277  *
2278  * Return values:
2279  * NULL: Failure
2280  * Else: Pointer to symlink entry
2281  *------------------------------------------------------------------------*/
2282 struct usb_symlink *
2283 usb_alloc_symlink(const char *target)
2284 {
2285 	struct usb_symlink *ps;
2286 
2287 	ps = malloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2288 	/* XXX no longer needed */
2289 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2290 	ps->src_len = strlen(ps->src_path);
2291 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2292 	ps->dst_len = strlen(ps->dst_path);
2293 
2294 	sx_xlock(&usb_sym_lock);
2295 	TAILQ_INSERT_TAIL(&usb_sym_head, ps, sym_entry);
2296 	sx_unlock(&usb_sym_lock);
2297 	return (ps);
2298 }
2299 
2300 /*------------------------------------------------------------------------*
2301  *	usb_free_symlink
2302  *------------------------------------------------------------------------*/
2303 void
2304 usb_free_symlink(struct usb_symlink *ps)
2305 {
2306 	if (ps == NULL) {
2307 		return;
2308 	}
2309 	sx_xlock(&usb_sym_lock);
2310 	TAILQ_REMOVE(&usb_sym_head, ps, sym_entry);
2311 	sx_unlock(&usb_sym_lock);
2312 
2313 	free(ps, M_USBDEV);
2314 }
2315 
2316 /*------------------------------------------------------------------------*
2317  *	usb_read_symlink
2318  *
2319  * Return value:
2320  * 0: Success
2321  * Else: Failure
2322  *------------------------------------------------------------------------*/
2323 int
2324 usb_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2325 {
2326 	struct usb_symlink *ps;
2327 	uint32_t temp;
2328 	uint32_t delta = 0;
2329 	uint8_t len;
2330 	int error = 0;
2331 
2332 	sx_xlock(&usb_sym_lock);
2333 
2334 	TAILQ_FOREACH(ps, &usb_sym_head, sym_entry) {
2335 		/*
2336 		 * Compute total length of source and destination symlink
2337 		 * strings pluss one length byte and two NUL bytes:
2338 		 */
2339 		temp = ps->src_len + ps->dst_len + 3;
2340 
2341 		if (temp > 255) {
2342 			/*
2343 			 * Skip entry because this length cannot fit
2344 			 * into one byte:
2345 			 */
2346 			continue;
2347 		}
2348 		if (startentry != 0) {
2349 			/* decrement read offset */
2350 			startentry--;
2351 			continue;
2352 		}
2353 		if (temp > user_len) {
2354 			/* out of buffer space */
2355 			break;
2356 		}
2357 		len = temp;
2358 
2359 		/* copy out total length */
2360 
2361 		error = copyout(&len,
2362 		    USB_ADD_BYTES(user_ptr, delta), 1);
2363 		if (error) {
2364 			break;
2365 		}
2366 		delta += 1;
2367 
2368 		/* copy out source string */
2369 
2370 		error = copyout(ps->src_path,
2371 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2372 		if (error) {
2373 			break;
2374 		}
2375 		len = 0;
2376 		delta += ps->src_len;
2377 		error = copyout(&len,
2378 		    USB_ADD_BYTES(user_ptr, delta), 1);
2379 		if (error) {
2380 			break;
2381 		}
2382 		delta += 1;
2383 
2384 		/* copy out destination string */
2385 
2386 		error = copyout(ps->dst_path,
2387 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2388 		if (error) {
2389 			break;
2390 		}
2391 		len = 0;
2392 		delta += ps->dst_len;
2393 		error = copyout(&len,
2394 		    USB_ADD_BYTES(user_ptr, delta), 1);
2395 		if (error) {
2396 			break;
2397 		}
2398 		delta += 1;
2399 
2400 		user_len -= temp;
2401 	}
2402 
2403 	/* a zero length entry indicates the end */
2404 
2405 	if ((user_len != 0) && (error == 0)) {
2406 		len = 0;
2407 
2408 		error = copyout(&len,
2409 		    USB_ADD_BYTES(user_ptr, delta), 1);
2410 	}
2411 	sx_unlock(&usb_sym_lock);
2412 	return (error);
2413 }
2414 
2415 void
2416 usb_fifo_set_close_zlp(struct usb_fifo *f, uint8_t onoff)
2417 {
2418 	if (f == NULL)
2419 		return;
2420 
2421 	/* send a Zero Length Packet, ZLP, before close */
2422 	f->flag_short = onoff;
2423 }
2424 
2425 void
2426 usb_fifo_set_write_defrag(struct usb_fifo *f, uint8_t onoff)
2427 {
2428 	if (f == NULL)
2429 		return;
2430 
2431 	/* defrag written data */
2432 	f->flag_write_defrag = onoff;
2433 	/* reset defrag state */
2434 	f->flag_have_fragment = 0;
2435 }
2436 
2437 void *
2438 usb_fifo_softc(struct usb_fifo *f)
2439 {
2440 	return (f->priv_sc0);
2441 }
2442 #endif	/* USB_HAVE_UGEN */
2443