xref: /dragonfly/sys/bus/u4b/usb_dev.c (revision ae071d8d)
1 /* $FreeBSD$ */
2 /*-
3  * Copyright (c) 2006-2008 Hans Petter Selasky. All rights reserved.
4  *
5  * Redistribution and use in source and binary forms, with or without
6  * modification, are permitted provided that the following conditions
7  * are met:
8  * 1. Redistributions of source code must retain the above copyright
9  *    notice, this list of conditions and the following disclaimer.
10  * 2. Redistributions in binary form must reproduce the above copyright
11  *    notice, this list of conditions and the following disclaimer in the
12  *    documentation and/or other materials provided with the distribution.
13  *
14  * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
15  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
16  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
17  * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
18  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
19  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
20  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
21  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
22  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
23  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
24  * SUCH DAMAGE.
25  *
26  *
27  * usb_dev.c - An abstraction layer for creating devices under /dev/...
28  */
29 
30 #include <sys/stdint.h>
31 #include <sys/param.h>
32 #include <sys/queue.h>
33 #include <sys/types.h>
34 #include <sys/systm.h>
35 #include <sys/kernel.h>
36 #include <sys/thread2.h>
37 #include <sys/bus.h>
38 #include <sys/module.h>
39 #include <sys/lock.h>
40 #include <sys/mutex.h>
41 #include <sys/condvar.h>
42 #include <sys/sysctl.h>
43 #include <sys/unistd.h>
44 #include <sys/callout.h>
45 #include <sys/malloc.h>
46 #include <sys/priv.h>
47 #include <sys/vnode.h>
48 #include <sys/conf.h>
49 #include <sys/fcntl.h>
50 #include <sys/devfs.h>
51 
52 #include <bus/u4b/usb.h>
53 #include <bus/u4b/usb_ioctl.h>
54 #include <bus/u4b/usbdi.h>
55 #include <bus/u4b/usbdi_util.h>
56 
57 #define	USB_DEBUG_VAR usb_fifo_debug
58 
59 #include <bus/u4b/usb_core.h>
60 #include <bus/u4b/usb_dev.h>
61 #include <bus/u4b/usb_mbuf.h>
62 #include <bus/u4b/usb_process.h>
63 #include <bus/u4b/usb_device.h>
64 #include <bus/u4b/usb_debug.h>
65 #include <bus/u4b/usb_busdma.h>
66 #include <bus/u4b/usb_generic.h>
67 #include <bus/u4b/usb_dynamic.h>
68 #include <bus/u4b/usb_util.h>
69 
70 #include <bus/u4b/usb_controller.h>
71 #include <bus/u4b/usb_bus.h>
72 
73 #include <sys/filio.h>
74 #include <sys/ttycom.h>
75 #include <sys/kern_syscall.h>
76 
77 #include <machine/stdarg.h>
78 
79 #if USB_HAVE_UGEN
80 
81 #ifdef USB_DEBUG
82 static int usb_fifo_debug = 0;
83 
84 static SYSCTL_NODE(_hw_usb, OID_AUTO, dev, CTLFLAG_RW, 0, "USB device");
85 SYSCTL_INT(_hw_usb_dev, OID_AUTO, debug, CTLFLAG_RW,
86     &usb_fifo_debug, 0, "Debug Level");
87 
88 TUNABLE_INT("hw.usb.dev.debug", &usb_fifo_debug);
89 #endif
90 
91 #define	USB_UCRED struct ucred *ucred,
92 
93 /* prototypes */
94 
95 static int	usb_fifo_open(struct usb_cdev_privdata *,
96 		    struct usb_fifo *, int);
97 static void	usb_fifo_close(struct usb_fifo *, int);
98 static void	usb_dev_init(void *);
99 static void	usb_dev_init_post(void *);
100 static void	usb_dev_uninit(void *);
101 static int	usb_fifo_uiomove(struct usb_fifo *, void *, int,
102 		    struct uio *);
103 static void	usb_fifo_check_methods(struct usb_fifo_methods *);
104 static struct	usb_fifo *usb_fifo_alloc(void);
105 static struct	usb_endpoint *usb_dev_get_ep(struct usb_device *, uint8_t,
106 		    uint8_t);
107 static void	usb_loc_fill(struct usb_fs_privdata *,
108 		    struct usb_cdev_privdata *);
109 static usb_error_t usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *, int);
110 static usb_error_t usb_usb_ref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
111 static void	usb_unref_device(struct usb_cdev_privdata *, struct usb_cdev_refdata *);
112 
113 static void usb_filter_detach(struct knote *kn);
114 static int usb_filter_read(struct knote *kn, long hint);
115 static int usb_filter_write(struct knote *kn, long hint);
116 
117 static d_open_t usb_open;
118 static d_close_t usb_close;
119 static d_ioctl_t usb_ioctl;
120 static d_read_t usb_read;
121 static d_write_t usb_write;
122 static d_kqfilter_t usb_kqfilter;
123 
124 static d_ioctl_t usb_static_ioctl;
125 static d_open_t usb_static_open;
126 static d_close_t usb_static_close;
127 
128 static usb_fifo_open_t usb_fifo_dummy_open;
129 static usb_fifo_close_t usb_fifo_dummy_close;
130 static usb_fifo_ioctl_t usb_fifo_dummy_ioctl;
131 static usb_fifo_cmd_t usb_fifo_dummy_cmd;
132 
133 /* character device structure used for devices (/dev/ugenX.Y and /dev/uXXX) */
134 struct dev_ops usb_ops = {
135 	{ "usbdev", 0, D_MEM },
136 	.d_open = usb_open,
137 	.d_close = usb_close,
138 	.d_ioctl = usb_ioctl,
139 	.d_read = usb_read,
140 	.d_write = usb_write,
141 	.d_kqfilter = usb_kqfilter
142 };
143 
144 static struct cdev* usb_dev = NULL;
145 
146 /* character device structure used for /bus/u4b */
147 static struct dev_ops usb_static_ops = {
148 	{ "usb", 0, D_MEM },
149 	.d_open = usb_static_open,
150 	.d_close = usb_static_close,
151 	.d_ioctl = usb_static_ioctl,
152 };
153 
154 static TAILQ_HEAD(, usb_symlink) usb_sym_head;
155 static struct lock usb_sym_lock;
156 
157 struct lock usb_ref_lock;
158 
159 static struct kqinfo usb_kqevent;
160 
161 static void
162 usb_cdevpriv_dtor(void *cpd)
163 {
164 }
165 
166 /*------------------------------------------------------------------------*
167  *	usb_loc_fill
168  *
169  * This is used to fill out a usb_cdev_privdata structure based on the
170  * device's address as contained in usb_fs_privdata.
171  *------------------------------------------------------------------------*/
172 static void
173 usb_loc_fill(struct usb_fs_privdata* pd, struct usb_cdev_privdata *cpd)
174 {
175 	cpd->bus_index = pd->bus_index;
176 	cpd->dev_index = pd->dev_index;
177 	cpd->ep_addr = pd->ep_addr;
178 	cpd->fifo_index = pd->fifo_index;
179 }
180 
181 /*------------------------------------------------------------------------*
182  *	usb_ref_device
183  *
184  * This function is used to atomically refer an USB device by its
185  * device location. If this function returns success the USB device
186  * will not dissappear until the USB device is unreferenced.
187  *
188  * Return values:
189  *  0: Success, refcount incremented on the given USB device.
190  *  Else: Failure.
191  *------------------------------------------------------------------------*/
192 static usb_error_t
193 usb_ref_device(struct usb_cdev_privdata *cpd,
194     struct usb_cdev_refdata *crd, int need_uref)
195 {
196 	struct usb_fifo **ppf;
197 	struct usb_fifo *f;
198 
199 	DPRINTFN(2, "cpd=%p need uref=%d\n", cpd, need_uref);
200 
201 	/* clear all refs */
202 	memset(crd, 0, sizeof(*crd));
203 
204 	lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
205 	cpd->bus = devclass_get_softc(usb_devclass_ptr, cpd->bus_index);
206 	if (cpd->bus == NULL) {
207 		DPRINTFN(2, "no bus at %u\n", cpd->bus_index);
208 		goto error;
209 	}
210 	cpd->udev = cpd->bus->devices[cpd->dev_index];
211 	if (cpd->udev == NULL) {
212 		DPRINTFN(2, "no device at %u\n", cpd->dev_index);
213 		goto error;
214 	}
215 	if (cpd->udev->refcount == USB_DEV_REF_MAX) {
216 		DPRINTFN(2, "no dev ref\n");
217 		goto error;
218 	}
219 	if (need_uref) {
220 		DPRINTFN(2, "ref udev - needed\n");
221 		cpd->udev->refcount++;
222 
223 		lockmgr(&usb_ref_lock, LK_RELEASE);
224 
225 		/*
226 		 * We need to grab the sx-lock before grabbing the
227 		 * FIFO refs to avoid deadlock at detach!
228 		 */
229 		crd->do_unlock = usbd_enum_lock(cpd->udev);
230 
231 		lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
232 
233 		/*
234 		 * Set "is_uref" after grabbing the default SX lock
235 		 */
236 		crd->is_uref = 1;
237 	}
238 
239 	/* check if we are doing an open */
240 	if (cpd->fflags == 0) {
241 		/* use zero defaults */
242 	} else {
243 		/* check for write */
244 		if (cpd->fflags & FWRITE) {
245 			ppf = cpd->udev->fifo;
246 			f = ppf[cpd->fifo_index + USB_FIFO_TX];
247 			crd->txfifo = f;
248 			crd->is_write = 1;	/* ref */
249 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
250 				goto error;
251 			if (f->curr_cpd != cpd)
252 				goto error;
253 			/* check if USB-FS is active */
254 			if (f->fs_ep_max != 0) {
255 				crd->is_usbfs = 1;
256 			}
257 		}
258 
259 		/* check for read */
260 		if (cpd->fflags & FREAD) {
261 			ppf = cpd->udev->fifo;
262 			f = ppf[cpd->fifo_index + USB_FIFO_RX];
263 			crd->rxfifo = f;
264 			crd->is_read = 1;	/* ref */
265 			if (f == NULL || f->refcount == USB_FIFO_REF_MAX)
266 				goto error;
267 			if (f->curr_cpd != cpd)
268 				goto error;
269 			/* check if USB-FS is active */
270 			if (f->fs_ep_max != 0) {
271 				crd->is_usbfs = 1;
272 			}
273 		}
274 	}
275 
276 	/* when everything is OK we increment the refcounts */
277 	if (crd->is_write) {
278 		DPRINTFN(2, "ref write\n");
279 		crd->txfifo->refcount++;
280 	}
281 	if (crd->is_read) {
282 		DPRINTFN(2, "ref read\n");
283 		crd->rxfifo->refcount++;
284 	}
285 	lockmgr(&usb_ref_lock, LK_RELEASE);
286 
287 	return (0);
288 
289 error:
290 	if (crd->do_unlock)
291 		usbd_enum_unlock(cpd->udev);
292 
293 	if (crd->is_uref) {
294 		if (--(cpd->udev->refcount) == 0) {
295 			cv_signal(&cpd->udev->ref_cv);
296 		}
297 	}
298 	lockmgr(&usb_ref_lock, LK_RELEASE);
299 	DPRINTFN(2, "fail\n");
300 	return (USB_ERR_INVAL);
301 }
302 
303 /*------------------------------------------------------------------------*
304  *	usb_usb_ref_device
305  *
306  * This function is used to upgrade an USB reference to include the
307  * USB device reference on a USB location.
308  *
309  * Return values:
310  *  0: Success, refcount incremented on the given USB device.
311  *  Else: Failure.
312  *------------------------------------------------------------------------*/
313 static usb_error_t
314 usb_usb_ref_device(struct usb_cdev_privdata *cpd,
315     struct usb_cdev_refdata *crd)
316 {
317 	/*
318 	 * Check if we already got an USB reference on this location:
319 	 */
320 	if (crd->is_uref)
321 		return (0);		/* success */
322 
323 	/*
324 	 * To avoid deadlock at detach we need to drop the FIFO ref
325 	 * and re-acquire a new ref!
326 	 */
327 	usb_unref_device(cpd, crd);
328 
329 	return (usb_ref_device(cpd, crd, 1 /* need uref */));
330 }
331 
332 /*------------------------------------------------------------------------*
333  *	usb_unref_device
334  *
335  * This function will release the reference count by one unit for the
336  * given USB device.
337  *------------------------------------------------------------------------*/
338 static void
339 usb_unref_device(struct usb_cdev_privdata *cpd,
340     struct usb_cdev_refdata *crd)
341 {
342 
343 	DPRINTFN(2, "cpd=%p is_uref=%d\n", cpd, crd->is_uref);
344 
345 	if (crd->do_unlock)
346 		usbd_enum_unlock(cpd->udev);
347 
348 	lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
349 	if (crd->is_read) {
350 		if (--(crd->rxfifo->refcount) == 0) {
351 			cv_signal(&crd->rxfifo->cv_drain);
352 		}
353 		crd->is_read = 0;
354 	}
355 	if (crd->is_write) {
356 		if (--(crd->txfifo->refcount) == 0) {
357 			cv_signal(&crd->txfifo->cv_drain);
358 		}
359 		crd->is_write = 0;
360 	}
361 	if (crd->is_uref) {
362 		if (--(cpd->udev->refcount) == 0) {
363 			cv_signal(&cpd->udev->ref_cv);
364 		}
365 		crd->is_uref = 0;
366 	}
367 	lockmgr(&usb_ref_lock, LK_RELEASE);
368 }
369 
370 static struct usb_fifo *
371 usb_fifo_alloc(void)
372 {
373 	struct usb_fifo *f;
374 
375 	f = kmalloc(sizeof(*f), M_USBDEV, M_WAITOK | M_ZERO);
376 	if (f) {
377 		cv_init(&f->cv_io, "FIFO-IO");
378 		cv_init(&f->cv_drain, "FIFO-DRAIN");
379 		f->refcount = 1;
380 	}
381 	return (f);
382 }
383 
384 /*------------------------------------------------------------------------*
385  *	usb_fifo_create
386  *------------------------------------------------------------------------*/
387 static int
388 usb_fifo_create(struct usb_cdev_privdata *cpd,
389     struct usb_cdev_refdata *crd)
390 {
391 	struct usb_device *udev = cpd->udev;
392 	struct usb_fifo *f;
393 	struct usb_endpoint *ep;
394 	uint8_t n;
395 	uint8_t is_tx;
396 	uint8_t is_rx;
397 	uint8_t no_null;
398 	uint8_t is_busy;
399 	int e = cpd->ep_addr;
400 
401 	is_tx = (cpd->fflags & FWRITE) ? 1 : 0;
402 	is_rx = (cpd->fflags & FREAD) ? 1 : 0;
403 	no_null = 1;
404 	is_busy = 0;
405 
406 	/* Preallocated FIFO */
407 	if (e < 0) {
408 		DPRINTFN(5, "Preallocated FIFO\n");
409 		if (is_tx) {
410 			f = udev->fifo[cpd->fifo_index + USB_FIFO_TX];
411 			if (f == NULL)
412 				return (EINVAL);
413 			crd->txfifo = f;
414 		}
415 		if (is_rx) {
416 			f = udev->fifo[cpd->fifo_index + USB_FIFO_RX];
417 			if (f == NULL)
418 				return (EINVAL);
419 			crd->rxfifo = f;
420 		}
421 		return (0);
422 	}
423 
424 	KASSERT(e >= 0 && e <= 15, ("endpoint %d out of range", e));
425 
426 	/* search for a free FIFO slot */
427 	DPRINTFN(5, "Endpoint device, searching for 0x%02x\n", e);
428 	for (n = 0;; n += 2) {
429 
430 		if (n == USB_FIFO_MAX) {
431 			if (no_null) {
432 				no_null = 0;
433 				n = 0;
434 			} else {
435 				/* end of FIFOs reached */
436 				DPRINTFN(5, "out of FIFOs\n");
437 				return (ENOMEM);
438 			}
439 		}
440 		/* Check for TX FIFO */
441 		if (is_tx) {
442 			f = udev->fifo[n + USB_FIFO_TX];
443 			if (f != NULL) {
444 				if (f->dev_ep_index != e) {
445 					/* wrong endpoint index */
446 					continue;
447 				}
448 				if (f->curr_cpd != NULL) {
449 					/* FIFO is opened */
450 					is_busy = 1;
451 					continue;
452 				}
453 			} else if (no_null) {
454 				continue;
455 			}
456 		}
457 		/* Check for RX FIFO */
458 		if (is_rx) {
459 			f = udev->fifo[n + USB_FIFO_RX];
460 			if (f != NULL) {
461 				if (f->dev_ep_index != e) {
462 					/* wrong endpoint index */
463 					continue;
464 				}
465 				if (f->curr_cpd != NULL) {
466 					/* FIFO is opened */
467 					is_busy = 1;
468 					continue;
469 				}
470 			} else if (no_null) {
471 				continue;
472 			}
473 		}
474 		break;
475 	}
476 
477 	if (no_null == 0) {
478 		if (e >= (USB_EP_MAX / 2)) {
479 			/* we don't create any endpoints in this range */
480 			DPRINTFN(5, "ep out of range\n");
481 			return (is_busy ? EBUSY : EINVAL);
482 		}
483 	}
484 
485 	if ((e != 0) && is_busy) {
486 		/*
487 		 * Only the default control endpoint is allowed to be
488 		 * opened multiple times!
489 		 */
490 		DPRINTFN(5, "busy\n");
491 		return (EBUSY);
492 	}
493 
494 	/* Check TX FIFO */
495 	if (is_tx &&
496 	    (udev->fifo[n + USB_FIFO_TX] == NULL)) {
497 		ep = usb_dev_get_ep(udev, e, USB_FIFO_TX);
498 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_TX);
499 		if (ep == NULL) {
500 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
501 			return (EINVAL);
502 		}
503 		f = usb_fifo_alloc();
504 		if (f == NULL) {
505 			DPRINTFN(5, "could not alloc tx fifo\n");
506 			return (ENOMEM);
507 		}
508 		/* update some fields */
509 		f->fifo_index = n + USB_FIFO_TX;
510 		f->dev_ep_index = e;
511 		f->priv_lock = &udev->device_lock;
512 		f->priv_sc0 = ep;
513 		f->methods = &usb_ugen_methods;
514 		f->iface_index = ep->iface_index;
515 		f->udev = udev;
516 		lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
517 		udev->fifo[n + USB_FIFO_TX] = f;
518 		lockmgr(&usb_ref_lock, LK_RELEASE);
519 	}
520 	/* Check RX FIFO */
521 	if (is_rx &&
522 	    (udev->fifo[n + USB_FIFO_RX] == NULL)) {
523 
524 		ep = usb_dev_get_ep(udev, e, USB_FIFO_RX);
525 		DPRINTFN(5, "dev_get_endpoint(%d, 0x%x)\n", e, USB_FIFO_RX);
526 		if (ep == NULL) {
527 			DPRINTFN(5, "dev_get_endpoint returned NULL\n");
528 			return (EINVAL);
529 		}
530 		f = usb_fifo_alloc();
531 		if (f == NULL) {
532 			DPRINTFN(5, "could not alloc rx fifo\n");
533 			return (ENOMEM);
534 		}
535 		/* update some fields */
536 		f->fifo_index = n + USB_FIFO_RX;
537 		f->dev_ep_index = e;
538 		f->priv_lock = &udev->device_lock;
539 		f->priv_sc0 = ep;
540 		f->methods = &usb_ugen_methods;
541 		f->iface_index = ep->iface_index;
542 		f->udev = udev;
543 		lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
544 		udev->fifo[n + USB_FIFO_RX] = f;
545 		lockmgr(&usb_ref_lock, LK_RELEASE);
546 	}
547 	if (is_tx) {
548 		crd->txfifo = udev->fifo[n + USB_FIFO_TX];
549 	}
550 	if (is_rx) {
551 		crd->rxfifo = udev->fifo[n + USB_FIFO_RX];
552 	}
553 	/* fill out fifo index */
554 	DPRINTFN(5, "fifo index = %d\n", n);
555 	cpd->fifo_index = n;
556 
557 	/* complete */
558 
559 	return (0);
560 }
561 
562 void
563 usb_fifo_free(struct usb_fifo *f)
564 {
565 	uint8_t n;
566 
567 	if (f == NULL) {
568 		/* be NULL safe */
569 		return;
570 	}
571 	/* destroy symlink devices, if any */
572 	for (n = 0; n != 2; n++) {
573 		if (f->symlink[n]) {
574 			usb_free_symlink(f->symlink[n]);
575 			f->symlink[n] = NULL;
576 		}
577 	}
578 	lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
579 
580 	/* delink ourselves to stop calls from userland */
581 	if ((f->fifo_index < USB_FIFO_MAX) &&
582 	    (f->udev != NULL) &&
583 	    (f->udev->fifo[f->fifo_index] == f)) {
584 		f->udev->fifo[f->fifo_index] = NULL;
585 	} else {
586 		DPRINTFN(0, "USB FIFO %p has not been linked\n", f);
587 	}
588 
589 	/* decrease refcount */
590 	f->refcount--;
591 	/* prevent any write flush */
592 	f->flag_iserror = 1;
593 	/* need to wait until all callers have exited */
594 	while (f->refcount != 0) {
595 		lockmgr(&usb_ref_lock, LK_RELEASE);	/* avoid LOR */
596 		lockmgr(f->priv_lock, LK_EXCLUSIVE);
597 		/* get I/O thread out of any sleep state */
598 		if (f->flag_sleeping) {
599 			f->flag_sleeping = 0;
600 			cv_broadcast(&f->cv_io);
601 		}
602 		lockmgr(f->priv_lock, LK_RELEASE);
603 		lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
604 
605 		/* wait for sync */
606 		cv_wait(&f->cv_drain, &usb_ref_lock);
607 	}
608 	lockmgr(&usb_ref_lock, LK_RELEASE);
609 
610 	/* take care of closing the device here, if any */
611 	usb_fifo_close(f, 0);
612 
613 	cv_destroy(&f->cv_io);
614 	cv_destroy(&f->cv_drain);
615 
616 	kfree(f, M_USBDEV);
617 }
618 
619 static struct usb_endpoint *
620 usb_dev_get_ep(struct usb_device *udev, uint8_t ep_index, uint8_t dir)
621 {
622 	struct usb_endpoint *ep;
623 	uint8_t ep_dir;
624 
625 	if (ep_index == 0) {
626 		ep = &udev->ctrl_ep;
627 	} else {
628 		if (dir == USB_FIFO_RX) {
629 			if (udev->flags.usb_mode == USB_MODE_HOST) {
630 				ep_dir = UE_DIR_IN;
631 			} else {
632 				ep_dir = UE_DIR_OUT;
633 			}
634 		} else {
635 			if (udev->flags.usb_mode == USB_MODE_HOST) {
636 				ep_dir = UE_DIR_OUT;
637 			} else {
638 				ep_dir = UE_DIR_IN;
639 			}
640 		}
641 		ep = usbd_get_ep_by_addr(udev, ep_index | ep_dir);
642 	}
643 
644 	if (ep == NULL) {
645 		/* if the endpoint does not exist then return */
646 		return (NULL);
647 	}
648 	if (ep->edesc == NULL) {
649 		/* invalid endpoint */
650 		return (NULL);
651 	}
652 	return (ep);			/* success */
653 }
654 
655 /*------------------------------------------------------------------------*
656  *	usb_fifo_open
657  *
658  * Returns:
659  * 0: Success
660  * Else: Failure
661  *------------------------------------------------------------------------*/
662 static int
663 usb_fifo_open(struct usb_cdev_privdata *cpd,
664     struct usb_fifo *f, int fflags)
665 {
666 	int err;
667 
668 	if (f == NULL) {
669 		/* no FIFO there */
670 		DPRINTFN(2, "no FIFO\n");
671 		return (ENXIO);
672 	}
673 	/* remove FWRITE and FREAD flags */
674 	fflags &= ~(FWRITE | FREAD);
675 
676 	/* set correct file flags */
677 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
678 		fflags |= FWRITE;
679 	} else {
680 		fflags |= FREAD;
681 	}
682 
683 	/* check if we are already opened */
684 	/* we don't need any locks when checking this variable */
685 	if (f->curr_cpd != NULL) {
686 		err = EBUSY;
687 		goto done;
688 	}
689 
690 	/* reset short flag before open */
691 	f->flag_short = 0;
692 
693 	/* call open method */
694 	err = (f->methods->f_open) (f, fflags);
695 	if (err) {
696 		goto done;
697 	}
698 	lockmgr(f->priv_lock, LK_EXCLUSIVE);
699 
700 	/* reset sleep flag */
701 	f->flag_sleeping = 0;
702 
703 	/* reset error flag */
704 	f->flag_iserror = 0;
705 
706 	/* reset complete flag */
707 	f->flag_iscomplete = 0;
708 
709 	/* reset select flag */
710 	f->flag_isselect = 0;
711 
712 	/* reset flushing flag */
713 	f->flag_flushing = 0;
714 
715 	/* reset ASYNC proc flag */
716 	f->async_p = NULL;
717 
718 	lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
719 	/* flag the fifo as opened to prevent others */
720 	f->curr_cpd = cpd;
721 	lockmgr(&usb_ref_lock, LK_RELEASE);
722 
723 	/* reset queue */
724 	usb_fifo_reset(f);
725 
726 	lockmgr(f->priv_lock, LK_RELEASE);
727 done:
728 	return (err);
729 }
730 
731 /*------------------------------------------------------------------------*
732  *	usb_fifo_reset
733  *------------------------------------------------------------------------*/
734 void
735 usb_fifo_reset(struct usb_fifo *f)
736 {
737 	struct usb_mbuf *m;
738 
739 	if (f == NULL) {
740 		return;
741 	}
742 	while (1) {
743 		USB_IF_DEQUEUE(&f->used_q, m);
744 		if (m) {
745 			USB_IF_ENQUEUE(&f->free_q, m);
746 		} else {
747 			break;
748 		}
749 	}
750 	/* reset have fragment flag */
751 	f->flag_have_fragment = 0;
752 }
753 
754 /*------------------------------------------------------------------------*
755  *	usb_fifo_close
756  *------------------------------------------------------------------------*/
757 static void
758 usb_fifo_close(struct usb_fifo *f, int fflags)
759 {
760 	int err;
761 
762 	/* check if we are not opened */
763 	if (f->curr_cpd == NULL) {
764 		/* nothing to do - already closed */
765 		return;
766 	}
767 	lockmgr(f->priv_lock, LK_EXCLUSIVE);
768 
769 	/* clear current cdev private data pointer */
770 	f->curr_cpd = NULL;
771 
772 	/* check if we are selected */
773 	if (f->flag_isselect) {
774 #if 0 /* XXXDF */
775 		selwakeup(&f->selinfo);
776 #endif
777 		f->flag_isselect = 0;
778 	}
779 	/* check if a thread wants SIGIO */
780 	if (f->async_p != NULL && lwkt_trytoken(&f->async_p->p_token)) {
781 		ksignal(f->async_p, SIGIO);
782 		lwkt_reltoken(&f->async_p->p_token);
783 		f->async_p = NULL;
784 	}
785 	/* remove FWRITE and FREAD flags */
786 	fflags &= ~(FWRITE | FREAD);
787 
788 	/* flush written data, if any */
789 	if ((f->fifo_index & 1) == USB_FIFO_TX) {
790 
791 		if (!f->flag_iserror) {
792 
793 			/* set flushing flag */
794 			f->flag_flushing = 1;
795 
796 			/* get the last packet in */
797 			if (f->flag_have_fragment) {
798 				struct usb_mbuf *m;
799 				f->flag_have_fragment = 0;
800 				USB_IF_DEQUEUE(&f->free_q, m);
801 				if (m) {
802 					USB_IF_ENQUEUE(&f->used_q, m);
803 				}
804 			}
805 
806 			/* start write transfer, if not already started */
807 			(f->methods->f_start_write) (f);
808 
809 			/* check if flushed already */
810 			while (f->flag_flushing &&
811 			    (!f->flag_iserror)) {
812 				/* wait until all data has been written */
813 				f->flag_sleeping = 1;
814 				err = cv_wait_sig(&f->cv_io, f->priv_lock);
815 				if (err) {
816 					DPRINTF("signal received\n");
817 					break;
818 				}
819 			}
820 		}
821 		fflags |= FWRITE;
822 
823 		/* stop write transfer, if not already stopped */
824 		(f->methods->f_stop_write) (f);
825 	} else {
826 		fflags |= FREAD;
827 
828 		/* stop write transfer, if not already stopped */
829 		(f->methods->f_stop_read) (f);
830 	}
831 
832 	/* check if we are sleeping */
833 	if (f->flag_sleeping) {
834 		DPRINTFN(2, "Sleeping at close!\n");
835 	}
836 	lockmgr(f->priv_lock, LK_RELEASE);
837 
838 	/* call close method */
839 	(f->methods->f_close) (f, fflags);
840 
841 	DPRINTF("closed\n");
842 }
843 
844 /*------------------------------------------------------------------------*
845  *	usb_open - cdev callback
846  *------------------------------------------------------------------------*/
847 static int
848 usb_open(struct dev_open_args *ap)
849 {
850 	struct cdev *dev = ap->a_head.a_dev;
851 	int fflags = ap->a_oflags;
852 	struct usb_fs_privdata* pd = (struct usb_fs_privdata*)dev->si_drv1;
853 	struct usb_cdev_refdata refs;
854 	struct usb_cdev_privdata *cpd;
855 	int err, ep;
856 
857 	DPRINTFN(2, "%s fflags=0x%08x\n", devtoname(dev), fflags);
858 
859 	KASSERT(fflags & (FREAD|FWRITE), ("invalid open flags"));
860 	if (((fflags & FREAD) && !(pd->mode & FREAD)) ||
861 	    ((fflags & FWRITE) && !(pd->mode & FWRITE))) {
862 		DPRINTFN(2, "access mode not supported\n");
863 		return (EPERM);
864 	}
865 
866 	cpd = kmalloc(sizeof(*cpd), M_USBDEV, M_WAITOK | M_ZERO);
867 	ep = cpd->ep_addr = pd->ep_addr;
868 
869 	usb_loc_fill(pd, cpd);
870 	err = usb_ref_device(cpd, &refs, 1);
871 	if (err) {
872 		DPRINTFN(2, "cannot ref device\n");
873 		kfree(cpd, M_USBDEV);
874 		return (ENXIO);
875 	}
876 	cpd->fflags = fflags;	/* access mode for open lifetime */
877 
878 	/* create FIFOs, if any */
879 	err = usb_fifo_create(cpd, &refs);
880 	/* check for error */
881 	if (err) {
882 		DPRINTFN(2, "cannot create fifo\n");
883 		usb_unref_device(cpd, &refs);
884 		kfree(cpd, M_USBDEV);
885 		return (err);
886 	}
887 	if (fflags & FREAD) {
888 		err = usb_fifo_open(cpd, refs.rxfifo, fflags);
889 		if (err) {
890 			DPRINTFN(2, "read open failed\n");
891 			usb_unref_device(cpd, &refs);
892 			kfree(cpd, M_USBDEV);
893 			return (err);
894 		}
895 	}
896 	if (fflags & FWRITE) {
897 		err = usb_fifo_open(cpd, refs.txfifo, fflags);
898 		if (err) {
899 			DPRINTFN(2, "write open failed\n");
900 			if (fflags & FREAD) {
901 				usb_fifo_close(refs.rxfifo, fflags);
902 			}
903 			usb_unref_device(cpd, &refs);
904 			kfree(cpd, M_USBDEV);
905 			return (err);
906 		}
907 	}
908 	usb_unref_device(cpd, &refs);
909 	err = devfs_set_cdevpriv(ap->a_fp, cpd, usb_cdevpriv_dtor);
910 	if(err) {
911 		DPRINTFN(2, "devfs_set_cdevpriv failed in %s\n", __FUNC__);
912 		kfree(cpd, M_USBDEV);
913 		return(err);
914 	}
915 	return (0);
916 }
917 
918 /*------------------------------------------------------------------------*
919  *	usb_close - cdev callback
920  *------------------------------------------------------------------------*/
921 static int
922 usb_close(struct dev_close_args *ap)
923 {
924 	struct usb_cdev_refdata refs;
925 	struct usb_cdev_privdata *cpd;
926 	int err;
927 
928 	DPRINTFN(2, "cpd=%p\n", cpd);
929 
930 	err = devfs_get_cdevpriv(ap->a_fp, (void **)&cpd);
931  	if (err != 0)
932  		return (err);
933 
934 	err = usb_ref_device(cpd, &refs, 0);
935 	if (err)
936 		goto done;
937 
938 	/*
939 	 * If this function is not called directly from the root HUB
940 	 * thread, there is usually a need to lock the enumeration
941 	 * lock. Check this.
942 	 */
943 	if (!usbd_enum_is_locked(cpd->udev)) {
944 
945 		DPRINTFN(2, "Locking enumeration\n");
946 
947 		/* reference device */
948 		err = usb_usb_ref_device(cpd, &refs);
949 		if (err)
950 			goto done;
951 	}
952 	if (cpd->fflags & FREAD) {
953 		usb_fifo_close(refs.rxfifo, cpd->fflags);
954 	}
955 	if (cpd->fflags & FWRITE) {
956 		usb_fifo_close(refs.txfifo, cpd->fflags);
957 	}
958 	usb_unref_device(cpd, &refs);
959 done:
960 	kfree(cpd, M_USBDEV);
961 	return 0;
962 }
963 
964 static void
965 usb_dev_init(void *arg)
966 {
967 	lockinit(&usb_ref_lock, "USB ref mutex", 0, 0);
968 	lockinit(&usb_sym_lock, "USB sym mutex", 0, 0);
969 	TAILQ_INIT(&usb_sym_head);
970 
971 	/* check the UGEN methods */
972 	usb_fifo_check_methods(&usb_ugen_methods);
973 }
974 
975 /* XXX SI_SUB_KLD? */
976 SYSINIT(usb_dev_init, SI_SUB_PRE_DRIVERS, SI_ORDER_FIRST, usb_dev_init, NULL);
977 
978 static void
979 usb_dev_init_post(void *arg)
980 {
981 	/*
982 	 * Create /dev/usb - this is needed for usbconfig(8), which
983 	 * needs a well-known device name to access.
984 	 */
985 	usb_dev = make_dev(&usb_static_ops, 0, UID_ROOT, GID_OPERATOR,
986 	    0644, USB_DEVICE_NAME);
987 	if (usb_dev == NULL) {
988 		DPRINTFN(0, "Could not create usb bus device\n");
989 	}
990 }
991 
992 SYSINIT(usb_dev_init_post, SI_SUB_DRIVERS, SI_ORDER_FIRST, usb_dev_init_post,
993     NULL);
994 
995 static void
996 usb_dev_uninit(void *arg)
997 {
998 	if (usb_dev != NULL) {
999 		destroy_dev(usb_dev);
1000 		usb_dev = NULL;
1001 	}
1002 	lockuninit(&usb_ref_lock);
1003 	lockuninit(&usb_sym_lock);
1004 }
1005 
1006 SYSUNINIT(usb_dev_uninit, SI_SUB_KICK_SCHEDULER, SI_ORDER_ANY, usb_dev_uninit, NULL);
1007 
1008 static int
1009 usb_ioctl_f_sub(struct usb_fifo *f, u_long cmd, void *addr,
1010     struct thread *td)
1011 {
1012 	int error = 0;
1013 
1014 	switch (cmd) {
1015 	case FIODTYPE:
1016 		*(int *)addr = 0;	/* character device */
1017 		break;
1018 
1019 	case FIONBIO:
1020 		/* handled by upper FS layer */
1021 		break;
1022 
1023 	case FIOASYNC:
1024 		if (*(int *)addr) {
1025 			if (f->async_p != NULL) {
1026 				error = EBUSY;
1027 				break;
1028 			}
1029 			f->async_p = USB_TD_GET_PROC(td);
1030 		} else {
1031 			f->async_p = NULL;
1032 		}
1033 		break;
1034 
1035 		/* XXX this is not the most general solution */
1036 	case TIOCSPGRP:
1037 		if (f->async_p == NULL) {
1038 			error = EINVAL;
1039 			break;
1040 		}
1041 		if (*(int *)addr != USB_PROC_GET_GID(f->async_p)) {
1042 			error = EPERM;
1043 			break;
1044 		}
1045 		break;
1046 	default:
1047 		return (ENOIOCTL);
1048 	}
1049 	DPRINTFN(3, "cmd 0x%lx = %d\n", cmd, error);
1050 	return (error);
1051 }
1052 
1053 /*------------------------------------------------------------------------*
1054  *	usb_ioctl - cdev callback
1055  *------------------------------------------------------------------------*/
1056 static int
1057 usb_ioctl(struct dev_ioctl_args *ap)
1058 {
1059 	u_long cmd = ap->a_cmd;
1060 	caddr_t addr = ap->a_data;
1061 	struct thread *td = curthread;
1062 	struct usb_cdev_refdata refs;
1063 	struct usb_cdev_privdata* cpd;
1064 	struct usb_fifo *f;
1065 	int fflags;
1066 	int err;
1067 
1068 	DPRINTFN(2, "cmd=0x%lx\n", cmd);
1069 
1070 	err = devfs_get_cdevpriv(ap->a_fp, (void **)&cpd);
1071 	if (err != 0)
1072 		return (err);
1073 
1074 	/*
1075 	 * Performance optimisation: We try to check for IOCTL's that
1076 	 * don't need the USB reference first. Then we grab the USB
1077 	 * reference if we need it!
1078 	 */
1079 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1080 	if (err)
1081 		return (ENXIO);
1082 
1083 	fflags = cpd->fflags;
1084 
1085 	f = NULL;			/* set default value */
1086 	err = ENOIOCTL;			/* set default value */
1087 
1088 	if (fflags & FWRITE) {
1089 		f = refs.txfifo;
1090 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1091 	}
1092 	if (fflags & FREAD) {
1093 		f = refs.rxfifo;
1094 		err = usb_ioctl_f_sub(f, cmd, addr, td);
1095 	}
1096 	KASSERT(f != NULL, ("fifo not found"));
1097 	if (err != ENOIOCTL)
1098 		goto done;
1099 
1100 	err = (f->methods->f_ioctl) (f, cmd, addr, fflags);
1101 
1102 	DPRINTFN(2, "f_ioctl cmd 0x%lx = %d\n", cmd, err);
1103 
1104 	if (err != ENOIOCTL)
1105 		goto done;
1106 
1107 	if (usb_usb_ref_device(cpd, &refs)) {
1108 		err = ENXIO;
1109 		goto done;
1110 	}
1111 
1112 	err = (f->methods->f_ioctl_post) (f, cmd, addr, fflags);
1113 
1114 	DPRINTFN(2, "f_ioctl_post cmd 0x%lx = %d\n", cmd, err);
1115 
1116 	if (err == ENOIOCTL)
1117 		err = ENOTTY;
1118 
1119 	if (err)
1120 		goto done;
1121 
1122 	/* Wait for re-enumeration, if any */
1123 
1124 	while (f->udev->re_enumerate_wait != 0) {
1125 
1126 		usb_unref_device(cpd, &refs);
1127 
1128 		usb_pause_mtx(NULL, hz / 128);
1129 
1130 		if (usb_ref_device(cpd, &refs, 1 /* need uref */)) {
1131 			err = ENXIO;
1132 			goto done;
1133 		}
1134 	}
1135 
1136 done:
1137 	usb_unref_device(cpd, &refs);
1138 	return (err);
1139 }
1140 
1141 static struct filterops usb_filtops_read =
1142     { FILTEROP_ISFD, NULL, usb_filter_detach, usb_filter_read };
1143 
1144 static struct filterops usb_filtops_write =
1145     { FILTEROP_ISFD, NULL, usb_filter_detach, usb_filter_write };
1146 
1147 static int
1148 usb_kqfilter(struct dev_kqfilter_args *ap)
1149 {
1150 	cdev_t dev = ap->a_head.a_dev;
1151 	struct knote *kn = ap->a_kn;
1152 	struct klist *klist;
1153 
1154 	ap->a_result = 0;
1155 
1156 	switch(kn->kn_filter) {
1157 	case EVFILT_READ:
1158 		kn->kn_fop = &usb_filtops_read;
1159 		kn->kn_hook = (caddr_t)dev;
1160 		break;
1161 	case EVFILT_WRITE:
1162 		kn->kn_fop = &usb_filtops_write;
1163 		kn->kn_hook = (caddr_t)dev;
1164 		break;
1165 	default:
1166 		ap->a_result = EOPNOTSUPP;
1167 		return(0);
1168 	}
1169 
1170 	klist = &usb_kqevent.ki_note;
1171 	knote_insert(klist, kn);
1172 
1173 	return(0);
1174 }
1175 
1176 static void
1177 usb_filter_detach(struct knote *kn)
1178 {
1179 	struct klist *klist;
1180 
1181 	klist = &usb_kqevent.ki_note;
1182 	knote_remove(klist, kn);
1183 }
1184 
1185 static int
1186 usb_filter_read(struct knote *kn, long hint)
1187 {
1188 	/*cdev_t dev = (cdev_t)kn->kn_hook;
1189 	*/
1190 	int ready = 0;
1191 
1192 	return(ready);
1193 }
1194 
1195 static int
1196 usb_filter_write(struct knote *kn, long hint)
1197 {
1198 	/*cdev_t dev = (cdev_t)kn->kn_hook;
1199 	*/
1200 	int ready = 0;
1201 
1202 	return(ready);
1203 }
1204 
1205 #if 0 /* XXX implement using kqfilter */
1206 /* ARGSUSED */
1207 static int
1208 usb_poll(struct cdev* dev, int events, struct thread* td)
1209 {
1210 	struct usb_cdev_refdata refs;
1211 	struct usb_cdev_privdata* cpd;
1212 	struct usb_fifo *f;
1213 	struct usb_mbuf *m;
1214 	int fflags, revents;
1215 
1216 	if (devfs_get_cdevpriv((void **)&cpd) != 0 ||
1217 	    usb_ref_device(cpd, &refs, 0) != 0)
1218 		return (events &
1219 		    (POLLHUP|POLLIN|POLLRDNORM|POLLOUT|POLLWRNORM));
1220 
1221 	fflags = cpd->fflags;
1222 
1223 	/* Figure out who needs service */
1224 	revents = 0;
1225 	if ((events & (POLLOUT | POLLWRNORM)) &&
1226 	    (fflags & FWRITE)) {
1227 
1228 		f = refs.txfifo;
1229 
1230 		lockmgr(f->priv_lock, LK_EXCLUSIVE);
1231 
1232 		if (!refs.is_usbfs) {
1233 			if (f->flag_iserror) {
1234 				/* we got an error */
1235 				m = (void *)1;
1236 			} else {
1237 				if (f->queue_data == NULL) {
1238 					/*
1239 					 * start write transfer, if not
1240 					 * already started
1241 					 */
1242 					(f->methods->f_start_write) (f);
1243 				}
1244 				/* check if any packets are available */
1245 				USB_IF_POLL(&f->free_q, m);
1246 			}
1247 		} else {
1248 			if (f->flag_iscomplete) {
1249 				m = (void *)1;
1250 			} else {
1251 				m = NULL;
1252 			}
1253 		}
1254 
1255 		if (m) {
1256 			revents |= events & (POLLOUT | POLLWRNORM);
1257 		} else {
1258 			f->flag_isselect = 1;
1259 			selrecord(td, &f->selinfo);
1260 		}
1261 
1262 		lockmgr(f->priv_lock);
1263 	}
1264 	if ((events & (POLLIN | POLLRDNORM)) &&
1265 	    (fflags & FREAD)) {
1266 
1267 		f = refs.rxfifo;
1268 
1269 		lockmgr(f->priv_lock, LK_EXCLUSIVE);
1270 
1271 		if (!refs.is_usbfs) {
1272 			if (f->flag_iserror) {
1273 				/* we have and error */
1274 				m = (void *)1;
1275 			} else {
1276 				if (f->queue_data == NULL) {
1277 					/*
1278 					 * start read transfer, if not
1279 					 * already started
1280 					 */
1281 					(f->methods->f_start_read) (f);
1282 				}
1283 				/* check if any packets are available */
1284 				USB_IF_POLL(&f->used_q, m);
1285 			}
1286 		} else {
1287 			if (f->flag_iscomplete) {
1288 				m = (void *)1;
1289 			} else {
1290 				m = NULL;
1291 			}
1292 		}
1293 
1294 		if (m) {
1295 			revents |= events & (POLLIN | POLLRDNORM);
1296 		} else {
1297 			f->flag_isselect = 1;
1298 			selrecord(td, &f->selinfo);
1299 
1300 			if (!refs.is_usbfs) {
1301 				/* start reading data */
1302 				(f->methods->f_start_read) (f);
1303 			}
1304 		}
1305 
1306 		lockmgr(f->priv_lock, LK_RELEASE);
1307 	}
1308 	usb_unref_device(cpd, &refs);
1309 	return (revents);
1310 }
1311 #endif
1312 
1313 static int
1314 usb_read(struct dev_read_args *ap)
1315 {
1316 	struct uio *uio = ap->a_uio;
1317 	int ioflag = ap->a_ioflag;
1318 	struct usb_cdev_refdata refs;
1319 	struct usb_cdev_privdata* cpd;
1320 	struct usb_fifo *f;
1321 	struct usb_mbuf *m;
1322 	int fflags;
1323 	int resid;
1324 	int io_len;
1325 	int err;
1326 	uint8_t tr_data = 0;
1327 
1328 	err = devfs_get_cdevpriv(ap->a_fp, (void **)&cpd);
1329 	if (err != 0)
1330 		return (err);
1331 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1332 	if (err) {
1333 		return (ENXIO);
1334 	}
1335 	fflags = cpd->fflags;
1336 
1337 	f = refs.rxfifo;
1338 	if (f == NULL) {
1339 		/* should not happen */
1340 		usb_unref_device(cpd, &refs);
1341 		return (EPERM);
1342 	}
1343 
1344 	resid = uio->uio_resid;
1345 
1346 	lockmgr(f->priv_lock, LK_EXCLUSIVE);
1347 
1348 	/* check for permanent read error */
1349 	if (f->flag_iserror) {
1350 		err = EIO;
1351 		goto done;
1352 	}
1353 	/* check if USB-FS interface is active */
1354 	if (refs.is_usbfs) {
1355 		/*
1356 		 * The queue is used for events that should be
1357 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1358 		 */
1359 		err = EINVAL;
1360 		goto done;
1361 	}
1362 	while (uio->uio_resid > 0) {
1363 
1364 		USB_IF_DEQUEUE(&f->used_q, m);
1365 
1366 		if (m == NULL) {
1367 
1368 			/* start read transfer, if not already started */
1369 
1370 			(f->methods->f_start_read) (f);
1371 
1372 			if (ioflag & IO_NDELAY) {
1373 				if (tr_data) {
1374 					/* return length before error */
1375 					break;
1376 				}
1377 				err = EWOULDBLOCK;
1378 				break;
1379 			}
1380 			DPRINTF("sleeping\n");
1381 
1382 			err = usb_fifo_wait(f);
1383 			if (err) {
1384 				break;
1385 			}
1386 			continue;
1387 		}
1388 		if (f->methods->f_filter_read) {
1389 			/*
1390 			 * Sometimes it is convenient to process data at the
1391 			 * expense of a userland process instead of a kernel
1392 			 * process.
1393 			 */
1394 			(f->methods->f_filter_read) (f, m);
1395 		}
1396 		tr_data = 1;
1397 
1398 		io_len = MIN(m->cur_data_len, uio->uio_resid);
1399 
1400 		DPRINTFN(2, "transfer %d bytes from %p\n",
1401 		    io_len, m->cur_data_ptr);
1402 
1403 		err = usb_fifo_uiomove(f,
1404 		    m->cur_data_ptr, io_len, uio);
1405 
1406 		m->cur_data_len -= io_len;
1407 		m->cur_data_ptr += io_len;
1408 
1409 		if (m->cur_data_len == 0) {
1410 
1411 			uint8_t last_packet;
1412 
1413 			last_packet = m->last_packet;
1414 
1415 			USB_IF_ENQUEUE(&f->free_q, m);
1416 
1417 			if (last_packet) {
1418 				/* keep framing */
1419 				break;
1420 			}
1421 		} else {
1422 			USB_IF_PREPEND(&f->used_q, m);
1423 		}
1424 
1425 		if (err) {
1426 			break;
1427 		}
1428 	}
1429 done:
1430 	lockmgr(f->priv_lock, LK_RELEASE);
1431 
1432 	usb_unref_device(cpd, &refs);
1433 
1434 	return (err);
1435 }
1436 
1437 static int
1438 usb_write(struct dev_write_args *ap)
1439 {
1440 	struct uio *uio = ap->a_uio;
1441 	int ioflag = ap->a_ioflag;
1442 	struct usb_cdev_refdata refs;
1443 	struct usb_cdev_privdata* cpd;
1444 	struct usb_fifo *f;
1445 	struct usb_mbuf *m;
1446 	uint8_t *pdata;
1447 	int fflags;
1448 	int resid;
1449 	int io_len;
1450 	int err;
1451 	uint8_t tr_data = 0;
1452 
1453 	DPRINTFN(2, "\n");
1454 
1455 	err = devfs_get_cdevpriv(ap->a_fp, (void **)&cpd);
1456 	if (err != 0)
1457 		return (err);
1458 	err = usb_ref_device(cpd, &refs, 0 /* no uref */ );
1459 	if (err) {
1460 		return (ENXIO);
1461 	}
1462 	fflags = cpd->fflags;
1463 
1464 	f = refs.txfifo;
1465 	if (f == NULL) {
1466 		/* should not happen */
1467 		usb_unref_device(cpd, &refs);
1468 		return (EPERM);
1469 	}
1470 	resid = uio->uio_resid;
1471 
1472 	lockmgr(f->priv_lock, LK_EXCLUSIVE);
1473 
1474 	/* check for permanent write error */
1475 	if (f->flag_iserror) {
1476 		err = EIO;
1477 		goto done;
1478 	}
1479 	/* check if USB-FS interface is active */
1480 	if (refs.is_usbfs) {
1481 		/*
1482 		 * The queue is used for events that should be
1483 		 * retrieved using the "USB_FS_COMPLETE" ioctl.
1484 		 */
1485 		err = EINVAL;
1486 		goto done;
1487 	}
1488 	if (f->queue_data == NULL) {
1489 		/* start write transfer, if not already started */
1490 		(f->methods->f_start_write) (f);
1491 	}
1492 	/* we allow writing zero length data */
1493 	do {
1494 		USB_IF_DEQUEUE(&f->free_q, m);
1495 
1496 		if (m == NULL) {
1497 
1498 			if (ioflag & IO_NDELAY) {
1499 				if (tr_data) {
1500 					/* return length before error */
1501 					break;
1502 				}
1503 				err = EWOULDBLOCK;
1504 				break;
1505 			}
1506 			DPRINTF("sleeping\n");
1507 
1508 			err = usb_fifo_wait(f);
1509 			if (err) {
1510 				break;
1511 			}
1512 			continue;
1513 		}
1514 		tr_data = 1;
1515 
1516 		if (f->flag_have_fragment == 0) {
1517 			USB_MBUF_RESET(m);
1518 			io_len = m->cur_data_len;
1519 			pdata = m->cur_data_ptr;
1520 			if (io_len > uio->uio_resid)
1521 				io_len = uio->uio_resid;
1522 			m->cur_data_len = io_len;
1523 		} else {
1524 			io_len = m->max_data_len - m->cur_data_len;
1525 			pdata = m->cur_data_ptr + m->cur_data_len;
1526 			if (io_len > uio->uio_resid)
1527 				io_len = uio->uio_resid;
1528 			m->cur_data_len += io_len;
1529 		}
1530 
1531 		DPRINTFN(2, "transfer %d bytes to %p\n",
1532 		    io_len, pdata);
1533 
1534 		err = usb_fifo_uiomove(f, pdata, io_len, uio);
1535 
1536 		if (err) {
1537 			f->flag_have_fragment = 0;
1538 			USB_IF_ENQUEUE(&f->free_q, m);
1539 			break;
1540 		}
1541 
1542 		/* check if the buffer is ready to be transmitted */
1543 
1544 		if ((f->flag_write_defrag == 0) ||
1545 		    (m->cur_data_len == m->max_data_len)) {
1546 			f->flag_have_fragment = 0;
1547 
1548 			/*
1549 			 * Check for write filter:
1550 			 *
1551 			 * Sometimes it is convenient to process data
1552 			 * at the expense of a userland process
1553 			 * instead of a kernel process.
1554 			 */
1555 			if (f->methods->f_filter_write) {
1556 				(f->methods->f_filter_write) (f, m);
1557 			}
1558 
1559 			/* Put USB mbuf in the used queue */
1560 			USB_IF_ENQUEUE(&f->used_q, m);
1561 
1562 			/* Start writing data, if not already started */
1563 			(f->methods->f_start_write) (f);
1564 		} else {
1565 			/* Wait for more data or close */
1566 			f->flag_have_fragment = 1;
1567 			USB_IF_PREPEND(&f->free_q, m);
1568 		}
1569 
1570 	} while (uio->uio_resid > 0);
1571 done:
1572 	lockmgr(f->priv_lock, LK_RELEASE);
1573 
1574 	usb_unref_device(cpd, &refs);
1575 
1576 	return (err);
1577 }
1578 
1579 int
1580 usb_static_open(struct dev_open_args *ap)
1581 {
1582 	return 0;
1583 }
1584 
1585 int
1586 usb_static_close(struct dev_close_args *ap)
1587 {
1588 	return 0;
1589 }
1590 
1591 int
1592 usb_static_ioctl(struct dev_ioctl_args *ap)
1593 {
1594 	u_long cmd = ap->a_cmd;
1595 	caddr_t data = ap->a_data;
1596 	struct thread *td = curthread; /* XXX: curthread the correct choice? */
1597 	int fflag = ap->a_fflag;
1598 	union {
1599 		struct usb_read_dir *urd;
1600 		void* data;
1601 	} u;
1602 	int err;
1603 
1604 	u.data = data;
1605 	switch (cmd) {
1606 		case USB_READ_DIR:
1607 			err = usb_read_symlink(u.urd->urd_data,
1608 			    u.urd->urd_startentry, u.urd->urd_maxlen);
1609 			break;
1610 		case USB_DEV_QUIRK_GET:
1611 		case USB_QUIRK_NAME_GET:
1612 		case USB_DEV_QUIRK_ADD:
1613 		case USB_DEV_QUIRK_REMOVE:
1614 			err = usb_quirk_ioctl_p(cmd, data, fflag, td);
1615 			break;
1616 		case USB_GET_TEMPLATE:
1617 			*(int *)data = usb_template;
1618 			err = 0;
1619 			break;
1620 		case USB_SET_TEMPLATE:
1621 			err = priv_check(curthread, PRIV_DRIVER);
1622 			if (err)
1623 				break;
1624 			usb_template = *(int *)data;
1625 			break;
1626 		default:
1627 			err = ENOTTY;
1628 			break;
1629 	}
1630 	return (err);
1631 }
1632 
1633 static int
1634 usb_fifo_uiomove(struct usb_fifo *f, void *cp,
1635     int n, struct uio *uio)
1636 {
1637 	int error;
1638 
1639 	lockmgr(f->priv_lock, LK_RELEASE);
1640 
1641 	/*
1642 	 * "uiomove()" can sleep so one needs to make a wrapper,
1643 	 * exiting the mutex and checking things:
1644 	 */
1645 	error = uiomove(cp, n, uio);
1646 
1647 	lockmgr(f->priv_lock, LK_EXCLUSIVE);
1648 
1649 	return (error);
1650 }
1651 
1652 int
1653 usb_fifo_wait(struct usb_fifo *f)
1654 {
1655 	int err;
1656 
1657 	KKASSERT(lockowned(f->priv_lock));
1658 
1659 	if (f->flag_iserror) {
1660 		/* we are gone */
1661 		return (EIO);
1662 	}
1663 	f->flag_sleeping = 1;
1664 
1665 	err = cv_wait_sig(&f->cv_io, f->priv_lock);
1666 
1667 	if (f->flag_iserror) {
1668 		/* we are gone */
1669 		err = EIO;
1670 	}
1671 	return (err);
1672 }
1673 
1674 void
1675 usb_fifo_signal(struct usb_fifo *f)
1676 {
1677 	if (f->flag_sleeping) {
1678 		f->flag_sleeping = 0;
1679 		cv_broadcast(&f->cv_io);
1680 	}
1681 }
1682 
1683 void
1684 usb_fifo_wakeup(struct usb_fifo *f)
1685 {
1686 	usb_fifo_signal(f);
1687 
1688 	if (f->flag_isselect) {
1689 #if 0 /* XXXDF */
1690 		selwakeup(&f->selinfo);
1691 #endif
1692 		f->flag_isselect = 0;
1693 	}
1694 	if (f->async_p != NULL && lwkt_trytoken(&f->async_p->p_token)) {
1695 		ksignal(f->async_p, SIGIO);
1696 		lwkt_reltoken(&f->async_p->p_token);
1697 	}
1698 }
1699 
1700 static int
1701 usb_fifo_dummy_open(struct usb_fifo *fifo, int fflags)
1702 {
1703 	return (0);
1704 }
1705 
1706 static void
1707 usb_fifo_dummy_close(struct usb_fifo *fifo, int fflags)
1708 {
1709 	return;
1710 }
1711 
1712 static int
1713 usb_fifo_dummy_ioctl(struct usb_fifo *fifo, u_long cmd, void *addr, int fflags)
1714 {
1715 	return (ENOIOCTL);
1716 }
1717 
1718 static void
1719 usb_fifo_dummy_cmd(struct usb_fifo *fifo)
1720 {
1721 	fifo->flag_flushing = 0;	/* not flushing */
1722 }
1723 
1724 static void
1725 usb_fifo_check_methods(struct usb_fifo_methods *pm)
1726 {
1727 	/* check that all callback functions are OK */
1728 
1729 	if (pm->f_open == NULL)
1730 		pm->f_open = &usb_fifo_dummy_open;
1731 
1732 	if (pm->f_close == NULL)
1733 		pm->f_close = &usb_fifo_dummy_close;
1734 
1735 	if (pm->f_ioctl == NULL)
1736 		pm->f_ioctl = &usb_fifo_dummy_ioctl;
1737 
1738 	if (pm->f_ioctl_post == NULL)
1739 		pm->f_ioctl_post = &usb_fifo_dummy_ioctl;
1740 
1741 	if (pm->f_start_read == NULL)
1742 		pm->f_start_read = &usb_fifo_dummy_cmd;
1743 
1744 	if (pm->f_stop_read == NULL)
1745 		pm->f_stop_read = &usb_fifo_dummy_cmd;
1746 
1747 	if (pm->f_start_write == NULL)
1748 		pm->f_start_write = &usb_fifo_dummy_cmd;
1749 
1750 	if (pm->f_stop_write == NULL)
1751 		pm->f_stop_write = &usb_fifo_dummy_cmd;
1752 }
1753 
1754 /*------------------------------------------------------------------------*
1755  *	usb_fifo_attach
1756  *
1757  * The following function will create a duplex FIFO.
1758  *
1759  * Return values:
1760  * 0: Success.
1761  * Else: Failure.
1762  *------------------------------------------------------------------------*/
1763 int
1764 usb_fifo_attach(struct usb_device *udev, void *priv_sc,
1765     struct lock *priv_lock, struct usb_fifo_methods *pm,
1766     struct usb_fifo_sc *f_sc, uint16_t unit, int16_t subunit,
1767     uint8_t iface_index, uid_t uid, gid_t gid, int mode)
1768 {
1769 	struct usb_fifo *f_tx;
1770 	struct usb_fifo *f_rx;
1771 	char devname[32];
1772 	uint8_t n;
1773 
1774 	f_sc->fp[USB_FIFO_TX] = NULL;
1775 	f_sc->fp[USB_FIFO_RX] = NULL;
1776 
1777 	if (pm == NULL)
1778 		return (EINVAL);
1779 
1780 	/* check the methods */
1781 	usb_fifo_check_methods(pm);
1782 
1783 	/* search for a free FIFO slot */
1784 	for (n = 0;; n += 2) {
1785 
1786 		if (n == USB_FIFO_MAX) {
1787 			/* end of FIFOs reached */
1788 			return (ENOMEM);
1789 		}
1790 		/* Check for TX FIFO */
1791 		if (udev->fifo[n + USB_FIFO_TX] != NULL) {
1792 			continue;
1793 		}
1794 		/* Check for RX FIFO */
1795 		if (udev->fifo[n + USB_FIFO_RX] != NULL) {
1796 			continue;
1797 		}
1798 		break;
1799 	}
1800 
1801 	f_tx = usb_fifo_alloc();
1802 	f_rx = usb_fifo_alloc();
1803 
1804 	if ((f_tx == NULL) || (f_rx == NULL)) {
1805 		usb_fifo_free(f_tx);
1806 		usb_fifo_free(f_rx);
1807 		return (ENOMEM);
1808 	}
1809 	/* initialise FIFO structures */
1810 
1811 	f_tx->fifo_index = n + USB_FIFO_TX;
1812 	f_tx->dev_ep_index = -1;
1813 	f_tx->priv_lock = priv_lock;
1814 	f_tx->priv_sc0 = priv_sc;
1815 	f_tx->methods = pm;
1816 	f_tx->iface_index = iface_index;
1817 	f_tx->udev = udev;
1818 
1819 	f_rx->fifo_index = n + USB_FIFO_RX;
1820 	f_rx->dev_ep_index = -1;
1821 	f_rx->priv_lock = priv_lock;
1822 	f_rx->priv_sc0 = priv_sc;
1823 	f_rx->methods = pm;
1824 	f_rx->iface_index = iface_index;
1825 	f_rx->udev = udev;
1826 
1827 	f_sc->fp[USB_FIFO_TX] = f_tx;
1828 	f_sc->fp[USB_FIFO_RX] = f_rx;
1829 
1830 	lockmgr(&usb_ref_lock, LK_EXCLUSIVE);
1831 	udev->fifo[f_tx->fifo_index] = f_tx;
1832 	udev->fifo[f_rx->fifo_index] = f_rx;
1833 	lockmgr(&usb_ref_lock, LK_RELEASE);
1834 
1835 	for (n = 0; n != 4; n++) {
1836 
1837 		if (pm->basename[n] == NULL) {
1838 			continue;
1839 		}
1840 		if (subunit == 0xFFFF) {
1841 			if (ksnprintf(devname, sizeof(devname),
1842 			    "%s%u%s", pm->basename[n],
1843 			    unit, pm->postfix[n] ?
1844 			    pm->postfix[n] : "")) {
1845 				/* ignore */
1846 			}
1847 		} else {
1848 			if (ksnprintf(devname, sizeof(devname),
1849 			    "%s%u.%u%s", pm->basename[n],
1850 			    unit, subunit, pm->postfix[n] ?
1851 			    pm->postfix[n] : "")) {
1852 				/* ignore */
1853 			}
1854 		}
1855 
1856 		/*
1857 		 * Distribute the symbolic links into two FIFO structures:
1858 		 */
1859 		if (n & 1) {
1860 			f_rx->symlink[n / 2] =
1861 			    usb_alloc_symlink(devname);
1862 		} else {
1863 			f_tx->symlink[n / 2] =
1864 			    usb_alloc_symlink(devname);
1865 		}
1866 
1867 		/* Create the device */
1868 		f_sc->dev = usb_make_dev(udev, devname, -1,
1869 		    f_tx->fifo_index & f_rx->fifo_index,
1870 		    FREAD|FWRITE, uid, gid, mode);
1871 	}
1872 
1873 	DPRINTFN(2, "attached %p/%p\n", f_tx, f_rx);
1874 	return (0);
1875 }
1876 
1877 /*------------------------------------------------------------------------*
1878  *	usb_fifo_alloc_buffer
1879  *
1880  * Return values:
1881  * 0: Success
1882  * Else failure
1883  *------------------------------------------------------------------------*/
1884 int
1885 usb_fifo_alloc_buffer(struct usb_fifo *f, usb_size_t bufsize,
1886     uint16_t nbuf)
1887 {
1888 	usb_fifo_free_buffer(f);
1889 
1890 	/* allocate an endpoint */
1891 	f->free_q.ifq_maxlen = nbuf;
1892 	f->used_q.ifq_maxlen = nbuf;
1893 
1894 	f->queue_data = usb_alloc_mbufs(
1895 	    M_USBDEV, &f->free_q, bufsize, nbuf);
1896 
1897 	if ((f->queue_data == NULL) && bufsize && nbuf) {
1898 		return (ENOMEM);
1899 	}
1900 	return (0);			/* success */
1901 }
1902 
1903 /*------------------------------------------------------------------------*
1904  *	usb_fifo_free_buffer
1905  *
1906  * This function will free the buffers associated with a FIFO. This
1907  * function can be called multiple times in a row.
1908  *------------------------------------------------------------------------*/
1909 void
1910 usb_fifo_free_buffer(struct usb_fifo *f)
1911 {
1912 	if (f->queue_data) {
1913 		/* free old buffer */
1914 		kfree(f->queue_data, M_USBDEV);
1915 		f->queue_data = NULL;
1916 	}
1917 	/* reset queues */
1918 
1919 	memset(&f->free_q, 0, sizeof(f->free_q));
1920 	memset(&f->used_q, 0, sizeof(f->used_q));
1921 }
1922 
1923 void
1924 usb_fifo_detach(struct usb_fifo_sc *f_sc)
1925 {
1926 	if (f_sc == NULL) {
1927 		return;
1928 	}
1929 	usb_fifo_free(f_sc->fp[USB_FIFO_TX]);
1930 	usb_fifo_free(f_sc->fp[USB_FIFO_RX]);
1931 
1932 	f_sc->fp[USB_FIFO_TX] = NULL;
1933 	f_sc->fp[USB_FIFO_RX] = NULL;
1934 
1935 	usb_destroy_dev(f_sc->dev);
1936 
1937 	f_sc->dev = NULL;
1938 
1939 	DPRINTFN(2, "detached %p\n", f_sc);
1940 }
1941 
1942 usb_size_t
1943 usb_fifo_put_bytes_max(struct usb_fifo *f)
1944 {
1945 	struct usb_mbuf *m;
1946 	usb_size_t len;
1947 
1948 	USB_IF_POLL(&f->free_q, m);
1949 
1950 	if (m) {
1951 		len = m->max_data_len;
1952 	} else {
1953 		len = 0;
1954 	}
1955 	return (len);
1956 }
1957 
1958 /*------------------------------------------------------------------------*
1959  *	usb_fifo_put_data
1960  *
1961  * what:
1962  *  0 - normal operation
1963  *  1 - set last packet flag to enforce framing
1964  *------------------------------------------------------------------------*/
1965 void
1966 usb_fifo_put_data(struct usb_fifo *f, struct usb_page_cache *pc,
1967     usb_frlength_t offset, usb_frlength_t len, uint8_t what)
1968 {
1969 	struct usb_mbuf *m;
1970 	usb_frlength_t io_len;
1971 
1972 	while (len || (what == 1)) {
1973 
1974 		USB_IF_DEQUEUE(&f->free_q, m);
1975 
1976 		if (m) {
1977 			USB_MBUF_RESET(m);
1978 
1979 			io_len = MIN(len, m->cur_data_len);
1980 
1981 			usbd_copy_out(pc, offset, m->cur_data_ptr, io_len);
1982 
1983 			m->cur_data_len = io_len;
1984 			offset += io_len;
1985 			len -= io_len;
1986 
1987 			if ((len == 0) && (what == 1)) {
1988 				m->last_packet = 1;
1989 			}
1990 			USB_IF_ENQUEUE(&f->used_q, m);
1991 
1992 			usb_fifo_wakeup(f);
1993 
1994 			if ((len == 0) || (what == 1)) {
1995 				break;
1996 			}
1997 		} else {
1998 			break;
1999 		}
2000 	}
2001 }
2002 
2003 void
2004 usb_fifo_put_data_linear(struct usb_fifo *f, void *ptr,
2005     usb_size_t len, uint8_t what)
2006 {
2007 	struct usb_mbuf *m;
2008 	usb_size_t io_len;
2009 
2010 	while (len || (what == 1)) {
2011 
2012 		USB_IF_DEQUEUE(&f->free_q, m);
2013 
2014 		if (m) {
2015 			USB_MBUF_RESET(m);
2016 
2017 			io_len = MIN(len, m->cur_data_len);
2018 
2019 			memcpy(m->cur_data_ptr, ptr, io_len);
2020 
2021 			m->cur_data_len = io_len;
2022 			ptr = USB_ADD_BYTES(ptr, io_len);
2023 			len -= io_len;
2024 
2025 			if ((len == 0) && (what == 1)) {
2026 				m->last_packet = 1;
2027 			}
2028 			USB_IF_ENQUEUE(&f->used_q, m);
2029 
2030 			usb_fifo_wakeup(f);
2031 
2032 			if ((len == 0) || (what == 1)) {
2033 				break;
2034 			}
2035 		} else {
2036 			break;
2037 		}
2038 	}
2039 }
2040 
2041 uint8_t
2042 usb_fifo_put_data_buffer(struct usb_fifo *f, void *ptr, usb_size_t len)
2043 {
2044 	struct usb_mbuf *m;
2045 
2046 	USB_IF_DEQUEUE(&f->free_q, m);
2047 
2048 	if (m) {
2049 		m->cur_data_len = len;
2050 		m->cur_data_ptr = ptr;
2051 		USB_IF_ENQUEUE(&f->used_q, m);
2052 		usb_fifo_wakeup(f);
2053 		return (1);
2054 	}
2055 	return (0);
2056 }
2057 
2058 void
2059 usb_fifo_put_data_error(struct usb_fifo *f)
2060 {
2061 	f->flag_iserror = 1;
2062 	usb_fifo_wakeup(f);
2063 }
2064 
2065 /*------------------------------------------------------------------------*
2066  *	usb_fifo_get_data
2067  *
2068  * what:
2069  *  0 - normal operation
2070  *  1 - only get one "usb_mbuf"
2071  *
2072  * returns:
2073  *  0 - no more data
2074  *  1 - data in buffer
2075  *------------------------------------------------------------------------*/
2076 uint8_t
2077 usb_fifo_get_data(struct usb_fifo *f, struct usb_page_cache *pc,
2078     usb_frlength_t offset, usb_frlength_t len, usb_frlength_t *actlen,
2079     uint8_t what)
2080 {
2081 	struct usb_mbuf *m;
2082 	usb_frlength_t io_len;
2083 	uint8_t tr_data = 0;
2084 
2085 	actlen[0] = 0;
2086 
2087 	while (1) {
2088 
2089 		USB_IF_DEQUEUE(&f->used_q, m);
2090 
2091 		if (m) {
2092 
2093 			tr_data = 1;
2094 
2095 			io_len = MIN(len, m->cur_data_len);
2096 
2097 			usbd_copy_in(pc, offset, m->cur_data_ptr, io_len);
2098 
2099 			len -= io_len;
2100 			offset += io_len;
2101 			actlen[0] += io_len;
2102 			m->cur_data_ptr += io_len;
2103 			m->cur_data_len -= io_len;
2104 
2105 			if ((m->cur_data_len == 0) || (what == 1)) {
2106 				USB_IF_ENQUEUE(&f->free_q, m);
2107 
2108 				usb_fifo_wakeup(f);
2109 
2110 				if (what == 1) {
2111 					break;
2112 				}
2113 			} else {
2114 				USB_IF_PREPEND(&f->used_q, m);
2115 			}
2116 		} else {
2117 
2118 			if (tr_data) {
2119 				/* wait for data to be written out */
2120 				break;
2121 			}
2122 			if (f->flag_flushing) {
2123 				/* check if we should send a short packet */
2124 				if (f->flag_short != 0) {
2125 					f->flag_short = 0;
2126 					tr_data = 1;
2127 					break;
2128 				}
2129 				/* flushing complete */
2130 				f->flag_flushing = 0;
2131 				usb_fifo_wakeup(f);
2132 			}
2133 			break;
2134 		}
2135 		if (len == 0) {
2136 			break;
2137 		}
2138 	}
2139 	return (tr_data);
2140 }
2141 
2142 uint8_t
2143 usb_fifo_get_data_linear(struct usb_fifo *f, void *ptr,
2144     usb_size_t len, usb_size_t *actlen, uint8_t what)
2145 {
2146 	struct usb_mbuf *m;
2147 	usb_size_t io_len;
2148 	uint8_t tr_data = 0;
2149 
2150 	actlen[0] = 0;
2151 
2152 	while (1) {
2153 
2154 		USB_IF_DEQUEUE(&f->used_q, m);
2155 
2156 		if (m) {
2157 
2158 			tr_data = 1;
2159 
2160 			io_len = MIN(len, m->cur_data_len);
2161 
2162 			memcpy(ptr, m->cur_data_ptr, io_len);
2163 
2164 			len -= io_len;
2165 			ptr = USB_ADD_BYTES(ptr, io_len);
2166 			actlen[0] += io_len;
2167 			m->cur_data_ptr += io_len;
2168 			m->cur_data_len -= io_len;
2169 
2170 			if ((m->cur_data_len == 0) || (what == 1)) {
2171 				USB_IF_ENQUEUE(&f->free_q, m);
2172 
2173 				usb_fifo_wakeup(f);
2174 
2175 				if (what == 1) {
2176 					break;
2177 				}
2178 			} else {
2179 				USB_IF_PREPEND(&f->used_q, m);
2180 			}
2181 		} else {
2182 
2183 			if (tr_data) {
2184 				/* wait for data to be written out */
2185 				break;
2186 			}
2187 			if (f->flag_flushing) {
2188 				/* check if we should send a short packet */
2189 				if (f->flag_short != 0) {
2190 					f->flag_short = 0;
2191 					tr_data = 1;
2192 					break;
2193 				}
2194 				/* flushing complete */
2195 				f->flag_flushing = 0;
2196 				usb_fifo_wakeup(f);
2197 			}
2198 			break;
2199 		}
2200 		if (len == 0) {
2201 			break;
2202 		}
2203 	}
2204 	return (tr_data);
2205 }
2206 
2207 uint8_t
2208 usb_fifo_get_data_buffer(struct usb_fifo *f, void **pptr, usb_size_t *plen)
2209 {
2210 	struct usb_mbuf *m;
2211 
2212 	USB_IF_POLL(&f->used_q, m);
2213 
2214 	if (m) {
2215 		*plen = m->cur_data_len;
2216 		*pptr = m->cur_data_ptr;
2217 
2218 		return (1);
2219 	}
2220 	return (0);
2221 }
2222 
2223 void
2224 usb_fifo_get_data_error(struct usb_fifo *f)
2225 {
2226 	f->flag_iserror = 1;
2227 	usb_fifo_wakeup(f);
2228 }
2229 
2230 /*------------------------------------------------------------------------*
2231  *	usb_alloc_symlink
2232  *
2233  * Return values:
2234  * NULL: Failure
2235  * Else: Pointer to symlink entry
2236  *------------------------------------------------------------------------*/
2237 struct usb_symlink *
2238 usb_alloc_symlink(const char *target)
2239 {
2240 	struct usb_symlink *ps;
2241 
2242 	ps = kmalloc(sizeof(*ps), M_USBDEV, M_WAITOK);
2243 	if (ps == NULL) {
2244 		return (ps);
2245 	}
2246 	/* XXX no longer needed */
2247 	strlcpy(ps->src_path, target, sizeof(ps->src_path));
2248 	ps->src_len = strlen(ps->src_path);
2249 	strlcpy(ps->dst_path, target, sizeof(ps->dst_path));
2250 	ps->dst_len = strlen(ps->dst_path);
2251 
2252 	lockmgr(&usb_sym_lock, LK_EXCLUSIVE);
2253 	TAILQ_INSERT_TAIL(&usb_sym_head, ps, sym_entry);
2254 	lockmgr(&usb_sym_lock, LK_RELEASE);
2255 	return (ps);
2256 }
2257 
2258 /*------------------------------------------------------------------------*
2259  *	usb_free_symlink
2260  *------------------------------------------------------------------------*/
2261 void
2262 usb_free_symlink(struct usb_symlink *ps)
2263 {
2264 	if (ps == NULL) {
2265 		return;
2266 	}
2267 	lockmgr(&usb_sym_lock, LK_EXCLUSIVE);
2268 	TAILQ_REMOVE(&usb_sym_head, ps, sym_entry);
2269 	lockmgr(&usb_sym_lock, LK_RELEASE);
2270 
2271 	kfree(ps, M_USBDEV);
2272 }
2273 
2274 /*------------------------------------------------------------------------*
2275  *	usb_read_symlink
2276  *
2277  * Return value:
2278  * 0: Success
2279  * Else: Failure
2280  *------------------------------------------------------------------------*/
2281 int
2282 usb_read_symlink(uint8_t *user_ptr, uint32_t startentry, uint32_t user_len)
2283 {
2284 	struct usb_symlink *ps;
2285 	uint32_t temp;
2286 	uint32_t delta = 0;
2287 	uint8_t len;
2288 	int error = 0;
2289 
2290 	lockmgr(&usb_sym_lock, LK_EXCLUSIVE);
2291 
2292 	TAILQ_FOREACH(ps, &usb_sym_head, sym_entry) {
2293 
2294 		/*
2295 		 * Compute total length of source and destination symlink
2296 		 * strings pluss one length byte and two NUL bytes:
2297 		 */
2298 		temp = ps->src_len + ps->dst_len + 3;
2299 
2300 		if (temp > 255) {
2301 			/*
2302 			 * Skip entry because this length cannot fit
2303 			 * into one byte:
2304 			 */
2305 			continue;
2306 		}
2307 		if (startentry != 0) {
2308 			/* decrement read offset */
2309 			startentry--;
2310 			continue;
2311 		}
2312 		if (temp > user_len) {
2313 			/* out of buffer space */
2314 			break;
2315 		}
2316 		len = temp;
2317 
2318 		/* copy out total length */
2319 
2320 		error = copyout(&len,
2321 		    USB_ADD_BYTES(user_ptr, delta), 1);
2322 		if (error) {
2323 			break;
2324 		}
2325 		delta += 1;
2326 
2327 		/* copy out source string */
2328 
2329 		error = copyout(ps->src_path,
2330 		    USB_ADD_BYTES(user_ptr, delta), ps->src_len);
2331 		if (error) {
2332 			break;
2333 		}
2334 		len = 0;
2335 		delta += ps->src_len;
2336 		error = copyout(&len,
2337 		    USB_ADD_BYTES(user_ptr, delta), 1);
2338 		if (error) {
2339 			break;
2340 		}
2341 		delta += 1;
2342 
2343 		/* copy out destination string */
2344 
2345 		error = copyout(ps->dst_path,
2346 		    USB_ADD_BYTES(user_ptr, delta), ps->dst_len);
2347 		if (error) {
2348 			break;
2349 		}
2350 		len = 0;
2351 		delta += ps->dst_len;
2352 		error = copyout(&len,
2353 		    USB_ADD_BYTES(user_ptr, delta), 1);
2354 		if (error) {
2355 			break;
2356 		}
2357 		delta += 1;
2358 
2359 		user_len -= temp;
2360 	}
2361 
2362 	/* a zero length entry indicates the end */
2363 
2364 	if ((user_len != 0) && (error == 0)) {
2365 
2366 		len = 0;
2367 
2368 		error = copyout(&len,
2369 		    USB_ADD_BYTES(user_ptr, delta), 1);
2370 	}
2371 	lockmgr(&usb_sym_lock, LK_RELEASE);
2372 	return (error);
2373 }
2374 
2375 void
2376 usb_fifo_set_close_zlp(struct usb_fifo *f, uint8_t onoff)
2377 {
2378 	if (f == NULL)
2379 		return;
2380 
2381 	/* send a Zero Length Packet, ZLP, before close */
2382 	f->flag_short = onoff;
2383 }
2384 
2385 void
2386 usb_fifo_set_write_defrag(struct usb_fifo *f, uint8_t onoff)
2387 {
2388 	if (f == NULL)
2389 		return;
2390 
2391 	/* defrag written data */
2392 	f->flag_write_defrag = onoff;
2393 	/* reset defrag state */
2394 	f->flag_have_fragment = 0;
2395 }
2396 
2397 void *
2398 usb_fifo_softc(struct usb_fifo *f)
2399 {
2400 	return (f->priv_sc0);
2401 }
2402 #endif	/* USB_HAVE_UGEN */
2403