1 /* $FreeBSD: head/sys/dev/usb/controller/xhci.c 278477 2015-02-09 21:47:12Z hselasky $ */ 2 /*- 3 * Copyright (c) 2010 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 /* 28 * USB eXtensible Host Controller Interface, a.k.a. USB 3.0 controller. 29 * 30 * The XHCI 1.0 spec can be found at 31 * http://www.intel.com/technology/usb/download/xHCI_Specification_for_USB.pdf 32 * and the USB 3.0 spec at 33 * http://www.usb.org/developers/docs/usb_30_spec_060910.zip 34 */ 35 36 /* 37 * A few words about the design implementation: This driver emulates 38 * the concept about TDs which is found in EHCI specification. This 39 * way we achieve that the USB controller drivers look similar to 40 * eachother which makes it easier to understand the code. 41 */ 42 43 #include <sys/stdint.h> 44 #include <sys/param.h> 45 #include <sys/queue.h> 46 #include <sys/types.h> 47 #include <sys/systm.h> 48 #include <sys/kernel.h> 49 #include <sys/bus.h> 50 #include <sys/module.h> 51 #include <sys/lock.h> 52 #include <sys/condvar.h> 53 #include <sys/sysctl.h> 54 #include <sys/unistd.h> 55 #include <sys/callout.h> 56 #include <sys/malloc.h> 57 #include <sys/priv.h> 58 59 #include <bus/u4b/usb.h> 60 #include <bus/u4b/usbdi.h> 61 62 #define USB_DEBUG_VAR xhcidebug 63 64 #include <bus/u4b/usb_core.h> 65 #include <bus/u4b/usb_debug.h> 66 #include <bus/u4b/usb_busdma.h> 67 #include <bus/u4b/usb_process.h> 68 #include <bus/u4b/usb_transfer.h> 69 #include <bus/u4b/usb_device.h> 70 #include <bus/u4b/usb_hub.h> 71 #include <bus/u4b/usb_util.h> 72 73 #include <bus/u4b/usb_controller.h> 74 #include <bus/u4b/usb_bus.h> 75 #include <bus/u4b/controller/xhci.h> 76 #include <bus/u4b/controller/xhcireg.h> 77 78 #define XHCI_BUS2SC(bus) \ 79 ((struct xhci_softc *)(((uint8_t *)(bus)) - \ 80 ((uint8_t *)&(((struct xhci_softc *)0)->sc_bus)))) 81 82 static SYSCTL_NODE(_hw_usb, OID_AUTO, xhci, CTLFLAG_RW, 0, "USB XHCI"); 83 84 static int xhcistreams; 85 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, streams, CTLFLAG_RW, 86 &xhcistreams, 0, "Set to enable streams mode support"); 87 TUNABLE_INT("hw.usb.xhci.streams", &xhcistreams); 88 89 90 #ifdef USB_DEBUG 91 static int xhcidebug = 0; 92 static int xhciroute = 0; 93 static int xhcipolling = 0; 94 static int xhcidma32; 95 96 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, debug, CTLFLAG_RW, 97 &xhcidebug, 0, "Debug level"); 98 TUNABLE_INT("hw.usb.xhci.debug", &xhcidebug); 99 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, xhci_port_route, CTLFLAG_RW, 100 &xhciroute, 0, "Routing bitmap for switching EHCI ports to XHCI controller"); 101 TUNABLE_INT("hw.usb.xhci.xhci_port_route", &xhciroute); 102 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, use_polling, CTLFLAG_RW, 103 &xhcipolling, 0, "Set to enable software interrupt polling for XHCI controller"); 104 TUNABLE_INT("hw.usb.xhci.use_polling", &xhcipolling); 105 106 SYSCTL_INT(_hw_usb_xhci, OID_AUTO, dma32, CTLFLAG_RW, 107 &xhcidma32, 0, "Set to only use 32-bit DMA for the XHCI controller"); 108 TUNABLE_INT("hw.usb.xhci.dma32", &xhcidma32); 109 110 #else 111 #define xhciroute 0 112 #define xhcidma32 0 113 #endif 114 115 #define XHCI_INTR_ENDPT 1 116 117 struct xhci_std_temp { 118 struct xhci_softc *sc; 119 struct usb_page_cache *pc; 120 struct xhci_td *td; 121 struct xhci_td *td_next; 122 uint32_t len; 123 uint32_t offset; 124 uint32_t max_packet_size; 125 uint32_t average; 126 uint16_t isoc_delta; 127 uint16_t isoc_frame; 128 uint8_t shortpkt; 129 uint8_t multishort; 130 uint8_t last_frame; 131 uint8_t trb_type; 132 uint8_t direction; 133 uint8_t tbc; 134 uint8_t tlbpc; 135 uint8_t step_td; 136 uint8_t do_isoc_sync; 137 }; 138 139 static void xhci_do_poll(struct usb_bus *); 140 static void xhci_device_done(struct usb_xfer *, usb_error_t); 141 static void xhci_root_intr(struct xhci_softc *); 142 static void xhci_free_device_ext(struct usb_device *); 143 static struct xhci_endpoint_ext *xhci_get_endpoint_ext(struct usb_device *, 144 struct usb_endpoint_descriptor *); 145 static usb_proc_callback_t xhci_configure_msg; 146 static usb_error_t xhci_configure_device(struct usb_device *); 147 static usb_error_t xhci_configure_endpoint(struct usb_device *, 148 struct usb_endpoint_descriptor *, struct xhci_endpoint_ext *, 149 uint16_t, uint8_t, uint8_t, uint8_t, uint16_t, uint16_t, 150 uint8_t); 151 static usb_error_t xhci_configure_mask(struct usb_device *, 152 uint32_t, uint8_t); 153 static usb_error_t xhci_cmd_evaluate_ctx(struct xhci_softc *, 154 uint64_t, uint8_t); 155 static void xhci_endpoint_doorbell(struct usb_xfer *); 156 static void xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val); 157 static uint32_t xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr); 158 static void xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val); 159 #ifdef USB_DEBUG 160 static uint64_t xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr); 161 #endif 162 163 static const struct usb_bus_methods xhci_bus_methods; 164 165 #ifdef USB_DEBUG 166 static void 167 xhci_dump_trb(struct xhci_trb *trb) 168 { 169 DPRINTFN(5, "trb = %p\n", trb); 170 DPRINTFN(5, "qwTrb0 = 0x%016llx\n", (long long)le64toh(trb->qwTrb0)); 171 DPRINTFN(5, "dwTrb2 = 0x%08x\n", le32toh(trb->dwTrb2)); 172 DPRINTFN(5, "dwTrb3 = 0x%08x\n", le32toh(trb->dwTrb3)); 173 } 174 175 static void 176 xhci_dump_endpoint(struct xhci_softc *sc, struct xhci_endp_ctx *pep) 177 { 178 DPRINTFN(5, "pep = %p\n", pep); 179 DPRINTFN(5, "dwEpCtx0=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx0)); 180 DPRINTFN(5, "dwEpCtx1=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx1)); 181 DPRINTFN(5, "qwEpCtx2=0x%016llx\n", (long long)xhci_ctx_get_le64(sc, &pep->qwEpCtx2)); 182 DPRINTFN(5, "dwEpCtx4=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx4)); 183 DPRINTFN(5, "dwEpCtx5=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx5)); 184 DPRINTFN(5, "dwEpCtx6=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx6)); 185 DPRINTFN(5, "dwEpCtx7=0x%08x\n", xhci_ctx_get_le32(sc, &pep->dwEpCtx7)); 186 } 187 188 static void 189 xhci_dump_device(struct xhci_softc *sc, struct xhci_slot_ctx *psl) 190 { 191 DPRINTFN(5, "psl = %p\n", psl); 192 DPRINTFN(5, "dwSctx0=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx0)); 193 DPRINTFN(5, "dwSctx1=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx1)); 194 DPRINTFN(5, "dwSctx2=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx2)); 195 DPRINTFN(5, "dwSctx3=0x%08x\n", xhci_ctx_get_le32(sc, &psl->dwSctx3)); 196 } 197 #endif 198 199 uint8_t 200 xhci_use_polling(void) 201 { 202 #ifdef USB_DEBUG 203 return (xhcipolling != 0); 204 #else 205 return (0); 206 #endif 207 } 208 209 static void 210 xhci_iterate_hw_softc(struct usb_bus *bus, usb_bus_mem_sub_cb_t *cb) 211 { 212 struct xhci_softc *sc = XHCI_BUS2SC(bus); 213 uint8_t i; 214 215 cb(bus, &sc->sc_hw.root_pc, &sc->sc_hw.root_pg, 216 sizeof(struct xhci_hw_root), XHCI_PAGE_SIZE); 217 218 cb(bus, &sc->sc_hw.ctx_pc, &sc->sc_hw.ctx_pg, 219 sizeof(struct xhci_dev_ctx_addr), XHCI_PAGE_SIZE); 220 221 for (i = 0; i != XHCI_MAX_SCRATCHPADS; i++) { 222 cb(bus, &sc->sc_hw.scratch_pc[i], &sc->sc_hw.scratch_pg[i], 223 XHCI_PAGE_SIZE, XHCI_PAGE_SIZE); 224 } 225 } 226 227 static void 228 xhci_ctx_set_le32(struct xhci_softc *sc, volatile uint32_t *ptr, uint32_t val) 229 { 230 if (sc->sc_ctx_is_64_byte) { 231 uint32_t offset; 232 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 233 /* all contexts are initially 32-bytes */ 234 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 235 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 236 } 237 *ptr = htole32(val); 238 } 239 240 static uint32_t 241 xhci_ctx_get_le32(struct xhci_softc *sc, volatile uint32_t *ptr) 242 { 243 if (sc->sc_ctx_is_64_byte) { 244 uint32_t offset; 245 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 246 /* all contexts are initially 32-bytes */ 247 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 248 ptr = (volatile uint32_t *)(((volatile uint8_t *)ptr) + offset); 249 } 250 return (le32toh(*ptr)); 251 } 252 253 static void 254 xhci_ctx_set_le64(struct xhci_softc *sc, volatile uint64_t *ptr, uint64_t val) 255 { 256 if (sc->sc_ctx_is_64_byte) { 257 uint32_t offset; 258 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 259 /* all contexts are initially 32-bytes */ 260 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 261 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 262 } 263 *ptr = htole64(val); 264 } 265 266 #ifdef USB_DEBUG 267 static uint64_t 268 xhci_ctx_get_le64(struct xhci_softc *sc, volatile uint64_t *ptr) 269 { 270 if (sc->sc_ctx_is_64_byte) { 271 uint32_t offset; 272 /* exploit the fact that our structures are XHCI_PAGE_SIZE aligned */ 273 /* all contexts are initially 32-bytes */ 274 offset = ((uintptr_t)ptr) & ((XHCI_PAGE_SIZE - 1) & ~(31U)); 275 ptr = (volatile uint64_t *)(((volatile uint8_t *)ptr) + offset); 276 } 277 return (le64toh(*ptr)); 278 } 279 #endif 280 281 static int 282 xhci_reset_command_queue_locked(struct xhci_softc *sc) 283 { 284 struct usb_page_search buf_res; 285 struct xhci_hw_root *phwr; 286 uint64_t addr; 287 uint32_t temp; 288 289 DPRINTF("\n"); 290 291 temp = XREAD4(sc, oper, XHCI_CRCR_LO); 292 if (temp & XHCI_CRCR_LO_CRR) { 293 DPRINTF("Command ring running\n"); 294 temp &= ~(XHCI_CRCR_LO_CS | XHCI_CRCR_LO_CA); 295 296 /* 297 * Try to abort the last command as per section 298 * 4.6.1.2 "Aborting a Command" of the XHCI 299 * specification: 300 */ 301 302 /* stop and cancel */ 303 XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CS); 304 XWRITE4(sc, oper, XHCI_CRCR_HI, 0); 305 306 XWRITE4(sc, oper, XHCI_CRCR_LO, temp | XHCI_CRCR_LO_CA); 307 XWRITE4(sc, oper, XHCI_CRCR_HI, 0); 308 309 /* wait 250ms */ 310 usb_pause_mtx(&sc->sc_bus.bus_lock, hz / 4); 311 312 /* check if command ring is still running */ 313 temp = XREAD4(sc, oper, XHCI_CRCR_LO); 314 if (temp & XHCI_CRCR_LO_CRR) { 315 DPRINTF("Command ring still running\n"); 316 return (USB_ERR_IOERROR); 317 } 318 } 319 320 /* reset command ring */ 321 sc->sc_command_ccs = 1; 322 sc->sc_command_idx = 0; 323 324 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 325 326 /* set up command ring control base address */ 327 addr = buf_res.physaddr; 328 phwr = buf_res.buffer; 329 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0]; 330 331 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr); 332 333 memset(phwr->hwr_commands, 0, sizeof(phwr->hwr_commands)); 334 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr); 335 336 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 337 338 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS); 339 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32)); 340 341 return (0); 342 } 343 344 usb_error_t 345 xhci_start_controller(struct xhci_softc *sc) 346 { 347 struct usb_page_search buf_res; 348 struct xhci_hw_root *phwr; 349 struct xhci_dev_ctx_addr *pdctxa; 350 uint64_t addr; 351 uint32_t temp; 352 uint16_t i; 353 354 DPRINTF("\n"); 355 356 sc->sc_event_ccs = 1; 357 sc->sc_event_idx = 0; 358 sc->sc_command_ccs = 1; 359 sc->sc_command_idx = 0; 360 361 /* Reset controller */ 362 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_HCRST); 363 364 for (i = 0; i != 100; i++) { 365 usb_pause_mtx(NULL, hz / 100); 366 temp = (XREAD4(sc, oper, XHCI_USBCMD) & XHCI_CMD_HCRST) | 367 (XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_CNR); 368 if (!temp) 369 break; 370 } 371 372 if (temp) { 373 device_printf(sc->sc_bus.parent, "Controller " 374 "reset timeout.\n"); 375 return (USB_ERR_IOERROR); 376 } 377 378 if (!(XREAD4(sc, oper, XHCI_PAGESIZE) & XHCI_PAGESIZE_4K)) { 379 device_printf(sc->sc_bus.parent, "Controller does " 380 "not support 4K page size.\n"); 381 return (USB_ERR_IOERROR); 382 } 383 384 temp = XREAD4(sc, capa, XHCI_HCSPARAMS1); 385 386 i = XHCI_HCS1_N_PORTS(temp); 387 388 if (i == 0) { 389 device_printf(sc->sc_bus.parent, "Invalid number " 390 "of ports: %u\n", i); 391 return (USB_ERR_IOERROR); 392 } 393 394 sc->sc_noport = i; 395 sc->sc_noslot = XHCI_HCS1_DEVSLOT_MAX(temp); 396 397 if (sc->sc_noslot > XHCI_MAX_DEVICES) 398 sc->sc_noslot = XHCI_MAX_DEVICES; 399 400 /* set up number of device slots */ 401 402 DPRINTF("CONFIG=0x%08x -> 0x%08x\n", 403 XREAD4(sc, oper, XHCI_CONFIG), sc->sc_noslot); 404 405 XWRITE4(sc, oper, XHCI_CONFIG, sc->sc_noslot); 406 407 DPRINTF("Max slots: %u\n", sc->sc_noslot); 408 409 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2); 410 411 sc->sc_noscratch = XHCI_HCS2_SPB_MAX(temp); 412 413 if (sc->sc_noscratch > XHCI_MAX_SCRATCHPADS) { 414 device_printf(sc->sc_bus.parent, "XHCI request " 415 "too many scratchpads\n"); 416 return (USB_ERR_NOMEM); 417 } 418 419 DPRINTF("Max scratch: %u\n", sc->sc_noscratch); 420 421 temp = XREAD4(sc, capa, XHCI_HCSPARAMS3); 422 423 sc->sc_exit_lat_max = XHCI_HCS3_U1_DEL(temp) + 424 XHCI_HCS3_U2_DEL(temp) + 250 /* us */; 425 426 temp = XREAD4(sc, oper, XHCI_USBSTS); 427 428 /* clear interrupts */ 429 XWRITE4(sc, oper, XHCI_USBSTS, temp); 430 /* disable all device notifications */ 431 XWRITE4(sc, oper, XHCI_DNCTRL, 0); 432 433 /* set up device context base address */ 434 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 435 pdctxa = buf_res.buffer; 436 memset(pdctxa, 0, sizeof(*pdctxa)); 437 438 addr = buf_res.physaddr; 439 addr += (uintptr_t)&((struct xhci_dev_ctx_addr *)0)->qwSpBufPtr[0]; 440 441 /* slot 0 points to the table of scratchpad pointers */ 442 pdctxa->qwBaaDevCtxAddr[0] = htole64(addr); 443 444 for (i = 0; i != sc->sc_noscratch; i++) { 445 struct usb_page_search buf_scp; 446 usbd_get_page(&sc->sc_hw.scratch_pc[i], 0, &buf_scp); 447 pdctxa->qwSpBufPtr[i] = htole64((uint64_t)buf_scp.physaddr); 448 } 449 450 addr = buf_res.physaddr; 451 452 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 453 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 454 XWRITE4(sc, oper, XHCI_DCBAAP_LO, (uint32_t)addr); 455 XWRITE4(sc, oper, XHCI_DCBAAP_HI, (uint32_t)(addr >> 32)); 456 457 /* Setup event table size */ 458 459 temp = XREAD4(sc, capa, XHCI_HCSPARAMS2); 460 461 DPRINTF("HCS2=0x%08x\n", temp); 462 463 temp = XHCI_HCS2_ERST_MAX(temp); 464 temp = 1U << temp; 465 if (temp > XHCI_MAX_RSEG) 466 temp = XHCI_MAX_RSEG; 467 468 sc->sc_erst_max = temp; 469 470 DPRINTF("ERSTSZ=0x%08x -> 0x%08x\n", 471 XREAD4(sc, runt, XHCI_ERSTSZ(0)), temp); 472 473 XWRITE4(sc, runt, XHCI_ERSTSZ(0), XHCI_ERSTS_SET(temp)); 474 475 /* Check if we should use the default IMOD value */ 476 if (sc->sc_imod_default == 0) 477 sc->sc_imod_default = XHCI_IMOD_DEFAULT; 478 479 /* Setup interrupt rate */ 480 XWRITE4(sc, runt, XHCI_IMOD(0), sc->sc_imod_default); 481 482 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 483 484 phwr = buf_res.buffer; 485 addr = buf_res.physaddr; 486 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[0]; 487 488 /* reset hardware root structure */ 489 memset(phwr, 0, sizeof(*phwr)); 490 491 phwr->hwr_ring_seg[0].qwEvrsTablePtr = htole64(addr); 492 phwr->hwr_ring_seg[0].dwEvrsTableSize = htole32(XHCI_MAX_EVENTS); 493 494 DPRINTF("ERDP(0)=0x%016llx\n", (unsigned long long)addr); 495 496 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 497 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 498 499 addr = buf_res.physaddr; 500 501 DPRINTF("ERSTBA(0)=0x%016llx\n", (unsigned long long)addr); 502 503 XWRITE4(sc, runt, XHCI_ERSTBA_LO(0), (uint32_t)addr); 504 XWRITE4(sc, runt, XHCI_ERSTBA_HI(0), (uint32_t)(addr >> 32)); 505 506 /* Setup interrupter registers */ 507 508 temp = XREAD4(sc, runt, XHCI_IMAN(0)); 509 temp |= XHCI_IMAN_INTR_ENA; 510 XWRITE4(sc, runt, XHCI_IMAN(0), temp); 511 512 /* set up command ring control base address */ 513 addr = buf_res.physaddr; 514 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[0]; 515 516 DPRINTF("CRCR=0x%016llx\n", (unsigned long long)addr); 517 518 XWRITE4(sc, oper, XHCI_CRCR_LO, ((uint32_t)addr) | XHCI_CRCR_LO_RCS); 519 XWRITE4(sc, oper, XHCI_CRCR_HI, (uint32_t)(addr >> 32)); 520 521 phwr->hwr_commands[XHCI_MAX_COMMANDS - 1].qwTrb0 = htole64(addr); 522 523 usb_bus_mem_flush_all(&sc->sc_bus, &xhci_iterate_hw_softc); 524 525 /* Go! */ 526 XWRITE4(sc, oper, XHCI_USBCMD, XHCI_CMD_RS | 527 XHCI_CMD_INTE | XHCI_CMD_HSEE); 528 529 for (i = 0; i != 100; i++) { 530 usb_pause_mtx(NULL, hz / 100); 531 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 532 if (!temp) 533 break; 534 } 535 if (temp) { 536 XWRITE4(sc, oper, XHCI_USBCMD, 0); 537 device_printf(sc->sc_bus.parent, "Run timeout.\n"); 538 return (USB_ERR_IOERROR); 539 } 540 541 /* catch any lost interrupts */ 542 xhci_do_poll(&sc->sc_bus); 543 544 if (sc->sc_port_route != NULL) { 545 /* Route all ports to the XHCI by default */ 546 sc->sc_port_route(sc->sc_bus.parent, 547 ~xhciroute, xhciroute); 548 } 549 return (0); 550 } 551 552 usb_error_t 553 xhci_halt_controller(struct xhci_softc *sc) 554 { 555 uint32_t temp; 556 uint16_t i; 557 558 DPRINTF("\n"); 559 560 sc->sc_event_ccs = 1; 561 sc->sc_event_idx = 0; 562 sc->sc_command_ccs = 1; 563 sc->sc_command_idx = 0; 564 565 /* Halt controller */ 566 XWRITE4(sc, oper, XHCI_USBCMD, 0); 567 568 for (i = 0; i != 100; i++) { 569 usb_pause_mtx(NULL, hz / 100); 570 temp = XREAD4(sc, oper, XHCI_USBSTS) & XHCI_STS_HCH; 571 if (temp) 572 break; 573 } 574 575 if (!temp) { 576 device_printf(sc->sc_bus.parent, "Controller halt timeout.\n"); 577 return (USB_ERR_IOERROR); 578 } 579 return (0); 580 } 581 582 usb_error_t 583 xhci_init(struct xhci_softc *sc, device_t self, uint8_t dma32) 584 { 585 uint32_t temp; 586 587 DPRINTF("\n"); 588 589 /* initialise some bus fields */ 590 sc->sc_bus.parent = self; 591 592 /* set the bus revision */ 593 sc->sc_bus.usbrev = USB_REV_3_0; 594 595 /* set up the bus struct */ 596 sc->sc_bus.methods = &xhci_bus_methods; 597 598 /* set up devices array */ 599 sc->sc_bus.devices = sc->sc_devices; 600 sc->sc_bus.devices_max = XHCI_MAX_DEVICES; 601 602 /* set default cycle state in case of early interrupts */ 603 sc->sc_event_ccs = 1; 604 sc->sc_command_ccs = 1; 605 606 /* set up bus space offsets */ 607 sc->sc_capa_off = 0; 608 sc->sc_oper_off = XREAD1(sc, capa, XHCI_CAPLENGTH); 609 sc->sc_runt_off = XREAD4(sc, capa, XHCI_RTSOFF) & ~0x1F; 610 sc->sc_door_off = XREAD4(sc, capa, XHCI_DBOFF) & ~0x3; 611 612 DPRINTF("CAPLENGTH=0x%x\n", sc->sc_oper_off); 613 DPRINTF("RUNTIMEOFFSET=0x%x\n", sc->sc_runt_off); 614 DPRINTF("DOOROFFSET=0x%x\n", sc->sc_door_off); 615 616 DPRINTF("xHCI version = 0x%04x\n", XREAD2(sc, capa, XHCI_HCIVERSION)); 617 618 temp = XREAD4(sc, capa, XHCI_HCSPARAMS0); 619 620 DPRINTF("HCS0 = 0x%08x\n", temp); 621 622 /* set up context size */ 623 if (XHCI_HCS0_CSZ(temp)) { 624 sc->sc_ctx_is_64_byte = 1; 625 } else { 626 sc->sc_ctx_is_64_byte = 0; 627 } 628 629 /* get DMA bits */ 630 sc->sc_bus.dma_bits = (XHCI_HCS0_AC64(temp) && 631 xhcidma32 == 0 && dma32 == 0) ? 64 : 32; 632 633 device_printf(self, "%d bytes context size, %d-bit DMA\n", 634 sc->sc_ctx_is_64_byte ? 64 : 32, (int)sc->sc_bus.dma_bits); 635 636 /* get all DMA memory */ 637 if (usb_bus_mem_alloc_all(&sc->sc_bus, 638 USB_GET_DMA_TAG(self), &xhci_iterate_hw_softc)) { 639 return (ENOMEM); 640 } 641 642 /* setup command queue mutex and condition varible */ 643 cv_init(&sc->sc_cmd_cv, "CMDQ"); 644 lockinit(&sc->sc_cmd_lock, "CMDQ lock", 0, LK_CANRECURSE); 645 646 /* set up command queue mutex and condition varible */ 647 sc->sc_config_msg[0].hdr.pm_callback = &xhci_configure_msg; 648 sc->sc_config_msg[0].bus = &sc->sc_bus; 649 sc->sc_config_msg[1].hdr.pm_callback = &xhci_configure_msg; 650 sc->sc_config_msg[1].bus = &sc->sc_bus; 651 652 return (0); 653 } 654 655 void 656 xhci_uninit(struct xhci_softc *sc) 657 { 658 /* 659 * NOTE: At this point the control transfer process is gone 660 * and "xhci_configure_msg" is no longer called. Consequently 661 * waiting for the configuration messages to complete is not 662 * needed. 663 */ 664 usb_bus_mem_free_all(&sc->sc_bus, &xhci_iterate_hw_softc); 665 666 cv_destroy(&sc->sc_cmd_cv); 667 lockuninit(&sc->sc_cmd_lock); 668 } 669 670 static void 671 xhci_set_hw_power_sleep(struct usb_bus *bus, uint32_t state) 672 { 673 struct xhci_softc *sc = XHCI_BUS2SC(bus); 674 675 switch (state) { 676 case USB_HW_POWER_SUSPEND: 677 DPRINTF("Stopping the XHCI\n"); 678 xhci_halt_controller(sc); 679 break; 680 case USB_HW_POWER_SHUTDOWN: 681 DPRINTF("Stopping the XHCI\n"); 682 xhci_halt_controller(sc); 683 break; 684 case USB_HW_POWER_RESUME: 685 DPRINTF("Starting the XHCI\n"); 686 xhci_start_controller(sc); 687 break; 688 default: 689 break; 690 } 691 } 692 693 static usb_error_t 694 xhci_generic_done_sub(struct usb_xfer *xfer) 695 { 696 struct xhci_td *td; 697 struct xhci_td *td_alt_next; 698 uint32_t len; 699 uint8_t status; 700 701 td = xfer->td_transfer_cache; 702 td_alt_next = td->alt_next; 703 704 if (xfer->aframes != xfer->nframes) 705 usbd_xfer_set_frame_len(xfer, xfer->aframes, 0); 706 707 while (1) { 708 709 usb_pc_cpu_invalidate(td->page_cache); 710 711 status = td->status; 712 len = td->remainder; 713 714 DPRINTFN(4, "xfer=%p[%u/%u] rem=%u/%u status=%u\n", 715 xfer, (unsigned int)xfer->aframes, 716 (unsigned int)xfer->nframes, 717 (unsigned int)len, (unsigned int)td->len, 718 (unsigned int)status); 719 720 /* 721 * Verify the status length and 722 * add the length to "frlengths[]": 723 */ 724 if (len > td->len) { 725 /* should not happen */ 726 DPRINTF("Invalid status length, " 727 "0x%04x/0x%04x bytes\n", len, td->len); 728 status = XHCI_TRB_ERROR_LENGTH; 729 } else if (xfer->aframes != xfer->nframes) { 730 xfer->frlengths[xfer->aframes] += td->len - len; 731 } 732 /* Check for last transfer */ 733 if (((void *)td) == xfer->td_transfer_last) { 734 td = NULL; 735 break; 736 } 737 /* Check for transfer error */ 738 if (status != XHCI_TRB_ERROR_SHORT_PKT && 739 status != XHCI_TRB_ERROR_SUCCESS) { 740 /* the transfer is finished */ 741 td = NULL; 742 break; 743 } 744 /* Check for short transfer */ 745 if (len > 0) { 746 if (xfer->flags_int.short_frames_ok || 747 xfer->flags_int.isochronous_xfr || 748 xfer->flags_int.control_xfr) { 749 /* follow alt next */ 750 td = td->alt_next; 751 } else { 752 /* the transfer is finished */ 753 td = NULL; 754 } 755 break; 756 } 757 td = td->obj_next; 758 759 if (td->alt_next != td_alt_next) { 760 /* this USB frame is complete */ 761 break; 762 } 763 } 764 765 /* update transfer cache */ 766 767 xfer->td_transfer_cache = td; 768 769 return ((status == XHCI_TRB_ERROR_STALL) ? USB_ERR_STALLED : 770 (status != XHCI_TRB_ERROR_SHORT_PKT && 771 status != XHCI_TRB_ERROR_SUCCESS) ? USB_ERR_IOERROR : 772 USB_ERR_NORMAL_COMPLETION); 773 } 774 775 static void 776 xhci_generic_done(struct usb_xfer *xfer) 777 { 778 usb_error_t err = 0; 779 780 DPRINTFN(13, "xfer=%p endpoint=%p transfer done\n", 781 xfer, xfer->endpoint); 782 783 /* reset scanner */ 784 785 xfer->td_transfer_cache = xfer->td_transfer_first; 786 787 if (xfer->flags_int.control_xfr) { 788 789 if (xfer->flags_int.control_hdr) 790 err = xhci_generic_done_sub(xfer); 791 792 xfer->aframes = 1; 793 794 if (xfer->td_transfer_cache == NULL) 795 goto done; 796 } 797 798 while (xfer->aframes != xfer->nframes) { 799 800 err = xhci_generic_done_sub(xfer); 801 xfer->aframes++; 802 803 if (xfer->td_transfer_cache == NULL) 804 goto done; 805 } 806 807 if (xfer->flags_int.control_xfr && 808 !xfer->flags_int.control_act) 809 err = xhci_generic_done_sub(xfer); 810 done: 811 /* transfer is complete */ 812 xhci_device_done(xfer, err); 813 } 814 815 static void 816 xhci_activate_transfer(struct usb_xfer *xfer) 817 { 818 struct xhci_td *td; 819 820 td = xfer->td_transfer_cache; 821 822 usb_pc_cpu_invalidate(td->page_cache); 823 824 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 825 826 /* activate the transfer */ 827 828 td->td_trb[0].dwTrb3 |= htole32(XHCI_TRB_3_CYCLE_BIT); 829 usb_pc_cpu_flush(td->page_cache); 830 831 xhci_endpoint_doorbell(xfer); 832 } 833 } 834 835 static void 836 xhci_skip_transfer(struct usb_xfer *xfer) 837 { 838 struct xhci_td *td; 839 struct xhci_td *td_last; 840 841 td = xfer->td_transfer_cache; 842 td_last = xfer->td_transfer_last; 843 844 td = td->alt_next; 845 846 usb_pc_cpu_invalidate(td->page_cache); 847 848 if (!(td->td_trb[0].dwTrb3 & htole32(XHCI_TRB_3_CYCLE_BIT))) { 849 850 usb_pc_cpu_invalidate(td_last->page_cache); 851 852 /* copy LINK TRB to current waiting location */ 853 854 td->td_trb[0].qwTrb0 = td_last->td_trb[td_last->ntrb].qwTrb0; 855 td->td_trb[0].dwTrb2 = td_last->td_trb[td_last->ntrb].dwTrb2; 856 usb_pc_cpu_flush(td->page_cache); 857 858 td->td_trb[0].dwTrb3 = td_last->td_trb[td_last->ntrb].dwTrb3; 859 usb_pc_cpu_flush(td->page_cache); 860 861 xhci_endpoint_doorbell(xfer); 862 } 863 } 864 865 /*------------------------------------------------------------------------* 866 * xhci_check_transfer 867 *------------------------------------------------------------------------*/ 868 static void 869 xhci_check_transfer(struct xhci_softc *sc, struct xhci_trb *trb) 870 { 871 struct xhci_endpoint_ext *pepext; 872 int64_t offset; 873 uint64_t td_event; 874 uint32_t temp; 875 uint32_t remainder; 876 uint16_t stream_id; 877 uint16_t i; 878 uint8_t status; 879 uint8_t halted; 880 uint8_t epno; 881 uint8_t index; 882 883 /* decode TRB */ 884 td_event = le64toh(trb->qwTrb0); 885 temp = le32toh(trb->dwTrb2); 886 887 remainder = XHCI_TRB_2_REM_GET(temp); 888 status = XHCI_TRB_2_ERROR_GET(temp); 889 stream_id = XHCI_TRB_2_STREAM_GET(temp); 890 891 temp = le32toh(trb->dwTrb3); 892 epno = XHCI_TRB_3_EP_GET(temp); 893 index = XHCI_TRB_3_SLOT_GET(temp); 894 895 /* check if error means halted */ 896 halted = (status != XHCI_TRB_ERROR_SHORT_PKT && 897 status != XHCI_TRB_ERROR_SUCCESS); 898 899 DPRINTF("slot=%u epno=%u stream=%u remainder=%u status=%u\n", 900 index, epno, stream_id, remainder, status); 901 902 if (index > sc->sc_noslot) { 903 DPRINTF("Invalid slot.\n"); 904 return; 905 } 906 907 if ((epno == 0) || (epno >= XHCI_MAX_ENDPOINTS)) { 908 DPRINTF("Invalid endpoint.\n"); 909 return; 910 } 911 912 pepext = &sc->sc_hw.devs[index].endp[epno]; 913 914 if (pepext->trb_ep_mode != USB_EP_MODE_STREAMS) { 915 stream_id = 0; 916 DPRINTF("stream_id=0\n"); 917 } else if (stream_id >= XHCI_MAX_STREAMS) { 918 DPRINTF("Invalid stream ID.\n"); 919 return; 920 } 921 922 /* try to find the USB transfer that generated the event */ 923 for (i = 0; i != (XHCI_MAX_TRANSFERS - 1); i++) { 924 struct usb_xfer *xfer; 925 struct xhci_td *td; 926 927 xfer = pepext->xfer[i + (XHCI_MAX_TRANSFERS * stream_id)]; 928 if (xfer == NULL) 929 continue; 930 931 td = xfer->td_transfer_cache; 932 933 DPRINTFN(5, "Checking if 0x%016llx == (0x%016llx .. 0x%016llx)\n", 934 (long long)td_event, 935 (long long)td->td_self, 936 (long long)td->td_self + sizeof(td->td_trb)); 937 938 /* 939 * NOTE: Some XHCI implementations might not trigger 940 * an event on the last LINK TRB so we need to 941 * consider both the last and second last event 942 * address as conditions for a successful transfer. 943 * 944 * NOTE: We assume that the XHCI will only trigger one 945 * event per chain of TRBs. 946 */ 947 948 offset = td_event - td->td_self; 949 950 if (offset >= 0 && 951 offset < (int64_t)sizeof(td->td_trb)) { 952 953 usb_pc_cpu_invalidate(td->page_cache); 954 955 /* compute rest of remainder, if any */ 956 for (i = (offset / 16) + 1; i < td->ntrb; i++) { 957 temp = le32toh(td->td_trb[i].dwTrb2); 958 remainder += XHCI_TRB_2_BYTES_GET(temp); 959 } 960 961 DPRINTFN(5, "New remainder: %u\n", remainder); 962 963 /* clear isochronous transfer errors */ 964 if (xfer->flags_int.isochronous_xfr) { 965 if (halted) { 966 halted = 0; 967 status = XHCI_TRB_ERROR_SUCCESS; 968 remainder = td->len; 969 } 970 } 971 972 /* "td->remainder" is verified later */ 973 td->remainder = remainder; 974 td->status = status; 975 976 usb_pc_cpu_flush(td->page_cache); 977 978 /* 979 * 1) Last transfer descriptor makes the 980 * transfer done 981 */ 982 if (((void *)td) == xfer->td_transfer_last) { 983 DPRINTF("TD is last\n"); 984 xhci_generic_done(xfer); 985 break; 986 } 987 988 /* 989 * 2) Any kind of error makes the transfer 990 * done 991 */ 992 if (halted) { 993 DPRINTF("TD has I/O error\n"); 994 xhci_generic_done(xfer); 995 break; 996 } 997 998 /* 999 * 3) If there is no alternate next transfer, 1000 * a short packet also makes the transfer done 1001 */ 1002 if (td->remainder > 0) { 1003 if (td->alt_next == NULL) { 1004 DPRINTF( 1005 "short TD has no alternate next\n"); 1006 xhci_generic_done(xfer); 1007 break; 1008 } 1009 DPRINTF("TD has short pkt\n"); 1010 if (xfer->flags_int.short_frames_ok || 1011 xfer->flags_int.isochronous_xfr || 1012 xfer->flags_int.control_xfr) { 1013 /* follow the alt next */ 1014 xfer->td_transfer_cache = td->alt_next; 1015 xhci_activate_transfer(xfer); 1016 break; 1017 } 1018 xhci_skip_transfer(xfer); 1019 xhci_generic_done(xfer); 1020 break; 1021 } 1022 1023 /* 1024 * 4) Transfer complete - go to next TD 1025 */ 1026 DPRINTF("Following next TD\n"); 1027 xfer->td_transfer_cache = td->obj_next; 1028 xhci_activate_transfer(xfer); 1029 break; /* there should only be one match */ 1030 } 1031 } 1032 } 1033 1034 static int 1035 xhci_check_command(struct xhci_softc *sc, struct xhci_trb *trb) 1036 { 1037 if (sc->sc_cmd_addr == trb->qwTrb0) { 1038 DPRINTF("Received command event\n"); 1039 sc->sc_cmd_result[0] = trb->dwTrb2; 1040 sc->sc_cmd_result[1] = trb->dwTrb3; 1041 cv_signal(&sc->sc_cmd_cv); 1042 return (1); /* command match */ 1043 } 1044 return (0); 1045 } 1046 1047 static int 1048 xhci_interrupt_poll(struct xhci_softc *sc) 1049 { 1050 struct usb_page_search buf_res; 1051 struct xhci_hw_root *phwr; 1052 uint64_t addr; 1053 uint32_t temp; 1054 int retval = 0; 1055 uint16_t i; 1056 uint8_t event; 1057 uint8_t j; 1058 uint8_t k; 1059 uint8_t t; 1060 1061 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 1062 1063 phwr = buf_res.buffer; 1064 1065 /* Receive any events */ 1066 1067 usb_pc_cpu_invalidate(&sc->sc_hw.root_pc); 1068 1069 i = sc->sc_event_idx; 1070 j = sc->sc_event_ccs; 1071 t = 2; 1072 1073 while (1) { 1074 1075 temp = le32toh(phwr->hwr_events[i].dwTrb3); 1076 1077 k = (temp & XHCI_TRB_3_CYCLE_BIT) ? 1 : 0; 1078 1079 if (j != k) 1080 break; 1081 1082 event = XHCI_TRB_3_TYPE_GET(temp); 1083 1084 DPRINTFN(10, "event[%u] = %u (0x%016llx 0x%08lx 0x%08lx)\n", 1085 i, event, (long long)le64toh(phwr->hwr_events[i].qwTrb0), 1086 (long)le32toh(phwr->hwr_events[i].dwTrb2), 1087 (long)le32toh(phwr->hwr_events[i].dwTrb3)); 1088 1089 switch (event) { 1090 case XHCI_TRB_EVENT_TRANSFER: 1091 xhci_check_transfer(sc, &phwr->hwr_events[i]); 1092 break; 1093 case XHCI_TRB_EVENT_CMD_COMPLETE: 1094 retval |= xhci_check_command(sc, &phwr->hwr_events[i]); 1095 break; 1096 default: 1097 DPRINTF("Unhandled event = %u\n", event); 1098 break; 1099 } 1100 1101 i++; 1102 1103 if (i == XHCI_MAX_EVENTS) { 1104 i = 0; 1105 j ^= 1; 1106 1107 /* check for timeout */ 1108 if (!--t) 1109 break; 1110 } 1111 } 1112 1113 sc->sc_event_idx = i; 1114 sc->sc_event_ccs = j; 1115 1116 /* 1117 * NOTE: The Event Ring Dequeue Pointer Register is 64-bit 1118 * latched. That means to activate the register we need to 1119 * write both the low and high double word of the 64-bit 1120 * register. 1121 */ 1122 1123 addr = buf_res.physaddr; 1124 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_events[i]; 1125 1126 /* try to clear busy bit */ 1127 addr |= XHCI_ERDP_LO_BUSY; 1128 1129 XWRITE4(sc, runt, XHCI_ERDP_LO(0), (uint32_t)addr); 1130 XWRITE4(sc, runt, XHCI_ERDP_HI(0), (uint32_t)(addr >> 32)); 1131 1132 return (retval); 1133 } 1134 1135 static usb_error_t 1136 xhci_do_command(struct xhci_softc *sc, struct xhci_trb *trb, 1137 uint16_t timeout_ms) 1138 { 1139 struct usb_page_search buf_res; 1140 struct xhci_hw_root *phwr; 1141 uint64_t addr; 1142 uint32_t temp; 1143 uint8_t i; 1144 uint8_t j; 1145 uint8_t timeout = 0; 1146 int err; 1147 1148 XHCI_CMD_ASSERT_LOCKED(sc); 1149 1150 /* get hardware root structure */ 1151 1152 usbd_get_page(&sc->sc_hw.root_pc, 0, &buf_res); 1153 1154 phwr = buf_res.buffer; 1155 1156 /* Queue command */ 1157 1158 USB_BUS_LOCK(&sc->sc_bus); 1159 retry: 1160 i = sc->sc_command_idx; 1161 j = sc->sc_command_ccs; 1162 1163 DPRINTFN(10, "command[%u] = %u (0x%016llx, 0x%08lx, 0x%08lx)\n", 1164 i, XHCI_TRB_3_TYPE_GET(le32toh(trb->dwTrb3)), 1165 (long long)le64toh(trb->qwTrb0), 1166 (long)le32toh(trb->dwTrb2), 1167 (long)le32toh(trb->dwTrb3)); 1168 1169 phwr->hwr_commands[i].qwTrb0 = trb->qwTrb0; 1170 phwr->hwr_commands[i].dwTrb2 = trb->dwTrb2; 1171 1172 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1173 1174 temp = trb->dwTrb3; 1175 1176 if (j) 1177 temp |= htole32(XHCI_TRB_3_CYCLE_BIT); 1178 else 1179 temp &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1180 1181 temp &= ~htole32(XHCI_TRB_3_TC_BIT); 1182 1183 phwr->hwr_commands[i].dwTrb3 = temp; 1184 1185 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1186 1187 addr = buf_res.physaddr; 1188 addr += (uintptr_t)&((struct xhci_hw_root *)0)->hwr_commands[i]; 1189 1190 sc->sc_cmd_addr = htole64(addr); 1191 1192 i++; 1193 1194 if (i == (XHCI_MAX_COMMANDS - 1)) { 1195 1196 if (j) { 1197 temp = htole32(XHCI_TRB_3_TC_BIT | 1198 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1199 XHCI_TRB_3_CYCLE_BIT); 1200 } else { 1201 temp = htole32(XHCI_TRB_3_TC_BIT | 1202 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 1203 } 1204 1205 phwr->hwr_commands[i].dwTrb3 = temp; 1206 1207 usb_pc_cpu_flush(&sc->sc_hw.root_pc); 1208 1209 i = 0; 1210 j ^= 1; 1211 } 1212 1213 sc->sc_command_idx = i; 1214 sc->sc_command_ccs = j; 1215 1216 XWRITE4(sc, door, XHCI_DOORBELL(0), 0); 1217 1218 err = cv_timedwait(&sc->sc_cmd_cv, &sc->sc_bus.bus_lock, 1219 USB_MS_TO_TICKS(timeout_ms)); 1220 1221 /* 1222 * In some error cases event interrupts are not generated. 1223 * Poll one time to see if the command has completed. 1224 */ 1225 if (err != 0 && xhci_interrupt_poll(sc) != 0) { 1226 DPRINTF("Command was completed when polling\n"); 1227 err = 0; 1228 } 1229 if (err != 0) { 1230 DPRINTF("Command timeout!\n"); 1231 /* 1232 * After some weeks of continuous operation, it has 1233 * been observed that the ASMedia Technology, ASM1042 1234 * SuperSpeed USB Host Controller can suddenly stop 1235 * accepting commands via the command queue. Try to 1236 * first reset the command queue. If that fails do a 1237 * host controller reset. 1238 */ 1239 if (timeout == 0 && 1240 xhci_reset_command_queue_locked(sc) == 0) { 1241 temp = le32toh(trb->dwTrb3); 1242 1243 /* 1244 * Avoid infinite XHCI reset loops if the set 1245 * address command fails to respond due to a 1246 * non-enumerating device: 1247 */ 1248 if (XHCI_TRB_3_TYPE_GET(temp) == XHCI_TRB_TYPE_ADDRESS_DEVICE && 1249 (temp & XHCI_TRB_3_BSR_BIT) == 0) { 1250 DPRINTF("Set address timeout\n"); 1251 } else { 1252 timeout = 1; 1253 goto retry; 1254 } 1255 } else { 1256 DPRINTF("Controller reset!\n"); 1257 usb_bus_reset_async_locked(&sc->sc_bus); 1258 } 1259 err = USB_ERR_TIMEOUT; 1260 trb->dwTrb2 = 0; 1261 trb->dwTrb3 = 0; 1262 } else { 1263 temp = le32toh(sc->sc_cmd_result[0]); 1264 if (XHCI_TRB_2_ERROR_GET(temp) != XHCI_TRB_ERROR_SUCCESS) 1265 err = USB_ERR_IOERROR; 1266 1267 trb->dwTrb2 = sc->sc_cmd_result[0]; 1268 trb->dwTrb3 = sc->sc_cmd_result[1]; 1269 } 1270 1271 USB_BUS_UNLOCK(&sc->sc_bus); 1272 1273 return (err); 1274 } 1275 1276 #if 0 1277 static usb_error_t 1278 xhci_cmd_nop(struct xhci_softc *sc) 1279 { 1280 struct xhci_trb trb; 1281 uint32_t temp; 1282 1283 DPRINTF("\n"); 1284 1285 trb.qwTrb0 = 0; 1286 trb.dwTrb2 = 0; 1287 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NOOP); 1288 1289 trb.dwTrb3 = htole32(temp); 1290 1291 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1292 } 1293 #endif 1294 1295 static usb_error_t 1296 xhci_cmd_enable_slot(struct xhci_softc *sc, uint8_t *pslot) 1297 { 1298 struct xhci_trb trb; 1299 uint32_t temp; 1300 usb_error_t err; 1301 1302 DPRINTF("\n"); 1303 1304 trb.qwTrb0 = 0; 1305 trb.dwTrb2 = 0; 1306 trb.dwTrb3 = htole32(XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ENABLE_SLOT)); 1307 1308 err = xhci_do_command(sc, &trb, 100 /* ms */); 1309 if (err) 1310 goto done; 1311 1312 temp = le32toh(trb.dwTrb3); 1313 1314 *pslot = XHCI_TRB_3_SLOT_GET(temp); 1315 1316 done: 1317 return (err); 1318 } 1319 1320 static usb_error_t 1321 xhci_cmd_disable_slot(struct xhci_softc *sc, uint8_t slot_id) 1322 { 1323 struct xhci_trb trb; 1324 uint32_t temp; 1325 1326 DPRINTF("\n"); 1327 1328 trb.qwTrb0 = 0; 1329 trb.dwTrb2 = 0; 1330 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DISABLE_SLOT) | 1331 XHCI_TRB_3_SLOT_SET(slot_id); 1332 1333 trb.dwTrb3 = htole32(temp); 1334 1335 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1336 } 1337 1338 static usb_error_t 1339 xhci_cmd_set_address(struct xhci_softc *sc, uint64_t input_ctx, 1340 uint8_t bsr, uint8_t slot_id) 1341 { 1342 struct xhci_trb trb; 1343 uint32_t temp; 1344 1345 DPRINTF("\n"); 1346 1347 trb.qwTrb0 = htole64(input_ctx); 1348 trb.dwTrb2 = 0; 1349 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ADDRESS_DEVICE) | 1350 XHCI_TRB_3_SLOT_SET(slot_id); 1351 1352 if (bsr) 1353 temp |= XHCI_TRB_3_BSR_BIT; 1354 1355 trb.dwTrb3 = htole32(temp); 1356 1357 return (xhci_do_command(sc, &trb, 500 /* ms */)); 1358 } 1359 1360 static usb_error_t 1361 xhci_set_address(struct usb_device *udev, struct lock *lock, uint16_t address) 1362 { 1363 struct usb_page_search buf_inp; 1364 struct usb_page_search buf_dev; 1365 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 1366 struct xhci_hw_dev *hdev; 1367 struct xhci_dev_ctx *pdev; 1368 struct xhci_endpoint_ext *pepext; 1369 uint32_t temp; 1370 uint16_t mps; 1371 usb_error_t err; 1372 uint8_t index; 1373 1374 /* the root HUB case is not handled here */ 1375 if (udev->parent_hub == NULL) 1376 return (USB_ERR_INVAL); 1377 1378 index = udev->controller_slot_id; 1379 1380 hdev = &sc->sc_hw.devs[index]; 1381 1382 if (lock != NULL) 1383 lockmgr(lock, LK_RELEASE); 1384 1385 XHCI_CMD_LOCK(sc); 1386 1387 switch (hdev->state) { 1388 case XHCI_ST_DEFAULT: 1389 case XHCI_ST_ENABLED: 1390 1391 hdev->state = XHCI_ST_ENABLED; 1392 1393 /* set configure mask to slot and EP0 */ 1394 xhci_configure_mask(udev, 3, 0); 1395 1396 /* configure input slot context structure */ 1397 err = xhci_configure_device(udev); 1398 1399 if (err != 0) { 1400 DPRINTF("Could not configure device\n"); 1401 break; 1402 } 1403 1404 /* configure input endpoint context structure */ 1405 switch (udev->speed) { 1406 case USB_SPEED_LOW: 1407 case USB_SPEED_FULL: 1408 mps = 8; 1409 break; 1410 case USB_SPEED_HIGH: 1411 mps = 64; 1412 break; 1413 default: 1414 mps = 512; 1415 break; 1416 } 1417 1418 pepext = xhci_get_endpoint_ext(udev, 1419 &udev->ctrl_ep_desc); 1420 1421 /* ensure the control endpoint is setup again */ 1422 USB_BUS_LOCK(udev->bus); 1423 pepext->trb_halted = 1; 1424 pepext->trb_running = 0; 1425 USB_BUS_UNLOCK(udev->bus); 1426 1427 err = xhci_configure_endpoint(udev, 1428 &udev->ctrl_ep_desc, pepext, 1429 0, 1, 1, 0, mps, mps, USB_EP_MODE_DEFAULT); 1430 1431 if (err != 0) { 1432 DPRINTF("Could not configure default endpoint\n"); 1433 break; 1434 } 1435 1436 /* execute set address command */ 1437 usbd_get_page(&hdev->input_pc, 0, &buf_inp); 1438 1439 err = xhci_cmd_set_address(sc, buf_inp.physaddr, 1440 (address == 0), index); 1441 1442 if (err != 0) { 1443 temp = le32toh(sc->sc_cmd_result[0]); 1444 if (address == 0 && sc->sc_port_route != NULL && 1445 XHCI_TRB_2_ERROR_GET(temp) == 1446 XHCI_TRB_ERROR_PARAMETER) { 1447 /* LynxPoint XHCI - ports are not switchable */ 1448 /* Un-route all ports from the XHCI */ 1449 sc->sc_port_route(sc->sc_bus.parent, 0, ~0); 1450 } 1451 DPRINTF("Could not set address " 1452 "for slot %u.\n", index); 1453 if (address != 0) 1454 break; 1455 } 1456 1457 /* update device address to new value */ 1458 1459 usbd_get_page(&hdev->device_pc, 0, &buf_dev); 1460 pdev = buf_dev.buffer; 1461 usb_pc_cpu_invalidate(&hdev->device_pc); 1462 1463 temp = xhci_ctx_get_le32(sc, &pdev->ctx_slot.dwSctx3); 1464 udev->address = XHCI_SCTX_3_DEV_ADDR_GET(temp); 1465 1466 /* update device state to new value */ 1467 1468 if (address != 0) 1469 hdev->state = XHCI_ST_ADDRESSED; 1470 else 1471 hdev->state = XHCI_ST_DEFAULT; 1472 break; 1473 1474 default: 1475 DPRINTF("Wrong state for set address.\n"); 1476 err = USB_ERR_IOERROR; 1477 break; 1478 } 1479 XHCI_CMD_UNLOCK(sc); 1480 1481 if (lock != NULL) 1482 lockmgr(lock, LK_EXCLUSIVE); 1483 1484 return (err); 1485 } 1486 1487 static usb_error_t 1488 xhci_cmd_configure_ep(struct xhci_softc *sc, uint64_t input_ctx, 1489 uint8_t deconfigure, uint8_t slot_id) 1490 { 1491 struct xhci_trb trb; 1492 uint32_t temp; 1493 1494 DPRINTF("\n"); 1495 1496 trb.qwTrb0 = htole64(input_ctx); 1497 trb.dwTrb2 = 0; 1498 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_CONFIGURE_EP) | 1499 XHCI_TRB_3_SLOT_SET(slot_id); 1500 1501 if (deconfigure) 1502 temp |= XHCI_TRB_3_DCEP_BIT; 1503 1504 trb.dwTrb3 = htole32(temp); 1505 1506 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1507 } 1508 1509 static usb_error_t 1510 xhci_cmd_evaluate_ctx(struct xhci_softc *sc, uint64_t input_ctx, 1511 uint8_t slot_id) 1512 { 1513 struct xhci_trb trb; 1514 uint32_t temp; 1515 1516 DPRINTF("\n"); 1517 1518 trb.qwTrb0 = htole64(input_ctx); 1519 trb.dwTrb2 = 0; 1520 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_EVALUATE_CTX) | 1521 XHCI_TRB_3_SLOT_SET(slot_id); 1522 trb.dwTrb3 = htole32(temp); 1523 1524 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1525 } 1526 1527 static usb_error_t 1528 xhci_cmd_reset_ep(struct xhci_softc *sc, uint8_t preserve, 1529 uint8_t ep_id, uint8_t slot_id) 1530 { 1531 struct xhci_trb trb; 1532 uint32_t temp; 1533 1534 DPRINTF("\n"); 1535 1536 trb.qwTrb0 = 0; 1537 trb.dwTrb2 = 0; 1538 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_EP) | 1539 XHCI_TRB_3_SLOT_SET(slot_id) | 1540 XHCI_TRB_3_EP_SET(ep_id); 1541 1542 if (preserve) 1543 temp |= XHCI_TRB_3_PRSV_BIT; 1544 1545 trb.dwTrb3 = htole32(temp); 1546 1547 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1548 } 1549 1550 static usb_error_t 1551 xhci_cmd_set_tr_dequeue_ptr(struct xhci_softc *sc, uint64_t dequeue_ptr, 1552 uint16_t stream_id, uint8_t ep_id, uint8_t slot_id) 1553 { 1554 struct xhci_trb trb; 1555 uint32_t temp; 1556 1557 DPRINTF("\n"); 1558 1559 trb.qwTrb0 = htole64(dequeue_ptr); 1560 1561 temp = XHCI_TRB_2_STREAM_SET(stream_id); 1562 trb.dwTrb2 = htole32(temp); 1563 1564 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SET_TR_DEQUEUE) | 1565 XHCI_TRB_3_SLOT_SET(slot_id) | 1566 XHCI_TRB_3_EP_SET(ep_id); 1567 trb.dwTrb3 = htole32(temp); 1568 1569 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1570 } 1571 1572 static usb_error_t 1573 xhci_cmd_stop_ep(struct xhci_softc *sc, uint8_t suspend, 1574 uint8_t ep_id, uint8_t slot_id) 1575 { 1576 struct xhci_trb trb; 1577 uint32_t temp; 1578 1579 DPRINTF("\n"); 1580 1581 trb.qwTrb0 = 0; 1582 trb.dwTrb2 = 0; 1583 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STOP_EP) | 1584 XHCI_TRB_3_SLOT_SET(slot_id) | 1585 XHCI_TRB_3_EP_SET(ep_id); 1586 1587 if (suspend) 1588 temp |= XHCI_TRB_3_SUSP_EP_BIT; 1589 1590 trb.dwTrb3 = htole32(temp); 1591 1592 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1593 } 1594 1595 static usb_error_t 1596 xhci_cmd_reset_dev(struct xhci_softc *sc, uint8_t slot_id) 1597 { 1598 struct xhci_trb trb; 1599 uint32_t temp; 1600 1601 DPRINTF("\n"); 1602 1603 trb.qwTrb0 = 0; 1604 trb.dwTrb2 = 0; 1605 temp = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_RESET_DEVICE) | 1606 XHCI_TRB_3_SLOT_SET(slot_id); 1607 1608 trb.dwTrb3 = htole32(temp); 1609 1610 return (xhci_do_command(sc, &trb, 100 /* ms */)); 1611 } 1612 1613 /*------------------------------------------------------------------------* 1614 * xhci_interrupt - XHCI interrupt handler 1615 *------------------------------------------------------------------------*/ 1616 void 1617 xhci_interrupt(struct xhci_softc *sc) 1618 { 1619 uint32_t status; 1620 uint32_t temp; 1621 1622 USB_BUS_LOCK(&sc->sc_bus); 1623 1624 status = XREAD4(sc, oper, XHCI_USBSTS); 1625 1626 /* acknowledge interrupts, if any */ 1627 if (status != 0) { 1628 XWRITE4(sc, oper, XHCI_USBSTS, status); 1629 DPRINTFN(16, "real interrupt (status=0x%08x)\n", status); 1630 } 1631 1632 temp = XREAD4(sc, runt, XHCI_IMAN(0)); 1633 1634 /* force clearing of pending interrupts */ 1635 if (temp & XHCI_IMAN_INTR_PEND) 1636 XWRITE4(sc, runt, XHCI_IMAN(0), temp); 1637 1638 /* check for event(s) */ 1639 xhci_interrupt_poll(sc); 1640 1641 if (status & (XHCI_STS_PCD | XHCI_STS_HCH | 1642 XHCI_STS_HSE | XHCI_STS_HCE)) { 1643 1644 if (status & XHCI_STS_PCD) { 1645 xhci_root_intr(sc); 1646 } 1647 1648 if (status & XHCI_STS_HCH) { 1649 kprintf("%s: host controller halted\n", 1650 __func__); 1651 } 1652 1653 if (status & XHCI_STS_HSE) { 1654 kprintf("%s: host system error\n", 1655 __func__); 1656 } 1657 1658 if (status & XHCI_STS_HCE) { 1659 kprintf("%s: host controller error\n", 1660 __func__); 1661 } 1662 } 1663 USB_BUS_UNLOCK(&sc->sc_bus); 1664 } 1665 1666 /*------------------------------------------------------------------------* 1667 * xhci_timeout - XHCI timeout handler 1668 *------------------------------------------------------------------------*/ 1669 static void 1670 xhci_timeout(void *arg) 1671 { 1672 struct usb_xfer *xfer = arg; 1673 1674 DPRINTF("xfer=%p\n", xfer); 1675 1676 USB_BUS_LOCK_ASSERT(xfer->xroot->bus); 1677 1678 /* transfer is transferred */ 1679 xhci_device_done(xfer, USB_ERR_TIMEOUT); 1680 } 1681 1682 static void 1683 xhci_do_poll(struct usb_bus *bus) 1684 { 1685 struct xhci_softc *sc = XHCI_BUS2SC(bus); 1686 1687 USB_BUS_LOCK(&sc->sc_bus); 1688 xhci_interrupt_poll(sc); 1689 USB_BUS_UNLOCK(&sc->sc_bus); 1690 } 1691 1692 static void 1693 xhci_setup_generic_chain_sub(struct xhci_std_temp *temp) 1694 { 1695 struct usb_page_search buf_res; 1696 struct xhci_td *td; 1697 struct xhci_td *td_next; 1698 struct xhci_td *td_alt_next; 1699 struct xhci_td *td_first; 1700 uint32_t buf_offset; 1701 uint32_t average; 1702 uint32_t len_old; 1703 uint32_t npkt_off; 1704 uint32_t dword; 1705 uint8_t shortpkt_old; 1706 uint8_t precompute; 1707 uint8_t x; 1708 1709 td_alt_next = NULL; 1710 buf_offset = 0; 1711 shortpkt_old = temp->shortpkt; 1712 len_old = temp->len; 1713 npkt_off = 0; 1714 precompute = 1; 1715 1716 restart: 1717 1718 td = temp->td; 1719 td_next = td_first = temp->td_next; 1720 1721 while (1) { 1722 1723 if (temp->len == 0) { 1724 1725 if (temp->shortpkt) 1726 break; 1727 1728 /* send a Zero Length Packet, ZLP, last */ 1729 1730 temp->shortpkt = 1; 1731 average = 0; 1732 1733 } else { 1734 1735 average = temp->average; 1736 1737 if (temp->len < average) { 1738 if (temp->len % temp->max_packet_size) { 1739 temp->shortpkt = 1; 1740 } 1741 average = temp->len; 1742 } 1743 } 1744 1745 if (td_next == NULL) 1746 panic("%s: out of XHCI transfer descriptors!", __func__); 1747 1748 /* get next TD */ 1749 1750 td = td_next; 1751 td_next = td->obj_next; 1752 1753 /* check if we are pre-computing */ 1754 1755 if (precompute) { 1756 1757 /* update remaining length */ 1758 1759 temp->len -= average; 1760 1761 continue; 1762 } 1763 /* fill out current TD */ 1764 1765 td->len = average; 1766 td->remainder = 0; 1767 td->status = 0; 1768 1769 /* update remaining length */ 1770 1771 temp->len -= average; 1772 1773 /* reset TRB index */ 1774 1775 x = 0; 1776 1777 if (temp->trb_type == XHCI_TRB_TYPE_SETUP_STAGE) { 1778 /* immediate data */ 1779 1780 if (average > 8) 1781 average = 8; 1782 1783 td->td_trb[0].qwTrb0 = 0; 1784 1785 usbd_copy_out(temp->pc, temp->offset + buf_offset, 1786 (uint8_t *)(uintptr_t)&td->td_trb[0].qwTrb0, 1787 average); 1788 1789 dword = XHCI_TRB_2_BYTES_SET(8) | 1790 XHCI_TRB_2_TDSZ_SET(0) | 1791 XHCI_TRB_2_IRQ_SET(0); 1792 1793 td->td_trb[0].dwTrb2 = htole32(dword); 1794 1795 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_SETUP_STAGE) | 1796 XHCI_TRB_3_IDT_BIT | XHCI_TRB_3_CYCLE_BIT; 1797 1798 /* check wLength */ 1799 if (td->td_trb[0].qwTrb0 & 1800 htole64(XHCI_TRB_0_WLENGTH_MASK)) { 1801 if (td->td_trb[0].qwTrb0 & 1802 htole64(XHCI_TRB_0_DIR_IN_MASK)) 1803 dword |= XHCI_TRB_3_TRT_IN; 1804 else 1805 dword |= XHCI_TRB_3_TRT_OUT; 1806 } 1807 1808 td->td_trb[0].dwTrb3 = htole32(dword); 1809 #ifdef USB_DEBUG 1810 xhci_dump_trb(&td->td_trb[x]); 1811 #endif 1812 x++; 1813 1814 } else do { 1815 1816 uint32_t npkt; 1817 1818 /* fill out buffer pointers */ 1819 1820 if (average == 0) { 1821 memset(&buf_res, 0, sizeof(buf_res)); 1822 } else { 1823 usbd_get_page(temp->pc, temp->offset + 1824 buf_offset, &buf_res); 1825 1826 /* get length to end of page */ 1827 if (buf_res.length > average) 1828 buf_res.length = average; 1829 1830 /* check for maximum length */ 1831 if (buf_res.length > XHCI_TD_PAGE_SIZE) 1832 buf_res.length = XHCI_TD_PAGE_SIZE; 1833 1834 npkt_off += buf_res.length; 1835 } 1836 1837 /* set up npkt */ 1838 npkt = (len_old - npkt_off + temp->max_packet_size - 1) / 1839 temp->max_packet_size; 1840 1841 if (npkt == 0) 1842 npkt = 1; 1843 else if (npkt > 31) 1844 npkt = 31; 1845 1846 /* fill out TRB's */ 1847 td->td_trb[x].qwTrb0 = 1848 htole64((uint64_t)buf_res.physaddr); 1849 1850 dword = 1851 XHCI_TRB_2_BYTES_SET(buf_res.length) | 1852 XHCI_TRB_2_TDSZ_SET(npkt) | 1853 XHCI_TRB_2_IRQ_SET(0); 1854 1855 td->td_trb[x].dwTrb2 = htole32(dword); 1856 1857 switch (temp->trb_type) { 1858 case XHCI_TRB_TYPE_ISOCH: 1859 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1860 XHCI_TRB_3_TBC_SET(temp->tbc) | 1861 XHCI_TRB_3_TLBPC_SET(temp->tlbpc); 1862 if (td != td_first) { 1863 dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL); 1864 } else if (temp->do_isoc_sync != 0) { 1865 temp->do_isoc_sync = 0; 1866 /* wait until "isoc_frame" */ 1867 dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) | 1868 XHCI_TRB_3_FRID_SET(temp->isoc_frame / 8); 1869 } else { 1870 /* start data transfer at next interval */ 1871 dword |= XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_ISOCH) | 1872 XHCI_TRB_3_ISO_SIA_BIT; 1873 } 1874 if (temp->direction == UE_DIR_IN) 1875 dword |= XHCI_TRB_3_ISP_BIT; 1876 break; 1877 case XHCI_TRB_TYPE_DATA_STAGE: 1878 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1879 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_DATA_STAGE); 1880 if (temp->direction == UE_DIR_IN) 1881 dword |= XHCI_TRB_3_DIR_IN | XHCI_TRB_3_ISP_BIT; 1882 /* 1883 * Section 3.2.9 in the XHCI 1884 * specification about control 1885 * transfers says that we should use a 1886 * normal-TRB if there are more TRBs 1887 * extending the data-stage 1888 * TRB. Update the "trb_type". 1889 */ 1890 temp->trb_type = XHCI_TRB_TYPE_NORMAL; 1891 break; 1892 case XHCI_TRB_TYPE_STATUS_STAGE: 1893 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1894 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_STATUS_STAGE); 1895 if (temp->direction == UE_DIR_IN) 1896 dword |= XHCI_TRB_3_DIR_IN; 1897 break; 1898 default: /* XHCI_TRB_TYPE_NORMAL */ 1899 dword = XHCI_TRB_3_CHAIN_BIT | XHCI_TRB_3_CYCLE_BIT | 1900 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_NORMAL); 1901 if (temp->direction == UE_DIR_IN) 1902 dword |= XHCI_TRB_3_ISP_BIT; 1903 break; 1904 } 1905 td->td_trb[x].dwTrb3 = htole32(dword); 1906 1907 average -= buf_res.length; 1908 buf_offset += buf_res.length; 1909 #ifdef USB_DEBUG 1910 xhci_dump_trb(&td->td_trb[x]); 1911 #endif 1912 x++; 1913 1914 } while (average != 0); 1915 1916 td->td_trb[x-1].dwTrb3 |= htole32(XHCI_TRB_3_IOC_BIT); 1917 1918 /* store number of data TRB's */ 1919 1920 td->ntrb = x; 1921 1922 DPRINTF("NTRB=%u\n", x); 1923 1924 /* fill out link TRB */ 1925 1926 if (td_next != NULL) { 1927 /* link the current TD with the next one */ 1928 td->td_trb[x].qwTrb0 = htole64((uint64_t)td_next->td_self); 1929 DPRINTF("LINK=0x%08llx\n", (long long)td_next->td_self); 1930 } else { 1931 /* this field will get updated later */ 1932 DPRINTF("NOLINK\n"); 1933 } 1934 1935 dword = XHCI_TRB_2_IRQ_SET(0); 1936 1937 td->td_trb[x].dwTrb2 = htole32(dword); 1938 1939 dword = XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK) | 1940 XHCI_TRB_3_CYCLE_BIT | XHCI_TRB_3_IOC_BIT | 1941 /* 1942 * CHAIN-BIT: Ensure that a multi-TRB IN-endpoint 1943 * frame only receives a single short packet event 1944 * by setting the CHAIN bit in the LINK field. In 1945 * addition some XHCI controllers have problems 1946 * sending a ZLP unless the CHAIN-BIT is set in 1947 * the LINK TRB. 1948 */ 1949 XHCI_TRB_3_CHAIN_BIT; 1950 1951 td->td_trb[x].dwTrb3 = htole32(dword); 1952 1953 td->alt_next = td_alt_next; 1954 #ifdef USB_DEBUG 1955 xhci_dump_trb(&td->td_trb[x]); 1956 #endif 1957 usb_pc_cpu_flush(td->page_cache); 1958 } 1959 1960 if (precompute) { 1961 precompute = 0; 1962 1963 /* set up alt next pointer, if any */ 1964 if (temp->last_frame) { 1965 td_alt_next = NULL; 1966 } else { 1967 /* we use this field internally */ 1968 td_alt_next = td_next; 1969 } 1970 1971 /* restore */ 1972 temp->shortpkt = shortpkt_old; 1973 temp->len = len_old; 1974 goto restart; 1975 } 1976 1977 /* 1978 * Remove cycle bit from the first TRB if we are 1979 * stepping them: 1980 */ 1981 if (temp->step_td != 0) { 1982 td_first->td_trb[0].dwTrb3 &= ~htole32(XHCI_TRB_3_CYCLE_BIT); 1983 usb_pc_cpu_flush(td_first->page_cache); 1984 } 1985 1986 /* clear TD SIZE to zero, hence this is the last TRB */ 1987 /* remove chain bit because this is the last data TRB in the chain */ 1988 td->td_trb[td->ntrb - 1].dwTrb2 &= ~htole32(XHCI_TRB_2_TDSZ_SET(15)); 1989 td->td_trb[td->ntrb - 1].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT); 1990 /* remove CHAIN-BIT from last LINK TRB */ 1991 td->td_trb[td->ntrb].dwTrb3 &= ~htole32(XHCI_TRB_3_CHAIN_BIT); 1992 1993 usb_pc_cpu_flush(td->page_cache); 1994 1995 temp->td = td; 1996 temp->td_next = td_next; 1997 } 1998 1999 static void 2000 xhci_setup_generic_chain(struct usb_xfer *xfer) 2001 { 2002 struct xhci_std_temp temp; 2003 struct xhci_td *td; 2004 uint32_t x; 2005 uint32_t y; 2006 uint8_t mult; 2007 2008 temp.do_isoc_sync = 0; 2009 temp.step_td = 0; 2010 temp.tbc = 0; 2011 temp.tlbpc = 0; 2012 temp.average = xfer->max_hc_frame_size; 2013 temp.max_packet_size = xfer->max_packet_size; 2014 temp.sc = XHCI_BUS2SC(xfer->xroot->bus); 2015 temp.pc = NULL; 2016 temp.last_frame = 0; 2017 temp.offset = 0; 2018 temp.multishort = xfer->flags_int.isochronous_xfr || 2019 xfer->flags_int.control_xfr || 2020 xfer->flags_int.short_frames_ok; 2021 2022 /* toggle the DMA set we are using */ 2023 xfer->flags_int.curr_dma_set ^= 1; 2024 2025 /* get next DMA set */ 2026 td = xfer->td_start[xfer->flags_int.curr_dma_set]; 2027 2028 temp.td = NULL; 2029 temp.td_next = td; 2030 2031 xfer->td_transfer_first = td; 2032 xfer->td_transfer_cache = td; 2033 2034 if (xfer->flags_int.isochronous_xfr) { 2035 uint8_t shift; 2036 2037 /* compute multiplier for ISOCHRONOUS transfers */ 2038 mult = xfer->endpoint->ecomp ? 2039 UE_GET_SS_ISO_MULT(xfer->endpoint->ecomp->bmAttributes) 2040 : 0; 2041 /* check for USB 2.0 multiplier */ 2042 if (mult == 0) { 2043 mult = (xfer->endpoint->edesc-> 2044 wMaxPacketSize[1] >> 3) & 3; 2045 } 2046 /* range check */ 2047 if (mult > 2) 2048 mult = 3; 2049 else 2050 mult++; 2051 2052 x = XREAD4(temp.sc, runt, XHCI_MFINDEX); 2053 2054 DPRINTF("MFINDEX=0x%08x\n", x); 2055 2056 switch (usbd_get_speed(xfer->xroot->udev)) { 2057 case USB_SPEED_FULL: 2058 shift = 3; 2059 temp.isoc_delta = 8; /* 1ms */ 2060 x += temp.isoc_delta - 1; 2061 x &= ~(temp.isoc_delta - 1); 2062 break; 2063 default: 2064 shift = usbd_xfer_get_fps_shift(xfer); 2065 temp.isoc_delta = 1U << shift; 2066 x += temp.isoc_delta - 1; 2067 x &= ~(temp.isoc_delta - 1); 2068 /* simple frame load balancing */ 2069 x += xfer->endpoint->usb_uframe; 2070 break; 2071 } 2072 2073 y = XHCI_MFINDEX_GET(x - xfer->endpoint->isoc_next); 2074 2075 if ((xfer->endpoint->is_synced == 0) || 2076 (y < (xfer->nframes << shift)) || 2077 (XHCI_MFINDEX_GET(-y) >= (128 * 8))) { 2078 /* 2079 * If there is data underflow or the pipe 2080 * queue is empty we schedule the transfer a 2081 * few frames ahead of the current frame 2082 * position. Else two isochronous transfers 2083 * might overlap. 2084 */ 2085 xfer->endpoint->isoc_next = XHCI_MFINDEX_GET(x + (3 * 8)); 2086 xfer->endpoint->is_synced = 1; 2087 temp.do_isoc_sync = 1; 2088 2089 DPRINTFN(3, "start next=%d\n", xfer->endpoint->isoc_next); 2090 } 2091 2092 /* compute isochronous completion time */ 2093 2094 y = XHCI_MFINDEX_GET(xfer->endpoint->isoc_next - (x & ~7)); 2095 2096 xfer->isoc_time_complete = 2097 usb_isoc_time_expand(&temp.sc->sc_bus, x / 8) + 2098 (y / 8) + (((xfer->nframes << shift) + 7) / 8); 2099 2100 x = 0; 2101 temp.isoc_frame = xfer->endpoint->isoc_next; 2102 temp.trb_type = XHCI_TRB_TYPE_ISOCH; 2103 2104 xfer->endpoint->isoc_next += xfer->nframes << shift; 2105 2106 } else if (xfer->flags_int.control_xfr) { 2107 2108 /* check if we should prepend a setup message */ 2109 2110 if (xfer->flags_int.control_hdr) { 2111 2112 temp.len = xfer->frlengths[0]; 2113 temp.pc = xfer->frbuffers + 0; 2114 temp.shortpkt = temp.len ? 1 : 0; 2115 temp.trb_type = XHCI_TRB_TYPE_SETUP_STAGE; 2116 temp.direction = 0; 2117 2118 /* check for last frame */ 2119 if (xfer->nframes == 1) { 2120 /* no STATUS stage yet, SETUP is last */ 2121 if (xfer->flags_int.control_act) 2122 temp.last_frame = 1; 2123 } 2124 2125 xhci_setup_generic_chain_sub(&temp); 2126 } 2127 x = 1; 2128 mult = 1; 2129 temp.isoc_delta = 0; 2130 temp.isoc_frame = 0; 2131 temp.trb_type = xfer->flags_int.control_did_data ? 2132 XHCI_TRB_TYPE_NORMAL : XHCI_TRB_TYPE_DATA_STAGE; 2133 } else { 2134 x = 0; 2135 mult = 1; 2136 temp.isoc_delta = 0; 2137 temp.isoc_frame = 0; 2138 temp.trb_type = XHCI_TRB_TYPE_NORMAL; 2139 } 2140 2141 if (x != xfer->nframes) { 2142 /* set up page_cache pointer */ 2143 temp.pc = xfer->frbuffers + x; 2144 /* set endpoint direction */ 2145 temp.direction = UE_GET_DIR(xfer->endpointno); 2146 } 2147 2148 while (x != xfer->nframes) { 2149 2150 /* DATA0 / DATA1 message */ 2151 2152 temp.len = xfer->frlengths[x]; 2153 temp.step_td = ((xfer->endpointno & UE_DIR_IN) && 2154 x != 0 && temp.multishort == 0); 2155 2156 x++; 2157 2158 if (x == xfer->nframes) { 2159 if (xfer->flags_int.control_xfr) { 2160 /* no STATUS stage yet, DATA is last */ 2161 if (xfer->flags_int.control_act) 2162 temp.last_frame = 1; 2163 } else { 2164 temp.last_frame = 1; 2165 } 2166 } 2167 if (temp.len == 0) { 2168 2169 /* make sure that we send an USB packet */ 2170 2171 temp.shortpkt = 0; 2172 2173 temp.tbc = 0; 2174 temp.tlbpc = mult - 1; 2175 2176 } else if (xfer->flags_int.isochronous_xfr) { 2177 2178 uint8_t tdpc; 2179 2180 /* 2181 * Isochronous transfers don't have short 2182 * packet termination: 2183 */ 2184 2185 temp.shortpkt = 1; 2186 2187 /* isochronous transfers have a transfer limit */ 2188 2189 if (temp.len > xfer->max_frame_size) 2190 temp.len = xfer->max_frame_size; 2191 2192 /* compute TD packet count */ 2193 tdpc = (temp.len + xfer->max_packet_size - 1) / 2194 xfer->max_packet_size; 2195 2196 temp.tbc = ((tdpc + mult - 1) / mult) - 1; 2197 temp.tlbpc = (tdpc % mult); 2198 2199 if (temp.tlbpc == 0) 2200 temp.tlbpc = mult - 1; 2201 else 2202 temp.tlbpc--; 2203 } else { 2204 2205 /* regular data transfer */ 2206 2207 temp.shortpkt = xfer->flags.force_short_xfer ? 0 : 1; 2208 } 2209 2210 xhci_setup_generic_chain_sub(&temp); 2211 2212 if (xfer->flags_int.isochronous_xfr) { 2213 temp.offset += xfer->frlengths[x - 1]; 2214 temp.isoc_frame += temp.isoc_delta; 2215 } else { 2216 /* get next Page Cache pointer */ 2217 temp.pc = xfer->frbuffers + x; 2218 } 2219 } 2220 2221 /* check if we should append a status stage */ 2222 2223 if (xfer->flags_int.control_xfr && 2224 !xfer->flags_int.control_act) { 2225 2226 /* 2227 * Send a DATA1 message and invert the current 2228 * endpoint direction. 2229 */ 2230 temp.step_td = (xfer->nframes != 0); 2231 temp.direction = UE_GET_DIR(xfer->endpointno) ^ UE_DIR_IN; 2232 temp.len = 0; 2233 temp.pc = NULL; 2234 temp.shortpkt = 0; 2235 temp.last_frame = 1; 2236 temp.trb_type = XHCI_TRB_TYPE_STATUS_STAGE; 2237 2238 xhci_setup_generic_chain_sub(&temp); 2239 } 2240 2241 td = temp.td; 2242 2243 /* must have at least one frame! */ 2244 2245 xfer->td_transfer_last = td; 2246 2247 DPRINTF("first=%p last=%p\n", xfer->td_transfer_first, td); 2248 } 2249 2250 static void 2251 xhci_set_slot_pointer(struct xhci_softc *sc, uint8_t index, uint64_t dev_addr) 2252 { 2253 struct usb_page_search buf_res; 2254 struct xhci_dev_ctx_addr *pdctxa; 2255 2256 usbd_get_page(&sc->sc_hw.ctx_pc, 0, &buf_res); 2257 2258 pdctxa = buf_res.buffer; 2259 2260 DPRINTF("addr[%u]=0x%016llx\n", index, (long long)dev_addr); 2261 2262 pdctxa->qwBaaDevCtxAddr[index] = htole64(dev_addr); 2263 2264 usb_pc_cpu_flush(&sc->sc_hw.ctx_pc); 2265 } 2266 2267 static usb_error_t 2268 xhci_configure_mask(struct usb_device *udev, uint32_t mask, uint8_t drop) 2269 { 2270 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2271 struct usb_page_search buf_inp; 2272 struct xhci_input_dev_ctx *pinp; 2273 uint32_t temp; 2274 uint8_t index; 2275 uint8_t x; 2276 2277 index = udev->controller_slot_id; 2278 2279 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2280 2281 pinp = buf_inp.buffer; 2282 2283 if (drop) { 2284 mask &= XHCI_INCTX_NON_CTRL_MASK; 2285 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, mask); 2286 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, 0); 2287 } else { 2288 /* 2289 * Some hardware requires that we drop the endpoint 2290 * context before adding it again: 2291 */ 2292 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx0, 2293 mask & XHCI_INCTX_NON_CTRL_MASK); 2294 2295 /* Add new endpoint context */ 2296 xhci_ctx_set_le32(sc, &pinp->ctx_input.dwInCtx1, mask); 2297 2298 /* find most significant set bit */ 2299 for (x = 31; x != 1; x--) { 2300 if (mask & (1 << x)) 2301 break; 2302 } 2303 2304 /* adjust */ 2305 x--; 2306 2307 /* figure out the maximum number of contexts */ 2308 if (x > sc->sc_hw.devs[index].context_num) 2309 sc->sc_hw.devs[index].context_num = x; 2310 else 2311 x = sc->sc_hw.devs[index].context_num; 2312 2313 /* update number of contexts */ 2314 temp = xhci_ctx_get_le32(sc, &pinp->ctx_slot.dwSctx0); 2315 temp &= ~XHCI_SCTX_0_CTX_NUM_SET(31); 2316 temp |= XHCI_SCTX_0_CTX_NUM_SET(x + 1); 2317 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2318 } 2319 usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc); 2320 return (0); 2321 } 2322 2323 static usb_error_t 2324 xhci_configure_endpoint(struct usb_device *udev, 2325 struct usb_endpoint_descriptor *edesc, struct xhci_endpoint_ext *pepext, 2326 uint16_t interval, uint8_t max_packet_count, 2327 uint8_t mult, uint8_t fps_shift, uint16_t max_packet_size, 2328 uint16_t max_frame_size, uint8_t ep_mode) 2329 { 2330 struct usb_page_search buf_inp; 2331 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2332 struct xhci_input_dev_ctx *pinp; 2333 uint64_t ring_addr = pepext->physaddr; 2334 uint32_t temp; 2335 uint8_t index; 2336 uint8_t epno; 2337 uint8_t type; 2338 2339 index = udev->controller_slot_id; 2340 2341 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 2342 2343 pinp = buf_inp.buffer; 2344 2345 epno = edesc->bEndpointAddress; 2346 type = edesc->bmAttributes & UE_XFERTYPE; 2347 2348 if (type == UE_CONTROL) 2349 epno |= UE_DIR_IN; 2350 2351 epno = XHCI_EPNO2EPID(epno); 2352 2353 if (epno == 0) 2354 return (USB_ERR_NO_PIPE); /* invalid */ 2355 2356 if (max_packet_count == 0) 2357 return (USB_ERR_BAD_BUFSIZE); 2358 2359 max_packet_count--; 2360 2361 if (mult == 0) 2362 return (USB_ERR_BAD_BUFSIZE); 2363 2364 /* store endpoint mode */ 2365 pepext->trb_ep_mode = ep_mode; 2366 usb_pc_cpu_flush(pepext->page_cache); 2367 2368 if (ep_mode == USB_EP_MODE_STREAMS) { 2369 temp = XHCI_EPCTX_0_EPSTATE_SET(0) | 2370 XHCI_EPCTX_0_MAXP_STREAMS_SET(XHCI_MAX_STREAMS_LOG - 1) | 2371 XHCI_EPCTX_0_LSA_SET(1); 2372 2373 ring_addr += sizeof(struct xhci_trb) * 2374 XHCI_MAX_TRANSFERS * XHCI_MAX_STREAMS; 2375 } else { 2376 temp = XHCI_EPCTX_0_EPSTATE_SET(0) | 2377 XHCI_EPCTX_0_MAXP_STREAMS_SET(0) | 2378 XHCI_EPCTX_0_LSA_SET(0); 2379 2380 ring_addr |= XHCI_EPCTX_2_DCS_SET(1); 2381 } 2382 2383 switch (udev->speed) { 2384 case USB_SPEED_FULL: 2385 case USB_SPEED_LOW: 2386 /* 1ms -> 125us */ 2387 fps_shift += 3; 2388 break; 2389 default: 2390 break; 2391 } 2392 2393 switch (type) { 2394 case UE_INTERRUPT: 2395 if (fps_shift > 3) 2396 fps_shift--; 2397 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2398 break; 2399 case UE_ISOCHRONOUS: 2400 temp |= XHCI_EPCTX_0_IVAL_SET(fps_shift); 2401 2402 switch (udev->speed) { 2403 case USB_SPEED_SUPER: 2404 if (mult > 3) 2405 mult = 3; 2406 temp |= XHCI_EPCTX_0_MULT_SET(mult - 1); 2407 max_packet_count /= mult; 2408 break; 2409 default: 2410 break; 2411 } 2412 break; 2413 default: 2414 break; 2415 } 2416 2417 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx0, temp); 2418 2419 temp = 2420 XHCI_EPCTX_1_HID_SET(0) | 2421 XHCI_EPCTX_1_MAXB_SET(max_packet_count) | 2422 XHCI_EPCTX_1_MAXP_SIZE_SET(max_packet_size); 2423 2424 /* 2425 * Always enable the "three strikes and you are gone" feature 2426 * except for ISOCHRONOUS endpoints. This is suggested by 2427 * section 4.3.3 in the XHCI specification about device slot 2428 * initialisation. 2429 */ 2430 if (type != UE_ISOCHRONOUS) 2431 temp |= XHCI_EPCTX_1_CERR_SET(3); 2432 2433 switch (type) { 2434 case UE_CONTROL: 2435 temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2436 break; 2437 case UE_ISOCHRONOUS: 2438 temp |= XHCI_EPCTX_1_EPTYPE_SET(1); 2439 break; 2440 case UE_BULK: 2441 temp |= XHCI_EPCTX_1_EPTYPE_SET(2); 2442 break; 2443 default: 2444 temp |= XHCI_EPCTX_1_EPTYPE_SET(3); 2445 break; 2446 } 2447 2448 /* check for IN direction */ 2449 if (epno & 1) 2450 temp |= XHCI_EPCTX_1_EPTYPE_SET(4); 2451 2452 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx1, temp); 2453 xhci_ctx_set_le64(sc, &pinp->ctx_ep[epno - 1].qwEpCtx2, ring_addr); 2454 2455 switch (edesc->bmAttributes & UE_XFERTYPE) { 2456 case UE_INTERRUPT: 2457 case UE_ISOCHRONOUS: 2458 temp = XHCI_EPCTX_4_MAX_ESIT_PAYLOAD_SET(max_frame_size) | 2459 XHCI_EPCTX_4_AVG_TRB_LEN_SET(MIN(XHCI_PAGE_SIZE, 2460 max_frame_size)); 2461 break; 2462 case UE_CONTROL: 2463 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(8); 2464 break; 2465 default: 2466 temp = XHCI_EPCTX_4_AVG_TRB_LEN_SET(XHCI_PAGE_SIZE); 2467 break; 2468 } 2469 2470 xhci_ctx_set_le32(sc, &pinp->ctx_ep[epno - 1].dwEpCtx4, temp); 2471 2472 #ifdef USB_DEBUG 2473 xhci_dump_endpoint(sc, &pinp->ctx_ep[epno - 1]); 2474 #endif 2475 usb_pc_cpu_flush(&sc->sc_hw.devs[index].input_pc); 2476 2477 return (0); /* success */ 2478 } 2479 2480 static usb_error_t 2481 xhci_configure_endpoint_by_xfer(struct usb_xfer *xfer) 2482 { 2483 struct xhci_endpoint_ext *pepext; 2484 struct usb_endpoint_ss_comp_descriptor *ecomp; 2485 usb_stream_t x; 2486 2487 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2488 xfer->endpoint->edesc); 2489 2490 ecomp = xfer->endpoint->ecomp; 2491 2492 for (x = 0; x != XHCI_MAX_STREAMS; x++) { 2493 uint64_t temp; 2494 2495 /* halt any transfers */ 2496 pepext->trb[x * XHCI_MAX_TRANSFERS].dwTrb3 = 0; 2497 2498 /* compute start of TRB ring for stream "x" */ 2499 temp = pepext->physaddr + 2500 (x * XHCI_MAX_TRANSFERS * sizeof(struct xhci_trb)) + 2501 XHCI_SCTX_0_SCT_SEC_TR_RING; 2502 2503 /* make tree structure */ 2504 pepext->trb[(XHCI_MAX_TRANSFERS * 2505 XHCI_MAX_STREAMS) + x].qwTrb0 = htole64(temp); 2506 2507 /* reserved fields */ 2508 pepext->trb[(XHCI_MAX_TRANSFERS * 2509 XHCI_MAX_STREAMS) + x].dwTrb2 = 0; 2510 pepext->trb[(XHCI_MAX_TRANSFERS * 2511 XHCI_MAX_STREAMS) + x].dwTrb3 = 0; 2512 } 2513 usb_pc_cpu_flush(pepext->page_cache); 2514 2515 return (xhci_configure_endpoint(xfer->xroot->udev, 2516 xfer->endpoint->edesc, pepext, 2517 xfer->interval, xfer->max_packet_count, 2518 (ecomp != NULL) ? UE_GET_SS_ISO_MULT(ecomp->bmAttributes) + 1 : 1, 2519 usbd_xfer_get_fps_shift(xfer), xfer->max_packet_size, 2520 xfer->max_frame_size, xfer->endpoint->ep_mode)); 2521 } 2522 2523 static usb_error_t 2524 xhci_configure_device(struct usb_device *udev) 2525 { 2526 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2527 struct usb_page_search buf_inp; 2528 struct usb_page_cache *pcinp; 2529 struct xhci_input_dev_ctx *pinp; 2530 struct usb_device *hubdev; 2531 uint32_t temp; 2532 uint32_t route; 2533 uint32_t rh_port; 2534 uint8_t is_hub; 2535 uint8_t index; 2536 uint8_t depth; 2537 2538 index = udev->controller_slot_id; 2539 2540 DPRINTF("index=%u\n", index); 2541 2542 pcinp = &sc->sc_hw.devs[index].input_pc; 2543 2544 usbd_get_page(pcinp, 0, &buf_inp); 2545 2546 pinp = buf_inp.buffer; 2547 2548 rh_port = 0; 2549 route = 0; 2550 2551 /* figure out route string and root HUB port number */ 2552 2553 for (hubdev = udev; hubdev != NULL; hubdev = hubdev->parent_hub) { 2554 2555 if (hubdev->parent_hub == NULL) 2556 break; 2557 2558 depth = hubdev->parent_hub->depth; 2559 2560 /* 2561 * NOTE: HS/FS/LS devices and the SS root HUB can have 2562 * more than 15 ports 2563 */ 2564 2565 rh_port = hubdev->port_no; 2566 2567 if (depth == 0) 2568 break; 2569 2570 if (rh_port > 15) 2571 rh_port = 15; 2572 2573 if (depth < 6) 2574 route |= rh_port << (4 * (depth - 1)); 2575 } 2576 2577 DPRINTF("Route=0x%08x\n", route); 2578 2579 temp = XHCI_SCTX_0_ROUTE_SET(route) | 2580 XHCI_SCTX_0_CTX_NUM_SET( 2581 sc->sc_hw.devs[index].context_num + 1); 2582 2583 switch (udev->speed) { 2584 case USB_SPEED_LOW: 2585 temp |= XHCI_SCTX_0_SPEED_SET(2); 2586 if (udev->parent_hs_hub != NULL && 2587 udev->parent_hs_hub->ddesc.bDeviceProtocol == 2588 UDPROTO_HSHUBMTT) { 2589 DPRINTF("Device inherits MTT\n"); 2590 temp |= XHCI_SCTX_0_MTT_SET(1); 2591 } 2592 break; 2593 case USB_SPEED_HIGH: 2594 temp |= XHCI_SCTX_0_SPEED_SET(3); 2595 if (sc->sc_hw.devs[index].nports != 0 && 2596 udev->ddesc.bDeviceProtocol == UDPROTO_HSHUBMTT) { 2597 DPRINTF("HUB supports MTT\n"); 2598 temp |= XHCI_SCTX_0_MTT_SET(1); 2599 } 2600 break; 2601 case USB_SPEED_FULL: 2602 temp |= XHCI_SCTX_0_SPEED_SET(1); 2603 if (udev->parent_hs_hub != NULL && 2604 udev->parent_hs_hub->ddesc.bDeviceProtocol == 2605 UDPROTO_HSHUBMTT) { 2606 DPRINTF("Device inherits MTT\n"); 2607 temp |= XHCI_SCTX_0_MTT_SET(1); 2608 } 2609 break; 2610 default: 2611 temp |= XHCI_SCTX_0_SPEED_SET(4); 2612 break; 2613 } 2614 2615 is_hub = sc->sc_hw.devs[index].nports != 0 && 2616 (udev->speed == USB_SPEED_SUPER || 2617 udev->speed == USB_SPEED_HIGH); 2618 2619 if (is_hub) { 2620 temp |= XHCI_SCTX_0_HUB_SET(1); 2621 } 2622 2623 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx0, temp); 2624 2625 temp = XHCI_SCTX_1_RH_PORT_SET(rh_port); 2626 2627 if (is_hub) { 2628 temp |= XHCI_SCTX_1_NUM_PORTS_SET( 2629 sc->sc_hw.devs[index].nports); 2630 } 2631 2632 switch (udev->speed) { 2633 case USB_SPEED_SUPER: 2634 switch (sc->sc_hw.devs[index].state) { 2635 case XHCI_ST_ADDRESSED: 2636 case XHCI_ST_CONFIGURED: 2637 /* enable power save */ 2638 temp |= XHCI_SCTX_1_MAX_EL_SET(sc->sc_exit_lat_max); 2639 break; 2640 default: 2641 /* disable power save */ 2642 break; 2643 } 2644 break; 2645 default: 2646 break; 2647 } 2648 2649 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx1, temp); 2650 2651 temp = XHCI_SCTX_2_IRQ_TARGET_SET(0); 2652 2653 if (is_hub) { 2654 temp |= XHCI_SCTX_2_TT_THINK_TIME_SET( 2655 sc->sc_hw.devs[index].tt); 2656 } 2657 2658 hubdev = udev->parent_hs_hub; 2659 2660 /* check if we should activate the transaction translator */ 2661 switch (udev->speed) { 2662 case USB_SPEED_FULL: 2663 case USB_SPEED_LOW: 2664 if (hubdev != NULL) { 2665 temp |= XHCI_SCTX_2_TT_HUB_SID_SET( 2666 hubdev->controller_slot_id); 2667 temp |= XHCI_SCTX_2_TT_PORT_NUM_SET( 2668 udev->hs_port_no); 2669 } 2670 break; 2671 default: 2672 break; 2673 } 2674 2675 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx2, temp); 2676 2677 /* 2678 * These fields should be initialized to zero, according to 2679 * XHCI section 6.2.2 - slot context: 2680 */ 2681 temp = XHCI_SCTX_3_DEV_ADDR_SET(0) | 2682 XHCI_SCTX_3_SLOT_STATE_SET(0); 2683 2684 xhci_ctx_set_le32(sc, &pinp->ctx_slot.dwSctx3, temp); 2685 2686 #ifdef USB_DEBUG 2687 xhci_dump_device(sc, &pinp->ctx_slot); 2688 #endif 2689 usb_pc_cpu_flush(pcinp); 2690 2691 return (0); /* success */ 2692 } 2693 2694 static usb_error_t 2695 xhci_alloc_device_ext(struct usb_device *udev) 2696 { 2697 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2698 struct usb_page_search buf_dev; 2699 struct usb_page_search buf_ep; 2700 struct xhci_trb *trb; 2701 struct usb_page_cache *pc; 2702 struct usb_page *pg; 2703 uint64_t addr; 2704 uint8_t index; 2705 uint8_t i; 2706 2707 index = udev->controller_slot_id; 2708 2709 pc = &sc->sc_hw.devs[index].device_pc; 2710 pg = &sc->sc_hw.devs[index].device_pg; 2711 2712 /* need to initialize the page cache */ 2713 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2714 2715 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2716 (2 * sizeof(struct xhci_dev_ctx)) : 2717 sizeof(struct xhci_dev_ctx), XHCI_PAGE_SIZE)) 2718 goto error; 2719 2720 usbd_get_page(pc, 0, &buf_dev); 2721 2722 pc = &sc->sc_hw.devs[index].input_pc; 2723 pg = &sc->sc_hw.devs[index].input_pg; 2724 2725 /* need to initialize the page cache */ 2726 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2727 2728 if (usb_pc_alloc_mem(pc, pg, sc->sc_ctx_is_64_byte ? 2729 (2 * sizeof(struct xhci_input_dev_ctx)) : 2730 sizeof(struct xhci_input_dev_ctx), XHCI_PAGE_SIZE)) { 2731 goto error; 2732 } 2733 2734 /* initialize all endpoint LINK TRBs */ 2735 2736 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) { 2737 2738 pc = &sc->sc_hw.devs[index].endpoint_pc[i]; 2739 pg = &sc->sc_hw.devs[index].endpoint_pg[i]; 2740 2741 /* need to initialize the page cache */ 2742 pc->tag_parent = sc->sc_bus.dma_parent_tag; 2743 2744 if (usb_pc_alloc_mem(pc, pg, 2745 sizeof(struct xhci_dev_endpoint_trbs), XHCI_TRB_ALIGN)) { 2746 goto error; 2747 } 2748 2749 /* lookup endpoint TRB ring */ 2750 usbd_get_page(pc, 0, &buf_ep); 2751 2752 /* get TRB pointer */ 2753 trb = buf_ep.buffer; 2754 trb += XHCI_MAX_TRANSFERS - 1; 2755 2756 /* get TRB start address */ 2757 addr = buf_ep.physaddr; 2758 2759 /* create LINK TRB */ 2760 trb->qwTrb0 = htole64(addr); 2761 trb->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2762 trb->dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2763 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2764 2765 usb_pc_cpu_flush(pc); 2766 } 2767 2768 xhci_set_slot_pointer(sc, index, buf_dev.physaddr); 2769 2770 return (0); 2771 2772 error: 2773 xhci_free_device_ext(udev); 2774 2775 return (USB_ERR_NOMEM); 2776 } 2777 2778 static void 2779 xhci_free_device_ext(struct usb_device *udev) 2780 { 2781 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2782 uint8_t index; 2783 uint8_t i; 2784 2785 index = udev->controller_slot_id; 2786 xhci_set_slot_pointer(sc, index, 0); 2787 2788 usb_pc_free_mem(&sc->sc_hw.devs[index].device_pc); 2789 usb_pc_free_mem(&sc->sc_hw.devs[index].input_pc); 2790 for (i = 0; i != XHCI_MAX_ENDPOINTS; i++) 2791 usb_pc_free_mem(&sc->sc_hw.devs[index].endpoint_pc[i]); 2792 } 2793 2794 static struct xhci_endpoint_ext * 2795 xhci_get_endpoint_ext(struct usb_device *udev, struct usb_endpoint_descriptor *edesc) 2796 { 2797 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 2798 struct xhci_endpoint_ext *pepext; 2799 struct usb_page_cache *pc; 2800 struct usb_page_search buf_ep; 2801 uint8_t epno; 2802 uint8_t index; 2803 2804 epno = edesc->bEndpointAddress; 2805 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 2806 epno |= UE_DIR_IN; 2807 2808 epno = XHCI_EPNO2EPID(epno); 2809 2810 index = udev->controller_slot_id; 2811 2812 pc = &sc->sc_hw.devs[index].endpoint_pc[epno]; 2813 2814 usbd_get_page(pc, 0, &buf_ep); 2815 2816 pepext = &sc->sc_hw.devs[index].endp[epno]; 2817 pepext->page_cache = pc; 2818 pepext->trb = buf_ep.buffer; 2819 pepext->physaddr = buf_ep.physaddr; 2820 2821 return (pepext); 2822 } 2823 2824 static void 2825 xhci_endpoint_doorbell(struct usb_xfer *xfer) 2826 { 2827 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2828 uint8_t epno; 2829 uint8_t index; 2830 2831 epno = xfer->endpointno; 2832 if (xfer->flags_int.control_xfr) 2833 epno |= UE_DIR_IN; 2834 2835 epno = XHCI_EPNO2EPID(epno); 2836 index = xfer->xroot->udev->controller_slot_id; 2837 2838 if (xfer->xroot->udev->flags.self_suspended == 0) { 2839 XWRITE4(sc, door, XHCI_DOORBELL(index), 2840 epno | XHCI_DB_SID_SET(xfer->stream_id)); 2841 } 2842 } 2843 2844 static void 2845 xhci_transfer_remove(struct usb_xfer *xfer, usb_error_t error) 2846 { 2847 struct xhci_endpoint_ext *pepext; 2848 2849 if (xfer->flags_int.bandwidth_reclaimed) { 2850 xfer->flags_int.bandwidth_reclaimed = 0; 2851 2852 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2853 xfer->endpoint->edesc); 2854 2855 pepext->trb_used[xfer->stream_id]--; 2856 2857 pepext->xfer[xfer->qh_pos] = NULL; 2858 2859 if (error && pepext->trb_running != 0) { 2860 pepext->trb_halted = 1; 2861 pepext->trb_running = 0; 2862 } 2863 } 2864 } 2865 2866 static usb_error_t 2867 xhci_transfer_insert(struct usb_xfer *xfer) 2868 { 2869 struct xhci_td *td_first; 2870 struct xhci_td *td_last; 2871 struct xhci_trb *trb_link; 2872 struct xhci_endpoint_ext *pepext; 2873 uint64_t addr; 2874 usb_stream_t id; 2875 uint8_t i; 2876 uint8_t inext; 2877 uint8_t trb_limit; 2878 2879 DPRINTFN(8, "\n"); 2880 2881 id = xfer->stream_id; 2882 2883 /* check if already inserted */ 2884 if (xfer->flags_int.bandwidth_reclaimed) { 2885 DPRINTFN(8, "Already in schedule\n"); 2886 return (0); 2887 } 2888 2889 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 2890 xfer->endpoint->edesc); 2891 2892 td_first = xfer->td_transfer_first; 2893 td_last = xfer->td_transfer_last; 2894 addr = pepext->physaddr; 2895 2896 switch (xfer->endpoint->edesc->bmAttributes & UE_XFERTYPE) { 2897 case UE_CONTROL: 2898 case UE_INTERRUPT: 2899 /* single buffered */ 2900 trb_limit = 1; 2901 break; 2902 default: 2903 /* multi buffered */ 2904 trb_limit = (XHCI_MAX_TRANSFERS - 2); 2905 break; 2906 } 2907 2908 if (pepext->trb_used[id] >= trb_limit) { 2909 DPRINTFN(8, "Too many TDs queued.\n"); 2910 return (USB_ERR_NOMEM); 2911 } 2912 2913 /* check for stopped condition, after putting transfer on interrupt queue */ 2914 if (pepext->trb_running == 0) { 2915 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 2916 2917 DPRINTFN(8, "Not running\n"); 2918 2919 /* start configuration */ 2920 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus), 2921 &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 2922 return (0); 2923 } 2924 2925 pepext->trb_used[id]++; 2926 2927 /* get current TRB index */ 2928 i = pepext->trb_index[id]; 2929 2930 /* get next TRB index */ 2931 inext = (i + 1); 2932 2933 /* the last entry of the ring is a hardcoded link TRB */ 2934 if (inext >= (XHCI_MAX_TRANSFERS - 1)) 2935 inext = 0; 2936 2937 /* store next TRB index, before stream ID offset is added */ 2938 pepext->trb_index[id] = inext; 2939 2940 /* offset for stream */ 2941 i += id * XHCI_MAX_TRANSFERS; 2942 inext += id * XHCI_MAX_TRANSFERS; 2943 2944 /* compute terminating return address */ 2945 addr += (inext * sizeof(struct xhci_trb)); 2946 2947 /* compute link TRB pointer */ 2948 trb_link = td_last->td_trb + td_last->ntrb; 2949 2950 /* update next pointer of last link TRB */ 2951 trb_link->qwTrb0 = htole64(addr); 2952 trb_link->dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2953 trb_link->dwTrb3 = htole32(XHCI_TRB_3_IOC_BIT | 2954 XHCI_TRB_3_CYCLE_BIT | 2955 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2956 2957 #ifdef USB_DEBUG 2958 xhci_dump_trb(&td_last->td_trb[td_last->ntrb]); 2959 #endif 2960 usb_pc_cpu_flush(td_last->page_cache); 2961 2962 /* write ahead chain end marker */ 2963 2964 pepext->trb[inext].qwTrb0 = 0; 2965 pepext->trb[inext].dwTrb2 = 0; 2966 pepext->trb[inext].dwTrb3 = 0; 2967 2968 /* update next pointer of link TRB */ 2969 2970 pepext->trb[i].qwTrb0 = htole64((uint64_t)td_first->td_self); 2971 pepext->trb[i].dwTrb2 = htole32(XHCI_TRB_2_IRQ_SET(0)); 2972 2973 #ifdef USB_DEBUG 2974 xhci_dump_trb(&pepext->trb[i]); 2975 #endif 2976 usb_pc_cpu_flush(pepext->page_cache); 2977 2978 /* toggle cycle bit which activates the transfer chain */ 2979 2980 pepext->trb[i].dwTrb3 = htole32(XHCI_TRB_3_CYCLE_BIT | 2981 XHCI_TRB_3_TYPE_SET(XHCI_TRB_TYPE_LINK)); 2982 2983 usb_pc_cpu_flush(pepext->page_cache); 2984 2985 DPRINTF("qh_pos = %u\n", i); 2986 2987 pepext->xfer[i] = xfer; 2988 2989 xfer->qh_pos = i; 2990 2991 xfer->flags_int.bandwidth_reclaimed = 1; 2992 2993 xhci_endpoint_doorbell(xfer); 2994 2995 return (0); 2996 } 2997 2998 static void 2999 xhci_root_intr(struct xhci_softc *sc) 3000 { 3001 uint16_t i; 3002 3003 USB_BUS_LOCK_ASSERT(&sc->sc_bus); 3004 3005 /* clear any old interrupt data */ 3006 memset(sc->sc_hub_idata, 0, sizeof(sc->sc_hub_idata)); 3007 3008 for (i = 1; i <= sc->sc_noport; i++) { 3009 /* pick out CHANGE bits from the status register */ 3010 if (XREAD4(sc, oper, XHCI_PORTSC(i)) & ( 3011 XHCI_PS_CSC | XHCI_PS_PEC | 3012 XHCI_PS_OCC | XHCI_PS_WRC | 3013 XHCI_PS_PRC | XHCI_PS_PLC | 3014 XHCI_PS_CEC)) { 3015 sc->sc_hub_idata[i / 8] |= 1 << (i % 8); 3016 DPRINTF("port %d changed\n", i); 3017 } 3018 } 3019 uhub_root_intr(&sc->sc_bus, sc->sc_hub_idata, 3020 sizeof(sc->sc_hub_idata)); 3021 } 3022 3023 /*------------------------------------------------------------------------* 3024 * xhci_device_done - XHCI done handler 3025 * 3026 * NOTE: This function can be called two times in a row on 3027 * the same USB transfer. From close and from interrupt. 3028 *------------------------------------------------------------------------*/ 3029 static void 3030 xhci_device_done(struct usb_xfer *xfer, usb_error_t error) 3031 { 3032 DPRINTFN(2, "xfer=%p, endpoint=%p, error=%d\n", 3033 xfer, xfer->endpoint, error); 3034 3035 /* remove transfer from HW queue */ 3036 xhci_transfer_remove(xfer, error); 3037 3038 /* dequeue transfer and start next transfer */ 3039 usbd_transfer_done(xfer, error); 3040 } 3041 3042 /*------------------------------------------------------------------------* 3043 * XHCI data transfer support (generic type) 3044 *------------------------------------------------------------------------*/ 3045 static void 3046 xhci_device_generic_open(struct usb_xfer *xfer) 3047 { 3048 if (xfer->flags_int.isochronous_xfr) { 3049 switch (xfer->xroot->udev->speed) { 3050 case USB_SPEED_FULL: 3051 break; 3052 default: 3053 usb_hs_bandwidth_alloc(xfer); 3054 break; 3055 } 3056 } 3057 } 3058 3059 static void 3060 xhci_device_generic_close(struct usb_xfer *xfer) 3061 { 3062 DPRINTF("\n"); 3063 3064 xhci_device_done(xfer, USB_ERR_CANCELLED); 3065 3066 if (xfer->flags_int.isochronous_xfr) { 3067 switch (xfer->xroot->udev->speed) { 3068 case USB_SPEED_FULL: 3069 break; 3070 default: 3071 usb_hs_bandwidth_free(xfer); 3072 break; 3073 } 3074 } 3075 } 3076 3077 static void 3078 xhci_device_generic_multi_enter(struct usb_endpoint *ep, 3079 usb_stream_t stream_id, struct usb_xfer *enter_xfer) 3080 { 3081 struct usb_xfer *xfer; 3082 3083 /* check if there is a current transfer */ 3084 xfer = ep->endpoint_q[stream_id].curr; 3085 if (xfer == NULL) 3086 return; 3087 3088 /* 3089 * Check if the current transfer is started and then pickup 3090 * the next one, if any. Else wait for next start event due to 3091 * block on failure feature. 3092 */ 3093 if (!xfer->flags_int.bandwidth_reclaimed) 3094 return; 3095 3096 xfer = TAILQ_FIRST(&ep->endpoint_q[stream_id].head); 3097 if (xfer == NULL) { 3098 /* 3099 * In case of enter we have to consider that the 3100 * transfer is queued by the USB core after the enter 3101 * method is called. 3102 */ 3103 xfer = enter_xfer; 3104 3105 if (xfer == NULL) 3106 return; 3107 } 3108 3109 /* try to multi buffer */ 3110 xhci_transfer_insert(xfer); 3111 } 3112 3113 static void 3114 xhci_device_generic_enter(struct usb_xfer *xfer) 3115 { 3116 DPRINTF("\n"); 3117 3118 /* set up TD's and QH */ 3119 xhci_setup_generic_chain(xfer); 3120 3121 xhci_device_generic_multi_enter(xfer->endpoint, 3122 xfer->stream_id, xfer); 3123 } 3124 3125 static void 3126 xhci_device_generic_start(struct usb_xfer *xfer) 3127 { 3128 DPRINTF("\n"); 3129 3130 /* try to insert xfer on HW queue */ 3131 xhci_transfer_insert(xfer); 3132 3133 /* try to multi buffer */ 3134 xhci_device_generic_multi_enter(xfer->endpoint, 3135 xfer->stream_id, NULL); 3136 3137 /* add transfer last on interrupt queue */ 3138 usbd_transfer_enqueue(&xfer->xroot->bus->intr_q, xfer); 3139 3140 /* start timeout, if any */ 3141 if (xfer->timeout != 0) 3142 usbd_transfer_timeout_ms(xfer, &xhci_timeout, xfer->timeout); 3143 } 3144 3145 static const struct usb_pipe_methods xhci_device_generic_methods = 3146 { 3147 .open = xhci_device_generic_open, 3148 .close = xhci_device_generic_close, 3149 .enter = xhci_device_generic_enter, 3150 .start = xhci_device_generic_start, 3151 }; 3152 3153 /*------------------------------------------------------------------------* 3154 * xhci root HUB support 3155 *------------------------------------------------------------------------* 3156 * Simulate a hardware HUB by handling all the necessary requests. 3157 *------------------------------------------------------------------------*/ 3158 3159 #define HSETW(ptr, val) ptr = { (uint8_t)(val), (uint8_t)((val) >> 8) } 3160 3161 static const 3162 struct usb_device_descriptor xhci_devd = 3163 { 3164 .bLength = sizeof(xhci_devd), 3165 .bDescriptorType = UDESC_DEVICE, /* type */ 3166 HSETW(.bcdUSB, 0x0300), /* USB version */ 3167 .bDeviceClass = UDCLASS_HUB, /* class */ 3168 .bDeviceSubClass = UDSUBCLASS_HUB, /* subclass */ 3169 .bDeviceProtocol = UDPROTO_SSHUB, /* protocol */ 3170 .bMaxPacketSize = 9, /* max packet size */ 3171 HSETW(.idVendor, 0x0000), /* vendor */ 3172 HSETW(.idProduct, 0x0000), /* product */ 3173 HSETW(.bcdDevice, 0x0100), /* device version */ 3174 .iManufacturer = 1, 3175 .iProduct = 2, 3176 .iSerialNumber = 0, 3177 .bNumConfigurations = 1, /* # of configurations */ 3178 }; 3179 3180 static const 3181 struct xhci_bos_desc xhci_bosd = { 3182 .bosd = { 3183 .bLength = sizeof(xhci_bosd.bosd), 3184 .bDescriptorType = UDESC_BOS, 3185 HSETW(.wTotalLength, sizeof(xhci_bosd)), 3186 .bNumDeviceCaps = 3, 3187 }, 3188 .usb2extd = { 3189 .bLength = sizeof(xhci_bosd.usb2extd), 3190 .bDescriptorType = 1, 3191 .bDevCapabilityType = 2, 3192 .bmAttributes[0] = 2, 3193 }, 3194 .usbdcd = { 3195 .bLength = sizeof(xhci_bosd.usbdcd), 3196 .bDescriptorType = UDESC_DEVICE_CAPABILITY, 3197 .bDevCapabilityType = 3, 3198 .bmAttributes = 0, /* XXX */ 3199 HSETW(.wSpeedsSupported, 0x000C), 3200 .bFunctionalitySupport = 8, 3201 .bU1DevExitLat = 255, /* dummy - not used */ 3202 .wU2DevExitLat = { 0x00, 0x08 }, 3203 }, 3204 .cidd = { 3205 .bLength = sizeof(xhci_bosd.cidd), 3206 .bDescriptorType = 1, 3207 .bDevCapabilityType = 4, 3208 .bReserved = 0, 3209 .bContainerID = 0, /* XXX */ 3210 }, 3211 }; 3212 3213 static const 3214 struct xhci_config_desc xhci_confd = { 3215 .confd = { 3216 .bLength = sizeof(xhci_confd.confd), 3217 .bDescriptorType = UDESC_CONFIG, 3218 .wTotalLength[0] = sizeof(xhci_confd), 3219 .bNumInterface = 1, 3220 .bConfigurationValue = 1, 3221 .iConfiguration = 0, 3222 .bmAttributes = UC_SELF_POWERED, 3223 .bMaxPower = 0 /* max power */ 3224 }, 3225 .ifcd = { 3226 .bLength = sizeof(xhci_confd.ifcd), 3227 .bDescriptorType = UDESC_INTERFACE, 3228 .bNumEndpoints = 1, 3229 .bInterfaceClass = UICLASS_HUB, 3230 .bInterfaceSubClass = UISUBCLASS_HUB, 3231 .bInterfaceProtocol = 0, 3232 }, 3233 .endpd = { 3234 .bLength = sizeof(xhci_confd.endpd), 3235 .bDescriptorType = UDESC_ENDPOINT, 3236 .bEndpointAddress = UE_DIR_IN | XHCI_INTR_ENDPT, 3237 .bmAttributes = UE_INTERRUPT, 3238 .wMaxPacketSize[0] = 2, /* max 15 ports */ 3239 .bInterval = 255, 3240 }, 3241 .endpcd = { 3242 .bLength = sizeof(xhci_confd.endpcd), 3243 .bDescriptorType = UDESC_ENDPOINT_SS_COMP, 3244 .bMaxBurst = 0, 3245 .bmAttributes = 0, 3246 }, 3247 }; 3248 3249 static const 3250 struct usb_hub_ss_descriptor xhci_hubd = { 3251 .bLength = sizeof(xhci_hubd), 3252 .bDescriptorType = UDESC_SS_HUB, 3253 }; 3254 3255 static usb_error_t 3256 xhci_roothub_exec(struct usb_device *udev, 3257 struct usb_device_request *req, const void **pptr, uint16_t *plength) 3258 { 3259 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 3260 const char *str_ptr; 3261 const void *ptr; 3262 uint32_t port; 3263 uint32_t v; 3264 uint16_t len; 3265 uint16_t i; 3266 uint16_t value; 3267 uint16_t index; 3268 uint8_t j; 3269 usb_error_t err; 3270 3271 USB_BUS_LOCK_ASSERT(&sc->sc_bus); 3272 3273 /* buffer reset */ 3274 ptr = (const void *)&sc->sc_hub_desc; 3275 len = 0; 3276 err = 0; 3277 3278 value = UGETW(req->wValue); 3279 index = UGETW(req->wIndex); 3280 3281 DPRINTFN(3, "type=0x%02x request=0x%02x wLen=0x%04x " 3282 "wValue=0x%04x wIndex=0x%04x\n", 3283 req->bmRequestType, req->bRequest, 3284 UGETW(req->wLength), value, index); 3285 3286 #define C(x,y) ((x) | ((y) << 8)) 3287 switch (C(req->bRequest, req->bmRequestType)) { 3288 case C(UR_CLEAR_FEATURE, UT_WRITE_DEVICE): 3289 case C(UR_CLEAR_FEATURE, UT_WRITE_INTERFACE): 3290 case C(UR_CLEAR_FEATURE, UT_WRITE_ENDPOINT): 3291 /* 3292 * DEVICE_REMOTE_WAKEUP and ENDPOINT_HALT are no-ops 3293 * for the integrated root hub. 3294 */ 3295 break; 3296 case C(UR_GET_CONFIG, UT_READ_DEVICE): 3297 len = 1; 3298 sc->sc_hub_desc.temp[0] = sc->sc_conf; 3299 break; 3300 case C(UR_GET_DESCRIPTOR, UT_READ_DEVICE): 3301 switch (value >> 8) { 3302 case UDESC_DEVICE: 3303 if ((value & 0xff) != 0) { 3304 err = USB_ERR_IOERROR; 3305 goto done; 3306 } 3307 len = sizeof(xhci_devd); 3308 ptr = (const void *)&xhci_devd; 3309 break; 3310 3311 case UDESC_BOS: 3312 if ((value & 0xff) != 0) { 3313 err = USB_ERR_IOERROR; 3314 goto done; 3315 } 3316 len = sizeof(xhci_bosd); 3317 ptr = (const void *)&xhci_bosd; 3318 break; 3319 3320 case UDESC_CONFIG: 3321 if ((value & 0xff) != 0) { 3322 err = USB_ERR_IOERROR; 3323 goto done; 3324 } 3325 len = sizeof(xhci_confd); 3326 ptr = (const void *)&xhci_confd; 3327 break; 3328 3329 case UDESC_STRING: 3330 switch (value & 0xff) { 3331 case 0: /* Language table */ 3332 str_ptr = "\001"; 3333 break; 3334 3335 case 1: /* Vendor */ 3336 str_ptr = sc->sc_vendor; 3337 break; 3338 3339 case 2: /* Product */ 3340 str_ptr = "XHCI root HUB"; 3341 break; 3342 3343 default: 3344 str_ptr = ""; 3345 break; 3346 } 3347 3348 len = usb_make_str_desc( 3349 sc->sc_hub_desc.temp, 3350 sizeof(sc->sc_hub_desc.temp), 3351 str_ptr); 3352 break; 3353 3354 default: 3355 err = USB_ERR_IOERROR; 3356 goto done; 3357 } 3358 break; 3359 case C(UR_GET_INTERFACE, UT_READ_INTERFACE): 3360 len = 1; 3361 sc->sc_hub_desc.temp[0] = 0; 3362 break; 3363 case C(UR_GET_STATUS, UT_READ_DEVICE): 3364 len = 2; 3365 USETW(sc->sc_hub_desc.stat.wStatus, UDS_SELF_POWERED); 3366 break; 3367 case C(UR_GET_STATUS, UT_READ_INTERFACE): 3368 case C(UR_GET_STATUS, UT_READ_ENDPOINT): 3369 len = 2; 3370 USETW(sc->sc_hub_desc.stat.wStatus, 0); 3371 break; 3372 case C(UR_SET_ADDRESS, UT_WRITE_DEVICE): 3373 if (value >= XHCI_MAX_DEVICES) { 3374 err = USB_ERR_IOERROR; 3375 goto done; 3376 } 3377 break; 3378 case C(UR_SET_CONFIG, UT_WRITE_DEVICE): 3379 if (value != 0 && value != 1) { 3380 err = USB_ERR_IOERROR; 3381 goto done; 3382 } 3383 sc->sc_conf = value; 3384 break; 3385 case C(UR_SET_DESCRIPTOR, UT_WRITE_DEVICE): 3386 break; 3387 case C(UR_SET_FEATURE, UT_WRITE_DEVICE): 3388 case C(UR_SET_FEATURE, UT_WRITE_INTERFACE): 3389 case C(UR_SET_FEATURE, UT_WRITE_ENDPOINT): 3390 err = USB_ERR_IOERROR; 3391 goto done; 3392 case C(UR_SET_INTERFACE, UT_WRITE_INTERFACE): 3393 break; 3394 case C(UR_SYNCH_FRAME, UT_WRITE_ENDPOINT): 3395 break; 3396 /* Hub requests */ 3397 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_DEVICE): 3398 break; 3399 case C(UR_CLEAR_FEATURE, UT_WRITE_CLASS_OTHER): 3400 DPRINTFN(9, "UR_CLEAR_PORT_FEATURE\n"); 3401 3402 if ((index < 1) || 3403 (index > sc->sc_noport)) { 3404 err = USB_ERR_IOERROR; 3405 goto done; 3406 } 3407 port = XHCI_PORTSC(index); 3408 3409 v = XREAD4(sc, oper, port); 3410 i = XHCI_PS_PLS_GET(v); 3411 v &= ~XHCI_PS_CLEAR; 3412 3413 switch (value) { 3414 case UHF_C_BH_PORT_RESET: 3415 XWRITE4(sc, oper, port, v | XHCI_PS_WRC); 3416 break; 3417 case UHF_C_PORT_CONFIG_ERROR: 3418 XWRITE4(sc, oper, port, v | XHCI_PS_CEC); 3419 break; 3420 case UHF_C_PORT_SUSPEND: 3421 case UHF_C_PORT_LINK_STATE: 3422 XWRITE4(sc, oper, port, v | XHCI_PS_PLC); 3423 break; 3424 case UHF_C_PORT_CONNECTION: 3425 XWRITE4(sc, oper, port, v | XHCI_PS_CSC); 3426 break; 3427 case UHF_C_PORT_ENABLE: 3428 XWRITE4(sc, oper, port, v | XHCI_PS_PEC); 3429 break; 3430 case UHF_C_PORT_OVER_CURRENT: 3431 XWRITE4(sc, oper, port, v | XHCI_PS_OCC); 3432 break; 3433 case UHF_C_PORT_RESET: 3434 XWRITE4(sc, oper, port, v | XHCI_PS_PRC); 3435 break; 3436 case UHF_PORT_ENABLE: 3437 XWRITE4(sc, oper, port, v | XHCI_PS_PED); 3438 break; 3439 case UHF_PORT_POWER: 3440 XWRITE4(sc, oper, port, v & ~XHCI_PS_PP); 3441 break; 3442 case UHF_PORT_INDICATOR: 3443 XWRITE4(sc, oper, port, v & ~XHCI_PS_PIC_SET(3)); 3444 break; 3445 case UHF_PORT_SUSPEND: 3446 3447 /* U3 -> U15 */ 3448 if (i == 3) { 3449 XWRITE4(sc, oper, port, v | 3450 XHCI_PS_PLS_SET(0xF) | XHCI_PS_LWS); 3451 } 3452 3453 /* wait 20ms for resume sequence to complete */ 3454 usb_pause_mtx(&sc->sc_bus.bus_lock, hz / 50); 3455 3456 /* U0 */ 3457 XWRITE4(sc, oper, port, v | 3458 XHCI_PS_PLS_SET(0) | XHCI_PS_LWS); 3459 break; 3460 default: 3461 err = USB_ERR_IOERROR; 3462 goto done; 3463 } 3464 break; 3465 3466 case C(UR_GET_DESCRIPTOR, UT_READ_CLASS_DEVICE): 3467 if ((value & 0xff) != 0) { 3468 err = USB_ERR_IOERROR; 3469 goto done; 3470 } 3471 3472 v = XREAD4(sc, capa, XHCI_HCSPARAMS0); 3473 3474 sc->sc_hub_desc.hubd = xhci_hubd; 3475 3476 sc->sc_hub_desc.hubd.bNbrPorts = sc->sc_noport; 3477 3478 if (XHCI_HCS0_PPC(v)) 3479 i = UHD_PWR_INDIVIDUAL; 3480 else 3481 i = UHD_PWR_GANGED; 3482 3483 if (XHCI_HCS0_PIND(v)) 3484 i |= UHD_PORT_IND; 3485 3486 i |= UHD_OC_INDIVIDUAL; 3487 3488 USETW(sc->sc_hub_desc.hubd.wHubCharacteristics, i); 3489 3490 /* see XHCI section 5.4.9: */ 3491 sc->sc_hub_desc.hubd.bPwrOn2PwrGood = 10; 3492 3493 for (j = 1; j <= sc->sc_noport; j++) { 3494 3495 v = XREAD4(sc, oper, XHCI_PORTSC(j)); 3496 if (v & XHCI_PS_DR) { 3497 sc->sc_hub_desc.hubd. 3498 DeviceRemovable[j / 8] |= 1U << (j % 8); 3499 } 3500 } 3501 len = sc->sc_hub_desc.hubd.bLength; 3502 break; 3503 3504 case C(UR_GET_STATUS, UT_READ_CLASS_DEVICE): 3505 len = 16; 3506 memset(sc->sc_hub_desc.temp, 0, 16); 3507 break; 3508 3509 case C(UR_GET_STATUS, UT_READ_CLASS_OTHER): 3510 DPRINTFN(9, "UR_GET_STATUS i=%d\n", index); 3511 3512 if ((index < 1) || 3513 (index > sc->sc_noport)) { 3514 err = USB_ERR_IOERROR; 3515 goto done; 3516 } 3517 3518 v = XREAD4(sc, oper, XHCI_PORTSC(index)); 3519 3520 DPRINTFN(9, "port status=0x%08x\n", v); 3521 3522 i = UPS_PORT_LINK_STATE_SET(XHCI_PS_PLS_GET(v)); 3523 3524 switch (XHCI_PS_SPEED_GET(v)) { 3525 case 3: 3526 i |= UPS_HIGH_SPEED; 3527 break; 3528 case 2: 3529 i |= UPS_LOW_SPEED; 3530 break; 3531 case 1: 3532 /* FULL speed */ 3533 break; 3534 default: 3535 i |= UPS_OTHER_SPEED; 3536 break; 3537 } 3538 3539 if (v & XHCI_PS_CCS) 3540 i |= UPS_CURRENT_CONNECT_STATUS; 3541 if (v & XHCI_PS_PED) 3542 i |= UPS_PORT_ENABLED; 3543 if (v & XHCI_PS_OCA) 3544 i |= UPS_OVERCURRENT_INDICATOR; 3545 if (v & XHCI_PS_PR) 3546 i |= UPS_RESET; 3547 if (v & XHCI_PS_PP) { 3548 /* 3549 * The USB 3.0 RH is using the 3550 * USB 2.0's power bit 3551 */ 3552 i |= UPS_PORT_POWER; 3553 } 3554 USETW(sc->sc_hub_desc.ps.wPortStatus, i); 3555 3556 i = 0; 3557 if (v & XHCI_PS_CSC) 3558 i |= UPS_C_CONNECT_STATUS; 3559 if (v & XHCI_PS_PEC) 3560 i |= UPS_C_PORT_ENABLED; 3561 if (v & XHCI_PS_OCC) 3562 i |= UPS_C_OVERCURRENT_INDICATOR; 3563 if (v & XHCI_PS_WRC) 3564 i |= UPS_C_BH_PORT_RESET; 3565 if (v & XHCI_PS_PRC) 3566 i |= UPS_C_PORT_RESET; 3567 if (v & XHCI_PS_PLC) 3568 i |= UPS_C_PORT_LINK_STATE; 3569 if (v & XHCI_PS_CEC) 3570 i |= UPS_C_PORT_CONFIG_ERROR; 3571 3572 USETW(sc->sc_hub_desc.ps.wPortChange, i); 3573 len = sizeof(sc->sc_hub_desc.ps); 3574 break; 3575 3576 case C(UR_SET_DESCRIPTOR, UT_WRITE_CLASS_DEVICE): 3577 err = USB_ERR_IOERROR; 3578 goto done; 3579 3580 case C(UR_SET_FEATURE, UT_WRITE_CLASS_DEVICE): 3581 break; 3582 3583 case C(UR_SET_FEATURE, UT_WRITE_CLASS_OTHER): 3584 3585 i = index >> 8; 3586 index &= 0x00FF; 3587 3588 if ((index < 1) || 3589 (index > sc->sc_noport)) { 3590 err = USB_ERR_IOERROR; 3591 goto done; 3592 } 3593 3594 port = XHCI_PORTSC(index); 3595 v = XREAD4(sc, oper, port) & ~XHCI_PS_CLEAR; 3596 3597 switch (value) { 3598 case UHF_PORT_U1_TIMEOUT: 3599 if (XHCI_PS_SPEED_GET(v) != 4) { 3600 err = USB_ERR_IOERROR; 3601 goto done; 3602 } 3603 port = XHCI_PORTPMSC(index); 3604 v = XREAD4(sc, oper, port); 3605 v &= ~XHCI_PM3_U1TO_SET(0xFF); 3606 v |= XHCI_PM3_U1TO_SET(i); 3607 XWRITE4(sc, oper, port, v); 3608 break; 3609 case UHF_PORT_U2_TIMEOUT: 3610 if (XHCI_PS_SPEED_GET(v) != 4) { 3611 err = USB_ERR_IOERROR; 3612 goto done; 3613 } 3614 port = XHCI_PORTPMSC(index); 3615 v = XREAD4(sc, oper, port); 3616 v &= ~XHCI_PM3_U2TO_SET(0xFF); 3617 v |= XHCI_PM3_U2TO_SET(i); 3618 XWRITE4(sc, oper, port, v); 3619 break; 3620 case UHF_BH_PORT_RESET: 3621 XWRITE4(sc, oper, port, v | XHCI_PS_WPR); 3622 break; 3623 case UHF_PORT_LINK_STATE: 3624 XWRITE4(sc, oper, port, v | 3625 XHCI_PS_PLS_SET(i) | XHCI_PS_LWS); 3626 /* 4ms settle time */ 3627 usb_pause_mtx(&sc->sc_bus.bus_lock, hz / 250); 3628 break; 3629 case UHF_PORT_ENABLE: 3630 DPRINTFN(3, "set port enable %d\n", index); 3631 break; 3632 case UHF_PORT_SUSPEND: 3633 DPRINTFN(6, "suspend port %u (LPM=%u)\n", index, i); 3634 j = XHCI_PS_SPEED_GET(v); 3635 if ((j < 1) || (j > 3)) { 3636 /* non-supported speed */ 3637 err = USB_ERR_IOERROR; 3638 goto done; 3639 } 3640 XWRITE4(sc, oper, port, v | 3641 XHCI_PS_PLS_SET(i ? 2 /* LPM */ : 3) | XHCI_PS_LWS); 3642 break; 3643 case UHF_PORT_RESET: 3644 DPRINTFN(6, "reset port %d\n", index); 3645 XWRITE4(sc, oper, port, v | XHCI_PS_PR); 3646 break; 3647 case UHF_PORT_POWER: 3648 DPRINTFN(3, "set port power %d\n", index); 3649 XWRITE4(sc, oper, port, v | XHCI_PS_PP); 3650 break; 3651 case UHF_PORT_TEST: 3652 DPRINTFN(3, "set port test %d\n", index); 3653 break; 3654 case UHF_PORT_INDICATOR: 3655 DPRINTFN(3, "set port indicator %d\n", index); 3656 3657 v &= ~XHCI_PS_PIC_SET(3); 3658 v |= XHCI_PS_PIC_SET(1); 3659 3660 XWRITE4(sc, oper, port, v); 3661 break; 3662 default: 3663 err = USB_ERR_IOERROR; 3664 goto done; 3665 } 3666 break; 3667 3668 case C(UR_CLEAR_TT_BUFFER, UT_WRITE_CLASS_OTHER): 3669 case C(UR_RESET_TT, UT_WRITE_CLASS_OTHER): 3670 case C(UR_GET_TT_STATE, UT_READ_CLASS_OTHER): 3671 case C(UR_STOP_TT, UT_WRITE_CLASS_OTHER): 3672 break; 3673 default: 3674 err = USB_ERR_IOERROR; 3675 goto done; 3676 } 3677 done: 3678 *plength = len; 3679 *pptr = ptr; 3680 return (err); 3681 } 3682 3683 static void 3684 xhci_xfer_setup(struct usb_setup_params *parm) 3685 { 3686 struct usb_page_search page_info; 3687 struct usb_page_cache *pc; 3688 struct xhci_softc *sc; 3689 struct usb_xfer *xfer; 3690 void *last_obj; 3691 uint32_t ntd; 3692 uint32_t n; 3693 3694 sc = XHCI_BUS2SC(parm->udev->bus); 3695 xfer = parm->curr_xfer; 3696 3697 /* 3698 * The proof for the "ntd" formula is illustrated like this: 3699 * 3700 * +------------------------------------+ 3701 * | | 3702 * | |remainder -> | 3703 * | +-----+---+ | 3704 * | | xxx | x | frm 0 | 3705 * | +-----+---++ | 3706 * | | xxx | xx | frm 1 | 3707 * | +-----+----+ | 3708 * | ... | 3709 * +------------------------------------+ 3710 * 3711 * "xxx" means a completely full USB transfer descriptor 3712 * 3713 * "x" and "xx" means a short USB packet 3714 * 3715 * For the remainder of an USB transfer modulo 3716 * "max_data_length" we need two USB transfer descriptors. 3717 * One to transfer the remaining data and one to finalise with 3718 * a zero length packet in case the "force_short_xfer" flag is 3719 * set. We only need two USB transfer descriptors in the case 3720 * where the transfer length of the first one is a factor of 3721 * "max_frame_size". The rest of the needed USB transfer 3722 * descriptors is given by the buffer size divided by the 3723 * maximum data payload. 3724 */ 3725 parm->hc_max_packet_size = 0x400; 3726 parm->hc_max_packet_count = 16 * 3; 3727 parm->hc_max_frame_size = XHCI_TD_PAYLOAD_MAX; 3728 3729 xfer->flags_int.bdma_enable = 1; 3730 3731 usbd_transfer_setup_sub(parm); 3732 3733 if (xfer->flags_int.isochronous_xfr) { 3734 ntd = ((1 * xfer->nframes) 3735 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3736 } else if (xfer->flags_int.control_xfr) { 3737 ntd = ((2 * xfer->nframes) + 1 /* STATUS */ 3738 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3739 } else { 3740 ntd = ((2 * xfer->nframes) 3741 + (xfer->max_data_length / xfer->max_hc_frame_size)); 3742 } 3743 3744 alloc_dma_set: 3745 3746 if (parm->err) 3747 return; 3748 3749 /* 3750 * Allocate queue heads and transfer descriptors 3751 */ 3752 last_obj = NULL; 3753 3754 if (usbd_transfer_setup_sub_malloc( 3755 parm, &pc, sizeof(struct xhci_td), 3756 XHCI_TD_ALIGN, ntd)) { 3757 parm->err = USB_ERR_NOMEM; 3758 return; 3759 } 3760 if (parm->buf) { 3761 for (n = 0; n != ntd; n++) { 3762 struct xhci_td *td; 3763 3764 usbd_get_page(pc + n, 0, &page_info); 3765 3766 td = page_info.buffer; 3767 3768 /* init TD */ 3769 td->td_self = page_info.physaddr; 3770 td->obj_next = last_obj; 3771 td->page_cache = pc + n; 3772 3773 last_obj = td; 3774 3775 usb_pc_cpu_flush(pc + n); 3776 } 3777 } 3778 xfer->td_start[xfer->flags_int.curr_dma_set] = last_obj; 3779 3780 if (!xfer->flags_int.curr_dma_set) { 3781 xfer->flags_int.curr_dma_set = 1; 3782 goto alloc_dma_set; 3783 } 3784 } 3785 3786 static usb_error_t 3787 xhci_configure_reset_endpoint(struct usb_xfer *xfer) 3788 { 3789 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3790 struct usb_page_search buf_inp; 3791 struct usb_device *udev; 3792 struct xhci_endpoint_ext *pepext; 3793 struct usb_endpoint_descriptor *edesc; 3794 struct usb_page_cache *pcinp; 3795 usb_error_t err; 3796 usb_stream_t stream_id; 3797 uint8_t index; 3798 uint8_t epno; 3799 3800 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3801 xfer->endpoint->edesc); 3802 3803 udev = xfer->xroot->udev; 3804 index = udev->controller_slot_id; 3805 3806 pcinp = &sc->sc_hw.devs[index].input_pc; 3807 3808 usbd_get_page(pcinp, 0, &buf_inp); 3809 3810 edesc = xfer->endpoint->edesc; 3811 3812 epno = edesc->bEndpointAddress; 3813 stream_id = xfer->stream_id; 3814 3815 if ((edesc->bmAttributes & UE_XFERTYPE) == UE_CONTROL) 3816 epno |= UE_DIR_IN; 3817 3818 epno = XHCI_EPNO2EPID(epno); 3819 3820 if (epno == 0) 3821 return (USB_ERR_NO_PIPE); /* invalid */ 3822 3823 XHCI_CMD_LOCK(sc); 3824 3825 /* configure endpoint */ 3826 3827 err = xhci_configure_endpoint_by_xfer(xfer); 3828 3829 if (err != 0) { 3830 XHCI_CMD_UNLOCK(sc); 3831 return (err); 3832 } 3833 3834 /* 3835 * Get the endpoint into the stopped state according to the 3836 * endpoint context state diagram in the XHCI specification: 3837 */ 3838 3839 err = xhci_cmd_stop_ep(sc, 0, epno, index); 3840 3841 if (err != 0) 3842 DPRINTF("Could not stop endpoint %u\n", epno); 3843 3844 err = xhci_cmd_reset_ep(sc, 0, epno, index); 3845 3846 if (err != 0) 3847 DPRINTF("Could not reset endpoint %u\n", epno); 3848 3849 err = xhci_cmd_set_tr_dequeue_ptr(sc, 3850 (pepext->physaddr + (stream_id * sizeof(struct xhci_trb) * 3851 XHCI_MAX_TRANSFERS)) | XHCI_EPCTX_2_DCS_SET(1), 3852 stream_id, epno, index); 3853 3854 if (err != 0) 3855 DPRINTF("Could not set dequeue ptr for endpoint %u\n", epno); 3856 3857 /* 3858 * Get the endpoint into the running state according to the 3859 * endpoint context state diagram in the XHCI specification: 3860 */ 3861 3862 xhci_configure_mask(udev, (1U << epno) | 1U, 0); 3863 3864 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 3865 3866 if (err != 0) 3867 DPRINTF("Could not configure endpoint %u\n", epno); 3868 3869 err = xhci_cmd_configure_ep(sc, buf_inp.physaddr, 0, index); 3870 3871 if (err != 0) 3872 DPRINTF("Could not configure endpoint %u\n", epno); 3873 3874 XHCI_CMD_UNLOCK(sc); 3875 3876 return (0); 3877 } 3878 3879 static void 3880 xhci_xfer_unsetup(struct usb_xfer *xfer) 3881 { 3882 return; 3883 } 3884 3885 static void 3886 xhci_start_dma_delay(struct usb_xfer *xfer) 3887 { 3888 struct xhci_softc *sc = XHCI_BUS2SC(xfer->xroot->bus); 3889 3890 /* put transfer on interrupt queue (again) */ 3891 usbd_transfer_enqueue(&sc->sc_bus.intr_q, xfer); 3892 3893 (void)usb_proc_msignal(USB_BUS_CONTROL_XFER_PROC(&sc->sc_bus), 3894 &sc->sc_config_msg[0], &sc->sc_config_msg[1]); 3895 } 3896 3897 static void 3898 xhci_configure_msg(struct usb_proc_msg *pm) 3899 { 3900 struct xhci_softc *sc; 3901 struct xhci_endpoint_ext *pepext; 3902 struct usb_xfer *xfer; 3903 3904 sc = XHCI_BUS2SC(((struct usb_bus_msg *)pm)->bus); 3905 3906 restart: 3907 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3908 3909 pepext = xhci_get_endpoint_ext(xfer->xroot->udev, 3910 xfer->endpoint->edesc); 3911 3912 if ((pepext->trb_halted != 0) || 3913 (pepext->trb_running == 0)) { 3914 3915 uint16_t i; 3916 3917 /* clear halted and running */ 3918 pepext->trb_halted = 0; 3919 pepext->trb_running = 0; 3920 3921 /* nuke remaining buffered transfers */ 3922 3923 for (i = 0; i != (XHCI_MAX_TRANSFERS * 3924 XHCI_MAX_STREAMS); i++) { 3925 /* 3926 * NOTE: We need to use the timeout 3927 * error code here else existing 3928 * isochronous clients can get 3929 * confused: 3930 */ 3931 if (pepext->xfer[i] != NULL) { 3932 xhci_device_done(pepext->xfer[i], 3933 USB_ERR_TIMEOUT); 3934 } 3935 } 3936 3937 /* 3938 * NOTE: The USB transfer cannot vanish in 3939 * this state! 3940 */ 3941 3942 USB_BUS_UNLOCK(&sc->sc_bus); 3943 3944 xhci_configure_reset_endpoint(xfer); 3945 3946 USB_BUS_LOCK(&sc->sc_bus); 3947 3948 /* check if halted is still cleared */ 3949 if (pepext->trb_halted == 0) { 3950 pepext->trb_running = 1; 3951 memset(pepext->trb_index, 0, 3952 sizeof(pepext->trb_index)); 3953 } 3954 goto restart; 3955 } 3956 3957 if (xfer->flags_int.did_dma_delay) { 3958 3959 /* remove transfer from interrupt queue (again) */ 3960 usbd_transfer_dequeue(xfer); 3961 3962 /* we are finally done */ 3963 usb_dma_delay_done_cb(xfer); 3964 3965 /* queue changed - restart */ 3966 goto restart; 3967 } 3968 } 3969 3970 TAILQ_FOREACH(xfer, &sc->sc_bus.intr_q.head, wait_entry) { 3971 3972 /* try to insert xfer on HW queue */ 3973 xhci_transfer_insert(xfer); 3974 3975 /* try to multi buffer */ 3976 xhci_device_generic_multi_enter(xfer->endpoint, 3977 xfer->stream_id, NULL); 3978 } 3979 } 3980 3981 static void 3982 xhci_ep_init(struct usb_device *udev, struct usb_endpoint_descriptor *edesc, 3983 struct usb_endpoint *ep) 3984 { 3985 struct xhci_endpoint_ext *pepext; 3986 3987 DPRINTFN(2, "endpoint=%p, addr=%d, endpt=%d, mode=%d\n", 3988 ep, udev->address, edesc->bEndpointAddress, udev->flags.usb_mode); 3989 3990 if (udev->parent_hub == NULL) { 3991 /* root HUB has special endpoint handling */ 3992 return; 3993 } 3994 3995 ep->methods = &xhci_device_generic_methods; 3996 3997 pepext = xhci_get_endpoint_ext(udev, edesc); 3998 3999 USB_BUS_LOCK(udev->bus); 4000 pepext->trb_halted = 1; 4001 pepext->trb_running = 0; 4002 USB_BUS_UNLOCK(udev->bus); 4003 } 4004 4005 static void 4006 xhci_ep_uninit(struct usb_device *udev, struct usb_endpoint *ep) 4007 { 4008 4009 } 4010 4011 static void 4012 xhci_ep_clear_stall(struct usb_device *udev, struct usb_endpoint *ep) 4013 { 4014 struct xhci_endpoint_ext *pepext; 4015 4016 DPRINTF("\n"); 4017 4018 if (udev->flags.usb_mode != USB_MODE_HOST) { 4019 /* not supported */ 4020 return; 4021 } 4022 if (udev->parent_hub == NULL) { 4023 /* root HUB has special endpoint handling */ 4024 return; 4025 } 4026 4027 pepext = xhci_get_endpoint_ext(udev, ep->edesc); 4028 4029 USB_BUS_LOCK(udev->bus); 4030 pepext->trb_halted = 1; 4031 pepext->trb_running = 0; 4032 USB_BUS_UNLOCK(udev->bus); 4033 } 4034 4035 static usb_error_t 4036 xhci_device_init(struct usb_device *udev) 4037 { 4038 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4039 usb_error_t err; 4040 uint8_t temp; 4041 4042 /* no init for root HUB */ 4043 if (udev->parent_hub == NULL) 4044 return (0); 4045 4046 XHCI_CMD_LOCK(sc); 4047 4048 /* set invalid default */ 4049 4050 udev->controller_slot_id = sc->sc_noslot + 1; 4051 4052 /* try to get a new slot ID from the XHCI */ 4053 4054 err = xhci_cmd_enable_slot(sc, &temp); 4055 4056 if (err) { 4057 XHCI_CMD_UNLOCK(sc); 4058 return (err); 4059 } 4060 4061 if (temp > sc->sc_noslot) { 4062 XHCI_CMD_UNLOCK(sc); 4063 return (USB_ERR_BAD_ADDRESS); 4064 } 4065 4066 if (sc->sc_hw.devs[temp].state != XHCI_ST_DISABLED) { 4067 DPRINTF("slot %u already allocated.\n", temp); 4068 XHCI_CMD_UNLOCK(sc); 4069 return (USB_ERR_BAD_ADDRESS); 4070 } 4071 4072 /* store slot ID for later reference */ 4073 4074 udev->controller_slot_id = temp; 4075 4076 /* reset data structure */ 4077 4078 memset(&sc->sc_hw.devs[temp], 0, sizeof(sc->sc_hw.devs[0])); 4079 4080 /* set mark slot allocated */ 4081 4082 sc->sc_hw.devs[temp].state = XHCI_ST_ENABLED; 4083 4084 err = xhci_alloc_device_ext(udev); 4085 4086 XHCI_CMD_UNLOCK(sc); 4087 4088 /* get device into default state */ 4089 4090 if (err == 0) 4091 err = xhci_set_address(udev, NULL, 0); 4092 4093 return (err); 4094 } 4095 4096 static void 4097 xhci_device_uninit(struct usb_device *udev) 4098 { 4099 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4100 uint8_t index; 4101 4102 /* no init for root HUB */ 4103 if (udev->parent_hub == NULL) 4104 return; 4105 4106 XHCI_CMD_LOCK(sc); 4107 4108 index = udev->controller_slot_id; 4109 4110 if (index <= sc->sc_noslot) { 4111 xhci_cmd_disable_slot(sc, index); 4112 sc->sc_hw.devs[index].state = XHCI_ST_DISABLED; 4113 4114 /* free device extension */ 4115 xhci_free_device_ext(udev); 4116 } 4117 4118 XHCI_CMD_UNLOCK(sc); 4119 } 4120 4121 static void 4122 xhci_get_dma_delay(struct usb_device *udev, uint32_t *pus) 4123 { 4124 /* 4125 * Wait until the hardware has finished any possible use of 4126 * the transfer descriptor(s) 4127 */ 4128 *pus = 2048; /* microseconds */ 4129 } 4130 4131 static void 4132 xhci_device_resume(struct usb_device *udev) 4133 { 4134 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4135 uint8_t index; 4136 uint8_t n; 4137 uint8_t p; 4138 4139 DPRINTF("\n"); 4140 4141 /* check for root HUB */ 4142 if (udev->parent_hub == NULL) 4143 return; 4144 4145 index = udev->controller_slot_id; 4146 4147 XHCI_CMD_LOCK(sc); 4148 4149 /* blindly resume all endpoints */ 4150 4151 USB_BUS_LOCK(udev->bus); 4152 4153 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 4154 for (p = 0; p != XHCI_MAX_STREAMS; p++) { 4155 XWRITE4(sc, door, XHCI_DOORBELL(index), 4156 n | XHCI_DB_SID_SET(p)); 4157 } 4158 } 4159 4160 USB_BUS_UNLOCK(udev->bus); 4161 4162 XHCI_CMD_UNLOCK(sc); 4163 } 4164 4165 static void 4166 xhci_device_suspend(struct usb_device *udev) 4167 { 4168 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4169 uint8_t index; 4170 uint8_t n; 4171 usb_error_t err; 4172 4173 DPRINTF("\n"); 4174 4175 /* check for root HUB */ 4176 if (udev->parent_hub == NULL) 4177 return; 4178 4179 index = udev->controller_slot_id; 4180 4181 XHCI_CMD_LOCK(sc); 4182 4183 /* blindly suspend all endpoints */ 4184 4185 for (n = 1; n != XHCI_MAX_ENDPOINTS; n++) { 4186 err = xhci_cmd_stop_ep(sc, 1, n, index); 4187 if (err != 0) { 4188 DPRINTF("Failed to suspend endpoint " 4189 "%u on slot %u (ignored).\n", n, index); 4190 } 4191 } 4192 4193 XHCI_CMD_UNLOCK(sc); 4194 } 4195 4196 static void 4197 xhci_set_hw_power(struct usb_bus *bus) 4198 { 4199 DPRINTF("\n"); 4200 } 4201 4202 static void 4203 xhci_device_state_change(struct usb_device *udev) 4204 { 4205 struct xhci_softc *sc = XHCI_BUS2SC(udev->bus); 4206 struct usb_page_search buf_inp; 4207 usb_error_t err; 4208 uint8_t index; 4209 4210 /* check for root HUB */ 4211 if (udev->parent_hub == NULL) 4212 return; 4213 4214 index = udev->controller_slot_id; 4215 4216 DPRINTF("\n"); 4217 4218 if (usb_get_device_state(udev) == USB_STATE_CONFIGURED) { 4219 err = uhub_query_info(udev, &sc->sc_hw.devs[index].nports, 4220 &sc->sc_hw.devs[index].tt); 4221 if (err != 0) 4222 sc->sc_hw.devs[index].nports = 0; 4223 } 4224 4225 XHCI_CMD_LOCK(sc); 4226 4227 switch (usb_get_device_state(udev)) { 4228 case USB_STATE_POWERED: 4229 if (sc->sc_hw.devs[index].state == XHCI_ST_DEFAULT) 4230 break; 4231 4232 /* set default state */ 4233 sc->sc_hw.devs[index].state = XHCI_ST_DEFAULT; 4234 4235 /* reset number of contexts */ 4236 sc->sc_hw.devs[index].context_num = 0; 4237 4238 err = xhci_cmd_reset_dev(sc, index); 4239 4240 if (err != 0) { 4241 DPRINTF("Device reset failed " 4242 "for slot %u.\n", index); 4243 } 4244 break; 4245 4246 case USB_STATE_ADDRESSED: 4247 if (sc->sc_hw.devs[index].state == XHCI_ST_ADDRESSED) 4248 break; 4249 4250 sc->sc_hw.devs[index].state = XHCI_ST_ADDRESSED; 4251 4252 err = xhci_cmd_configure_ep(sc, 0, 1, index); 4253 4254 if (err) { 4255 DPRINTF("Failed to deconfigure " 4256 "slot %u.\n", index); 4257 } 4258 break; 4259 4260 case USB_STATE_CONFIGURED: 4261 if (sc->sc_hw.devs[index].state == XHCI_ST_CONFIGURED) 4262 break; 4263 4264 /* set configured state */ 4265 sc->sc_hw.devs[index].state = XHCI_ST_CONFIGURED; 4266 4267 /* reset number of contexts */ 4268 sc->sc_hw.devs[index].context_num = 0; 4269 4270 usbd_get_page(&sc->sc_hw.devs[index].input_pc, 0, &buf_inp); 4271 4272 xhci_configure_mask(udev, 3, 0); 4273 4274 err = xhci_configure_device(udev); 4275 if (err != 0) { 4276 DPRINTF("Could not configure device " 4277 "at slot %u.\n", index); 4278 } 4279 4280 err = xhci_cmd_evaluate_ctx(sc, buf_inp.physaddr, index); 4281 if (err != 0) { 4282 DPRINTF("Could not evaluate device " 4283 "context at slot %u.\n", index); 4284 } 4285 break; 4286 4287 default: 4288 break; 4289 } 4290 XHCI_CMD_UNLOCK(sc); 4291 } 4292 4293 static usb_error_t 4294 xhci_set_endpoint_mode(struct usb_device *udev, struct usb_endpoint *ep, 4295 uint8_t ep_mode) 4296 { 4297 switch (ep_mode) { 4298 case USB_EP_MODE_DEFAULT: 4299 return (0); 4300 case USB_EP_MODE_STREAMS: 4301 if (xhcistreams == 0 || 4302 (ep->edesc->bmAttributes & UE_XFERTYPE) != UE_BULK || 4303 udev->speed != USB_SPEED_SUPER) 4304 return (USB_ERR_INVAL); 4305 return (0); 4306 default: 4307 return (USB_ERR_INVAL); 4308 } 4309 } 4310 4311 static const struct usb_bus_methods xhci_bus_methods = { 4312 .endpoint_init = xhci_ep_init, 4313 .endpoint_uninit = xhci_ep_uninit, 4314 .xfer_setup = xhci_xfer_setup, 4315 .xfer_unsetup = xhci_xfer_unsetup, 4316 .get_dma_delay = xhci_get_dma_delay, 4317 .device_init = xhci_device_init, 4318 .device_uninit = xhci_device_uninit, 4319 .device_resume = xhci_device_resume, 4320 .device_suspend = xhci_device_suspend, 4321 .set_hw_power = xhci_set_hw_power, 4322 .roothub_exec = xhci_roothub_exec, 4323 .xfer_poll = xhci_do_poll, 4324 .start_dma_delay = xhci_start_dma_delay, 4325 .set_address = xhci_set_address, 4326 .clear_stall = xhci_ep_clear_stall, 4327 .device_state_change = xhci_device_state_change, 4328 .set_hw_power_sleep = xhci_set_hw_power_sleep, 4329 .set_endpoint_mode = xhci_set_endpoint_mode, 4330 }; 4331