1 /* $FreeBSD$ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/stdint.h> 28 #include <sys/param.h> 29 #include <sys/queue.h> 30 #include <sys/types.h> 31 #include <sys/systm.h> 32 #include <sys/kernel.h> 33 #include <sys/bus.h> 34 #include <sys/module.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/mutex2.h> 38 #include <sys/condvar.h> 39 #include <sys/sysctl.h> 40 #include <sys/unistd.h> 41 #include <sys/callout.h> 42 #include <sys/malloc.h> 43 #include <sys/priv.h> 44 45 #include <bus/u4b/usb.h> 46 #include <bus/u4b/usbdi.h> 47 #include <bus/u4b/usbdi_util.h> 48 49 #define USB_DEBUG_VAR usb_debug 50 51 #include <bus/u4b/usb_core.h> 52 #include <bus/u4b/usb_busdma.h> 53 #include <bus/u4b/usb_process.h> 54 #include <bus/u4b/usb_transfer.h> 55 #include <bus/u4b/usb_device.h> 56 #include <bus/u4b/usb_util.h> 57 #include <bus/u4b/usb_debug.h> 58 59 #include <bus/u4b/usb_controller.h> 60 #include <bus/u4b/usb_bus.h> 61 62 #if USB_HAVE_BUSDMA 63 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 64 static void usb_dma_tag_destroy(struct usb_dma_tag *); 65 #if 0 66 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 67 #endif 68 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 69 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 70 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 71 uint8_t); 72 #endif 73 74 /*------------------------------------------------------------------------* 75 * usbd_get_page - lookup DMA-able memory for the given offset 76 * 77 * NOTE: Only call this function when the "page_cache" structure has 78 * been properly initialized ! 79 *------------------------------------------------------------------------*/ 80 void 81 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 82 struct usb_page_search *res) 83 { 84 #if USB_HAVE_BUSDMA 85 struct usb_page *page; 86 87 if (pc->page_start) { 88 89 /* Case 1 - something has been loaded into DMA */ 90 91 if (pc->buffer) { 92 93 /* Case 1a - Kernel Virtual Address */ 94 95 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 96 } 97 offset += pc->page_offset_buf; 98 99 /* compute destination page */ 100 101 page = pc->page_start; 102 103 if (pc->ismultiseg) { 104 105 page += (offset / USB_PAGE_SIZE); 106 107 offset %= USB_PAGE_SIZE; 108 109 res->length = USB_PAGE_SIZE - offset; 110 res->physaddr = page->physaddr + offset; 111 } else { 112 res->length = 0 - 1; 113 res->physaddr = page->physaddr + offset; 114 } 115 if (!pc->buffer) { 116 117 /* Case 1b - Non Kernel Virtual Address */ 118 119 res->buffer = USB_ADD_BYTES(page->buffer, offset); 120 } 121 return; 122 } 123 #endif 124 /* Case 2 - Plain PIO */ 125 126 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 127 res->length = 0 - 1; 128 #if USB_HAVE_BUSDMA 129 res->physaddr = 0; 130 #endif 131 } 132 133 /*------------------------------------------------------------------------* 134 * usbd_copy_in - copy directly to DMA-able memory 135 *------------------------------------------------------------------------*/ 136 void 137 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 138 const void *ptr, usb_frlength_t len) 139 { 140 struct usb_page_search buf_res; 141 142 while (len != 0) { 143 144 usbd_get_page(cache, offset, &buf_res); 145 146 if (buf_res.length > len) { 147 buf_res.length = len; 148 } 149 memcpy(buf_res.buffer, ptr, buf_res.length); 150 151 offset += buf_res.length; 152 len -= buf_res.length; 153 ptr = USB_ADD_BYTES(ptr, buf_res.length); 154 } 155 } 156 157 /*------------------------------------------------------------------------* 158 * usbd_copy_in_user - copy directly to DMA-able memory from userland 159 * 160 * Return values: 161 * 0: Success 162 * Else: Failure 163 *------------------------------------------------------------------------*/ 164 #if USB_HAVE_USER_IO 165 int 166 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 167 const void *ptr, usb_frlength_t len) 168 { 169 struct usb_page_search buf_res; 170 int error; 171 172 while (len != 0) { 173 174 usbd_get_page(cache, offset, &buf_res); 175 176 if (buf_res.length > len) { 177 buf_res.length = len; 178 } 179 error = copyin(ptr, buf_res.buffer, buf_res.length); 180 if (error) 181 return (error); 182 183 offset += buf_res.length; 184 len -= buf_res.length; 185 ptr = USB_ADD_BYTES(ptr, buf_res.length); 186 } 187 return (0); /* success */ 188 } 189 #endif 190 191 /*------------------------------------------------------------------------* 192 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 193 *------------------------------------------------------------------------*/ 194 #if USB_HAVE_MBUF 195 struct usb_m_copy_in_arg { 196 struct usb_page_cache *cache; 197 usb_frlength_t dst_offset; 198 }; 199 200 static int 201 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 202 { 203 register struct usb_m_copy_in_arg *ua = arg; 204 205 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 206 ua->dst_offset += count; 207 return (0); 208 } 209 210 void 211 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 212 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 213 { 214 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 215 int error; 216 217 error = m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 218 } 219 #endif 220 221 /*------------------------------------------------------------------------* 222 * usb_uiomove - factored out code 223 *------------------------------------------------------------------------*/ 224 #if USB_HAVE_USER_IO 225 int 226 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 227 usb_frlength_t pc_offset, usb_frlength_t len) 228 { 229 struct usb_page_search res; 230 int error = 0; 231 232 while (len != 0) { 233 234 usbd_get_page(pc, pc_offset, &res); 235 236 if (res.length > len) { 237 res.length = len; 238 } 239 /* 240 * "uiomove()" can sleep so one needs to make a wrapper, 241 * exiting the mutex and checking things 242 */ 243 error = uiomove(res.buffer, res.length, uio); 244 245 if (error) { 246 break; 247 } 248 pc_offset += res.length; 249 len -= res.length; 250 } 251 return (error); 252 } 253 #endif 254 255 /*------------------------------------------------------------------------* 256 * usbd_copy_out - copy directly from DMA-able memory 257 *------------------------------------------------------------------------*/ 258 void 259 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 260 void *ptr, usb_frlength_t len) 261 { 262 struct usb_page_search res; 263 264 while (len != 0) { 265 266 usbd_get_page(cache, offset, &res); 267 268 if (res.length > len) { 269 res.length = len; 270 } 271 memcpy(ptr, res.buffer, res.length); 272 273 offset += res.length; 274 len -= res.length; 275 ptr = USB_ADD_BYTES(ptr, res.length); 276 } 277 } 278 279 /*------------------------------------------------------------------------* 280 * usbd_copy_out_user - copy directly from DMA-able memory to userland 281 * 282 * Return values: 283 * 0: Success 284 * Else: Failure 285 *------------------------------------------------------------------------*/ 286 #if USB_HAVE_USER_IO 287 int 288 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 289 void *ptr, usb_frlength_t len) 290 { 291 struct usb_page_search res; 292 int error; 293 294 while (len != 0) { 295 296 usbd_get_page(cache, offset, &res); 297 298 if (res.length > len) { 299 res.length = len; 300 } 301 error = copyout(res.buffer, ptr, res.length); 302 if (error) 303 return (error); 304 305 offset += res.length; 306 len -= res.length; 307 ptr = USB_ADD_BYTES(ptr, res.length); 308 } 309 return (0); /* success */ 310 } 311 #endif 312 313 /*------------------------------------------------------------------------* 314 * usbd_frame_zero - zero DMA-able memory 315 *------------------------------------------------------------------------*/ 316 void 317 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 318 usb_frlength_t len) 319 { 320 struct usb_page_search res; 321 322 while (len != 0) { 323 324 usbd_get_page(cache, offset, &res); 325 326 if (res.length > len) { 327 res.length = len; 328 } 329 memset(res.buffer, 0, res.length); 330 331 offset += res.length; 332 len -= res.length; 333 } 334 } 335 336 #if USB_HAVE_BUSDMA 337 338 /*------------------------------------------------------------------------* 339 * usb_dma_lock_cb - dummy callback 340 *------------------------------------------------------------------------*/ 341 #if 0 342 static void 343 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 344 { 345 /* we use "mtx_owned()" instead of this function */ 346 } 347 #endif 348 349 /*------------------------------------------------------------------------* 350 * usb_dma_tag_create - allocate a DMA tag 351 * 352 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 353 * allow multi-segment mappings. Else all mappings are single-segment. 354 *------------------------------------------------------------------------*/ 355 static void 356 usb_dma_tag_create(struct usb_dma_tag *udt, 357 usb_size_t size, usb_size_t align) 358 { 359 bus_dma_tag_t tag; 360 361 if (bus_dma_tag_create 362 ( /* parent */ udt->tag_parent->tag, 363 /* alignment */ align, 364 /* boundary */ (align == 1) ? 365 USB_PAGE_SIZE : 0, 366 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 367 /* highaddr */ BUS_SPACE_MAXADDR, 368 /* filter */ NULL, 369 /* filterarg */ NULL, 370 /* maxsize */ size, 371 /* nsegments */ (align == 1 && size > 1) ? 372 (2 + (size / USB_PAGE_SIZE)) : 1, 373 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? 374 USB_PAGE_SIZE : size, 375 /* flags */ BUS_DMA_KEEP_PG_OFFSET, /* XXX: Find out what this is supposed to do! */ 376 &tag)) { 377 tag = NULL; 378 } 379 udt->tag = tag; 380 } 381 382 /*------------------------------------------------------------------------* 383 * usb_dma_tag_free - free a DMA tag 384 *------------------------------------------------------------------------*/ 385 static void 386 usb_dma_tag_destroy(struct usb_dma_tag *udt) 387 { 388 bus_dma_tag_destroy(udt->tag); 389 } 390 391 /*------------------------------------------------------------------------* 392 * usb_pc_alloc_mem_cb - BUS-DMA callback function 393 *------------------------------------------------------------------------*/ 394 static void 395 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 396 int nseg, int error) 397 { 398 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 399 } 400 401 /*------------------------------------------------------------------------* 402 * usb_pc_load_mem_cb - BUS-DMA callback function 403 *------------------------------------------------------------------------*/ 404 static void 405 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 406 int nseg, int error) 407 { 408 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 409 } 410 411 /*------------------------------------------------------------------------* 412 * usb_pc_common_mem_cb - BUS-DMA callback function 413 *------------------------------------------------------------------------*/ 414 static void 415 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 416 int nseg, int error, uint8_t isload) 417 { 418 struct usb_dma_parent_tag *uptag; 419 struct usb_page_cache *pc; 420 struct usb_page *pg; 421 usb_size_t rem; 422 uint8_t owned; 423 424 pc = arg; 425 uptag = pc->tag_parent; 426 427 /* 428 * XXX There is sometimes recursive locking here. 429 * XXX We should try to find a better solution. 430 * XXX Until further the "owned" variable does 431 * XXX the trick. 432 */ 433 434 if (error) { 435 goto done; 436 } 437 pg = pc->page_start; 438 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 439 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 440 pc->page_offset_buf = rem; 441 pc->page_offset_end += rem; 442 nseg--; 443 #ifdef USB_DEBUG 444 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 445 /* 446 * This check verifies that the physical address is correct: 447 */ 448 DPRINTFN(0, "Page offset was not preserved\n"); 449 error = 1; 450 goto done; 451 } 452 #endif 453 while (nseg > 0) { 454 nseg--; 455 segs++; 456 pg++; 457 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 458 } 459 460 done: 461 owned = lockowned(uptag->lock); 462 if (!owned) 463 lockmgr(uptag->lock, LK_EXCLUSIVE); 464 465 uptag->dma_error = (error ? 1 : 0); 466 if (isload) { 467 (uptag->func) (uptag); 468 } else { 469 cv_broadcast(uptag->cv); 470 } 471 if (!owned) 472 lockmgr(uptag->lock, LK_RELEASE); 473 } 474 475 /*------------------------------------------------------------------------* 476 * usb_pc_alloc_mem - allocate DMA'able memory 477 * 478 * Returns: 479 * 0: Success 480 * Else: Failure 481 *------------------------------------------------------------------------*/ 482 uint8_t 483 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 484 usb_size_t size, usb_size_t align) 485 { 486 struct usb_dma_parent_tag *uptag; 487 struct usb_dma_tag *utag; 488 bus_dmamap_t map; 489 void *ptr; 490 int err; 491 492 uptag = pc->tag_parent; 493 494 if (align != 1) { 495 /* 496 * The alignment must be greater or equal to the 497 * "size" else the object can be split between two 498 * memory pages and we get a problem! 499 */ 500 while (align < size) { 501 align *= 2; 502 if (align == 0) { 503 goto error; 504 } 505 } 506 #if 1 507 /* 508 * XXX BUS-DMA workaround - FIXME later: 509 * 510 * We assume that that the aligment at this point of 511 * the code is greater than or equal to the size and 512 * less than two times the size, so that if we double 513 * the size, the size will be greater than the 514 * alignment. 515 * 516 * The bus-dma system has a check for "alignment" 517 * being less than "size". If that check fails we end 518 * up using contigmalloc which is page based even for 519 * small allocations. Try to avoid that to save 520 * memory, hence we sometimes to a large number of 521 * small allocations! 522 */ 523 if (size <= (USB_PAGE_SIZE / 2)) { 524 size *= 2; 525 } 526 #endif 527 } 528 /* get the correct DMA tag */ 529 utag = usb_dma_tag_find(uptag, size, align); 530 if (utag == NULL) { 531 goto error; 532 } 533 /* allocate memory */ 534 if (bus_dmamem_alloc( 535 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 536 goto error; 537 } 538 /* setup page cache */ 539 pc->buffer = ptr; 540 pc->page_start = pg; 541 pc->page_offset_buf = 0; 542 pc->page_offset_end = size; 543 pc->map = map; 544 pc->tag = utag->tag; 545 pc->ismultiseg = (align == 1); 546 547 lockmgr(uptag->lock, LK_EXCLUSIVE); 548 549 /* load memory into DMA */ 550 err = bus_dmamap_load( 551 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 552 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 553 554 if (err == EINPROGRESS) { 555 cv_wait(uptag->cv, uptag->lock); 556 err = 0; 557 } 558 lockmgr(uptag->lock, LK_RELEASE); 559 560 if (err || uptag->dma_error) { 561 bus_dmamem_free(utag->tag, ptr, map); 562 goto error; 563 } 564 memset(ptr, 0, size); 565 566 usb_pc_cpu_flush(pc); 567 568 return (0); 569 570 error: 571 /* reset most of the page cache */ 572 pc->buffer = NULL; 573 pc->page_start = NULL; 574 pc->page_offset_buf = 0; 575 pc->page_offset_end = 0; 576 pc->map = NULL; 577 pc->tag = NULL; 578 return (1); 579 } 580 581 /*------------------------------------------------------------------------* 582 * usb_pc_free_mem - free DMA memory 583 * 584 * This function is NULL safe. 585 *------------------------------------------------------------------------*/ 586 void 587 usb_pc_free_mem(struct usb_page_cache *pc) 588 { 589 if (pc && pc->buffer) { 590 591 bus_dmamap_unload(pc->tag, pc->map); 592 593 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 594 595 pc->buffer = NULL; 596 } 597 } 598 599 /*------------------------------------------------------------------------* 600 * usb_pc_load_mem - load virtual memory into DMA 601 * 602 * Return values: 603 * 0: Success 604 * Else: Error 605 *------------------------------------------------------------------------*/ 606 uint8_t 607 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 608 { 609 /* setup page cache */ 610 pc->page_offset_buf = 0; 611 pc->page_offset_end = size; 612 pc->ismultiseg = 1; 613 614 KKASSERT(lockowned(pc->tag_parent->lock)); 615 616 if (size > 0) { 617 if (sync) { 618 struct usb_dma_parent_tag *uptag; 619 int err; 620 621 uptag = pc->tag_parent; 622 623 /* 624 * We have to unload the previous loaded DMA 625 * pages before trying to load a new one! 626 */ 627 bus_dmamap_unload(pc->tag, pc->map); 628 629 /* 630 * Try to load memory into DMA. 631 */ 632 err = bus_dmamap_load( 633 pc->tag, pc->map, pc->buffer, size, 634 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 635 if (err == EINPROGRESS) { 636 cv_wait(uptag->cv, uptag->lock); 637 err = 0; 638 } 639 if (err || uptag->dma_error) { 640 return (1); 641 } 642 } else { 643 644 /* 645 * We have to unload the previous loaded DMA 646 * pages before trying to load a new one! 647 */ 648 bus_dmamap_unload(pc->tag, pc->map); 649 650 /* 651 * Try to load memory into DMA. The callback 652 * will be called in all cases: 653 */ 654 if (bus_dmamap_load( 655 pc->tag, pc->map, pc->buffer, size, 656 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 657 } 658 } 659 } else { 660 if (!sync) { 661 /* 662 * Call callback so that refcount is decremented 663 * properly: 664 */ 665 pc->tag_parent->dma_error = 0; 666 (pc->tag_parent->func) (pc->tag_parent); 667 } 668 } 669 return (0); 670 } 671 672 /*------------------------------------------------------------------------* 673 * usb_pc_cpu_invalidate - invalidate CPU cache 674 *------------------------------------------------------------------------*/ 675 void 676 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 677 { 678 if (pc->page_offset_end == pc->page_offset_buf) { 679 /* nothing has been loaded into this page cache! */ 680 return; 681 } 682 683 /* 684 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the 685 * same time, but in the future we should try to isolate the 686 * different cases to optimise the code. --HPS 687 */ 688 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); 689 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); 690 } 691 692 /*------------------------------------------------------------------------* 693 * usb_pc_cpu_flush - flush CPU cache 694 *------------------------------------------------------------------------*/ 695 void 696 usb_pc_cpu_flush(struct usb_page_cache *pc) 697 { 698 if (pc->page_offset_end == pc->page_offset_buf) { 699 /* nothing has been loaded into this page cache! */ 700 return; 701 } 702 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); 703 } 704 705 /*------------------------------------------------------------------------* 706 * usb_pc_dmamap_create - create a DMA map 707 * 708 * Returns: 709 * 0: Success 710 * Else: Failure 711 *------------------------------------------------------------------------*/ 712 uint8_t 713 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 714 { 715 struct usb_xfer_root *info; 716 struct usb_dma_tag *utag; 717 718 /* get info */ 719 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 720 721 /* sanity check */ 722 if (info == NULL) { 723 goto error; 724 } 725 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 726 if (utag == NULL) { 727 goto error; 728 } 729 /* create DMA map */ 730 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 731 goto error; 732 } 733 pc->tag = utag->tag; 734 return 0; /* success */ 735 736 error: 737 pc->map = NULL; 738 pc->tag = NULL; 739 return 1; /* failure */ 740 } 741 742 /*------------------------------------------------------------------------* 743 * usb_pc_dmamap_destroy 744 * 745 * This function is NULL safe. 746 *------------------------------------------------------------------------*/ 747 void 748 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 749 { 750 if (pc && pc->tag) { 751 bus_dmamap_destroy(pc->tag, pc->map); 752 pc->tag = NULL; 753 pc->map = NULL; 754 } 755 } 756 757 /*------------------------------------------------------------------------* 758 * usb_dma_tag_find - factored out code 759 *------------------------------------------------------------------------*/ 760 struct usb_dma_tag * 761 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 762 usb_size_t size, usb_size_t align) 763 { 764 struct usb_dma_tag *udt; 765 uint8_t nudt; 766 767 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n")); 768 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n")); 769 770 udt = udpt->utag_first; 771 nudt = udpt->utag_max; 772 773 while (nudt--) { 774 775 if (udt->align == 0) { 776 usb_dma_tag_create(udt, size, align); 777 if (udt->tag == NULL) { 778 return (NULL); 779 } 780 udt->align = align; 781 udt->size = size; 782 return (udt); 783 } 784 if ((udt->align == align) && (udt->size == size)) { 785 return (udt); 786 } 787 udt++; 788 } 789 return (NULL); 790 } 791 792 /*------------------------------------------------------------------------* 793 * usb_dma_tag_setup - initialise USB DMA tags 794 *------------------------------------------------------------------------*/ 795 void 796 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 797 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 798 struct lock *lock, usb_dma_callback_t *func, 799 uint8_t ndmabits, uint8_t nudt) 800 { 801 memset(udpt, 0, sizeof(*udpt)); 802 803 /* sanity checking */ 804 if ((nudt == 0) || 805 (ndmabits == 0) || 806 (lock == NULL)) { 807 /* something is corrupt */ 808 return; 809 } 810 /* initialise condition variable */ 811 cv_init(udpt->cv, "USB DMA CV"); 812 813 /* store some information */ 814 udpt->lock = lock; 815 udpt->func = func; 816 udpt->tag = dmat; 817 udpt->utag_first = udt; 818 udpt->utag_max = nudt; 819 udpt->dma_bits = ndmabits; 820 821 while (nudt--) { 822 memset(udt, 0, sizeof(*udt)); 823 udt->tag_parent = udpt; 824 udt++; 825 } 826 } 827 828 /*------------------------------------------------------------------------* 829 * usb_bus_tag_unsetup - factored out code 830 *------------------------------------------------------------------------*/ 831 void 832 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 833 { 834 struct usb_dma_tag *udt; 835 uint8_t nudt; 836 837 udt = udpt->utag_first; 838 nudt = udpt->utag_max; 839 840 while (nudt--) { 841 842 if (udt->align) { 843 /* destroy the USB DMA tag */ 844 usb_dma_tag_destroy(udt); 845 udt->align = 0; 846 } 847 udt++; 848 } 849 850 if (udpt->utag_max) { 851 /* destroy the condition variable */ 852 cv_destroy(udpt->cv); 853 } 854 } 855 856 /*------------------------------------------------------------------------* 857 * usb_bdma_work_loop 858 * 859 * This function handles loading of virtual buffers into DMA and is 860 * only called when "dma_refcount" is zero. 861 *------------------------------------------------------------------------*/ 862 void 863 usb_bdma_work_loop(struct usb_xfer_queue *pq) 864 { 865 struct usb_xfer_root *info; 866 struct usb_xfer *xfer; 867 usb_frcount_t nframes; 868 869 xfer = pq->curr; 870 info = xfer->xroot; 871 872 KKASSERT(lockowned(info->xfer_lock)); 873 874 if (xfer->error) { 875 /* some error happened */ 876 USB_BUS_LOCK(info->bus); 877 usbd_transfer_done(xfer, 0); 878 USB_BUS_UNLOCK(info->bus); 879 return; 880 } 881 if (!xfer->flags_int.bdma_setup) { 882 struct usb_page *pg; 883 usb_frlength_t frlength_0; 884 uint8_t isread; 885 886 xfer->flags_int.bdma_setup = 1; 887 888 /* reset BUS-DMA load state */ 889 890 info->dma_error = 0; 891 892 if (xfer->flags_int.isochronous_xfr) { 893 /* only one frame buffer */ 894 nframes = 1; 895 frlength_0 = xfer->sumlen; 896 } else { 897 /* can be multiple frame buffers */ 898 nframes = xfer->nframes; 899 frlength_0 = xfer->frlengths[0]; 900 } 901 902 /* 903 * Set DMA direction first. This is needed to 904 * select the correct cache invalidate and cache 905 * flush operations. 906 */ 907 isread = USB_GET_DATA_ISREAD(xfer); 908 pg = xfer->dma_page_ptr; 909 910 if (xfer->flags_int.control_xfr && 911 xfer->flags_int.control_hdr) { 912 /* special case */ 913 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 914 /* The device controller writes to memory */ 915 xfer->frbuffers[0].isread = 1; 916 } else { 917 /* The host controller reads from memory */ 918 xfer->frbuffers[0].isread = 0; 919 } 920 } else { 921 /* default case */ 922 xfer->frbuffers[0].isread = isread; 923 } 924 925 /* 926 * Setup the "page_start" pointer which points to an array of 927 * USB pages where information about the physical address of a 928 * page will be stored. Also initialise the "isread" field of 929 * the USB page caches. 930 */ 931 xfer->frbuffers[0].page_start = pg; 932 933 info->dma_nframes = nframes; 934 info->dma_currframe = 0; 935 info->dma_frlength_0 = frlength_0; 936 937 pg += (frlength_0 / USB_PAGE_SIZE); 938 pg += 2; 939 940 while (--nframes > 0) { 941 xfer->frbuffers[nframes].isread = isread; 942 xfer->frbuffers[nframes].page_start = pg; 943 944 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 945 pg += 2; 946 } 947 948 } 949 if (info->dma_error) { 950 USB_BUS_LOCK(info->bus); 951 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 952 USB_BUS_UNLOCK(info->bus); 953 return; 954 } 955 if (info->dma_currframe != info->dma_nframes) { 956 957 if (info->dma_currframe == 0) { 958 /* special case */ 959 usb_pc_load_mem(xfer->frbuffers, 960 info->dma_frlength_0, 0); 961 } else { 962 /* default case */ 963 nframes = info->dma_currframe; 964 usb_pc_load_mem(xfer->frbuffers + nframes, 965 xfer->frlengths[nframes], 0); 966 } 967 968 /* advance frame index */ 969 info->dma_currframe++; 970 971 return; 972 } 973 /* go ahead */ 974 usb_bdma_pre_sync(xfer); 975 976 /* start loading next USB transfer, if any */ 977 usb_command_wrapper(pq, NULL); 978 979 /* finally start the hardware */ 980 usbd_pipe_enter(xfer); 981 } 982 983 /*------------------------------------------------------------------------* 984 * usb_bdma_done_event 985 * 986 * This function is called when the BUS-DMA has loaded virtual memory 987 * into DMA, if any. 988 *------------------------------------------------------------------------*/ 989 void 990 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 991 { 992 struct usb_xfer_root *info; 993 994 info = USB_DMATAG_TO_XROOT(udpt); 995 996 KKASSERT(lockowned(info->xfer_lock)); 997 998 /* copy error */ 999 info->dma_error = udpt->dma_error; 1000 1001 /* enter workloop again */ 1002 usb_command_wrapper(&info->dma_q, 1003 info->dma_q.curr); 1004 } 1005 1006 /*------------------------------------------------------------------------* 1007 * usb_bdma_pre_sync 1008 * 1009 * This function handles DMA synchronisation that must be done before 1010 * an USB transfer is started. 1011 *------------------------------------------------------------------------*/ 1012 void 1013 usb_bdma_pre_sync(struct usb_xfer *xfer) 1014 { 1015 struct usb_page_cache *pc; 1016 usb_frcount_t nframes; 1017 1018 if (xfer->flags_int.isochronous_xfr) { 1019 /* only one frame buffer */ 1020 nframes = 1; 1021 } else { 1022 /* can be multiple frame buffers */ 1023 nframes = xfer->nframes; 1024 } 1025 1026 pc = xfer->frbuffers; 1027 1028 while (nframes--) { 1029 1030 if (pc->isread) { 1031 usb_pc_cpu_invalidate(pc); 1032 } else { 1033 usb_pc_cpu_flush(pc); 1034 } 1035 pc++; 1036 } 1037 } 1038 1039 /*------------------------------------------------------------------------* 1040 * usb_bdma_post_sync 1041 * 1042 * This function handles DMA synchronisation that must be done after 1043 * an USB transfer is complete. 1044 *------------------------------------------------------------------------*/ 1045 void 1046 usb_bdma_post_sync(struct usb_xfer *xfer) 1047 { 1048 struct usb_page_cache *pc; 1049 usb_frcount_t nframes; 1050 1051 if (xfer->flags_int.isochronous_xfr) { 1052 /* only one frame buffer */ 1053 nframes = 1; 1054 } else { 1055 /* can be multiple frame buffers */ 1056 nframes = xfer->nframes; 1057 } 1058 1059 pc = xfer->frbuffers; 1060 1061 while (nframes--) { 1062 if (pc->isread) { 1063 usb_pc_cpu_invalidate(pc); 1064 } 1065 pc++; 1066 } 1067 } 1068 1069 #endif 1070