1 /* $FreeBSD: head/sys/dev/usb/usb_busdma.c 261505 2014-02-05 08:02:52Z hselasky $ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/stdint.h> 28 #include <sys/param.h> 29 #include <sys/queue.h> 30 #include <sys/types.h> 31 #include <sys/systm.h> 32 #include <sys/kernel.h> 33 #include <sys/bus.h> 34 #include <sys/module.h> 35 #include <sys/lock.h> 36 #include <sys/condvar.h> 37 #include <sys/sysctl.h> 38 #include <sys/unistd.h> 39 #include <sys/callout.h> 40 #include <sys/malloc.h> 41 #include <sys/caps.h> 42 43 #include <bus/u4b/usb.h> 44 #include <bus/u4b/usbdi.h> 45 #include <bus/u4b/usbdi_util.h> 46 47 #define USB_DEBUG_VAR usb_debug 48 49 #include <bus/u4b/usb_core.h> 50 #include <bus/u4b/usb_busdma.h> 51 #include <bus/u4b/usb_process.h> 52 #include <bus/u4b/usb_transfer.h> 53 #include <bus/u4b/usb_device.h> 54 #include <bus/u4b/usb_util.h> 55 #include <bus/u4b/usb_debug.h> 56 57 #include <bus/u4b/usb_controller.h> 58 #include <bus/u4b/usb_bus.h> 59 60 #if USB_HAVE_BUSDMA 61 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 62 static void usb_dma_tag_destroy(struct usb_dma_tag *); 63 #if 0 64 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 65 #endif 66 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 67 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 68 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 69 uint8_t); 70 #endif 71 72 /*------------------------------------------------------------------------* 73 * usbd_get_page - lookup DMA-able memory for the given offset 74 * 75 * NOTE: Only call this function when the "page_cache" structure has 76 * been properly initialized ! 77 *------------------------------------------------------------------------*/ 78 void 79 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 80 struct usb_page_search *res) 81 { 82 #if USB_HAVE_BUSDMA 83 struct usb_page *page; 84 85 if (pc->page_start) { 86 87 /* Case 1 - something has been loaded into DMA */ 88 89 if (pc->buffer) { 90 91 /* Case 1a - Kernel Virtual Address */ 92 93 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 94 } 95 offset += pc->page_offset_buf; 96 97 /* compute destination page */ 98 99 page = pc->page_start; 100 101 if (pc->ismultiseg) { 102 103 page += (offset / USB_PAGE_SIZE); 104 105 offset %= USB_PAGE_SIZE; 106 107 res->length = USB_PAGE_SIZE - offset; 108 res->physaddr = page->physaddr + offset; 109 } else { 110 res->length = (usb_size_t)-1; 111 res->physaddr = page->physaddr + offset; 112 } 113 if (!pc->buffer) { 114 115 /* Case 1b - Non Kernel Virtual Address */ 116 117 res->buffer = USB_ADD_BYTES(page->buffer, offset); 118 } 119 return; 120 } 121 #endif 122 /* Case 2 - Plain PIO */ 123 124 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 125 res->length = (usb_size_t)-1; 126 #if USB_HAVE_BUSDMA 127 res->physaddr = 0; 128 #endif 129 } 130 131 /*------------------------------------------------------------------------* 132 * usbd_copy_in - copy directly to DMA-able memory 133 *------------------------------------------------------------------------*/ 134 void 135 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 136 const void *ptr, usb_frlength_t len) 137 { 138 struct usb_page_search buf_res; 139 140 while (len != 0) { 141 142 usbd_get_page(cache, offset, &buf_res); 143 144 if (buf_res.length > len) { 145 buf_res.length = len; 146 } 147 memcpy(buf_res.buffer, ptr, buf_res.length); 148 149 offset += buf_res.length; 150 len -= buf_res.length; 151 ptr = USB_ADD_BYTES(ptr, buf_res.length); 152 } 153 } 154 155 /*------------------------------------------------------------------------* 156 * usbd_copy_in_user - copy directly to DMA-able memory from userland 157 * 158 * Return values: 159 * 0: Success 160 * Else: Failure 161 *------------------------------------------------------------------------*/ 162 #if USB_HAVE_USER_IO 163 int 164 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 165 const void *ptr, usb_frlength_t len) 166 { 167 struct usb_page_search buf_res; 168 int error; 169 170 while (len != 0) { 171 172 usbd_get_page(cache, offset, &buf_res); 173 174 if (buf_res.length > len) { 175 buf_res.length = len; 176 } 177 error = copyin(ptr, buf_res.buffer, buf_res.length); 178 if (error) 179 return (error); 180 181 offset += buf_res.length; 182 len -= buf_res.length; 183 ptr = USB_ADD_BYTES(ptr, buf_res.length); 184 } 185 return (0); /* success */ 186 } 187 #endif 188 189 /*------------------------------------------------------------------------* 190 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 191 *------------------------------------------------------------------------*/ 192 #if USB_HAVE_MBUF 193 struct usb_m_copy_in_arg { 194 struct usb_page_cache *cache; 195 usb_frlength_t dst_offset; 196 }; 197 198 static int 199 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 200 { 201 register struct usb_m_copy_in_arg *ua = arg; 202 203 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 204 ua->dst_offset += count; 205 return (0); 206 } 207 208 void 209 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 210 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 211 { 212 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 213 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 214 } 215 #endif 216 217 /*------------------------------------------------------------------------* 218 * usb_uiomove - factored out code 219 *------------------------------------------------------------------------*/ 220 #if USB_HAVE_USER_IO 221 int 222 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 223 usb_frlength_t pc_offset, usb_frlength_t len) 224 { 225 struct usb_page_search res; 226 int error = 0; 227 228 while (len != 0) { 229 230 usbd_get_page(pc, pc_offset, &res); 231 232 if (res.length > len) { 233 res.length = len; 234 } 235 /* 236 * "uiomove()" can sleep so one needs to make a wrapper, 237 * exiting the mutex and checking things 238 */ 239 error = uiomove(res.buffer, res.length, uio); 240 241 if (error) { 242 break; 243 } 244 pc_offset += res.length; 245 len -= res.length; 246 } 247 return (error); 248 } 249 #endif 250 251 /*------------------------------------------------------------------------* 252 * usbd_copy_out - copy directly from DMA-able memory 253 *------------------------------------------------------------------------*/ 254 void 255 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 256 void *ptr, usb_frlength_t len) 257 { 258 struct usb_page_search res; 259 260 while (len != 0) { 261 262 usbd_get_page(cache, offset, &res); 263 264 if (res.length > len) { 265 res.length = len; 266 } 267 memcpy(ptr, res.buffer, res.length); 268 269 offset += res.length; 270 len -= res.length; 271 ptr = USB_ADD_BYTES(ptr, res.length); 272 } 273 } 274 275 /*------------------------------------------------------------------------* 276 * usbd_copy_out_user - copy directly from DMA-able memory to userland 277 * 278 * Return values: 279 * 0: Success 280 * Else: Failure 281 *------------------------------------------------------------------------*/ 282 #if USB_HAVE_USER_IO 283 int 284 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 285 void *ptr, usb_frlength_t len) 286 { 287 struct usb_page_search res; 288 int error; 289 290 while (len != 0) { 291 292 usbd_get_page(cache, offset, &res); 293 294 if (res.length > len) { 295 res.length = len; 296 } 297 error = copyout(res.buffer, ptr, res.length); 298 if (error) 299 return (error); 300 301 offset += res.length; 302 len -= res.length; 303 ptr = USB_ADD_BYTES(ptr, res.length); 304 } 305 return (0); /* success */ 306 } 307 #endif 308 309 /*------------------------------------------------------------------------* 310 * usbd_frame_zero - zero DMA-able memory 311 *------------------------------------------------------------------------*/ 312 void 313 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 314 usb_frlength_t len) 315 { 316 struct usb_page_search res; 317 318 while (len != 0) { 319 320 usbd_get_page(cache, offset, &res); 321 322 if (res.length > len) { 323 res.length = len; 324 } 325 memset(res.buffer, 0, res.length); 326 327 offset += res.length; 328 len -= res.length; 329 } 330 } 331 332 #if USB_HAVE_BUSDMA 333 334 /*------------------------------------------------------------------------* 335 * usb_dma_lock_cb - dummy callback 336 *------------------------------------------------------------------------*/ 337 #if 0 338 static void 339 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 340 { 341 /* we use "mtx_owned()" instead of this function */ 342 } 343 #endif 344 345 /*------------------------------------------------------------------------* 346 * usb_dma_tag_create - allocate a DMA tag 347 * 348 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 349 * allow multi-segment mappings. Else all mappings are single-segment. 350 *------------------------------------------------------------------------*/ 351 static void 352 usb_dma_tag_create(struct usb_dma_tag *udt, 353 usb_size_t size, usb_size_t align) 354 { 355 bus_dma_tag_t tag; 356 357 if (bus_dma_tag_create 358 ( /* parent */ udt->tag_parent->tag, 359 /* alignment */ align, 360 /* boundary */ 0, 361 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 362 /* highaddr */ BUS_SPACE_MAXADDR, 363 /* filter */ NULL, 364 /* filterarg */ NULL, 365 /* maxsize */ size, 366 /* nsegments */ (align == 1 && size > 1) ? (2 + (size / USB_PAGE_SIZE)) : 1, 367 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? USB_PAGE_SIZE : size, 368 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 369 &tag)) { 370 tag = NULL; 371 } 372 udt->tag = tag; 373 } 374 375 /*------------------------------------------------------------------------* 376 * usb_dma_tag_free - free a DMA tag 377 *------------------------------------------------------------------------*/ 378 static void 379 usb_dma_tag_destroy(struct usb_dma_tag *udt) 380 { 381 bus_dma_tag_destroy(udt->tag); 382 } 383 384 /*------------------------------------------------------------------------* 385 * usb_pc_alloc_mem_cb - BUS-DMA callback function 386 *------------------------------------------------------------------------*/ 387 static void 388 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 389 int nseg, int error) 390 { 391 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 392 } 393 394 /*------------------------------------------------------------------------* 395 * usb_pc_load_mem_cb - BUS-DMA callback function 396 *------------------------------------------------------------------------*/ 397 static void 398 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 399 int nseg, int error) 400 { 401 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 402 } 403 404 /*------------------------------------------------------------------------* 405 * usb_pc_common_mem_cb - BUS-DMA callback function 406 *------------------------------------------------------------------------*/ 407 static void 408 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 409 int nseg, int error, uint8_t isload) 410 { 411 struct usb_dma_parent_tag *uptag; 412 struct usb_page_cache *pc; 413 struct usb_page *pg; 414 usb_size_t rem; 415 bus_size_t off; 416 uint8_t owned; 417 418 pc = arg; 419 uptag = pc->tag_parent; 420 421 /* 422 * XXX There is sometimes recursive locking here. 423 * XXX We should try to find a better solution. 424 * XXX Until further the "owned" variable does 425 * XXX the trick. 426 */ 427 428 if (error) { 429 goto done; 430 } 431 432 off = 0; 433 pg = pc->page_start; 434 pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE); 435 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 436 pc->page_offset_buf = rem; 437 pc->page_offset_end += rem; 438 #ifdef USB_DEBUG 439 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 440 /* 441 * This check verifies that the physical address is correct: 442 */ 443 DPRINTFN(0, "Page offset was not preserved\n"); 444 error = 1; 445 goto done; 446 } 447 #endif 448 while (pc->ismultiseg) { 449 off += USB_PAGE_SIZE; 450 if (off >= (segs->ds_len + rem)) { 451 /* page crossing */ 452 nseg--; 453 segs++; 454 off = 0; 455 rem = 0; 456 if (nseg == 0) 457 break; 458 } 459 pg++; 460 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE); 461 } 462 463 done: 464 owned = lockowned(uptag->lock); 465 if (!owned) 466 lockmgr(uptag->lock, LK_EXCLUSIVE); 467 468 uptag->dma_error = (error ? 1 : 0); 469 if (isload) { 470 (uptag->func) (uptag); 471 } else { 472 cv_broadcast(uptag->cv); 473 } 474 if (!owned) 475 lockmgr(uptag->lock, LK_RELEASE); 476 } 477 478 /*------------------------------------------------------------------------* 479 * usb_pc_alloc_mem - allocate DMA'able memory 480 * 481 * Returns: 482 * 0: Success 483 * Else: Failure 484 *------------------------------------------------------------------------*/ 485 uint8_t 486 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 487 usb_size_t size, usb_size_t align) 488 { 489 struct usb_dma_parent_tag *uptag; 490 struct usb_dma_tag *utag; 491 bus_dmamap_t map; 492 void *ptr; 493 int err; 494 495 uptag = pc->tag_parent; 496 497 if (align != 1) { 498 /* 499 * The alignment must be greater or equal to the 500 * "size" else the object can be split between two 501 * memory pages and we get a problem! 502 */ 503 while (align < size) { 504 align *= 2; 505 if (align == 0) { 506 goto error; 507 } 508 } 509 #if 1 510 /* 511 * XXX BUS-DMA workaround - FIXME later: 512 * 513 * We assume that that the aligment at this point of 514 * the code is greater than or equal to the size and 515 * less than two times the size, so that if we double 516 * the size, the size will be greater than the 517 * alignment. 518 * 519 * The bus-dma system has a check for "alignment" 520 * being less than "size". If that check fails we end 521 * up using contigmalloc which is page based even for 522 * small allocations. Try to avoid that to save 523 * memory, hence we sometimes to a large number of 524 * small allocations! 525 */ 526 if (size <= (USB_PAGE_SIZE / 2)) { 527 size *= 2; 528 } 529 #endif 530 } 531 /* get the correct DMA tag */ 532 utag = usb_dma_tag_find(uptag, size, align); 533 if (utag == NULL) { 534 goto error; 535 } 536 /* allocate memory */ 537 if (bus_dmamem_alloc( 538 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 539 goto error; 540 } 541 /* setup page cache */ 542 pc->buffer = ptr; 543 pc->page_start = pg; 544 pc->page_offset_buf = 0; 545 pc->page_offset_end = size; 546 pc->map = map; 547 pc->tag = utag->tag; 548 pc->ismultiseg = (align == 1); 549 550 lockmgr(uptag->lock, LK_EXCLUSIVE); 551 552 /* load memory into DMA */ 553 err = bus_dmamap_load( 554 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 555 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 556 557 if (err == EINPROGRESS) { 558 cv_wait(uptag->cv, uptag->lock); 559 err = 0; 560 } 561 lockmgr(uptag->lock, LK_RELEASE); 562 563 if (err || uptag->dma_error) { 564 bus_dmamem_free(utag->tag, ptr, map); 565 goto error; 566 } 567 memset(ptr, 0, size); 568 569 usb_pc_cpu_flush(pc); 570 571 return (0); 572 573 error: 574 /* reset most of the page cache */ 575 pc->buffer = NULL; 576 pc->page_start = NULL; 577 pc->page_offset_buf = 0; 578 pc->page_offset_end = 0; 579 pc->map = NULL; 580 pc->tag = NULL; 581 return (1); 582 } 583 584 /*------------------------------------------------------------------------* 585 * usb_pc_free_mem - free DMA memory 586 * 587 * This function is NULL safe. 588 *------------------------------------------------------------------------*/ 589 void 590 usb_pc_free_mem(struct usb_page_cache *pc) 591 { 592 if (pc && pc->buffer) { 593 594 bus_dmamap_unload(pc->tag, pc->map); 595 596 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 597 598 pc->buffer = NULL; 599 } 600 } 601 602 /*------------------------------------------------------------------------* 603 * usb_pc_load_mem - load virtual memory into DMA 604 * 605 * Return values: 606 * 0: Success 607 * Else: Error 608 *------------------------------------------------------------------------*/ 609 uint8_t 610 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 611 { 612 /* setup page cache */ 613 pc->page_offset_buf = 0; 614 pc->page_offset_end = size; 615 pc->ismultiseg = 1; 616 617 KKASSERT(lockowned(pc->tag_parent->lock)); 618 619 if (size > 0) { 620 if (sync) { 621 struct usb_dma_parent_tag *uptag; 622 int err; 623 624 uptag = pc->tag_parent; 625 626 /* 627 * We have to unload the previous loaded DMA 628 * pages before trying to load a new one! 629 */ 630 bus_dmamap_unload(pc->tag, pc->map); 631 632 /* 633 * Try to load memory into DMA. 634 */ 635 err = bus_dmamap_load( 636 pc->tag, pc->map, pc->buffer, size, 637 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 638 if (err == EINPROGRESS) { 639 cv_wait(uptag->cv, uptag->lock); 640 err = 0; 641 } 642 if (err || uptag->dma_error) { 643 return (1); 644 } 645 } else { 646 647 /* 648 * We have to unload the previous loaded DMA 649 * pages before trying to load a new one! 650 */ 651 bus_dmamap_unload(pc->tag, pc->map); 652 653 /* 654 * Try to load memory into DMA. The callback 655 * will be called in all cases: 656 */ 657 if (bus_dmamap_load( 658 pc->tag, pc->map, pc->buffer, size, 659 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 660 } 661 } 662 } else { 663 if (!sync) { 664 /* 665 * Call callback so that refcount is decremented 666 * properly: 667 */ 668 pc->tag_parent->dma_error = 0; 669 (pc->tag_parent->func) (pc->tag_parent); 670 } 671 } 672 return (0); 673 } 674 675 /*------------------------------------------------------------------------* 676 * usb_pc_cpu_invalidate - invalidate CPU cache 677 *------------------------------------------------------------------------*/ 678 void 679 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 680 { 681 if (pc->page_offset_end == pc->page_offset_buf) { 682 /* nothing has been loaded into this page cache! */ 683 return; 684 } 685 686 /* 687 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the 688 * same time, but in the future we should try to isolate the 689 * different cases to optimise the code. --HPS 690 */ 691 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); 692 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); 693 } 694 695 /*------------------------------------------------------------------------* 696 * usb_pc_cpu_flush - flush CPU cache 697 *------------------------------------------------------------------------*/ 698 void 699 usb_pc_cpu_flush(struct usb_page_cache *pc) 700 { 701 if (pc->page_offset_end == pc->page_offset_buf) { 702 /* nothing has been loaded into this page cache! */ 703 return; 704 } 705 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); 706 } 707 708 /*------------------------------------------------------------------------* 709 * usb_pc_dmamap_create - create a DMA map 710 * 711 * Returns: 712 * 0: Success 713 * Else: Failure 714 *------------------------------------------------------------------------*/ 715 uint8_t 716 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 717 { 718 struct usb_xfer_root *info; 719 struct usb_dma_tag *utag; 720 721 /* get info */ 722 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 723 724 /* sanity check */ 725 if (info == NULL) { 726 goto error; 727 } 728 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 729 if (utag == NULL) { 730 goto error; 731 } 732 /* create DMA map */ 733 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 734 goto error; 735 } 736 pc->tag = utag->tag; 737 return 0; /* success */ 738 739 error: 740 pc->map = NULL; 741 pc->tag = NULL; 742 return 1; /* failure */ 743 } 744 745 /*------------------------------------------------------------------------* 746 * usb_pc_dmamap_destroy 747 * 748 * This function is NULL safe. 749 *------------------------------------------------------------------------*/ 750 void 751 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 752 { 753 if (pc && pc->tag) { 754 bus_dmamap_destroy(pc->tag, pc->map); 755 pc->tag = NULL; 756 pc->map = NULL; 757 } 758 } 759 760 /*------------------------------------------------------------------------* 761 * usb_dma_tag_find - factored out code 762 *------------------------------------------------------------------------*/ 763 struct usb_dma_tag * 764 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 765 usb_size_t size, usb_size_t align) 766 { 767 struct usb_dma_tag *udt; 768 uint8_t nudt; 769 770 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n")); 771 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n")); 772 773 udt = udpt->utag_first; 774 nudt = udpt->utag_max; 775 776 while (nudt--) { 777 778 if (udt->align == 0) { 779 usb_dma_tag_create(udt, size, align); 780 if (udt->tag == NULL) { 781 return (NULL); 782 } 783 udt->align = align; 784 udt->size = size; 785 return (udt); 786 } 787 if ((udt->align == align) && (udt->size == size)) { 788 return (udt); 789 } 790 udt++; 791 } 792 return (NULL); 793 } 794 795 /*------------------------------------------------------------------------* 796 * usb_dma_tag_setup - initialise USB DMA tags 797 *------------------------------------------------------------------------*/ 798 void 799 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 800 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 801 struct lock *lock, usb_dma_callback_t *func, 802 uint8_t ndmabits, uint8_t nudt) 803 { 804 memset(udpt, 0, sizeof(*udpt)); 805 806 /* sanity checking */ 807 if ((nudt == 0) || 808 (ndmabits == 0) || 809 (lock == NULL)) { 810 /* something is corrupt */ 811 return; 812 } 813 /* initialise condition variable */ 814 cv_init(udpt->cv, "USB DMA CV"); 815 816 /* store some information */ 817 udpt->lock = lock; 818 udpt->func = func; 819 udpt->tag = dmat; 820 udpt->utag_first = udt; 821 udpt->utag_max = nudt; 822 udpt->dma_bits = ndmabits; 823 824 while (nudt--) { 825 memset(udt, 0, sizeof(*udt)); 826 udt->tag_parent = udpt; 827 udt++; 828 } 829 } 830 831 /*------------------------------------------------------------------------* 832 * usb_bus_tag_unsetup - factored out code 833 *------------------------------------------------------------------------*/ 834 void 835 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 836 { 837 struct usb_dma_tag *udt; 838 uint8_t nudt; 839 840 udt = udpt->utag_first; 841 nudt = udpt->utag_max; 842 843 while (nudt--) { 844 845 if (udt->align) { 846 /* destroy the USB DMA tag */ 847 usb_dma_tag_destroy(udt); 848 udt->align = 0; 849 } 850 udt++; 851 } 852 853 if (udpt->utag_max) { 854 /* destroy the condition variable */ 855 cv_destroy(udpt->cv); 856 } 857 } 858 859 /*------------------------------------------------------------------------* 860 * usb_bdma_work_loop 861 * 862 * This function handles loading of virtual buffers into DMA and is 863 * only called when "dma_refcount" is zero. 864 *------------------------------------------------------------------------*/ 865 void 866 usb_bdma_work_loop(struct usb_xfer_queue *pq) 867 { 868 struct usb_xfer_root *info; 869 struct usb_xfer *xfer; 870 usb_frcount_t nframes; 871 872 xfer = pq->curr; 873 info = xfer->xroot; 874 875 KKASSERT(lockowned(info->xfer_lock)); 876 877 if (xfer->error) { 878 /* some error happened */ 879 USB_BUS_LOCK(info->bus); 880 usbd_transfer_done(xfer, 0); 881 USB_BUS_UNLOCK(info->bus); 882 return; 883 } 884 if (!xfer->flags_int.bdma_setup) { 885 struct usb_page *pg; 886 usb_frlength_t frlength_0; 887 uint8_t isread; 888 889 xfer->flags_int.bdma_setup = 1; 890 891 /* reset BUS-DMA load state */ 892 893 info->dma_error = 0; 894 895 if (xfer->flags_int.isochronous_xfr) { 896 /* only one frame buffer */ 897 nframes = 1; 898 frlength_0 = xfer->sumlen; 899 } else { 900 /* can be multiple frame buffers */ 901 nframes = xfer->nframes; 902 frlength_0 = xfer->frlengths[0]; 903 } 904 905 /* 906 * Set DMA direction first. This is needed to 907 * select the correct cache invalidate and cache 908 * flush operations. 909 */ 910 isread = USB_GET_DATA_ISREAD(xfer); 911 pg = xfer->dma_page_ptr; 912 913 if (xfer->flags_int.control_xfr && 914 xfer->flags_int.control_hdr) { 915 /* special case */ 916 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 917 /* The device controller writes to memory */ 918 xfer->frbuffers[0].isread = 1; 919 } else { 920 /* The host controller reads from memory */ 921 xfer->frbuffers[0].isread = 0; 922 } 923 } else { 924 /* default case */ 925 xfer->frbuffers[0].isread = isread; 926 } 927 928 /* 929 * Setup the "page_start" pointer which points to an array of 930 * USB pages where information about the physical address of a 931 * page will be stored. Also initialise the "isread" field of 932 * the USB page caches. 933 */ 934 xfer->frbuffers[0].page_start = pg; 935 936 info->dma_nframes = nframes; 937 info->dma_currframe = 0; 938 info->dma_frlength_0 = frlength_0; 939 940 pg += (frlength_0 / USB_PAGE_SIZE); 941 pg += 2; 942 943 while (--nframes > 0) { 944 xfer->frbuffers[nframes].isread = isread; 945 xfer->frbuffers[nframes].page_start = pg; 946 947 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 948 pg += 2; 949 } 950 951 } 952 if (info->dma_error) { 953 USB_BUS_LOCK(info->bus); 954 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 955 USB_BUS_UNLOCK(info->bus); 956 return; 957 } 958 if (info->dma_currframe != info->dma_nframes) { 959 960 if (info->dma_currframe == 0) { 961 /* special case */ 962 usb_pc_load_mem(xfer->frbuffers, 963 info->dma_frlength_0, 0); 964 } else { 965 /* default case */ 966 nframes = info->dma_currframe; 967 usb_pc_load_mem(xfer->frbuffers + nframes, 968 xfer->frlengths[nframes], 0); 969 } 970 971 /* advance frame index */ 972 info->dma_currframe++; 973 974 return; 975 } 976 /* go ahead */ 977 usb_bdma_pre_sync(xfer); 978 979 /* start loading next USB transfer, if any */ 980 usb_command_wrapper(pq, NULL); 981 982 /* finally start the hardware */ 983 usbd_pipe_enter(xfer); 984 } 985 986 /*------------------------------------------------------------------------* 987 * usb_bdma_done_event 988 * 989 * This function is called when the BUS-DMA has loaded virtual memory 990 * into DMA, if any. 991 *------------------------------------------------------------------------*/ 992 void 993 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 994 { 995 struct usb_xfer_root *info; 996 997 info = USB_DMATAG_TO_XROOT(udpt); 998 999 KKASSERT(lockowned(info->xfer_lock)); 1000 1001 /* copy error */ 1002 info->dma_error = udpt->dma_error; 1003 1004 /* enter workloop again */ 1005 usb_command_wrapper(&info->dma_q, 1006 info->dma_q.curr); 1007 } 1008 1009 /*------------------------------------------------------------------------* 1010 * usb_bdma_pre_sync 1011 * 1012 * This function handles DMA synchronisation that must be done before 1013 * an USB transfer is started. 1014 *------------------------------------------------------------------------*/ 1015 void 1016 usb_bdma_pre_sync(struct usb_xfer *xfer) 1017 { 1018 struct usb_page_cache *pc; 1019 usb_frcount_t nframes; 1020 1021 if (xfer->flags_int.isochronous_xfr) { 1022 /* only one frame buffer */ 1023 nframes = 1; 1024 } else { 1025 /* can be multiple frame buffers */ 1026 nframes = xfer->nframes; 1027 } 1028 1029 pc = xfer->frbuffers; 1030 1031 while (nframes--) { 1032 1033 if (pc->isread) { 1034 usb_pc_cpu_invalidate(pc); 1035 } else { 1036 usb_pc_cpu_flush(pc); 1037 } 1038 pc++; 1039 } 1040 } 1041 1042 /*------------------------------------------------------------------------* 1043 * usb_bdma_post_sync 1044 * 1045 * This function handles DMA synchronisation that must be done after 1046 * an USB transfer is complete. 1047 *------------------------------------------------------------------------*/ 1048 void 1049 usb_bdma_post_sync(struct usb_xfer *xfer) 1050 { 1051 struct usb_page_cache *pc; 1052 usb_frcount_t nframes; 1053 1054 if (xfer->flags_int.isochronous_xfr) { 1055 /* only one frame buffer */ 1056 nframes = 1; 1057 } else { 1058 /* can be multiple frame buffers */ 1059 nframes = xfer->nframes; 1060 } 1061 1062 pc = xfer->frbuffers; 1063 1064 while (nframes--) { 1065 if (pc->isread) { 1066 usb_pc_cpu_invalidate(pc); 1067 } 1068 pc++; 1069 } 1070 } 1071 1072 #endif 1073