1 /* $FreeBSD: head/sys/dev/usb/usb_busdma.c 261505 2014-02-05 08:02:52Z hselasky $ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/stdint.h> 28 #include <sys/param.h> 29 #include <sys/queue.h> 30 #include <sys/types.h> 31 #include <sys/systm.h> 32 #include <sys/kernel.h> 33 #include <sys/bus.h> 34 #include <sys/module.h> 35 #include <sys/lock.h> 36 #include <sys/condvar.h> 37 #include <sys/sysctl.h> 38 #include <sys/unistd.h> 39 #include <sys/callout.h> 40 #include <sys/malloc.h> 41 #include <sys/caps.h> 42 43 #include <bus/u4b/usb.h> 44 #include <bus/u4b/usbdi.h> 45 #include <bus/u4b/usbdi_util.h> 46 47 #define USB_DEBUG_VAR usb_debug 48 49 #include <bus/u4b/usb_core.h> 50 #include <bus/u4b/usb_busdma.h> 51 #include <bus/u4b/usb_process.h> 52 #include <bus/u4b/usb_transfer.h> 53 #include <bus/u4b/usb_device.h> 54 #include <bus/u4b/usb_util.h> 55 #include <bus/u4b/usb_debug.h> 56 57 #include <bus/u4b/usb_controller.h> 58 #include <bus/u4b/usb_bus.h> 59 60 #if USB_HAVE_BUSDMA 61 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 62 static void usb_dma_tag_destroy(struct usb_dma_tag *); 63 #if 0 64 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 65 #endif 66 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 67 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 68 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 69 uint8_t); 70 #endif 71 72 /*------------------------------------------------------------------------* 73 * usbd_get_page - lookup DMA-able memory for the given offset 74 * 75 * NOTE: Only call this function when the "page_cache" structure has 76 * been properly initialized ! 77 *------------------------------------------------------------------------*/ 78 void 79 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 80 struct usb_page_search *res) 81 { 82 #if USB_HAVE_BUSDMA 83 struct usb_page *page; 84 85 if (pc->page_start) { 86 87 /* Case 1 - something has been loaded into DMA */ 88 89 if (pc->buffer) { 90 91 /* Case 1a - Kernel Virtual Address */ 92 93 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 94 } 95 offset += pc->page_offset_buf; 96 97 /* compute destination page */ 98 99 page = pc->page_start; 100 101 if (pc->ismultiseg) { 102 103 page += (offset / USB_PAGE_SIZE); 104 105 offset %= USB_PAGE_SIZE; 106 107 res->length = USB_PAGE_SIZE - offset; 108 res->physaddr = page->physaddr + offset; 109 } else { 110 res->length = (usb_size_t)-1; 111 res->physaddr = page->physaddr + offset; 112 } 113 if (!pc->buffer) { 114 115 /* Case 1b - Non Kernel Virtual Address */ 116 117 res->buffer = USB_ADD_BYTES(page->buffer, offset); 118 } 119 return; 120 } 121 #endif 122 /* Case 2 - Plain PIO */ 123 124 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 125 res->length = (usb_size_t)-1; 126 #if USB_HAVE_BUSDMA 127 res->physaddr = 0; 128 #endif 129 } 130 131 /*------------------------------------------------------------------------* 132 * usbd_copy_in - copy directly to DMA-able memory 133 *------------------------------------------------------------------------*/ 134 void 135 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 136 const void *ptr, usb_frlength_t len) 137 { 138 struct usb_page_search buf_res; 139 140 while (len != 0) { 141 142 usbd_get_page(cache, offset, &buf_res); 143 144 if (buf_res.length > len) { 145 buf_res.length = len; 146 } 147 memcpy(buf_res.buffer, ptr, buf_res.length); 148 149 offset += buf_res.length; 150 len -= buf_res.length; 151 ptr = USB_ADD_BYTES(ptr, buf_res.length); 152 } 153 } 154 155 /*------------------------------------------------------------------------* 156 * usbd_copy_in_user - copy directly to DMA-able memory from userland 157 * 158 * Return values: 159 * 0: Success 160 * Else: Failure 161 *------------------------------------------------------------------------*/ 162 #if USB_HAVE_USER_IO 163 int 164 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 165 const void *ptr, usb_frlength_t len) 166 { 167 struct usb_page_search buf_res; 168 int error; 169 170 while (len != 0) { 171 172 usbd_get_page(cache, offset, &buf_res); 173 174 if (buf_res.length > len) { 175 buf_res.length = len; 176 } 177 error = copyin(ptr, buf_res.buffer, buf_res.length); 178 if (error) 179 return (error); 180 181 offset += buf_res.length; 182 len -= buf_res.length; 183 ptr = USB_ADD_BYTES(ptr, buf_res.length); 184 } 185 return (0); /* success */ 186 } 187 #endif 188 189 /*------------------------------------------------------------------------* 190 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 191 *------------------------------------------------------------------------*/ 192 #if USB_HAVE_MBUF 193 struct usb_m_copy_in_arg { 194 struct usb_page_cache *cache; 195 usb_frlength_t dst_offset; 196 }; 197 198 static int 199 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 200 { 201 register struct usb_m_copy_in_arg *ua = arg; 202 203 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 204 ua->dst_offset += count; 205 return (0); 206 } 207 208 void 209 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 210 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 211 { 212 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 213 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 214 } 215 #endif 216 217 /*------------------------------------------------------------------------* 218 * usb_uiomove - factored out code 219 *------------------------------------------------------------------------*/ 220 #if USB_HAVE_USER_IO 221 int 222 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 223 usb_frlength_t pc_offset, usb_frlength_t len) 224 { 225 struct usb_page_search res; 226 int error = 0; 227 228 while (len != 0) { 229 230 usbd_get_page(pc, pc_offset, &res); 231 232 if (res.length > len) { 233 res.length = len; 234 } 235 /* 236 * "uiomove()" can sleep so one needs to make a wrapper, 237 * exiting the mutex and checking things 238 */ 239 error = uiomove(res.buffer, res.length, uio); 240 241 if (error) { 242 break; 243 } 244 pc_offset += res.length; 245 len -= res.length; 246 } 247 return (error); 248 } 249 #endif 250 251 /*------------------------------------------------------------------------* 252 * usbd_copy_out - copy directly from DMA-able memory 253 *------------------------------------------------------------------------*/ 254 void 255 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 256 void *ptr, usb_frlength_t len) 257 { 258 struct usb_page_search res; 259 260 while (len != 0) { 261 262 usbd_get_page(cache, offset, &res); 263 264 if (res.length > len) { 265 res.length = len; 266 } 267 memcpy(ptr, res.buffer, res.length); 268 269 offset += res.length; 270 len -= res.length; 271 ptr = USB_ADD_BYTES(ptr, res.length); 272 } 273 } 274 275 /*------------------------------------------------------------------------* 276 * usbd_copy_out_user - copy directly from DMA-able memory to userland 277 * 278 * Return values: 279 * 0: Success 280 * Else: Failure 281 *------------------------------------------------------------------------*/ 282 #if USB_HAVE_USER_IO 283 int 284 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 285 void *ptr, usb_frlength_t len) 286 { 287 struct usb_page_search res; 288 int error; 289 290 while (len != 0) { 291 292 usbd_get_page(cache, offset, &res); 293 294 if (res.length > len) { 295 res.length = len; 296 } 297 error = copyout(res.buffer, ptr, res.length); 298 if (error) 299 return (error); 300 301 offset += res.length; 302 len -= res.length; 303 ptr = USB_ADD_BYTES(ptr, res.length); 304 } 305 return (0); /* success */ 306 } 307 #endif 308 309 /*------------------------------------------------------------------------* 310 * usbd_frame_zero - zero DMA-able memory 311 *------------------------------------------------------------------------*/ 312 void 313 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 314 usb_frlength_t len) 315 { 316 struct usb_page_search res; 317 318 while (len != 0) { 319 320 usbd_get_page(cache, offset, &res); 321 322 if (res.length > len) { 323 res.length = len; 324 } 325 memset(res.buffer, 0, res.length); 326 327 offset += res.length; 328 len -= res.length; 329 } 330 } 331 332 #if USB_HAVE_BUSDMA 333 334 /*------------------------------------------------------------------------* 335 * usb_dma_lock_cb - dummy callback 336 *------------------------------------------------------------------------*/ 337 #if 0 338 static void 339 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 340 { 341 /* we use "mtx_owned()" instead of this function */ 342 } 343 #endif 344 345 /*------------------------------------------------------------------------* 346 * usb_dma_tag_create - allocate a DMA tag 347 * 348 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 349 * allow multi-segment mappings. Else all mappings are single-segment. 350 *------------------------------------------------------------------------*/ 351 static void 352 usb_dma_tag_create(struct usb_dma_tag *udt, 353 usb_size_t size, usb_size_t align) 354 { 355 bus_dma_tag_t tag; 356 357 if (bus_dma_tag_create 358 ( /* parent */ udt->tag_parent->tag, 359 /* alignment */ align, 360 /* boundary */ 0, 361 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 362 /* highaddr */ BUS_SPACE_MAXADDR, 363 /* maxsize */ size, 364 /* nsegments */ (align == 1 && size > 1) ? (2 + (size / USB_PAGE_SIZE)) : 1, 365 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? USB_PAGE_SIZE : size, 366 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 367 &tag)) { 368 tag = NULL; 369 } 370 udt->tag = tag; 371 } 372 373 /*------------------------------------------------------------------------* 374 * usb_dma_tag_free - free a DMA tag 375 *------------------------------------------------------------------------*/ 376 static void 377 usb_dma_tag_destroy(struct usb_dma_tag *udt) 378 { 379 bus_dma_tag_destroy(udt->tag); 380 } 381 382 /*------------------------------------------------------------------------* 383 * usb_pc_alloc_mem_cb - BUS-DMA callback function 384 *------------------------------------------------------------------------*/ 385 static void 386 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 387 int nseg, int error) 388 { 389 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 390 } 391 392 /*------------------------------------------------------------------------* 393 * usb_pc_load_mem_cb - BUS-DMA callback function 394 *------------------------------------------------------------------------*/ 395 static void 396 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 397 int nseg, int error) 398 { 399 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 400 } 401 402 /*------------------------------------------------------------------------* 403 * usb_pc_common_mem_cb - BUS-DMA callback function 404 *------------------------------------------------------------------------*/ 405 static void 406 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 407 int nseg, int error, uint8_t isload) 408 { 409 struct usb_dma_parent_tag *uptag; 410 struct usb_page_cache *pc; 411 struct usb_page *pg; 412 usb_size_t rem; 413 bus_size_t off; 414 uint8_t owned; 415 416 pc = arg; 417 uptag = pc->tag_parent; 418 419 /* 420 * XXX There is sometimes recursive locking here. 421 * XXX We should try to find a better solution. 422 * XXX Until further the "owned" variable does 423 * XXX the trick. 424 */ 425 426 if (error) { 427 goto done; 428 } 429 430 off = 0; 431 pg = pc->page_start; 432 pg->physaddr = rounddown2(segs->ds_addr, USB_PAGE_SIZE); 433 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 434 pc->page_offset_buf = rem; 435 pc->page_offset_end += rem; 436 #ifdef USB_DEBUG 437 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 438 /* 439 * This check verifies that the physical address is correct: 440 */ 441 DPRINTFN(0, "Page offset was not preserved\n"); 442 error = 1; 443 goto done; 444 } 445 #endif 446 while (pc->ismultiseg) { 447 off += USB_PAGE_SIZE; 448 if (off >= (segs->ds_len + rem)) { 449 /* page crossing */ 450 nseg--; 451 segs++; 452 off = 0; 453 rem = 0; 454 if (nseg == 0) 455 break; 456 } 457 pg++; 458 pg->physaddr = rounddown2(segs->ds_addr + off, USB_PAGE_SIZE); 459 } 460 461 done: 462 owned = lockowned(uptag->lock); 463 if (!owned) 464 lockmgr(uptag->lock, LK_EXCLUSIVE); 465 466 uptag->dma_error = (error ? 1 : 0); 467 if (isload) { 468 (uptag->func) (uptag); 469 } else { 470 cv_broadcast(uptag->cv); 471 } 472 if (!owned) 473 lockmgr(uptag->lock, LK_RELEASE); 474 } 475 476 /*------------------------------------------------------------------------* 477 * usb_pc_alloc_mem - allocate DMA'able memory 478 * 479 * Returns: 480 * 0: Success 481 * Else: Failure 482 *------------------------------------------------------------------------*/ 483 uint8_t 484 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 485 usb_size_t size, usb_size_t align) 486 { 487 struct usb_dma_parent_tag *uptag; 488 struct usb_dma_tag *utag; 489 bus_dmamap_t map; 490 void *ptr; 491 int err; 492 493 uptag = pc->tag_parent; 494 495 if (align != 1) { 496 /* 497 * The alignment must be greater or equal to the 498 * "size" else the object can be split between two 499 * memory pages and we get a problem! 500 */ 501 while (align < size) { 502 align *= 2; 503 if (align == 0) { 504 goto error; 505 } 506 } 507 #if 1 508 /* 509 * XXX BUS-DMA workaround - FIXME later: 510 * 511 * We assume that that the aligment at this point of 512 * the code is greater than or equal to the size and 513 * less than two times the size, so that if we double 514 * the size, the size will be greater than the 515 * alignment. 516 * 517 * The bus-dma system has a check for "alignment" 518 * being less than "size". If that check fails we end 519 * up using contigmalloc which is page based even for 520 * small allocations. Try to avoid that to save 521 * memory, hence we sometimes to a large number of 522 * small allocations! 523 */ 524 if (size <= (USB_PAGE_SIZE / 2)) { 525 size *= 2; 526 } 527 #endif 528 } 529 /* get the correct DMA tag */ 530 utag = usb_dma_tag_find(uptag, size, align); 531 if (utag == NULL) { 532 goto error; 533 } 534 /* allocate memory */ 535 if (bus_dmamem_alloc( 536 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 537 goto error; 538 } 539 /* setup page cache */ 540 pc->buffer = ptr; 541 pc->page_start = pg; 542 pc->page_offset_buf = 0; 543 pc->page_offset_end = size; 544 pc->map = map; 545 pc->tag = utag->tag; 546 pc->ismultiseg = (align == 1); 547 548 lockmgr(uptag->lock, LK_EXCLUSIVE); 549 550 /* load memory into DMA */ 551 err = bus_dmamap_load( 552 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 553 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 554 555 if (err == EINPROGRESS) { 556 cv_wait(uptag->cv, uptag->lock); 557 err = 0; 558 } 559 lockmgr(uptag->lock, LK_RELEASE); 560 561 if (err || uptag->dma_error) { 562 bus_dmamem_free(utag->tag, ptr, map); 563 goto error; 564 } 565 memset(ptr, 0, size); 566 567 usb_pc_cpu_flush(pc); 568 569 return (0); 570 571 error: 572 /* reset most of the page cache */ 573 pc->buffer = NULL; 574 pc->page_start = NULL; 575 pc->page_offset_buf = 0; 576 pc->page_offset_end = 0; 577 pc->map = NULL; 578 pc->tag = NULL; 579 return (1); 580 } 581 582 /*------------------------------------------------------------------------* 583 * usb_pc_free_mem - free DMA memory 584 * 585 * This function is NULL safe. 586 *------------------------------------------------------------------------*/ 587 void 588 usb_pc_free_mem(struct usb_page_cache *pc) 589 { 590 if (pc && pc->buffer) { 591 592 bus_dmamap_unload(pc->tag, pc->map); 593 594 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 595 596 pc->buffer = NULL; 597 } 598 } 599 600 /*------------------------------------------------------------------------* 601 * usb_pc_load_mem - load virtual memory into DMA 602 * 603 * Return values: 604 * 0: Success 605 * Else: Error 606 *------------------------------------------------------------------------*/ 607 uint8_t 608 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 609 { 610 /* setup page cache */ 611 pc->page_offset_buf = 0; 612 pc->page_offset_end = size; 613 pc->ismultiseg = 1; 614 615 KKASSERT(lockowned(pc->tag_parent->lock)); 616 617 if (size > 0) { 618 if (sync) { 619 struct usb_dma_parent_tag *uptag; 620 int err; 621 622 uptag = pc->tag_parent; 623 624 /* 625 * We have to unload the previous loaded DMA 626 * pages before trying to load a new one! 627 */ 628 bus_dmamap_unload(pc->tag, pc->map); 629 630 /* 631 * Try to load memory into DMA. 632 */ 633 err = bus_dmamap_load( 634 pc->tag, pc->map, pc->buffer, size, 635 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 636 if (err == EINPROGRESS) { 637 cv_wait(uptag->cv, uptag->lock); 638 err = 0; 639 } 640 if (err || uptag->dma_error) { 641 return (1); 642 } 643 } else { 644 645 /* 646 * We have to unload the previous loaded DMA 647 * pages before trying to load a new one! 648 */ 649 bus_dmamap_unload(pc->tag, pc->map); 650 651 /* 652 * Try to load memory into DMA. The callback 653 * will be called in all cases: 654 */ 655 if (bus_dmamap_load( 656 pc->tag, pc->map, pc->buffer, size, 657 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 658 } 659 } 660 } else { 661 if (!sync) { 662 /* 663 * Call callback so that refcount is decremented 664 * properly: 665 */ 666 pc->tag_parent->dma_error = 0; 667 (pc->tag_parent->func) (pc->tag_parent); 668 } 669 } 670 return (0); 671 } 672 673 /*------------------------------------------------------------------------* 674 * usb_pc_cpu_invalidate - invalidate CPU cache 675 *------------------------------------------------------------------------*/ 676 void 677 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 678 { 679 if (pc->page_offset_end == pc->page_offset_buf) { 680 /* nothing has been loaded into this page cache! */ 681 return; 682 } 683 684 /* 685 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the 686 * same time, but in the future we should try to isolate the 687 * different cases to optimise the code. --HPS 688 */ 689 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); 690 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); 691 } 692 693 /*------------------------------------------------------------------------* 694 * usb_pc_cpu_flush - flush CPU cache 695 *------------------------------------------------------------------------*/ 696 void 697 usb_pc_cpu_flush(struct usb_page_cache *pc) 698 { 699 if (pc->page_offset_end == pc->page_offset_buf) { 700 /* nothing has been loaded into this page cache! */ 701 return; 702 } 703 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); 704 } 705 706 /*------------------------------------------------------------------------* 707 * usb_pc_dmamap_create - create a DMA map 708 * 709 * Returns: 710 * 0: Success 711 * Else: Failure 712 *------------------------------------------------------------------------*/ 713 uint8_t 714 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 715 { 716 struct usb_xfer_root *info; 717 struct usb_dma_tag *utag; 718 719 /* get info */ 720 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 721 722 /* sanity check */ 723 if (info == NULL) { 724 goto error; 725 } 726 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 727 if (utag == NULL) { 728 goto error; 729 } 730 /* create DMA map */ 731 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 732 goto error; 733 } 734 pc->tag = utag->tag; 735 return 0; /* success */ 736 737 error: 738 pc->map = NULL; 739 pc->tag = NULL; 740 return 1; /* failure */ 741 } 742 743 /*------------------------------------------------------------------------* 744 * usb_pc_dmamap_destroy 745 * 746 * This function is NULL safe. 747 *------------------------------------------------------------------------*/ 748 void 749 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 750 { 751 if (pc && pc->tag) { 752 bus_dmamap_destroy(pc->tag, pc->map); 753 pc->tag = NULL; 754 pc->map = NULL; 755 } 756 } 757 758 /*------------------------------------------------------------------------* 759 * usb_dma_tag_find - factored out code 760 *------------------------------------------------------------------------*/ 761 struct usb_dma_tag * 762 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 763 usb_size_t size, usb_size_t align) 764 { 765 struct usb_dma_tag *udt; 766 uint8_t nudt; 767 768 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n")); 769 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n")); 770 771 udt = udpt->utag_first; 772 nudt = udpt->utag_max; 773 774 while (nudt--) { 775 776 if (udt->align == 0) { 777 usb_dma_tag_create(udt, size, align); 778 if (udt->tag == NULL) { 779 return (NULL); 780 } 781 udt->align = align; 782 udt->size = size; 783 return (udt); 784 } 785 if ((udt->align == align) && (udt->size == size)) { 786 return (udt); 787 } 788 udt++; 789 } 790 return (NULL); 791 } 792 793 /*------------------------------------------------------------------------* 794 * usb_dma_tag_setup - initialise USB DMA tags 795 *------------------------------------------------------------------------*/ 796 void 797 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 798 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 799 struct lock *lock, usb_dma_callback_t *func, 800 uint8_t ndmabits, uint8_t nudt) 801 { 802 memset(udpt, 0, sizeof(*udpt)); 803 804 /* sanity checking */ 805 if ((nudt == 0) || 806 (ndmabits == 0) || 807 (lock == NULL)) { 808 /* something is corrupt */ 809 return; 810 } 811 /* initialise condition variable */ 812 cv_init(udpt->cv, "USB DMA CV"); 813 814 /* store some information */ 815 udpt->lock = lock; 816 udpt->func = func; 817 udpt->tag = dmat; 818 udpt->utag_first = udt; 819 udpt->utag_max = nudt; 820 udpt->dma_bits = ndmabits; 821 822 while (nudt--) { 823 memset(udt, 0, sizeof(*udt)); 824 udt->tag_parent = udpt; 825 udt++; 826 } 827 } 828 829 /*------------------------------------------------------------------------* 830 * usb_bus_tag_unsetup - factored out code 831 *------------------------------------------------------------------------*/ 832 void 833 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 834 { 835 struct usb_dma_tag *udt; 836 uint8_t nudt; 837 838 udt = udpt->utag_first; 839 nudt = udpt->utag_max; 840 841 while (nudt--) { 842 843 if (udt->align) { 844 /* destroy the USB DMA tag */ 845 usb_dma_tag_destroy(udt); 846 udt->align = 0; 847 } 848 udt++; 849 } 850 851 if (udpt->utag_max) { 852 /* destroy the condition variable */ 853 cv_destroy(udpt->cv); 854 } 855 } 856 857 /*------------------------------------------------------------------------* 858 * usb_bdma_work_loop 859 * 860 * This function handles loading of virtual buffers into DMA and is 861 * only called when "dma_refcount" is zero. 862 *------------------------------------------------------------------------*/ 863 void 864 usb_bdma_work_loop(struct usb_xfer_queue *pq) 865 { 866 struct usb_xfer_root *info; 867 struct usb_xfer *xfer; 868 usb_frcount_t nframes; 869 870 xfer = pq->curr; 871 info = xfer->xroot; 872 873 KKASSERT(lockowned(info->xfer_lock)); 874 875 if (xfer->error) { 876 /* some error happened */ 877 USB_BUS_LOCK(info->bus); 878 usbd_transfer_done(xfer, 0); 879 USB_BUS_UNLOCK(info->bus); 880 return; 881 } 882 if (!xfer->flags_int.bdma_setup) { 883 struct usb_page *pg; 884 usb_frlength_t frlength_0; 885 uint8_t isread; 886 887 xfer->flags_int.bdma_setup = 1; 888 889 /* reset BUS-DMA load state */ 890 891 info->dma_error = 0; 892 893 if (xfer->flags_int.isochronous_xfr) { 894 /* only one frame buffer */ 895 nframes = 1; 896 frlength_0 = xfer->sumlen; 897 } else { 898 /* can be multiple frame buffers */ 899 nframes = xfer->nframes; 900 frlength_0 = xfer->frlengths[0]; 901 } 902 903 /* 904 * Set DMA direction first. This is needed to 905 * select the correct cache invalidate and cache 906 * flush operations. 907 */ 908 isread = USB_GET_DATA_ISREAD(xfer); 909 pg = xfer->dma_page_ptr; 910 911 if (xfer->flags_int.control_xfr && 912 xfer->flags_int.control_hdr) { 913 /* special case */ 914 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 915 /* The device controller writes to memory */ 916 xfer->frbuffers[0].isread = 1; 917 } else { 918 /* The host controller reads from memory */ 919 xfer->frbuffers[0].isread = 0; 920 } 921 } else { 922 /* default case */ 923 xfer->frbuffers[0].isread = isread; 924 } 925 926 /* 927 * Setup the "page_start" pointer which points to an array of 928 * USB pages where information about the physical address of a 929 * page will be stored. Also initialise the "isread" field of 930 * the USB page caches. 931 */ 932 xfer->frbuffers[0].page_start = pg; 933 934 info->dma_nframes = nframes; 935 info->dma_currframe = 0; 936 info->dma_frlength_0 = frlength_0; 937 938 pg += (frlength_0 / USB_PAGE_SIZE); 939 pg += 2; 940 941 while (--nframes > 0) { 942 xfer->frbuffers[nframes].isread = isread; 943 xfer->frbuffers[nframes].page_start = pg; 944 945 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 946 pg += 2; 947 } 948 949 } 950 if (info->dma_error) { 951 USB_BUS_LOCK(info->bus); 952 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 953 USB_BUS_UNLOCK(info->bus); 954 return; 955 } 956 if (info->dma_currframe != info->dma_nframes) { 957 958 if (info->dma_currframe == 0) { 959 /* special case */ 960 usb_pc_load_mem(xfer->frbuffers, 961 info->dma_frlength_0, 0); 962 } else { 963 /* default case */ 964 nframes = info->dma_currframe; 965 usb_pc_load_mem(xfer->frbuffers + nframes, 966 xfer->frlengths[nframes], 0); 967 } 968 969 /* advance frame index */ 970 info->dma_currframe++; 971 972 return; 973 } 974 /* go ahead */ 975 usb_bdma_pre_sync(xfer); 976 977 /* start loading next USB transfer, if any */ 978 usb_command_wrapper(pq, NULL); 979 980 /* finally start the hardware */ 981 usbd_pipe_enter(xfer); 982 } 983 984 /*------------------------------------------------------------------------* 985 * usb_bdma_done_event 986 * 987 * This function is called when the BUS-DMA has loaded virtual memory 988 * into DMA, if any. 989 *------------------------------------------------------------------------*/ 990 void 991 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 992 { 993 struct usb_xfer_root *info; 994 995 info = USB_DMATAG_TO_XROOT(udpt); 996 997 KKASSERT(lockowned(info->xfer_lock)); 998 999 /* copy error */ 1000 info->dma_error = udpt->dma_error; 1001 1002 /* enter workloop again */ 1003 usb_command_wrapper(&info->dma_q, 1004 info->dma_q.curr); 1005 } 1006 1007 /*------------------------------------------------------------------------* 1008 * usb_bdma_pre_sync 1009 * 1010 * This function handles DMA synchronisation that must be done before 1011 * an USB transfer is started. 1012 *------------------------------------------------------------------------*/ 1013 void 1014 usb_bdma_pre_sync(struct usb_xfer *xfer) 1015 { 1016 struct usb_page_cache *pc; 1017 usb_frcount_t nframes; 1018 1019 if (xfer->flags_int.isochronous_xfr) { 1020 /* only one frame buffer */ 1021 nframes = 1; 1022 } else { 1023 /* can be multiple frame buffers */ 1024 nframes = xfer->nframes; 1025 } 1026 1027 pc = xfer->frbuffers; 1028 1029 while (nframes--) { 1030 1031 if (pc->isread) { 1032 usb_pc_cpu_invalidate(pc); 1033 } else { 1034 usb_pc_cpu_flush(pc); 1035 } 1036 pc++; 1037 } 1038 } 1039 1040 /*------------------------------------------------------------------------* 1041 * usb_bdma_post_sync 1042 * 1043 * This function handles DMA synchronisation that must be done after 1044 * an USB transfer is complete. 1045 *------------------------------------------------------------------------*/ 1046 void 1047 usb_bdma_post_sync(struct usb_xfer *xfer) 1048 { 1049 struct usb_page_cache *pc; 1050 usb_frcount_t nframes; 1051 1052 if (xfer->flags_int.isochronous_xfr) { 1053 /* only one frame buffer */ 1054 nframes = 1; 1055 } else { 1056 /* can be multiple frame buffers */ 1057 nframes = xfer->nframes; 1058 } 1059 1060 pc = xfer->frbuffers; 1061 1062 while (nframes--) { 1063 if (pc->isread) { 1064 usb_pc_cpu_invalidate(pc); 1065 } 1066 pc++; 1067 } 1068 } 1069 1070 #endif 1071