1 /* $FreeBSD: head/sys/dev/usb/usb_busdma.c 261505 2014-02-05 08:02:52Z hselasky $ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/stdint.h> 28 #include <sys/param.h> 29 #include <sys/queue.h> 30 #include <sys/types.h> 31 #include <sys/systm.h> 32 #include <sys/kernel.h> 33 #include <sys/bus.h> 34 #include <sys/module.h> 35 #include <sys/lock.h> 36 #include <sys/mutex.h> 37 #include <sys/mutex2.h> 38 #include <sys/condvar.h> 39 #include <sys/sysctl.h> 40 #include <sys/unistd.h> 41 #include <sys/callout.h> 42 #include <sys/malloc.h> 43 #include <sys/priv.h> 44 45 #include <bus/u4b/usb.h> 46 #include <bus/u4b/usbdi.h> 47 #include <bus/u4b/usbdi_util.h> 48 49 #define USB_DEBUG_VAR usb_debug 50 51 #include <bus/u4b/usb_core.h> 52 #include <bus/u4b/usb_busdma.h> 53 #include <bus/u4b/usb_process.h> 54 #include <bus/u4b/usb_transfer.h> 55 #include <bus/u4b/usb_device.h> 56 #include <bus/u4b/usb_util.h> 57 #include <bus/u4b/usb_debug.h> 58 59 #include <bus/u4b/usb_controller.h> 60 #include <bus/u4b/usb_bus.h> 61 62 #if USB_HAVE_BUSDMA 63 static void usb_dma_tag_create(struct usb_dma_tag *, usb_size_t, usb_size_t); 64 static void usb_dma_tag_destroy(struct usb_dma_tag *); 65 #if 0 66 static void usb_dma_lock_cb(void *, bus_dma_lock_op_t); 67 #endif 68 static void usb_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 69 static void usb_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 70 static void usb_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 71 uint8_t); 72 #endif 73 74 /*------------------------------------------------------------------------* 75 * usbd_get_page - lookup DMA-able memory for the given offset 76 * 77 * NOTE: Only call this function when the "page_cache" structure has 78 * been properly initialized ! 79 *------------------------------------------------------------------------*/ 80 void 81 usbd_get_page(struct usb_page_cache *pc, usb_frlength_t offset, 82 struct usb_page_search *res) 83 { 84 #if USB_HAVE_BUSDMA 85 struct usb_page *page; 86 87 if (pc->page_start) { 88 89 /* Case 1 - something has been loaded into DMA */ 90 91 if (pc->buffer) { 92 93 /* Case 1a - Kernel Virtual Address */ 94 95 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 96 } 97 offset += pc->page_offset_buf; 98 99 /* compute destination page */ 100 101 page = pc->page_start; 102 103 if (pc->ismultiseg) { 104 105 page += (offset / USB_PAGE_SIZE); 106 107 offset %= USB_PAGE_SIZE; 108 109 res->length = USB_PAGE_SIZE - offset; 110 res->physaddr = page->physaddr + offset; 111 } else { 112 res->length = (usb_size_t)-1; 113 res->physaddr = page->physaddr + offset; 114 } 115 if (!pc->buffer) { 116 117 /* Case 1b - Non Kernel Virtual Address */ 118 119 res->buffer = USB_ADD_BYTES(page->buffer, offset); 120 } 121 return; 122 } 123 #endif 124 /* Case 2 - Plain PIO */ 125 126 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 127 res->length = (usb_size_t)-1; 128 #if USB_HAVE_BUSDMA 129 res->physaddr = 0; 130 #endif 131 } 132 133 /*------------------------------------------------------------------------* 134 * usbd_copy_in - copy directly to DMA-able memory 135 *------------------------------------------------------------------------*/ 136 void 137 usbd_copy_in(struct usb_page_cache *cache, usb_frlength_t offset, 138 const void *ptr, usb_frlength_t len) 139 { 140 struct usb_page_search buf_res; 141 142 while (len != 0) { 143 144 usbd_get_page(cache, offset, &buf_res); 145 146 if (buf_res.length > len) { 147 buf_res.length = len; 148 } 149 memcpy(buf_res.buffer, ptr, buf_res.length); 150 151 offset += buf_res.length; 152 len -= buf_res.length; 153 ptr = USB_ADD_BYTES(ptr, buf_res.length); 154 } 155 } 156 157 /*------------------------------------------------------------------------* 158 * usbd_copy_in_user - copy directly to DMA-able memory from userland 159 * 160 * Return values: 161 * 0: Success 162 * Else: Failure 163 *------------------------------------------------------------------------*/ 164 #if USB_HAVE_USER_IO 165 int 166 usbd_copy_in_user(struct usb_page_cache *cache, usb_frlength_t offset, 167 const void *ptr, usb_frlength_t len) 168 { 169 struct usb_page_search buf_res; 170 int error; 171 172 while (len != 0) { 173 174 usbd_get_page(cache, offset, &buf_res); 175 176 if (buf_res.length > len) { 177 buf_res.length = len; 178 } 179 error = copyin(ptr, buf_res.buffer, buf_res.length); 180 if (error) 181 return (error); 182 183 offset += buf_res.length; 184 len -= buf_res.length; 185 ptr = USB_ADD_BYTES(ptr, buf_res.length); 186 } 187 return (0); /* success */ 188 } 189 #endif 190 191 /*------------------------------------------------------------------------* 192 * usbd_m_copy_in - copy a mbuf chain directly into DMA-able memory 193 *------------------------------------------------------------------------*/ 194 #if USB_HAVE_MBUF 195 struct usb_m_copy_in_arg { 196 struct usb_page_cache *cache; 197 usb_frlength_t dst_offset; 198 }; 199 200 static int 201 usbd_m_copy_in_cb(void *arg, void *src, uint32_t count) 202 { 203 register struct usb_m_copy_in_arg *ua = arg; 204 205 usbd_copy_in(ua->cache, ua->dst_offset, src, count); 206 ua->dst_offset += count; 207 return (0); 208 } 209 210 void 211 usbd_m_copy_in(struct usb_page_cache *cache, usb_frlength_t dst_offset, 212 struct mbuf *m, usb_size_t src_offset, usb_frlength_t src_len) 213 { 214 struct usb_m_copy_in_arg arg = {cache, dst_offset}; 215 (void) m_apply(m, src_offset, src_len, &usbd_m_copy_in_cb, &arg); 216 } 217 #endif 218 219 /*------------------------------------------------------------------------* 220 * usb_uiomove - factored out code 221 *------------------------------------------------------------------------*/ 222 #if USB_HAVE_USER_IO 223 int 224 usb_uiomove(struct usb_page_cache *pc, struct uio *uio, 225 usb_frlength_t pc_offset, usb_frlength_t len) 226 { 227 struct usb_page_search res; 228 int error = 0; 229 230 while (len != 0) { 231 232 usbd_get_page(pc, pc_offset, &res); 233 234 if (res.length > len) { 235 res.length = len; 236 } 237 /* 238 * "uiomove()" can sleep so one needs to make a wrapper, 239 * exiting the mutex and checking things 240 */ 241 error = uiomove(res.buffer, res.length, uio); 242 243 if (error) { 244 break; 245 } 246 pc_offset += res.length; 247 len -= res.length; 248 } 249 return (error); 250 } 251 #endif 252 253 /*------------------------------------------------------------------------* 254 * usbd_copy_out - copy directly from DMA-able memory 255 *------------------------------------------------------------------------*/ 256 void 257 usbd_copy_out(struct usb_page_cache *cache, usb_frlength_t offset, 258 void *ptr, usb_frlength_t len) 259 { 260 struct usb_page_search res; 261 262 while (len != 0) { 263 264 usbd_get_page(cache, offset, &res); 265 266 if (res.length > len) { 267 res.length = len; 268 } 269 memcpy(ptr, res.buffer, res.length); 270 271 offset += res.length; 272 len -= res.length; 273 ptr = USB_ADD_BYTES(ptr, res.length); 274 } 275 } 276 277 /*------------------------------------------------------------------------* 278 * usbd_copy_out_user - copy directly from DMA-able memory to userland 279 * 280 * Return values: 281 * 0: Success 282 * Else: Failure 283 *------------------------------------------------------------------------*/ 284 #if USB_HAVE_USER_IO 285 int 286 usbd_copy_out_user(struct usb_page_cache *cache, usb_frlength_t offset, 287 void *ptr, usb_frlength_t len) 288 { 289 struct usb_page_search res; 290 int error; 291 292 while (len != 0) { 293 294 usbd_get_page(cache, offset, &res); 295 296 if (res.length > len) { 297 res.length = len; 298 } 299 error = copyout(res.buffer, ptr, res.length); 300 if (error) 301 return (error); 302 303 offset += res.length; 304 len -= res.length; 305 ptr = USB_ADD_BYTES(ptr, res.length); 306 } 307 return (0); /* success */ 308 } 309 #endif 310 311 /*------------------------------------------------------------------------* 312 * usbd_frame_zero - zero DMA-able memory 313 *------------------------------------------------------------------------*/ 314 void 315 usbd_frame_zero(struct usb_page_cache *cache, usb_frlength_t offset, 316 usb_frlength_t len) 317 { 318 struct usb_page_search res; 319 320 while (len != 0) { 321 322 usbd_get_page(cache, offset, &res); 323 324 if (res.length > len) { 325 res.length = len; 326 } 327 memset(res.buffer, 0, res.length); 328 329 offset += res.length; 330 len -= res.length; 331 } 332 } 333 334 #if USB_HAVE_BUSDMA 335 336 /*------------------------------------------------------------------------* 337 * usb_dma_lock_cb - dummy callback 338 *------------------------------------------------------------------------*/ 339 #if 0 340 static void 341 usb_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 342 { 343 /* we use "mtx_owned()" instead of this function */ 344 } 345 #endif 346 347 /*------------------------------------------------------------------------* 348 * usb_dma_tag_create - allocate a DMA tag 349 * 350 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 351 * allow multi-segment mappings. Else all mappings are single-segment. 352 *------------------------------------------------------------------------*/ 353 static void 354 usb_dma_tag_create(struct usb_dma_tag *udt, 355 usb_size_t size, usb_size_t align) 356 { 357 bus_dma_tag_t tag; 358 359 if (bus_dma_tag_create 360 ( /* parent */ udt->tag_parent->tag, 361 /* alignment */ align, 362 /* boundary */ 0, 363 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 364 /* highaddr */ BUS_SPACE_MAXADDR, 365 /* filter */ NULL, 366 /* filterarg */ NULL, 367 /* maxsize */ size, 368 /* nsegments */ (align == 1 && size > 1) ? (2 + (size / USB_PAGE_SIZE)) : 1, 369 /* maxsegsz */ (align == 1 && size > USB_PAGE_SIZE) ? USB_PAGE_SIZE : size, 370 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 371 &tag)) { 372 tag = NULL; 373 } 374 udt->tag = tag; 375 } 376 377 /*------------------------------------------------------------------------* 378 * usb_dma_tag_free - free a DMA tag 379 *------------------------------------------------------------------------*/ 380 static void 381 usb_dma_tag_destroy(struct usb_dma_tag *udt) 382 { 383 bus_dma_tag_destroy(udt->tag); 384 } 385 386 /*------------------------------------------------------------------------* 387 * usb_pc_alloc_mem_cb - BUS-DMA callback function 388 *------------------------------------------------------------------------*/ 389 static void 390 usb_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 391 int nseg, int error) 392 { 393 usb_pc_common_mem_cb(arg, segs, nseg, error, 0); 394 } 395 396 /*------------------------------------------------------------------------* 397 * usb_pc_load_mem_cb - BUS-DMA callback function 398 *------------------------------------------------------------------------*/ 399 static void 400 usb_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 401 int nseg, int error) 402 { 403 usb_pc_common_mem_cb(arg, segs, nseg, error, 1); 404 } 405 406 /*------------------------------------------------------------------------* 407 * usb_pc_common_mem_cb - BUS-DMA callback function 408 *------------------------------------------------------------------------*/ 409 static void 410 usb_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 411 int nseg, int error, uint8_t isload) 412 { 413 struct usb_dma_parent_tag *uptag; 414 struct usb_page_cache *pc; 415 struct usb_page *pg; 416 usb_size_t rem; 417 bus_size_t off; 418 uint8_t owned; 419 420 pc = arg; 421 uptag = pc->tag_parent; 422 423 /* 424 * XXX There is sometimes recursive locking here. 425 * XXX We should try to find a better solution. 426 * XXX Until further the "owned" variable does 427 * XXX the trick. 428 */ 429 430 if (error) { 431 goto done; 432 } 433 434 off = 0; 435 pg = pc->page_start; 436 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 437 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 438 pc->page_offset_buf = rem; 439 pc->page_offset_end += rem; 440 #ifdef USB_DEBUG 441 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 442 /* 443 * This check verifies that the physical address is correct: 444 */ 445 DPRINTFN(0, "Page offset was not preserved\n"); 446 error = 1; 447 goto done; 448 } 449 #endif 450 while (pc->ismultiseg) { 451 off += USB_PAGE_SIZE; 452 if (off >= (segs->ds_len + rem)) { 453 /* page crossing */ 454 nseg--; 455 segs++; 456 off = 0; 457 rem = 0; 458 if (nseg == 0) 459 break; 460 } 461 pg++; 462 pg->physaddr = (segs->ds_addr + off) & ~(USB_PAGE_SIZE - 1); 463 } 464 465 done: 466 owned = lockowned(uptag->lock); 467 if (!owned) 468 lockmgr(uptag->lock, LK_EXCLUSIVE); 469 470 uptag->dma_error = (error ? 1 : 0); 471 if (isload) { 472 (uptag->func) (uptag); 473 } else { 474 cv_broadcast(uptag->cv); 475 } 476 if (!owned) 477 lockmgr(uptag->lock, LK_RELEASE); 478 } 479 480 /*------------------------------------------------------------------------* 481 * usb_pc_alloc_mem - allocate DMA'able memory 482 * 483 * Returns: 484 * 0: Success 485 * Else: Failure 486 *------------------------------------------------------------------------*/ 487 uint8_t 488 usb_pc_alloc_mem(struct usb_page_cache *pc, struct usb_page *pg, 489 usb_size_t size, usb_size_t align) 490 { 491 struct usb_dma_parent_tag *uptag; 492 struct usb_dma_tag *utag; 493 bus_dmamap_t map; 494 void *ptr; 495 int err; 496 497 uptag = pc->tag_parent; 498 499 if (align != 1) { 500 /* 501 * The alignment must be greater or equal to the 502 * "size" else the object can be split between two 503 * memory pages and we get a problem! 504 */ 505 while (align < size) { 506 align *= 2; 507 if (align == 0) { 508 goto error; 509 } 510 } 511 #if 1 512 /* 513 * XXX BUS-DMA workaround - FIXME later: 514 * 515 * We assume that that the aligment at this point of 516 * the code is greater than or equal to the size and 517 * less than two times the size, so that if we double 518 * the size, the size will be greater than the 519 * alignment. 520 * 521 * The bus-dma system has a check for "alignment" 522 * being less than "size". If that check fails we end 523 * up using contigmalloc which is page based even for 524 * small allocations. Try to avoid that to save 525 * memory, hence we sometimes to a large number of 526 * small allocations! 527 */ 528 if (size <= (USB_PAGE_SIZE / 2)) { 529 size *= 2; 530 } 531 #endif 532 } 533 /* get the correct DMA tag */ 534 utag = usb_dma_tag_find(uptag, size, align); 535 if (utag == NULL) { 536 goto error; 537 } 538 /* allocate memory */ 539 if (bus_dmamem_alloc( 540 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 541 goto error; 542 } 543 /* setup page cache */ 544 pc->buffer = ptr; 545 pc->page_start = pg; 546 pc->page_offset_buf = 0; 547 pc->page_offset_end = size; 548 pc->map = map; 549 pc->tag = utag->tag; 550 pc->ismultiseg = (align == 1); 551 552 lockmgr(uptag->lock, LK_EXCLUSIVE); 553 554 /* load memory into DMA */ 555 err = bus_dmamap_load( 556 utag->tag, map, ptr, size, &usb_pc_alloc_mem_cb, 557 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 558 559 if (err == EINPROGRESS) { 560 cv_wait(uptag->cv, uptag->lock); 561 err = 0; 562 } 563 lockmgr(uptag->lock, LK_RELEASE); 564 565 if (err || uptag->dma_error) { 566 bus_dmamem_free(utag->tag, ptr, map); 567 goto error; 568 } 569 memset(ptr, 0, size); 570 571 usb_pc_cpu_flush(pc); 572 573 return (0); 574 575 error: 576 /* reset most of the page cache */ 577 pc->buffer = NULL; 578 pc->page_start = NULL; 579 pc->page_offset_buf = 0; 580 pc->page_offset_end = 0; 581 pc->map = NULL; 582 pc->tag = NULL; 583 return (1); 584 } 585 586 /*------------------------------------------------------------------------* 587 * usb_pc_free_mem - free DMA memory 588 * 589 * This function is NULL safe. 590 *------------------------------------------------------------------------*/ 591 void 592 usb_pc_free_mem(struct usb_page_cache *pc) 593 { 594 if (pc && pc->buffer) { 595 596 bus_dmamap_unload(pc->tag, pc->map); 597 598 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 599 600 pc->buffer = NULL; 601 } 602 } 603 604 /*------------------------------------------------------------------------* 605 * usb_pc_load_mem - load virtual memory into DMA 606 * 607 * Return values: 608 * 0: Success 609 * Else: Error 610 *------------------------------------------------------------------------*/ 611 uint8_t 612 usb_pc_load_mem(struct usb_page_cache *pc, usb_size_t size, uint8_t sync) 613 { 614 /* setup page cache */ 615 pc->page_offset_buf = 0; 616 pc->page_offset_end = size; 617 pc->ismultiseg = 1; 618 619 KKASSERT(lockowned(pc->tag_parent->lock)); 620 621 if (size > 0) { 622 if (sync) { 623 struct usb_dma_parent_tag *uptag; 624 int err; 625 626 uptag = pc->tag_parent; 627 628 /* 629 * We have to unload the previous loaded DMA 630 * pages before trying to load a new one! 631 */ 632 bus_dmamap_unload(pc->tag, pc->map); 633 634 /* 635 * Try to load memory into DMA. 636 */ 637 err = bus_dmamap_load( 638 pc->tag, pc->map, pc->buffer, size, 639 &usb_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 640 if (err == EINPROGRESS) { 641 cv_wait(uptag->cv, uptag->lock); 642 err = 0; 643 } 644 if (err || uptag->dma_error) { 645 return (1); 646 } 647 } else { 648 649 /* 650 * We have to unload the previous loaded DMA 651 * pages before trying to load a new one! 652 */ 653 bus_dmamap_unload(pc->tag, pc->map); 654 655 /* 656 * Try to load memory into DMA. The callback 657 * will be called in all cases: 658 */ 659 if (bus_dmamap_load( 660 pc->tag, pc->map, pc->buffer, size, 661 &usb_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 662 } 663 } 664 } else { 665 if (!sync) { 666 /* 667 * Call callback so that refcount is decremented 668 * properly: 669 */ 670 pc->tag_parent->dma_error = 0; 671 (pc->tag_parent->func) (pc->tag_parent); 672 } 673 } 674 return (0); 675 } 676 677 /*------------------------------------------------------------------------* 678 * usb_pc_cpu_invalidate - invalidate CPU cache 679 *------------------------------------------------------------------------*/ 680 void 681 usb_pc_cpu_invalidate(struct usb_page_cache *pc) 682 { 683 if (pc->page_offset_end == pc->page_offset_buf) { 684 /* nothing has been loaded into this page cache! */ 685 return; 686 } 687 688 /* 689 * TODO: We currently do XXX_POSTREAD and XXX_PREREAD at the 690 * same time, but in the future we should try to isolate the 691 * different cases to optimise the code. --HPS 692 */ 693 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_POSTREAD); 694 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREREAD); 695 } 696 697 /*------------------------------------------------------------------------* 698 * usb_pc_cpu_flush - flush CPU cache 699 *------------------------------------------------------------------------*/ 700 void 701 usb_pc_cpu_flush(struct usb_page_cache *pc) 702 { 703 if (pc->page_offset_end == pc->page_offset_buf) { 704 /* nothing has been loaded into this page cache! */ 705 return; 706 } 707 bus_dmamap_sync(pc->tag, pc->map, BUS_DMASYNC_PREWRITE); 708 } 709 710 /*------------------------------------------------------------------------* 711 * usb_pc_dmamap_create - create a DMA map 712 * 713 * Returns: 714 * 0: Success 715 * Else: Failure 716 *------------------------------------------------------------------------*/ 717 uint8_t 718 usb_pc_dmamap_create(struct usb_page_cache *pc, usb_size_t size) 719 { 720 struct usb_xfer_root *info; 721 struct usb_dma_tag *utag; 722 723 /* get info */ 724 info = USB_DMATAG_TO_XROOT(pc->tag_parent); 725 726 /* sanity check */ 727 if (info == NULL) { 728 goto error; 729 } 730 utag = usb_dma_tag_find(pc->tag_parent, size, 1); 731 if (utag == NULL) { 732 goto error; 733 } 734 /* create DMA map */ 735 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 736 goto error; 737 } 738 pc->tag = utag->tag; 739 return 0; /* success */ 740 741 error: 742 pc->map = NULL; 743 pc->tag = NULL; 744 return 1; /* failure */ 745 } 746 747 /*------------------------------------------------------------------------* 748 * usb_pc_dmamap_destroy 749 * 750 * This function is NULL safe. 751 *------------------------------------------------------------------------*/ 752 void 753 usb_pc_dmamap_destroy(struct usb_page_cache *pc) 754 { 755 if (pc && pc->tag) { 756 bus_dmamap_destroy(pc->tag, pc->map); 757 pc->tag = NULL; 758 pc->map = NULL; 759 } 760 } 761 762 /*------------------------------------------------------------------------* 763 * usb_dma_tag_find - factored out code 764 *------------------------------------------------------------------------*/ 765 struct usb_dma_tag * 766 usb_dma_tag_find(struct usb_dma_parent_tag *udpt, 767 usb_size_t size, usb_size_t align) 768 { 769 struct usb_dma_tag *udt; 770 uint8_t nudt; 771 772 USB_ASSERT(align > 0, ("Invalid parameter align = 0\n")); 773 USB_ASSERT(size > 0, ("Invalid parameter size = 0\n")); 774 775 udt = udpt->utag_first; 776 nudt = udpt->utag_max; 777 778 while (nudt--) { 779 780 if (udt->align == 0) { 781 usb_dma_tag_create(udt, size, align); 782 if (udt->tag == NULL) { 783 return (NULL); 784 } 785 udt->align = align; 786 udt->size = size; 787 return (udt); 788 } 789 if ((udt->align == align) && (udt->size == size)) { 790 return (udt); 791 } 792 udt++; 793 } 794 return (NULL); 795 } 796 797 /*------------------------------------------------------------------------* 798 * usb_dma_tag_setup - initialise USB DMA tags 799 *------------------------------------------------------------------------*/ 800 void 801 usb_dma_tag_setup(struct usb_dma_parent_tag *udpt, 802 struct usb_dma_tag *udt, bus_dma_tag_t dmat, 803 struct lock *lock, usb_dma_callback_t *func, 804 uint8_t ndmabits, uint8_t nudt) 805 { 806 memset(udpt, 0, sizeof(*udpt)); 807 808 /* sanity checking */ 809 if ((nudt == 0) || 810 (ndmabits == 0) || 811 (lock == NULL)) { 812 /* something is corrupt */ 813 return; 814 } 815 /* initialise condition variable */ 816 cv_init(udpt->cv, "USB DMA CV"); 817 818 /* store some information */ 819 udpt->lock = lock; 820 udpt->func = func; 821 udpt->tag = dmat; 822 udpt->utag_first = udt; 823 udpt->utag_max = nudt; 824 udpt->dma_bits = ndmabits; 825 826 while (nudt--) { 827 memset(udt, 0, sizeof(*udt)); 828 udt->tag_parent = udpt; 829 udt++; 830 } 831 } 832 833 /*------------------------------------------------------------------------* 834 * usb_bus_tag_unsetup - factored out code 835 *------------------------------------------------------------------------*/ 836 void 837 usb_dma_tag_unsetup(struct usb_dma_parent_tag *udpt) 838 { 839 struct usb_dma_tag *udt; 840 uint8_t nudt; 841 842 udt = udpt->utag_first; 843 nudt = udpt->utag_max; 844 845 while (nudt--) { 846 847 if (udt->align) { 848 /* destroy the USB DMA tag */ 849 usb_dma_tag_destroy(udt); 850 udt->align = 0; 851 } 852 udt++; 853 } 854 855 if (udpt->utag_max) { 856 /* destroy the condition variable */ 857 cv_destroy(udpt->cv); 858 } 859 } 860 861 /*------------------------------------------------------------------------* 862 * usb_bdma_work_loop 863 * 864 * This function handles loading of virtual buffers into DMA and is 865 * only called when "dma_refcount" is zero. 866 *------------------------------------------------------------------------*/ 867 void 868 usb_bdma_work_loop(struct usb_xfer_queue *pq) 869 { 870 struct usb_xfer_root *info; 871 struct usb_xfer *xfer; 872 usb_frcount_t nframes; 873 874 xfer = pq->curr; 875 info = xfer->xroot; 876 877 KKASSERT(lockowned(info->xfer_lock)); 878 879 if (xfer->error) { 880 /* some error happened */ 881 USB_BUS_LOCK(info->bus); 882 usbd_transfer_done(xfer, 0); 883 USB_BUS_UNLOCK(info->bus); 884 return; 885 } 886 if (!xfer->flags_int.bdma_setup) { 887 struct usb_page *pg; 888 usb_frlength_t frlength_0; 889 uint8_t isread; 890 891 xfer->flags_int.bdma_setup = 1; 892 893 /* reset BUS-DMA load state */ 894 895 info->dma_error = 0; 896 897 if (xfer->flags_int.isochronous_xfr) { 898 /* only one frame buffer */ 899 nframes = 1; 900 frlength_0 = xfer->sumlen; 901 } else { 902 /* can be multiple frame buffers */ 903 nframes = xfer->nframes; 904 frlength_0 = xfer->frlengths[0]; 905 } 906 907 /* 908 * Set DMA direction first. This is needed to 909 * select the correct cache invalidate and cache 910 * flush operations. 911 */ 912 isread = USB_GET_DATA_ISREAD(xfer); 913 pg = xfer->dma_page_ptr; 914 915 if (xfer->flags_int.control_xfr && 916 xfer->flags_int.control_hdr) { 917 /* special case */ 918 if (xfer->flags_int.usb_mode == USB_MODE_DEVICE) { 919 /* The device controller writes to memory */ 920 xfer->frbuffers[0].isread = 1; 921 } else { 922 /* The host controller reads from memory */ 923 xfer->frbuffers[0].isread = 0; 924 } 925 } else { 926 /* default case */ 927 xfer->frbuffers[0].isread = isread; 928 } 929 930 /* 931 * Setup the "page_start" pointer which points to an array of 932 * USB pages where information about the physical address of a 933 * page will be stored. Also initialise the "isread" field of 934 * the USB page caches. 935 */ 936 xfer->frbuffers[0].page_start = pg; 937 938 info->dma_nframes = nframes; 939 info->dma_currframe = 0; 940 info->dma_frlength_0 = frlength_0; 941 942 pg += (frlength_0 / USB_PAGE_SIZE); 943 pg += 2; 944 945 while (--nframes > 0) { 946 xfer->frbuffers[nframes].isread = isread; 947 xfer->frbuffers[nframes].page_start = pg; 948 949 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 950 pg += 2; 951 } 952 953 } 954 if (info->dma_error) { 955 USB_BUS_LOCK(info->bus); 956 usbd_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 957 USB_BUS_UNLOCK(info->bus); 958 return; 959 } 960 if (info->dma_currframe != info->dma_nframes) { 961 962 if (info->dma_currframe == 0) { 963 /* special case */ 964 usb_pc_load_mem(xfer->frbuffers, 965 info->dma_frlength_0, 0); 966 } else { 967 /* default case */ 968 nframes = info->dma_currframe; 969 usb_pc_load_mem(xfer->frbuffers + nframes, 970 xfer->frlengths[nframes], 0); 971 } 972 973 /* advance frame index */ 974 info->dma_currframe++; 975 976 return; 977 } 978 /* go ahead */ 979 usb_bdma_pre_sync(xfer); 980 981 /* start loading next USB transfer, if any */ 982 usb_command_wrapper(pq, NULL); 983 984 /* finally start the hardware */ 985 usbd_pipe_enter(xfer); 986 } 987 988 /*------------------------------------------------------------------------* 989 * usb_bdma_done_event 990 * 991 * This function is called when the BUS-DMA has loaded virtual memory 992 * into DMA, if any. 993 *------------------------------------------------------------------------*/ 994 void 995 usb_bdma_done_event(struct usb_dma_parent_tag *udpt) 996 { 997 struct usb_xfer_root *info; 998 999 info = USB_DMATAG_TO_XROOT(udpt); 1000 1001 KKASSERT(lockowned(info->xfer_lock)); 1002 1003 /* copy error */ 1004 info->dma_error = udpt->dma_error; 1005 1006 /* enter workloop again */ 1007 usb_command_wrapper(&info->dma_q, 1008 info->dma_q.curr); 1009 } 1010 1011 /*------------------------------------------------------------------------* 1012 * usb_bdma_pre_sync 1013 * 1014 * This function handles DMA synchronisation that must be done before 1015 * an USB transfer is started. 1016 *------------------------------------------------------------------------*/ 1017 void 1018 usb_bdma_pre_sync(struct usb_xfer *xfer) 1019 { 1020 struct usb_page_cache *pc; 1021 usb_frcount_t nframes; 1022 1023 if (xfer->flags_int.isochronous_xfr) { 1024 /* only one frame buffer */ 1025 nframes = 1; 1026 } else { 1027 /* can be multiple frame buffers */ 1028 nframes = xfer->nframes; 1029 } 1030 1031 pc = xfer->frbuffers; 1032 1033 while (nframes--) { 1034 1035 if (pc->isread) { 1036 usb_pc_cpu_invalidate(pc); 1037 } else { 1038 usb_pc_cpu_flush(pc); 1039 } 1040 pc++; 1041 } 1042 } 1043 1044 /*------------------------------------------------------------------------* 1045 * usb_bdma_post_sync 1046 * 1047 * This function handles DMA synchronisation that must be done after 1048 * an USB transfer is complete. 1049 *------------------------------------------------------------------------*/ 1050 void 1051 usb_bdma_post_sync(struct usb_xfer *xfer) 1052 { 1053 struct usb_page_cache *pc; 1054 usb_frcount_t nframes; 1055 1056 if (xfer->flags_int.isochronous_xfr) { 1057 /* only one frame buffer */ 1058 nframes = 1; 1059 } else { 1060 /* can be multiple frame buffers */ 1061 nframes = xfer->nframes; 1062 } 1063 1064 pc = xfer->frbuffers; 1065 1066 while (nframes--) { 1067 if (pc->isread) { 1068 usb_pc_cpu_invalidate(pc); 1069 } 1070 pc++; 1071 } 1072 } 1073 1074 #endif 1075