1 /* $FreeBSD$ */ 2 /*- 3 * Copyright (c) 2008 Hans Petter Selasky. All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <dev/usb/usb_mfunc.h> 28 #include <dev/usb/usb_error.h> 29 #include <dev/usb/usb.h> 30 #include <dev/usb/usb_defs.h> 31 32 #define USB_DEBUG_VAR usb2_debug 33 34 #include <dev/usb/usb_core.h> 35 #include <dev/usb/usb_busdma.h> 36 #include <dev/usb/usb_process.h> 37 #include <dev/usb/usb_transfer.h> 38 #include <dev/usb/usb_device.h> 39 #include <dev/usb/usb_util.h> 40 #include <dev/usb/usb_debug.h> 41 42 #include <dev/usb/usb_controller.h> 43 #include <dev/usb/usb_bus.h> 44 45 static void usb2_dma_tag_create(struct usb2_dma_tag *, uint32_t, uint32_t); 46 static void usb2_dma_tag_destroy(struct usb2_dma_tag *); 47 48 #ifdef __FreeBSD__ 49 static void usb2_dma_lock_cb(void *, bus_dma_lock_op_t); 50 static int32_t usb2_m_copy_in_cb(void *, void *, uint32_t); 51 static void usb2_pc_alloc_mem_cb(void *, bus_dma_segment_t *, int, int); 52 static void usb2_pc_load_mem_cb(void *, bus_dma_segment_t *, int, int); 53 static void usb2_pc_common_mem_cb(void *, bus_dma_segment_t *, int, int, 54 uint8_t); 55 #endif 56 57 #ifdef __NetBSD__ 58 static int32_t usb2_m_copy_in_cb(void *, caddr_t, uint32_t); 59 static void usb2_pc_common_mem_cb(struct usb2_page_cache *, 60 bus_dma_segment_t *, int, int, uint8_t); 61 #endif 62 63 /*------------------------------------------------------------------------* 64 * usb2_get_page - lookup DMA-able memory for the given offset 65 * 66 * NOTE: Only call this function when the "page_cache" structure has 67 * been properly initialized ! 68 *------------------------------------------------------------------------*/ 69 void 70 usb2_get_page(struct usb2_page_cache *pc, uint32_t offset, 71 struct usb2_page_search *res) 72 { 73 struct usb2_page *page; 74 75 if (pc->page_start) { 76 77 /* Case 1 - something has been loaded into DMA */ 78 79 if (pc->buffer) { 80 81 /* Case 1a - Kernel Virtual Address */ 82 83 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 84 } 85 offset += pc->page_offset_buf; 86 87 /* compute destination page */ 88 89 page = pc->page_start; 90 91 if (pc->ismultiseg) { 92 93 page += (offset / USB_PAGE_SIZE); 94 95 offset %= USB_PAGE_SIZE; 96 97 res->length = USB_PAGE_SIZE - offset; 98 res->physaddr = page->physaddr + offset; 99 } else { 100 res->length = 0 - 1; 101 res->physaddr = page->physaddr + offset; 102 } 103 if (!pc->buffer) { 104 105 /* Case 1b - Non Kernel Virtual Address */ 106 107 res->buffer = USB_ADD_BYTES(page->buffer, offset); 108 } 109 } else { 110 111 /* Case 2 - Plain PIO */ 112 113 res->buffer = USB_ADD_BYTES(pc->buffer, offset); 114 res->length = 0 - 1; 115 res->physaddr = 0; 116 } 117 } 118 119 /*------------------------------------------------------------------------* 120 * usb2_copy_in - copy directly to DMA-able memory 121 *------------------------------------------------------------------------*/ 122 void 123 usb2_copy_in(struct usb2_page_cache *cache, uint32_t offset, 124 const void *ptr, uint32_t len) 125 { 126 struct usb2_page_search buf_res; 127 128 while (len != 0) { 129 130 usb2_get_page(cache, offset, &buf_res); 131 132 if (buf_res.length > len) { 133 buf_res.length = len; 134 } 135 bcopy(ptr, buf_res.buffer, buf_res.length); 136 137 offset += buf_res.length; 138 len -= buf_res.length; 139 ptr = USB_ADD_BYTES(ptr, buf_res.length); 140 } 141 } 142 143 /*------------------------------------------------------------------------* 144 * usb2_copy_in_user - copy directly to DMA-able memory from userland 145 * 146 * Return values: 147 * 0: Success 148 * Else: Failure 149 *------------------------------------------------------------------------*/ 150 int 151 usb2_copy_in_user(struct usb2_page_cache *cache, uint32_t offset, 152 const void *ptr, uint32_t len) 153 { 154 struct usb2_page_search buf_res; 155 int error; 156 157 while (len != 0) { 158 159 usb2_get_page(cache, offset, &buf_res); 160 161 if (buf_res.length > len) { 162 buf_res.length = len; 163 } 164 error = copyin(ptr, buf_res.buffer, buf_res.length); 165 if (error) 166 return (error); 167 168 offset += buf_res.length; 169 len -= buf_res.length; 170 ptr = USB_ADD_BYTES(ptr, buf_res.length); 171 } 172 return (0); /* success */ 173 } 174 175 /*------------------------------------------------------------------------* 176 * usb2_m_copy_in - copy a mbuf chain directly into DMA-able memory 177 *------------------------------------------------------------------------*/ 178 struct usb2_m_copy_in_arg { 179 struct usb2_page_cache *cache; 180 uint32_t dst_offset; 181 }; 182 183 static int32_t 184 #ifdef __FreeBSD__ 185 usb2_m_copy_in_cb(void *arg, void *src, uint32_t count) 186 #else 187 usb2_m_copy_in_cb(void *arg, caddr_t src, uint32_t count) 188 #endif 189 { 190 register struct usb2_m_copy_in_arg *ua = arg; 191 192 usb2_copy_in(ua->cache, ua->dst_offset, src, count); 193 ua->dst_offset += count; 194 return (0); 195 } 196 197 void 198 usb2_m_copy_in(struct usb2_page_cache *cache, uint32_t dst_offset, 199 struct mbuf *m, uint32_t src_offset, uint32_t src_len) 200 { 201 struct usb2_m_copy_in_arg arg = {cache, dst_offset}; 202 register int error; 203 204 error = m_apply(m, src_offset, src_len, &usb2_m_copy_in_cb, &arg); 205 } 206 207 /*------------------------------------------------------------------------* 208 * usb2_uiomove - factored out code 209 *------------------------------------------------------------------------*/ 210 int 211 usb2_uiomove(struct usb2_page_cache *pc, struct uio *uio, 212 uint32_t pc_offset, uint32_t len) 213 { 214 struct usb2_page_search res; 215 int error = 0; 216 217 while (len != 0) { 218 219 usb2_get_page(pc, pc_offset, &res); 220 221 if (res.length > len) { 222 res.length = len; 223 } 224 /* 225 * "uiomove()" can sleep so one needs to make a wrapper, 226 * exiting the mutex and checking things 227 */ 228 error = uiomove(res.buffer, res.length, uio); 229 230 if (error) { 231 break; 232 } 233 pc_offset += res.length; 234 len -= res.length; 235 } 236 return (error); 237 } 238 239 /*------------------------------------------------------------------------* 240 * usb2_copy_out - copy directly from DMA-able memory 241 *------------------------------------------------------------------------*/ 242 void 243 usb2_copy_out(struct usb2_page_cache *cache, uint32_t offset, 244 void *ptr, uint32_t len) 245 { 246 struct usb2_page_search res; 247 248 while (len != 0) { 249 250 usb2_get_page(cache, offset, &res); 251 252 if (res.length > len) { 253 res.length = len; 254 } 255 bcopy(res.buffer, ptr, res.length); 256 257 offset += res.length; 258 len -= res.length; 259 ptr = USB_ADD_BYTES(ptr, res.length); 260 } 261 } 262 263 /*------------------------------------------------------------------------* 264 * usb2_copy_out_user - copy directly from DMA-able memory to userland 265 * 266 * Return values: 267 * 0: Success 268 * Else: Failure 269 *------------------------------------------------------------------------*/ 270 int 271 usb2_copy_out_user(struct usb2_page_cache *cache, uint32_t offset, 272 void *ptr, uint32_t len) 273 { 274 struct usb2_page_search res; 275 int error; 276 277 while (len != 0) { 278 279 usb2_get_page(cache, offset, &res); 280 281 if (res.length > len) { 282 res.length = len; 283 } 284 error = copyout(res.buffer, ptr, res.length); 285 if (error) 286 return (error); 287 288 offset += res.length; 289 len -= res.length; 290 ptr = USB_ADD_BYTES(ptr, res.length); 291 } 292 return (0); /* success */ 293 } 294 295 /*------------------------------------------------------------------------* 296 * usb2_bzero - zero DMA-able memory 297 *------------------------------------------------------------------------*/ 298 void 299 usb2_bzero(struct usb2_page_cache *cache, uint32_t offset, uint32_t len) 300 { 301 struct usb2_page_search res; 302 303 while (len != 0) { 304 305 usb2_get_page(cache, offset, &res); 306 307 if (res.length > len) { 308 res.length = len; 309 } 310 bzero(res.buffer, res.length); 311 312 offset += res.length; 313 len -= res.length; 314 } 315 } 316 317 318 #ifdef __FreeBSD__ 319 320 /*------------------------------------------------------------------------* 321 * usb2_dma_lock_cb - dummy callback 322 *------------------------------------------------------------------------*/ 323 static void 324 usb2_dma_lock_cb(void *arg, bus_dma_lock_op_t op) 325 { 326 /* we use "mtx_owned()" instead of this function */ 327 } 328 329 /*------------------------------------------------------------------------* 330 * usb2_dma_tag_create - allocate a DMA tag 331 * 332 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 333 * allow multi-segment mappings. Else all mappings are single-segment. 334 *------------------------------------------------------------------------*/ 335 static void 336 usb2_dma_tag_create(struct usb2_dma_tag *udt, 337 uint32_t size, uint32_t align) 338 { 339 bus_dma_tag_t tag; 340 341 if (bus_dma_tag_create 342 ( /* parent */ udt->tag_parent->tag, 343 /* alignment */ align, 344 /* boundary */ USB_PAGE_SIZE, 345 /* lowaddr */ (2ULL << (udt->tag_parent->dma_bits - 1)) - 1, 346 /* highaddr */ BUS_SPACE_MAXADDR, 347 /* filter */ NULL, 348 /* filterarg */ NULL, 349 /* maxsize */ size, 350 /* nsegments */ (align == 1) ? 351 (2 + (size / USB_PAGE_SIZE)) : 1, 352 /* maxsegsz */ (align == 1) ? 353 USB_PAGE_SIZE : size, 354 /* flags */ BUS_DMA_KEEP_PG_OFFSET, 355 /* lockfn */ &usb2_dma_lock_cb, 356 /* lockarg */ NULL, 357 &tag)) { 358 tag = NULL; 359 } 360 udt->tag = tag; 361 } 362 363 /*------------------------------------------------------------------------* 364 * usb2_dma_tag_free - free a DMA tag 365 *------------------------------------------------------------------------*/ 366 static void 367 usb2_dma_tag_destroy(struct usb2_dma_tag *udt) 368 { 369 bus_dma_tag_destroy(udt->tag); 370 } 371 372 /*------------------------------------------------------------------------* 373 * usb2_pc_alloc_mem_cb - BUS-DMA callback function 374 *------------------------------------------------------------------------*/ 375 static void 376 usb2_pc_alloc_mem_cb(void *arg, bus_dma_segment_t *segs, 377 int nseg, int error) 378 { 379 usb2_pc_common_mem_cb(arg, segs, nseg, error, 0); 380 } 381 382 /*------------------------------------------------------------------------* 383 * usb2_pc_load_mem_cb - BUS-DMA callback function 384 *------------------------------------------------------------------------*/ 385 static void 386 usb2_pc_load_mem_cb(void *arg, bus_dma_segment_t *segs, 387 int nseg, int error) 388 { 389 usb2_pc_common_mem_cb(arg, segs, nseg, error, 1); 390 } 391 392 /*------------------------------------------------------------------------* 393 * usb2_pc_common_mem_cb - BUS-DMA callback function 394 *------------------------------------------------------------------------*/ 395 static void 396 usb2_pc_common_mem_cb(void *arg, bus_dma_segment_t *segs, 397 int nseg, int error, uint8_t isload) 398 { 399 struct usb2_dma_parent_tag *uptag; 400 struct usb2_page_cache *pc; 401 struct usb2_page *pg; 402 uint32_t rem; 403 uint8_t owned; 404 405 pc = arg; 406 uptag = pc->tag_parent; 407 408 /* 409 * XXX There is sometimes recursive locking here. 410 * XXX We should try to find a better solution. 411 * XXX Until further the "owned" variable does 412 * XXX the trick. 413 */ 414 415 if (error) { 416 goto done; 417 } 418 pg = pc->page_start; 419 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 420 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 421 pc->page_offset_buf = rem; 422 pc->page_offset_end += rem; 423 nseg--; 424 #if (USB_DEBUG != 0) 425 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 426 /* 427 * This check verifies that the physical address is correct: 428 */ 429 DPRINTFN(0, "Page offset was not preserved!\n"); 430 error = 1; 431 goto done; 432 } 433 #endif 434 while (nseg > 0) { 435 nseg--; 436 segs++; 437 pg++; 438 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 439 } 440 441 done: 442 owned = mtx_owned(uptag->mtx); 443 if (!owned) 444 mtx_lock(uptag->mtx); 445 446 uptag->dma_error = (error ? 1 : 0); 447 if (isload) { 448 (uptag->func) (uptag); 449 } else { 450 usb2_cv_broadcast(uptag->cv); 451 } 452 if (!owned) 453 mtx_unlock(uptag->mtx); 454 } 455 456 /*------------------------------------------------------------------------* 457 * usb2_pc_alloc_mem - allocate DMA'able memory 458 * 459 * Returns: 460 * 0: Success 461 * Else: Failure 462 *------------------------------------------------------------------------*/ 463 uint8_t 464 usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, 465 uint32_t size, uint32_t align) 466 { 467 struct usb2_dma_parent_tag *uptag; 468 struct usb2_dma_tag *utag; 469 bus_dmamap_t map; 470 void *ptr; 471 int err; 472 473 uptag = pc->tag_parent; 474 475 if (align != 1) { 476 /* 477 * The alignment must be greater or equal to the 478 * "size" else the object can be split between two 479 * memory pages and we get a problem! 480 */ 481 while (align < size) { 482 align *= 2; 483 if (align == 0) { 484 goto error; 485 } 486 } 487 #if 1 488 /* 489 * XXX BUS-DMA workaround - FIXME later: 490 * 491 * We assume that that the aligment at this point of 492 * the code is greater than or equal to the size and 493 * less than two times the size, so that if we double 494 * the size, the size will be greater than the 495 * alignment. 496 * 497 * The bus-dma system has a check for "alignment" 498 * being less than "size". If that check fails we end 499 * up using contigmalloc which is page based even for 500 * small allocations. Try to avoid that to save 501 * memory, hence we sometimes to a large number of 502 * small allocations! 503 */ 504 if (size <= (USB_PAGE_SIZE / 2)) { 505 size *= 2; 506 } 507 #endif 508 } 509 /* get the correct DMA tag */ 510 utag = usb2_dma_tag_find(uptag, size, align); 511 if (utag == NULL) { 512 goto error; 513 } 514 /* allocate memory */ 515 if (bus_dmamem_alloc( 516 utag->tag, &ptr, (BUS_DMA_WAITOK | BUS_DMA_COHERENT), &map)) { 517 goto error; 518 } 519 /* setup page cache */ 520 pc->buffer = ptr; 521 pc->page_start = pg; 522 pc->page_offset_buf = 0; 523 pc->page_offset_end = size; 524 pc->map = map; 525 pc->tag = utag->tag; 526 pc->ismultiseg = (align == 1); 527 528 mtx_lock(uptag->mtx); 529 530 /* load memory into DMA */ 531 err = bus_dmamap_load( 532 utag->tag, map, ptr, size, &usb2_pc_alloc_mem_cb, 533 pc, (BUS_DMA_WAITOK | BUS_DMA_COHERENT)); 534 535 if (err == EINPROGRESS) { 536 usb2_cv_wait(uptag->cv, uptag->mtx); 537 err = 0; 538 } 539 mtx_unlock(uptag->mtx); 540 541 if (err || uptag->dma_error) { 542 bus_dmamem_free(utag->tag, ptr, map); 543 goto error; 544 } 545 bzero(ptr, size); 546 547 usb2_pc_cpu_flush(pc); 548 549 return (0); 550 551 error: 552 /* reset most of the page cache */ 553 pc->buffer = NULL; 554 pc->page_start = NULL; 555 pc->page_offset_buf = 0; 556 pc->page_offset_end = 0; 557 pc->map = NULL; 558 pc->tag = NULL; 559 return (1); 560 } 561 562 /*------------------------------------------------------------------------* 563 * usb2_pc_free_mem - free DMA memory 564 * 565 * This function is NULL safe. 566 *------------------------------------------------------------------------*/ 567 void 568 usb2_pc_free_mem(struct usb2_page_cache *pc) 569 { 570 if (pc && pc->buffer) { 571 572 bus_dmamap_unload(pc->tag, pc->map); 573 574 bus_dmamem_free(pc->tag, pc->buffer, pc->map); 575 576 pc->buffer = NULL; 577 } 578 } 579 580 /*------------------------------------------------------------------------* 581 * usb2_pc_load_mem - load virtual memory into DMA 582 * 583 * Return values: 584 * 0: Success 585 * Else: Error 586 *------------------------------------------------------------------------*/ 587 uint8_t 588 usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync) 589 { 590 /* setup page cache */ 591 pc->page_offset_buf = 0; 592 pc->page_offset_end = size; 593 pc->ismultiseg = 1; 594 595 mtx_assert(pc->tag_parent->mtx, MA_OWNED); 596 597 if (size > 0) { 598 if (sync) { 599 struct usb2_dma_parent_tag *uptag; 600 int err; 601 602 uptag = pc->tag_parent; 603 604 /* 605 * We have to unload the previous loaded DMA 606 * pages before trying to load a new one! 607 */ 608 bus_dmamap_unload(pc->tag, pc->map); 609 610 /* 611 * Try to load memory into DMA. 612 */ 613 err = bus_dmamap_load( 614 pc->tag, pc->map, pc->buffer, size, 615 &usb2_pc_alloc_mem_cb, pc, BUS_DMA_WAITOK); 616 if (err == EINPROGRESS) { 617 usb2_cv_wait(uptag->cv, uptag->mtx); 618 err = 0; 619 } 620 if (err || uptag->dma_error) { 621 return (1); 622 } 623 } else { 624 625 /* 626 * We have to unload the previous loaded DMA 627 * pages before trying to load a new one! 628 */ 629 bus_dmamap_unload(pc->tag, pc->map); 630 631 /* 632 * Try to load memory into DMA. The callback 633 * will be called in all cases: 634 */ 635 if (bus_dmamap_load( 636 pc->tag, pc->map, pc->buffer, size, 637 &usb2_pc_load_mem_cb, pc, BUS_DMA_WAITOK)) { 638 } 639 } 640 } else { 641 if (!sync) { 642 /* 643 * Call callback so that refcount is decremented 644 * properly: 645 */ 646 pc->tag_parent->dma_error = 0; 647 (pc->tag_parent->func) (pc->tag_parent); 648 } 649 } 650 return (0); 651 } 652 653 /*------------------------------------------------------------------------* 654 * usb2_pc_cpu_invalidate - invalidate CPU cache 655 *------------------------------------------------------------------------*/ 656 void 657 usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) 658 { 659 if (pc->page_offset_end == pc->page_offset_buf) { 660 /* nothing has been loaded into this page cache! */ 661 return; 662 } 663 bus_dmamap_sync(pc->tag, pc->map, 664 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 665 } 666 667 /*------------------------------------------------------------------------* 668 * usb2_pc_cpu_flush - flush CPU cache 669 *------------------------------------------------------------------------*/ 670 void 671 usb2_pc_cpu_flush(struct usb2_page_cache *pc) 672 { 673 if (pc->page_offset_end == pc->page_offset_buf) { 674 /* nothing has been loaded into this page cache! */ 675 return; 676 } 677 bus_dmamap_sync(pc->tag, pc->map, 678 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 679 } 680 681 /*------------------------------------------------------------------------* 682 * usb2_pc_dmamap_create - create a DMA map 683 * 684 * Returns: 685 * 0: Success 686 * Else: Failure 687 *------------------------------------------------------------------------*/ 688 uint8_t 689 usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size) 690 { 691 struct usb2_xfer_root *info; 692 struct usb2_dma_tag *utag; 693 694 /* get info */ 695 info = pc->tag_parent->info; 696 697 /* sanity check */ 698 if (info == NULL) { 699 goto error; 700 } 701 utag = usb2_dma_tag_find(pc->tag_parent, size, 1); 702 if (utag == NULL) { 703 goto error; 704 } 705 /* create DMA map */ 706 if (bus_dmamap_create(utag->tag, 0, &pc->map)) { 707 goto error; 708 } 709 pc->tag = utag->tag; 710 return 0; /* success */ 711 712 error: 713 pc->map = NULL; 714 pc->tag = NULL; 715 return 1; /* failure */ 716 } 717 718 /*------------------------------------------------------------------------* 719 * usb2_pc_dmamap_destroy 720 * 721 * This function is NULL safe. 722 *------------------------------------------------------------------------*/ 723 void 724 usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) 725 { 726 if (pc && pc->tag) { 727 bus_dmamap_destroy(pc->tag, pc->map); 728 pc->tag = NULL; 729 pc->map = NULL; 730 } 731 } 732 733 #endif 734 735 #ifdef __NetBSD__ 736 737 /*------------------------------------------------------------------------* 738 * usb2_dma_tag_create - allocate a DMA tag 739 * 740 * NOTE: If the "align" parameter has a value of 1 the DMA-tag will 741 * allow multi-segment mappings. Else all mappings are single-segment. 742 *------------------------------------------------------------------------*/ 743 static void 744 usb2_dma_tag_create(struct usb2_dma_tag *udt, 745 uint32_t size, uint32_t align) 746 { 747 uint32_t nseg; 748 749 if (align == 1) { 750 nseg = (2 + (size / USB_PAGE_SIZE)); 751 } else { 752 nseg = 1; 753 } 754 755 udt->p_seg = malloc(nseg * sizeof(*(udt->p_seg)), 756 M_USB, M_WAITOK | M_ZERO); 757 758 if (udt->p_seg == NULL) { 759 return; 760 } 761 udt->tag = udt->tag_parent->tag; 762 udt->n_seg = nseg; 763 } 764 765 /*------------------------------------------------------------------------* 766 * usb2_dma_tag_free - free a DMA tag 767 *------------------------------------------------------------------------*/ 768 static void 769 usb2_dma_tag_destroy(struct usb2_dma_tag *udt) 770 { 771 free(udt->p_seg, M_USB); 772 } 773 774 /*------------------------------------------------------------------------* 775 * usb2_pc_common_mem_cb - BUS-DMA callback function 776 *------------------------------------------------------------------------*/ 777 static void 778 usb2_pc_common_mem_cb(struct usb2_page_cache *pc, bus_dma_segment_t *segs, 779 int nseg, int error, uint8_t isload, uint8_t dolock) 780 { 781 struct usb2_dma_parent_tag *uptag; 782 struct usb2_page *pg; 783 uint32_t rem; 784 uint8_t ext_seg; /* extend last segment */ 785 786 uptag = pc->tag_parent; 787 788 if (error) { 789 goto done; 790 } 791 pg = pc->page_start; 792 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 793 rem = segs->ds_addr & (USB_PAGE_SIZE - 1); 794 pc->page_offset_buf = rem; 795 pc->page_offset_end += rem; 796 if (nseg < ((pc->page_offset_end + 797 (USB_PAGE_SIZE - 1)) / USB_PAGE_SIZE)) { 798 ext_seg = 1; 799 } else { 800 ext_seg = 0; 801 } 802 nseg--; 803 #if (USB_DEBUG != 0) 804 if (rem != (USB_P2U(pc->buffer) & (USB_PAGE_SIZE - 1))) { 805 /* 806 * This check verifies that the physical address is correct: 807 */ 808 DPRINTFN(0, "Page offset was not preserved!\n"); 809 error = 1; 810 goto done; 811 } 812 #endif 813 while (nseg > 0) { 814 nseg--; 815 segs++; 816 pg++; 817 pg->physaddr = segs->ds_addr & ~(USB_PAGE_SIZE - 1); 818 } 819 820 /* 821 * XXX The segments we get from BUS-DMA are not aligned, 822 * XXX so we need to extend the last segment if we are 823 * XXX unaligned and cross the segment boundary! 824 */ 825 if (ext_seg && pc->ismultiseg) { 826 (pg + 1)->physaddr = pg->physaddr + USB_PAGE_SIZE; 827 } 828 done: 829 if (dolock) 830 mtx_lock(uptag->mtx); 831 832 uptag->dma_error = (error ? 1 : 0); 833 if (isload) { 834 (uptag->func) (uptag); 835 } 836 if (dolock) 837 mtx_unlock(uptag->mtx); 838 } 839 840 /*------------------------------------------------------------------------* 841 * usb2_pc_alloc_mem - allocate DMA'able memory 842 * 843 * Returns: 844 * 0: Success 845 * Else: Failure 846 *------------------------------------------------------------------------*/ 847 uint8_t 848 usb2_pc_alloc_mem(struct usb2_page_cache *pc, struct usb2_page *pg, 849 uint32_t size, uint32_t align) 850 { 851 struct usb2_dma_parent_tag *uptag; 852 struct usb2_dma_tag *utag; 853 caddr_t ptr = NULL; 854 bus_dmamap_t map; 855 int seg_count; 856 857 uptag = pc->tag_parent; 858 859 if (align != 1) { 860 /* 861 * The alignment must be greater or equal to the 862 * "size" else the object can be split between two 863 * memory pages and we get a problem! 864 */ 865 while (align < size) { 866 align *= 2; 867 if (align == 0) { 868 goto done_5; 869 } 870 } 871 } 872 /* get the correct DMA tag */ 873 utag = usb2_dma_tag_find(pc->tag_parent, size, align); 874 if (utag == NULL) { 875 goto done_5; 876 } 877 if (bus_dmamem_alloc(utag->tag, size, align, 0, utag->p_seg, 878 utag->n_seg, &seg_count, BUS_DMA_WAITOK)) { 879 goto done_4; 880 } 881 if (bus_dmamem_map(utag->tag, utag->p_seg, seg_count, size, 882 &ptr, BUS_DMA_WAITOK | BUS_DMA_COHERENT)) { 883 goto done_3; 884 } 885 if (bus_dmamap_create(utag->tag, size, utag->n_seg, (align == 1) ? 886 USB_PAGE_SIZE : size, 0, BUS_DMA_WAITOK, &map)) { 887 goto done_2; 888 } 889 if (bus_dmamap_load(utag->tag, map, ptr, size, NULL, 890 BUS_DMA_WAITOK)) { 891 goto done_1; 892 } 893 pc->p_seg = malloc(seg_count * sizeof(*(pc->p_seg)), 894 M_USB, M_WAITOK | M_ZERO); 895 if (pc->p_seg == NULL) { 896 goto done_0; 897 } 898 /* store number if actual segments used */ 899 pc->n_seg = seg_count; 900 901 /* make a copy of the segments */ 902 bcopy(utag->p_seg, pc->p_seg, 903 seg_count * sizeof(*(pc->p_seg))); 904 905 /* setup page cache */ 906 pc->buffer = ptr; 907 pc->page_start = pg; 908 pc->page_offset_buf = 0; 909 pc->page_offset_end = size; 910 pc->map = map; 911 pc->tag = utag->tag; 912 pc->ismultiseg = (align == 1); 913 914 usb2_pc_common_mem_cb(pc, utag->p_seg, seg_count, 0, 0, 1); 915 916 bzero(ptr, size); 917 918 usb2_pc_cpu_flush(pc); 919 920 return (0); 921 922 done_0: 923 bus_dmamap_unload(utag->tag, map); 924 done_1: 925 bus_dmamap_destroy(utag->tag, map); 926 done_2: 927 bus_dmamem_unmap(utag->tag, ptr, size); 928 done_3: 929 bus_dmamem_free(utag->tag, utag->p_seg, seg_count); 930 done_4: 931 /* utag is destroyed later */ 932 done_5: 933 /* reset most of the page cache */ 934 pc->buffer = NULL; 935 pc->page_start = NULL; 936 pc->page_offset_buf = 0; 937 pc->page_offset_end = 0; 938 pc->map = NULL; 939 pc->tag = NULL; 940 pc->n_seg = 0; 941 pc->p_seg = NULL; 942 return (1); 943 } 944 945 /*------------------------------------------------------------------------* 946 * usb2_pc_free_mem - free DMA memory 947 * 948 * This function is NULL safe. 949 *------------------------------------------------------------------------*/ 950 void 951 usb2_pc_free_mem(struct usb2_page_cache *pc) 952 { 953 if (pc && pc->buffer) { 954 bus_dmamap_unload(pc->tag, pc->map); 955 bus_dmamap_destroy(pc->tag, pc->map); 956 bus_dmamem_unmap(pc->tag, pc->buffer, 957 pc->page_offset_end - pc->page_offset_buf); 958 bus_dmamem_free(pc->tag, pc->p_seg, pc->n_seg); 959 free(pc->p_seg, M_USB); 960 pc->buffer = NULL; 961 } 962 } 963 964 /*------------------------------------------------------------------------* 965 * usb2_pc_load_mem - load virtual memory into DMA 966 * 967 * Return values: 968 * 0: Success 969 * Else: Error 970 *------------------------------------------------------------------------*/ 971 uint8_t 972 usb2_pc_load_mem(struct usb2_page_cache *pc, uint32_t size, uint8_t sync) 973 { 974 int error; 975 976 /* setup page cache */ 977 pc->page_offset_buf = 0; 978 pc->page_offset_end = size; 979 pc->ismultiseg = 1; 980 981 if (size > 0) { 982 983 /* 984 * We have to unload the previous loaded DMA 985 * pages before trying to load a new one! 986 */ 987 bus_dmamap_unload(pc->tag, pc->map); 988 989 /* try to load memory into DMA using using no wait option */ 990 if (bus_dmamap_load(pc->tag, pc->map, pc->buffer, 991 size, NULL, BUS_DMA_NOWAIT)) { 992 error = ENOMEM; 993 } else { 994 error = 0; 995 } 996 997 usb2_pc_common_mem_cb(pc, pc->map->dm_segs, 998 pc->map->dm_nsegs, error, !sync); 999 1000 if (error) { 1001 return (1); 1002 } 1003 } else { 1004 if (!sync) { 1005 /* 1006 * Call callback so that refcount is decremented 1007 * properly: 1008 */ 1009 pc->tag_parent->dma_error = 0; 1010 (pc->tag_parent->func) (pc->tag_parent); 1011 } 1012 } 1013 return (0); 1014 } 1015 1016 /*------------------------------------------------------------------------* 1017 * usb2_pc_cpu_invalidate - invalidate CPU cache 1018 *------------------------------------------------------------------------*/ 1019 void 1020 usb2_pc_cpu_invalidate(struct usb2_page_cache *pc) 1021 { 1022 uint32_t len; 1023 1024 len = pc->page_offset_end - pc->page_offset_buf; 1025 1026 if (len == 0) { 1027 /* nothing has been loaded into this page cache */ 1028 return; 1029 } 1030 bus_dmamap_sync(pc->tag, pc->map, 0, len, 1031 BUS_DMASYNC_POSTWRITE | BUS_DMASYNC_POSTREAD); 1032 } 1033 1034 /*------------------------------------------------------------------------* 1035 * usb2_pc_cpu_flush - flush CPU cache 1036 *------------------------------------------------------------------------*/ 1037 void 1038 usb2_pc_cpu_flush(struct usb2_page_cache *pc) 1039 { 1040 uint32_t len; 1041 1042 len = pc->page_offset_end - pc->page_offset_buf; 1043 1044 if (len == 0) { 1045 /* nothing has been loaded into this page cache */ 1046 return; 1047 } 1048 bus_dmamap_sync(pc->tag, pc->map, 0, len, 1049 BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD); 1050 } 1051 1052 /*------------------------------------------------------------------------* 1053 * usb2_pc_dmamap_create - create a DMA map 1054 * 1055 * Returns: 1056 * 0: Success 1057 * Else: Failure 1058 *------------------------------------------------------------------------*/ 1059 uint8_t 1060 usb2_pc_dmamap_create(struct usb2_page_cache *pc, uint32_t size) 1061 { 1062 struct usb2_xfer_root *info; 1063 struct usb2_dma_tag *utag; 1064 1065 /* get info */ 1066 info = pc->tag_parent->info; 1067 1068 /* sanity check */ 1069 if (info == NULL) { 1070 goto error; 1071 } 1072 utag = usb2_dma_tag_find(pc->tag_parent, size, 1); 1073 if (utag == NULL) { 1074 goto error; 1075 } 1076 if (bus_dmamap_create(utag->tag, size, utag->n_seg, 1077 USB_PAGE_SIZE, 0, BUS_DMA_WAITOK, &pc->map)) { 1078 goto error; 1079 } 1080 pc->tag = utag->tag; 1081 pc->p_seg = utag->p_seg; 1082 pc->n_seg = utag->n_seg; 1083 return 0; /* success */ 1084 1085 error: 1086 pc->map = NULL; 1087 pc->tag = NULL; 1088 pc->p_seg = NULL; 1089 pc->n_seg = 0; 1090 return 1; /* failure */ 1091 } 1092 1093 /*------------------------------------------------------------------------* 1094 * usb2_pc_dmamap_destroy 1095 * 1096 * This function is NULL safe. 1097 *------------------------------------------------------------------------*/ 1098 void 1099 usb2_pc_dmamap_destroy(struct usb2_page_cache *pc) 1100 { 1101 if (pc && pc->tag) { 1102 bus_dmamap_destroy(pc->tag, pc->map); 1103 pc->tag = NULL; 1104 pc->map = NULL; 1105 } 1106 } 1107 1108 #endif 1109 1110 /*------------------------------------------------------------------------* 1111 * usb2_dma_tag_find - factored out code 1112 *------------------------------------------------------------------------*/ 1113 struct usb2_dma_tag * 1114 usb2_dma_tag_find(struct usb2_dma_parent_tag *udpt, 1115 uint32_t size, uint32_t align) 1116 { 1117 struct usb2_dma_tag *udt; 1118 uint8_t nudt; 1119 1120 USB_ASSERT(align > 0, ("Invalid parameter align = 0!\n")); 1121 USB_ASSERT(size > 0, ("Invalid parameter size = 0!\n")); 1122 1123 udt = udpt->utag_first; 1124 nudt = udpt->utag_max; 1125 1126 while (nudt--) { 1127 1128 if (udt->align == 0) { 1129 usb2_dma_tag_create(udt, size, align); 1130 if (udt->tag == NULL) { 1131 return (NULL); 1132 } 1133 udt->align = align; 1134 udt->size = size; 1135 return (udt); 1136 } 1137 if ((udt->align == align) && (udt->size == size)) { 1138 return (udt); 1139 } 1140 udt++; 1141 } 1142 return (NULL); 1143 } 1144 1145 /*------------------------------------------------------------------------* 1146 * usb2_dma_tag_setup - initialise USB DMA tags 1147 *------------------------------------------------------------------------*/ 1148 void 1149 usb2_dma_tag_setup(struct usb2_dma_parent_tag *udpt, 1150 struct usb2_dma_tag *udt, bus_dma_tag_t dmat, 1151 struct mtx *mtx, usb2_dma_callback_t *func, 1152 struct usb2_xfer_root *info, uint8_t ndmabits, 1153 uint8_t nudt) 1154 { 1155 bzero(udpt, sizeof(*udpt)); 1156 1157 /* sanity checking */ 1158 if ((nudt == 0) || 1159 (ndmabits == 0) || 1160 (mtx == NULL)) { 1161 /* something is corrupt */ 1162 return; 1163 } 1164 #ifdef __FreeBSD__ 1165 /* initialise condition variable */ 1166 usb2_cv_init(udpt->cv, "USB DMA CV"); 1167 #endif 1168 1169 /* store some information */ 1170 udpt->mtx = mtx; 1171 udpt->info = info; 1172 udpt->func = func; 1173 udpt->tag = dmat; 1174 udpt->utag_first = udt; 1175 udpt->utag_max = nudt; 1176 udpt->dma_bits = ndmabits; 1177 1178 while (nudt--) { 1179 bzero(udt, sizeof(*udt)); 1180 udt->tag_parent = udpt; 1181 udt++; 1182 } 1183 } 1184 1185 /*------------------------------------------------------------------------* 1186 * usb2_bus_tag_unsetup - factored out code 1187 *------------------------------------------------------------------------*/ 1188 void 1189 usb2_dma_tag_unsetup(struct usb2_dma_parent_tag *udpt) 1190 { 1191 struct usb2_dma_tag *udt; 1192 uint8_t nudt; 1193 1194 udt = udpt->utag_first; 1195 nudt = udpt->utag_max; 1196 1197 while (nudt--) { 1198 1199 if (udt->align) { 1200 /* destroy the USB DMA tag */ 1201 usb2_dma_tag_destroy(udt); 1202 udt->align = 0; 1203 } 1204 udt++; 1205 } 1206 1207 if (udpt->utag_max) { 1208 #ifdef __FreeBSD__ 1209 /* destroy the condition variable */ 1210 usb2_cv_destroy(udpt->cv); 1211 #endif 1212 } 1213 } 1214 1215 /*------------------------------------------------------------------------* 1216 * usb2_bdma_work_loop 1217 * 1218 * This function handles loading of virtual buffers into DMA and is 1219 * only called when "dma_refcount" is zero. 1220 *------------------------------------------------------------------------*/ 1221 void 1222 usb2_bdma_work_loop(struct usb2_xfer_queue *pq) 1223 { 1224 struct usb2_xfer_root *info; 1225 struct usb2_xfer *xfer; 1226 uint32_t nframes; 1227 1228 xfer = pq->curr; 1229 info = xfer->xroot; 1230 1231 mtx_assert(info->xfer_mtx, MA_OWNED); 1232 1233 if (xfer->error) { 1234 /* some error happened */ 1235 USB_BUS_LOCK(info->bus); 1236 usb2_transfer_done(xfer, 0); 1237 USB_BUS_UNLOCK(info->bus); 1238 return; 1239 } 1240 if (!xfer->flags_int.bdma_setup) { 1241 struct usb2_page *pg; 1242 uint32_t frlength_0; 1243 uint8_t isread; 1244 1245 xfer->flags_int.bdma_setup = 1; 1246 1247 /* reset BUS-DMA load state */ 1248 1249 info->dma_error = 0; 1250 1251 if (xfer->flags_int.isochronous_xfr) { 1252 /* only one frame buffer */ 1253 nframes = 1; 1254 frlength_0 = xfer->sumlen; 1255 } else { 1256 /* can be multiple frame buffers */ 1257 nframes = xfer->nframes; 1258 frlength_0 = xfer->frlengths[0]; 1259 } 1260 1261 /* 1262 * Set DMA direction first. This is needed to 1263 * select the correct cache invalidate and cache 1264 * flush operations. 1265 */ 1266 isread = USB_GET_DATA_ISREAD(xfer); 1267 pg = xfer->dma_page_ptr; 1268 1269 if (xfer->flags_int.control_xfr && 1270 xfer->flags_int.control_hdr) { 1271 /* special case */ 1272 if (xfer->flags_int.usb2_mode == USB_MODE_DEVICE) { 1273 /* The device controller writes to memory */ 1274 xfer->frbuffers[0].isread = 1; 1275 } else { 1276 /* The host controller reads from memory */ 1277 xfer->frbuffers[0].isread = 0; 1278 } 1279 } else { 1280 /* default case */ 1281 xfer->frbuffers[0].isread = isread; 1282 } 1283 1284 /* 1285 * Setup the "page_start" pointer which points to an array of 1286 * USB pages where information about the physical address of a 1287 * page will be stored. Also initialise the "isread" field of 1288 * the USB page caches. 1289 */ 1290 xfer->frbuffers[0].page_start = pg; 1291 1292 info->dma_nframes = nframes; 1293 info->dma_currframe = 0; 1294 info->dma_frlength_0 = frlength_0; 1295 1296 pg += (frlength_0 / USB_PAGE_SIZE); 1297 pg += 2; 1298 1299 while (--nframes > 0) { 1300 xfer->frbuffers[nframes].isread = isread; 1301 xfer->frbuffers[nframes].page_start = pg; 1302 1303 pg += (xfer->frlengths[nframes] / USB_PAGE_SIZE); 1304 pg += 2; 1305 } 1306 1307 } 1308 if (info->dma_error) { 1309 USB_BUS_LOCK(info->bus); 1310 usb2_transfer_done(xfer, USB_ERR_DMA_LOAD_FAILED); 1311 USB_BUS_UNLOCK(info->bus); 1312 return; 1313 } 1314 if (info->dma_currframe != info->dma_nframes) { 1315 1316 if (info->dma_currframe == 0) { 1317 /* special case */ 1318 usb2_pc_load_mem(xfer->frbuffers, 1319 info->dma_frlength_0, 0); 1320 } else { 1321 /* default case */ 1322 nframes = info->dma_currframe; 1323 usb2_pc_load_mem(xfer->frbuffers + nframes, 1324 xfer->frlengths[nframes], 0); 1325 } 1326 1327 /* advance frame index */ 1328 info->dma_currframe++; 1329 1330 return; 1331 } 1332 /* go ahead */ 1333 usb2_bdma_pre_sync(xfer); 1334 1335 /* start loading next USB transfer, if any */ 1336 usb2_command_wrapper(pq, NULL); 1337 1338 /* finally start the hardware */ 1339 usb2_pipe_enter(xfer); 1340 } 1341 1342 /*------------------------------------------------------------------------* 1343 * usb2_bdma_done_event 1344 * 1345 * This function is called when the BUS-DMA has loaded virtual memory 1346 * into DMA, if any. 1347 *------------------------------------------------------------------------*/ 1348 void 1349 usb2_bdma_done_event(struct usb2_dma_parent_tag *udpt) 1350 { 1351 struct usb2_xfer_root *info; 1352 1353 info = udpt->info; 1354 1355 mtx_assert(info->xfer_mtx, MA_OWNED); 1356 1357 /* copy error */ 1358 info->dma_error = udpt->dma_error; 1359 1360 /* enter workloop again */ 1361 usb2_command_wrapper(&info->dma_q, 1362 info->dma_q.curr); 1363 } 1364 1365 /*------------------------------------------------------------------------* 1366 * usb2_bdma_pre_sync 1367 * 1368 * This function handles DMA synchronisation that must be done before 1369 * an USB transfer is started. 1370 *------------------------------------------------------------------------*/ 1371 void 1372 usb2_bdma_pre_sync(struct usb2_xfer *xfer) 1373 { 1374 struct usb2_page_cache *pc; 1375 uint32_t nframes; 1376 1377 if (xfer->flags_int.isochronous_xfr) { 1378 /* only one frame buffer */ 1379 nframes = 1; 1380 } else { 1381 /* can be multiple frame buffers */ 1382 nframes = xfer->nframes; 1383 } 1384 1385 pc = xfer->frbuffers; 1386 1387 while (nframes--) { 1388 1389 if (pc->isread) { 1390 usb2_pc_cpu_invalidate(pc); 1391 } else { 1392 usb2_pc_cpu_flush(pc); 1393 } 1394 pc++; 1395 } 1396 } 1397 1398 /*------------------------------------------------------------------------* 1399 * usb2_bdma_post_sync 1400 * 1401 * This function handles DMA synchronisation that must be done after 1402 * an USB transfer is complete. 1403 *------------------------------------------------------------------------*/ 1404 void 1405 usb2_bdma_post_sync(struct usb2_xfer *xfer) 1406 { 1407 struct usb2_page_cache *pc; 1408 uint32_t nframes; 1409 1410 if (xfer->flags_int.isochronous_xfr) { 1411 /* only one frame buffer */ 1412 nframes = 1; 1413 } else { 1414 /* can be multiple frame buffers */ 1415 nframes = xfer->nframes; 1416 } 1417 1418 pc = xfer->frbuffers; 1419 1420 while (nframes--) { 1421 if (pc->isread) { 1422 usb2_pc_cpu_invalidate(pc); 1423 } 1424 pc++; 1425 } 1426 } 1427