1 /* 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.94 2008/08/15 20:51:31 kmacy Exp $ 27 */ 28 29 #include <sys/param.h> 30 #include <sys/systm.h> 31 #include <sys/malloc.h> 32 #include <sys/mbuf.h> 33 #include <sys/uio.h> 34 #include <sys/thread2.h> 35 #include <sys/bus_dma.h> 36 #include <sys/kernel.h> 37 #include <sys/sysctl.h> 38 #include <sys/lock.h> 39 #include <sys/spinlock2.h> 40 41 #include <vm/vm.h> 42 #include <vm/vm_page.h> 43 44 /* XXX needed for to access pmap to convert per-proc virtual to physical */ 45 #include <sys/proc.h> 46 #include <sys/lock.h> 47 #include <vm/vm_map.h> 48 49 #include <machine/md_var.h> 50 51 #define MAX_BPAGES 1024 52 53 struct bounce_zone; 54 struct bus_dmamap; 55 56 struct bus_dma_tag { 57 bus_dma_tag_t parent; 58 bus_size_t alignment; 59 bus_size_t boundary; 60 bus_addr_t lowaddr; 61 bus_addr_t highaddr; 62 bus_dma_filter_t *filter; 63 void *filterarg; 64 bus_size_t maxsize; 65 u_int nsegments; 66 bus_size_t maxsegsz; 67 int flags; 68 int ref_count; 69 int map_count; 70 bus_dma_segment_t *segments; 71 struct bounce_zone *bounce_zone; 72 }; 73 74 /* 75 * bus_dma_tag private flags 76 */ 77 #define BUS_DMA_BOUNCE_ALIGN BUS_DMA_BUS2 78 #define BUS_DMA_BOUNCE_LOWADDR BUS_DMA_BUS3 79 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 80 81 #define BUS_DMA_COULD_BOUNCE (BUS_DMA_BOUNCE_LOWADDR | BUS_DMA_BOUNCE_ALIGN) 82 83 #define BUS_DMAMEM_KMALLOC(dmat) \ 84 ((dmat)->maxsize <= PAGE_SIZE && \ 85 (dmat)->alignment <= PAGE_SIZE && \ 86 (dmat)->lowaddr >= ptoa(Maxmem)) 87 88 struct bounce_page { 89 vm_offset_t vaddr; /* kva of bounce buffer */ 90 bus_addr_t busaddr; /* Physical address */ 91 vm_offset_t datavaddr; /* kva of client data */ 92 bus_size_t datacount; /* client data count */ 93 STAILQ_ENTRY(bounce_page) links; 94 }; 95 96 struct bounce_zone { 97 STAILQ_ENTRY(bounce_zone) links; 98 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 99 STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 100 #ifdef SMP 101 struct spinlock spin; 102 #else 103 int unused0; 104 #endif 105 int total_bpages; 106 int free_bpages; 107 int reserved_bpages; 108 int active_bpages; 109 int total_bounced; 110 int total_deferred; 111 int reserve_failed; 112 bus_size_t alignment; 113 bus_addr_t lowaddr; 114 char zoneid[8]; 115 char lowaddrid[20]; 116 struct sysctl_ctx_list sysctl_ctx; 117 struct sysctl_oid *sysctl_tree; 118 }; 119 120 #ifdef SMP 121 #define BZ_LOCK(bz) spin_lock(&(bz)->spin) 122 #define BZ_UNLOCK(bz) spin_unlock(&(bz)->spin) 123 #else 124 #define BZ_LOCK(bz) crit_enter() 125 #define BZ_UNLOCK(bz) crit_exit() 126 #endif 127 128 static struct lwkt_token bounce_zone_tok = 129 LWKT_TOKEN_INITIALIZER(bounce_zone_token); 130 static int busdma_zonecount; 131 static STAILQ_HEAD(, bounce_zone) bounce_zone_list = 132 STAILQ_HEAD_INITIALIZER(bounce_zone_list); 133 134 int busdma_swi_pending; 135 static int total_bounce_pages; 136 static int max_bounce_pages = MAX_BPAGES; 137 static int bounce_alignment = 1; /* XXX temporary */ 138 139 TUNABLE_INT("hw.busdma.max_bpages", &max_bounce_pages); 140 TUNABLE_INT("hw.busdma.bounce_alignment", &bounce_alignment); 141 142 struct bus_dmamap { 143 struct bp_list bpages; 144 int pagesneeded; 145 int pagesreserved; 146 bus_dma_tag_t dmat; 147 void *buf; /* unmapped buffer pointer */ 148 bus_size_t buflen; /* unmapped buffer length */ 149 bus_dmamap_callback_t *callback; 150 void *callback_arg; 151 STAILQ_ENTRY(bus_dmamap) links; 152 }; 153 154 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist = 155 STAILQ_HEAD_INITIALIZER(bounce_map_callbacklist); 156 157 static struct bus_dmamap nobounce_dmamap; 158 159 static int alloc_bounce_zone(bus_dma_tag_t); 160 static int alloc_bounce_pages(bus_dma_tag_t, u_int, int); 161 static int reserve_bounce_pages(bus_dma_tag_t, bus_dmamap_t, int); 162 static void return_bounce_pages(bus_dma_tag_t, bus_dmamap_t); 163 static bus_addr_t add_bounce_page(bus_dma_tag_t, bus_dmamap_t, 164 vm_offset_t, bus_size_t); 165 static void free_bounce_page(bus_dma_tag_t, struct bounce_page *); 166 167 static bus_dmamap_t get_map_waiting(bus_dma_tag_t); 168 static void add_map_callback(bus_dmamap_t); 169 170 SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 171 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bounce_pages, 172 0, "Total bounce pages"); 173 SYSCTL_INT(_hw_busdma, OID_AUTO, max_bpages, CTLFLAG_RD, &max_bounce_pages, 174 0, "Max bounce pages per bounce zone"); 175 SYSCTL_INT(_hw_busdma, OID_AUTO, bounce_alignment, CTLFLAG_RD, 176 &bounce_alignment, 0, "Obey alignment constraint"); 177 178 static __inline int 179 run_filter(bus_dma_tag_t dmat, bus_addr_t paddr) 180 { 181 int retval; 182 183 retval = 0; 184 do { 185 if (((paddr > dmat->lowaddr && paddr <= dmat->highaddr) || 186 (bounce_alignment && (paddr & (dmat->alignment - 1)) != 0)) 187 && (dmat->filter == NULL || 188 dmat->filter(dmat->filterarg, paddr) != 0)) 189 retval = 1; 190 191 dmat = dmat->parent; 192 } while (retval == 0 && dmat != NULL); 193 return (retval); 194 } 195 196 /* 197 * Allocate a device specific dma_tag. 198 */ 199 int 200 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 201 bus_size_t boundary, bus_addr_t lowaddr, 202 bus_addr_t highaddr, bus_dma_filter_t *filter, 203 void *filterarg, bus_size_t maxsize, int nsegments, 204 bus_size_t maxsegsz, int flags, bus_dma_tag_t *dmat) 205 { 206 bus_dma_tag_t newtag; 207 int error = 0; 208 209 /* 210 * Sanity checks 211 */ 212 213 if (alignment == 0) 214 alignment = 1; 215 if (alignment & (alignment - 1)) 216 panic("alignment must be power of 2"); 217 218 if (boundary != 0) { 219 if (boundary & (boundary - 1)) 220 panic("boundary must be power of 2"); 221 if (boundary < maxsegsz) { 222 kprintf("boundary < maxsegsz:\n"); 223 print_backtrace(-1); 224 maxsegsz = boundary; 225 } 226 } 227 228 /* Return a NULL tag on failure */ 229 *dmat = NULL; 230 231 newtag = kmalloc(sizeof(*newtag), M_DEVBUF, M_INTWAIT); 232 233 newtag->parent = parent; 234 newtag->alignment = alignment; 235 newtag->boundary = boundary; 236 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 237 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 238 newtag->filter = filter; 239 newtag->filterarg = filterarg; 240 newtag->maxsize = maxsize; 241 newtag->nsegments = nsegments; 242 newtag->maxsegsz = maxsegsz; 243 newtag->flags = flags; 244 newtag->ref_count = 1; /* Count ourself */ 245 newtag->map_count = 0; 246 newtag->segments = NULL; 247 newtag->bounce_zone = NULL; 248 249 /* Take into account any restrictions imposed by our parent tag */ 250 if (parent != NULL) { 251 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 252 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 253 254 if (newtag->boundary == 0) { 255 newtag->boundary = parent->boundary; 256 } else if (parent->boundary != 0) { 257 newtag->boundary = MIN(parent->boundary, 258 newtag->boundary); 259 } 260 261 #ifdef notyet 262 newtag->alignment = MAX(parent->alignment, newtag->alignment); 263 #endif 264 265 if (newtag->filter == NULL) { 266 /* 267 * Short circuit looking at our parent directly 268 * since we have encapsulated all of its information 269 */ 270 newtag->filter = parent->filter; 271 newtag->filterarg = parent->filterarg; 272 newtag->parent = parent->parent; 273 } 274 if (newtag->parent != NULL) 275 parent->ref_count++; 276 } 277 278 if (newtag->lowaddr < ptoa(Maxmem)) 279 newtag->flags |= BUS_DMA_BOUNCE_LOWADDR; 280 if (bounce_alignment && newtag->alignment > 1 && 281 !(newtag->flags & BUS_DMA_ALIGNED)) 282 newtag->flags |= BUS_DMA_BOUNCE_ALIGN; 283 284 if ((newtag->flags & BUS_DMA_COULD_BOUNCE) && 285 (flags & BUS_DMA_ALLOCNOW) != 0) { 286 struct bounce_zone *bz; 287 288 /* Must bounce */ 289 290 error = alloc_bounce_zone(newtag); 291 if (error) 292 goto back; 293 bz = newtag->bounce_zone; 294 295 if (ptoa(bz->total_bpages) < maxsize) { 296 int pages; 297 298 if (flags & BUS_DMA_ONEBPAGE) { 299 pages = 1; 300 } else { 301 pages = atop(round_page(maxsize)) - 302 bz->total_bpages; 303 pages = MAX(pages, 1); 304 } 305 306 /* Add pages to our bounce pool */ 307 if (alloc_bounce_pages(newtag, pages, flags) < pages) 308 error = ENOMEM; 309 310 /* Performed initial allocation */ 311 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 312 } 313 } 314 back: 315 if (error) 316 kfree(newtag, M_DEVBUF); 317 else 318 *dmat = newtag; 319 return error; 320 } 321 322 int 323 bus_dma_tag_destroy(bus_dma_tag_t dmat) 324 { 325 if (dmat != NULL) { 326 if (dmat->map_count != 0) 327 return (EBUSY); 328 329 while (dmat != NULL) { 330 bus_dma_tag_t parent; 331 332 parent = dmat->parent; 333 dmat->ref_count--; 334 if (dmat->ref_count == 0) { 335 if (dmat->segments != NULL) 336 kfree(dmat->segments, M_DEVBUF); 337 kfree(dmat, M_DEVBUF); 338 /* 339 * Last reference count, so 340 * release our reference 341 * count on our parent. 342 */ 343 dmat = parent; 344 } else 345 dmat = NULL; 346 } 347 } 348 return (0); 349 } 350 351 /* 352 * Allocate a handle for mapping from kva/uva/physical 353 * address space into bus device space. 354 */ 355 int 356 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 357 { 358 int error; 359 360 error = 0; 361 362 if (dmat->segments == NULL) { 363 KKASSERT(dmat->nsegments && dmat->nsegments < 16384); 364 dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * 365 dmat->nsegments, M_DEVBUF, M_INTWAIT); 366 } 367 368 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 369 struct bounce_zone *bz; 370 int maxpages; 371 372 /* Must bounce */ 373 374 if (dmat->bounce_zone == NULL) { 375 error = alloc_bounce_zone(dmat); 376 if (error) 377 return error; 378 } 379 bz = dmat->bounce_zone; 380 381 *mapp = kmalloc(sizeof(**mapp), M_DEVBUF, M_INTWAIT | M_ZERO); 382 383 /* Initialize the new map */ 384 STAILQ_INIT(&((*mapp)->bpages)); 385 386 /* 387 * Attempt to add pages to our pool on a per-instance 388 * basis up to a sane limit. 389 */ 390 if (dmat->flags & BUS_DMA_BOUNCE_ALIGN) { 391 maxpages = max_bounce_pages; 392 } else { 393 maxpages = MIN(max_bounce_pages, 394 Maxmem - atop(dmat->lowaddr)); 395 } 396 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 397 || (dmat->map_count > 0 398 && bz->total_bpages < maxpages)) { 399 int pages; 400 401 if (flags & BUS_DMA_ONEBPAGE) { 402 pages = 1; 403 } else { 404 pages = atop(round_page(dmat->maxsize)); 405 pages = MIN(maxpages - bz->total_bpages, pages); 406 pages = MAX(pages, 1); 407 } 408 if (alloc_bounce_pages(dmat, pages, flags) < pages) 409 error = ENOMEM; 410 411 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 412 if (!error) 413 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 414 } else { 415 error = 0; 416 } 417 } 418 } else { 419 *mapp = NULL; 420 } 421 if (!error) 422 dmat->map_count++; 423 return error; 424 } 425 426 /* 427 * Destroy a handle for mapping from kva/uva/physical 428 * address space into bus device space. 429 */ 430 int 431 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 432 { 433 if (map != NULL) { 434 if (STAILQ_FIRST(&map->bpages) != NULL) 435 return (EBUSY); 436 kfree(map, M_DEVBUF); 437 } 438 dmat->map_count--; 439 return (0); 440 } 441 442 static __inline bus_size_t 443 check_kmalloc(bus_dma_tag_t dmat, const void *vaddr0, int verify) 444 { 445 bus_size_t maxsize = 0; 446 uintptr_t vaddr = (uintptr_t)vaddr0; 447 448 if ((vaddr ^ (vaddr + dmat->maxsize - 1)) & ~PAGE_MASK) { 449 kprintf("boundary check failed\n"); 450 if (verify) 451 print_backtrace(-1); /* XXX panic */ 452 maxsize = dmat->maxsize; 453 } 454 if (vaddr & (dmat->alignment - 1)) { 455 kprintf("alignment check failed\n"); 456 if (verify) 457 print_backtrace(-1); /* XXX panic */ 458 if (dmat->maxsize < dmat->alignment) 459 maxsize = dmat->alignment; 460 else 461 maxsize = dmat->maxsize; 462 } 463 return maxsize; 464 } 465 466 /* 467 * Allocate a piece of memory that can be efficiently mapped into 468 * bus device space based on the constraints lited in the dma tag. 469 * 470 * mapp is degenerate. By definition this allocation should not require 471 * bounce buffers so do not allocate a dma map. 472 */ 473 int 474 bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, 475 bus_dmamap_t *mapp) 476 { 477 int mflags; 478 479 /* If we succeed, no mapping/bouncing will be required */ 480 *mapp = NULL; 481 482 if (dmat->segments == NULL) { 483 KKASSERT(dmat->nsegments < 16384); 484 dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * 485 dmat->nsegments, M_DEVBUF, M_INTWAIT); 486 } 487 488 if (flags & BUS_DMA_NOWAIT) 489 mflags = M_NOWAIT; 490 else 491 mflags = M_WAITOK; 492 if (flags & BUS_DMA_ZERO) 493 mflags |= M_ZERO; 494 495 if (BUS_DMAMEM_KMALLOC(dmat)) { 496 bus_size_t maxsize; 497 498 *vaddr = kmalloc(dmat->maxsize, M_DEVBUF, mflags); 499 500 /* 501 * XXX 502 * Check whether the allocation 503 * - crossed a page boundary 504 * - was not aligned 505 * Retry with power-of-2 alignment in the above cases. 506 */ 507 maxsize = check_kmalloc(dmat, *vaddr, 0); 508 if (maxsize) { 509 size_t size; 510 511 kfree(*vaddr, M_DEVBUF); 512 /* XXX check for overflow? */ 513 for (size = 1; size <= maxsize; size <<= 1) 514 ; 515 *vaddr = kmalloc(size, M_DEVBUF, mflags); 516 check_kmalloc(dmat, *vaddr, 1); 517 } 518 } else { 519 /* 520 * XXX Use Contigmalloc until it is merged into this facility 521 * and handles multi-seg allocations. Nobody is doing 522 * multi-seg allocations yet though. 523 */ 524 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 525 0ul, dmat->lowaddr, dmat->alignment, dmat->boundary); 526 } 527 if (*vaddr == NULL) 528 return (ENOMEM); 529 return (0); 530 } 531 532 /* 533 * Free a piece of memory and it's allociated dmamap, that was allocated 534 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 535 */ 536 void 537 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 538 { 539 /* 540 * dmamem does not need to be bounced, so the map should be 541 * NULL 542 */ 543 if (map != NULL) 544 panic("bus_dmamem_free: Invalid map freed"); 545 if (BUS_DMAMEM_KMALLOC(dmat)) 546 kfree(vaddr, M_DEVBUF); 547 else 548 contigfree(vaddr, dmat->maxsize, M_DEVBUF); 549 } 550 551 static __inline vm_paddr_t 552 _bus_dma_extract(pmap_t pmap, vm_offset_t vaddr) 553 { 554 if (pmap) 555 return pmap_extract(pmap, vaddr); 556 else 557 return pmap_kextract(vaddr); 558 } 559 560 /* 561 * Utility function to load a linear buffer. lastaddrp holds state 562 * between invocations (for multiple-buffer loads). segp contains 563 * the segment following the starting one on entrace, and the ending 564 * segment on exit. first indicates if this is the first invocation 565 * of this function. 566 */ 567 static int 568 _bus_dmamap_load_buffer(bus_dma_tag_t dmat, 569 bus_dmamap_t map, 570 void *buf, bus_size_t buflen, 571 bus_dma_segment_t *segments, 572 int nsegments, 573 pmap_t pmap, 574 int flags, 575 vm_paddr_t *lastpaddrp, 576 int *segp, 577 int first) 578 { 579 vm_offset_t vaddr; 580 vm_paddr_t paddr, nextpaddr; 581 bus_dma_segment_t *sg; 582 bus_addr_t bmask; 583 int seg, error = 0; 584 585 if (map == NULL) 586 map = &nobounce_dmamap; 587 588 #ifdef INVARIANTS 589 if (dmat->flags & BUS_DMA_ALIGNED) 590 KKASSERT(((uintptr_t)buf & (dmat->alignment - 1)) == 0); 591 #endif 592 593 /* 594 * If we are being called during a callback, pagesneeded will 595 * be non-zero, so we can avoid doing the work twice. 596 */ 597 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) && 598 map != &nobounce_dmamap && map->pagesneeded == 0) { 599 vm_offset_t vendaddr; 600 601 /* 602 * Count the number of bounce pages 603 * needed in order to complete this transfer 604 */ 605 vaddr = (vm_offset_t)buf; 606 vendaddr = (vm_offset_t)buf + buflen; 607 608 while (vaddr < vendaddr) { 609 paddr = _bus_dma_extract(pmap, vaddr); 610 if (run_filter(dmat, paddr) != 0) 611 map->pagesneeded++; 612 vaddr += (PAGE_SIZE - (vaddr & PAGE_MASK)); 613 } 614 } 615 616 /* Reserve Necessary Bounce Pages */ 617 if (map->pagesneeded != 0) { 618 struct bounce_zone *bz; 619 620 bz = dmat->bounce_zone; 621 BZ_LOCK(bz); 622 if (flags & BUS_DMA_NOWAIT) { 623 if (reserve_bounce_pages(dmat, map, 0) != 0) { 624 BZ_UNLOCK(bz); 625 error = ENOMEM; 626 goto free_bounce; 627 } 628 } else { 629 if (reserve_bounce_pages(dmat, map, 1) != 0) { 630 /* Queue us for resources */ 631 map->dmat = dmat; 632 map->buf = buf; 633 map->buflen = buflen; 634 635 STAILQ_INSERT_TAIL( 636 &dmat->bounce_zone->bounce_map_waitinglist, 637 map, links); 638 BZ_UNLOCK(bz); 639 640 return (EINPROGRESS); 641 } 642 } 643 BZ_UNLOCK(bz); 644 } 645 646 KKASSERT(*segp >= 1 && *segp <= nsegments); 647 seg = *segp; 648 sg = &segments[seg - 1]; 649 650 vaddr = (vm_offset_t)buf; 651 nextpaddr = *lastpaddrp; 652 bmask = ~(dmat->boundary - 1); /* note: will be 0 if boundary is 0 */ 653 654 /* force at least one segment */ 655 do { 656 bus_size_t size; 657 658 /* 659 * Per-page main loop 660 */ 661 paddr = _bus_dma_extract(pmap, vaddr); 662 size = PAGE_SIZE - (paddr & PAGE_MASK); 663 if (size > buflen) 664 size = buflen; 665 if (map->pagesneeded != 0 && run_filter(dmat, paddr)) { 666 /* 667 * note: this paddr has the same in-page offset 668 * as vaddr and thus the paddr above, so the 669 * size does not have to be recalculated 670 */ 671 paddr = add_bounce_page(dmat, map, vaddr, size); 672 } 673 674 /* 675 * Fill in the bus_dma_segment 676 */ 677 if (first) { 678 sg->ds_addr = paddr; 679 sg->ds_len = size; 680 first = 0; 681 } else if (paddr == nextpaddr) { 682 sg->ds_len += size; 683 } else { 684 sg++; 685 seg++; 686 if (seg > nsegments) 687 break; 688 sg->ds_addr = paddr; 689 sg->ds_len = size; 690 } 691 nextpaddr = paddr + size; 692 693 /* 694 * Handle maxsegsz and boundary issues with a nested loop 695 */ 696 for (;;) { 697 bus_size_t tmpsize; 698 699 /* 700 * Limit to the boundary and maximum segment size 701 */ 702 if (((nextpaddr - 1) ^ sg->ds_addr) & bmask) { 703 tmpsize = dmat->boundary - 704 (sg->ds_addr & ~bmask); 705 if (tmpsize > dmat->maxsegsz) 706 tmpsize = dmat->maxsegsz; 707 KKASSERT(tmpsize < sg->ds_len); 708 } else if (sg->ds_len > dmat->maxsegsz) { 709 tmpsize = dmat->maxsegsz; 710 } else { 711 break; 712 } 713 714 /* 715 * Futz, split the data into a new segment. 716 */ 717 if (seg >= nsegments) 718 goto fail; 719 sg[1].ds_len = sg[0].ds_len - tmpsize; 720 sg[1].ds_addr = sg[0].ds_addr + tmpsize; 721 sg[0].ds_len = tmpsize; 722 sg++; 723 seg++; 724 } 725 726 /* 727 * Adjust for loop 728 */ 729 buflen -= size; 730 vaddr += size; 731 } while (buflen > 0); 732 fail: 733 if (buflen != 0) 734 error = EFBIG; 735 736 *segp = seg; 737 *lastpaddrp = nextpaddr; 738 739 free_bounce: 740 if (error && (dmat->flags & BUS_DMA_COULD_BOUNCE) && 741 map != &nobounce_dmamap) { 742 _bus_dmamap_unload(dmat, map); 743 return_bounce_pages(dmat, map); 744 } 745 return error; 746 } 747 748 /* 749 * Map the buffer buf into bus space using the dmamap map. 750 */ 751 int 752 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 753 bus_size_t buflen, bus_dmamap_callback_t *callback, 754 void *callback_arg, int flags) 755 { 756 vm_paddr_t lastaddr = 0; 757 int error, nsegs = 1; 758 759 if (map != NULL) { 760 /* 761 * XXX 762 * Follow old semantics. Once all of the callers are fixed, 763 * we should get rid of these internal flag "adjustment". 764 */ 765 flags &= ~BUS_DMA_NOWAIT; 766 flags |= BUS_DMA_WAITOK; 767 768 map->callback = callback; 769 map->callback_arg = callback_arg; 770 } 771 772 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, 773 dmat->segments, dmat->nsegments, 774 NULL, flags, &lastaddr, &nsegs, 1); 775 if (error == EINPROGRESS) 776 return error; 777 778 callback(callback_arg, dmat->segments, nsegs, error); 779 return 0; 780 } 781 782 /* 783 * Like _bus_dmamap_load(), but for mbufs. 784 */ 785 int 786 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 787 struct mbuf *m0, 788 bus_dmamap_callback2_t *callback, void *callback_arg, 789 int flags) 790 { 791 int nsegs, error; 792 793 /* 794 * XXX 795 * Follow old semantics. Once all of the callers are fixed, 796 * we should get rid of these internal flag "adjustment". 797 */ 798 flags &= ~BUS_DMA_WAITOK; 799 flags |= BUS_DMA_NOWAIT; 800 801 error = bus_dmamap_load_mbuf_segment(dmat, map, m0, 802 dmat->segments, dmat->nsegments, &nsegs, flags); 803 if (error) { 804 /* force "no valid mappings" in callback */ 805 callback(callback_arg, dmat->segments, 0, 0, error); 806 } else { 807 callback(callback_arg, dmat->segments, nsegs, 808 m0->m_pkthdr.len, error); 809 } 810 return error; 811 } 812 813 int 814 bus_dmamap_load_mbuf_segment(bus_dma_tag_t dmat, bus_dmamap_t map, 815 struct mbuf *m0, 816 bus_dma_segment_t *segs, int maxsegs, 817 int *nsegs, int flags) 818 { 819 int error; 820 821 M_ASSERTPKTHDR(m0); 822 823 KASSERT(maxsegs >= 1, ("invalid maxsegs %d", maxsegs)); 824 KASSERT(maxsegs <= dmat->nsegments, 825 ("%d too many segments, dmat only supports %d segments", 826 maxsegs, dmat->nsegments)); 827 KASSERT(flags & BUS_DMA_NOWAIT, 828 ("only BUS_DMA_NOWAIT is supported")); 829 830 if (m0->m_pkthdr.len <= dmat->maxsize) { 831 int first = 1; 832 vm_paddr_t lastaddr = 0; 833 struct mbuf *m; 834 835 *nsegs = 1; 836 error = 0; 837 for (m = m0; m != NULL && error == 0; m = m->m_next) { 838 if (m->m_len == 0) 839 continue; 840 841 error = _bus_dmamap_load_buffer(dmat, map, 842 m->m_data, m->m_len, 843 segs, maxsegs, 844 NULL, flags, &lastaddr, 845 nsegs, first); 846 if (error == ENOMEM && !first) { 847 /* 848 * Out of bounce pages due to too many 849 * fragments in the mbuf chain; return 850 * EFBIG instead. 851 */ 852 error = EFBIG; 853 } 854 first = 0; 855 } 856 #ifdef INVARIANTS 857 if (!error) 858 KKASSERT(*nsegs <= maxsegs && *nsegs >= 1); 859 #endif 860 } else { 861 *nsegs = 0; 862 error = EINVAL; 863 } 864 KKASSERT(error != EINPROGRESS); 865 return error; 866 } 867 868 /* 869 * Like _bus_dmamap_load(), but for uios. 870 */ 871 int 872 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 873 struct uio *uio, 874 bus_dmamap_callback2_t *callback, void *callback_arg, 875 int flags) 876 { 877 vm_paddr_t lastaddr; 878 int nsegs, error, first, i; 879 bus_size_t resid; 880 struct iovec *iov; 881 pmap_t pmap; 882 883 /* 884 * XXX 885 * Follow old semantics. Once all of the callers are fixed, 886 * we should get rid of these internal flag "adjustment". 887 */ 888 flags &= ~BUS_DMA_WAITOK; 889 flags |= BUS_DMA_NOWAIT; 890 891 resid = (bus_size_t)uio->uio_resid; 892 iov = uio->uio_iov; 893 894 if (uio->uio_segflg == UIO_USERSPACE) { 895 struct thread *td; 896 897 td = uio->uio_td; 898 KASSERT(td != NULL && td->td_proc != NULL, 899 ("bus_dmamap_load_uio: USERSPACE but no proc")); 900 pmap = vmspace_pmap(td->td_proc->p_vmspace); 901 } else { 902 pmap = NULL; 903 } 904 905 error = 0; 906 nsegs = 1; 907 first = 1; 908 lastaddr = 0; 909 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 910 /* 911 * Now at the first iovec to load. Load each iovec 912 * until we have exhausted the residual count. 913 */ 914 bus_size_t minlen = 915 resid < iov[i].iov_len ? resid : iov[i].iov_len; 916 caddr_t addr = (caddr_t) iov[i].iov_base; 917 918 error = _bus_dmamap_load_buffer(dmat, map, addr, minlen, 919 dmat->segments, dmat->nsegments, 920 pmap, flags, &lastaddr, &nsegs, first); 921 first = 0; 922 923 resid -= minlen; 924 } 925 926 if (error) { 927 /* force "no valid mappings" in callback */ 928 callback(callback_arg, dmat->segments, 0, 0, error); 929 } else { 930 callback(callback_arg, dmat->segments, nsegs, 931 (bus_size_t)uio->uio_resid, error); 932 } 933 return error; 934 } 935 936 /* 937 * Release the mapping held by map. 938 */ 939 void 940 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 941 { 942 struct bounce_page *bpage; 943 944 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 945 STAILQ_REMOVE_HEAD(&map->bpages, links); 946 free_bounce_page(dmat, bpage); 947 } 948 } 949 950 void 951 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 952 { 953 struct bounce_page *bpage; 954 955 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 956 /* 957 * Handle data bouncing. We might also 958 * want to add support for invalidating 959 * the caches on broken hardware 960 */ 961 switch (op) { 962 case BUS_DMASYNC_PREWRITE: 963 while (bpage != NULL) { 964 bcopy((void *)bpage->datavaddr, 965 (void *)bpage->vaddr, 966 bpage->datacount); 967 bpage = STAILQ_NEXT(bpage, links); 968 } 969 dmat->bounce_zone->total_bounced++; 970 break; 971 972 case BUS_DMASYNC_POSTREAD: 973 while (bpage != NULL) { 974 bcopy((void *)bpage->vaddr, 975 (void *)bpage->datavaddr, 976 bpage->datacount); 977 bpage = STAILQ_NEXT(bpage, links); 978 } 979 dmat->bounce_zone->total_bounced++; 980 break; 981 982 case BUS_DMASYNC_PREREAD: 983 case BUS_DMASYNC_POSTWRITE: 984 /* No-ops */ 985 break; 986 } 987 } 988 } 989 990 static int 991 alloc_bounce_zone(bus_dma_tag_t dmat) 992 { 993 struct bounce_zone *bz, *new_bz; 994 995 KASSERT(dmat->bounce_zone == NULL, 996 ("bounce zone was already assigned")); 997 998 new_bz = kmalloc(sizeof(*new_bz), M_DEVBUF, M_INTWAIT | M_ZERO); 999 1000 lwkt_gettoken(&bounce_zone_tok); 1001 1002 /* Check to see if we already have a suitable zone */ 1003 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1004 if (dmat->alignment <= bz->alignment && 1005 dmat->lowaddr >= bz->lowaddr) { 1006 lwkt_reltoken(&bounce_zone_tok); 1007 1008 dmat->bounce_zone = bz; 1009 kfree(new_bz, M_DEVBUF); 1010 return 0; 1011 } 1012 } 1013 bz = new_bz; 1014 1015 #ifdef SMP 1016 spin_init(&bz->spin); 1017 #endif 1018 STAILQ_INIT(&bz->bounce_page_list); 1019 STAILQ_INIT(&bz->bounce_map_waitinglist); 1020 bz->free_bpages = 0; 1021 bz->reserved_bpages = 0; 1022 bz->active_bpages = 0; 1023 bz->lowaddr = dmat->lowaddr; 1024 bz->alignment = round_page(dmat->alignment); 1025 ksnprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1026 busdma_zonecount++; 1027 ksnprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1028 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1029 1030 lwkt_reltoken(&bounce_zone_tok); 1031 1032 dmat->bounce_zone = bz; 1033 1034 sysctl_ctx_init(&bz->sysctl_ctx); 1035 bz->sysctl_tree = SYSCTL_ADD_NODE(&bz->sysctl_ctx, 1036 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1037 CTLFLAG_RD, 0, ""); 1038 if (bz->sysctl_tree == NULL) { 1039 sysctl_ctx_free(&bz->sysctl_ctx); 1040 return 0; /* XXX error code? */ 1041 } 1042 1043 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1044 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1045 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1046 "Total bounce pages"); 1047 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1048 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1049 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1050 "Free bounce pages"); 1051 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1052 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1053 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1054 "Reserved bounce pages"); 1055 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1056 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1057 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1058 "Active bounce pages"); 1059 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1060 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1061 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1062 "Total bounce requests"); 1063 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1064 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1065 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1066 "Total bounce requests that were deferred"); 1067 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1068 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1069 "reserve_failed", CTLFLAG_RD, &bz->reserve_failed, 0, 1070 "Total bounce page reservations that were failed"); 1071 SYSCTL_ADD_STRING(&bz->sysctl_ctx, 1072 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1073 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1074 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1075 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1076 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1077 1078 return 0; 1079 } 1080 1081 static int 1082 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages, int flags) 1083 { 1084 struct bounce_zone *bz = dmat->bounce_zone; 1085 int count = 0, mflags; 1086 1087 if (flags & BUS_DMA_NOWAIT) 1088 mflags = M_NOWAIT; 1089 else 1090 mflags = M_WAITOK; 1091 1092 while (numpages > 0) { 1093 struct bounce_page *bpage; 1094 1095 bpage = kmalloc(sizeof(*bpage), M_DEVBUF, M_INTWAIT | M_ZERO); 1096 1097 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1098 mflags, 0ul, 1099 bz->lowaddr, 1100 bz->alignment, 0); 1101 if (bpage->vaddr == 0) { 1102 kfree(bpage, M_DEVBUF); 1103 break; 1104 } 1105 bpage->busaddr = pmap_kextract(bpage->vaddr); 1106 1107 BZ_LOCK(bz); 1108 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1109 total_bounce_pages++; 1110 bz->total_bpages++; 1111 bz->free_bpages++; 1112 BZ_UNLOCK(bz); 1113 1114 count++; 1115 numpages--; 1116 } 1117 return count; 1118 } 1119 1120 /* Assume caller holds bounce zone spinlock */ 1121 static int 1122 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1123 { 1124 struct bounce_zone *bz = dmat->bounce_zone; 1125 int pages; 1126 1127 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1128 if (!commit && map->pagesneeded > (map->pagesreserved + pages)) { 1129 bz->reserve_failed++; 1130 return (map->pagesneeded - (map->pagesreserved + pages)); 1131 } 1132 1133 bz->free_bpages -= pages; 1134 1135 bz->reserved_bpages += pages; 1136 KKASSERT(bz->reserved_bpages <= bz->total_bpages); 1137 1138 map->pagesreserved += pages; 1139 pages = map->pagesneeded - map->pagesreserved; 1140 1141 return pages; 1142 } 1143 1144 static void 1145 return_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) 1146 { 1147 struct bounce_zone *bz = dmat->bounce_zone; 1148 int reserved = map->pagesreserved; 1149 bus_dmamap_t wait_map; 1150 1151 map->pagesreserved = 0; 1152 map->pagesneeded = 0; 1153 1154 if (reserved == 0) 1155 return; 1156 1157 BZ_LOCK(bz); 1158 1159 bz->free_bpages += reserved; 1160 KKASSERT(bz->free_bpages <= bz->total_bpages); 1161 1162 KKASSERT(bz->reserved_bpages >= reserved); 1163 bz->reserved_bpages -= reserved; 1164 1165 wait_map = get_map_waiting(dmat); 1166 1167 BZ_UNLOCK(bz); 1168 1169 if (wait_map != NULL) 1170 add_map_callback(map); 1171 } 1172 1173 static bus_addr_t 1174 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1175 bus_size_t size) 1176 { 1177 struct bounce_zone *bz = dmat->bounce_zone; 1178 struct bounce_page *bpage; 1179 1180 KASSERT(map->pagesneeded > 0, ("map doesn't need any pages")); 1181 map->pagesneeded--; 1182 1183 KASSERT(map->pagesreserved > 0, ("map doesn't reserve any pages")); 1184 map->pagesreserved--; 1185 1186 BZ_LOCK(bz); 1187 1188 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1189 KASSERT(bpage != NULL, ("free page list is empty")); 1190 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1191 1192 KKASSERT(bz->reserved_bpages > 0); 1193 bz->reserved_bpages--; 1194 1195 bz->active_bpages++; 1196 KKASSERT(bz->active_bpages <= bz->total_bpages); 1197 1198 BZ_UNLOCK(bz); 1199 1200 bpage->datavaddr = vaddr; 1201 bpage->datacount = size; 1202 STAILQ_INSERT_TAIL(&map->bpages, bpage, links); 1203 return bpage->busaddr; 1204 } 1205 1206 static void 1207 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1208 { 1209 struct bounce_zone *bz = dmat->bounce_zone; 1210 bus_dmamap_t map; 1211 1212 bpage->datavaddr = 0; 1213 bpage->datacount = 0; 1214 1215 BZ_LOCK(bz); 1216 1217 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1218 1219 bz->free_bpages++; 1220 KKASSERT(bz->free_bpages <= bz->total_bpages); 1221 1222 KKASSERT(bz->active_bpages > 0); 1223 bz->active_bpages--; 1224 1225 map = get_map_waiting(dmat); 1226 1227 BZ_UNLOCK(bz); 1228 1229 if (map != NULL) 1230 add_map_callback(map); 1231 } 1232 1233 /* Assume caller holds bounce zone spinlock */ 1234 static bus_dmamap_t 1235 get_map_waiting(bus_dma_tag_t dmat) 1236 { 1237 struct bounce_zone *bz = dmat->bounce_zone; 1238 bus_dmamap_t map; 1239 1240 map = STAILQ_FIRST(&bz->bounce_map_waitinglist); 1241 if (map != NULL) { 1242 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1243 STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links); 1244 bz->total_deferred++; 1245 } else { 1246 map = NULL; 1247 } 1248 } 1249 return map; 1250 } 1251 1252 static void 1253 add_map_callback(bus_dmamap_t map) 1254 { 1255 #ifdef notyet 1256 /* XXX callbacklist is not MPSAFE */ 1257 crit_enter(); 1258 get_mplock(); 1259 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links); 1260 busdma_swi_pending = 1; 1261 setsoftvm(); 1262 rel_mplock(); 1263 crit_exit(); 1264 #else 1265 panic("%s uncoded", __func__); 1266 #endif 1267 } 1268 1269 #ifdef notyet 1270 void 1271 busdma_swi(void) 1272 { 1273 bus_dmamap_t map; 1274 1275 crit_enter(); 1276 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1277 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1278 crit_exit(); 1279 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 1280 map->callback, map->callback_arg, /*flags*/0); 1281 crit_enter(); 1282 } 1283 crit_exit(); 1284 } 1285 #endif 1286