1 /* 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions, and the following disclaimer, 10 * without modification, immediately at the beginning of the file. 11 * 2. The name of the author may not be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 18 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.94 2008/08/15 20:51:31 kmacy Exp $ 27 */ 28 29 #include <sys/param.h> 30 #include <sys/systm.h> 31 #include <sys/malloc.h> 32 #include <sys/mbuf.h> 33 #include <sys/uio.h> 34 #include <sys/bus_dma.h> 35 #include <sys/kernel.h> 36 #include <sys/sysctl.h> 37 #include <sys/lock.h> 38 39 #include <sys/spinlock2.h> 40 41 #include <vm/vm.h> 42 #include <vm/vm_page.h> 43 44 /* XXX needed for to access pmap to convert per-proc virtual to physical */ 45 #include <sys/proc.h> 46 #include <vm/vm_map.h> 47 48 #include <machine/md_var.h> 49 #include <machine/pmap.h> 50 51 #include <bus/cam/cam.h> 52 #include <bus/cam/cam_ccb.h> 53 54 #define MAX_BPAGES 1024 55 56 /* 57 * 16 x N declared on stack. 58 */ 59 #define BUS_DMA_CACHE_SEGMENTS 8 60 61 struct bounce_zone; 62 struct bus_dmamap; 63 64 struct bus_dma_tag { 65 bus_size_t alignment; 66 bus_size_t boundary; 67 bus_addr_t lowaddr; 68 bus_addr_t highaddr; 69 bus_size_t maxsize; 70 u_int nsegments; 71 bus_size_t maxsegsz; 72 int flags; 73 int map_count; 74 bus_dma_segment_t *segments; 75 struct bounce_zone *bounce_zone; 76 struct spinlock spin; 77 }; 78 79 /* 80 * bus_dma_tag private flags 81 */ 82 #define BUS_DMA_BOUNCE_ALIGN BUS_DMA_BUS2 83 #define BUS_DMA_BOUNCE_LOWADDR BUS_DMA_BUS3 84 #define BUS_DMA_MIN_ALLOC_COMP BUS_DMA_BUS4 85 86 #define BUS_DMA_COULD_BOUNCE (BUS_DMA_BOUNCE_LOWADDR | BUS_DMA_BOUNCE_ALIGN) 87 88 #define BUS_DMAMEM_KMALLOC(dmat) \ 89 ((dmat)->maxsize <= PAGE_SIZE && \ 90 (dmat)->alignment <= PAGE_SIZE && \ 91 (dmat)->lowaddr >= ptoa(Maxmem)) 92 93 struct bounce_page { 94 vm_offset_t vaddr; /* kva of bounce buffer */ 95 bus_addr_t busaddr; /* Physical address */ 96 vm_offset_t datavaddr; /* kva of client data */ 97 bus_size_t datacount; /* client data count */ 98 STAILQ_ENTRY(bounce_page) links; 99 }; 100 101 struct bounce_zone { 102 STAILQ_ENTRY(bounce_zone) links; 103 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 104 STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 105 struct spinlock spin; 106 int total_bpages; 107 int free_bpages; 108 int reserved_bpages; 109 int active_bpages; 110 int total_bounced; 111 int total_deferred; 112 int reserve_failed; 113 bus_size_t alignment; 114 bus_addr_t lowaddr; 115 char zoneid[8]; 116 char lowaddrid[20]; 117 struct sysctl_ctx_list sysctl_ctx; 118 struct sysctl_oid *sysctl_tree; 119 }; 120 121 #define BZ_LOCK(bz) spin_lock(&(bz)->spin) 122 #define BZ_UNLOCK(bz) spin_unlock(&(bz)->spin) 123 124 static struct lwkt_token bounce_zone_tok = 125 LWKT_TOKEN_INITIALIZER(bounce_zone_token); 126 static int busdma_zonecount; 127 static STAILQ_HEAD(, bounce_zone) bounce_zone_list = 128 STAILQ_HEAD_INITIALIZER(bounce_zone_list); 129 130 static int busdma_priv_zonecount = -1; 131 132 int busdma_swi_pending; 133 static int total_bounce_pages; 134 static int max_bounce_pages = MAX_BPAGES; 135 static int bounce_alignment = 1; /* XXX temporary */ 136 137 TUNABLE_INT("hw.busdma.max_bpages", &max_bounce_pages); 138 TUNABLE_INT("hw.busdma.bounce_alignment", &bounce_alignment); 139 140 struct bus_dmamap { 141 struct bp_list bpages; 142 int pagesneeded; 143 int pagesreserved; 144 bus_dma_tag_t dmat; 145 void *buf; /* unmapped buffer pointer */ 146 bus_size_t buflen; /* unmapped buffer length */ 147 bus_dmamap_callback_t *callback; 148 void *callback_arg; 149 STAILQ_ENTRY(bus_dmamap) links; 150 }; 151 152 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist = 153 STAILQ_HEAD_INITIALIZER(bounce_map_callbacklist); 154 static struct spinlock bounce_map_list_spin = 155 SPINLOCK_INITIALIZER(&bounce_map_list_spin, "bounce_map_list_spin"); 156 157 static struct bus_dmamap nobounce_dmamap; 158 159 static int alloc_bounce_zone(bus_dma_tag_t); 160 static int alloc_bounce_pages(bus_dma_tag_t, u_int, int); 161 static void free_bounce_pages_all(bus_dma_tag_t); 162 static void free_bounce_zone(bus_dma_tag_t); 163 static int reserve_bounce_pages(bus_dma_tag_t, bus_dmamap_t, int); 164 static void return_bounce_pages(bus_dma_tag_t, bus_dmamap_t); 165 static bus_addr_t add_bounce_page(bus_dma_tag_t, bus_dmamap_t, 166 vm_offset_t, bus_size_t *); 167 static void free_bounce_page(bus_dma_tag_t, struct bounce_page *); 168 169 static bus_dmamap_t get_map_waiting(bus_dma_tag_t); 170 static void add_map_callback(bus_dmamap_t); 171 172 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD, 0, "Busdma parameters"); 173 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bounce_pages, 174 0, "Total bounce pages"); 175 SYSCTL_INT(_hw_busdma, OID_AUTO, max_bpages, CTLFLAG_RD, &max_bounce_pages, 176 0, "Max bounce pages per bounce zone"); 177 SYSCTL_INT(_hw_busdma, OID_AUTO, bounce_alignment, CTLFLAG_RD, 178 &bounce_alignment, 0, "Obey alignment constraint"); 179 180 181 /* 182 * Returns true if the address falls within the tag's exclusion window, or 183 * fails to meet its alignment requirements. 184 */ 185 static __inline int 186 addr_needs_bounce(bus_dma_tag_t dmat, bus_addr_t paddr) 187 { 188 if ((paddr > dmat->lowaddr && paddr <= dmat->highaddr) || 189 (bounce_alignment && (paddr & (dmat->alignment - 1)) != 0)) 190 return (1); 191 192 return (0); 193 } 194 195 static __inline 196 bus_dma_segment_t * 197 bus_dma_tag_lock(bus_dma_tag_t tag, bus_dma_segment_t *cache) 198 { 199 if (tag->flags & BUS_DMA_PROTECTED) 200 return(tag->segments); 201 202 if (tag->nsegments <= BUS_DMA_CACHE_SEGMENTS) 203 return(cache); 204 spin_lock(&tag->spin); 205 return(tag->segments); 206 } 207 208 static __inline 209 void 210 bus_dma_tag_unlock(bus_dma_tag_t tag) 211 { 212 if (tag->flags & BUS_DMA_PROTECTED) 213 return; 214 215 if (tag->nsegments > BUS_DMA_CACHE_SEGMENTS) 216 spin_unlock(&tag->spin); 217 } 218 219 /* 220 * Allocate a device specific dma_tag. 221 */ 222 int 223 bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 224 bus_size_t boundary, bus_addr_t lowaddr, 225 bus_addr_t highaddr, bus_size_t maxsize, int nsegments, 226 bus_size_t maxsegsz, int flags, bus_dma_tag_t *dmat) 227 { 228 bus_dma_tag_t newtag; 229 int error = 0; 230 231 /* 232 * Sanity checks 233 */ 234 235 if (alignment == 0) 236 alignment = 1; 237 if (alignment & (alignment - 1)) 238 panic("alignment must be power of 2"); 239 240 if (boundary != 0) { 241 if (boundary & (boundary - 1)) 242 panic("boundary must be power of 2"); 243 if (boundary < maxsegsz) { 244 kprintf("boundary < maxsegsz:\n"); 245 print_backtrace(-1); 246 maxsegsz = boundary; 247 } 248 } 249 250 /* Return a NULL tag on failure */ 251 *dmat = NULL; 252 253 newtag = kmalloc(sizeof(*newtag), M_DEVBUF, M_INTWAIT | M_ZERO); 254 255 spin_init(&newtag->spin, "busdmacreate"); 256 newtag->alignment = alignment; 257 newtag->boundary = boundary; 258 newtag->lowaddr = trunc_page((vm_paddr_t)lowaddr) + (PAGE_SIZE - 1); 259 newtag->highaddr = trunc_page((vm_paddr_t)highaddr) + (PAGE_SIZE - 1); 260 newtag->maxsize = maxsize; 261 newtag->nsegments = nsegments; 262 newtag->maxsegsz = maxsegsz; 263 newtag->flags = flags; 264 newtag->map_count = 0; 265 newtag->segments = NULL; 266 newtag->bounce_zone = NULL; 267 268 /* Take into account any restrictions imposed by our parent tag */ 269 if (parent != NULL) { 270 newtag->lowaddr = MIN(parent->lowaddr, newtag->lowaddr); 271 newtag->highaddr = MAX(parent->highaddr, newtag->highaddr); 272 273 if (newtag->boundary == 0) { 274 newtag->boundary = parent->boundary; 275 } else if (parent->boundary != 0) { 276 newtag->boundary = MIN(parent->boundary, 277 newtag->boundary); 278 } 279 280 #ifdef notyet 281 newtag->alignment = MAX(parent->alignment, newtag->alignment); 282 #endif 283 284 } 285 286 if (newtag->lowaddr < ptoa(Maxmem)) 287 newtag->flags |= BUS_DMA_BOUNCE_LOWADDR; 288 if (bounce_alignment && newtag->alignment > 1 && 289 !(newtag->flags & BUS_DMA_ALIGNED)) 290 newtag->flags |= BUS_DMA_BOUNCE_ALIGN; 291 292 if ((newtag->flags & BUS_DMA_COULD_BOUNCE) && 293 (flags & BUS_DMA_ALLOCNOW) != 0) { 294 struct bounce_zone *bz; 295 296 /* Must bounce */ 297 298 error = alloc_bounce_zone(newtag); 299 if (error) 300 goto back; 301 bz = newtag->bounce_zone; 302 303 if ((newtag->flags & BUS_DMA_ALLOCALL) == 0 && 304 ptoa(bz->total_bpages) < maxsize) { 305 int pages; 306 307 if (flags & BUS_DMA_ONEBPAGE) { 308 pages = 1; 309 } else { 310 pages = atop(round_page(maxsize)) - 311 bz->total_bpages; 312 pages = MAX(pages, 1); 313 } 314 315 /* Add pages to our bounce pool */ 316 if (alloc_bounce_pages(newtag, pages, flags) < pages) 317 error = ENOMEM; 318 319 /* Performed initial allocation */ 320 newtag->flags |= BUS_DMA_MIN_ALLOC_COMP; 321 } 322 } 323 back: 324 if (error) { 325 free_bounce_zone(newtag); 326 kfree(newtag, M_DEVBUF); 327 } else { 328 *dmat = newtag; 329 } 330 return error; 331 } 332 333 int 334 bus_dma_tag_destroy(bus_dma_tag_t dmat) 335 { 336 if (dmat != NULL) { 337 if (dmat->map_count != 0) 338 return (EBUSY); 339 340 free_bounce_zone(dmat); 341 if (dmat->segments != NULL) 342 kfree(dmat->segments, M_DEVBUF); 343 kfree(dmat, M_DEVBUF); 344 } 345 return (0); 346 } 347 348 bus_size_t 349 bus_dma_tag_getmaxsize(bus_dma_tag_t tag) 350 { 351 return(tag->maxsize); 352 } 353 354 /* 355 * Allocate a handle for mapping from kva/uva/physical 356 * address space into bus device space. 357 */ 358 int 359 bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 360 { 361 int error; 362 363 error = 0; 364 365 if (dmat->segments == NULL) { 366 KKASSERT(dmat->nsegments && dmat->nsegments < 16384); 367 dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * 368 dmat->nsegments, M_DEVBUF, M_INTWAIT); 369 } 370 371 if (dmat->flags & BUS_DMA_COULD_BOUNCE) { 372 struct bounce_zone *bz; 373 int maxpages; 374 375 /* Must bounce */ 376 377 if (dmat->bounce_zone == NULL) { 378 error = alloc_bounce_zone(dmat); 379 if (error) 380 return error; 381 } 382 bz = dmat->bounce_zone; 383 384 *mapp = kmalloc(sizeof(**mapp), M_DEVBUF, M_INTWAIT | M_ZERO); 385 386 /* Initialize the new map */ 387 STAILQ_INIT(&((*mapp)->bpages)); 388 389 /* 390 * Attempt to add pages to our pool on a per-instance 391 * basis up to a sane limit. 392 */ 393 if (dmat->flags & BUS_DMA_ALLOCALL) { 394 maxpages = Maxmem - atop(dmat->lowaddr); 395 } else if (dmat->flags & BUS_DMA_BOUNCE_ALIGN) { 396 maxpages = max_bounce_pages; 397 } else { 398 maxpages = MIN(max_bounce_pages, 399 Maxmem - atop(dmat->lowaddr)); 400 } 401 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0 || 402 (dmat->map_count > 0 && bz->total_bpages < maxpages)) { 403 int pages; 404 405 if (flags & BUS_DMA_ONEBPAGE) { 406 pages = 1; 407 } else { 408 pages = atop(round_page(dmat->maxsize)); 409 pages = MIN(maxpages - bz->total_bpages, pages); 410 pages = MAX(pages, 1); 411 } 412 if (alloc_bounce_pages(dmat, pages, flags) < pages) 413 error = ENOMEM; 414 415 if ((dmat->flags & BUS_DMA_MIN_ALLOC_COMP) == 0) { 416 if (!error && 417 (dmat->flags & BUS_DMA_ALLOCALL) == 0) 418 dmat->flags |= BUS_DMA_MIN_ALLOC_COMP; 419 } else { 420 error = 0; 421 } 422 } 423 } else { 424 *mapp = NULL; 425 } 426 if (!error) { 427 dmat->map_count++; 428 } else { 429 kfree(*mapp, M_DEVBUF); 430 *mapp = NULL; 431 } 432 return error; 433 } 434 435 /* 436 * Destroy a handle for mapping from kva/uva/physical 437 * address space into bus device space. 438 */ 439 int 440 bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 441 { 442 if (map != NULL && map != (void *)-1) { 443 if (STAILQ_FIRST(&map->bpages) != NULL) 444 return (EBUSY); 445 kfree(map, M_DEVBUF); 446 } 447 dmat->map_count--; 448 return (0); 449 } 450 451 static __inline bus_size_t 452 check_kmalloc(bus_dma_tag_t dmat, const void *vaddr0, int verify) 453 { 454 bus_size_t maxsize = 0; 455 uintptr_t vaddr = (uintptr_t)vaddr0; 456 457 if ((vaddr ^ (vaddr + dmat->maxsize - 1)) & ~PAGE_MASK) { 458 if (verify) 459 panic("boundary check failed\n"); 460 maxsize = dmat->maxsize; 461 } 462 if (vaddr & (dmat->alignment - 1)) { 463 if (verify) 464 panic("alignment check failed\n"); 465 if (dmat->maxsize < dmat->alignment) 466 maxsize = dmat->alignment; 467 else 468 maxsize = dmat->maxsize; 469 } 470 return maxsize; 471 } 472 473 /* 474 * Allocate a piece of memory that can be efficiently mapped into 475 * bus device space based on the constraints lited in the dma tag. 476 * 477 * Use *mapp to record whether we were able to use kmalloc() 478 * or whether we had to use contigmalloc(). 479 */ 480 int 481 bus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags, 482 bus_dmamap_t *mapp) 483 { 484 vm_memattr_t attr; 485 int mflags; 486 487 /* If we succeed, no mapping/bouncing will be required */ 488 *mapp = NULL; 489 490 if (dmat->segments == NULL) { 491 KKASSERT(dmat->nsegments < 16384); 492 dmat->segments = kmalloc(sizeof(bus_dma_segment_t) * 493 dmat->nsegments, M_DEVBUF, M_INTWAIT); 494 } 495 496 if (flags & BUS_DMA_NOWAIT) 497 mflags = M_NOWAIT; 498 else 499 mflags = M_WAITOK; 500 if (flags & BUS_DMA_ZERO) 501 mflags |= M_ZERO; 502 if (flags & BUS_DMA_NOCACHE) 503 attr = VM_MEMATTR_UNCACHEABLE; 504 else 505 attr = VM_MEMATTR_DEFAULT; 506 507 /* XXX must alloc with correct mem attribute here */ 508 if (BUS_DMAMEM_KMALLOC(dmat) && attr == VM_MEMATTR_DEFAULT) { 509 bus_size_t maxsize; 510 511 *vaddr = kmalloc(dmat->maxsize, M_DEVBUF, mflags); 512 513 /* 514 * XXX 515 * Check whether the allocation 516 * - crossed a page boundary 517 * - was not aligned 518 * Retry with power-of-2 alignment in the above cases. 519 */ 520 maxsize = check_kmalloc(dmat, *vaddr, 0); 521 if (maxsize) { 522 kfree(*vaddr, M_DEVBUF); 523 *vaddr = kmalloc(maxsize, M_DEVBUF, 524 mflags | M_POWEROF2); 525 check_kmalloc(dmat, *vaddr, 1); 526 } 527 } else { 528 /* 529 * XXX Use Contigmalloc until it is merged into this facility 530 * and handles multi-seg allocations. Nobody is doing 531 * multi-seg allocations yet though. 532 */ 533 *vaddr = contigmalloc(dmat->maxsize, M_DEVBUF, mflags, 534 0ul, dmat->lowaddr, 535 dmat->alignment, dmat->boundary); 536 *mapp = (void *)-1; 537 } 538 if (*vaddr == NULL) 539 return (ENOMEM); 540 541 if (attr != VM_MEMATTR_DEFAULT) { 542 pmap_change_attr((vm_offset_t)(*vaddr), 543 dmat->maxsize / PAGE_SIZE, attr); 544 } 545 return (0); 546 } 547 548 /* 549 * Free a piece of memory and it's allociated dmamap, that was allocated 550 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 551 */ 552 void 553 bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 554 { 555 /* 556 * dmamem does not need to be bounced, so the map should be 557 * NULL 558 */ 559 if (map != NULL && map != (void *)-1) 560 panic("bus_dmamem_free: Invalid map freed"); 561 if (map == NULL) 562 kfree(vaddr, M_DEVBUF); 563 else 564 contigfree(vaddr, dmat->maxsize, M_DEVBUF); 565 } 566 567 static __inline vm_paddr_t 568 _bus_dma_extract(pmap_t pmap, vm_offset_t vaddr) 569 { 570 if (pmap) 571 return pmap_extract(pmap, vaddr, NULL); 572 else 573 return pmap_kextract(vaddr); 574 } 575 576 /* 577 * Utility function to load a linear buffer. lastaddrp holds state 578 * between invocations (for multiple-buffer loads). segp contains 579 * the segment following the starting one on entrace, and the ending 580 * segment on exit. first indicates if this is the first invocation 581 * of this function. 582 */ 583 static int 584 _bus_dmamap_load_buffer(bus_dma_tag_t dmat, 585 bus_dmamap_t map, 586 void *buf, bus_size_t buflen, 587 bus_dma_segment_t *segments, 588 int nsegments, 589 pmap_t pmap, 590 int flags, 591 vm_paddr_t *lastpaddrp, 592 int *segp, 593 int first) 594 { 595 vm_offset_t vaddr; 596 vm_paddr_t paddr, nextpaddr; 597 bus_dma_segment_t *sg; 598 bus_addr_t bmask; 599 int seg, error = 0; 600 601 if (map == NULL || map == (void *)-1) 602 map = &nobounce_dmamap; 603 604 #ifdef INVARIANTS 605 if (dmat->flags & BUS_DMA_ALIGNED) 606 KKASSERT(((uintptr_t)buf & (dmat->alignment - 1)) == 0); 607 #endif 608 609 /* 610 * If we are being called during a callback, pagesneeded will 611 * be non-zero, so we can avoid doing the work twice. 612 */ 613 if ((dmat->flags & BUS_DMA_COULD_BOUNCE) && 614 map != &nobounce_dmamap && map->pagesneeded == 0) { 615 vm_offset_t vendaddr; 616 617 /* 618 * Count the number of bounce pages 619 * needed in order to complete this transfer 620 */ 621 vaddr = (vm_offset_t)buf; 622 vendaddr = (vm_offset_t)buf + buflen; 623 624 while (vaddr < vendaddr) { 625 paddr = _bus_dma_extract(pmap, vaddr); 626 if (addr_needs_bounce(dmat, paddr)) 627 map->pagesneeded++; 628 vaddr += (PAGE_SIZE - (vaddr & PAGE_MASK)); 629 } 630 } 631 632 /* Reserve Necessary Bounce Pages */ 633 if (map->pagesneeded != 0) { 634 struct bounce_zone *bz; 635 636 bz = dmat->bounce_zone; 637 BZ_LOCK(bz); 638 if (flags & BUS_DMA_NOWAIT) { 639 if (reserve_bounce_pages(dmat, map, 0) != 0) { 640 BZ_UNLOCK(bz); 641 error = ENOMEM; 642 goto free_bounce; 643 } 644 } else { 645 if (reserve_bounce_pages(dmat, map, 1) != 0) { 646 /* Queue us for resources */ 647 map->dmat = dmat; 648 map->buf = buf; 649 map->buflen = buflen; 650 651 STAILQ_INSERT_TAIL( 652 &dmat->bounce_zone->bounce_map_waitinglist, 653 map, links); 654 BZ_UNLOCK(bz); 655 656 return (EINPROGRESS); 657 } 658 } 659 BZ_UNLOCK(bz); 660 } 661 662 KKASSERT(*segp >= 1 && *segp <= nsegments); 663 seg = *segp; 664 sg = &segments[seg - 1]; 665 666 vaddr = (vm_offset_t)buf; 667 nextpaddr = *lastpaddrp; 668 bmask = ~(dmat->boundary - 1); /* note: will be 0 if boundary is 0 */ 669 670 /* force at least one segment */ 671 do { 672 bus_size_t size; 673 674 /* 675 * Per-page main loop 676 */ 677 paddr = _bus_dma_extract(pmap, vaddr); 678 size = PAGE_SIZE - (paddr & PAGE_MASK); 679 if (size > buflen) 680 size = buflen; 681 if (map->pagesneeded != 0 && addr_needs_bounce(dmat, paddr)) { 682 /* 683 * NOTE: paddr may have different in-page offset, 684 * unless BUS_DMA_KEEP_PG_OFFSET is set. 685 */ 686 paddr = add_bounce_page(dmat, map, vaddr, &size); 687 } 688 689 /* 690 * Fill in the bus_dma_segment 691 */ 692 if (first) { 693 sg->ds_addr = paddr; 694 sg->ds_len = size; 695 first = 0; 696 } else if (paddr == nextpaddr) { 697 sg->ds_len += size; 698 } else { 699 sg++; 700 seg++; 701 if (seg > nsegments) 702 break; 703 sg->ds_addr = paddr; 704 sg->ds_len = size; 705 } 706 nextpaddr = paddr + size; 707 708 /* 709 * Handle maxsegsz and boundary issues with a nested loop 710 */ 711 for (;;) { 712 bus_size_t tmpsize; 713 714 /* 715 * Limit to the boundary and maximum segment size 716 */ 717 if (((nextpaddr - 1) ^ sg->ds_addr) & bmask) { 718 tmpsize = dmat->boundary - 719 (sg->ds_addr & ~bmask); 720 if (tmpsize > dmat->maxsegsz) 721 tmpsize = dmat->maxsegsz; 722 KKASSERT(tmpsize < sg->ds_len); 723 } else if (sg->ds_len > dmat->maxsegsz) { 724 tmpsize = dmat->maxsegsz; 725 } else { 726 break; 727 } 728 729 /* 730 * Futz, split the data into a new segment. 731 */ 732 if (seg >= nsegments) 733 goto fail; 734 sg[1].ds_len = sg[0].ds_len - tmpsize; 735 sg[1].ds_addr = sg[0].ds_addr + tmpsize; 736 sg[0].ds_len = tmpsize; 737 sg++; 738 seg++; 739 } 740 741 /* 742 * Adjust for loop 743 */ 744 buflen -= size; 745 vaddr += size; 746 } while (buflen > 0); 747 fail: 748 if (buflen != 0) 749 error = EFBIG; 750 751 *segp = seg; 752 *lastpaddrp = nextpaddr; 753 754 free_bounce: 755 if (error && (dmat->flags & BUS_DMA_COULD_BOUNCE) && 756 map != &nobounce_dmamap) { 757 _bus_dmamap_unload(dmat, map); 758 return_bounce_pages(dmat, map); 759 } 760 return error; 761 } 762 763 /* 764 * Map the buffer buf into bus space using the dmamap map. 765 */ 766 int 767 bus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 768 bus_size_t buflen, bus_dmamap_callback_t *callback, 769 void *callback_arg, int flags) 770 { 771 bus_dma_segment_t cache_segments[BUS_DMA_CACHE_SEGMENTS]; 772 bus_dma_segment_t *segments; 773 vm_paddr_t lastaddr = 0; 774 int error, nsegs = 1; 775 776 if (map != NULL && map != (void *)-1) { 777 /* 778 * XXX 779 * Follow old semantics. Once all of the callers are fixed, 780 * we should get rid of these internal flag "adjustment". 781 */ 782 flags &= ~BUS_DMA_NOWAIT; 783 flags |= BUS_DMA_WAITOK; 784 785 map->callback = callback; 786 map->callback_arg = callback_arg; 787 } 788 789 segments = bus_dma_tag_lock(dmat, cache_segments); 790 error = _bus_dmamap_load_buffer(dmat, map, buf, buflen, 791 segments, dmat->nsegments, 792 NULL, flags, &lastaddr, &nsegs, 1); 793 if (error == EINPROGRESS) { 794 KKASSERT((dmat->flags & 795 (BUS_DMA_PRIVBZONE | BUS_DMA_ALLOCALL)) != 796 (BUS_DMA_PRIVBZONE | BUS_DMA_ALLOCALL)); 797 798 if (dmat->flags & BUS_DMA_PROTECTED) 799 panic("protected dmamap callback will be defered"); 800 801 bus_dma_tag_unlock(dmat); 802 return error; 803 } 804 callback(callback_arg, segments, nsegs, error); 805 bus_dma_tag_unlock(dmat); 806 return 0; 807 } 808 809 /* 810 * Like _bus_dmamap_load(), but for ccb. 811 */ 812 int 813 bus_dmamap_load_ccb(bus_dma_tag_t dmat, bus_dmamap_t map, union ccb *ccb, 814 bus_dmamap_callback_t *callback, void *callback_arg, int flags) 815 { 816 const struct ccb_scsiio *csio; 817 struct ccb_hdr *ccb_h; 818 819 ccb_h = &ccb->ccb_h; 820 KASSERT(ccb_h->func_code == XPT_SCSI_IO || 821 ccb_h->func_code == XPT_CONT_TARGET_IO, 822 ("invalid ccb func_code %u", ccb_h->func_code)); 823 if ((ccb_h->flags & CAM_DIR_MASK) == CAM_DIR_NONE) { 824 callback(callback_arg, NULL, 0, 0); 825 return 0; 826 } 827 csio = &ccb->csio; 828 829 return (bus_dmamap_load(dmat, map, csio->data_ptr, csio->dxfer_len, 830 callback, callback_arg, flags)); 831 } 832 833 /* 834 * Like _bus_dmamap_load(), but for mbufs. 835 */ 836 int 837 bus_dmamap_load_mbuf(bus_dma_tag_t dmat, bus_dmamap_t map, 838 struct mbuf *m0, 839 bus_dmamap_callback2_t *callback, void *callback_arg, 840 int flags) 841 { 842 bus_dma_segment_t cache_segments[BUS_DMA_CACHE_SEGMENTS]; 843 bus_dma_segment_t *segments; 844 int nsegs, error; 845 846 /* 847 * XXX 848 * Follow old semantics. Once all of the callers are fixed, 849 * we should get rid of these internal flag "adjustment". 850 */ 851 flags &= ~BUS_DMA_WAITOK; 852 flags |= BUS_DMA_NOWAIT; 853 854 segments = bus_dma_tag_lock(dmat, cache_segments); 855 error = bus_dmamap_load_mbuf_segment(dmat, map, m0, 856 segments, dmat->nsegments, &nsegs, flags); 857 if (error) { 858 /* force "no valid mappings" in callback */ 859 callback(callback_arg, segments, 0, 860 0, error); 861 } else { 862 callback(callback_arg, segments, nsegs, 863 m0->m_pkthdr.len, error); 864 } 865 bus_dma_tag_unlock(dmat); 866 return error; 867 } 868 869 int 870 bus_dmamap_load_mbuf_segment(bus_dma_tag_t dmat, bus_dmamap_t map, 871 struct mbuf *m0, 872 bus_dma_segment_t *segs, int maxsegs, 873 int *nsegs, int flags) 874 { 875 int error; 876 877 M_ASSERTPKTHDR(m0); 878 879 KASSERT(maxsegs >= 1, ("invalid maxsegs %d", maxsegs)); 880 KASSERT(maxsegs <= dmat->nsegments, 881 ("%d too many segments, dmat only supports %d segments", 882 maxsegs, dmat->nsegments)); 883 KASSERT(flags & BUS_DMA_NOWAIT, 884 ("only BUS_DMA_NOWAIT is supported")); 885 886 if (m0->m_pkthdr.len <= dmat->maxsize) { 887 int first = 1; 888 vm_paddr_t lastaddr = 0; 889 struct mbuf *m; 890 891 *nsegs = 1; 892 error = 0; 893 for (m = m0; m != NULL && error == 0; m = m->m_next) { 894 if (m->m_len == 0) 895 continue; 896 897 error = _bus_dmamap_load_buffer(dmat, map, 898 m->m_data, m->m_len, 899 segs, maxsegs, 900 NULL, flags, &lastaddr, 901 nsegs, first); 902 if (error == ENOMEM && !first) { 903 /* 904 * Out of bounce pages due to too many 905 * fragments in the mbuf chain; return 906 * EFBIG instead. 907 */ 908 error = EFBIG; 909 break; 910 } 911 first = 0; 912 } 913 #ifdef INVARIANTS 914 if (!error) 915 KKASSERT(*nsegs <= maxsegs && *nsegs >= 1); 916 #endif 917 } else { 918 *nsegs = 0; 919 error = EINVAL; 920 } 921 KKASSERT(error != EINPROGRESS); 922 return error; 923 } 924 925 /* 926 * Like _bus_dmamap_load(), but for uios. 927 */ 928 int 929 bus_dmamap_load_uio(bus_dma_tag_t dmat, bus_dmamap_t map, 930 struct uio *uio, 931 bus_dmamap_callback2_t *callback, void *callback_arg, 932 int flags) 933 { 934 vm_paddr_t lastaddr; 935 int nsegs, error, first, i; 936 bus_size_t resid; 937 struct iovec *iov; 938 pmap_t pmap; 939 bus_dma_segment_t cache_segments[BUS_DMA_CACHE_SEGMENTS]; 940 bus_dma_segment_t *segments; 941 bus_dma_segment_t *segs; 942 int nsegs_left; 943 944 if (dmat->nsegments <= BUS_DMA_CACHE_SEGMENTS) 945 segments = cache_segments; 946 else 947 segments = kmalloc(sizeof(bus_dma_segment_t) * dmat->nsegments, 948 M_DEVBUF, M_WAITOK | M_ZERO); 949 950 /* 951 * XXX 952 * Follow old semantics. Once all of the callers are fixed, 953 * we should get rid of these internal flag "adjustment". 954 */ 955 flags &= ~BUS_DMA_WAITOK; 956 flags |= BUS_DMA_NOWAIT; 957 958 resid = (bus_size_t)uio->uio_resid; 959 iov = uio->uio_iov; 960 961 segs = segments; 962 nsegs_left = dmat->nsegments; 963 964 if (uio->uio_segflg == UIO_USERSPACE) { 965 struct thread *td; 966 967 td = uio->uio_td; 968 KASSERT(td != NULL && td->td_proc != NULL, 969 ("bus_dmamap_load_uio: USERSPACE but no proc")); 970 pmap = vmspace_pmap(td->td_proc->p_vmspace); 971 } else { 972 pmap = NULL; 973 } 974 975 error = 0; 976 nsegs = 1; 977 first = 1; 978 lastaddr = 0; 979 for (i = 0; i < uio->uio_iovcnt && resid != 0 && !error; i++) { 980 /* 981 * Now at the first iovec to load. Load each iovec 982 * until we have exhausted the residual count. 983 */ 984 bus_size_t minlen = 985 resid < iov[i].iov_len ? resid : iov[i].iov_len; 986 caddr_t addr = (caddr_t) iov[i].iov_base; 987 988 error = _bus_dmamap_load_buffer(dmat, map, addr, minlen, 989 segs, nsegs_left, 990 pmap, flags, &lastaddr, &nsegs, first); 991 first = 0; 992 993 resid -= minlen; 994 if (error == 0) { 995 nsegs_left -= nsegs; 996 segs += nsegs; 997 } 998 } 999 1000 /* 1001 * Minimum one DMA segment, even if 0-length buffer. 1002 */ 1003 if (nsegs_left == dmat->nsegments) 1004 --nsegs_left; 1005 1006 if (error) { 1007 /* force "no valid mappings" in callback */ 1008 callback(callback_arg, segments, 0, 1009 0, error); 1010 } else { 1011 callback(callback_arg, segments, dmat->nsegments - nsegs_left, 1012 (bus_size_t)uio->uio_resid, error); 1013 } 1014 if (dmat->nsegments > BUS_DMA_CACHE_SEGMENTS) 1015 kfree(segments, M_DEVBUF); 1016 return error; 1017 } 1018 1019 /* 1020 * Release the mapping held by map. 1021 */ 1022 void 1023 _bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 1024 { 1025 struct bounce_page *bpage; 1026 1027 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1028 STAILQ_REMOVE_HEAD(&map->bpages, links); 1029 free_bounce_page(dmat, bpage); 1030 } 1031 } 1032 1033 void 1034 _bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op) 1035 { 1036 struct bounce_page *bpage; 1037 1038 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1039 /* 1040 * Handle data bouncing. We might also 1041 * want to add support for invalidating 1042 * the caches on broken hardware 1043 */ 1044 if (op & BUS_DMASYNC_PREWRITE) { 1045 while (bpage != NULL) { 1046 bcopy((void *)bpage->datavaddr, 1047 (void *)bpage->vaddr, 1048 bpage->datacount); 1049 bpage = STAILQ_NEXT(bpage, links); 1050 } 1051 cpu_sfence(); 1052 dmat->bounce_zone->total_bounced++; 1053 } 1054 if (op & BUS_DMASYNC_POSTREAD) { 1055 cpu_lfence(); 1056 while (bpage != NULL) { 1057 bcopy((void *)bpage->vaddr, 1058 (void *)bpage->datavaddr, 1059 bpage->datacount); 1060 bpage = STAILQ_NEXT(bpage, links); 1061 } 1062 dmat->bounce_zone->total_bounced++; 1063 } 1064 /* BUS_DMASYNC_PREREAD - no operation on intel */ 1065 /* BUS_DMASYNC_POSTWRITE - no operation on intel */ 1066 } 1067 } 1068 1069 static int 1070 alloc_bounce_zone(bus_dma_tag_t dmat) 1071 { 1072 struct bounce_zone *bz, *new_bz; 1073 1074 KASSERT(dmat->bounce_zone == NULL, 1075 ("bounce zone was already assigned")); 1076 1077 new_bz = kmalloc(sizeof(*new_bz), M_DEVBUF, M_INTWAIT | M_ZERO); 1078 1079 lwkt_gettoken(&bounce_zone_tok); 1080 1081 if ((dmat->flags & BUS_DMA_PRIVBZONE) == 0) { 1082 /* 1083 * For shared bounce zone, check to see 1084 * if we already have a suitable zone 1085 */ 1086 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1087 if (dmat->alignment <= bz->alignment && 1088 dmat->lowaddr >= bz->lowaddr) { 1089 lwkt_reltoken(&bounce_zone_tok); 1090 1091 dmat->bounce_zone = bz; 1092 kfree(new_bz, M_DEVBUF); 1093 return 0; 1094 } 1095 } 1096 } 1097 bz = new_bz; 1098 1099 spin_init(&bz->spin, "allocbouncezone"); 1100 STAILQ_INIT(&bz->bounce_page_list); 1101 STAILQ_INIT(&bz->bounce_map_waitinglist); 1102 bz->free_bpages = 0; 1103 bz->reserved_bpages = 0; 1104 bz->active_bpages = 0; 1105 bz->lowaddr = dmat->lowaddr; 1106 bz->alignment = round_page(dmat->alignment); 1107 ksnprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1108 1109 if ((dmat->flags & BUS_DMA_PRIVBZONE) == 0) { 1110 ksnprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1111 busdma_zonecount++; 1112 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1113 } else { 1114 ksnprintf(bz->zoneid, 8, "zone%d", busdma_priv_zonecount); 1115 busdma_priv_zonecount--; 1116 } 1117 1118 lwkt_reltoken(&bounce_zone_tok); 1119 1120 dmat->bounce_zone = bz; 1121 1122 sysctl_ctx_init(&bz->sysctl_ctx); 1123 bz->sysctl_tree = SYSCTL_ADD_NODE(&bz->sysctl_ctx, 1124 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1125 CTLFLAG_RD, 0, ""); 1126 if (bz->sysctl_tree == NULL) { 1127 sysctl_ctx_free(&bz->sysctl_ctx); 1128 return 0; /* XXX error code? */ 1129 } 1130 1131 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1132 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1133 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1134 "Total bounce pages"); 1135 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1136 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1137 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1138 "Free bounce pages"); 1139 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1140 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1141 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1142 "Reserved bounce pages"); 1143 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1144 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1145 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1146 "Active bounce pages"); 1147 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1148 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1149 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1150 "Total bounce requests"); 1151 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1152 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1153 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1154 "Total bounce requests that were deferred"); 1155 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1156 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1157 "reserve_failed", CTLFLAG_RD, &bz->reserve_failed, 0, 1158 "Total bounce page reservations that were failed"); 1159 SYSCTL_ADD_STRING(&bz->sysctl_ctx, 1160 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1161 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1162 SYSCTL_ADD_INT(&bz->sysctl_ctx, 1163 SYSCTL_CHILDREN(bz->sysctl_tree), OID_AUTO, 1164 "alignment", CTLFLAG_RD, &bz->alignment, 0, ""); 1165 1166 return 0; 1167 } 1168 1169 static int 1170 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages, int flags) 1171 { 1172 struct bounce_zone *bz = dmat->bounce_zone; 1173 int count = 0, mflags; 1174 1175 if (flags & BUS_DMA_NOWAIT) 1176 mflags = M_NOWAIT; 1177 else 1178 mflags = M_WAITOK; 1179 1180 while (numpages > 0) { 1181 struct bounce_page *bpage; 1182 1183 bpage = kmalloc(sizeof(*bpage), M_DEVBUF, M_INTWAIT | M_ZERO); 1184 1185 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1186 mflags, 0ul, 1187 bz->lowaddr, 1188 bz->alignment, 0); 1189 if (bpage->vaddr == 0) { 1190 kfree(bpage, M_DEVBUF); 1191 break; 1192 } 1193 bpage->busaddr = pmap_kextract(bpage->vaddr); 1194 1195 BZ_LOCK(bz); 1196 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1197 total_bounce_pages++; 1198 bz->total_bpages++; 1199 bz->free_bpages++; 1200 BZ_UNLOCK(bz); 1201 1202 count++; 1203 numpages--; 1204 } 1205 return count; 1206 } 1207 1208 static void 1209 free_bounce_pages_all(bus_dma_tag_t dmat) 1210 { 1211 struct bounce_zone *bz = dmat->bounce_zone; 1212 struct bounce_page *bpage; 1213 1214 BZ_LOCK(bz); 1215 1216 while ((bpage = STAILQ_FIRST(&bz->bounce_page_list)) != NULL) { 1217 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1218 1219 KKASSERT(total_bounce_pages > 0); 1220 total_bounce_pages--; 1221 1222 KKASSERT(bz->total_bpages > 0); 1223 bz->total_bpages--; 1224 1225 KKASSERT(bz->free_bpages > 0); 1226 bz->free_bpages--; 1227 1228 BZ_UNLOCK(bz); 1229 contigfree((void *)bpage->vaddr, PAGE_SIZE, M_DEVBUF); 1230 kfree(bpage, M_DEVBUF); 1231 BZ_LOCK(bz); 1232 } 1233 if (bz->total_bpages) { 1234 kprintf("#%d bounce pages are still in use\n", 1235 bz->total_bpages); 1236 print_backtrace(-1); 1237 } 1238 1239 BZ_UNLOCK(bz); 1240 } 1241 1242 static void 1243 free_bounce_zone(bus_dma_tag_t dmat) 1244 { 1245 struct bounce_zone *bz = dmat->bounce_zone; 1246 1247 if (bz == NULL) 1248 return; 1249 1250 if ((dmat->flags & BUS_DMA_PRIVBZONE) == 0) 1251 return; 1252 1253 free_bounce_pages_all(dmat); 1254 dmat->bounce_zone = NULL; 1255 1256 if (bz->sysctl_tree != NULL) 1257 sysctl_ctx_free(&bz->sysctl_ctx); 1258 kfree(bz, M_DEVBUF); 1259 } 1260 1261 /* Assume caller holds bounce zone spinlock */ 1262 static int 1263 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1264 { 1265 struct bounce_zone *bz = dmat->bounce_zone; 1266 int pages; 1267 1268 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1269 if (!commit && map->pagesneeded > (map->pagesreserved + pages)) { 1270 bz->reserve_failed++; 1271 return (map->pagesneeded - (map->pagesreserved + pages)); 1272 } 1273 1274 bz->free_bpages -= pages; 1275 1276 bz->reserved_bpages += pages; 1277 KKASSERT(bz->reserved_bpages <= bz->total_bpages); 1278 1279 map->pagesreserved += pages; 1280 pages = map->pagesneeded - map->pagesreserved; 1281 1282 return pages; 1283 } 1284 1285 static void 1286 return_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map) 1287 { 1288 struct bounce_zone *bz = dmat->bounce_zone; 1289 int reserved = map->pagesreserved; 1290 bus_dmamap_t wait_map; 1291 1292 map->pagesreserved = 0; 1293 map->pagesneeded = 0; 1294 1295 if (reserved == 0) 1296 return; 1297 1298 BZ_LOCK(bz); 1299 1300 bz->free_bpages += reserved; 1301 KKASSERT(bz->free_bpages <= bz->total_bpages); 1302 1303 KKASSERT(bz->reserved_bpages >= reserved); 1304 bz->reserved_bpages -= reserved; 1305 1306 wait_map = get_map_waiting(dmat); 1307 1308 BZ_UNLOCK(bz); 1309 1310 if (wait_map != NULL) 1311 add_map_callback(map); 1312 } 1313 1314 static bus_addr_t 1315 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1316 bus_size_t *sizep) 1317 { 1318 struct bounce_zone *bz = dmat->bounce_zone; 1319 struct bounce_page *bpage; 1320 bus_size_t size; 1321 1322 KASSERT(map->pagesneeded > 0, ("map doesn't need any pages")); 1323 map->pagesneeded--; 1324 1325 KASSERT(map->pagesreserved > 0, ("map doesn't reserve any pages")); 1326 map->pagesreserved--; 1327 1328 BZ_LOCK(bz); 1329 1330 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1331 KASSERT(bpage != NULL, ("free page list is empty")); 1332 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1333 1334 KKASSERT(bz->reserved_bpages > 0); 1335 bz->reserved_bpages--; 1336 1337 bz->active_bpages++; 1338 KKASSERT(bz->active_bpages <= bz->total_bpages); 1339 1340 BZ_UNLOCK(bz); 1341 1342 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1343 /* 1344 * Page offset needs to be preserved. No size adjustments 1345 * needed. 1346 */ 1347 bpage->vaddr |= vaddr & PAGE_MASK; 1348 bpage->busaddr |= vaddr & PAGE_MASK; 1349 size = *sizep; 1350 } else { 1351 /* 1352 * Realign to bounce page base address, reduce size if 1353 * necessary. Bounce pages are typically already 1354 * page-aligned. 1355 */ 1356 size = PAGE_SIZE - (bpage->busaddr & PAGE_MASK); 1357 if (size < *sizep) { 1358 *sizep = size; 1359 } else { 1360 size = *sizep; 1361 } 1362 } 1363 1364 bpage->datavaddr = vaddr; 1365 bpage->datacount = size; 1366 STAILQ_INSERT_TAIL(&map->bpages, bpage, links); 1367 return bpage->busaddr; 1368 } 1369 1370 static void 1371 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1372 { 1373 struct bounce_zone *bz = dmat->bounce_zone; 1374 bus_dmamap_t map; 1375 1376 bpage->datavaddr = 0; 1377 bpage->datacount = 0; 1378 1379 if (dmat->flags & BUS_DMA_KEEP_PG_OFFSET) { 1380 /* 1381 * Reset the bounce page to start at offset 0. Other uses 1382 * of this bounce page may need to store a full page of 1383 * data and/or assume it starts on a page boundary. 1384 */ 1385 bpage->vaddr &= ~PAGE_MASK; 1386 bpage->busaddr &= ~PAGE_MASK; 1387 } 1388 1389 BZ_LOCK(bz); 1390 1391 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1392 1393 bz->free_bpages++; 1394 KKASSERT(bz->free_bpages <= bz->total_bpages); 1395 1396 KKASSERT(bz->active_bpages > 0); 1397 bz->active_bpages--; 1398 1399 map = get_map_waiting(dmat); 1400 1401 BZ_UNLOCK(bz); 1402 1403 if (map != NULL && map != (void *)-1) 1404 add_map_callback(map); 1405 } 1406 1407 /* Assume caller holds bounce zone spinlock */ 1408 static bus_dmamap_t 1409 get_map_waiting(bus_dma_tag_t dmat) 1410 { 1411 struct bounce_zone *bz = dmat->bounce_zone; 1412 bus_dmamap_t map; 1413 1414 map = STAILQ_FIRST(&bz->bounce_map_waitinglist); 1415 if (map != NULL && map != (void *)-1) { 1416 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1417 STAILQ_REMOVE_HEAD(&bz->bounce_map_waitinglist, links); 1418 bz->total_deferred++; 1419 } else { 1420 map = NULL; 1421 } 1422 } 1423 return map; 1424 } 1425 1426 static void 1427 add_map_callback(bus_dmamap_t map) 1428 { 1429 spin_lock(&bounce_map_list_spin); 1430 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, map, links); 1431 busdma_swi_pending = 1; 1432 setsoftvm(); 1433 spin_unlock(&bounce_map_list_spin); 1434 } 1435 1436 void 1437 busdma_swi(void) 1438 { 1439 bus_dmamap_t map; 1440 1441 spin_lock(&bounce_map_list_spin); 1442 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1443 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1444 spin_unlock(&bounce_map_list_spin); 1445 bus_dmamap_load(map->dmat, map, map->buf, map->buflen, 1446 map->callback, map->callback_arg, /*flags*/0); 1447 spin_lock(&bounce_map_list_spin); 1448 } 1449 spin_unlock(&bounce_map_list_spin); 1450 } 1451 1452 int 1453 bus_space_map(bus_space_tag_t t __unused, bus_addr_t addr, bus_size_t size, 1454 int flags __unused, bus_space_handle_t *bshp) 1455 { 1456 1457 if (t == X86_64_BUS_SPACE_MEM) 1458 *bshp = (uintptr_t)pmap_mapdev(addr, size); 1459 else 1460 *bshp = addr; 1461 return (0); 1462 } 1463 1464 void 1465 bus_space_unmap(bus_space_tag_t t, bus_space_handle_t bsh, bus_size_t size) 1466 { 1467 if (t == X86_64_BUS_SPACE_MEM) 1468 pmap_unmapdev(bsh, size); 1469 } 1470