1 /*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * Copyright (c) 2015-2016 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship of the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by Semihalf 10 * under sponsorship of the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions, and the following disclaimer, 17 * without modification, immediately at the beginning of the file. 18 * 2. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 25 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/bus.h> 41 #include <sys/interrupt.h> 42 #include <sys/kernel.h> 43 #include <sys/ktr.h> 44 #include <sys/lock.h> 45 #include <sys/proc.h> 46 #include <sys/memdesc.h> 47 #include <sys/mutex.h> 48 #include <sys/sysctl.h> 49 #include <sys/uio.h> 50 51 #include <vm/vm.h> 52 #include <vm/vm_extern.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_page.h> 55 #include <vm/vm_map.h> 56 57 #include <machine/atomic.h> 58 #include <machine/bus.h> 59 #include <machine/md_var.h> 60 #include <arm64/include/bus_dma_impl.h> 61 62 #define MAX_BPAGES 4096 63 64 enum { 65 BF_COULD_BOUNCE = 0x01, 66 BF_MIN_ALLOC_COMP = 0x02, 67 BF_KMEM_ALLOC = 0x04, 68 BF_COHERENT = 0x10, 69 }; 70 71 struct bounce_zone; 72 73 struct bus_dma_tag { 74 struct bus_dma_tag_common common; 75 size_t alloc_size; 76 size_t alloc_alignment; 77 int map_count; 78 int bounce_flags; 79 bus_dma_segment_t *segments; 80 struct bounce_zone *bounce_zone; 81 }; 82 83 struct bounce_page { 84 vm_offset_t vaddr; /* kva of bounce buffer */ 85 bus_addr_t busaddr; /* Physical address */ 86 vm_offset_t datavaddr; /* kva of client data */ 87 vm_page_t datapage; /* physical page of client data */ 88 vm_offset_t dataoffs; /* page offset of client data */ 89 bus_size_t datacount; /* client data count */ 90 STAILQ_ENTRY(bounce_page) links; 91 }; 92 93 int busdma_swi_pending; 94 95 struct bounce_zone { 96 STAILQ_ENTRY(bounce_zone) links; 97 STAILQ_HEAD(bp_list, bounce_page) bounce_page_list; 98 int total_bpages; 99 int free_bpages; 100 int reserved_bpages; 101 int active_bpages; 102 int total_bounced; 103 int total_deferred; 104 int map_count; 105 bus_size_t alignment; 106 bus_addr_t lowaddr; 107 char zoneid[8]; 108 char lowaddrid[20]; 109 struct sysctl_ctx_list sysctl_tree; 110 struct sysctl_oid *sysctl_tree_top; 111 }; 112 113 static struct mtx bounce_lock; 114 static int total_bpages; 115 static int busdma_zonecount; 116 static STAILQ_HEAD(, bounce_zone) bounce_zone_list; 117 118 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 119 "Busdma parameters"); 120 SYSCTL_INT(_hw_busdma, OID_AUTO, total_bpages, CTLFLAG_RD, &total_bpages, 0, 121 "Total bounce pages"); 122 123 struct sync_list { 124 vm_offset_t vaddr; /* kva of client data */ 125 bus_addr_t paddr; /* physical address */ 126 vm_page_t pages; /* starting page of client data */ 127 bus_size_t datacount; /* client data count */ 128 }; 129 130 struct bus_dmamap { 131 struct bp_list bpages; 132 int pagesneeded; 133 int pagesreserved; 134 bus_dma_tag_t dmat; 135 struct memdesc mem; 136 bus_dmamap_callback_t *callback; 137 void *callback_arg; 138 STAILQ_ENTRY(bus_dmamap) links; 139 u_int flags; 140 #define DMAMAP_COHERENT (1 << 0) 141 #define DMAMAP_FROM_DMAMEM (1 << 1) 142 #define DMAMAP_MBUF (1 << 2) 143 int sync_count; 144 struct sync_list slist[]; 145 }; 146 147 static STAILQ_HEAD(, bus_dmamap) bounce_map_waitinglist; 148 static STAILQ_HEAD(, bus_dmamap) bounce_map_callbacklist; 149 150 static void init_bounce_pages(void *dummy); 151 static int alloc_bounce_zone(bus_dma_tag_t dmat); 152 static int alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages); 153 static int reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 154 int commit); 155 static bus_addr_t add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, 156 vm_offset_t vaddr, bus_addr_t addr, bus_size_t size); 157 static void free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage); 158 int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 159 static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, 160 vm_paddr_t buf, bus_size_t buflen, int *pagesneeded); 161 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 162 pmap_t pmap, void *buf, bus_size_t buflen, int flags); 163 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 164 vm_paddr_t buf, bus_size_t buflen, int flags); 165 static int _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 166 int flags); 167 168 /* 169 * Return true if the DMA should bounce because the start or end does not fall 170 * on a cacheline boundary (which would require a partial cacheline flush). 171 * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by 172 * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a 173 * strict rule that such memory cannot be accessed by the CPU while DMA is in 174 * progress (or by multiple DMA engines at once), so that it's always safe to do 175 * full cacheline flushes even if that affects memory outside the range of a 176 * given DMA operation that doesn't involve the full allocated buffer. If we're 177 * mapping an mbuf, that follows the same rules as a buffer we allocated. 178 */ 179 static bool 180 cacheline_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 181 bus_size_t size) 182 { 183 184 #define DMAMAP_CACHELINE_FLAGS \ 185 (DMAMAP_FROM_DMAMEM | DMAMAP_COHERENT | DMAMAP_MBUF) 186 if ((dmat->bounce_flags & BF_COHERENT) != 0) 187 return (false); 188 if (map != NULL && (map->flags & DMAMAP_CACHELINE_FLAGS) != 0) 189 return (false); 190 return (((paddr | size) & (dcache_line_size - 1)) != 0); 191 #undef DMAMAP_CACHELINE_FLAGS 192 } 193 194 /* 195 * Return true if the given address does not fall on the alignment boundary. 196 */ 197 static bool 198 alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) 199 { 200 201 return ((addr & (dmat->common.alignment - 1)) != 0); 202 } 203 204 static bool 205 might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 206 bus_size_t size) 207 { 208 209 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) 210 return (true); 211 212 if (cacheline_bounce(dmat, map, paddr, size)) 213 return (true); 214 215 if (alignment_bounce(dmat, paddr)) 216 return (true); 217 218 return (false); 219 } 220 221 static bool 222 must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 223 bus_size_t size) 224 { 225 226 if (cacheline_bounce(dmat, map, paddr, size)) 227 return (true); 228 229 if (alignment_bounce(dmat, paddr)) 230 return (true); 231 232 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 && 233 bus_dma_run_filter(&dmat->common, paddr)) 234 return (true); 235 236 return (false); 237 } 238 239 /* 240 * Allocate a device specific dma_tag. 241 */ 242 static int 243 bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 244 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 245 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 246 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 247 void *lockfuncarg, bus_dma_tag_t *dmat) 248 { 249 bus_dma_tag_t newtag; 250 int error; 251 252 *dmat = NULL; 253 error = common_bus_dma_tag_create(parent != NULL ? &parent->common : 254 NULL, alignment, boundary, lowaddr, highaddr, filter, filterarg, 255 maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, 256 sizeof (struct bus_dma_tag), (void **)&newtag); 257 if (error != 0) 258 return (error); 259 260 newtag->common.impl = &bus_dma_bounce_impl; 261 newtag->map_count = 0; 262 newtag->segments = NULL; 263 264 if ((flags & BUS_DMA_COHERENT) != 0) { 265 newtag->bounce_flags |= BF_COHERENT; 266 newtag->alloc_alignment = newtag->common.alignment; 267 newtag->alloc_size = newtag->common.maxsize; 268 } else { 269 /* 270 * Ensure the buffer is aligned to a cacheline when allocating 271 * a non-coherent buffer. This is so we don't have any data 272 * that another CPU may be accessing around DMA buffer 273 * causing the cache to become dirty. 274 */ 275 newtag->alloc_alignment = MAX(newtag->common.alignment, 276 dcache_line_size); 277 newtag->alloc_size = roundup2(newtag->common.maxsize, 278 dcache_line_size); 279 } 280 281 if (parent != NULL) { 282 if ((newtag->common.filter != NULL || 283 (parent->bounce_flags & BF_COULD_BOUNCE) != 0)) 284 newtag->bounce_flags |= BF_COULD_BOUNCE; 285 286 /* Copy some flags from the parent */ 287 newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT; 288 } 289 290 if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) || 291 newtag->common.alignment > 1) 292 newtag->bounce_flags |= BF_COULD_BOUNCE; 293 294 if ((flags & BUS_DMA_ALLOCNOW) != 0) { 295 struct bounce_zone *bz; 296 297 /* Must bounce */ 298 if ((error = alloc_bounce_zone(newtag)) != 0) { 299 free(newtag, M_DEVBUF); 300 return (error); 301 } 302 bz = newtag->bounce_zone; 303 304 if (ptoa(bz->total_bpages) < maxsize) { 305 int pages; 306 307 pages = atop(round_page(maxsize)) - bz->total_bpages; 308 309 /* Add pages to our bounce pool */ 310 if (alloc_bounce_pages(newtag, pages) < pages) 311 error = ENOMEM; 312 } 313 /* Performed initial allocation */ 314 newtag->bounce_flags |= BF_MIN_ALLOC_COMP; 315 } else 316 error = 0; 317 318 if (error != 0) 319 free(newtag, M_DEVBUF); 320 else 321 *dmat = newtag; 322 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 323 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), 324 error); 325 return (error); 326 } 327 328 static int 329 bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) 330 { 331 bus_dma_tag_t dmat_copy, parent; 332 int error; 333 334 error = 0; 335 dmat_copy = dmat; 336 337 if (dmat != NULL) { 338 if (dmat->map_count != 0) { 339 error = EBUSY; 340 goto out; 341 } 342 while (dmat != NULL) { 343 parent = (bus_dma_tag_t)dmat->common.parent; 344 atomic_subtract_int(&dmat->common.ref_count, 1); 345 if (dmat->common.ref_count == 0) { 346 if (dmat->segments != NULL) 347 free(dmat->segments, M_DEVBUF); 348 free(dmat, M_DEVBUF); 349 /* 350 * Last reference count, so 351 * release our reference 352 * count on our parent. 353 */ 354 dmat = parent; 355 } else 356 dmat = NULL; 357 } 358 } 359 out: 360 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 361 return (error); 362 } 363 364 static bool 365 bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) 366 { 367 368 if (!might_bounce(dmat, NULL, buf, buflen)) 369 return (true); 370 return (!_bus_dmamap_pagesneeded(dmat, NULL, buf, buflen, NULL)); 371 } 372 373 static bus_dmamap_t 374 alloc_dmamap(bus_dma_tag_t dmat, int flags) 375 { 376 u_long mapsize; 377 bus_dmamap_t map; 378 379 mapsize = sizeof(*map); 380 mapsize += sizeof(struct sync_list) * dmat->common.nsegments; 381 map = malloc(mapsize, M_DEVBUF, flags | M_ZERO); 382 if (map == NULL) 383 return (NULL); 384 385 /* Initialize the new map */ 386 STAILQ_INIT(&map->bpages); 387 388 return (map); 389 } 390 391 /* 392 * Allocate a handle for mapping from kva/uva/physical 393 * address space into bus device space. 394 */ 395 static int 396 bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 397 { 398 struct bounce_zone *bz; 399 int error, maxpages, pages; 400 401 error = 0; 402 403 if (dmat->segments == NULL) { 404 dmat->segments = (bus_dma_segment_t *)malloc( 405 sizeof(bus_dma_segment_t) * dmat->common.nsegments, 406 M_DEVBUF, M_NOWAIT); 407 if (dmat->segments == NULL) { 408 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 409 __func__, dmat, ENOMEM); 410 return (ENOMEM); 411 } 412 } 413 414 *mapp = alloc_dmamap(dmat, M_NOWAIT); 415 if (*mapp == NULL) { 416 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 417 __func__, dmat, ENOMEM); 418 return (ENOMEM); 419 } 420 421 /* 422 * Bouncing might be required if the driver asks for an active 423 * exclusion region, a data alignment that is stricter than 1, and/or 424 * an active address boundary. 425 */ 426 if (dmat->bounce_zone == NULL) { 427 if ((error = alloc_bounce_zone(dmat)) != 0) { 428 free(*mapp, M_DEVBUF); 429 return (error); 430 } 431 } 432 bz = dmat->bounce_zone; 433 434 /* 435 * Attempt to add pages to our pool on a per-instance 436 * basis up to a sane limit. 437 */ 438 if (dmat->common.alignment > 1) 439 maxpages = MAX_BPAGES; 440 else 441 maxpages = MIN(MAX_BPAGES, Maxmem - 442 atop(dmat->common.lowaddr)); 443 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 || 444 (bz->map_count > 0 && bz->total_bpages < maxpages)) { 445 pages = MAX(atop(dmat->common.maxsize), 1); 446 pages = MIN(maxpages - bz->total_bpages, pages); 447 pages = MAX(pages, 1); 448 if (alloc_bounce_pages(dmat, pages) < pages) 449 error = ENOMEM; 450 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) { 451 if (error == 0) { 452 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 453 } 454 } else 455 error = 0; 456 } 457 bz->map_count++; 458 459 if (error == 0) { 460 dmat->map_count++; 461 if ((dmat->bounce_flags & BF_COHERENT) != 0) 462 (*mapp)->flags |= DMAMAP_COHERENT; 463 } else { 464 free(*mapp, M_DEVBUF); 465 } 466 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 467 __func__, dmat, dmat->common.flags, error); 468 return (error); 469 } 470 471 /* 472 * Destroy a handle for mapping from kva/uva/physical 473 * address space into bus device space. 474 */ 475 static int 476 bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 477 { 478 479 /* Check we are destroying the correct map type */ 480 if ((map->flags & DMAMAP_FROM_DMAMEM) != 0) 481 panic("bounce_bus_dmamap_destroy: Invalid map freed\n"); 482 483 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 484 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); 485 return (EBUSY); 486 } 487 if (dmat->bounce_zone) 488 dmat->bounce_zone->map_count--; 489 free(map, M_DEVBUF); 490 dmat->map_count--; 491 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 492 return (0); 493 } 494 495 /* 496 * Allocate a piece of memory that can be efficiently mapped into 497 * bus device space based on the constraints lited in the dma tag. 498 * A dmamap to for use with dmamap_load is also allocated. 499 */ 500 static int 501 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 502 bus_dmamap_t *mapp) 503 { 504 /* 505 * XXX ARM64TODO: 506 * This bus_dma implementation requires IO-Coherent architecutre. 507 * If IO-Coherency is not guaranteed, the BUS_DMA_COHERENT flag has 508 * to be implented using non-cacheable memory. 509 */ 510 511 vm_memattr_t attr; 512 int mflags; 513 514 if (flags & BUS_DMA_NOWAIT) 515 mflags = M_NOWAIT; 516 else 517 mflags = M_WAITOK; 518 519 if (dmat->segments == NULL) { 520 dmat->segments = (bus_dma_segment_t *)malloc( 521 sizeof(bus_dma_segment_t) * dmat->common.nsegments, 522 M_DEVBUF, mflags); 523 if (dmat->segments == NULL) { 524 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 525 __func__, dmat, dmat->common.flags, ENOMEM); 526 return (ENOMEM); 527 } 528 } 529 if (flags & BUS_DMA_ZERO) 530 mflags |= M_ZERO; 531 if (flags & BUS_DMA_NOCACHE) 532 attr = VM_MEMATTR_UNCACHEABLE; 533 else if ((flags & BUS_DMA_COHERENT) != 0 && 534 (dmat->bounce_flags & BF_COHERENT) == 0) 535 /* 536 * If we have a non-coherent tag, and are trying to allocate 537 * a coherent block of memory it needs to be uncached. 538 */ 539 attr = VM_MEMATTR_UNCACHEABLE; 540 else 541 attr = VM_MEMATTR_DEFAULT; 542 543 /* 544 * Create the map, but don't set the could bounce flag as 545 * this allocation should never bounce; 546 */ 547 *mapp = alloc_dmamap(dmat, mflags); 548 if (*mapp == NULL) { 549 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 550 __func__, dmat, dmat->common.flags, ENOMEM); 551 return (ENOMEM); 552 } 553 554 /* 555 * Mark the map as coherent if we used uncacheable memory or the 556 * tag was already marked as coherent. 557 */ 558 if (attr == VM_MEMATTR_UNCACHEABLE || 559 (dmat->bounce_flags & BF_COHERENT) != 0) 560 (*mapp)->flags |= DMAMAP_COHERENT; 561 562 (*mapp)->flags |= DMAMAP_FROM_DMAMEM; 563 564 /* 565 * Allocate the buffer from the malloc(9) allocator if... 566 * - It's small enough to fit into a single power of two sized bucket. 567 * - The alignment is less than or equal to the maximum size 568 * - The low address requirement is fulfilled. 569 * else allocate non-contiguous pages if... 570 * - The page count that could get allocated doesn't exceed 571 * nsegments also when the maximum segment size is less 572 * than PAGE_SIZE. 573 * - The alignment constraint isn't larger than a page boundary. 574 * - There are no boundary-crossing constraints. 575 * else allocate a block of contiguous pages because one or more of the 576 * constraints is something that only the contig allocator can fulfill. 577 * 578 * NOTE: The (dmat->common.alignment <= dmat->maxsize) check 579 * below is just a quick hack. The exact alignment guarantees 580 * of malloc(9) need to be nailed down, and the code below 581 * should be rewritten to take that into account. 582 * 583 * In the meantime warn the user if malloc gets it wrong. 584 */ 585 if ((dmat->alloc_size <= PAGE_SIZE) && 586 (dmat->alloc_alignment <= dmat->alloc_size) && 587 dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && 588 attr == VM_MEMATTR_DEFAULT) { 589 *vaddr = malloc(dmat->alloc_size, M_DEVBUF, mflags); 590 } else if (dmat->common.nsegments >= 591 howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && 592 dmat->alloc_alignment <= PAGE_SIZE && 593 (dmat->common.boundary % PAGE_SIZE) == 0) { 594 /* Page-based multi-segment allocations allowed */ 595 *vaddr = (void *)kmem_alloc_attr(dmat->alloc_size, mflags, 596 0ul, dmat->common.lowaddr, attr); 597 dmat->bounce_flags |= BF_KMEM_ALLOC; 598 } else { 599 *vaddr = (void *)kmem_alloc_contig(dmat->alloc_size, mflags, 600 0ul, dmat->common.lowaddr, dmat->alloc_alignment != 0 ? 601 dmat->alloc_alignment : 1ul, dmat->common.boundary, attr); 602 dmat->bounce_flags |= BF_KMEM_ALLOC; 603 } 604 if (*vaddr == NULL) { 605 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 606 __func__, dmat, dmat->common.flags, ENOMEM); 607 free(*mapp, M_DEVBUF); 608 return (ENOMEM); 609 } else if (vtophys(*vaddr) & (dmat->alloc_alignment - 1)) { 610 printf("bus_dmamem_alloc failed to align memory properly.\n"); 611 } 612 dmat->map_count++; 613 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 614 __func__, dmat, dmat->common.flags, 0); 615 return (0); 616 } 617 618 /* 619 * Free a piece of memory and it's allociated dmamap, that was allocated 620 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 621 */ 622 static void 623 bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 624 { 625 626 /* 627 * Check the map came from bounce_bus_dmamem_alloc, so the map 628 * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc() 629 * was used and set if kmem_alloc_contig() was used. 630 */ 631 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 632 panic("bus_dmamem_free: Invalid map freed\n"); 633 if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0) 634 free(vaddr, M_DEVBUF); 635 else 636 kmem_free((vm_offset_t)vaddr, dmat->alloc_size); 637 free(map, M_DEVBUF); 638 dmat->map_count--; 639 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, 640 dmat->bounce_flags); 641 } 642 643 static bool 644 _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 645 bus_size_t buflen, int *pagesneeded) 646 { 647 bus_addr_t curaddr; 648 bus_size_t sgsize; 649 int count; 650 651 /* 652 * Count the number of bounce pages needed in order to 653 * complete this transfer 654 */ 655 count = 0; 656 curaddr = buf; 657 while (buflen != 0) { 658 sgsize = MIN(buflen, dmat->common.maxsegsz); 659 if (must_bounce(dmat, map, curaddr, sgsize)) { 660 sgsize = MIN(sgsize, 661 PAGE_SIZE - (curaddr & PAGE_MASK)); 662 if (pagesneeded == NULL) 663 return (true); 664 count++; 665 } 666 curaddr += sgsize; 667 buflen -= sgsize; 668 } 669 670 if (pagesneeded != NULL) 671 *pagesneeded = count; 672 return (count != 0); 673 } 674 675 static void 676 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 677 bus_size_t buflen, int flags) 678 { 679 680 if (map->pagesneeded == 0) { 681 _bus_dmamap_pagesneeded(dmat, map, buf, buflen, 682 &map->pagesneeded); 683 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 684 } 685 } 686 687 static void 688 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 689 void *buf, bus_size_t buflen, int flags) 690 { 691 vm_offset_t vaddr; 692 vm_offset_t vendaddr; 693 bus_addr_t paddr; 694 bus_size_t sg_len; 695 696 if (map->pagesneeded == 0) { 697 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 698 "alignment= %d", dmat->common.lowaddr, 699 ptoa((vm_paddr_t)Maxmem), 700 dmat->common.boundary, dmat->common.alignment); 701 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, 702 map->pagesneeded); 703 /* 704 * Count the number of bounce pages 705 * needed in order to complete this transfer 706 */ 707 vaddr = (vm_offset_t)buf; 708 vendaddr = (vm_offset_t)buf + buflen; 709 710 while (vaddr < vendaddr) { 711 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 712 if (pmap == kernel_pmap) 713 paddr = pmap_kextract(vaddr); 714 else 715 paddr = pmap_extract(pmap, vaddr); 716 if (must_bounce(dmat, map, paddr, 717 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & 718 PAGE_MASK)))) != 0) { 719 sg_len = roundup2(sg_len, 720 dmat->common.alignment); 721 map->pagesneeded++; 722 } 723 vaddr += sg_len; 724 } 725 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 726 } 727 } 728 729 static int 730 _bus_dmamap_reserve_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int flags) 731 { 732 733 /* Reserve Necessary Bounce Pages */ 734 mtx_lock(&bounce_lock); 735 if (flags & BUS_DMA_NOWAIT) { 736 if (reserve_bounce_pages(dmat, map, 0) != 0) { 737 mtx_unlock(&bounce_lock); 738 return (ENOMEM); 739 } 740 } else { 741 if (reserve_bounce_pages(dmat, map, 1) != 0) { 742 /* Queue us for resources */ 743 STAILQ_INSERT_TAIL(&bounce_map_waitinglist, map, links); 744 mtx_unlock(&bounce_lock); 745 return (EINPROGRESS); 746 } 747 } 748 mtx_unlock(&bounce_lock); 749 750 return (0); 751 } 752 753 /* 754 * Add a single contiguous physical range to the segment list. 755 */ 756 static bus_size_t 757 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 758 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 759 { 760 bus_addr_t baddr, bmask; 761 int seg; 762 763 /* 764 * Make sure we don't cross any boundaries. 765 */ 766 bmask = ~(dmat->common.boundary - 1); 767 if (dmat->common.boundary > 0) { 768 baddr = (curaddr + dmat->common.boundary) & bmask; 769 if (sgsize > (baddr - curaddr)) 770 sgsize = (baddr - curaddr); 771 } 772 773 /* 774 * Insert chunk into a segment, coalescing with 775 * previous segment if possible. 776 */ 777 seg = *segp; 778 if (seg == -1) { 779 seg = 0; 780 segs[seg].ds_addr = curaddr; 781 segs[seg].ds_len = sgsize; 782 } else { 783 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 784 (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz && 785 (dmat->common.boundary == 0 || 786 (segs[seg].ds_addr & bmask) == (curaddr & bmask))) 787 segs[seg].ds_len += sgsize; 788 else { 789 if (++seg >= dmat->common.nsegments) 790 return (0); 791 segs[seg].ds_addr = curaddr; 792 segs[seg].ds_len = sgsize; 793 } 794 } 795 *segp = seg; 796 return (sgsize); 797 } 798 799 /* 800 * Utility function to load a physical buffer. segp contains 801 * the starting segment on entrace, and the ending segment on exit. 802 */ 803 static int 804 bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 805 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, 806 int *segp) 807 { 808 struct sync_list *sl; 809 bus_size_t sgsize; 810 bus_addr_t curaddr, sl_end; 811 int error; 812 813 if (segs == NULL) 814 segs = dmat->segments; 815 816 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 817 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 818 if (map->pagesneeded != 0) { 819 error = _bus_dmamap_reserve_pages(dmat, map, flags); 820 if (error) 821 return (error); 822 } 823 } 824 825 sl = map->slist + map->sync_count - 1; 826 sl_end = 0; 827 828 while (buflen > 0) { 829 curaddr = buf; 830 sgsize = MIN(buflen, dmat->common.maxsegsz); 831 if (map->pagesneeded != 0 && 832 must_bounce(dmat, map, curaddr, sgsize)) { 833 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 834 curaddr = add_bounce_page(dmat, map, 0, curaddr, 835 sgsize); 836 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 837 if (map->sync_count > 0) 838 sl_end = sl->paddr + sl->datacount; 839 840 if (map->sync_count == 0 || curaddr != sl_end) { 841 if (++map->sync_count > dmat->common.nsegments) 842 break; 843 sl++; 844 sl->vaddr = 0; 845 sl->paddr = curaddr; 846 sl->datacount = sgsize; 847 sl->pages = PHYS_TO_VM_PAGE(curaddr); 848 KASSERT(sl->pages != NULL, 849 ("%s: page at PA:0x%08lx is not in " 850 "vm_page_array", __func__, curaddr)); 851 } else 852 sl->datacount += sgsize; 853 } 854 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 855 segp); 856 if (sgsize == 0) 857 break; 858 buf += sgsize; 859 buflen -= sgsize; 860 } 861 862 /* 863 * Did we fit? 864 */ 865 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 866 } 867 868 /* 869 * Utility function to load a linear buffer. segp contains 870 * the starting segment on entrace, and the ending segment on exit. 871 */ 872 static int 873 bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 874 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 875 int *segp) 876 { 877 struct sync_list *sl; 878 bus_size_t sgsize, max_sgsize; 879 bus_addr_t curaddr, sl_pend; 880 vm_offset_t kvaddr, vaddr, sl_vend; 881 int error; 882 883 if (segs == NULL) 884 segs = dmat->segments; 885 886 if (flags & BUS_DMA_LOAD_MBUF) 887 map->flags |= DMAMAP_MBUF; 888 889 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 890 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 891 if (map->pagesneeded != 0) { 892 error = _bus_dmamap_reserve_pages(dmat, map, flags); 893 if (error) 894 return (error); 895 } 896 } 897 898 sl = map->slist + map->sync_count - 1; 899 vaddr = (vm_offset_t)buf; 900 sl_pend = 0; 901 sl_vend = 0; 902 903 while (buflen > 0) { 904 /* 905 * Get the physical address for this segment. 906 */ 907 if (pmap == kernel_pmap) { 908 curaddr = pmap_kextract(vaddr); 909 kvaddr = vaddr; 910 } else { 911 curaddr = pmap_extract(pmap, vaddr); 912 kvaddr = 0; 913 } 914 915 /* 916 * Compute the segment size, and adjust counts. 917 */ 918 max_sgsize = MIN(buflen, dmat->common.maxsegsz); 919 sgsize = PAGE_SIZE - (curaddr & PAGE_MASK); 920 if (map->pagesneeded != 0 && 921 must_bounce(dmat, map, curaddr, sgsize)) { 922 sgsize = roundup2(sgsize, dmat->common.alignment); 923 sgsize = MIN(sgsize, max_sgsize); 924 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 925 sgsize); 926 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 927 sgsize = MIN(sgsize, max_sgsize); 928 if (map->sync_count > 0) { 929 sl_pend = sl->paddr + sl->datacount; 930 sl_vend = sl->vaddr + sl->datacount; 931 } 932 933 if (map->sync_count == 0 || 934 (kvaddr != 0 && kvaddr != sl_vend) || 935 (curaddr != sl_pend)) { 936 if (++map->sync_count > dmat->common.nsegments) 937 goto cleanup; 938 sl++; 939 sl->vaddr = kvaddr; 940 sl->paddr = curaddr; 941 if (kvaddr != 0) { 942 sl->pages = NULL; 943 } else { 944 sl->pages = PHYS_TO_VM_PAGE(curaddr); 945 KASSERT(sl->pages != NULL, 946 ("%s: page at PA:0x%08lx is not " 947 "in vm_page_array", __func__, 948 curaddr)); 949 } 950 sl->datacount = sgsize; 951 } else 952 sl->datacount += sgsize; 953 } else { 954 sgsize = MIN(sgsize, max_sgsize); 955 } 956 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 957 segp); 958 if (sgsize == 0) 959 break; 960 vaddr += sgsize; 961 buflen -= sgsize; 962 } 963 964 cleanup: 965 /* 966 * Did we fit? 967 */ 968 return (buflen != 0 ? EFBIG : 0); /* XXX better return value here? */ 969 } 970 971 static void 972 bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 973 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 974 { 975 976 map->mem = *mem; 977 map->dmat = dmat; 978 map->callback = callback; 979 map->callback_arg = callback_arg; 980 } 981 982 static bus_dma_segment_t * 983 bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 984 bus_dma_segment_t *segs, int nsegs, int error) 985 { 986 987 if (segs == NULL) 988 segs = dmat->segments; 989 return (segs); 990 } 991 992 /* 993 * Release the mapping held by map. 994 */ 995 static void 996 bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 997 { 998 struct bounce_page *bpage; 999 1000 while ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1001 STAILQ_REMOVE_HEAD(&map->bpages, links); 1002 free_bounce_page(dmat, bpage); 1003 } 1004 1005 map->sync_count = 0; 1006 map->flags &= ~DMAMAP_MBUF; 1007 } 1008 1009 static void 1010 dma_preread_safe(vm_offset_t va, vm_size_t size) 1011 { 1012 /* 1013 * Write back any partial cachelines immediately before and 1014 * after the DMA region. 1015 */ 1016 if (va & (dcache_line_size - 1)) 1017 cpu_dcache_wb_range(va, 1); 1018 if ((va + size) & (dcache_line_size - 1)) 1019 cpu_dcache_wb_range(va + size, 1); 1020 1021 cpu_dcache_inv_range(va, size); 1022 } 1023 1024 static void 1025 dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) 1026 { 1027 uint32_t len, offset; 1028 vm_page_t m; 1029 vm_paddr_t pa; 1030 vm_offset_t va, tempva; 1031 bus_size_t size; 1032 1033 offset = sl->paddr & PAGE_MASK; 1034 m = sl->pages; 1035 size = sl->datacount; 1036 pa = sl->paddr; 1037 1038 for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { 1039 tempva = 0; 1040 if (sl->vaddr == 0) { 1041 len = min(PAGE_SIZE - offset, size); 1042 tempva = pmap_quick_enter_page(m); 1043 va = tempva | offset; 1044 KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), 1045 ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx", 1046 VM_PAGE_TO_PHYS(m) | offset, pa)); 1047 } else { 1048 len = sl->datacount; 1049 va = sl->vaddr; 1050 } 1051 1052 switch (op) { 1053 case BUS_DMASYNC_PREWRITE: 1054 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1055 cpu_dcache_wb_range(va, len); 1056 break; 1057 case BUS_DMASYNC_PREREAD: 1058 /* 1059 * An mbuf may start in the middle of a cacheline. There 1060 * will be no cpu writes to the beginning of that line 1061 * (which contains the mbuf header) while dma is in 1062 * progress. Handle that case by doing a writeback of 1063 * just the first cacheline before invalidating the 1064 * overall buffer. Any mbuf in a chain may have this 1065 * misalignment. Buffers which are not mbufs bounce if 1066 * they are not aligned to a cacheline. 1067 */ 1068 dma_preread_safe(va, len); 1069 break; 1070 case BUS_DMASYNC_POSTREAD: 1071 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1072 cpu_dcache_inv_range(va, len); 1073 break; 1074 default: 1075 panic("unsupported combination of sync operations: " 1076 "0x%08x\n", op); 1077 } 1078 1079 if (tempva != 0) 1080 pmap_quick_remove_page(tempva); 1081 } 1082 } 1083 1084 static void 1085 bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, 1086 bus_dmasync_op_t op) 1087 { 1088 struct bounce_page *bpage; 1089 struct sync_list *sl, *end; 1090 vm_offset_t datavaddr, tempvaddr; 1091 1092 if (op == BUS_DMASYNC_POSTWRITE) 1093 return; 1094 1095 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1096 /* 1097 * Wait for any DMA operations to complete before the bcopy. 1098 */ 1099 dsb(sy); 1100 } 1101 1102 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1103 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1104 "performing bounce", __func__, dmat, dmat->common.flags, 1105 op); 1106 1107 if ((op & BUS_DMASYNC_PREWRITE) != 0) { 1108 while (bpage != NULL) { 1109 tempvaddr = 0; 1110 datavaddr = bpage->datavaddr; 1111 if (datavaddr == 0) { 1112 tempvaddr = pmap_quick_enter_page( 1113 bpage->datapage); 1114 datavaddr = tempvaddr | bpage->dataoffs; 1115 } 1116 1117 bcopy((void *)datavaddr, 1118 (void *)bpage->vaddr, bpage->datacount); 1119 if (tempvaddr != 0) 1120 pmap_quick_remove_page(tempvaddr); 1121 if ((map->flags & DMAMAP_COHERENT) == 0) 1122 cpu_dcache_wb_range(bpage->vaddr, 1123 bpage->datacount); 1124 bpage = STAILQ_NEXT(bpage, links); 1125 } 1126 dmat->bounce_zone->total_bounced++; 1127 } else if ((op & BUS_DMASYNC_PREREAD) != 0) { 1128 while (bpage != NULL) { 1129 if ((map->flags & DMAMAP_COHERENT) == 0) 1130 cpu_dcache_wbinv_range(bpage->vaddr, 1131 bpage->datacount); 1132 bpage = STAILQ_NEXT(bpage, links); 1133 } 1134 } 1135 1136 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1137 while (bpage != NULL) { 1138 if ((map->flags & DMAMAP_COHERENT) == 0) 1139 cpu_dcache_inv_range(bpage->vaddr, 1140 bpage->datacount); 1141 tempvaddr = 0; 1142 datavaddr = bpage->datavaddr; 1143 if (datavaddr == 0) { 1144 tempvaddr = pmap_quick_enter_page( 1145 bpage->datapage); 1146 datavaddr = tempvaddr | bpage->dataoffs; 1147 } 1148 1149 bcopy((void *)bpage->vaddr, 1150 (void *)datavaddr, bpage->datacount); 1151 1152 if (tempvaddr != 0) 1153 pmap_quick_remove_page(tempvaddr); 1154 bpage = STAILQ_NEXT(bpage, links); 1155 } 1156 dmat->bounce_zone->total_bounced++; 1157 } 1158 } 1159 1160 /* 1161 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. 1162 */ 1163 if (map->sync_count != 0) { 1164 sl = &map->slist[0]; 1165 end = &map->slist[map->sync_count]; 1166 CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x " 1167 "performing sync", __func__, dmat, op); 1168 1169 for ( ; sl != end; ++sl) 1170 dma_dcache_sync(sl, op); 1171 } 1172 1173 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) { 1174 /* 1175 * Wait for the bcopy to complete before any DMA operations. 1176 */ 1177 dsb(sy); 1178 } 1179 } 1180 1181 static void 1182 init_bounce_pages(void *dummy __unused) 1183 { 1184 1185 total_bpages = 0; 1186 STAILQ_INIT(&bounce_zone_list); 1187 STAILQ_INIT(&bounce_map_waitinglist); 1188 STAILQ_INIT(&bounce_map_callbacklist); 1189 mtx_init(&bounce_lock, "bounce pages lock", NULL, MTX_DEF); 1190 } 1191 SYSINIT(bpages, SI_SUB_LOCK, SI_ORDER_ANY, init_bounce_pages, NULL); 1192 1193 static struct sysctl_ctx_list * 1194 busdma_sysctl_tree(struct bounce_zone *bz) 1195 { 1196 1197 return (&bz->sysctl_tree); 1198 } 1199 1200 static struct sysctl_oid * 1201 busdma_sysctl_tree_top(struct bounce_zone *bz) 1202 { 1203 1204 return (bz->sysctl_tree_top); 1205 } 1206 1207 static int 1208 alloc_bounce_zone(bus_dma_tag_t dmat) 1209 { 1210 struct bounce_zone *bz; 1211 1212 /* Check to see if we already have a suitable zone */ 1213 STAILQ_FOREACH(bz, &bounce_zone_list, links) { 1214 if ((dmat->common.alignment <= bz->alignment) && 1215 (dmat->common.lowaddr >= bz->lowaddr)) { 1216 dmat->bounce_zone = bz; 1217 return (0); 1218 } 1219 } 1220 1221 if ((bz = (struct bounce_zone *)malloc(sizeof(*bz), M_DEVBUF, 1222 M_NOWAIT | M_ZERO)) == NULL) 1223 return (ENOMEM); 1224 1225 STAILQ_INIT(&bz->bounce_page_list); 1226 bz->free_bpages = 0; 1227 bz->reserved_bpages = 0; 1228 bz->active_bpages = 0; 1229 bz->lowaddr = dmat->common.lowaddr; 1230 bz->alignment = MAX(dmat->common.alignment, PAGE_SIZE); 1231 bz->map_count = 0; 1232 snprintf(bz->zoneid, 8, "zone%d", busdma_zonecount); 1233 busdma_zonecount++; 1234 snprintf(bz->lowaddrid, 18, "%#jx", (uintmax_t)bz->lowaddr); 1235 STAILQ_INSERT_TAIL(&bounce_zone_list, bz, links); 1236 dmat->bounce_zone = bz; 1237 1238 sysctl_ctx_init(&bz->sysctl_tree); 1239 bz->sysctl_tree_top = SYSCTL_ADD_NODE(&bz->sysctl_tree, 1240 SYSCTL_STATIC_CHILDREN(_hw_busdma), OID_AUTO, bz->zoneid, 1241 CTLFLAG_RD | CTLFLAG_MPSAFE, 0, ""); 1242 if (bz->sysctl_tree_top == NULL) { 1243 sysctl_ctx_free(&bz->sysctl_tree); 1244 return (0); /* XXX error code? */ 1245 } 1246 1247 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1248 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1249 "total_bpages", CTLFLAG_RD, &bz->total_bpages, 0, 1250 "Total bounce pages"); 1251 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1252 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1253 "free_bpages", CTLFLAG_RD, &bz->free_bpages, 0, 1254 "Free bounce pages"); 1255 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1256 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1257 "reserved_bpages", CTLFLAG_RD, &bz->reserved_bpages, 0, 1258 "Reserved bounce pages"); 1259 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1260 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1261 "active_bpages", CTLFLAG_RD, &bz->active_bpages, 0, 1262 "Active bounce pages"); 1263 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1264 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1265 "total_bounced", CTLFLAG_RD, &bz->total_bounced, 0, 1266 "Total bounce requests"); 1267 SYSCTL_ADD_INT(busdma_sysctl_tree(bz), 1268 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1269 "total_deferred", CTLFLAG_RD, &bz->total_deferred, 0, 1270 "Total bounce requests that were deferred"); 1271 SYSCTL_ADD_STRING(busdma_sysctl_tree(bz), 1272 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1273 "lowaddr", CTLFLAG_RD, bz->lowaddrid, 0, ""); 1274 SYSCTL_ADD_UAUTO(busdma_sysctl_tree(bz), 1275 SYSCTL_CHILDREN(busdma_sysctl_tree_top(bz)), OID_AUTO, 1276 "alignment", CTLFLAG_RD, &bz->alignment, ""); 1277 1278 return (0); 1279 } 1280 1281 static int 1282 alloc_bounce_pages(bus_dma_tag_t dmat, u_int numpages) 1283 { 1284 struct bounce_zone *bz; 1285 int count; 1286 1287 bz = dmat->bounce_zone; 1288 count = 0; 1289 while (numpages > 0) { 1290 struct bounce_page *bpage; 1291 1292 bpage = (struct bounce_page *)malloc(sizeof(*bpage), M_DEVBUF, 1293 M_NOWAIT | M_ZERO); 1294 1295 if (bpage == NULL) 1296 break; 1297 bpage->vaddr = (vm_offset_t)contigmalloc(PAGE_SIZE, M_DEVBUF, 1298 M_NOWAIT, 0ul, bz->lowaddr, PAGE_SIZE, 0); 1299 if (bpage->vaddr == 0) { 1300 free(bpage, M_DEVBUF); 1301 break; 1302 } 1303 bpage->busaddr = pmap_kextract(bpage->vaddr); 1304 mtx_lock(&bounce_lock); 1305 STAILQ_INSERT_TAIL(&bz->bounce_page_list, bpage, links); 1306 total_bpages++; 1307 bz->total_bpages++; 1308 bz->free_bpages++; 1309 mtx_unlock(&bounce_lock); 1310 count++; 1311 numpages--; 1312 } 1313 return (count); 1314 } 1315 1316 static int 1317 reserve_bounce_pages(bus_dma_tag_t dmat, bus_dmamap_t map, int commit) 1318 { 1319 struct bounce_zone *bz; 1320 int pages; 1321 1322 mtx_assert(&bounce_lock, MA_OWNED); 1323 bz = dmat->bounce_zone; 1324 pages = MIN(bz->free_bpages, map->pagesneeded - map->pagesreserved); 1325 if (commit == 0 && map->pagesneeded > (map->pagesreserved + pages)) 1326 return (map->pagesneeded - (map->pagesreserved + pages)); 1327 bz->free_bpages -= pages; 1328 bz->reserved_bpages += pages; 1329 map->pagesreserved += pages; 1330 pages = map->pagesneeded - map->pagesreserved; 1331 1332 return (pages); 1333 } 1334 1335 static bus_addr_t 1336 add_bounce_page(bus_dma_tag_t dmat, bus_dmamap_t map, vm_offset_t vaddr, 1337 bus_addr_t addr, bus_size_t size) 1338 { 1339 struct bounce_zone *bz; 1340 struct bounce_page *bpage; 1341 1342 KASSERT(dmat->bounce_zone != NULL, ("no bounce zone in dma tag")); 1343 1344 bz = dmat->bounce_zone; 1345 if (map->pagesneeded == 0) 1346 panic("add_bounce_page: map doesn't need any pages"); 1347 map->pagesneeded--; 1348 1349 if (map->pagesreserved == 0) 1350 panic("add_bounce_page: map doesn't need any pages"); 1351 map->pagesreserved--; 1352 1353 mtx_lock(&bounce_lock); 1354 bpage = STAILQ_FIRST(&bz->bounce_page_list); 1355 if (bpage == NULL) 1356 panic("add_bounce_page: free page list is empty"); 1357 1358 STAILQ_REMOVE_HEAD(&bz->bounce_page_list, links); 1359 bz->reserved_bpages--; 1360 bz->active_bpages++; 1361 mtx_unlock(&bounce_lock); 1362 1363 if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) { 1364 /* Page offset needs to be preserved. */ 1365 bpage->vaddr |= addr & PAGE_MASK; 1366 bpage->busaddr |= addr & PAGE_MASK; 1367 } 1368 bpage->datavaddr = vaddr; 1369 bpage->datapage = PHYS_TO_VM_PAGE(addr); 1370 bpage->dataoffs = addr & PAGE_MASK; 1371 bpage->datacount = size; 1372 STAILQ_INSERT_TAIL(&(map->bpages), bpage, links); 1373 return (bpage->busaddr); 1374 } 1375 1376 static void 1377 free_bounce_page(bus_dma_tag_t dmat, struct bounce_page *bpage) 1378 { 1379 struct bus_dmamap *map; 1380 struct bounce_zone *bz; 1381 1382 bz = dmat->bounce_zone; 1383 bpage->datavaddr = 0; 1384 bpage->datacount = 0; 1385 if (dmat->common.flags & BUS_DMA_KEEP_PG_OFFSET) { 1386 /* 1387 * Reset the bounce page to start at offset 0. Other uses 1388 * of this bounce page may need to store a full page of 1389 * data and/or assume it starts on a page boundary. 1390 */ 1391 bpage->vaddr &= ~PAGE_MASK; 1392 bpage->busaddr &= ~PAGE_MASK; 1393 } 1394 1395 mtx_lock(&bounce_lock); 1396 STAILQ_INSERT_HEAD(&bz->bounce_page_list, bpage, links); 1397 bz->free_bpages++; 1398 bz->active_bpages--; 1399 if ((map = STAILQ_FIRST(&bounce_map_waitinglist)) != NULL) { 1400 if (reserve_bounce_pages(map->dmat, map, 1) == 0) { 1401 STAILQ_REMOVE_HEAD(&bounce_map_waitinglist, links); 1402 STAILQ_INSERT_TAIL(&bounce_map_callbacklist, 1403 map, links); 1404 busdma_swi_pending = 1; 1405 bz->total_deferred++; 1406 swi_sched(vm_ih, 0); 1407 } 1408 } 1409 mtx_unlock(&bounce_lock); 1410 } 1411 1412 void 1413 busdma_swi(void) 1414 { 1415 bus_dma_tag_t dmat; 1416 struct bus_dmamap *map; 1417 1418 mtx_lock(&bounce_lock); 1419 while ((map = STAILQ_FIRST(&bounce_map_callbacklist)) != NULL) { 1420 STAILQ_REMOVE_HEAD(&bounce_map_callbacklist, links); 1421 mtx_unlock(&bounce_lock); 1422 dmat = map->dmat; 1423 (dmat->common.lockfunc)(dmat->common.lockfuncarg, BUS_DMA_LOCK); 1424 bus_dmamap_load_mem(map->dmat, map, &map->mem, 1425 map->callback, map->callback_arg, BUS_DMA_WAITOK); 1426 (dmat->common.lockfunc)(dmat->common.lockfuncarg, 1427 BUS_DMA_UNLOCK); 1428 mtx_lock(&bounce_lock); 1429 } 1430 mtx_unlock(&bounce_lock); 1431 } 1432 1433 struct bus_dma_impl bus_dma_bounce_impl = { 1434 .tag_create = bounce_bus_dma_tag_create, 1435 .tag_destroy = bounce_bus_dma_tag_destroy, 1436 .id_mapped = bounce_bus_dma_id_mapped, 1437 .map_create = bounce_bus_dmamap_create, 1438 .map_destroy = bounce_bus_dmamap_destroy, 1439 .mem_alloc = bounce_bus_dmamem_alloc, 1440 .mem_free = bounce_bus_dmamem_free, 1441 .load_phys = bounce_bus_dmamap_load_phys, 1442 .load_buffer = bounce_bus_dmamap_load_buffer, 1443 .load_ma = bus_dmamap_load_ma_triv, 1444 .map_waitok = bounce_bus_dmamap_waitok, 1445 .map_complete = bounce_bus_dmamap_complete, 1446 .map_unload = bounce_bus_dmamap_unload, 1447 .map_sync = bounce_bus_dmamap_sync 1448 }; 1449