1 /*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * Copyright (c) 2015-2016 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship of the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by Semihalf 10 * under sponsorship of the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions, and the following disclaimer, 17 * without modification, immediately at the beginning of the file. 18 * 2. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 25 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/malloc.h> 40 #include <sys/bus.h> 41 #include <sys/interrupt.h> 42 #include <sys/kernel.h> 43 #include <sys/ktr.h> 44 #include <sys/lock.h> 45 #include <sys/proc.h> 46 #include <sys/memdesc.h> 47 #include <sys/mutex.h> 48 #include <sys/sysctl.h> 49 #include <sys/uio.h> 50 51 #include <vm/vm.h> 52 #include <vm/vm_extern.h> 53 #include <vm/vm_kern.h> 54 #include <vm/vm_page.h> 55 #include <vm/vm_map.h> 56 57 #include <machine/atomic.h> 58 #include <machine/bus.h> 59 #include <machine/md_var.h> 60 #include <arm64/include/bus_dma_impl.h> 61 62 #define MAX_BPAGES 4096 63 64 enum { 65 BF_COULD_BOUNCE = 0x01, 66 BF_MIN_ALLOC_COMP = 0x02, 67 BF_KMEM_ALLOC = 0x04, 68 BF_COHERENT = 0x10, 69 }; 70 71 struct bounce_page; 72 struct bounce_zone; 73 74 struct bus_dma_tag { 75 struct bus_dma_tag_common common; 76 size_t alloc_size; 77 size_t alloc_alignment; 78 int map_count; 79 int bounce_flags; 80 bus_dma_segment_t *segments; 81 struct bounce_zone *bounce_zone; 82 }; 83 84 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 85 "Busdma parameters"); 86 87 struct sync_list { 88 vm_offset_t vaddr; /* kva of client data */ 89 bus_addr_t paddr; /* physical address */ 90 vm_page_t pages; /* starting page of client data */ 91 bus_size_t datacount; /* client data count */ 92 }; 93 94 struct bus_dmamap { 95 STAILQ_HEAD(, bounce_page) bpages; 96 int pagesneeded; 97 int pagesreserved; 98 bus_dma_tag_t dmat; 99 struct memdesc mem; 100 bus_dmamap_callback_t *callback; 101 void *callback_arg; 102 STAILQ_ENTRY(bus_dmamap) links; 103 u_int flags; 104 #define DMAMAP_COHERENT (1 << 0) 105 #define DMAMAP_FROM_DMAMEM (1 << 1) 106 #define DMAMAP_MBUF (1 << 2) 107 int sync_count; 108 struct sync_list slist[]; 109 }; 110 111 int run_filter(bus_dma_tag_t dmat, bus_addr_t paddr); 112 static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, 113 vm_paddr_t buf, bus_size_t buflen, int *pagesneeded); 114 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 115 pmap_t pmap, void *buf, bus_size_t buflen, int flags); 116 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 117 vm_paddr_t buf, bus_size_t buflen, int flags); 118 119 static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); 120 121 #define dmat_alignment(dmat) ((dmat)->common.alignment) 122 #define dmat_flags(dmat) ((dmat)->common.flags) 123 #define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) 124 #define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) 125 #define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) 126 127 #include "../../kern/subr_busdma_bounce.c" 128 129 /* 130 * Return true if the DMA should bounce because the start or end does not fall 131 * on a cacheline boundary (which would require a partial cacheline flush). 132 * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by 133 * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a 134 * strict rule that such memory cannot be accessed by the CPU while DMA is in 135 * progress (or by multiple DMA engines at once), so that it's always safe to do 136 * full cacheline flushes even if that affects memory outside the range of a 137 * given DMA operation that doesn't involve the full allocated buffer. If we're 138 * mapping an mbuf, that follows the same rules as a buffer we allocated. 139 */ 140 static bool 141 cacheline_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 142 bus_size_t size) 143 { 144 145 #define DMAMAP_CACHELINE_FLAGS \ 146 (DMAMAP_FROM_DMAMEM | DMAMAP_COHERENT | DMAMAP_MBUF) 147 if ((dmat->bounce_flags & BF_COHERENT) != 0) 148 return (false); 149 if (map != NULL && (map->flags & DMAMAP_CACHELINE_FLAGS) != 0) 150 return (false); 151 return (((paddr | size) & (dcache_line_size - 1)) != 0); 152 #undef DMAMAP_CACHELINE_FLAGS 153 } 154 155 /* 156 * Return true if the given address does not fall on the alignment boundary. 157 */ 158 static bool 159 alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) 160 { 161 162 return (!vm_addr_align_ok(addr, dmat->common.alignment)); 163 } 164 165 static bool 166 might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 167 bus_size_t size) 168 { 169 170 /* Memory allocated by bounce_bus_dmamem_alloc won't bounce */ 171 if (map && (map->flags & DMAMAP_FROM_DMAMEM) != 0) 172 return (false); 173 174 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) 175 return (true); 176 177 if (cacheline_bounce(dmat, map, paddr, size)) 178 return (true); 179 180 if (alignment_bounce(dmat, paddr)) 181 return (true); 182 183 return (false); 184 } 185 186 static bool 187 must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 188 bus_size_t size) 189 { 190 191 if (cacheline_bounce(dmat, map, paddr, size)) 192 return (true); 193 194 if (alignment_bounce(dmat, paddr)) 195 return (true); 196 197 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 && 198 bus_dma_run_filter(&dmat->common, paddr)) 199 return (true); 200 201 return (false); 202 } 203 204 /* 205 * Allocate a device specific dma_tag. 206 */ 207 static int 208 bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 209 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 210 bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize, 211 int nsegments, bus_size_t maxsegsz, int flags, bus_dma_lock_t *lockfunc, 212 void *lockfuncarg, bus_dma_tag_t *dmat) 213 { 214 bus_dma_tag_t newtag; 215 int error; 216 217 *dmat = NULL; 218 error = common_bus_dma_tag_create(parent != NULL ? &parent->common : 219 NULL, alignment, boundary, lowaddr, highaddr, filter, filterarg, 220 maxsize, nsegments, maxsegsz, flags, lockfunc, lockfuncarg, 221 sizeof (struct bus_dma_tag), (void **)&newtag); 222 if (error != 0) 223 return (error); 224 225 newtag->common.impl = &bus_dma_bounce_impl; 226 newtag->map_count = 0; 227 newtag->segments = NULL; 228 229 if ((flags & BUS_DMA_COHERENT) != 0) { 230 newtag->bounce_flags |= BF_COHERENT; 231 } 232 233 if (parent != NULL) { 234 if ((newtag->common.filter != NULL || 235 (parent->bounce_flags & BF_COULD_BOUNCE) != 0)) 236 newtag->bounce_flags |= BF_COULD_BOUNCE; 237 238 /* Copy some flags from the parent */ 239 newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT; 240 } 241 242 if ((newtag->bounce_flags & BF_COHERENT) != 0) { 243 newtag->alloc_alignment = newtag->common.alignment; 244 newtag->alloc_size = newtag->common.maxsize; 245 } else { 246 /* 247 * Ensure the buffer is aligned to a cacheline when allocating 248 * a non-coherent buffer. This is so we don't have any data 249 * that another CPU may be accessing around DMA buffer 250 * causing the cache to become dirty. 251 */ 252 newtag->alloc_alignment = MAX(newtag->common.alignment, 253 dcache_line_size); 254 newtag->alloc_size = roundup2(newtag->common.maxsize, 255 dcache_line_size); 256 } 257 258 if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) || 259 newtag->common.alignment > 1) 260 newtag->bounce_flags |= BF_COULD_BOUNCE; 261 262 if ((flags & BUS_DMA_ALLOCNOW) != 0) { 263 struct bounce_zone *bz; 264 /* 265 * Round size up to a full page, and add one more page because 266 * there can always be one more boundary crossing than the 267 * number of pages in a transfer. 268 */ 269 maxsize = roundup2(maxsize, PAGE_SIZE) + PAGE_SIZE; 270 271 /* Must bounce */ 272 if ((error = alloc_bounce_zone(newtag)) != 0) { 273 free(newtag, M_DEVBUF); 274 return (error); 275 } 276 bz = newtag->bounce_zone; 277 278 if (ptoa(bz->total_bpages) < maxsize) { 279 int pages; 280 281 pages = atop(maxsize) + 1 - bz->total_bpages; 282 283 /* Add pages to our bounce pool */ 284 if (alloc_bounce_pages(newtag, pages) < pages) 285 error = ENOMEM; 286 } 287 /* Performed initial allocation */ 288 newtag->bounce_flags |= BF_MIN_ALLOC_COMP; 289 } else 290 error = 0; 291 292 if (error != 0) 293 free(newtag, M_DEVBUF); 294 else 295 *dmat = newtag; 296 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 297 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), 298 error); 299 return (error); 300 } 301 302 static int 303 bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) 304 { 305 #ifdef KTR 306 bus_dma_tag_t dmat_copy; 307 #endif 308 bus_dma_tag_t parent; 309 int error; 310 311 error = 0; 312 #ifdef KTR 313 dmat_copy = dmat; 314 #endif 315 316 317 if (dmat != NULL) { 318 if (dmat->map_count != 0) { 319 error = EBUSY; 320 goto out; 321 } 322 while (dmat != NULL) { 323 parent = (bus_dma_tag_t)dmat->common.parent; 324 atomic_subtract_int(&dmat->common.ref_count, 1); 325 if (dmat->common.ref_count == 0) { 326 if (dmat->segments != NULL) 327 free(dmat->segments, M_DEVBUF); 328 free(dmat, M_DEVBUF); 329 /* 330 * Last reference count, so 331 * release our reference 332 * count on our parent. 333 */ 334 dmat = parent; 335 } else 336 dmat = NULL; 337 } 338 } 339 out: 340 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat_copy, error); 341 return (error); 342 } 343 344 static bool 345 bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) 346 { 347 348 if (!might_bounce(dmat, NULL, buf, buflen)) 349 return (true); 350 return (!_bus_dmamap_pagesneeded(dmat, NULL, buf, buflen, NULL)); 351 } 352 353 static bus_dmamap_t 354 alloc_dmamap(bus_dma_tag_t dmat, int flags) 355 { 356 u_long mapsize; 357 bus_dmamap_t map; 358 359 mapsize = sizeof(*map); 360 mapsize += sizeof(struct sync_list) * dmat->common.nsegments; 361 map = malloc(mapsize, M_DEVBUF, flags | M_ZERO); 362 if (map == NULL) 363 return (NULL); 364 365 /* Initialize the new map */ 366 STAILQ_INIT(&map->bpages); 367 368 return (map); 369 } 370 371 /* 372 * Allocate a handle for mapping from kva/uva/physical 373 * address space into bus device space. 374 */ 375 static int 376 bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 377 { 378 struct bounce_zone *bz; 379 int error, maxpages, pages; 380 381 error = 0; 382 383 if (dmat->segments == NULL) { 384 dmat->segments = (bus_dma_segment_t *)malloc( 385 sizeof(bus_dma_segment_t) * dmat->common.nsegments, 386 M_DEVBUF, M_NOWAIT); 387 if (dmat->segments == NULL) { 388 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 389 __func__, dmat, ENOMEM); 390 return (ENOMEM); 391 } 392 } 393 394 *mapp = alloc_dmamap(dmat, M_NOWAIT); 395 if (*mapp == NULL) { 396 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 397 __func__, dmat, ENOMEM); 398 return (ENOMEM); 399 } 400 401 /* 402 * Bouncing might be required if the driver asks for an active 403 * exclusion region, a data alignment that is stricter than 1, and/or 404 * an active address boundary. 405 */ 406 if (dmat->bounce_zone == NULL) { 407 if ((error = alloc_bounce_zone(dmat)) != 0) { 408 free(*mapp, M_DEVBUF); 409 return (error); 410 } 411 } 412 bz = dmat->bounce_zone; 413 414 /* 415 * Attempt to add pages to our pool on a per-instance basis up to a sane 416 * limit. Even if the tag isn't subject of bouncing due to alignment 417 * and boundary constraints, it could still auto-bounce due to 418 * cacheline alignment, which requires at most two bounce pages. 419 */ 420 if (dmat->common.alignment > 1) 421 maxpages = MAX_BPAGES; 422 else 423 maxpages = MIN(MAX_BPAGES, Maxmem - 424 atop(dmat->common.lowaddr)); 425 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 || 426 (bz->map_count > 0 && bz->total_bpages < maxpages)) { 427 pages = atop(roundup2(dmat->common.maxsize, PAGE_SIZE)) + 1; 428 pages = MIN(maxpages - bz->total_bpages, pages); 429 pages = MAX(pages, 2); 430 if (alloc_bounce_pages(dmat, pages) < pages) 431 error = ENOMEM; 432 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) { 433 if (error == 0) { 434 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 435 } 436 } else 437 error = 0; 438 } 439 bz->map_count++; 440 441 if (error == 0) { 442 dmat->map_count++; 443 if ((dmat->bounce_flags & BF_COHERENT) != 0) 444 (*mapp)->flags |= DMAMAP_COHERENT; 445 } else { 446 free(*mapp, M_DEVBUF); 447 } 448 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 449 __func__, dmat, dmat->common.flags, error); 450 return (error); 451 } 452 453 /* 454 * Destroy a handle for mapping from kva/uva/physical 455 * address space into bus device space. 456 */ 457 static int 458 bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 459 { 460 461 /* Check we are destroying the correct map type */ 462 if ((map->flags & DMAMAP_FROM_DMAMEM) != 0) 463 panic("bounce_bus_dmamap_destroy: Invalid map freed\n"); 464 465 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 466 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); 467 return (EBUSY); 468 } 469 if (dmat->bounce_zone) 470 dmat->bounce_zone->map_count--; 471 free(map, M_DEVBUF); 472 dmat->map_count--; 473 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 474 return (0); 475 } 476 477 /* 478 * Allocate a piece of memory that can be efficiently mapped into 479 * bus device space based on the constraints lited in the dma tag. 480 * A dmamap to for use with dmamap_load is also allocated. 481 */ 482 static int 483 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 484 bus_dmamap_t *mapp) 485 { 486 vm_memattr_t attr; 487 int mflags; 488 489 if (flags & BUS_DMA_NOWAIT) 490 mflags = M_NOWAIT; 491 else 492 mflags = M_WAITOK; 493 494 if (dmat->segments == NULL) { 495 dmat->segments = (bus_dma_segment_t *)malloc( 496 sizeof(bus_dma_segment_t) * dmat->common.nsegments, 497 M_DEVBUF, mflags); 498 if (dmat->segments == NULL) { 499 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 500 __func__, dmat, dmat->common.flags, ENOMEM); 501 return (ENOMEM); 502 } 503 } 504 if (flags & BUS_DMA_ZERO) 505 mflags |= M_ZERO; 506 if (flags & BUS_DMA_NOCACHE) 507 attr = VM_MEMATTR_UNCACHEABLE; 508 else if ((flags & BUS_DMA_COHERENT) != 0 && 509 (dmat->bounce_flags & BF_COHERENT) == 0) 510 /* 511 * If we have a non-coherent tag, and are trying to allocate 512 * a coherent block of memory it needs to be uncached. 513 */ 514 attr = VM_MEMATTR_UNCACHEABLE; 515 else 516 attr = VM_MEMATTR_DEFAULT; 517 518 /* 519 * Create the map, but don't set the could bounce flag as 520 * this allocation should never bounce; 521 */ 522 *mapp = alloc_dmamap(dmat, mflags); 523 if (*mapp == NULL) { 524 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 525 __func__, dmat, dmat->common.flags, ENOMEM); 526 return (ENOMEM); 527 } 528 529 /* 530 * Mark the map as coherent if we used uncacheable memory or the 531 * tag was already marked as coherent. 532 */ 533 if (attr == VM_MEMATTR_UNCACHEABLE || 534 (dmat->bounce_flags & BF_COHERENT) != 0) 535 (*mapp)->flags |= DMAMAP_COHERENT; 536 537 (*mapp)->flags |= DMAMAP_FROM_DMAMEM; 538 539 /* 540 * Allocate the buffer from the malloc(9) allocator if... 541 * - It's small enough to fit into a single power of two sized bucket. 542 * - The alignment is less than or equal to the maximum size 543 * - The low address requirement is fulfilled. 544 * else allocate non-contiguous pages if... 545 * - The page count that could get allocated doesn't exceed 546 * nsegments also when the maximum segment size is less 547 * than PAGE_SIZE. 548 * - The alignment constraint isn't larger than a page boundary. 549 * - There are no boundary-crossing constraints. 550 * else allocate a block of contiguous pages because one or more of the 551 * constraints is something that only the contig allocator can fulfill. 552 * 553 * NOTE: The (dmat->common.alignment <= dmat->maxsize) check 554 * below is just a quick hack. The exact alignment guarantees 555 * of malloc(9) need to be nailed down, and the code below 556 * should be rewritten to take that into account. 557 * 558 * In the meantime warn the user if malloc gets it wrong. 559 */ 560 if ((dmat->alloc_size <= PAGE_SIZE) && 561 (dmat->alloc_alignment <= dmat->alloc_size) && 562 dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && 563 attr == VM_MEMATTR_DEFAULT) { 564 *vaddr = malloc(dmat->alloc_size, M_DEVBUF, mflags); 565 } else if (dmat->common.nsegments >= 566 howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && 567 dmat->alloc_alignment <= PAGE_SIZE && 568 (dmat->common.boundary % PAGE_SIZE) == 0) { 569 /* Page-based multi-segment allocations allowed */ 570 *vaddr = kmem_alloc_attr(dmat->alloc_size, mflags, 571 0ul, dmat->common.lowaddr, attr); 572 dmat->bounce_flags |= BF_KMEM_ALLOC; 573 } else { 574 *vaddr = kmem_alloc_contig(dmat->alloc_size, mflags, 575 0ul, dmat->common.lowaddr, dmat->alloc_alignment != 0 ? 576 dmat->alloc_alignment : 1ul, dmat->common.boundary, attr); 577 dmat->bounce_flags |= BF_KMEM_ALLOC; 578 } 579 if (*vaddr == NULL) { 580 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 581 __func__, dmat, dmat->common.flags, ENOMEM); 582 free(*mapp, M_DEVBUF); 583 return (ENOMEM); 584 } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->alloc_alignment)) { 585 printf("bus_dmamem_alloc failed to align memory properly.\n"); 586 } 587 dmat->map_count++; 588 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 589 __func__, dmat, dmat->common.flags, 0); 590 return (0); 591 } 592 593 /* 594 * Free a piece of memory and it's allociated dmamap, that was allocated 595 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 596 */ 597 static void 598 bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 599 { 600 601 /* 602 * Check the map came from bounce_bus_dmamem_alloc, so the map 603 * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc() 604 * was used and set if kmem_alloc_contig() was used. 605 */ 606 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 607 panic("bus_dmamem_free: Invalid map freed\n"); 608 if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0) 609 free(vaddr, M_DEVBUF); 610 else 611 kmem_free(vaddr, dmat->alloc_size); 612 free(map, M_DEVBUF); 613 dmat->map_count--; 614 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, 615 dmat->bounce_flags); 616 } 617 618 static bool 619 _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 620 bus_size_t buflen, int *pagesneeded) 621 { 622 bus_addr_t curaddr; 623 bus_size_t sgsize; 624 int count; 625 626 /* 627 * Count the number of bounce pages needed in order to 628 * complete this transfer 629 */ 630 count = 0; 631 curaddr = buf; 632 while (buflen != 0) { 633 sgsize = MIN(buflen, dmat->common.maxsegsz); 634 if (must_bounce(dmat, map, curaddr, sgsize)) { 635 sgsize = MIN(sgsize, 636 PAGE_SIZE - (curaddr & PAGE_MASK)); 637 if (pagesneeded == NULL) 638 return (true); 639 count++; 640 } 641 curaddr += sgsize; 642 buflen -= sgsize; 643 } 644 645 if (pagesneeded != NULL) 646 *pagesneeded = count; 647 return (count != 0); 648 } 649 650 static void 651 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 652 bus_size_t buflen, int flags) 653 { 654 655 if (map->pagesneeded == 0) { 656 _bus_dmamap_pagesneeded(dmat, map, buf, buflen, 657 &map->pagesneeded); 658 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 659 } 660 } 661 662 static void 663 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 664 void *buf, bus_size_t buflen, int flags) 665 { 666 vm_offset_t vaddr; 667 vm_offset_t vendaddr; 668 bus_addr_t paddr; 669 bus_size_t sg_len; 670 671 if (map->pagesneeded == 0) { 672 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 673 "alignment= %d", dmat->common.lowaddr, 674 ptoa((vm_paddr_t)Maxmem), 675 dmat->common.boundary, dmat->common.alignment); 676 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, 677 map->pagesneeded); 678 /* 679 * Count the number of bounce pages 680 * needed in order to complete this transfer 681 */ 682 vaddr = (vm_offset_t)buf; 683 vendaddr = (vm_offset_t)buf + buflen; 684 685 while (vaddr < vendaddr) { 686 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 687 if (pmap == kernel_pmap) 688 paddr = pmap_kextract(vaddr); 689 else 690 paddr = pmap_extract(pmap, vaddr); 691 if (must_bounce(dmat, map, paddr, 692 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & 693 PAGE_MASK)))) != 0) { 694 sg_len = roundup2(sg_len, 695 dmat->common.alignment); 696 map->pagesneeded++; 697 } 698 vaddr += sg_len; 699 } 700 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 701 } 702 } 703 704 /* 705 * Add a single contiguous physical range to the segment list. 706 */ 707 static bus_size_t 708 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 709 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 710 { 711 int seg; 712 713 /* 714 * Make sure we don't cross any boundaries. 715 */ 716 if (!vm_addr_bound_ok(curaddr, sgsize, dmat->common.boundary)) 717 sgsize = roundup2(curaddr, dmat->common.boundary) - curaddr; 718 719 /* 720 * Insert chunk into a segment, coalescing with 721 * previous segment if possible. 722 */ 723 seg = *segp; 724 if (seg == -1) { 725 seg = 0; 726 segs[seg].ds_addr = curaddr; 727 segs[seg].ds_len = sgsize; 728 } else { 729 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 730 (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz && 731 vm_addr_bound_ok(segs[seg].ds_addr, 732 segs[seg].ds_len + sgsize, dmat->common.boundary)) 733 segs[seg].ds_len += sgsize; 734 else { 735 if (++seg >= dmat->common.nsegments) 736 return (0); 737 segs[seg].ds_addr = curaddr; 738 segs[seg].ds_len = sgsize; 739 } 740 } 741 *segp = seg; 742 return (sgsize); 743 } 744 745 /* 746 * Utility function to load a physical buffer. segp contains 747 * the starting segment on entrace, and the ending segment on exit. 748 */ 749 static int 750 bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 751 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, 752 int *segp) 753 { 754 struct sync_list *sl; 755 bus_size_t sgsize; 756 bus_addr_t curaddr, sl_end; 757 int error; 758 759 if (segs == NULL) 760 segs = dmat->segments; 761 762 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 763 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 764 if (map->pagesneeded != 0) { 765 error = _bus_dmamap_reserve_pages(dmat, map, flags); 766 if (error) 767 return (error); 768 } 769 } 770 771 sl = map->slist + map->sync_count - 1; 772 sl_end = 0; 773 774 while (buflen > 0) { 775 curaddr = buf; 776 sgsize = MIN(buflen, dmat->common.maxsegsz); 777 if (map->pagesneeded != 0 && 778 must_bounce(dmat, map, curaddr, sgsize)) { 779 /* 780 * The attempt to split a physically continuous buffer 781 * seems very controversial, it's unclear whether we 782 * can do this in all cases. Also, memory for bounced 783 * buffers is allocated as pages, so we cannot 784 * guarantee multipage alignment. 785 */ 786 KASSERT(dmat->common.alignment <= PAGE_SIZE, 787 ("bounced buffer cannot have alignment bigger " 788 "than PAGE_SIZE: %lu", dmat->common.alignment)); 789 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 790 curaddr = add_bounce_page(dmat, map, 0, curaddr, 791 sgsize); 792 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 793 if (map->sync_count > 0) 794 sl_end = sl->paddr + sl->datacount; 795 796 if (map->sync_count == 0 || curaddr != sl_end) { 797 if (++map->sync_count > dmat->common.nsegments) 798 break; 799 sl++; 800 sl->vaddr = 0; 801 sl->paddr = curaddr; 802 sl->pages = PHYS_TO_VM_PAGE(curaddr); 803 KASSERT(sl->pages != NULL, 804 ("%s: page at PA:0x%08lx is not in " 805 "vm_page_array", __func__, curaddr)); 806 sl->datacount = sgsize; 807 } else 808 sl->datacount += sgsize; 809 } 810 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 811 segp); 812 if (sgsize == 0) 813 break; 814 buf += sgsize; 815 buflen -= sgsize; 816 } 817 818 /* 819 * Did we fit? 820 */ 821 if (buflen != 0) { 822 bus_dmamap_unload(dmat, map); 823 return (EFBIG); /* XXX better return value here? */ 824 } 825 return (0); 826 } 827 828 /* 829 * Utility function to load a linear buffer. segp contains 830 * the starting segment on entrace, and the ending segment on exit. 831 */ 832 static int 833 bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 834 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 835 int *segp) 836 { 837 struct sync_list *sl; 838 bus_size_t sgsize; 839 bus_addr_t curaddr, sl_pend; 840 vm_offset_t kvaddr, vaddr, sl_vend; 841 int error; 842 843 KASSERT((map->flags & DMAMAP_FROM_DMAMEM) != 0 || 844 dmat->common.alignment <= PAGE_SIZE, 845 ("loading user buffer with alignment bigger than PAGE_SIZE is not " 846 "supported")); 847 848 if (segs == NULL) 849 segs = dmat->segments; 850 851 if (flags & BUS_DMA_LOAD_MBUF) 852 map->flags |= DMAMAP_MBUF; 853 854 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 855 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 856 if (map->pagesneeded != 0) { 857 error = _bus_dmamap_reserve_pages(dmat, map, flags); 858 if (error) 859 return (error); 860 } 861 } 862 863 /* 864 * XXX Optimally we should parse input buffer for physically 865 * continuous segments first and then pass these segment into 866 * load loop. 867 */ 868 sl = map->slist + map->sync_count - 1; 869 vaddr = (vm_offset_t)buf; 870 sl_pend = 0; 871 sl_vend = 0; 872 873 while (buflen > 0) { 874 /* 875 * Get the physical address for this segment. 876 */ 877 if (__predict_true(pmap == kernel_pmap)) { 878 curaddr = pmap_kextract(vaddr); 879 kvaddr = vaddr; 880 } else { 881 curaddr = pmap_extract(pmap, vaddr); 882 kvaddr = 0; 883 } 884 885 /* 886 * Compute the segment size, and adjust counts. 887 */ 888 sgsize = MIN(buflen, dmat->common.maxsegsz); 889 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 890 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 891 892 if (map->pagesneeded != 0 && 893 must_bounce(dmat, map, curaddr, sgsize)) { 894 /* See comment in bounce_bus_dmamap_load_phys */ 895 KASSERT(dmat->common.alignment <= PAGE_SIZE, 896 ("bounced buffer cannot have alignment bigger " 897 "than PAGE_SIZE: %lu", dmat->common.alignment)); 898 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 899 sgsize); 900 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 901 if (map->sync_count > 0) { 902 sl_pend = sl->paddr + sl->datacount; 903 sl_vend = sl->vaddr + sl->datacount; 904 } 905 906 if (map->sync_count == 0 || 907 (kvaddr != 0 && kvaddr != sl_vend) || 908 (curaddr != sl_pend)) { 909 if (++map->sync_count > dmat->common.nsegments) 910 break; 911 sl++; 912 sl->vaddr = kvaddr; 913 sl->paddr = curaddr; 914 if (kvaddr != 0) { 915 sl->pages = NULL; 916 } else { 917 sl->pages = PHYS_TO_VM_PAGE(curaddr); 918 KASSERT(sl->pages != NULL, 919 ("%s: page at PA:0x%08lx is not " 920 "in vm_page_array", __func__, 921 curaddr)); 922 } 923 sl->datacount = sgsize; 924 } else 925 sl->datacount += sgsize; 926 } 927 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 928 segp); 929 if (sgsize == 0) 930 break; 931 vaddr += sgsize; 932 buflen -= sgsize; 933 } 934 935 /* 936 * Did we fit? 937 */ 938 if (buflen != 0) { 939 bus_dmamap_unload(dmat, map); 940 return (EFBIG); /* XXX better return value here? */ 941 } 942 return (0); 943 } 944 945 static void 946 bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 947 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 948 { 949 950 map->mem = *mem; 951 map->dmat = dmat; 952 map->callback = callback; 953 map->callback_arg = callback_arg; 954 } 955 956 static bus_dma_segment_t * 957 bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 958 bus_dma_segment_t *segs, int nsegs, int error) 959 { 960 961 if (segs == NULL) 962 segs = dmat->segments; 963 return (segs); 964 } 965 966 /* 967 * Release the mapping held by map. 968 */ 969 static void 970 bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 971 { 972 free_bounce_pages(dmat, map); 973 map->sync_count = 0; 974 map->flags &= ~DMAMAP_MBUF; 975 } 976 977 static void 978 dma_preread_safe(vm_offset_t va, vm_size_t size) 979 { 980 /* 981 * Write back any partial cachelines immediately before and 982 * after the DMA region. 983 */ 984 if (va & (dcache_line_size - 1)) 985 cpu_dcache_wb_range(va, 1); 986 if ((va + size) & (dcache_line_size - 1)) 987 cpu_dcache_wb_range(va + size, 1); 988 989 cpu_dcache_inv_range(va, size); 990 } 991 992 static void 993 dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) 994 { 995 uint32_t len, offset; 996 vm_page_t m; 997 vm_paddr_t pa; 998 vm_offset_t va, tempva; 999 bus_size_t size; 1000 1001 offset = sl->paddr & PAGE_MASK; 1002 m = sl->pages; 1003 size = sl->datacount; 1004 pa = sl->paddr; 1005 1006 for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { 1007 tempva = 0; 1008 if (sl->vaddr == 0) { 1009 len = min(PAGE_SIZE - offset, size); 1010 tempva = pmap_quick_enter_page(m); 1011 va = tempva | offset; 1012 KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), 1013 ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx", 1014 VM_PAGE_TO_PHYS(m) | offset, pa)); 1015 } else { 1016 len = sl->datacount; 1017 va = sl->vaddr; 1018 } 1019 1020 switch (op) { 1021 case BUS_DMASYNC_PREWRITE: 1022 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1023 cpu_dcache_wb_range(va, len); 1024 break; 1025 case BUS_DMASYNC_PREREAD: 1026 /* 1027 * An mbuf may start in the middle of a cacheline. There 1028 * will be no cpu writes to the beginning of that line 1029 * (which contains the mbuf header) while dma is in 1030 * progress. Handle that case by doing a writeback of 1031 * just the first cacheline before invalidating the 1032 * overall buffer. Any mbuf in a chain may have this 1033 * misalignment. Buffers which are not mbufs bounce if 1034 * they are not aligned to a cacheline. 1035 */ 1036 dma_preread_safe(va, len); 1037 break; 1038 case BUS_DMASYNC_POSTREAD: 1039 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1040 cpu_dcache_inv_range(va, len); 1041 break; 1042 default: 1043 panic("unsupported combination of sync operations: " 1044 "0x%08x\n", op); 1045 } 1046 1047 if (tempva != 0) 1048 pmap_quick_remove_page(tempva); 1049 } 1050 } 1051 1052 static void 1053 bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, 1054 bus_dmasync_op_t op) 1055 { 1056 struct bounce_page *bpage; 1057 struct sync_list *sl, *end; 1058 vm_offset_t datavaddr, tempvaddr; 1059 1060 if (op == BUS_DMASYNC_POSTWRITE) 1061 return; 1062 1063 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1064 /* 1065 * Wait for any DMA operations to complete before the bcopy. 1066 */ 1067 dsb(sy); 1068 } 1069 1070 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1071 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1072 "performing bounce", __func__, dmat, dmat->common.flags, 1073 op); 1074 1075 if ((op & BUS_DMASYNC_PREWRITE) != 0) { 1076 while (bpage != NULL) { 1077 tempvaddr = 0; 1078 datavaddr = bpage->datavaddr; 1079 if (datavaddr == 0) { 1080 tempvaddr = pmap_quick_enter_page( 1081 bpage->datapage); 1082 datavaddr = tempvaddr | bpage->dataoffs; 1083 } 1084 1085 bcopy((void *)datavaddr, 1086 (void *)bpage->vaddr, bpage->datacount); 1087 if (tempvaddr != 0) 1088 pmap_quick_remove_page(tempvaddr); 1089 if ((map->flags & DMAMAP_COHERENT) == 0) 1090 cpu_dcache_wb_range(bpage->vaddr, 1091 bpage->datacount); 1092 bpage = STAILQ_NEXT(bpage, links); 1093 } 1094 dmat->bounce_zone->total_bounced++; 1095 } else if ((op & BUS_DMASYNC_PREREAD) != 0) { 1096 while (bpage != NULL) { 1097 if ((map->flags & DMAMAP_COHERENT) == 0) 1098 cpu_dcache_wbinv_range(bpage->vaddr, 1099 bpage->datacount); 1100 bpage = STAILQ_NEXT(bpage, links); 1101 } 1102 } 1103 1104 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1105 while (bpage != NULL) { 1106 if ((map->flags & DMAMAP_COHERENT) == 0) 1107 cpu_dcache_inv_range(bpage->vaddr, 1108 bpage->datacount); 1109 tempvaddr = 0; 1110 datavaddr = bpage->datavaddr; 1111 if (datavaddr == 0) { 1112 tempvaddr = pmap_quick_enter_page( 1113 bpage->datapage); 1114 datavaddr = tempvaddr | bpage->dataoffs; 1115 } 1116 1117 bcopy((void *)bpage->vaddr, 1118 (void *)datavaddr, bpage->datacount); 1119 1120 if (tempvaddr != 0) 1121 pmap_quick_remove_page(tempvaddr); 1122 bpage = STAILQ_NEXT(bpage, links); 1123 } 1124 dmat->bounce_zone->total_bounced++; 1125 } 1126 } 1127 1128 /* 1129 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. 1130 */ 1131 if (map->sync_count != 0) { 1132 sl = &map->slist[0]; 1133 end = &map->slist[map->sync_count]; 1134 CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x " 1135 "performing sync", __func__, dmat, op); 1136 1137 for ( ; sl != end; ++sl) 1138 dma_dcache_sync(sl, op); 1139 } 1140 1141 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) { 1142 /* 1143 * Wait for the bcopy to complete before any DMA operations. 1144 */ 1145 dsb(sy); 1146 } 1147 } 1148 1149 struct bus_dma_impl bus_dma_bounce_impl = { 1150 .tag_create = bounce_bus_dma_tag_create, 1151 .tag_destroy = bounce_bus_dma_tag_destroy, 1152 .id_mapped = bounce_bus_dma_id_mapped, 1153 .map_create = bounce_bus_dmamap_create, 1154 .map_destroy = bounce_bus_dmamap_destroy, 1155 .mem_alloc = bounce_bus_dmamem_alloc, 1156 .mem_free = bounce_bus_dmamem_free, 1157 .load_phys = bounce_bus_dmamap_load_phys, 1158 .load_buffer = bounce_bus_dmamap_load_buffer, 1159 .load_ma = bus_dmamap_load_ma_triv, 1160 .map_waitok = bounce_bus_dmamap_waitok, 1161 .map_complete = bounce_bus_dmamap_complete, 1162 .map_unload = bounce_bus_dmamap_unload, 1163 .map_sync = bounce_bus_dmamap_sync 1164 }; 1165