1 /*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * Copyright (c) 2015-2016 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship of the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by Semihalf 10 * under sponsorship of the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions, and the following disclaimer, 17 * without modification, immediately at the beginning of the file. 18 * 2. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 25 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/domainset.h> 37 #include <sys/malloc.h> 38 #include <sys/bus.h> 39 #include <sys/interrupt.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/lock.h> 43 #include <sys/proc.h> 44 #include <sys/memdesc.h> 45 #include <sys/mutex.h> 46 #include <sys/sysctl.h> 47 #include <sys/uio.h> 48 49 #include <vm/vm.h> 50 #include <vm/vm_extern.h> 51 #include <vm/vm_kern.h> 52 #include <vm/vm_page.h> 53 #include <vm/vm_map.h> 54 55 #include <machine/atomic.h> 56 #include <machine/bus.h> 57 #include <machine/md_var.h> 58 #include <arm64/include/bus_dma_impl.h> 59 60 #define MAX_BPAGES 4096 61 62 enum { 63 BF_COULD_BOUNCE = 0x01, 64 BF_MIN_ALLOC_COMP = 0x02, 65 BF_KMEM_ALLOC = 0x04, 66 BF_COHERENT = 0x10, 67 }; 68 69 struct bounce_page; 70 struct bounce_zone; 71 72 struct bus_dma_tag { 73 struct bus_dma_tag_common common; 74 size_t alloc_size; 75 size_t alloc_alignment; 76 int map_count; 77 int bounce_flags; 78 bus_dma_segment_t *segments; 79 struct bounce_zone *bounce_zone; 80 }; 81 82 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 83 "Busdma parameters"); 84 85 struct sync_list { 86 vm_offset_t vaddr; /* kva of client data */ 87 bus_addr_t paddr; /* physical address */ 88 vm_page_t pages; /* starting page of client data */ 89 bus_size_t datacount; /* client data count */ 90 }; 91 92 struct bus_dmamap { 93 STAILQ_HEAD(, bounce_page) bpages; 94 int pagesneeded; 95 int pagesreserved; 96 bus_dma_tag_t dmat; 97 struct memdesc mem; 98 bus_dmamap_callback_t *callback; 99 void *callback_arg; 100 __sbintime_t queued_time; 101 STAILQ_ENTRY(bus_dmamap) links; 102 u_int flags; 103 #define DMAMAP_COHERENT (1 << 0) 104 #define DMAMAP_FROM_DMAMEM (1 << 1) 105 #define DMAMAP_MBUF (1 << 2) 106 int sync_count; 107 struct sync_list slist[]; 108 }; 109 110 static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, 111 vm_paddr_t buf, bus_size_t buflen, int *pagesneeded); 112 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 113 pmap_t pmap, void *buf, bus_size_t buflen, int flags); 114 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 115 vm_paddr_t buf, bus_size_t buflen, int flags); 116 117 static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); 118 119 #define dmat_alignment(dmat) ((dmat)->common.alignment) 120 #define dmat_domain(dmat) ((dmat)->common.domain) 121 #define dmat_flags(dmat) ((dmat)->common.flags) 122 #define dmat_highaddr(dmat) ((dmat)->common.highaddr) 123 #define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) 124 #define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) 125 #define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) 126 127 #include "../../kern/subr_busdma_bounce.c" 128 129 static int 130 bounce_bus_dma_zone_setup(bus_dma_tag_t dmat) 131 { 132 struct bounce_zone *bz; 133 bus_size_t maxsize; 134 int error; 135 136 /* 137 * Round size up to a full page, and add one more page because 138 * there can always be one more boundary crossing than the 139 * number of pages in a transfer. 140 */ 141 maxsize = roundup2(dmat->common.maxsize, PAGE_SIZE) + PAGE_SIZE; 142 143 /* Must bounce */ 144 if ((error = alloc_bounce_zone(dmat)) != 0) 145 return (error); 146 bz = dmat->bounce_zone; 147 148 if (ptoa(bz->total_bpages) < maxsize) { 149 int pages; 150 151 pages = atop(maxsize) + 1 - bz->total_bpages; 152 153 /* Add pages to our bounce pool */ 154 if (alloc_bounce_pages(dmat, pages) < pages) 155 return (ENOMEM); 156 } 157 /* Performed initial allocation */ 158 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 159 160 return (error); 161 } 162 163 /* 164 * Return true if the DMA should bounce because the start or end does not fall 165 * on a cacheline boundary (which would require a partial cacheline flush). 166 * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by 167 * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a 168 * strict rule that such memory cannot be accessed by the CPU while DMA is in 169 * progress (or by multiple DMA engines at once), so that it's always safe to do 170 * full cacheline flushes even if that affects memory outside the range of a 171 * given DMA operation that doesn't involve the full allocated buffer. If we're 172 * mapping an mbuf, that follows the same rules as a buffer we allocated. 173 */ 174 static bool 175 cacheline_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 176 bus_size_t size) 177 { 178 179 #define DMAMAP_CACHELINE_FLAGS \ 180 (DMAMAP_FROM_DMAMEM | DMAMAP_COHERENT | DMAMAP_MBUF) 181 if ((dmat->bounce_flags & BF_COHERENT) != 0) 182 return (false); 183 if (map != NULL && (map->flags & DMAMAP_CACHELINE_FLAGS) != 0) 184 return (false); 185 return (((paddr | size) & (dcache_line_size - 1)) != 0); 186 #undef DMAMAP_CACHELINE_FLAGS 187 } 188 189 /* 190 * Return true if the given address does not fall on the alignment boundary. 191 */ 192 static bool 193 alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) 194 { 195 196 return (!vm_addr_align_ok(addr, dmat->common.alignment)); 197 } 198 199 static bool 200 might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 201 bus_size_t size) 202 { 203 204 /* Memory allocated by bounce_bus_dmamem_alloc won't bounce */ 205 if (map && (map->flags & DMAMAP_FROM_DMAMEM) != 0) 206 return (false); 207 208 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) 209 return (true); 210 211 if (cacheline_bounce(dmat, map, paddr, size)) 212 return (true); 213 214 if (alignment_bounce(dmat, paddr)) 215 return (true); 216 217 return (false); 218 } 219 220 static bool 221 must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 222 bus_size_t size) 223 { 224 225 if (cacheline_bounce(dmat, map, paddr, size)) 226 return (true); 227 228 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 && 229 addr_needs_bounce(dmat, paddr)) 230 return (true); 231 232 return (false); 233 } 234 235 /* 236 * Allocate a device specific dma_tag. 237 */ 238 static int 239 bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 240 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 241 bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, 242 bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) 243 { 244 bus_dma_tag_t newtag; 245 int error; 246 247 *dmat = NULL; 248 error = common_bus_dma_tag_create(parent != NULL ? &parent->common : 249 NULL, alignment, boundary, lowaddr, highaddr, maxsize, nsegments, 250 maxsegsz, flags, lockfunc, lockfuncarg, 251 sizeof (struct bus_dma_tag), (void **)&newtag); 252 if (error != 0) 253 return (error); 254 255 newtag->common.impl = &bus_dma_bounce_impl; 256 newtag->map_count = 0; 257 newtag->segments = NULL; 258 259 if ((flags & BUS_DMA_COHERENT) != 0) { 260 newtag->bounce_flags |= BF_COHERENT; 261 } 262 263 if (parent != NULL) { 264 if ((parent->bounce_flags & BF_COULD_BOUNCE) != 0) 265 newtag->bounce_flags |= BF_COULD_BOUNCE; 266 267 /* Copy some flags from the parent */ 268 newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT; 269 } 270 271 if ((newtag->bounce_flags & BF_COHERENT) != 0) { 272 newtag->alloc_alignment = newtag->common.alignment; 273 newtag->alloc_size = newtag->common.maxsize; 274 } else { 275 /* 276 * Ensure the buffer is aligned to a cacheline when allocating 277 * a non-coherent buffer. This is so we don't have any data 278 * that another CPU may be accessing around DMA buffer 279 * causing the cache to become dirty. 280 */ 281 newtag->alloc_alignment = MAX(newtag->common.alignment, 282 dcache_line_size); 283 newtag->alloc_size = roundup2(newtag->common.maxsize, 284 dcache_line_size); 285 } 286 287 if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) || 288 newtag->common.alignment > 1) 289 newtag->bounce_flags |= BF_COULD_BOUNCE; 290 291 if ((flags & BUS_DMA_ALLOCNOW) != 0) 292 error = bounce_bus_dma_zone_setup(newtag); 293 else 294 error = 0; 295 296 if (error != 0) 297 free(newtag, M_DEVBUF); 298 else 299 *dmat = newtag; 300 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 301 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), 302 error); 303 return (error); 304 } 305 306 static int 307 bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) 308 { 309 int error = 0; 310 311 if (dmat != NULL) { 312 if (dmat->map_count != 0) { 313 error = EBUSY; 314 goto out; 315 } 316 if (dmat->segments != NULL) 317 free(dmat->segments, M_DEVBUF); 318 free(dmat, M_DEVBUF); 319 } 320 out: 321 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error); 322 return (error); 323 } 324 325 /* 326 * Update the domain for the tag. We may need to reallocate the zone and 327 * bounce pages. 328 */ 329 static int 330 bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat) 331 { 332 333 KASSERT(dmat->map_count == 0, 334 ("bounce_bus_dma_tag_set_domain: Domain set after use.\n")); 335 if ((dmat->bounce_flags & BF_COULD_BOUNCE) == 0 || 336 dmat->bounce_zone == NULL) 337 return (0); 338 dmat->bounce_flags &= ~BF_MIN_ALLOC_COMP; 339 return (bounce_bus_dma_zone_setup(dmat)); 340 } 341 342 static bool 343 bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) 344 { 345 346 if (!might_bounce(dmat, NULL, buf, buflen)) 347 return (true); 348 return (!_bus_dmamap_pagesneeded(dmat, NULL, buf, buflen, NULL)); 349 } 350 351 static bus_dmamap_t 352 alloc_dmamap(bus_dma_tag_t dmat, int flags) 353 { 354 u_long mapsize; 355 bus_dmamap_t map; 356 357 mapsize = sizeof(*map); 358 mapsize += sizeof(struct sync_list) * dmat->common.nsegments; 359 map = malloc_domainset(mapsize, M_DEVBUF, 360 DOMAINSET_PREF(dmat->common.domain), flags | M_ZERO); 361 if (map == NULL) 362 return (NULL); 363 364 /* Initialize the new map */ 365 STAILQ_INIT(&map->bpages); 366 367 return (map); 368 } 369 370 /* 371 * Allocate a handle for mapping from kva/uva/physical 372 * address space into bus device space. 373 */ 374 static int 375 bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 376 { 377 struct bounce_zone *bz; 378 int error, maxpages, pages; 379 380 error = 0; 381 382 if (dmat->segments == NULL) { 383 dmat->segments = mallocarray_domainset(dmat->common.nsegments, 384 sizeof(bus_dma_segment_t), M_DEVBUF, 385 DOMAINSET_PREF(dmat->common.domain), M_NOWAIT); 386 if (dmat->segments == NULL) { 387 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 388 __func__, dmat, ENOMEM); 389 return (ENOMEM); 390 } 391 } 392 393 *mapp = alloc_dmamap(dmat, M_NOWAIT); 394 if (*mapp == NULL) { 395 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 396 __func__, dmat, ENOMEM); 397 return (ENOMEM); 398 } 399 400 /* 401 * Bouncing might be required if the driver asks for an active 402 * exclusion region, a data alignment that is stricter than 1, and/or 403 * an active address boundary. 404 */ 405 if (dmat->bounce_zone == NULL) { 406 if ((error = alloc_bounce_zone(dmat)) != 0) { 407 free(*mapp, M_DEVBUF); 408 return (error); 409 } 410 } 411 bz = dmat->bounce_zone; 412 413 /* 414 * Attempt to add pages to our pool on a per-instance basis up to a sane 415 * limit. Even if the tag isn't subject of bouncing due to alignment 416 * and boundary constraints, it could still auto-bounce due to 417 * cacheline alignment, which requires at most two bounce pages. 418 */ 419 if (dmat->common.alignment > 1) 420 maxpages = MAX_BPAGES; 421 else 422 maxpages = MIN(MAX_BPAGES, Maxmem - 423 atop(dmat->common.lowaddr)); 424 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 || 425 (bz->map_count > 0 && bz->total_bpages < maxpages)) { 426 pages = atop(roundup2(dmat->common.maxsize, PAGE_SIZE)) + 1; 427 pages = MIN(maxpages - bz->total_bpages, pages); 428 pages = MAX(pages, 2); 429 if (alloc_bounce_pages(dmat, pages) < pages) 430 error = ENOMEM; 431 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) { 432 if (error == 0) { 433 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 434 } 435 } else 436 error = 0; 437 } 438 bz->map_count++; 439 440 if (error == 0) { 441 dmat->map_count++; 442 if ((dmat->bounce_flags & BF_COHERENT) != 0) 443 (*mapp)->flags |= DMAMAP_COHERENT; 444 } else { 445 free(*mapp, M_DEVBUF); 446 } 447 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 448 __func__, dmat, dmat->common.flags, error); 449 return (error); 450 } 451 452 /* 453 * Destroy a handle for mapping from kva/uva/physical 454 * address space into bus device space. 455 */ 456 static int 457 bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 458 { 459 460 /* Check we are destroying the correct map type */ 461 if ((map->flags & DMAMAP_FROM_DMAMEM) != 0) 462 panic("bounce_bus_dmamap_destroy: Invalid map freed\n"); 463 464 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 465 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); 466 return (EBUSY); 467 } 468 if (dmat->bounce_zone) 469 dmat->bounce_zone->map_count--; 470 free(map, M_DEVBUF); 471 dmat->map_count--; 472 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 473 return (0); 474 } 475 476 /* 477 * Allocate a piece of memory that can be efficiently mapped into 478 * bus device space based on the constraints lited in the dma tag. 479 * A dmamap to for use with dmamap_load is also allocated. 480 */ 481 static int 482 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 483 bus_dmamap_t *mapp) 484 { 485 vm_memattr_t attr; 486 int mflags; 487 488 if (flags & BUS_DMA_NOWAIT) 489 mflags = M_NOWAIT; 490 else 491 mflags = M_WAITOK; 492 493 if (dmat->segments == NULL) { 494 dmat->segments = mallocarray_domainset(dmat->common.nsegments, 495 sizeof(bus_dma_segment_t), M_DEVBUF, 496 DOMAINSET_PREF(dmat->common.domain), mflags); 497 if (dmat->segments == NULL) { 498 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 499 __func__, dmat, dmat->common.flags, ENOMEM); 500 return (ENOMEM); 501 } 502 } 503 if (flags & BUS_DMA_ZERO) 504 mflags |= M_ZERO; 505 if (flags & BUS_DMA_NOCACHE) 506 attr = VM_MEMATTR_UNCACHEABLE; 507 else if ((flags & BUS_DMA_COHERENT) != 0 && 508 (dmat->bounce_flags & BF_COHERENT) == 0) 509 /* 510 * If we have a non-coherent tag, and are trying to allocate 511 * a coherent block of memory it needs to be uncached. 512 */ 513 attr = VM_MEMATTR_UNCACHEABLE; 514 else 515 attr = VM_MEMATTR_DEFAULT; 516 517 /* 518 * Create the map, but don't set the could bounce flag as 519 * this allocation should never bounce; 520 */ 521 *mapp = alloc_dmamap(dmat, mflags); 522 if (*mapp == NULL) { 523 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 524 __func__, dmat, dmat->common.flags, ENOMEM); 525 return (ENOMEM); 526 } 527 528 /* 529 * Mark the map as coherent if we used uncacheable memory or the 530 * tag was already marked as coherent. 531 */ 532 if (attr == VM_MEMATTR_UNCACHEABLE || 533 (dmat->bounce_flags & BF_COHERENT) != 0) 534 (*mapp)->flags |= DMAMAP_COHERENT; 535 536 (*mapp)->flags |= DMAMAP_FROM_DMAMEM; 537 538 /* 539 * Allocate the buffer from the malloc(9) allocator if... 540 * - It's small enough to fit into a single page. 541 * - Its alignment requirement is also smaller than the page size. 542 * - The low address requirement is fulfilled. 543 * - Default cache attributes are requested (WB). 544 * else allocate non-contiguous pages if... 545 * - The page count that could get allocated doesn't exceed 546 * nsegments also when the maximum segment size is less 547 * than PAGE_SIZE. 548 * - The alignment constraint isn't larger than a page boundary. 549 * - There are no boundary-crossing constraints. 550 * else allocate a block of contiguous pages because one or more of the 551 * constraints is something that only the contig allocator can fulfill. 552 * 553 * NOTE: The (dmat->common.alignment <= dmat->maxsize) check 554 * below is just a quick hack. The exact alignment guarantees 555 * of malloc(9) need to be nailed down, and the code below 556 * should be rewritten to take that into account. 557 * 558 * In the meantime warn the user if malloc gets it wrong. 559 */ 560 if (dmat->alloc_size <= PAGE_SIZE && 561 dmat->alloc_alignment <= PAGE_SIZE && 562 dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && 563 attr == VM_MEMATTR_DEFAULT) { 564 *vaddr = malloc_domainset_aligned(dmat->alloc_size, 565 dmat->alloc_alignment, M_DEVBUF, 566 DOMAINSET_PREF(dmat->common.domain), mflags); 567 } else if (dmat->common.nsegments >= 568 howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && 569 dmat->alloc_alignment <= PAGE_SIZE && 570 (dmat->common.boundary % PAGE_SIZE) == 0) { 571 /* Page-based multi-segment allocations allowed */ 572 *vaddr = kmem_alloc_attr_domainset( 573 DOMAINSET_PREF(dmat->common.domain), dmat->alloc_size, 574 mflags, 0ul, dmat->common.lowaddr, attr); 575 dmat->bounce_flags |= BF_KMEM_ALLOC; 576 } else { 577 *vaddr = kmem_alloc_contig_domainset( 578 DOMAINSET_PREF(dmat->common.domain), dmat->alloc_size, 579 mflags, 0ul, dmat->common.lowaddr, 580 dmat->alloc_alignment != 0 ? dmat->alloc_alignment : 1ul, 581 dmat->common.boundary, attr); 582 dmat->bounce_flags |= BF_KMEM_ALLOC; 583 } 584 if (*vaddr == NULL) { 585 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 586 __func__, dmat, dmat->common.flags, ENOMEM); 587 free(*mapp, M_DEVBUF); 588 return (ENOMEM); 589 } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->alloc_alignment)) { 590 printf("bus_dmamem_alloc failed to align memory properly.\n"); 591 } 592 dmat->map_count++; 593 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 594 __func__, dmat, dmat->common.flags, 0); 595 return (0); 596 } 597 598 /* 599 * Free a piece of memory and it's allociated dmamap, that was allocated 600 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 601 */ 602 static void 603 bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 604 { 605 606 /* 607 * Check the map came from bounce_bus_dmamem_alloc, so the map 608 * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc() 609 * was used and set if kmem_alloc_contig() was used. 610 */ 611 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 612 panic("bus_dmamem_free: Invalid map freed\n"); 613 if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0) 614 free(vaddr, M_DEVBUF); 615 else 616 kmem_free(vaddr, dmat->alloc_size); 617 free(map, M_DEVBUF); 618 dmat->map_count--; 619 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, 620 dmat->bounce_flags); 621 } 622 623 static bool 624 _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 625 bus_size_t buflen, int *pagesneeded) 626 { 627 bus_addr_t curaddr; 628 bus_size_t sgsize; 629 int count; 630 631 /* 632 * Count the number of bounce pages needed in order to 633 * complete this transfer 634 */ 635 count = 0; 636 curaddr = buf; 637 while (buflen != 0) { 638 sgsize = MIN(buflen, dmat->common.maxsegsz); 639 if (must_bounce(dmat, map, curaddr, sgsize)) { 640 sgsize = MIN(sgsize, 641 PAGE_SIZE - (curaddr & PAGE_MASK)); 642 if (pagesneeded == NULL) 643 return (true); 644 count++; 645 } 646 curaddr += sgsize; 647 buflen -= sgsize; 648 } 649 650 if (pagesneeded != NULL) 651 *pagesneeded = count; 652 return (count != 0); 653 } 654 655 static void 656 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 657 bus_size_t buflen, int flags) 658 { 659 660 if (map->pagesneeded == 0) { 661 _bus_dmamap_pagesneeded(dmat, map, buf, buflen, 662 &map->pagesneeded); 663 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 664 } 665 } 666 667 static void 668 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 669 void *buf, bus_size_t buflen, int flags) 670 { 671 vm_offset_t vaddr; 672 vm_offset_t vendaddr; 673 bus_addr_t paddr; 674 bus_size_t sg_len; 675 676 if (map->pagesneeded == 0) { 677 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 678 "alignment= %d", dmat->common.lowaddr, 679 ptoa((vm_paddr_t)Maxmem), 680 dmat->common.boundary, dmat->common.alignment); 681 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, 682 map->pagesneeded); 683 /* 684 * Count the number of bounce pages 685 * needed in order to complete this transfer 686 */ 687 vaddr = (vm_offset_t)buf; 688 vendaddr = (vm_offset_t)buf + buflen; 689 690 while (vaddr < vendaddr) { 691 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 692 if (pmap == kernel_pmap) 693 paddr = pmap_kextract(vaddr); 694 else 695 paddr = pmap_extract(pmap, vaddr); 696 if (must_bounce(dmat, map, paddr, 697 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & 698 PAGE_MASK)))) != 0) { 699 sg_len = roundup2(sg_len, 700 dmat->common.alignment); 701 map->pagesneeded++; 702 } 703 vaddr += sg_len; 704 } 705 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 706 } 707 } 708 709 /* 710 * Add a single contiguous physical range to the segment list. 711 */ 712 static bus_size_t 713 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 714 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 715 { 716 int seg; 717 718 /* 719 * Make sure we don't cross any boundaries. 720 */ 721 if (!vm_addr_bound_ok(curaddr, sgsize, dmat->common.boundary)) 722 sgsize = roundup2(curaddr, dmat->common.boundary) - curaddr; 723 724 /* 725 * Insert chunk into a segment, coalescing with 726 * previous segment if possible. 727 */ 728 seg = *segp; 729 if (seg == -1) { 730 seg = 0; 731 segs[seg].ds_addr = curaddr; 732 segs[seg].ds_len = sgsize; 733 } else { 734 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 735 (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz && 736 vm_addr_bound_ok(segs[seg].ds_addr, 737 segs[seg].ds_len + sgsize, dmat->common.boundary)) 738 segs[seg].ds_len += sgsize; 739 else { 740 if (++seg >= dmat->common.nsegments) 741 return (0); 742 segs[seg].ds_addr = curaddr; 743 segs[seg].ds_len = sgsize; 744 } 745 } 746 *segp = seg; 747 return (sgsize); 748 } 749 750 /* 751 * Utility function to load a physical buffer. segp contains 752 * the starting segment on entrace, and the ending segment on exit. 753 */ 754 static int 755 bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 756 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, 757 int *segp) 758 { 759 struct sync_list *sl; 760 bus_size_t sgsize; 761 bus_addr_t curaddr, sl_end; 762 int error; 763 764 if (segs == NULL) 765 segs = dmat->segments; 766 767 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 768 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 769 if (map->pagesneeded != 0) { 770 error = _bus_dmamap_reserve_pages(dmat, map, flags); 771 if (error) 772 return (error); 773 } 774 } 775 776 sl = map->slist + map->sync_count - 1; 777 sl_end = 0; 778 779 while (buflen > 0) { 780 curaddr = buf; 781 sgsize = MIN(buflen, dmat->common.maxsegsz); 782 if (map->pagesneeded != 0 && 783 must_bounce(dmat, map, curaddr, sgsize)) { 784 /* 785 * The attempt to split a physically continuous buffer 786 * seems very controversial, it's unclear whether we 787 * can do this in all cases. Also, memory for bounced 788 * buffers is allocated as pages, so we cannot 789 * guarantee multipage alignment. 790 */ 791 KASSERT(dmat->common.alignment <= PAGE_SIZE, 792 ("bounced buffer cannot have alignment bigger " 793 "than PAGE_SIZE: %lu", dmat->common.alignment)); 794 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 795 curaddr = add_bounce_page(dmat, map, 0, curaddr, 796 sgsize); 797 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 798 if (map->sync_count > 0) 799 sl_end = sl->paddr + sl->datacount; 800 801 if (map->sync_count == 0 || curaddr != sl_end) { 802 if (++map->sync_count > dmat->common.nsegments) 803 break; 804 sl++; 805 sl->vaddr = 0; 806 sl->paddr = curaddr; 807 sl->pages = PHYS_TO_VM_PAGE(curaddr); 808 KASSERT(sl->pages != NULL, 809 ("%s: page at PA:0x%08lx is not in " 810 "vm_page_array", __func__, curaddr)); 811 sl->datacount = sgsize; 812 } else 813 sl->datacount += sgsize; 814 } 815 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 816 segp); 817 if (sgsize == 0) 818 break; 819 buf += sgsize; 820 buflen -= sgsize; 821 } 822 823 /* 824 * Did we fit? 825 */ 826 if (buflen != 0) { 827 bus_dmamap_unload(dmat, map); 828 return (EFBIG); /* XXX better return value here? */ 829 } 830 return (0); 831 } 832 833 /* 834 * Utility function to load a linear buffer. segp contains 835 * the starting segment on entrace, and the ending segment on exit. 836 */ 837 static int 838 bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 839 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 840 int *segp) 841 { 842 struct sync_list *sl; 843 bus_size_t sgsize; 844 bus_addr_t curaddr, sl_pend; 845 vm_offset_t kvaddr, vaddr, sl_vend; 846 int error; 847 848 KASSERT((map->flags & DMAMAP_FROM_DMAMEM) != 0 || 849 dmat->common.alignment <= PAGE_SIZE, 850 ("loading user buffer with alignment bigger than PAGE_SIZE is not " 851 "supported")); 852 853 if (segs == NULL) 854 segs = dmat->segments; 855 856 if (flags & BUS_DMA_LOAD_MBUF) 857 map->flags |= DMAMAP_MBUF; 858 859 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 860 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 861 if (map->pagesneeded != 0) { 862 error = _bus_dmamap_reserve_pages(dmat, map, flags); 863 if (error) 864 return (error); 865 } 866 } 867 868 /* 869 * XXX Optimally we should parse input buffer for physically 870 * continuous segments first and then pass these segment into 871 * load loop. 872 */ 873 sl = map->slist + map->sync_count - 1; 874 vaddr = (vm_offset_t)buf; 875 sl_pend = 0; 876 sl_vend = 0; 877 878 while (buflen > 0) { 879 /* 880 * Get the physical address for this segment. 881 */ 882 if (__predict_true(pmap == kernel_pmap)) { 883 curaddr = pmap_kextract(vaddr); 884 kvaddr = vaddr; 885 } else { 886 curaddr = pmap_extract(pmap, vaddr); 887 kvaddr = 0; 888 } 889 890 /* 891 * Compute the segment size, and adjust counts. 892 */ 893 sgsize = MIN(buflen, dmat->common.maxsegsz); 894 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 895 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 896 897 if (map->pagesneeded != 0 && 898 must_bounce(dmat, map, curaddr, sgsize)) { 899 /* See comment in bounce_bus_dmamap_load_phys */ 900 KASSERT(dmat->common.alignment <= PAGE_SIZE, 901 ("bounced buffer cannot have alignment bigger " 902 "than PAGE_SIZE: %lu", dmat->common.alignment)); 903 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 904 sgsize); 905 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 906 if (map->sync_count > 0) { 907 sl_pend = sl->paddr + sl->datacount; 908 sl_vend = sl->vaddr + sl->datacount; 909 } 910 911 if (map->sync_count == 0 || 912 (kvaddr != 0 && kvaddr != sl_vend) || 913 (curaddr != sl_pend)) { 914 if (++map->sync_count > dmat->common.nsegments) 915 break; 916 sl++; 917 sl->vaddr = kvaddr; 918 sl->paddr = curaddr; 919 if (kvaddr != 0) { 920 sl->pages = NULL; 921 } else { 922 sl->pages = PHYS_TO_VM_PAGE(curaddr); 923 KASSERT(sl->pages != NULL, 924 ("%s: page at PA:0x%08lx is not " 925 "in vm_page_array", __func__, 926 curaddr)); 927 } 928 sl->datacount = sgsize; 929 } else 930 sl->datacount += sgsize; 931 } 932 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 933 segp); 934 if (sgsize == 0) 935 break; 936 vaddr += sgsize; 937 buflen -= sgsize; 938 } 939 940 /* 941 * Did we fit? 942 */ 943 if (buflen != 0) { 944 bus_dmamap_unload(dmat, map); 945 return (EFBIG); /* XXX better return value here? */ 946 } 947 return (0); 948 } 949 950 static void 951 bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 952 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 953 { 954 955 map->mem = *mem; 956 map->dmat = dmat; 957 map->callback = callback; 958 map->callback_arg = callback_arg; 959 } 960 961 static bus_dma_segment_t * 962 bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 963 bus_dma_segment_t *segs, int nsegs, int error) 964 { 965 966 if (segs == NULL) 967 segs = dmat->segments; 968 return (segs); 969 } 970 971 /* 972 * Release the mapping held by map. 973 */ 974 static void 975 bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 976 { 977 free_bounce_pages(dmat, map); 978 map->sync_count = 0; 979 map->flags &= ~DMAMAP_MBUF; 980 } 981 982 static void 983 dma_preread_safe(vm_offset_t va, vm_size_t size) 984 { 985 /* 986 * Write back any partial cachelines immediately before and 987 * after the DMA region. 988 */ 989 if (va & (dcache_line_size - 1)) 990 cpu_dcache_wb_range(va, 1); 991 if ((va + size) & (dcache_line_size - 1)) 992 cpu_dcache_wb_range(va + size, 1); 993 994 cpu_dcache_inv_range(va, size); 995 } 996 997 static void 998 dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) 999 { 1000 uint32_t len, offset; 1001 vm_page_t m; 1002 vm_paddr_t pa; 1003 vm_offset_t va, tempva; 1004 bus_size_t size; 1005 1006 offset = sl->paddr & PAGE_MASK; 1007 m = sl->pages; 1008 size = sl->datacount; 1009 pa = sl->paddr; 1010 1011 for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { 1012 tempva = 0; 1013 if (sl->vaddr == 0) { 1014 len = min(PAGE_SIZE - offset, size); 1015 tempva = pmap_quick_enter_page(m); 1016 va = tempva | offset; 1017 KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), 1018 ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx", 1019 VM_PAGE_TO_PHYS(m) | offset, pa)); 1020 } else { 1021 len = sl->datacount; 1022 va = sl->vaddr; 1023 } 1024 1025 switch (op) { 1026 case BUS_DMASYNC_PREWRITE: 1027 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1028 cpu_dcache_wb_range(va, len); 1029 break; 1030 case BUS_DMASYNC_PREREAD: 1031 /* 1032 * An mbuf may start in the middle of a cacheline. There 1033 * will be no cpu writes to the beginning of that line 1034 * (which contains the mbuf header) while dma is in 1035 * progress. Handle that case by doing a writeback of 1036 * just the first cacheline before invalidating the 1037 * overall buffer. Any mbuf in a chain may have this 1038 * misalignment. Buffers which are not mbufs bounce if 1039 * they are not aligned to a cacheline. 1040 */ 1041 dma_preread_safe(va, len); 1042 break; 1043 case BUS_DMASYNC_POSTREAD: 1044 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1045 cpu_dcache_inv_range(va, len); 1046 break; 1047 default: 1048 panic("unsupported combination of sync operations: " 1049 "0x%08x\n", op); 1050 } 1051 1052 if (tempva != 0) 1053 pmap_quick_remove_page(tempva); 1054 } 1055 } 1056 1057 static void 1058 bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, 1059 bus_dmasync_op_t op) 1060 { 1061 struct bounce_page *bpage; 1062 struct sync_list *sl, *end; 1063 vm_offset_t datavaddr, tempvaddr; 1064 1065 if (op == BUS_DMASYNC_POSTWRITE) 1066 return; 1067 1068 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1069 /* 1070 * Wait for any DMA operations to complete before the bcopy. 1071 */ 1072 dsb(sy); 1073 } 1074 1075 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1076 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1077 "performing bounce", __func__, dmat, dmat->common.flags, 1078 op); 1079 1080 if ((op & BUS_DMASYNC_PREWRITE) != 0) { 1081 while (bpage != NULL) { 1082 tempvaddr = 0; 1083 datavaddr = bpage->datavaddr; 1084 if (datavaddr == 0) { 1085 tempvaddr = pmap_quick_enter_page( 1086 bpage->datapage); 1087 datavaddr = tempvaddr | bpage->dataoffs; 1088 } 1089 1090 bcopy((void *)datavaddr, 1091 (void *)bpage->vaddr, bpage->datacount); 1092 if (tempvaddr != 0) 1093 pmap_quick_remove_page(tempvaddr); 1094 if ((map->flags & DMAMAP_COHERENT) == 0) 1095 cpu_dcache_wb_range(bpage->vaddr, 1096 bpage->datacount); 1097 bpage = STAILQ_NEXT(bpage, links); 1098 } 1099 dmat->bounce_zone->total_bounced++; 1100 } else if ((op & BUS_DMASYNC_PREREAD) != 0) { 1101 while (bpage != NULL) { 1102 if ((map->flags & DMAMAP_COHERENT) == 0) 1103 cpu_dcache_wbinv_range(bpage->vaddr, 1104 bpage->datacount); 1105 bpage = STAILQ_NEXT(bpage, links); 1106 } 1107 } 1108 1109 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1110 while (bpage != NULL) { 1111 if ((map->flags & DMAMAP_COHERENT) == 0) 1112 cpu_dcache_inv_range(bpage->vaddr, 1113 bpage->datacount); 1114 tempvaddr = 0; 1115 datavaddr = bpage->datavaddr; 1116 if (datavaddr == 0) { 1117 tempvaddr = pmap_quick_enter_page( 1118 bpage->datapage); 1119 datavaddr = tempvaddr | bpage->dataoffs; 1120 } 1121 1122 bcopy((void *)bpage->vaddr, 1123 (void *)datavaddr, bpage->datacount); 1124 1125 if (tempvaddr != 0) 1126 pmap_quick_remove_page(tempvaddr); 1127 bpage = STAILQ_NEXT(bpage, links); 1128 } 1129 dmat->bounce_zone->total_bounced++; 1130 } 1131 } 1132 1133 /* 1134 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. 1135 */ 1136 if (map->sync_count != 0) { 1137 sl = &map->slist[0]; 1138 end = &map->slist[map->sync_count]; 1139 CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x " 1140 "performing sync", __func__, dmat, op); 1141 1142 for ( ; sl != end; ++sl) 1143 dma_dcache_sync(sl, op); 1144 } 1145 1146 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) { 1147 /* 1148 * Wait for the bcopy to complete before any DMA operations. 1149 */ 1150 dsb(sy); 1151 } 1152 } 1153 1154 struct bus_dma_impl bus_dma_bounce_impl = { 1155 .tag_create = bounce_bus_dma_tag_create, 1156 .tag_destroy = bounce_bus_dma_tag_destroy, 1157 .tag_set_domain = bounce_bus_dma_tag_set_domain, 1158 .id_mapped = bounce_bus_dma_id_mapped, 1159 .map_create = bounce_bus_dmamap_create, 1160 .map_destroy = bounce_bus_dmamap_destroy, 1161 .mem_alloc = bounce_bus_dmamem_alloc, 1162 .mem_free = bounce_bus_dmamem_free, 1163 .load_phys = bounce_bus_dmamap_load_phys, 1164 .load_buffer = bounce_bus_dmamap_load_buffer, 1165 .load_ma = bus_dmamap_load_ma_triv, 1166 .map_waitok = bounce_bus_dmamap_waitok, 1167 .map_complete = bounce_bus_dmamap_complete, 1168 .map_unload = bounce_bus_dmamap_unload, 1169 .map_sync = bounce_bus_dmamap_sync 1170 }; 1171