1 /*- 2 * Copyright (c) 1997, 1998 Justin T. Gibbs. 3 * Copyright (c) 2015-2016 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship of the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by Semihalf 10 * under sponsorship of the FreeBSD Foundation. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 1. Redistributions of source code must retain the above copyright 16 * notice, this list of conditions, and the following disclaimer, 17 * without modification, immediately at the beginning of the file. 18 * 2. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR 25 * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/param.h> 35 #include <sys/systm.h> 36 #include <sys/domainset.h> 37 #include <sys/malloc.h> 38 #include <sys/bus.h> 39 #include <sys/interrupt.h> 40 #include <sys/kernel.h> 41 #include <sys/ktr.h> 42 #include <sys/lock.h> 43 #include <sys/memdesc.h> 44 #include <sys/msan.h> 45 #include <sys/mutex.h> 46 #include <sys/proc.h> 47 #include <sys/sysctl.h> 48 #include <sys/uio.h> 49 50 #include <vm/vm.h> 51 #include <vm/vm_extern.h> 52 #include <vm/vm_kern.h> 53 #include <vm/vm_page.h> 54 #include <vm/vm_map.h> 55 56 #include <machine/atomic.h> 57 #include <machine/bus.h> 58 #include <machine/md_var.h> 59 #include <arm64/include/bus_dma_impl.h> 60 61 #define MAX_BPAGES 4096 62 63 enum { 64 BF_COULD_BOUNCE = 0x01, 65 BF_MIN_ALLOC_COMP = 0x02, 66 BF_KMEM_ALLOC = 0x04, 67 BF_COHERENT = 0x10, 68 }; 69 70 struct bounce_page; 71 struct bounce_zone; 72 73 struct bus_dma_tag { 74 struct bus_dma_tag_common common; 75 size_t alloc_size; 76 size_t alloc_alignment; 77 int map_count; 78 int bounce_flags; 79 bus_dma_segment_t *segments; 80 struct bounce_zone *bounce_zone; 81 }; 82 83 static SYSCTL_NODE(_hw, OID_AUTO, busdma, CTLFLAG_RD | CTLFLAG_MPSAFE, 0, 84 "Busdma parameters"); 85 86 struct sync_list { 87 vm_offset_t vaddr; /* kva of client data */ 88 bus_addr_t paddr; /* physical address */ 89 vm_page_t pages; /* starting page of client data */ 90 bus_size_t datacount; /* client data count */ 91 }; 92 93 struct bus_dmamap { 94 STAILQ_HEAD(, bounce_page) bpages; 95 int pagesneeded; 96 int pagesreserved; 97 bus_dma_tag_t dmat; 98 struct memdesc mem; 99 bus_dmamap_callback_t *callback; 100 void *callback_arg; 101 __sbintime_t queued_time; 102 STAILQ_ENTRY(bus_dmamap) links; 103 u_int flags; 104 #define DMAMAP_COHERENT (1 << 0) 105 #define DMAMAP_FROM_DMAMEM (1 << 1) 106 #define DMAMAP_MBUF (1 << 2) 107 int sync_count; 108 #ifdef KMSAN 109 struct memdesc kmsan_mem; 110 #endif 111 struct sync_list slist[]; 112 }; 113 114 static bool _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, 115 vm_paddr_t buf, bus_size_t buflen, int *pagesneeded); 116 static void _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, 117 pmap_t pmap, void *buf, bus_size_t buflen, int flags); 118 static void _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 119 vm_paddr_t buf, bus_size_t buflen, int flags); 120 121 static MALLOC_DEFINE(M_BUSDMA, "busdma", "busdma metadata"); 122 123 #define dmat_alignment(dmat) ((dmat)->common.alignment) 124 #define dmat_domain(dmat) ((dmat)->common.domain) 125 #define dmat_flags(dmat) ((dmat)->common.flags) 126 #define dmat_highaddr(dmat) ((dmat)->common.highaddr) 127 #define dmat_lowaddr(dmat) ((dmat)->common.lowaddr) 128 #define dmat_lockfunc(dmat) ((dmat)->common.lockfunc) 129 #define dmat_lockfuncarg(dmat) ((dmat)->common.lockfuncarg) 130 131 #include "../../kern/subr_busdma_bounce.c" 132 133 static int 134 bounce_bus_dma_zone_setup(bus_dma_tag_t dmat) 135 { 136 struct bounce_zone *bz; 137 bus_size_t maxsize; 138 int error; 139 140 /* 141 * Round size up to a full page, and add one more page because 142 * there can always be one more boundary crossing than the 143 * number of pages in a transfer. 144 */ 145 maxsize = roundup2(dmat->common.maxsize, PAGE_SIZE) + PAGE_SIZE; 146 147 /* Must bounce */ 148 if ((error = alloc_bounce_zone(dmat)) != 0) 149 return (error); 150 bz = dmat->bounce_zone; 151 152 if (ptoa(bz->total_bpages) < maxsize) { 153 int pages; 154 155 pages = atop(maxsize) + 1 - bz->total_bpages; 156 157 /* Add pages to our bounce pool */ 158 if (alloc_bounce_pages(dmat, pages) < pages) 159 return (ENOMEM); 160 } 161 /* Performed initial allocation */ 162 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 163 164 return (error); 165 } 166 167 /* 168 * Return true if the DMA should bounce because the start or end does not fall 169 * on a cacheline boundary (which would require a partial cacheline flush). 170 * COHERENT memory doesn't trigger cacheline flushes. Memory allocated by 171 * bus_dmamem_alloc() is always aligned to cacheline boundaries, and there's a 172 * strict rule that such memory cannot be accessed by the CPU while DMA is in 173 * progress (or by multiple DMA engines at once), so that it's always safe to do 174 * full cacheline flushes even if that affects memory outside the range of a 175 * given DMA operation that doesn't involve the full allocated buffer. If we're 176 * mapping an mbuf, that follows the same rules as a buffer we allocated. 177 */ 178 static bool 179 cacheline_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 180 bus_size_t size) 181 { 182 183 #define DMAMAP_CACHELINE_FLAGS \ 184 (DMAMAP_FROM_DMAMEM | DMAMAP_COHERENT | DMAMAP_MBUF) 185 if ((dmat->bounce_flags & BF_COHERENT) != 0) 186 return (false); 187 if (map != NULL && (map->flags & DMAMAP_CACHELINE_FLAGS) != 0) 188 return (false); 189 return (((paddr | size) & (dcache_line_size - 1)) != 0); 190 #undef DMAMAP_CACHELINE_FLAGS 191 } 192 193 /* 194 * Return true if the given address does not fall on the alignment boundary. 195 */ 196 static bool 197 alignment_bounce(bus_dma_tag_t dmat, bus_addr_t addr) 198 { 199 200 return (!vm_addr_align_ok(addr, dmat->common.alignment)); 201 } 202 203 static bool 204 might_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 205 bus_size_t size) 206 { 207 208 /* Memory allocated by bounce_bus_dmamem_alloc won't bounce */ 209 if (map && (map->flags & DMAMAP_FROM_DMAMEM) != 0) 210 return (false); 211 212 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0) 213 return (true); 214 215 if (cacheline_bounce(dmat, map, paddr, size)) 216 return (true); 217 218 if (alignment_bounce(dmat, paddr)) 219 return (true); 220 221 return (false); 222 } 223 224 static bool 225 must_bounce(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t paddr, 226 bus_size_t size) 227 { 228 229 if (cacheline_bounce(dmat, map, paddr, size)) 230 return (true); 231 232 if ((dmat->bounce_flags & BF_COULD_BOUNCE) != 0 && 233 addr_needs_bounce(dmat, paddr)) 234 return (true); 235 236 return (false); 237 } 238 239 /* 240 * Allocate a device specific dma_tag. 241 */ 242 static int 243 bounce_bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment, 244 bus_addr_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr, 245 bus_size_t maxsize, int nsegments, bus_size_t maxsegsz, int flags, 246 bus_dma_lock_t *lockfunc, void *lockfuncarg, bus_dma_tag_t *dmat) 247 { 248 bus_dma_tag_t newtag; 249 int error; 250 251 *dmat = NULL; 252 error = common_bus_dma_tag_create(parent != NULL ? &parent->common : 253 NULL, alignment, boundary, lowaddr, highaddr, maxsize, nsegments, 254 maxsegsz, flags, lockfunc, lockfuncarg, 255 sizeof (struct bus_dma_tag), (void **)&newtag); 256 if (error != 0) 257 return (error); 258 259 newtag->common.impl = &bus_dma_bounce_impl; 260 newtag->map_count = 0; 261 newtag->segments = NULL; 262 263 if ((flags & BUS_DMA_COHERENT) != 0) { 264 newtag->bounce_flags |= BF_COHERENT; 265 } 266 267 if (parent != NULL) { 268 if ((parent->bounce_flags & BF_COULD_BOUNCE) != 0) 269 newtag->bounce_flags |= BF_COULD_BOUNCE; 270 271 /* Copy some flags from the parent */ 272 newtag->bounce_flags |= parent->bounce_flags & BF_COHERENT; 273 } 274 275 if ((newtag->bounce_flags & BF_COHERENT) != 0) { 276 newtag->alloc_alignment = newtag->common.alignment; 277 newtag->alloc_size = newtag->common.maxsize; 278 } else { 279 /* 280 * Ensure the buffer is aligned to a cacheline when allocating 281 * a non-coherent buffer. This is so we don't have any data 282 * that another CPU may be accessing around DMA buffer 283 * causing the cache to become dirty. 284 */ 285 newtag->alloc_alignment = MAX(newtag->common.alignment, 286 dcache_line_size); 287 newtag->alloc_size = roundup2(newtag->common.maxsize, 288 dcache_line_size); 289 } 290 291 if (newtag->common.lowaddr < ptoa((vm_paddr_t)Maxmem) || 292 newtag->common.alignment > 1) 293 newtag->bounce_flags |= BF_COULD_BOUNCE; 294 295 if ((flags & BUS_DMA_ALLOCNOW) != 0) 296 error = bounce_bus_dma_zone_setup(newtag); 297 else 298 error = 0; 299 300 if (error != 0) 301 free(newtag, M_DEVBUF); 302 else 303 *dmat = newtag; 304 CTR4(KTR_BUSDMA, "%s returned tag %p tag flags 0x%x error %d", 305 __func__, newtag, (newtag != NULL ? newtag->common.flags : 0), 306 error); 307 return (error); 308 } 309 310 static int 311 bounce_bus_dma_tag_destroy(bus_dma_tag_t dmat) 312 { 313 int error = 0; 314 315 if (dmat != NULL) { 316 if (dmat->map_count != 0) { 317 error = EBUSY; 318 goto out; 319 } 320 if (dmat->segments != NULL) 321 free(dmat->segments, M_DEVBUF); 322 free(dmat, M_DEVBUF); 323 } 324 out: 325 CTR3(KTR_BUSDMA, "%s tag %p error %d", __func__, dmat, error); 326 return (error); 327 } 328 329 /* 330 * Update the domain for the tag. We may need to reallocate the zone and 331 * bounce pages. 332 */ 333 static int 334 bounce_bus_dma_tag_set_domain(bus_dma_tag_t dmat) 335 { 336 337 KASSERT(dmat->map_count == 0, 338 ("bounce_bus_dma_tag_set_domain: Domain set after use.\n")); 339 if ((dmat->bounce_flags & BF_COULD_BOUNCE) == 0 || 340 dmat->bounce_zone == NULL) 341 return (0); 342 dmat->bounce_flags &= ~BF_MIN_ALLOC_COMP; 343 return (bounce_bus_dma_zone_setup(dmat)); 344 } 345 346 static bool 347 bounce_bus_dma_id_mapped(bus_dma_tag_t dmat, vm_paddr_t buf, bus_size_t buflen) 348 { 349 350 if (!might_bounce(dmat, NULL, buf, buflen)) 351 return (true); 352 return (!_bus_dmamap_pagesneeded(dmat, NULL, buf, buflen, NULL)); 353 } 354 355 static bus_dmamap_t 356 alloc_dmamap(bus_dma_tag_t dmat, int flags) 357 { 358 u_long mapsize; 359 bus_dmamap_t map; 360 361 mapsize = sizeof(*map); 362 mapsize += sizeof(struct sync_list) * dmat->common.nsegments; 363 map = malloc_domainset(mapsize, M_DEVBUF, 364 DOMAINSET_PREF(dmat->common.domain), flags | M_ZERO); 365 if (map == NULL) 366 return (NULL); 367 368 /* Initialize the new map */ 369 STAILQ_INIT(&map->bpages); 370 371 return (map); 372 } 373 374 /* 375 * Allocate a handle for mapping from kva/uva/physical 376 * address space into bus device space. 377 */ 378 static int 379 bounce_bus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp) 380 { 381 struct bounce_zone *bz; 382 int error, maxpages, pages; 383 384 error = 0; 385 386 if (dmat->segments == NULL) { 387 dmat->segments = mallocarray_domainset(dmat->common.nsegments, 388 sizeof(bus_dma_segment_t), M_DEVBUF, 389 DOMAINSET_PREF(dmat->common.domain), M_NOWAIT); 390 if (dmat->segments == NULL) { 391 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 392 __func__, dmat, ENOMEM); 393 return (ENOMEM); 394 } 395 } 396 397 *mapp = alloc_dmamap(dmat, M_NOWAIT); 398 if (*mapp == NULL) { 399 CTR3(KTR_BUSDMA, "%s: tag %p error %d", 400 __func__, dmat, ENOMEM); 401 return (ENOMEM); 402 } 403 404 /* 405 * Bouncing might be required if the driver asks for an active 406 * exclusion region, a data alignment that is stricter than 1, and/or 407 * an active address boundary. 408 */ 409 if (dmat->bounce_zone == NULL) { 410 if ((error = alloc_bounce_zone(dmat)) != 0) { 411 free(*mapp, M_DEVBUF); 412 return (error); 413 } 414 } 415 bz = dmat->bounce_zone; 416 417 /* 418 * Attempt to add pages to our pool on a per-instance basis up to a sane 419 * limit. Even if the tag isn't subject of bouncing due to alignment 420 * and boundary constraints, it could still auto-bounce due to 421 * cacheline alignment, which requires at most two bounce pages. 422 */ 423 if (dmat->common.alignment > 1) 424 maxpages = MAX_BPAGES; 425 else 426 maxpages = MIN(MAX_BPAGES, Maxmem - 427 atop(dmat->common.lowaddr)); 428 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0 || 429 (bz->map_count > 0 && bz->total_bpages < maxpages)) { 430 pages = atop(roundup2(dmat->common.maxsize, PAGE_SIZE)) + 1; 431 pages = MIN(maxpages - bz->total_bpages, pages); 432 pages = MAX(pages, 2); 433 if (alloc_bounce_pages(dmat, pages) < pages) 434 error = ENOMEM; 435 if ((dmat->bounce_flags & BF_MIN_ALLOC_COMP) == 0) { 436 if (error == 0) { 437 dmat->bounce_flags |= BF_MIN_ALLOC_COMP; 438 } 439 } else 440 error = 0; 441 } 442 bz->map_count++; 443 444 if (error == 0) { 445 dmat->map_count++; 446 if ((dmat->bounce_flags & BF_COHERENT) != 0) 447 (*mapp)->flags |= DMAMAP_COHERENT; 448 } else { 449 free(*mapp, M_DEVBUF); 450 } 451 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 452 __func__, dmat, dmat->common.flags, error); 453 return (error); 454 } 455 456 /* 457 * Destroy a handle for mapping from kva/uva/physical 458 * address space into bus device space. 459 */ 460 static int 461 bounce_bus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map) 462 { 463 464 /* Check we are destroying the correct map type */ 465 if ((map->flags & DMAMAP_FROM_DMAMEM) != 0) 466 panic("bounce_bus_dmamap_destroy: Invalid map freed\n"); 467 468 if (STAILQ_FIRST(&map->bpages) != NULL || map->sync_count != 0) { 469 CTR3(KTR_BUSDMA, "%s: tag %p error %d", __func__, dmat, EBUSY); 470 return (EBUSY); 471 } 472 if (dmat->bounce_zone) 473 dmat->bounce_zone->map_count--; 474 free(map, M_DEVBUF); 475 dmat->map_count--; 476 CTR2(KTR_BUSDMA, "%s: tag %p error 0", __func__, dmat); 477 return (0); 478 } 479 480 /* 481 * Allocate a piece of memory that can be efficiently mapped into 482 * bus device space based on the constraints lited in the dma tag. 483 * A dmamap to for use with dmamap_load is also allocated. 484 */ 485 static int 486 bounce_bus_dmamem_alloc(bus_dma_tag_t dmat, void** vaddr, int flags, 487 bus_dmamap_t *mapp) 488 { 489 vm_memattr_t attr; 490 int mflags; 491 492 if (flags & BUS_DMA_NOWAIT) 493 mflags = M_NOWAIT; 494 else 495 mflags = M_WAITOK; 496 497 if (dmat->segments == NULL) { 498 dmat->segments = mallocarray_domainset(dmat->common.nsegments, 499 sizeof(bus_dma_segment_t), M_DEVBUF, 500 DOMAINSET_PREF(dmat->common.domain), mflags); 501 if (dmat->segments == NULL) { 502 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 503 __func__, dmat, dmat->common.flags, ENOMEM); 504 return (ENOMEM); 505 } 506 } 507 if (flags & BUS_DMA_ZERO) 508 mflags |= M_ZERO; 509 if (flags & BUS_DMA_NOCACHE) 510 attr = VM_MEMATTR_UNCACHEABLE; 511 else if ((flags & BUS_DMA_COHERENT) != 0 && 512 (dmat->bounce_flags & BF_COHERENT) == 0) 513 /* 514 * If we have a non-coherent tag, and are trying to allocate 515 * a coherent block of memory it needs to be uncached. 516 */ 517 attr = VM_MEMATTR_UNCACHEABLE; 518 else 519 attr = VM_MEMATTR_DEFAULT; 520 521 /* 522 * Create the map, but don't set the could bounce flag as 523 * this allocation should never bounce; 524 */ 525 *mapp = alloc_dmamap(dmat, mflags); 526 if (*mapp == NULL) { 527 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 528 __func__, dmat, dmat->common.flags, ENOMEM); 529 return (ENOMEM); 530 } 531 532 /* 533 * Mark the map as coherent if we used uncacheable memory or the 534 * tag was already marked as coherent. 535 */ 536 if (attr == VM_MEMATTR_UNCACHEABLE || 537 (dmat->bounce_flags & BF_COHERENT) != 0) 538 (*mapp)->flags |= DMAMAP_COHERENT; 539 540 (*mapp)->flags |= DMAMAP_FROM_DMAMEM; 541 542 /* 543 * Allocate the buffer from the malloc(9) allocator if... 544 * - It's small enough to fit into a single page. 545 * - Its alignment requirement is also smaller than the page size. 546 * - The low address requirement is fulfilled. 547 * - Default cache attributes are requested (WB). 548 * else allocate non-contiguous pages if... 549 * - The page count that could get allocated doesn't exceed 550 * nsegments also when the maximum segment size is less 551 * than PAGE_SIZE. 552 * - The alignment constraint isn't larger than a page boundary. 553 * - There are no boundary-crossing constraints. 554 * else allocate a block of contiguous pages because one or more of the 555 * constraints is something that only the contig allocator can fulfill. 556 * 557 * NOTE: The (dmat->common.alignment <= dmat->maxsize) check 558 * below is just a quick hack. The exact alignment guarantees 559 * of malloc(9) need to be nailed down, and the code below 560 * should be rewritten to take that into account. 561 * 562 * In the meantime warn the user if malloc gets it wrong. 563 */ 564 if (dmat->alloc_size <= PAGE_SIZE && 565 dmat->alloc_alignment <= PAGE_SIZE && 566 dmat->common.lowaddr >= ptoa((vm_paddr_t)Maxmem) && 567 attr == VM_MEMATTR_DEFAULT) { 568 *vaddr = malloc_domainset_aligned(dmat->alloc_size, 569 dmat->alloc_alignment, M_DEVBUF, 570 DOMAINSET_PREF(dmat->common.domain), mflags); 571 } else if (dmat->common.nsegments >= 572 howmany(dmat->alloc_size, MIN(dmat->common.maxsegsz, PAGE_SIZE)) && 573 dmat->alloc_alignment <= PAGE_SIZE && 574 (dmat->common.boundary % PAGE_SIZE) == 0) { 575 /* Page-based multi-segment allocations allowed */ 576 *vaddr = kmem_alloc_attr_domainset( 577 DOMAINSET_PREF(dmat->common.domain), dmat->alloc_size, 578 mflags, 0ul, dmat->common.lowaddr, attr); 579 dmat->bounce_flags |= BF_KMEM_ALLOC; 580 } else { 581 *vaddr = kmem_alloc_contig_domainset( 582 DOMAINSET_PREF(dmat->common.domain), dmat->alloc_size, 583 mflags, 0ul, dmat->common.lowaddr, 584 dmat->alloc_alignment != 0 ? dmat->alloc_alignment : 1ul, 585 dmat->common.boundary, attr); 586 dmat->bounce_flags |= BF_KMEM_ALLOC; 587 } 588 if (*vaddr == NULL) { 589 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 590 __func__, dmat, dmat->common.flags, ENOMEM); 591 free(*mapp, M_DEVBUF); 592 return (ENOMEM); 593 } else if (!vm_addr_align_ok(vtophys(*vaddr), dmat->alloc_alignment)) { 594 printf("bus_dmamem_alloc failed to align memory properly.\n"); 595 } 596 dmat->map_count++; 597 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x error %d", 598 __func__, dmat, dmat->common.flags, 0); 599 return (0); 600 } 601 602 /* 603 * Free a piece of memory and it's allociated dmamap, that was allocated 604 * via bus_dmamem_alloc. Make the same choice for free/contigfree. 605 */ 606 static void 607 bounce_bus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map) 608 { 609 610 /* 611 * Check the map came from bounce_bus_dmamem_alloc, so the map 612 * should be NULL and the BF_KMEM_ALLOC flag cleared if malloc() 613 * was used and set if kmem_alloc_contig() was used. 614 */ 615 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 616 panic("bus_dmamem_free: Invalid map freed\n"); 617 if ((dmat->bounce_flags & BF_KMEM_ALLOC) == 0) 618 free(vaddr, M_DEVBUF); 619 else 620 kmem_free(vaddr, dmat->alloc_size); 621 free(map, M_DEVBUF); 622 dmat->map_count--; 623 CTR3(KTR_BUSDMA, "%s: tag %p flags 0x%x", __func__, dmat, 624 dmat->bounce_flags); 625 } 626 627 static bool 628 _bus_dmamap_pagesneeded(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 629 bus_size_t buflen, int *pagesneeded) 630 { 631 bus_addr_t curaddr; 632 bus_size_t sgsize; 633 int count; 634 635 /* 636 * Count the number of bounce pages needed in order to 637 * complete this transfer 638 */ 639 count = 0; 640 curaddr = buf; 641 while (buflen != 0) { 642 sgsize = MIN(buflen, dmat->common.maxsegsz); 643 if (must_bounce(dmat, map, curaddr, sgsize)) { 644 sgsize = MIN(sgsize, 645 PAGE_SIZE - (curaddr & PAGE_MASK)); 646 if (pagesneeded == NULL) 647 return (true); 648 count++; 649 } 650 curaddr += sgsize; 651 buflen -= sgsize; 652 } 653 654 if (pagesneeded != NULL) 655 *pagesneeded = count; 656 return (count != 0); 657 } 658 659 static void 660 _bus_dmamap_count_phys(bus_dma_tag_t dmat, bus_dmamap_t map, vm_paddr_t buf, 661 bus_size_t buflen, int flags) 662 { 663 664 if (map->pagesneeded == 0) { 665 _bus_dmamap_pagesneeded(dmat, map, buf, buflen, 666 &map->pagesneeded); 667 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 668 } 669 } 670 671 static void 672 _bus_dmamap_count_pages(bus_dma_tag_t dmat, bus_dmamap_t map, pmap_t pmap, 673 void *buf, bus_size_t buflen, int flags) 674 { 675 vm_offset_t vaddr; 676 vm_offset_t vendaddr; 677 bus_addr_t paddr; 678 bus_size_t sg_len; 679 680 if (map->pagesneeded == 0) { 681 CTR4(KTR_BUSDMA, "lowaddr= %d Maxmem= %d, boundary= %d, " 682 "alignment= %d", dmat->common.lowaddr, 683 ptoa((vm_paddr_t)Maxmem), 684 dmat->common.boundary, dmat->common.alignment); 685 CTR2(KTR_BUSDMA, "map= %p, pagesneeded= %d", map, 686 map->pagesneeded); 687 /* 688 * Count the number of bounce pages 689 * needed in order to complete this transfer 690 */ 691 vaddr = (vm_offset_t)buf; 692 vendaddr = (vm_offset_t)buf + buflen; 693 694 while (vaddr < vendaddr) { 695 sg_len = PAGE_SIZE - ((vm_offset_t)vaddr & PAGE_MASK); 696 sg_len = MIN(sg_len, dmat->common.maxsegsz); 697 if (pmap == kernel_pmap) 698 paddr = pmap_kextract(vaddr); 699 else 700 paddr = pmap_extract(pmap, vaddr); 701 if (must_bounce(dmat, map, paddr, 702 min(vendaddr - vaddr, (PAGE_SIZE - ((vm_offset_t)vaddr & 703 PAGE_MASK)))) != 0) { 704 sg_len = roundup2(sg_len, 705 dmat->common.alignment); 706 map->pagesneeded++; 707 } 708 vaddr += sg_len; 709 } 710 CTR1(KTR_BUSDMA, "pagesneeded= %d\n", map->pagesneeded); 711 } 712 } 713 714 /* 715 * Add a single contiguous physical range to the segment list. 716 */ 717 static bus_size_t 718 _bus_dmamap_addseg(bus_dma_tag_t dmat, bus_dmamap_t map, bus_addr_t curaddr, 719 bus_size_t sgsize, bus_dma_segment_t *segs, int *segp) 720 { 721 int seg; 722 723 /* 724 * Make sure we don't cross any boundaries. 725 */ 726 if (!vm_addr_bound_ok(curaddr, sgsize, dmat->common.boundary)) 727 sgsize = roundup2(curaddr, dmat->common.boundary) - curaddr; 728 729 /* 730 * Insert chunk into a segment, coalescing with 731 * previous segment if possible. 732 */ 733 seg = *segp; 734 if (seg == -1) { 735 seg = 0; 736 segs[seg].ds_addr = curaddr; 737 segs[seg].ds_len = sgsize; 738 } else { 739 if (curaddr == segs[seg].ds_addr + segs[seg].ds_len && 740 (segs[seg].ds_len + sgsize) <= dmat->common.maxsegsz && 741 vm_addr_bound_ok(segs[seg].ds_addr, 742 segs[seg].ds_len + sgsize, dmat->common.boundary)) 743 segs[seg].ds_len += sgsize; 744 else { 745 if (++seg >= dmat->common.nsegments) 746 return (0); 747 segs[seg].ds_addr = curaddr; 748 segs[seg].ds_len = sgsize; 749 } 750 } 751 *segp = seg; 752 return (sgsize); 753 } 754 755 /* 756 * Utility function to load a physical buffer. segp contains 757 * the starting segment on entrace, and the ending segment on exit. 758 */ 759 static int 760 bounce_bus_dmamap_load_phys(bus_dma_tag_t dmat, bus_dmamap_t map, 761 vm_paddr_t buf, bus_size_t buflen, int flags, bus_dma_segment_t *segs, 762 int *segp) 763 { 764 struct sync_list *sl; 765 bus_size_t sgsize; 766 bus_addr_t curaddr, sl_end; 767 int error; 768 769 if (segs == NULL) 770 segs = dmat->segments; 771 772 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 773 _bus_dmamap_count_phys(dmat, map, buf, buflen, flags); 774 if (map->pagesneeded != 0) { 775 error = _bus_dmamap_reserve_pages(dmat, map, flags); 776 if (error) 777 return (error); 778 } 779 } 780 781 sl = map->slist + map->sync_count - 1; 782 sl_end = 0; 783 784 while (buflen > 0) { 785 curaddr = buf; 786 sgsize = MIN(buflen, dmat->common.maxsegsz); 787 if (map->pagesneeded != 0 && 788 must_bounce(dmat, map, curaddr, sgsize)) { 789 /* 790 * The attempt to split a physically continuous buffer 791 * seems very controversial, it's unclear whether we 792 * can do this in all cases. Also, memory for bounced 793 * buffers is allocated as pages, so we cannot 794 * guarantee multipage alignment. 795 */ 796 KASSERT(dmat->common.alignment <= PAGE_SIZE, 797 ("bounced buffer cannot have alignment bigger " 798 "than PAGE_SIZE: %lu", dmat->common.alignment)); 799 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 800 curaddr = add_bounce_page(dmat, map, 0, curaddr, 801 sgsize); 802 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 803 if (map->sync_count > 0) 804 sl_end = sl->paddr + sl->datacount; 805 806 if (map->sync_count == 0 || curaddr != sl_end) { 807 if (++map->sync_count > dmat->common.nsegments) 808 break; 809 sl++; 810 sl->vaddr = 0; 811 sl->paddr = curaddr; 812 sl->pages = PHYS_TO_VM_PAGE(curaddr); 813 KASSERT(sl->pages != NULL, 814 ("%s: page at PA:0x%08lx is not in " 815 "vm_page_array", __func__, curaddr)); 816 sl->datacount = sgsize; 817 } else 818 sl->datacount += sgsize; 819 } 820 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 821 segp); 822 if (sgsize == 0) 823 break; 824 buf += sgsize; 825 buflen -= sgsize; 826 } 827 828 /* 829 * Did we fit? 830 */ 831 if (buflen != 0) { 832 bus_dmamap_unload(dmat, map); 833 return (EFBIG); /* XXX better return value here? */ 834 } 835 return (0); 836 } 837 838 /* 839 * Utility function to load a linear buffer. segp contains 840 * the starting segment on entrace, and the ending segment on exit. 841 */ 842 static int 843 bounce_bus_dmamap_load_buffer(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf, 844 bus_size_t buflen, pmap_t pmap, int flags, bus_dma_segment_t *segs, 845 int *segp) 846 { 847 struct sync_list *sl; 848 bus_size_t sgsize; 849 bus_addr_t curaddr, sl_pend; 850 vm_offset_t kvaddr, vaddr, sl_vend; 851 int error; 852 853 KASSERT((map->flags & DMAMAP_FROM_DMAMEM) != 0 || 854 dmat->common.alignment <= PAGE_SIZE, 855 ("loading user buffer with alignment bigger than PAGE_SIZE is not " 856 "supported")); 857 858 if (segs == NULL) 859 segs = dmat->segments; 860 861 if (flags & BUS_DMA_LOAD_MBUF) 862 map->flags |= DMAMAP_MBUF; 863 864 if (might_bounce(dmat, map, (bus_addr_t)buf, buflen)) { 865 _bus_dmamap_count_pages(dmat, map, pmap, buf, buflen, flags); 866 if (map->pagesneeded != 0) { 867 error = _bus_dmamap_reserve_pages(dmat, map, flags); 868 if (error) 869 return (error); 870 } 871 } 872 873 /* 874 * XXX Optimally we should parse input buffer for physically 875 * continuous segments first and then pass these segment into 876 * load loop. 877 */ 878 sl = map->slist + map->sync_count - 1; 879 vaddr = (vm_offset_t)buf; 880 sl_pend = 0; 881 sl_vend = 0; 882 883 while (buflen > 0) { 884 /* 885 * Get the physical address for this segment. 886 */ 887 if (__predict_true(pmap == kernel_pmap)) { 888 curaddr = pmap_kextract(vaddr); 889 kvaddr = vaddr; 890 } else { 891 curaddr = pmap_extract(pmap, vaddr); 892 kvaddr = 0; 893 } 894 895 /* 896 * Compute the segment size, and adjust counts. 897 */ 898 sgsize = MIN(buflen, dmat->common.maxsegsz); 899 if ((map->flags & DMAMAP_FROM_DMAMEM) == 0) 900 sgsize = MIN(sgsize, PAGE_SIZE - (curaddr & PAGE_MASK)); 901 902 if (map->pagesneeded != 0 && 903 must_bounce(dmat, map, curaddr, sgsize)) { 904 /* See comment in bounce_bus_dmamap_load_phys */ 905 KASSERT(dmat->common.alignment <= PAGE_SIZE, 906 ("bounced buffer cannot have alignment bigger " 907 "than PAGE_SIZE: %lu", dmat->common.alignment)); 908 curaddr = add_bounce_page(dmat, map, kvaddr, curaddr, 909 sgsize); 910 } else if ((map->flags & DMAMAP_COHERENT) == 0) { 911 if (map->sync_count > 0) { 912 sl_pend = sl->paddr + sl->datacount; 913 sl_vend = sl->vaddr + sl->datacount; 914 } 915 916 if (map->sync_count == 0 || 917 (kvaddr != 0 && kvaddr != sl_vend) || 918 (curaddr != sl_pend)) { 919 if (++map->sync_count > dmat->common.nsegments) 920 break; 921 sl++; 922 sl->vaddr = kvaddr; 923 sl->paddr = curaddr; 924 if (kvaddr != 0) { 925 sl->pages = NULL; 926 } else { 927 sl->pages = PHYS_TO_VM_PAGE(curaddr); 928 KASSERT(sl->pages != NULL, 929 ("%s: page at PA:0x%08lx is not " 930 "in vm_page_array", __func__, 931 curaddr)); 932 } 933 sl->datacount = sgsize; 934 } else 935 sl->datacount += sgsize; 936 } 937 sgsize = _bus_dmamap_addseg(dmat, map, curaddr, sgsize, segs, 938 segp); 939 if (sgsize == 0) 940 break; 941 vaddr += sgsize; 942 buflen -= sgsize; 943 } 944 945 /* 946 * Did we fit? 947 */ 948 if (buflen != 0) { 949 bus_dmamap_unload(dmat, map); 950 return (EFBIG); /* XXX better return value here? */ 951 } 952 return (0); 953 } 954 955 static void 956 bounce_bus_dmamap_waitok(bus_dma_tag_t dmat, bus_dmamap_t map, 957 struct memdesc *mem, bus_dmamap_callback_t *callback, void *callback_arg) 958 { 959 960 map->mem = *mem; 961 map->dmat = dmat; 962 map->callback = callback; 963 map->callback_arg = callback_arg; 964 } 965 966 static bus_dma_segment_t * 967 bounce_bus_dmamap_complete(bus_dma_tag_t dmat, bus_dmamap_t map, 968 bus_dma_segment_t *segs, int nsegs, int error) 969 { 970 971 if (segs == NULL) 972 segs = dmat->segments; 973 return (segs); 974 } 975 976 /* 977 * Release the mapping held by map. 978 */ 979 static void 980 bounce_bus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map) 981 { 982 free_bounce_pages(dmat, map); 983 map->sync_count = 0; 984 map->flags &= ~DMAMAP_MBUF; 985 } 986 987 static void 988 dma_preread_safe(char *va, vm_size_t size) 989 { 990 /* 991 * Write back any partial cachelines immediately before and 992 * after the DMA region. 993 */ 994 if (!__is_aligned(va, dcache_line_size)) 995 cpu_dcache_wb_range(va, 1); 996 if (!__is_aligned(va + size, dcache_line_size)) 997 cpu_dcache_wb_range(va + size, 1); 998 999 cpu_dcache_inv_range(va, size); 1000 } 1001 1002 static void 1003 dma_dcache_sync(struct sync_list *sl, bus_dmasync_op_t op) 1004 { 1005 uint32_t len, offset; 1006 vm_page_t m; 1007 vm_paddr_t pa; 1008 vm_offset_t va, tempva; 1009 bus_size_t size; 1010 1011 offset = sl->paddr & PAGE_MASK; 1012 m = sl->pages; 1013 size = sl->datacount; 1014 pa = sl->paddr; 1015 1016 for ( ; size != 0; size -= len, pa += len, offset = 0, ++m) { 1017 tempva = 0; 1018 if (sl->vaddr == 0) { 1019 len = min(PAGE_SIZE - offset, size); 1020 tempva = pmap_quick_enter_page(m); 1021 va = tempva | offset; 1022 KASSERT(pa == (VM_PAGE_TO_PHYS(m) | offset), 1023 ("unexpected vm_page_t phys: 0x%16lx != 0x%16lx", 1024 VM_PAGE_TO_PHYS(m) | offset, pa)); 1025 } else { 1026 len = sl->datacount; 1027 va = sl->vaddr; 1028 } 1029 1030 switch (op) { 1031 case BUS_DMASYNC_PREWRITE: 1032 case BUS_DMASYNC_PREWRITE | BUS_DMASYNC_PREREAD: 1033 cpu_dcache_wb_range((void *)va, len); 1034 break; 1035 case BUS_DMASYNC_PREREAD: 1036 /* 1037 * An mbuf may start in the middle of a cacheline. There 1038 * will be no cpu writes to the beginning of that line 1039 * (which contains the mbuf header) while dma is in 1040 * progress. Handle that case by doing a writeback of 1041 * just the first cacheline before invalidating the 1042 * overall buffer. Any mbuf in a chain may have this 1043 * misalignment. Buffers which are not mbufs bounce if 1044 * they are not aligned to a cacheline. 1045 */ 1046 dma_preread_safe((void *)va, len); 1047 break; 1048 case BUS_DMASYNC_POSTREAD: 1049 case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE: 1050 cpu_dcache_inv_range((void *)va, len); 1051 break; 1052 default: 1053 panic("unsupported combination of sync operations: " 1054 "0x%08x\n", op); 1055 } 1056 1057 if (tempva != 0) 1058 pmap_quick_remove_page(tempva); 1059 } 1060 } 1061 1062 static void 1063 bounce_bus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, 1064 bus_dmasync_op_t op) 1065 { 1066 struct bounce_page *bpage; 1067 struct sync_list *sl, *end; 1068 vm_offset_t datavaddr, tempvaddr; 1069 1070 if (op == BUS_DMASYNC_POSTWRITE) 1071 return; 1072 1073 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1074 /* 1075 * Wait for any DMA operations to complete before the bcopy. 1076 */ 1077 dsb(sy); 1078 } 1079 1080 if ((bpage = STAILQ_FIRST(&map->bpages)) != NULL) { 1081 CTR4(KTR_BUSDMA, "%s: tag %p tag flags 0x%x op 0x%x " 1082 "performing bounce", __func__, dmat, dmat->common.flags, 1083 op); 1084 1085 if ((op & BUS_DMASYNC_PREWRITE) != 0) { 1086 while (bpage != NULL) { 1087 tempvaddr = 0; 1088 datavaddr = bpage->datavaddr; 1089 if (datavaddr == 0) { 1090 tempvaddr = pmap_quick_enter_page( 1091 bpage->datapage); 1092 datavaddr = tempvaddr | bpage->dataoffs; 1093 } 1094 1095 bcopy((void *)datavaddr, 1096 (void *)bpage->vaddr, bpage->datacount); 1097 if (tempvaddr != 0) 1098 pmap_quick_remove_page(tempvaddr); 1099 if ((map->flags & DMAMAP_COHERENT) == 0) 1100 cpu_dcache_wb_range((void *)bpage->vaddr, 1101 bpage->datacount); 1102 bpage = STAILQ_NEXT(bpage, links); 1103 } 1104 dmat->bounce_zone->total_bounced++; 1105 } else if ((op & BUS_DMASYNC_PREREAD) != 0) { 1106 while (bpage != NULL) { 1107 if ((map->flags & DMAMAP_COHERENT) == 0) 1108 cpu_dcache_wbinv_range((void *)bpage->vaddr, 1109 bpage->datacount); 1110 bpage = STAILQ_NEXT(bpage, links); 1111 } 1112 } 1113 1114 if ((op & BUS_DMASYNC_POSTREAD) != 0) { 1115 while (bpage != NULL) { 1116 if ((map->flags & DMAMAP_COHERENT) == 0) 1117 cpu_dcache_inv_range((void *)bpage->vaddr, 1118 bpage->datacount); 1119 tempvaddr = 0; 1120 datavaddr = bpage->datavaddr; 1121 if (datavaddr == 0) { 1122 tempvaddr = pmap_quick_enter_page( 1123 bpage->datapage); 1124 datavaddr = tempvaddr | bpage->dataoffs; 1125 } 1126 1127 bcopy((void *)bpage->vaddr, 1128 (void *)datavaddr, bpage->datacount); 1129 1130 if (tempvaddr != 0) 1131 pmap_quick_remove_page(tempvaddr); 1132 bpage = STAILQ_NEXT(bpage, links); 1133 } 1134 dmat->bounce_zone->total_bounced++; 1135 } 1136 } 1137 1138 /* 1139 * Cache maintenance for normal (non-COHERENT non-bounce) buffers. 1140 */ 1141 if (map->sync_count != 0) { 1142 sl = &map->slist[0]; 1143 end = &map->slist[map->sync_count]; 1144 CTR3(KTR_BUSDMA, "%s: tag %p op 0x%x " 1145 "performing sync", __func__, dmat, op); 1146 1147 for ( ; sl != end; ++sl) 1148 dma_dcache_sync(sl, op); 1149 } 1150 1151 if ((op & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) != 0) { 1152 /* 1153 * Wait for the bcopy to complete before any DMA operations. 1154 */ 1155 dsb(sy); 1156 } 1157 1158 kmsan_bus_dmamap_sync(&map->kmsan_mem, op); 1159 } 1160 1161 #ifdef KMSAN 1162 static void 1163 bounce_bus_dmamap_load_kmsan(bus_dmamap_t map, struct memdesc *mem) 1164 { 1165 if (map == NULL) 1166 return; 1167 memcpy(&map->kmsan_mem, mem, sizeof(map->kmsan_mem)); 1168 } 1169 #endif 1170 1171 struct bus_dma_impl bus_dma_bounce_impl = { 1172 .tag_create = bounce_bus_dma_tag_create, 1173 .tag_destroy = bounce_bus_dma_tag_destroy, 1174 .tag_set_domain = bounce_bus_dma_tag_set_domain, 1175 .id_mapped = bounce_bus_dma_id_mapped, 1176 .map_create = bounce_bus_dmamap_create, 1177 .map_destroy = bounce_bus_dmamap_destroy, 1178 .mem_alloc = bounce_bus_dmamem_alloc, 1179 .mem_free = bounce_bus_dmamem_free, 1180 .load_phys = bounce_bus_dmamap_load_phys, 1181 .load_buffer = bounce_bus_dmamap_load_buffer, 1182 .load_ma = bus_dmamap_load_ma_triv, 1183 .map_waitok = bounce_bus_dmamap_waitok, 1184 .map_complete = bounce_bus_dmamap_complete, 1185 .map_unload = bounce_bus_dmamap_unload, 1186 .map_sync = bounce_bus_dmamap_sync, 1187 #ifdef KMSAN 1188 .load_kmsan = bounce_bus_dmamap_load_kmsan, 1189 #endif 1190 }; 1191