1 /* $OpenBSD: iommu.c,v 1.73 2016/05/04 18:26:12 kettenis Exp $ */ 2 /* $NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $ */ 3 4 /* 5 * Copyright (c) 2003 Henric Jungheim 6 * Copyright (c) 2001, 2002 Eduardo Horvath 7 * Copyright (c) 1999, 2000 Matthew R. Green 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * UltraSPARC IOMMU support; used by both the sbus and pci code. 36 */ 37 #include <sys/param.h> 38 #include <sys/extent.h> 39 #include <sys/malloc.h> 40 #include <sys/systm.h> 41 #include <sys/proc.h> 42 #include <sys/device.h> 43 #include <sys/mbuf.h> 44 45 #include <uvm/uvm_extern.h> 46 47 #include <machine/bus.h> 48 #include <sparc64/sparc64/cache.h> 49 #include <sparc64/dev/iommureg.h> 50 #include <sparc64/dev/iommuvar.h> 51 52 #include <machine/autoconf.h> 53 #include <machine/cpu.h> 54 55 #ifdef DDB 56 #include <machine/db_machdep.h> 57 #include <ddb/db_sym.h> 58 #include <ddb/db_extern.h> 59 #endif 60 61 #ifdef DEBUG 62 #define IDB_BUSDMA 0x1 63 #define IDB_IOMMU 0x2 64 #define IDB_INFO 0x4 65 #define IDB_SYNC 0x8 66 #define IDB_XXX 0x10 67 #define IDB_PRINT_MAP 0x20 68 #define IDB_BREAK 0x40 69 int iommudebug = IDB_INFO; 70 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 71 #else 72 #define DPRINTF(l, s) 73 #endif 74 75 void iommu_enter(struct iommu_state *, struct strbuf_ctl *, bus_addr_t, 76 paddr_t, int); 77 void iommu_remove(struct iommu_state *, struct strbuf_ctl *, bus_addr_t); 78 int iommu_dvmamap_sync_range(struct strbuf_ctl*, bus_addr_t, bus_size_t); 79 int iommu_strbuf_flush_done(struct iommu_map_state *); 80 int iommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *, 81 bus_dmamap_t, bus_dma_segment_t *, int, int, bus_size_t, bus_size_t); 82 int iommu_dvmamap_load_mlist(bus_dma_tag_t, struct iommu_state *, 83 bus_dmamap_t, struct pglist *, int, bus_size_t, bus_size_t); 84 int iommu_dvmamap_validate_map(bus_dma_tag_t, struct iommu_state *, 85 bus_dmamap_t); 86 void iommu_dvmamap_print_map(bus_dma_tag_t, struct iommu_state *, 87 bus_dmamap_t); 88 int iommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t, 89 bus_size_t, int, bus_size_t); 90 int64_t iommu_tsb_entry(struct iommu_state *, bus_addr_t); 91 void strbuf_reset(struct strbuf_ctl *); 92 int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t); 93 bus_addr_t iommu_iomap_translate(struct iommu_map_state *, paddr_t); 94 void iommu_iomap_load_map(struct iommu_state *, struct iommu_map_state *, 95 bus_addr_t, int); 96 void iommu_iomap_unload_map(struct iommu_state *, struct iommu_map_state *); 97 struct iommu_map_state *iommu_iomap_create(int); 98 void iommu_iomap_destroy(struct iommu_map_state *); 99 void iommu_iomap_clear_pages(struct iommu_map_state *); 100 void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t, 101 bus_addr_t, bus_size_t, int); 102 103 /* 104 * Initiate an STC entry flush. 105 */ 106 static inline void 107 iommu_strbuf_flush(struct strbuf_ctl *sb, bus_addr_t va) 108 { 109 #ifdef DEBUG 110 if (sb->sb_flush == NULL) { 111 printf("iommu_strbuf_flush: attempting to flush w/o STC\n"); 112 return; 113 } 114 #endif 115 116 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 117 STRBUFREG(strbuf_pgflush), va); 118 } 119 120 /* 121 * initialise the UltraSPARC IOMMU (SBus or PCI): 122 * - allocate and setup the iotsb. 123 * - enable the IOMMU 124 * - initialise the streaming buffers (if they exist) 125 * - create a private DVMA map. 126 */ 127 void 128 iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase) 129 { 130 psize_t size; 131 vaddr_t va; 132 paddr_t pa; 133 struct vm_page *m; 134 struct pglist mlist; 135 136 /* 137 * Setup the iommu. 138 * 139 * The sun4u iommu is part of the SBus or PCI controller so we will 140 * deal with it here.. 141 * 142 * For sysio and psycho/psycho+ the IOMMU address space always ends at 143 * 0xffffe000, but the starting address depends on the size of the 144 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 145 * entry is 8 bytes. The start of the map can be calculated by 146 * (0xffffe000 << (8 + is->is_tsbsize)). 147 * 148 * But sabre and hummingbird use a different scheme that seems to 149 * be hard-wired, so we read the start and size from the PROM and 150 * just use those values. 151 */ 152 if (strncmp(name, "pyro", 4) == 0) { 153 is->is_cr = IOMMUREG_READ(is, iommu_cr); 154 is->is_cr &= ~IOMMUCR_FIRE_BE; 155 is->is_cr |= (IOMMUCR_FIRE_SE | IOMMUCR_FIRE_CM_EN | 156 IOMMUCR_FIRE_TE); 157 } else 158 is->is_cr = IOMMUCR_EN; 159 is->is_tsbsize = tsbsize; 160 if (iovabase == (u_int32_t)-1) { 161 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 162 is->is_dvmaend = IOTSB_VEND; 163 } else { 164 is->is_dvmabase = iovabase; 165 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; 166 } 167 168 /* 169 * Allocate memory for I/O pagetables. They need to be physically 170 * contiguous. 171 */ 172 173 size = PAGE_SIZE << is->is_tsbsize; 174 TAILQ_INIT(&mlist); 175 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 176 (paddr_t)PAGE_SIZE, (paddr_t)0, &mlist, 1, UVM_PLA_NOWAIT) != 0) 177 panic("iommu_init: no memory"); 178 179 va = (vaddr_t)km_alloc(size, &kv_any, &kp_none, &kd_nowait); 180 if (va == 0) 181 panic("iommu_init: no memory"); 182 is->is_tsb = (int64_t *)va; 183 184 m = TAILQ_FIRST(&mlist); 185 is->is_ptsb = VM_PAGE_TO_PHYS(m); 186 187 /* Map the pages */ 188 for (; m != NULL; m = TAILQ_NEXT(m,pageq)) { 189 pa = VM_PAGE_TO_PHYS(m); 190 pmap_enter(pmap_kernel(), va, pa | PMAP_NVC, 191 PROT_READ | PROT_WRITE, 192 PROT_READ | PROT_WRITE | PMAP_WIRED); 193 va += PAGE_SIZE; 194 } 195 pmap_update(pmap_kernel()); 196 memset(is->is_tsb, 0, size); 197 198 TAILQ_INIT(&mlist); 199 if (uvm_pglistalloc(PAGE_SIZE, 0, -1, PAGE_SIZE, 0, &mlist, 1, 200 UVM_PLA_NOWAIT | UVM_PLA_ZERO) != 0) 201 panic("%s: no memory", __func__); 202 m = TAILQ_FIRST(&mlist); 203 is->is_scratch = VM_PAGE_TO_PHYS(m); 204 205 #ifdef DEBUG 206 if (iommudebug & IDB_INFO) { 207 /* Probe the iommu */ 208 /* The address or contents of the regs...? */ 209 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 210 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 211 IOMMUREG(iommu_cr), 212 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 213 IOMMUREG(iommu_tsb), 214 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 215 IOMMUREG(iommu_flush)); 216 printf("iommu cr=%llx tsb=%llx\n", 217 IOMMUREG_READ(is, iommu_cr), 218 IOMMUREG_READ(is, iommu_tsb)); 219 printf("TSB base %p phys %llx\n", 220 (void *)is->is_tsb, (unsigned long long)is->is_ptsb); 221 delay(1000000); /* 1 s */ 222 } 223 #endif 224 225 /* 226 * Now all the hardware's working we need to allocate a dvma map. 227 */ 228 printf("dvma map %x-%x", is->is_dvmabase, is->is_dvmaend); 229 #ifdef DEBUG 230 printf(", iotdb %llx-%llx", 231 (unsigned long long)is->is_ptsb, 232 (unsigned long long)(is->is_ptsb + size)); 233 #endif 234 is->is_dvmamap = extent_create(name, 235 is->is_dvmabase, (u_long)is->is_dvmaend + 1, 236 M_DEVBUF, NULL, 0, EX_NOCOALESCE); 237 mtx_init(&is->is_mtx, IPL_HIGH); 238 239 /* 240 * Set the TSB size. The relevant bits were moved to the TSB 241 * base register in the PCIe host bridges. 242 */ 243 if (strncmp(name, "pyro", 4) == 0) 244 is->is_ptsb |= is->is_tsbsize; 245 else 246 is->is_cr |= (is->is_tsbsize << 16); 247 248 /* 249 * Now actually start up the IOMMU. 250 */ 251 iommu_reset(is); 252 printf("\n"); 253 } 254 255 /* 256 * Streaming buffers don't exist on the UltraSPARC IIi/e; we should have 257 * detected that already and disabled them. If not, we will notice that 258 * they aren't there when the STRBUF_EN bit does not remain. 259 */ 260 void 261 iommu_reset(struct iommu_state *is) 262 { 263 int i; 264 265 IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); 266 267 /* Enable IOMMU */ 268 IOMMUREG_WRITE(is, iommu_cr, is->is_cr); 269 270 for (i = 0; i < 2; ++i) { 271 struct strbuf_ctl *sb = is->is_sb[i]; 272 273 if (sb == NULL) 274 continue; 275 276 sb->sb_iommu = is; 277 strbuf_reset(sb); 278 279 if (sb->sb_flush) 280 printf(", STC%d enabled", i); 281 } 282 283 if (is->is_flags & IOMMU_FLUSH_CACHE) 284 IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL); 285 } 286 287 /* 288 * Initialize one STC. 289 */ 290 void 291 strbuf_reset(struct strbuf_ctl *sb) 292 { 293 if(sb->sb_flush == NULL) 294 return; 295 296 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 297 STRBUFREG(strbuf_ctl), STRBUF_EN); 298 299 membar(Lookaside); 300 301 /* No streaming buffers? Disable them */ 302 if (bus_space_read_8(sb->sb_bustag, sb->sb_sb, 303 STRBUFREG(strbuf_ctl)) == 0) { 304 sb->sb_flush = NULL; 305 } else { 306 /* 307 * locate the pa of the flush buffer 308 */ 309 if (pmap_extract(pmap_kernel(), 310 (vaddr_t)sb->sb_flush, &sb->sb_flushpa) == FALSE) 311 sb->sb_flush = NULL; 312 mtx_init(&sb->sb_mtx, IPL_HIGH); 313 } 314 } 315 316 /* 317 * Add an entry to the IOMMU table. 318 * 319 * The entry is marked streaming if an STC was detected and 320 * the BUS_DMA_STREAMING flag is set. 321 */ 322 void 323 iommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va, 324 paddr_t pa, int flags) 325 { 326 int64_t tte; 327 volatile int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]; 328 329 #ifdef DIAGNOSTIC 330 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 331 panic("iommu_enter: va %#lx not in DVMA space", va); 332 333 tte = *tte_ptr; 334 335 if (tte & IOTTE_V) { 336 printf("Overwriting valid tte entry (dva %lx pa %lx " 337 "&tte %p tte %llx)\n", va, pa, tte_ptr, tte); 338 extent_print(is->is_dvmamap); 339 panic("IOMMU overwrite"); 340 } 341 #endif 342 343 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 344 !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING)); 345 346 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 347 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 348 349 *tte_ptr = tte; 350 351 /* 352 * Why bother to flush this va? It should only be relevant for 353 * V ==> V or V ==> non-V transitions. The former is illegal and 354 * the latter is never done here. It is true that this provides 355 * some protection against a misbehaving master using an address 356 * after it should. The IOMMU documentations specifically warns 357 * that the consequences of a simultaneous IOMMU flush and DVMA 358 * access to the same address are undefined. (By that argument, 359 * the STC should probably be flushed as well.) Note that if 360 * a bus master keeps using a memory region after it has been 361 * unmapped, the specific behavior of the IOMMU is likely to 362 * be the least of our worries. 363 */ 364 IOMMUREG_WRITE(is, iommu_flush, va); 365 366 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 367 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 368 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 369 (u_long)tte)); 370 } 371 372 /* 373 * Remove an entry from the IOMMU table. 374 * 375 * The entry is flushed from the STC if an STC is detected and the TSB 376 * entry has the IOTTE_STREAM flags set. It should be impossible for 377 * the TSB entry to have this flag set without the BUS_DMA_STREAMING 378 * flag, but better to be safe. (The IOMMU will be ignored as long 379 * as an STC entry exists.) 380 */ 381 void 382 iommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va) 383 { 384 int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]; 385 int64_t tte; 386 387 #ifdef DIAGNOSTIC 388 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 389 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 390 if (va != trunc_page(va)) { 391 printf("iommu_remove: unaligned va: %lx\n", va); 392 va = trunc_page(va); 393 } 394 #endif 395 tte = *tte_ptr; 396 397 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%llx]@%p\n", 398 va, tte, tte_ptr)); 399 400 #ifdef DIAGNOSTIC 401 if ((tte & IOTTE_V) == 0) { 402 printf("Removing invalid tte entry (dva %lx &tte %p " 403 "tte %llx)\n", va, tte_ptr, tte); 404 extent_print(is->is_dvmamap); 405 panic("IOMMU remove overwrite"); 406 } 407 #endif 408 409 *tte_ptr = tte & ~IOTTE_V; 410 411 /* 412 * IO operations are strongly ordered WRT each other. It is 413 * unclear how they relate to normal memory accesses. 414 */ 415 membar(StoreStore); 416 417 IOMMUREG_WRITE(is, iommu_flush, va); 418 419 if (sb && (tte & IOTTE_STREAM)) 420 iommu_strbuf_flush(sb, va); 421 422 /* Should we sync the iommu and stc here? */ 423 } 424 425 /* 426 * Find the physical address of a DVMA address (debug routine). 427 */ 428 paddr_t 429 iommu_extract(struct iommu_state *is, bus_addr_t dva) 430 { 431 int64_t tte = 0; 432 433 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 434 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 435 436 return (tte & IOTTE_PAMASK); 437 } 438 439 /* 440 * Lookup a TSB entry for a given DVMA (debug routine). 441 */ 442 int64_t 443 iommu_lookup_tte(struct iommu_state *is, bus_addr_t dva) 444 { 445 int64_t tte = 0; 446 447 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 448 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 449 450 return (tte); 451 } 452 453 /* 454 * Lookup a TSB entry at a given physical address (debug routine). 455 */ 456 int64_t 457 iommu_fetch_tte(struct iommu_state *is, paddr_t pa) 458 { 459 int64_t tte = 0; 460 461 if (pa >= is->is_ptsb && pa < is->is_ptsb + 462 (PAGE_SIZE << is->is_tsbsize)) 463 tte = ldxa(pa, ASI_PHYS_CACHED); 464 465 return (tte); 466 } 467 468 /* 469 * Fetch a TSB entry with some sanity checking. 470 */ 471 int64_t 472 iommu_tsb_entry(struct iommu_state *is, bus_addr_t dva) 473 { 474 int64_t tte; 475 476 if (dva < is->is_dvmabase || dva > is->is_dvmaend) 477 panic("invalid dva: %llx", (long long)dva); 478 479 tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)]; 480 481 if ((tte & IOTTE_V) == 0) 482 panic("iommu_tsb_entry: invalid entry %lx", dva); 483 484 return (tte); 485 } 486 487 /* 488 * Initiate and then block until an STC flush synchronization has completed. 489 */ 490 int 491 iommu_strbuf_flush_done(struct iommu_map_state *ims) 492 { 493 struct strbuf_ctl *sb = ims->ims_sb; 494 struct strbuf_flush *sf = &ims->ims_flush; 495 struct timeval cur, flushtimeout; 496 struct timeval to = { 0, 500000 }; 497 u_int64_t flush; 498 int timeout_started = 0; 499 500 #ifdef DIAGNOSTIC 501 if (sb == NULL) { 502 panic("iommu_strbuf_flush_done: invalid flush buffer"); 503 } 504 #endif 505 506 mtx_enter(&sb->sb_mtx); 507 508 /* 509 * Streaming buffer flushes: 510 * 511 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. 512 * 2 Store 0 in flag 513 * 3 Store pointer to flag in flushsync 514 * 4 wait till flushsync becomes 0x1 515 * 516 * If it takes more than .5 sec, something went very, very wrong. 517 */ 518 519 /* 520 * If we're reading from ASI_PHYS_CACHED, then we'll write to 521 * it too. No need to tempt fate or learn about Si bugs or such. 522 * FreeBSD just uses normal "volatile" reads/writes... 523 */ 524 525 stxa(sf->sbf_flushpa, ASI_PHYS_CACHED, 0); 526 527 /* 528 * Insure any previous strbuf operations are complete and that 529 * memory is initialized before the IOMMU uses it. 530 * Is this Needed? How are IO and memory operations ordered? 531 */ 532 membar(StoreStore); 533 534 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 535 STRBUFREG(strbuf_flushsync), sf->sbf_flushpa); 536 537 DPRINTF(IDB_IOMMU, 538 ("iommu_strbuf_flush_done: flush = %llx pa = %lx\n", 539 ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), sf->sbf_flushpa)); 540 541 membar(StoreLoad | Lookaside); 542 543 for(;;) { 544 int i; 545 546 /* 547 * Try to shave a few instruction cycles off the average 548 * latency by only checking the elapsed time every few 549 * fetches. 550 */ 551 for (i = 0; i < 1000; ++i) { 552 membar(LoadLoad); 553 /* Bypass non-coherent D$ */ 554 /* non-coherent...? Huh? */ 555 flush = ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED); 556 557 if (flush) { 558 DPRINTF(IDB_IOMMU, 559 ("iommu_strbuf_flush_done: flushed\n")); 560 mtx_leave(&sb->sb_mtx); 561 return (0); 562 } 563 } 564 565 microtime(&cur); 566 567 if (timeout_started) { 568 if (timercmp(&cur, &flushtimeout, >)) 569 panic("STC timeout at %lx (%lld)", 570 sf->sbf_flushpa, flush); 571 } else { 572 timeradd(&cur, &to, &flushtimeout); 573 574 timeout_started = 1; 575 576 DPRINTF(IDB_IOMMU, 577 ("iommu_strbuf_flush_done: flush = %llx pa = %lx " 578 "now=%lx:%lx until = %lx:%lx\n", 579 ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), 580 sf->sbf_flushpa, cur.tv_sec, cur.tv_usec, 581 flushtimeout.tv_sec, flushtimeout.tv_usec)); 582 } 583 } 584 } 585 586 /* 587 * IOMMU DVMA operations, common to SBus and PCI. 588 */ 589 590 #define BUS_DMA_FIND_PARENT(t, fn) \ 591 if (t->_parent == NULL) \ 592 panic("null bus_dma parent (" #fn ")"); \ 593 for (t = t->_parent; t->fn == NULL; t = t->_parent) \ 594 if (t->_parent == NULL) \ 595 panic("no bus_dma " #fn " located"); 596 597 int 598 iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb, 599 bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, 600 int flags, bus_dmamap_t *dmamap) 601 { 602 int ret; 603 bus_dmamap_t map; 604 struct iommu_map_state *ims; 605 606 BUS_DMA_FIND_PARENT(t, _dmamap_create); 607 ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary, 608 flags, &map); 609 610 if (ret) 611 return (ret); 612 613 ims = iommu_iomap_create(atop(round_page(size))); 614 615 if (ims == NULL) { 616 bus_dmamap_destroy(t0, map); 617 return (ENOMEM); 618 } 619 620 ims->ims_sb = sb; 621 map->_dm_cookie = ims; 622 623 #ifdef DIAGNOSTIC 624 if (ims->ims_sb == NULL) 625 panic("iommu_dvmamap_create: null sb"); 626 if (ims->ims_sb->sb_iommu == NULL) 627 panic("iommu_dvmamap_create: null iommu"); 628 #endif 629 *dmamap = map; 630 631 return (0); 632 } 633 634 void 635 iommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) 636 { 637 /* 638 * The specification (man page) requires a loaded 639 * map to be unloaded before it is destroyed. 640 */ 641 if (map->dm_nsegs) 642 bus_dmamap_unload(t0, map); 643 644 if (map->_dm_cookie) 645 iommu_iomap_destroy(map->_dm_cookie); 646 map->_dm_cookie = NULL; 647 648 BUS_DMA_FIND_PARENT(t, _dmamap_destroy); 649 (*t->_dmamap_destroy)(t, t0, map); 650 } 651 652 /* 653 * Load a contiguous kva buffer into a dmamap. The physical pages are 654 * not assumed to be contiguous. Two passes are made through the buffer 655 * and both call pmap_extract() for the same va->pa translations. It 656 * is possible to run out of pa->dvma mappings; the code should be smart 657 * enough to resize the iomap (when the "flags" permit allocation). It 658 * is trivial to compute the number of entries required (round the length 659 * up to the page size and then divide by the page size)... 660 */ 661 int 662 iommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 663 void *buf, bus_size_t buflen, struct proc *p, int flags) 664 { 665 int err = 0; 666 bus_size_t sgsize; 667 u_long dvmaddr, sgstart, sgend; 668 bus_size_t align, boundary; 669 struct iommu_state *is; 670 struct iommu_map_state *ims = map->_dm_cookie; 671 pmap_t pmap; 672 673 #ifdef DIAGNOSTIC 674 if (ims == NULL) 675 panic("iommu_dvmamap_load: null map state"); 676 #endif 677 #ifdef DEBUG 678 if (ims->ims_sb == NULL) 679 panic("iommu_dvmamap_load: null sb"); 680 if (ims->ims_sb->sb_iommu == NULL) 681 panic("iommu_dvmamap_load: null iommu"); 682 #endif /* DEBUG */ 683 is = ims->ims_sb->sb_iommu; 684 685 if (map->dm_nsegs) { 686 /* 687 * Is it still in use? _bus_dmamap_load should have taken care 688 * of this. 689 */ 690 #ifdef DIAGNOSTIC 691 panic("iommu_dvmamap_load: map still in use"); 692 #endif 693 bus_dmamap_unload(t0, map); 694 } 695 696 /* 697 * Make sure that on error condition we return "no valid mappings". 698 */ 699 map->dm_nsegs = 0; 700 701 if (buflen < 1 || buflen > map->_dm_size) { 702 DPRINTF(IDB_BUSDMA, 703 ("iommu_dvmamap_load(): error %d > %d -- " 704 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 705 return (EINVAL); 706 } 707 708 /* 709 * A boundary presented to bus_dmamem_alloc() takes precedence 710 * over boundary in the map. 711 */ 712 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 713 boundary = map->_dm_boundary; 714 align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE); 715 716 pmap = p ? p->p_vmspace->vm_map.pmap : pmap_kernel(); 717 718 /* Count up the total number of pages we need */ 719 iommu_iomap_clear_pages(ims); 720 { /* Scope */ 721 bus_addr_t a, aend; 722 bus_addr_t addr = (bus_addr_t)buf; 723 int seg_len = buflen; 724 725 aend = round_page(addr + seg_len); 726 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 727 paddr_t pa; 728 729 if (pmap_extract(pmap, a, &pa) == FALSE) 730 panic("iomap pmap error addr 0x%lx\n", a); 731 732 err = iommu_iomap_insert_page(ims, pa); 733 if (err) { 734 printf("iomap insert error: %d for " 735 "va 0x%lx pa 0x%lx " 736 "(buf %p len %ld/%lx)\n", 737 err, a, pa, buf, buflen, buflen); 738 iommu_dvmamap_print_map(t, is, map); 739 iommu_iomap_clear_pages(ims); 740 return (EFBIG); 741 } 742 } 743 } 744 if (flags & BUS_DMA_OVERRUN) { 745 err = iommu_iomap_insert_page(ims, is->is_scratch); 746 if (err) { 747 iommu_iomap_clear_pages(ims); 748 return (EFBIG); 749 } 750 } 751 sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; 752 753 mtx_enter(&is->is_mtx); 754 if (flags & BUS_DMA_24BIT) { 755 sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); 756 sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); 757 } else { 758 sgstart = is->is_dvmamap->ex_start; 759 sgend = is->is_dvmamap->ex_end; 760 } 761 762 /* 763 * If our segment size is larger than the boundary we need to 764 * split the transfer up into little pieces ourselves. 765 */ 766 err = extent_alloc_subregion_with_descr(is->is_dvmamap, sgstart, sgend, 767 sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 768 EX_NOWAIT | EX_BOUNDZERO, &ims->ims_er, (u_long *)&dvmaddr); 769 mtx_leave(&is->is_mtx); 770 771 #ifdef DEBUG 772 if (err || (dvmaddr == (bus_addr_t)-1)) { 773 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 774 (int)sgsize, flags); 775 #ifdef DDB 776 if (iommudebug & IDB_BREAK) 777 Debugger(); 778 #endif 779 } 780 #endif 781 if (err != 0) { 782 iommu_iomap_clear_pages(ims); 783 return (err); 784 } 785 786 /* Set the active DVMA map */ 787 map->_dm_dvmastart = dvmaddr; 788 map->_dm_dvmasize = sgsize; 789 790 map->dm_mapsize = buflen; 791 792 #ifdef DEBUG 793 iommu_dvmamap_validate_map(t, is, map); 794 #endif 795 796 iommu_iomap_load_map(is, ims, dvmaddr, flags); 797 798 { /* Scope */ 799 bus_addr_t a, aend; 800 bus_addr_t addr = (bus_addr_t)buf; 801 int seg_len = buflen; 802 803 aend = round_page(addr + seg_len); 804 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 805 bus_addr_t pgstart; 806 bus_addr_t pgend; 807 paddr_t pa; 808 int pglen; 809 810 /* Yuck... Redoing the same pmap_extract... */ 811 if (pmap_extract(pmap, a, &pa) == FALSE) 812 panic("iomap pmap error addr 0x%lx\n", a); 813 814 pgstart = pa | (MAX(a, addr) & PAGE_MASK); 815 pgend = pa | (MIN(a + PAGE_SIZE - 1, 816 addr + seg_len - 1) & PAGE_MASK); 817 pglen = pgend - pgstart + 1; 818 819 if (pglen < 1) 820 continue; 821 822 err = iommu_dvmamap_append_range(t, map, pgstart, 823 pglen, flags, boundary); 824 if (err == EFBIG) 825 break; 826 else if (err) { 827 printf("iomap load seg page: %d for " 828 "va 0x%lx pa %lx (%lx - %lx) " 829 "for %d/0x%x\n", 830 err, a, pa, pgstart, pgend, pglen, pglen); 831 break; 832 } 833 } 834 } 835 #ifdef DEBUG 836 iommu_dvmamap_validate_map(t, is, map); 837 838 if (err) 839 printf("**** iommu_dvmamap_load failed with error %d\n", 840 err); 841 842 if (err || (iommudebug & IDB_PRINT_MAP)) { 843 iommu_dvmamap_print_map(t, is, map); 844 #ifdef DDB 845 if (iommudebug & IDB_BREAK) 846 Debugger(); 847 #endif 848 } 849 #endif 850 if (err) 851 iommu_dvmamap_unload(t, t0, map); 852 853 return (err); 854 } 855 856 /* 857 * Load a dvmamap from an array of segs or an mlist (if the first 858 * "segs" entry's mlist is non-null). It calls iommu_dvmamap_load_segs() 859 * or iommu_dvmamap_load_mlist() for part of the 2nd pass through the 860 * mapping. This is ugly. A better solution would probably be to have 861 * function pointers for implementing the traversal. That way, there 862 * could be one core load routine for each of the three required algorithms 863 * (buffer, seg, and mlist). That would also mean that the traversal 864 * algorithm would then only need one implementation for each algorithm 865 * instead of two (one for populating the iomap and one for populating 866 * the dvma map). 867 */ 868 int 869 iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 870 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 871 { 872 int i; 873 int left; 874 int err = 0; 875 bus_size_t sgsize; 876 bus_size_t boundary, align; 877 u_long dvmaddr, sgstart, sgend; 878 struct iommu_state *is; 879 struct iommu_map_state *ims = map->_dm_cookie; 880 881 #ifdef DIAGNOSTIC 882 if (ims == NULL) 883 panic("iommu_dvmamap_load_raw: null map state"); 884 #endif 885 #ifdef DEBUG 886 if (ims->ims_sb == NULL) 887 panic("iommu_dvmamap_load_raw: null sb"); 888 if (ims->ims_sb->sb_iommu == NULL) 889 panic("iommu_dvmamap_load_raw: null iommu"); 890 #endif /* DEBUG */ 891 is = ims->ims_sb->sb_iommu; 892 893 if (map->dm_nsegs) { 894 /* Already in use?? */ 895 #ifdef DIAGNOSTIC 896 panic("iommu_dvmamap_load_raw: map still in use"); 897 #endif 898 bus_dmamap_unload(t0, map); 899 } 900 901 /* 902 * A boundary presented to bus_dmamem_alloc() takes precedence 903 * over boundary in the map. 904 */ 905 if ((boundary = segs[0]._ds_boundary) == 0) 906 boundary = map->_dm_boundary; 907 908 align = MAX(segs[0]._ds_align, PAGE_SIZE); 909 910 /* 911 * Make sure that on error condition we return "no valid mappings". 912 */ 913 map->dm_nsegs = 0; 914 915 iommu_iomap_clear_pages(ims); 916 if (segs[0]._ds_mlist) { 917 struct pglist *mlist = segs[0]._ds_mlist; 918 struct vm_page *m; 919 for (m = TAILQ_FIRST(mlist); m != NULL; 920 m = TAILQ_NEXT(m,pageq)) { 921 err = iommu_iomap_insert_page(ims, VM_PAGE_TO_PHYS(m)); 922 923 if(err) { 924 printf("iomap insert error: %d for " 925 "pa 0x%lx\n", err, VM_PAGE_TO_PHYS(m)); 926 iommu_dvmamap_print_map(t, is, map); 927 iommu_iomap_clear_pages(ims); 928 return (EFBIG); 929 } 930 } 931 } else { 932 /* Count up the total number of pages we need */ 933 for (i = 0, left = size; left > 0 && i < nsegs; i++) { 934 bus_addr_t a, aend; 935 bus_size_t len = segs[i].ds_len; 936 bus_addr_t addr = segs[i].ds_addr; 937 int seg_len = MIN(left, len); 938 939 if (len < 1) 940 continue; 941 942 aend = round_page(addr + seg_len); 943 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 944 945 err = iommu_iomap_insert_page(ims, a); 946 if (err) { 947 printf("iomap insert error: %d for " 948 "pa 0x%lx\n", err, a); 949 iommu_dvmamap_print_map(t, is, map); 950 iommu_iomap_clear_pages(ims); 951 return (EFBIG); 952 } 953 } 954 955 left -= seg_len; 956 } 957 } 958 if (flags & BUS_DMA_OVERRUN) { 959 err = iommu_iomap_insert_page(ims, is->is_scratch); 960 if (err) { 961 iommu_iomap_clear_pages(ims); 962 return (EFBIG); 963 } 964 } 965 sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; 966 967 mtx_enter(&is->is_mtx); 968 if (flags & BUS_DMA_24BIT) { 969 sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); 970 sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); 971 } else { 972 sgstart = is->is_dvmamap->ex_start; 973 sgend = is->is_dvmamap->ex_end; 974 } 975 976 /* 977 * If our segment size is larger than the boundary we need to 978 * split the transfer up into little pieces ourselves. 979 */ 980 err = extent_alloc_subregion_with_descr(is->is_dvmamap, sgstart, sgend, 981 sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 982 EX_NOWAIT | EX_BOUNDZERO, &ims->ims_er, (u_long *)&dvmaddr); 983 mtx_leave(&is->is_mtx); 984 985 if (err != 0) { 986 iommu_iomap_clear_pages(ims); 987 return (err); 988 } 989 990 #ifdef DEBUG 991 if (dvmaddr == (bus_addr_t)-1) { 992 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) " 993 "failed!\n", (int)sgsize, flags); 994 #ifdef DDB 995 if (iommudebug & IDB_BREAK) 996 Debugger(); 997 #else 998 panic(""); 999 #endif 1000 } 1001 #endif 1002 1003 /* Set the active DVMA map */ 1004 map->_dm_dvmastart = dvmaddr; 1005 map->_dm_dvmasize = sgsize; 1006 1007 map->dm_mapsize = size; 1008 1009 #ifdef DEBUG 1010 iommu_dvmamap_validate_map(t, is, map); 1011 #endif 1012 1013 iommu_iomap_load_map(is, ims, dvmaddr, flags); 1014 1015 if (segs[0]._ds_mlist) 1016 err = iommu_dvmamap_load_mlist(t, is, map, segs[0]._ds_mlist, 1017 flags, size, boundary); 1018 else 1019 err = iommu_dvmamap_load_seg(t, is, map, segs, nsegs, 1020 flags, size, boundary); 1021 1022 #ifdef DEBUG 1023 /* The map should be valid even if the load failed */ 1024 if (iommu_dvmamap_validate_map(t, is, map)) { 1025 printf("load size %ld/0x%lx\n", size, size); 1026 if (segs[0]._ds_mlist) 1027 printf("mlist %p\n", segs[0]._ds_mlist); 1028 else { 1029 long tot_len = 0; 1030 long clip_len = 0; 1031 printf("segs %p nsegs %d\n", segs, nsegs); 1032 1033 left = size; 1034 for(i = 0; i < nsegs; i++) { 1035 bus_size_t len = segs[i].ds_len; 1036 bus_addr_t addr = segs[i].ds_addr; 1037 int seg_len = MIN(left, len); 1038 1039 printf("addr %lx len %ld/0x%lx seg_len " 1040 "%ld/0x%lx left %ld/0xl%x\n", addr, 1041 len, len, seg_len, seg_len, left, left); 1042 1043 left -= seg_len; 1044 1045 clip_len += seg_len; 1046 tot_len += segs[i].ds_len; 1047 } 1048 printf("total length %ld/0x%lx total seg. " 1049 "length %ld/0x%lx\n", tot_len, tot_len, clip_len, 1050 clip_len); 1051 } 1052 1053 if (err == 0) 1054 err = 1; 1055 } 1056 1057 if (err) 1058 printf("**** iommu_dvmamap_load_raw failed with error %d\n", 1059 err); 1060 1061 if (err || (iommudebug & IDB_PRINT_MAP)) { 1062 iommu_dvmamap_print_map(t, is, map); 1063 #ifdef DDB 1064 if (iommudebug & IDB_BREAK) 1065 Debugger(); 1066 #endif 1067 } 1068 #endif 1069 if (err) 1070 iommu_dvmamap_unload(t, t0, map); 1071 1072 return (err); 1073 } 1074 1075 /* 1076 * Insert a range of addresses into a loaded map respecting the specified 1077 * boundary and alignment restrictions. The range is specified by its 1078 * physical address and length. The range cannot cross a page boundary. 1079 * This code (along with most of the rest of the function in this file) 1080 * assumes that the IOMMU page size is equal to PAGE_SIZE. 1081 */ 1082 int 1083 iommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa, 1084 bus_size_t length, int flags, bus_size_t boundary) 1085 { 1086 struct iommu_map_state *ims = map->_dm_cookie; 1087 bus_addr_t sgstart, sgend, bd_mask; 1088 bus_dma_segment_t *seg = NULL; 1089 int i = map->dm_nsegs; 1090 1091 #ifdef DEBUG 1092 if (ims == NULL) 1093 panic("iommu_dvmamap_append_range: null map state"); 1094 #endif 1095 1096 sgstart = iommu_iomap_translate(ims, pa); 1097 sgend = sgstart + length - 1; 1098 1099 #ifdef DIAGNOSTIC 1100 if (sgstart == 0 || sgstart > sgend) { 1101 printf("append range invalid mapping for %lx " 1102 "(0x%lx - 0x%lx)\n", pa, sgstart, sgend); 1103 map->dm_nsegs = 0; 1104 return (EINVAL); 1105 } 1106 #endif 1107 1108 #ifdef DEBUG 1109 if (trunc_page(sgstart) != trunc_page(sgend)) { 1110 printf("append range crossing page boundary! " 1111 "pa %lx length %ld/0x%lx sgstart %lx sgend %lx\n", 1112 pa, length, length, sgstart, sgend); 1113 } 1114 #endif 1115 1116 /* 1117 * We will attempt to merge this range with the previous entry 1118 * (if there is one). 1119 */ 1120 if (i > 0) { 1121 seg = &map->dm_segs[i - 1]; 1122 if (sgstart == seg->ds_addr + seg->ds_len) { 1123 length += seg->ds_len; 1124 sgstart = seg->ds_addr; 1125 sgend = sgstart + length - 1; 1126 } else 1127 seg = NULL; 1128 } 1129 1130 if (seg == NULL) { 1131 seg = &map->dm_segs[i]; 1132 if (++i > map->_dm_segcnt) { 1133 map->dm_nsegs = 0; 1134 return (EFBIG); 1135 } 1136 } 1137 1138 /* 1139 * At this point, "i" is the index of the *next* bus_dma_segment_t 1140 * (the segment count, aka map->dm_nsegs) and "seg" points to the 1141 * *current* entry. "length", "sgstart", and "sgend" reflect what 1142 * we intend to put in "*seg". No assumptions should be made about 1143 * the contents of "*seg". Only "boundary" issue can change this 1144 * and "boundary" is often zero, so explicitly test for that case 1145 * (the test is strictly an optimization). 1146 */ 1147 if (boundary != 0) { 1148 bd_mask = ~(boundary - 1); 1149 1150 while ((sgstart & bd_mask) != (sgend & bd_mask)) { 1151 /* 1152 * We are crossing a boundary so fill in the current 1153 * segment with as much as possible, then grab a new 1154 * one. 1155 */ 1156 1157 seg->ds_addr = sgstart; 1158 seg->ds_len = boundary - (sgstart & ~bd_mask); 1159 1160 sgstart += seg->ds_len; /* sgend stays the same */ 1161 length -= seg->ds_len; 1162 1163 seg = &map->dm_segs[i]; 1164 if (++i > map->_dm_segcnt) { 1165 map->dm_nsegs = 0; 1166 return (EFBIG); 1167 } 1168 } 1169 } 1170 1171 seg->ds_addr = sgstart; 1172 seg->ds_len = length; 1173 map->dm_nsegs = i; 1174 1175 return (0); 1176 } 1177 1178 /* 1179 * Populate the iomap from a bus_dma_segment_t array. See note for 1180 * iommu_dvmamap_load() * regarding page entry exhaustion of the iomap. 1181 * This is less of a problem for load_seg, as the number of pages 1182 * is usually similar to the number of segments (nsegs). 1183 */ 1184 int 1185 iommu_dvmamap_load_seg(bus_dma_tag_t t, struct iommu_state *is, 1186 bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags, 1187 bus_size_t size, bus_size_t boundary) 1188 { 1189 int i; 1190 int left; 1191 int seg; 1192 1193 /* 1194 * This segs is made up of individual physical 1195 * segments, probably by _bus_dmamap_load_uio() or 1196 * _bus_dmamap_load_mbuf(). Ignore the mlist and 1197 * load each one individually. 1198 */ 1199 1200 /* 1201 * Keep in mind that each segment could span 1202 * multiple pages and that these are not always 1203 * adjacent. The code is no longer adding dvma 1204 * aliases to the IOMMU. The STC will not cross 1205 * page boundaries anyway and a IOMMU table walk 1206 * vs. what may be a streamed PCI DMA to a ring 1207 * descriptor is probably a wash. It eases TLB 1208 * pressure and in the worst possible case, it is 1209 * only as bad a non-IOMMUed architecture. More 1210 * importantly, the code is not quite as hairy. 1211 * (It's bad enough as it is.) 1212 */ 1213 left = size; 1214 seg = 0; 1215 for (i = 0; left > 0 && i < nsegs; i++) { 1216 bus_addr_t a, aend; 1217 bus_size_t len = segs[i].ds_len; 1218 bus_addr_t addr = segs[i].ds_addr; 1219 int seg_len = MIN(left, len); 1220 1221 if (len < 1) 1222 continue; 1223 1224 aend = round_page(addr + seg_len); 1225 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 1226 bus_addr_t pgstart; 1227 bus_addr_t pgend; 1228 int pglen; 1229 int err; 1230 1231 pgstart = MAX(a, addr); 1232 pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1); 1233 pglen = pgend - pgstart + 1; 1234 1235 if (pglen < 1) 1236 continue; 1237 1238 err = iommu_dvmamap_append_range(t, map, pgstart, 1239 pglen, flags, boundary); 1240 if (err == EFBIG) 1241 return (err); 1242 if (err) { 1243 printf("iomap load seg page: %d for " 1244 "pa 0x%lx (%lx - %lx for %d/%x\n", 1245 err, a, pgstart, pgend, pglen, pglen); 1246 return (err); 1247 } 1248 1249 } 1250 1251 left -= seg_len; 1252 } 1253 return (0); 1254 } 1255 1256 /* 1257 * Populate the iomap from an mlist. See note for iommu_dvmamap_load() 1258 * regarding page entry exhaustion of the iomap. 1259 */ 1260 int 1261 iommu_dvmamap_load_mlist(bus_dma_tag_t t, struct iommu_state *is, 1262 bus_dmamap_t map, struct pglist *mlist, int flags, 1263 bus_size_t size, bus_size_t boundary) 1264 { 1265 struct vm_page *m; 1266 paddr_t pa; 1267 int err; 1268 1269 /* 1270 * This was allocated with bus_dmamem_alloc. 1271 * The pages are on an `mlist'. 1272 */ 1273 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 1274 pa = VM_PAGE_TO_PHYS(m); 1275 1276 err = iommu_dvmamap_append_range(t, map, pa, 1277 MIN(PAGE_SIZE, size), flags, boundary); 1278 if (err == EFBIG) 1279 return (err); 1280 if (err) { 1281 printf("iomap load seg page: %d for pa 0x%lx " 1282 "(%lx - %lx for %d/%x\n", err, pa, pa, 1283 pa + PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1284 return (err); 1285 } 1286 if (size < PAGE_SIZE) 1287 break; 1288 size -= PAGE_SIZE; 1289 } 1290 1291 return (0); 1292 } 1293 1294 /* 1295 * Unload a dvmamap. 1296 */ 1297 void 1298 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) 1299 { 1300 struct iommu_state *is; 1301 struct iommu_map_state *ims = map->_dm_cookie; 1302 bus_addr_t dvmaddr = map->_dm_dvmastart; 1303 bus_size_t sgsize = map->_dm_dvmasize; 1304 int error; 1305 1306 #ifdef DEBUG 1307 if (ims == NULL) 1308 panic("iommu_dvmamap_unload: null map state"); 1309 if (ims->ims_sb == NULL) 1310 panic("iommu_dvmamap_unload: null sb"); 1311 if (ims->ims_sb->sb_iommu == NULL) 1312 panic("iommu_dvmamap_unload: null iommu"); 1313 #endif /* DEBUG */ 1314 1315 is = ims->ims_sb->sb_iommu; 1316 1317 /* Flush the iommu */ 1318 #ifdef DEBUG 1319 if (dvmaddr == 0) { 1320 printf("iommu_dvmamap_unload: No dvmastart\n"); 1321 #ifdef DDB 1322 if (iommudebug & IDB_BREAK) 1323 Debugger(); 1324 #endif 1325 return; 1326 } 1327 1328 iommu_dvmamap_validate_map(t, is, map); 1329 1330 if (iommudebug & IDB_PRINT_MAP) 1331 iommu_dvmamap_print_map(t, is, map); 1332 #endif /* DEBUG */ 1333 1334 /* Remove the IOMMU entries */ 1335 iommu_iomap_unload_map(is, ims); 1336 1337 /* Clear the iomap */ 1338 iommu_iomap_clear_pages(ims); 1339 1340 bus_dmamap_unload(t->_parent, map); 1341 1342 /* Mark the mappings as invalid. */ 1343 map->dm_mapsize = 0; 1344 map->dm_nsegs = 0; 1345 1346 mtx_enter(&is->is_mtx); 1347 error = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT); 1348 map->_dm_dvmastart = 0; 1349 map->_dm_dvmasize = 0; 1350 mtx_leave(&is->is_mtx); 1351 if (error != 0) 1352 printf("warning: %ld of DVMA space lost\n", sgsize); 1353 } 1354 1355 #ifdef DEBUG 1356 /* 1357 * Perform internal consistency checking on a dvmamap. 1358 */ 1359 int 1360 iommu_dvmamap_validate_map(bus_dma_tag_t t, struct iommu_state *is, 1361 bus_dmamap_t map) 1362 { 1363 int err = 0; 1364 int seg; 1365 1366 if (trunc_page(map->_dm_dvmastart) != map->_dm_dvmastart) { 1367 printf("**** dvmastart address not page aligned: %llx", 1368 map->_dm_dvmastart); 1369 err = 1; 1370 } 1371 if (trunc_page(map->_dm_dvmasize) != map->_dm_dvmasize) { 1372 printf("**** dvmasize not a multiple of page size: %llx", 1373 map->_dm_dvmasize); 1374 err = 1; 1375 } 1376 if (map->_dm_dvmastart < is->is_dvmabase || 1377 (round_page(map->_dm_dvmastart + map->_dm_dvmasize) - 1) > 1378 is->is_dvmaend) { 1379 printf("dvmaddr %llx len %llx out of range %x - %x\n", 1380 map->_dm_dvmastart, map->_dm_dvmasize, 1381 is->is_dvmabase, is->is_dvmaend); 1382 err = 1; 1383 } 1384 for (seg = 0; seg < map->dm_nsegs; seg++) { 1385 if (map->dm_segs[seg].ds_addr == 0 || 1386 map->dm_segs[seg].ds_len == 0) { 1387 printf("seg %d null segment dvmaddr %llx len %llx for " 1388 "range %llx len %llx\n", 1389 seg, 1390 map->dm_segs[seg].ds_addr, 1391 map->dm_segs[seg].ds_len, 1392 map->_dm_dvmastart, map->_dm_dvmasize); 1393 err = 1; 1394 } else if (map->dm_segs[seg].ds_addr < map->_dm_dvmastart || 1395 round_page(map->dm_segs[seg].ds_addr + 1396 map->dm_segs[seg].ds_len) > 1397 map->_dm_dvmastart + map->_dm_dvmasize) { 1398 printf("seg %d dvmaddr %llx len %llx out of " 1399 "range %llx len %llx\n", 1400 seg, 1401 map->dm_segs[seg].ds_addr, 1402 map->dm_segs[seg].ds_len, 1403 map->_dm_dvmastart, map->_dm_dvmasize); 1404 err = 1; 1405 } 1406 } 1407 1408 if (err) { 1409 iommu_dvmamap_print_map(t, is, map); 1410 #if defined(DDB) && defined(DEBUG) 1411 if (iommudebug & IDB_BREAK) 1412 Debugger(); 1413 #endif 1414 } 1415 1416 return (err); 1417 } 1418 #endif /* DEBUG */ 1419 1420 void 1421 iommu_dvmamap_print_map(bus_dma_tag_t t, struct iommu_state *is, 1422 bus_dmamap_t map) 1423 { 1424 int seg, i; 1425 long full_len, source_len; 1426 struct mbuf *m; 1427 1428 printf("DVMA %x for %x, mapping %p: dvstart %lx dvsize %lx " 1429 "size %ld/%lx maxsegsz %lx boundary %lx segcnt %d " 1430 "flags %x type %d source %p " 1431 "cookie %p mapsize %lx nsegs %d\n", 1432 is ? is->is_dvmabase : 0, is ? is->is_dvmaend : 0, map, 1433 map->_dm_dvmastart, map->_dm_dvmasize, 1434 map->_dm_size, map->_dm_size, map->_dm_maxsegsz, map->_dm_boundary, 1435 map->_dm_segcnt, map->_dm_flags, map->_dm_type, 1436 map->_dm_source, map->_dm_cookie, map->dm_mapsize, 1437 map->dm_nsegs); 1438 1439 full_len = 0; 1440 for (seg = 0; seg < map->dm_nsegs; seg++) { 1441 printf("seg %d dvmaddr %lx pa %lx len %lx (tte %llx)\n", 1442 seg, map->dm_segs[seg].ds_addr, 1443 is ? iommu_extract(is, map->dm_segs[seg].ds_addr) : 0, 1444 map->dm_segs[seg].ds_len, 1445 is ? iommu_lookup_tte(is, map->dm_segs[seg].ds_addr) : 0); 1446 full_len += map->dm_segs[seg].ds_len; 1447 } 1448 printf("total length = %ld/0x%lx\n", full_len, full_len); 1449 1450 if (map->_dm_source) switch (map->_dm_type) { 1451 case _DM_TYPE_MBUF: 1452 m = map->_dm_source; 1453 if (m->m_flags & M_PKTHDR) 1454 printf("source PKTHDR mbuf (%p) hdr len = %d/0x%x:\n", 1455 m, m->m_pkthdr.len, m->m_pkthdr.len); 1456 else 1457 printf("source mbuf (%p):\n", m); 1458 1459 source_len = 0; 1460 for ( ; m; m = m->m_next) { 1461 vaddr_t vaddr = mtod(m, vaddr_t); 1462 long len = m->m_len; 1463 paddr_t pa; 1464 1465 if (pmap_extract(pmap_kernel(), vaddr, &pa)) 1466 printf("kva %lx pa %lx len %ld/0x%lx\n", 1467 vaddr, pa, len, len); 1468 else 1469 printf("kva %lx pa <invalid> len %ld/0x%lx\n", 1470 vaddr, len, len); 1471 1472 source_len += len; 1473 } 1474 1475 if (full_len != source_len) 1476 printf("mbuf length %ld/0x%lx is %s than mapping " 1477 "length %ld/0x%lx\n", source_len, source_len, 1478 (source_len > full_len) ? "greater" : "less", 1479 full_len, full_len); 1480 else 1481 printf("mbuf length %ld/0x%lx\n", source_len, 1482 source_len); 1483 break; 1484 case _DM_TYPE_LOAD: 1485 case _DM_TYPE_SEGS: 1486 case _DM_TYPE_UIO: 1487 default: 1488 break; 1489 } 1490 1491 if (map->_dm_cookie) { 1492 struct iommu_map_state *ims = map->_dm_cookie; 1493 struct iommu_page_map *ipm = &ims->ims_map; 1494 1495 printf("page map (%p) of size %d with %d entries\n", 1496 ipm, ipm->ipm_maxpage, ipm->ipm_pagecnt); 1497 for (i = 0; i < ipm->ipm_pagecnt; ++i) { 1498 struct iommu_page_entry *e = &ipm->ipm_map[i]; 1499 printf("%d: vmaddr 0x%lx pa 0x%lx\n", i, 1500 e->ipe_va, e->ipe_pa); 1501 } 1502 } else 1503 printf("iommu map state (cookie) is NULL\n"); 1504 } 1505 1506 void 1507 _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 1508 bus_addr_t offset, bus_size_t len, int ops) 1509 { 1510 struct iommu_state *is; 1511 struct iommu_map_state *ims = map->_dm_cookie; 1512 struct strbuf_ctl *sb; 1513 bus_size_t count; 1514 int i, needsflush = 0; 1515 1516 sb = ims->ims_sb; 1517 is = sb->sb_iommu; 1518 1519 for (i = 0; i < map->dm_nsegs; i++) { 1520 if (offset < map->dm_segs[i].ds_len) 1521 break; 1522 offset -= map->dm_segs[i].ds_len; 1523 } 1524 1525 if (i == map->dm_nsegs) 1526 panic("iommu_dvmamap_sync: too short %lu", offset); 1527 1528 for (; len > 0 && i < map->dm_nsegs; i++) { 1529 count = MIN(map->dm_segs[i].ds_len - offset, len); 1530 if (count > 0 && iommu_dvmamap_sync_range(sb, 1531 map->dm_segs[i].ds_addr + offset, count)) 1532 needsflush = 1; 1533 len -= count; 1534 } 1535 1536 #ifdef DIAGNOSTIC 1537 if (i == map->dm_nsegs && len > 0) 1538 panic("iommu_dvmamap_sync: leftover %lu", len); 1539 #endif 1540 1541 if (needsflush) 1542 iommu_strbuf_flush_done(ims); 1543 } 1544 1545 void 1546 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 1547 bus_addr_t offset, bus_size_t len, int ops) 1548 { 1549 struct iommu_map_state *ims = map->_dm_cookie; 1550 1551 #ifdef DIAGNOSTIC 1552 if (ims == NULL) 1553 panic("iommu_dvmamap_sync: null map state"); 1554 if (ims->ims_sb == NULL) 1555 panic("iommu_dvmamap_sync: null sb"); 1556 if (ims->ims_sb->sb_iommu == NULL) 1557 panic("iommu_dvmamap_sync: null iommu"); 1558 #endif 1559 if (len == 0) 1560 return; 1561 1562 if (ops & BUS_DMASYNC_PREWRITE) 1563 membar(MemIssue); 1564 1565 if ((ims->ims_flags & IOMMU_MAP_STREAM) && 1566 (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE))) 1567 _iommu_dvmamap_sync(t, t0, map, offset, len, ops); 1568 1569 if (ops & BUS_DMASYNC_POSTREAD) 1570 membar(MemIssue); 1571 } 1572 1573 /* 1574 * Flush an individual dma segment, returns non-zero if the streaming buffers 1575 * need flushing afterwards. 1576 */ 1577 int 1578 iommu_dvmamap_sync_range(struct strbuf_ctl *sb, bus_addr_t va, bus_size_t len) 1579 { 1580 bus_addr_t vaend; 1581 #ifdef DIAGNOSTIC 1582 struct iommu_state *is = sb->sb_iommu; 1583 1584 if (va < is->is_dvmabase || va > is->is_dvmaend) 1585 panic("invalid va: %llx", (long long)va); 1586 1587 if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) { 1588 printf("iommu_dvmamap_sync_range: attempting to flush " 1589 "non-streaming entry\n"); 1590 return (0); 1591 } 1592 #endif 1593 1594 vaend = (va + len + PAGE_MASK) & ~PAGE_MASK; 1595 va &= ~PAGE_MASK; 1596 1597 #ifdef DIAGNOSTIC 1598 if (va < is->is_dvmabase || (vaend - 1) > is->is_dvmaend) 1599 panic("invalid va range: %llx to %llx (%x to %x)", 1600 (long long)va, (long long)vaend, 1601 is->is_dvmabase, 1602 is->is_dvmaend); 1603 #endif 1604 1605 for ( ; va <= vaend; va += PAGE_SIZE) { 1606 DPRINTF(IDB_BUSDMA, 1607 ("iommu_dvmamap_sync_range: flushing va %p\n", 1608 (void *)(u_long)va)); 1609 iommu_strbuf_flush(sb, va); 1610 } 1611 1612 return (1); 1613 } 1614 1615 int 1616 iommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size, 1617 bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, 1618 int nsegs, int *rsegs, int flags) 1619 { 1620 1621 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx " 1622 "bound %llx segp %p flags %d\n", (unsigned long long)size, 1623 (unsigned long long)alignment, (unsigned long long)boundary, 1624 segs, flags)); 1625 BUS_DMA_FIND_PARENT(t, _dmamem_alloc); 1626 return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary, 1627 segs, nsegs, rsegs, flags | BUS_DMA_DVMA)); 1628 } 1629 1630 void 1631 iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs, 1632 int nsegs) 1633 { 1634 1635 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1636 segs, nsegs)); 1637 BUS_DMA_FIND_PARENT(t, _dmamem_free); 1638 (*t->_dmamem_free)(t, t0, segs, nsegs); 1639 } 1640 1641 /* 1642 * Create a new iomap. 1643 */ 1644 struct iommu_map_state * 1645 iommu_iomap_create(int n) 1646 { 1647 struct iommu_map_state *ims; 1648 struct strbuf_flush *sbf; 1649 vaddr_t va; 1650 1651 /* Safety for heavily fragmented data, such as mbufs */ 1652 n += 4; 1653 if (n < 16) 1654 n = 16; 1655 1656 ims = malloc(sizeof(*ims) + (n - 1) * sizeof(ims->ims_map.ipm_map[0]), 1657 M_DEVBUF, M_NOWAIT | M_ZERO); 1658 if (ims == NULL) 1659 return (NULL); 1660 1661 /* Initialize the map. */ 1662 ims->ims_map.ipm_maxpage = n; 1663 SPLAY_INIT(&ims->ims_map.ipm_tree); 1664 1665 /* Initialize the flush area. */ 1666 sbf = &ims->ims_flush; 1667 va = (vaddr_t)&sbf->sbf_area[0x40]; 1668 va &= ~0x3f; 1669 pmap_extract(pmap_kernel(), va, &sbf->sbf_flushpa); 1670 sbf->sbf_flush = (void *)va; 1671 1672 return (ims); 1673 } 1674 1675 /* 1676 * Destroy an iomap. 1677 */ 1678 void 1679 iommu_iomap_destroy(struct iommu_map_state *ims) 1680 { 1681 #ifdef DIAGNOSTIC 1682 if (ims->ims_map.ipm_pagecnt > 0) 1683 printf("iommu_iomap_destroy: %d page entries in use\n", 1684 ims->ims_map.ipm_pagecnt); 1685 #endif 1686 1687 free(ims, M_DEVBUF, 0); 1688 } 1689 1690 /* 1691 * Utility function used by splay tree to order page entries by pa. 1692 */ 1693 static inline int 1694 iomap_compare(struct iommu_page_entry *a, struct iommu_page_entry *b) 1695 { 1696 return ((a->ipe_pa > b->ipe_pa) ? 1 : 1697 (a->ipe_pa < b->ipe_pa) ? -1 : 0); 1698 } 1699 1700 SPLAY_PROTOTYPE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare); 1701 1702 SPLAY_GENERATE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare); 1703 1704 /* 1705 * Insert a pa entry in the iomap. 1706 */ 1707 int 1708 iommu_iomap_insert_page(struct iommu_map_state *ims, paddr_t pa) 1709 { 1710 struct iommu_page_map *ipm = &ims->ims_map; 1711 struct iommu_page_entry *e; 1712 1713 if (ipm->ipm_pagecnt >= ipm->ipm_maxpage) { 1714 struct iommu_page_entry ipe; 1715 1716 ipe.ipe_pa = pa; 1717 if (SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &ipe)) 1718 return (0); 1719 1720 return (ENOMEM); 1721 } 1722 1723 e = &ipm->ipm_map[ipm->ipm_pagecnt]; 1724 1725 e->ipe_pa = pa; 1726 e->ipe_va = 0; 1727 1728 e = SPLAY_INSERT(iommu_page_tree, &ipm->ipm_tree, e); 1729 1730 /* Duplicates are okay, but only count them once. */ 1731 if (e) 1732 return (0); 1733 1734 ++ipm->ipm_pagecnt; 1735 1736 return (0); 1737 } 1738 1739 /* 1740 * Locate the iomap by filling in the pa->va mapping and inserting it 1741 * into the IOMMU tables. 1742 */ 1743 void 1744 iommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims, 1745 bus_addr_t vmaddr, int flags) 1746 { 1747 struct iommu_page_map *ipm = &ims->ims_map; 1748 struct iommu_page_entry *e; 1749 struct strbuf_ctl *sb = ims->ims_sb; 1750 int i, slot; 1751 1752 if (sb->sb_flush == NULL) 1753 flags &= ~BUS_DMA_STREAMING; 1754 1755 if (flags & BUS_DMA_STREAMING) 1756 ims->ims_flags |= IOMMU_MAP_STREAM; 1757 else 1758 ims->ims_flags &= ~IOMMU_MAP_STREAM; 1759 1760 for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) { 1761 e->ipe_va = vmaddr; 1762 iommu_enter(is, sb, e->ipe_va, e->ipe_pa, flags); 1763 1764 /* Flush cache if necessary. */ 1765 slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); 1766 if (is->is_flags & IOMMU_FLUSH_CACHE && 1767 (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) 1768 IOMMUREG_WRITE(is, iommu_cache_flush, 1769 is->is_ptsb + slot * 8); 1770 1771 vmaddr += PAGE_SIZE; 1772 } 1773 } 1774 1775 /* 1776 * Remove the iomap from the IOMMU. 1777 */ 1778 void 1779 iommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims) 1780 { 1781 struct iommu_page_map *ipm = &ims->ims_map; 1782 struct iommu_page_entry *e; 1783 struct strbuf_ctl *sb = ims->ims_sb; 1784 int i, slot; 1785 1786 for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) { 1787 iommu_remove(is, sb, e->ipe_va); 1788 1789 /* Flush cache if necessary. */ 1790 slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); 1791 if (is->is_flags & IOMMU_FLUSH_CACHE && 1792 (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) 1793 IOMMUREG_WRITE(is, iommu_cache_flush, 1794 is->is_ptsb + slot * 8); 1795 } 1796 } 1797 1798 /* 1799 * Translate a physical address (pa) into a DVMA address. 1800 */ 1801 bus_addr_t 1802 iommu_iomap_translate(struct iommu_map_state *ims, paddr_t pa) 1803 { 1804 struct iommu_page_map *ipm = &ims->ims_map; 1805 struct iommu_page_entry *e; 1806 struct iommu_page_entry pe; 1807 paddr_t offset = pa & PAGE_MASK; 1808 1809 pe.ipe_pa = trunc_page(pa); 1810 1811 e = SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &pe); 1812 1813 if (e == NULL) 1814 return (0); 1815 1816 return (e->ipe_va | offset); 1817 } 1818 1819 /* 1820 * Clear the iomap table and tree. 1821 */ 1822 void 1823 iommu_iomap_clear_pages(struct iommu_map_state *ims) 1824 { 1825 ims->ims_map.ipm_pagecnt = 0; 1826 SPLAY_INIT(&ims->ims_map.ipm_tree); 1827 } 1828 1829