1 /* $OpenBSD: iommu.c,v 1.80 2020/01/01 15:00:07 kn Exp $ */ 2 /* $NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $ */ 3 4 /* 5 * Copyright (c) 2003 Henric Jungheim 6 * Copyright (c) 2001, 2002 Eduardo Horvath 7 * Copyright (c) 1999, 2000 Matthew R. Green 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * UltraSPARC IOMMU support; used by both the sbus and pci code. 36 */ 37 #include <sys/param.h> 38 #include <sys/extent.h> 39 #include <sys/malloc.h> 40 #include <sys/systm.h> 41 #include <sys/proc.h> 42 #include <sys/device.h> 43 #include <sys/mbuf.h> 44 45 #include <uvm/uvm_extern.h> 46 47 #include <machine/bus.h> 48 #include <sparc64/sparc64/cache.h> 49 #include <sparc64/dev/iommureg.h> 50 #include <sparc64/dev/iommuvar.h> 51 52 #include <machine/autoconf.h> 53 #include <machine/cpu.h> 54 55 #ifdef DDB 56 #include <machine/db_machdep.h> 57 #include <ddb/db_sym.h> 58 #include <ddb/db_extern.h> 59 #endif 60 61 #ifdef DEBUG 62 #define IDB_BUSDMA 0x1 63 #define IDB_IOMMU 0x2 64 #define IDB_INFO 0x4 65 #define IDB_SYNC 0x8 66 #define IDB_XXX 0x10 67 #define IDB_PRINT_MAP 0x20 68 #define IDB_BREAK 0x40 69 int iommudebug = IDB_INFO; 70 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 71 #else 72 #define DPRINTF(l, s) 73 #endif 74 75 void iommu_enter(struct iommu_state *, struct strbuf_ctl *, bus_addr_t, 76 paddr_t, int); 77 void iommu_remove(struct iommu_state *, struct strbuf_ctl *, bus_addr_t); 78 int iommu_dvmamap_sync_range(struct strbuf_ctl*, bus_addr_t, bus_size_t); 79 int iommu_strbuf_flush_done(struct iommu_map_state *); 80 int iommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *, 81 bus_dmamap_t, bus_dma_segment_t *, int, int, bus_size_t, bus_size_t); 82 int iommu_dvmamap_load_mlist(bus_dma_tag_t, struct iommu_state *, 83 bus_dmamap_t, struct pglist *, int, bus_size_t, bus_size_t); 84 int iommu_dvmamap_validate_map(bus_dma_tag_t, struct iommu_state *, 85 bus_dmamap_t); 86 void iommu_dvmamap_print_map(bus_dma_tag_t, struct iommu_state *, 87 bus_dmamap_t); 88 int iommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t, 89 bus_size_t, int, bus_size_t); 90 int iommu_dvmamap_insert(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 91 bus_size_t, int, bus_size_t); 92 int64_t iommu_tsb_entry(struct iommu_state *, bus_addr_t); 93 void strbuf_reset(struct strbuf_ctl *); 94 int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t); 95 bus_addr_t iommu_iomap_translate(struct iommu_map_state *, paddr_t); 96 void iommu_iomap_load_map(struct iommu_state *, struct iommu_map_state *, 97 bus_addr_t, int); 98 void iommu_iomap_unload_map(struct iommu_state *, struct iommu_map_state *); 99 struct iommu_map_state *iommu_iomap_create(int); 100 void iommu_iomap_destroy(struct iommu_map_state *); 101 void iommu_iomap_clear_pages(struct iommu_map_state *); 102 void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t, 103 bus_addr_t, bus_size_t, int); 104 105 void iommu_hw_enable(struct iommu_state *); 106 107 const struct iommu_hw iommu_hw_default = { 108 .ihw_enable = iommu_hw_enable, 109 110 .ihw_dvma_pa = IOTTE_PAMASK, 111 112 .ihw_bypass = 0x3fffUL << 50, 113 .ihw_bypass_nc = 0, 114 .ihw_bypass_ro = 0, 115 }; 116 117 void 118 iommu_hw_enable(struct iommu_state *is) 119 { 120 IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); 121 IOMMUREG_WRITE(is, iommu_cr, IOMMUCR_EN | (is->is_tsbsize << 16)); 122 } 123 124 /* 125 * Initiate an STC entry flush. 126 */ 127 static inline void 128 iommu_strbuf_flush(struct strbuf_ctl *sb, bus_addr_t va) 129 { 130 #ifdef DEBUG 131 if (sb->sb_flush == NULL) { 132 printf("iommu_strbuf_flush: attempting to flush w/o STC\n"); 133 return; 134 } 135 #endif 136 137 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 138 STRBUFREG(strbuf_pgflush), va); 139 } 140 141 /* 142 * initialise the UltraSPARC IOMMU (SBus or PCI): 143 * - allocate and setup the iotsb. 144 * - enable the IOMMU 145 * - initialise the streaming buffers (if they exist) 146 * - create a private DVMA map. 147 */ 148 void 149 iommu_init(char *name, const struct iommu_hw *ihw, struct iommu_state *is, 150 int tsbsize, u_int32_t iovabase) 151 { 152 psize_t size; 153 vaddr_t va; 154 paddr_t pa; 155 struct vm_page *m; 156 struct pglist mlist; 157 158 /* 159 * Setup the iommu. 160 * 161 * The sun4u iommu is part of the SBus or PCI controller so we will 162 * deal with it here.. 163 * 164 * For sysio and psycho/psycho+ the IOMMU address space always ends at 165 * 0xffffe000, but the starting address depends on the size of the 166 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 167 * entry is 8 bytes. The start of the map can be calculated by 168 * (0xffffe000 << (8 + is->is_tsbsize)). 169 * 170 * But sabre and hummingbird use a different scheme that seems to 171 * be hard-wired, so we read the start and size from the PROM and 172 * just use those values. 173 */ 174 175 is->is_hw = ihw; 176 177 is->is_tsbsize = tsbsize; 178 if (iovabase == (u_int32_t)-1) { 179 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 180 is->is_dvmaend = IOTSB_VEND; 181 } else { 182 is->is_dvmabase = iovabase; 183 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; 184 } 185 186 /* 187 * Allocate memory for I/O pagetables. They need to be physically 188 * contiguous. 189 */ 190 191 size = PAGE_SIZE << is->is_tsbsize; 192 TAILQ_INIT(&mlist); 193 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 194 (paddr_t)PAGE_SIZE, (paddr_t)0, &mlist, 1, UVM_PLA_NOWAIT) != 0) 195 panic("iommu_init: no memory"); 196 197 va = (vaddr_t)km_alloc(size, &kv_any, &kp_none, &kd_nowait); 198 if (va == 0) 199 panic("iommu_init: no memory"); 200 is->is_tsb = (int64_t *)va; 201 202 m = TAILQ_FIRST(&mlist); 203 is->is_ptsb = VM_PAGE_TO_PHYS(m); 204 205 /* Map the pages */ 206 for (; m != NULL; m = TAILQ_NEXT(m,pageq)) { 207 pa = VM_PAGE_TO_PHYS(m); 208 pmap_enter(pmap_kernel(), va, pa | PMAP_NVC, 209 PROT_READ | PROT_WRITE, 210 PROT_READ | PROT_WRITE | PMAP_WIRED); 211 va += PAGE_SIZE; 212 } 213 pmap_update(pmap_kernel()); 214 memset(is->is_tsb, 0, size); 215 216 TAILQ_INIT(&mlist); 217 if (uvm_pglistalloc(PAGE_SIZE, 0, -1, PAGE_SIZE, 0, &mlist, 1, 218 UVM_PLA_NOWAIT | UVM_PLA_ZERO) != 0) 219 panic("%s: no memory", __func__); 220 m = TAILQ_FIRST(&mlist); 221 is->is_scratch = VM_PAGE_TO_PHYS(m); 222 223 #ifdef DEBUG 224 if (iommudebug & IDB_INFO) { 225 /* Probe the iommu */ 226 /* The address or contents of the regs...? */ 227 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 228 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 229 IOMMUREG(iommu_cr), 230 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 231 IOMMUREG(iommu_tsb), 232 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 233 IOMMUREG(iommu_flush)); 234 printf("iommu cr=%llx tsb=%llx\n", 235 IOMMUREG_READ(is, iommu_cr), 236 IOMMUREG_READ(is, iommu_tsb)); 237 printf("TSB base %p phys %llx\n", 238 (void *)is->is_tsb, (unsigned long long)is->is_ptsb); 239 delay(1000000); /* 1 s */ 240 } 241 #endif 242 243 /* 244 * Now all the hardware's working we need to allocate a dvma map. 245 */ 246 printf("dvma map %x-%x", is->is_dvmabase, is->is_dvmaend); 247 #ifdef DEBUG 248 printf(", iotdb %llx-%llx", 249 (unsigned long long)is->is_ptsb, 250 (unsigned long long)(is->is_ptsb + size)); 251 #endif 252 is->is_dvmamap = extent_create(name, 253 is->is_dvmabase, (u_long)is->is_dvmaend + 1, 254 M_DEVBUF, NULL, 0, EX_NOCOALESCE); 255 mtx_init(&is->is_mtx, IPL_HIGH); 256 257 /* 258 * Now actually start up the IOMMU. 259 */ 260 iommu_reset(is); 261 printf("\n"); 262 } 263 264 /* 265 * Streaming buffers don't exist on the UltraSPARC IIi/e; we should have 266 * detected that already and disabled them. If not, we will notice that 267 * they aren't there when the STRBUF_EN bit does not remain. 268 */ 269 void 270 iommu_reset(struct iommu_state *is) 271 { 272 int i; 273 274 (*is->is_hw->ihw_enable)(is); 275 276 for (i = 0; i < 2; ++i) { 277 struct strbuf_ctl *sb = is->is_sb[i]; 278 279 if (sb == NULL) 280 continue; 281 282 sb->sb_iommu = is; 283 strbuf_reset(sb); 284 285 if (sb->sb_flush) 286 printf(", STC%d enabled", i); 287 } 288 289 if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE)) 290 IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL); 291 } 292 293 /* 294 * Initialize one STC. 295 */ 296 void 297 strbuf_reset(struct strbuf_ctl *sb) 298 { 299 if(sb->sb_flush == NULL) 300 return; 301 302 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 303 STRBUFREG(strbuf_ctl), STRBUF_EN); 304 305 __membar("#Lookaside"); 306 307 /* No streaming buffers? Disable them */ 308 if (bus_space_read_8(sb->sb_bustag, sb->sb_sb, 309 STRBUFREG(strbuf_ctl)) == 0) { 310 sb->sb_flush = NULL; 311 } else { 312 /* 313 * locate the pa of the flush buffer 314 */ 315 if (pmap_extract(pmap_kernel(), 316 (vaddr_t)sb->sb_flush, &sb->sb_flushpa) == FALSE) 317 sb->sb_flush = NULL; 318 mtx_init(&sb->sb_mtx, IPL_HIGH); 319 } 320 } 321 322 /* 323 * Add an entry to the IOMMU table. 324 * 325 * The entry is marked streaming if an STC was detected and 326 * the BUS_DMA_STREAMING flag is set. 327 */ 328 void 329 iommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va, 330 paddr_t pa, int flags) 331 { 332 int64_t tte; 333 volatile int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]; 334 335 #ifdef DIAGNOSTIC 336 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 337 panic("iommu_enter: va %#lx not in DVMA space", va); 338 339 tte = *tte_ptr; 340 341 if (tte & IOTTE_V) { 342 printf("Overwriting valid tte entry (dva %lx pa %lx " 343 "&tte %p tte %llx)\n", va, pa, tte_ptr, tte); 344 extent_print(is->is_dvmamap); 345 panic("IOMMU overwrite"); 346 } 347 #endif 348 349 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 350 !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING)); 351 352 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 353 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 354 355 *tte_ptr = tte; 356 357 /* 358 * Why bother to flush this va? It should only be relevant for 359 * V ==> V or V ==> non-V transitions. The former is illegal and 360 * the latter is never done here. It is true that this provides 361 * some protection against a misbehaving master using an address 362 * after it should. The IOMMU documentations specifically warns 363 * that the consequences of a simultaneous IOMMU flush and DVMA 364 * access to the same address are undefined. (By that argument, 365 * the STC should probably be flushed as well.) Note that if 366 * a bus master keeps using a memory region after it has been 367 * unmapped, the specific behavior of the IOMMU is likely to 368 * be the least of our worries. 369 */ 370 IOMMUREG_WRITE(is, iommu_flush, va); 371 372 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 373 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 374 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 375 (u_long)tte)); 376 } 377 378 /* 379 * Remove an entry from the IOMMU table. 380 * 381 * The entry is flushed from the STC if an STC is detected and the TSB 382 * entry has the IOTTE_STREAM flags set. It should be impossible for 383 * the TSB entry to have this flag set without the BUS_DMA_STREAMING 384 * flag, but better to be safe. (The IOMMU will be ignored as long 385 * as an STC entry exists.) 386 */ 387 void 388 iommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, bus_addr_t va) 389 { 390 int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]; 391 int64_t tte; 392 393 #ifdef DIAGNOSTIC 394 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 395 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 396 if (va != trunc_page(va)) { 397 printf("iommu_remove: unaligned va: %lx\n", va); 398 va = trunc_page(va); 399 } 400 #endif 401 tte = *tte_ptr; 402 403 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%llx]@%p\n", 404 va, tte, tte_ptr)); 405 406 #ifdef DIAGNOSTIC 407 if ((tte & IOTTE_V) == 0) { 408 printf("Removing invalid tte entry (dva %lx &tte %p " 409 "tte %llx)\n", va, tte_ptr, tte); 410 extent_print(is->is_dvmamap); 411 panic("IOMMU remove overwrite"); 412 } 413 #endif 414 415 *tte_ptr = tte & ~IOTTE_V; 416 417 /* 418 * IO operations are strongly ordered WRT each other. It is 419 * unclear how they relate to normal memory accesses. 420 */ 421 __membar("#StoreStore"); 422 423 IOMMUREG_WRITE(is, iommu_flush, va); 424 425 if (sb && (tte & IOTTE_STREAM)) 426 iommu_strbuf_flush(sb, va); 427 428 /* Should we sync the iommu and stc here? */ 429 } 430 431 /* 432 * Find the physical address of a DVMA address (debug routine). 433 */ 434 paddr_t 435 iommu_extract(struct iommu_state *is, bus_addr_t dva) 436 { 437 int64_t tte = 0; 438 439 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 440 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 441 442 return (tte & is->is_hw->ihw_dvma_pa); 443 } 444 445 /* 446 * Lookup a TSB entry for a given DVMA (debug routine). 447 */ 448 int64_t 449 iommu_lookup_tte(struct iommu_state *is, bus_addr_t dva) 450 { 451 int64_t tte = 0; 452 453 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 454 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 455 456 return (tte); 457 } 458 459 /* 460 * Lookup a TSB entry at a given physical address (debug routine). 461 */ 462 int64_t 463 iommu_fetch_tte(struct iommu_state *is, paddr_t pa) 464 { 465 int64_t tte = 0; 466 467 if (pa >= is->is_ptsb && pa < is->is_ptsb + 468 (PAGE_SIZE << is->is_tsbsize)) 469 tte = ldxa(pa, ASI_PHYS_CACHED); 470 471 return (tte); 472 } 473 474 /* 475 * Fetch a TSB entry with some sanity checking. 476 */ 477 int64_t 478 iommu_tsb_entry(struct iommu_state *is, bus_addr_t dva) 479 { 480 int64_t tte; 481 482 if (dva < is->is_dvmabase || dva > is->is_dvmaend) 483 panic("invalid dva: %llx", (long long)dva); 484 485 tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)]; 486 487 if ((tte & IOTTE_V) == 0) 488 panic("iommu_tsb_entry: invalid entry %lx", dva); 489 490 return (tte); 491 } 492 493 /* 494 * Initiate and then block until an STC flush synchronization has completed. 495 */ 496 int 497 iommu_strbuf_flush_done(struct iommu_map_state *ims) 498 { 499 struct strbuf_ctl *sb = ims->ims_sb; 500 struct strbuf_flush *sf = &ims->ims_flush; 501 struct timeval cur, flushtimeout; 502 struct timeval to = { 0, 500000 }; 503 u_int64_t flush; 504 int timeout_started = 0; 505 506 #ifdef DIAGNOSTIC 507 if (sb == NULL) { 508 panic("iommu_strbuf_flush_done: invalid flush buffer"); 509 } 510 #endif 511 512 mtx_enter(&sb->sb_mtx); 513 514 /* 515 * Streaming buffer flushes: 516 * 517 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. 518 * 2 Store 0 in flag 519 * 3 Store pointer to flag in flushsync 520 * 4 wait till flushsync becomes 0x1 521 * 522 * If it takes more than .5 sec, something went very, very wrong. 523 */ 524 525 /* 526 * If we're reading from ASI_PHYS_CACHED, then we'll write to 527 * it too. No need to tempt fate or learn about Si bugs or such. 528 * FreeBSD just uses normal "volatile" reads/writes... 529 */ 530 531 stxa(sf->sbf_flushpa, ASI_PHYS_CACHED, 0); 532 533 /* 534 * Insure any previous strbuf operations are complete and that 535 * memory is initialized before the IOMMU uses it. 536 * Is this Needed? How are IO and memory operations ordered? 537 */ 538 __membar("#StoreStore"); 539 540 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 541 STRBUFREG(strbuf_flushsync), sf->sbf_flushpa); 542 543 DPRINTF(IDB_IOMMU, 544 ("iommu_strbuf_flush_done: flush = %llx pa = %lx\n", 545 ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), sf->sbf_flushpa)); 546 547 __membar("#StoreLoad | #Lookaside"); 548 549 for(;;) { 550 int i; 551 552 /* 553 * Try to shave a few instruction cycles off the average 554 * latency by only checking the elapsed time every few 555 * fetches. 556 */ 557 for (i = 0; i < 1000; ++i) { 558 __membar("#LoadLoad"); 559 /* Bypass non-coherent D$ */ 560 /* non-coherent...? Huh? */ 561 flush = ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED); 562 563 if (flush) { 564 DPRINTF(IDB_IOMMU, 565 ("iommu_strbuf_flush_done: flushed\n")); 566 mtx_leave(&sb->sb_mtx); 567 return (0); 568 } 569 } 570 571 microtime(&cur); 572 573 if (timeout_started) { 574 if (timercmp(&cur, &flushtimeout, >)) 575 panic("STC timeout at %lx (%lld)", 576 sf->sbf_flushpa, flush); 577 } else { 578 timeradd(&cur, &to, &flushtimeout); 579 580 timeout_started = 1; 581 582 DPRINTF(IDB_IOMMU, 583 ("iommu_strbuf_flush_done: flush = %llx pa = %lx " 584 "now=%llx:%lx until = %llx:%lx\n", 585 ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), 586 sf->sbf_flushpa, cur.tv_sec, cur.tv_usec, 587 flushtimeout.tv_sec, flushtimeout.tv_usec)); 588 } 589 } 590 } 591 592 /* 593 * IOMMU DVMA operations, common to SBus and PCI. 594 */ 595 596 #define BUS_DMA_FIND_PARENT(t, fn) \ 597 if (t->_parent == NULL) \ 598 panic("null bus_dma parent (" #fn ")"); \ 599 for (t = t->_parent; t->fn == NULL; t = t->_parent) \ 600 if (t->_parent == NULL) \ 601 panic("no bus_dma " #fn " located"); 602 603 int 604 iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb, 605 bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, 606 int flags, bus_dmamap_t *dmamap) 607 { 608 int ret; 609 bus_dmamap_t map; 610 struct iommu_state *is = sb->sb_iommu; 611 struct iommu_map_state *ims; 612 613 BUS_DMA_FIND_PARENT(t, _dmamap_create); 614 ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary, 615 flags, &map); 616 617 if (ret) 618 return (ret); 619 620 if (flags & BUS_DMA_64BIT) { 621 map->_dm_cookie = is; 622 *dmamap = map; 623 return (0); 624 } 625 626 ims = iommu_iomap_create(atop(round_page(size))); 627 628 if (ims == NULL) { 629 bus_dmamap_destroy(t0, map); 630 return (ENOMEM); 631 } 632 633 ims->ims_sb = sb; 634 map->_dm_cookie = ims; 635 636 #ifdef DIAGNOSTIC 637 if (ims->ims_sb == NULL) 638 panic("iommu_dvmamap_create: null sb"); 639 if (ims->ims_sb->sb_iommu == NULL) 640 panic("iommu_dvmamap_create: null iommu"); 641 #endif 642 *dmamap = map; 643 644 return (0); 645 } 646 647 void 648 iommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) 649 { 650 /* 651 * The specification (man page) requires a loaded 652 * map to be unloaded before it is destroyed. 653 */ 654 if (map->dm_nsegs) 655 bus_dmamap_unload(t0, map); 656 657 if (!ISSET(map->_dm_flags, BUS_DMA_64BIT)) { 658 if (map->_dm_cookie) 659 iommu_iomap_destroy(map->_dm_cookie); 660 } 661 map->_dm_cookie = NULL; 662 663 BUS_DMA_FIND_PARENT(t, _dmamap_destroy); 664 (*t->_dmamap_destroy)(t, t0, map); 665 } 666 667 /* 668 * Load a contiguous kva buffer into a dmamap. The physical pages are 669 * not assumed to be contiguous. Two passes are made through the buffer 670 * and both call pmap_extract() for the same va->pa translations. It 671 * is possible to run out of pa->dvma mappings; the code should be smart 672 * enough to resize the iomap (when the "flags" permit allocation). It 673 * is trivial to compute the number of entries required (round the length 674 * up to the page size and then divide by the page size)... 675 */ 676 int 677 iommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 678 void *buf, bus_size_t buflen, struct proc *p, int flags) 679 { 680 int err = 0; 681 bus_size_t sgsize; 682 u_long dvmaddr, sgstart, sgend; 683 bus_size_t align, boundary; 684 struct iommu_state *is; 685 struct iommu_map_state *ims; 686 pmap_t pmap; 687 688 /* 689 * Make sure that on error condition we return "no valid mappings". 690 */ 691 KASSERTMSG(map->dm_nsegs == 0, "map still in use"); 692 693 if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) { 694 unsigned long bypass; 695 int i; 696 697 is = map->_dm_cookie; 698 bypass = is->is_hw->ihw_bypass; 699 700 /* Bypass translation by the IOMMU. */ 701 702 BUS_DMA_FIND_PARENT(t, _dmamap_load); 703 err = (*t->_dmamap_load)(t, t0, map, buf, buflen, p, flags); 704 if (err != 0) 705 return (err); 706 707 for (i = 0; i < map->dm_nsegs; i++) 708 map->dm_segs[i].ds_addr |= bypass; 709 710 return (0); 711 } 712 713 ims = map->_dm_cookie; 714 is = ims->ims_sb->sb_iommu; 715 716 if (buflen < 1 || buflen > map->_dm_size) { 717 DPRINTF(IDB_BUSDMA, 718 ("iommu_dvmamap_load(): error %d > %d -- " 719 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 720 return (EINVAL); 721 } 722 723 /* 724 * A boundary presented to bus_dmamem_alloc() takes precedence 725 * over boundary in the map. 726 */ 727 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 728 boundary = map->_dm_boundary; 729 align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE); 730 731 pmap = p ? p->p_vmspace->vm_map.pmap : pmap_kernel(); 732 733 /* Count up the total number of pages we need */ 734 iommu_iomap_clear_pages(ims); 735 { /* Scope */ 736 bus_addr_t a, aend; 737 bus_addr_t addr = (bus_addr_t)buf; 738 int seg_len = buflen; 739 740 aend = round_page(addr + seg_len); 741 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 742 paddr_t pa; 743 744 if (pmap_extract(pmap, a, &pa) == FALSE) 745 panic("iomap pmap error addr 0x%lx\n", a); 746 747 err = iommu_iomap_insert_page(ims, pa); 748 if (err) { 749 printf("iomap insert error: %d for " 750 "va 0x%lx pa 0x%lx " 751 "(buf %p len %ld/%lx)\n", 752 err, a, pa, buf, buflen, buflen); 753 iommu_dvmamap_print_map(t, is, map); 754 iommu_iomap_clear_pages(ims); 755 return (EFBIG); 756 } 757 } 758 } 759 if (flags & BUS_DMA_OVERRUN) { 760 err = iommu_iomap_insert_page(ims, is->is_scratch); 761 if (err) { 762 iommu_iomap_clear_pages(ims); 763 return (EFBIG); 764 } 765 } 766 sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; 767 768 mtx_enter(&is->is_mtx); 769 if (flags & BUS_DMA_24BIT) { 770 sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); 771 sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); 772 } else { 773 sgstart = is->is_dvmamap->ex_start; 774 sgend = is->is_dvmamap->ex_end; 775 } 776 777 /* 778 * If our segment size is larger than the boundary we need to 779 * split the transfer up into little pieces ourselves. 780 */ 781 err = extent_alloc_subregion_with_descr(is->is_dvmamap, sgstart, sgend, 782 sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 783 EX_NOWAIT | EX_BOUNDZERO, &ims->ims_er, (u_long *)&dvmaddr); 784 mtx_leave(&is->is_mtx); 785 786 #ifdef DEBUG 787 if (err || (dvmaddr == (bus_addr_t)-1)) { 788 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 789 (int)sgsize, flags); 790 #ifdef DDB 791 if (iommudebug & IDB_BREAK) 792 db_enter(); 793 #endif 794 } 795 #endif 796 if (err != 0) { 797 iommu_iomap_clear_pages(ims); 798 return (err); 799 } 800 801 /* Set the active DVMA map */ 802 map->_dm_dvmastart = dvmaddr; 803 map->_dm_dvmasize = sgsize; 804 805 map->dm_mapsize = buflen; 806 807 #ifdef DEBUG 808 iommu_dvmamap_validate_map(t, is, map); 809 #endif 810 811 iommu_iomap_load_map(is, ims, dvmaddr, flags); 812 813 { /* Scope */ 814 bus_addr_t a, aend; 815 bus_addr_t addr = (bus_addr_t)buf; 816 int seg_len = buflen; 817 818 aend = round_page(addr + seg_len); 819 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 820 bus_addr_t pgstart; 821 bus_addr_t pgend; 822 paddr_t pa; 823 int pglen; 824 825 /* Yuck... Redoing the same pmap_extract... */ 826 if (pmap_extract(pmap, a, &pa) == FALSE) 827 panic("iomap pmap error addr 0x%lx\n", a); 828 829 pgstart = pa | (MAX(a, addr) & PAGE_MASK); 830 pgend = pa | (MIN(a + PAGE_SIZE - 1, 831 addr + seg_len - 1) & PAGE_MASK); 832 pglen = pgend - pgstart + 1; 833 834 if (pglen < 1) 835 continue; 836 837 err = iommu_dvmamap_append_range(t, map, pgstart, 838 pglen, flags, boundary); 839 if (err == EFBIG) 840 break; 841 else if (err) { 842 printf("iomap load seg page: %d for " 843 "va 0x%lx pa %lx (%lx - %lx) " 844 "for %d/0x%x\n", 845 err, a, pa, pgstart, pgend, pglen, pglen); 846 break; 847 } 848 } 849 } 850 #ifdef DEBUG 851 iommu_dvmamap_validate_map(t, is, map); 852 853 if (err) 854 printf("**** iommu_dvmamap_load failed with error %d\n", 855 err); 856 857 if (err || (iommudebug & IDB_PRINT_MAP)) { 858 iommu_dvmamap_print_map(t, is, map); 859 #ifdef DDB 860 if (iommudebug & IDB_BREAK) 861 db_enter(); 862 #endif 863 } 864 #endif 865 if (err) 866 iommu_dvmamap_unload(t, t0, map); 867 868 return (err); 869 } 870 871 /* 872 * Load a dvmamap from an array of segs or an mlist (if the first 873 * "segs" entry's mlist is non-null). It calls iommu_dvmamap_load_segs() 874 * or iommu_dvmamap_load_mlist() for part of the 2nd pass through the 875 * mapping. This is ugly. A better solution would probably be to have 876 * function pointers for implementing the traversal. That way, there 877 * could be one core load routine for each of the three required algorithms 878 * (buffer, seg, and mlist). That would also mean that the traversal 879 * algorithm would then only need one implementation for each algorithm 880 * instead of two (one for populating the iomap and one for populating 881 * the dvma map). 882 */ 883 int 884 iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 885 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 886 { 887 int i; 888 int left; 889 int err = 0; 890 bus_size_t sgsize; 891 bus_size_t boundary, align; 892 u_long dvmaddr, sgstart, sgend; 893 struct iommu_state *is; 894 struct iommu_map_state *ims; 895 896 KASSERTMSG(map->dm_nsegs == 0, "map stil in use"); 897 898 /* 899 * A boundary presented to bus_dmamem_alloc() takes precedence 900 * over boundary in the map. 901 */ 902 if ((boundary = segs[0]._ds_boundary) == 0) 903 boundary = map->_dm_boundary; 904 905 if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) { 906 unsigned long bypass; 907 908 is = map->_dm_cookie; 909 bypass = is->is_hw->ihw_bypass; 910 911 /* Bypass translation by the IOMMU. */ 912 for (i = 0; i < nsegs; i++) { 913 err = iommu_dvmamap_insert(t, map, 914 bypass | segs[i].ds_addr, segs[i].ds_len, 915 0, boundary); 916 if (err != 0) { 917 map->dm_nsegs = 0; 918 return (err); 919 } 920 } 921 922 map->dm_mapsize = size; 923 924 return (0); 925 } 926 927 ims = map->_dm_cookie; 928 is = ims->ims_sb->sb_iommu; 929 930 align = MAX(segs[0]._ds_align, PAGE_SIZE); 931 932 /* 933 * Make sure that on error condition we return "no valid mappings". 934 */ 935 map->dm_nsegs = 0; 936 937 iommu_iomap_clear_pages(ims); 938 if (segs[0]._ds_mlist) { 939 struct pglist *mlist = segs[0]._ds_mlist; 940 struct vm_page *m; 941 for (m = TAILQ_FIRST(mlist); m != NULL; 942 m = TAILQ_NEXT(m,pageq)) { 943 err = iommu_iomap_insert_page(ims, VM_PAGE_TO_PHYS(m)); 944 945 if(err) { 946 printf("iomap insert error: %d for " 947 "pa 0x%lx\n", err, VM_PAGE_TO_PHYS(m)); 948 iommu_dvmamap_print_map(t, is, map); 949 iommu_iomap_clear_pages(ims); 950 return (EFBIG); 951 } 952 } 953 } else { 954 /* Count up the total number of pages we need */ 955 for (i = 0, left = size; left > 0 && i < nsegs; i++) { 956 bus_addr_t a, aend; 957 bus_size_t len = segs[i].ds_len; 958 bus_addr_t addr = segs[i].ds_addr; 959 int seg_len = MIN(left, len); 960 961 if (len < 1) 962 continue; 963 964 aend = round_page(addr + seg_len); 965 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 966 967 err = iommu_iomap_insert_page(ims, a); 968 if (err) { 969 printf("iomap insert error: %d for " 970 "pa 0x%lx\n", err, a); 971 iommu_dvmamap_print_map(t, is, map); 972 iommu_iomap_clear_pages(ims); 973 return (EFBIG); 974 } 975 } 976 977 left -= seg_len; 978 } 979 } 980 if (flags & BUS_DMA_OVERRUN) { 981 err = iommu_iomap_insert_page(ims, is->is_scratch); 982 if (err) { 983 iommu_iomap_clear_pages(ims); 984 return (EFBIG); 985 } 986 } 987 sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; 988 989 mtx_enter(&is->is_mtx); 990 if (flags & BUS_DMA_24BIT) { 991 sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); 992 sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); 993 } else { 994 sgstart = is->is_dvmamap->ex_start; 995 sgend = is->is_dvmamap->ex_end; 996 } 997 998 /* 999 * If our segment size is larger than the boundary we need to 1000 * split the transfer up into little pieces ourselves. 1001 */ 1002 err = extent_alloc_subregion_with_descr(is->is_dvmamap, sgstart, sgend, 1003 sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 1004 EX_NOWAIT | EX_BOUNDZERO, &ims->ims_er, (u_long *)&dvmaddr); 1005 mtx_leave(&is->is_mtx); 1006 1007 if (err != 0) { 1008 iommu_iomap_clear_pages(ims); 1009 return (err); 1010 } 1011 1012 #ifdef DEBUG 1013 if (dvmaddr == (bus_addr_t)-1) { 1014 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) " 1015 "failed!\n", (int)sgsize, flags); 1016 #ifdef DDB 1017 if (iommudebug & IDB_BREAK) 1018 db_enter(); 1019 #else 1020 panic(""); 1021 #endif 1022 } 1023 #endif 1024 1025 /* Set the active DVMA map */ 1026 map->_dm_dvmastart = dvmaddr; 1027 map->_dm_dvmasize = sgsize; 1028 1029 map->dm_mapsize = size; 1030 1031 #ifdef DEBUG 1032 iommu_dvmamap_validate_map(t, is, map); 1033 #endif 1034 1035 iommu_iomap_load_map(is, ims, dvmaddr, flags); 1036 1037 if (segs[0]._ds_mlist) 1038 err = iommu_dvmamap_load_mlist(t, is, map, segs[0]._ds_mlist, 1039 flags, size, boundary); 1040 else 1041 err = iommu_dvmamap_load_seg(t, is, map, segs, nsegs, 1042 flags, size, boundary); 1043 1044 #ifdef DEBUG 1045 /* The map should be valid even if the load failed */ 1046 if (iommu_dvmamap_validate_map(t, is, map)) { 1047 printf("load size %ld/0x%lx\n", size, size); 1048 if (segs[0]._ds_mlist) 1049 printf("mlist %p\n", segs[0]._ds_mlist); 1050 else { 1051 long tot_len = 0; 1052 long clip_len = 0; 1053 printf("segs %p nsegs %d\n", segs, nsegs); 1054 1055 left = size; 1056 for(i = 0; i < nsegs; i++) { 1057 bus_size_t len = segs[i].ds_len; 1058 bus_addr_t addr = segs[i].ds_addr; 1059 int seg_len = MIN(left, len); 1060 1061 printf("addr %lx len %ld/0x%lx seg_len " 1062 "%d/0x%x left %d/0x%x\n", addr, 1063 len, len, seg_len, seg_len, left, left); 1064 1065 left -= seg_len; 1066 1067 clip_len += seg_len; 1068 tot_len += segs[i].ds_len; 1069 } 1070 printf("total length %ld/0x%lx total seg. " 1071 "length %ld/0x%lx\n", tot_len, tot_len, clip_len, 1072 clip_len); 1073 } 1074 1075 if (err == 0) 1076 err = 1; 1077 } 1078 1079 if (err) 1080 printf("**** iommu_dvmamap_load_raw failed with error %d\n", 1081 err); 1082 1083 if (err || (iommudebug & IDB_PRINT_MAP)) { 1084 iommu_dvmamap_print_map(t, is, map); 1085 #ifdef DDB 1086 if (iommudebug & IDB_BREAK) 1087 db_enter(); 1088 #endif 1089 } 1090 #endif 1091 if (err) 1092 iommu_dvmamap_unload(t, t0, map); 1093 1094 return (err); 1095 } 1096 1097 /* 1098 * Insert a range of addresses into a loaded map respecting the specified 1099 * boundary and alignment restrictions. The range is specified by its 1100 * physical address and length. The range cannot cross a page boundary. 1101 * This code (along with most of the rest of the function in this file) 1102 * assumes that the IOMMU page size is equal to PAGE_SIZE. 1103 */ 1104 int 1105 iommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa, 1106 bus_size_t length, int flags, bus_size_t boundary) 1107 { 1108 struct iommu_map_state *ims = map->_dm_cookie; 1109 bus_addr_t sgstart = iommu_iomap_translate(ims, pa); 1110 1111 return (iommu_dvmamap_insert(t, map, sgstart, length, flags, boundary)); 1112 } 1113 1114 int 1115 iommu_dvmamap_insert(bus_dma_tag_t t, bus_dmamap_t map, 1116 bus_addr_t sgstart, bus_size_t length, int flags, bus_size_t boundary) 1117 { 1118 bus_addr_t sgend = sgstart + length - 1; 1119 bus_addr_t bd_mask; 1120 bus_dma_segment_t *seg = NULL; 1121 int i = map->dm_nsegs; 1122 1123 #ifdef DIAGNOSTIC 1124 if (sgstart == 0 || sgstart > sgend) { 1125 printf("append range invalid mapping for " 1126 "0x%lx - 0x%lx\n", sgstart, sgend); 1127 map->dm_nsegs = 0; 1128 return (EINVAL); 1129 } 1130 #endif 1131 1132 #ifdef DEBUG 1133 if (trunc_page(sgstart) != trunc_page(sgend)) { 1134 printf("append range crossing page boundary! " 1135 "length %ld/0x%lx sgstart %lx sgend %lx\n", 1136 length, length, sgstart, sgend); 1137 } 1138 #endif 1139 1140 /* 1141 * We will attempt to merge this range with the previous entry 1142 * (if there is one). 1143 */ 1144 if (i > 0) { 1145 seg = &map->dm_segs[i - 1]; 1146 if (sgstart == seg->ds_addr + seg->ds_len) { 1147 length += seg->ds_len; 1148 sgstart = seg->ds_addr; 1149 sgend = sgstart + length - 1; 1150 } else 1151 seg = NULL; 1152 } 1153 1154 if (seg == NULL) { 1155 seg = &map->dm_segs[i]; 1156 if (++i > map->_dm_segcnt) { 1157 map->dm_nsegs = 0; 1158 return (EFBIG); 1159 } 1160 } 1161 1162 /* 1163 * At this point, "i" is the index of the *next* bus_dma_segment_t 1164 * (the segment count, aka map->dm_nsegs) and "seg" points to the 1165 * *current* entry. "length", "sgstart", and "sgend" reflect what 1166 * we intend to put in "*seg". No assumptions should be made about 1167 * the contents of "*seg". Only "boundary" issue can change this 1168 * and "boundary" is often zero, so explicitly test for that case 1169 * (the test is strictly an optimization). 1170 */ 1171 if (boundary != 0) { 1172 bd_mask = ~(boundary - 1); 1173 1174 while ((sgstart & bd_mask) != (sgend & bd_mask)) { 1175 /* 1176 * We are crossing a boundary so fill in the current 1177 * segment with as much as possible, then grab a new 1178 * one. 1179 */ 1180 1181 seg->ds_addr = sgstart; 1182 seg->ds_len = boundary - (sgstart & ~bd_mask); 1183 1184 sgstart += seg->ds_len; /* sgend stays the same */ 1185 length -= seg->ds_len; 1186 1187 seg = &map->dm_segs[i]; 1188 if (++i > map->_dm_segcnt) { 1189 map->dm_nsegs = 0; 1190 return (EFBIG); 1191 } 1192 } 1193 } 1194 1195 seg->ds_addr = sgstart; 1196 seg->ds_len = length; 1197 map->dm_nsegs = i; 1198 1199 return (0); 1200 } 1201 1202 /* 1203 * Populate the iomap from a bus_dma_segment_t array. See note for 1204 * iommu_dvmamap_load() * regarding page entry exhaustion of the iomap. 1205 * This is less of a problem for load_seg, as the number of pages 1206 * is usually similar to the number of segments (nsegs). 1207 */ 1208 int 1209 iommu_dvmamap_load_seg(bus_dma_tag_t t, struct iommu_state *is, 1210 bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags, 1211 bus_size_t size, bus_size_t boundary) 1212 { 1213 int i; 1214 int left; 1215 int seg; 1216 1217 /* 1218 * This segs is made up of individual physical 1219 * segments, probably by _bus_dmamap_load_uio() or 1220 * _bus_dmamap_load_mbuf(). Ignore the mlist and 1221 * load each one individually. 1222 */ 1223 1224 /* 1225 * Keep in mind that each segment could span 1226 * multiple pages and that these are not always 1227 * adjacent. The code is no longer adding dvma 1228 * aliases to the IOMMU. The STC will not cross 1229 * page boundaries anyway and a IOMMU table walk 1230 * vs. what may be a streamed PCI DMA to a ring 1231 * descriptor is probably a wash. It eases TLB 1232 * pressure and in the worst possible case, it is 1233 * only as bad a non-IOMMUed architecture. More 1234 * importantly, the code is not quite as hairy. 1235 * (It's bad enough as it is.) 1236 */ 1237 left = size; 1238 seg = 0; 1239 for (i = 0; left > 0 && i < nsegs; i++) { 1240 bus_addr_t a, aend; 1241 bus_size_t len = segs[i].ds_len; 1242 bus_addr_t addr = segs[i].ds_addr; 1243 int seg_len = MIN(left, len); 1244 1245 if (len < 1) 1246 continue; 1247 1248 aend = round_page(addr + seg_len); 1249 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 1250 bus_addr_t pgstart; 1251 bus_addr_t pgend; 1252 int pglen; 1253 int err; 1254 1255 pgstart = MAX(a, addr); 1256 pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1); 1257 pglen = pgend - pgstart + 1; 1258 1259 if (pglen < 1) 1260 continue; 1261 1262 err = iommu_dvmamap_append_range(t, map, pgstart, 1263 pglen, flags, boundary); 1264 if (err == EFBIG) 1265 return (err); 1266 if (err) { 1267 printf("iomap load seg page: %d for " 1268 "pa 0x%lx (%lx - %lx for %d/%x\n", 1269 err, a, pgstart, pgend, pglen, pglen); 1270 return (err); 1271 } 1272 1273 } 1274 1275 left -= seg_len; 1276 } 1277 return (0); 1278 } 1279 1280 /* 1281 * Populate the iomap from an mlist. See note for iommu_dvmamap_load() 1282 * regarding page entry exhaustion of the iomap. 1283 */ 1284 int 1285 iommu_dvmamap_load_mlist(bus_dma_tag_t t, struct iommu_state *is, 1286 bus_dmamap_t map, struct pglist *mlist, int flags, 1287 bus_size_t size, bus_size_t boundary) 1288 { 1289 struct vm_page *m; 1290 paddr_t pa; 1291 int err; 1292 1293 /* 1294 * This was allocated with bus_dmamem_alloc. 1295 * The pages are on an `mlist'. 1296 */ 1297 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 1298 pa = VM_PAGE_TO_PHYS(m); 1299 1300 err = iommu_dvmamap_append_range(t, map, pa, 1301 MIN(PAGE_SIZE, size), flags, boundary); 1302 if (err == EFBIG) 1303 return (err); 1304 if (err) { 1305 printf("iomap load seg page: %d for pa 0x%lx " 1306 "(%lx - %lx for %d/%x\n", err, pa, pa, 1307 pa + PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1308 return (err); 1309 } 1310 if (size < PAGE_SIZE) 1311 break; 1312 size -= PAGE_SIZE; 1313 } 1314 1315 return (0); 1316 } 1317 1318 /* 1319 * Unload a dvmamap. 1320 */ 1321 void 1322 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) 1323 { 1324 struct iommu_state *is; 1325 struct iommu_map_state *ims; 1326 bus_addr_t dvmaddr = map->_dm_dvmastart; 1327 bus_size_t sgsize = map->_dm_dvmasize; 1328 int error; 1329 1330 if (ISSET(map->_dm_flags, BUS_DMA_64BIT)) { 1331 bus_dmamap_unload(t->_parent, map); 1332 return; 1333 } 1334 1335 ims = map->_dm_cookie; 1336 is = ims->ims_sb->sb_iommu; 1337 1338 /* Flush the iommu */ 1339 #ifdef DEBUG 1340 if (dvmaddr == 0) { 1341 printf("iommu_dvmamap_unload: No dvmastart\n"); 1342 #ifdef DDB 1343 if (iommudebug & IDB_BREAK) 1344 db_enter(); 1345 #endif 1346 return; 1347 } 1348 1349 iommu_dvmamap_validate_map(t, is, map); 1350 1351 if (iommudebug & IDB_PRINT_MAP) 1352 iommu_dvmamap_print_map(t, is, map); 1353 #endif /* DEBUG */ 1354 1355 /* Remove the IOMMU entries */ 1356 iommu_iomap_unload_map(is, ims); 1357 1358 /* Clear the iomap */ 1359 iommu_iomap_clear_pages(ims); 1360 1361 bus_dmamap_unload(t->_parent, map); 1362 1363 /* Mark the mappings as invalid. */ 1364 map->dm_mapsize = 0; 1365 map->dm_nsegs = 0; 1366 1367 mtx_enter(&is->is_mtx); 1368 error = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT); 1369 map->_dm_dvmastart = 0; 1370 map->_dm_dvmasize = 0; 1371 mtx_leave(&is->is_mtx); 1372 if (error != 0) 1373 printf("warning: %ld of DVMA space lost\n", sgsize); 1374 } 1375 1376 #ifdef DEBUG 1377 /* 1378 * Perform internal consistency checking on a dvmamap. 1379 */ 1380 int 1381 iommu_dvmamap_validate_map(bus_dma_tag_t t, struct iommu_state *is, 1382 bus_dmamap_t map) 1383 { 1384 int err = 0; 1385 int seg; 1386 1387 if (trunc_page(map->_dm_dvmastart) != map->_dm_dvmastart) { 1388 printf("**** dvmastart address not page aligned: %lx", 1389 map->_dm_dvmastart); 1390 err = 1; 1391 } 1392 if (trunc_page(map->_dm_dvmasize) != map->_dm_dvmasize) { 1393 printf("**** dvmasize not a multiple of page size: %lx", 1394 map->_dm_dvmasize); 1395 err = 1; 1396 } 1397 if (map->_dm_dvmastart < is->is_dvmabase || 1398 (round_page(map->_dm_dvmastart + map->_dm_dvmasize) - 1) > 1399 is->is_dvmaend) { 1400 printf("dvmaddr %lx len %lx out of range %x - %x\n", 1401 map->_dm_dvmastart, map->_dm_dvmasize, 1402 is->is_dvmabase, is->is_dvmaend); 1403 err = 1; 1404 } 1405 for (seg = 0; seg < map->dm_nsegs; seg++) { 1406 if (map->dm_segs[seg].ds_addr == 0 || 1407 map->dm_segs[seg].ds_len == 0) { 1408 printf("seg %d null segment dvmaddr %lx len %lx for " 1409 "range %lx len %lx\n", 1410 seg, 1411 map->dm_segs[seg].ds_addr, 1412 map->dm_segs[seg].ds_len, 1413 map->_dm_dvmastart, map->_dm_dvmasize); 1414 err = 1; 1415 } else if (map->dm_segs[seg].ds_addr < map->_dm_dvmastart || 1416 round_page(map->dm_segs[seg].ds_addr + 1417 map->dm_segs[seg].ds_len) > 1418 map->_dm_dvmastart + map->_dm_dvmasize) { 1419 printf("seg %d dvmaddr %lx len %lx out of " 1420 "range %lx len %lx\n", 1421 seg, 1422 map->dm_segs[seg].ds_addr, 1423 map->dm_segs[seg].ds_len, 1424 map->_dm_dvmastart, map->_dm_dvmasize); 1425 err = 1; 1426 } 1427 } 1428 1429 if (err) { 1430 iommu_dvmamap_print_map(t, is, map); 1431 #if defined(DDB) && defined(DEBUG) 1432 if (iommudebug & IDB_BREAK) 1433 db_enter(); 1434 #endif 1435 } 1436 1437 return (err); 1438 } 1439 #endif /* DEBUG */ 1440 1441 void 1442 iommu_dvmamap_print_map(bus_dma_tag_t t, struct iommu_state *is, 1443 bus_dmamap_t map) 1444 { 1445 int seg, i; 1446 long full_len, source_len; 1447 struct mbuf *m; 1448 1449 printf("DVMA %x for %x, mapping %p: dvstart %lx dvsize %lx " 1450 "size %ld/%lx maxsegsz %lx boundary %lx segcnt %d " 1451 "flags %x type %d source %p " 1452 "cookie %p mapsize %lx nsegs %d\n", 1453 is ? is->is_dvmabase : 0, is ? is->is_dvmaend : 0, map, 1454 map->_dm_dvmastart, map->_dm_dvmasize, 1455 map->_dm_size, map->_dm_size, map->_dm_maxsegsz, map->_dm_boundary, 1456 map->_dm_segcnt, map->_dm_flags, map->_dm_type, 1457 map->_dm_source, map->_dm_cookie, map->dm_mapsize, 1458 map->dm_nsegs); 1459 1460 full_len = 0; 1461 for (seg = 0; seg < map->dm_nsegs; seg++) { 1462 printf("seg %d dvmaddr %lx pa %lx len %lx (tte %llx)\n", 1463 seg, map->dm_segs[seg].ds_addr, 1464 is ? iommu_extract(is, map->dm_segs[seg].ds_addr) : 0, 1465 map->dm_segs[seg].ds_len, 1466 is ? iommu_lookup_tte(is, map->dm_segs[seg].ds_addr) : 0); 1467 full_len += map->dm_segs[seg].ds_len; 1468 } 1469 printf("total length = %ld/0x%lx\n", full_len, full_len); 1470 1471 if (map->_dm_source) switch (map->_dm_type) { 1472 case _DM_TYPE_MBUF: 1473 m = map->_dm_source; 1474 if (m->m_flags & M_PKTHDR) 1475 printf("source PKTHDR mbuf (%p) hdr len = %d/0x%x:\n", 1476 m, m->m_pkthdr.len, m->m_pkthdr.len); 1477 else 1478 printf("source mbuf (%p):\n", m); 1479 1480 source_len = 0; 1481 for ( ; m; m = m->m_next) { 1482 vaddr_t vaddr = mtod(m, vaddr_t); 1483 long len = m->m_len; 1484 paddr_t pa; 1485 1486 if (pmap_extract(pmap_kernel(), vaddr, &pa)) 1487 printf("kva %lx pa %lx len %ld/0x%lx\n", 1488 vaddr, pa, len, len); 1489 else 1490 printf("kva %lx pa <invalid> len %ld/0x%lx\n", 1491 vaddr, len, len); 1492 1493 source_len += len; 1494 } 1495 1496 if (full_len != source_len) 1497 printf("mbuf length %ld/0x%lx is %s than mapping " 1498 "length %ld/0x%lx\n", source_len, source_len, 1499 (source_len > full_len) ? "greater" : "less", 1500 full_len, full_len); 1501 else 1502 printf("mbuf length %ld/0x%lx\n", source_len, 1503 source_len); 1504 break; 1505 case _DM_TYPE_LOAD: 1506 case _DM_TYPE_SEGS: 1507 case _DM_TYPE_UIO: 1508 default: 1509 break; 1510 } 1511 1512 if (!ISSET(map->_dm_flags, BUS_DMA_64BIT) && map->_dm_cookie != NULL) { 1513 struct iommu_map_state *ims = map->_dm_cookie; 1514 struct iommu_page_map *ipm = &ims->ims_map; 1515 1516 printf("page map (%p) of size %d with %d entries\n", 1517 ipm, ipm->ipm_maxpage, ipm->ipm_pagecnt); 1518 for (i = 0; i < ipm->ipm_pagecnt; ++i) { 1519 struct iommu_page_entry *e = &ipm->ipm_map[i]; 1520 printf("%d: vmaddr 0x%lx pa 0x%lx\n", i, 1521 e->ipe_va, e->ipe_pa); 1522 } 1523 } else 1524 printf("iommu map state (cookie) is NULL\n"); 1525 } 1526 1527 void 1528 _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 1529 bus_addr_t offset, bus_size_t len, int ops) 1530 { 1531 struct iommu_state *is; 1532 struct iommu_map_state *ims = map->_dm_cookie; 1533 struct strbuf_ctl *sb; 1534 bus_size_t count; 1535 int i, needsflush = 0; 1536 1537 sb = ims->ims_sb; 1538 is = sb->sb_iommu; 1539 1540 for (i = 0; i < map->dm_nsegs; i++) { 1541 if (offset < map->dm_segs[i].ds_len) 1542 break; 1543 offset -= map->dm_segs[i].ds_len; 1544 } 1545 1546 if (i == map->dm_nsegs) 1547 panic("iommu_dvmamap_sync: too short %lu", offset); 1548 1549 for (; len > 0 && i < map->dm_nsegs; i++) { 1550 count = MIN(map->dm_segs[i].ds_len - offset, len); 1551 if (count > 0 && iommu_dvmamap_sync_range(sb, 1552 map->dm_segs[i].ds_addr + offset, count)) 1553 needsflush = 1; 1554 len -= count; 1555 } 1556 1557 #ifdef DIAGNOSTIC 1558 if (i == map->dm_nsegs && len > 0) 1559 panic("iommu_dvmamap_sync: leftover %lu", len); 1560 #endif 1561 1562 if (needsflush) 1563 iommu_strbuf_flush_done(ims); 1564 } 1565 1566 void 1567 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 1568 bus_addr_t offset, bus_size_t len, int ops) 1569 { 1570 struct iommu_map_state *ims; 1571 1572 if (len == 0) 1573 return; 1574 1575 if (map->_dm_flags & BUS_DMA_64BIT) { 1576 if (ops & (BUS_DMASYNC_PREWRITE | BUS_DMASYNC_POSTREAD)) 1577 __membar("#MemIssue"); 1578 return; 1579 } 1580 1581 ims = map->_dm_cookie; 1582 1583 if (ops & BUS_DMASYNC_PREWRITE) 1584 __membar("#MemIssue"); 1585 1586 if ((ims->ims_flags & IOMMU_MAP_STREAM) && 1587 (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE))) 1588 _iommu_dvmamap_sync(t, t0, map, offset, len, ops); 1589 1590 if (ops & BUS_DMASYNC_POSTREAD) 1591 __membar("#MemIssue"); 1592 } 1593 1594 /* 1595 * Flush an individual dma segment, returns non-zero if the streaming buffers 1596 * need flushing afterwards. 1597 */ 1598 int 1599 iommu_dvmamap_sync_range(struct strbuf_ctl *sb, bus_addr_t va, bus_size_t len) 1600 { 1601 bus_addr_t vaend; 1602 #ifdef DIAGNOSTIC 1603 struct iommu_state *is = sb->sb_iommu; 1604 1605 if (va < is->is_dvmabase || va > is->is_dvmaend) 1606 panic("invalid va: %llx", (long long)va); 1607 1608 if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) { 1609 printf("iommu_dvmamap_sync_range: attempting to flush " 1610 "non-streaming entry\n"); 1611 return (0); 1612 } 1613 #endif 1614 1615 vaend = (va + len + PAGE_MASK) & ~PAGE_MASK; 1616 va &= ~PAGE_MASK; 1617 1618 #ifdef DIAGNOSTIC 1619 if (va < is->is_dvmabase || (vaend - 1) > is->is_dvmaend) 1620 panic("invalid va range: %llx to %llx (%x to %x)", 1621 (long long)va, (long long)vaend, 1622 is->is_dvmabase, 1623 is->is_dvmaend); 1624 #endif 1625 1626 for ( ; va <= vaend; va += PAGE_SIZE) { 1627 DPRINTF(IDB_BUSDMA, 1628 ("iommu_dvmamap_sync_range: flushing va %p\n", 1629 (void *)(u_long)va)); 1630 iommu_strbuf_flush(sb, va); 1631 } 1632 1633 return (1); 1634 } 1635 1636 int 1637 iommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size, 1638 bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, 1639 int nsegs, int *rsegs, int flags) 1640 { 1641 1642 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx " 1643 "bound %llx segp %p flags %d\n", (unsigned long long)size, 1644 (unsigned long long)alignment, (unsigned long long)boundary, 1645 segs, flags)); 1646 1647 if ((flags & BUS_DMA_64BIT) == 0) 1648 flags |= BUS_DMA_DVMA; 1649 1650 BUS_DMA_FIND_PARENT(t, _dmamem_alloc); 1651 return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary, 1652 segs, nsegs, rsegs, flags)); 1653 } 1654 1655 void 1656 iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs, 1657 int nsegs) 1658 { 1659 1660 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1661 segs, nsegs)); 1662 BUS_DMA_FIND_PARENT(t, _dmamem_free); 1663 (*t->_dmamem_free)(t, t0, segs, nsegs); 1664 } 1665 1666 /* 1667 * Create a new iomap. 1668 */ 1669 struct iommu_map_state * 1670 iommu_iomap_create(int n) 1671 { 1672 struct iommu_map_state *ims; 1673 struct strbuf_flush *sbf; 1674 vaddr_t va; 1675 1676 /* Safety for heavily fragmented data, such as mbufs */ 1677 n += 4; 1678 if (n < 16) 1679 n = 16; 1680 1681 ims = malloc(sizeof(*ims) + (n - 1) * sizeof(ims->ims_map.ipm_map[0]), 1682 M_DEVBUF, M_NOWAIT | M_ZERO); 1683 if (ims == NULL) 1684 return (NULL); 1685 1686 /* Initialize the map. */ 1687 ims->ims_map.ipm_maxpage = n; 1688 SPLAY_INIT(&ims->ims_map.ipm_tree); 1689 1690 /* Initialize the flush area. */ 1691 sbf = &ims->ims_flush; 1692 va = (vaddr_t)&sbf->sbf_area[0x40]; 1693 va &= ~0x3f; 1694 pmap_extract(pmap_kernel(), va, &sbf->sbf_flushpa); 1695 sbf->sbf_flush = (void *)va; 1696 1697 return (ims); 1698 } 1699 1700 /* 1701 * Destroy an iomap. 1702 */ 1703 void 1704 iommu_iomap_destroy(struct iommu_map_state *ims) 1705 { 1706 #ifdef DIAGNOSTIC 1707 if (ims->ims_map.ipm_pagecnt > 0) 1708 printf("iommu_iomap_destroy: %d page entries in use\n", 1709 ims->ims_map.ipm_pagecnt); 1710 #endif 1711 1712 free(ims, M_DEVBUF, 0); 1713 } 1714 1715 /* 1716 * Utility function used by splay tree to order page entries by pa. 1717 */ 1718 static inline int 1719 iomap_compare(struct iommu_page_entry *a, struct iommu_page_entry *b) 1720 { 1721 return ((a->ipe_pa > b->ipe_pa) ? 1 : 1722 (a->ipe_pa < b->ipe_pa) ? -1 : 0); 1723 } 1724 1725 SPLAY_PROTOTYPE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare); 1726 1727 SPLAY_GENERATE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare); 1728 1729 /* 1730 * Insert a pa entry in the iomap. 1731 */ 1732 int 1733 iommu_iomap_insert_page(struct iommu_map_state *ims, paddr_t pa) 1734 { 1735 struct iommu_page_map *ipm = &ims->ims_map; 1736 struct iommu_page_entry *e; 1737 1738 if (ipm->ipm_pagecnt >= ipm->ipm_maxpage) { 1739 struct iommu_page_entry ipe; 1740 1741 ipe.ipe_pa = pa; 1742 if (SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &ipe)) 1743 return (0); 1744 1745 return (ENOMEM); 1746 } 1747 1748 e = &ipm->ipm_map[ipm->ipm_pagecnt]; 1749 1750 e->ipe_pa = pa; 1751 e->ipe_va = 0; 1752 1753 e = SPLAY_INSERT(iommu_page_tree, &ipm->ipm_tree, e); 1754 1755 /* Duplicates are okay, but only count them once. */ 1756 if (e) 1757 return (0); 1758 1759 ++ipm->ipm_pagecnt; 1760 1761 return (0); 1762 } 1763 1764 /* 1765 * Locate the iomap by filling in the pa->va mapping and inserting it 1766 * into the IOMMU tables. 1767 */ 1768 void 1769 iommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims, 1770 bus_addr_t vmaddr, int flags) 1771 { 1772 struct iommu_page_map *ipm = &ims->ims_map; 1773 struct iommu_page_entry *e; 1774 struct strbuf_ctl *sb = ims->ims_sb; 1775 int i, slot; 1776 1777 if (sb->sb_flush == NULL) 1778 flags &= ~BUS_DMA_STREAMING; 1779 1780 if (flags & BUS_DMA_STREAMING) 1781 ims->ims_flags |= IOMMU_MAP_STREAM; 1782 else 1783 ims->ims_flags &= ~IOMMU_MAP_STREAM; 1784 1785 for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) { 1786 e->ipe_va = vmaddr; 1787 iommu_enter(is, sb, e->ipe_va, e->ipe_pa, flags); 1788 1789 /* Flush cache if necessary. */ 1790 slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); 1791 if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) && 1792 (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) 1793 IOMMUREG_WRITE(is, iommu_cache_flush, 1794 is->is_ptsb + slot * 8); 1795 1796 vmaddr += PAGE_SIZE; 1797 } 1798 } 1799 1800 /* 1801 * Remove the iomap from the IOMMU. 1802 */ 1803 void 1804 iommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims) 1805 { 1806 struct iommu_page_map *ipm = &ims->ims_map; 1807 struct iommu_page_entry *e; 1808 struct strbuf_ctl *sb = ims->ims_sb; 1809 int i, slot; 1810 1811 for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) { 1812 iommu_remove(is, sb, e->ipe_va); 1813 1814 /* Flush cache if necessary. */ 1815 slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); 1816 if (ISSET(is->is_hw->ihw_flags, IOMMU_HW_FLUSH_CACHE) && 1817 (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) 1818 IOMMUREG_WRITE(is, iommu_cache_flush, 1819 is->is_ptsb + slot * 8); 1820 } 1821 } 1822 1823 /* 1824 * Translate a physical address (pa) into a DVMA address. 1825 */ 1826 bus_addr_t 1827 iommu_iomap_translate(struct iommu_map_state *ims, paddr_t pa) 1828 { 1829 struct iommu_page_map *ipm = &ims->ims_map; 1830 struct iommu_page_entry *e; 1831 struct iommu_page_entry pe; 1832 paddr_t offset = pa & PAGE_MASK; 1833 1834 pe.ipe_pa = trunc_page(pa); 1835 1836 e = SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &pe); 1837 1838 if (e == NULL) 1839 return (0); 1840 1841 return (e->ipe_va | offset); 1842 } 1843 1844 /* 1845 * Clear the iomap table and tree. 1846 */ 1847 void 1848 iommu_iomap_clear_pages(struct iommu_map_state *ims) 1849 { 1850 ims->ims_map.ipm_pagecnt = 0; 1851 SPLAY_INIT(&ims->ims_map.ipm_tree); 1852 } 1853 1854