1 /* $OpenBSD: smmu.c,v 1.18 2021/06/25 19:55:22 patrick Exp $ */ 2 /* 3 * Copyright (c) 2008-2009,2014-2016 Dale Rahn <drahn@dalerahn.com> 4 * Copyright (c) 2021 Patrick Wildt <patrick@blueri.se> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/param.h> 20 #include <sys/systm.h> 21 #include <sys/device.h> 22 #include <sys/pool.h> 23 #include <sys/atomic.h> 24 25 #include <machine/bus.h> 26 27 #include <uvm/uvm_extern.h> 28 #include <arm64/vmparam.h> 29 #include <arm64/pmap.h> 30 31 #include <dev/pci/pcivar.h> 32 #include <arm64/dev/smmuvar.h> 33 #include <arm64/dev/smmureg.h> 34 35 struct smmu_map_state { 36 struct extent_region sms_er; 37 bus_addr_t sms_dva; 38 bus_size_t sms_len; 39 bus_size_t sms_loaded; 40 }; 41 42 struct smmuvp0 { 43 uint64_t l0[VP_IDX0_CNT]; 44 struct smmuvp1 *vp[VP_IDX0_CNT]; 45 }; 46 47 struct smmuvp1 { 48 uint64_t l1[VP_IDX1_CNT]; 49 struct smmuvp2 *vp[VP_IDX1_CNT]; 50 }; 51 52 struct smmuvp2 { 53 uint64_t l2[VP_IDX2_CNT]; 54 struct smmuvp3 *vp[VP_IDX2_CNT]; 55 }; 56 57 struct smmuvp3 { 58 uint64_t l3[VP_IDX3_CNT]; 59 }; 60 61 CTASSERT(sizeof(struct smmuvp0) == sizeof(struct smmuvp1)); 62 CTASSERT(sizeof(struct smmuvp0) == sizeof(struct smmuvp2)); 63 CTASSERT(sizeof(struct smmuvp0) != sizeof(struct smmuvp3)); 64 65 uint32_t smmu_gr0_read_4(struct smmu_softc *, bus_size_t); 66 void smmu_gr0_write_4(struct smmu_softc *, bus_size_t, uint32_t); 67 uint32_t smmu_gr1_read_4(struct smmu_softc *, bus_size_t); 68 void smmu_gr1_write_4(struct smmu_softc *, bus_size_t, uint32_t); 69 uint32_t smmu_cb_read_4(struct smmu_softc *, int, bus_size_t); 70 void smmu_cb_write_4(struct smmu_softc *, int, bus_size_t, uint32_t); 71 uint64_t smmu_cb_read_8(struct smmu_softc *, int, bus_size_t); 72 void smmu_cb_write_8(struct smmu_softc *, int, bus_size_t, uint64_t); 73 74 void smmu_tlb_sync_global(struct smmu_softc *); 75 void smmu_tlb_sync_context(struct smmu_domain *); 76 77 struct smmu_domain *smmu_domain_lookup(struct smmu_softc *, uint32_t); 78 struct smmu_domain *smmu_domain_create(struct smmu_softc *, uint32_t); 79 80 void smmu_set_l1(struct smmu_domain *, uint64_t, struct smmuvp1 *); 81 void smmu_set_l2(struct smmu_domain *, uint64_t, struct smmuvp1 *, 82 struct smmuvp2 *); 83 void smmu_set_l3(struct smmu_domain *, uint64_t, struct smmuvp2 *, 84 struct smmuvp3 *); 85 86 int smmu_vp_lookup(struct smmu_domain *, vaddr_t, uint64_t **); 87 int smmu_vp_enter(struct smmu_domain *, vaddr_t, uint64_t **, int); 88 89 uint64_t smmu_fill_pte(struct smmu_domain *, vaddr_t, paddr_t, 90 vm_prot_t, int, int); 91 void smmu_pte_update(struct smmu_domain *, uint64_t, uint64_t *); 92 void smmu_pte_remove(struct smmu_domain *, vaddr_t); 93 94 int smmu_enter(struct smmu_domain *, vaddr_t, paddr_t, vm_prot_t, int, int); 95 void smmu_map(struct smmu_domain *, vaddr_t, paddr_t, vm_prot_t, int, int); 96 void smmu_unmap(struct smmu_domain *, vaddr_t); 97 void smmu_remove(struct smmu_domain *, vaddr_t); 98 99 int smmu_load_map(struct smmu_domain *, bus_dmamap_t); 100 void smmu_unload_map(struct smmu_domain *, bus_dmamap_t); 101 102 int smmu_dmamap_create(bus_dma_tag_t , bus_size_t, int, 103 bus_size_t, bus_size_t, int, bus_dmamap_t *); 104 void smmu_dmamap_destroy(bus_dma_tag_t , bus_dmamap_t); 105 int smmu_dmamap_load(bus_dma_tag_t , bus_dmamap_t, void *, 106 bus_size_t, struct proc *, int); 107 int smmu_dmamap_load_mbuf(bus_dma_tag_t , bus_dmamap_t, 108 struct mbuf *, int); 109 int smmu_dmamap_load_uio(bus_dma_tag_t , bus_dmamap_t, 110 struct uio *, int); 111 int smmu_dmamap_load_raw(bus_dma_tag_t , bus_dmamap_t, 112 bus_dma_segment_t *, int, bus_size_t, int); 113 void smmu_dmamap_unload(bus_dma_tag_t , bus_dmamap_t); 114 115 struct cfdriver smmu_cd = { 116 NULL, "smmu", DV_DULL 117 }; 118 119 int 120 smmu_attach(struct smmu_softc *sc) 121 { 122 uint32_t reg; 123 int i; 124 125 SIMPLEQ_INIT(&sc->sc_domains); 126 127 pool_init(&sc->sc_vp_pool, sizeof(struct smmuvp0), PAGE_SIZE, IPL_VM, 0, 128 "smmu_vp", NULL); 129 pool_setlowat(&sc->sc_vp_pool, 20); 130 pool_init(&sc->sc_vp3_pool, sizeof(struct smmuvp3), PAGE_SIZE, IPL_VM, 0, 131 "smmu_vp3", NULL); 132 pool_setlowat(&sc->sc_vp3_pool, 20); 133 134 reg = smmu_gr0_read_4(sc, SMMU_IDR0); 135 if (reg & SMMU_IDR0_S1TS) 136 sc->sc_has_s1 = 1; 137 /* 138 * Marvell's 8040 does not support 64-bit writes, hence it 139 * is not possible to invalidate stage-2, because the ASID 140 * is part of the upper 32-bits and they'd be ignored. 141 */ 142 if (sc->sc_is_ap806) 143 sc->sc_has_s1 = 0; 144 if (reg & SMMU_IDR0_S2TS) 145 sc->sc_has_s2 = 1; 146 if (!sc->sc_has_s1 && !sc->sc_has_s2) 147 return 1; 148 if (reg & SMMU_IDR0_EXIDS) 149 sc->sc_has_exids = 1; 150 151 sc->sc_num_streams = 1 << SMMU_IDR0_NUMSIDB(reg); 152 if (sc->sc_has_exids) 153 sc->sc_num_streams = 1 << 16; 154 sc->sc_stream_mask = sc->sc_num_streams - 1; 155 if (reg & SMMU_IDR0_SMS) { 156 sc->sc_num_streams = SMMU_IDR0_NUMSMRG(reg); 157 if (sc->sc_num_streams == 0) 158 return 1; 159 sc->sc_smr = mallocarray(sc->sc_num_streams, 160 sizeof(*sc->sc_smr), M_DEVBUF, M_WAITOK | M_ZERO); 161 } 162 163 reg = smmu_gr0_read_4(sc, SMMU_IDR1); 164 sc->sc_pagesize = 4 * 1024; 165 if (reg & SMMU_IDR1_PAGESIZE_64K) 166 sc->sc_pagesize = 64 * 1024; 167 sc->sc_numpage = 1 << (SMMU_IDR1_NUMPAGENDXB(reg) + 1); 168 169 /* 0 to NUMS2CB == stage-2, NUMS2CB to NUMCB == stage-1 */ 170 sc->sc_num_context_banks = SMMU_IDR1_NUMCB(reg); 171 sc->sc_num_s2_context_banks = SMMU_IDR1_NUMS2CB(reg); 172 if (sc->sc_num_s2_context_banks > sc->sc_num_context_banks) 173 return 1; 174 sc->sc_cb = mallocarray(sc->sc_num_context_banks, 175 sizeof(*sc->sc_cb), M_DEVBUF, M_WAITOK | M_ZERO); 176 177 reg = smmu_gr0_read_4(sc, SMMU_IDR2); 178 if (reg & SMMU_IDR2_VMID16S) 179 sc->sc_has_vmid16s = 1; 180 181 switch (SMMU_IDR2_IAS(reg)) { 182 case SMMU_IDR2_IAS_32BIT: 183 sc->sc_ipa_bits = 32; 184 break; 185 case SMMU_IDR2_IAS_36BIT: 186 sc->sc_ipa_bits = 36; 187 break; 188 case SMMU_IDR2_IAS_40BIT: 189 sc->sc_ipa_bits = 40; 190 break; 191 case SMMU_IDR2_IAS_42BIT: 192 sc->sc_ipa_bits = 42; 193 break; 194 case SMMU_IDR2_IAS_44BIT: 195 sc->sc_ipa_bits = 44; 196 break; 197 case SMMU_IDR2_IAS_48BIT: 198 default: 199 sc->sc_ipa_bits = 48; 200 break; 201 } 202 switch (SMMU_IDR2_OAS(reg)) { 203 case SMMU_IDR2_OAS_32BIT: 204 sc->sc_pa_bits = 32; 205 break; 206 case SMMU_IDR2_OAS_36BIT: 207 sc->sc_pa_bits = 36; 208 break; 209 case SMMU_IDR2_OAS_40BIT: 210 sc->sc_pa_bits = 40; 211 break; 212 case SMMU_IDR2_OAS_42BIT: 213 sc->sc_pa_bits = 42; 214 break; 215 case SMMU_IDR2_OAS_44BIT: 216 sc->sc_pa_bits = 44; 217 break; 218 case SMMU_IDR2_OAS_48BIT: 219 default: 220 sc->sc_pa_bits = 48; 221 break; 222 } 223 switch (SMMU_IDR2_UBS(reg)) { 224 case SMMU_IDR2_UBS_32BIT: 225 sc->sc_va_bits = 32; 226 break; 227 case SMMU_IDR2_UBS_36BIT: 228 sc->sc_va_bits = 36; 229 break; 230 case SMMU_IDR2_UBS_40BIT: 231 sc->sc_va_bits = 40; 232 break; 233 case SMMU_IDR2_UBS_42BIT: 234 sc->sc_va_bits = 42; 235 break; 236 case SMMU_IDR2_UBS_44BIT: 237 sc->sc_va_bits = 44; 238 break; 239 case SMMU_IDR2_UBS_49BIT: 240 default: 241 sc->sc_va_bits = 48; 242 break; 243 } 244 245 printf(": %u CBs (%u S2-only)\n", 246 sc->sc_num_context_banks, sc->sc_num_s2_context_banks); 247 248 /* Clear Global Fault Status Register */ 249 smmu_gr0_write_4(sc, SMMU_SGFSR, smmu_gr0_read_4(sc, SMMU_SGFSR)); 250 251 for (i = 0; i < sc->sc_num_streams; i++) { 252 #if 1 253 /* Setup all streams to fault by default */ 254 smmu_gr0_write_4(sc, SMMU_S2CR(i), SMMU_S2CR_TYPE_FAULT); 255 #else 256 /* For stream indexing, USFCFG bypass isn't enough! */ 257 smmu_gr0_write_4(sc, SMMU_S2CR(i), SMMU_S2CR_TYPE_BYPASS); 258 #endif 259 /* Disable all stream map registers */ 260 if (sc->sc_smr) 261 smmu_gr0_write_4(sc, SMMU_SMR(i), 0); 262 } 263 264 for (i = 0; i < sc->sc_num_context_banks; i++) { 265 /* Disable Context Bank */ 266 smmu_cb_write_4(sc, i, SMMU_CB_SCTLR, 0); 267 /* Clear Context Bank Fault Status Register */ 268 smmu_cb_write_4(sc, i, SMMU_CB_FSR, SMMU_CB_FSR_MASK); 269 } 270 271 /* Invalidate TLB */ 272 smmu_gr0_write_4(sc, SMMU_TLBIALLH, ~0); 273 smmu_gr0_write_4(sc, SMMU_TLBIALLNSNH, ~0); 274 275 if (sc->sc_is_mmu500) { 276 reg = smmu_gr0_read_4(sc, SMMU_SACR); 277 if (SMMU_IDR7_MAJOR(smmu_gr0_read_4(sc, SMMU_IDR7)) >= 2) 278 reg &= ~SMMU_SACR_MMU500_CACHE_LOCK; 279 reg |= SMMU_SACR_MMU500_SMTNMB_TLBEN | 280 SMMU_SACR_MMU500_S2CRB_TLBEN; 281 smmu_gr0_write_4(sc, SMMU_SACR, reg); 282 for (i = 0; i < sc->sc_num_context_banks; i++) { 283 reg = smmu_cb_read_4(sc, i, SMMU_CB_ACTLR); 284 reg &= ~SMMU_CB_ACTLR_CPRE; 285 smmu_cb_write_4(sc, i, SMMU_CB_ACTLR, reg); 286 } 287 } 288 289 /* Enable SMMU */ 290 reg = smmu_gr0_read_4(sc, SMMU_SCR0); 291 reg &= ~(SMMU_SCR0_CLIENTPD | 292 SMMU_SCR0_FB | SMMU_SCR0_BSU_MASK); 293 #if 1 294 /* Disable bypass for unknown streams */ 295 reg |= SMMU_SCR0_USFCFG; 296 #else 297 /* Enable bypass for unknown streams */ 298 reg &= ~SMMU_SCR0_USFCFG; 299 #endif 300 reg |= SMMU_SCR0_GFRE | SMMU_SCR0_GFIE | 301 SMMU_SCR0_GCFGFRE | SMMU_SCR0_GCFGFIE | 302 SMMU_SCR0_VMIDPNE | SMMU_SCR0_PTM; 303 if (sc->sc_has_exids) 304 reg |= SMMU_SCR0_EXIDENABLE; 305 if (sc->sc_has_vmid16s) 306 reg |= SMMU_SCR0_VMID16EN; 307 308 smmu_tlb_sync_global(sc); 309 smmu_gr0_write_4(sc, SMMU_SCR0, reg); 310 311 return 0; 312 } 313 314 int 315 smmu_global_irq(void *cookie) 316 { 317 struct smmu_softc *sc = cookie; 318 uint32_t reg; 319 320 reg = smmu_gr0_read_4(sc, SMMU_SGFSR); 321 if (reg == 0) 322 return 0; 323 324 printf("%s: SGFSR 0x%08x SGFSYNR0 0x%08x SGFSYNR1 0x%08x " 325 "SGFSYNR2 0x%08x\n", sc->sc_dev.dv_xname, reg, 326 smmu_gr0_read_4(sc, SMMU_SGFSYNR0), 327 smmu_gr0_read_4(sc, SMMU_SGFSYNR1), 328 smmu_gr0_read_4(sc, SMMU_SGFSYNR2)); 329 330 smmu_gr0_write_4(sc, SMMU_SGFSR, reg); 331 332 return 1; 333 } 334 335 int 336 smmu_context_irq(void *cookie) 337 { 338 struct smmu_cb_irq *cbi = cookie; 339 struct smmu_softc *sc = cbi->cbi_sc; 340 uint32_t reg; 341 342 reg = smmu_cb_read_4(sc, cbi->cbi_idx, SMMU_CB_FSR); 343 if ((reg & SMMU_CB_FSR_MASK) == 0) 344 return 0; 345 346 printf("%s: FSR 0x%08x FSYNR0 0x%08x FAR 0x%llx " 347 "CBFRSYNRA 0x%08x\n", sc->sc_dev.dv_xname, reg, 348 smmu_cb_read_4(sc, cbi->cbi_idx, SMMU_CB_FSYNR0), 349 smmu_cb_read_8(sc, cbi->cbi_idx, SMMU_CB_FAR), 350 smmu_gr1_read_4(sc, SMMU_CBFRSYNRA(cbi->cbi_idx))); 351 352 smmu_cb_write_4(sc, cbi->cbi_idx, SMMU_CB_FSR, reg); 353 354 return 1; 355 } 356 357 void 358 smmu_tlb_sync_global(struct smmu_softc *sc) 359 { 360 int i; 361 362 smmu_gr0_write_4(sc, SMMU_STLBGSYNC, ~0); 363 for (i = 1000; i > 0; i--) { 364 if ((smmu_gr0_read_4(sc, SMMU_STLBGSTATUS) & 365 SMMU_STLBGSTATUS_GSACTIVE) == 0) 366 return; 367 } 368 369 printf("%s: global TLB sync timeout\n", 370 sc->sc_dev.dv_xname); 371 } 372 373 void 374 smmu_tlb_sync_context(struct smmu_domain *dom) 375 { 376 struct smmu_softc *sc = dom->sd_sc; 377 int i; 378 379 smmu_cb_write_4(sc, dom->sd_cb_idx, SMMU_CB_TLBSYNC, ~0); 380 for (i = 1000; i > 0; i--) { 381 if ((smmu_cb_read_4(sc, dom->sd_cb_idx, SMMU_CB_TLBSTATUS) & 382 SMMU_CB_TLBSTATUS_SACTIVE) == 0) 383 return; 384 } 385 386 printf("%s: context TLB sync timeout\n", 387 sc->sc_dev.dv_xname); 388 } 389 390 uint32_t 391 smmu_gr0_read_4(struct smmu_softc *sc, bus_size_t off) 392 { 393 uint32_t base = 0 * sc->sc_pagesize; 394 395 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, base + off); 396 } 397 398 void 399 smmu_gr0_write_4(struct smmu_softc *sc, bus_size_t off, uint32_t val) 400 { 401 uint32_t base = 0 * sc->sc_pagesize; 402 403 bus_space_write_4(sc->sc_iot, sc->sc_ioh, base + off, val); 404 } 405 406 uint32_t 407 smmu_gr1_read_4(struct smmu_softc *sc, bus_size_t off) 408 { 409 uint32_t base = 1 * sc->sc_pagesize; 410 411 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, base + off); 412 } 413 414 void 415 smmu_gr1_write_4(struct smmu_softc *sc, bus_size_t off, uint32_t val) 416 { 417 uint32_t base = 1 * sc->sc_pagesize; 418 419 bus_space_write_4(sc->sc_iot, sc->sc_ioh, base + off, val); 420 } 421 422 uint32_t 423 smmu_cb_read_4(struct smmu_softc *sc, int idx, bus_size_t off) 424 { 425 uint32_t base; 426 427 base = sc->sc_numpage * sc->sc_pagesize; /* SMMU_CB_BASE */ 428 base += idx * sc->sc_pagesize; /* SMMU_CBn_BASE */ 429 430 return bus_space_read_4(sc->sc_iot, sc->sc_ioh, base + off); 431 } 432 433 void 434 smmu_cb_write_4(struct smmu_softc *sc, int idx, bus_size_t off, uint32_t val) 435 { 436 uint32_t base; 437 438 base = sc->sc_numpage * sc->sc_pagesize; /* SMMU_CB_BASE */ 439 base += idx * sc->sc_pagesize; /* SMMU_CBn_BASE */ 440 441 bus_space_write_4(sc->sc_iot, sc->sc_ioh, base + off, val); 442 } 443 444 uint64_t 445 smmu_cb_read_8(struct smmu_softc *sc, int idx, bus_size_t off) 446 { 447 uint64_t reg; 448 uint32_t base; 449 450 base = sc->sc_numpage * sc->sc_pagesize; /* SMMU_CB_BASE */ 451 base += idx * sc->sc_pagesize; /* SMMU_CBn_BASE */ 452 453 if (sc->sc_is_ap806) { 454 reg = bus_space_read_4(sc->sc_iot, sc->sc_ioh, base + off + 4); 455 reg <<= 32; 456 reg |= bus_space_read_4(sc->sc_iot, sc->sc_ioh, base + off + 0); 457 return reg; 458 } 459 460 return bus_space_read_8(sc->sc_iot, sc->sc_ioh, base + off); 461 } 462 463 void 464 smmu_cb_write_8(struct smmu_softc *sc, int idx, bus_size_t off, uint64_t val) 465 { 466 uint32_t base; 467 468 base = sc->sc_numpage * sc->sc_pagesize; /* SMMU_CB_BASE */ 469 base += idx * sc->sc_pagesize; /* SMMU_CBn_BASE */ 470 471 if (sc->sc_is_ap806) { 472 bus_space_write_4(sc->sc_iot, sc->sc_ioh, base + off + 4, 473 val >> 32); 474 bus_space_write_4(sc->sc_iot, sc->sc_ioh, base + off + 0, 475 val & 0xffffffff); 476 return; 477 } 478 479 bus_space_write_8(sc->sc_iot, sc->sc_ioh, base + off, val); 480 } 481 482 bus_dma_tag_t 483 smmu_device_map(void *cookie, uint32_t sid, bus_dma_tag_t dmat) 484 { 485 struct smmu_softc *sc = cookie; 486 struct smmu_domain *dom; 487 488 dom = smmu_domain_lookup(sc, sid); 489 if (dom == NULL) 490 return dmat; 491 492 if (dom->sd_dmat == NULL) { 493 dom->sd_dmat = malloc(sizeof(*dom->sd_dmat), 494 M_DEVBUF, M_WAITOK); 495 memcpy(dom->sd_dmat, sc->sc_dmat, 496 sizeof(*dom->sd_dmat)); 497 dom->sd_dmat->_cookie = dom; 498 dom->sd_dmat->_dmamap_create = smmu_dmamap_create; 499 dom->sd_dmat->_dmamap_destroy = smmu_dmamap_destroy; 500 dom->sd_dmat->_dmamap_load = smmu_dmamap_load; 501 dom->sd_dmat->_dmamap_load_mbuf = smmu_dmamap_load_mbuf; 502 dom->sd_dmat->_dmamap_load_uio = smmu_dmamap_load_uio; 503 dom->sd_dmat->_dmamap_load_raw = smmu_dmamap_load_raw; 504 dom->sd_dmat->_dmamap_unload = smmu_dmamap_unload; 505 dom->sd_dmat->_flags |= BUS_DMA_COHERENT; 506 } 507 508 return dom->sd_dmat; 509 } 510 511 struct smmu_domain * 512 smmu_domain_lookup(struct smmu_softc *sc, uint32_t sid) 513 { 514 struct smmu_domain *dom; 515 516 SIMPLEQ_FOREACH(dom, &sc->sc_domains, sd_list) { 517 if (dom->sd_sid == sid) 518 return dom; 519 } 520 521 return smmu_domain_create(sc, sid); 522 } 523 524 struct smmu_domain * 525 smmu_domain_create(struct smmu_softc *sc, uint32_t sid) 526 { 527 struct smmu_domain *dom; 528 uint32_t iovabits, reg; 529 paddr_t pa; 530 vaddr_t l0va; 531 int i, start, end; 532 533 dom = malloc(sizeof(*dom), M_DEVBUF, M_WAITOK | M_ZERO); 534 mtx_init(&dom->sd_iova_mtx, IPL_VM); 535 mtx_init(&dom->sd_pmap_mtx, IPL_VM); 536 dom->sd_sc = sc; 537 dom->sd_sid = sid; 538 539 /* Prefer stage 1 if possible! */ 540 if (sc->sc_has_s1) { 541 start = sc->sc_num_s2_context_banks; 542 end = sc->sc_num_context_banks; 543 dom->sd_stage = 1; 544 } else { 545 start = 0; 546 end = sc->sc_num_context_banks; 547 dom->sd_stage = 2; 548 } 549 550 for (i = start; i < end; i++) { 551 if (sc->sc_cb[i] != NULL) 552 continue; 553 sc->sc_cb[i] = malloc(sizeof(struct smmu_cb), 554 M_DEVBUF, M_WAITOK | M_ZERO); 555 dom->sd_cb_idx = i; 556 break; 557 } 558 if (i >= end) { 559 printf("%s: out of context blocks, I/O device will fail\n", 560 sc->sc_dev.dv_xname); 561 free(dom, M_DEVBUF, sizeof(*dom)); 562 return NULL; 563 } 564 565 /* Stream indexing is easy */ 566 dom->sd_smr_idx = sid; 567 568 /* Stream mapping is a bit more effort */ 569 if (sc->sc_smr) { 570 for (i = 0; i < sc->sc_num_streams; i++) { 571 if (sc->sc_smr[i] != NULL) 572 continue; 573 sc->sc_smr[i] = malloc(sizeof(struct smmu_smr), 574 M_DEVBUF, M_WAITOK | M_ZERO); 575 dom->sd_smr_idx = i; 576 break; 577 } 578 579 if (i >= sc->sc_num_streams) { 580 free(sc->sc_cb[dom->sd_cb_idx], M_DEVBUF, 581 sizeof(struct smmu_cb)); 582 sc->sc_cb[dom->sd_cb_idx] = NULL; 583 free(dom, M_DEVBUF, sizeof(*dom)); 584 printf("%s: out of streams, I/O device will fail\n", 585 sc->sc_dev.dv_xname); 586 return NULL; 587 } 588 } 589 590 reg = SMMU_CBA2R_VA64; 591 if (sc->sc_has_vmid16s) 592 reg |= (dom->sd_cb_idx + 1) << SMMU_CBA2R_VMID16_SHIFT; 593 smmu_gr1_write_4(sc, SMMU_CBA2R(dom->sd_cb_idx), reg); 594 595 if (dom->sd_stage == 1) { 596 reg = SMMU_CBAR_TYPE_S1_TRANS_S2_BYPASS | 597 SMMU_CBAR_BPSHCFG_NSH | SMMU_CBAR_MEMATTR_WB; 598 } else { 599 reg = SMMU_CBAR_TYPE_S2_TRANS; 600 if (!sc->sc_has_vmid16s) 601 reg |= (dom->sd_cb_idx + 1) << SMMU_CBAR_VMID_SHIFT; 602 } 603 smmu_gr1_write_4(sc, SMMU_CBAR(dom->sd_cb_idx), reg); 604 605 if (dom->sd_stage == 1) { 606 reg = SMMU_CB_TCR2_AS | SMMU_CB_TCR2_SEP_UPSTREAM; 607 switch (sc->sc_ipa_bits) { 608 case 32: 609 reg |= SMMU_CB_TCR2_PASIZE_32BIT; 610 break; 611 case 36: 612 reg |= SMMU_CB_TCR2_PASIZE_36BIT; 613 break; 614 case 40: 615 reg |= SMMU_CB_TCR2_PASIZE_40BIT; 616 break; 617 case 42: 618 reg |= SMMU_CB_TCR2_PASIZE_42BIT; 619 break; 620 case 44: 621 reg |= SMMU_CB_TCR2_PASIZE_44BIT; 622 break; 623 case 48: 624 reg |= SMMU_CB_TCR2_PASIZE_48BIT; 625 break; 626 } 627 smmu_cb_write_4(sc, dom->sd_cb_idx, SMMU_CB_TCR2, reg); 628 } 629 630 if (dom->sd_stage == 1) 631 iovabits = sc->sc_va_bits; 632 else 633 iovabits = sc->sc_ipa_bits; 634 /* 635 * Marvell's 8040 does not support 64-bit writes, hence we 636 * can only address 44-bits of VA space for TLB invalidation. 637 */ 638 if (sc->sc_is_ap806) 639 iovabits = min(44, iovabits); 640 if (iovabits >= 40) 641 dom->sd_4level = 1; 642 643 reg = SMMU_CB_TCR_TG0_4KB | SMMU_CB_TCR_T0SZ(64 - iovabits); 644 if (dom->sd_stage == 1) { 645 reg |= SMMU_CB_TCR_EPD1; 646 } else { 647 if (dom->sd_4level) 648 reg |= SMMU_CB_TCR_S2_SL0_4KB_L0; 649 else 650 reg |= SMMU_CB_TCR_S2_SL0_4KB_L1; 651 switch (sc->sc_pa_bits) { 652 case 32: 653 reg |= SMMU_CB_TCR_S2_PASIZE_32BIT; 654 break; 655 case 36: 656 reg |= SMMU_CB_TCR_S2_PASIZE_36BIT; 657 break; 658 case 40: 659 reg |= SMMU_CB_TCR_S2_PASIZE_40BIT; 660 break; 661 case 42: 662 reg |= SMMU_CB_TCR_S2_PASIZE_42BIT; 663 break; 664 case 44: 665 reg |= SMMU_CB_TCR_S2_PASIZE_44BIT; 666 break; 667 case 48: 668 reg |= SMMU_CB_TCR_S2_PASIZE_48BIT; 669 break; 670 } 671 } 672 if (sc->sc_coherent) 673 reg |= SMMU_CB_TCR_IRGN0_WBWA | SMMU_CB_TCR_ORGN0_WBWA | 674 SMMU_CB_TCR_SH0_ISH; 675 else 676 reg |= SMMU_CB_TCR_IRGN0_NC | SMMU_CB_TCR_ORGN0_NC | 677 SMMU_CB_TCR_SH0_OSH; 678 smmu_cb_write_4(sc, dom->sd_cb_idx, SMMU_CB_TCR, reg); 679 680 if (dom->sd_4level) { 681 while (dom->sd_vp.l0 == NULL) { 682 dom->sd_vp.l0 = pool_get(&sc->sc_vp_pool, 683 PR_WAITOK | PR_ZERO); 684 } 685 l0va = (vaddr_t)dom->sd_vp.l0->l0; /* top level is l0 */ 686 } else { 687 while (dom->sd_vp.l1 == NULL) { 688 dom->sd_vp.l1 = pool_get(&sc->sc_vp_pool, 689 PR_WAITOK | PR_ZERO); 690 } 691 l0va = (vaddr_t)dom->sd_vp.l1->l1; /* top level is l1 */ 692 } 693 pmap_extract(pmap_kernel(), l0va, &pa); 694 695 if (dom->sd_stage == 1) { 696 smmu_cb_write_8(sc, dom->sd_cb_idx, SMMU_CB_TTBR0, 697 (uint64_t)dom->sd_cb_idx << SMMU_CB_TTBR_ASID_SHIFT | pa); 698 smmu_cb_write_8(sc, dom->sd_cb_idx, SMMU_CB_TTBR1, 699 (uint64_t)dom->sd_cb_idx << SMMU_CB_TTBR_ASID_SHIFT); 700 } else 701 smmu_cb_write_8(sc, dom->sd_cb_idx, SMMU_CB_TTBR0, pa); 702 703 if (dom->sd_stage == 1) { 704 smmu_cb_write_4(sc, dom->sd_cb_idx, SMMU_CB_MAIR0, 705 SMMU_CB_MAIR_MAIR_ATTR(SMMU_CB_MAIR_DEVICE_nGnRnE, 0) | 706 SMMU_CB_MAIR_MAIR_ATTR(SMMU_CB_MAIR_DEVICE_nGnRE, 1) | 707 SMMU_CB_MAIR_MAIR_ATTR(SMMU_CB_MAIR_DEVICE_NC, 2) | 708 SMMU_CB_MAIR_MAIR_ATTR(SMMU_CB_MAIR_DEVICE_WB, 3)); 709 smmu_cb_write_4(sc, dom->sd_cb_idx, SMMU_CB_MAIR1, 710 SMMU_CB_MAIR_MAIR_ATTR(SMMU_CB_MAIR_DEVICE_WT, 0)); 711 } 712 713 reg = SMMU_CB_SCTLR_M | SMMU_CB_SCTLR_TRE | SMMU_CB_SCTLR_AFE | 714 SMMU_CB_SCTLR_CFRE | SMMU_CB_SCTLR_CFIE; 715 if (dom->sd_stage == 1) 716 reg |= SMMU_CB_SCTLR_ASIDPNE; 717 smmu_cb_write_4(sc, dom->sd_cb_idx, SMMU_CB_SCTLR, reg); 718 719 /* Point stream to context block */ 720 reg = SMMU_S2CR_TYPE_TRANS | dom->sd_cb_idx; 721 if (sc->sc_has_exids && sc->sc_smr) 722 reg |= SMMU_S2CR_EXIDVALID; 723 smmu_gr0_write_4(sc, SMMU_S2CR(dom->sd_smr_idx), reg); 724 725 /* Map stream idx to S2CR idx */ 726 if (sc->sc_smr) { 727 reg = sid; 728 if (!sc->sc_has_exids) 729 reg |= SMMU_SMR_VALID; 730 smmu_gr0_write_4(sc, SMMU_SMR(dom->sd_smr_idx), reg); 731 } 732 733 snprintf(dom->sd_exname, sizeof(dom->sd_exname), "%s:%x", 734 sc->sc_dev.dv_xname, sid); 735 dom->sd_iovamap = extent_create(dom->sd_exname, 0, 736 (1LL << iovabits) - 1, M_DEVBUF, NULL, 0, EX_WAITOK | 737 EX_NOCOALESCE); 738 739 /* Reserve first page (to catch NULL access) */ 740 extent_alloc_region(dom->sd_iovamap, 0, PAGE_SIZE, EX_WAITOK); 741 742 SIMPLEQ_INSERT_TAIL(&sc->sc_domains, dom, sd_list); 743 return dom; 744 } 745 746 void 747 smmu_reserve_region(void *cookie, uint32_t sid, bus_addr_t addr, 748 bus_size_t size) 749 { 750 struct smmu_softc *sc = cookie; 751 struct smmu_domain *dom; 752 753 dom = smmu_domain_lookup(sc, sid); 754 if (dom == NULL) 755 return; 756 757 extent_alloc_region(dom->sd_iovamap, addr, size, 758 EX_WAITOK | EX_CONFLICTOK); 759 } 760 761 /* basically pmap follows */ 762 763 /* virtual to physical helpers */ 764 static inline int 765 VP_IDX0(vaddr_t va) 766 { 767 return (va >> VP_IDX0_POS) & VP_IDX0_MASK; 768 } 769 770 static inline int 771 VP_IDX1(vaddr_t va) 772 { 773 return (va >> VP_IDX1_POS) & VP_IDX1_MASK; 774 } 775 776 static inline int 777 VP_IDX2(vaddr_t va) 778 { 779 return (va >> VP_IDX2_POS) & VP_IDX2_MASK; 780 } 781 782 static inline int 783 VP_IDX3(vaddr_t va) 784 { 785 return (va >> VP_IDX3_POS) & VP_IDX3_MASK; 786 } 787 788 static inline uint64_t 789 VP_Lx(paddr_t pa) 790 { 791 /* 792 * This function takes the pa address given and manipulates it 793 * into the form that should be inserted into the VM table. 794 */ 795 return pa | Lx_TYPE_PT; 796 } 797 798 void 799 smmu_set_l1(struct smmu_domain *dom, uint64_t va, struct smmuvp1 *l1_va) 800 { 801 uint64_t pg_entry; 802 paddr_t l1_pa; 803 int idx0; 804 805 if (pmap_extract(pmap_kernel(), (vaddr_t)l1_va, &l1_pa) == 0) 806 panic("%s: unable to find vp pa mapping %p", __func__, l1_va); 807 808 if (l1_pa & (Lx_TABLE_ALIGN-1)) 809 panic("%s: misaligned L2 table", __func__); 810 811 pg_entry = VP_Lx(l1_pa); 812 813 idx0 = VP_IDX0(va); 814 dom->sd_vp.l0->vp[idx0] = l1_va; 815 dom->sd_vp.l0->l0[idx0] = pg_entry; 816 } 817 818 void 819 smmu_set_l2(struct smmu_domain *dom, uint64_t va, struct smmuvp1 *vp1, 820 struct smmuvp2 *l2_va) 821 { 822 uint64_t pg_entry; 823 paddr_t l2_pa; 824 int idx1; 825 826 if (pmap_extract(pmap_kernel(), (vaddr_t)l2_va, &l2_pa) == 0) 827 panic("%s: unable to find vp pa mapping %p", __func__, l2_va); 828 829 if (l2_pa & (Lx_TABLE_ALIGN-1)) 830 panic("%s: misaligned L2 table", __func__); 831 832 pg_entry = VP_Lx(l2_pa); 833 834 idx1 = VP_IDX1(va); 835 vp1->vp[idx1] = l2_va; 836 vp1->l1[idx1] = pg_entry; 837 } 838 839 void 840 smmu_set_l3(struct smmu_domain *dom, uint64_t va, struct smmuvp2 *vp2, 841 struct smmuvp3 *l3_va) 842 { 843 uint64_t pg_entry; 844 paddr_t l3_pa; 845 int idx2; 846 847 if (pmap_extract(pmap_kernel(), (vaddr_t)l3_va, &l3_pa) == 0) 848 panic("%s: unable to find vp pa mapping %p", __func__, l3_va); 849 850 if (l3_pa & (Lx_TABLE_ALIGN-1)) 851 panic("%s: misaligned L2 table", __func__); 852 853 pg_entry = VP_Lx(l3_pa); 854 855 idx2 = VP_IDX2(va); 856 vp2->vp[idx2] = l3_va; 857 vp2->l2[idx2] = pg_entry; 858 } 859 860 int 861 smmu_vp_lookup(struct smmu_domain *dom, vaddr_t va, uint64_t **pl3entry) 862 { 863 struct smmuvp1 *vp1; 864 struct smmuvp2 *vp2; 865 struct smmuvp3 *vp3; 866 867 if (dom->sd_4level) { 868 if (dom->sd_vp.l0 == NULL) { 869 return ENXIO; 870 } 871 vp1 = dom->sd_vp.l0->vp[VP_IDX0(va)]; 872 } else { 873 vp1 = dom->sd_vp.l1; 874 } 875 if (vp1 == NULL) { 876 return ENXIO; 877 } 878 879 vp2 = vp1->vp[VP_IDX1(va)]; 880 if (vp2 == NULL) { 881 return ENXIO; 882 } 883 884 vp3 = vp2->vp[VP_IDX2(va)]; 885 if (vp3 == NULL) { 886 return ENXIO; 887 } 888 889 if (pl3entry != NULL) 890 *pl3entry = &(vp3->l3[VP_IDX3(va)]); 891 892 return 0; 893 } 894 895 int 896 smmu_vp_enter(struct smmu_domain *dom, vaddr_t va, uint64_t **pl3entry, 897 int flags) 898 { 899 struct smmu_softc *sc = dom->sd_sc; 900 struct smmuvp1 *vp1; 901 struct smmuvp2 *vp2; 902 struct smmuvp3 *vp3; 903 904 if (dom->sd_4level) { 905 vp1 = dom->sd_vp.l0->vp[VP_IDX0(va)]; 906 if (vp1 == NULL) { 907 mtx_enter(&dom->sd_pmap_mtx); 908 vp1 = dom->sd_vp.l0->vp[VP_IDX0(va)]; 909 if (vp1 == NULL) { 910 vp1 = pool_get(&sc->sc_vp_pool, 911 PR_NOWAIT | PR_ZERO); 912 if (vp1 == NULL) { 913 mtx_leave(&dom->sd_pmap_mtx); 914 return ENOMEM; 915 } 916 smmu_set_l1(dom, va, vp1); 917 } 918 mtx_leave(&dom->sd_pmap_mtx); 919 } 920 } else { 921 vp1 = dom->sd_vp.l1; 922 } 923 924 vp2 = vp1->vp[VP_IDX1(va)]; 925 if (vp2 == NULL) { 926 mtx_enter(&dom->sd_pmap_mtx); 927 vp2 = vp1->vp[VP_IDX1(va)]; 928 if (vp2 == NULL) { 929 vp2 = pool_get(&sc->sc_vp_pool, PR_NOWAIT | PR_ZERO); 930 if (vp2 == NULL) { 931 mtx_leave(&dom->sd_pmap_mtx); 932 return ENOMEM; 933 } 934 smmu_set_l2(dom, va, vp1, vp2); 935 } 936 mtx_leave(&dom->sd_pmap_mtx); 937 } 938 939 vp3 = vp2->vp[VP_IDX2(va)]; 940 if (vp3 == NULL) { 941 mtx_enter(&dom->sd_pmap_mtx); 942 vp3 = vp2->vp[VP_IDX2(va)]; 943 if (vp3 == NULL) { 944 vp3 = pool_get(&sc->sc_vp3_pool, PR_NOWAIT | PR_ZERO); 945 if (vp3 == NULL) { 946 mtx_leave(&dom->sd_pmap_mtx); 947 return ENOMEM; 948 } 949 smmu_set_l3(dom, va, vp2, vp3); 950 } 951 mtx_leave(&dom->sd_pmap_mtx); 952 } 953 954 if (pl3entry != NULL) 955 *pl3entry = &(vp3->l3[VP_IDX3(va)]); 956 957 return 0; 958 } 959 960 uint64_t 961 smmu_fill_pte(struct smmu_domain *dom, vaddr_t va, paddr_t pa, 962 vm_prot_t prot, int flags, int cache) 963 { 964 uint64_t pted; 965 966 pted = pa & PTE_RPGN; 967 968 switch (cache) { 969 case PMAP_CACHE_WB: 970 break; 971 case PMAP_CACHE_WT: 972 break; 973 case PMAP_CACHE_CI: 974 break; 975 case PMAP_CACHE_DEV_NGNRNE: 976 break; 977 case PMAP_CACHE_DEV_NGNRE: 978 break; 979 default: 980 panic("%s: invalid cache mode", __func__); 981 } 982 983 pted |= cache; 984 pted |= flags & (PROT_READ|PROT_WRITE|PROT_EXEC); 985 return pted; 986 } 987 988 void 989 smmu_pte_update(struct smmu_domain *dom, uint64_t pted, uint64_t *pl3) 990 { 991 uint64_t pte, access_bits; 992 uint64_t attr = 0; 993 994 /* see mair in locore.S */ 995 switch (pted & PMAP_CACHE_BITS) { 996 case PMAP_CACHE_WB: 997 /* inner and outer writeback */ 998 if (dom->sd_stage == 1) 999 attr |= ATTR_IDX(PTE_ATTR_WB); 1000 else 1001 attr |= ATTR_IDX(PTE_MEMATTR_WB); 1002 attr |= ATTR_SH(SH_INNER); 1003 break; 1004 case PMAP_CACHE_WT: 1005 /* inner and outer writethrough */ 1006 if (dom->sd_stage == 1) 1007 attr |= ATTR_IDX(PTE_ATTR_WT); 1008 else 1009 attr |= ATTR_IDX(PTE_MEMATTR_WT); 1010 attr |= ATTR_SH(SH_INNER); 1011 break; 1012 case PMAP_CACHE_CI: 1013 if (dom->sd_stage == 1) 1014 attr |= ATTR_IDX(PTE_ATTR_CI); 1015 else 1016 attr |= ATTR_IDX(PTE_MEMATTR_CI); 1017 attr |= ATTR_SH(SH_INNER); 1018 break; 1019 case PMAP_CACHE_DEV_NGNRNE: 1020 if (dom->sd_stage == 1) 1021 attr |= ATTR_IDX(PTE_ATTR_DEV_NGNRNE); 1022 else 1023 attr |= ATTR_IDX(PTE_MEMATTR_DEV_NGNRNE); 1024 attr |= ATTR_SH(SH_INNER); 1025 break; 1026 case PMAP_CACHE_DEV_NGNRE: 1027 if (dom->sd_stage == 1) 1028 attr |= ATTR_IDX(PTE_ATTR_DEV_NGNRE); 1029 else 1030 attr |= ATTR_IDX(PTE_MEMATTR_DEV_NGNRE); 1031 attr |= ATTR_SH(SH_INNER); 1032 break; 1033 default: 1034 panic("%s: invalid cache mode", __func__); 1035 } 1036 1037 access_bits = ATTR_PXN | ATTR_AF; 1038 if (dom->sd_stage == 1) { 1039 attr |= ATTR_nG; 1040 access_bits |= ATTR_AP(1); 1041 if ((pted & PROT_READ) && 1042 !(pted & PROT_WRITE)) 1043 access_bits |= ATTR_AP(2); 1044 } else { 1045 if (pted & PROT_READ) 1046 access_bits |= ATTR_AP(1); 1047 if (pted & PROT_WRITE) 1048 access_bits |= ATTR_AP(2); 1049 } 1050 1051 pte = (pted & PTE_RPGN) | attr | access_bits | L3_P; 1052 *pl3 = pte; 1053 } 1054 1055 void 1056 smmu_pte_remove(struct smmu_domain *dom, vaddr_t va) 1057 { 1058 /* put entry into table */ 1059 /* need to deal with ref/change here */ 1060 struct smmuvp1 *vp1; 1061 struct smmuvp2 *vp2; 1062 struct smmuvp3 *vp3; 1063 1064 if (dom->sd_4level) 1065 vp1 = dom->sd_vp.l0->vp[VP_IDX0(va)]; 1066 else 1067 vp1 = dom->sd_vp.l1; 1068 if (vp1 == NULL) { 1069 panic("%s: missing the l1 for va %lx domain %p", __func__, 1070 va, dom); 1071 } 1072 vp2 = vp1->vp[VP_IDX1(va)]; 1073 if (vp2 == NULL) { 1074 panic("%s: missing the l2 for va %lx domain %p", __func__, 1075 va, dom); 1076 } 1077 vp3 = vp2->vp[VP_IDX2(va)]; 1078 if (vp3 == NULL) { 1079 panic("%s: missing the l3 for va %lx domain %p", __func__, 1080 va, dom); 1081 } 1082 vp3->l3[VP_IDX3(va)] = 0; 1083 } 1084 1085 int 1086 smmu_enter(struct smmu_domain *dom, vaddr_t va, paddr_t pa, vm_prot_t prot, 1087 int flags, int cache) 1088 { 1089 uint64_t *pl3; 1090 1091 if (smmu_vp_lookup(dom, va, &pl3) != 0) { 1092 if (smmu_vp_enter(dom, va, &pl3, flags)) 1093 return ENOMEM; 1094 } 1095 1096 if (flags & (PROT_READ|PROT_WRITE|PROT_EXEC)) 1097 smmu_map(dom, va, pa, prot, flags, cache); 1098 1099 return 0; 1100 } 1101 1102 void 1103 smmu_map(struct smmu_domain *dom, vaddr_t va, paddr_t pa, vm_prot_t prot, 1104 int flags, int cache) 1105 { 1106 uint64_t *pl3; 1107 uint64_t pted; 1108 int ret; 1109 1110 /* IOVA must already be allocated */ 1111 ret = smmu_vp_lookup(dom, va, &pl3); 1112 KASSERT(ret == 0); 1113 1114 /* Update PTED information for physical address */ 1115 pted = smmu_fill_pte(dom, va, pa, prot, flags, cache); 1116 1117 /* Insert updated information */ 1118 smmu_pte_update(dom, pted, pl3); 1119 membar_producer(); /* XXX bus dma sync? */ 1120 } 1121 1122 void 1123 smmu_unmap(struct smmu_domain *dom, vaddr_t va) 1124 { 1125 struct smmu_softc *sc = dom->sd_sc; 1126 int ret; 1127 1128 /* IOVA must already be allocated */ 1129 ret = smmu_vp_lookup(dom, va, NULL); 1130 KASSERT(ret == 0); 1131 1132 /* Remove mapping from pagetable */ 1133 smmu_pte_remove(dom, va); 1134 membar_producer(); /* XXX bus dma sync? */ 1135 1136 /* Invalidate IOTLB */ 1137 if (dom->sd_stage == 1) 1138 smmu_cb_write_8(sc, dom->sd_cb_idx, SMMU_CB_TLBIVAL, 1139 (uint64_t)dom->sd_cb_idx << 48 | va >> PAGE_SHIFT); 1140 else 1141 smmu_cb_write_8(sc, dom->sd_cb_idx, SMMU_CB_TLBIIPAS2L, 1142 va >> PAGE_SHIFT); 1143 } 1144 1145 void 1146 smmu_remove(struct smmu_domain *dom, vaddr_t va) 1147 { 1148 /* TODO: garbage collect page tables? */ 1149 } 1150 1151 int 1152 smmu_load_map(struct smmu_domain *dom, bus_dmamap_t map) 1153 { 1154 struct smmu_map_state *sms = map->_dm_cookie; 1155 u_long dva, maplen; 1156 int seg; 1157 1158 maplen = 0; 1159 for (seg = 0; seg < map->dm_nsegs; seg++) { 1160 paddr_t pa = map->dm_segs[seg]._ds_paddr; 1161 psize_t off = pa - trunc_page(pa); 1162 maplen += round_page(map->dm_segs[seg].ds_len + off); 1163 } 1164 KASSERT(maplen <= sms->sms_len); 1165 1166 dva = sms->sms_dva; 1167 for (seg = 0; seg < map->dm_nsegs; seg++) { 1168 paddr_t pa = map->dm_segs[seg]._ds_paddr; 1169 psize_t off = pa - trunc_page(pa); 1170 u_long len = round_page(map->dm_segs[seg].ds_len + off); 1171 1172 map->dm_segs[seg].ds_addr = dva + off; 1173 1174 pa = trunc_page(pa); 1175 while (len > 0) { 1176 smmu_map(dom, dva, pa, 1177 PROT_READ | PROT_WRITE, 1178 PROT_READ | PROT_WRITE, PMAP_CACHE_WB); 1179 1180 dva += PAGE_SIZE; 1181 pa += PAGE_SIZE; 1182 len -= PAGE_SIZE; 1183 sms->sms_loaded += PAGE_SIZE; 1184 } 1185 } 1186 1187 return 0; 1188 } 1189 1190 void 1191 smmu_unload_map(struct smmu_domain *dom, bus_dmamap_t map) 1192 { 1193 struct smmu_map_state *sms = map->_dm_cookie; 1194 u_long len, dva; 1195 1196 if (sms->sms_loaded == 0) 1197 return; 1198 1199 dva = sms->sms_dva; 1200 len = sms->sms_loaded; 1201 1202 while (len > 0) { 1203 smmu_unmap(dom, dva); 1204 1205 dva += PAGE_SIZE; 1206 len -= PAGE_SIZE; 1207 } 1208 1209 sms->sms_loaded = 0; 1210 1211 smmu_tlb_sync_context(dom); 1212 } 1213 1214 int 1215 smmu_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 1216 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamap) 1217 { 1218 struct smmu_domain *dom = t->_cookie; 1219 struct smmu_softc *sc = dom->sd_sc; 1220 struct smmu_map_state *sms; 1221 bus_dmamap_t map; 1222 u_long dva, len; 1223 int error; 1224 1225 error = sc->sc_dmat->_dmamap_create(sc->sc_dmat, size, 1226 nsegments, maxsegsz, boundary, flags, &map); 1227 if (error) 1228 return error; 1229 1230 sms = malloc(sizeof(*sms), M_DEVBUF, (flags & BUS_DMA_NOWAIT) ? 1231 (M_NOWAIT|M_ZERO) : (M_WAITOK|M_ZERO)); 1232 if (sms == NULL) { 1233 sc->sc_dmat->_dmamap_destroy(sc->sc_dmat, map); 1234 return ENOMEM; 1235 } 1236 1237 /* Approximation of maximum pages needed. */ 1238 len = round_page(size) + nsegments * PAGE_SIZE; 1239 1240 /* Allocate IOVA, and a guard page at the end. */ 1241 mtx_enter(&dom->sd_iova_mtx); 1242 error = extent_alloc_with_descr(dom->sd_iovamap, len + PAGE_SIZE, 1243 PAGE_SIZE, 0, 0, EX_NOWAIT, &sms->sms_er, &dva); 1244 mtx_leave(&dom->sd_iova_mtx); 1245 if (error) { 1246 sc->sc_dmat->_dmamap_destroy(sc->sc_dmat, map); 1247 free(sms, M_DEVBUF, sizeof(*sms)); 1248 return error; 1249 } 1250 1251 sms->sms_dva = dva; 1252 sms->sms_len = len; 1253 1254 while (len > 0) { 1255 error = smmu_enter(dom, dva, dva, PROT_READ | PROT_WRITE, 1256 PROT_NONE, PMAP_CACHE_WB); 1257 KASSERT(error == 0); /* FIXME: rollback smmu_enter() */ 1258 dva += PAGE_SIZE; 1259 len -= PAGE_SIZE; 1260 } 1261 1262 map->_dm_cookie = sms; 1263 *dmamap = map; 1264 return 0; 1265 } 1266 1267 void 1268 smmu_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 1269 { 1270 struct smmu_domain *dom = t->_cookie; 1271 struct smmu_softc *sc = dom->sd_sc; 1272 struct smmu_map_state *sms = map->_dm_cookie; 1273 u_long dva, len; 1274 int error; 1275 1276 if (sms->sms_loaded) 1277 smmu_dmamap_unload(t, map); 1278 1279 dva = sms->sms_dva; 1280 len = sms->sms_len; 1281 1282 while (len > 0) { 1283 smmu_remove(dom, dva); 1284 dva += PAGE_SIZE; 1285 len -= PAGE_SIZE; 1286 } 1287 1288 mtx_enter(&dom->sd_iova_mtx); 1289 error = extent_free(dom->sd_iovamap, sms->sms_dva, 1290 sms->sms_len + PAGE_SIZE, EX_NOWAIT); 1291 mtx_leave(&dom->sd_iova_mtx); 1292 KASSERT(error == 0); 1293 1294 free(sms, M_DEVBUF, sizeof(*sms)); 1295 sc->sc_dmat->_dmamap_destroy(sc->sc_dmat, map); 1296 } 1297 1298 int 1299 smmu_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 1300 bus_size_t buflen, struct proc *p, int flags) 1301 { 1302 struct smmu_domain *dom = t->_cookie; 1303 struct smmu_softc *sc = dom->sd_sc; 1304 int error; 1305 1306 error = sc->sc_dmat->_dmamap_load(sc->sc_dmat, map, 1307 buf, buflen, p, flags); 1308 if (error) 1309 return error; 1310 1311 error = smmu_load_map(dom, map); 1312 if (error) 1313 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 1314 1315 return error; 1316 } 1317 1318 int 1319 smmu_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, 1320 int flags) 1321 { 1322 struct smmu_domain *dom = t->_cookie; 1323 struct smmu_softc *sc = dom->sd_sc; 1324 int error; 1325 1326 error = sc->sc_dmat->_dmamap_load_mbuf(sc->sc_dmat, map, 1327 m0, flags); 1328 if (error) 1329 return error; 1330 1331 error = smmu_load_map(dom, map); 1332 if (error) 1333 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 1334 1335 return error; 1336 } 1337 1338 int 1339 smmu_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, 1340 int flags) 1341 { 1342 struct smmu_domain *dom = t->_cookie; 1343 struct smmu_softc *sc = dom->sd_sc; 1344 int error; 1345 1346 error = sc->sc_dmat->_dmamap_load_uio(sc->sc_dmat, map, 1347 uio, flags); 1348 if (error) 1349 return error; 1350 1351 error = smmu_load_map(dom, map); 1352 if (error) 1353 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 1354 1355 return error; 1356 } 1357 1358 int 1359 smmu_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, bus_dma_segment_t *segs, 1360 int nsegs, bus_size_t size, int flags) 1361 { 1362 struct smmu_domain *dom = t->_cookie; 1363 struct smmu_softc *sc = dom->sd_sc; 1364 int error; 1365 1366 error = sc->sc_dmat->_dmamap_load_raw(sc->sc_dmat, map, 1367 segs, nsegs, size, flags); 1368 if (error) 1369 return error; 1370 1371 error = smmu_load_map(dom, map); 1372 if (error) 1373 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 1374 1375 return error; 1376 } 1377 1378 void 1379 smmu_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 1380 { 1381 struct smmu_domain *dom = t->_cookie; 1382 struct smmu_softc *sc = dom->sd_sc; 1383 1384 smmu_unload_map(dom, map); 1385 sc->sc_dmat->_dmamap_unload(sc->sc_dmat, map); 1386 } 1387