1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_acpi.h" 36 37 #include <sys/param.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/memdesc.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/taskqueue.h> 45 #include <sys/time.h> 46 #include <sys/tree.h> 47 #include <sys/vmem.h> 48 #include <vm/vm.h> 49 #include <vm/vm_extern.h> 50 #include <vm/vm_kern.h> 51 #include <vm/vm_page.h> 52 #include <vm/vm_map.h> 53 #include <contrib/dev/acpica/include/acpi.h> 54 #include <contrib/dev/acpica/include/accommon.h> 55 #include <dev/acpica/acpivar.h> 56 #include <dev/pci/pcireg.h> 57 #include <machine/bus.h> 58 #include <machine/cpu.h> 59 #include <x86/include/busdma_impl.h> 60 #include <dev/iommu/busdma_iommu.h> 61 #include <x86/iommu/intel_reg.h> 62 #include <x86/iommu/intel_dmar.h> 63 64 static bool 65 dmar_qi_seq_processed(const struct dmar_unit *unit, 66 const struct iommu_qi_genseq *pseq) 67 { 68 69 return (pseq->gen < unit->inv_waitd_gen || 70 (pseq->gen == unit->inv_waitd_gen && 71 pseq->seq <= unit->inv_waitd_seq_hw)); 72 } 73 74 static int 75 dmar_enable_qi(struct dmar_unit *unit) 76 { 77 int error; 78 79 DMAR_ASSERT_LOCKED(unit); 80 unit->hw_gcmd |= DMAR_GCMD_QIE; 81 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 82 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 83 != 0)); 84 return (error); 85 } 86 87 static int 88 dmar_disable_qi(struct dmar_unit *unit) 89 { 90 int error; 91 92 DMAR_ASSERT_LOCKED(unit); 93 unit->hw_gcmd &= ~DMAR_GCMD_QIE; 94 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 95 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 96 == 0)); 97 return (error); 98 } 99 100 static void 101 dmar_qi_advance_tail(struct dmar_unit *unit) 102 { 103 104 DMAR_ASSERT_LOCKED(unit); 105 dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail); 106 } 107 108 static void 109 dmar_qi_ensure(struct dmar_unit *unit, int descr_count) 110 { 111 uint32_t head; 112 int bytes; 113 114 DMAR_ASSERT_LOCKED(unit); 115 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; 116 for (;;) { 117 if (bytes <= unit->inv_queue_avail) 118 break; 119 /* refill */ 120 head = dmar_read4(unit, DMAR_IQH_REG); 121 head &= DMAR_IQH_MASK; 122 unit->inv_queue_avail = head - unit->inv_queue_tail - 123 DMAR_IQ_DESCR_SZ; 124 if (head <= unit->inv_queue_tail) 125 unit->inv_queue_avail += unit->inv_queue_size; 126 if (bytes <= unit->inv_queue_avail) 127 break; 128 129 /* 130 * No space in the queue, do busy wait. Hardware must 131 * make a progress. But first advance the tail to 132 * inform the descriptor streamer about entries we 133 * might have already filled, otherwise they could 134 * clog the whole queue.. 135 */ 136 dmar_qi_advance_tail(unit); 137 unit->inv_queue_full++; 138 cpu_spinwait(); 139 } 140 unit->inv_queue_avail -= bytes; 141 } 142 143 static void 144 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) 145 { 146 147 DMAR_ASSERT_LOCKED(unit); 148 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1; 149 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 150 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 151 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 152 (uintmax_t)unit->inv_queue_size)); 153 unit->inv_queue_tail &= unit->inv_queue_size - 1; 154 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2; 155 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 156 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 157 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 158 (uintmax_t)unit->inv_queue_size)); 159 unit->inv_queue_tail &= unit->inv_queue_size - 1; 160 } 161 162 static void 163 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr, 164 bool memw, bool fence) 165 { 166 167 DMAR_ASSERT_LOCKED(unit); 168 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | 169 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | 170 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | 171 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | 172 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), 173 memw ? unit->inv_waitd_seq_hw_phys : 0); 174 } 175 176 static void 177 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct iommu_qi_genseq *pseq, 178 bool emit_wait) 179 { 180 struct iommu_qi_genseq gsec; 181 uint32_t seq; 182 183 KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); 184 DMAR_ASSERT_LOCKED(unit); 185 if (unit->inv_waitd_seq == 0xffffffff) { 186 gsec.gen = unit->inv_waitd_gen; 187 gsec.seq = unit->inv_waitd_seq; 188 dmar_qi_ensure(unit, 1); 189 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false); 190 dmar_qi_advance_tail(unit); 191 while (!dmar_qi_seq_processed(unit, &gsec)) 192 cpu_spinwait(); 193 unit->inv_waitd_gen++; 194 unit->inv_waitd_seq = 1; 195 } 196 seq = unit->inv_waitd_seq++; 197 pseq->gen = unit->inv_waitd_gen; 198 pseq->seq = seq; 199 if (emit_wait) { 200 dmar_qi_ensure(unit, 1); 201 dmar_qi_emit_wait_descr(unit, seq, true, true, false); 202 } 203 } 204 205 static void 206 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct iommu_qi_genseq *gseq, 207 bool nowait) 208 { 209 210 DMAR_ASSERT_LOCKED(unit); 211 unit->inv_seq_waiters++; 212 while (!dmar_qi_seq_processed(unit, gseq)) { 213 if (cold || nowait) { 214 cpu_spinwait(); 215 } else { 216 msleep(&unit->inv_seq_waiters, &unit->iommu.lock, 0, 217 "dmarse", hz); 218 } 219 } 220 unit->inv_seq_waiters--; 221 } 222 223 void 224 dmar_qi_invalidate_locked(struct dmar_domain *domain, iommu_gaddr_t base, 225 iommu_gaddr_t size, struct iommu_qi_genseq *pseq, bool emit_wait) 226 { 227 struct dmar_unit *unit; 228 iommu_gaddr_t isize; 229 int am; 230 231 unit = domain->dmar; 232 DMAR_ASSERT_LOCKED(unit); 233 for (; size > 0; base += isize, size -= isize) { 234 am = calc_am(unit, base, size, &isize); 235 dmar_qi_ensure(unit, 1); 236 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | 237 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | 238 DMAR_IQ_DESCR_IOTLB_DR | 239 DMAR_IQ_DESCR_IOTLB_DID(domain->domain), 240 base | am); 241 } 242 dmar_qi_emit_wait_seq(unit, pseq, emit_wait); 243 dmar_qi_advance_tail(unit); 244 } 245 246 void 247 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) 248 { 249 struct iommu_qi_genseq gseq; 250 251 DMAR_ASSERT_LOCKED(unit); 252 dmar_qi_ensure(unit, 2); 253 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); 254 dmar_qi_emit_wait_seq(unit, &gseq, true); 255 dmar_qi_advance_tail(unit); 256 dmar_qi_wait_for_seq(unit, &gseq, false); 257 } 258 259 void 260 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) 261 { 262 struct iommu_qi_genseq gseq; 263 264 DMAR_ASSERT_LOCKED(unit); 265 dmar_qi_ensure(unit, 2); 266 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | 267 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); 268 dmar_qi_emit_wait_seq(unit, &gseq, true); 269 dmar_qi_advance_tail(unit); 270 dmar_qi_wait_for_seq(unit, &gseq, false); 271 } 272 273 void 274 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) 275 { 276 struct iommu_qi_genseq gseq; 277 278 DMAR_ASSERT_LOCKED(unit); 279 dmar_qi_ensure(unit, 2); 280 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); 281 dmar_qi_emit_wait_seq(unit, &gseq, true); 282 dmar_qi_advance_tail(unit); 283 dmar_qi_wait_for_seq(unit, &gseq, false); 284 } 285 286 void 287 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) 288 { 289 struct iommu_qi_genseq gseq; 290 u_int c, l; 291 292 DMAR_ASSERT_LOCKED(unit); 293 KASSERT(start < unit->irte_cnt && start < start + cnt && 294 start + cnt <= unit->irte_cnt, 295 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); 296 for (; cnt > 0; cnt -= c, start += c) { 297 l = ffs(start | cnt) - 1; 298 c = 1 << l; 299 dmar_qi_ensure(unit, 1); 300 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | 301 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) | 302 DMAR_IQ_DESCR_IEC_IM(l), 0); 303 } 304 dmar_qi_ensure(unit, 1); 305 dmar_qi_emit_wait_seq(unit, &gseq, true); 306 dmar_qi_advance_tail(unit); 307 308 /* 309 * The caller of the function, in particular, 310 * dmar_ir_program_irte(), may be called from the context 311 * where the sleeping is forbidden (in fact, the 312 * intr_table_lock mutex may be held, locked from 313 * intr_shuffle_irqs()). Wait for the invalidation completion 314 * using the busy wait. 315 * 316 * The impact on the interrupt input setup code is small, the 317 * expected overhead is comparable with the chipset register 318 * read. It is more harmful for the parallel DMA operations, 319 * since we own the dmar unit lock until whole invalidation 320 * queue is processed, which includes requests possibly issued 321 * before our request. 322 */ 323 dmar_qi_wait_for_seq(unit, &gseq, true); 324 } 325 326 int 327 dmar_qi_intr(void *arg) 328 { 329 struct dmar_unit *unit; 330 331 unit = arg; 332 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", 333 unit->iommu.unit)); 334 taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task); 335 return (FILTER_HANDLED); 336 } 337 338 static void 339 dmar_qi_task(void *arg, int pending __unused) 340 { 341 struct dmar_unit *unit; 342 struct iommu_map_entry *entry; 343 uint32_t ics; 344 345 unit = arg; 346 347 DMAR_LOCK(unit); 348 for (;;) { 349 entry = TAILQ_FIRST(&unit->tlb_flush_entries); 350 if (entry == NULL) 351 break; 352 if (!dmar_qi_seq_processed(unit, &entry->gseq)) 353 break; 354 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); 355 DMAR_UNLOCK(unit); 356 dmar_domain_free_entry(entry, (entry->flags & 357 IOMMU_MAP_ENTRY_QI_NF) == 0); 358 DMAR_LOCK(unit); 359 } 360 ics = dmar_read4(unit, DMAR_ICS_REG); 361 if ((ics & DMAR_ICS_IWC) != 0) { 362 ics = DMAR_ICS_IWC; 363 dmar_write4(unit, DMAR_ICS_REG, ics); 364 } 365 if (unit->inv_seq_waiters > 0) 366 wakeup(&unit->inv_seq_waiters); 367 DMAR_UNLOCK(unit); 368 } 369 370 int 371 dmar_init_qi(struct dmar_unit *unit) 372 { 373 uint64_t iqa; 374 uint32_t ics; 375 int qi_sz; 376 377 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0) 378 return (0); 379 unit->qi_enabled = 1; 380 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled); 381 if (!unit->qi_enabled) 382 return (0); 383 384 TAILQ_INIT(&unit->tlb_flush_entries); 385 TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit); 386 unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK, 387 taskqueue_thread_enqueue, &unit->qi_taskqueue); 388 taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV, 389 "dmar%d qi taskq", unit->iommu.unit); 390 391 unit->inv_waitd_gen = 0; 392 unit->inv_waitd_seq = 1; 393 394 qi_sz = DMAR_IQA_QS_DEF; 395 TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz); 396 if (qi_sz > DMAR_IQA_QS_MAX) 397 qi_sz = DMAR_IQA_QS_MAX; 398 unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE; 399 /* Reserve one descriptor to prevent wraparound. */ 400 unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ; 401 402 /* The invalidation queue reads by DMARs are always coherent. */ 403 unit->inv_queue = kmem_alloc_contig(unit->inv_queue_size, M_WAITOK | 404 M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 405 unit->inv_waitd_seq_hw_phys = pmap_kextract( 406 (vm_offset_t)&unit->inv_waitd_seq_hw); 407 408 DMAR_LOCK(unit); 409 dmar_write8(unit, DMAR_IQT_REG, 0); 410 iqa = pmap_kextract(unit->inv_queue); 411 iqa |= qi_sz; 412 dmar_write8(unit, DMAR_IQA_REG, iqa); 413 dmar_enable_qi(unit); 414 ics = dmar_read4(unit, DMAR_ICS_REG); 415 if ((ics & DMAR_ICS_IWC) != 0) { 416 ics = DMAR_ICS_IWC; 417 dmar_write4(unit, DMAR_ICS_REG, ics); 418 } 419 dmar_enable_qi_intr(unit); 420 DMAR_UNLOCK(unit); 421 422 return (0); 423 } 424 425 void 426 dmar_fini_qi(struct dmar_unit *unit) 427 { 428 struct iommu_qi_genseq gseq; 429 430 if (!unit->qi_enabled) 431 return; 432 taskqueue_drain(unit->qi_taskqueue, &unit->qi_task); 433 taskqueue_free(unit->qi_taskqueue); 434 unit->qi_taskqueue = NULL; 435 436 DMAR_LOCK(unit); 437 /* quisce */ 438 dmar_qi_ensure(unit, 1); 439 dmar_qi_emit_wait_seq(unit, &gseq, true); 440 dmar_qi_advance_tail(unit); 441 dmar_qi_wait_for_seq(unit, &gseq, false); 442 /* only after the quisce, disable queue */ 443 dmar_disable_qi_intr(unit); 444 dmar_disable_qi(unit); 445 KASSERT(unit->inv_seq_waiters == 0, 446 ("dmar%d: waiters on disabled queue", unit->iommu.unit)); 447 DMAR_UNLOCK(unit); 448 449 kmem_free(unit->inv_queue, unit->inv_queue_size); 450 unit->inv_queue = 0; 451 unit->inv_queue_size = 0; 452 unit->qi_enabled = 0; 453 } 454 455 void 456 dmar_enable_qi_intr(struct dmar_unit *unit) 457 { 458 uint32_t iectl; 459 460 DMAR_ASSERT_LOCKED(unit); 461 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", 462 unit->iommu.unit)); 463 iectl = dmar_read4(unit, DMAR_IECTL_REG); 464 iectl &= ~DMAR_IECTL_IM; 465 dmar_write4(unit, DMAR_IECTL_REG, iectl); 466 } 467 468 void 469 dmar_disable_qi_intr(struct dmar_unit *unit) 470 { 471 uint32_t iectl; 472 473 DMAR_ASSERT_LOCKED(unit); 474 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", 475 unit->iommu.unit)); 476 iectl = dmar_read4(unit, DMAR_IECTL_REG); 477 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); 478 } 479