1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2013 The FreeBSD Foundation 5 * All rights reserved. 6 * 7 * This software was developed by Konstantin Belousov <kib@FreeBSD.org> 8 * under sponsorship from the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_acpi.h" 36 37 #include <sys/param.h> 38 #include <sys/bus.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/memdesc.h> 42 #include <sys/module.h> 43 #include <sys/rman.h> 44 #include <sys/taskqueue.h> 45 #include <sys/time.h> 46 #include <sys/tree.h> 47 #include <sys/vmem.h> 48 #include <machine/bus.h> 49 #include <contrib/dev/acpica/include/acpi.h> 50 #include <contrib/dev/acpica/include/accommon.h> 51 #include <dev/acpica/acpivar.h> 52 #include <vm/vm.h> 53 #include <vm/vm_extern.h> 54 #include <vm/vm_kern.h> 55 #include <vm/vm_page.h> 56 #include <vm/vm_map.h> 57 #include <machine/cpu.h> 58 #include <x86/include/busdma_impl.h> 59 #include <x86/iommu/intel_reg.h> 60 #include <x86/iommu/busdma_dmar.h> 61 #include <x86/iommu/intel_dmar.h> 62 63 static bool 64 dmar_qi_seq_processed(const struct dmar_unit *unit, 65 const struct dmar_qi_genseq *pseq) 66 { 67 68 return (pseq->gen < unit->inv_waitd_gen || 69 (pseq->gen == unit->inv_waitd_gen && 70 pseq->seq <= unit->inv_waitd_seq_hw)); 71 } 72 73 static int 74 dmar_enable_qi(struct dmar_unit *unit) 75 { 76 int error; 77 78 DMAR_ASSERT_LOCKED(unit); 79 unit->hw_gcmd |= DMAR_GCMD_QIE; 80 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 81 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 82 != 0)); 83 return (error); 84 } 85 86 static int 87 dmar_disable_qi(struct dmar_unit *unit) 88 { 89 int error; 90 91 DMAR_ASSERT_LOCKED(unit); 92 unit->hw_gcmd &= ~DMAR_GCMD_QIE; 93 dmar_write4(unit, DMAR_GCMD_REG, unit->hw_gcmd); 94 DMAR_WAIT_UNTIL(((dmar_read4(unit, DMAR_GSTS_REG) & DMAR_GSTS_QIES) 95 == 0)); 96 return (error); 97 } 98 99 static void 100 dmar_qi_advance_tail(struct dmar_unit *unit) 101 { 102 103 DMAR_ASSERT_LOCKED(unit); 104 dmar_write4(unit, DMAR_IQT_REG, unit->inv_queue_tail); 105 } 106 107 static void 108 dmar_qi_ensure(struct dmar_unit *unit, int descr_count) 109 { 110 uint32_t head; 111 int bytes; 112 113 DMAR_ASSERT_LOCKED(unit); 114 bytes = descr_count << DMAR_IQ_DESCR_SZ_SHIFT; 115 for (;;) { 116 if (bytes <= unit->inv_queue_avail) 117 break; 118 /* refill */ 119 head = dmar_read4(unit, DMAR_IQH_REG); 120 head &= DMAR_IQH_MASK; 121 unit->inv_queue_avail = head - unit->inv_queue_tail - 122 DMAR_IQ_DESCR_SZ; 123 if (head <= unit->inv_queue_tail) 124 unit->inv_queue_avail += unit->inv_queue_size; 125 if (bytes <= unit->inv_queue_avail) 126 break; 127 128 /* 129 * No space in the queue, do busy wait. Hardware must 130 * make a progress. But first advance the tail to 131 * inform the descriptor streamer about entries we 132 * might have already filled, otherwise they could 133 * clog the whole queue.. 134 */ 135 dmar_qi_advance_tail(unit); 136 unit->inv_queue_full++; 137 cpu_spinwait(); 138 } 139 unit->inv_queue_avail -= bytes; 140 } 141 142 static void 143 dmar_qi_emit(struct dmar_unit *unit, uint64_t data1, uint64_t data2) 144 { 145 146 DMAR_ASSERT_LOCKED(unit); 147 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data1; 148 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 149 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 150 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 151 (uintmax_t)unit->inv_queue_size)); 152 unit->inv_queue_tail &= unit->inv_queue_size - 1; 153 *(volatile uint64_t *)(unit->inv_queue + unit->inv_queue_tail) = data2; 154 unit->inv_queue_tail += DMAR_IQ_DESCR_SZ / 2; 155 KASSERT(unit->inv_queue_tail <= unit->inv_queue_size, 156 ("tail overflow 0x%x 0x%jx", unit->inv_queue_tail, 157 (uintmax_t)unit->inv_queue_size)); 158 unit->inv_queue_tail &= unit->inv_queue_size - 1; 159 } 160 161 static void 162 dmar_qi_emit_wait_descr(struct dmar_unit *unit, uint32_t seq, bool intr, 163 bool memw, bool fence) 164 { 165 166 DMAR_ASSERT_LOCKED(unit); 167 dmar_qi_emit(unit, DMAR_IQ_DESCR_WAIT_ID | 168 (intr ? DMAR_IQ_DESCR_WAIT_IF : 0) | 169 (memw ? DMAR_IQ_DESCR_WAIT_SW : 0) | 170 (fence ? DMAR_IQ_DESCR_WAIT_FN : 0) | 171 (memw ? DMAR_IQ_DESCR_WAIT_SD(seq) : 0), 172 memw ? unit->inv_waitd_seq_hw_phys : 0); 173 } 174 175 static void 176 dmar_qi_emit_wait_seq(struct dmar_unit *unit, struct dmar_qi_genseq *pseq, 177 bool emit_wait) 178 { 179 struct dmar_qi_genseq gsec; 180 uint32_t seq; 181 182 KASSERT(pseq != NULL, ("wait descriptor with no place for seq")); 183 DMAR_ASSERT_LOCKED(unit); 184 if (unit->inv_waitd_seq == 0xffffffff) { 185 gsec.gen = unit->inv_waitd_gen; 186 gsec.seq = unit->inv_waitd_seq; 187 dmar_qi_ensure(unit, 1); 188 dmar_qi_emit_wait_descr(unit, gsec.seq, false, true, false); 189 dmar_qi_advance_tail(unit); 190 while (!dmar_qi_seq_processed(unit, &gsec)) 191 cpu_spinwait(); 192 unit->inv_waitd_gen++; 193 unit->inv_waitd_seq = 1; 194 } 195 seq = unit->inv_waitd_seq++; 196 pseq->gen = unit->inv_waitd_gen; 197 pseq->seq = seq; 198 if (emit_wait) { 199 dmar_qi_ensure(unit, 1); 200 dmar_qi_emit_wait_descr(unit, seq, true, true, false); 201 } 202 } 203 204 static void 205 dmar_qi_wait_for_seq(struct dmar_unit *unit, const struct dmar_qi_genseq *gseq, 206 bool nowait) 207 { 208 209 DMAR_ASSERT_LOCKED(unit); 210 unit->inv_seq_waiters++; 211 while (!dmar_qi_seq_processed(unit, gseq)) { 212 if (cold || nowait) { 213 cpu_spinwait(); 214 } else { 215 msleep(&unit->inv_seq_waiters, &unit->lock, 0, 216 "dmarse", hz); 217 } 218 } 219 unit->inv_seq_waiters--; 220 } 221 222 void 223 dmar_qi_invalidate_locked(struct dmar_domain *domain, dmar_gaddr_t base, 224 dmar_gaddr_t size, struct dmar_qi_genseq *pseq, bool emit_wait) 225 { 226 struct dmar_unit *unit; 227 dmar_gaddr_t isize; 228 int am; 229 230 unit = domain->dmar; 231 DMAR_ASSERT_LOCKED(unit); 232 for (; size > 0; base += isize, size -= isize) { 233 am = calc_am(unit, base, size, &isize); 234 dmar_qi_ensure(unit, 1); 235 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | 236 DMAR_IQ_DESCR_IOTLB_PAGE | DMAR_IQ_DESCR_IOTLB_DW | 237 DMAR_IQ_DESCR_IOTLB_DR | 238 DMAR_IQ_DESCR_IOTLB_DID(domain->domain), 239 base | am); 240 } 241 dmar_qi_emit_wait_seq(unit, pseq, emit_wait); 242 dmar_qi_advance_tail(unit); 243 } 244 245 void 246 dmar_qi_invalidate_ctx_glob_locked(struct dmar_unit *unit) 247 { 248 struct dmar_qi_genseq gseq; 249 250 DMAR_ASSERT_LOCKED(unit); 251 dmar_qi_ensure(unit, 2); 252 dmar_qi_emit(unit, DMAR_IQ_DESCR_CTX_INV | DMAR_IQ_DESCR_CTX_GLOB, 0); 253 dmar_qi_emit_wait_seq(unit, &gseq, true); 254 dmar_qi_advance_tail(unit); 255 dmar_qi_wait_for_seq(unit, &gseq, false); 256 } 257 258 void 259 dmar_qi_invalidate_iotlb_glob_locked(struct dmar_unit *unit) 260 { 261 struct dmar_qi_genseq gseq; 262 263 DMAR_ASSERT_LOCKED(unit); 264 dmar_qi_ensure(unit, 2); 265 dmar_qi_emit(unit, DMAR_IQ_DESCR_IOTLB_INV | DMAR_IQ_DESCR_IOTLB_GLOB | 266 DMAR_IQ_DESCR_IOTLB_DW | DMAR_IQ_DESCR_IOTLB_DR, 0); 267 dmar_qi_emit_wait_seq(unit, &gseq, true); 268 dmar_qi_advance_tail(unit); 269 dmar_qi_wait_for_seq(unit, &gseq, false); 270 } 271 272 void 273 dmar_qi_invalidate_iec_glob(struct dmar_unit *unit) 274 { 275 struct dmar_qi_genseq gseq; 276 277 DMAR_ASSERT_LOCKED(unit); 278 dmar_qi_ensure(unit, 2); 279 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV, 0); 280 dmar_qi_emit_wait_seq(unit, &gseq, true); 281 dmar_qi_advance_tail(unit); 282 dmar_qi_wait_for_seq(unit, &gseq, false); 283 } 284 285 void 286 dmar_qi_invalidate_iec(struct dmar_unit *unit, u_int start, u_int cnt) 287 { 288 struct dmar_qi_genseq gseq; 289 u_int c, l; 290 291 DMAR_ASSERT_LOCKED(unit); 292 KASSERT(start < unit->irte_cnt && start < start + cnt && 293 start + cnt <= unit->irte_cnt, 294 ("inv iec overflow %d %d %d", unit->irte_cnt, start, cnt)); 295 for (; cnt > 0; cnt -= c, start += c) { 296 l = ffs(start | cnt) - 1; 297 c = 1 << l; 298 dmar_qi_ensure(unit, 1); 299 dmar_qi_emit(unit, DMAR_IQ_DESCR_IEC_INV | 300 DMAR_IQ_DESCR_IEC_IDX | DMAR_IQ_DESCR_IEC_IIDX(start) | 301 DMAR_IQ_DESCR_IEC_IM(l), 0); 302 } 303 dmar_qi_ensure(unit, 1); 304 dmar_qi_emit_wait_seq(unit, &gseq, true); 305 dmar_qi_advance_tail(unit); 306 307 /* 308 * The caller of the function, in particular, 309 * dmar_ir_program_irte(), may be called from the context 310 * where the sleeping is forbidden (in fact, the 311 * intr_table_lock mutex may be held, locked from 312 * intr_shuffle_irqs()). Wait for the invalidation completion 313 * using the busy wait. 314 * 315 * The impact on the interrupt input setup code is small, the 316 * expected overhead is comparable with the chipset register 317 * read. It is more harmful for the parallel DMA operations, 318 * since we own the dmar unit lock until whole invalidation 319 * queue is processed, which includes requests possibly issued 320 * before our request. 321 */ 322 dmar_qi_wait_for_seq(unit, &gseq, true); 323 } 324 325 int 326 dmar_qi_intr(void *arg) 327 { 328 struct dmar_unit *unit; 329 330 unit = arg; 331 KASSERT(unit->qi_enabled, ("dmar%d: QI is not enabled", unit->unit)); 332 taskqueue_enqueue(unit->qi_taskqueue, &unit->qi_task); 333 return (FILTER_HANDLED); 334 } 335 336 static void 337 dmar_qi_task(void *arg, int pending __unused) 338 { 339 struct dmar_unit *unit; 340 struct dmar_map_entry *entry; 341 uint32_t ics; 342 343 unit = arg; 344 345 DMAR_LOCK(unit); 346 for (;;) { 347 entry = TAILQ_FIRST(&unit->tlb_flush_entries); 348 if (entry == NULL) 349 break; 350 if (!dmar_qi_seq_processed(unit, &entry->gseq)) 351 break; 352 TAILQ_REMOVE(&unit->tlb_flush_entries, entry, dmamap_link); 353 DMAR_UNLOCK(unit); 354 dmar_domain_free_entry(entry, (entry->flags & 355 DMAR_MAP_ENTRY_QI_NF) == 0); 356 DMAR_LOCK(unit); 357 } 358 ics = dmar_read4(unit, DMAR_ICS_REG); 359 if ((ics & DMAR_ICS_IWC) != 0) { 360 ics = DMAR_ICS_IWC; 361 dmar_write4(unit, DMAR_ICS_REG, ics); 362 } 363 if (unit->inv_seq_waiters > 0) 364 wakeup(&unit->inv_seq_waiters); 365 DMAR_UNLOCK(unit); 366 } 367 368 int 369 dmar_init_qi(struct dmar_unit *unit) 370 { 371 uint64_t iqa; 372 uint32_t ics; 373 int qi_sz; 374 375 if (!DMAR_HAS_QI(unit) || (unit->hw_cap & DMAR_CAP_CM) != 0) 376 return (0); 377 unit->qi_enabled = 1; 378 TUNABLE_INT_FETCH("hw.dmar.qi", &unit->qi_enabled); 379 if (!unit->qi_enabled) 380 return (0); 381 382 TAILQ_INIT(&unit->tlb_flush_entries); 383 TASK_INIT(&unit->qi_task, 0, dmar_qi_task, unit); 384 unit->qi_taskqueue = taskqueue_create_fast("dmarqf", M_WAITOK, 385 taskqueue_thread_enqueue, &unit->qi_taskqueue); 386 taskqueue_start_threads(&unit->qi_taskqueue, 1, PI_AV, 387 "dmar%d qi taskq", unit->unit); 388 389 unit->inv_waitd_gen = 0; 390 unit->inv_waitd_seq = 1; 391 392 qi_sz = DMAR_IQA_QS_DEF; 393 TUNABLE_INT_FETCH("hw.dmar.qi_size", &qi_sz); 394 if (qi_sz > DMAR_IQA_QS_MAX) 395 qi_sz = DMAR_IQA_QS_MAX; 396 unit->inv_queue_size = (1ULL << qi_sz) * PAGE_SIZE; 397 /* Reserve one descriptor to prevent wraparound. */ 398 unit->inv_queue_avail = unit->inv_queue_size - DMAR_IQ_DESCR_SZ; 399 400 /* The invalidation queue reads by DMARs are always coherent. */ 401 unit->inv_queue = kmem_alloc_contig(kernel_arena, unit->inv_queue_size, 402 M_WAITOK | M_ZERO, 0, dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT); 403 unit->inv_waitd_seq_hw_phys = pmap_kextract( 404 (vm_offset_t)&unit->inv_waitd_seq_hw); 405 406 DMAR_LOCK(unit); 407 dmar_write8(unit, DMAR_IQT_REG, 0); 408 iqa = pmap_kextract(unit->inv_queue); 409 iqa |= qi_sz; 410 dmar_write8(unit, DMAR_IQA_REG, iqa); 411 dmar_enable_qi(unit); 412 ics = dmar_read4(unit, DMAR_ICS_REG); 413 if ((ics & DMAR_ICS_IWC) != 0) { 414 ics = DMAR_ICS_IWC; 415 dmar_write4(unit, DMAR_ICS_REG, ics); 416 } 417 dmar_enable_qi_intr(unit); 418 DMAR_UNLOCK(unit); 419 420 return (0); 421 } 422 423 void 424 dmar_fini_qi(struct dmar_unit *unit) 425 { 426 struct dmar_qi_genseq gseq; 427 428 if (unit->qi_enabled) 429 return; 430 taskqueue_drain(unit->qi_taskqueue, &unit->qi_task); 431 taskqueue_free(unit->qi_taskqueue); 432 unit->qi_taskqueue = NULL; 433 434 DMAR_LOCK(unit); 435 /* quisce */ 436 dmar_qi_ensure(unit, 1); 437 dmar_qi_emit_wait_seq(unit, &gseq, true); 438 dmar_qi_advance_tail(unit); 439 dmar_qi_wait_for_seq(unit, &gseq, false); 440 /* only after the quisce, disable queue */ 441 dmar_disable_qi_intr(unit); 442 dmar_disable_qi(unit); 443 KASSERT(unit->inv_seq_waiters == 0, 444 ("dmar%d: waiters on disabled queue", unit->unit)); 445 DMAR_UNLOCK(unit); 446 447 kmem_free(kernel_arena, unit->inv_queue, unit->inv_queue_size); 448 unit->inv_queue = 0; 449 unit->inv_queue_size = 0; 450 unit->qi_enabled = 0; 451 } 452 453 void 454 dmar_enable_qi_intr(struct dmar_unit *unit) 455 { 456 uint32_t iectl; 457 458 DMAR_ASSERT_LOCKED(unit); 459 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); 460 iectl = dmar_read4(unit, DMAR_IECTL_REG); 461 iectl &= ~DMAR_IECTL_IM; 462 dmar_write4(unit, DMAR_IECTL_REG, iectl); 463 } 464 465 void 466 dmar_disable_qi_intr(struct dmar_unit *unit) 467 { 468 uint32_t iectl; 469 470 DMAR_ASSERT_LOCKED(unit); 471 KASSERT(DMAR_HAS_QI(unit), ("dmar%d: QI is not supported", unit->unit)); 472 iectl = dmar_read4(unit, DMAR_IECTL_REG); 473 dmar_write4(unit, DMAR_IECTL_REG, iectl | DMAR_IECTL_IM); 474 } 475