1 /* 2 * QEMU emulation of an Intel IOMMU (VT-d) 3 * (DMA Remapping device) 4 * 5 * Copyright (C) 2013 Knut Omang, Oracle <knut.omang@oracle.com> 6 * Copyright (C) 2014 Le Tan, <tamlokveer@gmail.com> 7 * 8 * This program is free software; you can redistribute it and/or modify 9 * it under the terms of the GNU General Public License as published by 10 * the Free Software Foundation; either version 2 of the License, or 11 * (at your option) any later version. 12 13 * This program is distributed in the hope that it will be useful, 14 * but WITHOUT ANY WARRANTY; without even the implied warranty of 15 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 16 * GNU General Public License for more details. 17 18 * You should have received a copy of the GNU General Public License along 19 * with this program; if not, see <http://www.gnu.org/licenses/>. 20 */ 21 22 #include "qemu/osdep.h" 23 #include "hw/sysbus.h" 24 #include "exec/address-spaces.h" 25 #include "intel_iommu_internal.h" 26 #include "hw/pci/pci.h" 27 #include "hw/pci/pci_bus.h" 28 29 /*#define DEBUG_INTEL_IOMMU*/ 30 #ifdef DEBUG_INTEL_IOMMU 31 enum { 32 DEBUG_GENERAL, DEBUG_CSR, DEBUG_INV, DEBUG_MMU, DEBUG_FLOG, 33 DEBUG_CACHE, 34 }; 35 #define VTD_DBGBIT(x) (1 << DEBUG_##x) 36 static int vtd_dbgflags = VTD_DBGBIT(GENERAL) | VTD_DBGBIT(CSR); 37 38 #define VTD_DPRINTF(what, fmt, ...) do { \ 39 if (vtd_dbgflags & VTD_DBGBIT(what)) { \ 40 fprintf(stderr, "(vtd)%s: " fmt "\n", __func__, \ 41 ## __VA_ARGS__); } \ 42 } while (0) 43 #else 44 #define VTD_DPRINTF(what, fmt, ...) do {} while (0) 45 #endif 46 47 static void vtd_define_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val, 48 uint64_t wmask, uint64_t w1cmask) 49 { 50 stq_le_p(&s->csr[addr], val); 51 stq_le_p(&s->wmask[addr], wmask); 52 stq_le_p(&s->w1cmask[addr], w1cmask); 53 } 54 55 static void vtd_define_quad_wo(IntelIOMMUState *s, hwaddr addr, uint64_t mask) 56 { 57 stq_le_p(&s->womask[addr], mask); 58 } 59 60 static void vtd_define_long(IntelIOMMUState *s, hwaddr addr, uint32_t val, 61 uint32_t wmask, uint32_t w1cmask) 62 { 63 stl_le_p(&s->csr[addr], val); 64 stl_le_p(&s->wmask[addr], wmask); 65 stl_le_p(&s->w1cmask[addr], w1cmask); 66 } 67 68 static void vtd_define_long_wo(IntelIOMMUState *s, hwaddr addr, uint32_t mask) 69 { 70 stl_le_p(&s->womask[addr], mask); 71 } 72 73 /* "External" get/set operations */ 74 static void vtd_set_quad(IntelIOMMUState *s, hwaddr addr, uint64_t val) 75 { 76 uint64_t oldval = ldq_le_p(&s->csr[addr]); 77 uint64_t wmask = ldq_le_p(&s->wmask[addr]); 78 uint64_t w1cmask = ldq_le_p(&s->w1cmask[addr]); 79 stq_le_p(&s->csr[addr], 80 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 81 } 82 83 static void vtd_set_long(IntelIOMMUState *s, hwaddr addr, uint32_t val) 84 { 85 uint32_t oldval = ldl_le_p(&s->csr[addr]); 86 uint32_t wmask = ldl_le_p(&s->wmask[addr]); 87 uint32_t w1cmask = ldl_le_p(&s->w1cmask[addr]); 88 stl_le_p(&s->csr[addr], 89 ((oldval & ~wmask) | (val & wmask)) & ~(w1cmask & val)); 90 } 91 92 static uint64_t vtd_get_quad(IntelIOMMUState *s, hwaddr addr) 93 { 94 uint64_t val = ldq_le_p(&s->csr[addr]); 95 uint64_t womask = ldq_le_p(&s->womask[addr]); 96 return val & ~womask; 97 } 98 99 static uint32_t vtd_get_long(IntelIOMMUState *s, hwaddr addr) 100 { 101 uint32_t val = ldl_le_p(&s->csr[addr]); 102 uint32_t womask = ldl_le_p(&s->womask[addr]); 103 return val & ~womask; 104 } 105 106 /* "Internal" get/set operations */ 107 static uint64_t vtd_get_quad_raw(IntelIOMMUState *s, hwaddr addr) 108 { 109 return ldq_le_p(&s->csr[addr]); 110 } 111 112 static uint32_t vtd_get_long_raw(IntelIOMMUState *s, hwaddr addr) 113 { 114 return ldl_le_p(&s->csr[addr]); 115 } 116 117 static void vtd_set_quad_raw(IntelIOMMUState *s, hwaddr addr, uint64_t val) 118 { 119 stq_le_p(&s->csr[addr], val); 120 } 121 122 static uint32_t vtd_set_clear_mask_long(IntelIOMMUState *s, hwaddr addr, 123 uint32_t clear, uint32_t mask) 124 { 125 uint32_t new_val = (ldl_le_p(&s->csr[addr]) & ~clear) | mask; 126 stl_le_p(&s->csr[addr], new_val); 127 return new_val; 128 } 129 130 static uint64_t vtd_set_clear_mask_quad(IntelIOMMUState *s, hwaddr addr, 131 uint64_t clear, uint64_t mask) 132 { 133 uint64_t new_val = (ldq_le_p(&s->csr[addr]) & ~clear) | mask; 134 stq_le_p(&s->csr[addr], new_val); 135 return new_val; 136 } 137 138 /* GHashTable functions */ 139 static gboolean vtd_uint64_equal(gconstpointer v1, gconstpointer v2) 140 { 141 return *((const uint64_t *)v1) == *((const uint64_t *)v2); 142 } 143 144 static guint vtd_uint64_hash(gconstpointer v) 145 { 146 return (guint)*(const uint64_t *)v; 147 } 148 149 static gboolean vtd_hash_remove_by_domain(gpointer key, gpointer value, 150 gpointer user_data) 151 { 152 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 153 uint16_t domain_id = *(uint16_t *)user_data; 154 return entry->domain_id == domain_id; 155 } 156 157 /* The shift of an addr for a certain level of paging structure */ 158 static inline uint32_t vtd_slpt_level_shift(uint32_t level) 159 { 160 return VTD_PAGE_SHIFT_4K + (level - 1) * VTD_SL_LEVEL_BITS; 161 } 162 163 static inline uint64_t vtd_slpt_level_page_mask(uint32_t level) 164 { 165 return ~((1ULL << vtd_slpt_level_shift(level)) - 1); 166 } 167 168 static gboolean vtd_hash_remove_by_page(gpointer key, gpointer value, 169 gpointer user_data) 170 { 171 VTDIOTLBEntry *entry = (VTDIOTLBEntry *)value; 172 VTDIOTLBPageInvInfo *info = (VTDIOTLBPageInvInfo *)user_data; 173 uint64_t gfn = (info->addr >> VTD_PAGE_SHIFT_4K) & info->mask; 174 uint64_t gfn_tlb = (info->addr & entry->mask) >> VTD_PAGE_SHIFT_4K; 175 return (entry->domain_id == info->domain_id) && 176 (((entry->gfn & info->mask) == gfn) || 177 (entry->gfn == gfn_tlb)); 178 } 179 180 /* Reset all the gen of VTDAddressSpace to zero and set the gen of 181 * IntelIOMMUState to 1. 182 */ 183 static void vtd_reset_context_cache(IntelIOMMUState *s) 184 { 185 VTDAddressSpace *vtd_as; 186 VTDBus *vtd_bus; 187 GHashTableIter bus_it; 188 uint32_t devfn_it; 189 190 g_hash_table_iter_init(&bus_it, s->vtd_as_by_busptr); 191 192 VTD_DPRINTF(CACHE, "global context_cache_gen=1"); 193 while (g_hash_table_iter_next (&bus_it, NULL, (void**)&vtd_bus)) { 194 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 195 vtd_as = vtd_bus->dev_as[devfn_it]; 196 if (!vtd_as) { 197 continue; 198 } 199 vtd_as->context_cache_entry.context_cache_gen = 0; 200 } 201 } 202 s->context_cache_gen = 1; 203 } 204 205 static void vtd_reset_iotlb(IntelIOMMUState *s) 206 { 207 assert(s->iotlb); 208 g_hash_table_remove_all(s->iotlb); 209 } 210 211 static uint64_t vtd_get_iotlb_key(uint64_t gfn, uint8_t source_id, 212 uint32_t level) 213 { 214 return gfn | ((uint64_t)(source_id) << VTD_IOTLB_SID_SHIFT) | 215 ((uint64_t)(level) << VTD_IOTLB_LVL_SHIFT); 216 } 217 218 static uint64_t vtd_get_iotlb_gfn(hwaddr addr, uint32_t level) 219 { 220 return (addr & vtd_slpt_level_page_mask(level)) >> VTD_PAGE_SHIFT_4K; 221 } 222 223 static VTDIOTLBEntry *vtd_lookup_iotlb(IntelIOMMUState *s, uint16_t source_id, 224 hwaddr addr) 225 { 226 VTDIOTLBEntry *entry; 227 uint64_t key; 228 int level; 229 230 for (level = VTD_SL_PT_LEVEL; level < VTD_SL_PML4_LEVEL; level++) { 231 key = vtd_get_iotlb_key(vtd_get_iotlb_gfn(addr, level), 232 source_id, level); 233 entry = g_hash_table_lookup(s->iotlb, &key); 234 if (entry) { 235 goto out; 236 } 237 } 238 239 out: 240 return entry; 241 } 242 243 static void vtd_update_iotlb(IntelIOMMUState *s, uint16_t source_id, 244 uint16_t domain_id, hwaddr addr, uint64_t slpte, 245 bool read_flags, bool write_flags, 246 uint32_t level) 247 { 248 VTDIOTLBEntry *entry = g_malloc(sizeof(*entry)); 249 uint64_t *key = g_malloc(sizeof(*key)); 250 uint64_t gfn = vtd_get_iotlb_gfn(addr, level); 251 252 VTD_DPRINTF(CACHE, "update iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 253 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, slpte, 254 domain_id); 255 if (g_hash_table_size(s->iotlb) >= VTD_IOTLB_MAX_SIZE) { 256 VTD_DPRINTF(CACHE, "iotlb exceeds size limit, forced to reset"); 257 vtd_reset_iotlb(s); 258 } 259 260 entry->gfn = gfn; 261 entry->domain_id = domain_id; 262 entry->slpte = slpte; 263 entry->read_flags = read_flags; 264 entry->write_flags = write_flags; 265 entry->mask = vtd_slpt_level_page_mask(level); 266 *key = vtd_get_iotlb_key(gfn, source_id, level); 267 g_hash_table_replace(s->iotlb, key, entry); 268 } 269 270 /* Given the reg addr of both the message data and address, generate an 271 * interrupt via MSI. 272 */ 273 static void vtd_generate_interrupt(IntelIOMMUState *s, hwaddr mesg_addr_reg, 274 hwaddr mesg_data_reg) 275 { 276 hwaddr addr; 277 uint32_t data; 278 279 assert(mesg_data_reg < DMAR_REG_SIZE); 280 assert(mesg_addr_reg < DMAR_REG_SIZE); 281 282 addr = vtd_get_long_raw(s, mesg_addr_reg); 283 data = vtd_get_long_raw(s, mesg_data_reg); 284 285 VTD_DPRINTF(FLOG, "msi: addr 0x%"PRIx64 " data 0x%"PRIx32, addr, data); 286 address_space_stl_le(&address_space_memory, addr, data, 287 MEMTXATTRS_UNSPECIFIED, NULL); 288 } 289 290 /* Generate a fault event to software via MSI if conditions are met. 291 * Notice that the value of FSTS_REG being passed to it should be the one 292 * before any update. 293 */ 294 static void vtd_generate_fault_event(IntelIOMMUState *s, uint32_t pre_fsts) 295 { 296 if (pre_fsts & VTD_FSTS_PPF || pre_fsts & VTD_FSTS_PFO || 297 pre_fsts & VTD_FSTS_IQE) { 298 VTD_DPRINTF(FLOG, "there are previous interrupt conditions " 299 "to be serviced by software, fault event is not generated " 300 "(FSTS_REG 0x%"PRIx32 ")", pre_fsts); 301 return; 302 } 303 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, 0, VTD_FECTL_IP); 304 if (vtd_get_long_raw(s, DMAR_FECTL_REG) & VTD_FECTL_IM) { 305 VTD_DPRINTF(FLOG, "Interrupt Mask set, fault event is not generated"); 306 } else { 307 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 308 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 309 } 310 } 311 312 /* Check if the Fault (F) field of the Fault Recording Register referenced by 313 * @index is Set. 314 */ 315 static bool vtd_is_frcd_set(IntelIOMMUState *s, uint16_t index) 316 { 317 /* Each reg is 128-bit */ 318 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 319 addr += 8; /* Access the high 64-bit half */ 320 321 assert(index < DMAR_FRCD_REG_NR); 322 323 return vtd_get_quad_raw(s, addr) & VTD_FRCD_F; 324 } 325 326 /* Update the PPF field of Fault Status Register. 327 * Should be called whenever change the F field of any fault recording 328 * registers. 329 */ 330 static void vtd_update_fsts_ppf(IntelIOMMUState *s) 331 { 332 uint32_t i; 333 uint32_t ppf_mask = 0; 334 335 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 336 if (vtd_is_frcd_set(s, i)) { 337 ppf_mask = VTD_FSTS_PPF; 338 break; 339 } 340 } 341 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_PPF, ppf_mask); 342 VTD_DPRINTF(FLOG, "set PPF of FSTS_REG to %d", ppf_mask ? 1 : 0); 343 } 344 345 static void vtd_set_frcd_and_update_ppf(IntelIOMMUState *s, uint16_t index) 346 { 347 /* Each reg is 128-bit */ 348 hwaddr addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 349 addr += 8; /* Access the high 64-bit half */ 350 351 assert(index < DMAR_FRCD_REG_NR); 352 353 vtd_set_clear_mask_quad(s, addr, 0, VTD_FRCD_F); 354 vtd_update_fsts_ppf(s); 355 } 356 357 /* Must not update F field now, should be done later */ 358 static void vtd_record_frcd(IntelIOMMUState *s, uint16_t index, 359 uint16_t source_id, hwaddr addr, 360 VTDFaultReason fault, bool is_write) 361 { 362 uint64_t hi = 0, lo; 363 hwaddr frcd_reg_addr = DMAR_FRCD_REG_OFFSET + (((uint64_t)index) << 4); 364 365 assert(index < DMAR_FRCD_REG_NR); 366 367 lo = VTD_FRCD_FI(addr); 368 hi = VTD_FRCD_SID(source_id) | VTD_FRCD_FR(fault); 369 if (!is_write) { 370 hi |= VTD_FRCD_T; 371 } 372 vtd_set_quad_raw(s, frcd_reg_addr, lo); 373 vtd_set_quad_raw(s, frcd_reg_addr + 8, hi); 374 VTD_DPRINTF(FLOG, "record to FRCD_REG #%"PRIu16 ": hi 0x%"PRIx64 375 ", lo 0x%"PRIx64, index, hi, lo); 376 } 377 378 /* Try to collapse multiple pending faults from the same requester */ 379 static bool vtd_try_collapse_fault(IntelIOMMUState *s, uint16_t source_id) 380 { 381 uint32_t i; 382 uint64_t frcd_reg; 383 hwaddr addr = DMAR_FRCD_REG_OFFSET + 8; /* The high 64-bit half */ 384 385 for (i = 0; i < DMAR_FRCD_REG_NR; i++) { 386 frcd_reg = vtd_get_quad_raw(s, addr); 387 VTD_DPRINTF(FLOG, "frcd_reg #%d 0x%"PRIx64, i, frcd_reg); 388 if ((frcd_reg & VTD_FRCD_F) && 389 ((frcd_reg & VTD_FRCD_SID_MASK) == source_id)) { 390 return true; 391 } 392 addr += 16; /* 128-bit for each */ 393 } 394 return false; 395 } 396 397 /* Log and report an DMAR (address translation) fault to software */ 398 static void vtd_report_dmar_fault(IntelIOMMUState *s, uint16_t source_id, 399 hwaddr addr, VTDFaultReason fault, 400 bool is_write) 401 { 402 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 403 404 assert(fault < VTD_FR_MAX); 405 406 if (fault == VTD_FR_RESERVED_ERR) { 407 /* This is not a normal fault reason case. Drop it. */ 408 return; 409 } 410 VTD_DPRINTF(FLOG, "sid 0x%"PRIx16 ", fault %d, addr 0x%"PRIx64 411 ", is_write %d", source_id, fault, addr, is_write); 412 if (fsts_reg & VTD_FSTS_PFO) { 413 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 414 "Primary Fault Overflow"); 415 return; 416 } 417 if (vtd_try_collapse_fault(s, source_id)) { 418 VTD_DPRINTF(FLOG, "new fault is not recorded due to " 419 "compression of faults"); 420 return; 421 } 422 if (vtd_is_frcd_set(s, s->next_frcd_reg)) { 423 VTD_DPRINTF(FLOG, "Primary Fault Overflow and " 424 "new fault is not recorded, set PFO field"); 425 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_PFO); 426 return; 427 } 428 429 vtd_record_frcd(s, s->next_frcd_reg, source_id, addr, fault, is_write); 430 431 if (fsts_reg & VTD_FSTS_PPF) { 432 VTD_DPRINTF(FLOG, "there are pending faults already, " 433 "fault event is not generated"); 434 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); 435 s->next_frcd_reg++; 436 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 437 s->next_frcd_reg = 0; 438 } 439 } else { 440 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, VTD_FSTS_FRI_MASK, 441 VTD_FSTS_FRI(s->next_frcd_reg)); 442 vtd_set_frcd_and_update_ppf(s, s->next_frcd_reg); /* Will set PPF */ 443 s->next_frcd_reg++; 444 if (s->next_frcd_reg == DMAR_FRCD_REG_NR) { 445 s->next_frcd_reg = 0; 446 } 447 /* This case actually cause the PPF to be Set. 448 * So generate fault event (interrupt). 449 */ 450 vtd_generate_fault_event(s, fsts_reg); 451 } 452 } 453 454 /* Handle Invalidation Queue Errors of queued invalidation interface error 455 * conditions. 456 */ 457 static void vtd_handle_inv_queue_error(IntelIOMMUState *s) 458 { 459 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 460 461 vtd_set_clear_mask_long(s, DMAR_FSTS_REG, 0, VTD_FSTS_IQE); 462 vtd_generate_fault_event(s, fsts_reg); 463 } 464 465 /* Set the IWC field and try to generate an invalidation completion interrupt */ 466 static void vtd_generate_completion_event(IntelIOMMUState *s) 467 { 468 VTD_DPRINTF(INV, "completes an invalidation wait command with " 469 "Interrupt Flag"); 470 if (vtd_get_long_raw(s, DMAR_ICS_REG) & VTD_ICS_IWC) { 471 VTD_DPRINTF(INV, "there is a previous interrupt condition to be " 472 "serviced by software, " 473 "new invalidation event is not generated"); 474 return; 475 } 476 vtd_set_clear_mask_long(s, DMAR_ICS_REG, 0, VTD_ICS_IWC); 477 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, 0, VTD_IECTL_IP); 478 if (vtd_get_long_raw(s, DMAR_IECTL_REG) & VTD_IECTL_IM) { 479 VTD_DPRINTF(INV, "IM filed in IECTL_REG is set, new invalidation " 480 "event is not generated"); 481 return; 482 } else { 483 /* Generate the interrupt event */ 484 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 485 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 486 } 487 } 488 489 static inline bool vtd_root_entry_present(VTDRootEntry *root) 490 { 491 return root->val & VTD_ROOT_ENTRY_P; 492 } 493 494 static int vtd_get_root_entry(IntelIOMMUState *s, uint8_t index, 495 VTDRootEntry *re) 496 { 497 dma_addr_t addr; 498 499 addr = s->root + index * sizeof(*re); 500 if (dma_memory_read(&address_space_memory, addr, re, sizeof(*re))) { 501 VTD_DPRINTF(GENERAL, "error: fail to access root-entry at 0x%"PRIx64 502 " + %"PRIu8, s->root, index); 503 re->val = 0; 504 return -VTD_FR_ROOT_TABLE_INV; 505 } 506 re->val = le64_to_cpu(re->val); 507 return 0; 508 } 509 510 static inline bool vtd_context_entry_present(VTDContextEntry *context) 511 { 512 return context->lo & VTD_CONTEXT_ENTRY_P; 513 } 514 515 static int vtd_get_context_entry_from_root(VTDRootEntry *root, uint8_t index, 516 VTDContextEntry *ce) 517 { 518 dma_addr_t addr; 519 520 if (!vtd_root_entry_present(root)) { 521 VTD_DPRINTF(GENERAL, "error: root-entry is not present"); 522 return -VTD_FR_ROOT_ENTRY_P; 523 } 524 addr = (root->val & VTD_ROOT_ENTRY_CTP) + index * sizeof(*ce); 525 if (dma_memory_read(&address_space_memory, addr, ce, sizeof(*ce))) { 526 VTD_DPRINTF(GENERAL, "error: fail to access context-entry at 0x%"PRIx64 527 " + %"PRIu8, 528 (uint64_t)(root->val & VTD_ROOT_ENTRY_CTP), index); 529 return -VTD_FR_CONTEXT_TABLE_INV; 530 } 531 ce->lo = le64_to_cpu(ce->lo); 532 ce->hi = le64_to_cpu(ce->hi); 533 return 0; 534 } 535 536 static inline dma_addr_t vtd_get_slpt_base_from_context(VTDContextEntry *ce) 537 { 538 return ce->lo & VTD_CONTEXT_ENTRY_SLPTPTR; 539 } 540 541 static inline uint64_t vtd_get_slpte_addr(uint64_t slpte) 542 { 543 return slpte & VTD_SL_PT_BASE_ADDR_MASK; 544 } 545 546 /* Whether the pte indicates the address of the page frame */ 547 static inline bool vtd_is_last_slpte(uint64_t slpte, uint32_t level) 548 { 549 return level == VTD_SL_PT_LEVEL || (slpte & VTD_SL_PT_PAGE_SIZE_MASK); 550 } 551 552 /* Get the content of a spte located in @base_addr[@index] */ 553 static uint64_t vtd_get_slpte(dma_addr_t base_addr, uint32_t index) 554 { 555 uint64_t slpte; 556 557 assert(index < VTD_SL_PT_ENTRY_NR); 558 559 if (dma_memory_read(&address_space_memory, 560 base_addr + index * sizeof(slpte), &slpte, 561 sizeof(slpte))) { 562 slpte = (uint64_t)-1; 563 return slpte; 564 } 565 slpte = le64_to_cpu(slpte); 566 return slpte; 567 } 568 569 /* Given a gpa and the level of paging structure, return the offset of current 570 * level. 571 */ 572 static inline uint32_t vtd_gpa_level_offset(uint64_t gpa, uint32_t level) 573 { 574 return (gpa >> vtd_slpt_level_shift(level)) & 575 ((1ULL << VTD_SL_LEVEL_BITS) - 1); 576 } 577 578 /* Check Capability Register to see if the @level of page-table is supported */ 579 static inline bool vtd_is_level_supported(IntelIOMMUState *s, uint32_t level) 580 { 581 return VTD_CAP_SAGAW_MASK & s->cap & 582 (1ULL << (level - 2 + VTD_CAP_SAGAW_SHIFT)); 583 } 584 585 /* Get the page-table level that hardware should use for the second-level 586 * page-table walk from the Address Width field of context-entry. 587 */ 588 static inline uint32_t vtd_get_level_from_context_entry(VTDContextEntry *ce) 589 { 590 return 2 + (ce->hi & VTD_CONTEXT_ENTRY_AW); 591 } 592 593 static inline uint32_t vtd_get_agaw_from_context_entry(VTDContextEntry *ce) 594 { 595 return 30 + (ce->hi & VTD_CONTEXT_ENTRY_AW) * 9; 596 } 597 598 static const uint64_t vtd_paging_entry_rsvd_field[] = { 599 [0] = ~0ULL, 600 /* For not large page */ 601 [1] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 602 [2] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 603 [3] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 604 [4] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 605 /* For large page */ 606 [5] = 0x800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 607 [6] = 0x1ff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 608 [7] = 0x3ffff800ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 609 [8] = 0x880ULL | ~(VTD_HAW_MASK | VTD_SL_IGN_COM), 610 }; 611 612 static bool vtd_slpte_nonzero_rsvd(uint64_t slpte, uint32_t level) 613 { 614 if (slpte & VTD_SL_PT_PAGE_SIZE_MASK) { 615 /* Maybe large page */ 616 return slpte & vtd_paging_entry_rsvd_field[level + 4]; 617 } else { 618 return slpte & vtd_paging_entry_rsvd_field[level]; 619 } 620 } 621 622 /* Given the @gpa, get relevant @slptep. @slpte_level will be the last level 623 * of the translation, can be used for deciding the size of large page. 624 */ 625 static int vtd_gpa_to_slpte(VTDContextEntry *ce, uint64_t gpa, bool is_write, 626 uint64_t *slptep, uint32_t *slpte_level, 627 bool *reads, bool *writes) 628 { 629 dma_addr_t addr = vtd_get_slpt_base_from_context(ce); 630 uint32_t level = vtd_get_level_from_context_entry(ce); 631 uint32_t offset; 632 uint64_t slpte; 633 uint32_t ce_agaw = vtd_get_agaw_from_context_entry(ce); 634 uint64_t access_right_check; 635 636 /* Check if @gpa is above 2^X-1, where X is the minimum of MGAW in CAP_REG 637 * and AW in context-entry. 638 */ 639 if (gpa & ~((1ULL << MIN(ce_agaw, VTD_MGAW)) - 1)) { 640 VTD_DPRINTF(GENERAL, "error: gpa 0x%"PRIx64 " exceeds limits", gpa); 641 return -VTD_FR_ADDR_BEYOND_MGAW; 642 } 643 644 /* FIXME: what is the Atomics request here? */ 645 access_right_check = is_write ? VTD_SL_W : VTD_SL_R; 646 647 while (true) { 648 offset = vtd_gpa_level_offset(gpa, level); 649 slpte = vtd_get_slpte(addr, offset); 650 651 if (slpte == (uint64_t)-1) { 652 VTD_DPRINTF(GENERAL, "error: fail to access second-level paging " 653 "entry at level %"PRIu32 " for gpa 0x%"PRIx64, 654 level, gpa); 655 if (level == vtd_get_level_from_context_entry(ce)) { 656 /* Invalid programming of context-entry */ 657 return -VTD_FR_CONTEXT_ENTRY_INV; 658 } else { 659 return -VTD_FR_PAGING_ENTRY_INV; 660 } 661 } 662 *reads = (*reads) && (slpte & VTD_SL_R); 663 *writes = (*writes) && (slpte & VTD_SL_W); 664 if (!(slpte & access_right_check)) { 665 VTD_DPRINTF(GENERAL, "error: lack of %s permission for " 666 "gpa 0x%"PRIx64 " slpte 0x%"PRIx64, 667 (is_write ? "write" : "read"), gpa, slpte); 668 return is_write ? -VTD_FR_WRITE : -VTD_FR_READ; 669 } 670 if (vtd_slpte_nonzero_rsvd(slpte, level)) { 671 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in second " 672 "level paging entry level %"PRIu32 " slpte 0x%"PRIx64, 673 level, slpte); 674 return -VTD_FR_PAGING_ENTRY_RSVD; 675 } 676 677 if (vtd_is_last_slpte(slpte, level)) { 678 *slptep = slpte; 679 *slpte_level = level; 680 return 0; 681 } 682 addr = vtd_get_slpte_addr(slpte); 683 level--; 684 } 685 } 686 687 /* Map a device to its corresponding domain (context-entry) */ 688 static int vtd_dev_to_context_entry(IntelIOMMUState *s, uint8_t bus_num, 689 uint8_t devfn, VTDContextEntry *ce) 690 { 691 VTDRootEntry re; 692 int ret_fr; 693 694 ret_fr = vtd_get_root_entry(s, bus_num, &re); 695 if (ret_fr) { 696 return ret_fr; 697 } 698 699 if (!vtd_root_entry_present(&re)) { 700 VTD_DPRINTF(GENERAL, "error: root-entry #%"PRIu8 " is not present", 701 bus_num); 702 return -VTD_FR_ROOT_ENTRY_P; 703 } else if (re.rsvd || (re.val & VTD_ROOT_ENTRY_RSVD)) { 704 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in root-entry " 705 "hi 0x%"PRIx64 " lo 0x%"PRIx64, re.rsvd, re.val); 706 return -VTD_FR_ROOT_ENTRY_RSVD; 707 } 708 709 ret_fr = vtd_get_context_entry_from_root(&re, devfn, ce); 710 if (ret_fr) { 711 return ret_fr; 712 } 713 714 if (!vtd_context_entry_present(ce)) { 715 VTD_DPRINTF(GENERAL, 716 "error: context-entry #%"PRIu8 "(bus #%"PRIu8 ") " 717 "is not present", devfn, bus_num); 718 return -VTD_FR_CONTEXT_ENTRY_P; 719 } else if ((ce->hi & VTD_CONTEXT_ENTRY_RSVD_HI) || 720 (ce->lo & VTD_CONTEXT_ENTRY_RSVD_LO)) { 721 VTD_DPRINTF(GENERAL, 722 "error: non-zero reserved field in context-entry " 723 "hi 0x%"PRIx64 " lo 0x%"PRIx64, ce->hi, ce->lo); 724 return -VTD_FR_CONTEXT_ENTRY_RSVD; 725 } 726 /* Check if the programming of context-entry is valid */ 727 if (!vtd_is_level_supported(s, vtd_get_level_from_context_entry(ce))) { 728 VTD_DPRINTF(GENERAL, "error: unsupported Address Width value in " 729 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 730 ce->hi, ce->lo); 731 return -VTD_FR_CONTEXT_ENTRY_INV; 732 } else if (ce->lo & VTD_CONTEXT_ENTRY_TT) { 733 VTD_DPRINTF(GENERAL, "error: unsupported Translation Type in " 734 "context-entry hi 0x%"PRIx64 " lo 0x%"PRIx64, 735 ce->hi, ce->lo); 736 return -VTD_FR_CONTEXT_ENTRY_INV; 737 } 738 return 0; 739 } 740 741 static inline uint16_t vtd_make_source_id(uint8_t bus_num, uint8_t devfn) 742 { 743 return ((bus_num & 0xffUL) << 8) | (devfn & 0xffUL); 744 } 745 746 static const bool vtd_qualified_faults[] = { 747 [VTD_FR_RESERVED] = false, 748 [VTD_FR_ROOT_ENTRY_P] = false, 749 [VTD_FR_CONTEXT_ENTRY_P] = true, 750 [VTD_FR_CONTEXT_ENTRY_INV] = true, 751 [VTD_FR_ADDR_BEYOND_MGAW] = true, 752 [VTD_FR_WRITE] = true, 753 [VTD_FR_READ] = true, 754 [VTD_FR_PAGING_ENTRY_INV] = true, 755 [VTD_FR_ROOT_TABLE_INV] = false, 756 [VTD_FR_CONTEXT_TABLE_INV] = false, 757 [VTD_FR_ROOT_ENTRY_RSVD] = false, 758 [VTD_FR_PAGING_ENTRY_RSVD] = true, 759 [VTD_FR_CONTEXT_ENTRY_TT] = true, 760 [VTD_FR_RESERVED_ERR] = false, 761 [VTD_FR_MAX] = false, 762 }; 763 764 /* To see if a fault condition is "qualified", which is reported to software 765 * only if the FPD field in the context-entry used to process the faulting 766 * request is 0. 767 */ 768 static inline bool vtd_is_qualified_fault(VTDFaultReason fault) 769 { 770 return vtd_qualified_faults[fault]; 771 } 772 773 static inline bool vtd_is_interrupt_addr(hwaddr addr) 774 { 775 return VTD_INTERRUPT_ADDR_FIRST <= addr && addr <= VTD_INTERRUPT_ADDR_LAST; 776 } 777 778 /* Map dev to context-entry then do a paging-structures walk to do a iommu 779 * translation. 780 * 781 * Called from RCU critical section. 782 * 783 * @bus_num: The bus number 784 * @devfn: The devfn, which is the combined of device and function number 785 * @is_write: The access is a write operation 786 * @entry: IOMMUTLBEntry that contain the addr to be translated and result 787 */ 788 static void vtd_do_iommu_translate(VTDAddressSpace *vtd_as, PCIBus *bus, 789 uint8_t devfn, hwaddr addr, bool is_write, 790 IOMMUTLBEntry *entry) 791 { 792 IntelIOMMUState *s = vtd_as->iommu_state; 793 VTDContextEntry ce; 794 uint8_t bus_num = pci_bus_num(bus); 795 VTDContextCacheEntry *cc_entry = &vtd_as->context_cache_entry; 796 uint64_t slpte, page_mask; 797 uint32_t level; 798 uint16_t source_id = vtd_make_source_id(bus_num, devfn); 799 int ret_fr; 800 bool is_fpd_set = false; 801 bool reads = true; 802 bool writes = true; 803 VTDIOTLBEntry *iotlb_entry; 804 805 /* Check if the request is in interrupt address range */ 806 if (vtd_is_interrupt_addr(addr)) { 807 if (is_write) { 808 /* FIXME: since we don't know the length of the access here, we 809 * treat Non-DWORD length write requests without PASID as 810 * interrupt requests, too. Withoud interrupt remapping support, 811 * we just use 1:1 mapping. 812 */ 813 VTD_DPRINTF(MMU, "write request to interrupt address " 814 "gpa 0x%"PRIx64, addr); 815 entry->iova = addr & VTD_PAGE_MASK_4K; 816 entry->translated_addr = addr & VTD_PAGE_MASK_4K; 817 entry->addr_mask = ~VTD_PAGE_MASK_4K; 818 entry->perm = IOMMU_WO; 819 return; 820 } else { 821 VTD_DPRINTF(GENERAL, "error: read request from interrupt address " 822 "gpa 0x%"PRIx64, addr); 823 vtd_report_dmar_fault(s, source_id, addr, VTD_FR_READ, is_write); 824 return; 825 } 826 } 827 /* Try to fetch slpte form IOTLB */ 828 iotlb_entry = vtd_lookup_iotlb(s, source_id, addr); 829 if (iotlb_entry) { 830 VTD_DPRINTF(CACHE, "hit iotlb sid 0x%"PRIx16 " gpa 0x%"PRIx64 831 " slpte 0x%"PRIx64 " did 0x%"PRIx16, source_id, addr, 832 iotlb_entry->slpte, iotlb_entry->domain_id); 833 slpte = iotlb_entry->slpte; 834 reads = iotlb_entry->read_flags; 835 writes = iotlb_entry->write_flags; 836 page_mask = iotlb_entry->mask; 837 goto out; 838 } 839 /* Try to fetch context-entry from cache first */ 840 if (cc_entry->context_cache_gen == s->context_cache_gen) { 841 VTD_DPRINTF(CACHE, "hit context-cache bus %d devfn %d " 842 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 ")", 843 bus_num, devfn, cc_entry->context_entry.hi, 844 cc_entry->context_entry.lo, cc_entry->context_cache_gen); 845 ce = cc_entry->context_entry; 846 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 847 } else { 848 ret_fr = vtd_dev_to_context_entry(s, bus_num, devfn, &ce); 849 is_fpd_set = ce.lo & VTD_CONTEXT_ENTRY_FPD; 850 if (ret_fr) { 851 ret_fr = -ret_fr; 852 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 853 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA " 854 "requests through this context-entry " 855 "(with FPD Set)"); 856 } else { 857 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 858 } 859 return; 860 } 861 /* Update context-cache */ 862 VTD_DPRINTF(CACHE, "update context-cache bus %d devfn %d " 863 "(hi %"PRIx64 " lo %"PRIx64 " gen %"PRIu32 "->%"PRIu32 ")", 864 bus_num, devfn, ce.hi, ce.lo, 865 cc_entry->context_cache_gen, s->context_cache_gen); 866 cc_entry->context_entry = ce; 867 cc_entry->context_cache_gen = s->context_cache_gen; 868 } 869 870 ret_fr = vtd_gpa_to_slpte(&ce, addr, is_write, &slpte, &level, 871 &reads, &writes); 872 if (ret_fr) { 873 ret_fr = -ret_fr; 874 if (is_fpd_set && vtd_is_qualified_fault(ret_fr)) { 875 VTD_DPRINTF(FLOG, "fault processing is disabled for DMA requests " 876 "through this context-entry (with FPD Set)"); 877 } else { 878 vtd_report_dmar_fault(s, source_id, addr, ret_fr, is_write); 879 } 880 return; 881 } 882 883 page_mask = vtd_slpt_level_page_mask(level); 884 vtd_update_iotlb(s, source_id, VTD_CONTEXT_ENTRY_DID(ce.hi), addr, slpte, 885 reads, writes, level); 886 out: 887 entry->iova = addr & page_mask; 888 entry->translated_addr = vtd_get_slpte_addr(slpte) & page_mask; 889 entry->addr_mask = ~page_mask; 890 entry->perm = (writes ? 2 : 0) + (reads ? 1 : 0); 891 } 892 893 static void vtd_root_table_setup(IntelIOMMUState *s) 894 { 895 s->root = vtd_get_quad_raw(s, DMAR_RTADDR_REG); 896 s->root_extended = s->root & VTD_RTADDR_RTT; 897 s->root &= VTD_RTADDR_ADDR_MASK; 898 899 VTD_DPRINTF(CSR, "root_table addr 0x%"PRIx64 " %s", s->root, 900 (s->root_extended ? "(extended)" : "")); 901 } 902 903 static void vtd_context_global_invalidate(IntelIOMMUState *s) 904 { 905 s->context_cache_gen++; 906 if (s->context_cache_gen == VTD_CONTEXT_CACHE_GEN_MAX) { 907 vtd_reset_context_cache(s); 908 } 909 } 910 911 912 /* Find the VTD address space currently associated with a given bus number, 913 */ 914 static VTDBus *vtd_find_as_from_bus_num(IntelIOMMUState *s, uint8_t bus_num) 915 { 916 VTDBus *vtd_bus = s->vtd_as_by_bus_num[bus_num]; 917 if (!vtd_bus) { 918 /* Iterate over the registered buses to find the one 919 * which currently hold this bus number, and update the bus_num lookup table: 920 */ 921 GHashTableIter iter; 922 923 g_hash_table_iter_init(&iter, s->vtd_as_by_busptr); 924 while (g_hash_table_iter_next (&iter, NULL, (void**)&vtd_bus)) { 925 if (pci_bus_num(vtd_bus->bus) == bus_num) { 926 s->vtd_as_by_bus_num[bus_num] = vtd_bus; 927 return vtd_bus; 928 } 929 } 930 } 931 return vtd_bus; 932 } 933 934 /* Do a context-cache device-selective invalidation. 935 * @func_mask: FM field after shifting 936 */ 937 static void vtd_context_device_invalidate(IntelIOMMUState *s, 938 uint16_t source_id, 939 uint16_t func_mask) 940 { 941 uint16_t mask; 942 VTDBus *vtd_bus; 943 VTDAddressSpace *vtd_as; 944 uint16_t devfn; 945 uint16_t devfn_it; 946 947 switch (func_mask & 3) { 948 case 0: 949 mask = 0; /* No bits in the SID field masked */ 950 break; 951 case 1: 952 mask = 4; /* Mask bit 2 in the SID field */ 953 break; 954 case 2: 955 mask = 6; /* Mask bit 2:1 in the SID field */ 956 break; 957 case 3: 958 mask = 7; /* Mask bit 2:0 in the SID field */ 959 break; 960 } 961 VTD_DPRINTF(INV, "device-selective invalidation source 0x%"PRIx16 962 " mask %"PRIu16, source_id, mask); 963 vtd_bus = vtd_find_as_from_bus_num(s, VTD_SID_TO_BUS(source_id)); 964 if (vtd_bus) { 965 devfn = VTD_SID_TO_DEVFN(source_id); 966 for (devfn_it = 0; devfn_it < VTD_PCI_DEVFN_MAX; ++devfn_it) { 967 vtd_as = vtd_bus->dev_as[devfn_it]; 968 if (vtd_as && ((devfn_it & mask) == (devfn & mask))) { 969 VTD_DPRINTF(INV, "invalidate context-cahce of devfn 0x%"PRIx16, 970 devfn_it); 971 vtd_as->context_cache_entry.context_cache_gen = 0; 972 } 973 } 974 } 975 } 976 977 /* Context-cache invalidation 978 * Returns the Context Actual Invalidation Granularity. 979 * @val: the content of the CCMD_REG 980 */ 981 static uint64_t vtd_context_cache_invalidate(IntelIOMMUState *s, uint64_t val) 982 { 983 uint64_t caig; 984 uint64_t type = val & VTD_CCMD_CIRG_MASK; 985 986 switch (type) { 987 case VTD_CCMD_DOMAIN_INVL: 988 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 989 (uint16_t)VTD_CCMD_DID(val)); 990 /* Fall through */ 991 case VTD_CCMD_GLOBAL_INVL: 992 VTD_DPRINTF(INV, "global invalidation"); 993 caig = VTD_CCMD_GLOBAL_INVL_A; 994 vtd_context_global_invalidate(s); 995 break; 996 997 case VTD_CCMD_DEVICE_INVL: 998 caig = VTD_CCMD_DEVICE_INVL_A; 999 vtd_context_device_invalidate(s, VTD_CCMD_SID(val), VTD_CCMD_FM(val)); 1000 break; 1001 1002 default: 1003 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 1004 caig = 0; 1005 } 1006 return caig; 1007 } 1008 1009 static void vtd_iotlb_global_invalidate(IntelIOMMUState *s) 1010 { 1011 vtd_reset_iotlb(s); 1012 } 1013 1014 static void vtd_iotlb_domain_invalidate(IntelIOMMUState *s, uint16_t domain_id) 1015 { 1016 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_domain, 1017 &domain_id); 1018 } 1019 1020 static void vtd_iotlb_page_invalidate(IntelIOMMUState *s, uint16_t domain_id, 1021 hwaddr addr, uint8_t am) 1022 { 1023 VTDIOTLBPageInvInfo info; 1024 1025 assert(am <= VTD_MAMV); 1026 info.domain_id = domain_id; 1027 info.addr = addr; 1028 info.mask = ~((1 << am) - 1); 1029 g_hash_table_foreach_remove(s->iotlb, vtd_hash_remove_by_page, &info); 1030 } 1031 1032 /* Flush IOTLB 1033 * Returns the IOTLB Actual Invalidation Granularity. 1034 * @val: the content of the IOTLB_REG 1035 */ 1036 static uint64_t vtd_iotlb_flush(IntelIOMMUState *s, uint64_t val) 1037 { 1038 uint64_t iaig; 1039 uint64_t type = val & VTD_TLB_FLUSH_GRANU_MASK; 1040 uint16_t domain_id; 1041 hwaddr addr; 1042 uint8_t am; 1043 1044 switch (type) { 1045 case VTD_TLB_GLOBAL_FLUSH: 1046 VTD_DPRINTF(INV, "global invalidation"); 1047 iaig = VTD_TLB_GLOBAL_FLUSH_A; 1048 vtd_iotlb_global_invalidate(s); 1049 break; 1050 1051 case VTD_TLB_DSI_FLUSH: 1052 domain_id = VTD_TLB_DID(val); 1053 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1054 domain_id); 1055 iaig = VTD_TLB_DSI_FLUSH_A; 1056 vtd_iotlb_domain_invalidate(s, domain_id); 1057 break; 1058 1059 case VTD_TLB_PSI_FLUSH: 1060 domain_id = VTD_TLB_DID(val); 1061 addr = vtd_get_quad_raw(s, DMAR_IVA_REG); 1062 am = VTD_IVA_AM(addr); 1063 addr = VTD_IVA_ADDR(addr); 1064 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1065 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1066 if (am > VTD_MAMV) { 1067 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1068 "%"PRIu8, (uint8_t)VTD_MAMV); 1069 iaig = 0; 1070 break; 1071 } 1072 iaig = VTD_TLB_PSI_FLUSH_A; 1073 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1074 break; 1075 1076 default: 1077 VTD_DPRINTF(GENERAL, "error: invalid granularity"); 1078 iaig = 0; 1079 } 1080 return iaig; 1081 } 1082 1083 static inline bool vtd_queued_inv_enable_check(IntelIOMMUState *s) 1084 { 1085 return s->iq_tail == 0; 1086 } 1087 1088 static inline bool vtd_queued_inv_disable_check(IntelIOMMUState *s) 1089 { 1090 return s->qi_enabled && (s->iq_tail == s->iq_head) && 1091 (s->iq_last_desc_type == VTD_INV_DESC_WAIT); 1092 } 1093 1094 static void vtd_handle_gcmd_qie(IntelIOMMUState *s, bool en) 1095 { 1096 uint64_t iqa_val = vtd_get_quad_raw(s, DMAR_IQA_REG); 1097 1098 VTD_DPRINTF(INV, "Queued Invalidation Enable %s", (en ? "on" : "off")); 1099 if (en) { 1100 if (vtd_queued_inv_enable_check(s)) { 1101 s->iq = iqa_val & VTD_IQA_IQA_MASK; 1102 /* 2^(x+8) entries */ 1103 s->iq_size = 1UL << ((iqa_val & VTD_IQA_QS) + 8); 1104 s->qi_enabled = true; 1105 VTD_DPRINTF(INV, "DMAR_IQA_REG 0x%"PRIx64, iqa_val); 1106 VTD_DPRINTF(INV, "Invalidation Queue addr 0x%"PRIx64 " size %d", 1107 s->iq, s->iq_size); 1108 /* Ok - report back to driver */ 1109 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_QIES); 1110 } else { 1111 VTD_DPRINTF(GENERAL, "error: can't enable Queued Invalidation: " 1112 "tail %"PRIu16, s->iq_tail); 1113 } 1114 } else { 1115 if (vtd_queued_inv_disable_check(s)) { 1116 /* disable Queued Invalidation */ 1117 vtd_set_quad_raw(s, DMAR_IQH_REG, 0); 1118 s->iq_head = 0; 1119 s->qi_enabled = false; 1120 /* Ok - report back to driver */ 1121 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_QIES, 0); 1122 } else { 1123 VTD_DPRINTF(GENERAL, "error: can't disable Queued Invalidation: " 1124 "head %"PRIu16 ", tail %"PRIu16 1125 ", last_descriptor %"PRIu8, 1126 s->iq_head, s->iq_tail, s->iq_last_desc_type); 1127 } 1128 } 1129 } 1130 1131 /* Set Root Table Pointer */ 1132 static void vtd_handle_gcmd_srtp(IntelIOMMUState *s) 1133 { 1134 VTD_DPRINTF(CSR, "set Root Table Pointer"); 1135 1136 vtd_root_table_setup(s); 1137 /* Ok - report back to driver */ 1138 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_RTPS); 1139 } 1140 1141 /* Handle Translation Enable/Disable */ 1142 static void vtd_handle_gcmd_te(IntelIOMMUState *s, bool en) 1143 { 1144 VTD_DPRINTF(CSR, "Translation Enable %s", (en ? "on" : "off")); 1145 1146 if (en) { 1147 s->dmar_enabled = true; 1148 /* Ok - report back to driver */ 1149 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, 0, VTD_GSTS_TES); 1150 } else { 1151 s->dmar_enabled = false; 1152 1153 /* Clear the index of Fault Recording Register */ 1154 s->next_frcd_reg = 0; 1155 /* Ok - report back to driver */ 1156 vtd_set_clear_mask_long(s, DMAR_GSTS_REG, VTD_GSTS_TES, 0); 1157 } 1158 } 1159 1160 /* Handle write to Global Command Register */ 1161 static void vtd_handle_gcmd_write(IntelIOMMUState *s) 1162 { 1163 uint32_t status = vtd_get_long_raw(s, DMAR_GSTS_REG); 1164 uint32_t val = vtd_get_long_raw(s, DMAR_GCMD_REG); 1165 uint32_t changed = status ^ val; 1166 1167 VTD_DPRINTF(CSR, "value 0x%"PRIx32 " status 0x%"PRIx32, val, status); 1168 if (changed & VTD_GCMD_TE) { 1169 /* Translation enable/disable */ 1170 vtd_handle_gcmd_te(s, val & VTD_GCMD_TE); 1171 } 1172 if (val & VTD_GCMD_SRTP) { 1173 /* Set/update the root-table pointer */ 1174 vtd_handle_gcmd_srtp(s); 1175 } 1176 if (changed & VTD_GCMD_QIE) { 1177 /* Queued Invalidation Enable */ 1178 vtd_handle_gcmd_qie(s, val & VTD_GCMD_QIE); 1179 } 1180 } 1181 1182 /* Handle write to Context Command Register */ 1183 static void vtd_handle_ccmd_write(IntelIOMMUState *s) 1184 { 1185 uint64_t ret; 1186 uint64_t val = vtd_get_quad_raw(s, DMAR_CCMD_REG); 1187 1188 /* Context-cache invalidation request */ 1189 if (val & VTD_CCMD_ICC) { 1190 if (s->qi_enabled) { 1191 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1192 "should not use register-based invalidation"); 1193 return; 1194 } 1195 ret = vtd_context_cache_invalidate(s, val); 1196 /* Invalidation completed. Change something to show */ 1197 vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_ICC, 0ULL); 1198 ret = vtd_set_clear_mask_quad(s, DMAR_CCMD_REG, VTD_CCMD_CAIG_MASK, 1199 ret); 1200 VTD_DPRINTF(INV, "CCMD_REG write-back val: 0x%"PRIx64, ret); 1201 } 1202 } 1203 1204 /* Handle write to IOTLB Invalidation Register */ 1205 static void vtd_handle_iotlb_write(IntelIOMMUState *s) 1206 { 1207 uint64_t ret; 1208 uint64_t val = vtd_get_quad_raw(s, DMAR_IOTLB_REG); 1209 1210 /* IOTLB invalidation request */ 1211 if (val & VTD_TLB_IVT) { 1212 if (s->qi_enabled) { 1213 VTD_DPRINTF(GENERAL, "error: Queued Invalidation enabled, " 1214 "should not use register-based invalidation"); 1215 return; 1216 } 1217 ret = vtd_iotlb_flush(s, val); 1218 /* Invalidation completed. Change something to show */ 1219 vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, VTD_TLB_IVT, 0ULL); 1220 ret = vtd_set_clear_mask_quad(s, DMAR_IOTLB_REG, 1221 VTD_TLB_FLUSH_GRANU_MASK_A, ret); 1222 VTD_DPRINTF(INV, "IOTLB_REG write-back val: 0x%"PRIx64, ret); 1223 } 1224 } 1225 1226 /* Fetch an Invalidation Descriptor from the Invalidation Queue */ 1227 static bool vtd_get_inv_desc(dma_addr_t base_addr, uint32_t offset, 1228 VTDInvDesc *inv_desc) 1229 { 1230 dma_addr_t addr = base_addr + offset * sizeof(*inv_desc); 1231 if (dma_memory_read(&address_space_memory, addr, inv_desc, 1232 sizeof(*inv_desc))) { 1233 VTD_DPRINTF(GENERAL, "error: fail to fetch Invalidation Descriptor " 1234 "base_addr 0x%"PRIx64 " offset %"PRIu32, base_addr, offset); 1235 inv_desc->lo = 0; 1236 inv_desc->hi = 0; 1237 1238 return false; 1239 } 1240 inv_desc->lo = le64_to_cpu(inv_desc->lo); 1241 inv_desc->hi = le64_to_cpu(inv_desc->hi); 1242 return true; 1243 } 1244 1245 static bool vtd_process_wait_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1246 { 1247 if ((inv_desc->hi & VTD_INV_DESC_WAIT_RSVD_HI) || 1248 (inv_desc->lo & VTD_INV_DESC_WAIT_RSVD_LO)) { 1249 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Invalidation " 1250 "Wait Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1251 inv_desc->hi, inv_desc->lo); 1252 return false; 1253 } 1254 if (inv_desc->lo & VTD_INV_DESC_WAIT_SW) { 1255 /* Status Write */ 1256 uint32_t status_data = (uint32_t)(inv_desc->lo >> 1257 VTD_INV_DESC_WAIT_DATA_SHIFT); 1258 1259 assert(!(inv_desc->lo & VTD_INV_DESC_WAIT_IF)); 1260 1261 /* FIXME: need to be masked with HAW? */ 1262 dma_addr_t status_addr = inv_desc->hi; 1263 VTD_DPRINTF(INV, "status data 0x%x, status addr 0x%"PRIx64, 1264 status_data, status_addr); 1265 status_data = cpu_to_le32(status_data); 1266 if (dma_memory_write(&address_space_memory, status_addr, &status_data, 1267 sizeof(status_data))) { 1268 VTD_DPRINTF(GENERAL, "error: fail to perform a coherent write"); 1269 return false; 1270 } 1271 } else if (inv_desc->lo & VTD_INV_DESC_WAIT_IF) { 1272 /* Interrupt flag */ 1273 VTD_DPRINTF(INV, "Invalidation Wait Descriptor interrupt completion"); 1274 vtd_generate_completion_event(s); 1275 } else { 1276 VTD_DPRINTF(GENERAL, "error: invalid Invalidation Wait Descriptor: " 1277 "hi 0x%"PRIx64 " lo 0x%"PRIx64, inv_desc->hi, inv_desc->lo); 1278 return false; 1279 } 1280 return true; 1281 } 1282 1283 static bool vtd_process_context_cache_desc(IntelIOMMUState *s, 1284 VTDInvDesc *inv_desc) 1285 { 1286 if ((inv_desc->lo & VTD_INV_DESC_CC_RSVD) || inv_desc->hi) { 1287 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in Context-cache " 1288 "Invalidate Descriptor"); 1289 return false; 1290 } 1291 switch (inv_desc->lo & VTD_INV_DESC_CC_G) { 1292 case VTD_INV_DESC_CC_DOMAIN: 1293 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1294 (uint16_t)VTD_INV_DESC_CC_DID(inv_desc->lo)); 1295 /* Fall through */ 1296 case VTD_INV_DESC_CC_GLOBAL: 1297 VTD_DPRINTF(INV, "global invalidation"); 1298 vtd_context_global_invalidate(s); 1299 break; 1300 1301 case VTD_INV_DESC_CC_DEVICE: 1302 vtd_context_device_invalidate(s, VTD_INV_DESC_CC_SID(inv_desc->lo), 1303 VTD_INV_DESC_CC_FM(inv_desc->lo)); 1304 break; 1305 1306 default: 1307 VTD_DPRINTF(GENERAL, "error: invalid granularity in Context-cache " 1308 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1309 inv_desc->hi, inv_desc->lo); 1310 return false; 1311 } 1312 return true; 1313 } 1314 1315 static bool vtd_process_iotlb_desc(IntelIOMMUState *s, VTDInvDesc *inv_desc) 1316 { 1317 uint16_t domain_id; 1318 uint8_t am; 1319 hwaddr addr; 1320 1321 if ((inv_desc->lo & VTD_INV_DESC_IOTLB_RSVD_LO) || 1322 (inv_desc->hi & VTD_INV_DESC_IOTLB_RSVD_HI)) { 1323 VTD_DPRINTF(GENERAL, "error: non-zero reserved field in IOTLB " 1324 "Invalidate Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1325 inv_desc->hi, inv_desc->lo); 1326 return false; 1327 } 1328 1329 switch (inv_desc->lo & VTD_INV_DESC_IOTLB_G) { 1330 case VTD_INV_DESC_IOTLB_GLOBAL: 1331 VTD_DPRINTF(INV, "global invalidation"); 1332 vtd_iotlb_global_invalidate(s); 1333 break; 1334 1335 case VTD_INV_DESC_IOTLB_DOMAIN: 1336 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1337 VTD_DPRINTF(INV, "domain-selective invalidation domain 0x%"PRIx16, 1338 domain_id); 1339 vtd_iotlb_domain_invalidate(s, domain_id); 1340 break; 1341 1342 case VTD_INV_DESC_IOTLB_PAGE: 1343 domain_id = VTD_INV_DESC_IOTLB_DID(inv_desc->lo); 1344 addr = VTD_INV_DESC_IOTLB_ADDR(inv_desc->hi); 1345 am = VTD_INV_DESC_IOTLB_AM(inv_desc->hi); 1346 VTD_DPRINTF(INV, "page-selective invalidation domain 0x%"PRIx16 1347 " addr 0x%"PRIx64 " mask %"PRIu8, domain_id, addr, am); 1348 if (am > VTD_MAMV) { 1349 VTD_DPRINTF(GENERAL, "error: supported max address mask value is " 1350 "%"PRIu8, (uint8_t)VTD_MAMV); 1351 return false; 1352 } 1353 vtd_iotlb_page_invalidate(s, domain_id, addr, am); 1354 break; 1355 1356 default: 1357 VTD_DPRINTF(GENERAL, "error: invalid granularity in IOTLB Invalidate " 1358 "Descriptor hi 0x%"PRIx64 " lo 0x%"PRIx64, 1359 inv_desc->hi, inv_desc->lo); 1360 return false; 1361 } 1362 return true; 1363 } 1364 1365 static bool vtd_process_inv_desc(IntelIOMMUState *s) 1366 { 1367 VTDInvDesc inv_desc; 1368 uint8_t desc_type; 1369 1370 VTD_DPRINTF(INV, "iq head %"PRIu16, s->iq_head); 1371 if (!vtd_get_inv_desc(s->iq, s->iq_head, &inv_desc)) { 1372 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1373 return false; 1374 } 1375 desc_type = inv_desc.lo & VTD_INV_DESC_TYPE; 1376 /* FIXME: should update at first or at last? */ 1377 s->iq_last_desc_type = desc_type; 1378 1379 switch (desc_type) { 1380 case VTD_INV_DESC_CC: 1381 VTD_DPRINTF(INV, "Context-cache Invalidate Descriptor hi 0x%"PRIx64 1382 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1383 if (!vtd_process_context_cache_desc(s, &inv_desc)) { 1384 return false; 1385 } 1386 break; 1387 1388 case VTD_INV_DESC_IOTLB: 1389 VTD_DPRINTF(INV, "IOTLB Invalidate Descriptor hi 0x%"PRIx64 1390 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1391 if (!vtd_process_iotlb_desc(s, &inv_desc)) { 1392 return false; 1393 } 1394 break; 1395 1396 case VTD_INV_DESC_WAIT: 1397 VTD_DPRINTF(INV, "Invalidation Wait Descriptor hi 0x%"PRIx64 1398 " lo 0x%"PRIx64, inv_desc.hi, inv_desc.lo); 1399 if (!vtd_process_wait_desc(s, &inv_desc)) { 1400 return false; 1401 } 1402 break; 1403 1404 default: 1405 VTD_DPRINTF(GENERAL, "error: unkonw Invalidation Descriptor type " 1406 "hi 0x%"PRIx64 " lo 0x%"PRIx64 " type %"PRIu8, 1407 inv_desc.hi, inv_desc.lo, desc_type); 1408 return false; 1409 } 1410 s->iq_head++; 1411 if (s->iq_head == s->iq_size) { 1412 s->iq_head = 0; 1413 } 1414 return true; 1415 } 1416 1417 /* Try to fetch and process more Invalidation Descriptors */ 1418 static void vtd_fetch_inv_desc(IntelIOMMUState *s) 1419 { 1420 VTD_DPRINTF(INV, "fetch Invalidation Descriptors"); 1421 if (s->iq_tail >= s->iq_size) { 1422 /* Detects an invalid Tail pointer */ 1423 VTD_DPRINTF(GENERAL, "error: iq_tail is %"PRIu16 1424 " while iq_size is %"PRIu16, s->iq_tail, s->iq_size); 1425 vtd_handle_inv_queue_error(s); 1426 return; 1427 } 1428 while (s->iq_head != s->iq_tail) { 1429 if (!vtd_process_inv_desc(s)) { 1430 /* Invalidation Queue Errors */ 1431 vtd_handle_inv_queue_error(s); 1432 break; 1433 } 1434 /* Must update the IQH_REG in time */ 1435 vtd_set_quad_raw(s, DMAR_IQH_REG, 1436 (((uint64_t)(s->iq_head)) << VTD_IQH_QH_SHIFT) & 1437 VTD_IQH_QH_MASK); 1438 } 1439 } 1440 1441 /* Handle write to Invalidation Queue Tail Register */ 1442 static void vtd_handle_iqt_write(IntelIOMMUState *s) 1443 { 1444 uint64_t val = vtd_get_quad_raw(s, DMAR_IQT_REG); 1445 1446 s->iq_tail = VTD_IQT_QT(val); 1447 VTD_DPRINTF(INV, "set iq tail %"PRIu16, s->iq_tail); 1448 if (s->qi_enabled && !(vtd_get_long_raw(s, DMAR_FSTS_REG) & VTD_FSTS_IQE)) { 1449 /* Process Invalidation Queue here */ 1450 vtd_fetch_inv_desc(s); 1451 } 1452 } 1453 1454 static void vtd_handle_fsts_write(IntelIOMMUState *s) 1455 { 1456 uint32_t fsts_reg = vtd_get_long_raw(s, DMAR_FSTS_REG); 1457 uint32_t fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1458 uint32_t status_fields = VTD_FSTS_PFO | VTD_FSTS_PPF | VTD_FSTS_IQE; 1459 1460 if ((fectl_reg & VTD_FECTL_IP) && !(fsts_reg & status_fields)) { 1461 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1462 VTD_DPRINTF(FLOG, "all pending interrupt conditions serviced, clear " 1463 "IP field of FECTL_REG"); 1464 } 1465 /* FIXME: when IQE is Clear, should we try to fetch some Invalidation 1466 * Descriptors if there are any when Queued Invalidation is enabled? 1467 */ 1468 } 1469 1470 static void vtd_handle_fectl_write(IntelIOMMUState *s) 1471 { 1472 uint32_t fectl_reg; 1473 /* FIXME: when software clears the IM field, check the IP field. But do we 1474 * need to compare the old value and the new value to conclude that 1475 * software clears the IM field? Or just check if the IM field is zero? 1476 */ 1477 fectl_reg = vtd_get_long_raw(s, DMAR_FECTL_REG); 1478 if ((fectl_reg & VTD_FECTL_IP) && !(fectl_reg & VTD_FECTL_IM)) { 1479 vtd_generate_interrupt(s, DMAR_FEADDR_REG, DMAR_FEDATA_REG); 1480 vtd_set_clear_mask_long(s, DMAR_FECTL_REG, VTD_FECTL_IP, 0); 1481 VTD_DPRINTF(FLOG, "IM field is cleared, generate " 1482 "fault event interrupt"); 1483 } 1484 } 1485 1486 static void vtd_handle_ics_write(IntelIOMMUState *s) 1487 { 1488 uint32_t ics_reg = vtd_get_long_raw(s, DMAR_ICS_REG); 1489 uint32_t iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1490 1491 if ((iectl_reg & VTD_IECTL_IP) && !(ics_reg & VTD_ICS_IWC)) { 1492 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1493 VTD_DPRINTF(INV, "pending completion interrupt condition serviced, " 1494 "clear IP field of IECTL_REG"); 1495 } 1496 } 1497 1498 static void vtd_handle_iectl_write(IntelIOMMUState *s) 1499 { 1500 uint32_t iectl_reg; 1501 /* FIXME: when software clears the IM field, check the IP field. But do we 1502 * need to compare the old value and the new value to conclude that 1503 * software clears the IM field? Or just check if the IM field is zero? 1504 */ 1505 iectl_reg = vtd_get_long_raw(s, DMAR_IECTL_REG); 1506 if ((iectl_reg & VTD_IECTL_IP) && !(iectl_reg & VTD_IECTL_IM)) { 1507 vtd_generate_interrupt(s, DMAR_IEADDR_REG, DMAR_IEDATA_REG); 1508 vtd_set_clear_mask_long(s, DMAR_IECTL_REG, VTD_IECTL_IP, 0); 1509 VTD_DPRINTF(INV, "IM field is cleared, generate " 1510 "invalidation event interrupt"); 1511 } 1512 } 1513 1514 static uint64_t vtd_mem_read(void *opaque, hwaddr addr, unsigned size) 1515 { 1516 IntelIOMMUState *s = opaque; 1517 uint64_t val; 1518 1519 if (addr + size > DMAR_REG_SIZE) { 1520 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1521 ", got 0x%"PRIx64 " %d", 1522 (uint64_t)DMAR_REG_SIZE, addr, size); 1523 return (uint64_t)-1; 1524 } 1525 1526 switch (addr) { 1527 /* Root Table Address Register, 64-bit */ 1528 case DMAR_RTADDR_REG: 1529 if (size == 4) { 1530 val = s->root & ((1ULL << 32) - 1); 1531 } else { 1532 val = s->root; 1533 } 1534 break; 1535 1536 case DMAR_RTADDR_REG_HI: 1537 assert(size == 4); 1538 val = s->root >> 32; 1539 break; 1540 1541 /* Invalidation Queue Address Register, 64-bit */ 1542 case DMAR_IQA_REG: 1543 val = s->iq | (vtd_get_quad(s, DMAR_IQA_REG) & VTD_IQA_QS); 1544 if (size == 4) { 1545 val = val & ((1ULL << 32) - 1); 1546 } 1547 break; 1548 1549 case DMAR_IQA_REG_HI: 1550 assert(size == 4); 1551 val = s->iq >> 32; 1552 break; 1553 1554 default: 1555 if (size == 4) { 1556 val = vtd_get_long(s, addr); 1557 } else { 1558 val = vtd_get_quad(s, addr); 1559 } 1560 } 1561 VTD_DPRINTF(CSR, "addr 0x%"PRIx64 " size %d val 0x%"PRIx64, 1562 addr, size, val); 1563 return val; 1564 } 1565 1566 static void vtd_mem_write(void *opaque, hwaddr addr, 1567 uint64_t val, unsigned size) 1568 { 1569 IntelIOMMUState *s = opaque; 1570 1571 if (addr + size > DMAR_REG_SIZE) { 1572 VTD_DPRINTF(GENERAL, "error: addr outside region: max 0x%"PRIx64 1573 ", got 0x%"PRIx64 " %d", 1574 (uint64_t)DMAR_REG_SIZE, addr, size); 1575 return; 1576 } 1577 1578 switch (addr) { 1579 /* Global Command Register, 32-bit */ 1580 case DMAR_GCMD_REG: 1581 VTD_DPRINTF(CSR, "DMAR_GCMD_REG write addr 0x%"PRIx64 1582 ", size %d, val 0x%"PRIx64, addr, size, val); 1583 vtd_set_long(s, addr, val); 1584 vtd_handle_gcmd_write(s); 1585 break; 1586 1587 /* Context Command Register, 64-bit */ 1588 case DMAR_CCMD_REG: 1589 VTD_DPRINTF(CSR, "DMAR_CCMD_REG write addr 0x%"PRIx64 1590 ", size %d, val 0x%"PRIx64, addr, size, val); 1591 if (size == 4) { 1592 vtd_set_long(s, addr, val); 1593 } else { 1594 vtd_set_quad(s, addr, val); 1595 vtd_handle_ccmd_write(s); 1596 } 1597 break; 1598 1599 case DMAR_CCMD_REG_HI: 1600 VTD_DPRINTF(CSR, "DMAR_CCMD_REG_HI write addr 0x%"PRIx64 1601 ", size %d, val 0x%"PRIx64, addr, size, val); 1602 assert(size == 4); 1603 vtd_set_long(s, addr, val); 1604 vtd_handle_ccmd_write(s); 1605 break; 1606 1607 /* IOTLB Invalidation Register, 64-bit */ 1608 case DMAR_IOTLB_REG: 1609 VTD_DPRINTF(INV, "DMAR_IOTLB_REG write addr 0x%"PRIx64 1610 ", size %d, val 0x%"PRIx64, addr, size, val); 1611 if (size == 4) { 1612 vtd_set_long(s, addr, val); 1613 } else { 1614 vtd_set_quad(s, addr, val); 1615 vtd_handle_iotlb_write(s); 1616 } 1617 break; 1618 1619 case DMAR_IOTLB_REG_HI: 1620 VTD_DPRINTF(INV, "DMAR_IOTLB_REG_HI write addr 0x%"PRIx64 1621 ", size %d, val 0x%"PRIx64, addr, size, val); 1622 assert(size == 4); 1623 vtd_set_long(s, addr, val); 1624 vtd_handle_iotlb_write(s); 1625 break; 1626 1627 /* Invalidate Address Register, 64-bit */ 1628 case DMAR_IVA_REG: 1629 VTD_DPRINTF(INV, "DMAR_IVA_REG write addr 0x%"PRIx64 1630 ", size %d, val 0x%"PRIx64, addr, size, val); 1631 if (size == 4) { 1632 vtd_set_long(s, addr, val); 1633 } else { 1634 vtd_set_quad(s, addr, val); 1635 } 1636 break; 1637 1638 case DMAR_IVA_REG_HI: 1639 VTD_DPRINTF(INV, "DMAR_IVA_REG_HI write addr 0x%"PRIx64 1640 ", size %d, val 0x%"PRIx64, addr, size, val); 1641 assert(size == 4); 1642 vtd_set_long(s, addr, val); 1643 break; 1644 1645 /* Fault Status Register, 32-bit */ 1646 case DMAR_FSTS_REG: 1647 VTD_DPRINTF(FLOG, "DMAR_FSTS_REG write addr 0x%"PRIx64 1648 ", size %d, val 0x%"PRIx64, addr, size, val); 1649 assert(size == 4); 1650 vtd_set_long(s, addr, val); 1651 vtd_handle_fsts_write(s); 1652 break; 1653 1654 /* Fault Event Control Register, 32-bit */ 1655 case DMAR_FECTL_REG: 1656 VTD_DPRINTF(FLOG, "DMAR_FECTL_REG write addr 0x%"PRIx64 1657 ", size %d, val 0x%"PRIx64, addr, size, val); 1658 assert(size == 4); 1659 vtd_set_long(s, addr, val); 1660 vtd_handle_fectl_write(s); 1661 break; 1662 1663 /* Fault Event Data Register, 32-bit */ 1664 case DMAR_FEDATA_REG: 1665 VTD_DPRINTF(FLOG, "DMAR_FEDATA_REG write addr 0x%"PRIx64 1666 ", size %d, val 0x%"PRIx64, addr, size, val); 1667 assert(size == 4); 1668 vtd_set_long(s, addr, val); 1669 break; 1670 1671 /* Fault Event Address Register, 32-bit */ 1672 case DMAR_FEADDR_REG: 1673 VTD_DPRINTF(FLOG, "DMAR_FEADDR_REG write addr 0x%"PRIx64 1674 ", size %d, val 0x%"PRIx64, addr, size, val); 1675 assert(size == 4); 1676 vtd_set_long(s, addr, val); 1677 break; 1678 1679 /* Fault Event Upper Address Register, 32-bit */ 1680 case DMAR_FEUADDR_REG: 1681 VTD_DPRINTF(FLOG, "DMAR_FEUADDR_REG write addr 0x%"PRIx64 1682 ", size %d, val 0x%"PRIx64, addr, size, val); 1683 assert(size == 4); 1684 vtd_set_long(s, addr, val); 1685 break; 1686 1687 /* Protected Memory Enable Register, 32-bit */ 1688 case DMAR_PMEN_REG: 1689 VTD_DPRINTF(CSR, "DMAR_PMEN_REG write addr 0x%"PRIx64 1690 ", size %d, val 0x%"PRIx64, addr, size, val); 1691 assert(size == 4); 1692 vtd_set_long(s, addr, val); 1693 break; 1694 1695 /* Root Table Address Register, 64-bit */ 1696 case DMAR_RTADDR_REG: 1697 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG write addr 0x%"PRIx64 1698 ", size %d, val 0x%"PRIx64, addr, size, val); 1699 if (size == 4) { 1700 vtd_set_long(s, addr, val); 1701 } else { 1702 vtd_set_quad(s, addr, val); 1703 } 1704 break; 1705 1706 case DMAR_RTADDR_REG_HI: 1707 VTD_DPRINTF(CSR, "DMAR_RTADDR_REG_HI write addr 0x%"PRIx64 1708 ", size %d, val 0x%"PRIx64, addr, size, val); 1709 assert(size == 4); 1710 vtd_set_long(s, addr, val); 1711 break; 1712 1713 /* Invalidation Queue Tail Register, 64-bit */ 1714 case DMAR_IQT_REG: 1715 VTD_DPRINTF(INV, "DMAR_IQT_REG write addr 0x%"PRIx64 1716 ", size %d, val 0x%"PRIx64, addr, size, val); 1717 if (size == 4) { 1718 vtd_set_long(s, addr, val); 1719 } else { 1720 vtd_set_quad(s, addr, val); 1721 } 1722 vtd_handle_iqt_write(s); 1723 break; 1724 1725 case DMAR_IQT_REG_HI: 1726 VTD_DPRINTF(INV, "DMAR_IQT_REG_HI write addr 0x%"PRIx64 1727 ", size %d, val 0x%"PRIx64, addr, size, val); 1728 assert(size == 4); 1729 vtd_set_long(s, addr, val); 1730 /* 19:63 of IQT_REG is RsvdZ, do nothing here */ 1731 break; 1732 1733 /* Invalidation Queue Address Register, 64-bit */ 1734 case DMAR_IQA_REG: 1735 VTD_DPRINTF(INV, "DMAR_IQA_REG write addr 0x%"PRIx64 1736 ", size %d, val 0x%"PRIx64, addr, size, val); 1737 if (size == 4) { 1738 vtd_set_long(s, addr, val); 1739 } else { 1740 vtd_set_quad(s, addr, val); 1741 } 1742 break; 1743 1744 case DMAR_IQA_REG_HI: 1745 VTD_DPRINTF(INV, "DMAR_IQA_REG_HI write addr 0x%"PRIx64 1746 ", size %d, val 0x%"PRIx64, addr, size, val); 1747 assert(size == 4); 1748 vtd_set_long(s, addr, val); 1749 break; 1750 1751 /* Invalidation Completion Status Register, 32-bit */ 1752 case DMAR_ICS_REG: 1753 VTD_DPRINTF(INV, "DMAR_ICS_REG write addr 0x%"PRIx64 1754 ", size %d, val 0x%"PRIx64, addr, size, val); 1755 assert(size == 4); 1756 vtd_set_long(s, addr, val); 1757 vtd_handle_ics_write(s); 1758 break; 1759 1760 /* Invalidation Event Control Register, 32-bit */ 1761 case DMAR_IECTL_REG: 1762 VTD_DPRINTF(INV, "DMAR_IECTL_REG write addr 0x%"PRIx64 1763 ", size %d, val 0x%"PRIx64, addr, size, val); 1764 assert(size == 4); 1765 vtd_set_long(s, addr, val); 1766 vtd_handle_iectl_write(s); 1767 break; 1768 1769 /* Invalidation Event Data Register, 32-bit */ 1770 case DMAR_IEDATA_REG: 1771 VTD_DPRINTF(INV, "DMAR_IEDATA_REG write addr 0x%"PRIx64 1772 ", size %d, val 0x%"PRIx64, addr, size, val); 1773 assert(size == 4); 1774 vtd_set_long(s, addr, val); 1775 break; 1776 1777 /* Invalidation Event Address Register, 32-bit */ 1778 case DMAR_IEADDR_REG: 1779 VTD_DPRINTF(INV, "DMAR_IEADDR_REG write addr 0x%"PRIx64 1780 ", size %d, val 0x%"PRIx64, addr, size, val); 1781 assert(size == 4); 1782 vtd_set_long(s, addr, val); 1783 break; 1784 1785 /* Invalidation Event Upper Address Register, 32-bit */ 1786 case DMAR_IEUADDR_REG: 1787 VTD_DPRINTF(INV, "DMAR_IEUADDR_REG write addr 0x%"PRIx64 1788 ", size %d, val 0x%"PRIx64, addr, size, val); 1789 assert(size == 4); 1790 vtd_set_long(s, addr, val); 1791 break; 1792 1793 /* Fault Recording Registers, 128-bit */ 1794 case DMAR_FRCD_REG_0_0: 1795 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_0 write addr 0x%"PRIx64 1796 ", size %d, val 0x%"PRIx64, addr, size, val); 1797 if (size == 4) { 1798 vtd_set_long(s, addr, val); 1799 } else { 1800 vtd_set_quad(s, addr, val); 1801 } 1802 break; 1803 1804 case DMAR_FRCD_REG_0_1: 1805 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_1 write addr 0x%"PRIx64 1806 ", size %d, val 0x%"PRIx64, addr, size, val); 1807 assert(size == 4); 1808 vtd_set_long(s, addr, val); 1809 break; 1810 1811 case DMAR_FRCD_REG_0_2: 1812 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_2 write addr 0x%"PRIx64 1813 ", size %d, val 0x%"PRIx64, addr, size, val); 1814 if (size == 4) { 1815 vtd_set_long(s, addr, val); 1816 } else { 1817 vtd_set_quad(s, addr, val); 1818 /* May clear bit 127 (Fault), update PPF */ 1819 vtd_update_fsts_ppf(s); 1820 } 1821 break; 1822 1823 case DMAR_FRCD_REG_0_3: 1824 VTD_DPRINTF(FLOG, "DMAR_FRCD_REG_0_3 write addr 0x%"PRIx64 1825 ", size %d, val 0x%"PRIx64, addr, size, val); 1826 assert(size == 4); 1827 vtd_set_long(s, addr, val); 1828 /* May clear bit 127 (Fault), update PPF */ 1829 vtd_update_fsts_ppf(s); 1830 break; 1831 1832 default: 1833 VTD_DPRINTF(GENERAL, "error: unhandled reg write addr 0x%"PRIx64 1834 ", size %d, val 0x%"PRIx64, addr, size, val); 1835 if (size == 4) { 1836 vtd_set_long(s, addr, val); 1837 } else { 1838 vtd_set_quad(s, addr, val); 1839 } 1840 } 1841 } 1842 1843 static IOMMUTLBEntry vtd_iommu_translate(MemoryRegion *iommu, hwaddr addr, 1844 bool is_write) 1845 { 1846 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 1847 IntelIOMMUState *s = vtd_as->iommu_state; 1848 IOMMUTLBEntry ret = { 1849 .target_as = &address_space_memory, 1850 .iova = addr, 1851 .translated_addr = 0, 1852 .addr_mask = ~(hwaddr)0, 1853 .perm = IOMMU_NONE, 1854 }; 1855 1856 if (!s->dmar_enabled) { 1857 /* DMAR disabled, passthrough, use 4k-page*/ 1858 ret.iova = addr & VTD_PAGE_MASK_4K; 1859 ret.translated_addr = addr & VTD_PAGE_MASK_4K; 1860 ret.addr_mask = ~VTD_PAGE_MASK_4K; 1861 ret.perm = IOMMU_RW; 1862 return ret; 1863 } 1864 1865 vtd_do_iommu_translate(vtd_as, vtd_as->bus, vtd_as->devfn, addr, 1866 is_write, &ret); 1867 VTD_DPRINTF(MMU, 1868 "bus %"PRIu8 " slot %"PRIu8 " func %"PRIu8 " devfn %"PRIu8 1869 " gpa 0x%"PRIx64 " hpa 0x%"PRIx64, pci_bus_num(vtd_as->bus), 1870 VTD_PCI_SLOT(vtd_as->devfn), VTD_PCI_FUNC(vtd_as->devfn), 1871 vtd_as->devfn, addr, ret.translated_addr); 1872 return ret; 1873 } 1874 1875 static void vtd_iommu_notify_started(MemoryRegion *iommu) 1876 { 1877 VTDAddressSpace *vtd_as = container_of(iommu, VTDAddressSpace, iommu); 1878 1879 hw_error("Device at bus %s addr %02x.%d requires iommu notifier which " 1880 "is currently not supported by intel-iommu emulation", 1881 vtd_as->bus->qbus.name, PCI_SLOT(vtd_as->devfn), 1882 PCI_FUNC(vtd_as->devfn)); 1883 } 1884 1885 static const VMStateDescription vtd_vmstate = { 1886 .name = "iommu-intel", 1887 .unmigratable = 1, 1888 }; 1889 1890 static const MemoryRegionOps vtd_mem_ops = { 1891 .read = vtd_mem_read, 1892 .write = vtd_mem_write, 1893 .endianness = DEVICE_LITTLE_ENDIAN, 1894 .impl = { 1895 .min_access_size = 4, 1896 .max_access_size = 8, 1897 }, 1898 .valid = { 1899 .min_access_size = 4, 1900 .max_access_size = 8, 1901 }, 1902 }; 1903 1904 static Property vtd_properties[] = { 1905 DEFINE_PROP_UINT32("version", IntelIOMMUState, version, 0), 1906 DEFINE_PROP_END_OF_LIST(), 1907 }; 1908 1909 1910 VTDAddressSpace *vtd_find_add_as(IntelIOMMUState *s, PCIBus *bus, int devfn) 1911 { 1912 uintptr_t key = (uintptr_t)bus; 1913 VTDBus *vtd_bus = g_hash_table_lookup(s->vtd_as_by_busptr, &key); 1914 VTDAddressSpace *vtd_dev_as; 1915 1916 if (!vtd_bus) { 1917 /* No corresponding free() */ 1918 vtd_bus = g_malloc0(sizeof(VTDBus) + sizeof(VTDAddressSpace *) * VTD_PCI_DEVFN_MAX); 1919 vtd_bus->bus = bus; 1920 key = (uintptr_t)bus; 1921 g_hash_table_insert(s->vtd_as_by_busptr, &key, vtd_bus); 1922 } 1923 1924 vtd_dev_as = vtd_bus->dev_as[devfn]; 1925 1926 if (!vtd_dev_as) { 1927 vtd_bus->dev_as[devfn] = vtd_dev_as = g_malloc0(sizeof(VTDAddressSpace)); 1928 1929 vtd_dev_as->bus = bus; 1930 vtd_dev_as->devfn = (uint8_t)devfn; 1931 vtd_dev_as->iommu_state = s; 1932 vtd_dev_as->context_cache_entry.context_cache_gen = 0; 1933 memory_region_init_iommu(&vtd_dev_as->iommu, OBJECT(s), 1934 &s->iommu_ops, "intel_iommu", UINT64_MAX); 1935 address_space_init(&vtd_dev_as->as, 1936 &vtd_dev_as->iommu, "intel_iommu"); 1937 } 1938 return vtd_dev_as; 1939 } 1940 1941 /* Do the initialization. It will also be called when reset, so pay 1942 * attention when adding new initialization stuff. 1943 */ 1944 static void vtd_init(IntelIOMMUState *s) 1945 { 1946 memset(s->csr, 0, DMAR_REG_SIZE); 1947 memset(s->wmask, 0, DMAR_REG_SIZE); 1948 memset(s->w1cmask, 0, DMAR_REG_SIZE); 1949 memset(s->womask, 0, DMAR_REG_SIZE); 1950 1951 s->iommu_ops.translate = vtd_iommu_translate; 1952 s->iommu_ops.notify_started = vtd_iommu_notify_started; 1953 s->root = 0; 1954 s->root_extended = false; 1955 s->dmar_enabled = false; 1956 s->iq_head = 0; 1957 s->iq_tail = 0; 1958 s->iq = 0; 1959 s->iq_size = 0; 1960 s->qi_enabled = false; 1961 s->iq_last_desc_type = VTD_INV_DESC_NONE; 1962 s->next_frcd_reg = 0; 1963 s->cap = VTD_CAP_FRO | VTD_CAP_NFR | VTD_CAP_ND | VTD_CAP_MGAW | 1964 VTD_CAP_SAGAW | VTD_CAP_MAMV | VTD_CAP_PSI | VTD_CAP_SLLPS; 1965 s->ecap = VTD_ECAP_QI | VTD_ECAP_IRO; 1966 1967 vtd_reset_context_cache(s); 1968 vtd_reset_iotlb(s); 1969 1970 /* Define registers with default values and bit semantics */ 1971 vtd_define_long(s, DMAR_VER_REG, 0x10UL, 0, 0); 1972 vtd_define_quad(s, DMAR_CAP_REG, s->cap, 0, 0); 1973 vtd_define_quad(s, DMAR_ECAP_REG, s->ecap, 0, 0); 1974 vtd_define_long(s, DMAR_GCMD_REG, 0, 0xff800000UL, 0); 1975 vtd_define_long_wo(s, DMAR_GCMD_REG, 0xff800000UL); 1976 vtd_define_long(s, DMAR_GSTS_REG, 0, 0, 0); 1977 vtd_define_quad(s, DMAR_RTADDR_REG, 0, 0xfffffffffffff000ULL, 0); 1978 vtd_define_quad(s, DMAR_CCMD_REG, 0, 0xe0000003ffffffffULL, 0); 1979 vtd_define_quad_wo(s, DMAR_CCMD_REG, 0x3ffff0000ULL); 1980 1981 /* Advanced Fault Logging not supported */ 1982 vtd_define_long(s, DMAR_FSTS_REG, 0, 0, 0x11UL); 1983 vtd_define_long(s, DMAR_FECTL_REG, 0x80000000UL, 0x80000000UL, 0); 1984 vtd_define_long(s, DMAR_FEDATA_REG, 0, 0x0000ffffUL, 0); 1985 vtd_define_long(s, DMAR_FEADDR_REG, 0, 0xfffffffcUL, 0); 1986 1987 /* Treated as RsvdZ when EIM in ECAP_REG is not supported 1988 * vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0xffffffffUL, 0); 1989 */ 1990 vtd_define_long(s, DMAR_FEUADDR_REG, 0, 0, 0); 1991 1992 /* Treated as RO for implementations that PLMR and PHMR fields reported 1993 * as Clear in the CAP_REG. 1994 * vtd_define_long(s, DMAR_PMEN_REG, 0, 0x80000000UL, 0); 1995 */ 1996 vtd_define_long(s, DMAR_PMEN_REG, 0, 0, 0); 1997 1998 vtd_define_quad(s, DMAR_IQH_REG, 0, 0, 0); 1999 vtd_define_quad(s, DMAR_IQT_REG, 0, 0x7fff0ULL, 0); 2000 vtd_define_quad(s, DMAR_IQA_REG, 0, 0xfffffffffffff007ULL, 0); 2001 vtd_define_long(s, DMAR_ICS_REG, 0, 0, 0x1UL); 2002 vtd_define_long(s, DMAR_IECTL_REG, 0x80000000UL, 0x80000000UL, 0); 2003 vtd_define_long(s, DMAR_IEDATA_REG, 0, 0xffffffffUL, 0); 2004 vtd_define_long(s, DMAR_IEADDR_REG, 0, 0xfffffffcUL, 0); 2005 /* Treadted as RsvdZ when EIM in ECAP_REG is not supported */ 2006 vtd_define_long(s, DMAR_IEUADDR_REG, 0, 0, 0); 2007 2008 /* IOTLB registers */ 2009 vtd_define_quad(s, DMAR_IOTLB_REG, 0, 0Xb003ffff00000000ULL, 0); 2010 vtd_define_quad(s, DMAR_IVA_REG, 0, 0xfffffffffffff07fULL, 0); 2011 vtd_define_quad_wo(s, DMAR_IVA_REG, 0xfffffffffffff07fULL); 2012 2013 /* Fault Recording Registers, 128-bit */ 2014 vtd_define_quad(s, DMAR_FRCD_REG_0_0, 0, 0, 0); 2015 vtd_define_quad(s, DMAR_FRCD_REG_0_2, 0, 0, 0x8000000000000000ULL); 2016 } 2017 2018 /* Should not reset address_spaces when reset because devices will still use 2019 * the address space they got at first (won't ask the bus again). 2020 */ 2021 static void vtd_reset(DeviceState *dev) 2022 { 2023 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 2024 2025 VTD_DPRINTF(GENERAL, ""); 2026 vtd_init(s); 2027 } 2028 2029 static void vtd_realize(DeviceState *dev, Error **errp) 2030 { 2031 IntelIOMMUState *s = INTEL_IOMMU_DEVICE(dev); 2032 2033 VTD_DPRINTF(GENERAL, ""); 2034 memset(s->vtd_as_by_bus_num, 0, sizeof(s->vtd_as_by_bus_num)); 2035 memory_region_init_io(&s->csrmem, OBJECT(s), &vtd_mem_ops, s, 2036 "intel_iommu", DMAR_REG_SIZE); 2037 sysbus_init_mmio(SYS_BUS_DEVICE(s), &s->csrmem); 2038 /* No corresponding destroy */ 2039 s->iotlb = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 2040 g_free, g_free); 2041 s->vtd_as_by_busptr = g_hash_table_new_full(vtd_uint64_hash, vtd_uint64_equal, 2042 g_free, g_free); 2043 vtd_init(s); 2044 } 2045 2046 static void vtd_class_init(ObjectClass *klass, void *data) 2047 { 2048 DeviceClass *dc = DEVICE_CLASS(klass); 2049 2050 dc->reset = vtd_reset; 2051 dc->realize = vtd_realize; 2052 dc->vmsd = &vtd_vmstate; 2053 dc->props = vtd_properties; 2054 } 2055 2056 static const TypeInfo vtd_info = { 2057 .name = TYPE_INTEL_IOMMU_DEVICE, 2058 .parent = TYPE_SYS_BUS_DEVICE, 2059 .instance_size = sizeof(IntelIOMMUState), 2060 .class_init = vtd_class_init, 2061 }; 2062 2063 static void vtd_register_types(void) 2064 { 2065 VTD_DPRINTF(GENERAL, ""); 2066 type_register_static(&vtd_info); 2067 } 2068 2069 type_init(vtd_register_types) 2070