1366f6083SPeter Grehan /*- 2c49761ddSPedro F. Giffuni * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3c49761ddSPedro F. Giffuni * 4366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 5366f6083SPeter Grehan * All rights reserved. 6366f6083SPeter Grehan * 7366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 8366f6083SPeter Grehan * modification, are permitted provided that the following conditions 9366f6083SPeter Grehan * are met: 10366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 12366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 13366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 14366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 15366f6083SPeter Grehan * 16366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 17366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 20366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26366f6083SPeter Grehan * SUCH DAMAGE. 27366f6083SPeter Grehan * 28366f6083SPeter Grehan * $FreeBSD$ 29366f6083SPeter Grehan */ 30366f6083SPeter Grehan 31366f6083SPeter Grehan #include <sys/cdefs.h> 32366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 33366f6083SPeter Grehan 34366f6083SPeter Grehan #include <sys/param.h> 35366f6083SPeter Grehan #include <sys/kernel.h> 36366f6083SPeter Grehan #include <sys/systm.h> 37366f6083SPeter Grehan #include <sys/malloc.h> 38366f6083SPeter Grehan 39366f6083SPeter Grehan #include <vm/vm.h> 40366f6083SPeter Grehan #include <vm/pmap.h> 41366f6083SPeter Grehan 42366f6083SPeter Grehan #include <dev/pci/pcireg.h> 43366f6083SPeter Grehan 44366f6083SPeter Grehan #include <machine/vmparam.h> 45f77e9829SNeel Natu #include <contrib/dev/acpica/include/acpi.h> 46366f6083SPeter Grehan 47366f6083SPeter Grehan #include "io/iommu.h" 48366f6083SPeter Grehan 49366f6083SPeter Grehan /* 50366f6083SPeter Grehan * Documented in the "Intel Virtualization Technology for Directed I/O", 51366f6083SPeter Grehan * Architecture Spec, September 2008. 52366f6083SPeter Grehan */ 53366f6083SPeter Grehan 54da761f3bSScott Long #define VTD_DRHD_INCLUDE_PCI_ALL(Flags) (((Flags) >> 0) & 0x1) 55da761f3bSScott Long 56366f6083SPeter Grehan /* Section 10.4 "Register Descriptions" */ 57366f6083SPeter Grehan struct vtdmap { 58366f6083SPeter Grehan volatile uint32_t version; 59366f6083SPeter Grehan volatile uint32_t res0; 60366f6083SPeter Grehan volatile uint64_t cap; 61366f6083SPeter Grehan volatile uint64_t ext_cap; 62366f6083SPeter Grehan volatile uint32_t gcr; 63366f6083SPeter Grehan volatile uint32_t gsr; 64366f6083SPeter Grehan volatile uint64_t rta; 65366f6083SPeter Grehan volatile uint64_t ccr; 66366f6083SPeter Grehan }; 67366f6083SPeter Grehan 68366f6083SPeter Grehan #define VTD_CAP_SAGAW(cap) (((cap) >> 8) & 0x1F) 69366f6083SPeter Grehan #define VTD_CAP_ND(cap) ((cap) & 0x7) 70366f6083SPeter Grehan #define VTD_CAP_CM(cap) (((cap) >> 7) & 0x1) 71366f6083SPeter Grehan #define VTD_CAP_SPS(cap) (((cap) >> 34) & 0xF) 72366f6083SPeter Grehan #define VTD_CAP_RWBF(cap) (((cap) >> 4) & 0x1) 73366f6083SPeter Grehan 74366f6083SPeter Grehan #define VTD_ECAP_DI(ecap) (((ecap) >> 2) & 0x1) 75366f6083SPeter Grehan #define VTD_ECAP_COHERENCY(ecap) ((ecap) & 0x1) 76366f6083SPeter Grehan #define VTD_ECAP_IRO(ecap) (((ecap) >> 8) & 0x3FF) 77366f6083SPeter Grehan 78366f6083SPeter Grehan #define VTD_GCR_WBF (1 << 27) 79366f6083SPeter Grehan #define VTD_GCR_SRTP (1 << 30) 807a22215cSEitan Adler #define VTD_GCR_TE (1U << 31) 81366f6083SPeter Grehan 82366f6083SPeter Grehan #define VTD_GSR_WBFS (1 << 27) 83366f6083SPeter Grehan #define VTD_GSR_RTPS (1 << 30) 847a22215cSEitan Adler #define VTD_GSR_TES (1U << 31) 85366f6083SPeter Grehan 86366f6083SPeter Grehan #define VTD_CCR_ICC (1UL << 63) /* invalidate context cache */ 87366f6083SPeter Grehan #define VTD_CCR_CIRG_GLOBAL (1UL << 61) /* global invalidation */ 88366f6083SPeter Grehan 89366f6083SPeter Grehan #define VTD_IIR_IVT (1UL << 63) /* invalidation IOTLB */ 90366f6083SPeter Grehan #define VTD_IIR_IIRG_GLOBAL (1ULL << 60) /* global IOTLB invalidation */ 91366f6083SPeter Grehan #define VTD_IIR_IIRG_DOMAIN (2ULL << 60) /* domain IOTLB invalidation */ 92366f6083SPeter Grehan #define VTD_IIR_IIRG_PAGE (3ULL << 60) /* page IOTLB invalidation */ 93366f6083SPeter Grehan #define VTD_IIR_DRAIN_READS (1ULL << 49) /* drain pending DMA reads */ 94366f6083SPeter Grehan #define VTD_IIR_DRAIN_WRITES (1ULL << 48) /* drain pending DMA writes */ 95366f6083SPeter Grehan #define VTD_IIR_DOMAIN_P 32 96366f6083SPeter Grehan 97366f6083SPeter Grehan #define VTD_ROOT_PRESENT 0x1 98366f6083SPeter Grehan #define VTD_CTX_PRESENT 0x1 99366f6083SPeter Grehan #define VTD_CTX_TT_ALL (1UL << 2) 100366f6083SPeter Grehan 101366f6083SPeter Grehan #define VTD_PTE_RD (1UL << 0) 102366f6083SPeter Grehan #define VTD_PTE_WR (1UL << 1) 103366f6083SPeter Grehan #define VTD_PTE_SUPERPAGE (1UL << 7) 104366f6083SPeter Grehan #define VTD_PTE_ADDR_M (0x000FFFFFFFFFF000UL) 105366f6083SPeter Grehan 106a8667250SRyan Stone #define VTD_RID2IDX(rid) (((rid) & 0xff) * 2) 107a8667250SRyan Stone 108366f6083SPeter Grehan struct domain { 109366f6083SPeter Grehan uint64_t *ptp; /* first level page table page */ 110366f6083SPeter Grehan int pt_levels; /* number of page table levels */ 111366f6083SPeter Grehan int addrwidth; /* 'AW' field in context entry */ 112366f6083SPeter Grehan int spsmask; /* supported super page sizes */ 113366f6083SPeter Grehan u_int id; /* domain id */ 114366f6083SPeter Grehan vm_paddr_t maxaddr; /* highest address to be mapped */ 115366f6083SPeter Grehan SLIST_ENTRY(domain) next; 116366f6083SPeter Grehan }; 117366f6083SPeter Grehan 118366f6083SPeter Grehan static SLIST_HEAD(, domain) domhead; 119366f6083SPeter Grehan 120366f6083SPeter Grehan #define DRHD_MAX_UNITS 8 121da761f3bSScott Long static ACPI_DMAR_HARDWARE_UNIT *drhds[DRHD_MAX_UNITS]; 122366f6083SPeter Grehan static int drhd_num; 123366f6083SPeter Grehan static struct vtdmap *vtdmaps[DRHD_MAX_UNITS]; 124366f6083SPeter Grehan static int max_domains; 125366f6083SPeter Grehan typedef int (*drhd_ident_func_t)(void); 126366f6083SPeter Grehan 127366f6083SPeter Grehan static uint64_t root_table[PAGE_SIZE / sizeof(uint64_t)] __aligned(4096); 128366f6083SPeter Grehan static uint64_t ctx_tables[256][PAGE_SIZE / sizeof(uint64_t)] __aligned(4096); 129366f6083SPeter Grehan 130366f6083SPeter Grehan static MALLOC_DEFINE(M_VTD, "vtd", "vtd"); 131366f6083SPeter Grehan 132366f6083SPeter Grehan static int 133366f6083SPeter Grehan vtd_max_domains(struct vtdmap *vtdmap) 134366f6083SPeter Grehan { 135366f6083SPeter Grehan int nd; 136366f6083SPeter Grehan 137366f6083SPeter Grehan nd = VTD_CAP_ND(vtdmap->cap); 138366f6083SPeter Grehan 139366f6083SPeter Grehan switch (nd) { 140366f6083SPeter Grehan case 0: 141366f6083SPeter Grehan return (16); 142366f6083SPeter Grehan case 1: 143366f6083SPeter Grehan return (64); 144366f6083SPeter Grehan case 2: 145366f6083SPeter Grehan return (256); 146366f6083SPeter Grehan case 3: 147366f6083SPeter Grehan return (1024); 148366f6083SPeter Grehan case 4: 149366f6083SPeter Grehan return (4 * 1024); 150366f6083SPeter Grehan case 5: 151366f6083SPeter Grehan return (16 * 1024); 152366f6083SPeter Grehan case 6: 153366f6083SPeter Grehan return (64 * 1024); 154366f6083SPeter Grehan default: 155366f6083SPeter Grehan panic("vtd_max_domains: invalid value of nd (0x%0x)", nd); 156366f6083SPeter Grehan } 157366f6083SPeter Grehan } 158366f6083SPeter Grehan 159366f6083SPeter Grehan static u_int 160366f6083SPeter Grehan domain_id(void) 161366f6083SPeter Grehan { 162366f6083SPeter Grehan u_int id; 163366f6083SPeter Grehan struct domain *dom; 164366f6083SPeter Grehan 165366f6083SPeter Grehan /* Skip domain id 0 - it is reserved when Caching Mode field is set */ 166366f6083SPeter Grehan for (id = 1; id < max_domains; id++) { 167366f6083SPeter Grehan SLIST_FOREACH(dom, &domhead, next) { 168366f6083SPeter Grehan if (dom->id == id) 169366f6083SPeter Grehan break; 170366f6083SPeter Grehan } 171366f6083SPeter Grehan if (dom == NULL) 172366f6083SPeter Grehan break; /* found it */ 173366f6083SPeter Grehan } 174366f6083SPeter Grehan 175366f6083SPeter Grehan if (id >= max_domains) 176366f6083SPeter Grehan panic("domain ids exhausted"); 177366f6083SPeter Grehan 178366f6083SPeter Grehan return (id); 179366f6083SPeter Grehan } 180366f6083SPeter Grehan 181da761f3bSScott Long static struct vtdmap * 182da761f3bSScott Long vtd_device_scope(uint16_t rid) 183da761f3bSScott Long { 184da761f3bSScott Long int i, remaining, pathremaining; 185da761f3bSScott Long char *end, *pathend; 186da761f3bSScott Long struct vtdmap *vtdmap; 187da761f3bSScott Long ACPI_DMAR_HARDWARE_UNIT *drhd; 188da761f3bSScott Long ACPI_DMAR_DEVICE_SCOPE *device_scope; 189da761f3bSScott Long ACPI_DMAR_PCI_PATH *path; 190da761f3bSScott Long 191da761f3bSScott Long for (i = 0; i < drhd_num; i++) { 192da761f3bSScott Long drhd = drhds[i]; 193da761f3bSScott Long 194da761f3bSScott Long if (VTD_DRHD_INCLUDE_PCI_ALL(drhd->Flags)) { 195da761f3bSScott Long /* 196da761f3bSScott Long * From Intel VT-d arch spec, version 3.0: 197da761f3bSScott Long * If a DRHD structure with INCLUDE_PCI_ALL flag Set is reported 198da761f3bSScott Long * for a Segment, it must be enumerated by BIOS after all other 199da761f3bSScott Long * DRHD structures for the same Segment. 200da761f3bSScott Long */ 201da761f3bSScott Long vtdmap = vtdmaps[i]; 202da761f3bSScott Long return(vtdmap); 203da761f3bSScott Long } 204da761f3bSScott Long 205da761f3bSScott Long end = (char *)drhd + drhd->Header.Length; 206da761f3bSScott Long remaining = drhd->Header.Length - sizeof(ACPI_DMAR_HARDWARE_UNIT); 207da761f3bSScott Long while (remaining > sizeof(ACPI_DMAR_DEVICE_SCOPE)) { 208da761f3bSScott Long device_scope = (ACPI_DMAR_DEVICE_SCOPE *)(end - remaining); 209da761f3bSScott Long remaining -= device_scope->Length; 210da761f3bSScott Long 211da761f3bSScott Long switch (device_scope->EntryType){ 212da761f3bSScott Long /* 0x01 and 0x02 are PCI device entries */ 213da761f3bSScott Long case 0x01: 214da761f3bSScott Long case 0x02: 215da761f3bSScott Long break; 216da761f3bSScott Long default: 217da761f3bSScott Long continue; 218da761f3bSScott Long } 219da761f3bSScott Long 220da761f3bSScott Long if (PCI_RID2BUS(rid) != device_scope->Bus) 221da761f3bSScott Long continue; 222da761f3bSScott Long 223da761f3bSScott Long pathend = (char *)device_scope + device_scope->Length; 224da761f3bSScott Long pathremaining = device_scope->Length - sizeof(ACPI_DMAR_DEVICE_SCOPE); 225da761f3bSScott Long while (pathremaining >= sizeof(ACPI_DMAR_PCI_PATH)) { 226da761f3bSScott Long path = (ACPI_DMAR_PCI_PATH *)(pathend - pathremaining); 227da761f3bSScott Long pathremaining -= sizeof(ACPI_DMAR_PCI_PATH); 228da761f3bSScott Long 229da761f3bSScott Long if (PCI_RID2SLOT(rid) != path->Device) 230da761f3bSScott Long continue; 231da761f3bSScott Long if (PCI_RID2FUNC(rid) != path->Function) 232da761f3bSScott Long continue; 233da761f3bSScott Long 234da761f3bSScott Long vtdmap = vtdmaps[i]; 235da761f3bSScott Long return (vtdmap); 236da761f3bSScott Long } 237da761f3bSScott Long } 238da761f3bSScott Long } 239da761f3bSScott Long 240da761f3bSScott Long /* No matching scope */ 241da761f3bSScott Long return (NULL); 242da761f3bSScott Long } 243da761f3bSScott Long 244366f6083SPeter Grehan static void 245366f6083SPeter Grehan vtd_wbflush(struct vtdmap *vtdmap) 246366f6083SPeter Grehan { 247366f6083SPeter Grehan 248366f6083SPeter Grehan if (VTD_ECAP_COHERENCY(vtdmap->ext_cap) == 0) 249366f6083SPeter Grehan pmap_invalidate_cache(); 250366f6083SPeter Grehan 251366f6083SPeter Grehan if (VTD_CAP_RWBF(vtdmap->cap)) { 252366f6083SPeter Grehan vtdmap->gcr = VTD_GCR_WBF; 253366f6083SPeter Grehan while ((vtdmap->gsr & VTD_GSR_WBFS) != 0) 254366f6083SPeter Grehan ; 255366f6083SPeter Grehan } 256366f6083SPeter Grehan } 257366f6083SPeter Grehan 258366f6083SPeter Grehan static void 259366f6083SPeter Grehan vtd_ctx_global_invalidate(struct vtdmap *vtdmap) 260366f6083SPeter Grehan { 261366f6083SPeter Grehan 262366f6083SPeter Grehan vtdmap->ccr = VTD_CCR_ICC | VTD_CCR_CIRG_GLOBAL; 263366f6083SPeter Grehan while ((vtdmap->ccr & VTD_CCR_ICC) != 0) 264366f6083SPeter Grehan ; 265366f6083SPeter Grehan } 266366f6083SPeter Grehan 267366f6083SPeter Grehan static void 268366f6083SPeter Grehan vtd_iotlb_global_invalidate(struct vtdmap *vtdmap) 269366f6083SPeter Grehan { 270366f6083SPeter Grehan int offset; 271366f6083SPeter Grehan volatile uint64_t *iotlb_reg, val; 272366f6083SPeter Grehan 273366f6083SPeter Grehan vtd_wbflush(vtdmap); 274366f6083SPeter Grehan 275366f6083SPeter Grehan offset = VTD_ECAP_IRO(vtdmap->ext_cap) * 16; 276366f6083SPeter Grehan iotlb_reg = (volatile uint64_t *)((caddr_t)vtdmap + offset + 8); 277366f6083SPeter Grehan 278366f6083SPeter Grehan *iotlb_reg = VTD_IIR_IVT | VTD_IIR_IIRG_GLOBAL | 279366f6083SPeter Grehan VTD_IIR_DRAIN_READS | VTD_IIR_DRAIN_WRITES; 280366f6083SPeter Grehan 281366f6083SPeter Grehan while (1) { 282366f6083SPeter Grehan val = *iotlb_reg; 283366f6083SPeter Grehan if ((val & VTD_IIR_IVT) == 0) 284366f6083SPeter Grehan break; 285366f6083SPeter Grehan } 286366f6083SPeter Grehan } 287366f6083SPeter Grehan 288366f6083SPeter Grehan static void 289366f6083SPeter Grehan vtd_translation_enable(struct vtdmap *vtdmap) 290366f6083SPeter Grehan { 291366f6083SPeter Grehan 292366f6083SPeter Grehan vtdmap->gcr = VTD_GCR_TE; 293366f6083SPeter Grehan while ((vtdmap->gsr & VTD_GSR_TES) == 0) 294366f6083SPeter Grehan ; 295366f6083SPeter Grehan } 296366f6083SPeter Grehan 297366f6083SPeter Grehan static void 298366f6083SPeter Grehan vtd_translation_disable(struct vtdmap *vtdmap) 299366f6083SPeter Grehan { 300366f6083SPeter Grehan 301366f6083SPeter Grehan vtdmap->gcr = 0; 302366f6083SPeter Grehan while ((vtdmap->gsr & VTD_GSR_TES) != 0) 303366f6083SPeter Grehan ; 304366f6083SPeter Grehan } 305366f6083SPeter Grehan 306366f6083SPeter Grehan static int 307366f6083SPeter Grehan vtd_init(void) 308366f6083SPeter Grehan { 309da761f3bSScott Long int i, units, remaining, tmp; 310366f6083SPeter Grehan struct vtdmap *vtdmap; 311366f6083SPeter Grehan vm_paddr_t ctx_paddr; 312f77e9829SNeel Natu char *end, envname[32]; 313f77e9829SNeel Natu unsigned long mapaddr; 314f77e9829SNeel Natu ACPI_STATUS status; 315f77e9829SNeel Natu ACPI_TABLE_DMAR *dmar; 316f77e9829SNeel Natu ACPI_DMAR_HEADER *hdr; 317f77e9829SNeel Natu ACPI_DMAR_HARDWARE_UNIT *drhd; 318366f6083SPeter Grehan 319f77e9829SNeel Natu /* 320f77e9829SNeel Natu * Allow the user to override the ACPI DMAR table by specifying the 321f77e9829SNeel Natu * physical address of each remapping unit. 322f77e9829SNeel Natu * 323f77e9829SNeel Natu * The following example specifies two remapping units at 324f77e9829SNeel Natu * physical addresses 0xfed90000 and 0xfeda0000 respectively. 325f77e9829SNeel Natu * set vtd.regmap.0.addr=0xfed90000 326f77e9829SNeel Natu * set vtd.regmap.1.addr=0xfeda0000 327f77e9829SNeel Natu */ 328f77e9829SNeel Natu for (units = 0; units < DRHD_MAX_UNITS; units++) { 329f77e9829SNeel Natu snprintf(envname, sizeof(envname), "vtd.regmap.%d.addr", units); 330f77e9829SNeel Natu if (getenv_ulong(envname, &mapaddr) == 0) 331366f6083SPeter Grehan break; 332f77e9829SNeel Natu vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(mapaddr); 333f77e9829SNeel Natu } 334f77e9829SNeel Natu 335f77e9829SNeel Natu if (units > 0) 336f77e9829SNeel Natu goto skip_dmar; 337f77e9829SNeel Natu 338f77e9829SNeel Natu /* Search for DMAR table. */ 339f77e9829SNeel Natu status = AcpiGetTable(ACPI_SIG_DMAR, 0, (ACPI_TABLE_HEADER **)&dmar); 340f77e9829SNeel Natu if (ACPI_FAILURE(status)) 341f77e9829SNeel Natu return (ENXIO); 342f77e9829SNeel Natu 343f77e9829SNeel Natu end = (char *)dmar + dmar->Header.Length; 344f77e9829SNeel Natu remaining = dmar->Header.Length - sizeof(ACPI_TABLE_DMAR); 345f77e9829SNeel Natu while (remaining > sizeof(ACPI_DMAR_HEADER)) { 346f77e9829SNeel Natu hdr = (ACPI_DMAR_HEADER *)(end - remaining); 347f77e9829SNeel Natu if (hdr->Length > remaining) 348f77e9829SNeel Natu break; 349f77e9829SNeel Natu /* 350f77e9829SNeel Natu * From Intel VT-d arch spec, version 1.3: 351f77e9829SNeel Natu * BIOS implementations must report mapping structures 352f77e9829SNeel Natu * in numerical order, i.e. All remapping structures of 353f77e9829SNeel Natu * type 0 (DRHD) enumerated before remapping structures of 354f77e9829SNeel Natu * type 1 (RMRR) and so forth. 355f77e9829SNeel Natu */ 356f77e9829SNeel Natu if (hdr->Type != ACPI_DMAR_TYPE_HARDWARE_UNIT) 357f77e9829SNeel Natu break; 358f77e9829SNeel Natu 359f77e9829SNeel Natu drhd = (ACPI_DMAR_HARDWARE_UNIT *)hdr; 360da761f3bSScott Long drhds[units] = drhd; 361da761f3bSScott Long vtdmaps[units] = (struct vtdmap *)PHYS_TO_DMAP(drhd->Address); 362da761f3bSScott Long if (++units >= DRHD_MAX_UNITS) 363f77e9829SNeel Natu break; 364f77e9829SNeel Natu remaining -= hdr->Length; 365366f6083SPeter Grehan } 366366f6083SPeter Grehan 367366f6083SPeter Grehan if (units <= 0) 368366f6083SPeter Grehan return (ENXIO); 369366f6083SPeter Grehan 370f77e9829SNeel Natu skip_dmar: 371366f6083SPeter Grehan drhd_num = units; 372da761f3bSScott Long 373da761f3bSScott Long max_domains = 64 * 1024; /* maximum valid value */ 374da761f3bSScott Long for (i = 0; i < drhd_num; i++){ 375da761f3bSScott Long vtdmap = vtdmaps[i]; 376366f6083SPeter Grehan 377366f6083SPeter Grehan if (VTD_CAP_CM(vtdmap->cap) != 0) 378366f6083SPeter Grehan panic("vtd_init: invalid caching mode"); 379366f6083SPeter Grehan 380da761f3bSScott Long /* take most compatible (minimum) value */ 381da761f3bSScott Long if ((tmp = vtd_max_domains(vtdmap)) < max_domains) 382da761f3bSScott Long max_domains = tmp; 383da761f3bSScott Long } 384366f6083SPeter Grehan 385366f6083SPeter Grehan /* 386366f6083SPeter Grehan * Set up the root-table to point to the context-entry tables 387366f6083SPeter Grehan */ 388366f6083SPeter Grehan for (i = 0; i < 256; i++) { 389366f6083SPeter Grehan ctx_paddr = vtophys(ctx_tables[i]); 390366f6083SPeter Grehan if (ctx_paddr & PAGE_MASK) 391366f6083SPeter Grehan panic("ctx table (0x%0lx) not page aligned", ctx_paddr); 392366f6083SPeter Grehan 393366f6083SPeter Grehan root_table[i * 2] = ctx_paddr | VTD_ROOT_PRESENT; 394366f6083SPeter Grehan } 395366f6083SPeter Grehan 396366f6083SPeter Grehan return (0); 397366f6083SPeter Grehan } 398366f6083SPeter Grehan 399366f6083SPeter Grehan static void 400366f6083SPeter Grehan vtd_cleanup(void) 401366f6083SPeter Grehan { 402366f6083SPeter Grehan } 403366f6083SPeter Grehan 404366f6083SPeter Grehan static void 405366f6083SPeter Grehan vtd_enable(void) 406366f6083SPeter Grehan { 407366f6083SPeter Grehan int i; 408366f6083SPeter Grehan struct vtdmap *vtdmap; 409366f6083SPeter Grehan 410366f6083SPeter Grehan for (i = 0; i < drhd_num; i++) { 411366f6083SPeter Grehan vtdmap = vtdmaps[i]; 412366f6083SPeter Grehan vtd_wbflush(vtdmap); 413366f6083SPeter Grehan 414366f6083SPeter Grehan /* Update the root table address */ 415366f6083SPeter Grehan vtdmap->rta = vtophys(root_table); 416366f6083SPeter Grehan vtdmap->gcr = VTD_GCR_SRTP; 417366f6083SPeter Grehan while ((vtdmap->gsr & VTD_GSR_RTPS) == 0) 418366f6083SPeter Grehan ; 419366f6083SPeter Grehan 420366f6083SPeter Grehan vtd_ctx_global_invalidate(vtdmap); 421366f6083SPeter Grehan vtd_iotlb_global_invalidate(vtdmap); 422366f6083SPeter Grehan 423366f6083SPeter Grehan vtd_translation_enable(vtdmap); 424366f6083SPeter Grehan } 425366f6083SPeter Grehan } 426366f6083SPeter Grehan 427366f6083SPeter Grehan static void 428366f6083SPeter Grehan vtd_disable(void) 429366f6083SPeter Grehan { 430366f6083SPeter Grehan int i; 431366f6083SPeter Grehan struct vtdmap *vtdmap; 432366f6083SPeter Grehan 433366f6083SPeter Grehan for (i = 0; i < drhd_num; i++) { 434366f6083SPeter Grehan vtdmap = vtdmaps[i]; 435366f6083SPeter Grehan vtd_translation_disable(vtdmap); 436366f6083SPeter Grehan } 437366f6083SPeter Grehan } 438366f6083SPeter Grehan 439366f6083SPeter Grehan static void 440a8667250SRyan Stone vtd_add_device(void *arg, uint16_t rid) 441366f6083SPeter Grehan { 442366f6083SPeter Grehan int idx; 443366f6083SPeter Grehan uint64_t *ctxp; 444366f6083SPeter Grehan struct domain *dom = arg; 445366f6083SPeter Grehan vm_paddr_t pt_paddr; 446366f6083SPeter Grehan struct vtdmap *vtdmap; 447a8667250SRyan Stone uint8_t bus; 448366f6083SPeter Grehan 44927029bc0SRobert Wing KASSERT(dom != NULL, ("domain is NULL")); 45027029bc0SRobert Wing 451a8667250SRyan Stone bus = PCI_RID2BUS(rid); 452366f6083SPeter Grehan ctxp = ctx_tables[bus]; 453366f6083SPeter Grehan pt_paddr = vtophys(dom->ptp); 454a8667250SRyan Stone idx = VTD_RID2IDX(rid); 455366f6083SPeter Grehan 456366f6083SPeter Grehan if (ctxp[idx] & VTD_CTX_PRESENT) { 457a8667250SRyan Stone panic("vtd_add_device: device %x is already owned by " 458a8667250SRyan Stone "domain %d", rid, 459366f6083SPeter Grehan (uint16_t)(ctxp[idx + 1] >> 8)); 460366f6083SPeter Grehan } 461366f6083SPeter Grehan 462da761f3bSScott Long if ((vtdmap = vtd_device_scope(rid)) == NULL) 463da761f3bSScott Long panic("vtd_add_device: device %x is not in scope for " 464da761f3bSScott Long "any DMA remapping unit", rid); 465da761f3bSScott Long 466366f6083SPeter Grehan /* 467366f6083SPeter Grehan * Order is important. The 'present' bit is set only after all fields 468366f6083SPeter Grehan * of the context pointer are initialized. 469366f6083SPeter Grehan */ 470366f6083SPeter Grehan ctxp[idx + 1] = dom->addrwidth | (dom->id << 8); 471366f6083SPeter Grehan 472366f6083SPeter Grehan if (VTD_ECAP_DI(vtdmap->ext_cap)) 473366f6083SPeter Grehan ctxp[idx] = VTD_CTX_TT_ALL; 474366f6083SPeter Grehan else 475366f6083SPeter Grehan ctxp[idx] = 0; 476366f6083SPeter Grehan 477366f6083SPeter Grehan ctxp[idx] |= pt_paddr | VTD_CTX_PRESENT; 478366f6083SPeter Grehan 479366f6083SPeter Grehan /* 480366f6083SPeter Grehan * 'Not Present' entries are not cached in either the Context Cache 481366f6083SPeter Grehan * or in the IOTLB, so there is no need to invalidate either of them. 482366f6083SPeter Grehan */ 483366f6083SPeter Grehan } 484366f6083SPeter Grehan 485366f6083SPeter Grehan static void 486a8667250SRyan Stone vtd_remove_device(void *arg, uint16_t rid) 487366f6083SPeter Grehan { 488366f6083SPeter Grehan int i, idx; 489366f6083SPeter Grehan uint64_t *ctxp; 490366f6083SPeter Grehan struct vtdmap *vtdmap; 491a8667250SRyan Stone uint8_t bus; 492366f6083SPeter Grehan 493a8667250SRyan Stone bus = PCI_RID2BUS(rid); 494366f6083SPeter Grehan ctxp = ctx_tables[bus]; 495a8667250SRyan Stone idx = VTD_RID2IDX(rid); 496366f6083SPeter Grehan 497366f6083SPeter Grehan /* 498366f6083SPeter Grehan * Order is important. The 'present' bit is must be cleared first. 499366f6083SPeter Grehan */ 500366f6083SPeter Grehan ctxp[idx] = 0; 501366f6083SPeter Grehan ctxp[idx + 1] = 0; 502366f6083SPeter Grehan 503366f6083SPeter Grehan /* 504366f6083SPeter Grehan * Invalidate the Context Cache and the IOTLB. 505366f6083SPeter Grehan * 506366f6083SPeter Grehan * XXX use device-selective invalidation for Context Cache 507366f6083SPeter Grehan * XXX use domain-selective invalidation for IOTLB 508366f6083SPeter Grehan */ 509366f6083SPeter Grehan for (i = 0; i < drhd_num; i++) { 510366f6083SPeter Grehan vtdmap = vtdmaps[i]; 511366f6083SPeter Grehan vtd_ctx_global_invalidate(vtdmap); 512366f6083SPeter Grehan vtd_iotlb_global_invalidate(vtdmap); 513366f6083SPeter Grehan } 514366f6083SPeter Grehan } 515366f6083SPeter Grehan 5167ce04d0aSNeel Natu #define CREATE_MAPPING 0 5177ce04d0aSNeel Natu #define REMOVE_MAPPING 1 5187ce04d0aSNeel Natu 519366f6083SPeter Grehan static uint64_t 5207ce04d0aSNeel Natu vtd_update_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len, 5217ce04d0aSNeel Natu int remove) 522366f6083SPeter Grehan { 523366f6083SPeter Grehan struct domain *dom; 524366f6083SPeter Grehan int i, spshift, ptpshift, ptpindex, nlevels; 525366f6083SPeter Grehan uint64_t spsize, *ptp; 526366f6083SPeter Grehan 527366f6083SPeter Grehan dom = arg; 528366f6083SPeter Grehan ptpindex = 0; 529366f6083SPeter Grehan ptpshift = 0; 530366f6083SPeter Grehan 531477867a0SNeel Natu KASSERT(gpa + len > gpa, ("%s: invalid gpa range %#lx/%#lx", __func__, 532477867a0SNeel Natu gpa, len)); 533477867a0SNeel Natu KASSERT(gpa + len <= dom->maxaddr, ("%s: gpa range %#lx/%#lx beyond " 534477867a0SNeel Natu "domain maxaddr %#lx", __func__, gpa, len, dom->maxaddr)); 535477867a0SNeel Natu 536366f6083SPeter Grehan if (gpa & PAGE_MASK) 537366f6083SPeter Grehan panic("vtd_create_mapping: unaligned gpa 0x%0lx", gpa); 538366f6083SPeter Grehan 539366f6083SPeter Grehan if (hpa & PAGE_MASK) 540366f6083SPeter Grehan panic("vtd_create_mapping: unaligned hpa 0x%0lx", hpa); 541366f6083SPeter Grehan 542366f6083SPeter Grehan if (len & PAGE_MASK) 543366f6083SPeter Grehan panic("vtd_create_mapping: unaligned len 0x%0lx", len); 544366f6083SPeter Grehan 545366f6083SPeter Grehan /* 546500eb14aSPedro F. Giffuni * Compute the size of the mapping that we can accommodate. 547366f6083SPeter Grehan * 548366f6083SPeter Grehan * This is based on three factors: 549366f6083SPeter Grehan * - supported super page size 550366f6083SPeter Grehan * - alignment of the region starting at 'gpa' and 'hpa' 551366f6083SPeter Grehan * - length of the region 'len' 552366f6083SPeter Grehan */ 553366f6083SPeter Grehan spshift = 48; 554366f6083SPeter Grehan for (i = 3; i >= 0; i--) { 555366f6083SPeter Grehan spsize = 1UL << spshift; 556366f6083SPeter Grehan if ((dom->spsmask & (1 << i)) != 0 && 557366f6083SPeter Grehan (gpa & (spsize - 1)) == 0 && 558366f6083SPeter Grehan (hpa & (spsize - 1)) == 0 && 559366f6083SPeter Grehan (len >= spsize)) { 560366f6083SPeter Grehan break; 561366f6083SPeter Grehan } 562366f6083SPeter Grehan spshift -= 9; 563366f6083SPeter Grehan } 564366f6083SPeter Grehan 565366f6083SPeter Grehan ptp = dom->ptp; 566366f6083SPeter Grehan nlevels = dom->pt_levels; 567366f6083SPeter Grehan while (--nlevels >= 0) { 568366f6083SPeter Grehan ptpshift = 12 + nlevels * 9; 569366f6083SPeter Grehan ptpindex = (gpa >> ptpshift) & 0x1FF; 570366f6083SPeter Grehan 571366f6083SPeter Grehan /* We have reached the leaf mapping */ 572366f6083SPeter Grehan if (spshift >= ptpshift) { 573366f6083SPeter Grehan break; 574366f6083SPeter Grehan } 575366f6083SPeter Grehan 576366f6083SPeter Grehan /* 577366f6083SPeter Grehan * We are working on a non-leaf page table page. 578366f6083SPeter Grehan * 579366f6083SPeter Grehan * Create a downstream page table page if necessary and point 580366f6083SPeter Grehan * to it from the current page table. 581366f6083SPeter Grehan */ 582366f6083SPeter Grehan if (ptp[ptpindex] == 0) { 583366f6083SPeter Grehan void *nlp = malloc(PAGE_SIZE, M_VTD, M_WAITOK | M_ZERO); 584366f6083SPeter Grehan ptp[ptpindex] = vtophys(nlp)| VTD_PTE_RD | VTD_PTE_WR; 585366f6083SPeter Grehan } 586366f6083SPeter Grehan 587366f6083SPeter Grehan ptp = (uint64_t *)PHYS_TO_DMAP(ptp[ptpindex] & VTD_PTE_ADDR_M); 588366f6083SPeter Grehan } 589366f6083SPeter Grehan 590366f6083SPeter Grehan if ((gpa & ((1UL << ptpshift) - 1)) != 0) 591366f6083SPeter Grehan panic("gpa 0x%lx and ptpshift %d mismatch", gpa, ptpshift); 592366f6083SPeter Grehan 593366f6083SPeter Grehan /* 5947ce04d0aSNeel Natu * Update the 'gpa' -> 'hpa' mapping 595366f6083SPeter Grehan */ 5967ce04d0aSNeel Natu if (remove) { 5977ce04d0aSNeel Natu ptp[ptpindex] = 0; 5987ce04d0aSNeel Natu } else { 599366f6083SPeter Grehan ptp[ptpindex] = hpa | VTD_PTE_RD | VTD_PTE_WR; 600366f6083SPeter Grehan 601366f6083SPeter Grehan if (nlevels > 0) 602366f6083SPeter Grehan ptp[ptpindex] |= VTD_PTE_SUPERPAGE; 6037ce04d0aSNeel Natu } 604366f6083SPeter Grehan 605366f6083SPeter Grehan return (1UL << ptpshift); 606366f6083SPeter Grehan } 607366f6083SPeter Grehan 6087ce04d0aSNeel Natu static uint64_t 6097ce04d0aSNeel Natu vtd_create_mapping(void *arg, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 6107ce04d0aSNeel Natu { 6117ce04d0aSNeel Natu 6127ce04d0aSNeel Natu return (vtd_update_mapping(arg, gpa, hpa, len, CREATE_MAPPING)); 6137ce04d0aSNeel Natu } 6147ce04d0aSNeel Natu 6157ce04d0aSNeel Natu static uint64_t 6167ce04d0aSNeel Natu vtd_remove_mapping(void *arg, vm_paddr_t gpa, uint64_t len) 6177ce04d0aSNeel Natu { 6187ce04d0aSNeel Natu 6197ce04d0aSNeel Natu return (vtd_update_mapping(arg, gpa, 0, len, REMOVE_MAPPING)); 6207ce04d0aSNeel Natu } 6217ce04d0aSNeel Natu 6227ce04d0aSNeel Natu static void 6237ce04d0aSNeel Natu vtd_invalidate_tlb(void *dom) 6247ce04d0aSNeel Natu { 6257ce04d0aSNeel Natu int i; 6267ce04d0aSNeel Natu struct vtdmap *vtdmap; 6277ce04d0aSNeel Natu 6287ce04d0aSNeel Natu /* 6297ce04d0aSNeel Natu * Invalidate the IOTLB. 6307ce04d0aSNeel Natu * XXX use domain-selective invalidation for IOTLB 6317ce04d0aSNeel Natu */ 6327ce04d0aSNeel Natu for (i = 0; i < drhd_num; i++) { 6337ce04d0aSNeel Natu vtdmap = vtdmaps[i]; 6347ce04d0aSNeel Natu vtd_iotlb_global_invalidate(vtdmap); 6357ce04d0aSNeel Natu } 6367ce04d0aSNeel Natu } 6377ce04d0aSNeel Natu 638366f6083SPeter Grehan static void * 639366f6083SPeter Grehan vtd_create_domain(vm_paddr_t maxaddr) 640366f6083SPeter Grehan { 641366f6083SPeter Grehan struct domain *dom; 642366f6083SPeter Grehan vm_paddr_t addr; 643366f6083SPeter Grehan int tmp, i, gaw, agaw, sagaw, res, pt_levels, addrwidth; 644366f6083SPeter Grehan struct vtdmap *vtdmap; 645366f6083SPeter Grehan 646366f6083SPeter Grehan if (drhd_num <= 0) 647366f6083SPeter Grehan panic("vtd_create_domain: no dma remapping hardware available"); 648366f6083SPeter Grehan 649366f6083SPeter Grehan /* 650366f6083SPeter Grehan * Calculate AGAW. 651366f6083SPeter Grehan * Section 3.4.2 "Adjusted Guest Address Width", Architecture Spec. 652366f6083SPeter Grehan */ 653366f6083SPeter Grehan addr = 0; 654366f6083SPeter Grehan for (gaw = 0; addr < maxaddr; gaw++) 655366f6083SPeter Grehan addr = 1ULL << gaw; 656366f6083SPeter Grehan 657366f6083SPeter Grehan res = (gaw - 12) % 9; 658366f6083SPeter Grehan if (res == 0) 659366f6083SPeter Grehan agaw = gaw; 660366f6083SPeter Grehan else 661366f6083SPeter Grehan agaw = gaw + 9 - res; 662366f6083SPeter Grehan 663366f6083SPeter Grehan if (agaw > 64) 664366f6083SPeter Grehan agaw = 64; 665366f6083SPeter Grehan 666366f6083SPeter Grehan /* 667366f6083SPeter Grehan * Select the smallest Supported AGAW and the corresponding number 668366f6083SPeter Grehan * of page table levels. 669366f6083SPeter Grehan */ 670366f6083SPeter Grehan pt_levels = 2; 671366f6083SPeter Grehan sagaw = 30; 672366f6083SPeter Grehan addrwidth = 0; 673da761f3bSScott Long 674da761f3bSScott Long tmp = ~0; 675da761f3bSScott Long for (i = 0; i < drhd_num; i++) { 676da761f3bSScott Long vtdmap = vtdmaps[i]; 677da761f3bSScott Long /* take most compatible value */ 678da761f3bSScott Long tmp &= VTD_CAP_SAGAW(vtdmap->cap); 679da761f3bSScott Long } 680da761f3bSScott Long 681366f6083SPeter Grehan for (i = 0; i < 5; i++) { 682366f6083SPeter Grehan if ((tmp & (1 << i)) != 0 && sagaw >= agaw) 683366f6083SPeter Grehan break; 684366f6083SPeter Grehan pt_levels++; 685366f6083SPeter Grehan addrwidth++; 686366f6083SPeter Grehan sagaw += 9; 687366f6083SPeter Grehan if (sagaw > 64) 688366f6083SPeter Grehan sagaw = 64; 689366f6083SPeter Grehan } 690366f6083SPeter Grehan 691366f6083SPeter Grehan if (i >= 5) { 692da761f3bSScott Long panic("vtd_create_domain: SAGAW 0x%x does not support AGAW %d", 693da761f3bSScott Long tmp, agaw); 694366f6083SPeter Grehan } 695366f6083SPeter Grehan 696366f6083SPeter Grehan dom = malloc(sizeof(struct domain), M_VTD, M_ZERO | M_WAITOK); 697366f6083SPeter Grehan dom->pt_levels = pt_levels; 698366f6083SPeter Grehan dom->addrwidth = addrwidth; 699366f6083SPeter Grehan dom->id = domain_id(); 700366f6083SPeter Grehan dom->maxaddr = maxaddr; 701366f6083SPeter Grehan dom->ptp = malloc(PAGE_SIZE, M_VTD, M_ZERO | M_WAITOK); 702366f6083SPeter Grehan if ((uintptr_t)dom->ptp & PAGE_MASK) 703366f6083SPeter Grehan panic("vtd_create_domain: ptp (%p) not page aligned", dom->ptp); 704366f6083SPeter Grehan 705b98940e5SNeel Natu #ifdef notyet 706b98940e5SNeel Natu /* 707b98940e5SNeel Natu * XXX superpage mappings for the iommu do not work correctly. 708b98940e5SNeel Natu * 709b98940e5SNeel Natu * By default all physical memory is mapped into the host_domain. 710b98940e5SNeel Natu * When a VM is allocated wired memory the pages belonging to it 711b98940e5SNeel Natu * are removed from the host_domain and added to the vm's domain. 712b98940e5SNeel Natu * 713b98940e5SNeel Natu * If the page being removed was mapped using a superpage mapping 714b98940e5SNeel Natu * in the host_domain then we need to demote the mapping before 715b98940e5SNeel Natu * removing the page. 716b98940e5SNeel Natu * 717b98940e5SNeel Natu * There is not any code to deal with the demotion at the moment 718b98940e5SNeel Natu * so we disable superpage mappings altogether. 719b98940e5SNeel Natu */ 720da761f3bSScott Long dom->spsmask = ~0; 721da761f3bSScott Long for (i = 0; i < drhd_num; i++) { 722da761f3bSScott Long vtdmap = vtdmaps[i]; 723da761f3bSScott Long /* take most compatible value */ 724da761f3bSScott Long dom->spsmask &= VTD_CAP_SPS(vtdmap->cap); 725da761f3bSScott Long } 726b98940e5SNeel Natu #endif 727b98940e5SNeel Natu 728366f6083SPeter Grehan SLIST_INSERT_HEAD(&domhead, dom, next); 729366f6083SPeter Grehan 730366f6083SPeter Grehan return (dom); 731366f6083SPeter Grehan } 732366f6083SPeter Grehan 733366f6083SPeter Grehan static void 734366f6083SPeter Grehan vtd_free_ptp(uint64_t *ptp, int level) 735366f6083SPeter Grehan { 736366f6083SPeter Grehan int i; 737366f6083SPeter Grehan uint64_t *nlp; 738366f6083SPeter Grehan 739366f6083SPeter Grehan if (level > 1) { 740366f6083SPeter Grehan for (i = 0; i < 512; i++) { 741366f6083SPeter Grehan if ((ptp[i] & (VTD_PTE_RD | VTD_PTE_WR)) == 0) 742366f6083SPeter Grehan continue; 743366f6083SPeter Grehan if ((ptp[i] & VTD_PTE_SUPERPAGE) != 0) 744366f6083SPeter Grehan continue; 745366f6083SPeter Grehan nlp = (uint64_t *)PHYS_TO_DMAP(ptp[i] & VTD_PTE_ADDR_M); 746366f6083SPeter Grehan vtd_free_ptp(nlp, level - 1); 747366f6083SPeter Grehan } 748366f6083SPeter Grehan } 749366f6083SPeter Grehan 750366f6083SPeter Grehan bzero(ptp, PAGE_SIZE); 751366f6083SPeter Grehan free(ptp, M_VTD); 752366f6083SPeter Grehan } 753366f6083SPeter Grehan 754366f6083SPeter Grehan static void 755366f6083SPeter Grehan vtd_destroy_domain(void *arg) 756366f6083SPeter Grehan { 757366f6083SPeter Grehan struct domain *dom; 758366f6083SPeter Grehan 759366f6083SPeter Grehan dom = arg; 760366f6083SPeter Grehan 761366f6083SPeter Grehan SLIST_REMOVE(&domhead, dom, domain, next); 762366f6083SPeter Grehan vtd_free_ptp(dom->ptp, dom->pt_levels); 763366f6083SPeter Grehan free(dom, M_VTD); 764366f6083SPeter Grehan } 765366f6083SPeter Grehan 76641335c6bSMark Johnston const struct iommu_ops iommu_ops_intel = { 76741335c6bSMark Johnston .init = vtd_init, 76841335c6bSMark Johnston .cleanup = vtd_cleanup, 76941335c6bSMark Johnston .enable = vtd_enable, 77041335c6bSMark Johnston .disable = vtd_disable, 77141335c6bSMark Johnston .create_domain = vtd_create_domain, 77241335c6bSMark Johnston .destroy_domain = vtd_destroy_domain, 77341335c6bSMark Johnston .create_mapping = vtd_create_mapping, 77441335c6bSMark Johnston .remove_mapping = vtd_remove_mapping, 77541335c6bSMark Johnston .add_device = vtd_add_device, 77641335c6bSMark Johnston .remove_device = vtd_remove_device, 77741335c6bSMark Johnston .invalidate_tlb = vtd_invalidate_tlb, 778366f6083SPeter Grehan }; 779