1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 33366f6083SPeter Grehan #include <sys/types.h> 34366f6083SPeter Grehan #include <sys/systm.h> 35366f6083SPeter Grehan #include <sys/bus.h> 3651f45d01SNeel Natu #include <sys/sysctl.h> 37366f6083SPeter Grehan 38366f6083SPeter Grehan #include <dev/pci/pcivar.h> 39366f6083SPeter Grehan #include <dev/pci/pcireg.h> 40366f6083SPeter Grehan 41ffe1b10dSJohn Baldwin #include <machine/cpu.h> 42366f6083SPeter Grehan #include <machine/md_var.h> 43366f6083SPeter Grehan 44366f6083SPeter Grehan #include "vmm_util.h" 457ce04d0aSNeel Natu #include "vmm_mem.h" 46366f6083SPeter Grehan #include "iommu.h" 47366f6083SPeter Grehan 4851f45d01SNeel Natu SYSCTL_DECL(_hw_vmm); 4951f45d01SNeel Natu SYSCTL_NODE(_hw_vmm, OID_AUTO, iommu, CTLFLAG_RW, 0, "bhyve iommu parameters"); 5051f45d01SNeel Natu 5151f45d01SNeel Natu static int iommu_avail; 5251f45d01SNeel Natu SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, initialized, CTLFLAG_RD, &iommu_avail, 5351f45d01SNeel Natu 0, "bhyve iommu initialized?"); 5451f45d01SNeel Natu 55ffe1b10dSJohn Baldwin static int iommu_enable = 1; 56ffe1b10dSJohn Baldwin SYSCTL_INT(_hw_vmm_iommu, OID_AUTO, enable, CTLFLAG_RDTUN, &iommu_enable, 0, 57ffe1b10dSJohn Baldwin "Enable use of I/O MMU (required for PCI passthrough)."); 58ffe1b10dSJohn Baldwin 59366f6083SPeter Grehan static struct iommu_ops *ops; 60366f6083SPeter Grehan static void *host_domain; 6164414cc0SJohn Baldwin static eventhandler_tag add_tag, delete_tag; 62366f6083SPeter Grehan 63366f6083SPeter Grehan static __inline int 64366f6083SPeter Grehan IOMMU_INIT(void) 65366f6083SPeter Grehan { 66366f6083SPeter Grehan if (ops != NULL) 67366f6083SPeter Grehan return ((*ops->init)()); 68366f6083SPeter Grehan else 69366f6083SPeter Grehan return (ENXIO); 70366f6083SPeter Grehan } 71366f6083SPeter Grehan 72366f6083SPeter Grehan static __inline void 73366f6083SPeter Grehan IOMMU_CLEANUP(void) 74366f6083SPeter Grehan { 75366f6083SPeter Grehan if (ops != NULL && iommu_avail) 76366f6083SPeter Grehan (*ops->cleanup)(); 77366f6083SPeter Grehan } 78366f6083SPeter Grehan 79366f6083SPeter Grehan static __inline void * 80366f6083SPeter Grehan IOMMU_CREATE_DOMAIN(vm_paddr_t maxaddr) 81366f6083SPeter Grehan { 82366f6083SPeter Grehan 83366f6083SPeter Grehan if (ops != NULL && iommu_avail) 84366f6083SPeter Grehan return ((*ops->create_domain)(maxaddr)); 85366f6083SPeter Grehan else 86366f6083SPeter Grehan return (NULL); 87366f6083SPeter Grehan } 88366f6083SPeter Grehan 89366f6083SPeter Grehan static __inline void 90366f6083SPeter Grehan IOMMU_DESTROY_DOMAIN(void *dom) 91366f6083SPeter Grehan { 92366f6083SPeter Grehan 93366f6083SPeter Grehan if (ops != NULL && iommu_avail) 94366f6083SPeter Grehan (*ops->destroy_domain)(dom); 95366f6083SPeter Grehan } 96366f6083SPeter Grehan 97366f6083SPeter Grehan static __inline uint64_t 98366f6083SPeter Grehan IOMMU_CREATE_MAPPING(void *domain, vm_paddr_t gpa, vm_paddr_t hpa, uint64_t len) 99366f6083SPeter Grehan { 100366f6083SPeter Grehan 101366f6083SPeter Grehan if (ops != NULL && iommu_avail) 102366f6083SPeter Grehan return ((*ops->create_mapping)(domain, gpa, hpa, len)); 103366f6083SPeter Grehan else 104366f6083SPeter Grehan return (len); /* XXX */ 105366f6083SPeter Grehan } 106366f6083SPeter Grehan 1077ce04d0aSNeel Natu static __inline uint64_t 1087ce04d0aSNeel Natu IOMMU_REMOVE_MAPPING(void *domain, vm_paddr_t gpa, uint64_t len) 1097ce04d0aSNeel Natu { 1107ce04d0aSNeel Natu 1117ce04d0aSNeel Natu if (ops != NULL && iommu_avail) 1127ce04d0aSNeel Natu return ((*ops->remove_mapping)(domain, gpa, len)); 1137ce04d0aSNeel Natu else 1147ce04d0aSNeel Natu return (len); /* XXX */ 1157ce04d0aSNeel Natu } 1167ce04d0aSNeel Natu 117366f6083SPeter Grehan static __inline void 118a8667250SRyan Stone IOMMU_ADD_DEVICE(void *domain, uint16_t rid) 119366f6083SPeter Grehan { 120366f6083SPeter Grehan 121366f6083SPeter Grehan if (ops != NULL && iommu_avail) 122a8667250SRyan Stone (*ops->add_device)(domain, rid); 123366f6083SPeter Grehan } 124366f6083SPeter Grehan 125366f6083SPeter Grehan static __inline void 126a8667250SRyan Stone IOMMU_REMOVE_DEVICE(void *domain, uint16_t rid) 127366f6083SPeter Grehan { 128366f6083SPeter Grehan 129366f6083SPeter Grehan if (ops != NULL && iommu_avail) 130a8667250SRyan Stone (*ops->remove_device)(domain, rid); 131366f6083SPeter Grehan } 132366f6083SPeter Grehan 133366f6083SPeter Grehan static __inline void 1347ce04d0aSNeel Natu IOMMU_INVALIDATE_TLB(void *domain) 1357ce04d0aSNeel Natu { 1367ce04d0aSNeel Natu 1377ce04d0aSNeel Natu if (ops != NULL && iommu_avail) 1387ce04d0aSNeel Natu (*ops->invalidate_tlb)(domain); 1397ce04d0aSNeel Natu } 1407ce04d0aSNeel Natu 1417ce04d0aSNeel Natu static __inline void 142366f6083SPeter Grehan IOMMU_ENABLE(void) 143366f6083SPeter Grehan { 144366f6083SPeter Grehan 145366f6083SPeter Grehan if (ops != NULL && iommu_avail) 146366f6083SPeter Grehan (*ops->enable)(); 147366f6083SPeter Grehan } 148366f6083SPeter Grehan 149366f6083SPeter Grehan static __inline void 150366f6083SPeter Grehan IOMMU_DISABLE(void) 151366f6083SPeter Grehan { 152366f6083SPeter Grehan 153366f6083SPeter Grehan if (ops != NULL && iommu_avail) 154366f6083SPeter Grehan (*ops->disable)(); 155366f6083SPeter Grehan } 156366f6083SPeter Grehan 157ffe1b10dSJohn Baldwin static void 15864414cc0SJohn Baldwin iommu_pci_add(void *arg, device_t dev) 15964414cc0SJohn Baldwin { 16064414cc0SJohn Baldwin 16164414cc0SJohn Baldwin /* Add new devices to the host domain. */ 16264414cc0SJohn Baldwin iommu_add_device(host_domain, pci_get_rid(dev)); 16364414cc0SJohn Baldwin } 16464414cc0SJohn Baldwin 16564414cc0SJohn Baldwin static void 16664414cc0SJohn Baldwin iommu_pci_delete(void *arg, device_t dev) 16764414cc0SJohn Baldwin { 16864414cc0SJohn Baldwin 16964414cc0SJohn Baldwin iommu_remove_device(host_domain, pci_get_rid(dev)); 17064414cc0SJohn Baldwin } 17164414cc0SJohn Baldwin 17264414cc0SJohn Baldwin static void 173366f6083SPeter Grehan iommu_init(void) 174366f6083SPeter Grehan { 175366f6083SPeter Grehan int error, bus, slot, func; 176366f6083SPeter Grehan vm_paddr_t maxaddr; 1776db55a0fSJohn Baldwin devclass_t dc; 178366f6083SPeter Grehan device_t dev; 179366f6083SPeter Grehan 180ffe1b10dSJohn Baldwin if (!iommu_enable) 181ffe1b10dSJohn Baldwin return; 182ffe1b10dSJohn Baldwin 183366f6083SPeter Grehan if (vmm_is_intel()) 184366f6083SPeter Grehan ops = &iommu_ops_intel; 185366f6083SPeter Grehan else if (vmm_is_amd()) 186366f6083SPeter Grehan ops = &iommu_ops_amd; 187366f6083SPeter Grehan else 188366f6083SPeter Grehan ops = NULL; 189366f6083SPeter Grehan 190366f6083SPeter Grehan error = IOMMU_INIT(); 191366f6083SPeter Grehan if (error) 192366f6083SPeter Grehan return; 193366f6083SPeter Grehan 19451f45d01SNeel Natu iommu_avail = 1; 195366f6083SPeter Grehan 196366f6083SPeter Grehan /* 197366f6083SPeter Grehan * Create a domain for the devices owned by the host 198366f6083SPeter Grehan */ 1997ce04d0aSNeel Natu maxaddr = vmm_mem_maxaddr(); 200366f6083SPeter Grehan host_domain = IOMMU_CREATE_DOMAIN(maxaddr); 201ffe1b10dSJohn Baldwin if (host_domain == NULL) { 202ffe1b10dSJohn Baldwin printf("iommu_init: unable to create a host domain"); 203ffe1b10dSJohn Baldwin IOMMU_CLEANUP(); 204ffe1b10dSJohn Baldwin ops = NULL; 205ffe1b10dSJohn Baldwin iommu_avail = 0; 206ffe1b10dSJohn Baldwin return; 207ffe1b10dSJohn Baldwin } 208366f6083SPeter Grehan 209366f6083SPeter Grehan /* 2107ce04d0aSNeel Natu * Create 1:1 mappings from '0' to 'maxaddr' for devices assigned to 211366f6083SPeter Grehan * the host 212366f6083SPeter Grehan */ 213366f6083SPeter Grehan iommu_create_mapping(host_domain, 0, 0, maxaddr); 214366f6083SPeter Grehan 21564414cc0SJohn Baldwin add_tag = EVENTHANDLER_REGISTER(pci_add_device, iommu_pci_add, NULL, 0); 21664414cc0SJohn Baldwin delete_tag = EVENTHANDLER_REGISTER(pci_delete_device, iommu_pci_delete, 21764414cc0SJohn Baldwin NULL, 0); 2186db55a0fSJohn Baldwin dc = devclass_find("ppt"); 219366f6083SPeter Grehan for (bus = 0; bus <= PCI_BUSMAX; bus++) { 220366f6083SPeter Grehan for (slot = 0; slot <= PCI_SLOTMAX; slot++) { 221366f6083SPeter Grehan for (func = 0; func <= PCI_FUNCMAX; func++) { 222366f6083SPeter Grehan dev = pci_find_dbsf(0, bus, slot, func); 223366f6083SPeter Grehan if (dev == NULL) 224366f6083SPeter Grehan continue; 225366f6083SPeter Grehan 2266db55a0fSJohn Baldwin /* Skip passthrough devices. */ 2276db55a0fSJohn Baldwin if (dc != NULL && 2286db55a0fSJohn Baldwin device_get_devclass(dev) == dc) 2296db55a0fSJohn Baldwin continue; 2306db55a0fSJohn Baldwin 2316db55a0fSJohn Baldwin /* 2326db55a0fSJohn Baldwin * Everything else belongs to the host 2336db55a0fSJohn Baldwin * domain. 2346db55a0fSJohn Baldwin */ 235a8667250SRyan Stone iommu_add_device(host_domain, 236a8667250SRyan Stone pci_get_rid(dev)); 237366f6083SPeter Grehan } 238366f6083SPeter Grehan } 239366f6083SPeter Grehan } 240366f6083SPeter Grehan IOMMU_ENABLE(); 241366f6083SPeter Grehan 242366f6083SPeter Grehan } 243366f6083SPeter Grehan 244366f6083SPeter Grehan void 245366f6083SPeter Grehan iommu_cleanup(void) 246366f6083SPeter Grehan { 24764414cc0SJohn Baldwin 24864414cc0SJohn Baldwin if (add_tag != NULL) { 24964414cc0SJohn Baldwin EVENTHANDLER_DEREGISTER(pci_add_device, add_tag); 25064414cc0SJohn Baldwin add_tag = NULL; 25164414cc0SJohn Baldwin } 25264414cc0SJohn Baldwin if (delete_tag != NULL) { 25364414cc0SJohn Baldwin EVENTHANDLER_DEREGISTER(pci_delete_device, delete_tag); 25464414cc0SJohn Baldwin delete_tag = NULL; 25564414cc0SJohn Baldwin } 256366f6083SPeter Grehan IOMMU_DISABLE(); 257366f6083SPeter Grehan IOMMU_DESTROY_DOMAIN(host_domain); 258366f6083SPeter Grehan IOMMU_CLEANUP(); 259366f6083SPeter Grehan } 260366f6083SPeter Grehan 261366f6083SPeter Grehan void * 262366f6083SPeter Grehan iommu_create_domain(vm_paddr_t maxaddr) 263366f6083SPeter Grehan { 264ffe1b10dSJohn Baldwin static volatile int iommu_initted; 265366f6083SPeter Grehan 266ffe1b10dSJohn Baldwin if (iommu_initted < 2) { 267ffe1b10dSJohn Baldwin if (atomic_cmpset_int(&iommu_initted, 0, 1)) { 268ffe1b10dSJohn Baldwin iommu_init(); 269ffe1b10dSJohn Baldwin atomic_store_rel_int(&iommu_initted, 2); 270ffe1b10dSJohn Baldwin } else 271ffe1b10dSJohn Baldwin while (iommu_initted == 1) 272ffe1b10dSJohn Baldwin cpu_spinwait(); 273ffe1b10dSJohn Baldwin } 274366f6083SPeter Grehan return (IOMMU_CREATE_DOMAIN(maxaddr)); 275366f6083SPeter Grehan } 276366f6083SPeter Grehan 277366f6083SPeter Grehan void 278366f6083SPeter Grehan iommu_destroy_domain(void *dom) 279366f6083SPeter Grehan { 280366f6083SPeter Grehan 281366f6083SPeter Grehan IOMMU_DESTROY_DOMAIN(dom); 282366f6083SPeter Grehan } 283366f6083SPeter Grehan 284366f6083SPeter Grehan void 285366f6083SPeter Grehan iommu_create_mapping(void *dom, vm_paddr_t gpa, vm_paddr_t hpa, size_t len) 286366f6083SPeter Grehan { 287366f6083SPeter Grehan uint64_t mapped, remaining; 288366f6083SPeter Grehan 289366f6083SPeter Grehan remaining = len; 290366f6083SPeter Grehan 291366f6083SPeter Grehan while (remaining > 0) { 292366f6083SPeter Grehan mapped = IOMMU_CREATE_MAPPING(dom, gpa, hpa, remaining); 293366f6083SPeter Grehan gpa += mapped; 294366f6083SPeter Grehan hpa += mapped; 295366f6083SPeter Grehan remaining -= mapped; 296366f6083SPeter Grehan } 297366f6083SPeter Grehan } 298366f6083SPeter Grehan 299366f6083SPeter Grehan void 3007ce04d0aSNeel Natu iommu_remove_mapping(void *dom, vm_paddr_t gpa, size_t len) 3017ce04d0aSNeel Natu { 3027ce04d0aSNeel Natu uint64_t unmapped, remaining; 3037ce04d0aSNeel Natu 3047ce04d0aSNeel Natu remaining = len; 3057ce04d0aSNeel Natu 3067ce04d0aSNeel Natu while (remaining > 0) { 3077ce04d0aSNeel Natu unmapped = IOMMU_REMOVE_MAPPING(dom, gpa, remaining); 3087ce04d0aSNeel Natu gpa += unmapped; 3097ce04d0aSNeel Natu remaining -= unmapped; 3107ce04d0aSNeel Natu } 3117ce04d0aSNeel Natu } 3127ce04d0aSNeel Natu 3137ce04d0aSNeel Natu void * 3147ce04d0aSNeel Natu iommu_host_domain(void) 3157ce04d0aSNeel Natu { 3167ce04d0aSNeel Natu 3177ce04d0aSNeel Natu return (host_domain); 3187ce04d0aSNeel Natu } 3197ce04d0aSNeel Natu 3207ce04d0aSNeel Natu void 321a8667250SRyan Stone iommu_add_device(void *dom, uint16_t rid) 322366f6083SPeter Grehan { 323366f6083SPeter Grehan 324a8667250SRyan Stone IOMMU_ADD_DEVICE(dom, rid); 325366f6083SPeter Grehan } 326366f6083SPeter Grehan 327366f6083SPeter Grehan void 328a8667250SRyan Stone iommu_remove_device(void *dom, uint16_t rid) 329366f6083SPeter Grehan { 330366f6083SPeter Grehan 331a8667250SRyan Stone IOMMU_REMOVE_DEVICE(dom, rid); 332366f6083SPeter Grehan } 3337ce04d0aSNeel Natu 3347ce04d0aSNeel Natu void 3357ce04d0aSNeel Natu iommu_invalidate_tlb(void *domain) 3367ce04d0aSNeel Natu { 3377ce04d0aSNeel Natu 3387ce04d0aSNeel Natu IOMMU_INVALIDATE_TLB(domain); 3397ce04d0aSNeel Natu } 340