1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 3338f1b189SPeter Grehan #include <sys/systm.h> 34366f6083SPeter Grehan #include <sys/kernel.h> 35366f6083SPeter Grehan #include <sys/module.h> 36366f6083SPeter Grehan #include <sys/sysctl.h> 37366f6083SPeter Grehan #include <sys/malloc.h> 38366f6083SPeter Grehan #include <sys/pcpu.h> 39366f6083SPeter Grehan #include <sys/lock.h> 40366f6083SPeter Grehan #include <sys/mutex.h> 41366f6083SPeter Grehan #include <sys/proc.h> 42318224bbSNeel Natu #include <sys/rwlock.h> 43366f6083SPeter Grehan #include <sys/sched.h> 44366f6083SPeter Grehan #include <sys/smp.h> 45366f6083SPeter Grehan #include <sys/systm.h> 46366f6083SPeter Grehan 47366f6083SPeter Grehan #include <vm/vm.h> 48318224bbSNeel Natu #include <vm/vm_object.h> 49318224bbSNeel Natu #include <vm/vm_page.h> 50318224bbSNeel Natu #include <vm/pmap.h> 51318224bbSNeel Natu #include <vm/vm_map.h> 52318224bbSNeel Natu #include <vm/vm_extern.h> 53318224bbSNeel Natu #include <vm/vm_param.h> 54366f6083SPeter Grehan 5563e62d39SJohn Baldwin #include <machine/cpu.h> 56366f6083SPeter Grehan #include <machine/vm.h> 57366f6083SPeter Grehan #include <machine/pcb.h> 5875dd3366SNeel Natu #include <machine/smp.h> 591c052192SNeel Natu #include <x86/psl.h> 6034a6b2d6SJohn Baldwin #include <x86/apicreg.h> 61318224bbSNeel Natu #include <machine/vmparam.h> 62366f6083SPeter Grehan 63366f6083SPeter Grehan #include <machine/vmm.h> 64565bbb86SNeel Natu #include <machine/vmm_dev.h> 65565bbb86SNeel Natu 66318224bbSNeel Natu #include "vmm_ktr.h" 67b01c2033SNeel Natu #include "vmm_host.h" 68366f6083SPeter Grehan #include "vmm_mem.h" 69366f6083SPeter Grehan #include "vmm_util.h" 7008e3ff32SNeel Natu #include "vhpet.h" 71565bbb86SNeel Natu #include "vioapic.h" 72366f6083SPeter Grehan #include "vlapic.h" 73366f6083SPeter Grehan #include "vmm_msr.h" 74366f6083SPeter Grehan #include "vmm_ipi.h" 75366f6083SPeter Grehan #include "vmm_stat.h" 76f76fc5d4SNeel Natu #include "vmm_lapic.h" 77366f6083SPeter Grehan 78366f6083SPeter Grehan #include "io/ppt.h" 79366f6083SPeter Grehan #include "io/iommu.h" 80366f6083SPeter Grehan 81366f6083SPeter Grehan struct vlapic; 82366f6083SPeter Grehan 83366f6083SPeter Grehan struct vcpu { 84366f6083SPeter Grehan int flags; 8575dd3366SNeel Natu enum vcpu_state state; 8675dd3366SNeel Natu struct mtx mtx; 87366f6083SPeter Grehan int hostcpu; /* host cpuid this vcpu last ran on */ 88366f6083SPeter Grehan uint64_t guest_msrs[VMM_MSR_NUM]; 89366f6083SPeter Grehan struct vlapic *vlapic; 90366f6083SPeter Grehan int vcpuid; 9138f1b189SPeter Grehan struct savefpu *guestfpu; /* guest fpu state */ 92abb023fbSJohn Baldwin uint64_t guest_xcr0; 93366f6083SPeter Grehan void *stats; 9498ed632cSNeel Natu struct vm_exit exitinfo; 95e9027382SNeel Natu enum x2apic_state x2apic_state; 96f352ff0cSNeel Natu int nmi_pending; 97366f6083SPeter Grehan }; 98366f6083SPeter Grehan 99f76fc5d4SNeel Natu #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 100f76fc5d4SNeel Natu #define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 101f76fc5d4SNeel Natu #define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 102318224bbSNeel Natu #define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED) 10375dd3366SNeel Natu 104318224bbSNeel Natu struct mem_seg { 105318224bbSNeel Natu vm_paddr_t gpa; 106318224bbSNeel Natu size_t len; 107318224bbSNeel Natu boolean_t wired; 108318224bbSNeel Natu vm_object_t object; 109318224bbSNeel Natu }; 110366f6083SPeter Grehan #define VM_MAX_MEMORY_SEGMENTS 2 111366f6083SPeter Grehan 112366f6083SPeter Grehan struct vm { 113366f6083SPeter Grehan void *cookie; /* processor-specific data */ 114366f6083SPeter Grehan void *iommu; /* iommu-specific data */ 11508e3ff32SNeel Natu struct vhpet *vhpet; /* virtual HPET */ 116565bbb86SNeel Natu struct vioapic *vioapic; /* virtual ioapic */ 117318224bbSNeel Natu struct vmspace *vmspace; /* guest's address space */ 118366f6083SPeter Grehan struct vcpu vcpu[VM_MAXCPU]; 119366f6083SPeter Grehan int num_mem_segs; 120318224bbSNeel Natu struct mem_seg mem_segs[VM_MAX_MEMORY_SEGMENTS]; 121366f6083SPeter Grehan char name[VM_MAX_NAMELEN]; 122366f6083SPeter Grehan 123366f6083SPeter Grehan /* 124a5615c90SPeter Grehan * Set of active vcpus. 125366f6083SPeter Grehan * An active vcpu is one that has been started implicitly (BSP) or 126366f6083SPeter Grehan * explicitly (AP) by sending it a startup ipi. 127366f6083SPeter Grehan */ 128a5615c90SPeter Grehan cpuset_t active_cpus; 1295b8a8cd1SNeel Natu 1305b8a8cd1SNeel Natu struct mtx rendezvous_mtx; 1315b8a8cd1SNeel Natu cpuset_t rendezvous_req_cpus; 1325b8a8cd1SNeel Natu cpuset_t rendezvous_done_cpus; 1335b8a8cd1SNeel Natu void *rendezvous_arg; 1345b8a8cd1SNeel Natu vm_rendezvous_func_t rendezvous_func; 135366f6083SPeter Grehan }; 136366f6083SPeter Grehan 137d5408b1dSNeel Natu static int vmm_initialized; 138d5408b1dSNeel Natu 139366f6083SPeter Grehan static struct vmm_ops *ops; 140add611fdSNeel Natu #define VMM_INIT(num) (ops != NULL ? (*ops->init)(num) : 0) 141366f6083SPeter Grehan #define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 14263e62d39SJohn Baldwin #define VMM_RESUME() (ops != NULL ? (*ops->resume)() : 0) 143366f6083SPeter Grehan 144318224bbSNeel Natu #define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL) 1455b8a8cd1SNeel Natu #define VMRUN(vmi, vcpu, rip, pmap, rptr) \ 1465b8a8cd1SNeel Natu (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap, rptr) : ENXIO) 147366f6083SPeter Grehan #define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 148318224bbSNeel Natu #define VMSPACE_ALLOC(min, max) \ 149318224bbSNeel Natu (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL) 150318224bbSNeel Natu #define VMSPACE_FREE(vmspace) \ 151318224bbSNeel Natu (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO) 152366f6083SPeter Grehan #define VMGETREG(vmi, vcpu, num, retval) \ 153366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 154366f6083SPeter Grehan #define VMSETREG(vmi, vcpu, num, val) \ 155366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 156366f6083SPeter Grehan #define VMGETDESC(vmi, vcpu, num, desc) \ 157366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 158366f6083SPeter Grehan #define VMSETDESC(vmi, vcpu, num, desc) \ 159366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 160366f6083SPeter Grehan #define VMINJECT(vmi, vcpu, type, vec, ec, ecv) \ 161366f6083SPeter Grehan (ops != NULL ? (*ops->vminject)(vmi, vcpu, type, vec, ec, ecv) : ENXIO) 162366f6083SPeter Grehan #define VMGETCAP(vmi, vcpu, num, retval) \ 163366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 164366f6083SPeter Grehan #define VMSETCAP(vmi, vcpu, num, val) \ 165366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 166de5ea6b6SNeel Natu #define VLAPIC_INIT(vmi, vcpu) \ 167de5ea6b6SNeel Natu (ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL) 168de5ea6b6SNeel Natu #define VLAPIC_CLEANUP(vmi, vlapic) \ 169de5ea6b6SNeel Natu (ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL) 170366f6083SPeter Grehan 171014a52f3SNeel Natu #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 172014a52f3SNeel Natu #define fpu_stop_emulating() clts() 173366f6083SPeter Grehan 174366f6083SPeter Grehan static MALLOC_DEFINE(M_VM, "vm", "vm"); 175366f6083SPeter Grehan CTASSERT(VMM_MSR_NUM <= 64); /* msr_mask can keep track of up to 64 msrs */ 176366f6083SPeter Grehan 177366f6083SPeter Grehan /* statistics */ 17861592433SNeel Natu static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 179366f6083SPeter Grehan 180add611fdSNeel Natu SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 181add611fdSNeel Natu 182add611fdSNeel Natu static int vmm_ipinum; 183add611fdSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0, 184add611fdSNeel Natu "IPI vector used for vcpu notifications"); 185add611fdSNeel Natu 1865b8a8cd1SNeel Natu static void vm_deactivate_cpu(struct vm *vm, int vcpuid); 1875b8a8cd1SNeel Natu 188366f6083SPeter Grehan static void 189de5ea6b6SNeel Natu vcpu_cleanup(struct vm *vm, int i) 190366f6083SPeter Grehan { 191de5ea6b6SNeel Natu struct vcpu *vcpu = &vm->vcpu[i]; 192de5ea6b6SNeel Natu 193de5ea6b6SNeel Natu VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic); 194366f6083SPeter Grehan vmm_stat_free(vcpu->stats); 19538f1b189SPeter Grehan fpu_save_area_free(vcpu->guestfpu); 196366f6083SPeter Grehan } 197366f6083SPeter Grehan 198366f6083SPeter Grehan static void 199366f6083SPeter Grehan vcpu_init(struct vm *vm, uint32_t vcpu_id) 200366f6083SPeter Grehan { 201366f6083SPeter Grehan struct vcpu *vcpu; 202366f6083SPeter Grehan 203366f6083SPeter Grehan vcpu = &vm->vcpu[vcpu_id]; 204366f6083SPeter Grehan 20575dd3366SNeel Natu vcpu_lock_init(vcpu); 20675dd3366SNeel Natu vcpu->hostcpu = NOCPU; 207366f6083SPeter Grehan vcpu->vcpuid = vcpu_id; 208de5ea6b6SNeel Natu vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id); 20952e5c8a2SNeel Natu vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED); 210abb023fbSJohn Baldwin vcpu->guest_xcr0 = XFEATURE_ENABLED_X87; 21138f1b189SPeter Grehan vcpu->guestfpu = fpu_save_area_alloc(); 21238f1b189SPeter Grehan fpu_save_area_reset(vcpu->guestfpu); 213366f6083SPeter Grehan vcpu->stats = vmm_stat_alloc(); 214366f6083SPeter Grehan } 215366f6083SPeter Grehan 21698ed632cSNeel Natu struct vm_exit * 21798ed632cSNeel Natu vm_exitinfo(struct vm *vm, int cpuid) 21898ed632cSNeel Natu { 21998ed632cSNeel Natu struct vcpu *vcpu; 22098ed632cSNeel Natu 22198ed632cSNeel Natu if (cpuid < 0 || cpuid >= VM_MAXCPU) 22298ed632cSNeel Natu panic("vm_exitinfo: invalid cpuid %d", cpuid); 22398ed632cSNeel Natu 22498ed632cSNeel Natu vcpu = &vm->vcpu[cpuid]; 22598ed632cSNeel Natu 22698ed632cSNeel Natu return (&vcpu->exitinfo); 22798ed632cSNeel Natu } 22898ed632cSNeel Natu 22963e62d39SJohn Baldwin static void 23063e62d39SJohn Baldwin vmm_resume(void) 23163e62d39SJohn Baldwin { 23263e62d39SJohn Baldwin VMM_RESUME(); 23363e62d39SJohn Baldwin } 23463e62d39SJohn Baldwin 235366f6083SPeter Grehan static int 236366f6083SPeter Grehan vmm_init(void) 237366f6083SPeter Grehan { 238366f6083SPeter Grehan int error; 239366f6083SPeter Grehan 240b01c2033SNeel Natu vmm_host_state_init(); 241add611fdSNeel Natu 242add611fdSNeel Natu vmm_ipinum = vmm_ipi_alloc(); 243add611fdSNeel Natu if (vmm_ipinum == 0) 244add611fdSNeel Natu vmm_ipinum = IPI_AST; 245366f6083SPeter Grehan 246366f6083SPeter Grehan error = vmm_mem_init(); 247366f6083SPeter Grehan if (error) 248366f6083SPeter Grehan return (error); 249366f6083SPeter Grehan 250366f6083SPeter Grehan if (vmm_is_intel()) 251366f6083SPeter Grehan ops = &vmm_ops_intel; 252366f6083SPeter Grehan else if (vmm_is_amd()) 253366f6083SPeter Grehan ops = &vmm_ops_amd; 254366f6083SPeter Grehan else 255366f6083SPeter Grehan return (ENXIO); 256366f6083SPeter Grehan 257366f6083SPeter Grehan vmm_msr_init(); 25863e62d39SJohn Baldwin vmm_resume_p = vmm_resume; 259366f6083SPeter Grehan 260add611fdSNeel Natu return (VMM_INIT(vmm_ipinum)); 261366f6083SPeter Grehan } 262366f6083SPeter Grehan 263366f6083SPeter Grehan static int 264366f6083SPeter Grehan vmm_handler(module_t mod, int what, void *arg) 265366f6083SPeter Grehan { 266366f6083SPeter Grehan int error; 267366f6083SPeter Grehan 268366f6083SPeter Grehan switch (what) { 269366f6083SPeter Grehan case MOD_LOAD: 270366f6083SPeter Grehan vmmdev_init(); 27151f45d01SNeel Natu if (ppt_avail_devices() > 0) 272366f6083SPeter Grehan iommu_init(); 273366f6083SPeter Grehan error = vmm_init(); 274d5408b1dSNeel Natu if (error == 0) 275d5408b1dSNeel Natu vmm_initialized = 1; 276366f6083SPeter Grehan break; 277366f6083SPeter Grehan case MOD_UNLOAD: 278cdc5b9e7SNeel Natu error = vmmdev_cleanup(); 279cdc5b9e7SNeel Natu if (error == 0) { 28063e62d39SJohn Baldwin vmm_resume_p = NULL; 281366f6083SPeter Grehan iommu_cleanup(); 282add611fdSNeel Natu if (vmm_ipinum != IPI_AST) 283add611fdSNeel Natu vmm_ipi_free(vmm_ipinum); 284366f6083SPeter Grehan error = VMM_CLEANUP(); 28581ef6611SPeter Grehan /* 28681ef6611SPeter Grehan * Something bad happened - prevent new 28781ef6611SPeter Grehan * VMs from being created 28881ef6611SPeter Grehan */ 28981ef6611SPeter Grehan if (error) 290d5408b1dSNeel Natu vmm_initialized = 0; 29181ef6611SPeter Grehan } 292366f6083SPeter Grehan break; 293366f6083SPeter Grehan default: 294366f6083SPeter Grehan error = 0; 295366f6083SPeter Grehan break; 296366f6083SPeter Grehan } 297366f6083SPeter Grehan return (error); 298366f6083SPeter Grehan } 299366f6083SPeter Grehan 300366f6083SPeter Grehan static moduledata_t vmm_kmod = { 301366f6083SPeter Grehan "vmm", 302366f6083SPeter Grehan vmm_handler, 303366f6083SPeter Grehan NULL 304366f6083SPeter Grehan }; 305366f6083SPeter Grehan 306366f6083SPeter Grehan /* 307e3f0800bSNeel Natu * vmm initialization has the following dependencies: 308e3f0800bSNeel Natu * 309e3f0800bSNeel Natu * - iommu initialization must happen after the pci passthru driver has had 310e3f0800bSNeel Natu * a chance to attach to any passthru devices (after SI_SUB_CONFIGURE). 311e3f0800bSNeel Natu * 312e3f0800bSNeel Natu * - VT-x initialization requires smp_rendezvous() and therefore must happen 313e3f0800bSNeel Natu * after SMP is fully functional (after SI_SUB_SMP). 314366f6083SPeter Grehan */ 315e3f0800bSNeel Natu DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY); 316366f6083SPeter Grehan MODULE_VERSION(vmm, 1); 317366f6083SPeter Grehan 318d5408b1dSNeel Natu int 319d5408b1dSNeel Natu vm_create(const char *name, struct vm **retvm) 320366f6083SPeter Grehan { 321366f6083SPeter Grehan int i; 322366f6083SPeter Grehan struct vm *vm; 323318224bbSNeel Natu struct vmspace *vmspace; 324366f6083SPeter Grehan 325366f6083SPeter Grehan const int BSP = 0; 326366f6083SPeter Grehan 327d5408b1dSNeel Natu /* 328d5408b1dSNeel Natu * If vmm.ko could not be successfully initialized then don't attempt 329d5408b1dSNeel Natu * to create the virtual machine. 330d5408b1dSNeel Natu */ 331d5408b1dSNeel Natu if (!vmm_initialized) 332d5408b1dSNeel Natu return (ENXIO); 333d5408b1dSNeel Natu 334366f6083SPeter Grehan if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 335d5408b1dSNeel Natu return (EINVAL); 336366f6083SPeter Grehan 337318224bbSNeel Natu vmspace = VMSPACE_ALLOC(VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS); 338318224bbSNeel Natu if (vmspace == NULL) 339318224bbSNeel Natu return (ENOMEM); 340318224bbSNeel Natu 341366f6083SPeter Grehan vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 342366f6083SPeter Grehan strcpy(vm->name, name); 34388c4b8d1SNeel Natu vm->vmspace = vmspace; 3445b8a8cd1SNeel Natu mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF); 345318224bbSNeel Natu vm->cookie = VMINIT(vm, vmspace_pmap(vmspace)); 346565bbb86SNeel Natu vm->vioapic = vioapic_init(vm); 34708e3ff32SNeel Natu vm->vhpet = vhpet_init(vm); 348366f6083SPeter Grehan 349366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) { 350366f6083SPeter Grehan vcpu_init(vm, i); 351366f6083SPeter Grehan guest_msrs_init(vm, i); 352366f6083SPeter Grehan } 353366f6083SPeter Grehan 354366f6083SPeter Grehan vm_activate_cpu(vm, BSP); 355366f6083SPeter Grehan 356d5408b1dSNeel Natu *retvm = vm; 357d5408b1dSNeel Natu return (0); 358366f6083SPeter Grehan } 359366f6083SPeter Grehan 360f7d51510SNeel Natu static void 361318224bbSNeel Natu vm_free_mem_seg(struct vm *vm, struct mem_seg *seg) 362f7d51510SNeel Natu { 3637ce04d0aSNeel Natu 364318224bbSNeel Natu if (seg->object != NULL) 365318224bbSNeel Natu vmm_mem_free(vm->vmspace, seg->gpa, seg->len); 366f7d51510SNeel Natu 367318224bbSNeel Natu bzero(seg, sizeof(*seg)); 368f7d51510SNeel Natu } 369f7d51510SNeel Natu 370366f6083SPeter Grehan void 371366f6083SPeter Grehan vm_destroy(struct vm *vm) 372366f6083SPeter Grehan { 373366f6083SPeter Grehan int i; 374366f6083SPeter Grehan 375366f6083SPeter Grehan ppt_unassign_all(vm); 376366f6083SPeter Grehan 377318224bbSNeel Natu if (vm->iommu != NULL) 378318224bbSNeel Natu iommu_destroy_domain(vm->iommu); 379318224bbSNeel Natu 38008e3ff32SNeel Natu vhpet_cleanup(vm->vhpet); 38108e3ff32SNeel Natu vioapic_cleanup(vm->vioapic); 38208e3ff32SNeel Natu 383366f6083SPeter Grehan for (i = 0; i < vm->num_mem_segs; i++) 384f7d51510SNeel Natu vm_free_mem_seg(vm, &vm->mem_segs[i]); 385f7d51510SNeel Natu 386f7d51510SNeel Natu vm->num_mem_segs = 0; 387366f6083SPeter Grehan 388366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) 389de5ea6b6SNeel Natu vcpu_cleanup(vm, i); 390366f6083SPeter Grehan 391318224bbSNeel Natu VMSPACE_FREE(vm->vmspace); 392366f6083SPeter Grehan 393366f6083SPeter Grehan VMCLEANUP(vm->cookie); 394366f6083SPeter Grehan 395366f6083SPeter Grehan free(vm, M_VM); 396366f6083SPeter Grehan } 397366f6083SPeter Grehan 398366f6083SPeter Grehan const char * 399366f6083SPeter Grehan vm_name(struct vm *vm) 400366f6083SPeter Grehan { 401366f6083SPeter Grehan return (vm->name); 402366f6083SPeter Grehan } 403366f6083SPeter Grehan 404366f6083SPeter Grehan int 405366f6083SPeter Grehan vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 406366f6083SPeter Grehan { 407318224bbSNeel Natu vm_object_t obj; 408366f6083SPeter Grehan 409318224bbSNeel Natu if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL) 410318224bbSNeel Natu return (ENOMEM); 411318224bbSNeel Natu else 412318224bbSNeel Natu return (0); 413366f6083SPeter Grehan } 414366f6083SPeter Grehan 415366f6083SPeter Grehan int 416366f6083SPeter Grehan vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 417366f6083SPeter Grehan { 418366f6083SPeter Grehan 419318224bbSNeel Natu vmm_mmio_free(vm->vmspace, gpa, len); 420318224bbSNeel Natu return (0); 421366f6083SPeter Grehan } 422366f6083SPeter Grehan 423318224bbSNeel Natu boolean_t 424318224bbSNeel Natu vm_mem_allocated(struct vm *vm, vm_paddr_t gpa) 425366f6083SPeter Grehan { 426341f19c9SNeel Natu int i; 427341f19c9SNeel Natu vm_paddr_t gpabase, gpalimit; 428341f19c9SNeel Natu 429341f19c9SNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 430341f19c9SNeel Natu gpabase = vm->mem_segs[i].gpa; 431341f19c9SNeel Natu gpalimit = gpabase + vm->mem_segs[i].len; 432341f19c9SNeel Natu if (gpa >= gpabase && gpa < gpalimit) 433318224bbSNeel Natu return (TRUE); /* 'gpa' is regular memory */ 434341f19c9SNeel Natu } 435341f19c9SNeel Natu 436318224bbSNeel Natu if (ppt_is_mmio(vm, gpa)) 437318224bbSNeel Natu return (TRUE); /* 'gpa' is pci passthru mmio */ 438318224bbSNeel Natu 439318224bbSNeel Natu return (FALSE); 440341f19c9SNeel Natu } 441341f19c9SNeel Natu 442341f19c9SNeel Natu int 443341f19c9SNeel Natu vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len) 444341f19c9SNeel Natu { 445318224bbSNeel Natu int available, allocated; 446318224bbSNeel Natu struct mem_seg *seg; 447318224bbSNeel Natu vm_object_t object; 448318224bbSNeel Natu vm_paddr_t g; 449366f6083SPeter Grehan 450341f19c9SNeel Natu if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0) 451341f19c9SNeel Natu return (EINVAL); 452341f19c9SNeel Natu 453341f19c9SNeel Natu available = allocated = 0; 454341f19c9SNeel Natu g = gpa; 455341f19c9SNeel Natu while (g < gpa + len) { 456318224bbSNeel Natu if (vm_mem_allocated(vm, g)) 457341f19c9SNeel Natu allocated++; 458318224bbSNeel Natu else 459318224bbSNeel Natu available++; 460341f19c9SNeel Natu 461341f19c9SNeel Natu g += PAGE_SIZE; 462341f19c9SNeel Natu } 463341f19c9SNeel Natu 464366f6083SPeter Grehan /* 465341f19c9SNeel Natu * If there are some allocated and some available pages in the address 466341f19c9SNeel Natu * range then it is an error. 467366f6083SPeter Grehan */ 468341f19c9SNeel Natu if (allocated && available) 469341f19c9SNeel Natu return (EINVAL); 470341f19c9SNeel Natu 471341f19c9SNeel Natu /* 472341f19c9SNeel Natu * If the entire address range being requested has already been 473341f19c9SNeel Natu * allocated then there isn't anything more to do. 474341f19c9SNeel Natu */ 475341f19c9SNeel Natu if (allocated && available == 0) 476341f19c9SNeel Natu return (0); 477366f6083SPeter Grehan 478366f6083SPeter Grehan if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS) 479366f6083SPeter Grehan return (E2BIG); 480366f6083SPeter Grehan 481f7d51510SNeel Natu seg = &vm->mem_segs[vm->num_mem_segs]; 482366f6083SPeter Grehan 483318224bbSNeel Natu if ((object = vmm_mem_alloc(vm->vmspace, gpa, len)) == NULL) 484318224bbSNeel Natu return (ENOMEM); 485318224bbSNeel Natu 486f7d51510SNeel Natu seg->gpa = gpa; 487318224bbSNeel Natu seg->len = len; 488318224bbSNeel Natu seg->object = object; 489318224bbSNeel Natu seg->wired = FALSE; 4907ce04d0aSNeel Natu 491366f6083SPeter Grehan vm->num_mem_segs++; 492341f19c9SNeel Natu 493366f6083SPeter Grehan return (0); 494366f6083SPeter Grehan } 495366f6083SPeter Grehan 496318224bbSNeel Natu static void 497318224bbSNeel Natu vm_gpa_unwire(struct vm *vm) 498366f6083SPeter Grehan { 499318224bbSNeel Natu int i, rv; 500318224bbSNeel Natu struct mem_seg *seg; 5014db4fb2cSNeel Natu 502318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 503318224bbSNeel Natu seg = &vm->mem_segs[i]; 504318224bbSNeel Natu if (!seg->wired) 505318224bbSNeel Natu continue; 506366f6083SPeter Grehan 507318224bbSNeel Natu rv = vm_map_unwire(&vm->vmspace->vm_map, 508318224bbSNeel Natu seg->gpa, seg->gpa + seg->len, 509318224bbSNeel Natu VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 510318224bbSNeel Natu KASSERT(rv == KERN_SUCCESS, ("vm(%s) memory segment " 511318224bbSNeel Natu "%#lx/%ld could not be unwired: %d", 512318224bbSNeel Natu vm_name(vm), seg->gpa, seg->len, rv)); 513318224bbSNeel Natu 514318224bbSNeel Natu seg->wired = FALSE; 515318224bbSNeel Natu } 516318224bbSNeel Natu } 517318224bbSNeel Natu 518318224bbSNeel Natu static int 519318224bbSNeel Natu vm_gpa_wire(struct vm *vm) 520318224bbSNeel Natu { 521318224bbSNeel Natu int i, rv; 522318224bbSNeel Natu struct mem_seg *seg; 523318224bbSNeel Natu 524318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 525318224bbSNeel Natu seg = &vm->mem_segs[i]; 526318224bbSNeel Natu if (seg->wired) 527318224bbSNeel Natu continue; 528318224bbSNeel Natu 529318224bbSNeel Natu /* XXX rlimits? */ 530318224bbSNeel Natu rv = vm_map_wire(&vm->vmspace->vm_map, 531318224bbSNeel Natu seg->gpa, seg->gpa + seg->len, 532318224bbSNeel Natu VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 533318224bbSNeel Natu if (rv != KERN_SUCCESS) 534318224bbSNeel Natu break; 535318224bbSNeel Natu 536318224bbSNeel Natu seg->wired = TRUE; 537318224bbSNeel Natu } 538318224bbSNeel Natu 539318224bbSNeel Natu if (i < vm->num_mem_segs) { 540318224bbSNeel Natu /* 541318224bbSNeel Natu * Undo the wiring before returning an error. 542318224bbSNeel Natu */ 543318224bbSNeel Natu vm_gpa_unwire(vm); 544318224bbSNeel Natu return (EAGAIN); 545318224bbSNeel Natu } 546318224bbSNeel Natu 547318224bbSNeel Natu return (0); 548318224bbSNeel Natu } 549318224bbSNeel Natu 550318224bbSNeel Natu static void 551318224bbSNeel Natu vm_iommu_modify(struct vm *vm, boolean_t map) 552318224bbSNeel Natu { 553318224bbSNeel Natu int i, sz; 554318224bbSNeel Natu vm_paddr_t gpa, hpa; 555318224bbSNeel Natu struct mem_seg *seg; 556318224bbSNeel Natu void *vp, *cookie, *host_domain; 557318224bbSNeel Natu 558318224bbSNeel Natu sz = PAGE_SIZE; 559318224bbSNeel Natu host_domain = iommu_host_domain(); 560318224bbSNeel Natu 561318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 562318224bbSNeel Natu seg = &vm->mem_segs[i]; 563318224bbSNeel Natu KASSERT(seg->wired, ("vm(%s) memory segment %#lx/%ld not wired", 564318224bbSNeel Natu vm_name(vm), seg->gpa, seg->len)); 565318224bbSNeel Natu 566318224bbSNeel Natu gpa = seg->gpa; 567318224bbSNeel Natu while (gpa < seg->gpa + seg->len) { 568318224bbSNeel Natu vp = vm_gpa_hold(vm, gpa, PAGE_SIZE, VM_PROT_WRITE, 569318224bbSNeel Natu &cookie); 570318224bbSNeel Natu KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx", 571318224bbSNeel Natu vm_name(vm), gpa)); 572318224bbSNeel Natu 573318224bbSNeel Natu vm_gpa_release(cookie); 574318224bbSNeel Natu 575318224bbSNeel Natu hpa = DMAP_TO_PHYS((uintptr_t)vp); 576318224bbSNeel Natu if (map) { 577318224bbSNeel Natu iommu_create_mapping(vm->iommu, gpa, hpa, sz); 578318224bbSNeel Natu iommu_remove_mapping(host_domain, hpa, sz); 579318224bbSNeel Natu } else { 580318224bbSNeel Natu iommu_remove_mapping(vm->iommu, gpa, sz); 581318224bbSNeel Natu iommu_create_mapping(host_domain, hpa, hpa, sz); 582318224bbSNeel Natu } 583318224bbSNeel Natu 584318224bbSNeel Natu gpa += PAGE_SIZE; 585318224bbSNeel Natu } 586318224bbSNeel Natu } 587318224bbSNeel Natu 588318224bbSNeel Natu /* 589318224bbSNeel Natu * Invalidate the cached translations associated with the domain 590318224bbSNeel Natu * from which pages were removed. 591318224bbSNeel Natu */ 592318224bbSNeel Natu if (map) 593318224bbSNeel Natu iommu_invalidate_tlb(host_domain); 594318224bbSNeel Natu else 595318224bbSNeel Natu iommu_invalidate_tlb(vm->iommu); 596318224bbSNeel Natu } 597318224bbSNeel Natu 598318224bbSNeel Natu #define vm_iommu_unmap(vm) vm_iommu_modify((vm), FALSE) 599318224bbSNeel Natu #define vm_iommu_map(vm) vm_iommu_modify((vm), TRUE) 600318224bbSNeel Natu 601318224bbSNeel Natu int 602318224bbSNeel Natu vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func) 603318224bbSNeel Natu { 604318224bbSNeel Natu int error; 605318224bbSNeel Natu 606318224bbSNeel Natu error = ppt_unassign_device(vm, bus, slot, func); 607318224bbSNeel Natu if (error) 608318224bbSNeel Natu return (error); 609318224bbSNeel Natu 61051f45d01SNeel Natu if (ppt_assigned_devices(vm) == 0) { 611318224bbSNeel Natu vm_iommu_unmap(vm); 612318224bbSNeel Natu vm_gpa_unwire(vm); 613318224bbSNeel Natu } 614318224bbSNeel Natu return (0); 615318224bbSNeel Natu } 616318224bbSNeel Natu 617318224bbSNeel Natu int 618318224bbSNeel Natu vm_assign_pptdev(struct vm *vm, int bus, int slot, int func) 619318224bbSNeel Natu { 620318224bbSNeel Natu int error; 621318224bbSNeel Natu vm_paddr_t maxaddr; 622318224bbSNeel Natu 623318224bbSNeel Natu /* 624318224bbSNeel Natu * Virtual machines with pci passthru devices get special treatment: 625318224bbSNeel Natu * - the guest physical memory is wired 626318224bbSNeel Natu * - the iommu is programmed to do the 'gpa' to 'hpa' translation 627318224bbSNeel Natu * 628318224bbSNeel Natu * We need to do this before the first pci passthru device is attached. 629318224bbSNeel Natu */ 63051f45d01SNeel Natu if (ppt_assigned_devices(vm) == 0) { 631318224bbSNeel Natu KASSERT(vm->iommu == NULL, 632318224bbSNeel Natu ("vm_assign_pptdev: iommu must be NULL")); 633318224bbSNeel Natu maxaddr = vmm_mem_maxaddr(); 634318224bbSNeel Natu vm->iommu = iommu_create_domain(maxaddr); 635318224bbSNeel Natu 636318224bbSNeel Natu error = vm_gpa_wire(vm); 637318224bbSNeel Natu if (error) 638318224bbSNeel Natu return (error); 639318224bbSNeel Natu 640318224bbSNeel Natu vm_iommu_map(vm); 641318224bbSNeel Natu } 642318224bbSNeel Natu 643318224bbSNeel Natu error = ppt_assign_device(vm, bus, slot, func); 644318224bbSNeel Natu return (error); 645318224bbSNeel Natu } 646318224bbSNeel Natu 647318224bbSNeel Natu void * 648318224bbSNeel Natu vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 649318224bbSNeel Natu void **cookie) 650318224bbSNeel Natu { 651318224bbSNeel Natu int count, pageoff; 652318224bbSNeel Natu vm_page_t m; 653318224bbSNeel Natu 654318224bbSNeel Natu pageoff = gpa & PAGE_MASK; 655318224bbSNeel Natu if (len > PAGE_SIZE - pageoff) 656318224bbSNeel Natu panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 657318224bbSNeel Natu 658318224bbSNeel Natu count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map, 659318224bbSNeel Natu trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1); 660318224bbSNeel Natu 661318224bbSNeel Natu if (count == 1) { 662318224bbSNeel Natu *cookie = m; 663318224bbSNeel Natu return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 664318224bbSNeel Natu } else { 665318224bbSNeel Natu *cookie = NULL; 666318224bbSNeel Natu return (NULL); 667318224bbSNeel Natu } 668318224bbSNeel Natu } 669318224bbSNeel Natu 670318224bbSNeel Natu void 671318224bbSNeel Natu vm_gpa_release(void *cookie) 672318224bbSNeel Natu { 673318224bbSNeel Natu vm_page_t m = cookie; 674318224bbSNeel Natu 675318224bbSNeel Natu vm_page_lock(m); 676318224bbSNeel Natu vm_page_unhold(m); 677318224bbSNeel Natu vm_page_unlock(m); 678366f6083SPeter Grehan } 679366f6083SPeter Grehan 680366f6083SPeter Grehan int 681366f6083SPeter Grehan vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase, 682366f6083SPeter Grehan struct vm_memory_segment *seg) 683366f6083SPeter Grehan { 684366f6083SPeter Grehan int i; 685366f6083SPeter Grehan 686366f6083SPeter Grehan for (i = 0; i < vm->num_mem_segs; i++) { 687366f6083SPeter Grehan if (gpabase == vm->mem_segs[i].gpa) { 688318224bbSNeel Natu seg->gpa = vm->mem_segs[i].gpa; 689318224bbSNeel Natu seg->len = vm->mem_segs[i].len; 690318224bbSNeel Natu seg->wired = vm->mem_segs[i].wired; 691366f6083SPeter Grehan return (0); 692366f6083SPeter Grehan } 693366f6083SPeter Grehan } 694366f6083SPeter Grehan return (-1); 695366f6083SPeter Grehan } 696366f6083SPeter Grehan 697366f6083SPeter Grehan int 698318224bbSNeel Natu vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len, 699318224bbSNeel Natu vm_offset_t *offset, struct vm_object **object) 700318224bbSNeel Natu { 701318224bbSNeel Natu int i; 702318224bbSNeel Natu size_t seg_len; 703318224bbSNeel Natu vm_paddr_t seg_gpa; 704318224bbSNeel Natu vm_object_t seg_obj; 705318224bbSNeel Natu 706318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 707318224bbSNeel Natu if ((seg_obj = vm->mem_segs[i].object) == NULL) 708318224bbSNeel Natu continue; 709318224bbSNeel Natu 710318224bbSNeel Natu seg_gpa = vm->mem_segs[i].gpa; 711318224bbSNeel Natu seg_len = vm->mem_segs[i].len; 712318224bbSNeel Natu 713318224bbSNeel Natu if (gpa >= seg_gpa && gpa < seg_gpa + seg_len) { 714318224bbSNeel Natu *offset = gpa - seg_gpa; 715318224bbSNeel Natu *object = seg_obj; 716318224bbSNeel Natu vm_object_reference(seg_obj); 717318224bbSNeel Natu return (0); 718318224bbSNeel Natu } 719318224bbSNeel Natu } 720318224bbSNeel Natu 721318224bbSNeel Natu return (EINVAL); 722318224bbSNeel Natu } 723318224bbSNeel Natu 724318224bbSNeel Natu int 725366f6083SPeter Grehan vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 726366f6083SPeter Grehan { 727366f6083SPeter Grehan 728366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 729366f6083SPeter Grehan return (EINVAL); 730366f6083SPeter Grehan 731366f6083SPeter Grehan if (reg >= VM_REG_LAST) 732366f6083SPeter Grehan return (EINVAL); 733366f6083SPeter Grehan 734366f6083SPeter Grehan return (VMGETREG(vm->cookie, vcpu, reg, retval)); 735366f6083SPeter Grehan } 736366f6083SPeter Grehan 737366f6083SPeter Grehan int 738366f6083SPeter Grehan vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val) 739366f6083SPeter Grehan { 740366f6083SPeter Grehan 741366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 742366f6083SPeter Grehan return (EINVAL); 743366f6083SPeter Grehan 744366f6083SPeter Grehan if (reg >= VM_REG_LAST) 745366f6083SPeter Grehan return (EINVAL); 746366f6083SPeter Grehan 747366f6083SPeter Grehan return (VMSETREG(vm->cookie, vcpu, reg, val)); 748366f6083SPeter Grehan } 749366f6083SPeter Grehan 750366f6083SPeter Grehan static boolean_t 751366f6083SPeter Grehan is_descriptor_table(int reg) 752366f6083SPeter Grehan { 753366f6083SPeter Grehan 754366f6083SPeter Grehan switch (reg) { 755366f6083SPeter Grehan case VM_REG_GUEST_IDTR: 756366f6083SPeter Grehan case VM_REG_GUEST_GDTR: 757366f6083SPeter Grehan return (TRUE); 758366f6083SPeter Grehan default: 759366f6083SPeter Grehan return (FALSE); 760366f6083SPeter Grehan } 761366f6083SPeter Grehan } 762366f6083SPeter Grehan 763366f6083SPeter Grehan static boolean_t 764366f6083SPeter Grehan is_segment_register(int reg) 765366f6083SPeter Grehan { 766366f6083SPeter Grehan 767366f6083SPeter Grehan switch (reg) { 768366f6083SPeter Grehan case VM_REG_GUEST_ES: 769366f6083SPeter Grehan case VM_REG_GUEST_CS: 770366f6083SPeter Grehan case VM_REG_GUEST_SS: 771366f6083SPeter Grehan case VM_REG_GUEST_DS: 772366f6083SPeter Grehan case VM_REG_GUEST_FS: 773366f6083SPeter Grehan case VM_REG_GUEST_GS: 774366f6083SPeter Grehan case VM_REG_GUEST_TR: 775366f6083SPeter Grehan case VM_REG_GUEST_LDTR: 776366f6083SPeter Grehan return (TRUE); 777366f6083SPeter Grehan default: 778366f6083SPeter Grehan return (FALSE); 779366f6083SPeter Grehan } 780366f6083SPeter Grehan } 781366f6083SPeter Grehan 782366f6083SPeter Grehan int 783366f6083SPeter Grehan vm_get_seg_desc(struct vm *vm, int vcpu, int reg, 784366f6083SPeter Grehan struct seg_desc *desc) 785366f6083SPeter Grehan { 786366f6083SPeter Grehan 787366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 788366f6083SPeter Grehan return (EINVAL); 789366f6083SPeter Grehan 790366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 791366f6083SPeter Grehan return (EINVAL); 792366f6083SPeter Grehan 793366f6083SPeter Grehan return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 794366f6083SPeter Grehan } 795366f6083SPeter Grehan 796366f6083SPeter Grehan int 797366f6083SPeter Grehan vm_set_seg_desc(struct vm *vm, int vcpu, int reg, 798366f6083SPeter Grehan struct seg_desc *desc) 799366f6083SPeter Grehan { 800366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 801366f6083SPeter Grehan return (EINVAL); 802366f6083SPeter Grehan 803366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 804366f6083SPeter Grehan return (EINVAL); 805366f6083SPeter Grehan 806366f6083SPeter Grehan return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 807366f6083SPeter Grehan } 808366f6083SPeter Grehan 809366f6083SPeter Grehan static void 810366f6083SPeter Grehan restore_guest_fpustate(struct vcpu *vcpu) 811366f6083SPeter Grehan { 812366f6083SPeter Grehan 81338f1b189SPeter Grehan /* flush host state to the pcb */ 81438f1b189SPeter Grehan fpuexit(curthread); 815bd8572e0SNeel Natu 816bd8572e0SNeel Natu /* restore guest FPU state */ 817366f6083SPeter Grehan fpu_stop_emulating(); 81838f1b189SPeter Grehan fpurestore(vcpu->guestfpu); 819bd8572e0SNeel Natu 820abb023fbSJohn Baldwin /* restore guest XCR0 if XSAVE is enabled in the host */ 821abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) 822abb023fbSJohn Baldwin load_xcr(0, vcpu->guest_xcr0); 823abb023fbSJohn Baldwin 824bd8572e0SNeel Natu /* 825bd8572e0SNeel Natu * The FPU is now "dirty" with the guest's state so turn on emulation 826bd8572e0SNeel Natu * to trap any access to the FPU by the host. 827bd8572e0SNeel Natu */ 828bd8572e0SNeel Natu fpu_start_emulating(); 829366f6083SPeter Grehan } 830366f6083SPeter Grehan 831366f6083SPeter Grehan static void 832366f6083SPeter Grehan save_guest_fpustate(struct vcpu *vcpu) 833366f6083SPeter Grehan { 834366f6083SPeter Grehan 835bd8572e0SNeel Natu if ((rcr0() & CR0_TS) == 0) 836bd8572e0SNeel Natu panic("fpu emulation not enabled in host!"); 837bd8572e0SNeel Natu 838abb023fbSJohn Baldwin /* save guest XCR0 and restore host XCR0 */ 839abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) { 840abb023fbSJohn Baldwin vcpu->guest_xcr0 = rxcr(0); 841abb023fbSJohn Baldwin load_xcr(0, vmm_get_host_xcr0()); 842abb023fbSJohn Baldwin } 843abb023fbSJohn Baldwin 844bd8572e0SNeel Natu /* save guest FPU state */ 845bd8572e0SNeel Natu fpu_stop_emulating(); 84638f1b189SPeter Grehan fpusave(vcpu->guestfpu); 847366f6083SPeter Grehan fpu_start_emulating(); 848366f6083SPeter Grehan } 849366f6083SPeter Grehan 85061592433SNeel Natu static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 851f76fc5d4SNeel Natu 852318224bbSNeel Natu static int 853f80330a8SNeel Natu vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate, 854f80330a8SNeel Natu bool from_idle) 855366f6083SPeter Grehan { 856318224bbSNeel Natu int error; 857366f6083SPeter Grehan 858318224bbSNeel Natu vcpu_assert_locked(vcpu); 859366f6083SPeter Grehan 860f76fc5d4SNeel Natu /* 861f80330a8SNeel Natu * State transitions from the vmmdev_ioctl() must always begin from 862f80330a8SNeel Natu * the VCPU_IDLE state. This guarantees that there is only a single 863f80330a8SNeel Natu * ioctl() operating on a vcpu at any point. 864f80330a8SNeel Natu */ 865f80330a8SNeel Natu if (from_idle) { 866f80330a8SNeel Natu while (vcpu->state != VCPU_IDLE) 867f80330a8SNeel Natu msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz); 868f80330a8SNeel Natu } else { 869f80330a8SNeel Natu KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from " 870f80330a8SNeel Natu "vcpu idle state")); 871f80330a8SNeel Natu } 872f80330a8SNeel Natu 873f80330a8SNeel Natu /* 874318224bbSNeel Natu * The following state transitions are allowed: 875318224bbSNeel Natu * IDLE -> FROZEN -> IDLE 876318224bbSNeel Natu * FROZEN -> RUNNING -> FROZEN 877318224bbSNeel Natu * FROZEN -> SLEEPING -> FROZEN 878f76fc5d4SNeel Natu */ 879318224bbSNeel Natu switch (vcpu->state) { 880318224bbSNeel Natu case VCPU_IDLE: 881318224bbSNeel Natu case VCPU_RUNNING: 882318224bbSNeel Natu case VCPU_SLEEPING: 883318224bbSNeel Natu error = (newstate != VCPU_FROZEN); 884318224bbSNeel Natu break; 885318224bbSNeel Natu case VCPU_FROZEN: 886318224bbSNeel Natu error = (newstate == VCPU_FROZEN); 887318224bbSNeel Natu break; 888318224bbSNeel Natu default: 889318224bbSNeel Natu error = 1; 890318224bbSNeel Natu break; 891318224bbSNeel Natu } 892318224bbSNeel Natu 893f80330a8SNeel Natu if (error) 894f80330a8SNeel Natu return (EBUSY); 895318224bbSNeel Natu 896f80330a8SNeel Natu vcpu->state = newstate; 897f80330a8SNeel Natu if (newstate == VCPU_IDLE) 898f80330a8SNeel Natu wakeup(&vcpu->state); 899f80330a8SNeel Natu 900f80330a8SNeel Natu return (0); 901318224bbSNeel Natu } 902318224bbSNeel Natu 903318224bbSNeel Natu static void 904318224bbSNeel Natu vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 905318224bbSNeel Natu { 906318224bbSNeel Natu int error; 907318224bbSNeel Natu 908f80330a8SNeel Natu if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0) 909318224bbSNeel Natu panic("Error %d setting state to %d\n", error, newstate); 910318224bbSNeel Natu } 911318224bbSNeel Natu 912318224bbSNeel Natu static void 913318224bbSNeel Natu vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) 914318224bbSNeel Natu { 915318224bbSNeel Natu int error; 916318224bbSNeel Natu 917f80330a8SNeel Natu if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0) 918318224bbSNeel Natu panic("Error %d setting state to %d", error, newstate); 919318224bbSNeel Natu } 920318224bbSNeel Natu 9215b8a8cd1SNeel Natu static void 9225b8a8cd1SNeel Natu vm_set_rendezvous_func(struct vm *vm, vm_rendezvous_func_t func) 9235b8a8cd1SNeel Natu { 9245b8a8cd1SNeel Natu 9255b8a8cd1SNeel Natu KASSERT(mtx_owned(&vm->rendezvous_mtx), ("rendezvous_mtx not locked")); 9265b8a8cd1SNeel Natu 9275b8a8cd1SNeel Natu /* 9285b8a8cd1SNeel Natu * Update 'rendezvous_func' and execute a write memory barrier to 9295b8a8cd1SNeel Natu * ensure that it is visible across all host cpus. This is not needed 9305b8a8cd1SNeel Natu * for correctness but it does ensure that all the vcpus will notice 9315b8a8cd1SNeel Natu * that the rendezvous is requested immediately. 9325b8a8cd1SNeel Natu */ 9335b8a8cd1SNeel Natu vm->rendezvous_func = func; 9345b8a8cd1SNeel Natu wmb(); 9355b8a8cd1SNeel Natu } 9365b8a8cd1SNeel Natu 9375b8a8cd1SNeel Natu #define RENDEZVOUS_CTR0(vm, vcpuid, fmt) \ 9385b8a8cd1SNeel Natu do { \ 9395b8a8cd1SNeel Natu if (vcpuid >= 0) \ 9405b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, fmt); \ 9415b8a8cd1SNeel Natu else \ 9425b8a8cd1SNeel Natu VM_CTR0(vm, fmt); \ 9435b8a8cd1SNeel Natu } while (0) 9445b8a8cd1SNeel Natu 9455b8a8cd1SNeel Natu static void 9465b8a8cd1SNeel Natu vm_handle_rendezvous(struct vm *vm, int vcpuid) 9475b8a8cd1SNeel Natu { 9485b8a8cd1SNeel Natu 9495b8a8cd1SNeel Natu KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU), 9505b8a8cd1SNeel Natu ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid)); 9515b8a8cd1SNeel Natu 9525b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 9535b8a8cd1SNeel Natu while (vm->rendezvous_func != NULL) { 9545b8a8cd1SNeel Natu if (vcpuid != -1 && 9555b8a8cd1SNeel Natu CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus)) { 9565b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Calling rendezvous func"); 9575b8a8cd1SNeel Natu (*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg); 9585b8a8cd1SNeel Natu CPU_SET(vcpuid, &vm->rendezvous_done_cpus); 9595b8a8cd1SNeel Natu } 9605b8a8cd1SNeel Natu if (CPU_CMP(&vm->rendezvous_req_cpus, 9615b8a8cd1SNeel Natu &vm->rendezvous_done_cpus) == 0) { 9625b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Rendezvous completed"); 9635b8a8cd1SNeel Natu vm_set_rendezvous_func(vm, NULL); 9645b8a8cd1SNeel Natu wakeup(&vm->rendezvous_func); 9655b8a8cd1SNeel Natu break; 9665b8a8cd1SNeel Natu } 9675b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion"); 9685b8a8cd1SNeel Natu mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0, 9695b8a8cd1SNeel Natu "vmrndv", 0); 9705b8a8cd1SNeel Natu } 9715b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 9725b8a8cd1SNeel Natu } 9735b8a8cd1SNeel Natu 974318224bbSNeel Natu /* 975318224bbSNeel Natu * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run. 976318224bbSNeel Natu */ 977318224bbSNeel Natu static int 978becd9849SNeel Natu vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu) 979318224bbSNeel Natu { 9801c052192SNeel Natu struct vm_exit *vmexit; 981318224bbSNeel Natu struct vcpu *vcpu; 982fb03ca4eSNeel Natu int t, timo; 983318224bbSNeel Natu 984318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 985318224bbSNeel Natu 986f76fc5d4SNeel Natu vcpu_lock(vcpu); 987f76fc5d4SNeel Natu 988f76fc5d4SNeel Natu /* 989f76fc5d4SNeel Natu * Do a final check for pending NMI or interrupts before 990f76fc5d4SNeel Natu * really putting this thread to sleep. 991f76fc5d4SNeel Natu * 992f76fc5d4SNeel Natu * These interrupts could have happened any time after we 993f76fc5d4SNeel Natu * returned from VMRUN() and before we grabbed the vcpu lock. 994f76fc5d4SNeel Natu */ 9951c052192SNeel Natu if (!vm_nmi_pending(vm, vcpuid) && 9964d1e82a8SNeel Natu (intr_disabled || !vlapic_pending_intr(vcpu->vlapic, NULL))) { 997f76fc5d4SNeel Natu t = ticks; 998318224bbSNeel Natu vcpu_require_state_locked(vcpu, VCPU_SLEEPING); 9991c052192SNeel Natu if (vlapic_enabled(vcpu->vlapic)) { 1000fb03ca4eSNeel Natu /* 1001fb03ca4eSNeel Natu * XXX msleep_spin() is not interruptible so use the 1002fb03ca4eSNeel Natu * 'timo' to put an upper bound on the sleep time. 1003fb03ca4eSNeel Natu */ 1004fb03ca4eSNeel Natu timo = hz; 1005fb03ca4eSNeel Natu msleep_spin(vcpu, &vcpu->mtx, "vmidle", timo); 10061c052192SNeel Natu } else { 10071c052192SNeel Natu /* 10081c052192SNeel Natu * Spindown the vcpu if the apic is disabled and it 10091c052192SNeel Natu * had entered the halted state. 10101c052192SNeel Natu */ 1011becd9849SNeel Natu *retu = true; 10121c052192SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 10131c052192SNeel Natu vmexit->exitcode = VM_EXITCODE_SPINDOWN_CPU; 10145b8a8cd1SNeel Natu vm_deactivate_cpu(vm, vcpuid); 10151c052192SNeel Natu VCPU_CTR0(vm, vcpuid, "spinning down cpu"); 10161c052192SNeel Natu } 1017318224bbSNeel Natu vcpu_require_state_locked(vcpu, VCPU_FROZEN); 1018f76fc5d4SNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 1019f76fc5d4SNeel Natu } 1020f76fc5d4SNeel Natu vcpu_unlock(vcpu); 1021f76fc5d4SNeel Natu 1022318224bbSNeel Natu return (0); 1023318224bbSNeel Natu } 1024318224bbSNeel Natu 1025318224bbSNeel Natu static int 1026becd9849SNeel Natu vm_handle_paging(struct vm *vm, int vcpuid, bool *retu) 1027318224bbSNeel Natu { 1028318224bbSNeel Natu int rv, ftype; 1029318224bbSNeel Natu struct vm_map *map; 1030318224bbSNeel Natu struct vcpu *vcpu; 1031318224bbSNeel Natu struct vm_exit *vme; 1032318224bbSNeel Natu 1033318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1034318224bbSNeel Natu vme = &vcpu->exitinfo; 1035318224bbSNeel Natu 1036318224bbSNeel Natu ftype = vme->u.paging.fault_type; 1037318224bbSNeel Natu KASSERT(ftype == VM_PROT_READ || 1038318224bbSNeel Natu ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE, 1039318224bbSNeel Natu ("vm_handle_paging: invalid fault_type %d", ftype)); 1040318224bbSNeel Natu 1041318224bbSNeel Natu if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { 1042318224bbSNeel Natu rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace), 1043318224bbSNeel Natu vme->u.paging.gpa, ftype); 1044318224bbSNeel Natu if (rv == 0) 1045318224bbSNeel Natu goto done; 1046318224bbSNeel Natu } 1047318224bbSNeel Natu 1048318224bbSNeel Natu map = &vm->vmspace->vm_map; 1049318224bbSNeel Natu rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL); 1050318224bbSNeel Natu 1051513c8d33SNeel Natu VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, " 1052513c8d33SNeel Natu "ftype = %d", rv, vme->u.paging.gpa, ftype); 1053318224bbSNeel Natu 1054318224bbSNeel Natu if (rv != KERN_SUCCESS) 1055318224bbSNeel Natu return (EFAULT); 1056318224bbSNeel Natu done: 1057318224bbSNeel Natu /* restart execution at the faulting instruction */ 1058318224bbSNeel Natu vme->inst_length = 0; 1059318224bbSNeel Natu 1060318224bbSNeel Natu return (0); 1061318224bbSNeel Natu } 1062318224bbSNeel Natu 1063318224bbSNeel Natu static int 1064becd9849SNeel Natu vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu) 1065318224bbSNeel Natu { 1066318224bbSNeel Natu struct vie *vie; 1067318224bbSNeel Natu struct vcpu *vcpu; 1068318224bbSNeel Natu struct vm_exit *vme; 1069318224bbSNeel Natu int error, inst_length; 1070318224bbSNeel Natu uint64_t rip, gla, gpa, cr3; 107100f3efe1SJohn Baldwin enum vie_cpu_mode cpu_mode; 107200f3efe1SJohn Baldwin enum vie_paging_mode paging_mode; 1073565bbb86SNeel Natu mem_region_read_t mread; 1074565bbb86SNeel Natu mem_region_write_t mwrite; 1075318224bbSNeel Natu 1076318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1077318224bbSNeel Natu vme = &vcpu->exitinfo; 1078318224bbSNeel Natu 1079318224bbSNeel Natu rip = vme->rip; 1080318224bbSNeel Natu inst_length = vme->inst_length; 1081318224bbSNeel Natu 1082318224bbSNeel Natu gla = vme->u.inst_emul.gla; 1083318224bbSNeel Natu gpa = vme->u.inst_emul.gpa; 1084318224bbSNeel Natu cr3 = vme->u.inst_emul.cr3; 108500f3efe1SJohn Baldwin cpu_mode = vme->u.inst_emul.cpu_mode; 108600f3efe1SJohn Baldwin paging_mode = vme->u.inst_emul.paging_mode; 1087318224bbSNeel Natu vie = &vme->u.inst_emul.vie; 1088318224bbSNeel Natu 1089318224bbSNeel Natu vie_init(vie); 1090318224bbSNeel Natu 1091318224bbSNeel Natu /* Fetch, decode and emulate the faulting instruction */ 109200f3efe1SJohn Baldwin if (vmm_fetch_instruction(vm, vcpuid, rip, inst_length, cr3, 109300f3efe1SJohn Baldwin paging_mode, vie) != 0) 1094318224bbSNeel Natu return (EFAULT); 1095318224bbSNeel Natu 109600f3efe1SJohn Baldwin if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, vie) != 0) 1097318224bbSNeel Natu return (EFAULT); 1098318224bbSNeel Natu 109908e3ff32SNeel Natu /* return to userland unless this is an in-kernel emulated device */ 1100565bbb86SNeel Natu if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) { 1101565bbb86SNeel Natu mread = lapic_mmio_read; 1102565bbb86SNeel Natu mwrite = lapic_mmio_write; 1103565bbb86SNeel Natu } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) { 1104565bbb86SNeel Natu mread = vioapic_mmio_read; 1105565bbb86SNeel Natu mwrite = vioapic_mmio_write; 110608e3ff32SNeel Natu } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) { 110708e3ff32SNeel Natu mread = vhpet_mmio_read; 110808e3ff32SNeel Natu mwrite = vhpet_mmio_write; 1109565bbb86SNeel Natu } else { 1110becd9849SNeel Natu *retu = true; 1111318224bbSNeel Natu return (0); 1112318224bbSNeel Natu } 1113318224bbSNeel Natu 1114becd9849SNeel Natu error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, mread, mwrite, 1115becd9849SNeel Natu retu); 1116318224bbSNeel Natu 1117318224bbSNeel Natu return (error); 1118318224bbSNeel Natu } 1119318224bbSNeel Natu 1120318224bbSNeel Natu int 1121318224bbSNeel Natu vm_run(struct vm *vm, struct vm_run *vmrun) 1122318224bbSNeel Natu { 1123318224bbSNeel Natu int error, vcpuid; 1124318224bbSNeel Natu struct vcpu *vcpu; 1125318224bbSNeel Natu struct pcb *pcb; 1126318224bbSNeel Natu uint64_t tscval, rip; 1127318224bbSNeel Natu struct vm_exit *vme; 1128becd9849SNeel Natu bool retu, intr_disabled; 1129318224bbSNeel Natu pmap_t pmap; 1130318224bbSNeel Natu 1131318224bbSNeel Natu vcpuid = vmrun->cpuid; 1132318224bbSNeel Natu 1133318224bbSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1134318224bbSNeel Natu return (EINVAL); 1135318224bbSNeel Natu 1136318224bbSNeel Natu pmap = vmspace_pmap(vm->vmspace); 1137318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1138318224bbSNeel Natu vme = &vcpu->exitinfo; 1139318224bbSNeel Natu rip = vmrun->rip; 1140318224bbSNeel Natu restart: 1141318224bbSNeel Natu critical_enter(); 1142318224bbSNeel Natu 1143318224bbSNeel Natu KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 1144318224bbSNeel Natu ("vm_run: absurd pm_active")); 1145318224bbSNeel Natu 1146318224bbSNeel Natu tscval = rdtsc(); 1147318224bbSNeel Natu 1148318224bbSNeel Natu pcb = PCPU_GET(curpcb); 1149318224bbSNeel Natu set_pcb_flags(pcb, PCB_FULL_IRET); 1150318224bbSNeel Natu 1151318224bbSNeel Natu restore_guest_msrs(vm, vcpuid); 1152318224bbSNeel Natu restore_guest_fpustate(vcpu); 1153318224bbSNeel Natu 1154318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_RUNNING); 1155318224bbSNeel Natu vcpu->hostcpu = curcpu; 11565b8a8cd1SNeel Natu error = VMRUN(vm->cookie, vcpuid, rip, pmap, &vm->rendezvous_func); 1157318224bbSNeel Natu vcpu->hostcpu = NOCPU; 1158318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_FROZEN); 1159318224bbSNeel Natu 1160318224bbSNeel Natu save_guest_fpustate(vcpu); 1161318224bbSNeel Natu restore_host_msrs(vm, vcpuid); 1162318224bbSNeel Natu 1163318224bbSNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 1164318224bbSNeel Natu 1165318224bbSNeel Natu critical_exit(); 1166318224bbSNeel Natu 1167318224bbSNeel Natu if (error == 0) { 1168becd9849SNeel Natu retu = false; 1169318224bbSNeel Natu switch (vme->exitcode) { 117030b94db8SNeel Natu case VM_EXITCODE_IOAPIC_EOI: 117130b94db8SNeel Natu vioapic_process_eoi(vm, vcpuid, 117230b94db8SNeel Natu vme->u.ioapic_eoi.vector); 117330b94db8SNeel Natu break; 11745b8a8cd1SNeel Natu case VM_EXITCODE_RENDEZVOUS: 11755b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 11765b8a8cd1SNeel Natu error = 0; 11775b8a8cd1SNeel Natu break; 1178318224bbSNeel Natu case VM_EXITCODE_HLT: 1179becd9849SNeel Natu intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0); 11801c052192SNeel Natu error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu); 1181318224bbSNeel Natu break; 1182318224bbSNeel Natu case VM_EXITCODE_PAGING: 1183318224bbSNeel Natu error = vm_handle_paging(vm, vcpuid, &retu); 1184318224bbSNeel Natu break; 1185318224bbSNeel Natu case VM_EXITCODE_INST_EMUL: 1186318224bbSNeel Natu error = vm_handle_inst_emul(vm, vcpuid, &retu); 1187318224bbSNeel Natu break; 1188318224bbSNeel Natu default: 1189becd9849SNeel Natu retu = true; /* handled in userland */ 1190318224bbSNeel Natu break; 1191318224bbSNeel Natu } 1192318224bbSNeel Natu } 1193318224bbSNeel Natu 1194becd9849SNeel Natu if (error == 0 && retu == false) { 1195f76fc5d4SNeel Natu rip = vme->rip + vme->inst_length; 1196f76fc5d4SNeel Natu goto restart; 1197f76fc5d4SNeel Natu } 1198f76fc5d4SNeel Natu 1199318224bbSNeel Natu /* copy the exit information */ 1200318224bbSNeel Natu bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit)); 1201366f6083SPeter Grehan return (error); 1202366f6083SPeter Grehan } 1203366f6083SPeter Grehan 1204366f6083SPeter Grehan int 1205366f6083SPeter Grehan vm_inject_event(struct vm *vm, int vcpuid, int type, 1206366f6083SPeter Grehan int vector, uint32_t code, int code_valid) 1207366f6083SPeter Grehan { 1208366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1209366f6083SPeter Grehan return (EINVAL); 1210366f6083SPeter Grehan 1211366f6083SPeter Grehan if ((type > VM_EVENT_NONE && type < VM_EVENT_MAX) == 0) 1212366f6083SPeter Grehan return (EINVAL); 1213366f6083SPeter Grehan 1214366f6083SPeter Grehan if (vector < 0 || vector > 255) 1215366f6083SPeter Grehan return (EINVAL); 1216366f6083SPeter Grehan 1217366f6083SPeter Grehan return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid)); 1218366f6083SPeter Grehan } 1219366f6083SPeter Grehan 122061592433SNeel Natu static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 1221366f6083SPeter Grehan 1222f352ff0cSNeel Natu int 1223f352ff0cSNeel Natu vm_inject_nmi(struct vm *vm, int vcpuid) 1224f352ff0cSNeel Natu { 1225f352ff0cSNeel Natu struct vcpu *vcpu; 1226f352ff0cSNeel Natu 1227f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1228366f6083SPeter Grehan return (EINVAL); 1229366f6083SPeter Grehan 1230f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1231f352ff0cSNeel Natu 1232f352ff0cSNeel Natu vcpu->nmi_pending = 1; 1233de5ea6b6SNeel Natu vcpu_notify_event(vm, vcpuid, false); 1234f352ff0cSNeel Natu return (0); 1235f352ff0cSNeel Natu } 1236f352ff0cSNeel Natu 1237f352ff0cSNeel Natu int 1238f352ff0cSNeel Natu vm_nmi_pending(struct vm *vm, int vcpuid) 1239f352ff0cSNeel Natu { 1240f352ff0cSNeel Natu struct vcpu *vcpu; 1241f352ff0cSNeel Natu 1242f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1243f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1244f352ff0cSNeel Natu 1245f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1246f352ff0cSNeel Natu 1247f352ff0cSNeel Natu return (vcpu->nmi_pending); 1248f352ff0cSNeel Natu } 1249f352ff0cSNeel Natu 1250f352ff0cSNeel Natu void 1251f352ff0cSNeel Natu vm_nmi_clear(struct vm *vm, int vcpuid) 1252f352ff0cSNeel Natu { 1253f352ff0cSNeel Natu struct vcpu *vcpu; 1254f352ff0cSNeel Natu 1255f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1256f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1257f352ff0cSNeel Natu 1258f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1259f352ff0cSNeel Natu 1260f352ff0cSNeel Natu if (vcpu->nmi_pending == 0) 1261f352ff0cSNeel Natu panic("vm_nmi_clear: inconsistent nmi_pending state"); 1262f352ff0cSNeel Natu 1263f352ff0cSNeel Natu vcpu->nmi_pending = 0; 1264f352ff0cSNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 1265366f6083SPeter Grehan } 1266366f6083SPeter Grehan 1267366f6083SPeter Grehan int 1268366f6083SPeter Grehan vm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 1269366f6083SPeter Grehan { 1270366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 1271366f6083SPeter Grehan return (EINVAL); 1272366f6083SPeter Grehan 1273366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 1274366f6083SPeter Grehan return (EINVAL); 1275366f6083SPeter Grehan 1276366f6083SPeter Grehan return (VMGETCAP(vm->cookie, vcpu, type, retval)); 1277366f6083SPeter Grehan } 1278366f6083SPeter Grehan 1279366f6083SPeter Grehan int 1280366f6083SPeter Grehan vm_set_capability(struct vm *vm, int vcpu, int type, int val) 1281366f6083SPeter Grehan { 1282366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 1283366f6083SPeter Grehan return (EINVAL); 1284366f6083SPeter Grehan 1285366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 1286366f6083SPeter Grehan return (EINVAL); 1287366f6083SPeter Grehan 1288366f6083SPeter Grehan return (VMSETCAP(vm->cookie, vcpu, type, val)); 1289366f6083SPeter Grehan } 1290366f6083SPeter Grehan 1291366f6083SPeter Grehan uint64_t * 1292366f6083SPeter Grehan vm_guest_msrs(struct vm *vm, int cpu) 1293366f6083SPeter Grehan { 1294366f6083SPeter Grehan return (vm->vcpu[cpu].guest_msrs); 1295366f6083SPeter Grehan } 1296366f6083SPeter Grehan 1297366f6083SPeter Grehan struct vlapic * 1298366f6083SPeter Grehan vm_lapic(struct vm *vm, int cpu) 1299366f6083SPeter Grehan { 1300366f6083SPeter Grehan return (vm->vcpu[cpu].vlapic); 1301366f6083SPeter Grehan } 1302366f6083SPeter Grehan 1303565bbb86SNeel Natu struct vioapic * 1304565bbb86SNeel Natu vm_ioapic(struct vm *vm) 1305565bbb86SNeel Natu { 1306565bbb86SNeel Natu 1307565bbb86SNeel Natu return (vm->vioapic); 1308565bbb86SNeel Natu } 1309565bbb86SNeel Natu 131008e3ff32SNeel Natu struct vhpet * 131108e3ff32SNeel Natu vm_hpet(struct vm *vm) 131208e3ff32SNeel Natu { 131308e3ff32SNeel Natu 131408e3ff32SNeel Natu return (vm->vhpet); 131508e3ff32SNeel Natu } 131608e3ff32SNeel Natu 1317366f6083SPeter Grehan boolean_t 1318366f6083SPeter Grehan vmm_is_pptdev(int bus, int slot, int func) 1319366f6083SPeter Grehan { 132007044a96SNeel Natu int found, i, n; 132107044a96SNeel Natu int b, s, f; 1322366f6083SPeter Grehan char *val, *cp, *cp2; 1323366f6083SPeter Grehan 1324366f6083SPeter Grehan /* 132507044a96SNeel Natu * XXX 132607044a96SNeel Natu * The length of an environment variable is limited to 128 bytes which 132707044a96SNeel Natu * puts an upper limit on the number of passthru devices that may be 132807044a96SNeel Natu * specified using a single environment variable. 132907044a96SNeel Natu * 133007044a96SNeel Natu * Work around this by scanning multiple environment variable 133107044a96SNeel Natu * names instead of a single one - yuck! 1332366f6083SPeter Grehan */ 133307044a96SNeel Natu const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL }; 133407044a96SNeel Natu 133507044a96SNeel Natu /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */ 1336366f6083SPeter Grehan found = 0; 133707044a96SNeel Natu for (i = 0; names[i] != NULL && !found; i++) { 133807044a96SNeel Natu cp = val = getenv(names[i]); 1339366f6083SPeter Grehan while (cp != NULL && *cp != '\0') { 1340366f6083SPeter Grehan if ((cp2 = strchr(cp, ' ')) != NULL) 1341366f6083SPeter Grehan *cp2 = '\0'; 1342366f6083SPeter Grehan 1343366f6083SPeter Grehan n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 1344366f6083SPeter Grehan if (n == 3 && bus == b && slot == s && func == f) { 1345366f6083SPeter Grehan found = 1; 1346366f6083SPeter Grehan break; 1347366f6083SPeter Grehan } 1348366f6083SPeter Grehan 1349366f6083SPeter Grehan if (cp2 != NULL) 1350366f6083SPeter Grehan *cp2++ = ' '; 1351366f6083SPeter Grehan 1352366f6083SPeter Grehan cp = cp2; 1353366f6083SPeter Grehan } 1354366f6083SPeter Grehan freeenv(val); 135507044a96SNeel Natu } 1356366f6083SPeter Grehan return (found); 1357366f6083SPeter Grehan } 1358366f6083SPeter Grehan 1359366f6083SPeter Grehan void * 1360366f6083SPeter Grehan vm_iommu_domain(struct vm *vm) 1361366f6083SPeter Grehan { 1362366f6083SPeter Grehan 1363366f6083SPeter Grehan return (vm->iommu); 1364366f6083SPeter Grehan } 1365366f6083SPeter Grehan 136675dd3366SNeel Natu int 1367f80330a8SNeel Natu vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate, 1368f80330a8SNeel Natu bool from_idle) 1369366f6083SPeter Grehan { 137075dd3366SNeel Natu int error; 1371366f6083SPeter Grehan struct vcpu *vcpu; 1372366f6083SPeter Grehan 1373366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1374366f6083SPeter Grehan panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 1375366f6083SPeter Grehan 1376366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 1377366f6083SPeter Grehan 137875dd3366SNeel Natu vcpu_lock(vcpu); 1379f80330a8SNeel Natu error = vcpu_set_state_locked(vcpu, newstate, from_idle); 138075dd3366SNeel Natu vcpu_unlock(vcpu); 138175dd3366SNeel Natu 138275dd3366SNeel Natu return (error); 138375dd3366SNeel Natu } 138475dd3366SNeel Natu 138575dd3366SNeel Natu enum vcpu_state 1386d3c11f40SPeter Grehan vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu) 1387366f6083SPeter Grehan { 1388366f6083SPeter Grehan struct vcpu *vcpu; 138975dd3366SNeel Natu enum vcpu_state state; 1390366f6083SPeter Grehan 1391366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1392366f6083SPeter Grehan panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 1393366f6083SPeter Grehan 1394366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 1395366f6083SPeter Grehan 139675dd3366SNeel Natu vcpu_lock(vcpu); 139775dd3366SNeel Natu state = vcpu->state; 1398d3c11f40SPeter Grehan if (hostcpu != NULL) 1399d3c11f40SPeter Grehan *hostcpu = vcpu->hostcpu; 140075dd3366SNeel Natu vcpu_unlock(vcpu); 1401366f6083SPeter Grehan 140275dd3366SNeel Natu return (state); 1403366f6083SPeter Grehan } 1404366f6083SPeter Grehan 1405366f6083SPeter Grehan void 1406366f6083SPeter Grehan vm_activate_cpu(struct vm *vm, int vcpuid) 1407366f6083SPeter Grehan { 1408366f6083SPeter Grehan 1409366f6083SPeter Grehan if (vcpuid >= 0 && vcpuid < VM_MAXCPU) 1410a5615c90SPeter Grehan CPU_SET(vcpuid, &vm->active_cpus); 1411366f6083SPeter Grehan } 1412366f6083SPeter Grehan 14135b8a8cd1SNeel Natu static void 14145b8a8cd1SNeel Natu vm_deactivate_cpu(struct vm *vm, int vcpuid) 14155b8a8cd1SNeel Natu { 14165b8a8cd1SNeel Natu 14175b8a8cd1SNeel Natu if (vcpuid >= 0 && vcpuid < VM_MAXCPU) 14185b8a8cd1SNeel Natu CPU_CLR(vcpuid, &vm->active_cpus); 14195b8a8cd1SNeel Natu } 14205b8a8cd1SNeel Natu 1421a5615c90SPeter Grehan cpuset_t 1422366f6083SPeter Grehan vm_active_cpus(struct vm *vm) 1423366f6083SPeter Grehan { 1424366f6083SPeter Grehan 1425366f6083SPeter Grehan return (vm->active_cpus); 1426366f6083SPeter Grehan } 1427366f6083SPeter Grehan 1428366f6083SPeter Grehan void * 1429366f6083SPeter Grehan vcpu_stats(struct vm *vm, int vcpuid) 1430366f6083SPeter Grehan { 1431366f6083SPeter Grehan 1432366f6083SPeter Grehan return (vm->vcpu[vcpuid].stats); 1433366f6083SPeter Grehan } 1434e9027382SNeel Natu 1435e9027382SNeel Natu int 1436e9027382SNeel Natu vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 1437e9027382SNeel Natu { 1438e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1439e9027382SNeel Natu return (EINVAL); 1440e9027382SNeel Natu 1441e9027382SNeel Natu *state = vm->vcpu[vcpuid].x2apic_state; 1442e9027382SNeel Natu 1443e9027382SNeel Natu return (0); 1444e9027382SNeel Natu } 1445e9027382SNeel Natu 1446e9027382SNeel Natu int 1447e9027382SNeel Natu vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 1448e9027382SNeel Natu { 1449e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1450e9027382SNeel Natu return (EINVAL); 1451e9027382SNeel Natu 14523f23d3caSNeel Natu if (state >= X2APIC_STATE_LAST) 1453e9027382SNeel Natu return (EINVAL); 1454e9027382SNeel Natu 1455e9027382SNeel Natu vm->vcpu[vcpuid].x2apic_state = state; 1456e9027382SNeel Natu 145773820fb0SNeel Natu vlapic_set_x2apic_state(vm, vcpuid, state); 145873820fb0SNeel Natu 1459e9027382SNeel Natu return (0); 1460e9027382SNeel Natu } 146175dd3366SNeel Natu 146222821874SNeel Natu /* 146322821874SNeel Natu * This function is called to ensure that a vcpu "sees" a pending event 146422821874SNeel Natu * as soon as possible: 146522821874SNeel Natu * - If the vcpu thread is sleeping then it is woken up. 146622821874SNeel Natu * - If the vcpu is running on a different host_cpu then an IPI will be directed 146722821874SNeel Natu * to the host_cpu to cause the vcpu to trap into the hypervisor. 146822821874SNeel Natu */ 146975dd3366SNeel Natu void 1470de5ea6b6SNeel Natu vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr) 147175dd3366SNeel Natu { 147275dd3366SNeel Natu int hostcpu; 147375dd3366SNeel Natu struct vcpu *vcpu; 147475dd3366SNeel Natu 147575dd3366SNeel Natu vcpu = &vm->vcpu[vcpuid]; 147675dd3366SNeel Natu 1477f76fc5d4SNeel Natu vcpu_lock(vcpu); 147875dd3366SNeel Natu hostcpu = vcpu->hostcpu; 1479f76fc5d4SNeel Natu if (hostcpu == NOCPU) { 1480318224bbSNeel Natu if (vcpu->state == VCPU_SLEEPING) 1481f76fc5d4SNeel Natu wakeup_one(vcpu); 1482f76fc5d4SNeel Natu } else { 1483f76fc5d4SNeel Natu if (vcpu->state != VCPU_RUNNING) 1484f76fc5d4SNeel Natu panic("invalid vcpu state %d", vcpu->state); 1485de5ea6b6SNeel Natu if (hostcpu != curcpu) { 1486de5ea6b6SNeel Natu if (lapic_intr) 1487add611fdSNeel Natu vlapic_post_intr(vcpu->vlapic, hostcpu, 1488add611fdSNeel Natu vmm_ipinum); 1489de5ea6b6SNeel Natu else 149075dd3366SNeel Natu ipi_cpu(hostcpu, vmm_ipinum); 149175dd3366SNeel Natu } 1492de5ea6b6SNeel Natu } 1493f76fc5d4SNeel Natu vcpu_unlock(vcpu); 1494f76fc5d4SNeel Natu } 1495318224bbSNeel Natu 1496318224bbSNeel Natu struct vmspace * 1497318224bbSNeel Natu vm_get_vmspace(struct vm *vm) 1498318224bbSNeel Natu { 1499318224bbSNeel Natu 1500318224bbSNeel Natu return (vm->vmspace); 1501318224bbSNeel Natu } 1502565bbb86SNeel Natu 1503565bbb86SNeel Natu int 1504565bbb86SNeel Natu vm_apicid2vcpuid(struct vm *vm, int apicid) 1505565bbb86SNeel Natu { 1506565bbb86SNeel Natu /* 1507565bbb86SNeel Natu * XXX apic id is assumed to be numerically identical to vcpu id 1508565bbb86SNeel Natu */ 1509565bbb86SNeel Natu return (apicid); 1510565bbb86SNeel Natu } 15115b8a8cd1SNeel Natu 15125b8a8cd1SNeel Natu void 15135b8a8cd1SNeel Natu vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest, 15145b8a8cd1SNeel Natu vm_rendezvous_func_t func, void *arg) 15155b8a8cd1SNeel Natu { 15165b8a8cd1SNeel Natu /* 15175b8a8cd1SNeel Natu * Enforce that this function is called without any locks 15185b8a8cd1SNeel Natu */ 15195b8a8cd1SNeel Natu WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous"); 15205b8a8cd1SNeel Natu KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU), 15215b8a8cd1SNeel Natu ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid)); 15225b8a8cd1SNeel Natu 15235b8a8cd1SNeel Natu restart: 15245b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 15255b8a8cd1SNeel Natu if (vm->rendezvous_func != NULL) { 15265b8a8cd1SNeel Natu /* 15275b8a8cd1SNeel Natu * If a rendezvous is already in progress then we need to 15285b8a8cd1SNeel Natu * call the rendezvous handler in case this 'vcpuid' is one 15295b8a8cd1SNeel Natu * of the targets of the rendezvous. 15305b8a8cd1SNeel Natu */ 15315b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress"); 15325b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 15335b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 15345b8a8cd1SNeel Natu goto restart; 15355b8a8cd1SNeel Natu } 15365b8a8cd1SNeel Natu KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous " 15375b8a8cd1SNeel Natu "rendezvous is still in progress")); 15385b8a8cd1SNeel Natu 15395b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous"); 15405b8a8cd1SNeel Natu vm->rendezvous_req_cpus = dest; 15415b8a8cd1SNeel Natu CPU_ZERO(&vm->rendezvous_done_cpus); 15425b8a8cd1SNeel Natu vm->rendezvous_arg = arg; 15435b8a8cd1SNeel Natu vm_set_rendezvous_func(vm, func); 15445b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 15455b8a8cd1SNeel Natu 15465b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 15475b8a8cd1SNeel Natu } 1548