1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 3338f1b189SPeter Grehan #include <sys/systm.h> 34366f6083SPeter Grehan #include <sys/kernel.h> 35366f6083SPeter Grehan #include <sys/module.h> 36366f6083SPeter Grehan #include <sys/sysctl.h> 37366f6083SPeter Grehan #include <sys/malloc.h> 38366f6083SPeter Grehan #include <sys/pcpu.h> 39366f6083SPeter Grehan #include <sys/lock.h> 40366f6083SPeter Grehan #include <sys/mutex.h> 41366f6083SPeter Grehan #include <sys/proc.h> 42366f6083SPeter Grehan #include <sys/sched.h> 43366f6083SPeter Grehan #include <sys/smp.h> 44366f6083SPeter Grehan #include <sys/systm.h> 45366f6083SPeter Grehan 46366f6083SPeter Grehan #include <vm/vm.h> 47366f6083SPeter Grehan 48366f6083SPeter Grehan #include <machine/vm.h> 49366f6083SPeter Grehan #include <machine/pcb.h> 5075dd3366SNeel Natu #include <machine/smp.h> 5134a6b2d6SJohn Baldwin #include <x86/apicreg.h> 52366f6083SPeter Grehan 53366f6083SPeter Grehan #include <machine/vmm.h> 54b01c2033SNeel Natu #include "vmm_host.h" 55366f6083SPeter Grehan #include "vmm_mem.h" 56366f6083SPeter Grehan #include "vmm_util.h" 57366f6083SPeter Grehan #include <machine/vmm_dev.h> 58366f6083SPeter Grehan #include "vlapic.h" 59366f6083SPeter Grehan #include "vmm_msr.h" 60366f6083SPeter Grehan #include "vmm_ipi.h" 61366f6083SPeter Grehan #include "vmm_stat.h" 62f76fc5d4SNeel Natu #include "vmm_lapic.h" 63366f6083SPeter Grehan 64366f6083SPeter Grehan #include "io/ppt.h" 65366f6083SPeter Grehan #include "io/iommu.h" 66366f6083SPeter Grehan 67366f6083SPeter Grehan struct vlapic; 68366f6083SPeter Grehan 69366f6083SPeter Grehan struct vcpu { 70366f6083SPeter Grehan int flags; 7175dd3366SNeel Natu enum vcpu_state state; 7275dd3366SNeel Natu struct mtx mtx; 73366f6083SPeter Grehan int pincpu; /* host cpuid this vcpu is bound to */ 74366f6083SPeter Grehan int hostcpu; /* host cpuid this vcpu last ran on */ 75366f6083SPeter Grehan uint64_t guest_msrs[VMM_MSR_NUM]; 76366f6083SPeter Grehan struct vlapic *vlapic; 77366f6083SPeter Grehan int vcpuid; 7838f1b189SPeter Grehan struct savefpu *guestfpu; /* guest fpu state */ 79366f6083SPeter Grehan void *stats; 8098ed632cSNeel Natu struct vm_exit exitinfo; 81e9027382SNeel Natu enum x2apic_state x2apic_state; 82f352ff0cSNeel Natu int nmi_pending; 83366f6083SPeter Grehan }; 84366f6083SPeter Grehan #define VCPU_F_PINNED 0x0001 85366f6083SPeter Grehan 86366f6083SPeter Grehan #define VCPU_PINCPU(vm, vcpuid) \ 87366f6083SPeter Grehan ((vm->vcpu[vcpuid].flags & VCPU_F_PINNED) ? vm->vcpu[vcpuid].pincpu : -1) 88366f6083SPeter Grehan 89366f6083SPeter Grehan #define VCPU_UNPIN(vm, vcpuid) (vm->vcpu[vcpuid].flags &= ~VCPU_F_PINNED) 90366f6083SPeter Grehan 91366f6083SPeter Grehan #define VCPU_PIN(vm, vcpuid, host_cpuid) \ 92366f6083SPeter Grehan do { \ 93366f6083SPeter Grehan vm->vcpu[vcpuid].flags |= VCPU_F_PINNED; \ 94366f6083SPeter Grehan vm->vcpu[vcpuid].pincpu = host_cpuid; \ 95366f6083SPeter Grehan } while(0) 96366f6083SPeter Grehan 97f76fc5d4SNeel Natu #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 98f76fc5d4SNeel Natu #define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 99f76fc5d4SNeel Natu #define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 10075dd3366SNeel Natu 101366f6083SPeter Grehan #define VM_MAX_MEMORY_SEGMENTS 2 102366f6083SPeter Grehan 103366f6083SPeter Grehan struct vm { 104366f6083SPeter Grehan void *cookie; /* processor-specific data */ 105366f6083SPeter Grehan void *iommu; /* iommu-specific data */ 106366f6083SPeter Grehan struct vcpu vcpu[VM_MAXCPU]; 107366f6083SPeter Grehan int num_mem_segs; 108366f6083SPeter Grehan struct vm_memory_segment mem_segs[VM_MAX_MEMORY_SEGMENTS]; 109366f6083SPeter Grehan char name[VM_MAX_NAMELEN]; 110366f6083SPeter Grehan 111366f6083SPeter Grehan /* 112a5615c90SPeter Grehan * Set of active vcpus. 113366f6083SPeter Grehan * An active vcpu is one that has been started implicitly (BSP) or 114366f6083SPeter Grehan * explicitly (AP) by sending it a startup ipi. 115366f6083SPeter Grehan */ 116a5615c90SPeter Grehan cpuset_t active_cpus; 117366f6083SPeter Grehan }; 118366f6083SPeter Grehan 119366f6083SPeter Grehan static struct vmm_ops *ops; 120366f6083SPeter Grehan #define VMM_INIT() (ops != NULL ? (*ops->init)() : 0) 121366f6083SPeter Grehan #define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 122366f6083SPeter Grehan 123366f6083SPeter Grehan #define VMINIT(vm) (ops != NULL ? (*ops->vminit)(vm): NULL) 12498ed632cSNeel Natu #define VMRUN(vmi, vcpu, rip) \ 12598ed632cSNeel Natu (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip) : ENXIO) 126366f6083SPeter Grehan #define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 127bda273f2SNeel Natu #define VMMMAP_SET(vmi, gpa, hpa, len, attr, prot, spm) \ 128bda273f2SNeel Natu (ops != NULL ? \ 129bda273f2SNeel Natu (*ops->vmmmap_set)(vmi, gpa, hpa, len, attr, prot, spm) : \ 130bda273f2SNeel Natu ENXIO) 131bda273f2SNeel Natu #define VMMMAP_GET(vmi, gpa) \ 132bda273f2SNeel Natu (ops != NULL ? (*ops->vmmmap_get)(vmi, gpa) : ENXIO) 133366f6083SPeter Grehan #define VMGETREG(vmi, vcpu, num, retval) \ 134366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 135366f6083SPeter Grehan #define VMSETREG(vmi, vcpu, num, val) \ 136366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 137366f6083SPeter Grehan #define VMGETDESC(vmi, vcpu, num, desc) \ 138366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 139366f6083SPeter Grehan #define VMSETDESC(vmi, vcpu, num, desc) \ 140366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 141366f6083SPeter Grehan #define VMINJECT(vmi, vcpu, type, vec, ec, ecv) \ 142366f6083SPeter Grehan (ops != NULL ? (*ops->vminject)(vmi, vcpu, type, vec, ec, ecv) : ENXIO) 143366f6083SPeter Grehan #define VMGETCAP(vmi, vcpu, num, retval) \ 144366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 145366f6083SPeter Grehan #define VMSETCAP(vmi, vcpu, num, val) \ 146366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 147366f6083SPeter Grehan 148014a52f3SNeel Natu #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 149014a52f3SNeel Natu #define fpu_stop_emulating() clts() 150366f6083SPeter Grehan 151366f6083SPeter Grehan static MALLOC_DEFINE(M_VM, "vm", "vm"); 152366f6083SPeter Grehan CTASSERT(VMM_MSR_NUM <= 64); /* msr_mask can keep track of up to 64 msrs */ 153366f6083SPeter Grehan 154366f6083SPeter Grehan /* statistics */ 155366f6083SPeter Grehan static VMM_STAT_DEFINE(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 156366f6083SPeter Grehan 157366f6083SPeter Grehan static void 158366f6083SPeter Grehan vcpu_cleanup(struct vcpu *vcpu) 159366f6083SPeter Grehan { 160366f6083SPeter Grehan vlapic_cleanup(vcpu->vlapic); 161366f6083SPeter Grehan vmm_stat_free(vcpu->stats); 16238f1b189SPeter Grehan fpu_save_area_free(vcpu->guestfpu); 163366f6083SPeter Grehan } 164366f6083SPeter Grehan 165366f6083SPeter Grehan static void 166366f6083SPeter Grehan vcpu_init(struct vm *vm, uint32_t vcpu_id) 167366f6083SPeter Grehan { 168366f6083SPeter Grehan struct vcpu *vcpu; 169366f6083SPeter Grehan 170366f6083SPeter Grehan vcpu = &vm->vcpu[vcpu_id]; 171366f6083SPeter Grehan 17275dd3366SNeel Natu vcpu_lock_init(vcpu); 17375dd3366SNeel Natu vcpu->hostcpu = NOCPU; 174366f6083SPeter Grehan vcpu->vcpuid = vcpu_id; 175366f6083SPeter Grehan vcpu->vlapic = vlapic_init(vm, vcpu_id); 17673820fb0SNeel Natu vm_set_x2apic_state(vm, vcpu_id, X2APIC_ENABLED); 17738f1b189SPeter Grehan vcpu->guestfpu = fpu_save_area_alloc(); 17838f1b189SPeter Grehan fpu_save_area_reset(vcpu->guestfpu); 179366f6083SPeter Grehan vcpu->stats = vmm_stat_alloc(); 180366f6083SPeter Grehan } 181366f6083SPeter Grehan 18298ed632cSNeel Natu struct vm_exit * 18398ed632cSNeel Natu vm_exitinfo(struct vm *vm, int cpuid) 18498ed632cSNeel Natu { 18598ed632cSNeel Natu struct vcpu *vcpu; 18698ed632cSNeel Natu 18798ed632cSNeel Natu if (cpuid < 0 || cpuid >= VM_MAXCPU) 18898ed632cSNeel Natu panic("vm_exitinfo: invalid cpuid %d", cpuid); 18998ed632cSNeel Natu 19098ed632cSNeel Natu vcpu = &vm->vcpu[cpuid]; 19198ed632cSNeel Natu 19298ed632cSNeel Natu return (&vcpu->exitinfo); 19398ed632cSNeel Natu } 19498ed632cSNeel Natu 195366f6083SPeter Grehan static int 196366f6083SPeter Grehan vmm_init(void) 197366f6083SPeter Grehan { 198366f6083SPeter Grehan int error; 199366f6083SPeter Grehan 200b01c2033SNeel Natu vmm_host_state_init(); 201366f6083SPeter Grehan vmm_ipi_init(); 202366f6083SPeter Grehan 203366f6083SPeter Grehan error = vmm_mem_init(); 204366f6083SPeter Grehan if (error) 205366f6083SPeter Grehan return (error); 206366f6083SPeter Grehan 207366f6083SPeter Grehan if (vmm_is_intel()) 208366f6083SPeter Grehan ops = &vmm_ops_intel; 209366f6083SPeter Grehan else if (vmm_is_amd()) 210366f6083SPeter Grehan ops = &vmm_ops_amd; 211366f6083SPeter Grehan else 212366f6083SPeter Grehan return (ENXIO); 213366f6083SPeter Grehan 214366f6083SPeter Grehan vmm_msr_init(); 215366f6083SPeter Grehan 216366f6083SPeter Grehan return (VMM_INIT()); 217366f6083SPeter Grehan } 218366f6083SPeter Grehan 219366f6083SPeter Grehan static int 220366f6083SPeter Grehan vmm_handler(module_t mod, int what, void *arg) 221366f6083SPeter Grehan { 222366f6083SPeter Grehan int error; 223366f6083SPeter Grehan 224366f6083SPeter Grehan switch (what) { 225366f6083SPeter Grehan case MOD_LOAD: 226366f6083SPeter Grehan vmmdev_init(); 227366f6083SPeter Grehan iommu_init(); 228366f6083SPeter Grehan error = vmm_init(); 229366f6083SPeter Grehan break; 230366f6083SPeter Grehan case MOD_UNLOAD: 231cdc5b9e7SNeel Natu error = vmmdev_cleanup(); 232cdc5b9e7SNeel Natu if (error == 0) { 233366f6083SPeter Grehan iommu_cleanup(); 234366f6083SPeter Grehan vmm_ipi_cleanup(); 235366f6083SPeter Grehan error = VMM_CLEANUP(); 236cdc5b9e7SNeel Natu } 237366f6083SPeter Grehan break; 238366f6083SPeter Grehan default: 239366f6083SPeter Grehan error = 0; 240366f6083SPeter Grehan break; 241366f6083SPeter Grehan } 242366f6083SPeter Grehan return (error); 243366f6083SPeter Grehan } 244366f6083SPeter Grehan 245366f6083SPeter Grehan static moduledata_t vmm_kmod = { 246366f6083SPeter Grehan "vmm", 247366f6083SPeter Grehan vmm_handler, 248366f6083SPeter Grehan NULL 249366f6083SPeter Grehan }; 250366f6083SPeter Grehan 251366f6083SPeter Grehan /* 252366f6083SPeter Grehan * Execute the module load handler after the pci passthru driver has had 253366f6083SPeter Grehan * a chance to claim devices. We need this information at the time we do 254366f6083SPeter Grehan * iommu initialization. 255366f6083SPeter Grehan */ 256366f6083SPeter Grehan DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_CONFIGURE + 1, SI_ORDER_ANY); 257366f6083SPeter Grehan MODULE_VERSION(vmm, 1); 258366f6083SPeter Grehan 259366f6083SPeter Grehan SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 260366f6083SPeter Grehan 261366f6083SPeter Grehan struct vm * 262366f6083SPeter Grehan vm_create(const char *name) 263366f6083SPeter Grehan { 264366f6083SPeter Grehan int i; 265366f6083SPeter Grehan struct vm *vm; 266366f6083SPeter Grehan vm_paddr_t maxaddr; 267366f6083SPeter Grehan 268366f6083SPeter Grehan const int BSP = 0; 269366f6083SPeter Grehan 270366f6083SPeter Grehan if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 271366f6083SPeter Grehan return (NULL); 272366f6083SPeter Grehan 273366f6083SPeter Grehan vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 274366f6083SPeter Grehan strcpy(vm->name, name); 275366f6083SPeter Grehan vm->cookie = VMINIT(vm); 276366f6083SPeter Grehan 277366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) { 278366f6083SPeter Grehan vcpu_init(vm, i); 279366f6083SPeter Grehan guest_msrs_init(vm, i); 280366f6083SPeter Grehan } 281366f6083SPeter Grehan 282366f6083SPeter Grehan maxaddr = vmm_mem_maxaddr(); 283366f6083SPeter Grehan vm->iommu = iommu_create_domain(maxaddr); 284366f6083SPeter Grehan vm_activate_cpu(vm, BSP); 285366f6083SPeter Grehan 286366f6083SPeter Grehan return (vm); 287366f6083SPeter Grehan } 288366f6083SPeter Grehan 289f7d51510SNeel Natu static void 290f7d51510SNeel Natu vm_free_mem_seg(struct vm *vm, struct vm_memory_segment *seg) 291f7d51510SNeel Natu { 292f7d51510SNeel Natu size_t len; 293f7d51510SNeel Natu vm_paddr_t hpa; 2947ce04d0aSNeel Natu void *host_domain; 2957ce04d0aSNeel Natu 2967ce04d0aSNeel Natu host_domain = iommu_host_domain(); 297f7d51510SNeel Natu 298f7d51510SNeel Natu len = 0; 299f7d51510SNeel Natu while (len < seg->len) { 300f7d51510SNeel Natu hpa = vm_gpa2hpa(vm, seg->gpa + len, PAGE_SIZE); 301f7d51510SNeel Natu if (hpa == (vm_paddr_t)-1) { 302f7d51510SNeel Natu panic("vm_free_mem_segs: cannot free hpa " 303f7d51510SNeel Natu "associated with gpa 0x%016lx", seg->gpa + len); 304f7d51510SNeel Natu } 305f7d51510SNeel Natu 3067ce04d0aSNeel Natu /* 3077ce04d0aSNeel Natu * Remove the 'gpa' to 'hpa' mapping in VMs domain. 3087ce04d0aSNeel Natu * And resurrect the 1:1 mapping for 'hpa' in 'host_domain'. 3097ce04d0aSNeel Natu */ 3107ce04d0aSNeel Natu iommu_remove_mapping(vm->iommu, seg->gpa + len, PAGE_SIZE); 3117ce04d0aSNeel Natu iommu_create_mapping(host_domain, hpa, hpa, PAGE_SIZE); 3127ce04d0aSNeel Natu 313f7d51510SNeel Natu vmm_mem_free(hpa, PAGE_SIZE); 314f7d51510SNeel Natu 315f7d51510SNeel Natu len += PAGE_SIZE; 316f7d51510SNeel Natu } 317f7d51510SNeel Natu 3187ce04d0aSNeel Natu /* 3197ce04d0aSNeel Natu * Invalidate cached translations associated with 'vm->iommu' since 3207ce04d0aSNeel Natu * we have now moved some pages from it. 3217ce04d0aSNeel Natu */ 3227ce04d0aSNeel Natu iommu_invalidate_tlb(vm->iommu); 3237ce04d0aSNeel Natu 324f7d51510SNeel Natu bzero(seg, sizeof(struct vm_memory_segment)); 325f7d51510SNeel Natu } 326f7d51510SNeel Natu 327366f6083SPeter Grehan void 328366f6083SPeter Grehan vm_destroy(struct vm *vm) 329366f6083SPeter Grehan { 330366f6083SPeter Grehan int i; 331366f6083SPeter Grehan 332366f6083SPeter Grehan ppt_unassign_all(vm); 333366f6083SPeter Grehan 334366f6083SPeter Grehan for (i = 0; i < vm->num_mem_segs; i++) 335f7d51510SNeel Natu vm_free_mem_seg(vm, &vm->mem_segs[i]); 336f7d51510SNeel Natu 337f7d51510SNeel Natu vm->num_mem_segs = 0; 338366f6083SPeter Grehan 339366f6083SPeter Grehan for (i = 0; i < VM_MAXCPU; i++) 340366f6083SPeter Grehan vcpu_cleanup(&vm->vcpu[i]); 341366f6083SPeter Grehan 342366f6083SPeter Grehan iommu_destroy_domain(vm->iommu); 343366f6083SPeter Grehan 344366f6083SPeter Grehan VMCLEANUP(vm->cookie); 345366f6083SPeter Grehan 346366f6083SPeter Grehan free(vm, M_VM); 347366f6083SPeter Grehan } 348366f6083SPeter Grehan 349366f6083SPeter Grehan const char * 350366f6083SPeter Grehan vm_name(struct vm *vm) 351366f6083SPeter Grehan { 352366f6083SPeter Grehan return (vm->name); 353366f6083SPeter Grehan } 354366f6083SPeter Grehan 355366f6083SPeter Grehan int 356366f6083SPeter Grehan vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 357366f6083SPeter Grehan { 358366f6083SPeter Grehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 359366f6083SPeter Grehan 360bda273f2SNeel Natu return (VMMMAP_SET(vm->cookie, gpa, hpa, len, VM_MEMATTR_UNCACHEABLE, 361366f6083SPeter Grehan VM_PROT_RW, spok)); 362366f6083SPeter Grehan } 363366f6083SPeter Grehan 364366f6083SPeter Grehan int 365366f6083SPeter Grehan vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 366366f6083SPeter Grehan { 367366f6083SPeter Grehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 368366f6083SPeter Grehan 369bda273f2SNeel Natu return (VMMMAP_SET(vm->cookie, gpa, 0, len, 0, 370366f6083SPeter Grehan VM_PROT_NONE, spok)); 371366f6083SPeter Grehan } 372366f6083SPeter Grehan 373341f19c9SNeel Natu /* 374341f19c9SNeel Natu * Returns TRUE if 'gpa' is available for allocation and FALSE otherwise 375341f19c9SNeel Natu */ 376341f19c9SNeel Natu static boolean_t 377341f19c9SNeel Natu vm_gpa_available(struct vm *vm, vm_paddr_t gpa) 378366f6083SPeter Grehan { 379341f19c9SNeel Natu int i; 380341f19c9SNeel Natu vm_paddr_t gpabase, gpalimit; 381341f19c9SNeel Natu 382341f19c9SNeel Natu if (gpa & PAGE_MASK) 383341f19c9SNeel Natu panic("vm_gpa_available: gpa (0x%016lx) not page aligned", gpa); 384341f19c9SNeel Natu 385341f19c9SNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 386341f19c9SNeel Natu gpabase = vm->mem_segs[i].gpa; 387341f19c9SNeel Natu gpalimit = gpabase + vm->mem_segs[i].len; 388341f19c9SNeel Natu if (gpa >= gpabase && gpa < gpalimit) 389341f19c9SNeel Natu return (FALSE); 390341f19c9SNeel Natu } 391341f19c9SNeel Natu 392341f19c9SNeel Natu return (TRUE); 393341f19c9SNeel Natu } 394341f19c9SNeel Natu 395341f19c9SNeel Natu int 396341f19c9SNeel Natu vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len) 397341f19c9SNeel Natu { 398341f19c9SNeel Natu int error, available, allocated; 399f7d51510SNeel Natu struct vm_memory_segment *seg; 400341f19c9SNeel Natu vm_paddr_t g, hpa; 4017ce04d0aSNeel Natu void *host_domain; 402366f6083SPeter Grehan 403366f6083SPeter Grehan const boolean_t spok = TRUE; /* superpage mappings are ok */ 404366f6083SPeter Grehan 405341f19c9SNeel Natu if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0) 406341f19c9SNeel Natu return (EINVAL); 407341f19c9SNeel Natu 408341f19c9SNeel Natu available = allocated = 0; 409341f19c9SNeel Natu g = gpa; 410341f19c9SNeel Natu while (g < gpa + len) { 411341f19c9SNeel Natu if (vm_gpa_available(vm, g)) 412341f19c9SNeel Natu available++; 413341f19c9SNeel Natu else 414341f19c9SNeel Natu allocated++; 415341f19c9SNeel Natu 416341f19c9SNeel Natu g += PAGE_SIZE; 417341f19c9SNeel Natu } 418341f19c9SNeel Natu 419366f6083SPeter Grehan /* 420341f19c9SNeel Natu * If there are some allocated and some available pages in the address 421341f19c9SNeel Natu * range then it is an error. 422366f6083SPeter Grehan */ 423341f19c9SNeel Natu if (allocated && available) 424341f19c9SNeel Natu return (EINVAL); 425341f19c9SNeel Natu 426341f19c9SNeel Natu /* 427341f19c9SNeel Natu * If the entire address range being requested has already been 428341f19c9SNeel Natu * allocated then there isn't anything more to do. 429341f19c9SNeel Natu */ 430341f19c9SNeel Natu if (allocated && available == 0) 431341f19c9SNeel Natu return (0); 432366f6083SPeter Grehan 433366f6083SPeter Grehan if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS) 434366f6083SPeter Grehan return (E2BIG); 435366f6083SPeter Grehan 4367ce04d0aSNeel Natu host_domain = iommu_host_domain(); 4377ce04d0aSNeel Natu 438f7d51510SNeel Natu seg = &vm->mem_segs[vm->num_mem_segs]; 439366f6083SPeter Grehan 4407ce04d0aSNeel Natu error = 0; 441f7d51510SNeel Natu seg->gpa = gpa; 442f7d51510SNeel Natu seg->len = 0; 443f7d51510SNeel Natu while (seg->len < len) { 444f7d51510SNeel Natu hpa = vmm_mem_alloc(PAGE_SIZE); 445f7d51510SNeel Natu if (hpa == 0) { 446f7d51510SNeel Natu error = ENOMEM; 447f7d51510SNeel Natu break; 448f7d51510SNeel Natu } 449f7d51510SNeel Natu 450f7d51510SNeel Natu error = VMMMAP_SET(vm->cookie, gpa + seg->len, hpa, PAGE_SIZE, 451f7d51510SNeel Natu VM_MEMATTR_WRITE_BACK, VM_PROT_ALL, spok); 452f7d51510SNeel Natu if (error) 453f7d51510SNeel Natu break; 454f7d51510SNeel Natu 4557ce04d0aSNeel Natu /* 4567ce04d0aSNeel Natu * Remove the 1:1 mapping for 'hpa' from the 'host_domain'. 4577ce04d0aSNeel Natu * Add mapping for 'gpa + seg->len' to 'hpa' in the VMs domain. 4587ce04d0aSNeel Natu */ 4597ce04d0aSNeel Natu iommu_remove_mapping(host_domain, hpa, PAGE_SIZE); 460f7d51510SNeel Natu iommu_create_mapping(vm->iommu, gpa + seg->len, hpa, PAGE_SIZE); 461f7d51510SNeel Natu 462f7d51510SNeel Natu seg->len += PAGE_SIZE; 463f7d51510SNeel Natu } 464f7d51510SNeel Natu 4657ce04d0aSNeel Natu if (error) { 466f7d51510SNeel Natu vm_free_mem_seg(vm, seg); 467366f6083SPeter Grehan return (error); 468366f6083SPeter Grehan } 469366f6083SPeter Grehan 4707ce04d0aSNeel Natu /* 4717ce04d0aSNeel Natu * Invalidate cached translations associated with 'host_domain' since 4727ce04d0aSNeel Natu * we have now moved some pages from it. 4737ce04d0aSNeel Natu */ 4747ce04d0aSNeel Natu iommu_invalidate_tlb(host_domain); 4757ce04d0aSNeel Natu 476366f6083SPeter Grehan vm->num_mem_segs++; 477341f19c9SNeel Natu 478366f6083SPeter Grehan return (0); 479366f6083SPeter Grehan } 480366f6083SPeter Grehan 481366f6083SPeter Grehan vm_paddr_t 482366f6083SPeter Grehan vm_gpa2hpa(struct vm *vm, vm_paddr_t gpa, size_t len) 483366f6083SPeter Grehan { 4844db4fb2cSNeel Natu vm_paddr_t nextpage; 4854db4fb2cSNeel Natu 4864db4fb2cSNeel Natu nextpage = rounddown(gpa + PAGE_SIZE, PAGE_SIZE); 4874db4fb2cSNeel Natu if (len > nextpage - gpa) 4884db4fb2cSNeel Natu panic("vm_gpa2hpa: invalid gpa/len: 0x%016lx/%lu", gpa, len); 489366f6083SPeter Grehan 490bda273f2SNeel Natu return (VMMMAP_GET(vm->cookie, gpa)); 491366f6083SPeter Grehan } 492366f6083SPeter Grehan 493366f6083SPeter Grehan int 494366f6083SPeter Grehan vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase, 495366f6083SPeter Grehan struct vm_memory_segment *seg) 496366f6083SPeter Grehan { 497366f6083SPeter Grehan int i; 498366f6083SPeter Grehan 499366f6083SPeter Grehan for (i = 0; i < vm->num_mem_segs; i++) { 500366f6083SPeter Grehan if (gpabase == vm->mem_segs[i].gpa) { 501366f6083SPeter Grehan *seg = vm->mem_segs[i]; 502366f6083SPeter Grehan return (0); 503366f6083SPeter Grehan } 504366f6083SPeter Grehan } 505366f6083SPeter Grehan return (-1); 506366f6083SPeter Grehan } 507366f6083SPeter Grehan 508366f6083SPeter Grehan int 509366f6083SPeter Grehan vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 510366f6083SPeter Grehan { 511366f6083SPeter Grehan 512366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 513366f6083SPeter Grehan return (EINVAL); 514366f6083SPeter Grehan 515366f6083SPeter Grehan if (reg >= VM_REG_LAST) 516366f6083SPeter Grehan return (EINVAL); 517366f6083SPeter Grehan 518366f6083SPeter Grehan return (VMGETREG(vm->cookie, vcpu, reg, retval)); 519366f6083SPeter Grehan } 520366f6083SPeter Grehan 521366f6083SPeter Grehan int 522366f6083SPeter Grehan vm_set_register(struct vm *vm, int vcpu, int reg, uint64_t val) 523366f6083SPeter Grehan { 524366f6083SPeter Grehan 525366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 526366f6083SPeter Grehan return (EINVAL); 527366f6083SPeter Grehan 528366f6083SPeter Grehan if (reg >= VM_REG_LAST) 529366f6083SPeter Grehan return (EINVAL); 530366f6083SPeter Grehan 531366f6083SPeter Grehan return (VMSETREG(vm->cookie, vcpu, reg, val)); 532366f6083SPeter Grehan } 533366f6083SPeter Grehan 534366f6083SPeter Grehan static boolean_t 535366f6083SPeter Grehan is_descriptor_table(int reg) 536366f6083SPeter Grehan { 537366f6083SPeter Grehan 538366f6083SPeter Grehan switch (reg) { 539366f6083SPeter Grehan case VM_REG_GUEST_IDTR: 540366f6083SPeter Grehan case VM_REG_GUEST_GDTR: 541366f6083SPeter Grehan return (TRUE); 542366f6083SPeter Grehan default: 543366f6083SPeter Grehan return (FALSE); 544366f6083SPeter Grehan } 545366f6083SPeter Grehan } 546366f6083SPeter Grehan 547366f6083SPeter Grehan static boolean_t 548366f6083SPeter Grehan is_segment_register(int reg) 549366f6083SPeter Grehan { 550366f6083SPeter Grehan 551366f6083SPeter Grehan switch (reg) { 552366f6083SPeter Grehan case VM_REG_GUEST_ES: 553366f6083SPeter Grehan case VM_REG_GUEST_CS: 554366f6083SPeter Grehan case VM_REG_GUEST_SS: 555366f6083SPeter Grehan case VM_REG_GUEST_DS: 556366f6083SPeter Grehan case VM_REG_GUEST_FS: 557366f6083SPeter Grehan case VM_REG_GUEST_GS: 558366f6083SPeter Grehan case VM_REG_GUEST_TR: 559366f6083SPeter Grehan case VM_REG_GUEST_LDTR: 560366f6083SPeter Grehan return (TRUE); 561366f6083SPeter Grehan default: 562366f6083SPeter Grehan return (FALSE); 563366f6083SPeter Grehan } 564366f6083SPeter Grehan } 565366f6083SPeter Grehan 566366f6083SPeter Grehan int 567366f6083SPeter Grehan vm_get_seg_desc(struct vm *vm, int vcpu, int reg, 568366f6083SPeter Grehan struct seg_desc *desc) 569366f6083SPeter Grehan { 570366f6083SPeter Grehan 571366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 572366f6083SPeter Grehan return (EINVAL); 573366f6083SPeter Grehan 574366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 575366f6083SPeter Grehan return (EINVAL); 576366f6083SPeter Grehan 577366f6083SPeter Grehan return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 578366f6083SPeter Grehan } 579366f6083SPeter Grehan 580366f6083SPeter Grehan int 581366f6083SPeter Grehan vm_set_seg_desc(struct vm *vm, int vcpu, int reg, 582366f6083SPeter Grehan struct seg_desc *desc) 583366f6083SPeter Grehan { 584366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 585366f6083SPeter Grehan return (EINVAL); 586366f6083SPeter Grehan 587366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 588366f6083SPeter Grehan return (EINVAL); 589366f6083SPeter Grehan 590366f6083SPeter Grehan return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 591366f6083SPeter Grehan } 592366f6083SPeter Grehan 593366f6083SPeter Grehan int 594366f6083SPeter Grehan vm_get_pinning(struct vm *vm, int vcpuid, int *cpuid) 595366f6083SPeter Grehan { 596366f6083SPeter Grehan 597366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 598366f6083SPeter Grehan return (EINVAL); 599366f6083SPeter Grehan 600366f6083SPeter Grehan *cpuid = VCPU_PINCPU(vm, vcpuid); 601366f6083SPeter Grehan 602366f6083SPeter Grehan return (0); 603366f6083SPeter Grehan } 604366f6083SPeter Grehan 605366f6083SPeter Grehan int 606366f6083SPeter Grehan vm_set_pinning(struct vm *vm, int vcpuid, int host_cpuid) 607366f6083SPeter Grehan { 608366f6083SPeter Grehan struct thread *td; 609366f6083SPeter Grehan 610366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 611366f6083SPeter Grehan return (EINVAL); 612366f6083SPeter Grehan 613366f6083SPeter Grehan td = curthread; /* XXXSMP only safe when muxing vcpus */ 614366f6083SPeter Grehan 615366f6083SPeter Grehan /* unpin */ 616366f6083SPeter Grehan if (host_cpuid < 0) { 617366f6083SPeter Grehan VCPU_UNPIN(vm, vcpuid); 618366f6083SPeter Grehan thread_lock(td); 619366f6083SPeter Grehan sched_unbind(td); 620366f6083SPeter Grehan thread_unlock(td); 621366f6083SPeter Grehan return (0); 622366f6083SPeter Grehan } 623366f6083SPeter Grehan 624366f6083SPeter Grehan if (CPU_ABSENT(host_cpuid)) 625366f6083SPeter Grehan return (EINVAL); 626366f6083SPeter Grehan 627366f6083SPeter Grehan /* 628366f6083SPeter Grehan * XXX we should check that 'host_cpuid' has not already been pinned 629366f6083SPeter Grehan * by another vm. 630366f6083SPeter Grehan */ 631366f6083SPeter Grehan thread_lock(td); 632366f6083SPeter Grehan sched_bind(td, host_cpuid); 633366f6083SPeter Grehan thread_unlock(td); 634366f6083SPeter Grehan VCPU_PIN(vm, vcpuid, host_cpuid); 635366f6083SPeter Grehan 636366f6083SPeter Grehan return (0); 637366f6083SPeter Grehan } 638366f6083SPeter Grehan 639366f6083SPeter Grehan static void 640366f6083SPeter Grehan restore_guest_fpustate(struct vcpu *vcpu) 641366f6083SPeter Grehan { 642366f6083SPeter Grehan 64338f1b189SPeter Grehan /* flush host state to the pcb */ 64438f1b189SPeter Grehan fpuexit(curthread); 645bd8572e0SNeel Natu 646bd8572e0SNeel Natu /* restore guest FPU state */ 647366f6083SPeter Grehan fpu_stop_emulating(); 64838f1b189SPeter Grehan fpurestore(vcpu->guestfpu); 649bd8572e0SNeel Natu 650bd8572e0SNeel Natu /* 651bd8572e0SNeel Natu * The FPU is now "dirty" with the guest's state so turn on emulation 652bd8572e0SNeel Natu * to trap any access to the FPU by the host. 653bd8572e0SNeel Natu */ 654bd8572e0SNeel Natu fpu_start_emulating(); 655366f6083SPeter Grehan } 656366f6083SPeter Grehan 657366f6083SPeter Grehan static void 658366f6083SPeter Grehan save_guest_fpustate(struct vcpu *vcpu) 659366f6083SPeter Grehan { 660366f6083SPeter Grehan 661bd8572e0SNeel Natu if ((rcr0() & CR0_TS) == 0) 662bd8572e0SNeel Natu panic("fpu emulation not enabled in host!"); 663bd8572e0SNeel Natu 664bd8572e0SNeel Natu /* save guest FPU state */ 665bd8572e0SNeel Natu fpu_stop_emulating(); 66638f1b189SPeter Grehan fpusave(vcpu->guestfpu); 667366f6083SPeter Grehan fpu_start_emulating(); 668366f6083SPeter Grehan } 669366f6083SPeter Grehan 670f76fc5d4SNeel Natu static VMM_STAT_DEFINE(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 671f76fc5d4SNeel Natu 672366f6083SPeter Grehan int 673366f6083SPeter Grehan vm_run(struct vm *vm, struct vm_run *vmrun) 674366f6083SPeter Grehan { 675f76fc5d4SNeel Natu int error, vcpuid, sleepticks, t; 676366f6083SPeter Grehan struct vcpu *vcpu; 677366f6083SPeter Grehan struct pcb *pcb; 678f76fc5d4SNeel Natu uint64_t tscval, rip; 679f76fc5d4SNeel Natu struct vm_exit *vme; 680366f6083SPeter Grehan 681366f6083SPeter Grehan vcpuid = vmrun->cpuid; 682366f6083SPeter Grehan 683366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 684366f6083SPeter Grehan return (EINVAL); 685366f6083SPeter Grehan 686366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 687f76fc5d4SNeel Natu vme = &vmrun->vm_exit; 688f76fc5d4SNeel Natu rip = vmrun->rip; 689f76fc5d4SNeel Natu restart: 690366f6083SPeter Grehan critical_enter(); 691366f6083SPeter Grehan 692366f6083SPeter Grehan tscval = rdtsc(); 693366f6083SPeter Grehan 694366f6083SPeter Grehan pcb = PCPU_GET(curpcb); 69534a6b2d6SJohn Baldwin set_pcb_flags(pcb, PCB_FULL_IRET); 696366f6083SPeter Grehan 697366f6083SPeter Grehan restore_guest_msrs(vm, vcpuid); 698366f6083SPeter Grehan restore_guest_fpustate(vcpu); 69975dd3366SNeel Natu 70075dd3366SNeel Natu vcpu->hostcpu = curcpu; 701f76fc5d4SNeel Natu error = VMRUN(vm->cookie, vcpuid, rip); 70275dd3366SNeel Natu vcpu->hostcpu = NOCPU; 70375dd3366SNeel Natu 704366f6083SPeter Grehan save_guest_fpustate(vcpu); 705366f6083SPeter Grehan restore_host_msrs(vm, vcpuid); 706366f6083SPeter Grehan 707366f6083SPeter Grehan vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 708366f6083SPeter Grehan 70998ed632cSNeel Natu /* copy the exit information */ 710f76fc5d4SNeel Natu bcopy(&vcpu->exitinfo, vme, sizeof(struct vm_exit)); 71198ed632cSNeel Natu 712366f6083SPeter Grehan critical_exit(); 713366f6083SPeter Grehan 714f76fc5d4SNeel Natu /* 715f76fc5d4SNeel Natu * Oblige the guest's desire to 'hlt' by sleeping until the vcpu 716f76fc5d4SNeel Natu * is ready to run. 717f76fc5d4SNeel Natu */ 718f76fc5d4SNeel Natu if (error == 0 && vme->exitcode == VM_EXITCODE_HLT) { 719f76fc5d4SNeel Natu vcpu_lock(vcpu); 720f76fc5d4SNeel Natu 721f76fc5d4SNeel Natu /* 722f76fc5d4SNeel Natu * Figure out the number of host ticks until the next apic 723f76fc5d4SNeel Natu * timer interrupt in the guest. 724f76fc5d4SNeel Natu */ 725f76fc5d4SNeel Natu sleepticks = lapic_timer_tick(vm, vcpuid); 726f76fc5d4SNeel Natu 727f76fc5d4SNeel Natu /* 728f76fc5d4SNeel Natu * If the guest local apic timer is disabled then sleep for 729f76fc5d4SNeel Natu * a long time but not forever. 730f76fc5d4SNeel Natu */ 731f76fc5d4SNeel Natu if (sleepticks < 0) 732f76fc5d4SNeel Natu sleepticks = hz; 733f76fc5d4SNeel Natu 734f76fc5d4SNeel Natu /* 735f76fc5d4SNeel Natu * Do a final check for pending NMI or interrupts before 736f76fc5d4SNeel Natu * really putting this thread to sleep. 737f76fc5d4SNeel Natu * 738f76fc5d4SNeel Natu * These interrupts could have happened any time after we 739f76fc5d4SNeel Natu * returned from VMRUN() and before we grabbed the vcpu lock. 740f76fc5d4SNeel Natu */ 741f76fc5d4SNeel Natu if (!vm_nmi_pending(vm, vcpuid) && 742f76fc5d4SNeel Natu lapic_pending_intr(vm, vcpuid) < 0) { 743f76fc5d4SNeel Natu if (sleepticks <= 0) 744f76fc5d4SNeel Natu panic("invalid sleepticks %d", sleepticks); 745f76fc5d4SNeel Natu t = ticks; 746f76fc5d4SNeel Natu msleep_spin(vcpu, &vcpu->mtx, "vmidle", sleepticks); 747f76fc5d4SNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 748f76fc5d4SNeel Natu } 749f76fc5d4SNeel Natu 750f76fc5d4SNeel Natu vcpu_unlock(vcpu); 751f76fc5d4SNeel Natu 752f76fc5d4SNeel Natu rip = vme->rip + vme->inst_length; 753f76fc5d4SNeel Natu goto restart; 754f76fc5d4SNeel Natu } 755f76fc5d4SNeel Natu 756366f6083SPeter Grehan return (error); 757366f6083SPeter Grehan } 758366f6083SPeter Grehan 759366f6083SPeter Grehan int 760366f6083SPeter Grehan vm_inject_event(struct vm *vm, int vcpuid, int type, 761366f6083SPeter Grehan int vector, uint32_t code, int code_valid) 762366f6083SPeter Grehan { 763366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 764366f6083SPeter Grehan return (EINVAL); 765366f6083SPeter Grehan 766366f6083SPeter Grehan if ((type > VM_EVENT_NONE && type < VM_EVENT_MAX) == 0) 767366f6083SPeter Grehan return (EINVAL); 768366f6083SPeter Grehan 769366f6083SPeter Grehan if (vector < 0 || vector > 255) 770366f6083SPeter Grehan return (EINVAL); 771366f6083SPeter Grehan 772366f6083SPeter Grehan return (VMINJECT(vm->cookie, vcpuid, type, vector, code, code_valid)); 773366f6083SPeter Grehan } 774366f6083SPeter Grehan 775f76fc5d4SNeel Natu static VMM_STAT_DEFINE(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 776366f6083SPeter Grehan 777f352ff0cSNeel Natu int 778f352ff0cSNeel Natu vm_inject_nmi(struct vm *vm, int vcpuid) 779f352ff0cSNeel Natu { 780f352ff0cSNeel Natu struct vcpu *vcpu; 781f352ff0cSNeel Natu 782f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 783366f6083SPeter Grehan return (EINVAL); 784366f6083SPeter Grehan 785f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 786f352ff0cSNeel Natu 787f352ff0cSNeel Natu vcpu->nmi_pending = 1; 788f352ff0cSNeel Natu vm_interrupt_hostcpu(vm, vcpuid); 789f352ff0cSNeel Natu return (0); 790f352ff0cSNeel Natu } 791f352ff0cSNeel Natu 792f352ff0cSNeel Natu int 793f352ff0cSNeel Natu vm_nmi_pending(struct vm *vm, int vcpuid) 794f352ff0cSNeel Natu { 795f352ff0cSNeel Natu struct vcpu *vcpu; 796f352ff0cSNeel Natu 797f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 798f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 799f352ff0cSNeel Natu 800f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 801f352ff0cSNeel Natu 802f352ff0cSNeel Natu return (vcpu->nmi_pending); 803f352ff0cSNeel Natu } 804f352ff0cSNeel Natu 805f352ff0cSNeel Natu void 806f352ff0cSNeel Natu vm_nmi_clear(struct vm *vm, int vcpuid) 807f352ff0cSNeel Natu { 808f352ff0cSNeel Natu struct vcpu *vcpu; 809f352ff0cSNeel Natu 810f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 811f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 812f352ff0cSNeel Natu 813f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 814f352ff0cSNeel Natu 815f352ff0cSNeel Natu if (vcpu->nmi_pending == 0) 816f352ff0cSNeel Natu panic("vm_nmi_clear: inconsistent nmi_pending state"); 817f352ff0cSNeel Natu 818f352ff0cSNeel Natu vcpu->nmi_pending = 0; 819f352ff0cSNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 820366f6083SPeter Grehan } 821366f6083SPeter Grehan 822366f6083SPeter Grehan int 823366f6083SPeter Grehan vm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 824366f6083SPeter Grehan { 825366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 826366f6083SPeter Grehan return (EINVAL); 827366f6083SPeter Grehan 828366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 829366f6083SPeter Grehan return (EINVAL); 830366f6083SPeter Grehan 831366f6083SPeter Grehan return (VMGETCAP(vm->cookie, vcpu, type, retval)); 832366f6083SPeter Grehan } 833366f6083SPeter Grehan 834366f6083SPeter Grehan int 835366f6083SPeter Grehan vm_set_capability(struct vm *vm, int vcpu, int type, int val) 836366f6083SPeter Grehan { 837366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 838366f6083SPeter Grehan return (EINVAL); 839366f6083SPeter Grehan 840366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 841366f6083SPeter Grehan return (EINVAL); 842366f6083SPeter Grehan 843366f6083SPeter Grehan return (VMSETCAP(vm->cookie, vcpu, type, val)); 844366f6083SPeter Grehan } 845366f6083SPeter Grehan 846366f6083SPeter Grehan uint64_t * 847366f6083SPeter Grehan vm_guest_msrs(struct vm *vm, int cpu) 848366f6083SPeter Grehan { 849366f6083SPeter Grehan return (vm->vcpu[cpu].guest_msrs); 850366f6083SPeter Grehan } 851366f6083SPeter Grehan 852366f6083SPeter Grehan struct vlapic * 853366f6083SPeter Grehan vm_lapic(struct vm *vm, int cpu) 854366f6083SPeter Grehan { 855366f6083SPeter Grehan return (vm->vcpu[cpu].vlapic); 856366f6083SPeter Grehan } 857366f6083SPeter Grehan 858366f6083SPeter Grehan boolean_t 859366f6083SPeter Grehan vmm_is_pptdev(int bus, int slot, int func) 860366f6083SPeter Grehan { 861366f6083SPeter Grehan int found, b, s, f, n; 862366f6083SPeter Grehan char *val, *cp, *cp2; 863366f6083SPeter Grehan 864366f6083SPeter Grehan /* 865366f6083SPeter Grehan * setenv pptdevs "1/2/3 4/5/6 7/8/9 10/11/12" 866366f6083SPeter Grehan */ 867366f6083SPeter Grehan found = 0; 868366f6083SPeter Grehan cp = val = getenv("pptdevs"); 869366f6083SPeter Grehan while (cp != NULL && *cp != '\0') { 870366f6083SPeter Grehan if ((cp2 = strchr(cp, ' ')) != NULL) 871366f6083SPeter Grehan *cp2 = '\0'; 872366f6083SPeter Grehan 873366f6083SPeter Grehan n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 874366f6083SPeter Grehan if (n == 3 && bus == b && slot == s && func == f) { 875366f6083SPeter Grehan found = 1; 876366f6083SPeter Grehan break; 877366f6083SPeter Grehan } 878366f6083SPeter Grehan 879366f6083SPeter Grehan if (cp2 != NULL) 880366f6083SPeter Grehan *cp2++ = ' '; 881366f6083SPeter Grehan 882366f6083SPeter Grehan cp = cp2; 883366f6083SPeter Grehan } 884366f6083SPeter Grehan freeenv(val); 885366f6083SPeter Grehan return (found); 886366f6083SPeter Grehan } 887366f6083SPeter Grehan 888366f6083SPeter Grehan void * 889366f6083SPeter Grehan vm_iommu_domain(struct vm *vm) 890366f6083SPeter Grehan { 891366f6083SPeter Grehan 892366f6083SPeter Grehan return (vm->iommu); 893366f6083SPeter Grehan } 894366f6083SPeter Grehan 89575dd3366SNeel Natu int 89675dd3366SNeel Natu vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state state) 897366f6083SPeter Grehan { 89875dd3366SNeel Natu int error; 899366f6083SPeter Grehan struct vcpu *vcpu; 900366f6083SPeter Grehan 901366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 902366f6083SPeter Grehan panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 903366f6083SPeter Grehan 904366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 905366f6083SPeter Grehan 90675dd3366SNeel Natu vcpu_lock(vcpu); 90775dd3366SNeel Natu 90875dd3366SNeel Natu /* 90975dd3366SNeel Natu * The following state transitions are allowed: 91075dd3366SNeel Natu * IDLE -> RUNNING -> IDLE 91175dd3366SNeel Natu * IDLE -> CANNOT_RUN -> IDLE 91275dd3366SNeel Natu */ 91375dd3366SNeel Natu if ((vcpu->state == VCPU_IDLE && state != VCPU_IDLE) || 91475dd3366SNeel Natu (vcpu->state != VCPU_IDLE && state == VCPU_IDLE)) { 91575dd3366SNeel Natu error = 0; 91675dd3366SNeel Natu vcpu->state = state; 917366f6083SPeter Grehan } else { 91875dd3366SNeel Natu error = EBUSY; 919366f6083SPeter Grehan } 920366f6083SPeter Grehan 92175dd3366SNeel Natu vcpu_unlock(vcpu); 92275dd3366SNeel Natu 92375dd3366SNeel Natu return (error); 92475dd3366SNeel Natu } 92575dd3366SNeel Natu 92675dd3366SNeel Natu enum vcpu_state 92775dd3366SNeel Natu vcpu_get_state(struct vm *vm, int vcpuid) 928366f6083SPeter Grehan { 929366f6083SPeter Grehan struct vcpu *vcpu; 93075dd3366SNeel Natu enum vcpu_state state; 931366f6083SPeter Grehan 932366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 933366f6083SPeter Grehan panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 934366f6083SPeter Grehan 935366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 936366f6083SPeter Grehan 93775dd3366SNeel Natu vcpu_lock(vcpu); 93875dd3366SNeel Natu state = vcpu->state; 93975dd3366SNeel Natu vcpu_unlock(vcpu); 940366f6083SPeter Grehan 94175dd3366SNeel Natu return (state); 942366f6083SPeter Grehan } 943366f6083SPeter Grehan 944366f6083SPeter Grehan void 945366f6083SPeter Grehan vm_activate_cpu(struct vm *vm, int vcpuid) 946366f6083SPeter Grehan { 947366f6083SPeter Grehan 948366f6083SPeter Grehan if (vcpuid >= 0 && vcpuid < VM_MAXCPU) 949a5615c90SPeter Grehan CPU_SET(vcpuid, &vm->active_cpus); 950366f6083SPeter Grehan } 951366f6083SPeter Grehan 952a5615c90SPeter Grehan cpuset_t 953366f6083SPeter Grehan vm_active_cpus(struct vm *vm) 954366f6083SPeter Grehan { 955366f6083SPeter Grehan 956366f6083SPeter Grehan return (vm->active_cpus); 957366f6083SPeter Grehan } 958366f6083SPeter Grehan 959366f6083SPeter Grehan void * 960366f6083SPeter Grehan vcpu_stats(struct vm *vm, int vcpuid) 961366f6083SPeter Grehan { 962366f6083SPeter Grehan 963366f6083SPeter Grehan return (vm->vcpu[vcpuid].stats); 964366f6083SPeter Grehan } 965e9027382SNeel Natu 966e9027382SNeel Natu int 967e9027382SNeel Natu vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 968e9027382SNeel Natu { 969e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 970e9027382SNeel Natu return (EINVAL); 971e9027382SNeel Natu 972e9027382SNeel Natu *state = vm->vcpu[vcpuid].x2apic_state; 973e9027382SNeel Natu 974e9027382SNeel Natu return (0); 975e9027382SNeel Natu } 976e9027382SNeel Natu 977e9027382SNeel Natu int 978e9027382SNeel Natu vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 979e9027382SNeel Natu { 980e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 981e9027382SNeel Natu return (EINVAL); 982e9027382SNeel Natu 983e9027382SNeel Natu if (state < 0 || state >= X2APIC_STATE_LAST) 984e9027382SNeel Natu return (EINVAL); 985e9027382SNeel Natu 986e9027382SNeel Natu vm->vcpu[vcpuid].x2apic_state = state; 987e9027382SNeel Natu 98873820fb0SNeel Natu vlapic_set_x2apic_state(vm, vcpuid, state); 98973820fb0SNeel Natu 990e9027382SNeel Natu return (0); 991e9027382SNeel Natu } 99275dd3366SNeel Natu 99375dd3366SNeel Natu void 99475dd3366SNeel Natu vm_interrupt_hostcpu(struct vm *vm, int vcpuid) 99575dd3366SNeel Natu { 99675dd3366SNeel Natu int hostcpu; 99775dd3366SNeel Natu struct vcpu *vcpu; 99875dd3366SNeel Natu 99975dd3366SNeel Natu vcpu = &vm->vcpu[vcpuid]; 100075dd3366SNeel Natu 1001f76fc5d4SNeel Natu vcpu_lock(vcpu); 100275dd3366SNeel Natu hostcpu = vcpu->hostcpu; 1003f76fc5d4SNeel Natu if (hostcpu == NOCPU) { 1004f76fc5d4SNeel Natu /* 1005f76fc5d4SNeel Natu * If the vcpu is 'RUNNING' but without a valid 'hostcpu' then 1006f76fc5d4SNeel Natu * the host thread must be sleeping waiting for an event to 1007f76fc5d4SNeel Natu * kick the vcpu out of 'hlt'. 1008f76fc5d4SNeel Natu * 1009f76fc5d4SNeel Natu * XXX this is racy because the condition exists right before 1010f76fc5d4SNeel Natu * and after calling VMRUN() in vm_run(). The wakeup() is 1011f76fc5d4SNeel Natu * benign in this case. 1012f76fc5d4SNeel Natu */ 1013f76fc5d4SNeel Natu if (vcpu->state == VCPU_RUNNING) 1014f76fc5d4SNeel Natu wakeup_one(vcpu); 1015f76fc5d4SNeel Natu } else { 1016f76fc5d4SNeel Natu if (vcpu->state != VCPU_RUNNING) 1017f76fc5d4SNeel Natu panic("invalid vcpu state %d", vcpu->state); 1018f76fc5d4SNeel Natu if (hostcpu != curcpu) 101975dd3366SNeel Natu ipi_cpu(hostcpu, vmm_ipinum); 102075dd3366SNeel Natu } 1021f76fc5d4SNeel Natu vcpu_unlock(vcpu); 1022f76fc5d4SNeel Natu } 1023