1366f6083SPeter Grehan /*- 2366f6083SPeter Grehan * Copyright (c) 2011 NetApp, Inc. 3366f6083SPeter Grehan * All rights reserved. 4366f6083SPeter Grehan * 5366f6083SPeter Grehan * Redistribution and use in source and binary forms, with or without 6366f6083SPeter Grehan * modification, are permitted provided that the following conditions 7366f6083SPeter Grehan * are met: 8366f6083SPeter Grehan * 1. Redistributions of source code must retain the above copyright 9366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer. 10366f6083SPeter Grehan * 2. Redistributions in binary form must reproduce the above copyright 11366f6083SPeter Grehan * notice, this list of conditions and the following disclaimer in the 12366f6083SPeter Grehan * documentation and/or other materials provided with the distribution. 13366f6083SPeter Grehan * 14366f6083SPeter Grehan * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 15366f6083SPeter Grehan * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16366f6083SPeter Grehan * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17366f6083SPeter Grehan * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 18366f6083SPeter Grehan * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19366f6083SPeter Grehan * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20366f6083SPeter Grehan * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21366f6083SPeter Grehan * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22366f6083SPeter Grehan * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23366f6083SPeter Grehan * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24366f6083SPeter Grehan * SUCH DAMAGE. 25366f6083SPeter Grehan * 26366f6083SPeter Grehan * $FreeBSD$ 27366f6083SPeter Grehan */ 28366f6083SPeter Grehan 29366f6083SPeter Grehan #include <sys/cdefs.h> 30366f6083SPeter Grehan __FBSDID("$FreeBSD$"); 31366f6083SPeter Grehan 32366f6083SPeter Grehan #include <sys/param.h> 3338f1b189SPeter Grehan #include <sys/systm.h> 34366f6083SPeter Grehan #include <sys/kernel.h> 35366f6083SPeter Grehan #include <sys/module.h> 36366f6083SPeter Grehan #include <sys/sysctl.h> 37366f6083SPeter Grehan #include <sys/malloc.h> 38366f6083SPeter Grehan #include <sys/pcpu.h> 39366f6083SPeter Grehan #include <sys/lock.h> 40366f6083SPeter Grehan #include <sys/mutex.h> 41366f6083SPeter Grehan #include <sys/proc.h> 42318224bbSNeel Natu #include <sys/rwlock.h> 43366f6083SPeter Grehan #include <sys/sched.h> 44366f6083SPeter Grehan #include <sys/smp.h> 45366f6083SPeter Grehan #include <sys/systm.h> 46366f6083SPeter Grehan 47366f6083SPeter Grehan #include <vm/vm.h> 48318224bbSNeel Natu #include <vm/vm_object.h> 49318224bbSNeel Natu #include <vm/vm_page.h> 50318224bbSNeel Natu #include <vm/pmap.h> 51318224bbSNeel Natu #include <vm/vm_map.h> 52318224bbSNeel Natu #include <vm/vm_extern.h> 53318224bbSNeel Natu #include <vm/vm_param.h> 54366f6083SPeter Grehan 5563e62d39SJohn Baldwin #include <machine/cpu.h> 56366f6083SPeter Grehan #include <machine/vm.h> 57366f6083SPeter Grehan #include <machine/pcb.h> 5875dd3366SNeel Natu #include <machine/smp.h> 591c052192SNeel Natu #include <x86/psl.h> 6034a6b2d6SJohn Baldwin #include <x86/apicreg.h> 61318224bbSNeel Natu #include <machine/vmparam.h> 62366f6083SPeter Grehan 63366f6083SPeter Grehan #include <machine/vmm.h> 64565bbb86SNeel Natu #include <machine/vmm_dev.h> 65e813a873SNeel Natu #include <machine/vmm_instruction_emul.h> 66565bbb86SNeel Natu 67d17b5104SNeel Natu #include "vmm_ioport.h" 68318224bbSNeel Natu #include "vmm_ktr.h" 69b01c2033SNeel Natu #include "vmm_host.h" 70366f6083SPeter Grehan #include "vmm_mem.h" 71366f6083SPeter Grehan #include "vmm_util.h" 72762fd208STycho Nightingale #include "vatpic.h" 73e883c9bbSTycho Nightingale #include "vatpit.h" 7408e3ff32SNeel Natu #include "vhpet.h" 75565bbb86SNeel Natu #include "vioapic.h" 76366f6083SPeter Grehan #include "vlapic.h" 77160ef77aSNeel Natu #include "vpmtmr.h" 780dafa5cdSNeel Natu #include "vrtc.h" 79366f6083SPeter Grehan #include "vmm_stat.h" 80f76fc5d4SNeel Natu #include "vmm_lapic.h" 81366f6083SPeter Grehan 82366f6083SPeter Grehan #include "io/ppt.h" 83366f6083SPeter Grehan #include "io/iommu.h" 84366f6083SPeter Grehan 85366f6083SPeter Grehan struct vlapic; 86366f6083SPeter Grehan 875fcf252fSNeel Natu /* 885fcf252fSNeel Natu * Initialization: 895fcf252fSNeel Natu * (a) allocated when vcpu is created 905fcf252fSNeel Natu * (i) initialized when vcpu is created and when it is reinitialized 915fcf252fSNeel Natu * (o) initialized the first time the vcpu is created 925fcf252fSNeel Natu * (x) initialized before use 935fcf252fSNeel Natu */ 94366f6083SPeter Grehan struct vcpu { 955fcf252fSNeel Natu struct mtx mtx; /* (o) protects 'state' and 'hostcpu' */ 965fcf252fSNeel Natu enum vcpu_state state; /* (o) vcpu state */ 975fcf252fSNeel Natu int hostcpu; /* (o) vcpu's host cpu */ 985fcf252fSNeel Natu struct vlapic *vlapic; /* (i) APIC device model */ 995fcf252fSNeel Natu enum x2apic_state x2apic_state; /* (i) APIC mode */ 100091d4532SNeel Natu uint64_t exitintinfo; /* (i) events pending at VM exit */ 1015fcf252fSNeel Natu int nmi_pending; /* (i) NMI pending */ 1025fcf252fSNeel Natu int extint_pending; /* (i) INTR pending */ 1035fcf252fSNeel Natu int exception_pending; /* (i) exception pending */ 104c9c75df4SNeel Natu int exc_vector; /* (x) exception collateral */ 105c9c75df4SNeel Natu int exc_errcode_valid; 106c9c75df4SNeel Natu uint32_t exc_errcode; 1075fcf252fSNeel Natu struct savefpu *guestfpu; /* (a,i) guest fpu state */ 1085fcf252fSNeel Natu uint64_t guest_xcr0; /* (i) guest %xcr0 register */ 1095fcf252fSNeel Natu void *stats; /* (a,i) statistics */ 1105fcf252fSNeel Natu struct vm_exit exitinfo; /* (x) exit reason and collateral */ 111d087a399SNeel Natu uint64_t nextrip; /* (x) next instruction to execute */ 112366f6083SPeter Grehan }; 113366f6083SPeter Grehan 1145fcf252fSNeel Natu #define vcpu_lock_initialized(v) mtx_initialized(&((v)->mtx)) 115f76fc5d4SNeel Natu #define vcpu_lock_init(v) mtx_init(&((v)->mtx), "vcpu lock", 0, MTX_SPIN) 116f76fc5d4SNeel Natu #define vcpu_lock(v) mtx_lock_spin(&((v)->mtx)) 117f76fc5d4SNeel Natu #define vcpu_unlock(v) mtx_unlock_spin(&((v)->mtx)) 118318224bbSNeel Natu #define vcpu_assert_locked(v) mtx_assert(&((v)->mtx), MA_OWNED) 11975dd3366SNeel Natu 120318224bbSNeel Natu struct mem_seg { 121318224bbSNeel Natu vm_paddr_t gpa; 122318224bbSNeel Natu size_t len; 123318224bbSNeel Natu boolean_t wired; 124318224bbSNeel Natu vm_object_t object; 125318224bbSNeel Natu }; 126366f6083SPeter Grehan #define VM_MAX_MEMORY_SEGMENTS 2 127366f6083SPeter Grehan 128366f6083SPeter Grehan /* 1295fcf252fSNeel Natu * Initialization: 1305fcf252fSNeel Natu * (o) initialized the first time the VM is created 1315fcf252fSNeel Natu * (i) initialized when VM is created and when it is reinitialized 1325fcf252fSNeel Natu * (x) initialized before use 133366f6083SPeter Grehan */ 1345fcf252fSNeel Natu struct vm { 1355fcf252fSNeel Natu void *cookie; /* (i) cpu-specific data */ 1365fcf252fSNeel Natu void *iommu; /* (x) iommu-specific data */ 1375fcf252fSNeel Natu struct vhpet *vhpet; /* (i) virtual HPET */ 1385fcf252fSNeel Natu struct vioapic *vioapic; /* (i) virtual ioapic */ 1395fcf252fSNeel Natu struct vatpic *vatpic; /* (i) virtual atpic */ 1405fcf252fSNeel Natu struct vatpit *vatpit; /* (i) virtual atpit */ 141160ef77aSNeel Natu struct vpmtmr *vpmtmr; /* (i) virtual ACPI PM timer */ 1420dafa5cdSNeel Natu struct vrtc *vrtc; /* (o) virtual RTC */ 1435fcf252fSNeel Natu volatile cpuset_t active_cpus; /* (i) active vcpus */ 1445fcf252fSNeel Natu int suspend; /* (i) stop VM execution */ 1455fcf252fSNeel Natu volatile cpuset_t suspended_cpus; /* (i) suspended vcpus */ 1465fcf252fSNeel Natu volatile cpuset_t halted_cpus; /* (x) cpus in a hard halt */ 1475fcf252fSNeel Natu cpuset_t rendezvous_req_cpus; /* (x) rendezvous requested */ 1485fcf252fSNeel Natu cpuset_t rendezvous_done_cpus; /* (x) rendezvous finished */ 1495fcf252fSNeel Natu void *rendezvous_arg; /* (x) rendezvous func/arg */ 1505b8a8cd1SNeel Natu vm_rendezvous_func_t rendezvous_func; 1515fcf252fSNeel Natu struct mtx rendezvous_mtx; /* (o) rendezvous lock */ 1525fcf252fSNeel Natu int num_mem_segs; /* (o) guest memory segments */ 1535fcf252fSNeel Natu struct mem_seg mem_segs[VM_MAX_MEMORY_SEGMENTS]; 1545fcf252fSNeel Natu struct vmspace *vmspace; /* (o) guest's address space */ 1555fcf252fSNeel Natu char name[VM_MAX_NAMELEN]; /* (o) virtual machine name */ 1565fcf252fSNeel Natu struct vcpu vcpu[VM_MAXCPU]; /* (i) guest vcpus */ 157366f6083SPeter Grehan }; 158366f6083SPeter Grehan 159d5408b1dSNeel Natu static int vmm_initialized; 160d5408b1dSNeel Natu 161366f6083SPeter Grehan static struct vmm_ops *ops; 162add611fdSNeel Natu #define VMM_INIT(num) (ops != NULL ? (*ops->init)(num) : 0) 163366f6083SPeter Grehan #define VMM_CLEANUP() (ops != NULL ? (*ops->cleanup)() : 0) 16463e62d39SJohn Baldwin #define VMM_RESUME() (ops != NULL ? (*ops->resume)() : 0) 165366f6083SPeter Grehan 166318224bbSNeel Natu #define VMINIT(vm, pmap) (ops != NULL ? (*ops->vminit)(vm, pmap): NULL) 167b15a09c0SNeel Natu #define VMRUN(vmi, vcpu, rip, pmap, rptr, sptr) \ 168b15a09c0SNeel Natu (ops != NULL ? (*ops->vmrun)(vmi, vcpu, rip, pmap, rptr, sptr) : ENXIO) 169366f6083SPeter Grehan #define VMCLEANUP(vmi) (ops != NULL ? (*ops->vmcleanup)(vmi) : NULL) 170318224bbSNeel Natu #define VMSPACE_ALLOC(min, max) \ 171318224bbSNeel Natu (ops != NULL ? (*ops->vmspace_alloc)(min, max) : NULL) 172318224bbSNeel Natu #define VMSPACE_FREE(vmspace) \ 173318224bbSNeel Natu (ops != NULL ? (*ops->vmspace_free)(vmspace) : ENXIO) 174366f6083SPeter Grehan #define VMGETREG(vmi, vcpu, num, retval) \ 175366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetreg)(vmi, vcpu, num, retval) : ENXIO) 176366f6083SPeter Grehan #define VMSETREG(vmi, vcpu, num, val) \ 177366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetreg)(vmi, vcpu, num, val) : ENXIO) 178366f6083SPeter Grehan #define VMGETDESC(vmi, vcpu, num, desc) \ 179366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetdesc)(vmi, vcpu, num, desc) : ENXIO) 180366f6083SPeter Grehan #define VMSETDESC(vmi, vcpu, num, desc) \ 181366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetdesc)(vmi, vcpu, num, desc) : ENXIO) 182366f6083SPeter Grehan #define VMGETCAP(vmi, vcpu, num, retval) \ 183366f6083SPeter Grehan (ops != NULL ? (*ops->vmgetcap)(vmi, vcpu, num, retval) : ENXIO) 184366f6083SPeter Grehan #define VMSETCAP(vmi, vcpu, num, val) \ 185366f6083SPeter Grehan (ops != NULL ? (*ops->vmsetcap)(vmi, vcpu, num, val) : ENXIO) 186de5ea6b6SNeel Natu #define VLAPIC_INIT(vmi, vcpu) \ 187de5ea6b6SNeel Natu (ops != NULL ? (*ops->vlapic_init)(vmi, vcpu) : NULL) 188de5ea6b6SNeel Natu #define VLAPIC_CLEANUP(vmi, vlapic) \ 189de5ea6b6SNeel Natu (ops != NULL ? (*ops->vlapic_cleanup)(vmi, vlapic) : NULL) 190366f6083SPeter Grehan 191014a52f3SNeel Natu #define fpu_start_emulating() load_cr0(rcr0() | CR0_TS) 192014a52f3SNeel Natu #define fpu_stop_emulating() clts() 193366f6083SPeter Grehan 194366f6083SPeter Grehan static MALLOC_DEFINE(M_VM, "vm", "vm"); 195366f6083SPeter Grehan 196366f6083SPeter Grehan /* statistics */ 19761592433SNeel Natu static VMM_STAT(VCPU_TOTAL_RUNTIME, "vcpu total runtime"); 198366f6083SPeter Grehan 199add611fdSNeel Natu SYSCTL_NODE(_hw, OID_AUTO, vmm, CTLFLAG_RW, NULL, NULL); 200add611fdSNeel Natu 201055fc2cbSNeel Natu /* 202055fc2cbSNeel Natu * Halt the guest if all vcpus are executing a HLT instruction with 203055fc2cbSNeel Natu * interrupts disabled. 204055fc2cbSNeel Natu */ 205055fc2cbSNeel Natu static int halt_detection_enabled = 1; 206055fc2cbSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, halt_detection, CTLFLAG_RDTUN, 207055fc2cbSNeel Natu &halt_detection_enabled, 0, 208055fc2cbSNeel Natu "Halt VM if all vcpus execute HLT with interrupts disabled"); 209055fc2cbSNeel Natu 210add611fdSNeel Natu static int vmm_ipinum; 211add611fdSNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, ipinum, CTLFLAG_RD, &vmm_ipinum, 0, 212add611fdSNeel Natu "IPI vector used for vcpu notifications"); 213add611fdSNeel Natu 214b0538143SNeel Natu static int trace_guest_exceptions; 215b0538143SNeel Natu SYSCTL_INT(_hw_vmm, OID_AUTO, trace_guest_exceptions, CTLFLAG_RDTUN, 216b0538143SNeel Natu &trace_guest_exceptions, 0, 217b0538143SNeel Natu "Trap into hypervisor on all guest exceptions and reflect them back"); 218b0538143SNeel Natu 219a15f820aSRyan Stone static int vmm_force_iommu = 0; 220a15f820aSRyan Stone TUNABLE_INT("hw.vmm.force_iommu", &vmm_force_iommu); 221a15f820aSRyan Stone SYSCTL_INT(_hw_vmm, OID_AUTO, force_iommu, CTLFLAG_RDTUN, &vmm_force_iommu, 0, 222a15f820aSRyan Stone "Force use of I/O MMU even if no passthrough devices were found."); 223a15f820aSRyan Stone 224366f6083SPeter Grehan static void 2255fcf252fSNeel Natu vcpu_cleanup(struct vm *vm, int i, bool destroy) 226366f6083SPeter Grehan { 227de5ea6b6SNeel Natu struct vcpu *vcpu = &vm->vcpu[i]; 228de5ea6b6SNeel Natu 229de5ea6b6SNeel Natu VLAPIC_CLEANUP(vm->cookie, vcpu->vlapic); 2305fcf252fSNeel Natu if (destroy) { 231366f6083SPeter Grehan vmm_stat_free(vcpu->stats); 23238f1b189SPeter Grehan fpu_save_area_free(vcpu->guestfpu); 233366f6083SPeter Grehan } 2345fcf252fSNeel Natu } 235366f6083SPeter Grehan 236366f6083SPeter Grehan static void 2375fcf252fSNeel Natu vcpu_init(struct vm *vm, int vcpu_id, bool create) 238366f6083SPeter Grehan { 239366f6083SPeter Grehan struct vcpu *vcpu; 240366f6083SPeter Grehan 2415fcf252fSNeel Natu KASSERT(vcpu_id >= 0 && vcpu_id < VM_MAXCPU, 2425fcf252fSNeel Natu ("vcpu_init: invalid vcpu %d", vcpu_id)); 2435fcf252fSNeel Natu 244366f6083SPeter Grehan vcpu = &vm->vcpu[vcpu_id]; 245366f6083SPeter Grehan 2465fcf252fSNeel Natu if (create) { 2475fcf252fSNeel Natu KASSERT(!vcpu_lock_initialized(vcpu), ("vcpu %d already " 2485fcf252fSNeel Natu "initialized", vcpu_id)); 24975dd3366SNeel Natu vcpu_lock_init(vcpu); 2505fcf252fSNeel Natu vcpu->state = VCPU_IDLE; 25175dd3366SNeel Natu vcpu->hostcpu = NOCPU; 2525fcf252fSNeel Natu vcpu->guestfpu = fpu_save_area_alloc(); 2535fcf252fSNeel Natu vcpu->stats = vmm_stat_alloc(); 2545fcf252fSNeel Natu } 2555fcf252fSNeel Natu 256de5ea6b6SNeel Natu vcpu->vlapic = VLAPIC_INIT(vm->cookie, vcpu_id); 25752e5c8a2SNeel Natu vm_set_x2apic_state(vm, vcpu_id, X2APIC_DISABLED); 258091d4532SNeel Natu vcpu->exitintinfo = 0; 2595fcf252fSNeel Natu vcpu->nmi_pending = 0; 2605fcf252fSNeel Natu vcpu->extint_pending = 0; 2615fcf252fSNeel Natu vcpu->exception_pending = 0; 262abb023fbSJohn Baldwin vcpu->guest_xcr0 = XFEATURE_ENABLED_X87; 26338f1b189SPeter Grehan fpu_save_area_reset(vcpu->guestfpu); 2645fcf252fSNeel Natu vmm_stat_init(vcpu->stats); 265366f6083SPeter Grehan } 266366f6083SPeter Grehan 267b0538143SNeel Natu int 268b0538143SNeel Natu vcpu_trace_exceptions(struct vm *vm, int vcpuid) 269b0538143SNeel Natu { 270b0538143SNeel Natu 271b0538143SNeel Natu return (trace_guest_exceptions); 272b0538143SNeel Natu } 273b0538143SNeel Natu 27498ed632cSNeel Natu struct vm_exit * 27598ed632cSNeel Natu vm_exitinfo(struct vm *vm, int cpuid) 27698ed632cSNeel Natu { 27798ed632cSNeel Natu struct vcpu *vcpu; 27898ed632cSNeel Natu 27998ed632cSNeel Natu if (cpuid < 0 || cpuid >= VM_MAXCPU) 28098ed632cSNeel Natu panic("vm_exitinfo: invalid cpuid %d", cpuid); 28198ed632cSNeel Natu 28298ed632cSNeel Natu vcpu = &vm->vcpu[cpuid]; 28398ed632cSNeel Natu 28498ed632cSNeel Natu return (&vcpu->exitinfo); 28598ed632cSNeel Natu } 28698ed632cSNeel Natu 28763e62d39SJohn Baldwin static void 28863e62d39SJohn Baldwin vmm_resume(void) 28963e62d39SJohn Baldwin { 29063e62d39SJohn Baldwin VMM_RESUME(); 29163e62d39SJohn Baldwin } 29263e62d39SJohn Baldwin 293366f6083SPeter Grehan static int 294366f6083SPeter Grehan vmm_init(void) 295366f6083SPeter Grehan { 296366f6083SPeter Grehan int error; 297366f6083SPeter Grehan 298b01c2033SNeel Natu vmm_host_state_init(); 299add611fdSNeel Natu 30018a2b08eSNeel Natu vmm_ipinum = lapic_ipi_alloc(&IDTVEC(justreturn)); 30118a2b08eSNeel Natu if (vmm_ipinum < 0) 302add611fdSNeel Natu vmm_ipinum = IPI_AST; 303366f6083SPeter Grehan 304366f6083SPeter Grehan error = vmm_mem_init(); 305366f6083SPeter Grehan if (error) 306366f6083SPeter Grehan return (error); 307366f6083SPeter Grehan 308366f6083SPeter Grehan if (vmm_is_intel()) 309366f6083SPeter Grehan ops = &vmm_ops_intel; 310366f6083SPeter Grehan else if (vmm_is_amd()) 311366f6083SPeter Grehan ops = &vmm_ops_amd; 312366f6083SPeter Grehan else 313366f6083SPeter Grehan return (ENXIO); 314366f6083SPeter Grehan 31563e62d39SJohn Baldwin vmm_resume_p = vmm_resume; 316366f6083SPeter Grehan 317add611fdSNeel Natu return (VMM_INIT(vmm_ipinum)); 318366f6083SPeter Grehan } 319366f6083SPeter Grehan 320366f6083SPeter Grehan static int 321366f6083SPeter Grehan vmm_handler(module_t mod, int what, void *arg) 322366f6083SPeter Grehan { 323366f6083SPeter Grehan int error; 324366f6083SPeter Grehan 325366f6083SPeter Grehan switch (what) { 326366f6083SPeter Grehan case MOD_LOAD: 327366f6083SPeter Grehan vmmdev_init(); 328a15f820aSRyan Stone if (vmm_force_iommu || ppt_avail_devices() > 0) 329366f6083SPeter Grehan iommu_init(); 330366f6083SPeter Grehan error = vmm_init(); 331d5408b1dSNeel Natu if (error == 0) 332d5408b1dSNeel Natu vmm_initialized = 1; 333366f6083SPeter Grehan break; 334366f6083SPeter Grehan case MOD_UNLOAD: 335cdc5b9e7SNeel Natu error = vmmdev_cleanup(); 336cdc5b9e7SNeel Natu if (error == 0) { 33763e62d39SJohn Baldwin vmm_resume_p = NULL; 338366f6083SPeter Grehan iommu_cleanup(); 339add611fdSNeel Natu if (vmm_ipinum != IPI_AST) 34018a2b08eSNeel Natu lapic_ipi_free(vmm_ipinum); 341366f6083SPeter Grehan error = VMM_CLEANUP(); 34281ef6611SPeter Grehan /* 34381ef6611SPeter Grehan * Something bad happened - prevent new 34481ef6611SPeter Grehan * VMs from being created 34581ef6611SPeter Grehan */ 34681ef6611SPeter Grehan if (error) 347d5408b1dSNeel Natu vmm_initialized = 0; 34881ef6611SPeter Grehan } 349366f6083SPeter Grehan break; 350366f6083SPeter Grehan default: 351366f6083SPeter Grehan error = 0; 352366f6083SPeter Grehan break; 353366f6083SPeter Grehan } 354366f6083SPeter Grehan return (error); 355366f6083SPeter Grehan } 356366f6083SPeter Grehan 357366f6083SPeter Grehan static moduledata_t vmm_kmod = { 358366f6083SPeter Grehan "vmm", 359366f6083SPeter Grehan vmm_handler, 360366f6083SPeter Grehan NULL 361366f6083SPeter Grehan }; 362366f6083SPeter Grehan 363366f6083SPeter Grehan /* 364e3f0800bSNeel Natu * vmm initialization has the following dependencies: 365e3f0800bSNeel Natu * 366e3f0800bSNeel Natu * - iommu initialization must happen after the pci passthru driver has had 367e3f0800bSNeel Natu * a chance to attach to any passthru devices (after SI_SUB_CONFIGURE). 368e3f0800bSNeel Natu * 369e3f0800bSNeel Natu * - VT-x initialization requires smp_rendezvous() and therefore must happen 370e3f0800bSNeel Natu * after SMP is fully functional (after SI_SUB_SMP). 371366f6083SPeter Grehan */ 372e3f0800bSNeel Natu DECLARE_MODULE(vmm, vmm_kmod, SI_SUB_SMP + 1, SI_ORDER_ANY); 373366f6083SPeter Grehan MODULE_VERSION(vmm, 1); 374366f6083SPeter Grehan 3755fcf252fSNeel Natu static void 3765fcf252fSNeel Natu vm_init(struct vm *vm, bool create) 3775fcf252fSNeel Natu { 3785fcf252fSNeel Natu int i; 3795fcf252fSNeel Natu 3805fcf252fSNeel Natu vm->cookie = VMINIT(vm, vmspace_pmap(vm->vmspace)); 3815fcf252fSNeel Natu vm->iommu = NULL; 3825fcf252fSNeel Natu vm->vioapic = vioapic_init(vm); 3835fcf252fSNeel Natu vm->vhpet = vhpet_init(vm); 3845fcf252fSNeel Natu vm->vatpic = vatpic_init(vm); 3855fcf252fSNeel Natu vm->vatpit = vatpit_init(vm); 386160ef77aSNeel Natu vm->vpmtmr = vpmtmr_init(vm); 3870dafa5cdSNeel Natu if (create) 3880dafa5cdSNeel Natu vm->vrtc = vrtc_init(vm); 3895fcf252fSNeel Natu 3905fcf252fSNeel Natu CPU_ZERO(&vm->active_cpus); 3915fcf252fSNeel Natu 3925fcf252fSNeel Natu vm->suspend = 0; 3935fcf252fSNeel Natu CPU_ZERO(&vm->suspended_cpus); 3945fcf252fSNeel Natu 3955fcf252fSNeel Natu for (i = 0; i < VM_MAXCPU; i++) 3965fcf252fSNeel Natu vcpu_init(vm, i, create); 3975fcf252fSNeel Natu } 3985fcf252fSNeel Natu 399d5408b1dSNeel Natu int 400d5408b1dSNeel Natu vm_create(const char *name, struct vm **retvm) 401366f6083SPeter Grehan { 402366f6083SPeter Grehan struct vm *vm; 403318224bbSNeel Natu struct vmspace *vmspace; 404366f6083SPeter Grehan 405d5408b1dSNeel Natu /* 406d5408b1dSNeel Natu * If vmm.ko could not be successfully initialized then don't attempt 407d5408b1dSNeel Natu * to create the virtual machine. 408d5408b1dSNeel Natu */ 409d5408b1dSNeel Natu if (!vmm_initialized) 410d5408b1dSNeel Natu return (ENXIO); 411d5408b1dSNeel Natu 412366f6083SPeter Grehan if (name == NULL || strlen(name) >= VM_MAX_NAMELEN) 413d5408b1dSNeel Natu return (EINVAL); 414366f6083SPeter Grehan 415526c8885SPeter Grehan vmspace = VMSPACE_ALLOC(0, VM_MAXUSER_ADDRESS); 416318224bbSNeel Natu if (vmspace == NULL) 417318224bbSNeel Natu return (ENOMEM); 418318224bbSNeel Natu 419366f6083SPeter Grehan vm = malloc(sizeof(struct vm), M_VM, M_WAITOK | M_ZERO); 420366f6083SPeter Grehan strcpy(vm->name, name); 4215fcf252fSNeel Natu vm->num_mem_segs = 0; 42288c4b8d1SNeel Natu vm->vmspace = vmspace; 4235b8a8cd1SNeel Natu mtx_init(&vm->rendezvous_mtx, "vm rendezvous lock", 0, MTX_DEF); 424366f6083SPeter Grehan 4255fcf252fSNeel Natu vm_init(vm, true); 426366f6083SPeter Grehan 427d5408b1dSNeel Natu *retvm = vm; 428d5408b1dSNeel Natu return (0); 429366f6083SPeter Grehan } 430366f6083SPeter Grehan 431f7d51510SNeel Natu static void 432318224bbSNeel Natu vm_free_mem_seg(struct vm *vm, struct mem_seg *seg) 433f7d51510SNeel Natu { 4347ce04d0aSNeel Natu 435318224bbSNeel Natu if (seg->object != NULL) 436318224bbSNeel Natu vmm_mem_free(vm->vmspace, seg->gpa, seg->len); 437f7d51510SNeel Natu 438318224bbSNeel Natu bzero(seg, sizeof(*seg)); 439f7d51510SNeel Natu } 440f7d51510SNeel Natu 4415fcf252fSNeel Natu static void 4425fcf252fSNeel Natu vm_cleanup(struct vm *vm, bool destroy) 443366f6083SPeter Grehan { 444366f6083SPeter Grehan int i; 445366f6083SPeter Grehan 446366f6083SPeter Grehan ppt_unassign_all(vm); 447366f6083SPeter Grehan 448318224bbSNeel Natu if (vm->iommu != NULL) 449318224bbSNeel Natu iommu_destroy_domain(vm->iommu); 450318224bbSNeel Natu 4510dafa5cdSNeel Natu if (destroy) 4520dafa5cdSNeel Natu vrtc_cleanup(vm->vrtc); 4530dafa5cdSNeel Natu else 4540dafa5cdSNeel Natu vrtc_reset(vm->vrtc); 455160ef77aSNeel Natu vpmtmr_cleanup(vm->vpmtmr); 456e883c9bbSTycho Nightingale vatpit_cleanup(vm->vatpit); 45708e3ff32SNeel Natu vhpet_cleanup(vm->vhpet); 458762fd208STycho Nightingale vatpic_cleanup(vm->vatpic); 45908e3ff32SNeel Natu vioapic_cleanup(vm->vioapic); 46008e3ff32SNeel Natu 4615fcf252fSNeel Natu for (i = 0; i < VM_MAXCPU; i++) 4625fcf252fSNeel Natu vcpu_cleanup(vm, i, destroy); 4635fcf252fSNeel Natu 4645fcf252fSNeel Natu VMCLEANUP(vm->cookie); 4655fcf252fSNeel Natu 4665fcf252fSNeel Natu if (destroy) { 467366f6083SPeter Grehan for (i = 0; i < vm->num_mem_segs; i++) 468f7d51510SNeel Natu vm_free_mem_seg(vm, &vm->mem_segs[i]); 469f7d51510SNeel Natu 470f7d51510SNeel Natu vm->num_mem_segs = 0; 471366f6083SPeter Grehan 472318224bbSNeel Natu VMSPACE_FREE(vm->vmspace); 4735fcf252fSNeel Natu vm->vmspace = NULL; 4745fcf252fSNeel Natu } 4755fcf252fSNeel Natu } 476366f6083SPeter Grehan 4775fcf252fSNeel Natu void 4785fcf252fSNeel Natu vm_destroy(struct vm *vm) 4795fcf252fSNeel Natu { 4805fcf252fSNeel Natu vm_cleanup(vm, true); 481366f6083SPeter Grehan free(vm, M_VM); 482366f6083SPeter Grehan } 483366f6083SPeter Grehan 4845fcf252fSNeel Natu int 4855fcf252fSNeel Natu vm_reinit(struct vm *vm) 4865fcf252fSNeel Natu { 4875fcf252fSNeel Natu int error; 4885fcf252fSNeel Natu 4895fcf252fSNeel Natu /* 4905fcf252fSNeel Natu * A virtual machine can be reset only if all vcpus are suspended. 4915fcf252fSNeel Natu */ 4925fcf252fSNeel Natu if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { 4935fcf252fSNeel Natu vm_cleanup(vm, false); 4945fcf252fSNeel Natu vm_init(vm, false); 4955fcf252fSNeel Natu error = 0; 4965fcf252fSNeel Natu } else { 4975fcf252fSNeel Natu error = EBUSY; 4985fcf252fSNeel Natu } 4995fcf252fSNeel Natu 5005fcf252fSNeel Natu return (error); 5015fcf252fSNeel Natu } 5025fcf252fSNeel Natu 503366f6083SPeter Grehan const char * 504366f6083SPeter Grehan vm_name(struct vm *vm) 505366f6083SPeter Grehan { 506366f6083SPeter Grehan return (vm->name); 507366f6083SPeter Grehan } 508366f6083SPeter Grehan 509366f6083SPeter Grehan int 510366f6083SPeter Grehan vm_map_mmio(struct vm *vm, vm_paddr_t gpa, size_t len, vm_paddr_t hpa) 511366f6083SPeter Grehan { 512318224bbSNeel Natu vm_object_t obj; 513366f6083SPeter Grehan 514318224bbSNeel Natu if ((obj = vmm_mmio_alloc(vm->vmspace, gpa, len, hpa)) == NULL) 515318224bbSNeel Natu return (ENOMEM); 516318224bbSNeel Natu else 517318224bbSNeel Natu return (0); 518366f6083SPeter Grehan } 519366f6083SPeter Grehan 520366f6083SPeter Grehan int 521366f6083SPeter Grehan vm_unmap_mmio(struct vm *vm, vm_paddr_t gpa, size_t len) 522366f6083SPeter Grehan { 523366f6083SPeter Grehan 524318224bbSNeel Natu vmm_mmio_free(vm->vmspace, gpa, len); 525318224bbSNeel Natu return (0); 526366f6083SPeter Grehan } 527366f6083SPeter Grehan 528318224bbSNeel Natu boolean_t 529318224bbSNeel Natu vm_mem_allocated(struct vm *vm, vm_paddr_t gpa) 530366f6083SPeter Grehan { 531341f19c9SNeel Natu int i; 532341f19c9SNeel Natu vm_paddr_t gpabase, gpalimit; 533341f19c9SNeel Natu 534341f19c9SNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 535341f19c9SNeel Natu gpabase = vm->mem_segs[i].gpa; 536341f19c9SNeel Natu gpalimit = gpabase + vm->mem_segs[i].len; 537341f19c9SNeel Natu if (gpa >= gpabase && gpa < gpalimit) 538318224bbSNeel Natu return (TRUE); /* 'gpa' is regular memory */ 539341f19c9SNeel Natu } 540341f19c9SNeel Natu 541318224bbSNeel Natu if (ppt_is_mmio(vm, gpa)) 542318224bbSNeel Natu return (TRUE); /* 'gpa' is pci passthru mmio */ 543318224bbSNeel Natu 544318224bbSNeel Natu return (FALSE); 545341f19c9SNeel Natu } 546341f19c9SNeel Natu 547341f19c9SNeel Natu int 548341f19c9SNeel Natu vm_malloc(struct vm *vm, vm_paddr_t gpa, size_t len) 549341f19c9SNeel Natu { 550318224bbSNeel Natu int available, allocated; 551318224bbSNeel Natu struct mem_seg *seg; 552318224bbSNeel Natu vm_object_t object; 553318224bbSNeel Natu vm_paddr_t g; 554366f6083SPeter Grehan 555341f19c9SNeel Natu if ((gpa & PAGE_MASK) || (len & PAGE_MASK) || len == 0) 556341f19c9SNeel Natu return (EINVAL); 557341f19c9SNeel Natu 558341f19c9SNeel Natu available = allocated = 0; 559341f19c9SNeel Natu g = gpa; 560341f19c9SNeel Natu while (g < gpa + len) { 561318224bbSNeel Natu if (vm_mem_allocated(vm, g)) 562341f19c9SNeel Natu allocated++; 563318224bbSNeel Natu else 564318224bbSNeel Natu available++; 565341f19c9SNeel Natu 566341f19c9SNeel Natu g += PAGE_SIZE; 567341f19c9SNeel Natu } 568341f19c9SNeel Natu 569366f6083SPeter Grehan /* 570341f19c9SNeel Natu * If there are some allocated and some available pages in the address 571341f19c9SNeel Natu * range then it is an error. 572366f6083SPeter Grehan */ 573341f19c9SNeel Natu if (allocated && available) 574341f19c9SNeel Natu return (EINVAL); 575341f19c9SNeel Natu 576341f19c9SNeel Natu /* 577341f19c9SNeel Natu * If the entire address range being requested has already been 578341f19c9SNeel Natu * allocated then there isn't anything more to do. 579341f19c9SNeel Natu */ 580341f19c9SNeel Natu if (allocated && available == 0) 581341f19c9SNeel Natu return (0); 582366f6083SPeter Grehan 583366f6083SPeter Grehan if (vm->num_mem_segs >= VM_MAX_MEMORY_SEGMENTS) 584366f6083SPeter Grehan return (E2BIG); 585366f6083SPeter Grehan 586f7d51510SNeel Natu seg = &vm->mem_segs[vm->num_mem_segs]; 587366f6083SPeter Grehan 588318224bbSNeel Natu if ((object = vmm_mem_alloc(vm->vmspace, gpa, len)) == NULL) 589318224bbSNeel Natu return (ENOMEM); 590318224bbSNeel Natu 591f7d51510SNeel Natu seg->gpa = gpa; 592318224bbSNeel Natu seg->len = len; 593318224bbSNeel Natu seg->object = object; 594318224bbSNeel Natu seg->wired = FALSE; 5957ce04d0aSNeel Natu 596366f6083SPeter Grehan vm->num_mem_segs++; 597341f19c9SNeel Natu 598366f6083SPeter Grehan return (0); 599366f6083SPeter Grehan } 600366f6083SPeter Grehan 601477867a0SNeel Natu static vm_paddr_t 602477867a0SNeel Natu vm_maxmem(struct vm *vm) 603477867a0SNeel Natu { 604477867a0SNeel Natu int i; 605477867a0SNeel Natu vm_paddr_t gpa, maxmem; 606477867a0SNeel Natu 607477867a0SNeel Natu maxmem = 0; 608477867a0SNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 609477867a0SNeel Natu gpa = vm->mem_segs[i].gpa + vm->mem_segs[i].len; 610477867a0SNeel Natu if (gpa > maxmem) 611477867a0SNeel Natu maxmem = gpa; 612477867a0SNeel Natu } 613477867a0SNeel Natu return (maxmem); 614477867a0SNeel Natu } 615477867a0SNeel Natu 616318224bbSNeel Natu static void 617318224bbSNeel Natu vm_gpa_unwire(struct vm *vm) 618366f6083SPeter Grehan { 619318224bbSNeel Natu int i, rv; 620318224bbSNeel Natu struct mem_seg *seg; 6214db4fb2cSNeel Natu 622318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 623318224bbSNeel Natu seg = &vm->mem_segs[i]; 624318224bbSNeel Natu if (!seg->wired) 625318224bbSNeel Natu continue; 626366f6083SPeter Grehan 627318224bbSNeel Natu rv = vm_map_unwire(&vm->vmspace->vm_map, 628318224bbSNeel Natu seg->gpa, seg->gpa + seg->len, 629318224bbSNeel Natu VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 630318224bbSNeel Natu KASSERT(rv == KERN_SUCCESS, ("vm(%s) memory segment " 631318224bbSNeel Natu "%#lx/%ld could not be unwired: %d", 632318224bbSNeel Natu vm_name(vm), seg->gpa, seg->len, rv)); 633318224bbSNeel Natu 634318224bbSNeel Natu seg->wired = FALSE; 635318224bbSNeel Natu } 636318224bbSNeel Natu } 637318224bbSNeel Natu 638318224bbSNeel Natu static int 639318224bbSNeel Natu vm_gpa_wire(struct vm *vm) 640318224bbSNeel Natu { 641318224bbSNeel Natu int i, rv; 642318224bbSNeel Natu struct mem_seg *seg; 643318224bbSNeel Natu 644318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 645318224bbSNeel Natu seg = &vm->mem_segs[i]; 646318224bbSNeel Natu if (seg->wired) 647318224bbSNeel Natu continue; 648318224bbSNeel Natu 649318224bbSNeel Natu /* XXX rlimits? */ 650318224bbSNeel Natu rv = vm_map_wire(&vm->vmspace->vm_map, 651318224bbSNeel Natu seg->gpa, seg->gpa + seg->len, 652318224bbSNeel Natu VM_MAP_WIRE_USER | VM_MAP_WIRE_NOHOLES); 653318224bbSNeel Natu if (rv != KERN_SUCCESS) 654318224bbSNeel Natu break; 655318224bbSNeel Natu 656318224bbSNeel Natu seg->wired = TRUE; 657318224bbSNeel Natu } 658318224bbSNeel Natu 659318224bbSNeel Natu if (i < vm->num_mem_segs) { 660318224bbSNeel Natu /* 661318224bbSNeel Natu * Undo the wiring before returning an error. 662318224bbSNeel Natu */ 663318224bbSNeel Natu vm_gpa_unwire(vm); 664318224bbSNeel Natu return (EAGAIN); 665318224bbSNeel Natu } 666318224bbSNeel Natu 667318224bbSNeel Natu return (0); 668318224bbSNeel Natu } 669318224bbSNeel Natu 670318224bbSNeel Natu static void 671318224bbSNeel Natu vm_iommu_modify(struct vm *vm, boolean_t map) 672318224bbSNeel Natu { 673318224bbSNeel Natu int i, sz; 674318224bbSNeel Natu vm_paddr_t gpa, hpa; 675318224bbSNeel Natu struct mem_seg *seg; 676318224bbSNeel Natu void *vp, *cookie, *host_domain; 677318224bbSNeel Natu 678318224bbSNeel Natu sz = PAGE_SIZE; 679318224bbSNeel Natu host_domain = iommu_host_domain(); 680318224bbSNeel Natu 681318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 682318224bbSNeel Natu seg = &vm->mem_segs[i]; 683318224bbSNeel Natu KASSERT(seg->wired, ("vm(%s) memory segment %#lx/%ld not wired", 684318224bbSNeel Natu vm_name(vm), seg->gpa, seg->len)); 685318224bbSNeel Natu 686318224bbSNeel Natu gpa = seg->gpa; 687318224bbSNeel Natu while (gpa < seg->gpa + seg->len) { 688318224bbSNeel Natu vp = vm_gpa_hold(vm, gpa, PAGE_SIZE, VM_PROT_WRITE, 689318224bbSNeel Natu &cookie); 690318224bbSNeel Natu KASSERT(vp != NULL, ("vm(%s) could not map gpa %#lx", 691318224bbSNeel Natu vm_name(vm), gpa)); 692318224bbSNeel Natu 693318224bbSNeel Natu vm_gpa_release(cookie); 694318224bbSNeel Natu 695318224bbSNeel Natu hpa = DMAP_TO_PHYS((uintptr_t)vp); 696318224bbSNeel Natu if (map) { 697318224bbSNeel Natu iommu_create_mapping(vm->iommu, gpa, hpa, sz); 698318224bbSNeel Natu iommu_remove_mapping(host_domain, hpa, sz); 699318224bbSNeel Natu } else { 700318224bbSNeel Natu iommu_remove_mapping(vm->iommu, gpa, sz); 701318224bbSNeel Natu iommu_create_mapping(host_domain, hpa, hpa, sz); 702318224bbSNeel Natu } 703318224bbSNeel Natu 704318224bbSNeel Natu gpa += PAGE_SIZE; 705318224bbSNeel Natu } 706318224bbSNeel Natu } 707318224bbSNeel Natu 708318224bbSNeel Natu /* 709318224bbSNeel Natu * Invalidate the cached translations associated with the domain 710318224bbSNeel Natu * from which pages were removed. 711318224bbSNeel Natu */ 712318224bbSNeel Natu if (map) 713318224bbSNeel Natu iommu_invalidate_tlb(host_domain); 714318224bbSNeel Natu else 715318224bbSNeel Natu iommu_invalidate_tlb(vm->iommu); 716318224bbSNeel Natu } 717318224bbSNeel Natu 718318224bbSNeel Natu #define vm_iommu_unmap(vm) vm_iommu_modify((vm), FALSE) 719318224bbSNeel Natu #define vm_iommu_map(vm) vm_iommu_modify((vm), TRUE) 720318224bbSNeel Natu 721318224bbSNeel Natu int 722318224bbSNeel Natu vm_unassign_pptdev(struct vm *vm, int bus, int slot, int func) 723318224bbSNeel Natu { 724318224bbSNeel Natu int error; 725318224bbSNeel Natu 726318224bbSNeel Natu error = ppt_unassign_device(vm, bus, slot, func); 727318224bbSNeel Natu if (error) 728318224bbSNeel Natu return (error); 729318224bbSNeel Natu 73051f45d01SNeel Natu if (ppt_assigned_devices(vm) == 0) { 731318224bbSNeel Natu vm_iommu_unmap(vm); 732318224bbSNeel Natu vm_gpa_unwire(vm); 733318224bbSNeel Natu } 734318224bbSNeel Natu return (0); 735318224bbSNeel Natu } 736318224bbSNeel Natu 737318224bbSNeel Natu int 738318224bbSNeel Natu vm_assign_pptdev(struct vm *vm, int bus, int slot, int func) 739318224bbSNeel Natu { 740318224bbSNeel Natu int error; 741318224bbSNeel Natu vm_paddr_t maxaddr; 742318224bbSNeel Natu 743318224bbSNeel Natu /* 744318224bbSNeel Natu * Virtual machines with pci passthru devices get special treatment: 745318224bbSNeel Natu * - the guest physical memory is wired 746318224bbSNeel Natu * - the iommu is programmed to do the 'gpa' to 'hpa' translation 747318224bbSNeel Natu * 748318224bbSNeel Natu * We need to do this before the first pci passthru device is attached. 749318224bbSNeel Natu */ 75051f45d01SNeel Natu if (ppt_assigned_devices(vm) == 0) { 751318224bbSNeel Natu KASSERT(vm->iommu == NULL, 752318224bbSNeel Natu ("vm_assign_pptdev: iommu must be NULL")); 753477867a0SNeel Natu maxaddr = vm_maxmem(vm); 754318224bbSNeel Natu vm->iommu = iommu_create_domain(maxaddr); 755318224bbSNeel Natu 756318224bbSNeel Natu error = vm_gpa_wire(vm); 757318224bbSNeel Natu if (error) 758318224bbSNeel Natu return (error); 759318224bbSNeel Natu 760318224bbSNeel Natu vm_iommu_map(vm); 761318224bbSNeel Natu } 762318224bbSNeel Natu 763318224bbSNeel Natu error = ppt_assign_device(vm, bus, slot, func); 764318224bbSNeel Natu return (error); 765318224bbSNeel Natu } 766318224bbSNeel Natu 767318224bbSNeel Natu void * 768318224bbSNeel Natu vm_gpa_hold(struct vm *vm, vm_paddr_t gpa, size_t len, int reqprot, 769318224bbSNeel Natu void **cookie) 770318224bbSNeel Natu { 771318224bbSNeel Natu int count, pageoff; 772318224bbSNeel Natu vm_page_t m; 773318224bbSNeel Natu 774318224bbSNeel Natu pageoff = gpa & PAGE_MASK; 775318224bbSNeel Natu if (len > PAGE_SIZE - pageoff) 776318224bbSNeel Natu panic("vm_gpa_hold: invalid gpa/len: 0x%016lx/%lu", gpa, len); 777318224bbSNeel Natu 778318224bbSNeel Natu count = vm_fault_quick_hold_pages(&vm->vmspace->vm_map, 779318224bbSNeel Natu trunc_page(gpa), PAGE_SIZE, reqprot, &m, 1); 780318224bbSNeel Natu 781318224bbSNeel Natu if (count == 1) { 782318224bbSNeel Natu *cookie = m; 783318224bbSNeel Natu return ((void *)(PHYS_TO_DMAP(VM_PAGE_TO_PHYS(m)) + pageoff)); 784318224bbSNeel Natu } else { 785318224bbSNeel Natu *cookie = NULL; 786318224bbSNeel Natu return (NULL); 787318224bbSNeel Natu } 788318224bbSNeel Natu } 789318224bbSNeel Natu 790318224bbSNeel Natu void 791318224bbSNeel Natu vm_gpa_release(void *cookie) 792318224bbSNeel Natu { 793318224bbSNeel Natu vm_page_t m = cookie; 794318224bbSNeel Natu 795318224bbSNeel Natu vm_page_lock(m); 796318224bbSNeel Natu vm_page_unhold(m); 797318224bbSNeel Natu vm_page_unlock(m); 798366f6083SPeter Grehan } 799366f6083SPeter Grehan 800366f6083SPeter Grehan int 801366f6083SPeter Grehan vm_gpabase2memseg(struct vm *vm, vm_paddr_t gpabase, 802366f6083SPeter Grehan struct vm_memory_segment *seg) 803366f6083SPeter Grehan { 804366f6083SPeter Grehan int i; 805366f6083SPeter Grehan 806366f6083SPeter Grehan for (i = 0; i < vm->num_mem_segs; i++) { 807366f6083SPeter Grehan if (gpabase == vm->mem_segs[i].gpa) { 808318224bbSNeel Natu seg->gpa = vm->mem_segs[i].gpa; 809318224bbSNeel Natu seg->len = vm->mem_segs[i].len; 810318224bbSNeel Natu seg->wired = vm->mem_segs[i].wired; 811366f6083SPeter Grehan return (0); 812366f6083SPeter Grehan } 813366f6083SPeter Grehan } 814366f6083SPeter Grehan return (-1); 815366f6083SPeter Grehan } 816366f6083SPeter Grehan 817366f6083SPeter Grehan int 818318224bbSNeel Natu vm_get_memobj(struct vm *vm, vm_paddr_t gpa, size_t len, 819318224bbSNeel Natu vm_offset_t *offset, struct vm_object **object) 820318224bbSNeel Natu { 821318224bbSNeel Natu int i; 822318224bbSNeel Natu size_t seg_len; 823318224bbSNeel Natu vm_paddr_t seg_gpa; 824318224bbSNeel Natu vm_object_t seg_obj; 825318224bbSNeel Natu 826318224bbSNeel Natu for (i = 0; i < vm->num_mem_segs; i++) { 827318224bbSNeel Natu if ((seg_obj = vm->mem_segs[i].object) == NULL) 828318224bbSNeel Natu continue; 829318224bbSNeel Natu 830318224bbSNeel Natu seg_gpa = vm->mem_segs[i].gpa; 831318224bbSNeel Natu seg_len = vm->mem_segs[i].len; 832318224bbSNeel Natu 833318224bbSNeel Natu if (gpa >= seg_gpa && gpa < seg_gpa + seg_len) { 834318224bbSNeel Natu *offset = gpa - seg_gpa; 835318224bbSNeel Natu *object = seg_obj; 836318224bbSNeel Natu vm_object_reference(seg_obj); 837318224bbSNeel Natu return (0); 838318224bbSNeel Natu } 839318224bbSNeel Natu } 840318224bbSNeel Natu 841318224bbSNeel Natu return (EINVAL); 842318224bbSNeel Natu } 843318224bbSNeel Natu 844318224bbSNeel Natu int 845366f6083SPeter Grehan vm_get_register(struct vm *vm, int vcpu, int reg, uint64_t *retval) 846366f6083SPeter Grehan { 847366f6083SPeter Grehan 848366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 849366f6083SPeter Grehan return (EINVAL); 850366f6083SPeter Grehan 851366f6083SPeter Grehan if (reg >= VM_REG_LAST) 852366f6083SPeter Grehan return (EINVAL); 853366f6083SPeter Grehan 854366f6083SPeter Grehan return (VMGETREG(vm->cookie, vcpu, reg, retval)); 855366f6083SPeter Grehan } 856366f6083SPeter Grehan 857366f6083SPeter Grehan int 858d087a399SNeel Natu vm_set_register(struct vm *vm, int vcpuid, int reg, uint64_t val) 859366f6083SPeter Grehan { 860d087a399SNeel Natu struct vcpu *vcpu; 861d087a399SNeel Natu int error; 862366f6083SPeter Grehan 863d087a399SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 864366f6083SPeter Grehan return (EINVAL); 865366f6083SPeter Grehan 866366f6083SPeter Grehan if (reg >= VM_REG_LAST) 867366f6083SPeter Grehan return (EINVAL); 868366f6083SPeter Grehan 869d087a399SNeel Natu error = VMSETREG(vm->cookie, vcpuid, reg, val); 870d087a399SNeel Natu if (error || reg != VM_REG_GUEST_RIP) 871d087a399SNeel Natu return (error); 872d087a399SNeel Natu 873d087a399SNeel Natu /* Set 'nextrip' to match the value of %rip */ 874d087a399SNeel Natu VCPU_CTR1(vm, vcpuid, "Setting nextrip to %#lx", val); 875d087a399SNeel Natu vcpu = &vm->vcpu[vcpuid]; 876d087a399SNeel Natu vcpu->nextrip = val; 877d087a399SNeel Natu return (0); 878366f6083SPeter Grehan } 879366f6083SPeter Grehan 880366f6083SPeter Grehan static boolean_t 881366f6083SPeter Grehan is_descriptor_table(int reg) 882366f6083SPeter Grehan { 883366f6083SPeter Grehan 884366f6083SPeter Grehan switch (reg) { 885366f6083SPeter Grehan case VM_REG_GUEST_IDTR: 886366f6083SPeter Grehan case VM_REG_GUEST_GDTR: 887366f6083SPeter Grehan return (TRUE); 888366f6083SPeter Grehan default: 889366f6083SPeter Grehan return (FALSE); 890366f6083SPeter Grehan } 891366f6083SPeter Grehan } 892366f6083SPeter Grehan 893366f6083SPeter Grehan static boolean_t 894366f6083SPeter Grehan is_segment_register(int reg) 895366f6083SPeter Grehan { 896366f6083SPeter Grehan 897366f6083SPeter Grehan switch (reg) { 898366f6083SPeter Grehan case VM_REG_GUEST_ES: 899366f6083SPeter Grehan case VM_REG_GUEST_CS: 900366f6083SPeter Grehan case VM_REG_GUEST_SS: 901366f6083SPeter Grehan case VM_REG_GUEST_DS: 902366f6083SPeter Grehan case VM_REG_GUEST_FS: 903366f6083SPeter Grehan case VM_REG_GUEST_GS: 904366f6083SPeter Grehan case VM_REG_GUEST_TR: 905366f6083SPeter Grehan case VM_REG_GUEST_LDTR: 906366f6083SPeter Grehan return (TRUE); 907366f6083SPeter Grehan default: 908366f6083SPeter Grehan return (FALSE); 909366f6083SPeter Grehan } 910366f6083SPeter Grehan } 911366f6083SPeter Grehan 912366f6083SPeter Grehan int 913366f6083SPeter Grehan vm_get_seg_desc(struct vm *vm, int vcpu, int reg, 914366f6083SPeter Grehan struct seg_desc *desc) 915366f6083SPeter Grehan { 916366f6083SPeter Grehan 917366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 918366f6083SPeter Grehan return (EINVAL); 919366f6083SPeter Grehan 920366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 921366f6083SPeter Grehan return (EINVAL); 922366f6083SPeter Grehan 923366f6083SPeter Grehan return (VMGETDESC(vm->cookie, vcpu, reg, desc)); 924366f6083SPeter Grehan } 925366f6083SPeter Grehan 926366f6083SPeter Grehan int 927366f6083SPeter Grehan vm_set_seg_desc(struct vm *vm, int vcpu, int reg, 928366f6083SPeter Grehan struct seg_desc *desc) 929366f6083SPeter Grehan { 930366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 931366f6083SPeter Grehan return (EINVAL); 932366f6083SPeter Grehan 933366f6083SPeter Grehan if (!is_segment_register(reg) && !is_descriptor_table(reg)) 934366f6083SPeter Grehan return (EINVAL); 935366f6083SPeter Grehan 936366f6083SPeter Grehan return (VMSETDESC(vm->cookie, vcpu, reg, desc)); 937366f6083SPeter Grehan } 938366f6083SPeter Grehan 939366f6083SPeter Grehan static void 940366f6083SPeter Grehan restore_guest_fpustate(struct vcpu *vcpu) 941366f6083SPeter Grehan { 942366f6083SPeter Grehan 94338f1b189SPeter Grehan /* flush host state to the pcb */ 94438f1b189SPeter Grehan fpuexit(curthread); 945bd8572e0SNeel Natu 946bd8572e0SNeel Natu /* restore guest FPU state */ 947366f6083SPeter Grehan fpu_stop_emulating(); 94838f1b189SPeter Grehan fpurestore(vcpu->guestfpu); 949bd8572e0SNeel Natu 950abb023fbSJohn Baldwin /* restore guest XCR0 if XSAVE is enabled in the host */ 951abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) 952abb023fbSJohn Baldwin load_xcr(0, vcpu->guest_xcr0); 953abb023fbSJohn Baldwin 954bd8572e0SNeel Natu /* 955bd8572e0SNeel Natu * The FPU is now "dirty" with the guest's state so turn on emulation 956bd8572e0SNeel Natu * to trap any access to the FPU by the host. 957bd8572e0SNeel Natu */ 958bd8572e0SNeel Natu fpu_start_emulating(); 959366f6083SPeter Grehan } 960366f6083SPeter Grehan 961366f6083SPeter Grehan static void 962366f6083SPeter Grehan save_guest_fpustate(struct vcpu *vcpu) 963366f6083SPeter Grehan { 964366f6083SPeter Grehan 965bd8572e0SNeel Natu if ((rcr0() & CR0_TS) == 0) 966bd8572e0SNeel Natu panic("fpu emulation not enabled in host!"); 967bd8572e0SNeel Natu 968abb023fbSJohn Baldwin /* save guest XCR0 and restore host XCR0 */ 969abb023fbSJohn Baldwin if (rcr4() & CR4_XSAVE) { 970abb023fbSJohn Baldwin vcpu->guest_xcr0 = rxcr(0); 971abb023fbSJohn Baldwin load_xcr(0, vmm_get_host_xcr0()); 972abb023fbSJohn Baldwin } 973abb023fbSJohn Baldwin 974bd8572e0SNeel Natu /* save guest FPU state */ 975bd8572e0SNeel Natu fpu_stop_emulating(); 97638f1b189SPeter Grehan fpusave(vcpu->guestfpu); 977366f6083SPeter Grehan fpu_start_emulating(); 978366f6083SPeter Grehan } 979366f6083SPeter Grehan 98061592433SNeel Natu static VMM_STAT(VCPU_IDLE_TICKS, "number of ticks vcpu was idle"); 981f76fc5d4SNeel Natu 982318224bbSNeel Natu static int 983f80330a8SNeel Natu vcpu_set_state_locked(struct vcpu *vcpu, enum vcpu_state newstate, 984f80330a8SNeel Natu bool from_idle) 985366f6083SPeter Grehan { 986318224bbSNeel Natu int error; 987366f6083SPeter Grehan 988318224bbSNeel Natu vcpu_assert_locked(vcpu); 989366f6083SPeter Grehan 990f76fc5d4SNeel Natu /* 991f80330a8SNeel Natu * State transitions from the vmmdev_ioctl() must always begin from 992f80330a8SNeel Natu * the VCPU_IDLE state. This guarantees that there is only a single 993f80330a8SNeel Natu * ioctl() operating on a vcpu at any point. 994f80330a8SNeel Natu */ 995f80330a8SNeel Natu if (from_idle) { 996f80330a8SNeel Natu while (vcpu->state != VCPU_IDLE) 997f80330a8SNeel Natu msleep_spin(&vcpu->state, &vcpu->mtx, "vmstat", hz); 998f80330a8SNeel Natu } else { 999f80330a8SNeel Natu KASSERT(vcpu->state != VCPU_IDLE, ("invalid transition from " 1000f80330a8SNeel Natu "vcpu idle state")); 1001f80330a8SNeel Natu } 1002f80330a8SNeel Natu 1003ef39d7e9SNeel Natu if (vcpu->state == VCPU_RUNNING) { 1004ef39d7e9SNeel Natu KASSERT(vcpu->hostcpu == curcpu, ("curcpu %d and hostcpu %d " 1005ef39d7e9SNeel Natu "mismatch for running vcpu", curcpu, vcpu->hostcpu)); 1006ef39d7e9SNeel Natu } else { 1007ef39d7e9SNeel Natu KASSERT(vcpu->hostcpu == NOCPU, ("Invalid hostcpu %d for a " 1008ef39d7e9SNeel Natu "vcpu that is not running", vcpu->hostcpu)); 1009ef39d7e9SNeel Natu } 1010ef39d7e9SNeel Natu 1011f80330a8SNeel Natu /* 1012318224bbSNeel Natu * The following state transitions are allowed: 1013318224bbSNeel Natu * IDLE -> FROZEN -> IDLE 1014318224bbSNeel Natu * FROZEN -> RUNNING -> FROZEN 1015318224bbSNeel Natu * FROZEN -> SLEEPING -> FROZEN 1016f76fc5d4SNeel Natu */ 1017318224bbSNeel Natu switch (vcpu->state) { 1018318224bbSNeel Natu case VCPU_IDLE: 1019318224bbSNeel Natu case VCPU_RUNNING: 1020318224bbSNeel Natu case VCPU_SLEEPING: 1021318224bbSNeel Natu error = (newstate != VCPU_FROZEN); 1022318224bbSNeel Natu break; 1023318224bbSNeel Natu case VCPU_FROZEN: 1024318224bbSNeel Natu error = (newstate == VCPU_FROZEN); 1025318224bbSNeel Natu break; 1026318224bbSNeel Natu default: 1027318224bbSNeel Natu error = 1; 1028318224bbSNeel Natu break; 1029318224bbSNeel Natu } 1030318224bbSNeel Natu 1031f80330a8SNeel Natu if (error) 1032f80330a8SNeel Natu return (EBUSY); 1033318224bbSNeel Natu 1034f80330a8SNeel Natu vcpu->state = newstate; 1035ef39d7e9SNeel Natu if (newstate == VCPU_RUNNING) 1036ef39d7e9SNeel Natu vcpu->hostcpu = curcpu; 1037ef39d7e9SNeel Natu else 1038ef39d7e9SNeel Natu vcpu->hostcpu = NOCPU; 1039ef39d7e9SNeel Natu 1040f80330a8SNeel Natu if (newstate == VCPU_IDLE) 1041f80330a8SNeel Natu wakeup(&vcpu->state); 1042f80330a8SNeel Natu 1043f80330a8SNeel Natu return (0); 1044318224bbSNeel Natu } 1045318224bbSNeel Natu 1046318224bbSNeel Natu static void 1047318224bbSNeel Natu vcpu_require_state(struct vm *vm, int vcpuid, enum vcpu_state newstate) 1048318224bbSNeel Natu { 1049318224bbSNeel Natu int error; 1050318224bbSNeel Natu 1051f80330a8SNeel Natu if ((error = vcpu_set_state(vm, vcpuid, newstate, false)) != 0) 1052318224bbSNeel Natu panic("Error %d setting state to %d\n", error, newstate); 1053318224bbSNeel Natu } 1054318224bbSNeel Natu 1055318224bbSNeel Natu static void 1056318224bbSNeel Natu vcpu_require_state_locked(struct vcpu *vcpu, enum vcpu_state newstate) 1057318224bbSNeel Natu { 1058318224bbSNeel Natu int error; 1059318224bbSNeel Natu 1060f80330a8SNeel Natu if ((error = vcpu_set_state_locked(vcpu, newstate, false)) != 0) 1061318224bbSNeel Natu panic("Error %d setting state to %d", error, newstate); 1062318224bbSNeel Natu } 1063318224bbSNeel Natu 10645b8a8cd1SNeel Natu static void 10655b8a8cd1SNeel Natu vm_set_rendezvous_func(struct vm *vm, vm_rendezvous_func_t func) 10665b8a8cd1SNeel Natu { 10675b8a8cd1SNeel Natu 10685b8a8cd1SNeel Natu KASSERT(mtx_owned(&vm->rendezvous_mtx), ("rendezvous_mtx not locked")); 10695b8a8cd1SNeel Natu 10705b8a8cd1SNeel Natu /* 10715b8a8cd1SNeel Natu * Update 'rendezvous_func' and execute a write memory barrier to 10725b8a8cd1SNeel Natu * ensure that it is visible across all host cpus. This is not needed 10735b8a8cd1SNeel Natu * for correctness but it does ensure that all the vcpus will notice 10745b8a8cd1SNeel Natu * that the rendezvous is requested immediately. 10755b8a8cd1SNeel Natu */ 10765b8a8cd1SNeel Natu vm->rendezvous_func = func; 10775b8a8cd1SNeel Natu wmb(); 10785b8a8cd1SNeel Natu } 10795b8a8cd1SNeel Natu 10805b8a8cd1SNeel Natu #define RENDEZVOUS_CTR0(vm, vcpuid, fmt) \ 10815b8a8cd1SNeel Natu do { \ 10825b8a8cd1SNeel Natu if (vcpuid >= 0) \ 10835b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, fmt); \ 10845b8a8cd1SNeel Natu else \ 10855b8a8cd1SNeel Natu VM_CTR0(vm, fmt); \ 10865b8a8cd1SNeel Natu } while (0) 10875b8a8cd1SNeel Natu 10885b8a8cd1SNeel Natu static void 10895b8a8cd1SNeel Natu vm_handle_rendezvous(struct vm *vm, int vcpuid) 10905b8a8cd1SNeel Natu { 10915b8a8cd1SNeel Natu 10925b8a8cd1SNeel Natu KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU), 10935b8a8cd1SNeel Natu ("vm_handle_rendezvous: invalid vcpuid %d", vcpuid)); 10945b8a8cd1SNeel Natu 10955b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 10965b8a8cd1SNeel Natu while (vm->rendezvous_func != NULL) { 109722d822c6SNeel Natu /* 'rendezvous_req_cpus' must be a subset of 'active_cpus' */ 109822d822c6SNeel Natu CPU_AND(&vm->rendezvous_req_cpus, &vm->active_cpus); 109922d822c6SNeel Natu 11005b8a8cd1SNeel Natu if (vcpuid != -1 && 110122d822c6SNeel Natu CPU_ISSET(vcpuid, &vm->rendezvous_req_cpus) && 110222d822c6SNeel Natu !CPU_ISSET(vcpuid, &vm->rendezvous_done_cpus)) { 11035b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Calling rendezvous func"); 11045b8a8cd1SNeel Natu (*vm->rendezvous_func)(vm, vcpuid, vm->rendezvous_arg); 11055b8a8cd1SNeel Natu CPU_SET(vcpuid, &vm->rendezvous_done_cpus); 11065b8a8cd1SNeel Natu } 11075b8a8cd1SNeel Natu if (CPU_CMP(&vm->rendezvous_req_cpus, 11085b8a8cd1SNeel Natu &vm->rendezvous_done_cpus) == 0) { 11095b8a8cd1SNeel Natu VCPU_CTR0(vm, vcpuid, "Rendezvous completed"); 11105b8a8cd1SNeel Natu vm_set_rendezvous_func(vm, NULL); 11115b8a8cd1SNeel Natu wakeup(&vm->rendezvous_func); 11125b8a8cd1SNeel Natu break; 11135b8a8cd1SNeel Natu } 11145b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Wait for rendezvous completion"); 11155b8a8cd1SNeel Natu mtx_sleep(&vm->rendezvous_func, &vm->rendezvous_mtx, 0, 11165b8a8cd1SNeel Natu "vmrndv", 0); 11175b8a8cd1SNeel Natu } 11185b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 11195b8a8cd1SNeel Natu } 11205b8a8cd1SNeel Natu 1121318224bbSNeel Natu /* 1122318224bbSNeel Natu * Emulate a guest 'hlt' by sleeping until the vcpu is ready to run. 1123318224bbSNeel Natu */ 1124318224bbSNeel Natu static int 1125becd9849SNeel Natu vm_handle_hlt(struct vm *vm, int vcpuid, bool intr_disabled, bool *retu) 1126318224bbSNeel Natu { 1127318224bbSNeel Natu struct vcpu *vcpu; 1128c6a0cc2eSNeel Natu const char *wmesg; 11292ce12423SNeel Natu int t, vcpu_halted, vm_halted; 1130e50ce2aaSNeel Natu 1131e50ce2aaSNeel Natu KASSERT(!CPU_ISSET(vcpuid, &vm->halted_cpus), ("vcpu already halted")); 1132318224bbSNeel Natu 1133318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1134e50ce2aaSNeel Natu vcpu_halted = 0; 1135e50ce2aaSNeel Natu vm_halted = 0; 1136318224bbSNeel Natu 1137f76fc5d4SNeel Natu vcpu_lock(vcpu); 1138c6a0cc2eSNeel Natu while (1) { 1139f76fc5d4SNeel Natu /* 1140f76fc5d4SNeel Natu * Do a final check for pending NMI or interrupts before 1141c6a0cc2eSNeel Natu * really putting this thread to sleep. Also check for 1142c6a0cc2eSNeel Natu * software events that would cause this vcpu to wakeup. 1143f76fc5d4SNeel Natu * 1144c6a0cc2eSNeel Natu * These interrupts/events could have happened after the 1145c6a0cc2eSNeel Natu * vcpu returned from VMRUN() and before it acquired the 1146c6a0cc2eSNeel Natu * vcpu lock above. 1147f76fc5d4SNeel Natu */ 1148c6a0cc2eSNeel Natu if (vm->rendezvous_func != NULL || vm->suspend) 1149c6a0cc2eSNeel Natu break; 1150c6a0cc2eSNeel Natu if (vm_nmi_pending(vm, vcpuid)) 1151c6a0cc2eSNeel Natu break; 1152c6a0cc2eSNeel Natu if (!intr_disabled) { 1153c6a0cc2eSNeel Natu if (vm_extint_pending(vm, vcpuid) || 1154c6a0cc2eSNeel Natu vlapic_pending_intr(vcpu->vlapic, NULL)) { 1155c6a0cc2eSNeel Natu break; 1156c6a0cc2eSNeel Natu } 1157c6a0cc2eSNeel Natu } 1158c6a0cc2eSNeel Natu 1159f008d157SNeel Natu /* Don't go to sleep if the vcpu thread needs to yield */ 1160f008d157SNeel Natu if (vcpu_should_yield(vm, vcpuid)) 1161f008d157SNeel Natu break; 1162f008d157SNeel Natu 1163e50ce2aaSNeel Natu /* 1164e50ce2aaSNeel Natu * Some Linux guests implement "halt" by having all vcpus 1165e50ce2aaSNeel Natu * execute HLT with interrupts disabled. 'halted_cpus' keeps 1166e50ce2aaSNeel Natu * track of the vcpus that have entered this state. When all 1167e50ce2aaSNeel Natu * vcpus enter the halted state the virtual machine is halted. 1168e50ce2aaSNeel Natu */ 1169e50ce2aaSNeel Natu if (intr_disabled) { 1170c6a0cc2eSNeel Natu wmesg = "vmhalt"; 1171e50ce2aaSNeel Natu VCPU_CTR0(vm, vcpuid, "Halted"); 1172055fc2cbSNeel Natu if (!vcpu_halted && halt_detection_enabled) { 1173e50ce2aaSNeel Natu vcpu_halted = 1; 1174e50ce2aaSNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->halted_cpus); 1175e50ce2aaSNeel Natu } 1176e50ce2aaSNeel Natu if (CPU_CMP(&vm->halted_cpus, &vm->active_cpus) == 0) { 1177e50ce2aaSNeel Natu vm_halted = 1; 1178e50ce2aaSNeel Natu break; 1179e50ce2aaSNeel Natu } 1180e50ce2aaSNeel Natu } else { 1181e50ce2aaSNeel Natu wmesg = "vmidle"; 1182e50ce2aaSNeel Natu } 1183c6a0cc2eSNeel Natu 1184f76fc5d4SNeel Natu t = ticks; 1185318224bbSNeel Natu vcpu_require_state_locked(vcpu, VCPU_SLEEPING); 1186f008d157SNeel Natu /* 1187f008d157SNeel Natu * XXX msleep_spin() cannot be interrupted by signals so 1188f008d157SNeel Natu * wake up periodically to check pending signals. 1189f008d157SNeel Natu */ 1190f008d157SNeel Natu msleep_spin(vcpu, &vcpu->mtx, wmesg, hz); 1191318224bbSNeel Natu vcpu_require_state_locked(vcpu, VCPU_FROZEN); 1192f76fc5d4SNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_IDLE_TICKS, ticks - t); 1193f76fc5d4SNeel Natu } 1194e50ce2aaSNeel Natu 1195e50ce2aaSNeel Natu if (vcpu_halted) 1196e50ce2aaSNeel Natu CPU_CLR_ATOMIC(vcpuid, &vm->halted_cpus); 1197e50ce2aaSNeel Natu 1198f76fc5d4SNeel Natu vcpu_unlock(vcpu); 1199f76fc5d4SNeel Natu 1200e50ce2aaSNeel Natu if (vm_halted) 1201e50ce2aaSNeel Natu vm_suspend(vm, VM_SUSPEND_HALT); 1202e50ce2aaSNeel Natu 1203318224bbSNeel Natu return (0); 1204318224bbSNeel Natu } 1205318224bbSNeel Natu 1206318224bbSNeel Natu static int 1207becd9849SNeel Natu vm_handle_paging(struct vm *vm, int vcpuid, bool *retu) 1208318224bbSNeel Natu { 1209318224bbSNeel Natu int rv, ftype; 1210318224bbSNeel Natu struct vm_map *map; 1211318224bbSNeel Natu struct vcpu *vcpu; 1212318224bbSNeel Natu struct vm_exit *vme; 1213318224bbSNeel Natu 1214318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1215318224bbSNeel Natu vme = &vcpu->exitinfo; 1216318224bbSNeel Natu 1217d087a399SNeel Natu KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 1218d087a399SNeel Natu __func__, vme->inst_length)); 1219d087a399SNeel Natu 1220318224bbSNeel Natu ftype = vme->u.paging.fault_type; 1221318224bbSNeel Natu KASSERT(ftype == VM_PROT_READ || 1222318224bbSNeel Natu ftype == VM_PROT_WRITE || ftype == VM_PROT_EXECUTE, 1223318224bbSNeel Natu ("vm_handle_paging: invalid fault_type %d", ftype)); 1224318224bbSNeel Natu 1225318224bbSNeel Natu if (ftype == VM_PROT_READ || ftype == VM_PROT_WRITE) { 1226318224bbSNeel Natu rv = pmap_emulate_accessed_dirty(vmspace_pmap(vm->vmspace), 1227318224bbSNeel Natu vme->u.paging.gpa, ftype); 12289d8d8e3eSNeel Natu if (rv == 0) { 12299d8d8e3eSNeel Natu VCPU_CTR2(vm, vcpuid, "%s bit emulation for gpa %#lx", 12309d8d8e3eSNeel Natu ftype == VM_PROT_READ ? "accessed" : "dirty", 12319d8d8e3eSNeel Natu vme->u.paging.gpa); 1232318224bbSNeel Natu goto done; 1233318224bbSNeel Natu } 12349d8d8e3eSNeel Natu } 1235318224bbSNeel Natu 1236318224bbSNeel Natu map = &vm->vmspace->vm_map; 1237318224bbSNeel Natu rv = vm_fault(map, vme->u.paging.gpa, ftype, VM_FAULT_NORMAL); 1238318224bbSNeel Natu 1239513c8d33SNeel Natu VCPU_CTR3(vm, vcpuid, "vm_handle_paging rv = %d, gpa = %#lx, " 1240513c8d33SNeel Natu "ftype = %d", rv, vme->u.paging.gpa, ftype); 1241318224bbSNeel Natu 1242318224bbSNeel Natu if (rv != KERN_SUCCESS) 1243318224bbSNeel Natu return (EFAULT); 1244318224bbSNeel Natu done: 1245318224bbSNeel Natu return (0); 1246318224bbSNeel Natu } 1247318224bbSNeel Natu 1248318224bbSNeel Natu static int 1249becd9849SNeel Natu vm_handle_inst_emul(struct vm *vm, int vcpuid, bool *retu) 1250318224bbSNeel Natu { 1251318224bbSNeel Natu struct vie *vie; 1252318224bbSNeel Natu struct vcpu *vcpu; 1253318224bbSNeel Natu struct vm_exit *vme; 1254e4f605eeSTycho Nightingale uint64_t gla, gpa, cs_base; 1255e813a873SNeel Natu struct vm_guest_paging *paging; 1256565bbb86SNeel Natu mem_region_read_t mread; 1257565bbb86SNeel Natu mem_region_write_t mwrite; 1258f7a9f178SNeel Natu enum vm_cpu_mode cpu_mode; 12591c73ea3eSNeel Natu int cs_d, error, fault; 1260318224bbSNeel Natu 1261318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1262318224bbSNeel Natu vme = &vcpu->exitinfo; 1263318224bbSNeel Natu 12641c73ea3eSNeel Natu KASSERT(vme->inst_length == 0, ("%s: invalid inst_length %d", 12651c73ea3eSNeel Natu __func__, vme->inst_length)); 12661c73ea3eSNeel Natu 1267318224bbSNeel Natu gla = vme->u.inst_emul.gla; 1268318224bbSNeel Natu gpa = vme->u.inst_emul.gpa; 1269e4f605eeSTycho Nightingale cs_base = vme->u.inst_emul.cs_base; 1270f7a9f178SNeel Natu cs_d = vme->u.inst_emul.cs_d; 1271318224bbSNeel Natu vie = &vme->u.inst_emul.vie; 1272e813a873SNeel Natu paging = &vme->u.inst_emul.paging; 1273f7a9f178SNeel Natu cpu_mode = paging->cpu_mode; 1274318224bbSNeel Natu 12759d8d8e3eSNeel Natu VCPU_CTR1(vm, vcpuid, "inst_emul fault accessing gpa %#lx", gpa); 12769d8d8e3eSNeel Natu 1277318224bbSNeel Natu /* Fetch, decode and emulate the faulting instruction */ 1278c2a875f9SNeel Natu if (vie->num_valid == 0) { 1279e4f605eeSTycho Nightingale error = vmm_fetch_instruction(vm, vcpuid, paging, vme->rip + 12801c73ea3eSNeel Natu cs_base, VIE_INST_SIZE, vie, &fault); 1281c2a875f9SNeel Natu } else { 1282c2a875f9SNeel Natu /* 1283c2a875f9SNeel Natu * The instruction bytes have already been copied into 'vie' 1284c2a875f9SNeel Natu */ 12859c4d5478SNeel Natu error = fault = 0; 1286c2a875f9SNeel Natu } 12879c4d5478SNeel Natu if (error || fault) 12889c4d5478SNeel Natu return (error); 1289318224bbSNeel Natu 1290c07a0648SNeel Natu if (vmm_decode_instruction(vm, vcpuid, gla, cpu_mode, cs_d, vie) != 0) { 1291c07a0648SNeel Natu VCPU_CTR1(vm, vcpuid, "Error decoding instruction at %#lx", 1292c07a0648SNeel Natu vme->rip + cs_base); 1293c07a0648SNeel Natu *retu = true; /* dump instruction bytes in userspace */ 1294c07a0648SNeel Natu return (0); 1295c07a0648SNeel Natu } 1296318224bbSNeel Natu 1297a0b78f09SPeter Grehan /* 12981c73ea3eSNeel Natu * Update 'nextrip' based on the length of the emulated instruction. 1299a0b78f09SPeter Grehan */ 1300a0b78f09SPeter Grehan vme->inst_length = vie->num_processed; 1301d087a399SNeel Natu vcpu->nextrip += vie->num_processed; 13021c73ea3eSNeel Natu VCPU_CTR1(vm, vcpuid, "nextrip updated to %#lx after instruction " 13031c73ea3eSNeel Natu "decoding", vcpu->nextrip); 1304a0b78f09SPeter Grehan 130508e3ff32SNeel Natu /* return to userland unless this is an in-kernel emulated device */ 1306565bbb86SNeel Natu if (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE) { 1307565bbb86SNeel Natu mread = lapic_mmio_read; 1308565bbb86SNeel Natu mwrite = lapic_mmio_write; 1309565bbb86SNeel Natu } else if (gpa >= VIOAPIC_BASE && gpa < VIOAPIC_BASE + VIOAPIC_SIZE) { 1310565bbb86SNeel Natu mread = vioapic_mmio_read; 1311565bbb86SNeel Natu mwrite = vioapic_mmio_write; 131208e3ff32SNeel Natu } else if (gpa >= VHPET_BASE && gpa < VHPET_BASE + VHPET_SIZE) { 131308e3ff32SNeel Natu mread = vhpet_mmio_read; 131408e3ff32SNeel Natu mwrite = vhpet_mmio_write; 1315565bbb86SNeel Natu } else { 1316becd9849SNeel Natu *retu = true; 1317318224bbSNeel Natu return (0); 1318318224bbSNeel Natu } 1319318224bbSNeel Natu 1320d665d229SNeel Natu error = vmm_emulate_instruction(vm, vcpuid, gpa, vie, paging, 1321d665d229SNeel Natu mread, mwrite, retu); 1322318224bbSNeel Natu 1323318224bbSNeel Natu return (error); 1324318224bbSNeel Natu } 1325318224bbSNeel Natu 1326b15a09c0SNeel Natu static int 1327b15a09c0SNeel Natu vm_handle_suspend(struct vm *vm, int vcpuid, bool *retu) 1328b15a09c0SNeel Natu { 1329b15a09c0SNeel Natu int i, done; 1330b15a09c0SNeel Natu struct vcpu *vcpu; 1331b15a09c0SNeel Natu 1332b15a09c0SNeel Natu done = 0; 1333b15a09c0SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1334b15a09c0SNeel Natu 1335b15a09c0SNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->suspended_cpus); 1336b15a09c0SNeel Natu 1337b15a09c0SNeel Natu /* 1338b15a09c0SNeel Natu * Wait until all 'active_cpus' have suspended themselves. 1339b15a09c0SNeel Natu * 1340b15a09c0SNeel Natu * Since a VM may be suspended at any time including when one or 1341b15a09c0SNeel Natu * more vcpus are doing a rendezvous we need to call the rendezvous 1342b15a09c0SNeel Natu * handler while we are waiting to prevent a deadlock. 1343b15a09c0SNeel Natu */ 1344b15a09c0SNeel Natu vcpu_lock(vcpu); 1345b15a09c0SNeel Natu while (1) { 1346b15a09c0SNeel Natu if (CPU_CMP(&vm->suspended_cpus, &vm->active_cpus) == 0) { 1347b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "All vcpus suspended"); 1348b15a09c0SNeel Natu break; 1349b15a09c0SNeel Natu } 1350b15a09c0SNeel Natu 1351b15a09c0SNeel Natu if (vm->rendezvous_func == NULL) { 1352b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "Sleeping during suspend"); 1353b15a09c0SNeel Natu vcpu_require_state_locked(vcpu, VCPU_SLEEPING); 1354b15a09c0SNeel Natu msleep_spin(vcpu, &vcpu->mtx, "vmsusp", hz); 1355b15a09c0SNeel Natu vcpu_require_state_locked(vcpu, VCPU_FROZEN); 1356b15a09c0SNeel Natu } else { 1357b15a09c0SNeel Natu VCPU_CTR0(vm, vcpuid, "Rendezvous during suspend"); 1358b15a09c0SNeel Natu vcpu_unlock(vcpu); 1359b15a09c0SNeel Natu vm_handle_rendezvous(vm, vcpuid); 1360b15a09c0SNeel Natu vcpu_lock(vcpu); 1361b15a09c0SNeel Natu } 1362b15a09c0SNeel Natu } 1363b15a09c0SNeel Natu vcpu_unlock(vcpu); 1364b15a09c0SNeel Natu 1365b15a09c0SNeel Natu /* 1366b15a09c0SNeel Natu * Wakeup the other sleeping vcpus and return to userspace. 1367b15a09c0SNeel Natu */ 1368b15a09c0SNeel Natu for (i = 0; i < VM_MAXCPU; i++) { 1369b15a09c0SNeel Natu if (CPU_ISSET(i, &vm->suspended_cpus)) { 1370b15a09c0SNeel Natu vcpu_notify_event(vm, i, false); 1371b15a09c0SNeel Natu } 1372b15a09c0SNeel Natu } 1373b15a09c0SNeel Natu 1374b15a09c0SNeel Natu *retu = true; 1375b15a09c0SNeel Natu return (0); 1376b15a09c0SNeel Natu } 1377b15a09c0SNeel Natu 1378b15a09c0SNeel Natu int 1379f0fdcfe2SNeel Natu vm_suspend(struct vm *vm, enum vm_suspend_how how) 1380b15a09c0SNeel Natu { 1381f0fdcfe2SNeel Natu int i; 1382b15a09c0SNeel Natu 1383f0fdcfe2SNeel Natu if (how <= VM_SUSPEND_NONE || how >= VM_SUSPEND_LAST) 1384f0fdcfe2SNeel Natu return (EINVAL); 1385f0fdcfe2SNeel Natu 1386f0fdcfe2SNeel Natu if (atomic_cmpset_int(&vm->suspend, 0, how) == 0) { 1387f0fdcfe2SNeel Natu VM_CTR2(vm, "virtual machine already suspended %d/%d", 1388f0fdcfe2SNeel Natu vm->suspend, how); 1389b15a09c0SNeel Natu return (EALREADY); 1390b15a09c0SNeel Natu } 1391f0fdcfe2SNeel Natu 1392f0fdcfe2SNeel Natu VM_CTR1(vm, "virtual machine successfully suspended %d", how); 1393f0fdcfe2SNeel Natu 1394f0fdcfe2SNeel Natu /* 1395f0fdcfe2SNeel Natu * Notify all active vcpus that they are now suspended. 1396f0fdcfe2SNeel Natu */ 1397f0fdcfe2SNeel Natu for (i = 0; i < VM_MAXCPU; i++) { 1398f0fdcfe2SNeel Natu if (CPU_ISSET(i, &vm->active_cpus)) 1399f0fdcfe2SNeel Natu vcpu_notify_event(vm, i, false); 1400f0fdcfe2SNeel Natu } 1401f0fdcfe2SNeel Natu 1402f0fdcfe2SNeel Natu return (0); 1403f0fdcfe2SNeel Natu } 1404f0fdcfe2SNeel Natu 1405f0fdcfe2SNeel Natu void 1406f0fdcfe2SNeel Natu vm_exit_suspended(struct vm *vm, int vcpuid, uint64_t rip) 1407f0fdcfe2SNeel Natu { 1408f0fdcfe2SNeel Natu struct vm_exit *vmexit; 1409f0fdcfe2SNeel Natu 1410f0fdcfe2SNeel Natu KASSERT(vm->suspend > VM_SUSPEND_NONE && vm->suspend < VM_SUSPEND_LAST, 1411f0fdcfe2SNeel Natu ("vm_exit_suspended: invalid suspend type %d", vm->suspend)); 1412f0fdcfe2SNeel Natu 1413f0fdcfe2SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 1414f0fdcfe2SNeel Natu vmexit->rip = rip; 1415f0fdcfe2SNeel Natu vmexit->inst_length = 0; 1416f0fdcfe2SNeel Natu vmexit->exitcode = VM_EXITCODE_SUSPENDED; 1417f0fdcfe2SNeel Natu vmexit->u.suspended.how = vm->suspend; 1418b15a09c0SNeel Natu } 1419b15a09c0SNeel Natu 142040487465SNeel Natu void 142140487465SNeel Natu vm_exit_rendezvous(struct vm *vm, int vcpuid, uint64_t rip) 142240487465SNeel Natu { 142340487465SNeel Natu struct vm_exit *vmexit; 142440487465SNeel Natu 142540487465SNeel Natu KASSERT(vm->rendezvous_func != NULL, ("rendezvous not in progress")); 142640487465SNeel Natu 142740487465SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 142840487465SNeel Natu vmexit->rip = rip; 142940487465SNeel Natu vmexit->inst_length = 0; 143040487465SNeel Natu vmexit->exitcode = VM_EXITCODE_RENDEZVOUS; 143140487465SNeel Natu vmm_stat_incr(vm, vcpuid, VMEXIT_RENDEZVOUS, 1); 143240487465SNeel Natu } 143340487465SNeel Natu 143440487465SNeel Natu void 143540487465SNeel Natu vm_exit_astpending(struct vm *vm, int vcpuid, uint64_t rip) 143640487465SNeel Natu { 143740487465SNeel Natu struct vm_exit *vmexit; 143840487465SNeel Natu 143940487465SNeel Natu vmexit = vm_exitinfo(vm, vcpuid); 144040487465SNeel Natu vmexit->rip = rip; 144140487465SNeel Natu vmexit->inst_length = 0; 144240487465SNeel Natu vmexit->exitcode = VM_EXITCODE_BOGUS; 144340487465SNeel Natu vmm_stat_incr(vm, vcpuid, VMEXIT_ASTPENDING, 1); 144440487465SNeel Natu } 144540487465SNeel Natu 1446318224bbSNeel Natu int 1447318224bbSNeel Natu vm_run(struct vm *vm, struct vm_run *vmrun) 1448318224bbSNeel Natu { 1449318224bbSNeel Natu int error, vcpuid; 1450318224bbSNeel Natu struct vcpu *vcpu; 1451318224bbSNeel Natu struct pcb *pcb; 1452d087a399SNeel Natu uint64_t tscval; 1453318224bbSNeel Natu struct vm_exit *vme; 1454becd9849SNeel Natu bool retu, intr_disabled; 1455318224bbSNeel Natu pmap_t pmap; 1456b15a09c0SNeel Natu void *rptr, *sptr; 1457318224bbSNeel Natu 1458318224bbSNeel Natu vcpuid = vmrun->cpuid; 1459318224bbSNeel Natu 1460318224bbSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1461318224bbSNeel Natu return (EINVAL); 1462318224bbSNeel Natu 146395ebc360SNeel Natu if (!CPU_ISSET(vcpuid, &vm->active_cpus)) 146495ebc360SNeel Natu return (EINVAL); 146595ebc360SNeel Natu 146695ebc360SNeel Natu if (CPU_ISSET(vcpuid, &vm->suspended_cpus)) 146795ebc360SNeel Natu return (EINVAL); 146895ebc360SNeel Natu 1469b15a09c0SNeel Natu rptr = &vm->rendezvous_func; 1470b15a09c0SNeel Natu sptr = &vm->suspend; 1471318224bbSNeel Natu pmap = vmspace_pmap(vm->vmspace); 1472318224bbSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1473318224bbSNeel Natu vme = &vcpu->exitinfo; 1474318224bbSNeel Natu restart: 1475318224bbSNeel Natu critical_enter(); 1476318224bbSNeel Natu 1477318224bbSNeel Natu KASSERT(!CPU_ISSET(curcpu, &pmap->pm_active), 1478318224bbSNeel Natu ("vm_run: absurd pm_active")); 1479318224bbSNeel Natu 1480318224bbSNeel Natu tscval = rdtsc(); 1481318224bbSNeel Natu 1482318224bbSNeel Natu pcb = PCPU_GET(curpcb); 1483318224bbSNeel Natu set_pcb_flags(pcb, PCB_FULL_IRET); 1484318224bbSNeel Natu 1485318224bbSNeel Natu restore_guest_fpustate(vcpu); 1486318224bbSNeel Natu 1487318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_RUNNING); 1488d087a399SNeel Natu error = VMRUN(vm->cookie, vcpuid, vcpu->nextrip, pmap, rptr, sptr); 1489318224bbSNeel Natu vcpu_require_state(vm, vcpuid, VCPU_FROZEN); 1490318224bbSNeel Natu 1491318224bbSNeel Natu save_guest_fpustate(vcpu); 1492318224bbSNeel Natu 1493318224bbSNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_TOTAL_RUNTIME, rdtsc() - tscval); 1494318224bbSNeel Natu 1495318224bbSNeel Natu critical_exit(); 1496318224bbSNeel Natu 1497318224bbSNeel Natu if (error == 0) { 1498becd9849SNeel Natu retu = false; 1499d087a399SNeel Natu vcpu->nextrip = vme->rip + vme->inst_length; 1500318224bbSNeel Natu switch (vme->exitcode) { 1501b15a09c0SNeel Natu case VM_EXITCODE_SUSPENDED: 1502b15a09c0SNeel Natu error = vm_handle_suspend(vm, vcpuid, &retu); 1503b15a09c0SNeel Natu break; 150430b94db8SNeel Natu case VM_EXITCODE_IOAPIC_EOI: 150530b94db8SNeel Natu vioapic_process_eoi(vm, vcpuid, 150630b94db8SNeel Natu vme->u.ioapic_eoi.vector); 150730b94db8SNeel Natu break; 15085b8a8cd1SNeel Natu case VM_EXITCODE_RENDEZVOUS: 15095b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 15105b8a8cd1SNeel Natu error = 0; 15115b8a8cd1SNeel Natu break; 1512318224bbSNeel Natu case VM_EXITCODE_HLT: 1513becd9849SNeel Natu intr_disabled = ((vme->u.hlt.rflags & PSL_I) == 0); 15141c052192SNeel Natu error = vm_handle_hlt(vm, vcpuid, intr_disabled, &retu); 1515318224bbSNeel Natu break; 1516318224bbSNeel Natu case VM_EXITCODE_PAGING: 1517318224bbSNeel Natu error = vm_handle_paging(vm, vcpuid, &retu); 1518318224bbSNeel Natu break; 1519318224bbSNeel Natu case VM_EXITCODE_INST_EMUL: 1520318224bbSNeel Natu error = vm_handle_inst_emul(vm, vcpuid, &retu); 1521318224bbSNeel Natu break; 1522d17b5104SNeel Natu case VM_EXITCODE_INOUT: 1523d17b5104SNeel Natu case VM_EXITCODE_INOUT_STR: 1524d17b5104SNeel Natu error = vm_handle_inout(vm, vcpuid, vme, &retu); 1525d17b5104SNeel Natu break; 152665145c7fSNeel Natu case VM_EXITCODE_MONITOR: 152765145c7fSNeel Natu case VM_EXITCODE_MWAIT: 152865145c7fSNeel Natu vm_inject_ud(vm, vcpuid); 152965145c7fSNeel Natu break; 1530318224bbSNeel Natu default: 1531becd9849SNeel Natu retu = true; /* handled in userland */ 1532318224bbSNeel Natu break; 1533318224bbSNeel Natu } 1534318224bbSNeel Natu } 1535318224bbSNeel Natu 1536d087a399SNeel Natu if (error == 0 && retu == false) 1537f76fc5d4SNeel Natu goto restart; 1538f76fc5d4SNeel Natu 1539318224bbSNeel Natu /* copy the exit information */ 1540318224bbSNeel Natu bcopy(vme, &vmrun->vm_exit, sizeof(struct vm_exit)); 1541366f6083SPeter Grehan return (error); 1542366f6083SPeter Grehan } 1543366f6083SPeter Grehan 1544366f6083SPeter Grehan int 1545c9c75df4SNeel Natu vm_restart_instruction(void *arg, int vcpuid) 1546c9c75df4SNeel Natu { 1547d087a399SNeel Natu struct vm *vm; 1548c9c75df4SNeel Natu struct vcpu *vcpu; 1549d087a399SNeel Natu enum vcpu_state state; 1550d087a399SNeel Natu uint64_t rip; 1551d087a399SNeel Natu int error; 1552c9c75df4SNeel Natu 1553d087a399SNeel Natu vm = arg; 1554c9c75df4SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1555c9c75df4SNeel Natu return (EINVAL); 1556c9c75df4SNeel Natu 1557c9c75df4SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1558d087a399SNeel Natu state = vcpu_get_state(vm, vcpuid, NULL); 1559d087a399SNeel Natu if (state == VCPU_RUNNING) { 1560d087a399SNeel Natu /* 1561d087a399SNeel Natu * When a vcpu is "running" the next instruction is determined 1562d087a399SNeel Natu * by adding 'rip' and 'inst_length' in the vcpu's 'exitinfo'. 1563d087a399SNeel Natu * Thus setting 'inst_length' to zero will cause the current 1564d087a399SNeel Natu * instruction to be restarted. 1565d087a399SNeel Natu */ 1566c9c75df4SNeel Natu vcpu->exitinfo.inst_length = 0; 1567d087a399SNeel Natu VCPU_CTR1(vm, vcpuid, "restarting instruction at %#lx by " 1568d087a399SNeel Natu "setting inst_length to zero", vcpu->exitinfo.rip); 1569d087a399SNeel Natu } else if (state == VCPU_FROZEN) { 1570d087a399SNeel Natu /* 1571d087a399SNeel Natu * When a vcpu is "frozen" it is outside the critical section 1572d087a399SNeel Natu * around VMRUN() and 'nextrip' points to the next instruction. 1573d087a399SNeel Natu * Thus instruction restart is achieved by setting 'nextrip' 1574d087a399SNeel Natu * to the vcpu's %rip. 1575d087a399SNeel Natu */ 1576d087a399SNeel Natu error = vm_get_register(vm, vcpuid, VM_REG_GUEST_RIP, &rip); 1577d087a399SNeel Natu KASSERT(!error, ("%s: error %d getting rip", __func__, error)); 1578d087a399SNeel Natu VCPU_CTR2(vm, vcpuid, "restarting instruction by updating " 1579d087a399SNeel Natu "nextrip from %#lx to %#lx", vcpu->nextrip, rip); 1580d087a399SNeel Natu vcpu->nextrip = rip; 1581d087a399SNeel Natu } else { 1582d087a399SNeel Natu panic("%s: invalid state %d", __func__, state); 1583d087a399SNeel Natu } 1584c9c75df4SNeel Natu return (0); 1585c9c75df4SNeel Natu } 1586c9c75df4SNeel Natu 1587c9c75df4SNeel Natu int 1588091d4532SNeel Natu vm_exit_intinfo(struct vm *vm, int vcpuid, uint64_t info) 1589091d4532SNeel Natu { 1590091d4532SNeel Natu struct vcpu *vcpu; 1591091d4532SNeel Natu int type, vector; 1592091d4532SNeel Natu 1593091d4532SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1594091d4532SNeel Natu return (EINVAL); 1595091d4532SNeel Natu 1596091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1597091d4532SNeel Natu 1598091d4532SNeel Natu if (info & VM_INTINFO_VALID) { 1599091d4532SNeel Natu type = info & VM_INTINFO_TYPE; 1600091d4532SNeel Natu vector = info & 0xff; 1601091d4532SNeel Natu if (type == VM_INTINFO_NMI && vector != IDT_NMI) 1602091d4532SNeel Natu return (EINVAL); 1603091d4532SNeel Natu if (type == VM_INTINFO_HWEXCEPTION && vector >= 32) 1604091d4532SNeel Natu return (EINVAL); 1605091d4532SNeel Natu if (info & VM_INTINFO_RSVD) 1606091d4532SNeel Natu return (EINVAL); 1607091d4532SNeel Natu } else { 1608091d4532SNeel Natu info = 0; 1609091d4532SNeel Natu } 1610091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "%s: info1(%#lx)", __func__, info); 1611091d4532SNeel Natu vcpu->exitintinfo = info; 1612091d4532SNeel Natu return (0); 1613091d4532SNeel Natu } 1614091d4532SNeel Natu 1615091d4532SNeel Natu enum exc_class { 1616091d4532SNeel Natu EXC_BENIGN, 1617091d4532SNeel Natu EXC_CONTRIBUTORY, 1618091d4532SNeel Natu EXC_PAGEFAULT 1619091d4532SNeel Natu }; 1620091d4532SNeel Natu 1621091d4532SNeel Natu #define IDT_VE 20 /* Virtualization Exception (Intel specific) */ 1622091d4532SNeel Natu 1623091d4532SNeel Natu static enum exc_class 1624091d4532SNeel Natu exception_class(uint64_t info) 1625091d4532SNeel Natu { 1626091d4532SNeel Natu int type, vector; 1627091d4532SNeel Natu 1628091d4532SNeel Natu KASSERT(info & VM_INTINFO_VALID, ("intinfo must be valid: %#lx", info)); 1629091d4532SNeel Natu type = info & VM_INTINFO_TYPE; 1630091d4532SNeel Natu vector = info & 0xff; 1631091d4532SNeel Natu 1632091d4532SNeel Natu /* Table 6-4, "Interrupt and Exception Classes", Intel SDM, Vol 3 */ 1633091d4532SNeel Natu switch (type) { 1634091d4532SNeel Natu case VM_INTINFO_HWINTR: 1635091d4532SNeel Natu case VM_INTINFO_SWINTR: 1636091d4532SNeel Natu case VM_INTINFO_NMI: 1637091d4532SNeel Natu return (EXC_BENIGN); 1638091d4532SNeel Natu default: 1639091d4532SNeel Natu /* 1640091d4532SNeel Natu * Hardware exception. 1641091d4532SNeel Natu * 1642091d4532SNeel Natu * SVM and VT-x use identical type values to represent NMI, 1643091d4532SNeel Natu * hardware interrupt and software interrupt. 1644091d4532SNeel Natu * 1645091d4532SNeel Natu * SVM uses type '3' for all exceptions. VT-x uses type '3' 1646091d4532SNeel Natu * for exceptions except #BP and #OF. #BP and #OF use a type 1647091d4532SNeel Natu * value of '5' or '6'. Therefore we don't check for explicit 1648091d4532SNeel Natu * values of 'type' to classify 'intinfo' into a hardware 1649091d4532SNeel Natu * exception. 1650091d4532SNeel Natu */ 1651091d4532SNeel Natu break; 1652091d4532SNeel Natu } 1653091d4532SNeel Natu 1654091d4532SNeel Natu switch (vector) { 1655091d4532SNeel Natu case IDT_PF: 1656091d4532SNeel Natu case IDT_VE: 1657091d4532SNeel Natu return (EXC_PAGEFAULT); 1658091d4532SNeel Natu case IDT_DE: 1659091d4532SNeel Natu case IDT_TS: 1660091d4532SNeel Natu case IDT_NP: 1661091d4532SNeel Natu case IDT_SS: 1662091d4532SNeel Natu case IDT_GP: 1663091d4532SNeel Natu return (EXC_CONTRIBUTORY); 1664091d4532SNeel Natu default: 1665091d4532SNeel Natu return (EXC_BENIGN); 1666091d4532SNeel Natu } 1667091d4532SNeel Natu } 1668091d4532SNeel Natu 1669091d4532SNeel Natu static int 1670091d4532SNeel Natu nested_fault(struct vm *vm, int vcpuid, uint64_t info1, uint64_t info2, 1671091d4532SNeel Natu uint64_t *retinfo) 1672091d4532SNeel Natu { 1673091d4532SNeel Natu enum exc_class exc1, exc2; 1674091d4532SNeel Natu int type1, vector1; 1675091d4532SNeel Natu 1676091d4532SNeel Natu KASSERT(info1 & VM_INTINFO_VALID, ("info1 %#lx is not valid", info1)); 1677091d4532SNeel Natu KASSERT(info2 & VM_INTINFO_VALID, ("info2 %#lx is not valid", info2)); 1678091d4532SNeel Natu 1679091d4532SNeel Natu /* 1680091d4532SNeel Natu * If an exception occurs while attempting to call the double-fault 1681091d4532SNeel Natu * handler the processor enters shutdown mode (aka triple fault). 1682091d4532SNeel Natu */ 1683091d4532SNeel Natu type1 = info1 & VM_INTINFO_TYPE; 1684091d4532SNeel Natu vector1 = info1 & 0xff; 1685091d4532SNeel Natu if (type1 == VM_INTINFO_HWEXCEPTION && vector1 == IDT_DF) { 1686091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "triple fault: info1(%#lx), info2(%#lx)", 1687091d4532SNeel Natu info1, info2); 1688091d4532SNeel Natu vm_suspend(vm, VM_SUSPEND_TRIPLEFAULT); 1689091d4532SNeel Natu *retinfo = 0; 1690091d4532SNeel Natu return (0); 1691091d4532SNeel Natu } 1692091d4532SNeel Natu 1693091d4532SNeel Natu /* 1694091d4532SNeel Natu * Table 6-5 "Conditions for Generating a Double Fault", Intel SDM, Vol3 1695091d4532SNeel Natu */ 1696091d4532SNeel Natu exc1 = exception_class(info1); 1697091d4532SNeel Natu exc2 = exception_class(info2); 1698091d4532SNeel Natu if ((exc1 == EXC_CONTRIBUTORY && exc2 == EXC_CONTRIBUTORY) || 1699091d4532SNeel Natu (exc1 == EXC_PAGEFAULT && exc2 != EXC_BENIGN)) { 1700091d4532SNeel Natu /* Convert nested fault into a double fault. */ 1701091d4532SNeel Natu *retinfo = IDT_DF; 1702091d4532SNeel Natu *retinfo |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 1703091d4532SNeel Natu *retinfo |= VM_INTINFO_DEL_ERRCODE; 1704091d4532SNeel Natu } else { 1705091d4532SNeel Natu /* Handle exceptions serially */ 1706091d4532SNeel Natu *retinfo = info2; 1707091d4532SNeel Natu } 1708091d4532SNeel Natu return (1); 1709091d4532SNeel Natu } 1710091d4532SNeel Natu 1711091d4532SNeel Natu static uint64_t 1712091d4532SNeel Natu vcpu_exception_intinfo(struct vcpu *vcpu) 1713091d4532SNeel Natu { 1714091d4532SNeel Natu uint64_t info = 0; 1715091d4532SNeel Natu 1716091d4532SNeel Natu if (vcpu->exception_pending) { 1717c9c75df4SNeel Natu info = vcpu->exc_vector & 0xff; 1718091d4532SNeel Natu info |= VM_INTINFO_VALID | VM_INTINFO_HWEXCEPTION; 1719c9c75df4SNeel Natu if (vcpu->exc_errcode_valid) { 1720091d4532SNeel Natu info |= VM_INTINFO_DEL_ERRCODE; 1721c9c75df4SNeel Natu info |= (uint64_t)vcpu->exc_errcode << 32; 1722091d4532SNeel Natu } 1723091d4532SNeel Natu } 1724091d4532SNeel Natu return (info); 1725091d4532SNeel Natu } 1726091d4532SNeel Natu 1727091d4532SNeel Natu int 1728091d4532SNeel Natu vm_entry_intinfo(struct vm *vm, int vcpuid, uint64_t *retinfo) 1729091d4532SNeel Natu { 1730091d4532SNeel Natu struct vcpu *vcpu; 1731091d4532SNeel Natu uint64_t info1, info2; 1732091d4532SNeel Natu int valid; 1733091d4532SNeel Natu 1734091d4532SNeel Natu KASSERT(vcpuid >= 0 && vcpuid < VM_MAXCPU, ("invalid vcpu %d", vcpuid)); 1735091d4532SNeel Natu 1736091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1737091d4532SNeel Natu 1738091d4532SNeel Natu info1 = vcpu->exitintinfo; 1739091d4532SNeel Natu vcpu->exitintinfo = 0; 1740091d4532SNeel Natu 1741091d4532SNeel Natu info2 = 0; 1742091d4532SNeel Natu if (vcpu->exception_pending) { 1743091d4532SNeel Natu info2 = vcpu_exception_intinfo(vcpu); 1744091d4532SNeel Natu vcpu->exception_pending = 0; 1745091d4532SNeel Natu VCPU_CTR2(vm, vcpuid, "Exception %d delivered: %#lx", 1746c9c75df4SNeel Natu vcpu->exc_vector, info2); 1747091d4532SNeel Natu } 1748091d4532SNeel Natu 1749091d4532SNeel Natu if ((info1 & VM_INTINFO_VALID) && (info2 & VM_INTINFO_VALID)) { 1750091d4532SNeel Natu valid = nested_fault(vm, vcpuid, info1, info2, retinfo); 1751091d4532SNeel Natu } else if (info1 & VM_INTINFO_VALID) { 1752091d4532SNeel Natu *retinfo = info1; 1753091d4532SNeel Natu valid = 1; 1754091d4532SNeel Natu } else if (info2 & VM_INTINFO_VALID) { 1755091d4532SNeel Natu *retinfo = info2; 1756091d4532SNeel Natu valid = 1; 1757091d4532SNeel Natu } else { 1758091d4532SNeel Natu valid = 0; 1759091d4532SNeel Natu } 1760091d4532SNeel Natu 1761091d4532SNeel Natu if (valid) { 1762091d4532SNeel Natu VCPU_CTR4(vm, vcpuid, "%s: info1(%#lx), info2(%#lx), " 1763091d4532SNeel Natu "retinfo(%#lx)", __func__, info1, info2, *retinfo); 1764091d4532SNeel Natu } 1765091d4532SNeel Natu 1766091d4532SNeel Natu return (valid); 1767091d4532SNeel Natu } 1768091d4532SNeel Natu 1769091d4532SNeel Natu int 1770091d4532SNeel Natu vm_get_intinfo(struct vm *vm, int vcpuid, uint64_t *info1, uint64_t *info2) 1771091d4532SNeel Natu { 1772091d4532SNeel Natu struct vcpu *vcpu; 1773091d4532SNeel Natu 1774091d4532SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1775091d4532SNeel Natu return (EINVAL); 1776091d4532SNeel Natu 1777091d4532SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1778091d4532SNeel Natu *info1 = vcpu->exitintinfo; 1779091d4532SNeel Natu *info2 = vcpu_exception_intinfo(vcpu); 1780091d4532SNeel Natu return (0); 1781091d4532SNeel Natu } 1782091d4532SNeel Natu 1783091d4532SNeel Natu int 1784c9c75df4SNeel Natu vm_inject_exception(struct vm *vm, int vcpuid, int vector, int errcode_valid, 1785c9c75df4SNeel Natu uint32_t errcode, int restart_instruction) 1786366f6083SPeter Grehan { 1787dc506506SNeel Natu struct vcpu *vcpu; 17882ce12423SNeel Natu int error; 1789dc506506SNeel Natu 1790366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1791366f6083SPeter Grehan return (EINVAL); 1792366f6083SPeter Grehan 1793c9c75df4SNeel Natu if (vector < 0 || vector >= 32) 1794366f6083SPeter Grehan return (EINVAL); 1795366f6083SPeter Grehan 1796091d4532SNeel Natu /* 1797091d4532SNeel Natu * A double fault exception should never be injected directly into 1798091d4532SNeel Natu * the guest. It is a derived exception that results from specific 1799091d4532SNeel Natu * combinations of nested faults. 1800091d4532SNeel Natu */ 1801c9c75df4SNeel Natu if (vector == IDT_DF) 1802091d4532SNeel Natu return (EINVAL); 1803091d4532SNeel Natu 1804dc506506SNeel Natu vcpu = &vm->vcpu[vcpuid]; 1805366f6083SPeter Grehan 1806dc506506SNeel Natu if (vcpu->exception_pending) { 1807dc506506SNeel Natu VCPU_CTR2(vm, vcpuid, "Unable to inject exception %d due to " 1808c9c75df4SNeel Natu "pending exception %d", vector, vcpu->exc_vector); 1809dc506506SNeel Natu return (EBUSY); 1810dc506506SNeel Natu } 1811dc506506SNeel Natu 18122ce12423SNeel Natu /* 18132ce12423SNeel Natu * From section 26.6.1 "Interruptibility State" in Intel SDM: 18142ce12423SNeel Natu * 18152ce12423SNeel Natu * Event blocking by "STI" or "MOV SS" is cleared after guest executes 18162ce12423SNeel Natu * one instruction or incurs an exception. 18172ce12423SNeel Natu */ 18182ce12423SNeel Natu error = vm_set_register(vm, vcpuid, VM_REG_GUEST_INTR_SHADOW, 0); 18192ce12423SNeel Natu KASSERT(error == 0, ("%s: error %d clearing interrupt shadow", 18202ce12423SNeel Natu __func__, error)); 18212ce12423SNeel Natu 1822c9c75df4SNeel Natu if (restart_instruction) 1823c9c75df4SNeel Natu vm_restart_instruction(vm, vcpuid); 1824c9c75df4SNeel Natu 1825dc506506SNeel Natu vcpu->exception_pending = 1; 1826c9c75df4SNeel Natu vcpu->exc_vector = vector; 1827c9c75df4SNeel Natu vcpu->exc_errcode = errcode; 1828c9c75df4SNeel Natu vcpu->exc_errcode_valid = errcode_valid; 1829c9c75df4SNeel Natu VCPU_CTR1(vm, vcpuid, "Exception %d pending", vector); 1830dc506506SNeel Natu return (0); 1831dc506506SNeel Natu } 1832dc506506SNeel Natu 1833d37f2adbSNeel Natu void 1834d37f2adbSNeel Natu vm_inject_fault(void *vmarg, int vcpuid, int vector, int errcode_valid, 1835d37f2adbSNeel Natu int errcode) 1836dc506506SNeel Natu { 1837d37f2adbSNeel Natu struct vm *vm; 1838c9c75df4SNeel Natu int error, restart_instruction; 1839dc506506SNeel Natu 1840d37f2adbSNeel Natu vm = vmarg; 1841c9c75df4SNeel Natu restart_instruction = 1; 1842d37f2adbSNeel Natu 1843c9c75df4SNeel Natu error = vm_inject_exception(vm, vcpuid, vector, errcode_valid, 1844c9c75df4SNeel Natu errcode, restart_instruction); 1845dc506506SNeel Natu KASSERT(error == 0, ("vm_inject_exception error %d", error)); 1846dc506506SNeel Natu } 1847dc506506SNeel Natu 1848dc506506SNeel Natu void 1849d37f2adbSNeel Natu vm_inject_pf(void *vmarg, int vcpuid, int error_code, uint64_t cr2) 1850fd949af6SNeel Natu { 1851d37f2adbSNeel Natu struct vm *vm; 185237a723a5SNeel Natu int error; 185337a723a5SNeel Natu 1854d37f2adbSNeel Natu vm = vmarg; 185537a723a5SNeel Natu VCPU_CTR2(vm, vcpuid, "Injecting page fault: error_code %#x, cr2 %#lx", 185637a723a5SNeel Natu error_code, cr2); 185737a723a5SNeel Natu 185837a723a5SNeel Natu error = vm_set_register(vm, vcpuid, VM_REG_GUEST_CR2, cr2); 185937a723a5SNeel Natu KASSERT(error == 0, ("vm_set_register(cr2) error %d", error)); 1860fd949af6SNeel Natu 1861d37f2adbSNeel Natu vm_inject_fault(vm, vcpuid, IDT_PF, 1, error_code); 1862366f6083SPeter Grehan } 1863366f6083SPeter Grehan 186461592433SNeel Natu static VMM_STAT(VCPU_NMI_COUNT, "number of NMIs delivered to vcpu"); 1865366f6083SPeter Grehan 1866f352ff0cSNeel Natu int 1867f352ff0cSNeel Natu vm_inject_nmi(struct vm *vm, int vcpuid) 1868f352ff0cSNeel Natu { 1869f352ff0cSNeel Natu struct vcpu *vcpu; 1870f352ff0cSNeel Natu 1871f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1872366f6083SPeter Grehan return (EINVAL); 1873366f6083SPeter Grehan 1874f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1875f352ff0cSNeel Natu 1876f352ff0cSNeel Natu vcpu->nmi_pending = 1; 1877de5ea6b6SNeel Natu vcpu_notify_event(vm, vcpuid, false); 1878f352ff0cSNeel Natu return (0); 1879f352ff0cSNeel Natu } 1880f352ff0cSNeel Natu 1881f352ff0cSNeel Natu int 1882f352ff0cSNeel Natu vm_nmi_pending(struct vm *vm, int vcpuid) 1883f352ff0cSNeel Natu { 1884f352ff0cSNeel Natu struct vcpu *vcpu; 1885f352ff0cSNeel Natu 1886f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1887f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1888f352ff0cSNeel Natu 1889f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1890f352ff0cSNeel Natu 1891f352ff0cSNeel Natu return (vcpu->nmi_pending); 1892f352ff0cSNeel Natu } 1893f352ff0cSNeel Natu 1894f352ff0cSNeel Natu void 1895f352ff0cSNeel Natu vm_nmi_clear(struct vm *vm, int vcpuid) 1896f352ff0cSNeel Natu { 1897f352ff0cSNeel Natu struct vcpu *vcpu; 1898f352ff0cSNeel Natu 1899f352ff0cSNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 1900f352ff0cSNeel Natu panic("vm_nmi_pending: invalid vcpuid %d", vcpuid); 1901f352ff0cSNeel Natu 1902f352ff0cSNeel Natu vcpu = &vm->vcpu[vcpuid]; 1903f352ff0cSNeel Natu 1904f352ff0cSNeel Natu if (vcpu->nmi_pending == 0) 1905f352ff0cSNeel Natu panic("vm_nmi_clear: inconsistent nmi_pending state"); 1906f352ff0cSNeel Natu 1907f352ff0cSNeel Natu vcpu->nmi_pending = 0; 1908f352ff0cSNeel Natu vmm_stat_incr(vm, vcpuid, VCPU_NMI_COUNT, 1); 1909366f6083SPeter Grehan } 1910366f6083SPeter Grehan 19110775fbb4STycho Nightingale static VMM_STAT(VCPU_EXTINT_COUNT, "number of ExtINTs delivered to vcpu"); 19120775fbb4STycho Nightingale 19130775fbb4STycho Nightingale int 19140775fbb4STycho Nightingale vm_inject_extint(struct vm *vm, int vcpuid) 19150775fbb4STycho Nightingale { 19160775fbb4STycho Nightingale struct vcpu *vcpu; 19170775fbb4STycho Nightingale 19180775fbb4STycho Nightingale if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 19190775fbb4STycho Nightingale return (EINVAL); 19200775fbb4STycho Nightingale 19210775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 19220775fbb4STycho Nightingale 19230775fbb4STycho Nightingale vcpu->extint_pending = 1; 19240775fbb4STycho Nightingale vcpu_notify_event(vm, vcpuid, false); 19250775fbb4STycho Nightingale return (0); 19260775fbb4STycho Nightingale } 19270775fbb4STycho Nightingale 19280775fbb4STycho Nightingale int 19290775fbb4STycho Nightingale vm_extint_pending(struct vm *vm, int vcpuid) 19300775fbb4STycho Nightingale { 19310775fbb4STycho Nightingale struct vcpu *vcpu; 19320775fbb4STycho Nightingale 19330775fbb4STycho Nightingale if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 19340775fbb4STycho Nightingale panic("vm_extint_pending: invalid vcpuid %d", vcpuid); 19350775fbb4STycho Nightingale 19360775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 19370775fbb4STycho Nightingale 19380775fbb4STycho Nightingale return (vcpu->extint_pending); 19390775fbb4STycho Nightingale } 19400775fbb4STycho Nightingale 19410775fbb4STycho Nightingale void 19420775fbb4STycho Nightingale vm_extint_clear(struct vm *vm, int vcpuid) 19430775fbb4STycho Nightingale { 19440775fbb4STycho Nightingale struct vcpu *vcpu; 19450775fbb4STycho Nightingale 19460775fbb4STycho Nightingale if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 19470775fbb4STycho Nightingale panic("vm_extint_pending: invalid vcpuid %d", vcpuid); 19480775fbb4STycho Nightingale 19490775fbb4STycho Nightingale vcpu = &vm->vcpu[vcpuid]; 19500775fbb4STycho Nightingale 19510775fbb4STycho Nightingale if (vcpu->extint_pending == 0) 19520775fbb4STycho Nightingale panic("vm_extint_clear: inconsistent extint_pending state"); 19530775fbb4STycho Nightingale 19540775fbb4STycho Nightingale vcpu->extint_pending = 0; 19550775fbb4STycho Nightingale vmm_stat_incr(vm, vcpuid, VCPU_EXTINT_COUNT, 1); 19560775fbb4STycho Nightingale } 19570775fbb4STycho Nightingale 1958366f6083SPeter Grehan int 1959366f6083SPeter Grehan vm_get_capability(struct vm *vm, int vcpu, int type, int *retval) 1960366f6083SPeter Grehan { 1961366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 1962366f6083SPeter Grehan return (EINVAL); 1963366f6083SPeter Grehan 1964366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 1965366f6083SPeter Grehan return (EINVAL); 1966366f6083SPeter Grehan 1967366f6083SPeter Grehan return (VMGETCAP(vm->cookie, vcpu, type, retval)); 1968366f6083SPeter Grehan } 1969366f6083SPeter Grehan 1970366f6083SPeter Grehan int 1971366f6083SPeter Grehan vm_set_capability(struct vm *vm, int vcpu, int type, int val) 1972366f6083SPeter Grehan { 1973366f6083SPeter Grehan if (vcpu < 0 || vcpu >= VM_MAXCPU) 1974366f6083SPeter Grehan return (EINVAL); 1975366f6083SPeter Grehan 1976366f6083SPeter Grehan if (type < 0 || type >= VM_CAP_MAX) 1977366f6083SPeter Grehan return (EINVAL); 1978366f6083SPeter Grehan 1979366f6083SPeter Grehan return (VMSETCAP(vm->cookie, vcpu, type, val)); 1980366f6083SPeter Grehan } 1981366f6083SPeter Grehan 1982366f6083SPeter Grehan struct vlapic * 1983366f6083SPeter Grehan vm_lapic(struct vm *vm, int cpu) 1984366f6083SPeter Grehan { 1985366f6083SPeter Grehan return (vm->vcpu[cpu].vlapic); 1986366f6083SPeter Grehan } 1987366f6083SPeter Grehan 1988565bbb86SNeel Natu struct vioapic * 1989565bbb86SNeel Natu vm_ioapic(struct vm *vm) 1990565bbb86SNeel Natu { 1991565bbb86SNeel Natu 1992565bbb86SNeel Natu return (vm->vioapic); 1993565bbb86SNeel Natu } 1994565bbb86SNeel Natu 199508e3ff32SNeel Natu struct vhpet * 199608e3ff32SNeel Natu vm_hpet(struct vm *vm) 199708e3ff32SNeel Natu { 199808e3ff32SNeel Natu 199908e3ff32SNeel Natu return (vm->vhpet); 200008e3ff32SNeel Natu } 200108e3ff32SNeel Natu 2002366f6083SPeter Grehan boolean_t 2003366f6083SPeter Grehan vmm_is_pptdev(int bus, int slot, int func) 2004366f6083SPeter Grehan { 200507044a96SNeel Natu int found, i, n; 200607044a96SNeel Natu int b, s, f; 2007366f6083SPeter Grehan char *val, *cp, *cp2; 2008366f6083SPeter Grehan 2009366f6083SPeter Grehan /* 201007044a96SNeel Natu * XXX 201107044a96SNeel Natu * The length of an environment variable is limited to 128 bytes which 201207044a96SNeel Natu * puts an upper limit on the number of passthru devices that may be 201307044a96SNeel Natu * specified using a single environment variable. 201407044a96SNeel Natu * 201507044a96SNeel Natu * Work around this by scanning multiple environment variable 201607044a96SNeel Natu * names instead of a single one - yuck! 2017366f6083SPeter Grehan */ 201807044a96SNeel Natu const char *names[] = { "pptdevs", "pptdevs2", "pptdevs3", NULL }; 201907044a96SNeel Natu 202007044a96SNeel Natu /* set pptdevs="1/2/3 4/5/6 7/8/9 10/11/12" */ 2021366f6083SPeter Grehan found = 0; 202207044a96SNeel Natu for (i = 0; names[i] != NULL && !found; i++) { 20232be111bfSDavide Italiano cp = val = kern_getenv(names[i]); 2024366f6083SPeter Grehan while (cp != NULL && *cp != '\0') { 2025366f6083SPeter Grehan if ((cp2 = strchr(cp, ' ')) != NULL) 2026366f6083SPeter Grehan *cp2 = '\0'; 2027366f6083SPeter Grehan 2028366f6083SPeter Grehan n = sscanf(cp, "%d/%d/%d", &b, &s, &f); 2029366f6083SPeter Grehan if (n == 3 && bus == b && slot == s && func == f) { 2030366f6083SPeter Grehan found = 1; 2031366f6083SPeter Grehan break; 2032366f6083SPeter Grehan } 2033366f6083SPeter Grehan 2034366f6083SPeter Grehan if (cp2 != NULL) 2035366f6083SPeter Grehan *cp2++ = ' '; 2036366f6083SPeter Grehan 2037366f6083SPeter Grehan cp = cp2; 2038366f6083SPeter Grehan } 2039366f6083SPeter Grehan freeenv(val); 204007044a96SNeel Natu } 2041366f6083SPeter Grehan return (found); 2042366f6083SPeter Grehan } 2043366f6083SPeter Grehan 2044366f6083SPeter Grehan void * 2045366f6083SPeter Grehan vm_iommu_domain(struct vm *vm) 2046366f6083SPeter Grehan { 2047366f6083SPeter Grehan 2048366f6083SPeter Grehan return (vm->iommu); 2049366f6083SPeter Grehan } 2050366f6083SPeter Grehan 205175dd3366SNeel Natu int 2052f80330a8SNeel Natu vcpu_set_state(struct vm *vm, int vcpuid, enum vcpu_state newstate, 2053f80330a8SNeel Natu bool from_idle) 2054366f6083SPeter Grehan { 205575dd3366SNeel Natu int error; 2056366f6083SPeter Grehan struct vcpu *vcpu; 2057366f6083SPeter Grehan 2058366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2059366f6083SPeter Grehan panic("vm_set_run_state: invalid vcpuid %d", vcpuid); 2060366f6083SPeter Grehan 2061366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 2062366f6083SPeter Grehan 206375dd3366SNeel Natu vcpu_lock(vcpu); 2064f80330a8SNeel Natu error = vcpu_set_state_locked(vcpu, newstate, from_idle); 206575dd3366SNeel Natu vcpu_unlock(vcpu); 206675dd3366SNeel Natu 206775dd3366SNeel Natu return (error); 206875dd3366SNeel Natu } 206975dd3366SNeel Natu 207075dd3366SNeel Natu enum vcpu_state 2071d3c11f40SPeter Grehan vcpu_get_state(struct vm *vm, int vcpuid, int *hostcpu) 2072366f6083SPeter Grehan { 2073366f6083SPeter Grehan struct vcpu *vcpu; 207475dd3366SNeel Natu enum vcpu_state state; 2075366f6083SPeter Grehan 2076366f6083SPeter Grehan if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2077366f6083SPeter Grehan panic("vm_get_run_state: invalid vcpuid %d", vcpuid); 2078366f6083SPeter Grehan 2079366f6083SPeter Grehan vcpu = &vm->vcpu[vcpuid]; 2080366f6083SPeter Grehan 208175dd3366SNeel Natu vcpu_lock(vcpu); 208275dd3366SNeel Natu state = vcpu->state; 2083d3c11f40SPeter Grehan if (hostcpu != NULL) 2084d3c11f40SPeter Grehan *hostcpu = vcpu->hostcpu; 208575dd3366SNeel Natu vcpu_unlock(vcpu); 2086366f6083SPeter Grehan 208775dd3366SNeel Natu return (state); 2088366f6083SPeter Grehan } 2089366f6083SPeter Grehan 209095ebc360SNeel Natu int 2091366f6083SPeter Grehan vm_activate_cpu(struct vm *vm, int vcpuid) 2092366f6083SPeter Grehan { 2093366f6083SPeter Grehan 209495ebc360SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 209595ebc360SNeel Natu return (EINVAL); 209695ebc360SNeel Natu 209795ebc360SNeel Natu if (CPU_ISSET(vcpuid, &vm->active_cpus)) 209895ebc360SNeel Natu return (EBUSY); 209922d822c6SNeel Natu 210022d822c6SNeel Natu VCPU_CTR0(vm, vcpuid, "activated"); 210122d822c6SNeel Natu CPU_SET_ATOMIC(vcpuid, &vm->active_cpus); 210295ebc360SNeel Natu return (0); 2103366f6083SPeter Grehan } 2104366f6083SPeter Grehan 2105a5615c90SPeter Grehan cpuset_t 2106366f6083SPeter Grehan vm_active_cpus(struct vm *vm) 2107366f6083SPeter Grehan { 2108366f6083SPeter Grehan 2109366f6083SPeter Grehan return (vm->active_cpus); 2110366f6083SPeter Grehan } 2111366f6083SPeter Grehan 211295ebc360SNeel Natu cpuset_t 211395ebc360SNeel Natu vm_suspended_cpus(struct vm *vm) 211495ebc360SNeel Natu { 211595ebc360SNeel Natu 211695ebc360SNeel Natu return (vm->suspended_cpus); 211795ebc360SNeel Natu } 211895ebc360SNeel Natu 2119366f6083SPeter Grehan void * 2120366f6083SPeter Grehan vcpu_stats(struct vm *vm, int vcpuid) 2121366f6083SPeter Grehan { 2122366f6083SPeter Grehan 2123366f6083SPeter Grehan return (vm->vcpu[vcpuid].stats); 2124366f6083SPeter Grehan } 2125e9027382SNeel Natu 2126e9027382SNeel Natu int 2127e9027382SNeel Natu vm_get_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state *state) 2128e9027382SNeel Natu { 2129e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2130e9027382SNeel Natu return (EINVAL); 2131e9027382SNeel Natu 2132e9027382SNeel Natu *state = vm->vcpu[vcpuid].x2apic_state; 2133e9027382SNeel Natu 2134e9027382SNeel Natu return (0); 2135e9027382SNeel Natu } 2136e9027382SNeel Natu 2137e9027382SNeel Natu int 2138e9027382SNeel Natu vm_set_x2apic_state(struct vm *vm, int vcpuid, enum x2apic_state state) 2139e9027382SNeel Natu { 2140e9027382SNeel Natu if (vcpuid < 0 || vcpuid >= VM_MAXCPU) 2141e9027382SNeel Natu return (EINVAL); 2142e9027382SNeel Natu 21433f23d3caSNeel Natu if (state >= X2APIC_STATE_LAST) 2144e9027382SNeel Natu return (EINVAL); 2145e9027382SNeel Natu 2146e9027382SNeel Natu vm->vcpu[vcpuid].x2apic_state = state; 2147e9027382SNeel Natu 214873820fb0SNeel Natu vlapic_set_x2apic_state(vm, vcpuid, state); 214973820fb0SNeel Natu 2150e9027382SNeel Natu return (0); 2151e9027382SNeel Natu } 215275dd3366SNeel Natu 215322821874SNeel Natu /* 215422821874SNeel Natu * This function is called to ensure that a vcpu "sees" a pending event 215522821874SNeel Natu * as soon as possible: 215622821874SNeel Natu * - If the vcpu thread is sleeping then it is woken up. 215722821874SNeel Natu * - If the vcpu is running on a different host_cpu then an IPI will be directed 215822821874SNeel Natu * to the host_cpu to cause the vcpu to trap into the hypervisor. 215922821874SNeel Natu */ 216075dd3366SNeel Natu void 2161de5ea6b6SNeel Natu vcpu_notify_event(struct vm *vm, int vcpuid, bool lapic_intr) 216275dd3366SNeel Natu { 216375dd3366SNeel Natu int hostcpu; 216475dd3366SNeel Natu struct vcpu *vcpu; 216575dd3366SNeel Natu 216675dd3366SNeel Natu vcpu = &vm->vcpu[vcpuid]; 216775dd3366SNeel Natu 2168f76fc5d4SNeel Natu vcpu_lock(vcpu); 216975dd3366SNeel Natu hostcpu = vcpu->hostcpu; 2170ef39d7e9SNeel Natu if (vcpu->state == VCPU_RUNNING) { 2171ef39d7e9SNeel Natu KASSERT(hostcpu != NOCPU, ("vcpu running on invalid hostcpu")); 2172de5ea6b6SNeel Natu if (hostcpu != curcpu) { 2173ef39d7e9SNeel Natu if (lapic_intr) { 2174add611fdSNeel Natu vlapic_post_intr(vcpu->vlapic, hostcpu, 2175add611fdSNeel Natu vmm_ipinum); 2176ef39d7e9SNeel Natu } else { 217775dd3366SNeel Natu ipi_cpu(hostcpu, vmm_ipinum); 217875dd3366SNeel Natu } 2179ef39d7e9SNeel Natu } else { 2180ef39d7e9SNeel Natu /* 2181ef39d7e9SNeel Natu * If the 'vcpu' is running on 'curcpu' then it must 2182ef39d7e9SNeel Natu * be sending a notification to itself (e.g. SELF_IPI). 2183ef39d7e9SNeel Natu * The pending event will be picked up when the vcpu 2184ef39d7e9SNeel Natu * transitions back to guest context. 2185ef39d7e9SNeel Natu */ 2186ef39d7e9SNeel Natu } 2187ef39d7e9SNeel Natu } else { 2188ef39d7e9SNeel Natu KASSERT(hostcpu == NOCPU, ("vcpu state %d not consistent " 2189ef39d7e9SNeel Natu "with hostcpu %d", vcpu->state, hostcpu)); 2190366f6083SPeter Grehan if (vcpu->state == VCPU_SLEEPING) 2191366f6083SPeter Grehan wakeup_one(vcpu); 2192366f6083SPeter Grehan } 2193f76fc5d4SNeel Natu vcpu_unlock(vcpu); 2194f76fc5d4SNeel Natu } 2195318224bbSNeel Natu 2196318224bbSNeel Natu struct vmspace * 2197318224bbSNeel Natu vm_get_vmspace(struct vm *vm) 2198318224bbSNeel Natu { 2199318224bbSNeel Natu 2200318224bbSNeel Natu return (vm->vmspace); 2201318224bbSNeel Natu } 2202565bbb86SNeel Natu 2203565bbb86SNeel Natu int 2204565bbb86SNeel Natu vm_apicid2vcpuid(struct vm *vm, int apicid) 2205565bbb86SNeel Natu { 2206565bbb86SNeel Natu /* 2207565bbb86SNeel Natu * XXX apic id is assumed to be numerically identical to vcpu id 2208565bbb86SNeel Natu */ 2209565bbb86SNeel Natu return (apicid); 2210565bbb86SNeel Natu } 22115b8a8cd1SNeel Natu 22125b8a8cd1SNeel Natu void 22135b8a8cd1SNeel Natu vm_smp_rendezvous(struct vm *vm, int vcpuid, cpuset_t dest, 22145b8a8cd1SNeel Natu vm_rendezvous_func_t func, void *arg) 22155b8a8cd1SNeel Natu { 2216970955e4SNeel Natu int i; 2217970955e4SNeel Natu 22185b8a8cd1SNeel Natu /* 22195b8a8cd1SNeel Natu * Enforce that this function is called without any locks 22205b8a8cd1SNeel Natu */ 22215b8a8cd1SNeel Natu WITNESS_WARN(WARN_PANIC, NULL, "vm_smp_rendezvous"); 22225b8a8cd1SNeel Natu KASSERT(vcpuid == -1 || (vcpuid >= 0 && vcpuid < VM_MAXCPU), 22235b8a8cd1SNeel Natu ("vm_smp_rendezvous: invalid vcpuid %d", vcpuid)); 22245b8a8cd1SNeel Natu 22255b8a8cd1SNeel Natu restart: 22265b8a8cd1SNeel Natu mtx_lock(&vm->rendezvous_mtx); 22275b8a8cd1SNeel Natu if (vm->rendezvous_func != NULL) { 22285b8a8cd1SNeel Natu /* 22295b8a8cd1SNeel Natu * If a rendezvous is already in progress then we need to 22305b8a8cd1SNeel Natu * call the rendezvous handler in case this 'vcpuid' is one 22315b8a8cd1SNeel Natu * of the targets of the rendezvous. 22325b8a8cd1SNeel Natu */ 22335b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Rendezvous already in progress"); 22345b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 22355b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 22365b8a8cd1SNeel Natu goto restart; 22375b8a8cd1SNeel Natu } 22385b8a8cd1SNeel Natu KASSERT(vm->rendezvous_func == NULL, ("vm_smp_rendezvous: previous " 22395b8a8cd1SNeel Natu "rendezvous is still in progress")); 22405b8a8cd1SNeel Natu 22415b8a8cd1SNeel Natu RENDEZVOUS_CTR0(vm, vcpuid, "Initiating rendezvous"); 22425b8a8cd1SNeel Natu vm->rendezvous_req_cpus = dest; 22435b8a8cd1SNeel Natu CPU_ZERO(&vm->rendezvous_done_cpus); 22445b8a8cd1SNeel Natu vm->rendezvous_arg = arg; 22455b8a8cd1SNeel Natu vm_set_rendezvous_func(vm, func); 22465b8a8cd1SNeel Natu mtx_unlock(&vm->rendezvous_mtx); 22475b8a8cd1SNeel Natu 2248970955e4SNeel Natu /* 2249970955e4SNeel Natu * Wake up any sleeping vcpus and trigger a VM-exit in any running 2250970955e4SNeel Natu * vcpus so they handle the rendezvous as soon as possible. 2251970955e4SNeel Natu */ 2252970955e4SNeel Natu for (i = 0; i < VM_MAXCPU; i++) { 2253970955e4SNeel Natu if (CPU_ISSET(i, &dest)) 2254970955e4SNeel Natu vcpu_notify_event(vm, i, false); 2255970955e4SNeel Natu } 2256970955e4SNeel Natu 22575b8a8cd1SNeel Natu vm_handle_rendezvous(vm, vcpuid); 22585b8a8cd1SNeel Natu } 2259762fd208STycho Nightingale 2260762fd208STycho Nightingale struct vatpic * 2261762fd208STycho Nightingale vm_atpic(struct vm *vm) 2262762fd208STycho Nightingale { 2263762fd208STycho Nightingale return (vm->vatpic); 2264762fd208STycho Nightingale } 2265e883c9bbSTycho Nightingale 2266e883c9bbSTycho Nightingale struct vatpit * 2267e883c9bbSTycho Nightingale vm_atpit(struct vm *vm) 2268e883c9bbSTycho Nightingale { 2269e883c9bbSTycho Nightingale return (vm->vatpit); 2270e883c9bbSTycho Nightingale } 2271d17b5104SNeel Natu 2272160ef77aSNeel Natu struct vpmtmr * 2273160ef77aSNeel Natu vm_pmtmr(struct vm *vm) 2274160ef77aSNeel Natu { 2275160ef77aSNeel Natu 2276160ef77aSNeel Natu return (vm->vpmtmr); 2277160ef77aSNeel Natu } 2278160ef77aSNeel Natu 22790dafa5cdSNeel Natu struct vrtc * 22800dafa5cdSNeel Natu vm_rtc(struct vm *vm) 22810dafa5cdSNeel Natu { 22820dafa5cdSNeel Natu 22830dafa5cdSNeel Natu return (vm->vrtc); 22840dafa5cdSNeel Natu } 22850dafa5cdSNeel Natu 2286d17b5104SNeel Natu enum vm_reg_name 2287d17b5104SNeel Natu vm_segment_name(int seg) 2288d17b5104SNeel Natu { 2289d17b5104SNeel Natu static enum vm_reg_name seg_names[] = { 2290d17b5104SNeel Natu VM_REG_GUEST_ES, 2291d17b5104SNeel Natu VM_REG_GUEST_CS, 2292d17b5104SNeel Natu VM_REG_GUEST_SS, 2293d17b5104SNeel Natu VM_REG_GUEST_DS, 2294d17b5104SNeel Natu VM_REG_GUEST_FS, 2295d17b5104SNeel Natu VM_REG_GUEST_GS 2296d17b5104SNeel Natu }; 2297d17b5104SNeel Natu 2298d17b5104SNeel Natu KASSERT(seg >= 0 && seg < nitems(seg_names), 2299d17b5104SNeel Natu ("%s: invalid segment encoding %d", __func__, seg)); 2300d17b5104SNeel Natu return (seg_names[seg]); 2301d17b5104SNeel Natu } 2302cf1d80d8SPeter Grehan 2303d665d229SNeel Natu void 2304d665d229SNeel Natu vm_copy_teardown(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, 2305d665d229SNeel Natu int num_copyinfo) 2306d665d229SNeel Natu { 2307d665d229SNeel Natu int idx; 2308d665d229SNeel Natu 2309d665d229SNeel Natu for (idx = 0; idx < num_copyinfo; idx++) { 2310d665d229SNeel Natu if (copyinfo[idx].cookie != NULL) 2311d665d229SNeel Natu vm_gpa_release(copyinfo[idx].cookie); 2312d665d229SNeel Natu } 2313d665d229SNeel Natu bzero(copyinfo, num_copyinfo * sizeof(struct vm_copyinfo)); 2314d665d229SNeel Natu } 2315d665d229SNeel Natu 2316d665d229SNeel Natu int 2317d665d229SNeel Natu vm_copy_setup(struct vm *vm, int vcpuid, struct vm_guest_paging *paging, 2318d665d229SNeel Natu uint64_t gla, size_t len, int prot, struct vm_copyinfo *copyinfo, 23199c4d5478SNeel Natu int num_copyinfo, int *fault) 2320d665d229SNeel Natu { 2321d665d229SNeel Natu int error, idx, nused; 2322d665d229SNeel Natu size_t n, off, remaining; 2323d665d229SNeel Natu void *hva, *cookie; 2324d665d229SNeel Natu uint64_t gpa; 2325d665d229SNeel Natu 2326d665d229SNeel Natu bzero(copyinfo, sizeof(struct vm_copyinfo) * num_copyinfo); 2327d665d229SNeel Natu 2328d665d229SNeel Natu nused = 0; 2329d665d229SNeel Natu remaining = len; 2330d665d229SNeel Natu while (remaining > 0) { 2331d665d229SNeel Natu KASSERT(nused < num_copyinfo, ("insufficient vm_copyinfo")); 23329c4d5478SNeel Natu error = vm_gla2gpa(vm, vcpuid, paging, gla, prot, &gpa, fault); 23339c4d5478SNeel Natu if (error || *fault) 2334d665d229SNeel Natu return (error); 2335d665d229SNeel Natu off = gpa & PAGE_MASK; 2336d665d229SNeel Natu n = min(remaining, PAGE_SIZE - off); 2337d665d229SNeel Natu copyinfo[nused].gpa = gpa; 2338d665d229SNeel Natu copyinfo[nused].len = n; 2339d665d229SNeel Natu remaining -= n; 2340d665d229SNeel Natu gla += n; 2341d665d229SNeel Natu nused++; 2342d665d229SNeel Natu } 2343d665d229SNeel Natu 2344d665d229SNeel Natu for (idx = 0; idx < nused; idx++) { 2345d665d229SNeel Natu hva = vm_gpa_hold(vm, copyinfo[idx].gpa, copyinfo[idx].len, 2346d665d229SNeel Natu prot, &cookie); 2347d665d229SNeel Natu if (hva == NULL) 2348d665d229SNeel Natu break; 2349d665d229SNeel Natu copyinfo[idx].hva = hva; 2350d665d229SNeel Natu copyinfo[idx].cookie = cookie; 2351d665d229SNeel Natu } 2352d665d229SNeel Natu 2353d665d229SNeel Natu if (idx != nused) { 2354d665d229SNeel Natu vm_copy_teardown(vm, vcpuid, copyinfo, num_copyinfo); 23559c4d5478SNeel Natu return (EFAULT); 2356d665d229SNeel Natu } else { 23579c4d5478SNeel Natu *fault = 0; 2358d665d229SNeel Natu return (0); 2359d665d229SNeel Natu } 2360d665d229SNeel Natu } 2361d665d229SNeel Natu 2362d665d229SNeel Natu void 2363d665d229SNeel Natu vm_copyin(struct vm *vm, int vcpuid, struct vm_copyinfo *copyinfo, void *kaddr, 2364d665d229SNeel Natu size_t len) 2365d665d229SNeel Natu { 2366d665d229SNeel Natu char *dst; 2367d665d229SNeel Natu int idx; 2368d665d229SNeel Natu 2369d665d229SNeel Natu dst = kaddr; 2370d665d229SNeel Natu idx = 0; 2371d665d229SNeel Natu while (len > 0) { 2372d665d229SNeel Natu bcopy(copyinfo[idx].hva, dst, copyinfo[idx].len); 2373d665d229SNeel Natu len -= copyinfo[idx].len; 2374d665d229SNeel Natu dst += copyinfo[idx].len; 2375d665d229SNeel Natu idx++; 2376d665d229SNeel Natu } 2377d665d229SNeel Natu } 2378d665d229SNeel Natu 2379d665d229SNeel Natu void 2380d665d229SNeel Natu vm_copyout(struct vm *vm, int vcpuid, const void *kaddr, 2381d665d229SNeel Natu struct vm_copyinfo *copyinfo, size_t len) 2382d665d229SNeel Natu { 2383d665d229SNeel Natu const char *src; 2384d665d229SNeel Natu int idx; 2385d665d229SNeel Natu 2386d665d229SNeel Natu src = kaddr; 2387d665d229SNeel Natu idx = 0; 2388d665d229SNeel Natu while (len > 0) { 2389d665d229SNeel Natu bcopy(src, copyinfo[idx].hva, copyinfo[idx].len); 2390d665d229SNeel Natu len -= copyinfo[idx].len; 2391d665d229SNeel Natu src += copyinfo[idx].len; 2392d665d229SNeel Natu idx++; 2393d665d229SNeel Natu } 2394d665d229SNeel Natu } 2395cf1d80d8SPeter Grehan 2396cf1d80d8SPeter Grehan /* 2397cf1d80d8SPeter Grehan * Return the amount of in-use and wired memory for the VM. Since 2398cf1d80d8SPeter Grehan * these are global stats, only return the values with for vCPU 0 2399cf1d80d8SPeter Grehan */ 2400cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_RESIDENT); 2401cf1d80d8SPeter Grehan VMM_STAT_DECLARE(VMM_MEM_WIRED); 2402cf1d80d8SPeter Grehan 2403cf1d80d8SPeter Grehan static void 2404cf1d80d8SPeter Grehan vm_get_rescnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat) 2405cf1d80d8SPeter Grehan { 2406cf1d80d8SPeter Grehan 2407cf1d80d8SPeter Grehan if (vcpu == 0) { 2408cf1d80d8SPeter Grehan vmm_stat_set(vm, vcpu, VMM_MEM_RESIDENT, 2409cf1d80d8SPeter Grehan PAGE_SIZE * vmspace_resident_count(vm->vmspace)); 2410cf1d80d8SPeter Grehan } 2411cf1d80d8SPeter Grehan } 2412cf1d80d8SPeter Grehan 2413cf1d80d8SPeter Grehan static void 2414cf1d80d8SPeter Grehan vm_get_wiredcnt(struct vm *vm, int vcpu, struct vmm_stat_type *stat) 2415cf1d80d8SPeter Grehan { 2416cf1d80d8SPeter Grehan 2417cf1d80d8SPeter Grehan if (vcpu == 0) { 2418cf1d80d8SPeter Grehan vmm_stat_set(vm, vcpu, VMM_MEM_WIRED, 2419cf1d80d8SPeter Grehan PAGE_SIZE * pmap_wired_count(vmspace_pmap(vm->vmspace))); 2420cf1d80d8SPeter Grehan } 2421cf1d80d8SPeter Grehan } 2422cf1d80d8SPeter Grehan 2423cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_RESIDENT, "Resident memory", vm_get_rescnt); 2424cf1d80d8SPeter Grehan VMM_STAT_FUNC(VMM_MEM_WIRED, "Wired memory", vm_get_wiredcnt); 2425