1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright (c) 2011 NetApp, Inc. 5 * All rights reserved. 6 * Copyright (c) 2018 Joyent, Inc. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY NETAPP, INC ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL NETAPP, INC OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 * 29 * $FreeBSD$ 30 */ 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include "opt_bhyve_snapshot.h" 36 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/smp.h> 40 #include <sys/kernel.h> 41 #include <sys/malloc.h> 42 #include <sys/pcpu.h> 43 #include <sys/proc.h> 44 #include <sys/sysctl.h> 45 46 #include <vm/vm.h> 47 #include <vm/pmap.h> 48 49 #include <machine/psl.h> 50 #include <machine/cpufunc.h> 51 #include <machine/md_var.h> 52 #include <machine/reg.h> 53 #include <machine/segments.h> 54 #include <machine/smp.h> 55 #include <machine/specialreg.h> 56 #include <machine/vmparam.h> 57 58 #include <machine/vmm.h> 59 #include <machine/vmm_dev.h> 60 #include <machine/vmm_instruction_emul.h> 61 #include <machine/vmm_snapshot.h> 62 63 #include "vmm_lapic.h" 64 #include "vmm_host.h" 65 #include "vmm_ioport.h" 66 #include "vmm_ktr.h" 67 #include "vmm_stat.h" 68 #include "vatpic.h" 69 #include "vlapic.h" 70 #include "vlapic_priv.h" 71 72 #include "ept.h" 73 #include "vmx_cpufunc.h" 74 #include "vmx.h" 75 #include "vmx_msr.h" 76 #include "x86.h" 77 #include "vmx_controls.h" 78 79 #define PINBASED_CTLS_ONE_SETTING \ 80 (PINBASED_EXTINT_EXITING | \ 81 PINBASED_NMI_EXITING | \ 82 PINBASED_VIRTUAL_NMI) 83 #define PINBASED_CTLS_ZERO_SETTING 0 84 85 #define PROCBASED_CTLS_WINDOW_SETTING \ 86 (PROCBASED_INT_WINDOW_EXITING | \ 87 PROCBASED_NMI_WINDOW_EXITING) 88 89 #define PROCBASED_CTLS_ONE_SETTING \ 90 (PROCBASED_SECONDARY_CONTROLS | \ 91 PROCBASED_MWAIT_EXITING | \ 92 PROCBASED_MONITOR_EXITING | \ 93 PROCBASED_IO_EXITING | \ 94 PROCBASED_MSR_BITMAPS | \ 95 PROCBASED_CTLS_WINDOW_SETTING | \ 96 PROCBASED_CR8_LOAD_EXITING | \ 97 PROCBASED_CR8_STORE_EXITING) 98 #define PROCBASED_CTLS_ZERO_SETTING \ 99 (PROCBASED_CR3_LOAD_EXITING | \ 100 PROCBASED_CR3_STORE_EXITING | \ 101 PROCBASED_IO_BITMAPS) 102 103 #define PROCBASED_CTLS2_ONE_SETTING PROCBASED2_ENABLE_EPT 104 #define PROCBASED_CTLS2_ZERO_SETTING 0 105 106 #define VM_EXIT_CTLS_ONE_SETTING \ 107 (VM_EXIT_SAVE_DEBUG_CONTROLS | \ 108 VM_EXIT_HOST_LMA | \ 109 VM_EXIT_SAVE_EFER | \ 110 VM_EXIT_LOAD_EFER | \ 111 VM_EXIT_ACKNOWLEDGE_INTERRUPT) 112 113 #define VM_EXIT_CTLS_ZERO_SETTING 0 114 115 #define VM_ENTRY_CTLS_ONE_SETTING \ 116 (VM_ENTRY_LOAD_DEBUG_CONTROLS | \ 117 VM_ENTRY_LOAD_EFER) 118 119 #define VM_ENTRY_CTLS_ZERO_SETTING \ 120 (VM_ENTRY_INTO_SMM | \ 121 VM_ENTRY_DEACTIVATE_DUAL_MONITOR) 122 123 #define HANDLED 1 124 #define UNHANDLED 0 125 126 static MALLOC_DEFINE(M_VMX, "vmx", "vmx"); 127 static MALLOC_DEFINE(M_VLAPIC, "vlapic", "vlapic"); 128 129 SYSCTL_DECL(_hw_vmm); 130 SYSCTL_NODE(_hw_vmm, OID_AUTO, vmx, CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 131 NULL); 132 133 int vmxon_enabled[MAXCPU]; 134 static char vmxon_region[MAXCPU][PAGE_SIZE] __aligned(PAGE_SIZE); 135 136 static uint32_t pinbased_ctls, procbased_ctls, procbased_ctls2; 137 static uint32_t exit_ctls, entry_ctls; 138 139 static uint64_t cr0_ones_mask, cr0_zeros_mask; 140 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_ones_mask, CTLFLAG_RD, 141 &cr0_ones_mask, 0, NULL); 142 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr0_zeros_mask, CTLFLAG_RD, 143 &cr0_zeros_mask, 0, NULL); 144 145 static uint64_t cr4_ones_mask, cr4_zeros_mask; 146 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_ones_mask, CTLFLAG_RD, 147 &cr4_ones_mask, 0, NULL); 148 SYSCTL_ULONG(_hw_vmm_vmx, OID_AUTO, cr4_zeros_mask, CTLFLAG_RD, 149 &cr4_zeros_mask, 0, NULL); 150 151 static int vmx_initialized; 152 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, initialized, CTLFLAG_RD, 153 &vmx_initialized, 0, "Intel VMX initialized"); 154 155 /* 156 * Optional capabilities 157 */ 158 static SYSCTL_NODE(_hw_vmm_vmx, OID_AUTO, cap, 159 CTLFLAG_RW | CTLFLAG_MPSAFE, NULL, 160 NULL); 161 162 static int cap_halt_exit; 163 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, halt_exit, CTLFLAG_RD, &cap_halt_exit, 0, 164 "HLT triggers a VM-exit"); 165 166 static int cap_pause_exit; 167 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, pause_exit, CTLFLAG_RD, &cap_pause_exit, 168 0, "PAUSE triggers a VM-exit"); 169 170 static int cap_unrestricted_guest; 171 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, unrestricted_guest, CTLFLAG_RD, 172 &cap_unrestricted_guest, 0, "Unrestricted guests"); 173 174 static int cap_monitor_trap; 175 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, monitor_trap, CTLFLAG_RD, 176 &cap_monitor_trap, 0, "Monitor trap flag"); 177 178 static int cap_invpcid; 179 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, invpcid, CTLFLAG_RD, &cap_invpcid, 180 0, "Guests are allowed to use INVPCID"); 181 182 static int tpr_shadowing; 183 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, tpr_shadowing, CTLFLAG_RD, 184 &tpr_shadowing, 0, "TPR shadowing support"); 185 186 static int virtual_interrupt_delivery; 187 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, virtual_interrupt_delivery, CTLFLAG_RD, 188 &virtual_interrupt_delivery, 0, "APICv virtual interrupt delivery support"); 189 190 static int posted_interrupts; 191 SYSCTL_INT(_hw_vmm_vmx_cap, OID_AUTO, posted_interrupts, CTLFLAG_RD, 192 &posted_interrupts, 0, "APICv posted interrupt support"); 193 194 static int pirvec = -1; 195 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, posted_interrupt_vector, CTLFLAG_RD, 196 &pirvec, 0, "APICv posted interrupt vector"); 197 198 static struct unrhdr *vpid_unr; 199 static u_int vpid_alloc_failed; 200 SYSCTL_UINT(_hw_vmm_vmx, OID_AUTO, vpid_alloc_failed, CTLFLAG_RD, 201 &vpid_alloc_failed, 0, NULL); 202 203 int guest_l1d_flush; 204 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush, CTLFLAG_RD, 205 &guest_l1d_flush, 0, NULL); 206 int guest_l1d_flush_sw; 207 SYSCTL_INT(_hw_vmm_vmx, OID_AUTO, l1d_flush_sw, CTLFLAG_RD, 208 &guest_l1d_flush_sw, 0, NULL); 209 210 static struct msr_entry msr_load_list[1] __aligned(16); 211 212 /* 213 * The definitions of SDT probes for VMX. 214 */ 215 216 SDT_PROBE_DEFINE3(vmm, vmx, exit, entry, 217 "struct vmx *", "int", "struct vm_exit *"); 218 219 SDT_PROBE_DEFINE4(vmm, vmx, exit, taskswitch, 220 "struct vmx *", "int", "struct vm_exit *", "struct vm_task_switch *"); 221 222 SDT_PROBE_DEFINE4(vmm, vmx, exit, craccess, 223 "struct vmx *", "int", "struct vm_exit *", "uint64_t"); 224 225 SDT_PROBE_DEFINE4(vmm, vmx, exit, rdmsr, 226 "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 227 228 SDT_PROBE_DEFINE5(vmm, vmx, exit, wrmsr, 229 "struct vmx *", "int", "struct vm_exit *", "uint32_t", "uint64_t"); 230 231 SDT_PROBE_DEFINE3(vmm, vmx, exit, halt, 232 "struct vmx *", "int", "struct vm_exit *"); 233 234 SDT_PROBE_DEFINE3(vmm, vmx, exit, mtrap, 235 "struct vmx *", "int", "struct vm_exit *"); 236 237 SDT_PROBE_DEFINE3(vmm, vmx, exit, pause, 238 "struct vmx *", "int", "struct vm_exit *"); 239 240 SDT_PROBE_DEFINE3(vmm, vmx, exit, intrwindow, 241 "struct vmx *", "int", "struct vm_exit *"); 242 243 SDT_PROBE_DEFINE4(vmm, vmx, exit, interrupt, 244 "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 245 246 SDT_PROBE_DEFINE3(vmm, vmx, exit, nmiwindow, 247 "struct vmx *", "int", "struct vm_exit *"); 248 249 SDT_PROBE_DEFINE3(vmm, vmx, exit, inout, 250 "struct vmx *", "int", "struct vm_exit *"); 251 252 SDT_PROBE_DEFINE3(vmm, vmx, exit, cpuid, 253 "struct vmx *", "int", "struct vm_exit *"); 254 255 SDT_PROBE_DEFINE5(vmm, vmx, exit, exception, 256 "struct vmx *", "int", "struct vm_exit *", "uint32_t", "int"); 257 258 SDT_PROBE_DEFINE5(vmm, vmx, exit, nestedfault, 259 "struct vmx *", "int", "struct vm_exit *", "uint64_t", "uint64_t"); 260 261 SDT_PROBE_DEFINE4(vmm, vmx, exit, mmiofault, 262 "struct vmx *", "int", "struct vm_exit *", "uint64_t"); 263 264 SDT_PROBE_DEFINE3(vmm, vmx, exit, eoi, 265 "struct vmx *", "int", "struct vm_exit *"); 266 267 SDT_PROBE_DEFINE3(vmm, vmx, exit, apicaccess, 268 "struct vmx *", "int", "struct vm_exit *"); 269 270 SDT_PROBE_DEFINE4(vmm, vmx, exit, apicwrite, 271 "struct vmx *", "int", "struct vm_exit *", "struct vlapic *"); 272 273 SDT_PROBE_DEFINE3(vmm, vmx, exit, xsetbv, 274 "struct vmx *", "int", "struct vm_exit *"); 275 276 SDT_PROBE_DEFINE3(vmm, vmx, exit, monitor, 277 "struct vmx *", "int", "struct vm_exit *"); 278 279 SDT_PROBE_DEFINE3(vmm, vmx, exit, mwait, 280 "struct vmx *", "int", "struct vm_exit *"); 281 282 SDT_PROBE_DEFINE3(vmm, vmx, exit, vminsn, 283 "struct vmx *", "int", "struct vm_exit *"); 284 285 SDT_PROBE_DEFINE4(vmm, vmx, exit, unknown, 286 "struct vmx *", "int", "struct vm_exit *", "uint32_t"); 287 288 SDT_PROBE_DEFINE4(vmm, vmx, exit, return, 289 "struct vmx *", "int", "struct vm_exit *", "int"); 290 291 /* 292 * Use the last page below 4GB as the APIC access address. This address is 293 * occupied by the boot firmware so it is guaranteed that it will not conflict 294 * with a page in system memory. 295 */ 296 #define APIC_ACCESS_ADDRESS 0xFFFFF000 297 298 static int vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc); 299 static int vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval); 300 static int vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val); 301 static void vmx_inject_pir(struct vlapic *vlapic); 302 #ifdef BHYVE_SNAPSHOT 303 static int vmx_restore_tsc(void *arg, int vcpu, uint64_t now); 304 #endif 305 306 #ifdef KTR 307 static const char * 308 exit_reason_to_str(int reason) 309 { 310 static char reasonbuf[32]; 311 312 switch (reason) { 313 case EXIT_REASON_EXCEPTION: 314 return "exception"; 315 case EXIT_REASON_EXT_INTR: 316 return "extint"; 317 case EXIT_REASON_TRIPLE_FAULT: 318 return "triplefault"; 319 case EXIT_REASON_INIT: 320 return "init"; 321 case EXIT_REASON_SIPI: 322 return "sipi"; 323 case EXIT_REASON_IO_SMI: 324 return "iosmi"; 325 case EXIT_REASON_SMI: 326 return "smi"; 327 case EXIT_REASON_INTR_WINDOW: 328 return "intrwindow"; 329 case EXIT_REASON_NMI_WINDOW: 330 return "nmiwindow"; 331 case EXIT_REASON_TASK_SWITCH: 332 return "taskswitch"; 333 case EXIT_REASON_CPUID: 334 return "cpuid"; 335 case EXIT_REASON_GETSEC: 336 return "getsec"; 337 case EXIT_REASON_HLT: 338 return "hlt"; 339 case EXIT_REASON_INVD: 340 return "invd"; 341 case EXIT_REASON_INVLPG: 342 return "invlpg"; 343 case EXIT_REASON_RDPMC: 344 return "rdpmc"; 345 case EXIT_REASON_RDTSC: 346 return "rdtsc"; 347 case EXIT_REASON_RSM: 348 return "rsm"; 349 case EXIT_REASON_VMCALL: 350 return "vmcall"; 351 case EXIT_REASON_VMCLEAR: 352 return "vmclear"; 353 case EXIT_REASON_VMLAUNCH: 354 return "vmlaunch"; 355 case EXIT_REASON_VMPTRLD: 356 return "vmptrld"; 357 case EXIT_REASON_VMPTRST: 358 return "vmptrst"; 359 case EXIT_REASON_VMREAD: 360 return "vmread"; 361 case EXIT_REASON_VMRESUME: 362 return "vmresume"; 363 case EXIT_REASON_VMWRITE: 364 return "vmwrite"; 365 case EXIT_REASON_VMXOFF: 366 return "vmxoff"; 367 case EXIT_REASON_VMXON: 368 return "vmxon"; 369 case EXIT_REASON_CR_ACCESS: 370 return "craccess"; 371 case EXIT_REASON_DR_ACCESS: 372 return "draccess"; 373 case EXIT_REASON_INOUT: 374 return "inout"; 375 case EXIT_REASON_RDMSR: 376 return "rdmsr"; 377 case EXIT_REASON_WRMSR: 378 return "wrmsr"; 379 case EXIT_REASON_INVAL_VMCS: 380 return "invalvmcs"; 381 case EXIT_REASON_INVAL_MSR: 382 return "invalmsr"; 383 case EXIT_REASON_MWAIT: 384 return "mwait"; 385 case EXIT_REASON_MTF: 386 return "mtf"; 387 case EXIT_REASON_MONITOR: 388 return "monitor"; 389 case EXIT_REASON_PAUSE: 390 return "pause"; 391 case EXIT_REASON_MCE_DURING_ENTRY: 392 return "mce-during-entry"; 393 case EXIT_REASON_TPR: 394 return "tpr"; 395 case EXIT_REASON_APIC_ACCESS: 396 return "apic-access"; 397 case EXIT_REASON_GDTR_IDTR: 398 return "gdtridtr"; 399 case EXIT_REASON_LDTR_TR: 400 return "ldtrtr"; 401 case EXIT_REASON_EPT_FAULT: 402 return "eptfault"; 403 case EXIT_REASON_EPT_MISCONFIG: 404 return "eptmisconfig"; 405 case EXIT_REASON_INVEPT: 406 return "invept"; 407 case EXIT_REASON_RDTSCP: 408 return "rdtscp"; 409 case EXIT_REASON_VMX_PREEMPT: 410 return "vmxpreempt"; 411 case EXIT_REASON_INVVPID: 412 return "invvpid"; 413 case EXIT_REASON_WBINVD: 414 return "wbinvd"; 415 case EXIT_REASON_XSETBV: 416 return "xsetbv"; 417 case EXIT_REASON_APIC_WRITE: 418 return "apic-write"; 419 default: 420 snprintf(reasonbuf, sizeof(reasonbuf), "%d", reason); 421 return (reasonbuf); 422 } 423 } 424 #endif /* KTR */ 425 426 static int 427 vmx_allow_x2apic_msrs(struct vmx *vmx) 428 { 429 int i, error; 430 431 error = 0; 432 433 /* 434 * Allow readonly access to the following x2APIC MSRs from the guest. 435 */ 436 error += guest_msr_ro(vmx, MSR_APIC_ID); 437 error += guest_msr_ro(vmx, MSR_APIC_VERSION); 438 error += guest_msr_ro(vmx, MSR_APIC_LDR); 439 error += guest_msr_ro(vmx, MSR_APIC_SVR); 440 441 for (i = 0; i < 8; i++) 442 error += guest_msr_ro(vmx, MSR_APIC_ISR0 + i); 443 444 for (i = 0; i < 8; i++) 445 error += guest_msr_ro(vmx, MSR_APIC_TMR0 + i); 446 447 for (i = 0; i < 8; i++) 448 error += guest_msr_ro(vmx, MSR_APIC_IRR0 + i); 449 450 error += guest_msr_ro(vmx, MSR_APIC_ESR); 451 error += guest_msr_ro(vmx, MSR_APIC_LVT_TIMER); 452 error += guest_msr_ro(vmx, MSR_APIC_LVT_THERMAL); 453 error += guest_msr_ro(vmx, MSR_APIC_LVT_PCINT); 454 error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT0); 455 error += guest_msr_ro(vmx, MSR_APIC_LVT_LINT1); 456 error += guest_msr_ro(vmx, MSR_APIC_LVT_ERROR); 457 error += guest_msr_ro(vmx, MSR_APIC_ICR_TIMER); 458 error += guest_msr_ro(vmx, MSR_APIC_DCR_TIMER); 459 error += guest_msr_ro(vmx, MSR_APIC_ICR); 460 461 /* 462 * Allow TPR, EOI and SELF_IPI MSRs to be read and written by the guest. 463 * 464 * These registers get special treatment described in the section 465 * "Virtualizing MSR-Based APIC Accesses". 466 */ 467 error += guest_msr_rw(vmx, MSR_APIC_TPR); 468 error += guest_msr_rw(vmx, MSR_APIC_EOI); 469 error += guest_msr_rw(vmx, MSR_APIC_SELF_IPI); 470 471 return (error); 472 } 473 474 u_long 475 vmx_fix_cr0(u_long cr0) 476 { 477 478 return ((cr0 | cr0_ones_mask) & ~cr0_zeros_mask); 479 } 480 481 u_long 482 vmx_fix_cr4(u_long cr4) 483 { 484 485 return ((cr4 | cr4_ones_mask) & ~cr4_zeros_mask); 486 } 487 488 static void 489 vpid_free(int vpid) 490 { 491 if (vpid < 0 || vpid > 0xffff) 492 panic("vpid_free: invalid vpid %d", vpid); 493 494 /* 495 * VPIDs [0,VM_MAXCPU] are special and are not allocated from 496 * the unit number allocator. 497 */ 498 499 if (vpid > VM_MAXCPU) 500 free_unr(vpid_unr, vpid); 501 } 502 503 static void 504 vpid_alloc(uint16_t *vpid, int num) 505 { 506 int i, x; 507 508 if (num <= 0 || num > VM_MAXCPU) 509 panic("invalid number of vpids requested: %d", num); 510 511 /* 512 * If the "enable vpid" execution control is not enabled then the 513 * VPID is required to be 0 for all vcpus. 514 */ 515 if ((procbased_ctls2 & PROCBASED2_ENABLE_VPID) == 0) { 516 for (i = 0; i < num; i++) 517 vpid[i] = 0; 518 return; 519 } 520 521 /* 522 * Allocate a unique VPID for each vcpu from the unit number allocator. 523 */ 524 for (i = 0; i < num; i++) { 525 x = alloc_unr(vpid_unr); 526 if (x == -1) 527 break; 528 else 529 vpid[i] = x; 530 } 531 532 if (i < num) { 533 atomic_add_int(&vpid_alloc_failed, 1); 534 535 /* 536 * If the unit number allocator does not have enough unique 537 * VPIDs then we need to allocate from the [1,VM_MAXCPU] range. 538 * 539 * These VPIDs are not be unique across VMs but this does not 540 * affect correctness because the combined mappings are also 541 * tagged with the EP4TA which is unique for each VM. 542 * 543 * It is still sub-optimal because the invvpid will invalidate 544 * combined mappings for a particular VPID across all EP4TAs. 545 */ 546 while (i-- > 0) 547 vpid_free(vpid[i]); 548 549 for (i = 0; i < num; i++) 550 vpid[i] = i + 1; 551 } 552 } 553 554 static void 555 vpid_init(void) 556 { 557 /* 558 * VPID 0 is required when the "enable VPID" execution control is 559 * disabled. 560 * 561 * VPIDs [1,VM_MAXCPU] are used as the "overflow namespace" when the 562 * unit number allocator does not have sufficient unique VPIDs to 563 * satisfy the allocation. 564 * 565 * The remaining VPIDs are managed by the unit number allocator. 566 */ 567 vpid_unr = new_unrhdr(VM_MAXCPU + 1, 0xffff, NULL); 568 } 569 570 static void 571 vmx_disable(void *arg __unused) 572 { 573 struct invvpid_desc invvpid_desc = { 0 }; 574 struct invept_desc invept_desc = { 0 }; 575 576 if (vmxon_enabled[curcpu]) { 577 /* 578 * See sections 25.3.3.3 and 25.3.3.4 in Intel Vol 3b. 579 * 580 * VMXON or VMXOFF are not required to invalidate any TLB 581 * caching structures. This prevents potential retention of 582 * cached information in the TLB between distinct VMX episodes. 583 */ 584 invvpid(INVVPID_TYPE_ALL_CONTEXTS, invvpid_desc); 585 invept(INVEPT_TYPE_ALL_CONTEXTS, invept_desc); 586 vmxoff(); 587 } 588 load_cr4(rcr4() & ~CR4_VMXE); 589 } 590 591 static int 592 vmx_cleanup(void) 593 { 594 595 if (pirvec >= 0) 596 lapic_ipi_free(pirvec); 597 598 if (vpid_unr != NULL) { 599 delete_unrhdr(vpid_unr); 600 vpid_unr = NULL; 601 } 602 603 if (nmi_flush_l1d_sw == 1) 604 nmi_flush_l1d_sw = 0; 605 606 smp_rendezvous(NULL, vmx_disable, NULL, NULL); 607 608 return (0); 609 } 610 611 static void 612 vmx_enable(void *arg __unused) 613 { 614 int error; 615 uint64_t feature_control; 616 617 feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 618 if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 0 || 619 (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 620 wrmsr(MSR_IA32_FEATURE_CONTROL, 621 feature_control | IA32_FEATURE_CONTROL_VMX_EN | 622 IA32_FEATURE_CONTROL_LOCK); 623 } 624 625 load_cr4(rcr4() | CR4_VMXE); 626 627 *(uint32_t *)vmxon_region[curcpu] = vmx_revision(); 628 error = vmxon(vmxon_region[curcpu]); 629 if (error == 0) 630 vmxon_enabled[curcpu] = 1; 631 } 632 633 static void 634 vmx_restore(void) 635 { 636 637 if (vmxon_enabled[curcpu]) 638 vmxon(vmxon_region[curcpu]); 639 } 640 641 static int 642 vmx_init(int ipinum) 643 { 644 int error; 645 uint64_t basic, fixed0, fixed1, feature_control; 646 uint32_t tmp, procbased2_vid_bits; 647 648 /* CPUID.1:ECX[bit 5] must be 1 for processor to support VMX */ 649 if (!(cpu_feature2 & CPUID2_VMX)) { 650 printf("vmx_init: processor does not support VMX operation\n"); 651 return (ENXIO); 652 } 653 654 /* 655 * Verify that MSR_IA32_FEATURE_CONTROL lock and VMXON enable bits 656 * are set (bits 0 and 2 respectively). 657 */ 658 feature_control = rdmsr(MSR_IA32_FEATURE_CONTROL); 659 if ((feature_control & IA32_FEATURE_CONTROL_LOCK) == 1 && 660 (feature_control & IA32_FEATURE_CONTROL_VMX_EN) == 0) { 661 printf("vmx_init: VMX operation disabled by BIOS\n"); 662 return (ENXIO); 663 } 664 665 /* 666 * Verify capabilities MSR_VMX_BASIC: 667 * - bit 54 indicates support for INS/OUTS decoding 668 */ 669 basic = rdmsr(MSR_VMX_BASIC); 670 if ((basic & (1UL << 54)) == 0) { 671 printf("vmx_init: processor does not support desired basic " 672 "capabilities\n"); 673 return (EINVAL); 674 } 675 676 /* Check support for primary processor-based VM-execution controls */ 677 error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 678 MSR_VMX_TRUE_PROCBASED_CTLS, 679 PROCBASED_CTLS_ONE_SETTING, 680 PROCBASED_CTLS_ZERO_SETTING, &procbased_ctls); 681 if (error) { 682 printf("vmx_init: processor does not support desired primary " 683 "processor-based controls\n"); 684 return (error); 685 } 686 687 /* Clear the processor-based ctl bits that are set on demand */ 688 procbased_ctls &= ~PROCBASED_CTLS_WINDOW_SETTING; 689 690 /* Check support for secondary processor-based VM-execution controls */ 691 error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 692 MSR_VMX_PROCBASED_CTLS2, 693 PROCBASED_CTLS2_ONE_SETTING, 694 PROCBASED_CTLS2_ZERO_SETTING, &procbased_ctls2); 695 if (error) { 696 printf("vmx_init: processor does not support desired secondary " 697 "processor-based controls\n"); 698 return (error); 699 } 700 701 /* Check support for VPID */ 702 error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 703 PROCBASED2_ENABLE_VPID, 0, &tmp); 704 if (error == 0) 705 procbased_ctls2 |= PROCBASED2_ENABLE_VPID; 706 707 /* Check support for pin-based VM-execution controls */ 708 error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 709 MSR_VMX_TRUE_PINBASED_CTLS, 710 PINBASED_CTLS_ONE_SETTING, 711 PINBASED_CTLS_ZERO_SETTING, &pinbased_ctls); 712 if (error) { 713 printf("vmx_init: processor does not support desired " 714 "pin-based controls\n"); 715 return (error); 716 } 717 718 /* Check support for VM-exit controls */ 719 error = vmx_set_ctlreg(MSR_VMX_EXIT_CTLS, MSR_VMX_TRUE_EXIT_CTLS, 720 VM_EXIT_CTLS_ONE_SETTING, 721 VM_EXIT_CTLS_ZERO_SETTING, 722 &exit_ctls); 723 if (error) { 724 printf("vmx_init: processor does not support desired " 725 "exit controls\n"); 726 return (error); 727 } 728 729 /* Check support for VM-entry controls */ 730 error = vmx_set_ctlreg(MSR_VMX_ENTRY_CTLS, MSR_VMX_TRUE_ENTRY_CTLS, 731 VM_ENTRY_CTLS_ONE_SETTING, VM_ENTRY_CTLS_ZERO_SETTING, 732 &entry_ctls); 733 if (error) { 734 printf("vmx_init: processor does not support desired " 735 "entry controls\n"); 736 return (error); 737 } 738 739 /* 740 * Check support for optional features by testing them 741 * as individual bits 742 */ 743 cap_halt_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 744 MSR_VMX_TRUE_PROCBASED_CTLS, 745 PROCBASED_HLT_EXITING, 0, 746 &tmp) == 0); 747 748 cap_monitor_trap = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 749 MSR_VMX_PROCBASED_CTLS, 750 PROCBASED_MTF, 0, 751 &tmp) == 0); 752 753 cap_pause_exit = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 754 MSR_VMX_TRUE_PROCBASED_CTLS, 755 PROCBASED_PAUSE_EXITING, 0, 756 &tmp) == 0); 757 758 cap_unrestricted_guest = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 759 MSR_VMX_PROCBASED_CTLS2, 760 PROCBASED2_UNRESTRICTED_GUEST, 0, 761 &tmp) == 0); 762 763 cap_invpcid = (vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, 764 MSR_VMX_PROCBASED_CTLS2, PROCBASED2_ENABLE_INVPCID, 0, 765 &tmp) == 0); 766 767 /* 768 * Check support for TPR shadow. 769 */ 770 error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS, 771 MSR_VMX_TRUE_PROCBASED_CTLS, PROCBASED_USE_TPR_SHADOW, 0, 772 &tmp); 773 if (error == 0) { 774 tpr_shadowing = 1; 775 TUNABLE_INT_FETCH("hw.vmm.vmx.use_tpr_shadowing", 776 &tpr_shadowing); 777 } 778 779 if (tpr_shadowing) { 780 procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 781 procbased_ctls &= ~PROCBASED_CR8_LOAD_EXITING; 782 procbased_ctls &= ~PROCBASED_CR8_STORE_EXITING; 783 } 784 785 /* 786 * Check support for virtual interrupt delivery. 787 */ 788 procbased2_vid_bits = (PROCBASED2_VIRTUALIZE_APIC_ACCESSES | 789 PROCBASED2_VIRTUALIZE_X2APIC_MODE | 790 PROCBASED2_APIC_REGISTER_VIRTUALIZATION | 791 PROCBASED2_VIRTUAL_INTERRUPT_DELIVERY); 792 793 error = vmx_set_ctlreg(MSR_VMX_PROCBASED_CTLS2, MSR_VMX_PROCBASED_CTLS2, 794 procbased2_vid_bits, 0, &tmp); 795 if (error == 0 && tpr_shadowing) { 796 virtual_interrupt_delivery = 1; 797 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_vid", 798 &virtual_interrupt_delivery); 799 } 800 801 if (virtual_interrupt_delivery) { 802 procbased_ctls |= PROCBASED_USE_TPR_SHADOW; 803 procbased_ctls2 |= procbased2_vid_bits; 804 procbased_ctls2 &= ~PROCBASED2_VIRTUALIZE_X2APIC_MODE; 805 806 /* 807 * Check for Posted Interrupts only if Virtual Interrupt 808 * Delivery is enabled. 809 */ 810 error = vmx_set_ctlreg(MSR_VMX_PINBASED_CTLS, 811 MSR_VMX_TRUE_PINBASED_CTLS, PINBASED_POSTED_INTERRUPT, 0, 812 &tmp); 813 if (error == 0) { 814 pirvec = lapic_ipi_alloc(pti ? &IDTVEC(justreturn1_pti) : 815 &IDTVEC(justreturn)); 816 if (pirvec < 0) { 817 if (bootverbose) { 818 printf("vmx_init: unable to allocate " 819 "posted interrupt vector\n"); 820 } 821 } else { 822 posted_interrupts = 1; 823 TUNABLE_INT_FETCH("hw.vmm.vmx.use_apic_pir", 824 &posted_interrupts); 825 } 826 } 827 } 828 829 if (posted_interrupts) 830 pinbased_ctls |= PINBASED_POSTED_INTERRUPT; 831 832 /* Initialize EPT */ 833 error = ept_init(ipinum); 834 if (error) { 835 printf("vmx_init: ept initialization failed (%d)\n", error); 836 return (error); 837 } 838 839 guest_l1d_flush = (cpu_ia32_arch_caps & 840 IA32_ARCH_CAP_SKIP_L1DFL_VMENTRY) == 0; 841 TUNABLE_INT_FETCH("hw.vmm.l1d_flush", &guest_l1d_flush); 842 843 /* 844 * L1D cache flush is enabled. Use IA32_FLUSH_CMD MSR when 845 * available. Otherwise fall back to the software flush 846 * method which loads enough data from the kernel text to 847 * flush existing L1D content, both on VMX entry and on NMI 848 * return. 849 */ 850 if (guest_l1d_flush) { 851 if ((cpu_stdext_feature3 & CPUID_STDEXT3_L1D_FLUSH) == 0) { 852 guest_l1d_flush_sw = 1; 853 TUNABLE_INT_FETCH("hw.vmm.l1d_flush_sw", 854 &guest_l1d_flush_sw); 855 } 856 if (guest_l1d_flush_sw) { 857 if (nmi_flush_l1d_sw <= 1) 858 nmi_flush_l1d_sw = 1; 859 } else { 860 msr_load_list[0].index = MSR_IA32_FLUSH_CMD; 861 msr_load_list[0].val = IA32_FLUSH_CMD_L1D; 862 } 863 } 864 865 /* 866 * Stash the cr0 and cr4 bits that must be fixed to 0 or 1 867 */ 868 fixed0 = rdmsr(MSR_VMX_CR0_FIXED0); 869 fixed1 = rdmsr(MSR_VMX_CR0_FIXED1); 870 cr0_ones_mask = fixed0 & fixed1; 871 cr0_zeros_mask = ~fixed0 & ~fixed1; 872 873 /* 874 * CR0_PE and CR0_PG can be set to zero in VMX non-root operation 875 * if unrestricted guest execution is allowed. 876 */ 877 if (cap_unrestricted_guest) 878 cr0_ones_mask &= ~(CR0_PG | CR0_PE); 879 880 /* 881 * Do not allow the guest to set CR0_NW or CR0_CD. 882 */ 883 cr0_zeros_mask |= (CR0_NW | CR0_CD); 884 885 fixed0 = rdmsr(MSR_VMX_CR4_FIXED0); 886 fixed1 = rdmsr(MSR_VMX_CR4_FIXED1); 887 cr4_ones_mask = fixed0 & fixed1; 888 cr4_zeros_mask = ~fixed0 & ~fixed1; 889 890 vpid_init(); 891 892 vmx_msr_init(); 893 894 /* enable VMX operation */ 895 smp_rendezvous(NULL, vmx_enable, NULL, NULL); 896 897 vmx_initialized = 1; 898 899 return (0); 900 } 901 902 static void 903 vmx_trigger_hostintr(int vector) 904 { 905 uintptr_t func; 906 struct gate_descriptor *gd; 907 908 gd = &idt[vector]; 909 910 KASSERT(vector >= 32 && vector <= 255, ("vmx_trigger_hostintr: " 911 "invalid vector %d", vector)); 912 KASSERT(gd->gd_p == 1, ("gate descriptor for vector %d not present", 913 vector)); 914 KASSERT(gd->gd_type == SDT_SYSIGT, ("gate descriptor for vector %d " 915 "has invalid type %d", vector, gd->gd_type)); 916 KASSERT(gd->gd_dpl == SEL_KPL, ("gate descriptor for vector %d " 917 "has invalid dpl %d", vector, gd->gd_dpl)); 918 KASSERT(gd->gd_selector == GSEL(GCODE_SEL, SEL_KPL), ("gate descriptor " 919 "for vector %d has invalid selector %d", vector, gd->gd_selector)); 920 KASSERT(gd->gd_ist == 0, ("gate descriptor for vector %d has invalid " 921 "IST %d", vector, gd->gd_ist)); 922 923 func = ((long)gd->gd_hioffset << 16 | gd->gd_looffset); 924 vmx_call_isr(func); 925 } 926 927 static int 928 vmx_setup_cr_shadow(int which, struct vmcs *vmcs, uint32_t initial) 929 { 930 int error, mask_ident, shadow_ident; 931 uint64_t mask_value; 932 933 if (which != 0 && which != 4) 934 panic("vmx_setup_cr_shadow: unknown cr%d", which); 935 936 if (which == 0) { 937 mask_ident = VMCS_CR0_MASK; 938 mask_value = cr0_ones_mask | cr0_zeros_mask; 939 shadow_ident = VMCS_CR0_SHADOW; 940 } else { 941 mask_ident = VMCS_CR4_MASK; 942 mask_value = cr4_ones_mask | cr4_zeros_mask; 943 shadow_ident = VMCS_CR4_SHADOW; 944 } 945 946 error = vmcs_setreg(vmcs, 0, VMCS_IDENT(mask_ident), mask_value); 947 if (error) 948 return (error); 949 950 error = vmcs_setreg(vmcs, 0, VMCS_IDENT(shadow_ident), initial); 951 if (error) 952 return (error); 953 954 return (0); 955 } 956 #define vmx_setup_cr0_shadow(vmcs,init) vmx_setup_cr_shadow(0, (vmcs), (init)) 957 #define vmx_setup_cr4_shadow(vmcs,init) vmx_setup_cr_shadow(4, (vmcs), (init)) 958 959 static void * 960 vmx_vminit(struct vm *vm, pmap_t pmap) 961 { 962 uint16_t vpid[VM_MAXCPU]; 963 int i, error; 964 struct vmx *vmx; 965 struct vmcs *vmcs; 966 uint32_t exc_bitmap; 967 uint16_t maxcpus; 968 969 vmx = malloc(sizeof(struct vmx), M_VMX, M_WAITOK | M_ZERO); 970 if ((uintptr_t)vmx & PAGE_MASK) { 971 panic("malloc of struct vmx not aligned on %d byte boundary", 972 PAGE_SIZE); 973 } 974 vmx->vm = vm; 975 976 vmx->eptp = eptp(vtophys((vm_offset_t)pmap->pm_pml4)); 977 978 /* 979 * Clean up EPTP-tagged guest physical and combined mappings 980 * 981 * VMX transitions are not required to invalidate any guest physical 982 * mappings. So, it may be possible for stale guest physical mappings 983 * to be present in the processor TLBs. 984 * 985 * Combined mappings for this EP4TA are also invalidated for all VPIDs. 986 */ 987 ept_invalidate_mappings(vmx->eptp); 988 989 msr_bitmap_initialize(vmx->msr_bitmap); 990 991 /* 992 * It is safe to allow direct access to MSR_GSBASE and MSR_FSBASE. 993 * The guest FSBASE and GSBASE are saved and restored during 994 * vm-exit and vm-entry respectively. The host FSBASE and GSBASE are 995 * always restored from the vmcs host state area on vm-exit. 996 * 997 * The SYSENTER_CS/ESP/EIP MSRs are identical to FS/GSBASE in 998 * how they are saved/restored so can be directly accessed by the 999 * guest. 1000 * 1001 * MSR_EFER is saved and restored in the guest VMCS area on a 1002 * VM exit and entry respectively. It is also restored from the 1003 * host VMCS area on a VM exit. 1004 * 1005 * The TSC MSR is exposed read-only. Writes are disallowed as 1006 * that will impact the host TSC. If the guest does a write 1007 * the "use TSC offsetting" execution control is enabled and the 1008 * difference between the host TSC and the guest TSC is written 1009 * into the TSC offset in the VMCS. 1010 */ 1011 if (guest_msr_rw(vmx, MSR_GSBASE) || 1012 guest_msr_rw(vmx, MSR_FSBASE) || 1013 guest_msr_rw(vmx, MSR_SYSENTER_CS_MSR) || 1014 guest_msr_rw(vmx, MSR_SYSENTER_ESP_MSR) || 1015 guest_msr_rw(vmx, MSR_SYSENTER_EIP_MSR) || 1016 guest_msr_rw(vmx, MSR_EFER) || 1017 guest_msr_ro(vmx, MSR_TSC)) 1018 panic("vmx_vminit: error setting guest msr access"); 1019 1020 vpid_alloc(vpid, VM_MAXCPU); 1021 1022 if (virtual_interrupt_delivery) { 1023 error = vm_map_mmio(vm, DEFAULT_APIC_BASE, PAGE_SIZE, 1024 APIC_ACCESS_ADDRESS); 1025 /* XXX this should really return an error to the caller */ 1026 KASSERT(error == 0, ("vm_map_mmio(apicbase) error %d", error)); 1027 } 1028 1029 maxcpus = vm_get_maxcpus(vm); 1030 for (i = 0; i < maxcpus; i++) { 1031 vmcs = &vmx->vmcs[i]; 1032 vmcs->identifier = vmx_revision(); 1033 error = vmclear(vmcs); 1034 if (error != 0) { 1035 panic("vmx_vminit: vmclear error %d on vcpu %d\n", 1036 error, i); 1037 } 1038 1039 vmx_msr_guest_init(vmx, i); 1040 1041 error = vmcs_init(vmcs); 1042 KASSERT(error == 0, ("vmcs_init error %d", error)); 1043 1044 VMPTRLD(vmcs); 1045 error = 0; 1046 error += vmwrite(VMCS_HOST_RSP, (u_long)&vmx->ctx[i]); 1047 error += vmwrite(VMCS_EPTP, vmx->eptp); 1048 error += vmwrite(VMCS_PIN_BASED_CTLS, pinbased_ctls); 1049 error += vmwrite(VMCS_PRI_PROC_BASED_CTLS, procbased_ctls); 1050 error += vmwrite(VMCS_SEC_PROC_BASED_CTLS, procbased_ctls2); 1051 error += vmwrite(VMCS_EXIT_CTLS, exit_ctls); 1052 error += vmwrite(VMCS_ENTRY_CTLS, entry_ctls); 1053 error += vmwrite(VMCS_MSR_BITMAP, vtophys(vmx->msr_bitmap)); 1054 error += vmwrite(VMCS_VPID, vpid[i]); 1055 1056 if (guest_l1d_flush && !guest_l1d_flush_sw) { 1057 vmcs_write(VMCS_ENTRY_MSR_LOAD, pmap_kextract( 1058 (vm_offset_t)&msr_load_list[0])); 1059 vmcs_write(VMCS_ENTRY_MSR_LOAD_COUNT, 1060 nitems(msr_load_list)); 1061 vmcs_write(VMCS_EXIT_MSR_STORE, 0); 1062 vmcs_write(VMCS_EXIT_MSR_STORE_COUNT, 0); 1063 } 1064 1065 /* exception bitmap */ 1066 if (vcpu_trace_exceptions(vm, i)) 1067 exc_bitmap = 0xffffffff; 1068 else 1069 exc_bitmap = 1 << IDT_MC; 1070 error += vmwrite(VMCS_EXCEPTION_BITMAP, exc_bitmap); 1071 1072 vmx->ctx[i].guest_dr6 = DBREG_DR6_RESERVED1; 1073 error += vmwrite(VMCS_GUEST_DR7, DBREG_DR7_RESERVED1); 1074 1075 if (tpr_shadowing) { 1076 error += vmwrite(VMCS_VIRTUAL_APIC, 1077 vtophys(&vmx->apic_page[i])); 1078 } 1079 1080 if (virtual_interrupt_delivery) { 1081 error += vmwrite(VMCS_APIC_ACCESS, APIC_ACCESS_ADDRESS); 1082 error += vmwrite(VMCS_EOI_EXIT0, 0); 1083 error += vmwrite(VMCS_EOI_EXIT1, 0); 1084 error += vmwrite(VMCS_EOI_EXIT2, 0); 1085 error += vmwrite(VMCS_EOI_EXIT3, 0); 1086 } 1087 if (posted_interrupts) { 1088 error += vmwrite(VMCS_PIR_VECTOR, pirvec); 1089 error += vmwrite(VMCS_PIR_DESC, 1090 vtophys(&vmx->pir_desc[i])); 1091 } 1092 VMCLEAR(vmcs); 1093 KASSERT(error == 0, ("vmx_vminit: error customizing the vmcs")); 1094 1095 vmx->cap[i].set = 0; 1096 vmx->cap[i].proc_ctls = procbased_ctls; 1097 vmx->cap[i].proc_ctls2 = procbased_ctls2; 1098 vmx->cap[i].exc_bitmap = exc_bitmap; 1099 1100 vmx->state[i].nextrip = ~0; 1101 vmx->state[i].lastcpu = NOCPU; 1102 vmx->state[i].vpid = vpid[i]; 1103 1104 /* 1105 * Set up the CR0/4 shadows, and init the read shadow 1106 * to the power-on register value from the Intel Sys Arch. 1107 * CR0 - 0x60000010 1108 * CR4 - 0 1109 */ 1110 error = vmx_setup_cr0_shadow(vmcs, 0x60000010); 1111 if (error != 0) 1112 panic("vmx_setup_cr0_shadow %d", error); 1113 1114 error = vmx_setup_cr4_shadow(vmcs, 0); 1115 if (error != 0) 1116 panic("vmx_setup_cr4_shadow %d", error); 1117 1118 vmx->ctx[i].pmap = pmap; 1119 } 1120 1121 return (vmx); 1122 } 1123 1124 static int 1125 vmx_handle_cpuid(struct vm *vm, int vcpu, struct vmxctx *vmxctx) 1126 { 1127 int handled, func; 1128 1129 func = vmxctx->guest_rax; 1130 1131 handled = x86_emulate_cpuid(vm, vcpu, 1132 (uint32_t*)(&vmxctx->guest_rax), 1133 (uint32_t*)(&vmxctx->guest_rbx), 1134 (uint32_t*)(&vmxctx->guest_rcx), 1135 (uint32_t*)(&vmxctx->guest_rdx)); 1136 return (handled); 1137 } 1138 1139 static __inline void 1140 vmx_run_trace(struct vmx *vmx, int vcpu) 1141 { 1142 #ifdef KTR 1143 VCPU_CTR1(vmx->vm, vcpu, "Resume execution at %#lx", vmcs_guest_rip()); 1144 #endif 1145 } 1146 1147 static __inline void 1148 vmx_exit_trace(struct vmx *vmx, int vcpu, uint64_t rip, uint32_t exit_reason, 1149 int handled) 1150 { 1151 #ifdef KTR 1152 VCPU_CTR3(vmx->vm, vcpu, "%s %s vmexit at 0x%0lx", 1153 handled ? "handled" : "unhandled", 1154 exit_reason_to_str(exit_reason), rip); 1155 #endif 1156 } 1157 1158 static __inline void 1159 vmx_astpending_trace(struct vmx *vmx, int vcpu, uint64_t rip) 1160 { 1161 #ifdef KTR 1162 VCPU_CTR1(vmx->vm, vcpu, "astpending vmexit at 0x%0lx", rip); 1163 #endif 1164 } 1165 1166 static VMM_STAT_INTEL(VCPU_INVVPID_SAVED, "Number of vpid invalidations saved"); 1167 static VMM_STAT_INTEL(VCPU_INVVPID_DONE, "Number of vpid invalidations done"); 1168 1169 /* 1170 * Invalidate guest mappings identified by its vpid from the TLB. 1171 */ 1172 static __inline void 1173 vmx_invvpid(struct vmx *vmx, int vcpu, pmap_t pmap, int running) 1174 { 1175 struct vmxstate *vmxstate; 1176 struct invvpid_desc invvpid_desc; 1177 1178 vmxstate = &vmx->state[vcpu]; 1179 if (vmxstate->vpid == 0) 1180 return; 1181 1182 if (!running) { 1183 /* 1184 * Set the 'lastcpu' to an invalid host cpu. 1185 * 1186 * This will invalidate TLB entries tagged with the vcpu's 1187 * vpid the next time it runs via vmx_set_pcpu_defaults(). 1188 */ 1189 vmxstate->lastcpu = NOCPU; 1190 return; 1191 } 1192 1193 KASSERT(curthread->td_critnest > 0, ("%s: vcpu %d running outside " 1194 "critical section", __func__, vcpu)); 1195 1196 /* 1197 * Invalidate all mappings tagged with 'vpid' 1198 * 1199 * We do this because this vcpu was executing on a different host 1200 * cpu when it last ran. We do not track whether it invalidated 1201 * mappings associated with its 'vpid' during that run. So we must 1202 * assume that the mappings associated with 'vpid' on 'curcpu' are 1203 * stale and invalidate them. 1204 * 1205 * Note that we incur this penalty only when the scheduler chooses to 1206 * move the thread associated with this vcpu between host cpus. 1207 * 1208 * Note also that this will invalidate mappings tagged with 'vpid' 1209 * for "all" EP4TAs. 1210 */ 1211 if (pmap->pm_eptgen == vmx->eptgen[curcpu]) { 1212 invvpid_desc._res1 = 0; 1213 invvpid_desc._res2 = 0; 1214 invvpid_desc.vpid = vmxstate->vpid; 1215 invvpid_desc.linear_addr = 0; 1216 invvpid(INVVPID_TYPE_SINGLE_CONTEXT, invvpid_desc); 1217 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_DONE, 1); 1218 } else { 1219 /* 1220 * The invvpid can be skipped if an invept is going to 1221 * be performed before entering the guest. The invept 1222 * will invalidate combined mappings tagged with 1223 * 'vmx->eptp' for all vpids. 1224 */ 1225 vmm_stat_incr(vmx->vm, vcpu, VCPU_INVVPID_SAVED, 1); 1226 } 1227 } 1228 1229 static void 1230 vmx_set_pcpu_defaults(struct vmx *vmx, int vcpu, pmap_t pmap) 1231 { 1232 struct vmxstate *vmxstate; 1233 1234 vmxstate = &vmx->state[vcpu]; 1235 if (vmxstate->lastcpu == curcpu) 1236 return; 1237 1238 vmxstate->lastcpu = curcpu; 1239 1240 vmm_stat_incr(vmx->vm, vcpu, VCPU_MIGRATIONS, 1); 1241 1242 vmcs_write(VMCS_HOST_TR_BASE, vmm_get_host_trbase()); 1243 vmcs_write(VMCS_HOST_GDTR_BASE, vmm_get_host_gdtrbase()); 1244 vmcs_write(VMCS_HOST_GS_BASE, vmm_get_host_gsbase()); 1245 vmx_invvpid(vmx, vcpu, pmap, 1); 1246 } 1247 1248 /* 1249 * We depend on 'procbased_ctls' to have the Interrupt Window Exiting bit set. 1250 */ 1251 CTASSERT((PROCBASED_CTLS_ONE_SETTING & PROCBASED_INT_WINDOW_EXITING) != 0); 1252 1253 static void __inline 1254 vmx_set_int_window_exiting(struct vmx *vmx, int vcpu) 1255 { 1256 1257 if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) == 0) { 1258 vmx->cap[vcpu].proc_ctls |= PROCBASED_INT_WINDOW_EXITING; 1259 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 1260 VCPU_CTR0(vmx->vm, vcpu, "Enabling interrupt window exiting"); 1261 } 1262 } 1263 1264 static void __inline 1265 vmx_clear_int_window_exiting(struct vmx *vmx, int vcpu) 1266 { 1267 1268 KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0, 1269 ("intr_window_exiting not set: %#x", vmx->cap[vcpu].proc_ctls)); 1270 vmx->cap[vcpu].proc_ctls &= ~PROCBASED_INT_WINDOW_EXITING; 1271 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 1272 VCPU_CTR0(vmx->vm, vcpu, "Disabling interrupt window exiting"); 1273 } 1274 1275 static void __inline 1276 vmx_set_nmi_window_exiting(struct vmx *vmx, int vcpu) 1277 { 1278 1279 if ((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) == 0) { 1280 vmx->cap[vcpu].proc_ctls |= PROCBASED_NMI_WINDOW_EXITING; 1281 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 1282 VCPU_CTR0(vmx->vm, vcpu, "Enabling NMI window exiting"); 1283 } 1284 } 1285 1286 static void __inline 1287 vmx_clear_nmi_window_exiting(struct vmx *vmx, int vcpu) 1288 { 1289 1290 KASSERT((vmx->cap[vcpu].proc_ctls & PROCBASED_NMI_WINDOW_EXITING) != 0, 1291 ("nmi_window_exiting not set %#x", vmx->cap[vcpu].proc_ctls)); 1292 vmx->cap[vcpu].proc_ctls &= ~PROCBASED_NMI_WINDOW_EXITING; 1293 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 1294 VCPU_CTR0(vmx->vm, vcpu, "Disabling NMI window exiting"); 1295 } 1296 1297 int 1298 vmx_set_tsc_offset(struct vmx *vmx, int vcpu, uint64_t offset) 1299 { 1300 int error; 1301 1302 if ((vmx->cap[vcpu].proc_ctls & PROCBASED_TSC_OFFSET) == 0) { 1303 vmx->cap[vcpu].proc_ctls |= PROCBASED_TSC_OFFSET; 1304 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, vmx->cap[vcpu].proc_ctls); 1305 VCPU_CTR0(vmx->vm, vcpu, "Enabling TSC offsetting"); 1306 } 1307 1308 error = vmwrite(VMCS_TSC_OFFSET, offset); 1309 #ifdef BHYVE_SNAPSHOT 1310 if (error == 0) 1311 error = vm_set_tsc_offset(vmx->vm, vcpu, offset); 1312 #endif 1313 return (error); 1314 } 1315 1316 #define NMI_BLOCKING (VMCS_INTERRUPTIBILITY_NMI_BLOCKING | \ 1317 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 1318 #define HWINTR_BLOCKING (VMCS_INTERRUPTIBILITY_STI_BLOCKING | \ 1319 VMCS_INTERRUPTIBILITY_MOVSS_BLOCKING) 1320 1321 static void 1322 vmx_inject_nmi(struct vmx *vmx, int vcpu) 1323 { 1324 uint32_t gi, info; 1325 1326 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1327 KASSERT((gi & NMI_BLOCKING) == 0, ("vmx_inject_nmi: invalid guest " 1328 "interruptibility-state %#x", gi)); 1329 1330 info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1331 KASSERT((info & VMCS_INTR_VALID) == 0, ("vmx_inject_nmi: invalid " 1332 "VM-entry interruption information %#x", info)); 1333 1334 /* 1335 * Inject the virtual NMI. The vector must be the NMI IDT entry 1336 * or the VMCS entry check will fail. 1337 */ 1338 info = IDT_NMI | VMCS_INTR_T_NMI | VMCS_INTR_VALID; 1339 vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1340 1341 VCPU_CTR0(vmx->vm, vcpu, "Injecting vNMI"); 1342 1343 /* Clear the request */ 1344 vm_nmi_clear(vmx->vm, vcpu); 1345 } 1346 1347 static void 1348 vmx_inject_interrupts(struct vmx *vmx, int vcpu, struct vlapic *vlapic, 1349 uint64_t guestrip) 1350 { 1351 int vector, need_nmi_exiting, extint_pending; 1352 uint64_t rflags, entryinfo; 1353 uint32_t gi, info; 1354 1355 if (vmx->state[vcpu].nextrip != guestrip) { 1356 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1357 if (gi & HWINTR_BLOCKING) { 1358 VCPU_CTR2(vmx->vm, vcpu, "Guest interrupt blocking " 1359 "cleared due to rip change: %#lx/%#lx", 1360 vmx->state[vcpu].nextrip, guestrip); 1361 gi &= ~HWINTR_BLOCKING; 1362 vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1363 } 1364 } 1365 1366 if (vm_entry_intinfo(vmx->vm, vcpu, &entryinfo)) { 1367 KASSERT((entryinfo & VMCS_INTR_VALID) != 0, ("%s: entry " 1368 "intinfo is not valid: %#lx", __func__, entryinfo)); 1369 1370 info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1371 KASSERT((info & VMCS_INTR_VALID) == 0, ("%s: cannot inject " 1372 "pending exception: %#lx/%#x", __func__, entryinfo, info)); 1373 1374 info = entryinfo; 1375 vector = info & 0xff; 1376 if (vector == IDT_BP || vector == IDT_OF) { 1377 /* 1378 * VT-x requires #BP and #OF to be injected as software 1379 * exceptions. 1380 */ 1381 info &= ~VMCS_INTR_T_MASK; 1382 info |= VMCS_INTR_T_SWEXCEPTION; 1383 } 1384 1385 if (info & VMCS_INTR_DEL_ERRCODE) 1386 vmcs_write(VMCS_ENTRY_EXCEPTION_ERROR, entryinfo >> 32); 1387 1388 vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1389 } 1390 1391 if (vm_nmi_pending(vmx->vm, vcpu)) { 1392 /* 1393 * If there are no conditions blocking NMI injection then 1394 * inject it directly here otherwise enable "NMI window 1395 * exiting" to inject it as soon as we can. 1396 * 1397 * We also check for STI_BLOCKING because some implementations 1398 * don't allow NMI injection in this case. If we are running 1399 * on a processor that doesn't have this restriction it will 1400 * immediately exit and the NMI will be injected in the 1401 * "NMI window exiting" handler. 1402 */ 1403 need_nmi_exiting = 1; 1404 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1405 if ((gi & (HWINTR_BLOCKING | NMI_BLOCKING)) == 0) { 1406 info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1407 if ((info & VMCS_INTR_VALID) == 0) { 1408 vmx_inject_nmi(vmx, vcpu); 1409 need_nmi_exiting = 0; 1410 } else { 1411 VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI " 1412 "due to VM-entry intr info %#x", info); 1413 } 1414 } else { 1415 VCPU_CTR1(vmx->vm, vcpu, "Cannot inject NMI due to " 1416 "Guest Interruptibility-state %#x", gi); 1417 } 1418 1419 if (need_nmi_exiting) 1420 vmx_set_nmi_window_exiting(vmx, vcpu); 1421 } 1422 1423 extint_pending = vm_extint_pending(vmx->vm, vcpu); 1424 1425 if (!extint_pending && virtual_interrupt_delivery) { 1426 vmx_inject_pir(vlapic); 1427 return; 1428 } 1429 1430 /* 1431 * If interrupt-window exiting is already in effect then don't bother 1432 * checking for pending interrupts. This is just an optimization and 1433 * not needed for correctness. 1434 */ 1435 if ((vmx->cap[vcpu].proc_ctls & PROCBASED_INT_WINDOW_EXITING) != 0) { 1436 VCPU_CTR0(vmx->vm, vcpu, "Skip interrupt injection due to " 1437 "pending int_window_exiting"); 1438 return; 1439 } 1440 1441 if (!extint_pending) { 1442 /* Ask the local apic for a vector to inject */ 1443 if (!vlapic_pending_intr(vlapic, &vector)) 1444 return; 1445 1446 /* 1447 * From the Intel SDM, Volume 3, Section "Maskable 1448 * Hardware Interrupts": 1449 * - maskable interrupt vectors [16,255] can be delivered 1450 * through the local APIC. 1451 */ 1452 KASSERT(vector >= 16 && vector <= 255, 1453 ("invalid vector %d from local APIC", vector)); 1454 } else { 1455 /* Ask the legacy pic for a vector to inject */ 1456 vatpic_pending_intr(vmx->vm, &vector); 1457 1458 /* 1459 * From the Intel SDM, Volume 3, Section "Maskable 1460 * Hardware Interrupts": 1461 * - maskable interrupt vectors [0,255] can be delivered 1462 * through the INTR pin. 1463 */ 1464 KASSERT(vector >= 0 && vector <= 255, 1465 ("invalid vector %d from INTR", vector)); 1466 } 1467 1468 /* Check RFLAGS.IF and the interruptibility state of the guest */ 1469 rflags = vmcs_read(VMCS_GUEST_RFLAGS); 1470 if ((rflags & PSL_I) == 0) { 1471 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 1472 "rflags %#lx", vector, rflags); 1473 goto cantinject; 1474 } 1475 1476 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1477 if (gi & HWINTR_BLOCKING) { 1478 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 1479 "Guest Interruptibility-state %#x", vector, gi); 1480 goto cantinject; 1481 } 1482 1483 info = vmcs_read(VMCS_ENTRY_INTR_INFO); 1484 if (info & VMCS_INTR_VALID) { 1485 /* 1486 * This is expected and could happen for multiple reasons: 1487 * - A vectoring VM-entry was aborted due to astpending 1488 * - A VM-exit happened during event injection. 1489 * - An exception was injected above. 1490 * - An NMI was injected above or after "NMI window exiting" 1491 */ 1492 VCPU_CTR2(vmx->vm, vcpu, "Cannot inject vector %d due to " 1493 "VM-entry intr info %#x", vector, info); 1494 goto cantinject; 1495 } 1496 1497 /* Inject the interrupt */ 1498 info = VMCS_INTR_T_HWINTR | VMCS_INTR_VALID; 1499 info |= vector; 1500 vmcs_write(VMCS_ENTRY_INTR_INFO, info); 1501 1502 if (!extint_pending) { 1503 /* Update the Local APIC ISR */ 1504 vlapic_intr_accepted(vlapic, vector); 1505 } else { 1506 vm_extint_clear(vmx->vm, vcpu); 1507 vatpic_intr_accepted(vmx->vm, vector); 1508 1509 /* 1510 * After we accepted the current ExtINT the PIC may 1511 * have posted another one. If that is the case, set 1512 * the Interrupt Window Exiting execution control so 1513 * we can inject that one too. 1514 * 1515 * Also, interrupt window exiting allows us to inject any 1516 * pending APIC vector that was preempted by the ExtINT 1517 * as soon as possible. This applies both for the software 1518 * emulated vlapic and the hardware assisted virtual APIC. 1519 */ 1520 vmx_set_int_window_exiting(vmx, vcpu); 1521 } 1522 1523 VCPU_CTR1(vmx->vm, vcpu, "Injecting hwintr at vector %d", vector); 1524 1525 return; 1526 1527 cantinject: 1528 /* 1529 * Set the Interrupt Window Exiting execution control so we can inject 1530 * the interrupt as soon as blocking condition goes away. 1531 */ 1532 vmx_set_int_window_exiting(vmx, vcpu); 1533 } 1534 1535 /* 1536 * If the Virtual NMIs execution control is '1' then the logical processor 1537 * tracks virtual-NMI blocking in the Guest Interruptibility-state field of 1538 * the VMCS. An IRET instruction in VMX non-root operation will remove any 1539 * virtual-NMI blocking. 1540 * 1541 * This unblocking occurs even if the IRET causes a fault. In this case the 1542 * hypervisor needs to restore virtual-NMI blocking before resuming the guest. 1543 */ 1544 static void 1545 vmx_restore_nmi_blocking(struct vmx *vmx, int vcpuid) 1546 { 1547 uint32_t gi; 1548 1549 VCPU_CTR0(vmx->vm, vcpuid, "Restore Virtual-NMI blocking"); 1550 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1551 gi |= VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1552 vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1553 } 1554 1555 static void 1556 vmx_clear_nmi_blocking(struct vmx *vmx, int vcpuid) 1557 { 1558 uint32_t gi; 1559 1560 VCPU_CTR0(vmx->vm, vcpuid, "Clear Virtual-NMI blocking"); 1561 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1562 gi &= ~VMCS_INTERRUPTIBILITY_NMI_BLOCKING; 1563 vmcs_write(VMCS_GUEST_INTERRUPTIBILITY, gi); 1564 } 1565 1566 static void 1567 vmx_assert_nmi_blocking(struct vmx *vmx, int vcpuid) 1568 { 1569 uint32_t gi; 1570 1571 gi = vmcs_read(VMCS_GUEST_INTERRUPTIBILITY); 1572 KASSERT(gi & VMCS_INTERRUPTIBILITY_NMI_BLOCKING, 1573 ("NMI blocking is not in effect %#x", gi)); 1574 } 1575 1576 static int 1577 vmx_emulate_xsetbv(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 1578 { 1579 struct vmxctx *vmxctx; 1580 uint64_t xcrval; 1581 const struct xsave_limits *limits; 1582 1583 vmxctx = &vmx->ctx[vcpu]; 1584 limits = vmm_get_xsave_limits(); 1585 1586 /* 1587 * Note that the processor raises a GP# fault on its own if 1588 * xsetbv is executed for CPL != 0, so we do not have to 1589 * emulate that fault here. 1590 */ 1591 1592 /* Only xcr0 is supported. */ 1593 if (vmxctx->guest_rcx != 0) { 1594 vm_inject_gp(vmx->vm, vcpu); 1595 return (HANDLED); 1596 } 1597 1598 /* We only handle xcr0 if both the host and guest have XSAVE enabled. */ 1599 if (!limits->xsave_enabled || !(vmcs_read(VMCS_GUEST_CR4) & CR4_XSAVE)) { 1600 vm_inject_ud(vmx->vm, vcpu); 1601 return (HANDLED); 1602 } 1603 1604 xcrval = vmxctx->guest_rdx << 32 | (vmxctx->guest_rax & 0xffffffff); 1605 if ((xcrval & ~limits->xcr0_allowed) != 0) { 1606 vm_inject_gp(vmx->vm, vcpu); 1607 return (HANDLED); 1608 } 1609 1610 if (!(xcrval & XFEATURE_ENABLED_X87)) { 1611 vm_inject_gp(vmx->vm, vcpu); 1612 return (HANDLED); 1613 } 1614 1615 /* AVX (YMM_Hi128) requires SSE. */ 1616 if (xcrval & XFEATURE_ENABLED_AVX && 1617 (xcrval & XFEATURE_AVX) != XFEATURE_AVX) { 1618 vm_inject_gp(vmx->vm, vcpu); 1619 return (HANDLED); 1620 } 1621 1622 /* 1623 * AVX512 requires base AVX (YMM_Hi128) as well as OpMask, 1624 * ZMM_Hi256, and Hi16_ZMM. 1625 */ 1626 if (xcrval & XFEATURE_AVX512 && 1627 (xcrval & (XFEATURE_AVX512 | XFEATURE_AVX)) != 1628 (XFEATURE_AVX512 | XFEATURE_AVX)) { 1629 vm_inject_gp(vmx->vm, vcpu); 1630 return (HANDLED); 1631 } 1632 1633 /* 1634 * Intel MPX requires both bound register state flags to be 1635 * set. 1636 */ 1637 if (((xcrval & XFEATURE_ENABLED_BNDREGS) != 0) != 1638 ((xcrval & XFEATURE_ENABLED_BNDCSR) != 0)) { 1639 vm_inject_gp(vmx->vm, vcpu); 1640 return (HANDLED); 1641 } 1642 1643 /* 1644 * This runs "inside" vmrun() with the guest's FPU state, so 1645 * modifying xcr0 directly modifies the guest's xcr0, not the 1646 * host's. 1647 */ 1648 load_xcr(0, xcrval); 1649 return (HANDLED); 1650 } 1651 1652 static uint64_t 1653 vmx_get_guest_reg(struct vmx *vmx, int vcpu, int ident) 1654 { 1655 const struct vmxctx *vmxctx; 1656 1657 vmxctx = &vmx->ctx[vcpu]; 1658 1659 switch (ident) { 1660 case 0: 1661 return (vmxctx->guest_rax); 1662 case 1: 1663 return (vmxctx->guest_rcx); 1664 case 2: 1665 return (vmxctx->guest_rdx); 1666 case 3: 1667 return (vmxctx->guest_rbx); 1668 case 4: 1669 return (vmcs_read(VMCS_GUEST_RSP)); 1670 case 5: 1671 return (vmxctx->guest_rbp); 1672 case 6: 1673 return (vmxctx->guest_rsi); 1674 case 7: 1675 return (vmxctx->guest_rdi); 1676 case 8: 1677 return (vmxctx->guest_r8); 1678 case 9: 1679 return (vmxctx->guest_r9); 1680 case 10: 1681 return (vmxctx->guest_r10); 1682 case 11: 1683 return (vmxctx->guest_r11); 1684 case 12: 1685 return (vmxctx->guest_r12); 1686 case 13: 1687 return (vmxctx->guest_r13); 1688 case 14: 1689 return (vmxctx->guest_r14); 1690 case 15: 1691 return (vmxctx->guest_r15); 1692 default: 1693 panic("invalid vmx register %d", ident); 1694 } 1695 } 1696 1697 static void 1698 vmx_set_guest_reg(struct vmx *vmx, int vcpu, int ident, uint64_t regval) 1699 { 1700 struct vmxctx *vmxctx; 1701 1702 vmxctx = &vmx->ctx[vcpu]; 1703 1704 switch (ident) { 1705 case 0: 1706 vmxctx->guest_rax = regval; 1707 break; 1708 case 1: 1709 vmxctx->guest_rcx = regval; 1710 break; 1711 case 2: 1712 vmxctx->guest_rdx = regval; 1713 break; 1714 case 3: 1715 vmxctx->guest_rbx = regval; 1716 break; 1717 case 4: 1718 vmcs_write(VMCS_GUEST_RSP, regval); 1719 break; 1720 case 5: 1721 vmxctx->guest_rbp = regval; 1722 break; 1723 case 6: 1724 vmxctx->guest_rsi = regval; 1725 break; 1726 case 7: 1727 vmxctx->guest_rdi = regval; 1728 break; 1729 case 8: 1730 vmxctx->guest_r8 = regval; 1731 break; 1732 case 9: 1733 vmxctx->guest_r9 = regval; 1734 break; 1735 case 10: 1736 vmxctx->guest_r10 = regval; 1737 break; 1738 case 11: 1739 vmxctx->guest_r11 = regval; 1740 break; 1741 case 12: 1742 vmxctx->guest_r12 = regval; 1743 break; 1744 case 13: 1745 vmxctx->guest_r13 = regval; 1746 break; 1747 case 14: 1748 vmxctx->guest_r14 = regval; 1749 break; 1750 case 15: 1751 vmxctx->guest_r15 = regval; 1752 break; 1753 default: 1754 panic("invalid vmx register %d", ident); 1755 } 1756 } 1757 1758 static int 1759 vmx_emulate_cr0_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1760 { 1761 uint64_t crval, regval; 1762 1763 /* We only handle mov to %cr0 at this time */ 1764 if ((exitqual & 0xf0) != 0x00) 1765 return (UNHANDLED); 1766 1767 regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); 1768 1769 vmcs_write(VMCS_CR0_SHADOW, regval); 1770 1771 crval = regval | cr0_ones_mask; 1772 crval &= ~cr0_zeros_mask; 1773 vmcs_write(VMCS_GUEST_CR0, crval); 1774 1775 if (regval & CR0_PG) { 1776 uint64_t efer, entry_ctls; 1777 1778 /* 1779 * If CR0.PG is 1 and EFER.LME is 1 then EFER.LMA and 1780 * the "IA-32e mode guest" bit in VM-entry control must be 1781 * equal. 1782 */ 1783 efer = vmcs_read(VMCS_GUEST_IA32_EFER); 1784 if (efer & EFER_LME) { 1785 efer |= EFER_LMA; 1786 vmcs_write(VMCS_GUEST_IA32_EFER, efer); 1787 entry_ctls = vmcs_read(VMCS_ENTRY_CTLS); 1788 entry_ctls |= VM_ENTRY_GUEST_LMA; 1789 vmcs_write(VMCS_ENTRY_CTLS, entry_ctls); 1790 } 1791 } 1792 1793 return (HANDLED); 1794 } 1795 1796 static int 1797 vmx_emulate_cr4_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1798 { 1799 uint64_t crval, regval; 1800 1801 /* We only handle mov to %cr4 at this time */ 1802 if ((exitqual & 0xf0) != 0x00) 1803 return (UNHANDLED); 1804 1805 regval = vmx_get_guest_reg(vmx, vcpu, (exitqual >> 8) & 0xf); 1806 1807 vmcs_write(VMCS_CR4_SHADOW, regval); 1808 1809 crval = regval | cr4_ones_mask; 1810 crval &= ~cr4_zeros_mask; 1811 vmcs_write(VMCS_GUEST_CR4, crval); 1812 1813 return (HANDLED); 1814 } 1815 1816 static int 1817 vmx_emulate_cr8_access(struct vmx *vmx, int vcpu, uint64_t exitqual) 1818 { 1819 struct vlapic *vlapic; 1820 uint64_t cr8; 1821 int regnum; 1822 1823 /* We only handle mov %cr8 to/from a register at this time. */ 1824 if ((exitqual & 0xe0) != 0x00) { 1825 return (UNHANDLED); 1826 } 1827 1828 vlapic = vm_lapic(vmx->vm, vcpu); 1829 regnum = (exitqual >> 8) & 0xf; 1830 if (exitqual & 0x10) { 1831 cr8 = vlapic_get_cr8(vlapic); 1832 vmx_set_guest_reg(vmx, vcpu, regnum, cr8); 1833 } else { 1834 cr8 = vmx_get_guest_reg(vmx, vcpu, regnum); 1835 vlapic_set_cr8(vlapic, cr8); 1836 } 1837 1838 return (HANDLED); 1839 } 1840 1841 /* 1842 * From section "Guest Register State" in the Intel SDM: CPL = SS.DPL 1843 */ 1844 static int 1845 vmx_cpl(void) 1846 { 1847 uint32_t ssar; 1848 1849 ssar = vmcs_read(VMCS_GUEST_SS_ACCESS_RIGHTS); 1850 return ((ssar >> 5) & 0x3); 1851 } 1852 1853 static enum vm_cpu_mode 1854 vmx_cpu_mode(void) 1855 { 1856 uint32_t csar; 1857 1858 if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LMA) { 1859 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 1860 if (csar & 0x2000) 1861 return (CPU_MODE_64BIT); /* CS.L = 1 */ 1862 else 1863 return (CPU_MODE_COMPATIBILITY); 1864 } else if (vmcs_read(VMCS_GUEST_CR0) & CR0_PE) { 1865 return (CPU_MODE_PROTECTED); 1866 } else { 1867 return (CPU_MODE_REAL); 1868 } 1869 } 1870 1871 static enum vm_paging_mode 1872 vmx_paging_mode(void) 1873 { 1874 1875 if (!(vmcs_read(VMCS_GUEST_CR0) & CR0_PG)) 1876 return (PAGING_MODE_FLAT); 1877 if (!(vmcs_read(VMCS_GUEST_CR4) & CR4_PAE)) 1878 return (PAGING_MODE_32); 1879 if (vmcs_read(VMCS_GUEST_IA32_EFER) & EFER_LME) 1880 return (PAGING_MODE_64); 1881 else 1882 return (PAGING_MODE_PAE); 1883 } 1884 1885 static uint64_t 1886 inout_str_index(struct vmx *vmx, int vcpuid, int in) 1887 { 1888 uint64_t val; 1889 int error; 1890 enum vm_reg_name reg; 1891 1892 reg = in ? VM_REG_GUEST_RDI : VM_REG_GUEST_RSI; 1893 error = vmx_getreg(vmx, vcpuid, reg, &val); 1894 KASSERT(error == 0, ("%s: vmx_getreg error %d", __func__, error)); 1895 return (val); 1896 } 1897 1898 static uint64_t 1899 inout_str_count(struct vmx *vmx, int vcpuid, int rep) 1900 { 1901 uint64_t val; 1902 int error; 1903 1904 if (rep) { 1905 error = vmx_getreg(vmx, vcpuid, VM_REG_GUEST_RCX, &val); 1906 KASSERT(!error, ("%s: vmx_getreg error %d", __func__, error)); 1907 } else { 1908 val = 1; 1909 } 1910 return (val); 1911 } 1912 1913 static int 1914 inout_str_addrsize(uint32_t inst_info) 1915 { 1916 uint32_t size; 1917 1918 size = (inst_info >> 7) & 0x7; 1919 switch (size) { 1920 case 0: 1921 return (2); /* 16 bit */ 1922 case 1: 1923 return (4); /* 32 bit */ 1924 case 2: 1925 return (8); /* 64 bit */ 1926 default: 1927 panic("%s: invalid size encoding %d", __func__, size); 1928 } 1929 } 1930 1931 static void 1932 inout_str_seginfo(struct vmx *vmx, int vcpuid, uint32_t inst_info, int in, 1933 struct vm_inout_str *vis) 1934 { 1935 int error, s; 1936 1937 if (in) { 1938 vis->seg_name = VM_REG_GUEST_ES; 1939 } else { 1940 s = (inst_info >> 15) & 0x7; 1941 vis->seg_name = vm_segment_name(s); 1942 } 1943 1944 error = vmx_getdesc(vmx, vcpuid, vis->seg_name, &vis->seg_desc); 1945 KASSERT(error == 0, ("%s: vmx_getdesc error %d", __func__, error)); 1946 } 1947 1948 static void 1949 vmx_paging_info(struct vm_guest_paging *paging) 1950 { 1951 paging->cr3 = vmcs_guest_cr3(); 1952 paging->cpl = vmx_cpl(); 1953 paging->cpu_mode = vmx_cpu_mode(); 1954 paging->paging_mode = vmx_paging_mode(); 1955 } 1956 1957 static void 1958 vmexit_inst_emul(struct vm_exit *vmexit, uint64_t gpa, uint64_t gla) 1959 { 1960 struct vm_guest_paging *paging; 1961 uint32_t csar; 1962 1963 paging = &vmexit->u.inst_emul.paging; 1964 1965 vmexit->exitcode = VM_EXITCODE_INST_EMUL; 1966 vmexit->inst_length = 0; 1967 vmexit->u.inst_emul.gpa = gpa; 1968 vmexit->u.inst_emul.gla = gla; 1969 vmx_paging_info(paging); 1970 switch (paging->cpu_mode) { 1971 case CPU_MODE_REAL: 1972 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 1973 vmexit->u.inst_emul.cs_d = 0; 1974 break; 1975 case CPU_MODE_PROTECTED: 1976 case CPU_MODE_COMPATIBILITY: 1977 vmexit->u.inst_emul.cs_base = vmcs_read(VMCS_GUEST_CS_BASE); 1978 csar = vmcs_read(VMCS_GUEST_CS_ACCESS_RIGHTS); 1979 vmexit->u.inst_emul.cs_d = SEG_DESC_DEF32(csar); 1980 break; 1981 default: 1982 vmexit->u.inst_emul.cs_base = 0; 1983 vmexit->u.inst_emul.cs_d = 0; 1984 break; 1985 } 1986 vie_init(&vmexit->u.inst_emul.vie, NULL, 0); 1987 } 1988 1989 static int 1990 ept_fault_type(uint64_t ept_qual) 1991 { 1992 int fault_type; 1993 1994 if (ept_qual & EPT_VIOLATION_DATA_WRITE) 1995 fault_type = VM_PROT_WRITE; 1996 else if (ept_qual & EPT_VIOLATION_INST_FETCH) 1997 fault_type = VM_PROT_EXECUTE; 1998 else 1999 fault_type= VM_PROT_READ; 2000 2001 return (fault_type); 2002 } 2003 2004 static bool 2005 ept_emulation_fault(uint64_t ept_qual) 2006 { 2007 int read, write; 2008 2009 /* EPT fault on an instruction fetch doesn't make sense here */ 2010 if (ept_qual & EPT_VIOLATION_INST_FETCH) 2011 return (false); 2012 2013 /* EPT fault must be a read fault or a write fault */ 2014 read = ept_qual & EPT_VIOLATION_DATA_READ ? 1 : 0; 2015 write = ept_qual & EPT_VIOLATION_DATA_WRITE ? 1 : 0; 2016 if ((read | write) == 0) 2017 return (false); 2018 2019 /* 2020 * The EPT violation must have been caused by accessing a 2021 * guest-physical address that is a translation of a guest-linear 2022 * address. 2023 */ 2024 if ((ept_qual & EPT_VIOLATION_GLA_VALID) == 0 || 2025 (ept_qual & EPT_VIOLATION_XLAT_VALID) == 0) { 2026 return (false); 2027 } 2028 2029 return (true); 2030 } 2031 2032 static __inline int 2033 apic_access_virtualization(struct vmx *vmx, int vcpuid) 2034 { 2035 uint32_t proc_ctls2; 2036 2037 proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 2038 return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) ? 1 : 0); 2039 } 2040 2041 static __inline int 2042 x2apic_virtualization(struct vmx *vmx, int vcpuid) 2043 { 2044 uint32_t proc_ctls2; 2045 2046 proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 2047 return ((proc_ctls2 & PROCBASED2_VIRTUALIZE_X2APIC_MODE) ? 1 : 0); 2048 } 2049 2050 static int 2051 vmx_handle_apic_write(struct vmx *vmx, int vcpuid, struct vlapic *vlapic, 2052 uint64_t qual) 2053 { 2054 int error, handled, offset; 2055 uint32_t *apic_regs, vector; 2056 bool retu; 2057 2058 handled = HANDLED; 2059 offset = APIC_WRITE_OFFSET(qual); 2060 2061 if (!apic_access_virtualization(vmx, vcpuid)) { 2062 /* 2063 * In general there should not be any APIC write VM-exits 2064 * unless APIC-access virtualization is enabled. 2065 * 2066 * However self-IPI virtualization can legitimately trigger 2067 * an APIC-write VM-exit so treat it specially. 2068 */ 2069 if (x2apic_virtualization(vmx, vcpuid) && 2070 offset == APIC_OFFSET_SELF_IPI) { 2071 apic_regs = (uint32_t *)(vlapic->apic_page); 2072 vector = apic_regs[APIC_OFFSET_SELF_IPI / 4]; 2073 vlapic_self_ipi_handler(vlapic, vector); 2074 return (HANDLED); 2075 } else 2076 return (UNHANDLED); 2077 } 2078 2079 switch (offset) { 2080 case APIC_OFFSET_ID: 2081 vlapic_id_write_handler(vlapic); 2082 break; 2083 case APIC_OFFSET_LDR: 2084 vlapic_ldr_write_handler(vlapic); 2085 break; 2086 case APIC_OFFSET_DFR: 2087 vlapic_dfr_write_handler(vlapic); 2088 break; 2089 case APIC_OFFSET_SVR: 2090 vlapic_svr_write_handler(vlapic); 2091 break; 2092 case APIC_OFFSET_ESR: 2093 vlapic_esr_write_handler(vlapic); 2094 break; 2095 case APIC_OFFSET_ICR_LOW: 2096 retu = false; 2097 error = vlapic_icrlo_write_handler(vlapic, &retu); 2098 if (error != 0 || retu) 2099 handled = UNHANDLED; 2100 break; 2101 case APIC_OFFSET_CMCI_LVT: 2102 case APIC_OFFSET_TIMER_LVT ... APIC_OFFSET_ERROR_LVT: 2103 vlapic_lvt_write_handler(vlapic, offset); 2104 break; 2105 case APIC_OFFSET_TIMER_ICR: 2106 vlapic_icrtmr_write_handler(vlapic); 2107 break; 2108 case APIC_OFFSET_TIMER_DCR: 2109 vlapic_dcr_write_handler(vlapic); 2110 break; 2111 default: 2112 handled = UNHANDLED; 2113 break; 2114 } 2115 return (handled); 2116 } 2117 2118 static bool 2119 apic_access_fault(struct vmx *vmx, int vcpuid, uint64_t gpa) 2120 { 2121 2122 if (apic_access_virtualization(vmx, vcpuid) && 2123 (gpa >= DEFAULT_APIC_BASE && gpa < DEFAULT_APIC_BASE + PAGE_SIZE)) 2124 return (true); 2125 else 2126 return (false); 2127 } 2128 2129 static int 2130 vmx_handle_apic_access(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 2131 { 2132 uint64_t qual; 2133 int access_type, offset, allowed; 2134 2135 if (!apic_access_virtualization(vmx, vcpuid)) 2136 return (UNHANDLED); 2137 2138 qual = vmexit->u.vmx.exit_qualification; 2139 access_type = APIC_ACCESS_TYPE(qual); 2140 offset = APIC_ACCESS_OFFSET(qual); 2141 2142 allowed = 0; 2143 if (access_type == 0) { 2144 /* 2145 * Read data access to the following registers is expected. 2146 */ 2147 switch (offset) { 2148 case APIC_OFFSET_APR: 2149 case APIC_OFFSET_PPR: 2150 case APIC_OFFSET_RRR: 2151 case APIC_OFFSET_CMCI_LVT: 2152 case APIC_OFFSET_TIMER_CCR: 2153 allowed = 1; 2154 break; 2155 default: 2156 break; 2157 } 2158 } else if (access_type == 1) { 2159 /* 2160 * Write data access to the following registers is expected. 2161 */ 2162 switch (offset) { 2163 case APIC_OFFSET_VER: 2164 case APIC_OFFSET_APR: 2165 case APIC_OFFSET_PPR: 2166 case APIC_OFFSET_RRR: 2167 case APIC_OFFSET_ISR0 ... APIC_OFFSET_ISR7: 2168 case APIC_OFFSET_TMR0 ... APIC_OFFSET_TMR7: 2169 case APIC_OFFSET_IRR0 ... APIC_OFFSET_IRR7: 2170 case APIC_OFFSET_CMCI_LVT: 2171 case APIC_OFFSET_TIMER_CCR: 2172 allowed = 1; 2173 break; 2174 default: 2175 break; 2176 } 2177 } 2178 2179 if (allowed) { 2180 vmexit_inst_emul(vmexit, DEFAULT_APIC_BASE + offset, 2181 VIE_INVALID_GLA); 2182 } 2183 2184 /* 2185 * Regardless of whether the APIC-access is allowed this handler 2186 * always returns UNHANDLED: 2187 * - if the access is allowed then it is handled by emulating the 2188 * instruction that caused the VM-exit (outside the critical section) 2189 * - if the access is not allowed then it will be converted to an 2190 * exitcode of VM_EXITCODE_VMX and will be dealt with in userland. 2191 */ 2192 return (UNHANDLED); 2193 } 2194 2195 static enum task_switch_reason 2196 vmx_task_switch_reason(uint64_t qual) 2197 { 2198 int reason; 2199 2200 reason = (qual >> 30) & 0x3; 2201 switch (reason) { 2202 case 0: 2203 return (TSR_CALL); 2204 case 1: 2205 return (TSR_IRET); 2206 case 2: 2207 return (TSR_JMP); 2208 case 3: 2209 return (TSR_IDT_GATE); 2210 default: 2211 panic("%s: invalid reason %d", __func__, reason); 2212 } 2213 } 2214 2215 static int 2216 emulate_wrmsr(struct vmx *vmx, int vcpuid, u_int num, uint64_t val, bool *retu) 2217 { 2218 int error; 2219 2220 if (lapic_msr(num)) 2221 error = lapic_wrmsr(vmx->vm, vcpuid, num, val, retu); 2222 else 2223 error = vmx_wrmsr(vmx, vcpuid, num, val, retu); 2224 2225 return (error); 2226 } 2227 2228 static int 2229 emulate_rdmsr(struct vmx *vmx, int vcpuid, u_int num, bool *retu) 2230 { 2231 struct vmxctx *vmxctx; 2232 uint64_t result; 2233 uint32_t eax, edx; 2234 int error; 2235 2236 if (lapic_msr(num)) 2237 error = lapic_rdmsr(vmx->vm, vcpuid, num, &result, retu); 2238 else 2239 error = vmx_rdmsr(vmx, vcpuid, num, &result, retu); 2240 2241 if (error == 0) { 2242 eax = result; 2243 vmxctx = &vmx->ctx[vcpuid]; 2244 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RAX, eax); 2245 KASSERT(error == 0, ("vmxctx_setreg(rax) error %d", error)); 2246 2247 edx = result >> 32; 2248 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_RDX, edx); 2249 KASSERT(error == 0, ("vmxctx_setreg(rdx) error %d", error)); 2250 } 2251 2252 return (error); 2253 } 2254 2255 static int 2256 vmx_exit_process(struct vmx *vmx, int vcpu, struct vm_exit *vmexit) 2257 { 2258 int error, errcode, errcode_valid, handled, in; 2259 struct vmxctx *vmxctx; 2260 struct vlapic *vlapic; 2261 struct vm_inout_str *vis; 2262 struct vm_task_switch *ts; 2263 uint32_t eax, ecx, edx, idtvec_info, idtvec_err, intr_info, inst_info; 2264 uint32_t intr_type, intr_vec, reason; 2265 uint64_t exitintinfo, qual, gpa; 2266 bool retu; 2267 2268 CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_VIRTUAL_NMI) != 0); 2269 CTASSERT((PINBASED_CTLS_ONE_SETTING & PINBASED_NMI_EXITING) != 0); 2270 2271 handled = UNHANDLED; 2272 vmxctx = &vmx->ctx[vcpu]; 2273 2274 qual = vmexit->u.vmx.exit_qualification; 2275 reason = vmexit->u.vmx.exit_reason; 2276 vmexit->exitcode = VM_EXITCODE_BOGUS; 2277 2278 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_COUNT, 1); 2279 SDT_PROBE3(vmm, vmx, exit, entry, vmx, vcpu, vmexit); 2280 2281 /* 2282 * VM-entry failures during or after loading guest state. 2283 * 2284 * These VM-exits are uncommon but must be handled specially 2285 * as most VM-exit fields are not populated as usual. 2286 */ 2287 if (__predict_false(reason == EXIT_REASON_MCE_DURING_ENTRY)) { 2288 VCPU_CTR0(vmx->vm, vcpu, "Handling MCE during VM-entry"); 2289 __asm __volatile("int $18"); 2290 return (1); 2291 } 2292 2293 /* 2294 * VM exits that can be triggered during event delivery need to 2295 * be handled specially by re-injecting the event if the IDT 2296 * vectoring information field's valid bit is set. 2297 * 2298 * See "Information for VM Exits During Event Delivery" in Intel SDM 2299 * for details. 2300 */ 2301 idtvec_info = vmcs_idt_vectoring_info(); 2302 if (idtvec_info & VMCS_IDT_VEC_VALID) { 2303 idtvec_info &= ~(1 << 12); /* clear undefined bit */ 2304 exitintinfo = idtvec_info; 2305 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 2306 idtvec_err = vmcs_idt_vectoring_err(); 2307 exitintinfo |= (uint64_t)idtvec_err << 32; 2308 } 2309 error = vm_exit_intinfo(vmx->vm, vcpu, exitintinfo); 2310 KASSERT(error == 0, ("%s: vm_set_intinfo error %d", 2311 __func__, error)); 2312 2313 /* 2314 * If 'virtual NMIs' are being used and the VM-exit 2315 * happened while injecting an NMI during the previous 2316 * VM-entry, then clear "blocking by NMI" in the 2317 * Guest Interruptibility-State so the NMI can be 2318 * reinjected on the subsequent VM-entry. 2319 * 2320 * However, if the NMI was being delivered through a task 2321 * gate, then the new task must start execution with NMIs 2322 * blocked so don't clear NMI blocking in this case. 2323 */ 2324 intr_type = idtvec_info & VMCS_INTR_T_MASK; 2325 if (intr_type == VMCS_INTR_T_NMI) { 2326 if (reason != EXIT_REASON_TASK_SWITCH) 2327 vmx_clear_nmi_blocking(vmx, vcpu); 2328 else 2329 vmx_assert_nmi_blocking(vmx, vcpu); 2330 } 2331 2332 /* 2333 * Update VM-entry instruction length if the event being 2334 * delivered was a software interrupt or software exception. 2335 */ 2336 if (intr_type == VMCS_INTR_T_SWINTR || 2337 intr_type == VMCS_INTR_T_PRIV_SWEXCEPTION || 2338 intr_type == VMCS_INTR_T_SWEXCEPTION) { 2339 vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2340 } 2341 } 2342 2343 switch (reason) { 2344 case EXIT_REASON_TASK_SWITCH: 2345 ts = &vmexit->u.task_switch; 2346 ts->tsssel = qual & 0xffff; 2347 ts->reason = vmx_task_switch_reason(qual); 2348 ts->ext = 0; 2349 ts->errcode_valid = 0; 2350 vmx_paging_info(&ts->paging); 2351 /* 2352 * If the task switch was due to a CALL, JMP, IRET, software 2353 * interrupt (INT n) or software exception (INT3, INTO), 2354 * then the saved %rip references the instruction that caused 2355 * the task switch. The instruction length field in the VMCS 2356 * is valid in this case. 2357 * 2358 * In all other cases (e.g., NMI, hardware exception) the 2359 * saved %rip is one that would have been saved in the old TSS 2360 * had the task switch completed normally so the instruction 2361 * length field is not needed in this case and is explicitly 2362 * set to 0. 2363 */ 2364 if (ts->reason == TSR_IDT_GATE) { 2365 KASSERT(idtvec_info & VMCS_IDT_VEC_VALID, 2366 ("invalid idtvec_info %#x for IDT task switch", 2367 idtvec_info)); 2368 intr_type = idtvec_info & VMCS_INTR_T_MASK; 2369 if (intr_type != VMCS_INTR_T_SWINTR && 2370 intr_type != VMCS_INTR_T_SWEXCEPTION && 2371 intr_type != VMCS_INTR_T_PRIV_SWEXCEPTION) { 2372 /* Task switch triggered by external event */ 2373 ts->ext = 1; 2374 vmexit->inst_length = 0; 2375 if (idtvec_info & VMCS_IDT_VEC_ERRCODE_VALID) { 2376 ts->errcode_valid = 1; 2377 ts->errcode = vmcs_idt_vectoring_err(); 2378 } 2379 } 2380 } 2381 vmexit->exitcode = VM_EXITCODE_TASK_SWITCH; 2382 SDT_PROBE4(vmm, vmx, exit, taskswitch, vmx, vcpu, vmexit, ts); 2383 VCPU_CTR4(vmx->vm, vcpu, "task switch reason %d, tss 0x%04x, " 2384 "%s errcode 0x%016lx", ts->reason, ts->tsssel, 2385 ts->ext ? "external" : "internal", 2386 ((uint64_t)ts->errcode << 32) | ts->errcode_valid); 2387 break; 2388 case EXIT_REASON_CR_ACCESS: 2389 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CR_ACCESS, 1); 2390 SDT_PROBE4(vmm, vmx, exit, craccess, vmx, vcpu, vmexit, qual); 2391 switch (qual & 0xf) { 2392 case 0: 2393 handled = vmx_emulate_cr0_access(vmx, vcpu, qual); 2394 break; 2395 case 4: 2396 handled = vmx_emulate_cr4_access(vmx, vcpu, qual); 2397 break; 2398 case 8: 2399 handled = vmx_emulate_cr8_access(vmx, vcpu, qual); 2400 break; 2401 } 2402 break; 2403 case EXIT_REASON_RDMSR: 2404 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_RDMSR, 1); 2405 retu = false; 2406 ecx = vmxctx->guest_rcx; 2407 VCPU_CTR1(vmx->vm, vcpu, "rdmsr 0x%08x", ecx); 2408 SDT_PROBE4(vmm, vmx, exit, rdmsr, vmx, vcpu, vmexit, ecx); 2409 error = emulate_rdmsr(vmx, vcpu, ecx, &retu); 2410 if (error) { 2411 vmexit->exitcode = VM_EXITCODE_RDMSR; 2412 vmexit->u.msr.code = ecx; 2413 } else if (!retu) { 2414 handled = HANDLED; 2415 } else { 2416 /* Return to userspace with a valid exitcode */ 2417 KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2418 ("emulate_rdmsr retu with bogus exitcode")); 2419 } 2420 break; 2421 case EXIT_REASON_WRMSR: 2422 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_WRMSR, 1); 2423 retu = false; 2424 eax = vmxctx->guest_rax; 2425 ecx = vmxctx->guest_rcx; 2426 edx = vmxctx->guest_rdx; 2427 VCPU_CTR2(vmx->vm, vcpu, "wrmsr 0x%08x value 0x%016lx", 2428 ecx, (uint64_t)edx << 32 | eax); 2429 SDT_PROBE5(vmm, vmx, exit, wrmsr, vmx, vmexit, vcpu, ecx, 2430 (uint64_t)edx << 32 | eax); 2431 error = emulate_wrmsr(vmx, vcpu, ecx, 2432 (uint64_t)edx << 32 | eax, &retu); 2433 if (error) { 2434 vmexit->exitcode = VM_EXITCODE_WRMSR; 2435 vmexit->u.msr.code = ecx; 2436 vmexit->u.msr.wval = (uint64_t)edx << 32 | eax; 2437 } else if (!retu) { 2438 handled = HANDLED; 2439 } else { 2440 /* Return to userspace with a valid exitcode */ 2441 KASSERT(vmexit->exitcode != VM_EXITCODE_BOGUS, 2442 ("emulate_wrmsr retu with bogus exitcode")); 2443 } 2444 break; 2445 case EXIT_REASON_HLT: 2446 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_HLT, 1); 2447 SDT_PROBE3(vmm, vmx, exit, halt, vmx, vcpu, vmexit); 2448 vmexit->exitcode = VM_EXITCODE_HLT; 2449 vmexit->u.hlt.rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2450 if (virtual_interrupt_delivery) 2451 vmexit->u.hlt.intr_status = 2452 vmcs_read(VMCS_GUEST_INTR_STATUS); 2453 else 2454 vmexit->u.hlt.intr_status = 0; 2455 break; 2456 case EXIT_REASON_MTF: 2457 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_MTRAP, 1); 2458 SDT_PROBE3(vmm, vmx, exit, mtrap, vmx, vcpu, vmexit); 2459 vmexit->exitcode = VM_EXITCODE_MTRAP; 2460 vmexit->inst_length = 0; 2461 break; 2462 case EXIT_REASON_PAUSE: 2463 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_PAUSE, 1); 2464 SDT_PROBE3(vmm, vmx, exit, pause, vmx, vcpu, vmexit); 2465 vmexit->exitcode = VM_EXITCODE_PAUSE; 2466 break; 2467 case EXIT_REASON_INTR_WINDOW: 2468 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INTR_WINDOW, 1); 2469 SDT_PROBE3(vmm, vmx, exit, intrwindow, vmx, vcpu, vmexit); 2470 vmx_clear_int_window_exiting(vmx, vcpu); 2471 return (1); 2472 case EXIT_REASON_EXT_INTR: 2473 /* 2474 * External interrupts serve only to cause VM exits and allow 2475 * the host interrupt handler to run. 2476 * 2477 * If this external interrupt triggers a virtual interrupt 2478 * to a VM, then that state will be recorded by the 2479 * host interrupt handler in the VM's softc. We will inject 2480 * this virtual interrupt during the subsequent VM enter. 2481 */ 2482 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 2483 SDT_PROBE4(vmm, vmx, exit, interrupt, 2484 vmx, vcpu, vmexit, intr_info); 2485 2486 /* 2487 * XXX: Ignore this exit if VMCS_INTR_VALID is not set. 2488 * This appears to be a bug in VMware Fusion? 2489 */ 2490 if (!(intr_info & VMCS_INTR_VALID)) 2491 return (1); 2492 KASSERT((intr_info & VMCS_INTR_VALID) != 0 && 2493 (intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_HWINTR, 2494 ("VM exit interruption info invalid: %#x", intr_info)); 2495 vmx_trigger_hostintr(intr_info & 0xff); 2496 2497 /* 2498 * This is special. We want to treat this as an 'handled' 2499 * VM-exit but not increment the instruction pointer. 2500 */ 2501 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXTINT, 1); 2502 return (1); 2503 case EXIT_REASON_NMI_WINDOW: 2504 SDT_PROBE3(vmm, vmx, exit, nmiwindow, vmx, vcpu, vmexit); 2505 /* Exit to allow the pending virtual NMI to be injected */ 2506 if (vm_nmi_pending(vmx->vm, vcpu)) 2507 vmx_inject_nmi(vmx, vcpu); 2508 vmx_clear_nmi_window_exiting(vmx, vcpu); 2509 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NMI_WINDOW, 1); 2510 return (1); 2511 case EXIT_REASON_INOUT: 2512 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INOUT, 1); 2513 vmexit->exitcode = VM_EXITCODE_INOUT; 2514 vmexit->u.inout.bytes = (qual & 0x7) + 1; 2515 vmexit->u.inout.in = in = (qual & 0x8) ? 1 : 0; 2516 vmexit->u.inout.string = (qual & 0x10) ? 1 : 0; 2517 vmexit->u.inout.rep = (qual & 0x20) ? 1 : 0; 2518 vmexit->u.inout.port = (uint16_t)(qual >> 16); 2519 vmexit->u.inout.eax = (uint32_t)(vmxctx->guest_rax); 2520 if (vmexit->u.inout.string) { 2521 inst_info = vmcs_read(VMCS_EXIT_INSTRUCTION_INFO); 2522 vmexit->exitcode = VM_EXITCODE_INOUT_STR; 2523 vis = &vmexit->u.inout_str; 2524 vmx_paging_info(&vis->paging); 2525 vis->rflags = vmcs_read(VMCS_GUEST_RFLAGS); 2526 vis->cr0 = vmcs_read(VMCS_GUEST_CR0); 2527 vis->index = inout_str_index(vmx, vcpu, in); 2528 vis->count = inout_str_count(vmx, vcpu, vis->inout.rep); 2529 vis->addrsize = inout_str_addrsize(inst_info); 2530 inout_str_seginfo(vmx, vcpu, inst_info, in, vis); 2531 } 2532 SDT_PROBE3(vmm, vmx, exit, inout, vmx, vcpu, vmexit); 2533 break; 2534 case EXIT_REASON_CPUID: 2535 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_CPUID, 1); 2536 SDT_PROBE3(vmm, vmx, exit, cpuid, vmx, vcpu, vmexit); 2537 handled = vmx_handle_cpuid(vmx->vm, vcpu, vmxctx); 2538 break; 2539 case EXIT_REASON_EXCEPTION: 2540 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_EXCEPTION, 1); 2541 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 2542 KASSERT((intr_info & VMCS_INTR_VALID) != 0, 2543 ("VM exit interruption info invalid: %#x", intr_info)); 2544 2545 intr_vec = intr_info & 0xff; 2546 intr_type = intr_info & VMCS_INTR_T_MASK; 2547 2548 /* 2549 * If Virtual NMIs control is 1 and the VM-exit is due to a 2550 * fault encountered during the execution of IRET then we must 2551 * restore the state of "virtual-NMI blocking" before resuming 2552 * the guest. 2553 * 2554 * See "Resuming Guest Software after Handling an Exception". 2555 * See "Information for VM Exits Due to Vectored Events". 2556 */ 2557 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2558 (intr_vec != IDT_DF) && 2559 (intr_info & EXIT_QUAL_NMIUDTI) != 0) 2560 vmx_restore_nmi_blocking(vmx, vcpu); 2561 2562 /* 2563 * The NMI has already been handled in vmx_exit_handle_nmi(). 2564 */ 2565 if (intr_type == VMCS_INTR_T_NMI) 2566 return (1); 2567 2568 /* 2569 * Call the machine check handler by hand. Also don't reflect 2570 * the machine check back into the guest. 2571 */ 2572 if (intr_vec == IDT_MC) { 2573 VCPU_CTR0(vmx->vm, vcpu, "Vectoring to MCE handler"); 2574 __asm __volatile("int $18"); 2575 return (1); 2576 } 2577 2578 /* 2579 * If the hypervisor has requested user exits for 2580 * debug exceptions, bounce them out to userland. 2581 */ 2582 if (intr_type == VMCS_INTR_T_SWEXCEPTION && intr_vec == IDT_BP && 2583 (vmx->cap[vcpu].set & (1 << VM_CAP_BPT_EXIT))) { 2584 vmexit->exitcode = VM_EXITCODE_BPT; 2585 vmexit->u.bpt.inst_length = vmexit->inst_length; 2586 vmexit->inst_length = 0; 2587 break; 2588 } 2589 2590 if (intr_vec == IDT_PF) { 2591 error = vmxctx_setreg(vmxctx, VM_REG_GUEST_CR2, qual); 2592 KASSERT(error == 0, ("%s: vmxctx_setreg(cr2) error %d", 2593 __func__, error)); 2594 } 2595 2596 /* 2597 * Software exceptions exhibit trap-like behavior. This in 2598 * turn requires populating the VM-entry instruction length 2599 * so that the %rip in the trap frame is past the INT3/INTO 2600 * instruction. 2601 */ 2602 if (intr_type == VMCS_INTR_T_SWEXCEPTION) 2603 vmcs_write(VMCS_ENTRY_INST_LENGTH, vmexit->inst_length); 2604 2605 /* Reflect all other exceptions back into the guest */ 2606 errcode_valid = errcode = 0; 2607 if (intr_info & VMCS_INTR_DEL_ERRCODE) { 2608 errcode_valid = 1; 2609 errcode = vmcs_read(VMCS_EXIT_INTR_ERRCODE); 2610 } 2611 VCPU_CTR2(vmx->vm, vcpu, "Reflecting exception %d/%#x into " 2612 "the guest", intr_vec, errcode); 2613 SDT_PROBE5(vmm, vmx, exit, exception, 2614 vmx, vcpu, vmexit, intr_vec, errcode); 2615 error = vm_inject_exception(vmx->vm, vcpu, intr_vec, 2616 errcode_valid, errcode, 0); 2617 KASSERT(error == 0, ("%s: vm_inject_exception error %d", 2618 __func__, error)); 2619 return (1); 2620 2621 case EXIT_REASON_EPT_FAULT: 2622 /* 2623 * If 'gpa' lies within the address space allocated to 2624 * memory then this must be a nested page fault otherwise 2625 * this must be an instruction that accesses MMIO space. 2626 */ 2627 gpa = vmcs_gpa(); 2628 if (vm_mem_allocated(vmx->vm, vcpu, gpa) || 2629 apic_access_fault(vmx, vcpu, gpa)) { 2630 vmexit->exitcode = VM_EXITCODE_PAGING; 2631 vmexit->inst_length = 0; 2632 vmexit->u.paging.gpa = gpa; 2633 vmexit->u.paging.fault_type = ept_fault_type(qual); 2634 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_NESTED_FAULT, 1); 2635 SDT_PROBE5(vmm, vmx, exit, nestedfault, 2636 vmx, vcpu, vmexit, gpa, qual); 2637 } else if (ept_emulation_fault(qual)) { 2638 vmexit_inst_emul(vmexit, gpa, vmcs_gla()); 2639 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_INST_EMUL, 1); 2640 SDT_PROBE4(vmm, vmx, exit, mmiofault, 2641 vmx, vcpu, vmexit, gpa); 2642 } 2643 /* 2644 * If Virtual NMIs control is 1 and the VM-exit is due to an 2645 * EPT fault during the execution of IRET then we must restore 2646 * the state of "virtual-NMI blocking" before resuming. 2647 * 2648 * See description of "NMI unblocking due to IRET" in 2649 * "Exit Qualification for EPT Violations". 2650 */ 2651 if ((idtvec_info & VMCS_IDT_VEC_VALID) == 0 && 2652 (qual & EXIT_QUAL_NMIUDTI) != 0) 2653 vmx_restore_nmi_blocking(vmx, vcpu); 2654 break; 2655 case EXIT_REASON_VIRTUALIZED_EOI: 2656 vmexit->exitcode = VM_EXITCODE_IOAPIC_EOI; 2657 vmexit->u.ioapic_eoi.vector = qual & 0xFF; 2658 SDT_PROBE3(vmm, vmx, exit, eoi, vmx, vcpu, vmexit); 2659 vmexit->inst_length = 0; /* trap-like */ 2660 break; 2661 case EXIT_REASON_APIC_ACCESS: 2662 SDT_PROBE3(vmm, vmx, exit, apicaccess, vmx, vcpu, vmexit); 2663 handled = vmx_handle_apic_access(vmx, vcpu, vmexit); 2664 break; 2665 case EXIT_REASON_APIC_WRITE: 2666 /* 2667 * APIC-write VM exit is trap-like so the %rip is already 2668 * pointing to the next instruction. 2669 */ 2670 vmexit->inst_length = 0; 2671 vlapic = vm_lapic(vmx->vm, vcpu); 2672 SDT_PROBE4(vmm, vmx, exit, apicwrite, 2673 vmx, vcpu, vmexit, vlapic); 2674 handled = vmx_handle_apic_write(vmx, vcpu, vlapic, qual); 2675 break; 2676 case EXIT_REASON_XSETBV: 2677 SDT_PROBE3(vmm, vmx, exit, xsetbv, vmx, vcpu, vmexit); 2678 handled = vmx_emulate_xsetbv(vmx, vcpu, vmexit); 2679 break; 2680 case EXIT_REASON_MONITOR: 2681 SDT_PROBE3(vmm, vmx, exit, monitor, vmx, vcpu, vmexit); 2682 vmexit->exitcode = VM_EXITCODE_MONITOR; 2683 break; 2684 case EXIT_REASON_MWAIT: 2685 SDT_PROBE3(vmm, vmx, exit, mwait, vmx, vcpu, vmexit); 2686 vmexit->exitcode = VM_EXITCODE_MWAIT; 2687 break; 2688 case EXIT_REASON_TPR: 2689 vlapic = vm_lapic(vmx->vm, vcpu); 2690 vlapic_sync_tpr(vlapic); 2691 vmexit->inst_length = 0; 2692 handled = HANDLED; 2693 break; 2694 case EXIT_REASON_VMCALL: 2695 case EXIT_REASON_VMCLEAR: 2696 case EXIT_REASON_VMLAUNCH: 2697 case EXIT_REASON_VMPTRLD: 2698 case EXIT_REASON_VMPTRST: 2699 case EXIT_REASON_VMREAD: 2700 case EXIT_REASON_VMRESUME: 2701 case EXIT_REASON_VMWRITE: 2702 case EXIT_REASON_VMXOFF: 2703 case EXIT_REASON_VMXON: 2704 SDT_PROBE3(vmm, vmx, exit, vminsn, vmx, vcpu, vmexit); 2705 vmexit->exitcode = VM_EXITCODE_VMINSN; 2706 break; 2707 default: 2708 SDT_PROBE4(vmm, vmx, exit, unknown, 2709 vmx, vcpu, vmexit, reason); 2710 vmm_stat_incr(vmx->vm, vcpu, VMEXIT_UNKNOWN, 1); 2711 break; 2712 } 2713 2714 if (handled) { 2715 /* 2716 * It is possible that control is returned to userland 2717 * even though we were able to handle the VM exit in the 2718 * kernel. 2719 * 2720 * In such a case we want to make sure that the userland 2721 * restarts guest execution at the instruction *after* 2722 * the one we just processed. Therefore we update the 2723 * guest rip in the VMCS and in 'vmexit'. 2724 */ 2725 vmexit->rip += vmexit->inst_length; 2726 vmexit->inst_length = 0; 2727 vmcs_write(VMCS_GUEST_RIP, vmexit->rip); 2728 } else { 2729 if (vmexit->exitcode == VM_EXITCODE_BOGUS) { 2730 /* 2731 * If this VM exit was not claimed by anybody then 2732 * treat it as a generic VMX exit. 2733 */ 2734 vmexit->exitcode = VM_EXITCODE_VMX; 2735 vmexit->u.vmx.status = VM_SUCCESS; 2736 vmexit->u.vmx.inst_type = 0; 2737 vmexit->u.vmx.inst_error = 0; 2738 } else { 2739 /* 2740 * The exitcode and collateral have been populated. 2741 * The VM exit will be processed further in userland. 2742 */ 2743 } 2744 } 2745 2746 SDT_PROBE4(vmm, vmx, exit, return, 2747 vmx, vcpu, vmexit, handled); 2748 return (handled); 2749 } 2750 2751 static __inline void 2752 vmx_exit_inst_error(struct vmxctx *vmxctx, int rc, struct vm_exit *vmexit) 2753 { 2754 2755 KASSERT(vmxctx->inst_fail_status != VM_SUCCESS, 2756 ("vmx_exit_inst_error: invalid inst_fail_status %d", 2757 vmxctx->inst_fail_status)); 2758 2759 vmexit->inst_length = 0; 2760 vmexit->exitcode = VM_EXITCODE_VMX; 2761 vmexit->u.vmx.status = vmxctx->inst_fail_status; 2762 vmexit->u.vmx.inst_error = vmcs_instruction_error(); 2763 vmexit->u.vmx.exit_reason = ~0; 2764 vmexit->u.vmx.exit_qualification = ~0; 2765 2766 switch (rc) { 2767 case VMX_VMRESUME_ERROR: 2768 case VMX_VMLAUNCH_ERROR: 2769 case VMX_INVEPT_ERROR: 2770 vmexit->u.vmx.inst_type = rc; 2771 break; 2772 default: 2773 panic("vm_exit_inst_error: vmx_enter_guest returned %d", rc); 2774 } 2775 } 2776 2777 /* 2778 * If the NMI-exiting VM execution control is set to '1' then an NMI in 2779 * non-root operation causes a VM-exit. NMI blocking is in effect so it is 2780 * sufficient to simply vector to the NMI handler via a software interrupt. 2781 * However, this must be done before maskable interrupts are enabled 2782 * otherwise the "iret" issued by an interrupt handler will incorrectly 2783 * clear NMI blocking. 2784 */ 2785 static __inline void 2786 vmx_exit_handle_nmi(struct vmx *vmx, int vcpuid, struct vm_exit *vmexit) 2787 { 2788 uint32_t intr_info; 2789 2790 KASSERT((read_rflags() & PSL_I) == 0, ("interrupts enabled")); 2791 2792 if (vmexit->u.vmx.exit_reason != EXIT_REASON_EXCEPTION) 2793 return; 2794 2795 intr_info = vmcs_read(VMCS_EXIT_INTR_INFO); 2796 KASSERT((intr_info & VMCS_INTR_VALID) != 0, 2797 ("VM exit interruption info invalid: %#x", intr_info)); 2798 2799 if ((intr_info & VMCS_INTR_T_MASK) == VMCS_INTR_T_NMI) { 2800 KASSERT((intr_info & 0xff) == IDT_NMI, ("VM exit due " 2801 "to NMI has invalid vector: %#x", intr_info)); 2802 VCPU_CTR0(vmx->vm, vcpuid, "Vectoring to NMI handler"); 2803 __asm __volatile("int $2"); 2804 } 2805 } 2806 2807 static __inline void 2808 vmx_dr_enter_guest(struct vmxctx *vmxctx) 2809 { 2810 register_t rflags; 2811 2812 /* Save host control debug registers. */ 2813 vmxctx->host_dr7 = rdr7(); 2814 vmxctx->host_debugctl = rdmsr(MSR_DEBUGCTLMSR); 2815 2816 /* 2817 * Disable debugging in DR7 and DEBUGCTL to avoid triggering 2818 * exceptions in the host based on the guest DRx values. The 2819 * guest DR7 and DEBUGCTL are saved/restored in the VMCS. 2820 */ 2821 load_dr7(0); 2822 wrmsr(MSR_DEBUGCTLMSR, 0); 2823 2824 /* 2825 * Disable single stepping the kernel to avoid corrupting the 2826 * guest DR6. A debugger might still be able to corrupt the 2827 * guest DR6 by setting a breakpoint after this point and then 2828 * single stepping. 2829 */ 2830 rflags = read_rflags(); 2831 vmxctx->host_tf = rflags & PSL_T; 2832 write_rflags(rflags & ~PSL_T); 2833 2834 /* Save host debug registers. */ 2835 vmxctx->host_dr0 = rdr0(); 2836 vmxctx->host_dr1 = rdr1(); 2837 vmxctx->host_dr2 = rdr2(); 2838 vmxctx->host_dr3 = rdr3(); 2839 vmxctx->host_dr6 = rdr6(); 2840 2841 /* Restore guest debug registers. */ 2842 load_dr0(vmxctx->guest_dr0); 2843 load_dr1(vmxctx->guest_dr1); 2844 load_dr2(vmxctx->guest_dr2); 2845 load_dr3(vmxctx->guest_dr3); 2846 load_dr6(vmxctx->guest_dr6); 2847 } 2848 2849 static __inline void 2850 vmx_dr_leave_guest(struct vmxctx *vmxctx) 2851 { 2852 2853 /* Save guest debug registers. */ 2854 vmxctx->guest_dr0 = rdr0(); 2855 vmxctx->guest_dr1 = rdr1(); 2856 vmxctx->guest_dr2 = rdr2(); 2857 vmxctx->guest_dr3 = rdr3(); 2858 vmxctx->guest_dr6 = rdr6(); 2859 2860 /* 2861 * Restore host debug registers. Restore DR7, DEBUGCTL, and 2862 * PSL_T last. 2863 */ 2864 load_dr0(vmxctx->host_dr0); 2865 load_dr1(vmxctx->host_dr1); 2866 load_dr2(vmxctx->host_dr2); 2867 load_dr3(vmxctx->host_dr3); 2868 load_dr6(vmxctx->host_dr6); 2869 wrmsr(MSR_DEBUGCTLMSR, vmxctx->host_debugctl); 2870 load_dr7(vmxctx->host_dr7); 2871 write_rflags(read_rflags() | vmxctx->host_tf); 2872 } 2873 2874 static int 2875 vmx_run(void *arg, int vcpu, register_t rip, pmap_t pmap, 2876 struct vm_eventinfo *evinfo) 2877 { 2878 int rc, handled, launched; 2879 struct vmx *vmx; 2880 struct vm *vm; 2881 struct vmxctx *vmxctx; 2882 struct vmcs *vmcs; 2883 struct vm_exit *vmexit; 2884 struct vlapic *vlapic; 2885 uint32_t exit_reason; 2886 struct region_descriptor gdtr, idtr; 2887 uint16_t ldt_sel; 2888 2889 vmx = arg; 2890 vm = vmx->vm; 2891 vmcs = &vmx->vmcs[vcpu]; 2892 vmxctx = &vmx->ctx[vcpu]; 2893 vlapic = vm_lapic(vm, vcpu); 2894 vmexit = vm_exitinfo(vm, vcpu); 2895 launched = 0; 2896 2897 KASSERT(vmxctx->pmap == pmap, 2898 ("pmap %p different than ctx pmap %p", pmap, vmxctx->pmap)); 2899 2900 vmx_msr_guest_enter(vmx, vcpu); 2901 2902 VMPTRLD(vmcs); 2903 2904 /* 2905 * XXX 2906 * We do this every time because we may setup the virtual machine 2907 * from a different process than the one that actually runs it. 2908 * 2909 * If the life of a virtual machine was spent entirely in the context 2910 * of a single process we could do this once in vmx_vminit(). 2911 */ 2912 vmcs_write(VMCS_HOST_CR3, rcr3()); 2913 2914 vmcs_write(VMCS_GUEST_RIP, rip); 2915 vmx_set_pcpu_defaults(vmx, vcpu, pmap); 2916 do { 2917 KASSERT(vmcs_guest_rip() == rip, ("%s: vmcs guest rip mismatch " 2918 "%#lx/%#lx", __func__, vmcs_guest_rip(), rip)); 2919 2920 handled = UNHANDLED; 2921 /* 2922 * Interrupts are disabled from this point on until the 2923 * guest starts executing. This is done for the following 2924 * reasons: 2925 * 2926 * If an AST is asserted on this thread after the check below, 2927 * then the IPI_AST notification will not be lost, because it 2928 * will cause a VM exit due to external interrupt as soon as 2929 * the guest state is loaded. 2930 * 2931 * A posted interrupt after 'vmx_inject_interrupts()' will 2932 * not be "lost" because it will be held pending in the host 2933 * APIC because interrupts are disabled. The pending interrupt 2934 * will be recognized as soon as the guest state is loaded. 2935 * 2936 * The same reasoning applies to the IPI generated by 2937 * pmap_invalidate_ept(). 2938 */ 2939 disable_intr(); 2940 vmx_inject_interrupts(vmx, vcpu, vlapic, rip); 2941 2942 /* 2943 * Check for vcpu suspension after injecting events because 2944 * vmx_inject_interrupts() can suspend the vcpu due to a 2945 * triple fault. 2946 */ 2947 if (vcpu_suspended(evinfo)) { 2948 enable_intr(); 2949 vm_exit_suspended(vmx->vm, vcpu, rip); 2950 break; 2951 } 2952 2953 if (vcpu_rendezvous_pending(evinfo)) { 2954 enable_intr(); 2955 vm_exit_rendezvous(vmx->vm, vcpu, rip); 2956 break; 2957 } 2958 2959 if (vcpu_reqidle(evinfo)) { 2960 enable_intr(); 2961 vm_exit_reqidle(vmx->vm, vcpu, rip); 2962 break; 2963 } 2964 2965 if (vcpu_should_yield(vm, vcpu)) { 2966 enable_intr(); 2967 vm_exit_astpending(vmx->vm, vcpu, rip); 2968 vmx_astpending_trace(vmx, vcpu, rip); 2969 handled = HANDLED; 2970 break; 2971 } 2972 2973 if (vcpu_debugged(vm, vcpu)) { 2974 enable_intr(); 2975 vm_exit_debug(vmx->vm, vcpu, rip); 2976 break; 2977 } 2978 2979 /* 2980 * If TPR Shadowing is enabled, the TPR Threshold 2981 * must be updated right before entering the guest. 2982 */ 2983 if (tpr_shadowing && !virtual_interrupt_delivery) { 2984 if ((vmx->cap[vcpu].proc_ctls & PROCBASED_USE_TPR_SHADOW) != 0) { 2985 vmcs_write(VMCS_TPR_THRESHOLD, vlapic_get_cr8(vlapic)); 2986 } 2987 } 2988 2989 /* 2990 * VM exits restore the base address but not the 2991 * limits of GDTR and IDTR. The VMCS only stores the 2992 * base address, so VM exits set the limits to 0xffff. 2993 * Save and restore the full GDTR and IDTR to restore 2994 * the limits. 2995 * 2996 * The VMCS does not save the LDTR at all, and VM 2997 * exits clear LDTR as if a NULL selector were loaded. 2998 * The userspace hypervisor probably doesn't use a 2999 * LDT, but save and restore it to be safe. 3000 */ 3001 sgdt(&gdtr); 3002 sidt(&idtr); 3003 ldt_sel = sldt(); 3004 3005 vmx_run_trace(vmx, vcpu); 3006 vmx_dr_enter_guest(vmxctx); 3007 rc = vmx_enter_guest(vmxctx, vmx, launched); 3008 vmx_dr_leave_guest(vmxctx); 3009 3010 bare_lgdt(&gdtr); 3011 lidt(&idtr); 3012 lldt(ldt_sel); 3013 3014 /* Collect some information for VM exit processing */ 3015 vmexit->rip = rip = vmcs_guest_rip(); 3016 vmexit->inst_length = vmexit_instruction_length(); 3017 vmexit->u.vmx.exit_reason = exit_reason = vmcs_exit_reason(); 3018 vmexit->u.vmx.exit_qualification = vmcs_exit_qualification(); 3019 3020 /* Update 'nextrip' */ 3021 vmx->state[vcpu].nextrip = rip; 3022 3023 if (rc == VMX_GUEST_VMEXIT) { 3024 vmx_exit_handle_nmi(vmx, vcpu, vmexit); 3025 enable_intr(); 3026 handled = vmx_exit_process(vmx, vcpu, vmexit); 3027 } else { 3028 enable_intr(); 3029 vmx_exit_inst_error(vmxctx, rc, vmexit); 3030 } 3031 launched = 1; 3032 vmx_exit_trace(vmx, vcpu, rip, exit_reason, handled); 3033 rip = vmexit->rip; 3034 } while (handled); 3035 3036 /* 3037 * If a VM exit has been handled then the exitcode must be BOGUS 3038 * If a VM exit is not handled then the exitcode must not be BOGUS 3039 */ 3040 if ((handled && vmexit->exitcode != VM_EXITCODE_BOGUS) || 3041 (!handled && vmexit->exitcode == VM_EXITCODE_BOGUS)) { 3042 panic("Mismatch between handled (%d) and exitcode (%d)", 3043 handled, vmexit->exitcode); 3044 } 3045 3046 if (!handled) 3047 vmm_stat_incr(vm, vcpu, VMEXIT_USERSPACE, 1); 3048 3049 VCPU_CTR1(vm, vcpu, "returning from vmx_run: exitcode %d", 3050 vmexit->exitcode); 3051 3052 VMCLEAR(vmcs); 3053 vmx_msr_guest_exit(vmx, vcpu); 3054 3055 return (0); 3056 } 3057 3058 static void 3059 vmx_vmcleanup(void *arg) 3060 { 3061 int i; 3062 struct vmx *vmx = arg; 3063 uint16_t maxcpus; 3064 3065 if (apic_access_virtualization(vmx, 0)) 3066 vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 3067 3068 maxcpus = vm_get_maxcpus(vmx->vm); 3069 for (i = 0; i < maxcpus; i++) 3070 vpid_free(vmx->state[i].vpid); 3071 3072 free(vmx, M_VMX); 3073 3074 return; 3075 } 3076 3077 static register_t * 3078 vmxctx_regptr(struct vmxctx *vmxctx, int reg) 3079 { 3080 3081 switch (reg) { 3082 case VM_REG_GUEST_RAX: 3083 return (&vmxctx->guest_rax); 3084 case VM_REG_GUEST_RBX: 3085 return (&vmxctx->guest_rbx); 3086 case VM_REG_GUEST_RCX: 3087 return (&vmxctx->guest_rcx); 3088 case VM_REG_GUEST_RDX: 3089 return (&vmxctx->guest_rdx); 3090 case VM_REG_GUEST_RSI: 3091 return (&vmxctx->guest_rsi); 3092 case VM_REG_GUEST_RDI: 3093 return (&vmxctx->guest_rdi); 3094 case VM_REG_GUEST_RBP: 3095 return (&vmxctx->guest_rbp); 3096 case VM_REG_GUEST_R8: 3097 return (&vmxctx->guest_r8); 3098 case VM_REG_GUEST_R9: 3099 return (&vmxctx->guest_r9); 3100 case VM_REG_GUEST_R10: 3101 return (&vmxctx->guest_r10); 3102 case VM_REG_GUEST_R11: 3103 return (&vmxctx->guest_r11); 3104 case VM_REG_GUEST_R12: 3105 return (&vmxctx->guest_r12); 3106 case VM_REG_GUEST_R13: 3107 return (&vmxctx->guest_r13); 3108 case VM_REG_GUEST_R14: 3109 return (&vmxctx->guest_r14); 3110 case VM_REG_GUEST_R15: 3111 return (&vmxctx->guest_r15); 3112 case VM_REG_GUEST_CR2: 3113 return (&vmxctx->guest_cr2); 3114 case VM_REG_GUEST_DR0: 3115 return (&vmxctx->guest_dr0); 3116 case VM_REG_GUEST_DR1: 3117 return (&vmxctx->guest_dr1); 3118 case VM_REG_GUEST_DR2: 3119 return (&vmxctx->guest_dr2); 3120 case VM_REG_GUEST_DR3: 3121 return (&vmxctx->guest_dr3); 3122 case VM_REG_GUEST_DR6: 3123 return (&vmxctx->guest_dr6); 3124 default: 3125 break; 3126 } 3127 return (NULL); 3128 } 3129 3130 static int 3131 vmxctx_getreg(struct vmxctx *vmxctx, int reg, uint64_t *retval) 3132 { 3133 register_t *regp; 3134 3135 if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 3136 *retval = *regp; 3137 return (0); 3138 } else 3139 return (EINVAL); 3140 } 3141 3142 static int 3143 vmxctx_setreg(struct vmxctx *vmxctx, int reg, uint64_t val) 3144 { 3145 register_t *regp; 3146 3147 if ((regp = vmxctx_regptr(vmxctx, reg)) != NULL) { 3148 *regp = val; 3149 return (0); 3150 } else 3151 return (EINVAL); 3152 } 3153 3154 static int 3155 vmx_get_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t *retval) 3156 { 3157 uint64_t gi; 3158 int error; 3159 3160 error = vmcs_getreg(&vmx->vmcs[vcpu], running, 3161 VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY), &gi); 3162 *retval = (gi & HWINTR_BLOCKING) ? 1 : 0; 3163 return (error); 3164 } 3165 3166 static int 3167 vmx_modify_intr_shadow(struct vmx *vmx, int vcpu, int running, uint64_t val) 3168 { 3169 struct vmcs *vmcs; 3170 uint64_t gi; 3171 int error, ident; 3172 3173 /* 3174 * Forcing the vcpu into an interrupt shadow is not supported. 3175 */ 3176 if (val) { 3177 error = EINVAL; 3178 goto done; 3179 } 3180 3181 vmcs = &vmx->vmcs[vcpu]; 3182 ident = VMCS_IDENT(VMCS_GUEST_INTERRUPTIBILITY); 3183 error = vmcs_getreg(vmcs, running, ident, &gi); 3184 if (error == 0) { 3185 gi &= ~HWINTR_BLOCKING; 3186 error = vmcs_setreg(vmcs, running, ident, gi); 3187 } 3188 done: 3189 VCPU_CTR2(vmx->vm, vcpu, "Setting intr_shadow to %#lx %s", val, 3190 error ? "failed" : "succeeded"); 3191 return (error); 3192 } 3193 3194 static int 3195 vmx_shadow_reg(int reg) 3196 { 3197 int shreg; 3198 3199 shreg = -1; 3200 3201 switch (reg) { 3202 case VM_REG_GUEST_CR0: 3203 shreg = VMCS_CR0_SHADOW; 3204 break; 3205 case VM_REG_GUEST_CR4: 3206 shreg = VMCS_CR4_SHADOW; 3207 break; 3208 default: 3209 break; 3210 } 3211 3212 return (shreg); 3213 } 3214 3215 static int 3216 vmx_getreg(void *arg, int vcpu, int reg, uint64_t *retval) 3217 { 3218 int running, hostcpu; 3219 struct vmx *vmx = arg; 3220 3221 running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3222 if (running && hostcpu != curcpu) 3223 panic("vmx_getreg: %s%d is running", vm_name(vmx->vm), vcpu); 3224 3225 if (reg == VM_REG_GUEST_INTR_SHADOW) 3226 return (vmx_get_intr_shadow(vmx, vcpu, running, retval)); 3227 3228 if (vmxctx_getreg(&vmx->ctx[vcpu], reg, retval) == 0) 3229 return (0); 3230 3231 return (vmcs_getreg(&vmx->vmcs[vcpu], running, reg, retval)); 3232 } 3233 3234 static int 3235 vmx_setreg(void *arg, int vcpu, int reg, uint64_t val) 3236 { 3237 int error, hostcpu, running, shadow; 3238 uint64_t ctls; 3239 pmap_t pmap; 3240 struct vmx *vmx = arg; 3241 3242 running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3243 if (running && hostcpu != curcpu) 3244 panic("vmx_setreg: %s%d is running", vm_name(vmx->vm), vcpu); 3245 3246 if (reg == VM_REG_GUEST_INTR_SHADOW) 3247 return (vmx_modify_intr_shadow(vmx, vcpu, running, val)); 3248 3249 if (vmxctx_setreg(&vmx->ctx[vcpu], reg, val) == 0) 3250 return (0); 3251 3252 error = vmcs_setreg(&vmx->vmcs[vcpu], running, reg, val); 3253 3254 if (error == 0) { 3255 /* 3256 * If the "load EFER" VM-entry control is 1 then the 3257 * value of EFER.LMA must be identical to "IA-32e mode guest" 3258 * bit in the VM-entry control. 3259 */ 3260 if ((entry_ctls & VM_ENTRY_LOAD_EFER) != 0 && 3261 (reg == VM_REG_GUEST_EFER)) { 3262 vmcs_getreg(&vmx->vmcs[vcpu], running, 3263 VMCS_IDENT(VMCS_ENTRY_CTLS), &ctls); 3264 if (val & EFER_LMA) 3265 ctls |= VM_ENTRY_GUEST_LMA; 3266 else 3267 ctls &= ~VM_ENTRY_GUEST_LMA; 3268 vmcs_setreg(&vmx->vmcs[vcpu], running, 3269 VMCS_IDENT(VMCS_ENTRY_CTLS), ctls); 3270 } 3271 3272 shadow = vmx_shadow_reg(reg); 3273 if (shadow > 0) { 3274 /* 3275 * Store the unmodified value in the shadow 3276 */ 3277 error = vmcs_setreg(&vmx->vmcs[vcpu], running, 3278 VMCS_IDENT(shadow), val); 3279 } 3280 3281 if (reg == VM_REG_GUEST_CR3) { 3282 /* 3283 * Invalidate the guest vcpu's TLB mappings to emulate 3284 * the behavior of updating %cr3. 3285 * 3286 * XXX the processor retains global mappings when %cr3 3287 * is updated but vmx_invvpid() does not. 3288 */ 3289 pmap = vmx->ctx[vcpu].pmap; 3290 vmx_invvpid(vmx, vcpu, pmap, running); 3291 } 3292 } 3293 3294 return (error); 3295 } 3296 3297 static int 3298 vmx_getdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 3299 { 3300 int hostcpu, running; 3301 struct vmx *vmx = arg; 3302 3303 running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3304 if (running && hostcpu != curcpu) 3305 panic("vmx_getdesc: %s%d is running", vm_name(vmx->vm), vcpu); 3306 3307 return (vmcs_getdesc(&vmx->vmcs[vcpu], running, reg, desc)); 3308 } 3309 3310 static int 3311 vmx_setdesc(void *arg, int vcpu, int reg, struct seg_desc *desc) 3312 { 3313 int hostcpu, running; 3314 struct vmx *vmx = arg; 3315 3316 running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3317 if (running && hostcpu != curcpu) 3318 panic("vmx_setdesc: %s%d is running", vm_name(vmx->vm), vcpu); 3319 3320 return (vmcs_setdesc(&vmx->vmcs[vcpu], running, reg, desc)); 3321 } 3322 3323 static int 3324 vmx_getcap(void *arg, int vcpu, int type, int *retval) 3325 { 3326 struct vmx *vmx = arg; 3327 int vcap; 3328 int ret; 3329 3330 ret = ENOENT; 3331 3332 vcap = vmx->cap[vcpu].set; 3333 3334 switch (type) { 3335 case VM_CAP_HALT_EXIT: 3336 if (cap_halt_exit) 3337 ret = 0; 3338 break; 3339 case VM_CAP_PAUSE_EXIT: 3340 if (cap_pause_exit) 3341 ret = 0; 3342 break; 3343 case VM_CAP_MTRAP_EXIT: 3344 if (cap_monitor_trap) 3345 ret = 0; 3346 break; 3347 case VM_CAP_UNRESTRICTED_GUEST: 3348 if (cap_unrestricted_guest) 3349 ret = 0; 3350 break; 3351 case VM_CAP_ENABLE_INVPCID: 3352 if (cap_invpcid) 3353 ret = 0; 3354 break; 3355 case VM_CAP_BPT_EXIT: 3356 ret = 0; 3357 break; 3358 default: 3359 break; 3360 } 3361 3362 if (ret == 0) 3363 *retval = (vcap & (1 << type)) ? 1 : 0; 3364 3365 return (ret); 3366 } 3367 3368 static int 3369 vmx_setcap(void *arg, int vcpu, int type, int val) 3370 { 3371 struct vmx *vmx = arg; 3372 struct vmcs *vmcs = &vmx->vmcs[vcpu]; 3373 uint32_t baseval; 3374 uint32_t *pptr; 3375 int error; 3376 int flag; 3377 int reg; 3378 int retval; 3379 3380 retval = ENOENT; 3381 pptr = NULL; 3382 3383 switch (type) { 3384 case VM_CAP_HALT_EXIT: 3385 if (cap_halt_exit) { 3386 retval = 0; 3387 pptr = &vmx->cap[vcpu].proc_ctls; 3388 baseval = *pptr; 3389 flag = PROCBASED_HLT_EXITING; 3390 reg = VMCS_PRI_PROC_BASED_CTLS; 3391 } 3392 break; 3393 case VM_CAP_MTRAP_EXIT: 3394 if (cap_monitor_trap) { 3395 retval = 0; 3396 pptr = &vmx->cap[vcpu].proc_ctls; 3397 baseval = *pptr; 3398 flag = PROCBASED_MTF; 3399 reg = VMCS_PRI_PROC_BASED_CTLS; 3400 } 3401 break; 3402 case VM_CAP_PAUSE_EXIT: 3403 if (cap_pause_exit) { 3404 retval = 0; 3405 pptr = &vmx->cap[vcpu].proc_ctls; 3406 baseval = *pptr; 3407 flag = PROCBASED_PAUSE_EXITING; 3408 reg = VMCS_PRI_PROC_BASED_CTLS; 3409 } 3410 break; 3411 case VM_CAP_UNRESTRICTED_GUEST: 3412 if (cap_unrestricted_guest) { 3413 retval = 0; 3414 pptr = &vmx->cap[vcpu].proc_ctls2; 3415 baseval = *pptr; 3416 flag = PROCBASED2_UNRESTRICTED_GUEST; 3417 reg = VMCS_SEC_PROC_BASED_CTLS; 3418 } 3419 break; 3420 case VM_CAP_ENABLE_INVPCID: 3421 if (cap_invpcid) { 3422 retval = 0; 3423 pptr = &vmx->cap[vcpu].proc_ctls2; 3424 baseval = *pptr; 3425 flag = PROCBASED2_ENABLE_INVPCID; 3426 reg = VMCS_SEC_PROC_BASED_CTLS; 3427 } 3428 break; 3429 case VM_CAP_BPT_EXIT: 3430 retval = 0; 3431 3432 /* Don't change the bitmap if we are tracing all exceptions. */ 3433 if (vmx->cap[vcpu].exc_bitmap != 0xffffffff) { 3434 pptr = &vmx->cap[vcpu].exc_bitmap; 3435 baseval = *pptr; 3436 flag = (1 << IDT_BP); 3437 reg = VMCS_EXCEPTION_BITMAP; 3438 } 3439 break; 3440 default: 3441 break; 3442 } 3443 3444 if (retval) 3445 return (retval); 3446 3447 if (pptr != NULL) { 3448 if (val) { 3449 baseval |= flag; 3450 } else { 3451 baseval &= ~flag; 3452 } 3453 VMPTRLD(vmcs); 3454 error = vmwrite(reg, baseval); 3455 VMCLEAR(vmcs); 3456 3457 if (error) 3458 return (error); 3459 3460 /* 3461 * Update optional stored flags, and record 3462 * setting 3463 */ 3464 *pptr = baseval; 3465 } 3466 3467 if (val) { 3468 vmx->cap[vcpu].set |= (1 << type); 3469 } else { 3470 vmx->cap[vcpu].set &= ~(1 << type); 3471 } 3472 3473 return (0); 3474 } 3475 3476 struct vlapic_vtx { 3477 struct vlapic vlapic; 3478 struct pir_desc *pir_desc; 3479 struct vmx *vmx; 3480 u_int pending_prio; 3481 }; 3482 3483 #define VPR_PRIO_BIT(vpr) (1 << ((vpr) >> 4)) 3484 3485 #define VMX_CTR_PIR(vm, vcpuid, pir_desc, notify, vector, level, msg) \ 3486 do { \ 3487 VCPU_CTR2(vm, vcpuid, msg " assert %s-triggered vector %d", \ 3488 level ? "level" : "edge", vector); \ 3489 VCPU_CTR1(vm, vcpuid, msg " pir0 0x%016lx", pir_desc->pir[0]); \ 3490 VCPU_CTR1(vm, vcpuid, msg " pir1 0x%016lx", pir_desc->pir[1]); \ 3491 VCPU_CTR1(vm, vcpuid, msg " pir2 0x%016lx", pir_desc->pir[2]); \ 3492 VCPU_CTR1(vm, vcpuid, msg " pir3 0x%016lx", pir_desc->pir[3]); \ 3493 VCPU_CTR1(vm, vcpuid, msg " notify: %s", notify ? "yes" : "no");\ 3494 } while (0) 3495 3496 /* 3497 * vlapic->ops handlers that utilize the APICv hardware assist described in 3498 * Chapter 29 of the Intel SDM. 3499 */ 3500 static int 3501 vmx_set_intr_ready(struct vlapic *vlapic, int vector, bool level) 3502 { 3503 struct vlapic_vtx *vlapic_vtx; 3504 struct pir_desc *pir_desc; 3505 uint64_t mask; 3506 int idx, notify = 0; 3507 3508 vlapic_vtx = (struct vlapic_vtx *)vlapic; 3509 pir_desc = vlapic_vtx->pir_desc; 3510 3511 /* 3512 * Keep track of interrupt requests in the PIR descriptor. This is 3513 * because the virtual APIC page pointed to by the VMCS cannot be 3514 * modified if the vcpu is running. 3515 */ 3516 idx = vector / 64; 3517 mask = 1UL << (vector % 64); 3518 atomic_set_long(&pir_desc->pir[idx], mask); 3519 3520 /* 3521 * A notification is required whenever the 'pending' bit makes a 3522 * transition from 0->1. 3523 * 3524 * Even if the 'pending' bit is already asserted, notification about 3525 * the incoming interrupt may still be necessary. For example, if a 3526 * vCPU is HLTed with a high PPR, a low priority interrupt would cause 3527 * the 0->1 'pending' transition with a notification, but the vCPU 3528 * would ignore the interrupt for the time being. The same vCPU would 3529 * need to then be notified if a high-priority interrupt arrived which 3530 * satisfied the PPR. 3531 * 3532 * The priorities of interrupts injected while 'pending' is asserted 3533 * are tracked in a custom bitfield 'pending_prio'. Should the 3534 * to-be-injected interrupt exceed the priorities already present, the 3535 * notification is sent. The priorities recorded in 'pending_prio' are 3536 * cleared whenever the 'pending' bit makes another 0->1 transition. 3537 */ 3538 if (atomic_cmpset_long(&pir_desc->pending, 0, 1) != 0) { 3539 notify = 1; 3540 vlapic_vtx->pending_prio = 0; 3541 } else { 3542 const u_int old_prio = vlapic_vtx->pending_prio; 3543 const u_int prio_bit = VPR_PRIO_BIT(vector & APIC_TPR_INT); 3544 3545 if ((old_prio & prio_bit) == 0 && prio_bit > old_prio) { 3546 atomic_set_int(&vlapic_vtx->pending_prio, prio_bit); 3547 notify = 1; 3548 } 3549 } 3550 3551 VMX_CTR_PIR(vlapic->vm, vlapic->vcpuid, pir_desc, notify, vector, 3552 level, "vmx_set_intr_ready"); 3553 return (notify); 3554 } 3555 3556 static int 3557 vmx_pending_intr(struct vlapic *vlapic, int *vecptr) 3558 { 3559 struct vlapic_vtx *vlapic_vtx; 3560 struct pir_desc *pir_desc; 3561 struct LAPIC *lapic; 3562 uint64_t pending, pirval; 3563 uint32_t ppr, vpr; 3564 int i; 3565 3566 /* 3567 * This function is only expected to be called from the 'HLT' exit 3568 * handler which does not care about the vector that is pending. 3569 */ 3570 KASSERT(vecptr == NULL, ("vmx_pending_intr: vecptr must be NULL")); 3571 3572 vlapic_vtx = (struct vlapic_vtx *)vlapic; 3573 pir_desc = vlapic_vtx->pir_desc; 3574 3575 pending = atomic_load_acq_long(&pir_desc->pending); 3576 if (!pending) { 3577 /* 3578 * While a virtual interrupt may have already been 3579 * processed the actual delivery maybe pending the 3580 * interruptibility of the guest. Recognize a pending 3581 * interrupt by reevaluating virtual interrupts 3582 * following Section 29.2.1 in the Intel SDM Volume 3. 3583 */ 3584 struct vm_exit *vmexit; 3585 uint8_t rvi, ppr; 3586 3587 vmexit = vm_exitinfo(vlapic->vm, vlapic->vcpuid); 3588 KASSERT(vmexit->exitcode == VM_EXITCODE_HLT, 3589 ("vmx_pending_intr: exitcode not 'HLT'")); 3590 rvi = vmexit->u.hlt.intr_status & APIC_TPR_INT; 3591 lapic = vlapic->apic_page; 3592 ppr = lapic->ppr & APIC_TPR_INT; 3593 if (rvi > ppr) { 3594 return (1); 3595 } 3596 3597 return (0); 3598 } 3599 3600 /* 3601 * If there is an interrupt pending then it will be recognized only 3602 * if its priority is greater than the processor priority. 3603 * 3604 * Special case: if the processor priority is zero then any pending 3605 * interrupt will be recognized. 3606 */ 3607 lapic = vlapic->apic_page; 3608 ppr = lapic->ppr & APIC_TPR_INT; 3609 if (ppr == 0) 3610 return (1); 3611 3612 VCPU_CTR1(vlapic->vm, vlapic->vcpuid, "HLT with non-zero PPR %d", 3613 lapic->ppr); 3614 3615 vpr = 0; 3616 for (i = 3; i >= 0; i--) { 3617 pirval = pir_desc->pir[i]; 3618 if (pirval != 0) { 3619 vpr = (i * 64 + flsl(pirval) - 1) & APIC_TPR_INT; 3620 break; 3621 } 3622 } 3623 3624 /* 3625 * If the highest-priority pending interrupt falls short of the 3626 * processor priority of this vCPU, ensure that 'pending_prio' does not 3627 * have any stale bits which would preclude a higher-priority interrupt 3628 * from incurring a notification later. 3629 */ 3630 if (vpr <= ppr) { 3631 const u_int prio_bit = VPR_PRIO_BIT(vpr); 3632 const u_int old = vlapic_vtx->pending_prio; 3633 3634 if (old > prio_bit && (old & prio_bit) == 0) { 3635 vlapic_vtx->pending_prio = prio_bit; 3636 } 3637 return (0); 3638 } 3639 return (1); 3640 } 3641 3642 static void 3643 vmx_intr_accepted(struct vlapic *vlapic, int vector) 3644 { 3645 3646 panic("vmx_intr_accepted: not expected to be called"); 3647 } 3648 3649 static void 3650 vmx_set_tmr(struct vlapic *vlapic, int vector, bool level) 3651 { 3652 struct vlapic_vtx *vlapic_vtx; 3653 struct vmx *vmx; 3654 struct vmcs *vmcs; 3655 uint64_t mask, val; 3656 3657 KASSERT(vector >= 0 && vector <= 255, ("invalid vector %d", vector)); 3658 KASSERT(!vcpu_is_running(vlapic->vm, vlapic->vcpuid, NULL), 3659 ("vmx_set_tmr: vcpu cannot be running")); 3660 3661 vlapic_vtx = (struct vlapic_vtx *)vlapic; 3662 vmx = vlapic_vtx->vmx; 3663 vmcs = &vmx->vmcs[vlapic->vcpuid]; 3664 mask = 1UL << (vector % 64); 3665 3666 VMPTRLD(vmcs); 3667 val = vmcs_read(VMCS_EOI_EXIT(vector)); 3668 if (level) 3669 val |= mask; 3670 else 3671 val &= ~mask; 3672 vmcs_write(VMCS_EOI_EXIT(vector), val); 3673 VMCLEAR(vmcs); 3674 } 3675 3676 static void 3677 vmx_enable_x2apic_mode_ts(struct vlapic *vlapic) 3678 { 3679 struct vmx *vmx; 3680 struct vmcs *vmcs; 3681 uint32_t proc_ctls; 3682 int vcpuid; 3683 3684 vcpuid = vlapic->vcpuid; 3685 vmx = ((struct vlapic_vtx *)vlapic)->vmx; 3686 vmcs = &vmx->vmcs[vcpuid]; 3687 3688 proc_ctls = vmx->cap[vcpuid].proc_ctls; 3689 proc_ctls &= ~PROCBASED_USE_TPR_SHADOW; 3690 proc_ctls |= PROCBASED_CR8_LOAD_EXITING; 3691 proc_ctls |= PROCBASED_CR8_STORE_EXITING; 3692 vmx->cap[vcpuid].proc_ctls = proc_ctls; 3693 3694 VMPTRLD(vmcs); 3695 vmcs_write(VMCS_PRI_PROC_BASED_CTLS, proc_ctls); 3696 VMCLEAR(vmcs); 3697 } 3698 3699 static void 3700 vmx_enable_x2apic_mode_vid(struct vlapic *vlapic) 3701 { 3702 struct vmx *vmx; 3703 struct vmcs *vmcs; 3704 uint32_t proc_ctls2; 3705 int vcpuid, error; 3706 3707 vcpuid = vlapic->vcpuid; 3708 vmx = ((struct vlapic_vtx *)vlapic)->vmx; 3709 vmcs = &vmx->vmcs[vcpuid]; 3710 3711 proc_ctls2 = vmx->cap[vcpuid].proc_ctls2; 3712 KASSERT((proc_ctls2 & PROCBASED2_VIRTUALIZE_APIC_ACCESSES) != 0, 3713 ("%s: invalid proc_ctls2 %#x", __func__, proc_ctls2)); 3714 3715 proc_ctls2 &= ~PROCBASED2_VIRTUALIZE_APIC_ACCESSES; 3716 proc_ctls2 |= PROCBASED2_VIRTUALIZE_X2APIC_MODE; 3717 vmx->cap[vcpuid].proc_ctls2 = proc_ctls2; 3718 3719 VMPTRLD(vmcs); 3720 vmcs_write(VMCS_SEC_PROC_BASED_CTLS, proc_ctls2); 3721 VMCLEAR(vmcs); 3722 3723 if (vlapic->vcpuid == 0) { 3724 /* 3725 * The nested page table mappings are shared by all vcpus 3726 * so unmap the APIC access page just once. 3727 */ 3728 error = vm_unmap_mmio(vmx->vm, DEFAULT_APIC_BASE, PAGE_SIZE); 3729 KASSERT(error == 0, ("%s: vm_unmap_mmio error %d", 3730 __func__, error)); 3731 3732 /* 3733 * The MSR bitmap is shared by all vcpus so modify it only 3734 * once in the context of vcpu 0. 3735 */ 3736 error = vmx_allow_x2apic_msrs(vmx); 3737 KASSERT(error == 0, ("%s: vmx_allow_x2apic_msrs error %d", 3738 __func__, error)); 3739 } 3740 } 3741 3742 static void 3743 vmx_post_intr(struct vlapic *vlapic, int hostcpu) 3744 { 3745 3746 ipi_cpu(hostcpu, pirvec); 3747 } 3748 3749 /* 3750 * Transfer the pending interrupts in the PIR descriptor to the IRR 3751 * in the virtual APIC page. 3752 */ 3753 static void 3754 vmx_inject_pir(struct vlapic *vlapic) 3755 { 3756 struct vlapic_vtx *vlapic_vtx; 3757 struct pir_desc *pir_desc; 3758 struct LAPIC *lapic; 3759 uint64_t val, pirval; 3760 int rvi, pirbase = -1; 3761 uint16_t intr_status_old, intr_status_new; 3762 3763 vlapic_vtx = (struct vlapic_vtx *)vlapic; 3764 pir_desc = vlapic_vtx->pir_desc; 3765 if (atomic_cmpset_long(&pir_desc->pending, 1, 0) == 0) { 3766 VCPU_CTR0(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 3767 "no posted interrupt pending"); 3768 return; 3769 } 3770 3771 pirval = 0; 3772 pirbase = -1; 3773 lapic = vlapic->apic_page; 3774 3775 val = atomic_readandclear_long(&pir_desc->pir[0]); 3776 if (val != 0) { 3777 lapic->irr0 |= val; 3778 lapic->irr1 |= val >> 32; 3779 pirbase = 0; 3780 pirval = val; 3781 } 3782 3783 val = atomic_readandclear_long(&pir_desc->pir[1]); 3784 if (val != 0) { 3785 lapic->irr2 |= val; 3786 lapic->irr3 |= val >> 32; 3787 pirbase = 64; 3788 pirval = val; 3789 } 3790 3791 val = atomic_readandclear_long(&pir_desc->pir[2]); 3792 if (val != 0) { 3793 lapic->irr4 |= val; 3794 lapic->irr5 |= val >> 32; 3795 pirbase = 128; 3796 pirval = val; 3797 } 3798 3799 val = atomic_readandclear_long(&pir_desc->pir[3]); 3800 if (val != 0) { 3801 lapic->irr6 |= val; 3802 lapic->irr7 |= val >> 32; 3803 pirbase = 192; 3804 pirval = val; 3805 } 3806 3807 VLAPIC_CTR_IRR(vlapic, "vmx_inject_pir"); 3808 3809 /* 3810 * Update RVI so the processor can evaluate pending virtual 3811 * interrupts on VM-entry. 3812 * 3813 * It is possible for pirval to be 0 here, even though the 3814 * pending bit has been set. The scenario is: 3815 * CPU-Y is sending a posted interrupt to CPU-X, which 3816 * is running a guest and processing posted interrupts in h/w. 3817 * CPU-X will eventually exit and the state seen in s/w is 3818 * the pending bit set, but no PIR bits set. 3819 * 3820 * CPU-X CPU-Y 3821 * (vm running) (host running) 3822 * rx posted interrupt 3823 * CLEAR pending bit 3824 * SET PIR bit 3825 * READ/CLEAR PIR bits 3826 * SET pending bit 3827 * (vm exit) 3828 * pending bit set, PIR 0 3829 */ 3830 if (pirval != 0) { 3831 rvi = pirbase + flsl(pirval) - 1; 3832 intr_status_old = vmcs_read(VMCS_GUEST_INTR_STATUS); 3833 intr_status_new = (intr_status_old & 0xFF00) | rvi; 3834 if (intr_status_new > intr_status_old) { 3835 vmcs_write(VMCS_GUEST_INTR_STATUS, intr_status_new); 3836 VCPU_CTR2(vlapic->vm, vlapic->vcpuid, "vmx_inject_pir: " 3837 "guest_intr_status changed from 0x%04x to 0x%04x", 3838 intr_status_old, intr_status_new); 3839 } 3840 } 3841 } 3842 3843 static struct vlapic * 3844 vmx_vlapic_init(void *arg, int vcpuid) 3845 { 3846 struct vmx *vmx; 3847 struct vlapic *vlapic; 3848 struct vlapic_vtx *vlapic_vtx; 3849 3850 vmx = arg; 3851 3852 vlapic = malloc(sizeof(struct vlapic_vtx), M_VLAPIC, M_WAITOK | M_ZERO); 3853 vlapic->vm = vmx->vm; 3854 vlapic->vcpuid = vcpuid; 3855 vlapic->apic_page = (struct LAPIC *)&vmx->apic_page[vcpuid]; 3856 3857 vlapic_vtx = (struct vlapic_vtx *)vlapic; 3858 vlapic_vtx->pir_desc = &vmx->pir_desc[vcpuid]; 3859 vlapic_vtx->vmx = vmx; 3860 3861 if (tpr_shadowing) { 3862 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_ts; 3863 } 3864 3865 if (virtual_interrupt_delivery) { 3866 vlapic->ops.set_intr_ready = vmx_set_intr_ready; 3867 vlapic->ops.pending_intr = vmx_pending_intr; 3868 vlapic->ops.intr_accepted = vmx_intr_accepted; 3869 vlapic->ops.set_tmr = vmx_set_tmr; 3870 vlapic->ops.enable_x2apic_mode = vmx_enable_x2apic_mode_vid; 3871 } 3872 3873 if (posted_interrupts) 3874 vlapic->ops.post_intr = vmx_post_intr; 3875 3876 vlapic_init(vlapic); 3877 3878 return (vlapic); 3879 } 3880 3881 static void 3882 vmx_vlapic_cleanup(void *arg, struct vlapic *vlapic) 3883 { 3884 3885 vlapic_cleanup(vlapic); 3886 free(vlapic, M_VLAPIC); 3887 } 3888 3889 #ifdef BHYVE_SNAPSHOT 3890 static int 3891 vmx_snapshot_vmi(void *arg, struct vm_snapshot_meta *meta) 3892 { 3893 struct vmx *vmx; 3894 struct vmxctx *vmxctx; 3895 int i; 3896 int ret; 3897 3898 vmx = arg; 3899 3900 KASSERT(vmx != NULL, ("%s: arg was NULL", __func__)); 3901 3902 for (i = 0; i < VM_MAXCPU; i++) { 3903 SNAPSHOT_BUF_OR_LEAVE(vmx->guest_msrs[i], 3904 sizeof(vmx->guest_msrs[i]), meta, ret, done); 3905 3906 vmxctx = &vmx->ctx[i]; 3907 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdi, meta, ret, done); 3908 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rsi, meta, ret, done); 3909 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rdx, meta, ret, done); 3910 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rcx, meta, ret, done); 3911 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r8, meta, ret, done); 3912 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r9, meta, ret, done); 3913 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rax, meta, ret, done); 3914 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbx, meta, ret, done); 3915 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_rbp, meta, ret, done); 3916 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r10, meta, ret, done); 3917 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r11, meta, ret, done); 3918 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r12, meta, ret, done); 3919 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r13, meta, ret, done); 3920 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r14, meta, ret, done); 3921 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_r15, meta, ret, done); 3922 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_cr2, meta, ret, done); 3923 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr0, meta, ret, done); 3924 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr1, meta, ret, done); 3925 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr2, meta, ret, done); 3926 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr3, meta, ret, done); 3927 SNAPSHOT_VAR_OR_LEAVE(vmxctx->guest_dr6, meta, ret, done); 3928 } 3929 3930 done: 3931 return (ret); 3932 } 3933 3934 static int 3935 vmx_snapshot_vmcx(void *arg, struct vm_snapshot_meta *meta, int vcpu) 3936 { 3937 struct vmcs *vmcs; 3938 struct vmx *vmx; 3939 int err, run, hostcpu; 3940 3941 vmx = (struct vmx *)arg; 3942 err = 0; 3943 3944 KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); 3945 vmcs = &vmx->vmcs[vcpu]; 3946 3947 run = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 3948 if (run && hostcpu != curcpu) { 3949 printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); 3950 return (EINVAL); 3951 } 3952 3953 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR0, meta); 3954 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR3, meta); 3955 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CR4, meta); 3956 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DR7, meta); 3957 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RSP, meta); 3958 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RIP, meta); 3959 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_RFLAGS, meta); 3960 3961 /* Guest segments */ 3962 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_ES, meta); 3963 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_ES, meta); 3964 3965 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_CS, meta); 3966 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_CS, meta); 3967 3968 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_SS, meta); 3969 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_SS, meta); 3970 3971 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_DS, meta); 3972 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_DS, meta); 3973 3974 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_FS, meta); 3975 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_FS, meta); 3976 3977 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_GS, meta); 3978 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GS, meta); 3979 3980 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_TR, meta); 3981 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_TR, meta); 3982 3983 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_LDTR, meta); 3984 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_LDTR, meta); 3985 3986 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_EFER, meta); 3987 3988 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_IDTR, meta); 3989 err += vmcs_snapshot_desc(vmcs, run, VM_REG_GUEST_GDTR, meta); 3990 3991 /* Guest page tables */ 3992 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE0, meta); 3993 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE1, meta); 3994 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE2, meta); 3995 err += vmcs_snapshot_reg(vmcs, run, VM_REG_GUEST_PDPTE3, meta); 3996 3997 /* Other guest state */ 3998 err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_CS, meta); 3999 err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_ESP, meta); 4000 err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_IA32_SYSENTER_EIP, meta); 4001 err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_INTERRUPTIBILITY, meta); 4002 err += vmcs_snapshot_any(vmcs, run, VMCS_GUEST_ACTIVITY, meta); 4003 err += vmcs_snapshot_any(vmcs, run, VMCS_ENTRY_CTLS, meta); 4004 err += vmcs_snapshot_any(vmcs, run, VMCS_EXIT_CTLS, meta); 4005 4006 return (err); 4007 } 4008 4009 static int 4010 vmx_restore_tsc(void *arg, int vcpu, uint64_t offset) 4011 { 4012 struct vmcs *vmcs; 4013 struct vmx *vmx = (struct vmx *)arg; 4014 int error, running, hostcpu; 4015 4016 KASSERT(arg != NULL, ("%s: arg was NULL", __func__)); 4017 vmcs = &vmx->vmcs[vcpu]; 4018 4019 running = vcpu_is_running(vmx->vm, vcpu, &hostcpu); 4020 if (running && hostcpu != curcpu) { 4021 printf("%s: %s%d is running", __func__, vm_name(vmx->vm), vcpu); 4022 return (EINVAL); 4023 } 4024 4025 if (!running) 4026 VMPTRLD(vmcs); 4027 4028 error = vmx_set_tsc_offset(vmx, vcpu, offset); 4029 4030 if (!running) 4031 VMCLEAR(vmcs); 4032 return (error); 4033 } 4034 #endif 4035 4036 struct vmm_ops vmm_ops_intel = { 4037 .init = vmx_init, 4038 .cleanup = vmx_cleanup, 4039 .resume = vmx_restore, 4040 .vminit = vmx_vminit, 4041 .vmrun = vmx_run, 4042 .vmcleanup = vmx_vmcleanup, 4043 .vmgetreg = vmx_getreg, 4044 .vmsetreg = vmx_setreg, 4045 .vmgetdesc = vmx_getdesc, 4046 .vmsetdesc = vmx_setdesc, 4047 .vmgetcap = vmx_getcap, 4048 .vmsetcap = vmx_setcap, 4049 .vmspace_alloc = ept_vmspace_alloc, 4050 .vmspace_free = ept_vmspace_free, 4051 .vlapic_init = vmx_vlapic_init, 4052 .vlapic_cleanup = vmx_vlapic_cleanup, 4053 #ifdef BHYVE_SNAPSHOT 4054 .vmsnapshot = vmx_snapshot_vmi, 4055 .vmcx_snapshot = vmx_snapshot_vmcx, 4056 .vm_restore_tsc = vmx_restore_tsc, 4057 #endif 4058 }; 4059