1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2008, 2013 Citrix Systems, Inc. 5 * Copyright (c) 2012 Spectra Logic Corporation 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 #include <vm/vm_param.h> 41 42 #include <dev/pci/pcivar.h> 43 44 #include <machine/cpufunc.h> 45 #include <machine/cpu.h> 46 #include <machine/smp.h> 47 48 #include <x86/apicreg.h> 49 50 #include <xen/xen-os.h> 51 #include <xen/error.h> 52 #include <xen/features.h> 53 #include <xen/gnttab.h> 54 #include <xen/hypervisor.h> 55 #include <xen/hvm.h> 56 #include <xen/xen_intr.h> 57 58 #include <contrib/xen/arch-x86/cpuid.h> 59 #include <contrib/xen/hvm/params.h> 60 #include <contrib/xen/vcpu.h> 61 62 /*--------------------------- Forward Declarations ---------------------------*/ 63 static void xen_hvm_cpu_init(void); 64 65 /*-------------------------------- Global Data -------------------------------*/ 66 #ifdef SMP 67 struct cpu_ops xen_hvm_cpu_ops = { 68 .cpu_init = xen_hvm_cpu_init, 69 .cpu_resume = xen_hvm_cpu_init 70 }; 71 #endif 72 73 static MALLOC_DEFINE(M_XENHVM, "xen_hvm", "Xen HVM PV Support"); 74 75 /** 76 * If non-zero, the hypervisor has been configured to use a direct 77 * IDT event callback for interrupt injection. 78 */ 79 int xen_vector_callback_enabled; 80 81 /** 82 * Signal whether the vector injected for the event channel upcall requires to 83 * be EOI'ed on the local APIC. 84 */ 85 bool xen_evtchn_needs_ack; 86 87 /*------------------------------- Per-CPU Data -------------------------------*/ 88 DPCPU_DECLARE(struct vcpu_info *, vcpu_info); 89 90 /*------------------------------ Sysctl tunables -----------------------------*/ 91 int xen_disable_pv_disks = 0; 92 int xen_disable_pv_nics = 0; 93 TUNABLE_INT("hw.xen.disable_pv_disks", &xen_disable_pv_disks); 94 TUNABLE_INT("hw.xen.disable_pv_nics", &xen_disable_pv_nics); 95 96 /*---------------------- XEN Hypervisor Probe and Setup ----------------------*/ 97 98 uint32_t xen_cpuid_base; 99 100 static uint32_t 101 xen_hvm_cpuid_base(void) 102 { 103 uint32_t base, regs[4]; 104 105 for (base = 0x40000000; base < 0x40010000; base += 0x100) { 106 do_cpuid(base, regs); 107 if (!memcmp("XenVMMXenVMM", ®s[1], 12) 108 && (regs[0] - base) >= 2) 109 return (base); 110 } 111 return (0); 112 } 113 114 static void 115 hypervisor_quirks(unsigned int major, unsigned int minor) 116 { 117 #ifdef SMP 118 if (((major < 4) || (major == 4 && minor <= 5)) && 119 msix_disable_migration == -1) { 120 /* 121 * Xen hypervisors prior to 4.6.0 do not properly 122 * handle updates to enabled MSI-X table entries, 123 * so disable MSI-X interrupt migration in that 124 * case. 125 */ 126 if (bootverbose) 127 printf( 128 "Disabling MSI-X interrupt migration due to Xen hypervisor bug.\n" 129 "Set machdep.msix_disable_migration=0 to forcefully enable it.\n"); 130 msix_disable_migration = 1; 131 } 132 #endif 133 } 134 135 static void 136 hypervisor_version(void) 137 { 138 uint32_t regs[4]; 139 int major, minor; 140 141 do_cpuid(xen_cpuid_base + 1, regs); 142 143 major = regs[0] >> 16; 144 minor = regs[0] & 0xffff; 145 printf("XEN: Hypervisor version %d.%d detected.\n", major, minor); 146 147 hypervisor_quirks(major, minor); 148 } 149 150 /* 151 * Allocate and fill in the hypcall page. 152 */ 153 int 154 xen_hvm_init_hypercall_stubs(enum xen_hvm_init_type init_type) 155 { 156 uint32_t regs[4]; 157 158 if (xen_cpuid_base != 0) 159 /* Already setup. */ 160 goto out; 161 162 xen_cpuid_base = xen_hvm_cpuid_base(); 163 if (xen_cpuid_base == 0) 164 return (ENXIO); 165 166 /* 167 * Find the hypercall pages. 168 */ 169 do_cpuid(xen_cpuid_base + 2, regs); 170 if (regs[0] != 1) 171 return (EINVAL); 172 173 wrmsr(regs[1], (init_type == XEN_HVM_INIT_EARLY) 174 ? (vm_paddr_t)((uintptr_t)&hypercall_page - KERNBASE) 175 : vtophys(&hypercall_page)); 176 177 out: 178 hypervisor_version(); 179 return (0); 180 } 181 182 static void 183 xen_hvm_init_shared_info_page(void) 184 { 185 struct xen_add_to_physmap xatp; 186 187 if (xen_pv_domain()) { 188 /* 189 * Already setup in the PV case, shared_info is passed inside 190 * of the start_info struct at start of day. 191 */ 192 return; 193 } 194 195 if (HYPERVISOR_shared_info == NULL) { 196 HYPERVISOR_shared_info = malloc(PAGE_SIZE, M_XENHVM, M_NOWAIT); 197 if (HYPERVISOR_shared_info == NULL) 198 panic("Unable to allocate Xen shared info page"); 199 } 200 201 xatp.domid = DOMID_SELF; 202 xatp.idx = 0; 203 xatp.space = XENMAPSPACE_shared_info; 204 xatp.gpfn = vtophys(HYPERVISOR_shared_info) >> PAGE_SHIFT; 205 if (HYPERVISOR_memory_op(XENMEM_add_to_physmap, &xatp)) 206 panic("HYPERVISOR_memory_op failed"); 207 } 208 209 static int 210 set_percpu_callback(unsigned int vcpu) 211 { 212 struct xen_hvm_evtchn_upcall_vector vec; 213 int error; 214 215 vec.vcpu = vcpu; 216 vec.vector = IDT_EVTCHN; 217 error = HYPERVISOR_hvm_op(HVMOP_set_evtchn_upcall_vector, &vec); 218 219 return (error != 0 ? xen_translate_error(error) : 0); 220 } 221 222 /* 223 * Tell the hypervisor how to contact us for event channel callbacks. 224 */ 225 void 226 xen_hvm_set_callback(device_t dev) 227 { 228 struct xen_hvm_param xhp; 229 int irq; 230 231 if (xen_vector_callback_enabled) 232 return; 233 234 xhp.domid = DOMID_SELF; 235 xhp.index = HVM_PARAM_CALLBACK_IRQ; 236 if (xen_feature(XENFEAT_hvm_callback_vector) != 0) { 237 int error; 238 239 error = set_percpu_callback(0); 240 if (error == 0) { 241 xen_evtchn_needs_ack = true; 242 /* Trick toolstack to think we are enlightened */ 243 xhp.value = 1; 244 } else 245 xhp.value = HVM_CALLBACK_VECTOR(IDT_EVTCHN); 246 error = HYPERVISOR_hvm_op(HVMOP_set_param, &xhp); 247 if (error == 0) { 248 xen_vector_callback_enabled = 1; 249 return; 250 } else if (xen_evtchn_needs_ack) 251 panic("Unable to setup fake HVM param: %d", error); 252 253 printf("Xen HVM callback vector registration failed (%d). " 254 "Falling back to emulated device interrupt\n", error); 255 } 256 xen_vector_callback_enabled = 0; 257 if (dev == NULL) { 258 /* 259 * Called from early boot or resume. 260 * xenpci will invoke us again later. 261 */ 262 return; 263 } 264 265 irq = pci_get_irq(dev); 266 if (irq < 16) { 267 xhp.value = HVM_CALLBACK_GSI(irq); 268 } else { 269 u_int slot; 270 u_int pin; 271 272 slot = pci_get_slot(dev); 273 pin = pci_get_intpin(dev) - 1; 274 xhp.value = HVM_CALLBACK_PCI_INTX(slot, pin); 275 } 276 277 if (HYPERVISOR_hvm_op(HVMOP_set_param, &xhp) != 0) 278 panic("Can't set evtchn callback"); 279 } 280 281 #define XEN_MAGIC_IOPORT 0x10 282 enum { 283 XMI_MAGIC = 0x49d2, 284 XMI_UNPLUG_IDE_DISKS = 0x01, 285 XMI_UNPLUG_NICS = 0x02, 286 XMI_UNPLUG_IDE_EXCEPT_PRI_MASTER = 0x04 287 }; 288 289 static void 290 xen_hvm_disable_emulated_devices(void) 291 { 292 u_short disable_devs = 0; 293 294 if (xen_pv_domain()) { 295 /* 296 * No emulated devices in the PV case, so no need to unplug 297 * anything. 298 */ 299 if (xen_disable_pv_disks != 0 || xen_disable_pv_nics != 0) 300 printf("PV devices cannot be disabled in PV guests\n"); 301 return; 302 } 303 304 if (inw(XEN_MAGIC_IOPORT) != XMI_MAGIC) 305 return; 306 307 if (xen_disable_pv_disks == 0) { 308 if (bootverbose) 309 printf("XEN: disabling emulated disks\n"); 310 disable_devs |= XMI_UNPLUG_IDE_DISKS; 311 } 312 if (xen_disable_pv_nics == 0) { 313 if (bootverbose) 314 printf("XEN: disabling emulated nics\n"); 315 disable_devs |= XMI_UNPLUG_NICS; 316 } 317 318 if (disable_devs != 0) 319 outw(XEN_MAGIC_IOPORT, disable_devs); 320 } 321 322 static void 323 xen_hvm_init(enum xen_hvm_init_type init_type) 324 { 325 int error; 326 int i; 327 328 if (!xen_domain() || 329 init_type == XEN_HVM_INIT_CANCELLED_SUSPEND) 330 return; 331 332 error = xen_hvm_init_hypercall_stubs(init_type); 333 334 switch (init_type) { 335 case XEN_HVM_INIT_LATE: 336 if (error != 0) 337 return; 338 339 setup_xen_features(); 340 #ifdef SMP 341 cpu_ops = xen_hvm_cpu_ops; 342 #endif 343 break; 344 case XEN_HVM_INIT_RESUME: 345 if (error != 0) 346 panic("Unable to init Xen hypercall stubs on resume"); 347 348 /* Clear stale vcpu_info. */ 349 CPU_FOREACH(i) 350 DPCPU_ID_SET(i, vcpu_info, NULL); 351 break; 352 default: 353 panic("Unsupported HVM initialization type"); 354 } 355 356 xen_vector_callback_enabled = 0; 357 xen_evtchn_needs_ack = false; 358 xen_hvm_set_callback(NULL); 359 360 /* 361 * On (PV)HVM domains we need to request the hypervisor to 362 * fill the shared info page, for PVH guest the shared_info page 363 * is passed inside the start_info struct and is already set, so this 364 * functions are no-ops. 365 */ 366 xen_hvm_init_shared_info_page(); 367 xen_hvm_disable_emulated_devices(); 368 } 369 370 void 371 xen_hvm_suspend(void) 372 { 373 } 374 375 void 376 xen_hvm_resume(bool suspend_cancelled) 377 { 378 379 xen_hvm_init(suspend_cancelled ? 380 XEN_HVM_INIT_CANCELLED_SUSPEND : XEN_HVM_INIT_RESUME); 381 382 /* Register vcpu_info area for CPU#0. */ 383 xen_hvm_cpu_init(); 384 } 385 386 static void 387 xen_hvm_sysinit(void *arg __unused) 388 { 389 xen_hvm_init(XEN_HVM_INIT_LATE); 390 } 391 SYSINIT(xen_hvm_init, SI_SUB_HYPERVISOR, SI_ORDER_FIRST, xen_hvm_sysinit, NULL); 392 393 static void 394 xen_hvm_cpu_init(void) 395 { 396 uint32_t regs[4]; 397 int rc; 398 399 if (!xen_domain()) 400 return; 401 402 if (DPCPU_GET(vcpu_info) != NULL) { 403 /* 404 * vcpu_info is already set. We're resuming 405 * from a failed migration and our pre-suspend 406 * configuration is still valid. 407 */ 408 return; 409 } 410 411 /* 412 * Set vCPU ID. If available fetch the ID from CPUID, if not just use 413 * the ACPI ID. 414 */ 415 KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf")); 416 cpuid_count(xen_cpuid_base + 4, 0, regs); 417 KASSERT((regs[0] & XEN_HVM_CPUID_VCPU_ID_PRESENT) || 418 !xen_pv_domain(), 419 ("Xen PV domain without vcpu_id in cpuid")); 420 PCPU_SET(vcpu_id, (regs[0] & XEN_HVM_CPUID_VCPU_ID_PRESENT) ? 421 regs[1] : PCPU_GET(acpi_id)); 422 423 if (xen_evtchn_needs_ack && !IS_BSP()) { 424 /* 425 * Setup the per-vpcu event channel upcall vector. This is only 426 * required when using the new HVMOP_set_evtchn_upcall_vector 427 * hypercall, which allows using a different vector for each 428 * vCPU. Note that FreeBSD uses the same vector for all vCPUs 429 * because it's not dynamically allocated. 430 */ 431 rc = set_percpu_callback(PCPU_GET(vcpu_id)); 432 if (rc != 0) 433 panic("Event channel upcall vector setup failed: %d", 434 rc); 435 } 436 437 xen_setup_vcpu_info(); 438 } 439 SYSINIT(xen_hvm_cpu_init, SI_SUB_INTR, SI_ORDER_FIRST, xen_hvm_cpu_init, NULL); 440 441 bool 442 xen_has_iommu_maps(void) 443 { 444 uint32_t regs[4]; 445 446 KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf")); 447 cpuid_count(xen_cpuid_base + 4, 0, regs); 448 449 return (regs[0] & XEN_HVM_CPUID_IOMMU_MAPPINGS); 450 } 451