1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 #include <sys/param.h> 29 #include <sys/bus.h> 30 #include <sys/kernel.h> 31 #include <sys/malloc.h> 32 #include <sys/proc.h> 33 #include <sys/smp.h> 34 #include <sys/systm.h> 35 36 #include <vm/vm.h> 37 #include <vm/pmap.h> 38 39 #include <machine/cpufunc.h> 40 #include <machine/cpu.h> 41 #include <machine/intr_machdep.h> 42 #include <machine/md_var.h> 43 #include <machine/smp.h> 44 45 #include <x86/apicreg.h> 46 #include <x86/apicvar.h> 47 48 #include <xen/xen-os.h> 49 #include <xen/features.h> 50 #include <xen/gnttab.h> 51 #include <xen/hypervisor.h> 52 #include <xen/hvm.h> 53 #include <xen/xen_intr.h> 54 55 #include <contrib/xen/arch-x86/cpuid.h> 56 #include <contrib/xen/vcpu.h> 57 58 /*--------------------------- Forward Declarations ---------------------------*/ 59 static driver_filter_t xen_smp_rendezvous_action; 60 #ifdef __amd64__ 61 static driver_filter_t xen_invlop; 62 #else 63 static driver_filter_t xen_invltlb; 64 static driver_filter_t xen_invlpg; 65 static driver_filter_t xen_invlrng; 66 static driver_filter_t xen_invlcache; 67 #endif 68 static driver_filter_t xen_ipi_bitmap_handler; 69 static driver_filter_t xen_cpustop_handler; 70 static driver_filter_t xen_cpususpend_handler; 71 static driver_filter_t xen_ipi_swi_handler; 72 73 /*---------------------------------- Macros ----------------------------------*/ 74 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 75 76 /*--------------------------------- Xen IPIs ---------------------------------*/ 77 struct xen_ipi_handler 78 { 79 driver_filter_t *filter; 80 const char *description; 81 }; 82 83 static struct xen_ipi_handler xen_ipis[] = 84 { 85 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 86 #ifdef __amd64__ 87 [IPI_TO_IDX(IPI_INVLOP)] = { xen_invlop, "itlb"}, 88 #else 89 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 90 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 91 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 92 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 93 #endif 94 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 95 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 96 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 97 [IPI_TO_IDX(IPI_SWI)] = { xen_ipi_swi_handler, "sw" }, 98 }; 99 100 /* 101 * Save previous (native) handler as a fallback. Xen < 4.7 doesn't support 102 * VCPUOP_send_nmi for HVM guests, and thus we need a fallback in that case: 103 * 104 * https://lists.freebsd.org/archives/freebsd-xen/2022-January/000032.html 105 */ 106 void (*native_ipi_vectored)(u_int, int); 107 108 /*------------------------------- Per-CPU Data -------------------------------*/ 109 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 110 111 /*------------------------------- Xen PV APIC --------------------------------*/ 112 113 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 114 static int 115 send_nmi(int dest) 116 { 117 unsigned int cpu; 118 int rc = 0; 119 120 /* 121 * NMIs are not routed over event channels, and instead delivered as on 122 * native using the exception vector (#2). Triggering them can be done 123 * using the local APIC, or an hypercall as a shortcut like it's done 124 * below. 125 */ 126 switch(dest) { 127 case APIC_IPI_DEST_SELF: 128 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 129 break; 130 case APIC_IPI_DEST_ALL: 131 CPU_FOREACH(cpu) { 132 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 133 PCPU_ID_GET(cpu, vcpu_id), NULL); 134 if (rc != 0) 135 break; 136 } 137 break; 138 case APIC_IPI_DEST_OTHERS: 139 CPU_FOREACH(cpu) { 140 if (cpu != PCPU_GET(cpuid)) { 141 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 142 PCPU_ID_GET(cpu, vcpu_id), NULL); 143 if (rc != 0) 144 break; 145 } 146 } 147 break; 148 default: 149 rc = HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 150 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 151 break; 152 } 153 154 return rc; 155 } 156 #undef PCPU_ID_GET 157 158 static void 159 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 160 { 161 xen_intr_handle_t *ipi_handle; 162 int ipi_idx, to_cpu, self; 163 static bool pvnmi = true; 164 165 if (vector >= IPI_NMI_FIRST) { 166 if (pvnmi) { 167 int rc = send_nmi(dest); 168 169 if (rc != 0) { 170 printf( 171 "Sending NMI using hypercall failed (%d) switching to APIC\n", rc); 172 pvnmi = false; 173 native_ipi_vectored(vector, dest); 174 } 175 } else 176 native_ipi_vectored(vector, dest); 177 178 return; 179 } 180 181 ipi_idx = IPI_TO_IDX(vector); 182 if (ipi_idx >= nitems(xen_ipis)) 183 panic("IPI out of range"); 184 185 switch(dest) { 186 case APIC_IPI_DEST_SELF: 187 ipi_handle = DPCPU_GET(ipi_handle); 188 xen_intr_signal(ipi_handle[ipi_idx]); 189 break; 190 case APIC_IPI_DEST_ALL: 191 CPU_FOREACH(to_cpu) { 192 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 193 xen_intr_signal(ipi_handle[ipi_idx]); 194 } 195 break; 196 case APIC_IPI_DEST_OTHERS: 197 self = PCPU_GET(cpuid); 198 CPU_FOREACH(to_cpu) { 199 if (to_cpu != self) { 200 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 201 xen_intr_signal(ipi_handle[ipi_idx]); 202 } 203 } 204 break; 205 default: 206 to_cpu = apic_cpuid(dest); 207 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 208 xen_intr_signal(ipi_handle[ipi_idx]); 209 break; 210 } 211 } 212 213 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 214 /* 215 * These are C clones of the ASM functions found in apic_vector. 216 */ 217 static int 218 xen_ipi_bitmap_handler(void *arg) 219 { 220 struct trapframe *frame; 221 222 frame = arg; 223 ipi_bitmap_handler(*frame); 224 return (FILTER_HANDLED); 225 } 226 227 static int 228 xen_smp_rendezvous_action(void *arg) 229 { 230 #ifdef COUNT_IPIS 231 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 232 #endif /* COUNT_IPIS */ 233 234 smp_rendezvous_action(); 235 return (FILTER_HANDLED); 236 } 237 238 #ifdef __amd64__ 239 static int 240 xen_invlop(void *arg) 241 { 242 243 invlop_handler(); 244 return (FILTER_HANDLED); 245 } 246 247 #else /* __i386__ */ 248 249 static int 250 xen_invltlb(void *arg) 251 { 252 253 invltlb_handler(); 254 return (FILTER_HANDLED); 255 } 256 257 static int 258 xen_invlpg(void *arg) 259 { 260 261 invlpg_handler(); 262 return (FILTER_HANDLED); 263 } 264 265 static int 266 xen_invlrng(void *arg) 267 { 268 269 invlrng_handler(); 270 return (FILTER_HANDLED); 271 } 272 273 static int 274 xen_invlcache(void *arg) 275 { 276 277 invlcache_handler(); 278 return (FILTER_HANDLED); 279 } 280 #endif /* __amd64__ */ 281 282 static int 283 xen_cpustop_handler(void *arg) 284 { 285 286 cpustop_handler(); 287 return (FILTER_HANDLED); 288 } 289 290 static int 291 xen_cpususpend_handler(void *arg) 292 { 293 294 cpususpend_handler(); 295 return (FILTER_HANDLED); 296 } 297 298 static int 299 xen_ipi_swi_handler(void *arg) 300 { 301 struct trapframe *frame = arg; 302 303 ipi_swi_handler(*frame); 304 return (FILTER_HANDLED); 305 } 306 307 /*----------------------------- XEN PV IPI setup -----------------------------*/ 308 /* 309 * Those functions are provided outside of the Xen PV APIC implementation 310 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 311 * because on PVHVM there's an emulated LAPIC provided by Xen. 312 */ 313 static void 314 xen_cpu_ipi_init(int cpu) 315 { 316 xen_intr_handle_t *ipi_handle; 317 const struct xen_ipi_handler *ipi; 318 int idx, rc; 319 320 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 321 322 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 323 if (ipi->filter == NULL) { 324 ipi_handle[idx] = NULL; 325 continue; 326 } 327 328 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 329 INTR_TYPE_TTY, &ipi_handle[idx]); 330 if (rc != 0) 331 panic("Unable to allocate a XEN IPI port"); 332 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 333 } 334 } 335 336 static void 337 xen_setup_cpus(void) 338 { 339 uint32_t regs[4]; 340 int i; 341 342 if (!xen_vector_callback_enabled) 343 return; 344 345 /* 346 * Check whether the APIC virtualization is hardware assisted, as 347 * that's faster than using event channels because it avoids the VM 348 * exit. 349 */ 350 KASSERT(xen_cpuid_base != 0, ("Invalid base Xen CPUID leaf")); 351 cpuid_count(xen_cpuid_base + 4, 0, regs); 352 if ((x2apic_mode && (regs[0] & XEN_HVM_CPUID_X2APIC_VIRT)) || 353 (!x2apic_mode && (regs[0] & XEN_HVM_CPUID_APIC_ACCESS_VIRT))) 354 return; 355 356 CPU_FOREACH(i) 357 xen_cpu_ipi_init(i); 358 359 /* Set the xen pv ipi ops to replace the native ones */ 360 ipi_vectored = xen_pv_lapic_ipi_vectored; 361 native_ipi_vectored = ipi_vectored; 362 } 363 364 /* Switch to using PV IPIs as soon as the vcpu_id is set. */ 365 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 366