1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/md_var.h> 45 #include <machine/smp.h> 46 47 #include <x86/apicreg.h> 48 #include <x86/apicvar.h> 49 50 #include <xen/xen-os.h> 51 #include <xen/features.h> 52 #include <xen/gnttab.h> 53 #include <xen/hypervisor.h> 54 #include <xen/hvm.h> 55 #include <xen/xen_intr.h> 56 57 #include <xen/interface/vcpu.h> 58 59 /*--------------------------------- Macros -----------------------------------*/ 60 61 #define XEN_APIC_UNSUPPORTED \ 62 panic("%s: not available in Xen PV port.", __func__) 63 64 65 /*--------------------------- Forward Declarations ---------------------------*/ 66 #ifdef SMP 67 static driver_filter_t xen_smp_rendezvous_action; 68 static driver_filter_t xen_invltlb; 69 static driver_filter_t xen_invlpg; 70 static driver_filter_t xen_invlrng; 71 static driver_filter_t xen_invlcache; 72 static driver_filter_t xen_ipi_bitmap_handler; 73 static driver_filter_t xen_cpustop_handler; 74 static driver_filter_t xen_cpususpend_handler; 75 static driver_filter_t xen_cpustophard_handler; 76 #endif 77 78 /*---------------------------------- Macros ----------------------------------*/ 79 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 80 81 /*--------------------------------- Xen IPIs ---------------------------------*/ 82 #ifdef SMP 83 struct xen_ipi_handler 84 { 85 driver_filter_t *filter; 86 const char *description; 87 }; 88 89 static struct xen_ipi_handler xen_ipis[] = 90 { 91 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 92 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 93 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 94 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 95 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 96 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 97 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 98 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 99 [IPI_TO_IDX(IPI_STOP_HARD)] = { xen_cpustophard_handler, "sth" }, 100 }; 101 #endif 102 103 /*------------------------------- Per-CPU Data -------------------------------*/ 104 #ifdef SMP 105 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 106 #endif 107 108 /*------------------------------- Xen PV APIC --------------------------------*/ 109 110 static void 111 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 112 { 113 #ifdef SMP 114 cpu_add(apic_id, boot_cpu); 115 #endif 116 } 117 118 static void 119 xen_pv_lapic_init(vm_paddr_t addr) 120 { 121 122 } 123 124 static void 125 xen_pv_lapic_setup(int boot) 126 { 127 128 } 129 130 static void 131 xen_pv_lapic_dump(const char *str) 132 { 133 134 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 135 } 136 137 static void 138 xen_pv_lapic_disable(void) 139 { 140 141 } 142 143 static bool 144 xen_pv_lapic_is_x2apic(void) 145 { 146 147 return (false); 148 } 149 150 static void 151 xen_pv_lapic_eoi(void) 152 { 153 154 XEN_APIC_UNSUPPORTED; 155 } 156 157 static int 158 xen_pv_lapic_id(void) 159 { 160 161 return (PCPU_GET(apic_id)); 162 } 163 164 static int 165 xen_pv_lapic_intr_pending(u_int vector) 166 { 167 168 XEN_APIC_UNSUPPORTED; 169 return (0); 170 } 171 172 static u_int 173 xen_pv_apic_cpuid(u_int apic_id) 174 { 175 #ifdef SMP 176 return (apic_cpuids[apic_id]); 177 #else 178 return (0); 179 #endif 180 } 181 182 static u_int 183 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 184 { 185 186 XEN_APIC_UNSUPPORTED; 187 return (0); 188 } 189 190 static u_int 191 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 192 { 193 194 XEN_APIC_UNSUPPORTED; 195 return (0); 196 } 197 198 static void 199 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 200 { 201 202 XEN_APIC_UNSUPPORTED; 203 } 204 205 static void 206 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 207 { 208 209 XEN_APIC_UNSUPPORTED; 210 } 211 212 static void 213 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 214 { 215 216 XEN_APIC_UNSUPPORTED; 217 } 218 219 static void 220 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 221 { 222 223 XEN_APIC_UNSUPPORTED; 224 } 225 226 static int 227 xen_pv_lapic_enable_pmc(void) 228 { 229 230 XEN_APIC_UNSUPPORTED; 231 return (0); 232 } 233 234 static void 235 xen_pv_lapic_disable_pmc(void) 236 { 237 238 XEN_APIC_UNSUPPORTED; 239 } 240 241 static void 242 xen_pv_lapic_reenable_pmc(void) 243 { 244 245 XEN_APIC_UNSUPPORTED; 246 } 247 248 static void 249 xen_pv_lapic_enable_cmc(void) 250 { 251 252 } 253 254 #ifdef SMP 255 static void 256 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 257 { 258 259 XEN_APIC_UNSUPPORTED; 260 } 261 262 static void 263 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 264 { 265 xen_intr_handle_t *ipi_handle; 266 int ipi_idx, to_cpu, self; 267 268 ipi_idx = IPI_TO_IDX(vector); 269 if (ipi_idx >= nitems(xen_ipis)) 270 panic("IPI out of range"); 271 272 switch(dest) { 273 case APIC_IPI_DEST_SELF: 274 ipi_handle = DPCPU_GET(ipi_handle); 275 xen_intr_signal(ipi_handle[ipi_idx]); 276 break; 277 case APIC_IPI_DEST_ALL: 278 CPU_FOREACH(to_cpu) { 279 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 280 xen_intr_signal(ipi_handle[ipi_idx]); 281 } 282 break; 283 case APIC_IPI_DEST_OTHERS: 284 self = PCPU_GET(cpuid); 285 CPU_FOREACH(to_cpu) { 286 if (to_cpu != self) { 287 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 288 xen_intr_signal(ipi_handle[ipi_idx]); 289 } 290 } 291 break; 292 default: 293 to_cpu = apic_cpuid(dest); 294 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 295 xen_intr_signal(ipi_handle[ipi_idx]); 296 break; 297 } 298 } 299 300 static int 301 xen_pv_lapic_ipi_wait(int delay) 302 { 303 304 XEN_APIC_UNSUPPORTED; 305 return (0); 306 } 307 #endif /* SMP */ 308 309 static int 310 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 311 { 312 313 XEN_APIC_UNSUPPORTED; 314 return (-1); 315 } 316 317 static void 318 xen_pv_lapic_ipi_free(int vector) 319 { 320 321 XEN_APIC_UNSUPPORTED; 322 } 323 324 static int 325 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 326 { 327 328 XEN_APIC_UNSUPPORTED; 329 return (0); 330 } 331 332 static int 333 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 334 { 335 336 XEN_APIC_UNSUPPORTED; 337 return (0); 338 } 339 340 static int 341 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 342 { 343 344 XEN_APIC_UNSUPPORTED; 345 return (0); 346 } 347 348 static int 349 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 350 enum intr_trigger trigger) 351 { 352 353 XEN_APIC_UNSUPPORTED; 354 return (0); 355 } 356 357 /* Xen apic_ops implementation */ 358 struct apic_ops xen_apic_ops = { 359 .create = xen_pv_lapic_create, 360 .init = xen_pv_lapic_init, 361 .xapic_mode = xen_pv_lapic_disable, 362 .is_x2apic = xen_pv_lapic_is_x2apic, 363 .setup = xen_pv_lapic_setup, 364 .dump = xen_pv_lapic_dump, 365 .disable = xen_pv_lapic_disable, 366 .eoi = xen_pv_lapic_eoi, 367 .id = xen_pv_lapic_id, 368 .intr_pending = xen_pv_lapic_intr_pending, 369 .set_logical_id = xen_pv_lapic_set_logical_id, 370 .cpuid = xen_pv_apic_cpuid, 371 .alloc_vector = xen_pv_apic_alloc_vector, 372 .alloc_vectors = xen_pv_apic_alloc_vectors, 373 .enable_vector = xen_pv_apic_enable_vector, 374 .disable_vector = xen_pv_apic_disable_vector, 375 .free_vector = xen_pv_apic_free_vector, 376 .enable_pmc = xen_pv_lapic_enable_pmc, 377 .disable_pmc = xen_pv_lapic_disable_pmc, 378 .reenable_pmc = xen_pv_lapic_reenable_pmc, 379 .enable_cmc = xen_pv_lapic_enable_cmc, 380 #ifdef SMP 381 .ipi_raw = xen_pv_lapic_ipi_raw, 382 .ipi_vectored = xen_pv_lapic_ipi_vectored, 383 .ipi_wait = xen_pv_lapic_ipi_wait, 384 #endif 385 .ipi_alloc = xen_pv_lapic_ipi_alloc, 386 .ipi_free = xen_pv_lapic_ipi_free, 387 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 388 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 389 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 390 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 391 }; 392 393 #ifdef SMP 394 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 395 /* 396 * These are C clones of the ASM functions found in apic_vector. 397 */ 398 static int 399 xen_ipi_bitmap_handler(void *arg) 400 { 401 struct trapframe *frame; 402 403 frame = arg; 404 ipi_bitmap_handler(*frame); 405 return (FILTER_HANDLED); 406 } 407 408 static int 409 xen_smp_rendezvous_action(void *arg) 410 { 411 #ifdef COUNT_IPIS 412 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 413 #endif /* COUNT_IPIS */ 414 415 smp_rendezvous_action(); 416 return (FILTER_HANDLED); 417 } 418 419 static int 420 xen_invltlb(void *arg) 421 { 422 423 invltlb_handler(); 424 return (FILTER_HANDLED); 425 } 426 427 #ifdef __amd64__ 428 static int 429 xen_invltlb_invpcid(void *arg) 430 { 431 432 invltlb_invpcid_handler(); 433 return (FILTER_HANDLED); 434 } 435 436 static int 437 xen_invltlb_pcid(void *arg) 438 { 439 440 invltlb_pcid_handler(); 441 return (FILTER_HANDLED); 442 } 443 444 static int 445 xen_invltlb_invpcid_pti(void *arg) 446 { 447 448 invltlb_invpcid_pti_handler(); 449 return (FILTER_HANDLED); 450 } 451 452 static int 453 xen_invlpg_invpcid_handler(void *arg) 454 { 455 456 invlpg_invpcid_handler(); 457 return (FILTER_HANDLED); 458 } 459 460 static int 461 xen_invlpg_pcid_handler(void *arg) 462 { 463 464 invlpg_pcid_handler(); 465 return (FILTER_HANDLED); 466 } 467 468 static int 469 xen_invlrng_invpcid_handler(void *arg) 470 { 471 472 invlrng_invpcid_handler(); 473 return (FILTER_HANDLED); 474 } 475 476 static int 477 xen_invlrng_pcid_handler(void *arg) 478 { 479 480 invlrng_pcid_handler(); 481 return (FILTER_HANDLED); 482 } 483 #endif 484 485 static int 486 xen_invlpg(void *arg) 487 { 488 489 invlpg_handler(); 490 return (FILTER_HANDLED); 491 } 492 493 static int 494 xen_invlrng(void *arg) 495 { 496 497 invlrng_handler(); 498 return (FILTER_HANDLED); 499 } 500 501 static int 502 xen_invlcache(void *arg) 503 { 504 505 invlcache_handler(); 506 return (FILTER_HANDLED); 507 } 508 509 static int 510 xen_cpustop_handler(void *arg) 511 { 512 513 cpustop_handler(); 514 return (FILTER_HANDLED); 515 } 516 517 static int 518 xen_cpususpend_handler(void *arg) 519 { 520 521 cpususpend_handler(); 522 return (FILTER_HANDLED); 523 } 524 525 static int 526 xen_cpustophard_handler(void *arg) 527 { 528 529 ipi_nmi_handler(); 530 return (FILTER_HANDLED); 531 } 532 533 /*----------------------------- XEN PV IPI setup -----------------------------*/ 534 /* 535 * Those functions are provided outside of the Xen PV APIC implementation 536 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 537 * because on PVHVM there's an emulated LAPIC provided by Xen. 538 */ 539 static void 540 xen_cpu_ipi_init(int cpu) 541 { 542 xen_intr_handle_t *ipi_handle; 543 const struct xen_ipi_handler *ipi; 544 int idx, rc; 545 546 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 547 548 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 549 550 if (ipi->filter == NULL) { 551 ipi_handle[idx] = NULL; 552 continue; 553 } 554 555 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 556 INTR_TYPE_TTY, &ipi_handle[idx]); 557 if (rc != 0) 558 panic("Unable to allocate a XEN IPI port"); 559 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 560 } 561 } 562 563 static void 564 xen_setup_cpus(void) 565 { 566 int i; 567 568 if (!xen_vector_callback_enabled) 569 return; 570 571 #ifdef __amd64__ 572 if (pmap_pcid_enabled) { 573 if (pti) 574 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 575 invpcid_works ? xen_invltlb_invpcid_pti : 576 xen_invltlb_pcid; 577 else 578 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 579 invpcid_works ? xen_invltlb_invpcid : 580 xen_invltlb_pcid; 581 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = invpcid_works ? 582 xen_invlpg_invpcid_handler : xen_invlpg_pcid_handler; 583 xen_ipis[IPI_TO_IDX(IPI_INVLRNG)].filter = invpcid_works ? 584 xen_invlrng_invpcid_handler : xen_invlrng_pcid_handler; 585 } 586 #endif 587 CPU_FOREACH(i) 588 xen_cpu_ipi_init(i); 589 590 /* Set the xen pv ipi ops to replace the native ones */ 591 if (xen_hvm_domain()) 592 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 593 } 594 595 /* Switch to using PV IPIs as soon as the vcpu_id is set. */ 596 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 597 #endif /* SMP */ 598