1 /* 2 * Copyright (c) 2014 Roger Pau Monné <roger.pau@citrix.com> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include <sys/param.h> 31 #include <sys/bus.h> 32 #include <sys/kernel.h> 33 #include <sys/malloc.h> 34 #include <sys/proc.h> 35 #include <sys/smp.h> 36 #include <sys/systm.h> 37 38 #include <vm/vm.h> 39 #include <vm/pmap.h> 40 41 #include <machine/cpufunc.h> 42 #include <machine/cpu.h> 43 #include <machine/intr_machdep.h> 44 #include <machine/md_var.h> 45 #include <machine/smp.h> 46 47 #include <x86/apicreg.h> 48 #include <x86/apicvar.h> 49 50 #include <xen/xen-os.h> 51 #include <xen/features.h> 52 #include <xen/gnttab.h> 53 #include <xen/hypervisor.h> 54 #include <xen/hvm.h> 55 #include <xen/xen_intr.h> 56 57 #include <xen/interface/vcpu.h> 58 59 /*--------------------------------- Macros -----------------------------------*/ 60 61 #define XEN_APIC_UNSUPPORTED \ 62 panic("%s: not available in Xen PV port.", __func__) 63 64 65 /*--------------------------- Forward Declarations ---------------------------*/ 66 #ifdef SMP 67 static driver_filter_t xen_smp_rendezvous_action; 68 static driver_filter_t xen_invltlb; 69 static driver_filter_t xen_invlpg; 70 static driver_filter_t xen_invlrng; 71 static driver_filter_t xen_invlcache; 72 static driver_filter_t xen_ipi_bitmap_handler; 73 static driver_filter_t xen_cpustop_handler; 74 static driver_filter_t xen_cpususpend_handler; 75 #endif 76 77 /*---------------------------------- Macros ----------------------------------*/ 78 #define IPI_TO_IDX(ipi) ((ipi) - APIC_IPI_INTS) 79 80 /*--------------------------------- Xen IPIs ---------------------------------*/ 81 #ifdef SMP 82 struct xen_ipi_handler 83 { 84 driver_filter_t *filter; 85 const char *description; 86 }; 87 88 static struct xen_ipi_handler xen_ipis[] = 89 { 90 [IPI_TO_IDX(IPI_RENDEZVOUS)] = { xen_smp_rendezvous_action, "r" }, 91 [IPI_TO_IDX(IPI_INVLTLB)] = { xen_invltlb, "itlb"}, 92 [IPI_TO_IDX(IPI_INVLPG)] = { xen_invlpg, "ipg" }, 93 [IPI_TO_IDX(IPI_INVLRNG)] = { xen_invlrng, "irg" }, 94 [IPI_TO_IDX(IPI_INVLCACHE)] = { xen_invlcache, "ic" }, 95 [IPI_TO_IDX(IPI_BITMAP_VECTOR)] = { xen_ipi_bitmap_handler, "b" }, 96 [IPI_TO_IDX(IPI_STOP)] = { xen_cpustop_handler, "st" }, 97 [IPI_TO_IDX(IPI_SUSPEND)] = { xen_cpususpend_handler, "sp" }, 98 }; 99 #endif 100 101 /*------------------------------- Per-CPU Data -------------------------------*/ 102 #ifdef SMP 103 DPCPU_DEFINE(xen_intr_handle_t, ipi_handle[nitems(xen_ipis)]); 104 #endif 105 106 /*------------------------------- Xen PV APIC --------------------------------*/ 107 108 static void 109 xen_pv_lapic_create(u_int apic_id, int boot_cpu) 110 { 111 #ifdef SMP 112 cpu_add(apic_id, boot_cpu); 113 #endif 114 } 115 116 static void 117 xen_pv_lapic_init(vm_paddr_t addr) 118 { 119 120 } 121 122 static void 123 xen_pv_lapic_setup(int boot) 124 { 125 126 } 127 128 static void 129 xen_pv_lapic_dump(const char *str) 130 { 131 132 printf("cpu%d %s XEN PV LAPIC\n", PCPU_GET(cpuid), str); 133 } 134 135 static void 136 xen_pv_lapic_disable(void) 137 { 138 139 } 140 141 static bool 142 xen_pv_lapic_is_x2apic(void) 143 { 144 145 return (false); 146 } 147 148 static void 149 xen_pv_lapic_eoi(void) 150 { 151 152 XEN_APIC_UNSUPPORTED; 153 } 154 155 static int 156 xen_pv_lapic_id(void) 157 { 158 159 return (PCPU_GET(apic_id)); 160 } 161 162 static int 163 xen_pv_lapic_intr_pending(u_int vector) 164 { 165 166 XEN_APIC_UNSUPPORTED; 167 return (0); 168 } 169 170 static u_int 171 xen_pv_apic_cpuid(u_int apic_id) 172 { 173 #ifdef SMP 174 return (apic_cpuids[apic_id]); 175 #else 176 return (0); 177 #endif 178 } 179 180 static u_int 181 xen_pv_apic_alloc_vector(u_int apic_id, u_int irq) 182 { 183 184 XEN_APIC_UNSUPPORTED; 185 return (0); 186 } 187 188 static u_int 189 xen_pv_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 190 { 191 192 XEN_APIC_UNSUPPORTED; 193 return (0); 194 } 195 196 static void 197 xen_pv_apic_disable_vector(u_int apic_id, u_int vector) 198 { 199 200 XEN_APIC_UNSUPPORTED; 201 } 202 203 static void 204 xen_pv_apic_enable_vector(u_int apic_id, u_int vector) 205 { 206 207 XEN_APIC_UNSUPPORTED; 208 } 209 210 static void 211 xen_pv_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 212 { 213 214 XEN_APIC_UNSUPPORTED; 215 } 216 217 static void 218 xen_pv_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 219 { 220 221 XEN_APIC_UNSUPPORTED; 222 } 223 224 static int 225 xen_pv_lapic_enable_pmc(void) 226 { 227 228 XEN_APIC_UNSUPPORTED; 229 return (0); 230 } 231 232 static void 233 xen_pv_lapic_disable_pmc(void) 234 { 235 236 XEN_APIC_UNSUPPORTED; 237 } 238 239 static void 240 xen_pv_lapic_reenable_pmc(void) 241 { 242 243 XEN_APIC_UNSUPPORTED; 244 } 245 246 static void 247 xen_pv_lapic_enable_cmc(void) 248 { 249 250 } 251 252 #ifdef SMP 253 static void 254 xen_pv_lapic_ipi_raw(register_t icrlo, u_int dest) 255 { 256 257 XEN_APIC_UNSUPPORTED; 258 } 259 260 #define PCPU_ID_GET(id, field) (pcpu_find(id)->pc_##field) 261 static void 262 send_nmi(int dest) 263 { 264 unsigned int cpu; 265 266 /* 267 * NMIs are not routed over event channels, and instead delivered as on 268 * native using the exception vector (#2). Triggering them can be done 269 * using the local APIC, or an hypercall as a shortcut like it's done 270 * below. 271 */ 272 switch(dest) { 273 case APIC_IPI_DEST_SELF: 274 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, PCPU_GET(vcpu_id), NULL); 275 break; 276 case APIC_IPI_DEST_ALL: 277 CPU_FOREACH(cpu) 278 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 279 PCPU_ID_GET(cpu, vcpu_id), NULL); 280 break; 281 case APIC_IPI_DEST_OTHERS: 282 CPU_FOREACH(cpu) 283 if (cpu != PCPU_GET(cpuid)) 284 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 285 PCPU_ID_GET(cpu, vcpu_id), NULL); 286 break; 287 default: 288 HYPERVISOR_vcpu_op(VCPUOP_send_nmi, 289 PCPU_ID_GET(apic_cpuid(dest), vcpu_id), NULL); 290 break; 291 } 292 } 293 #undef PCPU_ID_GET 294 295 static void 296 xen_pv_lapic_ipi_vectored(u_int vector, int dest) 297 { 298 xen_intr_handle_t *ipi_handle; 299 int ipi_idx, to_cpu, self; 300 301 if (vector >= IPI_NMI_FIRST) { 302 send_nmi(dest); 303 return; 304 } 305 306 ipi_idx = IPI_TO_IDX(vector); 307 if (ipi_idx >= nitems(xen_ipis)) 308 panic("IPI out of range"); 309 310 switch(dest) { 311 case APIC_IPI_DEST_SELF: 312 ipi_handle = DPCPU_GET(ipi_handle); 313 xen_intr_signal(ipi_handle[ipi_idx]); 314 break; 315 case APIC_IPI_DEST_ALL: 316 CPU_FOREACH(to_cpu) { 317 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 318 xen_intr_signal(ipi_handle[ipi_idx]); 319 } 320 break; 321 case APIC_IPI_DEST_OTHERS: 322 self = PCPU_GET(cpuid); 323 CPU_FOREACH(to_cpu) { 324 if (to_cpu != self) { 325 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 326 xen_intr_signal(ipi_handle[ipi_idx]); 327 } 328 } 329 break; 330 default: 331 to_cpu = apic_cpuid(dest); 332 ipi_handle = DPCPU_ID_GET(to_cpu, ipi_handle); 333 xen_intr_signal(ipi_handle[ipi_idx]); 334 break; 335 } 336 } 337 338 static int 339 xen_pv_lapic_ipi_wait(int delay) 340 { 341 342 XEN_APIC_UNSUPPORTED; 343 return (0); 344 } 345 #endif /* SMP */ 346 347 static int 348 xen_pv_lapic_ipi_alloc(inthand_t *ipifunc) 349 { 350 351 XEN_APIC_UNSUPPORTED; 352 return (-1); 353 } 354 355 static void 356 xen_pv_lapic_ipi_free(int vector) 357 { 358 359 XEN_APIC_UNSUPPORTED; 360 } 361 362 static int 363 xen_pv_lapic_set_lvt_mask(u_int apic_id, u_int lvt, u_char masked) 364 { 365 366 XEN_APIC_UNSUPPORTED; 367 return (0); 368 } 369 370 static int 371 xen_pv_lapic_set_lvt_mode(u_int apic_id, u_int lvt, uint32_t mode) 372 { 373 374 XEN_APIC_UNSUPPORTED; 375 return (0); 376 } 377 378 static int 379 xen_pv_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, enum intr_polarity pol) 380 { 381 382 XEN_APIC_UNSUPPORTED; 383 return (0); 384 } 385 386 static int 387 xen_pv_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 388 enum intr_trigger trigger) 389 { 390 391 XEN_APIC_UNSUPPORTED; 392 return (0); 393 } 394 395 /* Xen apic_ops implementation */ 396 struct apic_ops xen_apic_ops = { 397 .create = xen_pv_lapic_create, 398 .init = xen_pv_lapic_init, 399 .xapic_mode = xen_pv_lapic_disable, 400 .is_x2apic = xen_pv_lapic_is_x2apic, 401 .setup = xen_pv_lapic_setup, 402 .dump = xen_pv_lapic_dump, 403 .disable = xen_pv_lapic_disable, 404 .eoi = xen_pv_lapic_eoi, 405 .id = xen_pv_lapic_id, 406 .intr_pending = xen_pv_lapic_intr_pending, 407 .set_logical_id = xen_pv_lapic_set_logical_id, 408 .cpuid = xen_pv_apic_cpuid, 409 .alloc_vector = xen_pv_apic_alloc_vector, 410 .alloc_vectors = xen_pv_apic_alloc_vectors, 411 .enable_vector = xen_pv_apic_enable_vector, 412 .disable_vector = xen_pv_apic_disable_vector, 413 .free_vector = xen_pv_apic_free_vector, 414 .enable_pmc = xen_pv_lapic_enable_pmc, 415 .disable_pmc = xen_pv_lapic_disable_pmc, 416 .reenable_pmc = xen_pv_lapic_reenable_pmc, 417 .enable_cmc = xen_pv_lapic_enable_cmc, 418 #ifdef SMP 419 .ipi_raw = xen_pv_lapic_ipi_raw, 420 .ipi_vectored = xen_pv_lapic_ipi_vectored, 421 .ipi_wait = xen_pv_lapic_ipi_wait, 422 #endif 423 .ipi_alloc = xen_pv_lapic_ipi_alloc, 424 .ipi_free = xen_pv_lapic_ipi_free, 425 .set_lvt_mask = xen_pv_lapic_set_lvt_mask, 426 .set_lvt_mode = xen_pv_lapic_set_lvt_mode, 427 .set_lvt_polarity = xen_pv_lapic_set_lvt_polarity, 428 .set_lvt_triggermode = xen_pv_lapic_set_lvt_triggermode, 429 }; 430 431 #ifdef SMP 432 /*---------------------------- XEN PV IPI Handlers ---------------------------*/ 433 /* 434 * These are C clones of the ASM functions found in apic_vector. 435 */ 436 static int 437 xen_ipi_bitmap_handler(void *arg) 438 { 439 struct trapframe *frame; 440 441 frame = arg; 442 ipi_bitmap_handler(*frame); 443 return (FILTER_HANDLED); 444 } 445 446 static int 447 xen_smp_rendezvous_action(void *arg) 448 { 449 #ifdef COUNT_IPIS 450 (*ipi_rendezvous_counts[PCPU_GET(cpuid)])++; 451 #endif /* COUNT_IPIS */ 452 453 smp_rendezvous_action(); 454 return (FILTER_HANDLED); 455 } 456 457 static int 458 xen_invltlb(void *arg) 459 { 460 461 invltlb_handler(); 462 return (FILTER_HANDLED); 463 } 464 465 #ifdef __amd64__ 466 static int 467 xen_invltlb_invpcid(void *arg) 468 { 469 470 invltlb_invpcid_handler(); 471 return (FILTER_HANDLED); 472 } 473 474 static int 475 xen_invltlb_pcid(void *arg) 476 { 477 478 invltlb_pcid_handler(); 479 return (FILTER_HANDLED); 480 } 481 482 static int 483 xen_invltlb_invpcid_pti(void *arg) 484 { 485 486 invltlb_invpcid_pti_handler(); 487 return (FILTER_HANDLED); 488 } 489 490 static int 491 xen_invlpg_invpcid_handler(void *arg) 492 { 493 494 invlpg_invpcid_handler(); 495 return (FILTER_HANDLED); 496 } 497 498 static int 499 xen_invlpg_pcid_handler(void *arg) 500 { 501 502 invlpg_pcid_handler(); 503 return (FILTER_HANDLED); 504 } 505 506 static int 507 xen_invlrng_invpcid_handler(void *arg) 508 { 509 510 invlrng_invpcid_handler(); 511 return (FILTER_HANDLED); 512 } 513 514 static int 515 xen_invlrng_pcid_handler(void *arg) 516 { 517 518 invlrng_pcid_handler(); 519 return (FILTER_HANDLED); 520 } 521 #endif 522 523 static int 524 xen_invlpg(void *arg) 525 { 526 527 invlpg_handler(); 528 return (FILTER_HANDLED); 529 } 530 531 static int 532 xen_invlrng(void *arg) 533 { 534 535 invlrng_handler(); 536 return (FILTER_HANDLED); 537 } 538 539 static int 540 xen_invlcache(void *arg) 541 { 542 543 invlcache_handler(); 544 return (FILTER_HANDLED); 545 } 546 547 static int 548 xen_cpustop_handler(void *arg) 549 { 550 551 cpustop_handler(); 552 return (FILTER_HANDLED); 553 } 554 555 static int 556 xen_cpususpend_handler(void *arg) 557 { 558 559 cpususpend_handler(); 560 return (FILTER_HANDLED); 561 } 562 563 /*----------------------------- XEN PV IPI setup -----------------------------*/ 564 /* 565 * Those functions are provided outside of the Xen PV APIC implementation 566 * so PVHVM guests can also use PV IPIs without having an actual Xen PV APIC, 567 * because on PVHVM there's an emulated LAPIC provided by Xen. 568 */ 569 static void 570 xen_cpu_ipi_init(int cpu) 571 { 572 xen_intr_handle_t *ipi_handle; 573 const struct xen_ipi_handler *ipi; 574 int idx, rc; 575 576 ipi_handle = DPCPU_ID_GET(cpu, ipi_handle); 577 578 for (ipi = xen_ipis, idx = 0; idx < nitems(xen_ipis); ipi++, idx++) { 579 580 if (ipi->filter == NULL) { 581 ipi_handle[idx] = NULL; 582 continue; 583 } 584 585 rc = xen_intr_alloc_and_bind_ipi(cpu, ipi->filter, 586 INTR_TYPE_TTY, &ipi_handle[idx]); 587 if (rc != 0) 588 panic("Unable to allocate a XEN IPI port"); 589 xen_intr_describe(ipi_handle[idx], "%s", ipi->description); 590 } 591 } 592 593 static void 594 xen_setup_cpus(void) 595 { 596 int i; 597 598 if (!xen_vector_callback_enabled) 599 return; 600 601 #ifdef __amd64__ 602 if (pmap_pcid_enabled) { 603 if (pti) 604 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 605 invpcid_works ? xen_invltlb_invpcid_pti : 606 xen_invltlb_pcid; 607 else 608 xen_ipis[IPI_TO_IDX(IPI_INVLTLB)].filter = 609 invpcid_works ? xen_invltlb_invpcid : 610 xen_invltlb_pcid; 611 xen_ipis[IPI_TO_IDX(IPI_INVLPG)].filter = invpcid_works ? 612 xen_invlpg_invpcid_handler : xen_invlpg_pcid_handler; 613 xen_ipis[IPI_TO_IDX(IPI_INVLRNG)].filter = invpcid_works ? 614 xen_invlrng_invpcid_handler : xen_invlrng_pcid_handler; 615 } 616 #endif 617 CPU_FOREACH(i) 618 xen_cpu_ipi_init(i); 619 620 /* Set the xen pv ipi ops to replace the native ones */ 621 if (xen_hvm_domain()) 622 apic_ops.ipi_vectored = xen_pv_lapic_ipi_vectored; 623 } 624 625 /* Switch to using PV IPIs as soon as the vcpu_id is set. */ 626 SYSINIT(xen_setup_cpus, SI_SUB_SMP, SI_ORDER_SECOND, xen_setup_cpus, NULL); 627 #endif /* SMP */ 628