1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 5 * Copyright (c) 1996, by Steve Passe 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. The name of the developer may NOT be used to endorse or promote products 14 * derived from this software without specific prior written permission. 15 * 3. Neither the name of the author nor the names of any co-contributors 16 * may be used to endorse or promote products derived from this software 17 * without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * Local APIC support on Pentium and later processors. 34 */ 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include "opt_atpic.h" 40 #include "opt_hwpmc_hooks.h" 41 42 #include "opt_ddb.h" 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/bus.h> 47 #include <sys/kernel.h> 48 #include <sys/lock.h> 49 #include <sys/malloc.h> 50 #include <sys/mutex.h> 51 #include <sys/pcpu.h> 52 #include <sys/proc.h> 53 #include <sys/sched.h> 54 #include <sys/smp.h> 55 #include <sys/sysctl.h> 56 #include <sys/timeet.h> 57 58 #include <vm/vm.h> 59 #include <vm/pmap.h> 60 61 #include <x86/apicreg.h> 62 #include <machine/clock.h> 63 #include <machine/cpufunc.h> 64 #include <machine/cputypes.h> 65 #include <machine/frame.h> 66 #include <machine/intr_machdep.h> 67 #include <x86/apicvar.h> 68 #include <x86/mca.h> 69 #include <machine/md_var.h> 70 #include <machine/smp.h> 71 #include <machine/specialreg.h> 72 #include <x86/init.h> 73 74 #ifdef DDB 75 #include <sys/interrupt.h> 76 #include <ddb/ddb.h> 77 #endif 78 79 #ifdef __amd64__ 80 #define SDT_APIC SDT_SYSIGT 81 #define GSEL_APIC 0 82 #else 83 #define SDT_APIC SDT_SYS386IGT 84 #define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL) 85 #endif 86 87 static MALLOC_DEFINE(M_LAPIC, "local_apic", "Local APIC items"); 88 89 /* Sanity checks on IDT vectors. */ 90 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); 91 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); 92 CTASSERT(APIC_LOCAL_INTS == 240); 93 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT); 94 95 /* 96 * I/O interrupts use non-negative IRQ values. These values are used 97 * to mark unused IDT entries or IDT entries reserved for a non-I/O 98 * interrupt. 99 */ 100 #define IRQ_FREE -1 101 #define IRQ_TIMER -2 102 #define IRQ_SYSCALL -3 103 #define IRQ_DTRACE_RET -4 104 #define IRQ_EVTCHN -5 105 106 enum lat_timer_mode { 107 LAT_MODE_UNDEF = 0, 108 LAT_MODE_PERIODIC = 1, 109 LAT_MODE_ONESHOT = 2, 110 LAT_MODE_DEADLINE = 3, 111 }; 112 113 /* 114 * Support for local APICs. Local APICs manage interrupts on each 115 * individual processor as opposed to I/O APICs which receive interrupts 116 * from I/O devices and then forward them on to the local APICs. 117 * 118 * Local APICs can also send interrupts to each other thus providing the 119 * mechanism for IPIs. 120 */ 121 122 struct lvt { 123 u_int lvt_edgetrigger:1; 124 u_int lvt_activehi:1; 125 u_int lvt_masked:1; 126 u_int lvt_active:1; 127 u_int lvt_mode:16; 128 u_int lvt_vector:8; 129 }; 130 131 struct lapic { 132 struct lvt la_lvts[APIC_LVT_MAX + 1]; 133 struct lvt la_elvts[APIC_ELVT_MAX + 1];; 134 u_int la_id:8; 135 u_int la_cluster:4; 136 u_int la_cluster_id:2; 137 u_int la_present:1; 138 u_long *la_timer_count; 139 uint64_t la_timer_period; 140 enum lat_timer_mode la_timer_mode; 141 uint32_t lvt_timer_base; 142 uint32_t lvt_timer_last; 143 /* Include IDT_SYSCALL to make indexing easier. */ 144 int la_ioint_irqs[APIC_NUM_IOINTS + 1]; 145 } static *lapics; 146 147 /* Global defaults for local APIC LVT entries. */ 148 static struct lvt lvts[APIC_LVT_MAX + 1] = { 149 { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 }, /* LINT0: masked ExtINT */ 150 { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 }, /* LINT1: NMI */ 151 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT }, /* Timer */ 152 { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT }, /* Error */ 153 { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 }, /* PMC */ 154 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT }, /* Thermal */ 155 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT }, /* CMCI */ 156 }; 157 158 /* Global defaults for AMD local APIC ELVT entries. */ 159 static struct lvt elvts[APIC_ELVT_MAX + 1] = { 160 { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 }, 161 { 1, 1, 1, 0, APIC_LVT_DM_FIXED, APIC_CMC_INT }, 162 { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 }, 163 { 1, 1, 1, 0, APIC_LVT_DM_FIXED, 0 }, 164 }; 165 166 static inthand_t *ioint_handlers[] = { 167 NULL, /* 0 - 31 */ 168 IDTVEC(apic_isr1), /* 32 - 63 */ 169 IDTVEC(apic_isr2), /* 64 - 95 */ 170 IDTVEC(apic_isr3), /* 96 - 127 */ 171 IDTVEC(apic_isr4), /* 128 - 159 */ 172 IDTVEC(apic_isr5), /* 160 - 191 */ 173 IDTVEC(apic_isr6), /* 192 - 223 */ 174 IDTVEC(apic_isr7), /* 224 - 255 */ 175 }; 176 177 static inthand_t *ioint_pti_handlers[] = { 178 NULL, /* 0 - 31 */ 179 IDTVEC(apic_isr1_pti), /* 32 - 63 */ 180 IDTVEC(apic_isr2_pti), /* 64 - 95 */ 181 IDTVEC(apic_isr3_pti), /* 96 - 127 */ 182 IDTVEC(apic_isr4_pti), /* 128 - 159 */ 183 IDTVEC(apic_isr5_pti), /* 160 - 191 */ 184 IDTVEC(apic_isr6_pti), /* 192 - 223 */ 185 IDTVEC(apic_isr7_pti), /* 224 - 255 */ 186 }; 187 188 static u_int32_t lapic_timer_divisors[] = { 189 APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, 190 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128 191 }; 192 193 extern inthand_t IDTVEC(rsvd_pti), IDTVEC(rsvd); 194 195 volatile char *lapic_map; 196 vm_paddr_t lapic_paddr; 197 int x2apic_mode; 198 int lapic_eoi_suppression; 199 static int lapic_timer_tsc_deadline; 200 static u_long lapic_timer_divisor, count_freq; 201 static struct eventtimer lapic_et; 202 #ifdef SMP 203 static uint64_t lapic_ipi_wait_mult; 204 #endif 205 unsigned int max_apic_id; 206 207 SYSCTL_NODE(_hw, OID_AUTO, apic, CTLFLAG_RD, 0, "APIC options"); 208 SYSCTL_INT(_hw_apic, OID_AUTO, x2apic_mode, CTLFLAG_RD, &x2apic_mode, 0, ""); 209 SYSCTL_INT(_hw_apic, OID_AUTO, eoi_suppression, CTLFLAG_RD, 210 &lapic_eoi_suppression, 0, ""); 211 SYSCTL_INT(_hw_apic, OID_AUTO, timer_tsc_deadline, CTLFLAG_RD, 212 &lapic_timer_tsc_deadline, 0, ""); 213 214 static void lapic_calibrate_initcount(struct lapic *la); 215 static void lapic_calibrate_deadline(struct lapic *la); 216 217 static uint32_t 218 lapic_read32(enum LAPIC_REGISTERS reg) 219 { 220 uint32_t res; 221 222 if (x2apic_mode) { 223 res = rdmsr32(MSR_APIC_000 + reg); 224 } else { 225 res = *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL); 226 } 227 return (res); 228 } 229 230 static void 231 lapic_write32(enum LAPIC_REGISTERS reg, uint32_t val) 232 { 233 234 if (x2apic_mode) { 235 mfence(); 236 lfence(); 237 wrmsr(MSR_APIC_000 + reg, val); 238 } else { 239 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val; 240 } 241 } 242 243 static void 244 lapic_write32_nofence(enum LAPIC_REGISTERS reg, uint32_t val) 245 { 246 247 if (x2apic_mode) { 248 wrmsr(MSR_APIC_000 + reg, val); 249 } else { 250 *(volatile uint32_t *)(lapic_map + reg * LAPIC_MEM_MUL) = val; 251 } 252 } 253 254 #ifdef SMP 255 static uint64_t 256 lapic_read_icr(void) 257 { 258 uint64_t v; 259 uint32_t vhi, vlo; 260 261 if (x2apic_mode) { 262 v = rdmsr(MSR_APIC_000 + LAPIC_ICR_LO); 263 } else { 264 vhi = lapic_read32(LAPIC_ICR_HI); 265 vlo = lapic_read32(LAPIC_ICR_LO); 266 v = ((uint64_t)vhi << 32) | vlo; 267 } 268 return (v); 269 } 270 271 static uint64_t 272 lapic_read_icr_lo(void) 273 { 274 275 return (lapic_read32(LAPIC_ICR_LO)); 276 } 277 278 static void 279 lapic_write_icr(uint32_t vhi, uint32_t vlo) 280 { 281 uint64_t v; 282 283 if (x2apic_mode) { 284 v = ((uint64_t)vhi << 32) | vlo; 285 mfence(); 286 wrmsr(MSR_APIC_000 + LAPIC_ICR_LO, v); 287 } else { 288 lapic_write32(LAPIC_ICR_HI, vhi); 289 lapic_write32(LAPIC_ICR_LO, vlo); 290 } 291 } 292 #endif /* SMP */ 293 294 static void 295 native_lapic_enable_x2apic(void) 296 { 297 uint64_t apic_base; 298 299 apic_base = rdmsr(MSR_APICBASE); 300 apic_base |= APICBASE_X2APIC | APICBASE_ENABLED; 301 wrmsr(MSR_APICBASE, apic_base); 302 } 303 304 static bool 305 native_lapic_is_x2apic(void) 306 { 307 uint64_t apic_base; 308 309 apic_base = rdmsr(MSR_APICBASE); 310 return ((apic_base & (APICBASE_X2APIC | APICBASE_ENABLED)) == 311 (APICBASE_X2APIC | APICBASE_ENABLED)); 312 } 313 314 static void lapic_enable(void); 315 static void lapic_resume(struct pic *pic, bool suspend_cancelled); 316 static void lapic_timer_oneshot(struct lapic *); 317 static void lapic_timer_oneshot_nointr(struct lapic *, uint32_t); 318 static void lapic_timer_periodic(struct lapic *); 319 static void lapic_timer_deadline(struct lapic *); 320 static void lapic_timer_stop(struct lapic *); 321 static void lapic_timer_set_divisor(u_int divisor); 322 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value); 323 static int lapic_et_start(struct eventtimer *et, 324 sbintime_t first, sbintime_t period); 325 static int lapic_et_stop(struct eventtimer *et); 326 static u_int apic_idt_to_irq(u_int apic_id, u_int vector); 327 static void lapic_set_tpr(u_int vector); 328 329 struct pic lapic_pic = { .pic_resume = lapic_resume }; 330 331 /* Forward declarations for apic_ops */ 332 static void native_lapic_create(u_int apic_id, int boot_cpu); 333 static void native_lapic_init(vm_paddr_t addr); 334 static void native_lapic_xapic_mode(void); 335 static void native_lapic_setup(int boot); 336 static void native_lapic_dump(const char *str); 337 static void native_lapic_disable(void); 338 static void native_lapic_eoi(void); 339 static int native_lapic_id(void); 340 static int native_lapic_intr_pending(u_int vector); 341 static u_int native_apic_cpuid(u_int apic_id); 342 static u_int native_apic_alloc_vector(u_int apic_id, u_int irq); 343 static u_int native_apic_alloc_vectors(u_int apic_id, u_int *irqs, 344 u_int count, u_int align); 345 static void native_apic_disable_vector(u_int apic_id, u_int vector); 346 static void native_apic_enable_vector(u_int apic_id, u_int vector); 347 static void native_apic_free_vector(u_int apic_id, u_int vector, u_int irq); 348 static void native_lapic_set_logical_id(u_int apic_id, u_int cluster, 349 u_int cluster_id); 350 static int native_lapic_enable_pmc(void); 351 static void native_lapic_disable_pmc(void); 352 static void native_lapic_reenable_pmc(void); 353 static void native_lapic_enable_cmc(void); 354 static int native_lapic_enable_mca_elvt(void); 355 static int native_lapic_set_lvt_mask(u_int apic_id, u_int lvt, 356 u_char masked); 357 static int native_lapic_set_lvt_mode(u_int apic_id, u_int lvt, 358 uint32_t mode); 359 static int native_lapic_set_lvt_polarity(u_int apic_id, u_int lvt, 360 enum intr_polarity pol); 361 static int native_lapic_set_lvt_triggermode(u_int apic_id, u_int lvt, 362 enum intr_trigger trigger); 363 #ifdef SMP 364 static void native_lapic_ipi_raw(register_t icrlo, u_int dest); 365 static void native_lapic_ipi_vectored(u_int vector, int dest); 366 static int native_lapic_ipi_wait(int delay); 367 #endif /* SMP */ 368 static int native_lapic_ipi_alloc(inthand_t *ipifunc); 369 static void native_lapic_ipi_free(int vector); 370 371 struct apic_ops apic_ops = { 372 .create = native_lapic_create, 373 .init = native_lapic_init, 374 .xapic_mode = native_lapic_xapic_mode, 375 .is_x2apic = native_lapic_is_x2apic, 376 .setup = native_lapic_setup, 377 .dump = native_lapic_dump, 378 .disable = native_lapic_disable, 379 .eoi = native_lapic_eoi, 380 .id = native_lapic_id, 381 .intr_pending = native_lapic_intr_pending, 382 .set_logical_id = native_lapic_set_logical_id, 383 .cpuid = native_apic_cpuid, 384 .alloc_vector = native_apic_alloc_vector, 385 .alloc_vectors = native_apic_alloc_vectors, 386 .enable_vector = native_apic_enable_vector, 387 .disable_vector = native_apic_disable_vector, 388 .free_vector = native_apic_free_vector, 389 .enable_pmc = native_lapic_enable_pmc, 390 .disable_pmc = native_lapic_disable_pmc, 391 .reenable_pmc = native_lapic_reenable_pmc, 392 .enable_cmc = native_lapic_enable_cmc, 393 .enable_mca_elvt = native_lapic_enable_mca_elvt, 394 #ifdef SMP 395 .ipi_raw = native_lapic_ipi_raw, 396 .ipi_vectored = native_lapic_ipi_vectored, 397 .ipi_wait = native_lapic_ipi_wait, 398 #endif 399 .ipi_alloc = native_lapic_ipi_alloc, 400 .ipi_free = native_lapic_ipi_free, 401 .set_lvt_mask = native_lapic_set_lvt_mask, 402 .set_lvt_mode = native_lapic_set_lvt_mode, 403 .set_lvt_polarity = native_lapic_set_lvt_polarity, 404 .set_lvt_triggermode = native_lapic_set_lvt_triggermode, 405 }; 406 407 static uint32_t 408 lvt_mode_impl(struct lapic *la, struct lvt *lvt, u_int pin, uint32_t value) 409 { 410 411 value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM | 412 APIC_LVT_VECTOR); 413 if (lvt->lvt_edgetrigger == 0) 414 value |= APIC_LVT_TM; 415 if (lvt->lvt_activehi == 0) 416 value |= APIC_LVT_IIPP_INTALO; 417 if (lvt->lvt_masked) 418 value |= APIC_LVT_M; 419 value |= lvt->lvt_mode; 420 switch (lvt->lvt_mode) { 421 case APIC_LVT_DM_NMI: 422 case APIC_LVT_DM_SMI: 423 case APIC_LVT_DM_INIT: 424 case APIC_LVT_DM_EXTINT: 425 if (!lvt->lvt_edgetrigger && bootverbose) { 426 printf("lapic%u: Forcing LINT%u to edge trigger\n", 427 la->la_id, pin); 428 value &= ~APIC_LVT_TM; 429 } 430 /* Use a vector of 0. */ 431 break; 432 case APIC_LVT_DM_FIXED: 433 value |= lvt->lvt_vector; 434 break; 435 default: 436 panic("bad APIC LVT delivery mode: %#x\n", value); 437 } 438 return (value); 439 } 440 441 static uint32_t 442 lvt_mode(struct lapic *la, u_int pin, uint32_t value) 443 { 444 struct lvt *lvt; 445 446 KASSERT(pin <= APIC_LVT_MAX, 447 ("%s: pin %u out of range", __func__, pin)); 448 if (la->la_lvts[pin].lvt_active) 449 lvt = &la->la_lvts[pin]; 450 else 451 lvt = &lvts[pin]; 452 453 return (lvt_mode_impl(la, lvt, pin, value)); 454 } 455 456 static uint32_t 457 elvt_mode(struct lapic *la, u_int idx, uint32_t value) 458 { 459 struct lvt *elvt; 460 461 KASSERT(idx <= APIC_ELVT_MAX, 462 ("%s: idx %u out of range", __func__, idx)); 463 464 elvt = &la->la_elvts[idx]; 465 KASSERT(elvt->lvt_active, ("%s: ELVT%u is not active", __func__, idx)); 466 KASSERT(elvt->lvt_edgetrigger, 467 ("%s: ELVT%u is not edge triggered", __func__, idx)); 468 KASSERT(elvt->lvt_activehi, 469 ("%s: ELVT%u is not active high", __func__, idx)); 470 return (lvt_mode_impl(la, elvt, idx, value)); 471 } 472 473 /* 474 * Map the local APIC and setup necessary interrupt vectors. 475 */ 476 static void 477 native_lapic_init(vm_paddr_t addr) 478 { 479 #ifdef SMP 480 uint64_t r, r1, r2, rx; 481 #endif 482 uint32_t ver; 483 u_int regs[4]; 484 int i, arat; 485 486 /* 487 * Enable x2APIC mode if possible. Map the local APIC 488 * registers page. 489 * 490 * Keep the LAPIC registers page mapped uncached for x2APIC 491 * mode too, to have direct map page attribute set to 492 * uncached. This is needed to work around CPU errata present 493 * on all Intel processors. 494 */ 495 KASSERT(trunc_page(addr) == addr, 496 ("local APIC not aligned on a page boundary")); 497 lapic_paddr = addr; 498 lapic_map = pmap_mapdev(addr, PAGE_SIZE); 499 if (x2apic_mode) { 500 native_lapic_enable_x2apic(); 501 lapic_map = NULL; 502 } 503 504 /* Setup the spurious interrupt handler. */ 505 setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL, 506 GSEL_APIC); 507 508 /* Perform basic initialization of the BSP's local APIC. */ 509 lapic_enable(); 510 511 /* Set BSP's per-CPU local APIC ID. */ 512 PCPU_SET(apic_id, lapic_id()); 513 514 /* Local APIC timer interrupt. */ 515 setidt(APIC_TIMER_INT, pti ? IDTVEC(timerint_pti) : IDTVEC(timerint), 516 SDT_APIC, SEL_KPL, GSEL_APIC); 517 518 /* Local APIC error interrupt. */ 519 setidt(APIC_ERROR_INT, pti ? IDTVEC(errorint_pti) : IDTVEC(errorint), 520 SDT_APIC, SEL_KPL, GSEL_APIC); 521 522 /* XXX: Thermal interrupt */ 523 524 /* Local APIC CMCI. */ 525 setidt(APIC_CMC_INT, pti ? IDTVEC(cmcint_pti) : IDTVEC(cmcint), 526 SDT_APIC, SEL_KPL, GSEL_APIC); 527 528 if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) { 529 arat = 0; 530 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */ 531 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) { 532 do_cpuid(0x06, regs); 533 if ((regs[0] & CPUTPM1_ARAT) != 0) 534 arat = 1; 535 } else if (cpu_vendor_id == CPU_VENDOR_AMD && 536 CPUID_TO_FAMILY(cpu_id) >= 0x12) { 537 arat = 1; 538 } 539 bzero(&lapic_et, sizeof(lapic_et)); 540 lapic_et.et_name = "LAPIC"; 541 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | 542 ET_FLAGS_PERCPU; 543 lapic_et.et_quality = 600; 544 if (!arat) { 545 lapic_et.et_flags |= ET_FLAGS_C3STOP; 546 lapic_et.et_quality = 100; 547 } 548 if ((cpu_feature & CPUID_TSC) != 0 && 549 (cpu_feature2 & CPUID2_TSCDLT) != 0 && 550 tsc_is_invariant && tsc_freq != 0) { 551 lapic_timer_tsc_deadline = 1; 552 TUNABLE_INT_FETCH("hw.lapic_tsc_deadline", 553 &lapic_timer_tsc_deadline); 554 } 555 556 lapic_et.et_frequency = 0; 557 /* We don't know frequency yet, so trying to guess. */ 558 lapic_et.et_min_period = 0x00001000LL; 559 lapic_et.et_max_period = SBT_1S; 560 lapic_et.et_start = lapic_et_start; 561 lapic_et.et_stop = lapic_et_stop; 562 lapic_et.et_priv = NULL; 563 et_register(&lapic_et); 564 } 565 566 /* 567 * Set lapic_eoi_suppression after lapic_enable(), to not 568 * enable suppression in the hardware prematurely. Note that 569 * we by default enable suppression even when system only has 570 * one IO-APIC, since EOI is broadcasted to all APIC agents, 571 * including CPUs, otherwise. 572 * 573 * It seems that at least some KVM versions report 574 * EOI_SUPPRESSION bit, but auto-EOI does not work. 575 */ 576 ver = lapic_read32(LAPIC_VERSION); 577 if ((ver & APIC_VER_EOI_SUPPRESSION) != 0) { 578 lapic_eoi_suppression = 1; 579 if (vm_guest == VM_GUEST_KVM) { 580 if (bootverbose) 581 printf( 582 "KVM -- disabling lapic eoi suppression\n"); 583 lapic_eoi_suppression = 0; 584 } 585 TUNABLE_INT_FETCH("hw.lapic_eoi_suppression", 586 &lapic_eoi_suppression); 587 } 588 589 #ifdef SMP 590 #define LOOPS 100000 591 /* 592 * Calibrate the busy loop waiting for IPI ack in xAPIC mode. 593 * lapic_ipi_wait_mult contains the number of iterations which 594 * approximately delay execution for 1 microsecond (the 595 * argument to native_lapic_ipi_wait() is in microseconds). 596 * 597 * We assume that TSC is present and already measured. 598 * Possible TSC frequency jumps are irrelevant to the 599 * calibration loop below, the CPU clock management code is 600 * not yet started, and we do not enter sleep states. 601 */ 602 KASSERT((cpu_feature & CPUID_TSC) != 0 && tsc_freq != 0, 603 ("TSC not initialized")); 604 if (!x2apic_mode) { 605 r = rdtsc(); 606 for (rx = 0; rx < LOOPS; rx++) { 607 (void)lapic_read_icr_lo(); 608 ia32_pause(); 609 } 610 r = rdtsc() - r; 611 r1 = tsc_freq * LOOPS; 612 r2 = r * 1000000; 613 lapic_ipi_wait_mult = r1 >= r2 ? r1 / r2 : 1; 614 if (bootverbose) { 615 printf("LAPIC: ipi_wait() us multiplier %ju (r %ju " 616 "tsc %ju)\n", (uintmax_t)lapic_ipi_wait_mult, 617 (uintmax_t)r, (uintmax_t)tsc_freq); 618 } 619 } 620 #undef LOOPS 621 #endif /* SMP */ 622 } 623 624 /* 625 * Create a local APIC instance. 626 */ 627 static void 628 native_lapic_create(u_int apic_id, int boot_cpu) 629 { 630 int i; 631 632 if (apic_id > max_apic_id) { 633 printf("APIC: Ignoring local APIC with ID %d\n", apic_id); 634 if (boot_cpu) 635 panic("Can't ignore BSP"); 636 return; 637 } 638 KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u", 639 apic_id)); 640 641 /* 642 * Assume no local LVT overrides and a cluster of 0 and 643 * intra-cluster ID of 0. 644 */ 645 lapics[apic_id].la_present = 1; 646 lapics[apic_id].la_id = apic_id; 647 for (i = 0; i <= APIC_LVT_MAX; i++) { 648 lapics[apic_id].la_lvts[i] = lvts[i]; 649 lapics[apic_id].la_lvts[i].lvt_active = 0; 650 } 651 for (i = 0; i <= APIC_ELVT_MAX; i++) { 652 lapics[apic_id].la_elvts[i] = elvts[i]; 653 lapics[apic_id].la_elvts[i].lvt_active = 0; 654 } 655 for (i = 0; i <= APIC_NUM_IOINTS; i++) 656 lapics[apic_id].la_ioint_irqs[i] = IRQ_FREE; 657 lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL; 658 lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = 659 IRQ_TIMER; 660 #ifdef KDTRACE_HOOKS 661 lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] = 662 IRQ_DTRACE_RET; 663 #endif 664 #ifdef XENHVM 665 lapics[apic_id].la_ioint_irqs[IDT_EVTCHN - APIC_IO_INTS] = IRQ_EVTCHN; 666 #endif 667 668 669 #ifdef SMP 670 cpu_add(apic_id, boot_cpu); 671 #endif 672 } 673 674 static inline uint32_t 675 amd_read_ext_features(void) 676 { 677 uint32_t version; 678 679 if (cpu_vendor_id != CPU_VENDOR_AMD) 680 return (0); 681 version = lapic_read32(LAPIC_VERSION); 682 if ((version & APIC_VER_AMD_EXT_SPACE) != 0) 683 return (lapic_read32(LAPIC_EXT_FEATURES)); 684 else 685 return (0); 686 } 687 688 static inline uint32_t 689 amd_read_elvt_count(void) 690 { 691 uint32_t extf; 692 uint32_t count; 693 694 extf = amd_read_ext_features(); 695 count = (extf & APIC_EXTF_ELVT_MASK) >> APIC_EXTF_ELVT_SHIFT; 696 count = min(count, APIC_ELVT_MAX + 1); 697 return (count); 698 } 699 700 /* 701 * Dump contents of local APIC registers 702 */ 703 static void 704 native_lapic_dump(const char* str) 705 { 706 uint32_t version; 707 uint32_t maxlvt; 708 uint32_t extf; 709 int elvt_count; 710 int i; 711 712 version = lapic_read32(LAPIC_VERSION); 713 maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 714 printf("cpu%d %s:\n", PCPU_GET(cpuid), str); 715 printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x", 716 lapic_read32(LAPIC_ID), version, 717 lapic_read32(LAPIC_LDR), x2apic_mode ? 0 : lapic_read32(LAPIC_DFR)); 718 if ((cpu_feature2 & CPUID2_X2APIC) != 0) 719 printf(" x2APIC: %d", x2apic_mode); 720 printf("\n lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", 721 lapic_read32(LAPIC_LVT_LINT0), lapic_read32(LAPIC_LVT_LINT1), 722 lapic_read32(LAPIC_TPR), lapic_read32(LAPIC_SVR)); 723 printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x", 724 lapic_read32(LAPIC_LVT_TIMER), lapic_read32(LAPIC_LVT_THERMAL), 725 lapic_read32(LAPIC_LVT_ERROR)); 726 if (maxlvt >= APIC_LVT_PMC) 727 printf(" pmc: 0x%08x", lapic_read32(LAPIC_LVT_PCINT)); 728 printf("\n"); 729 if (maxlvt >= APIC_LVT_CMCI) 730 printf(" cmci: 0x%08x\n", lapic_read32(LAPIC_LVT_CMCI)); 731 extf = amd_read_ext_features(); 732 if (extf != 0) { 733 printf(" AMD ext features: 0x%08x\n", extf); 734 elvt_count = amd_read_elvt_count(); 735 for (i = 0; i < elvt_count; i++) 736 printf(" AMD elvt%d: 0x%08x\n", i, 737 lapic_read32(LAPIC_EXT_LVT0 + i)); 738 } 739 } 740 741 static void 742 native_lapic_xapic_mode(void) 743 { 744 register_t saveintr; 745 746 saveintr = intr_disable(); 747 if (x2apic_mode) 748 native_lapic_enable_x2apic(); 749 intr_restore(saveintr); 750 } 751 752 static void 753 native_lapic_setup(int boot) 754 { 755 struct lapic *la; 756 uint32_t version; 757 uint32_t maxlvt; 758 register_t saveintr; 759 int elvt_count; 760 int i; 761 762 saveintr = intr_disable(); 763 764 la = &lapics[lapic_id()]; 765 KASSERT(la->la_present, ("missing APIC structure")); 766 version = lapic_read32(LAPIC_VERSION); 767 maxlvt = (version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 768 769 /* Initialize the TPR to allow all interrupts. */ 770 lapic_set_tpr(0); 771 772 /* Setup spurious vector and enable the local APIC. */ 773 lapic_enable(); 774 775 /* Program LINT[01] LVT entries. */ 776 lapic_write32(LAPIC_LVT_LINT0, lvt_mode(la, APIC_LVT_LINT0, 777 lapic_read32(LAPIC_LVT_LINT0))); 778 lapic_write32(LAPIC_LVT_LINT1, lvt_mode(la, APIC_LVT_LINT1, 779 lapic_read32(LAPIC_LVT_LINT1))); 780 781 /* Program the PMC LVT entry if present. */ 782 if (maxlvt >= APIC_LVT_PMC) { 783 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC, 784 LAPIC_LVT_PCINT)); 785 } 786 787 /* Program timer LVT. */ 788 la->lvt_timer_base = lvt_mode(la, APIC_LVT_TIMER, 789 lapic_read32(LAPIC_LVT_TIMER)); 790 la->lvt_timer_last = la->lvt_timer_base; 791 lapic_write32(LAPIC_LVT_TIMER, la->lvt_timer_base); 792 793 /* Calibrate the timer parameters using BSP. */ 794 if (boot && IS_BSP()) { 795 lapic_calibrate_initcount(la); 796 if (lapic_timer_tsc_deadline) 797 lapic_calibrate_deadline(la); 798 } 799 800 /* Setup the timer if configured. */ 801 if (la->la_timer_mode != LAT_MODE_UNDEF) { 802 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor", 803 lapic_id())); 804 switch (la->la_timer_mode) { 805 case LAT_MODE_PERIODIC: 806 lapic_timer_set_divisor(lapic_timer_divisor); 807 lapic_timer_periodic(la); 808 break; 809 case LAT_MODE_ONESHOT: 810 lapic_timer_set_divisor(lapic_timer_divisor); 811 lapic_timer_oneshot(la); 812 break; 813 case LAT_MODE_DEADLINE: 814 lapic_timer_deadline(la); 815 break; 816 default: 817 panic("corrupted la_timer_mode %p %d", la, 818 la->la_timer_mode); 819 } 820 } 821 822 /* Program error LVT and clear any existing errors. */ 823 lapic_write32(LAPIC_LVT_ERROR, lvt_mode(la, APIC_LVT_ERROR, 824 lapic_read32(LAPIC_LVT_ERROR))); 825 lapic_write32(LAPIC_ESR, 0); 826 827 /* XXX: Thermal LVT */ 828 829 /* Program the CMCI LVT entry if present. */ 830 if (maxlvt >= APIC_LVT_CMCI) { 831 lapic_write32(LAPIC_LVT_CMCI, lvt_mode(la, APIC_LVT_CMCI, 832 lapic_read32(LAPIC_LVT_CMCI))); 833 } 834 835 elvt_count = amd_read_elvt_count(); 836 for (i = 0; i < elvt_count; i++) { 837 if (la->la_elvts[i].lvt_active) 838 lapic_write32(LAPIC_EXT_LVT0 + i, 839 elvt_mode(la, i, lapic_read32(LAPIC_EXT_LVT0 + i))); 840 } 841 842 intr_restore(saveintr); 843 } 844 845 static void 846 native_lapic_intrcnt(void *dummy __unused) 847 { 848 struct pcpu *pc; 849 struct lapic *la; 850 char buf[MAXCOMLEN + 1]; 851 852 /* If there are no APICs, skip this function. */ 853 if (lapics == NULL) 854 return; 855 856 STAILQ_FOREACH(pc, &cpuhead, pc_allcpu) { 857 la = &lapics[pc->pc_apic_id]; 858 if (!la->la_present) 859 continue; 860 861 snprintf(buf, sizeof(buf), "cpu%d:timer", pc->pc_cpuid); 862 intrcnt_add(buf, &la->la_timer_count); 863 } 864 } 865 SYSINIT(native_lapic_intrcnt, SI_SUB_INTR, SI_ORDER_MIDDLE, native_lapic_intrcnt, 866 NULL); 867 868 static void 869 native_lapic_reenable_pmc(void) 870 { 871 #ifdef HWPMC_HOOKS 872 uint32_t value; 873 874 value = lapic_read32(LAPIC_LVT_PCINT); 875 value &= ~APIC_LVT_M; 876 lapic_write32(LAPIC_LVT_PCINT, value); 877 #endif 878 } 879 880 #ifdef HWPMC_HOOKS 881 static void 882 lapic_update_pmc(void *dummy) 883 { 884 struct lapic *la; 885 886 la = &lapics[lapic_id()]; 887 lapic_write32(LAPIC_LVT_PCINT, lvt_mode(la, APIC_LVT_PMC, 888 lapic_read32(LAPIC_LVT_PCINT))); 889 } 890 #endif 891 892 static int 893 native_lapic_enable_pmc(void) 894 { 895 #ifdef HWPMC_HOOKS 896 u_int32_t maxlvt; 897 898 /* Fail if the local APIC is not present. */ 899 if (!x2apic_mode && lapic_map == NULL) 900 return (0); 901 902 /* Fail if the PMC LVT is not present. */ 903 maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 904 if (maxlvt < APIC_LVT_PMC) 905 return (0); 906 907 lvts[APIC_LVT_PMC].lvt_masked = 0; 908 909 #ifdef EARLY_AP_STARTUP 910 MPASS(mp_ncpus == 1 || smp_started); 911 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 912 #else 913 #ifdef SMP 914 /* 915 * If hwpmc was loaded at boot time then the APs may not be 916 * started yet. In that case, don't forward the request to 917 * them as they will program the lvt when they start. 918 */ 919 if (smp_started) 920 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 921 else 922 #endif 923 lapic_update_pmc(NULL); 924 #endif 925 return (1); 926 #else 927 return (0); 928 #endif 929 } 930 931 static void 932 native_lapic_disable_pmc(void) 933 { 934 #ifdef HWPMC_HOOKS 935 u_int32_t maxlvt; 936 937 /* Fail if the local APIC is not present. */ 938 if (!x2apic_mode && lapic_map == NULL) 939 return; 940 941 /* Fail if the PMC LVT is not present. */ 942 maxlvt = (lapic_read32(LAPIC_VERSION) & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 943 if (maxlvt < APIC_LVT_PMC) 944 return; 945 946 lvts[APIC_LVT_PMC].lvt_masked = 1; 947 948 #ifdef SMP 949 /* The APs should always be started when hwpmc is unloaded. */ 950 KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early")); 951 #endif 952 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 953 #endif 954 } 955 956 static void 957 lapic_calibrate_initcount(struct lapic *la) 958 { 959 u_long value; 960 961 /* Start off with a divisor of 2 (power on reset default). */ 962 lapic_timer_divisor = 2; 963 /* Try to calibrate the local APIC timer. */ 964 do { 965 lapic_timer_set_divisor(lapic_timer_divisor); 966 lapic_timer_oneshot_nointr(la, APIC_TIMER_MAX_COUNT); 967 DELAY(1000000); 968 value = APIC_TIMER_MAX_COUNT - lapic_read32(LAPIC_CCR_TIMER); 969 if (value != APIC_TIMER_MAX_COUNT) 970 break; 971 lapic_timer_divisor <<= 1; 972 } while (lapic_timer_divisor <= 128); 973 if (lapic_timer_divisor > 128) 974 panic("lapic: Divisor too big"); 975 if (bootverbose) { 976 printf("lapic: Divisor %lu, Frequency %lu Hz\n", 977 lapic_timer_divisor, value); 978 } 979 count_freq = value; 980 } 981 982 static void 983 lapic_calibrate_deadline(struct lapic *la __unused) 984 { 985 986 if (bootverbose) { 987 printf("lapic: deadline tsc mode, Frequency %ju Hz\n", 988 (uintmax_t)tsc_freq); 989 } 990 } 991 992 static void 993 lapic_change_mode(struct eventtimer *et, struct lapic *la, 994 enum lat_timer_mode newmode) 995 { 996 997 if (la->la_timer_mode == newmode) 998 return; 999 switch (newmode) { 1000 case LAT_MODE_PERIODIC: 1001 lapic_timer_set_divisor(lapic_timer_divisor); 1002 et->et_frequency = count_freq; 1003 break; 1004 case LAT_MODE_DEADLINE: 1005 et->et_frequency = tsc_freq; 1006 break; 1007 case LAT_MODE_ONESHOT: 1008 lapic_timer_set_divisor(lapic_timer_divisor); 1009 et->et_frequency = count_freq; 1010 break; 1011 default: 1012 panic("lapic_change_mode %d", newmode); 1013 } 1014 la->la_timer_mode = newmode; 1015 et->et_min_period = (0x00000002LLU << 32) / et->et_frequency; 1016 et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency; 1017 } 1018 1019 static int 1020 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) 1021 { 1022 struct lapic *la; 1023 1024 la = &lapics[PCPU_GET(apic_id)]; 1025 if (period != 0) { 1026 lapic_change_mode(et, la, LAT_MODE_PERIODIC); 1027 la->la_timer_period = ((uint32_t)et->et_frequency * period) >> 1028 32; 1029 lapic_timer_periodic(la); 1030 } else if (lapic_timer_tsc_deadline) { 1031 lapic_change_mode(et, la, LAT_MODE_DEADLINE); 1032 la->la_timer_period = (et->et_frequency * first) >> 32; 1033 lapic_timer_deadline(la); 1034 } else { 1035 lapic_change_mode(et, la, LAT_MODE_ONESHOT); 1036 la->la_timer_period = ((uint32_t)et->et_frequency * first) >> 1037 32; 1038 lapic_timer_oneshot(la); 1039 } 1040 return (0); 1041 } 1042 1043 static int 1044 lapic_et_stop(struct eventtimer *et) 1045 { 1046 struct lapic *la; 1047 1048 la = &lapics[PCPU_GET(apic_id)]; 1049 lapic_timer_stop(la); 1050 la->la_timer_mode = LAT_MODE_UNDEF; 1051 return (0); 1052 } 1053 1054 static void 1055 native_lapic_disable(void) 1056 { 1057 uint32_t value; 1058 1059 /* Software disable the local APIC. */ 1060 value = lapic_read32(LAPIC_SVR); 1061 value &= ~APIC_SVR_SWEN; 1062 lapic_write32(LAPIC_SVR, value); 1063 } 1064 1065 static void 1066 lapic_enable(void) 1067 { 1068 uint32_t value; 1069 1070 /* Program the spurious vector to enable the local APIC. */ 1071 value = lapic_read32(LAPIC_SVR); 1072 value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); 1073 value |= APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT; 1074 if (lapic_eoi_suppression) 1075 value |= APIC_SVR_EOI_SUPPRESSION; 1076 lapic_write32(LAPIC_SVR, value); 1077 } 1078 1079 /* Reset the local APIC on the BSP during resume. */ 1080 static void 1081 lapic_resume(struct pic *pic, bool suspend_cancelled) 1082 { 1083 1084 lapic_setup(0); 1085 } 1086 1087 static int 1088 native_lapic_id(void) 1089 { 1090 uint32_t v; 1091 1092 KASSERT(x2apic_mode || lapic_map != NULL, ("local APIC is not mapped")); 1093 v = lapic_read32(LAPIC_ID); 1094 if (!x2apic_mode) 1095 v >>= APIC_ID_SHIFT; 1096 return (v); 1097 } 1098 1099 static int 1100 native_lapic_intr_pending(u_int vector) 1101 { 1102 uint32_t irr; 1103 1104 /* 1105 * The IRR registers are an array of registers each of which 1106 * only describes 32 interrupts in the low 32 bits. Thus, we 1107 * divide the vector by 32 to get the register index. 1108 * Finally, we modulus the vector by 32 to determine the 1109 * individual bit to test. 1110 */ 1111 irr = lapic_read32(LAPIC_IRR0 + vector / 32); 1112 return (irr & 1 << (vector % 32)); 1113 } 1114 1115 static void 1116 native_lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 1117 { 1118 struct lapic *la; 1119 1120 KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist", 1121 __func__, apic_id)); 1122 KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big", 1123 __func__, cluster)); 1124 KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID, 1125 ("%s: intra cluster id %u too big", __func__, cluster_id)); 1126 la = &lapics[apic_id]; 1127 la->la_cluster = cluster; 1128 la->la_cluster_id = cluster_id; 1129 } 1130 1131 static int 1132 native_lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked) 1133 { 1134 1135 if (pin > APIC_LVT_MAX) 1136 return (EINVAL); 1137 if (apic_id == APIC_ID_ALL) { 1138 lvts[pin].lvt_masked = masked; 1139 if (bootverbose) 1140 printf("lapic:"); 1141 } else { 1142 KASSERT(lapics[apic_id].la_present, 1143 ("%s: missing APIC %u", __func__, apic_id)); 1144 lapics[apic_id].la_lvts[pin].lvt_masked = masked; 1145 lapics[apic_id].la_lvts[pin].lvt_active = 1; 1146 if (bootverbose) 1147 printf("lapic%u:", apic_id); 1148 } 1149 if (bootverbose) 1150 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked"); 1151 return (0); 1152 } 1153 1154 static int 1155 native_lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode) 1156 { 1157 struct lvt *lvt; 1158 1159 if (pin > APIC_LVT_MAX) 1160 return (EINVAL); 1161 if (apic_id == APIC_ID_ALL) { 1162 lvt = &lvts[pin]; 1163 if (bootverbose) 1164 printf("lapic:"); 1165 } else { 1166 KASSERT(lapics[apic_id].la_present, 1167 ("%s: missing APIC %u", __func__, apic_id)); 1168 lvt = &lapics[apic_id].la_lvts[pin]; 1169 lvt->lvt_active = 1; 1170 if (bootverbose) 1171 printf("lapic%u:", apic_id); 1172 } 1173 lvt->lvt_mode = mode; 1174 switch (mode) { 1175 case APIC_LVT_DM_NMI: 1176 case APIC_LVT_DM_SMI: 1177 case APIC_LVT_DM_INIT: 1178 case APIC_LVT_DM_EXTINT: 1179 lvt->lvt_edgetrigger = 1; 1180 lvt->lvt_activehi = 1; 1181 if (mode == APIC_LVT_DM_EXTINT) 1182 lvt->lvt_masked = 1; 1183 else 1184 lvt->lvt_masked = 0; 1185 break; 1186 default: 1187 panic("Unsupported delivery mode: 0x%x\n", mode); 1188 } 1189 if (bootverbose) { 1190 printf(" Routing "); 1191 switch (mode) { 1192 case APIC_LVT_DM_NMI: 1193 printf("NMI"); 1194 break; 1195 case APIC_LVT_DM_SMI: 1196 printf("SMI"); 1197 break; 1198 case APIC_LVT_DM_INIT: 1199 printf("INIT"); 1200 break; 1201 case APIC_LVT_DM_EXTINT: 1202 printf("ExtINT"); 1203 break; 1204 } 1205 printf(" -> LINT%u\n", pin); 1206 } 1207 return (0); 1208 } 1209 1210 static int 1211 native_lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol) 1212 { 1213 1214 if (pin > APIC_LVT_MAX || pol == INTR_POLARITY_CONFORM) 1215 return (EINVAL); 1216 if (apic_id == APIC_ID_ALL) { 1217 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH); 1218 if (bootverbose) 1219 printf("lapic:"); 1220 } else { 1221 KASSERT(lapics[apic_id].la_present, 1222 ("%s: missing APIC %u", __func__, apic_id)); 1223 lapics[apic_id].la_lvts[pin].lvt_active = 1; 1224 lapics[apic_id].la_lvts[pin].lvt_activehi = 1225 (pol == INTR_POLARITY_HIGH); 1226 if (bootverbose) 1227 printf("lapic%u:", apic_id); 1228 } 1229 if (bootverbose) 1230 printf(" LINT%u polarity: %s\n", pin, 1231 pol == INTR_POLARITY_HIGH ? "high" : "low"); 1232 return (0); 1233 } 1234 1235 static int 1236 native_lapic_set_lvt_triggermode(u_int apic_id, u_int pin, 1237 enum intr_trigger trigger) 1238 { 1239 1240 if (pin > APIC_LVT_MAX || trigger == INTR_TRIGGER_CONFORM) 1241 return (EINVAL); 1242 if (apic_id == APIC_ID_ALL) { 1243 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE); 1244 if (bootverbose) 1245 printf("lapic:"); 1246 } else { 1247 KASSERT(lapics[apic_id].la_present, 1248 ("%s: missing APIC %u", __func__, apic_id)); 1249 lapics[apic_id].la_lvts[pin].lvt_edgetrigger = 1250 (trigger == INTR_TRIGGER_EDGE); 1251 lapics[apic_id].la_lvts[pin].lvt_active = 1; 1252 if (bootverbose) 1253 printf("lapic%u:", apic_id); 1254 } 1255 if (bootverbose) 1256 printf(" LINT%u trigger: %s\n", pin, 1257 trigger == INTR_TRIGGER_EDGE ? "edge" : "level"); 1258 return (0); 1259 } 1260 1261 /* 1262 * Adjust the TPR of the current CPU so that it blocks all interrupts below 1263 * the passed in vector. 1264 */ 1265 static void 1266 lapic_set_tpr(u_int vector) 1267 { 1268 #ifdef CHEAP_TPR 1269 lapic_write32(LAPIC_TPR, vector); 1270 #else 1271 uint32_t tpr; 1272 1273 tpr = lapic_read32(LAPIC_TPR) & ~APIC_TPR_PRIO; 1274 tpr |= vector; 1275 lapic_write32(LAPIC_TPR, tpr); 1276 #endif 1277 } 1278 1279 static void 1280 native_lapic_eoi(void) 1281 { 1282 1283 lapic_write32_nofence(LAPIC_EOI, 0); 1284 } 1285 1286 void 1287 lapic_handle_intr(int vector, struct trapframe *frame) 1288 { 1289 struct intsrc *isrc; 1290 1291 isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id), 1292 vector)); 1293 intr_execute_handlers(isrc, frame); 1294 } 1295 1296 void 1297 lapic_handle_timer(struct trapframe *frame) 1298 { 1299 struct lapic *la; 1300 struct trapframe *oldframe; 1301 struct thread *td; 1302 1303 /* Send EOI first thing. */ 1304 lapic_eoi(); 1305 1306 #if defined(SMP) && !defined(SCHED_ULE) 1307 /* 1308 * Don't do any accounting for the disabled HTT cores, since it 1309 * will provide misleading numbers for the userland. 1310 * 1311 * No locking is necessary here, since even if we lose the race 1312 * when hlt_cpus_mask changes it is not a big deal, really. 1313 * 1314 * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask 1315 * and unlike other schedulers it actually schedules threads to 1316 * those CPUs. 1317 */ 1318 if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)) 1319 return; 1320 #endif 1321 1322 /* Look up our local APIC structure for the tick counters. */ 1323 la = &lapics[PCPU_GET(apic_id)]; 1324 (*la->la_timer_count)++; 1325 critical_enter(); 1326 if (lapic_et.et_active) { 1327 td = curthread; 1328 td->td_intr_nesting_level++; 1329 oldframe = td->td_intr_frame; 1330 td->td_intr_frame = frame; 1331 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg); 1332 td->td_intr_frame = oldframe; 1333 td->td_intr_nesting_level--; 1334 } 1335 critical_exit(); 1336 } 1337 1338 static void 1339 lapic_timer_set_divisor(u_int divisor) 1340 { 1341 1342 KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor)); 1343 KASSERT(ffs(divisor) <= nitems(lapic_timer_divisors), 1344 ("lapic: invalid divisor %u", divisor)); 1345 lapic_write32(LAPIC_DCR_TIMER, lapic_timer_divisors[ffs(divisor) - 1]); 1346 } 1347 1348 static void 1349 lapic_timer_oneshot(struct lapic *la) 1350 { 1351 uint32_t value; 1352 1353 value = la->lvt_timer_base; 1354 value &= ~(APIC_LVTT_TM | APIC_LVT_M); 1355 value |= APIC_LVTT_TM_ONE_SHOT; 1356 la->lvt_timer_last = value; 1357 lapic_write32(LAPIC_LVT_TIMER, value); 1358 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period); 1359 } 1360 1361 static void 1362 lapic_timer_oneshot_nointr(struct lapic *la, uint32_t count) 1363 { 1364 uint32_t value; 1365 1366 value = la->lvt_timer_base; 1367 value &= ~APIC_LVTT_TM; 1368 value |= APIC_LVTT_TM_ONE_SHOT | APIC_LVT_M; 1369 la->lvt_timer_last = value; 1370 lapic_write32(LAPIC_LVT_TIMER, value); 1371 lapic_write32(LAPIC_ICR_TIMER, count); 1372 } 1373 1374 static void 1375 lapic_timer_periodic(struct lapic *la) 1376 { 1377 uint32_t value; 1378 1379 value = la->lvt_timer_base; 1380 value &= ~(APIC_LVTT_TM | APIC_LVT_M); 1381 value |= APIC_LVTT_TM_PERIODIC; 1382 la->lvt_timer_last = value; 1383 lapic_write32(LAPIC_LVT_TIMER, value); 1384 lapic_write32(LAPIC_ICR_TIMER, la->la_timer_period); 1385 } 1386 1387 static void 1388 lapic_timer_deadline(struct lapic *la) 1389 { 1390 uint32_t value; 1391 1392 value = la->lvt_timer_base; 1393 value &= ~(APIC_LVTT_TM | APIC_LVT_M); 1394 value |= APIC_LVTT_TM_TSCDLT; 1395 if (value != la->lvt_timer_last) { 1396 la->lvt_timer_last = value; 1397 lapic_write32_nofence(LAPIC_LVT_TIMER, value); 1398 if (!x2apic_mode) 1399 mfence(); 1400 } 1401 wrmsr(MSR_TSC_DEADLINE, la->la_timer_period + rdtsc()); 1402 } 1403 1404 static void 1405 lapic_timer_stop(struct lapic *la) 1406 { 1407 uint32_t value; 1408 1409 if (la->la_timer_mode == LAT_MODE_DEADLINE) { 1410 wrmsr(MSR_TSC_DEADLINE, 0); 1411 mfence(); 1412 } else { 1413 value = la->lvt_timer_base; 1414 value &= ~APIC_LVTT_TM; 1415 value |= APIC_LVT_M; 1416 la->lvt_timer_last = value; 1417 lapic_write32(LAPIC_LVT_TIMER, value); 1418 } 1419 } 1420 1421 void 1422 lapic_handle_cmc(void) 1423 { 1424 1425 lapic_eoi(); 1426 cmc_intr(); 1427 } 1428 1429 /* 1430 * Called from the mca_init() to activate the CMC interrupt if this CPU is 1431 * responsible for monitoring any MC banks for CMC events. Since mca_init() 1432 * is called prior to lapic_setup() during boot, this just needs to unmask 1433 * this CPU's LVT_CMCI entry. 1434 */ 1435 static void 1436 native_lapic_enable_cmc(void) 1437 { 1438 u_int apic_id; 1439 1440 #ifdef DEV_ATPIC 1441 if (!x2apic_mode && lapic_map == NULL) 1442 return; 1443 #endif 1444 apic_id = PCPU_GET(apic_id); 1445 KASSERT(lapics[apic_id].la_present, 1446 ("%s: missing APIC %u", __func__, apic_id)); 1447 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_masked = 0; 1448 lapics[apic_id].la_lvts[APIC_LVT_CMCI].lvt_active = 1; 1449 if (bootverbose) 1450 printf("lapic%u: CMCI unmasked\n", apic_id); 1451 } 1452 1453 static int 1454 native_lapic_enable_mca_elvt(void) 1455 { 1456 u_int apic_id; 1457 uint32_t value; 1458 int elvt_count; 1459 1460 #ifdef DEV_ATPIC 1461 if (lapic_map == NULL) 1462 return (-1); 1463 #endif 1464 1465 apic_id = PCPU_GET(apic_id); 1466 KASSERT(lapics[apic_id].la_present, 1467 ("%s: missing APIC %u", __func__, apic_id)); 1468 elvt_count = amd_read_elvt_count(); 1469 if (elvt_count <= APIC_ELVT_MCA) 1470 return (-1); 1471 1472 value = lapic_read32(LAPIC_EXT_LVT0 + APIC_ELVT_MCA); 1473 if ((value & APIC_LVT_M) == 0) { 1474 if (bootverbose) 1475 printf("AMD MCE Thresholding Extended LVT is already active\n"); 1476 return (APIC_ELVT_MCA); 1477 } 1478 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_masked = 0; 1479 lapics[apic_id].la_elvts[APIC_ELVT_MCA].lvt_active = 1; 1480 if (bootverbose) 1481 printf("lapic%u: MCE Thresholding ELVT unmasked\n", apic_id); 1482 return (APIC_ELVT_MCA); 1483 } 1484 1485 void 1486 lapic_handle_error(void) 1487 { 1488 uint32_t esr; 1489 1490 /* 1491 * Read the contents of the error status register. Write to 1492 * the register first before reading from it to force the APIC 1493 * to update its value to indicate any errors that have 1494 * occurred since the previous write to the register. 1495 */ 1496 lapic_write32(LAPIC_ESR, 0); 1497 esr = lapic_read32(LAPIC_ESR); 1498 1499 printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr); 1500 lapic_eoi(); 1501 } 1502 1503 static u_int 1504 native_apic_cpuid(u_int apic_id) 1505 { 1506 #ifdef SMP 1507 return apic_cpuids[apic_id]; 1508 #else 1509 return 0; 1510 #endif 1511 } 1512 1513 /* Request a free IDT vector to be used by the specified IRQ. */ 1514 static u_int 1515 native_apic_alloc_vector(u_int apic_id, u_int irq) 1516 { 1517 u_int vector; 1518 1519 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq)); 1520 1521 /* 1522 * Search for a free vector. Currently we just use a very simple 1523 * algorithm to find the first free vector. 1524 */ 1525 mtx_lock_spin(&icu_lock); 1526 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 1527 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE) 1528 continue; 1529 lapics[apic_id].la_ioint_irqs[vector] = irq; 1530 mtx_unlock_spin(&icu_lock); 1531 return (vector + APIC_IO_INTS); 1532 } 1533 mtx_unlock_spin(&icu_lock); 1534 return (0); 1535 } 1536 1537 /* 1538 * Request 'count' free contiguous IDT vectors to be used by 'count' 1539 * IRQs. 'count' must be a power of two and the vectors will be 1540 * aligned on a boundary of 'align'. If the request cannot be 1541 * satisfied, 0 is returned. 1542 */ 1543 static u_int 1544 native_apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 1545 { 1546 u_int first, run, vector; 1547 1548 KASSERT(powerof2(count), ("bad count")); 1549 KASSERT(powerof2(align), ("bad align")); 1550 KASSERT(align >= count, ("align < count")); 1551 #ifdef INVARIANTS 1552 for (run = 0; run < count; run++) 1553 KASSERT(irqs[run] < num_io_irqs, ("Invalid IRQ %u at index %u", 1554 irqs[run], run)); 1555 #endif 1556 1557 /* 1558 * Search for 'count' free vectors. As with apic_alloc_vector(), 1559 * this just uses a simple first fit algorithm. 1560 */ 1561 run = 0; 1562 first = 0; 1563 mtx_lock_spin(&icu_lock); 1564 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 1565 1566 /* Vector is in use, end run. */ 1567 if (lapics[apic_id].la_ioint_irqs[vector] != IRQ_FREE) { 1568 run = 0; 1569 first = 0; 1570 continue; 1571 } 1572 1573 /* Start a new run if run == 0 and vector is aligned. */ 1574 if (run == 0) { 1575 if ((vector & (align - 1)) != 0) 1576 continue; 1577 first = vector; 1578 } 1579 run++; 1580 1581 /* Keep looping if the run isn't long enough yet. */ 1582 if (run < count) 1583 continue; 1584 1585 /* Found a run, assign IRQs and return the first vector. */ 1586 for (vector = 0; vector < count; vector++) 1587 lapics[apic_id].la_ioint_irqs[first + vector] = 1588 irqs[vector]; 1589 mtx_unlock_spin(&icu_lock); 1590 return (first + APIC_IO_INTS); 1591 } 1592 mtx_unlock_spin(&icu_lock); 1593 printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count); 1594 return (0); 1595 } 1596 1597 /* 1598 * Enable a vector for a particular apic_id. Since all lapics share idt 1599 * entries and ioint_handlers this enables the vector on all lapics. lapics 1600 * which do not have the vector configured would report spurious interrupts 1601 * should it fire. 1602 */ 1603 static void 1604 native_apic_enable_vector(u_int apic_id, u_int vector) 1605 { 1606 1607 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1608 KASSERT(ioint_handlers[vector / 32] != NULL, 1609 ("No ISR handler for vector %u", vector)); 1610 #ifdef KDTRACE_HOOKS 1611 KASSERT(vector != IDT_DTRACE_RET, 1612 ("Attempt to overwrite DTrace entry")); 1613 #endif 1614 setidt(vector, (pti ? ioint_pti_handlers : ioint_handlers)[vector / 32], 1615 SDT_APIC, SEL_KPL, GSEL_APIC); 1616 } 1617 1618 static void 1619 native_apic_disable_vector(u_int apic_id, u_int vector) 1620 { 1621 1622 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1623 #ifdef KDTRACE_HOOKS 1624 KASSERT(vector != IDT_DTRACE_RET, 1625 ("Attempt to overwrite DTrace entry")); 1626 #endif 1627 KASSERT(ioint_handlers[vector / 32] != NULL, 1628 ("No ISR handler for vector %u", vector)); 1629 #ifdef notyet 1630 /* 1631 * We can not currently clear the idt entry because other cpus 1632 * may have a valid vector at this offset. 1633 */ 1634 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC, 1635 SEL_KPL, GSEL_APIC); 1636 #endif 1637 } 1638 1639 /* Release an APIC vector when it's no longer in use. */ 1640 static void 1641 native_apic_free_vector(u_int apic_id, u_int vector, u_int irq) 1642 { 1643 struct thread *td; 1644 1645 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1646 vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1647 ("Vector %u does not map to an IRQ line", vector)); 1648 KASSERT(irq < num_io_irqs, ("Invalid IRQ %u", irq)); 1649 KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] == 1650 irq, ("IRQ mismatch")); 1651 #ifdef KDTRACE_HOOKS 1652 KASSERT(vector != IDT_DTRACE_RET, 1653 ("Attempt to overwrite DTrace entry")); 1654 #endif 1655 1656 /* 1657 * Bind us to the cpu that owned the vector before freeing it so 1658 * we don't lose an interrupt delivery race. 1659 */ 1660 td = curthread; 1661 if (!rebooting) { 1662 thread_lock(td); 1663 if (sched_is_bound(td)) 1664 panic("apic_free_vector: Thread already bound.\n"); 1665 sched_bind(td, apic_cpuid(apic_id)); 1666 thread_unlock(td); 1667 } 1668 mtx_lock_spin(&icu_lock); 1669 lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = IRQ_FREE; 1670 mtx_unlock_spin(&icu_lock); 1671 if (!rebooting) { 1672 thread_lock(td); 1673 sched_unbind(td); 1674 thread_unlock(td); 1675 } 1676 } 1677 1678 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */ 1679 static u_int 1680 apic_idt_to_irq(u_int apic_id, u_int vector) 1681 { 1682 int irq; 1683 1684 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1685 vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1686 ("Vector %u does not map to an IRQ line", vector)); 1687 #ifdef KDTRACE_HOOKS 1688 KASSERT(vector != IDT_DTRACE_RET, 1689 ("Attempt to overwrite DTrace entry")); 1690 #endif 1691 irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS]; 1692 if (irq < 0) 1693 irq = 0; 1694 return (irq); 1695 } 1696 1697 #ifdef DDB 1698 /* 1699 * Dump data about APIC IDT vector mappings. 1700 */ 1701 DB_SHOW_COMMAND(apic, db_show_apic) 1702 { 1703 struct intsrc *isrc; 1704 int i, verbose; 1705 u_int apic_id; 1706 u_int irq; 1707 1708 if (strcmp(modif, "vv") == 0) 1709 verbose = 2; 1710 else if (strcmp(modif, "v") == 0) 1711 verbose = 1; 1712 else 1713 verbose = 0; 1714 for (apic_id = 0; apic_id <= max_apic_id; apic_id++) { 1715 if (lapics[apic_id].la_present == 0) 1716 continue; 1717 db_printf("Interrupts bound to lapic %u\n", apic_id); 1718 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) { 1719 irq = lapics[apic_id].la_ioint_irqs[i]; 1720 if (irq == IRQ_FREE || irq == IRQ_SYSCALL) 1721 continue; 1722 #ifdef KDTRACE_HOOKS 1723 if (irq == IRQ_DTRACE_RET) 1724 continue; 1725 #endif 1726 #ifdef XENHVM 1727 if (irq == IRQ_EVTCHN) 1728 continue; 1729 #endif 1730 db_printf("vec 0x%2x -> ", i + APIC_IO_INTS); 1731 if (irq == IRQ_TIMER) 1732 db_printf("lapic timer\n"); 1733 else if (irq < num_io_irqs) { 1734 isrc = intr_lookup_source(irq); 1735 if (isrc == NULL || verbose == 0) 1736 db_printf("IRQ %u\n", irq); 1737 else 1738 db_dump_intr_event(isrc->is_event, 1739 verbose == 2); 1740 } else 1741 db_printf("IRQ %u ???\n", irq); 1742 } 1743 } 1744 } 1745 1746 static void 1747 dump_mask(const char *prefix, uint32_t v, int base) 1748 { 1749 int i, first; 1750 1751 first = 1; 1752 for (i = 0; i < 32; i++) 1753 if (v & (1 << i)) { 1754 if (first) { 1755 db_printf("%s:", prefix); 1756 first = 0; 1757 } 1758 db_printf(" %02x", base + i); 1759 } 1760 if (!first) 1761 db_printf("\n"); 1762 } 1763 1764 /* Show info from the lapic regs for this CPU. */ 1765 DB_SHOW_COMMAND(lapic, db_show_lapic) 1766 { 1767 uint32_t v; 1768 1769 db_printf("lapic ID = %d\n", lapic_id()); 1770 v = lapic_read32(LAPIC_VERSION); 1771 db_printf("version = %d.%d\n", (v & APIC_VER_VERSION) >> 4, 1772 v & 0xf); 1773 db_printf("max LVT = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT); 1774 v = lapic_read32(LAPIC_SVR); 1775 db_printf("SVR = %02x (%s)\n", v & APIC_SVR_VECTOR, 1776 v & APIC_SVR_ENABLE ? "enabled" : "disabled"); 1777 db_printf("TPR = %02x\n", lapic_read32(LAPIC_TPR)); 1778 1779 #define dump_field(prefix, regn, index) \ 1780 dump_mask(__XSTRING(prefix ## index), \ 1781 lapic_read32(LAPIC_ ## regn ## index), \ 1782 index * 32) 1783 1784 db_printf("In-service Interrupts:\n"); 1785 dump_field(isr, ISR, 0); 1786 dump_field(isr, ISR, 1); 1787 dump_field(isr, ISR, 2); 1788 dump_field(isr, ISR, 3); 1789 dump_field(isr, ISR, 4); 1790 dump_field(isr, ISR, 5); 1791 dump_field(isr, ISR, 6); 1792 dump_field(isr, ISR, 7); 1793 1794 db_printf("TMR Interrupts:\n"); 1795 dump_field(tmr, TMR, 0); 1796 dump_field(tmr, TMR, 1); 1797 dump_field(tmr, TMR, 2); 1798 dump_field(tmr, TMR, 3); 1799 dump_field(tmr, TMR, 4); 1800 dump_field(tmr, TMR, 5); 1801 dump_field(tmr, TMR, 6); 1802 dump_field(tmr, TMR, 7); 1803 1804 db_printf("IRR Interrupts:\n"); 1805 dump_field(irr, IRR, 0); 1806 dump_field(irr, IRR, 1); 1807 dump_field(irr, IRR, 2); 1808 dump_field(irr, IRR, 3); 1809 dump_field(irr, IRR, 4); 1810 dump_field(irr, IRR, 5); 1811 dump_field(irr, IRR, 6); 1812 dump_field(irr, IRR, 7); 1813 1814 #undef dump_field 1815 } 1816 #endif 1817 1818 /* 1819 * APIC probing support code. This includes code to manage enumerators. 1820 */ 1821 1822 static SLIST_HEAD(, apic_enumerator) enumerators = 1823 SLIST_HEAD_INITIALIZER(enumerators); 1824 static struct apic_enumerator *best_enum; 1825 1826 void 1827 apic_register_enumerator(struct apic_enumerator *enumerator) 1828 { 1829 #ifdef INVARIANTS 1830 struct apic_enumerator *apic_enum; 1831 1832 SLIST_FOREACH(apic_enum, &enumerators, apic_next) { 1833 if (apic_enum == enumerator) 1834 panic("%s: Duplicate register of %s", __func__, 1835 enumerator->apic_name); 1836 } 1837 #endif 1838 SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next); 1839 } 1840 1841 /* 1842 * We have to look for CPU's very, very early because certain subsystems 1843 * want to know how many CPU's we have extremely early on in the boot 1844 * process. 1845 */ 1846 static void 1847 apic_init(void *dummy __unused) 1848 { 1849 struct apic_enumerator *enumerator; 1850 int retval, best; 1851 1852 /* We only support built in local APICs. */ 1853 if (!(cpu_feature & CPUID_APIC)) 1854 return; 1855 1856 /* Don't probe if APIC mode is disabled. */ 1857 if (resource_disabled("apic", 0)) 1858 return; 1859 1860 /* Probe all the enumerators to find the best match. */ 1861 best_enum = NULL; 1862 best = 0; 1863 SLIST_FOREACH(enumerator, &enumerators, apic_next) { 1864 retval = enumerator->apic_probe(); 1865 if (retval > 0) 1866 continue; 1867 if (best_enum == NULL || best < retval) { 1868 best_enum = enumerator; 1869 best = retval; 1870 } 1871 } 1872 if (best_enum == NULL) { 1873 if (bootverbose) 1874 printf("APIC: Could not find any APICs.\n"); 1875 #ifndef DEV_ATPIC 1876 panic("running without device atpic requires a local APIC"); 1877 #endif 1878 return; 1879 } 1880 1881 if (bootverbose) 1882 printf("APIC: Using the %s enumerator.\n", 1883 best_enum->apic_name); 1884 1885 #ifdef I686_CPU 1886 /* 1887 * To work around an errata, we disable the local APIC on some 1888 * CPUs during early startup. We need to turn the local APIC back 1889 * on on such CPUs now. 1890 */ 1891 ppro_reenable_apic(); 1892 #endif 1893 1894 /* Probe the CPU's in the system. */ 1895 retval = best_enum->apic_probe_cpus(); 1896 if (retval != 0) 1897 printf("%s: Failed to probe CPUs: returned %d\n", 1898 best_enum->apic_name, retval); 1899 1900 } 1901 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL); 1902 1903 /* 1904 * Setup the local APIC. We have to do this prior to starting up the APs 1905 * in the SMP case. 1906 */ 1907 static void 1908 apic_setup_local(void *dummy __unused) 1909 { 1910 int retval; 1911 1912 if (best_enum == NULL) 1913 return; 1914 1915 lapics = malloc(sizeof(*lapics) * (max_apic_id + 1), M_LAPIC, 1916 M_WAITOK | M_ZERO); 1917 1918 /* Initialize the local APIC. */ 1919 retval = best_enum->apic_setup_local(); 1920 if (retval != 0) 1921 printf("%s: Failed to setup the local APIC: returned %d\n", 1922 best_enum->apic_name, retval); 1923 } 1924 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL); 1925 1926 /* 1927 * Setup the I/O APICs. 1928 */ 1929 static void 1930 apic_setup_io(void *dummy __unused) 1931 { 1932 int retval; 1933 1934 if (best_enum == NULL) 1935 return; 1936 1937 /* 1938 * Local APIC must be registered before other PICs and pseudo PICs 1939 * for proper suspend/resume order. 1940 */ 1941 intr_register_pic(&lapic_pic); 1942 1943 retval = best_enum->apic_setup_io(); 1944 if (retval != 0) 1945 printf("%s: Failed to setup I/O APICs: returned %d\n", 1946 best_enum->apic_name, retval); 1947 1948 /* 1949 * Finish setting up the local APIC on the BSP once we know 1950 * how to properly program the LINT pins. In particular, this 1951 * enables the EOI suppression mode, if LAPIC supports it and 1952 * user did not disable the mode. 1953 */ 1954 lapic_setup(1); 1955 if (bootverbose) 1956 lapic_dump("BSP"); 1957 1958 /* Enable the MSI "pic". */ 1959 init_ops.msi_init(); 1960 1961 #ifdef XENHVM 1962 xen_intr_alloc_irqs(); 1963 #endif 1964 } 1965 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_THIRD, apic_setup_io, NULL); 1966 1967 #ifdef SMP 1968 /* 1969 * Inter Processor Interrupt functions. The lapic_ipi_*() functions are 1970 * private to the MD code. The public interface for the rest of the 1971 * kernel is defined in mp_machdep.c. 1972 */ 1973 1974 /* 1975 * Wait delay microseconds for IPI to be sent. If delay is -1, we 1976 * wait forever. 1977 */ 1978 static int 1979 native_lapic_ipi_wait(int delay) 1980 { 1981 uint64_t rx; 1982 1983 /* LAPIC_ICR.APIC_DELSTAT_MASK is undefined in x2APIC mode */ 1984 if (x2apic_mode) 1985 return (1); 1986 1987 for (rx = 0; delay == -1 || rx < lapic_ipi_wait_mult * delay; rx++) { 1988 if ((lapic_read_icr_lo() & APIC_DELSTAT_MASK) == 1989 APIC_DELSTAT_IDLE) 1990 return (1); 1991 ia32_pause(); 1992 } 1993 return (0); 1994 } 1995 1996 static void 1997 native_lapic_ipi_raw(register_t icrlo, u_int dest) 1998 { 1999 uint64_t icr; 2000 uint32_t vhi, vlo; 2001 register_t saveintr; 2002 2003 /* XXX: Need more sanity checking of icrlo? */ 2004 KASSERT(x2apic_mode || lapic_map != NULL, 2005 ("%s called too early", __func__)); 2006 KASSERT(x2apic_mode || 2007 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 2008 ("%s: invalid dest field", __func__)); 2009 KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, 2010 ("%s: reserved bits set in ICR LO register", __func__)); 2011 2012 /* Set destination in ICR HI register if it is being used. */ 2013 if (!x2apic_mode) { 2014 saveintr = intr_disable(); 2015 icr = lapic_read_icr(); 2016 } 2017 2018 if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { 2019 if (x2apic_mode) { 2020 vhi = dest; 2021 } else { 2022 vhi = icr >> 32; 2023 vhi &= ~APIC_ID_MASK; 2024 vhi |= dest << APIC_ID_SHIFT; 2025 } 2026 } else { 2027 vhi = 0; 2028 } 2029 2030 /* Program the contents of the IPI and dispatch it. */ 2031 if (x2apic_mode) { 2032 vlo = icrlo; 2033 } else { 2034 vlo = icr; 2035 vlo &= APIC_ICRLO_RESV_MASK; 2036 vlo |= icrlo; 2037 } 2038 lapic_write_icr(vhi, vlo); 2039 if (!x2apic_mode) 2040 intr_restore(saveintr); 2041 } 2042 2043 #define BEFORE_SPIN 50000 2044 #ifdef DETECT_DEADLOCK 2045 #define AFTER_SPIN 50 2046 #endif 2047 2048 static void 2049 native_lapic_ipi_vectored(u_int vector, int dest) 2050 { 2051 register_t icrlo, destfield; 2052 2053 KASSERT((vector & ~APIC_VECTOR_MASK) == 0, 2054 ("%s: invalid vector %d", __func__, vector)); 2055 2056 icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE | APIC_LEVEL_ASSERT; 2057 2058 /* 2059 * NMI IPIs are just fake vectors used to send a NMI. Use special rules 2060 * regarding NMIs if passed, otherwise specify the vector. 2061 */ 2062 if (vector >= IPI_NMI_FIRST) 2063 icrlo |= APIC_DELMODE_NMI; 2064 else 2065 icrlo |= vector | APIC_DELMODE_FIXED; 2066 destfield = 0; 2067 switch (dest) { 2068 case APIC_IPI_DEST_SELF: 2069 icrlo |= APIC_DEST_SELF; 2070 break; 2071 case APIC_IPI_DEST_ALL: 2072 icrlo |= APIC_DEST_ALLISELF; 2073 break; 2074 case APIC_IPI_DEST_OTHERS: 2075 icrlo |= APIC_DEST_ALLESELF; 2076 break; 2077 default: 2078 KASSERT(x2apic_mode || 2079 (dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 2080 ("%s: invalid destination 0x%x", __func__, dest)); 2081 destfield = dest; 2082 } 2083 2084 /* Wait for an earlier IPI to finish. */ 2085 if (!lapic_ipi_wait(BEFORE_SPIN)) { 2086 if (panicstr != NULL) 2087 return; 2088 else 2089 panic("APIC: Previous IPI is stuck"); 2090 } 2091 2092 lapic_ipi_raw(icrlo, destfield); 2093 2094 #ifdef DETECT_DEADLOCK 2095 /* Wait for IPI to be delivered. */ 2096 if (!lapic_ipi_wait(AFTER_SPIN)) { 2097 #ifdef needsattention 2098 /* 2099 * XXX FIXME: 2100 * 2101 * The above function waits for the message to actually be 2102 * delivered. It breaks out after an arbitrary timeout 2103 * since the message should eventually be delivered (at 2104 * least in theory) and that if it wasn't we would catch 2105 * the failure with the check above when the next IPI is 2106 * sent. 2107 * 2108 * We could skip this wait entirely, EXCEPT it probably 2109 * protects us from other routines that assume that the 2110 * message was delivered and acted upon when this function 2111 * returns. 2112 */ 2113 printf("APIC: IPI might be stuck\n"); 2114 #else /* !needsattention */ 2115 /* Wait until mesage is sent without a timeout. */ 2116 while (lapic_read_icr_lo() & APIC_DELSTAT_PEND) 2117 ia32_pause(); 2118 #endif /* needsattention */ 2119 } 2120 #endif /* DETECT_DEADLOCK */ 2121 } 2122 2123 #endif /* SMP */ 2124 2125 /* 2126 * Since the IDT is shared by all CPUs the IPI slot update needs to be globally 2127 * visible. 2128 * 2129 * Consider the case where an IPI is generated immediately after allocation: 2130 * vector = lapic_ipi_alloc(ipifunc); 2131 * ipi_selected(other_cpus, vector); 2132 * 2133 * In xAPIC mode a write to ICR_LO has serializing semantics because the 2134 * APIC page is mapped as an uncached region. In x2APIC mode there is an 2135 * explicit 'mfence' before the ICR MSR is written. Therefore in both cases 2136 * the IDT slot update is globally visible before the IPI is delivered. 2137 */ 2138 static int 2139 native_lapic_ipi_alloc(inthand_t *ipifunc) 2140 { 2141 struct gate_descriptor *ip; 2142 long func; 2143 int idx, vector; 2144 2145 KASSERT(ipifunc != &IDTVEC(rsvd) && ipifunc != &IDTVEC(rsvd_pti), 2146 ("invalid ipifunc %p", ipifunc)); 2147 2148 vector = -1; 2149 mtx_lock_spin(&icu_lock); 2150 for (idx = IPI_DYN_FIRST; idx <= IPI_DYN_LAST; idx++) { 2151 ip = &idt[idx]; 2152 func = (ip->gd_hioffset << 16) | ip->gd_looffset; 2153 if ((!pti && func == (uintptr_t)&IDTVEC(rsvd)) || 2154 (pti && func == (uintptr_t)&IDTVEC(rsvd_pti))) { 2155 vector = idx; 2156 setidt(vector, ipifunc, SDT_APIC, SEL_KPL, GSEL_APIC); 2157 break; 2158 } 2159 } 2160 mtx_unlock_spin(&icu_lock); 2161 return (vector); 2162 } 2163 2164 static void 2165 native_lapic_ipi_free(int vector) 2166 { 2167 struct gate_descriptor *ip; 2168 long func; 2169 2170 KASSERT(vector >= IPI_DYN_FIRST && vector <= IPI_DYN_LAST, 2171 ("%s: invalid vector %d", __func__, vector)); 2172 2173 mtx_lock_spin(&icu_lock); 2174 ip = &idt[vector]; 2175 func = (ip->gd_hioffset << 16) | ip->gd_looffset; 2176 KASSERT(func != (uintptr_t)&IDTVEC(rsvd) && 2177 func != (uintptr_t)&IDTVEC(rsvd_pti), 2178 ("invalid idtfunc %#lx", func)); 2179 setidt(vector, pti ? &IDTVEC(rsvd_pti) : &IDTVEC(rsvd), SDT_APIC, 2180 SEL_KPL, GSEL_APIC); 2181 mtx_unlock_spin(&icu_lock); 2182 } 2183