1 /*- 2 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3 * Copyright (c) 1996, by Steve Passe 4 * All rights reserved. 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. The name of the developer may NOT be used to endorse or promote products 12 * derived from this software without specific prior written permission. 13 * 3. Neither the name of the author nor the names of any co-contributors 14 * may be used to endorse or promote products derived from this software 15 * without specific prior written permission. 16 * 17 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 18 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 19 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 20 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 21 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 22 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 23 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 24 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 25 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 26 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 27 * SUCH DAMAGE. 28 */ 29 30 /* 31 * Local APIC support on Pentium and later processors. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_atpic.h" 38 #include "opt_hwpmc_hooks.h" 39 #include "opt_kdtrace.h" 40 41 #include "opt_ddb.h" 42 43 #include <sys/param.h> 44 #include <sys/systm.h> 45 #include <sys/bus.h> 46 #include <sys/kernel.h> 47 #include <sys/lock.h> 48 #include <sys/mutex.h> 49 #include <sys/pcpu.h> 50 #include <sys/proc.h> 51 #include <sys/sched.h> 52 #include <sys/smp.h> 53 #include <sys/timeet.h> 54 55 #include <vm/vm.h> 56 #include <vm/pmap.h> 57 58 #include <x86/apicreg.h> 59 #include <machine/cpu.h> 60 #include <machine/cputypes.h> 61 #include <machine/frame.h> 62 #include <machine/intr_machdep.h> 63 #include <machine/apicvar.h> 64 #include <x86/mca.h> 65 #include <machine/md_var.h> 66 #include <machine/smp.h> 67 #include <machine/specialreg.h> 68 69 #ifdef DDB 70 #include <sys/interrupt.h> 71 #include <ddb/ddb.h> 72 #endif 73 74 #ifdef __amd64__ 75 #define SDT_APIC SDT_SYSIGT 76 #define SDT_APICT SDT_SYSIGT 77 #define GSEL_APIC 0 78 #else 79 #define SDT_APIC SDT_SYS386IGT 80 #define SDT_APICT SDT_SYS386TGT 81 #define GSEL_APIC GSEL(GCODE_SEL, SEL_KPL) 82 #endif 83 84 /* Sanity checks on IDT vectors. */ 85 CTASSERT(APIC_IO_INTS + APIC_NUM_IOINTS == APIC_TIMER_INT); 86 CTASSERT(APIC_TIMER_INT < APIC_LOCAL_INTS); 87 CTASSERT(APIC_LOCAL_INTS == 240); 88 CTASSERT(IPI_STOP < APIC_SPURIOUS_INT); 89 90 /* Magic IRQ values for the timer and syscalls. */ 91 #define IRQ_TIMER (NUM_IO_INTS + 1) 92 #define IRQ_SYSCALL (NUM_IO_INTS + 2) 93 #define IRQ_DTRACE_RET (NUM_IO_INTS + 3) 94 95 /* 96 * Support for local APICs. Local APICs manage interrupts on each 97 * individual processor as opposed to I/O APICs which receive interrupts 98 * from I/O devices and then forward them on to the local APICs. 99 * 100 * Local APICs can also send interrupts to each other thus providing the 101 * mechanism for IPIs. 102 */ 103 104 struct lvt { 105 u_int lvt_edgetrigger:1; 106 u_int lvt_activehi:1; 107 u_int lvt_masked:1; 108 u_int lvt_active:1; 109 u_int lvt_mode:16; 110 u_int lvt_vector:8; 111 }; 112 113 struct lapic { 114 struct lvt la_lvts[LVT_MAX + 1]; 115 u_int la_id:8; 116 u_int la_cluster:4; 117 u_int la_cluster_id:2; 118 u_int la_present:1; 119 u_long *la_timer_count; 120 u_long la_timer_period; 121 u_int la_timer_mode; 122 uint32_t lvt_timer_cache; 123 /* Include IDT_SYSCALL to make indexing easier. */ 124 int la_ioint_irqs[APIC_NUM_IOINTS + 1]; 125 } static lapics[MAX_APIC_ID + 1]; 126 127 /* Global defaults for local APIC LVT entries. */ 128 static struct lvt lvts[LVT_MAX + 1] = { 129 { 1, 1, 1, 1, APIC_LVT_DM_EXTINT, 0 }, /* LINT0: masked ExtINT */ 130 { 1, 1, 0, 1, APIC_LVT_DM_NMI, 0 }, /* LINT1: NMI */ 131 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_TIMER_INT }, /* Timer */ 132 { 1, 1, 0, 1, APIC_LVT_DM_FIXED, APIC_ERROR_INT }, /* Error */ 133 { 1, 1, 1, 1, APIC_LVT_DM_NMI, 0 }, /* PMC */ 134 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_THERMAL_INT }, /* Thermal */ 135 { 1, 1, 1, 1, APIC_LVT_DM_FIXED, APIC_CMC_INT }, /* CMCI */ 136 }; 137 138 static inthand_t *ioint_handlers[] = { 139 NULL, /* 0 - 31 */ 140 IDTVEC(apic_isr1), /* 32 - 63 */ 141 IDTVEC(apic_isr2), /* 64 - 95 */ 142 IDTVEC(apic_isr3), /* 96 - 127 */ 143 IDTVEC(apic_isr4), /* 128 - 159 */ 144 IDTVEC(apic_isr5), /* 160 - 191 */ 145 IDTVEC(apic_isr6), /* 192 - 223 */ 146 IDTVEC(apic_isr7), /* 224 - 255 */ 147 }; 148 149 150 static u_int32_t lapic_timer_divisors[] = { 151 APIC_TDCR_1, APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, 152 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128 153 }; 154 155 extern inthand_t IDTVEC(rsvd); 156 157 volatile lapic_t *lapic; 158 vm_paddr_t lapic_paddr; 159 static u_long lapic_timer_divisor; 160 static struct eventtimer lapic_et; 161 162 static void lapic_enable(void); 163 static void lapic_resume(struct pic *pic); 164 static void lapic_timer_oneshot(struct lapic *, 165 u_int count, int enable_int); 166 static void lapic_timer_periodic(struct lapic *, 167 u_int count, int enable_int); 168 static void lapic_timer_stop(struct lapic *); 169 static void lapic_timer_set_divisor(u_int divisor); 170 static uint32_t lvt_mode(struct lapic *la, u_int pin, uint32_t value); 171 static int lapic_et_start(struct eventtimer *et, 172 sbintime_t first, sbintime_t period); 173 static int lapic_et_stop(struct eventtimer *et); 174 175 struct pic lapic_pic = { .pic_resume = lapic_resume }; 176 177 static uint32_t 178 lvt_mode(struct lapic *la, u_int pin, uint32_t value) 179 { 180 struct lvt *lvt; 181 182 KASSERT(pin <= LVT_MAX, ("%s: pin %u out of range", __func__, pin)); 183 if (la->la_lvts[pin].lvt_active) 184 lvt = &la->la_lvts[pin]; 185 else 186 lvt = &lvts[pin]; 187 188 value &= ~(APIC_LVT_M | APIC_LVT_TM | APIC_LVT_IIPP | APIC_LVT_DM | 189 APIC_LVT_VECTOR); 190 if (lvt->lvt_edgetrigger == 0) 191 value |= APIC_LVT_TM; 192 if (lvt->lvt_activehi == 0) 193 value |= APIC_LVT_IIPP_INTALO; 194 if (lvt->lvt_masked) 195 value |= APIC_LVT_M; 196 value |= lvt->lvt_mode; 197 switch (lvt->lvt_mode) { 198 case APIC_LVT_DM_NMI: 199 case APIC_LVT_DM_SMI: 200 case APIC_LVT_DM_INIT: 201 case APIC_LVT_DM_EXTINT: 202 if (!lvt->lvt_edgetrigger) { 203 printf("lapic%u: Forcing LINT%u to edge trigger\n", 204 la->la_id, pin); 205 value |= APIC_LVT_TM; 206 } 207 /* Use a vector of 0. */ 208 break; 209 case APIC_LVT_DM_FIXED: 210 value |= lvt->lvt_vector; 211 break; 212 default: 213 panic("bad APIC LVT delivery mode: %#x\n", value); 214 } 215 return (value); 216 } 217 218 /* 219 * Map the local APIC and setup necessary interrupt vectors. 220 */ 221 void 222 lapic_init(vm_paddr_t addr) 223 { 224 u_int regs[4]; 225 int i, arat; 226 227 /* Map the local APIC and setup the spurious interrupt handler. */ 228 KASSERT(trunc_page(addr) == addr, 229 ("local APIC not aligned on a page boundary")); 230 lapic_paddr = addr; 231 lapic = pmap_mapdev(addr, sizeof(lapic_t)); 232 setidt(APIC_SPURIOUS_INT, IDTVEC(spuriousint), SDT_APIC, SEL_KPL, 233 GSEL_APIC); 234 235 /* Perform basic initialization of the BSP's local APIC. */ 236 lapic_enable(); 237 238 /* Set BSP's per-CPU local APIC ID. */ 239 PCPU_SET(apic_id, lapic_id()); 240 241 /* Local APIC timer interrupt. */ 242 setidt(APIC_TIMER_INT, IDTVEC(timerint), SDT_APIC, SEL_KPL, GSEL_APIC); 243 244 /* Local APIC error interrupt. */ 245 setidt(APIC_ERROR_INT, IDTVEC(errorint), SDT_APIC, SEL_KPL, GSEL_APIC); 246 247 /* XXX: Thermal interrupt */ 248 249 /* Local APIC CMCI. */ 250 setidt(APIC_CMC_INT, IDTVEC(cmcint), SDT_APICT, SEL_KPL, GSEL_APIC); 251 252 if ((resource_int_value("apic", 0, "clock", &i) != 0 || i != 0)) { 253 arat = 0; 254 /* Intel CPUID 0x06 EAX[2] set if APIC timer runs in C3. */ 255 if (cpu_vendor_id == CPU_VENDOR_INTEL && cpu_high >= 6) { 256 do_cpuid(0x06, regs); 257 if ((regs[0] & CPUTPM1_ARAT) != 0) 258 arat = 1; 259 } 260 bzero(&lapic_et, sizeof(lapic_et)); 261 lapic_et.et_name = "LAPIC"; 262 lapic_et.et_flags = ET_FLAGS_PERIODIC | ET_FLAGS_ONESHOT | 263 ET_FLAGS_PERCPU; 264 lapic_et.et_quality = 600; 265 if (!arat) { 266 lapic_et.et_flags |= ET_FLAGS_C3STOP; 267 lapic_et.et_quality -= 200; 268 } 269 lapic_et.et_frequency = 0; 270 /* We don't know frequency yet, so trying to guess. */ 271 lapic_et.et_min_period = 0x00001000LL; 272 lapic_et.et_max_period = SBT_1S; 273 lapic_et.et_start = lapic_et_start; 274 lapic_et.et_stop = lapic_et_stop; 275 lapic_et.et_priv = NULL; 276 et_register(&lapic_et); 277 } 278 } 279 280 /* 281 * Create a local APIC instance. 282 */ 283 void 284 lapic_create(u_int apic_id, int boot_cpu) 285 { 286 int i; 287 288 if (apic_id > MAX_APIC_ID) { 289 printf("APIC: Ignoring local APIC with ID %d\n", apic_id); 290 if (boot_cpu) 291 panic("Can't ignore BSP"); 292 return; 293 } 294 KASSERT(!lapics[apic_id].la_present, ("duplicate local APIC %u", 295 apic_id)); 296 297 /* 298 * Assume no local LVT overrides and a cluster of 0 and 299 * intra-cluster ID of 0. 300 */ 301 lapics[apic_id].la_present = 1; 302 lapics[apic_id].la_id = apic_id; 303 for (i = 0; i <= LVT_MAX; i++) { 304 lapics[apic_id].la_lvts[i] = lvts[i]; 305 lapics[apic_id].la_lvts[i].lvt_active = 0; 306 } 307 for (i = 0; i <= APIC_NUM_IOINTS; i++) 308 lapics[apic_id].la_ioint_irqs[i] = -1; 309 lapics[apic_id].la_ioint_irqs[IDT_SYSCALL - APIC_IO_INTS] = IRQ_SYSCALL; 310 lapics[apic_id].la_ioint_irqs[APIC_TIMER_INT - APIC_IO_INTS] = 311 IRQ_TIMER; 312 #ifdef KDTRACE_HOOKS 313 lapics[apic_id].la_ioint_irqs[IDT_DTRACE_RET - APIC_IO_INTS] = 314 IRQ_DTRACE_RET; 315 #endif 316 317 318 #ifdef SMP 319 cpu_add(apic_id, boot_cpu); 320 #endif 321 } 322 323 /* 324 * Dump contents of local APIC registers 325 */ 326 void 327 lapic_dump(const char* str) 328 { 329 uint32_t maxlvt; 330 331 maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 332 printf("cpu%d %s:\n", PCPU_GET(cpuid), str); 333 printf(" ID: 0x%08x VER: 0x%08x LDR: 0x%08x DFR: 0x%08x\n", 334 lapic->id, lapic->version, lapic->ldr, lapic->dfr); 335 printf(" lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", 336 lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr); 337 printf(" timer: 0x%08x therm: 0x%08x err: 0x%08x", 338 lapic->lvt_timer, lapic->lvt_thermal, lapic->lvt_error); 339 if (maxlvt >= LVT_PMC) 340 printf(" pmc: 0x%08x", lapic->lvt_pcint); 341 printf("\n"); 342 if (maxlvt >= LVT_CMCI) 343 printf(" cmci: 0x%08x\n", lapic->lvt_cmci); 344 } 345 346 void 347 lapic_setup(int boot) 348 { 349 struct lapic *la; 350 u_int32_t maxlvt; 351 register_t saveintr; 352 char buf[MAXCOMLEN + 1]; 353 354 la = &lapics[lapic_id()]; 355 KASSERT(la->la_present, ("missing APIC structure")); 356 saveintr = intr_disable(); 357 maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 358 359 /* Initialize the TPR to allow all interrupts. */ 360 lapic_set_tpr(0); 361 362 /* Setup spurious vector and enable the local APIC. */ 363 lapic_enable(); 364 365 /* Program LINT[01] LVT entries. */ 366 lapic->lvt_lint0 = lvt_mode(la, LVT_LINT0, lapic->lvt_lint0); 367 lapic->lvt_lint1 = lvt_mode(la, LVT_LINT1, lapic->lvt_lint1); 368 369 /* Program the PMC LVT entry if present. */ 370 if (maxlvt >= LVT_PMC) 371 lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); 372 373 /* Program timer LVT and setup handler. */ 374 la->lvt_timer_cache = lapic->lvt_timer = 375 lvt_mode(la, LVT_TIMER, lapic->lvt_timer); 376 if (boot) { 377 snprintf(buf, sizeof(buf), "cpu%d:timer", PCPU_GET(cpuid)); 378 intrcnt_add(buf, &la->la_timer_count); 379 } 380 381 /* Setup the timer if configured. */ 382 if (la->la_timer_mode != 0) { 383 KASSERT(la->la_timer_period != 0, ("lapic%u: zero divisor", 384 lapic_id())); 385 lapic_timer_set_divisor(lapic_timer_divisor); 386 if (la->la_timer_mode == 1) 387 lapic_timer_periodic(la, la->la_timer_period, 1); 388 else 389 lapic_timer_oneshot(la, la->la_timer_period, 1); 390 } 391 392 /* Program error LVT and clear any existing errors. */ 393 lapic->lvt_error = lvt_mode(la, LVT_ERROR, lapic->lvt_error); 394 lapic->esr = 0; 395 396 /* XXX: Thermal LVT */ 397 398 /* Program the CMCI LVT entry if present. */ 399 if (maxlvt >= LVT_CMCI) 400 lapic->lvt_cmci = lvt_mode(la, LVT_CMCI, lapic->lvt_cmci); 401 402 intr_restore(saveintr); 403 } 404 405 void 406 lapic_reenable_pmc(void) 407 { 408 #ifdef HWPMC_HOOKS 409 uint32_t value; 410 411 value = lapic->lvt_pcint; 412 value &= ~APIC_LVT_M; 413 lapic->lvt_pcint = value; 414 #endif 415 } 416 417 #ifdef HWPMC_HOOKS 418 static void 419 lapic_update_pmc(void *dummy) 420 { 421 struct lapic *la; 422 423 la = &lapics[lapic_id()]; 424 lapic->lvt_pcint = lvt_mode(la, LVT_PMC, lapic->lvt_pcint); 425 } 426 #endif 427 428 int 429 lapic_enable_pmc(void) 430 { 431 #ifdef HWPMC_HOOKS 432 u_int32_t maxlvt; 433 434 /* Fail if the local APIC is not present. */ 435 if (lapic == NULL) 436 return (0); 437 438 /* Fail if the PMC LVT is not present. */ 439 maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 440 if (maxlvt < LVT_PMC) 441 return (0); 442 443 lvts[LVT_PMC].lvt_masked = 0; 444 445 #ifdef SMP 446 /* 447 * If hwpmc was loaded at boot time then the APs may not be 448 * started yet. In that case, don't forward the request to 449 * them as they will program the lvt when they start. 450 */ 451 if (smp_started) 452 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 453 else 454 #endif 455 lapic_update_pmc(NULL); 456 return (1); 457 #else 458 return (0); 459 #endif 460 } 461 462 void 463 lapic_disable_pmc(void) 464 { 465 #ifdef HWPMC_HOOKS 466 u_int32_t maxlvt; 467 468 /* Fail if the local APIC is not present. */ 469 if (lapic == NULL) 470 return; 471 472 /* Fail if the PMC LVT is not present. */ 473 maxlvt = (lapic->version & APIC_VER_MAXLVT) >> MAXLVTSHIFT; 474 if (maxlvt < LVT_PMC) 475 return; 476 477 lvts[LVT_PMC].lvt_masked = 1; 478 479 #ifdef SMP 480 /* The APs should always be started when hwpmc is unloaded. */ 481 KASSERT(mp_ncpus == 1 || smp_started, ("hwpmc unloaded too early")); 482 #endif 483 smp_rendezvous(NULL, lapic_update_pmc, NULL, NULL); 484 #endif 485 } 486 487 static int 488 lapic_et_start(struct eventtimer *et, sbintime_t first, sbintime_t period) 489 { 490 struct lapic *la; 491 u_long value; 492 493 la = &lapics[PCPU_GET(apic_id)]; 494 if (et->et_frequency == 0) { 495 /* Start off with a divisor of 2 (power on reset default). */ 496 lapic_timer_divisor = 2; 497 /* Try to calibrate the local APIC timer. */ 498 do { 499 lapic_timer_set_divisor(lapic_timer_divisor); 500 lapic_timer_oneshot(la, APIC_TIMER_MAX_COUNT, 0); 501 DELAY(1000000); 502 value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer; 503 if (value != APIC_TIMER_MAX_COUNT) 504 break; 505 lapic_timer_divisor <<= 1; 506 } while (lapic_timer_divisor <= 128); 507 if (lapic_timer_divisor > 128) 508 panic("lapic: Divisor too big"); 509 if (bootverbose) 510 printf("lapic: Divisor %lu, Frequency %lu Hz\n", 511 lapic_timer_divisor, value); 512 et->et_frequency = value; 513 et->et_min_period = (0x00000002LLU << 32) / et->et_frequency; 514 et->et_max_period = (0xfffffffeLLU << 32) / et->et_frequency; 515 } 516 if (la->la_timer_mode == 0) 517 lapic_timer_set_divisor(lapic_timer_divisor); 518 if (period != 0) { 519 la->la_timer_mode = 1; 520 la->la_timer_period = ((uint32_t)et->et_frequency * period) >> 32; 521 lapic_timer_periodic(la, la->la_timer_period, 1); 522 } else { 523 la->la_timer_mode = 2; 524 la->la_timer_period = ((uint32_t)et->et_frequency * first) >> 32; 525 lapic_timer_oneshot(la, la->la_timer_period, 1); 526 } 527 return (0); 528 } 529 530 static int 531 lapic_et_stop(struct eventtimer *et) 532 { 533 struct lapic *la = &lapics[PCPU_GET(apic_id)]; 534 535 la->la_timer_mode = 0; 536 lapic_timer_stop(la); 537 return (0); 538 } 539 540 void 541 lapic_disable(void) 542 { 543 uint32_t value; 544 545 /* Software disable the local APIC. */ 546 value = lapic->svr; 547 value &= ~APIC_SVR_SWEN; 548 lapic->svr = value; 549 } 550 551 static void 552 lapic_enable(void) 553 { 554 u_int32_t value; 555 556 /* Program the spurious vector to enable the local APIC. */ 557 value = lapic->svr; 558 value &= ~(APIC_SVR_VECTOR | APIC_SVR_FOCUS); 559 value |= (APIC_SVR_FEN | APIC_SVR_SWEN | APIC_SPURIOUS_INT); 560 lapic->svr = value; 561 } 562 563 /* Reset the local APIC on the BSP during resume. */ 564 static void 565 lapic_resume(struct pic *pic) 566 { 567 568 lapic_setup(0); 569 } 570 571 int 572 lapic_id(void) 573 { 574 575 KASSERT(lapic != NULL, ("local APIC is not mapped")); 576 return (lapic->id >> APIC_ID_SHIFT); 577 } 578 579 int 580 lapic_intr_pending(u_int vector) 581 { 582 volatile u_int32_t *irr; 583 584 /* 585 * The IRR registers are an array of 128-bit registers each of 586 * which only describes 32 interrupts in the low 32 bits.. Thus, 587 * we divide the vector by 32 to get the 128-bit index. We then 588 * multiply that index by 4 to get the equivalent index from 589 * treating the IRR as an array of 32-bit registers. Finally, we 590 * modulus the vector by 32 to determine the individual bit to 591 * test. 592 */ 593 irr = &lapic->irr0; 594 return (irr[(vector / 32) * 4] & 1 << (vector % 32)); 595 } 596 597 void 598 lapic_set_logical_id(u_int apic_id, u_int cluster, u_int cluster_id) 599 { 600 struct lapic *la; 601 602 KASSERT(lapics[apic_id].la_present, ("%s: APIC %u doesn't exist", 603 __func__, apic_id)); 604 KASSERT(cluster <= APIC_MAX_CLUSTER, ("%s: cluster %u too big", 605 __func__, cluster)); 606 KASSERT(cluster_id <= APIC_MAX_INTRACLUSTER_ID, 607 ("%s: intra cluster id %u too big", __func__, cluster_id)); 608 la = &lapics[apic_id]; 609 la->la_cluster = cluster; 610 la->la_cluster_id = cluster_id; 611 } 612 613 int 614 lapic_set_lvt_mask(u_int apic_id, u_int pin, u_char masked) 615 { 616 617 if (pin > LVT_MAX) 618 return (EINVAL); 619 if (apic_id == APIC_ID_ALL) { 620 lvts[pin].lvt_masked = masked; 621 if (bootverbose) 622 printf("lapic:"); 623 } else { 624 KASSERT(lapics[apic_id].la_present, 625 ("%s: missing APIC %u", __func__, apic_id)); 626 lapics[apic_id].la_lvts[pin].lvt_masked = masked; 627 lapics[apic_id].la_lvts[pin].lvt_active = 1; 628 if (bootverbose) 629 printf("lapic%u:", apic_id); 630 } 631 if (bootverbose) 632 printf(" LINT%u %s\n", pin, masked ? "masked" : "unmasked"); 633 return (0); 634 } 635 636 int 637 lapic_set_lvt_mode(u_int apic_id, u_int pin, u_int32_t mode) 638 { 639 struct lvt *lvt; 640 641 if (pin > LVT_MAX) 642 return (EINVAL); 643 if (apic_id == APIC_ID_ALL) { 644 lvt = &lvts[pin]; 645 if (bootverbose) 646 printf("lapic:"); 647 } else { 648 KASSERT(lapics[apic_id].la_present, 649 ("%s: missing APIC %u", __func__, apic_id)); 650 lvt = &lapics[apic_id].la_lvts[pin]; 651 lvt->lvt_active = 1; 652 if (bootverbose) 653 printf("lapic%u:", apic_id); 654 } 655 lvt->lvt_mode = mode; 656 switch (mode) { 657 case APIC_LVT_DM_NMI: 658 case APIC_LVT_DM_SMI: 659 case APIC_LVT_DM_INIT: 660 case APIC_LVT_DM_EXTINT: 661 lvt->lvt_edgetrigger = 1; 662 lvt->lvt_activehi = 1; 663 if (mode == APIC_LVT_DM_EXTINT) 664 lvt->lvt_masked = 1; 665 else 666 lvt->lvt_masked = 0; 667 break; 668 default: 669 panic("Unsupported delivery mode: 0x%x\n", mode); 670 } 671 if (bootverbose) { 672 printf(" Routing "); 673 switch (mode) { 674 case APIC_LVT_DM_NMI: 675 printf("NMI"); 676 break; 677 case APIC_LVT_DM_SMI: 678 printf("SMI"); 679 break; 680 case APIC_LVT_DM_INIT: 681 printf("INIT"); 682 break; 683 case APIC_LVT_DM_EXTINT: 684 printf("ExtINT"); 685 break; 686 } 687 printf(" -> LINT%u\n", pin); 688 } 689 return (0); 690 } 691 692 int 693 lapic_set_lvt_polarity(u_int apic_id, u_int pin, enum intr_polarity pol) 694 { 695 696 if (pin > LVT_MAX || pol == INTR_POLARITY_CONFORM) 697 return (EINVAL); 698 if (apic_id == APIC_ID_ALL) { 699 lvts[pin].lvt_activehi = (pol == INTR_POLARITY_HIGH); 700 if (bootverbose) 701 printf("lapic:"); 702 } else { 703 KASSERT(lapics[apic_id].la_present, 704 ("%s: missing APIC %u", __func__, apic_id)); 705 lapics[apic_id].la_lvts[pin].lvt_active = 1; 706 lapics[apic_id].la_lvts[pin].lvt_activehi = 707 (pol == INTR_POLARITY_HIGH); 708 if (bootverbose) 709 printf("lapic%u:", apic_id); 710 } 711 if (bootverbose) 712 printf(" LINT%u polarity: %s\n", pin, 713 pol == INTR_POLARITY_HIGH ? "high" : "low"); 714 return (0); 715 } 716 717 int 718 lapic_set_lvt_triggermode(u_int apic_id, u_int pin, enum intr_trigger trigger) 719 { 720 721 if (pin > LVT_MAX || trigger == INTR_TRIGGER_CONFORM) 722 return (EINVAL); 723 if (apic_id == APIC_ID_ALL) { 724 lvts[pin].lvt_edgetrigger = (trigger == INTR_TRIGGER_EDGE); 725 if (bootverbose) 726 printf("lapic:"); 727 } else { 728 KASSERT(lapics[apic_id].la_present, 729 ("%s: missing APIC %u", __func__, apic_id)); 730 lapics[apic_id].la_lvts[pin].lvt_edgetrigger = 731 (trigger == INTR_TRIGGER_EDGE); 732 lapics[apic_id].la_lvts[pin].lvt_active = 1; 733 if (bootverbose) 734 printf("lapic%u:", apic_id); 735 } 736 if (bootverbose) 737 printf(" LINT%u trigger: %s\n", pin, 738 trigger == INTR_TRIGGER_EDGE ? "edge" : "level"); 739 return (0); 740 } 741 742 /* 743 * Adjust the TPR of the current CPU so that it blocks all interrupts below 744 * the passed in vector. 745 */ 746 void 747 lapic_set_tpr(u_int vector) 748 { 749 #ifdef CHEAP_TPR 750 lapic->tpr = vector; 751 #else 752 u_int32_t tpr; 753 754 tpr = lapic->tpr & ~APIC_TPR_PRIO; 755 tpr |= vector; 756 lapic->tpr = tpr; 757 #endif 758 } 759 760 void 761 lapic_eoi(void) 762 { 763 764 lapic->eoi = 0; 765 } 766 767 void 768 lapic_handle_intr(int vector, struct trapframe *frame) 769 { 770 struct intsrc *isrc; 771 772 isrc = intr_lookup_source(apic_idt_to_irq(PCPU_GET(apic_id), 773 vector)); 774 intr_execute_handlers(isrc, frame); 775 } 776 777 void 778 lapic_handle_timer(struct trapframe *frame) 779 { 780 struct lapic *la; 781 struct trapframe *oldframe; 782 struct thread *td; 783 784 /* Send EOI first thing. */ 785 lapic_eoi(); 786 787 #if defined(SMP) && !defined(SCHED_ULE) 788 /* 789 * Don't do any accounting for the disabled HTT cores, since it 790 * will provide misleading numbers for the userland. 791 * 792 * No locking is necessary here, since even if we lose the race 793 * when hlt_cpus_mask changes it is not a big deal, really. 794 * 795 * Don't do that for ULE, since ULE doesn't consider hlt_cpus_mask 796 * and unlike other schedulers it actually schedules threads to 797 * those CPUs. 798 */ 799 if (CPU_ISSET(PCPU_GET(cpuid), &hlt_cpus_mask)) 800 return; 801 #endif 802 803 /* Look up our local APIC structure for the tick counters. */ 804 la = &lapics[PCPU_GET(apic_id)]; 805 (*la->la_timer_count)++; 806 critical_enter(); 807 if (lapic_et.et_active) { 808 td = curthread; 809 td->td_intr_nesting_level++; 810 oldframe = td->td_intr_frame; 811 td->td_intr_frame = frame; 812 lapic_et.et_event_cb(&lapic_et, lapic_et.et_arg); 813 td->td_intr_frame = oldframe; 814 td->td_intr_nesting_level--; 815 } 816 critical_exit(); 817 } 818 819 static void 820 lapic_timer_set_divisor(u_int divisor) 821 { 822 823 KASSERT(powerof2(divisor), ("lapic: invalid divisor %u", divisor)); 824 KASSERT(ffs(divisor) <= sizeof(lapic_timer_divisors) / 825 sizeof(u_int32_t), ("lapic: invalid divisor %u", divisor)); 826 lapic->dcr_timer = lapic_timer_divisors[ffs(divisor) - 1]; 827 } 828 829 static void 830 lapic_timer_oneshot(struct lapic *la, u_int count, int enable_int) 831 { 832 u_int32_t value; 833 834 value = la->lvt_timer_cache; 835 value &= ~APIC_LVTT_TM; 836 value |= APIC_LVTT_TM_ONE_SHOT; 837 if (enable_int) 838 value &= ~APIC_LVT_M; 839 lapic->lvt_timer = value; 840 lapic->icr_timer = count; 841 } 842 843 static void 844 lapic_timer_periodic(struct lapic *la, u_int count, int enable_int) 845 { 846 u_int32_t value; 847 848 value = la->lvt_timer_cache; 849 value &= ~APIC_LVTT_TM; 850 value |= APIC_LVTT_TM_PERIODIC; 851 if (enable_int) 852 value &= ~APIC_LVT_M; 853 lapic->lvt_timer = value; 854 lapic->icr_timer = count; 855 } 856 857 static void 858 lapic_timer_stop(struct lapic *la) 859 { 860 u_int32_t value; 861 862 value = la->lvt_timer_cache; 863 value &= ~APIC_LVTT_TM; 864 value |= APIC_LVT_M; 865 lapic->lvt_timer = value; 866 } 867 868 void 869 lapic_handle_cmc(void) 870 { 871 872 lapic_eoi(); 873 cmc_intr(); 874 } 875 876 /* 877 * Called from the mca_init() to activate the CMC interrupt if this CPU is 878 * responsible for monitoring any MC banks for CMC events. Since mca_init() 879 * is called prior to lapic_setup() during boot, this just needs to unmask 880 * this CPU's LVT_CMCI entry. 881 */ 882 void 883 lapic_enable_cmc(void) 884 { 885 u_int apic_id; 886 887 #ifdef DEV_ATPIC 888 if (lapic == NULL) 889 return; 890 #endif 891 apic_id = PCPU_GET(apic_id); 892 KASSERT(lapics[apic_id].la_present, 893 ("%s: missing APIC %u", __func__, apic_id)); 894 lapics[apic_id].la_lvts[LVT_CMCI].lvt_masked = 0; 895 lapics[apic_id].la_lvts[LVT_CMCI].lvt_active = 1; 896 if (bootverbose) 897 printf("lapic%u: CMCI unmasked\n", apic_id); 898 } 899 900 void 901 lapic_handle_error(void) 902 { 903 u_int32_t esr; 904 905 /* 906 * Read the contents of the error status register. Write to 907 * the register first before reading from it to force the APIC 908 * to update its value to indicate any errors that have 909 * occurred since the previous write to the register. 910 */ 911 lapic->esr = 0; 912 esr = lapic->esr; 913 914 printf("CPU%d: local APIC error 0x%x\n", PCPU_GET(cpuid), esr); 915 lapic_eoi(); 916 } 917 918 u_int 919 apic_cpuid(u_int apic_id) 920 { 921 #ifdef SMP 922 return apic_cpuids[apic_id]; 923 #else 924 return 0; 925 #endif 926 } 927 928 /* Request a free IDT vector to be used by the specified IRQ. */ 929 u_int 930 apic_alloc_vector(u_int apic_id, u_int irq) 931 { 932 u_int vector; 933 934 KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 935 936 /* 937 * Search for a free vector. Currently we just use a very simple 938 * algorithm to find the first free vector. 939 */ 940 mtx_lock_spin(&icu_lock); 941 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 942 if (lapics[apic_id].la_ioint_irqs[vector] != -1) 943 continue; 944 lapics[apic_id].la_ioint_irqs[vector] = irq; 945 mtx_unlock_spin(&icu_lock); 946 return (vector + APIC_IO_INTS); 947 } 948 mtx_unlock_spin(&icu_lock); 949 return (0); 950 } 951 952 /* 953 * Request 'count' free contiguous IDT vectors to be used by 'count' 954 * IRQs. 'count' must be a power of two and the vectors will be 955 * aligned on a boundary of 'align'. If the request cannot be 956 * satisfied, 0 is returned. 957 */ 958 u_int 959 apic_alloc_vectors(u_int apic_id, u_int *irqs, u_int count, u_int align) 960 { 961 u_int first, run, vector; 962 963 KASSERT(powerof2(count), ("bad count")); 964 KASSERT(powerof2(align), ("bad align")); 965 KASSERT(align >= count, ("align < count")); 966 #ifdef INVARIANTS 967 for (run = 0; run < count; run++) 968 KASSERT(irqs[run] < NUM_IO_INTS, ("Invalid IRQ %u at index %u", 969 irqs[run], run)); 970 #endif 971 972 /* 973 * Search for 'count' free vectors. As with apic_alloc_vector(), 974 * this just uses a simple first fit algorithm. 975 */ 976 run = 0; 977 first = 0; 978 mtx_lock_spin(&icu_lock); 979 for (vector = 0; vector < APIC_NUM_IOINTS; vector++) { 980 981 /* Vector is in use, end run. */ 982 if (lapics[apic_id].la_ioint_irqs[vector] != -1) { 983 run = 0; 984 first = 0; 985 continue; 986 } 987 988 /* Start a new run if run == 0 and vector is aligned. */ 989 if (run == 0) { 990 if ((vector & (align - 1)) != 0) 991 continue; 992 first = vector; 993 } 994 run++; 995 996 /* Keep looping if the run isn't long enough yet. */ 997 if (run < count) 998 continue; 999 1000 /* Found a run, assign IRQs and return the first vector. */ 1001 for (vector = 0; vector < count; vector++) 1002 lapics[apic_id].la_ioint_irqs[first + vector] = 1003 irqs[vector]; 1004 mtx_unlock_spin(&icu_lock); 1005 return (first + APIC_IO_INTS); 1006 } 1007 mtx_unlock_spin(&icu_lock); 1008 printf("APIC: Couldn't find APIC vectors for %u IRQs\n", count); 1009 return (0); 1010 } 1011 1012 /* 1013 * Enable a vector for a particular apic_id. Since all lapics share idt 1014 * entries and ioint_handlers this enables the vector on all lapics. lapics 1015 * which do not have the vector configured would report spurious interrupts 1016 * should it fire. 1017 */ 1018 void 1019 apic_enable_vector(u_int apic_id, u_int vector) 1020 { 1021 1022 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1023 KASSERT(ioint_handlers[vector / 32] != NULL, 1024 ("No ISR handler for vector %u", vector)); 1025 #ifdef KDTRACE_HOOKS 1026 KASSERT(vector != IDT_DTRACE_RET, 1027 ("Attempt to overwrite DTrace entry")); 1028 #endif 1029 setidt(vector, ioint_handlers[vector / 32], SDT_APIC, SEL_KPL, 1030 GSEL_APIC); 1031 } 1032 1033 void 1034 apic_disable_vector(u_int apic_id, u_int vector) 1035 { 1036 1037 KASSERT(vector != IDT_SYSCALL, ("Attempt to overwrite syscall entry")); 1038 #ifdef KDTRACE_HOOKS 1039 KASSERT(vector != IDT_DTRACE_RET, 1040 ("Attempt to overwrite DTrace entry")); 1041 #endif 1042 KASSERT(ioint_handlers[vector / 32] != NULL, 1043 ("No ISR handler for vector %u", vector)); 1044 #ifdef notyet 1045 /* 1046 * We can not currently clear the idt entry because other cpus 1047 * may have a valid vector at this offset. 1048 */ 1049 setidt(vector, &IDTVEC(rsvd), SDT_APICT, SEL_KPL, GSEL_APIC); 1050 #endif 1051 } 1052 1053 /* Release an APIC vector when it's no longer in use. */ 1054 void 1055 apic_free_vector(u_int apic_id, u_int vector, u_int irq) 1056 { 1057 struct thread *td; 1058 1059 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1060 vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1061 ("Vector %u does not map to an IRQ line", vector)); 1062 KASSERT(irq < NUM_IO_INTS, ("Invalid IRQ %u", irq)); 1063 KASSERT(lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] == 1064 irq, ("IRQ mismatch")); 1065 #ifdef KDTRACE_HOOKS 1066 KASSERT(vector != IDT_DTRACE_RET, 1067 ("Attempt to overwrite DTrace entry")); 1068 #endif 1069 1070 /* 1071 * Bind us to the cpu that owned the vector before freeing it so 1072 * we don't lose an interrupt delivery race. 1073 */ 1074 td = curthread; 1075 if (!rebooting) { 1076 thread_lock(td); 1077 if (sched_is_bound(td)) 1078 panic("apic_free_vector: Thread already bound.\n"); 1079 sched_bind(td, apic_cpuid(apic_id)); 1080 thread_unlock(td); 1081 } 1082 mtx_lock_spin(&icu_lock); 1083 lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS] = -1; 1084 mtx_unlock_spin(&icu_lock); 1085 if (!rebooting) { 1086 thread_lock(td); 1087 sched_unbind(td); 1088 thread_unlock(td); 1089 } 1090 } 1091 1092 /* Map an IDT vector (APIC) to an IRQ (interrupt source). */ 1093 u_int 1094 apic_idt_to_irq(u_int apic_id, u_int vector) 1095 { 1096 int irq; 1097 1098 KASSERT(vector >= APIC_IO_INTS && vector != IDT_SYSCALL && 1099 vector <= APIC_IO_INTS + APIC_NUM_IOINTS, 1100 ("Vector %u does not map to an IRQ line", vector)); 1101 #ifdef KDTRACE_HOOKS 1102 KASSERT(vector != IDT_DTRACE_RET, 1103 ("Attempt to overwrite DTrace entry")); 1104 #endif 1105 irq = lapics[apic_id].la_ioint_irqs[vector - APIC_IO_INTS]; 1106 if (irq < 0) 1107 irq = 0; 1108 return (irq); 1109 } 1110 1111 #ifdef DDB 1112 /* 1113 * Dump data about APIC IDT vector mappings. 1114 */ 1115 DB_SHOW_COMMAND(apic, db_show_apic) 1116 { 1117 struct intsrc *isrc; 1118 int i, verbose; 1119 u_int apic_id; 1120 u_int irq; 1121 1122 if (strcmp(modif, "vv") == 0) 1123 verbose = 2; 1124 else if (strcmp(modif, "v") == 0) 1125 verbose = 1; 1126 else 1127 verbose = 0; 1128 for (apic_id = 0; apic_id <= MAX_APIC_ID; apic_id++) { 1129 if (lapics[apic_id].la_present == 0) 1130 continue; 1131 db_printf("Interrupts bound to lapic %u\n", apic_id); 1132 for (i = 0; i < APIC_NUM_IOINTS + 1 && !db_pager_quit; i++) { 1133 irq = lapics[apic_id].la_ioint_irqs[i]; 1134 if (irq == -1 || irq == IRQ_SYSCALL) 1135 continue; 1136 #ifdef KDTRACE_HOOKS 1137 if (irq == IRQ_DTRACE_RET) 1138 continue; 1139 #endif 1140 db_printf("vec 0x%2x -> ", i + APIC_IO_INTS); 1141 if (irq == IRQ_TIMER) 1142 db_printf("lapic timer\n"); 1143 else if (irq < NUM_IO_INTS) { 1144 isrc = intr_lookup_source(irq); 1145 if (isrc == NULL || verbose == 0) 1146 db_printf("IRQ %u\n", irq); 1147 else 1148 db_dump_intr_event(isrc->is_event, 1149 verbose == 2); 1150 } else 1151 db_printf("IRQ %u ???\n", irq); 1152 } 1153 } 1154 } 1155 1156 static void 1157 dump_mask(const char *prefix, uint32_t v, int base) 1158 { 1159 int i, first; 1160 1161 first = 1; 1162 for (i = 0; i < 32; i++) 1163 if (v & (1 << i)) { 1164 if (first) { 1165 db_printf("%s:", prefix); 1166 first = 0; 1167 } 1168 db_printf(" %02x", base + i); 1169 } 1170 if (!first) 1171 db_printf("\n"); 1172 } 1173 1174 /* Show info from the lapic regs for this CPU. */ 1175 DB_SHOW_COMMAND(lapic, db_show_lapic) 1176 { 1177 uint32_t v; 1178 1179 db_printf("lapic ID = %d\n", lapic_id()); 1180 v = lapic->version; 1181 db_printf("version = %d.%d\n", (v & APIC_VER_VERSION) >> 4, 1182 v & 0xf); 1183 db_printf("max LVT = %d\n", (v & APIC_VER_MAXLVT) >> MAXLVTSHIFT); 1184 v = lapic->svr; 1185 db_printf("SVR = %02x (%s)\n", v & APIC_SVR_VECTOR, 1186 v & APIC_SVR_ENABLE ? "enabled" : "disabled"); 1187 db_printf("TPR = %02x\n", lapic->tpr); 1188 1189 #define dump_field(prefix, index) \ 1190 dump_mask(__XSTRING(prefix ## index), lapic->prefix ## index, \ 1191 index * 32) 1192 1193 db_printf("In-service Interrupts:\n"); 1194 dump_field(isr, 0); 1195 dump_field(isr, 1); 1196 dump_field(isr, 2); 1197 dump_field(isr, 3); 1198 dump_field(isr, 4); 1199 dump_field(isr, 5); 1200 dump_field(isr, 6); 1201 dump_field(isr, 7); 1202 1203 db_printf("TMR Interrupts:\n"); 1204 dump_field(tmr, 0); 1205 dump_field(tmr, 1); 1206 dump_field(tmr, 2); 1207 dump_field(tmr, 3); 1208 dump_field(tmr, 4); 1209 dump_field(tmr, 5); 1210 dump_field(tmr, 6); 1211 dump_field(tmr, 7); 1212 1213 db_printf("IRR Interrupts:\n"); 1214 dump_field(irr, 0); 1215 dump_field(irr, 1); 1216 dump_field(irr, 2); 1217 dump_field(irr, 3); 1218 dump_field(irr, 4); 1219 dump_field(irr, 5); 1220 dump_field(irr, 6); 1221 dump_field(irr, 7); 1222 1223 #undef dump_field 1224 } 1225 #endif 1226 1227 /* 1228 * APIC probing support code. This includes code to manage enumerators. 1229 */ 1230 1231 static SLIST_HEAD(, apic_enumerator) enumerators = 1232 SLIST_HEAD_INITIALIZER(enumerators); 1233 static struct apic_enumerator *best_enum; 1234 1235 void 1236 apic_register_enumerator(struct apic_enumerator *enumerator) 1237 { 1238 #ifdef INVARIANTS 1239 struct apic_enumerator *apic_enum; 1240 1241 SLIST_FOREACH(apic_enum, &enumerators, apic_next) { 1242 if (apic_enum == enumerator) 1243 panic("%s: Duplicate register of %s", __func__, 1244 enumerator->apic_name); 1245 } 1246 #endif 1247 SLIST_INSERT_HEAD(&enumerators, enumerator, apic_next); 1248 } 1249 1250 /* 1251 * We have to look for CPU's very, very early because certain subsystems 1252 * want to know how many CPU's we have extremely early on in the boot 1253 * process. 1254 */ 1255 static void 1256 apic_init(void *dummy __unused) 1257 { 1258 struct apic_enumerator *enumerator; 1259 #ifndef __amd64__ 1260 uint64_t apic_base; 1261 #endif 1262 int retval, best; 1263 1264 /* We only support built in local APICs. */ 1265 if (!(cpu_feature & CPUID_APIC)) 1266 return; 1267 1268 /* Don't probe if APIC mode is disabled. */ 1269 if (resource_disabled("apic", 0)) 1270 return; 1271 1272 /* Probe all the enumerators to find the best match. */ 1273 best_enum = NULL; 1274 best = 0; 1275 SLIST_FOREACH(enumerator, &enumerators, apic_next) { 1276 retval = enumerator->apic_probe(); 1277 if (retval > 0) 1278 continue; 1279 if (best_enum == NULL || best < retval) { 1280 best_enum = enumerator; 1281 best = retval; 1282 } 1283 } 1284 if (best_enum == NULL) { 1285 if (bootverbose) 1286 printf("APIC: Could not find any APICs.\n"); 1287 #ifndef DEV_ATPIC 1288 panic("running without device atpic requires a local APIC"); 1289 #endif 1290 return; 1291 } 1292 1293 if (bootverbose) 1294 printf("APIC: Using the %s enumerator.\n", 1295 best_enum->apic_name); 1296 1297 #ifndef __amd64__ 1298 /* 1299 * To work around an errata, we disable the local APIC on some 1300 * CPUs during early startup. We need to turn the local APIC back 1301 * on on such CPUs now. 1302 */ 1303 if (cpu == CPU_686 && cpu_vendor_id == CPU_VENDOR_INTEL && 1304 (cpu_id & 0xff0) == 0x610) { 1305 apic_base = rdmsr(MSR_APICBASE); 1306 apic_base |= APICBASE_ENABLED; 1307 wrmsr(MSR_APICBASE, apic_base); 1308 } 1309 #endif 1310 1311 /* Probe the CPU's in the system. */ 1312 retval = best_enum->apic_probe_cpus(); 1313 if (retval != 0) 1314 printf("%s: Failed to probe CPUs: returned %d\n", 1315 best_enum->apic_name, retval); 1316 1317 } 1318 SYSINIT(apic_init, SI_SUB_TUNABLES - 1, SI_ORDER_SECOND, apic_init, NULL); 1319 1320 /* 1321 * Setup the local APIC. We have to do this prior to starting up the APs 1322 * in the SMP case. 1323 */ 1324 static void 1325 apic_setup_local(void *dummy __unused) 1326 { 1327 int retval; 1328 1329 if (best_enum == NULL) 1330 return; 1331 1332 /* Initialize the local APIC. */ 1333 retval = best_enum->apic_setup_local(); 1334 if (retval != 0) 1335 printf("%s: Failed to setup the local APIC: returned %d\n", 1336 best_enum->apic_name, retval); 1337 } 1338 SYSINIT(apic_setup_local, SI_SUB_CPU, SI_ORDER_SECOND, apic_setup_local, NULL); 1339 1340 /* 1341 * Setup the I/O APICs. 1342 */ 1343 static void 1344 apic_setup_io(void *dummy __unused) 1345 { 1346 int retval; 1347 1348 if (best_enum == NULL) 1349 return; 1350 1351 /* 1352 * Local APIC must be registered before other PICs and pseudo PICs 1353 * for proper suspend/resume order. 1354 */ 1355 #ifndef XEN 1356 intr_register_pic(&lapic_pic); 1357 #endif 1358 1359 retval = best_enum->apic_setup_io(); 1360 if (retval != 0) 1361 printf("%s: Failed to setup I/O APICs: returned %d\n", 1362 best_enum->apic_name, retval); 1363 #ifdef XEN 1364 return; 1365 #endif 1366 /* 1367 * Finish setting up the local APIC on the BSP once we know how to 1368 * properly program the LINT pins. 1369 */ 1370 lapic_setup(1); 1371 if (bootverbose) 1372 lapic_dump("BSP"); 1373 1374 /* Enable the MSI "pic". */ 1375 msi_init(); 1376 } 1377 SYSINIT(apic_setup_io, SI_SUB_INTR, SI_ORDER_SECOND, apic_setup_io, NULL); 1378 1379 #ifdef SMP 1380 /* 1381 * Inter Processor Interrupt functions. The lapic_ipi_*() functions are 1382 * private to the MD code. The public interface for the rest of the 1383 * kernel is defined in mp_machdep.c. 1384 */ 1385 int 1386 lapic_ipi_wait(int delay) 1387 { 1388 int x, incr; 1389 1390 /* 1391 * Wait delay loops for IPI to be sent. This is highly bogus 1392 * since this is sensitive to CPU clock speed. If delay is 1393 * -1, we wait forever. 1394 */ 1395 if (delay == -1) { 1396 incr = 0; 1397 delay = 1; 1398 } else 1399 incr = 1; 1400 for (x = 0; x < delay; x += incr) { 1401 if ((lapic->icr_lo & APIC_DELSTAT_MASK) == APIC_DELSTAT_IDLE) 1402 return (1); 1403 ia32_pause(); 1404 } 1405 return (0); 1406 } 1407 1408 void 1409 lapic_ipi_raw(register_t icrlo, u_int dest) 1410 { 1411 register_t value, saveintr; 1412 1413 /* XXX: Need more sanity checking of icrlo? */ 1414 KASSERT(lapic != NULL, ("%s called too early", __func__)); 1415 KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1416 ("%s: invalid dest field", __func__)); 1417 KASSERT((icrlo & APIC_ICRLO_RESV_MASK) == 0, 1418 ("%s: reserved bits set in ICR LO register", __func__)); 1419 1420 /* Set destination in ICR HI register if it is being used. */ 1421 saveintr = intr_disable(); 1422 if ((icrlo & APIC_DEST_MASK) == APIC_DEST_DESTFLD) { 1423 value = lapic->icr_hi; 1424 value &= ~APIC_ID_MASK; 1425 value |= dest << APIC_ID_SHIFT; 1426 lapic->icr_hi = value; 1427 } 1428 1429 /* Program the contents of the IPI and dispatch it. */ 1430 value = lapic->icr_lo; 1431 value &= APIC_ICRLO_RESV_MASK; 1432 value |= icrlo; 1433 lapic->icr_lo = value; 1434 intr_restore(saveintr); 1435 } 1436 1437 #define BEFORE_SPIN 1000000 1438 #ifdef DETECT_DEADLOCK 1439 #define AFTER_SPIN 1000 1440 #endif 1441 1442 void 1443 lapic_ipi_vectored(u_int vector, int dest) 1444 { 1445 register_t icrlo, destfield; 1446 1447 KASSERT((vector & ~APIC_VECTOR_MASK) == 0, 1448 ("%s: invalid vector %d", __func__, vector)); 1449 1450 icrlo = APIC_DESTMODE_PHY | APIC_TRIGMOD_EDGE; 1451 1452 /* 1453 * IPI_STOP_HARD is just a "fake" vector used to send a NMI. 1454 * Use special rules regard NMI if passed, otherwise specify 1455 * the vector. 1456 */ 1457 if (vector == IPI_STOP_HARD) 1458 icrlo |= APIC_DELMODE_NMI | APIC_LEVEL_ASSERT; 1459 else 1460 icrlo |= vector | APIC_DELMODE_FIXED | APIC_LEVEL_DEASSERT; 1461 destfield = 0; 1462 switch (dest) { 1463 case APIC_IPI_DEST_SELF: 1464 icrlo |= APIC_DEST_SELF; 1465 break; 1466 case APIC_IPI_DEST_ALL: 1467 icrlo |= APIC_DEST_ALLISELF; 1468 break; 1469 case APIC_IPI_DEST_OTHERS: 1470 icrlo |= APIC_DEST_ALLESELF; 1471 break; 1472 default: 1473 KASSERT((dest & ~(APIC_ID_MASK >> APIC_ID_SHIFT)) == 0, 1474 ("%s: invalid destination 0x%x", __func__, dest)); 1475 destfield = dest; 1476 } 1477 1478 /* Wait for an earlier IPI to finish. */ 1479 if (!lapic_ipi_wait(BEFORE_SPIN)) { 1480 if (panicstr != NULL) 1481 return; 1482 else 1483 panic("APIC: Previous IPI is stuck"); 1484 } 1485 1486 lapic_ipi_raw(icrlo, destfield); 1487 1488 #ifdef DETECT_DEADLOCK 1489 /* Wait for IPI to be delivered. */ 1490 if (!lapic_ipi_wait(AFTER_SPIN)) { 1491 #ifdef needsattention 1492 /* 1493 * XXX FIXME: 1494 * 1495 * The above function waits for the message to actually be 1496 * delivered. It breaks out after an arbitrary timeout 1497 * since the message should eventually be delivered (at 1498 * least in theory) and that if it wasn't we would catch 1499 * the failure with the check above when the next IPI is 1500 * sent. 1501 * 1502 * We could skip this wait entirely, EXCEPT it probably 1503 * protects us from other routines that assume that the 1504 * message was delivered and acted upon when this function 1505 * returns. 1506 */ 1507 printf("APIC: IPI might be stuck\n"); 1508 #else /* !needsattention */ 1509 /* Wait until mesage is sent without a timeout. */ 1510 while (lapic->icr_lo & APIC_DELSTAT_PEND) 1511 ia32_pause(); 1512 #endif /* needsattention */ 1513 } 1514 #endif /* DETECT_DEADLOCK */ 1515 } 1516 #endif /* SMP */ 1517