1 /* 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. The name of the developer may NOT be used to endorse or promote products 11 * derived from this software without specific prior written permission. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.37.2.7 2003/01/25 02:31:47 peter Exp $ 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/kernel.h> 31 #include <sys/bus.h> 32 #include <sys/machintr.h> 33 #include <machine/globaldata.h> 34 #include <machine/smp.h> 35 #include <machine/md_var.h> 36 #include <machine/pmap.h> 37 #include <machine/specialreg.h> 38 #include <machine_base/apic/lapic.h> 39 #include <machine_base/apic/ioapic.h> 40 #include <machine_base/apic/ioapic_abi.h> 41 #include <machine_base/apic/apicvar.h> 42 #include <machine_base/icu/icu_var.h> 43 #include <machine/segments.h> 44 #include <sys/thread2.h> 45 46 #include <machine/cputypes.h> 47 #include <machine/intr_machdep.h> 48 49 extern int naps; 50 51 volatile lapic_t *lapic; 52 53 static void lapic_timer_calibrate(void); 54 static void lapic_timer_set_divisor(int); 55 static void lapic_timer_fixup_handler(void *); 56 static void lapic_timer_restart_handler(void *); 57 58 void lapic_timer_process(void); 59 void lapic_timer_process_frame(struct intrframe *); 60 void lapic_timer_always(struct intrframe *); 61 62 static int lapic_timer_enable = 1; 63 TUNABLE_INT("hw.lapic_timer_enable", &lapic_timer_enable); 64 65 static void lapic_timer_intr_reload(struct cputimer_intr *, sysclock_t); 66 static void lapic_timer_intr_enable(struct cputimer_intr *); 67 static void lapic_timer_intr_restart(struct cputimer_intr *); 68 static void lapic_timer_intr_pmfixup(struct cputimer_intr *); 69 70 static struct cputimer_intr lapic_cputimer_intr = { 71 .freq = 0, 72 .reload = lapic_timer_intr_reload, 73 .enable = lapic_timer_intr_enable, 74 .config = cputimer_intr_default_config, 75 .restart = lapic_timer_intr_restart, 76 .pmfixup = lapic_timer_intr_pmfixup, 77 .initclock = cputimer_intr_default_initclock, 78 .next = SLIST_ENTRY_INITIALIZER, 79 .name = "lapic", 80 .type = CPUTIMER_INTR_LAPIC, 81 .prio = CPUTIMER_INTR_PRIO_LAPIC, 82 .caps = CPUTIMER_INTR_CAP_NONE 83 }; 84 85 static int lapic_timer_divisor_idx = -1; 86 static const uint32_t lapic_timer_divisors[] = { 87 APIC_TDCR_2, APIC_TDCR_4, APIC_TDCR_8, APIC_TDCR_16, 88 APIC_TDCR_32, APIC_TDCR_64, APIC_TDCR_128, APIC_TDCR_1 89 }; 90 #define APIC_TIMER_NDIVISORS (int)(NELEM(lapic_timer_divisors)) 91 92 /* 93 * APIC ID <-> CPU ID mapping structures. 94 */ 95 int cpu_id_to_apic_id[NAPICID]; 96 int apic_id_to_cpu_id[NAPICID]; 97 int lapic_enable = 1; 98 99 /* 100 * Enable LAPIC, configure interrupts. 101 */ 102 void 103 lapic_init(boolean_t bsp) 104 { 105 uint32_t timer; 106 u_int temp; 107 108 /* 109 * Install vectors 110 * 111 * Since IDT is shared between BSP and APs, these vectors 112 * only need to be installed once; we do it on BSP. 113 */ 114 if (bsp) { 115 if (cpu_vendor_id == CPU_VENDOR_AMD && 116 CPUID_TO_FAMILY(cpu_id) >= 0xf) { 117 uint32_t tcr; 118 119 /* 120 * Set the LINTEN bit in the HyperTransport 121 * Transaction Control Register. 122 * 123 * This will cause EXTINT and NMI interrupts 124 * routed over the hypertransport bus to be 125 * fed into the LAPIC LINT0/LINT1. If the bit 126 * isn't set, the interrupts will go to the 127 * general cpu INTR/NMI pins. On a dual-core 128 * cpu the interrupt winds up going to BOTH cpus. 129 * The first cpu that does the interrupt ack 130 * cycle will get the correct interrupt. The 131 * second cpu that does it will get a spurious 132 * interrupt vector (typically IRQ 7). 133 */ 134 outl(0x0cf8, 135 (1 << 31) | /* enable */ 136 (0 << 16) | /* bus */ 137 (0x18 << 11) | /* dev (cpu + 0x18) */ 138 (0 << 8) | /* func */ 139 0x68 /* reg */ 140 ); 141 tcr = inl(0xcfc); 142 if ((tcr & 0x00010000) == 0) { 143 kprintf("LAPIC: AMD LINTEN on\n"); 144 outl(0xcfc, tcr|0x00010000); 145 } 146 outl(0x0cf8, 0); 147 } 148 149 /* Install a 'Spurious INTerrupt' vector */ 150 setidt_global(XSPURIOUSINT_OFFSET, Xspuriousint, 151 SDT_SYSIGT, SEL_KPL, 0); 152 153 /* Install a timer vector */ 154 setidt_global(XTIMER_OFFSET, Xtimer, 155 SDT_SYSIGT, SEL_KPL, 0); 156 157 /* Install an inter-CPU IPI for TLB invalidation */ 158 setidt_global(XINVLTLB_OFFSET, Xinvltlb, 159 SDT_SYSIGT, SEL_KPL, 0); 160 161 /* Install an inter-CPU IPI for IPIQ messaging */ 162 setidt_global(XIPIQ_OFFSET, Xipiq, 163 SDT_SYSIGT, SEL_KPL, 0); 164 165 /* Install an inter-CPU IPI for CPU stop/restart */ 166 setidt_global(XCPUSTOP_OFFSET, Xcpustop, 167 SDT_SYSIGT, SEL_KPL, 0); 168 } 169 170 /* 171 * Setup LINT0 as ExtINT on the BSP. This is theoretically an 172 * aggregate interrupt input from the 8259. The INTA cycle 173 * will be routed to the external controller (the 8259) which 174 * is expected to supply the vector. 175 * 176 * Must be setup edge triggered, active high. 177 * 178 * Disable LINT0 on BSP, if I/O APIC is enabled. 179 * 180 * Disable LINT0 on the APs. It doesn't matter what delivery 181 * mode we use because we leave it masked. 182 */ 183 temp = lapic->lvt_lint0; 184 temp &= ~(APIC_LVT_MASKED | APIC_LVT_TRIG_MASK | 185 APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK); 186 if (bsp) { 187 temp |= APIC_LVT_DM_EXTINT; 188 if (ioapic_enable) 189 temp |= APIC_LVT_MASKED; 190 } else { 191 temp |= APIC_LVT_DM_FIXED | APIC_LVT_MASKED; 192 } 193 lapic->lvt_lint0 = temp; 194 195 /* 196 * Setup LINT1 as NMI. 197 * 198 * Must be setup edge trigger, active high. 199 * 200 * Enable LINT1 on BSP, if I/O APIC is enabled. 201 * 202 * Disable LINT1 on the APs. 203 */ 204 temp = lapic->lvt_lint1; 205 temp &= ~(APIC_LVT_MASKED | APIC_LVT_TRIG_MASK | 206 APIC_LVT_POLARITY_MASK | APIC_LVT_DM_MASK); 207 temp |= APIC_LVT_MASKED | APIC_LVT_DM_NMI; 208 if (bsp && ioapic_enable) 209 temp &= ~APIC_LVT_MASKED; 210 lapic->lvt_lint1 = temp; 211 212 /* 213 * Mask the LAPIC error interrupt, LAPIC performance counter 214 * interrupt. 215 */ 216 lapic->lvt_error = lapic->lvt_error | APIC_LVT_MASKED; 217 lapic->lvt_pcint = lapic->lvt_pcint | APIC_LVT_MASKED; 218 219 /* 220 * Set LAPIC timer vector and mask the LAPIC timer interrupt. 221 */ 222 timer = lapic->lvt_timer; 223 timer &= ~APIC_LVTT_VECTOR; 224 timer |= XTIMER_OFFSET; 225 timer |= APIC_LVTT_MASKED; 226 lapic->lvt_timer = timer; 227 228 /* 229 * Set the Task Priority Register as needed. At the moment allow 230 * interrupts on all cpus (the APs will remain CLId until they are 231 * ready to deal). 232 */ 233 temp = lapic->tpr; 234 temp &= ~APIC_TPR_PRIO; /* clear priority field */ 235 lapic->tpr = temp; 236 237 /* 238 * Enable the LAPIC 239 */ 240 temp = lapic->svr; 241 temp |= APIC_SVR_ENABLE; /* enable the LAPIC */ 242 temp &= ~APIC_SVR_FOCUS_DISABLE; /* enable lopri focus processor */ 243 244 /* 245 * Set the spurious interrupt vector. The low 4 bits of the vector 246 * must be 1111. 247 */ 248 if ((XSPURIOUSINT_OFFSET & 0x0F) != 0x0F) 249 panic("bad XSPURIOUSINT_OFFSET: 0x%08x", XSPURIOUSINT_OFFSET); 250 temp &= ~APIC_SVR_VECTOR; 251 temp |= XSPURIOUSINT_OFFSET; 252 253 lapic->svr = temp; 254 255 /* 256 * Pump out a few EOIs to clean out interrupts that got through 257 * before we were able to set the TPR. 258 */ 259 lapic->eoi = 0; 260 lapic->eoi = 0; 261 lapic->eoi = 0; 262 263 if (bsp) { 264 lapic_timer_calibrate(); 265 if (lapic_timer_enable) { 266 if (cpu_thermal_feature & CPUID_THERMAL_ARAT) { 267 /* 268 * Local APIC timer will not stop 269 * in deep C-state. 270 */ 271 lapic_cputimer_intr.caps |= 272 CPUTIMER_INTR_CAP_PS; 273 } 274 cputimer_intr_register(&lapic_cputimer_intr); 275 cputimer_intr_select(&lapic_cputimer_intr, 0); 276 } 277 } else { 278 lapic_timer_set_divisor(lapic_timer_divisor_idx); 279 } 280 281 if (bootverbose) 282 apic_dump("apic_initialize()"); 283 } 284 285 static void 286 lapic_timer_set_divisor(int divisor_idx) 287 { 288 KKASSERT(divisor_idx >= 0 && divisor_idx < APIC_TIMER_NDIVISORS); 289 lapic->dcr_timer = lapic_timer_divisors[divisor_idx]; 290 } 291 292 static void 293 lapic_timer_oneshot(u_int count) 294 { 295 uint32_t value; 296 297 value = lapic->lvt_timer; 298 value &= ~APIC_LVTT_PERIODIC; 299 lapic->lvt_timer = value; 300 lapic->icr_timer = count; 301 } 302 303 static void 304 lapic_timer_oneshot_quick(u_int count) 305 { 306 lapic->icr_timer = count; 307 } 308 309 static void 310 lapic_timer_calibrate(void) 311 { 312 sysclock_t value; 313 314 /* Try to calibrate the local APIC timer. */ 315 for (lapic_timer_divisor_idx = 0; 316 lapic_timer_divisor_idx < APIC_TIMER_NDIVISORS; 317 lapic_timer_divisor_idx++) { 318 lapic_timer_set_divisor(lapic_timer_divisor_idx); 319 lapic_timer_oneshot(APIC_TIMER_MAX_COUNT); 320 DELAY(2000000); 321 value = APIC_TIMER_MAX_COUNT - lapic->ccr_timer; 322 if (value != APIC_TIMER_MAX_COUNT) 323 break; 324 } 325 if (lapic_timer_divisor_idx >= APIC_TIMER_NDIVISORS) 326 panic("lapic: no proper timer divisor?!"); 327 lapic_cputimer_intr.freq = value / 2; 328 329 kprintf("lapic: divisor index %d, frequency %u Hz\n", 330 lapic_timer_divisor_idx, lapic_cputimer_intr.freq); 331 } 332 333 static void 334 lapic_timer_process_oncpu(struct globaldata *gd, struct intrframe *frame) 335 { 336 sysclock_t count; 337 338 gd->gd_timer_running = 0; 339 340 count = sys_cputimer->count(); 341 if (TAILQ_FIRST(&gd->gd_systimerq) != NULL) 342 systimer_intr(&count, 0, frame); 343 } 344 345 void 346 lapic_timer_process(void) 347 { 348 lapic_timer_process_oncpu(mycpu, NULL); 349 } 350 351 void 352 lapic_timer_process_frame(struct intrframe *frame) 353 { 354 lapic_timer_process_oncpu(mycpu, frame); 355 } 356 357 /* 358 * This manual debugging code is called unconditionally from Xtimer 359 * (the lapic timer interrupt) whether the current thread is in a 360 * critical section or not) and can be useful in tracking down lockups. 361 * 362 * NOTE: MANUAL DEBUG CODE 363 */ 364 #if 0 365 static int saveticks[SMP_MAXCPU]; 366 static int savecounts[SMP_MAXCPU]; 367 #endif 368 369 void 370 lapic_timer_always(struct intrframe *frame) 371 { 372 #if 0 373 globaldata_t gd = mycpu; 374 int cpu = gd->gd_cpuid; 375 char buf[64]; 376 short *gptr; 377 int i; 378 379 if (cpu <= 20) { 380 gptr = (short *)0xFFFFFFFF800b8000 + 80 * cpu; 381 *gptr = ((*gptr + 1) & 0x00FF) | 0x0700; 382 ++gptr; 383 384 ksnprintf(buf, sizeof(buf), " %p %16s %d %16s ", 385 (void *)frame->if_rip, gd->gd_curthread->td_comm, ticks, 386 gd->gd_infomsg); 387 for (i = 0; buf[i]; ++i) { 388 gptr[i] = 0x0700 | (unsigned char)buf[i]; 389 } 390 } 391 #if 0 392 if (saveticks[gd->gd_cpuid] != ticks) { 393 saveticks[gd->gd_cpuid] = ticks; 394 savecounts[gd->gd_cpuid] = 0; 395 } 396 ++savecounts[gd->gd_cpuid]; 397 if (savecounts[gd->gd_cpuid] > 2000 && panicstr == NULL) { 398 panic("cpud %d panicing on ticks failure", 399 gd->gd_cpuid); 400 } 401 for (i = 0; i < ncpus; ++i) { 402 int delta; 403 if (saveticks[i] && panicstr == NULL) { 404 delta = saveticks[i] - ticks; 405 if (delta < -10 || delta > 10) { 406 panic("cpu %d panicing on cpu %d watchdog", 407 gd->gd_cpuid, i); 408 } 409 } 410 } 411 #endif 412 #endif 413 } 414 415 static void 416 lapic_timer_intr_reload(struct cputimer_intr *cti, sysclock_t reload) 417 { 418 struct globaldata *gd = mycpu; 419 420 reload = (int64_t)reload * cti->freq / sys_cputimer->freq; 421 if (reload < 2) 422 reload = 2; 423 424 if (gd->gd_timer_running) { 425 if (reload < lapic->ccr_timer) 426 lapic_timer_oneshot_quick(reload); 427 } else { 428 gd->gd_timer_running = 1; 429 lapic_timer_oneshot_quick(reload); 430 } 431 } 432 433 static void 434 lapic_timer_intr_enable(struct cputimer_intr *cti __unused) 435 { 436 uint32_t timer; 437 438 timer = lapic->lvt_timer; 439 timer &= ~(APIC_LVTT_MASKED | APIC_LVTT_PERIODIC); 440 lapic->lvt_timer = timer; 441 442 lapic_timer_fixup_handler(NULL); 443 } 444 445 static void 446 lapic_timer_fixup_handler(void *arg) 447 { 448 int *started = arg; 449 450 if (started != NULL) 451 *started = 0; 452 453 if (cpu_vendor_id == CPU_VENDOR_AMD) { 454 /* 455 * Detect the presence of C1E capability mostly on latest 456 * dual-cores (or future) k8 family. This feature renders 457 * the local APIC timer dead, so we disable it by reading 458 * the Interrupt Pending Message register and clearing both 459 * C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27). 460 * 461 * Reference: 462 * "BIOS and Kernel Developer's Guide for AMD NPT 463 * Family 0Fh Processors" 464 * #32559 revision 3.00 465 */ 466 if ((cpu_id & 0x00000f00) == 0x00000f00 && 467 (cpu_id & 0x0fff0000) >= 0x00040000) { 468 uint64_t msr; 469 470 msr = rdmsr(0xc0010055); 471 if (msr & 0x18000000) { 472 struct globaldata *gd = mycpu; 473 474 kprintf("cpu%d: AMD C1E detected\n", 475 gd->gd_cpuid); 476 wrmsr(0xc0010055, msr & ~0x18000000ULL); 477 478 /* 479 * We are kinda stalled; 480 * kick start again. 481 */ 482 gd->gd_timer_running = 1; 483 lapic_timer_oneshot_quick(2); 484 485 if (started != NULL) 486 *started = 1; 487 } 488 } 489 } 490 } 491 492 static void 493 lapic_timer_restart_handler(void *dummy __unused) 494 { 495 int started; 496 497 lapic_timer_fixup_handler(&started); 498 if (!started) { 499 struct globaldata *gd = mycpu; 500 501 gd->gd_timer_running = 1; 502 lapic_timer_oneshot_quick(2); 503 } 504 } 505 506 /* 507 * This function is called only by ACPICA code currently: 508 * - AMD C1E fixup. AMD C1E only seems to happen after ACPI 509 * module controls PM. So once ACPICA is attached, we try 510 * to apply the fixup to prevent LAPIC timer from hanging. 511 */ 512 static void 513 lapic_timer_intr_pmfixup(struct cputimer_intr *cti __unused) 514 { 515 lwkt_send_ipiq_mask(smp_active_mask, 516 lapic_timer_fixup_handler, NULL); 517 } 518 519 static void 520 lapic_timer_intr_restart(struct cputimer_intr *cti __unused) 521 { 522 lwkt_send_ipiq_mask(smp_active_mask, lapic_timer_restart_handler, NULL); 523 } 524 525 526 /* 527 * dump contents of local APIC registers 528 */ 529 void 530 apic_dump(char* str) 531 { 532 kprintf("SMP: CPU%d %s:\n", mycpu->gd_cpuid, str); 533 kprintf(" lint0: 0x%08x lint1: 0x%08x TPR: 0x%08x SVR: 0x%08x\n", 534 lapic->lvt_lint0, lapic->lvt_lint1, lapic->tpr, lapic->svr); 535 } 536 537 /* 538 * Inter Processor Interrupt functions. 539 */ 540 541 /* 542 * Send APIC IPI 'vector' to 'destType' via 'deliveryMode'. 543 * 544 * destType is 1 of: APIC_DEST_SELF, APIC_DEST_ALLISELF, APIC_DEST_ALLESELF 545 * vector is any valid SYSTEM INT vector 546 * delivery_mode is 1 of: APIC_DELMODE_FIXED, APIC_DELMODE_LOWPRIO 547 * 548 * WARNINGS! 549 * 550 * We now implement a per-cpu interlock (gd->gd_npoll) to prevent more than 551 * one IPI from being sent to any given cpu at a time. Thus we no longer 552 * have to process incoming IPIs while waiting for the status to clear. 553 * No deadlock should be possible. 554 * 555 * We now physically disable interrupts for the lapic ICR operation. If 556 * we do not do this then it looks like an EOI sent to the lapic (which 557 * occurs even with a critical section) can interfere with the command 558 * register ready status and cause an IPI to be lost. 559 * 560 * e.g. an interrupt can occur, issue the EOI, IRET, and cause the command 561 * register to busy just before we write to icr_lo, resulting in a lost 562 * issuance. This only appears to occur on Intel cpus and is not 563 * documented. It could simply be that cpus are so fast these days that 564 * it was always an issue, but is only now rearing its ugly head. This 565 * is conjecture. 566 */ 567 int 568 apic_ipi(int dest_type, int vector, int delivery_mode) 569 { 570 unsigned long rflags; 571 u_long icr_lo; 572 573 rflags = read_rflags(); 574 cpu_disable_intr(); 575 while ((lapic->icr_lo & APIC_DELSTAT_MASK) != 0) { 576 cpu_pause(); 577 } 578 icr_lo = (lapic->icr_lo & APIC_ICRLO_RESV_MASK) | dest_type | 579 delivery_mode | vector; 580 lapic->icr_lo = icr_lo; 581 write_rflags(rflags); 582 583 return 0; 584 } 585 586 void 587 single_apic_ipi(int cpu, int vector, int delivery_mode) 588 { 589 unsigned long rflags; 590 u_long icr_lo; 591 u_long icr_hi; 592 593 rflags = read_rflags(); 594 cpu_disable_intr(); 595 while ((lapic->icr_lo & APIC_DELSTAT_MASK) != 0) { 596 cpu_pause(); 597 } 598 icr_hi = lapic->icr_hi & ~APIC_ID_MASK; 599 icr_hi |= (CPUID_TO_APICID(cpu) << 24); 600 lapic->icr_hi = icr_hi; 601 602 /* build ICR_LOW */ 603 icr_lo = (lapic->icr_lo & APIC_ICRLO_RESV_MASK) | 604 APIC_DEST_DESTFLD | delivery_mode | vector; 605 606 /* write APIC ICR */ 607 lapic->icr_lo = icr_lo; 608 write_rflags(rflags); 609 } 610 611 #if 0 612 613 /* 614 * Returns 0 if the apic is busy, 1 if we were able to queue the request. 615 * 616 * NOT WORKING YET! The code as-is may end up not queueing an IPI at all 617 * to the target, and the scheduler does not 'poll' for IPI messages. 618 */ 619 int 620 single_apic_ipi_passive(int cpu, int vector, int delivery_mode) 621 { 622 u_long icr_lo; 623 u_long icr_hi; 624 625 crit_enter(); 626 if ((lapic->icr_lo & APIC_DELSTAT_MASK) != 0) { 627 crit_exit(); 628 return(0); 629 } 630 icr_hi = lapic->icr_hi & ~APIC_ID_MASK; 631 icr_hi |= (CPUID_TO_APICID(cpu) << 24); 632 lapic->icr_hi = icr_hi; 633 634 /* build IRC_LOW */ 635 icr_lo = (lapic->icr_lo & APIC_RESV2_MASK) 636 | APIC_DEST_DESTFLD | delivery_mode | vector; 637 638 /* write APIC ICR */ 639 lapic->icr_lo = icr_lo; 640 crit_exit(); 641 return(1); 642 } 643 644 #endif 645 646 /* 647 * Send APIC IPI 'vector' to 'target's via 'delivery_mode'. 648 * 649 * target is a bitmask of destination cpus. Vector is any 650 * valid system INT vector. Delivery mode may be either 651 * APIC_DELMODE_FIXED or APIC_DELMODE_LOWPRIO. 652 */ 653 void 654 selected_apic_ipi(cpumask_t target, int vector, int delivery_mode) 655 { 656 crit_enter(); 657 while (CPUMASK_TESTNZERO(target)) { 658 int n = BSFCPUMASK(target); 659 CPUMASK_NANDBIT(target, n); 660 single_apic_ipi(n, vector, delivery_mode); 661 } 662 crit_exit(); 663 } 664 665 /* 666 * Timer code, in development... 667 * - suggested by rgrimes@gndrsh.aac.dev.com 668 */ 669 int 670 get_apic_timer_frequency(void) 671 { 672 return(lapic_cputimer_intr.freq); 673 } 674 675 /* 676 * Load a 'downcount time' in uSeconds. 677 */ 678 void 679 set_apic_timer(int us) 680 { 681 u_int count; 682 683 /* 684 * When we reach here, lapic timer's frequency 685 * must have been calculated as well as the 686 * divisor (lapic->dcr_timer is setup during the 687 * divisor calculation). 688 */ 689 KKASSERT(lapic_cputimer_intr.freq != 0 && 690 lapic_timer_divisor_idx >= 0); 691 692 count = ((us * (int64_t)lapic_cputimer_intr.freq) + 999999) / 1000000; 693 lapic_timer_oneshot(count); 694 } 695 696 697 /* 698 * Read remaining time in timer. 699 */ 700 int 701 read_apic_timer(void) 702 { 703 #if 0 704 /** XXX FIXME: we need to return the actual remaining time, 705 * for now we just return the remaining count. 706 */ 707 #else 708 return lapic->ccr_timer; 709 #endif 710 } 711 712 713 /* 714 * Spin-style delay, set delay time in uS, spin till it drains. 715 */ 716 void 717 u_sleep(int count) 718 { 719 set_apic_timer(count); 720 while (read_apic_timer()) 721 /* spin */ ; 722 } 723 724 int 725 lapic_unused_apic_id(int start) 726 { 727 int i; 728 729 for (i = start; i < APICID_MAX; ++i) { 730 if (APICID_TO_CPUID(i) == -1) 731 return i; 732 } 733 return NAPICID; 734 } 735 736 void 737 lapic_map(vm_paddr_t lapic_addr) 738 { 739 lapic = pmap_mapdev_uncacheable(lapic_addr, sizeof(struct LAPIC)); 740 } 741 742 static TAILQ_HEAD(, lapic_enumerator) lapic_enumerators = 743 TAILQ_HEAD_INITIALIZER(lapic_enumerators); 744 745 int 746 lapic_config(void) 747 { 748 struct lapic_enumerator *e; 749 int error, i, ap_max; 750 751 KKASSERT(lapic_enable); 752 753 for (i = 0; i < NAPICID; ++i) 754 APICID_TO_CPUID(i) = -1; 755 756 TAILQ_FOREACH(e, &lapic_enumerators, lapic_link) { 757 error = e->lapic_probe(e); 758 if (!error) 759 break; 760 } 761 if (e == NULL) { 762 kprintf("LAPIC: Can't find LAPIC\n"); 763 return ENXIO; 764 } 765 766 error = e->lapic_enumerate(e); 767 if (error) { 768 kprintf("LAPIC: enumeration failed\n"); 769 return ENXIO; 770 } 771 772 ap_max = MAXCPU - 1; 773 TUNABLE_INT_FETCH("hw.ap_max", &ap_max); 774 if (ap_max > MAXCPU - 1) 775 ap_max = MAXCPU - 1; 776 777 if (naps > ap_max) { 778 kprintf("LAPIC: Warning use only %d out of %d " 779 "available APs\n", 780 ap_max, naps); 781 naps = ap_max; 782 } 783 784 return 0; 785 } 786 787 void 788 lapic_enumerator_register(struct lapic_enumerator *ne) 789 { 790 struct lapic_enumerator *e; 791 792 TAILQ_FOREACH(e, &lapic_enumerators, lapic_link) { 793 if (e->lapic_prio < ne->lapic_prio) { 794 TAILQ_INSERT_BEFORE(e, ne, lapic_link); 795 return; 796 } 797 } 798 TAILQ_INSERT_TAIL(&lapic_enumerators, ne, lapic_link); 799 } 800 801 void 802 lapic_set_cpuid(int cpu_id, int apic_id) 803 { 804 CPUID_TO_APICID(cpu_id) = apic_id; 805 APICID_TO_CPUID(apic_id) = cpu_id; 806 } 807 808 void 809 lapic_fixup_noioapic(void) 810 { 811 u_int temp; 812 813 /* Only allowed on BSP */ 814 KKASSERT(mycpuid == 0); 815 KKASSERT(!ioapic_enable); 816 817 temp = lapic->lvt_lint0; 818 temp &= ~APIC_LVT_MASKED; 819 lapic->lvt_lint0 = temp; 820 821 temp = lapic->lvt_lint1; 822 temp |= APIC_LVT_MASKED; 823 lapic->lvt_lint1 = temp; 824 } 825 826 static void 827 lapic_sysinit(void *dummy __unused) 828 { 829 if (lapic_enable) { 830 int error; 831 832 error = lapic_config(); 833 if (error) 834 lapic_enable = 0; 835 } 836 837 if (lapic_enable) { 838 /* Initialize BSP's local APIC */ 839 lapic_init(TRUE); 840 } else if (ioapic_enable) { 841 ioapic_enable = 0; 842 icu_reinit_noioapic(); 843 } 844 } 845 SYSINIT(lapic, SI_BOOT2_LAPIC, SI_ORDER_FIRST, lapic_sysinit, NULL); 846