1 /* $NetBSD: lapic.c,v 1.89 2022/09/07 00:40:19 knakahara Exp $ */
2
3 /*-
4 * Copyright (c) 2000, 2008, 2020 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by RedBack Networks Inc.
9 *
10 * Author: Bill Sommerfeld
11 *
12 * Redistribution and use in source and binary forms, with or without
13 * modification, are permitted provided that the following conditions
14 * are met:
15 * 1. Redistributions of source code must retain the above copyright
16 * notice, this list of conditions and the following disclaimer.
17 * 2. Redistributions in binary form must reproduce the above copyright
18 * notice, this list of conditions and the following disclaimer in the
19 * documentation and/or other materials provided with the distribution.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
22 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
23 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
24 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
25 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
26 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
27 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
28 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
29 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
30 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
31 * POSSIBILITY OF SUCH DAMAGE.
32 */
33
34 #include <sys/cdefs.h>
35 __KERNEL_RCSID(0, "$NetBSD: lapic.c,v 1.89 2022/09/07 00:40:19 knakahara Exp $");
36
37 #include "acpica.h"
38 #include "ioapic.h"
39 #include "opt_acpi.h"
40 #include "opt_ddb.h"
41 #include "opt_mpbios.h" /* for MPDEBUG */
42 #include "opt_multiprocessor.h"
43 #include "opt_ntp.h"
44 #include "opt_xen.h"
45
46
47 #include <sys/param.h>
48 #include <sys/proc.h>
49 #include <sys/systm.h>
50 #include <sys/device.h>
51 #include <sys/timetc.h>
52
53 #include <uvm/uvm_extern.h>
54
55 #include <dev/ic/i8253reg.h>
56
57 #include <x86/machdep.h>
58 #include <machine/cpu.h>
59 #include <machine/cpu_counter.h>
60 #include <machine/cpufunc.h>
61 #include <machine/cpuvar.h>
62 #include <machine/pmap.h>
63 #include <machine/vmparam.h>
64 #include <machine/mpacpi.h>
65 #include <machine/mpbiosvar.h>
66 #include <machine/pcb.h>
67 #include <machine/pmap_private.h>
68 #include <machine/specialreg.h>
69 #include <machine/segments.h>
70 #include <x86/x86/tsc.h>
71 #include <x86/i82093var.h>
72
73 #include <machine/apicvar.h>
74 #include <machine/i82489reg.h>
75 #include <machine/i82489var.h>
76
77 #ifndef XENPV
78 #if NACPICA > 0
79 #include <dev/acpi/acpica.h>
80 #include <dev/acpi/acpivar.h>
81 #endif
82
83 #ifdef DDB
84 #include <machine/db_machdep.h>
85 #ifdef MULTIPROCESSOR
86 #ifdef __x86_64__
87 typedef void (vector)(void);
88 extern vector Xintr_x2apic_ddbipi;
89 extern int ddb_vec;
90 #endif
91 #endif
92 #endif
93
94 #include <dev/vmt/vmtreg.h> /* for vmt_hvcall() */
95 #include <dev/vmt/vmtvar.h> /* for vmt_hvcall() */
96
97 /* Referenced from vector.S */
98 void lapic_clockintr(void *, struct intrframe *);
99
100 static void lapic_delay(unsigned int);
101 static uint32_t lapic_gettick(void);
102 static void lapic_setup_bsp(paddr_t);
103 static void lapic_map(paddr_t);
104
105 static void lapic_hwmask(struct pic *, int);
106 static void lapic_hwunmask(struct pic *, int);
107 static void lapic_setup(struct pic *, struct cpu_info *, int, int, int);
108 /* Make it public to call via ddb */
109 void lapic_dump(void);
110
111 struct pic local_pic = {
112 .pic_name = "lapic",
113 .pic_type = PIC_LAPIC,
114 .pic_lock = __SIMPLELOCK_UNLOCKED,
115 .pic_hwmask = lapic_hwmask,
116 .pic_hwunmask = lapic_hwunmask,
117 .pic_addroute = lapic_setup,
118 .pic_delroute = lapic_setup,
119 .pic_intr_get_devname = x86_intr_get_devname,
120 .pic_intr_get_assigned = x86_intr_get_assigned,
121 .pic_intr_get_count = x86_intr_get_count,
122 };
123
124 static int i82489_ipi(int vec, int target, int dl);
125 static int x2apic_ipi(int vec, int target, int dl);
126 int (*x86_ipi)(int, int, int) = i82489_ipi;
127
128 bool x2apic_mode __read_mostly;
129 #ifdef LAPIC_ENABLE_X2APIC
130 bool x2apic_enable = true;
131 #else
132 bool x2apic_enable = false;
133 #endif
134
135 static bool lapic_broken_periodic __read_mostly;
136
137 static uint32_t
i82489_readreg(u_int reg)138 i82489_readreg(u_int reg)
139 {
140 return *((volatile uint32_t *)(local_apic_va + reg));
141 }
142
143 static void
i82489_writereg(u_int reg,uint32_t val)144 i82489_writereg(u_int reg, uint32_t val)
145 {
146 *((volatile uint32_t *)(local_apic_va + reg)) = val;
147 }
148
149 static uint32_t
i82489_cpu_number(void)150 i82489_cpu_number(void)
151 {
152 return i82489_readreg(LAPIC_ID) >> LAPIC_ID_SHIFT;
153 }
154
155 static uint32_t
x2apic_readreg(u_int reg)156 x2apic_readreg(u_int reg)
157 {
158 return rdmsr(MSR_X2APIC_BASE + (reg >> 4));
159 }
160
161 static void
x2apic_writereg(u_int reg,uint32_t val)162 x2apic_writereg(u_int reg, uint32_t val)
163 {
164 x86_mfence();
165 wrmsr(MSR_X2APIC_BASE + (reg >> 4), val);
166 }
167
168 static void
x2apic_writereg64(u_int reg,uint64_t val)169 x2apic_writereg64(u_int reg, uint64_t val)
170 {
171 KDASSERT(reg == LAPIC_ICRLO);
172 x86_mfence();
173 wrmsr(MSR_X2APIC_BASE + (reg >> 4), val);
174 }
175
176 static void
x2apic_write_icr(uint32_t hi,uint32_t lo)177 x2apic_write_icr(uint32_t hi, uint32_t lo)
178 {
179 x2apic_writereg64(LAPIC_ICRLO, ((uint64_t)hi << 32) | lo);
180 }
181
182 static uint32_t
x2apic_cpu_number(void)183 x2apic_cpu_number(void)
184 {
185 return x2apic_readreg(LAPIC_ID);
186 }
187
188 uint32_t
lapic_readreg(u_int reg)189 lapic_readreg(u_int reg)
190 {
191 if (x2apic_mode)
192 return x2apic_readreg(reg);
193 return i82489_readreg(reg);
194 }
195
196 void
lapic_writereg(u_int reg,uint32_t val)197 lapic_writereg(u_int reg, uint32_t val)
198 {
199 if (x2apic_mode)
200 x2apic_writereg(reg, val);
201 else
202 i82489_writereg(reg, val);
203 }
204
205 void
lapic_write_tpri(uint32_t val)206 lapic_write_tpri(uint32_t val)
207 {
208
209 val &= LAPIC_TPRI_MASK;
210 #ifdef i386
211 lapic_writereg(LAPIC_TPRI, val);
212 #else
213 lcr8(val >> 4);
214 #endif
215 }
216
217 uint32_t
lapic_cpu_number(void)218 lapic_cpu_number(void)
219 {
220 if (x2apic_mode)
221 return x2apic_cpu_number();
222 return i82489_cpu_number();
223 }
224
225 static void
lapic_enable_x2apic(void)226 lapic_enable_x2apic(void)
227 {
228 uint64_t apicbase;
229
230 apicbase = rdmsr(MSR_APICBASE);
231 if (!ISSET(apicbase, APICBASE_EN)) {
232 apicbase |= APICBASE_EN;
233 wrmsr(MSR_APICBASE, apicbase);
234 }
235 apicbase |= APICBASE_EXTD;
236 wrmsr(MSR_APICBASE, apicbase);
237 }
238
239 bool
lapic_is_x2apic(void)240 lapic_is_x2apic(void)
241 {
242 uint64_t msr;
243
244 if (!ISSET(cpu_feature[0], CPUID_APIC) ||
245 rdmsr_safe(MSR_APICBASE, &msr) == EFAULT)
246 return false;
247 return (msr & (APICBASE_EN | APICBASE_EXTD)) ==
248 (APICBASE_EN | APICBASE_EXTD);
249 }
250
251 /*
252 * Initialize the local APIC on the BSP.
253 */
254 static void
lapic_setup_bsp(paddr_t lapic_base)255 lapic_setup_bsp(paddr_t lapic_base)
256 {
257 u_int regs[6];
258 const char *reason = NULL;
259 const char *hw_vendor;
260 bool bios_x2apic;
261
262 if (ISSET(cpu_feature[1], CPUID2_X2APIC)) {
263 #if NACPICA > 0
264 if (acpi_present) {
265 ACPI_TABLE_DMAR *dmar;
266 ACPI_STATUS status;
267
268 /*
269 * Automatically detect several configurations where
270 * x2APIC mode is known to cause troubles. User can
271 * override the setting with hw.x2apic_enable tunable.
272 */
273 status = AcpiGetTable(ACPI_SIG_DMAR, 1,
274 (ACPI_TABLE_HEADER **)&dmar);
275 if (ACPI_SUCCESS(status)) {
276 if (ISSET(dmar->Flags, ACPI_DMAR_X2APIC_OPT_OUT)) {
277 reason = "by DMAR table";
278 }
279 AcpiPutTable(&dmar->Header);
280 }
281 }
282 #endif /* NACPICA > 0 */
283 if (vm_guest == VM_GUEST_VMWARE) {
284 vmt_hvcall(VM_CMD_GET_VCPU_INFO, regs);
285 if (ISSET(regs[0], VCPUINFO_VCPU_RESERVED) ||
286 !ISSET(regs[0], VCPUINFO_LEGACY_X2APIC))
287 reason = "inside VMWare without intr "
288 "redirection";
289 } else if (vm_guest == VM_GUEST_XENHVM) {
290 reason = "due to running under XEN";
291 } else if (vm_guest == VM_GUEST_NO &&
292 CPUID_TO_FAMILY(curcpu()->ci_signature) == 6 &&
293 CPUID_TO_MODEL(curcpu()->ci_signature) == 0x2a) {
294 hw_vendor = pmf_get_platform("board-vendor");
295 if (hw_vendor != NULL) {
296 /*
297 * It seems that some Lenovo and ASUS
298 * SandyBridge-based notebook BIOSes have a bug
299 * which prevents booting AP in x2APIC mode.
300 * Since the only way to detect mobile CPU is
301 * to check northbridge pci id, which cannot be
302 * done that early, disable x2APIC for all
303 * Lenovo and ASUS SandyBridge machines.
304 */
305 if (strcmp(hw_vendor, "LENOVO") == 0 ||
306 strcmp(hw_vendor, "ASUSTeK Computer Inc.") == 0) {
307 reason = "for a suspected SandyBridge "
308 "BIOS bug";
309 }
310 }
311 }
312 bios_x2apic = lapic_is_x2apic();
313 if (reason != NULL && bios_x2apic) {
314 aprint_verbose("x2APIC should be disabled %s but "
315 "already enabled by BIOS; enabling.\n", reason);
316 reason = NULL;
317 }
318 if (reason == NULL)
319 x2apic_mode = true;
320 else
321 aprint_verbose("x2APIC available but disabled %s\n",
322 reason);
323 if (x2apic_enable != x2apic_mode) {
324 if (bios_x2apic && !x2apic_enable)
325 aprint_verbose("x2APIC disabled by user and "
326 "enabled by BIOS; ignoring user setting.\n");
327 else
328 x2apic_mode = x2apic_enable;
329 }
330 }
331 if (x2apic_mode) {
332 x86_ipi = x2apic_ipi;
333 #if NIOAPIC > 0
334 struct ioapic_softc *ioapic;
335 for (ioapic = ioapics; ioapic != NULL; ioapic = ioapic->sc_next) {
336 ioapic->sc_pic.pic_edge_stubs = x2apic_edge_stubs;
337 ioapic->sc_pic.pic_level_stubs = x2apic_level_stubs;
338 }
339 #endif
340 #if defined(DDB) && defined(MULTIPROCESSOR)
341 #ifdef __x86_64__
342 struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
343 idt_descriptor_t *idt = iv->iv_idt;
344 set_idtgate(&idt[ddb_vec], &Xintr_x2apic_ddbipi, 1,
345 SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
346 #else
347 /*
348 * Set DDB IPI handler in cpu_set_tss_gates() when cpu0 is
349 * attached.
350 */
351 #endif
352 #endif
353
354 x86_disable_intr();
355 lapic_enable_x2apic();
356 #ifdef MULTIPROCESSOR
357 cpu_init_first(); /* Catch up to changed cpu_number() */
358 #endif
359 lapic_write_tpri(0);
360 x86_enable_intr();
361 } else
362 lapic_map(lapic_base);
363 }
364
365 static void
lapic_map(paddr_t lapic_base)366 lapic_map(paddr_t lapic_base)
367 {
368 pt_entry_t *pte;
369 vaddr_t va = local_apic_va;
370
371 /*
372 * If the CPU has an APIC MSR, use it and ignore the supplied value:
373 * some ACPI implementations have been observed to pass bad values.
374 * Additionally, ensure that the lapic is enabled as we are committed
375 * to using it at this point. Be conservative and assume that the MSR
376 * is not present on the Pentium (is it?).
377 */
378 if (CPUID_TO_FAMILY(curcpu()->ci_signature) >= 6) {
379 lapic_base = (paddr_t)rdmsr(MSR_APICBASE);
380 if ((lapic_base & APICBASE_PHYSADDR) == 0) {
381 lapic_base |= LAPIC_BASE;
382 }
383 wrmsr(MSR_APICBASE, lapic_base | APICBASE_EN);
384 lapic_base &= APICBASE_PHYSADDR;
385 }
386
387 x86_disable_intr();
388
389 /*
390 * Map local apic. If we have a local apic, it's safe to assume
391 * we're on a 486 or better and can use invlpg and non-cacheable PTE's
392 *
393 * Whap the PTE "by hand" rather than calling pmap_kenter_pa because
394 * the latter will attempt to invoke TLB shootdown code just as we
395 * might have changed the value of cpu_number()..
396 */
397
398 pte = kvtopte(va);
399 *pte = lapic_base | PTE_W | PTE_P | PTE_PCD | pmap_pg_g | pmap_pg_nx;
400 invlpg(va);
401
402 #ifdef MULTIPROCESSOR
403 cpu_init_first(); /* Catch up to changed cpu_number() */
404 #endif
405
406 lapic_write_tpri(0);
407 x86_enable_intr();
408 }
409
410 /*
411 * enable local apic
412 */
413 void
lapic_enable(void)414 lapic_enable(void)
415 {
416 lapic_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR);
417 }
418
419 void
lapic_set_lvt(void)420 lapic_set_lvt(void)
421 {
422 struct cpu_info *ci = curcpu();
423 int i;
424 struct mp_intr_map *mpi;
425 uint32_t lint0, lint1;
426
427 #ifdef MULTIPROCESSOR
428 if (mp_verbose) {
429 apic_format_redir(device_xname(ci->ci_dev), "prelint", 0,
430 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT0));
431 apic_format_redir(device_xname(ci->ci_dev), "prelint", 1,
432 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(LAPIC_LVT_LINT1));
433 }
434 #endif
435
436 /*
437 * If an I/O APIC has been attached, assume that it is used instead of
438 * the 8259A for interrupt delivery. Otherwise request the LAPIC to
439 * get external interrupts via LINT0 for the primary CPU.
440 */
441 lint0 = LAPIC_DLMODE_EXTINT;
442 if (nioapics > 0 || !CPU_IS_PRIMARY(curcpu()))
443 lint0 |= LAPIC_LVT_MASKED;
444 lapic_writereg(LAPIC_LVT_LINT0, lint0);
445
446 /*
447 * Non Maskable Interrupts are to be delivered to the primary CPU.
448 */
449 lint1 = LAPIC_DLMODE_NMI;
450 if (!CPU_IS_PRIMARY(curcpu()))
451 lint1 |= LAPIC_LVT_MASKED;
452 lapic_writereg(LAPIC_LVT_LINT1, lint1);
453
454 for (i = 0; i < mp_nintr; i++) {
455 mpi = &mp_intrs[i];
456 if (mpi->ioapic == NULL && (mpi->cpu_id == MPS_ALL_APICS ||
457 mpi->cpu_id == ci->ci_cpuid)) {
458 if (mpi->ioapic_pin > 1)
459 aprint_error_dev(ci->ci_dev,
460 "%s: WARNING: bad pin value %d\n",
461 __func__, mpi->ioapic_pin);
462 if (mpi->ioapic_pin == 0)
463 lapic_writereg(LAPIC_LVT_LINT0, mpi->redir);
464 else
465 lapic_writereg(LAPIC_LVT_LINT1, mpi->redir);
466 }
467 }
468
469 #ifdef MULTIPROCESSOR
470 if (mp_verbose)
471 lapic_dump();
472 #endif
473 }
474
475 /*
476 * Initialize fixed idt vectors for use by local apic.
477 */
478 void
lapic_boot_init(paddr_t lapic_base)479 lapic_boot_init(paddr_t lapic_base)
480 {
481 struct idt_vec *iv = &(cpu_info_primary.ci_idtvec);
482
483 lapic_setup_bsp(lapic_base);
484
485 #ifdef MULTIPROCESSOR
486 idt_vec_reserve(iv, LAPIC_IPI_VECTOR);
487 idt_vec_set(iv, LAPIC_IPI_VECTOR,
488 x2apic_mode ? Xintr_x2apic_ipi : Xintr_lapic_ipi);
489
490 idt_vec_reserve(iv, LAPIC_TLB_VECTOR);
491 idt_vec_set(iv, LAPIC_TLB_VECTOR,
492 x2apic_mode ? Xintr_x2apic_tlb : Xintr_lapic_tlb);
493 #endif
494 idt_vec_reserve(iv, LAPIC_SPURIOUS_VECTOR);
495 idt_vec_set(iv, LAPIC_SPURIOUS_VECTOR, Xintrspurious);
496
497 idt_vec_reserve(iv, LAPIC_TIMER_VECTOR);
498 idt_vec_set(iv, LAPIC_TIMER_VECTOR,
499 x2apic_mode ? Xintr_x2apic_ltimer : Xintr_lapic_ltimer);
500 }
501
502 static uint32_t
lapic_gettick(void)503 lapic_gettick(void)
504 {
505 return lapic_readreg(LAPIC_CCR_TIMER);
506 }
507
508 #include <sys/kernel.h> /* for hz */
509
510 uint32_t lapic_tval;
511
512 /*
513 * this gets us up to a 4GHz busclock....
514 */
515 uint32_t lapic_per_second;
516 uint32_t lapic_frac_usec_per_cycle;
517 uint64_t lapic_frac_cycle_per_usec;
518 uint32_t lapic_delaytab[26];
519
520 static u_int
lapic_get_timecount(struct timecounter * tc)521 lapic_get_timecount(struct timecounter *tc)
522 {
523 struct cpu_info *ci;
524 uint32_t cur_timer;
525 int s;
526
527 s = splhigh();
528 ci = curcpu();
529
530 /*
531 * Check for a race against the clockinterrupt.
532 * The update of ci_lapic_counter is blocked by splhigh() and
533 * the check for a pending clockinterrupt compensates for that.
534 *
535 * If the current tick is almost the Initial Counter, explicitly
536 * check for the pending interrupt bit as the interrupt delivery
537 * could be asynchronious and compensate as well.
538 *
539 * This can't be done without splhigh() as the calling code might
540 * have masked the clockinterrupt already.
541 *
542 * This code assumes that clockinterrupts are not missed.
543 */
544 cur_timer = lapic_gettick();
545 if (cur_timer >= lapic_tval - 1) {
546 uint16_t reg = LAPIC_IRR + LAPIC_TIMER_VECTOR / 32 * 16;
547
548 if (lapic_readreg(reg) & (1 << (LAPIC_TIMER_VECTOR % 32))) {
549 cur_timer -= lapic_tval;
550 }
551 } else if (ci->ci_ipending & (1ULL << LIR_TIMER))
552 cur_timer = lapic_gettick() - lapic_tval;
553 cur_timer = ci->ci_lapic_counter - cur_timer;
554 splx(s);
555
556 return cur_timer;
557 }
558
559 static struct timecounter lapic_timecounter = {
560 .tc_get_timecount = lapic_get_timecount,
561 .tc_counter_mask = ~0u,
562 .tc_name = "lapic",
563 .tc_quality =
564 #ifndef MULTIPROCESSOR
565 2100,
566 #else
567 -100, /* per CPU state */
568 #endif
569 };
570
571 extern u_int i8254_get_timecount(struct timecounter *);
572
573 void
lapic_clockintr(void * arg,struct intrframe * frame)574 lapic_clockintr(void *arg, struct intrframe *frame)
575 {
576 struct cpu_info *ci = curcpu();
577
578 ci->ci_lapic_counter += lapic_tval;
579 ci->ci_isources[LIR_TIMER]->is_evcnt.ev_count++;
580 hardclock((struct clockframe *)frame);
581 }
582
583 void
lapic_reset(void)584 lapic_reset(void)
585 {
586
587 /*
588 * Mask the clock interrupt and set mode,
589 * then set divisor,
590 * then unmask and set the vector.
591 */
592 lapic_writereg(LAPIC_LVT_TIMER,
593 LAPIC_LVT_TMM_PERIODIC | LAPIC_LVT_MASKED);
594 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
595 lapic_writereg(LAPIC_ICR_TIMER, lapic_tval);
596 lapic_writereg(LAPIC_LVT_TIMER,
597 LAPIC_LVT_TMM_PERIODIC | LAPIC_TIMER_VECTOR);
598 lapic_writereg(LAPIC_EOI, 0);
599 }
600
601 static void
lapic_initclock(void)602 lapic_initclock(void)
603 {
604
605 if (curcpu() == &cpu_info_primary) {
606 /*
607 * Recalibrate the timer using the cycle counter, now that
608 * the cycle counter itself has been recalibrated.
609 */
610 lapic_calibrate_timer(true);
611
612 /*
613 * Hook up time counter. This assume that all LAPICs have
614 * the same frequency.
615 */
616 lapic_timecounter.tc_frequency = lapic_per_second;
617 tc_init(&lapic_timecounter);
618 }
619
620 /* Start local apic countdown timer running, in repeated mode. */
621 lapic_reset();
622 }
623
624 /*
625 * Calibrate the local apic count-down timer (which is running at
626 * bus-clock speed) vs. the i8254 counter/timer (which is running at
627 * a fixed rate).
628 *
629 * The Intel MP spec says: "An MP operating system may use the IRQ8
630 * real-time clock as a reference to determine the actual APIC timer clock
631 * speed."
632 *
633 * We're actually using the IRQ0 timer. Hmm.
634 */
635 void
lapic_calibrate_timer(bool secondpass)636 lapic_calibrate_timer(bool secondpass)
637 {
638 struct cpu_info *ci = curcpu();
639 uint64_t tmp;
640 int i;
641 char tbuf[9];
642
643 KASSERT(ci == &cpu_info_primary);
644
645 aprint_debug_dev(ci->ci_dev, "[re]calibrating local timer\n");
646
647 /*
648 * Configure timer to one-shot, interrupt masked,
649 * large positive number.
650 */
651 x86_disable_intr();
652 lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_MASKED);
653 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
654 lapic_writereg(LAPIC_ICR_TIMER, 0x80000000);
655 (void)lapic_gettick();
656
657 if (secondpass && cpu_hascounter()) {
658 /*
659 * Second pass calibration, using the TSC which has ideally
660 * been calibrated using the HPET or information gleaned
661 * from MSRs by this point.
662 */
663 uint64_t l0, l1, t0, t1;
664
665 (void)cpu_counter();
666 t0 = cpu_counter();
667 l0 = lapic_gettick();
668 t0 += cpu_counter();
669 DELAY(50000);
670 t1 = cpu_counter();
671 l1 = lapic_gettick();
672 t1 += cpu_counter();
673
674 tmp = (l0 - l1) * cpu_frequency(ci) / ((t1 - t0 + 1) / 2);
675 lapic_per_second = rounddown(tmp + 500, 1000);
676 } else if (lapic_per_second == 0) {
677 /*
678 * Inaccurate first pass calibration using the i8254.
679 */
680 unsigned int seen, delta, initial_i8254, initial_lapic;
681 unsigned int cur_i8254, cur_lapic;
682
683 (void)gettick();
684 initial_lapic = lapic_gettick();
685 initial_i8254 = gettick();
686 for (seen = 0; seen < TIMER_FREQ / 100; seen += delta) {
687 cur_i8254 = gettick();
688 if (cur_i8254 > initial_i8254)
689 delta = x86_rtclock_tval - (cur_i8254 - initial_i8254);
690 else
691 delta = initial_i8254 - cur_i8254;
692 initial_i8254 = cur_i8254;
693 }
694 cur_lapic = lapic_gettick();
695 tmp = initial_lapic - cur_lapic;
696 lapic_per_second = (tmp * TIMER_FREQ + seen / 2) / seen;
697 }
698 x86_enable_intr();
699
700 humanize_number(tbuf, sizeof(tbuf), lapic_per_second, "Hz", 1000);
701 aprint_debug_dev(ci->ci_dev, "apic clock running at %s\n", tbuf);
702
703 if (lapic_per_second != 0) {
704 /*
705 * reprogram the apic timer to run in periodic mode.
706 * XXX need to program timer on other CPUs, too.
707 */
708 lapic_tval = (lapic_per_second * 2) / hz;
709 lapic_tval = (lapic_tval / 2) + (lapic_tval & 0x1);
710
711 lapic_writereg(LAPIC_LVT_TIMER, LAPIC_LVT_TMM_PERIODIC
712 | LAPIC_LVT_MASKED | LAPIC_TIMER_VECTOR);
713 lapic_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
714 lapic_writereg(LAPIC_ICR_TIMER, lapic_tval);
715
716 /*
717 * Compute fixed-point ratios between cycles and
718 * microseconds to avoid having to do any division
719 * in lapic_delay.
720 */
721
722 tmp = (1000000 * (uint64_t)1 << 32) / lapic_per_second;
723 lapic_frac_usec_per_cycle = tmp;
724
725 tmp = (lapic_per_second * (uint64_t)1 << 32) / 1000000;
726
727 lapic_frac_cycle_per_usec = tmp;
728
729 /*
730 * Compute delay in cycles for likely short delays in usec.
731 */
732 for (i = 0; i < 26; i++)
733 lapic_delaytab[i] = (lapic_frac_cycle_per_usec * i) >>
734 32;
735
736 /*
737 * Apply workaround for broken periodic timer under KVM
738 */
739 if (vm_guest == VM_GUEST_KVM) {
740 lapic_broken_periodic = true;
741 lapic_timecounter.tc_quality = -100;
742 aprint_debug_dev(ci->ci_dev,
743 "applying KVM timer workaround\n");
744 }
745
746 /*
747 * Now that the timer's calibrated, use the apic timer routines
748 * for all our timing needs..
749 */
750 if (!secondpass) {
751 delay_func = lapic_delay;
752 x86_initclock_func = lapic_initclock;
753 initrtclock(0);
754 }
755 }
756 }
757
758 /*
759 * delay for N usec.
760 */
761
762 static void
lapic_delay(unsigned int usec)763 lapic_delay(unsigned int usec)
764 {
765 int32_t xtick, otick;
766 int64_t deltat;
767
768 /* XXX Bad to disable preemption, but it's tied to the cpu. */
769 kpreempt_disable();
770 otick = lapic_gettick();
771
772 if (usec <= 0) {
773 kpreempt_enable();
774 return;
775 }
776
777 if (usec <= 25)
778 deltat = lapic_delaytab[usec];
779 else
780 deltat = (lapic_frac_cycle_per_usec * usec) >> 32;
781
782 while (deltat > 0) {
783 xtick = lapic_gettick();
784 if (lapic_broken_periodic && xtick == 0 && otick == 0) {
785 lapic_reset();
786 xtick = lapic_gettick();
787 if (xtick == 0)
788 panic("lapic timer stopped ticking");
789 }
790 if (xtick > otick)
791 deltat -= lapic_tval - (xtick - otick);
792 else
793 deltat -= otick - xtick;
794 otick = xtick;
795
796 x86_pause();
797 }
798 kpreempt_enable();
799 }
800
801 /*
802 * XXX the following belong mostly or partly elsewhere..
803 */
804
805 static void
i82489_icr_wait(void)806 i82489_icr_wait(void)
807 {
808 #ifdef DIAGNOSTIC
809 unsigned j = 100000;
810 #endif /* DIAGNOSTIC */
811
812 while ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) {
813 x86_pause();
814 #ifdef DIAGNOSTIC
815 j--;
816 if (j == 0)
817 panic("i82489_icr_wait: busy");
818 #endif /* DIAGNOSTIC */
819 }
820 }
821
822 static int
i82489_ipi_init(int target)823 i82489_ipi_init(int target)
824 {
825 uint32_t esr;
826
827 i82489_writereg(LAPIC_ESR, 0);
828 (void)i82489_readreg(LAPIC_ESR);
829
830 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
831
832 i82489_writereg(LAPIC_ICRLO, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT);
833 i82489_icr_wait();
834 delay_func(10000);
835 i82489_writereg(LAPIC_ICRLO,
836 LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT);
837 i82489_icr_wait();
838
839 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0)
840 return EBUSY;
841
842 esr = i82489_readreg(LAPIC_ESR);
843 if (esr != 0)
844 aprint_debug("%s: ESR %08x\n", __func__, esr);
845
846 return 0;
847 }
848
849 static int
i82489_ipi_startup(int target,int vec)850 i82489_ipi_startup(int target, int vec)
851 {
852 uint32_t esr;
853
854 i82489_writereg(LAPIC_ESR, 0);
855 (void)i82489_readreg(LAPIC_ESR);
856
857 i82489_icr_wait();
858 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
859 i82489_writereg(LAPIC_ICRLO, vec | LAPIC_DLMODE_STARTUP |
860 LAPIC_LEVEL_ASSERT);
861 i82489_icr_wait();
862
863 if ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0)
864 return EBUSY;
865
866 esr = i82489_readreg(LAPIC_ESR);
867 if (esr != 0)
868 aprint_debug("%s: ESR %08x\n", __func__, esr);
869
870 return 0;
871 }
872
873 static int
i82489_ipi(int vec,int target,int dl)874 i82489_ipi(int vec, int target, int dl)
875 {
876 int result, s;
877
878 s = splhigh();
879
880 i82489_icr_wait();
881
882 if ((target & LAPIC_DEST_MASK) == 0)
883 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
884
885 i82489_writereg(LAPIC_ICRLO,
886 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT);
887
888 #ifdef DIAGNOSTIC
889 i82489_icr_wait();
890 result = (i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) ? EBUSY : 0;
891 #else
892 /* Don't wait - if it doesn't go, we're in big trouble anyway. */
893 result = 0;
894 #endif
895 splx(s);
896
897 return result;
898 }
899
900 static int
x2apic_ipi_init(int target)901 x2apic_ipi_init(int target)
902 {
903
904 x2apic_write_icr(target, LAPIC_DLMODE_INIT | LAPIC_LEVEL_ASSERT);
905
906 delay_func(10000);
907
908 x2apic_write_icr(0,
909 LAPIC_DLMODE_INIT | LAPIC_TRIGMODE_LEVEL | LAPIC_LEVEL_DEASSERT);
910
911 return 0;
912 }
913
914 static int
x2apic_ipi_startup(int target,int vec)915 x2apic_ipi_startup(int target, int vec)
916 {
917
918 x2apic_write_icr(target,
919 vec | LAPIC_DLMODE_STARTUP | LAPIC_LEVEL_ASSERT);
920
921 return 0;
922 }
923
924 static int
x2apic_ipi(int vec,int target,int dl)925 x2apic_ipi(int vec, int target, int dl)
926 {
927 uint32_t dest_id = 0;
928
929 if ((target & LAPIC_DEST_MASK) == 0)
930 dest_id = target;
931
932 x2apic_write_icr(dest_id,
933 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LEVEL_ASSERT);
934
935 return 0;
936 }
937
938 int
x86_ipi_init(int target)939 x86_ipi_init(int target)
940 {
941 if (x2apic_mode)
942 return x2apic_ipi_init(target);
943 return i82489_ipi_init(target);
944 }
945
946 int
x86_ipi_startup(int target,int vec)947 x86_ipi_startup(int target, int vec)
948 {
949 if (x2apic_mode)
950 return x2apic_ipi_startup(target, vec);
951 return i82489_ipi_startup(target, vec);
952 }
953
954 /*
955 * Using 'pin numbers' as:
956 * 0 - timer
957 * 1 - thermal
958 * 2 - PCINT
959 * 3 - LVINT0
960 * 4 - LVINT1
961 * 5 - LVERR
962 */
963
964 static void
lapic_hwmask(struct pic * pic,int pin)965 lapic_hwmask(struct pic *pic, int pin)
966 {
967 int reg;
968 uint32_t val;
969
970 reg = LAPIC_LVT_TIMER + (pin << 4);
971 val = lapic_readreg(reg);
972 val |= LAPIC_LVT_MASKED;
973 lapic_writereg(reg, val);
974 }
975
976 static void
lapic_hwunmask(struct pic * pic,int pin)977 lapic_hwunmask(struct pic *pic, int pin)
978 {
979 int reg;
980 uint32_t val;
981
982 reg = LAPIC_LVT_TIMER + (pin << 4);
983 val = lapic_readreg(reg);
984 val &= ~LAPIC_LVT_MASKED;
985 lapic_writereg(reg, val);
986 }
987
988 static void
lapic_setup(struct pic * pic,struct cpu_info * ci,int pin,int idtvec,int type)989 lapic_setup(struct pic *pic, struct cpu_info *ci,
990 int pin, int idtvec, int type)
991 {
992 }
993
994 void
lapic_dump(void)995 lapic_dump(void)
996 {
997 struct cpu_info *ci = curcpu();
998
999 #define APIC_LVT_PRINT(ci, where, idx, lvtreg) \
1000 apic_format_redir(device_xname(ci->ci_dev), where, (idx), \
1001 APIC_VECTYPE_LAPIC_LVT, 0, lapic_readreg(lvtreg))
1002
1003 APIC_LVT_PRINT(ci, "cmci", 0, LAPIC_LVT_CMCI);
1004 APIC_LVT_PRINT(ci, "timer", 0, LAPIC_LVT_TIMER);
1005 APIC_LVT_PRINT(ci, "thermal", 0, LAPIC_LVT_THERM);
1006 APIC_LVT_PRINT(ci, "pcint", 0, LAPIC_LVT_PCINT);
1007 APIC_LVT_PRINT(ci, "lint", 0, LAPIC_LVT_LINT0);
1008 APIC_LVT_PRINT(ci, "lint", 1, LAPIC_LVT_LINT1);
1009 APIC_LVT_PRINT(ci, "err", 0, LAPIC_LVT_ERR);
1010
1011 #undef APIC_LVT_PRINT
1012 }
1013 #else /* XENPV */
1014 void
lapic_boot_init(paddr_t lapic_base)1015 lapic_boot_init(paddr_t lapic_base)
1016 {
1017 }
1018 #endif /* XENPV */
1019