1 /* $OpenBSD: lapic.c,v 1.58 2023/09/17 14:50:51 cheloha Exp $ */
2 /* $NetBSD: lapic.c,v 1.1.2.8 2000/02/23 06:10:50 sommerfeld Exp $ */
3
4 /*-
5 * Copyright (c) 2000 The NetBSD Foundation, Inc.
6 * All rights reserved.
7 *
8 * This code is derived from software contributed to The NetBSD Foundation
9 * by RedBack Networks Inc.
10 *
11 * Author: Bill Sommerfeld
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
23 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
24 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
25 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
26 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
27 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
28 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
29 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
30 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
31 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
32 * POSSIBILITY OF SUCH DAMAGE.
33 */
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/clockintr.h>
38 #include <sys/device.h>
39 #include <sys/stdint.h>
40
41 #include <uvm/uvm_extern.h>
42
43 #include <machine/cpufunc.h>
44 #include <machine/cpuvar.h>
45 #include <machine/pmap.h>
46 #include <machine/mpbiosvar.h>
47 #include <machine/specialreg.h>
48 #include <machine/segments.h>
49
50 #include <machine/apicvar.h>
51 #include <machine/i82489reg.h>
52 #include <machine/i82489var.h>
53 #include <machine/pctr.h>
54
55 #include <dev/ic/i8253reg.h>
56
57 /* #define LAPIC_DEBUG */
58
59 #ifdef LAPIC_DEBUG
60 #define DPRINTF(x...) do { printf(x); } while(0)
61 #else
62 #define DPRINTF(x...)
63 #endif /* LAPIC_DEBUG */
64
65 struct evcount clk_count;
66 #ifdef MULTIPROCESSOR
67 struct evcount ipi_count;
68 #endif
69
70 static u_int32_t lapic_gettick(void);
71 void lapic_clockintr(void *);
72 void lapic_initclocks(void);
73 void lapic_map(paddr_t);
74
75 void
lapic_map(paddr_t lapic_base)76 lapic_map(paddr_t lapic_base)
77 {
78 vaddr_t va = (vaddr_t)&local_apic;
79 u_long s;
80 int tpr;
81
82 s = intr_disable();
83 tpr = lapic_tpr;
84
85 /*
86 * Map local apic.
87 *
88 * Whap the PTE "by hand" rather than calling pmap_kenter_pa because
89 * the latter will attempt to invoke TLB shootdown code just as we
90 * might have changed the value of cpu_number()..
91 */
92
93 pmap_pte_set(va, lapic_base, PG_RW | PG_V | PG_N);
94 invlpg(va);
95
96 pmap_enter_special(va, lapic_base, PROT_READ | PROT_WRITE, PG_N);
97 DPRINTF("%s: entered lapic page va 0x%08lx pa 0x%08lx\n", __func__,
98 va, lapic_base);
99
100 #ifdef MULTIPROCESSOR
101 cpu_init_first();
102 #endif
103
104 lapic_tpr = tpr;
105 intr_restore(s);
106 }
107
108 /*
109 * enable local apic
110 */
111 void
lapic_enable(void)112 lapic_enable(void)
113 {
114 i82489_writereg(LAPIC_SVR, LAPIC_SVR_ENABLE | LAPIC_SPURIOUS_VECTOR);
115 }
116
117 void
lapic_disable(void)118 lapic_disable(void)
119 {
120 i82489_writereg(LAPIC_SVR, 0);
121 }
122
123 void
lapic_set_softvectors(void)124 lapic_set_softvectors(void)
125 {
126 idt_vec_set(LAPIC_SOFTCLOCK_VECTOR, Xintrsoftclock);
127 idt_vec_set(LAPIC_SOFTNET_VECTOR, Xintrsoftnet);
128 idt_vec_set(LAPIC_SOFTTTY_VECTOR, Xintrsofttty);
129 }
130
131 void
lapic_set_lvt(void)132 lapic_set_lvt(void)
133 {
134 struct cpu_info *ci = curcpu();
135 int i;
136 struct mp_intr_map *mpi;
137
138 #ifdef MULTIPROCESSOR
139 if (mp_verbose) {
140 apic_format_redir(ci->ci_dev->dv_xname, "prelint", 0, 0,
141 i82489_readreg(LAPIC_LVINT0));
142 apic_format_redir(ci->ci_dev->dv_xname, "prelint", 1, 0,
143 i82489_readreg(LAPIC_LVINT1));
144 }
145 #endif
146
147 if (strcmp(cpu_vendor, "AuthenticAMD") == 0) {
148 /*
149 * Detect the presence of C1E capability mostly on latest
150 * dual-cores (or future) k8 family. This mis-feature renders
151 * the local APIC timer dead, so we disable it by reading
152 * the Interrupt Pending Message register and clearing both
153 * C1eOnCmpHalt (bit 28) and SmiOnCmpHalt (bit 27).
154 *
155 * Reference:
156 * "BIOS and Kernel Developer's Guide for AMD NPT
157 * Family 0Fh Processors"
158 * #32559 revision 3.00
159 */
160 if (ci->ci_family == 0xf || ci->ci_family == 0x10) {
161 uint64_t msr;
162
163 msr = rdmsr(MSR_INT_PEN_MSG);
164 if (msr & (IPM_C1E_CMP_HLT|IPM_SMI_CMP_HLT)) {
165 msr &= ~(IPM_C1E_CMP_HLT|IPM_SMI_CMP_HLT);
166 wrmsr(MSR_INT_PEN_MSG, msr);
167 }
168 }
169 }
170
171 for (i = 0; i < mp_nintrs; i++) {
172 mpi = &mp_intrs[i];
173 if (mpi->ioapic == NULL && (mpi->cpu_id == MPS_ALL_APICS
174 || mpi->cpu_id == ci->ci_apicid)) {
175 #ifdef DIAGNOSTIC
176 if (mpi->ioapic_pin > 1)
177 panic("lapic_set_lvt: bad pin value %d",
178 mpi->ioapic_pin);
179 #endif
180 if (mpi->ioapic_pin == 0)
181 i82489_writereg(LAPIC_LVINT0, mpi->redir);
182 else
183 i82489_writereg(LAPIC_LVINT1, mpi->redir);
184 }
185 }
186
187 #ifdef MULTIPROCESSOR
188 if (mp_verbose) {
189 apic_format_redir(ci->ci_dev->dv_xname, "timer", 0, 0,
190 i82489_readreg(LAPIC_LVTT));
191 apic_format_redir(ci->ci_dev->dv_xname, "pcint", 0, 0,
192 i82489_readreg(LAPIC_PCINT));
193 apic_format_redir(ci->ci_dev->dv_xname, "lint", 0, 0,
194 i82489_readreg(LAPIC_LVINT0));
195 apic_format_redir(ci->ci_dev->dv_xname, "lint", 1, 0,
196 i82489_readreg(LAPIC_LVINT1));
197 apic_format_redir(ci->ci_dev->dv_xname, "err", 0, 0,
198 i82489_readreg(LAPIC_LVERR));
199 }
200 #endif
201 }
202
203 /*
204 * Initialize fixed idt vectors for use by local apic.
205 */
206 void
lapic_boot_init(paddr_t lapic_base)207 lapic_boot_init(paddr_t lapic_base)
208 {
209 static int clk_irq = 0;
210 #ifdef MULTIPROCESSOR
211 static int ipi_irq = 0;
212 #endif
213
214 lapic_map(lapic_base);
215
216 #ifdef MULTIPROCESSOR
217 idt_vec_set(LAPIC_IPI_VECTOR, Xintripi);
218 idt_vec_set(LAPIC_IPI_INVLTLB, Xintripi_invltlb);
219 idt_vec_set(LAPIC_IPI_INVLPG, Xintripi_invlpg);
220 idt_vec_set(LAPIC_IPI_INVLRANGE, Xintripi_invlrange);
221 idt_vec_set(LAPIC_IPI_RELOADCR3, Xintripi_reloadcr3);
222 #endif
223 idt_vec_set(LAPIC_SPURIOUS_VECTOR, Xintrspurious);
224 idt_vec_set(LAPIC_TIMER_VECTOR, Xintrltimer);
225
226 evcount_attach(&clk_count, "clock", &clk_irq);
227 #ifdef MULTIPROCESSOR
228 evcount_attach(&ipi_count, "ipi", &ipi_irq);
229 #endif
230 }
231
232 static __inline u_int32_t
lapic_gettick(void)233 lapic_gettick(void)
234 {
235 return i82489_readreg(LAPIC_CCR_TIMER);
236 }
237
238 #include <sys/kernel.h> /* for hz */
239
240 /*
241 * this gets us up to a 4GHz busclock....
242 */
243 u_int32_t lapic_per_second = 0;
244 uint64_t lapic_timer_nsec_cycle_ratio;
245 uint64_t lapic_timer_nsec_max;
246
247 void lapic_timer_rearm(void *, uint64_t);
248 void lapic_timer_trigger(void *);
249
250 struct intrclock lapic_timer_intrclock = {
251 .ic_rearm = lapic_timer_rearm,
252 .ic_trigger = lapic_timer_trigger
253 };
254
255 void lapic_timer_oneshot(uint32_t, uint32_t);
256 void lapic_timer_periodic(uint32_t, uint32_t);
257
258 void
lapic_timer_rearm(void * unused,uint64_t nsecs)259 lapic_timer_rearm(void *unused, uint64_t nsecs)
260 {
261 uint32_t cycles;
262
263 if (nsecs > lapic_timer_nsec_max)
264 nsecs = lapic_timer_nsec_max;
265 cycles = (nsecs * lapic_timer_nsec_cycle_ratio) >> 32;
266 if (cycles == 0)
267 cycles = 1;
268 lapic_timer_oneshot(0, cycles);
269 }
270
271 void
lapic_timer_trigger(void * unused)272 lapic_timer_trigger(void *unused)
273 {
274 u_long s;
275
276 s = intr_disable();
277 lapic_timer_oneshot(0, 1);
278 intr_restore(s);
279 }
280
281 /*
282 * Start the local apic countdown timer.
283 *
284 * First set the mode, mask, and vector. Then set the
285 * divisor. Last, set the cycle count: this restarts
286 * the countdown.
287 */
288 static inline void
lapic_timer_start(uint32_t mode,uint32_t mask,uint32_t cycles)289 lapic_timer_start(uint32_t mode, uint32_t mask, uint32_t cycles)
290 {
291 i82489_writereg(LAPIC_LVTT, mode | mask | LAPIC_TIMER_VECTOR);
292 i82489_writereg(LAPIC_DCR_TIMER, LAPIC_DCRT_DIV1);
293 i82489_writereg(LAPIC_ICR_TIMER, cycles);
294 }
295
296 void
lapic_timer_oneshot(uint32_t mask,uint32_t cycles)297 lapic_timer_oneshot(uint32_t mask, uint32_t cycles)
298 {
299 lapic_timer_start(LAPIC_LVTT_TM_ONESHOT, mask, cycles);
300 }
301
302 void
lapic_timer_periodic(uint32_t mask,uint32_t cycles)303 lapic_timer_periodic(uint32_t mask, uint32_t cycles)
304 {
305 lapic_timer_start(LAPIC_LVTT_TM_PERIODIC, mask, cycles);
306 }
307
308 void
lapic_clockintr(void * frame)309 lapic_clockintr(void *frame)
310 {
311 clockintr_dispatch(frame);
312 clk_count.ec_count++;
313 }
314
315 void
lapic_startclock(void)316 lapic_startclock(void)
317 {
318 clockintr_cpu_init(&lapic_timer_intrclock);
319 clockintr_trigger();
320 }
321
322 void
lapic_initclocks(void)323 lapic_initclocks(void)
324 {
325 i8254_inittimecounter_simple();
326
327 stathz = hz;
328 profhz = stathz * 10;
329 statclock_is_randomized = 1;
330 }
331
332 extern int gettick(void); /* XXX put in header file */
333 extern u_long rtclock_tval; /* XXX put in header file */
334
335 static __inline void
wait_next_cycle(void)336 wait_next_cycle(void)
337 {
338 unsigned int tick, tlast;
339
340 tlast = (1 << 16); /* i8254 counter has 16 bits at most */
341 for (;;) {
342 tick = gettick();
343 if (tick > tlast)
344 return;
345 tlast = tick;
346 }
347 }
348
349 /*
350 * Calibrate the local apic count-down timer (which is running at
351 * bus-clock speed) vs. the i8254 counter/timer (which is running at
352 * a fixed rate).
353 *
354 * The Intel MP spec says: "An MP operating system may use the IRQ8
355 * real-time clock as a reference to determine the actual APIC timer clock
356 * speed."
357 *
358 * We're actually using the IRQ0 timer. Hmm.
359 */
360 void
lapic_calibrate_timer(struct cpu_info * ci)361 lapic_calibrate_timer(struct cpu_info *ci)
362 {
363 unsigned int startapic, endapic;
364 u_int64_t dtick, dapic, tmp;
365 u_long s;
366 int i;
367
368 if (mp_verbose)
369 printf("%s: calibrating local timer\n", ci->ci_dev->dv_xname);
370
371 /*
372 * Configure timer to one-shot, interrupt masked,
373 * large positive number.
374 */
375 lapic_timer_oneshot(LAPIC_LVTT_M, 0x80000000);
376
377 if (delay_func == i8254_delay) {
378 s = intr_disable();
379
380 /* wait for current cycle to finish */
381 wait_next_cycle();
382
383 startapic = lapic_gettick();
384
385 /* wait the next hz cycles */
386 for (i = 0; i < hz; i++)
387 wait_next_cycle();
388
389 endapic = lapic_gettick();
390
391 intr_restore(s);
392
393 dtick = hz * rtclock_tval;
394 dapic = startapic-endapic;
395
396 /*
397 * there are TIMER_FREQ ticks per second.
398 * in dtick ticks, there are dapic bus clocks.
399 */
400 tmp = (TIMER_FREQ * dapic) / dtick;
401
402 lapic_per_second = tmp;
403 } else {
404 s = intr_disable();
405 startapic = lapic_gettick();
406 delay(1 * 1000 * 1000);
407 endapic = lapic_gettick();
408 intr_restore(s);
409 lapic_per_second = startapic - endapic;
410 }
411
412 printf("%s: apic clock running at %dMHz\n",
413 ci->ci_dev->dv_xname, lapic_per_second / (1000 * 1000));
414
415 /* XXX What should we do here if the timer frequency is zero? */
416 if (lapic_per_second == 0)
417 return;
418
419 lapic_timer_nsec_cycle_ratio =
420 lapic_per_second * (1ULL << 32) / 1000000000;
421 lapic_timer_nsec_max = UINT64_MAX / lapic_timer_nsec_cycle_ratio;
422 initclock_func = lapic_initclocks;
423 startclock_func = lapic_startclock;
424 }
425
426 /*
427 * XXX the following belong mostly or partly elsewhere..
428 */
429
430 #ifdef MULTIPROCESSOR
431 static __inline void i82489_icr_wait(void);
432
433 static __inline void
i82489_icr_wait(void)434 i82489_icr_wait(void)
435 {
436 #ifdef DIAGNOSTIC
437 unsigned j = 100000;
438 #endif /* DIAGNOSTIC */
439
440 while ((i82489_readreg(LAPIC_ICRLO) & LAPIC_DLSTAT_BUSY) != 0) {
441 __asm volatile("pause": : :"memory");
442 #ifdef DIAGNOSTIC
443 j--;
444 if (j == 0)
445 panic("i82489_icr_wait: busy");
446 #endif /* DIAGNOSTIC */
447 }
448 }
449
450 void
i386_ipi_init(int target)451 i386_ipi_init(int target)
452 {
453 if ((target & LAPIC_DEST_MASK) == 0)
454 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
455
456 i82489_writereg(LAPIC_ICRLO, (target & LAPIC_DEST_MASK) |
457 LAPIC_DLMODE_INIT | LAPIC_LVL_ASSERT );
458
459 i82489_icr_wait();
460
461 i8254_delay(10000);
462
463 i82489_writereg(LAPIC_ICRLO, (target & LAPIC_DEST_MASK) |
464 LAPIC_DLMODE_INIT | LAPIC_LVL_TRIG | LAPIC_LVL_DEASSERT);
465
466 i82489_icr_wait();
467 }
468
469 void
i386_ipi(int vec,int target,int dl)470 i386_ipi(int vec, int target, int dl)
471 {
472 int s;
473
474 s = splhigh();
475
476 i82489_icr_wait();
477
478 if ((target & LAPIC_DEST_MASK) == 0)
479 i82489_writereg(LAPIC_ICRHI, target << LAPIC_ID_SHIFT);
480
481 i82489_writereg(LAPIC_ICRLO,
482 (target & LAPIC_DEST_MASK) | vec | dl | LAPIC_LVL_ASSERT);
483
484 i82489_icr_wait();
485
486 splx(s);
487 }
488 #endif /* MULTIPROCESSOR */
489