1 /*
2 * Copyright (c) 1996, by Steve Passe
3 * All rights reserved.
4 *
5 * Redistribution and use in source and binary forms, with or without
6 * modification, are permitted provided that the following conditions
7 * are met:
8 * 1. Redistributions of source code must retain the above copyright
9 * notice, this list of conditions and the following disclaimer.
10 * 2. The name of the developer may NOT be used to endorse or promote products
11 * derived from this software without specific prior written permission.
12 *
13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
23 * SUCH DAMAGE.
24 *
25 * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.37.2.7 2003/01/25 02:31:47 peter Exp $
26 */
27
28 #include <sys/param.h>
29 #include <sys/systm.h>
30 #include <sys/kernel.h>
31 #include <sys/bus.h>
32 #include <sys/machintr.h>
33 #include <sys/malloc.h>
34 #include <sys/thread2.h>
35
36 #include <machine/pmap.h>
37 #include <machine_base/isa/isa_intr.h>
38 #include <machine_base/icu/icu_var.h>
39 #include <machine_base/apic/lapic.h>
40 #include <machine_base/apic/ioapic.h>
41 #include <machine_base/apic/ioapic_abi.h>
42 #include <machine_base/apic/apicvar.h>
43
44 #define IOAPIC_COUNT_MAX 16
45 #define IOAPIC_ID_MASK (IOAPIC_COUNT_MAX - 1)
46
47 struct ioapic_info {
48 int io_idx;
49 int io_apic_id;
50 void *io_addr;
51 int io_npin;
52 int io_gsi_base;
53
54 TAILQ_ENTRY(ioapic_info) io_link;
55 };
56 TAILQ_HEAD(ioapic_info_list, ioapic_info);
57
58 struct ioapic_intsrc {
59 int int_gsi;
60 enum intr_trigger int_trig;
61 enum intr_polarity int_pola;
62 };
63
64 struct ioapic_conf {
65 struct ioapic_info_list ioc_list;
66 struct ioapic_intsrc ioc_intsrc[ISA_IRQ_CNT];
67 };
68
69 static int ioapic_config(void);
70 static void ioapic_setup(const struct ioapic_info *);
71 static int ioapic_alloc_apic_id(int);
72 static void ioapic_set_apic_id(const struct ioapic_info *);
73 static void ioapic_gsi_setup(int);
74 static const struct ioapic_info *
75 ioapic_gsi_search(int);
76 static void ioapic_pin_prog(void *, int, int,
77 enum intr_trigger, enum intr_polarity, uint32_t, int);
78
79 static struct ioapic_conf ioapic_conf;
80
81 static TAILQ_HEAD(, ioapic_enumerator) ioapic_enumerators =
82 TAILQ_HEAD_INITIALIZER(ioapic_enumerators);
83
84 int ioapic_enable = -1; /* I/O APIC auto-enable mode */
85
86 static int
ioapic_config(void)87 ioapic_config(void)
88 {
89 struct ioapic_enumerator *e;
90 struct ioapic_info *info;
91 int start_apic_id = 0;
92 int error, i, probe;
93 register_t ef = 0;
94
95 TAILQ_INIT(&ioapic_conf.ioc_list);
96 for (i = 0; i < ISA_IRQ_CNT; ++i)
97 ioapic_conf.ioc_intsrc[i].int_gsi = -1;
98
99 probe = 1;
100 TUNABLE_INT_FETCH("hw.ioapic_probe", &probe);
101 if (!probe) {
102 kprintf("IOAPIC: warning I/O APIC will not be probed\n");
103 return ENXIO;
104 }
105
106 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) {
107 error = e->ioapic_probe(e);
108 if (!error)
109 break;
110 }
111 if (e == NULL) {
112 kprintf("IOAPIC: can't find I/O APIC\n");
113 return ENXIO;
114 }
115
116 crit_enter();
117
118 ef = read_rflags();
119 cpu_disable_intr();
120
121 /*
122 * Switch to I/O APIC MachIntrABI and reconfigure
123 * the default IDT entries.
124 */
125 MachIntrABI = MachIntrABI_IOAPIC;
126 MachIntrABI.setdefault();
127
128 e->ioapic_enumerate(e);
129
130 /*
131 * Setup index
132 */
133 i = 0;
134 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link)
135 info->io_idx = i++;
136
137 if (i > IOAPIC_COUNT_MAX)
138 panic("ioapic_config: more than 16 I/O APIC");
139
140 /*
141 * Setup APIC ID
142 */
143 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
144 int apic_id;
145
146 apic_id = ioapic_alloc_apic_id(start_apic_id);
147 if (apic_id == NAPICID) {
148 kprintf("IOAPIC: can't alloc APIC ID for "
149 "%dth I/O APIC\n", info->io_idx);
150 break;
151 }
152 info->io_apic_id = apic_id;
153
154 start_apic_id = apic_id + 1;
155 }
156 if (info != NULL) {
157 /*
158 * xAPIC allows I/O APIC's APIC ID to be same
159 * as the LAPIC's APIC ID
160 */
161 kprintf("IOAPIC: use xAPIC model to alloc APIC ID "
162 "for I/O APIC\n");
163
164 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link)
165 info->io_apic_id = info->io_idx;
166 }
167
168 /*
169 * Warning about any GSI holes
170 */
171 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
172 const struct ioapic_info *prev_info;
173
174 prev_info = TAILQ_PREV(info, ioapic_info_list, io_link);
175 if (prev_info != NULL) {
176 if (info->io_gsi_base !=
177 prev_info->io_gsi_base + prev_info->io_npin) {
178 kprintf("IOAPIC: warning gsi hole "
179 "[%d, %d]\n",
180 prev_info->io_gsi_base +
181 prev_info->io_npin,
182 info->io_gsi_base - 1);
183 }
184 }
185 }
186
187 if (bootverbose) {
188 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
189 kprintf("IOAPIC: idx %d, apic id %d, "
190 "gsi base %d, npin %d\n",
191 info->io_idx,
192 info->io_apic_id,
193 info->io_gsi_base,
194 info->io_npin);
195 }
196 }
197
198 /*
199 * Setup all I/O APIC
200 */
201 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link)
202 ioapic_setup(info);
203 ioapic_fixup_legacy_irqmaps();
204
205 write_rflags(ef);
206
207 MachIntrABI.cleanup();
208
209 crit_exit();
210
211 return 0;
212 }
213
214 void
ioapic_enumerator_register(struct ioapic_enumerator * ne)215 ioapic_enumerator_register(struct ioapic_enumerator *ne)
216 {
217 struct ioapic_enumerator *e;
218
219 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) {
220 if (e->ioapic_prio < ne->ioapic_prio) {
221 TAILQ_INSERT_BEFORE(e, ne, ioapic_link);
222 return;
223 }
224 }
225 TAILQ_INSERT_TAIL(&ioapic_enumerators, ne, ioapic_link);
226 }
227
228 void
ioapic_add(void * addr,int gsi_base,int npin)229 ioapic_add(void *addr, int gsi_base, int npin)
230 {
231 struct ioapic_info *info, *ninfo;
232 int gsi_end;
233
234 gsi_end = gsi_base + npin - 1;
235 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
236 if ((gsi_base >= info->io_gsi_base &&
237 gsi_base < info->io_gsi_base + info->io_npin) ||
238 (gsi_end >= info->io_gsi_base &&
239 gsi_end < info->io_gsi_base + info->io_npin)) {
240 panic("ioapic_add: overlapped gsi, base %d npin %d, "
241 "hit base %d, npin %d\n", gsi_base, npin,
242 info->io_gsi_base, info->io_npin);
243 }
244 if (info->io_addr == addr)
245 panic("ioapic_add: duplicated addr %p", addr);
246 }
247
248 ninfo = kmalloc(sizeof(*ninfo), M_DEVBUF, M_WAITOK | M_ZERO);
249 ninfo->io_addr = addr;
250 ninfo->io_npin = npin;
251 ninfo->io_gsi_base = gsi_base;
252 ninfo->io_apic_id = -1;
253
254 /*
255 * Create IOAPIC list in ascending order of GSI base
256 */
257 TAILQ_FOREACH_REVERSE(info, &ioapic_conf.ioc_list,
258 ioapic_info_list, io_link) {
259 if (ninfo->io_gsi_base > info->io_gsi_base) {
260 TAILQ_INSERT_AFTER(&ioapic_conf.ioc_list,
261 info, ninfo, io_link);
262 break;
263 }
264 }
265 if (info == NULL)
266 TAILQ_INSERT_HEAD(&ioapic_conf.ioc_list, ninfo, io_link);
267 }
268
269 void
ioapic_intsrc(int irq,int gsi,enum intr_trigger trig,enum intr_polarity pola)270 ioapic_intsrc(int irq, int gsi, enum intr_trigger trig, enum intr_polarity pola)
271 {
272 struct ioapic_intsrc *int_src;
273
274 KKASSERT(irq < ISA_IRQ_CNT);
275 int_src = &ioapic_conf.ioc_intsrc[irq];
276
277 if (gsi == 0) {
278 /* Don't allow mixed mode */
279 kprintf("IOAPIC: warning intsrc irq %d -> gsi 0\n", irq);
280 return;
281 }
282
283 if (int_src->int_gsi != -1) {
284 if (int_src->int_gsi != gsi) {
285 kprintf("IOAPIC: warning intsrc irq %d, gsi "
286 "%d -> %d\n", irq, int_src->int_gsi, gsi);
287 }
288 if (int_src->int_trig != trig) {
289 kprintf("IOAPIC: warning intsrc irq %d, trig "
290 "%s -> %s\n", irq,
291 intr_str_trigger(int_src->int_trig),
292 intr_str_trigger(trig));
293 }
294 if (int_src->int_pola != pola) {
295 kprintf("IOAPIC: warning intsrc irq %d, pola "
296 "%s -> %s\n", irq,
297 intr_str_polarity(int_src->int_pola),
298 intr_str_polarity(pola));
299 }
300 }
301 int_src->int_gsi = gsi;
302 int_src->int_trig = trig;
303 int_src->int_pola = pola;
304 }
305
306 static void
ioapic_set_apic_id(const struct ioapic_info * info)307 ioapic_set_apic_id(const struct ioapic_info *info)
308 {
309 uint32_t id;
310 int apic_id;
311
312 id = ioapic_read(info->io_addr, IOAPIC_ID);
313
314 id &= ~APIC_ID_MASK;
315 id |= (info->io_apic_id << 24);
316
317 ioapic_write(info->io_addr, IOAPIC_ID, id);
318
319 /*
320 * Re-read && test
321 */
322 id = ioapic_read(info->io_addr, IOAPIC_ID);
323 apic_id = (id & APIC_ID_MASK) >> 24;
324
325 /*
326 * I/O APIC ID is a 4bits field
327 */
328 if ((apic_id & IOAPIC_ID_MASK) !=
329 (info->io_apic_id & IOAPIC_ID_MASK)) {
330 panic("ioapic_set_apic_id: can't set apic id to %d, "
331 "currently set to %d\n", info->io_apic_id, apic_id);
332 }
333 }
334
335 static void
ioapic_gsi_setup(int gsi)336 ioapic_gsi_setup(int gsi)
337 {
338 enum intr_trigger trig;
339 enum intr_polarity pola;
340 int irq;
341
342 if (gsi == 0) {
343 /* ExtINT */
344 imen_lock();
345 ioapic_extpin_setup(ioapic_gsi_ioaddr(gsi),
346 ioapic_gsi_pin(gsi), 0);
347 imen_unlock();
348 return;
349 }
350
351 trig = 0; /* silence older gcc's */
352 pola = 0; /* silence older gcc's */
353
354 for (irq = 0; irq < ISA_IRQ_CNT; ++irq) {
355 const struct ioapic_intsrc *int_src =
356 &ioapic_conf.ioc_intsrc[irq];
357
358 if (gsi == int_src->int_gsi) {
359 trig = int_src->int_trig;
360 pola = int_src->int_pola;
361 break;
362 }
363 }
364
365 if (irq == ISA_IRQ_CNT) {
366 /*
367 * No explicit IRQ to GSI mapping;
368 * use the default 1:1 mapping
369 */
370 irq = gsi;
371 if (irq < ISA_IRQ_CNT) {
372 if (ioapic_conf.ioc_intsrc[irq].int_gsi >= 0) {
373 /*
374 * This IRQ is mapped to different GSI,
375 * don't do the default configuration.
376 * The configuration of the target GSI
377 * will finally setup this IRQ.
378 *
379 * This GSI is not used, disable it.
380 */
381 imen_lock();
382 ioapic_pin_setup(ioapic_gsi_ioaddr(gsi),
383 ioapic_gsi_pin(gsi), 0,
384 INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH, 0);
385 imen_unlock();
386 return;
387 }
388 trig = INTR_TRIGGER_EDGE;
389 pola = INTR_POLARITY_HIGH;
390 } else {
391 trig = INTR_TRIGGER_LEVEL;
392 pola = INTR_POLARITY_LOW;
393 }
394 }
395
396 ioapic_set_legacy_irqmap(irq, gsi, trig, pola);
397 }
398
399 void *
ioapic_gsi_ioaddr(int gsi)400 ioapic_gsi_ioaddr(int gsi)
401 {
402 const struct ioapic_info *info;
403
404 info = ioapic_gsi_search(gsi);
405 return info->io_addr;
406 }
407
408 int
ioapic_gsi_pin(int gsi)409 ioapic_gsi_pin(int gsi)
410 {
411 const struct ioapic_info *info;
412
413 info = ioapic_gsi_search(gsi);
414 return gsi - info->io_gsi_base;
415 }
416
417 static const struct ioapic_info *
ioapic_gsi_search(int gsi)418 ioapic_gsi_search(int gsi)
419 {
420 const struct ioapic_info *info;
421
422 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
423 if (gsi >= info->io_gsi_base &&
424 gsi < info->io_gsi_base + info->io_npin)
425 return info;
426 }
427 panic("ioapic_gsi_search: no I/O APIC");
428 }
429
430 int
ioapic_gsi(int idx,int pin)431 ioapic_gsi(int idx, int pin)
432 {
433 const struct ioapic_info *info;
434
435 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
436 if (info->io_idx == idx)
437 break;
438 }
439 if (info == NULL)
440 return -1;
441 if (pin >= info->io_npin)
442 return -1;
443 return info->io_gsi_base + pin;
444 }
445
446 void
ioapic_extpin_setup(void * addr,int pin,int vec)447 ioapic_extpin_setup(void *addr, int pin, int vec)
448 {
449 ioapic_pin_prog(addr, pin, vec,
450 INTR_TRIGGER_CONFORM, INTR_POLARITY_CONFORM, IOART_DELEXINT, 0);
451 }
452
453 int
ioapic_extpin_gsi(void)454 ioapic_extpin_gsi(void)
455 {
456 return 0;
457 }
458
459 void
ioapic_pin_setup(void * addr,int pin,int vec,enum intr_trigger trig,enum intr_polarity pola,int cpuid)460 ioapic_pin_setup(void *addr, int pin, int vec,
461 enum intr_trigger trig, enum intr_polarity pola, int cpuid)
462 {
463 /*
464 * Always clear an I/O APIC pin before [re]programming it. This is
465 * particularly important if the pin is set up for a level interrupt
466 * as the IOART_REM_IRR bit might be set. When we reprogram the
467 * vector any EOI from pending ints on this pin could be lost and
468 * IRR might never get reset.
469 *
470 * To fix this problem, clear the vector and make sure it is
471 * programmed as an edge interrupt. This should theoretically
472 * clear IRR so we can later, safely program it as a level
473 * interrupt.
474 */
475 ioapic_pin_prog(addr, pin, vec, INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH,
476 IOART_DELFIXED, cpuid);
477 ioapic_pin_prog(addr, pin, vec, trig, pola, IOART_DELFIXED, cpuid);
478 }
479
480 static void
ioapic_pin_prog(void * addr,int pin,int vec,enum intr_trigger trig,enum intr_polarity pola,uint32_t del_mode,int cpuid)481 ioapic_pin_prog(void *addr, int pin, int vec,
482 enum intr_trigger trig, enum intr_polarity pola,
483 uint32_t del_mode, int cpuid)
484 {
485 uint32_t flags, target;
486 int select;
487
488 KKASSERT(del_mode == IOART_DELEXINT || del_mode == IOART_DELFIXED);
489
490 select = IOAPIC_REDTBL0 + (2 * pin);
491
492 flags = ioapic_read(addr, select) & IOART_RESV;
493 flags |= IOART_INTMSET | IOART_DESTPHY;
494 #ifdef foo
495 flags |= del_mode;
496 #else
497 /*
498 * We only support limited I/O APIC mixed mode,
499 * so even for ExtINT, we still use "fixed"
500 * delivery mode.
501 */
502 flags |= IOART_DELFIXED;
503 #endif
504
505 if (del_mode == IOART_DELEXINT) {
506 KKASSERT(trig == INTR_TRIGGER_CONFORM &&
507 pola == INTR_POLARITY_CONFORM);
508 flags |= IOART_TRGREDG | IOART_INTAHI;
509 } else {
510 switch (trig) {
511 case INTR_TRIGGER_EDGE:
512 flags |= IOART_TRGREDG;
513 break;
514
515 case INTR_TRIGGER_LEVEL:
516 flags |= IOART_TRGRLVL;
517 break;
518
519 case INTR_TRIGGER_CONFORM:
520 panic("ioapic_pin_prog: trig conform is not "
521 "supported\n");
522 }
523 switch (pola) {
524 case INTR_POLARITY_HIGH:
525 flags |= IOART_INTAHI;
526 break;
527
528 case INTR_POLARITY_LOW:
529 flags |= IOART_INTALO;
530 break;
531
532 case INTR_POLARITY_CONFORM:
533 panic("ioapic_pin_prog: pola conform is not "
534 "supported\n");
535 }
536 }
537
538 target = ioapic_read(addr, select + 1) & IOART_HI_DEST_RESV;
539 target |= (CPUID_TO_APICID(cpuid) << IOART_HI_DEST_SHIFT) &
540 IOART_HI_DEST_MASK;
541
542 ioapic_write(addr, select, flags | vec);
543 ioapic_write(addr, select + 1, target);
544 }
545
546 static void
ioapic_setup(const struct ioapic_info * info)547 ioapic_setup(const struct ioapic_info *info)
548 {
549 int i;
550
551 ioapic_set_apic_id(info);
552
553 for (i = 0; i < info->io_npin; ++i)
554 ioapic_gsi_setup(info->io_gsi_base + i);
555 }
556
557 static int
ioapic_alloc_apic_id(int start)558 ioapic_alloc_apic_id(int start)
559 {
560 for (;;) {
561 const struct ioapic_info *info;
562 int apic_id, apic_id16;
563
564 apic_id = lapic_unused_apic_id(start);
565 if (apic_id == NAPICID) {
566 kprintf("IOAPIC: can't find unused APIC ID\n");
567 return apic_id;
568 }
569 apic_id16 = apic_id & IOAPIC_ID_MASK;
570
571 /*
572 * Check against other I/O APIC's APIC ID's lower 4bits.
573 *
574 * The new APIC ID will have to be different from others
575 * in the lower 4bits, no matter whether xAPIC is used
576 * or not.
577 */
578 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) {
579 if (info->io_apic_id == -1) {
580 info = NULL;
581 break;
582 }
583 if ((info->io_apic_id & IOAPIC_ID_MASK) == apic_id16)
584 break;
585 }
586 if (info == NULL)
587 return apic_id;
588
589 kprintf("IOAPIC: APIC ID %d has same lower 4bits as "
590 "%dth I/O APIC, keep searching...\n",
591 apic_id, info->io_idx);
592
593 start = apic_id + 1;
594 }
595 panic("ioapic_unused_apic_id: never reached");
596 }
597
598 /*
599 * Map a physical memory address representing I/O into KVA. The I/O
600 * block is assumed not to cross a page boundary.
601 */
602 void *
ioapic_map(vm_paddr_t pa)603 ioapic_map(vm_paddr_t pa)
604 {
605 KKASSERT(pa < 0x100000000LL);
606
607 return pmap_mapdev_uncacheable(pa, PAGE_SIZE);
608 }
609
610 static void
ioapic_sysinit(void * dummy __unused)611 ioapic_sysinit(void *dummy __unused)
612 {
613 int error;
614
615 if (!ioapic_enable)
616 return;
617
618 KASSERT(lapic_enable, ("I/O APIC is enabled, but LAPIC is disabled"));
619 error = ioapic_config();
620 if (error) {
621 kprintf("IOAPIC disabled - error during ioapic_config()\n");
622 ioapic_enable = 0;
623 icu_reinit_noioapic();
624 lapic_fixup_noioapic();
625 }
626 }
627 SYSINIT(ioapic, SI_BOOT2_IOAPIC, SI_ORDER_FIRST, ioapic_sysinit, NULL);
628