1 /* 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. The name of the developer may NOT be used to endorse or promote products 11 * derived from this software without specific prior written permission. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.37.2.7 2003/01/25 02:31:47 peter Exp $ 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/kernel.h> 31 #include <sys/bus.h> 32 #include <sys/machintr.h> 33 #include <sys/thread2.h> 34 35 #include <machine/pmap.h> 36 #include <machine_base/isa/isa_intr.h> 37 #include <machine_base/icu/icu_var.h> 38 #include <machine_base/apic/lapic.h> 39 #include <machine_base/apic/ioapic.h> 40 #include <machine_base/apic/ioapic_abi.h> 41 #include <machine_base/apic/apicvar.h> 42 43 #define IOAPIC_COUNT_MAX 16 44 #define IOAPIC_ID_MASK (IOAPIC_COUNT_MAX - 1) 45 46 struct ioapic_info { 47 int io_idx; 48 int io_apic_id; 49 void *io_addr; 50 int io_npin; 51 int io_gsi_base; 52 53 TAILQ_ENTRY(ioapic_info) io_link; 54 }; 55 TAILQ_HEAD(ioapic_info_list, ioapic_info); 56 57 struct ioapic_intsrc { 58 int int_gsi; 59 enum intr_trigger int_trig; 60 enum intr_polarity int_pola; 61 }; 62 63 struct ioapic_conf { 64 struct ioapic_info_list ioc_list; 65 struct ioapic_intsrc ioc_intsrc[ISA_IRQ_CNT]; 66 }; 67 68 static int ioapic_config(void); 69 static void ioapic_setup(const struct ioapic_info *); 70 static int ioapic_alloc_apic_id(int); 71 static void ioapic_set_apic_id(const struct ioapic_info *); 72 static void ioapic_gsi_setup(int); 73 static const struct ioapic_info * 74 ioapic_gsi_search(int); 75 static void ioapic_pin_prog(void *, int, int, 76 enum intr_trigger, enum intr_polarity, uint32_t, int); 77 78 static struct ioapic_conf ioapic_conf; 79 80 static TAILQ_HEAD(, ioapic_enumerator) ioapic_enumerators = 81 TAILQ_HEAD_INITIALIZER(ioapic_enumerators); 82 83 int ioapic_enable = -1; /* I/O APIC auto-enable mode */ 84 85 static int 86 ioapic_config(void) 87 { 88 struct ioapic_enumerator *e; 89 struct ioapic_info *info; 90 int start_apic_id = 0; 91 int error, i, probe; 92 register_t ef = 0; 93 94 TAILQ_INIT(&ioapic_conf.ioc_list); 95 for (i = 0; i < ISA_IRQ_CNT; ++i) 96 ioapic_conf.ioc_intsrc[i].int_gsi = -1; 97 98 probe = 1; 99 TUNABLE_INT_FETCH("hw.ioapic_probe", &probe); 100 if (!probe) { 101 kprintf("IOAPIC: warning I/O APIC will not be probed\n"); 102 return ENXIO; 103 } 104 105 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) { 106 error = e->ioapic_probe(e); 107 if (!error) 108 break; 109 } 110 if (e == NULL) { 111 kprintf("IOAPIC: can't find I/O APIC\n"); 112 return ENXIO; 113 } 114 115 crit_enter(); 116 117 ef = read_rflags(); 118 cpu_disable_intr(); 119 120 /* 121 * Switch to I/O APIC MachIntrABI and reconfigure 122 * the default IDT entries. 123 */ 124 MachIntrABI = MachIntrABI_IOAPIC; 125 MachIntrABI.setdefault(); 126 127 e->ioapic_enumerate(e); 128 129 /* 130 * Setup index 131 */ 132 i = 0; 133 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 134 info->io_idx = i++; 135 136 if (i > IOAPIC_COUNT_MAX) 137 panic("ioapic_config: more than 16 I/O APIC"); 138 139 /* 140 * Setup APIC ID 141 */ 142 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 143 int apic_id; 144 145 apic_id = ioapic_alloc_apic_id(start_apic_id); 146 if (apic_id == NAPICID) { 147 kprintf("IOAPIC: can't alloc APIC ID for " 148 "%dth I/O APIC\n", info->io_idx); 149 break; 150 } 151 info->io_apic_id = apic_id; 152 153 start_apic_id = apic_id + 1; 154 } 155 if (info != NULL) { 156 /* 157 * xAPIC allows I/O APIC's APIC ID to be same 158 * as the LAPIC's APIC ID 159 */ 160 kprintf("IOAPIC: use xAPIC model to alloc APIC ID " 161 "for I/O APIC\n"); 162 163 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 164 info->io_apic_id = info->io_idx; 165 } 166 167 /* 168 * Warning about any GSI holes 169 */ 170 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 171 const struct ioapic_info *prev_info; 172 173 prev_info = TAILQ_PREV(info, ioapic_info_list, io_link); 174 if (prev_info != NULL) { 175 if (info->io_gsi_base != 176 prev_info->io_gsi_base + prev_info->io_npin) { 177 kprintf("IOAPIC: warning gsi hole " 178 "[%d, %d]\n", 179 prev_info->io_gsi_base + 180 prev_info->io_npin, 181 info->io_gsi_base - 1); 182 } 183 } 184 } 185 186 if (bootverbose) { 187 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 188 kprintf("IOAPIC: idx %d, apic id %d, " 189 "gsi base %d, npin %d\n", 190 info->io_idx, 191 info->io_apic_id, 192 info->io_gsi_base, 193 info->io_npin); 194 } 195 } 196 197 /* 198 * Setup all I/O APIC 199 */ 200 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 201 ioapic_setup(info); 202 ioapic_fixup_legacy_irqmaps(); 203 204 write_rflags(ef); 205 206 MachIntrABI.cleanup(); 207 208 crit_exit(); 209 210 return 0; 211 } 212 213 void 214 ioapic_enumerator_register(struct ioapic_enumerator *ne) 215 { 216 struct ioapic_enumerator *e; 217 218 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) { 219 if (e->ioapic_prio < ne->ioapic_prio) { 220 TAILQ_INSERT_BEFORE(e, ne, ioapic_link); 221 return; 222 } 223 } 224 TAILQ_INSERT_TAIL(&ioapic_enumerators, ne, ioapic_link); 225 } 226 227 void 228 ioapic_add(void *addr, int gsi_base, int npin) 229 { 230 struct ioapic_info *info, *ninfo; 231 int gsi_end; 232 233 gsi_end = gsi_base + npin - 1; 234 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 235 if ((gsi_base >= info->io_gsi_base && 236 gsi_base < info->io_gsi_base + info->io_npin) || 237 (gsi_end >= info->io_gsi_base && 238 gsi_end < info->io_gsi_base + info->io_npin)) { 239 panic("ioapic_add: overlapped gsi, base %d npin %d, " 240 "hit base %d, npin %d\n", gsi_base, npin, 241 info->io_gsi_base, info->io_npin); 242 } 243 if (info->io_addr == addr) 244 panic("ioapic_add: duplicated addr %p", addr); 245 } 246 247 ninfo = kmalloc(sizeof(*ninfo), M_DEVBUF, M_WAITOK | M_ZERO); 248 ninfo->io_addr = addr; 249 ninfo->io_npin = npin; 250 ninfo->io_gsi_base = gsi_base; 251 ninfo->io_apic_id = -1; 252 253 /* 254 * Create IOAPIC list in ascending order of GSI base 255 */ 256 TAILQ_FOREACH_REVERSE(info, &ioapic_conf.ioc_list, 257 ioapic_info_list, io_link) { 258 if (ninfo->io_gsi_base > info->io_gsi_base) { 259 TAILQ_INSERT_AFTER(&ioapic_conf.ioc_list, 260 info, ninfo, io_link); 261 break; 262 } 263 } 264 if (info == NULL) 265 TAILQ_INSERT_HEAD(&ioapic_conf.ioc_list, ninfo, io_link); 266 } 267 268 void 269 ioapic_intsrc(int irq, int gsi, enum intr_trigger trig, enum intr_polarity pola) 270 { 271 struct ioapic_intsrc *int_src; 272 273 KKASSERT(irq < ISA_IRQ_CNT); 274 int_src = &ioapic_conf.ioc_intsrc[irq]; 275 276 if (gsi == 0) { 277 /* Don't allow mixed mode */ 278 kprintf("IOAPIC: warning intsrc irq %d -> gsi 0\n", irq); 279 return; 280 } 281 282 if (int_src->int_gsi != -1) { 283 if (int_src->int_gsi != gsi) { 284 kprintf("IOAPIC: warning intsrc irq %d, gsi " 285 "%d -> %d\n", irq, int_src->int_gsi, gsi); 286 } 287 if (int_src->int_trig != trig) { 288 kprintf("IOAPIC: warning intsrc irq %d, trig " 289 "%s -> %s\n", irq, 290 intr_str_trigger(int_src->int_trig), 291 intr_str_trigger(trig)); 292 } 293 if (int_src->int_pola != pola) { 294 kprintf("IOAPIC: warning intsrc irq %d, pola " 295 "%s -> %s\n", irq, 296 intr_str_polarity(int_src->int_pola), 297 intr_str_polarity(pola)); 298 } 299 } 300 int_src->int_gsi = gsi; 301 int_src->int_trig = trig; 302 int_src->int_pola = pola; 303 } 304 305 static void 306 ioapic_set_apic_id(const struct ioapic_info *info) 307 { 308 uint32_t id; 309 int apic_id; 310 311 id = ioapic_read(info->io_addr, IOAPIC_ID); 312 313 id &= ~APIC_ID_MASK; 314 id |= (info->io_apic_id << 24); 315 316 ioapic_write(info->io_addr, IOAPIC_ID, id); 317 318 /* 319 * Re-read && test 320 */ 321 id = ioapic_read(info->io_addr, IOAPIC_ID); 322 apic_id = (id & APIC_ID_MASK) >> 24; 323 324 /* 325 * I/O APIC ID is a 4bits field 326 */ 327 if ((apic_id & IOAPIC_ID_MASK) != 328 (info->io_apic_id & IOAPIC_ID_MASK)) { 329 panic("ioapic_set_apic_id: can't set apic id to %d, " 330 "currently set to %d\n", info->io_apic_id, apic_id); 331 } 332 } 333 334 static void 335 ioapic_gsi_setup(int gsi) 336 { 337 enum intr_trigger trig; 338 enum intr_polarity pola; 339 int irq; 340 341 if (gsi == 0) { 342 /* ExtINT */ 343 imen_lock(); 344 ioapic_extpin_setup(ioapic_gsi_ioaddr(gsi), 345 ioapic_gsi_pin(gsi), 0); 346 imen_unlock(); 347 return; 348 } 349 350 trig = 0; /* silence older gcc's */ 351 pola = 0; /* silence older gcc's */ 352 353 for (irq = 0; irq < ISA_IRQ_CNT; ++irq) { 354 const struct ioapic_intsrc *int_src = 355 &ioapic_conf.ioc_intsrc[irq]; 356 357 if (gsi == int_src->int_gsi) { 358 trig = int_src->int_trig; 359 pola = int_src->int_pola; 360 break; 361 } 362 } 363 364 if (irq == ISA_IRQ_CNT) { 365 /* 366 * No explicit IRQ to GSI mapping; 367 * use the default 1:1 mapping 368 */ 369 irq = gsi; 370 if (irq < ISA_IRQ_CNT) { 371 if (ioapic_conf.ioc_intsrc[irq].int_gsi >= 0) { 372 /* 373 * This IRQ is mapped to different GSI, 374 * don't do the default configuration. 375 * The configuration of the target GSI 376 * will finally setup this IRQ. 377 * 378 * This GSI is not used, disable it. 379 */ 380 imen_lock(); 381 ioapic_pin_setup(ioapic_gsi_ioaddr(gsi), 382 ioapic_gsi_pin(gsi), 0, 383 INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH, 0); 384 imen_unlock(); 385 return; 386 } 387 trig = INTR_TRIGGER_EDGE; 388 pola = INTR_POLARITY_HIGH; 389 } else { 390 trig = INTR_TRIGGER_LEVEL; 391 pola = INTR_POLARITY_LOW; 392 } 393 } 394 395 ioapic_set_legacy_irqmap(irq, gsi, trig, pola); 396 } 397 398 void * 399 ioapic_gsi_ioaddr(int gsi) 400 { 401 const struct ioapic_info *info; 402 403 info = ioapic_gsi_search(gsi); 404 return info->io_addr; 405 } 406 407 int 408 ioapic_gsi_pin(int gsi) 409 { 410 const struct ioapic_info *info; 411 412 info = ioapic_gsi_search(gsi); 413 return gsi - info->io_gsi_base; 414 } 415 416 static const struct ioapic_info * 417 ioapic_gsi_search(int gsi) 418 { 419 const struct ioapic_info *info; 420 421 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 422 if (gsi >= info->io_gsi_base && 423 gsi < info->io_gsi_base + info->io_npin) 424 return info; 425 } 426 panic("ioapic_gsi_search: no I/O APIC"); 427 } 428 429 int 430 ioapic_gsi(int idx, int pin) 431 { 432 const struct ioapic_info *info; 433 434 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 435 if (info->io_idx == idx) 436 break; 437 } 438 if (info == NULL) 439 return -1; 440 if (pin >= info->io_npin) 441 return -1; 442 return info->io_gsi_base + pin; 443 } 444 445 void 446 ioapic_extpin_setup(void *addr, int pin, int vec) 447 { 448 ioapic_pin_prog(addr, pin, vec, 449 INTR_TRIGGER_CONFORM, INTR_POLARITY_CONFORM, IOART_DELEXINT, 0); 450 } 451 452 int 453 ioapic_extpin_gsi(void) 454 { 455 return 0; 456 } 457 458 void 459 ioapic_pin_setup(void *addr, int pin, int vec, 460 enum intr_trigger trig, enum intr_polarity pola, int cpuid) 461 { 462 /* 463 * Always clear an I/O APIC pin before [re]programming it. This is 464 * particularly important if the pin is set up for a level interrupt 465 * as the IOART_REM_IRR bit might be set. When we reprogram the 466 * vector any EOI from pending ints on this pin could be lost and 467 * IRR might never get reset. 468 * 469 * To fix this problem, clear the vector and make sure it is 470 * programmed as an edge interrupt. This should theoretically 471 * clear IRR so we can later, safely program it as a level 472 * interrupt. 473 */ 474 ioapic_pin_prog(addr, pin, vec, INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH, 475 IOART_DELFIXED, cpuid); 476 ioapic_pin_prog(addr, pin, vec, trig, pola, IOART_DELFIXED, cpuid); 477 } 478 479 static void 480 ioapic_pin_prog(void *addr, int pin, int vec, 481 enum intr_trigger trig, enum intr_polarity pola, 482 uint32_t del_mode, int cpuid) 483 { 484 uint32_t flags, target; 485 int select; 486 487 KKASSERT(del_mode == IOART_DELEXINT || del_mode == IOART_DELFIXED); 488 489 select = IOAPIC_REDTBL0 + (2 * pin); 490 491 flags = ioapic_read(addr, select) & IOART_RESV; 492 flags |= IOART_INTMSET | IOART_DESTPHY; 493 #ifdef foo 494 flags |= del_mode; 495 #else 496 /* 497 * We only support limited I/O APIC mixed mode, 498 * so even for ExtINT, we still use "fixed" 499 * delivery mode. 500 */ 501 flags |= IOART_DELFIXED; 502 #endif 503 504 if (del_mode == IOART_DELEXINT) { 505 KKASSERT(trig == INTR_TRIGGER_CONFORM && 506 pola == INTR_POLARITY_CONFORM); 507 flags |= IOART_TRGREDG | IOART_INTAHI; 508 } else { 509 switch (trig) { 510 case INTR_TRIGGER_EDGE: 511 flags |= IOART_TRGREDG; 512 break; 513 514 case INTR_TRIGGER_LEVEL: 515 flags |= IOART_TRGRLVL; 516 break; 517 518 case INTR_TRIGGER_CONFORM: 519 panic("ioapic_pin_prog: trig conform is not " 520 "supported\n"); 521 } 522 switch (pola) { 523 case INTR_POLARITY_HIGH: 524 flags |= IOART_INTAHI; 525 break; 526 527 case INTR_POLARITY_LOW: 528 flags |= IOART_INTALO; 529 break; 530 531 case INTR_POLARITY_CONFORM: 532 panic("ioapic_pin_prog: pola conform is not " 533 "supported\n"); 534 } 535 } 536 537 target = ioapic_read(addr, select + 1) & IOART_HI_DEST_RESV; 538 target |= (CPUID_TO_APICID(cpuid) << IOART_HI_DEST_SHIFT) & 539 IOART_HI_DEST_MASK; 540 541 ioapic_write(addr, select, flags | vec); 542 ioapic_write(addr, select + 1, target); 543 } 544 545 static void 546 ioapic_setup(const struct ioapic_info *info) 547 { 548 int i; 549 550 ioapic_set_apic_id(info); 551 552 for (i = 0; i < info->io_npin; ++i) 553 ioapic_gsi_setup(info->io_gsi_base + i); 554 } 555 556 static int 557 ioapic_alloc_apic_id(int start) 558 { 559 for (;;) { 560 const struct ioapic_info *info; 561 int apic_id, apic_id16; 562 563 apic_id = lapic_unused_apic_id(start); 564 if (apic_id == NAPICID) { 565 kprintf("IOAPIC: can't find unused APIC ID\n"); 566 return apic_id; 567 } 568 apic_id16 = apic_id & IOAPIC_ID_MASK; 569 570 /* 571 * Check against other I/O APIC's APIC ID's lower 4bits. 572 * 573 * The new APIC ID will have to be different from others 574 * in the lower 4bits, no matter whether xAPIC is used 575 * or not. 576 */ 577 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 578 if (info->io_apic_id == -1) { 579 info = NULL; 580 break; 581 } 582 if ((info->io_apic_id & IOAPIC_ID_MASK) == apic_id16) 583 break; 584 } 585 if (info == NULL) 586 return apic_id; 587 588 kprintf("IOAPIC: APIC ID %d has same lower 4bits as " 589 "%dth I/O APIC, keep searching...\n", 590 apic_id, info->io_idx); 591 592 start = apic_id + 1; 593 } 594 panic("ioapic_unused_apic_id: never reached"); 595 } 596 597 /* 598 * Map a physical memory address representing I/O into KVA. The I/O 599 * block is assumed not to cross a page boundary. 600 */ 601 void * 602 ioapic_map(vm_paddr_t pa) 603 { 604 KKASSERT(pa < 0x100000000LL); 605 606 return pmap_mapdev_uncacheable(pa, PAGE_SIZE); 607 } 608 609 static void 610 ioapic_sysinit(void *dummy __unused) 611 { 612 int error; 613 614 if (!ioapic_enable) 615 return; 616 617 KASSERT(lapic_enable, ("I/O APIC is enabled, but LAPIC is disabled")); 618 error = ioapic_config(); 619 if (error) { 620 ioapic_enable = 0; 621 icu_reinit_noioapic(); 622 lapic_fixup_noioapic(); 623 } 624 } 625 SYSINIT(ioapic, SI_BOOT2_IOAPIC, SI_ORDER_FIRST, ioapic_sysinit, NULL); 626