1 /* 2 * Copyright (c) 1996, by Steve Passe 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. The name of the developer may NOT be used to endorse or promote products 11 * derived from this software without specific prior written permission. 12 * 13 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 14 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 15 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 16 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 17 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 18 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 19 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 20 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 21 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 22 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 23 * SUCH DAMAGE. 24 * 25 * $FreeBSD: src/sys/i386/i386/mpapic.c,v 1.37.2.7 2003/01/25 02:31:47 peter Exp $ 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/kernel.h> 31 #include <sys/bus.h> 32 #include <sys/machintr.h> 33 #include <sys/malloc.h> 34 #include <sys/thread2.h> 35 36 #include <machine/pmap.h> 37 #include <machine_base/isa/isa_intr.h> 38 #include <machine_base/icu/icu_var.h> 39 #include <machine_base/apic/lapic.h> 40 #include <machine_base/apic/ioapic.h> 41 #include <machine_base/apic/ioapic_abi.h> 42 #include <machine_base/apic/apicvar.h> 43 44 #define IOAPIC_COUNT_MAX 16 45 #define IOAPIC_ID_MASK (IOAPIC_COUNT_MAX - 1) 46 47 struct ioapic_info { 48 int io_idx; 49 int io_apic_id; 50 void *io_addr; 51 int io_npin; 52 int io_gsi_base; 53 54 TAILQ_ENTRY(ioapic_info) io_link; 55 }; 56 TAILQ_HEAD(ioapic_info_list, ioapic_info); 57 58 struct ioapic_intsrc { 59 int int_gsi; 60 enum intr_trigger int_trig; 61 enum intr_polarity int_pola; 62 }; 63 64 struct ioapic_conf { 65 struct ioapic_info_list ioc_list; 66 struct ioapic_intsrc ioc_intsrc[ISA_IRQ_CNT]; 67 }; 68 69 static int ioapic_config(void); 70 static void ioapic_setup(const struct ioapic_info *); 71 static int ioapic_alloc_apic_id(int); 72 static void ioapic_set_apic_id(const struct ioapic_info *); 73 static void ioapic_gsi_setup(int); 74 static const struct ioapic_info * 75 ioapic_gsi_search(int); 76 static void ioapic_pin_prog(void *, int, int, 77 enum intr_trigger, enum intr_polarity, uint32_t, int); 78 79 static struct ioapic_conf ioapic_conf; 80 81 static TAILQ_HEAD(, ioapic_enumerator) ioapic_enumerators = 82 TAILQ_HEAD_INITIALIZER(ioapic_enumerators); 83 84 int ioapic_enable = -1; /* I/O APIC auto-enable mode */ 85 86 static int 87 ioapic_config(void) 88 { 89 struct ioapic_enumerator *e; 90 struct ioapic_info *info; 91 int start_apic_id = 0; 92 int error, i, probe; 93 register_t ef = 0; 94 95 TAILQ_INIT(&ioapic_conf.ioc_list); 96 for (i = 0; i < ISA_IRQ_CNT; ++i) 97 ioapic_conf.ioc_intsrc[i].int_gsi = -1; 98 99 probe = 1; 100 TUNABLE_INT_FETCH("hw.ioapic_probe", &probe); 101 if (!probe) { 102 kprintf("IOAPIC: warning I/O APIC will not be probed\n"); 103 return ENXIO; 104 } 105 106 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) { 107 error = e->ioapic_probe(e); 108 if (!error) 109 break; 110 } 111 if (e == NULL) { 112 kprintf("IOAPIC: can't find I/O APIC\n"); 113 return ENXIO; 114 } 115 116 crit_enter(); 117 118 ef = read_rflags(); 119 cpu_disable_intr(); 120 121 /* 122 * Switch to I/O APIC MachIntrABI and reconfigure 123 * the default IDT entries. 124 */ 125 MachIntrABI = MachIntrABI_IOAPIC; 126 MachIntrABI.setdefault(); 127 128 e->ioapic_enumerate(e); 129 130 /* 131 * Setup index 132 */ 133 i = 0; 134 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 135 info->io_idx = i++; 136 137 if (i > IOAPIC_COUNT_MAX) 138 panic("ioapic_config: more than 16 I/O APIC"); 139 140 /* 141 * Setup APIC ID 142 */ 143 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 144 int apic_id; 145 146 apic_id = ioapic_alloc_apic_id(start_apic_id); 147 if (apic_id == NAPICID) { 148 kprintf("IOAPIC: can't alloc APIC ID for " 149 "%dth I/O APIC\n", info->io_idx); 150 break; 151 } 152 info->io_apic_id = apic_id; 153 154 start_apic_id = apic_id + 1; 155 } 156 if (info != NULL) { 157 /* 158 * xAPIC allows I/O APIC's APIC ID to be same 159 * as the LAPIC's APIC ID 160 */ 161 kprintf("IOAPIC: use xAPIC model to alloc APIC ID " 162 "for I/O APIC\n"); 163 164 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 165 info->io_apic_id = info->io_idx; 166 } 167 168 /* 169 * Warning about any GSI holes 170 */ 171 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 172 const struct ioapic_info *prev_info; 173 174 prev_info = TAILQ_PREV(info, ioapic_info_list, io_link); 175 if (prev_info != NULL) { 176 if (info->io_gsi_base != 177 prev_info->io_gsi_base + prev_info->io_npin) { 178 kprintf("IOAPIC: warning gsi hole " 179 "[%d, %d]\n", 180 prev_info->io_gsi_base + 181 prev_info->io_npin, 182 info->io_gsi_base - 1); 183 } 184 } 185 } 186 187 if (bootverbose) { 188 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 189 kprintf("IOAPIC: idx %d, apic id %d, " 190 "gsi base %d, npin %d\n", 191 info->io_idx, 192 info->io_apic_id, 193 info->io_gsi_base, 194 info->io_npin); 195 } 196 } 197 198 /* 199 * Setup all I/O APIC 200 */ 201 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) 202 ioapic_setup(info); 203 ioapic_fixup_legacy_irqmaps(); 204 205 write_rflags(ef); 206 207 MachIntrABI.cleanup(); 208 209 crit_exit(); 210 211 return 0; 212 } 213 214 void 215 ioapic_enumerator_register(struct ioapic_enumerator *ne) 216 { 217 struct ioapic_enumerator *e; 218 219 TAILQ_FOREACH(e, &ioapic_enumerators, ioapic_link) { 220 if (e->ioapic_prio < ne->ioapic_prio) { 221 TAILQ_INSERT_BEFORE(e, ne, ioapic_link); 222 return; 223 } 224 } 225 TAILQ_INSERT_TAIL(&ioapic_enumerators, ne, ioapic_link); 226 } 227 228 void 229 ioapic_add(void *addr, int gsi_base, int npin) 230 { 231 struct ioapic_info *info, *ninfo; 232 int gsi_end; 233 234 gsi_end = gsi_base + npin - 1; 235 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 236 if ((gsi_base >= info->io_gsi_base && 237 gsi_base < info->io_gsi_base + info->io_npin) || 238 (gsi_end >= info->io_gsi_base && 239 gsi_end < info->io_gsi_base + info->io_npin)) { 240 panic("ioapic_add: overlapped gsi, base %d npin %d, " 241 "hit base %d, npin %d\n", gsi_base, npin, 242 info->io_gsi_base, info->io_npin); 243 } 244 if (info->io_addr == addr) 245 panic("ioapic_add: duplicated addr %p", addr); 246 } 247 248 ninfo = kmalloc(sizeof(*ninfo), M_DEVBUF, M_WAITOK | M_ZERO); 249 ninfo->io_addr = addr; 250 ninfo->io_npin = npin; 251 ninfo->io_gsi_base = gsi_base; 252 ninfo->io_apic_id = -1; 253 254 /* 255 * Create IOAPIC list in ascending order of GSI base 256 */ 257 TAILQ_FOREACH_REVERSE(info, &ioapic_conf.ioc_list, 258 ioapic_info_list, io_link) { 259 if (ninfo->io_gsi_base > info->io_gsi_base) { 260 TAILQ_INSERT_AFTER(&ioapic_conf.ioc_list, 261 info, ninfo, io_link); 262 break; 263 } 264 } 265 if (info == NULL) 266 TAILQ_INSERT_HEAD(&ioapic_conf.ioc_list, ninfo, io_link); 267 } 268 269 void 270 ioapic_intsrc(int irq, int gsi, enum intr_trigger trig, enum intr_polarity pola) 271 { 272 struct ioapic_intsrc *int_src; 273 274 KKASSERT(irq < ISA_IRQ_CNT); 275 int_src = &ioapic_conf.ioc_intsrc[irq]; 276 277 if (gsi == 0) { 278 /* Don't allow mixed mode */ 279 kprintf("IOAPIC: warning intsrc irq %d -> gsi 0\n", irq); 280 return; 281 } 282 283 if (int_src->int_gsi != -1) { 284 if (int_src->int_gsi != gsi) { 285 kprintf("IOAPIC: warning intsrc irq %d, gsi " 286 "%d -> %d\n", irq, int_src->int_gsi, gsi); 287 } 288 if (int_src->int_trig != trig) { 289 kprintf("IOAPIC: warning intsrc irq %d, trig " 290 "%s -> %s\n", irq, 291 intr_str_trigger(int_src->int_trig), 292 intr_str_trigger(trig)); 293 } 294 if (int_src->int_pola != pola) { 295 kprintf("IOAPIC: warning intsrc irq %d, pola " 296 "%s -> %s\n", irq, 297 intr_str_polarity(int_src->int_pola), 298 intr_str_polarity(pola)); 299 } 300 } 301 int_src->int_gsi = gsi; 302 int_src->int_trig = trig; 303 int_src->int_pola = pola; 304 } 305 306 static void 307 ioapic_set_apic_id(const struct ioapic_info *info) 308 { 309 uint32_t id; 310 int apic_id; 311 312 id = ioapic_read(info->io_addr, IOAPIC_ID); 313 314 id &= ~APIC_ID_MASK; 315 id |= (info->io_apic_id << 24); 316 317 ioapic_write(info->io_addr, IOAPIC_ID, id); 318 319 /* 320 * Re-read && test 321 */ 322 id = ioapic_read(info->io_addr, IOAPIC_ID); 323 apic_id = (id & APIC_ID_MASK) >> 24; 324 325 /* 326 * I/O APIC ID is a 4bits field 327 */ 328 if ((apic_id & IOAPIC_ID_MASK) != 329 (info->io_apic_id & IOAPIC_ID_MASK)) { 330 panic("ioapic_set_apic_id: can't set apic id to %d, " 331 "currently set to %d\n", info->io_apic_id, apic_id); 332 } 333 } 334 335 static void 336 ioapic_gsi_setup(int gsi) 337 { 338 enum intr_trigger trig; 339 enum intr_polarity pola; 340 int irq; 341 342 if (gsi == 0) { 343 /* ExtINT */ 344 imen_lock(); 345 ioapic_extpin_setup(ioapic_gsi_ioaddr(gsi), 346 ioapic_gsi_pin(gsi), 0); 347 imen_unlock(); 348 return; 349 } 350 351 trig = 0; /* silence older gcc's */ 352 pola = 0; /* silence older gcc's */ 353 354 for (irq = 0; irq < ISA_IRQ_CNT; ++irq) { 355 const struct ioapic_intsrc *int_src = 356 &ioapic_conf.ioc_intsrc[irq]; 357 358 if (gsi == int_src->int_gsi) { 359 trig = int_src->int_trig; 360 pola = int_src->int_pola; 361 break; 362 } 363 } 364 365 if (irq == ISA_IRQ_CNT) { 366 /* 367 * No explicit IRQ to GSI mapping; 368 * use the default 1:1 mapping 369 */ 370 irq = gsi; 371 if (irq < ISA_IRQ_CNT) { 372 if (ioapic_conf.ioc_intsrc[irq].int_gsi >= 0) { 373 /* 374 * This IRQ is mapped to different GSI, 375 * don't do the default configuration. 376 * The configuration of the target GSI 377 * will finally setup this IRQ. 378 * 379 * This GSI is not used, disable it. 380 */ 381 imen_lock(); 382 ioapic_pin_setup(ioapic_gsi_ioaddr(gsi), 383 ioapic_gsi_pin(gsi), 0, 384 INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH, 0); 385 imen_unlock(); 386 return; 387 } 388 trig = INTR_TRIGGER_EDGE; 389 pola = INTR_POLARITY_HIGH; 390 } else { 391 trig = INTR_TRIGGER_LEVEL; 392 pola = INTR_POLARITY_LOW; 393 } 394 } 395 396 ioapic_set_legacy_irqmap(irq, gsi, trig, pola); 397 } 398 399 void * 400 ioapic_gsi_ioaddr(int gsi) 401 { 402 const struct ioapic_info *info; 403 404 info = ioapic_gsi_search(gsi); 405 return info->io_addr; 406 } 407 408 int 409 ioapic_gsi_pin(int gsi) 410 { 411 const struct ioapic_info *info; 412 413 info = ioapic_gsi_search(gsi); 414 return gsi - info->io_gsi_base; 415 } 416 417 static const struct ioapic_info * 418 ioapic_gsi_search(int gsi) 419 { 420 const struct ioapic_info *info; 421 422 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 423 if (gsi >= info->io_gsi_base && 424 gsi < info->io_gsi_base + info->io_npin) 425 return info; 426 } 427 panic("ioapic_gsi_search: no I/O APIC"); 428 } 429 430 int 431 ioapic_gsi(int idx, int pin) 432 { 433 const struct ioapic_info *info; 434 435 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 436 if (info->io_idx == idx) 437 break; 438 } 439 if (info == NULL) 440 return -1; 441 if (pin >= info->io_npin) 442 return -1; 443 return info->io_gsi_base + pin; 444 } 445 446 void 447 ioapic_extpin_setup(void *addr, int pin, int vec) 448 { 449 ioapic_pin_prog(addr, pin, vec, 450 INTR_TRIGGER_CONFORM, INTR_POLARITY_CONFORM, IOART_DELEXINT, 0); 451 } 452 453 int 454 ioapic_extpin_gsi(void) 455 { 456 return 0; 457 } 458 459 void 460 ioapic_pin_setup(void *addr, int pin, int vec, 461 enum intr_trigger trig, enum intr_polarity pola, int cpuid) 462 { 463 /* 464 * Always clear an I/O APIC pin before [re]programming it. This is 465 * particularly important if the pin is set up for a level interrupt 466 * as the IOART_REM_IRR bit might be set. When we reprogram the 467 * vector any EOI from pending ints on this pin could be lost and 468 * IRR might never get reset. 469 * 470 * To fix this problem, clear the vector and make sure it is 471 * programmed as an edge interrupt. This should theoretically 472 * clear IRR so we can later, safely program it as a level 473 * interrupt. 474 */ 475 ioapic_pin_prog(addr, pin, vec, INTR_TRIGGER_EDGE, INTR_POLARITY_HIGH, 476 IOART_DELFIXED, cpuid); 477 ioapic_pin_prog(addr, pin, vec, trig, pola, IOART_DELFIXED, cpuid); 478 } 479 480 static void 481 ioapic_pin_prog(void *addr, int pin, int vec, 482 enum intr_trigger trig, enum intr_polarity pola, 483 uint32_t del_mode, int cpuid) 484 { 485 uint32_t flags, target; 486 int select; 487 488 KKASSERT(del_mode == IOART_DELEXINT || del_mode == IOART_DELFIXED); 489 490 select = IOAPIC_REDTBL0 + (2 * pin); 491 492 flags = ioapic_read(addr, select) & IOART_RESV; 493 flags |= IOART_INTMSET | IOART_DESTPHY; 494 #ifdef foo 495 flags |= del_mode; 496 #else 497 /* 498 * We only support limited I/O APIC mixed mode, 499 * so even for ExtINT, we still use "fixed" 500 * delivery mode. 501 */ 502 flags |= IOART_DELFIXED; 503 #endif 504 505 if (del_mode == IOART_DELEXINT) { 506 KKASSERT(trig == INTR_TRIGGER_CONFORM && 507 pola == INTR_POLARITY_CONFORM); 508 flags |= IOART_TRGREDG | IOART_INTAHI; 509 } else { 510 switch (trig) { 511 case INTR_TRIGGER_EDGE: 512 flags |= IOART_TRGREDG; 513 break; 514 515 case INTR_TRIGGER_LEVEL: 516 flags |= IOART_TRGRLVL; 517 break; 518 519 case INTR_TRIGGER_CONFORM: 520 panic("ioapic_pin_prog: trig conform is not " 521 "supported\n"); 522 } 523 switch (pola) { 524 case INTR_POLARITY_HIGH: 525 flags |= IOART_INTAHI; 526 break; 527 528 case INTR_POLARITY_LOW: 529 flags |= IOART_INTALO; 530 break; 531 532 case INTR_POLARITY_CONFORM: 533 panic("ioapic_pin_prog: pola conform is not " 534 "supported\n"); 535 } 536 } 537 538 target = ioapic_read(addr, select + 1) & IOART_HI_DEST_RESV; 539 target |= (CPUID_TO_APICID(cpuid) << IOART_HI_DEST_SHIFT) & 540 IOART_HI_DEST_MASK; 541 542 ioapic_write(addr, select, flags | vec); 543 ioapic_write(addr, select + 1, target); 544 } 545 546 static void 547 ioapic_setup(const struct ioapic_info *info) 548 { 549 int i; 550 551 ioapic_set_apic_id(info); 552 553 for (i = 0; i < info->io_npin; ++i) 554 ioapic_gsi_setup(info->io_gsi_base + i); 555 } 556 557 static int 558 ioapic_alloc_apic_id(int start) 559 { 560 for (;;) { 561 const struct ioapic_info *info; 562 int apic_id, apic_id16; 563 564 apic_id = lapic_unused_apic_id(start); 565 if (apic_id == NAPICID) { 566 kprintf("IOAPIC: can't find unused APIC ID\n"); 567 return apic_id; 568 } 569 apic_id16 = apic_id & IOAPIC_ID_MASK; 570 571 /* 572 * Check against other I/O APIC's APIC ID's lower 4bits. 573 * 574 * The new APIC ID will have to be different from others 575 * in the lower 4bits, no matter whether xAPIC is used 576 * or not. 577 */ 578 TAILQ_FOREACH(info, &ioapic_conf.ioc_list, io_link) { 579 if (info->io_apic_id == -1) { 580 info = NULL; 581 break; 582 } 583 if ((info->io_apic_id & IOAPIC_ID_MASK) == apic_id16) 584 break; 585 } 586 if (info == NULL) 587 return apic_id; 588 589 kprintf("IOAPIC: APIC ID %d has same lower 4bits as " 590 "%dth I/O APIC, keep searching...\n", 591 apic_id, info->io_idx); 592 593 start = apic_id + 1; 594 } 595 panic("ioapic_unused_apic_id: never reached"); 596 } 597 598 /* 599 * Map a physical memory address representing I/O into KVA. The I/O 600 * block is assumed not to cross a page boundary. 601 */ 602 void * 603 ioapic_map(vm_paddr_t pa) 604 { 605 KKASSERT(pa < 0x100000000LL); 606 607 return pmap_mapdev_uncacheable(pa, PAGE_SIZE); 608 } 609 610 static void 611 ioapic_sysinit(void *dummy __unused) 612 { 613 int error; 614 615 if (!ioapic_enable) 616 return; 617 618 KASSERT(lapic_enable, ("I/O APIC is enabled, but LAPIC is disabled")); 619 error = ioapic_config(); 620 if (error) { 621 kprintf("IOAPIC disabled - error during ioapic_config()\n"); 622 ioapic_enable = 0; 623 icu_reinit_noioapic(); 624 lapic_fixup_noioapic(); 625 } 626 } 627 SYSINIT(ioapic, SI_BOOT2_IOAPIC, SI_ORDER_FIRST, ioapic_sysinit, NULL); 628