1 /*- 2 * Copyright (c) 2003 John Baldwin <jhb@FreeBSD.org> 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 */ 26 27 #include <sys/cdefs.h> 28 __FBSDID("$FreeBSD$"); 29 30 #include "opt_acpi.h" 31 #include "opt_isa.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/bus.h> 36 #include <sys/kernel.h> 37 #include <sys/lock.h> 38 #include <sys/malloc.h> 39 #include <sys/module.h> 40 #include <sys/mutex.h> 41 #include <sys/sysctl.h> 42 43 #include <dev/pci/pcireg.h> 44 #include <dev/pci/pcivar.h> 45 46 #include <vm/vm.h> 47 #include <vm/pmap.h> 48 49 #include <x86/apicreg.h> 50 #include <machine/frame.h> 51 #include <machine/intr_machdep.h> 52 #include <x86/apicvar.h> 53 #include <machine/resource.h> 54 #include <machine/segments.h> 55 #include <x86/iommu/iommu_intrmap.h> 56 57 #define IOAPIC_ISA_INTS 16 58 #define IOAPIC_MEM_REGION 32 59 #define IOAPIC_REDTBL_LO(i) (IOAPIC_REDTBL + (i) * 2) 60 #define IOAPIC_REDTBL_HI(i) (IOAPIC_REDTBL_LO(i) + 1) 61 62 static MALLOC_DEFINE(M_IOAPIC, "io_apic", "I/O APIC structures"); 63 64 /* 65 * I/O APIC interrupt source driver. Each pin is assigned an IRQ cookie 66 * as laid out in the ACPI System Interrupt number model where each I/O 67 * APIC has a contiguous chunk of the System Interrupt address space. 68 * We assume that IRQs 1 - 15 behave like ISA IRQs and that all other 69 * IRQs behave as PCI IRQs by default. We also assume that the pin for 70 * IRQ 0 is actually an ExtINT pin. The apic enumerators override the 71 * configuration of individual pins as indicated by their tables. 72 * 73 * Documentation for the I/O APIC: "82093AA I/O Advanced Programmable 74 * Interrupt Controller (IOAPIC)", May 1996, Intel Corp. 75 * ftp://download.intel.com/design/chipsets/datashts/29056601.pdf 76 */ 77 78 struct ioapic_intsrc { 79 struct intsrc io_intsrc; 80 u_int io_irq; 81 u_int io_intpin:8; 82 u_int io_vector:8; 83 u_int io_cpu; 84 u_int io_activehi:1; 85 u_int io_edgetrigger:1; 86 u_int io_masked:1; 87 int io_bus:4; 88 uint32_t io_lowreg; 89 u_int io_remap_cookie; 90 }; 91 92 struct ioapic { 93 struct pic io_pic; 94 u_int io_id:8; /* logical ID */ 95 u_int io_apic_id:4; 96 u_int io_intbase:8; /* System Interrupt base */ 97 u_int io_numintr:8; 98 u_int io_haseoi:1; 99 volatile ioapic_t *io_addr; /* XXX: should use bus_space */ 100 vm_paddr_t io_paddr; 101 STAILQ_ENTRY(ioapic) io_next; 102 struct ioapic_intsrc io_pins[0]; 103 }; 104 105 static u_int ioapic_read(volatile ioapic_t *apic, int reg); 106 static void ioapic_write(volatile ioapic_t *apic, int reg, u_int val); 107 static const char *ioapic_bus_string(int bus_type); 108 static void ioapic_print_irq(struct ioapic_intsrc *intpin); 109 static void ioapic_enable_source(struct intsrc *isrc); 110 static void ioapic_disable_source(struct intsrc *isrc, int eoi); 111 static void ioapic_eoi_source(struct intsrc *isrc); 112 static void ioapic_enable_intr(struct intsrc *isrc); 113 static void ioapic_disable_intr(struct intsrc *isrc); 114 static int ioapic_vector(struct intsrc *isrc); 115 static int ioapic_source_pending(struct intsrc *isrc); 116 static int ioapic_config_intr(struct intsrc *isrc, enum intr_trigger trig, 117 enum intr_polarity pol); 118 static void ioapic_resume(struct pic *pic, bool suspend_cancelled); 119 static int ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id); 120 static void ioapic_program_intpin(struct ioapic_intsrc *intpin); 121 static void ioapic_reprogram_intpin(struct intsrc *isrc); 122 123 static STAILQ_HEAD(,ioapic) ioapic_list = STAILQ_HEAD_INITIALIZER(ioapic_list); 124 struct pic ioapic_template = { 125 .pic_enable_source = ioapic_enable_source, 126 .pic_disable_source = ioapic_disable_source, 127 .pic_eoi_source = ioapic_eoi_source, 128 .pic_enable_intr = ioapic_enable_intr, 129 .pic_disable_intr = ioapic_disable_intr, 130 .pic_vector = ioapic_vector, 131 .pic_source_pending = ioapic_source_pending, 132 .pic_suspend = NULL, 133 .pic_resume = ioapic_resume, 134 .pic_config_intr = ioapic_config_intr, 135 .pic_assign_cpu = ioapic_assign_cpu, 136 .pic_reprogram_pin = ioapic_reprogram_intpin, 137 }; 138 139 static int next_ioapic_base; 140 static u_int next_id; 141 142 static int enable_extint; 143 SYSCTL_INT(_hw_apic, OID_AUTO, enable_extint, CTLFLAG_RDTUN, &enable_extint, 0, 144 "Enable the ExtINT pin in the first I/O APIC"); 145 146 static void 147 _ioapic_eoi_source(struct intsrc *isrc, int locked) 148 { 149 struct ioapic_intsrc *src; 150 struct ioapic *io; 151 volatile uint32_t *apic_eoi; 152 uint32_t low1; 153 154 lapic_eoi(); 155 if (!lapic_eoi_suppression) 156 return; 157 src = (struct ioapic_intsrc *)isrc; 158 if (src->io_edgetrigger) 159 return; 160 io = (struct ioapic *)isrc->is_pic; 161 162 /* 163 * Handle targeted EOI for level-triggered pins, if broadcast 164 * EOI suppression is supported by LAPICs. 165 */ 166 if (io->io_haseoi) { 167 /* 168 * If IOAPIC has EOI Register, simply write vector 169 * number into the reg. 170 */ 171 apic_eoi = (volatile uint32_t *)((volatile char *) 172 io->io_addr + IOAPIC_EOIR); 173 *apic_eoi = src->io_vector; 174 } else { 175 /* 176 * Otherwise, if IO-APIC is too old to provide EOIR, 177 * do what Intel did for the Linux kernel. Temporary 178 * switch the pin to edge-trigger and back, masking 179 * the pin during the trick. 180 */ 181 if (!locked) 182 mtx_lock_spin(&icu_lock); 183 low1 = src->io_lowreg; 184 low1 &= ~IOART_TRGRLVL; 185 low1 |= IOART_TRGREDG | IOART_INTMSET; 186 ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(src->io_intpin), 187 low1); 188 ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(src->io_intpin), 189 src->io_lowreg); 190 if (!locked) 191 mtx_unlock_spin(&icu_lock); 192 } 193 } 194 195 static u_int 196 ioapic_read(volatile ioapic_t *apic, int reg) 197 { 198 199 mtx_assert(&icu_lock, MA_OWNED); 200 apic->ioregsel = reg; 201 return (apic->iowin); 202 } 203 204 static void 205 ioapic_write(volatile ioapic_t *apic, int reg, u_int val) 206 { 207 208 mtx_assert(&icu_lock, MA_OWNED); 209 apic->ioregsel = reg; 210 apic->iowin = val; 211 } 212 213 static const char * 214 ioapic_bus_string(int bus_type) 215 { 216 217 switch (bus_type) { 218 case APIC_BUS_ISA: 219 return ("ISA"); 220 case APIC_BUS_EISA: 221 return ("EISA"); 222 case APIC_BUS_PCI: 223 return ("PCI"); 224 default: 225 return ("unknown"); 226 } 227 } 228 229 static void 230 ioapic_print_irq(struct ioapic_intsrc *intpin) 231 { 232 233 switch (intpin->io_irq) { 234 case IRQ_DISABLED: 235 printf("disabled"); 236 break; 237 case IRQ_EXTINT: 238 printf("ExtINT"); 239 break; 240 case IRQ_NMI: 241 printf("NMI"); 242 break; 243 case IRQ_SMI: 244 printf("SMI"); 245 break; 246 default: 247 printf("%s IRQ %u", ioapic_bus_string(intpin->io_bus), 248 intpin->io_irq); 249 } 250 } 251 252 static void 253 ioapic_enable_source(struct intsrc *isrc) 254 { 255 struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; 256 struct ioapic *io = (struct ioapic *)isrc->is_pic; 257 uint32_t flags; 258 259 mtx_lock_spin(&icu_lock); 260 if (intpin->io_masked) { 261 flags = intpin->io_lowreg & ~IOART_INTMASK; 262 ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), 263 flags); 264 intpin->io_masked = 0; 265 } 266 mtx_unlock_spin(&icu_lock); 267 } 268 269 static void 270 ioapic_disable_source(struct intsrc *isrc, int eoi) 271 { 272 struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; 273 struct ioapic *io = (struct ioapic *)isrc->is_pic; 274 uint32_t flags; 275 276 mtx_lock_spin(&icu_lock); 277 if (!intpin->io_masked && !intpin->io_edgetrigger) { 278 flags = intpin->io_lowreg | IOART_INTMSET; 279 ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), 280 flags); 281 intpin->io_masked = 1; 282 } 283 284 if (eoi == PIC_EOI) 285 _ioapic_eoi_source(isrc, 1); 286 287 mtx_unlock_spin(&icu_lock); 288 } 289 290 static void 291 ioapic_eoi_source(struct intsrc *isrc) 292 { 293 294 _ioapic_eoi_source(isrc, 0); 295 } 296 297 /* 298 * Completely program an intpin based on the data in its interrupt source 299 * structure. 300 */ 301 static void 302 ioapic_program_intpin(struct ioapic_intsrc *intpin) 303 { 304 struct ioapic *io = (struct ioapic *)intpin->io_intsrc.is_pic; 305 uint32_t low, high, value; 306 #ifdef ACPI_DMAR 307 int error; 308 #endif 309 310 /* 311 * If a pin is completely invalid or if it is valid but hasn't 312 * been enabled yet, just ensure that the pin is masked. 313 */ 314 mtx_assert(&icu_lock, MA_OWNED); 315 if (intpin->io_irq == IRQ_DISABLED || (intpin->io_irq < NUM_IO_INTS && 316 intpin->io_vector == 0)) { 317 low = ioapic_read(io->io_addr, 318 IOAPIC_REDTBL_LO(intpin->io_intpin)); 319 if ((low & IOART_INTMASK) == IOART_INTMCLR) 320 ioapic_write(io->io_addr, 321 IOAPIC_REDTBL_LO(intpin->io_intpin), 322 low | IOART_INTMSET); 323 #ifdef ACPI_DMAR 324 mtx_unlock_spin(&icu_lock); 325 iommu_unmap_ioapic_intr(io->io_apic_id, 326 &intpin->io_remap_cookie); 327 mtx_lock_spin(&icu_lock); 328 #endif 329 return; 330 } 331 332 #ifdef ACPI_DMAR 333 mtx_unlock_spin(&icu_lock); 334 error = iommu_map_ioapic_intr(io->io_apic_id, 335 intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger, 336 intpin->io_activehi, intpin->io_irq, &intpin->io_remap_cookie, 337 &high, &low); 338 mtx_lock_spin(&icu_lock); 339 if (error == 0) { 340 ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin), 341 high); 342 intpin->io_lowreg = low; 343 ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), 344 low); 345 return; 346 } else if (error != EOPNOTSUPP) { 347 return; 348 } 349 #endif 350 351 /* Set the destination. */ 352 low = IOART_DESTPHY; 353 high = intpin->io_cpu << APIC_ID_SHIFT; 354 355 /* Program the rest of the low word. */ 356 if (intpin->io_edgetrigger) 357 low |= IOART_TRGREDG; 358 else 359 low |= IOART_TRGRLVL; 360 if (intpin->io_activehi) 361 low |= IOART_INTAHI; 362 else 363 low |= IOART_INTALO; 364 if (intpin->io_masked) 365 low |= IOART_INTMSET; 366 switch (intpin->io_irq) { 367 case IRQ_EXTINT: 368 KASSERT(intpin->io_edgetrigger, 369 ("ExtINT not edge triggered")); 370 low |= IOART_DELEXINT; 371 break; 372 case IRQ_NMI: 373 KASSERT(intpin->io_edgetrigger, 374 ("NMI not edge triggered")); 375 low |= IOART_DELNMI; 376 break; 377 case IRQ_SMI: 378 KASSERT(intpin->io_edgetrigger, 379 ("SMI not edge triggered")); 380 low |= IOART_DELSMI; 381 break; 382 default: 383 KASSERT(intpin->io_vector != 0, ("No vector for IRQ %u", 384 intpin->io_irq)); 385 low |= IOART_DELFIXED | intpin->io_vector; 386 } 387 388 /* Write the values to the APIC. */ 389 value = ioapic_read(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin)); 390 value &= ~IOART_DEST; 391 value |= high; 392 ioapic_write(io->io_addr, IOAPIC_REDTBL_HI(intpin->io_intpin), value); 393 intpin->io_lowreg = low; 394 ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), low); 395 } 396 397 static void 398 ioapic_reprogram_intpin(struct intsrc *isrc) 399 { 400 401 mtx_lock_spin(&icu_lock); 402 ioapic_program_intpin((struct ioapic_intsrc *)isrc); 403 mtx_unlock_spin(&icu_lock); 404 } 405 406 static int 407 ioapic_assign_cpu(struct intsrc *isrc, u_int apic_id) 408 { 409 struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; 410 struct ioapic *io = (struct ioapic *)isrc->is_pic; 411 u_int old_vector, new_vector; 412 u_int old_id; 413 414 /* 415 * On Hyper-V: 416 * - Stick to the first cpu for all I/O APIC pins. 417 * - And don't allow destination cpu changes. 418 */ 419 if (vm_guest == VM_GUEST_HV) { 420 if (intpin->io_vector) 421 return (EINVAL); 422 else 423 apic_id = 0; 424 } 425 426 /* 427 * keep 1st core as the destination for NMI 428 */ 429 if (intpin->io_irq == IRQ_NMI) 430 apic_id = 0; 431 432 /* 433 * Set us up to free the old irq. 434 */ 435 old_vector = intpin->io_vector; 436 old_id = intpin->io_cpu; 437 if (old_vector && apic_id == old_id) 438 return (0); 439 440 /* 441 * Allocate an APIC vector for this interrupt pin. Once 442 * we have a vector we program the interrupt pin. 443 */ 444 new_vector = apic_alloc_vector(apic_id, intpin->io_irq); 445 if (new_vector == 0) 446 return (ENOSPC); 447 448 /* 449 * Mask the old intpin if it is enabled while it is migrated. 450 * 451 * At least some level-triggered interrupts seem to need the 452 * extra DELAY() to avoid being stuck in a non-EOI'd state. 453 */ 454 mtx_lock_spin(&icu_lock); 455 if (!intpin->io_masked && !intpin->io_edgetrigger) { 456 ioapic_write(io->io_addr, IOAPIC_REDTBL_LO(intpin->io_intpin), 457 intpin->io_lowreg | IOART_INTMSET); 458 mtx_unlock_spin(&icu_lock); 459 DELAY(100); 460 mtx_lock_spin(&icu_lock); 461 } 462 463 intpin->io_cpu = apic_id; 464 intpin->io_vector = new_vector; 465 if (isrc->is_handlers > 0) 466 apic_enable_vector(intpin->io_cpu, intpin->io_vector); 467 if (bootverbose) { 468 printf("ioapic%u: routing intpin %u (", io->io_id, 469 intpin->io_intpin); 470 ioapic_print_irq(intpin); 471 printf(") to lapic %u vector %u\n", intpin->io_cpu, 472 intpin->io_vector); 473 } 474 ioapic_program_intpin(intpin); 475 mtx_unlock_spin(&icu_lock); 476 477 /* 478 * Free the old vector after the new one is established. This is done 479 * to prevent races where we could miss an interrupt. 480 */ 481 if (old_vector) { 482 if (isrc->is_handlers > 0) 483 apic_disable_vector(old_id, old_vector); 484 apic_free_vector(old_id, old_vector, intpin->io_irq); 485 } 486 return (0); 487 } 488 489 static void 490 ioapic_enable_intr(struct intsrc *isrc) 491 { 492 struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; 493 494 if (intpin->io_vector == 0) 495 if (ioapic_assign_cpu(isrc, intr_next_cpu()) != 0) 496 panic("Couldn't find an APIC vector for IRQ %d", 497 intpin->io_irq); 498 apic_enable_vector(intpin->io_cpu, intpin->io_vector); 499 } 500 501 502 static void 503 ioapic_disable_intr(struct intsrc *isrc) 504 { 505 struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; 506 u_int vector; 507 508 if (intpin->io_vector != 0) { 509 /* Mask this interrupt pin and free its APIC vector. */ 510 vector = intpin->io_vector; 511 apic_disable_vector(intpin->io_cpu, vector); 512 mtx_lock_spin(&icu_lock); 513 intpin->io_masked = 1; 514 intpin->io_vector = 0; 515 ioapic_program_intpin(intpin); 516 mtx_unlock_spin(&icu_lock); 517 apic_free_vector(intpin->io_cpu, vector, intpin->io_irq); 518 } 519 } 520 521 static int 522 ioapic_vector(struct intsrc *isrc) 523 { 524 struct ioapic_intsrc *pin; 525 526 pin = (struct ioapic_intsrc *)isrc; 527 return (pin->io_irq); 528 } 529 530 static int 531 ioapic_source_pending(struct intsrc *isrc) 532 { 533 struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; 534 535 if (intpin->io_vector == 0) 536 return 0; 537 return (lapic_intr_pending(intpin->io_vector)); 538 } 539 540 static int 541 ioapic_config_intr(struct intsrc *isrc, enum intr_trigger trig, 542 enum intr_polarity pol) 543 { 544 struct ioapic_intsrc *intpin = (struct ioapic_intsrc *)isrc; 545 struct ioapic *io = (struct ioapic *)isrc->is_pic; 546 int changed; 547 548 KASSERT(!(trig == INTR_TRIGGER_CONFORM || pol == INTR_POLARITY_CONFORM), 549 ("%s: Conforming trigger or polarity\n", __func__)); 550 551 /* 552 * EISA interrupts always use active high polarity, so don't allow 553 * them to be set to active low. 554 * 555 * XXX: Should we write to the ELCR if the trigger mode changes for 556 * an EISA IRQ or an ISA IRQ with the ELCR present? 557 */ 558 mtx_lock_spin(&icu_lock); 559 if (intpin->io_bus == APIC_BUS_EISA) 560 pol = INTR_POLARITY_HIGH; 561 changed = 0; 562 if (intpin->io_edgetrigger != (trig == INTR_TRIGGER_EDGE)) { 563 if (bootverbose) 564 printf("ioapic%u: Changing trigger for pin %u to %s\n", 565 io->io_id, intpin->io_intpin, 566 trig == INTR_TRIGGER_EDGE ? "edge" : "level"); 567 intpin->io_edgetrigger = (trig == INTR_TRIGGER_EDGE); 568 changed++; 569 } 570 if (intpin->io_activehi != (pol == INTR_POLARITY_HIGH)) { 571 if (bootverbose) 572 printf("ioapic%u: Changing polarity for pin %u to %s\n", 573 io->io_id, intpin->io_intpin, 574 pol == INTR_POLARITY_HIGH ? "high" : "low"); 575 intpin->io_activehi = (pol == INTR_POLARITY_HIGH); 576 changed++; 577 } 578 if (changed) 579 ioapic_program_intpin(intpin); 580 mtx_unlock_spin(&icu_lock); 581 return (0); 582 } 583 584 static void 585 ioapic_resume(struct pic *pic, bool suspend_cancelled) 586 { 587 struct ioapic *io = (struct ioapic *)pic; 588 int i; 589 590 mtx_lock_spin(&icu_lock); 591 for (i = 0; i < io->io_numintr; i++) 592 ioapic_program_intpin(&io->io_pins[i]); 593 mtx_unlock_spin(&icu_lock); 594 } 595 596 /* 597 * Create a plain I/O APIC object. 598 */ 599 void * 600 ioapic_create(vm_paddr_t addr, int32_t apic_id, int intbase) 601 { 602 struct ioapic *io; 603 struct ioapic_intsrc *intpin; 604 volatile ioapic_t *apic; 605 u_int numintr, i; 606 uint32_t value; 607 608 /* Map the register window so we can access the device. */ 609 apic = pmap_mapdev(addr, IOAPIC_MEM_REGION); 610 mtx_lock_spin(&icu_lock); 611 value = ioapic_read(apic, IOAPIC_VER); 612 mtx_unlock_spin(&icu_lock); 613 614 /* If it's version register doesn't seem to work, punt. */ 615 if (value == 0xffffffff) { 616 pmap_unmapdev((vm_offset_t)apic, IOAPIC_MEM_REGION); 617 return (NULL); 618 } 619 620 /* Determine the number of vectors and set the APIC ID. */ 621 numintr = ((value & IOART_VER_MAXREDIR) >> MAXREDIRSHIFT) + 1; 622 io = malloc(sizeof(struct ioapic) + 623 numintr * sizeof(struct ioapic_intsrc), M_IOAPIC, M_WAITOK); 624 io->io_pic = ioapic_template; 625 mtx_lock_spin(&icu_lock); 626 io->io_id = next_id++; 627 io->io_apic_id = ioapic_read(apic, IOAPIC_ID) >> APIC_ID_SHIFT; 628 if (apic_id != -1 && io->io_apic_id != apic_id) { 629 ioapic_write(apic, IOAPIC_ID, apic_id << APIC_ID_SHIFT); 630 mtx_unlock_spin(&icu_lock); 631 io->io_apic_id = apic_id; 632 printf("ioapic%u: Changing APIC ID to %d\n", io->io_id, 633 apic_id); 634 } else 635 mtx_unlock_spin(&icu_lock); 636 if (intbase == -1) { 637 intbase = next_ioapic_base; 638 printf("ioapic%u: Assuming intbase of %d\n", io->io_id, 639 intbase); 640 } else if (intbase != next_ioapic_base && bootverbose) 641 printf("ioapic%u: WARNING: intbase %d != expected base %d\n", 642 io->io_id, intbase, next_ioapic_base); 643 io->io_intbase = intbase; 644 next_ioapic_base = intbase + numintr; 645 io->io_numintr = numintr; 646 io->io_addr = apic; 647 io->io_paddr = addr; 648 649 if (bootverbose) { 650 printf("ioapic%u: ver 0x%02x maxredir 0x%02x\n", io->io_id, 651 (value & IOART_VER_VERSION), (value & IOART_VER_MAXREDIR) 652 >> MAXREDIRSHIFT); 653 } 654 /* 655 * The summary information about IO-APIC versions is taken from 656 * the Linux kernel source: 657 * 0Xh 82489DX 658 * 1Xh I/OAPIC or I/O(x)APIC which are not PCI 2.2 Compliant 659 * 2Xh I/O(x)APIC which is PCI 2.2 Compliant 660 * 30h-FFh Reserved 661 * IO-APICs with version >= 0x20 have working EOIR register. 662 */ 663 io->io_haseoi = (value & IOART_VER_VERSION) >= 0x20; 664 665 /* 666 * Initialize pins. Start off with interrupts disabled. Default 667 * to active-hi and edge-triggered for ISA interrupts and active-lo 668 * and level-triggered for all others. 669 */ 670 bzero(io->io_pins, sizeof(struct ioapic_intsrc) * numintr); 671 mtx_lock_spin(&icu_lock); 672 for (i = 0, intpin = io->io_pins; i < numintr; i++, intpin++) { 673 intpin->io_intsrc.is_pic = (struct pic *)io; 674 intpin->io_intpin = i; 675 intpin->io_irq = intbase + i; 676 677 /* 678 * Assume that pin 0 on the first I/O APIC is an ExtINT pin. 679 * Assume that pins 1-15 are ISA interrupts and that all 680 * other pins are PCI interrupts. 681 */ 682 if (intpin->io_irq == 0) 683 ioapic_set_extint(io, i); 684 else if (intpin->io_irq < IOAPIC_ISA_INTS) { 685 intpin->io_bus = APIC_BUS_ISA; 686 intpin->io_activehi = 1; 687 intpin->io_edgetrigger = 1; 688 intpin->io_masked = 1; 689 } else { 690 intpin->io_bus = APIC_BUS_PCI; 691 intpin->io_activehi = 0; 692 intpin->io_edgetrigger = 0; 693 intpin->io_masked = 1; 694 } 695 696 /* 697 * Route interrupts to the BSP by default. Interrupts may 698 * be routed to other CPUs later after they are enabled. 699 */ 700 intpin->io_cpu = PCPU_GET(apic_id); 701 value = ioapic_read(apic, IOAPIC_REDTBL_LO(i)); 702 ioapic_write(apic, IOAPIC_REDTBL_LO(i), value | IOART_INTMSET); 703 #ifdef ACPI_DMAR 704 /* dummy, but sets cookie */ 705 mtx_unlock_spin(&icu_lock); 706 iommu_map_ioapic_intr(io->io_apic_id, 707 intpin->io_cpu, intpin->io_vector, intpin->io_edgetrigger, 708 intpin->io_activehi, intpin->io_irq, 709 &intpin->io_remap_cookie, NULL, NULL); 710 mtx_lock_spin(&icu_lock); 711 #endif 712 } 713 mtx_unlock_spin(&icu_lock); 714 715 return (io); 716 } 717 718 int 719 ioapic_get_vector(void *cookie, u_int pin) 720 { 721 struct ioapic *io; 722 723 io = (struct ioapic *)cookie; 724 if (pin >= io->io_numintr) 725 return (-1); 726 return (io->io_pins[pin].io_irq); 727 } 728 729 int 730 ioapic_disable_pin(void *cookie, u_int pin) 731 { 732 struct ioapic *io; 733 734 io = (struct ioapic *)cookie; 735 if (pin >= io->io_numintr) 736 return (EINVAL); 737 if (io->io_pins[pin].io_irq == IRQ_DISABLED) 738 return (EINVAL); 739 io->io_pins[pin].io_irq = IRQ_DISABLED; 740 if (bootverbose) 741 printf("ioapic%u: intpin %d disabled\n", io->io_id, pin); 742 return (0); 743 } 744 745 int 746 ioapic_remap_vector(void *cookie, u_int pin, int vector) 747 { 748 struct ioapic *io; 749 750 io = (struct ioapic *)cookie; 751 if (pin >= io->io_numintr || vector < 0) 752 return (EINVAL); 753 if (io->io_pins[pin].io_irq >= NUM_IO_INTS) 754 return (EINVAL); 755 io->io_pins[pin].io_irq = vector; 756 if (bootverbose) 757 printf("ioapic%u: Routing IRQ %d -> intpin %d\n", io->io_id, 758 vector, pin); 759 return (0); 760 } 761 762 int 763 ioapic_set_bus(void *cookie, u_int pin, int bus_type) 764 { 765 struct ioapic *io; 766 767 if (bus_type < 0 || bus_type > APIC_BUS_MAX) 768 return (EINVAL); 769 io = (struct ioapic *)cookie; 770 if (pin >= io->io_numintr) 771 return (EINVAL); 772 if (io->io_pins[pin].io_irq >= NUM_IO_INTS) 773 return (EINVAL); 774 if (io->io_pins[pin].io_bus == bus_type) 775 return (0); 776 io->io_pins[pin].io_bus = bus_type; 777 if (bootverbose) 778 printf("ioapic%u: intpin %d bus %s\n", io->io_id, pin, 779 ioapic_bus_string(bus_type)); 780 return (0); 781 } 782 783 int 784 ioapic_set_nmi(void *cookie, u_int pin) 785 { 786 struct ioapic *io; 787 788 io = (struct ioapic *)cookie; 789 if (pin >= io->io_numintr) 790 return (EINVAL); 791 if (io->io_pins[pin].io_irq == IRQ_NMI) 792 return (0); 793 if (io->io_pins[pin].io_irq >= NUM_IO_INTS) 794 return (EINVAL); 795 io->io_pins[pin].io_bus = APIC_BUS_UNKNOWN; 796 io->io_pins[pin].io_irq = IRQ_NMI; 797 io->io_pins[pin].io_masked = 0; 798 io->io_pins[pin].io_edgetrigger = 1; 799 io->io_pins[pin].io_activehi = 1; 800 if (bootverbose) 801 printf("ioapic%u: Routing NMI -> intpin %d\n", 802 io->io_id, pin); 803 return (0); 804 } 805 806 int 807 ioapic_set_smi(void *cookie, u_int pin) 808 { 809 struct ioapic *io; 810 811 io = (struct ioapic *)cookie; 812 if (pin >= io->io_numintr) 813 return (EINVAL); 814 if (io->io_pins[pin].io_irq == IRQ_SMI) 815 return (0); 816 if (io->io_pins[pin].io_irq >= NUM_IO_INTS) 817 return (EINVAL); 818 io->io_pins[pin].io_bus = APIC_BUS_UNKNOWN; 819 io->io_pins[pin].io_irq = IRQ_SMI; 820 io->io_pins[pin].io_masked = 0; 821 io->io_pins[pin].io_edgetrigger = 1; 822 io->io_pins[pin].io_activehi = 1; 823 if (bootverbose) 824 printf("ioapic%u: Routing SMI -> intpin %d\n", 825 io->io_id, pin); 826 return (0); 827 } 828 829 int 830 ioapic_set_extint(void *cookie, u_int pin) 831 { 832 struct ioapic *io; 833 834 io = (struct ioapic *)cookie; 835 if (pin >= io->io_numintr) 836 return (EINVAL); 837 if (io->io_pins[pin].io_irq == IRQ_EXTINT) 838 return (0); 839 if (io->io_pins[pin].io_irq >= NUM_IO_INTS) 840 return (EINVAL); 841 io->io_pins[pin].io_bus = APIC_BUS_UNKNOWN; 842 io->io_pins[pin].io_irq = IRQ_EXTINT; 843 if (enable_extint) 844 io->io_pins[pin].io_masked = 0; 845 else 846 io->io_pins[pin].io_masked = 1; 847 io->io_pins[pin].io_edgetrigger = 1; 848 io->io_pins[pin].io_activehi = 1; 849 if (bootverbose) 850 printf("ioapic%u: Routing external 8259A's -> intpin %d\n", 851 io->io_id, pin); 852 return (0); 853 } 854 855 int 856 ioapic_set_polarity(void *cookie, u_int pin, enum intr_polarity pol) 857 { 858 struct ioapic *io; 859 int activehi; 860 861 io = (struct ioapic *)cookie; 862 if (pin >= io->io_numintr || pol == INTR_POLARITY_CONFORM) 863 return (EINVAL); 864 if (io->io_pins[pin].io_irq >= NUM_IO_INTS) 865 return (EINVAL); 866 activehi = (pol == INTR_POLARITY_HIGH); 867 if (io->io_pins[pin].io_activehi == activehi) 868 return (0); 869 io->io_pins[pin].io_activehi = activehi; 870 if (bootverbose) 871 printf("ioapic%u: intpin %d polarity: %s\n", io->io_id, pin, 872 pol == INTR_POLARITY_HIGH ? "high" : "low"); 873 return (0); 874 } 875 876 int 877 ioapic_set_triggermode(void *cookie, u_int pin, enum intr_trigger trigger) 878 { 879 struct ioapic *io; 880 int edgetrigger; 881 882 io = (struct ioapic *)cookie; 883 if (pin >= io->io_numintr || trigger == INTR_TRIGGER_CONFORM) 884 return (EINVAL); 885 if (io->io_pins[pin].io_irq >= NUM_IO_INTS) 886 return (EINVAL); 887 edgetrigger = (trigger == INTR_TRIGGER_EDGE); 888 if (io->io_pins[pin].io_edgetrigger == edgetrigger) 889 return (0); 890 io->io_pins[pin].io_edgetrigger = edgetrigger; 891 if (bootverbose) 892 printf("ioapic%u: intpin %d trigger: %s\n", io->io_id, pin, 893 trigger == INTR_TRIGGER_EDGE ? "edge" : "level"); 894 return (0); 895 } 896 897 /* 898 * Register a complete I/O APIC object with the interrupt subsystem. 899 */ 900 void 901 ioapic_register(void *cookie) 902 { 903 struct ioapic_intsrc *pin; 904 struct ioapic *io; 905 volatile ioapic_t *apic; 906 uint32_t flags; 907 int i; 908 909 io = (struct ioapic *)cookie; 910 apic = io->io_addr; 911 mtx_lock_spin(&icu_lock); 912 flags = ioapic_read(apic, IOAPIC_VER) & IOART_VER_VERSION; 913 STAILQ_INSERT_TAIL(&ioapic_list, io, io_next); 914 mtx_unlock_spin(&icu_lock); 915 printf("ioapic%u <Version %u.%u> irqs %u-%u on motherboard\n", 916 io->io_id, flags >> 4, flags & 0xf, io->io_intbase, 917 io->io_intbase + io->io_numintr - 1); 918 919 /* 920 * Reprogram pins to handle special case pins (such as NMI and 921 * SMI) and register valid pins as interrupt sources. 922 */ 923 intr_register_pic(&io->io_pic); 924 for (i = 0, pin = io->io_pins; i < io->io_numintr; i++, pin++) { 925 ioapic_reprogram_intpin(&pin->io_intsrc); 926 if (pin->io_irq < NUM_IO_INTS) 927 intr_register_source(&pin->io_intsrc); 928 } 929 } 930 931 /* A simple new-bus driver to consume PCI I/O APIC devices. */ 932 static int 933 ioapic_pci_probe(device_t dev) 934 { 935 936 if (pci_get_class(dev) == PCIC_BASEPERIPH && 937 pci_get_subclass(dev) == PCIS_BASEPERIPH_PIC) { 938 switch (pci_get_progif(dev)) { 939 case PCIP_BASEPERIPH_PIC_IO_APIC: 940 device_set_desc(dev, "IO APIC"); 941 break; 942 case PCIP_BASEPERIPH_PIC_IOX_APIC: 943 device_set_desc(dev, "IO(x) APIC"); 944 break; 945 default: 946 return (ENXIO); 947 } 948 device_quiet(dev); 949 return (-10000); 950 } 951 return (ENXIO); 952 } 953 954 static int 955 ioapic_pci_attach(device_t dev) 956 { 957 958 return (0); 959 } 960 961 static device_method_t ioapic_pci_methods[] = { 962 /* Device interface */ 963 DEVMETHOD(device_probe, ioapic_pci_probe), 964 DEVMETHOD(device_attach, ioapic_pci_attach), 965 966 { 0, 0 } 967 }; 968 969 DEFINE_CLASS_0(ioapic, ioapic_pci_driver, ioapic_pci_methods, 0); 970 971 static devclass_t ioapic_devclass; 972 DRIVER_MODULE(ioapic, pci, ioapic_pci_driver, ioapic_devclass, 0, 0); 973 974 /* 975 * A new-bus driver to consume the memory resources associated with 976 * the APICs in the system. On some systems ACPI or PnPBIOS system 977 * resource devices may already claim these resources. To keep from 978 * breaking those devices, we attach ourself to the nexus device after 979 * legacy0 and acpi0 and ignore any allocation failures. 980 */ 981 static void 982 apic_identify(driver_t *driver, device_t parent) 983 { 984 985 /* 986 * Add at order 12. acpi0 is probed at order 10 and legacy0 987 * is probed at order 11. 988 */ 989 if (lapic_paddr != 0) 990 BUS_ADD_CHILD(parent, 12, "apic", 0); 991 } 992 993 static int 994 apic_probe(device_t dev) 995 { 996 997 device_set_desc(dev, "APIC resources"); 998 device_quiet(dev); 999 return (0); 1000 } 1001 1002 static void 1003 apic_add_resource(device_t dev, int rid, vm_paddr_t base, size_t length) 1004 { 1005 int error; 1006 1007 error = bus_set_resource(dev, SYS_RES_MEMORY, rid, base, length); 1008 if (error) 1009 panic("apic_add_resource: resource %d failed set with %d", rid, 1010 error); 1011 bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 0); 1012 } 1013 1014 static int 1015 apic_attach(device_t dev) 1016 { 1017 struct ioapic *io; 1018 int i; 1019 1020 /* Reserve the local APIC. */ 1021 apic_add_resource(dev, 0, lapic_paddr, LAPIC_MEM_REGION); 1022 i = 1; 1023 STAILQ_FOREACH(io, &ioapic_list, io_next) { 1024 apic_add_resource(dev, i, io->io_paddr, IOAPIC_MEM_REGION); 1025 i++; 1026 } 1027 return (0); 1028 } 1029 1030 static device_method_t apic_methods[] = { 1031 /* Device interface */ 1032 DEVMETHOD(device_identify, apic_identify), 1033 DEVMETHOD(device_probe, apic_probe), 1034 DEVMETHOD(device_attach, apic_attach), 1035 1036 { 0, 0 } 1037 }; 1038 1039 DEFINE_CLASS_0(apic, apic_driver, apic_methods, 0); 1040 1041 static devclass_t apic_devclass; 1042 DRIVER_MODULE(apic, nexus, apic_driver, apic_devclass, 0, 0); 1043 1044 #include "opt_ddb.h" 1045 1046 #ifdef DDB 1047 #include <ddb/ddb.h> 1048 1049 static const char * 1050 ioapic_delivery_mode(uint32_t mode) 1051 { 1052 1053 switch (mode) { 1054 case IOART_DELFIXED: 1055 return ("fixed"); 1056 case IOART_DELLOPRI: 1057 return ("lowestpri"); 1058 case IOART_DELSMI: 1059 return ("SMI"); 1060 case IOART_DELRSV1: 1061 return ("rsrvd1"); 1062 case IOART_DELNMI: 1063 return ("NMI"); 1064 case IOART_DELINIT: 1065 return ("INIT"); 1066 case IOART_DELRSV2: 1067 return ("rsrvd2"); 1068 case IOART_DELEXINT: 1069 return ("ExtINT"); 1070 default: 1071 return (""); 1072 } 1073 } 1074 1075 static u_int 1076 db_ioapic_read(volatile ioapic_t *apic, int reg) 1077 { 1078 1079 apic->ioregsel = reg; 1080 return (apic->iowin); 1081 } 1082 1083 static void 1084 db_show_ioapic_one(volatile ioapic_t *io_addr) 1085 { 1086 uint32_t r, lo, hi; 1087 int mre, i; 1088 1089 r = db_ioapic_read(io_addr, IOAPIC_VER); 1090 mre = (r & IOART_VER_MAXREDIR) >> MAXREDIRSHIFT; 1091 db_printf("Id 0x%08x Ver 0x%02x MRE %d\n", 1092 db_ioapic_read(io_addr, IOAPIC_ID), r & IOART_VER_VERSION, mre); 1093 for (i = 0; i < mre; i++) { 1094 lo = db_ioapic_read(io_addr, IOAPIC_REDTBL_LO(i)); 1095 hi = db_ioapic_read(io_addr, IOAPIC_REDTBL_HI(i)); 1096 db_printf(" pin %d Dest %s/%x %smasked Trig %s RemoteIRR %d " 1097 "Polarity %s Status %s DeliveryMode %s Vec %d\n", i, 1098 (lo & IOART_DESTMOD) == IOART_DESTLOG ? "log" : "phy", 1099 (hi & IOART_DEST) >> 24, 1100 (lo & IOART_INTMASK) == IOART_INTMSET ? "" : "not", 1101 (lo & IOART_TRGRMOD) == IOART_TRGRLVL ? "lvl" : "edge", 1102 (lo & IOART_REM_IRR) == IOART_REM_IRR ? 1 : 0, 1103 (lo & IOART_INTPOL) == IOART_INTALO ? "low" : "high", 1104 (lo & IOART_DELIVS) == IOART_DELIVS ? "pend" : "idle", 1105 ioapic_delivery_mode(lo & IOART_DELMOD), 1106 (lo & IOART_INTVEC)); 1107 } 1108 } 1109 1110 DB_SHOW_COMMAND(ioapic, db_show_ioapic) 1111 { 1112 struct ioapic *ioapic; 1113 int idx, i; 1114 1115 if (!have_addr) { 1116 db_printf("usage: show ioapic index\n"); 1117 return; 1118 } 1119 1120 idx = (int)addr; 1121 i = 0; 1122 STAILQ_FOREACH(ioapic, &ioapic_list, io_next) { 1123 if (idx == i) { 1124 db_show_ioapic_one(ioapic->io_addr); 1125 break; 1126 } 1127 i++; 1128 } 1129 } 1130 1131 DB_SHOW_ALL_COMMAND(ioapics, db_show_all_ioapics) 1132 { 1133 struct ioapic *ioapic; 1134 1135 STAILQ_FOREACH(ioapic, &ioapic_list, io_next) 1136 db_show_ioapic_one(ioapic->io_addr); 1137 } 1138 #endif 1139