1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 2005,2008 The DragonFly Project. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The DragonFly Project 7 * by Matthew Dillon <dillon@backplane.com> 8 * 9 * This code is derived from software contributed to Berkeley by 10 * William Jolitz. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in 20 * the documentation and/or other materials provided with the 21 * distribution. 22 * 3. Neither the name of The DragonFly Project nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific, prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 29 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 30 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 31 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 32 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 33 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 34 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 35 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 36 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 */ 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/kernel.h> 43 #include <sys/machintr.h> 44 #include <sys/interrupt.h> 45 #include <sys/rman.h> 46 #include <sys/bus.h> 47 48 #include <machine/segments.h> 49 #include <machine/md_var.h> 50 #include <machine/intr_machdep.h> 51 #include <machine/globaldata.h> 52 #include <machine/smp.h> 53 #include <machine/msi_var.h> 54 55 #include <sys/thread2.h> 56 57 #include <machine_base/icu/elcr_var.h> 58 59 #include <machine_base/icu/icu.h> 60 #include <machine_base/icu/icu_ipl.h> 61 #include <machine_base/apic/ioapic.h> 62 63 extern inthand_t 64 IDTVEC(icu_intr0), IDTVEC(icu_intr1), 65 IDTVEC(icu_intr2), IDTVEC(icu_intr3), 66 IDTVEC(icu_intr4), IDTVEC(icu_intr5), 67 IDTVEC(icu_intr6), IDTVEC(icu_intr7), 68 IDTVEC(icu_intr8), IDTVEC(icu_intr9), 69 IDTVEC(icu_intr10), IDTVEC(icu_intr11), 70 IDTVEC(icu_intr12), IDTVEC(icu_intr13), 71 IDTVEC(icu_intr14), IDTVEC(icu_intr15); 72 73 static inthand_t *icu_intr[ICU_HWI_VECTORS] = { 74 &IDTVEC(icu_intr0), &IDTVEC(icu_intr1), 75 &IDTVEC(icu_intr2), &IDTVEC(icu_intr3), 76 &IDTVEC(icu_intr4), &IDTVEC(icu_intr5), 77 &IDTVEC(icu_intr6), &IDTVEC(icu_intr7), 78 &IDTVEC(icu_intr8), &IDTVEC(icu_intr9), 79 &IDTVEC(icu_intr10), &IDTVEC(icu_intr11), 80 &IDTVEC(icu_intr12), &IDTVEC(icu_intr13), 81 &IDTVEC(icu_intr14), &IDTVEC(icu_intr15) 82 }; 83 84 static struct icu_irqmap { 85 int im_type; /* ICU_IMT_ */ 86 enum intr_trigger im_trig; 87 int im_msi_base; 88 } icu_irqmaps[MAXCPU][IDT_HWI_VECTORS]; 89 90 static struct lwkt_token icu_irqmap_tok = 91 LWKT_TOKEN_INITIALIZER(icu_irqmap_token); 92 93 #define ICU_IMT_UNUSED 0 /* KEEP THIS */ 94 #define ICU_IMT_RESERVED 1 95 #define ICU_IMT_LEGACY 2 96 #define ICU_IMT_SYSCALL 3 97 #define ICU_IMT_MSI 4 98 #define ICU_IMT_MSIX 5 99 100 #define ICU_IMT_ISHWI(map) ((map)->im_type != ICU_IMT_RESERVED && \ 101 (map)->im_type != ICU_IMT_SYSCALL) 102 103 extern void ICU_INTREN(int); 104 extern void ICU_INTRDIS(int); 105 106 extern int imcr_present; 107 108 static void icu_abi_intr_enable(int); 109 static void icu_abi_intr_disable(int); 110 static void icu_abi_intr_setup(int, int); 111 static void icu_abi_intr_teardown(int); 112 113 static void icu_abi_legacy_intr_config(int, enum intr_trigger, 114 enum intr_polarity); 115 static int icu_abi_legacy_intr_cpuid(int); 116 117 static int icu_abi_msi_alloc(int [], int, int); 118 static void icu_abi_msi_release(const int [], int, int); 119 static void icu_abi_msi_map(int, uint64_t *, uint32_t *, int); 120 static int icu_abi_msix_alloc(int *, int); 121 static void icu_abi_msix_release(int, int); 122 123 static int icu_abi_msi_alloc_intern(int, const char *, 124 int [], int, int); 125 static void icu_abi_msi_release_intern(int, const char *, 126 const int [], int, int); 127 128 static void icu_abi_finalize(void); 129 static void icu_abi_cleanup(void); 130 static void icu_abi_setdefault(void); 131 static void icu_abi_stabilize(void); 132 static void icu_abi_initmap(void); 133 static void icu_abi_rman_setup(struct rman *); 134 135 struct machintr_abi MachIntrABI_ICU = { 136 MACHINTR_ICU, 137 .intr_disable = icu_abi_intr_disable, 138 .intr_enable = icu_abi_intr_enable, 139 .intr_setup = icu_abi_intr_setup, 140 .intr_teardown = icu_abi_intr_teardown, 141 142 .legacy_intr_config = icu_abi_legacy_intr_config, 143 .legacy_intr_cpuid = icu_abi_legacy_intr_cpuid, 144 145 .msi_alloc = icu_abi_msi_alloc, 146 .msi_release = icu_abi_msi_release, 147 .msi_map = icu_abi_msi_map, 148 .msix_alloc = icu_abi_msix_alloc, 149 .msix_release = icu_abi_msix_release, 150 151 .finalize = icu_abi_finalize, 152 .cleanup = icu_abi_cleanup, 153 .setdefault = icu_abi_setdefault, 154 .stabilize = icu_abi_stabilize, 155 .initmap = icu_abi_initmap, 156 .rman_setup = icu_abi_rman_setup 157 }; 158 159 static int icu_abi_msi_start; /* NOTE: for testing only */ 160 161 /* 162 * WARNING! SMP builds can use the ICU now so this code must be MP safe. 163 */ 164 165 static void 166 icu_abi_intr_enable(int irq) 167 { 168 const struct icu_irqmap *map; 169 170 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS, 171 ("icu enable, invalid irq %d", irq)); 172 173 map = &icu_irqmaps[mycpuid][irq]; 174 KASSERT(ICU_IMT_ISHWI(map), 175 ("icu enable, not hwi irq %d, type %d, cpu%d", 176 irq, map->im_type, mycpuid)); 177 if (map->im_type != ICU_IMT_LEGACY) 178 return; 179 180 ICU_INTREN(irq); 181 } 182 183 static void 184 icu_abi_intr_disable(int irq) 185 { 186 const struct icu_irqmap *map; 187 188 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS, 189 ("icu disable, invalid irq %d", irq)); 190 191 map = &icu_irqmaps[mycpuid][irq]; 192 KASSERT(ICU_IMT_ISHWI(map), 193 ("icu disable, not hwi irq %d, type %d, cpu%d", 194 irq, map->im_type, mycpuid)); 195 if (map->im_type != ICU_IMT_LEGACY) 196 return; 197 198 ICU_INTRDIS(irq); 199 } 200 201 /* 202 * Called before interrupts are physically enabled 203 */ 204 static void 205 icu_abi_stabilize(void) 206 { 207 int intr; 208 209 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) 210 ICU_INTRDIS(intr); 211 ICU_INTREN(ICU_IRQ_SLAVE); 212 } 213 214 /* 215 * Called after interrupts physically enabled but before the 216 * critical section is released. 217 */ 218 static void 219 icu_abi_cleanup(void) 220 { 221 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 222 } 223 224 /* 225 * Called after stablize and cleanup; critical section is not 226 * held and interrupts are not physically disabled. 227 */ 228 static void 229 icu_abi_finalize(void) 230 { 231 KKASSERT(MachIntrABI.type == MACHINTR_ICU); 232 KKASSERT(!ioapic_enable); 233 234 /* 235 * If an IMCR is present, programming bit 0 disconnects the 8259 236 * from the BSP. The 8259 may still be connected to LINT0 on the 237 * BSP's LAPIC. 238 * 239 * If we are running SMP the LAPIC is active, try to use virtual 240 * wire mode so we can use other interrupt sources within the LAPIC 241 * in addition to the 8259. 242 */ 243 if (imcr_present) { 244 outb(0x22, 0x70); 245 outb(0x23, 0x01); 246 } 247 } 248 249 static void 250 icu_abi_intr_setup(int intr, int flags) 251 { 252 const struct icu_irqmap *map; 253 register_t ef; 254 255 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 256 ("icu setup, invalid irq %d", intr)); 257 258 map = &icu_irqmaps[mycpuid][intr]; 259 KASSERT(ICU_IMT_ISHWI(map), 260 ("icu setup, not hwi irq %d, type %d, cpu%d", 261 intr, map->im_type, mycpuid)); 262 if (map->im_type != ICU_IMT_LEGACY) 263 return; 264 265 ef = read_rflags(); 266 cpu_disable_intr(); 267 268 ICU_INTREN(intr); 269 270 write_rflags(ef); 271 } 272 273 static void 274 icu_abi_intr_teardown(int intr) 275 { 276 const struct icu_irqmap *map; 277 register_t ef; 278 279 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 280 ("icu teardown, invalid irq %d", intr)); 281 282 map = &icu_irqmaps[mycpuid][intr]; 283 KASSERT(ICU_IMT_ISHWI(map), 284 ("icu teardown, not hwi irq %d, type %d, cpu%d", 285 intr, map->im_type, mycpuid)); 286 if (map->im_type != ICU_IMT_LEGACY) 287 return; 288 289 ef = read_rflags(); 290 cpu_disable_intr(); 291 292 ICU_INTRDIS(intr); 293 294 write_rflags(ef); 295 } 296 297 static void 298 icu_abi_setdefault(void) 299 { 300 int intr; 301 302 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) { 303 if (intr == ICU_IRQ_SLAVE) 304 continue; 305 setidt_global(IDT_OFFSET + intr, icu_intr[intr], 306 SDT_SYSIGT, SEL_KPL, 0); 307 } 308 } 309 310 static void 311 icu_abi_initmap(void) 312 { 313 int cpu; 314 315 kgetenv_int("hw.icu.msi_start", &icu_abi_msi_start); 316 icu_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 317 318 /* 319 * NOTE: ncpus is not ready yet 320 */ 321 for (cpu = 0; cpu < MAXCPU; ++cpu) { 322 int i; 323 324 if (cpu != 0) { 325 for (i = 0; i < ICU_HWI_VECTORS; ++i) 326 icu_irqmaps[cpu][i].im_type = ICU_IMT_RESERVED; 327 } else { 328 for (i = 0; i < ICU_HWI_VECTORS; ++i) 329 icu_irqmaps[cpu][i].im_type = ICU_IMT_LEGACY; 330 icu_irqmaps[cpu][ICU_IRQ_SLAVE].im_type = 331 ICU_IMT_RESERVED; 332 333 if (elcr_found) { 334 for (i = 0; i < ICU_HWI_VECTORS; ++i) { 335 icu_irqmaps[cpu][i].im_trig = 336 elcr_read_trigger(i); 337 } 338 } else { 339 /* 340 * NOTE: Trigger mode does not matter at all 341 */ 342 for (i = 0; i < ICU_HWI_VECTORS; ++i) { 343 icu_irqmaps[cpu][i].im_trig = 344 INTR_TRIGGER_EDGE; 345 } 346 } 347 } 348 349 for (i = 0; i < IDT_HWI_VECTORS; ++i) 350 icu_irqmaps[cpu][i].im_msi_base = -1; 351 352 icu_irqmaps[cpu][IDT_OFFSET_SYSCALL - IDT_OFFSET].im_type = 353 ICU_IMT_SYSCALL; 354 } 355 } 356 357 static void 358 icu_abi_legacy_intr_config(int irq, enum intr_trigger trig, 359 enum intr_polarity pola __unused) 360 { 361 struct icu_irqmap *map; 362 363 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 364 365 KKASSERT(irq >= 0 && irq < IDT_HWI_VECTORS); 366 map = &icu_irqmaps[0][irq]; 367 368 KKASSERT(map->im_type == ICU_IMT_LEGACY); 369 370 /* TODO: Check whether it is configured or not */ 371 372 if (trig == map->im_trig) 373 return; 374 375 if (bootverbose) { 376 kprintf("ICU: irq %d, %s -> %s\n", irq, 377 intr_str_trigger(map->im_trig), 378 intr_str_trigger(trig)); 379 } 380 map->im_trig = trig; 381 382 if (!elcr_found) { 383 if (bootverbose) 384 kprintf("ICU: no ELCR, skip irq %d config\n", irq); 385 return; 386 } 387 elcr_write_trigger(irq, map->im_trig); 388 } 389 390 static int 391 icu_abi_legacy_intr_cpuid(int irq __unused) 392 { 393 return 0; 394 } 395 396 static void 397 icu_abi_rman_setup(struct rman *rm) 398 { 399 int start, end, i; 400 401 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 402 ("invalid rman cpuid %d", rm->rm_cpuid)); 403 404 start = end = -1; 405 for (i = 0; i < IDT_HWI_VECTORS; ++i) { 406 const struct icu_irqmap *map = &icu_irqmaps[rm->rm_cpuid][i]; 407 408 if (start < 0) { 409 if (ICU_IMT_ISHWI(map)) 410 start = end = i; 411 } else { 412 if (ICU_IMT_ISHWI(map)) { 413 end = i; 414 } else { 415 KKASSERT(end >= 0); 416 if (bootverbose) { 417 kprintf("ICU: rman cpu%d %d - %d\n", 418 rm->rm_cpuid, start, end); 419 } 420 if (rman_manage_region(rm, start, end)) { 421 panic("rman_manage_region" 422 "(cpu%d %d - %d)", rm->rm_cpuid, 423 start, end); 424 } 425 start = end = -1; 426 } 427 } 428 } 429 if (start >= 0) { 430 KKASSERT(end >= 0); 431 if (bootverbose) { 432 kprintf("ICU: rman cpu%d %d - %d\n", 433 rm->rm_cpuid, start, end); 434 } 435 if (rman_manage_region(rm, start, end)) { 436 panic("rman_manage_region(cpu%d %d - %d)", 437 rm->rm_cpuid, start, end); 438 } 439 } 440 } 441 442 static int 443 icu_abi_msi_alloc_intern(int type, const char *desc, 444 int intrs[], int count, int cpuid) 445 { 446 int i, error; 447 448 KASSERT(cpuid >= 0 && cpuid < ncpus, 449 ("invalid cpuid %d", cpuid)); 450 451 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 452 KASSERT((count & (count - 1)) == 0, 453 ("count %d is not power of 2", count)); 454 455 lwkt_gettoken(&icu_irqmap_tok); 456 457 /* 458 * NOTE: 459 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 460 * we do not need to find out the first properly aligned 461 * interrupt vector. 462 */ 463 464 error = EMSGSIZE; 465 for (i = icu_abi_msi_start; i < IDT_HWI_VECTORS; i += count) { 466 int j; 467 468 if (icu_irqmaps[cpuid][i].im_type != ICU_IMT_UNUSED) 469 continue; 470 471 for (j = 1; j < count; ++j) { 472 if (icu_irqmaps[cpuid][i + j].im_type != ICU_IMT_UNUSED) 473 break; 474 } 475 if (j != count) 476 continue; 477 478 for (j = 0; j < count; ++j) { 479 struct icu_irqmap *map; 480 int intr = i + j; 481 482 map = &icu_irqmaps[cpuid][intr]; 483 KASSERT(map->im_msi_base < 0, 484 ("intr %d, stale %s-base %d", 485 intr, desc, map->im_msi_base)); 486 487 map->im_type = type; 488 map->im_msi_base = i; 489 490 intrs[j] = intr; 491 msi_setup(intr, cpuid); 492 493 if (bootverbose) { 494 kprintf("alloc %s intr %d on cpu%d\n", 495 desc, intr, cpuid); 496 } 497 } 498 error = 0; 499 break; 500 } 501 502 lwkt_reltoken(&icu_irqmap_tok); 503 504 return error; 505 } 506 507 static void 508 icu_abi_msi_release_intern(int type, const char *desc, 509 const int intrs[], int count, int cpuid) 510 { 511 int i, msi_base = -1, intr_next = -1, mask; 512 513 KASSERT(cpuid >= 0 && cpuid < ncpus, 514 ("invalid cpuid %d", cpuid)); 515 516 KASSERT(count > 0 && count <= 32, ("invalid count %d", count)); 517 518 mask = count - 1; 519 KASSERT((count & mask) == 0, ("count %d is not power of 2", count)); 520 521 lwkt_gettoken(&icu_irqmap_tok); 522 523 for (i = 0; i < count; ++i) { 524 struct icu_irqmap *map; 525 int intr = intrs[i]; 526 527 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 528 ("invalid intr %d", intr)); 529 530 map = &icu_irqmaps[cpuid][intr]; 531 KASSERT(map->im_type == type, 532 ("trying to release non-%s intr %d, type %d", desc, 533 intr, map->im_type)); 534 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 535 ("intr %d, invalid %s-base %d", intr, desc, 536 map->im_msi_base)); 537 KASSERT((map->im_msi_base & mask) == 0, 538 ("intr %d, %s-base %d is not properly aligned %d", 539 intr, desc, map->im_msi_base, count)); 540 541 if (msi_base < 0) { 542 msi_base = map->im_msi_base; 543 } else { 544 KASSERT(map->im_msi_base == msi_base, 545 ("intr %d, inconsistent %s-base, " 546 "was %d, now %d", 547 intr, desc, msi_base, map->im_msi_base)); 548 } 549 550 if (intr_next < intr) 551 intr_next = intr; 552 553 map->im_type = ICU_IMT_UNUSED; 554 map->im_msi_base = -1; 555 556 if (bootverbose) { 557 kprintf("release %s intr %d on cpu%d\n", 558 desc, intr, cpuid); 559 } 560 } 561 562 KKASSERT(intr_next > 0); 563 KKASSERT(msi_base >= 0); 564 565 ++intr_next; 566 if (intr_next < IDT_HWI_VECTORS) { 567 const struct icu_irqmap *map = &icu_irqmaps[cpuid][intr_next]; 568 569 if (map->im_type == type) { 570 KASSERT(map->im_msi_base != msi_base, 571 ("more than %d %s was allocated", count, desc)); 572 } 573 } 574 575 lwkt_reltoken(&icu_irqmap_tok); 576 } 577 578 static int 579 icu_abi_msi_alloc(int intrs[], int count, int cpuid) 580 { 581 return icu_abi_msi_alloc_intern(ICU_IMT_MSI, "MSI", 582 intrs, count, cpuid); 583 } 584 585 static void 586 icu_abi_msi_release(const int intrs[], int count, int cpuid) 587 { 588 icu_abi_msi_release_intern(ICU_IMT_MSI, "MSI", 589 intrs, count, cpuid); 590 } 591 592 static int 593 icu_abi_msix_alloc(int *intr, int cpuid) 594 { 595 return icu_abi_msi_alloc_intern(ICU_IMT_MSIX, "MSI-X", 596 intr, 1, cpuid); 597 } 598 599 static void 600 icu_abi_msix_release(int intr, int cpuid) 601 { 602 icu_abi_msi_release_intern(ICU_IMT_MSIX, "MXI-X", 603 &intr, 1, cpuid); 604 } 605 606 static void 607 icu_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 608 { 609 const struct icu_irqmap *map; 610 611 KASSERT(cpuid >= 0 && cpuid < ncpus, 612 ("invalid cpuid %d", cpuid)); 613 614 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 615 ("invalid intr %d", intr)); 616 617 lwkt_gettoken(&icu_irqmap_tok); 618 619 map = &icu_irqmaps[cpuid][intr]; 620 KASSERT(map->im_type == ICU_IMT_MSI || 621 map->im_type == ICU_IMT_MSIX, 622 ("trying to map non-MSI/MSI-X intr %d, type %d", intr, map->im_type)); 623 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 624 ("intr %d, invalid %s-base %d", intr, 625 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X", 626 map->im_msi_base)); 627 628 msi_map(map->im_msi_base, addr, data, cpuid); 629 630 if (bootverbose) { 631 kprintf("map %s intr %d on cpu%d\n", 632 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X", 633 intr, cpuid); 634 } 635 636 lwkt_reltoken(&icu_irqmap_tok); 637 } 638