1 /* 2 * Copyright (c) 1991 The Regents of the University of California. 3 * Copyright (c) 2005,2008 The DragonFly Project. 4 * All rights reserved. 5 * 6 * This code is derived from software contributed to The DragonFly Project 7 * by Matthew Dillon <dillon@backplane.com> 8 * 9 * This code is derived from software contributed to Berkeley by 10 * William Jolitz. 11 * 12 * Redistribution and use in source and binary forms, with or without 13 * modification, are permitted provided that the following conditions 14 * are met: 15 * 16 * 1. Redistributions of source code must retain the above copyright 17 * notice, this list of conditions and the following disclaimer. 18 * 2. Redistributions in binary form must reproduce the above copyright 19 * notice, this list of conditions and the following disclaimer in 20 * the documentation and/or other materials provided with the 21 * distribution. 22 * 3. Neither the name of The DragonFly Project nor the names of its 23 * contributors may be used to endorse or promote products derived 24 * from this software without specific, prior written permission. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 27 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 28 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS 29 * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE 30 * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, 31 * INCIDENTAL, SPECIAL, EXEMPLARY OR CONSEQUENTIAL DAMAGES (INCLUDING, 32 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 33 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 34 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 35 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT 36 * OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 37 * SUCH DAMAGE. 38 * 39 * $DragonFly: src/sys/platform/pc64/icu/icu_abi.c,v 1.1 2008/08/29 17:07:16 dillon Exp $ 40 */ 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/kernel.h> 45 #include <sys/machintr.h> 46 #include <sys/interrupt.h> 47 #include <sys/rman.h> 48 #include <sys/bus.h> 49 50 #include <machine/segments.h> 51 #include <machine/md_var.h> 52 #include <machine/intr_machdep.h> 53 #include <machine/globaldata.h> 54 #include <machine/smp.h> 55 #include <machine/msi_var.h> 56 57 #include <sys/thread2.h> 58 59 #include <machine_base/icu/elcr_var.h> 60 61 #include <machine_base/icu/icu.h> 62 #include <machine_base/icu/icu_ipl.h> 63 #include <machine_base/apic/ioapic.h> 64 65 extern inthand_t 66 IDTVEC(icu_intr0), IDTVEC(icu_intr1), 67 IDTVEC(icu_intr2), IDTVEC(icu_intr3), 68 IDTVEC(icu_intr4), IDTVEC(icu_intr5), 69 IDTVEC(icu_intr6), IDTVEC(icu_intr7), 70 IDTVEC(icu_intr8), IDTVEC(icu_intr9), 71 IDTVEC(icu_intr10), IDTVEC(icu_intr11), 72 IDTVEC(icu_intr12), IDTVEC(icu_intr13), 73 IDTVEC(icu_intr14), IDTVEC(icu_intr15); 74 75 static inthand_t *icu_intr[ICU_HWI_VECTORS] = { 76 &IDTVEC(icu_intr0), &IDTVEC(icu_intr1), 77 &IDTVEC(icu_intr2), &IDTVEC(icu_intr3), 78 &IDTVEC(icu_intr4), &IDTVEC(icu_intr5), 79 &IDTVEC(icu_intr6), &IDTVEC(icu_intr7), 80 &IDTVEC(icu_intr8), &IDTVEC(icu_intr9), 81 &IDTVEC(icu_intr10), &IDTVEC(icu_intr11), 82 &IDTVEC(icu_intr12), &IDTVEC(icu_intr13), 83 &IDTVEC(icu_intr14), &IDTVEC(icu_intr15) 84 }; 85 86 static struct icu_irqmap { 87 int im_type; /* ICU_IMT_ */ 88 enum intr_trigger im_trig; 89 int im_msi_base; 90 } icu_irqmaps[MAXCPU][IDT_HWI_VECTORS]; 91 92 static struct lwkt_token icu_irqmap_tok = 93 LWKT_TOKEN_INITIALIZER(icu_irqmap_token); 94 95 #define ICU_IMT_UNUSED 0 /* KEEP THIS */ 96 #define ICU_IMT_RESERVED 1 97 #define ICU_IMT_LEGACY 2 98 #define ICU_IMT_SYSCALL 3 99 #define ICU_IMT_MSI 4 100 #define ICU_IMT_MSIX 5 101 102 #define ICU_IMT_ISHWI(map) ((map)->im_type != ICU_IMT_RESERVED && \ 103 (map)->im_type != ICU_IMT_SYSCALL) 104 105 extern void ICU_INTREN(int); 106 extern void ICU_INTRDIS(int); 107 108 extern int imcr_present; 109 110 static void icu_abi_intr_enable(int); 111 static void icu_abi_intr_disable(int); 112 static void icu_abi_intr_setup(int, int); 113 static void icu_abi_intr_teardown(int); 114 115 static void icu_abi_legacy_intr_config(int, enum intr_trigger, 116 enum intr_polarity); 117 static int icu_abi_legacy_intr_cpuid(int); 118 119 static int icu_abi_msi_alloc(int [], int, int); 120 static void icu_abi_msi_release(const int [], int, int); 121 static void icu_abi_msi_map(int, uint64_t *, uint32_t *, int); 122 static int icu_abi_msix_alloc(int *, int); 123 static void icu_abi_msix_release(int, int); 124 125 static int icu_abi_msi_alloc_intern(int, const char *, 126 int [], int, int); 127 static void icu_abi_msi_release_intern(int, const char *, 128 const int [], int, int); 129 130 static void icu_abi_finalize(void); 131 static void icu_abi_cleanup(void); 132 static void icu_abi_setdefault(void); 133 static void icu_abi_stabilize(void); 134 static void icu_abi_initmap(void); 135 static void icu_abi_rman_setup(struct rman *); 136 137 struct machintr_abi MachIntrABI_ICU = { 138 MACHINTR_ICU, 139 .intr_disable = icu_abi_intr_disable, 140 .intr_enable = icu_abi_intr_enable, 141 .intr_setup = icu_abi_intr_setup, 142 .intr_teardown = icu_abi_intr_teardown, 143 144 .legacy_intr_config = icu_abi_legacy_intr_config, 145 .legacy_intr_cpuid = icu_abi_legacy_intr_cpuid, 146 147 .msi_alloc = icu_abi_msi_alloc, 148 .msi_release = icu_abi_msi_release, 149 .msi_map = icu_abi_msi_map, 150 .msix_alloc = icu_abi_msix_alloc, 151 .msix_release = icu_abi_msix_release, 152 153 .finalize = icu_abi_finalize, 154 .cleanup = icu_abi_cleanup, 155 .setdefault = icu_abi_setdefault, 156 .stabilize = icu_abi_stabilize, 157 .initmap = icu_abi_initmap, 158 .rman_setup = icu_abi_rman_setup 159 }; 160 161 static int icu_abi_msi_start; /* NOTE: for testing only */ 162 163 /* 164 * WARNING! SMP builds can use the ICU now so this code must be MP safe. 165 */ 166 167 static void 168 icu_abi_intr_enable(int irq) 169 { 170 const struct icu_irqmap *map; 171 172 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS, 173 ("icu enable, invalid irq %d\n", irq)); 174 175 map = &icu_irqmaps[mycpuid][irq]; 176 KASSERT(ICU_IMT_ISHWI(map), 177 ("icu enable, not hwi irq %d, type %d, cpu%d\n", 178 irq, map->im_type, mycpuid)); 179 if (map->im_type != ICU_IMT_LEGACY) 180 return; 181 182 ICU_INTREN(irq); 183 } 184 185 static void 186 icu_abi_intr_disable(int irq) 187 { 188 const struct icu_irqmap *map; 189 190 KASSERT(irq >= 0 && irq < IDT_HWI_VECTORS, 191 ("icu disable, invalid irq %d\n", irq)); 192 193 map = &icu_irqmaps[mycpuid][irq]; 194 KASSERT(ICU_IMT_ISHWI(map), 195 ("icu disable, not hwi irq %d, type %d, cpu%d\n", 196 irq, map->im_type, mycpuid)); 197 if (map->im_type != ICU_IMT_LEGACY) 198 return; 199 200 ICU_INTRDIS(irq); 201 } 202 203 /* 204 * Called before interrupts are physically enabled 205 */ 206 static void 207 icu_abi_stabilize(void) 208 { 209 int intr; 210 211 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) 212 ICU_INTRDIS(intr); 213 ICU_INTREN(ICU_IRQ_SLAVE); 214 } 215 216 /* 217 * Called after interrupts physically enabled but before the 218 * critical section is released. 219 */ 220 static void 221 icu_abi_cleanup(void) 222 { 223 bzero(mdcpu->gd_ipending, sizeof(mdcpu->gd_ipending)); 224 } 225 226 /* 227 * Called after stablize and cleanup; critical section is not 228 * held and interrupts are not physically disabled. 229 */ 230 static void 231 icu_abi_finalize(void) 232 { 233 KKASSERT(MachIntrABI.type == MACHINTR_ICU); 234 KKASSERT(!ioapic_enable); 235 236 /* 237 * If an IMCR is present, programming bit 0 disconnects the 8259 238 * from the BSP. The 8259 may still be connected to LINT0 on the 239 * BSP's LAPIC. 240 * 241 * If we are running SMP the LAPIC is active, try to use virtual 242 * wire mode so we can use other interrupt sources within the LAPIC 243 * in addition to the 8259. 244 */ 245 if (imcr_present) { 246 outb(0x22, 0x70); 247 outb(0x23, 0x01); 248 } 249 } 250 251 static void 252 icu_abi_intr_setup(int intr, int flags) 253 { 254 const struct icu_irqmap *map; 255 register_t ef; 256 257 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 258 ("icu setup, invalid irq %d\n", intr)); 259 260 map = &icu_irqmaps[mycpuid][intr]; 261 KASSERT(ICU_IMT_ISHWI(map), 262 ("icu setup, not hwi irq %d, type %d, cpu%d\n", 263 intr, map->im_type, mycpuid)); 264 if (map->im_type != ICU_IMT_LEGACY) 265 return; 266 267 ef = read_rflags(); 268 cpu_disable_intr(); 269 270 ICU_INTREN(intr); 271 272 write_rflags(ef); 273 } 274 275 static void 276 icu_abi_intr_teardown(int intr) 277 { 278 const struct icu_irqmap *map; 279 register_t ef; 280 281 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 282 ("icu teardown, invalid irq %d\n", intr)); 283 284 map = &icu_irqmaps[mycpuid][intr]; 285 KASSERT(ICU_IMT_ISHWI(map), 286 ("icu teardown, not hwi irq %d, type %d, cpu%d\n", 287 intr, map->im_type, mycpuid)); 288 if (map->im_type != ICU_IMT_LEGACY) 289 return; 290 291 ef = read_rflags(); 292 cpu_disable_intr(); 293 294 ICU_INTRDIS(intr); 295 296 write_rflags(ef); 297 } 298 299 static void 300 icu_abi_setdefault(void) 301 { 302 int intr; 303 304 for (intr = 0; intr < ICU_HWI_VECTORS; ++intr) { 305 if (intr == ICU_IRQ_SLAVE) 306 continue; 307 setidt_global(IDT_OFFSET + intr, icu_intr[intr], 308 SDT_SYSIGT, SEL_KPL, 0); 309 } 310 } 311 312 static void 313 icu_abi_initmap(void) 314 { 315 int cpu; 316 317 kgetenv_int("hw.icu.msi_start", &icu_abi_msi_start); 318 icu_abi_msi_start &= ~0x1f; /* MUST be 32 aligned */ 319 320 /* 321 * NOTE: ncpus is not ready yet 322 */ 323 for (cpu = 0; cpu < MAXCPU; ++cpu) { 324 int i; 325 326 if (cpu != 0) { 327 for (i = 0; i < ICU_HWI_VECTORS; ++i) 328 icu_irqmaps[cpu][i].im_type = ICU_IMT_RESERVED; 329 } else { 330 for (i = 0; i < ICU_HWI_VECTORS; ++i) 331 icu_irqmaps[cpu][i].im_type = ICU_IMT_LEGACY; 332 icu_irqmaps[cpu][ICU_IRQ_SLAVE].im_type = 333 ICU_IMT_RESERVED; 334 335 if (elcr_found) { 336 for (i = 0; i < ICU_HWI_VECTORS; ++i) { 337 icu_irqmaps[cpu][i].im_trig = 338 elcr_read_trigger(i); 339 } 340 } else { 341 /* 342 * NOTE: Trigger mode does not matter at all 343 */ 344 for (i = 0; i < ICU_HWI_VECTORS; ++i) { 345 icu_irqmaps[cpu][i].im_trig = 346 INTR_TRIGGER_EDGE; 347 } 348 } 349 } 350 351 for (i = 0; i < IDT_HWI_VECTORS; ++i) 352 icu_irqmaps[cpu][i].im_msi_base = -1; 353 354 icu_irqmaps[cpu][IDT_OFFSET_SYSCALL - IDT_OFFSET].im_type = 355 ICU_IMT_SYSCALL; 356 } 357 } 358 359 static void 360 icu_abi_legacy_intr_config(int irq, enum intr_trigger trig, 361 enum intr_polarity pola __unused) 362 { 363 struct icu_irqmap *map; 364 365 KKASSERT(trig == INTR_TRIGGER_EDGE || trig == INTR_TRIGGER_LEVEL); 366 367 KKASSERT(irq >= 0 && irq < IDT_HWI_VECTORS); 368 map = &icu_irqmaps[0][irq]; 369 370 KKASSERT(map->im_type == ICU_IMT_LEGACY); 371 372 /* TODO: Check whether it is configured or not */ 373 374 if (trig == map->im_trig) 375 return; 376 377 if (bootverbose) { 378 kprintf("ICU: irq %d, %s -> %s\n", irq, 379 intr_str_trigger(map->im_trig), 380 intr_str_trigger(trig)); 381 } 382 map->im_trig = trig; 383 384 if (!elcr_found) { 385 if (bootverbose) 386 kprintf("ICU: no ELCR, skip irq %d config\n", irq); 387 return; 388 } 389 elcr_write_trigger(irq, map->im_trig); 390 } 391 392 static int 393 icu_abi_legacy_intr_cpuid(int irq __unused) 394 { 395 return 0; 396 } 397 398 static void 399 icu_abi_rman_setup(struct rman *rm) 400 { 401 int start, end, i; 402 403 KASSERT(rm->rm_cpuid >= 0 && rm->rm_cpuid < MAXCPU, 404 ("invalid rman cpuid %d", rm->rm_cpuid)); 405 406 start = end = -1; 407 for (i = 0; i < IDT_HWI_VECTORS; ++i) { 408 const struct icu_irqmap *map = &icu_irqmaps[rm->rm_cpuid][i]; 409 410 if (start < 0) { 411 if (ICU_IMT_ISHWI(map)) 412 start = end = i; 413 } else { 414 if (ICU_IMT_ISHWI(map)) { 415 end = i; 416 } else { 417 KKASSERT(end >= 0); 418 if (bootverbose) { 419 kprintf("ICU: rman cpu%d %d - %d\n", 420 rm->rm_cpuid, start, end); 421 } 422 if (rman_manage_region(rm, start, end)) { 423 panic("rman_manage_region" 424 "(cpu%d %d - %d)", rm->rm_cpuid, 425 start, end); 426 } 427 start = end = -1; 428 } 429 } 430 } 431 if (start >= 0) { 432 KKASSERT(end >= 0); 433 if (bootverbose) { 434 kprintf("ICU: rman cpu%d %d - %d\n", 435 rm->rm_cpuid, start, end); 436 } 437 if (rman_manage_region(rm, start, end)) { 438 panic("rman_manage_region(cpu%d %d - %d)", 439 rm->rm_cpuid, start, end); 440 } 441 } 442 } 443 444 static int 445 icu_abi_msi_alloc_intern(int type, const char *desc, 446 int intrs[], int count, int cpuid) 447 { 448 int i, error; 449 450 KASSERT(cpuid >= 0 && cpuid < ncpus, 451 ("invalid cpuid %d", cpuid)); 452 453 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count)); 454 KASSERT((count & (count - 1)) == 0, 455 ("count %d is not power of 2\n", count)); 456 457 lwkt_gettoken(&icu_irqmap_tok); 458 459 /* 460 * NOTE: 461 * Since IDT_OFFSET is 32, which is the maximum valid 'count', 462 * we do not need to find out the first properly aligned 463 * interrupt vector. 464 */ 465 466 error = EMSGSIZE; 467 for (i = icu_abi_msi_start; i < IDT_HWI_VECTORS; i += count) { 468 int j; 469 470 if (icu_irqmaps[cpuid][i].im_type != ICU_IMT_UNUSED) 471 continue; 472 473 for (j = 1; j < count; ++j) { 474 if (icu_irqmaps[cpuid][i + j].im_type != ICU_IMT_UNUSED) 475 break; 476 } 477 if (j != count) 478 continue; 479 480 for (j = 0; j < count; ++j) { 481 struct icu_irqmap *map; 482 int intr = i + j; 483 484 map = &icu_irqmaps[cpuid][intr]; 485 KASSERT(map->im_msi_base < 0, 486 ("intr %d, stale %s-base %d\n", 487 intr, desc, map->im_msi_base)); 488 489 map->im_type = type; 490 map->im_msi_base = i; 491 492 intrs[j] = intr; 493 msi_setup(intr, cpuid); 494 495 if (bootverbose) { 496 kprintf("alloc %s intr %d on cpu%d\n", 497 desc, intr, cpuid); 498 } 499 } 500 error = 0; 501 break; 502 } 503 504 lwkt_reltoken(&icu_irqmap_tok); 505 506 return error; 507 } 508 509 static void 510 icu_abi_msi_release_intern(int type, const char *desc, 511 const int intrs[], int count, int cpuid) 512 { 513 int i, msi_base = -1, intr_next = -1, mask; 514 515 KASSERT(cpuid >= 0 && cpuid < ncpus, 516 ("invalid cpuid %d", cpuid)); 517 518 KASSERT(count > 0 && count <= 32, ("invalid count %d\n", count)); 519 520 mask = count - 1; 521 KASSERT((count & mask) == 0, ("count %d is not power of 2\n", count)); 522 523 lwkt_gettoken(&icu_irqmap_tok); 524 525 for (i = 0; i < count; ++i) { 526 struct icu_irqmap *map; 527 int intr = intrs[i]; 528 529 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 530 ("invalid intr %d\n", intr)); 531 532 map = &icu_irqmaps[cpuid][intr]; 533 KASSERT(map->im_type == type, 534 ("try release non-%s intr %d, type %d\n", desc, 535 intr, map->im_type)); 536 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 537 ("intr %d, invalid %s-base %d\n", intr, desc, 538 map->im_msi_base)); 539 KASSERT((map->im_msi_base & mask) == 0, 540 ("intr %d, %s-base %d is not proper aligned %d\n", 541 intr, desc, map->im_msi_base, count)); 542 543 if (msi_base < 0) { 544 msi_base = map->im_msi_base; 545 } else { 546 KASSERT(map->im_msi_base == msi_base, 547 ("intr %d, inconsistent %s-base, " 548 "was %d, now %d\n", 549 intr, desc, msi_base, map->im_msi_base)); 550 } 551 552 if (intr_next < intr) 553 intr_next = intr; 554 555 map->im_type = ICU_IMT_UNUSED; 556 map->im_msi_base = -1; 557 558 if (bootverbose) { 559 kprintf("release %s intr %d on cpu%d\n", 560 desc, intr, cpuid); 561 } 562 } 563 564 KKASSERT(intr_next > 0); 565 KKASSERT(msi_base >= 0); 566 567 ++intr_next; 568 if (intr_next < IDT_HWI_VECTORS) { 569 const struct icu_irqmap *map = &icu_irqmaps[cpuid][intr_next]; 570 571 if (map->im_type == type) { 572 KASSERT(map->im_msi_base != msi_base, 573 ("more than %d %s was allocated\n", count, desc)); 574 } 575 } 576 577 lwkt_reltoken(&icu_irqmap_tok); 578 } 579 580 static int 581 icu_abi_msi_alloc(int intrs[], int count, int cpuid) 582 { 583 return icu_abi_msi_alloc_intern(ICU_IMT_MSI, "MSI", 584 intrs, count, cpuid); 585 } 586 587 static void 588 icu_abi_msi_release(const int intrs[], int count, int cpuid) 589 { 590 icu_abi_msi_release_intern(ICU_IMT_MSI, "MSI", 591 intrs, count, cpuid); 592 } 593 594 static int 595 icu_abi_msix_alloc(int *intr, int cpuid) 596 { 597 return icu_abi_msi_alloc_intern(ICU_IMT_MSIX, "MSI-X", 598 intr, 1, cpuid); 599 } 600 601 static void 602 icu_abi_msix_release(int intr, int cpuid) 603 { 604 icu_abi_msi_release_intern(ICU_IMT_MSIX, "MXI-X", 605 &intr, 1, cpuid); 606 } 607 608 static void 609 icu_abi_msi_map(int intr, uint64_t *addr, uint32_t *data, int cpuid) 610 { 611 const struct icu_irqmap *map; 612 613 KASSERT(cpuid >= 0 && cpuid < ncpus, 614 ("invalid cpuid %d", cpuid)); 615 616 KASSERT(intr >= 0 && intr < IDT_HWI_VECTORS, 617 ("invalid intr %d\n", intr)); 618 619 lwkt_gettoken(&icu_irqmap_tok); 620 621 map = &icu_irqmaps[cpuid][intr]; 622 KASSERT(map->im_type == ICU_IMT_MSI || 623 map->im_type == ICU_IMT_MSIX, 624 ("try map non-MSI/MSI-X intr %d, type %d\n", intr, map->im_type)); 625 KASSERT(map->im_msi_base >= 0 && map->im_msi_base <= intr, 626 ("intr %d, invalid %s-base %d\n", intr, 627 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X", 628 map->im_msi_base)); 629 630 msi_map(map->im_msi_base, addr, data, cpuid); 631 632 if (bootverbose) { 633 kprintf("map %s intr %d on cpu%d\n", 634 map->im_type == ICU_IMT_MSI ? "MSI" : "MSI-X", 635 intr, cpuid); 636 } 637 638 lwkt_reltoken(&icu_irqmap_tok); 639 } 640