1 /*- 2 * SPDX-License-Identifier: BSD-3-Clause 3 * 4 * Copyright (c) 1991 The Regents of the University of California. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to Berkeley by 8 * William Jolitz. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. Neither the name of the University nor the names of its contributors 19 * may be used to endorse or promote products derived from this software 20 * without specific prior written permission. 21 * 22 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 25 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 32 * SUCH DAMAGE. 33 */ 34 /*- 35 * Copyright (c) 2002 Benno Rice. 36 * All rights reserved. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 47 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 48 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 49 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 50 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 51 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 52 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 53 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 54 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 55 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 56 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 57 * SUCH DAMAGE. 58 * 59 * from: @(#)isa.c 7.2 (Berkeley) 5/13/91 60 * form: src/sys/i386/isa/intr_machdep.c,v 1.57 2001/07/20 61 */ 62 63 #include "opt_isa.h" 64 65 #include <sys/param.h> 66 #include <sys/systm.h> 67 #include <sys/kernel.h> 68 #include <sys/queue.h> 69 #include <sys/bus.h> 70 #include <sys/cpuset.h> 71 #include <sys/interrupt.h> 72 #include <sys/ktr.h> 73 #include <sys/lock.h> 74 #include <sys/malloc.h> 75 #include <sys/mutex.h> 76 #include <sys/pcpu.h> 77 #include <sys/smp.h> 78 #include <sys/syslog.h> 79 #include <sys/vmmeter.h> 80 #include <sys/proc.h> 81 82 #include <machine/frame.h> 83 #include <machine/intr_machdep.h> 84 #include <machine/md_var.h> 85 #include <machine/smp.h> 86 #include <machine/trap.h> 87 88 #include "pic_if.h" 89 90 #define MAX_STRAY_LOG 5 91 92 static MALLOC_DEFINE(M_INTR, "intr", "interrupt handler data"); 93 94 struct powerpc_intr { 95 struct intr_event *event; 96 long *cntp; 97 void *priv; /* PIC-private data */ 98 device_t pic; 99 u_int irq; 100 u_int intline; 101 u_int vector; 102 u_int cntindex; 103 int fwcode; 104 int ipi; 105 int pi_domain; 106 enum intr_trigger trig; 107 enum intr_polarity pol; 108 cpuset_t pi_cpuset; 109 }; 110 111 struct pic { 112 device_t dev; 113 uint32_t node; 114 u_int irqs; 115 u_int ipis; 116 int base; 117 }; 118 119 static u_int intrcnt_index = 0; 120 static struct mtx intr_table_lock; 121 static struct powerpc_intr **powerpc_intrs; 122 static struct pic piclist[MAX_PICS]; 123 static u_int nvectors; /* Allocated vectors */ 124 static u_int npics; /* PICs registered */ 125 #ifdef DEV_ISA 126 static u_int nirqs = 16; /* Allocated IRQS (ISA pre-allocated). */ 127 #else 128 static u_int nirqs = 0; /* Allocated IRQs. */ 129 #endif 130 static u_int stray_count; 131 132 u_long *intrcnt; 133 char *intrnames; 134 size_t sintrcnt = sizeof(intrcnt); 135 size_t sintrnames = sizeof(intrnames); 136 int nintrcnt; 137 138 /* 139 * Just to start 140 */ 141 #ifdef __powerpc64__ 142 u_int num_io_irqs = 768; 143 #else 144 u_int num_io_irqs = 256; 145 #endif 146 147 device_t root_pic; 148 149 #ifdef SMP 150 static void *ipi_cookie; 151 #endif 152 153 static void 154 intrcnt_setname(const char *name, int index) 155 { 156 157 snprintf(intrnames + (MAXCOMLEN + 1) * index, MAXCOMLEN + 1, "%-*s", 158 MAXCOMLEN, name); 159 } 160 161 static void 162 intr_init(void *dummy __unused) 163 { 164 165 mtx_init(&intr_table_lock, "intr sources lock", NULL, MTX_DEF); 166 } 167 SYSINIT(intr_init, SI_SUB_INTR, SI_ORDER_FIRST, intr_init, NULL); 168 169 static void 170 intr_init_sources(void *arg __unused) 171 { 172 173 powerpc_intrs = mallocarray(num_io_irqs, sizeof(*powerpc_intrs), 174 M_INTR, M_WAITOK | M_ZERO); 175 nintrcnt = 1 + num_io_irqs * 2 + mp_ncpus * 2; 176 #ifdef COUNT_IPIS 177 if (mp_ncpus > 1) 178 nintrcnt += 8 * mp_ncpus; 179 #endif 180 intrcnt = mallocarray(nintrcnt, sizeof(u_long), M_INTR, M_WAITOK | 181 M_ZERO); 182 intrnames = mallocarray(nintrcnt, MAXCOMLEN + 1, M_INTR, M_WAITOK | 183 M_ZERO); 184 sintrcnt = nintrcnt * sizeof(u_long); 185 sintrnames = nintrcnt * (MAXCOMLEN + 1); 186 187 intrcnt_setname("???", 0); 188 intrcnt_index = 1; 189 } 190 /* 191 * This needs to happen before SI_SUB_CPU 192 */ 193 SYSINIT(intr_init_sources, SI_SUB_KLD, SI_ORDER_ANY, intr_init_sources, NULL); 194 195 #ifdef SMP 196 static void 197 smp_intr_init(void *dummy __unused) 198 { 199 struct powerpc_intr *i; 200 int vector; 201 202 for (vector = 0; vector < nvectors; vector++) { 203 i = powerpc_intrs[vector]; 204 if (i != NULL && i->event != NULL && i->pic == root_pic) 205 PIC_BIND(i->pic, i->intline, i->pi_cpuset, &i->priv); 206 } 207 } 208 SYSINIT(smp_intr_init, SI_SUB_SMP, SI_ORDER_ANY, smp_intr_init, NULL); 209 #endif 210 211 void 212 intrcnt_add(const char *name, u_long **countp) 213 { 214 int idx; 215 216 idx = atomic_fetchadd_int(&intrcnt_index, 1); 217 KASSERT(idx < nintrcnt, ("intrcnt_add: Interrupt counter index %d/%d" 218 "reached nintrcnt : %d", intrcnt_index, idx, nintrcnt)); 219 *countp = &intrcnt[idx]; 220 intrcnt_setname(name, idx); 221 } 222 223 extern void kdb_backtrace(void); 224 static struct powerpc_intr * 225 intr_lookup(u_int irq) 226 { 227 char intrname[16]; 228 struct powerpc_intr *i, *iscan; 229 int vector; 230 231 mtx_lock(&intr_table_lock); 232 for (vector = 0; vector < nvectors; vector++) { 233 i = powerpc_intrs[vector]; 234 if (i != NULL && i->irq == irq) { 235 mtx_unlock(&intr_table_lock); 236 return (i); 237 } 238 } 239 240 i = malloc(sizeof(*i), M_INTR, M_NOWAIT); 241 if (i == NULL) { 242 mtx_unlock(&intr_table_lock); 243 return (NULL); 244 } 245 246 i->event = NULL; 247 i->cntp = NULL; 248 i->priv = NULL; 249 i->trig = INTR_TRIGGER_CONFORM; 250 i->pol = INTR_POLARITY_CONFORM; 251 i->irq = irq; 252 i->pic = NULL; 253 i->vector = -1; 254 i->fwcode = 0; 255 i->ipi = 0; 256 257 #ifdef SMP 258 i->pi_cpuset = all_cpus; 259 #else 260 CPU_SETOF(0, &i->pi_cpuset); 261 #endif 262 263 for (vector = 0; vector < num_io_irqs && vector <= nvectors; 264 vector++) { 265 iscan = powerpc_intrs[vector]; 266 if (iscan != NULL && iscan->irq == irq) 267 break; 268 if (iscan == NULL && i->vector == -1) 269 i->vector = vector; 270 iscan = NULL; 271 } 272 273 if (iscan == NULL && i->vector != -1) { 274 powerpc_intrs[i->vector] = i; 275 i->cntindex = atomic_fetchadd_int(&intrcnt_index, 1); 276 i->cntp = &intrcnt[i->cntindex]; 277 sprintf(intrname, "irq%u:", i->irq); 278 intrcnt_setname(intrname, i->cntindex); 279 nvectors++; 280 } 281 mtx_unlock(&intr_table_lock); 282 283 if (iscan != NULL || i->vector == -1) { 284 free(i, M_INTR); 285 i = iscan; 286 } 287 288 return (i); 289 } 290 291 static int 292 powerpc_map_irq(struct powerpc_intr *i) 293 { 294 struct pic *p; 295 u_int cnt; 296 int idx; 297 298 for (idx = 0; idx < npics; idx++) { 299 p = &piclist[idx]; 300 cnt = p->irqs + p->ipis; 301 if (i->irq >= p->base && i->irq < p->base + cnt) 302 break; 303 } 304 if (idx == npics) 305 return (EINVAL); 306 307 i->intline = i->irq - p->base; 308 i->pic = p->dev; 309 310 /* Try a best guess if that failed */ 311 if (i->pic == NULL) 312 i->pic = root_pic; 313 314 return (0); 315 } 316 317 static void 318 powerpc_intr_eoi(void *arg) 319 { 320 struct powerpc_intr *i = arg; 321 322 PIC_EOI(i->pic, i->intline, i->priv); 323 } 324 325 static void 326 powerpc_intr_pre_ithread(void *arg) 327 { 328 struct powerpc_intr *i = arg; 329 330 PIC_MASK(i->pic, i->intline, i->priv); 331 PIC_EOI(i->pic, i->intline, i->priv); 332 } 333 334 static void 335 powerpc_intr_post_ithread(void *arg) 336 { 337 struct powerpc_intr *i = arg; 338 339 PIC_UNMASK(i->pic, i->intline, i->priv); 340 } 341 342 static int 343 powerpc_assign_intr_cpu(void *arg, int cpu) 344 { 345 #ifdef SMP 346 struct powerpc_intr *i = arg; 347 348 if (cpu == NOCPU) 349 i->pi_cpuset = all_cpus; 350 else 351 CPU_SETOF(cpu, &i->pi_cpuset); 352 353 if (!cold && i->pic != NULL && i->pic == root_pic) 354 PIC_BIND(i->pic, i->intline, i->pi_cpuset, &i->priv); 355 356 return (0); 357 #else 358 return (EOPNOTSUPP); 359 #endif 360 } 361 362 u_int 363 powerpc_register_pic(device_t dev, uint32_t node, u_int irqs, u_int ipis, 364 u_int atpic) 365 { 366 struct pic *p; 367 u_int irq; 368 int idx; 369 370 mtx_lock(&intr_table_lock); 371 372 /* XXX see powerpc_get_irq(). */ 373 for (idx = 0; idx < npics; idx++) { 374 p = &piclist[idx]; 375 if (p->node != node) 376 continue; 377 if (node != 0 || p->dev == dev) 378 break; 379 } 380 p = &piclist[idx]; 381 382 p->dev = dev; 383 p->node = node; 384 p->irqs = irqs; 385 p->ipis = ipis; 386 if (idx == npics) { 387 #ifdef DEV_ISA 388 p->base = (atpic) ? 0 : nirqs; 389 #else 390 p->base = nirqs; 391 #endif 392 irq = p->base + irqs + ipis; 393 nirqs = MAX(nirqs, irq); 394 npics++; 395 } 396 397 KASSERT(npics < MAX_PICS, 398 ("Number of PICs exceeds maximum (%d)", MAX_PICS)); 399 400 mtx_unlock(&intr_table_lock); 401 402 return (p->base); 403 } 404 405 u_int 406 powerpc_get_irq(uint32_t node, u_int pin) 407 { 408 int idx; 409 410 if (node == 0) 411 return (pin); 412 413 mtx_lock(&intr_table_lock); 414 for (idx = 0; idx < npics; idx++) { 415 if (piclist[idx].node == node) { 416 mtx_unlock(&intr_table_lock); 417 return (piclist[idx].base + pin); 418 } 419 } 420 421 /* 422 * XXX we should never encounter an unregistered PIC, but that 423 * can only be done when we properly support bus enumeration 424 * using multiple passes. Until then, fake an entry and give it 425 * some adhoc maximum number of IRQs and IPIs. 426 */ 427 piclist[idx].dev = NULL; 428 piclist[idx].node = node; 429 piclist[idx].irqs = 124; 430 piclist[idx].ipis = 4; 431 piclist[idx].base = nirqs; 432 nirqs += (1 << 25); 433 npics++; 434 435 KASSERT(npics < MAX_PICS, 436 ("Number of PICs exceeds maximum (%d)", MAX_PICS)); 437 438 mtx_unlock(&intr_table_lock); 439 440 return (piclist[idx].base + pin); 441 } 442 443 int 444 powerpc_enable_intr(void) 445 { 446 struct powerpc_intr *i; 447 int error, vector; 448 #ifdef SMP 449 int n; 450 #endif 451 452 if (npics == 0) 453 panic("no PIC detected\n"); 454 455 if (root_pic == NULL) 456 root_pic = piclist[0].dev; 457 458 KASSERT(root_pic != NULL, ("no root PIC!")); 459 460 #ifdef SMP 461 /* Install an IPI handler. */ 462 if (mp_ncpus > 1) { 463 for (n = 0; n < npics; n++) { 464 if (piclist[n].dev != root_pic) 465 continue; 466 467 KASSERT(piclist[n].ipis != 0, 468 ("%s: SMP root PIC does not supply any IPIs", 469 __func__)); 470 error = powerpc_setup_intr("IPI", 471 MAP_IRQ(piclist[n].node, piclist[n].irqs), 472 powerpc_ipi_handler, NULL, NULL, 473 INTR_TYPE_MISC | INTR_EXCL, &ipi_cookie, 474 0 /* domain XXX */); 475 if (error) { 476 printf("unable to setup IPI handler\n"); 477 return (error); 478 } 479 480 /* 481 * Some subterfuge: disable late EOI and mark this 482 * as an IPI to the dispatch layer. 483 */ 484 i = intr_lookup(MAP_IRQ(piclist[n].node, 485 piclist[n].irqs)); 486 i->event->ie_post_filter = NULL; 487 i->ipi = 1; 488 } 489 } 490 #endif 491 492 for (vector = 0; vector < nvectors; vector++) { 493 i = powerpc_intrs[vector]; 494 if (i == NULL) 495 continue; 496 497 error = powerpc_map_irq(i); 498 if (error) 499 continue; 500 501 if (i->trig == INTR_TRIGGER_INVALID) 502 PIC_TRANSLATE_CODE(i->pic, i->intline, i->fwcode, 503 &i->trig, &i->pol); 504 if (i->trig != INTR_TRIGGER_CONFORM || 505 i->pol != INTR_POLARITY_CONFORM) 506 PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); 507 508 if (i->event != NULL) 509 PIC_ENABLE(i->pic, i->intline, vector, &i->priv); 510 } 511 512 return (0); 513 } 514 515 int 516 powerpc_setup_intr(const char *name, u_int irq, driver_filter_t filter, 517 driver_intr_t handler, void *arg, enum intr_type flags, void **cookiep, 518 int domain) 519 { 520 struct powerpc_intr *i; 521 int error, enable = 0; 522 523 i = intr_lookup(irq); 524 if (i == NULL) 525 return (ENOMEM); 526 527 if (i->event == NULL) { 528 error = intr_event_create(&i->event, (void *)i, 0, irq, 529 powerpc_intr_pre_ithread, powerpc_intr_post_ithread, 530 powerpc_intr_eoi, powerpc_assign_intr_cpu, "irq%u:", irq); 531 if (error) 532 return (error); 533 534 enable = 1; 535 } 536 537 error = intr_event_add_handler(i->event, name, filter, handler, arg, 538 intr_priority(flags), flags, cookiep); 539 if (error) 540 return (error); 541 i->pi_domain = domain; 542 if (strcmp(name, "IPI") != 0) { 543 CPU_ZERO(&i->pi_cpuset); 544 CPU_COPY(&cpuset_domain[domain], &i->pi_cpuset); 545 } 546 mtx_lock(&intr_table_lock); 547 intrcnt_setname(i->event->ie_fullname, i->cntindex); 548 mtx_unlock(&intr_table_lock); 549 550 if (!cold) { 551 error = powerpc_map_irq(i); 552 553 if (!error) { 554 if (i->trig == INTR_TRIGGER_INVALID) 555 PIC_TRANSLATE_CODE(i->pic, i->intline, 556 i->fwcode, &i->trig, &i->pol); 557 558 if (i->trig != INTR_TRIGGER_CONFORM || 559 i->pol != INTR_POLARITY_CONFORM) 560 PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); 561 562 if (i->pic == root_pic) 563 PIC_BIND(i->pic, i->intline, i->pi_cpuset, &i->priv); 564 565 if (enable) 566 PIC_ENABLE(i->pic, i->intline, i->vector, 567 &i->priv); 568 } 569 } 570 return (error); 571 } 572 573 int 574 powerpc_teardown_intr(void *cookie) 575 { 576 577 return (intr_event_remove_handler(cookie)); 578 } 579 580 #ifdef SMP 581 int 582 powerpc_bind_intr(u_int irq, u_char cpu) 583 { 584 struct powerpc_intr *i; 585 586 i = intr_lookup(irq); 587 if (i == NULL) 588 return (ENOMEM); 589 590 return (intr_event_bind(i->event, cpu)); 591 } 592 #endif 593 594 int 595 powerpc_fw_config_intr(int irq, int sense_code) 596 { 597 struct powerpc_intr *i; 598 599 i = intr_lookup(irq); 600 if (i == NULL) 601 return (ENOMEM); 602 603 i->trig = INTR_TRIGGER_INVALID; 604 i->pol = INTR_POLARITY_CONFORM; 605 i->fwcode = sense_code; 606 607 if (!cold && i->pic != NULL) { 608 PIC_TRANSLATE_CODE(i->pic, i->intline, i->fwcode, &i->trig, 609 &i->pol); 610 PIC_CONFIG(i->pic, i->intline, i->trig, i->pol); 611 } 612 613 return (0); 614 } 615 616 int 617 powerpc_config_intr(int irq, enum intr_trigger trig, enum intr_polarity pol) 618 { 619 struct powerpc_intr *i; 620 621 i = intr_lookup(irq); 622 if (i == NULL) 623 return (ENOMEM); 624 625 i->trig = trig; 626 i->pol = pol; 627 628 if (!cold && i->pic != NULL) 629 PIC_CONFIG(i->pic, i->intline, trig, pol); 630 631 return (0); 632 } 633 634 void 635 powerpc_dispatch_intr(u_int vector, struct trapframe *tf) 636 { 637 struct powerpc_intr *i; 638 struct intr_event *ie; 639 640 i = powerpc_intrs[vector]; 641 if (i == NULL) 642 goto stray; 643 644 (*i->cntp)++; 645 646 ie = i->event; 647 KASSERT(ie != NULL, ("%s: interrupt without an event", __func__)); 648 649 /* 650 * IPIs are magical and need to be EOI'ed before filtering. 651 * This prevents races in IPI handling. 652 */ 653 if (i->ipi) 654 PIC_EOI(i->pic, i->intline, i->priv); 655 656 if (intr_event_handle(ie, tf) != 0) { 657 goto stray; 658 } 659 return; 660 661 stray: 662 stray_count++; 663 if (stray_count <= MAX_STRAY_LOG) { 664 printf("stray irq %d\n", i ? i->irq : -1); 665 if (stray_count >= MAX_STRAY_LOG) { 666 printf("got %d stray interrupts, not logging anymore\n", 667 MAX_STRAY_LOG); 668 } 669 } 670 if (i != NULL) 671 PIC_MASK(i->pic, i->intline, i->priv); 672 } 673 674 void 675 powerpc_intr_mask(u_int irq) 676 { 677 struct powerpc_intr *i; 678 679 i = intr_lookup(irq); 680 if (i == NULL || i->pic == NULL) 681 return; 682 683 PIC_MASK(i->pic, i->intline, i->priv); 684 } 685 686 void 687 powerpc_intr_unmask(u_int irq) 688 { 689 struct powerpc_intr *i; 690 691 i = intr_lookup(irq); 692 if (i == NULL || i->pic == NULL) 693 return; 694 695 PIC_UNMASK(i->pic, i->intline, i->priv); 696 } 697