1 /*- 2 * Copyright (c) 2011 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * Developed by Damjan Marion <damjan.marion@gmail.com> 6 * 7 * Based on OMAP4 GIC code by Ben Gray 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. The name of the company nor the name of the author may be used to 18 * endorse or promote products derived from this software without specific 19 * prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_platform.h" 38 39 #include "opt_platform.h" 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/bus.h> 44 #include <sys/kernel.h> 45 #include <sys/ktr.h> 46 #include <sys/module.h> 47 #include <sys/malloc.h> 48 #include <sys/rman.h> 49 #include <sys/pcpu.h> 50 #include <sys/proc.h> 51 #include <sys/cpuset.h> 52 #include <sys/lock.h> 53 #include <sys/mutex.h> 54 #include <sys/smp.h> 55 #ifdef ARM_INTRNG 56 #include <sys/sched.h> 57 #endif 58 #include <machine/bus.h> 59 #include <machine/intr.h> 60 #include <machine/smp.h> 61 62 #include <dev/fdt/fdt_common.h> 63 #include <dev/ofw/openfirm.h> 64 #include <dev/ofw/ofw_bus.h> 65 #include <dev/ofw/ofw_bus_subr.h> 66 67 #ifdef ARM_INTRNG 68 #include "pic_if.h" 69 #endif 70 71 #define GIC_DEBUG_SPURIOUS 72 73 /* We are using GICv2 register naming */ 74 75 /* Distributor Registers */ 76 #define GICD_CTLR 0x000 /* v1 ICDDCR */ 77 #define GICD_TYPER 0x004 /* v1 ICDICTR */ 78 #define GICD_IIDR 0x008 /* v1 ICDIIDR */ 79 #define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ 80 #define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ 81 #define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ 82 #define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ 83 #define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ 84 #define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ 85 #define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ 86 #define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ 87 #define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ 88 #define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ 89 90 /* CPU Registers */ 91 #define GICC_CTLR 0x0000 /* v1 ICCICR */ 92 #define GICC_PMR 0x0004 /* v1 ICCPMR */ 93 #define GICC_BPR 0x0008 /* v1 ICCBPR */ 94 #define GICC_IAR 0x000C /* v1 ICCIAR */ 95 #define GICC_EOIR 0x0010 /* v1 ICCEOIR */ 96 #define GICC_RPR 0x0014 /* v1 ICCRPR */ 97 #define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ 98 #define GICC_ABPR 0x001C /* v1 ICCABPR */ 99 #define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ 100 101 #define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ 102 #define GIC_LAST_SGI 15 103 #define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ 104 #define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ 105 #define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ 106 107 /* First bit is a polarity bit (0 - low, 1 - high) */ 108 #define GICD_ICFGR_POL_LOW (0 << 0) 109 #define GICD_ICFGR_POL_HIGH (1 << 0) 110 #define GICD_ICFGR_POL_MASK 0x1 111 /* Second bit is a trigger bit (0 - level, 1 - edge) */ 112 #define GICD_ICFGR_TRIG_LVL (0 << 1) 113 #define GICD_ICFGR_TRIG_EDGE (1 << 1) 114 #define GICD_ICFGR_TRIG_MASK 0x2 115 116 #ifndef GIC_DEFAULT_ICFGR_INIT 117 #define GIC_DEFAULT_ICFGR_INIT 0x00000000 118 #endif 119 120 #ifdef ARM_INTRNG 121 static u_int gic_irq_cpu; 122 static int arm_gic_intr(void *); 123 static int arm_gic_bind(device_t dev, struct intr_irqsrc *isrc); 124 #endif 125 126 struct arm_gic_softc { 127 device_t gic_dev; 128 #ifdef ARM_INTRNG 129 void * gic_intrhand; 130 struct intr_irqsrc ** gic_irqs; 131 #endif 132 struct resource * gic_res[3]; 133 bus_space_tag_t gic_c_bst; 134 bus_space_tag_t gic_d_bst; 135 bus_space_handle_t gic_c_bsh; 136 bus_space_handle_t gic_d_bsh; 137 uint8_t ver; 138 struct mtx mutex; 139 uint32_t nirqs; 140 #ifdef GIC_DEBUG_SPURIOUS 141 uint32_t last_irq[MAXCPU]; 142 #endif 143 }; 144 145 static struct resource_spec arm_gic_spec[] = { 146 { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ 147 { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ 148 #ifdef ARM_INTRNG 149 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ 150 #endif 151 { -1, 0 } 152 }; 153 154 static struct arm_gic_softc *gic_sc = NULL; 155 156 #define gic_c_read_4(_sc, _reg) \ 157 bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) 158 #define gic_c_write_4(_sc, _reg, _val) \ 159 bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) 160 #define gic_d_read_4(_sc, _reg) \ 161 bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) 162 #define gic_d_write_1(_sc, _reg, _val) \ 163 bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 164 #define gic_d_write_4(_sc, _reg, _val) \ 165 bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 166 167 #ifndef ARM_INTRNG 168 static int gic_config_irq(int irq, enum intr_trigger trig, 169 enum intr_polarity pol); 170 static void gic_post_filter(void *); 171 #endif 172 173 static struct ofw_compat_data compat_data[] = { 174 {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ 175 {"arm,gic-400", true}, 176 {"arm,cortex-a15-gic", true}, 177 {"arm,cortex-a9-gic", true}, 178 {"arm,cortex-a7-gic", true}, 179 {"arm,arm11mp-gic", true}, 180 {"brcm,brahma-b15-gic", true}, 181 {NULL, false} 182 }; 183 184 static int 185 arm_gic_probe(device_t dev) 186 { 187 188 if (!ofw_bus_status_okay(dev)) 189 return (ENXIO); 190 191 if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) 192 return (ENXIO); 193 device_set_desc(dev, "ARM Generic Interrupt Controller"); 194 return (BUS_PROBE_DEFAULT); 195 } 196 197 #ifdef ARM_INTRNG 198 static inline void 199 gic_irq_unmask(struct arm_gic_softc *sc, u_int irq) 200 { 201 202 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 203 } 204 205 static inline void 206 gic_irq_mask(struct arm_gic_softc *sc, u_int irq) 207 { 208 209 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 210 } 211 #endif 212 213 #ifdef SMP 214 #ifdef ARM_INTRNG 215 static void 216 arm_gic_init_secondary(device_t dev) 217 { 218 struct arm_gic_softc *sc = device_get_softc(dev); 219 struct intr_irqsrc *isrc; 220 u_int irq; 221 222 for (irq = 0; irq < sc->nirqs; irq += 4) 223 gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0); 224 225 /* Set all the interrupts to be in Group 0 (secure) */ 226 for (irq = 0; irq < sc->nirqs; irq += 32) { 227 gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0); 228 } 229 230 /* Enable CPU interface */ 231 gic_c_write_4(sc, GICC_CTLR, 1); 232 233 /* Set priority mask register. */ 234 gic_c_write_4(sc, GICC_PMR, 0xff); 235 236 /* Enable interrupt distribution */ 237 gic_d_write_4(sc, GICD_CTLR, 0x01); 238 239 /* Unmask attached SGI interrupts. */ 240 for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) { 241 isrc = sc->gic_irqs[irq]; 242 if (isrc != NULL && isrc->isrc_handlers != 0) { 243 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 244 gic_irq_unmask(sc, irq); 245 } 246 } 247 248 /* Unmask attached PPI interrupts. */ 249 for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) { 250 isrc = sc->gic_irqs[irq]; 251 if (isrc == NULL || isrc->isrc_handlers == 0) 252 continue; 253 if (isrc->isrc_flags & INTR_ISRCF_BOUND) { 254 if (CPU_ISSET(PCPU_GET(cpuid), &isrc->isrc_cpu)) 255 gic_irq_unmask(sc, irq); 256 } else { 257 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 258 gic_irq_unmask(sc, irq); 259 } 260 } 261 } 262 #else 263 static void 264 arm_gic_init_secondary(device_t dev) 265 { 266 struct arm_gic_softc *sc = device_get_softc(dev); 267 int i; 268 269 for (i = 0; i < sc->nirqs; i += 4) 270 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 271 272 /* Set all the interrupts to be in Group 0 (secure) */ 273 for (i = 0; i < sc->nirqs; i += 32) { 274 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 275 } 276 277 /* Enable CPU interface */ 278 gic_c_write_4(sc, GICC_CTLR, 1); 279 280 /* Set priority mask register. */ 281 gic_c_write_4(sc, GICC_PMR, 0xff); 282 283 /* Enable interrupt distribution */ 284 gic_d_write_4(sc, GICD_CTLR, 0x01); 285 286 /* 287 * Activate the timer interrupts: virtual, secure, and non-secure. 288 */ 289 gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); 290 gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); 291 gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); 292 } 293 #endif /* ARM_INTRNG */ 294 #endif /* SMP */ 295 296 #ifndef ARM_INTRNG 297 int 298 gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, 299 int *trig, int *pol) 300 { 301 static u_int num_intr_cells; 302 static phandle_t self; 303 struct ofw_compat_data *ocd; 304 305 if (self == 0) { 306 for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) { 307 if (fdt_is_compatible(iparent, ocd->ocd_str)) { 308 self = iparent; 309 break; 310 } 311 } 312 } 313 if (self != iparent) 314 return (ENXIO); 315 316 if (num_intr_cells == 0) { 317 if (OF_searchencprop(OF_node_from_xref(iparent), 318 "#interrupt-cells", &num_intr_cells, 319 sizeof(num_intr_cells)) == -1) { 320 num_intr_cells = 1; 321 } 322 } 323 324 if (num_intr_cells == 1) { 325 *interrupt = fdt32_to_cpu(intr[0]); 326 *trig = INTR_TRIGGER_CONFORM; 327 *pol = INTR_POLARITY_CONFORM; 328 } else { 329 if (fdt32_to_cpu(intr[0]) == 0) 330 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; 331 else 332 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; 333 /* 334 * In intr[2], bits[3:0] are trigger type and level flags. 335 * 1 = low-to-high edge triggered 336 * 2 = high-to-low edge triggered 337 * 4 = active high level-sensitive 338 * 8 = active low level-sensitive 339 * The hardware only supports active-high-level or rising-edge. 340 */ 341 if (fdt32_to_cpu(intr[2]) & 0x0a) { 342 printf("unsupported trigger/polarity configuration " 343 "0x%02x\n", fdt32_to_cpu(intr[2]) & 0x0f); 344 } 345 *pol = INTR_POLARITY_CONFORM; 346 if (fdt32_to_cpu(intr[2]) & 0x03) 347 *trig = INTR_TRIGGER_EDGE; 348 else 349 *trig = INTR_TRIGGER_LEVEL; 350 } 351 return (0); 352 } 353 #endif 354 355 #ifdef ARM_INTRNG 356 static inline intptr_t 357 gic_xref(device_t dev) 358 { 359 #ifdef FDT 360 return (OF_xref_from_node(ofw_bus_get_node(dev))); 361 #else 362 return (0); 363 #endif 364 } 365 #endif 366 367 static int 368 arm_gic_attach(device_t dev) 369 { 370 struct arm_gic_softc *sc; 371 int i; 372 uint32_t icciidr; 373 #ifdef ARM_INTRNG 374 phandle_t pxref; 375 intptr_t xref = gic_xref(dev); 376 #endif 377 378 if (gic_sc) 379 return (ENXIO); 380 381 sc = device_get_softc(dev); 382 383 if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { 384 device_printf(dev, "could not allocate resources\n"); 385 return (ENXIO); 386 } 387 388 sc->gic_dev = dev; 389 gic_sc = sc; 390 391 /* Initialize mutex */ 392 mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); 393 394 /* Distributor Interface */ 395 sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); 396 sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); 397 398 /* CPU Interface */ 399 sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); 400 sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); 401 402 /* Disable interrupt forwarding to the CPU interface */ 403 gic_d_write_4(sc, GICD_CTLR, 0x00); 404 405 /* Get the number of interrupts */ 406 sc->nirqs = gic_d_read_4(sc, GICD_TYPER); 407 sc->nirqs = 32 * ((sc->nirqs & 0x1f) + 1); 408 409 #ifdef ARM_INTRNG 410 sc->gic_irqs = malloc(sc->nirqs * sizeof (*sc->gic_irqs), M_DEVBUF, 411 M_WAITOK | M_ZERO); 412 #else 413 /* Set up function pointers */ 414 arm_post_filter = gic_post_filter; 415 arm_config_irq = gic_config_irq; 416 #endif 417 418 icciidr = gic_c_read_4(sc, GICC_IIDR); 419 device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", 420 icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, 421 (icciidr & 0xfff), sc->nirqs); 422 423 /* Set all global interrupts to be level triggered, active low. */ 424 for (i = 32; i < sc->nirqs; i += 16) { 425 gic_d_write_4(sc, GICD_ICFGR(i >> 4), GIC_DEFAULT_ICFGR_INIT); 426 } 427 428 /* Disable all interrupts. */ 429 for (i = 32; i < sc->nirqs; i += 32) { 430 gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); 431 } 432 433 for (i = 0; i < sc->nirqs; i += 4) { 434 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 435 gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), 436 1 << 0 | 1 << 8 | 1 << 16 | 1 << 24); 437 } 438 439 /* Set all the interrupts to be in Group 0 (secure) */ 440 for (i = 0; i < sc->nirqs; i += 32) { 441 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 442 } 443 444 /* Enable CPU interface */ 445 gic_c_write_4(sc, GICC_CTLR, 1); 446 447 /* Set priority mask register. */ 448 gic_c_write_4(sc, GICC_PMR, 0xff); 449 450 /* Enable interrupt distribution */ 451 gic_d_write_4(sc, GICD_CTLR, 0x01); 452 #ifndef ARM_INTRNG 453 return (0); 454 #else 455 /* 456 * Now, when everything is initialized, it's right time to 457 * register interrupt controller to interrupt framefork. 458 */ 459 if (intr_pic_register(dev, xref) != 0) { 460 device_printf(dev, "could not register PIC\n"); 461 goto cleanup; 462 } 463 464 /* 465 * Controller is root if: 466 * - doesn't have interrupt parent 467 * - his interrupt parent is this controller 468 */ 469 pxref = ofw_bus_find_iparent(ofw_bus_get_node(dev)); 470 if (pxref == 0 || xref == pxref) { 471 if (intr_pic_claim_root(dev, xref, arm_gic_intr, sc, 472 GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { 473 device_printf(dev, "could not set PIC as a root\n"); 474 intr_pic_unregister(dev, xref); 475 goto cleanup; 476 } 477 } else { 478 if (sc->gic_res[2] == NULL) { 479 device_printf(dev, 480 "not root PIC must have defined interrupt\n"); 481 intr_pic_unregister(dev, xref); 482 goto cleanup; 483 } 484 if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, 485 arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { 486 device_printf(dev, "could not setup irq handler\n"); 487 intr_pic_unregister(dev, xref); 488 goto cleanup; 489 } 490 } 491 492 OF_device_register_xref(xref, dev); 493 return (0); 494 495 cleanup: 496 /* 497 * XXX - not implemented arm_gic_detach() should be called ! 498 */ 499 if (sc->gic_irqs != NULL) 500 free(sc->gic_irqs, M_DEVBUF); 501 bus_release_resources(dev, arm_gic_spec, sc->gic_res); 502 return(ENXIO); 503 #endif 504 } 505 506 #ifdef ARM_INTRNG 507 static int 508 arm_gic_intr(void *arg) 509 { 510 struct arm_gic_softc *sc = arg; 511 struct intr_irqsrc *isrc; 512 uint32_t irq_active_reg, irq; 513 struct trapframe *tf; 514 515 irq_active_reg = gic_c_read_4(sc, GICC_IAR); 516 irq = irq_active_reg & 0x3FF; 517 518 /* 519 * 1. We do EOI here because recent read value from active interrupt 520 * register must be used for it. Another approach is to save this 521 * value into associated interrupt source. 522 * 2. EOI must be done on same CPU where interrupt has fired. Thus 523 * we must ensure that interrupted thread does not migrate to 524 * another CPU. 525 * 3. EOI cannot be delayed by any preemption which could happen on 526 * critical_exit() used in MI intr code, when interrupt thread is 527 * scheduled. See next point. 528 * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during 529 * an action and any use of critical_exit() could break this 530 * assumption. See comments within smp_rendezvous_action(). 531 * 5. We always return FILTER_HANDLED as this is an interrupt 532 * controller dispatch function. Otherwise, in cascaded interrupt 533 * case, the whole interrupt subtree would be masked. 534 */ 535 536 if (irq >= sc->nirqs) { 537 #ifdef GIC_DEBUG_SPURIOUS 538 device_printf(sc->gic_dev, 539 "Spurious interrupt detected: last irq: %d on CPU%d\n", 540 sc->last_irq[PCPU_GET(cpuid)], PCPU_GET(cpuid)); 541 #endif 542 return (FILTER_HANDLED); 543 } 544 545 tf = curthread->td_intr_frame; 546 dispatch_irq: 547 isrc = sc->gic_irqs[irq]; 548 if (isrc == NULL) { 549 device_printf(sc->gic_dev, "Stray interrupt %u detected\n", irq); 550 gic_irq_mask(sc, irq); 551 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 552 goto next_irq; 553 } 554 555 /* 556 * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement 557 * as compiler complains that comparing u_int >= 0 is always true. 558 */ 559 if (irq <= GIC_LAST_SGI) { 560 #ifdef SMP 561 /* Call EOI for all IPI before dispatch. */ 562 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 563 intr_ipi_dispatch(isrc, tf); 564 goto next_irq; 565 #else 566 device_printf(sc->gic_dev, "SGI %u on UP system detected\n", 567 irq - GIC_FIRST_SGI); 568 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 569 goto next_irq; 570 #endif 571 } 572 573 #ifdef GIC_DEBUG_SPURIOUS 574 sc->last_irq[PCPU_GET(cpuid)] = irq; 575 #endif 576 if (isrc->isrc_trig == INTR_TRIGGER_EDGE) 577 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 578 579 intr_irq_dispatch(isrc, tf); 580 581 next_irq: 582 arm_irq_memory_barrier(irq); 583 irq_active_reg = gic_c_read_4(sc, GICC_IAR); 584 irq = irq_active_reg & 0x3FF; 585 if (irq < sc->nirqs) 586 goto dispatch_irq; 587 588 return (FILTER_HANDLED); 589 } 590 591 static int 592 gic_attach_isrc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int irq) 593 { 594 const char *name; 595 596 /* 597 * 1. The link between ISRC and controller must be set atomically. 598 * 2. Just do things only once in rare case when consumers 599 * of shared interrupt came here at the same moment. 600 */ 601 mtx_lock_spin(&sc->mutex); 602 if (sc->gic_irqs[irq] != NULL) { 603 mtx_unlock_spin(&sc->mutex); 604 return (sc->gic_irqs[irq] == isrc ? 0 : EEXIST); 605 } 606 sc->gic_irqs[irq] = isrc; 607 isrc->isrc_data = irq; 608 mtx_unlock_spin(&sc->mutex); 609 610 name = device_get_nameunit(sc->gic_dev); 611 if (irq <= GIC_LAST_SGI) 612 intr_irq_set_name(isrc, "%s,i%u", name, irq - GIC_FIRST_SGI); 613 else if (irq <= GIC_LAST_PPI) 614 intr_irq_set_name(isrc, "%s,p%u", name, irq - GIC_FIRST_PPI); 615 else 616 intr_irq_set_name(isrc, "%s,s%u", name, irq - GIC_FIRST_SPI); 617 return (0); 618 } 619 620 static int 621 gic_detach_isrc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int irq) 622 { 623 624 mtx_lock_spin(&sc->mutex); 625 if (sc->gic_irqs[irq] != isrc) { 626 mtx_unlock_spin(&sc->mutex); 627 return (sc->gic_irqs[irq] == NULL ? 0 : EINVAL); 628 } 629 sc->gic_irqs[irq] = NULL; 630 isrc->isrc_data = 0; 631 mtx_unlock_spin(&sc->mutex); 632 633 intr_irq_set_name(isrc, ""); 634 return (0); 635 } 636 637 static void 638 gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig, 639 enum intr_polarity pol) 640 { 641 uint32_t reg; 642 uint32_t mask; 643 644 if (irq < GIC_FIRST_SPI) 645 return; 646 647 mtx_lock_spin(&sc->mutex); 648 649 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 650 mask = (reg >> 2*(irq % 16)) & 0x3; 651 652 if (pol == INTR_POLARITY_LOW) { 653 mask &= ~GICD_ICFGR_POL_MASK; 654 mask |= GICD_ICFGR_POL_LOW; 655 } else if (pol == INTR_POLARITY_HIGH) { 656 mask &= ~GICD_ICFGR_POL_MASK; 657 mask |= GICD_ICFGR_POL_HIGH; 658 } 659 660 if (trig == INTR_TRIGGER_LEVEL) { 661 mask &= ~GICD_ICFGR_TRIG_MASK; 662 mask |= GICD_ICFGR_TRIG_LVL; 663 } else if (trig == INTR_TRIGGER_EDGE) { 664 mask &= ~GICD_ICFGR_TRIG_MASK; 665 mask |= GICD_ICFGR_TRIG_EDGE; 666 } 667 668 /* Set mask */ 669 reg = reg & ~(0x3 << 2*(irq % 16)); 670 reg = reg | (mask << 2*(irq % 16)); 671 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 672 673 mtx_unlock_spin(&sc->mutex); 674 } 675 676 static int 677 gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus) 678 { 679 uint32_t cpu, end, mask; 680 681 end = min(mp_ncpus, 8); 682 for (cpu = end; cpu < MAXCPU; cpu++) 683 if (CPU_ISSET(cpu, cpus)) 684 return (EINVAL); 685 686 for (mask = 0, cpu = 0; cpu < end; cpu++) 687 if (CPU_ISSET(cpu, cpus)) 688 mask |= 1 << cpu; 689 690 gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask); 691 return (0); 692 } 693 694 static int 695 gic_irq_from_nspc(struct arm_gic_softc *sc, u_int type, u_int num, u_int *irqp) 696 { 697 698 switch (type) { 699 case INTR_IRQ_NSPC_PLAIN: 700 *irqp = num; 701 return (*irqp < sc->nirqs ? 0 : EINVAL); 702 703 case INTR_IRQ_NSPC_IRQ: 704 *irqp = num + GIC_FIRST_PPI; 705 return (*irqp < sc->nirqs ? 0 : EINVAL); 706 707 case INTR_IRQ_NSPC_IPI: 708 *irqp = num + GIC_FIRST_SGI; 709 return (*irqp < GIC_LAST_SGI ? 0 : EINVAL); 710 711 default: 712 return (EINVAL); 713 } 714 } 715 716 static int 717 gic_map_nspc(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int *irqp) 718 { 719 int error; 720 721 error = gic_irq_from_nspc(sc, isrc->isrc_nspc_type, isrc->isrc_nspc_num, 722 irqp); 723 if (error != 0) 724 return (error); 725 return (gic_attach_isrc(sc, isrc, *irqp)); 726 } 727 728 #ifdef FDT 729 static int 730 gic_map_fdt(struct arm_gic_softc *sc, struct intr_irqsrc *isrc, u_int *irqp) 731 { 732 u_int irq, tripol; 733 enum intr_trigger trig; 734 enum intr_polarity pol; 735 int error; 736 737 if (isrc->isrc_ncells == 1) { 738 irq = isrc->isrc_cells[0]; 739 pol = INTR_POLARITY_CONFORM; 740 trig = INTR_TRIGGER_CONFORM; 741 } else if (isrc->isrc_ncells == 3) { 742 if (isrc->isrc_cells[0] == 0) 743 irq = isrc->isrc_cells[1] + GIC_FIRST_SPI; 744 else 745 irq = isrc->isrc_cells[1] + GIC_FIRST_PPI; 746 747 /* 748 * In intr[2], bits[3:0] are trigger type and level flags. 749 * 1 = low-to-high edge triggered 750 * 2 = high-to-low edge triggered 751 * 4 = active high level-sensitive 752 * 8 = active low level-sensitive 753 * The hardware only supports active-high-level or rising-edge. 754 */ 755 tripol = isrc->isrc_cells[2]; 756 if (tripol & 0x0a) { 757 device_printf(sc->gic_dev, 758 "unsupported trigger/polarity configuration " 759 "0x%02x\n", tripol & 0x0f); 760 } 761 pol = INTR_POLARITY_CONFORM; 762 if (tripol & 0x03) 763 trig = INTR_TRIGGER_EDGE; 764 else 765 trig = INTR_TRIGGER_LEVEL; 766 } else 767 return (EINVAL); 768 769 if (irq >= sc->nirqs) 770 return (EINVAL); 771 772 error = gic_attach_isrc(sc, isrc, irq); 773 if (error != 0) 774 return (error); 775 776 isrc->isrc_nspc_type = INTR_IRQ_NSPC_PLAIN; 777 isrc->isrc_nspc_num = irq; 778 isrc->isrc_trig = trig; 779 isrc->isrc_pol = pol; 780 781 *irqp = irq; 782 return (0); 783 } 784 #endif 785 786 static int 787 arm_gic_register(device_t dev, struct intr_irqsrc *isrc, boolean_t *is_percpu) 788 { 789 struct arm_gic_softc *sc = device_get_softc(dev); 790 u_int irq; 791 int error; 792 793 if (isrc->isrc_type == INTR_ISRCT_NAMESPACE) 794 error = gic_map_nspc(sc, isrc, &irq); 795 #ifdef FDT 796 else if (isrc->isrc_type == INTR_ISRCT_FDT) 797 error = gic_map_fdt(sc, isrc, &irq); 798 #endif 799 else 800 return (EINVAL); 801 802 if (error == 0) 803 *is_percpu = irq < GIC_FIRST_SPI ? TRUE : FALSE; 804 return (error); 805 } 806 807 static void 808 arm_gic_enable_intr(device_t dev, struct intr_irqsrc *isrc) 809 { 810 struct arm_gic_softc *sc = device_get_softc(dev); 811 u_int irq = isrc->isrc_data; 812 813 if (isrc->isrc_trig == INTR_TRIGGER_CONFORM) 814 isrc->isrc_trig = INTR_TRIGGER_LEVEL; 815 816 /* 817 * XXX - In case that per CPU interrupt is going to be enabled in time 818 * when SMP is already started, we need some IPI call which 819 * enables it on others CPUs. Further, it's more complicated as 820 * pic_enable_source() and pic_disable_source() should act on 821 * per CPU basis only. Thus, it should be solved here somehow. 822 */ 823 if (isrc->isrc_flags & INTR_ISRCF_PERCPU) 824 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 825 826 gic_config(sc, irq, isrc->isrc_trig, isrc->isrc_pol); 827 arm_gic_bind(dev, isrc); 828 } 829 830 static void 831 arm_gic_enable_source(device_t dev, struct intr_irqsrc *isrc) 832 { 833 struct arm_gic_softc *sc = device_get_softc(dev); 834 u_int irq = isrc->isrc_data; 835 836 arm_irq_memory_barrier(irq); 837 gic_irq_unmask(sc, irq); 838 } 839 840 static void 841 arm_gic_disable_source(device_t dev, struct intr_irqsrc *isrc) 842 { 843 struct arm_gic_softc *sc = device_get_softc(dev); 844 u_int irq = isrc->isrc_data; 845 846 gic_irq_mask(sc, irq); 847 } 848 849 static int 850 arm_gic_unregister(device_t dev, struct intr_irqsrc *isrc) 851 { 852 struct arm_gic_softc *sc = device_get_softc(dev); 853 u_int irq = isrc->isrc_data; 854 855 return (gic_detach_isrc(sc, isrc, irq)); 856 } 857 858 static void 859 arm_gic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 860 { 861 struct arm_gic_softc *sc = device_get_softc(dev); 862 863 arm_gic_disable_source(dev, isrc); 864 gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data); 865 } 866 867 static void 868 arm_gic_post_ithread(device_t dev, struct intr_irqsrc *isrc) 869 { 870 871 arm_irq_memory_barrier(0); 872 arm_gic_enable_source(dev, isrc); 873 } 874 875 static void 876 arm_gic_post_filter(device_t dev, struct intr_irqsrc *isrc) 877 { 878 struct arm_gic_softc *sc = device_get_softc(dev); 879 880 /* EOI for edge-triggered done earlier. */ 881 if (isrc->isrc_trig == INTR_TRIGGER_EDGE) 882 return; 883 884 arm_irq_memory_barrier(0); 885 gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data); 886 } 887 888 static int 889 arm_gic_bind(device_t dev, struct intr_irqsrc *isrc) 890 { 891 struct arm_gic_softc *sc = device_get_softc(dev); 892 uint32_t irq = isrc->isrc_data; 893 894 if (irq < GIC_FIRST_SPI) 895 return (EINVAL); 896 897 if (CPU_EMPTY(&isrc->isrc_cpu)) { 898 gic_irq_cpu = intr_irq_next_cpu(gic_irq_cpu, &all_cpus); 899 CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); 900 } 901 return (gic_bind(sc, irq, &isrc->isrc_cpu)); 902 } 903 904 #ifdef SMP 905 static void 906 arm_gic_ipi_send(device_t dev, struct intr_irqsrc *isrc, cpuset_t cpus) 907 { 908 struct arm_gic_softc *sc = device_get_softc(dev); 909 uint32_t irq, val = 0, i; 910 911 irq = isrc->isrc_data; 912 913 for (i = 0; i < MAXCPU; i++) 914 if (CPU_ISSET(i, &cpus)) 915 val |= 1 << (16 + i); 916 917 gic_d_write_4(sc, GICD_SGIR(0), val | irq); 918 } 919 #endif 920 #else 921 static int 922 arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq) 923 { 924 uint32_t active_irq; 925 926 active_irq = gic_c_read_4(sc, GICC_IAR); 927 928 /* 929 * Immediatly EOIR the SGIs, because doing so requires the other 930 * bits (ie CPU number), not just the IRQ number, and we do not 931 * have this information later. 932 */ 933 if ((active_irq & 0x3ff) <= GIC_LAST_SGI) 934 gic_c_write_4(sc, GICC_EOIR, active_irq); 935 active_irq &= 0x3FF; 936 937 if (active_irq == 0x3FF) { 938 if (last_irq == -1) 939 device_printf(sc->gic_dev, 940 "Spurious interrupt detected\n"); 941 return -1; 942 } 943 944 return active_irq; 945 } 946 947 static int 948 arm_gic_config(device_t dev, int irq, enum intr_trigger trig, 949 enum intr_polarity pol) 950 { 951 struct arm_gic_softc *sc = device_get_softc(dev); 952 uint32_t reg; 953 uint32_t mask; 954 955 /* Function is public-accessible, so validate input arguments */ 956 if ((irq < 0) || (irq >= sc->nirqs)) 957 goto invalid_args; 958 if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && 959 (trig != INTR_TRIGGER_CONFORM)) 960 goto invalid_args; 961 if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && 962 (pol != INTR_POLARITY_CONFORM)) 963 goto invalid_args; 964 965 mtx_lock_spin(&sc->mutex); 966 967 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 968 mask = (reg >> 2*(irq % 16)) & 0x3; 969 970 if (pol == INTR_POLARITY_LOW) { 971 mask &= ~GICD_ICFGR_POL_MASK; 972 mask |= GICD_ICFGR_POL_LOW; 973 } else if (pol == INTR_POLARITY_HIGH) { 974 mask &= ~GICD_ICFGR_POL_MASK; 975 mask |= GICD_ICFGR_POL_HIGH; 976 } 977 978 if (trig == INTR_TRIGGER_LEVEL) { 979 mask &= ~GICD_ICFGR_TRIG_MASK; 980 mask |= GICD_ICFGR_TRIG_LVL; 981 } else if (trig == INTR_TRIGGER_EDGE) { 982 mask &= ~GICD_ICFGR_TRIG_MASK; 983 mask |= GICD_ICFGR_TRIG_EDGE; 984 } 985 986 /* Set mask */ 987 reg = reg & ~(0x3 << 2*(irq % 16)); 988 reg = reg | (mask << 2*(irq % 16)); 989 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 990 991 mtx_unlock_spin(&sc->mutex); 992 993 return (0); 994 995 invalid_args: 996 device_printf(dev, "gic_config_irg, invalid parameters\n"); 997 return (EINVAL); 998 } 999 1000 1001 static void 1002 arm_gic_mask(device_t dev, int irq) 1003 { 1004 struct arm_gic_softc *sc = device_get_softc(dev); 1005 1006 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 1007 gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */ 1008 } 1009 1010 static void 1011 arm_gic_unmask(device_t dev, int irq) 1012 { 1013 struct arm_gic_softc *sc = device_get_softc(dev); 1014 1015 if (irq > GIC_LAST_SGI) 1016 arm_irq_memory_barrier(irq); 1017 1018 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 1019 } 1020 1021 #ifdef SMP 1022 static void 1023 arm_gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) 1024 { 1025 struct arm_gic_softc *sc = device_get_softc(dev); 1026 uint32_t val = 0, i; 1027 1028 for (i = 0; i < MAXCPU; i++) 1029 if (CPU_ISSET(i, &cpus)) 1030 val |= 1 << (16 + i); 1031 1032 gic_d_write_4(sc, GICD_SGIR(0), val | ipi); 1033 } 1034 1035 static int 1036 arm_gic_ipi_read(device_t dev, int i) 1037 { 1038 1039 if (i != -1) { 1040 /* 1041 * The intr code will automagically give the frame pointer 1042 * if the interrupt argument is 0. 1043 */ 1044 if ((unsigned int)i > 16) 1045 return (0); 1046 return (i); 1047 } 1048 1049 return (0x3ff); 1050 } 1051 1052 static void 1053 arm_gic_ipi_clear(device_t dev, int ipi) 1054 { 1055 /* no-op */ 1056 } 1057 #endif 1058 1059 static void 1060 gic_post_filter(void *arg) 1061 { 1062 struct arm_gic_softc *sc = gic_sc; 1063 uintptr_t irq = (uintptr_t) arg; 1064 1065 if (irq > GIC_LAST_SGI) 1066 arm_irq_memory_barrier(irq); 1067 gic_c_write_4(sc, GICC_EOIR, irq); 1068 } 1069 1070 static int 1071 gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) 1072 { 1073 1074 return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol)); 1075 } 1076 1077 void 1078 arm_mask_irq(uintptr_t nb) 1079 { 1080 1081 arm_gic_mask(gic_sc->gic_dev, nb); 1082 } 1083 1084 void 1085 arm_unmask_irq(uintptr_t nb) 1086 { 1087 1088 arm_gic_unmask(gic_sc->gic_dev, nb); 1089 } 1090 1091 int 1092 arm_get_next_irq(int last_irq) 1093 { 1094 1095 return (arm_gic_next_irq(gic_sc, last_irq)); 1096 } 1097 1098 #ifdef SMP 1099 void 1100 intr_pic_init_secondary(void) 1101 { 1102 1103 arm_gic_init_secondary(gic_sc->gic_dev); 1104 } 1105 1106 void 1107 pic_ipi_send(cpuset_t cpus, u_int ipi) 1108 { 1109 1110 arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi); 1111 } 1112 1113 int 1114 pic_ipi_read(int i) 1115 { 1116 1117 return (arm_gic_ipi_read(gic_sc->gic_dev, i)); 1118 } 1119 1120 void 1121 pic_ipi_clear(int ipi) 1122 { 1123 1124 arm_gic_ipi_clear(gic_sc->gic_dev, ipi); 1125 } 1126 #endif 1127 #endif /* ARM_INTRNG */ 1128 1129 static device_method_t arm_gic_methods[] = { 1130 /* Device interface */ 1131 DEVMETHOD(device_probe, arm_gic_probe), 1132 DEVMETHOD(device_attach, arm_gic_attach), 1133 #ifdef ARM_INTRNG 1134 /* Interrupt controller interface */ 1135 DEVMETHOD(pic_disable_source, arm_gic_disable_source), 1136 DEVMETHOD(pic_enable_intr, arm_gic_enable_intr), 1137 DEVMETHOD(pic_enable_source, arm_gic_enable_source), 1138 DEVMETHOD(pic_post_filter, arm_gic_post_filter), 1139 DEVMETHOD(pic_post_ithread, arm_gic_post_ithread), 1140 DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread), 1141 DEVMETHOD(pic_register, arm_gic_register), 1142 DEVMETHOD(pic_unregister, arm_gic_unregister), 1143 #ifdef SMP 1144 DEVMETHOD(pic_bind, arm_gic_bind), 1145 DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), 1146 DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), 1147 #endif 1148 #endif 1149 { 0, 0 } 1150 }; 1151 1152 static driver_t arm_gic_driver = { 1153 "gic", 1154 arm_gic_methods, 1155 sizeof(struct arm_gic_softc), 1156 }; 1157 1158 static devclass_t arm_gic_devclass; 1159 1160 EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, 1161 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1162 EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, 1163 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1164