1 /*- 2 * Copyright (c) 2011 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * Developed by Damjan Marion <damjan.marion@gmail.com> 6 * 7 * Based on OMAP4 GIC code by Ben Gray 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 3. The name of the company nor the name of the author may be used to 18 * endorse or promote products derived from this software without specific 19 * prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 22 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 23 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 24 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 25 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 26 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 27 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 28 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 29 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 #include <sys/cdefs.h> 35 __FBSDID("$FreeBSD$"); 36 37 #include "opt_platform.h" 38 39 #include "opt_platform.h" 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/bus.h> 44 #include <sys/kernel.h> 45 #include <sys/ktr.h> 46 #include <sys/module.h> 47 #include <sys/malloc.h> 48 #include <sys/rman.h> 49 #include <sys/pcpu.h> 50 #include <sys/proc.h> 51 #include <sys/cpuset.h> 52 #include <sys/lock.h> 53 #include <sys/mutex.h> 54 #include <sys/smp.h> 55 #ifdef ARM_INTRNG 56 #include <sys/sched.h> 57 #endif 58 #include <machine/bus.h> 59 #include <machine/intr.h> 60 #include <machine/smp.h> 61 62 #include <dev/fdt/fdt_common.h> 63 #include <dev/ofw/openfirm.h> 64 #include <dev/ofw/ofw_bus.h> 65 #include <dev/ofw/ofw_bus_subr.h> 66 67 #ifdef ARM_INTRNG 68 #include "pic_if.h" 69 #endif 70 71 /* We are using GICv2 register naming */ 72 73 /* Distributor Registers */ 74 #define GICD_CTLR 0x000 /* v1 ICDDCR */ 75 #define GICD_TYPER 0x004 /* v1 ICDICTR */ 76 #define GICD_IIDR 0x008 /* v1 ICDIIDR */ 77 #define GICD_IGROUPR(n) (0x0080 + ((n) * 4)) /* v1 ICDISER */ 78 #define GICD_ISENABLER(n) (0x0100 + ((n) * 4)) /* v1 ICDISER */ 79 #define GICD_ICENABLER(n) (0x0180 + ((n) * 4)) /* v1 ICDICER */ 80 #define GICD_ISPENDR(n) (0x0200 + ((n) * 4)) /* v1 ICDISPR */ 81 #define GICD_ICPENDR(n) (0x0280 + ((n) * 4)) /* v1 ICDICPR */ 82 #define GICD_ICACTIVER(n) (0x0380 + ((n) * 4)) /* v1 ICDABR */ 83 #define GICD_IPRIORITYR(n) (0x0400 + ((n) * 4)) /* v1 ICDIPR */ 84 #define GICD_ITARGETSR(n) (0x0800 + ((n) * 4)) /* v1 ICDIPTR */ 85 #define GICD_ICFGR(n) (0x0C00 + ((n) * 4)) /* v1 ICDICFR */ 86 #define GICD_SGIR(n) (0x0F00 + ((n) * 4)) /* v1 ICDSGIR */ 87 88 /* CPU Registers */ 89 #define GICC_CTLR 0x0000 /* v1 ICCICR */ 90 #define GICC_PMR 0x0004 /* v1 ICCPMR */ 91 #define GICC_BPR 0x0008 /* v1 ICCBPR */ 92 #define GICC_IAR 0x000C /* v1 ICCIAR */ 93 #define GICC_EOIR 0x0010 /* v1 ICCEOIR */ 94 #define GICC_RPR 0x0014 /* v1 ICCRPR */ 95 #define GICC_HPPIR 0x0018 /* v1 ICCHPIR */ 96 #define GICC_ABPR 0x001C /* v1 ICCABPR */ 97 #define GICC_IIDR 0x00FC /* v1 ICCIIDR*/ 98 99 #define GIC_FIRST_SGI 0 /* Irqs 0-15 are SGIs/IPIs. */ 100 #define GIC_LAST_SGI 15 101 #define GIC_FIRST_PPI 16 /* Irqs 16-31 are private (per */ 102 #define GIC_LAST_PPI 31 /* core) peripheral interrupts. */ 103 #define GIC_FIRST_SPI 32 /* Irqs 32+ are shared peripherals. */ 104 105 /* First bit is a polarity bit (0 - low, 1 - high) */ 106 #define GICD_ICFGR_POL_LOW (0 << 0) 107 #define GICD_ICFGR_POL_HIGH (1 << 0) 108 #define GICD_ICFGR_POL_MASK 0x1 109 /* Second bit is a trigger bit (0 - level, 1 - edge) */ 110 #define GICD_ICFGR_TRIG_LVL (0 << 1) 111 #define GICD_ICFGR_TRIG_EDGE (1 << 1) 112 #define GICD_ICFGR_TRIG_MASK 0x2 113 114 #ifndef GIC_DEFAULT_ICFGR_INIT 115 #define GIC_DEFAULT_ICFGR_INIT 0x00000000 116 #endif 117 118 #ifdef ARM_INTRNG 119 static u_int gic_irq_cpu; 120 static int arm_gic_intr(void *); 121 static int arm_gic_bind(device_t dev, struct arm_irqsrc *isrc); 122 #endif 123 124 struct arm_gic_softc { 125 device_t gic_dev; 126 #ifdef ARM_INTRNG 127 void * gic_intrhand; 128 struct arm_irqsrc ** gic_irqs; 129 #endif 130 struct resource * gic_res[3]; 131 bus_space_tag_t gic_c_bst; 132 bus_space_tag_t gic_d_bst; 133 bus_space_handle_t gic_c_bsh; 134 bus_space_handle_t gic_d_bsh; 135 uint8_t ver; 136 struct mtx mutex; 137 uint32_t nirqs; 138 }; 139 140 static struct resource_spec arm_gic_spec[] = { 141 { SYS_RES_MEMORY, 0, RF_ACTIVE }, /* Distributor registers */ 142 { SYS_RES_MEMORY, 1, RF_ACTIVE }, /* CPU Interrupt Intf. registers */ 143 #ifdef ARM_INTRNG 144 { SYS_RES_IRQ, 0, RF_ACTIVE | RF_OPTIONAL }, /* Parent interrupt */ 145 #endif 146 { -1, 0 } 147 }; 148 149 static struct arm_gic_softc *gic_sc = NULL; 150 151 #define gic_c_read_4(_sc, _reg) \ 152 bus_space_read_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg)) 153 #define gic_c_write_4(_sc, _reg, _val) \ 154 bus_space_write_4((_sc)->gic_c_bst, (_sc)->gic_c_bsh, (_reg), (_val)) 155 #define gic_d_read_4(_sc, _reg) \ 156 bus_space_read_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg)) 157 #define gic_d_write_1(_sc, _reg, _val) \ 158 bus_space_write_1((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 159 #define gic_d_write_4(_sc, _reg, _val) \ 160 bus_space_write_4((_sc)->gic_d_bst, (_sc)->gic_d_bsh, (_reg), (_val)) 161 162 #ifndef ARM_INTRNG 163 static int gic_config_irq(int irq, enum intr_trigger trig, 164 enum intr_polarity pol); 165 static void gic_post_filter(void *); 166 #endif 167 168 static struct ofw_compat_data compat_data[] = { 169 {"arm,gic", true}, /* Non-standard, used in FreeBSD dts. */ 170 {"arm,gic-400", true}, 171 {"arm,cortex-a15-gic", true}, 172 {"arm,cortex-a9-gic", true}, 173 {"arm,cortex-a7-gic", true}, 174 {"arm,arm11mp-gic", true}, 175 {"brcm,brahma-b15-gic", true}, 176 {NULL, false} 177 }; 178 179 static int 180 arm_gic_probe(device_t dev) 181 { 182 183 if (!ofw_bus_status_okay(dev)) 184 return (ENXIO); 185 186 if (!ofw_bus_search_compatible(dev, compat_data)->ocd_data) 187 return (ENXIO); 188 device_set_desc(dev, "ARM Generic Interrupt Controller"); 189 return (BUS_PROBE_DEFAULT); 190 } 191 192 #ifdef ARM_INTRNG 193 static inline void 194 gic_irq_unmask(struct arm_gic_softc *sc, u_int irq) 195 { 196 197 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 198 } 199 200 static inline void 201 gic_irq_mask(struct arm_gic_softc *sc, u_int irq) 202 { 203 204 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 205 } 206 #endif 207 208 #ifdef SMP 209 #ifdef ARM_INTRNG 210 static void 211 arm_gic_init_secondary(device_t dev) 212 { 213 struct arm_gic_softc *sc = device_get_softc(dev); 214 struct arm_irqsrc *isrc; 215 u_int irq; 216 217 for (irq = 0; irq < sc->nirqs; irq += 4) 218 gic_d_write_4(sc, GICD_IPRIORITYR(irq >> 2), 0); 219 220 /* Set all the interrupts to be in Group 0 (secure) */ 221 for (irq = 0; irq < sc->nirqs; irq += 32) { 222 gic_d_write_4(sc, GICD_IGROUPR(irq >> 5), 0); 223 } 224 225 /* Enable CPU interface */ 226 gic_c_write_4(sc, GICC_CTLR, 1); 227 228 /* Set priority mask register. */ 229 gic_c_write_4(sc, GICC_PMR, 0xff); 230 231 /* Enable interrupt distribution */ 232 gic_d_write_4(sc, GICD_CTLR, 0x01); 233 234 /* Unmask attached SGI interrupts. */ 235 for (irq = GIC_FIRST_SGI; irq <= GIC_LAST_SGI; irq++) { 236 isrc = sc->gic_irqs[irq]; 237 if (isrc != NULL && isrc->isrc_handlers != 0) { 238 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 239 gic_irq_unmask(sc, irq); 240 } 241 } 242 243 /* Unmask attached PPI interrupts. */ 244 for (irq = GIC_FIRST_PPI; irq <= GIC_LAST_PPI; irq++) { 245 isrc = sc->gic_irqs[irq]; 246 if (isrc == NULL || isrc->isrc_handlers == 0) 247 continue; 248 if (isrc->isrc_flags & ARM_ISRCF_BOUND) { 249 if (CPU_ISSET(PCPU_GET(cpuid), &isrc->isrc_cpu)) 250 gic_irq_unmask(sc, irq); 251 } else { 252 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 253 gic_irq_unmask(sc, irq); 254 } 255 } 256 } 257 #else 258 static void 259 arm_gic_init_secondary(device_t dev) 260 { 261 struct arm_gic_softc *sc = device_get_softc(dev); 262 int i; 263 264 for (i = 0; i < sc->nirqs; i += 4) 265 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 266 267 /* Set all the interrupts to be in Group 0 (secure) */ 268 for (i = 0; i < sc->nirqs; i += 32) { 269 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 270 } 271 272 /* Enable CPU interface */ 273 gic_c_write_4(sc, GICC_CTLR, 1); 274 275 /* Set priority mask register. */ 276 gic_c_write_4(sc, GICC_PMR, 0xff); 277 278 /* Enable interrupt distribution */ 279 gic_d_write_4(sc, GICD_CTLR, 0x01); 280 281 /* 282 * Activate the timer interrupts: virtual, secure, and non-secure. 283 */ 284 gic_d_write_4(sc, GICD_ISENABLER(27 >> 5), (1UL << (27 & 0x1F))); 285 gic_d_write_4(sc, GICD_ISENABLER(29 >> 5), (1UL << (29 & 0x1F))); 286 gic_d_write_4(sc, GICD_ISENABLER(30 >> 5), (1UL << (30 & 0x1F))); 287 } 288 #endif /* ARM_INTRNG */ 289 #endif /* SMP */ 290 291 #ifndef ARM_INTRNG 292 int 293 gic_decode_fdt(phandle_t iparent, pcell_t *intr, int *interrupt, 294 int *trig, int *pol) 295 { 296 static u_int num_intr_cells; 297 static phandle_t self; 298 struct ofw_compat_data *ocd; 299 300 if (self == 0) { 301 for (ocd = compat_data; ocd->ocd_str != NULL; ocd++) { 302 if (fdt_is_compatible(iparent, ocd->ocd_str)) { 303 self = iparent; 304 break; 305 } 306 } 307 } 308 if (self != iparent) 309 return (ENXIO); 310 311 if (num_intr_cells == 0) { 312 if (OF_searchencprop(OF_node_from_xref(iparent), 313 "#interrupt-cells", &num_intr_cells, 314 sizeof(num_intr_cells)) == -1) { 315 num_intr_cells = 1; 316 } 317 } 318 319 if (num_intr_cells == 1) { 320 *interrupt = fdt32_to_cpu(intr[0]); 321 *trig = INTR_TRIGGER_CONFORM; 322 *pol = INTR_POLARITY_CONFORM; 323 } else { 324 if (fdt32_to_cpu(intr[0]) == 0) 325 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_SPI; 326 else 327 *interrupt = fdt32_to_cpu(intr[1]) + GIC_FIRST_PPI; 328 /* 329 * In intr[2], bits[3:0] are trigger type and level flags. 330 * 1 = low-to-high edge triggered 331 * 2 = high-to-low edge triggered 332 * 4 = active high level-sensitive 333 * 8 = active low level-sensitive 334 * The hardware only supports active-high-level or rising-edge. 335 */ 336 if (fdt32_to_cpu(intr[2]) & 0x0a) { 337 printf("unsupported trigger/polarity configuration " 338 "0x%2x\n", fdt32_to_cpu(intr[2]) & 0x0f); 339 return (ENOTSUP); 340 } 341 *pol = INTR_POLARITY_CONFORM; 342 if (fdt32_to_cpu(intr[2]) & 0x01) 343 *trig = INTR_TRIGGER_EDGE; 344 else 345 *trig = INTR_TRIGGER_LEVEL; 346 } 347 return (0); 348 } 349 #endif 350 351 #ifdef ARM_INTRNG 352 static inline intptr_t 353 gic_xref(device_t dev) 354 { 355 #ifdef FDT 356 return (OF_xref_from_node(ofw_bus_get_node(dev))); 357 #else 358 return (0); 359 #endif 360 } 361 #endif 362 363 static int 364 arm_gic_attach(device_t dev) 365 { 366 struct arm_gic_softc *sc; 367 int i; 368 uint32_t icciidr; 369 #ifdef ARM_INTRNG 370 intptr_t xref = gic_xref(dev); 371 #endif 372 373 if (gic_sc) 374 return (ENXIO); 375 376 sc = device_get_softc(dev); 377 378 if (bus_alloc_resources(dev, arm_gic_spec, sc->gic_res)) { 379 device_printf(dev, "could not allocate resources\n"); 380 return (ENXIO); 381 } 382 383 sc->gic_dev = dev; 384 gic_sc = sc; 385 386 /* Initialize mutex */ 387 mtx_init(&sc->mutex, "GIC lock", "", MTX_SPIN); 388 389 /* Distributor Interface */ 390 sc->gic_d_bst = rman_get_bustag(sc->gic_res[0]); 391 sc->gic_d_bsh = rman_get_bushandle(sc->gic_res[0]); 392 393 /* CPU Interface */ 394 sc->gic_c_bst = rman_get_bustag(sc->gic_res[1]); 395 sc->gic_c_bsh = rman_get_bushandle(sc->gic_res[1]); 396 397 /* Disable interrupt forwarding to the CPU interface */ 398 gic_d_write_4(sc, GICD_CTLR, 0x00); 399 400 /* Get the number of interrupts */ 401 sc->nirqs = gic_d_read_4(sc, GICD_TYPER); 402 sc->nirqs = 32 * ((sc->nirqs & 0x1f) + 1); 403 404 #ifdef ARM_INTRNG 405 sc->gic_irqs = malloc(sc->nirqs * sizeof (*sc->gic_irqs), M_DEVBUF, 406 M_WAITOK | M_ZERO); 407 #else 408 /* Set up function pointers */ 409 arm_post_filter = gic_post_filter; 410 arm_config_irq = gic_config_irq; 411 #endif 412 413 icciidr = gic_c_read_4(sc, GICC_IIDR); 414 device_printf(dev,"pn 0x%x, arch 0x%x, rev 0x%x, implementer 0x%x irqs %u\n", 415 icciidr>>20, (icciidr>>16) & 0xF, (icciidr>>12) & 0xf, 416 (icciidr & 0xfff), sc->nirqs); 417 418 /* Set all global interrupts to be level triggered, active low. */ 419 for (i = 32; i < sc->nirqs; i += 16) { 420 gic_d_write_4(sc, GICD_ICFGR(i >> 4), GIC_DEFAULT_ICFGR_INIT); 421 } 422 423 /* Disable all interrupts. */ 424 for (i = 32; i < sc->nirqs; i += 32) { 425 gic_d_write_4(sc, GICD_ICENABLER(i >> 5), 0xFFFFFFFF); 426 } 427 428 for (i = 0; i < sc->nirqs; i += 4) { 429 gic_d_write_4(sc, GICD_IPRIORITYR(i >> 2), 0); 430 gic_d_write_4(sc, GICD_ITARGETSR(i >> 2), 431 1 << 0 | 1 << 8 | 1 << 16 | 1 << 24); 432 } 433 434 /* Set all the interrupts to be in Group 0 (secure) */ 435 for (i = 0; i < sc->nirqs; i += 32) { 436 gic_d_write_4(sc, GICD_IGROUPR(i >> 5), 0); 437 } 438 439 /* Enable CPU interface */ 440 gic_c_write_4(sc, GICC_CTLR, 1); 441 442 /* Set priority mask register. */ 443 gic_c_write_4(sc, GICC_PMR, 0xff); 444 445 /* Enable interrupt distribution */ 446 gic_d_write_4(sc, GICD_CTLR, 0x01); 447 #ifndef ARM_INTRNG 448 return (0); 449 #else 450 /* 451 * Now, when everything is initialized, it's right time to 452 * register interrupt controller to interrupt framefork. 453 */ 454 if (arm_pic_register(dev, xref) != 0) { 455 device_printf(dev, "could not register PIC\n"); 456 goto cleanup; 457 } 458 459 if (sc->gic_res[2] == NULL) { 460 if (arm_pic_claim_root(dev, xref, arm_gic_intr, sc, 461 GIC_LAST_SGI - GIC_FIRST_SGI + 1) != 0) { 462 device_printf(dev, "could not set PIC as a root\n"); 463 arm_pic_unregister(dev, xref); 464 goto cleanup; 465 } 466 } else { 467 if (bus_setup_intr(dev, sc->gic_res[2], INTR_TYPE_CLK, 468 arm_gic_intr, NULL, sc, &sc->gic_intrhand)) { 469 device_printf(dev, "could not setup irq handler\n"); 470 arm_pic_unregister(dev, xref); 471 goto cleanup; 472 } 473 } 474 475 return (0); 476 477 cleanup: 478 /* 479 * XXX - not implemented arm_gic_detach() should be called ! 480 */ 481 if (sc->gic_irqs != NULL) 482 free(sc->gic_irqs, M_DEVBUF); 483 bus_release_resources(dev, arm_gic_spec, sc->gic_res); 484 return(ENXIO); 485 #endif 486 } 487 488 #ifdef ARM_INTRNG 489 static int 490 arm_gic_intr(void *arg) 491 { 492 struct arm_gic_softc *sc = arg; 493 struct arm_irqsrc *isrc; 494 uint32_t irq_active_reg, irq; 495 struct trapframe *tf; 496 497 irq_active_reg = gic_c_read_4(sc, GICC_IAR); 498 irq = irq_active_reg & 0x3FF; 499 500 /* 501 * 1. We do EOI here because recent read value from active interrupt 502 * register must be used for it. Another approach is to save this 503 * value into associated interrupt source. 504 * 2. EOI must be done on same CPU where interrupt has fired. Thus 505 * we must ensure that interrupted thread does not migrate to 506 * another CPU. 507 * 3. EOI cannot be delayed by any preemption which could happen on 508 * critical_exit() used in MI intr code, when interrupt thread is 509 * scheduled. See next point. 510 * 4. IPI_RENDEZVOUS assumes that no preemption is permitted during 511 * an action and any use of critical_exit() could break this 512 * assumption. See comments within smp_rendezvous_action(). 513 * 5. We always return FILTER_HANDLED as this is an interrupt 514 * controller dispatch function. Otherwise, in cascaded interrupt 515 * case, the whole interrupt subtree would be masked. 516 */ 517 518 if (irq >= sc->nirqs) { 519 device_printf(sc->gic_dev, "Spurious interrupt detected\n"); 520 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 521 return (FILTER_HANDLED); 522 } 523 524 tf = curthread->td_intr_frame; 525 dispatch_irq: 526 isrc = sc->gic_irqs[irq]; 527 if (isrc == NULL) { 528 device_printf(sc->gic_dev, "Stray interrupt %u detected\n", irq); 529 gic_irq_mask(sc, irq); 530 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 531 goto next_irq; 532 } 533 534 /* 535 * Note that GIC_FIRST_SGI is zero and is not used in 'if' statement 536 * as compiler complains that comparing u_int >= 0 is always true. 537 */ 538 if (irq <= GIC_LAST_SGI) { 539 #ifdef SMP 540 /* Call EOI for all IPI before dispatch. */ 541 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 542 arm_ipi_dispatch(isrc, tf); 543 goto next_irq; 544 #else 545 printf("SGI %u on UP system detected\n", irq - GIC_FIRST_SGI); 546 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 547 goto next_irq; 548 #endif 549 } 550 551 if (isrc->isrc_trig == INTR_TRIGGER_EDGE) 552 gic_c_write_4(sc, GICC_EOIR, irq_active_reg); 553 554 arm_irq_dispatch(isrc, tf); 555 556 next_irq: 557 // arm_irq_memory_barrier(irq); /* XXX */ 558 // irq_active_reg = gic_c_read_4(sc, GICC_IAR); 559 // irq = irq_active_reg & 0x3FF; 560 if (0 && irq < sc->nirqs) 561 goto dispatch_irq; 562 563 return (FILTER_HANDLED); 564 } 565 566 static int 567 gic_attach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq) 568 { 569 const char *name; 570 571 /* 572 * 1. The link between ISRC and controller must be set atomically. 573 * 2. Just do things only once in rare case when consumers 574 * of shared interrupt came here at the same moment. 575 */ 576 mtx_lock_spin(&sc->mutex); 577 if (sc->gic_irqs[irq] != NULL) { 578 mtx_unlock_spin(&sc->mutex); 579 return (sc->gic_irqs[irq] == isrc ? 0 : EEXIST); 580 } 581 sc->gic_irqs[irq] = isrc; 582 isrc->isrc_data = irq; 583 mtx_unlock_spin(&sc->mutex); 584 585 name = device_get_nameunit(sc->gic_dev); 586 if (irq <= GIC_LAST_SGI) 587 arm_irq_set_name(isrc, "%s,i%u", name, irq - GIC_FIRST_SGI); 588 else if (irq <= GIC_LAST_PPI) 589 arm_irq_set_name(isrc, "%s,p%u", name, irq - GIC_FIRST_PPI); 590 else 591 arm_irq_set_name(isrc, "%s,s%u", name, irq - GIC_FIRST_SPI); 592 return (0); 593 } 594 595 static int 596 gic_detach_isrc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int irq) 597 { 598 599 mtx_lock_spin(&sc->mutex); 600 if (sc->gic_irqs[irq] != isrc) { 601 mtx_unlock_spin(&sc->mutex); 602 return (sc->gic_irqs[irq] == NULL ? 0 : EINVAL); 603 } 604 sc->gic_irqs[irq] = NULL; 605 isrc->isrc_data = 0; 606 mtx_unlock_spin(&sc->mutex); 607 608 arm_irq_set_name(isrc, ""); 609 return (0); 610 } 611 612 static void 613 gic_config(struct arm_gic_softc *sc, u_int irq, enum intr_trigger trig, 614 enum intr_polarity pol) 615 { 616 uint32_t reg; 617 uint32_t mask; 618 619 if (irq < GIC_FIRST_SPI) 620 return; 621 622 mtx_lock_spin(&sc->mutex); 623 624 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 625 mask = (reg >> 2*(irq % 16)) & 0x3; 626 627 if (pol == INTR_POLARITY_LOW) { 628 mask &= ~GICD_ICFGR_POL_MASK; 629 mask |= GICD_ICFGR_POL_LOW; 630 } else if (pol == INTR_POLARITY_HIGH) { 631 mask &= ~GICD_ICFGR_POL_MASK; 632 mask |= GICD_ICFGR_POL_HIGH; 633 } 634 635 if (trig == INTR_TRIGGER_LEVEL) { 636 mask &= ~GICD_ICFGR_TRIG_MASK; 637 mask |= GICD_ICFGR_TRIG_LVL; 638 } else if (trig == INTR_TRIGGER_EDGE) { 639 mask &= ~GICD_ICFGR_TRIG_MASK; 640 mask |= GICD_ICFGR_TRIG_EDGE; 641 } 642 643 /* Set mask */ 644 reg = reg & ~(0x3 << 2*(irq % 16)); 645 reg = reg | (mask << 2*(irq % 16)); 646 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 647 648 mtx_unlock_spin(&sc->mutex); 649 } 650 651 static int 652 gic_bind(struct arm_gic_softc *sc, u_int irq, cpuset_t *cpus) 653 { 654 uint32_t cpu, end, mask; 655 656 end = min(mp_ncpus, 8); 657 for (cpu = end; cpu < MAXCPU; cpu++) 658 if (CPU_ISSET(cpu, cpus)) 659 return (EINVAL); 660 661 for (mask = 0, cpu = 0; cpu < end; cpu++) 662 if (CPU_ISSET(cpu, cpus)) 663 mask |= 1 << cpu; 664 665 gic_d_write_1(sc, GICD_ITARGETSR(0) + irq, mask); 666 return (0); 667 } 668 669 static int 670 gic_irq_from_nspc(struct arm_gic_softc *sc, u_int type, u_int num, u_int *irqp) 671 { 672 673 switch (type) { 674 case ARM_IRQ_NSPC_PLAIN: 675 *irqp = num; 676 return (*irqp < sc->nirqs ? 0 : EINVAL); 677 678 case ARM_IRQ_NSPC_IRQ: 679 *irqp = num + GIC_FIRST_PPI; 680 return (*irqp < sc->nirqs ? 0 : EINVAL); 681 682 case ARM_IRQ_NSPC_IPI: 683 *irqp = num + GIC_FIRST_SGI; 684 return (*irqp < GIC_LAST_SGI ? 0 : EINVAL); 685 686 default: 687 return (EINVAL); 688 } 689 } 690 691 static int 692 gic_map_nspc(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp) 693 { 694 int error; 695 696 error = gic_irq_from_nspc(sc, isrc->isrc_nspc_type, isrc->isrc_nspc_num, 697 irqp); 698 if (error != 0) 699 return (error); 700 return (gic_attach_isrc(sc, isrc, *irqp)); 701 } 702 703 #ifdef FDT 704 static int 705 gic_map_fdt(struct arm_gic_softc *sc, struct arm_irqsrc *isrc, u_int *irqp) 706 { 707 u_int irq, tripol; 708 enum intr_trigger trig; 709 enum intr_polarity pol; 710 int error; 711 712 if (isrc->isrc_ncells == 1) { 713 irq = isrc->isrc_cells[0]; 714 pol = INTR_POLARITY_CONFORM; 715 trig = INTR_TRIGGER_CONFORM; 716 } else if (isrc->isrc_ncells == 3) { 717 if (isrc->isrc_cells[0] == 0) 718 irq = isrc->isrc_cells[1] + GIC_FIRST_SPI; 719 else 720 irq = isrc->isrc_cells[1] + GIC_FIRST_PPI; 721 722 /* 723 * In intr[2], bits[3:0] are trigger type and level flags. 724 * 1 = low-to-high edge triggered 725 * 2 = high-to-low edge triggered 726 * 4 = active high level-sensitive 727 * 8 = active low level-sensitive 728 * The hardware only supports active-high-level or rising-edge. 729 */ 730 tripol = isrc->isrc_cells[2]; 731 if (tripol & 0x0a) { 732 printf("unsupported trigger/polarity configuration " 733 "0x%2x\n", tripol & 0x0f); 734 return (ENOTSUP); 735 } 736 pol = INTR_POLARITY_CONFORM; 737 if (tripol & 0x01) 738 trig = INTR_TRIGGER_EDGE; 739 else 740 trig = INTR_TRIGGER_LEVEL; 741 } else 742 return (EINVAL); 743 744 if (irq >= sc->nirqs) 745 return (EINVAL); 746 747 error = gic_attach_isrc(sc, isrc, irq); 748 if (error != 0) 749 return (error); 750 751 isrc->isrc_nspc_type = ARM_IRQ_NSPC_PLAIN; 752 isrc->isrc_nspc_num = irq; 753 isrc->isrc_trig = trig; 754 isrc->isrc_pol = pol; 755 756 *irqp = irq; 757 return (0); 758 } 759 #endif 760 761 static int 762 arm_gic_register(device_t dev, struct arm_irqsrc *isrc, boolean_t *is_percpu) 763 { 764 struct arm_gic_softc *sc = device_get_softc(dev); 765 u_int irq; 766 int error; 767 768 if (isrc->isrc_type == ARM_ISRCT_NAMESPACE) 769 error = gic_map_nspc(sc, isrc, &irq); 770 #ifdef FDT 771 else if (isrc->isrc_type == ARM_ISRCT_FDT) 772 error = gic_map_fdt(sc, isrc, &irq); 773 #endif 774 else 775 return (EINVAL); 776 777 if (error == 0) 778 *is_percpu = irq < GIC_FIRST_SPI ? TRUE : FALSE; 779 return (error); 780 } 781 782 static void 783 arm_gic_enable_intr(device_t dev, struct arm_irqsrc *isrc) 784 { 785 struct arm_gic_softc *sc = device_get_softc(dev); 786 u_int irq = isrc->isrc_data; 787 788 if (isrc->isrc_trig == INTR_TRIGGER_CONFORM) 789 isrc->isrc_trig = INTR_TRIGGER_LEVEL; 790 791 /* 792 * XXX - In case that per CPU interrupt is going to be enabled in time 793 * when SMP is already started, we need some IPI call which 794 * enables it on others CPUs. Further, it's more complicated as 795 * pic_enable_source() and pic_disable_source() should act on 796 * per CPU basis only. Thus, it should be solved here somehow. 797 */ 798 if (isrc->isrc_flags & ARM_ISRCF_PERCPU) 799 CPU_SET(PCPU_GET(cpuid), &isrc->isrc_cpu); 800 801 gic_config(sc, irq, isrc->isrc_trig, isrc->isrc_pol); 802 arm_gic_bind(dev, isrc); 803 } 804 805 static void 806 arm_gic_enable_source(device_t dev, struct arm_irqsrc *isrc) 807 { 808 struct arm_gic_softc *sc = device_get_softc(dev); 809 u_int irq = isrc->isrc_data; 810 811 arm_irq_memory_barrier(irq); 812 gic_irq_unmask(sc, irq); 813 } 814 815 static void 816 arm_gic_disable_source(device_t dev, struct arm_irqsrc *isrc) 817 { 818 struct arm_gic_softc *sc = device_get_softc(dev); 819 u_int irq = isrc->isrc_data; 820 821 gic_irq_mask(sc, irq); 822 } 823 824 static int 825 arm_gic_unregister(device_t dev, struct arm_irqsrc *isrc) 826 { 827 struct arm_gic_softc *sc = device_get_softc(dev); 828 u_int irq = isrc->isrc_data; 829 830 return (gic_detach_isrc(sc, isrc, irq)); 831 } 832 833 static void 834 arm_gic_pre_ithread(device_t dev, struct arm_irqsrc *isrc) 835 { 836 struct arm_gic_softc *sc = device_get_softc(dev); 837 838 arm_gic_disable_source(dev, isrc); 839 gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data); 840 } 841 842 static void 843 arm_gic_post_ithread(device_t dev, struct arm_irqsrc *isrc) 844 { 845 846 arm_irq_memory_barrier(0); 847 arm_gic_enable_source(dev, isrc); 848 } 849 850 static void 851 arm_gic_post_filter(device_t dev, struct arm_irqsrc *isrc) 852 { 853 struct arm_gic_softc *sc = device_get_softc(dev); 854 855 /* EOI for edge-triggered done earlier. */ 856 if (isrc->isrc_trig == INTR_TRIGGER_EDGE) 857 return; 858 859 arm_irq_memory_barrier(0); 860 gic_c_write_4(sc, GICC_EOIR, isrc->isrc_data); 861 } 862 863 #ifdef SMP 864 static int 865 arm_gic_bind(device_t dev, struct arm_irqsrc *isrc) 866 { 867 struct arm_gic_softc *sc = device_get_softc(dev); 868 uint32_t irq = isrc->isrc_data; 869 870 if (irq < GIC_FIRST_SPI) 871 return (EINVAL); 872 873 if (CPU_EMPTY(&isrc->isrc_cpu)) { 874 gic_irq_cpu = arm_irq_next_cpu(gic_irq_cpu, &all_cpus); 875 CPU_SETOF(gic_irq_cpu, &isrc->isrc_cpu); 876 } 877 return (gic_bind(sc, irq, &isrc->isrc_cpu)); 878 } 879 880 static void 881 arm_gic_ipi_send(device_t dev, struct arm_irqsrc *isrc, cpuset_t cpus) 882 { 883 struct arm_gic_softc *sc = device_get_softc(dev); 884 uint32_t irq, val = 0, i; 885 886 irq = isrc->isrc_data; 887 888 for (i = 0; i < MAXCPU; i++) 889 if (CPU_ISSET(i, &cpus)) 890 val |= 1 << (16 + i); 891 892 gic_d_write_4(sc, GICD_SGIR(0), val | irq); 893 } 894 #endif 895 #else 896 static int 897 arm_gic_next_irq(struct arm_gic_softc *sc, int last_irq) 898 { 899 uint32_t active_irq; 900 901 active_irq = gic_c_read_4(sc, GICC_IAR); 902 903 /* 904 * Immediatly EOIR the SGIs, because doing so requires the other 905 * bits (ie CPU number), not just the IRQ number, and we do not 906 * have this information later. 907 */ 908 if ((active_irq & 0x3ff) <= GIC_LAST_SGI) 909 gic_c_write_4(sc, GICC_EOIR, active_irq); 910 active_irq &= 0x3FF; 911 912 if (active_irq == 0x3FF) { 913 if (last_irq == -1) 914 printf("Spurious interrupt detected\n"); 915 return -1; 916 } 917 918 return active_irq; 919 } 920 921 static int 922 arm_gic_config(device_t dev, int irq, enum intr_trigger trig, 923 enum intr_polarity pol) 924 { 925 struct arm_gic_softc *sc = device_get_softc(dev); 926 uint32_t reg; 927 uint32_t mask; 928 929 /* Function is public-accessible, so validate input arguments */ 930 if ((irq < 0) || (irq >= sc->nirqs)) 931 goto invalid_args; 932 if ((trig != INTR_TRIGGER_EDGE) && (trig != INTR_TRIGGER_LEVEL) && 933 (trig != INTR_TRIGGER_CONFORM)) 934 goto invalid_args; 935 if ((pol != INTR_POLARITY_HIGH) && (pol != INTR_POLARITY_LOW) && 936 (pol != INTR_POLARITY_CONFORM)) 937 goto invalid_args; 938 939 mtx_lock_spin(&sc->mutex); 940 941 reg = gic_d_read_4(sc, GICD_ICFGR(irq >> 4)); 942 mask = (reg >> 2*(irq % 16)) & 0x3; 943 944 if (pol == INTR_POLARITY_LOW) { 945 mask &= ~GICD_ICFGR_POL_MASK; 946 mask |= GICD_ICFGR_POL_LOW; 947 } else if (pol == INTR_POLARITY_HIGH) { 948 mask &= ~GICD_ICFGR_POL_MASK; 949 mask |= GICD_ICFGR_POL_HIGH; 950 } 951 952 if (trig == INTR_TRIGGER_LEVEL) { 953 mask &= ~GICD_ICFGR_TRIG_MASK; 954 mask |= GICD_ICFGR_TRIG_LVL; 955 } else if (trig == INTR_TRIGGER_EDGE) { 956 mask &= ~GICD_ICFGR_TRIG_MASK; 957 mask |= GICD_ICFGR_TRIG_EDGE; 958 } 959 960 /* Set mask */ 961 reg = reg & ~(0x3 << 2*(irq % 16)); 962 reg = reg | (mask << 2*(irq % 16)); 963 gic_d_write_4(sc, GICD_ICFGR(irq >> 4), reg); 964 965 mtx_unlock_spin(&sc->mutex); 966 967 return (0); 968 969 invalid_args: 970 device_printf(dev, "gic_config_irg, invalid parameters\n"); 971 return (EINVAL); 972 } 973 974 975 static void 976 arm_gic_mask(device_t dev, int irq) 977 { 978 struct arm_gic_softc *sc = device_get_softc(dev); 979 980 gic_d_write_4(sc, GICD_ICENABLER(irq >> 5), (1UL << (irq & 0x1F))); 981 gic_c_write_4(sc, GICC_EOIR, irq); /* XXX - not allowed */ 982 } 983 984 static void 985 arm_gic_unmask(device_t dev, int irq) 986 { 987 struct arm_gic_softc *sc = device_get_softc(dev); 988 989 if (irq > GIC_LAST_SGI) 990 arm_irq_memory_barrier(irq); 991 992 gic_d_write_4(sc, GICD_ISENABLER(irq >> 5), (1UL << (irq & 0x1F))); 993 } 994 995 #ifdef SMP 996 static void 997 arm_gic_ipi_send(device_t dev, cpuset_t cpus, u_int ipi) 998 { 999 struct arm_gic_softc *sc = device_get_softc(dev); 1000 uint32_t val = 0, i; 1001 1002 for (i = 0; i < MAXCPU; i++) 1003 if (CPU_ISSET(i, &cpus)) 1004 val |= 1 << (16 + i); 1005 1006 gic_d_write_4(sc, GICD_SGIR(0), val | ipi); 1007 } 1008 1009 static int 1010 arm_gic_ipi_read(device_t dev, int i) 1011 { 1012 1013 if (i != -1) { 1014 /* 1015 * The intr code will automagically give the frame pointer 1016 * if the interrupt argument is 0. 1017 */ 1018 if ((unsigned int)i > 16) 1019 return (0); 1020 return (i); 1021 } 1022 1023 return (0x3ff); 1024 } 1025 1026 static void 1027 arm_gic_ipi_clear(device_t dev, int ipi) 1028 { 1029 /* no-op */ 1030 } 1031 #endif 1032 1033 static void 1034 gic_post_filter(void *arg) 1035 { 1036 struct arm_gic_softc *sc = gic_sc; 1037 uintptr_t irq = (uintptr_t) arg; 1038 1039 if (irq > GIC_LAST_SGI) 1040 arm_irq_memory_barrier(irq); 1041 gic_c_write_4(sc, GICC_EOIR, irq); 1042 } 1043 1044 static int 1045 gic_config_irq(int irq, enum intr_trigger trig, enum intr_polarity pol) 1046 { 1047 1048 return (arm_gic_config(gic_sc->gic_dev, irq, trig, pol)); 1049 } 1050 1051 void 1052 arm_mask_irq(uintptr_t nb) 1053 { 1054 1055 arm_gic_mask(gic_sc->gic_dev, nb); 1056 } 1057 1058 void 1059 arm_unmask_irq(uintptr_t nb) 1060 { 1061 1062 arm_gic_unmask(gic_sc->gic_dev, nb); 1063 } 1064 1065 int 1066 arm_get_next_irq(int last_irq) 1067 { 1068 1069 return (arm_gic_next_irq(gic_sc, last_irq)); 1070 } 1071 1072 #ifdef SMP 1073 void 1074 arm_pic_init_secondary(void) 1075 { 1076 1077 arm_gic_init_secondary(gic_sc->gic_dev); 1078 } 1079 1080 void 1081 pic_ipi_send(cpuset_t cpus, u_int ipi) 1082 { 1083 1084 arm_gic_ipi_send(gic_sc->gic_dev, cpus, ipi); 1085 } 1086 1087 int 1088 pic_ipi_read(int i) 1089 { 1090 1091 return (arm_gic_ipi_read(gic_sc->gic_dev, i)); 1092 } 1093 1094 void 1095 pic_ipi_clear(int ipi) 1096 { 1097 1098 arm_gic_ipi_clear(gic_sc->gic_dev, ipi); 1099 } 1100 #endif 1101 #endif /* ARM_INTRNG */ 1102 1103 static device_method_t arm_gic_methods[] = { 1104 /* Device interface */ 1105 DEVMETHOD(device_probe, arm_gic_probe), 1106 DEVMETHOD(device_attach, arm_gic_attach), 1107 #ifdef ARM_INTRNG 1108 /* Interrupt controller interface */ 1109 DEVMETHOD(pic_disable_source, arm_gic_disable_source), 1110 DEVMETHOD(pic_enable_intr, arm_gic_enable_intr), 1111 DEVMETHOD(pic_enable_source, arm_gic_enable_source), 1112 DEVMETHOD(pic_post_filter, arm_gic_post_filter), 1113 DEVMETHOD(pic_post_ithread, arm_gic_post_ithread), 1114 DEVMETHOD(pic_pre_ithread, arm_gic_pre_ithread), 1115 DEVMETHOD(pic_register, arm_gic_register), 1116 DEVMETHOD(pic_unregister, arm_gic_unregister), 1117 #ifdef SMP 1118 DEVMETHOD(pic_bind, arm_gic_bind), 1119 DEVMETHOD(pic_init_secondary, arm_gic_init_secondary), 1120 DEVMETHOD(pic_ipi_send, arm_gic_ipi_send), 1121 #endif 1122 #endif 1123 { 0, 0 } 1124 }; 1125 1126 static driver_t arm_gic_driver = { 1127 "gic", 1128 arm_gic_methods, 1129 sizeof(struct arm_gic_softc), 1130 }; 1131 1132 static devclass_t arm_gic_devclass; 1133 1134 EARLY_DRIVER_MODULE(gic, simplebus, arm_gic_driver, arm_gic_devclass, 0, 0, 1135 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1136 EARLY_DRIVER_MODULE(gic, ofwbus, arm_gic_driver, arm_gic_devclass, 0, 0, 1137 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1138