1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause 3 * 4 * Copyright (c) 2024 Himanshu Chauhan <hchauhan@thechauhan.dev> 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 16 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 18 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 19 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 20 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 21 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 22 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 23 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/param.h> 29 #include <sys/systm.h> 30 #include <sys/bus.h> 31 #include <sys/kernel.h> 32 #include <sys/ktr.h> 33 #include <sys/module.h> 34 #include <sys/proc.h> 35 #include <sys/rman.h> 36 #include <sys/smp.h> 37 38 #include <machine/bus.h> 39 #include <machine/intr.h> 40 #include <machine/riscvreg.h> 41 42 #include <dev/ofw/openfirm.h> 43 #include <dev/ofw/ofw_bus.h> 44 #include <dev/ofw/ofw_bus_subr.h> 45 46 #include "pic_if.h" 47 48 #define APLIC_MAX_IRQS 1023 49 50 /* Smaller priority number means higher priority */ 51 #define APLIC_INTR_DEF_PRIO 1 52 53 static pic_disable_intr_t aplic_disable_intr; 54 static pic_enable_intr_t aplic_enable_intr; 55 static pic_map_intr_t aplic_map_intr; 56 static pic_setup_intr_t aplic_setup_intr; 57 static pic_post_ithread_t aplic_post_ithread; 58 static pic_pre_ithread_t aplic_pre_ithread; 59 static pic_bind_intr_t aplic_bind_intr; 60 61 struct aplic_irqsrc { 62 struct intr_irqsrc isrc; 63 u_int irq; 64 }; 65 66 struct aplic_softc { 67 device_t dev; 68 struct resource *mem_res; 69 struct resource *irq_res; 70 void *ih; 71 struct aplic_irqsrc isrcs[APLIC_MAX_IRQS + 1]; 72 unsigned int hart_indices[MAXCPU]; 73 int ndev; 74 }; 75 76 #define APLIC_DOMAIN_CFG_IE (1UL << 8) /* Enable domain IRQs */ 77 #define APLIC_DOMAIN_CFG_DM (1UL << 2) /* IRQ delivery mode */ 78 #define APLIC_DOMAIN_CFG_BE (1UL << 0) /* Endianess */ 79 80 #define APLIC_MODE_DIRECT 0 /* Direct delivery mode */ 81 #define APLIC_MODE_MSI 1 /* MSI delivery mode */ 82 83 #define APLIC_SRC_CFG_DLGT (1UL << 10) /* Source delegation */ 84 #define APLIC_SRC_CFG_SM_SHIFT 0 85 #define APLIC_SRC_CFG_SM_MASK (0x7UL << APLIC_SRC_CFG_SM_SHIFT) 86 87 #define APLIC_SRC_CFG_SM_INACTIVE 0 /* APLIC inactive in domain */ 88 #define APLIC_SRC_CFG_SM_DETACHED 1 /* Detached from source wire */ 89 #define APLIC_SRC_CFG_SM_EDGE_RSE 4 /* Asserted on rising edge */ 90 #define APLIC_SRC_CFG_SM_EDGE_FLL 5 /* Asserted on falling edge */ 91 #define APLIC_SRC_CFG_SM_LVL_HI 6 /* Asserted when high */ 92 #define APLIC_SRC_CFG_SM_LVL_LO 7 /* Asserted when low */ 93 94 /* Register offsets in APLIC configuration space */ 95 #define APLIC_DOMAIN_CFG 0x0000 96 #define APLIC_SRC_CFG(_idx) (0x0004 + (((_idx) - 1) * 4)) 97 #define APLIC_TARGET(_idx) (0x3004 + (((_idx) - 1) * 4)) 98 #define APLIC_MMSIADDRCFG 0x1BC0 99 #define APLIC_MMSIADDRCFGH 0x1BC4 100 #define APLIC_SMSIADDRCFG 0x1BC8 101 #define APLIC_SMSIADDRCFGH 0x1BCC 102 #define APLIC_SETIPNUM 0x1CDC 103 #define APLIC_CLRIPNUM 0x1DDC 104 #define APLIC_SETIENUM 0x1EDC 105 #define APLIC_CLRIENUM 0x1FDC 106 #define APLIC_SETIPNUM_LE 0x2000 107 #define APLIC_SETIPNUM_BE 0x2004 108 #define APLIC_GENMSI 0x3000 109 #define APLIC_IDC_BASE 0x4000 110 111 #define APLIC_SETIE_BASE 0x1E00 112 #define APLIC_CLRIE_BASE 0x1F00 113 114 /* Interrupt delivery control structure */ 115 #define APLIC_IDC_IDELIVERY_OFFS 0x0000 116 #define APLIC_IDC_IFORCE_OFFS 0x0004 117 #define APLIC_IDC_ITHRESHOLD_OFFS 0x0008 118 #define APLIC_IDC_TOPI_OFFS 0x0018 119 #define APLIC_IDC_CLAIMI_OFFS 0x001C 120 121 #define APLIC_IDC_SZ 0x20 122 123 #define APLIC_IDC_IDELIVERY_DISABLE 0 124 #define APLIC_IDC_IDELIVERY_ENABLE 1 125 #define APLIC_IDC_ITHRESHOLD_DISABLE 0 126 127 #define APLIC_IDC_CLAIMI_PRIO_MASK 0xff 128 #define APLIC_IDC_CLAIMI_IRQ_SHIFT 16 129 #define APLIC_IDC_CLAIMI_IRQ_MASK 0x3ff 130 131 #define APLIC_IDC_CLAIMI_IRQ(_claimi) \ 132 (((_claimi) >> APLIC_IDC_CLAIMI_IRQ_SHIFT) \ 133 & APLIC_IDC_CLAIMI_IRQ_MASK) 134 135 #define APLIC_IDC_CLAIMI_PRIO(_claimi) ((_claimi) & APLIC_IDC_CLAIMI_PRIO_MASK) 136 137 #define APLIC_IDC_REG(_sc, _cpu, _field) \ 138 (APLIC_IDC_BASE + APLIC_IDC_##_field##_OFFS + \ 139 ((_sc->hart_indices[_cpu]) * APLIC_IDC_SZ)) 140 141 #define APLIC_IDC_IDELIVERY(_sc, _cpu) \ 142 APLIC_IDC_REG(_sc, _cpu, IDELIVERY) 143 144 #define APLIC_IDC_IFORCE(_sc, _cpu) \ 145 APLIC_IDC_REG(_sc, _cpu, IFORCE) 146 147 #define APLIC_IDC_ITHRESHOLD(_sc, _cpu) \ 148 APLIC_IDC_REG(_sc, _cpu, ITHRESHOLD) 149 150 #define APLIC_IDC_TOPI(_sc, _cpu) \ 151 APLIC_IDC_REG(_sc, _cpu, TOPI) 152 153 #define APLIC_IDC_CLAIMI(_sc, _cpu) \ 154 APLIC_IDC_REG(_sc, _cpu, CLAIMI) 155 156 #define APLIC_MK_IRQ_TARGET(_sc, _cpu, _prio) \ 157 (_sc->hart_indices[_cpu] << 18 | ((_prio) & 0xff)) 158 159 #define aplic_read(sc, reg) bus_read_4(sc->mem_res, (reg)) 160 #define aplic_write(sc, reg, val) bus_write_4(sc->mem_res, (reg), (val)) 161 162 static u_int aplic_irq_cpu; 163 164 static inline int 165 riscv_cpu_to_hartid(int cpu) 166 { 167 return pcpu_find(cpu)->pc_hart; 168 } 169 170 static inline int 171 riscv_hartid_to_cpu(int hartid) 172 { 173 int cpu; 174 175 CPU_FOREACH(cpu) { 176 if (riscv_cpu_to_hartid(cpu) == hartid) 177 return cpu; 178 } 179 180 return (-1); 181 } 182 183 static int 184 fdt_get_hartid(device_t dev, phandle_t aplic) 185 { 186 int hartid; 187 188 /* Check the interrupt controller layout. */ 189 if (OF_searchencprop(aplic, "#interrupt-cells", &hartid, 190 sizeof(hartid)) == -1) { 191 device_printf(dev, 192 "Could not find #interrupt-cells for phandle %u\n", aplic); 193 return (-1); 194 } 195 196 /* 197 * The parent of the interrupt-controller is the CPU we are 198 * interested in, so search for its hart ID. 199 */ 200 if (OF_searchencprop(OF_parent(aplic), "reg", (pcell_t *)&hartid, 201 sizeof(hartid)) == -1) { 202 device_printf(dev, "Could not find hartid\n"); 203 return (-1); 204 } 205 206 return (hartid); 207 } 208 209 static inline void 210 aplic_irq_dispatch(struct aplic_softc *sc, u_int irq, u_int prio, 211 struct trapframe *tf) 212 { 213 struct aplic_irqsrc *src; 214 215 src = &sc->isrcs[irq]; 216 217 if (intr_isrc_dispatch(&src->isrc, tf) != 0) 218 if (bootverbose) 219 device_printf(sc->dev, "Stray irq %u detected\n", irq); 220 } 221 222 static int 223 aplic_intr(void *arg) 224 { 225 struct aplic_softc *sc; 226 struct trapframe *tf; 227 u_int claimi, prio, irq; 228 int cpu; 229 230 sc = arg; 231 cpu = PCPU_GET(cpuid); 232 233 /* Claim any pending interrupt. */ 234 claimi = aplic_read(sc, APLIC_IDC_CLAIMI(sc, cpu)); 235 prio = APLIC_IDC_CLAIMI_PRIO(claimi); 236 irq = APLIC_IDC_CLAIMI_IRQ(claimi); 237 238 KASSERT((irq != 0), ("Invalid IRQ 0")); 239 240 tf = curthread->td_intr_frame; 241 aplic_irq_dispatch(sc, irq, prio, tf); 242 243 return (FILTER_HANDLED); 244 } 245 246 static void 247 aplic_disable_intr(device_t dev, struct intr_irqsrc *isrc) 248 { 249 struct aplic_softc *sc; 250 struct aplic_irqsrc *src; 251 252 sc = device_get_softc(dev); 253 src = (struct aplic_irqsrc *)isrc; 254 255 /* Disable the interrupt source */ 256 aplic_write(sc, APLIC_CLRIENUM, src->irq); 257 } 258 259 static void 260 aplic_enable_intr(device_t dev, struct intr_irqsrc *isrc) 261 { 262 struct aplic_softc *sc; 263 struct aplic_irqsrc *src; 264 265 sc = device_get_softc(dev); 266 src = (struct aplic_irqsrc *)isrc; 267 268 /* Enable the interrupt source */ 269 aplic_write(sc, APLIC_SETIENUM, src->irq); 270 } 271 272 static int 273 aplic_map_intr(device_t dev, struct intr_map_data *data, 274 struct intr_irqsrc **isrcp) 275 { 276 struct intr_map_data_fdt *daf; 277 struct aplic_softc *sc; 278 279 sc = device_get_softc(dev); 280 281 if (data->type != INTR_MAP_DATA_FDT) 282 return (ENOTSUP); 283 284 daf = (struct intr_map_data_fdt *)data; 285 if (daf->ncells != 2 || daf->cells[0] > sc->ndev) { 286 device_printf(dev, "Invalid cell data\n"); 287 return (EINVAL); 288 } 289 290 *isrcp = &sc->isrcs[daf->cells[0]].isrc; 291 292 return (0); 293 } 294 295 static int 296 aplic_probe(device_t dev) 297 { 298 if (!ofw_bus_status_okay(dev)) 299 return (ENXIO); 300 301 if (!ofw_bus_is_compatible(dev, "riscv,aplic")) 302 return (ENXIO); 303 304 device_set_desc(dev, "Advanced Platform-Level Interrupt Controller"); 305 306 return (BUS_PROBE_DEFAULT); 307 } 308 309 /* 310 * Setup APLIC in direct mode. 311 */ 312 static int 313 aplic_setup_direct_mode(device_t dev) 314 { 315 struct aplic_irqsrc *isrcs; 316 struct aplic_softc *sc; 317 struct intr_pic *pic; 318 const char *name; 319 phandle_t node, xref, iparent; 320 pcell_t *cells, cell; 321 int error = ENXIO; 322 u_int irq; 323 int cpu, hartid, rid, i, nintr, idc; 324 325 sc = device_get_softc(dev); 326 node = ofw_bus_get_node(dev); 327 328 sc->dev = dev; 329 330 if ((OF_getencprop(node, "riscv,num-sources", &sc->ndev, 331 sizeof(sc->ndev))) < 0) { 332 device_printf(dev, "Error: could not get number of devices\n"); 333 return (error); 334 } 335 336 if (sc->ndev > APLIC_MAX_IRQS) { 337 device_printf(dev, "Error: invalid ndev (%d)\n", sc->ndev); 338 return (error); 339 } 340 341 /* Request memory resources */ 342 rid = 0; 343 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 344 RF_ACTIVE); 345 if (sc->mem_res == NULL) { 346 device_printf(dev, 347 "Error: could not allocate memory resources\n"); 348 return (error); 349 } 350 351 /* Set APLIC in direct mode and enable all interrupts */ 352 aplic_write(sc, APLIC_DOMAIN_CFG, 353 APLIC_MODE_DIRECT | APLIC_DOMAIN_CFG_IE); 354 355 /* Register the interrupt sources */ 356 isrcs = sc->isrcs; 357 name = device_get_nameunit(sc->dev); 358 for (irq = 1; irq <= sc->ndev; irq++) { 359 isrcs[irq].irq = irq; 360 361 error = intr_isrc_register(&isrcs[irq].isrc, sc->dev, 362 0, "%s,%u", name, irq); 363 if (error != 0) 364 goto fail; 365 366 aplic_write(sc, APLIC_SRC_CFG(irq), 367 APLIC_SRC_CFG_SM_DETACHED); 368 } 369 370 nintr = OF_getencprop_alloc_multi(node, "interrupts-extended", 371 sizeof(uint32_t), (void **)&cells); 372 if (nintr <= 0) { 373 device_printf(dev, "Could not read interrupts-extended\n"); 374 goto fail; 375 } 376 377 /* interrupts-extended is a list of phandles and interrupt types. */ 378 for (i = 0, idc = 0; i < nintr; i += 2, idc++) { 379 /* Skip M-mode external interrupts */ 380 if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR) 381 continue; 382 383 /* Get the hart ID from the CLIC's phandle. */ 384 hartid = fdt_get_hartid(dev, OF_node_from_xref(cells[i])); 385 if (hartid < 0) { 386 OF_prop_free(cells); 387 goto fail; 388 } 389 390 /* Get the corresponding cpuid. */ 391 cpu = riscv_hartid_to_cpu(hartid); 392 if (cpu < 0) { 393 device_printf(dev, "Invalid cpu for hart %d\n", hartid); 394 OF_prop_free(cells); 395 goto fail; 396 } 397 398 sc->hart_indices[cpu] = idc; 399 } 400 OF_prop_free(cells); 401 402 /* Turn off the interrupt delivery for all CPUs within or out domain */ 403 CPU_FOREACH(cpu) { 404 aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu), 405 APLIC_IDC_IDELIVERY_DISABLE); 406 aplic_write(sc, APLIC_IDC_ITHRESHOLD(sc, cpu), 407 APLIC_IDC_ITHRESHOLD_DISABLE); 408 } 409 410 iparent = OF_xref_from_node(ofw_bus_get_node(intr_irq_root_dev)); 411 cell = IRQ_EXTERNAL_SUPERVISOR; 412 irq = ofw_bus_map_intr(dev, iparent, 1, &cell); 413 error = bus_set_resource(dev, SYS_RES_IRQ, 0, irq, 1); 414 if (error != 0) { 415 device_printf(dev, "Unable to register IRQ resource\n"); 416 return (ENXIO); 417 } 418 419 rid = 0; 420 sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid, 421 RF_ACTIVE); 422 if (sc->irq_res == NULL) { 423 device_printf(dev, 424 "Error: could not allocate IRQ resources\n"); 425 return (ENXIO); 426 } 427 428 xref = OF_xref_from_node(node); 429 pic = intr_pic_register(sc->dev, xref); 430 if (pic == NULL) { 431 error = ENXIO; 432 goto fail; 433 } 434 435 error = bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK, 436 aplic_intr, NULL, sc, &sc->ih); 437 if (error != 0) { 438 device_printf(dev, "Unable to setup IRQ resource\n"); 439 return (ENXIO); 440 } 441 442 fail: 443 return (error); 444 } 445 446 static int 447 aplic_attach(device_t dev) 448 { 449 int rc; 450 451 /* APLIC with IMSIC on hart is not supported */ 452 if (ofw_bus_has_prop(dev, "msi-parent")) { 453 device_printf(dev, "APLIC with IMSIC is unsupported\n"); 454 return (ENXIO); 455 } 456 457 rc = aplic_setup_direct_mode(dev); 458 459 return (rc); 460 } 461 462 static void 463 aplic_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 464 { 465 aplic_disable_intr(dev, isrc); 466 } 467 468 static void 469 aplic_post_ithread(device_t dev, struct intr_irqsrc *isrc) 470 { 471 aplic_enable_intr(dev, isrc); 472 } 473 474 static int 475 aplic_setup_intr(device_t dev, struct intr_irqsrc *isrc, struct resource *res, 476 struct intr_map_data *data) 477 { 478 struct aplic_irqsrc *src; 479 struct aplic_softc *sc; 480 481 CPU_ZERO(&isrc->isrc_cpu); 482 483 sc = device_get_softc(dev); 484 src = (struct aplic_irqsrc *)isrc; 485 486 aplic_write(sc, APLIC_SRC_CFG(src->irq), APLIC_SRC_CFG_SM_EDGE_RSE); 487 488 /* 489 * In uniprocessor system, bind_intr will not be called. 490 * So bind the interrupt on this CPU. If secondary CPUs 491 * are present, then bind_intr will be called again and 492 * interrupts will rebind to those CPUs. 493 */ 494 aplic_bind_intr(dev, isrc); 495 496 return (0); 497 } 498 499 static int 500 aplic_bind_intr(device_t dev, struct intr_irqsrc *isrc) 501 { 502 struct aplic_softc *sc; 503 struct aplic_irqsrc *src; 504 uint32_t cpu, hartid; 505 506 sc = device_get_softc(dev); 507 src = (struct aplic_irqsrc *)isrc; 508 509 /* Disable the interrupt source */ 510 aplic_write(sc, APLIC_CLRIENUM, src->irq); 511 512 if (CPU_EMPTY(&isrc->isrc_cpu)) { 513 cpu = aplic_irq_cpu = intr_irq_next_cpu(aplic_irq_cpu, 514 &all_cpus); 515 CPU_SETOF(cpu, &isrc->isrc_cpu); 516 } else { 517 cpu = CPU_FFS(&isrc->isrc_cpu) - 1; 518 } 519 520 hartid = riscv_cpu_to_hartid(cpu); 521 522 if (bootverbose) 523 device_printf(dev, "Bind irq %d to cpu%d (hart %d)\n", src->irq, 524 cpu, hartid); 525 526 aplic_write(sc, APLIC_TARGET(src->irq), 527 APLIC_MK_IRQ_TARGET(sc, cpu, APLIC_INTR_DEF_PRIO)); 528 aplic_write(sc, APLIC_IDC_IDELIVERY(sc, cpu), 529 APLIC_IDC_IDELIVERY_ENABLE); 530 aplic_enable_intr(dev, isrc); 531 532 return (0); 533 } 534 535 static device_method_t aplic_methods[] = { 536 DEVMETHOD(device_probe, aplic_probe), 537 DEVMETHOD(device_attach, aplic_attach), 538 539 DEVMETHOD(pic_disable_intr, aplic_disable_intr), 540 DEVMETHOD(pic_enable_intr, aplic_enable_intr), 541 DEVMETHOD(pic_map_intr, aplic_map_intr), 542 DEVMETHOD(pic_pre_ithread, aplic_pre_ithread), 543 DEVMETHOD(pic_post_ithread, aplic_post_ithread), 544 DEVMETHOD(pic_post_filter, aplic_post_ithread), 545 DEVMETHOD(pic_setup_intr, aplic_setup_intr), 546 DEVMETHOD(pic_bind_intr, aplic_bind_intr), 547 548 DEVMETHOD_END 549 }; 550 551 DEFINE_CLASS_0(aplic, aplic_driver, aplic_methods, sizeof(struct aplic_softc)); 552 553 EARLY_DRIVER_MODULE(aplic, simplebus, aplic_driver, 0, 0, 554 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 555