1 /* $OpenBSD: agintc.c,v 1.54 2023/09/22 01:10:43 jsg Exp $ */ 2 /* 3 * Copyright (c) 2007, 2009, 2011, 2017 Dale Rahn <drahn@dalerahn.com> 4 * Copyright (c) 2018 Mark Kettenis <kettenis@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 /* 20 * This is a device driver for the GICv3/GICv4 IP from ARM as specified 21 * in IHI0069C, an example of this hardware is the GIC 500. 22 */ 23 24 #include <sys/param.h> 25 #include <sys/systm.h> 26 #include <sys/queue.h> 27 #include <sys/malloc.h> 28 #include <sys/device.h> 29 #include <sys/evcount.h> 30 31 #include <machine/bus.h> 32 #include <machine/cpufunc.h> 33 #include <machine/fdt.h> 34 35 #include <dev/ofw/fdt.h> 36 #include <dev/ofw/openfirm.h> 37 38 #include <machine/simplebusvar.h> 39 40 #define ICC_PMR s3_0_c4_c6_0 41 #define ICC_IAR0 s3_0_c12_c8_0 42 #define ICC_EOIR0 s3_0_c12_c8_1 43 #define ICC_HPPIR0 s3_0_c12_c8_2 44 #define ICC_BPR0 s3_0_c12_c8_3 45 46 #define ICC_DIR s3_0_c12_c11_1 47 #define ICC_RPR s3_0_c12_c11_3 48 #define ICC_SGI1R s3_0_c12_c11_5 49 #define ICC_SGI0R s3_0_c12_c11_7 50 51 #define ICC_IAR1 s3_0_c12_c12_0 52 #define ICC_EOIR1 s3_0_c12_c12_1 53 #define ICC_HPPIR1 s3_0_c12_c12_2 54 #define ICC_BPR1 s3_0_c12_c12_3 55 #define ICC_CTLR s3_0_c12_c12_4 56 #define ICC_SRE_EL1 s3_0_c12_c12_5 57 #define ICC_SRE_EL1_EN 0x7 58 #define ICC_IGRPEN0 s3_0_c12_c12_6 59 #define ICC_IGRPEN1 s3_0_c12_c12_7 60 61 #define _STR(x) #x 62 #define STR(x) _STR(x) 63 64 /* distributor registers */ 65 #define GICD_CTLR 0x0000 66 /* non-secure */ 67 #define GICD_CTLR_RWP (1U << 31) 68 #define GICD_CTLR_EnableGrp1 (1 << 0) 69 #define GICD_CTLR_EnableGrp1A (1 << 1) 70 #define GICD_CTLR_ARE_NS (1 << 4) 71 #define GICD_CTLR_DS (1 << 6) 72 #define GICD_TYPER 0x0004 73 #define GICD_TYPER_MBIS (1 << 16) 74 #define GICD_TYPER_LPIS (1 << 17) 75 #define GICD_TYPER_ITLINE_M 0x1f 76 #define GICD_IIDR 0x0008 77 #define GICD_SETSPI_NSR 0x0040 78 #define GICD_CLRSPI_NSR 0x0048 79 #define GICD_IGROUPR(i) (0x0080 + (IRQ_TO_REG32(i) * 4)) 80 #define GICD_ISENABLER(i) (0x0100 + (IRQ_TO_REG32(i) * 4)) 81 #define GICD_ICENABLER(i) (0x0180 + (IRQ_TO_REG32(i) * 4)) 82 #define GICD_ISPENDR(i) (0x0200 + (IRQ_TO_REG32(i) * 4)) 83 #define GICD_ICPENDR(i) (0x0280 + (IRQ_TO_REG32(i) * 4)) 84 #define GICD_ISACTIVER(i) (0x0300 + (IRQ_TO_REG32(i) * 4)) 85 #define GICD_ICACTIVER(i) (0x0380 + (IRQ_TO_REG32(i) * 4)) 86 #define GICD_IPRIORITYR(i) (0x0400 + (i)) 87 #define GICD_ICFGR(i) (0x0c00 + (IRQ_TO_REG16(i) * 4)) 88 #define GICD_ICFGR_TRIG_LEVEL(i) (0x0 << (IRQ_TO_REG16BIT(i) * 2)) 89 #define GICD_ICFGR_TRIG_EDGE(i) (0x2 << (IRQ_TO_REG16BIT(i) * 2)) 90 #define GICD_ICFGR_TRIG_MASK(i) (0x2 << (IRQ_TO_REG16BIT(i) * 2)) 91 #define GICD_IGRPMODR(i) (0x0d00 + (IRQ_TO_REG32(i) * 4)) 92 #define GICD_NSACR(i) (0x0e00 + (IRQ_TO_REG16(i) * 4)) 93 #define GICD_IROUTER(i) (0x6000 + ((i) * 8)) 94 95 /* redistributor registers */ 96 #define GICR_CTLR 0x00000 97 #define GICR_CTLR_RWP ((1U << 31) | (1 << 3)) 98 #define GICR_CTLR_ENABLE_LPIS (1 << 0) 99 #define GICR_IIDR 0x00004 100 #define GICR_TYPER 0x00008 101 #define GICR_TYPER_LAST (1 << 4) 102 #define GICR_TYPER_VLPIS (1 << 1) 103 #define GICR_WAKER 0x00014 104 #define GICR_WAKER_X31 (1U << 31) 105 #define GICR_WAKER_CHILDRENASLEEP (1 << 2) 106 #define GICR_WAKER_PROCESSORSLEEP (1 << 1) 107 #define GICR_WAKER_X0 (1 << 0) 108 #define GICR_PROPBASER 0x00070 109 #define GICR_PROPBASER_ISH (1ULL << 10) 110 #define GICR_PROPBASER_IC_NORM_NC (1ULL << 7) 111 #define GICR_PENDBASER 0x00078 112 #define GICR_PENDBASER_PTZ (1ULL << 62) 113 #define GICR_PENDBASER_ISH (1ULL << 10) 114 #define GICR_PENDBASER_IC_NORM_NC (1ULL << 7) 115 #define GICR_IGROUPR0 0x10080 116 #define GICR_ISENABLE0 0x10100 117 #define GICR_ICENABLE0 0x10180 118 #define GICR_ISPENDR0 0x10200 119 #define GICR_ICPENDR0 0x10280 120 #define GICR_ISACTIVE0 0x10300 121 #define GICR_ICACTIVE0 0x10380 122 #define GICR_IPRIORITYR(i) (0x10400 + (i)) 123 #define GICR_ICFGR0 0x10c00 124 #define GICR_ICFGR1 0x10c04 125 #define GICR_IGRPMODR0 0x10d00 126 127 #define GICR_PROP_SIZE (64 * 1024) 128 #define GICR_PROP_GROUP1 (1 << 1) 129 #define GICR_PROP_ENABLE (1 << 0) 130 #define GICR_PEND_SIZE (64 * 1024) 131 132 #define PPI_BASE 16 133 #define SPI_BASE 32 134 #define LPI_BASE 8192 135 136 #define IRQ_TO_REG32(i) (((i) >> 5) & 0x1f) 137 #define IRQ_TO_REG32BIT(i) ((i) & 0x1f) 138 139 #define IRQ_TO_REG16(i) (((i) >> 4) & 0x3f) 140 #define IRQ_TO_REG16BIT(i) ((i) & 0xf) 141 142 #define IRQ_ENABLE 1 143 #define IRQ_DISABLE 0 144 145 struct agintc_mbi_range { 146 int mr_base; 147 int mr_span; 148 void **mr_mbi; 149 }; 150 151 struct agintc_lpi_info { 152 struct agintc_msi_softc *li_msic; 153 struct cpu_info *li_ci; 154 uint32_t li_deviceid; 155 uint32_t li_eventid; 156 struct intrhand *li_ih; 157 }; 158 159 struct agintc_softc { 160 struct simplebus_softc sc_sbus; 161 struct intrq *sc_handler; 162 struct agintc_lpi_info **sc_lpi; 163 bus_space_tag_t sc_iot; 164 bus_space_handle_t sc_d_ioh; 165 bus_space_handle_t *sc_r_ioh; 166 bus_space_handle_t sc_redist_base; 167 bus_dma_tag_t sc_dmat; 168 uint16_t *sc_processor; 169 int sc_cpuremap[MAXCPUS]; 170 int sc_nintr; 171 int sc_nlpi; 172 bus_addr_t sc_mbi_addr; 173 int sc_mbi_nranges; 174 struct agintc_mbi_range *sc_mbi_ranges; 175 int sc_prio_shift; 176 int sc_pmr_shift; 177 int sc_rk3399_quirk; 178 struct evcount sc_spur; 179 int sc_ncells; 180 int sc_num_redist; 181 struct agintc_dmamem *sc_prop; 182 struct agintc_dmamem *sc_pend; 183 struct interrupt_controller sc_ic; 184 int sc_ipi_num[3]; /* id for each ipi */ 185 int sc_ipi_reason[MAXCPUS]; /* cause of ipi */ 186 void *sc_ipi_irq[3]; /* irqhandle for each ipi */ 187 }; 188 struct agintc_softc *agintc_sc; 189 190 struct intrhand { 191 TAILQ_ENTRY(intrhand) ih_list; /* link on intrq list */ 192 int (*ih_func)(void *); /* handler */ 193 void *ih_arg; /* arg for handler */ 194 int ih_ipl; /* IPL_* */ 195 int ih_flags; 196 int ih_irq; /* IRQ number */ 197 struct evcount ih_count; 198 char *ih_name; 199 struct cpu_info *ih_ci; /* CPU the IRQ runs on */ 200 }; 201 202 struct intrq { 203 TAILQ_HEAD(, intrhand) iq_list; /* handler list */ 204 struct cpu_info *iq_ci; /* CPU the IRQ runs on */ 205 int iq_irq_max; /* IRQ to mask while handling */ 206 int iq_irq_min; /* lowest IRQ when shared */ 207 int iq_ist; /* share type */ 208 int iq_route; 209 }; 210 211 struct agintc_dmamem { 212 bus_dmamap_t adm_map; 213 bus_dma_segment_t adm_seg; 214 size_t adm_size; 215 caddr_t adm_kva; 216 }; 217 218 #define AGINTC_DMA_MAP(_adm) ((_adm)->adm_map) 219 #define AGINTC_DMA_LEN(_adm) ((_adm)->adm_size) 220 #define AGINTC_DMA_DVA(_adm) ((_adm)->adm_map->dm_segs[0].ds_addr) 221 #define AGINTC_DMA_KVA(_adm) ((void *)(_adm)->adm_kva) 222 223 struct agintc_dmamem *agintc_dmamem_alloc(bus_dma_tag_t, bus_size_t, 224 bus_size_t); 225 void agintc_dmamem_free(bus_dma_tag_t, struct agintc_dmamem *); 226 227 int agintc_match(struct device *, void *, void *); 228 void agintc_attach(struct device *, struct device *, void *); 229 void agintc_mbiinit(struct agintc_softc *, int, bus_addr_t); 230 void agintc_cpuinit(void); 231 int agintc_spllower(int); 232 void agintc_splx(int); 233 int agintc_splraise(int); 234 void agintc_setipl(int); 235 void agintc_enable_wakeup(void); 236 void agintc_disable_wakeup(void); 237 void agintc_calc_mask(void); 238 void agintc_calc_irq(struct agintc_softc *sc, int irq); 239 void *agintc_intr_establish(int, int, int, struct cpu_info *, 240 int (*)(void *), void *, char *); 241 void *agintc_intr_establish_fdt(void *cookie, int *cell, int level, 242 struct cpu_info *, int (*func)(void *), void *arg, char *name); 243 void *agintc_intr_establish_mbi(void *, uint64_t *, uint64_t *, 244 int , struct cpu_info *, int (*)(void *), void *, char *); 245 void agintc_intr_disestablish(void *); 246 void agintc_intr_set_wakeup(void *); 247 void agintc_irq_handler(void *); 248 uint32_t agintc_iack(void); 249 void agintc_eoi(uint32_t); 250 void agintc_set_priority(struct agintc_softc *sc, int, int); 251 void agintc_intr_enable(struct agintc_softc *, int); 252 void agintc_intr_disable(struct agintc_softc *, int); 253 void agintc_intr_config(struct agintc_softc *, int, int); 254 void agintc_route(struct agintc_softc *, int, int, 255 struct cpu_info *); 256 void agintc_route_irq(void *, int, struct cpu_info *); 257 void agintc_intr_barrier(void *); 258 void agintc_wait_rwp(struct agintc_softc *sc); 259 void agintc_r_wait_rwp(struct agintc_softc *sc); 260 uint32_t agintc_r_ictlr(void); 261 262 int agintc_ipi_ddb(void *v); 263 int agintc_ipi_halt(void *v); 264 int agintc_ipi_nop(void *v); 265 int agintc_ipi_combined(void *); 266 void agintc_send_ipi(struct cpu_info *, int); 267 268 void agintc_msi_discard(struct agintc_lpi_info *); 269 void agintc_msi_inv(struct agintc_lpi_info *); 270 271 const struct cfattach agintc_ca = { 272 sizeof (struct agintc_softc), agintc_match, agintc_attach 273 }; 274 275 struct cfdriver agintc_cd = { 276 NULL, "agintc", DV_DULL 277 }; 278 279 static char *agintc_compatibles[] = { 280 "arm,gic-v3", 281 "arm,gic-v4", 282 NULL 283 }; 284 285 int 286 agintc_match(struct device *parent, void *cfdata, void *aux) 287 { 288 struct fdt_attach_args *faa = aux; 289 int i; 290 291 for (i = 0; agintc_compatibles[i]; i++) 292 if (OF_is_compatible(faa->fa_node, agintc_compatibles[i])) 293 return (1); 294 295 return (0); 296 } 297 298 static void 299 __isb(void) 300 { 301 __asm volatile("isb"); 302 } 303 304 void 305 agintc_attach(struct device *parent, struct device *self, void *aux) 306 { 307 struct agintc_softc *sc = (struct agintc_softc *)self; 308 struct fdt_attach_args *faa = aux; 309 struct cpu_info *ci; 310 CPU_INFO_ITERATOR cii; 311 u_long psw; 312 uint32_t typer; 313 uint32_t nsacr, oldnsacr; 314 uint32_t pmr, oldpmr; 315 uint32_t ctrl, bits; 316 uint32_t affinity; 317 int i, nbits, nintr; 318 int offset, nredist; 319 #ifdef MULTIPROCESSOR 320 int nipi, ipiirq[3]; 321 #endif 322 323 psw = intr_disable(); 324 arm_init_smask(); 325 326 sc->sc_iot = faa->fa_iot; 327 sc->sc_dmat = faa->fa_dmat; 328 329 /* First row: distributor */ 330 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 331 faa->fa_reg[0].size, 0, &sc->sc_d_ioh)) 332 panic("%s: ICD bus_space_map failed!", __func__); 333 334 /* Second row: redistributor */ 335 if (bus_space_map(sc->sc_iot, faa->fa_reg[1].addr, 336 faa->fa_reg[1].size, 0, &sc->sc_redist_base)) 337 panic("%s: ICP bus_space_map failed!", __func__); 338 339 typer = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_TYPER); 340 341 if (typer & GICD_TYPER_LPIS) { 342 /* Allocate redistributor tables */ 343 sc->sc_prop = agintc_dmamem_alloc(sc->sc_dmat, 344 GICR_PROP_SIZE, GICR_PROP_SIZE); 345 if (sc->sc_prop == NULL) { 346 printf(": can't alloc LPI config table\n"); 347 goto unmap; 348 } 349 sc->sc_pend = agintc_dmamem_alloc(sc->sc_dmat, 350 GICR_PEND_SIZE, GICR_PEND_SIZE); 351 if (sc->sc_pend == NULL) { 352 printf(": can't alloc LPI pending table\n"); 353 goto unmap; 354 } 355 356 /* Minimum number of LPIs supported by any implementation. */ 357 sc->sc_nlpi = 8192; 358 } 359 360 if (typer & GICD_TYPER_MBIS) 361 agintc_mbiinit(sc, faa->fa_node, faa->fa_reg[0].addr); 362 363 /* 364 * We are guaranteed to have at least 16 priority levels, so 365 * in principle we just want to use the top 4 bits of the 366 * (non-secure) priority field. 367 */ 368 sc->sc_prio_shift = sc->sc_pmr_shift = 4; 369 370 /* 371 * If the system supports two security states and SCR_EL3.FIQ 372 * is zero, the non-secure shifted view applies. We detect 373 * this by checking whether the number of writable bits 374 * matches the number of implemented priority bits. If that 375 * is the case we will need to adjust the priorities that we 376 * write into ICC_PMR_EL1 accordingly. 377 * 378 * On Ampere eMAG it appears as if there are five writable 379 * bits when we write 0xff. But for higher priorities 380 * (smaller values) only the top 4 bits stick. So we use 0xbf 381 * instead to determine the number of writable bits. 382 */ 383 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 384 if ((ctrl & GICD_CTLR_DS) == 0) { 385 __asm volatile("mrs %x0, "STR(ICC_CTLR_EL1) : "=r"(ctrl)); 386 nbits = ICC_CTLR_EL1_PRIBITS(ctrl) + 1; 387 __asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(oldpmr)); 388 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xbf)); 389 __asm volatile("mrs %x0, "STR(ICC_PMR) : "=r"(pmr)); 390 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(oldpmr)); 391 if (nbits == 8 - (ffs(pmr) - 1)) 392 sc->sc_pmr_shift--; 393 } 394 395 /* 396 * The Rockchip RK3399 is busted. Its GIC-500 treats all 397 * access to its memory mapped registers as "secure". As a 398 * result, several registers don't behave as expected. For 399 * example, the GICD_IPRIORITYRn and GICR_IPRIORITYRn 400 * registers expose the full priority range available to 401 * secure interrupts. We need to be aware of this and write 402 * an adjusted priority value into these registers. We also 403 * need to be careful not to touch any bits that shouldn't be 404 * writable in non-secure mode. 405 * 406 * We check whether we have secure mode access to these 407 * registers by attempting to write to the GICD_NSACR register 408 * and check whether its contents actually change. In that 409 * case we need to adjust the priorities we write into 410 * GICD_IPRIORITYRn and GICRIPRIORITYRn accordingly. 411 */ 412 oldnsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32)); 413 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32), 414 oldnsacr ^ 0xffffffff); 415 nsacr = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32)); 416 if (nsacr != oldnsacr) { 417 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_NSACR(32), 418 oldnsacr); 419 sc->sc_rk3399_quirk = 1; 420 sc->sc_prio_shift--; 421 printf(" sec"); 422 } 423 424 printf(" shift %d:%d", sc->sc_prio_shift, sc->sc_pmr_shift); 425 426 evcount_attach(&sc->sc_spur, "irq1023/spur", NULL); 427 428 __asm volatile("msr "STR(ICC_SRE_EL1)", %x0" : : "r" (ICC_SRE_EL1_EN)); 429 __isb(); 430 431 nintr = 32 * (typer & GICD_TYPER_ITLINE_M); 432 nintr += 32; /* ICD_ICTR + 1, irq 0-31 is SGI, 32+ is PPI */ 433 sc->sc_nintr = nintr; 434 435 agintc_sc = sc; /* save this for global access */ 436 437 /* find the redistributors. */ 438 offset = 0; 439 for (nredist = 0; ; nredist++) { 440 int32_t sz = (64 * 1024 * 2); 441 uint64_t typer; 442 443 typer = bus_space_read_8(sc->sc_iot, sc->sc_redist_base, 444 offset + GICR_TYPER); 445 446 if (typer & GICR_TYPER_VLPIS) 447 sz += (64 * 1024 * 2); 448 449 #ifdef DEBUG_AGINTC 450 printf("probing redistributor %d %x\n", nredist, offset); 451 #endif 452 453 offset += sz; 454 455 if (typer & GICR_TYPER_LAST) { 456 sc->sc_num_redist = nredist + 1; 457 break; 458 } 459 } 460 461 printf(" nirq %d nredist %d", nintr, sc->sc_num_redist); 462 463 sc->sc_r_ioh = mallocarray(sc->sc_num_redist, 464 sizeof(*sc->sc_r_ioh), M_DEVBUF, M_WAITOK); 465 sc->sc_processor = mallocarray(sc->sc_num_redist, 466 sizeof(*sc->sc_processor), M_DEVBUF, M_WAITOK); 467 468 /* submap and configure the redistributors. */ 469 offset = 0; 470 for (nredist = 0; nredist < sc->sc_num_redist; nredist++) { 471 int32_t sz = (64 * 1024 * 2); 472 uint64_t typer; 473 474 typer = bus_space_read_8(sc->sc_iot, sc->sc_redist_base, 475 offset + GICR_TYPER); 476 477 if (typer & GICR_TYPER_VLPIS) 478 sz += (64 * 1024 * 2); 479 480 affinity = bus_space_read_8(sc->sc_iot, 481 sc->sc_redist_base, offset + GICR_TYPER) >> 32; 482 CPU_INFO_FOREACH(cii, ci) { 483 if (affinity == (((ci->ci_mpidr >> 8) & 0xff000000) | 484 (ci->ci_mpidr & 0x00ffffff))) 485 break; 486 } 487 if (ci != NULL) 488 sc->sc_cpuremap[ci->ci_cpuid] = nredist; 489 490 sc->sc_processor[nredist] = bus_space_read_8(sc->sc_iot, 491 sc->sc_redist_base, offset + GICR_TYPER) >> 8; 492 493 bus_space_subregion(sc->sc_iot, sc->sc_redist_base, 494 offset, sz, &sc->sc_r_ioh[nredist]); 495 496 if (sc->sc_nlpi > 0) { 497 bus_space_write_8(sc->sc_iot, sc->sc_redist_base, 498 offset + GICR_PROPBASER, 499 AGINTC_DMA_DVA(sc->sc_prop) | 500 GICR_PROPBASER_ISH | GICR_PROPBASER_IC_NORM_NC | 501 fls(LPI_BASE + sc->sc_nlpi - 1) - 1); 502 bus_space_write_8(sc->sc_iot, sc->sc_redist_base, 503 offset + GICR_PENDBASER, 504 AGINTC_DMA_DVA(sc->sc_pend) | 505 GICR_PENDBASER_ISH | GICR_PENDBASER_IC_NORM_NC | 506 GICR_PENDBASER_PTZ); 507 bus_space_write_4(sc->sc_iot, sc->sc_redist_base, 508 offset + GICR_CTLR, GICR_CTLR_ENABLE_LPIS); 509 } 510 511 offset += sz; 512 } 513 514 /* Disable all interrupts, clear all pending */ 515 for (i = 1; i < nintr / 32; i++) { 516 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 517 GICD_ICACTIVER(i * 32), ~0); 518 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 519 GICD_ICENABLER(i * 32), ~0); 520 } 521 522 for (i = 4; i < nintr; i += 4) { 523 /* lowest priority ?? */ 524 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 525 GICD_IPRIORITYR(i), 0xffffffff); 526 } 527 528 /* Set all interrupts to G1NS */ 529 for (i = 1; i < nintr / 32; i++) { 530 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 531 GICD_IGROUPR(i * 32), ~0); 532 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 533 GICD_IGRPMODR(i * 32), 0); 534 } 535 536 for (i = 2; i < nintr / 16; i++) { 537 /* irq 32 - N */ 538 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 539 GICD_ICFGR(i * 16), 0); 540 } 541 542 agintc_cpuinit(); 543 544 sc->sc_handler = mallocarray(nintr, 545 sizeof(*sc->sc_handler), M_DEVBUF, M_ZERO | M_WAITOK); 546 for (i = 0; i < nintr; i++) 547 TAILQ_INIT(&sc->sc_handler[i].iq_list); 548 sc->sc_lpi = mallocarray(sc->sc_nlpi, 549 sizeof(*sc->sc_lpi), M_DEVBUF, M_ZERO | M_WAITOK); 550 551 /* set priority to IPL_HIGH until configure lowers to desired IPL */ 552 agintc_setipl(IPL_HIGH); 553 554 /* initialize all interrupts as disabled */ 555 agintc_calc_mask(); 556 557 /* insert self as interrupt handler */ 558 arm_set_intr_handler(agintc_splraise, agintc_spllower, agintc_splx, 559 agintc_setipl, agintc_irq_handler, NULL, 560 agintc_enable_wakeup, agintc_disable_wakeup); 561 562 /* enable interrupts */ 563 ctrl = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 564 bits = GICD_CTLR_ARE_NS | GICD_CTLR_EnableGrp1A | GICD_CTLR_EnableGrp1; 565 if (sc->sc_rk3399_quirk) { 566 bits &= ~GICD_CTLR_EnableGrp1A; 567 bits <<= 1; 568 } 569 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR, ctrl | bits); 570 571 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff)); 572 __asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0)); 573 __asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1)); 574 575 #ifdef MULTIPROCESSOR 576 /* setup IPI interrupts */ 577 578 /* 579 * Ideally we want three IPI interrupts, one for NOP, one for 580 * DDB and one for HALT. However we can survive if only one 581 * is available; it is possible that most are not available to 582 * the non-secure OS. 583 */ 584 nipi = 0; 585 for (i = 0; i < 16; i++) { 586 int hwcpu = sc->sc_cpuremap[cpu_number()]; 587 int reg, oldreg; 588 589 oldreg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 590 GICR_IPRIORITYR(i)); 591 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 592 GICR_IPRIORITYR(i), oldreg ^ 0x20); 593 594 /* if this interrupt is not usable, pri will be unmodified */ 595 reg = bus_space_read_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 596 GICR_IPRIORITYR(i)); 597 if (reg == oldreg) 598 continue; 599 600 /* return to original value, will be set when used */ 601 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 602 GICR_IPRIORITYR(i), oldreg); 603 604 if (nipi == 0) 605 printf(" ipi: %d", i); 606 else 607 printf(", %d", i); 608 ipiirq[nipi++] = i; 609 if (nipi == 3) 610 break; 611 } 612 613 if (nipi == 0) 614 panic("no irq available for IPI"); 615 616 switch (nipi) { 617 case 1: 618 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 619 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 620 agintc_ipi_combined, sc, "ipi"); 621 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 622 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[0]; 623 sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[0]; 624 break; 625 case 2: 626 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 627 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 628 agintc_ipi_nop, sc, "ipinop"); 629 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 630 sc->sc_ipi_irq[1] = agintc_intr_establish(ipiirq[1], 631 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 632 agintc_ipi_combined, sc, "ipi"); 633 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[1]; 634 sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[1]; 635 break; 636 case 3: 637 sc->sc_ipi_irq[0] = agintc_intr_establish(ipiirq[0], 638 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 639 agintc_ipi_nop, sc, "ipinop"); 640 sc->sc_ipi_num[ARM_IPI_NOP] = ipiirq[0]; 641 sc->sc_ipi_irq[1] = agintc_intr_establish(ipiirq[1], 642 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 643 agintc_ipi_ddb, sc, "ipiddb"); 644 sc->sc_ipi_num[ARM_IPI_DDB] = ipiirq[1]; 645 sc->sc_ipi_irq[2] = agintc_intr_establish(ipiirq[2], 646 IST_EDGE_RISING, IPL_IPI|IPL_MPSAFE, NULL, 647 agintc_ipi_halt, sc, "ipihalt"); 648 sc->sc_ipi_num[ARM_IPI_HALT] = ipiirq[2]; 649 break; 650 default: 651 panic("nipi unexpected number %d", nipi); 652 } 653 654 intr_send_ipi_func = agintc_send_ipi; 655 #endif 656 657 sc->sc_ic.ic_node = faa->fa_node; 658 sc->sc_ic.ic_cookie = self; 659 sc->sc_ic.ic_establish = agintc_intr_establish_fdt; 660 sc->sc_ic.ic_disestablish = agintc_intr_disestablish; 661 sc->sc_ic.ic_route = agintc_route_irq; 662 sc->sc_ic.ic_cpu_enable = agintc_cpuinit; 663 sc->sc_ic.ic_barrier = agintc_intr_barrier; 664 if (sc->sc_mbi_nranges > 0) 665 sc->sc_ic.ic_establish_msi = agintc_intr_establish_mbi; 666 sc->sc_ic.ic_set_wakeup = agintc_intr_set_wakeup; 667 arm_intr_register_fdt(&sc->sc_ic); 668 669 intr_restore(psw); 670 671 /* Attach ITS. */ 672 simplebus_attach(parent, &sc->sc_sbus.sc_dev, faa); 673 674 return; 675 676 unmap: 677 if (sc->sc_r_ioh) { 678 free(sc->sc_r_ioh, M_DEVBUF, 679 sc->sc_num_redist * sizeof(*sc->sc_r_ioh)); 680 } 681 if (sc->sc_processor) { 682 free(sc->sc_processor, M_DEVBUF, 683 sc->sc_num_redist * sizeof(*sc->sc_processor)); 684 } 685 686 if (sc->sc_pend) 687 agintc_dmamem_free(sc->sc_dmat, sc->sc_pend); 688 if (sc->sc_prop) 689 agintc_dmamem_free(sc->sc_dmat, sc->sc_prop); 690 691 bus_space_unmap(sc->sc_iot, sc->sc_redist_base, faa->fa_reg[1].size); 692 bus_space_unmap(sc->sc_iot, sc->sc_d_ioh, faa->fa_reg[0].size); 693 } 694 695 void 696 agintc_mbiinit(struct agintc_softc *sc, int node, bus_addr_t addr) 697 { 698 uint32_t *ranges; 699 int i, len; 700 701 if (OF_getproplen(node, "msi-controller") != 0) 702 return; 703 704 len = OF_getproplen(node, "mbi-ranges"); 705 if (len <= 0 || len % 2 * sizeof(uint32_t) != 0) 706 return; 707 708 ranges = malloc(len, M_TEMP, M_WAITOK); 709 OF_getpropintarray(node, "mbi-ranges", ranges, len); 710 711 sc->sc_mbi_nranges = len / (2 * sizeof(uint32_t)); 712 sc->sc_mbi_ranges = mallocarray(sc->sc_mbi_nranges, 713 sizeof(struct agintc_mbi_range), M_DEVBUF, M_WAITOK); 714 715 for (i = 0; i < sc->sc_mbi_nranges; i++) { 716 sc->sc_mbi_ranges[i].mr_base = ranges[2 * i + 0]; 717 sc->sc_mbi_ranges[i].mr_span = ranges[2 * i + 1]; 718 sc->sc_mbi_ranges[i].mr_mbi = 719 mallocarray(sc->sc_mbi_ranges[i].mr_span, 720 sizeof(void *), M_DEVBUF, M_WAITOK | M_ZERO); 721 } 722 723 free(ranges, M_TEMP, len); 724 725 addr = OF_getpropint64(node, "mbi-alias", addr); 726 sc->sc_mbi_addr = addr + GICD_SETSPI_NSR; 727 728 printf(" mbi"); 729 } 730 731 /* Initialize redistributors on each core. */ 732 void 733 agintc_cpuinit(void) 734 { 735 struct agintc_softc *sc = agintc_sc; 736 uint32_t waker; 737 int timeout = 100000; 738 int hwcpu; 739 int i; 740 741 hwcpu = sc->sc_cpuremap[cpu_number()]; 742 waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 743 GICR_WAKER); 744 waker &= ~(GICR_WAKER_PROCESSORSLEEP); 745 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], GICR_WAKER, 746 waker); 747 748 do { 749 waker = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 750 GICR_WAKER); 751 } while (--timeout && (waker & GICR_WAKER_CHILDRENASLEEP)); 752 if (timeout == 0) 753 printf("%s: waker timed out\n", __func__); 754 755 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 756 GICR_ICENABLE0, ~0); 757 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 758 GICR_ICPENDR0, ~0); 759 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 760 GICR_ICACTIVE0, ~0); 761 for (i = 0; i < 32; i += 4) { 762 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 763 GICR_IPRIORITYR(i), ~0); 764 } 765 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 766 GICR_IGROUPR0, ~0); 767 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 768 GICR_IGRPMODR0, 0); 769 770 if (sc->sc_ipi_irq[0] != NULL) 771 agintc_route_irq(sc->sc_ipi_irq[0], IRQ_ENABLE, curcpu()); 772 if (sc->sc_ipi_irq[1] != NULL) 773 agintc_route_irq(sc->sc_ipi_irq[1], IRQ_ENABLE, curcpu()); 774 if (sc->sc_ipi_irq[2] != NULL) 775 agintc_route_irq(sc->sc_ipi_irq[2], IRQ_ENABLE, curcpu()); 776 777 __asm volatile("msr "STR(ICC_PMR)", %x0" :: "r"(0xff)); 778 __asm volatile("msr "STR(ICC_BPR1)", %x0" :: "r"(0)); 779 __asm volatile("msr "STR(ICC_IGRPEN1)", %x0" :: "r"(1)); 780 intr_enable(); 781 } 782 783 void 784 agintc_set_priority(struct agintc_softc *sc, int irq, int ipl) 785 { 786 struct cpu_info *ci = curcpu(); 787 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 788 uint32_t prival; 789 790 prival = ((0xff - ipl) << sc->sc_prio_shift) & 0xff; 791 792 if (irq >= SPI_BASE) { 793 bus_space_write_1(sc->sc_iot, sc->sc_d_ioh, 794 GICD_IPRIORITYR(irq), prival); 795 } else { 796 /* only sets local redistributor */ 797 bus_space_write_1(sc->sc_iot, sc->sc_r_ioh[hwcpu], 798 GICR_IPRIORITYR(irq), prival); 799 } 800 } 801 802 void 803 agintc_setipl(int ipl) 804 { 805 struct agintc_softc *sc = agintc_sc; 806 struct cpu_info *ci = curcpu(); 807 u_long psw; 808 uint32_t prival; 809 810 /* disable here is only to keep hardware in sync with ci->ci_cpl */ 811 psw = intr_disable(); 812 ci->ci_cpl = ipl; 813 814 prival = ((0xff - ipl) << sc->sc_pmr_shift) & 0xff; 815 __asm volatile("msr "STR(ICC_PMR)", %x0" : : "r" (prival)); 816 __isb(); 817 818 intr_restore(psw); 819 } 820 821 void 822 agintc_enable_wakeup(void) 823 { 824 struct agintc_softc *sc = agintc_sc; 825 struct intrhand *ih; 826 uint8_t *prop; 827 int irq, wakeup; 828 829 for (irq = 0; irq < sc->sc_nintr; irq++) { 830 /* No handler? Disabled already. */ 831 if (TAILQ_EMPTY(&sc->sc_handler[irq].iq_list)) 832 continue; 833 /* Unless we're WAKEUP, disable. */ 834 wakeup = 0; 835 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 836 if (ih->ih_flags & IPL_WAKEUP) { 837 wakeup = 1; 838 break; 839 } 840 } 841 if (!wakeup) 842 agintc_intr_disable(sc, irq); 843 } 844 845 for (irq = 0; irq < sc->sc_nlpi; irq++) { 846 if (sc->sc_lpi[irq] == NULL) 847 continue; 848 ih = sc->sc_lpi[irq]->li_ih; 849 KASSERT(ih != NULL); 850 if (ih->ih_flags & IPL_WAKEUP) 851 continue; 852 prop = AGINTC_DMA_KVA(sc->sc_prop); 853 prop[irq] &= ~GICR_PROP_ENABLE; 854 /* Make globally visible. */ 855 cpu_dcache_wb_range((vaddr_t)&prop[irq], 856 sizeof(*prop)); 857 __asm volatile("dsb sy"); 858 /* Invalidate cache */ 859 agintc_msi_inv(sc->sc_lpi[irq]); 860 } 861 } 862 863 void 864 agintc_disable_wakeup(void) 865 { 866 struct agintc_softc *sc = agintc_sc; 867 struct intrhand *ih; 868 uint8_t *prop; 869 int irq, wakeup; 870 871 for (irq = 0; irq < sc->sc_nintr; irq++) { 872 /* No handler? Keep disabled. */ 873 if (TAILQ_EMPTY(&sc->sc_handler[irq].iq_list)) 874 continue; 875 /* WAKEUPs are already enabled. */ 876 wakeup = 0; 877 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 878 if (ih->ih_flags & IPL_WAKEUP) { 879 wakeup = 1; 880 break; 881 } 882 } 883 if (!wakeup) 884 agintc_intr_enable(sc, irq); 885 } 886 887 for (irq = 0; irq < sc->sc_nlpi; irq++) { 888 if (sc->sc_lpi[irq] == NULL) 889 continue; 890 ih = sc->sc_lpi[irq]->li_ih; 891 KASSERT(ih != NULL); 892 if (ih->ih_flags & IPL_WAKEUP) 893 continue; 894 prop = AGINTC_DMA_KVA(sc->sc_prop); 895 prop[irq] |= GICR_PROP_ENABLE; 896 /* Make globally visible. */ 897 cpu_dcache_wb_range((vaddr_t)&prop[irq], 898 sizeof(*prop)); 899 __asm volatile("dsb sy"); 900 /* Invalidate cache */ 901 agintc_msi_inv(sc->sc_lpi[irq]); 902 } 903 } 904 905 void 906 agintc_intr_enable(struct agintc_softc *sc, int irq) 907 { 908 struct cpu_info *ci = curcpu(); 909 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 910 int bit = 1 << IRQ_TO_REG32BIT(irq); 911 912 if (irq >= 32) { 913 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 914 GICD_ISENABLER(irq), bit); 915 } else { 916 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 917 GICR_ISENABLE0, bit); 918 } 919 } 920 921 void 922 agintc_intr_disable(struct agintc_softc *sc, int irq) 923 { 924 struct cpu_info *ci = curcpu(); 925 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 926 927 if (irq >= 32) { 928 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, 929 GICD_ICENABLER(irq), 1 << IRQ_TO_REG32BIT(irq)); 930 } else { 931 bus_space_write_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 932 GICR_ICENABLE0, 1 << IRQ_TO_REG32BIT(irq)); 933 } 934 } 935 936 void 937 agintc_intr_config(struct agintc_softc *sc, int irq, int type) 938 { 939 uint32_t reg; 940 941 /* Don't dare to change SGIs or PPIs (yet) */ 942 if (irq < 32) 943 return; 944 945 reg = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_ICFGR(irq)); 946 reg &= ~GICD_ICFGR_TRIG_MASK(irq); 947 if (type == IST_EDGE_RISING) 948 reg |= GICD_ICFGR_TRIG_EDGE(irq); 949 else 950 reg |= GICD_ICFGR_TRIG_LEVEL(irq); 951 bus_space_write_4(sc->sc_iot, sc->sc_d_ioh, GICD_ICFGR(irq), reg); 952 } 953 954 void 955 agintc_calc_mask(void) 956 { 957 struct agintc_softc *sc = agintc_sc; 958 int irq; 959 960 for (irq = 0; irq < sc->sc_nintr; irq++) 961 agintc_calc_irq(sc, irq); 962 } 963 964 void 965 agintc_calc_irq(struct agintc_softc *sc, int irq) 966 { 967 struct cpu_info *ci = sc->sc_handler[irq].iq_ci; 968 struct intrhand *ih; 969 int max = IPL_NONE; 970 int min = IPL_HIGH; 971 972 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 973 if (ih->ih_ipl > max) 974 max = ih->ih_ipl; 975 976 if (ih->ih_ipl < min) 977 min = ih->ih_ipl; 978 } 979 980 if (max == IPL_NONE) 981 min = IPL_NONE; 982 983 if (sc->sc_handler[irq].iq_irq_max == max && 984 sc->sc_handler[irq].iq_irq_min == min) 985 return; 986 987 sc->sc_handler[irq].iq_irq_max = max; 988 sc->sc_handler[irq].iq_irq_min = min; 989 990 #ifdef DEBUG_AGINTC 991 if (min != IPL_NONE) 992 printf("irq %d to block at %d %d \n", irq, max, min ); 993 #endif 994 /* Enable interrupts at lower levels, clear -> enable */ 995 /* Set interrupt priority/enable */ 996 if (min != IPL_NONE) { 997 agintc_set_priority(sc, irq, min); 998 agintc_route(sc, irq, IRQ_ENABLE, ci); 999 agintc_intr_enable(sc, irq); 1000 } else { 1001 agintc_intr_disable(sc, irq); 1002 agintc_route(sc, irq, IRQ_DISABLE, ci); 1003 } 1004 } 1005 1006 void 1007 agintc_splx(int new) 1008 { 1009 struct cpu_info *ci = curcpu(); 1010 1011 if (ci->ci_ipending & arm_smask[new]) 1012 arm_do_pending_intr(new); 1013 1014 agintc_setipl(new); 1015 } 1016 1017 int 1018 agintc_spllower(int new) 1019 { 1020 struct cpu_info *ci = curcpu(); 1021 int old = ci->ci_cpl; 1022 1023 agintc_splx(new); 1024 return (old); 1025 } 1026 1027 int 1028 agintc_splraise(int new) 1029 { 1030 struct cpu_info *ci = curcpu(); 1031 int old = ci->ci_cpl; 1032 1033 /* 1034 * setipl must always be called because there is a race window 1035 * where the variable is updated before the mask is set 1036 * an interrupt occurs in that window without the mask always 1037 * being set, the hardware might not get updated on the next 1038 * splraise completely messing up spl protection. 1039 */ 1040 if (old > new) 1041 new = old; 1042 1043 agintc_setipl(new); 1044 return (old); 1045 } 1046 1047 uint32_t 1048 agintc_iack(void) 1049 { 1050 int irq; 1051 1052 __asm volatile("mrs %x0, "STR(ICC_IAR1) : "=r" (irq)); 1053 __asm volatile("dsb sy"); 1054 return irq; 1055 } 1056 1057 void 1058 agintc_route_irq(void *v, int enable, struct cpu_info *ci) 1059 { 1060 struct agintc_softc *sc = agintc_sc; 1061 struct intrhand *ih = v; 1062 1063 if (enable) { 1064 agintc_set_priority(sc, ih->ih_irq, 1065 sc->sc_handler[ih->ih_irq].iq_irq_min); 1066 agintc_route(sc, ih->ih_irq, IRQ_ENABLE, ci); 1067 agintc_intr_enable(sc, ih->ih_irq); 1068 } 1069 } 1070 1071 void 1072 agintc_route(struct agintc_softc *sc, int irq, int enable, struct cpu_info *ci) 1073 { 1074 /* XXX does not yet support 'participating node' */ 1075 if (irq >= 32) { 1076 #ifdef DEBUG_AGINTC 1077 printf("router %x irq %d val %016llx\n", GICD_IROUTER(irq), 1078 irq, ci->ci_mpidr & MPIDR_AFF); 1079 #endif 1080 bus_space_write_8(sc->sc_iot, sc->sc_d_ioh, 1081 GICD_IROUTER(irq), ci->ci_mpidr & MPIDR_AFF); 1082 } 1083 } 1084 1085 void 1086 agintc_intr_barrier(void *cookie) 1087 { 1088 struct intrhand *ih = cookie; 1089 1090 sched_barrier(ih->ih_ci); 1091 } 1092 1093 void 1094 agintc_run_handler(struct intrhand *ih, void *frame, int s) 1095 { 1096 void *arg; 1097 int handled; 1098 1099 #ifdef MULTIPROCESSOR 1100 int need_lock; 1101 1102 if (ih->ih_flags & IPL_MPSAFE) 1103 need_lock = 0; 1104 else 1105 need_lock = s < IPL_SCHED; 1106 1107 if (need_lock) 1108 KERNEL_LOCK(); 1109 #endif 1110 1111 if (ih->ih_arg) 1112 arg = ih->ih_arg; 1113 else 1114 arg = frame; 1115 1116 handled = ih->ih_func(arg); 1117 if (handled) 1118 ih->ih_count.ec_count++; 1119 1120 #ifdef MULTIPROCESSOR 1121 if (need_lock) 1122 KERNEL_UNLOCK(); 1123 #endif 1124 } 1125 1126 void 1127 agintc_irq_handler(void *frame) 1128 { 1129 struct agintc_softc *sc = agintc_sc; 1130 struct intrhand *ih; 1131 int irq, pri, s; 1132 1133 irq = agintc_iack(); 1134 1135 #ifdef DEBUG_AGINTC 1136 if (irq != 30) 1137 printf("irq %d fired\n", irq); 1138 else { 1139 static int cnt = 0; 1140 if ((cnt++ % 100) == 0) { 1141 printf("irq %d fired * _100\n", irq); 1142 #ifdef DDB 1143 db_enter(); 1144 #endif 1145 } 1146 } 1147 #endif 1148 1149 if (irq == 1023) { 1150 sc->sc_spur.ec_count++; 1151 return; 1152 } 1153 1154 if ((irq >= sc->sc_nintr && irq < LPI_BASE) || 1155 irq >= LPI_BASE + sc->sc_nlpi) { 1156 return; 1157 } 1158 1159 if (irq >= LPI_BASE) { 1160 if (sc->sc_lpi[irq - LPI_BASE] == NULL) 1161 return; 1162 ih = sc->sc_lpi[irq - LPI_BASE]->li_ih; 1163 KASSERT(ih != NULL); 1164 1165 s = agintc_splraise(ih->ih_ipl); 1166 intr_enable(); 1167 agintc_run_handler(ih, frame, s); 1168 intr_disable(); 1169 agintc_eoi(irq); 1170 1171 agintc_splx(s); 1172 return; 1173 } 1174 1175 pri = sc->sc_handler[irq].iq_irq_max; 1176 s = agintc_splraise(pri); 1177 intr_enable(); 1178 TAILQ_FOREACH(ih, &sc->sc_handler[irq].iq_list, ih_list) { 1179 agintc_run_handler(ih, frame, s); 1180 } 1181 intr_disable(); 1182 agintc_eoi(irq); 1183 1184 agintc_splx(s); 1185 } 1186 1187 void * 1188 agintc_intr_establish_fdt(void *cookie, int *cell, int level, 1189 struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 1190 { 1191 struct agintc_softc *sc = agintc_sc; 1192 int irq; 1193 int type; 1194 1195 /* 2nd cell contains the interrupt number */ 1196 irq = cell[1]; 1197 1198 /* 1st cell contains type: 0 SPI (32-X), 1 PPI (16-31) */ 1199 if (cell[0] == 0) 1200 irq += SPI_BASE; 1201 else if (cell[0] == 1) 1202 irq += PPI_BASE; 1203 else 1204 panic("%s: bogus interrupt type", sc->sc_sbus.sc_dev.dv_xname); 1205 1206 /* SPIs are only active-high level or low-to-high edge */ 1207 if (cell[2] & 0x3) 1208 type = IST_EDGE_RISING; 1209 else 1210 type = IST_LEVEL_HIGH; 1211 1212 return agintc_intr_establish(irq, type, level, ci, func, arg, name); 1213 } 1214 1215 void * 1216 agintc_intr_establish(int irqno, int type, int level, struct cpu_info *ci, 1217 int (*func)(void *), void *arg, char *name) 1218 { 1219 struct agintc_softc *sc = agintc_sc; 1220 struct intrhand *ih; 1221 u_long psw; 1222 1223 if (irqno < 0 || (irqno >= sc->sc_nintr && irqno < LPI_BASE) || 1224 irqno >= LPI_BASE + sc->sc_nlpi) 1225 panic("agintc_intr_establish: bogus irqnumber %d: %s", 1226 irqno, name); 1227 1228 if (ci == NULL) 1229 ci = &cpu_info_primary; 1230 1231 ih = malloc(sizeof *ih, M_DEVBUF, M_WAITOK); 1232 ih->ih_func = func; 1233 ih->ih_arg = arg; 1234 ih->ih_ipl = level & IPL_IRQMASK; 1235 ih->ih_flags = level & IPL_FLAGMASK; 1236 ih->ih_irq = irqno; 1237 ih->ih_name = name; 1238 ih->ih_ci = ci; 1239 1240 psw = intr_disable(); 1241 1242 if (irqno < LPI_BASE) { 1243 if (!TAILQ_EMPTY(&sc->sc_handler[irqno].iq_list) && 1244 sc->sc_handler[irqno].iq_ci != ci) { 1245 intr_restore(psw); 1246 free(ih, M_DEVBUF, sizeof *ih); 1247 return NULL; 1248 } 1249 TAILQ_INSERT_TAIL(&sc->sc_handler[irqno].iq_list, ih, ih_list); 1250 sc->sc_handler[irqno].iq_ci = ci; 1251 } 1252 1253 if (name != NULL) 1254 evcount_attach(&ih->ih_count, name, &ih->ih_irq); 1255 1256 #ifdef DEBUG_AGINTC 1257 printf("%s: irq %d level %d [%s]\n", __func__, irqno, level, name); 1258 #endif 1259 1260 if (irqno < LPI_BASE) { 1261 agintc_intr_config(sc, irqno, type); 1262 agintc_calc_irq(sc, irqno); 1263 } else { 1264 uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop); 1265 1266 prop[irqno - LPI_BASE] = (((0xff - ih->ih_ipl) << 4) & 0xff) | 1267 GICR_PROP_GROUP1 | GICR_PROP_ENABLE; 1268 1269 /* Make globally visible. */ 1270 cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE], 1271 sizeof(*prop)); 1272 __asm volatile("dsb sy"); 1273 } 1274 1275 intr_restore(psw); 1276 return (ih); 1277 } 1278 1279 void 1280 agintc_intr_disestablish(void *cookie) 1281 { 1282 struct agintc_softc *sc = agintc_sc; 1283 struct intrhand *ih = cookie; 1284 int irqno = ih->ih_irq; 1285 u_long psw; 1286 struct agintc_mbi_range *mr; 1287 int i; 1288 1289 psw = intr_disable(); 1290 1291 if (irqno < LPI_BASE) { 1292 TAILQ_REMOVE(&sc->sc_handler[irqno].iq_list, ih, ih_list); 1293 agintc_calc_irq(sc, irqno); 1294 1295 /* In case this is an MBI, free it */ 1296 for (i = 0; i < sc->sc_mbi_nranges; i++) { 1297 mr = &sc->sc_mbi_ranges[i]; 1298 if (irqno < mr->mr_base) 1299 continue; 1300 if (irqno >= mr->mr_base + mr->mr_span) 1301 break; 1302 if (mr->mr_mbi[irqno - mr->mr_base] != NULL) 1303 mr->mr_mbi[irqno - mr->mr_base] = NULL; 1304 } 1305 } else { 1306 uint8_t *prop = AGINTC_DMA_KVA(sc->sc_prop); 1307 1308 prop[irqno - LPI_BASE] = 0; 1309 1310 /* Make globally visible. */ 1311 cpu_dcache_wb_range((vaddr_t)&prop[irqno - LPI_BASE], 1312 sizeof(*prop)); 1313 __asm volatile("dsb sy"); 1314 } 1315 1316 if (ih->ih_name != NULL) 1317 evcount_detach(&ih->ih_count); 1318 1319 intr_restore(psw); 1320 1321 free(ih, M_DEVBUF, 0); 1322 } 1323 1324 void 1325 agintc_intr_set_wakeup(void *cookie) 1326 { 1327 struct intrhand *ih = cookie; 1328 1329 ih->ih_flags |= IPL_WAKEUP; 1330 } 1331 1332 void * 1333 agintc_intr_establish_mbi(void *self, uint64_t *addr, uint64_t *data, 1334 int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 1335 { 1336 struct agintc_softc *sc = agintc_sc; 1337 struct agintc_mbi_range *mr; 1338 void *cookie; 1339 int i, j, hwcpu; 1340 1341 if (ci == NULL) 1342 ci = &cpu_info_primary; 1343 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 1344 1345 for (i = 0; i < sc->sc_mbi_nranges; i++) { 1346 mr = &sc->sc_mbi_ranges[i]; 1347 for (j = 0; j < mr->mr_span; j++) { 1348 if (mr->mr_mbi[j] != NULL) 1349 continue; 1350 1351 cookie = agintc_intr_establish(mr->mr_base + j, 1352 IST_EDGE_RISING, level, ci, func, arg, name); 1353 if (cookie == NULL) 1354 return NULL; 1355 1356 *addr = sc->sc_mbi_addr; 1357 *data = mr->mr_base + j; 1358 1359 mr->mr_mbi[j] = cookie; 1360 return cookie; 1361 } 1362 } 1363 1364 return NULL; 1365 } 1366 1367 void 1368 agintc_eoi(uint32_t eoi) 1369 { 1370 __asm volatile("msr "STR(ICC_EOIR1)", %x0" :: "r" (eoi)); 1371 __isb(); 1372 } 1373 1374 void 1375 agintc_d_wait_rwp(struct agintc_softc *sc) 1376 { 1377 int count = 100000; 1378 uint32_t v; 1379 1380 do { 1381 v = bus_space_read_4(sc->sc_iot, sc->sc_d_ioh, GICD_CTLR); 1382 } while (--count && (v & GICD_CTLR_RWP)); 1383 1384 if (count == 0) 1385 panic("%s: RWP timed out 0x08%x", __func__, v); 1386 } 1387 1388 void 1389 agintc_r_wait_rwp(struct agintc_softc *sc) 1390 { 1391 struct cpu_info *ci = curcpu(); 1392 int hwcpu = sc->sc_cpuremap[ci->ci_cpuid]; 1393 int count = 100000; 1394 uint32_t v; 1395 1396 do { 1397 v = bus_space_read_4(sc->sc_iot, sc->sc_r_ioh[hwcpu], 1398 GICR_CTLR); 1399 } while (--count && (v & GICR_CTLR_RWP)); 1400 1401 if (count == 0) 1402 panic("%s: RWP timed out 0x08%x", __func__, v); 1403 } 1404 1405 #ifdef MULTIPROCESSOR 1406 int 1407 agintc_ipi_ddb(void *v) 1408 { 1409 /* XXX */ 1410 #ifdef DDB 1411 db_enter(); 1412 #endif 1413 return 1; 1414 } 1415 1416 int 1417 agintc_ipi_halt(void *v) 1418 { 1419 struct agintc_softc *sc = v; 1420 int old = curcpu()->ci_cpl; 1421 1422 intr_disable(); 1423 agintc_eoi(sc->sc_ipi_num[ARM_IPI_HALT]); 1424 agintc_setipl(IPL_NONE); 1425 1426 cpu_halt(); 1427 1428 agintc_setipl(old); 1429 intr_enable(); 1430 return 1; 1431 } 1432 1433 int 1434 agintc_ipi_nop(void *v) 1435 { 1436 /* Nothing to do here, just enough to wake up from WFI */ 1437 return 1; 1438 } 1439 1440 int 1441 agintc_ipi_combined(void *v) 1442 { 1443 struct agintc_softc *sc = v; 1444 1445 if (sc->sc_ipi_reason[cpu_number()] == ARM_IPI_DDB) { 1446 sc->sc_ipi_reason[cpu_number()] = ARM_IPI_NOP; 1447 return agintc_ipi_ddb(v); 1448 } else if (sc->sc_ipi_reason[cpu_number()] == ARM_IPI_HALT) { 1449 sc->sc_ipi_reason[cpu_number()] = ARM_IPI_NOP; 1450 return agintc_ipi_halt(v); 1451 } else { 1452 return agintc_ipi_nop(v); 1453 } 1454 } 1455 1456 void 1457 agintc_send_ipi(struct cpu_info *ci, int id) 1458 { 1459 struct agintc_softc *sc = agintc_sc; 1460 uint64_t sendmask; 1461 1462 if (ci == curcpu() && id == ARM_IPI_NOP) 1463 return; 1464 1465 /* never overwrite IPI_DDB or IPI_HALT with IPI_NOP */ 1466 if (id == ARM_IPI_DDB || id == ARM_IPI_HALT) 1467 sc->sc_ipi_reason[ci->ci_cpuid] = id; 1468 1469 /* will only send 1 cpu */ 1470 sendmask = (ci->ci_mpidr & MPIDR_AFF3) << 16; 1471 sendmask |= (ci->ci_mpidr & MPIDR_AFF2) << 16; 1472 sendmask |= (ci->ci_mpidr & MPIDR_AFF1) << 8; 1473 sendmask |= 1 << (ci->ci_mpidr & 0x0f); 1474 sendmask |= (sc->sc_ipi_num[id] << 24); 1475 1476 __asm volatile ("msr " STR(ICC_SGI1R)", %x0" ::"r"(sendmask)); 1477 } 1478 #endif 1479 1480 /* 1481 * GICv3 ITS controller for MSI interrupts. 1482 */ 1483 #define GITS_CTLR 0x0000 1484 #define GITS_CTLR_ENABLED (1UL << 0) 1485 #define GITS_TYPER 0x0008 1486 #define GITS_TYPER_CIL (1ULL << 36) 1487 #define GITS_TYPER_CIDBITS(x) (((x) >> 32) & 0xf) 1488 #define GITS_TYPER_HCC(x) (((x) >> 24) & 0xff) 1489 #define GITS_TYPER_PTA (1ULL << 19) 1490 #define GITS_TYPER_DEVBITS(x) (((x) >> 13) & 0x1f) 1491 #define GITS_TYPER_ITE_SZ(x) (((x) >> 4) & 0xf) 1492 #define GITS_TYPER_PHYS (1ULL << 0) 1493 #define GITS_CBASER 0x0080 1494 #define GITS_CBASER_VALID (1ULL << 63) 1495 #define GITS_CBASER_IC_NORM_NC (1ULL << 59) 1496 #define GITS_CBASER_MASK 0x1ffffffffff000ULL 1497 #define GITS_CWRITER 0x0088 1498 #define GITS_CREADR 0x0090 1499 #define GITS_BASER(i) (0x0100 + ((i) * 8)) 1500 #define GITS_BASER_VALID (1ULL << 63) 1501 #define GITS_BASER_INDIRECT (1ULL << 62) 1502 #define GITS_BASER_IC_NORM_NC (1ULL << 59) 1503 #define GITS_BASER_TYPE_MASK (7ULL << 56) 1504 #define GITS_BASER_TYPE_DEVICE (1ULL << 56) 1505 #define GITS_BASER_TYPE_COLL (4ULL << 56) 1506 #define GITS_BASER_TTE_SZ(x) (((x) >> 48) & 0x1f) 1507 #define GITS_BASER_PGSZ_MASK (3ULL << 8) 1508 #define GITS_BASER_PGSZ_4K (0ULL << 8) 1509 #define GITS_BASER_PGSZ_16K (1ULL << 8) 1510 #define GITS_BASER_PGSZ_64K (2ULL << 8) 1511 #define GITS_BASER_PA_MASK 0x7ffffffff000ULL 1512 #define GITS_TRANSLATER 0x10040 1513 1514 #define GITS_NUM_BASER 8 1515 1516 struct gits_cmd { 1517 uint8_t cmd; 1518 uint32_t deviceid; 1519 uint32_t eventid; 1520 uint32_t intid; 1521 uint64_t dw2; 1522 uint64_t dw3; 1523 }; 1524 1525 #define GITS_CMD_VALID (1ULL << 63) 1526 1527 /* ITS commands */ 1528 #define SYNC 0x05 1529 #define MAPD 0x08 1530 #define MAPC 0x09 1531 #define MAPTI 0x0a 1532 #define INV 0x0c 1533 #define INVALL 0x0d 1534 #define DISCARD 0x0f 1535 1536 #define GITS_CMDQ_SIZE (64 * 1024) 1537 #define GITS_CMDQ_NENTRIES (GITS_CMDQ_SIZE / sizeof(struct gits_cmd)) 1538 1539 struct agintc_msi_device { 1540 LIST_ENTRY(agintc_msi_device) md_list; 1541 1542 uint32_t md_deviceid; 1543 uint32_t md_eventid; 1544 struct agintc_dmamem *md_itt; 1545 }; 1546 1547 int agintc_msi_match(struct device *, void *, void *); 1548 void agintc_msi_attach(struct device *, struct device *, void *); 1549 void *agintc_intr_establish_msi(void *, uint64_t *, uint64_t *, 1550 int , struct cpu_info *, int (*)(void *), void *, char *); 1551 void agintc_intr_disestablish_msi(void *); 1552 void agintc_intr_barrier_msi(void *); 1553 1554 struct agintc_msi_softc { 1555 struct device sc_dev; 1556 bus_space_tag_t sc_iot; 1557 bus_space_handle_t sc_ioh; 1558 bus_dma_tag_t sc_dmat; 1559 1560 bus_addr_t sc_msi_addr; 1561 int sc_msi_delta; 1562 1563 struct agintc_dmamem *sc_cmdq; 1564 uint16_t sc_cmdidx; 1565 1566 int sc_devbits; 1567 struct agintc_dmamem *sc_dtt; 1568 size_t sc_dtt_pgsz; 1569 uint8_t sc_dte_sz; 1570 int sc_cidbits; 1571 struct agintc_dmamem *sc_ctt; 1572 size_t sc_ctt_pgsz; 1573 uint8_t sc_cte_sz; 1574 uint8_t sc_ite_sz; 1575 1576 LIST_HEAD(, agintc_msi_device) sc_msi_devices; 1577 1578 struct interrupt_controller sc_ic; 1579 }; 1580 1581 const struct cfattach agintcmsi_ca = { 1582 sizeof (struct agintc_msi_softc), agintc_msi_match, agintc_msi_attach 1583 }; 1584 1585 struct cfdriver agintcmsi_cd = { 1586 NULL, "agintcmsi", DV_DULL 1587 }; 1588 1589 void agintc_msi_send_cmd(struct agintc_msi_softc *, struct gits_cmd *); 1590 void agintc_msi_wait_cmd(struct agintc_msi_softc *); 1591 1592 int 1593 agintc_msi_match(struct device *parent, void *cfdata, void *aux) 1594 { 1595 struct fdt_attach_args *faa = aux; 1596 1597 return OF_is_compatible(faa->fa_node, "arm,gic-v3-its"); 1598 } 1599 1600 void 1601 agintc_msi_attach(struct device *parent, struct device *self, void *aux) 1602 { 1603 struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; 1604 struct fdt_attach_args *faa = aux; 1605 struct gits_cmd cmd; 1606 uint32_t pre_its[2]; 1607 uint64_t typer; 1608 int i, hwcpu; 1609 1610 if (faa->fa_nreg < 1) { 1611 printf(": no registers\n"); 1612 return; 1613 } 1614 1615 sc->sc_iot = faa->fa_iot; 1616 if (bus_space_map(sc->sc_iot, faa->fa_reg[0].addr, 1617 faa->fa_reg[0].size, 0, &sc->sc_ioh)) { 1618 printf(": can't map registers\n"); 1619 return; 1620 } 1621 sc->sc_dmat = faa->fa_dmat; 1622 1623 sc->sc_msi_addr = faa->fa_reg[0].addr + GITS_TRANSLATER; 1624 if (OF_getpropintarray(faa->fa_node, "socionext,synquacer-pre-its", 1625 pre_its, sizeof(pre_its)) == sizeof(pre_its)) { 1626 sc->sc_msi_addr = pre_its[0]; 1627 sc->sc_msi_delta = 4; 1628 } 1629 1630 typer = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_TYPER); 1631 if ((typer & GITS_TYPER_PHYS) == 0 || typer & GITS_TYPER_PTA) { 1632 printf(": unsupported type 0x%016llx\n", typer); 1633 goto unmap; 1634 } 1635 sc->sc_ite_sz = GITS_TYPER_ITE_SZ(typer) + 1; 1636 sc->sc_devbits = GITS_TYPER_DEVBITS(typer) + 1; 1637 if (typer & GITS_TYPER_CIL) 1638 sc->sc_cidbits = GITS_TYPER_CIDBITS(typer) + 1; 1639 else 1640 sc->sc_cidbits = 16; 1641 1642 /* Set up command queue. */ 1643 sc->sc_cmdq = agintc_dmamem_alloc(sc->sc_dmat, 1644 GITS_CMDQ_SIZE, GITS_CMDQ_SIZE); 1645 if (sc->sc_cmdq == NULL) { 1646 printf(": can't alloc command queue\n"); 1647 goto unmap; 1648 } 1649 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CBASER, 1650 AGINTC_DMA_DVA(sc->sc_cmdq) | GITS_CBASER_IC_NORM_NC | 1651 (GITS_CMDQ_SIZE / PAGE_SIZE) - 1 | GITS_CBASER_VALID); 1652 1653 /* Set up device translation table. */ 1654 for (i = 0; i < GITS_NUM_BASER; i++) { 1655 uint64_t baser; 1656 paddr_t dtt_pa; 1657 size_t size; 1658 1659 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1660 if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_DEVICE) 1661 continue; 1662 1663 /* Determine the maximum supported page size. */ 1664 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1665 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_64K); 1666 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1667 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_64K) 1668 goto dfound; 1669 1670 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1671 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_16K); 1672 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1673 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_16K) 1674 goto dfound; 1675 1676 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1677 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_4K); 1678 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1679 1680 dfound: 1681 switch (baser & GITS_BASER_PGSZ_MASK) { 1682 case GITS_BASER_PGSZ_4K: 1683 sc->sc_dtt_pgsz = PAGE_SIZE; 1684 break; 1685 case GITS_BASER_PGSZ_16K: 1686 sc->sc_dtt_pgsz = 4 * PAGE_SIZE; 1687 break; 1688 case GITS_BASER_PGSZ_64K: 1689 sc->sc_dtt_pgsz = 16 * PAGE_SIZE; 1690 break; 1691 } 1692 1693 /* Calculate table size. */ 1694 sc->sc_dte_sz = GITS_BASER_TTE_SZ(baser) + 1; 1695 size = (1ULL << sc->sc_devbits) * sc->sc_dte_sz; 1696 size = roundup(size, sc->sc_dtt_pgsz); 1697 1698 /* Allocate table. */ 1699 sc->sc_dtt = agintc_dmamem_alloc(sc->sc_dmat, 1700 size, sc->sc_dtt_pgsz); 1701 if (sc->sc_dtt == NULL) { 1702 printf(": can't alloc translation table\n"); 1703 goto unmap; 1704 } 1705 1706 /* Configure table. */ 1707 dtt_pa = AGINTC_DMA_DVA(sc->sc_dtt); 1708 KASSERT((dtt_pa & GITS_BASER_PA_MASK) == dtt_pa); 1709 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1710 GITS_BASER_IC_NORM_NC | baser & GITS_BASER_PGSZ_MASK | 1711 dtt_pa | (size / sc->sc_dtt_pgsz) - 1 | GITS_BASER_VALID); 1712 } 1713 1714 /* Set up collection translation table. */ 1715 for (i = 0; i < GITS_NUM_BASER; i++) { 1716 uint64_t baser; 1717 paddr_t ctt_pa; 1718 size_t size; 1719 1720 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1721 if ((baser & GITS_BASER_TYPE_MASK) != GITS_BASER_TYPE_COLL) 1722 continue; 1723 1724 /* Determine the maximum supported page size. */ 1725 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1726 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_64K); 1727 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1728 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_64K) 1729 goto cfound; 1730 1731 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1732 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_16K); 1733 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1734 if ((baser & GITS_BASER_PGSZ_MASK) == GITS_BASER_PGSZ_16K) 1735 goto cfound; 1736 1737 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1738 (baser & ~GITS_BASER_PGSZ_MASK) | GITS_BASER_PGSZ_4K); 1739 baser = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i)); 1740 1741 cfound: 1742 switch (baser & GITS_BASER_PGSZ_MASK) { 1743 case GITS_BASER_PGSZ_4K: 1744 sc->sc_ctt_pgsz = PAGE_SIZE; 1745 break; 1746 case GITS_BASER_PGSZ_16K: 1747 sc->sc_ctt_pgsz = 4 * PAGE_SIZE; 1748 break; 1749 case GITS_BASER_PGSZ_64K: 1750 sc->sc_ctt_pgsz = 16 * PAGE_SIZE; 1751 break; 1752 } 1753 1754 /* Calculate table size. */ 1755 sc->sc_cte_sz = GITS_BASER_TTE_SZ(baser) + 1; 1756 size = (1ULL << sc->sc_cidbits) * sc->sc_cte_sz; 1757 size = roundup(size, sc->sc_ctt_pgsz); 1758 1759 /* Allocate table. */ 1760 sc->sc_ctt = agintc_dmamem_alloc(sc->sc_dmat, 1761 size, sc->sc_ctt_pgsz); 1762 if (sc->sc_ctt == NULL) { 1763 printf(": can't alloc translation table\n"); 1764 goto unmap; 1765 } 1766 1767 /* Configure table. */ 1768 ctt_pa = AGINTC_DMA_DVA(sc->sc_ctt); 1769 KASSERT((ctt_pa & GITS_BASER_PA_MASK) == ctt_pa); 1770 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_BASER(i), 1771 GITS_BASER_IC_NORM_NC | baser & GITS_BASER_PGSZ_MASK | 1772 ctt_pa | (size / sc->sc_ctt_pgsz) - 1 | GITS_BASER_VALID); 1773 } 1774 1775 /* Enable ITS. */ 1776 bus_space_write_4(sc->sc_iot, sc->sc_ioh, GITS_CTLR, 1777 GITS_CTLR_ENABLED); 1778 1779 LIST_INIT(&sc->sc_msi_devices); 1780 1781 /* Create one collection per core. */ 1782 KASSERT(ncpus <= agintc_sc->sc_num_redist); 1783 for (i = 0; i < ncpus; i++) { 1784 hwcpu = agintc_sc->sc_cpuremap[i]; 1785 memset(&cmd, 0, sizeof(cmd)); 1786 cmd.cmd = MAPC; 1787 cmd.dw2 = GITS_CMD_VALID | 1788 (agintc_sc->sc_processor[hwcpu] << 16) | i; 1789 agintc_msi_send_cmd(sc, &cmd); 1790 agintc_msi_wait_cmd(sc); 1791 } 1792 1793 printf("\n"); 1794 1795 sc->sc_ic.ic_node = faa->fa_node; 1796 sc->sc_ic.ic_cookie = sc; 1797 sc->sc_ic.ic_establish_msi = agintc_intr_establish_msi; 1798 sc->sc_ic.ic_disestablish = agintc_intr_disestablish_msi; 1799 sc->sc_ic.ic_barrier = agintc_intr_barrier_msi; 1800 sc->sc_ic.ic_gic_its_id = OF_getpropint(faa->fa_node, 1801 "openbsd,gic-its-id", 0); 1802 arm_intr_register_fdt(&sc->sc_ic); 1803 return; 1804 1805 unmap: 1806 if (sc->sc_dtt) 1807 agintc_dmamem_free(sc->sc_dmat, sc->sc_dtt); 1808 if (sc->sc_cmdq) 1809 agintc_dmamem_free(sc->sc_dmat, sc->sc_cmdq); 1810 1811 bus_space_unmap(sc->sc_iot, sc->sc_ioh, faa->fa_reg[0].size); 1812 } 1813 1814 void 1815 agintc_msi_send_cmd(struct agintc_msi_softc *sc, struct gits_cmd *cmd) 1816 { 1817 struct gits_cmd *queue = AGINTC_DMA_KVA(sc->sc_cmdq); 1818 1819 memcpy(&queue[sc->sc_cmdidx], cmd, sizeof(*cmd)); 1820 1821 /* Make globally visible. */ 1822 cpu_dcache_wb_range((vaddr_t)&queue[sc->sc_cmdidx], sizeof(*cmd)); 1823 __asm volatile("dsb sy"); 1824 1825 sc->sc_cmdidx++; 1826 sc->sc_cmdidx %= GITS_CMDQ_NENTRIES; 1827 bus_space_write_8(sc->sc_iot, sc->sc_ioh, GITS_CWRITER, 1828 sc->sc_cmdidx * sizeof(*cmd)); 1829 } 1830 1831 void 1832 agintc_msi_wait_cmd(struct agintc_msi_softc *sc) 1833 { 1834 uint64_t creadr; 1835 int timo; 1836 1837 for (timo = 1000; timo > 0; timo--) { 1838 creadr = bus_space_read_8(sc->sc_iot, sc->sc_ioh, GITS_CREADR); 1839 if (creadr == sc->sc_cmdidx * sizeof(struct gits_cmd)) 1840 break; 1841 delay(1); 1842 } 1843 if (timo == 0) 1844 printf("%s: command queue timeout\n", sc->sc_dev.dv_xname); 1845 } 1846 1847 struct agintc_msi_device * 1848 agintc_msi_create_device(struct agintc_msi_softc *sc, uint32_t deviceid) 1849 { 1850 struct agintc_msi_device *md; 1851 struct gits_cmd cmd; 1852 1853 md = malloc(sizeof(*md), M_DEVBUF, M_ZERO | M_WAITOK); 1854 md->md_deviceid = deviceid; 1855 md->md_itt = agintc_dmamem_alloc(sc->sc_dmat, 1856 32 * sc->sc_ite_sz, PAGE_SIZE); 1857 LIST_INSERT_HEAD(&sc->sc_msi_devices, md, md_list); 1858 1859 memset(&cmd, 0, sizeof(cmd)); 1860 cmd.cmd = MAPD; 1861 cmd.deviceid = deviceid; 1862 cmd.eventid = 4; /* size */ 1863 cmd.dw2 = AGINTC_DMA_DVA(md->md_itt) | GITS_CMD_VALID; 1864 agintc_msi_send_cmd(sc, &cmd); 1865 agintc_msi_wait_cmd(sc); 1866 1867 return md; 1868 } 1869 1870 struct agintc_msi_device * 1871 agintc_msi_find_device(struct agintc_msi_softc *sc, uint32_t deviceid) 1872 { 1873 struct agintc_msi_device *md; 1874 1875 LIST_FOREACH(md, &sc->sc_msi_devices, md_list) { 1876 if (md->md_deviceid == deviceid) 1877 return md; 1878 } 1879 1880 return agintc_msi_create_device(sc, deviceid); 1881 } 1882 1883 void 1884 agintc_msi_discard(struct agintc_lpi_info *li) 1885 { 1886 struct agintc_msi_softc *sc; 1887 struct cpu_info *ci; 1888 struct gits_cmd cmd; 1889 int hwcpu; 1890 1891 sc = li->li_msic; 1892 ci = li->li_ci; 1893 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 1894 1895 memset(&cmd, 0, sizeof(cmd)); 1896 cmd.cmd = DISCARD; 1897 cmd.deviceid = li->li_deviceid; 1898 cmd.eventid = li->li_eventid; 1899 agintc_msi_send_cmd(sc, &cmd); 1900 1901 memset(&cmd, 0, sizeof(cmd)); 1902 cmd.cmd = SYNC; 1903 cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; 1904 agintc_msi_send_cmd(sc, &cmd); 1905 agintc_msi_wait_cmd(sc); 1906 } 1907 1908 void 1909 agintc_msi_inv(struct agintc_lpi_info *li) 1910 { 1911 struct agintc_msi_softc *sc; 1912 struct cpu_info *ci; 1913 struct gits_cmd cmd; 1914 int hwcpu; 1915 1916 sc = li->li_msic; 1917 ci = li->li_ci; 1918 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 1919 1920 memset(&cmd, 0, sizeof(cmd)); 1921 cmd.cmd = INV; 1922 cmd.deviceid = li->li_deviceid; 1923 cmd.eventid = li->li_eventid; 1924 agintc_msi_send_cmd(sc, &cmd); 1925 1926 memset(&cmd, 0, sizeof(cmd)); 1927 cmd.cmd = SYNC; 1928 cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; 1929 agintc_msi_send_cmd(sc, &cmd); 1930 agintc_msi_wait_cmd(sc); 1931 } 1932 1933 void * 1934 agintc_intr_establish_msi(void *self, uint64_t *addr, uint64_t *data, 1935 int level, struct cpu_info *ci, int (*func)(void *), void *arg, char *name) 1936 { 1937 struct agintc_msi_softc *sc = (struct agintc_msi_softc *)self; 1938 struct agintc_msi_device *md; 1939 struct gits_cmd cmd; 1940 uint32_t deviceid = *data; 1941 uint32_t eventid; 1942 int i, hwcpu; 1943 1944 if (ci == NULL) 1945 ci = &cpu_info_primary; 1946 hwcpu = agintc_sc->sc_cpuremap[ci->ci_cpuid]; 1947 1948 md = agintc_msi_find_device(sc, deviceid); 1949 if (md == NULL) 1950 return NULL; 1951 1952 eventid = md->md_eventid++; 1953 if (eventid >= 32) 1954 return NULL; 1955 1956 for (i = 0; i < agintc_sc->sc_nlpi; i++) { 1957 if (agintc_sc->sc_lpi[i] != NULL) 1958 continue; 1959 1960 agintc_sc->sc_lpi[i] = malloc(sizeof(struct agintc_lpi_info), 1961 M_DEVBUF, M_WAITOK | M_ZERO); 1962 agintc_sc->sc_lpi[i]->li_msic = sc; 1963 agintc_sc->sc_lpi[i]->li_ci = ci; 1964 agintc_sc->sc_lpi[i]->li_deviceid = deviceid; 1965 agintc_sc->sc_lpi[i]->li_eventid = eventid; 1966 agintc_sc->sc_lpi[i]->li_ih = 1967 agintc_intr_establish(LPI_BASE + i, 1968 IST_EDGE_RISING, level, ci, func, arg, name); 1969 if (agintc_sc->sc_lpi[i]->li_ih == NULL) { 1970 free(agintc_sc->sc_lpi[i], M_DEVBUF, 1971 sizeof(struct agintc_lpi_info)); 1972 agintc_sc->sc_lpi[i] = NULL; 1973 return NULL; 1974 } 1975 1976 memset(&cmd, 0, sizeof(cmd)); 1977 cmd.cmd = MAPTI; 1978 cmd.deviceid = deviceid; 1979 cmd.eventid = eventid; 1980 cmd.intid = LPI_BASE + i; 1981 cmd.dw2 = ci->ci_cpuid; 1982 agintc_msi_send_cmd(sc, &cmd); 1983 1984 memset(&cmd, 0, sizeof(cmd)); 1985 cmd.cmd = SYNC; 1986 cmd.dw2 = agintc_sc->sc_processor[hwcpu] << 16; 1987 agintc_msi_send_cmd(sc, &cmd); 1988 agintc_msi_wait_cmd(sc); 1989 1990 *addr = sc->sc_msi_addr + deviceid * sc->sc_msi_delta; 1991 *data = eventid; 1992 return &agintc_sc->sc_lpi[i]; 1993 } 1994 1995 return NULL; 1996 } 1997 1998 void 1999 agintc_intr_disestablish_msi(void *cookie) 2000 { 2001 struct agintc_lpi_info *li = *(void **)cookie; 2002 2003 agintc_intr_disestablish(li->li_ih); 2004 agintc_msi_discard(li); 2005 agintc_msi_inv(li); 2006 2007 free(li, M_DEVBUF, sizeof(*li)); 2008 *(void **)cookie = NULL; 2009 } 2010 2011 void 2012 agintc_intr_barrier_msi(void *cookie) 2013 { 2014 struct agintc_lpi_info *li = *(void **)cookie; 2015 2016 agintc_intr_barrier(li->li_ih); 2017 } 2018 2019 struct agintc_dmamem * 2020 agintc_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align) 2021 { 2022 struct agintc_dmamem *adm; 2023 int nsegs; 2024 2025 adm = malloc(sizeof(*adm), M_DEVBUF, M_WAITOK | M_ZERO); 2026 adm->adm_size = size; 2027 2028 if (bus_dmamap_create(dmat, size, 1, size, 0, 2029 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &adm->adm_map) != 0) 2030 goto admfree; 2031 2032 if (bus_dmamem_alloc(dmat, size, align, 0, &adm->adm_seg, 1, 2033 &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 2034 goto destroy; 2035 2036 if (bus_dmamem_map(dmat, &adm->adm_seg, nsegs, size, 2037 &adm->adm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0) 2038 goto free; 2039 2040 if (bus_dmamap_load_raw(dmat, adm->adm_map, &adm->adm_seg, 2041 nsegs, size, BUS_DMA_WAITOK) != 0) 2042 goto unmap; 2043 2044 /* Make globally visible. */ 2045 cpu_dcache_wb_range((vaddr_t)adm->adm_kva, size); 2046 __asm volatile("dsb sy"); 2047 return adm; 2048 2049 unmap: 2050 bus_dmamem_unmap(dmat, adm->adm_kva, size); 2051 free: 2052 bus_dmamem_free(dmat, &adm->adm_seg, 1); 2053 destroy: 2054 bus_dmamap_destroy(dmat, adm->adm_map); 2055 admfree: 2056 free(adm, M_DEVBUF, sizeof(*adm)); 2057 2058 return NULL; 2059 } 2060 2061 void 2062 agintc_dmamem_free(bus_dma_tag_t dmat, struct agintc_dmamem *adm) 2063 { 2064 bus_dmamem_unmap(dmat, adm->adm_kva, adm->adm_size); 2065 bus_dmamem_free(dmat, &adm->adm_seg, 1); 2066 bus_dmamap_destroy(dmat, adm->adm_map); 2067 free(adm, M_DEVBUF, sizeof(*adm)); 2068 } 2069