1 /* $NetBSD: bcm2835_intr.c,v 1.44 2022/11/19 09:29:26 yamt Exp $ */
2
3 /*-
4 * Copyright (c) 2012, 2015, 2019 The NetBSD Foundation, Inc.
5 * All rights reserved.
6 *
7 * This code is derived from software contributed to The NetBSD Foundation
8 * by Nick Hudson
9 *
10 * Redistribution and use in source and binary forms, with or without
11 * modification, are permitted provided that the following conditions
12 * are met:
13 * 1. Redistributions of source code must retain the above copyright
14 * notice, this list of conditions and the following disclaimer.
15 * 2. Redistributions in binary form must reproduce the above copyright
16 * notice, this list of conditions and the following disclaimer in the
17 * documentation and/or other materials provided with the distribution.
18 *
19 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
20 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
21 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
23 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
24 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
25 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
26 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
27 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
28 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
29 * POSSIBILITY OF SUCH DAMAGE.
30 */
31
32 #include <sys/cdefs.h>
33 __KERNEL_RCSID(0, "$NetBSD: bcm2835_intr.c,v 1.44 2022/11/19 09:29:26 yamt Exp $");
34
35 #define _INTR_PRIVATE
36
37 #include "opt_bcm283x.h"
38
39 #include <sys/param.h>
40 #include <sys/bus.h>
41 #include <sys/cpu.h>
42 #include <sys/device.h>
43 #include <sys/kernel.h>
44 #include <sys/kmem.h>
45 #include <sys/proc.h>
46
47 #include <dev/fdt/fdtvar.h>
48
49 #include <machine/intr.h>
50
51 #include <arm/locore.h>
52
53 #include <arm/pic/picvar.h>
54 #include <arm/cortex/gtmr_var.h>
55
56 #include <arm/broadcom/bcm2835_intr.h>
57 #include <arm/broadcom/bcm2835reg.h>
58 #include <arm/broadcom/bcm2835var.h>
59
60 #include <arm/fdt/arm_fdtvar.h>
61
62 static void bcm2835_irq_handler(void *);
63
64 static void bcm2835_pic_unblock_irqs(struct pic_softc *, size_t, uint32_t);
65 static void bcm2835_pic_block_irqs(struct pic_softc *, size_t, uint32_t);
66 static int bcm2835_pic_find_pending_irqs(struct pic_softc *);
67 static void bcm2835_pic_establish_irq(struct pic_softc *, struct intrsource *);
68 static void bcm2835_pic_source_name(struct pic_softc *, int, char *,
69 size_t);
70
71 static void bcm2836mp_pic_unblock_irqs(struct pic_softc *, size_t, uint32_t);
72 static void bcm2836mp_pic_block_irqs(struct pic_softc *, size_t, uint32_t);
73 static int bcm2836mp_pic_find_pending_irqs(struct pic_softc *);
74 static void bcm2836mp_pic_establish_irq(struct pic_softc *, struct intrsource *);
75 static void bcm2836mp_pic_source_name(struct pic_softc *, int, char *,
76 size_t);
77 #ifdef MULTIPROCESSOR
78 int bcm2836mp_ipi_handler(void *);
79 static void bcm2836mp_intr_init(struct cpu_info *);
80 static void bcm2836mp_cpu_init(struct pic_softc *, struct cpu_info *);
81 static void bcm2836mp_send_ipi(struct pic_softc *, const kcpuset_t *, u_long);
82 #endif
83
84 static int bcm2835_icu_fdt_decode_irq(u_int *);
85 static void *bcm2835_icu_fdt_establish(device_t, u_int *, int, int,
86 int (*)(void *), void *, const char *);
87 static void bcm2835_icu_fdt_disestablish(device_t, void *);
88 static bool bcm2835_icu_fdt_intrstr(device_t, u_int *, char *, size_t);
89
90 static int bcm2835_icu_intr(void *);
91
92 static int bcm2836mp_icu_fdt_decode_irq(u_int *);
93 static void *bcm2836mp_icu_fdt_establish(device_t, u_int *, int, int,
94 int (*)(void *), void *, const char *);
95 static void bcm2836mp_icu_fdt_disestablish(device_t, void *);
96 static bool bcm2836mp_icu_fdt_intrstr(device_t, u_int *, char *, size_t);
97
98 static int bcm2835_icu_match(device_t, cfdata_t, void *);
99 static void bcm2835_icu_attach(device_t, device_t, void *);
100
101 static int bcm2835_int_base;
102 #if defined(MULTIPROCESSOR)
103 #define _BCM2836_NCPUS BCM2836_NCPUS
104 #else
105 #define _BCM2836_NCPUS 1
106 #endif
107 static int bcm2836mp_int_base[_BCM2836_NCPUS];
108
109 #define BCM2835_INT_BASE bcm2835_int_base
110 #define BCM2836_INT_BASECPUN(n) bcm2836mp_int_base[(n)]
111
112 #define BCM2836_INT_CNTPSIRQ_CPUN(n) (BCM2836_INT_BASECPUN(n) + BCM2836_INT_CNTPSIRQ)
113 #define BCM2836_INT_CNTPNSIRQ_CPUN(n) (BCM2836_INT_BASECPUN(n) + BCM2836_INT_CNTPNSIRQ)
114 #define BCM2836_INT_CNTVIRQ_CPUN(n) (BCM2836_INT_BASECPUN(n) + BCM2836_INT_CNTVIRQ)
115 #define BCM2836_INT_CNTHPIRQ_CPUN(n) (BCM2836_INT_BASECPUN(n) + BCM2836_INT_CNTHPIRQ)
116 #define BCM2836_INT_MAILBOX0_CPUN(n) (BCM2836_INT_BASECPUN(n) + BCM2836_INT_MAILBOX0)
117
118 /* Periperal Interrupt sources */
119 #define BCM2835_NIRQ 96
120
121 #define BCM2835_INT_GPU0BASE (BCM2835_INT_BASE + 0)
122 #define BCM2835_INT_TIMER0 (BCM2835_INT_GPU0BASE + 0)
123 #define BCM2835_INT_TIMER1 (BCM2835_INT_GPU0BASE + 1)
124 #define BCM2835_INT_TIMER2 (BCM2835_INT_GPU0BASE + 2)
125 #define BCM2835_INT_TIMER3 (BCM2835_INT_GPU0BASE + 3)
126 #define BCM2835_INT_USB (BCM2835_INT_GPU0BASE + 9)
127 #define BCM2835_INT_DMA0 (BCM2835_INT_GPU0BASE + 16)
128 #define BCM2835_INT_DMA2 (BCM2835_INT_GPU0BASE + 18)
129 #define BCM2835_INT_DMA3 (BCM2835_INT_GPU0BASE + 19)
130 #define BCM2835_INT_AUX (BCM2835_INT_GPU0BASE + 29)
131 #define BCM2835_INT_ARM (BCM2835_INT_GPU0BASE + 30)
132
133 #define BCM2835_INT_GPU1BASE (BCM2835_INT_BASE + 32)
134 #define BCM2835_INT_GPIO0 (BCM2835_INT_GPU1BASE + 17)
135 #define BCM2835_INT_GPIO1 (BCM2835_INT_GPU1BASE + 18)
136 #define BCM2835_INT_GPIO2 (BCM2835_INT_GPU1BASE + 19)
137 #define BCM2835_INT_GPIO3 (BCM2835_INT_GPU1BASE + 20)
138 #define BCM2835_INT_BSC (BCM2835_INT_GPU1BASE + 21)
139 #define BCM2835_INT_SPI0 (BCM2835_INT_GPU1BASE + 22)
140 #define BCM2835_INT_PCM (BCM2835_INT_GPU1BASE + 23)
141 #define BCM2835_INT_SDHOST (BCM2835_INT_GPU1BASE + 24)
142 #define BCM2835_INT_UART0 (BCM2835_INT_GPU1BASE + 25)
143 #define BCM2835_INT_EMMC (BCM2835_INT_GPU1BASE + 30)
144
145 #define BCM2835_INT_BASICBASE (BCM2835_INT_BASE + 64)
146 #define BCM2835_INT_ARMTIMER (BCM2835_INT_BASICBASE + 0)
147 #define BCM2835_INT_ARMMAILBOX (BCM2835_INT_BASICBASE + 1)
148 #define BCM2835_INT_ARMDOORBELL0 (BCM2835_INT_BASICBASE + 2)
149 #define BCM2835_INT_ARMDOORBELL1 (BCM2835_INT_BASICBASE + 3)
150 #define BCM2835_INT_GPU0HALTED (BCM2835_INT_BASICBASE + 4)
151 #define BCM2835_INT_GPU1HALTED (BCM2835_INT_BASICBASE + 5)
152 #define BCM2835_INT_ILLEGALTYPE0 (BCM2835_INT_BASICBASE + 6)
153 #define BCM2835_INT_ILLEGALTYPE1 (BCM2835_INT_BASICBASE + 7)
154
155 static void
bcm2835_set_priority(struct pic_softc * pic,int ipl)156 bcm2835_set_priority(struct pic_softc *pic, int ipl)
157 {
158 curcpu()->ci_cpl = ipl;
159 }
160
161 static struct pic_ops bcm2835_picops = {
162 .pic_unblock_irqs = bcm2835_pic_unblock_irqs,
163 .pic_block_irqs = bcm2835_pic_block_irqs,
164 .pic_find_pending_irqs = bcm2835_pic_find_pending_irqs,
165 .pic_establish_irq = bcm2835_pic_establish_irq,
166 .pic_source_name = bcm2835_pic_source_name,
167 .pic_set_priority = bcm2835_set_priority,
168 };
169
170 static struct pic_softc bcm2835_pic = {
171 .pic_ops = &bcm2835_picops,
172 .pic_maxsources = BCM2835_NIRQ,
173 .pic_name = "bcm2835 pic",
174 };
175
176 static struct pic_ops bcm2836mp_picops = {
177 .pic_unblock_irqs = bcm2836mp_pic_unblock_irqs,
178 .pic_block_irqs = bcm2836mp_pic_block_irqs,
179 .pic_find_pending_irqs = bcm2836mp_pic_find_pending_irqs,
180 .pic_establish_irq = bcm2836mp_pic_establish_irq,
181 .pic_source_name = bcm2836mp_pic_source_name,
182 #if defined(MULTIPROCESSOR)
183 .pic_cpu_init = bcm2836mp_cpu_init,
184 .pic_ipi_send = bcm2836mp_send_ipi,
185 #endif
186 };
187
188 static struct pic_softc bcm2836mp_pic[_BCM2836_NCPUS] = {
189 [0 ... _BCM2836_NCPUS - 1] = {
190 .pic_ops = &bcm2836mp_picops,
191 .pic_maxsources = BCM2836_NIRQPERCPU,
192 .pic_name = "bcm2836 pic",
193 }
194 };
195
196 static struct fdtbus_interrupt_controller_func bcm2835icu_fdt_funcs = {
197 .establish = bcm2835_icu_fdt_establish,
198 .disestablish = bcm2835_icu_fdt_disestablish,
199 .intrstr = bcm2835_icu_fdt_intrstr
200 };
201
202 static struct fdtbus_interrupt_controller_func bcm2836mpicu_fdt_funcs = {
203 .establish = bcm2836mp_icu_fdt_establish,
204 .disestablish = bcm2836mp_icu_fdt_disestablish,
205 .intrstr = bcm2836mp_icu_fdt_intrstr
206 };
207
208 struct bcm2836mp_interrupt {
209 bool bi_done;
210 TAILQ_ENTRY(bcm2836mp_interrupt) bi_next;
211 int bi_irq;
212 int bi_ipl;
213 int bi_flags;
214 int (*bi_func)(void *);
215 void *bi_arg;
216 void *bi_ihs[_BCM2836_NCPUS];
217 };
218
219 static TAILQ_HEAD(, bcm2836mp_interrupt) bcm2836mp_interrupts =
220 TAILQ_HEAD_INITIALIZER(bcm2836mp_interrupts);
221
222 struct bcm2835icu_irqhandler;
223 struct bcm2835icu_irq;
224 struct bcm2835icu_softc;
225
226 struct bcm2835icu_irqhandler {
227 struct bcm2835icu_irq *ih_irq;
228 int (*ih_fn)(void *);
229 void *ih_arg;
230 TAILQ_ENTRY(bcm2835icu_irqhandler) ih_next;
231 };
232
233 struct bcm2835icu_irq {
234 struct bcm2835icu_softc *intr_sc;
235 void *intr_ih;
236 void *intr_arg;
237 int intr_refcnt;
238 int intr_ipl;
239 int intr_irq;
240 int intr_mpsafe;
241 TAILQ_HEAD(, bcm2835icu_irqhandler) intr_handlers;
242 };
243
244 struct bcm2835icu_softc {
245 device_t sc_dev;
246 bus_space_tag_t sc_iot;
247 bus_space_handle_t sc_ioh;
248
249 struct bcm2835icu_irq *sc_irq[BCM2835_NIRQ];
250
251 int sc_phandle;
252 };
253
254 static struct bcm2835icu_softc *bcml1icu_sc;
255 static struct bcm2835icu_softc *bcmicu_sc;
256
257 #define read_bcm2835reg(o) \
258 bus_space_read_4(bcmicu_sc->sc_iot, bcmicu_sc->sc_ioh, (o))
259
260 #define write_bcm2835reg(o, v) \
261 bus_space_write_4(bcmicu_sc->sc_iot, bcmicu_sc->sc_ioh, (o), (v))
262
263
264 #define bcm2835_barrier() \
265 bus_space_barrier(bcmicu_sc->sc_iot, bcmicu_sc->sc_ioh, 0, \
266 BCM2835_ARMICU_SIZE, BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE)
267
268 static const char * const bcm2835_sources[BCM2835_NIRQ] = {
269 "(unused 0)", "(unused 1)", "(unused 2)", "timer3",
270 "(unused 4)", "(unused 5)", "(unused 6)", "jpeg",
271 "(unused 8)", "usb", "(unused 10)", "(unused 11)",
272 "(unused 12)", "(unused 13)", "(unused 14)", "(unused 15)",
273 "dma0", "dma1", "dma2", "dma3",
274 "dma4", "dma5", "dma6", "dma7",
275 "dma8", "dma9", "dma10", "dma11",
276 "dma12", "aux", "(unused 30)", "(unused 31)",
277 "(unused 32)", "(unused 33)", "(unused 34)", "(unused 35)",
278 "(unused 36)", "(unused 37)", "(unused 38)", "(unused 39)",
279 "(unused 40)", "(unused 41)", "(unused 42)", "i2c spl slv",
280 "(unused 44)", "pwa0", "pwa1", "(unused 47)",
281 "smi", "gpio[0]", "gpio[1]", "gpio[2]",
282 "gpio[3]", "i2c", "spi", "pcm",
283 "sdhost", "uart", "(unused 58)", "(unused 59)",
284 "(unused 60)", "(unused 61)", "emmc", "(unused 63)",
285 "Timer", "Mailbox", "Doorbell0", "Doorbell1",
286 "GPU0 Halted", "GPU1 Halted", "Illegal #1", "Illegal #0"
287 };
288
289 static const char * const bcm2836mp_sources[BCM2836_NIRQPERCPU] = {
290 "cntpsirq", "cntpnsirq", "cnthpirq", "cntvirq",
291 "mailbox0", "mailbox1", "mailbox2", "mailbox3",
292 "gpu", "pmu"
293 };
294
295 #define BCM2836_INTBIT_GPUPENDING __BIT(8)
296
297 #define BCM2835_INTBIT_PENDING1 __BIT(8)
298 #define BCM2835_INTBIT_PENDING2 __BIT(9)
299 #define BCM2835_INTBIT_ARM __BITS(0,7)
300 #define BCM2835_INTBIT_GPU0 __BITS(10,14)
301 #define BCM2835_INTBIT_GPU1 __BITS(15,20)
302
303 CFATTACH_DECL_NEW(bcmicu, sizeof(struct bcm2835icu_softc),
304 bcm2835_icu_match, bcm2835_icu_attach, NULL, NULL);
305
306 static const struct device_compatible_entry compat_data[] = {
307 { .compat = "brcm,bcm2708-armctrl-ic", .value = 0 },
308 { .compat = "brcm,bcm2709-armctrl-ic", .value = 0 },
309 { .compat = "brcm,bcm2835-armctrl-ic", .value = 0 },
310 { .compat = "brcm,bcm2836-armctrl-ic", .value = 0 },
311 { .compat = "brcm,bcm2836-l1-intc", .value = 1 },
312 DEVICE_COMPAT_EOL
313 };
314
315 static int
bcm2835_icu_match(device_t parent,cfdata_t cf,void * aux)316 bcm2835_icu_match(device_t parent, cfdata_t cf, void *aux)
317 {
318 struct fdt_attach_args * const faa = aux;
319
320 return of_compatible_match(faa->faa_phandle, compat_data);
321 }
322
323 static void
bcm2835_icu_attach(device_t parent,device_t self,void * aux)324 bcm2835_icu_attach(device_t parent, device_t self, void *aux)
325 {
326 struct bcm2835icu_softc * const sc = device_private(self);
327 struct fdt_attach_args * const faa = aux;
328 struct fdtbus_interrupt_controller_func *ifuncs;
329 const struct device_compatible_entry *dce;
330 const int phandle = faa->faa_phandle;
331 bus_addr_t addr;
332 bus_size_t size;
333 bus_space_handle_t ioh;
334 int error;
335
336 if (fdtbus_get_reg(phandle, 0, &addr, &size) != 0) {
337 aprint_error(": couldn't get registers\n");
338 return;
339 }
340
341 sc->sc_dev = self;
342 sc->sc_iot = faa->faa_bst;
343
344 if (bus_space_map(sc->sc_iot, addr, size, 0, &ioh) != 0) {
345 aprint_error(": couldn't map device\n");
346 return;
347 }
348
349 sc->sc_ioh = ioh;
350 sc->sc_phandle = phandle;
351
352 dce = of_compatible_lookup(faa->faa_phandle, compat_data);
353 KASSERT(dce != NULL);
354
355 if (dce->value != 0) {
356 #if defined(MULTIPROCESSOR)
357 aprint_normal(": Multiprocessor");
358 #endif
359 bcml1icu_sc = sc;
360
361 bus_space_write_4(sc->sc_iot, sc->sc_ioh,
362 BCM2836_LOCAL_CONTROL, 0);
363 bus_space_write_4(sc->sc_iot, sc->sc_ioh,
364 BCM2836_LOCAL_PRESCALER, 0x80000000);
365
366 ifuncs = &bcm2836mpicu_fdt_funcs;
367
368 /*
369 * Register all PICs here in order to avoid pic_add() from
370 * cpu_hatch(). This is the only approved method.
371 */
372 CPU_INFO_ITERATOR cii;
373 struct cpu_info *ci;
374 for (CPU_INFO_FOREACH(cii, ci)) {
375 const cpuid_t cpuid = ci->ci_core_id;
376 struct pic_softc * const pic = &bcm2836mp_pic[cpuid];
377
378 KASSERT(cpuid < _BCM2836_NCPUS);
379 #if defined(MULTIPROCESSOR)
380 pic->pic_cpus = ci->ci_kcpuset;
381 /*
382 * Append "#n" to avoid duplication of .pic_name[]
383 * It should be a unique id for intr_get_source()
384 */
385 char suffix[sizeof("#00000")];
386 snprintf(suffix, sizeof(suffix), "#%lu", cpuid);
387 strlcat(pic->pic_name, suffix, sizeof(pic->pic_name));
388 #endif
389 bcm2836mp_int_base[cpuid] =
390 pic_add(pic, PIC_IRQBASE_ALLOC);
391 #if defined(MULTIPROCESSOR)
392 bcm2836mp_intr_init(ci);
393 #endif
394 }
395 } else {
396 if (bcml1icu_sc == NULL)
397 arm_fdt_irq_set_handler(bcm2835_irq_handler);
398 bcmicu_sc = sc;
399 sc->sc_ioh = ioh;
400 sc->sc_phandle = phandle;
401 bcm2835_int_base = pic_add(&bcm2835_pic, PIC_IRQBASE_ALLOC);
402 ifuncs = &bcm2835icu_fdt_funcs;
403 }
404
405 error = fdtbus_register_interrupt_controller(self, phandle, ifuncs);
406 if (error != 0) {
407 aprint_error(": couldn't register with fdtbus: %d\n", error);
408 return;
409 }
410 aprint_normal("\n");
411 }
412
413 static void
bcm2835_irq_handler(void * frame)414 bcm2835_irq_handler(void *frame)
415 {
416 struct cpu_info * const ci = curcpu();
417 const int oldipl = ci->ci_cpl;
418 const cpuid_t cpuid = ci->ci_core_id;
419 const uint32_t oldipl_mask = __BIT(oldipl);
420 int ipl_mask = 0;
421
422 KASSERT(cpuid < _BCM2836_NCPUS);
423
424 ci->ci_data.cpu_nintr++;
425
426 bcm2835_barrier();
427 if (cpuid == 0) {
428 ipl_mask = bcm2835_pic_find_pending_irqs(&bcm2835_pic);
429 }
430 #if defined(SOC_BCM2836)
431 ipl_mask |= bcm2836mp_pic_find_pending_irqs(&bcm2836mp_pic[cpuid]);
432 #endif
433
434 /*
435 * Record the pending_ipls and deliver them if we can.
436 */
437 if ((ipl_mask & ~oldipl_mask) > oldipl_mask)
438 pic_do_pending_ints(I32_bit, oldipl, frame);
439 }
440
441 static void
bcm2835_pic_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t irq_mask)442 bcm2835_pic_unblock_irqs(struct pic_softc *pic, size_t irqbase,
443 uint32_t irq_mask)
444 {
445
446 write_bcm2835reg(BCM2835_INTC_ENABLEBASE + (irqbase >> 3), irq_mask);
447 bcm2835_barrier();
448 }
449
450 static void
bcm2835_pic_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t irq_mask)451 bcm2835_pic_block_irqs(struct pic_softc *pic, size_t irqbase,
452 uint32_t irq_mask)
453 {
454
455 write_bcm2835reg(BCM2835_INTC_DISABLEBASE + (irqbase >> 3), irq_mask);
456 bcm2835_barrier();
457 }
458
459 /*
460 * Called with interrupts disabled
461 */
462 static int
bcm2835_pic_find_pending_irqs(struct pic_softc * pic)463 bcm2835_pic_find_pending_irqs(struct pic_softc *pic)
464 {
465 int ipl = 0;
466 uint32_t bpending, gpu0irq, gpu1irq, armirq;
467
468 bcm2835_barrier();
469 bpending = read_bcm2835reg(BCM2835_INTC_IRQBPENDING);
470 if (bpending == 0)
471 return 0;
472
473 armirq = bpending & BCM2835_INTBIT_ARM;
474 gpu0irq = bpending & BCM2835_INTBIT_GPU0;
475 gpu1irq = bpending & BCM2835_INTBIT_GPU1;
476
477 if (armirq) {
478 ipl |= pic_mark_pending_sources(pic,
479 BCM2835_INT_BASICBASE - BCM2835_INT_BASE, armirq);
480 }
481
482 if (gpu0irq || (bpending & BCM2835_INTBIT_PENDING1)) {
483 uint32_t pending1;
484
485 pending1 = read_bcm2835reg(BCM2835_INTC_IRQ1PENDING);
486 ipl |= pic_mark_pending_sources(pic,
487 BCM2835_INT_GPU0BASE - BCM2835_INT_BASE, pending1);
488 }
489 if (gpu1irq || (bpending & BCM2835_INTBIT_PENDING2)) {
490 uint32_t pending2;
491
492 pending2 = read_bcm2835reg(BCM2835_INTC_IRQ2PENDING);
493 ipl |= pic_mark_pending_sources(pic,
494 BCM2835_INT_GPU1BASE - BCM2835_INT_BASE, pending2);
495 }
496
497 return ipl;
498 }
499
500 static void
bcm2835_pic_establish_irq(struct pic_softc * pic,struct intrsource * is)501 bcm2835_pic_establish_irq(struct pic_softc *pic, struct intrsource *is)
502 {
503
504 /* Nothing really*/
505 KASSERT(is->is_irq < BCM2835_NIRQ);
506 KASSERT(is->is_type == IST_LEVEL);
507 }
508
509 static void
bcm2835_pic_source_name(struct pic_softc * pic,int irq,char * buf,size_t len)510 bcm2835_pic_source_name(struct pic_softc *pic, int irq, char *buf, size_t len)
511 {
512
513 strlcpy(buf, bcm2835_sources[irq], len);
514 }
515
516 static int
bcm2835_icu_fdt_decode_irq(u_int * specifier)517 bcm2835_icu_fdt_decode_irq(u_int *specifier)
518 {
519 u_int base;
520
521 if (!specifier)
522 return -1;
523
524 /* 1st cell is the bank number. 0 = ARM, 1 = GPU0, 2 = GPU1 */
525 /* 2nd cell is the irq relative to that bank */
526
527 const u_int bank = be32toh(specifier[0]);
528 switch (bank) {
529 case 0:
530 base = BCM2835_INT_BASICBASE;
531 break;
532 case 1:
533 base = BCM2835_INT_GPU0BASE;
534 break;
535 case 2:
536 base = BCM2835_INT_GPU1BASE;
537 break;
538 default:
539 return -1;
540 }
541 const u_int off = be32toh(specifier[1]);
542
543 return base + off;
544 }
545
546 static void *
bcm2835_icu_fdt_establish(device_t dev,u_int * specifier,int ipl,int flags,int (* func)(void *),void * arg,const char * xname)547 bcm2835_icu_fdt_establish(device_t dev, u_int *specifier, int ipl, int flags,
548 int (*func)(void *), void *arg, const char *xname)
549 {
550 struct bcm2835icu_softc * const sc = device_private(dev);
551 struct bcm2835icu_irq *firq;
552 struct bcm2835icu_irqhandler *firqh;
553 int iflags = (flags & FDT_INTR_MPSAFE) ? IST_MPSAFE : 0;
554 int irq, irqidx;
555
556 irq = bcm2835_icu_fdt_decode_irq(specifier);
557 if (irq == -1)
558 return NULL;
559 irqidx = irq - BCM2835_INT_BASE;
560
561 KASSERT(irqidx < BCM2835_NIRQ);
562
563 firq = sc->sc_irq[irqidx];
564 if (firq == NULL) {
565 firq = kmem_alloc(sizeof(*firq), KM_SLEEP);
566 firq->intr_sc = sc;
567 firq->intr_refcnt = 0;
568 firq->intr_arg = arg;
569 firq->intr_ipl = ipl;
570 firq->intr_mpsafe = iflags;
571 firq->intr_irq = irq;
572 TAILQ_INIT(&firq->intr_handlers);
573 if (arg == NULL) {
574 firq->intr_ih = intr_establish_xname(irq, ipl,
575 IST_LEVEL | iflags, func, NULL, xname);
576 } else {
577 firq->intr_ih = intr_establish_xname(irq, ipl,
578 IST_LEVEL | iflags, bcm2835_icu_intr, firq, xname);
579 }
580 if (firq->intr_ih == NULL) {
581 kmem_free(firq, sizeof(*firq));
582 return NULL;
583 }
584 sc->sc_irq[irqidx] = firq;
585 } else {
586 if (firq->intr_arg == NULL || arg == NULL) {
587 device_printf(dev,
588 "cannot share irq with NULL-arg handler\n");
589 return NULL;
590 }
591 if (firq->intr_ipl != ipl) {
592 device_printf(dev,
593 "cannot share irq with different ipl\n");
594 return NULL;
595 }
596 if (firq->intr_mpsafe != iflags) {
597 device_printf(dev,
598 "cannot share irq between mpsafe/non-mpsafe\n");
599 return NULL;
600 }
601 }
602
603 firqh = kmem_alloc(sizeof(*firqh), KM_SLEEP);
604 firqh->ih_irq = firq;
605 firqh->ih_fn = func;
606 firqh->ih_arg = arg;
607
608 firq->intr_refcnt++;
609 TAILQ_INSERT_TAIL(&firq->intr_handlers, firqh, ih_next);
610
611 /*
612 * XXX interrupt_distribute(9) assumes that any interrupt
613 * handle can be used as an input to the MD interrupt_distribute
614 * implementationm, so we are forced to return the handle
615 * we got back from intr_establish(). Upshot is that the
616 * input to bcm2835_icu_fdt_disestablish() is ambiguous for
617 * shared IRQs, rendering them un-disestablishable.
618 */
619
620 return firq->intr_ih;
621 }
622
623 static void
bcm2835_icu_fdt_disestablish(device_t dev,void * ih)624 bcm2835_icu_fdt_disestablish(device_t dev, void *ih)
625 {
626 struct bcm2835icu_softc * const sc = device_private(dev);
627 struct bcm2835icu_irqhandler *firqh;
628 struct bcm2835icu_irq *firq;
629 u_int n;
630
631 for (n = 0; n < BCM2835_NIRQ; n++) {
632 firq = sc->sc_irq[n];
633 if (firq == NULL || firq->intr_ih != ih)
634 continue;
635
636 KASSERT(firq->intr_refcnt > 0);
637 KASSERT(n == (firq->intr_irq - BCM2835_INT_BASE));
638
639 /* XXX see above */
640 if (firq->intr_refcnt > 1)
641 panic("%s: cannot disestablish shared irq", __func__);
642
643 intr_disestablish(firq->intr_ih);
644
645 firqh = TAILQ_FIRST(&firq->intr_handlers);
646 TAILQ_REMOVE(&firq->intr_handlers, firqh, ih_next);
647 kmem_free(firqh, sizeof(*firqh));
648
649 sc->sc_irq[n] = NULL;
650 kmem_free(firq, sizeof(*firq));
651
652 return;
653 }
654
655 panic("%s: interrupt not established", __func__);
656 }
657
658 static int
bcm2835_icu_intr(void * priv)659 bcm2835_icu_intr(void *priv)
660 {
661 struct bcm2835icu_irq *firq = priv;
662 struct bcm2835icu_irqhandler *firqh;
663 int handled = 0;
664
665 TAILQ_FOREACH(firqh, &firq->intr_handlers, ih_next) {
666 handled |= firqh->ih_fn(firqh->ih_arg);
667 }
668
669 return handled;
670 }
671
672 static bool
bcm2835_icu_fdt_intrstr(device_t dev,u_int * specifier,char * buf,size_t buflen)673 bcm2835_icu_fdt_intrstr(device_t dev, u_int *specifier, char *buf, size_t buflen)
674 {
675 int irq;
676
677 irq = bcm2835_icu_fdt_decode_irq(specifier);
678 if (irq == -1)
679 return false;
680
681 snprintf(buf, buflen, "icu irq %d", irq);
682
683 return true;
684 }
685
686 #define BCM2836MP_TIMER_IRQS __BITS(3,0)
687 #define BCM2836MP_MAILBOX_IRQS __BITS(4,7)
688 #define BCM2836MP_GPU_IRQ __BIT(8)
689 #define BCM2836MP_PMU_IRQ __BIT(9)
690 #define BCM2836MP_ALL_IRQS (BCM2836MP_TIMER_IRQS | BCM2836MP_MAILBOX_IRQS | BCM2836MP_GPU_IRQ | BCM2836MP_PMU_IRQ)
691
692 static void
bcm2836mp_pic_unblock_irqs(struct pic_softc * pic,size_t irqbase,uint32_t irq_mask)693 bcm2836mp_pic_unblock_irqs(struct pic_softc *pic, size_t irqbase,
694 uint32_t irq_mask)
695 {
696 const bus_space_tag_t iot = bcml1icu_sc->sc_iot;
697 const bus_space_handle_t ioh = bcml1icu_sc->sc_ioh;
698 const cpuid_t cpuid = pic - &bcm2836mp_pic[0];
699
700 KASSERT(cpuid < _BCM2836_NCPUS);
701 KASSERT(irqbase == 0);
702
703 if (irq_mask & BCM2836MP_TIMER_IRQS) {
704 uint32_t mask = __SHIFTOUT(irq_mask, BCM2836MP_TIMER_IRQS);
705 uint32_t val = bus_space_read_4(iot, ioh,
706 BCM2836_LOCAL_TIMER_IRQ_CONTROLN(cpuid));
707 val |= mask;
708 bus_space_write_4(iot, ioh,
709 BCM2836_LOCAL_TIMER_IRQ_CONTROLN(cpuid),
710 val);
711 bus_space_barrier(iot, ioh,
712 BCM2836_LOCAL_TIMER_IRQ_CONTROL_BASE,
713 BCM2836_LOCAL_TIMER_IRQ_CONTROL_SIZE,
714 BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE);
715 }
716 if (irq_mask & BCM2836MP_MAILBOX_IRQS) {
717 uint32_t mask = __SHIFTOUT(irq_mask, BCM2836MP_MAILBOX_IRQS);
718 uint32_t val = bus_space_read_4(iot, ioh,
719 BCM2836_LOCAL_MAILBOX_IRQ_CONTROLN(cpuid));
720 val |= mask;
721 bus_space_write_4(iot, ioh,
722 BCM2836_LOCAL_MAILBOX_IRQ_CONTROLN(cpuid),
723 val);
724 bus_space_barrier(iot, ioh,
725 BCM2836_LOCAL_MAILBOX_IRQ_CONTROL_BASE,
726 BCM2836_LOCAL_MAILBOX_IRQ_CONTROL_SIZE,
727 BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE);
728 }
729 if (irq_mask & BCM2836MP_PMU_IRQ) {
730 bus_space_write_4(iot, ioh, BCM2836_LOCAL_PM_ROUTING_SET,
731 __BIT(cpuid));
732 bus_space_barrier(iot, ioh, BCM2836_LOCAL_PM_ROUTING_SET, 4,
733 BUS_SPACE_BARRIER_READ|BUS_SPACE_BARRIER_WRITE);
734 }
735
736 return;
737 }
738
739 static void
bcm2836mp_pic_block_irqs(struct pic_softc * pic,size_t irqbase,uint32_t irq_mask)740 bcm2836mp_pic_block_irqs(struct pic_softc *pic, size_t irqbase,
741 uint32_t irq_mask)
742 {
743 const bus_space_tag_t iot = bcml1icu_sc->sc_iot;
744 const bus_space_handle_t ioh = bcml1icu_sc->sc_ioh;
745 const cpuid_t cpuid = pic - &bcm2836mp_pic[0];
746
747 KASSERT(cpuid < _BCM2836_NCPUS);
748 KASSERT(irqbase == 0);
749
750 if (irq_mask & BCM2836MP_TIMER_IRQS) {
751 uint32_t mask = __SHIFTOUT(irq_mask, BCM2836MP_TIMER_IRQS);
752 uint32_t val = bus_space_read_4(iot, ioh,
753 BCM2836_LOCAL_TIMER_IRQ_CONTROLN(cpuid));
754 val &= ~mask;
755 bus_space_write_4(iot, ioh,
756 BCM2836_LOCAL_TIMER_IRQ_CONTROLN(cpuid),
757 val);
758 }
759 if (irq_mask & BCM2836MP_MAILBOX_IRQS) {
760 uint32_t mask = __SHIFTOUT(irq_mask, BCM2836MP_MAILBOX_IRQS);
761 uint32_t val = bus_space_read_4(iot, ioh,
762 BCM2836_LOCAL_MAILBOX_IRQ_CONTROLN(cpuid));
763 val &= ~mask;
764 bus_space_write_4(iot, ioh,
765 BCM2836_LOCAL_MAILBOX_IRQ_CONTROLN(cpuid),
766 val);
767 }
768 if (irq_mask & BCM2836MP_PMU_IRQ) {
769 bus_space_write_4(iot, ioh, BCM2836_LOCAL_PM_ROUTING_CLR,
770 __BIT(cpuid));
771 }
772
773 bcm2835_barrier();
774 return;
775 }
776
777 static int
bcm2836mp_pic_find_pending_irqs(struct pic_softc * pic)778 bcm2836mp_pic_find_pending_irqs(struct pic_softc *pic)
779 {
780 struct cpu_info * const ci = curcpu();
781 const cpuid_t cpuid = ci->ci_core_id;
782 uint32_t lpending;
783 int ipl = 0;
784
785 KASSERT(cpuid < _BCM2836_NCPUS);
786 KASSERT(pic == &bcm2836mp_pic[cpuid]);
787
788 bcm2835_barrier();
789
790 lpending = bus_space_read_4(bcml1icu_sc->sc_iot, bcml1icu_sc->sc_ioh,
791 BCM2836_LOCAL_INTC_IRQPENDINGN(cpuid));
792
793 lpending &= ~BCM2836_INTBIT_GPUPENDING;
794 const uint32_t allirqs = lpending & BCM2836MP_ALL_IRQS;
795 if (allirqs) {
796 ipl |= pic_mark_pending_sources(pic, 0, allirqs);
797 }
798
799 return ipl;
800 }
801
802 static void
bcm2836mp_pic_establish_irq(struct pic_softc * pic,struct intrsource * is)803 bcm2836mp_pic_establish_irq(struct pic_softc *pic, struct intrsource *is)
804 {
805 /* Nothing really*/
806 KASSERT(is->is_irq >= 0);
807 KASSERT(is->is_irq < BCM2836_NIRQPERCPU);
808 }
809
810 static void
bcm2836mp_pic_source_name(struct pic_softc * pic,int irq,char * buf,size_t len)811 bcm2836mp_pic_source_name(struct pic_softc *pic, int irq, char *buf, size_t len)
812 {
813
814 irq %= BCM2836_NIRQPERCPU;
815 strlcpy(buf, bcm2836mp_sources[irq], len);
816 }
817
818
819 #if defined(MULTIPROCESSOR)
bcm2836mp_cpu_init(struct pic_softc * pic,struct cpu_info * ci)820 static void bcm2836mp_cpu_init(struct pic_softc *pic, struct cpu_info *ci)
821 {
822 const cpuid_t cpuid = ci->ci_core_id;
823
824 KASSERT(cpuid < _BCM2836_NCPUS);
825
826 /* Enable IRQ and not FIQ */
827 bus_space_write_4(bcml1icu_sc->sc_iot, bcml1icu_sc->sc_ioh,
828 BCM2836_LOCAL_MAILBOX_IRQ_CONTROLN(cpuid), 1);
829 }
830
831 static void
bcm2836mp_send_ipi(struct pic_softc * pic,const kcpuset_t * kcp,u_long ipi)832 bcm2836mp_send_ipi(struct pic_softc *pic, const kcpuset_t *kcp, u_long ipi)
833 {
834 KASSERT(pic != NULL);
835 KASSERT(pic != &bcm2835_pic);
836 KASSERT(pic->pic_cpus != NULL);
837
838 const cpuid_t cpuid = pic - &bcm2836mp_pic[0];
839 KASSERT(cpuid < _BCM2836_NCPUS);
840
841 bus_space_write_4(bcml1icu_sc->sc_iot, bcml1icu_sc->sc_ioh,
842 BCM2836_LOCAL_MAILBOX0_SETN(cpuid), __BIT(ipi));
843 }
844
845 int
bcm2836mp_ipi_handler(void * priv)846 bcm2836mp_ipi_handler(void *priv)
847 {
848 const struct cpu_info *ci = priv;
849 const cpuid_t cpuid = ci->ci_core_id;
850 uint32_t ipimask, bit;
851
852 KASSERT(cpuid < _BCM2836_NCPUS);
853
854 ipimask = bus_space_read_4(bcml1icu_sc->sc_iot, bcml1icu_sc->sc_ioh,
855 BCM2836_LOCAL_MAILBOX0_CLRN(cpuid));
856 bus_space_write_4(bcml1icu_sc->sc_iot, bcml1icu_sc->sc_ioh,
857 BCM2836_LOCAL_MAILBOX0_CLRN(cpuid), ipimask);
858
859 while ((bit = ffs(ipimask)) > 0) {
860 const u_int ipi = bit - 1;
861 switch (ipi) {
862 case IPI_AST:
863 pic_ipi_ast(priv);
864 break;
865 case IPI_NOP:
866 pic_ipi_nop(priv);
867 break;
868 #ifdef __HAVE_PREEMPTION
869 case IPI_KPREEMPT:
870 pic_ipi_kpreempt(priv);
871 break;
872 #endif
873 case IPI_XCALL:
874 pic_ipi_xcall(priv);
875 break;
876 case IPI_GENERIC:
877 pic_ipi_generic(priv);
878 break;
879 case IPI_SHOOTDOWN:
880 pic_ipi_shootdown(priv);
881 break;
882 #ifdef DDB
883 case IPI_DDB:
884 pic_ipi_ddb(priv);
885 break;
886 #endif
887 }
888 ipimask &= ~__BIT(ipi);
889 }
890
891 return 1;
892 }
893 #endif
894
895 #if defined(MULTIPROCESSOR)
896 static void
bcm2836mp_intr_init(struct cpu_info * ci)897 bcm2836mp_intr_init(struct cpu_info *ci)
898 {
899 const cpuid_t cpuid = ci->ci_core_id;
900
901 KASSERT(cpuid < _BCM2836_NCPUS);
902
903 intr_establish(BCM2836_INT_MAILBOX0_CPUN(cpuid), IPL_HIGH,
904 IST_LEVEL | IST_MPSAFE, bcm2836mp_ipi_handler, ci);
905 }
906 #endif
907
908 static int
bcm2836mp_icu_fdt_decode_irq(u_int * specifier)909 bcm2836mp_icu_fdt_decode_irq(u_int *specifier)
910 {
911
912 if (!specifier)
913 return -1;
914 return be32toh(specifier[0]);
915 }
916
917 static void *
bcm2836mp_icu_fdt_establish(device_t dev,u_int * specifier,int ipl,int flags,int (* func)(void *),void * arg,const char * xname)918 bcm2836mp_icu_fdt_establish(device_t dev, u_int *specifier, int ipl, int flags,
919 int (*func)(void *), void *arg, const char *xname)
920 {
921 int iflags = (flags & FDT_INTR_MPSAFE) ? IST_MPSAFE : 0;
922
923 int irq = bcm2836mp_icu_fdt_decode_irq(specifier);
924 if (irq == -1)
925 return NULL;
926
927 void *ihs[_BCM2836_NCPUS];
928 for (cpuid_t cpuid = 0; cpuid < _BCM2836_NCPUS; cpuid++) {
929 const int cpuirq = BCM2836_INT_BASECPUN(cpuid) + irq;
930 ihs[cpuid] = intr_establish_xname(cpuirq, ipl,
931 IST_LEVEL | iflags, func, arg, xname);
932 if (!ihs[cpuid]) {
933 for (cpuid_t undo = 0; undo < cpuid; undo++) {
934 intr_disestablish(ihs[undo]);
935 }
936 return NULL;
937 }
938
939 }
940
941 /*
942 * Return the intr_establish handle for cpu 0 for API compatibility.
943 * Any cpu would do here as these sources don't support set_affinity
944 * when the handle is used in interrupt_distribute(9)
945 */
946 return ihs[0];
947 }
948
949 static void
bcm2836mp_icu_fdt_disestablish(device_t dev,void * ih)950 bcm2836mp_icu_fdt_disestablish(device_t dev, void *ih)
951 {
952 intr_disestablish(ih);
953 }
954
955 static bool
bcm2836mp_icu_fdt_intrstr(device_t dev,u_int * specifier,char * buf,size_t buflen)956 bcm2836mp_icu_fdt_intrstr(device_t dev, u_int *specifier, char *buf,
957 size_t buflen)
958 {
959 int irq;
960
961 irq = bcm2836mp_icu_fdt_decode_irq(specifier);
962 if (irq == -1)
963 return false;
964
965 snprintf(buf, buflen, "local_intc irq %d", irq);
966
967 return true;
968 }
969