1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright (c) 2018 Ruslan Bukin <br@bsdpad.com>
5 * All rights reserved.
6 * Copyright (c) 2019 Mitchell Horne <mhorne@FreeBSD.org>
7 *
8 * Portions of this software were developed by SRI International and the
9 * University of Cambridge Computer Laboratory (Department of Computer Science
10 * and Technology) under DARPA contract HR0011-18-C-0016 ("ECATS"), as part of
11 * the DARPA SSITH research programme.
12 *
13 * Redistribution and use in source and binary forms, with or without
14 * modification, are permitted provided that the following conditions
15 * are met:
16 * 1. Redistributions of source code must retain the above copyright
17 * notice, this list of conditions and the following disclaimer.
18 * 2. Redistributions in binary form must reproduce the above copyright
19 * notice, this list of conditions and the following disclaimer in the
20 * documentation and/or other materials provided with the distribution.
21 *
22 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
23 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
24 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
25 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
26 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
27 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
28 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
29 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
30 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
31 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
32 * SUCH DAMAGE.
33 */
34
35 #include <sys/param.h>
36 #include <sys/systm.h>
37 #include <sys/bus.h>
38 #include <sys/kernel.h>
39 #include <sys/ktr.h>
40 #include <sys/module.h>
41 #include <sys/proc.h>
42 #include <sys/rman.h>
43 #include <sys/smp.h>
44
45 #include <machine/bus.h>
46 #include <machine/intr.h>
47
48 #include <dev/ofw/openfirm.h>
49 #include <dev/ofw/ofw_bus.h>
50 #include <dev/ofw/ofw_bus_subr.h>
51
52 #include "pic_if.h"
53
54 #define PLIC_MAX_IRQS 1024
55
56 #define PLIC_PRIORITY_BASE 0x000000U
57
58 #define PLIC_ENABLE_BASE 0x002000U
59 #define PLIC_ENABLE_STRIDE 0x80U
60
61 #define PLIC_CONTEXT_BASE 0x200000U
62 #define PLIC_CONTEXT_STRIDE 0x1000U
63 #define PLIC_CONTEXT_THRESHOLD 0x0U
64 #define PLIC_CONTEXT_CLAIM 0x4U
65
66 #define PLIC_PRIORITY(n) (PLIC_PRIORITY_BASE + (n) * sizeof(uint32_t))
67 #define PLIC_ENABLE(sc, n, h) \
68 (sc->contexts[h].enable_offset + ((n) / 32) * sizeof(uint32_t))
69 #define PLIC_THRESHOLD(sc, h) \
70 (sc->contexts[h].context_offset + PLIC_CONTEXT_THRESHOLD)
71 #define PLIC_CLAIM(sc, h) \
72 (sc->contexts[h].context_offset + PLIC_CONTEXT_CLAIM)
73
74 static pic_disable_intr_t plic_disable_intr;
75 static pic_enable_intr_t plic_enable_intr;
76 static pic_map_intr_t plic_map_intr;
77 static pic_setup_intr_t plic_setup_intr;
78 static pic_post_ithread_t plic_post_ithread;
79 static pic_pre_ithread_t plic_pre_ithread;
80 static pic_bind_intr_t plic_bind_intr;
81
82 struct plic_irqsrc {
83 struct intr_irqsrc isrc;
84 u_int irq;
85 };
86
87 struct plic_context {
88 bus_size_t enable_offset;
89 bus_size_t context_offset;
90 };
91
92 struct plic_softc {
93 device_t dev;
94 struct resource *mem_res;
95 struct resource *irq_res;
96 void *ih;
97 struct plic_irqsrc isrcs[PLIC_MAX_IRQS];
98 struct plic_context contexts[MAXCPU];
99 int ndev;
100 };
101
102 #define RD4(sc, reg) \
103 bus_read_4(sc->mem_res, (reg))
104 #define WR4(sc, reg, val) \
105 bus_write_4(sc->mem_res, (reg), (val))
106
107 static u_int plic_irq_cpu;
108
109 static int
riscv_hartid_to_cpu(int hartid)110 riscv_hartid_to_cpu(int hartid)
111 {
112 int i;
113
114 CPU_FOREACH(i) {
115 if (pcpu_find(i)->pc_hart == hartid)
116 return (i);
117 }
118
119 return (-1);
120 }
121
122 static int
plic_get_hartid(device_t dev,phandle_t intc)123 plic_get_hartid(device_t dev, phandle_t intc)
124 {
125 int hart;
126
127 /* Check the interrupt controller layout. */
128 if (OF_searchencprop(intc, "#interrupt-cells", &hart,
129 sizeof(hart)) == -1) {
130 device_printf(dev,
131 "Could not find #interrupt-cells for phandle %u\n", intc);
132 return (-1);
133 }
134
135 /*
136 * The parent of the interrupt-controller is the CPU we are
137 * interested in, so search for its hart ID.
138 */
139 if (OF_searchencprop(OF_parent(intc), "reg", (pcell_t *)&hart,
140 sizeof(hart)) == -1) {
141 device_printf(dev, "Could not find hartid\n");
142 return (-1);
143 }
144
145 return (hart);
146 }
147
148 static inline void
plic_irq_dispatch(struct plic_softc * sc,u_int irq,struct trapframe * tf)149 plic_irq_dispatch(struct plic_softc *sc, u_int irq,
150 struct trapframe *tf)
151 {
152 struct plic_irqsrc *src;
153
154 src = &sc->isrcs[irq];
155
156 if (intr_isrc_dispatch(&src->isrc, tf) != 0)
157 device_printf(sc->dev, "Stray irq %u detected\n", irq);
158 }
159
160 static int
plic_intr(void * arg)161 plic_intr(void *arg)
162 {
163 struct plic_softc *sc;
164 struct trapframe *tf;
165 uint32_t pending;
166 uint32_t cpu;
167
168 sc = arg;
169 cpu = PCPU_GET(cpuid);
170
171 /* Claim any pending interrupt. */
172 pending = RD4(sc, PLIC_CLAIM(sc, cpu));
173 if (pending) {
174 tf = curthread->td_intr_frame;
175 plic_irq_dispatch(sc, pending, tf);
176 }
177
178 return (FILTER_HANDLED);
179 }
180
181 static void
plic_disable_intr(device_t dev,struct intr_irqsrc * isrc)182 plic_disable_intr(device_t dev, struct intr_irqsrc *isrc)
183 {
184 struct plic_softc *sc;
185 struct plic_irqsrc *src;
186
187 sc = device_get_softc(dev);
188 src = (struct plic_irqsrc *)isrc;
189
190 WR4(sc, PLIC_PRIORITY(src->irq), 0);
191 }
192
193 static void
plic_enable_intr(device_t dev,struct intr_irqsrc * isrc)194 plic_enable_intr(device_t dev, struct intr_irqsrc *isrc)
195 {
196 struct plic_softc *sc;
197 struct plic_irqsrc *src;
198
199 sc = device_get_softc(dev);
200 src = (struct plic_irqsrc *)isrc;
201
202 WR4(sc, PLIC_PRIORITY(src->irq), 1);
203 }
204
205 static int
plic_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)206 plic_map_intr(device_t dev, struct intr_map_data *data,
207 struct intr_irqsrc **isrcp)
208 {
209 struct intr_map_data_fdt *daf;
210 struct plic_softc *sc;
211
212 sc = device_get_softc(dev);
213
214 if (data->type != INTR_MAP_DATA_FDT)
215 return (ENOTSUP);
216
217 daf = (struct intr_map_data_fdt *)data;
218 if (daf->ncells != 1 || daf->cells[0] > sc->ndev)
219 return (EINVAL);
220
221 *isrcp = &sc->isrcs[daf->cells[0]].isrc;
222
223 return (0);
224 }
225
226 static int
plic_probe(device_t dev)227 plic_probe(device_t dev)
228 {
229
230 if (!ofw_bus_status_okay(dev))
231 return (ENXIO);
232
233 if (!ofw_bus_is_compatible(dev, "riscv,plic0") &&
234 !ofw_bus_is_compatible(dev, "sifive,plic-1.0.0"))
235 return (ENXIO);
236
237 device_set_desc(dev, "RISC-V PLIC");
238
239 return (BUS_PROBE_DEFAULT);
240 }
241
242 static int
plic_attach(device_t dev)243 plic_attach(device_t dev)
244 {
245 struct plic_irqsrc *isrcs;
246 struct plic_softc *sc;
247 struct intr_pic *pic;
248 pcell_t *cells;
249 uint32_t irq;
250 const char *name;
251 phandle_t node;
252 phandle_t xref;
253 uint32_t cpu;
254 int error;
255 int rid;
256 int nintr;
257 int context;
258 int i;
259 int hart;
260
261 sc = device_get_softc(dev);
262
263 sc->dev = dev;
264
265 node = ofw_bus_get_node(dev);
266 if ((OF_getencprop(node, "riscv,ndev", &sc->ndev,
267 sizeof(sc->ndev))) < 0) {
268 device_printf(dev,
269 "Error: could not get number of devices\n");
270 return (ENXIO);
271 }
272
273 if (sc->ndev >= PLIC_MAX_IRQS) {
274 device_printf(dev,
275 "Error: invalid ndev (%d)\n", sc->ndev);
276 return (ENXIO);
277 }
278
279 /* Request memory resources */
280 rid = 0;
281 sc->mem_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
282 RF_ACTIVE);
283 if (sc->mem_res == NULL) {
284 device_printf(dev,
285 "Error: could not allocate memory resources\n");
286 return (ENXIO);
287 }
288
289 /* Register the interrupt sources */
290 isrcs = sc->isrcs;
291 name = device_get_nameunit(sc->dev);
292 for (irq = 1; irq <= sc->ndev; irq++) {
293 isrcs[irq].irq = irq;
294 error = intr_isrc_register(&isrcs[irq].isrc, sc->dev,
295 0, "%s,%u", name, irq);
296 if (error != 0)
297 return (error);
298
299 WR4(sc, PLIC_PRIORITY(irq), 0);
300 }
301
302 /*
303 * Calculate the per-cpu enable and context register offsets.
304 *
305 * This is tricky for a few reasons. The PLIC divides the interrupt
306 * enable, threshold, and claim bits by "context", where each context
307 * routes to a core's local interrupt controller.
308 *
309 * The tricky part is that the PLIC spec imposes no restrictions on how
310 * these contexts are laid out. So for example, there is no guarantee
311 * that each CPU will have both a machine mode and supervisor context,
312 * or that different PLIC implementations will organize the context
313 * registers in the same way. On top of this, we must handle the fact
314 * that cpuid != hartid, as they may have been renumbered during boot.
315 * We perform the following steps:
316 *
317 * 1. Examine the PLIC's "interrupts-extended" property and skip any
318 * entries that are not for supervisor external interrupts.
319 *
320 * 2. Walk up the device tree to find the corresponding CPU, and grab
321 * its hart ID.
322 *
323 * 3. Convert the hart to a cpuid, and calculate the register offsets
324 * based on the context number.
325 *
326 * 4. Save the index for the boot hart's S-mode external interrupt in
327 * order to allocate and setup the corresponding resource, since the
328 * local interrupt controller newbus device is associated with that
329 * specific node.
330 */
331 nintr = OF_getencprop_alloc_multi(node, "interrupts-extended",
332 sizeof(uint32_t), (void **)&cells);
333 if (nintr <= 0) {
334 device_printf(dev, "Could not read interrupts-extended\n");
335 return (ENXIO);
336 }
337
338 /* interrupts-extended is a list of phandles and interrupt types. */
339 rid = -1;
340 for (i = 0, context = 0; i < nintr; i += 2, context++) {
341 /* Skip M-mode external interrupts */
342 if (cells[i + 1] != IRQ_EXTERNAL_SUPERVISOR)
343 continue;
344
345 /*
346 * Get the hart ID from the core's interrupt controller
347 * phandle.
348 */
349 hart = plic_get_hartid(dev, OF_node_from_xref(cells[i]));
350 if (hart < 0) {
351 OF_prop_free(cells);
352 return (ENXIO);
353 }
354
355 /* Get the corresponding cpuid. */
356 cpu = riscv_hartid_to_cpu(hart);
357 if (cpu < 0) {
358 device_printf(dev, "Invalid hart!\n");
359 OF_prop_free(cells);
360 return (ENXIO);
361 }
362
363 if (cpu == 0)
364 rid = i / 2;
365
366 /* Set the enable and context register offsets for the CPU. */
367 sc->contexts[cpu].enable_offset = PLIC_ENABLE_BASE +
368 context * PLIC_ENABLE_STRIDE;
369 sc->contexts[cpu].context_offset = PLIC_CONTEXT_BASE +
370 context * PLIC_CONTEXT_STRIDE;
371 }
372 OF_prop_free(cells);
373
374 if (rid == -1) {
375 device_printf(dev,
376 "Could not find local interrupt controller\n");
377 return (ENXIO);
378 }
379
380 sc->irq_res = bus_alloc_resource_any(dev, SYS_RES_IRQ, &rid,
381 RF_ACTIVE);
382 if (sc->irq_res == NULL) {
383 device_printf(dev,
384 "Error: could not allocate IRQ resources\n");
385 return (ENXIO);
386 }
387
388 /* Set the threshold for each CPU to accept all priorities. */
389 CPU_FOREACH(cpu)
390 WR4(sc, PLIC_THRESHOLD(sc, cpu), 0);
391
392 xref = OF_xref_from_node(node);
393 pic = intr_pic_register(sc->dev, xref);
394 if (pic == NULL)
395 return (ENXIO);
396
397 return (bus_setup_intr(dev, sc->irq_res, INTR_TYPE_CLK | INTR_MPSAFE,
398 plic_intr, NULL, sc, &sc->ih));
399 }
400
401 static void
plic_pre_ithread(device_t dev,struct intr_irqsrc * isrc)402 plic_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
403 {
404
405 plic_disable_intr(dev, isrc);
406 }
407
408 static void
plic_post_ithread(device_t dev,struct intr_irqsrc * isrc)409 plic_post_ithread(device_t dev, struct intr_irqsrc *isrc)
410 {
411 struct plic_softc *sc;
412 struct plic_irqsrc *src;
413 uint32_t cpu;
414
415 sc = device_get_softc(dev);
416 src = (struct plic_irqsrc *)isrc;
417
418 cpu = CPU_FFS(&isrc->isrc_cpu) - 1;
419
420 /* Complete the interrupt. */
421 WR4(sc, PLIC_CLAIM(sc, cpu), src->irq);
422 plic_enable_intr(dev, isrc);
423 }
424
425 static int
plic_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)426 plic_setup_intr(device_t dev, struct intr_irqsrc *isrc,
427 struct resource *res, struct intr_map_data *data)
428 {
429 CPU_ZERO(&isrc->isrc_cpu);
430 plic_bind_intr(dev, isrc);
431
432 return (0);
433 }
434
435 static int
plic_bind_intr(device_t dev,struct intr_irqsrc * isrc)436 plic_bind_intr(device_t dev, struct intr_irqsrc *isrc)
437 {
438 struct plic_softc *sc;
439 struct plic_irqsrc *src;
440 uint32_t reg;
441 u_int cpu;
442
443 sc = device_get_softc(dev);
444 src = (struct plic_irqsrc *)isrc;
445
446 /* Disable the interrupt source on all CPUs. */
447 CPU_FOREACH(cpu) {
448 reg = RD4(sc, PLIC_ENABLE(sc, src->irq, cpu));
449 reg &= ~(1 << (src->irq % 32));
450 WR4(sc, PLIC_ENABLE(sc, src->irq, cpu), reg);
451 }
452
453 if (CPU_EMPTY(&isrc->isrc_cpu)) {
454 cpu = plic_irq_cpu = intr_irq_next_cpu(plic_irq_cpu, &all_cpus);
455 CPU_SETOF(cpu, &isrc->isrc_cpu);
456 } else {
457 /*
458 * We will only bind to a single CPU so select the first
459 * CPU found.
460 */
461 cpu = CPU_FFS(&isrc->isrc_cpu) - 1;
462 }
463
464 /* Enable the interrupt on the selected CPU only. */
465 reg = RD4(sc, PLIC_ENABLE(sc, src->irq, cpu));
466 reg |= (1 << (src->irq % 32));
467 WR4(sc, PLIC_ENABLE(sc, src->irq, cpu), reg);
468
469 return (0);
470 }
471
472 static device_method_t plic_methods[] = {
473 DEVMETHOD(device_probe, plic_probe),
474 DEVMETHOD(device_attach, plic_attach),
475
476 DEVMETHOD(pic_disable_intr, plic_disable_intr),
477 DEVMETHOD(pic_enable_intr, plic_enable_intr),
478 DEVMETHOD(pic_map_intr, plic_map_intr),
479 DEVMETHOD(pic_pre_ithread, plic_pre_ithread),
480 DEVMETHOD(pic_post_ithread, plic_post_ithread),
481 DEVMETHOD(pic_post_filter, plic_post_ithread),
482 DEVMETHOD(pic_setup_intr, plic_setup_intr),
483 DEVMETHOD(pic_bind_intr, plic_bind_intr),
484
485 DEVMETHOD_END
486 };
487
488 static driver_t plic_driver = {
489 "plic",
490 plic_methods,
491 sizeof(struct plic_softc),
492 };
493
494 EARLY_DRIVER_MODULE(plic, simplebus, plic_driver, 0, 0,
495 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
496