1 /*-
2 * SPDX-License-Identifier: BSD-2-Clause
3 *
4 * Copyright 2011 Nathan Whitehorn
5 *
6 * Redistribution and use in source and binary forms, with or without
7 * modification, are permitted provided that the following conditions
8 * are met:
9 * 1. Redistributions of source code must retain the above copyright
10 * notice, this list of conditions and the following disclaimer.
11 * 2. Redistributions in binary form must reproduce the above copyright
12 * notice, this list of conditions and the following disclaimer in the
13 * documentation and/or other materials provided with the distribution.
14 *
15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
25 * SUCH DAMAGE.
26 */
27
28 #include <sys/cdefs.h>
29 #include "opt_platform.h"
30
31 #include <sys/param.h>
32 #include <sys/systm.h>
33 #include <sys/module.h>
34 #include <sys/bus.h>
35 #include <sys/conf.h>
36 #include <sys/kernel.h>
37 #include <sys/lock.h>
38 #include <sys/malloc.h>
39 #include <sys/mutex.h>
40 #include <sys/smp.h>
41
42 #include <vm/vm.h>
43 #include <vm/pmap.h>
44
45 #include <machine/bus.h>
46 #include <machine/intr_machdep.h>
47 #include <machine/md_var.h>
48 #include <machine/rtas.h>
49
50 #include <dev/ofw/ofw_bus.h>
51 #include <dev/ofw/ofw_bus_subr.h>
52
53 #ifdef POWERNV
54 #include <powerpc/powernv/opal.h>
55 #endif
56
57 #include "phyp-hvcall.h"
58 #include "pic_if.h"
59
60 #define XICP_PRIORITY 5 /* Random non-zero number */
61 #define XICP_IPI 2
62 #define MAX_XICP_IRQS (1<<24) /* 24-bit XIRR field */
63
64 static int xicp_probe(device_t);
65 static int xicp_attach(device_t);
66 static int xics_probe(device_t);
67 static int xics_attach(device_t);
68
69 static void xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv);
70 static void xicp_dispatch(device_t, struct trapframe *);
71 static void xicp_enable(device_t, u_int, u_int, void **priv);
72 static void xicp_eoi(device_t, u_int, void *priv);
73 static void xicp_ipi(device_t, u_int);
74 static void xicp_mask(device_t, u_int, void *priv);
75 static void xicp_unmask(device_t, u_int, void *priv);
76
77 #ifdef POWERNV
78 extern void (*powernv_smp_ap_extra_init)(void);
79 static void xicp_smp_cpu_startup(void);
80 #endif
81
82 static device_method_t xicp_methods[] = {
83 /* Device interface */
84 DEVMETHOD(device_probe, xicp_probe),
85 DEVMETHOD(device_attach, xicp_attach),
86
87 /* PIC interface */
88 DEVMETHOD(pic_bind, xicp_bind),
89 DEVMETHOD(pic_dispatch, xicp_dispatch),
90 DEVMETHOD(pic_enable, xicp_enable),
91 DEVMETHOD(pic_eoi, xicp_eoi),
92 DEVMETHOD(pic_ipi, xicp_ipi),
93 DEVMETHOD(pic_mask, xicp_mask),
94 DEVMETHOD(pic_unmask, xicp_unmask),
95
96 DEVMETHOD_END
97 };
98
99 static device_method_t xics_methods[] = {
100 /* Device interface */
101 DEVMETHOD(device_probe, xics_probe),
102 DEVMETHOD(device_attach, xics_attach),
103
104 DEVMETHOD_END
105 };
106
107 struct xicp_intvec {
108 int irq;
109 int vector;
110 int cpu;
111 };
112
113 struct xicp_softc {
114 struct mtx sc_mtx;
115 struct resource *mem[MAXCPU];
116
117 int cpu_range[2];
118
119 int ibm_int_on;
120 int ibm_int_off;
121 int ibm_get_xive;
122 int ibm_set_xive;
123
124 /* XXX: inefficient -- hash table? tree? */
125 struct xicp_intvec intvecs[256];
126 int nintvecs;
127 int ipi_vec;
128 bool xics_emu;
129 };
130
131 static driver_t xicp_driver = {
132 "xicp",
133 xicp_methods,
134 sizeof(struct xicp_softc)
135 };
136
137 static driver_t xics_driver = {
138 "xics",
139 xics_methods,
140 0
141 };
142
143 #ifdef POWERNV
144 /* We can only pass physical addresses into OPAL. Kernel stacks are in the KVA,
145 * not in the direct map, so we need to somehow extract the physical address.
146 * However, pmap_kextract() takes locks, which is forbidden in a critical region
147 * (which PIC_DISPATCH() operates in). The kernel is mapped into the Direct
148 * Map (0xc000....), and the CPU implicitly drops the top two bits when doing
149 * real address by nature that the bus width is smaller than 64-bits. Placing
150 * cpu_xirr into the DMAP lets us take advantage of this and avoids the
151 * pmap_kextract() that would otherwise be needed if using the stack variable.
152 */
153 static uint32_t cpu_xirr[MAXCPU];
154 #endif
155
156 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, 0, 0, BUS_PASS_INTERRUPT - 1);
157 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, 0, 0, BUS_PASS_INTERRUPT);
158
159 #ifdef POWERNV
160 static struct resource *
xicp_mem_for_cpu(int cpu)161 xicp_mem_for_cpu(int cpu)
162 {
163 devclass_t dc;
164 device_t dev;
165 struct xicp_softc *sc;
166 int i;
167
168 dc = devclass_find(xicp_driver.name);
169 for (i = 0; (dev = devclass_get_device(dc, i)) != NULL; i++){
170 sc = device_get_softc(dev);
171 if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1])
172 return (sc->mem[cpu - sc->cpu_range[0]]);
173 }
174
175 return (NULL);
176 }
177 #endif
178
179 static int
xicp_probe(device_t dev)180 xicp_probe(device_t dev)
181 {
182
183 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") &&
184 !ofw_bus_is_compatible(dev, "ibm,opal-intc"))
185 return (ENXIO);
186
187 device_set_desc(dev, "External Interrupt Presentation Controller");
188 return (BUS_PROBE_GENERIC);
189 }
190
191 static int
xics_probe(device_t dev)192 xics_probe(device_t dev)
193 {
194
195 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") &&
196 !ofw_bus_is_compatible(dev, "IBM,opal-xics"))
197 return (ENXIO);
198
199 device_set_desc(dev, "External Interrupt Source Controller");
200 return (BUS_PROBE_GENERIC);
201 }
202
203 static int
xicp_attach(device_t dev)204 xicp_attach(device_t dev)
205 {
206 struct xicp_softc *sc = device_get_softc(dev);
207 phandle_t phandle = ofw_bus_get_node(dev);
208
209 if (rtas_exists()) {
210 sc->ibm_int_on = rtas_token_lookup("ibm,int-on");
211 sc->ibm_int_off = rtas_token_lookup("ibm,int-off");
212 sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive");
213 sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive");
214 #ifdef POWERNV
215 } else if (opal_check() == 0) {
216 /* No init needed */
217 #endif
218 } else {
219 device_printf(dev, "Cannot attach without RTAS or OPAL\n");
220 return (ENXIO);
221 }
222
223 if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) {
224 OF_getencprop(phandle, "ibm,interrupt-server-ranges",
225 sc->cpu_range, sizeof(sc->cpu_range));
226 sc->cpu_range[1] += sc->cpu_range[0];
227 device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0],
228 sc->cpu_range[1]-1);
229 #ifdef POWERNV
230 } else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) {
231 /*
232 * For now run POWER9 XIVE interrupt controller in XICS
233 * compatibility mode.
234 */
235 sc->xics_emu = true;
236 opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EMU);
237 #endif
238 } else {
239 sc->cpu_range[0] = 0;
240 sc->cpu_range[1] = mp_ncpus;
241 }
242
243 #ifdef POWERNV
244 if (mfmsr() & PSL_HV) {
245 int i;
246
247 if (sc->xics_emu) {
248 opal_call(OPAL_INT_SET_CPPR, 0xff);
249 for (i = 0; i < mp_ncpus; i++) {
250 opal_call(OPAL_INT_SET_MFRR,
251 pcpu_find(i)->pc_hwref, 0xff);
252 }
253 } else {
254 for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) {
255 sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
256 &i, RF_ACTIVE);
257 if (sc->mem[i] == NULL) {
258 device_printf(dev, "Could not alloc mem "
259 "resource %d\n", i);
260 return (ENXIO);
261 }
262
263 /* Unmask interrupts on all cores */
264 bus_write_1(sc->mem[i], 4, 0xff);
265 bus_write_1(sc->mem[i], 12, 0xff);
266 }
267 }
268 }
269 #endif
270
271 mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF);
272 sc->nintvecs = 0;
273
274 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS,
275 1 /* Number of IPIs */, FALSE);
276 root_pic = dev;
277
278 #ifdef POWERNV
279 if (sc->xics_emu)
280 powernv_smp_ap_extra_init = xicp_smp_cpu_startup;
281 #endif
282
283 return (0);
284 }
285
286 static int
xics_attach(device_t dev)287 xics_attach(device_t dev)
288 {
289 phandle_t phandle = ofw_bus_get_node(dev);
290
291 /* The XICP (root PIC) will handle all our interrupts */
292 powerpc_register_pic(root_pic, OF_xref_from_node(phandle),
293 MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE);
294
295 return (0);
296 }
297
298 static __inline struct xicp_intvec *
xicp_setup_priv(struct xicp_softc * sc,u_int irq,void ** priv)299 xicp_setup_priv(struct xicp_softc *sc, u_int irq, void **priv)
300 {
301 if (*priv == NULL) {
302 KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs),
303 ("Too many XICP interrupts"));
304 mtx_lock(&sc->sc_mtx);
305 *priv = &sc->intvecs[sc->nintvecs++];
306 mtx_unlock(&sc->sc_mtx);
307 }
308
309 return (*priv);
310 }
311
312 /*
313 * PIC I/F methods.
314 */
315
316 static void
xicp_bind(device_t dev,u_int irq,cpuset_t cpumask,void ** priv)317 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv)
318 {
319 struct xicp_softc *sc = device_get_softc(dev);
320 struct xicp_intvec *iv;
321 cell_t status, cpu;
322 int ncpus, i, error = -1;
323
324 /* Ignore IPIs */
325 if (irq == MAX_XICP_IRQS)
326 return;
327
328 iv = xicp_setup_priv(sc, irq, priv);
329
330 /*
331 * This doesn't appear to actually support affinity groups, so pick a
332 * random CPU.
333 */
334 ncpus = 0;
335 CPU_FOREACH(cpu)
336 if (CPU_ISSET(cpu, &cpumask)) ncpus++;
337
338 i = mftb() % ncpus;
339 ncpus = 0;
340 CPU_FOREACH(cpu) {
341 if (!CPU_ISSET(cpu, &cpumask))
342 continue;
343 if (ncpus == i)
344 break;
345 ncpus++;
346 }
347
348 cpu = pcpu_find(cpu)->pc_hwref;
349 iv->cpu = cpu;
350
351 if (rtas_exists())
352 error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
353 XICP_PRIORITY, &status);
354 #ifdef POWERNV
355 else
356 error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
357 #endif
358
359 if (error < 0)
360 panic("Cannot bind interrupt %d to CPU %d", irq, cpu);
361 }
362
363 static void
xicp_dispatch(device_t dev,struct trapframe * tf)364 xicp_dispatch(device_t dev, struct trapframe *tf)
365 {
366 struct xicp_softc *sc;
367 struct resource *regs = NULL;
368 uint64_t xirr, junk;
369 int i;
370
371 sc = device_get_softc(dev);
372 #ifdef POWERNV
373 if ((mfmsr() & PSL_HV) && !sc->xics_emu) {
374 regs = xicp_mem_for_cpu(PCPU_GET(hwref));
375 KASSERT(regs != NULL,
376 ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref)));
377 }
378 #endif
379
380 for (;;) {
381 /* Return value in R4, use the PFT call */
382 if (regs) {
383 xirr = bus_read_4(regs, 4);
384 #ifdef POWERNV
385 } else if (sc->xics_emu) {
386 opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)],
387 false);
388 xirr = cpu_xirr[PCPU_GET(cpuid)];
389 #endif
390 } else {
391 /* Return value in R4, use the PFT call */
392 phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk);
393 }
394 xirr &= 0x00ffffff;
395
396 if (xirr == 0) /* No more pending interrupts? */
397 break;
398
399 if (xirr == XICP_IPI) { /* Magic number for IPIs */
400 xirr = MAX_XICP_IRQS; /* Map to FreeBSD magic */
401
402 /* Clear IPI */
403 if (regs)
404 bus_write_1(regs, 12, 0xff);
405 #ifdef POWERNV
406 else if (sc->xics_emu)
407 opal_call(OPAL_INT_SET_MFRR,
408 PCPU_GET(hwref), 0xff);
409 #endif
410 else
411 phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)),
412 0xff);
413 i = sc->ipi_vec;
414 } else {
415 /* XXX: super inefficient */
416 for (i = 0; i < sc->nintvecs; i++) {
417 if (sc->intvecs[i].irq == xirr)
418 break;
419 }
420 KASSERT(i < sc->nintvecs, ("Unmapped XIRR"));
421 }
422
423 powerpc_dispatch_intr(sc->intvecs[i].vector, tf);
424 }
425 }
426
427 static void
xicp_enable(device_t dev,u_int irq,u_int vector,void ** priv)428 xicp_enable(device_t dev, u_int irq, u_int vector, void **priv)
429 {
430 struct xicp_softc *sc;
431 struct xicp_intvec *intr;
432 cell_t status, cpu;
433
434 sc = device_get_softc(dev);
435
436 /* Bind to this CPU to start: distrib. ID is last entry in gserver# */
437 cpu = PCPU_GET(hwref);
438
439 intr = xicp_setup_priv(sc, irq, priv);
440
441 intr->irq = irq;
442 intr->vector = vector;
443 intr->cpu = cpu;
444 mb();
445
446 /* IPIs are also enabled. Stash off the vector index */
447 if (irq == MAX_XICP_IRQS) {
448 sc->ipi_vec = intr - sc->intvecs;
449 return;
450 }
451
452 if (rtas_exists()) {
453 rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu,
454 XICP_PRIORITY, &status);
455 xicp_unmask(dev, irq, intr);
456 #ifdef POWERNV
457 } else {
458 status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY);
459 /* Unmask implicit for OPAL */
460
461 if (status != 0)
462 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq,
463 cpu, status);
464 #endif
465 }
466 }
467
468 static void
xicp_eoi(device_t dev,u_int irq,void * priv)469 xicp_eoi(device_t dev, u_int irq, void *priv)
470 {
471 #ifdef POWERNV
472 struct xicp_softc *sc;
473 #endif
474 uint64_t xirr;
475
476 if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */
477 irq = XICP_IPI;
478 xirr = irq | (0xff << 24);
479
480 #ifdef POWERNV
481 if (mfmsr() & PSL_HV) {
482 sc = device_get_softc(dev);
483 if (sc->xics_emu)
484 opal_call(OPAL_INT_EOI, xirr);
485 else
486 bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr);
487 } else
488 #endif
489 phyp_hcall(H_EOI, xirr);
490 }
491
492 static void
xicp_ipi(device_t dev,u_int cpu)493 xicp_ipi(device_t dev, u_int cpu)
494 {
495
496 #ifdef POWERNV
497 struct xicp_softc *sc;
498 cpu = pcpu_find(cpu)->pc_hwref;
499
500 if (mfmsr() & PSL_HV) {
501 sc = device_get_softc(dev);
502 if (sc->xics_emu) {
503 int64_t rv;
504 rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY);
505 if (rv != 0)
506 device_printf(dev, "IPI SET_MFRR result: %ld\n", rv);
507 } else
508 bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY);
509 } else
510 #endif
511 phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY);
512 }
513
514 static void
xicp_mask(device_t dev,u_int irq,void * priv)515 xicp_mask(device_t dev, u_int irq, void *priv)
516 {
517 struct xicp_softc *sc = device_get_softc(dev);
518 cell_t status;
519
520 if (irq == MAX_XICP_IRQS)
521 return;
522
523 if (rtas_exists()) {
524 rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status);
525 #ifdef POWERNV
526 } else {
527 struct xicp_intvec *ivec = priv;
528
529 KASSERT(ivec != NULL, ("Masking unconfigured interrupt"));
530 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, 0xff);
531 #endif
532 }
533 }
534
535 static void
xicp_unmask(device_t dev,u_int irq,void * priv)536 xicp_unmask(device_t dev, u_int irq, void *priv)
537 {
538 struct xicp_softc *sc = device_get_softc(dev);
539 cell_t status;
540
541 if (irq == MAX_XICP_IRQS)
542 return;
543
544 if (rtas_exists()) {
545 rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status);
546 #ifdef POWERNV
547 } else {
548 struct xicp_intvec *ivec = priv;
549
550 KASSERT(ivec != NULL, ("Unmasking unconfigured interrupt"));
551 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, XICP_PRIORITY);
552 #endif
553 }
554 }
555
556 #ifdef POWERNV
557 /* This is only used on POWER9 systems with the XIVE's XICS emulation. */
558 static void
xicp_smp_cpu_startup(void)559 xicp_smp_cpu_startup(void)
560 {
561 struct xicp_softc *sc;
562
563 if (mfmsr() & PSL_HV) {
564 sc = device_get_softc(root_pic);
565
566 if (sc->xics_emu)
567 opal_call(OPAL_INT_SET_CPPR, 0xff);
568 }
569 }
570 #endif
571