1 /* $OpenBSD: xive.c,v 1.17 2022/04/06 18:59:27 naddy Exp $ */
2 /*
3 * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org>
4 *
5 * Permission to use, copy, modify, and distribute this software for any
6 * purpose with or without fee is hereby granted, provided that the above
7 * copyright notice and this permission notice appear in all copies.
8 *
9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
16 */
17
18 #include <sys/param.h>
19 #include <sys/systm.h>
20 #include <sys/device.h>
21 #include <sys/evcount.h>
22 #include <sys/malloc.h>
23 #include <sys/queue.h>
24
25 #include <machine/bus.h>
26 #include <machine/fdt.h>
27 #include <machine/opal.h>
28
29 #include <dev/ofw/openfirm.h>
30 #include <dev/ofw/fdt.h>
31
32 #define XIVE_NUM_PRIORITIES 8
33 #define XIVE_NUM_IRQS 1024
34
35 #define XIVE_EQ_SIZE PAGE_SHIFT
36 #define XIVE_EQ_IDX_MASK ((1 << (PAGE_SHIFT - 2)) - 1)
37 #define XIVE_EQ_GEN_MASK 0x80000000
38
39 #define XIVE_TM_CPPR_HV 0x031
40
41 #define XIVE_TM_SPC_ACK_HV 0x830
42 #define XIVE_TM_SPC_ACK_HE_MASK 0xc000
43 #define XIVE_TM_SPC_ACK_HE_NONE 0x0000
44 #define XIVE_TM_SPC_ACK_HE_PHYS 0x8000
45
46 #define XIVE_ESB_STORE_TRIGGER 0x000
47 #define XIVE_ESB_LOAD_EOI 0x000
48 #define XIVE_ESB_STORE_EOI 0x400
49 #define XIVE_ESB_SET_PQ_00 0xc00
50 #define XIVE_ESB_SET_PQ_01 0xd00
51 #define XIVE_ESB_SET_PQ_10 0xe00
52 #define XIVE_ESB_SET_PQ_11 0xf00
53
54 #define XIVE_ESB_VAL_P 0x2
55 #define XIVE_ESB_VAL_Q 0x1
56
57 static inline uint8_t
xive_prio(int ipl)58 xive_prio(int ipl)
59 {
60 return ((IPL_IPI - ipl) > 7 ? 0xff : IPL_IPI - ipl);
61 }
62
63 static inline int
xive_ipl(uint8_t prio)64 xive_ipl(uint8_t prio)
65 {
66 return (IPL_IPI - prio);
67 }
68
69 struct intrhand {
70 TAILQ_ENTRY(intrhand) ih_list;
71 int (*ih_func)(void *);
72 void *ih_arg;
73 int ih_ipl;
74 int ih_flags;
75 uint32_t ih_girq;
76 struct evcount ih_count;
77 const char *ih_name;
78
79 bus_space_handle_t ih_esb_eoi;
80 bus_space_handle_t ih_esb_trig;
81 uint64_t ih_xive_flags;
82 };
83
84 struct xive_eq {
85 struct xive_dmamem *eq_queue;
86 uint32_t eq_idx;
87 uint32_t eq_gen;
88 };
89
90 struct xive_softc {
91 struct device sc_dev;
92 bus_space_tag_t sc_iot;
93 bus_space_handle_t sc_ioh;
94 bus_dma_tag_t sc_dmat;
95
96 struct intrhand *sc_handler[XIVE_NUM_IRQS];
97 struct xive_eq sc_eq[MAXCPUS][XIVE_NUM_PRIORITIES];
98
99 uint32_t sc_page_size;
100 uint32_t sc_lirq;
101 };
102
103 struct xive_softc *xive_sc;
104
105 struct xive_dmamem {
106 bus_dmamap_t xdm_map;
107 bus_dma_segment_t xdm_seg;
108 size_t xdm_size;
109 caddr_t xdm_kva;
110 };
111
112 #define XIVE_DMA_MAP(_xdm) ((_xdm)->xdm_map)
113 #define XIVE_DMA_LEN(_xdm) ((_xdm)->xdm_size)
114 #define XIVE_DMA_DVA(_xdm) ((_xdm)->xdm_map->dm_segs[0].ds_addr)
115 #define XIVE_DMA_KVA(_xdm) ((void *)(_xdm)->xdm_kva)
116
117 struct xive_dmamem *xive_dmamem_alloc(bus_dma_tag_t, bus_size_t,
118 bus_size_t);
119 void xive_dmamem_free(bus_dma_tag_t, struct xive_dmamem *);
120
121 static inline void
xive_write_1(struct xive_softc * sc,bus_size_t off,uint8_t val)122 xive_write_1(struct xive_softc *sc, bus_size_t off, uint8_t val)
123 {
124 bus_space_write_1(sc->sc_iot, sc->sc_ioh, off, val);
125 }
126
127 static inline uint16_t
xive_read_2(struct xive_softc * sc,bus_size_t off)128 xive_read_2(struct xive_softc *sc, bus_size_t off)
129 {
130 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, off);
131 }
132
133 static inline void
xive_unmask(struct xive_softc * sc,struct intrhand * ih)134 xive_unmask(struct xive_softc *sc, struct intrhand *ih)
135 {
136 bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi, XIVE_ESB_SET_PQ_00);
137 }
138
139 int xive_match(struct device *, void *, void *);
140 void xive_attach(struct device *, struct device *, void *);
141 int xive_activate(struct device *, int);
142
143 const struct cfattach xive_ca = {
144 sizeof (struct xive_softc), xive_match, xive_attach, NULL,
145 xive_activate
146 };
147
148 struct cfdriver xive_cd = {
149 NULL, "xive", DV_DULL
150 };
151
152 void xive_hvi(struct trapframe *);
153 void *xive_intr_establish(uint32_t, int, int, struct cpu_info *,
154 int (*)(void *), void *, const char *);
155 void xive_intr_send_ipi(void *);
156 void xive_setipl(int);
157
158 int
xive_match(struct device * parent,void * match,void * aux)159 xive_match(struct device *parent, void *match, void *aux)
160 {
161 struct fdt_attach_args *faa = aux;
162
163 return OF_is_compatible(faa->fa_node, "ibm,opal-xive-pe");
164 }
165
166 void
xive_attach(struct device * parent,struct device * self,void * aux)167 xive_attach(struct device *parent, struct device *self, void *aux)
168 {
169 struct xive_softc *sc = (struct xive_softc *)self;
170 struct fdt_attach_args *faa = aux;
171 struct cpu_info *ci;
172 CPU_INFO_ITERATOR cii;
173 int64_t error;
174 int i;
175
176 if (faa->fa_nreg < 2) {
177 printf(": no registers\n");
178 return;
179 }
180
181 sc->sc_iot = faa->fa_iot;
182 if (bus_space_map(sc->sc_iot, faa->fa_reg[1].addr,
183 faa->fa_reg[1].size, 0, &sc->sc_ioh)) {
184 printf(": can't map registers\n");
185 return;
186 }
187
188 sc->sc_dmat = faa->fa_dmat;
189
190 sc->sc_page_size = OF_getpropint(faa->fa_node,
191 "ibm,xive-provision-page-size", 0);
192
193 error = opal_xive_reset(OPAL_XIVE_MODE_EXPL);
194 if (error != OPAL_SUCCESS)
195 printf(": can't enable exploitation mode\n");
196
197 printf("\n");
198
199 CPU_INFO_FOREACH(cii, ci) {
200 for (i = 0; i < XIVE_NUM_PRIORITIES; i++) {
201 sc->sc_eq[ci->ci_cpuid][i].eq_queue =
202 xive_dmamem_alloc(sc->sc_dmat,
203 1 << XIVE_EQ_SIZE, 1 << XIVE_EQ_SIZE);
204 if (sc->sc_eq[ci->ci_cpuid][i].eq_queue == NULL) {
205 printf("%s: can't allocate event queue\n",
206 sc->sc_dev.dv_xname);
207 return;
208 }
209
210 error = opal_xive_set_queue_info(ci->ci_pir, i,
211 XIVE_DMA_DVA(sc->sc_eq[ci->ci_cpuid][i].eq_queue),
212 XIVE_EQ_SIZE, OPAL_XIVE_EQ_ENABLED |
213 OPAL_XIVE_EQ_ALWAYS_NOTIFY);
214 if (error != OPAL_SUCCESS) {
215 printf("%s: can't enable event queue\n",
216 sc->sc_dev.dv_xname);
217 return;
218 }
219
220 sc->sc_eq[ci->ci_cpuid][i].eq_gen = XIVE_EQ_GEN_MASK;
221 }
222 }
223
224 /* There can be only one. */
225 KASSERT(xive_sc == NULL);
226 xive_sc = sc;
227
228 _hvi = xive_hvi;
229 _intr_establish = xive_intr_establish;
230 _intr_send_ipi = xive_intr_send_ipi;
231 _setipl = xive_setipl;
232
233 /* Synchronize hardware state to software state. */
234 xive_write_1(sc, XIVE_TM_CPPR_HV, xive_prio(curcpu()->ci_cpl));
235 eieio();
236 }
237
238 int
xive_activate(struct device * self,int act)239 xive_activate(struct device *self, int act)
240 {
241 switch (act) {
242 case DVACT_POWERDOWN:
243 opal_xive_reset(OPAL_XIVE_MODE_EMU);
244 break;
245 }
246
247 return 0;
248 }
249
250 void *
xive_intr_establish(uint32_t girq,int type,int level,struct cpu_info * ci,int (* func)(void *),void * arg,const char * name)251 xive_intr_establish(uint32_t girq, int type, int level, struct cpu_info *ci,
252 int (*func)(void *), void *arg, const char *name)
253 {
254 struct xive_softc *sc = xive_sc;
255 struct intrhand *ih;
256 bus_space_handle_t eoi, trig;
257 bus_size_t page_size;
258 uint64_t flags, eoi_page, trig_page;
259 uint32_t esb_shift, lirq;
260 int64_t error;
261
262 if (ci == NULL)
263 ci = cpu_info_primary;
264
265 /* Allocate a logical IRQ. */
266 if (sc->sc_lirq >= XIVE_NUM_IRQS)
267 return NULL;
268 lirq = sc->sc_lirq++;
269
270 error = opal_xive_get_irq_info(girq, opal_phys(&flags),
271 opal_phys(&eoi_page), opal_phys(&trig_page),
272 opal_phys(&esb_shift), NULL);
273 if (error != OPAL_SUCCESS)
274 return NULL;
275 page_size = 1 << esb_shift;
276
277 /* Map EOI page. */
278 if (bus_space_map(sc->sc_iot, eoi_page, page_size, 0, &eoi))
279 return NULL;
280
281 /* Map trigger page. */
282 if (trig_page == eoi_page)
283 trig = eoi;
284 else if (trig_page == 0)
285 trig = 0;
286 else if (bus_space_map(sc->sc_iot, trig_page, page_size, 0, &trig)) {
287 bus_space_unmap(sc->sc_iot, trig, page_size);
288 return NULL;
289 }
290
291 error = opal_xive_set_irq_config(girq, ci->ci_pir,
292 xive_prio(level & IPL_IRQMASK), lirq);
293 if (error != OPAL_SUCCESS) {
294 if (trig != eoi && trig != 0)
295 bus_space_unmap(sc->sc_iot, trig, page_size);
296 bus_space_unmap(sc->sc_iot, eoi, page_size);
297 return NULL;
298 }
299
300 ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK);
301 ih->ih_func = func;
302 ih->ih_arg = arg;
303 ih->ih_ipl = level & IPL_IRQMASK;
304 ih->ih_flags = level & IPL_FLAGMASK;
305 ih->ih_girq = girq;
306 ih->ih_name = name;
307 ih->ih_esb_eoi = eoi;
308 ih->ih_esb_trig = trig;
309 ih->ih_xive_flags = flags;
310 sc->sc_handler[lirq] = ih;
311
312 if (name != NULL)
313 evcount_attach(&ih->ih_count, name, &ih->ih_girq);
314
315 xive_unmask(sc, ih);
316
317 return ih;
318 }
319
320 void
xive_intr_send_ipi(void * cookie)321 xive_intr_send_ipi(void *cookie)
322 {
323 struct xive_softc *sc = xive_sc;
324 struct intrhand *ih = cookie;
325
326 if (ih && ih->ih_esb_trig)
327 bus_space_write_8(sc->sc_iot, ih->ih_esb_trig,
328 XIVE_ESB_STORE_TRIGGER, 0);
329 }
330
331 void
xive_eoi(struct xive_softc * sc,struct intrhand * ih)332 xive_eoi(struct xive_softc *sc, struct intrhand *ih)
333 {
334 uint64_t eoi;
335
336 if (ih->ih_xive_flags & OPAL_XIVE_IRQ_STORE_EOI) {
337 bus_space_write_8(sc->sc_iot, ih->ih_esb_eoi,
338 XIVE_ESB_STORE_EOI, 0);
339 } else if (ih->ih_xive_flags & OPAL_XIVE_IRQ_LSI) {
340 eoi = bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi,
341 XIVE_ESB_LOAD_EOI);
342 } else {
343 eoi = bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi,
344 XIVE_ESB_SET_PQ_00);
345 if ((eoi & XIVE_ESB_VAL_Q) && ih->ih_esb_trig != 0)
346 bus_space_write_8(sc->sc_iot, ih->ih_esb_trig,
347 XIVE_ESB_STORE_TRIGGER, 0);
348 }
349 }
350
351 void
xive_setipl(int new)352 xive_setipl(int new)
353 {
354 struct xive_softc *sc = xive_sc;
355 struct cpu_info *ci = curcpu();
356 uint8_t oldprio = xive_prio(ci->ci_cpl);
357 uint8_t newprio = xive_prio(new);
358 u_long msr;
359
360 msr = intr_disable();
361 ci->ci_cpl = new;
362 if (newprio != oldprio) {
363 xive_write_1(sc, XIVE_TM_CPPR_HV, newprio);
364 eieio();
365 }
366 intr_restore(msr);
367 }
368
369 void
xive_run_handler(struct intrhand * ih)370 xive_run_handler(struct intrhand *ih)
371 {
372 int handled;
373
374 #ifdef MULTIPROCESSOR
375 int need_lock;
376
377 if (ih->ih_flags & IPL_MPSAFE)
378 need_lock = 0;
379 else
380 need_lock = (ih->ih_ipl < IPL_SCHED);
381
382 if (need_lock)
383 KERNEL_LOCK();
384 #endif
385 handled = ih->ih_func(ih->ih_arg);
386 if (handled)
387 ih->ih_count.ec_count++;
388 #ifdef MULTIPROCESSOR
389 if (need_lock)
390 KERNEL_UNLOCK();
391 #endif
392 }
393
394 void
xive_hvi(struct trapframe * frame)395 xive_hvi(struct trapframe *frame)
396 {
397 struct xive_softc *sc = xive_sc;
398 struct cpu_info *ci = curcpu();
399 struct intrhand *ih;
400 struct xive_eq *eq;
401 uint32_t *event;
402 uint32_t lirq;
403 int old, new;
404 uint16_t ack, he;
405 uint8_t cppr;
406
407 old = ci->ci_cpl;
408
409 while (1) {
410 ack = xive_read_2(sc, XIVE_TM_SPC_ACK_HV);
411
412 he = (ack & XIVE_TM_SPC_ACK_HE_MASK);
413 if (he == XIVE_TM_SPC_ACK_HE_NONE)
414 break;
415 KASSERT(he == XIVE_TM_SPC_ACK_HE_PHYS);
416
417 eieio();
418
419 /* Synchronize software state to hardware state. */
420 cppr = ack;
421 new = xive_ipl(cppr);
422 if (new <= old) {
423 /*
424 * QEMU generates spurious interrupts. It is
425 * unclear whether this can happen on real
426 * hardware as well. We just ignore the
427 * interrupt, but we need to reset the CPPR
428 * register since we did accept the interrupt.
429 */
430 goto spurious;
431 }
432 ci->ci_cpl = new;
433
434 KASSERT(cppr < XIVE_NUM_PRIORITIES);
435 eq = &sc->sc_eq[ci->ci_cpuid][cppr];
436 event = XIVE_DMA_KVA(eq->eq_queue);
437 while ((event[eq->eq_idx] & XIVE_EQ_GEN_MASK) == eq->eq_gen) {
438 lirq = event[eq->eq_idx] & ~XIVE_EQ_GEN_MASK;
439 KASSERT(lirq < XIVE_NUM_IRQS);
440 ih = sc->sc_handler[lirq];
441 if (ih != NULL) {
442 intr_enable();
443 xive_run_handler(ih);
444 intr_disable();
445 xive_eoi(sc, ih);
446 }
447 eq->eq_idx = (eq->eq_idx + 1) & XIVE_EQ_IDX_MASK;
448
449 /* Toggle generation on wrap around. */
450 if (eq->eq_idx == 0)
451 eq->eq_gen ^= XIVE_EQ_GEN_MASK;
452 }
453
454 ci->ci_cpl = old;
455 spurious:
456 xive_write_1(sc, XIVE_TM_CPPR_HV, xive_prio(old));
457 eieio();
458 }
459 }
460
461 struct xive_dmamem *
xive_dmamem_alloc(bus_dma_tag_t dmat,bus_size_t size,bus_size_t align)462 xive_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align)
463 {
464 struct xive_dmamem *xdm;
465 int nsegs;
466
467 xdm = malloc(sizeof(*xdm), M_DEVBUF, M_WAITOK | M_ZERO);
468 xdm->xdm_size = size;
469
470 if (bus_dmamap_create(dmat, size, 1, size, 0,
471 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &xdm->xdm_map) != 0)
472 goto xdmfree;
473
474 if (bus_dmamem_alloc(dmat, size, align, 0, &xdm->xdm_seg, 1,
475 &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0)
476 goto destroy;
477
478 if (bus_dmamem_map(dmat, &xdm->xdm_seg, nsegs, size,
479 &xdm->xdm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0)
480 goto free;
481
482 if (bus_dmamap_load_raw(dmat, xdm->xdm_map, &xdm->xdm_seg,
483 nsegs, size, BUS_DMA_WAITOK) != 0)
484 goto unmap;
485
486 return xdm;
487
488 unmap:
489 bus_dmamem_unmap(dmat, xdm->xdm_kva, size);
490 free:
491 bus_dmamem_free(dmat, &xdm->xdm_seg, 1);
492 destroy:
493 bus_dmamap_destroy(dmat, xdm->xdm_map);
494 xdmfree:
495 free(xdm, M_DEVBUF, sizeof(*xdm));
496
497 return NULL;
498 }
499
500 void
xive_dmamem_free(bus_dma_tag_t dmat,struct xive_dmamem * xdm)501 xive_dmamem_free(bus_dma_tag_t dmat, struct xive_dmamem *xdm)
502 {
503 bus_dmamem_unmap(dmat, xdm->xdm_kva, xdm->xdm_size);
504 bus_dmamem_free(dmat, &xdm->xdm_seg, 1);
505 bus_dmamap_destroy(dmat, xdm->xdm_map);
506 free(xdm, M_DEVBUF, sizeof(*xdm));
507 }
508