1 /* $OpenBSD: xive.c,v 1.17 2022/04/06 18:59:27 naddy Exp $ */ 2 /* 3 * Copyright (c) 2020 Mark Kettenis <kettenis@openbsd.org> 4 * 5 * Permission to use, copy, modify, and distribute this software for any 6 * purpose with or without fee is hereby granted, provided that the above 7 * copyright notice and this permission notice appear in all copies. 8 * 9 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 10 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 11 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 12 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 13 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 14 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 15 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 16 */ 17 18 #include <sys/param.h> 19 #include <sys/systm.h> 20 #include <sys/device.h> 21 #include <sys/evcount.h> 22 #include <sys/malloc.h> 23 #include <sys/queue.h> 24 25 #include <machine/bus.h> 26 #include <machine/fdt.h> 27 #include <machine/opal.h> 28 29 #include <dev/ofw/openfirm.h> 30 #include <dev/ofw/fdt.h> 31 32 #define XIVE_NUM_PRIORITIES 8 33 #define XIVE_NUM_IRQS 1024 34 35 #define XIVE_EQ_SIZE PAGE_SHIFT 36 #define XIVE_EQ_IDX_MASK ((1 << (PAGE_SHIFT - 2)) - 1) 37 #define XIVE_EQ_GEN_MASK 0x80000000 38 39 #define XIVE_TM_CPPR_HV 0x031 40 41 #define XIVE_TM_SPC_ACK_HV 0x830 42 #define XIVE_TM_SPC_ACK_HE_MASK 0xc000 43 #define XIVE_TM_SPC_ACK_HE_NONE 0x0000 44 #define XIVE_TM_SPC_ACK_HE_PHYS 0x8000 45 46 #define XIVE_ESB_STORE_TRIGGER 0x000 47 #define XIVE_ESB_LOAD_EOI 0x000 48 #define XIVE_ESB_STORE_EOI 0x400 49 #define XIVE_ESB_SET_PQ_00 0xc00 50 #define XIVE_ESB_SET_PQ_01 0xd00 51 #define XIVE_ESB_SET_PQ_10 0xe00 52 #define XIVE_ESB_SET_PQ_11 0xf00 53 54 #define XIVE_ESB_VAL_P 0x2 55 #define XIVE_ESB_VAL_Q 0x1 56 57 static inline uint8_t 58 xive_prio(int ipl) 59 { 60 return ((IPL_IPI - ipl) > 7 ? 0xff : IPL_IPI - ipl); 61 } 62 63 static inline int 64 xive_ipl(uint8_t prio) 65 { 66 return (IPL_IPI - prio); 67 } 68 69 struct intrhand { 70 TAILQ_ENTRY(intrhand) ih_list; 71 int (*ih_func)(void *); 72 void *ih_arg; 73 int ih_ipl; 74 int ih_flags; 75 uint32_t ih_girq; 76 struct evcount ih_count; 77 const char *ih_name; 78 79 bus_space_handle_t ih_esb_eoi; 80 bus_space_handle_t ih_esb_trig; 81 uint64_t ih_xive_flags; 82 }; 83 84 struct xive_eq { 85 struct xive_dmamem *eq_queue; 86 uint32_t eq_idx; 87 uint32_t eq_gen; 88 }; 89 90 struct xive_softc { 91 struct device sc_dev; 92 bus_space_tag_t sc_iot; 93 bus_space_handle_t sc_ioh; 94 bus_dma_tag_t sc_dmat; 95 96 struct intrhand *sc_handler[XIVE_NUM_IRQS]; 97 struct xive_eq sc_eq[MAXCPUS][XIVE_NUM_PRIORITIES]; 98 99 uint32_t sc_page_size; 100 uint32_t sc_lirq; 101 }; 102 103 struct xive_softc *xive_sc; 104 105 struct xive_dmamem { 106 bus_dmamap_t xdm_map; 107 bus_dma_segment_t xdm_seg; 108 size_t xdm_size; 109 caddr_t xdm_kva; 110 }; 111 112 #define XIVE_DMA_MAP(_xdm) ((_xdm)->xdm_map) 113 #define XIVE_DMA_LEN(_xdm) ((_xdm)->xdm_size) 114 #define XIVE_DMA_DVA(_xdm) ((_xdm)->xdm_map->dm_segs[0].ds_addr) 115 #define XIVE_DMA_KVA(_xdm) ((void *)(_xdm)->xdm_kva) 116 117 struct xive_dmamem *xive_dmamem_alloc(bus_dma_tag_t, bus_size_t, 118 bus_size_t); 119 void xive_dmamem_free(bus_dma_tag_t, struct xive_dmamem *); 120 121 static inline void 122 xive_write_1(struct xive_softc *sc, bus_size_t off, uint8_t val) 123 { 124 bus_space_write_1(sc->sc_iot, sc->sc_ioh, off, val); 125 } 126 127 static inline uint16_t 128 xive_read_2(struct xive_softc *sc, bus_size_t off) 129 { 130 return bus_space_read_2(sc->sc_iot, sc->sc_ioh, off); 131 } 132 133 static inline void 134 xive_unmask(struct xive_softc *sc, struct intrhand *ih) 135 { 136 bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi, XIVE_ESB_SET_PQ_00); 137 } 138 139 int xive_match(struct device *, void *, void *); 140 void xive_attach(struct device *, struct device *, void *); 141 int xive_activate(struct device *, int); 142 143 const struct cfattach xive_ca = { 144 sizeof (struct xive_softc), xive_match, xive_attach, NULL, 145 xive_activate 146 }; 147 148 struct cfdriver xive_cd = { 149 NULL, "xive", DV_DULL 150 }; 151 152 void xive_hvi(struct trapframe *); 153 void *xive_intr_establish(uint32_t, int, int, struct cpu_info *, 154 int (*)(void *), void *, const char *); 155 void xive_intr_send_ipi(void *); 156 void xive_setipl(int); 157 158 int 159 xive_match(struct device *parent, void *match, void *aux) 160 { 161 struct fdt_attach_args *faa = aux; 162 163 return OF_is_compatible(faa->fa_node, "ibm,opal-xive-pe"); 164 } 165 166 void 167 xive_attach(struct device *parent, struct device *self, void *aux) 168 { 169 struct xive_softc *sc = (struct xive_softc *)self; 170 struct fdt_attach_args *faa = aux; 171 struct cpu_info *ci; 172 CPU_INFO_ITERATOR cii; 173 int64_t error; 174 int i; 175 176 if (faa->fa_nreg < 2) { 177 printf(": no registers\n"); 178 return; 179 } 180 181 sc->sc_iot = faa->fa_iot; 182 if (bus_space_map(sc->sc_iot, faa->fa_reg[1].addr, 183 faa->fa_reg[1].size, 0, &sc->sc_ioh)) { 184 printf(": can't map registers\n"); 185 return; 186 } 187 188 sc->sc_dmat = faa->fa_dmat; 189 190 sc->sc_page_size = OF_getpropint(faa->fa_node, 191 "ibm,xive-provision-page-size", 0); 192 193 error = opal_xive_reset(OPAL_XIVE_MODE_EXPL); 194 if (error != OPAL_SUCCESS) 195 printf(": can't enable exploitation mode\n"); 196 197 printf("\n"); 198 199 CPU_INFO_FOREACH(cii, ci) { 200 for (i = 0; i < XIVE_NUM_PRIORITIES; i++) { 201 sc->sc_eq[ci->ci_cpuid][i].eq_queue = 202 xive_dmamem_alloc(sc->sc_dmat, 203 1 << XIVE_EQ_SIZE, 1 << XIVE_EQ_SIZE); 204 if (sc->sc_eq[ci->ci_cpuid][i].eq_queue == NULL) { 205 printf("%s: can't allocate event queue\n", 206 sc->sc_dev.dv_xname); 207 return; 208 } 209 210 error = opal_xive_set_queue_info(ci->ci_pir, i, 211 XIVE_DMA_DVA(sc->sc_eq[ci->ci_cpuid][i].eq_queue), 212 XIVE_EQ_SIZE, OPAL_XIVE_EQ_ENABLED | 213 OPAL_XIVE_EQ_ALWAYS_NOTIFY); 214 if (error != OPAL_SUCCESS) { 215 printf("%s: can't enable event queue\n", 216 sc->sc_dev.dv_xname); 217 return; 218 } 219 220 sc->sc_eq[ci->ci_cpuid][i].eq_gen = XIVE_EQ_GEN_MASK; 221 } 222 } 223 224 /* There can be only one. */ 225 KASSERT(xive_sc == NULL); 226 xive_sc = sc; 227 228 _hvi = xive_hvi; 229 _intr_establish = xive_intr_establish; 230 _intr_send_ipi = xive_intr_send_ipi; 231 _setipl = xive_setipl; 232 233 /* Synchronize hardware state to software state. */ 234 xive_write_1(sc, XIVE_TM_CPPR_HV, xive_prio(curcpu()->ci_cpl)); 235 eieio(); 236 } 237 238 int 239 xive_activate(struct device *self, int act) 240 { 241 switch (act) { 242 case DVACT_POWERDOWN: 243 opal_xive_reset(OPAL_XIVE_MODE_EMU); 244 break; 245 } 246 247 return 0; 248 } 249 250 void * 251 xive_intr_establish(uint32_t girq, int type, int level, struct cpu_info *ci, 252 int (*func)(void *), void *arg, const char *name) 253 { 254 struct xive_softc *sc = xive_sc; 255 struct intrhand *ih; 256 bus_space_handle_t eoi, trig; 257 bus_size_t page_size; 258 uint64_t flags, eoi_page, trig_page; 259 uint32_t esb_shift, lirq; 260 int64_t error; 261 262 if (ci == NULL) 263 ci = cpu_info_primary; 264 265 /* Allocate a logical IRQ. */ 266 if (sc->sc_lirq >= XIVE_NUM_IRQS) 267 return NULL; 268 lirq = sc->sc_lirq++; 269 270 error = opal_xive_get_irq_info(girq, opal_phys(&flags), 271 opal_phys(&eoi_page), opal_phys(&trig_page), 272 opal_phys(&esb_shift), NULL); 273 if (error != OPAL_SUCCESS) 274 return NULL; 275 page_size = 1 << esb_shift; 276 277 /* Map EOI page. */ 278 if (bus_space_map(sc->sc_iot, eoi_page, page_size, 0, &eoi)) 279 return NULL; 280 281 /* Map trigger page. */ 282 if (trig_page == eoi_page) 283 trig = eoi; 284 else if (trig_page == 0) 285 trig = 0; 286 else if (bus_space_map(sc->sc_iot, trig_page, page_size, 0, &trig)) { 287 bus_space_unmap(sc->sc_iot, trig, page_size); 288 return NULL; 289 } 290 291 error = opal_xive_set_irq_config(girq, ci->ci_pir, 292 xive_prio(level & IPL_IRQMASK), lirq); 293 if (error != OPAL_SUCCESS) { 294 if (trig != eoi && trig != 0) 295 bus_space_unmap(sc->sc_iot, trig, page_size); 296 bus_space_unmap(sc->sc_iot, eoi, page_size); 297 return NULL; 298 } 299 300 ih = malloc(sizeof(*ih), M_DEVBUF, M_WAITOK); 301 ih->ih_func = func; 302 ih->ih_arg = arg; 303 ih->ih_ipl = level & IPL_IRQMASK; 304 ih->ih_flags = level & IPL_FLAGMASK; 305 ih->ih_girq = girq; 306 ih->ih_name = name; 307 ih->ih_esb_eoi = eoi; 308 ih->ih_esb_trig = trig; 309 ih->ih_xive_flags = flags; 310 sc->sc_handler[lirq] = ih; 311 312 if (name != NULL) 313 evcount_attach(&ih->ih_count, name, &ih->ih_girq); 314 315 xive_unmask(sc, ih); 316 317 return ih; 318 } 319 320 void 321 xive_intr_send_ipi(void *cookie) 322 { 323 struct xive_softc *sc = xive_sc; 324 struct intrhand *ih = cookie; 325 326 if (ih && ih->ih_esb_trig) 327 bus_space_write_8(sc->sc_iot, ih->ih_esb_trig, 328 XIVE_ESB_STORE_TRIGGER, 0); 329 } 330 331 void 332 xive_eoi(struct xive_softc *sc, struct intrhand *ih) 333 { 334 uint64_t eoi; 335 336 if (ih->ih_xive_flags & OPAL_XIVE_IRQ_STORE_EOI) { 337 bus_space_write_8(sc->sc_iot, ih->ih_esb_eoi, 338 XIVE_ESB_STORE_EOI, 0); 339 } else if (ih->ih_xive_flags & OPAL_XIVE_IRQ_LSI) { 340 eoi = bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi, 341 XIVE_ESB_LOAD_EOI); 342 } else { 343 eoi = bus_space_read_8(sc->sc_iot, ih->ih_esb_eoi, 344 XIVE_ESB_SET_PQ_00); 345 if ((eoi & XIVE_ESB_VAL_Q) && ih->ih_esb_trig != 0) 346 bus_space_write_8(sc->sc_iot, ih->ih_esb_trig, 347 XIVE_ESB_STORE_TRIGGER, 0); 348 } 349 } 350 351 void 352 xive_setipl(int new) 353 { 354 struct xive_softc *sc = xive_sc; 355 struct cpu_info *ci = curcpu(); 356 uint8_t oldprio = xive_prio(ci->ci_cpl); 357 uint8_t newprio = xive_prio(new); 358 u_long msr; 359 360 msr = intr_disable(); 361 ci->ci_cpl = new; 362 if (newprio != oldprio) { 363 xive_write_1(sc, XIVE_TM_CPPR_HV, newprio); 364 eieio(); 365 } 366 intr_restore(msr); 367 } 368 369 void 370 xive_run_handler(struct intrhand *ih) 371 { 372 int handled; 373 374 #ifdef MULTIPROCESSOR 375 int need_lock; 376 377 if (ih->ih_flags & IPL_MPSAFE) 378 need_lock = 0; 379 else 380 need_lock = (ih->ih_ipl < IPL_SCHED); 381 382 if (need_lock) 383 KERNEL_LOCK(); 384 #endif 385 handled = ih->ih_func(ih->ih_arg); 386 if (handled) 387 ih->ih_count.ec_count++; 388 #ifdef MULTIPROCESSOR 389 if (need_lock) 390 KERNEL_UNLOCK(); 391 #endif 392 } 393 394 void 395 xive_hvi(struct trapframe *frame) 396 { 397 struct xive_softc *sc = xive_sc; 398 struct cpu_info *ci = curcpu(); 399 struct intrhand *ih; 400 struct xive_eq *eq; 401 uint32_t *event; 402 uint32_t lirq; 403 int old, new; 404 uint16_t ack, he; 405 uint8_t cppr; 406 407 old = ci->ci_cpl; 408 409 while (1) { 410 ack = xive_read_2(sc, XIVE_TM_SPC_ACK_HV); 411 412 he = (ack & XIVE_TM_SPC_ACK_HE_MASK); 413 if (he == XIVE_TM_SPC_ACK_HE_NONE) 414 break; 415 KASSERT(he == XIVE_TM_SPC_ACK_HE_PHYS); 416 417 eieio(); 418 419 /* Synchronize software state to hardware state. */ 420 cppr = ack; 421 new = xive_ipl(cppr); 422 if (new <= old) { 423 /* 424 * QEMU generates spurious interrupts. It is 425 * unclear whether this can happen on real 426 * hardware as well. We just ignore the 427 * interrupt, but we need to reset the CPPR 428 * register since we did accept the interrupt. 429 */ 430 goto spurious; 431 } 432 ci->ci_cpl = new; 433 434 KASSERT(cppr < XIVE_NUM_PRIORITIES); 435 eq = &sc->sc_eq[ci->ci_cpuid][cppr]; 436 event = XIVE_DMA_KVA(eq->eq_queue); 437 while ((event[eq->eq_idx] & XIVE_EQ_GEN_MASK) == eq->eq_gen) { 438 lirq = event[eq->eq_idx] & ~XIVE_EQ_GEN_MASK; 439 KASSERT(lirq < XIVE_NUM_IRQS); 440 ih = sc->sc_handler[lirq]; 441 if (ih != NULL) { 442 intr_enable(); 443 xive_run_handler(ih); 444 intr_disable(); 445 xive_eoi(sc, ih); 446 } 447 eq->eq_idx = (eq->eq_idx + 1) & XIVE_EQ_IDX_MASK; 448 449 /* Toggle generation on wrap around. */ 450 if (eq->eq_idx == 0) 451 eq->eq_gen ^= XIVE_EQ_GEN_MASK; 452 } 453 454 ci->ci_cpl = old; 455 spurious: 456 xive_write_1(sc, XIVE_TM_CPPR_HV, xive_prio(old)); 457 eieio(); 458 } 459 } 460 461 struct xive_dmamem * 462 xive_dmamem_alloc(bus_dma_tag_t dmat, bus_size_t size, bus_size_t align) 463 { 464 struct xive_dmamem *xdm; 465 int nsegs; 466 467 xdm = malloc(sizeof(*xdm), M_DEVBUF, M_WAITOK | M_ZERO); 468 xdm->xdm_size = size; 469 470 if (bus_dmamap_create(dmat, size, 1, size, 0, 471 BUS_DMA_WAITOK | BUS_DMA_ALLOCNOW, &xdm->xdm_map) != 0) 472 goto xdmfree; 473 474 if (bus_dmamem_alloc(dmat, size, align, 0, &xdm->xdm_seg, 1, 475 &nsegs, BUS_DMA_WAITOK | BUS_DMA_ZERO) != 0) 476 goto destroy; 477 478 if (bus_dmamem_map(dmat, &xdm->xdm_seg, nsegs, size, 479 &xdm->xdm_kva, BUS_DMA_WAITOK | BUS_DMA_NOCACHE) != 0) 480 goto free; 481 482 if (bus_dmamap_load_raw(dmat, xdm->xdm_map, &xdm->xdm_seg, 483 nsegs, size, BUS_DMA_WAITOK) != 0) 484 goto unmap; 485 486 return xdm; 487 488 unmap: 489 bus_dmamem_unmap(dmat, xdm->xdm_kva, size); 490 free: 491 bus_dmamem_free(dmat, &xdm->xdm_seg, 1); 492 destroy: 493 bus_dmamap_destroy(dmat, xdm->xdm_map); 494 xdmfree: 495 free(xdm, M_DEVBUF, sizeof(*xdm)); 496 497 return NULL; 498 } 499 500 void 501 xive_dmamem_free(bus_dma_tag_t dmat, struct xive_dmamem *xdm) 502 { 503 bus_dmamem_unmap(dmat, xdm->xdm_kva, xdm->xdm_size); 504 bus_dmamem_free(dmat, &xdm->xdm_seg, 1); 505 bus_dmamap_destroy(dmat, xdm->xdm_map); 506 free(xdm, M_DEVBUF, sizeof(*xdm)); 507 } 508