1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 2019 Justin Hibbits 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_platform.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/module.h> 36 #include <sys/bus.h> 37 #include <sys/conf.h> 38 #include <sys/endian.h> 39 #include <sys/kernel.h> 40 #include <sys/malloc.h> 41 #include <sys/smp.h> 42 43 #include <vm/vm.h> 44 #include <vm/pmap.h> 45 46 #include <machine/bus.h> 47 #include <machine/intr_machdep.h> 48 #include <machine/md_var.h> 49 50 #include <dev/ofw/ofw_bus.h> 51 #include <dev/ofw/ofw_bus_subr.h> 52 53 #ifdef POWERNV 54 #include <powerpc/powernv/opal.h> 55 #endif 56 57 #include "pic_if.h" 58 59 #define XIVE_PRIORITY 7 /* Random non-zero number */ 60 #define MAX_XIVE_IRQS (1<<24) /* 24-bit XIRR field */ 61 62 /* Registers */ 63 #define XIVE_TM_QW1_OS 0x010 /* Guest OS registers */ 64 #define XIVE_TM_QW2_HV_POOL 0x020 /* Hypervisor pool registers */ 65 #define XIVE_TM_QW3_HV 0x030 /* Hypervisor registers */ 66 67 #define XIVE_TM_NSR 0x00 68 #define XIVE_TM_CPPR 0x01 69 #define XIVE_TM_IPB 0x02 70 #define XIVE_TM_LSMFB 0x03 71 #define XIVE_TM_ACK_CNT 0x04 72 #define XIVE_TM_INC 0x05 73 #define XIVE_TM_AGE 0x06 74 #define XIVE_TM_PIPR 0x07 75 76 #define TM_WORD0 0x0 77 #define TM_WORD2 0x8 78 #define TM_QW2W2_VP 0x80000000 79 80 #define XIVE_TM_SPC_ACK 0x800 81 #define TM_QW3NSR_HE_SHIFT 14 82 #define TM_QW3_NSR_HE_NONE 0 83 #define TM_QW3_NSR_HE_POOL 1 84 #define TM_QW3_NSR_HE_PHYS 2 85 #define TM_QW3_NSR_HE_LSI 3 86 #define XIVE_TM_SPC_PULL_POOL_CTX 0x828 87 88 #define XIVE_IRQ_LOAD_EOI 0x000 89 #define XIVE_IRQ_STORE_EOI 0x400 90 #define XIVE_IRQ_PQ_00 0xc00 91 #define XIVE_IRQ_PQ_01 0xd00 92 93 #define XIVE_IRQ_VAL_P 0x02 94 #define XIVE_IRQ_VAL_Q 0x01 95 96 struct xive_softc; 97 struct xive_irq; 98 99 extern void (*powernv_smp_ap_extra_init)(void); 100 101 /* Private support */ 102 static void xive_setup_cpu(void); 103 static void xive_smp_cpu_startup(void); 104 static void xive_init_irq(struct xive_irq *irqd, u_int irq); 105 static struct xive_irq *xive_configure_irq(u_int irq); 106 static int xive_provision_page(struct xive_softc *sc); 107 108 109 /* Interfaces */ 110 static int xive_probe(device_t); 111 static int xive_attach(device_t); 112 static int xics_probe(device_t); 113 static int xics_attach(device_t); 114 115 static void xive_bind(device_t, u_int, cpuset_t, void **); 116 static void xive_dispatch(device_t, struct trapframe *); 117 static void xive_enable(device_t, u_int, u_int, void **); 118 static void xive_eoi(device_t, u_int, void *); 119 static void xive_ipi(device_t, u_int); 120 static void xive_mask(device_t, u_int, void *); 121 static void xive_unmask(device_t, u_int, void *); 122 static void xive_translate_code(device_t dev, u_int irq, int code, 123 enum intr_trigger *trig, enum intr_polarity *pol); 124 125 static device_method_t xive_methods[] = { 126 /* Device interface */ 127 DEVMETHOD(device_probe, xive_probe), 128 DEVMETHOD(device_attach, xive_attach), 129 130 /* PIC interface */ 131 DEVMETHOD(pic_bind, xive_bind), 132 DEVMETHOD(pic_dispatch, xive_dispatch), 133 DEVMETHOD(pic_enable, xive_enable), 134 DEVMETHOD(pic_eoi, xive_eoi), 135 DEVMETHOD(pic_ipi, xive_ipi), 136 DEVMETHOD(pic_mask, xive_mask), 137 DEVMETHOD(pic_unmask, xive_unmask), 138 DEVMETHOD(pic_translate_code, xive_translate_code), 139 140 DEVMETHOD_END 141 }; 142 143 static device_method_t xics_methods[] = { 144 /* Device interface */ 145 DEVMETHOD(device_probe, xics_probe), 146 DEVMETHOD(device_attach, xics_attach), 147 148 DEVMETHOD_END 149 }; 150 151 struct xive_softc { 152 struct mtx sc_mtx; 153 struct resource *sc_mem; 154 vm_size_t sc_prov_page_size; 155 uint32_t sc_offset; 156 }; 157 158 struct xive_queue { 159 uint32_t *q_page; 160 uint32_t *q_eoi_page; 161 uint32_t q_toggle; 162 uint32_t q_size; 163 uint32_t q_index; 164 uint32_t q_mask; 165 }; 166 167 struct xive_irq { 168 uint32_t girq; 169 uint32_t lirq; 170 uint64_t vp; 171 uint64_t flags; 172 #define OPAL_XIVE_IRQ_EOI_VIA_FW 0x00000020 173 #define OPAL_XIVE_IRQ_MASK_VIA_FW 0x00000010 174 #define OPAL_XIVE_IRQ_SHIFT_BUG 0x00000008 175 #define OPAL_XIVE_IRQ_LSI 0x00000004 176 #define OPAL_XIVE_IRQ_STORE_EOI 0x00000002 177 #define OPAL_XIVE_IRQ_TRIGGER_PAGE 0x00000001 178 uint8_t prio; 179 vm_offset_t eoi_page; 180 vm_offset_t trig_page; 181 vm_size_t esb_size; 182 int chip; 183 }; 184 185 struct xive_cpu { 186 uint64_t vp; 187 uint64_t flags; 188 struct xive_irq ipi_data; 189 struct xive_queue queue; /* We only use a single queue for now. */ 190 uint64_t cam; 191 uint32_t chip; 192 }; 193 194 static driver_t xive_driver = { 195 "xive", 196 xive_methods, 197 sizeof(struct xive_softc) 198 }; 199 200 static driver_t xics_driver = { 201 "xivevc", 202 xics_methods, 203 0 204 }; 205 206 static devclass_t xive_devclass; 207 static devclass_t xics_devclass; 208 209 EARLY_DRIVER_MODULE(xive, ofwbus, xive_driver, xive_devclass, 0, 0, 210 BUS_PASS_INTERRUPT-1); 211 EARLY_DRIVER_MODULE(xivevc, ofwbus, xics_driver, xics_devclass, 0, 0, 212 BUS_PASS_INTERRUPT); 213 214 MALLOC_DEFINE(M_XIVE, "xive", "XIVE Memory"); 215 216 DPCPU_DEFINE_STATIC(struct xive_cpu, xive_cpu_data); 217 218 static int xive_ipi_vector = -1; 219 220 /* 221 * XIVE Exploitation mode driver. 222 * 223 * The XIVE, present in the POWER9 CPU, can run in two modes: XICS emulation 224 * mode, and "Exploitation mode". XICS emulation mode is compatible with the 225 * POWER8 and earlier XICS interrupt controller, using OPAL calls to emulate 226 * hypervisor calls and memory accesses. Exploitation mode gives us raw access 227 * to the XIVE MMIO, improving performance significantly. 228 * 229 * The XIVE controller is a very bizarre interrupt controller. It uses queues 230 * in memory to pass interrupts around, and maps itself into 512GB of physical 231 * device address space, giving each interrupt in the system one or more pages 232 * of address space. An IRQ is tied to a virtual processor, which could be a 233 * physical CPU thread, or a guest CPU thread (LPAR running on a physical 234 * thread). Thus, the controller can route interrupts directly to guest OSes 235 * bypassing processing by the hypervisor, thereby improving performance of the 236 * guest OS. 237 * 238 * An IRQ, in addition to being tied to a virtual processor, has one or two 239 * page mappings: an EOI page, and an optional trigger page. The trigger page 240 * could be the same as the EOI page. Level-sensitive interrupts (LSIs) don't 241 * have a trigger page, as they're external interrupts controlled by physical 242 * lines. MSIs and IPIs have trigger pages. An IPI is really just another IRQ 243 * in the XIVE, which is triggered by software. 244 * 245 * An interesting behavior of the XIVE controller is that oftentimes the 246 * contents of an address location don't actually matter, but the direction of 247 * the action is the signifier (read vs write), and the address is significant. 248 * Hence, masking and unmasking an interrupt is done by reading different 249 * addresses in the EOI page, and triggering an interrupt consists of writing to 250 * the trigger page. 251 * 252 * Additionally, the MMIO region mapped is CPU-sensitive, just like the 253 * per-processor register space (private access) in OpenPIC. In order for a CPU 254 * to receive interrupts it must itself configure its CPPR (Current Processor 255 * Priority Register), it cannot be set by any other processor. This 256 * necessitates the xive_smp_cpu_startup() function. 257 * 258 * Queues are pages of memory, sized powers-of-two, that are shared with the 259 * XIVE. The XIVE writes into the queue with an alternating polarity bit, which 260 * flips when the queue wraps. 261 */ 262 263 /* 264 * Offset-based read/write interfaces. 265 */ 266 static uint16_t 267 xive_read_2(struct xive_softc *sc, bus_size_t offset) 268 { 269 270 return (bus_read_2(sc->sc_mem, sc->sc_offset + offset)); 271 } 272 273 static void 274 xive_write_1(struct xive_softc *sc, bus_size_t offset, uint8_t val) 275 { 276 277 bus_write_1(sc->sc_mem, sc->sc_offset + offset, val); 278 } 279 280 /* EOI and Trigger page access interfaces. */ 281 static uint64_t 282 xive_read_mmap8(vm_offset_t addr) 283 { 284 return (*(volatile uint64_t *)addr); 285 } 286 287 static void 288 xive_write_mmap8(vm_offset_t addr, uint64_t val) 289 { 290 *(uint64_t *)(addr) = val; 291 } 292 293 294 /* Device interfaces. */ 295 static int 296 xive_probe(device_t dev) 297 { 298 299 if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-pe")) 300 return (ENXIO); 301 302 device_set_desc(dev, "External Interrupt Virtualization Engine"); 303 304 /* Make sure we always win against the xicp driver. */ 305 return (BUS_PROBE_DEFAULT); 306 } 307 308 static int 309 xics_probe(device_t dev) 310 { 311 312 if (!ofw_bus_is_compatible(dev, "ibm,opal-xive-vc")) 313 return (ENXIO); 314 315 device_set_desc(dev, "External Interrupt Virtualization Engine Root"); 316 return (BUS_PROBE_DEFAULT); 317 } 318 319 static int 320 xive_attach(device_t dev) 321 { 322 struct xive_softc *sc = device_get_softc(dev); 323 struct xive_cpu *xive_cpud; 324 phandle_t phandle = ofw_bus_get_node(dev); 325 int64_t vp_block; 326 int error; 327 int rid; 328 int i, order; 329 uint64_t vp_id; 330 int64_t ipi_irq; 331 332 opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EXP); 333 334 error = OF_getencprop(phandle, "ibm,xive-provision-page-size", 335 (pcell_t *)&sc->sc_prov_page_size, sizeof(sc->sc_prov_page_size)); 336 337 rid = 1; /* Get the Hypervisor-level register set. */ 338 sc->sc_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 339 &rid, RF_ACTIVE); 340 sc->sc_offset = XIVE_TM_QW3_HV; 341 342 mtx_init(&sc->sc_mtx, "XIVE", NULL, MTX_DEF); 343 344 order = fls(mp_maxid + (mp_maxid - 1)) - 1; 345 346 do { 347 vp_block = opal_call(OPAL_XIVE_ALLOCATE_VP_BLOCK, order); 348 if (vp_block == OPAL_BUSY) 349 DELAY(10); 350 else if (vp_block == OPAL_XIVE_PROVISIONING) 351 xive_provision_page(sc); 352 else 353 break; 354 } while (1); 355 356 if (vp_block < 0) { 357 device_printf(dev, 358 "Unable to allocate VP block. Opal error %d\n", 359 (int)vp_block); 360 bus_release_resource(dev, SYS_RES_MEMORY, rid, sc->sc_mem); 361 return (ENXIO); 362 } 363 364 /* 365 * Set up the VPs. Try to do as much as we can in attach, to lessen 366 * what's needed at AP spawn time. 367 */ 368 CPU_FOREACH(i) { 369 vp_id = pcpu_find(i)->pc_hwref; 370 371 xive_cpud = DPCPU_ID_PTR(i, xive_cpu_data); 372 xive_cpud->vp = vp_id + vp_block; 373 opal_call(OPAL_XIVE_GET_VP_INFO, xive_cpud->vp, NULL, 374 vtophys(&xive_cpud->cam), NULL, vtophys(&xive_cpud->chip)); 375 376 /* Allocate the queue page and populate the queue state data. */ 377 xive_cpud->queue.q_page = contigmalloc(PAGE_SIZE, M_XIVE, 378 M_ZERO | M_WAITOK, 0, BUS_SPACE_MAXADDR, PAGE_SIZE, 0); 379 xive_cpud->queue.q_size = 1 << PAGE_SHIFT; 380 xive_cpud->queue.q_mask = 381 ((xive_cpud->queue.q_size / sizeof(int)) - 1); 382 xive_cpud->queue.q_toggle = 0; 383 xive_cpud->queue.q_index = 0; 384 do { 385 error = opal_call(OPAL_XIVE_SET_VP_INFO, xive_cpud->vp, 386 OPAL_XIVE_VP_ENABLED, 0); 387 } while (error == OPAL_BUSY); 388 error = opal_call(OPAL_XIVE_SET_QUEUE_INFO, vp_id, 389 XIVE_PRIORITY, vtophys(xive_cpud->queue.q_page), PAGE_SHIFT, 390 OPAL_XIVE_EQ_ALWAYS_NOTIFY | OPAL_XIVE_EQ_ENABLED); 391 392 do { 393 ipi_irq = opal_call(OPAL_XIVE_ALLOCATE_IRQ, 394 xive_cpud->chip); 395 } while (ipi_irq == OPAL_BUSY); 396 397 if (ipi_irq < 0) 398 device_printf(root_pic, 399 "Failed allocating IPI. OPAL error %d\n", 400 (int)ipi_irq); 401 else { 402 xive_init_irq(&xive_cpud->ipi_data, ipi_irq); 403 xive_cpud->ipi_data.vp = vp_id; 404 xive_cpud->ipi_data.lirq = MAX_XIVE_IRQS; 405 opal_call(OPAL_XIVE_SET_IRQ_CONFIG, ipi_irq, 406 xive_cpud->ipi_data.vp, XIVE_PRIORITY, 407 MAX_XIVE_IRQS); 408 } 409 } 410 411 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XIVE_IRQS, 412 1 /* Number of IPIs */, FALSE); 413 root_pic = dev; 414 415 xive_setup_cpu(); 416 powernv_smp_ap_extra_init = xive_smp_cpu_startup; 417 418 return (0); 419 } 420 421 static int 422 xics_attach(device_t dev) 423 { 424 phandle_t phandle = ofw_bus_get_node(dev); 425 426 /* The XIVE (root PIC) will handle all our interrupts */ 427 powerpc_register_pic(root_pic, OF_xref_from_node(phandle), 428 MAX_XIVE_IRQS, 1 /* Number of IPIs */, FALSE); 429 430 return (0); 431 } 432 433 /* 434 * PIC I/F methods. 435 */ 436 437 static void 438 xive_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv) 439 { 440 struct xive_irq *irqd; 441 int cpu; 442 int ncpus, i, error; 443 444 if (*priv == NULL) 445 *priv = xive_configure_irq(irq); 446 447 irqd = *priv; 448 449 /* 450 * This doesn't appear to actually support affinity groups, so pick a 451 * random CPU. 452 */ 453 ncpus = 0; 454 CPU_FOREACH(cpu) 455 if (CPU_ISSET(cpu, &cpumask)) ncpus++; 456 457 i = mftb() % ncpus; 458 ncpus = 0; 459 CPU_FOREACH(cpu) { 460 if (!CPU_ISSET(cpu, &cpumask)) 461 continue; 462 if (ncpus == i) 463 break; 464 ncpus++; 465 } 466 467 opal_call(OPAL_XIVE_SYNC); 468 469 irqd->vp = pcpu_find(cpu)->pc_hwref; 470 error = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, irqd->vp, 471 XIVE_PRIORITY, irqd->lirq); 472 473 if (error < 0) 474 panic("Cannot bind interrupt %d to CPU %d", irq, cpu); 475 476 xive_eoi(dev, irq, irqd); 477 } 478 479 /* Read the next entry in the queue page and update the index. */ 480 static int 481 xive_read_eq(struct xive_queue *q) 482 { 483 uint32_t i = be32toh(q->q_page[q->q_index]); 484 485 /* Check validity, using current queue polarity. */ 486 if ((i >> 31) == q->q_toggle) 487 return (0); 488 489 q->q_index = (q->q_index + 1) & q->q_mask; 490 491 if (q->q_index == 0) 492 q->q_toggle ^= 1; 493 494 return (i & 0x7fffffff); 495 } 496 497 static void 498 xive_dispatch(device_t dev, struct trapframe *tf) 499 { 500 struct xive_softc *sc; 501 struct xive_cpu *xive_cpud; 502 uint32_t vector; 503 uint16_t ack; 504 uint8_t cppr, he; 505 506 sc = device_get_softc(dev); 507 508 for (;;) { 509 ack = xive_read_2(sc, XIVE_TM_SPC_ACK); 510 cppr = (ack & 0xff); 511 512 he = ack >> TM_QW3NSR_HE_SHIFT; 513 514 if (he == TM_QW3_NSR_HE_NONE) 515 break; 516 switch (he) { 517 case TM_QW3_NSR_HE_NONE: 518 goto end; 519 case TM_QW3_NSR_HE_POOL: 520 case TM_QW3_NSR_HE_LSI: 521 device_printf(dev, 522 "Unexpected interrupt he type: %d\n", he); 523 goto end; 524 case TM_QW3_NSR_HE_PHYS: 525 break; 526 } 527 528 xive_cpud = DPCPU_PTR(xive_cpu_data); 529 xive_write_1(sc, XIVE_TM_CPPR, cppr); 530 531 for (;;) { 532 vector = xive_read_eq(&xive_cpud->queue); 533 534 if (vector == 0) 535 break; 536 537 if (vector == MAX_XIVE_IRQS) 538 vector = xive_ipi_vector; 539 540 powerpc_dispatch_intr(vector, tf); 541 } 542 } 543 end: 544 xive_write_1(sc, XIVE_TM_CPPR, 0xff); 545 } 546 547 static void 548 xive_enable(device_t dev, u_int irq, u_int vector, void **priv) 549 { 550 struct xive_irq *irqd; 551 cell_t status, cpu; 552 553 if (irq == MAX_XIVE_IRQS) { 554 if (xive_ipi_vector == -1) 555 xive_ipi_vector = vector; 556 return; 557 } 558 if (*priv == NULL) 559 *priv = xive_configure_irq(irq); 560 561 irqd = *priv; 562 563 /* Bind to this CPU to start */ 564 cpu = PCPU_GET(hwref); 565 irqd->lirq = vector; 566 567 for (;;) { 568 status = opal_call(OPAL_XIVE_SET_IRQ_CONFIG, irq, cpu, 569 XIVE_PRIORITY, vector); 570 if (status != OPAL_BUSY) 571 break; 572 DELAY(10); 573 } 574 575 if (status != 0) 576 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq, 577 cpu, status); 578 579 xive_unmask(dev, irq, *priv); 580 } 581 582 static void 583 xive_eoi(device_t dev, u_int irq, void *priv) 584 { 585 struct xive_irq *rirq; 586 struct xive_cpu *cpud; 587 uint8_t eoi_val; 588 589 if (irq == MAX_XIVE_IRQS) { 590 cpud = DPCPU_PTR(xive_cpu_data); 591 rirq = &cpud->ipi_data; 592 } else 593 rirq = priv; 594 595 if (rirq->flags & OPAL_XIVE_IRQ_EOI_VIA_FW) 596 opal_call(OPAL_INT_EOI, irq); 597 else if (rirq->flags & OPAL_XIVE_IRQ_STORE_EOI) 598 xive_write_mmap8(rirq->eoi_page + XIVE_IRQ_STORE_EOI, 0); 599 else if (rirq->flags & OPAL_XIVE_IRQ_LSI) 600 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_LOAD_EOI); 601 else { 602 eoi_val = xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00); 603 if ((eoi_val & XIVE_IRQ_VAL_Q) && rirq->trig_page != 0) 604 xive_write_mmap8(rirq->trig_page, 0); 605 } 606 } 607 608 static void 609 xive_ipi(device_t dev, u_int cpu) 610 { 611 struct xive_cpu *xive_cpud; 612 613 xive_cpud = DPCPU_ID_PTR(cpu, xive_cpu_data); 614 615 if (xive_cpud->ipi_data.trig_page == 0) 616 return; 617 xive_write_mmap8(xive_cpud->ipi_data.trig_page, 0); 618 } 619 620 static void 621 xive_mask(device_t dev, u_int irq, void *priv) 622 { 623 struct xive_irq *rirq; 624 625 /* Never mask IPIs */ 626 if (irq == MAX_XIVE_IRQS) 627 return; 628 629 rirq = priv; 630 631 if (!(rirq->flags & OPAL_XIVE_IRQ_LSI)) 632 return; 633 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_01); 634 } 635 636 static void 637 xive_unmask(device_t dev, u_int irq, void *priv) 638 { 639 struct xive_irq *rirq; 640 641 rirq = priv; 642 643 xive_read_mmap8(rirq->eoi_page + XIVE_IRQ_PQ_00); 644 } 645 646 static void 647 xive_translate_code(device_t dev, u_int irq, int code, 648 enum intr_trigger *trig, enum intr_polarity *pol) 649 { 650 switch (code) { 651 case 0: 652 /* L to H edge */ 653 *trig = INTR_TRIGGER_EDGE; 654 *pol = INTR_POLARITY_HIGH; 655 break; 656 case 1: 657 /* Active L level */ 658 *trig = INTR_TRIGGER_LEVEL; 659 *pol = INTR_POLARITY_LOW; 660 break; 661 default: 662 *trig = INTR_TRIGGER_CONFORM; 663 *pol = INTR_POLARITY_CONFORM; 664 } 665 } 666 667 /* Private functions. */ 668 /* 669 * Setup the current CPU. Called by the BSP at driver attachment, and by each 670 * AP at wakeup (via xive_smp_cpu_startup()). 671 */ 672 static void 673 xive_setup_cpu(void) 674 { 675 struct xive_softc *sc; 676 struct xive_cpu *cpup; 677 uint32_t val; 678 679 cpup = DPCPU_PTR(xive_cpu_data); 680 681 sc = device_get_softc(root_pic); 682 683 val = bus_read_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2); 684 if (val & TM_QW2W2_VP) 685 bus_read_8(sc->sc_mem, XIVE_TM_SPC_PULL_POOL_CTX); 686 687 bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD0, 0xff); 688 bus_write_4(sc->sc_mem, XIVE_TM_QW2_HV_POOL + TM_WORD2, 689 TM_QW2W2_VP | cpup->cam); 690 691 xive_unmask(root_pic, cpup->ipi_data.girq, &cpup->ipi_data); 692 xive_write_1(sc, XIVE_TM_CPPR, 0xff); 693 } 694 695 /* Populate an IRQ structure, mapping the EOI and trigger pages. */ 696 static void 697 xive_init_irq(struct xive_irq *irqd, u_int irq) 698 { 699 uint64_t eoi_phys, trig_phys; 700 uint32_t esb_shift; 701 702 opal_call(OPAL_XIVE_GET_IRQ_INFO, irq, 703 vtophys(&irqd->flags), vtophys(&eoi_phys), 704 vtophys(&trig_phys), vtophys(&esb_shift), 705 vtophys(&irqd->chip)); 706 707 irqd->girq = irq; 708 irqd->esb_size = 1 << esb_shift; 709 irqd->eoi_page = (vm_offset_t)pmap_mapdev(eoi_phys, irqd->esb_size); 710 711 if (eoi_phys == trig_phys) 712 irqd->trig_page = irqd->eoi_page; 713 else if (trig_phys != 0) 714 irqd->trig_page = (vm_offset_t)pmap_mapdev(trig_phys, 715 irqd->esb_size); 716 else 717 irqd->trig_page = 0; 718 719 opal_call(OPAL_XIVE_GET_IRQ_CONFIG, irq, vtophys(&irqd->vp), 720 vtophys(&irqd->prio), vtophys(&irqd->lirq)); 721 } 722 723 /* Allocate an IRQ struct before populating it. */ 724 static struct xive_irq * 725 xive_configure_irq(u_int irq) 726 { 727 struct xive_irq *irqd; 728 729 irqd = malloc(sizeof(struct xive_irq), M_XIVE, M_WAITOK); 730 731 xive_init_irq(irqd, irq); 732 733 return (irqd); 734 } 735 736 /* 737 * Part of the OPAL API. OPAL_XIVE_ALLOCATE_VP_BLOCK might require more pages, 738 * provisioned through this call. 739 */ 740 static int 741 xive_provision_page(struct xive_softc *sc) 742 { 743 void *prov_page; 744 int error; 745 746 do { 747 prov_page = contigmalloc(sc->sc_prov_page_size, M_XIVE, 0, 748 0, BUS_SPACE_MAXADDR, 749 sc->sc_prov_page_size, sc->sc_prov_page_size); 750 751 error = opal_call(OPAL_XIVE_DONATE_PAGE, -1, 752 vtophys(prov_page)); 753 } while (error == OPAL_XIVE_PROVISIONING); 754 755 return (0); 756 } 757 758 /* The XIVE_TM_CPPR register must be set by each thread */ 759 static void 760 xive_smp_cpu_startup(void) 761 { 762 763 xive_setup_cpu(); 764 } 765