1 /*- 2 * SPDX-License-Identifier: BSD-2-Clause-FreeBSD 3 * 4 * Copyright 2011 Nathan Whitehorn 5 * 6 * Redistribution and use in source and binary forms, with or without 7 * modification, are permitted provided that the following conditions 8 * are met: 9 * 1. Redistributions of source code must retain the above copyright 10 * notice, this list of conditions and the following disclaimer. 11 * 2. Redistributions in binary form must reproduce the above copyright 12 * notice, this list of conditions and the following disclaimer in the 13 * documentation and/or other materials provided with the distribution. 14 * 15 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 16 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 17 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 18 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 19 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 20 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 21 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 22 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 23 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 24 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 25 * SUCH DAMAGE. 26 */ 27 28 #include <sys/cdefs.h> 29 __FBSDID("$FreeBSD$"); 30 31 #include "opt_platform.h" 32 33 #include <sys/param.h> 34 #include <sys/systm.h> 35 #include <sys/module.h> 36 #include <sys/bus.h> 37 #include <sys/conf.h> 38 #include <sys/kernel.h> 39 #include <sys/malloc.h> 40 #include <sys/smp.h> 41 42 #include <vm/vm.h> 43 #include <vm/pmap.h> 44 45 #include <machine/bus.h> 46 #include <machine/intr_machdep.h> 47 #include <machine/md_var.h> 48 #include <machine/rtas.h> 49 50 #include <dev/ofw/ofw_bus.h> 51 #include <dev/ofw/ofw_bus_subr.h> 52 53 #ifdef POWERNV 54 #include <powerpc/powernv/opal.h> 55 #endif 56 57 #include "phyp-hvcall.h" 58 #include "pic_if.h" 59 60 #define XICP_PRIORITY 5 /* Random non-zero number */ 61 #define XICP_IPI 2 62 #define MAX_XICP_IRQS (1<<24) /* 24-bit XIRR field */ 63 64 static int xicp_probe(device_t); 65 static int xicp_attach(device_t); 66 static int xics_probe(device_t); 67 static int xics_attach(device_t); 68 69 static void xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv); 70 static void xicp_dispatch(device_t, struct trapframe *); 71 static void xicp_enable(device_t, u_int, u_int, void **priv); 72 static void xicp_eoi(device_t, u_int, void *priv); 73 static void xicp_ipi(device_t, u_int); 74 static void xicp_mask(device_t, u_int, void *priv); 75 static void xicp_unmask(device_t, u_int, void *priv); 76 77 #ifdef POWERNV 78 extern void (*powernv_smp_ap_extra_init)(void); 79 static void xicp_smp_cpu_startup(void); 80 #endif 81 82 static device_method_t xicp_methods[] = { 83 /* Device interface */ 84 DEVMETHOD(device_probe, xicp_probe), 85 DEVMETHOD(device_attach, xicp_attach), 86 87 /* PIC interface */ 88 DEVMETHOD(pic_bind, xicp_bind), 89 DEVMETHOD(pic_dispatch, xicp_dispatch), 90 DEVMETHOD(pic_enable, xicp_enable), 91 DEVMETHOD(pic_eoi, xicp_eoi), 92 DEVMETHOD(pic_ipi, xicp_ipi), 93 DEVMETHOD(pic_mask, xicp_mask), 94 DEVMETHOD(pic_unmask, xicp_unmask), 95 96 DEVMETHOD_END 97 }; 98 99 static device_method_t xics_methods[] = { 100 /* Device interface */ 101 DEVMETHOD(device_probe, xics_probe), 102 DEVMETHOD(device_attach, xics_attach), 103 104 DEVMETHOD_END 105 }; 106 107 struct xicp_intvec { 108 int irq; 109 int vector; 110 int cpu; 111 }; 112 113 struct xicp_softc { 114 struct mtx sc_mtx; 115 struct resource *mem[MAXCPU]; 116 117 int cpu_range[2]; 118 119 int ibm_int_on; 120 int ibm_int_off; 121 int ibm_get_xive; 122 int ibm_set_xive; 123 124 /* XXX: inefficient -- hash table? tree? */ 125 struct xicp_intvec intvecs[256]; 126 int nintvecs; 127 int ipi_vec; 128 bool xics_emu; 129 }; 130 131 static driver_t xicp_driver = { 132 "xicp", 133 xicp_methods, 134 sizeof(struct xicp_softc) 135 }; 136 137 static driver_t xics_driver = { 138 "xics", 139 xics_methods, 140 0 141 }; 142 143 #ifdef POWERNV 144 /* We can only pass physical addresses into OPAL. Kernel stacks are in the KVA, 145 * not in the direct map, so we need to somehow extract the physical address. 146 * However, pmap_kextract() takes locks, which is forbidden in a critical region 147 * (which PIC_DISPATCH() operates in). The kernel is mapped into the Direct 148 * Map (0xc000....), and the CPU implicitly drops the top two bits when doing 149 * real address by nature that the bus width is smaller than 64-bits. Placing 150 * cpu_xirr into the DMAP lets us take advantage of this and avoids the 151 * pmap_kextract() that would otherwise be needed if using the stack variable. 152 */ 153 static uint32_t cpu_xirr[MAXCPU]; 154 #endif 155 156 static devclass_t xicp_devclass; 157 static devclass_t xics_devclass; 158 159 EARLY_DRIVER_MODULE(xicp, ofwbus, xicp_driver, xicp_devclass, 0, 0, 160 BUS_PASS_INTERRUPT-1); 161 EARLY_DRIVER_MODULE(xics, ofwbus, xics_driver, xics_devclass, 0, 0, 162 BUS_PASS_INTERRUPT); 163 164 #ifdef POWERNV 165 static struct resource * 166 xicp_mem_for_cpu(int cpu) 167 { 168 device_t dev; 169 struct xicp_softc *sc; 170 int i; 171 172 for (i = 0; (dev = devclass_get_device(xicp_devclass, i)) != NULL; i++){ 173 sc = device_get_softc(dev); 174 if (cpu >= sc->cpu_range[0] && cpu < sc->cpu_range[1]) 175 return (sc->mem[cpu - sc->cpu_range[0]]); 176 } 177 178 return (NULL); 179 } 180 #endif 181 182 static int 183 xicp_probe(device_t dev) 184 { 185 186 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xicp") && 187 !ofw_bus_is_compatible(dev, "ibm,opal-intc")) 188 return (ENXIO); 189 190 device_set_desc(dev, "External Interrupt Presentation Controller"); 191 return (BUS_PROBE_GENERIC); 192 } 193 194 static int 195 xics_probe(device_t dev) 196 { 197 198 if (!ofw_bus_is_compatible(dev, "ibm,ppc-xics") && 199 !ofw_bus_is_compatible(dev, "IBM,opal-xics")) 200 return (ENXIO); 201 202 device_set_desc(dev, "External Interrupt Source Controller"); 203 return (BUS_PROBE_GENERIC); 204 } 205 206 static int 207 xicp_attach(device_t dev) 208 { 209 struct xicp_softc *sc = device_get_softc(dev); 210 phandle_t phandle = ofw_bus_get_node(dev); 211 212 if (rtas_exists()) { 213 sc->ibm_int_on = rtas_token_lookup("ibm,int-on"); 214 sc->ibm_int_off = rtas_token_lookup("ibm,int-off"); 215 sc->ibm_set_xive = rtas_token_lookup("ibm,set-xive"); 216 sc->ibm_get_xive = rtas_token_lookup("ibm,get-xive"); 217 #ifdef POWERNV 218 } else if (opal_check() == 0) { 219 /* No init needed */ 220 #endif 221 } else { 222 device_printf(dev, "Cannot attach without RTAS or OPAL\n"); 223 return (ENXIO); 224 } 225 226 if (OF_hasprop(phandle, "ibm,interrupt-server-ranges")) { 227 OF_getencprop(phandle, "ibm,interrupt-server-ranges", 228 sc->cpu_range, sizeof(sc->cpu_range)); 229 sc->cpu_range[1] += sc->cpu_range[0]; 230 device_printf(dev, "Handling CPUs %d-%d\n", sc->cpu_range[0], 231 sc->cpu_range[1]-1); 232 #ifdef POWERNV 233 } else if (ofw_bus_is_compatible(dev, "ibm,opal-intc")) { 234 /* 235 * For now run POWER9 XIVE interrupt controller in XICS 236 * compatibility mode. 237 */ 238 sc->xics_emu = true; 239 opal_call(OPAL_XIVE_RESET, OPAL_XIVE_XICS_MODE_EMU); 240 #endif 241 } else { 242 sc->cpu_range[0] = 0; 243 sc->cpu_range[1] = mp_ncpus; 244 } 245 246 #ifdef POWERNV 247 if (mfmsr() & PSL_HV) { 248 int i; 249 250 if (sc->xics_emu) { 251 opal_call(OPAL_INT_SET_CPPR, 0xff); 252 for (i = 0; i < mp_ncpus; i++) { 253 opal_call(OPAL_INT_SET_MFRR, 254 pcpu_find(i)->pc_hwref, 0xff); 255 } 256 } else { 257 for (i = 0; i < sc->cpu_range[1] - sc->cpu_range[0]; i++) { 258 sc->mem[i] = bus_alloc_resource_any(dev, SYS_RES_MEMORY, 259 &i, RF_ACTIVE); 260 if (sc->mem[i] == NULL) { 261 device_printf(dev, "Could not alloc mem " 262 "resource %d\n", i); 263 return (ENXIO); 264 } 265 266 /* Unmask interrupts on all cores */ 267 bus_write_1(sc->mem[i], 4, 0xff); 268 bus_write_1(sc->mem[i], 12, 0xff); 269 } 270 } 271 } 272 #endif 273 274 mtx_init(&sc->sc_mtx, "XICP", NULL, MTX_DEF); 275 sc->nintvecs = 0; 276 277 powerpc_register_pic(dev, OF_xref_from_node(phandle), MAX_XICP_IRQS, 278 1 /* Number of IPIs */, FALSE); 279 root_pic = dev; 280 281 #ifdef POWERNV 282 if (sc->xics_emu) 283 powernv_smp_ap_extra_init = xicp_smp_cpu_startup; 284 #endif 285 286 return (0); 287 } 288 289 static int 290 xics_attach(device_t dev) 291 { 292 phandle_t phandle = ofw_bus_get_node(dev); 293 294 /* The XICP (root PIC) will handle all our interrupts */ 295 powerpc_register_pic(root_pic, OF_xref_from_node(phandle), 296 MAX_XICP_IRQS, 1 /* Number of IPIs */, FALSE); 297 298 return (0); 299 } 300 301 static __inline struct xicp_intvec * 302 xicp_setup_priv(struct xicp_softc *sc, u_int irq, void **priv) 303 { 304 if (*priv == NULL) { 305 KASSERT(sc->nintvecs + 1 < nitems(sc->intvecs), 306 ("Too many XICP interrupts")); 307 mtx_lock(&sc->sc_mtx); 308 *priv = &sc->intvecs[sc->nintvecs++]; 309 mtx_unlock(&sc->sc_mtx); 310 } 311 312 return (*priv); 313 } 314 315 /* 316 * PIC I/F methods. 317 */ 318 319 static void 320 xicp_bind(device_t dev, u_int irq, cpuset_t cpumask, void **priv) 321 { 322 struct xicp_softc *sc = device_get_softc(dev); 323 struct xicp_intvec *iv; 324 cell_t status, cpu; 325 int ncpus, i, error; 326 327 /* Ignore IPIs */ 328 if (irq == MAX_XICP_IRQS) 329 return; 330 331 iv = xicp_setup_priv(sc, irq, priv); 332 333 /* 334 * This doesn't appear to actually support affinity groups, so pick a 335 * random CPU. 336 */ 337 ncpus = 0; 338 CPU_FOREACH(cpu) 339 if (CPU_ISSET(cpu, &cpumask)) ncpus++; 340 341 i = mftb() % ncpus; 342 ncpus = 0; 343 CPU_FOREACH(cpu) { 344 if (!CPU_ISSET(cpu, &cpumask)) 345 continue; 346 if (ncpus == i) 347 break; 348 ncpus++; 349 } 350 351 cpu = pcpu_find(cpu)->pc_hwref; 352 iv->cpu = cpu; 353 354 if (rtas_exists()) 355 error = rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 356 XICP_PRIORITY, &status); 357 #ifdef POWERNV 358 else 359 error = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 360 #endif 361 362 if (error < 0) 363 panic("Cannot bind interrupt %d to CPU %d", irq, cpu); 364 } 365 366 static void 367 xicp_dispatch(device_t dev, struct trapframe *tf) 368 { 369 struct xicp_softc *sc; 370 struct resource *regs = NULL; 371 uint64_t xirr, junk; 372 int i; 373 374 sc = device_get_softc(dev); 375 #ifdef POWERNV 376 if ((mfmsr() & PSL_HV) && !sc->xics_emu) { 377 regs = xicp_mem_for_cpu(PCPU_GET(hwref)); 378 KASSERT(regs != NULL, 379 ("Can't find regs for CPU %ld", (uintptr_t)PCPU_GET(hwref))); 380 } 381 #endif 382 383 for (;;) { 384 /* Return value in R4, use the PFT call */ 385 if (regs) { 386 xirr = bus_read_4(regs, 4); 387 #ifdef POWERNV 388 } else if (sc->xics_emu) { 389 opal_call(OPAL_INT_GET_XIRR, &cpu_xirr[PCPU_GET(cpuid)], 390 false); 391 xirr = cpu_xirr[PCPU_GET(cpuid)]; 392 #endif 393 } else { 394 /* Return value in R4, use the PFT call */ 395 phyp_pft_hcall(H_XIRR, 0, 0, 0, 0, &xirr, &junk, &junk); 396 } 397 xirr &= 0x00ffffff; 398 399 if (xirr == 0) /* No more pending interrupts? */ 400 break; 401 402 if (xirr == XICP_IPI) { /* Magic number for IPIs */ 403 xirr = MAX_XICP_IRQS; /* Map to FreeBSD magic */ 404 405 /* Clear IPI */ 406 if (regs) 407 bus_write_1(regs, 12, 0xff); 408 #ifdef POWERNV 409 else if (sc->xics_emu) 410 opal_call(OPAL_INT_SET_MFRR, 411 PCPU_GET(hwref), 0xff); 412 #endif 413 else 414 phyp_hcall(H_IPI, (uint64_t)(PCPU_GET(hwref)), 415 0xff); 416 i = sc->ipi_vec; 417 } else { 418 419 /* XXX: super inefficient */ 420 for (i = 0; i < sc->nintvecs; i++) { 421 if (sc->intvecs[i].irq == xirr) 422 break; 423 } 424 KASSERT(i < sc->nintvecs, ("Unmapped XIRR")); 425 } 426 427 powerpc_dispatch_intr(sc->intvecs[i].vector, tf); 428 } 429 } 430 431 static void 432 xicp_enable(device_t dev, u_int irq, u_int vector, void **priv) 433 { 434 struct xicp_softc *sc; 435 struct xicp_intvec *intr; 436 cell_t status, cpu; 437 438 sc = device_get_softc(dev); 439 440 /* Bind to this CPU to start: distrib. ID is last entry in gserver# */ 441 cpu = PCPU_GET(hwref); 442 443 intr = xicp_setup_priv(sc, irq, priv); 444 445 intr->irq = irq; 446 intr->vector = vector; 447 intr->cpu = cpu; 448 mb(); 449 450 /* IPIs are also enabled. Stash off the vector index */ 451 if (irq == MAX_XICP_IRQS) { 452 sc->ipi_vec = intr - sc->intvecs; 453 return; 454 } 455 456 if (rtas_exists()) { 457 rtas_call_method(sc->ibm_set_xive, 3, 1, irq, cpu, 458 XICP_PRIORITY, &status); 459 xicp_unmask(dev, irq, intr); 460 #ifdef POWERNV 461 } else { 462 status = opal_call(OPAL_SET_XIVE, irq, cpu << 2, XICP_PRIORITY); 463 /* Unmask implicit for OPAL */ 464 465 if (status != 0) 466 panic("OPAL_SET_XIVE IRQ %d -> cpu %d failed: %d", irq, 467 cpu, status); 468 #endif 469 } 470 } 471 472 static void 473 xicp_eoi(device_t dev, u_int irq, void *priv) 474 { 475 #ifdef POWERNV 476 struct xicp_softc *sc; 477 #endif 478 uint64_t xirr; 479 480 if (irq == MAX_XICP_IRQS) /* Remap IPI interrupt to internal value */ 481 irq = XICP_IPI; 482 xirr = irq | (0xff << 24); 483 484 #ifdef POWERNV 485 if (mfmsr() & PSL_HV) { 486 sc = device_get_softc(dev); 487 if (sc->xics_emu) 488 opal_call(OPAL_INT_EOI, xirr); 489 else 490 bus_write_4(xicp_mem_for_cpu(PCPU_GET(hwref)), 4, xirr); 491 } else 492 #endif 493 phyp_hcall(H_EOI, xirr); 494 } 495 496 static void 497 xicp_ipi(device_t dev, u_int cpu) 498 { 499 500 #ifdef POWERNV 501 struct xicp_softc *sc; 502 cpu = pcpu_find(cpu)->pc_hwref; 503 504 if (mfmsr() & PSL_HV) { 505 sc = device_get_softc(dev); 506 if (sc->xics_emu) { 507 int64_t rv; 508 rv = opal_call(OPAL_INT_SET_MFRR, cpu, XICP_PRIORITY); 509 if (rv != 0) 510 device_printf(dev, "IPI SET_MFRR result: %ld\n", rv); 511 } else 512 bus_write_1(xicp_mem_for_cpu(cpu), 12, XICP_PRIORITY); 513 } else 514 #endif 515 phyp_hcall(H_IPI, (uint64_t)cpu, XICP_PRIORITY); 516 } 517 518 static void 519 xicp_mask(device_t dev, u_int irq, void *priv) 520 { 521 struct xicp_softc *sc = device_get_softc(dev); 522 cell_t status; 523 524 if (irq == MAX_XICP_IRQS) 525 return; 526 527 if (rtas_exists()) { 528 rtas_call_method(sc->ibm_int_off, 1, 1, irq, &status); 529 #ifdef POWERNV 530 } else { 531 struct xicp_intvec *ivec = priv; 532 533 KASSERT(ivec != NULL, ("Masking unconfigured interrupt")); 534 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, 0xff); 535 #endif 536 } 537 } 538 539 static void 540 xicp_unmask(device_t dev, u_int irq, void *priv) 541 { 542 struct xicp_softc *sc = device_get_softc(dev); 543 cell_t status; 544 545 if (irq == MAX_XICP_IRQS) 546 return; 547 548 if (rtas_exists()) { 549 rtas_call_method(sc->ibm_int_on, 1, 1, irq, &status); 550 #ifdef POWERNV 551 } else { 552 struct xicp_intvec *ivec = priv; 553 554 KASSERT(ivec != NULL, ("Unmasking unconfigured interrupt")); 555 opal_call(OPAL_SET_XIVE, irq, ivec->cpu << 2, XICP_PRIORITY); 556 #endif 557 } 558 } 559 560 #ifdef POWERNV 561 /* This is only used on POWER9 systems with the XIVE's XICS emulation. */ 562 static void 563 xicp_smp_cpu_startup(void) 564 { 565 struct xicp_softc *sc; 566 567 if (mfmsr() & PSL_HV) { 568 sc = device_get_softc(root_pic); 569 570 if (sc->xics_emu) 571 opal_call(OPAL_INT_SET_CPPR, 0xff); 572 } 573 } 574 #endif 575