1 /*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Andrew Turner under 6 * the sponsorship of the FreeBSD Foundation. 7 * 8 * This software was developed by Semihalf under 9 * the sponsorship of the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include "opt_acpi.h" 34 #include "opt_platform.h" 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/bus.h> 42 #include <sys/cpuset.h> 43 #include <sys/endian.h> 44 #include <sys/kernel.h> 45 #include <sys/malloc.h> 46 #include <sys/module.h> 47 #include <sys/proc.h> 48 #include <sys/queue.h> 49 #include <sys/rman.h> 50 #include <sys/smp.h> 51 #include <sys/vmem.h> 52 53 #include <vm/vm.h> 54 #include <vm/pmap.h> 55 56 #include <machine/bus.h> 57 #include <machine/intr.h> 58 59 #include <arm/arm/gic_common.h> 60 #include <arm64/arm64/gic_v3_reg.h> 61 #include <arm64/arm64/gic_v3_var.h> 62 63 #ifdef FDT 64 #include <dev/ofw/openfirm.h> 65 #include <dev/ofw/ofw_bus.h> 66 #include <dev/ofw/ofw_bus_subr.h> 67 #endif 68 #include <dev/pci/pcireg.h> 69 #include <dev/pci/pcivar.h> 70 71 #include "pcib_if.h" 72 #include "pic_if.h" 73 #include "msi_if.h" 74 75 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 76 "ARM GICv3 Interrupt Translation Service"); 77 78 #define LPI_NIRQS (64 * 1024) 79 80 /* The size and alignment of the command circular buffer */ 81 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 82 #define ITS_CMDQ_ALIGN (64 * 1024) 83 84 #define LPI_CONFTAB_SIZE LPI_NIRQS 85 #define LPI_CONFTAB_ALIGN (64 * 1024) 86 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 87 88 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 89 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 90 #define LPI_PENDTAB_ALIGN (64 * 1024) 91 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 92 93 #define LPI_INT_TRANS_TAB_ALIGN 256 94 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 95 96 /* ITS commands encoding */ 97 #define ITS_CMD_MOVI (0x01) 98 #define ITS_CMD_SYNC (0x05) 99 #define ITS_CMD_MAPD (0x08) 100 #define ITS_CMD_MAPC (0x09) 101 #define ITS_CMD_MAPTI (0x0a) 102 #define ITS_CMD_MAPI (0x0b) 103 #define ITS_CMD_INV (0x0c) 104 #define ITS_CMD_INVALL (0x0d) 105 /* Command */ 106 #define CMD_COMMAND_MASK (0xFFUL) 107 /* PCI device ID */ 108 #define CMD_DEVID_SHIFT (32) 109 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 110 /* Size of IRQ ID bitfield */ 111 #define CMD_SIZE_MASK (0xFFUL) 112 /* Virtual LPI ID */ 113 #define CMD_ID_MASK (0xFFFFFFFFUL) 114 /* Physical LPI ID */ 115 #define CMD_PID_SHIFT (32) 116 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 117 /* Collection */ 118 #define CMD_COL_MASK (0xFFFFUL) 119 /* Target (CPU or Re-Distributor) */ 120 #define CMD_TARGET_SHIFT (16) 121 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 122 /* Interrupt Translation Table address */ 123 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 124 /* Valid command bit */ 125 #define CMD_VALID_SHIFT (63) 126 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 127 128 #define ITS_TARGET_NONE 0xFBADBEEF 129 130 /* LPI chunk owned by ITS device */ 131 struct lpi_chunk { 132 u_int lpi_base; 133 u_int lpi_free; /* First free LPI in set */ 134 u_int lpi_num; /* Total number of LPIs in chunk */ 135 u_int lpi_busy; /* Number of busy LPIs in chink */ 136 }; 137 138 /* ITS device */ 139 struct its_dev { 140 TAILQ_ENTRY(its_dev) entry; 141 /* PCI device */ 142 device_t pci_dev; 143 /* Device ID (i.e. PCI device ID) */ 144 uint32_t devid; 145 /* List of assigned LPIs */ 146 struct lpi_chunk lpis; 147 /* Virtual address of ITT */ 148 vm_offset_t itt; 149 size_t itt_size; 150 }; 151 152 /* 153 * ITS command descriptor. 154 * Idea for command description passing taken from Linux. 155 */ 156 struct its_cmd_desc { 157 uint8_t cmd_type; 158 159 union { 160 struct { 161 struct its_dev *its_dev; 162 struct its_col *col; 163 uint32_t id; 164 } cmd_desc_movi; 165 166 struct { 167 struct its_col *col; 168 } cmd_desc_sync; 169 170 struct { 171 struct its_col *col; 172 uint8_t valid; 173 } cmd_desc_mapc; 174 175 struct { 176 struct its_dev *its_dev; 177 struct its_col *col; 178 uint32_t pid; 179 uint32_t id; 180 } cmd_desc_mapvi; 181 182 struct { 183 struct its_dev *its_dev; 184 struct its_col *col; 185 uint32_t pid; 186 } cmd_desc_mapi; 187 188 struct { 189 struct its_dev *its_dev; 190 uint8_t valid; 191 } cmd_desc_mapd; 192 193 struct { 194 struct its_dev *its_dev; 195 struct its_col *col; 196 uint32_t pid; 197 } cmd_desc_inv; 198 199 struct { 200 struct its_col *col; 201 } cmd_desc_invall; 202 }; 203 }; 204 205 /* ITS command. Each command is 32 bytes long */ 206 struct its_cmd { 207 uint64_t cmd_dword[4]; /* ITS command double word */ 208 }; 209 210 /* An ITS private table */ 211 struct its_ptable { 212 vm_offset_t ptab_vaddr; 213 unsigned long ptab_size; 214 }; 215 216 /* ITS collection description. */ 217 struct its_col { 218 uint64_t col_target; /* Target Re-Distributor */ 219 uint64_t col_id; /* Collection ID */ 220 }; 221 222 struct gicv3_its_irqsrc { 223 struct intr_irqsrc gi_isrc; 224 u_int gi_irq; 225 struct its_dev *gi_its_dev; 226 }; 227 228 struct gicv3_its_softc { 229 struct intr_pic *sc_pic; 230 struct resource *sc_its_res; 231 232 cpuset_t sc_cpus; 233 u_int gic_irq_cpu; 234 235 struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 236 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 237 238 /* 239 * TODO: We should get these from the parent as we only want a 240 * single copy of each across the interrupt controller. 241 */ 242 vm_offset_t sc_conf_base; 243 vm_offset_t sc_pend_base[MAXCPU]; 244 245 /* Command handling */ 246 struct mtx sc_its_cmd_lock; 247 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 248 size_t sc_its_cmd_next_idx; 249 250 vmem_t *sc_irq_alloc; 251 struct gicv3_its_irqsrc *sc_irqs; 252 u_int sc_irq_base; 253 u_int sc_irq_length; 254 255 struct mtx sc_its_dev_lock; 256 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 257 258 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001 259 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 260 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 261 u_int sc_its_flags; 262 }; 263 264 typedef void (its_quirk_func_t)(device_t); 265 static its_quirk_func_t its_quirk_cavium_22375; 266 267 static const struct { 268 const char *desc; 269 uint32_t iidr; 270 uint32_t iidr_mask; 271 its_quirk_func_t *func; 272 } its_quirks[] = { 273 { 274 /* Cavium ThunderX Pass 1.x */ 275 .desc = "Cavoum ThunderX errata: 22375, 24313", 276 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 277 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 278 .iidr_mask = ~GITS_IIDR_REVISION_MASK, 279 .func = its_quirk_cavium_22375, 280 }, 281 }; 282 283 #define gic_its_read_4(sc, reg) \ 284 bus_read_4((sc)->sc_its_res, (reg)) 285 #define gic_its_read_8(sc, reg) \ 286 bus_read_8((sc)->sc_its_res, (reg)) 287 288 #define gic_its_write_4(sc, reg, val) \ 289 bus_write_4((sc)->sc_its_res, (reg), (val)) 290 #define gic_its_write_8(sc, reg, val) \ 291 bus_write_8((sc)->sc_its_res, (reg), (val)) 292 293 static device_attach_t gicv3_its_attach; 294 static device_detach_t gicv3_its_detach; 295 296 static pic_disable_intr_t gicv3_its_disable_intr; 297 static pic_enable_intr_t gicv3_its_enable_intr; 298 static pic_map_intr_t gicv3_its_map_intr; 299 static pic_setup_intr_t gicv3_its_setup_intr; 300 static pic_post_filter_t gicv3_its_post_filter; 301 static pic_post_ithread_t gicv3_its_post_ithread; 302 static pic_pre_ithread_t gicv3_its_pre_ithread; 303 static pic_bind_intr_t gicv3_its_bind_intr; 304 #ifdef SMP 305 static pic_init_secondary_t gicv3_its_init_secondary; 306 #endif 307 static msi_alloc_msi_t gicv3_its_alloc_msi; 308 static msi_release_msi_t gicv3_its_release_msi; 309 static msi_alloc_msix_t gicv3_its_alloc_msix; 310 static msi_release_msix_t gicv3_its_release_msix; 311 static msi_map_msi_t gicv3_its_map_msi; 312 313 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 314 static void its_cmd_mapc(device_t, struct its_col *, uint8_t); 315 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 316 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 317 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 318 static void its_cmd_invall(device_t, struct its_col *); 319 320 static device_method_t gicv3_its_methods[] = { 321 /* Device interface */ 322 DEVMETHOD(device_detach, gicv3_its_detach), 323 324 /* Interrupt controller interface */ 325 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 326 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 327 DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 328 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 329 DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 330 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 331 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 332 #ifdef SMP 333 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 334 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 335 #endif 336 337 /* MSI/MSI-X */ 338 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 339 DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 340 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 341 DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 342 DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 343 344 /* End */ 345 DEVMETHOD_END 346 }; 347 348 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 349 sizeof(struct gicv3_its_softc)); 350 351 static void 352 gicv3_its_cmdq_init(struct gicv3_its_softc *sc) 353 { 354 vm_paddr_t cmd_paddr; 355 uint64_t reg, tmp; 356 357 /* Set up the command circular buffer */ 358 sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS, 359 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); 360 sc->sc_its_cmd_next_idx = 0; 361 362 cmd_paddr = vtophys(sc->sc_its_cmd_base); 363 364 /* Set the base of the command buffer */ 365 reg = GITS_CBASER_VALID | 366 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 367 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 368 (ITS_CMDQ_SIZE / 4096 - 1); 369 gic_its_write_8(sc, GITS_CBASER, reg); 370 371 /* Read back to check for fixed value fields */ 372 tmp = gic_its_read_8(sc, GITS_CBASER); 373 374 if ((tmp & GITS_CBASER_SHARE_MASK) != 375 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 376 /* Check if the hardware reported non-shareable */ 377 if ((tmp & GITS_CBASER_SHARE_MASK) == 378 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 379 /* If so remove the cache attribute */ 380 reg &= ~GITS_CBASER_CACHE_MASK; 381 reg &= ~GITS_CBASER_SHARE_MASK; 382 /* Set to Non-cacheable, Non-shareable */ 383 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 384 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 385 386 gic_its_write_8(sc, GITS_CBASER, reg); 387 } 388 389 /* The command queue has to be flushed after each command */ 390 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 391 } 392 393 /* Get the next command from the start of the buffer */ 394 gic_its_write_8(sc, GITS_CWRITER, 0x0); 395 } 396 397 static int 398 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 399 { 400 vm_offset_t table; 401 vm_paddr_t paddr; 402 uint64_t cache, reg, share, tmp, type; 403 size_t esize, its_tbl_size, nidents, nitspages, npages; 404 int i, page_size; 405 int devbits; 406 407 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 408 /* 409 * GITS_TYPER[17:13] of ThunderX reports that device IDs 410 * are to be 21 bits in length. The entry size of the ITS 411 * table can be read from GITS_BASERn[52:48] and on ThunderX 412 * is supposed to be 8 bytes in length (for device table). 413 * Finally the page size that is to be used by ITS to access 414 * this table will be set to 64KB. 415 * 416 * This gives 0x200000 entries of size 0x8 bytes covered by 417 * 256 pages each of which 64KB in size. The number of pages 418 * (minus 1) should then be written to GITS_BASERn[7:0]. In 419 * that case this value would be 0xFF but on ThunderX the 420 * maximum value that HW accepts is 0xFD. 421 * 422 * Set an arbitrary number of device ID bits to 20 in order 423 * to limit the number of entries in ITS device table to 424 * 0x100000 and the table size to 8MB. 425 */ 426 devbits = 20; 427 cache = 0; 428 } else { 429 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 430 cache = GITS_BASER_CACHE_WAWB; 431 } 432 share = GITS_BASER_SHARE_IS; 433 page_size = PAGE_SIZE_64K; 434 435 for (i = 0; i < GITS_BASER_NUM; i++) { 436 reg = gic_its_read_8(sc, GITS_BASER(i)); 437 /* The type of table */ 438 type = GITS_BASER_TYPE(reg); 439 /* The table entry size */ 440 esize = GITS_BASER_ESIZE(reg); 441 442 switch(type) { 443 case GITS_BASER_TYPE_DEV: 444 nidents = (1 << devbits); 445 its_tbl_size = esize * nidents; 446 its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); 447 break; 448 case GITS_BASER_TYPE_VP: 449 case GITS_BASER_TYPE_PP: /* Undocumented? */ 450 case GITS_BASER_TYPE_IC: 451 its_tbl_size = page_size; 452 break; 453 default: 454 continue; 455 } 456 npages = howmany(its_tbl_size, PAGE_SIZE); 457 458 /* Allocate the table */ 459 table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, 460 M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, 461 PAGE_SIZE_64K, 0); 462 463 sc->sc_its_ptab[i].ptab_vaddr = table; 464 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; 465 466 paddr = vtophys(table); 467 468 while (1) { 469 nitspages = howmany(its_tbl_size, page_size); 470 471 /* Clear the fields we will be setting */ 472 reg &= ~(GITS_BASER_VALID | 473 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 474 GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | 475 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 476 GITS_BASER_SIZE_MASK); 477 /* Set the new values */ 478 reg |= GITS_BASER_VALID | 479 (cache << GITS_BASER_CACHE_SHIFT) | 480 (type << GITS_BASER_TYPE_SHIFT) | 481 ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | 482 paddr | (share << GITS_BASER_SHARE_SHIFT) | 483 (nitspages - 1); 484 485 switch (page_size) { 486 case PAGE_SIZE: /* 4KB */ 487 reg |= 488 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 489 break; 490 case PAGE_SIZE_16K: /* 16KB */ 491 reg |= 492 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 493 break; 494 case PAGE_SIZE_64K: /* 64KB */ 495 reg |= 496 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 497 break; 498 } 499 500 gic_its_write_8(sc, GITS_BASER(i), reg); 501 502 /* Read back to check */ 503 tmp = gic_its_read_8(sc, GITS_BASER(i)); 504 505 /* Do the shareability masks line up? */ 506 if ((tmp & GITS_BASER_SHARE_MASK) != 507 (reg & GITS_BASER_SHARE_MASK)) { 508 share = (tmp & GITS_BASER_SHARE_MASK) >> 509 GITS_BASER_SHARE_SHIFT; 510 continue; 511 } 512 513 if ((tmp & GITS_BASER_PSZ_MASK) != 514 (reg & GITS_BASER_PSZ_MASK)) { 515 switch (page_size) { 516 case PAGE_SIZE_16K: 517 page_size = PAGE_SIZE; 518 continue; 519 case PAGE_SIZE_64K: 520 page_size = PAGE_SIZE_16K; 521 continue; 522 } 523 } 524 525 if (tmp != reg) { 526 device_printf(dev, "GITS_BASER%d: " 527 "unable to be updated: %lx != %lx\n", 528 i, reg, tmp); 529 return (ENXIO); 530 } 531 532 /* We should have made all needed changes */ 533 break; 534 } 535 } 536 537 return (0); 538 } 539 540 static void 541 gicv3_its_conftable_init(struct gicv3_its_softc *sc) 542 { 543 544 sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, 545 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN, 546 0); 547 548 /* Set the default configuration */ 549 memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 550 LPI_CONFTAB_SIZE); 551 552 /* Flush the table to memory */ 553 cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE); 554 } 555 556 static void 557 gicv3_its_pendtables_init(struct gicv3_its_softc *sc) 558 { 559 int i; 560 561 for (i = 0; i <= mp_maxid; i++) { 562 if (CPU_ISSET(i, &sc->sc_cpus) == 0) 563 continue; 564 565 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( 566 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 567 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 568 569 /* Flush so the ITS can see the memory */ 570 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i], 571 LPI_PENDTAB_SIZE); 572 } 573 } 574 575 static int 576 its_init_cpu(device_t dev, struct gicv3_its_softc *sc) 577 { 578 device_t gicv3; 579 vm_paddr_t target; 580 uint64_t xbaser, tmp; 581 uint32_t ctlr; 582 u_int cpuid; 583 int domain; 584 585 if (!CPU_ISSET(PCPU_GET(cpuid), &sc->sc_cpus)) 586 return (0); 587 588 if (bus_get_domain(dev, &domain) == 0) { 589 if (PCPU_GET(domain) != domain) 590 return (0); 591 } 592 593 gicv3 = device_get_parent(dev); 594 cpuid = PCPU_GET(cpuid); 595 596 /* Check if the ITS is enabled on this CPU */ 597 if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) { 598 return (ENXIO); 599 } 600 601 /* Disable LPIs */ 602 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 603 ctlr &= ~GICR_CTLR_LPI_ENABLE; 604 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 605 606 /* Make sure changes are observable my the GIC */ 607 dsb(sy); 608 609 /* 610 * Set the redistributor base 611 */ 612 xbaser = vtophys(sc->sc_conf_base) | 613 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 614 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 615 (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 616 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 617 618 /* Check the cache attributes we set */ 619 tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 620 621 if ((tmp & GICR_PROPBASER_SHARE_MASK) != 622 (xbaser & GICR_PROPBASER_SHARE_MASK)) { 623 if ((tmp & GICR_PROPBASER_SHARE_MASK) == 624 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 625 /* We need to mark as non-cacheable */ 626 xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 627 GICR_PROPBASER_CACHE_MASK); 628 /* Non-cacheable */ 629 xbaser |= GICR_PROPBASER_CACHE_NIN << 630 GICR_PROPBASER_CACHE_SHIFT; 631 /* Non-sareable */ 632 xbaser |= GICR_PROPBASER_SHARE_NS << 633 GICR_PROPBASER_SHARE_SHIFT; 634 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 635 } 636 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 637 } 638 639 /* 640 * Set the LPI pending table base 641 */ 642 xbaser = vtophys(sc->sc_pend_base[cpuid]) | 643 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 644 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 645 646 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 647 648 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 649 650 if ((tmp & GICR_PENDBASER_SHARE_MASK) == 651 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 652 /* Clear the cahce and shareability bits */ 653 xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 654 GICR_PENDBASER_SHARE_MASK); 655 /* Mark as non-shareable */ 656 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 657 /* And non-cacheable */ 658 xbaser |= GICR_PENDBASER_CACHE_NIN << 659 GICR_PENDBASER_CACHE_SHIFT; 660 } 661 662 /* Enable LPIs */ 663 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 664 ctlr |= GICR_CTLR_LPI_ENABLE; 665 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 666 667 /* Make sure the GIC has seen everything */ 668 dsb(sy); 669 670 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 671 /* This ITS wants the redistributor physical address */ 672 target = vtophys(gicv3_get_redist_vaddr(dev)); 673 } else { 674 /* This ITS wants the unique processor number */ 675 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)); 676 } 677 678 sc->sc_its_cols[cpuid]->col_target = target; 679 sc->sc_its_cols[cpuid]->col_id = cpuid; 680 681 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 682 its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 683 684 return (0); 685 } 686 687 static int 688 gicv3_its_attach(device_t dev) 689 { 690 struct gicv3_its_softc *sc; 691 const char *name; 692 uint32_t iidr; 693 int domain, err, i, rid; 694 695 sc = device_get_softc(dev); 696 697 sc->sc_irq_length = gicv3_get_nirqs(dev); 698 sc->sc_irq_base = GIC_FIRST_LPI; 699 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; 700 701 rid = 0; 702 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 703 RF_ACTIVE); 704 if (sc->sc_its_res == NULL) { 705 device_printf(dev, "Could not allocate memory\n"); 706 return (ENXIO); 707 } 708 709 iidr = gic_its_read_4(sc, GITS_IIDR); 710 for (i = 0; i < nitems(its_quirks); i++) { 711 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 712 if (bootverbose) { 713 device_printf(dev, "Applying %s\n", 714 its_quirks[i].desc); 715 } 716 its_quirks[i].func(dev); 717 break; 718 } 719 } 720 721 /* Allocate the private tables */ 722 err = gicv3_its_table_init(dev, sc); 723 if (err != 0) 724 return (err); 725 726 /* Protects access to the device list */ 727 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 728 729 /* Protects access to the ITS command circular buffer. */ 730 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 731 732 if (bus_get_domain(dev, &domain) == 0) { 733 CPU_ZERO(&sc->sc_cpus); 734 if (domain < MAXMEMDOM) 735 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus); 736 } else { 737 CPU_COPY(&all_cpus, &sc->sc_cpus); 738 } 739 740 /* Allocate the command circular buffer */ 741 gicv3_its_cmdq_init(sc); 742 743 /* Allocate the per-CPU collections */ 744 for (int cpu = 0; cpu <= mp_maxid; cpu++) 745 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0) 746 sc->sc_its_cols[cpu] = malloc( 747 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 748 M_WAITOK | M_ZERO); 749 else 750 sc->sc_its_cols[cpu] = NULL; 751 752 /* Enable the ITS */ 753 gic_its_write_4(sc, GITS_CTLR, 754 gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); 755 756 /* Create the LPI configuration table */ 757 gicv3_its_conftable_init(sc); 758 759 /* And the pending tebles */ 760 gicv3_its_pendtables_init(sc); 761 762 /* Enable LPIs on this CPU */ 763 its_init_cpu(dev, sc); 764 765 TAILQ_INIT(&sc->sc_its_dev_list); 766 767 /* 768 * Create the vmem object to allocate INTRNG IRQs from. We try to 769 * use all IRQs not already used by the GICv3. 770 * XXX: This assumes there are no other interrupt controllers in the 771 * system. 772 */ 773 sc->sc_irq_alloc = vmem_create("GICv3 ITS IRQs", 0, 774 gicv3_get_nirqs(dev), 1, 1, M_FIRSTFIT | M_WAITOK); 775 776 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length, 777 M_GICV3_ITS, M_WAITOK | M_ZERO); 778 name = device_get_nameunit(dev); 779 for (i = 0; i < sc->sc_irq_length; i++) { 780 sc->sc_irqs[i].gi_irq = i; 781 err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0, 782 "%s,%u", name, i); 783 } 784 785 return (0); 786 } 787 788 static int 789 gicv3_its_detach(device_t dev) 790 { 791 792 return (ENXIO); 793 } 794 795 static void 796 its_quirk_cavium_22375(device_t dev) 797 { 798 struct gicv3_its_softc *sc; 799 800 sc = device_get_softc(dev); 801 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 802 } 803 804 static void 805 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 806 { 807 struct gicv3_its_softc *sc; 808 struct gicv3_its_irqsrc *girq; 809 uint8_t *conf; 810 811 sc = device_get_softc(dev); 812 girq = (struct gicv3_its_irqsrc *)isrc; 813 conf = (uint8_t *)sc->sc_conf_base; 814 815 conf[girq->gi_irq] &= ~LPI_CONF_ENABLE; 816 817 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 818 /* Clean D-cache under command. */ 819 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 820 } else { 821 /* DSB inner shareable, store */ 822 dsb(ishst); 823 } 824 825 its_cmd_inv(dev, girq->gi_its_dev, girq); 826 } 827 828 static void 829 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 830 { 831 struct gicv3_its_softc *sc; 832 struct gicv3_its_irqsrc *girq; 833 uint8_t *conf; 834 835 sc = device_get_softc(dev); 836 girq = (struct gicv3_its_irqsrc *)isrc; 837 conf = (uint8_t *)sc->sc_conf_base; 838 839 conf[girq->gi_irq] |= LPI_CONF_ENABLE; 840 841 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 842 /* Clean D-cache under command. */ 843 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 844 } else { 845 /* DSB inner shareable, store */ 846 dsb(ishst); 847 } 848 849 its_cmd_inv(dev, girq->gi_its_dev, girq); 850 } 851 852 static int 853 gicv3_its_intr(void *arg, uintptr_t irq) 854 { 855 struct gicv3_its_softc *sc = arg; 856 struct gicv3_its_irqsrc *girq; 857 struct trapframe *tf; 858 859 irq -= sc->sc_irq_base; 860 girq = &sc->sc_irqs[irq]; 861 if (girq == NULL) 862 panic("gicv3_its_intr: Invalid interrupt %ld", 863 irq + sc->sc_irq_base); 864 865 tf = curthread->td_intr_frame; 866 intr_isrc_dispatch(&girq->gi_isrc, tf); 867 return (FILTER_HANDLED); 868 } 869 870 static void 871 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 872 { 873 struct gicv3_its_irqsrc *girq; 874 struct gicv3_its_softc *sc; 875 876 sc = device_get_softc(dev); 877 girq = (struct gicv3_its_irqsrc *)isrc; 878 gicv3_its_disable_intr(dev, isrc); 879 gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base); 880 } 881 882 static void 883 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 884 { 885 886 gicv3_its_enable_intr(dev, isrc); 887 } 888 889 static void 890 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 891 { 892 struct gicv3_its_irqsrc *girq; 893 struct gicv3_its_softc *sc; 894 895 sc = device_get_softc(dev); 896 girq = (struct gicv3_its_irqsrc *)isrc; 897 gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base); 898 } 899 900 static int 901 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 902 { 903 struct gicv3_its_irqsrc *girq; 904 struct gicv3_its_softc *sc; 905 906 sc = device_get_softc(dev); 907 girq = (struct gicv3_its_irqsrc *)isrc; 908 if (CPU_EMPTY(&isrc->isrc_cpu)) { 909 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu, 910 &sc->sc_cpus); 911 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu); 912 } 913 914 its_cmd_movi(dev, girq); 915 916 return (0); 917 } 918 919 static int 920 gicv3_its_map_intr(device_t dev, struct intr_map_data *data, 921 struct intr_irqsrc **isrcp) 922 { 923 924 /* 925 * This should never happen, we only call this function to map 926 * interrupts found before the controller driver is ready. 927 */ 928 panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 929 } 930 931 static int 932 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 933 struct resource *res, struct intr_map_data *data) 934 { 935 936 /* Bind the interrupt to a CPU */ 937 gicv3_its_bind_intr(dev, isrc); 938 939 return (0); 940 } 941 942 #ifdef SMP 943 static void 944 gicv3_its_init_secondary(device_t dev) 945 { 946 struct gicv3_its_softc *sc; 947 948 sc = device_get_softc(dev); 949 950 /* 951 * This is fatal as otherwise we may bind interrupts to this CPU. 952 * We need a way to tell the interrupt framework to only bind to a 953 * subset of given CPUs when it performs the shuffle. 954 */ 955 if (its_init_cpu(dev, sc) != 0) 956 panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 957 PCPU_GET(cpuid)); 958 } 959 #endif 960 961 static uint32_t 962 its_get_devid(device_t pci_dev) 963 { 964 uintptr_t id; 965 966 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 967 panic("its_get_devid: Unable to get the MSI DeviceID"); 968 969 return (id); 970 } 971 972 static struct its_dev * 973 its_device_find(device_t dev, device_t child) 974 { 975 struct gicv3_its_softc *sc; 976 struct its_dev *its_dev = NULL; 977 978 sc = device_get_softc(dev); 979 980 mtx_lock_spin(&sc->sc_its_dev_lock); 981 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 982 if (its_dev->pci_dev == child) 983 break; 984 } 985 mtx_unlock_spin(&sc->sc_its_dev_lock); 986 987 return (its_dev); 988 } 989 990 static struct its_dev * 991 its_device_get(device_t dev, device_t child, u_int nvecs) 992 { 993 struct gicv3_its_softc *sc; 994 struct its_dev *its_dev; 995 vmem_addr_t irq_base; 996 size_t esize; 997 998 sc = device_get_softc(dev); 999 1000 its_dev = its_device_find(dev, child); 1001 if (its_dev != NULL) 1002 return (its_dev); 1003 1004 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 1005 if (its_dev == NULL) 1006 return (NULL); 1007 1008 its_dev->pci_dev = child; 1009 its_dev->devid = its_get_devid(child); 1010 1011 its_dev->lpis.lpi_busy = 0; 1012 its_dev->lpis.lpi_num = nvecs; 1013 its_dev->lpis.lpi_free = nvecs; 1014 1015 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 1016 &irq_base) != 0) { 1017 free(its_dev, M_GICV3_ITS); 1018 return (NULL); 1019 } 1020 its_dev->lpis.lpi_base = irq_base; 1021 1022 /* Get ITT entry size */ 1023 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 1024 1025 /* 1026 * Allocate ITT for this device. 1027 * PA has to be 256 B aligned. At least two entries for device. 1028 */ 1029 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 1030 its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size, 1031 M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, 1032 LPI_INT_TRANS_TAB_ALIGN, 0); 1033 if (its_dev->itt == 0) { 1034 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 1035 free(its_dev, M_GICV3_ITS); 1036 return (NULL); 1037 } 1038 1039 mtx_lock_spin(&sc->sc_its_dev_lock); 1040 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 1041 mtx_unlock_spin(&sc->sc_its_dev_lock); 1042 1043 /* Map device to its ITT */ 1044 its_cmd_mapd(dev, its_dev, 1); 1045 1046 return (its_dev); 1047 } 1048 1049 static void 1050 its_device_release(device_t dev, struct its_dev *its_dev) 1051 { 1052 struct gicv3_its_softc *sc; 1053 1054 KASSERT(its_dev->lpis.lpi_busy == 0, 1055 ("its_device_release: Trying to release an inuse ITS device")); 1056 1057 /* Unmap device in ITS */ 1058 its_cmd_mapd(dev, its_dev, 0); 1059 1060 sc = device_get_softc(dev); 1061 1062 /* Remove the device from the list of devices */ 1063 mtx_lock_spin(&sc->sc_its_dev_lock); 1064 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 1065 mtx_unlock_spin(&sc->sc_its_dev_lock); 1066 1067 /* Free ITT */ 1068 KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); 1069 contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 1070 1071 /* Free the IRQ allocation */ 1072 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 1073 its_dev->lpis.lpi_num); 1074 1075 free(its_dev, M_GICV3_ITS); 1076 } 1077 1078 static int 1079 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 1080 device_t *pic, struct intr_irqsrc **srcs) 1081 { 1082 struct gicv3_its_softc *sc; 1083 struct gicv3_its_irqsrc *girq; 1084 struct its_dev *its_dev; 1085 u_int irq; 1086 int i; 1087 1088 its_dev = its_device_get(dev, child, count); 1089 if (its_dev == NULL) 1090 return (ENXIO); 1091 1092 KASSERT(its_dev->lpis.lpi_free >= count, 1093 ("gicv3_its_alloc_msi: No free LPIs")); 1094 sc = device_get_softc(dev); 1095 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1096 its_dev->lpis.lpi_free; 1097 for (i = 0; i < count; i++, irq++) { 1098 its_dev->lpis.lpi_free--; 1099 girq = &sc->sc_irqs[irq]; 1100 girq->gi_its_dev = its_dev; 1101 srcs[i] = (struct intr_irqsrc *)girq; 1102 } 1103 its_dev->lpis.lpi_busy += count; 1104 *pic = dev; 1105 1106 return (0); 1107 } 1108 1109 static int 1110 gicv3_its_release_msi(device_t dev, device_t child, int count, 1111 struct intr_irqsrc **isrc) 1112 { 1113 struct gicv3_its_irqsrc *girq; 1114 struct its_dev *its_dev; 1115 int i; 1116 1117 its_dev = its_device_find(dev, child); 1118 1119 KASSERT(its_dev != NULL, 1120 ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1121 "no ITS device")); 1122 KASSERT(its_dev->lpis.lpi_busy >= count, 1123 ("gicv3_its_release_msi: Releasing more interrupts than " 1124 "were allocated: releasing %d, allocated %d", count, 1125 its_dev->lpis.lpi_busy)); 1126 for (i = 0; i < count; i++) { 1127 girq = (struct gicv3_its_irqsrc *)isrc[i]; 1128 girq->gi_its_dev = NULL; 1129 } 1130 its_dev->lpis.lpi_busy -= count; 1131 1132 if (its_dev->lpis.lpi_busy == 0) 1133 its_device_release(dev, its_dev); 1134 1135 return (0); 1136 } 1137 1138 static int 1139 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1140 struct intr_irqsrc **isrcp) 1141 { 1142 struct gicv3_its_softc *sc; 1143 struct gicv3_its_irqsrc *girq; 1144 struct its_dev *its_dev; 1145 u_int nvecs, irq; 1146 1147 nvecs = pci_msix_count(child); 1148 its_dev = its_device_get(dev, child, nvecs); 1149 if (its_dev == NULL) 1150 return (ENXIO); 1151 1152 KASSERT(its_dev->lpis.lpi_free > 0, 1153 ("gicv3_its_alloc_msix: No free LPIs")); 1154 sc = device_get_softc(dev); 1155 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1156 its_dev->lpis.lpi_free; 1157 its_dev->lpis.lpi_free--; 1158 its_dev->lpis.lpi_busy++; 1159 girq = &sc->sc_irqs[irq]; 1160 girq->gi_its_dev = its_dev; 1161 1162 *pic = dev; 1163 *isrcp = (struct intr_irqsrc *)girq; 1164 1165 return (0); 1166 } 1167 1168 static int 1169 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1170 { 1171 struct gicv3_its_irqsrc *girq; 1172 struct its_dev *its_dev; 1173 1174 its_dev = its_device_find(dev, child); 1175 1176 KASSERT(its_dev != NULL, 1177 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1178 "no ITS device")); 1179 KASSERT(its_dev->lpis.lpi_busy > 0, 1180 ("gicv3_its_release_msix: Releasing more interrupts than " 1181 "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1182 girq = (struct gicv3_its_irqsrc *)isrc; 1183 girq->gi_its_dev = NULL; 1184 its_dev->lpis.lpi_busy--; 1185 1186 if (its_dev->lpis.lpi_busy == 0) 1187 its_device_release(dev, its_dev); 1188 1189 return (0); 1190 } 1191 1192 static int 1193 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1194 uint64_t *addr, uint32_t *data) 1195 { 1196 struct gicv3_its_softc *sc; 1197 struct gicv3_its_irqsrc *girq; 1198 1199 sc = device_get_softc(dev); 1200 girq = (struct gicv3_its_irqsrc *)isrc; 1201 1202 /* Map the message to the given IRQ */ 1203 its_cmd_mapti(dev, girq); 1204 1205 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1206 *data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1207 1208 return (0); 1209 } 1210 1211 /* 1212 * Commands handling. 1213 */ 1214 1215 static __inline void 1216 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1217 { 1218 /* Command field: DW0 [7:0] */ 1219 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1220 cmd->cmd_dword[0] |= htole64(cmd_type); 1221 } 1222 1223 static __inline void 1224 cmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1225 { 1226 /* Device ID field: DW0 [63:32] */ 1227 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1228 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1229 } 1230 1231 static __inline void 1232 cmd_format_size(struct its_cmd *cmd, uint16_t size) 1233 { 1234 /* Size field: DW1 [4:0] */ 1235 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1236 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1237 } 1238 1239 static __inline void 1240 cmd_format_id(struct its_cmd *cmd, uint32_t id) 1241 { 1242 /* ID field: DW1 [31:0] */ 1243 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1244 cmd->cmd_dword[1] |= htole64(id); 1245 } 1246 1247 static __inline void 1248 cmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1249 { 1250 /* Physical ID field: DW1 [63:32] */ 1251 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1252 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1253 } 1254 1255 static __inline void 1256 cmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1257 { 1258 /* Collection field: DW2 [16:0] */ 1259 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1260 cmd->cmd_dword[2] |= htole64(col_id); 1261 } 1262 1263 static __inline void 1264 cmd_format_target(struct its_cmd *cmd, uint64_t target) 1265 { 1266 /* Target Address field: DW2 [47:16] */ 1267 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1268 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1269 } 1270 1271 static __inline void 1272 cmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1273 { 1274 /* ITT Address field: DW2 [47:8] */ 1275 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1276 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1277 } 1278 1279 static __inline void 1280 cmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1281 { 1282 /* Valid field: DW2 [63] */ 1283 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1284 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1285 } 1286 1287 static inline bool 1288 its_cmd_queue_full(struct gicv3_its_softc *sc) 1289 { 1290 size_t read_idx, next_write_idx; 1291 1292 /* Get the index of the next command */ 1293 next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1294 (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1295 /* And the index of the current command being read */ 1296 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1297 1298 /* 1299 * The queue is full when the write offset points 1300 * at the command before the current read offset. 1301 */ 1302 return (next_write_idx == read_idx); 1303 } 1304 1305 static inline void 1306 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1307 { 1308 1309 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1310 /* Clean D-cache under command. */ 1311 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); 1312 } else { 1313 /* DSB inner shareable, store */ 1314 dsb(ishst); 1315 } 1316 1317 } 1318 1319 static inline uint64_t 1320 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1321 { 1322 uint64_t off; 1323 1324 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1325 1326 return (off); 1327 } 1328 1329 static void 1330 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1331 struct its_cmd *cmd_last) 1332 { 1333 struct gicv3_its_softc *sc; 1334 uint64_t first, last, read; 1335 size_t us_left; 1336 1337 sc = device_get_softc(dev); 1338 1339 /* 1340 * XXX ARM64TODO: This is obviously a significant delay. 1341 * The reason for that is that currently the time frames for 1342 * the command to complete are not known. 1343 */ 1344 us_left = 1000000; 1345 1346 first = its_cmd_cwriter_offset(sc, cmd_first); 1347 last = its_cmd_cwriter_offset(sc, cmd_last); 1348 1349 for (;;) { 1350 read = gic_its_read_8(sc, GITS_CREADR); 1351 if (first < last) { 1352 if (read < first || read >= last) 1353 break; 1354 } else if (read < first && read >= last) 1355 break; 1356 1357 if (us_left-- == 0) { 1358 /* This means timeout */ 1359 device_printf(dev, 1360 "Timeout while waiting for CMD completion.\n"); 1361 return; 1362 } 1363 DELAY(1); 1364 } 1365 } 1366 1367 1368 static struct its_cmd * 1369 its_cmd_alloc_locked(device_t dev) 1370 { 1371 struct gicv3_its_softc *sc; 1372 struct its_cmd *cmd; 1373 size_t us_left; 1374 1375 sc = device_get_softc(dev); 1376 1377 /* 1378 * XXX ARM64TODO: This is obviously a significant delay. 1379 * The reason for that is that currently the time frames for 1380 * the command to complete (and therefore free the descriptor) 1381 * are not known. 1382 */ 1383 us_left = 1000000; 1384 1385 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1386 while (its_cmd_queue_full(sc)) { 1387 if (us_left-- == 0) { 1388 /* Timeout while waiting for free command */ 1389 device_printf(dev, 1390 "Timeout while waiting for free command\n"); 1391 return (NULL); 1392 } 1393 DELAY(1); 1394 } 1395 1396 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1397 sc->sc_its_cmd_next_idx++; 1398 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1399 1400 return (cmd); 1401 } 1402 1403 static uint64_t 1404 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1405 { 1406 uint64_t target; 1407 uint8_t cmd_type; 1408 u_int size; 1409 1410 cmd_type = desc->cmd_type; 1411 target = ITS_TARGET_NONE; 1412 1413 switch (cmd_type) { 1414 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1415 target = desc->cmd_desc_movi.col->col_target; 1416 cmd_format_command(cmd, ITS_CMD_MOVI); 1417 cmd_format_id(cmd, desc->cmd_desc_movi.id); 1418 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1419 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1420 break; 1421 case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1422 target = desc->cmd_desc_sync.col->col_target; 1423 cmd_format_command(cmd, ITS_CMD_SYNC); 1424 cmd_format_target(cmd, target); 1425 break; 1426 case ITS_CMD_MAPD: /* Assign ITT to device */ 1427 cmd_format_command(cmd, ITS_CMD_MAPD); 1428 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1429 /* 1430 * Size describes number of bits to encode interrupt IDs 1431 * supported by the device minus one. 1432 * When V (valid) bit is zero, this field should be written 1433 * as zero. 1434 */ 1435 if (desc->cmd_desc_mapd.valid != 0) { 1436 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1437 size = MAX(1, size) - 1; 1438 } else 1439 size = 0; 1440 1441 cmd_format_size(cmd, size); 1442 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1443 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1444 break; 1445 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1446 target = desc->cmd_desc_mapc.col->col_target; 1447 cmd_format_command(cmd, ITS_CMD_MAPC); 1448 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 1449 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 1450 cmd_format_target(cmd, target); 1451 break; 1452 case ITS_CMD_MAPTI: 1453 target = desc->cmd_desc_mapvi.col->col_target; 1454 cmd_format_command(cmd, ITS_CMD_MAPTI); 1455 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 1456 cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 1457 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 1458 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 1459 break; 1460 case ITS_CMD_MAPI: 1461 target = desc->cmd_desc_mapi.col->col_target; 1462 cmd_format_command(cmd, ITS_CMD_MAPI); 1463 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 1464 cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 1465 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 1466 break; 1467 case ITS_CMD_INV: 1468 target = desc->cmd_desc_inv.col->col_target; 1469 cmd_format_command(cmd, ITS_CMD_INV); 1470 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 1471 cmd_format_id(cmd, desc->cmd_desc_inv.pid); 1472 break; 1473 case ITS_CMD_INVALL: 1474 cmd_format_command(cmd, ITS_CMD_INVALL); 1475 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 1476 break; 1477 default: 1478 panic("its_cmd_prepare: Invalid command: %x", cmd_type); 1479 } 1480 1481 return (target); 1482 } 1483 1484 static int 1485 its_cmd_send(device_t dev, struct its_cmd_desc *desc) 1486 { 1487 struct gicv3_its_softc *sc; 1488 struct its_cmd *cmd, *cmd_sync, *cmd_write; 1489 struct its_col col_sync; 1490 struct its_cmd_desc desc_sync; 1491 uint64_t target, cwriter; 1492 1493 sc = device_get_softc(dev); 1494 mtx_lock_spin(&sc->sc_its_cmd_lock); 1495 cmd = its_cmd_alloc_locked(dev); 1496 if (cmd == NULL) { 1497 device_printf(dev, "could not allocate ITS command\n"); 1498 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1499 return (EBUSY); 1500 } 1501 1502 target = its_cmd_prepare(cmd, desc); 1503 its_cmd_sync(sc, cmd); 1504 1505 if (target != ITS_TARGET_NONE) { 1506 cmd_sync = its_cmd_alloc_locked(dev); 1507 if (cmd_sync != NULL) { 1508 desc_sync.cmd_type = ITS_CMD_SYNC; 1509 col_sync.col_target = target; 1510 desc_sync.cmd_desc_sync.col = &col_sync; 1511 its_cmd_prepare(cmd_sync, &desc_sync); 1512 its_cmd_sync(sc, cmd_sync); 1513 } 1514 } 1515 1516 /* Update GITS_CWRITER */ 1517 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 1518 gic_its_write_8(sc, GITS_CWRITER, cwriter); 1519 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1520 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1521 1522 its_cmd_wait_completion(dev, cmd, cmd_write); 1523 1524 return (0); 1525 } 1526 1527 /* Handlers to send commands */ 1528 static void 1529 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 1530 { 1531 struct gicv3_its_softc *sc; 1532 struct its_cmd_desc desc; 1533 struct its_col *col; 1534 1535 sc = device_get_softc(dev); 1536 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1537 1538 desc.cmd_type = ITS_CMD_MOVI; 1539 desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 1540 desc.cmd_desc_movi.col = col; 1541 desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1542 1543 its_cmd_send(dev, &desc); 1544 } 1545 1546 static void 1547 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 1548 { 1549 struct its_cmd_desc desc; 1550 1551 desc.cmd_type = ITS_CMD_MAPC; 1552 desc.cmd_desc_mapc.col = col; 1553 /* 1554 * Valid bit set - map the collection. 1555 * Valid bit cleared - unmap the collection. 1556 */ 1557 desc.cmd_desc_mapc.valid = valid; 1558 1559 its_cmd_send(dev, &desc); 1560 } 1561 1562 static void 1563 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 1564 { 1565 struct gicv3_its_softc *sc; 1566 struct its_cmd_desc desc; 1567 struct its_col *col; 1568 u_int col_id; 1569 1570 sc = device_get_softc(dev); 1571 1572 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 1573 col = sc->sc_its_cols[col_id]; 1574 1575 desc.cmd_type = ITS_CMD_MAPTI; 1576 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 1577 desc.cmd_desc_mapvi.col = col; 1578 /* The EventID sent to the device */ 1579 desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1580 /* The physical interrupt presented to softeware */ 1581 desc.cmd_desc_mapvi.pid = girq->gi_irq + sc->sc_irq_base; 1582 1583 its_cmd_send(dev, &desc); 1584 } 1585 1586 static void 1587 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 1588 { 1589 struct its_cmd_desc desc; 1590 1591 desc.cmd_type = ITS_CMD_MAPD; 1592 desc.cmd_desc_mapd.its_dev = its_dev; 1593 desc.cmd_desc_mapd.valid = valid; 1594 1595 its_cmd_send(dev, &desc); 1596 } 1597 1598 static void 1599 its_cmd_inv(device_t dev, struct its_dev *its_dev, 1600 struct gicv3_its_irqsrc *girq) 1601 { 1602 struct gicv3_its_softc *sc; 1603 struct its_cmd_desc desc; 1604 struct its_col *col; 1605 1606 sc = device_get_softc(dev); 1607 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1608 1609 desc.cmd_type = ITS_CMD_INV; 1610 /* The EventID sent to the device */ 1611 desc.cmd_desc_inv.pid = girq->gi_irq - its_dev->lpis.lpi_base; 1612 desc.cmd_desc_inv.its_dev = its_dev; 1613 desc.cmd_desc_inv.col = col; 1614 1615 its_cmd_send(dev, &desc); 1616 } 1617 1618 static void 1619 its_cmd_invall(device_t dev, struct its_col *col) 1620 { 1621 struct its_cmd_desc desc; 1622 1623 desc.cmd_type = ITS_CMD_INVALL; 1624 desc.cmd_desc_invall.col = col; 1625 1626 its_cmd_send(dev, &desc); 1627 } 1628 1629 #ifdef FDT 1630 static device_probe_t gicv3_its_fdt_probe; 1631 static device_attach_t gicv3_its_fdt_attach; 1632 1633 static device_method_t gicv3_its_fdt_methods[] = { 1634 /* Device interface */ 1635 DEVMETHOD(device_probe, gicv3_its_fdt_probe), 1636 DEVMETHOD(device_attach, gicv3_its_fdt_attach), 1637 1638 /* End */ 1639 DEVMETHOD_END 1640 }; 1641 1642 #define its_baseclasses its_fdt_baseclasses 1643 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 1644 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1645 #undef its_baseclasses 1646 static devclass_t gicv3_its_fdt_devclass; 1647 1648 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 1649 gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1650 1651 static int 1652 gicv3_its_fdt_probe(device_t dev) 1653 { 1654 1655 if (!ofw_bus_status_okay(dev)) 1656 return (ENXIO); 1657 1658 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 1659 return (ENXIO); 1660 1661 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1662 return (BUS_PROBE_DEFAULT); 1663 } 1664 1665 static int 1666 gicv3_its_fdt_attach(device_t dev) 1667 { 1668 struct gicv3_its_softc *sc; 1669 phandle_t xref; 1670 int err; 1671 1672 sc = device_get_softc(dev); 1673 err = gicv3_its_attach(dev); 1674 if (err != 0) 1675 return (err); 1676 1677 /* Register this device as a interrupt controller */ 1678 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 1679 sc->sc_pic = intr_pic_register(dev, xref); 1680 intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1681 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 1682 1683 /* Register this device to handle MSI interrupts */ 1684 intr_msi_register(dev, xref); 1685 1686 return (0); 1687 } 1688 #endif 1689 1690 #ifdef DEV_ACPI 1691 static device_probe_t gicv3_its_acpi_probe; 1692 static device_attach_t gicv3_its_acpi_attach; 1693 1694 static device_method_t gicv3_its_acpi_methods[] = { 1695 /* Device interface */ 1696 DEVMETHOD(device_probe, gicv3_its_acpi_probe), 1697 DEVMETHOD(device_attach, gicv3_its_acpi_attach), 1698 1699 /* End */ 1700 DEVMETHOD_END 1701 }; 1702 1703 #define its_baseclasses its_acpi_baseclasses 1704 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods, 1705 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1706 #undef its_baseclasses 1707 static devclass_t gicv3_its_acpi_devclass; 1708 1709 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 1710 gicv3_its_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1711 1712 static int 1713 gicv3_its_acpi_probe(device_t dev) 1714 { 1715 1716 if (gic_get_bus(dev) != GIC_BUS_ACPI) 1717 return (EINVAL); 1718 1719 if (gic_get_hw_rev(dev) < 3) 1720 return (EINVAL); 1721 1722 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1723 return (BUS_PROBE_DEFAULT); 1724 } 1725 1726 static int 1727 gicv3_its_acpi_attach(device_t dev) 1728 { 1729 struct gicv3_its_softc *sc; 1730 int err; 1731 1732 sc = device_get_softc(dev); 1733 err = gicv3_its_attach(dev); 1734 if (err != 0) 1735 return (err); 1736 1737 sc->sc_pic = intr_pic_register(dev, 1738 device_get_unit(dev) + ACPI_MSI_XREF); 1739 intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1740 gicv3_its_intr, sc, GIC_FIRST_LPI, LPI_NIRQS); 1741 1742 /* Register this device to handle MSI interrupts */ 1743 intr_msi_register(dev, device_get_unit(dev) + ACPI_MSI_XREF); 1744 1745 return (0); 1746 } 1747 #endif 1748