1 /*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * Copyright (c) 2023 Arm Ltd 4 * 5 * This software was developed by Andrew Turner under 6 * the sponsorship of the FreeBSD Foundation. 7 * 8 * This software was developed by Semihalf under 9 * the sponsorship of the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include "opt_acpi.h" 34 #include "opt_platform.h" 35 #include "opt_iommu.h" 36 37 #include <sys/cdefs.h> 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/bus.h> 41 #include <sys/cpuset.h> 42 #include <sys/domainset.h> 43 #include <sys/endian.h> 44 #include <sys/kernel.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/module.h> 48 #include <sys/mutex.h> 49 #include <sys/proc.h> 50 #include <sys/taskqueue.h> 51 #include <sys/tree.h> 52 #include <sys/queue.h> 53 #include <sys/rman.h> 54 #include <sys/sbuf.h> 55 #include <sys/smp.h> 56 #include <sys/sysctl.h> 57 #include <sys/vmem.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #include <vm/vm_page.h> 62 63 #include <machine/bus.h> 64 #include <machine/intr.h> 65 66 #include <arm/arm/gic_common.h> 67 #include <arm64/arm64/gic_v3_reg.h> 68 #include <arm64/arm64/gic_v3_var.h> 69 70 #ifdef FDT 71 #include <dev/ofw/openfirm.h> 72 #include <dev/ofw/ofw_bus.h> 73 #include <dev/ofw/ofw_bus_subr.h> 74 #endif 75 #include <dev/pci/pcireg.h> 76 #include <dev/pci/pcivar.h> 77 78 #ifdef IOMMU 79 #include <dev/iommu/iommu.h> 80 #include <dev/iommu/iommu_gas.h> 81 #endif 82 83 #include "pcib_if.h" 84 #include "pic_if.h" 85 #include "msi_if.h" 86 87 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 88 "ARM GICv3 Interrupt Translation Service"); 89 90 #define LPI_NIRQS (64 * 1024) 91 92 /* The size and alignment of the command circular buffer */ 93 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 94 #define ITS_CMDQ_ALIGN (64 * 1024) 95 96 #define LPI_CONFTAB_SIZE LPI_NIRQS 97 #define LPI_CONFTAB_ALIGN (64 * 1024) 98 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 99 100 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 101 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 102 #define LPI_PENDTAB_ALIGN (64 * 1024) 103 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 104 105 #define LPI_INT_TRANS_TAB_ALIGN 256 106 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 107 108 /* ITS commands encoding */ 109 #define ITS_CMD_MOVI (0x01) 110 #define ITS_CMD_SYNC (0x05) 111 #define ITS_CMD_MAPD (0x08) 112 #define ITS_CMD_MAPC (0x09) 113 #define ITS_CMD_MAPTI (0x0a) 114 #define ITS_CMD_MAPI (0x0b) 115 #define ITS_CMD_INV (0x0c) 116 #define ITS_CMD_INVALL (0x0d) 117 /* Command */ 118 #define CMD_COMMAND_MASK (0xFFUL) 119 /* PCI device ID */ 120 #define CMD_DEVID_SHIFT (32) 121 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 122 /* Size of IRQ ID bitfield */ 123 #define CMD_SIZE_MASK (0xFFUL) 124 /* Virtual LPI ID */ 125 #define CMD_ID_MASK (0xFFFFFFFFUL) 126 /* Physical LPI ID */ 127 #define CMD_PID_SHIFT (32) 128 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 129 /* Collection */ 130 #define CMD_COL_MASK (0xFFFFUL) 131 /* Target (CPU or Re-Distributor) */ 132 #define CMD_TARGET_SHIFT (16) 133 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 134 /* Interrupt Translation Table address */ 135 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 136 /* Valid command bit */ 137 #define CMD_VALID_SHIFT (63) 138 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 139 140 #define ITS_TARGET_NONE 0xFBADBEEF 141 142 /* LPI chunk owned by ITS device */ 143 struct lpi_chunk { 144 u_int lpi_base; 145 u_int lpi_free; /* First free LPI in set */ 146 u_int lpi_num; /* Total number of LPIs in chunk */ 147 u_int lpi_busy; /* Number of busy LPIs in chink */ 148 }; 149 150 /* ITS device */ 151 struct its_dev { 152 TAILQ_ENTRY(its_dev) entry; 153 /* PCI device */ 154 device_t pci_dev; 155 /* Device ID (i.e. PCI device ID) */ 156 uint32_t devid; 157 /* List of assigned LPIs */ 158 struct lpi_chunk lpis; 159 /* Virtual address of ITT */ 160 vm_offset_t itt; 161 size_t itt_size; 162 }; 163 164 /* 165 * ITS command descriptor. 166 * Idea for command description passing taken from Linux. 167 */ 168 struct its_cmd_desc { 169 uint8_t cmd_type; 170 171 union { 172 struct { 173 struct its_dev *its_dev; 174 struct its_col *col; 175 uint32_t id; 176 } cmd_desc_movi; 177 178 struct { 179 struct its_col *col; 180 } cmd_desc_sync; 181 182 struct { 183 struct its_col *col; 184 uint8_t valid; 185 } cmd_desc_mapc; 186 187 struct { 188 struct its_dev *its_dev; 189 struct its_col *col; 190 uint32_t pid; 191 uint32_t id; 192 } cmd_desc_mapvi; 193 194 struct { 195 struct its_dev *its_dev; 196 struct its_col *col; 197 uint32_t pid; 198 } cmd_desc_mapi; 199 200 struct { 201 struct its_dev *its_dev; 202 uint8_t valid; 203 } cmd_desc_mapd; 204 205 struct { 206 struct its_dev *its_dev; 207 struct its_col *col; 208 uint32_t pid; 209 } cmd_desc_inv; 210 211 struct { 212 struct its_col *col; 213 } cmd_desc_invall; 214 }; 215 }; 216 217 /* ITS command. Each command is 32 bytes long */ 218 struct its_cmd { 219 uint64_t cmd_dword[4]; /* ITS command double word */ 220 }; 221 222 /* An ITS private table */ 223 struct its_ptable { 224 vm_offset_t ptab_vaddr; 225 /* Size of the L1 and L2 tables */ 226 size_t ptab_l1_size; 227 size_t ptab_l2_size; 228 /* Number of L1 and L2 entries */ 229 int ptab_l1_nidents; 230 int ptab_l2_nidents; 231 232 int ptab_page_size; 233 int ptab_share; 234 bool ptab_indirect; 235 }; 236 237 /* ITS collection description. */ 238 struct its_col { 239 uint64_t col_target; /* Target Re-Distributor */ 240 uint64_t col_id; /* Collection ID */ 241 }; 242 243 struct gicv3_its_irqsrc { 244 struct intr_irqsrc gi_isrc; 245 u_int gi_id; 246 u_int gi_lpi; 247 struct its_dev *gi_its_dev; 248 TAILQ_ENTRY(gicv3_its_irqsrc) gi_link; 249 }; 250 251 struct gicv3_its_softc { 252 device_t dev; 253 struct intr_pic *sc_pic; 254 struct resource *sc_its_res; 255 256 cpuset_t sc_cpus; 257 struct domainset *sc_ds; 258 u_int gic_irq_cpu; 259 int sc_devbits; 260 int sc_dev_table_idx; 261 262 struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 263 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 264 265 /* 266 * TODO: We should get these from the parent as we only want a 267 * single copy of each across the interrupt controller. 268 */ 269 uint8_t *sc_conf_base; 270 vm_offset_t sc_pend_base[MAXCPU]; 271 272 /* Command handling */ 273 struct mtx sc_its_cmd_lock; 274 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 275 size_t sc_its_cmd_next_idx; 276 277 vmem_t *sc_irq_alloc; 278 struct gicv3_its_irqsrc **sc_irqs; 279 u_int sc_irq_base; 280 u_int sc_irq_length; 281 u_int sc_irq_count; 282 283 struct mtx sc_its_dev_lock; 284 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 285 TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs; 286 287 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001 288 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 289 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 290 u_int sc_its_flags; 291 bool trace_enable; 292 vm_page_t ma; /* fake msi page */ 293 }; 294 295 static void *conf_base; 296 297 typedef void (its_quirk_func_t)(device_t); 298 static its_quirk_func_t its_quirk_cavium_22375; 299 300 static const struct { 301 const char *desc; 302 uint32_t iidr; 303 uint32_t iidr_mask; 304 its_quirk_func_t *func; 305 } its_quirks[] = { 306 { 307 /* Cavium ThunderX Pass 1.x */ 308 .desc = "Cavium ThunderX errata: 22375, 24313", 309 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 310 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 311 .iidr_mask = ~GITS_IIDR_REVISION_MASK, 312 .func = its_quirk_cavium_22375, 313 }, 314 }; 315 316 #define gic_its_read_4(sc, reg) \ 317 bus_read_4((sc)->sc_its_res, (reg)) 318 #define gic_its_read_8(sc, reg) \ 319 bus_read_8((sc)->sc_its_res, (reg)) 320 321 #define gic_its_write_4(sc, reg, val) \ 322 bus_write_4((sc)->sc_its_res, (reg), (val)) 323 #define gic_its_write_8(sc, reg, val) \ 324 bus_write_8((sc)->sc_its_res, (reg), (val)) 325 326 static device_attach_t gicv3_its_attach; 327 static device_detach_t gicv3_its_detach; 328 329 static pic_disable_intr_t gicv3_its_disable_intr; 330 static pic_enable_intr_t gicv3_its_enable_intr; 331 static pic_map_intr_t gicv3_its_map_intr; 332 static pic_setup_intr_t gicv3_its_setup_intr; 333 static pic_post_filter_t gicv3_its_post_filter; 334 static pic_post_ithread_t gicv3_its_post_ithread; 335 static pic_pre_ithread_t gicv3_its_pre_ithread; 336 static pic_bind_intr_t gicv3_its_bind_intr; 337 #ifdef SMP 338 static pic_init_secondary_t gicv3_its_init_secondary; 339 #endif 340 static msi_alloc_msi_t gicv3_its_alloc_msi; 341 static msi_release_msi_t gicv3_its_release_msi; 342 static msi_alloc_msix_t gicv3_its_alloc_msix; 343 static msi_release_msix_t gicv3_its_release_msix; 344 static msi_map_msi_t gicv3_its_map_msi; 345 #ifdef IOMMU 346 static msi_iommu_init_t gicv3_iommu_init; 347 static msi_iommu_deinit_t gicv3_iommu_deinit; 348 #endif 349 350 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 351 static void its_cmd_mapc(device_t, struct its_col *, uint8_t); 352 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 353 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 354 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 355 static void its_cmd_invall(device_t, struct its_col *); 356 357 static device_method_t gicv3_its_methods[] = { 358 /* Device interface */ 359 DEVMETHOD(device_detach, gicv3_its_detach), 360 361 /* Interrupt controller interface */ 362 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 363 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 364 DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 365 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 366 DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 367 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 368 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 369 #ifdef SMP 370 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 371 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 372 #endif 373 374 /* MSI/MSI-X */ 375 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 376 DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 377 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 378 DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 379 DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 380 #ifdef IOMMU 381 DEVMETHOD(msi_iommu_init, gicv3_iommu_init), 382 DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit), 383 #endif 384 385 /* End */ 386 DEVMETHOD_END 387 }; 388 389 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 390 sizeof(struct gicv3_its_softc)); 391 392 static void 393 gicv3_its_cmdq_init(struct gicv3_its_softc *sc) 394 { 395 vm_paddr_t cmd_paddr; 396 uint64_t reg, tmp; 397 398 /* Set up the command circular buffer */ 399 sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS, 400 sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 401 0); 402 sc->sc_its_cmd_next_idx = 0; 403 404 cmd_paddr = vtophys(sc->sc_its_cmd_base); 405 406 /* Set the base of the command buffer */ 407 reg = GITS_CBASER_VALID | 408 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 409 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 410 (ITS_CMDQ_SIZE / 4096 - 1); 411 gic_its_write_8(sc, GITS_CBASER, reg); 412 413 /* Read back to check for fixed value fields */ 414 tmp = gic_its_read_8(sc, GITS_CBASER); 415 416 if ((tmp & GITS_CBASER_SHARE_MASK) != 417 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 418 /* Check if the hardware reported non-shareable */ 419 if ((tmp & GITS_CBASER_SHARE_MASK) == 420 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 421 /* If so remove the cache attribute */ 422 reg &= ~GITS_CBASER_CACHE_MASK; 423 reg &= ~GITS_CBASER_SHARE_MASK; 424 /* Set to Non-cacheable, Non-shareable */ 425 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 426 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 427 428 gic_its_write_8(sc, GITS_CBASER, reg); 429 } 430 431 /* The command queue has to be flushed after each command */ 432 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 433 } 434 435 /* Get the next command from the start of the buffer */ 436 gic_its_write_8(sc, GITS_CWRITER, 0x0); 437 } 438 439 static int 440 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table) 441 { 442 uint64_t reg, tmp; 443 int page_size; 444 445 page_size = PAGE_SIZE_64K; 446 reg = gic_its_read_8(sc, GITS_BASER(table)); 447 448 while (1) { 449 reg &= GITS_BASER_PSZ_MASK; 450 switch (page_size) { 451 case PAGE_SIZE_4K: /* 4KB */ 452 reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 453 break; 454 case PAGE_SIZE_16K: /* 16KB */ 455 reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 456 break; 457 case PAGE_SIZE_64K: /* 64KB */ 458 reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 459 break; 460 } 461 462 /* Write the new page size */ 463 gic_its_write_8(sc, GITS_BASER(table), reg); 464 465 /* Read back to check */ 466 tmp = gic_its_read_8(sc, GITS_BASER(table)); 467 468 /* The page size is correct */ 469 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK)) 470 return (page_size); 471 472 switch (page_size) { 473 default: 474 return (-1); 475 case PAGE_SIZE_16K: 476 page_size = PAGE_SIZE_4K; 477 break; 478 case PAGE_SIZE_64K: 479 page_size = PAGE_SIZE_16K; 480 break; 481 } 482 } 483 } 484 485 static bool 486 gicv3_its_table_supports_indirect(struct gicv3_its_softc *sc, int table) 487 { 488 uint64_t reg; 489 490 reg = gic_its_read_8(sc, GITS_BASER(table)); 491 492 /* Try setting the indirect flag */ 493 reg |= GITS_BASER_INDIRECT; 494 gic_its_write_8(sc, GITS_BASER(table), reg); 495 496 /* Read back to check */ 497 reg = gic_its_read_8(sc, GITS_BASER(table)); 498 return ((reg & GITS_BASER_INDIRECT) != 0); 499 } 500 501 502 static int 503 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 504 { 505 vm_offset_t table; 506 vm_paddr_t paddr; 507 uint64_t cache, reg, share, tmp, type; 508 size_t its_tbl_size, nitspages, npages; 509 size_t l1_esize, l2_esize, l1_nidents, l2_nidents; 510 int i, page_size; 511 int devbits; 512 bool indirect; 513 514 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 515 /* 516 * GITS_TYPER[17:13] of ThunderX reports that device IDs 517 * are to be 21 bits in length. The entry size of the ITS 518 * table can be read from GITS_BASERn[52:48] and on ThunderX 519 * is supposed to be 8 bytes in length (for device table). 520 * Finally the page size that is to be used by ITS to access 521 * this table will be set to 64KB. 522 * 523 * This gives 0x200000 entries of size 0x8 bytes covered by 524 * 256 pages each of which 64KB in size. The number of pages 525 * (minus 1) should then be written to GITS_BASERn[7:0]. In 526 * that case this value would be 0xFF but on ThunderX the 527 * maximum value that HW accepts is 0xFD. 528 * 529 * Set an arbitrary number of device ID bits to 20 in order 530 * to limit the number of entries in ITS device table to 531 * 0x100000 and the table size to 8MB. 532 */ 533 devbits = 20; 534 cache = 0; 535 } else { 536 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 537 cache = GITS_BASER_CACHE_WAWB; 538 } 539 sc->sc_devbits = devbits; 540 share = GITS_BASER_SHARE_IS; 541 542 for (i = 0; i < GITS_BASER_NUM; i++) { 543 reg = gic_its_read_8(sc, GITS_BASER(i)); 544 /* The type of table */ 545 type = GITS_BASER_TYPE(reg); 546 if (type == GITS_BASER_TYPE_UNIMPL) 547 continue; 548 549 /* The table entry size */ 550 l1_esize = GITS_BASER_ESIZE(reg); 551 552 /* Find the tables page size */ 553 page_size = gicv3_its_table_page_size(sc, i); 554 if (page_size == -1) { 555 device_printf(dev, "No valid page size for table %d\n", 556 i); 557 return (EINVAL); 558 } 559 560 indirect = false; 561 l2_nidents = 0; 562 l2_esize = 0; 563 switch(type) { 564 case GITS_BASER_TYPE_DEV: 565 if (sc->sc_dev_table_idx != -1) 566 device_printf(dev, 567 "Warning: Multiple device tables found\n"); 568 569 sc->sc_dev_table_idx = i; 570 l1_nidents = (1 << devbits); 571 if ((l1_esize * l1_nidents) > (page_size * 2)) { 572 indirect = 573 gicv3_its_table_supports_indirect(sc, i); 574 if (indirect) { 575 /* 576 * Each l1 entry is 8 bytes and points 577 * to an l2 table of size page_size. 578 * Calculate how many entries this is 579 * and use this to find how many 580 * 8 byte l1 idents we need. 581 */ 582 l2_esize = l1_esize; 583 l2_nidents = page_size / l2_esize; 584 l1_nidents = l1_nidents / l2_nidents; 585 l1_esize = GITS_INDIRECT_L1_ESIZE; 586 } 587 } 588 its_tbl_size = l1_esize * l1_nidents; 589 its_tbl_size = roundup2(its_tbl_size, page_size); 590 break; 591 case GITS_BASER_TYPE_VP: 592 case GITS_BASER_TYPE_PP: /* Undocumented? */ 593 case GITS_BASER_TYPE_IC: 594 its_tbl_size = page_size; 595 break; 596 default: 597 if (bootverbose) 598 device_printf(dev, "Unhandled table type %lx\n", 599 type); 600 continue; 601 } 602 npages = howmany(its_tbl_size, PAGE_SIZE); 603 604 /* Allocate the table */ 605 table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE, 606 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, 607 (1ul << 48) - 1, PAGE_SIZE_64K, 0); 608 609 sc->sc_its_ptab[i].ptab_vaddr = table; 610 sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size; 611 sc->sc_its_ptab[i].ptab_l1_nidents = l1_nidents; 612 sc->sc_its_ptab[i].ptab_l2_size = page_size; 613 sc->sc_its_ptab[i].ptab_l2_nidents = l2_nidents; 614 615 sc->sc_its_ptab[i].ptab_indirect = indirect; 616 sc->sc_its_ptab[i].ptab_page_size = page_size; 617 618 paddr = vtophys(table); 619 620 while (1) { 621 nitspages = howmany(its_tbl_size, page_size); 622 623 /* Clear the fields we will be setting */ 624 reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT | 625 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 626 GITS_BASER_PA_MASK | 627 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 628 GITS_BASER_SIZE_MASK); 629 /* Set the new values */ 630 reg |= GITS_BASER_VALID | 631 (indirect ? GITS_BASER_INDIRECT : 0) | 632 (cache << GITS_BASER_CACHE_SHIFT) | 633 (type << GITS_BASER_TYPE_SHIFT) | 634 paddr | (share << GITS_BASER_SHARE_SHIFT) | 635 (nitspages - 1); 636 637 switch (page_size) { 638 case PAGE_SIZE_4K: /* 4KB */ 639 reg |= 640 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 641 break; 642 case PAGE_SIZE_16K: /* 16KB */ 643 reg |= 644 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 645 break; 646 case PAGE_SIZE_64K: /* 64KB */ 647 reg |= 648 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 649 break; 650 } 651 652 gic_its_write_8(sc, GITS_BASER(i), reg); 653 654 /* Read back to check */ 655 tmp = gic_its_read_8(sc, GITS_BASER(i)); 656 657 /* Do the shareability masks line up? */ 658 if ((tmp & GITS_BASER_SHARE_MASK) != 659 (reg & GITS_BASER_SHARE_MASK)) { 660 share = (tmp & GITS_BASER_SHARE_MASK) >> 661 GITS_BASER_SHARE_SHIFT; 662 continue; 663 } 664 665 if (tmp != reg) { 666 device_printf(dev, "GITS_BASER%d: " 667 "unable to be updated: %lx != %lx\n", 668 i, reg, tmp); 669 return (ENXIO); 670 } 671 672 sc->sc_its_ptab[i].ptab_share = share; 673 /* We should have made all needed changes */ 674 break; 675 } 676 } 677 678 return (0); 679 } 680 681 static void 682 gicv3_its_conftable_init(struct gicv3_its_softc *sc) 683 { 684 void *conf_table; 685 686 conf_table = atomic_load_ptr(&conf_base); 687 if (conf_table == NULL) { 688 conf_table = contigmalloc(LPI_CONFTAB_SIZE, 689 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, 690 LPI_CONFTAB_ALIGN, 0); 691 692 if (atomic_cmpset_ptr((uintptr_t *)&conf_base, 693 (uintptr_t)NULL, (uintptr_t)conf_table) == 0) { 694 contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS); 695 conf_table = atomic_load_ptr(&conf_base); 696 } 697 } 698 sc->sc_conf_base = conf_table; 699 700 /* Set the default configuration */ 701 memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 702 LPI_CONFTAB_SIZE); 703 704 /* Flush the table to memory */ 705 cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE); 706 } 707 708 static void 709 gicv3_its_pendtables_init(struct gicv3_its_softc *sc) 710 { 711 int i; 712 713 for (i = 0; i <= mp_maxid; i++) { 714 if (CPU_ISSET(i, &sc->sc_cpus) == 0) 715 continue; 716 717 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( 718 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 719 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 720 721 /* Flush so the ITS can see the memory */ 722 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i], 723 LPI_PENDTAB_SIZE); 724 } 725 } 726 727 static void 728 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) 729 { 730 device_t gicv3; 731 uint64_t xbaser, tmp; 732 uint32_t ctlr; 733 u_int cpuid; 734 735 gicv3 = device_get_parent(dev); 736 cpuid = PCPU_GET(cpuid); 737 738 /* Disable LPIs */ 739 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 740 ctlr &= ~GICR_CTLR_LPI_ENABLE; 741 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 742 743 /* Make sure changes are observable my the GIC */ 744 dsb(sy); 745 746 /* 747 * Set the redistributor base 748 */ 749 xbaser = vtophys(sc->sc_conf_base) | 750 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 751 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 752 (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 753 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 754 755 /* Check the cache attributes we set */ 756 tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 757 758 if ((tmp & GICR_PROPBASER_SHARE_MASK) != 759 (xbaser & GICR_PROPBASER_SHARE_MASK)) { 760 if ((tmp & GICR_PROPBASER_SHARE_MASK) == 761 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 762 /* We need to mark as non-cacheable */ 763 xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 764 GICR_PROPBASER_CACHE_MASK); 765 /* Non-cacheable */ 766 xbaser |= GICR_PROPBASER_CACHE_NIN << 767 GICR_PROPBASER_CACHE_SHIFT; 768 /* Non-shareable */ 769 xbaser |= GICR_PROPBASER_SHARE_NS << 770 GICR_PROPBASER_SHARE_SHIFT; 771 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 772 } 773 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 774 } 775 776 /* 777 * Set the LPI pending table base 778 */ 779 xbaser = vtophys(sc->sc_pend_base[cpuid]) | 780 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 781 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 782 783 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 784 785 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 786 787 if ((tmp & GICR_PENDBASER_SHARE_MASK) == 788 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 789 /* Clear the cahce and shareability bits */ 790 xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 791 GICR_PENDBASER_SHARE_MASK); 792 /* Mark as non-shareable */ 793 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 794 /* And non-cacheable */ 795 xbaser |= GICR_PENDBASER_CACHE_NIN << 796 GICR_PENDBASER_CACHE_SHIFT; 797 } 798 799 /* Enable LPIs */ 800 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 801 ctlr |= GICR_CTLR_LPI_ENABLE; 802 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 803 804 /* Make sure the GIC has seen everything */ 805 dsb(sy); 806 } 807 808 static int 809 its_init_cpu(device_t dev, struct gicv3_its_softc *sc) 810 { 811 device_t gicv3; 812 vm_paddr_t target; 813 u_int cpuid; 814 struct redist_pcpu *rpcpu; 815 816 gicv3 = device_get_parent(dev); 817 cpuid = PCPU_GET(cpuid); 818 if (!CPU_ISSET(cpuid, &sc->sc_cpus)) 819 return (0); 820 821 /* Check if the ITS is enabled on this CPU */ 822 if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) 823 return (ENXIO); 824 825 rpcpu = gicv3_get_redist(dev); 826 827 /* Do per-cpu LPI init once */ 828 if (!rpcpu->lpi_enabled) { 829 its_init_cpu_lpi(dev, sc); 830 rpcpu->lpi_enabled = true; 831 } 832 833 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 834 /* This ITS wants the redistributor physical address */ 835 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) + 836 rpcpu->offset); 837 } else { 838 /* This ITS wants the unique processor number */ 839 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) << 840 CMD_TARGET_SHIFT; 841 } 842 843 sc->sc_its_cols[cpuid]->col_target = target; 844 sc->sc_its_cols[cpuid]->col_id = cpuid; 845 846 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 847 its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 848 849 return (0); 850 } 851 852 static int 853 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS) 854 { 855 struct gicv3_its_softc *sc; 856 int rv; 857 858 sc = arg1; 859 860 rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req); 861 if (rv != 0 || req->newptr == NULL) 862 return (rv); 863 if (sc->trace_enable) 864 gic_its_write_8(sc, GITS_TRKCTLR, 3); 865 else 866 gic_its_write_8(sc, GITS_TRKCTLR, 0); 867 868 return (0); 869 } 870 871 static int 872 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS) 873 { 874 struct gicv3_its_softc *sc; 875 struct sbuf *sb; 876 int err; 877 878 sc = arg1; 879 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 880 if (sb == NULL) { 881 device_printf(sc->dev, "Could not allocate sbuf for output.\n"); 882 return (ENOMEM); 883 } 884 sbuf_cat(sb, "\n"); 885 sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n", 886 gic_its_read_4(sc, GITS_TRKCTLR)); 887 sbuf_printf(sb, "GITS_TRKR: 0x%08X\n", 888 gic_its_read_4(sc, GITS_TRKR)); 889 sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n", 890 gic_its_read_4(sc, GITS_TRKDIDR)); 891 sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n", 892 gic_its_read_4(sc, GITS_TRKPIDR)); 893 sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n", 894 gic_its_read_4(sc, GITS_TRKVIDR)); 895 sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n", 896 gic_its_read_4(sc, GITS_TRKTGTR)); 897 898 err = sbuf_finish(sb); 899 if (err) 900 device_printf(sc->dev, "Error finishing sbuf: %d\n", err); 901 sbuf_delete(sb); 902 return(err); 903 } 904 905 static int 906 gicv3_its_init_sysctl(struct gicv3_its_softc *sc) 907 { 908 struct sysctl_oid *oid, *child; 909 struct sysctl_ctx_list *ctx_list; 910 911 ctx_list = device_get_sysctl_ctx(sc->dev); 912 child = device_get_sysctl_tree(sc->dev); 913 oid = SYSCTL_ADD_NODE(ctx_list, 914 SYSCTL_CHILDREN(child), OID_AUTO, "tracing", 915 CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing"); 916 if (oid == NULL) 917 return (ENXIO); 918 919 /* Add registers */ 920 SYSCTL_ADD_PROC(ctx_list, 921 SYSCTL_CHILDREN(oid), OID_AUTO, "enable", 922 CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 923 gicv3_its_sysctl_trace_enable, "CU", "Enable tracing"); 924 SYSCTL_ADD_PROC(ctx_list, 925 SYSCTL_CHILDREN(oid), OID_AUTO, "capture", 926 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 927 gicv3_its_sysctl_trace_regs, "", "Captured tracing registers."); 928 929 return (0); 930 } 931 932 static int 933 gicv3_its_attach(device_t dev) 934 { 935 struct gicv3_its_softc *sc; 936 int domain, err, i, rid; 937 uint64_t phys; 938 uint32_t ctlr, iidr; 939 940 sc = device_get_softc(dev); 941 942 sc->sc_dev_table_idx = -1; 943 sc->sc_irq_length = gicv3_get_nirqs(dev); 944 sc->sc_irq_base = GIC_FIRST_LPI; 945 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; 946 947 rid = 0; 948 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 949 RF_ACTIVE); 950 if (sc->sc_its_res == NULL) { 951 device_printf(dev, "Could not allocate memory\n"); 952 return (ENXIO); 953 } 954 955 phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) + 956 GITS_TRANSLATER, PAGE_SIZE); 957 sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO); 958 vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT); 959 960 CPU_COPY(&all_cpus, &sc->sc_cpus); 961 iidr = gic_its_read_4(sc, GITS_IIDR); 962 for (i = 0; i < nitems(its_quirks); i++) { 963 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 964 if (bootverbose) { 965 device_printf(dev, "Applying %s\n", 966 its_quirks[i].desc); 967 } 968 its_quirks[i].func(dev); 969 break; 970 } 971 } 972 973 if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) { 974 sc->sc_ds = DOMAINSET_PREF(domain); 975 } else { 976 sc->sc_ds = DOMAINSET_RR(); 977 } 978 979 /* 980 * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be 981 * coming in via, for instance, a kexec/kboot style setup where a 982 * previous kernel has configured then relinquished control. Clear it 983 * so that we can reconfigure GITS_BASER*. 984 */ 985 ctlr = gic_its_read_4(sc, GITS_CTLR); 986 if ((ctlr & GITS_CTLR_EN) != 0) { 987 ctlr &= ~GITS_CTLR_EN; 988 gic_its_write_4(sc, GITS_CTLR, ctlr); 989 } 990 991 /* Allocate the private tables */ 992 err = gicv3_its_table_init(dev, sc); 993 if (err != 0) 994 return (err); 995 996 /* Protects access to the device list */ 997 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 998 999 /* Protects access to the ITS command circular buffer. */ 1000 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 1001 1002 /* Allocate the command circular buffer */ 1003 gicv3_its_cmdq_init(sc); 1004 1005 /* Allocate the per-CPU collections */ 1006 for (int cpu = 0; cpu <= mp_maxid; cpu++) 1007 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0) 1008 sc->sc_its_cols[cpu] = malloc_domainset( 1009 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 1010 DOMAINSET_PREF(pcpu_find(cpu)->pc_domain), 1011 M_WAITOK | M_ZERO); 1012 else 1013 sc->sc_its_cols[cpu] = NULL; 1014 1015 /* Enable the ITS */ 1016 gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN); 1017 1018 /* Create the LPI configuration table */ 1019 gicv3_its_conftable_init(sc); 1020 1021 /* And the pending tebles */ 1022 gicv3_its_pendtables_init(sc); 1023 1024 /* Enable LPIs on this CPU */ 1025 its_init_cpu(dev, sc); 1026 1027 TAILQ_INIT(&sc->sc_its_dev_list); 1028 TAILQ_INIT(&sc->sc_free_irqs); 1029 1030 /* 1031 * Create the vmem object to allocate INTRNG IRQs from. We try to 1032 * use all IRQs not already used by the GICv3. 1033 * XXX: This assumes there are no other interrupt controllers in the 1034 * system. 1035 */ 1036 sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0, 1037 gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK); 1038 1039 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length, 1040 M_GICV3_ITS, M_WAITOK | M_ZERO); 1041 1042 /* For GIC-500 install tracking sysctls. */ 1043 if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) == 1044 GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0)) 1045 gicv3_its_init_sysctl(sc); 1046 1047 return (0); 1048 } 1049 1050 static int 1051 gicv3_its_detach(device_t dev) 1052 { 1053 1054 return (ENXIO); 1055 } 1056 1057 static void 1058 its_quirk_cavium_22375(device_t dev) 1059 { 1060 struct gicv3_its_softc *sc; 1061 int domain; 1062 1063 sc = device_get_softc(dev); 1064 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 1065 1066 /* 1067 * We need to limit which CPUs we send these interrupts to on 1068 * the original dual socket ThunderX as it is unable to 1069 * forward them between the two sockets. 1070 */ 1071 if (bus_get_domain(dev, &domain) == 0) { 1072 if (domain < MAXMEMDOM) { 1073 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus); 1074 } else { 1075 CPU_ZERO(&sc->sc_cpus); 1076 } 1077 } 1078 } 1079 1080 static void 1081 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 1082 { 1083 struct gicv3_its_softc *sc; 1084 struct gicv3_its_irqsrc *girq; 1085 uint8_t *conf; 1086 1087 sc = device_get_softc(dev); 1088 girq = (struct gicv3_its_irqsrc *)isrc; 1089 conf = sc->sc_conf_base; 1090 1091 conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE; 1092 1093 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 1094 /* Clean D-cache under command. */ 1095 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); 1096 } else { 1097 /* DSB inner shareable, store */ 1098 dsb(ishst); 1099 } 1100 1101 its_cmd_inv(dev, girq->gi_its_dev, girq); 1102 } 1103 1104 static void 1105 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 1106 { 1107 struct gicv3_its_softc *sc; 1108 struct gicv3_its_irqsrc *girq; 1109 uint8_t *conf; 1110 1111 sc = device_get_softc(dev); 1112 girq = (struct gicv3_its_irqsrc *)isrc; 1113 conf = sc->sc_conf_base; 1114 1115 conf[girq->gi_lpi] |= LPI_CONF_ENABLE; 1116 1117 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 1118 /* Clean D-cache under command. */ 1119 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); 1120 } else { 1121 /* DSB inner shareable, store */ 1122 dsb(ishst); 1123 } 1124 1125 its_cmd_inv(dev, girq->gi_its_dev, girq); 1126 } 1127 1128 static int 1129 gicv3_its_intr(void *arg, uintptr_t irq) 1130 { 1131 struct gicv3_its_softc *sc = arg; 1132 struct gicv3_its_irqsrc *girq; 1133 struct trapframe *tf; 1134 1135 irq -= sc->sc_irq_base; 1136 girq = sc->sc_irqs[irq]; 1137 if (girq == NULL) 1138 panic("gicv3_its_intr: Invalid interrupt %ld", 1139 irq + sc->sc_irq_base); 1140 1141 tf = curthread->td_intr_frame; 1142 intr_isrc_dispatch(&girq->gi_isrc, tf); 1143 return (FILTER_HANDLED); 1144 } 1145 1146 static void 1147 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 1148 { 1149 struct gicv3_its_irqsrc *girq; 1150 1151 girq = (struct gicv3_its_irqsrc *)isrc; 1152 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 1153 } 1154 1155 static void 1156 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 1157 { 1158 1159 } 1160 1161 static void 1162 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 1163 { 1164 struct gicv3_its_irqsrc *girq; 1165 1166 girq = (struct gicv3_its_irqsrc *)isrc; 1167 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 1168 } 1169 1170 static int 1171 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc) 1172 { 1173 struct gicv3_its_softc *sc; 1174 1175 sc = device_get_softc(dev); 1176 if (CPU_EMPTY(&isrc->isrc_cpu)) { 1177 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu, 1178 &sc->sc_cpus); 1179 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu); 1180 } 1181 1182 return (0); 1183 } 1184 1185 static int 1186 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 1187 { 1188 struct gicv3_its_irqsrc *girq; 1189 1190 gicv3_its_select_cpu(dev, isrc); 1191 1192 girq = (struct gicv3_its_irqsrc *)isrc; 1193 its_cmd_movi(dev, girq); 1194 return (0); 1195 } 1196 1197 static int 1198 gicv3_its_map_intr(device_t dev, struct intr_map_data *data, 1199 struct intr_irqsrc **isrcp) 1200 { 1201 1202 /* 1203 * This should never happen, we only call this function to map 1204 * interrupts found before the controller driver is ready. 1205 */ 1206 panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 1207 } 1208 1209 static int 1210 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 1211 struct resource *res, struct intr_map_data *data) 1212 { 1213 1214 /* Bind the interrupt to a CPU */ 1215 gicv3_its_bind_intr(dev, isrc); 1216 1217 return (0); 1218 } 1219 1220 #ifdef SMP 1221 static void 1222 gicv3_its_init_secondary(device_t dev) 1223 { 1224 struct gicv3_its_softc *sc; 1225 1226 sc = device_get_softc(dev); 1227 1228 /* 1229 * This is fatal as otherwise we may bind interrupts to this CPU. 1230 * We need a way to tell the interrupt framework to only bind to a 1231 * subset of given CPUs when it performs the shuffle. 1232 */ 1233 if (its_init_cpu(dev, sc) != 0) 1234 panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 1235 PCPU_GET(cpuid)); 1236 } 1237 #endif 1238 1239 static uint32_t 1240 its_get_devid(device_t pci_dev) 1241 { 1242 uintptr_t id; 1243 1244 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 1245 panic("%s: %s: Unable to get the MSI DeviceID", __func__, 1246 device_get_nameunit(pci_dev)); 1247 1248 return (id); 1249 } 1250 1251 static struct its_dev * 1252 its_device_find(device_t dev, device_t child) 1253 { 1254 struct gicv3_its_softc *sc; 1255 struct its_dev *its_dev = NULL; 1256 1257 sc = device_get_softc(dev); 1258 1259 mtx_lock_spin(&sc->sc_its_dev_lock); 1260 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 1261 if (its_dev->pci_dev == child) 1262 break; 1263 } 1264 mtx_unlock_spin(&sc->sc_its_dev_lock); 1265 1266 return (its_dev); 1267 } 1268 1269 static bool 1270 its_device_alloc(struct gicv3_its_softc *sc, int devid) 1271 { 1272 struct its_ptable *ptable; 1273 vm_offset_t l2_table; 1274 uint64_t *table; 1275 uint32_t index; 1276 bool shareable; 1277 1278 /* No device table */ 1279 if (sc->sc_dev_table_idx < 0) { 1280 if (devid >= (1 << sc->sc_devbits)) { 1281 if (bootverbose) { 1282 device_printf(sc->dev, 1283 "%s: Device out of range for hardware " 1284 "(%x >= %x)\n", __func__, devid, 1285 1 << sc->sc_devbits); 1286 } 1287 return (false); 1288 } 1289 return (true); 1290 } 1291 1292 ptable = &sc->sc_its_ptab[sc->sc_dev_table_idx]; 1293 /* Check the devid is within the table limit */ 1294 if (!ptable->ptab_indirect) { 1295 if (devid >= ptable->ptab_l1_nidents) { 1296 if (bootverbose) { 1297 device_printf(sc->dev, 1298 "%s: Device out of range for table " 1299 "(%x >= %x)\n", __func__, devid, 1300 ptable->ptab_l1_nidents); 1301 } 1302 return (false); 1303 } 1304 1305 return (true); 1306 } 1307 1308 /* Check the devid is within the allocated range */ 1309 index = devid / ptable->ptab_l2_nidents; 1310 if (index >= ptable->ptab_l1_nidents) { 1311 if (bootverbose) { 1312 device_printf(sc->dev, 1313 "%s: Index out of range for table (%x >= %x)\n", 1314 __func__, index, ptable->ptab_l1_nidents); 1315 } 1316 return (false); 1317 } 1318 1319 table = (uint64_t *)ptable->ptab_vaddr; 1320 /* We have an second level table */ 1321 if ((table[index] & GITS_BASER_VALID) != 0) 1322 return (true); 1323 1324 shareable = true; 1325 if ((ptable->ptab_share & GITS_BASER_SHARE_MASK) == GITS_BASER_SHARE_NS) 1326 shareable = false; 1327 1328 l2_table = (vm_offset_t)contigmalloc_domainset(ptable->ptab_l2_size, 1329 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, 1330 ptable->ptab_page_size, 0); 1331 1332 if (!shareable) 1333 cpu_dcache_wb_range((vm_offset_t)l2_table, 1334 ptable->ptab_l2_size); 1335 1336 table[index] = vtophys(l2_table) | GITS_BASER_VALID; 1337 if (!shareable) 1338 cpu_dcache_wb_range((vm_offset_t)&table[index], 1339 sizeof(table[index])); 1340 1341 dsb(sy); 1342 return (true); 1343 } 1344 1345 static struct its_dev * 1346 its_device_get(device_t dev, device_t child, u_int nvecs) 1347 { 1348 struct gicv3_its_softc *sc; 1349 struct its_dev *its_dev; 1350 vmem_addr_t irq_base; 1351 size_t esize; 1352 1353 sc = device_get_softc(dev); 1354 1355 its_dev = its_device_find(dev, child); 1356 if (its_dev != NULL) 1357 return (its_dev); 1358 1359 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 1360 if (its_dev == NULL) 1361 return (NULL); 1362 1363 its_dev->pci_dev = child; 1364 its_dev->devid = its_get_devid(child); 1365 1366 its_dev->lpis.lpi_busy = 0; 1367 its_dev->lpis.lpi_num = nvecs; 1368 its_dev->lpis.lpi_free = nvecs; 1369 1370 if (!its_device_alloc(sc, its_dev->devid)) { 1371 free(its_dev, M_GICV3_ITS); 1372 return (NULL); 1373 } 1374 1375 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 1376 &irq_base) != 0) { 1377 free(its_dev, M_GICV3_ITS); 1378 return (NULL); 1379 } 1380 its_dev->lpis.lpi_base = irq_base; 1381 1382 /* Get ITT entry size */ 1383 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 1384 1385 /* 1386 * Allocate ITT for this device. 1387 * PA has to be 256 B aligned. At least two entries for device. 1388 */ 1389 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 1390 its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size, 1391 M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0, 1392 LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0); 1393 if (its_dev->itt == 0) { 1394 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 1395 free(its_dev, M_GICV3_ITS); 1396 return (NULL); 1397 } 1398 1399 /* Make sure device sees zeroed ITT. */ 1400 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) 1401 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size); 1402 1403 mtx_lock_spin(&sc->sc_its_dev_lock); 1404 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 1405 mtx_unlock_spin(&sc->sc_its_dev_lock); 1406 1407 /* Map device to its ITT */ 1408 its_cmd_mapd(dev, its_dev, 1); 1409 1410 return (its_dev); 1411 } 1412 1413 static void 1414 its_device_release(device_t dev, struct its_dev *its_dev) 1415 { 1416 struct gicv3_its_softc *sc; 1417 1418 KASSERT(its_dev->lpis.lpi_busy == 0, 1419 ("its_device_release: Trying to release an inuse ITS device")); 1420 1421 /* Unmap device in ITS */ 1422 its_cmd_mapd(dev, its_dev, 0); 1423 1424 sc = device_get_softc(dev); 1425 1426 /* Remove the device from the list of devices */ 1427 mtx_lock_spin(&sc->sc_its_dev_lock); 1428 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 1429 mtx_unlock_spin(&sc->sc_its_dev_lock); 1430 1431 /* Free ITT */ 1432 KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); 1433 contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 1434 1435 /* Free the IRQ allocation */ 1436 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 1437 its_dev->lpis.lpi_num); 1438 1439 free(its_dev, M_GICV3_ITS); 1440 } 1441 1442 static struct gicv3_its_irqsrc * 1443 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq) 1444 { 1445 struct gicv3_its_irqsrc *girq = NULL; 1446 1447 KASSERT(sc->sc_irqs[irq] == NULL, 1448 ("%s: Interrupt %u already allocated", __func__, irq)); 1449 mtx_lock_spin(&sc->sc_its_dev_lock); 1450 if (!TAILQ_EMPTY(&sc->sc_free_irqs)) { 1451 girq = TAILQ_FIRST(&sc->sc_free_irqs); 1452 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link); 1453 } 1454 mtx_unlock_spin(&sc->sc_its_dev_lock); 1455 if (girq == NULL) { 1456 girq = malloc(sizeof(*girq), M_GICV3_ITS, 1457 M_NOWAIT | M_ZERO); 1458 if (girq == NULL) 1459 return (NULL); 1460 girq->gi_id = -1; 1461 if (intr_isrc_register(&girq->gi_isrc, dev, 0, 1462 "%s,%u", device_get_nameunit(dev), irq) != 0) { 1463 free(girq, M_GICV3_ITS); 1464 return (NULL); 1465 } 1466 } 1467 girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI; 1468 sc->sc_irqs[irq] = girq; 1469 1470 return (girq); 1471 } 1472 1473 static void 1474 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc, 1475 struct gicv3_its_irqsrc *girq) 1476 { 1477 u_int irq; 1478 1479 mtx_assert(&sc->sc_its_dev_lock, MA_OWNED); 1480 1481 irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base; 1482 sc->sc_irqs[irq] = NULL; 1483 1484 girq->gi_id = -1; 1485 girq->gi_its_dev = NULL; 1486 TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link); 1487 } 1488 1489 static int 1490 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 1491 device_t *pic, struct intr_irqsrc **srcs) 1492 { 1493 struct gicv3_its_softc *sc; 1494 struct gicv3_its_irqsrc *girq; 1495 struct its_dev *its_dev; 1496 u_int irq; 1497 int i; 1498 1499 its_dev = its_device_get(dev, child, count); 1500 if (its_dev == NULL) 1501 return (ENXIO); 1502 1503 KASSERT(its_dev->lpis.lpi_free >= count, 1504 ("gicv3_its_alloc_msi: No free LPIs")); 1505 sc = device_get_softc(dev); 1506 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1507 its_dev->lpis.lpi_free; 1508 1509 /* Allocate the irqsrc for each MSI */ 1510 for (i = 0; i < count; i++, irq++) { 1511 its_dev->lpis.lpi_free--; 1512 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev, 1513 sc, irq); 1514 if (srcs[i] == NULL) 1515 break; 1516 } 1517 1518 /* The allocation failed, release them */ 1519 if (i != count) { 1520 mtx_lock_spin(&sc->sc_its_dev_lock); 1521 for (i = 0; i < count; i++) { 1522 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1523 if (girq == NULL) 1524 break; 1525 gicv3_its_release_irqsrc(sc, girq); 1526 srcs[i] = NULL; 1527 } 1528 mtx_unlock_spin(&sc->sc_its_dev_lock); 1529 return (ENXIO); 1530 } 1531 1532 /* Finish the allocation now we have all MSI irqsrcs */ 1533 for (i = 0; i < count; i++) { 1534 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1535 girq->gi_id = i; 1536 girq->gi_its_dev = its_dev; 1537 1538 /* Map the message to the given IRQ */ 1539 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1540 its_cmd_mapti(dev, girq); 1541 } 1542 its_dev->lpis.lpi_busy += count; 1543 *pic = dev; 1544 1545 return (0); 1546 } 1547 1548 static int 1549 gicv3_its_release_msi(device_t dev, device_t child, int count, 1550 struct intr_irqsrc **isrc) 1551 { 1552 struct gicv3_its_softc *sc; 1553 struct gicv3_its_irqsrc *girq; 1554 struct its_dev *its_dev; 1555 int i; 1556 1557 its_dev = its_device_find(dev, child); 1558 1559 KASSERT(its_dev != NULL, 1560 ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1561 "no ITS device")); 1562 KASSERT(its_dev->lpis.lpi_busy >= count, 1563 ("gicv3_its_release_msi: Releasing more interrupts than " 1564 "were allocated: releasing %d, allocated %d", count, 1565 its_dev->lpis.lpi_busy)); 1566 1567 sc = device_get_softc(dev); 1568 mtx_lock_spin(&sc->sc_its_dev_lock); 1569 for (i = 0; i < count; i++) { 1570 girq = (struct gicv3_its_irqsrc *)isrc[i]; 1571 gicv3_its_release_irqsrc(sc, girq); 1572 } 1573 mtx_unlock_spin(&sc->sc_its_dev_lock); 1574 its_dev->lpis.lpi_busy -= count; 1575 1576 if (its_dev->lpis.lpi_busy == 0) 1577 its_device_release(dev, its_dev); 1578 1579 return (0); 1580 } 1581 1582 static int 1583 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1584 struct intr_irqsrc **isrcp) 1585 { 1586 struct gicv3_its_softc *sc; 1587 struct gicv3_its_irqsrc *girq; 1588 struct its_dev *its_dev; 1589 u_int nvecs, irq; 1590 1591 nvecs = pci_msix_count(child); 1592 its_dev = its_device_get(dev, child, nvecs); 1593 if (its_dev == NULL) 1594 return (ENXIO); 1595 1596 KASSERT(its_dev->lpis.lpi_free > 0, 1597 ("gicv3_its_alloc_msix: No free LPIs")); 1598 sc = device_get_softc(dev); 1599 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1600 its_dev->lpis.lpi_free; 1601 1602 girq = gicv3_its_alloc_irqsrc(dev, sc, irq); 1603 if (girq == NULL) 1604 return (ENXIO); 1605 girq->gi_id = its_dev->lpis.lpi_busy; 1606 girq->gi_its_dev = its_dev; 1607 1608 its_dev->lpis.lpi_free--; 1609 its_dev->lpis.lpi_busy++; 1610 1611 /* Map the message to the given IRQ */ 1612 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1613 its_cmd_mapti(dev, girq); 1614 1615 *pic = dev; 1616 *isrcp = (struct intr_irqsrc *)girq; 1617 1618 return (0); 1619 } 1620 1621 static int 1622 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1623 { 1624 struct gicv3_its_softc *sc; 1625 struct gicv3_its_irqsrc *girq; 1626 struct its_dev *its_dev; 1627 1628 its_dev = its_device_find(dev, child); 1629 1630 KASSERT(its_dev != NULL, 1631 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1632 "no ITS device")); 1633 KASSERT(its_dev->lpis.lpi_busy > 0, 1634 ("gicv3_its_release_msix: Releasing more interrupts than " 1635 "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1636 1637 sc = device_get_softc(dev); 1638 girq = (struct gicv3_its_irqsrc *)isrc; 1639 mtx_lock_spin(&sc->sc_its_dev_lock); 1640 gicv3_its_release_irqsrc(sc, girq); 1641 mtx_unlock_spin(&sc->sc_its_dev_lock); 1642 its_dev->lpis.lpi_busy--; 1643 1644 if (its_dev->lpis.lpi_busy == 0) 1645 its_device_release(dev, its_dev); 1646 1647 return (0); 1648 } 1649 1650 static int 1651 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1652 uint64_t *addr, uint32_t *data) 1653 { 1654 struct gicv3_its_softc *sc; 1655 struct gicv3_its_irqsrc *girq; 1656 1657 sc = device_get_softc(dev); 1658 girq = (struct gicv3_its_irqsrc *)isrc; 1659 1660 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1661 *data = girq->gi_id; 1662 1663 return (0); 1664 } 1665 1666 #ifdef IOMMU 1667 static int 1668 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain) 1669 { 1670 struct gicv3_its_softc *sc; 1671 struct iommu_ctx *ctx; 1672 int error; 1673 1674 sc = device_get_softc(dev); 1675 ctx = iommu_get_dev_ctx(child); 1676 if (ctx == NULL) 1677 return (ENXIO); 1678 /* Map the page containing the GITS_TRANSLATER register. */ 1679 error = iommu_map_msi(ctx, PAGE_SIZE, 0, 1680 IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma); 1681 *domain = iommu_get_ctx_domain(ctx); 1682 1683 return (error); 1684 } 1685 1686 static void 1687 gicv3_iommu_deinit(device_t dev, device_t child) 1688 { 1689 struct iommu_ctx *ctx; 1690 1691 ctx = iommu_get_dev_ctx(child); 1692 if (ctx == NULL) 1693 return; 1694 1695 iommu_unmap_msi(ctx); 1696 } 1697 #endif 1698 1699 /* 1700 * Commands handling. 1701 */ 1702 1703 static __inline void 1704 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1705 { 1706 /* Command field: DW0 [7:0] */ 1707 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1708 cmd->cmd_dword[0] |= htole64(cmd_type); 1709 } 1710 1711 static __inline void 1712 cmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1713 { 1714 /* Device ID field: DW0 [63:32] */ 1715 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1716 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1717 } 1718 1719 static __inline void 1720 cmd_format_size(struct its_cmd *cmd, uint16_t size) 1721 { 1722 /* Size field: DW1 [4:0] */ 1723 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1724 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1725 } 1726 1727 static __inline void 1728 cmd_format_id(struct its_cmd *cmd, uint32_t id) 1729 { 1730 /* ID field: DW1 [31:0] */ 1731 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1732 cmd->cmd_dword[1] |= htole64(id); 1733 } 1734 1735 static __inline void 1736 cmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1737 { 1738 /* Physical ID field: DW1 [63:32] */ 1739 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1740 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1741 } 1742 1743 static __inline void 1744 cmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1745 { 1746 /* Collection field: DW2 [16:0] */ 1747 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1748 cmd->cmd_dword[2] |= htole64(col_id); 1749 } 1750 1751 static __inline void 1752 cmd_format_target(struct its_cmd *cmd, uint64_t target) 1753 { 1754 /* Target Address field: DW2 [47:16] */ 1755 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1756 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1757 } 1758 1759 static __inline void 1760 cmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1761 { 1762 /* ITT Address field: DW2 [47:8] */ 1763 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1764 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1765 } 1766 1767 static __inline void 1768 cmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1769 { 1770 /* Valid field: DW2 [63] */ 1771 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1772 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1773 } 1774 1775 static inline bool 1776 its_cmd_queue_full(struct gicv3_its_softc *sc) 1777 { 1778 size_t read_idx, next_write_idx; 1779 1780 /* Get the index of the next command */ 1781 next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1782 (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1783 /* And the index of the current command being read */ 1784 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1785 1786 /* 1787 * The queue is full when the write offset points 1788 * at the command before the current read offset. 1789 */ 1790 return (next_write_idx == read_idx); 1791 } 1792 1793 static inline void 1794 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1795 { 1796 1797 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1798 /* Clean D-cache under command. */ 1799 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); 1800 } else { 1801 /* DSB inner shareable, store */ 1802 dsb(ishst); 1803 } 1804 1805 } 1806 1807 static inline uint64_t 1808 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1809 { 1810 uint64_t off; 1811 1812 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1813 1814 return (off); 1815 } 1816 1817 static void 1818 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1819 struct its_cmd *cmd_last) 1820 { 1821 struct gicv3_its_softc *sc; 1822 uint64_t first, last, read; 1823 size_t us_left; 1824 1825 sc = device_get_softc(dev); 1826 1827 /* 1828 * XXX ARM64TODO: This is obviously a significant delay. 1829 * The reason for that is that currently the time frames for 1830 * the command to complete are not known. 1831 */ 1832 us_left = 1000000; 1833 1834 first = its_cmd_cwriter_offset(sc, cmd_first); 1835 last = its_cmd_cwriter_offset(sc, cmd_last); 1836 1837 for (;;) { 1838 read = gic_its_read_8(sc, GITS_CREADR); 1839 if (first < last) { 1840 if (read < first || read >= last) 1841 break; 1842 } else if (read < first && read >= last) 1843 break; 1844 1845 if (us_left-- == 0) { 1846 /* This means timeout */ 1847 device_printf(dev, 1848 "Timeout while waiting for CMD completion.\n"); 1849 return; 1850 } 1851 DELAY(1); 1852 } 1853 } 1854 1855 static struct its_cmd * 1856 its_cmd_alloc_locked(device_t dev) 1857 { 1858 struct gicv3_its_softc *sc; 1859 struct its_cmd *cmd; 1860 size_t us_left; 1861 1862 sc = device_get_softc(dev); 1863 1864 /* 1865 * XXX ARM64TODO: This is obviously a significant delay. 1866 * The reason for that is that currently the time frames for 1867 * the command to complete (and therefore free the descriptor) 1868 * are not known. 1869 */ 1870 us_left = 1000000; 1871 1872 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1873 while (its_cmd_queue_full(sc)) { 1874 if (us_left-- == 0) { 1875 /* Timeout while waiting for free command */ 1876 device_printf(dev, 1877 "Timeout while waiting for free command\n"); 1878 return (NULL); 1879 } 1880 DELAY(1); 1881 } 1882 1883 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1884 sc->sc_its_cmd_next_idx++; 1885 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1886 1887 return (cmd); 1888 } 1889 1890 static uint64_t 1891 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1892 { 1893 uint64_t target; 1894 uint8_t cmd_type; 1895 u_int size; 1896 1897 cmd_type = desc->cmd_type; 1898 target = ITS_TARGET_NONE; 1899 1900 switch (cmd_type) { 1901 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1902 target = desc->cmd_desc_movi.col->col_target; 1903 cmd_format_command(cmd, ITS_CMD_MOVI); 1904 cmd_format_id(cmd, desc->cmd_desc_movi.id); 1905 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1906 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1907 break; 1908 case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1909 target = desc->cmd_desc_sync.col->col_target; 1910 cmd_format_command(cmd, ITS_CMD_SYNC); 1911 cmd_format_target(cmd, target); 1912 break; 1913 case ITS_CMD_MAPD: /* Assign ITT to device */ 1914 cmd_format_command(cmd, ITS_CMD_MAPD); 1915 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1916 /* 1917 * Size describes number of bits to encode interrupt IDs 1918 * supported by the device minus one. 1919 * When V (valid) bit is zero, this field should be written 1920 * as zero. 1921 */ 1922 if (desc->cmd_desc_mapd.valid != 0) { 1923 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1924 size = MAX(1, size) - 1; 1925 } else 1926 size = 0; 1927 1928 cmd_format_size(cmd, size); 1929 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1930 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1931 break; 1932 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1933 target = desc->cmd_desc_mapc.col->col_target; 1934 cmd_format_command(cmd, ITS_CMD_MAPC); 1935 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 1936 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 1937 cmd_format_target(cmd, target); 1938 break; 1939 case ITS_CMD_MAPTI: 1940 target = desc->cmd_desc_mapvi.col->col_target; 1941 cmd_format_command(cmd, ITS_CMD_MAPTI); 1942 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 1943 cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 1944 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 1945 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 1946 break; 1947 case ITS_CMD_MAPI: 1948 target = desc->cmd_desc_mapi.col->col_target; 1949 cmd_format_command(cmd, ITS_CMD_MAPI); 1950 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 1951 cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 1952 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 1953 break; 1954 case ITS_CMD_INV: 1955 target = desc->cmd_desc_inv.col->col_target; 1956 cmd_format_command(cmd, ITS_CMD_INV); 1957 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 1958 cmd_format_id(cmd, desc->cmd_desc_inv.pid); 1959 break; 1960 case ITS_CMD_INVALL: 1961 cmd_format_command(cmd, ITS_CMD_INVALL); 1962 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 1963 break; 1964 default: 1965 panic("its_cmd_prepare: Invalid command: %x", cmd_type); 1966 } 1967 1968 return (target); 1969 } 1970 1971 static int 1972 its_cmd_send(device_t dev, struct its_cmd_desc *desc) 1973 { 1974 struct gicv3_its_softc *sc; 1975 struct its_cmd *cmd, *cmd_sync, *cmd_write; 1976 struct its_col col_sync; 1977 struct its_cmd_desc desc_sync; 1978 uint64_t target, cwriter; 1979 1980 sc = device_get_softc(dev); 1981 mtx_lock_spin(&sc->sc_its_cmd_lock); 1982 cmd = its_cmd_alloc_locked(dev); 1983 if (cmd == NULL) { 1984 device_printf(dev, "could not allocate ITS command\n"); 1985 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1986 return (EBUSY); 1987 } 1988 1989 target = its_cmd_prepare(cmd, desc); 1990 its_cmd_sync(sc, cmd); 1991 1992 if (target != ITS_TARGET_NONE) { 1993 cmd_sync = its_cmd_alloc_locked(dev); 1994 if (cmd_sync != NULL) { 1995 desc_sync.cmd_type = ITS_CMD_SYNC; 1996 col_sync.col_target = target; 1997 desc_sync.cmd_desc_sync.col = &col_sync; 1998 its_cmd_prepare(cmd_sync, &desc_sync); 1999 its_cmd_sync(sc, cmd_sync); 2000 } 2001 } 2002 2003 /* Update GITS_CWRITER */ 2004 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 2005 gic_its_write_8(sc, GITS_CWRITER, cwriter); 2006 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 2007 mtx_unlock_spin(&sc->sc_its_cmd_lock); 2008 2009 its_cmd_wait_completion(dev, cmd, cmd_write); 2010 2011 return (0); 2012 } 2013 2014 /* Handlers to send commands */ 2015 static void 2016 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 2017 { 2018 struct gicv3_its_softc *sc; 2019 struct its_cmd_desc desc; 2020 struct its_col *col; 2021 2022 sc = device_get_softc(dev); 2023 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 2024 2025 desc.cmd_type = ITS_CMD_MOVI; 2026 desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 2027 desc.cmd_desc_movi.col = col; 2028 desc.cmd_desc_movi.id = girq->gi_id; 2029 2030 its_cmd_send(dev, &desc); 2031 } 2032 2033 static void 2034 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 2035 { 2036 struct its_cmd_desc desc; 2037 2038 desc.cmd_type = ITS_CMD_MAPC; 2039 desc.cmd_desc_mapc.col = col; 2040 /* 2041 * Valid bit set - map the collection. 2042 * Valid bit cleared - unmap the collection. 2043 */ 2044 desc.cmd_desc_mapc.valid = valid; 2045 2046 its_cmd_send(dev, &desc); 2047 } 2048 2049 static void 2050 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 2051 { 2052 struct gicv3_its_softc *sc; 2053 struct its_cmd_desc desc; 2054 struct its_col *col; 2055 u_int col_id; 2056 2057 sc = device_get_softc(dev); 2058 2059 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 2060 col = sc->sc_its_cols[col_id]; 2061 2062 desc.cmd_type = ITS_CMD_MAPTI; 2063 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 2064 desc.cmd_desc_mapvi.col = col; 2065 /* The EventID sent to the device */ 2066 desc.cmd_desc_mapvi.id = girq->gi_id; 2067 /* The physical interrupt presented to softeware */ 2068 desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI; 2069 2070 its_cmd_send(dev, &desc); 2071 } 2072 2073 static void 2074 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 2075 { 2076 struct its_cmd_desc desc; 2077 2078 desc.cmd_type = ITS_CMD_MAPD; 2079 desc.cmd_desc_mapd.its_dev = its_dev; 2080 desc.cmd_desc_mapd.valid = valid; 2081 2082 its_cmd_send(dev, &desc); 2083 } 2084 2085 static void 2086 its_cmd_inv(device_t dev, struct its_dev *its_dev, 2087 struct gicv3_its_irqsrc *girq) 2088 { 2089 struct gicv3_its_softc *sc; 2090 struct its_cmd_desc desc; 2091 struct its_col *col; 2092 2093 sc = device_get_softc(dev); 2094 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 2095 2096 desc.cmd_type = ITS_CMD_INV; 2097 /* The EventID sent to the device */ 2098 desc.cmd_desc_inv.pid = girq->gi_id; 2099 desc.cmd_desc_inv.its_dev = its_dev; 2100 desc.cmd_desc_inv.col = col; 2101 2102 its_cmd_send(dev, &desc); 2103 } 2104 2105 static void 2106 its_cmd_invall(device_t dev, struct its_col *col) 2107 { 2108 struct its_cmd_desc desc; 2109 2110 desc.cmd_type = ITS_CMD_INVALL; 2111 desc.cmd_desc_invall.col = col; 2112 2113 its_cmd_send(dev, &desc); 2114 } 2115 2116 #ifdef FDT 2117 static device_probe_t gicv3_its_fdt_probe; 2118 static device_attach_t gicv3_its_fdt_attach; 2119 2120 static device_method_t gicv3_its_fdt_methods[] = { 2121 /* Device interface */ 2122 DEVMETHOD(device_probe, gicv3_its_fdt_probe), 2123 DEVMETHOD(device_attach, gicv3_its_fdt_attach), 2124 2125 /* End */ 2126 DEVMETHOD_END 2127 }; 2128 2129 #define its_baseclasses its_fdt_baseclasses 2130 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 2131 sizeof(struct gicv3_its_softc), gicv3_its_driver); 2132 #undef its_baseclasses 2133 2134 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0, 2135 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 2136 2137 static int 2138 gicv3_its_fdt_probe(device_t dev) 2139 { 2140 2141 if (!ofw_bus_status_okay(dev)) 2142 return (ENXIO); 2143 2144 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 2145 return (ENXIO); 2146 2147 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 2148 return (BUS_PROBE_DEFAULT); 2149 } 2150 2151 static int 2152 gicv3_its_fdt_attach(device_t dev) 2153 { 2154 struct gicv3_its_softc *sc; 2155 phandle_t xref; 2156 int err; 2157 2158 sc = device_get_softc(dev); 2159 sc->dev = dev; 2160 err = gicv3_its_attach(dev); 2161 if (err != 0) 2162 return (err); 2163 2164 /* Register this device as a interrupt controller */ 2165 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 2166 sc->sc_pic = intr_pic_register(dev, xref); 2167 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 2168 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 2169 if (err != 0) { 2170 device_printf(dev, "Failed to add PIC handler: %d\n", err); 2171 return (err); 2172 } 2173 2174 /* Register this device to handle MSI interrupts */ 2175 err = intr_msi_register(dev, xref); 2176 if (err != 0) { 2177 device_printf(dev, "Failed to register for MSIs: %d\n", err); 2178 return (err); 2179 } 2180 2181 return (0); 2182 } 2183 #endif 2184 2185 #ifdef DEV_ACPI 2186 static device_probe_t gicv3_its_acpi_probe; 2187 static device_attach_t gicv3_its_acpi_attach; 2188 2189 static device_method_t gicv3_its_acpi_methods[] = { 2190 /* Device interface */ 2191 DEVMETHOD(device_probe, gicv3_its_acpi_probe), 2192 DEVMETHOD(device_attach, gicv3_its_acpi_attach), 2193 2194 /* End */ 2195 DEVMETHOD_END 2196 }; 2197 2198 #define its_baseclasses its_acpi_baseclasses 2199 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods, 2200 sizeof(struct gicv3_its_softc), gicv3_its_driver); 2201 #undef its_baseclasses 2202 2203 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0, 2204 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 2205 2206 static int 2207 gicv3_its_acpi_probe(device_t dev) 2208 { 2209 2210 if (gic_get_bus(dev) != GIC_BUS_ACPI) 2211 return (EINVAL); 2212 2213 if (gic_get_hw_rev(dev) < 3) 2214 return (EINVAL); 2215 2216 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 2217 return (BUS_PROBE_DEFAULT); 2218 } 2219 2220 static int 2221 gicv3_its_acpi_attach(device_t dev) 2222 { 2223 struct gicv3_its_softc *sc; 2224 struct gic_v3_devinfo *di; 2225 int err; 2226 2227 sc = device_get_softc(dev); 2228 sc->dev = dev; 2229 err = gicv3_its_attach(dev); 2230 if (err != 0) 2231 return (err); 2232 2233 di = device_get_ivars(dev); 2234 sc->sc_pic = intr_pic_register(dev, di->msi_xref); 2235 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 2236 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 2237 if (err != 0) { 2238 device_printf(dev, "Failed to add PIC handler: %d\n", err); 2239 return (err); 2240 } 2241 2242 /* Register this device to handle MSI interrupts */ 2243 err = intr_msi_register(dev, di->msi_xref); 2244 if (err != 0) { 2245 device_printf(dev, "Failed to register for MSIs: %d\n", err); 2246 return (err); 2247 } 2248 2249 return (0); 2250 } 2251 #endif 2252