1 /*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * 4 * This software was developed by Andrew Turner under 5 * the sponsorship of the FreeBSD Foundation. 6 * 7 * This software was developed by Semihalf under 8 * the sponsorship of the FreeBSD Foundation. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 20 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 21 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 22 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 23 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 24 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 25 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 26 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 27 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 #include "opt_acpi.h" 33 #include "opt_platform.h" 34 #include "opt_iommu.h" 35 36 #include <sys/cdefs.h> 37 #include <sys/param.h> 38 #include <sys/systm.h> 39 #include <sys/bus.h> 40 #include <sys/cpuset.h> 41 #include <sys/domainset.h> 42 #include <sys/endian.h> 43 #include <sys/kernel.h> 44 #include <sys/lock.h> 45 #include <sys/malloc.h> 46 #include <sys/module.h> 47 #include <sys/mutex.h> 48 #include <sys/proc.h> 49 #include <sys/taskqueue.h> 50 #include <sys/tree.h> 51 #include <sys/queue.h> 52 #include <sys/rman.h> 53 #include <sys/sbuf.h> 54 #include <sys/smp.h> 55 #include <sys/sysctl.h> 56 #include <sys/vmem.h> 57 58 #include <vm/vm.h> 59 #include <vm/pmap.h> 60 #include <vm/vm_page.h> 61 62 #include <machine/bus.h> 63 #include <machine/intr.h> 64 65 #include <arm/arm/gic_common.h> 66 #include <arm64/arm64/gic_v3_reg.h> 67 #include <arm64/arm64/gic_v3_var.h> 68 69 #ifdef FDT 70 #include <dev/ofw/openfirm.h> 71 #include <dev/ofw/ofw_bus.h> 72 #include <dev/ofw/ofw_bus_subr.h> 73 #endif 74 #include <dev/pci/pcireg.h> 75 #include <dev/pci/pcivar.h> 76 77 #ifdef IOMMU 78 #include <dev/iommu/iommu.h> 79 #include <dev/iommu/iommu_gas.h> 80 #endif 81 82 #include "pcib_if.h" 83 #include "pic_if.h" 84 #include "msi_if.h" 85 86 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 87 "ARM GICv3 Interrupt Translation Service"); 88 89 #define LPI_NIRQS (64 * 1024) 90 91 /* The size and alignment of the command circular buffer */ 92 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 93 #define ITS_CMDQ_ALIGN (64 * 1024) 94 95 #define LPI_CONFTAB_SIZE LPI_NIRQS 96 #define LPI_CONFTAB_ALIGN (64 * 1024) 97 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 98 99 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 100 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 101 #define LPI_PENDTAB_ALIGN (64 * 1024) 102 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 103 104 #define LPI_INT_TRANS_TAB_ALIGN 256 105 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 106 107 /* ITS commands encoding */ 108 #define ITS_CMD_MOVI (0x01) 109 #define ITS_CMD_SYNC (0x05) 110 #define ITS_CMD_MAPD (0x08) 111 #define ITS_CMD_MAPC (0x09) 112 #define ITS_CMD_MAPTI (0x0a) 113 #define ITS_CMD_MAPI (0x0b) 114 #define ITS_CMD_INV (0x0c) 115 #define ITS_CMD_INVALL (0x0d) 116 /* Command */ 117 #define CMD_COMMAND_MASK (0xFFUL) 118 /* PCI device ID */ 119 #define CMD_DEVID_SHIFT (32) 120 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 121 /* Size of IRQ ID bitfield */ 122 #define CMD_SIZE_MASK (0xFFUL) 123 /* Virtual LPI ID */ 124 #define CMD_ID_MASK (0xFFFFFFFFUL) 125 /* Physical LPI ID */ 126 #define CMD_PID_SHIFT (32) 127 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 128 /* Collection */ 129 #define CMD_COL_MASK (0xFFFFUL) 130 /* Target (CPU or Re-Distributor) */ 131 #define CMD_TARGET_SHIFT (16) 132 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 133 /* Interrupt Translation Table address */ 134 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 135 /* Valid command bit */ 136 #define CMD_VALID_SHIFT (63) 137 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 138 139 #define ITS_TARGET_NONE 0xFBADBEEF 140 141 /* LPI chunk owned by ITS device */ 142 struct lpi_chunk { 143 u_int lpi_base; 144 u_int lpi_free; /* First free LPI in set */ 145 u_int lpi_num; /* Total number of LPIs in chunk */ 146 u_int lpi_busy; /* Number of busy LPIs in chink */ 147 }; 148 149 /* ITS device */ 150 struct its_dev { 151 TAILQ_ENTRY(its_dev) entry; 152 /* PCI device */ 153 device_t pci_dev; 154 /* Device ID (i.e. PCI device ID) */ 155 uint32_t devid; 156 /* List of assigned LPIs */ 157 struct lpi_chunk lpis; 158 /* Virtual address of ITT */ 159 vm_offset_t itt; 160 size_t itt_size; 161 }; 162 163 /* 164 * ITS command descriptor. 165 * Idea for command description passing taken from Linux. 166 */ 167 struct its_cmd_desc { 168 uint8_t cmd_type; 169 170 union { 171 struct { 172 struct its_dev *its_dev; 173 struct its_col *col; 174 uint32_t id; 175 } cmd_desc_movi; 176 177 struct { 178 struct its_col *col; 179 } cmd_desc_sync; 180 181 struct { 182 struct its_col *col; 183 uint8_t valid; 184 } cmd_desc_mapc; 185 186 struct { 187 struct its_dev *its_dev; 188 struct its_col *col; 189 uint32_t pid; 190 uint32_t id; 191 } cmd_desc_mapvi; 192 193 struct { 194 struct its_dev *its_dev; 195 struct its_col *col; 196 uint32_t pid; 197 } cmd_desc_mapi; 198 199 struct { 200 struct its_dev *its_dev; 201 uint8_t valid; 202 } cmd_desc_mapd; 203 204 struct { 205 struct its_dev *its_dev; 206 struct its_col *col; 207 uint32_t pid; 208 } cmd_desc_inv; 209 210 struct { 211 struct its_col *col; 212 } cmd_desc_invall; 213 }; 214 }; 215 216 /* ITS command. Each command is 32 bytes long */ 217 struct its_cmd { 218 uint64_t cmd_dword[4]; /* ITS command double word */ 219 }; 220 221 /* An ITS private table */ 222 struct its_ptable { 223 vm_offset_t ptab_vaddr; 224 unsigned long ptab_size; 225 }; 226 227 /* ITS collection description. */ 228 struct its_col { 229 uint64_t col_target; /* Target Re-Distributor */ 230 uint64_t col_id; /* Collection ID */ 231 }; 232 233 struct gicv3_its_irqsrc { 234 struct intr_irqsrc gi_isrc; 235 u_int gi_id; 236 u_int gi_lpi; 237 struct its_dev *gi_its_dev; 238 TAILQ_ENTRY(gicv3_its_irqsrc) gi_link; 239 }; 240 241 struct gicv3_its_softc { 242 device_t dev; 243 struct intr_pic *sc_pic; 244 struct resource *sc_its_res; 245 246 cpuset_t sc_cpus; 247 struct domainset *sc_ds; 248 u_int gic_irq_cpu; 249 250 struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 251 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 252 253 /* 254 * TODO: We should get these from the parent as we only want a 255 * single copy of each across the interrupt controller. 256 */ 257 uint8_t *sc_conf_base; 258 vm_offset_t sc_pend_base[MAXCPU]; 259 260 /* Command handling */ 261 struct mtx sc_its_cmd_lock; 262 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 263 size_t sc_its_cmd_next_idx; 264 265 vmem_t *sc_irq_alloc; 266 struct gicv3_its_irqsrc **sc_irqs; 267 u_int sc_irq_base; 268 u_int sc_irq_length; 269 u_int sc_irq_count; 270 271 struct mtx sc_its_dev_lock; 272 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 273 TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs; 274 275 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001 276 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 277 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 278 u_int sc_its_flags; 279 bool trace_enable; 280 vm_page_t ma; /* fake msi page */ 281 }; 282 283 static void *conf_base; 284 285 typedef void (its_quirk_func_t)(device_t); 286 static its_quirk_func_t its_quirk_cavium_22375; 287 288 static const struct { 289 const char *desc; 290 uint32_t iidr; 291 uint32_t iidr_mask; 292 its_quirk_func_t *func; 293 } its_quirks[] = { 294 { 295 /* Cavium ThunderX Pass 1.x */ 296 .desc = "Cavium ThunderX errata: 22375, 24313", 297 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 298 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 299 .iidr_mask = ~GITS_IIDR_REVISION_MASK, 300 .func = its_quirk_cavium_22375, 301 }, 302 }; 303 304 #define gic_its_read_4(sc, reg) \ 305 bus_read_4((sc)->sc_its_res, (reg)) 306 #define gic_its_read_8(sc, reg) \ 307 bus_read_8((sc)->sc_its_res, (reg)) 308 309 #define gic_its_write_4(sc, reg, val) \ 310 bus_write_4((sc)->sc_its_res, (reg), (val)) 311 #define gic_its_write_8(sc, reg, val) \ 312 bus_write_8((sc)->sc_its_res, (reg), (val)) 313 314 static device_attach_t gicv3_its_attach; 315 static device_detach_t gicv3_its_detach; 316 317 static pic_disable_intr_t gicv3_its_disable_intr; 318 static pic_enable_intr_t gicv3_its_enable_intr; 319 static pic_map_intr_t gicv3_its_map_intr; 320 static pic_setup_intr_t gicv3_its_setup_intr; 321 static pic_post_filter_t gicv3_its_post_filter; 322 static pic_post_ithread_t gicv3_its_post_ithread; 323 static pic_pre_ithread_t gicv3_its_pre_ithread; 324 static pic_bind_intr_t gicv3_its_bind_intr; 325 #ifdef SMP 326 static pic_init_secondary_t gicv3_its_init_secondary; 327 #endif 328 static msi_alloc_msi_t gicv3_its_alloc_msi; 329 static msi_release_msi_t gicv3_its_release_msi; 330 static msi_alloc_msix_t gicv3_its_alloc_msix; 331 static msi_release_msix_t gicv3_its_release_msix; 332 static msi_map_msi_t gicv3_its_map_msi; 333 #ifdef IOMMU 334 static msi_iommu_init_t gicv3_iommu_init; 335 static msi_iommu_deinit_t gicv3_iommu_deinit; 336 #endif 337 338 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 339 static void its_cmd_mapc(device_t, struct its_col *, uint8_t); 340 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 341 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 342 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 343 static void its_cmd_invall(device_t, struct its_col *); 344 345 static device_method_t gicv3_its_methods[] = { 346 /* Device interface */ 347 DEVMETHOD(device_detach, gicv3_its_detach), 348 349 /* Interrupt controller interface */ 350 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 351 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 352 DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 353 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 354 DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 355 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 356 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 357 #ifdef SMP 358 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 359 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 360 #endif 361 362 /* MSI/MSI-X */ 363 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 364 DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 365 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 366 DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 367 DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 368 #ifdef IOMMU 369 DEVMETHOD(msi_iommu_init, gicv3_iommu_init), 370 DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit), 371 #endif 372 373 /* End */ 374 DEVMETHOD_END 375 }; 376 377 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 378 sizeof(struct gicv3_its_softc)); 379 380 static void 381 gicv3_its_cmdq_init(struct gicv3_its_softc *sc) 382 { 383 vm_paddr_t cmd_paddr; 384 uint64_t reg, tmp; 385 386 /* Set up the command circular buffer */ 387 sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS, 388 sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 389 0); 390 sc->sc_its_cmd_next_idx = 0; 391 392 cmd_paddr = vtophys(sc->sc_its_cmd_base); 393 394 /* Set the base of the command buffer */ 395 reg = GITS_CBASER_VALID | 396 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 397 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 398 (ITS_CMDQ_SIZE / 4096 - 1); 399 gic_its_write_8(sc, GITS_CBASER, reg); 400 401 /* Read back to check for fixed value fields */ 402 tmp = gic_its_read_8(sc, GITS_CBASER); 403 404 if ((tmp & GITS_CBASER_SHARE_MASK) != 405 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 406 /* Check if the hardware reported non-shareable */ 407 if ((tmp & GITS_CBASER_SHARE_MASK) == 408 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 409 /* If so remove the cache attribute */ 410 reg &= ~GITS_CBASER_CACHE_MASK; 411 reg &= ~GITS_CBASER_SHARE_MASK; 412 /* Set to Non-cacheable, Non-shareable */ 413 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 414 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 415 416 gic_its_write_8(sc, GITS_CBASER, reg); 417 } 418 419 /* The command queue has to be flushed after each command */ 420 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 421 } 422 423 /* Get the next command from the start of the buffer */ 424 gic_its_write_8(sc, GITS_CWRITER, 0x0); 425 } 426 427 static int 428 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 429 { 430 vm_offset_t table; 431 vm_paddr_t paddr; 432 uint64_t cache, reg, share, tmp, type; 433 size_t esize, its_tbl_size, nidents, nitspages, npages; 434 int i, page_size; 435 int devbits; 436 437 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 438 /* 439 * GITS_TYPER[17:13] of ThunderX reports that device IDs 440 * are to be 21 bits in length. The entry size of the ITS 441 * table can be read from GITS_BASERn[52:48] and on ThunderX 442 * is supposed to be 8 bytes in length (for device table). 443 * Finally the page size that is to be used by ITS to access 444 * this table will be set to 64KB. 445 * 446 * This gives 0x200000 entries of size 0x8 bytes covered by 447 * 256 pages each of which 64KB in size. The number of pages 448 * (minus 1) should then be written to GITS_BASERn[7:0]. In 449 * that case this value would be 0xFF but on ThunderX the 450 * maximum value that HW accepts is 0xFD. 451 * 452 * Set an arbitrary number of device ID bits to 20 in order 453 * to limit the number of entries in ITS device table to 454 * 0x100000 and the table size to 8MB. 455 */ 456 devbits = 20; 457 cache = 0; 458 } else { 459 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 460 cache = GITS_BASER_CACHE_WAWB; 461 } 462 share = GITS_BASER_SHARE_IS; 463 page_size = PAGE_SIZE_64K; 464 465 for (i = 0; i < GITS_BASER_NUM; i++) { 466 reg = gic_its_read_8(sc, GITS_BASER(i)); 467 /* The type of table */ 468 type = GITS_BASER_TYPE(reg); 469 /* The table entry size */ 470 esize = GITS_BASER_ESIZE(reg); 471 472 switch(type) { 473 case GITS_BASER_TYPE_DEV: 474 nidents = (1 << devbits); 475 its_tbl_size = esize * nidents; 476 its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); 477 break; 478 case GITS_BASER_TYPE_VP: 479 case GITS_BASER_TYPE_PP: /* Undocumented? */ 480 case GITS_BASER_TYPE_IC: 481 its_tbl_size = page_size; 482 break; 483 default: 484 continue; 485 } 486 npages = howmany(its_tbl_size, PAGE_SIZE); 487 488 /* Allocate the table */ 489 table = (vm_offset_t)contigmalloc_domainset(npages * PAGE_SIZE, 490 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, 491 (1ul << 48) - 1, PAGE_SIZE_64K, 0); 492 493 sc->sc_its_ptab[i].ptab_vaddr = table; 494 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; 495 496 paddr = vtophys(table); 497 498 while (1) { 499 nitspages = howmany(its_tbl_size, page_size); 500 501 /* Clear the fields we will be setting */ 502 reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT | 503 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 504 GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | 505 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 506 GITS_BASER_SIZE_MASK); 507 /* Set the new values */ 508 reg |= GITS_BASER_VALID | 509 (cache << GITS_BASER_CACHE_SHIFT) | 510 (type << GITS_BASER_TYPE_SHIFT) | 511 ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | 512 paddr | (share << GITS_BASER_SHARE_SHIFT) | 513 (nitspages - 1); 514 515 switch (page_size) { 516 case PAGE_SIZE_4K: /* 4KB */ 517 reg |= 518 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 519 break; 520 case PAGE_SIZE_16K: /* 16KB */ 521 reg |= 522 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 523 break; 524 case PAGE_SIZE_64K: /* 64KB */ 525 reg |= 526 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 527 break; 528 } 529 530 gic_its_write_8(sc, GITS_BASER(i), reg); 531 532 /* Read back to check */ 533 tmp = gic_its_read_8(sc, GITS_BASER(i)); 534 535 /* Do the shareability masks line up? */ 536 if ((tmp & GITS_BASER_SHARE_MASK) != 537 (reg & GITS_BASER_SHARE_MASK)) { 538 share = (tmp & GITS_BASER_SHARE_MASK) >> 539 GITS_BASER_SHARE_SHIFT; 540 continue; 541 } 542 543 if ((tmp & GITS_BASER_PSZ_MASK) != 544 (reg & GITS_BASER_PSZ_MASK)) { 545 switch (page_size) { 546 case PAGE_SIZE_16K: 547 page_size = PAGE_SIZE_4K; 548 continue; 549 case PAGE_SIZE_64K: 550 page_size = PAGE_SIZE_16K; 551 continue; 552 } 553 } 554 555 if (tmp != reg) { 556 device_printf(dev, "GITS_BASER%d: " 557 "unable to be updated: %lx != %lx\n", 558 i, reg, tmp); 559 return (ENXIO); 560 } 561 562 /* We should have made all needed changes */ 563 break; 564 } 565 } 566 567 return (0); 568 } 569 570 static void 571 gicv3_its_conftable_init(struct gicv3_its_softc *sc) 572 { 573 void *conf_table; 574 575 conf_table = atomic_load_ptr(&conf_base); 576 if (conf_table == NULL) { 577 conf_table = contigmalloc(LPI_CONFTAB_SIZE, 578 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, 579 LPI_CONFTAB_ALIGN, 0); 580 581 if (atomic_cmpset_ptr((uintptr_t *)&conf_base, 582 (uintptr_t)NULL, (uintptr_t)conf_table) == 0) { 583 contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS); 584 conf_table = atomic_load_ptr(&conf_base); 585 } 586 } 587 sc->sc_conf_base = conf_table; 588 589 /* Set the default configuration */ 590 memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 591 LPI_CONFTAB_SIZE); 592 593 /* Flush the table to memory */ 594 cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE); 595 } 596 597 static void 598 gicv3_its_pendtables_init(struct gicv3_its_softc *sc) 599 { 600 int i; 601 602 for (i = 0; i <= mp_maxid; i++) { 603 if (CPU_ISSET(i, &sc->sc_cpus) == 0) 604 continue; 605 606 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( 607 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 608 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 609 610 /* Flush so the ITS can see the memory */ 611 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i], 612 LPI_PENDTAB_SIZE); 613 } 614 } 615 616 static void 617 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) 618 { 619 device_t gicv3; 620 uint64_t xbaser, tmp; 621 uint32_t ctlr; 622 u_int cpuid; 623 624 gicv3 = device_get_parent(dev); 625 cpuid = PCPU_GET(cpuid); 626 627 /* Disable LPIs */ 628 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 629 ctlr &= ~GICR_CTLR_LPI_ENABLE; 630 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 631 632 /* Make sure changes are observable my the GIC */ 633 dsb(sy); 634 635 /* 636 * Set the redistributor base 637 */ 638 xbaser = vtophys(sc->sc_conf_base) | 639 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 640 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 641 (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 642 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 643 644 /* Check the cache attributes we set */ 645 tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 646 647 if ((tmp & GICR_PROPBASER_SHARE_MASK) != 648 (xbaser & GICR_PROPBASER_SHARE_MASK)) { 649 if ((tmp & GICR_PROPBASER_SHARE_MASK) == 650 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 651 /* We need to mark as non-cacheable */ 652 xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 653 GICR_PROPBASER_CACHE_MASK); 654 /* Non-cacheable */ 655 xbaser |= GICR_PROPBASER_CACHE_NIN << 656 GICR_PROPBASER_CACHE_SHIFT; 657 /* Non-shareable */ 658 xbaser |= GICR_PROPBASER_SHARE_NS << 659 GICR_PROPBASER_SHARE_SHIFT; 660 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 661 } 662 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 663 } 664 665 /* 666 * Set the LPI pending table base 667 */ 668 xbaser = vtophys(sc->sc_pend_base[cpuid]) | 669 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 670 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 671 672 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 673 674 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 675 676 if ((tmp & GICR_PENDBASER_SHARE_MASK) == 677 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 678 /* Clear the cahce and shareability bits */ 679 xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 680 GICR_PENDBASER_SHARE_MASK); 681 /* Mark as non-shareable */ 682 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 683 /* And non-cacheable */ 684 xbaser |= GICR_PENDBASER_CACHE_NIN << 685 GICR_PENDBASER_CACHE_SHIFT; 686 } 687 688 /* Enable LPIs */ 689 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 690 ctlr |= GICR_CTLR_LPI_ENABLE; 691 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 692 693 /* Make sure the GIC has seen everything */ 694 dsb(sy); 695 } 696 697 static int 698 its_init_cpu(device_t dev, struct gicv3_its_softc *sc) 699 { 700 device_t gicv3; 701 vm_paddr_t target; 702 u_int cpuid; 703 struct redist_pcpu *rpcpu; 704 705 gicv3 = device_get_parent(dev); 706 cpuid = PCPU_GET(cpuid); 707 if (!CPU_ISSET(cpuid, &sc->sc_cpus)) 708 return (0); 709 710 /* Check if the ITS is enabled on this CPU */ 711 if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) 712 return (ENXIO); 713 714 rpcpu = gicv3_get_redist(dev); 715 716 /* Do per-cpu LPI init once */ 717 if (!rpcpu->lpi_enabled) { 718 its_init_cpu_lpi(dev, sc); 719 rpcpu->lpi_enabled = true; 720 } 721 722 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 723 /* This ITS wants the redistributor physical address */ 724 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) + 725 rpcpu->offset); 726 } else { 727 /* This ITS wants the unique processor number */ 728 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) << 729 CMD_TARGET_SHIFT; 730 } 731 732 sc->sc_its_cols[cpuid]->col_target = target; 733 sc->sc_its_cols[cpuid]->col_id = cpuid; 734 735 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 736 its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 737 738 return (0); 739 } 740 741 static int 742 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS) 743 { 744 struct gicv3_its_softc *sc; 745 int rv; 746 747 sc = arg1; 748 749 rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req); 750 if (rv != 0 || req->newptr == NULL) 751 return (rv); 752 if (sc->trace_enable) 753 gic_its_write_8(sc, GITS_TRKCTLR, 3); 754 else 755 gic_its_write_8(sc, GITS_TRKCTLR, 0); 756 757 return (0); 758 } 759 760 static int 761 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS) 762 { 763 struct gicv3_its_softc *sc; 764 struct sbuf *sb; 765 int err; 766 767 sc = arg1; 768 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 769 if (sb == NULL) { 770 device_printf(sc->dev, "Could not allocate sbuf for output.\n"); 771 return (ENOMEM); 772 } 773 sbuf_cat(sb, "\n"); 774 sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n", 775 gic_its_read_4(sc, GITS_TRKCTLR)); 776 sbuf_printf(sb, "GITS_TRKR: 0x%08X\n", 777 gic_its_read_4(sc, GITS_TRKR)); 778 sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n", 779 gic_its_read_4(sc, GITS_TRKDIDR)); 780 sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n", 781 gic_its_read_4(sc, GITS_TRKPIDR)); 782 sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n", 783 gic_its_read_4(sc, GITS_TRKVIDR)); 784 sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n", 785 gic_its_read_4(sc, GITS_TRKTGTR)); 786 787 err = sbuf_finish(sb); 788 if (err) 789 device_printf(sc->dev, "Error finishing sbuf: %d\n", err); 790 sbuf_delete(sb); 791 return(err); 792 } 793 794 static int 795 gicv3_its_init_sysctl(struct gicv3_its_softc *sc) 796 { 797 struct sysctl_oid *oid, *child; 798 struct sysctl_ctx_list *ctx_list; 799 800 ctx_list = device_get_sysctl_ctx(sc->dev); 801 child = device_get_sysctl_tree(sc->dev); 802 oid = SYSCTL_ADD_NODE(ctx_list, 803 SYSCTL_CHILDREN(child), OID_AUTO, "tracing", 804 CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing"); 805 if (oid == NULL) 806 return (ENXIO); 807 808 /* Add registers */ 809 SYSCTL_ADD_PROC(ctx_list, 810 SYSCTL_CHILDREN(oid), OID_AUTO, "enable", 811 CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 812 gicv3_its_sysctl_trace_enable, "CU", "Enable tracing"); 813 SYSCTL_ADD_PROC(ctx_list, 814 SYSCTL_CHILDREN(oid), OID_AUTO, "capture", 815 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 816 gicv3_its_sysctl_trace_regs, "", "Captured tracing registers."); 817 818 return (0); 819 } 820 821 static int 822 gicv3_its_attach(device_t dev) 823 { 824 struct gicv3_its_softc *sc; 825 int domain, err, i, rid; 826 uint64_t phys; 827 uint32_t ctlr, iidr; 828 829 sc = device_get_softc(dev); 830 831 sc->sc_irq_length = gicv3_get_nirqs(dev); 832 sc->sc_irq_base = GIC_FIRST_LPI; 833 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; 834 835 rid = 0; 836 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 837 RF_ACTIVE); 838 if (sc->sc_its_res == NULL) { 839 device_printf(dev, "Could not allocate memory\n"); 840 return (ENXIO); 841 } 842 843 phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) + 844 GITS_TRANSLATER, PAGE_SIZE); 845 sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO); 846 vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT); 847 848 CPU_COPY(&all_cpus, &sc->sc_cpus); 849 iidr = gic_its_read_4(sc, GITS_IIDR); 850 for (i = 0; i < nitems(its_quirks); i++) { 851 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 852 if (bootverbose) { 853 device_printf(dev, "Applying %s\n", 854 its_quirks[i].desc); 855 } 856 its_quirks[i].func(dev); 857 break; 858 } 859 } 860 861 if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) { 862 sc->sc_ds = DOMAINSET_PREF(domain); 863 } else { 864 sc->sc_ds = DOMAINSET_RR(); 865 } 866 867 /* 868 * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be 869 * coming in via, for instance, a kexec/kboot style setup where a 870 * previous kernel has configured then relinquished control. Clear it 871 * so that we can reconfigure GITS_BASER*. 872 */ 873 ctlr = gic_its_read_4(sc, GITS_CTLR); 874 if ((ctlr & GITS_CTLR_EN) != 0) { 875 ctlr &= ~GITS_CTLR_EN; 876 gic_its_write_4(sc, GITS_CTLR, ctlr); 877 } 878 879 /* Allocate the private tables */ 880 err = gicv3_its_table_init(dev, sc); 881 if (err != 0) 882 return (err); 883 884 /* Protects access to the device list */ 885 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 886 887 /* Protects access to the ITS command circular buffer. */ 888 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 889 890 /* Allocate the command circular buffer */ 891 gicv3_its_cmdq_init(sc); 892 893 /* Allocate the per-CPU collections */ 894 for (int cpu = 0; cpu <= mp_maxid; cpu++) 895 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0) 896 sc->sc_its_cols[cpu] = malloc_domainset( 897 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 898 DOMAINSET_PREF(pcpu_find(cpu)->pc_domain), 899 M_WAITOK | M_ZERO); 900 else 901 sc->sc_its_cols[cpu] = NULL; 902 903 /* Enable the ITS */ 904 gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN); 905 906 /* Create the LPI configuration table */ 907 gicv3_its_conftable_init(sc); 908 909 /* And the pending tebles */ 910 gicv3_its_pendtables_init(sc); 911 912 /* Enable LPIs on this CPU */ 913 its_init_cpu(dev, sc); 914 915 TAILQ_INIT(&sc->sc_its_dev_list); 916 TAILQ_INIT(&sc->sc_free_irqs); 917 918 /* 919 * Create the vmem object to allocate INTRNG IRQs from. We try to 920 * use all IRQs not already used by the GICv3. 921 * XXX: This assumes there are no other interrupt controllers in the 922 * system. 923 */ 924 sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0, 925 gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK); 926 927 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length, 928 M_GICV3_ITS, M_WAITOK | M_ZERO); 929 930 /* For GIC-500 install tracking sysctls. */ 931 if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) == 932 GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0)) 933 gicv3_its_init_sysctl(sc); 934 935 return (0); 936 } 937 938 static int 939 gicv3_its_detach(device_t dev) 940 { 941 942 return (ENXIO); 943 } 944 945 static void 946 its_quirk_cavium_22375(device_t dev) 947 { 948 struct gicv3_its_softc *sc; 949 int domain; 950 951 sc = device_get_softc(dev); 952 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 953 954 /* 955 * We need to limit which CPUs we send these interrupts to on 956 * the original dual socket ThunderX as it is unable to 957 * forward them between the two sockets. 958 */ 959 if (bus_get_domain(dev, &domain) == 0) { 960 if (domain < MAXMEMDOM) { 961 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus); 962 } else { 963 CPU_ZERO(&sc->sc_cpus); 964 } 965 } 966 } 967 968 static void 969 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 970 { 971 struct gicv3_its_softc *sc; 972 struct gicv3_its_irqsrc *girq; 973 uint8_t *conf; 974 975 sc = device_get_softc(dev); 976 girq = (struct gicv3_its_irqsrc *)isrc; 977 conf = sc->sc_conf_base; 978 979 conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE; 980 981 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 982 /* Clean D-cache under command. */ 983 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); 984 } else { 985 /* DSB inner shareable, store */ 986 dsb(ishst); 987 } 988 989 its_cmd_inv(dev, girq->gi_its_dev, girq); 990 } 991 992 static void 993 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 994 { 995 struct gicv3_its_softc *sc; 996 struct gicv3_its_irqsrc *girq; 997 uint8_t *conf; 998 999 sc = device_get_softc(dev); 1000 girq = (struct gicv3_its_irqsrc *)isrc; 1001 conf = sc->sc_conf_base; 1002 1003 conf[girq->gi_lpi] |= LPI_CONF_ENABLE; 1004 1005 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 1006 /* Clean D-cache under command. */ 1007 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); 1008 } else { 1009 /* DSB inner shareable, store */ 1010 dsb(ishst); 1011 } 1012 1013 its_cmd_inv(dev, girq->gi_its_dev, girq); 1014 } 1015 1016 static int 1017 gicv3_its_intr(void *arg, uintptr_t irq) 1018 { 1019 struct gicv3_its_softc *sc = arg; 1020 struct gicv3_its_irqsrc *girq; 1021 struct trapframe *tf; 1022 1023 irq -= sc->sc_irq_base; 1024 girq = sc->sc_irqs[irq]; 1025 if (girq == NULL) 1026 panic("gicv3_its_intr: Invalid interrupt %ld", 1027 irq + sc->sc_irq_base); 1028 1029 tf = curthread->td_intr_frame; 1030 intr_isrc_dispatch(&girq->gi_isrc, tf); 1031 return (FILTER_HANDLED); 1032 } 1033 1034 static void 1035 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 1036 { 1037 struct gicv3_its_irqsrc *girq; 1038 1039 girq = (struct gicv3_its_irqsrc *)isrc; 1040 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 1041 } 1042 1043 static void 1044 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 1045 { 1046 1047 } 1048 1049 static void 1050 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 1051 { 1052 struct gicv3_its_irqsrc *girq; 1053 1054 girq = (struct gicv3_its_irqsrc *)isrc; 1055 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 1056 } 1057 1058 static int 1059 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc) 1060 { 1061 struct gicv3_its_softc *sc; 1062 1063 sc = device_get_softc(dev); 1064 if (CPU_EMPTY(&isrc->isrc_cpu)) { 1065 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu, 1066 &sc->sc_cpus); 1067 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu); 1068 } 1069 1070 return (0); 1071 } 1072 1073 static int 1074 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 1075 { 1076 struct gicv3_its_irqsrc *girq; 1077 1078 gicv3_its_select_cpu(dev, isrc); 1079 1080 girq = (struct gicv3_its_irqsrc *)isrc; 1081 its_cmd_movi(dev, girq); 1082 return (0); 1083 } 1084 1085 static int 1086 gicv3_its_map_intr(device_t dev, struct intr_map_data *data, 1087 struct intr_irqsrc **isrcp) 1088 { 1089 1090 /* 1091 * This should never happen, we only call this function to map 1092 * interrupts found before the controller driver is ready. 1093 */ 1094 panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 1095 } 1096 1097 static int 1098 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 1099 struct resource *res, struct intr_map_data *data) 1100 { 1101 1102 /* Bind the interrupt to a CPU */ 1103 gicv3_its_bind_intr(dev, isrc); 1104 1105 return (0); 1106 } 1107 1108 #ifdef SMP 1109 static void 1110 gicv3_its_init_secondary(device_t dev) 1111 { 1112 struct gicv3_its_softc *sc; 1113 1114 sc = device_get_softc(dev); 1115 1116 /* 1117 * This is fatal as otherwise we may bind interrupts to this CPU. 1118 * We need a way to tell the interrupt framework to only bind to a 1119 * subset of given CPUs when it performs the shuffle. 1120 */ 1121 if (its_init_cpu(dev, sc) != 0) 1122 panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 1123 PCPU_GET(cpuid)); 1124 } 1125 #endif 1126 1127 static uint32_t 1128 its_get_devid(device_t pci_dev) 1129 { 1130 uintptr_t id; 1131 1132 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 1133 panic("%s: %s: Unable to get the MSI DeviceID", __func__, 1134 device_get_nameunit(pci_dev)); 1135 1136 return (id); 1137 } 1138 1139 static struct its_dev * 1140 its_device_find(device_t dev, device_t child) 1141 { 1142 struct gicv3_its_softc *sc; 1143 struct its_dev *its_dev = NULL; 1144 1145 sc = device_get_softc(dev); 1146 1147 mtx_lock_spin(&sc->sc_its_dev_lock); 1148 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 1149 if (its_dev->pci_dev == child) 1150 break; 1151 } 1152 mtx_unlock_spin(&sc->sc_its_dev_lock); 1153 1154 return (its_dev); 1155 } 1156 1157 static struct its_dev * 1158 its_device_get(device_t dev, device_t child, u_int nvecs) 1159 { 1160 struct gicv3_its_softc *sc; 1161 struct its_dev *its_dev; 1162 vmem_addr_t irq_base; 1163 size_t esize; 1164 1165 sc = device_get_softc(dev); 1166 1167 its_dev = its_device_find(dev, child); 1168 if (its_dev != NULL) 1169 return (its_dev); 1170 1171 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 1172 if (its_dev == NULL) 1173 return (NULL); 1174 1175 its_dev->pci_dev = child; 1176 its_dev->devid = its_get_devid(child); 1177 1178 its_dev->lpis.lpi_busy = 0; 1179 its_dev->lpis.lpi_num = nvecs; 1180 its_dev->lpis.lpi_free = nvecs; 1181 1182 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 1183 &irq_base) != 0) { 1184 free(its_dev, M_GICV3_ITS); 1185 return (NULL); 1186 } 1187 its_dev->lpis.lpi_base = irq_base; 1188 1189 /* Get ITT entry size */ 1190 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 1191 1192 /* 1193 * Allocate ITT for this device. 1194 * PA has to be 256 B aligned. At least two entries for device. 1195 */ 1196 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 1197 its_dev->itt = (vm_offset_t)contigmalloc_domainset(its_dev->itt_size, 1198 M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0, 1199 LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0); 1200 if (its_dev->itt == 0) { 1201 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 1202 free(its_dev, M_GICV3_ITS); 1203 return (NULL); 1204 } 1205 1206 /* Make sure device sees zeroed ITT. */ 1207 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) 1208 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size); 1209 1210 mtx_lock_spin(&sc->sc_its_dev_lock); 1211 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 1212 mtx_unlock_spin(&sc->sc_its_dev_lock); 1213 1214 /* Map device to its ITT */ 1215 its_cmd_mapd(dev, its_dev, 1); 1216 1217 return (its_dev); 1218 } 1219 1220 static void 1221 its_device_release(device_t dev, struct its_dev *its_dev) 1222 { 1223 struct gicv3_its_softc *sc; 1224 1225 KASSERT(its_dev->lpis.lpi_busy == 0, 1226 ("its_device_release: Trying to release an inuse ITS device")); 1227 1228 /* Unmap device in ITS */ 1229 its_cmd_mapd(dev, its_dev, 0); 1230 1231 sc = device_get_softc(dev); 1232 1233 /* Remove the device from the list of devices */ 1234 mtx_lock_spin(&sc->sc_its_dev_lock); 1235 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 1236 mtx_unlock_spin(&sc->sc_its_dev_lock); 1237 1238 /* Free ITT */ 1239 KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); 1240 contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 1241 1242 /* Free the IRQ allocation */ 1243 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 1244 its_dev->lpis.lpi_num); 1245 1246 free(its_dev, M_GICV3_ITS); 1247 } 1248 1249 static struct gicv3_its_irqsrc * 1250 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq) 1251 { 1252 struct gicv3_its_irqsrc *girq = NULL; 1253 1254 KASSERT(sc->sc_irqs[irq] == NULL, 1255 ("%s: Interrupt %u already allocated", __func__, irq)); 1256 mtx_lock_spin(&sc->sc_its_dev_lock); 1257 if (!TAILQ_EMPTY(&sc->sc_free_irqs)) { 1258 girq = TAILQ_FIRST(&sc->sc_free_irqs); 1259 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link); 1260 } 1261 mtx_unlock_spin(&sc->sc_its_dev_lock); 1262 if (girq == NULL) { 1263 girq = malloc(sizeof(*girq), M_GICV3_ITS, 1264 M_NOWAIT | M_ZERO); 1265 if (girq == NULL) 1266 return (NULL); 1267 girq->gi_id = -1; 1268 if (intr_isrc_register(&girq->gi_isrc, dev, 0, 1269 "%s,%u", device_get_nameunit(dev), irq) != 0) { 1270 free(girq, M_GICV3_ITS); 1271 return (NULL); 1272 } 1273 } 1274 girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI; 1275 sc->sc_irqs[irq] = girq; 1276 1277 return (girq); 1278 } 1279 1280 static void 1281 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc, 1282 struct gicv3_its_irqsrc *girq) 1283 { 1284 u_int irq; 1285 1286 mtx_assert(&sc->sc_its_dev_lock, MA_OWNED); 1287 1288 irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base; 1289 sc->sc_irqs[irq] = NULL; 1290 1291 girq->gi_id = -1; 1292 girq->gi_its_dev = NULL; 1293 TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link); 1294 } 1295 1296 static int 1297 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 1298 device_t *pic, struct intr_irqsrc **srcs) 1299 { 1300 struct gicv3_its_softc *sc; 1301 struct gicv3_its_irqsrc *girq; 1302 struct its_dev *its_dev; 1303 u_int irq; 1304 int i; 1305 1306 its_dev = its_device_get(dev, child, count); 1307 if (its_dev == NULL) 1308 return (ENXIO); 1309 1310 KASSERT(its_dev->lpis.lpi_free >= count, 1311 ("gicv3_its_alloc_msi: No free LPIs")); 1312 sc = device_get_softc(dev); 1313 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1314 its_dev->lpis.lpi_free; 1315 1316 /* Allocate the irqsrc for each MSI */ 1317 for (i = 0; i < count; i++, irq++) { 1318 its_dev->lpis.lpi_free--; 1319 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev, 1320 sc, irq); 1321 if (srcs[i] == NULL) 1322 break; 1323 } 1324 1325 /* The allocation failed, release them */ 1326 if (i != count) { 1327 mtx_lock_spin(&sc->sc_its_dev_lock); 1328 for (i = 0; i < count; i++) { 1329 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1330 if (girq == NULL) 1331 break; 1332 gicv3_its_release_irqsrc(sc, girq); 1333 srcs[i] = NULL; 1334 } 1335 mtx_unlock_spin(&sc->sc_its_dev_lock); 1336 return (ENXIO); 1337 } 1338 1339 /* Finish the allocation now we have all MSI irqsrcs */ 1340 for (i = 0; i < count; i++) { 1341 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1342 girq->gi_id = i; 1343 girq->gi_its_dev = its_dev; 1344 1345 /* Map the message to the given IRQ */ 1346 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1347 its_cmd_mapti(dev, girq); 1348 } 1349 its_dev->lpis.lpi_busy += count; 1350 *pic = dev; 1351 1352 return (0); 1353 } 1354 1355 static int 1356 gicv3_its_release_msi(device_t dev, device_t child, int count, 1357 struct intr_irqsrc **isrc) 1358 { 1359 struct gicv3_its_softc *sc; 1360 struct gicv3_its_irqsrc *girq; 1361 struct its_dev *its_dev; 1362 int i; 1363 1364 its_dev = its_device_find(dev, child); 1365 1366 KASSERT(its_dev != NULL, 1367 ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1368 "no ITS device")); 1369 KASSERT(its_dev->lpis.lpi_busy >= count, 1370 ("gicv3_its_release_msi: Releasing more interrupts than " 1371 "were allocated: releasing %d, allocated %d", count, 1372 its_dev->lpis.lpi_busy)); 1373 1374 sc = device_get_softc(dev); 1375 mtx_lock_spin(&sc->sc_its_dev_lock); 1376 for (i = 0; i < count; i++) { 1377 girq = (struct gicv3_its_irqsrc *)isrc[i]; 1378 gicv3_its_release_irqsrc(sc, girq); 1379 } 1380 mtx_unlock_spin(&sc->sc_its_dev_lock); 1381 its_dev->lpis.lpi_busy -= count; 1382 1383 if (its_dev->lpis.lpi_busy == 0) 1384 its_device_release(dev, its_dev); 1385 1386 return (0); 1387 } 1388 1389 static int 1390 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1391 struct intr_irqsrc **isrcp) 1392 { 1393 struct gicv3_its_softc *sc; 1394 struct gicv3_its_irqsrc *girq; 1395 struct its_dev *its_dev; 1396 u_int nvecs, irq; 1397 1398 nvecs = pci_msix_count(child); 1399 its_dev = its_device_get(dev, child, nvecs); 1400 if (its_dev == NULL) 1401 return (ENXIO); 1402 1403 KASSERT(its_dev->lpis.lpi_free > 0, 1404 ("gicv3_its_alloc_msix: No free LPIs")); 1405 sc = device_get_softc(dev); 1406 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1407 its_dev->lpis.lpi_free; 1408 1409 girq = gicv3_its_alloc_irqsrc(dev, sc, irq); 1410 if (girq == NULL) 1411 return (ENXIO); 1412 girq->gi_id = its_dev->lpis.lpi_busy; 1413 girq->gi_its_dev = its_dev; 1414 1415 its_dev->lpis.lpi_free--; 1416 its_dev->lpis.lpi_busy++; 1417 1418 /* Map the message to the given IRQ */ 1419 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1420 its_cmd_mapti(dev, girq); 1421 1422 *pic = dev; 1423 *isrcp = (struct intr_irqsrc *)girq; 1424 1425 return (0); 1426 } 1427 1428 static int 1429 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1430 { 1431 struct gicv3_its_softc *sc; 1432 struct gicv3_its_irqsrc *girq; 1433 struct its_dev *its_dev; 1434 1435 its_dev = its_device_find(dev, child); 1436 1437 KASSERT(its_dev != NULL, 1438 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1439 "no ITS device")); 1440 KASSERT(its_dev->lpis.lpi_busy > 0, 1441 ("gicv3_its_release_msix: Releasing more interrupts than " 1442 "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1443 1444 sc = device_get_softc(dev); 1445 girq = (struct gicv3_its_irqsrc *)isrc; 1446 mtx_lock_spin(&sc->sc_its_dev_lock); 1447 gicv3_its_release_irqsrc(sc, girq); 1448 mtx_unlock_spin(&sc->sc_its_dev_lock); 1449 its_dev->lpis.lpi_busy--; 1450 1451 if (its_dev->lpis.lpi_busy == 0) 1452 its_device_release(dev, its_dev); 1453 1454 return (0); 1455 } 1456 1457 static int 1458 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1459 uint64_t *addr, uint32_t *data) 1460 { 1461 struct gicv3_its_softc *sc; 1462 struct gicv3_its_irqsrc *girq; 1463 1464 sc = device_get_softc(dev); 1465 girq = (struct gicv3_its_irqsrc *)isrc; 1466 1467 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1468 *data = girq->gi_id; 1469 1470 return (0); 1471 } 1472 1473 #ifdef IOMMU 1474 static int 1475 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain) 1476 { 1477 struct gicv3_its_softc *sc; 1478 struct iommu_ctx *ctx; 1479 int error; 1480 1481 sc = device_get_softc(dev); 1482 ctx = iommu_get_dev_ctx(child); 1483 if (ctx == NULL) 1484 return (ENXIO); 1485 /* Map the page containing the GITS_TRANSLATER register. */ 1486 error = iommu_map_msi(ctx, PAGE_SIZE, 0, 1487 IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma); 1488 *domain = iommu_get_ctx_domain(ctx); 1489 1490 return (error); 1491 } 1492 1493 static void 1494 gicv3_iommu_deinit(device_t dev, device_t child) 1495 { 1496 struct iommu_ctx *ctx; 1497 1498 ctx = iommu_get_dev_ctx(child); 1499 if (ctx == NULL) 1500 return; 1501 1502 iommu_unmap_msi(ctx); 1503 } 1504 #endif 1505 1506 /* 1507 * Commands handling. 1508 */ 1509 1510 static __inline void 1511 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1512 { 1513 /* Command field: DW0 [7:0] */ 1514 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1515 cmd->cmd_dword[0] |= htole64(cmd_type); 1516 } 1517 1518 static __inline void 1519 cmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1520 { 1521 /* Device ID field: DW0 [63:32] */ 1522 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1523 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1524 } 1525 1526 static __inline void 1527 cmd_format_size(struct its_cmd *cmd, uint16_t size) 1528 { 1529 /* Size field: DW1 [4:0] */ 1530 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1531 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1532 } 1533 1534 static __inline void 1535 cmd_format_id(struct its_cmd *cmd, uint32_t id) 1536 { 1537 /* ID field: DW1 [31:0] */ 1538 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1539 cmd->cmd_dword[1] |= htole64(id); 1540 } 1541 1542 static __inline void 1543 cmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1544 { 1545 /* Physical ID field: DW1 [63:32] */ 1546 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1547 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1548 } 1549 1550 static __inline void 1551 cmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1552 { 1553 /* Collection field: DW2 [16:0] */ 1554 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1555 cmd->cmd_dword[2] |= htole64(col_id); 1556 } 1557 1558 static __inline void 1559 cmd_format_target(struct its_cmd *cmd, uint64_t target) 1560 { 1561 /* Target Address field: DW2 [47:16] */ 1562 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1563 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1564 } 1565 1566 static __inline void 1567 cmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1568 { 1569 /* ITT Address field: DW2 [47:8] */ 1570 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1571 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1572 } 1573 1574 static __inline void 1575 cmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1576 { 1577 /* Valid field: DW2 [63] */ 1578 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1579 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1580 } 1581 1582 static inline bool 1583 its_cmd_queue_full(struct gicv3_its_softc *sc) 1584 { 1585 size_t read_idx, next_write_idx; 1586 1587 /* Get the index of the next command */ 1588 next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1589 (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1590 /* And the index of the current command being read */ 1591 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1592 1593 /* 1594 * The queue is full when the write offset points 1595 * at the command before the current read offset. 1596 */ 1597 return (next_write_idx == read_idx); 1598 } 1599 1600 static inline void 1601 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1602 { 1603 1604 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1605 /* Clean D-cache under command. */ 1606 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); 1607 } else { 1608 /* DSB inner shareable, store */ 1609 dsb(ishst); 1610 } 1611 1612 } 1613 1614 static inline uint64_t 1615 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1616 { 1617 uint64_t off; 1618 1619 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1620 1621 return (off); 1622 } 1623 1624 static void 1625 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1626 struct its_cmd *cmd_last) 1627 { 1628 struct gicv3_its_softc *sc; 1629 uint64_t first, last, read; 1630 size_t us_left; 1631 1632 sc = device_get_softc(dev); 1633 1634 /* 1635 * XXX ARM64TODO: This is obviously a significant delay. 1636 * The reason for that is that currently the time frames for 1637 * the command to complete are not known. 1638 */ 1639 us_left = 1000000; 1640 1641 first = its_cmd_cwriter_offset(sc, cmd_first); 1642 last = its_cmd_cwriter_offset(sc, cmd_last); 1643 1644 for (;;) { 1645 read = gic_its_read_8(sc, GITS_CREADR); 1646 if (first < last) { 1647 if (read < first || read >= last) 1648 break; 1649 } else if (read < first && read >= last) 1650 break; 1651 1652 if (us_left-- == 0) { 1653 /* This means timeout */ 1654 device_printf(dev, 1655 "Timeout while waiting for CMD completion.\n"); 1656 return; 1657 } 1658 DELAY(1); 1659 } 1660 } 1661 1662 static struct its_cmd * 1663 its_cmd_alloc_locked(device_t dev) 1664 { 1665 struct gicv3_its_softc *sc; 1666 struct its_cmd *cmd; 1667 size_t us_left; 1668 1669 sc = device_get_softc(dev); 1670 1671 /* 1672 * XXX ARM64TODO: This is obviously a significant delay. 1673 * The reason for that is that currently the time frames for 1674 * the command to complete (and therefore free the descriptor) 1675 * are not known. 1676 */ 1677 us_left = 1000000; 1678 1679 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1680 while (its_cmd_queue_full(sc)) { 1681 if (us_left-- == 0) { 1682 /* Timeout while waiting for free command */ 1683 device_printf(dev, 1684 "Timeout while waiting for free command\n"); 1685 return (NULL); 1686 } 1687 DELAY(1); 1688 } 1689 1690 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1691 sc->sc_its_cmd_next_idx++; 1692 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1693 1694 return (cmd); 1695 } 1696 1697 static uint64_t 1698 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1699 { 1700 uint64_t target; 1701 uint8_t cmd_type; 1702 u_int size; 1703 1704 cmd_type = desc->cmd_type; 1705 target = ITS_TARGET_NONE; 1706 1707 switch (cmd_type) { 1708 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1709 target = desc->cmd_desc_movi.col->col_target; 1710 cmd_format_command(cmd, ITS_CMD_MOVI); 1711 cmd_format_id(cmd, desc->cmd_desc_movi.id); 1712 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1713 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1714 break; 1715 case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1716 target = desc->cmd_desc_sync.col->col_target; 1717 cmd_format_command(cmd, ITS_CMD_SYNC); 1718 cmd_format_target(cmd, target); 1719 break; 1720 case ITS_CMD_MAPD: /* Assign ITT to device */ 1721 cmd_format_command(cmd, ITS_CMD_MAPD); 1722 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1723 /* 1724 * Size describes number of bits to encode interrupt IDs 1725 * supported by the device minus one. 1726 * When V (valid) bit is zero, this field should be written 1727 * as zero. 1728 */ 1729 if (desc->cmd_desc_mapd.valid != 0) { 1730 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1731 size = MAX(1, size) - 1; 1732 } else 1733 size = 0; 1734 1735 cmd_format_size(cmd, size); 1736 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1737 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1738 break; 1739 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1740 target = desc->cmd_desc_mapc.col->col_target; 1741 cmd_format_command(cmd, ITS_CMD_MAPC); 1742 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 1743 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 1744 cmd_format_target(cmd, target); 1745 break; 1746 case ITS_CMD_MAPTI: 1747 target = desc->cmd_desc_mapvi.col->col_target; 1748 cmd_format_command(cmd, ITS_CMD_MAPTI); 1749 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 1750 cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 1751 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 1752 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 1753 break; 1754 case ITS_CMD_MAPI: 1755 target = desc->cmd_desc_mapi.col->col_target; 1756 cmd_format_command(cmd, ITS_CMD_MAPI); 1757 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 1758 cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 1759 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 1760 break; 1761 case ITS_CMD_INV: 1762 target = desc->cmd_desc_inv.col->col_target; 1763 cmd_format_command(cmd, ITS_CMD_INV); 1764 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 1765 cmd_format_id(cmd, desc->cmd_desc_inv.pid); 1766 break; 1767 case ITS_CMD_INVALL: 1768 cmd_format_command(cmd, ITS_CMD_INVALL); 1769 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 1770 break; 1771 default: 1772 panic("its_cmd_prepare: Invalid command: %x", cmd_type); 1773 } 1774 1775 return (target); 1776 } 1777 1778 static int 1779 its_cmd_send(device_t dev, struct its_cmd_desc *desc) 1780 { 1781 struct gicv3_its_softc *sc; 1782 struct its_cmd *cmd, *cmd_sync, *cmd_write; 1783 struct its_col col_sync; 1784 struct its_cmd_desc desc_sync; 1785 uint64_t target, cwriter; 1786 1787 sc = device_get_softc(dev); 1788 mtx_lock_spin(&sc->sc_its_cmd_lock); 1789 cmd = its_cmd_alloc_locked(dev); 1790 if (cmd == NULL) { 1791 device_printf(dev, "could not allocate ITS command\n"); 1792 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1793 return (EBUSY); 1794 } 1795 1796 target = its_cmd_prepare(cmd, desc); 1797 its_cmd_sync(sc, cmd); 1798 1799 if (target != ITS_TARGET_NONE) { 1800 cmd_sync = its_cmd_alloc_locked(dev); 1801 if (cmd_sync != NULL) { 1802 desc_sync.cmd_type = ITS_CMD_SYNC; 1803 col_sync.col_target = target; 1804 desc_sync.cmd_desc_sync.col = &col_sync; 1805 its_cmd_prepare(cmd_sync, &desc_sync); 1806 its_cmd_sync(sc, cmd_sync); 1807 } 1808 } 1809 1810 /* Update GITS_CWRITER */ 1811 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 1812 gic_its_write_8(sc, GITS_CWRITER, cwriter); 1813 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1814 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1815 1816 its_cmd_wait_completion(dev, cmd, cmd_write); 1817 1818 return (0); 1819 } 1820 1821 /* Handlers to send commands */ 1822 static void 1823 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 1824 { 1825 struct gicv3_its_softc *sc; 1826 struct its_cmd_desc desc; 1827 struct its_col *col; 1828 1829 sc = device_get_softc(dev); 1830 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1831 1832 desc.cmd_type = ITS_CMD_MOVI; 1833 desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 1834 desc.cmd_desc_movi.col = col; 1835 desc.cmd_desc_movi.id = girq->gi_id; 1836 1837 its_cmd_send(dev, &desc); 1838 } 1839 1840 static void 1841 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 1842 { 1843 struct its_cmd_desc desc; 1844 1845 desc.cmd_type = ITS_CMD_MAPC; 1846 desc.cmd_desc_mapc.col = col; 1847 /* 1848 * Valid bit set - map the collection. 1849 * Valid bit cleared - unmap the collection. 1850 */ 1851 desc.cmd_desc_mapc.valid = valid; 1852 1853 its_cmd_send(dev, &desc); 1854 } 1855 1856 static void 1857 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 1858 { 1859 struct gicv3_its_softc *sc; 1860 struct its_cmd_desc desc; 1861 struct its_col *col; 1862 u_int col_id; 1863 1864 sc = device_get_softc(dev); 1865 1866 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 1867 col = sc->sc_its_cols[col_id]; 1868 1869 desc.cmd_type = ITS_CMD_MAPTI; 1870 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 1871 desc.cmd_desc_mapvi.col = col; 1872 /* The EventID sent to the device */ 1873 desc.cmd_desc_mapvi.id = girq->gi_id; 1874 /* The physical interrupt presented to softeware */ 1875 desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI; 1876 1877 its_cmd_send(dev, &desc); 1878 } 1879 1880 static void 1881 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 1882 { 1883 struct its_cmd_desc desc; 1884 1885 desc.cmd_type = ITS_CMD_MAPD; 1886 desc.cmd_desc_mapd.its_dev = its_dev; 1887 desc.cmd_desc_mapd.valid = valid; 1888 1889 its_cmd_send(dev, &desc); 1890 } 1891 1892 static void 1893 its_cmd_inv(device_t dev, struct its_dev *its_dev, 1894 struct gicv3_its_irqsrc *girq) 1895 { 1896 struct gicv3_its_softc *sc; 1897 struct its_cmd_desc desc; 1898 struct its_col *col; 1899 1900 sc = device_get_softc(dev); 1901 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1902 1903 desc.cmd_type = ITS_CMD_INV; 1904 /* The EventID sent to the device */ 1905 desc.cmd_desc_inv.pid = girq->gi_id; 1906 desc.cmd_desc_inv.its_dev = its_dev; 1907 desc.cmd_desc_inv.col = col; 1908 1909 its_cmd_send(dev, &desc); 1910 } 1911 1912 static void 1913 its_cmd_invall(device_t dev, struct its_col *col) 1914 { 1915 struct its_cmd_desc desc; 1916 1917 desc.cmd_type = ITS_CMD_INVALL; 1918 desc.cmd_desc_invall.col = col; 1919 1920 its_cmd_send(dev, &desc); 1921 } 1922 1923 #ifdef FDT 1924 static device_probe_t gicv3_its_fdt_probe; 1925 static device_attach_t gicv3_its_fdt_attach; 1926 1927 static device_method_t gicv3_its_fdt_methods[] = { 1928 /* Device interface */ 1929 DEVMETHOD(device_probe, gicv3_its_fdt_probe), 1930 DEVMETHOD(device_attach, gicv3_its_fdt_attach), 1931 1932 /* End */ 1933 DEVMETHOD_END 1934 }; 1935 1936 #define its_baseclasses its_fdt_baseclasses 1937 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 1938 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1939 #undef its_baseclasses 1940 1941 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0, 1942 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1943 1944 static int 1945 gicv3_its_fdt_probe(device_t dev) 1946 { 1947 1948 if (!ofw_bus_status_okay(dev)) 1949 return (ENXIO); 1950 1951 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 1952 return (ENXIO); 1953 1954 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1955 return (BUS_PROBE_DEFAULT); 1956 } 1957 1958 static int 1959 gicv3_its_fdt_attach(device_t dev) 1960 { 1961 struct gicv3_its_softc *sc; 1962 phandle_t xref; 1963 int err; 1964 1965 sc = device_get_softc(dev); 1966 sc->dev = dev; 1967 err = gicv3_its_attach(dev); 1968 if (err != 0) 1969 return (err); 1970 1971 /* Register this device as a interrupt controller */ 1972 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 1973 sc->sc_pic = intr_pic_register(dev, xref); 1974 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1975 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 1976 if (err != 0) { 1977 device_printf(dev, "Failed to add PIC handler: %d\n", err); 1978 return (err); 1979 } 1980 1981 /* Register this device to handle MSI interrupts */ 1982 err = intr_msi_register(dev, xref); 1983 if (err != 0) { 1984 device_printf(dev, "Failed to register for MSIs: %d\n", err); 1985 return (err); 1986 } 1987 1988 return (0); 1989 } 1990 #endif 1991 1992 #ifdef DEV_ACPI 1993 static device_probe_t gicv3_its_acpi_probe; 1994 static device_attach_t gicv3_its_acpi_attach; 1995 1996 static device_method_t gicv3_its_acpi_methods[] = { 1997 /* Device interface */ 1998 DEVMETHOD(device_probe, gicv3_its_acpi_probe), 1999 DEVMETHOD(device_attach, gicv3_its_acpi_attach), 2000 2001 /* End */ 2002 DEVMETHOD_END 2003 }; 2004 2005 #define its_baseclasses its_acpi_baseclasses 2006 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods, 2007 sizeof(struct gicv3_its_softc), gicv3_its_driver); 2008 #undef its_baseclasses 2009 2010 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0, 2011 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 2012 2013 static int 2014 gicv3_its_acpi_probe(device_t dev) 2015 { 2016 2017 if (gic_get_bus(dev) != GIC_BUS_ACPI) 2018 return (EINVAL); 2019 2020 if (gic_get_hw_rev(dev) < 3) 2021 return (EINVAL); 2022 2023 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 2024 return (BUS_PROBE_DEFAULT); 2025 } 2026 2027 static int 2028 gicv3_its_acpi_attach(device_t dev) 2029 { 2030 struct gicv3_its_softc *sc; 2031 struct gic_v3_devinfo *di; 2032 int err; 2033 2034 sc = device_get_softc(dev); 2035 sc->dev = dev; 2036 err = gicv3_its_attach(dev); 2037 if (err != 0) 2038 return (err); 2039 2040 di = device_get_ivars(dev); 2041 sc->sc_pic = intr_pic_register(dev, di->msi_xref); 2042 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 2043 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 2044 if (err != 0) { 2045 device_printf(dev, "Failed to add PIC handler: %d\n", err); 2046 return (err); 2047 } 2048 2049 /* Register this device to handle MSI interrupts */ 2050 err = intr_msi_register(dev, di->msi_xref); 2051 if (err != 0) { 2052 device_printf(dev, "Failed to register for MSIs: %d\n", err); 2053 return (err); 2054 } 2055 2056 return (0); 2057 } 2058 #endif 2059