1 /*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Andrew Turner under 6 * the sponsorship of the FreeBSD Foundation. 7 * 8 * This software was developed by Semihalf under 9 * the sponsorship of the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include "opt_acpi.h" 34 #include "opt_platform.h" 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/bus.h> 42 #include <sys/cpuset.h> 43 #include <sys/endian.h> 44 #include <sys/kernel.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/module.h> 48 #include <sys/mutex.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/rman.h> 52 #include <sys/smp.h> 53 #include <sys/vmem.h> 54 55 #include <vm/vm.h> 56 #include <vm/pmap.h> 57 58 #include <machine/bus.h> 59 #include <machine/intr.h> 60 61 #include <arm/arm/gic_common.h> 62 #include <arm64/arm64/gic_v3_reg.h> 63 #include <arm64/arm64/gic_v3_var.h> 64 65 #ifdef FDT 66 #include <dev/ofw/openfirm.h> 67 #include <dev/ofw/ofw_bus.h> 68 #include <dev/ofw/ofw_bus_subr.h> 69 #endif 70 #include <dev/pci/pcireg.h> 71 #include <dev/pci/pcivar.h> 72 73 #include "pcib_if.h" 74 #include "pic_if.h" 75 #include "msi_if.h" 76 77 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 78 "ARM GICv3 Interrupt Translation Service"); 79 80 #define LPI_NIRQS (64 * 1024) 81 82 /* The size and alignment of the command circular buffer */ 83 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 84 #define ITS_CMDQ_ALIGN (64 * 1024) 85 86 #define LPI_CONFTAB_SIZE LPI_NIRQS 87 #define LPI_CONFTAB_ALIGN (64 * 1024) 88 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 89 90 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 91 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 92 #define LPI_PENDTAB_ALIGN (64 * 1024) 93 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 94 95 #define LPI_INT_TRANS_TAB_ALIGN 256 96 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 97 98 /* ITS commands encoding */ 99 #define ITS_CMD_MOVI (0x01) 100 #define ITS_CMD_SYNC (0x05) 101 #define ITS_CMD_MAPD (0x08) 102 #define ITS_CMD_MAPC (0x09) 103 #define ITS_CMD_MAPTI (0x0a) 104 #define ITS_CMD_MAPI (0x0b) 105 #define ITS_CMD_INV (0x0c) 106 #define ITS_CMD_INVALL (0x0d) 107 /* Command */ 108 #define CMD_COMMAND_MASK (0xFFUL) 109 /* PCI device ID */ 110 #define CMD_DEVID_SHIFT (32) 111 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 112 /* Size of IRQ ID bitfield */ 113 #define CMD_SIZE_MASK (0xFFUL) 114 /* Virtual LPI ID */ 115 #define CMD_ID_MASK (0xFFFFFFFFUL) 116 /* Physical LPI ID */ 117 #define CMD_PID_SHIFT (32) 118 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 119 /* Collection */ 120 #define CMD_COL_MASK (0xFFFFUL) 121 /* Target (CPU or Re-Distributor) */ 122 #define CMD_TARGET_SHIFT (16) 123 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 124 /* Interrupt Translation Table address */ 125 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 126 /* Valid command bit */ 127 #define CMD_VALID_SHIFT (63) 128 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 129 130 #define ITS_TARGET_NONE 0xFBADBEEF 131 132 /* LPI chunk owned by ITS device */ 133 struct lpi_chunk { 134 u_int lpi_base; 135 u_int lpi_free; /* First free LPI in set */ 136 u_int lpi_num; /* Total number of LPIs in chunk */ 137 u_int lpi_busy; /* Number of busy LPIs in chink */ 138 }; 139 140 /* ITS device */ 141 struct its_dev { 142 TAILQ_ENTRY(its_dev) entry; 143 /* PCI device */ 144 device_t pci_dev; 145 /* Device ID (i.e. PCI device ID) */ 146 uint32_t devid; 147 /* List of assigned LPIs */ 148 struct lpi_chunk lpis; 149 /* Virtual address of ITT */ 150 vm_offset_t itt; 151 size_t itt_size; 152 }; 153 154 /* 155 * ITS command descriptor. 156 * Idea for command description passing taken from Linux. 157 */ 158 struct its_cmd_desc { 159 uint8_t cmd_type; 160 161 union { 162 struct { 163 struct its_dev *its_dev; 164 struct its_col *col; 165 uint32_t id; 166 } cmd_desc_movi; 167 168 struct { 169 struct its_col *col; 170 } cmd_desc_sync; 171 172 struct { 173 struct its_col *col; 174 uint8_t valid; 175 } cmd_desc_mapc; 176 177 struct { 178 struct its_dev *its_dev; 179 struct its_col *col; 180 uint32_t pid; 181 uint32_t id; 182 } cmd_desc_mapvi; 183 184 struct { 185 struct its_dev *its_dev; 186 struct its_col *col; 187 uint32_t pid; 188 } cmd_desc_mapi; 189 190 struct { 191 struct its_dev *its_dev; 192 uint8_t valid; 193 } cmd_desc_mapd; 194 195 struct { 196 struct its_dev *its_dev; 197 struct its_col *col; 198 uint32_t pid; 199 } cmd_desc_inv; 200 201 struct { 202 struct its_col *col; 203 } cmd_desc_invall; 204 }; 205 }; 206 207 /* ITS command. Each command is 32 bytes long */ 208 struct its_cmd { 209 uint64_t cmd_dword[4]; /* ITS command double word */ 210 }; 211 212 /* An ITS private table */ 213 struct its_ptable { 214 vm_offset_t ptab_vaddr; 215 unsigned long ptab_size; 216 }; 217 218 /* ITS collection description. */ 219 struct its_col { 220 uint64_t col_target; /* Target Re-Distributor */ 221 uint64_t col_id; /* Collection ID */ 222 }; 223 224 struct gicv3_its_irqsrc { 225 struct intr_irqsrc gi_isrc; 226 u_int gi_irq; 227 struct its_dev *gi_its_dev; 228 }; 229 230 struct gicv3_its_softc { 231 struct intr_pic *sc_pic; 232 struct resource *sc_its_res; 233 234 cpuset_t sc_cpus; 235 u_int gic_irq_cpu; 236 237 struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 238 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 239 240 /* 241 * TODO: We should get these from the parent as we only want a 242 * single copy of each across the interrupt controller. 243 */ 244 vm_offset_t sc_conf_base; 245 vm_offset_t sc_pend_base[MAXCPU]; 246 247 /* Command handling */ 248 struct mtx sc_its_cmd_lock; 249 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 250 size_t sc_its_cmd_next_idx; 251 252 vmem_t *sc_irq_alloc; 253 struct gicv3_its_irqsrc *sc_irqs; 254 u_int sc_irq_base; 255 u_int sc_irq_length; 256 257 struct mtx sc_its_dev_lock; 258 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 259 260 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001 261 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 262 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 263 u_int sc_its_flags; 264 }; 265 266 typedef void (its_quirk_func_t)(device_t); 267 static its_quirk_func_t its_quirk_cavium_22375; 268 269 static const struct { 270 const char *desc; 271 uint32_t iidr; 272 uint32_t iidr_mask; 273 its_quirk_func_t *func; 274 } its_quirks[] = { 275 { 276 /* Cavium ThunderX Pass 1.x */ 277 .desc = "Cavium ThunderX errata: 22375, 24313", 278 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 279 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 280 .iidr_mask = ~GITS_IIDR_REVISION_MASK, 281 .func = its_quirk_cavium_22375, 282 }, 283 }; 284 285 #define gic_its_read_4(sc, reg) \ 286 bus_read_4((sc)->sc_its_res, (reg)) 287 #define gic_its_read_8(sc, reg) \ 288 bus_read_8((sc)->sc_its_res, (reg)) 289 290 #define gic_its_write_4(sc, reg, val) \ 291 bus_write_4((sc)->sc_its_res, (reg), (val)) 292 #define gic_its_write_8(sc, reg, val) \ 293 bus_write_8((sc)->sc_its_res, (reg), (val)) 294 295 static device_attach_t gicv3_its_attach; 296 static device_detach_t gicv3_its_detach; 297 298 static pic_disable_intr_t gicv3_its_disable_intr; 299 static pic_enable_intr_t gicv3_its_enable_intr; 300 static pic_map_intr_t gicv3_its_map_intr; 301 static pic_setup_intr_t gicv3_its_setup_intr; 302 static pic_post_filter_t gicv3_its_post_filter; 303 static pic_post_ithread_t gicv3_its_post_ithread; 304 static pic_pre_ithread_t gicv3_its_pre_ithread; 305 static pic_bind_intr_t gicv3_its_bind_intr; 306 #ifdef SMP 307 static pic_init_secondary_t gicv3_its_init_secondary; 308 #endif 309 static msi_alloc_msi_t gicv3_its_alloc_msi; 310 static msi_release_msi_t gicv3_its_release_msi; 311 static msi_alloc_msix_t gicv3_its_alloc_msix; 312 static msi_release_msix_t gicv3_its_release_msix; 313 static msi_map_msi_t gicv3_its_map_msi; 314 315 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 316 static void its_cmd_mapc(device_t, struct its_col *, uint8_t); 317 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 318 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 319 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 320 static void its_cmd_invall(device_t, struct its_col *); 321 322 static device_method_t gicv3_its_methods[] = { 323 /* Device interface */ 324 DEVMETHOD(device_detach, gicv3_its_detach), 325 326 /* Interrupt controller interface */ 327 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 328 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 329 DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 330 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 331 DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 332 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 333 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 334 #ifdef SMP 335 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 336 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 337 #endif 338 339 /* MSI/MSI-X */ 340 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 341 DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 342 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 343 DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 344 DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 345 346 /* End */ 347 DEVMETHOD_END 348 }; 349 350 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 351 sizeof(struct gicv3_its_softc)); 352 353 static void 354 gicv3_its_cmdq_init(struct gicv3_its_softc *sc) 355 { 356 vm_paddr_t cmd_paddr; 357 uint64_t reg, tmp; 358 359 /* Set up the command circular buffer */ 360 sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS, 361 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); 362 sc->sc_its_cmd_next_idx = 0; 363 364 cmd_paddr = vtophys(sc->sc_its_cmd_base); 365 366 /* Set the base of the command buffer */ 367 reg = GITS_CBASER_VALID | 368 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 369 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 370 (ITS_CMDQ_SIZE / 4096 - 1); 371 gic_its_write_8(sc, GITS_CBASER, reg); 372 373 /* Read back to check for fixed value fields */ 374 tmp = gic_its_read_8(sc, GITS_CBASER); 375 376 if ((tmp & GITS_CBASER_SHARE_MASK) != 377 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 378 /* Check if the hardware reported non-shareable */ 379 if ((tmp & GITS_CBASER_SHARE_MASK) == 380 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 381 /* If so remove the cache attribute */ 382 reg &= ~GITS_CBASER_CACHE_MASK; 383 reg &= ~GITS_CBASER_SHARE_MASK; 384 /* Set to Non-cacheable, Non-shareable */ 385 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 386 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 387 388 gic_its_write_8(sc, GITS_CBASER, reg); 389 } 390 391 /* The command queue has to be flushed after each command */ 392 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 393 } 394 395 /* Get the next command from the start of the buffer */ 396 gic_its_write_8(sc, GITS_CWRITER, 0x0); 397 } 398 399 static int 400 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 401 { 402 vm_offset_t table; 403 vm_paddr_t paddr; 404 uint64_t cache, reg, share, tmp, type; 405 size_t esize, its_tbl_size, nidents, nitspages, npages; 406 int i, page_size; 407 int devbits; 408 409 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 410 /* 411 * GITS_TYPER[17:13] of ThunderX reports that device IDs 412 * are to be 21 bits in length. The entry size of the ITS 413 * table can be read from GITS_BASERn[52:48] and on ThunderX 414 * is supposed to be 8 bytes in length (for device table). 415 * Finally the page size that is to be used by ITS to access 416 * this table will be set to 64KB. 417 * 418 * This gives 0x200000 entries of size 0x8 bytes covered by 419 * 256 pages each of which 64KB in size. The number of pages 420 * (minus 1) should then be written to GITS_BASERn[7:0]. In 421 * that case this value would be 0xFF but on ThunderX the 422 * maximum value that HW accepts is 0xFD. 423 * 424 * Set an arbitrary number of device ID bits to 20 in order 425 * to limit the number of entries in ITS device table to 426 * 0x100000 and the table size to 8MB. 427 */ 428 devbits = 20; 429 cache = 0; 430 } else { 431 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 432 cache = GITS_BASER_CACHE_WAWB; 433 } 434 share = GITS_BASER_SHARE_IS; 435 page_size = PAGE_SIZE_64K; 436 437 for (i = 0; i < GITS_BASER_NUM; i++) { 438 reg = gic_its_read_8(sc, GITS_BASER(i)); 439 /* The type of table */ 440 type = GITS_BASER_TYPE(reg); 441 /* The table entry size */ 442 esize = GITS_BASER_ESIZE(reg); 443 444 switch(type) { 445 case GITS_BASER_TYPE_DEV: 446 nidents = (1 << devbits); 447 its_tbl_size = esize * nidents; 448 its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); 449 break; 450 case GITS_BASER_TYPE_VP: 451 case GITS_BASER_TYPE_PP: /* Undocumented? */ 452 case GITS_BASER_TYPE_IC: 453 its_tbl_size = page_size; 454 break; 455 default: 456 continue; 457 } 458 npages = howmany(its_tbl_size, PAGE_SIZE); 459 460 /* Allocate the table */ 461 table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, 462 M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, 463 PAGE_SIZE_64K, 0); 464 465 sc->sc_its_ptab[i].ptab_vaddr = table; 466 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; 467 468 paddr = vtophys(table); 469 470 while (1) { 471 nitspages = howmany(its_tbl_size, page_size); 472 473 /* Clear the fields we will be setting */ 474 reg &= ~(GITS_BASER_VALID | 475 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 476 GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | 477 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 478 GITS_BASER_SIZE_MASK); 479 /* Set the new values */ 480 reg |= GITS_BASER_VALID | 481 (cache << GITS_BASER_CACHE_SHIFT) | 482 (type << GITS_BASER_TYPE_SHIFT) | 483 ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | 484 paddr | (share << GITS_BASER_SHARE_SHIFT) | 485 (nitspages - 1); 486 487 switch (page_size) { 488 case PAGE_SIZE: /* 4KB */ 489 reg |= 490 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 491 break; 492 case PAGE_SIZE_16K: /* 16KB */ 493 reg |= 494 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 495 break; 496 case PAGE_SIZE_64K: /* 64KB */ 497 reg |= 498 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 499 break; 500 } 501 502 gic_its_write_8(sc, GITS_BASER(i), reg); 503 504 /* Read back to check */ 505 tmp = gic_its_read_8(sc, GITS_BASER(i)); 506 507 /* Do the shareability masks line up? */ 508 if ((tmp & GITS_BASER_SHARE_MASK) != 509 (reg & GITS_BASER_SHARE_MASK)) { 510 share = (tmp & GITS_BASER_SHARE_MASK) >> 511 GITS_BASER_SHARE_SHIFT; 512 continue; 513 } 514 515 if ((tmp & GITS_BASER_PSZ_MASK) != 516 (reg & GITS_BASER_PSZ_MASK)) { 517 switch (page_size) { 518 case PAGE_SIZE_16K: 519 page_size = PAGE_SIZE; 520 continue; 521 case PAGE_SIZE_64K: 522 page_size = PAGE_SIZE_16K; 523 continue; 524 } 525 } 526 527 if (tmp != reg) { 528 device_printf(dev, "GITS_BASER%d: " 529 "unable to be updated: %lx != %lx\n", 530 i, reg, tmp); 531 return (ENXIO); 532 } 533 534 /* We should have made all needed changes */ 535 break; 536 } 537 } 538 539 return (0); 540 } 541 542 static void 543 gicv3_its_conftable_init(struct gicv3_its_softc *sc) 544 { 545 546 sc->sc_conf_base = (vm_offset_t)contigmalloc(LPI_CONFTAB_SIZE, 547 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, LPI_CONFTAB_ALIGN, 548 0); 549 550 /* Set the default configuration */ 551 memset((void *)sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 552 LPI_CONFTAB_SIZE); 553 554 /* Flush the table to memory */ 555 cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE); 556 } 557 558 static void 559 gicv3_its_pendtables_init(struct gicv3_its_softc *sc) 560 { 561 int i; 562 563 for (i = 0; i <= mp_maxid; i++) { 564 if (CPU_ISSET(i, &sc->sc_cpus) == 0) 565 continue; 566 567 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( 568 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 569 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 570 571 /* Flush so the ITS can see the memory */ 572 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i], 573 LPI_PENDTAB_SIZE); 574 } 575 } 576 577 static void 578 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) 579 { 580 device_t gicv3; 581 uint64_t xbaser, tmp; 582 uint32_t ctlr; 583 u_int cpuid; 584 585 gicv3 = device_get_parent(dev); 586 cpuid = PCPU_GET(cpuid); 587 588 /* Disable LPIs */ 589 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 590 ctlr &= ~GICR_CTLR_LPI_ENABLE; 591 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 592 593 /* Make sure changes are observable my the GIC */ 594 dsb(sy); 595 596 /* 597 * Set the redistributor base 598 */ 599 xbaser = vtophys(sc->sc_conf_base) | 600 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 601 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 602 (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 603 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 604 605 /* Check the cache attributes we set */ 606 tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 607 608 if ((tmp & GICR_PROPBASER_SHARE_MASK) != 609 (xbaser & GICR_PROPBASER_SHARE_MASK)) { 610 if ((tmp & GICR_PROPBASER_SHARE_MASK) == 611 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 612 /* We need to mark as non-cacheable */ 613 xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 614 GICR_PROPBASER_CACHE_MASK); 615 /* Non-cacheable */ 616 xbaser |= GICR_PROPBASER_CACHE_NIN << 617 GICR_PROPBASER_CACHE_SHIFT; 618 /* Non-sareable */ 619 xbaser |= GICR_PROPBASER_SHARE_NS << 620 GICR_PROPBASER_SHARE_SHIFT; 621 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 622 } 623 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 624 } 625 626 /* 627 * Set the LPI pending table base 628 */ 629 xbaser = vtophys(sc->sc_pend_base[cpuid]) | 630 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 631 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 632 633 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 634 635 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 636 637 if ((tmp & GICR_PENDBASER_SHARE_MASK) == 638 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 639 /* Clear the cahce and shareability bits */ 640 xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 641 GICR_PENDBASER_SHARE_MASK); 642 /* Mark as non-shareable */ 643 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 644 /* And non-cacheable */ 645 xbaser |= GICR_PENDBASER_CACHE_NIN << 646 GICR_PENDBASER_CACHE_SHIFT; 647 } 648 649 /* Enable LPIs */ 650 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 651 ctlr |= GICR_CTLR_LPI_ENABLE; 652 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 653 654 /* Make sure the GIC has seen everything */ 655 dsb(sy); 656 } 657 658 static int 659 its_init_cpu(device_t dev, struct gicv3_its_softc *sc) 660 { 661 device_t gicv3; 662 vm_paddr_t target; 663 u_int cpuid; 664 struct redist_pcpu *rpcpu; 665 666 gicv3 = device_get_parent(dev); 667 cpuid = PCPU_GET(cpuid); 668 if (!CPU_ISSET(cpuid, &sc->sc_cpus)) 669 return (0); 670 671 /* Check if the ITS is enabled on this CPU */ 672 if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) 673 return (ENXIO); 674 675 rpcpu = gicv3_get_redist(dev); 676 677 /* Do per-cpu LPI init once */ 678 if (!rpcpu->lpi_enabled) { 679 its_init_cpu_lpi(dev, sc); 680 rpcpu->lpi_enabled = true; 681 } 682 683 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 684 /* This ITS wants the redistributor physical address */ 685 target = vtophys(rman_get_virtual(&rpcpu->res)); 686 } else { 687 /* This ITS wants the unique processor number */ 688 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)); 689 } 690 691 sc->sc_its_cols[cpuid]->col_target = target; 692 sc->sc_its_cols[cpuid]->col_id = cpuid; 693 694 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 695 its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 696 697 return (0); 698 } 699 700 static int 701 gicv3_its_attach(device_t dev) 702 { 703 struct gicv3_its_softc *sc; 704 const char *name; 705 uint32_t iidr; 706 int domain, err, i, rid; 707 708 sc = device_get_softc(dev); 709 710 sc->sc_irq_length = gicv3_get_nirqs(dev); 711 sc->sc_irq_base = GIC_FIRST_LPI; 712 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; 713 714 rid = 0; 715 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 716 RF_ACTIVE); 717 if (sc->sc_its_res == NULL) { 718 device_printf(dev, "Could not allocate memory\n"); 719 return (ENXIO); 720 } 721 722 iidr = gic_its_read_4(sc, GITS_IIDR); 723 for (i = 0; i < nitems(its_quirks); i++) { 724 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 725 if (bootverbose) { 726 device_printf(dev, "Applying %s\n", 727 its_quirks[i].desc); 728 } 729 its_quirks[i].func(dev); 730 break; 731 } 732 } 733 734 /* Allocate the private tables */ 735 err = gicv3_its_table_init(dev, sc); 736 if (err != 0) 737 return (err); 738 739 /* Protects access to the device list */ 740 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 741 742 /* Protects access to the ITS command circular buffer. */ 743 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 744 745 CPU_ZERO(&sc->sc_cpus); 746 if (bus_get_domain(dev, &domain) == 0) { 747 if (domain < MAXMEMDOM) 748 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus); 749 } else { 750 CPU_COPY(&all_cpus, &sc->sc_cpus); 751 } 752 753 /* Allocate the command circular buffer */ 754 gicv3_its_cmdq_init(sc); 755 756 /* Allocate the per-CPU collections */ 757 for (int cpu = 0; cpu <= mp_maxid; cpu++) 758 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0) 759 sc->sc_its_cols[cpu] = malloc( 760 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 761 M_WAITOK | M_ZERO); 762 else 763 sc->sc_its_cols[cpu] = NULL; 764 765 /* Enable the ITS */ 766 gic_its_write_4(sc, GITS_CTLR, 767 gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); 768 769 /* Create the LPI configuration table */ 770 gicv3_its_conftable_init(sc); 771 772 /* And the pending tebles */ 773 gicv3_its_pendtables_init(sc); 774 775 /* Enable LPIs on this CPU */ 776 its_init_cpu(dev, sc); 777 778 TAILQ_INIT(&sc->sc_its_dev_list); 779 780 /* 781 * Create the vmem object to allocate INTRNG IRQs from. We try to 782 * use all IRQs not already used by the GICv3. 783 * XXX: This assumes there are no other interrupt controllers in the 784 * system. 785 */ 786 sc->sc_irq_alloc = vmem_create("GICv3 ITS IRQs", 0, 787 gicv3_get_nirqs(dev), 1, 1, M_FIRSTFIT | M_WAITOK); 788 789 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length, 790 M_GICV3_ITS, M_WAITOK | M_ZERO); 791 name = device_get_nameunit(dev); 792 for (i = 0; i < sc->sc_irq_length; i++) { 793 sc->sc_irqs[i].gi_irq = i; 794 err = intr_isrc_register(&sc->sc_irqs[i].gi_isrc, dev, 0, 795 "%s,%u", name, i); 796 } 797 798 return (0); 799 } 800 801 static int 802 gicv3_its_detach(device_t dev) 803 { 804 805 return (ENXIO); 806 } 807 808 static void 809 its_quirk_cavium_22375(device_t dev) 810 { 811 struct gicv3_its_softc *sc; 812 813 sc = device_get_softc(dev); 814 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 815 } 816 817 static void 818 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 819 { 820 struct gicv3_its_softc *sc; 821 struct gicv3_its_irqsrc *girq; 822 uint8_t *conf; 823 824 sc = device_get_softc(dev); 825 girq = (struct gicv3_its_irqsrc *)isrc; 826 conf = (uint8_t *)sc->sc_conf_base; 827 828 conf[girq->gi_irq] &= ~LPI_CONF_ENABLE; 829 830 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 831 /* Clean D-cache under command. */ 832 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 833 } else { 834 /* DSB inner shareable, store */ 835 dsb(ishst); 836 } 837 838 its_cmd_inv(dev, girq->gi_its_dev, girq); 839 } 840 841 static void 842 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 843 { 844 struct gicv3_its_softc *sc; 845 struct gicv3_its_irqsrc *girq; 846 uint8_t *conf; 847 848 sc = device_get_softc(dev); 849 girq = (struct gicv3_its_irqsrc *)isrc; 850 conf = (uint8_t *)sc->sc_conf_base; 851 852 conf[girq->gi_irq] |= LPI_CONF_ENABLE; 853 854 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 855 /* Clean D-cache under command. */ 856 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_irq], 1); 857 } else { 858 /* DSB inner shareable, store */ 859 dsb(ishst); 860 } 861 862 its_cmd_inv(dev, girq->gi_its_dev, girq); 863 } 864 865 static int 866 gicv3_its_intr(void *arg, uintptr_t irq) 867 { 868 struct gicv3_its_softc *sc = arg; 869 struct gicv3_its_irqsrc *girq; 870 struct trapframe *tf; 871 872 irq -= sc->sc_irq_base; 873 girq = &sc->sc_irqs[irq]; 874 if (girq == NULL) 875 panic("gicv3_its_intr: Invalid interrupt %ld", 876 irq + sc->sc_irq_base); 877 878 tf = curthread->td_intr_frame; 879 intr_isrc_dispatch(&girq->gi_isrc, tf); 880 return (FILTER_HANDLED); 881 } 882 883 static void 884 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 885 { 886 struct gicv3_its_irqsrc *girq; 887 struct gicv3_its_softc *sc; 888 889 sc = device_get_softc(dev); 890 girq = (struct gicv3_its_irqsrc *)isrc; 891 gicv3_its_disable_intr(dev, isrc); 892 gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base); 893 } 894 895 static void 896 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 897 { 898 899 gicv3_its_enable_intr(dev, isrc); 900 } 901 902 static void 903 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 904 { 905 struct gicv3_its_irqsrc *girq; 906 struct gicv3_its_softc *sc; 907 908 sc = device_get_softc(dev); 909 girq = (struct gicv3_its_irqsrc *)isrc; 910 gic_icc_write(EOIR1, girq->gi_irq + sc->sc_irq_base); 911 } 912 913 static int 914 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 915 { 916 struct gicv3_its_irqsrc *girq; 917 struct gicv3_its_softc *sc; 918 919 sc = device_get_softc(dev); 920 girq = (struct gicv3_its_irqsrc *)isrc; 921 if (CPU_EMPTY(&isrc->isrc_cpu)) { 922 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu, 923 &sc->sc_cpus); 924 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu); 925 } 926 927 its_cmd_movi(dev, girq); 928 929 return (0); 930 } 931 932 static int 933 gicv3_its_map_intr(device_t dev, struct intr_map_data *data, 934 struct intr_irqsrc **isrcp) 935 { 936 937 /* 938 * This should never happen, we only call this function to map 939 * interrupts found before the controller driver is ready. 940 */ 941 panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 942 } 943 944 static int 945 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 946 struct resource *res, struct intr_map_data *data) 947 { 948 949 /* Bind the interrupt to a CPU */ 950 gicv3_its_bind_intr(dev, isrc); 951 952 return (0); 953 } 954 955 #ifdef SMP 956 static void 957 gicv3_its_init_secondary(device_t dev) 958 { 959 struct gicv3_its_softc *sc; 960 961 sc = device_get_softc(dev); 962 963 /* 964 * This is fatal as otherwise we may bind interrupts to this CPU. 965 * We need a way to tell the interrupt framework to only bind to a 966 * subset of given CPUs when it performs the shuffle. 967 */ 968 if (its_init_cpu(dev, sc) != 0) 969 panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 970 PCPU_GET(cpuid)); 971 } 972 #endif 973 974 static uint32_t 975 its_get_devid(device_t pci_dev) 976 { 977 uintptr_t id; 978 979 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 980 panic("its_get_devid: Unable to get the MSI DeviceID"); 981 982 return (id); 983 } 984 985 static struct its_dev * 986 its_device_find(device_t dev, device_t child) 987 { 988 struct gicv3_its_softc *sc; 989 struct its_dev *its_dev = NULL; 990 991 sc = device_get_softc(dev); 992 993 mtx_lock_spin(&sc->sc_its_dev_lock); 994 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 995 if (its_dev->pci_dev == child) 996 break; 997 } 998 mtx_unlock_spin(&sc->sc_its_dev_lock); 999 1000 return (its_dev); 1001 } 1002 1003 static struct its_dev * 1004 its_device_get(device_t dev, device_t child, u_int nvecs) 1005 { 1006 struct gicv3_its_softc *sc; 1007 struct its_dev *its_dev; 1008 vmem_addr_t irq_base; 1009 size_t esize; 1010 1011 sc = device_get_softc(dev); 1012 1013 its_dev = its_device_find(dev, child); 1014 if (its_dev != NULL) 1015 return (its_dev); 1016 1017 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 1018 if (its_dev == NULL) 1019 return (NULL); 1020 1021 its_dev->pci_dev = child; 1022 its_dev->devid = its_get_devid(child); 1023 1024 its_dev->lpis.lpi_busy = 0; 1025 its_dev->lpis.lpi_num = nvecs; 1026 its_dev->lpis.lpi_free = nvecs; 1027 1028 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 1029 &irq_base) != 0) { 1030 free(its_dev, M_GICV3_ITS); 1031 return (NULL); 1032 } 1033 its_dev->lpis.lpi_base = irq_base; 1034 1035 /* Get ITT entry size */ 1036 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 1037 1038 /* 1039 * Allocate ITT for this device. 1040 * PA has to be 256 B aligned. At least two entries for device. 1041 */ 1042 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 1043 its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size, 1044 M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, 1045 LPI_INT_TRANS_TAB_ALIGN, 0); 1046 if (its_dev->itt == 0) { 1047 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 1048 free(its_dev, M_GICV3_ITS); 1049 return (NULL); 1050 } 1051 1052 mtx_lock_spin(&sc->sc_its_dev_lock); 1053 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 1054 mtx_unlock_spin(&sc->sc_its_dev_lock); 1055 1056 /* Map device to its ITT */ 1057 its_cmd_mapd(dev, its_dev, 1); 1058 1059 return (its_dev); 1060 } 1061 1062 static void 1063 its_device_release(device_t dev, struct its_dev *its_dev) 1064 { 1065 struct gicv3_its_softc *sc; 1066 1067 KASSERT(its_dev->lpis.lpi_busy == 0, 1068 ("its_device_release: Trying to release an inuse ITS device")); 1069 1070 /* Unmap device in ITS */ 1071 its_cmd_mapd(dev, its_dev, 0); 1072 1073 sc = device_get_softc(dev); 1074 1075 /* Remove the device from the list of devices */ 1076 mtx_lock_spin(&sc->sc_its_dev_lock); 1077 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 1078 mtx_unlock_spin(&sc->sc_its_dev_lock); 1079 1080 /* Free ITT */ 1081 KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); 1082 contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 1083 1084 /* Free the IRQ allocation */ 1085 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 1086 its_dev->lpis.lpi_num); 1087 1088 free(its_dev, M_GICV3_ITS); 1089 } 1090 1091 static int 1092 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 1093 device_t *pic, struct intr_irqsrc **srcs) 1094 { 1095 struct gicv3_its_softc *sc; 1096 struct gicv3_its_irqsrc *girq; 1097 struct its_dev *its_dev; 1098 u_int irq; 1099 int i; 1100 1101 its_dev = its_device_get(dev, child, count); 1102 if (its_dev == NULL) 1103 return (ENXIO); 1104 1105 KASSERT(its_dev->lpis.lpi_free >= count, 1106 ("gicv3_its_alloc_msi: No free LPIs")); 1107 sc = device_get_softc(dev); 1108 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1109 its_dev->lpis.lpi_free; 1110 for (i = 0; i < count; i++, irq++) { 1111 its_dev->lpis.lpi_free--; 1112 girq = &sc->sc_irqs[irq]; 1113 girq->gi_its_dev = its_dev; 1114 srcs[i] = (struct intr_irqsrc *)girq; 1115 } 1116 its_dev->lpis.lpi_busy += count; 1117 *pic = dev; 1118 1119 return (0); 1120 } 1121 1122 static int 1123 gicv3_its_release_msi(device_t dev, device_t child, int count, 1124 struct intr_irqsrc **isrc) 1125 { 1126 struct gicv3_its_irqsrc *girq; 1127 struct its_dev *its_dev; 1128 int i; 1129 1130 its_dev = its_device_find(dev, child); 1131 1132 KASSERT(its_dev != NULL, 1133 ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1134 "no ITS device")); 1135 KASSERT(its_dev->lpis.lpi_busy >= count, 1136 ("gicv3_its_release_msi: Releasing more interrupts than " 1137 "were allocated: releasing %d, allocated %d", count, 1138 its_dev->lpis.lpi_busy)); 1139 for (i = 0; i < count; i++) { 1140 girq = (struct gicv3_its_irqsrc *)isrc[i]; 1141 girq->gi_its_dev = NULL; 1142 } 1143 its_dev->lpis.lpi_busy -= count; 1144 1145 if (its_dev->lpis.lpi_busy == 0) 1146 its_device_release(dev, its_dev); 1147 1148 return (0); 1149 } 1150 1151 static int 1152 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1153 struct intr_irqsrc **isrcp) 1154 { 1155 struct gicv3_its_softc *sc; 1156 struct gicv3_its_irqsrc *girq; 1157 struct its_dev *its_dev; 1158 u_int nvecs, irq; 1159 1160 nvecs = pci_msix_count(child); 1161 its_dev = its_device_get(dev, child, nvecs); 1162 if (its_dev == NULL) 1163 return (ENXIO); 1164 1165 KASSERT(its_dev->lpis.lpi_free > 0, 1166 ("gicv3_its_alloc_msix: No free LPIs")); 1167 sc = device_get_softc(dev); 1168 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1169 its_dev->lpis.lpi_free; 1170 its_dev->lpis.lpi_free--; 1171 its_dev->lpis.lpi_busy++; 1172 girq = &sc->sc_irqs[irq]; 1173 girq->gi_its_dev = its_dev; 1174 1175 *pic = dev; 1176 *isrcp = (struct intr_irqsrc *)girq; 1177 1178 return (0); 1179 } 1180 1181 static int 1182 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1183 { 1184 struct gicv3_its_irqsrc *girq; 1185 struct its_dev *its_dev; 1186 1187 its_dev = its_device_find(dev, child); 1188 1189 KASSERT(its_dev != NULL, 1190 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1191 "no ITS device")); 1192 KASSERT(its_dev->lpis.lpi_busy > 0, 1193 ("gicv3_its_release_msix: Releasing more interrupts than " 1194 "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1195 girq = (struct gicv3_its_irqsrc *)isrc; 1196 girq->gi_its_dev = NULL; 1197 its_dev->lpis.lpi_busy--; 1198 1199 if (its_dev->lpis.lpi_busy == 0) 1200 its_device_release(dev, its_dev); 1201 1202 return (0); 1203 } 1204 1205 static int 1206 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1207 uint64_t *addr, uint32_t *data) 1208 { 1209 struct gicv3_its_softc *sc; 1210 struct gicv3_its_irqsrc *girq; 1211 1212 sc = device_get_softc(dev); 1213 girq = (struct gicv3_its_irqsrc *)isrc; 1214 1215 /* Map the message to the given IRQ */ 1216 its_cmd_mapti(dev, girq); 1217 1218 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1219 *data = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1220 1221 return (0); 1222 } 1223 1224 /* 1225 * Commands handling. 1226 */ 1227 1228 static __inline void 1229 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1230 { 1231 /* Command field: DW0 [7:0] */ 1232 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1233 cmd->cmd_dword[0] |= htole64(cmd_type); 1234 } 1235 1236 static __inline void 1237 cmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1238 { 1239 /* Device ID field: DW0 [63:32] */ 1240 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1241 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1242 } 1243 1244 static __inline void 1245 cmd_format_size(struct its_cmd *cmd, uint16_t size) 1246 { 1247 /* Size field: DW1 [4:0] */ 1248 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1249 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1250 } 1251 1252 static __inline void 1253 cmd_format_id(struct its_cmd *cmd, uint32_t id) 1254 { 1255 /* ID field: DW1 [31:0] */ 1256 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1257 cmd->cmd_dword[1] |= htole64(id); 1258 } 1259 1260 static __inline void 1261 cmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1262 { 1263 /* Physical ID field: DW1 [63:32] */ 1264 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1265 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1266 } 1267 1268 static __inline void 1269 cmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1270 { 1271 /* Collection field: DW2 [16:0] */ 1272 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1273 cmd->cmd_dword[2] |= htole64(col_id); 1274 } 1275 1276 static __inline void 1277 cmd_format_target(struct its_cmd *cmd, uint64_t target) 1278 { 1279 /* Target Address field: DW2 [47:16] */ 1280 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1281 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1282 } 1283 1284 static __inline void 1285 cmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1286 { 1287 /* ITT Address field: DW2 [47:8] */ 1288 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1289 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1290 } 1291 1292 static __inline void 1293 cmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1294 { 1295 /* Valid field: DW2 [63] */ 1296 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1297 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1298 } 1299 1300 static inline bool 1301 its_cmd_queue_full(struct gicv3_its_softc *sc) 1302 { 1303 size_t read_idx, next_write_idx; 1304 1305 /* Get the index of the next command */ 1306 next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1307 (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1308 /* And the index of the current command being read */ 1309 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1310 1311 /* 1312 * The queue is full when the write offset points 1313 * at the command before the current read offset. 1314 */ 1315 return (next_write_idx == read_idx); 1316 } 1317 1318 static inline void 1319 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1320 { 1321 1322 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1323 /* Clean D-cache under command. */ 1324 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); 1325 } else { 1326 /* DSB inner shareable, store */ 1327 dsb(ishst); 1328 } 1329 1330 } 1331 1332 static inline uint64_t 1333 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1334 { 1335 uint64_t off; 1336 1337 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1338 1339 return (off); 1340 } 1341 1342 static void 1343 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1344 struct its_cmd *cmd_last) 1345 { 1346 struct gicv3_its_softc *sc; 1347 uint64_t first, last, read; 1348 size_t us_left; 1349 1350 sc = device_get_softc(dev); 1351 1352 /* 1353 * XXX ARM64TODO: This is obviously a significant delay. 1354 * The reason for that is that currently the time frames for 1355 * the command to complete are not known. 1356 */ 1357 us_left = 1000000; 1358 1359 first = its_cmd_cwriter_offset(sc, cmd_first); 1360 last = its_cmd_cwriter_offset(sc, cmd_last); 1361 1362 for (;;) { 1363 read = gic_its_read_8(sc, GITS_CREADR); 1364 if (first < last) { 1365 if (read < first || read >= last) 1366 break; 1367 } else if (read < first && read >= last) 1368 break; 1369 1370 if (us_left-- == 0) { 1371 /* This means timeout */ 1372 device_printf(dev, 1373 "Timeout while waiting for CMD completion.\n"); 1374 return; 1375 } 1376 DELAY(1); 1377 } 1378 } 1379 1380 1381 static struct its_cmd * 1382 its_cmd_alloc_locked(device_t dev) 1383 { 1384 struct gicv3_its_softc *sc; 1385 struct its_cmd *cmd; 1386 size_t us_left; 1387 1388 sc = device_get_softc(dev); 1389 1390 /* 1391 * XXX ARM64TODO: This is obviously a significant delay. 1392 * The reason for that is that currently the time frames for 1393 * the command to complete (and therefore free the descriptor) 1394 * are not known. 1395 */ 1396 us_left = 1000000; 1397 1398 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1399 while (its_cmd_queue_full(sc)) { 1400 if (us_left-- == 0) { 1401 /* Timeout while waiting for free command */ 1402 device_printf(dev, 1403 "Timeout while waiting for free command\n"); 1404 return (NULL); 1405 } 1406 DELAY(1); 1407 } 1408 1409 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1410 sc->sc_its_cmd_next_idx++; 1411 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1412 1413 return (cmd); 1414 } 1415 1416 static uint64_t 1417 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1418 { 1419 uint64_t target; 1420 uint8_t cmd_type; 1421 u_int size; 1422 1423 cmd_type = desc->cmd_type; 1424 target = ITS_TARGET_NONE; 1425 1426 switch (cmd_type) { 1427 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1428 target = desc->cmd_desc_movi.col->col_target; 1429 cmd_format_command(cmd, ITS_CMD_MOVI); 1430 cmd_format_id(cmd, desc->cmd_desc_movi.id); 1431 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1432 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1433 break; 1434 case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1435 target = desc->cmd_desc_sync.col->col_target; 1436 cmd_format_command(cmd, ITS_CMD_SYNC); 1437 cmd_format_target(cmd, target); 1438 break; 1439 case ITS_CMD_MAPD: /* Assign ITT to device */ 1440 cmd_format_command(cmd, ITS_CMD_MAPD); 1441 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1442 /* 1443 * Size describes number of bits to encode interrupt IDs 1444 * supported by the device minus one. 1445 * When V (valid) bit is zero, this field should be written 1446 * as zero. 1447 */ 1448 if (desc->cmd_desc_mapd.valid != 0) { 1449 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1450 size = MAX(1, size) - 1; 1451 } else 1452 size = 0; 1453 1454 cmd_format_size(cmd, size); 1455 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1456 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1457 break; 1458 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1459 target = desc->cmd_desc_mapc.col->col_target; 1460 cmd_format_command(cmd, ITS_CMD_MAPC); 1461 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 1462 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 1463 cmd_format_target(cmd, target); 1464 break; 1465 case ITS_CMD_MAPTI: 1466 target = desc->cmd_desc_mapvi.col->col_target; 1467 cmd_format_command(cmd, ITS_CMD_MAPTI); 1468 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 1469 cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 1470 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 1471 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 1472 break; 1473 case ITS_CMD_MAPI: 1474 target = desc->cmd_desc_mapi.col->col_target; 1475 cmd_format_command(cmd, ITS_CMD_MAPI); 1476 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 1477 cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 1478 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 1479 break; 1480 case ITS_CMD_INV: 1481 target = desc->cmd_desc_inv.col->col_target; 1482 cmd_format_command(cmd, ITS_CMD_INV); 1483 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 1484 cmd_format_id(cmd, desc->cmd_desc_inv.pid); 1485 break; 1486 case ITS_CMD_INVALL: 1487 cmd_format_command(cmd, ITS_CMD_INVALL); 1488 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 1489 break; 1490 default: 1491 panic("its_cmd_prepare: Invalid command: %x", cmd_type); 1492 } 1493 1494 return (target); 1495 } 1496 1497 static int 1498 its_cmd_send(device_t dev, struct its_cmd_desc *desc) 1499 { 1500 struct gicv3_its_softc *sc; 1501 struct its_cmd *cmd, *cmd_sync, *cmd_write; 1502 struct its_col col_sync; 1503 struct its_cmd_desc desc_sync; 1504 uint64_t target, cwriter; 1505 1506 sc = device_get_softc(dev); 1507 mtx_lock_spin(&sc->sc_its_cmd_lock); 1508 cmd = its_cmd_alloc_locked(dev); 1509 if (cmd == NULL) { 1510 device_printf(dev, "could not allocate ITS command\n"); 1511 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1512 return (EBUSY); 1513 } 1514 1515 target = its_cmd_prepare(cmd, desc); 1516 its_cmd_sync(sc, cmd); 1517 1518 if (target != ITS_TARGET_NONE) { 1519 cmd_sync = its_cmd_alloc_locked(dev); 1520 if (cmd_sync != NULL) { 1521 desc_sync.cmd_type = ITS_CMD_SYNC; 1522 col_sync.col_target = target; 1523 desc_sync.cmd_desc_sync.col = &col_sync; 1524 its_cmd_prepare(cmd_sync, &desc_sync); 1525 its_cmd_sync(sc, cmd_sync); 1526 } 1527 } 1528 1529 /* Update GITS_CWRITER */ 1530 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 1531 gic_its_write_8(sc, GITS_CWRITER, cwriter); 1532 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1533 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1534 1535 its_cmd_wait_completion(dev, cmd, cmd_write); 1536 1537 return (0); 1538 } 1539 1540 /* Handlers to send commands */ 1541 static void 1542 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 1543 { 1544 struct gicv3_its_softc *sc; 1545 struct its_cmd_desc desc; 1546 struct its_col *col; 1547 1548 sc = device_get_softc(dev); 1549 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1550 1551 desc.cmd_type = ITS_CMD_MOVI; 1552 desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 1553 desc.cmd_desc_movi.col = col; 1554 desc.cmd_desc_movi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1555 1556 its_cmd_send(dev, &desc); 1557 } 1558 1559 static void 1560 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 1561 { 1562 struct its_cmd_desc desc; 1563 1564 desc.cmd_type = ITS_CMD_MAPC; 1565 desc.cmd_desc_mapc.col = col; 1566 /* 1567 * Valid bit set - map the collection. 1568 * Valid bit cleared - unmap the collection. 1569 */ 1570 desc.cmd_desc_mapc.valid = valid; 1571 1572 its_cmd_send(dev, &desc); 1573 } 1574 1575 static void 1576 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 1577 { 1578 struct gicv3_its_softc *sc; 1579 struct its_cmd_desc desc; 1580 struct its_col *col; 1581 u_int col_id; 1582 1583 sc = device_get_softc(dev); 1584 1585 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 1586 col = sc->sc_its_cols[col_id]; 1587 1588 desc.cmd_type = ITS_CMD_MAPTI; 1589 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 1590 desc.cmd_desc_mapvi.col = col; 1591 /* The EventID sent to the device */ 1592 desc.cmd_desc_mapvi.id = girq->gi_irq - girq->gi_its_dev->lpis.lpi_base; 1593 /* The physical interrupt presented to softeware */ 1594 desc.cmd_desc_mapvi.pid = girq->gi_irq + sc->sc_irq_base; 1595 1596 its_cmd_send(dev, &desc); 1597 } 1598 1599 static void 1600 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 1601 { 1602 struct its_cmd_desc desc; 1603 1604 desc.cmd_type = ITS_CMD_MAPD; 1605 desc.cmd_desc_mapd.its_dev = its_dev; 1606 desc.cmd_desc_mapd.valid = valid; 1607 1608 its_cmd_send(dev, &desc); 1609 } 1610 1611 static void 1612 its_cmd_inv(device_t dev, struct its_dev *its_dev, 1613 struct gicv3_its_irqsrc *girq) 1614 { 1615 struct gicv3_its_softc *sc; 1616 struct its_cmd_desc desc; 1617 struct its_col *col; 1618 1619 sc = device_get_softc(dev); 1620 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1621 1622 desc.cmd_type = ITS_CMD_INV; 1623 /* The EventID sent to the device */ 1624 desc.cmd_desc_inv.pid = girq->gi_irq - its_dev->lpis.lpi_base; 1625 desc.cmd_desc_inv.its_dev = its_dev; 1626 desc.cmd_desc_inv.col = col; 1627 1628 its_cmd_send(dev, &desc); 1629 } 1630 1631 static void 1632 its_cmd_invall(device_t dev, struct its_col *col) 1633 { 1634 struct its_cmd_desc desc; 1635 1636 desc.cmd_type = ITS_CMD_INVALL; 1637 desc.cmd_desc_invall.col = col; 1638 1639 its_cmd_send(dev, &desc); 1640 } 1641 1642 #ifdef FDT 1643 static device_probe_t gicv3_its_fdt_probe; 1644 static device_attach_t gicv3_its_fdt_attach; 1645 1646 static device_method_t gicv3_its_fdt_methods[] = { 1647 /* Device interface */ 1648 DEVMETHOD(device_probe, gicv3_its_fdt_probe), 1649 DEVMETHOD(device_attach, gicv3_its_fdt_attach), 1650 1651 /* End */ 1652 DEVMETHOD_END 1653 }; 1654 1655 #define its_baseclasses its_fdt_baseclasses 1656 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 1657 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1658 #undef its_baseclasses 1659 static devclass_t gicv3_its_fdt_devclass; 1660 1661 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 1662 gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1663 1664 static int 1665 gicv3_its_fdt_probe(device_t dev) 1666 { 1667 1668 if (!ofw_bus_status_okay(dev)) 1669 return (ENXIO); 1670 1671 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 1672 return (ENXIO); 1673 1674 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1675 return (BUS_PROBE_DEFAULT); 1676 } 1677 1678 static int 1679 gicv3_its_fdt_attach(device_t dev) 1680 { 1681 struct gicv3_its_softc *sc; 1682 phandle_t xref; 1683 int err; 1684 1685 sc = device_get_softc(dev); 1686 err = gicv3_its_attach(dev); 1687 if (err != 0) 1688 return (err); 1689 1690 /* Register this device as a interrupt controller */ 1691 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 1692 sc->sc_pic = intr_pic_register(dev, xref); 1693 intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1694 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 1695 1696 /* Register this device to handle MSI interrupts */ 1697 intr_msi_register(dev, xref); 1698 1699 return (0); 1700 } 1701 #endif 1702 1703 #ifdef DEV_ACPI 1704 static device_probe_t gicv3_its_acpi_probe; 1705 static device_attach_t gicv3_its_acpi_attach; 1706 1707 static device_method_t gicv3_its_acpi_methods[] = { 1708 /* Device interface */ 1709 DEVMETHOD(device_probe, gicv3_its_acpi_probe), 1710 DEVMETHOD(device_attach, gicv3_its_acpi_attach), 1711 1712 /* End */ 1713 DEVMETHOD_END 1714 }; 1715 1716 #define its_baseclasses its_acpi_baseclasses 1717 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods, 1718 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1719 #undef its_baseclasses 1720 static devclass_t gicv3_its_acpi_devclass; 1721 1722 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 1723 gicv3_its_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1724 1725 static int 1726 gicv3_its_acpi_probe(device_t dev) 1727 { 1728 1729 if (gic_get_bus(dev) != GIC_BUS_ACPI) 1730 return (EINVAL); 1731 1732 if (gic_get_hw_rev(dev) < 3) 1733 return (EINVAL); 1734 1735 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1736 return (BUS_PROBE_DEFAULT); 1737 } 1738 1739 static int 1740 gicv3_its_acpi_attach(device_t dev) 1741 { 1742 struct gicv3_its_softc *sc; 1743 struct gic_v3_devinfo *di; 1744 int err; 1745 1746 sc = device_get_softc(dev); 1747 err = gicv3_its_attach(dev); 1748 if (err != 0) 1749 return (err); 1750 1751 di = device_get_ivars(dev); 1752 sc->sc_pic = intr_pic_register(dev, di->msi_xref); 1753 intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1754 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 1755 1756 /* Register this device to handle MSI interrupts */ 1757 intr_msi_register(dev, di->msi_xref); 1758 1759 return (0); 1760 } 1761 #endif 1762