1 /*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * All rights reserved. 4 * 5 * This software was developed by Andrew Turner under 6 * the sponsorship of the FreeBSD Foundation. 7 * 8 * This software was developed by Semihalf under 9 * the sponsorship of the FreeBSD Foundation. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 30 * SUCH DAMAGE. 31 */ 32 33 #include "opt_acpi.h" 34 #include "opt_platform.h" 35 36 #include <sys/cdefs.h> 37 __FBSDID("$FreeBSD$"); 38 39 #include <sys/param.h> 40 #include <sys/systm.h> 41 #include <sys/bus.h> 42 #include <sys/cpuset.h> 43 #include <sys/endian.h> 44 #include <sys/kernel.h> 45 #include <sys/lock.h> 46 #include <sys/malloc.h> 47 #include <sys/module.h> 48 #include <sys/mutex.h> 49 #include <sys/proc.h> 50 #include <sys/queue.h> 51 #include <sys/rman.h> 52 #include <sys/sbuf.h> 53 #include <sys/smp.h> 54 #include <sys/sysctl.h> 55 #include <sys/vmem.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 60 #include <machine/bus.h> 61 #include <machine/intr.h> 62 63 #include <arm/arm/gic_common.h> 64 #include <arm64/arm64/gic_v3_reg.h> 65 #include <arm64/arm64/gic_v3_var.h> 66 67 #ifdef FDT 68 #include <dev/ofw/openfirm.h> 69 #include <dev/ofw/ofw_bus.h> 70 #include <dev/ofw/ofw_bus_subr.h> 71 #endif 72 #include <dev/pci/pcireg.h> 73 #include <dev/pci/pcivar.h> 74 75 #include "pcib_if.h" 76 #include "pic_if.h" 77 #include "msi_if.h" 78 79 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS", 80 "ARM GICv3 Interrupt Translation Service"); 81 82 #define LPI_NIRQS (64 * 1024) 83 84 /* The size and alignment of the command circular buffer */ 85 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */ 86 #define ITS_CMDQ_ALIGN (64 * 1024) 87 88 #define LPI_CONFTAB_SIZE LPI_NIRQS 89 #define LPI_CONFTAB_ALIGN (64 * 1024) 90 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 91 92 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */ 93 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8) 94 #define LPI_PENDTAB_ALIGN (64 * 1024) 95 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */ 96 97 #define LPI_INT_TRANS_TAB_ALIGN 256 98 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1) 99 100 /* ITS commands encoding */ 101 #define ITS_CMD_MOVI (0x01) 102 #define ITS_CMD_SYNC (0x05) 103 #define ITS_CMD_MAPD (0x08) 104 #define ITS_CMD_MAPC (0x09) 105 #define ITS_CMD_MAPTI (0x0a) 106 #define ITS_CMD_MAPI (0x0b) 107 #define ITS_CMD_INV (0x0c) 108 #define ITS_CMD_INVALL (0x0d) 109 /* Command */ 110 #define CMD_COMMAND_MASK (0xFFUL) 111 /* PCI device ID */ 112 #define CMD_DEVID_SHIFT (32) 113 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT) 114 /* Size of IRQ ID bitfield */ 115 #define CMD_SIZE_MASK (0xFFUL) 116 /* Virtual LPI ID */ 117 #define CMD_ID_MASK (0xFFFFFFFFUL) 118 /* Physical LPI ID */ 119 #define CMD_PID_SHIFT (32) 120 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT) 121 /* Collection */ 122 #define CMD_COL_MASK (0xFFFFUL) 123 /* Target (CPU or Re-Distributor) */ 124 #define CMD_TARGET_SHIFT (16) 125 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT) 126 /* Interrupt Translation Table address */ 127 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL) 128 /* Valid command bit */ 129 #define CMD_VALID_SHIFT (63) 130 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT) 131 132 #define ITS_TARGET_NONE 0xFBADBEEF 133 134 /* LPI chunk owned by ITS device */ 135 struct lpi_chunk { 136 u_int lpi_base; 137 u_int lpi_free; /* First free LPI in set */ 138 u_int lpi_num; /* Total number of LPIs in chunk */ 139 u_int lpi_busy; /* Number of busy LPIs in chink */ 140 }; 141 142 /* ITS device */ 143 struct its_dev { 144 TAILQ_ENTRY(its_dev) entry; 145 /* PCI device */ 146 device_t pci_dev; 147 /* Device ID (i.e. PCI device ID) */ 148 uint32_t devid; 149 /* List of assigned LPIs */ 150 struct lpi_chunk lpis; 151 /* Virtual address of ITT */ 152 vm_offset_t itt; 153 size_t itt_size; 154 }; 155 156 /* 157 * ITS command descriptor. 158 * Idea for command description passing taken from Linux. 159 */ 160 struct its_cmd_desc { 161 uint8_t cmd_type; 162 163 union { 164 struct { 165 struct its_dev *its_dev; 166 struct its_col *col; 167 uint32_t id; 168 } cmd_desc_movi; 169 170 struct { 171 struct its_col *col; 172 } cmd_desc_sync; 173 174 struct { 175 struct its_col *col; 176 uint8_t valid; 177 } cmd_desc_mapc; 178 179 struct { 180 struct its_dev *its_dev; 181 struct its_col *col; 182 uint32_t pid; 183 uint32_t id; 184 } cmd_desc_mapvi; 185 186 struct { 187 struct its_dev *its_dev; 188 struct its_col *col; 189 uint32_t pid; 190 } cmd_desc_mapi; 191 192 struct { 193 struct its_dev *its_dev; 194 uint8_t valid; 195 } cmd_desc_mapd; 196 197 struct { 198 struct its_dev *its_dev; 199 struct its_col *col; 200 uint32_t pid; 201 } cmd_desc_inv; 202 203 struct { 204 struct its_col *col; 205 } cmd_desc_invall; 206 }; 207 }; 208 209 /* ITS command. Each command is 32 bytes long */ 210 struct its_cmd { 211 uint64_t cmd_dword[4]; /* ITS command double word */ 212 }; 213 214 /* An ITS private table */ 215 struct its_ptable { 216 vm_offset_t ptab_vaddr; 217 unsigned long ptab_size; 218 }; 219 220 /* ITS collection description. */ 221 struct its_col { 222 uint64_t col_target; /* Target Re-Distributor */ 223 uint64_t col_id; /* Collection ID */ 224 }; 225 226 struct gicv3_its_irqsrc { 227 struct intr_irqsrc gi_isrc; 228 u_int gi_id; 229 u_int gi_lpi; 230 struct its_dev *gi_its_dev; 231 TAILQ_ENTRY(gicv3_its_irqsrc) gi_link; 232 }; 233 234 struct gicv3_its_softc { 235 device_t dev; 236 struct intr_pic *sc_pic; 237 struct resource *sc_its_res; 238 239 cpuset_t sc_cpus; 240 u_int gic_irq_cpu; 241 242 struct its_ptable sc_its_ptab[GITS_BASER_NUM]; 243 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */ 244 245 /* 246 * TODO: We should get these from the parent as we only want a 247 * single copy of each across the interrupt controller. 248 */ 249 uint8_t *sc_conf_base; 250 vm_offset_t sc_pend_base[MAXCPU]; 251 252 /* Command handling */ 253 struct mtx sc_its_cmd_lock; 254 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */ 255 size_t sc_its_cmd_next_idx; 256 257 vmem_t *sc_irq_alloc; 258 struct gicv3_its_irqsrc **sc_irqs; 259 u_int sc_irq_base; 260 u_int sc_irq_length; 261 u_int sc_irq_count; 262 263 struct mtx sc_its_dev_lock; 264 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list; 265 TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs; 266 267 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001 268 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002 269 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004 270 u_int sc_its_flags; 271 bool trace_enable; 272 }; 273 274 static void *conf_base; 275 276 typedef void (its_quirk_func_t)(device_t); 277 static its_quirk_func_t its_quirk_cavium_22375; 278 279 static const struct { 280 const char *desc; 281 uint32_t iidr; 282 uint32_t iidr_mask; 283 its_quirk_func_t *func; 284 } its_quirks[] = { 285 { 286 /* Cavium ThunderX Pass 1.x */ 287 .desc = "Cavium ThunderX errata: 22375, 24313", 288 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM, 289 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0), 290 .iidr_mask = ~GITS_IIDR_REVISION_MASK, 291 .func = its_quirk_cavium_22375, 292 }, 293 }; 294 295 #define gic_its_read_4(sc, reg) \ 296 bus_read_4((sc)->sc_its_res, (reg)) 297 #define gic_its_read_8(sc, reg) \ 298 bus_read_8((sc)->sc_its_res, (reg)) 299 300 #define gic_its_write_4(sc, reg, val) \ 301 bus_write_4((sc)->sc_its_res, (reg), (val)) 302 #define gic_its_write_8(sc, reg, val) \ 303 bus_write_8((sc)->sc_its_res, (reg), (val)) 304 305 static device_attach_t gicv3_its_attach; 306 static device_detach_t gicv3_its_detach; 307 308 static pic_disable_intr_t gicv3_its_disable_intr; 309 static pic_enable_intr_t gicv3_its_enable_intr; 310 static pic_map_intr_t gicv3_its_map_intr; 311 static pic_setup_intr_t gicv3_its_setup_intr; 312 static pic_post_filter_t gicv3_its_post_filter; 313 static pic_post_ithread_t gicv3_its_post_ithread; 314 static pic_pre_ithread_t gicv3_its_pre_ithread; 315 static pic_bind_intr_t gicv3_its_bind_intr; 316 #ifdef SMP 317 static pic_init_secondary_t gicv3_its_init_secondary; 318 #endif 319 static msi_alloc_msi_t gicv3_its_alloc_msi; 320 static msi_release_msi_t gicv3_its_release_msi; 321 static msi_alloc_msix_t gicv3_its_alloc_msix; 322 static msi_release_msix_t gicv3_its_release_msix; 323 static msi_map_msi_t gicv3_its_map_msi; 324 325 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *); 326 static void its_cmd_mapc(device_t, struct its_col *, uint8_t); 327 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *); 328 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t); 329 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *); 330 static void its_cmd_invall(device_t, struct its_col *); 331 332 static device_method_t gicv3_its_methods[] = { 333 /* Device interface */ 334 DEVMETHOD(device_detach, gicv3_its_detach), 335 336 /* Interrupt controller interface */ 337 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr), 338 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr), 339 DEVMETHOD(pic_map_intr, gicv3_its_map_intr), 340 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr), 341 DEVMETHOD(pic_post_filter, gicv3_its_post_filter), 342 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread), 343 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread), 344 #ifdef SMP 345 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr), 346 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary), 347 #endif 348 349 /* MSI/MSI-X */ 350 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi), 351 DEVMETHOD(msi_release_msi, gicv3_its_release_msi), 352 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix), 353 DEVMETHOD(msi_release_msix, gicv3_its_release_msix), 354 DEVMETHOD(msi_map_msi, gicv3_its_map_msi), 355 356 /* End */ 357 DEVMETHOD_END 358 }; 359 360 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods, 361 sizeof(struct gicv3_its_softc)); 362 363 static void 364 gicv3_its_cmdq_init(struct gicv3_its_softc *sc) 365 { 366 vm_paddr_t cmd_paddr; 367 uint64_t reg, tmp; 368 369 /* Set up the command circular buffer */ 370 sc->sc_its_cmd_base = contigmalloc(ITS_CMDQ_SIZE, M_GICV3_ITS, 371 M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN, 0); 372 sc->sc_its_cmd_next_idx = 0; 373 374 cmd_paddr = vtophys(sc->sc_its_cmd_base); 375 376 /* Set the base of the command buffer */ 377 reg = GITS_CBASER_VALID | 378 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) | 379 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) | 380 (ITS_CMDQ_SIZE / 4096 - 1); 381 gic_its_write_8(sc, GITS_CBASER, reg); 382 383 /* Read back to check for fixed value fields */ 384 tmp = gic_its_read_8(sc, GITS_CBASER); 385 386 if ((tmp & GITS_CBASER_SHARE_MASK) != 387 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) { 388 /* Check if the hardware reported non-shareable */ 389 if ((tmp & GITS_CBASER_SHARE_MASK) == 390 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) { 391 /* If so remove the cache attribute */ 392 reg &= ~GITS_CBASER_CACHE_MASK; 393 reg &= ~GITS_CBASER_SHARE_MASK; 394 /* Set to Non-cacheable, Non-shareable */ 395 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT; 396 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT; 397 398 gic_its_write_8(sc, GITS_CBASER, reg); 399 } 400 401 /* The command queue has to be flushed after each command */ 402 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH; 403 } 404 405 /* Get the next command from the start of the buffer */ 406 gic_its_write_8(sc, GITS_CWRITER, 0x0); 407 } 408 409 static int 410 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc) 411 { 412 vm_offset_t table; 413 vm_paddr_t paddr; 414 uint64_t cache, reg, share, tmp, type; 415 size_t esize, its_tbl_size, nidents, nitspages, npages; 416 int i, page_size; 417 int devbits; 418 419 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) { 420 /* 421 * GITS_TYPER[17:13] of ThunderX reports that device IDs 422 * are to be 21 bits in length. The entry size of the ITS 423 * table can be read from GITS_BASERn[52:48] and on ThunderX 424 * is supposed to be 8 bytes in length (for device table). 425 * Finally the page size that is to be used by ITS to access 426 * this table will be set to 64KB. 427 * 428 * This gives 0x200000 entries of size 0x8 bytes covered by 429 * 256 pages each of which 64KB in size. The number of pages 430 * (minus 1) should then be written to GITS_BASERn[7:0]. In 431 * that case this value would be 0xFF but on ThunderX the 432 * maximum value that HW accepts is 0xFD. 433 * 434 * Set an arbitrary number of device ID bits to 20 in order 435 * to limit the number of entries in ITS device table to 436 * 0x100000 and the table size to 8MB. 437 */ 438 devbits = 20; 439 cache = 0; 440 } else { 441 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER)); 442 cache = GITS_BASER_CACHE_WAWB; 443 } 444 share = GITS_BASER_SHARE_IS; 445 page_size = PAGE_SIZE_64K; 446 447 for (i = 0; i < GITS_BASER_NUM; i++) { 448 reg = gic_its_read_8(sc, GITS_BASER(i)); 449 /* The type of table */ 450 type = GITS_BASER_TYPE(reg); 451 /* The table entry size */ 452 esize = GITS_BASER_ESIZE(reg); 453 454 switch(type) { 455 case GITS_BASER_TYPE_DEV: 456 nidents = (1 << devbits); 457 its_tbl_size = esize * nidents; 458 its_tbl_size = roundup2(its_tbl_size, PAGE_SIZE_64K); 459 break; 460 case GITS_BASER_TYPE_VP: 461 case GITS_BASER_TYPE_PP: /* Undocumented? */ 462 case GITS_BASER_TYPE_IC: 463 its_tbl_size = page_size; 464 break; 465 default: 466 continue; 467 } 468 npages = howmany(its_tbl_size, PAGE_SIZE); 469 470 /* Allocate the table */ 471 table = (vm_offset_t)contigmalloc(npages * PAGE_SIZE, 472 M_GICV3_ITS, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, 473 PAGE_SIZE_64K, 0); 474 475 sc->sc_its_ptab[i].ptab_vaddr = table; 476 sc->sc_its_ptab[i].ptab_size = npages * PAGE_SIZE; 477 478 paddr = vtophys(table); 479 480 while (1) { 481 nitspages = howmany(its_tbl_size, page_size); 482 483 /* Clear the fields we will be setting */ 484 reg &= ~(GITS_BASER_VALID | 485 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK | 486 GITS_BASER_ESIZE_MASK | GITS_BASER_PA_MASK | 487 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK | 488 GITS_BASER_SIZE_MASK); 489 /* Set the new values */ 490 reg |= GITS_BASER_VALID | 491 (cache << GITS_BASER_CACHE_SHIFT) | 492 (type << GITS_BASER_TYPE_SHIFT) | 493 ((esize - 1) << GITS_BASER_ESIZE_SHIFT) | 494 paddr | (share << GITS_BASER_SHARE_SHIFT) | 495 (nitspages - 1); 496 497 switch (page_size) { 498 case PAGE_SIZE: /* 4KB */ 499 reg |= 500 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT; 501 break; 502 case PAGE_SIZE_16K: /* 16KB */ 503 reg |= 504 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT; 505 break; 506 case PAGE_SIZE_64K: /* 64KB */ 507 reg |= 508 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT; 509 break; 510 } 511 512 gic_its_write_8(sc, GITS_BASER(i), reg); 513 514 /* Read back to check */ 515 tmp = gic_its_read_8(sc, GITS_BASER(i)); 516 517 /* Do the shareability masks line up? */ 518 if ((tmp & GITS_BASER_SHARE_MASK) != 519 (reg & GITS_BASER_SHARE_MASK)) { 520 share = (tmp & GITS_BASER_SHARE_MASK) >> 521 GITS_BASER_SHARE_SHIFT; 522 continue; 523 } 524 525 if ((tmp & GITS_BASER_PSZ_MASK) != 526 (reg & GITS_BASER_PSZ_MASK)) { 527 switch (page_size) { 528 case PAGE_SIZE_16K: 529 page_size = PAGE_SIZE; 530 continue; 531 case PAGE_SIZE_64K: 532 page_size = PAGE_SIZE_16K; 533 continue; 534 } 535 } 536 537 if (tmp != reg) { 538 device_printf(dev, "GITS_BASER%d: " 539 "unable to be updated: %lx != %lx\n", 540 i, reg, tmp); 541 return (ENXIO); 542 } 543 544 /* We should have made all needed changes */ 545 break; 546 } 547 } 548 549 return (0); 550 } 551 552 static void 553 gicv3_its_conftable_init(struct gicv3_its_softc *sc) 554 { 555 void *conf_table; 556 557 conf_table = atomic_load_ptr(&conf_base); 558 if (conf_table == NULL) { 559 conf_table = contigmalloc(LPI_CONFTAB_SIZE, 560 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR, 561 LPI_CONFTAB_ALIGN, 0); 562 563 if (atomic_cmpset_ptr((uintptr_t *)&conf_base, 564 (uintptr_t)NULL, (uintptr_t)conf_table) == 0) { 565 contigfree(conf_table, LPI_CONFTAB_SIZE, M_GICV3_ITS); 566 conf_table = atomic_load_ptr(&conf_base); 567 } 568 } 569 sc->sc_conf_base = conf_table; 570 571 /* Set the default configuration */ 572 memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1, 573 LPI_CONFTAB_SIZE); 574 575 /* Flush the table to memory */ 576 cpu_dcache_wb_range((vm_offset_t)sc->sc_conf_base, LPI_CONFTAB_SIZE); 577 } 578 579 static void 580 gicv3_its_pendtables_init(struct gicv3_its_softc *sc) 581 { 582 int i; 583 584 for (i = 0; i <= mp_maxid; i++) { 585 if (CPU_ISSET(i, &sc->sc_cpus) == 0) 586 continue; 587 588 sc->sc_pend_base[i] = (vm_offset_t)contigmalloc( 589 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO, 590 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0); 591 592 /* Flush so the ITS can see the memory */ 593 cpu_dcache_wb_range((vm_offset_t)sc->sc_pend_base[i], 594 LPI_PENDTAB_SIZE); 595 } 596 } 597 598 static void 599 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc) 600 { 601 device_t gicv3; 602 uint64_t xbaser, tmp; 603 uint32_t ctlr; 604 u_int cpuid; 605 606 gicv3 = device_get_parent(dev); 607 cpuid = PCPU_GET(cpuid); 608 609 /* Disable LPIs */ 610 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 611 ctlr &= ~GICR_CTLR_LPI_ENABLE; 612 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 613 614 /* Make sure changes are observable my the GIC */ 615 dsb(sy); 616 617 /* 618 * Set the redistributor base 619 */ 620 xbaser = vtophys(sc->sc_conf_base) | 621 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) | 622 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) | 623 (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1); 624 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 625 626 /* Check the cache attributes we set */ 627 tmp = gic_r_read_8(gicv3, GICR_PROPBASER); 628 629 if ((tmp & GICR_PROPBASER_SHARE_MASK) != 630 (xbaser & GICR_PROPBASER_SHARE_MASK)) { 631 if ((tmp & GICR_PROPBASER_SHARE_MASK) == 632 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) { 633 /* We need to mark as non-cacheable */ 634 xbaser &= ~(GICR_PROPBASER_SHARE_MASK | 635 GICR_PROPBASER_CACHE_MASK); 636 /* Non-cacheable */ 637 xbaser |= GICR_PROPBASER_CACHE_NIN << 638 GICR_PROPBASER_CACHE_SHIFT; 639 /* Non-sareable */ 640 xbaser |= GICR_PROPBASER_SHARE_NS << 641 GICR_PROPBASER_SHARE_SHIFT; 642 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser); 643 } 644 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH; 645 } 646 647 /* 648 * Set the LPI pending table base 649 */ 650 xbaser = vtophys(sc->sc_pend_base[cpuid]) | 651 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) | 652 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT); 653 654 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser); 655 656 tmp = gic_r_read_8(gicv3, GICR_PENDBASER); 657 658 if ((tmp & GICR_PENDBASER_SHARE_MASK) == 659 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) { 660 /* Clear the cahce and shareability bits */ 661 xbaser &= ~(GICR_PENDBASER_CACHE_MASK | 662 GICR_PENDBASER_SHARE_MASK); 663 /* Mark as non-shareable */ 664 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT; 665 /* And non-cacheable */ 666 xbaser |= GICR_PENDBASER_CACHE_NIN << 667 GICR_PENDBASER_CACHE_SHIFT; 668 } 669 670 /* Enable LPIs */ 671 ctlr = gic_r_read_4(gicv3, GICR_CTLR); 672 ctlr |= GICR_CTLR_LPI_ENABLE; 673 gic_r_write_4(gicv3, GICR_CTLR, ctlr); 674 675 /* Make sure the GIC has seen everything */ 676 dsb(sy); 677 } 678 679 static int 680 its_init_cpu(device_t dev, struct gicv3_its_softc *sc) 681 { 682 device_t gicv3; 683 vm_paddr_t target; 684 u_int cpuid; 685 struct redist_pcpu *rpcpu; 686 687 gicv3 = device_get_parent(dev); 688 cpuid = PCPU_GET(cpuid); 689 if (!CPU_ISSET(cpuid, &sc->sc_cpus)) 690 return (0); 691 692 /* Check if the ITS is enabled on this CPU */ 693 if ((gic_r_read_4(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0) 694 return (ENXIO); 695 696 rpcpu = gicv3_get_redist(dev); 697 698 /* Do per-cpu LPI init once */ 699 if (!rpcpu->lpi_enabled) { 700 its_init_cpu_lpi(dev, sc); 701 rpcpu->lpi_enabled = true; 702 } 703 704 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) { 705 /* This ITS wants the redistributor physical address */ 706 target = vtophys(rman_get_virtual(&rpcpu->res)); 707 } else { 708 /* This ITS wants the unique processor number */ 709 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) << 710 CMD_TARGET_SHIFT; 711 } 712 713 sc->sc_its_cols[cpuid]->col_target = target; 714 sc->sc_its_cols[cpuid]->col_id = cpuid; 715 716 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1); 717 its_cmd_invall(dev, sc->sc_its_cols[cpuid]); 718 719 return (0); 720 } 721 722 static int 723 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS) 724 { 725 struct gicv3_its_softc *sc; 726 int rv; 727 728 sc = arg1; 729 730 rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req); 731 if (rv != 0 || req->newptr == NULL) 732 return (rv); 733 if (sc->trace_enable) 734 gic_its_write_8(sc, GITS_TRKCTLR, 3); 735 else 736 gic_its_write_8(sc, GITS_TRKCTLR, 0); 737 738 return (0); 739 } 740 741 static int 742 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS) 743 { 744 struct gicv3_its_softc *sc; 745 struct sbuf *sb; 746 int err; 747 748 sc = arg1; 749 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req); 750 if (sb == NULL) { 751 device_printf(sc->dev, "Could not allocate sbuf for output.\n"); 752 return (ENOMEM); 753 } 754 sbuf_cat(sb, "\n"); 755 sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n", 756 gic_its_read_4(sc, GITS_TRKCTLR)); 757 sbuf_printf(sb, "GITS_TRKR: 0x%08X\n", 758 gic_its_read_4(sc, GITS_TRKR)); 759 sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n", 760 gic_its_read_4(sc, GITS_TRKDIDR)); 761 sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n", 762 gic_its_read_4(sc, GITS_TRKPIDR)); 763 sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n", 764 gic_its_read_4(sc, GITS_TRKVIDR)); 765 sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n", 766 gic_its_read_4(sc, GITS_TRKTGTR)); 767 768 err = sbuf_finish(sb); 769 if (err) 770 device_printf(sc->dev, "Error finishing sbuf: %d\n", err); 771 sbuf_delete(sb); 772 return(err); 773 } 774 775 static int 776 gicv3_its_init_sysctl(struct gicv3_its_softc *sc) 777 { 778 struct sysctl_oid *oid, *child; 779 struct sysctl_ctx_list *ctx_list; 780 781 ctx_list = device_get_sysctl_ctx(sc->dev); 782 child = device_get_sysctl_tree(sc->dev); 783 oid = SYSCTL_ADD_NODE(ctx_list, 784 SYSCTL_CHILDREN(child), OID_AUTO, "tracing", 785 CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing"); 786 if (oid == NULL) 787 return (ENXIO); 788 789 /* Add registers */ 790 SYSCTL_ADD_PROC(ctx_list, 791 SYSCTL_CHILDREN(oid), OID_AUTO, "enable", 792 CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 793 gicv3_its_sysctl_trace_enable, "CU", "Enable tracing"); 794 SYSCTL_ADD_PROC(ctx_list, 795 SYSCTL_CHILDREN(oid), OID_AUTO, "capture", 796 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0, 797 gicv3_its_sysctl_trace_regs, "", "Captured tracing registers."); 798 799 return (0); 800 } 801 802 static int 803 gicv3_its_attach(device_t dev) 804 { 805 struct gicv3_its_softc *sc; 806 uint32_t iidr; 807 int domain, err, i, rid; 808 809 sc = device_get_softc(dev); 810 811 sc->sc_irq_length = gicv3_get_nirqs(dev); 812 sc->sc_irq_base = GIC_FIRST_LPI; 813 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length; 814 815 rid = 0; 816 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid, 817 RF_ACTIVE); 818 if (sc->sc_its_res == NULL) { 819 device_printf(dev, "Could not allocate memory\n"); 820 return (ENXIO); 821 } 822 823 iidr = gic_its_read_4(sc, GITS_IIDR); 824 for (i = 0; i < nitems(its_quirks); i++) { 825 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) { 826 if (bootverbose) { 827 device_printf(dev, "Applying %s\n", 828 its_quirks[i].desc); 829 } 830 its_quirks[i].func(dev); 831 break; 832 } 833 } 834 835 /* Allocate the private tables */ 836 err = gicv3_its_table_init(dev, sc); 837 if (err != 0) 838 return (err); 839 840 /* Protects access to the device list */ 841 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN); 842 843 /* Protects access to the ITS command circular buffer. */ 844 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN); 845 846 CPU_ZERO(&sc->sc_cpus); 847 if (bus_get_domain(dev, &domain) == 0) { 848 if (domain < MAXMEMDOM) 849 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus); 850 } else { 851 CPU_COPY(&all_cpus, &sc->sc_cpus); 852 } 853 854 /* Allocate the command circular buffer */ 855 gicv3_its_cmdq_init(sc); 856 857 /* Allocate the per-CPU collections */ 858 for (int cpu = 0; cpu <= mp_maxid; cpu++) 859 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0) 860 sc->sc_its_cols[cpu] = malloc( 861 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS, 862 M_WAITOK | M_ZERO); 863 else 864 sc->sc_its_cols[cpu] = NULL; 865 866 /* Enable the ITS */ 867 gic_its_write_4(sc, GITS_CTLR, 868 gic_its_read_4(sc, GITS_CTLR) | GITS_CTLR_EN); 869 870 /* Create the LPI configuration table */ 871 gicv3_its_conftable_init(sc); 872 873 /* And the pending tebles */ 874 gicv3_its_pendtables_init(sc); 875 876 /* Enable LPIs on this CPU */ 877 its_init_cpu(dev, sc); 878 879 TAILQ_INIT(&sc->sc_its_dev_list); 880 TAILQ_INIT(&sc->sc_free_irqs); 881 882 /* 883 * Create the vmem object to allocate INTRNG IRQs from. We try to 884 * use all IRQs not already used by the GICv3. 885 * XXX: This assumes there are no other interrupt controllers in the 886 * system. 887 */ 888 sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0, 889 gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK); 890 891 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length, 892 M_GICV3_ITS, M_WAITOK | M_ZERO); 893 894 /* For GIC-500 install tracking sysctls. */ 895 if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) == 896 GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0)) 897 gicv3_its_init_sysctl(sc); 898 899 return (0); 900 } 901 902 static int 903 gicv3_its_detach(device_t dev) 904 { 905 906 return (ENXIO); 907 } 908 909 static void 910 its_quirk_cavium_22375(device_t dev) 911 { 912 struct gicv3_its_softc *sc; 913 914 sc = device_get_softc(dev); 915 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375; 916 } 917 918 static void 919 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc) 920 { 921 struct gicv3_its_softc *sc; 922 struct gicv3_its_irqsrc *girq; 923 uint8_t *conf; 924 925 sc = device_get_softc(dev); 926 girq = (struct gicv3_its_irqsrc *)isrc; 927 conf = sc->sc_conf_base; 928 929 conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE; 930 931 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 932 /* Clean D-cache under command. */ 933 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); 934 } else { 935 /* DSB inner shareable, store */ 936 dsb(ishst); 937 } 938 939 its_cmd_inv(dev, girq->gi_its_dev, girq); 940 } 941 942 static void 943 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc) 944 { 945 struct gicv3_its_softc *sc; 946 struct gicv3_its_irqsrc *girq; 947 uint8_t *conf; 948 949 sc = device_get_softc(dev); 950 girq = (struct gicv3_its_irqsrc *)isrc; 951 conf = sc->sc_conf_base; 952 953 conf[girq->gi_lpi] |= LPI_CONF_ENABLE; 954 955 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) { 956 /* Clean D-cache under command. */ 957 cpu_dcache_wb_range((vm_offset_t)&conf[girq->gi_lpi], 1); 958 } else { 959 /* DSB inner shareable, store */ 960 dsb(ishst); 961 } 962 963 its_cmd_inv(dev, girq->gi_its_dev, girq); 964 } 965 966 static int 967 gicv3_its_intr(void *arg, uintptr_t irq) 968 { 969 struct gicv3_its_softc *sc = arg; 970 struct gicv3_its_irqsrc *girq; 971 struct trapframe *tf; 972 973 irq -= sc->sc_irq_base; 974 girq = sc->sc_irqs[irq]; 975 if (girq == NULL) 976 panic("gicv3_its_intr: Invalid interrupt %ld", 977 irq + sc->sc_irq_base); 978 979 tf = curthread->td_intr_frame; 980 intr_isrc_dispatch(&girq->gi_isrc, tf); 981 return (FILTER_HANDLED); 982 } 983 984 static void 985 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc) 986 { 987 struct gicv3_its_irqsrc *girq; 988 struct gicv3_its_softc *sc; 989 990 sc = device_get_softc(dev); 991 girq = (struct gicv3_its_irqsrc *)isrc; 992 gicv3_its_disable_intr(dev, isrc); 993 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 994 } 995 996 static void 997 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc) 998 { 999 1000 gicv3_its_enable_intr(dev, isrc); 1001 } 1002 1003 static void 1004 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc) 1005 { 1006 struct gicv3_its_irqsrc *girq; 1007 struct gicv3_its_softc *sc; 1008 1009 sc = device_get_softc(dev); 1010 girq = (struct gicv3_its_irqsrc *)isrc; 1011 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI); 1012 } 1013 1014 static int 1015 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc) 1016 { 1017 struct gicv3_its_softc *sc; 1018 1019 sc = device_get_softc(dev); 1020 if (CPU_EMPTY(&isrc->isrc_cpu)) { 1021 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu, 1022 &sc->sc_cpus); 1023 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu); 1024 } 1025 1026 return (0); 1027 } 1028 1029 static int 1030 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc) 1031 { 1032 struct gicv3_its_irqsrc *girq; 1033 1034 gicv3_its_select_cpu(dev, isrc); 1035 1036 girq = (struct gicv3_its_irqsrc *)isrc; 1037 its_cmd_movi(dev, girq); 1038 return (0); 1039 } 1040 1041 static int 1042 gicv3_its_map_intr(device_t dev, struct intr_map_data *data, 1043 struct intr_irqsrc **isrcp) 1044 { 1045 1046 /* 1047 * This should never happen, we only call this function to map 1048 * interrupts found before the controller driver is ready. 1049 */ 1050 panic("gicv3_its_map_intr: Unable to map a MSI interrupt"); 1051 } 1052 1053 static int 1054 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc, 1055 struct resource *res, struct intr_map_data *data) 1056 { 1057 1058 /* Bind the interrupt to a CPU */ 1059 gicv3_its_bind_intr(dev, isrc); 1060 1061 return (0); 1062 } 1063 1064 #ifdef SMP 1065 static void 1066 gicv3_its_init_secondary(device_t dev) 1067 { 1068 struct gicv3_its_softc *sc; 1069 1070 sc = device_get_softc(dev); 1071 1072 /* 1073 * This is fatal as otherwise we may bind interrupts to this CPU. 1074 * We need a way to tell the interrupt framework to only bind to a 1075 * subset of given CPUs when it performs the shuffle. 1076 */ 1077 if (its_init_cpu(dev, sc) != 0) 1078 panic("gicv3_its_init_secondary: No usable ITS on CPU%d", 1079 PCPU_GET(cpuid)); 1080 } 1081 #endif 1082 1083 static uint32_t 1084 its_get_devid(device_t pci_dev) 1085 { 1086 uintptr_t id; 1087 1088 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0) 1089 panic("its_get_devid: Unable to get the MSI DeviceID"); 1090 1091 return (id); 1092 } 1093 1094 static struct its_dev * 1095 its_device_find(device_t dev, device_t child) 1096 { 1097 struct gicv3_its_softc *sc; 1098 struct its_dev *its_dev = NULL; 1099 1100 sc = device_get_softc(dev); 1101 1102 mtx_lock_spin(&sc->sc_its_dev_lock); 1103 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) { 1104 if (its_dev->pci_dev == child) 1105 break; 1106 } 1107 mtx_unlock_spin(&sc->sc_its_dev_lock); 1108 1109 return (its_dev); 1110 } 1111 1112 static struct its_dev * 1113 its_device_get(device_t dev, device_t child, u_int nvecs) 1114 { 1115 struct gicv3_its_softc *sc; 1116 struct its_dev *its_dev; 1117 vmem_addr_t irq_base; 1118 size_t esize; 1119 1120 sc = device_get_softc(dev); 1121 1122 its_dev = its_device_find(dev, child); 1123 if (its_dev != NULL) 1124 return (its_dev); 1125 1126 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO); 1127 if (its_dev == NULL) 1128 return (NULL); 1129 1130 its_dev->pci_dev = child; 1131 its_dev->devid = its_get_devid(child); 1132 1133 its_dev->lpis.lpi_busy = 0; 1134 its_dev->lpis.lpi_num = nvecs; 1135 its_dev->lpis.lpi_free = nvecs; 1136 1137 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT, 1138 &irq_base) != 0) { 1139 free(its_dev, M_GICV3_ITS); 1140 return (NULL); 1141 } 1142 its_dev->lpis.lpi_base = irq_base; 1143 1144 /* Get ITT entry size */ 1145 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER)); 1146 1147 /* 1148 * Allocate ITT for this device. 1149 * PA has to be 256 B aligned. At least two entries for device. 1150 */ 1151 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256); 1152 its_dev->itt = (vm_offset_t)contigmalloc(its_dev->itt_size, 1153 M_GICV3_ITS, M_NOWAIT | M_ZERO, 0, LPI_INT_TRANS_TAB_MAX_ADDR, 1154 LPI_INT_TRANS_TAB_ALIGN, 0); 1155 if (its_dev->itt == 0) { 1156 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs); 1157 free(its_dev, M_GICV3_ITS); 1158 return (NULL); 1159 } 1160 1161 mtx_lock_spin(&sc->sc_its_dev_lock); 1162 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry); 1163 mtx_unlock_spin(&sc->sc_its_dev_lock); 1164 1165 /* Map device to its ITT */ 1166 its_cmd_mapd(dev, its_dev, 1); 1167 1168 return (its_dev); 1169 } 1170 1171 static void 1172 its_device_release(device_t dev, struct its_dev *its_dev) 1173 { 1174 struct gicv3_its_softc *sc; 1175 1176 KASSERT(its_dev->lpis.lpi_busy == 0, 1177 ("its_device_release: Trying to release an inuse ITS device")); 1178 1179 /* Unmap device in ITS */ 1180 its_cmd_mapd(dev, its_dev, 0); 1181 1182 sc = device_get_softc(dev); 1183 1184 /* Remove the device from the list of devices */ 1185 mtx_lock_spin(&sc->sc_its_dev_lock); 1186 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry); 1187 mtx_unlock_spin(&sc->sc_its_dev_lock); 1188 1189 /* Free ITT */ 1190 KASSERT(its_dev->itt != 0, ("Invalid ITT in valid ITS device")); 1191 contigfree((void *)its_dev->itt, its_dev->itt_size, M_GICV3_ITS); 1192 1193 /* Free the IRQ allocation */ 1194 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, 1195 its_dev->lpis.lpi_num); 1196 1197 free(its_dev, M_GICV3_ITS); 1198 } 1199 1200 static struct gicv3_its_irqsrc * 1201 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq) 1202 { 1203 struct gicv3_its_irqsrc *girq = NULL; 1204 1205 KASSERT(sc->sc_irqs[irq] == NULL, 1206 ("%s: Interrupt %u already allocated", __func__, irq)); 1207 mtx_lock_spin(&sc->sc_its_dev_lock); 1208 if (!TAILQ_EMPTY(&sc->sc_free_irqs)) { 1209 girq = TAILQ_FIRST(&sc->sc_free_irqs); 1210 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link); 1211 } 1212 mtx_unlock_spin(&sc->sc_its_dev_lock); 1213 if (girq == NULL) { 1214 girq = malloc(sizeof(*girq), M_GICV3_ITS, 1215 M_NOWAIT | M_ZERO); 1216 if (girq == NULL) 1217 return (NULL); 1218 girq->gi_id = -1; 1219 if (intr_isrc_register(&girq->gi_isrc, dev, 0, 1220 "%s,%u", device_get_nameunit(dev), irq) != 0) { 1221 free(girq, M_GICV3_ITS); 1222 return (NULL); 1223 } 1224 } 1225 girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI; 1226 sc->sc_irqs[irq] = girq; 1227 1228 return (girq); 1229 } 1230 1231 static void 1232 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc, 1233 struct gicv3_its_irqsrc *girq) 1234 { 1235 u_int irq; 1236 1237 mtx_assert(&sc->sc_its_dev_lock, MA_OWNED); 1238 1239 irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base; 1240 sc->sc_irqs[irq] = NULL; 1241 1242 girq->gi_id = -1; 1243 girq->gi_its_dev = NULL; 1244 TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link); 1245 } 1246 1247 static int 1248 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount, 1249 device_t *pic, struct intr_irqsrc **srcs) 1250 { 1251 struct gicv3_its_softc *sc; 1252 struct gicv3_its_irqsrc *girq; 1253 struct its_dev *its_dev; 1254 u_int irq; 1255 int i; 1256 1257 its_dev = its_device_get(dev, child, count); 1258 if (its_dev == NULL) 1259 return (ENXIO); 1260 1261 KASSERT(its_dev->lpis.lpi_free >= count, 1262 ("gicv3_its_alloc_msi: No free LPIs")); 1263 sc = device_get_softc(dev); 1264 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1265 its_dev->lpis.lpi_free; 1266 1267 /* Allocate the irqsrc for each MSI */ 1268 for (i = 0; i < count; i++, irq++) { 1269 its_dev->lpis.lpi_free--; 1270 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev, 1271 sc, irq); 1272 if (srcs[i] == NULL) 1273 break; 1274 } 1275 1276 /* The allocation failed, release them */ 1277 if (i != count) { 1278 mtx_lock_spin(&sc->sc_its_dev_lock); 1279 for (i = 0; i < count; i++) { 1280 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1281 if (girq == NULL) 1282 break; 1283 gicv3_its_release_irqsrc(sc, girq); 1284 srcs[i] = NULL; 1285 } 1286 mtx_unlock_spin(&sc->sc_its_dev_lock); 1287 return (ENXIO); 1288 } 1289 1290 /* Finish the allocation now we have all MSI irqsrcs */ 1291 for (i = 0; i < count; i++) { 1292 girq = (struct gicv3_its_irqsrc *)srcs[i]; 1293 girq->gi_id = i; 1294 girq->gi_its_dev = its_dev; 1295 1296 /* Map the message to the given IRQ */ 1297 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1298 its_cmd_mapti(dev, girq); 1299 } 1300 its_dev->lpis.lpi_busy += count; 1301 *pic = dev; 1302 1303 return (0); 1304 } 1305 1306 static int 1307 gicv3_its_release_msi(device_t dev, device_t child, int count, 1308 struct intr_irqsrc **isrc) 1309 { 1310 struct gicv3_its_softc *sc; 1311 struct gicv3_its_irqsrc *girq; 1312 struct its_dev *its_dev; 1313 int i; 1314 1315 its_dev = its_device_find(dev, child); 1316 1317 KASSERT(its_dev != NULL, 1318 ("gicv3_its_release_msi: Releasing a MSI interrupt with " 1319 "no ITS device")); 1320 KASSERT(its_dev->lpis.lpi_busy >= count, 1321 ("gicv3_its_release_msi: Releasing more interrupts than " 1322 "were allocated: releasing %d, allocated %d", count, 1323 its_dev->lpis.lpi_busy)); 1324 1325 sc = device_get_softc(dev); 1326 mtx_lock_spin(&sc->sc_its_dev_lock); 1327 for (i = 0; i < count; i++) { 1328 girq = (struct gicv3_its_irqsrc *)isrc[i]; 1329 gicv3_its_release_irqsrc(sc, girq); 1330 } 1331 mtx_unlock_spin(&sc->sc_its_dev_lock); 1332 its_dev->lpis.lpi_busy -= count; 1333 1334 if (its_dev->lpis.lpi_busy == 0) 1335 its_device_release(dev, its_dev); 1336 1337 return (0); 1338 } 1339 1340 static int 1341 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic, 1342 struct intr_irqsrc **isrcp) 1343 { 1344 struct gicv3_its_softc *sc; 1345 struct gicv3_its_irqsrc *girq; 1346 struct its_dev *its_dev; 1347 u_int nvecs, irq; 1348 1349 nvecs = pci_msix_count(child); 1350 its_dev = its_device_get(dev, child, nvecs); 1351 if (its_dev == NULL) 1352 return (ENXIO); 1353 1354 KASSERT(its_dev->lpis.lpi_free > 0, 1355 ("gicv3_its_alloc_msix: No free LPIs")); 1356 sc = device_get_softc(dev); 1357 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num - 1358 its_dev->lpis.lpi_free; 1359 1360 girq = gicv3_its_alloc_irqsrc(dev, sc, irq); 1361 if (girq == NULL) 1362 return (ENXIO); 1363 girq->gi_id = its_dev->lpis.lpi_busy; 1364 girq->gi_its_dev = its_dev; 1365 1366 its_dev->lpis.lpi_free--; 1367 its_dev->lpis.lpi_busy++; 1368 1369 /* Map the message to the given IRQ */ 1370 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq); 1371 its_cmd_mapti(dev, girq); 1372 1373 *pic = dev; 1374 *isrcp = (struct intr_irqsrc *)girq; 1375 1376 return (0); 1377 } 1378 1379 static int 1380 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc) 1381 { 1382 struct gicv3_its_softc *sc; 1383 struct gicv3_its_irqsrc *girq; 1384 struct its_dev *its_dev; 1385 1386 its_dev = its_device_find(dev, child); 1387 1388 KASSERT(its_dev != NULL, 1389 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with " 1390 "no ITS device")); 1391 KASSERT(its_dev->lpis.lpi_busy > 0, 1392 ("gicv3_its_release_msix: Releasing more interrupts than " 1393 "were allocated: allocated %d", its_dev->lpis.lpi_busy)); 1394 1395 sc = device_get_softc(dev); 1396 girq = (struct gicv3_its_irqsrc *)isrc; 1397 gicv3_its_release_irqsrc(sc, girq); 1398 its_dev->lpis.lpi_busy--; 1399 1400 if (its_dev->lpis.lpi_busy == 0) 1401 its_device_release(dev, its_dev); 1402 1403 return (0); 1404 } 1405 1406 static int 1407 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc, 1408 uint64_t *addr, uint32_t *data) 1409 { 1410 struct gicv3_its_softc *sc; 1411 struct gicv3_its_irqsrc *girq; 1412 1413 sc = device_get_softc(dev); 1414 girq = (struct gicv3_its_irqsrc *)isrc; 1415 1416 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER; 1417 *data = girq->gi_id; 1418 1419 return (0); 1420 } 1421 1422 /* 1423 * Commands handling. 1424 */ 1425 1426 static __inline void 1427 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type) 1428 { 1429 /* Command field: DW0 [7:0] */ 1430 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK); 1431 cmd->cmd_dword[0] |= htole64(cmd_type); 1432 } 1433 1434 static __inline void 1435 cmd_format_devid(struct its_cmd *cmd, uint32_t devid) 1436 { 1437 /* Device ID field: DW0 [63:32] */ 1438 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK); 1439 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT); 1440 } 1441 1442 static __inline void 1443 cmd_format_size(struct its_cmd *cmd, uint16_t size) 1444 { 1445 /* Size field: DW1 [4:0] */ 1446 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK); 1447 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK)); 1448 } 1449 1450 static __inline void 1451 cmd_format_id(struct its_cmd *cmd, uint32_t id) 1452 { 1453 /* ID field: DW1 [31:0] */ 1454 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK); 1455 cmd->cmd_dword[1] |= htole64(id); 1456 } 1457 1458 static __inline void 1459 cmd_format_pid(struct its_cmd *cmd, uint32_t pid) 1460 { 1461 /* Physical ID field: DW1 [63:32] */ 1462 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK); 1463 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT); 1464 } 1465 1466 static __inline void 1467 cmd_format_col(struct its_cmd *cmd, uint16_t col_id) 1468 { 1469 /* Collection field: DW2 [16:0] */ 1470 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK); 1471 cmd->cmd_dword[2] |= htole64(col_id); 1472 } 1473 1474 static __inline void 1475 cmd_format_target(struct its_cmd *cmd, uint64_t target) 1476 { 1477 /* Target Address field: DW2 [47:16] */ 1478 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK); 1479 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK); 1480 } 1481 1482 static __inline void 1483 cmd_format_itt(struct its_cmd *cmd, uint64_t itt) 1484 { 1485 /* ITT Address field: DW2 [47:8] */ 1486 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK); 1487 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK); 1488 } 1489 1490 static __inline void 1491 cmd_format_valid(struct its_cmd *cmd, uint8_t valid) 1492 { 1493 /* Valid field: DW2 [63] */ 1494 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK); 1495 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT); 1496 } 1497 1498 static inline bool 1499 its_cmd_queue_full(struct gicv3_its_softc *sc) 1500 { 1501 size_t read_idx, next_write_idx; 1502 1503 /* Get the index of the next command */ 1504 next_write_idx = (sc->sc_its_cmd_next_idx + 1) % 1505 (ITS_CMDQ_SIZE / sizeof(struct its_cmd)); 1506 /* And the index of the current command being read */ 1507 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd); 1508 1509 /* 1510 * The queue is full when the write offset points 1511 * at the command before the current read offset. 1512 */ 1513 return (next_write_idx == read_idx); 1514 } 1515 1516 static inline void 1517 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1518 { 1519 1520 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) { 1521 /* Clean D-cache under command. */ 1522 cpu_dcache_wb_range((vm_offset_t)cmd, sizeof(*cmd)); 1523 } else { 1524 /* DSB inner shareable, store */ 1525 dsb(ishst); 1526 } 1527 1528 } 1529 1530 static inline uint64_t 1531 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd) 1532 { 1533 uint64_t off; 1534 1535 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd); 1536 1537 return (off); 1538 } 1539 1540 static void 1541 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first, 1542 struct its_cmd *cmd_last) 1543 { 1544 struct gicv3_its_softc *sc; 1545 uint64_t first, last, read; 1546 size_t us_left; 1547 1548 sc = device_get_softc(dev); 1549 1550 /* 1551 * XXX ARM64TODO: This is obviously a significant delay. 1552 * The reason for that is that currently the time frames for 1553 * the command to complete are not known. 1554 */ 1555 us_left = 1000000; 1556 1557 first = its_cmd_cwriter_offset(sc, cmd_first); 1558 last = its_cmd_cwriter_offset(sc, cmd_last); 1559 1560 for (;;) { 1561 read = gic_its_read_8(sc, GITS_CREADR); 1562 if (first < last) { 1563 if (read < first || read >= last) 1564 break; 1565 } else if (read < first && read >= last) 1566 break; 1567 1568 if (us_left-- == 0) { 1569 /* This means timeout */ 1570 device_printf(dev, 1571 "Timeout while waiting for CMD completion.\n"); 1572 return; 1573 } 1574 DELAY(1); 1575 } 1576 } 1577 1578 1579 static struct its_cmd * 1580 its_cmd_alloc_locked(device_t dev) 1581 { 1582 struct gicv3_its_softc *sc; 1583 struct its_cmd *cmd; 1584 size_t us_left; 1585 1586 sc = device_get_softc(dev); 1587 1588 /* 1589 * XXX ARM64TODO: This is obviously a significant delay. 1590 * The reason for that is that currently the time frames for 1591 * the command to complete (and therefore free the descriptor) 1592 * are not known. 1593 */ 1594 us_left = 1000000; 1595 1596 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED); 1597 while (its_cmd_queue_full(sc)) { 1598 if (us_left-- == 0) { 1599 /* Timeout while waiting for free command */ 1600 device_printf(dev, 1601 "Timeout while waiting for free command\n"); 1602 return (NULL); 1603 } 1604 DELAY(1); 1605 } 1606 1607 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1608 sc->sc_its_cmd_next_idx++; 1609 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd); 1610 1611 return (cmd); 1612 } 1613 1614 static uint64_t 1615 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc) 1616 { 1617 uint64_t target; 1618 uint8_t cmd_type; 1619 u_int size; 1620 1621 cmd_type = desc->cmd_type; 1622 target = ITS_TARGET_NONE; 1623 1624 switch (cmd_type) { 1625 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */ 1626 target = desc->cmd_desc_movi.col->col_target; 1627 cmd_format_command(cmd, ITS_CMD_MOVI); 1628 cmd_format_id(cmd, desc->cmd_desc_movi.id); 1629 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id); 1630 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid); 1631 break; 1632 case ITS_CMD_SYNC: /* Wait for previous commands completion */ 1633 target = desc->cmd_desc_sync.col->col_target; 1634 cmd_format_command(cmd, ITS_CMD_SYNC); 1635 cmd_format_target(cmd, target); 1636 break; 1637 case ITS_CMD_MAPD: /* Assign ITT to device */ 1638 cmd_format_command(cmd, ITS_CMD_MAPD); 1639 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt)); 1640 /* 1641 * Size describes number of bits to encode interrupt IDs 1642 * supported by the device minus one. 1643 * When V (valid) bit is zero, this field should be written 1644 * as zero. 1645 */ 1646 if (desc->cmd_desc_mapd.valid != 0) { 1647 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num); 1648 size = MAX(1, size) - 1; 1649 } else 1650 size = 0; 1651 1652 cmd_format_size(cmd, size); 1653 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid); 1654 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid); 1655 break; 1656 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */ 1657 target = desc->cmd_desc_mapc.col->col_target; 1658 cmd_format_command(cmd, ITS_CMD_MAPC); 1659 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id); 1660 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid); 1661 cmd_format_target(cmd, target); 1662 break; 1663 case ITS_CMD_MAPTI: 1664 target = desc->cmd_desc_mapvi.col->col_target; 1665 cmd_format_command(cmd, ITS_CMD_MAPTI); 1666 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid); 1667 cmd_format_id(cmd, desc->cmd_desc_mapvi.id); 1668 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid); 1669 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id); 1670 break; 1671 case ITS_CMD_MAPI: 1672 target = desc->cmd_desc_mapi.col->col_target; 1673 cmd_format_command(cmd, ITS_CMD_MAPI); 1674 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid); 1675 cmd_format_id(cmd, desc->cmd_desc_mapi.pid); 1676 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id); 1677 break; 1678 case ITS_CMD_INV: 1679 target = desc->cmd_desc_inv.col->col_target; 1680 cmd_format_command(cmd, ITS_CMD_INV); 1681 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid); 1682 cmd_format_id(cmd, desc->cmd_desc_inv.pid); 1683 break; 1684 case ITS_CMD_INVALL: 1685 cmd_format_command(cmd, ITS_CMD_INVALL); 1686 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id); 1687 break; 1688 default: 1689 panic("its_cmd_prepare: Invalid command: %x", cmd_type); 1690 } 1691 1692 return (target); 1693 } 1694 1695 static int 1696 its_cmd_send(device_t dev, struct its_cmd_desc *desc) 1697 { 1698 struct gicv3_its_softc *sc; 1699 struct its_cmd *cmd, *cmd_sync, *cmd_write; 1700 struct its_col col_sync; 1701 struct its_cmd_desc desc_sync; 1702 uint64_t target, cwriter; 1703 1704 sc = device_get_softc(dev); 1705 mtx_lock_spin(&sc->sc_its_cmd_lock); 1706 cmd = its_cmd_alloc_locked(dev); 1707 if (cmd == NULL) { 1708 device_printf(dev, "could not allocate ITS command\n"); 1709 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1710 return (EBUSY); 1711 } 1712 1713 target = its_cmd_prepare(cmd, desc); 1714 its_cmd_sync(sc, cmd); 1715 1716 if (target != ITS_TARGET_NONE) { 1717 cmd_sync = its_cmd_alloc_locked(dev); 1718 if (cmd_sync != NULL) { 1719 desc_sync.cmd_type = ITS_CMD_SYNC; 1720 col_sync.col_target = target; 1721 desc_sync.cmd_desc_sync.col = &col_sync; 1722 its_cmd_prepare(cmd_sync, &desc_sync); 1723 its_cmd_sync(sc, cmd_sync); 1724 } 1725 } 1726 1727 /* Update GITS_CWRITER */ 1728 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd); 1729 gic_its_write_8(sc, GITS_CWRITER, cwriter); 1730 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx]; 1731 mtx_unlock_spin(&sc->sc_its_cmd_lock); 1732 1733 its_cmd_wait_completion(dev, cmd, cmd_write); 1734 1735 return (0); 1736 } 1737 1738 /* Handlers to send commands */ 1739 static void 1740 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq) 1741 { 1742 struct gicv3_its_softc *sc; 1743 struct its_cmd_desc desc; 1744 struct its_col *col; 1745 1746 sc = device_get_softc(dev); 1747 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1748 1749 desc.cmd_type = ITS_CMD_MOVI; 1750 desc.cmd_desc_movi.its_dev = girq->gi_its_dev; 1751 desc.cmd_desc_movi.col = col; 1752 desc.cmd_desc_movi.id = girq->gi_id; 1753 1754 its_cmd_send(dev, &desc); 1755 } 1756 1757 static void 1758 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid) 1759 { 1760 struct its_cmd_desc desc; 1761 1762 desc.cmd_type = ITS_CMD_MAPC; 1763 desc.cmd_desc_mapc.col = col; 1764 /* 1765 * Valid bit set - map the collection. 1766 * Valid bit cleared - unmap the collection. 1767 */ 1768 desc.cmd_desc_mapc.valid = valid; 1769 1770 its_cmd_send(dev, &desc); 1771 } 1772 1773 static void 1774 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq) 1775 { 1776 struct gicv3_its_softc *sc; 1777 struct its_cmd_desc desc; 1778 struct its_col *col; 1779 u_int col_id; 1780 1781 sc = device_get_softc(dev); 1782 1783 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1; 1784 col = sc->sc_its_cols[col_id]; 1785 1786 desc.cmd_type = ITS_CMD_MAPTI; 1787 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev; 1788 desc.cmd_desc_mapvi.col = col; 1789 /* The EventID sent to the device */ 1790 desc.cmd_desc_mapvi.id = girq->gi_id; 1791 /* The physical interrupt presented to softeware */ 1792 desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI; 1793 1794 its_cmd_send(dev, &desc); 1795 } 1796 1797 static void 1798 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid) 1799 { 1800 struct its_cmd_desc desc; 1801 1802 desc.cmd_type = ITS_CMD_MAPD; 1803 desc.cmd_desc_mapd.its_dev = its_dev; 1804 desc.cmd_desc_mapd.valid = valid; 1805 1806 its_cmd_send(dev, &desc); 1807 } 1808 1809 static void 1810 its_cmd_inv(device_t dev, struct its_dev *its_dev, 1811 struct gicv3_its_irqsrc *girq) 1812 { 1813 struct gicv3_its_softc *sc; 1814 struct its_cmd_desc desc; 1815 struct its_col *col; 1816 1817 sc = device_get_softc(dev); 1818 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1]; 1819 1820 desc.cmd_type = ITS_CMD_INV; 1821 /* The EventID sent to the device */ 1822 desc.cmd_desc_inv.pid = girq->gi_id; 1823 desc.cmd_desc_inv.its_dev = its_dev; 1824 desc.cmd_desc_inv.col = col; 1825 1826 its_cmd_send(dev, &desc); 1827 } 1828 1829 static void 1830 its_cmd_invall(device_t dev, struct its_col *col) 1831 { 1832 struct its_cmd_desc desc; 1833 1834 desc.cmd_type = ITS_CMD_INVALL; 1835 desc.cmd_desc_invall.col = col; 1836 1837 its_cmd_send(dev, &desc); 1838 } 1839 1840 #ifdef FDT 1841 static device_probe_t gicv3_its_fdt_probe; 1842 static device_attach_t gicv3_its_fdt_attach; 1843 1844 static device_method_t gicv3_its_fdt_methods[] = { 1845 /* Device interface */ 1846 DEVMETHOD(device_probe, gicv3_its_fdt_probe), 1847 DEVMETHOD(device_attach, gicv3_its_fdt_attach), 1848 1849 /* End */ 1850 DEVMETHOD_END 1851 }; 1852 1853 #define its_baseclasses its_fdt_baseclasses 1854 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods, 1855 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1856 #undef its_baseclasses 1857 static devclass_t gicv3_its_fdt_devclass; 1858 1859 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 1860 gicv3_its_fdt_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1861 1862 static int 1863 gicv3_its_fdt_probe(device_t dev) 1864 { 1865 1866 if (!ofw_bus_status_okay(dev)) 1867 return (ENXIO); 1868 1869 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its")) 1870 return (ENXIO); 1871 1872 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1873 return (BUS_PROBE_DEFAULT); 1874 } 1875 1876 static int 1877 gicv3_its_fdt_attach(device_t dev) 1878 { 1879 struct gicv3_its_softc *sc; 1880 phandle_t xref; 1881 int err; 1882 1883 sc = device_get_softc(dev); 1884 sc->dev = dev; 1885 err = gicv3_its_attach(dev); 1886 if (err != 0) 1887 return (err); 1888 1889 /* Register this device as a interrupt controller */ 1890 xref = OF_xref_from_node(ofw_bus_get_node(dev)); 1891 sc->sc_pic = intr_pic_register(dev, xref); 1892 intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1893 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 1894 1895 /* Register this device to handle MSI interrupts */ 1896 intr_msi_register(dev, xref); 1897 1898 return (0); 1899 } 1900 #endif 1901 1902 #ifdef DEV_ACPI 1903 static device_probe_t gicv3_its_acpi_probe; 1904 static device_attach_t gicv3_its_acpi_attach; 1905 1906 static device_method_t gicv3_its_acpi_methods[] = { 1907 /* Device interface */ 1908 DEVMETHOD(device_probe, gicv3_its_acpi_probe), 1909 DEVMETHOD(device_attach, gicv3_its_acpi_attach), 1910 1911 /* End */ 1912 DEVMETHOD_END 1913 }; 1914 1915 #define its_baseclasses its_acpi_baseclasses 1916 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods, 1917 sizeof(struct gicv3_its_softc), gicv3_its_driver); 1918 #undef its_baseclasses 1919 static devclass_t gicv3_its_acpi_devclass; 1920 1921 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 1922 gicv3_its_acpi_devclass, 0, 0, BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE); 1923 1924 static int 1925 gicv3_its_acpi_probe(device_t dev) 1926 { 1927 1928 if (gic_get_bus(dev) != GIC_BUS_ACPI) 1929 return (EINVAL); 1930 1931 if (gic_get_hw_rev(dev) < 3) 1932 return (EINVAL); 1933 1934 device_set_desc(dev, "ARM GIC Interrupt Translation Service"); 1935 return (BUS_PROBE_DEFAULT); 1936 } 1937 1938 static int 1939 gicv3_its_acpi_attach(device_t dev) 1940 { 1941 struct gicv3_its_softc *sc; 1942 struct gic_v3_devinfo *di; 1943 int err; 1944 1945 sc = device_get_softc(dev); 1946 sc->dev = dev; 1947 err = gicv3_its_attach(dev); 1948 if (err != 0) 1949 return (err); 1950 1951 di = device_get_ivars(dev); 1952 sc->sc_pic = intr_pic_register(dev, di->msi_xref); 1953 intr_pic_add_handler(device_get_parent(dev), sc->sc_pic, 1954 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length); 1955 1956 /* Register this device to handle MSI interrupts */ 1957 intr_msi_register(dev, di->msi_xref); 1958 1959 return (0); 1960 } 1961 #endif 1962