1 /*-
2 * Copyright (c) 2015-2016 The FreeBSD Foundation
3 * Copyright (c) 2023 Arm Ltd
4 *
5 * This software was developed by Andrew Turner under
6 * the sponsorship of the FreeBSD Foundation.
7 *
8 * This software was developed by Semihalf under
9 * the sponsorship of the FreeBSD Foundation.
10 *
11 * Redistribution and use in source and binary forms, with or without
12 * modification, are permitted provided that the following conditions
13 * are met:
14 * 1. Redistributions of source code must retain the above copyright
15 * notice, this list of conditions and the following disclaimer.
16 * 2. Redistributions in binary form must reproduce the above copyright
17 * notice, this list of conditions and the following disclaimer in the
18 * documentation and/or other materials provided with the distribution.
19 *
20 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
21 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
22 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
23 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
24 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
25 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
26 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
27 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
28 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
29 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
30 * SUCH DAMAGE.
31 */
32
33 #include "opt_acpi.h"
34 #include "opt_platform.h"
35 #include "opt_iommu.h"
36
37 #include <sys/param.h>
38 #include <sys/systm.h>
39 #include <sys/bus.h>
40 #include <sys/cpuset.h>
41 #include <sys/domainset.h>
42 #include <sys/endian.h>
43 #include <sys/kernel.h>
44 #include <sys/lock.h>
45 #include <sys/malloc.h>
46 #include <sys/module.h>
47 #include <sys/mutex.h>
48 #include <sys/physmem.h>
49 #include <sys/proc.h>
50 #include <sys/taskqueue.h>
51 #include <sys/tree.h>
52 #include <sys/queue.h>
53 #include <sys/rman.h>
54 #include <sys/sbuf.h>
55 #include <sys/smp.h>
56 #include <sys/sysctl.h>
57 #include <sys/vmem.h>
58
59 #include <vm/vm.h>
60 #include <vm/pmap.h>
61 #include <vm/vm_page.h>
62
63 #include <machine/bus.h>
64 #include <machine/intr.h>
65
66 #include <arm/arm/gic_common.h>
67 #include <arm64/arm64/gic_v3_reg.h>
68 #include <arm64/arm64/gic_v3_var.h>
69
70 #ifdef FDT
71 #include <dev/ofw/openfirm.h>
72 #include <dev/ofw/ofw_bus.h>
73 #include <dev/ofw/ofw_bus_subr.h>
74 #endif
75 #include <dev/pci/pcireg.h>
76 #include <dev/pci/pcivar.h>
77
78 #ifdef IOMMU
79 #include <dev/iommu/iommu.h>
80 #include <dev/iommu/iommu_gas.h>
81 #endif
82
83 #include "pcib_if.h"
84 #include "pic_if.h"
85 #include "msi_if.h"
86
87 MALLOC_DEFINE(M_GICV3_ITS, "GICv3 ITS",
88 "ARM GICv3 Interrupt Translation Service");
89
90 #define LPI_NIRQS (64 * 1024)
91
92 /* The size and alignment of the command circular buffer */
93 #define ITS_CMDQ_SIZE (64 * 1024) /* Must be a multiple of 4K */
94 #define ITS_CMDQ_ALIGN (64 * 1024)
95
96 #define LPI_CONFTAB_SIZE LPI_NIRQS
97 #define LPI_CONFTAB_ALIGN (64 * 1024)
98 #define LPI_CONFTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */
99
100 /* 1 bit per SPI, PPI, and SGI (8k), and 1 bit per LPI (LPI_CONFTAB_SIZE) */
101 #define LPI_PENDTAB_SIZE ((LPI_NIRQS + GIC_FIRST_LPI) / 8)
102 #define LPI_PENDTAB_ALIGN (64 * 1024)
103 #define LPI_PENDTAB_MAX_ADDR ((1ul << 48) - 1) /* We need a 47 bit PA */
104
105 #define LPI_INT_TRANS_TAB_ALIGN 256
106 #define LPI_INT_TRANS_TAB_MAX_ADDR ((1ul << 48) - 1)
107
108 /* ITS commands encoding */
109 #define ITS_CMD_MOVI (0x01)
110 #define ITS_CMD_SYNC (0x05)
111 #define ITS_CMD_MAPD (0x08)
112 #define ITS_CMD_MAPC (0x09)
113 #define ITS_CMD_MAPTI (0x0a)
114 #define ITS_CMD_MAPI (0x0b)
115 #define ITS_CMD_INV (0x0c)
116 #define ITS_CMD_INVALL (0x0d)
117 /* Command */
118 #define CMD_COMMAND_MASK (0xFFUL)
119 /* PCI device ID */
120 #define CMD_DEVID_SHIFT (32)
121 #define CMD_DEVID_MASK (0xFFFFFFFFUL << CMD_DEVID_SHIFT)
122 /* Size of IRQ ID bitfield */
123 #define CMD_SIZE_MASK (0xFFUL)
124 /* Virtual LPI ID */
125 #define CMD_ID_MASK (0xFFFFFFFFUL)
126 /* Physical LPI ID */
127 #define CMD_PID_SHIFT (32)
128 #define CMD_PID_MASK (0xFFFFFFFFUL << CMD_PID_SHIFT)
129 /* Collection */
130 #define CMD_COL_MASK (0xFFFFUL)
131 /* Target (CPU or Re-Distributor) */
132 #define CMD_TARGET_SHIFT (16)
133 #define CMD_TARGET_MASK (0xFFFFFFFFUL << CMD_TARGET_SHIFT)
134 /* Interrupt Translation Table address */
135 #define CMD_ITT_MASK (0xFFFFFFFFFF00UL)
136 /* Valid command bit */
137 #define CMD_VALID_SHIFT (63)
138 #define CMD_VALID_MASK (1UL << CMD_VALID_SHIFT)
139
140 #define ITS_TARGET_NONE 0xFBADBEEF
141
142 /* LPI chunk owned by ITS device */
143 struct lpi_chunk {
144 u_int lpi_base;
145 u_int lpi_free; /* First free LPI in set */
146 u_int lpi_num; /* Total number of LPIs in chunk */
147 u_int lpi_busy; /* Number of busy LPIs in chink */
148 };
149
150 /* ITS device */
151 struct its_dev {
152 TAILQ_ENTRY(its_dev) entry;
153 /* PCI device */
154 device_t pci_dev;
155 /* Device ID (i.e. PCI device ID) */
156 uint32_t devid;
157 /* List of assigned LPIs */
158 struct lpi_chunk lpis;
159 /* Virtual address of ITT */
160 void *itt;
161 size_t itt_size;
162 };
163
164 /*
165 * ITS command descriptor.
166 * Idea for command description passing taken from Linux.
167 */
168 struct its_cmd_desc {
169 uint8_t cmd_type;
170
171 union {
172 struct {
173 struct its_dev *its_dev;
174 struct its_col *col;
175 uint32_t id;
176 } cmd_desc_movi;
177
178 struct {
179 struct its_col *col;
180 } cmd_desc_sync;
181
182 struct {
183 struct its_col *col;
184 uint8_t valid;
185 } cmd_desc_mapc;
186
187 struct {
188 struct its_dev *its_dev;
189 struct its_col *col;
190 uint32_t pid;
191 uint32_t id;
192 } cmd_desc_mapvi;
193
194 struct {
195 struct its_dev *its_dev;
196 struct its_col *col;
197 uint32_t pid;
198 } cmd_desc_mapi;
199
200 struct {
201 struct its_dev *its_dev;
202 uint8_t valid;
203 } cmd_desc_mapd;
204
205 struct {
206 struct its_dev *its_dev;
207 struct its_col *col;
208 uint32_t pid;
209 } cmd_desc_inv;
210
211 struct {
212 struct its_col *col;
213 } cmd_desc_invall;
214 };
215 };
216
217 /* ITS command. Each command is 32 bytes long */
218 struct its_cmd {
219 uint64_t cmd_dword[4]; /* ITS command double word */
220 };
221
222 /* An ITS private table */
223 struct its_ptable {
224 void *ptab_vaddr;
225 /* Size of the L1 and L2 tables */
226 size_t ptab_l1_size;
227 size_t ptab_l2_size;
228 /* Number of L1 and L2 entries */
229 int ptab_l1_nidents;
230 int ptab_l2_nidents;
231
232 int ptab_page_size;
233 int ptab_share;
234 bool ptab_indirect;
235 };
236
237 /* ITS collection description. */
238 struct its_col {
239 uint64_t col_target; /* Target Re-Distributor */
240 uint64_t col_id; /* Collection ID */
241 };
242
243 struct gicv3_its_irqsrc {
244 struct intr_irqsrc gi_isrc;
245 u_int gi_id;
246 u_int gi_lpi;
247 struct its_dev *gi_its_dev;
248 TAILQ_ENTRY(gicv3_its_irqsrc) gi_link;
249 };
250
251 struct gicv3_its_softc {
252 device_t dev;
253 struct intr_pic *sc_pic;
254 struct resource *sc_its_res;
255
256 cpuset_t sc_cpus;
257 struct domainset *sc_ds;
258 u_int gic_irq_cpu;
259 int sc_devbits;
260 int sc_dev_table_idx;
261
262 struct its_ptable sc_its_ptab[GITS_BASER_NUM];
263 struct its_col *sc_its_cols[MAXCPU]; /* Per-CPU collections */
264
265 /*
266 * TODO: We should get these from the parent as we only want a
267 * single copy of each across the interrupt controller.
268 */
269 uint8_t *sc_conf_base;
270 void *sc_pend_base[MAXCPU];
271
272 /* Command handling */
273 struct mtx sc_its_cmd_lock;
274 struct its_cmd *sc_its_cmd_base; /* Command circular buffer address */
275 size_t sc_its_cmd_next_idx;
276
277 vmem_t *sc_irq_alloc;
278 struct gicv3_its_irqsrc **sc_irqs;
279 u_int sc_irq_base;
280 u_int sc_irq_length;
281 u_int sc_irq_count;
282
283 struct mtx sc_its_dev_lock;
284 TAILQ_HEAD(its_dev_list, its_dev) sc_its_dev_list;
285 TAILQ_HEAD(free_irqs, gicv3_its_irqsrc) sc_free_irqs;
286
287 #define ITS_FLAGS_CMDQ_FLUSH 0x00000001
288 #define ITS_FLAGS_LPI_CONF_FLUSH 0x00000002
289 #define ITS_FLAGS_ERRATA_CAVIUM_22375 0x00000004
290 #define ITS_FLAGS_LPI_PREALLOC 0x00000008
291 u_int sc_its_flags;
292 bool trace_enable;
293 vm_page_t ma; /* fake msi page */
294 };
295
296 typedef void (its_quirk_func_t)(device_t);
297 static its_quirk_func_t its_quirk_cavium_22375;
298
299 static const struct {
300 const char *desc;
301 uint32_t iidr;
302 uint32_t iidr_mask;
303 its_quirk_func_t *func;
304 } its_quirks[] = {
305 {
306 /* Cavium ThunderX Pass 1.x */
307 .desc = "Cavium ThunderX errata: 22375, 24313",
308 .iidr = GITS_IIDR_RAW(GITS_IIDR_IMPL_CAVIUM,
309 GITS_IIDR_PROD_THUNDER, GITS_IIDR_VAR_THUNDER_1, 0),
310 .iidr_mask = ~GITS_IIDR_REVISION_MASK,
311 .func = its_quirk_cavium_22375,
312 },
313 };
314
315 #define gic_its_read_4(sc, reg) \
316 bus_read_4((sc)->sc_its_res, (reg))
317 #define gic_its_read_8(sc, reg) \
318 bus_read_8((sc)->sc_its_res, (reg))
319
320 #define gic_its_write_4(sc, reg, val) \
321 bus_write_4((sc)->sc_its_res, (reg), (val))
322 #define gic_its_write_8(sc, reg, val) \
323 bus_write_8((sc)->sc_its_res, (reg), (val))
324
325 static device_attach_t gicv3_its_attach;
326 static device_detach_t gicv3_its_detach;
327
328 static pic_disable_intr_t gicv3_its_disable_intr;
329 static pic_enable_intr_t gicv3_its_enable_intr;
330 static pic_map_intr_t gicv3_its_map_intr;
331 static pic_setup_intr_t gicv3_its_setup_intr;
332 static pic_post_filter_t gicv3_its_post_filter;
333 static pic_post_ithread_t gicv3_its_post_ithread;
334 static pic_pre_ithread_t gicv3_its_pre_ithread;
335 static pic_bind_intr_t gicv3_its_bind_intr;
336 #ifdef SMP
337 static pic_init_secondary_t gicv3_its_init_secondary;
338 #endif
339 static msi_alloc_msi_t gicv3_its_alloc_msi;
340 static msi_release_msi_t gicv3_its_release_msi;
341 static msi_alloc_msix_t gicv3_its_alloc_msix;
342 static msi_release_msix_t gicv3_its_release_msix;
343 static msi_map_msi_t gicv3_its_map_msi;
344 #ifdef IOMMU
345 static msi_iommu_init_t gicv3_iommu_init;
346 static msi_iommu_deinit_t gicv3_iommu_deinit;
347 #endif
348
349 static void its_cmd_movi(device_t, struct gicv3_its_irqsrc *);
350 static void its_cmd_mapc(device_t, struct its_col *, uint8_t);
351 static void its_cmd_mapti(device_t, struct gicv3_its_irqsrc *);
352 static void its_cmd_mapd(device_t, struct its_dev *, uint8_t);
353 static void its_cmd_inv(device_t, struct its_dev *, struct gicv3_its_irqsrc *);
354 static void its_cmd_invall(device_t, struct its_col *);
355
356 static device_method_t gicv3_its_methods[] = {
357 /* Device interface */
358 DEVMETHOD(device_detach, gicv3_its_detach),
359
360 /* Interrupt controller interface */
361 DEVMETHOD(pic_disable_intr, gicv3_its_disable_intr),
362 DEVMETHOD(pic_enable_intr, gicv3_its_enable_intr),
363 DEVMETHOD(pic_map_intr, gicv3_its_map_intr),
364 DEVMETHOD(pic_setup_intr, gicv3_its_setup_intr),
365 DEVMETHOD(pic_post_filter, gicv3_its_post_filter),
366 DEVMETHOD(pic_post_ithread, gicv3_its_post_ithread),
367 DEVMETHOD(pic_pre_ithread, gicv3_its_pre_ithread),
368 #ifdef SMP
369 DEVMETHOD(pic_bind_intr, gicv3_its_bind_intr),
370 DEVMETHOD(pic_init_secondary, gicv3_its_init_secondary),
371 #endif
372
373 /* MSI/MSI-X */
374 DEVMETHOD(msi_alloc_msi, gicv3_its_alloc_msi),
375 DEVMETHOD(msi_release_msi, gicv3_its_release_msi),
376 DEVMETHOD(msi_alloc_msix, gicv3_its_alloc_msix),
377 DEVMETHOD(msi_release_msix, gicv3_its_release_msix),
378 DEVMETHOD(msi_map_msi, gicv3_its_map_msi),
379 #ifdef IOMMU
380 DEVMETHOD(msi_iommu_init, gicv3_iommu_init),
381 DEVMETHOD(msi_iommu_deinit, gicv3_iommu_deinit),
382 #endif
383
384 /* End */
385 DEVMETHOD_END
386 };
387
388 static DEFINE_CLASS_0(gic, gicv3_its_driver, gicv3_its_methods,
389 sizeof(struct gicv3_its_softc));
390
391 static void
gicv3_its_cmdq_init(struct gicv3_its_softc * sc)392 gicv3_its_cmdq_init(struct gicv3_its_softc *sc)
393 {
394 vm_paddr_t cmd_paddr;
395 uint64_t reg, tmp;
396
397 /* Set up the command circular buffer */
398 sc->sc_its_cmd_base = contigmalloc_domainset(ITS_CMDQ_SIZE, M_GICV3_ITS,
399 sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1, ITS_CMDQ_ALIGN,
400 0);
401 sc->sc_its_cmd_next_idx = 0;
402
403 cmd_paddr = vtophys(sc->sc_its_cmd_base);
404
405 /* Set the base of the command buffer */
406 reg = GITS_CBASER_VALID |
407 (GITS_CBASER_CACHE_NIWAWB << GITS_CBASER_CACHE_SHIFT) |
408 cmd_paddr | (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT) |
409 (ITS_CMDQ_SIZE / 4096 - 1);
410 gic_its_write_8(sc, GITS_CBASER, reg);
411
412 /* Read back to check for fixed value fields */
413 tmp = gic_its_read_8(sc, GITS_CBASER);
414
415 if ((tmp & GITS_CBASER_SHARE_MASK) !=
416 (GITS_CBASER_SHARE_IS << GITS_CBASER_SHARE_SHIFT)) {
417 /* Check if the hardware reported non-shareable */
418 if ((tmp & GITS_CBASER_SHARE_MASK) ==
419 (GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT)) {
420 /* If so remove the cache attribute */
421 reg &= ~GITS_CBASER_CACHE_MASK;
422 reg &= ~GITS_CBASER_SHARE_MASK;
423 /* Set to Non-cacheable, Non-shareable */
424 reg |= GITS_CBASER_CACHE_NIN << GITS_CBASER_CACHE_SHIFT;
425 reg |= GITS_CBASER_SHARE_NS << GITS_CBASER_SHARE_SHIFT;
426
427 gic_its_write_8(sc, GITS_CBASER, reg);
428 }
429
430 /* The command queue has to be flushed after each command */
431 sc->sc_its_flags |= ITS_FLAGS_CMDQ_FLUSH;
432 }
433
434 /* Get the next command from the start of the buffer */
435 gic_its_write_8(sc, GITS_CWRITER, 0x0);
436 }
437
438 static int
gicv3_its_table_page_size(struct gicv3_its_softc * sc,int table)439 gicv3_its_table_page_size(struct gicv3_its_softc *sc, int table)
440 {
441 uint64_t reg, tmp;
442 int page_size;
443
444 page_size = PAGE_SIZE_64K;
445 reg = gic_its_read_8(sc, GITS_BASER(table));
446
447 while (1) {
448 reg &= GITS_BASER_PSZ_MASK;
449 switch (page_size) {
450 case PAGE_SIZE_4K: /* 4KB */
451 reg |= GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
452 break;
453 case PAGE_SIZE_16K: /* 16KB */
454 reg |= GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
455 break;
456 case PAGE_SIZE_64K: /* 64KB */
457 reg |= GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
458 break;
459 }
460
461 /* Write the new page size */
462 gic_its_write_8(sc, GITS_BASER(table), reg);
463
464 /* Read back to check */
465 tmp = gic_its_read_8(sc, GITS_BASER(table));
466
467 /* The page size is correct */
468 if ((tmp & GITS_BASER_PSZ_MASK) == (reg & GITS_BASER_PSZ_MASK))
469 return (page_size);
470
471 switch (page_size) {
472 default:
473 return (-1);
474 case PAGE_SIZE_16K:
475 page_size = PAGE_SIZE_4K;
476 break;
477 case PAGE_SIZE_64K:
478 page_size = PAGE_SIZE_16K;
479 break;
480 }
481 }
482 }
483
484 static bool
gicv3_its_table_supports_indirect(struct gicv3_its_softc * sc,int table)485 gicv3_its_table_supports_indirect(struct gicv3_its_softc *sc, int table)
486 {
487 uint64_t reg;
488
489 reg = gic_its_read_8(sc, GITS_BASER(table));
490
491 /* Try setting the indirect flag */
492 reg |= GITS_BASER_INDIRECT;
493 gic_its_write_8(sc, GITS_BASER(table), reg);
494
495 /* Read back to check */
496 reg = gic_its_read_8(sc, GITS_BASER(table));
497 return ((reg & GITS_BASER_INDIRECT) != 0);
498 }
499
500
501 static int
gicv3_its_table_init(device_t dev,struct gicv3_its_softc * sc)502 gicv3_its_table_init(device_t dev, struct gicv3_its_softc *sc)
503 {
504 void *table;
505 vm_paddr_t paddr;
506 uint64_t cache, reg, share, tmp, type;
507 size_t its_tbl_size, nitspages, npages;
508 size_t l1_esize, l2_esize, l1_nidents, l2_nidents;
509 int i, page_size;
510 int devbits;
511 bool indirect;
512
513 if ((sc->sc_its_flags & ITS_FLAGS_ERRATA_CAVIUM_22375) != 0) {
514 /*
515 * GITS_TYPER[17:13] of ThunderX reports that device IDs
516 * are to be 21 bits in length. The entry size of the ITS
517 * table can be read from GITS_BASERn[52:48] and on ThunderX
518 * is supposed to be 8 bytes in length (for device table).
519 * Finally the page size that is to be used by ITS to access
520 * this table will be set to 64KB.
521 *
522 * This gives 0x200000 entries of size 0x8 bytes covered by
523 * 256 pages each of which 64KB in size. The number of pages
524 * (minus 1) should then be written to GITS_BASERn[7:0]. In
525 * that case this value would be 0xFF but on ThunderX the
526 * maximum value that HW accepts is 0xFD.
527 *
528 * Set an arbitrary number of device ID bits to 20 in order
529 * to limit the number of entries in ITS device table to
530 * 0x100000 and the table size to 8MB.
531 */
532 devbits = 20;
533 cache = 0;
534 } else {
535 devbits = GITS_TYPER_DEVB(gic_its_read_8(sc, GITS_TYPER));
536 cache = GITS_BASER_CACHE_WAWB;
537 }
538 sc->sc_devbits = devbits;
539 share = GITS_BASER_SHARE_IS;
540
541 for (i = 0; i < GITS_BASER_NUM; i++) {
542 reg = gic_its_read_8(sc, GITS_BASER(i));
543 /* The type of table */
544 type = GITS_BASER_TYPE(reg);
545 if (type == GITS_BASER_TYPE_UNIMPL)
546 continue;
547
548 /* The table entry size */
549 l1_esize = GITS_BASER_ESIZE(reg);
550
551 /* Find the tables page size */
552 page_size = gicv3_its_table_page_size(sc, i);
553 if (page_size == -1) {
554 device_printf(dev, "No valid page size for table %d\n",
555 i);
556 return (EINVAL);
557 }
558
559 indirect = false;
560 l2_nidents = 0;
561 l2_esize = 0;
562 switch(type) {
563 case GITS_BASER_TYPE_DEV:
564 if (sc->sc_dev_table_idx != -1)
565 device_printf(dev,
566 "Warning: Multiple device tables found\n");
567
568 sc->sc_dev_table_idx = i;
569 l1_nidents = (1 << devbits);
570 if ((l1_esize * l1_nidents) > (page_size * 2)) {
571 indirect =
572 gicv3_its_table_supports_indirect(sc, i);
573 if (indirect) {
574 /*
575 * Each l1 entry is 8 bytes and points
576 * to an l2 table of size page_size.
577 * Calculate how many entries this is
578 * and use this to find how many
579 * 8 byte l1 idents we need.
580 */
581 l2_esize = l1_esize;
582 l2_nidents = page_size / l2_esize;
583 l1_nidents = l1_nidents / l2_nidents;
584 l1_esize = GITS_INDIRECT_L1_ESIZE;
585 }
586 }
587 its_tbl_size = l1_esize * l1_nidents;
588 its_tbl_size = roundup2(its_tbl_size, page_size);
589 break;
590 case GITS_BASER_TYPE_VP:
591 case GITS_BASER_TYPE_PP: /* Undocumented? */
592 case GITS_BASER_TYPE_IC:
593 its_tbl_size = page_size;
594 break;
595 default:
596 if (bootverbose)
597 device_printf(dev, "Unhandled table type %lx\n",
598 type);
599 continue;
600 }
601 npages = howmany(its_tbl_size, PAGE_SIZE);
602
603 /* Allocate the table */
604 table = contigmalloc_domainset(npages * PAGE_SIZE,
605 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0,
606 (1ul << 48) - 1, PAGE_SIZE_64K, 0);
607
608 sc->sc_its_ptab[i].ptab_vaddr = table;
609 sc->sc_its_ptab[i].ptab_l1_size = its_tbl_size;
610 sc->sc_its_ptab[i].ptab_l1_nidents = l1_nidents;
611 sc->sc_its_ptab[i].ptab_l2_size = page_size;
612 sc->sc_its_ptab[i].ptab_l2_nidents = l2_nidents;
613
614 sc->sc_its_ptab[i].ptab_indirect = indirect;
615 sc->sc_its_ptab[i].ptab_page_size = page_size;
616
617 paddr = vtophys(table);
618
619 while (1) {
620 nitspages = howmany(its_tbl_size, page_size);
621
622 /* Clear the fields we will be setting */
623 reg &= ~(GITS_BASER_VALID | GITS_BASER_INDIRECT |
624 GITS_BASER_CACHE_MASK | GITS_BASER_TYPE_MASK |
625 GITS_BASER_PA_MASK |
626 GITS_BASER_SHARE_MASK | GITS_BASER_PSZ_MASK |
627 GITS_BASER_SIZE_MASK);
628 /* Set the new values */
629 reg |= GITS_BASER_VALID |
630 (indirect ? GITS_BASER_INDIRECT : 0) |
631 (cache << GITS_BASER_CACHE_SHIFT) |
632 (type << GITS_BASER_TYPE_SHIFT) |
633 paddr | (share << GITS_BASER_SHARE_SHIFT) |
634 (nitspages - 1);
635
636 switch (page_size) {
637 case PAGE_SIZE_4K: /* 4KB */
638 reg |=
639 GITS_BASER_PSZ_4K << GITS_BASER_PSZ_SHIFT;
640 break;
641 case PAGE_SIZE_16K: /* 16KB */
642 reg |=
643 GITS_BASER_PSZ_16K << GITS_BASER_PSZ_SHIFT;
644 break;
645 case PAGE_SIZE_64K: /* 64KB */
646 reg |=
647 GITS_BASER_PSZ_64K << GITS_BASER_PSZ_SHIFT;
648 break;
649 }
650
651 gic_its_write_8(sc, GITS_BASER(i), reg);
652
653 /* Read back to check */
654 tmp = gic_its_read_8(sc, GITS_BASER(i));
655
656 /* Do the shareability masks line up? */
657 if ((tmp & GITS_BASER_SHARE_MASK) !=
658 (reg & GITS_BASER_SHARE_MASK)) {
659 share = (tmp & GITS_BASER_SHARE_MASK) >>
660 GITS_BASER_SHARE_SHIFT;
661 continue;
662 }
663
664 if (tmp != reg) {
665 device_printf(dev, "GITS_BASER%d: "
666 "unable to be updated: %lx != %lx\n",
667 i, reg, tmp);
668 return (ENXIO);
669 }
670
671 sc->sc_its_ptab[i].ptab_share = share;
672 /* We should have made all needed changes */
673 break;
674 }
675 }
676
677 return (0);
678 }
679
680 static void
gicv3_its_conftable_init(struct gicv3_its_softc * sc)681 gicv3_its_conftable_init(struct gicv3_its_softc *sc)
682 {
683 /* note: we assume the ITS children are serialized by the parent */
684 static void *conf_table;
685 int extra_flags = 0;
686 device_t gicv3;
687 uint32_t ctlr;
688 vm_paddr_t conf_pa;
689 vm_offset_t conf_va;
690
691 /*
692 * The PROPBASER is a singleton in our parent. We only set it up the
693 * first time through. conf_table is effectively global to all the units
694 * and we rely on subr_bus to serialize probe/attach.
695 */
696 if (conf_table != NULL) {
697 sc->sc_conf_base = conf_table;
698 return;
699 }
700
701 gicv3 = device_get_parent(sc->dev);
702 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
703 if ((ctlr & GICR_CTLR_LPI_ENABLE) != 0) {
704 conf_pa = gic_r_read_8(gicv3, GICR_PROPBASER);
705 conf_pa &= GICR_PROPBASER_PA_MASK;
706 /*
707 * If there was a pre-existing PROPBASER, then we need to honor
708 * it because implementation defined behavior in gicv3 makes it
709 * impossible to quiesce to change it out. We will only see a
710 * pre-existing one when we've been kexec'd from a Linux kernel,
711 * or from a LinuxBoot environment.
712 *
713 * Linux provides us with a MEMRESERVE table that we put into
714 * the excluded physmem area. If PROPBASER isn't in this tabke,
715 * the system cannot run due to random memory corruption,
716 * so we panic for this case.
717 */
718 if (!physmem_excluded(conf_pa, LPI_CONFTAB_SIZE))
719 panic("gicv3 PROPBASER needs to reuse %#lx, but not reserved",
720 conf_pa);
721 conf_va = PHYS_TO_DMAP(conf_pa);
722 if (!pmap_klookup(conf_va, NULL))
723 panic("Cannot map prior LPI mapping into KVA");
724 conf_table = (void *)conf_va;
725 extra_flags = ITS_FLAGS_LPI_PREALLOC | ITS_FLAGS_LPI_CONF_FLUSH;
726 if (bootverbose)
727 device_printf(sc->dev,
728 "LPI enabled, conf table using pa %#lx va %lx\n",
729 conf_pa, conf_va);
730 } else {
731 /*
732 * Otherwise just allocate contiguous pages. We'll configure the
733 * PROPBASER register later in its_init_cpu_lpi().
734 */
735 conf_table = contigmalloc(LPI_CONFTAB_SIZE,
736 M_GICV3_ITS, M_WAITOK, 0, LPI_CONFTAB_MAX_ADDR,
737 LPI_CONFTAB_ALIGN, 0);
738 }
739 sc->sc_conf_base = conf_table;
740 sc->sc_its_flags |= extra_flags;
741
742 /* Set the default configuration */
743 memset(sc->sc_conf_base, GIC_PRIORITY_MAX | LPI_CONF_GROUP1,
744 LPI_CONFTAB_SIZE);
745
746 /* Flush the table to memory */
747 cpu_dcache_wb_range(sc->sc_conf_base, LPI_CONFTAB_SIZE);
748 }
749
750 static void
gicv3_its_pendtables_init(struct gicv3_its_softc * sc)751 gicv3_its_pendtables_init(struct gicv3_its_softc *sc)
752 {
753
754 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
755 for (int i = 0; i <= mp_maxid; i++) {
756 if (CPU_ISSET(i, &sc->sc_cpus) == 0)
757 continue;
758
759 sc->sc_pend_base[i] = contigmalloc(
760 LPI_PENDTAB_SIZE, M_GICV3_ITS, M_WAITOK | M_ZERO,
761 0, LPI_PENDTAB_MAX_ADDR, LPI_PENDTAB_ALIGN, 0);
762
763 /* Flush so the ITS can see the memory */
764 cpu_dcache_wb_range(sc->sc_pend_base[i],
765 LPI_PENDTAB_SIZE);
766 }
767 }
768 }
769
770 static void
its_init_cpu_lpi(device_t dev,struct gicv3_its_softc * sc)771 its_init_cpu_lpi(device_t dev, struct gicv3_its_softc *sc)
772 {
773 device_t gicv3;
774 uint64_t xbaser, tmp, size;
775 uint32_t ctlr;
776 u_int cpuid;
777
778 gicv3 = device_get_parent(dev);
779 cpuid = PCPU_GET(cpuid);
780
781 /*
782 * Set the redistributor base. If we're reusing what we found on boot
783 * since the gic was already running, then don't touch it here. We also
784 * don't need to disable / enable LPI if we're not changing PROPBASER,
785 * so only do that if we're not prealloced.
786 */
787 if ((sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) == 0) {
788 /* Disable LPIs */
789 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
790 ctlr &= ~GICR_CTLR_LPI_ENABLE;
791 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
792
793 /* Make sure changes are observable my the GIC */
794 dsb(sy);
795
796 size = (flsl(LPI_CONFTAB_SIZE | GIC_FIRST_LPI) - 1);
797
798 xbaser = vtophys(sc->sc_conf_base) |
799 (GICR_PROPBASER_SHARE_IS << GICR_PROPBASER_SHARE_SHIFT) |
800 (GICR_PROPBASER_CACHE_NIWAWB << GICR_PROPBASER_CACHE_SHIFT) |
801 size;
802
803 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
804
805 /* Check the cache attributes we set */
806 tmp = gic_r_read_8(gicv3, GICR_PROPBASER);
807
808 if ((tmp & GICR_PROPBASER_SHARE_MASK) !=
809 (xbaser & GICR_PROPBASER_SHARE_MASK)) {
810 if ((tmp & GICR_PROPBASER_SHARE_MASK) ==
811 (GICR_PROPBASER_SHARE_NS << GICR_PROPBASER_SHARE_SHIFT)) {
812 /* We need to mark as non-cacheable */
813 xbaser &= ~(GICR_PROPBASER_SHARE_MASK |
814 GICR_PROPBASER_CACHE_MASK);
815 /* Non-cacheable */
816 xbaser |= GICR_PROPBASER_CACHE_NIN <<
817 GICR_PROPBASER_CACHE_SHIFT;
818 /* Non-shareable */
819 xbaser |= GICR_PROPBASER_SHARE_NS <<
820 GICR_PROPBASER_SHARE_SHIFT;
821 gic_r_write_8(gicv3, GICR_PROPBASER, xbaser);
822 }
823 sc->sc_its_flags |= ITS_FLAGS_LPI_CONF_FLUSH;
824 }
825
826 /*
827 * Set the LPI pending table base
828 */
829 xbaser = vtophys(sc->sc_pend_base[cpuid]) |
830 (GICR_PENDBASER_CACHE_NIWAWB << GICR_PENDBASER_CACHE_SHIFT) |
831 (GICR_PENDBASER_SHARE_IS << GICR_PENDBASER_SHARE_SHIFT);
832
833 gic_r_write_8(gicv3, GICR_PENDBASER, xbaser);
834
835 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
836
837 if ((tmp & GICR_PENDBASER_SHARE_MASK) ==
838 (GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT)) {
839 /* Clear the cahce and shareability bits */
840 xbaser &= ~(GICR_PENDBASER_CACHE_MASK |
841 GICR_PENDBASER_SHARE_MASK);
842 /* Mark as non-shareable */
843 xbaser |= GICR_PENDBASER_SHARE_NS << GICR_PENDBASER_SHARE_SHIFT;
844 /* And non-cacheable */
845 xbaser |= GICR_PENDBASER_CACHE_NIN <<
846 GICR_PENDBASER_CACHE_SHIFT;
847 }
848
849 /* Enable LPIs */
850 ctlr = gic_r_read_4(gicv3, GICR_CTLR);
851 ctlr |= GICR_CTLR_LPI_ENABLE;
852 gic_r_write_4(gicv3, GICR_CTLR, ctlr);
853
854 /* Make sure the GIC has seen everything */
855 dsb(sy);
856 } else {
857 KASSERT(sc->sc_pend_base[cpuid] == NULL,
858 ("PREALLOC too soon cpuid %d", cpuid));
859 tmp = gic_r_read_8(gicv3, GICR_PENDBASER);
860 tmp &= GICR_PENDBASER_PA_MASK;
861 if (!physmem_excluded(tmp, LPI_PENDTAB_SIZE))
862 panic("gicv3 PENDBASER on cpu %d needs to reuse 0x%#lx, but not reserved\n",
863 cpuid, tmp);
864 sc->sc_pend_base[cpuid] = (void *)PHYS_TO_DMAP(tmp);
865 }
866
867
868 if (bootverbose)
869 device_printf(gicv3, "using %sPENDBASE of %#lx on cpu %d\n",
870 (sc->sc_its_flags & ITS_FLAGS_LPI_PREALLOC) ? "pre-existing " : "",
871 vtophys(sc->sc_pend_base[cpuid]), cpuid);
872 }
873
874 static int
its_init_cpu(device_t dev,struct gicv3_its_softc * sc)875 its_init_cpu(device_t dev, struct gicv3_its_softc *sc)
876 {
877 device_t gicv3;
878 vm_paddr_t target;
879 u_int cpuid;
880 struct redist_pcpu *rpcpu;
881
882 gicv3 = device_get_parent(dev);
883 cpuid = PCPU_GET(cpuid);
884 if (!CPU_ISSET(cpuid, &sc->sc_cpus))
885 return (0);
886
887 /* Check if the ITS is enabled on this CPU */
888 if ((gic_r_read_8(gicv3, GICR_TYPER) & GICR_TYPER_PLPIS) == 0)
889 return (ENXIO);
890
891 rpcpu = gicv3_get_redist(dev);
892
893 /* Do per-cpu LPI init once */
894 if (!rpcpu->lpi_enabled) {
895 its_init_cpu_lpi(dev, sc);
896 rpcpu->lpi_enabled = true;
897 }
898
899 if ((gic_its_read_8(sc, GITS_TYPER) & GITS_TYPER_PTA) != 0) {
900 /* This ITS wants the redistributor physical address */
901 target = vtophys((vm_offset_t)rman_get_virtual(rpcpu->res) +
902 rpcpu->offset);
903 } else {
904 /* This ITS wants the unique processor number */
905 target = GICR_TYPER_CPUNUM(gic_r_read_8(gicv3, GICR_TYPER)) <<
906 CMD_TARGET_SHIFT;
907 }
908
909 sc->sc_its_cols[cpuid]->col_target = target;
910 sc->sc_its_cols[cpuid]->col_id = cpuid;
911
912 its_cmd_mapc(dev, sc->sc_its_cols[cpuid], 1);
913 its_cmd_invall(dev, sc->sc_its_cols[cpuid]);
914
915 return (0);
916 }
917
918 static int
gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)919 gicv3_its_sysctl_trace_enable(SYSCTL_HANDLER_ARGS)
920 {
921 struct gicv3_its_softc *sc;
922 int rv;
923
924 sc = arg1;
925
926 rv = sysctl_handle_bool(oidp, &sc->trace_enable, 0, req);
927 if (rv != 0 || req->newptr == NULL)
928 return (rv);
929 if (sc->trace_enable)
930 gic_its_write_8(sc, GITS_TRKCTLR, 3);
931 else
932 gic_its_write_8(sc, GITS_TRKCTLR, 0);
933
934 return (0);
935 }
936
937 static int
gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)938 gicv3_its_sysctl_trace_regs(SYSCTL_HANDLER_ARGS)
939 {
940 struct gicv3_its_softc *sc;
941 struct sbuf *sb;
942 int err;
943
944 sc = arg1;
945 sb = sbuf_new_for_sysctl(NULL, NULL, 128, req);
946 if (sb == NULL) {
947 device_printf(sc->dev, "Could not allocate sbuf for output.\n");
948 return (ENOMEM);
949 }
950 sbuf_cat(sb, "\n");
951 sbuf_printf(sb, "GITS_TRKCTLR: 0x%08X\n",
952 gic_its_read_4(sc, GITS_TRKCTLR));
953 sbuf_printf(sb, "GITS_TRKR: 0x%08X\n",
954 gic_its_read_4(sc, GITS_TRKR));
955 sbuf_printf(sb, "GITS_TRKDIDR: 0x%08X\n",
956 gic_its_read_4(sc, GITS_TRKDIDR));
957 sbuf_printf(sb, "GITS_TRKPIDR: 0x%08X\n",
958 gic_its_read_4(sc, GITS_TRKPIDR));
959 sbuf_printf(sb, "GITS_TRKVIDR: 0x%08X\n",
960 gic_its_read_4(sc, GITS_TRKVIDR));
961 sbuf_printf(sb, "GITS_TRKTGTR: 0x%08X\n",
962 gic_its_read_4(sc, GITS_TRKTGTR));
963
964 err = sbuf_finish(sb);
965 if (err)
966 device_printf(sc->dev, "Error finishing sbuf: %d\n", err);
967 sbuf_delete(sb);
968 return(err);
969 }
970
971 static int
gicv3_its_init_sysctl(struct gicv3_its_softc * sc)972 gicv3_its_init_sysctl(struct gicv3_its_softc *sc)
973 {
974 struct sysctl_oid *oid, *child;
975 struct sysctl_ctx_list *ctx_list;
976
977 ctx_list = device_get_sysctl_ctx(sc->dev);
978 child = device_get_sysctl_tree(sc->dev);
979 oid = SYSCTL_ADD_NODE(ctx_list,
980 SYSCTL_CHILDREN(child), OID_AUTO, "tracing",
981 CTLFLAG_RD| CTLFLAG_MPSAFE, NULL, "Messages tracing");
982 if (oid == NULL)
983 return (ENXIO);
984
985 /* Add registers */
986 SYSCTL_ADD_PROC(ctx_list,
987 SYSCTL_CHILDREN(oid), OID_AUTO, "enable",
988 CTLTYPE_U8 | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
989 gicv3_its_sysctl_trace_enable, "CU", "Enable tracing");
990 SYSCTL_ADD_PROC(ctx_list,
991 SYSCTL_CHILDREN(oid), OID_AUTO, "capture",
992 CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
993 gicv3_its_sysctl_trace_regs, "", "Captured tracing registers.");
994
995 return (0);
996 }
997
998 static int
gicv3_its_attach(device_t dev)999 gicv3_its_attach(device_t dev)
1000 {
1001 struct gicv3_its_softc *sc;
1002 int domain, err, i, rid;
1003 uint64_t phys;
1004 uint32_t ctlr, iidr;
1005
1006 sc = device_get_softc(dev);
1007
1008 sc->sc_dev_table_idx = -1;
1009 sc->sc_irq_length = gicv3_get_nirqs(dev);
1010 sc->sc_irq_base = GIC_FIRST_LPI;
1011 sc->sc_irq_base += device_get_unit(dev) * sc->sc_irq_length;
1012
1013 rid = 0;
1014 sc->sc_its_res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &rid,
1015 RF_ACTIVE);
1016 if (sc->sc_its_res == NULL) {
1017 device_printf(dev, "Could not allocate memory\n");
1018 return (ENXIO);
1019 }
1020
1021 phys = rounddown2(vtophys(rman_get_virtual(sc->sc_its_res)) +
1022 GITS_TRANSLATER, PAGE_SIZE);
1023 sc->ma = malloc(sizeof(struct vm_page), M_DEVBUF, M_WAITOK | M_ZERO);
1024 vm_page_initfake(sc->ma, phys, VM_MEMATTR_DEFAULT);
1025
1026 CPU_COPY(&all_cpus, &sc->sc_cpus);
1027 iidr = gic_its_read_4(sc, GITS_IIDR);
1028 for (i = 0; i < nitems(its_quirks); i++) {
1029 if ((iidr & its_quirks[i].iidr_mask) == its_quirks[i].iidr) {
1030 if (bootverbose) {
1031 device_printf(dev, "Applying %s\n",
1032 its_quirks[i].desc);
1033 }
1034 its_quirks[i].func(dev);
1035 break;
1036 }
1037 }
1038
1039 if (bus_get_domain(dev, &domain) == 0 && domain < MAXMEMDOM) {
1040 sc->sc_ds = DOMAINSET_PREF(domain);
1041 } else {
1042 sc->sc_ds = DOMAINSET_RR();
1043 }
1044
1045 /*
1046 * GIT_CTLR_EN is mandated to reset to 0 on a Warm reset, but we may be
1047 * coming in via, for instance, a kexec/kboot style setup where a
1048 * previous kernel has configured then relinquished control. Clear it
1049 * so that we can reconfigure GITS_BASER*.
1050 */
1051 ctlr = gic_its_read_4(sc, GITS_CTLR);
1052 if ((ctlr & GITS_CTLR_EN) != 0) {
1053 ctlr &= ~GITS_CTLR_EN;
1054 gic_its_write_4(sc, GITS_CTLR, ctlr);
1055 }
1056
1057 /* Allocate the private tables */
1058 err = gicv3_its_table_init(dev, sc);
1059 if (err != 0)
1060 return (err);
1061
1062 /* Protects access to the device list */
1063 mtx_init(&sc->sc_its_dev_lock, "ITS device lock", NULL, MTX_SPIN);
1064
1065 /* Protects access to the ITS command circular buffer. */
1066 mtx_init(&sc->sc_its_cmd_lock, "ITS cmd lock", NULL, MTX_SPIN);
1067
1068 /* Allocate the command circular buffer */
1069 gicv3_its_cmdq_init(sc);
1070
1071 /* Allocate the per-CPU collections */
1072 for (int cpu = 0; cpu <= mp_maxid; cpu++)
1073 if (CPU_ISSET(cpu, &sc->sc_cpus) != 0)
1074 sc->sc_its_cols[cpu] = malloc_domainset(
1075 sizeof(*sc->sc_its_cols[0]), M_GICV3_ITS,
1076 DOMAINSET_PREF(pcpu_find(cpu)->pc_domain),
1077 M_WAITOK | M_ZERO);
1078 else
1079 sc->sc_its_cols[cpu] = NULL;
1080
1081 /* Enable the ITS */
1082 gic_its_write_4(sc, GITS_CTLR, ctlr | GITS_CTLR_EN);
1083
1084 /* Create the LPI configuration table */
1085 gicv3_its_conftable_init(sc);
1086
1087 /* And the pending tebles */
1088 gicv3_its_pendtables_init(sc);
1089
1090 /* Enable LPIs on this CPU */
1091 its_init_cpu(dev, sc);
1092
1093 TAILQ_INIT(&sc->sc_its_dev_list);
1094 TAILQ_INIT(&sc->sc_free_irqs);
1095
1096 /*
1097 * Create the vmem object to allocate INTRNG IRQs from. We try to
1098 * use all IRQs not already used by the GICv3.
1099 * XXX: This assumes there are no other interrupt controllers in the
1100 * system.
1101 */
1102 sc->sc_irq_alloc = vmem_create(device_get_nameunit(dev), 0,
1103 gicv3_get_nirqs(dev), 1, 0, M_FIRSTFIT | M_WAITOK);
1104
1105 sc->sc_irqs = malloc(sizeof(*sc->sc_irqs) * sc->sc_irq_length,
1106 M_GICV3_ITS, M_WAITOK | M_ZERO);
1107
1108 /* For GIC-500 install tracking sysctls. */
1109 if ((iidr & (GITS_IIDR_PRODUCT_MASK | GITS_IIDR_IMPLEMENTOR_MASK)) ==
1110 GITS_IIDR_RAW(GITS_IIDR_IMPL_ARM, GITS_IIDR_PROD_GIC500, 0, 0))
1111 gicv3_its_init_sysctl(sc);
1112
1113 return (0);
1114 }
1115
1116 static int
gicv3_its_detach(device_t dev)1117 gicv3_its_detach(device_t dev)
1118 {
1119
1120 return (ENXIO);
1121 }
1122
1123 static void
its_quirk_cavium_22375(device_t dev)1124 its_quirk_cavium_22375(device_t dev)
1125 {
1126 struct gicv3_its_softc *sc;
1127 int domain;
1128
1129 sc = device_get_softc(dev);
1130 sc->sc_its_flags |= ITS_FLAGS_ERRATA_CAVIUM_22375;
1131
1132 /*
1133 * We need to limit which CPUs we send these interrupts to on
1134 * the original dual socket ThunderX as it is unable to
1135 * forward them between the two sockets.
1136 */
1137 if (bus_get_domain(dev, &domain) == 0) {
1138 if (domain < MAXMEMDOM) {
1139 CPU_COPY(&cpuset_domain[domain], &sc->sc_cpus);
1140 } else {
1141 CPU_ZERO(&sc->sc_cpus);
1142 }
1143 }
1144 }
1145
1146 static void
gicv3_its_disable_intr(device_t dev,struct intr_irqsrc * isrc)1147 gicv3_its_disable_intr(device_t dev, struct intr_irqsrc *isrc)
1148 {
1149 struct gicv3_its_softc *sc;
1150 struct gicv3_its_irqsrc *girq;
1151 uint8_t *conf;
1152
1153 sc = device_get_softc(dev);
1154 girq = (struct gicv3_its_irqsrc *)isrc;
1155 conf = sc->sc_conf_base;
1156
1157 conf[girq->gi_lpi] &= ~LPI_CONF_ENABLE;
1158
1159 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1160 /* Clean D-cache under command. */
1161 cpu_dcache_wb_range(&conf[girq->gi_lpi], 1);
1162 } else {
1163 /* DSB inner shareable, store */
1164 dsb(ishst);
1165 }
1166
1167 its_cmd_inv(dev, girq->gi_its_dev, girq);
1168 }
1169
1170 static void
gicv3_its_enable_intr(device_t dev,struct intr_irqsrc * isrc)1171 gicv3_its_enable_intr(device_t dev, struct intr_irqsrc *isrc)
1172 {
1173 struct gicv3_its_softc *sc;
1174 struct gicv3_its_irqsrc *girq;
1175 uint8_t *conf;
1176
1177 sc = device_get_softc(dev);
1178 girq = (struct gicv3_its_irqsrc *)isrc;
1179 conf = sc->sc_conf_base;
1180
1181 conf[girq->gi_lpi] |= LPI_CONF_ENABLE;
1182
1183 if ((sc->sc_its_flags & ITS_FLAGS_LPI_CONF_FLUSH) != 0) {
1184 /* Clean D-cache under command. */
1185 cpu_dcache_wb_range(&conf[girq->gi_lpi], 1);
1186 } else {
1187 /* DSB inner shareable, store */
1188 dsb(ishst);
1189 }
1190
1191 its_cmd_inv(dev, girq->gi_its_dev, girq);
1192 }
1193
1194 static int
gicv3_its_intr(void * arg,uintptr_t irq)1195 gicv3_its_intr(void *arg, uintptr_t irq)
1196 {
1197 struct gicv3_its_softc *sc = arg;
1198 struct gicv3_its_irqsrc *girq;
1199 struct trapframe *tf;
1200
1201 irq -= sc->sc_irq_base;
1202 girq = sc->sc_irqs[irq];
1203 if (girq == NULL)
1204 panic("gicv3_its_intr: Invalid interrupt %ld",
1205 irq + sc->sc_irq_base);
1206
1207 tf = curthread->td_intr_frame;
1208 intr_isrc_dispatch(&girq->gi_isrc, tf);
1209 return (FILTER_HANDLED);
1210 }
1211
1212 static void
gicv3_its_pre_ithread(device_t dev,struct intr_irqsrc * isrc)1213 gicv3_its_pre_ithread(device_t dev, struct intr_irqsrc *isrc)
1214 {
1215 struct gicv3_its_irqsrc *girq;
1216
1217 girq = (struct gicv3_its_irqsrc *)isrc;
1218 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1219 }
1220
1221 static void
gicv3_its_post_ithread(device_t dev,struct intr_irqsrc * isrc)1222 gicv3_its_post_ithread(device_t dev, struct intr_irqsrc *isrc)
1223 {
1224
1225 }
1226
1227 static void
gicv3_its_post_filter(device_t dev,struct intr_irqsrc * isrc)1228 gicv3_its_post_filter(device_t dev, struct intr_irqsrc *isrc)
1229 {
1230 struct gicv3_its_irqsrc *girq;
1231
1232 girq = (struct gicv3_its_irqsrc *)isrc;
1233 gic_icc_write(EOIR1, girq->gi_lpi + GIC_FIRST_LPI);
1234 }
1235
1236 static int
gicv3_its_select_cpu(device_t dev,struct intr_irqsrc * isrc)1237 gicv3_its_select_cpu(device_t dev, struct intr_irqsrc *isrc)
1238 {
1239 struct gicv3_its_softc *sc;
1240
1241 sc = device_get_softc(dev);
1242 if (CPU_EMPTY(&isrc->isrc_cpu)) {
1243 sc->gic_irq_cpu = intr_irq_next_cpu(sc->gic_irq_cpu,
1244 &sc->sc_cpus);
1245 CPU_SETOF(sc->gic_irq_cpu, &isrc->isrc_cpu);
1246 }
1247
1248 return (0);
1249 }
1250
1251 static int
gicv3_its_bind_intr(device_t dev,struct intr_irqsrc * isrc)1252 gicv3_its_bind_intr(device_t dev, struct intr_irqsrc *isrc)
1253 {
1254 struct gicv3_its_irqsrc *girq;
1255
1256 gicv3_its_select_cpu(dev, isrc);
1257
1258 girq = (struct gicv3_its_irqsrc *)isrc;
1259 its_cmd_movi(dev, girq);
1260 return (0);
1261 }
1262
1263 static int
gicv3_its_map_intr(device_t dev,struct intr_map_data * data,struct intr_irqsrc ** isrcp)1264 gicv3_its_map_intr(device_t dev, struct intr_map_data *data,
1265 struct intr_irqsrc **isrcp)
1266 {
1267
1268 /*
1269 * This should never happen, we only call this function to map
1270 * interrupts found before the controller driver is ready.
1271 */
1272 panic("gicv3_its_map_intr: Unable to map a MSI interrupt");
1273 }
1274
1275 static int
gicv3_its_setup_intr(device_t dev,struct intr_irqsrc * isrc,struct resource * res,struct intr_map_data * data)1276 gicv3_its_setup_intr(device_t dev, struct intr_irqsrc *isrc,
1277 struct resource *res, struct intr_map_data *data)
1278 {
1279
1280 /* Bind the interrupt to a CPU */
1281 gicv3_its_bind_intr(dev, isrc);
1282
1283 return (0);
1284 }
1285
1286 #ifdef SMP
1287 static void
gicv3_its_init_secondary(device_t dev)1288 gicv3_its_init_secondary(device_t dev)
1289 {
1290 struct gicv3_its_softc *sc;
1291
1292 sc = device_get_softc(dev);
1293
1294 /*
1295 * This is fatal as otherwise we may bind interrupts to this CPU.
1296 * We need a way to tell the interrupt framework to only bind to a
1297 * subset of given CPUs when it performs the shuffle.
1298 */
1299 if (its_init_cpu(dev, sc) != 0)
1300 panic("gicv3_its_init_secondary: No usable ITS on CPU%d",
1301 PCPU_GET(cpuid));
1302 }
1303 #endif
1304
1305 static uint32_t
its_get_devid(device_t pci_dev)1306 its_get_devid(device_t pci_dev)
1307 {
1308 uintptr_t id;
1309
1310 if (pci_get_id(pci_dev, PCI_ID_MSI, &id) != 0)
1311 panic("%s: %s: Unable to get the MSI DeviceID", __func__,
1312 device_get_nameunit(pci_dev));
1313
1314 return (id);
1315 }
1316
1317 static struct its_dev *
its_device_find(device_t dev,device_t child)1318 its_device_find(device_t dev, device_t child)
1319 {
1320 struct gicv3_its_softc *sc;
1321 struct its_dev *its_dev = NULL;
1322
1323 sc = device_get_softc(dev);
1324
1325 mtx_lock_spin(&sc->sc_its_dev_lock);
1326 TAILQ_FOREACH(its_dev, &sc->sc_its_dev_list, entry) {
1327 if (its_dev->pci_dev == child)
1328 break;
1329 }
1330 mtx_unlock_spin(&sc->sc_its_dev_lock);
1331
1332 return (its_dev);
1333 }
1334
1335 static bool
its_device_alloc(struct gicv3_its_softc * sc,int devid)1336 its_device_alloc(struct gicv3_its_softc *sc, int devid)
1337 {
1338 struct its_ptable *ptable;
1339 void *l2_table;
1340 uint64_t *table;
1341 uint32_t index;
1342 bool shareable;
1343
1344 /* No device table */
1345 if (sc->sc_dev_table_idx < 0) {
1346 if (devid >= (1 << sc->sc_devbits)) {
1347 if (bootverbose) {
1348 device_printf(sc->dev,
1349 "%s: Device out of range for hardware "
1350 "(%x >= %x)\n", __func__, devid,
1351 1 << sc->sc_devbits);
1352 }
1353 return (false);
1354 }
1355 return (true);
1356 }
1357
1358 ptable = &sc->sc_its_ptab[sc->sc_dev_table_idx];
1359 /* Check the devid is within the table limit */
1360 if (!ptable->ptab_indirect) {
1361 if (devid >= ptable->ptab_l1_nidents) {
1362 if (bootverbose) {
1363 device_printf(sc->dev,
1364 "%s: Device out of range for table "
1365 "(%x >= %x)\n", __func__, devid,
1366 ptable->ptab_l1_nidents);
1367 }
1368 return (false);
1369 }
1370
1371 return (true);
1372 }
1373
1374 /* Check the devid is within the allocated range */
1375 index = devid / ptable->ptab_l2_nidents;
1376 if (index >= ptable->ptab_l1_nidents) {
1377 if (bootverbose) {
1378 device_printf(sc->dev,
1379 "%s: Index out of range for table (%x >= %x)\n",
1380 __func__, index, ptable->ptab_l1_nidents);
1381 }
1382 return (false);
1383 }
1384
1385 table = (uint64_t *)ptable->ptab_vaddr;
1386 /* We have an second level table */
1387 if ((table[index] & GITS_BASER_VALID) != 0)
1388 return (true);
1389
1390 shareable = true;
1391 if ((ptable->ptab_share & GITS_BASER_SHARE_MASK) == GITS_BASER_SHARE_NS)
1392 shareable = false;
1393
1394 l2_table = contigmalloc_domainset(ptable->ptab_l2_size,
1395 M_GICV3_ITS, sc->sc_ds, M_WAITOK | M_ZERO, 0, (1ul << 48) - 1,
1396 ptable->ptab_page_size, 0);
1397
1398 if (!shareable)
1399 cpu_dcache_wb_range(l2_table, ptable->ptab_l2_size);
1400
1401 table[index] = vtophys(l2_table) | GITS_BASER_VALID;
1402 if (!shareable)
1403 cpu_dcache_wb_range(&table[index], sizeof(table[index]));
1404
1405 dsb(sy);
1406 return (true);
1407 }
1408
1409 static struct its_dev *
its_device_get(device_t dev,device_t child,u_int nvecs)1410 its_device_get(device_t dev, device_t child, u_int nvecs)
1411 {
1412 struct gicv3_its_softc *sc;
1413 struct its_dev *its_dev;
1414 vmem_addr_t irq_base;
1415 size_t esize;
1416
1417 sc = device_get_softc(dev);
1418
1419 its_dev = its_device_find(dev, child);
1420 if (its_dev != NULL)
1421 return (its_dev);
1422
1423 its_dev = malloc(sizeof(*its_dev), M_GICV3_ITS, M_NOWAIT | M_ZERO);
1424 if (its_dev == NULL)
1425 return (NULL);
1426
1427 its_dev->pci_dev = child;
1428 its_dev->devid = its_get_devid(child);
1429
1430 its_dev->lpis.lpi_busy = 0;
1431 its_dev->lpis.lpi_num = nvecs;
1432 its_dev->lpis.lpi_free = nvecs;
1433
1434 if (!its_device_alloc(sc, its_dev->devid)) {
1435 free(its_dev, M_GICV3_ITS);
1436 return (NULL);
1437 }
1438
1439 if (vmem_alloc(sc->sc_irq_alloc, nvecs, M_FIRSTFIT | M_NOWAIT,
1440 &irq_base) != 0) {
1441 free(its_dev, M_GICV3_ITS);
1442 return (NULL);
1443 }
1444 its_dev->lpis.lpi_base = irq_base;
1445
1446 /* Get ITT entry size */
1447 esize = GITS_TYPER_ITTES(gic_its_read_8(sc, GITS_TYPER));
1448
1449 /*
1450 * Allocate ITT for this device.
1451 * PA has to be 256 B aligned. At least two entries for device.
1452 */
1453 its_dev->itt_size = roundup2(MAX(nvecs, 2) * esize, 256);
1454 its_dev->itt = contigmalloc_domainset(its_dev->itt_size,
1455 M_GICV3_ITS, sc->sc_ds, M_NOWAIT | M_ZERO, 0,
1456 LPI_INT_TRANS_TAB_MAX_ADDR, LPI_INT_TRANS_TAB_ALIGN, 0);
1457 if (its_dev->itt == NULL) {
1458 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base, nvecs);
1459 free(its_dev, M_GICV3_ITS);
1460 return (NULL);
1461 }
1462
1463 /* Make sure device sees zeroed ITT. */
1464 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0)
1465 cpu_dcache_wb_range(its_dev->itt, its_dev->itt_size);
1466
1467 mtx_lock_spin(&sc->sc_its_dev_lock);
1468 TAILQ_INSERT_TAIL(&sc->sc_its_dev_list, its_dev, entry);
1469 mtx_unlock_spin(&sc->sc_its_dev_lock);
1470
1471 /* Map device to its ITT */
1472 its_cmd_mapd(dev, its_dev, 1);
1473
1474 return (its_dev);
1475 }
1476
1477 static void
its_device_release(device_t dev,struct its_dev * its_dev)1478 its_device_release(device_t dev, struct its_dev *its_dev)
1479 {
1480 struct gicv3_its_softc *sc;
1481
1482 KASSERT(its_dev->lpis.lpi_busy == 0,
1483 ("its_device_release: Trying to release an inuse ITS device"));
1484
1485 /* Unmap device in ITS */
1486 its_cmd_mapd(dev, its_dev, 0);
1487
1488 sc = device_get_softc(dev);
1489
1490 /* Remove the device from the list of devices */
1491 mtx_lock_spin(&sc->sc_its_dev_lock);
1492 TAILQ_REMOVE(&sc->sc_its_dev_list, its_dev, entry);
1493 mtx_unlock_spin(&sc->sc_its_dev_lock);
1494
1495 /* Free ITT */
1496 KASSERT(its_dev->itt != NULL, ("Invalid ITT in valid ITS device"));
1497 contigfree(its_dev->itt, its_dev->itt_size, M_GICV3_ITS);
1498
1499 /* Free the IRQ allocation */
1500 vmem_free(sc->sc_irq_alloc, its_dev->lpis.lpi_base,
1501 its_dev->lpis.lpi_num);
1502
1503 free(its_dev, M_GICV3_ITS);
1504 }
1505
1506 static struct gicv3_its_irqsrc *
gicv3_its_alloc_irqsrc(device_t dev,struct gicv3_its_softc * sc,u_int irq)1507 gicv3_its_alloc_irqsrc(device_t dev, struct gicv3_its_softc *sc, u_int irq)
1508 {
1509 struct gicv3_its_irqsrc *girq = NULL;
1510
1511 KASSERT(sc->sc_irqs[irq] == NULL,
1512 ("%s: Interrupt %u already allocated", __func__, irq));
1513 mtx_lock_spin(&sc->sc_its_dev_lock);
1514 if (!TAILQ_EMPTY(&sc->sc_free_irqs)) {
1515 girq = TAILQ_FIRST(&sc->sc_free_irqs);
1516 TAILQ_REMOVE(&sc->sc_free_irqs, girq, gi_link);
1517 }
1518 mtx_unlock_spin(&sc->sc_its_dev_lock);
1519 if (girq == NULL) {
1520 girq = malloc(sizeof(*girq), M_GICV3_ITS,
1521 M_NOWAIT | M_ZERO);
1522 if (girq == NULL)
1523 return (NULL);
1524 girq->gi_id = -1;
1525 if (intr_isrc_register(&girq->gi_isrc, dev, 0,
1526 "%s,%u", device_get_nameunit(dev), irq) != 0) {
1527 free(girq, M_GICV3_ITS);
1528 return (NULL);
1529 }
1530 }
1531 girq->gi_lpi = irq + sc->sc_irq_base - GIC_FIRST_LPI;
1532 sc->sc_irqs[irq] = girq;
1533
1534 return (girq);
1535 }
1536
1537 static void
gicv3_its_release_irqsrc(struct gicv3_its_softc * sc,struct gicv3_its_irqsrc * girq)1538 gicv3_its_release_irqsrc(struct gicv3_its_softc *sc,
1539 struct gicv3_its_irqsrc *girq)
1540 {
1541 u_int irq;
1542
1543 mtx_assert(&sc->sc_its_dev_lock, MA_OWNED);
1544
1545 irq = girq->gi_lpi + GIC_FIRST_LPI - sc->sc_irq_base;
1546 sc->sc_irqs[irq] = NULL;
1547
1548 girq->gi_id = -1;
1549 girq->gi_its_dev = NULL;
1550 TAILQ_INSERT_TAIL(&sc->sc_free_irqs, girq, gi_link);
1551 }
1552
1553 static int
gicv3_its_alloc_msi(device_t dev,device_t child,int count,int maxcount,device_t * pic,struct intr_irqsrc ** srcs)1554 gicv3_its_alloc_msi(device_t dev, device_t child, int count, int maxcount,
1555 device_t *pic, struct intr_irqsrc **srcs)
1556 {
1557 struct gicv3_its_softc *sc;
1558 struct gicv3_its_irqsrc *girq;
1559 struct its_dev *its_dev;
1560 u_int irq;
1561 int i;
1562
1563 its_dev = its_device_get(dev, child, count);
1564 if (its_dev == NULL)
1565 return (ENXIO);
1566
1567 KASSERT(its_dev->lpis.lpi_free >= count,
1568 ("gicv3_its_alloc_msi: No free LPIs"));
1569 sc = device_get_softc(dev);
1570 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1571 its_dev->lpis.lpi_free;
1572
1573 /* Allocate the irqsrc for each MSI */
1574 for (i = 0; i < count; i++, irq++) {
1575 its_dev->lpis.lpi_free--;
1576 srcs[i] = (struct intr_irqsrc *)gicv3_its_alloc_irqsrc(dev,
1577 sc, irq);
1578 if (srcs[i] == NULL)
1579 break;
1580 }
1581
1582 /* The allocation failed, release them */
1583 if (i != count) {
1584 mtx_lock_spin(&sc->sc_its_dev_lock);
1585 for (i = 0; i < count; i++) {
1586 girq = (struct gicv3_its_irqsrc *)srcs[i];
1587 if (girq == NULL)
1588 break;
1589 gicv3_its_release_irqsrc(sc, girq);
1590 srcs[i] = NULL;
1591 }
1592 mtx_unlock_spin(&sc->sc_its_dev_lock);
1593 return (ENXIO);
1594 }
1595
1596 /* Finish the allocation now we have all MSI irqsrcs */
1597 for (i = 0; i < count; i++) {
1598 girq = (struct gicv3_its_irqsrc *)srcs[i];
1599 girq->gi_id = i;
1600 girq->gi_its_dev = its_dev;
1601
1602 /* Map the message to the given IRQ */
1603 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1604 its_cmd_mapti(dev, girq);
1605 }
1606 its_dev->lpis.lpi_busy += count;
1607 *pic = dev;
1608
1609 return (0);
1610 }
1611
1612 static int
gicv3_its_release_msi(device_t dev,device_t child,int count,struct intr_irqsrc ** isrc)1613 gicv3_its_release_msi(device_t dev, device_t child, int count,
1614 struct intr_irqsrc **isrc)
1615 {
1616 struct gicv3_its_softc *sc;
1617 struct gicv3_its_irqsrc *girq;
1618 struct its_dev *its_dev;
1619 int i;
1620
1621 its_dev = its_device_find(dev, child);
1622
1623 KASSERT(its_dev != NULL,
1624 ("gicv3_its_release_msi: Releasing a MSI interrupt with "
1625 "no ITS device"));
1626 KASSERT(its_dev->lpis.lpi_busy >= count,
1627 ("gicv3_its_release_msi: Releasing more interrupts than "
1628 "were allocated: releasing %d, allocated %d", count,
1629 its_dev->lpis.lpi_busy));
1630
1631 sc = device_get_softc(dev);
1632 mtx_lock_spin(&sc->sc_its_dev_lock);
1633 for (i = 0; i < count; i++) {
1634 girq = (struct gicv3_its_irqsrc *)isrc[i];
1635 gicv3_its_release_irqsrc(sc, girq);
1636 }
1637 mtx_unlock_spin(&sc->sc_its_dev_lock);
1638 its_dev->lpis.lpi_busy -= count;
1639
1640 if (its_dev->lpis.lpi_busy == 0)
1641 its_device_release(dev, its_dev);
1642
1643 return (0);
1644 }
1645
1646 static int
gicv3_its_alloc_msix(device_t dev,device_t child,device_t * pic,struct intr_irqsrc ** isrcp)1647 gicv3_its_alloc_msix(device_t dev, device_t child, device_t *pic,
1648 struct intr_irqsrc **isrcp)
1649 {
1650 struct gicv3_its_softc *sc;
1651 struct gicv3_its_irqsrc *girq;
1652 struct its_dev *its_dev;
1653 u_int nvecs, irq;
1654
1655 nvecs = pci_msix_count(child);
1656 its_dev = its_device_get(dev, child, nvecs);
1657 if (its_dev == NULL)
1658 return (ENXIO);
1659
1660 KASSERT(its_dev->lpis.lpi_free > 0,
1661 ("gicv3_its_alloc_msix: No free LPIs"));
1662 sc = device_get_softc(dev);
1663 irq = its_dev->lpis.lpi_base + its_dev->lpis.lpi_num -
1664 its_dev->lpis.lpi_free;
1665
1666 girq = gicv3_its_alloc_irqsrc(dev, sc, irq);
1667 if (girq == NULL)
1668 return (ENXIO);
1669 girq->gi_id = its_dev->lpis.lpi_busy;
1670 girq->gi_its_dev = its_dev;
1671
1672 its_dev->lpis.lpi_free--;
1673 its_dev->lpis.lpi_busy++;
1674
1675 /* Map the message to the given IRQ */
1676 gicv3_its_select_cpu(dev, (struct intr_irqsrc *)girq);
1677 its_cmd_mapti(dev, girq);
1678
1679 *pic = dev;
1680 *isrcp = (struct intr_irqsrc *)girq;
1681
1682 return (0);
1683 }
1684
1685 static int
gicv3_its_release_msix(device_t dev,device_t child,struct intr_irqsrc * isrc)1686 gicv3_its_release_msix(device_t dev, device_t child, struct intr_irqsrc *isrc)
1687 {
1688 struct gicv3_its_softc *sc;
1689 struct gicv3_its_irqsrc *girq;
1690 struct its_dev *its_dev;
1691
1692 its_dev = its_device_find(dev, child);
1693
1694 KASSERT(its_dev != NULL,
1695 ("gicv3_its_release_msix: Releasing a MSI-X interrupt with "
1696 "no ITS device"));
1697 KASSERT(its_dev->lpis.lpi_busy > 0,
1698 ("gicv3_its_release_msix: Releasing more interrupts than "
1699 "were allocated: allocated %d", its_dev->lpis.lpi_busy));
1700
1701 sc = device_get_softc(dev);
1702 girq = (struct gicv3_its_irqsrc *)isrc;
1703 mtx_lock_spin(&sc->sc_its_dev_lock);
1704 gicv3_its_release_irqsrc(sc, girq);
1705 mtx_unlock_spin(&sc->sc_its_dev_lock);
1706 its_dev->lpis.lpi_busy--;
1707
1708 if (its_dev->lpis.lpi_busy == 0)
1709 its_device_release(dev, its_dev);
1710
1711 return (0);
1712 }
1713
1714 static int
gicv3_its_map_msi(device_t dev,device_t child,struct intr_irqsrc * isrc,uint64_t * addr,uint32_t * data)1715 gicv3_its_map_msi(device_t dev, device_t child, struct intr_irqsrc *isrc,
1716 uint64_t *addr, uint32_t *data)
1717 {
1718 struct gicv3_its_softc *sc;
1719 struct gicv3_its_irqsrc *girq;
1720
1721 sc = device_get_softc(dev);
1722 girq = (struct gicv3_its_irqsrc *)isrc;
1723
1724 *addr = vtophys(rman_get_virtual(sc->sc_its_res)) + GITS_TRANSLATER;
1725 *data = girq->gi_id;
1726
1727 return (0);
1728 }
1729
1730 #ifdef IOMMU
1731 static int
gicv3_iommu_init(device_t dev,device_t child,struct iommu_domain ** domain)1732 gicv3_iommu_init(device_t dev, device_t child, struct iommu_domain **domain)
1733 {
1734 struct gicv3_its_softc *sc;
1735 struct iommu_ctx *ctx;
1736 int error;
1737
1738 sc = device_get_softc(dev);
1739 ctx = iommu_get_dev_ctx(child);
1740 if (ctx == NULL)
1741 return (ENXIO);
1742 /* Map the page containing the GITS_TRANSLATER register. */
1743 error = iommu_map_msi(ctx, PAGE_SIZE, 0,
1744 IOMMU_MAP_ENTRY_WRITE, IOMMU_MF_CANWAIT, &sc->ma);
1745 *domain = iommu_get_ctx_domain(ctx);
1746
1747 return (error);
1748 }
1749
1750 static void
gicv3_iommu_deinit(device_t dev,device_t child)1751 gicv3_iommu_deinit(device_t dev, device_t child)
1752 {
1753 struct iommu_ctx *ctx;
1754
1755 ctx = iommu_get_dev_ctx(child);
1756 if (ctx == NULL)
1757 return;
1758
1759 iommu_unmap_msi(ctx);
1760 }
1761 #endif
1762
1763 /*
1764 * Commands handling.
1765 */
1766
1767 static __inline void
cmd_format_command(struct its_cmd * cmd,uint8_t cmd_type)1768 cmd_format_command(struct its_cmd *cmd, uint8_t cmd_type)
1769 {
1770 /* Command field: DW0 [7:0] */
1771 cmd->cmd_dword[0] &= htole64(~CMD_COMMAND_MASK);
1772 cmd->cmd_dword[0] |= htole64(cmd_type);
1773 }
1774
1775 static __inline void
cmd_format_devid(struct its_cmd * cmd,uint32_t devid)1776 cmd_format_devid(struct its_cmd *cmd, uint32_t devid)
1777 {
1778 /* Device ID field: DW0 [63:32] */
1779 cmd->cmd_dword[0] &= htole64(~CMD_DEVID_MASK);
1780 cmd->cmd_dword[0] |= htole64((uint64_t)devid << CMD_DEVID_SHIFT);
1781 }
1782
1783 static __inline void
cmd_format_size(struct its_cmd * cmd,uint16_t size)1784 cmd_format_size(struct its_cmd *cmd, uint16_t size)
1785 {
1786 /* Size field: DW1 [4:0] */
1787 cmd->cmd_dword[1] &= htole64(~CMD_SIZE_MASK);
1788 cmd->cmd_dword[1] |= htole64((size & CMD_SIZE_MASK));
1789 }
1790
1791 static __inline void
cmd_format_id(struct its_cmd * cmd,uint32_t id)1792 cmd_format_id(struct its_cmd *cmd, uint32_t id)
1793 {
1794 /* ID field: DW1 [31:0] */
1795 cmd->cmd_dword[1] &= htole64(~CMD_ID_MASK);
1796 cmd->cmd_dword[1] |= htole64(id);
1797 }
1798
1799 static __inline void
cmd_format_pid(struct its_cmd * cmd,uint32_t pid)1800 cmd_format_pid(struct its_cmd *cmd, uint32_t pid)
1801 {
1802 /* Physical ID field: DW1 [63:32] */
1803 cmd->cmd_dword[1] &= htole64(~CMD_PID_MASK);
1804 cmd->cmd_dword[1] |= htole64((uint64_t)pid << CMD_PID_SHIFT);
1805 }
1806
1807 static __inline void
cmd_format_col(struct its_cmd * cmd,uint16_t col_id)1808 cmd_format_col(struct its_cmd *cmd, uint16_t col_id)
1809 {
1810 /* Collection field: DW2 [16:0] */
1811 cmd->cmd_dword[2] &= htole64(~CMD_COL_MASK);
1812 cmd->cmd_dword[2] |= htole64(col_id);
1813 }
1814
1815 static __inline void
cmd_format_target(struct its_cmd * cmd,uint64_t target)1816 cmd_format_target(struct its_cmd *cmd, uint64_t target)
1817 {
1818 /* Target Address field: DW2 [47:16] */
1819 cmd->cmd_dword[2] &= htole64(~CMD_TARGET_MASK);
1820 cmd->cmd_dword[2] |= htole64(target & CMD_TARGET_MASK);
1821 }
1822
1823 static __inline void
cmd_format_itt(struct its_cmd * cmd,uint64_t itt)1824 cmd_format_itt(struct its_cmd *cmd, uint64_t itt)
1825 {
1826 /* ITT Address field: DW2 [47:8] */
1827 cmd->cmd_dword[2] &= htole64(~CMD_ITT_MASK);
1828 cmd->cmd_dword[2] |= htole64(itt & CMD_ITT_MASK);
1829 }
1830
1831 static __inline void
cmd_format_valid(struct its_cmd * cmd,uint8_t valid)1832 cmd_format_valid(struct its_cmd *cmd, uint8_t valid)
1833 {
1834 /* Valid field: DW2 [63] */
1835 cmd->cmd_dword[2] &= htole64(~CMD_VALID_MASK);
1836 cmd->cmd_dword[2] |= htole64((uint64_t)valid << CMD_VALID_SHIFT);
1837 }
1838
1839 static inline bool
its_cmd_queue_full(struct gicv3_its_softc * sc)1840 its_cmd_queue_full(struct gicv3_its_softc *sc)
1841 {
1842 size_t read_idx, next_write_idx;
1843
1844 /* Get the index of the next command */
1845 next_write_idx = (sc->sc_its_cmd_next_idx + 1) %
1846 (ITS_CMDQ_SIZE / sizeof(struct its_cmd));
1847 /* And the index of the current command being read */
1848 read_idx = gic_its_read_4(sc, GITS_CREADR) / sizeof(struct its_cmd);
1849
1850 /*
1851 * The queue is full when the write offset points
1852 * at the command before the current read offset.
1853 */
1854 return (next_write_idx == read_idx);
1855 }
1856
1857 static inline void
its_cmd_sync(struct gicv3_its_softc * sc,struct its_cmd * cmd)1858 its_cmd_sync(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1859 {
1860
1861 if ((sc->sc_its_flags & ITS_FLAGS_CMDQ_FLUSH) != 0) {
1862 /* Clean D-cache under command. */
1863 cpu_dcache_wb_range(cmd, sizeof(*cmd));
1864 } else {
1865 /* DSB inner shareable, store */
1866 dsb(ishst);
1867 }
1868
1869 }
1870
1871 static inline uint64_t
its_cmd_cwriter_offset(struct gicv3_its_softc * sc,struct its_cmd * cmd)1872 its_cmd_cwriter_offset(struct gicv3_its_softc *sc, struct its_cmd *cmd)
1873 {
1874 uint64_t off;
1875
1876 off = (cmd - sc->sc_its_cmd_base) * sizeof(*cmd);
1877
1878 return (off);
1879 }
1880
1881 static void
its_cmd_wait_completion(device_t dev,struct its_cmd * cmd_first,struct its_cmd * cmd_last)1882 its_cmd_wait_completion(device_t dev, struct its_cmd *cmd_first,
1883 struct its_cmd *cmd_last)
1884 {
1885 struct gicv3_its_softc *sc;
1886 uint64_t first, last, read;
1887 size_t us_left;
1888
1889 sc = device_get_softc(dev);
1890
1891 /*
1892 * XXX ARM64TODO: This is obviously a significant delay.
1893 * The reason for that is that currently the time frames for
1894 * the command to complete are not known.
1895 */
1896 us_left = 1000000;
1897
1898 first = its_cmd_cwriter_offset(sc, cmd_first);
1899 last = its_cmd_cwriter_offset(sc, cmd_last);
1900
1901 for (;;) {
1902 read = gic_its_read_8(sc, GITS_CREADR);
1903 if (first < last) {
1904 if (read < first || read >= last)
1905 break;
1906 } else if (read < first && read >= last)
1907 break;
1908
1909 if (us_left-- == 0) {
1910 /* This means timeout */
1911 device_printf(dev,
1912 "Timeout while waiting for CMD completion.\n");
1913 return;
1914 }
1915 DELAY(1);
1916 }
1917 }
1918
1919 static struct its_cmd *
its_cmd_alloc_locked(device_t dev)1920 its_cmd_alloc_locked(device_t dev)
1921 {
1922 struct gicv3_its_softc *sc;
1923 struct its_cmd *cmd;
1924 size_t us_left;
1925
1926 sc = device_get_softc(dev);
1927
1928 /*
1929 * XXX ARM64TODO: This is obviously a significant delay.
1930 * The reason for that is that currently the time frames for
1931 * the command to complete (and therefore free the descriptor)
1932 * are not known.
1933 */
1934 us_left = 1000000;
1935
1936 mtx_assert(&sc->sc_its_cmd_lock, MA_OWNED);
1937 while (its_cmd_queue_full(sc)) {
1938 if (us_left-- == 0) {
1939 /* Timeout while waiting for free command */
1940 device_printf(dev,
1941 "Timeout while waiting for free command\n");
1942 return (NULL);
1943 }
1944 DELAY(1);
1945 }
1946
1947 cmd = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
1948 sc->sc_its_cmd_next_idx++;
1949 sc->sc_its_cmd_next_idx %= ITS_CMDQ_SIZE / sizeof(struct its_cmd);
1950
1951 return (cmd);
1952 }
1953
1954 static uint64_t
its_cmd_prepare(struct its_cmd * cmd,struct its_cmd_desc * desc)1955 its_cmd_prepare(struct its_cmd *cmd, struct its_cmd_desc *desc)
1956 {
1957 uint64_t target;
1958 uint8_t cmd_type;
1959 u_int size;
1960
1961 cmd_type = desc->cmd_type;
1962 target = ITS_TARGET_NONE;
1963
1964 switch (cmd_type) {
1965 case ITS_CMD_MOVI: /* Move interrupt ID to another collection */
1966 target = desc->cmd_desc_movi.col->col_target;
1967 cmd_format_command(cmd, ITS_CMD_MOVI);
1968 cmd_format_id(cmd, desc->cmd_desc_movi.id);
1969 cmd_format_col(cmd, desc->cmd_desc_movi.col->col_id);
1970 cmd_format_devid(cmd, desc->cmd_desc_movi.its_dev->devid);
1971 break;
1972 case ITS_CMD_SYNC: /* Wait for previous commands completion */
1973 target = desc->cmd_desc_sync.col->col_target;
1974 cmd_format_command(cmd, ITS_CMD_SYNC);
1975 cmd_format_target(cmd, target);
1976 break;
1977 case ITS_CMD_MAPD: /* Assign ITT to device */
1978 cmd_format_command(cmd, ITS_CMD_MAPD);
1979 cmd_format_itt(cmd, vtophys(desc->cmd_desc_mapd.its_dev->itt));
1980 /*
1981 * Size describes number of bits to encode interrupt IDs
1982 * supported by the device minus one.
1983 * When V (valid) bit is zero, this field should be written
1984 * as zero.
1985 */
1986 if (desc->cmd_desc_mapd.valid != 0) {
1987 size = fls(desc->cmd_desc_mapd.its_dev->lpis.lpi_num);
1988 size = MAX(1, size) - 1;
1989 } else
1990 size = 0;
1991
1992 cmd_format_size(cmd, size);
1993 cmd_format_devid(cmd, desc->cmd_desc_mapd.its_dev->devid);
1994 cmd_format_valid(cmd, desc->cmd_desc_mapd.valid);
1995 break;
1996 case ITS_CMD_MAPC: /* Map collection to Re-Distributor */
1997 target = desc->cmd_desc_mapc.col->col_target;
1998 cmd_format_command(cmd, ITS_CMD_MAPC);
1999 cmd_format_col(cmd, desc->cmd_desc_mapc.col->col_id);
2000 cmd_format_valid(cmd, desc->cmd_desc_mapc.valid);
2001 cmd_format_target(cmd, target);
2002 break;
2003 case ITS_CMD_MAPTI:
2004 target = desc->cmd_desc_mapvi.col->col_target;
2005 cmd_format_command(cmd, ITS_CMD_MAPTI);
2006 cmd_format_devid(cmd, desc->cmd_desc_mapvi.its_dev->devid);
2007 cmd_format_id(cmd, desc->cmd_desc_mapvi.id);
2008 cmd_format_pid(cmd, desc->cmd_desc_mapvi.pid);
2009 cmd_format_col(cmd, desc->cmd_desc_mapvi.col->col_id);
2010 break;
2011 case ITS_CMD_MAPI:
2012 target = desc->cmd_desc_mapi.col->col_target;
2013 cmd_format_command(cmd, ITS_CMD_MAPI);
2014 cmd_format_devid(cmd, desc->cmd_desc_mapi.its_dev->devid);
2015 cmd_format_id(cmd, desc->cmd_desc_mapi.pid);
2016 cmd_format_col(cmd, desc->cmd_desc_mapi.col->col_id);
2017 break;
2018 case ITS_CMD_INV:
2019 target = desc->cmd_desc_inv.col->col_target;
2020 cmd_format_command(cmd, ITS_CMD_INV);
2021 cmd_format_devid(cmd, desc->cmd_desc_inv.its_dev->devid);
2022 cmd_format_id(cmd, desc->cmd_desc_inv.pid);
2023 break;
2024 case ITS_CMD_INVALL:
2025 cmd_format_command(cmd, ITS_CMD_INVALL);
2026 cmd_format_col(cmd, desc->cmd_desc_invall.col->col_id);
2027 break;
2028 default:
2029 panic("its_cmd_prepare: Invalid command: %x", cmd_type);
2030 }
2031
2032 return (target);
2033 }
2034
2035 static int
its_cmd_send(device_t dev,struct its_cmd_desc * desc)2036 its_cmd_send(device_t dev, struct its_cmd_desc *desc)
2037 {
2038 struct gicv3_its_softc *sc;
2039 struct its_cmd *cmd, *cmd_sync, *cmd_write;
2040 struct its_col col_sync;
2041 struct its_cmd_desc desc_sync;
2042 uint64_t target, cwriter;
2043
2044 sc = device_get_softc(dev);
2045 mtx_lock_spin(&sc->sc_its_cmd_lock);
2046 cmd = its_cmd_alloc_locked(dev);
2047 if (cmd == NULL) {
2048 device_printf(dev, "could not allocate ITS command\n");
2049 mtx_unlock_spin(&sc->sc_its_cmd_lock);
2050 return (EBUSY);
2051 }
2052
2053 target = its_cmd_prepare(cmd, desc);
2054 its_cmd_sync(sc, cmd);
2055
2056 if (target != ITS_TARGET_NONE) {
2057 cmd_sync = its_cmd_alloc_locked(dev);
2058 if (cmd_sync != NULL) {
2059 desc_sync.cmd_type = ITS_CMD_SYNC;
2060 col_sync.col_target = target;
2061 desc_sync.cmd_desc_sync.col = &col_sync;
2062 its_cmd_prepare(cmd_sync, &desc_sync);
2063 its_cmd_sync(sc, cmd_sync);
2064 }
2065 }
2066
2067 /* Update GITS_CWRITER */
2068 cwriter = sc->sc_its_cmd_next_idx * sizeof(struct its_cmd);
2069 gic_its_write_8(sc, GITS_CWRITER, cwriter);
2070 cmd_write = &sc->sc_its_cmd_base[sc->sc_its_cmd_next_idx];
2071 mtx_unlock_spin(&sc->sc_its_cmd_lock);
2072
2073 its_cmd_wait_completion(dev, cmd, cmd_write);
2074
2075 return (0);
2076 }
2077
2078 /* Handlers to send commands */
2079 static void
its_cmd_movi(device_t dev,struct gicv3_its_irqsrc * girq)2080 its_cmd_movi(device_t dev, struct gicv3_its_irqsrc *girq)
2081 {
2082 struct gicv3_its_softc *sc;
2083 struct its_cmd_desc desc;
2084 struct its_col *col;
2085
2086 sc = device_get_softc(dev);
2087 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2088
2089 desc.cmd_type = ITS_CMD_MOVI;
2090 desc.cmd_desc_movi.its_dev = girq->gi_its_dev;
2091 desc.cmd_desc_movi.col = col;
2092 desc.cmd_desc_movi.id = girq->gi_id;
2093
2094 its_cmd_send(dev, &desc);
2095 }
2096
2097 static void
its_cmd_mapc(device_t dev,struct its_col * col,uint8_t valid)2098 its_cmd_mapc(device_t dev, struct its_col *col, uint8_t valid)
2099 {
2100 struct its_cmd_desc desc;
2101
2102 desc.cmd_type = ITS_CMD_MAPC;
2103 desc.cmd_desc_mapc.col = col;
2104 /*
2105 * Valid bit set - map the collection.
2106 * Valid bit cleared - unmap the collection.
2107 */
2108 desc.cmd_desc_mapc.valid = valid;
2109
2110 its_cmd_send(dev, &desc);
2111 }
2112
2113 static void
its_cmd_mapti(device_t dev,struct gicv3_its_irqsrc * girq)2114 its_cmd_mapti(device_t dev, struct gicv3_its_irqsrc *girq)
2115 {
2116 struct gicv3_its_softc *sc;
2117 struct its_cmd_desc desc;
2118 struct its_col *col;
2119 u_int col_id;
2120
2121 sc = device_get_softc(dev);
2122
2123 col_id = CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1;
2124 col = sc->sc_its_cols[col_id];
2125
2126 desc.cmd_type = ITS_CMD_MAPTI;
2127 desc.cmd_desc_mapvi.its_dev = girq->gi_its_dev;
2128 desc.cmd_desc_mapvi.col = col;
2129 /* The EventID sent to the device */
2130 desc.cmd_desc_mapvi.id = girq->gi_id;
2131 /* The physical interrupt presented to softeware */
2132 desc.cmd_desc_mapvi.pid = girq->gi_lpi + GIC_FIRST_LPI;
2133
2134 its_cmd_send(dev, &desc);
2135 }
2136
2137 static void
its_cmd_mapd(device_t dev,struct its_dev * its_dev,uint8_t valid)2138 its_cmd_mapd(device_t dev, struct its_dev *its_dev, uint8_t valid)
2139 {
2140 struct its_cmd_desc desc;
2141
2142 desc.cmd_type = ITS_CMD_MAPD;
2143 desc.cmd_desc_mapd.its_dev = its_dev;
2144 desc.cmd_desc_mapd.valid = valid;
2145
2146 its_cmd_send(dev, &desc);
2147 }
2148
2149 static void
its_cmd_inv(device_t dev,struct its_dev * its_dev,struct gicv3_its_irqsrc * girq)2150 its_cmd_inv(device_t dev, struct its_dev *its_dev,
2151 struct gicv3_its_irqsrc *girq)
2152 {
2153 struct gicv3_its_softc *sc;
2154 struct its_cmd_desc desc;
2155 struct its_col *col;
2156
2157 sc = device_get_softc(dev);
2158 col = sc->sc_its_cols[CPU_FFS(&girq->gi_isrc.isrc_cpu) - 1];
2159
2160 desc.cmd_type = ITS_CMD_INV;
2161 /* The EventID sent to the device */
2162 desc.cmd_desc_inv.pid = girq->gi_id;
2163 desc.cmd_desc_inv.its_dev = its_dev;
2164 desc.cmd_desc_inv.col = col;
2165
2166 its_cmd_send(dev, &desc);
2167 }
2168
2169 static void
its_cmd_invall(device_t dev,struct its_col * col)2170 its_cmd_invall(device_t dev, struct its_col *col)
2171 {
2172 struct its_cmd_desc desc;
2173
2174 desc.cmd_type = ITS_CMD_INVALL;
2175 desc.cmd_desc_invall.col = col;
2176
2177 its_cmd_send(dev, &desc);
2178 }
2179
2180 #ifdef FDT
2181 static device_probe_t gicv3_its_fdt_probe;
2182 static device_attach_t gicv3_its_fdt_attach;
2183
2184 static device_method_t gicv3_its_fdt_methods[] = {
2185 /* Device interface */
2186 DEVMETHOD(device_probe, gicv3_its_fdt_probe),
2187 DEVMETHOD(device_attach, gicv3_its_fdt_attach),
2188
2189 /* End */
2190 DEVMETHOD_END
2191 };
2192
2193 #define its_baseclasses its_fdt_baseclasses
2194 DEFINE_CLASS_1(its, gicv3_its_fdt_driver, gicv3_its_fdt_methods,
2195 sizeof(struct gicv3_its_softc), gicv3_its_driver);
2196 #undef its_baseclasses
2197
2198 EARLY_DRIVER_MODULE(its_fdt, gic, gicv3_its_fdt_driver, 0, 0,
2199 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2200
2201 static int
gicv3_its_fdt_probe(device_t dev)2202 gicv3_its_fdt_probe(device_t dev)
2203 {
2204
2205 if (!ofw_bus_status_okay(dev))
2206 return (ENXIO);
2207
2208 if (!ofw_bus_is_compatible(dev, "arm,gic-v3-its"))
2209 return (ENXIO);
2210
2211 if (!gicv3_get_support_lpis(dev))
2212 return (ENXIO);
2213
2214 device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2215 return (BUS_PROBE_DEFAULT);
2216 }
2217
2218 static int
gicv3_its_fdt_attach(device_t dev)2219 gicv3_its_fdt_attach(device_t dev)
2220 {
2221 struct gicv3_its_softc *sc;
2222 phandle_t xref;
2223 int err;
2224
2225 sc = device_get_softc(dev);
2226 sc->dev = dev;
2227 err = gicv3_its_attach(dev);
2228 if (err != 0)
2229 return (err);
2230
2231 /* Register this device as a interrupt controller */
2232 xref = OF_xref_from_node(ofw_bus_get_node(dev));
2233 sc->sc_pic = intr_pic_register(dev, xref);
2234 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2235 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2236 if (err != 0) {
2237 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2238 return (err);
2239 }
2240
2241 /* Register this device to handle MSI interrupts */
2242 err = intr_msi_register(dev, xref);
2243 if (err != 0) {
2244 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2245 return (err);
2246 }
2247
2248 return (0);
2249 }
2250 #endif
2251
2252 #ifdef DEV_ACPI
2253 static device_probe_t gicv3_its_acpi_probe;
2254 static device_attach_t gicv3_its_acpi_attach;
2255
2256 static device_method_t gicv3_its_acpi_methods[] = {
2257 /* Device interface */
2258 DEVMETHOD(device_probe, gicv3_its_acpi_probe),
2259 DEVMETHOD(device_attach, gicv3_its_acpi_attach),
2260
2261 /* End */
2262 DEVMETHOD_END
2263 };
2264
2265 #define its_baseclasses its_acpi_baseclasses
2266 DEFINE_CLASS_1(its, gicv3_its_acpi_driver, gicv3_its_acpi_methods,
2267 sizeof(struct gicv3_its_softc), gicv3_its_driver);
2268 #undef its_baseclasses
2269
2270 EARLY_DRIVER_MODULE(its_acpi, gic, gicv3_its_acpi_driver, 0, 0,
2271 BUS_PASS_INTERRUPT + BUS_PASS_ORDER_MIDDLE);
2272
2273 static int
gicv3_its_acpi_probe(device_t dev)2274 gicv3_its_acpi_probe(device_t dev)
2275 {
2276
2277 if (gic_get_bus(dev) != GIC_BUS_ACPI)
2278 return (EINVAL);
2279
2280 if (gic_get_hw_rev(dev) < 3)
2281 return (EINVAL);
2282
2283 if (!gicv3_get_support_lpis(dev))
2284 return (ENXIO);
2285
2286 device_set_desc(dev, "ARM GIC Interrupt Translation Service");
2287 return (BUS_PROBE_DEFAULT);
2288 }
2289
2290 static int
gicv3_its_acpi_attach(device_t dev)2291 gicv3_its_acpi_attach(device_t dev)
2292 {
2293 struct gicv3_its_softc *sc;
2294 struct gic_v3_devinfo *di;
2295 int err;
2296
2297 sc = device_get_softc(dev);
2298 sc->dev = dev;
2299 err = gicv3_its_attach(dev);
2300 if (err != 0)
2301 return (err);
2302
2303 di = device_get_ivars(dev);
2304 sc->sc_pic = intr_pic_register(dev, di->msi_xref);
2305 err = intr_pic_add_handler(device_get_parent(dev), sc->sc_pic,
2306 gicv3_its_intr, sc, sc->sc_irq_base, sc->sc_irq_length);
2307 if (err != 0) {
2308 device_printf(dev, "Failed to add PIC handler: %d\n", err);
2309 return (err);
2310 }
2311
2312 /* Register this device to handle MSI interrupts */
2313 err = intr_msi_register(dev, di->msi_xref);
2314 if (err != 0) {
2315 device_printf(dev, "Failed to register for MSIs: %d\n", err);
2316 return (err);
2317 }
2318
2319 return (0);
2320 }
2321 #endif
2322