1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * Copyright (C) 2013-2017 ARM Limited, All Rights Reserved. 4 * Author: Marc Zyngier <marc.zyngier@arm.com> 5 */ 6 7 #define pr_fmt(fmt) "GICv3: " fmt 8 9 #include <linux/acpi.h> 10 #include <linux/cpu.h> 11 #include <linux/cpu_pm.h> 12 #include <linux/delay.h> 13 #include <linux/interrupt.h> 14 #include <linux/irqdomain.h> 15 #include <linux/of.h> 16 #include <linux/of_address.h> 17 #include <linux/of_irq.h> 18 #include <linux/percpu.h> 19 #include <linux/refcount.h> 20 #include <linux/slab.h> 21 22 #include <linux/irqchip.h> 23 #include <linux/irqchip/arm-gic-common.h> 24 #include <linux/irqchip/arm-gic-v3.h> 25 #include <linux/irqchip/irq-partition-percpu.h> 26 27 #include <asm/cputype.h> 28 #include <asm/exception.h> 29 #include <asm/smp_plat.h> 30 #include <asm/virt.h> 31 32 #include "irq-gic-common.h" 33 34 #define GICD_INT_NMI_PRI (GICD_INT_DEF_PRI & ~0x80) 35 36 #define FLAGS_WORKAROUND_GICR_WAKER_MSM8996 (1ULL << 0) 37 38 struct redist_region { 39 void __iomem *redist_base; 40 phys_addr_t phys_base; 41 bool single_redist; 42 }; 43 44 struct gic_chip_data { 45 struct fwnode_handle *fwnode; 46 void __iomem *dist_base; 47 struct redist_region *redist_regions; 48 struct rdists rdists; 49 struct irq_domain *domain; 50 u64 redist_stride; 51 u32 nr_redist_regions; 52 u64 flags; 53 bool has_rss; 54 unsigned int irq_nr; 55 struct partition_desc *ppi_descs[16]; 56 }; 57 58 static struct gic_chip_data gic_data __read_mostly; 59 static DEFINE_STATIC_KEY_TRUE(supports_deactivate_key); 60 61 /* 62 * The behaviours of RPR and PMR registers differ depending on the value of 63 * SCR_EL3.FIQ, and the behaviour of non-secure priority registers of the 64 * distributor and redistributors depends on whether security is enabled in the 65 * GIC. 66 * 67 * When security is enabled, non-secure priority values from the (re)distributor 68 * are presented to the GIC CPUIF as follow: 69 * (GIC_(R)DIST_PRI[irq] >> 1) | 0x80; 70 * 71 * If SCR_EL3.FIQ == 1, the values writen to/read from PMR and RPR at non-secure 72 * EL1 are subject to a similar operation thus matching the priorities presented 73 * from the (re)distributor when security is enabled. 74 * 75 * see GICv3/GICv4 Architecture Specification (IHI0069D): 76 * - section 4.8.1 Non-secure accesses to register fields for Secure interrupt 77 * priorities. 78 * - Figure 4-7 Secure read of the priority field for a Non-secure Group 1 79 * interrupt. 80 * 81 * For now, we only support pseudo-NMIs if we have non-secure view of 82 * priorities. 83 */ 84 static DEFINE_STATIC_KEY_FALSE(supports_pseudo_nmis); 85 86 /* ppi_nmi_refs[n] == number of cpus having ppi[n + 16] set as NMI */ 87 static refcount_t ppi_nmi_refs[16]; 88 89 static struct gic_kvm_info gic_v3_kvm_info; 90 static DEFINE_PER_CPU(bool, has_rss); 91 92 #define MPIDR_RS(mpidr) (((mpidr) & 0xF0UL) >> 4) 93 #define gic_data_rdist() (this_cpu_ptr(gic_data.rdists.rdist)) 94 #define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base) 95 #define gic_data_rdist_sgi_base() (gic_data_rdist_rd_base() + SZ_64K) 96 97 /* Our default, arbitrary priority value. Linux only uses one anyway. */ 98 #define DEFAULT_PMR_VALUE 0xf0 99 100 static inline unsigned int gic_irq(struct irq_data *d) 101 { 102 return d->hwirq; 103 } 104 105 static inline int gic_irq_in_rdist(struct irq_data *d) 106 { 107 return gic_irq(d) < 32; 108 } 109 110 static inline void __iomem *gic_dist_base(struct irq_data *d) 111 { 112 if (gic_irq_in_rdist(d)) /* SGI+PPI -> SGI_base for this CPU */ 113 return gic_data_rdist_sgi_base(); 114 115 if (d->hwirq <= 1023) /* SPI -> dist_base */ 116 return gic_data.dist_base; 117 118 return NULL; 119 } 120 121 static void gic_do_wait_for_rwp(void __iomem *base) 122 { 123 u32 count = 1000000; /* 1s! */ 124 125 while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) { 126 count--; 127 if (!count) { 128 pr_err_ratelimited("RWP timeout, gone fishing\n"); 129 return; 130 } 131 cpu_relax(); 132 udelay(1); 133 }; 134 } 135 136 /* Wait for completion of a distributor change */ 137 static void gic_dist_wait_for_rwp(void) 138 { 139 gic_do_wait_for_rwp(gic_data.dist_base); 140 } 141 142 /* Wait for completion of a redistributor change */ 143 static void gic_redist_wait_for_rwp(void) 144 { 145 gic_do_wait_for_rwp(gic_data_rdist_rd_base()); 146 } 147 148 #ifdef CONFIG_ARM64 149 150 static u64 __maybe_unused gic_read_iar(void) 151 { 152 if (cpus_have_const_cap(ARM64_WORKAROUND_CAVIUM_23154)) 153 return gic_read_iar_cavium_thunderx(); 154 else 155 return gic_read_iar_common(); 156 } 157 #endif 158 159 static void gic_enable_redist(bool enable) 160 { 161 void __iomem *rbase; 162 u32 count = 1000000; /* 1s! */ 163 u32 val; 164 165 if (gic_data.flags & FLAGS_WORKAROUND_GICR_WAKER_MSM8996) 166 return; 167 168 rbase = gic_data_rdist_rd_base(); 169 170 val = readl_relaxed(rbase + GICR_WAKER); 171 if (enable) 172 /* Wake up this CPU redistributor */ 173 val &= ~GICR_WAKER_ProcessorSleep; 174 else 175 val |= GICR_WAKER_ProcessorSleep; 176 writel_relaxed(val, rbase + GICR_WAKER); 177 178 if (!enable) { /* Check that GICR_WAKER is writeable */ 179 val = readl_relaxed(rbase + GICR_WAKER); 180 if (!(val & GICR_WAKER_ProcessorSleep)) 181 return; /* No PM support in this redistributor */ 182 } 183 184 while (--count) { 185 val = readl_relaxed(rbase + GICR_WAKER); 186 if (enable ^ (bool)(val & GICR_WAKER_ChildrenAsleep)) 187 break; 188 cpu_relax(); 189 udelay(1); 190 }; 191 if (!count) 192 pr_err_ratelimited("redistributor failed to %s...\n", 193 enable ? "wakeup" : "sleep"); 194 } 195 196 /* 197 * Routines to disable, enable, EOI and route interrupts 198 */ 199 static int gic_peek_irq(struct irq_data *d, u32 offset) 200 { 201 u32 mask = 1 << (gic_irq(d) % 32); 202 void __iomem *base; 203 204 if (gic_irq_in_rdist(d)) 205 base = gic_data_rdist_sgi_base(); 206 else 207 base = gic_data.dist_base; 208 209 return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask); 210 } 211 212 static void gic_poke_irq(struct irq_data *d, u32 offset) 213 { 214 u32 mask = 1 << (gic_irq(d) % 32); 215 void (*rwp_wait)(void); 216 void __iomem *base; 217 218 if (gic_irq_in_rdist(d)) { 219 base = gic_data_rdist_sgi_base(); 220 rwp_wait = gic_redist_wait_for_rwp; 221 } else { 222 base = gic_data.dist_base; 223 rwp_wait = gic_dist_wait_for_rwp; 224 } 225 226 writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4); 227 rwp_wait(); 228 } 229 230 static void gic_mask_irq(struct irq_data *d) 231 { 232 gic_poke_irq(d, GICD_ICENABLER); 233 } 234 235 static void gic_eoimode1_mask_irq(struct irq_data *d) 236 { 237 gic_mask_irq(d); 238 /* 239 * When masking a forwarded interrupt, make sure it is 240 * deactivated as well. 241 * 242 * This ensures that an interrupt that is getting 243 * disabled/masked will not get "stuck", because there is 244 * noone to deactivate it (guest is being terminated). 245 */ 246 if (irqd_is_forwarded_to_vcpu(d)) 247 gic_poke_irq(d, GICD_ICACTIVER); 248 } 249 250 static void gic_unmask_irq(struct irq_data *d) 251 { 252 gic_poke_irq(d, GICD_ISENABLER); 253 } 254 255 static inline bool gic_supports_nmi(void) 256 { 257 return IS_ENABLED(CONFIG_ARM64_PSEUDO_NMI) && 258 static_branch_likely(&supports_pseudo_nmis); 259 } 260 261 static int gic_irq_set_irqchip_state(struct irq_data *d, 262 enum irqchip_irq_state which, bool val) 263 { 264 u32 reg; 265 266 if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */ 267 return -EINVAL; 268 269 switch (which) { 270 case IRQCHIP_STATE_PENDING: 271 reg = val ? GICD_ISPENDR : GICD_ICPENDR; 272 break; 273 274 case IRQCHIP_STATE_ACTIVE: 275 reg = val ? GICD_ISACTIVER : GICD_ICACTIVER; 276 break; 277 278 case IRQCHIP_STATE_MASKED: 279 reg = val ? GICD_ICENABLER : GICD_ISENABLER; 280 break; 281 282 default: 283 return -EINVAL; 284 } 285 286 gic_poke_irq(d, reg); 287 return 0; 288 } 289 290 static int gic_irq_get_irqchip_state(struct irq_data *d, 291 enum irqchip_irq_state which, bool *val) 292 { 293 if (d->hwirq >= gic_data.irq_nr) /* PPI/SPI only */ 294 return -EINVAL; 295 296 switch (which) { 297 case IRQCHIP_STATE_PENDING: 298 *val = gic_peek_irq(d, GICD_ISPENDR); 299 break; 300 301 case IRQCHIP_STATE_ACTIVE: 302 *val = gic_peek_irq(d, GICD_ISACTIVER); 303 break; 304 305 case IRQCHIP_STATE_MASKED: 306 *val = !gic_peek_irq(d, GICD_ISENABLER); 307 break; 308 309 default: 310 return -EINVAL; 311 } 312 313 return 0; 314 } 315 316 static void gic_irq_set_prio(struct irq_data *d, u8 prio) 317 { 318 void __iomem *base = gic_dist_base(d); 319 320 writeb_relaxed(prio, base + GICD_IPRIORITYR + gic_irq(d)); 321 } 322 323 static int gic_irq_nmi_setup(struct irq_data *d) 324 { 325 struct irq_desc *desc = irq_to_desc(d->irq); 326 327 if (!gic_supports_nmi()) 328 return -EINVAL; 329 330 if (gic_peek_irq(d, GICD_ISENABLER)) { 331 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 332 return -EINVAL; 333 } 334 335 /* 336 * A secondary irq_chip should be in charge of LPI request, 337 * it should not be possible to get there 338 */ 339 if (WARN_ON(gic_irq(d) >= 8192)) 340 return -EINVAL; 341 342 /* desc lock should already be held */ 343 if (gic_irq(d) < 32) { 344 /* Setting up PPI as NMI, only switch handler for first NMI */ 345 if (!refcount_inc_not_zero(&ppi_nmi_refs[gic_irq(d) - 16])) { 346 refcount_set(&ppi_nmi_refs[gic_irq(d) - 16], 1); 347 desc->handle_irq = handle_percpu_devid_fasteoi_nmi; 348 } 349 } else { 350 desc->handle_irq = handle_fasteoi_nmi; 351 } 352 353 gic_irq_set_prio(d, GICD_INT_NMI_PRI); 354 355 return 0; 356 } 357 358 static void gic_irq_nmi_teardown(struct irq_data *d) 359 { 360 struct irq_desc *desc = irq_to_desc(d->irq); 361 362 if (WARN_ON(!gic_supports_nmi())) 363 return; 364 365 if (gic_peek_irq(d, GICD_ISENABLER)) { 366 pr_err("Cannot set NMI property of enabled IRQ %u\n", d->irq); 367 return; 368 } 369 370 /* 371 * A secondary irq_chip should be in charge of LPI request, 372 * it should not be possible to get there 373 */ 374 if (WARN_ON(gic_irq(d) >= 8192)) 375 return; 376 377 /* desc lock should already be held */ 378 if (gic_irq(d) < 32) { 379 /* Tearing down NMI, only switch handler for last NMI */ 380 if (refcount_dec_and_test(&ppi_nmi_refs[gic_irq(d) - 16])) 381 desc->handle_irq = handle_percpu_devid_irq; 382 } else { 383 desc->handle_irq = handle_fasteoi_irq; 384 } 385 386 gic_irq_set_prio(d, GICD_INT_DEF_PRI); 387 } 388 389 static void gic_eoi_irq(struct irq_data *d) 390 { 391 gic_write_eoir(gic_irq(d)); 392 } 393 394 static void gic_eoimode1_eoi_irq(struct irq_data *d) 395 { 396 /* 397 * No need to deactivate an LPI, or an interrupt that 398 * is is getting forwarded to a vcpu. 399 */ 400 if (gic_irq(d) >= 8192 || irqd_is_forwarded_to_vcpu(d)) 401 return; 402 gic_write_dir(gic_irq(d)); 403 } 404 405 static int gic_set_type(struct irq_data *d, unsigned int type) 406 { 407 unsigned int irq = gic_irq(d); 408 void (*rwp_wait)(void); 409 void __iomem *base; 410 411 /* Interrupt configuration for SGIs can't be changed */ 412 if (irq < 16) 413 return -EINVAL; 414 415 /* SPIs have restrictions on the supported types */ 416 if (irq >= 32 && type != IRQ_TYPE_LEVEL_HIGH && 417 type != IRQ_TYPE_EDGE_RISING) 418 return -EINVAL; 419 420 if (gic_irq_in_rdist(d)) { 421 base = gic_data_rdist_sgi_base(); 422 rwp_wait = gic_redist_wait_for_rwp; 423 } else { 424 base = gic_data.dist_base; 425 rwp_wait = gic_dist_wait_for_rwp; 426 } 427 428 return gic_configure_irq(irq, type, base, rwp_wait); 429 } 430 431 static int gic_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu) 432 { 433 if (vcpu) 434 irqd_set_forwarded_to_vcpu(d); 435 else 436 irqd_clr_forwarded_to_vcpu(d); 437 return 0; 438 } 439 440 static u64 gic_mpidr_to_affinity(unsigned long mpidr) 441 { 442 u64 aff; 443 444 aff = ((u64)MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 | 445 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 446 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 447 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 448 449 return aff; 450 } 451 452 static void gic_deactivate_unhandled(u32 irqnr) 453 { 454 if (static_branch_likely(&supports_deactivate_key)) { 455 if (irqnr < 8192) 456 gic_write_dir(irqnr); 457 } else { 458 gic_write_eoir(irqnr); 459 } 460 } 461 462 static inline void gic_handle_nmi(u32 irqnr, struct pt_regs *regs) 463 { 464 int err; 465 466 if (static_branch_likely(&supports_deactivate_key)) 467 gic_write_eoir(irqnr); 468 /* 469 * Leave the PSR.I bit set to prevent other NMIs to be 470 * received while handling this one. 471 * PSR.I will be restored when we ERET to the 472 * interrupted context. 473 */ 474 err = handle_domain_nmi(gic_data.domain, irqnr, regs); 475 if (err) 476 gic_deactivate_unhandled(irqnr); 477 } 478 479 static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs) 480 { 481 u32 irqnr; 482 483 irqnr = gic_read_iar(); 484 485 if (gic_supports_nmi() && 486 unlikely(gic_read_rpr() == GICD_INT_NMI_PRI)) { 487 gic_handle_nmi(irqnr, regs); 488 return; 489 } 490 491 if (gic_prio_masking_enabled()) { 492 gic_pmr_mask_irqs(); 493 gic_arch_enable_irqs(); 494 } 495 496 if (likely(irqnr > 15 && irqnr < 1020) || irqnr >= 8192) { 497 int err; 498 499 if (static_branch_likely(&supports_deactivate_key)) 500 gic_write_eoir(irqnr); 501 else 502 isb(); 503 504 err = handle_domain_irq(gic_data.domain, irqnr, regs); 505 if (err) { 506 WARN_ONCE(true, "Unexpected interrupt received!\n"); 507 gic_deactivate_unhandled(irqnr); 508 } 509 return; 510 } 511 if (irqnr < 16) { 512 gic_write_eoir(irqnr); 513 if (static_branch_likely(&supports_deactivate_key)) 514 gic_write_dir(irqnr); 515 #ifdef CONFIG_SMP 516 /* 517 * Unlike GICv2, we don't need an smp_rmb() here. 518 * The control dependency from gic_read_iar to 519 * the ISB in gic_write_eoir is enough to ensure 520 * that any shared data read by handle_IPI will 521 * be read after the ACK. 522 */ 523 handle_IPI(irqnr, regs); 524 #else 525 WARN_ONCE(true, "Unexpected SGI received!\n"); 526 #endif 527 } 528 } 529 530 static u32 gic_get_pribits(void) 531 { 532 u32 pribits; 533 534 pribits = gic_read_ctlr(); 535 pribits &= ICC_CTLR_EL1_PRI_BITS_MASK; 536 pribits >>= ICC_CTLR_EL1_PRI_BITS_SHIFT; 537 pribits++; 538 539 return pribits; 540 } 541 542 static bool gic_has_group0(void) 543 { 544 u32 val; 545 u32 old_pmr; 546 547 old_pmr = gic_read_pmr(); 548 549 /* 550 * Let's find out if Group0 is under control of EL3 or not by 551 * setting the highest possible, non-zero priority in PMR. 552 * 553 * If SCR_EL3.FIQ is set, the priority gets shifted down in 554 * order for the CPU interface to set bit 7, and keep the 555 * actual priority in the non-secure range. In the process, it 556 * looses the least significant bit and the actual priority 557 * becomes 0x80. Reading it back returns 0, indicating that 558 * we're don't have access to Group0. 559 */ 560 gic_write_pmr(BIT(8 - gic_get_pribits())); 561 val = gic_read_pmr(); 562 563 gic_write_pmr(old_pmr); 564 565 return val != 0; 566 } 567 568 static void __init gic_dist_init(void) 569 { 570 unsigned int i; 571 u64 affinity; 572 void __iomem *base = gic_data.dist_base; 573 574 /* Disable the distributor */ 575 writel_relaxed(0, base + GICD_CTLR); 576 gic_dist_wait_for_rwp(); 577 578 /* 579 * Configure SPIs as non-secure Group-1. This will only matter 580 * if the GIC only has a single security state. This will not 581 * do the right thing if the kernel is running in secure mode, 582 * but that's not the intended use case anyway. 583 */ 584 for (i = 32; i < gic_data.irq_nr; i += 32) 585 writel_relaxed(~0, base + GICD_IGROUPR + i / 8); 586 587 gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp); 588 589 /* Enable distributor with ARE, Group1 */ 590 writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1, 591 base + GICD_CTLR); 592 593 /* 594 * Set all global interrupts to the boot CPU only. ARE must be 595 * enabled. 596 */ 597 affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id())); 598 for (i = 32; i < gic_data.irq_nr; i++) 599 gic_write_irouter(affinity, base + GICD_IROUTER + i * 8); 600 } 601 602 static int gic_iterate_rdists(int (*fn)(struct redist_region *, void __iomem *)) 603 { 604 int ret = -ENODEV; 605 int i; 606 607 for (i = 0; i < gic_data.nr_redist_regions; i++) { 608 void __iomem *ptr = gic_data.redist_regions[i].redist_base; 609 u64 typer; 610 u32 reg; 611 612 reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK; 613 if (reg != GIC_PIDR2_ARCH_GICv3 && 614 reg != GIC_PIDR2_ARCH_GICv4) { /* We're in trouble... */ 615 pr_warn("No redistributor present @%p\n", ptr); 616 break; 617 } 618 619 do { 620 typer = gic_read_typer(ptr + GICR_TYPER); 621 ret = fn(gic_data.redist_regions + i, ptr); 622 if (!ret) 623 return 0; 624 625 if (gic_data.redist_regions[i].single_redist) 626 break; 627 628 if (gic_data.redist_stride) { 629 ptr += gic_data.redist_stride; 630 } else { 631 ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */ 632 if (typer & GICR_TYPER_VLPIS) 633 ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */ 634 } 635 } while (!(typer & GICR_TYPER_LAST)); 636 } 637 638 return ret ? -ENODEV : 0; 639 } 640 641 static int __gic_populate_rdist(struct redist_region *region, void __iomem *ptr) 642 { 643 unsigned long mpidr = cpu_logical_map(smp_processor_id()); 644 u64 typer; 645 u32 aff; 646 647 /* 648 * Convert affinity to a 32bit value that can be matched to 649 * GICR_TYPER bits [63:32]. 650 */ 651 aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 | 652 MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 | 653 MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 | 654 MPIDR_AFFINITY_LEVEL(mpidr, 0)); 655 656 typer = gic_read_typer(ptr + GICR_TYPER); 657 if ((typer >> 32) == aff) { 658 u64 offset = ptr - region->redist_base; 659 gic_data_rdist_rd_base() = ptr; 660 gic_data_rdist()->phys_base = region->phys_base + offset; 661 662 pr_info("CPU%d: found redistributor %lx region %d:%pa\n", 663 smp_processor_id(), mpidr, 664 (int)(region - gic_data.redist_regions), 665 &gic_data_rdist()->phys_base); 666 return 0; 667 } 668 669 /* Try next one */ 670 return 1; 671 } 672 673 static int gic_populate_rdist(void) 674 { 675 if (gic_iterate_rdists(__gic_populate_rdist) == 0) 676 return 0; 677 678 /* We couldn't even deal with ourselves... */ 679 WARN(true, "CPU%d: mpidr %lx has no re-distributor!\n", 680 smp_processor_id(), 681 (unsigned long)cpu_logical_map(smp_processor_id())); 682 return -ENODEV; 683 } 684 685 static int __gic_update_vlpi_properties(struct redist_region *region, 686 void __iomem *ptr) 687 { 688 u64 typer = gic_read_typer(ptr + GICR_TYPER); 689 gic_data.rdists.has_vlpis &= !!(typer & GICR_TYPER_VLPIS); 690 gic_data.rdists.has_direct_lpi &= !!(typer & GICR_TYPER_DirectLPIS); 691 692 return 1; 693 } 694 695 static void gic_update_vlpi_properties(void) 696 { 697 gic_iterate_rdists(__gic_update_vlpi_properties); 698 pr_info("%sVLPI support, %sdirect LPI support\n", 699 !gic_data.rdists.has_vlpis ? "no " : "", 700 !gic_data.rdists.has_direct_lpi ? "no " : ""); 701 } 702 703 /* Check whether it's single security state view */ 704 static inline bool gic_dist_security_disabled(void) 705 { 706 return readl_relaxed(gic_data.dist_base + GICD_CTLR) & GICD_CTLR_DS; 707 } 708 709 static void gic_cpu_sys_reg_init(void) 710 { 711 int i, cpu = smp_processor_id(); 712 u64 mpidr = cpu_logical_map(cpu); 713 u64 need_rss = MPIDR_RS(mpidr); 714 bool group0; 715 u32 pribits; 716 717 /* 718 * Need to check that the SRE bit has actually been set. If 719 * not, it means that SRE is disabled at EL2. We're going to 720 * die painfully, and there is nothing we can do about it. 721 * 722 * Kindly inform the luser. 723 */ 724 if (!gic_enable_sre()) 725 pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n"); 726 727 pribits = gic_get_pribits(); 728 729 group0 = gic_has_group0(); 730 731 /* Set priority mask register */ 732 if (!gic_prio_masking_enabled()) { 733 write_gicreg(DEFAULT_PMR_VALUE, ICC_PMR_EL1); 734 } else { 735 /* 736 * Mismatch configuration with boot CPU, the system is likely 737 * to die as interrupt masking will not work properly on all 738 * CPUs 739 */ 740 WARN_ON(gic_supports_nmi() && group0 && 741 !gic_dist_security_disabled()); 742 } 743 744 /* 745 * Some firmwares hand over to the kernel with the BPR changed from 746 * its reset value (and with a value large enough to prevent 747 * any pre-emptive interrupts from working at all). Writing a zero 748 * to BPR restores is reset value. 749 */ 750 gic_write_bpr1(0); 751 752 if (static_branch_likely(&supports_deactivate_key)) { 753 /* EOI drops priority only (mode 1) */ 754 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop); 755 } else { 756 /* EOI deactivates interrupt too (mode 0) */ 757 gic_write_ctlr(ICC_CTLR_EL1_EOImode_drop_dir); 758 } 759 760 /* Always whack Group0 before Group1 */ 761 if (group0) { 762 switch(pribits) { 763 case 8: 764 case 7: 765 write_gicreg(0, ICC_AP0R3_EL1); 766 write_gicreg(0, ICC_AP0R2_EL1); 767 case 6: 768 write_gicreg(0, ICC_AP0R1_EL1); 769 case 5: 770 case 4: 771 write_gicreg(0, ICC_AP0R0_EL1); 772 } 773 774 isb(); 775 } 776 777 switch(pribits) { 778 case 8: 779 case 7: 780 write_gicreg(0, ICC_AP1R3_EL1); 781 write_gicreg(0, ICC_AP1R2_EL1); 782 case 6: 783 write_gicreg(0, ICC_AP1R1_EL1); 784 case 5: 785 case 4: 786 write_gicreg(0, ICC_AP1R0_EL1); 787 } 788 789 isb(); 790 791 /* ... and let's hit the road... */ 792 gic_write_grpen1(1); 793 794 /* Keep the RSS capability status in per_cpu variable */ 795 per_cpu(has_rss, cpu) = !!(gic_read_ctlr() & ICC_CTLR_EL1_RSS); 796 797 /* Check all the CPUs have capable of sending SGIs to other CPUs */ 798 for_each_online_cpu(i) { 799 bool have_rss = per_cpu(has_rss, i) && per_cpu(has_rss, cpu); 800 801 need_rss |= MPIDR_RS(cpu_logical_map(i)); 802 if (need_rss && (!have_rss)) 803 pr_crit("CPU%d (%lx) can't SGI CPU%d (%lx), no RSS\n", 804 cpu, (unsigned long)mpidr, 805 i, (unsigned long)cpu_logical_map(i)); 806 } 807 808 /** 809 * GIC spec says, when ICC_CTLR_EL1.RSS==1 and GICD_TYPER.RSS==0, 810 * writing ICC_ASGI1R_EL1 register with RS != 0 is a CONSTRAINED 811 * UNPREDICTABLE choice of : 812 * - The write is ignored. 813 * - The RS field is treated as 0. 814 */ 815 if (need_rss && (!gic_data.has_rss)) 816 pr_crit_once("RSS is required but GICD doesn't support it\n"); 817 } 818 819 static bool gicv3_nolpi; 820 821 static int __init gicv3_nolpi_cfg(char *buf) 822 { 823 return strtobool(buf, &gicv3_nolpi); 824 } 825 early_param("irqchip.gicv3_nolpi", gicv3_nolpi_cfg); 826 827 static int gic_dist_supports_lpis(void) 828 { 829 return (IS_ENABLED(CONFIG_ARM_GIC_V3_ITS) && 830 !!(readl_relaxed(gic_data.dist_base + GICD_TYPER) & GICD_TYPER_LPIS) && 831 !gicv3_nolpi); 832 } 833 834 static void gic_cpu_init(void) 835 { 836 void __iomem *rbase; 837 838 /* Register ourselves with the rest of the world */ 839 if (gic_populate_rdist()) 840 return; 841 842 gic_enable_redist(true); 843 844 rbase = gic_data_rdist_sgi_base(); 845 846 /* Configure SGIs/PPIs as non-secure Group-1 */ 847 writel_relaxed(~0, rbase + GICR_IGROUPR0); 848 849 gic_cpu_config(rbase, gic_redist_wait_for_rwp); 850 851 /* initialise system registers */ 852 gic_cpu_sys_reg_init(); 853 } 854 855 #ifdef CONFIG_SMP 856 857 #define MPIDR_TO_SGI_RS(mpidr) (MPIDR_RS(mpidr) << ICC_SGI1R_RS_SHIFT) 858 #define MPIDR_TO_SGI_CLUSTER_ID(mpidr) ((mpidr) & ~0xFUL) 859 860 static int gic_starting_cpu(unsigned int cpu) 861 { 862 gic_cpu_init(); 863 864 if (gic_dist_supports_lpis()) 865 its_cpu_init(); 866 867 return 0; 868 } 869 870 static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask, 871 unsigned long cluster_id) 872 { 873 int next_cpu, cpu = *base_cpu; 874 unsigned long mpidr = cpu_logical_map(cpu); 875 u16 tlist = 0; 876 877 while (cpu < nr_cpu_ids) { 878 tlist |= 1 << (mpidr & 0xf); 879 880 next_cpu = cpumask_next(cpu, mask); 881 if (next_cpu >= nr_cpu_ids) 882 goto out; 883 cpu = next_cpu; 884 885 mpidr = cpu_logical_map(cpu); 886 887 if (cluster_id != MPIDR_TO_SGI_CLUSTER_ID(mpidr)) { 888 cpu--; 889 goto out; 890 } 891 } 892 out: 893 *base_cpu = cpu; 894 return tlist; 895 } 896 897 #define MPIDR_TO_SGI_AFFINITY(cluster_id, level) \ 898 (MPIDR_AFFINITY_LEVEL(cluster_id, level) \ 899 << ICC_SGI1R_AFFINITY_## level ##_SHIFT) 900 901 static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq) 902 { 903 u64 val; 904 905 val = (MPIDR_TO_SGI_AFFINITY(cluster_id, 3) | 906 MPIDR_TO_SGI_AFFINITY(cluster_id, 2) | 907 irq << ICC_SGI1R_SGI_ID_SHIFT | 908 MPIDR_TO_SGI_AFFINITY(cluster_id, 1) | 909 MPIDR_TO_SGI_RS(cluster_id) | 910 tlist << ICC_SGI1R_TARGET_LIST_SHIFT); 911 912 pr_devel("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val); 913 gic_write_sgi1r(val); 914 } 915 916 static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq) 917 { 918 int cpu; 919 920 if (WARN_ON(irq >= 16)) 921 return; 922 923 /* 924 * Ensure that stores to Normal memory are visible to the 925 * other CPUs before issuing the IPI. 926 */ 927 wmb(); 928 929 for_each_cpu(cpu, mask) { 930 u64 cluster_id = MPIDR_TO_SGI_CLUSTER_ID(cpu_logical_map(cpu)); 931 u16 tlist; 932 933 tlist = gic_compute_target_list(&cpu, mask, cluster_id); 934 gic_send_sgi(cluster_id, tlist, irq); 935 } 936 937 /* Force the above writes to ICC_SGI1R_EL1 to be executed */ 938 isb(); 939 } 940 941 static void gic_smp_init(void) 942 { 943 set_smp_cross_call(gic_raise_softirq); 944 cpuhp_setup_state_nocalls(CPUHP_AP_IRQ_GIC_STARTING, 945 "irqchip/arm/gicv3:starting", 946 gic_starting_cpu, NULL); 947 } 948 949 static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val, 950 bool force) 951 { 952 unsigned int cpu; 953 void __iomem *reg; 954 int enabled; 955 u64 val; 956 957 if (force) 958 cpu = cpumask_first(mask_val); 959 else 960 cpu = cpumask_any_and(mask_val, cpu_online_mask); 961 962 if (cpu >= nr_cpu_ids) 963 return -EINVAL; 964 965 if (gic_irq_in_rdist(d)) 966 return -EINVAL; 967 968 /* If interrupt was enabled, disable it first */ 969 enabled = gic_peek_irq(d, GICD_ISENABLER); 970 if (enabled) 971 gic_mask_irq(d); 972 973 reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8); 974 val = gic_mpidr_to_affinity(cpu_logical_map(cpu)); 975 976 gic_write_irouter(val, reg); 977 978 /* 979 * If the interrupt was enabled, enabled it again. Otherwise, 980 * just wait for the distributor to have digested our changes. 981 */ 982 if (enabled) 983 gic_unmask_irq(d); 984 else 985 gic_dist_wait_for_rwp(); 986 987 irq_data_update_effective_affinity(d, cpumask_of(cpu)); 988 989 return IRQ_SET_MASK_OK_DONE; 990 } 991 #else 992 #define gic_set_affinity NULL 993 #define gic_smp_init() do { } while(0) 994 #endif 995 996 #ifdef CONFIG_CPU_PM 997 static int gic_cpu_pm_notifier(struct notifier_block *self, 998 unsigned long cmd, void *v) 999 { 1000 if (cmd == CPU_PM_EXIT) { 1001 if (gic_dist_security_disabled()) 1002 gic_enable_redist(true); 1003 gic_cpu_sys_reg_init(); 1004 } else if (cmd == CPU_PM_ENTER && gic_dist_security_disabled()) { 1005 gic_write_grpen1(0); 1006 gic_enable_redist(false); 1007 } 1008 return NOTIFY_OK; 1009 } 1010 1011 static struct notifier_block gic_cpu_pm_notifier_block = { 1012 .notifier_call = gic_cpu_pm_notifier, 1013 }; 1014 1015 static void gic_cpu_pm_init(void) 1016 { 1017 cpu_pm_register_notifier(&gic_cpu_pm_notifier_block); 1018 } 1019 1020 #else 1021 static inline void gic_cpu_pm_init(void) { } 1022 #endif /* CONFIG_CPU_PM */ 1023 1024 static struct irq_chip gic_chip = { 1025 .name = "GICv3", 1026 .irq_mask = gic_mask_irq, 1027 .irq_unmask = gic_unmask_irq, 1028 .irq_eoi = gic_eoi_irq, 1029 .irq_set_type = gic_set_type, 1030 .irq_set_affinity = gic_set_affinity, 1031 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1032 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1033 .irq_nmi_setup = gic_irq_nmi_setup, 1034 .irq_nmi_teardown = gic_irq_nmi_teardown, 1035 .flags = IRQCHIP_SET_TYPE_MASKED | 1036 IRQCHIP_SKIP_SET_WAKE | 1037 IRQCHIP_MASK_ON_SUSPEND, 1038 }; 1039 1040 static struct irq_chip gic_eoimode1_chip = { 1041 .name = "GICv3", 1042 .irq_mask = gic_eoimode1_mask_irq, 1043 .irq_unmask = gic_unmask_irq, 1044 .irq_eoi = gic_eoimode1_eoi_irq, 1045 .irq_set_type = gic_set_type, 1046 .irq_set_affinity = gic_set_affinity, 1047 .irq_get_irqchip_state = gic_irq_get_irqchip_state, 1048 .irq_set_irqchip_state = gic_irq_set_irqchip_state, 1049 .irq_set_vcpu_affinity = gic_irq_set_vcpu_affinity, 1050 .irq_nmi_setup = gic_irq_nmi_setup, 1051 .irq_nmi_teardown = gic_irq_nmi_teardown, 1052 .flags = IRQCHIP_SET_TYPE_MASKED | 1053 IRQCHIP_SKIP_SET_WAKE | 1054 IRQCHIP_MASK_ON_SUSPEND, 1055 }; 1056 1057 #define GIC_ID_NR (1U << GICD_TYPER_ID_BITS(gic_data.rdists.gicd_typer)) 1058 1059 static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq, 1060 irq_hw_number_t hw) 1061 { 1062 struct irq_chip *chip = &gic_chip; 1063 1064 if (static_branch_likely(&supports_deactivate_key)) 1065 chip = &gic_eoimode1_chip; 1066 1067 /* SGIs are private to the core kernel */ 1068 if (hw < 16) 1069 return -EPERM; 1070 /* Nothing here */ 1071 if (hw >= gic_data.irq_nr && hw < 8192) 1072 return -EPERM; 1073 /* Off limits */ 1074 if (hw >= GIC_ID_NR) 1075 return -EPERM; 1076 1077 /* PPIs */ 1078 if (hw < 32) { 1079 irq_set_percpu_devid(irq); 1080 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1081 handle_percpu_devid_irq, NULL, NULL); 1082 irq_set_status_flags(irq, IRQ_NOAUTOEN); 1083 } 1084 /* SPIs */ 1085 if (hw >= 32 && hw < gic_data.irq_nr) { 1086 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1087 handle_fasteoi_irq, NULL, NULL); 1088 irq_set_probe(irq); 1089 irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(irq))); 1090 } 1091 /* LPIs */ 1092 if (hw >= 8192 && hw < GIC_ID_NR) { 1093 if (!gic_dist_supports_lpis()) 1094 return -EPERM; 1095 irq_domain_set_info(d, irq, hw, chip, d->host_data, 1096 handle_fasteoi_irq, NULL, NULL); 1097 } 1098 1099 return 0; 1100 } 1101 1102 #define GIC_IRQ_TYPE_PARTITION (GIC_IRQ_TYPE_LPI + 1) 1103 1104 static int gic_irq_domain_translate(struct irq_domain *d, 1105 struct irq_fwspec *fwspec, 1106 unsigned long *hwirq, 1107 unsigned int *type) 1108 { 1109 if (is_of_node(fwspec->fwnode)) { 1110 if (fwspec->param_count < 3) 1111 return -EINVAL; 1112 1113 switch (fwspec->param[0]) { 1114 case 0: /* SPI */ 1115 *hwirq = fwspec->param[1] + 32; 1116 break; 1117 case 1: /* PPI */ 1118 case GIC_IRQ_TYPE_PARTITION: 1119 *hwirq = fwspec->param[1] + 16; 1120 break; 1121 case GIC_IRQ_TYPE_LPI: /* LPI */ 1122 *hwirq = fwspec->param[1]; 1123 break; 1124 default: 1125 return -EINVAL; 1126 } 1127 1128 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1129 1130 /* 1131 * Make it clear that broken DTs are... broken. 1132 * Partitionned PPIs are an unfortunate exception. 1133 */ 1134 WARN_ON(*type == IRQ_TYPE_NONE && 1135 fwspec->param[0] != GIC_IRQ_TYPE_PARTITION); 1136 return 0; 1137 } 1138 1139 if (is_fwnode_irqchip(fwspec->fwnode)) { 1140 if(fwspec->param_count != 2) 1141 return -EINVAL; 1142 1143 *hwirq = fwspec->param[0]; 1144 *type = fwspec->param[1]; 1145 1146 WARN_ON(*type == IRQ_TYPE_NONE); 1147 return 0; 1148 } 1149 1150 return -EINVAL; 1151 } 1152 1153 static int gic_irq_domain_alloc(struct irq_domain *domain, unsigned int virq, 1154 unsigned int nr_irqs, void *arg) 1155 { 1156 int i, ret; 1157 irq_hw_number_t hwirq; 1158 unsigned int type = IRQ_TYPE_NONE; 1159 struct irq_fwspec *fwspec = arg; 1160 1161 ret = gic_irq_domain_translate(domain, fwspec, &hwirq, &type); 1162 if (ret) 1163 return ret; 1164 1165 for (i = 0; i < nr_irqs; i++) { 1166 ret = gic_irq_domain_map(domain, virq + i, hwirq + i); 1167 if (ret) 1168 return ret; 1169 } 1170 1171 return 0; 1172 } 1173 1174 static void gic_irq_domain_free(struct irq_domain *domain, unsigned int virq, 1175 unsigned int nr_irqs) 1176 { 1177 int i; 1178 1179 for (i = 0; i < nr_irqs; i++) { 1180 struct irq_data *d = irq_domain_get_irq_data(domain, virq + i); 1181 irq_set_handler(virq + i, NULL); 1182 irq_domain_reset_irq_data(d); 1183 } 1184 } 1185 1186 static int gic_irq_domain_select(struct irq_domain *d, 1187 struct irq_fwspec *fwspec, 1188 enum irq_domain_bus_token bus_token) 1189 { 1190 /* Not for us */ 1191 if (fwspec->fwnode != d->fwnode) 1192 return 0; 1193 1194 /* If this is not DT, then we have a single domain */ 1195 if (!is_of_node(fwspec->fwnode)) 1196 return 1; 1197 1198 /* 1199 * If this is a PPI and we have a 4th (non-null) parameter, 1200 * then we need to match the partition domain. 1201 */ 1202 if (fwspec->param_count >= 4 && 1203 fwspec->param[0] == 1 && fwspec->param[3] != 0) 1204 return d == partition_get_domain(gic_data.ppi_descs[fwspec->param[1]]); 1205 1206 return d == gic_data.domain; 1207 } 1208 1209 static const struct irq_domain_ops gic_irq_domain_ops = { 1210 .translate = gic_irq_domain_translate, 1211 .alloc = gic_irq_domain_alloc, 1212 .free = gic_irq_domain_free, 1213 .select = gic_irq_domain_select, 1214 }; 1215 1216 static int partition_domain_translate(struct irq_domain *d, 1217 struct irq_fwspec *fwspec, 1218 unsigned long *hwirq, 1219 unsigned int *type) 1220 { 1221 struct device_node *np; 1222 int ret; 1223 1224 np = of_find_node_by_phandle(fwspec->param[3]); 1225 if (WARN_ON(!np)) 1226 return -EINVAL; 1227 1228 ret = partition_translate_id(gic_data.ppi_descs[fwspec->param[1]], 1229 of_node_to_fwnode(np)); 1230 if (ret < 0) 1231 return ret; 1232 1233 *hwirq = ret; 1234 *type = fwspec->param[2] & IRQ_TYPE_SENSE_MASK; 1235 1236 return 0; 1237 } 1238 1239 static const struct irq_domain_ops partition_domain_ops = { 1240 .translate = partition_domain_translate, 1241 .select = gic_irq_domain_select, 1242 }; 1243 1244 static bool gic_enable_quirk_msm8996(void *data) 1245 { 1246 struct gic_chip_data *d = data; 1247 1248 d->flags |= FLAGS_WORKAROUND_GICR_WAKER_MSM8996; 1249 1250 return true; 1251 } 1252 1253 static void gic_enable_nmi_support(void) 1254 { 1255 int i; 1256 1257 for (i = 0; i < 16; i++) 1258 refcount_set(&ppi_nmi_refs[i], 0); 1259 1260 static_branch_enable(&supports_pseudo_nmis); 1261 1262 if (static_branch_likely(&supports_deactivate_key)) 1263 gic_eoimode1_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1264 else 1265 gic_chip.flags |= IRQCHIP_SUPPORTS_NMI; 1266 } 1267 1268 static int __init gic_init_bases(void __iomem *dist_base, 1269 struct redist_region *rdist_regs, 1270 u32 nr_redist_regions, 1271 u64 redist_stride, 1272 struct fwnode_handle *handle) 1273 { 1274 u32 typer; 1275 int gic_irqs; 1276 int err; 1277 1278 if (!is_hyp_mode_available()) 1279 static_branch_disable(&supports_deactivate_key); 1280 1281 if (static_branch_likely(&supports_deactivate_key)) 1282 pr_info("GIC: Using split EOI/Deactivate mode\n"); 1283 1284 gic_data.fwnode = handle; 1285 gic_data.dist_base = dist_base; 1286 gic_data.redist_regions = rdist_regs; 1287 gic_data.nr_redist_regions = nr_redist_regions; 1288 gic_data.redist_stride = redist_stride; 1289 1290 /* 1291 * Find out how many interrupts are supported. 1292 * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI) 1293 */ 1294 typer = readl_relaxed(gic_data.dist_base + GICD_TYPER); 1295 gic_data.rdists.gicd_typer = typer; 1296 gic_irqs = GICD_TYPER_IRQS(typer); 1297 if (gic_irqs > 1020) 1298 gic_irqs = 1020; 1299 gic_data.irq_nr = gic_irqs; 1300 1301 gic_data.domain = irq_domain_create_tree(handle, &gic_irq_domain_ops, 1302 &gic_data); 1303 irq_domain_update_bus_token(gic_data.domain, DOMAIN_BUS_WIRED); 1304 gic_data.rdists.rdist = alloc_percpu(typeof(*gic_data.rdists.rdist)); 1305 gic_data.rdists.has_vlpis = true; 1306 gic_data.rdists.has_direct_lpi = true; 1307 1308 if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdists.rdist)) { 1309 err = -ENOMEM; 1310 goto out_free; 1311 } 1312 1313 gic_data.has_rss = !!(typer & GICD_TYPER_RSS); 1314 pr_info("Distributor has %sRange Selector support\n", 1315 gic_data.has_rss ? "" : "no "); 1316 1317 if (typer & GICD_TYPER_MBIS) { 1318 err = mbi_init(handle, gic_data.domain); 1319 if (err) 1320 pr_err("Failed to initialize MBIs\n"); 1321 } 1322 1323 set_handle_irq(gic_handle_irq); 1324 1325 gic_update_vlpi_properties(); 1326 1327 gic_smp_init(); 1328 gic_dist_init(); 1329 gic_cpu_init(); 1330 gic_cpu_pm_init(); 1331 1332 if (gic_dist_supports_lpis()) { 1333 its_init(handle, &gic_data.rdists, gic_data.domain); 1334 its_cpu_init(); 1335 } 1336 1337 if (gic_prio_masking_enabled()) { 1338 if (!gic_has_group0() || gic_dist_security_disabled()) 1339 gic_enable_nmi_support(); 1340 else 1341 pr_warn("SCR_EL3.FIQ is cleared, cannot enable use of pseudo-NMIs\n"); 1342 } 1343 1344 return 0; 1345 1346 out_free: 1347 if (gic_data.domain) 1348 irq_domain_remove(gic_data.domain); 1349 free_percpu(gic_data.rdists.rdist); 1350 return err; 1351 } 1352 1353 static int __init gic_validate_dist_version(void __iomem *dist_base) 1354 { 1355 u32 reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1356 1357 if (reg != GIC_PIDR2_ARCH_GICv3 && reg != GIC_PIDR2_ARCH_GICv4) 1358 return -ENODEV; 1359 1360 return 0; 1361 } 1362 1363 /* Create all possible partitions at boot time */ 1364 static void __init gic_populate_ppi_partitions(struct device_node *gic_node) 1365 { 1366 struct device_node *parts_node, *child_part; 1367 int part_idx = 0, i; 1368 int nr_parts; 1369 struct partition_affinity *parts; 1370 1371 parts_node = of_get_child_by_name(gic_node, "ppi-partitions"); 1372 if (!parts_node) 1373 return; 1374 1375 nr_parts = of_get_child_count(parts_node); 1376 1377 if (!nr_parts) 1378 goto out_put_node; 1379 1380 parts = kcalloc(nr_parts, sizeof(*parts), GFP_KERNEL); 1381 if (WARN_ON(!parts)) 1382 goto out_put_node; 1383 1384 for_each_child_of_node(parts_node, child_part) { 1385 struct partition_affinity *part; 1386 int n; 1387 1388 part = &parts[part_idx]; 1389 1390 part->partition_id = of_node_to_fwnode(child_part); 1391 1392 pr_info("GIC: PPI partition %pOFn[%d] { ", 1393 child_part, part_idx); 1394 1395 n = of_property_count_elems_of_size(child_part, "affinity", 1396 sizeof(u32)); 1397 WARN_ON(n <= 0); 1398 1399 for (i = 0; i < n; i++) { 1400 int err, cpu; 1401 u32 cpu_phandle; 1402 struct device_node *cpu_node; 1403 1404 err = of_property_read_u32_index(child_part, "affinity", 1405 i, &cpu_phandle); 1406 if (WARN_ON(err)) 1407 continue; 1408 1409 cpu_node = of_find_node_by_phandle(cpu_phandle); 1410 if (WARN_ON(!cpu_node)) 1411 continue; 1412 1413 cpu = of_cpu_node_to_id(cpu_node); 1414 if (WARN_ON(cpu < 0)) 1415 continue; 1416 1417 pr_cont("%pOF[%d] ", cpu_node, cpu); 1418 1419 cpumask_set_cpu(cpu, &part->mask); 1420 } 1421 1422 pr_cont("}\n"); 1423 part_idx++; 1424 } 1425 1426 for (i = 0; i < 16; i++) { 1427 unsigned int irq; 1428 struct partition_desc *desc; 1429 struct irq_fwspec ppi_fwspec = { 1430 .fwnode = gic_data.fwnode, 1431 .param_count = 3, 1432 .param = { 1433 [0] = GIC_IRQ_TYPE_PARTITION, 1434 [1] = i, 1435 [2] = IRQ_TYPE_NONE, 1436 }, 1437 }; 1438 1439 irq = irq_create_fwspec_mapping(&ppi_fwspec); 1440 if (WARN_ON(!irq)) 1441 continue; 1442 desc = partition_create_desc(gic_data.fwnode, parts, nr_parts, 1443 irq, &partition_domain_ops); 1444 if (WARN_ON(!desc)) 1445 continue; 1446 1447 gic_data.ppi_descs[i] = desc; 1448 } 1449 1450 out_put_node: 1451 of_node_put(parts_node); 1452 } 1453 1454 static void __init gic_of_setup_kvm_info(struct device_node *node) 1455 { 1456 int ret; 1457 struct resource r; 1458 u32 gicv_idx; 1459 1460 gic_v3_kvm_info.type = GIC_V3; 1461 1462 gic_v3_kvm_info.maint_irq = irq_of_parse_and_map(node, 0); 1463 if (!gic_v3_kvm_info.maint_irq) 1464 return; 1465 1466 if (of_property_read_u32(node, "#redistributor-regions", 1467 &gicv_idx)) 1468 gicv_idx = 1; 1469 1470 gicv_idx += 3; /* Also skip GICD, GICC, GICH */ 1471 ret = of_address_to_resource(node, gicv_idx, &r); 1472 if (!ret) 1473 gic_v3_kvm_info.vcpu = r; 1474 1475 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 1476 gic_set_kvm_info(&gic_v3_kvm_info); 1477 } 1478 1479 static const struct gic_quirk gic_quirks[] = { 1480 { 1481 .desc = "GICv3: Qualcomm MSM8996 broken firmware", 1482 .compatible = "qcom,msm8996-gic-v3", 1483 .init = gic_enable_quirk_msm8996, 1484 }, 1485 { 1486 } 1487 }; 1488 1489 static int __init gic_of_init(struct device_node *node, struct device_node *parent) 1490 { 1491 void __iomem *dist_base; 1492 struct redist_region *rdist_regs; 1493 u64 redist_stride; 1494 u32 nr_redist_regions; 1495 int err, i; 1496 1497 dist_base = of_iomap(node, 0); 1498 if (!dist_base) { 1499 pr_err("%pOF: unable to map gic dist registers\n", node); 1500 return -ENXIO; 1501 } 1502 1503 err = gic_validate_dist_version(dist_base); 1504 if (err) { 1505 pr_err("%pOF: no distributor detected, giving up\n", node); 1506 goto out_unmap_dist; 1507 } 1508 1509 if (of_property_read_u32(node, "#redistributor-regions", &nr_redist_regions)) 1510 nr_redist_regions = 1; 1511 1512 rdist_regs = kcalloc(nr_redist_regions, sizeof(*rdist_regs), 1513 GFP_KERNEL); 1514 if (!rdist_regs) { 1515 err = -ENOMEM; 1516 goto out_unmap_dist; 1517 } 1518 1519 for (i = 0; i < nr_redist_regions; i++) { 1520 struct resource res; 1521 int ret; 1522 1523 ret = of_address_to_resource(node, 1 + i, &res); 1524 rdist_regs[i].redist_base = of_iomap(node, 1 + i); 1525 if (ret || !rdist_regs[i].redist_base) { 1526 pr_err("%pOF: couldn't map region %d\n", node, i); 1527 err = -ENODEV; 1528 goto out_unmap_rdist; 1529 } 1530 rdist_regs[i].phys_base = res.start; 1531 } 1532 1533 if (of_property_read_u64(node, "redistributor-stride", &redist_stride)) 1534 redist_stride = 0; 1535 1536 gic_enable_of_quirks(node, gic_quirks, &gic_data); 1537 1538 err = gic_init_bases(dist_base, rdist_regs, nr_redist_regions, 1539 redist_stride, &node->fwnode); 1540 if (err) 1541 goto out_unmap_rdist; 1542 1543 gic_populate_ppi_partitions(node); 1544 1545 if (static_branch_likely(&supports_deactivate_key)) 1546 gic_of_setup_kvm_info(node); 1547 return 0; 1548 1549 out_unmap_rdist: 1550 for (i = 0; i < nr_redist_regions; i++) 1551 if (rdist_regs[i].redist_base) 1552 iounmap(rdist_regs[i].redist_base); 1553 kfree(rdist_regs); 1554 out_unmap_dist: 1555 iounmap(dist_base); 1556 return err; 1557 } 1558 1559 IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init); 1560 1561 #ifdef CONFIG_ACPI 1562 static struct 1563 { 1564 void __iomem *dist_base; 1565 struct redist_region *redist_regs; 1566 u32 nr_redist_regions; 1567 bool single_redist; 1568 u32 maint_irq; 1569 int maint_irq_mode; 1570 phys_addr_t vcpu_base; 1571 } acpi_data __initdata; 1572 1573 static void __init 1574 gic_acpi_register_redist(phys_addr_t phys_base, void __iomem *redist_base) 1575 { 1576 static int count = 0; 1577 1578 acpi_data.redist_regs[count].phys_base = phys_base; 1579 acpi_data.redist_regs[count].redist_base = redist_base; 1580 acpi_data.redist_regs[count].single_redist = acpi_data.single_redist; 1581 count++; 1582 } 1583 1584 static int __init 1585 gic_acpi_parse_madt_redist(union acpi_subtable_headers *header, 1586 const unsigned long end) 1587 { 1588 struct acpi_madt_generic_redistributor *redist = 1589 (struct acpi_madt_generic_redistributor *)header; 1590 void __iomem *redist_base; 1591 1592 redist_base = ioremap(redist->base_address, redist->length); 1593 if (!redist_base) { 1594 pr_err("Couldn't map GICR region @%llx\n", redist->base_address); 1595 return -ENOMEM; 1596 } 1597 1598 gic_acpi_register_redist(redist->base_address, redist_base); 1599 return 0; 1600 } 1601 1602 static int __init 1603 gic_acpi_parse_madt_gicc(union acpi_subtable_headers *header, 1604 const unsigned long end) 1605 { 1606 struct acpi_madt_generic_interrupt *gicc = 1607 (struct acpi_madt_generic_interrupt *)header; 1608 u32 reg = readl_relaxed(acpi_data.dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK; 1609 u32 size = reg == GIC_PIDR2_ARCH_GICv4 ? SZ_64K * 4 : SZ_64K * 2; 1610 void __iomem *redist_base; 1611 1612 /* GICC entry which has !ACPI_MADT_ENABLED is not unusable so skip */ 1613 if (!(gicc->flags & ACPI_MADT_ENABLED)) 1614 return 0; 1615 1616 redist_base = ioremap(gicc->gicr_base_address, size); 1617 if (!redist_base) 1618 return -ENOMEM; 1619 1620 gic_acpi_register_redist(gicc->gicr_base_address, redist_base); 1621 return 0; 1622 } 1623 1624 static int __init gic_acpi_collect_gicr_base(void) 1625 { 1626 acpi_tbl_entry_handler redist_parser; 1627 enum acpi_madt_type type; 1628 1629 if (acpi_data.single_redist) { 1630 type = ACPI_MADT_TYPE_GENERIC_INTERRUPT; 1631 redist_parser = gic_acpi_parse_madt_gicc; 1632 } else { 1633 type = ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR; 1634 redist_parser = gic_acpi_parse_madt_redist; 1635 } 1636 1637 /* Collect redistributor base addresses in GICR entries */ 1638 if (acpi_table_parse_madt(type, redist_parser, 0) > 0) 1639 return 0; 1640 1641 pr_info("No valid GICR entries exist\n"); 1642 return -ENODEV; 1643 } 1644 1645 static int __init gic_acpi_match_gicr(union acpi_subtable_headers *header, 1646 const unsigned long end) 1647 { 1648 /* Subtable presence means that redist exists, that's it */ 1649 return 0; 1650 } 1651 1652 static int __init gic_acpi_match_gicc(union acpi_subtable_headers *header, 1653 const unsigned long end) 1654 { 1655 struct acpi_madt_generic_interrupt *gicc = 1656 (struct acpi_madt_generic_interrupt *)header; 1657 1658 /* 1659 * If GICC is enabled and has valid gicr base address, then it means 1660 * GICR base is presented via GICC 1661 */ 1662 if ((gicc->flags & ACPI_MADT_ENABLED) && gicc->gicr_base_address) 1663 return 0; 1664 1665 /* 1666 * It's perfectly valid firmware can pass disabled GICC entry, driver 1667 * should not treat as errors, skip the entry instead of probe fail. 1668 */ 1669 if (!(gicc->flags & ACPI_MADT_ENABLED)) 1670 return 0; 1671 1672 return -ENODEV; 1673 } 1674 1675 static int __init gic_acpi_count_gicr_regions(void) 1676 { 1677 int count; 1678 1679 /* 1680 * Count how many redistributor regions we have. It is not allowed 1681 * to mix redistributor description, GICR and GICC subtables have to be 1682 * mutually exclusive. 1683 */ 1684 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_REDISTRIBUTOR, 1685 gic_acpi_match_gicr, 0); 1686 if (count > 0) { 1687 acpi_data.single_redist = false; 1688 return count; 1689 } 1690 1691 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 1692 gic_acpi_match_gicc, 0); 1693 if (count > 0) 1694 acpi_data.single_redist = true; 1695 1696 return count; 1697 } 1698 1699 static bool __init acpi_validate_gic_table(struct acpi_subtable_header *header, 1700 struct acpi_probe_entry *ape) 1701 { 1702 struct acpi_madt_generic_distributor *dist; 1703 int count; 1704 1705 dist = (struct acpi_madt_generic_distributor *)header; 1706 if (dist->version != ape->driver_data) 1707 return false; 1708 1709 /* We need to do that exercise anyway, the sooner the better */ 1710 count = gic_acpi_count_gicr_regions(); 1711 if (count <= 0) 1712 return false; 1713 1714 acpi_data.nr_redist_regions = count; 1715 return true; 1716 } 1717 1718 static int __init gic_acpi_parse_virt_madt_gicc(union acpi_subtable_headers *header, 1719 const unsigned long end) 1720 { 1721 struct acpi_madt_generic_interrupt *gicc = 1722 (struct acpi_madt_generic_interrupt *)header; 1723 int maint_irq_mode; 1724 static int first_madt = true; 1725 1726 /* Skip unusable CPUs */ 1727 if (!(gicc->flags & ACPI_MADT_ENABLED)) 1728 return 0; 1729 1730 maint_irq_mode = (gicc->flags & ACPI_MADT_VGIC_IRQ_MODE) ? 1731 ACPI_EDGE_SENSITIVE : ACPI_LEVEL_SENSITIVE; 1732 1733 if (first_madt) { 1734 first_madt = false; 1735 1736 acpi_data.maint_irq = gicc->vgic_interrupt; 1737 acpi_data.maint_irq_mode = maint_irq_mode; 1738 acpi_data.vcpu_base = gicc->gicv_base_address; 1739 1740 return 0; 1741 } 1742 1743 /* 1744 * The maintenance interrupt and GICV should be the same for every CPU 1745 */ 1746 if ((acpi_data.maint_irq != gicc->vgic_interrupt) || 1747 (acpi_data.maint_irq_mode != maint_irq_mode) || 1748 (acpi_data.vcpu_base != gicc->gicv_base_address)) 1749 return -EINVAL; 1750 1751 return 0; 1752 } 1753 1754 static bool __init gic_acpi_collect_virt_info(void) 1755 { 1756 int count; 1757 1758 count = acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_INTERRUPT, 1759 gic_acpi_parse_virt_madt_gicc, 0); 1760 1761 return (count > 0); 1762 } 1763 1764 #define ACPI_GICV3_DIST_MEM_SIZE (SZ_64K) 1765 #define ACPI_GICV2_VCTRL_MEM_SIZE (SZ_4K) 1766 #define ACPI_GICV2_VCPU_MEM_SIZE (SZ_8K) 1767 1768 static void __init gic_acpi_setup_kvm_info(void) 1769 { 1770 int irq; 1771 1772 if (!gic_acpi_collect_virt_info()) { 1773 pr_warn("Unable to get hardware information used for virtualization\n"); 1774 return; 1775 } 1776 1777 gic_v3_kvm_info.type = GIC_V3; 1778 1779 irq = acpi_register_gsi(NULL, acpi_data.maint_irq, 1780 acpi_data.maint_irq_mode, 1781 ACPI_ACTIVE_HIGH); 1782 if (irq <= 0) 1783 return; 1784 1785 gic_v3_kvm_info.maint_irq = irq; 1786 1787 if (acpi_data.vcpu_base) { 1788 struct resource *vcpu = &gic_v3_kvm_info.vcpu; 1789 1790 vcpu->flags = IORESOURCE_MEM; 1791 vcpu->start = acpi_data.vcpu_base; 1792 vcpu->end = vcpu->start + ACPI_GICV2_VCPU_MEM_SIZE - 1; 1793 } 1794 1795 gic_v3_kvm_info.has_v4 = gic_data.rdists.has_vlpis; 1796 gic_set_kvm_info(&gic_v3_kvm_info); 1797 } 1798 1799 static int __init 1800 gic_acpi_init(struct acpi_subtable_header *header, const unsigned long end) 1801 { 1802 struct acpi_madt_generic_distributor *dist; 1803 struct fwnode_handle *domain_handle; 1804 size_t size; 1805 int i, err; 1806 1807 /* Get distributor base address */ 1808 dist = (struct acpi_madt_generic_distributor *)header; 1809 acpi_data.dist_base = ioremap(dist->base_address, 1810 ACPI_GICV3_DIST_MEM_SIZE); 1811 if (!acpi_data.dist_base) { 1812 pr_err("Unable to map GICD registers\n"); 1813 return -ENOMEM; 1814 } 1815 1816 err = gic_validate_dist_version(acpi_data.dist_base); 1817 if (err) { 1818 pr_err("No distributor detected at @%p, giving up\n", 1819 acpi_data.dist_base); 1820 goto out_dist_unmap; 1821 } 1822 1823 size = sizeof(*acpi_data.redist_regs) * acpi_data.nr_redist_regions; 1824 acpi_data.redist_regs = kzalloc(size, GFP_KERNEL); 1825 if (!acpi_data.redist_regs) { 1826 err = -ENOMEM; 1827 goto out_dist_unmap; 1828 } 1829 1830 err = gic_acpi_collect_gicr_base(); 1831 if (err) 1832 goto out_redist_unmap; 1833 1834 domain_handle = irq_domain_alloc_fwnode(acpi_data.dist_base); 1835 if (!domain_handle) { 1836 err = -ENOMEM; 1837 goto out_redist_unmap; 1838 } 1839 1840 err = gic_init_bases(acpi_data.dist_base, acpi_data.redist_regs, 1841 acpi_data.nr_redist_regions, 0, domain_handle); 1842 if (err) 1843 goto out_fwhandle_free; 1844 1845 acpi_set_irq_model(ACPI_IRQ_MODEL_GIC, domain_handle); 1846 1847 if (static_branch_likely(&supports_deactivate_key)) 1848 gic_acpi_setup_kvm_info(); 1849 1850 return 0; 1851 1852 out_fwhandle_free: 1853 irq_domain_free_fwnode(domain_handle); 1854 out_redist_unmap: 1855 for (i = 0; i < acpi_data.nr_redist_regions; i++) 1856 if (acpi_data.redist_regs[i].redist_base) 1857 iounmap(acpi_data.redist_regs[i].redist_base); 1858 kfree(acpi_data.redist_regs); 1859 out_dist_unmap: 1860 iounmap(acpi_data.dist_base); 1861 return err; 1862 } 1863 IRQCHIP_ACPI_DECLARE(gic_v3, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 1864 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V3, 1865 gic_acpi_init); 1866 IRQCHIP_ACPI_DECLARE(gic_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 1867 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_V4, 1868 gic_acpi_init); 1869 IRQCHIP_ACPI_DECLARE(gic_v3_or_v4, ACPI_MADT_TYPE_GENERIC_DISTRIBUTOR, 1870 acpi_validate_gic_table, ACPI_MADT_GIC_VERSION_NONE, 1871 gic_acpi_init); 1872 #endif 1873