1 /*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * 4 * This software was developed by Andrew Turner under 5 * sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30 #include "opt_acpi.h" 31 #include "opt_ddb.h" 32 #include "opt_kstack_pages.h" 33 #include "opt_platform.h" 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/bus.h> 38 #include <sys/cpu.h> 39 #include <sys/csan.h> 40 #include <sys/domainset.h> 41 #include <sys/kernel.h> 42 #include <sys/ktr.h> 43 #include <sys/malloc.h> 44 #include <sys/module.h> 45 #include <sys/mutex.h> 46 #include <sys/pcpu.h> 47 #include <sys/proc.h> 48 #include <sys/sched.h> 49 #include <sys/smp.h> 50 51 #include <vm/vm.h> 52 #include <vm/pmap.h> 53 #include <vm/vm_extern.h> 54 #include <vm/vm_kern.h> 55 #include <vm/vm_map.h> 56 57 #include <machine/machdep.h> 58 #include <machine/cpu.h> 59 #include <machine/debug_monitor.h> 60 #include <machine/intr.h> 61 #include <machine/smp.h> 62 #ifdef VFP 63 #include <machine/vfp.h> 64 #endif 65 66 #ifdef DEV_ACPI 67 #include <contrib/dev/acpica/include/acpi.h> 68 #include <dev/acpica/acpivar.h> 69 #endif 70 71 #ifdef FDT 72 #include <dev/ofw/openfirm.h> 73 #include <dev/ofw/ofw_bus.h> 74 #include <dev/ofw/ofw_bus_subr.h> 75 #include <dev/ofw/ofw_cpu.h> 76 #endif 77 78 #include <dev/psci/psci.h> 79 80 #include "pic_if.h" 81 82 #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE) 83 84 #define MP_QUIRK_CPULIST 0x01 /* The list of cpus may be wrong, */ 85 /* don't panic if one fails to start */ 86 static uint32_t mp_quirks; 87 88 #ifdef FDT 89 static struct { 90 const char *compat; 91 uint32_t quirks; 92 } fdt_quirks[] = { 93 { "arm,foundation-aarch64", MP_QUIRK_CPULIST }, 94 { "arm,fvp-base", MP_QUIRK_CPULIST }, 95 /* This is incorrect in some DTS files */ 96 { "arm,vfp-base", MP_QUIRK_CPULIST }, 97 { NULL, 0 }, 98 }; 99 #endif 100 101 typedef void intr_ipi_send_t(void *, cpuset_t, u_int); 102 typedef void intr_ipi_handler_t(void *); 103 104 #define INTR_IPI_NAMELEN (MAXCOMLEN + 1) 105 struct intr_ipi { 106 intr_ipi_handler_t * ii_handler; 107 void * ii_handler_arg; 108 intr_ipi_send_t * ii_send; 109 void * ii_send_arg; 110 char ii_name[INTR_IPI_NAMELEN]; 111 u_long * ii_count; 112 }; 113 114 static struct intr_ipi ipi_sources[INTR_IPI_COUNT]; 115 116 static struct intr_ipi *intr_ipi_lookup(u_int); 117 static void intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *, 118 void *); 119 120 static void ipi_ast(void *); 121 static void ipi_hardclock(void *); 122 static void ipi_preempt(void *); 123 static void ipi_rendezvous(void *); 124 static void ipi_stop(void *); 125 126 #ifdef FDT 127 static u_int fdt_cpuid; 128 #endif 129 130 void mpentry(unsigned long cpuid); 131 void init_secondary(uint64_t); 132 133 /* Synchronize AP startup. */ 134 static struct mtx ap_boot_mtx; 135 136 /* Used to initialize the PCPU ahead of calling init_secondary(). */ 137 void *bootpcpu; 138 139 /* Stacks for AP initialization, discarded once idle threads are started. */ 140 void *bootstack; 141 static void *bootstacks[MAXCPU]; 142 143 /* Count of started APs, used to synchronize access to bootstack. */ 144 static volatile int aps_started; 145 146 /* Set to 1 once we're ready to let the APs out of the pen. */ 147 static volatile int aps_ready; 148 149 /* Temporary variables for init_secondary() */ 150 static void *dpcpu[MAXCPU - 1]; 151 152 static bool 153 is_boot_cpu(uint64_t target_cpu) 154 { 155 156 return (PCPU_GET_MPIDR(cpuid_to_pcpu[0]) == (target_cpu & CPU_AFF_MASK)); 157 } 158 159 static void 160 release_aps(void *dummy __unused) 161 { 162 int i, started; 163 164 /* Only release CPUs if they exist */ 165 if (mp_ncpus == 1) 166 return; 167 168 intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); 169 intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); 170 intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); 171 intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); 172 intr_pic_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); 173 intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); 174 175 atomic_store_rel_int(&aps_ready, 1); 176 /* Wake up the other CPUs */ 177 __asm __volatile( 178 "dsb ishst \n" 179 "sev \n" 180 ::: "memory"); 181 182 printf("Release APs..."); 183 184 started = 0; 185 for (i = 0; i < 2000; i++) { 186 if (atomic_load_acq_int(&smp_started) != 0) { 187 printf("done\n"); 188 return; 189 } 190 /* 191 * Don't time out while we are making progress. Some large 192 * systems can take a while to start all CPUs. 193 */ 194 if (smp_cpus > started) { 195 i = 0; 196 started = smp_cpus; 197 } 198 DELAY(1000); 199 } 200 201 printf("APs not started\n"); 202 } 203 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 204 205 void 206 init_secondary(uint64_t cpu) 207 { 208 struct pcpu *pcpup; 209 pmap_t pmap0; 210 uint64_t mpidr; 211 212 ptrauth_mp_start(cpu); 213 214 /* 215 * Verify that the value passed in 'cpu' argument (aka context_id) is 216 * valid. Some older U-Boot based PSCI implementations are buggy, 217 * they can pass random value in it. 218 */ 219 mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; 220 if (cpu >= MAXCPU || cpuid_to_pcpu[cpu] == NULL || 221 PCPU_GET_MPIDR(cpuid_to_pcpu[cpu]) != mpidr) { 222 for (cpu = 0; cpu < mp_maxid; cpu++) 223 if (cpuid_to_pcpu[cpu] != NULL && 224 PCPU_GET_MPIDR(cpuid_to_pcpu[cpu]) == mpidr) 225 break; 226 if ( cpu >= MAXCPU) 227 panic("MPIDR for this CPU is not in pcpu table"); 228 } 229 230 /* 231 * Identify current CPU. This is necessary to setup 232 * affinity registers and to provide support for 233 * runtime chip identification. 234 * 235 * We need this before signalling the CPU is ready to 236 * let the boot CPU use the results. 237 */ 238 pcpup = cpuid_to_pcpu[cpu]; 239 pcpup->pc_midr = get_midr(); 240 identify_cpu(cpu); 241 242 /* Ensure the stores in identify_cpu have completed */ 243 atomic_thread_fence_acq_rel(); 244 245 /* Signal the BSP and spin until it has released all APs. */ 246 atomic_add_int(&aps_started, 1); 247 while (!atomic_load_int(&aps_ready)) 248 __asm __volatile("wfe"); 249 250 /* Initialize curthread */ 251 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 252 pcpup->pc_curthread = pcpup->pc_idlethread; 253 schedinit_ap(); 254 255 /* Initialize curpmap to match TTBR0's current setting. */ 256 pmap0 = vmspace_pmap(&vmspace0); 257 KASSERT(pmap_to_ttbr0(pmap0) == READ_SPECIALREG(ttbr0_el1), 258 ("pmap0 doesn't match cpu %ld's ttbr0", cpu)); 259 pcpup->pc_curpmap = pmap0; 260 261 install_cpu_errata(); 262 263 intr_pic_init_secondary(); 264 265 /* Start per-CPU event timers. */ 266 cpu_initclocks_ap(); 267 268 #ifdef VFP 269 vfp_init_secondary(); 270 #endif 271 272 dbg_init(); 273 pan_enable(); 274 275 mtx_lock_spin(&ap_boot_mtx); 276 atomic_add_rel_32(&smp_cpus, 1); 277 if (smp_cpus == mp_ncpus) { 278 /* enable IPI's, tlb shootdown, freezes etc */ 279 atomic_store_rel_int(&smp_started, 1); 280 } 281 mtx_unlock_spin(&ap_boot_mtx); 282 283 kcsan_cpu_init(cpu); 284 285 /* Enter the scheduler */ 286 sched_ap_entry(); 287 288 panic("scheduler returned us to init_secondary"); 289 /* NOTREACHED */ 290 } 291 292 static void 293 smp_after_idle_runnable(void *arg __unused) 294 { 295 int cpu; 296 297 if (mp_ncpus == 1) 298 return; 299 300 KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__)); 301 302 /* 303 * Wait for all APs to handle an interrupt. After that, we know that 304 * the APs have entered the scheduler at least once, so the boot stacks 305 * are safe to free. 306 */ 307 smp_rendezvous(smp_no_rendezvous_barrier, NULL, 308 smp_no_rendezvous_barrier, NULL); 309 310 for (cpu = 1; cpu < mp_ncpus; cpu++) { 311 if (bootstacks[cpu] != NULL) 312 kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE); 313 } 314 } 315 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, 316 smp_after_idle_runnable, NULL); 317 318 /* 319 * Send IPI thru interrupt controller. 320 */ 321 static void 322 pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi) 323 { 324 325 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 326 327 /* 328 * Ensure that this CPU's stores will be visible to IPI 329 * recipients before starting to send the interrupts. 330 */ 331 dsb(ishst); 332 333 PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi); 334 } 335 336 /* 337 * Setup IPI handler on interrupt controller. 338 * 339 * Not SMP coherent. 340 */ 341 static void 342 intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, 343 void *arg) 344 { 345 struct intr_irqsrc *isrc; 346 struct intr_ipi *ii; 347 int error; 348 349 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 350 KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); 351 352 error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc); 353 if (error != 0) 354 return; 355 356 isrc->isrc_handlers++; 357 358 ii = intr_ipi_lookup(ipi); 359 KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); 360 361 ii->ii_handler = hand; 362 ii->ii_handler_arg = arg; 363 ii->ii_send = pic_ipi_send; 364 ii->ii_send_arg = isrc; 365 strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); 366 ii->ii_count = intr_ipi_setup_counters(name); 367 368 PIC_ENABLE_INTR(intr_irq_root_dev, isrc); 369 } 370 371 static void 372 intr_ipi_send(cpuset_t cpus, u_int ipi) 373 { 374 struct intr_ipi *ii; 375 376 ii = intr_ipi_lookup(ipi); 377 if (ii->ii_count == NULL) 378 panic("%s: not setup IPI %u", __func__, ipi); 379 380 ii->ii_send(ii->ii_send_arg, cpus, ipi); 381 } 382 383 static void 384 ipi_ast(void *dummy __unused) 385 { 386 387 CTR0(KTR_SMP, "IPI_AST"); 388 } 389 390 static void 391 ipi_hardclock(void *dummy __unused) 392 { 393 394 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 395 hardclockintr(); 396 } 397 398 static void 399 ipi_preempt(void *dummy __unused) 400 { 401 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 402 sched_preempt(curthread); 403 } 404 405 static void 406 ipi_rendezvous(void *dummy __unused) 407 { 408 409 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 410 smp_rendezvous_action(); 411 } 412 413 static void 414 ipi_stop(void *dummy __unused) 415 { 416 u_int cpu; 417 418 CTR0(KTR_SMP, "IPI_STOP"); 419 420 cpu = PCPU_GET(cpuid); 421 savectx(&stoppcbs[cpu]); 422 423 /* Indicate we are stopped */ 424 CPU_SET_ATOMIC(cpu, &stopped_cpus); 425 426 /* Wait for restart */ 427 while (!CPU_ISSET(cpu, &started_cpus)) 428 cpu_spinwait(); 429 430 #ifdef DDB 431 dbg_register_sync(NULL); 432 #endif 433 434 CPU_CLR_ATOMIC(cpu, &started_cpus); 435 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 436 CTR0(KTR_SMP, "IPI_STOP (restart)"); 437 } 438 439 struct cpu_group * 440 cpu_topo(void) 441 { 442 struct cpu_group *dom, *root; 443 int i; 444 445 root = smp_topo_alloc(1); 446 dom = smp_topo_alloc(vm_ndomains); 447 448 root->cg_parent = NULL; 449 root->cg_child = dom; 450 CPU_COPY(&all_cpus, &root->cg_mask); 451 root->cg_count = mp_ncpus; 452 root->cg_children = vm_ndomains; 453 root->cg_level = CG_SHARE_NONE; 454 root->cg_flags = 0; 455 456 /* 457 * Redundant layers will be collapsed by the caller so we don't need a 458 * special case for a single domain. 459 */ 460 for (i = 0; i < vm_ndomains; i++, dom++) { 461 dom->cg_parent = root; 462 dom->cg_child = NULL; 463 CPU_COPY(&cpuset_domain[i], &dom->cg_mask); 464 dom->cg_count = CPU_COUNT(&dom->cg_mask); 465 dom->cg_children = 0; 466 dom->cg_level = CG_SHARE_L3; 467 dom->cg_flags = 0; 468 } 469 470 return (root); 471 } 472 473 /* Determine if we running MP machine */ 474 int 475 cpu_mp_probe(void) 476 { 477 478 /* ARM64TODO: Read the u bit of mpidr_el1 to determine this */ 479 return (1); 480 } 481 482 static int 483 enable_cpu_psci(uint64_t target_cpu, vm_paddr_t entry, u_int cpuid) 484 { 485 int err; 486 487 err = psci_cpu_on(target_cpu, entry, cpuid); 488 if (err != PSCI_RETVAL_SUCCESS) { 489 /* 490 * Panic here if INVARIANTS are enabled and PSCI failed to 491 * start the requested CPU. psci_cpu_on() returns PSCI_MISSING 492 * to indicate we are unable to use it to start the given CPU. 493 */ 494 KASSERT(err == PSCI_MISSING || 495 (mp_quirks & MP_QUIRK_CPULIST) == MP_QUIRK_CPULIST, 496 ("Failed to start CPU %u (%lx), error %d\n", 497 cpuid, target_cpu, err)); 498 return (EINVAL); 499 } 500 501 return (0); 502 } 503 504 static int 505 enable_cpu_spin(uint64_t cpu, vm_paddr_t entry, vm_paddr_t release_paddr) 506 { 507 vm_paddr_t *release_addr; 508 509 release_addr = pmap_mapdev(release_paddr, sizeof(*release_addr)); 510 if (release_addr == NULL) 511 return (ENOMEM); 512 513 *release_addr = entry; 514 pmap_unmapdev(release_addr, sizeof(*release_addr)); 515 516 __asm __volatile( 517 "dsb sy \n" 518 "sev \n" 519 ::: "memory"); 520 521 return (0); 522 } 523 524 /* 525 * Starts a given CPU. If the CPU is already running, i.e. it is the boot CPU, 526 * do nothing. Returns true if the CPU is present and running. 527 */ 528 static bool 529 start_cpu(u_int cpuid, uint64_t target_cpu, int domain, vm_paddr_t release_addr) 530 { 531 struct pcpu *pcpup; 532 vm_size_t size; 533 vm_paddr_t pa; 534 int err, naps; 535 536 /* Check we are able to start this cpu */ 537 if (cpuid > mp_maxid) 538 return (false); 539 540 /* Skip boot CPU */ 541 if (is_boot_cpu(target_cpu)) 542 return (true); 543 544 KASSERT(cpuid < MAXCPU, ("Too many CPUs")); 545 546 size = round_page(sizeof(*pcpup) + DPCPU_SIZE); 547 pcpup = kmem_malloc_domainset(DOMAINSET_PREF(domain), size, 548 M_WAITOK | M_ZERO); 549 pmap_disable_promotion((vm_offset_t)pcpup, size); 550 pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); 551 pcpup->pc_mpidr = target_cpu & CPU_AFF_MASK; 552 bootpcpu = pcpup; 553 554 dpcpu[cpuid - 1] = (void *)(pcpup + 1); 555 dpcpu_init(dpcpu[cpuid - 1], cpuid); 556 557 bootstacks[cpuid] = kmem_malloc_domainset(DOMAINSET_PREF(domain), 558 MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO); 559 560 naps = atomic_load_int(&aps_started); 561 bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE; 562 563 printf("Starting CPU %u (%lx)\n", cpuid, target_cpu); 564 pa = pmap_extract(kernel_pmap, (vm_offset_t)mpentry); 565 566 /* 567 * A limited set of hardware we support can only do spintables and 568 * remain useful, due to lack of EL3. Thus, we'll usually fall into the 569 * PSCI branch here. 570 */ 571 MPASS(release_addr == 0 || !psci_present); 572 if (release_addr != 0) 573 err = enable_cpu_spin(target_cpu, pa, release_addr); 574 else 575 err = enable_cpu_psci(target_cpu, pa, cpuid); 576 577 if (err != 0) { 578 pcpu_destroy(pcpup); 579 dpcpu[cpuid - 1] = NULL; 580 kmem_free(bootstacks[cpuid], MP_BOOTSTACK_SIZE); 581 kmem_free(pcpup, size); 582 bootstacks[cpuid] = NULL; 583 mp_ncpus--; 584 return (false); 585 } 586 587 /* Wait for the AP to switch to its boot stack. */ 588 while (atomic_load_int(&aps_started) < naps + 1) 589 cpu_spinwait(); 590 CPU_SET(cpuid, &all_cpus); 591 592 return (true); 593 } 594 595 #ifdef DEV_ACPI 596 static void 597 madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) 598 { 599 ACPI_MADT_GENERIC_INTERRUPT *intr; 600 u_int *cpuid; 601 u_int id; 602 int domain; 603 604 switch(entry->Type) { 605 case ACPI_MADT_TYPE_GENERIC_INTERRUPT: 606 intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; 607 cpuid = arg; 608 609 if (is_boot_cpu(intr->ArmMpidr)) 610 id = 0; 611 else 612 id = *cpuid; 613 614 domain = 0; 615 #ifdef NUMA 616 if (vm_ndomains > 1) 617 domain = acpi_pxm_get_cpu_locality(intr->Uid); 618 #endif 619 if (start_cpu(id, intr->ArmMpidr, domain, 0)) { 620 MPASS(cpuid_to_pcpu[id] != NULL); 621 cpuid_to_pcpu[id]->pc_acpi_id = intr->Uid; 622 /* 623 * Don't increment for the boot CPU, its CPU ID is 624 * reserved. 625 */ 626 if (!is_boot_cpu(intr->ArmMpidr)) 627 (*cpuid)++; 628 } 629 630 break; 631 default: 632 break; 633 } 634 } 635 636 static void 637 cpu_init_acpi(void) 638 { 639 ACPI_TABLE_MADT *madt; 640 vm_paddr_t physaddr; 641 u_int cpuid; 642 643 physaddr = acpi_find_table(ACPI_SIG_MADT); 644 if (physaddr == 0) 645 return; 646 647 madt = acpi_map_table(physaddr, ACPI_SIG_MADT); 648 if (madt == NULL) { 649 printf("Unable to map the MADT, not starting APs\n"); 650 return; 651 } 652 /* Boot CPU is always 0 */ 653 cpuid = 1; 654 acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, 655 madt_handler, &cpuid); 656 657 acpi_unmap_table(madt); 658 659 #if MAXMEMDOM > 1 660 acpi_pxm_set_cpu_locality(); 661 #endif 662 } 663 #endif 664 665 #ifdef FDT 666 /* 667 * Failure is indicated by failing to populate *release_addr. 668 */ 669 static void 670 populate_release_addr(phandle_t node, vm_paddr_t *release_addr) 671 { 672 pcell_t buf[2]; 673 674 if (OF_getencprop(node, "cpu-release-addr", buf, sizeof(buf)) != 675 sizeof(buf)) 676 return; 677 678 *release_addr = (((uintptr_t)buf[0] << 32) | buf[1]); 679 } 680 681 static bool 682 start_cpu_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 683 { 684 uint64_t target_cpu; 685 vm_paddr_t release_addr; 686 char *enable_method; 687 int domain; 688 int cpuid; 689 690 target_cpu = reg[0]; 691 if (addr_size == 2) { 692 target_cpu <<= 32; 693 target_cpu |= reg[1]; 694 } 695 696 if (is_boot_cpu(target_cpu)) 697 cpuid = 0; 698 else 699 cpuid = fdt_cpuid; 700 701 /* 702 * If PSCI is present, we'll always use that -- the cpu_on method is 703 * mandated in both v0.1 and v0.2. We'll check the enable-method if 704 * we don't have PSCI and use spin table if it's provided. 705 */ 706 release_addr = 0; 707 if (!psci_present && cpuid != 0) { 708 if (OF_getprop_alloc(node, "enable-method", 709 (void **)&enable_method) <= 0) 710 return (false); 711 712 if (strcmp(enable_method, "spin-table") != 0) { 713 OF_prop_free(enable_method); 714 return (false); 715 } 716 717 OF_prop_free(enable_method); 718 populate_release_addr(node, &release_addr); 719 if (release_addr == 0) { 720 printf("Failed to fetch release address for CPU %u", 721 cpuid); 722 return (false); 723 } 724 } 725 726 if (!start_cpu(cpuid, target_cpu, 0, release_addr)) 727 return (false); 728 729 /* 730 * Don't increment for the boot CPU, its CPU ID is reserved. 731 */ 732 if (!is_boot_cpu(target_cpu)) 733 fdt_cpuid++; 734 735 /* Try to read the numa node of this cpu */ 736 if (vm_ndomains == 1 || 737 OF_getencprop(node, "numa-node-id", &domain, sizeof(domain)) <= 0) 738 domain = 0; 739 cpuid_to_pcpu[cpuid]->pc_domain = domain; 740 if (domain < MAXMEMDOM) 741 CPU_SET(cpuid, &cpuset_domain[domain]); 742 return (true); 743 } 744 static void 745 cpu_init_fdt(void) 746 { 747 phandle_t node; 748 int i; 749 750 node = OF_peer(0); 751 for (i = 0; fdt_quirks[i].compat != NULL; i++) { 752 if (ofw_bus_node_is_compatible(node, 753 fdt_quirks[i].compat) != 0) { 754 mp_quirks = fdt_quirks[i].quirks; 755 } 756 } 757 fdt_cpuid = 1; 758 ofw_cpu_early_foreach(start_cpu_fdt, true); 759 } 760 #endif 761 762 /* Initialize and fire up non-boot processors */ 763 void 764 cpu_mp_start(void) 765 { 766 uint64_t mpidr; 767 768 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 769 770 /* CPU 0 is always boot CPU. */ 771 CPU_SET(0, &all_cpus); 772 mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; 773 cpuid_to_pcpu[0]->pc_mpidr = mpidr; 774 775 cpu_desc_init(); 776 777 switch(arm64_bus_method) { 778 #ifdef DEV_ACPI 779 case ARM64_BUS_ACPI: 780 mp_quirks = MP_QUIRK_CPULIST; 781 cpu_init_acpi(); 782 break; 783 #endif 784 #ifdef FDT 785 case ARM64_BUS_FDT: 786 cpu_init_fdt(); 787 break; 788 #endif 789 default: 790 break; 791 } 792 } 793 794 /* Introduce rest of cores to the world */ 795 void 796 cpu_mp_announce(void) 797 { 798 } 799 800 #ifdef DEV_ACPI 801 static void 802 cpu_count_acpi_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) 803 { 804 u_int *cores = arg; 805 806 switch(entry->Type) { 807 case ACPI_MADT_TYPE_GENERIC_INTERRUPT: 808 (*cores)++; 809 break; 810 default: 811 break; 812 } 813 } 814 815 static u_int 816 cpu_count_acpi(void) 817 { 818 ACPI_TABLE_MADT *madt; 819 vm_paddr_t physaddr; 820 u_int cores; 821 822 physaddr = acpi_find_table(ACPI_SIG_MADT); 823 if (physaddr == 0) 824 return (0); 825 826 madt = acpi_map_table(physaddr, ACPI_SIG_MADT); 827 if (madt == NULL) { 828 printf("Unable to map the MADT, not starting APs\n"); 829 return (0); 830 } 831 832 cores = 0; 833 acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, 834 cpu_count_acpi_handler, &cores); 835 836 acpi_unmap_table(madt); 837 838 return (cores); 839 } 840 #endif 841 842 void 843 cpu_mp_setmaxid(void) 844 { 845 int cores; 846 847 mp_ncpus = 1; 848 mp_maxid = 0; 849 850 switch(arm64_bus_method) { 851 #ifdef DEV_ACPI 852 case ARM64_BUS_ACPI: 853 cores = cpu_count_acpi(); 854 if (cores > 0) { 855 cores = MIN(cores, MAXCPU); 856 if (bootverbose) 857 printf("Found %d CPUs in the ACPI tables\n", 858 cores); 859 mp_ncpus = cores; 860 mp_maxid = cores - 1; 861 } 862 break; 863 #endif 864 #ifdef FDT 865 case ARM64_BUS_FDT: 866 cores = ofw_cpu_early_foreach(NULL, false); 867 if (cores > 0) { 868 cores = MIN(cores, MAXCPU); 869 if (bootverbose) 870 printf("Found %d CPUs in the device tree\n", 871 cores); 872 mp_ncpus = cores; 873 mp_maxid = cores - 1; 874 } 875 break; 876 #endif 877 default: 878 if (bootverbose) 879 printf("No CPU data, limiting to 1 core\n"); 880 break; 881 } 882 883 if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { 884 if (cores > 0 && cores < mp_ncpus) { 885 mp_ncpus = cores; 886 mp_maxid = cores - 1; 887 } 888 } 889 } 890 891 /* 892 * Lookup IPI source. 893 */ 894 static struct intr_ipi * 895 intr_ipi_lookup(u_int ipi) 896 { 897 898 if (ipi >= INTR_IPI_COUNT) 899 panic("%s: no such IPI %u", __func__, ipi); 900 901 return (&ipi_sources[ipi]); 902 } 903 904 /* 905 * interrupt controller dispatch function for IPIs. It should 906 * be called straight from the interrupt controller, when associated 907 * interrupt source is learned. Or from anybody who has an interrupt 908 * source mapped. 909 */ 910 void 911 intr_ipi_dispatch(u_int ipi) 912 { 913 struct intr_ipi *ii; 914 915 ii = intr_ipi_lookup(ipi); 916 if (ii->ii_count == NULL) 917 panic("%s: not setup IPI %u", __func__, ipi); 918 919 intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid)); 920 921 ii->ii_handler(ii->ii_handler_arg); 922 } 923 924 #ifdef notyet 925 /* 926 * Map IPI into interrupt controller. 927 * 928 * Not SMP coherent. 929 */ 930 static int 931 ipi_map(struct intr_irqsrc *isrc, u_int ipi) 932 { 933 boolean_t is_percpu; 934 int error; 935 936 if (ipi >= INTR_IPI_COUNT) 937 panic("%s: no such IPI %u", __func__, ipi); 938 939 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 940 941 isrc->isrc_type = INTR_ISRCT_NAMESPACE; 942 isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI; 943 isrc->isrc_nspc_num = ipi_next_num; 944 945 error = PIC_REGISTER(intr_irq_root_dev, isrc, &is_percpu); 946 if (error == 0) { 947 isrc->isrc_dev = intr_irq_root_dev; 948 ipi_next_num++; 949 } 950 return (error); 951 } 952 953 /* 954 * Setup IPI handler to interrupt source. 955 * 956 * Note that there could be more ways how to send and receive IPIs 957 * on a platform like fast interrupts for example. In that case, 958 * one can call this function with ASIF_NOALLOC flag set and then 959 * call intr_ipi_dispatch() when appropriate. 960 * 961 * Not SMP coherent. 962 */ 963 int 964 intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter, 965 void *arg, u_int flags) 966 { 967 struct intr_irqsrc *isrc; 968 int error; 969 970 if (filter == NULL) 971 return(EINVAL); 972 973 isrc = intr_ipi_lookup(ipi); 974 if (isrc->isrc_ipifilter != NULL) 975 return (EEXIST); 976 977 if ((flags & AISHF_NOALLOC) == 0) { 978 error = ipi_map(isrc, ipi); 979 if (error != 0) 980 return (error); 981 } 982 983 isrc->isrc_ipifilter = filter; 984 isrc->isrc_arg = arg; 985 isrc->isrc_handlers = 1; 986 isrc->isrc_count = intr_ipi_setup_counters(name); 987 isrc->isrc_index = 0; /* it should not be used in IPI case */ 988 989 if (isrc->isrc_dev != NULL) { 990 PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 991 PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc); 992 } 993 return (0); 994 } 995 #endif 996 997 /* Sending IPI */ 998 void 999 ipi_all_but_self(u_int ipi) 1000 { 1001 cpuset_t cpus; 1002 1003 cpus = all_cpus; 1004 CPU_CLR(PCPU_GET(cpuid), &cpus); 1005 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1006 intr_ipi_send(cpus, ipi); 1007 } 1008 1009 void 1010 ipi_cpu(int cpu, u_int ipi) 1011 { 1012 cpuset_t cpus; 1013 1014 CPU_ZERO(&cpus); 1015 CPU_SET(cpu, &cpus); 1016 1017 CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); 1018 intr_ipi_send(cpus, ipi); 1019 } 1020 1021 void 1022 ipi_selected(cpuset_t cpus, u_int ipi) 1023 { 1024 1025 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1026 intr_ipi_send(cpus, ipi); 1027 } 1028