1 /*- 2 * Copyright (c) 2015-2016 The FreeBSD Foundation 3 * 4 * This software was developed by Andrew Turner under 5 * sponsorship from the FreeBSD Foundation. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 17 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 18 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 19 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 20 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 21 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 22 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 23 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 24 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 * 28 */ 29 30 #include "opt_acpi.h" 31 #include "opt_ddb.h" 32 #include "opt_kstack_pages.h" 33 #include "opt_platform.h" 34 35 #include <sys/cdefs.h> 36 #include <sys/param.h> 37 #include <sys/systm.h> 38 #include <sys/bus.h> 39 #include <sys/cpu.h> 40 #include <sys/csan.h> 41 #include <sys/domainset.h> 42 #include <sys/kernel.h> 43 #include <sys/ktr.h> 44 #include <sys/malloc.h> 45 #include <sys/module.h> 46 #include <sys/mutex.h> 47 #include <sys/pcpu.h> 48 #include <sys/proc.h> 49 #include <sys/sched.h> 50 #include <sys/smp.h> 51 52 #include <vm/vm.h> 53 #include <vm/pmap.h> 54 #include <vm/vm_extern.h> 55 #include <vm/vm_kern.h> 56 #include <vm/vm_map.h> 57 58 #include <machine/machdep.h> 59 #include <machine/cpu.h> 60 #include <machine/debug_monitor.h> 61 #include <machine/intr.h> 62 #include <machine/smp.h> 63 #ifdef VFP 64 #include <machine/vfp.h> 65 #endif 66 67 #ifdef DEV_ACPI 68 #include <contrib/dev/acpica/include/acpi.h> 69 #include <dev/acpica/acpivar.h> 70 #endif 71 72 #ifdef FDT 73 #include <dev/ofw/openfirm.h> 74 #include <dev/ofw/ofw_bus.h> 75 #include <dev/ofw/ofw_bus_subr.h> 76 #include <dev/ofw/ofw_cpu.h> 77 #endif 78 79 #include <dev/psci/psci.h> 80 81 #include "pic_if.h" 82 83 #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE) 84 85 #define MP_QUIRK_CPULIST 0x01 /* The list of cpus may be wrong, */ 86 /* don't panic if one fails to start */ 87 static uint32_t mp_quirks; 88 89 #ifdef FDT 90 static struct { 91 const char *compat; 92 uint32_t quirks; 93 } fdt_quirks[] = { 94 { "arm,foundation-aarch64", MP_QUIRK_CPULIST }, 95 { "arm,fvp-base", MP_QUIRK_CPULIST }, 96 /* This is incorrect in some DTS files */ 97 { "arm,vfp-base", MP_QUIRK_CPULIST }, 98 { NULL, 0 }, 99 }; 100 #endif 101 102 typedef void intr_ipi_send_t(void *, cpuset_t, u_int); 103 typedef void intr_ipi_handler_t(void *); 104 105 #define INTR_IPI_NAMELEN (MAXCOMLEN + 1) 106 struct intr_ipi { 107 intr_ipi_handler_t * ii_handler; 108 void * ii_handler_arg; 109 intr_ipi_send_t * ii_send; 110 void * ii_send_arg; 111 char ii_name[INTR_IPI_NAMELEN]; 112 u_long * ii_count; 113 }; 114 115 static struct intr_ipi ipi_sources[INTR_IPI_COUNT]; 116 117 static struct intr_ipi *intr_ipi_lookup(u_int); 118 static void intr_pic_ipi_setup(u_int, const char *, intr_ipi_handler_t *, 119 void *); 120 121 static void ipi_ast(void *); 122 static void ipi_hardclock(void *); 123 static void ipi_preempt(void *); 124 static void ipi_rendezvous(void *); 125 static void ipi_stop(void *); 126 127 #ifdef FDT 128 static u_int fdt_cpuid; 129 #endif 130 131 void mpentry(unsigned long cpuid); 132 void init_secondary(uint64_t); 133 134 /* Synchronize AP startup. */ 135 static struct mtx ap_boot_mtx; 136 137 /* Stacks for AP initialization, discarded once idle threads are started. */ 138 void *bootstack; 139 static void *bootstacks[MAXCPU]; 140 141 /* Count of started APs, used to synchronize access to bootstack. */ 142 static volatile int aps_started; 143 144 /* Set to 1 once we're ready to let the APs out of the pen. */ 145 static volatile int aps_ready; 146 147 /* Temporary variables for init_secondary() */ 148 static void *dpcpu[MAXCPU - 1]; 149 150 static bool 151 is_boot_cpu(uint64_t target_cpu) 152 { 153 154 return (PCPU_GET_MPIDR(cpuid_to_pcpu[0]) == (target_cpu & CPU_AFF_MASK)); 155 } 156 157 static void 158 release_aps(void *dummy __unused) 159 { 160 int i, started; 161 162 /* Only release CPUs if they exist */ 163 if (mp_ncpus == 1) 164 return; 165 166 intr_pic_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); 167 intr_pic_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); 168 intr_pic_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); 169 intr_pic_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); 170 intr_pic_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); 171 intr_pic_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); 172 173 atomic_store_rel_int(&aps_ready, 1); 174 /* Wake up the other CPUs */ 175 __asm __volatile( 176 "dsb ishst \n" 177 "sev \n" 178 ::: "memory"); 179 180 printf("Release APs..."); 181 182 started = 0; 183 for (i = 0; i < 2000; i++) { 184 if (atomic_load_acq_int(&smp_started) != 0) { 185 printf("done\n"); 186 return; 187 } 188 /* 189 * Don't time out while we are making progress. Some large 190 * systems can take a while to start all CPUs. 191 */ 192 if (smp_cpus > started) { 193 i = 0; 194 started = smp_cpus; 195 } 196 DELAY(1000); 197 } 198 199 printf("APs not started\n"); 200 } 201 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 202 203 void 204 init_secondary(uint64_t cpu) 205 { 206 struct pcpu *pcpup; 207 pmap_t pmap0; 208 uint64_t mpidr; 209 210 ptrauth_mp_start(cpu); 211 212 /* 213 * Verify that the value passed in 'cpu' argument (aka context_id) is 214 * valid. Some older U-Boot based PSCI implementations are buggy, 215 * they can pass random value in it. 216 */ 217 mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; 218 if (cpu >= MAXCPU || cpuid_to_pcpu[cpu] == NULL || 219 PCPU_GET_MPIDR(cpuid_to_pcpu[cpu]) != mpidr) { 220 for (cpu = 0; cpu < mp_maxid; cpu++) 221 if (cpuid_to_pcpu[cpu] != NULL && 222 PCPU_GET_MPIDR(cpuid_to_pcpu[cpu]) == mpidr) 223 break; 224 if ( cpu >= MAXCPU) 225 panic("MPIDR for this CPU is not in pcpu table"); 226 } 227 228 pcpup = cpuid_to_pcpu[cpu]; 229 /* 230 * Set the pcpu pointer with a backup in tpidr_el1 to be 231 * loaded when entering the kernel from userland. 232 */ 233 __asm __volatile( 234 "mov x18, %0 \n" 235 "msr tpidr_el1, %0" :: "r"(pcpup)); 236 237 /* 238 * Identify current CPU. This is necessary to setup 239 * affinity registers and to provide support for 240 * runtime chip identification. 241 * 242 * We need this before signalling the CPU is ready to 243 * let the boot CPU use the results. 244 */ 245 pcpup->pc_midr = get_midr(); 246 identify_cpu(cpu); 247 248 /* Ensure the stores in identify_cpu have completed */ 249 atomic_thread_fence_acq_rel(); 250 251 /* Signal the BSP and spin until it has released all APs. */ 252 atomic_add_int(&aps_started, 1); 253 while (!atomic_load_int(&aps_ready)) 254 __asm __volatile("wfe"); 255 256 /* Initialize curthread */ 257 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 258 pcpup->pc_curthread = pcpup->pc_idlethread; 259 schedinit_ap(); 260 261 /* Initialize curpmap to match TTBR0's current setting. */ 262 pmap0 = vmspace_pmap(&vmspace0); 263 KASSERT(pmap_to_ttbr0(pmap0) == READ_SPECIALREG(ttbr0_el1), 264 ("pmap0 doesn't match cpu %ld's ttbr0", cpu)); 265 pcpup->pc_curpmap = pmap0; 266 267 install_cpu_errata(); 268 269 intr_pic_init_secondary(); 270 271 /* Start per-CPU event timers. */ 272 cpu_initclocks_ap(); 273 274 #ifdef VFP 275 vfp_init_secondary(); 276 #endif 277 278 dbg_init(); 279 pan_enable(); 280 281 mtx_lock_spin(&ap_boot_mtx); 282 atomic_add_rel_32(&smp_cpus, 1); 283 if (smp_cpus == mp_ncpus) { 284 /* enable IPI's, tlb shootdown, freezes etc */ 285 atomic_store_rel_int(&smp_started, 1); 286 } 287 mtx_unlock_spin(&ap_boot_mtx); 288 289 kcsan_cpu_init(cpu); 290 291 /* Enter the scheduler */ 292 sched_ap_entry(); 293 294 panic("scheduler returned us to init_secondary"); 295 /* NOTREACHED */ 296 } 297 298 static void 299 smp_after_idle_runnable(void *arg __unused) 300 { 301 int cpu; 302 303 if (mp_ncpus == 1) 304 return; 305 306 KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__)); 307 308 /* 309 * Wait for all APs to handle an interrupt. After that, we know that 310 * the APs have entered the scheduler at least once, so the boot stacks 311 * are safe to free. 312 */ 313 smp_rendezvous(smp_no_rendezvous_barrier, NULL, 314 smp_no_rendezvous_barrier, NULL); 315 316 for (cpu = 1; cpu < mp_ncpus; cpu++) { 317 if (bootstacks[cpu] != NULL) 318 kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE); 319 } 320 } 321 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, 322 smp_after_idle_runnable, NULL); 323 324 /* 325 * Send IPI thru interrupt controller. 326 */ 327 static void 328 pic_ipi_send(void *arg, cpuset_t cpus, u_int ipi) 329 { 330 331 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 332 333 /* 334 * Ensure that this CPU's stores will be visible to IPI 335 * recipients before starting to send the interrupts. 336 */ 337 dsb(ishst); 338 339 PIC_IPI_SEND(intr_irq_root_dev, arg, cpus, ipi); 340 } 341 342 /* 343 * Setup IPI handler on interrupt controller. 344 * 345 * Not SMP coherent. 346 */ 347 static void 348 intr_pic_ipi_setup(u_int ipi, const char *name, intr_ipi_handler_t *hand, 349 void *arg) 350 { 351 struct intr_irqsrc *isrc; 352 struct intr_ipi *ii; 353 int error; 354 355 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 356 KASSERT(hand != NULL, ("%s: ipi %u no handler", __func__, ipi)); 357 358 error = PIC_IPI_SETUP(intr_irq_root_dev, ipi, &isrc); 359 if (error != 0) 360 return; 361 362 isrc->isrc_handlers++; 363 364 ii = intr_ipi_lookup(ipi); 365 KASSERT(ii->ii_count == NULL, ("%s: ipi %u reused", __func__, ipi)); 366 367 ii->ii_handler = hand; 368 ii->ii_handler_arg = arg; 369 ii->ii_send = pic_ipi_send; 370 ii->ii_send_arg = isrc; 371 strlcpy(ii->ii_name, name, INTR_IPI_NAMELEN); 372 ii->ii_count = intr_ipi_setup_counters(name); 373 374 PIC_ENABLE_INTR(intr_irq_root_dev, isrc); 375 } 376 377 static void 378 intr_ipi_send(cpuset_t cpus, u_int ipi) 379 { 380 struct intr_ipi *ii; 381 382 ii = intr_ipi_lookup(ipi); 383 if (ii->ii_count == NULL) 384 panic("%s: not setup IPI %u", __func__, ipi); 385 386 ii->ii_send(ii->ii_send_arg, cpus, ipi); 387 } 388 389 static void 390 ipi_ast(void *dummy __unused) 391 { 392 393 CTR0(KTR_SMP, "IPI_AST"); 394 } 395 396 static void 397 ipi_hardclock(void *dummy __unused) 398 { 399 400 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 401 hardclockintr(); 402 } 403 404 static void 405 ipi_preempt(void *dummy __unused) 406 { 407 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 408 sched_preempt(curthread); 409 } 410 411 static void 412 ipi_rendezvous(void *dummy __unused) 413 { 414 415 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 416 smp_rendezvous_action(); 417 } 418 419 static void 420 ipi_stop(void *dummy __unused) 421 { 422 u_int cpu; 423 424 CTR0(KTR_SMP, "IPI_STOP"); 425 426 cpu = PCPU_GET(cpuid); 427 savectx(&stoppcbs[cpu]); 428 429 /* Indicate we are stopped */ 430 CPU_SET_ATOMIC(cpu, &stopped_cpus); 431 432 /* Wait for restart */ 433 while (!CPU_ISSET(cpu, &started_cpus)) 434 cpu_spinwait(); 435 436 #ifdef DDB 437 dbg_register_sync(NULL); 438 #endif 439 440 CPU_CLR_ATOMIC(cpu, &started_cpus); 441 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 442 CTR0(KTR_SMP, "IPI_STOP (restart)"); 443 } 444 445 struct cpu_group * 446 cpu_topo(void) 447 { 448 struct cpu_group *dom, *root; 449 int i; 450 451 root = smp_topo_alloc(1); 452 dom = smp_topo_alloc(vm_ndomains); 453 454 root->cg_parent = NULL; 455 root->cg_child = dom; 456 CPU_COPY(&all_cpus, &root->cg_mask); 457 root->cg_count = mp_ncpus; 458 root->cg_children = vm_ndomains; 459 root->cg_level = CG_SHARE_NONE; 460 root->cg_flags = 0; 461 462 /* 463 * Redundant layers will be collapsed by the caller so we don't need a 464 * special case for a single domain. 465 */ 466 for (i = 0; i < vm_ndomains; i++, dom++) { 467 dom->cg_parent = root; 468 dom->cg_child = NULL; 469 CPU_COPY(&cpuset_domain[i], &dom->cg_mask); 470 dom->cg_count = CPU_COUNT(&dom->cg_mask); 471 dom->cg_children = 0; 472 dom->cg_level = CG_SHARE_L3; 473 dom->cg_flags = 0; 474 } 475 476 return (root); 477 } 478 479 /* Determine if we running MP machine */ 480 int 481 cpu_mp_probe(void) 482 { 483 484 /* ARM64TODO: Read the u bit of mpidr_el1 to determine this */ 485 return (1); 486 } 487 488 static int 489 enable_cpu_psci(uint64_t target_cpu, vm_paddr_t entry, u_int cpuid) 490 { 491 int err; 492 493 err = psci_cpu_on(target_cpu, entry, cpuid); 494 if (err != PSCI_RETVAL_SUCCESS) { 495 /* 496 * Panic here if INVARIANTS are enabled and PSCI failed to 497 * start the requested CPU. psci_cpu_on() returns PSCI_MISSING 498 * to indicate we are unable to use it to start the given CPU. 499 */ 500 KASSERT(err == PSCI_MISSING || 501 (mp_quirks & MP_QUIRK_CPULIST) == MP_QUIRK_CPULIST, 502 ("Failed to start CPU %u (%lx), error %d\n", 503 cpuid, target_cpu, err)); 504 return (EINVAL); 505 } 506 507 return (0); 508 } 509 510 static int 511 enable_cpu_spin(uint64_t cpu, vm_paddr_t entry, vm_paddr_t release_paddr) 512 { 513 vm_paddr_t *release_addr; 514 515 release_addr = pmap_mapdev(release_paddr, sizeof(*release_addr)); 516 if (release_addr == NULL) 517 return (ENOMEM); 518 519 *release_addr = entry; 520 pmap_unmapdev(release_addr, sizeof(*release_addr)); 521 522 __asm __volatile( 523 "dsb sy \n" 524 "sev \n" 525 ::: "memory"); 526 527 return (0); 528 } 529 530 /* 531 * Starts a given CPU. If the CPU is already running, i.e. it is the boot CPU, 532 * do nothing. Returns true if the CPU is present and running. 533 */ 534 static bool 535 start_cpu(u_int cpuid, uint64_t target_cpu, int domain, vm_paddr_t release_addr) 536 { 537 struct pcpu *pcpup; 538 vm_size_t size; 539 vm_paddr_t pa; 540 int err, naps; 541 542 /* Check we are able to start this cpu */ 543 if (cpuid > mp_maxid) 544 return (false); 545 546 /* Skip boot CPU */ 547 if (is_boot_cpu(target_cpu)) 548 return (true); 549 550 KASSERT(cpuid < MAXCPU, ("Too many CPUs")); 551 552 size = round_page(sizeof(*pcpup) + DPCPU_SIZE); 553 pcpup = kmem_malloc_domainset(DOMAINSET_PREF(domain), size, 554 M_WAITOK | M_ZERO); 555 pmap_disable_promotion((vm_offset_t)pcpup, size); 556 pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); 557 pcpup->pc_mpidr = target_cpu & CPU_AFF_MASK; 558 559 dpcpu[cpuid - 1] = (void *)(pcpup + 1); 560 dpcpu_init(dpcpu[cpuid - 1], cpuid); 561 562 bootstacks[cpuid] = kmem_malloc_domainset(DOMAINSET_PREF(domain), 563 MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO); 564 565 naps = atomic_load_int(&aps_started); 566 bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE; 567 568 printf("Starting CPU %u (%lx)\n", cpuid, target_cpu); 569 pa = pmap_extract(kernel_pmap, (vm_offset_t)mpentry); 570 571 /* 572 * A limited set of hardware we support can only do spintables and 573 * remain useful, due to lack of EL3. Thus, we'll usually fall into the 574 * PSCI branch here. 575 */ 576 MPASS(release_addr == 0 || !psci_present); 577 if (release_addr != 0) 578 err = enable_cpu_spin(target_cpu, pa, release_addr); 579 else 580 err = enable_cpu_psci(target_cpu, pa, cpuid); 581 582 if (err != 0) { 583 pcpu_destroy(pcpup); 584 dpcpu[cpuid - 1] = NULL; 585 kmem_free(bootstacks[cpuid], MP_BOOTSTACK_SIZE); 586 kmem_free(pcpup, size); 587 bootstacks[cpuid] = NULL; 588 mp_ncpus--; 589 return (false); 590 } 591 592 /* Wait for the AP to switch to its boot stack. */ 593 while (atomic_load_int(&aps_started) < naps + 1) 594 cpu_spinwait(); 595 CPU_SET(cpuid, &all_cpus); 596 597 return (true); 598 } 599 600 #ifdef DEV_ACPI 601 static void 602 madt_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) 603 { 604 ACPI_MADT_GENERIC_INTERRUPT *intr; 605 u_int *cpuid; 606 u_int id; 607 int domain; 608 609 switch(entry->Type) { 610 case ACPI_MADT_TYPE_GENERIC_INTERRUPT: 611 intr = (ACPI_MADT_GENERIC_INTERRUPT *)entry; 612 cpuid = arg; 613 614 if (is_boot_cpu(intr->ArmMpidr)) 615 id = 0; 616 else 617 id = *cpuid; 618 619 domain = 0; 620 #ifdef NUMA 621 if (vm_ndomains > 1) 622 domain = acpi_pxm_get_cpu_locality(intr->Uid); 623 #endif 624 if (start_cpu(id, intr->ArmMpidr, domain, 0)) { 625 MPASS(cpuid_to_pcpu[id] != NULL); 626 cpuid_to_pcpu[id]->pc_acpi_id = intr->Uid; 627 /* 628 * Don't increment for the boot CPU, its CPU ID is 629 * reserved. 630 */ 631 if (!is_boot_cpu(intr->ArmMpidr)) 632 (*cpuid)++; 633 } 634 635 break; 636 default: 637 break; 638 } 639 } 640 641 static void 642 cpu_init_acpi(void) 643 { 644 ACPI_TABLE_MADT *madt; 645 vm_paddr_t physaddr; 646 u_int cpuid; 647 648 physaddr = acpi_find_table(ACPI_SIG_MADT); 649 if (physaddr == 0) 650 return; 651 652 madt = acpi_map_table(physaddr, ACPI_SIG_MADT); 653 if (madt == NULL) { 654 printf("Unable to map the MADT, not starting APs\n"); 655 return; 656 } 657 /* Boot CPU is always 0 */ 658 cpuid = 1; 659 acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, 660 madt_handler, &cpuid); 661 662 acpi_unmap_table(madt); 663 664 #if MAXMEMDOM > 1 665 acpi_pxm_set_cpu_locality(); 666 #endif 667 } 668 #endif 669 670 #ifdef FDT 671 /* 672 * Failure is indicated by failing to populate *release_addr. 673 */ 674 static void 675 populate_release_addr(phandle_t node, vm_paddr_t *release_addr) 676 { 677 pcell_t buf[2]; 678 679 if (OF_getencprop(node, "cpu-release-addr", buf, sizeof(buf)) != 680 sizeof(buf)) 681 return; 682 683 *release_addr = (((uintptr_t)buf[0] << 32) | buf[1]); 684 } 685 686 static bool 687 start_cpu_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 688 { 689 uint64_t target_cpu; 690 vm_paddr_t release_addr; 691 char *enable_method; 692 int domain; 693 int cpuid; 694 695 target_cpu = reg[0]; 696 if (addr_size == 2) { 697 target_cpu <<= 32; 698 target_cpu |= reg[1]; 699 } 700 701 if (is_boot_cpu(target_cpu)) 702 cpuid = 0; 703 else 704 cpuid = fdt_cpuid; 705 706 /* 707 * If PSCI is present, we'll always use that -- the cpu_on method is 708 * mandated in both v0.1 and v0.2. We'll check the enable-method if 709 * we don't have PSCI and use spin table if it's provided. 710 */ 711 release_addr = 0; 712 if (!psci_present && cpuid != 0) { 713 if (OF_getprop_alloc(node, "enable-method", 714 (void **)&enable_method) <= 0) 715 return (false); 716 717 if (strcmp(enable_method, "spin-table") != 0) { 718 OF_prop_free(enable_method); 719 return (false); 720 } 721 722 OF_prop_free(enable_method); 723 populate_release_addr(node, &release_addr); 724 if (release_addr == 0) { 725 printf("Failed to fetch release address for CPU %u", 726 cpuid); 727 return (false); 728 } 729 } 730 731 if (!start_cpu(cpuid, target_cpu, 0, release_addr)) 732 return (false); 733 734 /* 735 * Don't increment for the boot CPU, its CPU ID is reserved. 736 */ 737 if (!is_boot_cpu(target_cpu)) 738 fdt_cpuid++; 739 740 /* Try to read the numa node of this cpu */ 741 if (vm_ndomains == 1 || 742 OF_getencprop(node, "numa-node-id", &domain, sizeof(domain)) <= 0) 743 domain = 0; 744 cpuid_to_pcpu[cpuid]->pc_domain = domain; 745 if (domain < MAXMEMDOM) 746 CPU_SET(cpuid, &cpuset_domain[domain]); 747 return (true); 748 } 749 static void 750 cpu_init_fdt(void) 751 { 752 phandle_t node; 753 int i; 754 755 node = OF_peer(0); 756 for (i = 0; fdt_quirks[i].compat != NULL; i++) { 757 if (ofw_bus_node_is_compatible(node, 758 fdt_quirks[i].compat) != 0) { 759 mp_quirks = fdt_quirks[i].quirks; 760 } 761 } 762 fdt_cpuid = 1; 763 ofw_cpu_early_foreach(start_cpu_fdt, true); 764 } 765 #endif 766 767 /* Initialize and fire up non-boot processors */ 768 void 769 cpu_mp_start(void) 770 { 771 uint64_t mpidr; 772 773 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 774 775 /* CPU 0 is always boot CPU. */ 776 CPU_SET(0, &all_cpus); 777 mpidr = READ_SPECIALREG(mpidr_el1) & CPU_AFF_MASK; 778 cpuid_to_pcpu[0]->pc_mpidr = mpidr; 779 780 cpu_desc_init(); 781 782 switch(arm64_bus_method) { 783 #ifdef DEV_ACPI 784 case ARM64_BUS_ACPI: 785 mp_quirks = MP_QUIRK_CPULIST; 786 cpu_init_acpi(); 787 break; 788 #endif 789 #ifdef FDT 790 case ARM64_BUS_FDT: 791 cpu_init_fdt(); 792 break; 793 #endif 794 default: 795 break; 796 } 797 } 798 799 /* Introduce rest of cores to the world */ 800 void 801 cpu_mp_announce(void) 802 { 803 } 804 805 #ifdef DEV_ACPI 806 static void 807 cpu_count_acpi_handler(ACPI_SUBTABLE_HEADER *entry, void *arg) 808 { 809 u_int *cores = arg; 810 811 switch(entry->Type) { 812 case ACPI_MADT_TYPE_GENERIC_INTERRUPT: 813 (*cores)++; 814 break; 815 default: 816 break; 817 } 818 } 819 820 static u_int 821 cpu_count_acpi(void) 822 { 823 ACPI_TABLE_MADT *madt; 824 vm_paddr_t physaddr; 825 u_int cores; 826 827 physaddr = acpi_find_table(ACPI_SIG_MADT); 828 if (physaddr == 0) 829 return (0); 830 831 madt = acpi_map_table(physaddr, ACPI_SIG_MADT); 832 if (madt == NULL) { 833 printf("Unable to map the MADT, not starting APs\n"); 834 return (0); 835 } 836 837 cores = 0; 838 acpi_walk_subtables(madt + 1, (char *)madt + madt->Header.Length, 839 cpu_count_acpi_handler, &cores); 840 841 acpi_unmap_table(madt); 842 843 return (cores); 844 } 845 #endif 846 847 void 848 cpu_mp_setmaxid(void) 849 { 850 int cores; 851 852 mp_ncpus = 1; 853 mp_maxid = 0; 854 855 switch(arm64_bus_method) { 856 #ifdef DEV_ACPI 857 case ARM64_BUS_ACPI: 858 cores = cpu_count_acpi(); 859 if (cores > 0) { 860 cores = MIN(cores, MAXCPU); 861 if (bootverbose) 862 printf("Found %d CPUs in the ACPI tables\n", 863 cores); 864 mp_ncpus = cores; 865 mp_maxid = cores - 1; 866 } 867 break; 868 #endif 869 #ifdef FDT 870 case ARM64_BUS_FDT: 871 cores = ofw_cpu_early_foreach(NULL, false); 872 if (cores > 0) { 873 cores = MIN(cores, MAXCPU); 874 if (bootverbose) 875 printf("Found %d CPUs in the device tree\n", 876 cores); 877 mp_ncpus = cores; 878 mp_maxid = cores - 1; 879 } 880 break; 881 #endif 882 default: 883 if (bootverbose) 884 printf("No CPU data, limiting to 1 core\n"); 885 break; 886 } 887 888 if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { 889 if (cores > 0 && cores < mp_ncpus) { 890 mp_ncpus = cores; 891 mp_maxid = cores - 1; 892 } 893 } 894 } 895 896 /* 897 * Lookup IPI source. 898 */ 899 static struct intr_ipi * 900 intr_ipi_lookup(u_int ipi) 901 { 902 903 if (ipi >= INTR_IPI_COUNT) 904 panic("%s: no such IPI %u", __func__, ipi); 905 906 return (&ipi_sources[ipi]); 907 } 908 909 /* 910 * interrupt controller dispatch function for IPIs. It should 911 * be called straight from the interrupt controller, when associated 912 * interrupt source is learned. Or from anybody who has an interrupt 913 * source mapped. 914 */ 915 void 916 intr_ipi_dispatch(u_int ipi) 917 { 918 struct intr_ipi *ii; 919 920 ii = intr_ipi_lookup(ipi); 921 if (ii->ii_count == NULL) 922 panic("%s: not setup IPI %u", __func__, ipi); 923 924 intr_ipi_increment_count(ii->ii_count, PCPU_GET(cpuid)); 925 926 ii->ii_handler(ii->ii_handler_arg); 927 } 928 929 #ifdef notyet 930 /* 931 * Map IPI into interrupt controller. 932 * 933 * Not SMP coherent. 934 */ 935 static int 936 ipi_map(struct intr_irqsrc *isrc, u_int ipi) 937 { 938 boolean_t is_percpu; 939 int error; 940 941 if (ipi >= INTR_IPI_COUNT) 942 panic("%s: no such IPI %u", __func__, ipi); 943 944 KASSERT(intr_irq_root_dev != NULL, ("%s: no root attached", __func__)); 945 946 isrc->isrc_type = INTR_ISRCT_NAMESPACE; 947 isrc->isrc_nspc_type = INTR_IRQ_NSPC_IPI; 948 isrc->isrc_nspc_num = ipi_next_num; 949 950 error = PIC_REGISTER(intr_irq_root_dev, isrc, &is_percpu); 951 if (error == 0) { 952 isrc->isrc_dev = intr_irq_root_dev; 953 ipi_next_num++; 954 } 955 return (error); 956 } 957 958 /* 959 * Setup IPI handler to interrupt source. 960 * 961 * Note that there could be more ways how to send and receive IPIs 962 * on a platform like fast interrupts for example. In that case, 963 * one can call this function with ASIF_NOALLOC flag set and then 964 * call intr_ipi_dispatch() when appropriate. 965 * 966 * Not SMP coherent. 967 */ 968 int 969 intr_ipi_set_handler(u_int ipi, const char *name, intr_ipi_filter_t *filter, 970 void *arg, u_int flags) 971 { 972 struct intr_irqsrc *isrc; 973 int error; 974 975 if (filter == NULL) 976 return(EINVAL); 977 978 isrc = intr_ipi_lookup(ipi); 979 if (isrc->isrc_ipifilter != NULL) 980 return (EEXIST); 981 982 if ((flags & AISHF_NOALLOC) == 0) { 983 error = ipi_map(isrc, ipi); 984 if (error != 0) 985 return (error); 986 } 987 988 isrc->isrc_ipifilter = filter; 989 isrc->isrc_arg = arg; 990 isrc->isrc_handlers = 1; 991 isrc->isrc_count = intr_ipi_setup_counters(name); 992 isrc->isrc_index = 0; /* it should not be used in IPI case */ 993 994 if (isrc->isrc_dev != NULL) { 995 PIC_ENABLE_INTR(isrc->isrc_dev, isrc); 996 PIC_ENABLE_SOURCE(isrc->isrc_dev, isrc); 997 } 998 return (0); 999 } 1000 #endif 1001 1002 /* Sending IPI */ 1003 void 1004 ipi_all_but_self(u_int ipi) 1005 { 1006 cpuset_t cpus; 1007 1008 cpus = all_cpus; 1009 CPU_CLR(PCPU_GET(cpuid), &cpus); 1010 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1011 intr_ipi_send(cpus, ipi); 1012 } 1013 1014 void 1015 ipi_cpu(int cpu, u_int ipi) 1016 { 1017 cpuset_t cpus; 1018 1019 CPU_ZERO(&cpus); 1020 CPU_SET(cpu, &cpus); 1021 1022 CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); 1023 intr_ipi_send(cpus, ipi); 1024 } 1025 1026 void 1027 ipi_selected(cpuset_t cpus, u_int ipi) 1028 { 1029 1030 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 1031 intr_ipi_send(cpus, ipi); 1032 } 1033