1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by SRI International and the 10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 12 * 13 * Portions of this software were developed by the University of Cambridge 14 * Computer Laboratory as part of the CTSRD Project, with support from the 15 * UK Higher Education Innovation Fund (HEIF). 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_kstack_pages.h" 40 #include "opt_platform.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bus.h> 45 #include <sys/cpu.h> 46 #include <sys/cpuset.h> 47 #include <sys/kernel.h> 48 #include <sys/ktr.h> 49 #include <sys/malloc.h> 50 #include <sys/module.h> 51 #include <sys/mutex.h> 52 #include <sys/proc.h> 53 #include <sys/sched.h> 54 #include <sys/smp.h> 55 56 #include <vm/vm.h> 57 #include <vm/pmap.h> 58 #include <vm/vm_extern.h> 59 #include <vm/vm_kern.h> 60 #include <vm/vm_map.h> 61 62 #include <machine/intr.h> 63 #include <machine/smp.h> 64 #include <machine/sbi.h> 65 66 #ifdef FDT 67 #include <dev/ofw/openfirm.h> 68 #include <dev/ofw/ofw_cpu.h> 69 #endif 70 71 #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE) 72 73 uint32_t __riscv_boot_ap[MAXCPU]; 74 75 static enum { 76 CPUS_UNKNOWN, 77 #ifdef FDT 78 CPUS_FDT, 79 #endif 80 } cpu_enum_method; 81 82 static device_identify_t riscv64_cpu_identify; 83 static device_probe_t riscv64_cpu_probe; 84 static device_attach_t riscv64_cpu_attach; 85 86 static int ipi_handler(void *); 87 88 extern uint32_t boot_hart; 89 extern cpuset_t all_harts; 90 91 #ifdef INVARIANTS 92 static uint32_t cpu_reg[MAXCPU][2]; 93 #endif 94 static device_t cpu_list[MAXCPU]; 95 96 void mpentry(u_long hartid); 97 void init_secondary(uint64_t); 98 99 static struct mtx ap_boot_mtx; 100 101 /* Stacks for AP initialization, discarded once idle threads are started. */ 102 void *bootstack; 103 static void *bootstacks[MAXCPU]; 104 105 /* Count of started APs, used to synchronize access to bootstack. */ 106 static volatile int aps_started; 107 108 /* Set to 1 once we're ready to let the APs out of the pen. */ 109 static volatile int aps_ready; 110 111 /* Temporary variables for init_secondary() */ 112 void *dpcpu[MAXCPU - 1]; 113 114 static device_method_t riscv64_cpu_methods[] = { 115 /* Device interface */ 116 DEVMETHOD(device_identify, riscv64_cpu_identify), 117 DEVMETHOD(device_probe, riscv64_cpu_probe), 118 DEVMETHOD(device_attach, riscv64_cpu_attach), 119 120 DEVMETHOD_END 121 }; 122 123 static driver_t riscv64_cpu_driver = { 124 "riscv64_cpu", 125 riscv64_cpu_methods, 126 0 127 }; 128 129 DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, 0, 0); 130 131 static void 132 riscv64_cpu_identify(driver_t *driver, device_t parent) 133 { 134 135 if (device_find_child(parent, "riscv64_cpu", -1) != NULL) 136 return; 137 if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL) 138 device_printf(parent, "add child failed\n"); 139 } 140 141 static int 142 riscv64_cpu_probe(device_t dev) 143 { 144 u_int cpuid; 145 146 cpuid = device_get_unit(dev); 147 if (cpuid >= MAXCPU || cpuid > mp_maxid) 148 return (EINVAL); 149 150 device_quiet(dev); 151 return (0); 152 } 153 154 static int 155 riscv64_cpu_attach(device_t dev) 156 { 157 const uint32_t *reg; 158 size_t reg_size; 159 u_int cpuid; 160 int i; 161 162 cpuid = device_get_unit(dev); 163 164 if (cpuid >= MAXCPU || cpuid > mp_maxid) 165 return (EINVAL); 166 KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid)); 167 168 reg = cpu_get_cpuid(dev, ®_size); 169 if (reg == NULL) 170 return (EINVAL); 171 172 if (bootverbose) { 173 device_printf(dev, "register <"); 174 for (i = 0; i < reg_size; i++) 175 printf("%s%x", (i == 0) ? "" : " ", reg[i]); 176 printf(">\n"); 177 } 178 179 /* Set the device to start it later */ 180 cpu_list[cpuid] = dev; 181 182 return (0); 183 } 184 185 static void 186 release_aps(void *dummy __unused) 187 { 188 cpuset_t mask; 189 int i; 190 191 if (mp_ncpus == 1) 192 return; 193 194 /* Setup the IPI handler */ 195 riscv_setup_ipihandler(ipi_handler); 196 197 atomic_store_rel_int(&aps_ready, 1); 198 199 /* Wake up the other CPUs */ 200 mask = all_harts; 201 CPU_CLR(boot_hart, &mask); 202 203 printf("Release APs\n"); 204 205 sbi_send_ipi(mask.__bits); 206 207 for (i = 0; i < 2000; i++) { 208 if (atomic_load_acq_int(&smp_started)) 209 return; 210 DELAY(1000); 211 } 212 213 printf("APs not started\n"); 214 } 215 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 216 217 void 218 init_secondary(uint64_t hart) 219 { 220 struct pcpu *pcpup; 221 u_int cpuid; 222 223 /* Renumber this cpu */ 224 cpuid = hart; 225 if (cpuid < boot_hart) 226 cpuid += mp_maxid + 1; 227 cpuid -= boot_hart; 228 229 /* Setup the pcpu pointer */ 230 pcpup = &__pcpu[cpuid]; 231 __asm __volatile("mv tp, %0" :: "r"(pcpup)); 232 233 /* Workaround: make sure wfi doesn't halt the hart */ 234 csr_set(sie, SIE_SSIE); 235 csr_set(sip, SIE_SSIE); 236 237 /* Signal the BSP and spin until it has released all APs. */ 238 atomic_add_int(&aps_started, 1); 239 while (!atomic_load_int(&aps_ready)) 240 __asm __volatile("wfi"); 241 242 /* Initialize curthread */ 243 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 244 pcpup->pc_curthread = pcpup->pc_idlethread; 245 schedinit_ap(); 246 247 /* Enable software interrupts */ 248 riscv_unmask_ipi(); 249 250 #ifndef EARLY_AP_STARTUP 251 /* Start per-CPU event timers. */ 252 cpu_initclocks_ap(); 253 #endif 254 255 /* Enable external (PLIC) interrupts */ 256 csr_set(sie, SIE_SEIE); 257 258 /* Activate this hart in the kernel pmap. */ 259 CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active); 260 261 /* Activate process 0's pmap. */ 262 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); 263 264 mtx_lock_spin(&ap_boot_mtx); 265 266 atomic_add_rel_32(&smp_cpus, 1); 267 268 if (smp_cpus == mp_ncpus) { 269 /* enable IPI's, tlb shootdown, freezes etc */ 270 atomic_store_rel_int(&smp_started, 1); 271 } 272 273 mtx_unlock_spin(&ap_boot_mtx); 274 275 if (bootverbose) 276 printf("Secondary CPU %u fully online\n", cpuid); 277 278 /* Enter the scheduler */ 279 sched_ap_entry(); 280 281 panic("scheduler returned us to init_secondary"); 282 /* NOTREACHED */ 283 } 284 285 static void 286 smp_after_idle_runnable(void *arg __unused) 287 { 288 int cpu; 289 290 if (mp_ncpus == 1) 291 return; 292 293 KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__)); 294 295 /* 296 * Wait for all APs to handle an interrupt. After that, we know that 297 * the APs have entered the scheduler at least once, so the boot stacks 298 * are safe to free. 299 */ 300 smp_rendezvous(smp_no_rendezvous_barrier, NULL, 301 smp_no_rendezvous_barrier, NULL); 302 303 for (cpu = 1; cpu <= mp_maxid; cpu++) { 304 if (bootstacks[cpu] != NULL) 305 kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE); 306 } 307 } 308 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, 309 smp_after_idle_runnable, NULL); 310 311 static int 312 ipi_handler(void *arg) 313 { 314 u_int ipi_bitmap; 315 u_int cpu, ipi; 316 int bit; 317 318 csr_clear(sip, SIP_SSIP); 319 320 cpu = PCPU_GET(cpuid); 321 322 mb(); 323 324 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); 325 if (ipi_bitmap == 0) 326 return (FILTER_HANDLED); 327 328 while ((bit = ffs(ipi_bitmap))) { 329 bit = (bit - 1); 330 ipi = (1 << bit); 331 ipi_bitmap &= ~ipi; 332 333 mb(); 334 335 switch (ipi) { 336 case IPI_AST: 337 CTR0(KTR_SMP, "IPI_AST"); 338 break; 339 case IPI_PREEMPT: 340 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 341 sched_preempt(curthread); 342 break; 343 case IPI_RENDEZVOUS: 344 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 345 smp_rendezvous_action(); 346 break; 347 case IPI_STOP: 348 case IPI_STOP_HARD: 349 CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); 350 savectx(&stoppcbs[cpu]); 351 352 /* Indicate we are stopped */ 353 CPU_SET_ATOMIC(cpu, &stopped_cpus); 354 355 /* Wait for restart */ 356 while (!CPU_ISSET(cpu, &started_cpus)) 357 cpu_spinwait(); 358 359 CPU_CLR_ATOMIC(cpu, &started_cpus); 360 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 361 CTR0(KTR_SMP, "IPI_STOP (restart)"); 362 363 /* 364 * The kernel debugger might have set a breakpoint, 365 * so flush the instruction cache. 366 */ 367 fence_i(); 368 break; 369 case IPI_HARDCLOCK: 370 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 371 hardclockintr(); 372 break; 373 default: 374 panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); 375 } 376 } 377 378 return (FILTER_HANDLED); 379 } 380 381 struct cpu_group * 382 cpu_topo(void) 383 { 384 385 return (smp_topo_none()); 386 } 387 388 /* Determine if we running MP machine */ 389 int 390 cpu_mp_probe(void) 391 { 392 393 return (mp_ncpus > 1); 394 } 395 396 #ifdef FDT 397 static bool 398 cpu_check_mmu(u_int id __unused, phandle_t node, u_int addr_size __unused, 399 pcell_t *reg __unused) 400 { 401 char type[32]; 402 403 /* Check if this hart supports MMU. */ 404 if (OF_getprop(node, "mmu-type", (void *)type, sizeof(type)) == -1 || 405 strncmp(type, "riscv,none", 10) == 0) 406 return (false); 407 408 return (true); 409 } 410 411 static bool 412 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 413 { 414 struct pcpu *pcpup; 415 vm_paddr_t start_addr; 416 uint64_t hart; 417 u_int cpuid; 418 int naps; 419 int error; 420 421 if (!cpu_check_mmu(id, node, addr_size, reg)) 422 return (false); 423 424 KASSERT(id < MAXCPU, ("Too many CPUs")); 425 426 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); 427 #ifdef INVARIANTS 428 cpu_reg[id][0] = reg[0]; 429 if (addr_size == 2) 430 cpu_reg[id][1] = reg[1]; 431 #endif 432 433 hart = reg[0]; 434 if (addr_size == 2) { 435 hart <<= 32; 436 hart |= reg[1]; 437 } 438 439 KASSERT(hart < MAXCPU, ("Too many harts.")); 440 441 /* We are already running on this cpu */ 442 if (hart == boot_hart) 443 return (true); 444 445 /* 446 * Rotate the CPU IDs to put the boot CPU as CPU 0. 447 * We keep the other CPUs ordered. 448 */ 449 cpuid = hart; 450 if (cpuid < boot_hart) 451 cpuid += mp_maxid + 1; 452 cpuid -= boot_hart; 453 454 /* Check if we are able to start this cpu */ 455 if (cpuid > mp_maxid) 456 return (false); 457 458 /* 459 * Depending on the SBI implementation, APs are waiting either in 460 * locore.S or to be activated explicitly, via SBI call. 461 */ 462 if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) { 463 start_addr = pmap_kextract((vm_offset_t)mpentry); 464 error = sbi_hsm_hart_start(hart, start_addr, 0); 465 if (error != 0) { 466 mp_ncpus--; 467 468 /* Send a warning to the user and continue. */ 469 printf("AP %u (hart %lu) failed to start, error %d\n", 470 cpuid, hart, error); 471 return (false); 472 } 473 } 474 475 pcpup = &__pcpu[cpuid]; 476 pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); 477 pcpup->pc_hart = hart; 478 479 dpcpu[cpuid - 1] = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 480 dpcpu_init(dpcpu[cpuid - 1], cpuid); 481 482 bootstacks[cpuid] = kmem_malloc(MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO); 483 484 naps = atomic_load_int(&aps_started); 485 bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE; 486 487 if (bootverbose) 488 printf("Starting CPU %u (hart %lx)\n", cpuid, hart); 489 atomic_store_32(&__riscv_boot_ap[hart], 1); 490 491 /* Wait for the AP to switch to its boot stack. */ 492 while (atomic_load_int(&aps_started) < naps + 1) 493 cpu_spinwait(); 494 495 CPU_SET(cpuid, &all_cpus); 496 CPU_SET(hart, &all_harts); 497 498 return (true); 499 } 500 #endif 501 502 /* Initialize and fire up non-boot processors */ 503 void 504 cpu_mp_start(void) 505 { 506 u_int cpu; 507 508 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 509 510 CPU_SET(0, &all_cpus); 511 CPU_SET(boot_hart, &all_harts); 512 513 switch(cpu_enum_method) { 514 #ifdef FDT 515 case CPUS_FDT: 516 ofw_cpu_early_foreach(cpu_init_fdt, true); 517 break; 518 #endif 519 case CPUS_UNKNOWN: 520 break; 521 } 522 523 CPU_FOREACH(cpu) { 524 /* Already identified. */ 525 if (cpu == 0) 526 continue; 527 528 identify_cpu(cpu); 529 } 530 } 531 532 /* Introduce rest of cores to the world */ 533 void 534 cpu_mp_announce(void) 535 { 536 u_int cpu; 537 538 CPU_FOREACH(cpu) { 539 /* Already announced. */ 540 if (cpu == 0) 541 continue; 542 543 printcpuinfo(cpu); 544 } 545 } 546 547 void 548 cpu_mp_setmaxid(void) 549 { 550 int cores; 551 552 #ifdef FDT 553 cores = ofw_cpu_early_foreach(cpu_check_mmu, true); 554 if (cores > 0) { 555 cores = MIN(cores, MAXCPU); 556 if (bootverbose) 557 printf("Found %d CPUs in the device tree\n", cores); 558 mp_ncpus = cores; 559 mp_maxid = cores - 1; 560 cpu_enum_method = CPUS_FDT; 561 } else 562 #endif 563 { 564 if (bootverbose) 565 printf("No CPU data, limiting to 1 core\n"); 566 mp_ncpus = 1; 567 mp_maxid = 0; 568 } 569 570 if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { 571 if (cores > 0 && cores < mp_ncpus) { 572 mp_ncpus = cores; 573 mp_maxid = cores - 1; 574 } 575 } 576 } 577