1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by SRI International and the 10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 12 * 13 * Portions of this software were developed by the University of Cambridge 14 * Computer Laboratory as part of the CTSRD Project, with support from the 15 * UK Higher Education Innovation Fund (HEIF). 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_kstack_pages.h" 40 #include "opt_platform.h" 41 42 #include <sys/param.h> 43 #include <sys/systm.h> 44 #include <sys/bus.h> 45 #include <sys/cpu.h> 46 #include <sys/cpuset.h> 47 #include <sys/kernel.h> 48 #include <sys/ktr.h> 49 #include <sys/malloc.h> 50 #include <sys/module.h> 51 #include <sys/mutex.h> 52 #include <sys/proc.h> 53 #include <sys/sched.h> 54 #include <sys/smp.h> 55 56 #include <vm/vm.h> 57 #include <vm/pmap.h> 58 #include <vm/vm_extern.h> 59 #include <vm/vm_kern.h> 60 #include <vm/vm_map.h> 61 62 #include <machine/intr.h> 63 #include <machine/smp.h> 64 #include <machine/sbi.h> 65 66 #ifdef FDT 67 #include <dev/ofw/openfirm.h> 68 #include <dev/ofw/ofw_cpu.h> 69 #endif 70 71 #define MP_BOOTSTACK_SIZE (kstack_pages * PAGE_SIZE) 72 73 uint32_t __riscv_boot_ap[MAXCPU]; 74 75 static enum { 76 CPUS_UNKNOWN, 77 #ifdef FDT 78 CPUS_FDT, 79 #endif 80 } cpu_enum_method; 81 82 static void ipi_ast(void *); 83 static void ipi_hardclock(void *); 84 static void ipi_preempt(void *); 85 static void ipi_rendezvous(void *); 86 static void ipi_stop(void *); 87 88 extern uint32_t boot_hart; 89 extern cpuset_t all_harts; 90 91 #ifdef INVARIANTS 92 static uint32_t cpu_reg[MAXCPU][2]; 93 #endif 94 95 void mpentry(u_long hartid); 96 void init_secondary(uint64_t); 97 98 static struct mtx ap_boot_mtx; 99 100 /* Stacks for AP initialization, discarded once idle threads are started. */ 101 void *bootstack; 102 static void *bootstacks[MAXCPU]; 103 104 /* Count of started APs, used to synchronize access to bootstack. */ 105 static volatile int aps_started; 106 107 /* Set to 1 once we're ready to let the APs out of the pen. */ 108 static volatile int aps_ready; 109 110 /* Temporary variables for init_secondary() */ 111 void *dpcpu[MAXCPU - 1]; 112 113 static void 114 release_aps(void *dummy __unused) 115 { 116 cpuset_t mask; 117 int i; 118 119 if (mp_ncpus == 1) 120 return; 121 122 /* Setup the IPI handlers */ 123 intr_ipi_setup(IPI_AST, "ast", ipi_ast, NULL); 124 intr_ipi_setup(IPI_PREEMPT, "preempt", ipi_preempt, NULL); 125 intr_ipi_setup(IPI_RENDEZVOUS, "rendezvous", ipi_rendezvous, NULL); 126 intr_ipi_setup(IPI_STOP, "stop", ipi_stop, NULL); 127 intr_ipi_setup(IPI_STOP_HARD, "stop hard", ipi_stop, NULL); 128 intr_ipi_setup(IPI_HARDCLOCK, "hardclock", ipi_hardclock, NULL); 129 130 atomic_store_rel_int(&aps_ready, 1); 131 132 /* Wake up the other CPUs */ 133 mask = all_harts; 134 CPU_CLR(boot_hart, &mask); 135 136 printf("Release APs\n"); 137 138 sbi_send_ipi(mask.__bits); 139 140 for (i = 0; i < 2000; i++) { 141 if (atomic_load_acq_int(&smp_started)) 142 return; 143 DELAY(1000); 144 } 145 146 printf("APs not started\n"); 147 } 148 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 149 150 void 151 init_secondary(uint64_t hart) 152 { 153 struct pcpu *pcpup; 154 u_int cpuid; 155 156 /* Renumber this cpu */ 157 cpuid = hart; 158 if (cpuid < boot_hart) 159 cpuid += mp_maxid + 1; 160 cpuid -= boot_hart; 161 162 /* Setup the pcpu pointer */ 163 pcpup = &__pcpu[cpuid]; 164 __asm __volatile("mv tp, %0" :: "r"(pcpup)); 165 166 /* Workaround: make sure wfi doesn't halt the hart */ 167 csr_set(sie, SIE_SSIE); 168 csr_set(sip, SIE_SSIE); 169 170 /* Signal the BSP and spin until it has released all APs. */ 171 atomic_add_int(&aps_started, 1); 172 while (!atomic_load_int(&aps_ready)) 173 __asm __volatile("wfi"); 174 175 /* Initialize curthread */ 176 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 177 pcpup->pc_curthread = pcpup->pc_idlethread; 178 schedinit_ap(); 179 180 /* Setup and enable interrupts */ 181 intr_pic_init_secondary(); 182 183 #ifndef EARLY_AP_STARTUP 184 /* Start per-CPU event timers. */ 185 cpu_initclocks_ap(); 186 #endif 187 188 /* Activate this hart in the kernel pmap. */ 189 CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active); 190 191 /* Activate process 0's pmap. */ 192 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); 193 194 mtx_lock_spin(&ap_boot_mtx); 195 196 atomic_add_rel_32(&smp_cpus, 1); 197 198 if (smp_cpus == mp_ncpus) { 199 /* enable IPI's, tlb shootdown, freezes etc */ 200 atomic_store_rel_int(&smp_started, 1); 201 } 202 203 mtx_unlock_spin(&ap_boot_mtx); 204 205 if (bootverbose) 206 printf("Secondary CPU %u fully online\n", cpuid); 207 208 /* Enter the scheduler */ 209 sched_ap_entry(); 210 211 panic("scheduler returned us to init_secondary"); 212 /* NOTREACHED */ 213 } 214 215 static void 216 smp_after_idle_runnable(void *arg __unused) 217 { 218 int cpu; 219 220 if (mp_ncpus == 1) 221 return; 222 223 KASSERT(smp_started != 0, ("%s: SMP not started yet", __func__)); 224 225 /* 226 * Wait for all APs to handle an interrupt. After that, we know that 227 * the APs have entered the scheduler at least once, so the boot stacks 228 * are safe to free. 229 */ 230 smp_rendezvous(smp_no_rendezvous_barrier, NULL, 231 smp_no_rendezvous_barrier, NULL); 232 233 for (cpu = 1; cpu <= mp_maxid; cpu++) { 234 if (bootstacks[cpu] != NULL) 235 kmem_free(bootstacks[cpu], MP_BOOTSTACK_SIZE); 236 } 237 } 238 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, 239 smp_after_idle_runnable, NULL); 240 241 static void 242 ipi_ast(void *dummy __unused) 243 { 244 CTR0(KTR_SMP, "IPI_AST"); 245 } 246 247 static void 248 ipi_preempt(void *dummy __unused) 249 { 250 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 251 sched_preempt(curthread); 252 } 253 254 static void 255 ipi_rendezvous(void *dummy __unused) 256 { 257 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 258 smp_rendezvous_action(); 259 } 260 261 static void 262 ipi_stop(void *dummy __unused) 263 { 264 u_int cpu; 265 266 CTR0(KTR_SMP, "IPI_STOP"); 267 268 cpu = PCPU_GET(cpuid); 269 savectx(&stoppcbs[cpu]); 270 271 /* Indicate we are stopped */ 272 CPU_SET_ATOMIC(cpu, &stopped_cpus); 273 274 /* Wait for restart */ 275 while (!CPU_ISSET(cpu, &started_cpus)) 276 cpu_spinwait(); 277 278 CPU_CLR_ATOMIC(cpu, &started_cpus); 279 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 280 CTR0(KTR_SMP, "IPI_STOP (restart)"); 281 282 /* 283 * The kernel debugger might have set a breakpoint, 284 * so flush the instruction cache. 285 */ 286 fence_i(); 287 } 288 289 static void 290 ipi_hardclock(void *dummy __unused) 291 { 292 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 293 hardclockintr(); 294 } 295 296 struct cpu_group * 297 cpu_topo(void) 298 { 299 300 return (smp_topo_none()); 301 } 302 303 /* Determine if we running MP machine */ 304 int 305 cpu_mp_probe(void) 306 { 307 308 return (mp_ncpus > 1); 309 } 310 311 #ifdef FDT 312 static bool 313 cpu_check_mmu(u_int id __unused, phandle_t node, u_int addr_size __unused, 314 pcell_t *reg __unused) 315 { 316 char type[32]; 317 318 /* Check if this hart supports MMU. */ 319 if (OF_getprop(node, "mmu-type", (void *)type, sizeof(type)) == -1 || 320 strncmp(type, "riscv,none", 10) == 0) 321 return (false); 322 323 return (true); 324 } 325 326 static bool 327 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 328 { 329 struct pcpu *pcpup; 330 vm_paddr_t start_addr; 331 uint64_t hart; 332 u_int cpuid; 333 int naps; 334 int error; 335 336 if (!cpu_check_mmu(id, node, addr_size, reg)) 337 return (false); 338 339 KASSERT(id < MAXCPU, ("Too many CPUs")); 340 341 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); 342 #ifdef INVARIANTS 343 cpu_reg[id][0] = reg[0]; 344 if (addr_size == 2) 345 cpu_reg[id][1] = reg[1]; 346 #endif 347 348 hart = reg[0]; 349 if (addr_size == 2) { 350 hart <<= 32; 351 hart |= reg[1]; 352 } 353 354 KASSERT(hart < MAXCPU, ("Too many harts.")); 355 356 /* We are already running on this cpu */ 357 if (hart == boot_hart) 358 return (true); 359 360 /* 361 * Rotate the CPU IDs to put the boot CPU as CPU 0. 362 * We keep the other CPUs ordered. 363 */ 364 cpuid = hart; 365 if (cpuid < boot_hart) 366 cpuid += mp_maxid + 1; 367 cpuid -= boot_hart; 368 369 /* Check if we are able to start this cpu */ 370 if (cpuid > mp_maxid) 371 return (false); 372 373 /* 374 * Depending on the SBI implementation, APs are waiting either in 375 * locore.S or to be activated explicitly, via SBI call. 376 */ 377 if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) { 378 start_addr = pmap_kextract((vm_offset_t)mpentry); 379 error = sbi_hsm_hart_start(hart, start_addr, 0); 380 if (error != 0) { 381 mp_ncpus--; 382 383 /* Send a warning to the user and continue. */ 384 printf("AP %u (hart %lu) failed to start, error %d\n", 385 cpuid, hart, error); 386 return (false); 387 } 388 } 389 390 pcpup = &__pcpu[cpuid]; 391 pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); 392 pcpup->pc_hart = hart; 393 394 dpcpu[cpuid - 1] = kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 395 dpcpu_init(dpcpu[cpuid - 1], cpuid); 396 397 bootstacks[cpuid] = kmem_malloc(MP_BOOTSTACK_SIZE, M_WAITOK | M_ZERO); 398 399 naps = atomic_load_int(&aps_started); 400 bootstack = (char *)bootstacks[cpuid] + MP_BOOTSTACK_SIZE; 401 402 if (bootverbose) 403 printf("Starting CPU %u (hart %lx)\n", cpuid, hart); 404 atomic_store_32(&__riscv_boot_ap[hart], 1); 405 406 /* Wait for the AP to switch to its boot stack. */ 407 while (atomic_load_int(&aps_started) < naps + 1) 408 cpu_spinwait(); 409 410 CPU_SET(cpuid, &all_cpus); 411 CPU_SET(hart, &all_harts); 412 413 return (true); 414 } 415 #endif 416 417 /* Initialize and fire up non-boot processors */ 418 void 419 cpu_mp_start(void) 420 { 421 u_int cpu; 422 423 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 424 425 CPU_SET(0, &all_cpus); 426 CPU_SET(boot_hart, &all_harts); 427 428 switch(cpu_enum_method) { 429 #ifdef FDT 430 case CPUS_FDT: 431 ofw_cpu_early_foreach(cpu_init_fdt, true); 432 break; 433 #endif 434 case CPUS_UNKNOWN: 435 break; 436 } 437 438 CPU_FOREACH(cpu) { 439 /* Already identified. */ 440 if (cpu == 0) 441 continue; 442 443 identify_cpu(cpu); 444 } 445 } 446 447 /* Introduce rest of cores to the world */ 448 void 449 cpu_mp_announce(void) 450 { 451 u_int cpu; 452 453 CPU_FOREACH(cpu) { 454 /* Already announced. */ 455 if (cpu == 0) 456 continue; 457 458 printcpuinfo(cpu); 459 } 460 } 461 462 void 463 cpu_mp_setmaxid(void) 464 { 465 int cores; 466 467 #ifdef FDT 468 cores = ofw_cpu_early_foreach(cpu_check_mmu, true); 469 if (cores > 0) { 470 cores = MIN(cores, MAXCPU); 471 if (bootverbose) 472 printf("Found %d CPUs in the device tree\n", cores); 473 mp_ncpus = cores; 474 mp_maxid = cores - 1; 475 cpu_enum_method = CPUS_FDT; 476 } else 477 #endif 478 { 479 if (bootverbose) 480 printf("No CPU data, limiting to 1 core\n"); 481 mp_ncpus = 1; 482 mp_maxid = 0; 483 } 484 485 if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { 486 if (cores > 0 && cores < mp_ncpus) { 487 mp_ncpus = cores; 488 mp_maxid = cores - 1; 489 } 490 } 491 } 492 493 void 494 ipi_all_but_self(u_int ipi) 495 { 496 cpuset_t other_cpus; 497 498 other_cpus = all_cpus; 499 CPU_CLR(PCPU_GET(cpuid), &other_cpus); 500 501 CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); 502 intr_ipi_send(other_cpus, ipi); 503 } 504 505 void 506 ipi_cpu(int cpu, u_int ipi) 507 { 508 cpuset_t cpus; 509 510 CPU_ZERO(&cpus); 511 CPU_SET(cpu, &cpus); 512 513 CTR3(KTR_SMP, "%s: cpu: %d, ipi: %x", __func__, cpu, ipi); 514 intr_ipi_send(cpus, ipi); 515 } 516 517 void 518 ipi_selected(cpuset_t cpus, u_int ipi) 519 { 520 CTR1(KTR_SMP, "ipi_selected: ipi: %x", ipi); 521 intr_ipi_send(cpus, ipi); 522 } 523