1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by SRI International and the 10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 12 * 13 * Portions of this software were developed by the University of Cambridge 14 * Computer Laboratory as part of the CTSRD Project, with support from the 15 * UK Higher Education Innovation Fund (HEIF). 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_kstack_pages.h" 40 #include "opt_platform.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/bus.h> 48 #include <sys/cpu.h> 49 #include <sys/cpuset.h> 50 #include <sys/kernel.h> 51 #include <sys/ktr.h> 52 #include <sys/malloc.h> 53 #include <sys/module.h> 54 #include <sys/mutex.h> 55 #include <sys/proc.h> 56 #include <sys/sched.h> 57 #include <sys/smp.h> 58 59 #include <vm/vm.h> 60 #include <vm/pmap.h> 61 #include <vm/vm_extern.h> 62 #include <vm/vm_kern.h> 63 #include <vm/vm_map.h> 64 65 #include <machine/intr.h> 66 #include <machine/smp.h> 67 #include <machine/sbi.h> 68 69 #ifdef FDT 70 #include <dev/ofw/openfirm.h> 71 #include <dev/ofw/ofw_cpu.h> 72 #endif 73 74 boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *); 75 76 uint32_t __riscv_boot_ap[MAXCPU]; 77 78 static enum { 79 CPUS_UNKNOWN, 80 #ifdef FDT 81 CPUS_FDT, 82 #endif 83 } cpu_enum_method; 84 85 static device_identify_t riscv64_cpu_identify; 86 static device_probe_t riscv64_cpu_probe; 87 static device_attach_t riscv64_cpu_attach; 88 89 static int ipi_handler(void *); 90 91 struct pcb stoppcbs[MAXCPU]; 92 93 extern uint32_t boot_hart; 94 extern cpuset_t all_harts; 95 96 #ifdef INVARIANTS 97 static uint32_t cpu_reg[MAXCPU][2]; 98 #endif 99 static device_t cpu_list[MAXCPU]; 100 101 void mpentry(u_long hartid); 102 void init_secondary(uint64_t); 103 104 static struct mtx ap_boot_mtx; 105 106 /* Stacks for AP initialization, discarded once idle threads are started. */ 107 void *bootstack; 108 static void *bootstacks[MAXCPU]; 109 110 /* Count of started APs, used to synchronize access to bootstack. */ 111 static volatile int aps_started; 112 113 /* Set to 1 once we're ready to let the APs out of the pen. */ 114 static volatile int aps_ready; 115 116 /* Temporary variables for init_secondary() */ 117 void *dpcpu[MAXCPU - 1]; 118 119 static device_method_t riscv64_cpu_methods[] = { 120 /* Device interface */ 121 DEVMETHOD(device_identify, riscv64_cpu_identify), 122 DEVMETHOD(device_probe, riscv64_cpu_probe), 123 DEVMETHOD(device_attach, riscv64_cpu_attach), 124 125 DEVMETHOD_END 126 }; 127 128 static devclass_t riscv64_cpu_devclass; 129 static driver_t riscv64_cpu_driver = { 130 "riscv64_cpu", 131 riscv64_cpu_methods, 132 0 133 }; 134 135 DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, riscv64_cpu_devclass, 0, 0); 136 137 static void 138 riscv64_cpu_identify(driver_t *driver, device_t parent) 139 { 140 141 if (device_find_child(parent, "riscv64_cpu", -1) != NULL) 142 return; 143 if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL) 144 device_printf(parent, "add child failed\n"); 145 } 146 147 static int 148 riscv64_cpu_probe(device_t dev) 149 { 150 u_int cpuid; 151 152 cpuid = device_get_unit(dev); 153 if (cpuid >= MAXCPU || cpuid > mp_maxid) 154 return (EINVAL); 155 156 device_quiet(dev); 157 return (0); 158 } 159 160 static int 161 riscv64_cpu_attach(device_t dev) 162 { 163 const uint32_t *reg; 164 size_t reg_size; 165 u_int cpuid; 166 int i; 167 168 cpuid = device_get_unit(dev); 169 170 if (cpuid >= MAXCPU || cpuid > mp_maxid) 171 return (EINVAL); 172 KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid)); 173 174 reg = cpu_get_cpuid(dev, ®_size); 175 if (reg == NULL) 176 return (EINVAL); 177 178 if (bootverbose) { 179 device_printf(dev, "register <"); 180 for (i = 0; i < reg_size; i++) 181 printf("%s%x", (i == 0) ? "" : " ", reg[i]); 182 printf(">\n"); 183 } 184 185 /* Set the device to start it later */ 186 cpu_list[cpuid] = dev; 187 188 return (0); 189 } 190 191 static void 192 release_aps(void *dummy __unused) 193 { 194 cpuset_t mask; 195 int i; 196 197 if (mp_ncpus == 1) 198 return; 199 200 /* Setup the IPI handler */ 201 riscv_setup_ipihandler(ipi_handler); 202 203 atomic_store_rel_int(&aps_ready, 1); 204 205 /* Wake up the other CPUs */ 206 mask = all_harts; 207 CPU_CLR(boot_hart, &mask); 208 209 printf("Release APs\n"); 210 211 sbi_send_ipi(mask.__bits); 212 213 for (i = 0; i < 2000; i++) { 214 if (smp_started) 215 return; 216 DELAY(1000); 217 } 218 219 printf("APs not started\n"); 220 } 221 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 222 223 void 224 init_secondary(uint64_t hart) 225 { 226 struct pcpu *pcpup; 227 u_int cpuid; 228 229 /* Renumber this cpu */ 230 cpuid = hart; 231 if (cpuid < boot_hart) 232 cpuid += mp_maxid + 1; 233 cpuid -= boot_hart; 234 235 /* Setup the pcpu pointer */ 236 pcpup = &__pcpu[cpuid]; 237 __asm __volatile("mv tp, %0" :: "r"(pcpup)); 238 239 /* Workaround: make sure wfi doesn't halt the hart */ 240 csr_set(sie, SIE_SSIE); 241 csr_set(sip, SIE_SSIE); 242 243 /* Signal the BSP and spin until it has released all APs. */ 244 atomic_add_int(&aps_started, 1); 245 while (!atomic_load_int(&aps_ready)) 246 __asm __volatile("wfi"); 247 248 /* Initialize curthread */ 249 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 250 pcpup->pc_curthread = pcpup->pc_idlethread; 251 252 /* 253 * Identify current CPU. This is necessary to setup 254 * affinity registers and to provide support for 255 * runtime chip identification. 256 */ 257 identify_cpu(); 258 259 /* Enable software interrupts */ 260 riscv_unmask_ipi(); 261 262 #ifndef EARLY_AP_STARTUP 263 /* Start per-CPU event timers. */ 264 cpu_initclocks_ap(); 265 #endif 266 267 /* Enable external (PLIC) interrupts */ 268 csr_set(sie, SIE_SEIE); 269 270 /* Activate this hart in the kernel pmap. */ 271 CPU_SET_ATOMIC(hart, &kernel_pmap->pm_active); 272 273 /* Activate process 0's pmap. */ 274 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); 275 276 mtx_lock_spin(&ap_boot_mtx); 277 278 atomic_add_rel_32(&smp_cpus, 1); 279 280 if (smp_cpus == mp_ncpus) { 281 /* enable IPI's, tlb shootdown, freezes etc */ 282 atomic_store_rel_int(&smp_started, 1); 283 } 284 285 mtx_unlock_spin(&ap_boot_mtx); 286 287 /* 288 * Assert that smp_after_idle_runnable condition is reasonable. 289 */ 290 MPASS(PCPU_GET(curpcb) == NULL); 291 292 /* Enter the scheduler */ 293 sched_throw(NULL); 294 295 panic("scheduler returned us to init_secondary"); 296 /* NOTREACHED */ 297 } 298 299 static void 300 smp_after_idle_runnable(void *arg __unused) 301 { 302 struct pcpu *pc; 303 int cpu; 304 305 for (cpu = 1; cpu <= mp_maxid; cpu++) { 306 if (bootstacks[cpu] != NULL) { 307 pc = pcpu_find(cpu); 308 while (atomic_load_ptr(&pc->pc_curpcb) == NULL) 309 cpu_spinwait(); 310 kmem_free((vm_offset_t)bootstacks[cpu], PAGE_SIZE); 311 } 312 } 313 } 314 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, 315 smp_after_idle_runnable, NULL); 316 317 static int 318 ipi_handler(void *arg) 319 { 320 u_int ipi_bitmap; 321 u_int cpu, ipi; 322 int bit; 323 324 csr_clear(sip, SIP_SSIP); 325 326 cpu = PCPU_GET(cpuid); 327 328 mb(); 329 330 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); 331 if (ipi_bitmap == 0) 332 return (FILTER_HANDLED); 333 334 while ((bit = ffs(ipi_bitmap))) { 335 bit = (bit - 1); 336 ipi = (1 << bit); 337 ipi_bitmap &= ~ipi; 338 339 mb(); 340 341 switch (ipi) { 342 case IPI_AST: 343 CTR0(KTR_SMP, "IPI_AST"); 344 break; 345 case IPI_PREEMPT: 346 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 347 sched_preempt(curthread); 348 break; 349 case IPI_RENDEZVOUS: 350 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 351 smp_rendezvous_action(); 352 break; 353 case IPI_STOP: 354 case IPI_STOP_HARD: 355 CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); 356 savectx(&stoppcbs[cpu]); 357 358 /* Indicate we are stopped */ 359 CPU_SET_ATOMIC(cpu, &stopped_cpus); 360 361 /* Wait for restart */ 362 while (!CPU_ISSET(cpu, &started_cpus)) 363 cpu_spinwait(); 364 365 CPU_CLR_ATOMIC(cpu, &started_cpus); 366 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 367 CTR0(KTR_SMP, "IPI_STOP (restart)"); 368 369 /* 370 * The kernel debugger might have set a breakpoint, 371 * so flush the instruction cache. 372 */ 373 fence_i(); 374 break; 375 case IPI_HARDCLOCK: 376 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 377 hardclockintr(); 378 break; 379 default: 380 panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); 381 } 382 } 383 384 return (FILTER_HANDLED); 385 } 386 387 struct cpu_group * 388 cpu_topo(void) 389 { 390 391 return (smp_topo_none()); 392 } 393 394 /* Determine if we running MP machine */ 395 int 396 cpu_mp_probe(void) 397 { 398 399 return (mp_ncpus > 1); 400 } 401 402 #ifdef FDT 403 static boolean_t 404 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 405 { 406 struct pcpu *pcpup; 407 vm_paddr_t start_addr; 408 uint64_t hart; 409 u_int cpuid; 410 int naps; 411 int error; 412 413 /* Check if this hart supports MMU. */ 414 if (OF_getproplen(node, "mmu-type") < 0) 415 return (0); 416 417 KASSERT(id < MAXCPU, ("Too many CPUs")); 418 419 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); 420 #ifdef INVARIANTS 421 cpu_reg[id][0] = reg[0]; 422 if (addr_size == 2) 423 cpu_reg[id][1] = reg[1]; 424 #endif 425 426 hart = reg[0]; 427 if (addr_size == 2) { 428 hart <<= 32; 429 hart |= reg[1]; 430 } 431 432 KASSERT(hart < MAXCPU, ("Too many harts.")); 433 434 /* We are already running on this cpu */ 435 if (hart == boot_hart) 436 return (1); 437 438 /* 439 * Rotate the CPU IDs to put the boot CPU as CPU 0. 440 * We keep the other CPUs ordered. 441 */ 442 cpuid = hart; 443 if (cpuid < boot_hart) 444 cpuid += mp_maxid + 1; 445 cpuid -= boot_hart; 446 447 /* Check if we are able to start this cpu */ 448 if (cpuid > mp_maxid) 449 return (0); 450 451 /* 452 * Depending on the SBI implementation, APs are waiting either in 453 * locore.S or to be activated explicitly, via SBI call. 454 */ 455 if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) { 456 start_addr = pmap_kextract((vm_offset_t)mpentry); 457 error = sbi_hsm_hart_start(hart, start_addr, 0); 458 if (error != 0) { 459 mp_ncpus--; 460 461 /* Send a warning to the user and continue. */ 462 printf("AP %u (hart %lu) failed to start, error %d\n", 463 cpuid, hart, error); 464 return (0); 465 } 466 } 467 468 pcpup = &__pcpu[cpuid]; 469 pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); 470 pcpup->pc_hart = hart; 471 472 dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 473 dpcpu_init(dpcpu[cpuid - 1], cpuid); 474 475 bootstacks[cpuid] = (void *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO); 476 477 naps = atomic_load_int(&aps_started); 478 bootstack = (char *)bootstacks[cpuid] + PAGE_SIZE; 479 480 printf("Starting CPU %u (hart %lx)\n", cpuid, hart); 481 atomic_store_32(&__riscv_boot_ap[hart], 1); 482 483 /* Wait for the AP to switch to its boot stack. */ 484 while (atomic_load_int(&aps_started) < naps + 1) 485 cpu_spinwait(); 486 487 CPU_SET(cpuid, &all_cpus); 488 CPU_SET(hart, &all_harts); 489 490 return (1); 491 } 492 #endif 493 494 /* Initialize and fire up non-boot processors */ 495 void 496 cpu_mp_start(void) 497 { 498 499 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 500 501 CPU_SET(0, &all_cpus); 502 CPU_SET(boot_hart, &all_harts); 503 504 switch(cpu_enum_method) { 505 #ifdef FDT 506 case CPUS_FDT: 507 ofw_cpu_early_foreach(cpu_init_fdt, true); 508 break; 509 #endif 510 case CPUS_UNKNOWN: 511 break; 512 } 513 } 514 515 /* Introduce rest of cores to the world */ 516 void 517 cpu_mp_announce(void) 518 { 519 } 520 521 static boolean_t 522 cpu_check_mmu(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 523 { 524 525 /* Check if this hart supports MMU. */ 526 if (OF_getproplen(node, "mmu-type") < 0) 527 return (0); 528 529 return (1); 530 } 531 532 void 533 cpu_mp_setmaxid(void) 534 { 535 int cores; 536 537 #ifdef FDT 538 cores = ofw_cpu_early_foreach(cpu_check_mmu, true); 539 if (cores > 0) { 540 cores = MIN(cores, MAXCPU); 541 if (bootverbose) 542 printf("Found %d CPUs in the device tree\n", cores); 543 mp_ncpus = cores; 544 mp_maxid = cores - 1; 545 cpu_enum_method = CPUS_FDT; 546 } else 547 #endif 548 { 549 if (bootverbose) 550 printf("No CPU data, limiting to 1 core\n"); 551 mp_ncpus = 1; 552 mp_maxid = 0; 553 } 554 555 if (TUNABLE_INT_FETCH("hw.ncpu", &cores)) { 556 if (cores > 0 && cores < mp_ncpus) { 557 mp_ncpus = cores; 558 mp_maxid = cores - 1; 559 } 560 } 561 } 562