1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by SRI International and the 10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 12 * 13 * Portions of this software were developed by the University of Cambridge 14 * Computer Laboratory as part of the CTSRD Project, with support from the 15 * UK Higher Education Innovation Fund (HEIF). 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_kstack_pages.h" 40 #include "opt_platform.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/bus.h> 48 #include <sys/cpu.h> 49 #include <sys/kernel.h> 50 #include <sys/ktr.h> 51 #include <sys/malloc.h> 52 #include <sys/module.h> 53 #include <sys/mutex.h> 54 #include <sys/proc.h> 55 #include <sys/sched.h> 56 #include <sys/smp.h> 57 58 #include <vm/vm.h> 59 #include <vm/pmap.h> 60 #include <vm/vm_extern.h> 61 #include <vm/vm_kern.h> 62 #include <vm/vm_map.h> 63 64 #include <machine/intr.h> 65 #include <machine/smp.h> 66 #include <machine/sbi.h> 67 68 #ifdef FDT 69 #include <dev/ofw/openfirm.h> 70 #include <dev/ofw/ofw_cpu.h> 71 #endif 72 73 boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *); 74 75 uint32_t __riscv_boot_ap[MAXCPU]; 76 77 static enum { 78 CPUS_UNKNOWN, 79 #ifdef FDT 80 CPUS_FDT, 81 #endif 82 } cpu_enum_method; 83 84 static device_identify_t riscv64_cpu_identify; 85 static device_probe_t riscv64_cpu_probe; 86 static device_attach_t riscv64_cpu_attach; 87 88 static int ipi_handler(void *); 89 90 struct pcb stoppcbs[MAXCPU]; 91 92 extern uint32_t boot_hart; 93 extern cpuset_t all_harts; 94 95 #ifdef INVARIANTS 96 static uint32_t cpu_reg[MAXCPU][2]; 97 #endif 98 static device_t cpu_list[MAXCPU]; 99 100 void mpentry(u_long hartid); 101 void init_secondary(uint64_t); 102 103 static struct mtx ap_boot_mtx; 104 105 /* Stacks for AP initialization, discarded once idle threads are started. */ 106 void *bootstack; 107 static void *bootstacks[MAXCPU]; 108 109 /* Count of started APs, used to synchronize access to bootstack. */ 110 static volatile int aps_started; 111 112 /* Set to 1 once we're ready to let the APs out of the pen. */ 113 static volatile int aps_ready; 114 115 /* Temporary variables for init_secondary() */ 116 void *dpcpu[MAXCPU - 1]; 117 118 static device_method_t riscv64_cpu_methods[] = { 119 /* Device interface */ 120 DEVMETHOD(device_identify, riscv64_cpu_identify), 121 DEVMETHOD(device_probe, riscv64_cpu_probe), 122 DEVMETHOD(device_attach, riscv64_cpu_attach), 123 124 DEVMETHOD_END 125 }; 126 127 static devclass_t riscv64_cpu_devclass; 128 static driver_t riscv64_cpu_driver = { 129 "riscv64_cpu", 130 riscv64_cpu_methods, 131 0 132 }; 133 134 DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, riscv64_cpu_devclass, 0, 0); 135 136 static void 137 riscv64_cpu_identify(driver_t *driver, device_t parent) 138 { 139 140 if (device_find_child(parent, "riscv64_cpu", -1) != NULL) 141 return; 142 if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL) 143 device_printf(parent, "add child failed\n"); 144 } 145 146 static int 147 riscv64_cpu_probe(device_t dev) 148 { 149 u_int cpuid; 150 151 cpuid = device_get_unit(dev); 152 if (cpuid >= MAXCPU || cpuid > mp_maxid) 153 return (EINVAL); 154 155 device_quiet(dev); 156 return (0); 157 } 158 159 static int 160 riscv64_cpu_attach(device_t dev) 161 { 162 const uint32_t *reg; 163 size_t reg_size; 164 u_int cpuid; 165 int i; 166 167 cpuid = device_get_unit(dev); 168 169 if (cpuid >= MAXCPU || cpuid > mp_maxid) 170 return (EINVAL); 171 KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid)); 172 173 reg = cpu_get_cpuid(dev, ®_size); 174 if (reg == NULL) 175 return (EINVAL); 176 177 if (bootverbose) { 178 device_printf(dev, "register <"); 179 for (i = 0; i < reg_size; i++) 180 printf("%s%x", (i == 0) ? "" : " ", reg[i]); 181 printf(">\n"); 182 } 183 184 /* Set the device to start it later */ 185 cpu_list[cpuid] = dev; 186 187 return (0); 188 } 189 190 static void 191 release_aps(void *dummy __unused) 192 { 193 cpuset_t mask; 194 int i; 195 196 if (mp_ncpus == 1) 197 return; 198 199 /* Setup the IPI handler */ 200 riscv_setup_ipihandler(ipi_handler); 201 202 atomic_store_rel_int(&aps_ready, 1); 203 204 /* Wake up the other CPUs */ 205 mask = all_harts; 206 CPU_CLR(boot_hart, &mask); 207 208 printf("Release APs\n"); 209 210 sbi_send_ipi(mask.__bits); 211 212 for (i = 0; i < 2000; i++) { 213 if (smp_started) 214 return; 215 DELAY(1000); 216 } 217 218 printf("APs not started\n"); 219 } 220 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 221 222 void 223 init_secondary(uint64_t hart) 224 { 225 struct pcpu *pcpup; 226 u_int cpuid; 227 228 /* Renumber this cpu */ 229 cpuid = hart; 230 if (cpuid < boot_hart) 231 cpuid += mp_maxid + 1; 232 cpuid -= boot_hart; 233 234 /* Setup the pcpu pointer */ 235 pcpup = &__pcpu[cpuid]; 236 __asm __volatile("mv tp, %0" :: "r"(pcpup)); 237 238 /* Workaround: make sure wfi doesn't halt the hart */ 239 csr_set(sie, SIE_SSIE); 240 csr_set(sip, SIE_SSIE); 241 242 /* Signal the BSP and spin until it has released all APs. */ 243 atomic_add_int(&aps_started, 1); 244 while (!atomic_load_int(&aps_ready)) 245 __asm __volatile("wfi"); 246 247 /* Initialize curthread */ 248 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 249 pcpup->pc_curthread = pcpup->pc_idlethread; 250 251 /* 252 * Identify current CPU. This is necessary to setup 253 * affinity registers and to provide support for 254 * runtime chip identification. 255 */ 256 identify_cpu(); 257 258 /* Enable software interrupts */ 259 riscv_unmask_ipi(); 260 261 #ifndef EARLY_AP_STARTUP 262 /* Start per-CPU event timers. */ 263 cpu_initclocks_ap(); 264 #endif 265 266 /* Enable external (PLIC) interrupts */ 267 csr_set(sie, SIE_SEIE); 268 269 /* Activate process 0's pmap. */ 270 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); 271 272 mtx_lock_spin(&ap_boot_mtx); 273 274 atomic_add_rel_32(&smp_cpus, 1); 275 276 if (smp_cpus == mp_ncpus) { 277 /* enable IPI's, tlb shootdown, freezes etc */ 278 atomic_store_rel_int(&smp_started, 1); 279 } 280 281 mtx_unlock_spin(&ap_boot_mtx); 282 283 /* 284 * Assert that smp_after_idle_runnable condition is reasonable. 285 */ 286 MPASS(PCPU_GET(curpcb) == NULL); 287 288 /* Enter the scheduler */ 289 sched_throw(NULL); 290 291 panic("scheduler returned us to init_secondary"); 292 /* NOTREACHED */ 293 } 294 295 static void 296 smp_after_idle_runnable(void *arg __unused) 297 { 298 struct pcpu *pc; 299 int cpu; 300 301 for (cpu = 1; cpu <= mp_maxid; cpu++) { 302 if (bootstacks[cpu] != NULL) { 303 pc = pcpu_find(cpu); 304 while (atomic_load_ptr(&pc->pc_curpcb) == NULL) 305 cpu_spinwait(); 306 kmem_free((vm_offset_t)bootstacks[cpu], PAGE_SIZE); 307 } 308 } 309 } 310 SYSINIT(smp_after_idle_runnable, SI_SUB_SMP, SI_ORDER_ANY, 311 smp_after_idle_runnable, NULL); 312 313 static int 314 ipi_handler(void *arg) 315 { 316 u_int ipi_bitmap; 317 u_int cpu, ipi; 318 int bit; 319 320 sbi_clear_ipi(); 321 322 cpu = PCPU_GET(cpuid); 323 324 mb(); 325 326 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); 327 if (ipi_bitmap == 0) 328 return (FILTER_HANDLED); 329 330 while ((bit = ffs(ipi_bitmap))) { 331 bit = (bit - 1); 332 ipi = (1 << bit); 333 ipi_bitmap &= ~ipi; 334 335 mb(); 336 337 switch (ipi) { 338 case IPI_AST: 339 CTR0(KTR_SMP, "IPI_AST"); 340 break; 341 case IPI_PREEMPT: 342 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 343 sched_preempt(curthread); 344 break; 345 case IPI_RENDEZVOUS: 346 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 347 smp_rendezvous_action(); 348 break; 349 case IPI_STOP: 350 case IPI_STOP_HARD: 351 CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); 352 savectx(&stoppcbs[cpu]); 353 354 /* Indicate we are stopped */ 355 CPU_SET_ATOMIC(cpu, &stopped_cpus); 356 357 /* Wait for restart */ 358 while (!CPU_ISSET(cpu, &started_cpus)) 359 cpu_spinwait(); 360 361 CPU_CLR_ATOMIC(cpu, &started_cpus); 362 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 363 CTR0(KTR_SMP, "IPI_STOP (restart)"); 364 365 /* 366 * The kernel debugger might have set a breakpoint, 367 * so flush the instruction cache. 368 */ 369 fence_i(); 370 break; 371 case IPI_HARDCLOCK: 372 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 373 hardclockintr(); 374 break; 375 default: 376 panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); 377 } 378 } 379 380 return (FILTER_HANDLED); 381 } 382 383 struct cpu_group * 384 cpu_topo(void) 385 { 386 387 return (smp_topo_none()); 388 } 389 390 /* Determine if we running MP machine */ 391 int 392 cpu_mp_probe(void) 393 { 394 395 return (mp_ncpus > 1); 396 } 397 398 #ifdef FDT 399 static boolean_t 400 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 401 { 402 struct pcpu *pcpup; 403 vm_paddr_t start_addr; 404 uint64_t hart; 405 u_int cpuid; 406 int naps; 407 int error; 408 409 /* Check if this hart supports MMU. */ 410 if (OF_getproplen(node, "mmu-type") < 0) 411 return (0); 412 413 KASSERT(id < MAXCPU, ("Too many CPUs")); 414 415 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); 416 #ifdef INVARIANTS 417 cpu_reg[id][0] = reg[0]; 418 if (addr_size == 2) 419 cpu_reg[id][1] = reg[1]; 420 #endif 421 422 hart = reg[0]; 423 if (addr_size == 2) { 424 hart <<= 32; 425 hart |= reg[1]; 426 } 427 428 KASSERT(hart < MAXCPU, ("Too many harts.")); 429 430 /* We are already running on this cpu */ 431 if (hart == boot_hart) 432 return (1); 433 434 /* 435 * Rotate the CPU IDs to put the boot CPU as CPU 0. 436 * We keep the other CPUs ordered. 437 */ 438 cpuid = hart; 439 if (cpuid < boot_hart) 440 cpuid += mp_maxid + 1; 441 cpuid -= boot_hart; 442 443 /* Check if we are able to start this cpu */ 444 if (cpuid > mp_maxid) 445 return (0); 446 447 /* 448 * Depending on the SBI implementation, APs are waiting either in 449 * locore.S or to be activated explicitly, via SBI call. 450 */ 451 if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) { 452 start_addr = pmap_kextract((vm_offset_t)mpentry); 453 error = sbi_hsm_hart_start(hart, start_addr, 0); 454 if (error != 0) { 455 mp_ncpus--; 456 457 /* Send a warning to the user and continue. */ 458 printf("AP %u (hart %lu) failed to start, error %d\n", 459 cpuid, hart, error); 460 return (0); 461 } 462 } 463 464 pcpup = &__pcpu[cpuid]; 465 pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); 466 pcpup->pc_hart = hart; 467 468 dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 469 dpcpu_init(dpcpu[cpuid - 1], cpuid); 470 471 bootstacks[cpuid] = (void *)kmem_malloc(PAGE_SIZE, M_WAITOK | M_ZERO); 472 473 naps = atomic_load_int(&aps_started); 474 bootstack = (char *)bootstacks[cpuid] + PAGE_SIZE; 475 476 printf("Starting CPU %u (hart %lx)\n", cpuid, hart); 477 atomic_store_32(&__riscv_boot_ap[hart], 1); 478 479 /* Wait for the AP to switch to its boot stack. */ 480 while (atomic_load_int(&aps_started) < naps + 1) 481 cpu_spinwait(); 482 483 CPU_SET(cpuid, &all_cpus); 484 CPU_SET(hart, &all_harts); 485 486 return (1); 487 } 488 #endif 489 490 /* Initialize and fire up non-boot processors */ 491 void 492 cpu_mp_start(void) 493 { 494 495 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 496 497 CPU_SET(0, &all_cpus); 498 CPU_SET(boot_hart, &all_harts); 499 500 switch(cpu_enum_method) { 501 #ifdef FDT 502 case CPUS_FDT: 503 ofw_cpu_early_foreach(cpu_init_fdt, true); 504 break; 505 #endif 506 case CPUS_UNKNOWN: 507 break; 508 } 509 } 510 511 /* Introduce rest of cores to the world */ 512 void 513 cpu_mp_announce(void) 514 { 515 } 516 517 static boolean_t 518 cpu_check_mmu(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 519 { 520 521 /* Check if this hart supports MMU. */ 522 if (OF_getproplen(node, "mmu-type") < 0) 523 return (0); 524 525 return (1); 526 } 527 528 void 529 cpu_mp_setmaxid(void) 530 { 531 #ifdef FDT 532 int cores; 533 534 cores = ofw_cpu_early_foreach(cpu_check_mmu, true); 535 if (cores > 0) { 536 cores = MIN(cores, MAXCPU); 537 if (bootverbose) 538 printf("Found %d CPUs in the device tree\n", cores); 539 mp_ncpus = cores; 540 mp_maxid = cores - 1; 541 cpu_enum_method = CPUS_FDT; 542 return; 543 } 544 #endif 545 546 if (bootverbose) 547 printf("No CPU data, limiting to 1 core\n"); 548 mp_ncpus = 1; 549 mp_maxid = 0; 550 } 551