1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by SRI International and the 10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 12 * 13 * Portions of this software were developed by the University of Cambridge 14 * Computer Laboratory as part of the CTSRD Project, with support from the 15 * UK Higher Education Innovation Fund (HEIF). 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_kstack_pages.h" 40 #include "opt_platform.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/bus.h> 48 #include <sys/cpu.h> 49 #include <sys/kernel.h> 50 #include <sys/malloc.h> 51 #include <sys/module.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/sched.h> 55 #include <sys/smp.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 #include <vm/vm_extern.h> 60 #include <vm/vm_kern.h> 61 #include <vm/vm_map.h> 62 63 #include <machine/intr.h> 64 #include <machine/smp.h> 65 #include <machine/sbi.h> 66 67 #ifdef FDT 68 #include <dev/ofw/openfirm.h> 69 #include <dev/ofw/ofw_cpu.h> 70 #endif 71 72 boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *); 73 74 extern struct pcpu __pcpu[]; 75 76 uint32_t __riscv_boot_ap[MAXCPU]; 77 78 static enum { 79 CPUS_UNKNOWN, 80 #ifdef FDT 81 CPUS_FDT, 82 #endif 83 } cpu_enum_method; 84 85 static device_identify_t riscv64_cpu_identify; 86 static device_probe_t riscv64_cpu_probe; 87 static device_attach_t riscv64_cpu_attach; 88 89 static int ipi_handler(void *); 90 91 struct mtx ap_boot_mtx; 92 struct pcb stoppcbs[MAXCPU]; 93 94 #ifdef INVARIANTS 95 static uint32_t cpu_reg[MAXCPU][2]; 96 #endif 97 static device_t cpu_list[MAXCPU]; 98 99 void mpentry(unsigned long cpuid); 100 void init_secondary(uint64_t); 101 102 uint8_t secondary_stacks[MAXCPU - 1][PAGE_SIZE * KSTACK_PAGES] __aligned(16); 103 104 /* Set to 1 once we're ready to let the APs out of the pen. */ 105 volatile int aps_ready = 0; 106 107 /* Temporary variables for init_secondary() */ 108 void *dpcpu[MAXCPU - 1]; 109 110 static device_method_t riscv64_cpu_methods[] = { 111 /* Device interface */ 112 DEVMETHOD(device_identify, riscv64_cpu_identify), 113 DEVMETHOD(device_probe, riscv64_cpu_probe), 114 DEVMETHOD(device_attach, riscv64_cpu_attach), 115 116 DEVMETHOD_END 117 }; 118 119 static devclass_t riscv64_cpu_devclass; 120 static driver_t riscv64_cpu_driver = { 121 "riscv64_cpu", 122 riscv64_cpu_methods, 123 0 124 }; 125 126 DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, riscv64_cpu_devclass, 0, 0); 127 128 static void 129 riscv64_cpu_identify(driver_t *driver, device_t parent) 130 { 131 132 if (device_find_child(parent, "riscv64_cpu", -1) != NULL) 133 return; 134 if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL) 135 device_printf(parent, "add child failed\n"); 136 } 137 138 static int 139 riscv64_cpu_probe(device_t dev) 140 { 141 u_int cpuid; 142 143 cpuid = device_get_unit(dev); 144 if (cpuid >= MAXCPU || cpuid > mp_maxid) 145 return (EINVAL); 146 147 device_quiet(dev); 148 return (0); 149 } 150 151 static int 152 riscv64_cpu_attach(device_t dev) 153 { 154 const uint32_t *reg; 155 size_t reg_size; 156 u_int cpuid; 157 int i; 158 159 cpuid = device_get_unit(dev); 160 161 if (cpuid >= MAXCPU || cpuid > mp_maxid) 162 return (EINVAL); 163 KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid)); 164 165 reg = cpu_get_cpuid(dev, ®_size); 166 if (reg == NULL) 167 return (EINVAL); 168 169 if (bootverbose) { 170 device_printf(dev, "register <"); 171 for (i = 0; i < reg_size; i++) 172 printf("%s%x", (i == 0) ? "" : " ", reg[i]); 173 printf(">\n"); 174 } 175 176 /* Set the device to start it later */ 177 cpu_list[cpuid] = dev; 178 179 return (0); 180 } 181 182 static void 183 release_aps(void *dummy __unused) 184 { 185 u_long mask; 186 int cpu, i; 187 188 if (mp_ncpus == 1) 189 return; 190 191 /* Setup the IPI handler */ 192 riscv_setup_ipihandler(ipi_handler); 193 194 atomic_store_rel_int(&aps_ready, 1); 195 196 /* Wake up the other CPUs */ 197 mask = 0; 198 199 for (i = 1; i < mp_ncpus; i++) 200 mask |= (1 << i); 201 202 sbi_send_ipi(&mask); 203 204 printf("Release APs\n"); 205 206 for (i = 0; i < 2000; i++) { 207 if (smp_started) { 208 for (cpu = 0; cpu <= mp_maxid; cpu++) { 209 if (CPU_ABSENT(cpu)) 210 continue; 211 } 212 return; 213 } 214 DELAY(1000); 215 } 216 217 printf("APs not started\n"); 218 } 219 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 220 221 void 222 init_secondary(uint64_t cpu) 223 { 224 struct pcpu *pcpup; 225 226 /* Setup the pcpu pointer */ 227 pcpup = &__pcpu[cpu]; 228 __asm __volatile("mv gp, %0" :: "r"(pcpup)); 229 230 /* Workaround: make sure wfi doesn't halt the hart */ 231 csr_set(sie, SIE_SSIE); 232 csr_set(sip, SIE_SSIE); 233 234 /* Spin until the BSP releases the APs */ 235 while (!aps_ready) 236 __asm __volatile("wfi"); 237 238 /* Initialize curthread */ 239 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 240 pcpup->pc_curthread = pcpup->pc_idlethread; 241 pcpup->pc_curpcb = pcpup->pc_idlethread->td_pcb; 242 243 /* 244 * Identify current CPU. This is necessary to setup 245 * affinity registers and to provide support for 246 * runtime chip identification. 247 */ 248 identify_cpu(); 249 250 /* Enable software interrupts */ 251 riscv_unmask_ipi(); 252 253 /* Start per-CPU event timers. */ 254 cpu_initclocks_ap(); 255 256 /* Enable external (PLIC) interrupts */ 257 csr_set(sie, SIE_SEIE); 258 259 /* Activate process 0's pmap. */ 260 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); 261 262 mtx_lock_spin(&ap_boot_mtx); 263 264 atomic_add_rel_32(&smp_cpus, 1); 265 266 if (smp_cpus == mp_ncpus) { 267 /* enable IPI's, tlb shootdown, freezes etc */ 268 atomic_store_rel_int(&smp_started, 1); 269 } 270 271 mtx_unlock_spin(&ap_boot_mtx); 272 273 /* Enter the scheduler */ 274 sched_throw(NULL); 275 276 panic("scheduler returned us to init_secondary"); 277 /* NOTREACHED */ 278 } 279 280 static int 281 ipi_handler(void *arg) 282 { 283 u_int ipi_bitmap; 284 u_int cpu, ipi; 285 int bit; 286 287 sbi_clear_ipi(); 288 289 cpu = PCPU_GET(cpuid); 290 291 mb(); 292 293 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); 294 if (ipi_bitmap == 0) 295 return (FILTER_HANDLED); 296 297 while ((bit = ffs(ipi_bitmap))) { 298 bit = (bit - 1); 299 ipi = (1 << bit); 300 ipi_bitmap &= ~ipi; 301 302 mb(); 303 304 switch (ipi) { 305 case IPI_AST: 306 CTR0(KTR_SMP, "IPI_AST"); 307 break; 308 case IPI_PREEMPT: 309 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 310 sched_preempt(curthread); 311 break; 312 case IPI_RENDEZVOUS: 313 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 314 smp_rendezvous_action(); 315 break; 316 case IPI_STOP: 317 case IPI_STOP_HARD: 318 CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); 319 savectx(&stoppcbs[cpu]); 320 321 /* Indicate we are stopped */ 322 CPU_SET_ATOMIC(cpu, &stopped_cpus); 323 324 /* Wait for restart */ 325 while (!CPU_ISSET(cpu, &started_cpus)) 326 cpu_spinwait(); 327 328 CPU_CLR_ATOMIC(cpu, &started_cpus); 329 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 330 CTR0(KTR_SMP, "IPI_STOP (restart)"); 331 332 /* 333 * The kernel debugger might have set a breakpoint, 334 * so flush the instruction cache. 335 */ 336 fence_i(); 337 break; 338 case IPI_HARDCLOCK: 339 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 340 hardclockintr(); 341 break; 342 default: 343 panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); 344 } 345 } 346 347 return (FILTER_HANDLED); 348 } 349 350 struct cpu_group * 351 cpu_topo(void) 352 { 353 354 return (smp_topo_none()); 355 } 356 357 /* Determine if we running MP machine */ 358 int 359 cpu_mp_probe(void) 360 { 361 362 return (mp_ncpus > 1); 363 } 364 365 #ifdef FDT 366 static boolean_t 367 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 368 { 369 uint64_t target_cpu; 370 struct pcpu *pcpup; 371 372 /* Check we are able to start this cpu */ 373 if (id > mp_maxid) 374 return (0); 375 376 KASSERT(id < MAXCPU, ("Too many CPUs")); 377 378 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); 379 #ifdef INVARIANTS 380 cpu_reg[id][0] = reg[0]; 381 if (addr_size == 2) 382 cpu_reg[id][1] = reg[1]; 383 #endif 384 385 target_cpu = reg[0]; 386 if (addr_size == 2) { 387 target_cpu <<= 32; 388 target_cpu |= reg[1]; 389 } 390 391 pcpup = &__pcpu[id]; 392 393 /* We are already running on cpu 0 */ 394 if (id == 0) { 395 return (1); 396 } 397 398 pcpu_init(pcpup, id, sizeof(struct pcpu)); 399 400 dpcpu[id - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 401 dpcpu_init(dpcpu[id - 1], id); 402 403 printf("Starting CPU %u (%lx)\n", id, target_cpu); 404 __riscv_boot_ap[id] = 1; 405 406 CPU_SET(id, &all_cpus); 407 408 return (1); 409 } 410 #endif 411 412 /* Initialize and fire up non-boot processors */ 413 void 414 cpu_mp_start(void) 415 { 416 417 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 418 419 CPU_SET(0, &all_cpus); 420 421 switch(cpu_enum_method) { 422 #ifdef FDT 423 case CPUS_FDT: 424 ofw_cpu_early_foreach(cpu_init_fdt, true); 425 break; 426 #endif 427 case CPUS_UNKNOWN: 428 break; 429 } 430 } 431 432 /* Introduce rest of cores to the world */ 433 void 434 cpu_mp_announce(void) 435 { 436 } 437 438 void 439 cpu_mp_setmaxid(void) 440 { 441 #ifdef FDT 442 int cores; 443 444 cores = ofw_cpu_early_foreach(NULL, false); 445 if (cores > 0) { 446 cores = MIN(cores, MAXCPU); 447 if (bootverbose) 448 printf("Found %d CPUs in the device tree\n", cores); 449 mp_ncpus = cores; 450 mp_maxid = cores - 1; 451 cpu_enum_method = CPUS_FDT; 452 return; 453 } 454 #endif 455 456 if (bootverbose) 457 printf("No CPU data, limiting to 1 core\n"); 458 mp_ncpus = 1; 459 mp_maxid = 0; 460 } 461