1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by SRI International and the 10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 12 * 13 * Portions of this software were developed by the University of Cambridge 14 * Computer Laboratory as part of the CTSRD Project, with support from the 15 * UK Higher Education Innovation Fund (HEIF). 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_kstack_pages.h" 40 #include "opt_platform.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/bus.h> 48 #include <sys/cpu.h> 49 #include <sys/kernel.h> 50 #include <sys/malloc.h> 51 #include <sys/module.h> 52 #include <sys/mutex.h> 53 #include <sys/proc.h> 54 #include <sys/sched.h> 55 #include <sys/smp.h> 56 57 #include <vm/vm.h> 58 #include <vm/pmap.h> 59 #include <vm/vm_extern.h> 60 #include <vm/vm_kern.h> 61 62 #include <machine/intr.h> 63 #include <machine/smp.h> 64 #include <machine/sbi.h> 65 66 #ifdef FDT 67 #include <dev/ofw/openfirm.h> 68 #include <dev/ofw/ofw_cpu.h> 69 #endif 70 71 boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *); 72 73 extern struct pcpu __pcpu[]; 74 75 uint32_t __riscv_boot_ap[MAXCPU]; 76 77 static enum { 78 CPUS_UNKNOWN, 79 #ifdef FDT 80 CPUS_FDT, 81 #endif 82 } cpu_enum_method; 83 84 static device_identify_t riscv64_cpu_identify; 85 static device_probe_t riscv64_cpu_probe; 86 static device_attach_t riscv64_cpu_attach; 87 88 static int ipi_handler(void *); 89 90 struct mtx ap_boot_mtx; 91 struct pcb stoppcbs[MAXCPU]; 92 93 #ifdef INVARIANTS 94 static uint32_t cpu_reg[MAXCPU][2]; 95 #endif 96 static device_t cpu_list[MAXCPU]; 97 98 void mpentry(unsigned long cpuid); 99 void init_secondary(uint64_t); 100 101 uint8_t secondary_stacks[MAXCPU - 1][PAGE_SIZE * KSTACK_PAGES] __aligned(16); 102 103 /* Set to 1 once we're ready to let the APs out of the pen. */ 104 volatile int aps_ready = 0; 105 106 /* Temporary variables for init_secondary() */ 107 void *dpcpu[MAXCPU - 1]; 108 109 static device_method_t riscv64_cpu_methods[] = { 110 /* Device interface */ 111 DEVMETHOD(device_identify, riscv64_cpu_identify), 112 DEVMETHOD(device_probe, riscv64_cpu_probe), 113 DEVMETHOD(device_attach, riscv64_cpu_attach), 114 115 DEVMETHOD_END 116 }; 117 118 static devclass_t riscv64_cpu_devclass; 119 static driver_t riscv64_cpu_driver = { 120 "riscv64_cpu", 121 riscv64_cpu_methods, 122 0 123 }; 124 125 DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, riscv64_cpu_devclass, 0, 0); 126 127 static void 128 riscv64_cpu_identify(driver_t *driver, device_t parent) 129 { 130 131 if (device_find_child(parent, "riscv64_cpu", -1) != NULL) 132 return; 133 if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL) 134 device_printf(parent, "add child failed\n"); 135 } 136 137 static int 138 riscv64_cpu_probe(device_t dev) 139 { 140 u_int cpuid; 141 142 cpuid = device_get_unit(dev); 143 if (cpuid >= MAXCPU || cpuid > mp_maxid) 144 return (EINVAL); 145 146 device_quiet(dev); 147 return (0); 148 } 149 150 static int 151 riscv64_cpu_attach(device_t dev) 152 { 153 const uint32_t *reg; 154 size_t reg_size; 155 u_int cpuid; 156 int i; 157 158 cpuid = device_get_unit(dev); 159 160 if (cpuid >= MAXCPU || cpuid > mp_maxid) 161 return (EINVAL); 162 KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid)); 163 164 reg = cpu_get_cpuid(dev, ®_size); 165 if (reg == NULL) 166 return (EINVAL); 167 168 if (bootverbose) { 169 device_printf(dev, "register <"); 170 for (i = 0; i < reg_size; i++) 171 printf("%s%x", (i == 0) ? "" : " ", reg[i]); 172 printf(">\n"); 173 } 174 175 /* Set the device to start it later */ 176 cpu_list[cpuid] = dev; 177 178 return (0); 179 } 180 181 static void 182 release_aps(void *dummy __unused) 183 { 184 u_long mask; 185 int cpu, i; 186 187 if (mp_ncpus == 1) 188 return; 189 190 /* Setup the IPI handler */ 191 riscv_setup_ipihandler(ipi_handler); 192 193 atomic_store_rel_int(&aps_ready, 1); 194 195 /* Wake up the other CPUs */ 196 mask = 0; 197 198 for (i = 1; i < mp_ncpus; i++) 199 mask |= (1 << i); 200 201 sbi_send_ipi(&mask); 202 203 printf("Release APs\n"); 204 205 for (i = 0; i < 2000; i++) { 206 if (smp_started) { 207 for (cpu = 0; cpu <= mp_maxid; cpu++) { 208 if (CPU_ABSENT(cpu)) 209 continue; 210 } 211 return; 212 } 213 DELAY(1000); 214 } 215 216 printf("APs not started\n"); 217 } 218 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 219 220 void 221 init_secondary(uint64_t cpu) 222 { 223 struct pcpu *pcpup; 224 225 /* Setup the pcpu pointer */ 226 pcpup = &__pcpu[cpu]; 227 __asm __volatile("mv gp, %0" :: "r"(pcpup)); 228 229 /* Workaround: make sure wfi doesn't halt the hart */ 230 intr_disable(); 231 csr_set(sie, SIE_SSIE); 232 csr_set(sip, SIE_SSIE); 233 234 /* Spin until the BSP releases the APs */ 235 while (!aps_ready) 236 __asm __volatile("wfi"); 237 238 /* Initialize curthread */ 239 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 240 pcpup->pc_curthread = pcpup->pc_idlethread; 241 pcpup->pc_curpcb = pcpup->pc_idlethread->td_pcb; 242 243 /* 244 * Identify current CPU. This is necessary to setup 245 * affinity registers and to provide support for 246 * runtime chip identification. 247 */ 248 identify_cpu(); 249 250 /* Enable software interrupts */ 251 riscv_unmask_ipi(); 252 253 /* Start per-CPU event timers. */ 254 cpu_initclocks_ap(); 255 256 /* Enable interrupts */ 257 intr_enable(); 258 259 /* Enable external (PLIC) interrupts */ 260 csr_set(sie, SIE_SEIE); 261 262 mtx_lock_spin(&ap_boot_mtx); 263 264 atomic_add_rel_32(&smp_cpus, 1); 265 266 if (smp_cpus == mp_ncpus) { 267 /* enable IPI's, tlb shootdown, freezes etc */ 268 atomic_store_rel_int(&smp_started, 1); 269 } 270 271 mtx_unlock_spin(&ap_boot_mtx); 272 273 /* Enter the scheduler */ 274 sched_throw(NULL); 275 276 panic("scheduler returned us to init_secondary"); 277 /* NOTREACHED */ 278 } 279 280 static int 281 ipi_handler(void *arg) 282 { 283 u_int ipi_bitmap; 284 u_int cpu, ipi; 285 int bit; 286 287 sbi_clear_ipi(); 288 289 cpu = PCPU_GET(cpuid); 290 291 mb(); 292 293 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); 294 if (ipi_bitmap == 0) 295 return (FILTER_HANDLED); 296 297 while ((bit = ffs(ipi_bitmap))) { 298 bit = (bit - 1); 299 ipi = (1 << bit); 300 ipi_bitmap &= ~ipi; 301 302 mb(); 303 304 switch (ipi) { 305 case IPI_AST: 306 CTR0(KTR_SMP, "IPI_AST"); 307 break; 308 case IPI_PREEMPT: 309 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 310 sched_preempt(curthread); 311 break; 312 case IPI_RENDEZVOUS: 313 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 314 smp_rendezvous_action(); 315 break; 316 case IPI_STOP: 317 case IPI_STOP_HARD: 318 CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); 319 savectx(&stoppcbs[cpu]); 320 321 /* Indicate we are stopped */ 322 CPU_SET_ATOMIC(cpu, &stopped_cpus); 323 324 /* Wait for restart */ 325 while (!CPU_ISSET(cpu, &started_cpus)) 326 cpu_spinwait(); 327 328 CPU_CLR_ATOMIC(cpu, &started_cpus); 329 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 330 CTR0(KTR_SMP, "IPI_STOP (restart)"); 331 332 /* 333 * The kernel debugger might have set a breakpoint, 334 * so flush the instruction cache. 335 */ 336 fence_i(); 337 break; 338 case IPI_HARDCLOCK: 339 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 340 hardclockintr(); 341 break; 342 default: 343 panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); 344 } 345 } 346 347 return (FILTER_HANDLED); 348 } 349 350 struct cpu_group * 351 cpu_topo(void) 352 { 353 354 return (smp_topo_none()); 355 } 356 357 /* Determine if we running MP machine */ 358 int 359 cpu_mp_probe(void) 360 { 361 362 return (mp_ncpus > 1); 363 } 364 365 #ifdef FDT 366 static boolean_t 367 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 368 { 369 uint64_t target_cpu; 370 struct pcpu *pcpup; 371 372 /* Check we are able to start this cpu */ 373 if (id > mp_maxid) 374 return (0); 375 376 KASSERT(id < MAXCPU, ("Too many CPUs")); 377 378 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); 379 #ifdef INVARIANTS 380 cpu_reg[id][0] = reg[0]; 381 if (addr_size == 2) 382 cpu_reg[id][1] = reg[1]; 383 #endif 384 385 target_cpu = reg[0]; 386 if (addr_size == 2) { 387 target_cpu <<= 32; 388 target_cpu |= reg[1]; 389 } 390 391 pcpup = &__pcpu[id]; 392 393 /* We are already running on cpu 0 */ 394 if (id == 0) { 395 return (1); 396 } 397 398 pcpu_init(pcpup, id, sizeof(struct pcpu)); 399 400 dpcpu[id - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 401 dpcpu_init(dpcpu[id - 1], id); 402 403 printf("Starting CPU %u (%lx)\n", id, target_cpu); 404 __riscv_boot_ap[id] = 1; 405 406 CPU_SET(id, &all_cpus); 407 408 return (1); 409 } 410 #endif 411 412 /* Initialize and fire up non-boot processors */ 413 void 414 cpu_mp_start(void) 415 { 416 417 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 418 419 CPU_SET(0, &all_cpus); 420 421 switch(cpu_enum_method) { 422 #ifdef FDT 423 case CPUS_FDT: 424 ofw_cpu_early_foreach(cpu_init_fdt, true); 425 break; 426 #endif 427 case CPUS_UNKNOWN: 428 break; 429 } 430 } 431 432 /* Introduce rest of cores to the world */ 433 void 434 cpu_mp_announce(void) 435 { 436 } 437 438 void 439 cpu_mp_setmaxid(void) 440 { 441 #ifdef FDT 442 int cores; 443 444 cores = ofw_cpu_early_foreach(NULL, false); 445 if (cores > 0) { 446 cores = MIN(cores, MAXCPU); 447 if (bootverbose) 448 printf("Found %d CPUs in the device tree\n", cores); 449 mp_ncpus = cores; 450 mp_maxid = cores - 1; 451 cpu_enum_method = CPUS_FDT; 452 return; 453 } 454 #endif 455 456 if (bootverbose) 457 printf("No CPU data, limiting to 1 core\n"); 458 mp_ncpus = 1; 459 mp_maxid = 0; 460 } 461