1 /*- 2 * Copyright (c) 2015 The FreeBSD Foundation 3 * Copyright (c) 2016 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner under 7 * sponsorship from the FreeBSD Foundation. 8 * 9 * Portions of this software were developed by SRI International and the 10 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 11 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 12 * 13 * Portions of this software were developed by the University of Cambridge 14 * Computer Laboratory as part of the CTSRD Project, with support from the 15 * UK Higher Education Innovation Fund (HEIF). 16 * 17 * Redistribution and use in source and binary forms, with or without 18 * modification, are permitted provided that the following conditions 19 * are met: 20 * 1. Redistributions of source code must retain the above copyright 21 * notice, this list of conditions and the following disclaimer. 22 * 2. Redistributions in binary form must reproduce the above copyright 23 * notice, this list of conditions and the following disclaimer in the 24 * documentation and/or other materials provided with the distribution. 25 * 26 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 27 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 28 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 29 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 30 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 31 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 32 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 33 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 34 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 35 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 36 * SUCH DAMAGE. 37 */ 38 39 #include "opt_kstack_pages.h" 40 #include "opt_platform.h" 41 42 #include <sys/cdefs.h> 43 __FBSDID("$FreeBSD$"); 44 45 #include <sys/param.h> 46 #include <sys/systm.h> 47 #include <sys/bus.h> 48 #include <sys/cpu.h> 49 #include <sys/kernel.h> 50 #include <sys/ktr.h> 51 #include <sys/malloc.h> 52 #include <sys/module.h> 53 #include <sys/mutex.h> 54 #include <sys/proc.h> 55 #include <sys/sched.h> 56 #include <sys/smp.h> 57 58 #include <vm/vm.h> 59 #include <vm/pmap.h> 60 #include <vm/vm_extern.h> 61 #include <vm/vm_kern.h> 62 #include <vm/vm_map.h> 63 64 #include <machine/intr.h> 65 #include <machine/smp.h> 66 #include <machine/sbi.h> 67 68 #ifdef FDT 69 #include <dev/ofw/openfirm.h> 70 #include <dev/ofw/ofw_cpu.h> 71 #endif 72 73 boolean_t ofw_cpu_reg(phandle_t node, u_int, cell_t *); 74 75 uint32_t __riscv_boot_ap[MAXCPU]; 76 77 static enum { 78 CPUS_UNKNOWN, 79 #ifdef FDT 80 CPUS_FDT, 81 #endif 82 } cpu_enum_method; 83 84 static device_identify_t riscv64_cpu_identify; 85 static device_probe_t riscv64_cpu_probe; 86 static device_attach_t riscv64_cpu_attach; 87 88 static int ipi_handler(void *); 89 90 struct mtx ap_boot_mtx; 91 struct pcb stoppcbs[MAXCPU]; 92 93 extern uint32_t boot_hart; 94 extern cpuset_t all_harts; 95 96 #ifdef INVARIANTS 97 static uint32_t cpu_reg[MAXCPU][2]; 98 #endif 99 static device_t cpu_list[MAXCPU]; 100 101 void mpentry(unsigned long cpuid); 102 void init_secondary(uint64_t); 103 104 uint8_t secondary_stacks[MAXCPU][PAGE_SIZE * KSTACK_PAGES] __aligned(16); 105 106 /* Set to 1 once we're ready to let the APs out of the pen. */ 107 volatile int aps_ready = 0; 108 109 /* Temporary variables for init_secondary() */ 110 void *dpcpu[MAXCPU - 1]; 111 112 static device_method_t riscv64_cpu_methods[] = { 113 /* Device interface */ 114 DEVMETHOD(device_identify, riscv64_cpu_identify), 115 DEVMETHOD(device_probe, riscv64_cpu_probe), 116 DEVMETHOD(device_attach, riscv64_cpu_attach), 117 118 DEVMETHOD_END 119 }; 120 121 static devclass_t riscv64_cpu_devclass; 122 static driver_t riscv64_cpu_driver = { 123 "riscv64_cpu", 124 riscv64_cpu_methods, 125 0 126 }; 127 128 DRIVER_MODULE(riscv64_cpu, cpu, riscv64_cpu_driver, riscv64_cpu_devclass, 0, 0); 129 130 static void 131 riscv64_cpu_identify(driver_t *driver, device_t parent) 132 { 133 134 if (device_find_child(parent, "riscv64_cpu", -1) != NULL) 135 return; 136 if (BUS_ADD_CHILD(parent, 0, "riscv64_cpu", -1) == NULL) 137 device_printf(parent, "add child failed\n"); 138 } 139 140 static int 141 riscv64_cpu_probe(device_t dev) 142 { 143 u_int cpuid; 144 145 cpuid = device_get_unit(dev); 146 if (cpuid >= MAXCPU || cpuid > mp_maxid) 147 return (EINVAL); 148 149 device_quiet(dev); 150 return (0); 151 } 152 153 static int 154 riscv64_cpu_attach(device_t dev) 155 { 156 const uint32_t *reg; 157 size_t reg_size; 158 u_int cpuid; 159 int i; 160 161 cpuid = device_get_unit(dev); 162 163 if (cpuid >= MAXCPU || cpuid > mp_maxid) 164 return (EINVAL); 165 KASSERT(cpu_list[cpuid] == NULL, ("Already have cpu %u", cpuid)); 166 167 reg = cpu_get_cpuid(dev, ®_size); 168 if (reg == NULL) 169 return (EINVAL); 170 171 if (bootverbose) { 172 device_printf(dev, "register <"); 173 for (i = 0; i < reg_size; i++) 174 printf("%s%x", (i == 0) ? "" : " ", reg[i]); 175 printf(">\n"); 176 } 177 178 /* Set the device to start it later */ 179 cpu_list[cpuid] = dev; 180 181 return (0); 182 } 183 184 static void 185 release_aps(void *dummy __unused) 186 { 187 cpuset_t mask; 188 int i; 189 190 if (mp_ncpus == 1) 191 return; 192 193 /* Setup the IPI handler */ 194 riscv_setup_ipihandler(ipi_handler); 195 196 atomic_store_rel_int(&aps_ready, 1); 197 198 /* Wake up the other CPUs */ 199 mask = all_harts; 200 CPU_CLR(boot_hart, &mask); 201 202 printf("Release APs\n"); 203 204 sbi_send_ipi(mask.__bits); 205 206 for (i = 0; i < 2000; i++) { 207 if (smp_started) 208 return; 209 DELAY(1000); 210 } 211 212 printf("APs not started\n"); 213 } 214 SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); 215 216 void 217 init_secondary(uint64_t hart) 218 { 219 struct pcpu *pcpup; 220 u_int cpuid; 221 222 /* Renumber this cpu */ 223 cpuid = hart; 224 if (cpuid < boot_hart) 225 cpuid += mp_maxid + 1; 226 cpuid -= boot_hart; 227 228 /* Setup the pcpu pointer */ 229 pcpup = &__pcpu[cpuid]; 230 __asm __volatile("mv tp, %0" :: "r"(pcpup)); 231 232 /* Workaround: make sure wfi doesn't halt the hart */ 233 csr_set(sie, SIE_SSIE); 234 csr_set(sip, SIE_SSIE); 235 236 /* Spin until the BSP releases the APs */ 237 while (!aps_ready) 238 __asm __volatile("wfi"); 239 240 /* Initialize curthread */ 241 KASSERT(PCPU_GET(idlethread) != NULL, ("no idle thread")); 242 pcpup->pc_curthread = pcpup->pc_idlethread; 243 pcpup->pc_curpcb = pcpup->pc_idlethread->td_pcb; 244 245 /* 246 * Identify current CPU. This is necessary to setup 247 * affinity registers and to provide support for 248 * runtime chip identification. 249 */ 250 identify_cpu(); 251 252 /* Enable software interrupts */ 253 riscv_unmask_ipi(); 254 255 #ifndef EARLY_AP_STARTUP 256 /* Start per-CPU event timers. */ 257 cpu_initclocks_ap(); 258 #endif 259 260 /* Enable external (PLIC) interrupts */ 261 csr_set(sie, SIE_SEIE); 262 263 /* Activate process 0's pmap. */ 264 pmap_activate_boot(vmspace_pmap(proc0.p_vmspace)); 265 266 mtx_lock_spin(&ap_boot_mtx); 267 268 atomic_add_rel_32(&smp_cpus, 1); 269 270 if (smp_cpus == mp_ncpus) { 271 /* enable IPI's, tlb shootdown, freezes etc */ 272 atomic_store_rel_int(&smp_started, 1); 273 } 274 275 mtx_unlock_spin(&ap_boot_mtx); 276 277 /* Enter the scheduler */ 278 sched_throw(NULL); 279 280 panic("scheduler returned us to init_secondary"); 281 /* NOTREACHED */ 282 } 283 284 static int 285 ipi_handler(void *arg) 286 { 287 u_int ipi_bitmap; 288 u_int cpu, ipi; 289 int bit; 290 291 sbi_clear_ipi(); 292 293 cpu = PCPU_GET(cpuid); 294 295 mb(); 296 297 ipi_bitmap = atomic_readandclear_int(PCPU_PTR(pending_ipis)); 298 if (ipi_bitmap == 0) 299 return (FILTER_HANDLED); 300 301 while ((bit = ffs(ipi_bitmap))) { 302 bit = (bit - 1); 303 ipi = (1 << bit); 304 ipi_bitmap &= ~ipi; 305 306 mb(); 307 308 switch (ipi) { 309 case IPI_AST: 310 CTR0(KTR_SMP, "IPI_AST"); 311 break; 312 case IPI_PREEMPT: 313 CTR1(KTR_SMP, "%s: IPI_PREEMPT", __func__); 314 sched_preempt(curthread); 315 break; 316 case IPI_RENDEZVOUS: 317 CTR0(KTR_SMP, "IPI_RENDEZVOUS"); 318 smp_rendezvous_action(); 319 break; 320 case IPI_STOP: 321 case IPI_STOP_HARD: 322 CTR0(KTR_SMP, (ipi == IPI_STOP) ? "IPI_STOP" : "IPI_STOP_HARD"); 323 savectx(&stoppcbs[cpu]); 324 325 /* Indicate we are stopped */ 326 CPU_SET_ATOMIC(cpu, &stopped_cpus); 327 328 /* Wait for restart */ 329 while (!CPU_ISSET(cpu, &started_cpus)) 330 cpu_spinwait(); 331 332 CPU_CLR_ATOMIC(cpu, &started_cpus); 333 CPU_CLR_ATOMIC(cpu, &stopped_cpus); 334 CTR0(KTR_SMP, "IPI_STOP (restart)"); 335 336 /* 337 * The kernel debugger might have set a breakpoint, 338 * so flush the instruction cache. 339 */ 340 fence_i(); 341 break; 342 case IPI_HARDCLOCK: 343 CTR1(KTR_SMP, "%s: IPI_HARDCLOCK", __func__); 344 hardclockintr(); 345 break; 346 default: 347 panic("Unknown IPI %#0x on cpu %d", ipi, curcpu); 348 } 349 } 350 351 return (FILTER_HANDLED); 352 } 353 354 struct cpu_group * 355 cpu_topo(void) 356 { 357 358 return (smp_topo_none()); 359 } 360 361 /* Determine if we running MP machine */ 362 int 363 cpu_mp_probe(void) 364 { 365 366 return (mp_ncpus > 1); 367 } 368 369 #ifdef FDT 370 static boolean_t 371 cpu_init_fdt(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 372 { 373 struct pcpu *pcpup; 374 uint64_t hart; 375 u_int cpuid; 376 377 /* Check if this hart supports MMU. */ 378 if (OF_getproplen(node, "mmu-type") < 0) 379 return (0); 380 381 KASSERT(id < MAXCPU, ("Too many CPUs")); 382 383 KASSERT(addr_size == 1 || addr_size == 2, ("Invalid register size")); 384 #ifdef INVARIANTS 385 cpu_reg[id][0] = reg[0]; 386 if (addr_size == 2) 387 cpu_reg[id][1] = reg[1]; 388 #endif 389 390 hart = reg[0]; 391 if (addr_size == 2) { 392 hart <<= 32; 393 hart |= reg[1]; 394 } 395 396 KASSERT(hart < MAXCPU, ("Too many harts.")); 397 398 /* We are already running on this cpu */ 399 if (hart == boot_hart) 400 return (1); 401 402 /* 403 * Rotate the CPU IDs to put the boot CPU as CPU 0. 404 * We keep the other CPUs ordered. 405 */ 406 cpuid = hart; 407 if (cpuid < boot_hart) 408 cpuid += mp_maxid + 1; 409 cpuid -= boot_hart; 410 411 /* Check if we are able to start this cpu */ 412 if (cpuid > mp_maxid) 413 return (0); 414 415 pcpup = &__pcpu[cpuid]; 416 pcpu_init(pcpup, cpuid, sizeof(struct pcpu)); 417 pcpup->pc_hart = hart; 418 419 dpcpu[cpuid - 1] = (void *)kmem_malloc(DPCPU_SIZE, M_WAITOK | M_ZERO); 420 dpcpu_init(dpcpu[cpuid - 1], cpuid); 421 422 printf("Starting CPU %u (hart %lx)\n", cpuid, hart); 423 __riscv_boot_ap[hart] = 1; 424 425 CPU_SET(cpuid, &all_cpus); 426 CPU_SET(hart, &all_harts); 427 428 return (1); 429 } 430 #endif 431 432 /* Initialize and fire up non-boot processors */ 433 void 434 cpu_mp_start(void) 435 { 436 437 mtx_init(&ap_boot_mtx, "ap boot", NULL, MTX_SPIN); 438 439 CPU_SET(0, &all_cpus); 440 CPU_SET(boot_hart, &all_harts); 441 442 switch(cpu_enum_method) { 443 #ifdef FDT 444 case CPUS_FDT: 445 ofw_cpu_early_foreach(cpu_init_fdt, true); 446 break; 447 #endif 448 case CPUS_UNKNOWN: 449 break; 450 } 451 } 452 453 /* Introduce rest of cores to the world */ 454 void 455 cpu_mp_announce(void) 456 { 457 } 458 459 static boolean_t 460 cpu_check_mmu(u_int id, phandle_t node, u_int addr_size, pcell_t *reg) 461 { 462 463 /* Check if this hart supports MMU. */ 464 if (OF_getproplen(node, "mmu-type") < 0) 465 return (0); 466 467 return (1); 468 } 469 470 void 471 cpu_mp_setmaxid(void) 472 { 473 #ifdef FDT 474 int cores; 475 476 cores = ofw_cpu_early_foreach(cpu_check_mmu, true); 477 if (cores > 0) { 478 cores = MIN(cores, MAXCPU); 479 if (bootverbose) 480 printf("Found %d CPUs in the device tree\n", cores); 481 mp_ncpus = cores; 482 mp_maxid = cores - 1; 483 cpu_enum_method = CPUS_FDT; 484 return; 485 } 486 #endif 487 488 if (bootverbose) 489 printf("No CPU data, limiting to 1 core\n"); 490 mp_ncpus = 1; 491 mp_maxid = 0; 492 } 493