1 /*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2015-2017 Ruslan Bukin <br@bsdpad.com> 4 * All rights reserved. 5 * 6 * Portions of this software were developed by SRI International and the 7 * University of Cambridge Computer Laboratory under DARPA/AFRL contract 8 * FA8750-10-C-0237 ("CTSRD"), as part of the DARPA CRASH research programme. 9 * 10 * Portions of this software were developed by the University of Cambridge 11 * Computer Laboratory as part of the CTSRD Project, with support from the 12 * UK Higher Education Innovation Fund (HEIF). 13 * 14 * Redistribution and use in source and binary forms, with or without 15 * modification, are permitted provided that the following conditions 16 * are met: 17 * 1. Redistributions of source code must retain the above copyright 18 * notice, this list of conditions and the following disclaimer. 19 * 2. Redistributions in binary form must reproduce the above copyright 20 * notice, this list of conditions and the following disclaimer in the 21 * documentation and/or other materials provided with the distribution. 22 * 23 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 24 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 25 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 26 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 27 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 28 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 29 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 30 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 31 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 32 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 33 * SUCH DAMAGE. 34 */ 35 36 #include "opt_platform.h" 37 38 #include <sys/cdefs.h> 39 __FBSDID("$FreeBSD$"); 40 41 #include <sys/param.h> 42 #include <sys/systm.h> 43 #include <sys/boot.h> 44 #include <sys/buf.h> 45 #include <sys/bus.h> 46 #include <sys/cons.h> 47 #include <sys/cpu.h> 48 #include <sys/devmap.h> 49 #include <sys/exec.h> 50 #include <sys/imgact.h> 51 #include <sys/kdb.h> 52 #include <sys/kernel.h> 53 #include <sys/ktr.h> 54 #include <sys/limits.h> 55 #include <sys/linker.h> 56 #include <sys/msgbuf.h> 57 #include <sys/pcpu.h> 58 #include <sys/physmem.h> 59 #include <sys/proc.h> 60 #include <sys/ptrace.h> 61 #include <sys/reboot.h> 62 #include <sys/rwlock.h> 63 #include <sys/sched.h> 64 #include <sys/signalvar.h> 65 #include <sys/syscallsubr.h> 66 #include <sys/sysent.h> 67 #include <sys/sysproto.h> 68 #include <sys/tslog.h> 69 #include <sys/ucontext.h> 70 #include <sys/vmmeter.h> 71 72 #include <vm/vm.h> 73 #include <vm/vm_param.h> 74 #include <vm/vm_kern.h> 75 #include <vm/vm_object.h> 76 #include <vm/vm_page.h> 77 #include <vm/vm_phys.h> 78 #include <vm/pmap.h> 79 #include <vm/vm_map.h> 80 #include <vm/vm_pager.h> 81 82 #include <machine/cpu.h> 83 #include <machine/intr.h> 84 #include <machine/kdb.h> 85 #include <machine/machdep.h> 86 #include <machine/metadata.h> 87 #include <machine/pcb.h> 88 #include <machine/pte.h> 89 #include <machine/reg.h> 90 #include <machine/riscvreg.h> 91 #include <machine/sbi.h> 92 #include <machine/trap.h> 93 #include <machine/vmparam.h> 94 95 #ifdef FPE 96 #include <machine/fpe.h> 97 #endif 98 99 #ifdef FDT 100 #include <contrib/libfdt/libfdt.h> 101 #include <dev/fdt/fdt_common.h> 102 #include <dev/ofw/openfirm.h> 103 #endif 104 105 static void get_fpcontext(struct thread *td, mcontext_t *mcp); 106 static void set_fpcontext(struct thread *td, mcontext_t *mcp); 107 108 struct pcpu __pcpu[MAXCPU]; 109 110 static struct trapframe proc0_tf; 111 112 int early_boot = 1; 113 int cold = 1; 114 115 #define DTB_SIZE_MAX (1024 * 1024) 116 117 vm_paddr_t physmap[PHYS_AVAIL_ENTRIES]; 118 u_int physmap_idx; 119 120 struct kva_md_info kmi; 121 122 int64_t dcache_line_size; /* The minimum D cache line size */ 123 int64_t icache_line_size; /* The minimum I cache line size */ 124 int64_t idcache_line_size; /* The minimum cache line size */ 125 126 #define BOOT_HART_INVALID 0xffffffff 127 uint32_t boot_hart = BOOT_HART_INVALID; /* The hart we booted on. */ 128 129 cpuset_t all_harts; 130 131 extern int *end; 132 133 static void 134 cpu_startup(void *dummy) 135 { 136 137 sbi_print_version(); 138 identify_cpu(); 139 140 printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem), 141 ptoa((uintmax_t)realmem) / (1024 * 1024)); 142 143 /* 144 * Display any holes after the first chunk of extended memory. 145 */ 146 if (bootverbose) { 147 int indx; 148 149 printf("Physical memory chunk(s):\n"); 150 for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) { 151 vm_paddr_t size; 152 153 size = phys_avail[indx + 1] - phys_avail[indx]; 154 printf( 155 "0x%016jx - 0x%016jx, %ju bytes (%ju pages)\n", 156 (uintmax_t)phys_avail[indx], 157 (uintmax_t)phys_avail[indx + 1] - 1, 158 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE); 159 } 160 } 161 162 vm_ksubmap_init(&kmi); 163 164 printf("avail memory = %ju (%ju MB)\n", 165 ptoa((uintmax_t)vm_free_count()), 166 ptoa((uintmax_t)vm_free_count()) / (1024 * 1024)); 167 if (bootverbose) 168 devmap_print_table(); 169 170 bufinit(); 171 vm_pager_bufferinit(); 172 } 173 174 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 175 176 int 177 cpu_idle_wakeup(int cpu) 178 { 179 180 return (0); 181 } 182 183 int 184 fill_regs(struct thread *td, struct reg *regs) 185 { 186 struct trapframe *frame; 187 188 frame = td->td_frame; 189 regs->sepc = frame->tf_sepc; 190 regs->sstatus = frame->tf_sstatus; 191 regs->ra = frame->tf_ra; 192 regs->sp = frame->tf_sp; 193 regs->gp = frame->tf_gp; 194 regs->tp = frame->tf_tp; 195 196 memcpy(regs->t, frame->tf_t, sizeof(regs->t)); 197 memcpy(regs->s, frame->tf_s, sizeof(regs->s)); 198 memcpy(regs->a, frame->tf_a, sizeof(regs->a)); 199 200 return (0); 201 } 202 203 int 204 set_regs(struct thread *td, struct reg *regs) 205 { 206 struct trapframe *frame; 207 208 frame = td->td_frame; 209 frame->tf_sepc = regs->sepc; 210 frame->tf_ra = regs->ra; 211 frame->tf_sp = regs->sp; 212 frame->tf_gp = regs->gp; 213 frame->tf_tp = regs->tp; 214 215 memcpy(frame->tf_t, regs->t, sizeof(frame->tf_t)); 216 memcpy(frame->tf_s, regs->s, sizeof(frame->tf_s)); 217 memcpy(frame->tf_a, regs->a, sizeof(frame->tf_a)); 218 219 return (0); 220 } 221 222 int 223 fill_fpregs(struct thread *td, struct fpreg *regs) 224 { 225 #ifdef FPE 226 struct pcb *pcb; 227 228 pcb = td->td_pcb; 229 230 if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 231 /* 232 * If we have just been running FPE instructions we will 233 * need to save the state to memcpy it below. 234 */ 235 if (td == curthread) 236 fpe_state_save(td); 237 238 memcpy(regs->fp_x, pcb->pcb_x, sizeof(regs->fp_x)); 239 regs->fp_fcsr = pcb->pcb_fcsr; 240 } else 241 #endif 242 memset(regs, 0, sizeof(*regs)); 243 244 return (0); 245 } 246 247 int 248 set_fpregs(struct thread *td, struct fpreg *regs) 249 { 250 #ifdef FPE 251 struct trapframe *frame; 252 struct pcb *pcb; 253 254 frame = td->td_frame; 255 pcb = td->td_pcb; 256 257 memcpy(pcb->pcb_x, regs->fp_x, sizeof(regs->fp_x)); 258 pcb->pcb_fcsr = regs->fp_fcsr; 259 pcb->pcb_fpflags |= PCB_FP_STARTED; 260 frame->tf_sstatus &= ~SSTATUS_FS_MASK; 261 frame->tf_sstatus |= SSTATUS_FS_CLEAN; 262 #endif 263 264 return (0); 265 } 266 267 int 268 fill_dbregs(struct thread *td, struct dbreg *regs) 269 { 270 271 panic("fill_dbregs"); 272 } 273 274 int 275 set_dbregs(struct thread *td, struct dbreg *regs) 276 { 277 278 panic("set_dbregs"); 279 } 280 281 int 282 ptrace_set_pc(struct thread *td, u_long addr) 283 { 284 285 td->td_frame->tf_sepc = addr; 286 return (0); 287 } 288 289 int 290 ptrace_single_step(struct thread *td) 291 { 292 293 /* TODO; */ 294 return (EOPNOTSUPP); 295 } 296 297 int 298 ptrace_clear_single_step(struct thread *td) 299 { 300 301 /* TODO; */ 302 return (EOPNOTSUPP); 303 } 304 305 void 306 exec_setregs(struct thread *td, struct image_params *imgp, uintptr_t stack) 307 { 308 struct trapframe *tf; 309 struct pcb *pcb; 310 311 tf = td->td_frame; 312 pcb = td->td_pcb; 313 314 memset(tf, 0, sizeof(struct trapframe)); 315 316 tf->tf_a[0] = stack; 317 tf->tf_sp = STACKALIGN(stack); 318 tf->tf_ra = imgp->entry_addr; 319 tf->tf_sepc = imgp->entry_addr; 320 321 pcb->pcb_fpflags &= ~PCB_FP_STARTED; 322 } 323 324 /* Sanity check these are the same size, they will be memcpy'd to and fro */ 325 CTASSERT(sizeof(((struct trapframe *)0)->tf_a) == 326 sizeof((struct gpregs *)0)->gp_a); 327 CTASSERT(sizeof(((struct trapframe *)0)->tf_s) == 328 sizeof((struct gpregs *)0)->gp_s); 329 CTASSERT(sizeof(((struct trapframe *)0)->tf_t) == 330 sizeof((struct gpregs *)0)->gp_t); 331 CTASSERT(sizeof(((struct trapframe *)0)->tf_a) == 332 sizeof((struct reg *)0)->a); 333 CTASSERT(sizeof(((struct trapframe *)0)->tf_s) == 334 sizeof((struct reg *)0)->s); 335 CTASSERT(sizeof(((struct trapframe *)0)->tf_t) == 336 sizeof((struct reg *)0)->t); 337 338 /* Support for FDT configurations only. */ 339 CTASSERT(FDT); 340 341 int 342 get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) 343 { 344 struct trapframe *tf = td->td_frame; 345 346 memcpy(mcp->mc_gpregs.gp_t, tf->tf_t, sizeof(mcp->mc_gpregs.gp_t)); 347 memcpy(mcp->mc_gpregs.gp_s, tf->tf_s, sizeof(mcp->mc_gpregs.gp_s)); 348 memcpy(mcp->mc_gpregs.gp_a, tf->tf_a, sizeof(mcp->mc_gpregs.gp_a)); 349 350 if (clear_ret & GET_MC_CLEAR_RET) { 351 mcp->mc_gpregs.gp_a[0] = 0; 352 mcp->mc_gpregs.gp_t[0] = 0; /* clear syscall error */ 353 } 354 355 mcp->mc_gpregs.gp_ra = tf->tf_ra; 356 mcp->mc_gpregs.gp_sp = tf->tf_sp; 357 mcp->mc_gpregs.gp_gp = tf->tf_gp; 358 mcp->mc_gpregs.gp_tp = tf->tf_tp; 359 mcp->mc_gpregs.gp_sepc = tf->tf_sepc; 360 mcp->mc_gpregs.gp_sstatus = tf->tf_sstatus; 361 get_fpcontext(td, mcp); 362 363 return (0); 364 } 365 366 int 367 set_mcontext(struct thread *td, mcontext_t *mcp) 368 { 369 struct trapframe *tf; 370 371 tf = td->td_frame; 372 373 /* 374 * Permit changes to the USTATUS bits of SSTATUS. 375 * 376 * Ignore writes to read-only bits (SD, XS). 377 * 378 * Ignore writes to the FS field as set_fpcontext() will set 379 * it explicitly. 380 */ 381 if (((mcp->mc_gpregs.gp_sstatus ^ tf->tf_sstatus) & 382 ~(SSTATUS_SD | SSTATUS_XS_MASK | SSTATUS_FS_MASK | SSTATUS_UPIE | 383 SSTATUS_UIE)) != 0) 384 return (EINVAL); 385 386 memcpy(tf->tf_t, mcp->mc_gpregs.gp_t, sizeof(tf->tf_t)); 387 memcpy(tf->tf_s, mcp->mc_gpregs.gp_s, sizeof(tf->tf_s)); 388 memcpy(tf->tf_a, mcp->mc_gpregs.gp_a, sizeof(tf->tf_a)); 389 390 tf->tf_ra = mcp->mc_gpregs.gp_ra; 391 tf->tf_sp = mcp->mc_gpregs.gp_sp; 392 tf->tf_gp = mcp->mc_gpregs.gp_gp; 393 tf->tf_sepc = mcp->mc_gpregs.gp_sepc; 394 tf->tf_sstatus = mcp->mc_gpregs.gp_sstatus; 395 set_fpcontext(td, mcp); 396 397 return (0); 398 } 399 400 static void 401 get_fpcontext(struct thread *td, mcontext_t *mcp) 402 { 403 #ifdef FPE 404 struct pcb *curpcb; 405 406 critical_enter(); 407 408 curpcb = curthread->td_pcb; 409 410 KASSERT(td->td_pcb == curpcb, ("Invalid fpe pcb")); 411 412 if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 413 /* 414 * If we have just been running FPE instructions we will 415 * need to save the state to memcpy it below. 416 */ 417 fpe_state_save(td); 418 419 KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, 420 ("Non-userspace FPE flags set in get_fpcontext")); 421 memcpy(mcp->mc_fpregs.fp_x, curpcb->pcb_x, 422 sizeof(mcp->mc_fpregs)); 423 mcp->mc_fpregs.fp_fcsr = curpcb->pcb_fcsr; 424 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; 425 mcp->mc_flags |= _MC_FP_VALID; 426 } 427 428 critical_exit(); 429 #endif 430 } 431 432 static void 433 set_fpcontext(struct thread *td, mcontext_t *mcp) 434 { 435 #ifdef FPE 436 struct pcb *curpcb; 437 #endif 438 439 td->td_frame->tf_sstatus &= ~SSTATUS_FS_MASK; 440 td->td_frame->tf_sstatus |= SSTATUS_FS_OFF; 441 442 #ifdef FPE 443 critical_enter(); 444 445 if ((mcp->mc_flags & _MC_FP_VALID) != 0) { 446 curpcb = curthread->td_pcb; 447 /* FPE usage is enabled, override registers. */ 448 memcpy(curpcb->pcb_x, mcp->mc_fpregs.fp_x, 449 sizeof(mcp->mc_fpregs)); 450 curpcb->pcb_fcsr = mcp->mc_fpregs.fp_fcsr; 451 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK; 452 td->td_frame->tf_sstatus |= SSTATUS_FS_CLEAN; 453 } 454 455 critical_exit(); 456 #endif 457 } 458 459 void 460 cpu_idle(int busy) 461 { 462 463 spinlock_enter(); 464 if (!busy) 465 cpu_idleclock(); 466 if (!sched_runnable()) 467 __asm __volatile( 468 "fence \n" 469 "wfi \n"); 470 if (!busy) 471 cpu_activeclock(); 472 spinlock_exit(); 473 } 474 475 void 476 cpu_halt(void) 477 { 478 479 /* 480 * Try to power down using the HSM SBI extension and fall back to a 481 * simple wfi loop. 482 */ 483 intr_disable(); 484 if (sbi_probe_extension(SBI_EXT_ID_HSM) != 0) 485 sbi_hsm_hart_stop(); 486 for (;;) 487 __asm __volatile("wfi"); 488 /* NOTREACHED */ 489 } 490 491 /* 492 * Flush the D-cache for non-DMA I/O so that the I-cache can 493 * be made coherent later. 494 */ 495 void 496 cpu_flush_dcache(void *ptr, size_t len) 497 { 498 499 /* TBD */ 500 } 501 502 /* Get current clock frequency for the given CPU ID. */ 503 int 504 cpu_est_clockrate(int cpu_id, uint64_t *rate) 505 { 506 507 panic("cpu_est_clockrate"); 508 } 509 510 void 511 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 512 { 513 } 514 515 void 516 spinlock_enter(void) 517 { 518 struct thread *td; 519 register_t reg; 520 521 td = curthread; 522 if (td->td_md.md_spinlock_count == 0) { 523 reg = intr_disable(); 524 td->td_md.md_spinlock_count = 1; 525 td->td_md.md_saved_sstatus_ie = reg; 526 critical_enter(); 527 } else 528 td->td_md.md_spinlock_count++; 529 } 530 531 void 532 spinlock_exit(void) 533 { 534 struct thread *td; 535 register_t sstatus_ie; 536 537 td = curthread; 538 sstatus_ie = td->td_md.md_saved_sstatus_ie; 539 td->td_md.md_spinlock_count--; 540 if (td->td_md.md_spinlock_count == 0) { 541 critical_exit(); 542 intr_restore(sstatus_ie); 543 } 544 } 545 546 #ifndef _SYS_SYSPROTO_H_ 547 struct sigreturn_args { 548 ucontext_t *ucp; 549 }; 550 #endif 551 552 int 553 sys_sigreturn(struct thread *td, struct sigreturn_args *uap) 554 { 555 ucontext_t uc; 556 int error; 557 558 if (copyin(uap->sigcntxp, &uc, sizeof(uc))) 559 return (EFAULT); 560 561 error = set_mcontext(td, &uc.uc_mcontext); 562 if (error != 0) 563 return (error); 564 565 /* Restore signal mask. */ 566 kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); 567 568 return (EJUSTRETURN); 569 } 570 571 /* 572 * Construct a PCB from a trapframe. This is called from kdb_trap() where 573 * we want to start a backtrace from the function that caused us to enter 574 * the debugger. We have the context in the trapframe, but base the trace 575 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 576 * enough for a backtrace. 577 */ 578 void 579 makectx(struct trapframe *tf, struct pcb *pcb) 580 { 581 582 memcpy(pcb->pcb_s, tf->tf_s, sizeof(tf->tf_s)); 583 584 pcb->pcb_ra = tf->tf_sepc; 585 pcb->pcb_sp = tf->tf_sp; 586 pcb->pcb_gp = tf->tf_gp; 587 pcb->pcb_tp = tf->tf_tp; 588 } 589 590 void 591 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) 592 { 593 struct sigframe *fp, frame; 594 struct sysentvec *sysent; 595 struct trapframe *tf; 596 struct sigacts *psp; 597 struct thread *td; 598 struct proc *p; 599 int onstack; 600 int sig; 601 602 td = curthread; 603 p = td->td_proc; 604 PROC_LOCK_ASSERT(p, MA_OWNED); 605 606 sig = ksi->ksi_signo; 607 psp = p->p_sigacts; 608 mtx_assert(&psp->ps_mtx, MA_OWNED); 609 610 tf = td->td_frame; 611 onstack = sigonstack(tf->tf_sp); 612 613 CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, 614 catcher, sig); 615 616 /* Allocate and validate space for the signal handler context. */ 617 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && 618 SIGISMEMBER(psp->ps_sigonstack, sig)) { 619 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + 620 td->td_sigstk.ss_size); 621 } else { 622 fp = (struct sigframe *)td->td_frame->tf_sp; 623 } 624 625 /* Make room, keeping the stack aligned */ 626 fp--; 627 fp = (struct sigframe *)STACKALIGN(fp); 628 629 /* Fill in the frame to copy out */ 630 bzero(&frame, sizeof(frame)); 631 get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); 632 frame.sf_si = ksi->ksi_info; 633 frame.sf_uc.uc_sigmask = *mask; 634 frame.sf_uc.uc_stack = td->td_sigstk; 635 frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) != 0 ? 636 (onstack ? SS_ONSTACK : 0) : SS_DISABLE; 637 mtx_unlock(&psp->ps_mtx); 638 PROC_UNLOCK(td->td_proc); 639 640 /* Copy the sigframe out to the user's stack. */ 641 if (copyout(&frame, fp, sizeof(*fp)) != 0) { 642 /* Process has trashed its stack. Kill it. */ 643 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); 644 PROC_LOCK(p); 645 sigexit(td, SIGILL); 646 } 647 648 tf->tf_a[0] = sig; 649 tf->tf_a[1] = (register_t)&fp->sf_si; 650 tf->tf_a[2] = (register_t)&fp->sf_uc; 651 652 tf->tf_sepc = (register_t)catcher; 653 tf->tf_sp = (register_t)fp; 654 655 sysent = p->p_sysent; 656 if (sysent->sv_sigcode_base != 0) 657 tf->tf_ra = (register_t)sysent->sv_sigcode_base; 658 else 659 tf->tf_ra = (register_t)(sysent->sv_psstrings - 660 *(sysent->sv_szsigcode)); 661 662 CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_sepc, 663 tf->tf_sp); 664 665 PROC_LOCK(p); 666 mtx_lock(&psp->ps_mtx); 667 } 668 669 static void 670 init_proc0(vm_offset_t kstack) 671 { 672 struct pcpu *pcpup; 673 674 pcpup = &__pcpu[0]; 675 676 proc_linkup0(&proc0, &thread0); 677 thread0.td_kstack = kstack; 678 thread0.td_kstack_pages = KSTACK_PAGES; 679 thread0.td_pcb = (struct pcb *)(thread0.td_kstack + 680 thread0.td_kstack_pages * PAGE_SIZE) - 1; 681 thread0.td_pcb->pcb_fpflags = 0; 682 thread0.td_frame = &proc0_tf; 683 pcpup->pc_curpcb = thread0.td_pcb; 684 } 685 686 #ifdef FDT 687 static void 688 try_load_dtb(caddr_t kmdp) 689 { 690 vm_offset_t dtbp; 691 692 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 693 694 #if defined(FDT_DTB_STATIC) 695 /* 696 * In case the device tree blob was not retrieved (from metadata) try 697 * to use the statically embedded one. 698 */ 699 if (dtbp == (vm_offset_t)NULL) 700 dtbp = (vm_offset_t)&fdt_static_dtb; 701 #endif 702 703 if (dtbp == (vm_offset_t)NULL) { 704 printf("ERROR loading DTB\n"); 705 return; 706 } 707 708 if (OF_install(OFW_FDT, 0) == FALSE) 709 panic("Cannot install FDT"); 710 711 if (OF_init((void *)dtbp) != 0) 712 panic("OF_init failed with the found device tree"); 713 } 714 #endif 715 716 static void 717 cache_setup(void) 718 { 719 720 /* TODO */ 721 722 dcache_line_size = 0; 723 icache_line_size = 0; 724 idcache_line_size = 0; 725 } 726 727 /* 728 * Fake up a boot descriptor table. 729 */ 730 static void 731 fake_preload_metadata(struct riscv_bootparams *rvbp) 732 { 733 static uint32_t fake_preload[48]; 734 vm_offset_t lastaddr; 735 size_t fake_size, dtb_size; 736 737 #define PRELOAD_PUSH_VALUE(type, value) do { \ 738 *(type *)((char *)fake_preload + fake_size) = (value); \ 739 fake_size += sizeof(type); \ 740 } while (0) 741 742 #define PRELOAD_PUSH_STRING(str) do { \ 743 uint32_t ssize; \ 744 ssize = strlen(str) + 1; \ 745 PRELOAD_PUSH_VALUE(uint32_t, ssize); \ 746 strcpy(((char *)fake_preload + fake_size), str); \ 747 fake_size += ssize; \ 748 fake_size = roundup(fake_size, sizeof(u_long)); \ 749 } while (0) 750 751 fake_size = 0; 752 lastaddr = (vm_offset_t)&end; 753 754 PRELOAD_PUSH_VALUE(uint32_t, MODINFO_NAME); 755 PRELOAD_PUSH_STRING("kernel"); 756 PRELOAD_PUSH_VALUE(uint32_t, MODINFO_TYPE); 757 PRELOAD_PUSH_STRING("elf kernel"); 758 759 PRELOAD_PUSH_VALUE(uint32_t, MODINFO_ADDR); 760 PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t)); 761 PRELOAD_PUSH_VALUE(uint64_t, KERNBASE); 762 763 PRELOAD_PUSH_VALUE(uint32_t, MODINFO_SIZE); 764 PRELOAD_PUSH_VALUE(uint32_t, sizeof(size_t)); 765 PRELOAD_PUSH_VALUE(uint64_t, (size_t)((vm_offset_t)&end - KERNBASE)); 766 767 /* Copy the DTB to KVA space. */ 768 lastaddr = roundup(lastaddr, sizeof(int)); 769 PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_DTBP); 770 PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t)); 771 PRELOAD_PUSH_VALUE(vm_offset_t, lastaddr); 772 dtb_size = fdt_totalsize(rvbp->dtbp_virt); 773 memmove((void *)lastaddr, (const void *)rvbp->dtbp_virt, dtb_size); 774 lastaddr = roundup(lastaddr + dtb_size, sizeof(int)); 775 776 PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_KERNEND); 777 PRELOAD_PUSH_VALUE(uint32_t, sizeof(vm_offset_t)); 778 PRELOAD_PUSH_VALUE(vm_offset_t, lastaddr); 779 780 PRELOAD_PUSH_VALUE(uint32_t, MODINFO_METADATA | MODINFOMD_HOWTO); 781 PRELOAD_PUSH_VALUE(uint32_t, sizeof(int)); 782 PRELOAD_PUSH_VALUE(int, RB_VERBOSE); 783 784 /* End marker */ 785 PRELOAD_PUSH_VALUE(uint32_t, 0); 786 PRELOAD_PUSH_VALUE(uint32_t, 0); 787 preload_metadata = (caddr_t)fake_preload; 788 789 /* Check if bootloader clobbered part of the kernel with the DTB. */ 790 KASSERT(rvbp->dtbp_phys + dtb_size <= rvbp->kern_phys || 791 rvbp->dtbp_phys >= rvbp->kern_phys + (lastaddr - KERNBASE), 792 ("FDT (%lx-%lx) and kernel (%lx-%lx) overlap", rvbp->dtbp_phys, 793 rvbp->dtbp_phys + dtb_size, rvbp->kern_phys, 794 rvbp->kern_phys + (lastaddr - KERNBASE))); 795 KASSERT(fake_size < sizeof(fake_preload), 796 ("Too many fake_preload items")); 797 798 if (boothowto & RB_VERBOSE) 799 printf("FDT phys (%lx-%lx), kernel phys (%lx-%lx)\n", 800 rvbp->dtbp_phys, rvbp->dtbp_phys + dtb_size, 801 rvbp->kern_phys, rvbp->kern_phys + (lastaddr - KERNBASE)); 802 } 803 804 #ifdef FDT 805 static void 806 parse_fdt_bootargs(void) 807 { 808 char bootargs[512]; 809 810 bootargs[sizeof(bootargs) - 1] = '\0'; 811 if (fdt_get_chosen_bootargs(bootargs, sizeof(bootargs) - 1) == 0) { 812 boothowto |= boot_parse_cmdline(bootargs); 813 } 814 } 815 #endif 816 817 static vm_offset_t 818 parse_metadata(void) 819 { 820 caddr_t kmdp; 821 vm_offset_t lastaddr; 822 #ifdef DDB 823 vm_offset_t ksym_start, ksym_end; 824 #endif 825 char *kern_envp; 826 827 /* Find the kernel address */ 828 kmdp = preload_search_by_type("elf kernel"); 829 if (kmdp == NULL) 830 kmdp = preload_search_by_type("elf64 kernel"); 831 KASSERT(kmdp != NULL, ("No preload metadata found!")); 832 833 /* Read the boot metadata */ 834 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); 835 lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); 836 kern_envp = MD_FETCH(kmdp, MODINFOMD_ENVP, char *); 837 if (kern_envp != NULL) 838 init_static_kenv(kern_envp, 0); 839 #ifdef DDB 840 ksym_start = MD_FETCH(kmdp, MODINFOMD_SSYM, uintptr_t); 841 ksym_end = MD_FETCH(kmdp, MODINFOMD_ESYM, uintptr_t); 842 db_fetch_ksymtab(ksym_start, ksym_end); 843 #endif 844 #ifdef FDT 845 try_load_dtb(kmdp); 846 if (kern_envp == NULL) 847 parse_fdt_bootargs(); 848 #endif 849 return (lastaddr); 850 } 851 852 void 853 initriscv(struct riscv_bootparams *rvbp) 854 { 855 struct mem_region mem_regions[FDT_MEM_REGIONS]; 856 struct pcpu *pcpup; 857 int mem_regions_sz; 858 vm_offset_t lastaddr; 859 vm_size_t kernlen; 860 #ifdef FDT 861 phandle_t chosen; 862 uint32_t hart; 863 #endif 864 865 TSRAW(&thread0, TS_ENTER, __func__, NULL); 866 867 /* Set the pcpu data, this is needed by pmap_bootstrap */ 868 pcpup = &__pcpu[0]; 869 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 870 871 /* Set the pcpu pointer */ 872 __asm __volatile("mv tp, %0" :: "r"(pcpup)); 873 874 PCPU_SET(curthread, &thread0); 875 876 /* Initialize SBI interface. */ 877 sbi_init(); 878 879 /* Parse the boot metadata. */ 880 if (rvbp->modulep != 0) { 881 preload_metadata = (caddr_t)rvbp->modulep; 882 } else { 883 fake_preload_metadata(rvbp); 884 } 885 lastaddr = parse_metadata(); 886 887 #ifdef FDT 888 /* 889 * Look for the boot hart ID. This was either passed in directly from 890 * the SBI firmware and handled by locore, or was stored in the device 891 * tree by an earlier boot stage. 892 */ 893 chosen = OF_finddevice("/chosen"); 894 if (OF_getencprop(chosen, "boot-hartid", &hart, sizeof(hart)) != -1) { 895 boot_hart = hart; 896 } 897 #endif 898 if (boot_hart == BOOT_HART_INVALID) { 899 panic("Boot hart ID was not properly set"); 900 } 901 pcpup->pc_hart = boot_hart; 902 903 #ifdef FDT 904 /* 905 * Exclude reserved memory specified by the device tree. Typically, 906 * this contains an entry for memory used by the runtime SBI firmware. 907 */ 908 if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0) { 909 physmem_exclude_regions(mem_regions, mem_regions_sz, 910 EXFLAG_NODUMP | EXFLAG_NOALLOC); 911 } 912 913 /* Grab physical memory regions information from device tree. */ 914 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, NULL) != 0) { 915 panic("Cannot get physical memory regions"); 916 } 917 physmem_hardware_regions(mem_regions, mem_regions_sz); 918 #endif 919 920 /* Do basic tuning, hz etc */ 921 init_param1(); 922 923 cache_setup(); 924 925 /* Bootstrap enough of pmap to enter the kernel proper */ 926 kernlen = (lastaddr - KERNBASE); 927 pmap_bootstrap(rvbp->kern_l1pt, rvbp->kern_phys, kernlen); 928 929 #ifdef FDT 930 /* 931 * XXX: Exclude the lowest 2MB of physical memory, if it hasn't been 932 * already, as this area is assumed to contain the SBI firmware. This 933 * is a little fragile, but it is consistent with the platforms we 934 * support so far. 935 * 936 * TODO: remove this when the all regular booting methods properly 937 * report their reserved memory in the device tree. 938 */ 939 if (mem_regions[0].mr_start == physmap[0]) { 940 physmem_exclude_region(mem_regions[0].mr_start, L2_SIZE, 941 EXFLAG_NODUMP | EXFLAG_NOALLOC); 942 } 943 #endif 944 physmem_init_kernel_globals(); 945 946 /* Establish static device mappings */ 947 devmap_bootstrap(0, NULL); 948 949 cninit(); 950 951 init_proc0(rvbp->kern_stack); 952 953 msgbufinit(msgbufp, msgbufsize); 954 mutex_init(); 955 init_param2(physmem); 956 kdb_init(); 957 958 if (boothowto & RB_VERBOSE) 959 physmem_print_tables(); 960 961 early_boot = 0; 962 963 TSEXIT(); 964 } 965 966 #undef bzero 967 void 968 bzero(void *buf, size_t len) 969 { 970 uint8_t *p; 971 972 p = buf; 973 while(len-- > 0) 974 *p++ = 0; 975 } 976