1 /*- 2 * Copyright (c) 2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28 #include "opt_acpi.h" 29 #include "opt_platform.h" 30 #include "opt_ddb.h" 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/buf.h> 38 #include <sys/bus.h> 39 #include <sys/cons.h> 40 #include <sys/cpu.h> 41 #include <sys/devmap.h> 42 #include <sys/efi.h> 43 #include <sys/exec.h> 44 #include <sys/imgact.h> 45 #include <sys/kdb.h> 46 #include <sys/kernel.h> 47 #include <sys/limits.h> 48 #include <sys/linker.h> 49 #include <sys/msgbuf.h> 50 #include <sys/pcpu.h> 51 #include <sys/proc.h> 52 #include <sys/ptrace.h> 53 #include <sys/reboot.h> 54 #include <sys/rwlock.h> 55 #include <sys/sched.h> 56 #include <sys/signalvar.h> 57 #include <sys/syscallsubr.h> 58 #include <sys/sysent.h> 59 #include <sys/sysproto.h> 60 #include <sys/ucontext.h> 61 #include <sys/vdso.h> 62 63 #include <vm/vm.h> 64 #include <vm/vm_kern.h> 65 #include <vm/vm_object.h> 66 #include <vm/vm_page.h> 67 #include <vm/pmap.h> 68 #include <vm/vm_map.h> 69 #include <vm/vm_pager.h> 70 71 #include <machine/armreg.h> 72 #include <machine/cpu.h> 73 #include <machine/debug_monitor.h> 74 #include <machine/kdb.h> 75 #include <machine/machdep.h> 76 #include <machine/metadata.h> 77 #include <machine/md_var.h> 78 #include <machine/pcb.h> 79 #include <machine/reg.h> 80 #include <machine/vmparam.h> 81 82 #ifdef VFP 83 #include <machine/vfp.h> 84 #endif 85 86 #ifdef DEV_ACPI 87 #include <contrib/dev/acpica/include/acpi.h> 88 #include <machine/acpica_machdep.h> 89 #endif 90 91 #ifdef FDT 92 #include <dev/fdt/fdt_common.h> 93 #include <dev/ofw/openfirm.h> 94 #endif 95 96 97 enum arm64_bus arm64_bus_method = ARM64_BUS_NONE; 98 99 struct pcpu __pcpu[MAXCPU]; 100 101 static struct trapframe proc0_tf; 102 103 vm_paddr_t phys_avail[PHYS_AVAIL_SIZE + 2]; 104 vm_paddr_t dump_avail[PHYS_AVAIL_SIZE + 2]; 105 106 int early_boot = 1; 107 int cold = 1; 108 long realmem = 0; 109 long Maxmem = 0; 110 111 #define PHYSMAP_SIZE (2 * (VM_PHYSSEG_MAX - 1)) 112 vm_paddr_t physmap[PHYSMAP_SIZE]; 113 u_int physmap_idx; 114 115 struct kva_md_info kmi; 116 117 int64_t dcache_line_size; /* The minimum D cache line size */ 118 int64_t icache_line_size; /* The minimum I cache line size */ 119 int64_t idcache_line_size; /* The minimum cache line size */ 120 int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 121 int has_pan; 122 123 /* pagezero_* implementations are provided in support.S */ 124 void pagezero_simple(void *); 125 void pagezero_cache(void *); 126 127 /* pagezero_simple is default pagezero */ 128 void (*pagezero)(void *p) = pagezero_simple; 129 130 static void 131 pan_setup(void) 132 { 133 uint64_t id_aa64mfr1; 134 135 id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 136 if (ID_AA64MMFR1_PAN(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) 137 has_pan = 1; 138 } 139 140 void 141 pan_enable(void) 142 { 143 144 /* 145 * The LLVM integrated assembler doesn't understand the PAN 146 * PSTATE field. Because of this we need to manually create 147 * the instruction in an asm block. This is equivalent to: 148 * msr pan, #1 149 * 150 * This sets the PAN bit, stopping the kernel from accessing 151 * memory when userspace can also access it unless the kernel 152 * uses the userspace load/store instructions. 153 */ 154 if (has_pan) { 155 WRITE_SPECIALREG(sctlr_el1, 156 READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); 157 __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); 158 } 159 } 160 161 static void 162 cpu_startup(void *dummy) 163 { 164 165 identify_cpu(); 166 167 vm_ksubmap_init(&kmi); 168 bufinit(); 169 vm_pager_bufferinit(); 170 } 171 172 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 173 174 int 175 cpu_idle_wakeup(int cpu) 176 { 177 178 return (0); 179 } 180 181 int 182 fill_regs(struct thread *td, struct reg *regs) 183 { 184 struct trapframe *frame; 185 186 frame = td->td_frame; 187 regs->sp = frame->tf_sp; 188 regs->lr = frame->tf_lr; 189 regs->elr = frame->tf_elr; 190 regs->spsr = frame->tf_spsr; 191 192 memcpy(regs->x, frame->tf_x, sizeof(regs->x)); 193 194 return (0); 195 } 196 197 int 198 set_regs(struct thread *td, struct reg *regs) 199 { 200 struct trapframe *frame; 201 202 frame = td->td_frame; 203 frame->tf_sp = regs->sp; 204 frame->tf_lr = regs->lr; 205 frame->tf_elr = regs->elr; 206 frame->tf_spsr = regs->spsr; 207 208 memcpy(frame->tf_x, regs->x, sizeof(frame->tf_x)); 209 210 return (0); 211 } 212 213 int 214 fill_fpregs(struct thread *td, struct fpreg *regs) 215 { 216 #ifdef VFP 217 struct pcb *pcb; 218 219 pcb = td->td_pcb; 220 if ((pcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 221 /* 222 * If we have just been running VFP instructions we will 223 * need to save the state to memcpy it below. 224 */ 225 vfp_save_state(td, pcb); 226 227 KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, 228 ("Called fill_fpregs while the kernel is using the VFP")); 229 memcpy(regs->fp_q, pcb->pcb_fpustate.vfp_regs, 230 sizeof(regs->fp_q)); 231 regs->fp_cr = pcb->pcb_fpustate.vfp_fpcr; 232 regs->fp_sr = pcb->pcb_fpustate.vfp_fpsr; 233 } else 234 #endif 235 memset(regs->fp_q, 0, sizeof(regs->fp_q)); 236 return (0); 237 } 238 239 int 240 set_fpregs(struct thread *td, struct fpreg *regs) 241 { 242 #ifdef VFP 243 struct pcb *pcb; 244 245 pcb = td->td_pcb; 246 KASSERT(pcb->pcb_fpusaved == &pcb->pcb_fpustate, 247 ("Called set_fpregs while the kernel is using the VFP")); 248 memcpy(pcb->pcb_fpustate.vfp_regs, regs->fp_q, sizeof(regs->fp_q)); 249 pcb->pcb_fpustate.vfp_fpcr = regs->fp_cr; 250 pcb->pcb_fpustate.vfp_fpsr = regs->fp_sr; 251 #endif 252 return (0); 253 } 254 255 int 256 fill_dbregs(struct thread *td, struct dbreg *regs) 257 { 258 259 panic("ARM64TODO: fill_dbregs"); 260 } 261 262 int 263 set_dbregs(struct thread *td, struct dbreg *regs) 264 { 265 266 panic("ARM64TODO: set_dbregs"); 267 } 268 269 int 270 ptrace_set_pc(struct thread *td, u_long addr) 271 { 272 273 panic("ARM64TODO: ptrace_set_pc"); 274 return (0); 275 } 276 277 int 278 ptrace_single_step(struct thread *td) 279 { 280 281 td->td_frame->tf_spsr |= PSR_SS; 282 td->td_pcb->pcb_flags |= PCB_SINGLE_STEP; 283 return (0); 284 } 285 286 int 287 ptrace_clear_single_step(struct thread *td) 288 { 289 290 td->td_frame->tf_spsr &= ~PSR_SS; 291 td->td_pcb->pcb_flags &= ~PCB_SINGLE_STEP; 292 return (0); 293 } 294 295 void 296 exec_setregs(struct thread *td, struct image_params *imgp, u_long stack) 297 { 298 struct trapframe *tf = td->td_frame; 299 300 memset(tf, 0, sizeof(struct trapframe)); 301 302 /* 303 * We need to set x0 for init as it doesn't call 304 * cpu_set_syscall_retval to copy the value. We also 305 * need to set td_retval for the cases where we do. 306 */ 307 tf->tf_x[0] = td->td_retval[0] = stack; 308 tf->tf_sp = STACKALIGN(stack); 309 tf->tf_lr = imgp->entry_addr; 310 tf->tf_elr = imgp->entry_addr; 311 } 312 313 /* Sanity check these are the same size, they will be memcpy'd to and fro */ 314 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 315 sizeof((struct gpregs *)0)->gp_x); 316 CTASSERT(sizeof(((struct trapframe *)0)->tf_x) == 317 sizeof((struct reg *)0)->x); 318 319 int 320 get_mcontext(struct thread *td, mcontext_t *mcp, int clear_ret) 321 { 322 struct trapframe *tf = td->td_frame; 323 324 if (clear_ret & GET_MC_CLEAR_RET) { 325 mcp->mc_gpregs.gp_x[0] = 0; 326 mcp->mc_gpregs.gp_spsr = tf->tf_spsr & ~PSR_C; 327 } else { 328 mcp->mc_gpregs.gp_x[0] = tf->tf_x[0]; 329 mcp->mc_gpregs.gp_spsr = tf->tf_spsr; 330 } 331 332 memcpy(&mcp->mc_gpregs.gp_x[1], &tf->tf_x[1], 333 sizeof(mcp->mc_gpregs.gp_x[1]) * (nitems(mcp->mc_gpregs.gp_x) - 1)); 334 335 mcp->mc_gpregs.gp_sp = tf->tf_sp; 336 mcp->mc_gpregs.gp_lr = tf->tf_lr; 337 mcp->mc_gpregs.gp_elr = tf->tf_elr; 338 339 return (0); 340 } 341 342 int 343 set_mcontext(struct thread *td, mcontext_t *mcp) 344 { 345 struct trapframe *tf = td->td_frame; 346 347 memcpy(tf->tf_x, mcp->mc_gpregs.gp_x, sizeof(tf->tf_x)); 348 349 tf->tf_sp = mcp->mc_gpregs.gp_sp; 350 tf->tf_lr = mcp->mc_gpregs.gp_lr; 351 tf->tf_elr = mcp->mc_gpregs.gp_elr; 352 tf->tf_spsr = mcp->mc_gpregs.gp_spsr; 353 354 return (0); 355 } 356 357 static void 358 get_fpcontext(struct thread *td, mcontext_t *mcp) 359 { 360 #ifdef VFP 361 struct pcb *curpcb; 362 363 critical_enter(); 364 365 curpcb = curthread->td_pcb; 366 367 if ((curpcb->pcb_fpflags & PCB_FP_STARTED) != 0) { 368 /* 369 * If we have just been running VFP instructions we will 370 * need to save the state to memcpy it below. 371 */ 372 vfp_save_state(td, curpcb); 373 374 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, 375 ("Called get_fpcontext while the kernel is using the VFP")); 376 KASSERT((curpcb->pcb_fpflags & ~PCB_FP_USERMASK) == 0, 377 ("Non-userspace FPU flags set in get_fpcontext")); 378 memcpy(mcp->mc_fpregs.fp_q, curpcb->pcb_fpustate.vfp_regs, 379 sizeof(mcp->mc_fpregs)); 380 mcp->mc_fpregs.fp_cr = curpcb->pcb_fpustate.vfp_fpcr; 381 mcp->mc_fpregs.fp_sr = curpcb->pcb_fpustate.vfp_fpsr; 382 mcp->mc_fpregs.fp_flags = curpcb->pcb_fpflags; 383 mcp->mc_flags |= _MC_FP_VALID; 384 } 385 386 critical_exit(); 387 #endif 388 } 389 390 static void 391 set_fpcontext(struct thread *td, mcontext_t *mcp) 392 { 393 #ifdef VFP 394 struct pcb *curpcb; 395 396 critical_enter(); 397 398 if ((mcp->mc_flags & _MC_FP_VALID) != 0) { 399 curpcb = curthread->td_pcb; 400 401 /* 402 * Discard any vfp state for the current thread, we 403 * are about to override it. 404 */ 405 vfp_discard(td); 406 407 KASSERT(curpcb->pcb_fpusaved == &curpcb->pcb_fpustate, 408 ("Called set_fpcontext while the kernel is using the VFP")); 409 memcpy(curpcb->pcb_fpustate.vfp_regs, mcp->mc_fpregs.fp_q, 410 sizeof(mcp->mc_fpregs)); 411 curpcb->pcb_fpustate.vfp_fpcr = mcp->mc_fpregs.fp_cr; 412 curpcb->pcb_fpustate.vfp_fpsr = mcp->mc_fpregs.fp_sr; 413 curpcb->pcb_fpflags = mcp->mc_fpregs.fp_flags & PCB_FP_USERMASK; 414 } 415 416 critical_exit(); 417 #endif 418 } 419 420 void 421 cpu_idle(int busy) 422 { 423 424 spinlock_enter(); 425 if (!busy) 426 cpu_idleclock(); 427 if (!sched_runnable()) 428 __asm __volatile( 429 "dsb sy \n" 430 "wfi \n"); 431 if (!busy) 432 cpu_activeclock(); 433 spinlock_exit(); 434 } 435 436 void 437 cpu_halt(void) 438 { 439 440 /* We should have shutdown by now, if not enter a low power sleep */ 441 intr_disable(); 442 while (1) { 443 __asm __volatile("wfi"); 444 } 445 } 446 447 /* 448 * Flush the D-cache for non-DMA I/O so that the I-cache can 449 * be made coherent later. 450 */ 451 void 452 cpu_flush_dcache(void *ptr, size_t len) 453 { 454 455 /* ARM64TODO TBD */ 456 } 457 458 /* Get current clock frequency for the given CPU ID. */ 459 int 460 cpu_est_clockrate(int cpu_id, uint64_t *rate) 461 { 462 struct pcpu *pc; 463 464 pc = pcpu_find(cpu_id); 465 if (pc == NULL || rate == NULL) 466 return (EINVAL); 467 468 if (pc->pc_clock == 0) 469 return (EOPNOTSUPP); 470 471 *rate = pc->pc_clock; 472 return (0); 473 } 474 475 void 476 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 477 { 478 479 pcpu->pc_acpi_id = 0xffffffff; 480 } 481 482 void 483 spinlock_enter(void) 484 { 485 struct thread *td; 486 register_t daif; 487 488 td = curthread; 489 if (td->td_md.md_spinlock_count == 0) { 490 daif = intr_disable(); 491 td->td_md.md_spinlock_count = 1; 492 td->td_md.md_saved_daif = daif; 493 } else 494 td->td_md.md_spinlock_count++; 495 critical_enter(); 496 } 497 498 void 499 spinlock_exit(void) 500 { 501 struct thread *td; 502 register_t daif; 503 504 td = curthread; 505 critical_exit(); 506 daif = td->td_md.md_saved_daif; 507 td->td_md.md_spinlock_count--; 508 if (td->td_md.md_spinlock_count == 0) 509 intr_restore(daif); 510 } 511 512 #ifndef _SYS_SYSPROTO_H_ 513 struct sigreturn_args { 514 ucontext_t *ucp; 515 }; 516 #endif 517 518 int 519 sys_sigreturn(struct thread *td, struct sigreturn_args *uap) 520 { 521 ucontext_t uc; 522 uint32_t spsr; 523 524 if (uap == NULL) 525 return (EFAULT); 526 if (copyin(uap->sigcntxp, &uc, sizeof(uc))) 527 return (EFAULT); 528 529 spsr = uc.uc_mcontext.mc_gpregs.gp_spsr; 530 if ((spsr & PSR_M_MASK) != PSR_M_EL0t || 531 (spsr & (PSR_F | PSR_I | PSR_A | PSR_D)) != 0) 532 return (EINVAL); 533 534 set_mcontext(td, &uc.uc_mcontext); 535 set_fpcontext(td, &uc.uc_mcontext); 536 537 /* Restore signal mask. */ 538 kern_sigprocmask(td, SIG_SETMASK, &uc.uc_sigmask, NULL, 0); 539 540 return (EJUSTRETURN); 541 } 542 543 /* 544 * Construct a PCB from a trapframe. This is called from kdb_trap() where 545 * we want to start a backtrace from the function that caused us to enter 546 * the debugger. We have the context in the trapframe, but base the trace 547 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 548 * enough for a backtrace. 549 */ 550 void 551 makectx(struct trapframe *tf, struct pcb *pcb) 552 { 553 int i; 554 555 for (i = 0; i < PCB_LR; i++) 556 pcb->pcb_x[i] = tf->tf_x[i]; 557 558 pcb->pcb_x[PCB_LR] = tf->tf_lr; 559 pcb->pcb_pc = tf->tf_elr; 560 pcb->pcb_sp = tf->tf_sp; 561 } 562 563 void 564 sendsig(sig_t catcher, ksiginfo_t *ksi, sigset_t *mask) 565 { 566 struct thread *td; 567 struct proc *p; 568 struct trapframe *tf; 569 struct sigframe *fp, frame; 570 struct sigacts *psp; 571 struct sysentvec *sysent; 572 int code, onstack, sig; 573 574 td = curthread; 575 p = td->td_proc; 576 PROC_LOCK_ASSERT(p, MA_OWNED); 577 578 sig = ksi->ksi_signo; 579 code = ksi->ksi_code; 580 psp = p->p_sigacts; 581 mtx_assert(&psp->ps_mtx, MA_OWNED); 582 583 tf = td->td_frame; 584 onstack = sigonstack(tf->tf_sp); 585 586 CTR4(KTR_SIG, "sendsig: td=%p (%s) catcher=%p sig=%d", td, p->p_comm, 587 catcher, sig); 588 589 /* Allocate and validate space for the signal handler context. */ 590 if ((td->td_pflags & TDP_ALTSTACK) != 0 && !onstack && 591 SIGISMEMBER(psp->ps_sigonstack, sig)) { 592 fp = (struct sigframe *)((uintptr_t)td->td_sigstk.ss_sp + 593 td->td_sigstk.ss_size); 594 #if defined(COMPAT_43) 595 td->td_sigstk.ss_flags |= SS_ONSTACK; 596 #endif 597 } else { 598 fp = (struct sigframe *)td->td_frame->tf_sp; 599 } 600 601 /* Make room, keeping the stack aligned */ 602 fp--; 603 fp = (struct sigframe *)STACKALIGN(fp); 604 605 /* Fill in the frame to copy out */ 606 get_mcontext(td, &frame.sf_uc.uc_mcontext, 0); 607 get_fpcontext(td, &frame.sf_uc.uc_mcontext); 608 frame.sf_si = ksi->ksi_info; 609 frame.sf_uc.uc_sigmask = *mask; 610 frame.sf_uc.uc_stack.ss_flags = (td->td_pflags & TDP_ALTSTACK) ? 611 ((onstack) ? SS_ONSTACK : 0) : SS_DISABLE; 612 frame.sf_uc.uc_stack = td->td_sigstk; 613 mtx_unlock(&psp->ps_mtx); 614 PROC_UNLOCK(td->td_proc); 615 616 /* Copy the sigframe out to the user's stack. */ 617 if (copyout(&frame, fp, sizeof(*fp)) != 0) { 618 /* Process has trashed its stack. Kill it. */ 619 CTR2(KTR_SIG, "sendsig: sigexit td=%p fp=%p", td, fp); 620 PROC_LOCK(p); 621 sigexit(td, SIGILL); 622 } 623 624 tf->tf_x[0]= sig; 625 tf->tf_x[1] = (register_t)&fp->sf_si; 626 tf->tf_x[2] = (register_t)&fp->sf_uc; 627 628 tf->tf_elr = (register_t)catcher; 629 tf->tf_sp = (register_t)fp; 630 sysent = p->p_sysent; 631 if (sysent->sv_sigcode_base != 0) 632 tf->tf_lr = (register_t)sysent->sv_sigcode_base; 633 else 634 tf->tf_lr = (register_t)(sysent->sv_psstrings - 635 *(sysent->sv_szsigcode)); 636 637 CTR3(KTR_SIG, "sendsig: return td=%p pc=%#x sp=%#x", td, tf->tf_elr, 638 tf->tf_sp); 639 640 PROC_LOCK(p); 641 mtx_lock(&psp->ps_mtx); 642 } 643 644 static void 645 init_proc0(vm_offset_t kstack) 646 { 647 struct pcpu *pcpup = &__pcpu[0]; 648 649 proc_linkup0(&proc0, &thread0); 650 thread0.td_kstack = kstack; 651 thread0.td_pcb = (struct pcb *)(thread0.td_kstack) - 1; 652 thread0.td_pcb->pcb_fpflags = 0; 653 thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; 654 thread0.td_pcb->pcb_vfpcpu = UINT_MAX; 655 thread0.td_frame = &proc0_tf; 656 pcpup->pc_curpcb = thread0.td_pcb; 657 } 658 659 typedef struct { 660 uint32_t type; 661 uint64_t phys_start; 662 uint64_t virt_start; 663 uint64_t num_pages; 664 uint64_t attr; 665 } EFI_MEMORY_DESCRIPTOR; 666 667 static int 668 add_physmap_entry(uint64_t base, uint64_t length, vm_paddr_t *physmap, 669 u_int *physmap_idxp) 670 { 671 u_int i, insert_idx, _physmap_idx; 672 673 _physmap_idx = *physmap_idxp; 674 675 if (length == 0) 676 return (1); 677 678 /* 679 * Find insertion point while checking for overlap. Start off by 680 * assuming the new entry will be added to the end. 681 */ 682 insert_idx = _physmap_idx; 683 for (i = 0; i <= _physmap_idx; i += 2) { 684 if (base < physmap[i + 1]) { 685 if (base + length <= physmap[i]) { 686 insert_idx = i; 687 break; 688 } 689 if (boothowto & RB_VERBOSE) 690 printf( 691 "Overlapping memory regions, ignoring second region\n"); 692 return (1); 693 } 694 } 695 696 /* See if we can prepend to the next entry. */ 697 if (insert_idx <= _physmap_idx && 698 base + length == physmap[insert_idx]) { 699 physmap[insert_idx] = base; 700 return (1); 701 } 702 703 /* See if we can append to the previous entry. */ 704 if (insert_idx > 0 && base == physmap[insert_idx - 1]) { 705 physmap[insert_idx - 1] += length; 706 return (1); 707 } 708 709 _physmap_idx += 2; 710 *physmap_idxp = _physmap_idx; 711 if (_physmap_idx == PHYSMAP_SIZE) { 712 printf( 713 "Too many segments in the physical address map, giving up\n"); 714 return (0); 715 } 716 717 /* 718 * Move the last 'N' entries down to make room for the new 719 * entry if needed. 720 */ 721 for (i = _physmap_idx; i > insert_idx; i -= 2) { 722 physmap[i] = physmap[i - 2]; 723 physmap[i + 1] = physmap[i - 1]; 724 } 725 726 /* Insert the new entry. */ 727 physmap[insert_idx] = base; 728 physmap[insert_idx + 1] = base + length; 729 return (1); 730 } 731 732 #ifdef FDT 733 static void 734 add_fdt_mem_regions(struct mem_region *mr, int mrcnt, vm_paddr_t *physmap, 735 u_int *physmap_idxp) 736 { 737 738 for (int i = 0; i < mrcnt; i++) { 739 if (!add_physmap_entry(mr[i].mr_start, mr[i].mr_size, physmap, 740 physmap_idxp)) 741 break; 742 } 743 } 744 #endif 745 746 static void 747 add_efi_map_entries(struct efi_map_header *efihdr, vm_paddr_t *physmap, 748 u_int *physmap_idxp) 749 { 750 struct efi_md *map, *p; 751 const char *type; 752 size_t efisz; 753 int ndesc, i; 754 755 static const char *types[] = { 756 "Reserved", 757 "LoaderCode", 758 "LoaderData", 759 "BootServicesCode", 760 "BootServicesData", 761 "RuntimeServicesCode", 762 "RuntimeServicesData", 763 "ConventionalMemory", 764 "UnusableMemory", 765 "ACPIReclaimMemory", 766 "ACPIMemoryNVS", 767 "MemoryMappedIO", 768 "MemoryMappedIOPortSpace", 769 "PalCode", 770 "PersistentMemory" 771 }; 772 773 /* 774 * Memory map data provided by UEFI via the GetMemoryMap 775 * Boot Services API. 776 */ 777 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 778 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 779 780 if (efihdr->descriptor_size == 0) 781 return; 782 ndesc = efihdr->memory_size / efihdr->descriptor_size; 783 784 if (boothowto & RB_VERBOSE) 785 printf("%23s %12s %12s %8s %4s\n", 786 "Type", "Physical", "Virtual", "#Pages", "Attr"); 787 788 for (i = 0, p = map; i < ndesc; i++, 789 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 790 if (boothowto & RB_VERBOSE) { 791 if (p->md_type < nitems(types)) 792 type = types[p->md_type]; 793 else 794 type = "<INVALID>"; 795 printf("%23s %012lx %12p %08lx ", type, p->md_phys, 796 p->md_virt, p->md_pages); 797 if (p->md_attr & EFI_MD_ATTR_UC) 798 printf("UC "); 799 if (p->md_attr & EFI_MD_ATTR_WC) 800 printf("WC "); 801 if (p->md_attr & EFI_MD_ATTR_WT) 802 printf("WT "); 803 if (p->md_attr & EFI_MD_ATTR_WB) 804 printf("WB "); 805 if (p->md_attr & EFI_MD_ATTR_UCE) 806 printf("UCE "); 807 if (p->md_attr & EFI_MD_ATTR_WP) 808 printf("WP "); 809 if (p->md_attr & EFI_MD_ATTR_RP) 810 printf("RP "); 811 if (p->md_attr & EFI_MD_ATTR_XP) 812 printf("XP "); 813 if (p->md_attr & EFI_MD_ATTR_NV) 814 printf("NV "); 815 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) 816 printf("MORE_RELIABLE "); 817 if (p->md_attr & EFI_MD_ATTR_RO) 818 printf("RO "); 819 if (p->md_attr & EFI_MD_ATTR_RT) 820 printf("RUNTIME"); 821 printf("\n"); 822 } 823 824 switch (p->md_type) { 825 case EFI_MD_TYPE_CODE: 826 case EFI_MD_TYPE_DATA: 827 case EFI_MD_TYPE_BS_CODE: 828 case EFI_MD_TYPE_BS_DATA: 829 case EFI_MD_TYPE_FREE: 830 /* 831 * We're allowed to use any entry with these types. 832 */ 833 break; 834 default: 835 continue; 836 } 837 838 if (!add_physmap_entry(p->md_phys, (p->md_pages * PAGE_SIZE), 839 physmap, physmap_idxp)) 840 break; 841 } 842 } 843 844 #ifdef FDT 845 static void 846 try_load_dtb(caddr_t kmdp) 847 { 848 vm_offset_t dtbp; 849 850 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 851 if (dtbp == (vm_offset_t)NULL) { 852 printf("ERROR loading DTB\n"); 853 return; 854 } 855 856 if (OF_install(OFW_FDT, 0) == FALSE) 857 panic("Cannot install FDT"); 858 859 if (OF_init((void *)dtbp) != 0) 860 panic("OF_init failed with the found device tree"); 861 } 862 #endif 863 864 static bool 865 bus_probe(void) 866 { 867 bool has_acpi, has_fdt; 868 char *order, *env; 869 870 has_acpi = has_fdt = false; 871 872 #ifdef FDT 873 has_fdt = (OF_peer(0) != 0); 874 #endif 875 #ifdef DEV_ACPI 876 has_acpi = (acpi_find_table(ACPI_SIG_SPCR) != 0); 877 #endif 878 879 env = kern_getenv("kern.cfg.order"); 880 if (env != NULL) { 881 order = env; 882 while (order != NULL) { 883 if (has_acpi && 884 strncmp(order, "acpi", 4) == 0 && 885 (order[4] == ',' || order[4] == '\0')) { 886 arm64_bus_method = ARM64_BUS_ACPI; 887 break; 888 } 889 if (has_fdt && 890 strncmp(order, "fdt", 3) == 0 && 891 (order[3] == ',' || order[3] == '\0')) { 892 arm64_bus_method = ARM64_BUS_FDT; 893 break; 894 } 895 order = strchr(order, ','); 896 } 897 freeenv(env); 898 899 /* If we set the bus method it is valid */ 900 if (arm64_bus_method != ARM64_BUS_NONE) 901 return (true); 902 } 903 /* If no order or an invalid order was set use the default */ 904 if (arm64_bus_method == ARM64_BUS_NONE) { 905 if (has_fdt) 906 arm64_bus_method = ARM64_BUS_FDT; 907 else if (has_acpi) 908 arm64_bus_method = ARM64_BUS_ACPI; 909 } 910 911 /* 912 * If no option was set the default is valid, otherwise we are 913 * setting one to get cninit() working, then calling panic to tell 914 * the user about the invalid bus setup. 915 */ 916 return (env == NULL); 917 } 918 919 static void 920 cache_setup(void) 921 { 922 int dcache_line_shift, icache_line_shift, dczva_line_shift; 923 uint32_t ctr_el0; 924 uint32_t dczid_el0; 925 926 ctr_el0 = READ_SPECIALREG(ctr_el0); 927 928 /* Read the log2 words in each D cache line */ 929 dcache_line_shift = CTR_DLINE_SIZE(ctr_el0); 930 /* Get the D cache line size */ 931 dcache_line_size = sizeof(int) << dcache_line_shift; 932 933 /* And the same for the I cache */ 934 icache_line_shift = CTR_ILINE_SIZE(ctr_el0); 935 icache_line_size = sizeof(int) << icache_line_shift; 936 937 idcache_line_size = MIN(dcache_line_size, icache_line_size); 938 939 dczid_el0 = READ_SPECIALREG(dczid_el0); 940 941 /* Check if dc zva is not prohibited */ 942 if (dczid_el0 & DCZID_DZP) 943 dczva_line_size = 0; 944 else { 945 /* Same as with above calculations */ 946 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 947 dczva_line_size = sizeof(int) << dczva_line_shift; 948 949 /* Change pagezero function */ 950 pagezero = pagezero_cache; 951 } 952 } 953 954 void 955 initarm(struct arm64_bootparams *abp) 956 { 957 struct efi_map_header *efihdr; 958 struct pcpu *pcpup; 959 #ifdef FDT 960 struct mem_region mem_regions[FDT_MEM_REGIONS]; 961 int mem_regions_sz; 962 #endif 963 vm_offset_t lastaddr; 964 caddr_t kmdp; 965 vm_paddr_t mem_len; 966 bool valid; 967 int i; 968 969 /* Set the module data location */ 970 preload_metadata = (caddr_t)(uintptr_t)(abp->modulep); 971 972 /* Find the kernel address */ 973 kmdp = preload_search_by_type("elf kernel"); 974 if (kmdp == NULL) 975 kmdp = preload_search_by_type("elf64 kernel"); 976 977 boothowto = MD_FETCH(kmdp, MODINFOMD_HOWTO, int); 978 init_static_kenv(MD_FETCH(kmdp, MODINFOMD_ENVP, char *), 0); 979 980 #ifdef FDT 981 try_load_dtb(kmdp); 982 #endif 983 984 /* Find the address to start allocating from */ 985 lastaddr = MD_FETCH(kmdp, MODINFOMD_KERNEND, vm_offset_t); 986 987 /* Load the physical memory ranges */ 988 physmap_idx = 0; 989 efihdr = (struct efi_map_header *)preload_search_info(kmdp, 990 MODINFO_METADATA | MODINFOMD_EFI_MAP); 991 if (efihdr != NULL) 992 add_efi_map_entries(efihdr, physmap, &physmap_idx); 993 #ifdef FDT 994 else { 995 /* Grab physical memory regions information from device tree. */ 996 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, 997 NULL) != 0) 998 panic("Cannot get physical memory regions"); 999 add_fdt_mem_regions(mem_regions, mem_regions_sz, physmap, 1000 &physmap_idx); 1001 } 1002 #endif 1003 1004 /* Print the memory map */ 1005 mem_len = 0; 1006 for (i = 0; i < physmap_idx; i += 2) { 1007 dump_avail[i] = physmap[i]; 1008 dump_avail[i + 1] = physmap[i + 1]; 1009 mem_len += physmap[i + 1] - physmap[i]; 1010 } 1011 dump_avail[i] = 0; 1012 dump_avail[i + 1] = 0; 1013 1014 /* Set the pcpu data, this is needed by pmap_bootstrap */ 1015 pcpup = &__pcpu[0]; 1016 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 1017 1018 /* 1019 * Set the pcpu pointer with a backup in tpidr_el1 to be 1020 * loaded when entering the kernel from userland. 1021 */ 1022 __asm __volatile( 1023 "mov x18, %0 \n" 1024 "msr tpidr_el1, %0" :: "r"(pcpup)); 1025 1026 PCPU_SET(curthread, &thread0); 1027 1028 /* Do basic tuning, hz etc */ 1029 init_param1(); 1030 1031 cache_setup(); 1032 pan_setup(); 1033 1034 /* Bootstrap enough of pmap to enter the kernel proper */ 1035 pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, 1036 KERNBASE - abp->kern_delta, lastaddr - KERNBASE); 1037 1038 devmap_bootstrap(0, NULL); 1039 1040 valid = bus_probe(); 1041 1042 cninit(); 1043 1044 if (!valid) 1045 panic("Invalid bus configuration: %s", 1046 kern_getenv("kern.cfg.order")); 1047 1048 init_proc0(abp->kern_stack); 1049 msgbufinit(msgbufp, msgbufsize); 1050 mutex_init(); 1051 init_param2(physmem); 1052 1053 dbg_monitor_init(); 1054 kdb_init(); 1055 pan_enable(); 1056 1057 early_boot = 0; 1058 } 1059 1060 #ifdef DDB 1061 #include <ddb/ddb.h> 1062 1063 DB_SHOW_COMMAND(specialregs, db_show_spregs) 1064 { 1065 #define PRINT_REG(reg) \ 1066 db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) 1067 1068 PRINT_REG(actlr_el1); 1069 PRINT_REG(afsr0_el1); 1070 PRINT_REG(afsr1_el1); 1071 PRINT_REG(aidr_el1); 1072 PRINT_REG(amair_el1); 1073 PRINT_REG(ccsidr_el1); 1074 PRINT_REG(clidr_el1); 1075 PRINT_REG(contextidr_el1); 1076 PRINT_REG(cpacr_el1); 1077 PRINT_REG(csselr_el1); 1078 PRINT_REG(ctr_el0); 1079 PRINT_REG(currentel); 1080 PRINT_REG(daif); 1081 PRINT_REG(dczid_el0); 1082 PRINT_REG(elr_el1); 1083 PRINT_REG(esr_el1); 1084 PRINT_REG(far_el1); 1085 #if 0 1086 /* ARM64TODO: Enable VFP before reading floating-point registers */ 1087 PRINT_REG(fpcr); 1088 PRINT_REG(fpsr); 1089 #endif 1090 PRINT_REG(id_aa64afr0_el1); 1091 PRINT_REG(id_aa64afr1_el1); 1092 PRINT_REG(id_aa64dfr0_el1); 1093 PRINT_REG(id_aa64dfr1_el1); 1094 PRINT_REG(id_aa64isar0_el1); 1095 PRINT_REG(id_aa64isar1_el1); 1096 PRINT_REG(id_aa64pfr0_el1); 1097 PRINT_REG(id_aa64pfr1_el1); 1098 PRINT_REG(id_afr0_el1); 1099 PRINT_REG(id_dfr0_el1); 1100 PRINT_REG(id_isar0_el1); 1101 PRINT_REG(id_isar1_el1); 1102 PRINT_REG(id_isar2_el1); 1103 PRINT_REG(id_isar3_el1); 1104 PRINT_REG(id_isar4_el1); 1105 PRINT_REG(id_isar5_el1); 1106 PRINT_REG(id_mmfr0_el1); 1107 PRINT_REG(id_mmfr1_el1); 1108 PRINT_REG(id_mmfr2_el1); 1109 PRINT_REG(id_mmfr3_el1); 1110 #if 0 1111 /* Missing from llvm */ 1112 PRINT_REG(id_mmfr4_el1); 1113 #endif 1114 PRINT_REG(id_pfr0_el1); 1115 PRINT_REG(id_pfr1_el1); 1116 PRINT_REG(isr_el1); 1117 PRINT_REG(mair_el1); 1118 PRINT_REG(midr_el1); 1119 PRINT_REG(mpidr_el1); 1120 PRINT_REG(mvfr0_el1); 1121 PRINT_REG(mvfr1_el1); 1122 PRINT_REG(mvfr2_el1); 1123 PRINT_REG(revidr_el1); 1124 PRINT_REG(sctlr_el1); 1125 PRINT_REG(sp_el0); 1126 PRINT_REG(spsel); 1127 PRINT_REG(spsr_el1); 1128 PRINT_REG(tcr_el1); 1129 PRINT_REG(tpidr_el0); 1130 PRINT_REG(tpidr_el1); 1131 PRINT_REG(tpidrro_el0); 1132 PRINT_REG(ttbr0_el1); 1133 PRINT_REG(ttbr1_el1); 1134 PRINT_REG(vbar_el1); 1135 #undef PRINT_REG 1136 } 1137 1138 DB_SHOW_COMMAND(vtop, db_show_vtop) 1139 { 1140 uint64_t phys; 1141 1142 if (have_addr) { 1143 phys = arm64_address_translate_s1e1r(addr); 1144 db_printf("EL1 physical address reg (read): 0x%016lx\n", phys); 1145 phys = arm64_address_translate_s1e1w(addr); 1146 db_printf("EL1 physical address reg (write): 0x%016lx\n", phys); 1147 phys = arm64_address_translate_s1e0r(addr); 1148 db_printf("EL0 physical address reg (read): 0x%016lx\n", phys); 1149 phys = arm64_address_translate_s1e0w(addr); 1150 db_printf("EL0 physical address reg (write): 0x%016lx\n", phys); 1151 } else 1152 db_printf("show vtop <virt_addr>\n"); 1153 } 1154 #endif 1155