1 /*- 2 * Copyright (c) 2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28 #include "opt_acpi.h" 29 #include "opt_platform.h" 30 #include "opt_ddb.h" 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/buf.h> 38 #include <sys/bus.h> 39 #include <sys/cons.h> 40 #include <sys/cpu.h> 41 #include <sys/csan.h> 42 #include <sys/devmap.h> 43 #include <sys/efi.h> 44 #include <sys/exec.h> 45 #include <sys/imgact.h> 46 #include <sys/kdb.h> 47 #include <sys/kernel.h> 48 #include <sys/ktr.h> 49 #include <sys/limits.h> 50 #include <sys/linker.h> 51 #include <sys/msgbuf.h> 52 #include <sys/pcpu.h> 53 #include <sys/physmem.h> 54 #include <sys/proc.h> 55 #include <sys/ptrace.h> 56 #include <sys/reboot.h> 57 #include <sys/reg.h> 58 #include <sys/rwlock.h> 59 #include <sys/sched.h> 60 #include <sys/signalvar.h> 61 #include <sys/syscallsubr.h> 62 #include <sys/sysent.h> 63 #include <sys/sysproto.h> 64 #include <sys/ucontext.h> 65 #include <sys/vdso.h> 66 #include <sys/vmmeter.h> 67 68 #include <vm/vm.h> 69 #include <vm/vm_param.h> 70 #include <vm/vm_kern.h> 71 #include <vm/vm_object.h> 72 #include <vm/vm_page.h> 73 #include <vm/vm_phys.h> 74 #include <vm/pmap.h> 75 #include <vm/vm_map.h> 76 #include <vm/vm_pager.h> 77 78 #include <machine/armreg.h> 79 #include <machine/cpu.h> 80 #include <machine/debug_monitor.h> 81 #include <machine/kdb.h> 82 #include <machine/machdep.h> 83 #include <machine/metadata.h> 84 #include <machine/md_var.h> 85 #include <machine/pcb.h> 86 #include <machine/undefined.h> 87 #include <machine/vmparam.h> 88 89 #ifdef VFP 90 #include <machine/vfp.h> 91 #endif 92 93 #ifdef DEV_ACPI 94 #include <contrib/dev/acpica/include/acpi.h> 95 #include <machine/acpica_machdep.h> 96 #endif 97 98 #ifdef FDT 99 #include <dev/fdt/fdt_common.h> 100 #include <dev/ofw/openfirm.h> 101 #endif 102 103 enum arm64_bus arm64_bus_method = ARM64_BUS_NONE; 104 105 /* 106 * XXX: The .bss is assumed to be in the boot CPU NUMA domain. If not we 107 * could relocate this, but will need to keep the same virtual address as 108 * it's reverenced by the EARLY_COUNTER macro. 109 */ 110 struct pcpu pcpu0; 111 112 #if defined(PERTHREAD_SSP) 113 /* 114 * The boot SSP canary. Will be replaced with a per-thread canary when 115 * scheduling has started. 116 */ 117 uintptr_t boot_canary = 0x49a2d892bc05a0b1ul; 118 #endif 119 120 static struct trapframe proc0_tf; 121 122 int early_boot = 1; 123 int cold = 1; 124 static int boot_el; 125 126 struct kva_md_info kmi; 127 128 int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 129 int has_pan; 130 131 /* 132 * Physical address of the EFI System Table. Stashed from the metadata hints 133 * passed into the kernel and used by the EFI code to call runtime services. 134 */ 135 vm_paddr_t efi_systbl_phys; 136 static struct efi_map_header *efihdr; 137 138 /* pagezero_* implementations are provided in support.S */ 139 void pagezero_simple(void *); 140 void pagezero_cache(void *); 141 142 /* pagezero_simple is default pagezero */ 143 void (*pagezero)(void *p) = pagezero_simple; 144 145 int (*apei_nmi)(void); 146 147 #if defined(PERTHREAD_SSP_WARNING) 148 static void 149 print_ssp_warning(void *data __unused) 150 { 151 printf("WARNING: Per-thread SSP is enabled but the compiler is too old to support it\n"); 152 } 153 SYSINIT(ssp_warn, SI_SUB_COPYRIGHT, SI_ORDER_ANY, print_ssp_warning, NULL); 154 SYSINIT(ssp_warn2, SI_SUB_LAST, SI_ORDER_ANY, print_ssp_warning, NULL); 155 #endif 156 157 static void 158 pan_setup(void) 159 { 160 uint64_t id_aa64mfr1; 161 162 id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 163 if (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) 164 has_pan = 1; 165 } 166 167 void 168 pan_enable(void) 169 { 170 171 /* 172 * The LLVM integrated assembler doesn't understand the PAN 173 * PSTATE field. Because of this we need to manually create 174 * the instruction in an asm block. This is equivalent to: 175 * msr pan, #1 176 * 177 * This sets the PAN bit, stopping the kernel from accessing 178 * memory when userspace can also access it unless the kernel 179 * uses the userspace load/store instructions. 180 */ 181 if (has_pan) { 182 WRITE_SPECIALREG(sctlr_el1, 183 READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); 184 __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); 185 } 186 } 187 188 bool 189 has_hyp(void) 190 { 191 192 return (boot_el == 2); 193 } 194 195 static void 196 cpu_startup(void *dummy) 197 { 198 vm_paddr_t size; 199 int i; 200 201 printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem), 202 ptoa((uintmax_t)realmem) / 1024 / 1024); 203 204 if (bootverbose) { 205 printf("Physical memory chunk(s):\n"); 206 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 207 size = phys_avail[i + 1] - phys_avail[i]; 208 printf("%#016jx - %#016jx, %ju bytes (%ju pages)\n", 209 (uintmax_t)phys_avail[i], 210 (uintmax_t)phys_avail[i + 1] - 1, 211 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE); 212 } 213 } 214 215 printf("avail memory = %ju (%ju MB)\n", 216 ptoa((uintmax_t)vm_free_count()), 217 ptoa((uintmax_t)vm_free_count()) / 1024 / 1024); 218 219 undef_init(); 220 install_cpu_errata(); 221 222 vm_ksubmap_init(&kmi); 223 bufinit(); 224 vm_pager_bufferinit(); 225 } 226 227 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 228 229 static void 230 late_ifunc_resolve(void *dummy __unused) 231 { 232 link_elf_late_ireloc(); 233 } 234 SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL); 235 236 int 237 cpu_idle_wakeup(int cpu) 238 { 239 240 return (0); 241 } 242 243 void 244 cpu_idle(int busy) 245 { 246 247 spinlock_enter(); 248 if (!busy) 249 cpu_idleclock(); 250 if (!sched_runnable()) 251 __asm __volatile( 252 "dsb sy \n" 253 "wfi \n"); 254 if (!busy) 255 cpu_activeclock(); 256 spinlock_exit(); 257 } 258 259 void 260 cpu_halt(void) 261 { 262 263 /* We should have shutdown by now, if not enter a low power sleep */ 264 intr_disable(); 265 while (1) { 266 __asm __volatile("wfi"); 267 } 268 } 269 270 /* 271 * Flush the D-cache for non-DMA I/O so that the I-cache can 272 * be made coherent later. 273 */ 274 void 275 cpu_flush_dcache(void *ptr, size_t len) 276 { 277 278 /* ARM64TODO TBD */ 279 } 280 281 /* Get current clock frequency for the given CPU ID. */ 282 int 283 cpu_est_clockrate(int cpu_id, uint64_t *rate) 284 { 285 struct pcpu *pc; 286 287 pc = pcpu_find(cpu_id); 288 if (pc == NULL || rate == NULL) 289 return (EINVAL); 290 291 if (pc->pc_clock == 0) 292 return (EOPNOTSUPP); 293 294 *rate = pc->pc_clock; 295 return (0); 296 } 297 298 void 299 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 300 { 301 302 pcpu->pc_acpi_id = 0xffffffff; 303 pcpu->pc_mpidr = 0xffffffff; 304 } 305 306 void 307 spinlock_enter(void) 308 { 309 struct thread *td; 310 register_t daif; 311 312 td = curthread; 313 if (td->td_md.md_spinlock_count == 0) { 314 daif = intr_disable(); 315 td->td_md.md_spinlock_count = 1; 316 td->td_md.md_saved_daif = daif; 317 critical_enter(); 318 } else 319 td->td_md.md_spinlock_count++; 320 } 321 322 void 323 spinlock_exit(void) 324 { 325 struct thread *td; 326 register_t daif; 327 328 td = curthread; 329 daif = td->td_md.md_saved_daif; 330 td->td_md.md_spinlock_count--; 331 if (td->td_md.md_spinlock_count == 0) { 332 critical_exit(); 333 intr_restore(daif); 334 } 335 } 336 337 /* 338 * Construct a PCB from a trapframe. This is called from kdb_trap() where 339 * we want to start a backtrace from the function that caused us to enter 340 * the debugger. We have the context in the trapframe, but base the trace 341 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 342 * enough for a backtrace. 343 */ 344 void 345 makectx(struct trapframe *tf, struct pcb *pcb) 346 { 347 int i; 348 349 for (i = 0; i < nitems(pcb->pcb_x); i++) 350 pcb->pcb_x[i] = tf->tf_x[i]; 351 352 /* NB: pcb_lr is the PC, see PC_REGS() in db_machdep.h */ 353 pcb->pcb_lr = tf->tf_elr; 354 pcb->pcb_sp = tf->tf_sp; 355 } 356 357 static void 358 init_proc0(vm_offset_t kstack) 359 { 360 struct pcpu *pcpup; 361 362 pcpup = cpuid_to_pcpu[0]; 363 MPASS(pcpup != NULL); 364 365 proc_linkup0(&proc0, &thread0); 366 thread0.td_kstack = kstack; 367 thread0.td_kstack_pages = KSTACK_PAGES; 368 #if defined(PERTHREAD_SSP) 369 thread0.td_md.md_canary = boot_canary; 370 #endif 371 thread0.td_pcb = (struct pcb *)(thread0.td_kstack + 372 thread0.td_kstack_pages * PAGE_SIZE) - 1; 373 thread0.td_pcb->pcb_fpflags = 0; 374 thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; 375 thread0.td_pcb->pcb_vfpcpu = UINT_MAX; 376 thread0.td_frame = &proc0_tf; 377 ptrauth_thread0(&thread0); 378 pcpup->pc_curpcb = thread0.td_pcb; 379 380 /* 381 * Unmask SError exceptions. They are used to signal a RAS failure, 382 * or other hardware error. 383 */ 384 serror_enable(); 385 } 386 387 /* 388 * Get an address to be used to write to kernel data that may be mapped 389 * read-only, e.g. to patch kernel code. 390 */ 391 bool 392 arm64_get_writable_addr(vm_offset_t addr, vm_offset_t *out) 393 { 394 vm_paddr_t pa; 395 396 /* Check if the page is writable */ 397 if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { 398 *out = addr; 399 return (true); 400 } 401 402 /* 403 * Find the physical address of the given page. 404 */ 405 if (!pmap_klookup(addr, &pa)) { 406 return (false); 407 } 408 409 /* 410 * If it is within the DMAP region and is writable use that. 411 */ 412 if (PHYS_IN_DMAP(pa)) { 413 addr = PHYS_TO_DMAP(pa); 414 if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { 415 *out = addr; 416 return (true); 417 } 418 } 419 420 return (false); 421 } 422 423 typedef struct { 424 uint32_t type; 425 uint64_t phys_start; 426 uint64_t virt_start; 427 uint64_t num_pages; 428 uint64_t attr; 429 } EFI_MEMORY_DESCRIPTOR; 430 431 typedef void (*efi_map_entry_cb)(struct efi_md *); 432 433 static void 434 foreach_efi_map_entry(struct efi_map_header *efihdr, efi_map_entry_cb cb) 435 { 436 struct efi_md *map, *p; 437 size_t efisz; 438 int ndesc, i; 439 440 /* 441 * Memory map data provided by UEFI via the GetMemoryMap 442 * Boot Services API. 443 */ 444 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 445 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 446 447 if (efihdr->descriptor_size == 0) 448 return; 449 ndesc = efihdr->memory_size / efihdr->descriptor_size; 450 451 for (i = 0, p = map; i < ndesc; i++, 452 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 453 cb(p); 454 } 455 } 456 457 static void 458 exclude_efi_map_entry(struct efi_md *p) 459 { 460 461 switch (p->md_type) { 462 case EFI_MD_TYPE_CODE: 463 case EFI_MD_TYPE_DATA: 464 case EFI_MD_TYPE_BS_CODE: 465 case EFI_MD_TYPE_BS_DATA: 466 case EFI_MD_TYPE_FREE: 467 /* 468 * We're allowed to use any entry with these types. 469 */ 470 break; 471 default: 472 physmem_exclude_region(p->md_phys, p->md_pages * PAGE_SIZE, 473 EXFLAG_NOALLOC); 474 } 475 } 476 477 static void 478 exclude_efi_map_entries(struct efi_map_header *efihdr) 479 { 480 481 foreach_efi_map_entry(efihdr, exclude_efi_map_entry); 482 } 483 484 static void 485 add_efi_map_entry(struct efi_md *p) 486 { 487 488 switch (p->md_type) { 489 case EFI_MD_TYPE_RT_DATA: 490 /* 491 * Runtime data will be excluded after the DMAP 492 * region is created to stop it from being added 493 * to phys_avail. 494 */ 495 case EFI_MD_TYPE_CODE: 496 case EFI_MD_TYPE_DATA: 497 case EFI_MD_TYPE_BS_CODE: 498 case EFI_MD_TYPE_BS_DATA: 499 case EFI_MD_TYPE_FREE: 500 /* 501 * We're allowed to use any entry with these types. 502 */ 503 physmem_hardware_region(p->md_phys, 504 p->md_pages * PAGE_SIZE); 505 break; 506 } 507 } 508 509 static void 510 add_efi_map_entries(struct efi_map_header *efihdr) 511 { 512 513 foreach_efi_map_entry(efihdr, add_efi_map_entry); 514 } 515 516 static void 517 print_efi_map_entry(struct efi_md *p) 518 { 519 const char *type; 520 static const char *types[] = { 521 "Reserved", 522 "LoaderCode", 523 "LoaderData", 524 "BootServicesCode", 525 "BootServicesData", 526 "RuntimeServicesCode", 527 "RuntimeServicesData", 528 "ConventionalMemory", 529 "UnusableMemory", 530 "ACPIReclaimMemory", 531 "ACPIMemoryNVS", 532 "MemoryMappedIO", 533 "MemoryMappedIOPortSpace", 534 "PalCode", 535 "PersistentMemory" 536 }; 537 538 if (p->md_type < nitems(types)) 539 type = types[p->md_type]; 540 else 541 type = "<INVALID>"; 542 printf("%23s %012lx %012lx %08lx ", type, p->md_phys, 543 p->md_virt, p->md_pages); 544 if (p->md_attr & EFI_MD_ATTR_UC) 545 printf("UC "); 546 if (p->md_attr & EFI_MD_ATTR_WC) 547 printf("WC "); 548 if (p->md_attr & EFI_MD_ATTR_WT) 549 printf("WT "); 550 if (p->md_attr & EFI_MD_ATTR_WB) 551 printf("WB "); 552 if (p->md_attr & EFI_MD_ATTR_UCE) 553 printf("UCE "); 554 if (p->md_attr & EFI_MD_ATTR_WP) 555 printf("WP "); 556 if (p->md_attr & EFI_MD_ATTR_RP) 557 printf("RP "); 558 if (p->md_attr & EFI_MD_ATTR_XP) 559 printf("XP "); 560 if (p->md_attr & EFI_MD_ATTR_NV) 561 printf("NV "); 562 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) 563 printf("MORE_RELIABLE "); 564 if (p->md_attr & EFI_MD_ATTR_RO) 565 printf("RO "); 566 if (p->md_attr & EFI_MD_ATTR_RT) 567 printf("RUNTIME"); 568 printf("\n"); 569 } 570 571 static void 572 print_efi_map_entries(struct efi_map_header *efihdr) 573 { 574 575 printf("%23s %12s %12s %8s %4s\n", 576 "Type", "Physical", "Virtual", "#Pages", "Attr"); 577 foreach_efi_map_entry(efihdr, print_efi_map_entry); 578 } 579 580 #ifdef FDT 581 static void 582 try_load_dtb(caddr_t kmdp) 583 { 584 vm_offset_t dtbp; 585 586 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 587 #if defined(FDT_DTB_STATIC) 588 /* 589 * In case the device tree blob was not retrieved (from metadata) try 590 * to use the statically embedded one. 591 */ 592 if (dtbp == 0) 593 dtbp = (vm_offset_t)&fdt_static_dtb; 594 #endif 595 596 if (dtbp == (vm_offset_t)NULL) { 597 #ifndef TSLOG 598 printf("ERROR loading DTB\n"); 599 #endif 600 return; 601 } 602 603 if (OF_install(OFW_FDT, 0) == FALSE) 604 panic("Cannot install FDT"); 605 606 if (OF_init((void *)dtbp) != 0) 607 panic("OF_init failed with the found device tree"); 608 609 parse_fdt_bootargs(); 610 } 611 #endif 612 613 static bool 614 bus_probe(void) 615 { 616 bool has_acpi, has_fdt; 617 char *order, *env; 618 619 has_acpi = has_fdt = false; 620 621 #ifdef FDT 622 has_fdt = (OF_peer(0) != 0); 623 #endif 624 #ifdef DEV_ACPI 625 has_acpi = (AcpiOsGetRootPointer() != 0); 626 #endif 627 628 env = kern_getenv("kern.cfg.order"); 629 if (env != NULL) { 630 order = env; 631 while (order != NULL) { 632 if (has_acpi && 633 strncmp(order, "acpi", 4) == 0 && 634 (order[4] == ',' || order[4] == '\0')) { 635 arm64_bus_method = ARM64_BUS_ACPI; 636 break; 637 } 638 if (has_fdt && 639 strncmp(order, "fdt", 3) == 0 && 640 (order[3] == ',' || order[3] == '\0')) { 641 arm64_bus_method = ARM64_BUS_FDT; 642 break; 643 } 644 order = strchr(order, ','); 645 } 646 freeenv(env); 647 648 /* If we set the bus method it is valid */ 649 if (arm64_bus_method != ARM64_BUS_NONE) 650 return (true); 651 } 652 /* If no order or an invalid order was set use the default */ 653 if (arm64_bus_method == ARM64_BUS_NONE) { 654 if (has_fdt) 655 arm64_bus_method = ARM64_BUS_FDT; 656 else if (has_acpi) 657 arm64_bus_method = ARM64_BUS_ACPI; 658 } 659 660 /* 661 * If no option was set the default is valid, otherwise we are 662 * setting one to get cninit() working, then calling panic to tell 663 * the user about the invalid bus setup. 664 */ 665 return (env == NULL); 666 } 667 668 static void 669 cache_setup(void) 670 { 671 int dczva_line_shift; 672 uint32_t dczid_el0; 673 674 identify_cache(READ_SPECIALREG(ctr_el0)); 675 676 dczid_el0 = READ_SPECIALREG(dczid_el0); 677 678 /* Check if dc zva is not prohibited */ 679 if (dczid_el0 & DCZID_DZP) 680 dczva_line_size = 0; 681 else { 682 /* Same as with above calculations */ 683 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 684 dczva_line_size = sizeof(int) << dczva_line_shift; 685 686 /* Change pagezero function */ 687 pagezero = pagezero_cache; 688 } 689 } 690 691 int 692 memory_mapping_mode(vm_paddr_t pa) 693 { 694 struct efi_md *map, *p; 695 size_t efisz; 696 int ndesc, i; 697 698 if (efihdr == NULL) 699 return (VM_MEMATTR_WRITE_BACK); 700 701 /* 702 * Memory map data provided by UEFI via the GetMemoryMap 703 * Boot Services API. 704 */ 705 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 706 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 707 708 if (efihdr->descriptor_size == 0) 709 return (VM_MEMATTR_WRITE_BACK); 710 ndesc = efihdr->memory_size / efihdr->descriptor_size; 711 712 for (i = 0, p = map; i < ndesc; i++, 713 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 714 if (pa < p->md_phys || 715 pa >= p->md_phys + p->md_pages * EFI_PAGE_SIZE) 716 continue; 717 if (p->md_type == EFI_MD_TYPE_IOMEM || 718 p->md_type == EFI_MD_TYPE_IOPORT) 719 return (VM_MEMATTR_DEVICE); 720 else if ((p->md_attr & EFI_MD_ATTR_WB) != 0 || 721 p->md_type == EFI_MD_TYPE_RECLAIM) 722 return (VM_MEMATTR_WRITE_BACK); 723 else if ((p->md_attr & EFI_MD_ATTR_WT) != 0) 724 return (VM_MEMATTR_WRITE_THROUGH); 725 else if ((p->md_attr & EFI_MD_ATTR_WC) != 0) 726 return (VM_MEMATTR_WRITE_COMBINING); 727 break; 728 } 729 730 return (VM_MEMATTR_DEVICE); 731 } 732 733 void 734 initarm(struct arm64_bootparams *abp) 735 { 736 struct efi_fb *efifb; 737 struct pcpu *pcpup; 738 char *env; 739 #ifdef FDT 740 struct mem_region mem_regions[FDT_MEM_REGIONS]; 741 int mem_regions_sz; 742 phandle_t root; 743 char dts_version[255]; 744 #endif 745 vm_offset_t lastaddr; 746 caddr_t kmdp; 747 bool valid; 748 749 TSRAW(&thread0, TS_ENTER, __func__, NULL); 750 751 boot_el = abp->boot_el; 752 753 /* Parse loader or FDT boot parametes. Determine last used address. */ 754 lastaddr = parse_boot_param(abp); 755 756 /* Find the kernel address */ 757 kmdp = preload_search_by_type("elf kernel"); 758 if (kmdp == NULL) 759 kmdp = preload_search_by_type("elf64 kernel"); 760 761 identify_cpu(0); 762 update_special_regs(0); 763 764 link_elf_ireloc(kmdp); 765 try_load_dtb(kmdp); 766 767 efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t); 768 769 /* Load the physical memory ranges */ 770 efihdr = (struct efi_map_header *)preload_search_info(kmdp, 771 MODINFO_METADATA | MODINFOMD_EFI_MAP); 772 if (efihdr != NULL) 773 add_efi_map_entries(efihdr); 774 #ifdef FDT 775 else { 776 /* Grab physical memory regions information from device tree. */ 777 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, 778 NULL) != 0) 779 panic("Cannot get physical memory regions"); 780 physmem_hardware_regions(mem_regions, mem_regions_sz); 781 } 782 if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0) 783 physmem_exclude_regions(mem_regions, mem_regions_sz, 784 EXFLAG_NODUMP | EXFLAG_NOALLOC); 785 #endif 786 787 /* Exclude the EFI framebuffer from our view of physical memory. */ 788 efifb = (struct efi_fb *)preload_search_info(kmdp, 789 MODINFO_METADATA | MODINFOMD_EFI_FB); 790 if (efifb != NULL) 791 physmem_exclude_region(efifb->fb_addr, efifb->fb_size, 792 EXFLAG_NOALLOC); 793 794 /* Set the pcpu data, this is needed by pmap_bootstrap */ 795 pcpup = &pcpu0; 796 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 797 798 /* 799 * Set the pcpu pointer with a backup in tpidr_el1 to be 800 * loaded when entering the kernel from userland. 801 */ 802 __asm __volatile( 803 "mov x18, %0 \n" 804 "msr tpidr_el1, %0" :: "r"(pcpup)); 805 806 /* locore.S sets sp_el0 to &thread0 so no need to set it here. */ 807 PCPU_SET(curthread, &thread0); 808 PCPU_SET(midr, get_midr()); 809 810 /* Do basic tuning, hz etc */ 811 init_param1(); 812 813 cache_setup(); 814 pan_setup(); 815 816 /* Bootstrap enough of pmap to enter the kernel proper */ 817 pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, 818 KERNBASE - abp->kern_delta, lastaddr - KERNBASE); 819 /* Exclude entries neexed in teh DMAP region, but not phys_avail */ 820 if (efihdr != NULL) 821 exclude_efi_map_entries(efihdr); 822 physmem_init_kernel_globals(); 823 824 devmap_bootstrap(0, NULL); 825 826 valid = bus_probe(); 827 828 cninit(); 829 set_ttbr0(abp->kern_ttbr0); 830 cpu_tlb_flushID(); 831 832 if (!valid) 833 panic("Invalid bus configuration: %s", 834 kern_getenv("kern.cfg.order")); 835 836 /* 837 * Check if pointer authentication is available on this system, and 838 * if so enable its use. This needs to be called before init_proc0 839 * as that will configure the thread0 pointer authentication keys. 840 */ 841 ptrauth_init(); 842 843 /* 844 * Dump the boot metadata. We have to wait for cninit() since console 845 * output is required. If it's grossly incorrect the kernel will never 846 * make it this far. 847 */ 848 if (getenv_is_true("debug.dump_modinfo_at_boot")) 849 preload_dump(); 850 851 init_proc0(abp->kern_stack); 852 msgbufinit(msgbufp, msgbufsize); 853 mutex_init(); 854 init_param2(physmem); 855 856 dbg_init(); 857 kdb_init(); 858 #ifdef KDB 859 if ((boothowto & RB_KDB) != 0) 860 kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); 861 #endif 862 pan_enable(); 863 864 kcsan_cpu_init(0); 865 866 env = kern_getenv("kernelname"); 867 if (env != NULL) 868 strlcpy(kernelname, env, sizeof(kernelname)); 869 870 #ifdef FDT 871 if (arm64_bus_method == ARM64_BUS_FDT) { 872 root = OF_finddevice("/"); 873 if (OF_getprop(root, "freebsd,dts-version", dts_version, sizeof(dts_version)) > 0) { 874 if (strcmp(LINUX_DTS_VERSION, dts_version) != 0) 875 printf("WARNING: DTB version is %s while kernel expects %s, " 876 "please update the DTB in the ESP\n", 877 dts_version, 878 LINUX_DTS_VERSION); 879 } else { 880 printf("WARNING: Cannot find freebsd,dts-version property, " 881 "cannot check DTB compliance\n"); 882 } 883 } 884 #endif 885 886 if (boothowto & RB_VERBOSE) { 887 if (efihdr != NULL) 888 print_efi_map_entries(efihdr); 889 physmem_print_tables(); 890 } 891 892 early_boot = 0; 893 894 TSEXIT(); 895 } 896 897 void 898 dbg_init(void) 899 { 900 901 /* Clear OS lock */ 902 WRITE_SPECIALREG(oslar_el1, 0); 903 904 /* This permits DDB to use debug registers for watchpoints. */ 905 dbg_monitor_init(); 906 907 /* TODO: Eventually will need to initialize debug registers here. */ 908 } 909 910 #ifdef DDB 911 #include <ddb/ddb.h> 912 913 DB_SHOW_COMMAND(specialregs, db_show_spregs) 914 { 915 #define PRINT_REG(reg) \ 916 db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) 917 918 PRINT_REG(actlr_el1); 919 PRINT_REG(afsr0_el1); 920 PRINT_REG(afsr1_el1); 921 PRINT_REG(aidr_el1); 922 PRINT_REG(amair_el1); 923 PRINT_REG(ccsidr_el1); 924 PRINT_REG(clidr_el1); 925 PRINT_REG(contextidr_el1); 926 PRINT_REG(cpacr_el1); 927 PRINT_REG(csselr_el1); 928 PRINT_REG(ctr_el0); 929 PRINT_REG(currentel); 930 PRINT_REG(daif); 931 PRINT_REG(dczid_el0); 932 PRINT_REG(elr_el1); 933 PRINT_REG(esr_el1); 934 PRINT_REG(far_el1); 935 #if 0 936 /* ARM64TODO: Enable VFP before reading floating-point registers */ 937 PRINT_REG(fpcr); 938 PRINT_REG(fpsr); 939 #endif 940 PRINT_REG(id_aa64afr0_el1); 941 PRINT_REG(id_aa64afr1_el1); 942 PRINT_REG(id_aa64dfr0_el1); 943 PRINT_REG(id_aa64dfr1_el1); 944 PRINT_REG(id_aa64isar0_el1); 945 PRINT_REG(id_aa64isar1_el1); 946 PRINT_REG(id_aa64pfr0_el1); 947 PRINT_REG(id_aa64pfr1_el1); 948 PRINT_REG(id_afr0_el1); 949 PRINT_REG(id_dfr0_el1); 950 PRINT_REG(id_isar0_el1); 951 PRINT_REG(id_isar1_el1); 952 PRINT_REG(id_isar2_el1); 953 PRINT_REG(id_isar3_el1); 954 PRINT_REG(id_isar4_el1); 955 PRINT_REG(id_isar5_el1); 956 PRINT_REG(id_mmfr0_el1); 957 PRINT_REG(id_mmfr1_el1); 958 PRINT_REG(id_mmfr2_el1); 959 PRINT_REG(id_mmfr3_el1); 960 #if 0 961 /* Missing from llvm */ 962 PRINT_REG(id_mmfr4_el1); 963 #endif 964 PRINT_REG(id_pfr0_el1); 965 PRINT_REG(id_pfr1_el1); 966 PRINT_REG(isr_el1); 967 PRINT_REG(mair_el1); 968 PRINT_REG(midr_el1); 969 PRINT_REG(mpidr_el1); 970 PRINT_REG(mvfr0_el1); 971 PRINT_REG(mvfr1_el1); 972 PRINT_REG(mvfr2_el1); 973 PRINT_REG(revidr_el1); 974 PRINT_REG(sctlr_el1); 975 PRINT_REG(sp_el0); 976 PRINT_REG(spsel); 977 PRINT_REG(spsr_el1); 978 PRINT_REG(tcr_el1); 979 PRINT_REG(tpidr_el0); 980 PRINT_REG(tpidr_el1); 981 PRINT_REG(tpidrro_el0); 982 PRINT_REG(ttbr0_el1); 983 PRINT_REG(ttbr1_el1); 984 PRINT_REG(vbar_el1); 985 #undef PRINT_REG 986 } 987 988 DB_SHOW_COMMAND(vtop, db_show_vtop) 989 { 990 uint64_t phys; 991 992 if (have_addr) { 993 phys = arm64_address_translate_s1e1r(addr); 994 db_printf("EL1 physical address reg (read): 0x%016lx\n", phys); 995 phys = arm64_address_translate_s1e1w(addr); 996 db_printf("EL1 physical address reg (write): 0x%016lx\n", phys); 997 phys = arm64_address_translate_s1e0r(addr); 998 db_printf("EL0 physical address reg (read): 0x%016lx\n", phys); 999 phys = arm64_address_translate_s1e0w(addr); 1000 db_printf("EL0 physical address reg (write): 0x%016lx\n", phys); 1001 } else 1002 db_printf("show vtop <virt_addr>\n"); 1003 } 1004 #endif 1005