1 /*- 2 * Copyright (c) 2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28 #include "opt_acpi.h" 29 #include "opt_platform.h" 30 #include "opt_ddb.h" 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/buf.h> 38 #include <sys/bus.h> 39 #include <sys/cons.h> 40 #include <sys/cpu.h> 41 #include <sys/csan.h> 42 #include <sys/devmap.h> 43 #include <sys/efi.h> 44 #include <sys/exec.h> 45 #include <sys/imgact.h> 46 #include <sys/kdb.h> 47 #include <sys/kernel.h> 48 #include <sys/ktr.h> 49 #include <sys/limits.h> 50 #include <sys/linker.h> 51 #include <sys/msgbuf.h> 52 #include <sys/pcpu.h> 53 #include <sys/physmem.h> 54 #include <sys/proc.h> 55 #include <sys/ptrace.h> 56 #include <sys/reboot.h> 57 #include <sys/reg.h> 58 #include <sys/rwlock.h> 59 #include <sys/sched.h> 60 #include <sys/signalvar.h> 61 #include <sys/syscallsubr.h> 62 #include <sys/sysent.h> 63 #include <sys/sysproto.h> 64 #include <sys/ucontext.h> 65 #include <sys/vdso.h> 66 #include <sys/vmmeter.h> 67 68 #include <vm/vm.h> 69 #include <vm/vm_param.h> 70 #include <vm/vm_kern.h> 71 #include <vm/vm_object.h> 72 #include <vm/vm_page.h> 73 #include <vm/vm_phys.h> 74 #include <vm/pmap.h> 75 #include <vm/vm_map.h> 76 #include <vm/vm_pager.h> 77 78 #include <machine/armreg.h> 79 #include <machine/cpu.h> 80 #include <machine/debug_monitor.h> 81 #include <machine/kdb.h> 82 #include <machine/machdep.h> 83 #include <machine/metadata.h> 84 #include <machine/md_var.h> 85 #include <machine/pcb.h> 86 #include <machine/undefined.h> 87 #include <machine/vmparam.h> 88 89 #ifdef VFP 90 #include <machine/vfp.h> 91 #endif 92 93 #ifdef DEV_ACPI 94 #include <contrib/dev/acpica/include/acpi.h> 95 #include <machine/acpica_machdep.h> 96 #endif 97 98 #ifdef FDT 99 #include <dev/fdt/fdt_common.h> 100 #include <dev/ofw/openfirm.h> 101 #endif 102 103 enum arm64_bus arm64_bus_method = ARM64_BUS_NONE; 104 105 /* 106 * XXX: The .bss is assumed to be in the boot CPU NUMA domain. If not we 107 * could relocate this, but will need to keep the same virtual address as 108 * it's reverenced by the EARLY_COUNTER macro. 109 */ 110 struct pcpu pcpu0; 111 112 #if defined(PERTHREAD_SSP) 113 /* 114 * The boot SSP canary. Will be replaced with a per-thread canary when 115 * scheduling has started. 116 */ 117 uintptr_t boot_canary = 0x49a2d892bc05a0b1ul; 118 #endif 119 120 static struct trapframe proc0_tf; 121 122 int early_boot = 1; 123 int cold = 1; 124 static int boot_el; 125 126 struct kva_md_info kmi; 127 128 int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 129 int has_pan; 130 131 /* 132 * Physical address of the EFI System Table. Stashed from the metadata hints 133 * passed into the kernel and used by the EFI code to call runtime services. 134 */ 135 vm_paddr_t efi_systbl_phys; 136 static struct efi_map_header *efihdr; 137 138 /* pagezero_* implementations are provided in support.S */ 139 void pagezero_simple(void *); 140 void pagezero_cache(void *); 141 142 /* pagezero_simple is default pagezero */ 143 void (*pagezero)(void *p) = pagezero_simple; 144 145 int (*apei_nmi)(void); 146 147 #if defined(PERTHREAD_SSP_WARNING) 148 static void 149 print_ssp_warning(void *data __unused) 150 { 151 printf("WARNING: Per-thread SSP is enabled but the compiler is too old to support it\n"); 152 } 153 SYSINIT(ssp_warn, SI_SUB_COPYRIGHT, SI_ORDER_ANY, print_ssp_warning, NULL); 154 SYSINIT(ssp_warn2, SI_SUB_LAST, SI_ORDER_ANY, print_ssp_warning, NULL); 155 #endif 156 157 static void 158 pan_setup(void) 159 { 160 uint64_t id_aa64mfr1; 161 162 id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 163 if (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) 164 has_pan = 1; 165 } 166 167 void 168 pan_enable(void) 169 { 170 171 /* 172 * The LLVM integrated assembler doesn't understand the PAN 173 * PSTATE field. Because of this we need to manually create 174 * the instruction in an asm block. This is equivalent to: 175 * msr pan, #1 176 * 177 * This sets the PAN bit, stopping the kernel from accessing 178 * memory when userspace can also access it unless the kernel 179 * uses the userspace load/store instructions. 180 */ 181 if (has_pan) { 182 WRITE_SPECIALREG(sctlr_el1, 183 READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); 184 __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); 185 } 186 } 187 188 bool 189 has_hyp(void) 190 { 191 192 return (boot_el == 2); 193 } 194 195 static void 196 cpu_startup(void *dummy) 197 { 198 vm_paddr_t size; 199 int i; 200 201 printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem), 202 ptoa((uintmax_t)realmem) / 1024 / 1024); 203 204 if (bootverbose) { 205 printf("Physical memory chunk(s):\n"); 206 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 207 size = phys_avail[i + 1] - phys_avail[i]; 208 printf("%#016jx - %#016jx, %ju bytes (%ju pages)\n", 209 (uintmax_t)phys_avail[i], 210 (uintmax_t)phys_avail[i + 1] - 1, 211 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE); 212 } 213 } 214 215 printf("avail memory = %ju (%ju MB)\n", 216 ptoa((uintmax_t)vm_free_count()), 217 ptoa((uintmax_t)vm_free_count()) / 1024 / 1024); 218 219 undef_init(); 220 install_cpu_errata(); 221 222 vm_ksubmap_init(&kmi); 223 bufinit(); 224 vm_pager_bufferinit(); 225 } 226 227 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 228 229 static void 230 late_ifunc_resolve(void *dummy __unused) 231 { 232 link_elf_late_ireloc(); 233 } 234 SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL); 235 236 int 237 cpu_idle_wakeup(int cpu) 238 { 239 240 return (0); 241 } 242 243 void 244 cpu_idle(int busy) 245 { 246 247 spinlock_enter(); 248 if (!busy) 249 cpu_idleclock(); 250 if (!sched_runnable()) 251 __asm __volatile( 252 "dsb sy \n" 253 "wfi \n"); 254 if (!busy) 255 cpu_activeclock(); 256 spinlock_exit(); 257 } 258 259 void 260 cpu_halt(void) 261 { 262 263 /* We should have shutdown by now, if not enter a low power sleep */ 264 intr_disable(); 265 while (1) { 266 __asm __volatile("wfi"); 267 } 268 } 269 270 /* 271 * Flush the D-cache for non-DMA I/O so that the I-cache can 272 * be made coherent later. 273 */ 274 void 275 cpu_flush_dcache(void *ptr, size_t len) 276 { 277 278 /* ARM64TODO TBD */ 279 } 280 281 /* Get current clock frequency for the given CPU ID. */ 282 int 283 cpu_est_clockrate(int cpu_id, uint64_t *rate) 284 { 285 struct pcpu *pc; 286 287 pc = pcpu_find(cpu_id); 288 if (pc == NULL || rate == NULL) 289 return (EINVAL); 290 291 if (pc->pc_clock == 0) 292 return (EOPNOTSUPP); 293 294 *rate = pc->pc_clock; 295 return (0); 296 } 297 298 void 299 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 300 { 301 302 pcpu->pc_acpi_id = 0xffffffff; 303 pcpu->pc_mpidr = 0xffffffff; 304 } 305 306 void 307 spinlock_enter(void) 308 { 309 struct thread *td; 310 register_t daif; 311 312 td = curthread; 313 if (td->td_md.md_spinlock_count == 0) { 314 daif = intr_disable(); 315 td->td_md.md_spinlock_count = 1; 316 td->td_md.md_saved_daif = daif; 317 critical_enter(); 318 } else 319 td->td_md.md_spinlock_count++; 320 } 321 322 void 323 spinlock_exit(void) 324 { 325 struct thread *td; 326 register_t daif; 327 328 td = curthread; 329 daif = td->td_md.md_saved_daif; 330 td->td_md.md_spinlock_count--; 331 if (td->td_md.md_spinlock_count == 0) { 332 critical_exit(); 333 intr_restore(daif); 334 } 335 } 336 337 /* 338 * Construct a PCB from a trapframe. This is called from kdb_trap() where 339 * we want to start a backtrace from the function that caused us to enter 340 * the debugger. We have the context in the trapframe, but base the trace 341 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 342 * enough for a backtrace. 343 */ 344 void 345 makectx(struct trapframe *tf, struct pcb *pcb) 346 { 347 int i; 348 349 for (i = 0; i < nitems(pcb->pcb_x); i++) 350 pcb->pcb_x[i] = tf->tf_x[i]; 351 352 /* NB: pcb_lr is the PC, see PC_REGS() in db_machdep.h */ 353 pcb->pcb_lr = tf->tf_elr; 354 pcb->pcb_sp = tf->tf_sp; 355 } 356 357 static void 358 init_proc0(vm_offset_t kstack) 359 { 360 struct pcpu *pcpup; 361 362 pcpup = cpuid_to_pcpu[0]; 363 MPASS(pcpup != NULL); 364 365 proc_linkup0(&proc0, &thread0); 366 thread0.td_kstack = kstack; 367 thread0.td_kstack_pages = KSTACK_PAGES; 368 #if defined(PERTHREAD_SSP) 369 thread0.td_md.md_canary = boot_canary; 370 #endif 371 thread0.td_pcb = (struct pcb *)(thread0.td_kstack + 372 thread0.td_kstack_pages * PAGE_SIZE) - 1; 373 thread0.td_pcb->pcb_fpflags = 0; 374 thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; 375 thread0.td_pcb->pcb_vfpcpu = UINT_MAX; 376 thread0.td_frame = &proc0_tf; 377 pcpup->pc_curpcb = thread0.td_pcb; 378 379 /* 380 * Unmask SError exceptions. They are used to signal a RAS failure, 381 * or other hardware error. 382 */ 383 serror_enable(); 384 } 385 386 /* 387 * Get an address to be used to write to kernel data that may be mapped 388 * read-only, e.g. to patch kernel code. 389 */ 390 bool 391 arm64_get_writable_addr(vm_offset_t addr, vm_offset_t *out) 392 { 393 vm_paddr_t pa; 394 395 /* Check if the page is writable */ 396 if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { 397 *out = addr; 398 return (true); 399 } 400 401 /* 402 * Find the physical address of the given page. 403 */ 404 if (!pmap_klookup(addr, &pa)) { 405 return (false); 406 } 407 408 /* 409 * If it is within the DMAP region and is writable use that. 410 */ 411 if (PHYS_IN_DMAP(pa)) { 412 addr = PHYS_TO_DMAP(pa); 413 if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { 414 *out = addr; 415 return (true); 416 } 417 } 418 419 return (false); 420 } 421 422 typedef struct { 423 uint32_t type; 424 uint64_t phys_start; 425 uint64_t virt_start; 426 uint64_t num_pages; 427 uint64_t attr; 428 } EFI_MEMORY_DESCRIPTOR; 429 430 typedef void (*efi_map_entry_cb)(struct efi_md *); 431 432 static void 433 foreach_efi_map_entry(struct efi_map_header *efihdr, efi_map_entry_cb cb) 434 { 435 struct efi_md *map, *p; 436 size_t efisz; 437 int ndesc, i; 438 439 /* 440 * Memory map data provided by UEFI via the GetMemoryMap 441 * Boot Services API. 442 */ 443 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 444 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 445 446 if (efihdr->descriptor_size == 0) 447 return; 448 ndesc = efihdr->memory_size / efihdr->descriptor_size; 449 450 for (i = 0, p = map; i < ndesc; i++, 451 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 452 cb(p); 453 } 454 } 455 456 static void 457 exclude_efi_map_entry(struct efi_md *p) 458 { 459 460 switch (p->md_type) { 461 case EFI_MD_TYPE_CODE: 462 case EFI_MD_TYPE_DATA: 463 case EFI_MD_TYPE_BS_CODE: 464 case EFI_MD_TYPE_BS_DATA: 465 case EFI_MD_TYPE_FREE: 466 /* 467 * We're allowed to use any entry with these types. 468 */ 469 break; 470 default: 471 physmem_exclude_region(p->md_phys, p->md_pages * PAGE_SIZE, 472 EXFLAG_NOALLOC); 473 } 474 } 475 476 static void 477 exclude_efi_map_entries(struct efi_map_header *efihdr) 478 { 479 480 foreach_efi_map_entry(efihdr, exclude_efi_map_entry); 481 } 482 483 static void 484 add_efi_map_entry(struct efi_md *p) 485 { 486 487 switch (p->md_type) { 488 case EFI_MD_TYPE_RT_DATA: 489 /* 490 * Runtime data will be excluded after the DMAP 491 * region is created to stop it from being added 492 * to phys_avail. 493 */ 494 case EFI_MD_TYPE_CODE: 495 case EFI_MD_TYPE_DATA: 496 case EFI_MD_TYPE_BS_CODE: 497 case EFI_MD_TYPE_BS_DATA: 498 case EFI_MD_TYPE_FREE: 499 /* 500 * We're allowed to use any entry with these types. 501 */ 502 physmem_hardware_region(p->md_phys, 503 p->md_pages * PAGE_SIZE); 504 break; 505 } 506 } 507 508 static void 509 add_efi_map_entries(struct efi_map_header *efihdr) 510 { 511 512 foreach_efi_map_entry(efihdr, add_efi_map_entry); 513 } 514 515 static void 516 print_efi_map_entry(struct efi_md *p) 517 { 518 const char *type; 519 static const char *types[] = { 520 "Reserved", 521 "LoaderCode", 522 "LoaderData", 523 "BootServicesCode", 524 "BootServicesData", 525 "RuntimeServicesCode", 526 "RuntimeServicesData", 527 "ConventionalMemory", 528 "UnusableMemory", 529 "ACPIReclaimMemory", 530 "ACPIMemoryNVS", 531 "MemoryMappedIO", 532 "MemoryMappedIOPortSpace", 533 "PalCode", 534 "PersistentMemory" 535 }; 536 537 if (p->md_type < nitems(types)) 538 type = types[p->md_type]; 539 else 540 type = "<INVALID>"; 541 printf("%23s %012lx %012lx %08lx ", type, p->md_phys, 542 p->md_virt, p->md_pages); 543 if (p->md_attr & EFI_MD_ATTR_UC) 544 printf("UC "); 545 if (p->md_attr & EFI_MD_ATTR_WC) 546 printf("WC "); 547 if (p->md_attr & EFI_MD_ATTR_WT) 548 printf("WT "); 549 if (p->md_attr & EFI_MD_ATTR_WB) 550 printf("WB "); 551 if (p->md_attr & EFI_MD_ATTR_UCE) 552 printf("UCE "); 553 if (p->md_attr & EFI_MD_ATTR_WP) 554 printf("WP "); 555 if (p->md_attr & EFI_MD_ATTR_RP) 556 printf("RP "); 557 if (p->md_attr & EFI_MD_ATTR_XP) 558 printf("XP "); 559 if (p->md_attr & EFI_MD_ATTR_NV) 560 printf("NV "); 561 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) 562 printf("MORE_RELIABLE "); 563 if (p->md_attr & EFI_MD_ATTR_RO) 564 printf("RO "); 565 if (p->md_attr & EFI_MD_ATTR_RT) 566 printf("RUNTIME"); 567 printf("\n"); 568 } 569 570 static void 571 print_efi_map_entries(struct efi_map_header *efihdr) 572 { 573 574 printf("%23s %12s %12s %8s %4s\n", 575 "Type", "Physical", "Virtual", "#Pages", "Attr"); 576 foreach_efi_map_entry(efihdr, print_efi_map_entry); 577 } 578 579 #ifdef FDT 580 static void 581 try_load_dtb(caddr_t kmdp) 582 { 583 vm_offset_t dtbp; 584 585 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 586 #if defined(FDT_DTB_STATIC) 587 /* 588 * In case the device tree blob was not retrieved (from metadata) try 589 * to use the statically embedded one. 590 */ 591 if (dtbp == 0) 592 dtbp = (vm_offset_t)&fdt_static_dtb; 593 #endif 594 595 if (dtbp == (vm_offset_t)NULL) { 596 #ifndef TSLOG 597 printf("ERROR loading DTB\n"); 598 #endif 599 return; 600 } 601 602 if (OF_install(OFW_FDT, 0) == FALSE) 603 panic("Cannot install FDT"); 604 605 if (OF_init((void *)dtbp) != 0) 606 panic("OF_init failed with the found device tree"); 607 608 parse_fdt_bootargs(); 609 } 610 #endif 611 612 static bool 613 bus_probe(void) 614 { 615 bool has_acpi, has_fdt; 616 char *order, *env; 617 618 has_acpi = has_fdt = false; 619 620 #ifdef FDT 621 has_fdt = (OF_peer(0) != 0); 622 #endif 623 #ifdef DEV_ACPI 624 has_acpi = (AcpiOsGetRootPointer() != 0); 625 #endif 626 627 env = kern_getenv("kern.cfg.order"); 628 if (env != NULL) { 629 order = env; 630 while (order != NULL) { 631 if (has_acpi && 632 strncmp(order, "acpi", 4) == 0 && 633 (order[4] == ',' || order[4] == '\0')) { 634 arm64_bus_method = ARM64_BUS_ACPI; 635 break; 636 } 637 if (has_fdt && 638 strncmp(order, "fdt", 3) == 0 && 639 (order[3] == ',' || order[3] == '\0')) { 640 arm64_bus_method = ARM64_BUS_FDT; 641 break; 642 } 643 order = strchr(order, ','); 644 } 645 freeenv(env); 646 647 /* If we set the bus method it is valid */ 648 if (arm64_bus_method != ARM64_BUS_NONE) 649 return (true); 650 } 651 /* If no order or an invalid order was set use the default */ 652 if (arm64_bus_method == ARM64_BUS_NONE) { 653 if (has_fdt) 654 arm64_bus_method = ARM64_BUS_FDT; 655 else if (has_acpi) 656 arm64_bus_method = ARM64_BUS_ACPI; 657 } 658 659 /* 660 * If no option was set the default is valid, otherwise we are 661 * setting one to get cninit() working, then calling panic to tell 662 * the user about the invalid bus setup. 663 */ 664 return (env == NULL); 665 } 666 667 static void 668 cache_setup(void) 669 { 670 int dczva_line_shift; 671 uint32_t dczid_el0; 672 673 identify_cache(READ_SPECIALREG(ctr_el0)); 674 675 dczid_el0 = READ_SPECIALREG(dczid_el0); 676 677 /* Check if dc zva is not prohibited */ 678 if (dczid_el0 & DCZID_DZP) 679 dczva_line_size = 0; 680 else { 681 /* Same as with above calculations */ 682 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 683 dczva_line_size = sizeof(int) << dczva_line_shift; 684 685 /* Change pagezero function */ 686 pagezero = pagezero_cache; 687 } 688 } 689 690 int 691 memory_mapping_mode(vm_paddr_t pa) 692 { 693 struct efi_md *map, *p; 694 size_t efisz; 695 int ndesc, i; 696 697 if (efihdr == NULL) 698 return (VM_MEMATTR_WRITE_BACK); 699 700 /* 701 * Memory map data provided by UEFI via the GetMemoryMap 702 * Boot Services API. 703 */ 704 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 705 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 706 707 if (efihdr->descriptor_size == 0) 708 return (VM_MEMATTR_WRITE_BACK); 709 ndesc = efihdr->memory_size / efihdr->descriptor_size; 710 711 for (i = 0, p = map; i < ndesc; i++, 712 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 713 if (pa < p->md_phys || 714 pa >= p->md_phys + p->md_pages * EFI_PAGE_SIZE) 715 continue; 716 if (p->md_type == EFI_MD_TYPE_IOMEM || 717 p->md_type == EFI_MD_TYPE_IOPORT) 718 return (VM_MEMATTR_DEVICE); 719 else if ((p->md_attr & EFI_MD_ATTR_WB) != 0 || 720 p->md_type == EFI_MD_TYPE_RECLAIM) 721 return (VM_MEMATTR_WRITE_BACK); 722 else if ((p->md_attr & EFI_MD_ATTR_WT) != 0) 723 return (VM_MEMATTR_WRITE_THROUGH); 724 else if ((p->md_attr & EFI_MD_ATTR_WC) != 0) 725 return (VM_MEMATTR_WRITE_COMBINING); 726 break; 727 } 728 729 return (VM_MEMATTR_DEVICE); 730 } 731 732 void 733 initarm(struct arm64_bootparams *abp) 734 { 735 struct efi_fb *efifb; 736 struct pcpu *pcpup; 737 char *env; 738 #ifdef FDT 739 struct mem_region mem_regions[FDT_MEM_REGIONS]; 740 int mem_regions_sz; 741 phandle_t root; 742 char dts_version[255]; 743 #endif 744 vm_offset_t lastaddr; 745 caddr_t kmdp; 746 bool valid; 747 748 TSRAW(&thread0, TS_ENTER, __func__, NULL); 749 750 boot_el = abp->boot_el; 751 752 /* Parse loader or FDT boot parametes. Determine last used address. */ 753 lastaddr = parse_boot_param(abp); 754 755 /* Find the kernel address */ 756 kmdp = preload_search_by_type("elf kernel"); 757 if (kmdp == NULL) 758 kmdp = preload_search_by_type("elf64 kernel"); 759 760 identify_cpu(0); 761 update_special_regs(0); 762 763 link_elf_ireloc(kmdp); 764 try_load_dtb(kmdp); 765 766 efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t); 767 768 /* Load the physical memory ranges */ 769 efihdr = (struct efi_map_header *)preload_search_info(kmdp, 770 MODINFO_METADATA | MODINFOMD_EFI_MAP); 771 if (efihdr != NULL) 772 add_efi_map_entries(efihdr); 773 #ifdef FDT 774 else { 775 /* Grab physical memory regions information from device tree. */ 776 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, 777 NULL) != 0) 778 panic("Cannot get physical memory regions"); 779 physmem_hardware_regions(mem_regions, mem_regions_sz); 780 } 781 if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0) 782 physmem_exclude_regions(mem_regions, mem_regions_sz, 783 EXFLAG_NODUMP | EXFLAG_NOALLOC); 784 #endif 785 786 /* Exclude the EFI framebuffer from our view of physical memory. */ 787 efifb = (struct efi_fb *)preload_search_info(kmdp, 788 MODINFO_METADATA | MODINFOMD_EFI_FB); 789 if (efifb != NULL) 790 physmem_exclude_region(efifb->fb_addr, efifb->fb_size, 791 EXFLAG_NOALLOC); 792 793 /* Set the pcpu data, this is needed by pmap_bootstrap */ 794 pcpup = &pcpu0; 795 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 796 797 /* 798 * Set the pcpu pointer with a backup in tpidr_el1 to be 799 * loaded when entering the kernel from userland. 800 */ 801 __asm __volatile( 802 "mov x18, %0 \n" 803 "msr tpidr_el1, %0" :: "r"(pcpup)); 804 805 /* locore.S sets sp_el0 to &thread0 so no need to set it here. */ 806 PCPU_SET(curthread, &thread0); 807 PCPU_SET(midr, get_midr()); 808 809 /* Do basic tuning, hz etc */ 810 init_param1(); 811 812 cache_setup(); 813 pan_setup(); 814 815 /* Bootstrap enough of pmap to enter the kernel proper */ 816 pmap_bootstrap(abp->kern_l0pt, abp->kern_l1pt, 817 KERNBASE - abp->kern_delta, lastaddr - KERNBASE); 818 /* Exclude entries neexed in teh DMAP region, but not phys_avail */ 819 if (efihdr != NULL) 820 exclude_efi_map_entries(efihdr); 821 physmem_init_kernel_globals(); 822 823 devmap_bootstrap(0, NULL); 824 825 valid = bus_probe(); 826 827 cninit(); 828 set_ttbr0(abp->kern_ttbr0); 829 cpu_tlb_flushID(); 830 831 if (!valid) 832 panic("Invalid bus configuration: %s", 833 kern_getenv("kern.cfg.order")); 834 835 /* 836 * Dump the boot metadata. We have to wait for cninit() since console 837 * output is required. If it's grossly incorrect the kernel will never 838 * make it this far. 839 */ 840 if (getenv_is_true("debug.dump_modinfo_at_boot")) 841 preload_dump(); 842 843 init_proc0(abp->kern_stack); 844 msgbufinit(msgbufp, msgbufsize); 845 mutex_init(); 846 init_param2(physmem); 847 848 dbg_init(); 849 kdb_init(); 850 pan_enable(); 851 852 kcsan_cpu_init(0); 853 854 env = kern_getenv("kernelname"); 855 if (env != NULL) 856 strlcpy(kernelname, env, sizeof(kernelname)); 857 858 #ifdef FDT 859 if (arm64_bus_method == ARM64_BUS_FDT) { 860 root = OF_finddevice("/"); 861 if (OF_getprop(root, "freebsd,dts-version", dts_version, sizeof(dts_version)) > 0) { 862 if (strcmp(LINUX_DTS_VERSION, dts_version) != 0) 863 printf("WARNING: DTB version is %s while kernel expects %s, " 864 "please update the DTB in the ESP\n", 865 dts_version, 866 LINUX_DTS_VERSION); 867 } else { 868 printf("WARNING: Cannot find freebsd,dts-version property, " 869 "cannot check DTB compliance\n"); 870 } 871 } 872 #endif 873 874 if (boothowto & RB_VERBOSE) { 875 if (efihdr != NULL) 876 print_efi_map_entries(efihdr); 877 physmem_print_tables(); 878 } 879 880 early_boot = 0; 881 882 TSEXIT(); 883 } 884 885 void 886 dbg_init(void) 887 { 888 889 /* Clear OS lock */ 890 WRITE_SPECIALREG(oslar_el1, 0); 891 892 /* This permits DDB to use debug registers for watchpoints. */ 893 dbg_monitor_init(); 894 895 /* TODO: Eventually will need to initialize debug registers here. */ 896 } 897 898 #ifdef DDB 899 #include <ddb/ddb.h> 900 901 DB_SHOW_COMMAND(specialregs, db_show_spregs) 902 { 903 #define PRINT_REG(reg) \ 904 db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) 905 906 PRINT_REG(actlr_el1); 907 PRINT_REG(afsr0_el1); 908 PRINT_REG(afsr1_el1); 909 PRINT_REG(aidr_el1); 910 PRINT_REG(amair_el1); 911 PRINT_REG(ccsidr_el1); 912 PRINT_REG(clidr_el1); 913 PRINT_REG(contextidr_el1); 914 PRINT_REG(cpacr_el1); 915 PRINT_REG(csselr_el1); 916 PRINT_REG(ctr_el0); 917 PRINT_REG(currentel); 918 PRINT_REG(daif); 919 PRINT_REG(dczid_el0); 920 PRINT_REG(elr_el1); 921 PRINT_REG(esr_el1); 922 PRINT_REG(far_el1); 923 #if 0 924 /* ARM64TODO: Enable VFP before reading floating-point registers */ 925 PRINT_REG(fpcr); 926 PRINT_REG(fpsr); 927 #endif 928 PRINT_REG(id_aa64afr0_el1); 929 PRINT_REG(id_aa64afr1_el1); 930 PRINT_REG(id_aa64dfr0_el1); 931 PRINT_REG(id_aa64dfr1_el1); 932 PRINT_REG(id_aa64isar0_el1); 933 PRINT_REG(id_aa64isar1_el1); 934 PRINT_REG(id_aa64pfr0_el1); 935 PRINT_REG(id_aa64pfr1_el1); 936 PRINT_REG(id_afr0_el1); 937 PRINT_REG(id_dfr0_el1); 938 PRINT_REG(id_isar0_el1); 939 PRINT_REG(id_isar1_el1); 940 PRINT_REG(id_isar2_el1); 941 PRINT_REG(id_isar3_el1); 942 PRINT_REG(id_isar4_el1); 943 PRINT_REG(id_isar5_el1); 944 PRINT_REG(id_mmfr0_el1); 945 PRINT_REG(id_mmfr1_el1); 946 PRINT_REG(id_mmfr2_el1); 947 PRINT_REG(id_mmfr3_el1); 948 #if 0 949 /* Missing from llvm */ 950 PRINT_REG(id_mmfr4_el1); 951 #endif 952 PRINT_REG(id_pfr0_el1); 953 PRINT_REG(id_pfr1_el1); 954 PRINT_REG(isr_el1); 955 PRINT_REG(mair_el1); 956 PRINT_REG(midr_el1); 957 PRINT_REG(mpidr_el1); 958 PRINT_REG(mvfr0_el1); 959 PRINT_REG(mvfr1_el1); 960 PRINT_REG(mvfr2_el1); 961 PRINT_REG(revidr_el1); 962 PRINT_REG(sctlr_el1); 963 PRINT_REG(sp_el0); 964 PRINT_REG(spsel); 965 PRINT_REG(spsr_el1); 966 PRINT_REG(tcr_el1); 967 PRINT_REG(tpidr_el0); 968 PRINT_REG(tpidr_el1); 969 PRINT_REG(tpidrro_el0); 970 PRINT_REG(ttbr0_el1); 971 PRINT_REG(ttbr1_el1); 972 PRINT_REG(vbar_el1); 973 #undef PRINT_REG 974 } 975 976 DB_SHOW_COMMAND(vtop, db_show_vtop) 977 { 978 uint64_t phys; 979 980 if (have_addr) { 981 phys = arm64_address_translate_s1e1r(addr); 982 db_printf("EL1 physical address reg (read): 0x%016lx\n", phys); 983 phys = arm64_address_translate_s1e1w(addr); 984 db_printf("EL1 physical address reg (write): 0x%016lx\n", phys); 985 phys = arm64_address_translate_s1e0r(addr); 986 db_printf("EL0 physical address reg (read): 0x%016lx\n", phys); 987 phys = arm64_address_translate_s1e0w(addr); 988 db_printf("EL0 physical address reg (write): 0x%016lx\n", phys); 989 } else 990 db_printf("show vtop <virt_addr>\n"); 991 } 992 #endif 993