1 /*- 2 * Copyright (c) 2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 */ 27 28 #include "opt_acpi.h" 29 #include "opt_platform.h" 30 #include "opt_ddb.h" 31 32 #include <sys/cdefs.h> 33 __FBSDID("$FreeBSD$"); 34 35 #include <sys/param.h> 36 #include <sys/systm.h> 37 #include <sys/buf.h> 38 #include <sys/bus.h> 39 #include <sys/cons.h> 40 #include <sys/cpu.h> 41 #include <sys/csan.h> 42 #include <sys/devmap.h> 43 #include <sys/efi.h> 44 #include <sys/exec.h> 45 #include <sys/imgact.h> 46 #include <sys/kdb.h> 47 #include <sys/kernel.h> 48 #include <sys/ktr.h> 49 #include <sys/limits.h> 50 #include <sys/linker.h> 51 #include <sys/msgbuf.h> 52 #include <sys/pcpu.h> 53 #include <sys/physmem.h> 54 #include <sys/proc.h> 55 #include <sys/ptrace.h> 56 #include <sys/reboot.h> 57 #include <sys/reg.h> 58 #include <sys/rwlock.h> 59 #include <sys/sched.h> 60 #include <sys/signalvar.h> 61 #include <sys/syscallsubr.h> 62 #include <sys/sysent.h> 63 #include <sys/sysproto.h> 64 #include <sys/ucontext.h> 65 #include <sys/vdso.h> 66 #include <sys/vmmeter.h> 67 68 #include <vm/vm.h> 69 #include <vm/vm_param.h> 70 #include <vm/vm_kern.h> 71 #include <vm/vm_object.h> 72 #include <vm/vm_page.h> 73 #include <vm/vm_phys.h> 74 #include <vm/pmap.h> 75 #include <vm/vm_map.h> 76 #include <vm/vm_pager.h> 77 78 #include <machine/armreg.h> 79 #include <machine/cpu.h> 80 #include <machine/debug_monitor.h> 81 #include <machine/kdb.h> 82 #include <machine/machdep.h> 83 #include <machine/metadata.h> 84 #include <machine/md_var.h> 85 #include <machine/pcb.h> 86 #include <machine/undefined.h> 87 #include <machine/vmparam.h> 88 89 #ifdef VFP 90 #include <machine/vfp.h> 91 #endif 92 93 #ifdef DEV_ACPI 94 #include <contrib/dev/acpica/include/acpi.h> 95 #include <machine/acpica_machdep.h> 96 #endif 97 98 #ifdef FDT 99 #include <dev/fdt/fdt_common.h> 100 #include <dev/ofw/openfirm.h> 101 #endif 102 103 enum arm64_bus arm64_bus_method = ARM64_BUS_NONE; 104 105 /* 106 * XXX: The .bss is assumed to be in the boot CPU NUMA domain. If not we 107 * could relocate this, but will need to keep the same virtual address as 108 * it's reverenced by the EARLY_COUNTER macro. 109 */ 110 struct pcpu pcpu0; 111 112 #if defined(PERTHREAD_SSP) 113 /* 114 * The boot SSP canary. Will be replaced with a per-thread canary when 115 * scheduling has started. 116 */ 117 uintptr_t boot_canary = 0x49a2d892bc05a0b1ul; 118 #endif 119 120 static struct trapframe proc0_tf; 121 122 int early_boot = 1; 123 int cold = 1; 124 static int boot_el; 125 126 struct kva_md_info kmi; 127 128 int64_t dczva_line_size; /* The size of cache line the dc zva zeroes */ 129 int has_pan; 130 131 /* 132 * Physical address of the EFI System Table. Stashed from the metadata hints 133 * passed into the kernel and used by the EFI code to call runtime services. 134 */ 135 vm_paddr_t efi_systbl_phys; 136 static struct efi_map_header *efihdr; 137 138 /* pagezero_* implementations are provided in support.S */ 139 void pagezero_simple(void *); 140 void pagezero_cache(void *); 141 142 /* pagezero_simple is default pagezero */ 143 void (*pagezero)(void *p) = pagezero_simple; 144 145 int (*apei_nmi)(void); 146 147 #if defined(PERTHREAD_SSP_WARNING) 148 static void 149 print_ssp_warning(void *data __unused) 150 { 151 printf("WARNING: Per-thread SSP is enabled but the compiler is too old to support it\n"); 152 } 153 SYSINIT(ssp_warn, SI_SUB_COPYRIGHT, SI_ORDER_ANY, print_ssp_warning, NULL); 154 SYSINIT(ssp_warn2, SI_SUB_LAST, SI_ORDER_ANY, print_ssp_warning, NULL); 155 #endif 156 157 static void 158 pan_setup(void) 159 { 160 uint64_t id_aa64mfr1; 161 162 id_aa64mfr1 = READ_SPECIALREG(id_aa64mmfr1_el1); 163 if (ID_AA64MMFR1_PAN_VAL(id_aa64mfr1) != ID_AA64MMFR1_PAN_NONE) 164 has_pan = 1; 165 } 166 167 void 168 pan_enable(void) 169 { 170 171 /* 172 * The LLVM integrated assembler doesn't understand the PAN 173 * PSTATE field. Because of this we need to manually create 174 * the instruction in an asm block. This is equivalent to: 175 * msr pan, #1 176 * 177 * This sets the PAN bit, stopping the kernel from accessing 178 * memory when userspace can also access it unless the kernel 179 * uses the userspace load/store instructions. 180 */ 181 if (has_pan) { 182 WRITE_SPECIALREG(sctlr_el1, 183 READ_SPECIALREG(sctlr_el1) & ~SCTLR_SPAN); 184 __asm __volatile(".inst 0xd500409f | (0x1 << 8)"); 185 } 186 } 187 188 bool 189 has_hyp(void) 190 { 191 192 return (boot_el == 2); 193 } 194 195 static void 196 cpu_startup(void *dummy) 197 { 198 vm_paddr_t size; 199 int i; 200 201 printf("real memory = %ju (%ju MB)\n", ptoa((uintmax_t)realmem), 202 ptoa((uintmax_t)realmem) / 1024 / 1024); 203 204 if (bootverbose) { 205 printf("Physical memory chunk(s):\n"); 206 for (i = 0; phys_avail[i + 1] != 0; i += 2) { 207 size = phys_avail[i + 1] - phys_avail[i]; 208 printf("%#016jx - %#016jx, %ju bytes (%ju pages)\n", 209 (uintmax_t)phys_avail[i], 210 (uintmax_t)phys_avail[i + 1] - 1, 211 (uintmax_t)size, (uintmax_t)size / PAGE_SIZE); 212 } 213 } 214 215 printf("avail memory = %ju (%ju MB)\n", 216 ptoa((uintmax_t)vm_free_count()), 217 ptoa((uintmax_t)vm_free_count()) / 1024 / 1024); 218 219 undef_init(); 220 install_cpu_errata(); 221 222 vm_ksubmap_init(&kmi); 223 bufinit(); 224 vm_pager_bufferinit(); 225 } 226 227 SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL); 228 229 static void 230 late_ifunc_resolve(void *dummy __unused) 231 { 232 link_elf_late_ireloc(); 233 } 234 SYSINIT(late_ifunc_resolve, SI_SUB_CPU, SI_ORDER_ANY, late_ifunc_resolve, NULL); 235 236 int 237 cpu_idle_wakeup(int cpu) 238 { 239 240 return (0); 241 } 242 243 void 244 cpu_idle(int busy) 245 { 246 247 spinlock_enter(); 248 if (!busy) 249 cpu_idleclock(); 250 if (!sched_runnable()) 251 __asm __volatile( 252 "dsb sy \n" 253 "wfi \n"); 254 if (!busy) 255 cpu_activeclock(); 256 spinlock_exit(); 257 } 258 259 void 260 cpu_halt(void) 261 { 262 263 /* We should have shutdown by now, if not enter a low power sleep */ 264 intr_disable(); 265 while (1) { 266 __asm __volatile("wfi"); 267 } 268 } 269 270 /* 271 * Flush the D-cache for non-DMA I/O so that the I-cache can 272 * be made coherent later. 273 */ 274 void 275 cpu_flush_dcache(void *ptr, size_t len) 276 { 277 278 /* ARM64TODO TBD */ 279 } 280 281 /* Get current clock frequency for the given CPU ID. */ 282 int 283 cpu_est_clockrate(int cpu_id, uint64_t *rate) 284 { 285 struct pcpu *pc; 286 287 pc = pcpu_find(cpu_id); 288 if (pc == NULL || rate == NULL) 289 return (EINVAL); 290 291 if (pc->pc_clock == 0) 292 return (EOPNOTSUPP); 293 294 *rate = pc->pc_clock; 295 return (0); 296 } 297 298 void 299 cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t size) 300 { 301 302 pcpu->pc_acpi_id = 0xffffffff; 303 pcpu->pc_mpidr_low = 0xffffffff; 304 pcpu->pc_mpidr_high = 0xffffffff; 305 } 306 307 void 308 spinlock_enter(void) 309 { 310 struct thread *td; 311 register_t daif; 312 313 td = curthread; 314 if (td->td_md.md_spinlock_count == 0) { 315 daif = intr_disable(); 316 td->td_md.md_spinlock_count = 1; 317 td->td_md.md_saved_daif = daif; 318 critical_enter(); 319 } else 320 td->td_md.md_spinlock_count++; 321 } 322 323 void 324 spinlock_exit(void) 325 { 326 struct thread *td; 327 register_t daif; 328 329 td = curthread; 330 daif = td->td_md.md_saved_daif; 331 td->td_md.md_spinlock_count--; 332 if (td->td_md.md_spinlock_count == 0) { 333 critical_exit(); 334 intr_restore(daif); 335 } 336 } 337 338 /* 339 * Construct a PCB from a trapframe. This is called from kdb_trap() where 340 * we want to start a backtrace from the function that caused us to enter 341 * the debugger. We have the context in the trapframe, but base the trace 342 * on the PCB. The PCB doesn't have to be perfect, as long as it contains 343 * enough for a backtrace. 344 */ 345 void 346 makectx(struct trapframe *tf, struct pcb *pcb) 347 { 348 int i; 349 350 for (i = 0; i < nitems(pcb->pcb_x); i++) 351 pcb->pcb_x[i] = tf->tf_x[i]; 352 353 /* NB: pcb_lr is the PC, see PC_REGS() in db_machdep.h */ 354 pcb->pcb_lr = tf->tf_elr; 355 pcb->pcb_sp = tf->tf_sp; 356 } 357 358 static void 359 init_proc0(vm_offset_t kstack) 360 { 361 struct pcpu *pcpup; 362 363 pcpup = cpuid_to_pcpu[0]; 364 MPASS(pcpup != NULL); 365 366 proc_linkup0(&proc0, &thread0); 367 thread0.td_kstack = kstack; 368 thread0.td_kstack_pages = kstack_pages; 369 #if defined(PERTHREAD_SSP) 370 thread0.td_md.md_canary = boot_canary; 371 #endif 372 thread0.td_pcb = (struct pcb *)(thread0.td_kstack + 373 thread0.td_kstack_pages * PAGE_SIZE) - 1; 374 thread0.td_pcb->pcb_fpflags = 0; 375 thread0.td_pcb->pcb_fpusaved = &thread0.td_pcb->pcb_fpustate; 376 thread0.td_pcb->pcb_vfpcpu = UINT_MAX; 377 thread0.td_frame = &proc0_tf; 378 ptrauth_thread0(&thread0); 379 pcpup->pc_curpcb = thread0.td_pcb; 380 381 /* 382 * Unmask SError exceptions. They are used to signal a RAS failure, 383 * or other hardware error. 384 */ 385 serror_enable(); 386 } 387 388 /* 389 * Get an address to be used to write to kernel data that may be mapped 390 * read-only, e.g. to patch kernel code. 391 */ 392 bool 393 arm64_get_writable_addr(vm_offset_t addr, vm_offset_t *out) 394 { 395 vm_paddr_t pa; 396 397 /* Check if the page is writable */ 398 if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { 399 *out = addr; 400 return (true); 401 } 402 403 /* 404 * Find the physical address of the given page. 405 */ 406 if (!pmap_klookup(addr, &pa)) { 407 return (false); 408 } 409 410 /* 411 * If it is within the DMAP region and is writable use that. 412 */ 413 if (PHYS_IN_DMAP(pa)) { 414 addr = PHYS_TO_DMAP(pa); 415 if (PAR_SUCCESS(arm64_address_translate_s1e1w(addr))) { 416 *out = addr; 417 return (true); 418 } 419 } 420 421 return (false); 422 } 423 424 typedef void (*efi_map_entry_cb)(struct efi_md *, void *argp); 425 426 static void 427 foreach_efi_map_entry(struct efi_map_header *efihdr, efi_map_entry_cb cb, void *argp) 428 { 429 struct efi_md *map, *p; 430 size_t efisz; 431 int ndesc, i; 432 433 /* 434 * Memory map data provided by UEFI via the GetMemoryMap 435 * Boot Services API. 436 */ 437 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 438 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 439 440 if (efihdr->descriptor_size == 0) 441 return; 442 ndesc = efihdr->memory_size / efihdr->descriptor_size; 443 444 for (i = 0, p = map; i < ndesc; i++, 445 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 446 cb(p, argp); 447 } 448 } 449 450 static void 451 exclude_efi_map_entry(struct efi_md *p, void *argp __unused) 452 { 453 454 switch (p->md_type) { 455 case EFI_MD_TYPE_CODE: 456 case EFI_MD_TYPE_DATA: 457 case EFI_MD_TYPE_BS_CODE: 458 case EFI_MD_TYPE_BS_DATA: 459 case EFI_MD_TYPE_FREE: 460 /* 461 * We're allowed to use any entry with these types. 462 */ 463 break; 464 default: 465 physmem_exclude_region(p->md_phys, p->md_pages * EFI_PAGE_SIZE, 466 EXFLAG_NOALLOC); 467 } 468 } 469 470 static void 471 exclude_efi_map_entries(struct efi_map_header *efihdr) 472 { 473 474 foreach_efi_map_entry(efihdr, exclude_efi_map_entry, NULL); 475 } 476 477 static void 478 add_efi_map_entry(struct efi_md *p, void *argp __unused) 479 { 480 481 switch (p->md_type) { 482 case EFI_MD_TYPE_RECLAIM: 483 /* 484 * The recomended location for ACPI tables. Map into the 485 * DMAP so we can access them from userspace via /dev/mem. 486 */ 487 case EFI_MD_TYPE_RT_CODE: 488 /* 489 * Some UEFI implementations put the system table in the 490 * runtime code section. Include it in the DMAP, but will 491 * be excluded from phys_avail later. 492 */ 493 case EFI_MD_TYPE_RT_DATA: 494 /* 495 * Runtime data will be excluded after the DMAP 496 * region is created to stop it from being added 497 * to phys_avail. 498 */ 499 case EFI_MD_TYPE_CODE: 500 case EFI_MD_TYPE_DATA: 501 case EFI_MD_TYPE_BS_CODE: 502 case EFI_MD_TYPE_BS_DATA: 503 case EFI_MD_TYPE_FREE: 504 /* 505 * We're allowed to use any entry with these types. 506 */ 507 physmem_hardware_region(p->md_phys, 508 p->md_pages * EFI_PAGE_SIZE); 509 break; 510 } 511 } 512 513 static void 514 add_efi_map_entries(struct efi_map_header *efihdr) 515 { 516 foreach_efi_map_entry(efihdr, add_efi_map_entry, NULL); 517 } 518 519 static void 520 print_efi_map_entry(struct efi_md *p, void *argp __unused) 521 { 522 const char *type; 523 static const char *types[] = { 524 "Reserved", 525 "LoaderCode", 526 "LoaderData", 527 "BootServicesCode", 528 "BootServicesData", 529 "RuntimeServicesCode", 530 "RuntimeServicesData", 531 "ConventionalMemory", 532 "UnusableMemory", 533 "ACPIReclaimMemory", 534 "ACPIMemoryNVS", 535 "MemoryMappedIO", 536 "MemoryMappedIOPortSpace", 537 "PalCode", 538 "PersistentMemory" 539 }; 540 541 if (p->md_type < nitems(types)) 542 type = types[p->md_type]; 543 else 544 type = "<INVALID>"; 545 printf("%23s %012lx %012lx %08lx ", type, p->md_phys, 546 p->md_virt, p->md_pages); 547 if (p->md_attr & EFI_MD_ATTR_UC) 548 printf("UC "); 549 if (p->md_attr & EFI_MD_ATTR_WC) 550 printf("WC "); 551 if (p->md_attr & EFI_MD_ATTR_WT) 552 printf("WT "); 553 if (p->md_attr & EFI_MD_ATTR_WB) 554 printf("WB "); 555 if (p->md_attr & EFI_MD_ATTR_UCE) 556 printf("UCE "); 557 if (p->md_attr & EFI_MD_ATTR_WP) 558 printf("WP "); 559 if (p->md_attr & EFI_MD_ATTR_RP) 560 printf("RP "); 561 if (p->md_attr & EFI_MD_ATTR_XP) 562 printf("XP "); 563 if (p->md_attr & EFI_MD_ATTR_NV) 564 printf("NV "); 565 if (p->md_attr & EFI_MD_ATTR_MORE_RELIABLE) 566 printf("MORE_RELIABLE "); 567 if (p->md_attr & EFI_MD_ATTR_RO) 568 printf("RO "); 569 if (p->md_attr & EFI_MD_ATTR_RT) 570 printf("RUNTIME"); 571 printf("\n"); 572 } 573 574 static void 575 print_efi_map_entries(struct efi_map_header *efihdr) 576 { 577 578 printf("%23s %12s %12s %8s %4s\n", 579 "Type", "Physical", "Virtual", "#Pages", "Attr"); 580 foreach_efi_map_entry(efihdr, print_efi_map_entry, NULL); 581 } 582 583 /* 584 * Map the passed in VA in EFI space to a void * using the efi memory table to 585 * find the PA and return it in the DMAP, if it exists. We're used between the 586 * calls to pmap_bootstrap() and physmem_init_kernel_globals() to parse CFG 587 * tables We assume that either the entry you are mapping fits within its page, 588 * or if it spills to the next page, that's contiguous in PA and in the DMAP. 589 * All observed tables obey the first part of this precondition. 590 */ 591 struct early_map_data 592 { 593 vm_offset_t va; 594 vm_offset_t pa; 595 }; 596 597 static void 598 efi_early_map_entry(struct efi_md *p, void *argp) 599 { 600 struct early_map_data *emdp = argp; 601 vm_offset_t s, e; 602 603 if (emdp->pa != 0) 604 return; 605 if ((p->md_attr & EFI_MD_ATTR_RT) == 0) 606 return; 607 s = p->md_virt; 608 e = p->md_virt + p->md_pages * EFI_PAGE_SIZE; 609 if (emdp->va < s || emdp->va >= e) 610 return; 611 emdp->pa = p->md_phys + (emdp->va - p->md_virt); 612 } 613 614 static void * 615 efi_early_map(vm_offset_t va) 616 { 617 struct early_map_data emd = { .va = va }; 618 619 foreach_efi_map_entry(efihdr, efi_early_map_entry, &emd); 620 if (emd.pa == 0) 621 return NULL; 622 return (void *)PHYS_TO_DMAP(emd.pa); 623 } 624 625 626 /* 627 * When booted via kboot, the prior kernel will pass in reserved memory areas in 628 * a EFI config table. We need to find that table and walk through it excluding 629 * the memory ranges in it. btw, this is called too early for the printf to do 630 * anything since msgbufp isn't initialized, let alone a console... 631 */ 632 static void 633 exclude_efi_memreserve(vm_offset_t efi_systbl_phys) 634 { 635 struct efi_systbl *systbl; 636 struct uuid efi_memreserve = LINUX_EFI_MEMRESERVE_TABLE; 637 638 systbl = (struct efi_systbl *)PHYS_TO_DMAP(efi_systbl_phys); 639 if (systbl == NULL) { 640 printf("can't map systbl\n"); 641 return; 642 } 643 if (systbl->st_hdr.th_sig != EFI_SYSTBL_SIG) { 644 printf("Bad signature for systbl %#lx\n", systbl->st_hdr.th_sig); 645 return; 646 } 647 648 /* 649 * We don't yet have the pmap system booted enough to create a pmap for 650 * the efi firmware's preferred address space from the GetMemoryMap() 651 * table. The st_cfgtbl is a VA in this space, so we need to do the 652 * mapping ourselves to a kernel VA with efi_early_map. We assume that 653 * the cfgtbl entries don't span a page. Other pointers are PAs, as 654 * noted below. 655 */ 656 if (systbl->st_cfgtbl == 0) /* Failsafe st_entries should == 0 in this case */ 657 return; 658 for (int i = 0; i < systbl->st_entries; i++) { 659 struct efi_cfgtbl *cfgtbl; 660 struct linux_efi_memreserve *mr; 661 662 cfgtbl = efi_early_map(systbl->st_cfgtbl + i * sizeof(*cfgtbl)); 663 if (cfgtbl == NULL) 664 panic("Can't map the config table entry %d\n", i); 665 if (memcmp(&cfgtbl->ct_uuid, &efi_memreserve, sizeof(struct uuid)) != 0) 666 continue; 667 668 /* 669 * cfgtbl points are either VA or PA, depending on the GUID of 670 * the table. memreserve GUID pointers are PA and not converted 671 * after a SetVirtualAddressMap(). The list's mr_next pointer 672 * is also a PA. 673 */ 674 mr = (struct linux_efi_memreserve *)PHYS_TO_DMAP( 675 (vm_offset_t)cfgtbl->ct_data); 676 while (true) { 677 for (int j = 0; j < mr->mr_count; j++) { 678 struct linux_efi_memreserve_entry *mre; 679 680 mre = &mr->mr_entry[j]; 681 physmem_exclude_region(mre->mre_base, mre->mre_size, 682 EXFLAG_NODUMP | EXFLAG_NOALLOC); 683 } 684 if (mr->mr_next == 0) 685 break; 686 mr = (struct linux_efi_memreserve *)PHYS_TO_DMAP(mr->mr_next); 687 }; 688 } 689 690 } 691 692 #ifdef FDT 693 static void 694 try_load_dtb(caddr_t kmdp) 695 { 696 vm_offset_t dtbp; 697 698 dtbp = MD_FETCH(kmdp, MODINFOMD_DTBP, vm_offset_t); 699 #if defined(FDT_DTB_STATIC) 700 /* 701 * In case the device tree blob was not retrieved (from metadata) try 702 * to use the statically embedded one. 703 */ 704 if (dtbp == 0) 705 dtbp = (vm_offset_t)&fdt_static_dtb; 706 #endif 707 708 if (dtbp == (vm_offset_t)NULL) { 709 #ifndef TSLOG 710 printf("ERROR loading DTB\n"); 711 #endif 712 return; 713 } 714 715 if (OF_install(OFW_FDT, 0) == FALSE) 716 panic("Cannot install FDT"); 717 718 if (OF_init((void *)dtbp) != 0) 719 panic("OF_init failed with the found device tree"); 720 721 parse_fdt_bootargs(); 722 } 723 #endif 724 725 static bool 726 bus_probe(void) 727 { 728 bool has_acpi, has_fdt; 729 char *order, *env; 730 731 has_acpi = has_fdt = false; 732 733 #ifdef FDT 734 has_fdt = (OF_peer(0) != 0); 735 #endif 736 #ifdef DEV_ACPI 737 has_acpi = (AcpiOsGetRootPointer() != 0); 738 #endif 739 740 env = kern_getenv("kern.cfg.order"); 741 if (env != NULL) { 742 order = env; 743 while (order != NULL) { 744 if (has_acpi && 745 strncmp(order, "acpi", 4) == 0 && 746 (order[4] == ',' || order[4] == '\0')) { 747 arm64_bus_method = ARM64_BUS_ACPI; 748 break; 749 } 750 if (has_fdt && 751 strncmp(order, "fdt", 3) == 0 && 752 (order[3] == ',' || order[3] == '\0')) { 753 arm64_bus_method = ARM64_BUS_FDT; 754 break; 755 } 756 order = strchr(order, ','); 757 if (order != NULL) 758 order++; /* Skip comma */ 759 } 760 freeenv(env); 761 762 /* If we set the bus method it is valid */ 763 if (arm64_bus_method != ARM64_BUS_NONE) 764 return (true); 765 } 766 /* If no order or an invalid order was set use the default */ 767 if (arm64_bus_method == ARM64_BUS_NONE) { 768 if (has_fdt) 769 arm64_bus_method = ARM64_BUS_FDT; 770 else if (has_acpi) 771 arm64_bus_method = ARM64_BUS_ACPI; 772 } 773 774 /* 775 * If no option was set the default is valid, otherwise we are 776 * setting one to get cninit() working, then calling panic to tell 777 * the user about the invalid bus setup. 778 */ 779 return (env == NULL); 780 } 781 782 static void 783 cache_setup(void) 784 { 785 int dczva_line_shift; 786 uint32_t dczid_el0; 787 788 identify_cache(READ_SPECIALREG(ctr_el0)); 789 790 dczid_el0 = READ_SPECIALREG(dczid_el0); 791 792 /* Check if dc zva is not prohibited */ 793 if (dczid_el0 & DCZID_DZP) 794 dczva_line_size = 0; 795 else { 796 /* Same as with above calculations */ 797 dczva_line_shift = DCZID_BS_SIZE(dczid_el0); 798 dczva_line_size = sizeof(int) << dczva_line_shift; 799 800 /* Change pagezero function */ 801 pagezero = pagezero_cache; 802 } 803 } 804 805 int 806 memory_mapping_mode(vm_paddr_t pa) 807 { 808 struct efi_md *map, *p; 809 size_t efisz; 810 int ndesc, i; 811 812 if (efihdr == NULL) 813 return (VM_MEMATTR_WRITE_BACK); 814 815 /* 816 * Memory map data provided by UEFI via the GetMemoryMap 817 * Boot Services API. 818 */ 819 efisz = (sizeof(struct efi_map_header) + 0xf) & ~0xf; 820 map = (struct efi_md *)((uint8_t *)efihdr + efisz); 821 822 if (efihdr->descriptor_size == 0) 823 return (VM_MEMATTR_WRITE_BACK); 824 ndesc = efihdr->memory_size / efihdr->descriptor_size; 825 826 for (i = 0, p = map; i < ndesc; i++, 827 p = efi_next_descriptor(p, efihdr->descriptor_size)) { 828 if (pa < p->md_phys || 829 pa >= p->md_phys + p->md_pages * EFI_PAGE_SIZE) 830 continue; 831 if (p->md_type == EFI_MD_TYPE_IOMEM || 832 p->md_type == EFI_MD_TYPE_IOPORT) 833 return (VM_MEMATTR_DEVICE); 834 else if ((p->md_attr & EFI_MD_ATTR_WB) != 0 || 835 p->md_type == EFI_MD_TYPE_RECLAIM) 836 return (VM_MEMATTR_WRITE_BACK); 837 else if ((p->md_attr & EFI_MD_ATTR_WT) != 0) 838 return (VM_MEMATTR_WRITE_THROUGH); 839 else if ((p->md_attr & EFI_MD_ATTR_WC) != 0) 840 return (VM_MEMATTR_WRITE_COMBINING); 841 break; 842 } 843 844 return (VM_MEMATTR_DEVICE); 845 } 846 847 void 848 initarm(struct arm64_bootparams *abp) 849 { 850 struct efi_fb *efifb; 851 struct pcpu *pcpup; 852 char *env; 853 #ifdef FDT 854 struct mem_region mem_regions[FDT_MEM_REGIONS]; 855 int mem_regions_sz; 856 phandle_t root; 857 char dts_version[255]; 858 #endif 859 vm_offset_t lastaddr; 860 caddr_t kmdp; 861 bool valid; 862 863 TSRAW(&thread0, TS_ENTER, __func__, NULL); 864 865 boot_el = abp->boot_el; 866 867 /* Parse loader or FDT boot parametes. Determine last used address. */ 868 lastaddr = parse_boot_param(abp); 869 870 /* Find the kernel address */ 871 kmdp = preload_search_by_type("elf kernel"); 872 if (kmdp == NULL) 873 kmdp = preload_search_by_type("elf64 kernel"); 874 875 identify_cpu(0); 876 update_special_regs(0); 877 878 link_elf_ireloc(kmdp); 879 #ifdef FDT 880 try_load_dtb(kmdp); 881 #endif 882 883 efi_systbl_phys = MD_FETCH(kmdp, MODINFOMD_FW_HANDLE, vm_paddr_t); 884 885 /* Load the physical memory ranges */ 886 efihdr = (struct efi_map_header *)preload_search_info(kmdp, 887 MODINFO_METADATA | MODINFOMD_EFI_MAP); 888 if (efihdr != NULL) 889 add_efi_map_entries(efihdr); 890 #ifdef FDT 891 else { 892 /* Grab physical memory regions information from device tree. */ 893 if (fdt_get_mem_regions(mem_regions, &mem_regions_sz, 894 NULL) != 0) 895 panic("Cannot get physical memory regions"); 896 physmem_hardware_regions(mem_regions, mem_regions_sz); 897 } 898 if (fdt_get_reserved_mem(mem_regions, &mem_regions_sz) == 0) 899 physmem_exclude_regions(mem_regions, mem_regions_sz, 900 EXFLAG_NODUMP | EXFLAG_NOALLOC); 901 #endif 902 903 /* Exclude the EFI framebuffer from our view of physical memory. */ 904 efifb = (struct efi_fb *)preload_search_info(kmdp, 905 MODINFO_METADATA | MODINFOMD_EFI_FB); 906 if (efifb != NULL) 907 physmem_exclude_region(efifb->fb_addr, efifb->fb_size, 908 EXFLAG_NOALLOC); 909 910 /* Set the pcpu data, this is needed by pmap_bootstrap */ 911 pcpup = &pcpu0; 912 pcpu_init(pcpup, 0, sizeof(struct pcpu)); 913 914 /* 915 * Set the pcpu pointer with a backup in tpidr_el1 to be 916 * loaded when entering the kernel from userland. 917 */ 918 __asm __volatile( 919 "mov x18, %0 \n" 920 "msr tpidr_el1, %0" :: "r"(pcpup)); 921 922 /* locore.S sets sp_el0 to &thread0 so no need to set it here. */ 923 PCPU_SET(curthread, &thread0); 924 PCPU_SET(midr, get_midr()); 925 926 /* Do basic tuning, hz etc */ 927 init_param1(); 928 929 cache_setup(); 930 pan_setup(); 931 932 /* Bootstrap enough of pmap to enter the kernel proper */ 933 pmap_bootstrap(KERNBASE - abp->kern_delta, lastaddr - KERNBASE); 934 /* Exclude entries needed in the DMAP region, but not phys_avail */ 935 if (efihdr != NULL) 936 exclude_efi_map_entries(efihdr); 937 /* Do the same for reserve entries in the EFI MEMRESERVE table */ 938 if (efi_systbl_phys != 0) 939 exclude_efi_memreserve(efi_systbl_phys); 940 physmem_init_kernel_globals(); 941 942 devmap_bootstrap(0, NULL); 943 944 valid = bus_probe(); 945 946 cninit(); 947 set_ttbr0(abp->kern_ttbr0); 948 cpu_tlb_flushID(); 949 950 if (!valid) 951 panic("Invalid bus configuration: %s", 952 kern_getenv("kern.cfg.order")); 953 954 /* 955 * Check if pointer authentication is available on this system, and 956 * if so enable its use. This needs to be called before init_proc0 957 * as that will configure the thread0 pointer authentication keys. 958 */ 959 ptrauth_init(); 960 961 /* 962 * Dump the boot metadata. We have to wait for cninit() since console 963 * output is required. If it's grossly incorrect the kernel will never 964 * make it this far. 965 */ 966 if (getenv_is_true("debug.dump_modinfo_at_boot")) 967 preload_dump(); 968 969 init_proc0(abp->kern_stack); 970 msgbufinit(msgbufp, msgbufsize); 971 mutex_init(); 972 init_param2(physmem); 973 974 dbg_init(); 975 kdb_init(); 976 #ifdef KDB 977 if ((boothowto & RB_KDB) != 0) 978 kdb_enter(KDB_WHY_BOOTFLAGS, "Boot flags requested debugger"); 979 #endif 980 pan_enable(); 981 982 kcsan_cpu_init(0); 983 984 env = kern_getenv("kernelname"); 985 if (env != NULL) 986 strlcpy(kernelname, env, sizeof(kernelname)); 987 988 #ifdef FDT 989 if (arm64_bus_method == ARM64_BUS_FDT) { 990 root = OF_finddevice("/"); 991 if (OF_getprop(root, "freebsd,dts-version", dts_version, sizeof(dts_version)) > 0) { 992 if (strcmp(LINUX_DTS_VERSION, dts_version) != 0) 993 printf("WARNING: DTB version is %s while kernel expects %s, " 994 "please update the DTB in the ESP\n", 995 dts_version, 996 LINUX_DTS_VERSION); 997 } else { 998 printf("WARNING: Cannot find freebsd,dts-version property, " 999 "cannot check DTB compliance\n"); 1000 } 1001 } 1002 #endif 1003 1004 if (boothowto & RB_VERBOSE) { 1005 if (efihdr != NULL) 1006 print_efi_map_entries(efihdr); 1007 physmem_print_tables(); 1008 } 1009 1010 early_boot = 0; 1011 1012 TSEXIT(); 1013 } 1014 1015 void 1016 dbg_init(void) 1017 { 1018 1019 /* Clear OS lock */ 1020 WRITE_SPECIALREG(oslar_el1, 0); 1021 1022 /* This permits DDB to use debug registers for watchpoints. */ 1023 dbg_monitor_init(); 1024 1025 /* TODO: Eventually will need to initialize debug registers here. */ 1026 } 1027 1028 #ifdef DDB 1029 #include <ddb/ddb.h> 1030 1031 DB_SHOW_COMMAND(specialregs, db_show_spregs) 1032 { 1033 #define PRINT_REG(reg) \ 1034 db_printf(__STRING(reg) " = %#016lx\n", READ_SPECIALREG(reg)) 1035 1036 PRINT_REG(actlr_el1); 1037 PRINT_REG(afsr0_el1); 1038 PRINT_REG(afsr1_el1); 1039 PRINT_REG(aidr_el1); 1040 PRINT_REG(amair_el1); 1041 PRINT_REG(ccsidr_el1); 1042 PRINT_REG(clidr_el1); 1043 PRINT_REG(contextidr_el1); 1044 PRINT_REG(cpacr_el1); 1045 PRINT_REG(csselr_el1); 1046 PRINT_REG(ctr_el0); 1047 PRINT_REG(currentel); 1048 PRINT_REG(daif); 1049 PRINT_REG(dczid_el0); 1050 PRINT_REG(elr_el1); 1051 PRINT_REG(esr_el1); 1052 PRINT_REG(far_el1); 1053 #if 0 1054 /* ARM64TODO: Enable VFP before reading floating-point registers */ 1055 PRINT_REG(fpcr); 1056 PRINT_REG(fpsr); 1057 #endif 1058 PRINT_REG(id_aa64afr0_el1); 1059 PRINT_REG(id_aa64afr1_el1); 1060 PRINT_REG(id_aa64dfr0_el1); 1061 PRINT_REG(id_aa64dfr1_el1); 1062 PRINT_REG(id_aa64isar0_el1); 1063 PRINT_REG(id_aa64isar1_el1); 1064 PRINT_REG(id_aa64pfr0_el1); 1065 PRINT_REG(id_aa64pfr1_el1); 1066 PRINT_REG(id_afr0_el1); 1067 PRINT_REG(id_dfr0_el1); 1068 PRINT_REG(id_isar0_el1); 1069 PRINT_REG(id_isar1_el1); 1070 PRINT_REG(id_isar2_el1); 1071 PRINT_REG(id_isar3_el1); 1072 PRINT_REG(id_isar4_el1); 1073 PRINT_REG(id_isar5_el1); 1074 PRINT_REG(id_mmfr0_el1); 1075 PRINT_REG(id_mmfr1_el1); 1076 PRINT_REG(id_mmfr2_el1); 1077 PRINT_REG(id_mmfr3_el1); 1078 #if 0 1079 /* Missing from llvm */ 1080 PRINT_REG(id_mmfr4_el1); 1081 #endif 1082 PRINT_REG(id_pfr0_el1); 1083 PRINT_REG(id_pfr1_el1); 1084 PRINT_REG(isr_el1); 1085 PRINT_REG(mair_el1); 1086 PRINT_REG(midr_el1); 1087 PRINT_REG(mpidr_el1); 1088 PRINT_REG(mvfr0_el1); 1089 PRINT_REG(mvfr1_el1); 1090 PRINT_REG(mvfr2_el1); 1091 PRINT_REG(revidr_el1); 1092 PRINT_REG(sctlr_el1); 1093 PRINT_REG(sp_el0); 1094 PRINT_REG(spsel); 1095 PRINT_REG(spsr_el1); 1096 PRINT_REG(tcr_el1); 1097 PRINT_REG(tpidr_el0); 1098 PRINT_REG(tpidr_el1); 1099 PRINT_REG(tpidrro_el0); 1100 PRINT_REG(ttbr0_el1); 1101 PRINT_REG(ttbr1_el1); 1102 PRINT_REG(vbar_el1); 1103 #undef PRINT_REG 1104 } 1105 1106 DB_SHOW_COMMAND(vtop, db_show_vtop) 1107 { 1108 uint64_t phys; 1109 1110 if (have_addr) { 1111 phys = arm64_address_translate_s1e1r(addr); 1112 db_printf("EL1 physical address reg (read): 0x%016lx\n", phys); 1113 phys = arm64_address_translate_s1e1w(addr); 1114 db_printf("EL1 physical address reg (write): 0x%016lx\n", phys); 1115 phys = arm64_address_translate_s1e0r(addr); 1116 db_printf("EL0 physical address reg (read): 0x%016lx\n", phys); 1117 phys = arm64_address_translate_s1e0w(addr); 1118 db_printf("EL0 physical address reg (write): 0x%016lx\n", phys); 1119 } else 1120 db_printf("show vtop <virt_addr>\n"); 1121 } 1122 #endif 1123