1e5acd89cSAndrew Turner/*- 2e5acd89cSAndrew Turner * Copyright (c) 2012-2014 Andrew Turner 3e5acd89cSAndrew Turner * All rights reserved. 4e5acd89cSAndrew Turner * 5e5acd89cSAndrew Turner * Redistribution and use in source and binary forms, with or without 6e5acd89cSAndrew Turner * modification, are permitted provided that the following conditions 7e5acd89cSAndrew Turner * are met: 8e5acd89cSAndrew Turner * 1. Redistributions of source code must retain the above copyright 9e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer. 10e5acd89cSAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright 11e5acd89cSAndrew Turner * notice, this list of conditions and the following disclaimer in the 12e5acd89cSAndrew Turner * documentation and/or other materials provided with the distribution. 13e5acd89cSAndrew Turner * 14e5acd89cSAndrew Turner * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15e5acd89cSAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16e5acd89cSAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17e5acd89cSAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18e5acd89cSAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19e5acd89cSAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20e5acd89cSAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21e5acd89cSAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22e5acd89cSAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23e5acd89cSAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24e5acd89cSAndrew Turner * SUCH DAMAGE. 25e5acd89cSAndrew Turner */ 26e5acd89cSAndrew Turner 27fc2a8776SEd Maste#include "assym.inc" 28721555e7SZbigniew Bodek#include "opt_kstack_pages.h" 29e5acd89cSAndrew Turner#include <sys/syscall.h> 30e5acd89cSAndrew Turner#include <machine/asm.h> 31e5acd89cSAndrew Turner#include <machine/armreg.h> 32c78ebc69SAndrew Turner#include <machine/cpu.h> 33e5acd89cSAndrew Turner#include <machine/hypervisor.h> 34e5acd89cSAndrew Turner#include <machine/param.h> 35e5acd89cSAndrew Turner#include <machine/pte.h> 36ef3e1e13SAndrew Turner#include <machine/vm.h> 37c7d4b461SAndrew Turner#include <machine/vmparam.h> 38e5acd89cSAndrew Turner 39f2f21fafSAndrew Turner#define VIRT_BITS 48 40e5acd89cSAndrew Turner 4136f1526aSAndrew Turner#if PAGE_SIZE == PAGE_SIZE_16K 4236f1526aSAndrew Turner/* 4336f1526aSAndrew Turner * The number of level 3 tables to create. 32 will allow for 1G of address 4436f1526aSAndrew Turner * space, the same as a single level 2 page with 4k pages. 4536f1526aSAndrew Turner */ 4636f1526aSAndrew Turner#define L3_PAGE_COUNT 32 4736f1526aSAndrew Turner#endif 4836f1526aSAndrew Turner 4990372a9eSMark Johnston/* 5090372a9eSMark Johnston * The size of our bootstrap stack. 5190372a9eSMark Johnston */ 5290372a9eSMark Johnston#define BOOT_STACK_SIZE (KSTACK_PAGES * PAGE_SIZE) 5390372a9eSMark Johnston 54e5acd89cSAndrew Turner .globl kernbase 55e5acd89cSAndrew Turner .set kernbase, KERNBASE 56e5acd89cSAndrew Turner 57e5acd89cSAndrew Turner/* 58e5acd89cSAndrew Turner * We assume: 59e5acd89cSAndrew Turner * MMU on with an identity map, or off 60e5acd89cSAndrew Turner * D-Cache: off 61e5acd89cSAndrew Turner * I-Cache: on or off 62e5acd89cSAndrew Turner * We are loaded at a 2MiB aligned address 63e5acd89cSAndrew Turner */ 64e5acd89cSAndrew Turner 65edb48ff6SAndrew TurnerENTRY(_start) 66801160f4SAndrew Turner /* Enter the kernel exception level */ 67801160f4SAndrew Turner bl enter_kernel_el 68e5acd89cSAndrew Turner 69e5acd89cSAndrew Turner /* 70e5acd89cSAndrew Turner * Disable the MMU. We may have entered the kernel with it on and 71e5acd89cSAndrew Turner * will need to update the tables later. If this has been set up 72e5acd89cSAndrew Turner * with anything other than a VA == PA map then this will fail, 73e5acd89cSAndrew Turner * but in this case the code to find where we are running from 74e5acd89cSAndrew Turner * would have also failed. 75e5acd89cSAndrew Turner */ 76e5acd89cSAndrew Turner dsb sy 77e5acd89cSAndrew Turner mrs x2, sctlr_el1 78e5acd89cSAndrew Turner bic x2, x2, SCTLR_M 79e5acd89cSAndrew Turner msr sctlr_el1, x2 80e5acd89cSAndrew Turner isb 81e5acd89cSAndrew Turner 82b2b55077SAndrew Turner /* Set the context id */ 83b2b55077SAndrew Turner msr contextidr_el1, xzr 84e5acd89cSAndrew Turner 85e5acd89cSAndrew Turner /* Get the virt -> phys offset */ 86ba313626SAndrew Turner bl get_load_phys_addr 87e5acd89cSAndrew Turner 88e5acd89cSAndrew Turner /* 89e5acd89cSAndrew Turner * At this point: 90e5acd89cSAndrew Turner * x28 = Our physical load address 91e5acd89cSAndrew Turner */ 92e5acd89cSAndrew Turner 93e5acd89cSAndrew Turner /* Create the page tables */ 94e5acd89cSAndrew Turner bl create_pagetables 95e5acd89cSAndrew Turner 96e5acd89cSAndrew Turner /* 97e5acd89cSAndrew Turner * At this point: 98e5acd89cSAndrew Turner * x27 = TTBR0 table 99f2f21fafSAndrew Turner * x26 = Kernel L1 table 100f2f21fafSAndrew Turner * x24 = TTBR1 table 101e5acd89cSAndrew Turner */ 102e5acd89cSAndrew Turner 103e5acd89cSAndrew Turner /* Enable the mmu */ 104e5acd89cSAndrew Turner bl start_mmu 105e5acd89cSAndrew Turner 106857ab36fSAndrew Turner /* Load the new ttbr0 pagetable */ 107659f1a6aSAndrew Turner adrp x27, pagetable_l0_ttbr0 108659f1a6aSAndrew Turner add x27, x27, :lo12:pagetable_l0_ttbr0 109857ab36fSAndrew Turner 110e5acd89cSAndrew Turner /* Jump to the virtual address space */ 111e5acd89cSAndrew Turner ldr x15, .Lvirtdone 112e5acd89cSAndrew Turner br x15 113e5acd89cSAndrew Turner 114e5acd89cSAndrew Turnervirtdone: 115e340882dSAndrew Turner BTI_J 116e340882dSAndrew Turner 117e5acd89cSAndrew Turner /* Set up the stack */ 118659f1a6aSAndrew Turner adrp x25, initstack_end 119659f1a6aSAndrew Turner add x25, x25, :lo12:initstack_end 120e1fe3470SAlfonso Gregory sub sp, x25, #PCB_SIZE 121e5acd89cSAndrew Turner 122e5acd89cSAndrew Turner /* Zero the BSS */ 123e5acd89cSAndrew Turner ldr x15, .Lbss 124e5acd89cSAndrew Turner ldr x14, .Lend 125e5acd89cSAndrew Turner1: 126e5acd89cSAndrew Turner str xzr, [x15], #8 127e5acd89cSAndrew Turner cmp x15, x14 128e5acd89cSAndrew Turner b.lo 1b 129e5acd89cSAndrew Turner 130ae92ace0SAndrew Turner#if defined(PERTHREAD_SSP) 131ae92ace0SAndrew Turner /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 132ae92ace0SAndrew Turner adrp x15, boot_canary 133ae92ace0SAndrew Turner add x15, x15, :lo12:boot_canary 134ae92ace0SAndrew Turner msr sp_el0, x15 135ae92ace0SAndrew Turner#endif 136ae92ace0SAndrew Turner 137e5acd89cSAndrew Turner /* Backup the module pointer */ 138e5acd89cSAndrew Turner mov x1, x0 139e5acd89cSAndrew Turner 14015f8f720SAndrew Turner sub sp, sp, #BOOTPARAMS_SIZE 141e5acd89cSAndrew Turner mov x0, sp 142e5acd89cSAndrew Turner 14315f8f720SAndrew Turner str x1, [x0, #BP_MODULEP] 144659f1a6aSAndrew Turner adrp x25, initstack 145659f1a6aSAndrew Turner add x25, x25, :lo12:initstack 14615f8f720SAndrew Turner str x25, [x0, #BP_KERN_STACK] 147047110dfSAndrew Turner str x27, [x0, #BP_KERN_TTBR0] 148228b87bcSAndrew Turner str x23, [x0, #BP_BOOT_EL] 149d2ae03baSKyle Evans str x4, [x0, #BP_HCR_EL2] 150e5acd89cSAndrew Turner 15189c52f9dSKyle Evans#ifdef KASAN 15289c52f9dSKyle Evans /* Save bootparams */ 15389c52f9dSKyle Evans mov x19, x0 15489c52f9dSKyle Evans 15589c52f9dSKyle Evans /* Bootstrap an early shadow map for the boot stack. */ 15690372a9eSMark Johnston ldr x0, [x0, #BP_KERN_STACK] 15790372a9eSMark Johnston ldr x1, =BOOT_STACK_SIZE 15890372a9eSMark Johnston bl kasan_init_early 15989c52f9dSKyle Evans 16089c52f9dSKyle Evans /* Restore bootparams */ 16189c52f9dSKyle Evans mov x0, x19 16289c52f9dSKyle Evans#endif 16389c52f9dSKyle Evans 164e5acd89cSAndrew Turner /* trace back starts here */ 165e5acd89cSAndrew Turner mov fp, #0 166e5acd89cSAndrew Turner /* Branch to C code */ 167e5acd89cSAndrew Turner bl initarm 1683e2dc667SAndrew Turner /* We are done with the boot params */ 1693e2dc667SAndrew Turner add sp, sp, #BOOTPARAMS_SIZE 17085b7c566SAndrew Turner 17185b7c566SAndrew Turner /* 17285b7c566SAndrew Turner * Enable pointer authentication in the kernel. We set the keys for 17385b7c566SAndrew Turner * thread0 in initarm so have to wait until it returns to enable it. 17485b7c566SAndrew Turner * If we were to enable it in initarm then any authentication when 17585b7c566SAndrew Turner * returning would fail as it was called with pointer authentication 17685b7c566SAndrew Turner * disabled. 17785b7c566SAndrew Turner */ 17885b7c566SAndrew Turner bl ptrauth_start 17985b7c566SAndrew Turner 180e5acd89cSAndrew Turner bl mi_startup 181e5acd89cSAndrew Turner 182e5acd89cSAndrew Turner /* We should not get here */ 183e5acd89cSAndrew Turner brk 0 184e5acd89cSAndrew Turner 185e5acd89cSAndrew Turner .align 3 186e5acd89cSAndrew Turner.Lvirtdone: 187e5acd89cSAndrew Turner .quad virtdone 188e5acd89cSAndrew Turner.Lbss: 189e5acd89cSAndrew Turner .quad __bss_start 190e5acd89cSAndrew Turner.Lend: 191166ceb6fSAndrew Turner .quad __bss_end 192edb48ff6SAndrew TurnerEND(_start) 193e5acd89cSAndrew Turner 194b2b55077SAndrew Turner#ifdef SMP 195b2b55077SAndrew Turner/* 196c78ebc69SAndrew Turner * void 197c78ebc69SAndrew Turner * mpentry_psci(unsigned long) 198b2b55077SAndrew Turner * 199c78ebc69SAndrew Turner * Called by a core when it is being brought online with psci. 200b2b55077SAndrew Turner * The data in x0 is passed straight to init_secondary. 201b2b55077SAndrew Turner */ 202c78ebc69SAndrew TurnerENTRY(mpentry_psci) 203c78ebc69SAndrew Turner mov x26, xzr 204c78ebc69SAndrew Turner b mpentry_common 205c78ebc69SAndrew TurnerEND(mpentry_psci) 206c78ebc69SAndrew Turner 207c78ebc69SAndrew Turner/* 208c78ebc69SAndrew Turner * void 209c78ebc69SAndrew Turner * mpentry_spintable(void) 210c78ebc69SAndrew Turner * 211c78ebc69SAndrew Turner * Called by a core when it is being brought online with a spin-table. 212c78ebc69SAndrew Turner * Reads the new CPU ID and passes this to init_secondary. 213c78ebc69SAndrew Turner */ 214c78ebc69SAndrew TurnerENTRY(mpentry_spintable) 215c78ebc69SAndrew Turner ldr x26, =spintable_wait 216c78ebc69SAndrew Turner b mpentry_common 217c78ebc69SAndrew TurnerEND(mpentry_spintable) 218c78ebc69SAndrew Turner 219c78ebc69SAndrew Turner/* Wait for the current CPU to be released */ 220c78ebc69SAndrew TurnerLENTRY(spintable_wait) 221c78ebc69SAndrew Turner /* Read the affinity bits from mpidr_el1 */ 222c78ebc69SAndrew Turner mrs x1, mpidr_el1 223c78ebc69SAndrew Turner ldr x2, =CPU_AFF_MASK 224c78ebc69SAndrew Turner and x1, x1, x2 225c78ebc69SAndrew Turner 226c78ebc69SAndrew Turner adrp x2, ap_cpuid 227c78ebc69SAndrew Turner1: 228c78ebc69SAndrew Turner ldr x0, [x2, :lo12:ap_cpuid] 229c78ebc69SAndrew Turner cmp x0, x1 230c78ebc69SAndrew Turner b.ne 1b 231c78ebc69SAndrew Turner 232c78ebc69SAndrew Turner str xzr, [x2, :lo12:ap_cpuid] 233c78ebc69SAndrew Turner dsb sy 234c78ebc69SAndrew Turner sev 235c78ebc69SAndrew Turner 236c78ebc69SAndrew Turner ret 237c78ebc69SAndrew TurnerLEND(mpentry_spintable) 238c78ebc69SAndrew Turner 239c78ebc69SAndrew TurnerLENTRY(mpentry_common) 240b2b55077SAndrew Turner /* Disable interrupts */ 241337eb2abSAndrew Turner msr daifset, #DAIF_INTR 242b2b55077SAndrew Turner 243801160f4SAndrew Turner /* Enter the kernel exception level */ 244801160f4SAndrew Turner bl enter_kernel_el 245b2b55077SAndrew Turner 246b2b55077SAndrew Turner /* Set the context id */ 247e426794fSMichal Meloun msr contextidr_el1, xzr 248b2b55077SAndrew Turner 249b2b55077SAndrew Turner /* Load the kernel page table */ 250659f1a6aSAndrew Turner adrp x24, pagetable_l0_ttbr1 251659f1a6aSAndrew Turner add x24, x24, :lo12:pagetable_l0_ttbr1 252b2b55077SAndrew Turner /* Load the identity page table */ 253c6a6ec85SAlfredo Mazzinghi adrp x27, pagetable_l0_ttbr0_bootstrap 254c6a6ec85SAlfredo Mazzinghi add x27, x27, :lo12:pagetable_l0_ttbr0_bootstrap 255b2b55077SAndrew Turner 256b2b55077SAndrew Turner /* Enable the mmu */ 257b2b55077SAndrew Turner bl start_mmu 258b2b55077SAndrew Turner 259857ab36fSAndrew Turner /* Load the new ttbr0 pagetable */ 260659f1a6aSAndrew Turner adrp x27, pagetable_l0_ttbr0 261659f1a6aSAndrew Turner add x27, x27, :lo12:pagetable_l0_ttbr0 262857ab36fSAndrew Turner 263b2b55077SAndrew Turner /* Jump to the virtual address space */ 264b2b55077SAndrew Turner ldr x15, =mp_virtdone 265b2b55077SAndrew Turner br x15 266b2b55077SAndrew Turner 267b2b55077SAndrew Turnermp_virtdone: 268e340882dSAndrew Turner BTI_J 269e340882dSAndrew Turner 270c78ebc69SAndrew Turner /* 271c78ebc69SAndrew Turner * Allow this CPU to wait until the kernel is ready for it, 272c78ebc69SAndrew Turner * e.g. with spin-table but each CPU uses the same release address 273c78ebc69SAndrew Turner */ 274c78ebc69SAndrew Turner cbz x26, 1f 275c78ebc69SAndrew Turner blr x26 276c78ebc69SAndrew Turner1: 277c78ebc69SAndrew Turner 2788db2e8fdSMark Johnston /* Start using the AP boot stack */ 2797eb26be9SAndrew Turner adrp x4, bootstack 2807eb26be9SAndrew Turner ldr x4, [x4, :lo12:bootstack] 2818db2e8fdSMark Johnston mov sp, x4 282857ab36fSAndrew Turner 283ae92ace0SAndrew Turner#if defined(PERTHREAD_SSP) 284ae92ace0SAndrew Turner /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 285ae92ace0SAndrew Turner adrp x15, boot_canary 286ae92ace0SAndrew Turner add x15, x15, :lo12:boot_canary 287ae92ace0SAndrew Turner msr sp_el0, x15 288ae92ace0SAndrew Turner#endif 289ae92ace0SAndrew Turner 290857ab36fSAndrew Turner /* Load the kernel ttbr0 pagetable */ 291857ab36fSAndrew Turner msr ttbr0_el1, x27 292857ab36fSAndrew Turner isb 293857ab36fSAndrew Turner 294857ab36fSAndrew Turner /* Invalidate the TLB */ 295857ab36fSAndrew Turner tlbi vmalle1 296857ab36fSAndrew Turner dsb sy 297857ab36fSAndrew Turner isb 298857ab36fSAndrew Turner 299b9c0003fSMark Johnston /* 300b9c0003fSMark Johnston * Initialize the per-CPU pointer before calling into C code, for the 301b9c0003fSMark Johnston * benefit of kernel sanitizers. 302b9c0003fSMark Johnston */ 303b9c0003fSMark Johnston adrp x18, bootpcpu 304b9c0003fSMark Johnston ldr x18, [x18, :lo12:bootpcpu] 305b9c0003fSMark Johnston msr tpidr_el1, x18 306b9c0003fSMark Johnston 307b2b55077SAndrew Turner b init_secondary 308c78ebc69SAndrew TurnerLEND(mpentry_common) 309b2b55077SAndrew Turner#endif 310b2b55077SAndrew Turner 311e5acd89cSAndrew Turner/* 312e5acd89cSAndrew Turner * If we are started in EL2, configure the required hypervisor 313e5acd89cSAndrew Turner * registers and drop to EL1. 314e5acd89cSAndrew Turner */ 315801160f4SAndrew TurnerLENTRY(enter_kernel_el) 316228b87bcSAndrew Turner mrs x23, CurrentEL 31737563d39SAndrew Turner and x23, x23, #(CURRENTEL_EL_MASK) 31837563d39SAndrew Turner cmp x23, #(CURRENTEL_EL_EL2) 319e5acd89cSAndrew Turner b.eq 1f 320e5acd89cSAndrew Turner ret 321e5acd89cSAndrew Turner1: 32251adf913SKyle Evans /* 32351adf913SKyle Evans * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it 32451adf913SKyle Evans * which may break address translation. 32551adf913SKyle Evans */ 32651adf913SKyle Evans dsb sy 32751adf913SKyle Evans mrs x2, sctlr_el2 32851adf913SKyle Evans bic x2, x2, SCTLR_M 32951adf913SKyle Evans msr sctlr_el2, x2 33051adf913SKyle Evans isb 33151adf913SKyle Evans 332e5acd89cSAndrew Turner /* Configure the Hypervisor */ 33385b7c566SAndrew Turner ldr x2, =(HCR_RW | HCR_APK | HCR_API) 334e5acd89cSAndrew Turner msr hcr_el2, x2 335e5acd89cSAndrew Turner 336dc8616edSKyle Evans /* Stash value of HCR_EL2 for later */ 337dc8616edSKyle Evans isb 338dc8616edSKyle Evans mrs x4, hcr_el2 339dc8616edSKyle Evans 340e5acd89cSAndrew Turner /* Load the Virtualization Process ID Register */ 341e5acd89cSAndrew Turner mrs x2, midr_el1 342e5acd89cSAndrew Turner msr vpidr_el2, x2 343e5acd89cSAndrew Turner 344e5acd89cSAndrew Turner /* Load the Virtualization Multiprocess ID Register */ 345e5acd89cSAndrew Turner mrs x2, mpidr_el1 346e5acd89cSAndrew Turner msr vmpidr_el2, x2 347e5acd89cSAndrew Turner 348e5acd89cSAndrew Turner /* Set the bits that need to be 1 in sctlr_el1 */ 349e5acd89cSAndrew Turner ldr x2, .Lsctlr_res1 350e5acd89cSAndrew Turner msr sctlr_el1, x2 351e5acd89cSAndrew Turner 352dc8616edSKyle Evans /* 353dc8616edSKyle Evans * On some hardware, e.g., Apple M1, we can't clear E2H, so make sure we 354dc8616edSKyle Evans * don't trap to EL2 for SIMD register usage to have at least a 355dc8616edSKyle Evans * minimally usable system. 356dc8616edSKyle Evans */ 357dc8616edSKyle Evans tst x4, #HCR_E2H 358dc8616edSKyle Evans mov x3, #CPTR_RES1 /* HCR_E2H == 0 */ 359d2ae03baSKyle Evans mov x5, #CPTR_FPEN /* HCR_E2H == 1 */ 360d2ae03baSKyle Evans csel x2, x3, x5, eq 361e5acd89cSAndrew Turner msr cptr_el2, x2 362e5acd89cSAndrew Turner 363e5acd89cSAndrew Turner /* Don't trap to EL2 for CP15 traps */ 364e5acd89cSAndrew Turner msr hstr_el2, xzr 365e5acd89cSAndrew Turner 366f03aa10fSAndrew Turner /* Enable access to the physical timers at EL1 */ 367f03aa10fSAndrew Turner mrs x2, cnthctl_el2 368f03aa10fSAndrew Turner orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 369f03aa10fSAndrew Turner msr cnthctl_el2, x2 370f03aa10fSAndrew Turner 371f03aa10fSAndrew Turner /* Set the counter offset to a known value */ 372f03aa10fSAndrew Turner msr cntvoff_el2, xzr 373f03aa10fSAndrew Turner 374e5acd89cSAndrew Turner /* Hypervisor trap functions */ 3758a2adde1SAndrew Turner adrp x2, hyp_stub_vectors 3768a2adde1SAndrew Turner add x2, x2, :lo12:hyp_stub_vectors 377e5acd89cSAndrew Turner msr vbar_el2, x2 378e5acd89cSAndrew Turner 379ae43a817SAndrew Turner /* Zero vttbr_el2 so a hypervisor can tell the host and guest apart */ 380ae43a817SAndrew Turner msr vttbr_el2, xzr 381ae43a817SAndrew Turner 382136b8bd6SChristos Margiolis mov x2, #(PSR_DAIF | PSR_M_EL1h) 383e5acd89cSAndrew Turner msr spsr_el2, x2 384e5acd89cSAndrew Turner 38542cb216aSZbigniew Bodek /* Configure GICv3 CPU interface */ 38642cb216aSZbigniew Bodek mrs x2, id_aa64pfr0_el1 38742cb216aSZbigniew Bodek /* Extract GIC bits from the register */ 38842cb216aSZbigniew Bodek ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 38942cb216aSZbigniew Bodek /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 39042cb216aSZbigniew Bodek cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 39142cb216aSZbigniew Bodek b.ne 2f 39242cb216aSZbigniew Bodek 39342cb216aSZbigniew Bodek mrs x2, icc_sre_el2 39442cb216aSZbigniew Bodek orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 395b2552c46SWojciech Macek orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 39642cb216aSZbigniew Bodek msr icc_sre_el2, x2 39742cb216aSZbigniew Bodek2: 39842cb216aSZbigniew Bodek 399e5acd89cSAndrew Turner /* Set the address to return to our return address */ 400e5acd89cSAndrew Turner msr elr_el2, x30 401f452c301SAndrew Turner isb 402e5acd89cSAndrew Turner 403e5acd89cSAndrew Turner eret 404e5acd89cSAndrew Turner 405e5acd89cSAndrew Turner .align 3 406e5acd89cSAndrew Turner.Lsctlr_res1: 407e5acd89cSAndrew Turner .quad SCTLR_RES1 408801160f4SAndrew TurnerLEND(enter_kernel_el) 409e5acd89cSAndrew Turner 410e5acd89cSAndrew Turner/* 411ba313626SAndrew Turner * Get the physical address the kernel was loaded at. 412e5acd89cSAndrew Turner */ 413ba313626SAndrew TurnerLENTRY(get_load_phys_addr) 41461f14f1dSAndrew Turner /* Load the offset of get_load_phys_addr from KERNBASE */ 41561f14f1dSAndrew Turner ldr x28, =(get_load_phys_addr - KERNBASE) 416257b0445SAndrew Turner /* Load the physical address of get_load_phys_addr */ 41761f14f1dSAndrew Turner adr x29, get_load_phys_addr 41861f14f1dSAndrew Turner /* Find the physical address of KERNBASE, i.e. our load address */ 41961f14f1dSAndrew Turner sub x28, x29, x28 420e5acd89cSAndrew Turner ret 421ba313626SAndrew TurnerLEND(get_load_phys_addr) 422e5acd89cSAndrew Turner 423e5acd89cSAndrew Turner/* 424e5acd89cSAndrew Turner * This builds the page tables containing the identity map, and the kernel 425e5acd89cSAndrew Turner * virtual map. 426e5acd89cSAndrew Turner * 427e5acd89cSAndrew Turner * It relys on: 428e5acd89cSAndrew Turner * We were loaded to an address that is on a 2MiB boundary 429e5acd89cSAndrew Turner * All the memory must not cross a 1GiB boundaty 430e5acd89cSAndrew Turner * x28 contains the physical address we were loaded from 431e5acd89cSAndrew Turner * 432634dd430SAndrew Turner * There are 7 or 8 pages before that address for the page tables 433e5acd89cSAndrew Turner * The pages used are: 434634dd430SAndrew Turner * - The Kernel L3 tables (only for 16k kernel) 4359853d104SAndrew Turner * - The Kernel L2 table 4369853d104SAndrew Turner * - The Kernel L1 table 4379853d104SAndrew Turner * - The Kernel L0 table (TTBR1) 438634dd430SAndrew Turner * - The identity (PA = VA) L2 table 4399853d104SAndrew Turner * - The identity (PA = VA) L1 table 440634dd430SAndrew Turner * - The identity (PA = VA) L0 table (Early TTBR0) 441634dd430SAndrew Turner * - The Kernel empty L0 table (Late TTBR0) 442e5acd89cSAndrew Turner */ 443edb48ff6SAndrew TurnerLENTRY(create_pagetables) 444e5acd89cSAndrew Turner /* Save the Link register */ 445e5acd89cSAndrew Turner mov x5, x30 446e5acd89cSAndrew Turner 447e5acd89cSAndrew Turner /* Clean the page table */ 448659f1a6aSAndrew Turner adrp x6, pagetable 449659f1a6aSAndrew Turner add x6, x6, :lo12:pagetable 450e5acd89cSAndrew Turner mov x26, x6 451659f1a6aSAndrew Turner adrp x27, pagetable_end 452659f1a6aSAndrew Turner add x27, x27, :lo12:pagetable_end 453e5acd89cSAndrew Turner1: 454e5acd89cSAndrew Turner stp xzr, xzr, [x6], #16 455e5acd89cSAndrew Turner stp xzr, xzr, [x6], #16 456e5acd89cSAndrew Turner stp xzr, xzr, [x6], #16 457e5acd89cSAndrew Turner stp xzr, xzr, [x6], #16 458e5acd89cSAndrew Turner cmp x6, x27 459e5acd89cSAndrew Turner b.lo 1b 460e5acd89cSAndrew Turner 461e5acd89cSAndrew Turner /* 462e5acd89cSAndrew Turner * Build the TTBR1 maps. 463e5acd89cSAndrew Turner */ 464e5acd89cSAndrew Turner 465750d951fSJustin Hibbits /* Find the size of the kernel */ 466e5acd89cSAndrew Turner mov x6, #(KERNBASE) 4675641eda2SMichal Meloun 4685641eda2SMichal Meloun#if defined(LINUX_BOOT_ABI) 4695641eda2SMichal Meloun /* X19 is used as 'map FDT data' flag */ 4705641eda2SMichal Meloun mov x19, xzr 4715641eda2SMichal Meloun 4725641eda2SMichal Meloun /* No modules or FDT pointer ? */ 4735641eda2SMichal Meloun cbz x0, booti_no_fdt 4745641eda2SMichal Meloun 475722779c7SMichal Meloun /* 476722779c7SMichal Meloun * Test if x0 points to modules descriptor(virtual address) or 477722779c7SMichal Meloun * to FDT (physical address) 478722779c7SMichal Meloun */ 479722779c7SMichal Meloun cmp x0, x6 /* x6 is #(KERNBASE) */ 480722779c7SMichal Meloun b.lo booti_fdt 4815641eda2SMichal Meloun#endif 4825641eda2SMichal Meloun 4835641eda2SMichal Meloun /* Booted with modules pointer */ 484750d951fSJustin Hibbits /* Find modulep - begin */ 485750d951fSJustin Hibbits sub x8, x0, x6 48636f1526aSAndrew Turner /* 48736f1526aSAndrew Turner * Add space for the module data. When PAGE_SIZE is 4k this will 48836f1526aSAndrew Turner * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 48936f1526aSAndrew Turner * larger it will be at least as large as we use smaller level 3 49036f1526aSAndrew Turner * pages. 49136f1526aSAndrew Turner */ 49236f1526aSAndrew Turner ldr x7, =((6 * 1024 * 1024) - 1) 49378f23de5SAndrew Turner add x8, x8, x7 4945641eda2SMichal Meloun b common 4955641eda2SMichal Meloun 4965641eda2SMichal Meloun#if defined(LINUX_BOOT_ABI) 4975641eda2SMichal Melounbooti_fdt: 4985641eda2SMichal Meloun /* Booted by U-Boot booti with FDT data */ 4995641eda2SMichal Meloun /* Set 'map FDT data' flag */ 5005641eda2SMichal Meloun mov x19, #1 5015641eda2SMichal Meloun 5025641eda2SMichal Melounbooti_no_fdt: 5035641eda2SMichal Meloun /* Booted by U-Boot booti without FTD data */ 5045641eda2SMichal Meloun /* Find the end - begin */ 5055641eda2SMichal Meloun ldr x7, .Lend 5065641eda2SMichal Meloun sub x8, x7, x6 5075641eda2SMichal Meloun 5085641eda2SMichal Meloun /* 5095641eda2SMichal Meloun * Add one 2MiB page for copy of FDT data (maximum FDT size), 5105641eda2SMichal Meloun * one for metadata and round up 5115641eda2SMichal Meloun */ 5125641eda2SMichal Meloun ldr x7, =(3 * L2_SIZE - 1) 5135641eda2SMichal Meloun add x8, x8, x7 5145641eda2SMichal Meloun#endif 5155641eda2SMichal Meloun 5165641eda2SMichal Melouncommon: 51736f1526aSAndrew Turner#if PAGE_SIZE != PAGE_SIZE_4K 51836f1526aSAndrew Turner /* 519*94b09d38SAlan Cox * Create L3 and L3C pages. The kernel will be loaded at a 2M aligned 520*94b09d38SAlan Cox * address, enabling the creation of L3C pages. However, when the page 521*94b09d38SAlan Cox * size is larger than 4k, L2 blocks are too large to map the kernel 522*94b09d38SAlan Cox * with 2M alignment. 52336f1526aSAndrew Turner */ 524719908c8SAndrew Turner#define PTE_SHIFT L3_SHIFT 525719908c8SAndrew Turner#define BUILD_PTE_FUNC build_l3_page_pagetable 526719908c8SAndrew Turner#else 527719908c8SAndrew Turner#define PTE_SHIFT L2_SHIFT 528719908c8SAndrew Turner#define BUILD_PTE_FUNC build_l2_block_pagetable 529719908c8SAndrew Turner#endif 53036f1526aSAndrew Turner 531719908c8SAndrew Turner /* Get the number of blocks/pages to allocate, rounded down */ 532719908c8SAndrew Turner lsr x10, x8, #(PTE_SHIFT) 53336f1526aSAndrew Turner 534719908c8SAndrew Turner /* Create the kernel space PTE table */ 53536f1526aSAndrew Turner mov x6, x26 53636f1526aSAndrew Turner mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 53736f1526aSAndrew Turner mov x8, #(KERNBASE) 53836f1526aSAndrew Turner mov x9, x28 539719908c8SAndrew Turner bl BUILD_PTE_FUNC 54036f1526aSAndrew Turner 541719908c8SAndrew Turner#undef PTE_SHIFT 542719908c8SAndrew Turner#undef BUILD_PTE_FUNC 543719908c8SAndrew Turner 544719908c8SAndrew Turner#if PAGE_SIZE != PAGE_SIZE_4K 54536f1526aSAndrew Turner /* Move to the l2 table */ 54636f1526aSAndrew Turner ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 54736f1526aSAndrew Turner add x26, x26, x9 54836f1526aSAndrew Turner 54936f1526aSAndrew Turner /* Link the l2 -> l3 table */ 55036f1526aSAndrew Turner mov x9, x6 55136f1526aSAndrew Turner mov x6, x26 55236f1526aSAndrew Turner bl link_l2_pagetable 55336f1526aSAndrew Turner#endif 554e5acd89cSAndrew Turner 555e5acd89cSAndrew Turner /* Move to the l1 table */ 556e5acd89cSAndrew Turner add x26, x26, #PAGE_SIZE 557e5acd89cSAndrew Turner 558e5acd89cSAndrew Turner /* Link the l1 -> l2 table */ 559e5acd89cSAndrew Turner mov x9, x6 560e5acd89cSAndrew Turner mov x6, x26 561e5acd89cSAndrew Turner bl link_l1_pagetable 562e5acd89cSAndrew Turner 563f2f21fafSAndrew Turner /* Move to the l0 table */ 564f2f21fafSAndrew Turner add x24, x26, #PAGE_SIZE 565f2f21fafSAndrew Turner 566f2f21fafSAndrew Turner /* Link the l0 -> l1 table */ 567f2f21fafSAndrew Turner mov x9, x6 568f2f21fafSAndrew Turner mov x6, x24 56970e72785SAndrew Turner mov x10, #1 570f2f21fafSAndrew Turner bl link_l0_pagetable 571e5acd89cSAndrew Turner 572e5acd89cSAndrew Turner /* 573d153d023SAndrew Turner * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 57450e3ab6bSAlan Cox * They are only needed early on, so the VA = PA map is uncached. 575e5acd89cSAndrew Turner */ 576f2f21fafSAndrew Turner add x27, x24, #PAGE_SIZE 577e5acd89cSAndrew Turner 57880c4b9e5SAndrew Turner mov x6, x27 /* The initial page table */ 57948ba9b26SAndrew Turner 58048ba9b26SAndrew Turner /* Create the VA = PA map */ 58148ba9b26SAndrew Turner mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 58248ba9b26SAndrew Turner adrp x16, _start 58348ba9b26SAndrew Turner and x16, x16, #(~L2_OFFSET) 58448ba9b26SAndrew Turner mov x9, x16 /* PA start */ 58548ba9b26SAndrew Turner mov x8, x16 /* VA start (== PA start) */ 58648ba9b26SAndrew Turner mov x10, #1 58748ba9b26SAndrew Turner bl build_l2_block_pagetable 58848ba9b26SAndrew Turner 58948ba9b26SAndrew Turner#if defined(SOCDEV_PA) 590e5acd89cSAndrew Turner /* Create a table for the UART */ 591d153d023SAndrew Turner mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 59236f1526aSAndrew Turner ldr x9, =(L2_SIZE) 59336f1526aSAndrew Turner add x16, x16, x9 /* VA start */ 59448ba9b26SAndrew Turner mov x8, x16 59548ba9b26SAndrew Turner 59648ba9b26SAndrew Turner /* Store the socdev virtual address */ 59748ba9b26SAndrew Turner add x17, x8, #(SOCDEV_PA & L2_OFFSET) 59848ba9b26SAndrew Turner adrp x9, socdev_va 59948ba9b26SAndrew Turner str x17, [x9, :lo12:socdev_va] 60048ba9b26SAndrew Turner 60148ba9b26SAndrew Turner mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 6026aa751cfSAndrew Turner mov x10, #1 60348ba9b26SAndrew Turner bl build_l2_block_pagetable 604e5acd89cSAndrew Turner#endif 605e5acd89cSAndrew Turner 6065641eda2SMichal Meloun#if defined(LINUX_BOOT_ABI) 6075641eda2SMichal Meloun /* Map FDT data ? */ 6085641eda2SMichal Meloun cbz x19, 1f 6095641eda2SMichal Meloun 61048ba9b26SAndrew Turner /* Create the mapping for FDT data (2 MiB max) */ 611f10ab2d5SMichal Meloun mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 61236f1526aSAndrew Turner ldr x9, =(L2_SIZE) 61336f1526aSAndrew Turner add x16, x16, x9 /* VA start */ 61448ba9b26SAndrew Turner mov x8, x16 61548ba9b26SAndrew Turner mov x9, x0 /* PA start */ 61648ba9b26SAndrew Turner /* Update the module pointer to point at the allocated memory */ 61748ba9b26SAndrew Turner and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 61848ba9b26SAndrew Turner add x0, x0, x8 /* Add the aligned virtual address */ 61948ba9b26SAndrew Turner 6205641eda2SMichal Meloun mov x10, #1 62148ba9b26SAndrew Turner bl build_l2_block_pagetable 6225641eda2SMichal Meloun 6235641eda2SMichal Meloun1: 6245641eda2SMichal Meloun#endif 6255641eda2SMichal Meloun 62648ba9b26SAndrew Turner /* Move to the l1 table */ 62748ba9b26SAndrew Turner add x27, x27, #PAGE_SIZE 62848ba9b26SAndrew Turner 62948ba9b26SAndrew Turner /* Link the l1 -> l2 table */ 63048ba9b26SAndrew Turner mov x9, x6 63148ba9b26SAndrew Turner mov x6, x27 63248ba9b26SAndrew Turner bl link_l1_pagetable 633e5acd89cSAndrew Turner 63480c4b9e5SAndrew Turner /* Move to the l0 table */ 63580c4b9e5SAndrew Turner add x27, x27, #PAGE_SIZE 63680c4b9e5SAndrew Turner 63780c4b9e5SAndrew Turner /* Link the l0 -> l1 table */ 63880c4b9e5SAndrew Turner mov x9, x6 63980c4b9e5SAndrew Turner mov x6, x27 64070e72785SAndrew Turner mov x10, #1 64180c4b9e5SAndrew Turner bl link_l0_pagetable 64280c4b9e5SAndrew Turner 643e5acd89cSAndrew Turner /* Restore the Link register */ 644e5acd89cSAndrew Turner mov x30, x5 645e5acd89cSAndrew Turner ret 646edb48ff6SAndrew TurnerLEND(create_pagetables) 647e5acd89cSAndrew Turner 648e5acd89cSAndrew Turner/* 64980c4b9e5SAndrew Turner * Builds an L0 -> L1 table descriptor 65080c4b9e5SAndrew Turner * 65180c4b9e5SAndrew Turner * x6 = L0 table 65280c4b9e5SAndrew Turner * x8 = Virtual Address 65380c4b9e5SAndrew Turner * x9 = L1 PA (trashed) 65448ba9b26SAndrew Turner * x10 = Entry count (trashed) 65580c4b9e5SAndrew Turner * x11, x12 and x13 are trashed 65680c4b9e5SAndrew Turner */ 657edb48ff6SAndrew TurnerLENTRY(link_l0_pagetable) 65880c4b9e5SAndrew Turner /* 65980c4b9e5SAndrew Turner * Link an L0 -> L1 table entry. 66080c4b9e5SAndrew Turner */ 66180c4b9e5SAndrew Turner /* Find the table index */ 66280c4b9e5SAndrew Turner lsr x11, x8, #L0_SHIFT 663f2f21fafSAndrew Turner and x11, x11, #L0_ADDR_MASK 66480c4b9e5SAndrew Turner 66580c4b9e5SAndrew Turner /* Build the L0 block entry */ 66680c4b9e5SAndrew Turner mov x12, #L0_TABLE 667f6de51d3SAndrew Turner orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 66880c4b9e5SAndrew Turner 66980c4b9e5SAndrew Turner /* Only use the output address bits */ 67070e72785SAndrew Turner lsr x9, x9, #PAGE_SHIFT 67170e72785SAndrew Turner1: orr x13, x12, x9, lsl #PAGE_SHIFT 67280c4b9e5SAndrew Turner 67380c4b9e5SAndrew Turner /* Store the entry */ 67470e72785SAndrew Turner str x13, [x6, x11, lsl #3] 67570e72785SAndrew Turner 67670e72785SAndrew Turner sub x10, x10, #1 67770e72785SAndrew Turner add x11, x11, #1 67870e72785SAndrew Turner add x9, x9, #1 67970e72785SAndrew Turner cbnz x10, 1b 68080c4b9e5SAndrew Turner 68180c4b9e5SAndrew Turner ret 682edb48ff6SAndrew TurnerLEND(link_l0_pagetable) 68380c4b9e5SAndrew Turner 68480c4b9e5SAndrew Turner/* 685e5acd89cSAndrew Turner * Builds an L1 -> L2 table descriptor 686e5acd89cSAndrew Turner * 687e5acd89cSAndrew Turner * x6 = L1 table 688e5acd89cSAndrew Turner * x8 = Virtual Address 689e5acd89cSAndrew Turner * x9 = L2 PA (trashed) 690e5acd89cSAndrew Turner * x11, x12 and x13 are trashed 691e5acd89cSAndrew Turner */ 692edb48ff6SAndrew TurnerLENTRY(link_l1_pagetable) 693e5acd89cSAndrew Turner /* 694e5acd89cSAndrew Turner * Link an L1 -> L2 table entry. 695e5acd89cSAndrew Turner */ 696e5acd89cSAndrew Turner /* Find the table index */ 697e5acd89cSAndrew Turner lsr x11, x8, #L1_SHIFT 698e5acd89cSAndrew Turner and x11, x11, #Ln_ADDR_MASK 699e5acd89cSAndrew Turner 700e5acd89cSAndrew Turner /* Build the L1 block entry */ 701e5acd89cSAndrew Turner mov x12, #L1_TABLE 702e5acd89cSAndrew Turner 703e5acd89cSAndrew Turner /* Only use the output address bits */ 70470e72785SAndrew Turner lsr x9, x9, #PAGE_SHIFT 70570e72785SAndrew Turner orr x13, x12, x9, lsl #PAGE_SHIFT 706e5acd89cSAndrew Turner 707e5acd89cSAndrew Turner /* Store the entry */ 70870e72785SAndrew Turner str x13, [x6, x11, lsl #3] 709e5acd89cSAndrew Turner 710e5acd89cSAndrew Turner ret 711edb48ff6SAndrew TurnerLEND(link_l1_pagetable) 712e5acd89cSAndrew Turner 713e5acd89cSAndrew Turner/* 714e5acd89cSAndrew Turner * Builds count 2 MiB page table entry 715e5acd89cSAndrew Turner * x6 = L2 table 71623553d6bSAndrew Turner * x7 = Block attributes 717e5acd89cSAndrew Turner * x8 = VA start 718e5acd89cSAndrew Turner * x9 = PA start (trashed) 71948ba9b26SAndrew Turner * x10 = Entry count (trashed) 720e5acd89cSAndrew Turner * x11, x12 and x13 are trashed 721e5acd89cSAndrew Turner */ 722edb48ff6SAndrew TurnerLENTRY(build_l2_block_pagetable) 723e5acd89cSAndrew Turner /* 724e5acd89cSAndrew Turner * Build the L2 table entry. 725e5acd89cSAndrew Turner */ 726e5acd89cSAndrew Turner /* Find the table index */ 727e5acd89cSAndrew Turner lsr x11, x8, #L2_SHIFT 728e5acd89cSAndrew Turner and x11, x11, #Ln_ADDR_MASK 729e5acd89cSAndrew Turner 730e5acd89cSAndrew Turner /* Build the L2 block entry */ 73123553d6bSAndrew Turner orr x12, x7, #L2_BLOCK 732d6aa5fe5SAndrew Turner orr x12, x12, #(ATTR_DEFAULT) 733d153d023SAndrew Turner orr x12, x12, #(ATTR_S1_UXN) 7341b9096cdSAndrew Turner#ifdef __ARM_FEATURE_BTI_DEFAULT 7351b9096cdSAndrew Turner orr x12, x12, #(ATTR_S1_GP) 7361b9096cdSAndrew Turner#endif 737e5acd89cSAndrew Turner 738e5acd89cSAndrew Turner /* Only use the output address bits */ 739e5acd89cSAndrew Turner lsr x9, x9, #L2_SHIFT 740e5acd89cSAndrew Turner 741e5acd89cSAndrew Turner /* Set the physical address for this virtual address */ 74270e72785SAndrew Turner1: orr x13, x12, x9, lsl #L2_SHIFT 743e5acd89cSAndrew Turner 744e5acd89cSAndrew Turner /* Store the entry */ 74570e72785SAndrew Turner str x13, [x6, x11, lsl #3] 746e5acd89cSAndrew Turner 747e5acd89cSAndrew Turner sub x10, x10, #1 748e5acd89cSAndrew Turner add x11, x11, #1 749e5acd89cSAndrew Turner add x9, x9, #1 750e5acd89cSAndrew Turner cbnz x10, 1b 751e5acd89cSAndrew Turner 75270e72785SAndrew Turner ret 753edb48ff6SAndrew TurnerLEND(build_l2_block_pagetable) 754e5acd89cSAndrew Turner 75536f1526aSAndrew Turner#if PAGE_SIZE != PAGE_SIZE_4K 75636f1526aSAndrew Turner/* 75736f1526aSAndrew Turner * Builds an L2 -> L3 table descriptor 75836f1526aSAndrew Turner * 75936f1526aSAndrew Turner * x6 = L2 table 76036f1526aSAndrew Turner * x8 = Virtual Address 76136f1526aSAndrew Turner * x9 = L3 PA (trashed) 76236f1526aSAndrew Turner * x11, x12 and x13 are trashed 76336f1526aSAndrew Turner */ 76436f1526aSAndrew TurnerLENTRY(link_l2_pagetable) 76536f1526aSAndrew Turner /* 76636f1526aSAndrew Turner * Link an L2 -> L3 table entry. 76736f1526aSAndrew Turner */ 76836f1526aSAndrew Turner /* Find the table index */ 76936f1526aSAndrew Turner lsr x11, x8, #L2_SHIFT 77036f1526aSAndrew Turner and x11, x11, #Ln_ADDR_MASK 77136f1526aSAndrew Turner 77236f1526aSAndrew Turner /* Build the L1 block entry */ 77336f1526aSAndrew Turner mov x12, #L2_TABLE 77436f1526aSAndrew Turner 77536f1526aSAndrew Turner /* Only use the output address bits */ 77636f1526aSAndrew Turner lsr x9, x9, #PAGE_SHIFT 77736f1526aSAndrew Turner orr x13, x12, x9, lsl #PAGE_SHIFT 77836f1526aSAndrew Turner 77936f1526aSAndrew Turner /* Store the entry */ 78036f1526aSAndrew Turner str x13, [x6, x11, lsl #3] 78136f1526aSAndrew Turner 78236f1526aSAndrew Turner ret 78336f1526aSAndrew TurnerLEND(link_l2_pagetable) 78436f1526aSAndrew Turner 78536f1526aSAndrew Turner/* 786*94b09d38SAlan Cox * Builds count level 3 page table entries. Uses ATTR_CONTIGUOUS to create 787*94b09d38SAlan Cox * large page (L3C) mappings when the current VA and remaining count allow 788*94b09d38SAlan Cox * it. 78936f1526aSAndrew Turner * x6 = L3 table 79036f1526aSAndrew Turner * x7 = Block attributes 79136f1526aSAndrew Turner * x8 = VA start 79236f1526aSAndrew Turner * x9 = PA start (trashed) 79336f1526aSAndrew Turner * x10 = Entry count (trashed) 79436f1526aSAndrew Turner * x11, x12 and x13 are trashed 795*94b09d38SAlan Cox * 796*94b09d38SAlan Cox * VA start (x8) modulo L3C_SIZE must equal PA start (x9) modulo L3C_SIZE. 79736f1526aSAndrew Turner */ 79836f1526aSAndrew TurnerLENTRY(build_l3_page_pagetable) 79936f1526aSAndrew Turner /* 80036f1526aSAndrew Turner * Build the L3 table entry. 80136f1526aSAndrew Turner */ 80236f1526aSAndrew Turner /* Find the table index */ 80336f1526aSAndrew Turner lsr x11, x8, #L3_SHIFT 80436f1526aSAndrew Turner and x11, x11, #Ln_ADDR_MASK 80536f1526aSAndrew Turner 80636f1526aSAndrew Turner /* Build the L3 page entry */ 80736f1526aSAndrew Turner orr x12, x7, #L3_PAGE 80836f1526aSAndrew Turner orr x12, x12, #(ATTR_DEFAULT) 80936f1526aSAndrew Turner orr x12, x12, #(ATTR_S1_UXN) 8101b9096cdSAndrew Turner#ifdef __ARM_FEATURE_BTI_DEFAULT 8111b9096cdSAndrew Turner orr x12, x12, #(ATTR_S1_GP) 8121b9096cdSAndrew Turner#endif 81336f1526aSAndrew Turner 81436f1526aSAndrew Turner /* Only use the output address bits */ 81536f1526aSAndrew Turner lsr x9, x9, #L3_SHIFT 81636f1526aSAndrew Turner 817*94b09d38SAlan Cox /* Check if an ATTR_CONTIGUOUS mapping is possible */ 818*94b09d38SAlan Cox1: tst x11, #(L3C_ENTRIES - 1) 819*94b09d38SAlan Cox b.ne 2f 820*94b09d38SAlan Cox cmp x10, #L3C_ENTRIES 821*94b09d38SAlan Cox b.lo 3f 822*94b09d38SAlan Cox orr x12, x12, #(ATTR_CONTIGUOUS) 823*94b09d38SAlan Cox b 2f 824*94b09d38SAlan Cox3: and x12, x12, #(~ATTR_CONTIGUOUS) 825*94b09d38SAlan Cox 82636f1526aSAndrew Turner /* Set the physical address for this virtual address */ 827*94b09d38SAlan Cox2: orr x13, x12, x9, lsl #L3_SHIFT 82836f1526aSAndrew Turner 82936f1526aSAndrew Turner /* Store the entry */ 83036f1526aSAndrew Turner str x13, [x6, x11, lsl #3] 83136f1526aSAndrew Turner 83236f1526aSAndrew Turner sub x10, x10, #1 83336f1526aSAndrew Turner add x11, x11, #1 83436f1526aSAndrew Turner add x9, x9, #1 83536f1526aSAndrew Turner cbnz x10, 1b 83636f1526aSAndrew Turner 83736f1526aSAndrew Turner ret 83836f1526aSAndrew TurnerLEND(build_l3_page_pagetable) 83936f1526aSAndrew Turner#endif 84036f1526aSAndrew Turner 841edb48ff6SAndrew TurnerLENTRY(start_mmu) 842e5acd89cSAndrew Turner dsb sy 843e5acd89cSAndrew Turner 844e5acd89cSAndrew Turner /* Load the exception vectors */ 845e5acd89cSAndrew Turner ldr x2, =exception_vectors 846e5acd89cSAndrew Turner msr vbar_el1, x2 847e5acd89cSAndrew Turner 848e5acd89cSAndrew Turner /* Load ttbr0 and ttbr1 */ 849e5acd89cSAndrew Turner msr ttbr0_el1, x27 850f2f21fafSAndrew Turner msr ttbr1_el1, x24 851e5acd89cSAndrew Turner isb 852e5acd89cSAndrew Turner 853e5acd89cSAndrew Turner /* Clear the Monitor Debug System control register */ 854e5acd89cSAndrew Turner msr mdscr_el1, xzr 855e5acd89cSAndrew Turner 856e5acd89cSAndrew Turner /* Invalidate the TLB */ 857e5acd89cSAndrew Turner tlbi vmalle1is 858a48cf247SAndrew Turner dsb ish 859a48cf247SAndrew Turner isb 860e5acd89cSAndrew Turner 861e5acd89cSAndrew Turner ldr x2, mair 862e5acd89cSAndrew Turner msr mair_el1, x2 863e5acd89cSAndrew Turner 86480c4b9e5SAndrew Turner /* 86550e3ab6bSAlan Cox * Setup TCR according to the PARange and ASIDBits fields 866b0a0152aSAlan Cox * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 867b0a0152aSAlan Cox * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 86850e3ab6bSAlan Cox * to 1 only if the ASIDBits field equals 0b0010. 86980c4b9e5SAndrew Turner */ 870f2f21fafSAndrew Turner ldr x2, tcr 871e5acd89cSAndrew Turner mrs x3, id_aa64mmfr0_el1 87265565c97SAndrew Turner 87365565c97SAndrew Turner /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 87465565c97SAndrew Turner bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 87565565c97SAndrew Turner and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 87665565c97SAndrew Turner 87765565c97SAndrew Turner /* Check if the HW supports 16 bit ASIDS */ 87865565c97SAndrew Turner cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 87965565c97SAndrew Turner /* If so x3 == 1, else x3 == 0 */ 88050e3ab6bSAlan Cox cset x3, eq 88165565c97SAndrew Turner /* Set TCR.AS with x3 */ 88265565c97SAndrew Turner bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 88365565c97SAndrew Turner 884b0a0152aSAlan Cox /* 885b0a0152aSAlan Cox * Check if the HW supports access flag and dirty state updates, 886b0a0152aSAlan Cox * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 887b0a0152aSAlan Cox */ 888b0a0152aSAlan Cox mrs x3, id_aa64mmfr1_el1 889b0a0152aSAlan Cox and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 890b0a0152aSAlan Cox cmp x3, #1 891b0a0152aSAlan Cox b.ne 1f 892b0a0152aSAlan Cox orr x2, x2, #(TCR_HA) 893b0a0152aSAlan Cox b 2f 894b0a0152aSAlan Cox1: 895b0a0152aSAlan Cox cmp x3, #2 896b0a0152aSAlan Cox b.ne 2f 897b0a0152aSAlan Cox orr x2, x2, #(TCR_HA | TCR_HD) 898b0a0152aSAlan Cox2: 899e5acd89cSAndrew Turner msr tcr_el1, x2 900e5acd89cSAndrew Turner 90165565c97SAndrew Turner /* 90265565c97SAndrew Turner * Setup SCTLR. 90365565c97SAndrew Turner */ 904e5acd89cSAndrew Turner ldr x2, sctlr_set 905e5acd89cSAndrew Turner ldr x3, sctlr_clear 906e5acd89cSAndrew Turner mrs x1, sctlr_el1 907e5acd89cSAndrew Turner bic x1, x1, x3 /* Clear the required bits */ 908e5acd89cSAndrew Turner orr x1, x1, x2 /* Set the required bits */ 909e5acd89cSAndrew Turner msr sctlr_el1, x1 910e5acd89cSAndrew Turner isb 911e5acd89cSAndrew Turner 912e5acd89cSAndrew Turner ret 913e5acd89cSAndrew Turner 914e5acd89cSAndrew Turner .align 3 915e5acd89cSAndrew Turnermair: 91613ec5a6dSAndrew Turner .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 917ef3e1e13SAndrew Turner MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 918ef3e1e13SAndrew Turner MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 91913ec5a6dSAndrew Turner MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 92013ec5a6dSAndrew Turner MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 921e5acd89cSAndrew Turnertcr: 92236f1526aSAndrew Turner#if PAGE_SIZE == PAGE_SIZE_4K 92336f1526aSAndrew Turner#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 92436f1526aSAndrew Turner#elif PAGE_SIZE == PAGE_SIZE_16K 92536f1526aSAndrew Turner#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 92636f1526aSAndrew Turner#else 92736f1526aSAndrew Turner#error Unsupported page size 92836f1526aSAndrew Turner#endif 92936f1526aSAndrew Turner 93036f1526aSAndrew Turner .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 9311038d102SZbigniew Bodek TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 932e5acd89cSAndrew Turnersctlr_set: 933e5acd89cSAndrew Turner /* Bits to set */ 934aec085f4SAndrew Turner .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 935aec085f4SAndrew Turner SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 93609f966caSEd Schouten SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 937450f731bSAndrew Turner SCTLR_M | SCTLR_CP15BEN | SCTLR_BT1 | SCTLR_BT0) 938e5acd89cSAndrew Turnersctlr_clear: 939e5acd89cSAndrew Turner /* Bits to clear */ 940c0edde30SAndrew Turner .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 941a9725b63SAndrew Turner SCTLR_ITD | SCTLR_A) 942edb48ff6SAndrew TurnerLEND(start_mmu) 943e5acd89cSAndrew Turner 944edb48ff6SAndrew TurnerENTRY(abort) 945e5acd89cSAndrew Turner b abort 946edb48ff6SAndrew TurnerEND(abort) 947e5acd89cSAndrew Turner 9481a3cb489SMark Johnston.bss 9491a3cb489SMark Johnston .align PAGE_SHIFT 9501a3cb489SMark Johnstoninitstack: 95190372a9eSMark Johnston .space BOOT_STACK_SIZE 9521a3cb489SMark Johnstoninitstack_end: 9531a3cb489SMark Johnston 954166ceb6fSAndrew Turner .section .init_pagetable, "aw", %nobits 955166ceb6fSAndrew Turner .align PAGE_SHIFT 956e5acd89cSAndrew Turner /* 957857ab36fSAndrew Turner * 6 initial tables (in the following order): 958e5acd89cSAndrew Turner * L2 for kernel (High addresses) 959e5acd89cSAndrew Turner * L1 for kernel 960857ab36fSAndrew Turner * L0 for kernel 961857ab36fSAndrew Turner * L1 bootstrap for user (Low addresses) 962857ab36fSAndrew Turner * L0 bootstrap for user 963857ab36fSAndrew Turner * L0 for user 964e5acd89cSAndrew Turner */ 965bcd763b6SAndrew Turner .globl pagetable_l0_ttbr1 966e5acd89cSAndrew Turnerpagetable: 96736f1526aSAndrew Turner#if PAGE_SIZE != PAGE_SIZE_4K 96836f1526aSAndrew Turner .space (PAGE_SIZE * L3_PAGE_COUNT) 96936f1526aSAndrew Turnerpagetable_l2_ttbr1: 97036f1526aSAndrew Turner#endif 971e5acd89cSAndrew Turner .space PAGE_SIZE 972e5acd89cSAndrew Turnerpagetable_l1_ttbr1: 973e5acd89cSAndrew Turner .space PAGE_SIZE 974f2f21fafSAndrew Turnerpagetable_l0_ttbr1: 975f2f21fafSAndrew Turner .space PAGE_SIZE 97648ba9b26SAndrew Turnerpagetable_l2_ttbr0_bootstrap: 97748ba9b26SAndrew Turner .space PAGE_SIZE 978857ab36fSAndrew Turnerpagetable_l1_ttbr0_bootstrap: 979857ab36fSAndrew Turner .space PAGE_SIZE 980c6a6ec85SAlfredo Mazzinghipagetable_l0_ttbr0_bootstrap: 981e5acd89cSAndrew Turner .space PAGE_SIZE 98280c4b9e5SAndrew Turnerpagetable_l0_ttbr0: 98380c4b9e5SAndrew Turner .space PAGE_SIZE 984e5acd89cSAndrew Turnerpagetable_end: 985e5acd89cSAndrew Turner 986e5acd89cSAndrew Turnerel2_pagetable: 987e5acd89cSAndrew Turner .space PAGE_SIZE 988e5acd89cSAndrew Turner 98937c1ef5aSAndrew Turner .section .rodata, "a", %progbits 99037c1ef5aSAndrew Turner .globl aarch32_sigcode 99137c1ef5aSAndrew Turner .align 2 99237c1ef5aSAndrew Turneraarch32_sigcode: 9938c9c3144SOlivier Houchard .word 0xe1a0000d // mov r0, sp 9948c9c3144SOlivier Houchard .word 0xe2800040 // add r0, r0, #SIGF_UC 9958c9c3144SOlivier Houchard .word 0xe59f700c // ldr r7, [pc, #12] 9968c9c3144SOlivier Houchard .word 0xef000000 // swi #0 9978c9c3144SOlivier Houchard .word 0xe59f7008 // ldr r7, [pc, #8] 9988c9c3144SOlivier Houchard .word 0xef000000 // swi #0 9998c9c3144SOlivier Houchard .word 0xeafffffa // b . - 16 10008c9c3144SOlivier Houchard .word SYS_sigreturn 10018c9c3144SOlivier Houchard .word SYS_exit 10028c9c3144SOlivier Houchard .align 3 100337c1ef5aSAndrew Turner .size aarch32_sigcode, . - aarch32_sigcode 10048c9c3144SOlivier Houchardaarch32_esigcode: 10058c9c3144SOlivier Houchard .data 10068c9c3144SOlivier Houchard .global sz_aarch32_sigcode 10078c9c3144SOlivier Houchardsz_aarch32_sigcode: 10088c9c3144SOlivier Houchard .quad aarch32_esigcode - aarch32_sigcode 1009