1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.inc" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37#include <machine/vm.h> 38#include <machine/vmparam.h> 39 40#define VIRT_BITS 48 41 42#if PAGE_SIZE == PAGE_SIZE_16K 43/* 44 * The number of level 3 tables to create. 32 will allow for 1G of address 45 * space, the same as a single level 2 page with 4k pages. 46 */ 47#define L3_PAGE_COUNT 32 48#endif 49 50 .globl kernbase 51 .set kernbase, KERNBASE 52 53/* 54 * We assume: 55 * MMU on with an identity map, or off 56 * D-Cache: off 57 * I-Cache: on or off 58 * We are loaded at a 2MiB aligned address 59 */ 60 61ENTRY(_start) 62 /* Drop to EL1 */ 63 bl drop_to_el1 64 65 /* 66 * Disable the MMU. We may have entered the kernel with it on and 67 * will need to update the tables later. If this has been set up 68 * with anything other than a VA == PA map then this will fail, 69 * but in this case the code to find where we are running from 70 * would have also failed. 71 */ 72 dsb sy 73 mrs x2, sctlr_el1 74 bic x2, x2, SCTLR_M 75 msr sctlr_el1, x2 76 isb 77 78 /* Set the context id */ 79 msr contextidr_el1, xzr 80 81 /* Get the virt -> phys offset */ 82 bl get_virt_delta 83 84 /* 85 * At this point: 86 * x29 = PA - VA 87 * x28 = Our physical load address 88 */ 89 90 /* Create the page tables */ 91 bl create_pagetables 92 93 /* 94 * At this point: 95 * x27 = TTBR0 table 96 * x26 = Kernel L1 table 97 * x24 = TTBR1 table 98 */ 99 100 /* Enable the mmu */ 101 bl start_mmu 102 103 /* Load the new ttbr0 pagetable */ 104 adrp x27, pagetable_l0_ttbr0 105 add x27, x27, :lo12:pagetable_l0_ttbr0 106 107 /* Jump to the virtual address space */ 108 ldr x15, .Lvirtdone 109 br x15 110 111virtdone: 112 /* Set up the stack */ 113 adrp x25, initstack_end 114 add x25, x25, :lo12:initstack_end 115 mov sp, x25 116 sub sp, sp, #PCB_SIZE 117 118 /* Zero the BSS */ 119 ldr x15, .Lbss 120 ldr x14, .Lend 1211: 122 str xzr, [x15], #8 123 cmp x15, x14 124 b.lo 1b 125 126#if defined(PERTHREAD_SSP) 127 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 128 adrp x15, boot_canary 129 add x15, x15, :lo12:boot_canary 130 msr sp_el0, x15 131#endif 132 133 /* Backup the module pointer */ 134 mov x1, x0 135 136 /* Make the page table base a virtual address */ 137 sub x26, x26, x29 138 sub x24, x24, x29 139 140 sub sp, sp, #BOOTPARAMS_SIZE 141 mov x0, sp 142 143 /* Degate the delda so it is VA -> PA */ 144 neg x29, x29 145 146 str x1, [x0, #BP_MODULEP] 147 str x26, [x0, #BP_KERN_L1PT] 148 str x29, [x0, #BP_KERN_DELTA] 149 adrp x25, initstack 150 add x25, x25, :lo12:initstack 151 str x25, [x0, #BP_KERN_STACK] 152 str x24, [x0, #BP_KERN_L0PT] 153 str x27, [x0, #BP_KERN_TTBR0] 154 str x23, [x0, #BP_BOOT_EL] 155 156 /* trace back starts here */ 157 mov fp, #0 158 /* Branch to C code */ 159 bl initarm 160 /* We are done with the boot params */ 161 add sp, sp, #BOOTPARAMS_SIZE 162 163 /* 164 * Enable pointer authentication in the kernel. We set the keys for 165 * thread0 in initarm so have to wait until it returns to enable it. 166 * If we were to enable it in initarm then any authentication when 167 * returning would fail as it was called with pointer authentication 168 * disabled. 169 */ 170 bl ptrauth_start 171 172 bl mi_startup 173 174 /* We should not get here */ 175 brk 0 176 177 .align 3 178.Lvirtdone: 179 .quad virtdone 180.Lbss: 181 .quad __bss_start 182.Lend: 183 .quad __bss_end 184END(_start) 185 186#ifdef SMP 187/* 188 * mpentry(unsigned long) 189 * 190 * Called by a core when it is being brought online. 191 * The data in x0 is passed straight to init_secondary. 192 */ 193ENTRY(mpentry) 194 /* Disable interrupts */ 195 msr daifset, #DAIF_INTR 196 197 /* Drop to EL1 */ 198 bl drop_to_el1 199 200 /* Set the context id */ 201 msr contextidr_el1, xzr 202 203 /* Load the kernel page table */ 204 adrp x24, pagetable_l0_ttbr1 205 add x24, x24, :lo12:pagetable_l0_ttbr1 206 /* Load the identity page table */ 207 adrp x27, pagetable_l0_ttbr0_boostrap 208 add x27, x27, :lo12:pagetable_l0_ttbr0_boostrap 209 210 /* Enable the mmu */ 211 bl start_mmu 212 213 /* Load the new ttbr0 pagetable */ 214 adrp x27, pagetable_l0_ttbr0 215 add x27, x27, :lo12:pagetable_l0_ttbr0 216 217 /* Jump to the virtual address space */ 218 ldr x15, =mp_virtdone 219 br x15 220 221mp_virtdone: 222 /* Start using the AP boot stack */ 223 ldr x4, =bootstack 224 ldr x4, [x4] 225 mov sp, x4 226 227#if defined(PERTHREAD_SSP) 228 /* Set sp_el0 to the boot canary for early per-thread SSP to work */ 229 adrp x15, boot_canary 230 add x15, x15, :lo12:boot_canary 231 msr sp_el0, x15 232#endif 233 234 /* Load the kernel ttbr0 pagetable */ 235 msr ttbr0_el1, x27 236 isb 237 238 /* Invalidate the TLB */ 239 tlbi vmalle1 240 dsb sy 241 isb 242 243 b init_secondary 244END(mpentry) 245#endif 246 247/* 248 * If we are started in EL2, configure the required hypervisor 249 * registers and drop to EL1. 250 */ 251LENTRY(drop_to_el1) 252 mrs x23, CurrentEL 253 lsr x23, x23, #2 254 cmp x23, #0x2 255 b.eq 1f 256 ret 2571: 258 /* 259 * Disable the MMU. If the HCR_EL2.E2H field is set we will clear it 260 * which may break address translation. 261 */ 262 dsb sy 263 mrs x2, sctlr_el2 264 bic x2, x2, SCTLR_M 265 msr sctlr_el2, x2 266 isb 267 268 /* Configure the Hypervisor */ 269 ldr x2, =(HCR_RW | HCR_APK | HCR_API) 270 msr hcr_el2, x2 271 272 /* Load the Virtualization Process ID Register */ 273 mrs x2, midr_el1 274 msr vpidr_el2, x2 275 276 /* Load the Virtualization Multiprocess ID Register */ 277 mrs x2, mpidr_el1 278 msr vmpidr_el2, x2 279 280 /* Set the bits that need to be 1 in sctlr_el1 */ 281 ldr x2, .Lsctlr_res1 282 msr sctlr_el1, x2 283 284 /* Don't trap to EL2 for exceptions */ 285 mov x2, #CPTR_RES1 286 msr cptr_el2, x2 287 288 /* Don't trap to EL2 for CP15 traps */ 289 msr hstr_el2, xzr 290 291 /* Enable access to the physical timers at EL1 */ 292 mrs x2, cnthctl_el2 293 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 294 msr cnthctl_el2, x2 295 296 /* Set the counter offset to a known value */ 297 msr cntvoff_el2, xzr 298 299 /* Hypervisor trap functions */ 300 adrp x2, hyp_vectors 301 add x2, x2, :lo12:hyp_vectors 302 msr vbar_el2, x2 303 304 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 305 msr spsr_el2, x2 306 307 /* Configure GICv3 CPU interface */ 308 mrs x2, id_aa64pfr0_el1 309 /* Extract GIC bits from the register */ 310 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 311 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 312 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 313 b.ne 2f 314 315 mrs x2, icc_sre_el2 316 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 317 orr x2, x2, #ICC_SRE_EL2_SRE /* Enable system registers */ 318 msr icc_sre_el2, x2 3192: 320 321 /* Set the address to return to our return address */ 322 msr elr_el2, x30 323 isb 324 325 eret 326 327 .align 3 328.Lsctlr_res1: 329 .quad SCTLR_RES1 330LEND(drop_to_el1) 331 332#define VECT_EMPTY \ 333 .align 7; \ 334 1: b 1b 335 336 .align 11 337hyp_vectors: 338 VECT_EMPTY /* Synchronous EL2t */ 339 VECT_EMPTY /* IRQ EL2t */ 340 VECT_EMPTY /* FIQ EL2t */ 341 VECT_EMPTY /* Error EL2t */ 342 343 VECT_EMPTY /* Synchronous EL2h */ 344 VECT_EMPTY /* IRQ EL2h */ 345 VECT_EMPTY /* FIQ EL2h */ 346 VECT_EMPTY /* Error EL2h */ 347 348 VECT_EMPTY /* Synchronous 64-bit EL1 */ 349 VECT_EMPTY /* IRQ 64-bit EL1 */ 350 VECT_EMPTY /* FIQ 64-bit EL1 */ 351 VECT_EMPTY /* Error 64-bit EL1 */ 352 353 VECT_EMPTY /* Synchronous 32-bit EL1 */ 354 VECT_EMPTY /* IRQ 32-bit EL1 */ 355 VECT_EMPTY /* FIQ 32-bit EL1 */ 356 VECT_EMPTY /* Error 32-bit EL1 */ 357 358/* 359 * Get the delta between the physical address we were loaded to and the 360 * virtual address we expect to run from. This is used when building the 361 * initial page table. 362 */ 363LENTRY(get_virt_delta) 364 /* Load the physical address of virt_map */ 365 adrp x29, virt_map 366 add x29, x29, :lo12:virt_map 367 /* Load the virtual address of virt_map stored in virt_map */ 368 ldr x28, [x29] 369 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 370 sub x29, x29, x28 371 /* Find the load address for the kernel */ 372 mov x28, #(KERNBASE) 373 add x28, x28, x29 374 ret 375 376 .align 3 377virt_map: 378 .quad virt_map 379LEND(get_virt_delta) 380 381/* 382 * This builds the page tables containing the identity map, and the kernel 383 * virtual map. 384 * 385 * It relys on: 386 * We were loaded to an address that is on a 2MiB boundary 387 * All the memory must not cross a 1GiB boundaty 388 * x28 contains the physical address we were loaded from 389 * 390 * TODO: This is out of date. 391 * There are at least 5 pages before that address for the page tables 392 * The pages used are: 393 * - The Kernel L2 table 394 * - The Kernel L1 table 395 * - The Kernel L0 table (TTBR1) 396 * - The identity (PA = VA) L1 table 397 * - The identity (PA = VA) L0 table (TTBR0) 398 */ 399LENTRY(create_pagetables) 400 /* Save the Link register */ 401 mov x5, x30 402 403 /* Clean the page table */ 404 adrp x6, pagetable 405 add x6, x6, :lo12:pagetable 406 mov x26, x6 407 adrp x27, pagetable_end 408 add x27, x27, :lo12:pagetable_end 4091: 410 stp xzr, xzr, [x6], #16 411 stp xzr, xzr, [x6], #16 412 stp xzr, xzr, [x6], #16 413 stp xzr, xzr, [x6], #16 414 cmp x6, x27 415 b.lo 1b 416 417 /* 418 * Build the TTBR1 maps. 419 */ 420 421 /* Find the size of the kernel */ 422 mov x6, #(KERNBASE) 423 424#if defined(LINUX_BOOT_ABI) 425 /* X19 is used as 'map FDT data' flag */ 426 mov x19, xzr 427 428 /* No modules or FDT pointer ? */ 429 cbz x0, booti_no_fdt 430 431 /* 432 * Test if x0 points to modules descriptor(virtual address) or 433 * to FDT (physical address) 434 */ 435 cmp x0, x6 /* x6 is #(KERNBASE) */ 436 b.lo booti_fdt 437#endif 438 439 /* Booted with modules pointer */ 440 /* Find modulep - begin */ 441 sub x8, x0, x6 442 /* 443 * Add space for the module data. When PAGE_SIZE is 4k this will 444 * add at least 2 level 2 blocks (2 * 2MiB). When PAGE_SIZE is 445 * larger it will be at least as large as we use smaller level 3 446 * pages. 447 */ 448 ldr x7, =((6 * 1024 * 1024) - 1) 449 add x8, x8, x7 450 b common 451 452#if defined(LINUX_BOOT_ABI) 453booti_fdt: 454 /* Booted by U-Boot booti with FDT data */ 455 /* Set 'map FDT data' flag */ 456 mov x19, #1 457 458booti_no_fdt: 459 /* Booted by U-Boot booti without FTD data */ 460 /* Find the end - begin */ 461 ldr x7, .Lend 462 sub x8, x7, x6 463 464 /* 465 * Add one 2MiB page for copy of FDT data (maximum FDT size), 466 * one for metadata and round up 467 */ 468 ldr x7, =(3 * L2_SIZE - 1) 469 add x8, x8, x7 470#endif 471 472common: 473#if PAGE_SIZE != PAGE_SIZE_4K 474 /* 475 * Create L3 pages. The kernel will be loaded at a 2M aligned 476 * address, however L2 blocks are too large when the page size is 477 * not 4k to map the kernel with such an aligned address. However, 478 * when the page size is larger than 4k, L2 blocks are too large to 479 * map the kernel with such an alignment. 480 */ 481 482 /* Get the number of l3 pages to allocate, rounded down */ 483 lsr x10, x8, #(L3_SHIFT) 484 485 /* Create the kernel space L2 table */ 486 mov x6, x26 487 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 488 mov x8, #(KERNBASE) 489 mov x9, x28 490 bl build_l3_page_pagetable 491 492 /* Move to the l2 table */ 493 ldr x9, =(PAGE_SIZE * L3_PAGE_COUNT) 494 add x26, x26, x9 495 496 /* Link the l2 -> l3 table */ 497 mov x9, x6 498 mov x6, x26 499 bl link_l2_pagetable 500#else 501 /* Get the number of l2 pages to allocate, rounded down */ 502 lsr x10, x8, #(L2_SHIFT) 503 504 /* Create the kernel space L2 table */ 505 mov x6, x26 506 mov x7, #(ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 507 mov x8, #(KERNBASE) 508 mov x9, x28 509 bl build_l2_block_pagetable 510#endif 511 512 /* Move to the l1 table */ 513 add x26, x26, #PAGE_SIZE 514 515 /* Link the l1 -> l2 table */ 516 mov x9, x6 517 mov x6, x26 518 bl link_l1_pagetable 519 520 /* Move to the l0 table */ 521 add x24, x26, #PAGE_SIZE 522 523 /* Link the l0 -> l1 table */ 524 mov x9, x6 525 mov x6, x24 526 mov x10, #1 527 bl link_l0_pagetable 528 529 /* 530 * Build the TTBR0 maps. As TTBR0 maps, they must specify ATTR_S1_nG. 531 * They are only needed early on, so the VA = PA map is uncached. 532 */ 533 add x27, x24, #PAGE_SIZE 534 535 mov x6, x27 /* The initial page table */ 536 537 /* Create the VA = PA map */ 538 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 539 adrp x16, _start 540 and x16, x16, #(~L2_OFFSET) 541 mov x9, x16 /* PA start */ 542 mov x8, x16 /* VA start (== PA start) */ 543 mov x10, #1 544 bl build_l2_block_pagetable 545 546#if defined(SOCDEV_PA) 547 /* Create a table for the UART */ 548 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_DEVICE)) 549 ldr x9, =(L2_SIZE) 550 add x16, x16, x9 /* VA start */ 551 mov x8, x16 552 553 /* Store the socdev virtual address */ 554 add x17, x8, #(SOCDEV_PA & L2_OFFSET) 555 adrp x9, socdev_va 556 str x17, [x9, :lo12:socdev_va] 557 558 mov x9, #(SOCDEV_PA & ~L2_OFFSET) /* PA start */ 559 mov x10, #1 560 bl build_l2_block_pagetable 561#endif 562 563#if defined(LINUX_BOOT_ABI) 564 /* Map FDT data ? */ 565 cbz x19, 1f 566 567 /* Create the mapping for FDT data (2 MiB max) */ 568 mov x7, #(ATTR_S1_nG | ATTR_S1_IDX(VM_MEMATTR_WRITE_BACK)) 569 ldr x9, =(L2_SIZE) 570 add x16, x16, x9 /* VA start */ 571 mov x8, x16 572 mov x9, x0 /* PA start */ 573 /* Update the module pointer to point at the allocated memory */ 574 and x0, x0, #(L2_OFFSET) /* Keep the lower bits */ 575 add x0, x0, x8 /* Add the aligned virtual address */ 576 577 mov x10, #1 578 bl build_l2_block_pagetable 579 5801: 581#endif 582 583 /* Move to the l1 table */ 584 add x27, x27, #PAGE_SIZE 585 586 /* Link the l1 -> l2 table */ 587 mov x9, x6 588 mov x6, x27 589 bl link_l1_pagetable 590 591 /* Move to the l0 table */ 592 add x27, x27, #PAGE_SIZE 593 594 /* Link the l0 -> l1 table */ 595 mov x9, x6 596 mov x6, x27 597 mov x10, #1 598 bl link_l0_pagetable 599 600 /* Restore the Link register */ 601 mov x30, x5 602 ret 603LEND(create_pagetables) 604 605/* 606 * Builds an L0 -> L1 table descriptor 607 * 608 * x6 = L0 table 609 * x8 = Virtual Address 610 * x9 = L1 PA (trashed) 611 * x10 = Entry count (trashed) 612 * x11, x12 and x13 are trashed 613 */ 614LENTRY(link_l0_pagetable) 615 /* 616 * Link an L0 -> L1 table entry. 617 */ 618 /* Find the table index */ 619 lsr x11, x8, #L0_SHIFT 620 and x11, x11, #L0_ADDR_MASK 621 622 /* Build the L0 block entry */ 623 mov x12, #L0_TABLE 624 orr x12, x12, #(TATTR_UXN_TABLE | TATTR_AP_TABLE_NO_EL0) 625 626 /* Only use the output address bits */ 627 lsr x9, x9, #PAGE_SHIFT 6281: orr x13, x12, x9, lsl #PAGE_SHIFT 629 630 /* Store the entry */ 631 str x13, [x6, x11, lsl #3] 632 633 sub x10, x10, #1 634 add x11, x11, #1 635 add x9, x9, #1 636 cbnz x10, 1b 637 638 ret 639LEND(link_l0_pagetable) 640 641/* 642 * Builds an L1 -> L2 table descriptor 643 * 644 * x6 = L1 table 645 * x8 = Virtual Address 646 * x9 = L2 PA (trashed) 647 * x11, x12 and x13 are trashed 648 */ 649LENTRY(link_l1_pagetable) 650 /* 651 * Link an L1 -> L2 table entry. 652 */ 653 /* Find the table index */ 654 lsr x11, x8, #L1_SHIFT 655 and x11, x11, #Ln_ADDR_MASK 656 657 /* Build the L1 block entry */ 658 mov x12, #L1_TABLE 659 660 /* Only use the output address bits */ 661 lsr x9, x9, #PAGE_SHIFT 662 orr x13, x12, x9, lsl #PAGE_SHIFT 663 664 /* Store the entry */ 665 str x13, [x6, x11, lsl #3] 666 667 ret 668LEND(link_l1_pagetable) 669 670/* 671 * Builds count 2 MiB page table entry 672 * x6 = L2 table 673 * x7 = Block attributes 674 * x8 = VA start 675 * x9 = PA start (trashed) 676 * x10 = Entry count (trashed) 677 * x11, x12 and x13 are trashed 678 */ 679LENTRY(build_l2_block_pagetable) 680 /* 681 * Build the L2 table entry. 682 */ 683 /* Find the table index */ 684 lsr x11, x8, #L2_SHIFT 685 and x11, x11, #Ln_ADDR_MASK 686 687 /* Build the L2 block entry */ 688 orr x12, x7, #L2_BLOCK 689 orr x12, x12, #(ATTR_DEFAULT) 690 orr x12, x12, #(ATTR_S1_UXN) 691 692 /* Only use the output address bits */ 693 lsr x9, x9, #L2_SHIFT 694 695 /* Set the physical address for this virtual address */ 6961: orr x13, x12, x9, lsl #L2_SHIFT 697 698 /* Store the entry */ 699 str x13, [x6, x11, lsl #3] 700 701 sub x10, x10, #1 702 add x11, x11, #1 703 add x9, x9, #1 704 cbnz x10, 1b 705 706 ret 707LEND(build_l2_block_pagetable) 708 709#if PAGE_SIZE != PAGE_SIZE_4K 710/* 711 * Builds an L2 -> L3 table descriptor 712 * 713 * x6 = L2 table 714 * x8 = Virtual Address 715 * x9 = L3 PA (trashed) 716 * x11, x12 and x13 are trashed 717 */ 718LENTRY(link_l2_pagetable) 719 /* 720 * Link an L2 -> L3 table entry. 721 */ 722 /* Find the table index */ 723 lsr x11, x8, #L2_SHIFT 724 and x11, x11, #Ln_ADDR_MASK 725 726 /* Build the L1 block entry */ 727 mov x12, #L2_TABLE 728 729 /* Only use the output address bits */ 730 lsr x9, x9, #PAGE_SHIFT 731 orr x13, x12, x9, lsl #PAGE_SHIFT 732 733 /* Store the entry */ 734 str x13, [x6, x11, lsl #3] 735 736 ret 737LEND(link_l2_pagetable) 738 739/* 740 * Builds count level 3 page table entries 741 * x6 = L3 table 742 * x7 = Block attributes 743 * x8 = VA start 744 * x9 = PA start (trashed) 745 * x10 = Entry count (trashed) 746 * x11, x12 and x13 are trashed 747 */ 748LENTRY(build_l3_page_pagetable) 749 /* 750 * Build the L3 table entry. 751 */ 752 /* Find the table index */ 753 lsr x11, x8, #L3_SHIFT 754 and x11, x11, #Ln_ADDR_MASK 755 756 /* Build the L3 page entry */ 757 orr x12, x7, #L3_PAGE 758 orr x12, x12, #(ATTR_DEFAULT) 759 orr x12, x12, #(ATTR_S1_UXN) 760 761 /* Only use the output address bits */ 762 lsr x9, x9, #L3_SHIFT 763 764 /* Set the physical address for this virtual address */ 7651: orr x13, x12, x9, lsl #L3_SHIFT 766 767 /* Store the entry */ 768 str x13, [x6, x11, lsl #3] 769 770 sub x10, x10, #1 771 add x11, x11, #1 772 add x9, x9, #1 773 cbnz x10, 1b 774 775 ret 776LEND(build_l3_page_pagetable) 777#endif 778 779LENTRY(start_mmu) 780 dsb sy 781 782 /* Load the exception vectors */ 783 ldr x2, =exception_vectors 784 msr vbar_el1, x2 785 786 /* Load ttbr0 and ttbr1 */ 787 msr ttbr0_el1, x27 788 msr ttbr1_el1, x24 789 isb 790 791 /* Clear the Monitor Debug System control register */ 792 msr mdscr_el1, xzr 793 794 /* Invalidate the TLB */ 795 tlbi vmalle1is 796 dsb ish 797 isb 798 799 ldr x2, mair 800 msr mair_el1, x2 801 802 /* 803 * Setup TCR according to the PARange and ASIDBits fields 804 * from ID_AA64MMFR0_EL1 and the HAFDBS field from the 805 * ID_AA64MMFR1_EL1. More precisely, set TCR_EL1.AS 806 * to 1 only if the ASIDBits field equals 0b0010. 807 */ 808 ldr x2, tcr 809 mrs x3, id_aa64mmfr0_el1 810 811 /* Copy the bottom 3 bits from id_aa64mmfr0_el1 into TCR.IPS */ 812 bfi x2, x3, #(TCR_IPS_SHIFT), #(TCR_IPS_WIDTH) 813 and x3, x3, #(ID_AA64MMFR0_ASIDBits_MASK) 814 815 /* Check if the HW supports 16 bit ASIDS */ 816 cmp x3, #(ID_AA64MMFR0_ASIDBits_16) 817 /* If so x3 == 1, else x3 == 0 */ 818 cset x3, eq 819 /* Set TCR.AS with x3 */ 820 bfi x2, x3, #(TCR_ASID_SHIFT), #(TCR_ASID_WIDTH) 821 822 /* 823 * Check if the HW supports access flag and dirty state updates, 824 * and set TCR_EL1.HA and TCR_EL1.HD accordingly. 825 */ 826 mrs x3, id_aa64mmfr1_el1 827 and x3, x3, #(ID_AA64MMFR1_HAFDBS_MASK) 828 cmp x3, #1 829 b.ne 1f 830 orr x2, x2, #(TCR_HA) 831 b 2f 8321: 833 cmp x3, #2 834 b.ne 2f 835 orr x2, x2, #(TCR_HA | TCR_HD) 8362: 837 msr tcr_el1, x2 838 839 /* 840 * Setup SCTLR. 841 */ 842 ldr x2, sctlr_set 843 ldr x3, sctlr_clear 844 mrs x1, sctlr_el1 845 bic x1, x1, x3 /* Clear the required bits */ 846 orr x1, x1, x2 /* Set the required bits */ 847 msr sctlr_el1, x1 848 isb 849 850 ret 851 852 .align 3 853mair: 854 .quad MAIR_ATTR(MAIR_DEVICE_nGnRnE, VM_MEMATTR_DEVICE_nGnRnE) | \ 855 MAIR_ATTR(MAIR_NORMAL_NC, VM_MEMATTR_UNCACHEABLE) | \ 856 MAIR_ATTR(MAIR_NORMAL_WB, VM_MEMATTR_WRITE_BACK) | \ 857 MAIR_ATTR(MAIR_NORMAL_WT, VM_MEMATTR_WRITE_THROUGH) | \ 858 MAIR_ATTR(MAIR_DEVICE_nGnRE, VM_MEMATTR_DEVICE_nGnRE) 859tcr: 860#if PAGE_SIZE == PAGE_SIZE_4K 861#define TCR_TG (TCR_TG1_4K | TCR_TG0_4K) 862#elif PAGE_SIZE == PAGE_SIZE_16K 863#define TCR_TG (TCR_TG1_16K | TCR_TG0_16K) 864#else 865#error Unsupported page size 866#endif 867 868 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_TG | \ 869 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 870sctlr_set: 871 /* Bits to set */ 872 .quad (SCTLR_LSMAOE | SCTLR_nTLSMD | SCTLR_UCI | SCTLR_SPAN | \ 873 SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 874 SCTLR_I | SCTLR_SED | SCTLR_SA0 | SCTLR_SA | SCTLR_C | \ 875 SCTLR_M | SCTLR_CP15BEN) 876sctlr_clear: 877 /* Bits to clear */ 878 .quad (SCTLR_EE | SCTLR_E0E | SCTLR_IESB | SCTLR_WXN | SCTLR_UMA | \ 879 SCTLR_ITD | SCTLR_A) 880LEND(start_mmu) 881 882ENTRY(abort) 883 b abort 884END(abort) 885 886 .section .init_pagetable, "aw", %nobits 887 .align PAGE_SHIFT 888 /* 889 * 6 initial tables (in the following order): 890 * L2 for kernel (High addresses) 891 * L1 for kernel 892 * L0 for kernel 893 * L1 bootstrap for user (Low addresses) 894 * L0 bootstrap for user 895 * L0 for user 896 */ 897 .globl pagetable_l0_ttbr1 898pagetable: 899#if PAGE_SIZE != PAGE_SIZE_4K 900 .space (PAGE_SIZE * L3_PAGE_COUNT) 901pagetable_l2_ttbr1: 902#endif 903 .space PAGE_SIZE 904pagetable_l1_ttbr1: 905 .space PAGE_SIZE 906pagetable_l0_ttbr1: 907 .space PAGE_SIZE 908pagetable_l2_ttbr0_bootstrap: 909 .space PAGE_SIZE 910pagetable_l1_ttbr0_bootstrap: 911 .space PAGE_SIZE 912pagetable_l0_ttbr0_boostrap: 913 .space PAGE_SIZE 914pagetable_l0_ttbr0: 915 .space PAGE_SIZE 916 917pagetable_end: 918 919el2_pagetable: 920 .space PAGE_SIZE 921 922 .align 4 923initstack: 924 .space (PAGE_SIZE * KSTACK_PAGES) 925initstack_end: 926 927 928.text 929EENTRY(aarch32_sigcode) 930 .word 0xe1a0000d // mov r0, sp 931 .word 0xe2800040 // add r0, r0, #SIGF_UC 932 .word 0xe59f700c // ldr r7, [pc, #12] 933 .word 0xef000000 // swi #0 934 .word 0xe59f7008 // ldr r7, [pc, #8] 935 .word 0xef000000 // swi #0 936 .word 0xeafffffa // b . - 16 937EEND(aarch32_sigcode) 938 .word SYS_sigreturn 939 .word SYS_exit 940 .align 3 941aarch32_esigcode: 942 .data 943 .global sz_aarch32_sigcode 944sz_aarch32_sigcode: 945 .quad aarch32_esigcode - aarch32_sigcode 946