1/*- 2 * Copyright (c) 2012-2014 Andrew Turner 3 * All rights reserved. 4 * 5 * Redistribution and use in source and binary forms, with or without 6 * modification, are permitted provided that the following conditions 7 * are met: 8 * 1. Redistributions of source code must retain the above copyright 9 * notice, this list of conditions and the following disclaimer. 10 * 2. Redistributions in binary form must reproduce the above copyright 11 * notice, this list of conditions and the following disclaimer in the 12 * documentation and/or other materials provided with the distribution. 13 * 14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 15 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 16 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 17 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 18 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 19 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 20 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 21 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 22 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 23 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 24 * SUCH DAMAGE. 25 * 26 * $FreeBSD$ 27 */ 28 29#include "assym.s" 30#include "opt_kstack_pages.h" 31#include <sys/syscall.h> 32#include <machine/asm.h> 33#include <machine/armreg.h> 34#include <machine/hypervisor.h> 35#include <machine/param.h> 36#include <machine/pte.h> 37 38#define VIRT_BITS 39 39 40 .globl kernbase 41 .set kernbase, KERNBASE 42 43#define DEVICE_MEM 0 44#define NORMAL_UNCACHED 1 45#define NORMAL_MEM 2 46 47/* 48 * We assume: 49 * MMU on with an identity map, or off 50 * D-Cache: off 51 * I-Cache: on or off 52 * We are loaded at a 2MiB aligned address 53 */ 54 55 .text 56 .globl _start 57_start: 58 /* Drop to EL1 */ 59 bl drop_to_el1 60 61 /* 62 * Disable the MMU. We may have entered the kernel with it on and 63 * will need to update the tables later. If this has been set up 64 * with anything other than a VA == PA map then this will fail, 65 * but in this case the code to find where we are running from 66 * would have also failed. 67 */ 68 dsb sy 69 mrs x2, sctlr_el1 70 bic x2, x2, SCTLR_M 71 msr sctlr_el1, x2 72 isb 73 74 /* Set the context id */ 75 msr contextidr_el1, xzr 76 77 /* Get the virt -> phys offset */ 78 bl get_virt_delta 79 80 /* 81 * At this point: 82 * x29 = PA - VA 83 * x28 = Our physical load address 84 */ 85 86 /* Create the page tables */ 87 bl create_pagetables 88 89 /* 90 * At this point: 91 * x27 = TTBR0 table 92 * x26 = TTBR1 table 93 */ 94 95 /* Enable the mmu */ 96 bl start_mmu 97 98 /* Jump to the virtual address space */ 99 ldr x15, .Lvirtdone 100 br x15 101 102virtdone: 103 /* Set up the stack */ 104 adr x25, initstack_end 105 mov sp, x25 106 sub sp, sp, #PCB_SIZE 107 108 /* Zero the BSS */ 109 ldr x15, .Lbss 110 ldr x14, .Lend 1111: 112 str xzr, [x15], #8 113 cmp x15, x14 114 b.lo 1b 115 116 /* Backup the module pointer */ 117 mov x1, x0 118 119 /* Make the page table base a virtual address */ 120 sub x26, x26, x29 121 122 sub sp, sp, #(64 * 4) 123 mov x0, sp 124 125 /* Degate the delda so it is VA -> PA */ 126 neg x29, x29 127 128 str x1, [x0] /* modulep */ 129 str x26, [x0, 8] /* kern_l1pt */ 130 str x29, [x0, 16] /* kern_delta */ 131 str x25, [x0, 24] /* kern_stack */ 132 133 /* trace back starts here */ 134 mov fp, #0 135 /* Branch to C code */ 136 bl initarm 137 bl mi_startup 138 139 /* We should not get here */ 140 brk 0 141 142 .align 3 143.Lvirtdone: 144 .quad virtdone 145.Lbss: 146 .quad __bss_start 147.Lend: 148 .quad _end 149 150#ifdef SMP 151/* 152 * mpentry(unsigned long) 153 * 154 * Called by a core when it is being brought online. 155 * The data in x0 is passed straight to init_secondary. 156 */ 157ENTRY(mpentry) 158 /* Disable interrupts */ 159 msr daifset, #2 160 161 /* Drop to EL1 */ 162 bl drop_to_el1 163 164 /* Set the context id */ 165 msr contextidr_el1, x1 166 167 /* Load the kernel page table */ 168 adr x26, pagetable_l1_ttbr1 169 /* Load the identity page table */ 170 adr x27, pagetable_l1_ttbr0 171 172 /* Enable the mmu */ 173 bl start_mmu 174 175 /* Jump to the virtual address space */ 176 ldr x15, =mp_virtdone 177 br x15 178 179mp_virtdone: 180 ldr x4, =secondary_stacks 181 mov x5, #(PAGE_SIZE * KSTACK_PAGES) 182 mul x5, x0, x5 183 add sp, x4, x5 184 185 b init_secondary 186END(mpentry) 187#endif 188 189/* 190 * If we are started in EL2, configure the required hypervisor 191 * registers and drop to EL1. 192 */ 193drop_to_el1: 194 mrs x1, CurrentEL 195 lsr x1, x1, #2 196 cmp x1, #0x2 197 b.eq 1f 198 ret 1991: 200 /* Configure the Hypervisor */ 201 mov x2, #(HCR_RW) 202 msr hcr_el2, x2 203 204 /* Load the Virtualization Process ID Register */ 205 mrs x2, midr_el1 206 msr vpidr_el2, x2 207 208 /* Load the Virtualization Multiprocess ID Register */ 209 mrs x2, mpidr_el1 210 msr vmpidr_el2, x2 211 212 /* Set the bits that need to be 1 in sctlr_el1 */ 213 ldr x2, .Lsctlr_res1 214 msr sctlr_el1, x2 215 216 /* Don't trap to EL2 for exceptions */ 217 mov x2, #CPTR_RES1 218 msr cptr_el2, x2 219 220 /* Don't trap to EL2 for CP15 traps */ 221 msr hstr_el2, xzr 222 223 /* Enable access to the physical timers at EL1 */ 224 mrs x2, cnthctl_el2 225 orr x2, x2, #(CNTHCTL_EL1PCTEN | CNTHCTL_EL1PCEN) 226 msr cnthctl_el2, x2 227 228 /* Set the counter offset to a known value */ 229 msr cntvoff_el2, xzr 230 231 /* Hypervisor trap functions */ 232 adr x2, hyp_vectors 233 msr vbar_el2, x2 234 235 mov x2, #(PSR_F | PSR_I | PSR_A | PSR_D | PSR_M_EL1h) 236 msr spsr_el2, x2 237 238 /* Configure GICv3 CPU interface */ 239 mrs x2, id_aa64pfr0_el1 240 /* Extract GIC bits from the register */ 241 ubfx x2, x2, #ID_AA64PFR0_GIC_SHIFT, #ID_AA64PFR0_GIC_BITS 242 /* GIC[3:0] == 0001 - GIC CPU interface via special regs. supported */ 243 cmp x2, #(ID_AA64PFR0_GIC_CPUIF_EN >> ID_AA64PFR0_GIC_SHIFT) 244 b.ne 2f 245 246 mrs x2, icc_sre_el2 247 orr x2, x2, #ICC_SRE_EL2_EN /* Enable access from insecure EL1 */ 248 msr icc_sre_el2, x2 2492: 250 251 /* Set the address to return to our return address */ 252 msr elr_el2, x30 253 isb 254 255 eret 256 257 .align 3 258.Lsctlr_res1: 259 .quad SCTLR_RES1 260 261#define VECT_EMPTY \ 262 .align 7; \ 263 1: b 1b 264 265 .align 11 266hyp_vectors: 267 VECT_EMPTY /* Synchronous EL2t */ 268 VECT_EMPTY /* IRQ EL2t */ 269 VECT_EMPTY /* FIQ EL2t */ 270 VECT_EMPTY /* Error EL2t */ 271 272 VECT_EMPTY /* Synchronous EL2h */ 273 VECT_EMPTY /* IRQ EL2h */ 274 VECT_EMPTY /* FIQ EL2h */ 275 VECT_EMPTY /* Error EL2h */ 276 277 VECT_EMPTY /* Synchronous 64-bit EL1 */ 278 VECT_EMPTY /* IRQ 64-bit EL1 */ 279 VECT_EMPTY /* FIQ 64-bit EL1 */ 280 VECT_EMPTY /* Error 64-bit EL1 */ 281 282 VECT_EMPTY /* Synchronous 32-bit EL1 */ 283 VECT_EMPTY /* IRQ 32-bit EL1 */ 284 VECT_EMPTY /* FIQ 32-bit EL1 */ 285 VECT_EMPTY /* Error 32-bit EL1 */ 286 287/* 288 * Get the delta between the physical address we were loaded to and the 289 * virtual address we expect to run from. This is used when building the 290 * initial page table. 291 */ 292get_virt_delta: 293 /* Load the physical address of virt_map */ 294 adr x29, virt_map 295 /* Load the virtual address of virt_map stored in virt_map */ 296 ldr x28, [x29] 297 /* Find PA - VA as PA' = VA' - VA + PA = VA' + (PA - VA) = VA' + x29 */ 298 sub x29, x29, x28 299 /* Find the load address for the kernel */ 300 mov x28, #(KERNBASE) 301 add x28, x28, x29 302 ret 303 304 .align 3 305virt_map: 306 .quad virt_map 307 308/* 309 * This builds the page tables containing the identity map, and the kernel 310 * virtual map. 311 * 312 * It relys on: 313 * We were loaded to an address that is on a 2MiB boundary 314 * All the memory must not cross a 1GiB boundaty 315 * x28 contains the physical address we were loaded from 316 * 317 * TODO: This is out of date. 318 * There are at least 5 pages before that address for the page tables 319 * The pages used are: 320 * - The identity (PA = VA) table (TTBR0) 321 * - The Kernel L1 table (TTBR1)(not yet) 322 * - The PA != VA L2 table to jump into (not yet) 323 * - The FDT L2 table (not yet) 324 */ 325create_pagetables: 326 /* Save the Link register */ 327 mov x5, x30 328 329 /* Clean the page table */ 330 adr x6, pagetable 331 mov x26, x6 332 adr x27, pagetable_end 3331: 334 stp xzr, xzr, [x6], #16 335 stp xzr, xzr, [x6], #16 336 stp xzr, xzr, [x6], #16 337 stp xzr, xzr, [x6], #16 338 cmp x6, x27 339 b.lo 1b 340 341 /* 342 * Build the TTBR1 maps. 343 */ 344 345 /* Find the size of the kernel */ 346 mov x6, #(KERNBASE) 347 ldr x7, .Lend 348 /* Find the end - begin */ 349 sub x8, x7, x6 350 /* Get the number of l2 pages to allocate, rounded down */ 351 lsr x10, x8, #(L2_SHIFT) 352 /* Add 4 MiB for any rounding above and the module data */ 353 add x10, x10, #2 354 355 /* Create the kernel space L2 table */ 356 mov x6, x26 357 mov x7, #NORMAL_MEM 358 mov x8, #(KERNBASE & L2_BLOCK_MASK) 359 mov x9, x28 360 bl build_l2_block_pagetable 361 362 /* Move to the l1 table */ 363 add x26, x26, #PAGE_SIZE 364 365 /* Link the l1 -> l2 table */ 366 mov x9, x6 367 mov x6, x26 368 bl link_l1_pagetable 369 370 371 /* 372 * Build the TTBR0 maps. 373 */ 374 add x27, x26, #PAGE_SIZE 375 376#if defined(SOCDEV_PA) && defined(SOCDEV_VA) 377 /* Create a table for the UART */ 378 mov x6, x27 /* The initial page table */ 379 mov x7, #DEVICE_MEM 380 mov x8, #(SOCDEV_VA) /* VA start */ 381 mov x9, #(SOCDEV_PA) /* PA start */ 382 mov x10, #1 383 bl build_l1_block_pagetable 384#endif 385 386 /* Create the VA = PA map */ 387 mov x6, x27 /* The initial page table */ 388 mov x7, #NORMAL_UNCACHED /* Uncached as it's only needed early on */ 389 mov x9, x27 390 mov x8, x9 /* VA start (== PA start) */ 391 mov x10, #1 392 bl build_l1_block_pagetable 393 394 /* Restore the Link register */ 395 mov x30, x5 396 ret 397 398/* 399 * Builds an L1 -> L2 table descriptor 400 * 401 * This is a link for a 1GiB block of memory with up to 2MiB regions mapped 402 * within it by build_l2_block_pagetable. 403 * 404 * x6 = L1 table 405 * x8 = Virtual Address 406 * x9 = L2 PA (trashed) 407 * x11, x12 and x13 are trashed 408 */ 409link_l1_pagetable: 410 /* 411 * Link an L1 -> L2 table entry. 412 */ 413 /* Find the table index */ 414 lsr x11, x8, #L1_SHIFT 415 and x11, x11, #Ln_ADDR_MASK 416 417 /* Build the L1 block entry */ 418 mov x12, #L1_TABLE 419 420 /* Only use the output address bits */ 421 lsr x9, x9, #12 422 orr x12, x12, x9, lsl #12 423 424 /* Store the entry */ 425 str x12, [x6, x11, lsl #3] 426 427 ret 428 429/* 430 * Builds count 1 GiB page table entry 431 * x6 = L1 table 432 * x7 = Type (0 = Device, 1 = Normal) 433 * x8 = VA start 434 * x9 = PA start (trashed) 435 * x10 = Entry count (TODO) 436 * x11, x12 and x13 are trashed 437 */ 438build_l1_block_pagetable: 439 /* 440 * Build the L1 table entry. 441 */ 442 /* Find the table index */ 443 lsr x11, x8, #L1_SHIFT 444 and x11, x11, #Ln_ADDR_MASK 445 446 /* Build the L1 block entry */ 447 lsl x12, x7, #2 448 orr x12, x12, #L1_BLOCK 449 orr x12, x12, #(ATTR_AF) 450#ifdef SMP 451 orr x12, x12, ATTR_SH(ATTR_SH_IS) 452#endif 453 454 /* Only use the output address bits */ 455 lsr x9, x9, #L1_SHIFT 456 457 /* Set the physical address for this virtual address */ 4581: orr x12, x12, x9, lsl #L1_SHIFT 459 460 /* Store the entry */ 461 str x12, [x6, x11, lsl #3] 462 463 /* Clear the address bits */ 464 and x12, x12, #ATTR_MASK_L 465 466 sub x10, x10, #1 467 add x11, x11, #1 468 add x9, x9, #1 469 cbnz x10, 1b 470 4712: ret 472 473/* 474 * Builds count 2 MiB page table entry 475 * x6 = L2 table 476 * x7 = Type (0 = Device, 1 = Normal) 477 * x8 = VA start 478 * x9 = PA start (trashed) 479 * x10 = Entry count (TODO) 480 * x11, x12 and x13 are trashed 481 */ 482build_l2_block_pagetable: 483 /* 484 * Build the L2 table entry. 485 */ 486 /* Find the table index */ 487 lsr x11, x8, #L2_SHIFT 488 and x11, x11, #Ln_ADDR_MASK 489 490 /* Build the L2 block entry */ 491 lsl x12, x7, #2 492 orr x12, x12, #L2_BLOCK 493 orr x12, x12, #(ATTR_AF) 494#ifdef SMP 495 orr x12, x12, ATTR_SH(ATTR_SH_IS) 496#endif 497 498 /* Only use the output address bits */ 499 lsr x9, x9, #L2_SHIFT 500 501 /* Set the physical address for this virtual address */ 5021: orr x12, x12, x9, lsl #L2_SHIFT 503 504 /* Store the entry */ 505 str x12, [x6, x11, lsl #3] 506 507 /* Clear the address bits */ 508 and x12, x12, #ATTR_MASK_L 509 510 sub x10, x10, #1 511 add x11, x11, #1 512 add x9, x9, #1 513 cbnz x10, 1b 514 5152: ret 516 517start_mmu: 518 dsb sy 519 520 /* Load the exception vectors */ 521 ldr x2, =exception_vectors 522 msr vbar_el1, x2 523 524 /* Load ttbr0 and ttbr1 */ 525 msr ttbr0_el1, x27 526 msr ttbr1_el1, x26 527 isb 528 529 /* Clear the Monitor Debug System control register */ 530 msr mdscr_el1, xzr 531 532 /* Invalidate the TLB */ 533 tlbi vmalle1is 534 535 ldr x2, mair 536 msr mair_el1, x2 537 538 /* Setup TCR according to PARange bits from ID_AA64MMFR0_EL1 */ 539 ldr x2, tcr 540 mrs x3, id_aa64mmfr0_el1 541 bfi x2, x3, #32, #3 542 msr tcr_el1, x2 543 544 /* Setup SCTLR */ 545 ldr x2, sctlr_set 546 ldr x3, sctlr_clear 547 mrs x1, sctlr_el1 548 bic x1, x1, x3 /* Clear the required bits */ 549 orr x1, x1, x2 /* Set the required bits */ 550 msr sctlr_el1, x1 551 isb 552 553 ret 554 555 .align 3 556mair: 557 /* Device Normal, no cache Normal, write-back */ 558 .quad MAIR_ATTR(0x00, 0) | MAIR_ATTR(0x44, 1) | MAIR_ATTR(0xff, 2) 559tcr: 560 .quad (TCR_TxSZ(64 - VIRT_BITS) | TCR_ASID_16 | TCR_TG1_4K | \ 561 TCR_CACHE_ATTRS | TCR_SMP_ATTRS) 562sctlr_set: 563 /* Bits to set */ 564 .quad (SCTLR_UCI | SCTLR_nTWE | SCTLR_nTWI | SCTLR_UCT | SCTLR_DZE | \ 565 SCTLR_I | SCTLR_SED | SCTLR_C | SCTLR_M) 566sctlr_clear: 567 /* Bits to clear */ 568 .quad (SCTLR_EE | SCTLR_EOE | SCTLR_WXN | SCTLR_UMA | SCTLR_ITD | \ 569 SCTLR_THEE | SCTLR_CP15BEN | SCTLR_SA0 | SCTLR_SA | SCTLR_A) 570 571 .globl abort 572abort: 573 b abort 574 575 //.section .init_pagetable 576 .align 12 /* 4KiB aligned */ 577 /* 578 * 3 initial tables (in the following order): 579 * L2 for kernel (High addresses) 580 * L1 for kernel 581 * L1 for user (Low addresses) 582 */ 583pagetable: 584 .space PAGE_SIZE 585pagetable_l1_ttbr1: 586 .space PAGE_SIZE 587pagetable_l1_ttbr0: 588 .space PAGE_SIZE 589pagetable_end: 590 591el2_pagetable: 592 .space PAGE_SIZE 593 594 .globl init_pt_va 595init_pt_va: 596 .quad pagetable /* XXX: Keep page tables VA */ 597 598 .align 4 599initstack: 600 .space (PAGE_SIZE * KSTACK_PAGES) 601initstack_end: 602 603 604ENTRY(sigcode) 605 mov x0, sp 606 add x0, x0, #SF_UC 607 6081: 609 mov x8, #SYS_sigreturn 610 svc 0 611 612 /* sigreturn failed, exit */ 613 mov x8, #SYS_exit 614 svc 0 615 616 b 1b 617END(sigcode) 618 /* This may be copied to the stack, keep it 16-byte aligned */ 619 .align 3 620esigcode: 621 622 .data 623 .align 3 624 .global szsigcode 625szsigcode: 626 .quad esigcode - sigcode 627