147e07394SAndrew Turner /*-
247e07394SAndrew Turner * SPDX-License-Identifier: BSD-2-Clause
347e07394SAndrew Turner *
447e07394SAndrew Turner * Copyright (C) 2015 Mihai Carabas <mihai.carabas@gmail.com>
547e07394SAndrew Turner * All rights reserved.
647e07394SAndrew Turner *
747e07394SAndrew Turner * Redistribution and use in source and binary forms, with or without
847e07394SAndrew Turner * modification, are permitted provided that the following conditions
947e07394SAndrew Turner * are met:
1047e07394SAndrew Turner * 1. Redistributions of source code must retain the above copyright
1147e07394SAndrew Turner * notice, this list of conditions and the following disclaimer.
1247e07394SAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright
1347e07394SAndrew Turner * notice, this list of conditions and the following disclaimer in the
1447e07394SAndrew Turner * documentation and/or other materials provided with the distribution.
1547e07394SAndrew Turner *
1647e07394SAndrew Turner * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
1747e07394SAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
1847e07394SAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
1947e07394SAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
2047e07394SAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
2147e07394SAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
2247e07394SAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
2347e07394SAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
2447e07394SAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
2547e07394SAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
2647e07394SAndrew Turner * SUCH DAMAGE.
2747e07394SAndrew Turner */
2847e07394SAndrew Turner
2947e07394SAndrew Turner #include <sys/cdefs.h>
3047e07394SAndrew Turner #include <sys/param.h>
3147e07394SAndrew Turner #include <sys/systm.h>
3247e07394SAndrew Turner #include <sys/smp.h>
3347e07394SAndrew Turner #include <sys/kernel.h>
3447e07394SAndrew Turner #include <sys/malloc.h>
3547e07394SAndrew Turner #include <sys/mman.h>
3647e07394SAndrew Turner #include <sys/pcpu.h>
3747e07394SAndrew Turner #include <sys/proc.h>
3847e07394SAndrew Turner #include <sys/sysctl.h>
3947e07394SAndrew Turner #include <sys/lock.h>
4047e07394SAndrew Turner #include <sys/mutex.h>
4147e07394SAndrew Turner #include <sys/vmem.h>
4247e07394SAndrew Turner
4347e07394SAndrew Turner #include <vm/vm.h>
4447e07394SAndrew Turner #include <vm/pmap.h>
4547e07394SAndrew Turner #include <vm/vm_extern.h>
4647e07394SAndrew Turner #include <vm/vm_map.h>
4747e07394SAndrew Turner #include <vm/vm_page.h>
4847e07394SAndrew Turner #include <vm/vm_param.h>
4947e07394SAndrew Turner
5047e07394SAndrew Turner #include <machine/armreg.h>
5147e07394SAndrew Turner #include <machine/vm.h>
5247e07394SAndrew Turner #include <machine/cpufunc.h>
5347e07394SAndrew Turner #include <machine/cpu.h>
5447e07394SAndrew Turner #include <machine/machdep.h>
5547e07394SAndrew Turner #include <machine/vmm.h>
5647e07394SAndrew Turner #include <machine/vmm_dev.h>
5747e07394SAndrew Turner #include <machine/atomic.h>
5847e07394SAndrew Turner #include <machine/hypervisor.h>
5947e07394SAndrew Turner #include <machine/pmap.h>
6047e07394SAndrew Turner
6147e07394SAndrew Turner #include "mmu.h"
6247e07394SAndrew Turner #include "arm64.h"
6347e07394SAndrew Turner #include "hyp.h"
6447e07394SAndrew Turner #include "reset.h"
6547e07394SAndrew Turner #include "io/vgic.h"
6647e07394SAndrew Turner #include "io/vgic_v3.h"
6747e07394SAndrew Turner #include "io/vtimer.h"
6847e07394SAndrew Turner #include "vmm_stat.h"
6947e07394SAndrew Turner
7047e07394SAndrew Turner #define HANDLED 1
7147e07394SAndrew Turner #define UNHANDLED 0
7247e07394SAndrew Turner
7347e07394SAndrew Turner /* Number of bits in an EL2 virtual address */
7447e07394SAndrew Turner #define EL2_VIRT_BITS 48
7547e07394SAndrew Turner CTASSERT((1ul << EL2_VIRT_BITS) >= HYP_VM_MAX_ADDRESS);
7647e07394SAndrew Turner
7747e07394SAndrew Turner /* TODO: Move the host hypctx off the stack */
7847e07394SAndrew Turner #define VMM_STACK_PAGES 4
7947e07394SAndrew Turner #define VMM_STACK_SIZE (VMM_STACK_PAGES * PAGE_SIZE)
8047e07394SAndrew Turner
8147e07394SAndrew Turner static int vmm_pmap_levels, vmm_virt_bits, vmm_max_ipa_bits;
8247e07394SAndrew Turner
8347e07394SAndrew Turner /* Register values passed to arm_setup_vectors to set in the hypervisor */
8447e07394SAndrew Turner struct vmm_init_regs {
8547e07394SAndrew Turner uint64_t tcr_el2;
8647e07394SAndrew Turner uint64_t vtcr_el2;
8747e07394SAndrew Turner };
8847e07394SAndrew Turner
8947e07394SAndrew Turner MALLOC_DEFINE(M_HYP, "ARM VMM HYP", "ARM VMM HYP");
9047e07394SAndrew Turner
9147e07394SAndrew Turner extern char hyp_init_vectors[];
9247e07394SAndrew Turner extern char hyp_vectors[];
9347e07394SAndrew Turner extern char hyp_stub_vectors[];
9447e07394SAndrew Turner
9547e07394SAndrew Turner static vm_paddr_t hyp_code_base;
9647e07394SAndrew Turner static size_t hyp_code_len;
9747e07394SAndrew Turner
9847e07394SAndrew Turner static char *stack[MAXCPU];
9947e07394SAndrew Turner static vm_offset_t stack_hyp_va[MAXCPU];
10047e07394SAndrew Turner
10147e07394SAndrew Turner static vmem_t *el2_mem_alloc;
10247e07394SAndrew Turner
10347e07394SAndrew Turner static void arm_setup_vectors(void *arg);
10447e07394SAndrew Turner static void vmm_pmap_clean_stage2_tlbi(void);
10547e07394SAndrew Turner static void vmm_pmap_invalidate_range(uint64_t, vm_offset_t, vm_offset_t, bool);
10647e07394SAndrew Turner static void vmm_pmap_invalidate_all(uint64_t);
10747e07394SAndrew Turner
10847e07394SAndrew Turner DPCPU_DEFINE_STATIC(struct hypctx *, vcpu);
10947e07394SAndrew Turner
11047e07394SAndrew Turner static inline void
arm64_set_active_vcpu(struct hypctx * hypctx)11147e07394SAndrew Turner arm64_set_active_vcpu(struct hypctx *hypctx)
11247e07394SAndrew Turner {
11347e07394SAndrew Turner DPCPU_SET(vcpu, hypctx);
11447e07394SAndrew Turner }
11547e07394SAndrew Turner
11647e07394SAndrew Turner struct hypctx *
arm64_get_active_vcpu(void)11747e07394SAndrew Turner arm64_get_active_vcpu(void)
11847e07394SAndrew Turner {
11947e07394SAndrew Turner return (DPCPU_GET(vcpu));
12047e07394SAndrew Turner }
12147e07394SAndrew Turner
12247e07394SAndrew Turner static void
arm_setup_vectors(void * arg)12347e07394SAndrew Turner arm_setup_vectors(void *arg)
12447e07394SAndrew Turner {
12547e07394SAndrew Turner struct vmm_init_regs *el2_regs;
12647e07394SAndrew Turner uintptr_t stack_top;
12747e07394SAndrew Turner uint32_t sctlr_el2;
12847e07394SAndrew Turner register_t daif;
12947e07394SAndrew Turner
13047e07394SAndrew Turner el2_regs = arg;
13147e07394SAndrew Turner arm64_set_active_vcpu(NULL);
13247e07394SAndrew Turner
13347e07394SAndrew Turner daif = intr_disable();
13447e07394SAndrew Turner
13547e07394SAndrew Turner /*
13647e07394SAndrew Turner * Install the temporary vectors which will be responsible for
13747e07394SAndrew Turner * initializing the VMM when we next trap into EL2.
13847e07394SAndrew Turner *
13947e07394SAndrew Turner * x0: the exception vector table responsible for hypervisor
14047e07394SAndrew Turner * initialization on the next call.
14147e07394SAndrew Turner */
14247e07394SAndrew Turner vmm_call_hyp(vtophys(&vmm_hyp_code));
14347e07394SAndrew Turner
14447e07394SAndrew Turner /* Create and map the hypervisor stack */
14547e07394SAndrew Turner stack_top = stack_hyp_va[PCPU_GET(cpuid)] + VMM_STACK_SIZE;
14647e07394SAndrew Turner
14747e07394SAndrew Turner /*
14847e07394SAndrew Turner * Configure the system control register for EL2:
14947e07394SAndrew Turner *
15047e07394SAndrew Turner * SCTLR_EL2_M: MMU on
15147e07394SAndrew Turner * SCTLR_EL2_C: Data cacheability not affected
15247e07394SAndrew Turner * SCTLR_EL2_I: Instruction cacheability not affected
15347e07394SAndrew Turner * SCTLR_EL2_A: Instruction alignment check
15447e07394SAndrew Turner * SCTLR_EL2_SA: Stack pointer alignment check
15547e07394SAndrew Turner * SCTLR_EL2_WXN: Treat writable memory as execute never
15647e07394SAndrew Turner * ~SCTLR_EL2_EE: Data accesses are little-endian
15747e07394SAndrew Turner */
15847e07394SAndrew Turner sctlr_el2 = SCTLR_EL2_RES1;
15947e07394SAndrew Turner sctlr_el2 |= SCTLR_EL2_M | SCTLR_EL2_C | SCTLR_EL2_I;
16047e07394SAndrew Turner sctlr_el2 |= SCTLR_EL2_A | SCTLR_EL2_SA;
16147e07394SAndrew Turner sctlr_el2 |= SCTLR_EL2_WXN;
16247e07394SAndrew Turner sctlr_el2 &= ~SCTLR_EL2_EE;
16347e07394SAndrew Turner
16447e07394SAndrew Turner /* Special call to initialize EL2 */
16547e07394SAndrew Turner vmm_call_hyp(vmmpmap_to_ttbr0(), stack_top, el2_regs->tcr_el2,
16647e07394SAndrew Turner sctlr_el2, el2_regs->vtcr_el2);
16747e07394SAndrew Turner
16847e07394SAndrew Turner intr_restore(daif);
16947e07394SAndrew Turner }
17047e07394SAndrew Turner
17147e07394SAndrew Turner static void
arm_teardown_vectors(void * arg)17247e07394SAndrew Turner arm_teardown_vectors(void *arg)
17347e07394SAndrew Turner {
17447e07394SAndrew Turner register_t daif;
17547e07394SAndrew Turner
17647e07394SAndrew Turner /*
17747e07394SAndrew Turner * vmm_cleanup() will disable the MMU. For the next few instructions,
17847e07394SAndrew Turner * before the hardware disables the MMU, one of the following is
17947e07394SAndrew Turner * possible:
18047e07394SAndrew Turner *
18147e07394SAndrew Turner * a. The instruction addresses are fetched with the MMU disabled,
18247e07394SAndrew Turner * and they must represent the actual physical addresses. This will work
18347e07394SAndrew Turner * because we call the vmm_cleanup() function by its physical address.
18447e07394SAndrew Turner *
18547e07394SAndrew Turner * b. The instruction addresses are fetched using the old translation
18647e07394SAndrew Turner * tables. This will work because we have an identity mapping in place
18747e07394SAndrew Turner * in the translation tables and vmm_cleanup() is called by its physical
18847e07394SAndrew Turner * address.
18947e07394SAndrew Turner */
19047e07394SAndrew Turner daif = intr_disable();
19147e07394SAndrew Turner /* TODO: Invalidate the cache */
19247e07394SAndrew Turner vmm_call_hyp(HYP_CLEANUP, vtophys(hyp_stub_vectors));
19347e07394SAndrew Turner intr_restore(daif);
19447e07394SAndrew Turner
19547e07394SAndrew Turner arm64_set_active_vcpu(NULL);
19647e07394SAndrew Turner }
19747e07394SAndrew Turner
19847e07394SAndrew Turner static uint64_t
vmm_vtcr_el2_sl(u_int levels)19947e07394SAndrew Turner vmm_vtcr_el2_sl(u_int levels)
20047e07394SAndrew Turner {
20147e07394SAndrew Turner #if PAGE_SIZE == PAGE_SIZE_4K
20247e07394SAndrew Turner switch (levels) {
20347e07394SAndrew Turner case 2:
20447e07394SAndrew Turner return (VTCR_EL2_SL0_4K_LVL2);
20547e07394SAndrew Turner case 3:
20647e07394SAndrew Turner return (VTCR_EL2_SL0_4K_LVL1);
20747e07394SAndrew Turner case 4:
20847e07394SAndrew Turner return (VTCR_EL2_SL0_4K_LVL0);
20947e07394SAndrew Turner default:
21047e07394SAndrew Turner panic("%s: Invalid number of page table levels %u", __func__,
21147e07394SAndrew Turner levels);
21247e07394SAndrew Turner }
21347e07394SAndrew Turner #elif PAGE_SIZE == PAGE_SIZE_16K
21447e07394SAndrew Turner switch (levels) {
21547e07394SAndrew Turner case 2:
21647e07394SAndrew Turner return (VTCR_EL2_SL0_16K_LVL2);
21747e07394SAndrew Turner case 3:
21847e07394SAndrew Turner return (VTCR_EL2_SL0_16K_LVL1);
21947e07394SAndrew Turner case 4:
22047e07394SAndrew Turner return (VTCR_EL2_SL0_16K_LVL0);
22147e07394SAndrew Turner default:
22247e07394SAndrew Turner panic("%s: Invalid number of page table levels %u", __func__,
22347e07394SAndrew Turner levels);
22447e07394SAndrew Turner }
22547e07394SAndrew Turner #else
22647e07394SAndrew Turner #error Unsupported page size
22747e07394SAndrew Turner #endif
22847e07394SAndrew Turner }
22947e07394SAndrew Turner
23047e07394SAndrew Turner int
vmmops_modinit(int ipinum)23147e07394SAndrew Turner vmmops_modinit(int ipinum)
23247e07394SAndrew Turner {
23347e07394SAndrew Turner struct vmm_init_regs el2_regs;
23447e07394SAndrew Turner vm_offset_t next_hyp_va;
23547e07394SAndrew Turner vm_paddr_t vmm_base;
23647e07394SAndrew Turner uint64_t id_aa64mmfr0_el1, pa_range_bits, pa_range_field;
23747e07394SAndrew Turner uint64_t cnthctl_el2;
23847e07394SAndrew Turner register_t daif;
23947e07394SAndrew Turner int cpu, i;
24047e07394SAndrew Turner bool rv __diagused;
24147e07394SAndrew Turner
24247e07394SAndrew Turner if (!virt_enabled()) {
24347e07394SAndrew Turner printf(
24447e07394SAndrew Turner "vmm: Processor doesn't have support for virtualization\n");
24547e07394SAndrew Turner return (ENXIO);
24647e07394SAndrew Turner }
24747e07394SAndrew Turner
24847e07394SAndrew Turner /* TODO: Support VHE */
24947e07394SAndrew Turner if (in_vhe()) {
25047e07394SAndrew Turner printf("vmm: VHE is unsupported\n");
25147e07394SAndrew Turner return (ENXIO);
25247e07394SAndrew Turner }
25347e07394SAndrew Turner
25447e07394SAndrew Turner if (!vgic_present()) {
25547e07394SAndrew Turner printf("vmm: No vgic found\n");
25647e07394SAndrew Turner return (ENODEV);
25747e07394SAndrew Turner }
25847e07394SAndrew Turner
25947e07394SAndrew Turner if (!get_kernel_reg(ID_AA64MMFR0_EL1, &id_aa64mmfr0_el1)) {
26047e07394SAndrew Turner printf("vmm: Unable to read ID_AA64MMFR0_EL1\n");
26147e07394SAndrew Turner return (ENXIO);
26247e07394SAndrew Turner }
26347e07394SAndrew Turner pa_range_field = ID_AA64MMFR0_PARange_VAL(id_aa64mmfr0_el1);
26447e07394SAndrew Turner /*
26547e07394SAndrew Turner * Use 3 levels to give us up to 39 bits with 4k pages, or
26647e07394SAndrew Turner * 47 bits with 16k pages.
26747e07394SAndrew Turner */
26847e07394SAndrew Turner /* TODO: Check the number of levels for 64k pages */
26947e07394SAndrew Turner vmm_pmap_levels = 3;
27047e07394SAndrew Turner switch (pa_range_field) {
27147e07394SAndrew Turner case ID_AA64MMFR0_PARange_4G:
27247e07394SAndrew Turner printf("vmm: Not enough physical address bits\n");
27347e07394SAndrew Turner return (ENXIO);
27447e07394SAndrew Turner case ID_AA64MMFR0_PARange_64G:
27547e07394SAndrew Turner vmm_virt_bits = 36;
27647e07394SAndrew Turner #if PAGE_SIZE == PAGE_SIZE_16K
27747e07394SAndrew Turner vmm_pmap_levels = 2;
27847e07394SAndrew Turner #endif
27947e07394SAndrew Turner break;
28047e07394SAndrew Turner default:
28147e07394SAndrew Turner vmm_virt_bits = 39;
28247e07394SAndrew Turner break;
28347e07394SAndrew Turner }
28447e07394SAndrew Turner pa_range_bits = pa_range_field >> ID_AA64MMFR0_PARange_SHIFT;
28547e07394SAndrew Turner
28647e07394SAndrew Turner /* Initialise the EL2 MMU */
28747e07394SAndrew Turner if (!vmmpmap_init()) {
28847e07394SAndrew Turner printf("vmm: Failed to init the EL2 MMU\n");
28947e07394SAndrew Turner return (ENOMEM);
29047e07394SAndrew Turner }
29147e07394SAndrew Turner
29247e07394SAndrew Turner /* Set up the stage 2 pmap callbacks */
29347e07394SAndrew Turner MPASS(pmap_clean_stage2_tlbi == NULL);
29447e07394SAndrew Turner pmap_clean_stage2_tlbi = vmm_pmap_clean_stage2_tlbi;
29547e07394SAndrew Turner pmap_stage2_invalidate_range = vmm_pmap_invalidate_range;
29647e07394SAndrew Turner pmap_stage2_invalidate_all = vmm_pmap_invalidate_all;
29747e07394SAndrew Turner
29847e07394SAndrew Turner /*
29947e07394SAndrew Turner * Create an allocator for the virtual address space used by EL2.
30047e07394SAndrew Turner * EL2 code is identity-mapped; the allocator is used to find space for
30147e07394SAndrew Turner * VM structures.
30247e07394SAndrew Turner */
30347e07394SAndrew Turner el2_mem_alloc = vmem_create("VMM EL2", 0, 0, PAGE_SIZE, 0, M_WAITOK);
30447e07394SAndrew Turner
30547e07394SAndrew Turner /* Create the mappings for the hypervisor translation table. */
30647e07394SAndrew Turner hyp_code_len = round_page(&vmm_hyp_code_end - &vmm_hyp_code);
30747e07394SAndrew Turner
30847e07394SAndrew Turner /* We need an physical identity mapping for when we activate the MMU */
30947e07394SAndrew Turner hyp_code_base = vmm_base = vtophys(&vmm_hyp_code);
31047e07394SAndrew Turner rv = vmmpmap_enter(vmm_base, hyp_code_len, vmm_base,
31147e07394SAndrew Turner VM_PROT_READ | VM_PROT_EXECUTE);
31247e07394SAndrew Turner MPASS(rv);
31347e07394SAndrew Turner
31447e07394SAndrew Turner next_hyp_va = roundup2(vmm_base + hyp_code_len, L2_SIZE);
31547e07394SAndrew Turner
31647e07394SAndrew Turner /* Create a per-CPU hypervisor stack */
31747e07394SAndrew Turner CPU_FOREACH(cpu) {
31847e07394SAndrew Turner stack[cpu] = malloc(VMM_STACK_SIZE, M_HYP, M_WAITOK | M_ZERO);
31947e07394SAndrew Turner stack_hyp_va[cpu] = next_hyp_va;
32047e07394SAndrew Turner
32147e07394SAndrew Turner for (i = 0; i < VMM_STACK_PAGES; i++) {
32247e07394SAndrew Turner rv = vmmpmap_enter(stack_hyp_va[cpu] + ptoa(i),
32347e07394SAndrew Turner PAGE_SIZE, vtophys(stack[cpu] + ptoa(i)),
32447e07394SAndrew Turner VM_PROT_READ | VM_PROT_WRITE);
32547e07394SAndrew Turner MPASS(rv);
32647e07394SAndrew Turner }
32747e07394SAndrew Turner next_hyp_va += L2_SIZE;
32847e07394SAndrew Turner }
32947e07394SAndrew Turner
33047e07394SAndrew Turner el2_regs.tcr_el2 = TCR_EL2_RES1;
33147e07394SAndrew Turner el2_regs.tcr_el2 |= min(pa_range_bits << TCR_EL2_PS_SHIFT,
33247e07394SAndrew Turner TCR_EL2_PS_52BITS);
33347e07394SAndrew Turner el2_regs.tcr_el2 |= TCR_EL2_T0SZ(64 - EL2_VIRT_BITS);
33447e07394SAndrew Turner el2_regs.tcr_el2 |= TCR_EL2_IRGN0_WBWA | TCR_EL2_ORGN0_WBWA;
33547e07394SAndrew Turner #if PAGE_SIZE == PAGE_SIZE_4K
33647e07394SAndrew Turner el2_regs.tcr_el2 |= TCR_EL2_TG0_4K;
33747e07394SAndrew Turner #elif PAGE_SIZE == PAGE_SIZE_16K
33847e07394SAndrew Turner el2_regs.tcr_el2 |= TCR_EL2_TG0_16K;
33947e07394SAndrew Turner #else
34047e07394SAndrew Turner #error Unsupported page size
34147e07394SAndrew Turner #endif
34247e07394SAndrew Turner #ifdef SMP
34347e07394SAndrew Turner el2_regs.tcr_el2 |= TCR_EL2_SH0_IS;
34447e07394SAndrew Turner #endif
34547e07394SAndrew Turner
34647e07394SAndrew Turner switch (el2_regs.tcr_el2 & TCR_EL2_PS_MASK) {
34747e07394SAndrew Turner case TCR_EL2_PS_32BITS:
34847e07394SAndrew Turner vmm_max_ipa_bits = 32;
34947e07394SAndrew Turner break;
35047e07394SAndrew Turner case TCR_EL2_PS_36BITS:
35147e07394SAndrew Turner vmm_max_ipa_bits = 36;
35247e07394SAndrew Turner break;
35347e07394SAndrew Turner case TCR_EL2_PS_40BITS:
35447e07394SAndrew Turner vmm_max_ipa_bits = 40;
35547e07394SAndrew Turner break;
35647e07394SAndrew Turner case TCR_EL2_PS_42BITS:
35747e07394SAndrew Turner vmm_max_ipa_bits = 42;
35847e07394SAndrew Turner break;
35947e07394SAndrew Turner case TCR_EL2_PS_44BITS:
36047e07394SAndrew Turner vmm_max_ipa_bits = 44;
36147e07394SAndrew Turner break;
36247e07394SAndrew Turner case TCR_EL2_PS_48BITS:
36347e07394SAndrew Turner vmm_max_ipa_bits = 48;
36447e07394SAndrew Turner break;
36547e07394SAndrew Turner case TCR_EL2_PS_52BITS:
36647e07394SAndrew Turner default:
36747e07394SAndrew Turner vmm_max_ipa_bits = 52;
36847e07394SAndrew Turner break;
36947e07394SAndrew Turner }
37047e07394SAndrew Turner
37147e07394SAndrew Turner /*
37247e07394SAndrew Turner * Configure the Stage 2 translation control register:
37347e07394SAndrew Turner *
37447e07394SAndrew Turner * VTCR_IRGN0_WBWA: Translation table walks access inner cacheable
37547e07394SAndrew Turner * normal memory
37647e07394SAndrew Turner * VTCR_ORGN0_WBWA: Translation table walks access outer cacheable
37747e07394SAndrew Turner * normal memory
37847e07394SAndrew Turner * VTCR_EL2_TG0_4K/16K: Stage 2 uses the same page size as the kernel
37947e07394SAndrew Turner * VTCR_EL2_SL0_4K_LVL1: Stage 2 uses concatenated level 1 tables
38047e07394SAndrew Turner * VTCR_EL2_SH0_IS: Memory associated with Stage 2 walks is inner
38147e07394SAndrew Turner * shareable
38247e07394SAndrew Turner */
38347e07394SAndrew Turner el2_regs.vtcr_el2 = VTCR_EL2_RES1;
38447e07394SAndrew Turner el2_regs.vtcr_el2 |=
38547e07394SAndrew Turner min(pa_range_bits << VTCR_EL2_PS_SHIFT, VTCR_EL2_PS_48BIT);
38647e07394SAndrew Turner el2_regs.vtcr_el2 |= VTCR_EL2_IRGN0_WBWA | VTCR_EL2_ORGN0_WBWA;
38747e07394SAndrew Turner el2_regs.vtcr_el2 |= VTCR_EL2_T0SZ(64 - vmm_virt_bits);
38847e07394SAndrew Turner el2_regs.vtcr_el2 |= vmm_vtcr_el2_sl(vmm_pmap_levels);
38947e07394SAndrew Turner #if PAGE_SIZE == PAGE_SIZE_4K
39047e07394SAndrew Turner el2_regs.vtcr_el2 |= VTCR_EL2_TG0_4K;
39147e07394SAndrew Turner #elif PAGE_SIZE == PAGE_SIZE_16K
39247e07394SAndrew Turner el2_regs.vtcr_el2 |= VTCR_EL2_TG0_16K;
39347e07394SAndrew Turner #else
39447e07394SAndrew Turner #error Unsupported page size
39547e07394SAndrew Turner #endif
39647e07394SAndrew Turner #ifdef SMP
39747e07394SAndrew Turner el2_regs.vtcr_el2 |= VTCR_EL2_SH0_IS;
39847e07394SAndrew Turner #endif
39947e07394SAndrew Turner
40047e07394SAndrew Turner smp_rendezvous(NULL, arm_setup_vectors, NULL, &el2_regs);
40147e07394SAndrew Turner
40247e07394SAndrew Turner /* Add memory to the vmem allocator (checking there is space) */
40347e07394SAndrew Turner if (vmm_base > (L2_SIZE + PAGE_SIZE)) {
40447e07394SAndrew Turner /*
40547e07394SAndrew Turner * Ensure there is an L2 block before the vmm code to check
40647e07394SAndrew Turner * for buffer overflows on earlier data. Include the PAGE_SIZE
40747e07394SAndrew Turner * of the minimum we can allocate.
40847e07394SAndrew Turner */
40947e07394SAndrew Turner vmm_base -= L2_SIZE + PAGE_SIZE;
41047e07394SAndrew Turner vmm_base = rounddown2(vmm_base, L2_SIZE);
41147e07394SAndrew Turner
41247e07394SAndrew Turner /*
41347e07394SAndrew Turner * Check there is memory before the vmm code to add.
41447e07394SAndrew Turner *
41547e07394SAndrew Turner * Reserve the L2 block at address 0 so NULL dereference will
41647e07394SAndrew Turner * raise an exception.
41747e07394SAndrew Turner */
41847e07394SAndrew Turner if (vmm_base > L2_SIZE)
41947e07394SAndrew Turner vmem_add(el2_mem_alloc, L2_SIZE, vmm_base - L2_SIZE,
42047e07394SAndrew Turner M_WAITOK);
42147e07394SAndrew Turner }
42247e07394SAndrew Turner
42347e07394SAndrew Turner /*
42447e07394SAndrew Turner * Add the memory after the stacks. There is most of an L2 block
42547e07394SAndrew Turner * between the last stack and the first allocation so this should
42647e07394SAndrew Turner * be safe without adding more padding.
42747e07394SAndrew Turner */
42847e07394SAndrew Turner if (next_hyp_va < HYP_VM_MAX_ADDRESS - PAGE_SIZE)
42947e07394SAndrew Turner vmem_add(el2_mem_alloc, next_hyp_va,
43047e07394SAndrew Turner HYP_VM_MAX_ADDRESS - next_hyp_va, M_WAITOK);
43147e07394SAndrew Turner
43247e07394SAndrew Turner daif = intr_disable();
43347e07394SAndrew Turner cnthctl_el2 = vmm_call_hyp(HYP_READ_REGISTER, HYP_REG_CNTHCTL);
43447e07394SAndrew Turner intr_restore(daif);
43547e07394SAndrew Turner
43647e07394SAndrew Turner vgic_init();
43747e07394SAndrew Turner vtimer_init(cnthctl_el2);
43847e07394SAndrew Turner
43947e07394SAndrew Turner return (0);
44047e07394SAndrew Turner }
44147e07394SAndrew Turner
44247e07394SAndrew Turner int
vmmops_modcleanup(void)44347e07394SAndrew Turner vmmops_modcleanup(void)
44447e07394SAndrew Turner {
44547e07394SAndrew Turner int cpu;
44647e07394SAndrew Turner
44747e07394SAndrew Turner smp_rendezvous(NULL, arm_teardown_vectors, NULL, NULL);
44847e07394SAndrew Turner
44947e07394SAndrew Turner CPU_FOREACH(cpu) {
45047e07394SAndrew Turner vmmpmap_remove(stack_hyp_va[cpu], VMM_STACK_PAGES * PAGE_SIZE,
45147e07394SAndrew Turner false);
45247e07394SAndrew Turner }
45347e07394SAndrew Turner
45447e07394SAndrew Turner vmmpmap_remove(hyp_code_base, hyp_code_len, false);
45547e07394SAndrew Turner
45647e07394SAndrew Turner vtimer_cleanup();
45747e07394SAndrew Turner
45847e07394SAndrew Turner vmmpmap_fini();
45947e07394SAndrew Turner
46047e07394SAndrew Turner CPU_FOREACH(cpu)
46147e07394SAndrew Turner free(stack[cpu], M_HYP);
46247e07394SAndrew Turner
46347e07394SAndrew Turner pmap_clean_stage2_tlbi = NULL;
46447e07394SAndrew Turner pmap_stage2_invalidate_range = NULL;
46547e07394SAndrew Turner pmap_stage2_invalidate_all = NULL;
46647e07394SAndrew Turner
46747e07394SAndrew Turner return (0);
46847e07394SAndrew Turner }
46947e07394SAndrew Turner
47047e07394SAndrew Turner static vm_size_t
el2_hyp_size(struct vm * vm)47147e07394SAndrew Turner el2_hyp_size(struct vm *vm)
47247e07394SAndrew Turner {
47347e07394SAndrew Turner return (round_page(sizeof(struct hyp) +
47447e07394SAndrew Turner sizeof(struct hypctx *) * vm_get_maxcpus(vm)));
47547e07394SAndrew Turner }
47647e07394SAndrew Turner
47747e07394SAndrew Turner static vm_size_t
el2_hypctx_size(void)47847e07394SAndrew Turner el2_hypctx_size(void)
47947e07394SAndrew Turner {
48047e07394SAndrew Turner return (round_page(sizeof(struct hypctx)));
48147e07394SAndrew Turner }
48247e07394SAndrew Turner
48347e07394SAndrew Turner static vm_offset_t
el2_map_enter(vm_offset_t data,vm_size_t size,vm_prot_t prot)48447e07394SAndrew Turner el2_map_enter(vm_offset_t data, vm_size_t size, vm_prot_t prot)
48547e07394SAndrew Turner {
48647e07394SAndrew Turner vmem_addr_t addr;
48747e07394SAndrew Turner int err __diagused;
48847e07394SAndrew Turner bool rv __diagused;
48947e07394SAndrew Turner
49047e07394SAndrew Turner err = vmem_alloc(el2_mem_alloc, size, M_NEXTFIT | M_WAITOK, &addr);
49147e07394SAndrew Turner MPASS(err == 0);
49247e07394SAndrew Turner rv = vmmpmap_enter(addr, size, vtophys(data), prot);
49347e07394SAndrew Turner MPASS(rv);
49447e07394SAndrew Turner
49547e07394SAndrew Turner return (addr);
49647e07394SAndrew Turner }
49747e07394SAndrew Turner
49847e07394SAndrew Turner void *
vmmops_init(struct vm * vm,pmap_t pmap)49947e07394SAndrew Turner vmmops_init(struct vm *vm, pmap_t pmap)
50047e07394SAndrew Turner {
50147e07394SAndrew Turner struct hyp *hyp;
50247e07394SAndrew Turner vm_size_t size;
50347e07394SAndrew Turner
50447e07394SAndrew Turner size = el2_hyp_size(vm);
50547e07394SAndrew Turner hyp = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO);
50647e07394SAndrew Turner
50747e07394SAndrew Turner hyp->vm = vm;
50847e07394SAndrew Turner hyp->vgic_attached = false;
50947e07394SAndrew Turner
51047e07394SAndrew Turner vtimer_vminit(hyp);
51147e07394SAndrew Turner vgic_vminit(hyp);
51247e07394SAndrew Turner
51347e07394SAndrew Turner hyp->el2_addr = el2_map_enter((vm_offset_t)hyp, size,
51447e07394SAndrew Turner VM_PROT_READ | VM_PROT_WRITE);
51547e07394SAndrew Turner
51647e07394SAndrew Turner return (hyp);
51747e07394SAndrew Turner }
51847e07394SAndrew Turner
51947e07394SAndrew Turner void *
vmmops_vcpu_init(void * vmi,struct vcpu * vcpu1,int vcpuid)52047e07394SAndrew Turner vmmops_vcpu_init(void *vmi, struct vcpu *vcpu1, int vcpuid)
52147e07394SAndrew Turner {
52247e07394SAndrew Turner struct hyp *hyp = vmi;
52347e07394SAndrew Turner struct hypctx *hypctx;
52447e07394SAndrew Turner vm_size_t size;
52547e07394SAndrew Turner
52647e07394SAndrew Turner size = el2_hypctx_size();
52747e07394SAndrew Turner hypctx = malloc_aligned(size, PAGE_SIZE, M_HYP, M_WAITOK | M_ZERO);
52847e07394SAndrew Turner
52947e07394SAndrew Turner KASSERT(vcpuid >= 0 && vcpuid < vm_get_maxcpus(hyp->vm),
53047e07394SAndrew Turner ("%s: Invalid vcpuid %d", __func__, vcpuid));
53147e07394SAndrew Turner hyp->ctx[vcpuid] = hypctx;
53247e07394SAndrew Turner
53347e07394SAndrew Turner hypctx->hyp = hyp;
53447e07394SAndrew Turner hypctx->vcpu = vcpu1;
53547e07394SAndrew Turner
53647e07394SAndrew Turner reset_vm_el01_regs(hypctx);
53747e07394SAndrew Turner reset_vm_el2_regs(hypctx);
53847e07394SAndrew Turner
53947e07394SAndrew Turner vtimer_cpuinit(hypctx);
54047e07394SAndrew Turner vgic_cpuinit(hypctx);
54147e07394SAndrew Turner
54247e07394SAndrew Turner hypctx->el2_addr = el2_map_enter((vm_offset_t)hypctx, size,
54347e07394SAndrew Turner VM_PROT_READ | VM_PROT_WRITE);
54447e07394SAndrew Turner
54547e07394SAndrew Turner return (hypctx);
54647e07394SAndrew Turner }
54747e07394SAndrew Turner
54847e07394SAndrew Turner static int
arm_vmm_pinit(pmap_t pmap)54947e07394SAndrew Turner arm_vmm_pinit(pmap_t pmap)
55047e07394SAndrew Turner {
55147e07394SAndrew Turner
55247e07394SAndrew Turner pmap_pinit_stage(pmap, PM_STAGE2, vmm_pmap_levels);
55347e07394SAndrew Turner return (1);
55447e07394SAndrew Turner }
55547e07394SAndrew Turner
55647e07394SAndrew Turner struct vmspace *
vmmops_vmspace_alloc(vm_offset_t min,vm_offset_t max)55747e07394SAndrew Turner vmmops_vmspace_alloc(vm_offset_t min, vm_offset_t max)
55847e07394SAndrew Turner {
55947e07394SAndrew Turner return (vmspace_alloc(min, max, arm_vmm_pinit));
56047e07394SAndrew Turner }
56147e07394SAndrew Turner
56247e07394SAndrew Turner void
vmmops_vmspace_free(struct vmspace * vmspace)56347e07394SAndrew Turner vmmops_vmspace_free(struct vmspace *vmspace)
56447e07394SAndrew Turner {
56547e07394SAndrew Turner
56647e07394SAndrew Turner pmap_remove_pages(vmspace_pmap(vmspace));
56747e07394SAndrew Turner vmspace_free(vmspace);
56847e07394SAndrew Turner }
56947e07394SAndrew Turner
57047e07394SAndrew Turner static void
vmm_pmap_clean_stage2_tlbi(void)57147e07394SAndrew Turner vmm_pmap_clean_stage2_tlbi(void)
57247e07394SAndrew Turner {
57347e07394SAndrew Turner vmm_call_hyp(HYP_CLEAN_S2_TLBI);
57447e07394SAndrew Turner }
57547e07394SAndrew Turner
57647e07394SAndrew Turner static void
vmm_pmap_invalidate_range(uint64_t vttbr,vm_offset_t sva,vm_offset_t eva,bool final_only)57747e07394SAndrew Turner vmm_pmap_invalidate_range(uint64_t vttbr, vm_offset_t sva, vm_offset_t eva,
57847e07394SAndrew Turner bool final_only)
57947e07394SAndrew Turner {
58047e07394SAndrew Turner MPASS(eva > sva);
58147e07394SAndrew Turner vmm_call_hyp(HYP_S2_TLBI_RANGE, vttbr, sva, eva, final_only);
58247e07394SAndrew Turner }
58347e07394SAndrew Turner
58447e07394SAndrew Turner static void
vmm_pmap_invalidate_all(uint64_t vttbr)58547e07394SAndrew Turner vmm_pmap_invalidate_all(uint64_t vttbr)
58647e07394SAndrew Turner {
58747e07394SAndrew Turner vmm_call_hyp(HYP_S2_TLBI_ALL, vttbr);
58847e07394SAndrew Turner }
58947e07394SAndrew Turner
59047e07394SAndrew Turner static inline void
arm64_print_hyp_regs(struct vm_exit * vme)59147e07394SAndrew Turner arm64_print_hyp_regs(struct vm_exit *vme)
59247e07394SAndrew Turner {
59347e07394SAndrew Turner printf("esr_el2: 0x%016lx\n", vme->u.hyp.esr_el2);
59447e07394SAndrew Turner printf("far_el2: 0x%016lx\n", vme->u.hyp.far_el2);
59547e07394SAndrew Turner printf("hpfar_el2: 0x%016lx\n", vme->u.hyp.hpfar_el2);
59647e07394SAndrew Turner printf("elr_el2: 0x%016lx\n", vme->pc);
59747e07394SAndrew Turner }
59847e07394SAndrew Turner
59947e07394SAndrew Turner static void
arm64_gen_inst_emul_data(struct hypctx * hypctx,uint32_t esr_iss,struct vm_exit * vme_ret)60047e07394SAndrew Turner arm64_gen_inst_emul_data(struct hypctx *hypctx, uint32_t esr_iss,
60147e07394SAndrew Turner struct vm_exit *vme_ret)
60247e07394SAndrew Turner {
60347e07394SAndrew Turner struct vm_guest_paging *paging;
60447e07394SAndrew Turner struct vie *vie;
60547e07394SAndrew Turner uint32_t esr_sas, reg_num;
60647e07394SAndrew Turner
60747e07394SAndrew Turner /*
60847e07394SAndrew Turner * Get the page address from HPFAR_EL2.
60947e07394SAndrew Turner */
61047e07394SAndrew Turner vme_ret->u.inst_emul.gpa =
61147e07394SAndrew Turner HPFAR_EL2_FIPA_ADDR(hypctx->exit_info.hpfar_el2);
61247e07394SAndrew Turner /* Bits [11:0] are the same as bits [11:0] from the virtual address. */
61347e07394SAndrew Turner vme_ret->u.inst_emul.gpa += hypctx->exit_info.far_el2 &
61447e07394SAndrew Turner FAR_EL2_HPFAR_PAGE_MASK;
61547e07394SAndrew Turner
61647e07394SAndrew Turner esr_sas = (esr_iss & ISS_DATA_SAS_MASK) >> ISS_DATA_SAS_SHIFT;
61747e07394SAndrew Turner reg_num = (esr_iss & ISS_DATA_SRT_MASK) >> ISS_DATA_SRT_SHIFT;
61847e07394SAndrew Turner
61947e07394SAndrew Turner vie = &vme_ret->u.inst_emul.vie;
62047e07394SAndrew Turner vie->access_size = 1 << esr_sas;
62147e07394SAndrew Turner vie->sign_extend = (esr_iss & ISS_DATA_SSE) ? 1 : 0;
62247e07394SAndrew Turner vie->dir = (esr_iss & ISS_DATA_WnR) ? VM_DIR_WRITE : VM_DIR_READ;
62347e07394SAndrew Turner vie->reg = reg_num;
62447e07394SAndrew Turner
62547e07394SAndrew Turner paging = &vme_ret->u.inst_emul.paging;
62647e07394SAndrew Turner paging->ttbr0_addr = hypctx->ttbr0_el1 & ~(TTBR_ASID_MASK | TTBR_CnP);
62747e07394SAndrew Turner paging->ttbr1_addr = hypctx->ttbr1_el1 & ~(TTBR_ASID_MASK | TTBR_CnP);
62847e07394SAndrew Turner paging->tcr_el1 = hypctx->tcr_el1;
62947e07394SAndrew Turner paging->tcr2_el1 = hypctx->tcr2_el1;
63047e07394SAndrew Turner paging->flags = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32);
63147e07394SAndrew Turner if ((hypctx->sctlr_el1 & SCTLR_M) != 0)
63247e07394SAndrew Turner paging->flags |= VM_GP_MMU_ENABLED;
63347e07394SAndrew Turner }
63447e07394SAndrew Turner
63547e07394SAndrew Turner static void
arm64_gen_reg_emul_data(uint32_t esr_iss,struct vm_exit * vme_ret)63647e07394SAndrew Turner arm64_gen_reg_emul_data(uint32_t esr_iss, struct vm_exit *vme_ret)
63747e07394SAndrew Turner {
63847e07394SAndrew Turner uint32_t reg_num;
63947e07394SAndrew Turner struct vre *vre;
64047e07394SAndrew Turner
64147e07394SAndrew Turner /* u.hyp member will be replaced by u.reg_emul */
64247e07394SAndrew Turner vre = &vme_ret->u.reg_emul.vre;
64347e07394SAndrew Turner
64447e07394SAndrew Turner vre->inst_syndrome = esr_iss;
64547e07394SAndrew Turner /* ARMv8 Architecture Manual, p. D7-2273: 1 means read */
64647e07394SAndrew Turner vre->dir = (esr_iss & ISS_MSR_DIR) ? VM_DIR_READ : VM_DIR_WRITE;
64747e07394SAndrew Turner reg_num = ISS_MSR_Rt(esr_iss);
64847e07394SAndrew Turner vre->reg = reg_num;
64947e07394SAndrew Turner }
65047e07394SAndrew Turner
65147e07394SAndrew Turner void
raise_data_insn_abort(struct hypctx * hypctx,uint64_t far,bool dabort,int fsc)65247e07394SAndrew Turner raise_data_insn_abort(struct hypctx *hypctx, uint64_t far, bool dabort, int fsc)
65347e07394SAndrew Turner {
65447e07394SAndrew Turner uint64_t esr;
65547e07394SAndrew Turner
65647e07394SAndrew Turner if ((hypctx->tf.tf_spsr & PSR_M_MASK) == PSR_M_EL0t)
65747e07394SAndrew Turner esr = EXCP_INSN_ABORT_L << ESR_ELx_EC_SHIFT;
65847e07394SAndrew Turner else
65947e07394SAndrew Turner esr = EXCP_INSN_ABORT << ESR_ELx_EC_SHIFT;
66047e07394SAndrew Turner /* Set the bit that changes from insn -> data abort */
66147e07394SAndrew Turner if (dabort)
66247e07394SAndrew Turner esr |= EXCP_DATA_ABORT_L << ESR_ELx_EC_SHIFT;
66347e07394SAndrew Turner /* Set the IL bit if set by hardware */
66447e07394SAndrew Turner esr |= hypctx->tf.tf_esr & ESR_ELx_IL;
66547e07394SAndrew Turner
66647e07394SAndrew Turner vmmops_exception(hypctx, esr | fsc, far);
66747e07394SAndrew Turner }
66847e07394SAndrew Turner
66947e07394SAndrew Turner static int
handle_el1_sync_excp(struct hypctx * hypctx,struct vm_exit * vme_ret,pmap_t pmap)67047e07394SAndrew Turner handle_el1_sync_excp(struct hypctx *hypctx, struct vm_exit *vme_ret,
67147e07394SAndrew Turner pmap_t pmap)
67247e07394SAndrew Turner {
67347e07394SAndrew Turner uint64_t gpa;
67447e07394SAndrew Turner uint32_t esr_ec, esr_iss;
67547e07394SAndrew Turner
67647e07394SAndrew Turner esr_ec = ESR_ELx_EXCEPTION(hypctx->tf.tf_esr);
67747e07394SAndrew Turner esr_iss = hypctx->tf.tf_esr & ESR_ELx_ISS_MASK;
67847e07394SAndrew Turner
67947e07394SAndrew Turner switch (esr_ec) {
68047e07394SAndrew Turner case EXCP_UNKNOWN:
68147e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_UNKNOWN, 1);
68247e07394SAndrew Turner arm64_print_hyp_regs(vme_ret);
68347e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_HYP;
68447e07394SAndrew Turner break;
68547e07394SAndrew Turner case EXCP_TRAP_WFI_WFE:
68647e07394SAndrew Turner if ((hypctx->tf.tf_esr & 0x3) == 0) { /* WFI */
68747e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_WFI, 1);
68847e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_WFI;
68947e07394SAndrew Turner } else {
69047e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_WFE, 1);
69147e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_HYP;
69247e07394SAndrew Turner }
69347e07394SAndrew Turner break;
69447e07394SAndrew Turner case EXCP_HVC:
69547e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_HVC, 1);
69647e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_HVC;
69747e07394SAndrew Turner break;
69847e07394SAndrew Turner case EXCP_MSR:
69947e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_MSR, 1);
70047e07394SAndrew Turner arm64_gen_reg_emul_data(esr_iss, vme_ret);
70147e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_REG_EMUL;
70247e07394SAndrew Turner break;
70347e07394SAndrew Turner
70447e07394SAndrew Turner case EXCP_INSN_ABORT_L:
70547e07394SAndrew Turner case EXCP_DATA_ABORT_L:
70647e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, esr_ec == EXCP_DATA_ABORT_L ?
70747e07394SAndrew Turner VMEXIT_DATA_ABORT : VMEXIT_INSN_ABORT, 1);
70847e07394SAndrew Turner switch (hypctx->tf.tf_esr & ISS_DATA_DFSC_MASK) {
70947e07394SAndrew Turner case ISS_DATA_DFSC_TF_L0:
71047e07394SAndrew Turner case ISS_DATA_DFSC_TF_L1:
71147e07394SAndrew Turner case ISS_DATA_DFSC_TF_L2:
71247e07394SAndrew Turner case ISS_DATA_DFSC_TF_L3:
71347e07394SAndrew Turner case ISS_DATA_DFSC_AFF_L1:
71447e07394SAndrew Turner case ISS_DATA_DFSC_AFF_L2:
71547e07394SAndrew Turner case ISS_DATA_DFSC_AFF_L3:
71647e07394SAndrew Turner case ISS_DATA_DFSC_PF_L1:
71747e07394SAndrew Turner case ISS_DATA_DFSC_PF_L2:
71847e07394SAndrew Turner case ISS_DATA_DFSC_PF_L3:
71947e07394SAndrew Turner gpa = HPFAR_EL2_FIPA_ADDR(hypctx->exit_info.hpfar_el2);
72047e07394SAndrew Turner /* Check the IPA is valid */
72147e07394SAndrew Turner if (gpa >= (1ul << vmm_max_ipa_bits)) {
72247e07394SAndrew Turner raise_data_insn_abort(hypctx,
72347e07394SAndrew Turner hypctx->exit_info.far_el2,
72447e07394SAndrew Turner esr_ec == EXCP_DATA_ABORT_L,
72547e07394SAndrew Turner ISS_DATA_DFSC_ASF_L0);
72647e07394SAndrew Turner vme_ret->inst_length = 0;
72747e07394SAndrew Turner return (HANDLED);
72847e07394SAndrew Turner }
72947e07394SAndrew Turner
73047e07394SAndrew Turner if (vm_mem_allocated(hypctx->vcpu, gpa)) {
73147e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_PAGING;
73247e07394SAndrew Turner vme_ret->inst_length = 0;
73347e07394SAndrew Turner vme_ret->u.paging.esr = hypctx->tf.tf_esr;
73447e07394SAndrew Turner vme_ret->u.paging.gpa = gpa;
73547e07394SAndrew Turner } else if (esr_ec == EXCP_INSN_ABORT_L) {
73647e07394SAndrew Turner /*
73747e07394SAndrew Turner * Raise an external abort. Device memory is
73847e07394SAndrew Turner * not executable
73947e07394SAndrew Turner */
74047e07394SAndrew Turner raise_data_insn_abort(hypctx,
74147e07394SAndrew Turner hypctx->exit_info.far_el2, false,
74247e07394SAndrew Turner ISS_DATA_DFSC_EXT);
74347e07394SAndrew Turner vme_ret->inst_length = 0;
74447e07394SAndrew Turner return (HANDLED);
74547e07394SAndrew Turner } else {
74647e07394SAndrew Turner arm64_gen_inst_emul_data(hypctx, esr_iss,
74747e07394SAndrew Turner vme_ret);
74847e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_INST_EMUL;
74947e07394SAndrew Turner }
75047e07394SAndrew Turner break;
75147e07394SAndrew Turner default:
75247e07394SAndrew Turner arm64_print_hyp_regs(vme_ret);
75347e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_HYP;
75447e07394SAndrew Turner break;
75547e07394SAndrew Turner }
75647e07394SAndrew Turner
75747e07394SAndrew Turner break;
75847e07394SAndrew Turner
75947e07394SAndrew Turner default:
76047e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED_SYNC, 1);
76147e07394SAndrew Turner arm64_print_hyp_regs(vme_ret);
76247e07394SAndrew Turner vme_ret->exitcode = VM_EXITCODE_HYP;
76347e07394SAndrew Turner break;
76447e07394SAndrew Turner }
76547e07394SAndrew Turner
76647e07394SAndrew Turner /* We don't don't do any instruction emulation here */
76747e07394SAndrew Turner return (UNHANDLED);
76847e07394SAndrew Turner }
76947e07394SAndrew Turner
77047e07394SAndrew Turner static int
arm64_handle_world_switch(struct hypctx * hypctx,int excp_type,struct vm_exit * vme,pmap_t pmap)77147e07394SAndrew Turner arm64_handle_world_switch(struct hypctx *hypctx, int excp_type,
77247e07394SAndrew Turner struct vm_exit *vme, pmap_t pmap)
77347e07394SAndrew Turner {
77447e07394SAndrew Turner int handled;
77547e07394SAndrew Turner
77647e07394SAndrew Turner switch (excp_type) {
77747e07394SAndrew Turner case EXCP_TYPE_EL1_SYNC:
77847e07394SAndrew Turner /* The exit code will be set by handle_el1_sync_excp(). */
77947e07394SAndrew Turner handled = handle_el1_sync_excp(hypctx, vme, pmap);
78047e07394SAndrew Turner break;
78147e07394SAndrew Turner
78247e07394SAndrew Turner case EXCP_TYPE_EL1_IRQ:
78347e07394SAndrew Turner case EXCP_TYPE_EL1_FIQ:
78447e07394SAndrew Turner /* The host kernel will handle IRQs and FIQs. */
78547e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu,
78647e07394SAndrew Turner excp_type == EXCP_TYPE_EL1_IRQ ? VMEXIT_IRQ : VMEXIT_FIQ,1);
78747e07394SAndrew Turner vme->exitcode = VM_EXITCODE_BOGUS;
78847e07394SAndrew Turner handled = UNHANDLED;
78947e07394SAndrew Turner break;
79047e07394SAndrew Turner
79147e07394SAndrew Turner case EXCP_TYPE_EL1_ERROR:
79247e07394SAndrew Turner case EXCP_TYPE_EL2_SYNC:
79347e07394SAndrew Turner case EXCP_TYPE_EL2_IRQ:
79447e07394SAndrew Turner case EXCP_TYPE_EL2_FIQ:
79547e07394SAndrew Turner case EXCP_TYPE_EL2_ERROR:
79647e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED_EL2, 1);
79747e07394SAndrew Turner vme->exitcode = VM_EXITCODE_BOGUS;
79847e07394SAndrew Turner handled = UNHANDLED;
79947e07394SAndrew Turner break;
80047e07394SAndrew Turner
80147e07394SAndrew Turner default:
80247e07394SAndrew Turner vmm_stat_incr(hypctx->vcpu, VMEXIT_UNHANDLED, 1);
80347e07394SAndrew Turner vme->exitcode = VM_EXITCODE_BOGUS;
80447e07394SAndrew Turner handled = UNHANDLED;
80547e07394SAndrew Turner break;
80647e07394SAndrew Turner }
80747e07394SAndrew Turner
80847e07394SAndrew Turner return (handled);
80947e07394SAndrew Turner }
81047e07394SAndrew Turner
81147e07394SAndrew Turner static void
ptp_release(void ** cookie)81247e07394SAndrew Turner ptp_release(void **cookie)
81347e07394SAndrew Turner {
81447e07394SAndrew Turner if (*cookie != NULL) {
81547e07394SAndrew Turner vm_gpa_release(*cookie);
81647e07394SAndrew Turner *cookie = NULL;
81747e07394SAndrew Turner }
81847e07394SAndrew Turner }
81947e07394SAndrew Turner
82047e07394SAndrew Turner static void *
ptp_hold(struct vcpu * vcpu,vm_paddr_t ptpphys,size_t len,void ** cookie)82147e07394SAndrew Turner ptp_hold(struct vcpu *vcpu, vm_paddr_t ptpphys, size_t len, void **cookie)
82247e07394SAndrew Turner {
82347e07394SAndrew Turner void *ptr;
82447e07394SAndrew Turner
82547e07394SAndrew Turner ptp_release(cookie);
82647e07394SAndrew Turner ptr = vm_gpa_hold(vcpu, ptpphys, len, VM_PROT_RW, cookie);
82747e07394SAndrew Turner return (ptr);
82847e07394SAndrew Turner }
82947e07394SAndrew Turner
83047e07394SAndrew Turner /* log2 of the number of bytes in a page table entry */
83147e07394SAndrew Turner #define PTE_SHIFT 3
83247e07394SAndrew Turner int
vmmops_gla2gpa(void * vcpui,struct vm_guest_paging * paging,uint64_t gla,int prot,uint64_t * gpa,int * is_fault)83347e07394SAndrew Turner vmmops_gla2gpa(void *vcpui, struct vm_guest_paging *paging, uint64_t gla,
83447e07394SAndrew Turner int prot, uint64_t *gpa, int *is_fault)
83547e07394SAndrew Turner {
83647e07394SAndrew Turner struct hypctx *hypctx;
83747e07394SAndrew Turner void *cookie;
83847e07394SAndrew Turner uint64_t mask, *ptep, pte, pte_addr;
83947e07394SAndrew Turner int address_bits, granule_shift, ia_bits, levels, pte_shift, tsz;
84047e07394SAndrew Turner bool is_el0;
84147e07394SAndrew Turner
84247e07394SAndrew Turner /* Check if the MMU is off */
84347e07394SAndrew Turner if ((paging->flags & VM_GP_MMU_ENABLED) == 0) {
84447e07394SAndrew Turner *is_fault = 0;
84547e07394SAndrew Turner *gpa = gla;
84647e07394SAndrew Turner return (0);
84747e07394SAndrew Turner }
84847e07394SAndrew Turner
84947e07394SAndrew Turner is_el0 = (paging->flags & PSR_M_MASK) == PSR_M_EL0t;
85047e07394SAndrew Turner
85147e07394SAndrew Turner if (ADDR_IS_KERNEL(gla)) {
85247e07394SAndrew Turner /* If address translation is disabled raise an exception */
85347e07394SAndrew Turner if ((paging->tcr_el1 & TCR_EPD1) != 0) {
85447e07394SAndrew Turner *is_fault = 1;
85547e07394SAndrew Turner return (0);
85647e07394SAndrew Turner }
85747e07394SAndrew Turner if (is_el0 && (paging->tcr_el1 & TCR_E0PD1) != 0) {
85847e07394SAndrew Turner *is_fault = 1;
85947e07394SAndrew Turner return (0);
86047e07394SAndrew Turner }
86147e07394SAndrew Turner pte_addr = paging->ttbr1_addr;
86247e07394SAndrew Turner tsz = (paging->tcr_el1 & TCR_T1SZ_MASK) >> TCR_T1SZ_SHIFT;
86347e07394SAndrew Turner /* Clear the top byte if TBI is on */
86447e07394SAndrew Turner if ((paging->tcr_el1 & TCR_TBI1) != 0)
86547e07394SAndrew Turner gla |= (0xfful << 56);
86647e07394SAndrew Turner switch (paging->tcr_el1 & TCR_TG1_MASK) {
86747e07394SAndrew Turner case TCR_TG1_4K:
86847e07394SAndrew Turner granule_shift = PAGE_SHIFT_4K;
86947e07394SAndrew Turner break;
87047e07394SAndrew Turner case TCR_TG1_16K:
87147e07394SAndrew Turner granule_shift = PAGE_SHIFT_16K;
87247e07394SAndrew Turner break;
87347e07394SAndrew Turner case TCR_TG1_64K:
87447e07394SAndrew Turner granule_shift = PAGE_SHIFT_64K;
87547e07394SAndrew Turner break;
87647e07394SAndrew Turner default:
87747e07394SAndrew Turner *is_fault = 1;
87847e07394SAndrew Turner return (EINVAL);
87947e07394SAndrew Turner }
88047e07394SAndrew Turner } else {
88147e07394SAndrew Turner /* If address translation is disabled raise an exception */
88247e07394SAndrew Turner if ((paging->tcr_el1 & TCR_EPD0) != 0) {
88347e07394SAndrew Turner *is_fault = 1;
88447e07394SAndrew Turner return (0);
88547e07394SAndrew Turner }
88647e07394SAndrew Turner if (is_el0 && (paging->tcr_el1 & TCR_E0PD0) != 0) {
88747e07394SAndrew Turner *is_fault = 1;
88847e07394SAndrew Turner return (0);
88947e07394SAndrew Turner }
89047e07394SAndrew Turner pte_addr = paging->ttbr0_addr;
89147e07394SAndrew Turner tsz = (paging->tcr_el1 & TCR_T0SZ_MASK) >> TCR_T0SZ_SHIFT;
89247e07394SAndrew Turner /* Clear the top byte if TBI is on */
89347e07394SAndrew Turner if ((paging->tcr_el1 & TCR_TBI0) != 0)
89447e07394SAndrew Turner gla &= ~(0xfful << 56);
89547e07394SAndrew Turner switch (paging->tcr_el1 & TCR_TG0_MASK) {
89647e07394SAndrew Turner case TCR_TG0_4K:
89747e07394SAndrew Turner granule_shift = PAGE_SHIFT_4K;
89847e07394SAndrew Turner break;
89947e07394SAndrew Turner case TCR_TG0_16K:
90047e07394SAndrew Turner granule_shift = PAGE_SHIFT_16K;
90147e07394SAndrew Turner break;
90247e07394SAndrew Turner case TCR_TG0_64K:
90347e07394SAndrew Turner granule_shift = PAGE_SHIFT_64K;
90447e07394SAndrew Turner break;
90547e07394SAndrew Turner default:
90647e07394SAndrew Turner *is_fault = 1;
90747e07394SAndrew Turner return (EINVAL);
90847e07394SAndrew Turner }
90947e07394SAndrew Turner }
91047e07394SAndrew Turner
91147e07394SAndrew Turner /*
91247e07394SAndrew Turner * TODO: Support FEAT_TTST for smaller tsz values and FEAT_LPA2
91347e07394SAndrew Turner * for larger values.
91447e07394SAndrew Turner */
91547e07394SAndrew Turner switch (granule_shift) {
91647e07394SAndrew Turner case PAGE_SHIFT_4K:
91747e07394SAndrew Turner case PAGE_SHIFT_16K:
91847e07394SAndrew Turner /*
91947e07394SAndrew Turner * See "Table D8-11 4KB granule, determining stage 1 initial
92047e07394SAndrew Turner * lookup level" and "Table D8-21 16KB granule, determining
92147e07394SAndrew Turner * stage 1 initial lookup level" from the "Arm Architecture
92247e07394SAndrew Turner * Reference Manual for A-Profile architecture" revision I.a
92347e07394SAndrew Turner * for the minimum and maximum values.
92447e07394SAndrew Turner *
92547e07394SAndrew Turner * TODO: Support less than 16 when FEAT_LPA2 is implemented
92647e07394SAndrew Turner * and TCR_EL1.DS == 1
92747e07394SAndrew Turner * TODO: Support more than 39 when FEAT_TTST is implemented
92847e07394SAndrew Turner */
92947e07394SAndrew Turner if (tsz < 16 || tsz > 39) {
93047e07394SAndrew Turner *is_fault = 1;
93147e07394SAndrew Turner return (EINVAL);
93247e07394SAndrew Turner }
93347e07394SAndrew Turner break;
93447e07394SAndrew Turner case PAGE_SHIFT_64K:
93547e07394SAndrew Turner /* TODO: Support 64k granule. It will probably work, but is untested */
93647e07394SAndrew Turner default:
93747e07394SAndrew Turner *is_fault = 1;
93847e07394SAndrew Turner return (EINVAL);
93947e07394SAndrew Turner }
94047e07394SAndrew Turner
94147e07394SAndrew Turner /*
94247e07394SAndrew Turner * Calculate the input address bits. These are 64 bit in an address
94347e07394SAndrew Turner * with the top tsz bits being all 0 or all 1.
94447e07394SAndrew Turner */
94547e07394SAndrew Turner ia_bits = 64 - tsz;
94647e07394SAndrew Turner
94747e07394SAndrew Turner /*
94847e07394SAndrew Turner * Calculate the number of address bits used in the page table
94947e07394SAndrew Turner * calculation. This is ia_bits minus the bottom granule_shift
95047e07394SAndrew Turner * bits that are passed to the output address.
95147e07394SAndrew Turner */
95247e07394SAndrew Turner address_bits = ia_bits - granule_shift;
95347e07394SAndrew Turner
95447e07394SAndrew Turner /*
95547e07394SAndrew Turner * Calculate the number of levels. Each level uses
95647e07394SAndrew Turner * granule_shift - PTE_SHIFT bits of the input address.
95747e07394SAndrew Turner * This is because the table is 1 << granule_shift and each
95847e07394SAndrew Turner * entry is 1 << PTE_SHIFT bytes.
95947e07394SAndrew Turner */
96047e07394SAndrew Turner levels = howmany(address_bits, granule_shift - PTE_SHIFT);
96147e07394SAndrew Turner
96247e07394SAndrew Turner /* Mask of the upper unused bits in the virtual address */
96347e07394SAndrew Turner gla &= (1ul << ia_bits) - 1;
96447e07394SAndrew Turner hypctx = (struct hypctx *)vcpui;
96547e07394SAndrew Turner cookie = NULL;
96647e07394SAndrew Turner /* TODO: Check if the level supports block descriptors */
96747e07394SAndrew Turner for (;levels > 0; levels--) {
96847e07394SAndrew Turner int idx;
96947e07394SAndrew Turner
97047e07394SAndrew Turner pte_shift = (levels - 1) * (granule_shift - PTE_SHIFT) +
97147e07394SAndrew Turner granule_shift;
97247e07394SAndrew Turner idx = (gla >> pte_shift) &
97347e07394SAndrew Turner ((1ul << (granule_shift - PTE_SHIFT)) - 1);
97447e07394SAndrew Turner while (idx > PAGE_SIZE / sizeof(pte)) {
97547e07394SAndrew Turner idx -= PAGE_SIZE / sizeof(pte);
97647e07394SAndrew Turner pte_addr += PAGE_SIZE;
97747e07394SAndrew Turner }
97847e07394SAndrew Turner
97947e07394SAndrew Turner ptep = ptp_hold(hypctx->vcpu, pte_addr, PAGE_SIZE, &cookie);
98047e07394SAndrew Turner if (ptep == NULL)
98147e07394SAndrew Turner goto error;
98247e07394SAndrew Turner pte = ptep[idx];
98347e07394SAndrew Turner
98447e07394SAndrew Turner /* Calculate the level we are looking at */
98547e07394SAndrew Turner switch (levels) {
98647e07394SAndrew Turner default:
98747e07394SAndrew Turner goto fault;
98847e07394SAndrew Turner /* TODO: Level -1 when FEAT_LPA2 is implemented */
98947e07394SAndrew Turner case 4: /* Level 0 */
99047e07394SAndrew Turner if ((pte & ATTR_DESCR_MASK) != L0_TABLE)
99147e07394SAndrew Turner goto fault;
99247e07394SAndrew Turner /* FALLTHROUGH */
99347e07394SAndrew Turner case 3: /* Level 1 */
99447e07394SAndrew Turner case 2: /* Level 2 */
99547e07394SAndrew Turner switch (pte & ATTR_DESCR_MASK) {
99647e07394SAndrew Turner /* Use L1 macro as all levels are the same */
99747e07394SAndrew Turner case L1_TABLE:
99847e07394SAndrew Turner /* Check if EL0 can access this address space */
99947e07394SAndrew Turner if (is_el0 &&
100047e07394SAndrew Turner (pte & TATTR_AP_TABLE_NO_EL0) != 0)
100147e07394SAndrew Turner goto fault;
100247e07394SAndrew Turner /* Check if the address space is writable */
100347e07394SAndrew Turner if ((prot & PROT_WRITE) != 0 &&
100447e07394SAndrew Turner (pte & TATTR_AP_TABLE_RO) != 0)
100547e07394SAndrew Turner goto fault;
100647e07394SAndrew Turner if ((prot & PROT_EXEC) != 0) {
100747e07394SAndrew Turner /* Check the table exec attribute */
100847e07394SAndrew Turner if ((is_el0 &&
100947e07394SAndrew Turner (pte & TATTR_UXN_TABLE) != 0) ||
101047e07394SAndrew Turner (!is_el0 &&
101147e07394SAndrew Turner (pte & TATTR_PXN_TABLE) != 0))
101247e07394SAndrew Turner goto fault;
101347e07394SAndrew Turner }
101447e07394SAndrew Turner pte_addr = pte & ~ATTR_MASK;
101547e07394SAndrew Turner break;
101647e07394SAndrew Turner case L1_BLOCK:
101747e07394SAndrew Turner goto done;
101847e07394SAndrew Turner default:
101947e07394SAndrew Turner goto fault;
102047e07394SAndrew Turner }
102147e07394SAndrew Turner break;
102247e07394SAndrew Turner case 1: /* Level 3 */
102347e07394SAndrew Turner if ((pte & ATTR_DESCR_MASK) == L3_PAGE)
102447e07394SAndrew Turner goto done;
102547e07394SAndrew Turner goto fault;
102647e07394SAndrew Turner }
102747e07394SAndrew Turner }
102847e07394SAndrew Turner
102947e07394SAndrew Turner done:
103047e07394SAndrew Turner /* Check if EL0 has access to the block/page */
103147e07394SAndrew Turner if (is_el0 && (pte & ATTR_S1_AP(ATTR_S1_AP_USER)) == 0)
103247e07394SAndrew Turner goto fault;
103347e07394SAndrew Turner if ((prot & PROT_WRITE) != 0 && (pte & ATTR_S1_AP_RW_BIT) != 0)
103447e07394SAndrew Turner goto fault;
103547e07394SAndrew Turner if ((prot & PROT_EXEC) != 0) {
103647e07394SAndrew Turner if ((is_el0 && (pte & ATTR_S1_UXN) != 0) ||
103747e07394SAndrew Turner (!is_el0 && (pte & ATTR_S1_PXN) != 0))
103847e07394SAndrew Turner goto fault;
103947e07394SAndrew Turner }
104047e07394SAndrew Turner mask = (1ul << pte_shift) - 1;
104147e07394SAndrew Turner *gpa = (pte & ~ATTR_MASK) | (gla & mask);
104247e07394SAndrew Turner *is_fault = 0;
104347e07394SAndrew Turner ptp_release(&cookie);
104447e07394SAndrew Turner return (0);
104547e07394SAndrew Turner
104647e07394SAndrew Turner error:
104747e07394SAndrew Turner ptp_release(&cookie);
104847e07394SAndrew Turner return (EFAULT);
104947e07394SAndrew Turner fault:
105047e07394SAndrew Turner *is_fault = 1;
105147e07394SAndrew Turner ptp_release(&cookie);
105247e07394SAndrew Turner return (0);
105347e07394SAndrew Turner }
105447e07394SAndrew Turner
105547e07394SAndrew Turner int
vmmops_run(void * vcpui,register_t pc,pmap_t pmap,struct vm_eventinfo * evinfo)105647e07394SAndrew Turner vmmops_run(void *vcpui, register_t pc, pmap_t pmap, struct vm_eventinfo *evinfo)
105747e07394SAndrew Turner {
105847e07394SAndrew Turner uint64_t excp_type;
105947e07394SAndrew Turner int handled;
106047e07394SAndrew Turner register_t daif;
106147e07394SAndrew Turner struct hyp *hyp;
106247e07394SAndrew Turner struct hypctx *hypctx;
106347e07394SAndrew Turner struct vcpu *vcpu;
106447e07394SAndrew Turner struct vm_exit *vme;
106547e07394SAndrew Turner int mode;
106647e07394SAndrew Turner
106747e07394SAndrew Turner hypctx = (struct hypctx *)vcpui;
106847e07394SAndrew Turner hyp = hypctx->hyp;
106947e07394SAndrew Turner vcpu = hypctx->vcpu;
107047e07394SAndrew Turner vme = vm_exitinfo(vcpu);
107147e07394SAndrew Turner
107247e07394SAndrew Turner hypctx->tf.tf_elr = (uint64_t)pc;
107347e07394SAndrew Turner
107447e07394SAndrew Turner for (;;) {
107547e07394SAndrew Turner if (hypctx->has_exception) {
107647e07394SAndrew Turner hypctx->has_exception = false;
107747e07394SAndrew Turner hypctx->elr_el1 = hypctx->tf.tf_elr;
107847e07394SAndrew Turner
107947e07394SAndrew Turner mode = hypctx->tf.tf_spsr & (PSR_M_MASK | PSR_M_32);
108047e07394SAndrew Turner
108147e07394SAndrew Turner if (mode == PSR_M_EL1t) {
108247e07394SAndrew Turner hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x0;
108347e07394SAndrew Turner } else if (mode == PSR_M_EL1h) {
108447e07394SAndrew Turner hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x200;
108547e07394SAndrew Turner } else if ((mode & PSR_M_32) == PSR_M_64) {
108647e07394SAndrew Turner /* 64-bit EL0 */
108747e07394SAndrew Turner hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x400;
108847e07394SAndrew Turner } else {
108947e07394SAndrew Turner /* 32-bit EL0 */
109047e07394SAndrew Turner hypctx->tf.tf_elr = hypctx->vbar_el1 + 0x600;
109147e07394SAndrew Turner }
109247e07394SAndrew Turner
109347e07394SAndrew Turner /* Set the new spsr */
109447e07394SAndrew Turner hypctx->spsr_el1 = hypctx->tf.tf_spsr;
109547e07394SAndrew Turner
109647e07394SAndrew Turner /* Set the new cpsr */
109747e07394SAndrew Turner hypctx->tf.tf_spsr = hypctx->spsr_el1 & PSR_FLAGS;
109847e07394SAndrew Turner hypctx->tf.tf_spsr |= PSR_DAIF | PSR_M_EL1h;
109947e07394SAndrew Turner
110047e07394SAndrew Turner /*
110147e07394SAndrew Turner * Update fields that may change on exeption entry
110247e07394SAndrew Turner * based on how sctlr_el1 is configured.
110347e07394SAndrew Turner */
110447e07394SAndrew Turner if ((hypctx->sctlr_el1 & SCTLR_SPAN) != 0)
110547e07394SAndrew Turner hypctx->tf.tf_spsr |= PSR_PAN;
110647e07394SAndrew Turner if ((hypctx->sctlr_el1 & SCTLR_DSSBS) == 0)
110747e07394SAndrew Turner hypctx->tf.tf_spsr &= ~PSR_SSBS;
110847e07394SAndrew Turner else
110947e07394SAndrew Turner hypctx->tf.tf_spsr |= PSR_SSBS;
111047e07394SAndrew Turner }
111147e07394SAndrew Turner
111247e07394SAndrew Turner daif = intr_disable();
111347e07394SAndrew Turner
111447e07394SAndrew Turner /* Check if the vcpu is suspended */
111547e07394SAndrew Turner if (vcpu_suspended(evinfo)) {
111647e07394SAndrew Turner intr_restore(daif);
111747e07394SAndrew Turner vm_exit_suspended(vcpu, pc);
111847e07394SAndrew Turner break;
111947e07394SAndrew Turner }
112047e07394SAndrew Turner
112147e07394SAndrew Turner if (vcpu_debugged(vcpu)) {
112247e07394SAndrew Turner intr_restore(daif);
112347e07394SAndrew Turner vm_exit_debug(vcpu, pc);
112447e07394SAndrew Turner break;
112547e07394SAndrew Turner }
112647e07394SAndrew Turner
112747e07394SAndrew Turner /* Activate the stage2 pmap so the vmid is valid */
112847e07394SAndrew Turner pmap_activate_vm(pmap);
112947e07394SAndrew Turner hyp->vttbr_el2 = pmap_to_ttbr0(pmap);
113047e07394SAndrew Turner
113147e07394SAndrew Turner /*
113247e07394SAndrew Turner * TODO: What happens if a timer interrupt is asserted exactly
113347e07394SAndrew Turner * here, but for the previous VM?
113447e07394SAndrew Turner */
113547e07394SAndrew Turner arm64_set_active_vcpu(hypctx);
113647e07394SAndrew Turner vgic_flush_hwstate(hypctx);
113747e07394SAndrew Turner
113847e07394SAndrew Turner /* Call into EL2 to switch to the guest */
113947e07394SAndrew Turner excp_type = vmm_call_hyp(HYP_ENTER_GUEST,
114047e07394SAndrew Turner hyp->el2_addr, hypctx->el2_addr);
114147e07394SAndrew Turner
114247e07394SAndrew Turner vgic_sync_hwstate(hypctx);
114347e07394SAndrew Turner vtimer_sync_hwstate(hypctx);
114447e07394SAndrew Turner
114547e07394SAndrew Turner /*
114647e07394SAndrew Turner * Deactivate the stage2 pmap. vmm_pmap_clean_stage2_tlbi
114747e07394SAndrew Turner * depends on this meaning we activate the VM before entering
114847e07394SAndrew Turner * the vm again
114947e07394SAndrew Turner */
115047e07394SAndrew Turner PCPU_SET(curvmpmap, NULL);
115147e07394SAndrew Turner intr_restore(daif);
115247e07394SAndrew Turner
115347e07394SAndrew Turner vmm_stat_incr(vcpu, VMEXIT_COUNT, 1);
115447e07394SAndrew Turner if (excp_type == EXCP_TYPE_MAINT_IRQ)
115547e07394SAndrew Turner continue;
115647e07394SAndrew Turner
115747e07394SAndrew Turner vme->pc = hypctx->tf.tf_elr;
115847e07394SAndrew Turner vme->inst_length = INSN_SIZE;
115947e07394SAndrew Turner vme->u.hyp.exception_nr = excp_type;
116047e07394SAndrew Turner vme->u.hyp.esr_el2 = hypctx->tf.tf_esr;
116147e07394SAndrew Turner vme->u.hyp.far_el2 = hypctx->exit_info.far_el2;
116247e07394SAndrew Turner vme->u.hyp.hpfar_el2 = hypctx->exit_info.hpfar_el2;
116347e07394SAndrew Turner
116447e07394SAndrew Turner handled = arm64_handle_world_switch(hypctx, excp_type, vme,
116547e07394SAndrew Turner pmap);
116647e07394SAndrew Turner if (handled == UNHANDLED)
116747e07394SAndrew Turner /* Exit loop to emulate instruction. */
116847e07394SAndrew Turner break;
116947e07394SAndrew Turner else
117047e07394SAndrew Turner /* Resume guest execution from the next instruction. */
117147e07394SAndrew Turner hypctx->tf.tf_elr += vme->inst_length;
117247e07394SAndrew Turner }
117347e07394SAndrew Turner
117447e07394SAndrew Turner return (0);
117547e07394SAndrew Turner }
117647e07394SAndrew Turner
117747e07394SAndrew Turner static void
arm_pcpu_vmcleanup(void * arg)117847e07394SAndrew Turner arm_pcpu_vmcleanup(void *arg)
117947e07394SAndrew Turner {
118047e07394SAndrew Turner struct hyp *hyp;
118147e07394SAndrew Turner int i, maxcpus;
118247e07394SAndrew Turner
118347e07394SAndrew Turner hyp = arg;
118447e07394SAndrew Turner maxcpus = vm_get_maxcpus(hyp->vm);
118547e07394SAndrew Turner for (i = 0; i < maxcpus; i++) {
118647e07394SAndrew Turner if (arm64_get_active_vcpu() == hyp->ctx[i]) {
118747e07394SAndrew Turner arm64_set_active_vcpu(NULL);
118847e07394SAndrew Turner break;
118947e07394SAndrew Turner }
119047e07394SAndrew Turner }
119147e07394SAndrew Turner }
119247e07394SAndrew Turner
119347e07394SAndrew Turner void
vmmops_vcpu_cleanup(void * vcpui)119447e07394SAndrew Turner vmmops_vcpu_cleanup(void *vcpui)
119547e07394SAndrew Turner {
119647e07394SAndrew Turner struct hypctx *hypctx = vcpui;
119747e07394SAndrew Turner
119847e07394SAndrew Turner vtimer_cpucleanup(hypctx);
119947e07394SAndrew Turner vgic_cpucleanup(hypctx);
120047e07394SAndrew Turner
120147e07394SAndrew Turner vmmpmap_remove(hypctx->el2_addr, el2_hypctx_size(), true);
120247e07394SAndrew Turner
120347e07394SAndrew Turner free(hypctx, M_HYP);
120447e07394SAndrew Turner }
120547e07394SAndrew Turner
120647e07394SAndrew Turner void
vmmops_cleanup(void * vmi)120747e07394SAndrew Turner vmmops_cleanup(void *vmi)
120847e07394SAndrew Turner {
120947e07394SAndrew Turner struct hyp *hyp = vmi;
121047e07394SAndrew Turner
121147e07394SAndrew Turner vtimer_vmcleanup(hyp);
121247e07394SAndrew Turner vgic_vmcleanup(hyp);
121347e07394SAndrew Turner
121447e07394SAndrew Turner smp_rendezvous(NULL, arm_pcpu_vmcleanup, NULL, hyp);
121547e07394SAndrew Turner
121647e07394SAndrew Turner vmmpmap_remove(hyp->el2_addr, el2_hyp_size(hyp->vm), true);
121747e07394SAndrew Turner
121847e07394SAndrew Turner free(hyp, M_HYP);
121947e07394SAndrew Turner }
122047e07394SAndrew Turner
122147e07394SAndrew Turner /*
122247e07394SAndrew Turner * Return register value. Registers have different sizes and an explicit cast
122347e07394SAndrew Turner * must be made to ensure proper conversion.
122447e07394SAndrew Turner */
122547e07394SAndrew Turner static uint64_t *
hypctx_regptr(struct hypctx * hypctx,int reg)122647e07394SAndrew Turner hypctx_regptr(struct hypctx *hypctx, int reg)
122747e07394SAndrew Turner {
122847e07394SAndrew Turner switch (reg) {
122947e07394SAndrew Turner case VM_REG_GUEST_X0 ... VM_REG_GUEST_X29:
123047e07394SAndrew Turner return (&hypctx->tf.tf_x[reg]);
123147e07394SAndrew Turner case VM_REG_GUEST_LR:
123247e07394SAndrew Turner return (&hypctx->tf.tf_lr);
123347e07394SAndrew Turner case VM_REG_GUEST_SP:
123447e07394SAndrew Turner return (&hypctx->tf.tf_sp);
123547e07394SAndrew Turner case VM_REG_GUEST_CPSR:
123647e07394SAndrew Turner return (&hypctx->tf.tf_spsr);
123747e07394SAndrew Turner case VM_REG_GUEST_PC:
123847e07394SAndrew Turner return (&hypctx->tf.tf_elr);
123947e07394SAndrew Turner case VM_REG_GUEST_SCTLR_EL1:
124047e07394SAndrew Turner return (&hypctx->sctlr_el1);
124147e07394SAndrew Turner case VM_REG_GUEST_TTBR0_EL1:
124247e07394SAndrew Turner return (&hypctx->ttbr0_el1);
124347e07394SAndrew Turner case VM_REG_GUEST_TTBR1_EL1:
124447e07394SAndrew Turner return (&hypctx->ttbr1_el1);
124547e07394SAndrew Turner case VM_REG_GUEST_TCR_EL1:
124647e07394SAndrew Turner return (&hypctx->tcr_el1);
124747e07394SAndrew Turner case VM_REG_GUEST_TCR2_EL1:
124847e07394SAndrew Turner return (&hypctx->tcr2_el1);
124947e07394SAndrew Turner default:
125047e07394SAndrew Turner break;
125147e07394SAndrew Turner }
125247e07394SAndrew Turner return (NULL);
125347e07394SAndrew Turner }
125447e07394SAndrew Turner
125547e07394SAndrew Turner int
vmmops_getreg(void * vcpui,int reg,uint64_t * retval)125647e07394SAndrew Turner vmmops_getreg(void *vcpui, int reg, uint64_t *retval)
125747e07394SAndrew Turner {
125847e07394SAndrew Turner uint64_t *regp;
125947e07394SAndrew Turner int running, hostcpu;
126047e07394SAndrew Turner struct hypctx *hypctx = vcpui;
126147e07394SAndrew Turner
126247e07394SAndrew Turner running = vcpu_is_running(hypctx->vcpu, &hostcpu);
126347e07394SAndrew Turner if (running && hostcpu != curcpu)
126447e07394SAndrew Turner panic("arm_getreg: %s%d is running", vm_name(hypctx->hyp->vm),
126547e07394SAndrew Turner vcpu_vcpuid(hypctx->vcpu));
126647e07394SAndrew Turner
126747e07394SAndrew Turner regp = hypctx_regptr(hypctx, reg);
126847e07394SAndrew Turner if (regp == NULL)
126947e07394SAndrew Turner return (EINVAL);
127047e07394SAndrew Turner
127147e07394SAndrew Turner *retval = *regp;
127247e07394SAndrew Turner return (0);
127347e07394SAndrew Turner }
127447e07394SAndrew Turner
127547e07394SAndrew Turner int
vmmops_setreg(void * vcpui,int reg,uint64_t val)127647e07394SAndrew Turner vmmops_setreg(void *vcpui, int reg, uint64_t val)
127747e07394SAndrew Turner {
127847e07394SAndrew Turner uint64_t *regp;
127947e07394SAndrew Turner struct hypctx *hypctx = vcpui;
128047e07394SAndrew Turner int running, hostcpu;
128147e07394SAndrew Turner
128247e07394SAndrew Turner running = vcpu_is_running(hypctx->vcpu, &hostcpu);
128347e07394SAndrew Turner if (running && hostcpu != curcpu)
128447e07394SAndrew Turner panic("arm_setreg: %s%d is running", vm_name(hypctx->hyp->vm),
128547e07394SAndrew Turner vcpu_vcpuid(hypctx->vcpu));
128647e07394SAndrew Turner
128747e07394SAndrew Turner regp = hypctx_regptr(hypctx, reg);
128847e07394SAndrew Turner if (regp == NULL)
128947e07394SAndrew Turner return (EINVAL);
129047e07394SAndrew Turner
129147e07394SAndrew Turner *regp = val;
129247e07394SAndrew Turner return (0);
129347e07394SAndrew Turner }
129447e07394SAndrew Turner
129547e07394SAndrew Turner int
vmmops_exception(void * vcpui,uint64_t esr,uint64_t far)129647e07394SAndrew Turner vmmops_exception(void *vcpui, uint64_t esr, uint64_t far)
129747e07394SAndrew Turner {
129847e07394SAndrew Turner struct hypctx *hypctx = vcpui;
129947e07394SAndrew Turner int running, hostcpu;
130047e07394SAndrew Turner
130147e07394SAndrew Turner running = vcpu_is_running(hypctx->vcpu, &hostcpu);
130247e07394SAndrew Turner if (running && hostcpu != curcpu)
130347e07394SAndrew Turner panic("%s: %s%d is running", __func__, vm_name(hypctx->hyp->vm),
130447e07394SAndrew Turner vcpu_vcpuid(hypctx->vcpu));
130547e07394SAndrew Turner
130647e07394SAndrew Turner hypctx->far_el1 = far;
130747e07394SAndrew Turner hypctx->esr_el1 = esr;
130847e07394SAndrew Turner hypctx->has_exception = true;
130947e07394SAndrew Turner
131047e07394SAndrew Turner return (0);
131147e07394SAndrew Turner }
131247e07394SAndrew Turner
131347e07394SAndrew Turner int
vmmops_getcap(void * vcpui,int num,int * retval)131447e07394SAndrew Turner vmmops_getcap(void *vcpui, int num, int *retval)
131547e07394SAndrew Turner {
131647e07394SAndrew Turner int ret;
131747e07394SAndrew Turner
131847e07394SAndrew Turner ret = ENOENT;
131947e07394SAndrew Turner
132047e07394SAndrew Turner switch (num) {
132147e07394SAndrew Turner case VM_CAP_UNRESTRICTED_GUEST:
132247e07394SAndrew Turner *retval = 1;
132347e07394SAndrew Turner ret = 0;
132447e07394SAndrew Turner break;
132547e07394SAndrew Turner default:
132647e07394SAndrew Turner break;
132747e07394SAndrew Turner }
132847e07394SAndrew Turner
132947e07394SAndrew Turner return (ret);
133047e07394SAndrew Turner }
133147e07394SAndrew Turner
133247e07394SAndrew Turner int
vmmops_setcap(void * vcpui,int num,int val)133347e07394SAndrew Turner vmmops_setcap(void *vcpui, int num, int val)
133447e07394SAndrew Turner {
133547e07394SAndrew Turner
133647e07394SAndrew Turner return (ENOENT);
133747e07394SAndrew Turner }
1338