17a158e82SAndrew Turner /*-
27a158e82SAndrew Turner * Copyright (c) 2004 Marcel Moolenaar
37a158e82SAndrew Turner * Copyright (c) 2001 Doug Rabson
47a158e82SAndrew Turner * Copyright (c) 2016 The FreeBSD Foundation
57a158e82SAndrew Turner * Copyright (c) 2017 Andrew Turner
67a158e82SAndrew Turner * All rights reserved.
77a158e82SAndrew Turner *
87a158e82SAndrew Turner * Portions of this software were developed by Konstantin Belousov
97a158e82SAndrew Turner * under sponsorship from the FreeBSD Foundation.
107a158e82SAndrew Turner *
117a158e82SAndrew Turner * This software was developed by SRI International and the University of
127a158e82SAndrew Turner * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237
137a158e82SAndrew Turner * ("CTSRD"), as part of the DARPA CRASH research programme.
147a158e82SAndrew Turner *
157a158e82SAndrew Turner * Redistribution and use in source and binary forms, with or without
167a158e82SAndrew Turner * modification, are permitted provided that the following conditions
177a158e82SAndrew Turner * are met:
187a158e82SAndrew Turner * 1. Redistributions of source code must retain the above copyright
197a158e82SAndrew Turner * notice, this list of conditions and the following disclaimer.
207a158e82SAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright
217a158e82SAndrew Turner * notice, this list of conditions and the following disclaimer in the
227a158e82SAndrew Turner * documentation and/or other materials provided with the distribution.
237a158e82SAndrew Turner *
247a158e82SAndrew Turner * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
257a158e82SAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
267a158e82SAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
277a158e82SAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
287a158e82SAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
297a158e82SAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
307a158e82SAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
317a158e82SAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
327a158e82SAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
337a158e82SAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
347a158e82SAndrew Turner * SUCH DAMAGE.
357a158e82SAndrew Turner */
367a158e82SAndrew Turner
377a158e82SAndrew Turner #include <sys/param.h>
387a158e82SAndrew Turner #include <sys/efi.h>
397a158e82SAndrew Turner #include <sys/kernel.h>
407a158e82SAndrew Turner #include <sys/linker.h>
417a158e82SAndrew Turner #include <sys/lock.h>
427a158e82SAndrew Turner #include <sys/mutex.h>
437a158e82SAndrew Turner #include <sys/proc.h>
447a158e82SAndrew Turner #include <sys/rwlock.h>
457a158e82SAndrew Turner #include <sys/systm.h>
467a158e82SAndrew Turner #include <sys/vmmeter.h>
477a158e82SAndrew Turner
487a158e82SAndrew Turner #include <machine/pte.h>
497a158e82SAndrew Turner #include <machine/vmparam.h>
507a158e82SAndrew Turner
517a158e82SAndrew Turner #include <vm/vm.h>
527a158e82SAndrew Turner #include <vm/pmap.h>
537a158e82SAndrew Turner #include <vm/vm_map.h>
547a158e82SAndrew Turner #include <vm/vm_object.h>
557a158e82SAndrew Turner #include <vm/vm_page.h>
567a158e82SAndrew Turner #include <vm/vm_pager.h>
577a158e82SAndrew Turner
587a158e82SAndrew Turner static vm_object_t obj_1t1_pt;
59dde56027SKonstantin Belousov static vm_pindex_t efi_1t1_idx;
6050e3ab6bSAlan Cox static pd_entry_t *efi_l0;
6150e3ab6bSAlan Cox static uint64_t efi_ttbr0;
627a158e82SAndrew Turner
637a158e82SAndrew Turner void
efi_destroy_1t1_map(void)647a158e82SAndrew Turner efi_destroy_1t1_map(void)
657a158e82SAndrew Turner {
667a158e82SAndrew Turner vm_page_t m;
677a158e82SAndrew Turner
687a158e82SAndrew Turner if (obj_1t1_pt != NULL) {
697a158e82SAndrew Turner VM_OBJECT_RLOCK(obj_1t1_pt);
707a158e82SAndrew Turner TAILQ_FOREACH(m, &obj_1t1_pt->memq, listq)
71b119329dSMark Johnston m->ref_count = VPRC_OBJREF;
72e958ad4cSJeff Roberson vm_wire_sub(obj_1t1_pt->resident_page_count);
737a158e82SAndrew Turner VM_OBJECT_RUNLOCK(obj_1t1_pt);
747a158e82SAndrew Turner vm_object_deallocate(obj_1t1_pt);
757a158e82SAndrew Turner }
767a158e82SAndrew Turner
777a158e82SAndrew Turner obj_1t1_pt = NULL;
7850e3ab6bSAlan Cox efi_1t1_idx = 0;
797a158e82SAndrew Turner efi_l0 = NULL;
8050e3ab6bSAlan Cox efi_ttbr0 = 0;
817a158e82SAndrew Turner }
827a158e82SAndrew Turner
837a158e82SAndrew Turner static vm_page_t
efi_1t1_page(void)84dde56027SKonstantin Belousov efi_1t1_page(void)
857a158e82SAndrew Turner {
867a158e82SAndrew Turner
87dde56027SKonstantin Belousov return (vm_page_grab(obj_1t1_pt, efi_1t1_idx++, VM_ALLOC_NOBUSY |
887a158e82SAndrew Turner VM_ALLOC_WIRED | VM_ALLOC_ZERO));
897a158e82SAndrew Turner }
907a158e82SAndrew Turner
917a158e82SAndrew Turner static pt_entry_t *
efi_1t1_l3(vm_offset_t va)927a158e82SAndrew Turner efi_1t1_l3(vm_offset_t va)
937a158e82SAndrew Turner {
947a158e82SAndrew Turner pd_entry_t *l0, *l1, *l2;
957a158e82SAndrew Turner pt_entry_t *l3;
967a158e82SAndrew Turner vm_pindex_t l0_idx, l1_idx, l2_idx;
977a158e82SAndrew Turner vm_page_t m;
987a158e82SAndrew Turner vm_paddr_t mphys;
997a158e82SAndrew Turner
1007a158e82SAndrew Turner l0_idx = pmap_l0_index(va);
1017a158e82SAndrew Turner l0 = &efi_l0[l0_idx];
1027a158e82SAndrew Turner if (*l0 == 0) {
103dde56027SKonstantin Belousov m = efi_1t1_page();
1047a158e82SAndrew Turner mphys = VM_PAGE_TO_PHYS(m);
105012ea67dSZachary Leaf *l0 = PHYS_TO_PTE(mphys) | L0_TABLE;
1067a158e82SAndrew Turner } else {
1070f54e49dSZachary Leaf mphys = PTE_TO_PHYS(*l0);
1087a158e82SAndrew Turner }
1097a158e82SAndrew Turner
1107a158e82SAndrew Turner l1 = (pd_entry_t *)PHYS_TO_DMAP(mphys);
1117a158e82SAndrew Turner l1_idx = pmap_l1_index(va);
1127a158e82SAndrew Turner l1 += l1_idx;
1137a158e82SAndrew Turner if (*l1 == 0) {
114dde56027SKonstantin Belousov m = efi_1t1_page();
1157a158e82SAndrew Turner mphys = VM_PAGE_TO_PHYS(m);
116012ea67dSZachary Leaf *l1 = PHYS_TO_PTE(mphys) | L1_TABLE;
1177a158e82SAndrew Turner } else {
1180f54e49dSZachary Leaf mphys = PTE_TO_PHYS(*l1);
1197a158e82SAndrew Turner }
1207a158e82SAndrew Turner
1217a158e82SAndrew Turner l2 = (pd_entry_t *)PHYS_TO_DMAP(mphys);
1227a158e82SAndrew Turner l2_idx = pmap_l2_index(va);
1237a158e82SAndrew Turner l2 += l2_idx;
1247a158e82SAndrew Turner if (*l2 == 0) {
125dde56027SKonstantin Belousov m = efi_1t1_page();
1267a158e82SAndrew Turner mphys = VM_PAGE_TO_PHYS(m);
127012ea67dSZachary Leaf *l2 = PHYS_TO_PTE(mphys) | L2_TABLE;
1287a158e82SAndrew Turner } else {
1290f54e49dSZachary Leaf mphys = PTE_TO_PHYS(*l2);
1307a158e82SAndrew Turner }
1317a158e82SAndrew Turner
1327a158e82SAndrew Turner l3 = (pt_entry_t *)PHYS_TO_DMAP(mphys);
1337a158e82SAndrew Turner l3 += pmap_l3_index(va);
1347a158e82SAndrew Turner KASSERT(*l3 == 0, ("%s: Already mapped: va %#jx *pt %#jx", __func__,
1357a158e82SAndrew Turner va, *l3));
1367a158e82SAndrew Turner
1377a158e82SAndrew Turner return (l3);
1387a158e82SAndrew Turner }
1397a158e82SAndrew Turner
1407a158e82SAndrew Turner /*
1413395e43aSKyle Evans * Map a physical address from EFI runtime space into KVA space. Returns 0 to
1423395e43aSKyle Evans * indicate a failed mapping so that the caller may handle error.
1433395e43aSKyle Evans */
1443395e43aSKyle Evans vm_offset_t
efi_phys_to_kva(vm_paddr_t paddr)1453395e43aSKyle Evans efi_phys_to_kva(vm_paddr_t paddr)
1463395e43aSKyle Evans {
1479d40492eSAndrew Turner if (PHYS_IN_DMAP(paddr))
1489d40492eSAndrew Turner return (PHYS_TO_DMAP(paddr));
149f3ef799fSAndrew Turner
150f3ef799fSAndrew Turner /* TODO: Map memory not in the DMAP */
151f3ef799fSAndrew Turner
1523395e43aSKyle Evans return (0);
1533395e43aSKyle Evans }
1543395e43aSKyle Evans
1553395e43aSKyle Evans /*
1567a158e82SAndrew Turner * Create the 1:1 virtual to physical map for EFI
1577a158e82SAndrew Turner */
1587a158e82SAndrew Turner bool
efi_create_1t1_map(struct efi_md * map,int ndesc,int descsz)1597a158e82SAndrew Turner efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz)
1607a158e82SAndrew Turner {
1617a158e82SAndrew Turner struct efi_md *p;
162916e7b12SAndrew Turner pt_entry_t *l3, l3_attr;
1637a158e82SAndrew Turner vm_offset_t va;
16450e3ab6bSAlan Cox vm_page_t efi_l0_page;
1657a158e82SAndrew Turner uint64_t idx;
1667a158e82SAndrew Turner int i, mode;
1677a158e82SAndrew Turner
1687a158e82SAndrew Turner obj_1t1_pt = vm_pager_allocate(OBJT_PHYS, NULL, L0_ENTRIES +
1697a158e82SAndrew Turner L0_ENTRIES * Ln_ENTRIES + L0_ENTRIES * Ln_ENTRIES * Ln_ENTRIES +
1707a158e82SAndrew Turner L0_ENTRIES * Ln_ENTRIES * Ln_ENTRIES * Ln_ENTRIES,
1717a158e82SAndrew Turner VM_PROT_ALL, 0, NULL);
1727a158e82SAndrew Turner VM_OBJECT_WLOCK(obj_1t1_pt);
173dde56027SKonstantin Belousov efi_l0_page = efi_1t1_page();
1747a158e82SAndrew Turner VM_OBJECT_WUNLOCK(obj_1t1_pt);
1757a158e82SAndrew Turner efi_l0 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(efi_l0_page));
17650e3ab6bSAlan Cox efi_ttbr0 = ASID_TO_OPERAND(ASID_RESERVED_FOR_EFI) |
17750e3ab6bSAlan Cox VM_PAGE_TO_PHYS(efi_l0_page);
1787a158e82SAndrew Turner
1797a158e82SAndrew Turner for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p,
1807a158e82SAndrew Turner descsz)) {
1817a158e82SAndrew Turner if ((p->md_attr & EFI_MD_ATTR_RT) == 0)
1827a158e82SAndrew Turner continue;
183c78ad207SAndrew Turner if (p->md_virt != 0 && p->md_virt != p->md_phys) {
1847a158e82SAndrew Turner if (bootverbose)
1857a158e82SAndrew Turner printf("EFI Runtime entry %d is mapped\n", i);
1867a158e82SAndrew Turner goto fail;
1877a158e82SAndrew Turner }
1887a158e82SAndrew Turner if ((p->md_phys & EFI_PAGE_MASK) != 0) {
1897a158e82SAndrew Turner if (bootverbose)
1907a158e82SAndrew Turner printf("EFI Runtime entry %d is not aligned\n",
1917a158e82SAndrew Turner i);
1927a158e82SAndrew Turner goto fail;
1937a158e82SAndrew Turner }
1947a158e82SAndrew Turner if (p->md_phys + p->md_pages * EFI_PAGE_SIZE < p->md_phys ||
1957a158e82SAndrew Turner p->md_phys + p->md_pages * EFI_PAGE_SIZE >=
1967a158e82SAndrew Turner VM_MAXUSER_ADDRESS) {
1977a158e82SAndrew Turner printf("EFI Runtime entry %d is not in mappable for RT:"
1987a158e82SAndrew Turner "base %#016jx %#jx pages\n",
1997a158e82SAndrew Turner i, (uintmax_t)p->md_phys,
2007a158e82SAndrew Turner (uintmax_t)p->md_pages);
2017a158e82SAndrew Turner goto fail;
2027a158e82SAndrew Turner }
2037a158e82SAndrew Turner if ((p->md_attr & EFI_MD_ATTR_WB) != 0)
2047a158e82SAndrew Turner mode = VM_MEMATTR_WRITE_BACK;
2057a158e82SAndrew Turner else if ((p->md_attr & EFI_MD_ATTR_WT) != 0)
2067a158e82SAndrew Turner mode = VM_MEMATTR_WRITE_THROUGH;
2077a158e82SAndrew Turner else if ((p->md_attr & EFI_MD_ATTR_WC) != 0)
2087a158e82SAndrew Turner mode = VM_MEMATTR_WRITE_COMBINING;
2096c4395e3SEmmanuel Vadot else
2105428bb23SAndrew Turner mode = VM_MEMATTR_DEVICE;
2117a158e82SAndrew Turner
21250e1cc94SMark Johnston if (bootverbose) {
21350e1cc94SMark Johnston printf("MAP %lx mode %x pages %lu\n",
21450e1cc94SMark Johnston p->md_phys, mode, p->md_pages);
21550e1cc94SMark Johnston }
216916e7b12SAndrew Turner
217d153d023SAndrew Turner l3_attr = ATTR_DEFAULT | ATTR_S1_IDX(mode) |
218d153d023SAndrew Turner ATTR_S1_AP(ATTR_S1_AP_RW) | ATTR_S1_nG | L3_PAGE;
2196c4395e3SEmmanuel Vadot if (mode == VM_MEMATTR_DEVICE || p->md_attr & EFI_MD_ATTR_XP)
220d153d023SAndrew Turner l3_attr |= ATTR_S1_XN;
221916e7b12SAndrew Turner
2227a158e82SAndrew Turner VM_OBJECT_WLOCK(obj_1t1_pt);
223863f3220SAndrew Turner for (va = p->md_phys, idx = 0; idx < p->md_pages;
224863f3220SAndrew Turner idx += (PAGE_SIZE / EFI_PAGE_SIZE), va += PAGE_SIZE) {
2257a158e82SAndrew Turner l3 = efi_1t1_l3(va);
226916e7b12SAndrew Turner *l3 = va | l3_attr;
2277a158e82SAndrew Turner }
2287a158e82SAndrew Turner VM_OBJECT_WUNLOCK(obj_1t1_pt);
2297a158e82SAndrew Turner }
2307a158e82SAndrew Turner
2317a158e82SAndrew Turner return (true);
2327a158e82SAndrew Turner fail:
2337a158e82SAndrew Turner efi_destroy_1t1_map();
2347a158e82SAndrew Turner return (false);
2357a158e82SAndrew Turner }
2367a158e82SAndrew Turner
2377a158e82SAndrew Turner int
efi_arch_enter(void)2387a158e82SAndrew Turner efi_arch_enter(void)
2397a158e82SAndrew Turner {
2407a158e82SAndrew Turner
24150e3ab6bSAlan Cox CRITICAL_ASSERT(curthread);
24250e3ab6bSAlan Cox
24350e3ab6bSAlan Cox /*
24450e3ab6bSAlan Cox * Temporarily switch to EFI's page table. However, we leave curpmap
24550e3ab6bSAlan Cox * unchanged in order to prevent its ASID from being reclaimed before
24650e3ab6bSAlan Cox * we switch back to its page table in efi_arch_leave().
24750e3ab6bSAlan Cox */
24850e3ab6bSAlan Cox set_ttbr0(efi_ttbr0);
24950e3ab6bSAlan Cox if (PCPU_GET(bcast_tlbi_workaround) != 0)
25050e3ab6bSAlan Cox invalidate_local_icache();
2517a158e82SAndrew Turner
2527a158e82SAndrew Turner return (0);
2537a158e82SAndrew Turner }
2547a158e82SAndrew Turner
2557a158e82SAndrew Turner void
efi_arch_leave(void)2567a158e82SAndrew Turner efi_arch_leave(void)
2577a158e82SAndrew Turner {
2587a158e82SAndrew Turner
259fa19730cSAndrew Turner /*
260fa19730cSAndrew Turner * Restore the pcpu pointer. Some UEFI implementations trash it and
261fa19730cSAndrew Turner * we don't store it before calling into them. To fix this we need
262fa19730cSAndrew Turner * to restore it after returning to the kernel context. As reading
26350e3ab6bSAlan Cox * curpmap will access x18 we need to restore it before loading
26450e3ab6bSAlan Cox * the pmap pointer.
265fa19730cSAndrew Turner */
266fa19730cSAndrew Turner __asm __volatile(
267fa19730cSAndrew Turner "mrs x18, tpidr_el1 \n"
268fa19730cSAndrew Turner );
26950e3ab6bSAlan Cox set_ttbr0(pmap_to_ttbr0(PCPU_GET(curpmap)));
27050e3ab6bSAlan Cox if (PCPU_GET(bcast_tlbi_workaround) != 0)
27150e3ab6bSAlan Cox invalidate_local_icache();
2727a158e82SAndrew Turner }
27350cd0be7SKonstantin Belousov
27450cd0be7SKonstantin Belousov int
efi_rt_arch_call(struct efirt_callinfo * ec)27550cd0be7SKonstantin Belousov efi_rt_arch_call(struct efirt_callinfo *ec)
27650cd0be7SKonstantin Belousov {
27750cd0be7SKonstantin Belousov
27850cd0be7SKonstantin Belousov panic("not implemented");
27950cd0be7SKonstantin Belousov }
280