17a158e82SAndrew Turner /*- 27a158e82SAndrew Turner * Copyright (c) 2004 Marcel Moolenaar 37a158e82SAndrew Turner * Copyright (c) 2001 Doug Rabson 47a158e82SAndrew Turner * Copyright (c) 2016 The FreeBSD Foundation 57a158e82SAndrew Turner * Copyright (c) 2017 Andrew Turner 67a158e82SAndrew Turner * All rights reserved. 77a158e82SAndrew Turner * 87a158e82SAndrew Turner * Portions of this software were developed by Konstantin Belousov 97a158e82SAndrew Turner * under sponsorship from the FreeBSD Foundation. 107a158e82SAndrew Turner * 117a158e82SAndrew Turner * This software was developed by SRI International and the University of 127a158e82SAndrew Turner * Cambridge Computer Laboratory under DARPA/AFRL contract FA8750-10-C-0237 137a158e82SAndrew Turner * ("CTSRD"), as part of the DARPA CRASH research programme. 147a158e82SAndrew Turner * 157a158e82SAndrew Turner * Redistribution and use in source and binary forms, with or without 167a158e82SAndrew Turner * modification, are permitted provided that the following conditions 177a158e82SAndrew Turner * are met: 187a158e82SAndrew Turner * 1. Redistributions of source code must retain the above copyright 197a158e82SAndrew Turner * notice, this list of conditions and the following disclaimer. 207a158e82SAndrew Turner * 2. Redistributions in binary form must reproduce the above copyright 217a158e82SAndrew Turner * notice, this list of conditions and the following disclaimer in the 227a158e82SAndrew Turner * documentation and/or other materials provided with the distribution. 237a158e82SAndrew Turner * 247a158e82SAndrew Turner * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 257a158e82SAndrew Turner * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 267a158e82SAndrew Turner * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 277a158e82SAndrew Turner * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 287a158e82SAndrew Turner * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 297a158e82SAndrew Turner * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 307a158e82SAndrew Turner * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 317a158e82SAndrew Turner * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 327a158e82SAndrew Turner * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 337a158e82SAndrew Turner * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 347a158e82SAndrew Turner * SUCH DAMAGE. 357a158e82SAndrew Turner */ 367a158e82SAndrew Turner 377a158e82SAndrew Turner #include <sys/cdefs.h> 387a158e82SAndrew Turner __FBSDID("$FreeBSD$"); 397a158e82SAndrew Turner 407a158e82SAndrew Turner #include <sys/param.h> 417a158e82SAndrew Turner #include <sys/efi.h> 427a158e82SAndrew Turner #include <sys/kernel.h> 437a158e82SAndrew Turner #include <sys/linker.h> 447a158e82SAndrew Turner #include <sys/lock.h> 457a158e82SAndrew Turner #include <sys/mutex.h> 467a158e82SAndrew Turner #include <sys/proc.h> 477a158e82SAndrew Turner #include <sys/rwlock.h> 487a158e82SAndrew Turner #include <sys/systm.h> 497a158e82SAndrew Turner #include <sys/vmmeter.h> 507a158e82SAndrew Turner 517a158e82SAndrew Turner #include <machine/metadata.h> 527a158e82SAndrew Turner #include <machine/pcb.h> 537a158e82SAndrew Turner #include <machine/pte.h> 547a158e82SAndrew Turner #include <machine/vfp.h> 557a158e82SAndrew Turner #include <machine/vmparam.h> 567a158e82SAndrew Turner 577a158e82SAndrew Turner #include <vm/vm.h> 587a158e82SAndrew Turner #include <vm/pmap.h> 597a158e82SAndrew Turner #include <vm/vm_map.h> 607a158e82SAndrew Turner #include <vm/vm_object.h> 617a158e82SAndrew Turner #include <vm/vm_page.h> 627a158e82SAndrew Turner #include <vm/vm_pager.h> 637a158e82SAndrew Turner 647a158e82SAndrew Turner static vm_object_t obj_1t1_pt; 657a158e82SAndrew Turner static vm_page_t efi_l0_page; 667a158e82SAndrew Turner static pd_entry_t *efi_l0; 67dde56027SKonstantin Belousov static vm_pindex_t efi_1t1_idx; 687a158e82SAndrew Turner 697a158e82SAndrew Turner void 707a158e82SAndrew Turner efi_destroy_1t1_map(void) 717a158e82SAndrew Turner { 727a158e82SAndrew Turner vm_page_t m; 737a158e82SAndrew Turner 747a158e82SAndrew Turner if (obj_1t1_pt != NULL) { 757a158e82SAndrew Turner VM_OBJECT_RLOCK(obj_1t1_pt); 767a158e82SAndrew Turner TAILQ_FOREACH(m, &obj_1t1_pt->memq, listq) 777a158e82SAndrew Turner m->wire_count = 0; 78e958ad4cSJeff Roberson vm_wire_sub(obj_1t1_pt->resident_page_count); 797a158e82SAndrew Turner VM_OBJECT_RUNLOCK(obj_1t1_pt); 807a158e82SAndrew Turner vm_object_deallocate(obj_1t1_pt); 817a158e82SAndrew Turner } 827a158e82SAndrew Turner 837a158e82SAndrew Turner obj_1t1_pt = NULL; 847a158e82SAndrew Turner efi_l0 = NULL; 857a158e82SAndrew Turner efi_l0_page = NULL; 867a158e82SAndrew Turner } 877a158e82SAndrew Turner 887a158e82SAndrew Turner static vm_page_t 89dde56027SKonstantin Belousov efi_1t1_page(void) 907a158e82SAndrew Turner { 917a158e82SAndrew Turner 92dde56027SKonstantin Belousov return (vm_page_grab(obj_1t1_pt, efi_1t1_idx++, VM_ALLOC_NOBUSY | 937a158e82SAndrew Turner VM_ALLOC_WIRED | VM_ALLOC_ZERO)); 947a158e82SAndrew Turner } 957a158e82SAndrew Turner 967a158e82SAndrew Turner static pt_entry_t * 977a158e82SAndrew Turner efi_1t1_l3(vm_offset_t va) 987a158e82SAndrew Turner { 997a158e82SAndrew Turner pd_entry_t *l0, *l1, *l2; 1007a158e82SAndrew Turner pt_entry_t *l3; 1017a158e82SAndrew Turner vm_pindex_t l0_idx, l1_idx, l2_idx; 1027a158e82SAndrew Turner vm_page_t m; 1037a158e82SAndrew Turner vm_paddr_t mphys; 1047a158e82SAndrew Turner 1057a158e82SAndrew Turner l0_idx = pmap_l0_index(va); 1067a158e82SAndrew Turner l0 = &efi_l0[l0_idx]; 1077a158e82SAndrew Turner if (*l0 == 0) { 108dde56027SKonstantin Belousov m = efi_1t1_page(); 1097a158e82SAndrew Turner mphys = VM_PAGE_TO_PHYS(m); 1107a158e82SAndrew Turner *l0 = mphys | L0_TABLE; 1117a158e82SAndrew Turner } else { 1127a158e82SAndrew Turner mphys = *l0 & ~ATTR_MASK; 1137a158e82SAndrew Turner } 1147a158e82SAndrew Turner 1157a158e82SAndrew Turner l1 = (pd_entry_t *)PHYS_TO_DMAP(mphys); 1167a158e82SAndrew Turner l1_idx = pmap_l1_index(va); 1177a158e82SAndrew Turner l1 += l1_idx; 1187a158e82SAndrew Turner if (*l1 == 0) { 119dde56027SKonstantin Belousov m = efi_1t1_page(); 1207a158e82SAndrew Turner mphys = VM_PAGE_TO_PHYS(m); 1217a158e82SAndrew Turner *l1 = mphys | L1_TABLE; 1227a158e82SAndrew Turner } else { 1237a158e82SAndrew Turner mphys = *l1 & ~ATTR_MASK; 1247a158e82SAndrew Turner } 1257a158e82SAndrew Turner 1267a158e82SAndrew Turner l2 = (pd_entry_t *)PHYS_TO_DMAP(mphys); 1277a158e82SAndrew Turner l2_idx = pmap_l2_index(va); 1287a158e82SAndrew Turner l2 += l2_idx; 1297a158e82SAndrew Turner if (*l2 == 0) { 130dde56027SKonstantin Belousov m = efi_1t1_page(); 1317a158e82SAndrew Turner mphys = VM_PAGE_TO_PHYS(m); 1327a158e82SAndrew Turner *l2 = mphys | L2_TABLE; 1337a158e82SAndrew Turner } else { 1347a158e82SAndrew Turner mphys = *l2 & ~ATTR_MASK; 1357a158e82SAndrew Turner } 1367a158e82SAndrew Turner 1377a158e82SAndrew Turner l3 = (pt_entry_t *)PHYS_TO_DMAP(mphys); 1387a158e82SAndrew Turner l3 += pmap_l3_index(va); 1397a158e82SAndrew Turner KASSERT(*l3 == 0, ("%s: Already mapped: va %#jx *pt %#jx", __func__, 1407a158e82SAndrew Turner va, *l3)); 1417a158e82SAndrew Turner 1427a158e82SAndrew Turner return (l3); 1437a158e82SAndrew Turner } 1447a158e82SAndrew Turner 1457a158e82SAndrew Turner /* 1463395e43aSKyle Evans * Map a physical address from EFI runtime space into KVA space. Returns 0 to 1473395e43aSKyle Evans * indicate a failed mapping so that the caller may handle error. 1483395e43aSKyle Evans */ 1493395e43aSKyle Evans vm_offset_t 1503395e43aSKyle Evans efi_phys_to_kva(vm_paddr_t paddr) 1513395e43aSKyle Evans { 1523395e43aSKyle Evans 1533395e43aSKyle Evans if (!PHYS_IN_DMAP(paddr)) 1543395e43aSKyle Evans return (0); 1553395e43aSKyle Evans return (PHYS_TO_DMAP(paddr)); 1563395e43aSKyle Evans } 1573395e43aSKyle Evans 1583395e43aSKyle Evans /* 1597a158e82SAndrew Turner * Create the 1:1 virtual to physical map for EFI 1607a158e82SAndrew Turner */ 1617a158e82SAndrew Turner bool 1627a158e82SAndrew Turner efi_create_1t1_map(struct efi_md *map, int ndesc, int descsz) 1637a158e82SAndrew Turner { 1647a158e82SAndrew Turner struct efi_md *p; 165916e7b12SAndrew Turner pt_entry_t *l3, l3_attr; 1667a158e82SAndrew Turner vm_offset_t va; 1677a158e82SAndrew Turner uint64_t idx; 1687a158e82SAndrew Turner int i, mode; 1697a158e82SAndrew Turner 1707a158e82SAndrew Turner obj_1t1_pt = vm_pager_allocate(OBJT_PHYS, NULL, L0_ENTRIES + 1717a158e82SAndrew Turner L0_ENTRIES * Ln_ENTRIES + L0_ENTRIES * Ln_ENTRIES * Ln_ENTRIES + 1727a158e82SAndrew Turner L0_ENTRIES * Ln_ENTRIES * Ln_ENTRIES * Ln_ENTRIES, 1737a158e82SAndrew Turner VM_PROT_ALL, 0, NULL); 1747a158e82SAndrew Turner VM_OBJECT_WLOCK(obj_1t1_pt); 175dde56027SKonstantin Belousov efi_1t1_idx = 0; 176dde56027SKonstantin Belousov efi_l0_page = efi_1t1_page(); 1777a158e82SAndrew Turner VM_OBJECT_WUNLOCK(obj_1t1_pt); 1787a158e82SAndrew Turner efi_l0 = (pd_entry_t *)PHYS_TO_DMAP(VM_PAGE_TO_PHYS(efi_l0_page)); 1797a158e82SAndrew Turner bzero(efi_l0, L0_ENTRIES * sizeof(*efi_l0)); 1807a158e82SAndrew Turner 1817a158e82SAndrew Turner for (i = 0, p = map; i < ndesc; i++, p = efi_next_descriptor(p, 1827a158e82SAndrew Turner descsz)) { 1837a158e82SAndrew Turner if ((p->md_attr & EFI_MD_ATTR_RT) == 0) 1847a158e82SAndrew Turner continue; 18563ee68c2SKyle Evans if (p->md_virt != NULL && (uint64_t)p->md_virt != p->md_phys) { 1867a158e82SAndrew Turner if (bootverbose) 1877a158e82SAndrew Turner printf("EFI Runtime entry %d is mapped\n", i); 1887a158e82SAndrew Turner goto fail; 1897a158e82SAndrew Turner } 1907a158e82SAndrew Turner if ((p->md_phys & EFI_PAGE_MASK) != 0) { 1917a158e82SAndrew Turner if (bootverbose) 1927a158e82SAndrew Turner printf("EFI Runtime entry %d is not aligned\n", 1937a158e82SAndrew Turner i); 1947a158e82SAndrew Turner goto fail; 1957a158e82SAndrew Turner } 1967a158e82SAndrew Turner if (p->md_phys + p->md_pages * EFI_PAGE_SIZE < p->md_phys || 1977a158e82SAndrew Turner p->md_phys + p->md_pages * EFI_PAGE_SIZE >= 1987a158e82SAndrew Turner VM_MAXUSER_ADDRESS) { 1997a158e82SAndrew Turner printf("EFI Runtime entry %d is not in mappable for RT:" 2007a158e82SAndrew Turner "base %#016jx %#jx pages\n", 2017a158e82SAndrew Turner i, (uintmax_t)p->md_phys, 2027a158e82SAndrew Turner (uintmax_t)p->md_pages); 2037a158e82SAndrew Turner goto fail; 2047a158e82SAndrew Turner } 2057a158e82SAndrew Turner if ((p->md_attr & EFI_MD_ATTR_WB) != 0) 2067a158e82SAndrew Turner mode = VM_MEMATTR_WRITE_BACK; 2077a158e82SAndrew Turner else if ((p->md_attr & EFI_MD_ATTR_WT) != 0) 2087a158e82SAndrew Turner mode = VM_MEMATTR_WRITE_THROUGH; 2097a158e82SAndrew Turner else if ((p->md_attr & EFI_MD_ATTR_WC) != 0) 2107a158e82SAndrew Turner mode = VM_MEMATTR_WRITE_COMBINING; 2117a158e82SAndrew Turner else if ((p->md_attr & EFI_MD_ATTR_UC) != 0) 2125428bb23SAndrew Turner mode = VM_MEMATTR_DEVICE; 2137a158e82SAndrew Turner else { 2147a158e82SAndrew Turner if (bootverbose) 2157a158e82SAndrew Turner printf("EFI Runtime entry %d mapping " 2167a158e82SAndrew Turner "attributes unsupported\n", i); 2177a158e82SAndrew Turner mode = VM_MEMATTR_UNCACHEABLE; 2187a158e82SAndrew Turner } 2197a158e82SAndrew Turner 2207a158e82SAndrew Turner printf("MAP %lx mode %x pages %lu\n", p->md_phys, mode, p->md_pages); 221916e7b12SAndrew Turner 222916e7b12SAndrew Turner l3_attr = ATTR_DEFAULT | ATTR_IDX(mode) | ATTR_AP(ATTR_AP_RW) | 223916e7b12SAndrew Turner L3_PAGE; 224916e7b12SAndrew Turner if (mode == VM_MEMATTR_DEVICE) 225916e7b12SAndrew Turner l3_attr |= ATTR_UXN | ATTR_PXN; 226916e7b12SAndrew Turner 2277a158e82SAndrew Turner VM_OBJECT_WLOCK(obj_1t1_pt); 2287a158e82SAndrew Turner for (va = p->md_phys, idx = 0; idx < p->md_pages; idx++, 2297a158e82SAndrew Turner va += PAGE_SIZE) { 2307a158e82SAndrew Turner l3 = efi_1t1_l3(va); 231916e7b12SAndrew Turner *l3 = va | l3_attr; 2327a158e82SAndrew Turner } 2337a158e82SAndrew Turner VM_OBJECT_WUNLOCK(obj_1t1_pt); 2347a158e82SAndrew Turner } 2357a158e82SAndrew Turner 2367a158e82SAndrew Turner return (true); 2377a158e82SAndrew Turner fail: 2387a158e82SAndrew Turner efi_destroy_1t1_map(); 2397a158e82SAndrew Turner return (false); 2407a158e82SAndrew Turner } 2417a158e82SAndrew Turner 2427a158e82SAndrew Turner int 2437a158e82SAndrew Turner efi_arch_enter(void) 2447a158e82SAndrew Turner { 2457a158e82SAndrew Turner 2467a158e82SAndrew Turner __asm __volatile( 2477a158e82SAndrew Turner "msr ttbr0_el1, %0 \n" 2487a158e82SAndrew Turner "dsb ishst \n" 2497a158e82SAndrew Turner "tlbi vmalle1is \n" 2507a158e82SAndrew Turner "dsb ish \n" 2517a158e82SAndrew Turner "isb \n" 2527a158e82SAndrew Turner : : "r"(VM_PAGE_TO_PHYS(efi_l0_page))); 2537a158e82SAndrew Turner 2547a158e82SAndrew Turner return (0); 2557a158e82SAndrew Turner } 2567a158e82SAndrew Turner 2577a158e82SAndrew Turner void 2587a158e82SAndrew Turner efi_arch_leave(void) 2597a158e82SAndrew Turner { 2607a158e82SAndrew Turner struct thread *td; 2617a158e82SAndrew Turner 2627a158e82SAndrew Turner td = curthread; 2637a158e82SAndrew Turner __asm __volatile( 2647a158e82SAndrew Turner "msr ttbr0_el1, %0 \n" 2657a158e82SAndrew Turner "dsb ishst \n" 2667a158e82SAndrew Turner "tlbi vmalle1is \n" 2677a158e82SAndrew Turner "dsb ish \n" 2687a158e82SAndrew Turner "isb \n" 2697a158e82SAndrew Turner : : "r"(td->td_proc->p_md.md_l0addr)); 2707a158e82SAndrew Turner } 271