154936004Sbellard /* 254936004Sbellard * mmap support for qemu 354936004Sbellard * 454936004Sbellard * Copyright (c) 2003 Fabrice Bellard 554936004Sbellard * 654936004Sbellard * This program is free software; you can redistribute it and/or modify 754936004Sbellard * it under the terms of the GNU General Public License as published by 854936004Sbellard * the Free Software Foundation; either version 2 of the License, or 954936004Sbellard * (at your option) any later version. 1054936004Sbellard * 1154936004Sbellard * This program is distributed in the hope that it will be useful, 1254936004Sbellard * but WITHOUT ANY WARRANTY; without even the implied warranty of 1354936004Sbellard * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 1454936004Sbellard * GNU General Public License for more details. 1554936004Sbellard * 1654936004Sbellard * You should have received a copy of the GNU General Public License 178167ee88SBlue Swirl * along with this program; if not, see <http://www.gnu.org/licenses/>. 1854936004Sbellard */ 19d39594e9SPeter Maydell #include "qemu/osdep.h" 2011d96056SAlex Bennée #include "trace.h" 2110d0d505SAlex Bennée #include "exec/log.h" 2254936004Sbellard #include "qemu.h" 2354936004Sbellard 241e6eec8bSBlue Swirl static pthread_mutex_t mmap_mutex = PTHREAD_MUTEX_INITIALIZER; 25dfd3f85cSJuan Quintela static __thread int mmap_lock_count; 26c8a706feSpbrook 27c8a706feSpbrook void mmap_lock(void) 28c8a706feSpbrook { 29c8a706feSpbrook if (mmap_lock_count++ == 0) { 30c8a706feSpbrook pthread_mutex_lock(&mmap_mutex); 31c8a706feSpbrook } 32c8a706feSpbrook } 33c8a706feSpbrook 34c8a706feSpbrook void mmap_unlock(void) 35c8a706feSpbrook { 36c8a706feSpbrook if (--mmap_lock_count == 0) { 37c8a706feSpbrook pthread_mutex_unlock(&mmap_mutex); 38c8a706feSpbrook } 39c8a706feSpbrook } 40d5975363Spbrook 41301e40edSAlex Bennée bool have_mmap_lock(void) 42301e40edSAlex Bennée { 43301e40edSAlex Bennée return mmap_lock_count > 0 ? true : false; 44301e40edSAlex Bennée } 45301e40edSAlex Bennée 46d5975363Spbrook /* Grab lock to make sure things are in a consistent state after fork(). */ 47d5975363Spbrook void mmap_fork_start(void) 48d5975363Spbrook { 49d5975363Spbrook if (mmap_lock_count) 50d5975363Spbrook abort(); 51d5975363Spbrook pthread_mutex_lock(&mmap_mutex); 52d5975363Spbrook } 53d5975363Spbrook 54d5975363Spbrook void mmap_fork_end(int child) 55d5975363Spbrook { 56d5975363Spbrook if (child) 57d5975363Spbrook pthread_mutex_init(&mmap_mutex, NULL); 58d5975363Spbrook else 59d5975363Spbrook pthread_mutex_unlock(&mmap_mutex); 60d5975363Spbrook } 61c8a706feSpbrook 6253a5960aSpbrook /* NOTE: all the constants are the HOST ones, but addresses are target. */ 63992f48a0Sblueswir1 int target_mprotect(abi_ulong start, abi_ulong len, int prot) 6454936004Sbellard { 65992f48a0Sblueswir1 abi_ulong end, host_start, host_end, addr; 6654936004Sbellard int prot1, ret; 6754936004Sbellard 6811d96056SAlex Bennée trace_target_mprotect(start, len, prot); 6954936004Sbellard 7054936004Sbellard if ((start & ~TARGET_PAGE_MASK) != 0) 7178cf3390SMax Filippov return -TARGET_EINVAL; 7254936004Sbellard len = TARGET_PAGE_ALIGN(len); 7354936004Sbellard end = start + len; 74ebf9a363SMax Filippov if (!guest_range_valid(start, len)) { 7578cf3390SMax Filippov return -TARGET_ENOMEM; 76ebf9a363SMax Filippov } 77171cd1cdSbalrog prot &= PROT_READ | PROT_WRITE | PROT_EXEC; 7854936004Sbellard if (len == 0) 7954936004Sbellard return 0; 8054936004Sbellard 81c8a706feSpbrook mmap_lock(); 8283fb7adfSbellard host_start = start & qemu_host_page_mask; 8354936004Sbellard host_end = HOST_PAGE_ALIGN(end); 8454936004Sbellard if (start > host_start) { 8554936004Sbellard /* handle host page containing start */ 8654936004Sbellard prot1 = prot; 8754936004Sbellard for(addr = host_start; addr < start; addr += TARGET_PAGE_SIZE) { 8854936004Sbellard prot1 |= page_get_flags(addr); 8954936004Sbellard } 9083fb7adfSbellard if (host_end == host_start + qemu_host_page_size) { 91d418c81eSbellard for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) { 92d418c81eSbellard prot1 |= page_get_flags(addr); 93d418c81eSbellard } 94d418c81eSbellard end = host_end; 95d418c81eSbellard } 9653a5960aSpbrook ret = mprotect(g2h(host_start), qemu_host_page_size, prot1 & PAGE_BITS); 9754936004Sbellard if (ret != 0) 98c8a706feSpbrook goto error; 9983fb7adfSbellard host_start += qemu_host_page_size; 10054936004Sbellard } 10154936004Sbellard if (end < host_end) { 10254936004Sbellard prot1 = prot; 10354936004Sbellard for(addr = end; addr < host_end; addr += TARGET_PAGE_SIZE) { 10454936004Sbellard prot1 |= page_get_flags(addr); 10554936004Sbellard } 10653a5960aSpbrook ret = mprotect(g2h(host_end - qemu_host_page_size), qemu_host_page_size, 10754936004Sbellard prot1 & PAGE_BITS); 10854936004Sbellard if (ret != 0) 109c8a706feSpbrook goto error; 11083fb7adfSbellard host_end -= qemu_host_page_size; 11154936004Sbellard } 11254936004Sbellard 11354936004Sbellard /* handle the pages in the middle */ 11454936004Sbellard if (host_start < host_end) { 11553a5960aSpbrook ret = mprotect(g2h(host_start), host_end - host_start, prot); 11654936004Sbellard if (ret != 0) 117c8a706feSpbrook goto error; 11854936004Sbellard } 11954936004Sbellard page_set_flags(start, start + len, prot | PAGE_VALID); 120c8a706feSpbrook mmap_unlock(); 12154936004Sbellard return 0; 122c8a706feSpbrook error: 123c8a706feSpbrook mmap_unlock(); 124c8a706feSpbrook return ret; 12554936004Sbellard } 12654936004Sbellard 12754936004Sbellard /* map an incomplete host page */ 128992f48a0Sblueswir1 static int mmap_frag(abi_ulong real_start, 129992f48a0Sblueswir1 abi_ulong start, abi_ulong end, 130992f48a0Sblueswir1 int prot, int flags, int fd, abi_ulong offset) 13154936004Sbellard { 13280210bcdSths abi_ulong real_end, addr; 13353a5960aSpbrook void *host_start; 13454936004Sbellard int prot1, prot_new; 13554936004Sbellard 13653a5960aSpbrook real_end = real_start + qemu_host_page_size; 13753a5960aSpbrook host_start = g2h(real_start); 13854936004Sbellard 13954936004Sbellard /* get the protection of the target pages outside the mapping */ 14054936004Sbellard prot1 = 0; 14153a5960aSpbrook for(addr = real_start; addr < real_end; addr++) { 14254936004Sbellard if (addr < start || addr >= end) 14354936004Sbellard prot1 |= page_get_flags(addr); 14454936004Sbellard } 14554936004Sbellard 14654936004Sbellard if (prot1 == 0) { 14754936004Sbellard /* no page was there, so we allocate one */ 14880210bcdSths void *p = mmap(host_start, qemu_host_page_size, prot, 14954936004Sbellard flags | MAP_ANONYMOUS, -1, 0); 15080210bcdSths if (p == MAP_FAILED) 15180210bcdSths return -1; 15253a5960aSpbrook prot1 = prot; 15354936004Sbellard } 15454936004Sbellard prot1 &= PAGE_BITS; 15554936004Sbellard 15654936004Sbellard prot_new = prot | prot1; 15754936004Sbellard if (!(flags & MAP_ANONYMOUS)) { 15854936004Sbellard /* msync() won't work here, so we return an error if write is 15954936004Sbellard possible while it is a shared mapping */ 16054936004Sbellard if ((flags & MAP_TYPE) == MAP_SHARED && 16154936004Sbellard (prot & PROT_WRITE)) 162ee636500SJuan Quintela return -1; 16354936004Sbellard 16454936004Sbellard /* adjust protection to be able to read */ 16554936004Sbellard if (!(prot1 & PROT_WRITE)) 16653a5960aSpbrook mprotect(host_start, qemu_host_page_size, prot1 | PROT_WRITE); 16754936004Sbellard 16854936004Sbellard /* read the corresponding file data */ 169fb7e378cSKirill A. Shutemov if (pread(fd, g2h(start), end - start, offset) == -1) 170fb7e378cSKirill A. Shutemov return -1; 17154936004Sbellard 17254936004Sbellard /* put final protection */ 17354936004Sbellard if (prot_new != (prot1 | PROT_WRITE)) 17453a5960aSpbrook mprotect(host_start, qemu_host_page_size, prot_new); 17554936004Sbellard } else { 17654936004Sbellard if (prot_new != prot1) { 17753a5960aSpbrook mprotect(host_start, qemu_host_page_size, prot_new); 17854936004Sbellard } 179e6deac9cSChen Gang if (prot_new & PROT_WRITE) { 180e6deac9cSChen Gang memset(g2h(start), 0, end - start); 181e6deac9cSChen Gang } 18254936004Sbellard } 18354936004Sbellard return 0; 18454936004Sbellard } 18554936004Sbellard 18614f24e14SRichard Henderson #if HOST_LONG_BITS == 64 && TARGET_ABI_BITS == 64 187*aab613fbSLirong Yuan #ifdef TARGET_AARCH64 188*aab613fbSLirong Yuan # define TASK_UNMAPPED_BASE 0x5500000000 189*aab613fbSLirong Yuan #else 19014f24e14SRichard Henderson # define TASK_UNMAPPED_BASE (1ul << 38) 191*aab613fbSLirong Yuan #endif 192a03e2d42Sbellard #else 19314f24e14SRichard Henderson # define TASK_UNMAPPED_BASE 0x40000000 194a03e2d42Sbellard #endif 19559e9d91cSPeter Maydell abi_ulong mmap_next_start = TASK_UNMAPPED_BASE; 196a03e2d42Sbellard 1970776590dSpbrook unsigned long last_brk; 1980776590dSpbrook 19968a1c816SPaul Brook /* Subroutine of mmap_find_vma, used when we have pre-allocated a chunk 20068a1c816SPaul Brook of guest address space. */ 20130ab9ef2SRichard Henderson static abi_ulong mmap_find_vma_reserved(abi_ulong start, abi_ulong size, 20230ab9ef2SRichard Henderson abi_ulong align) 20368a1c816SPaul Brook { 20430ab9ef2SRichard Henderson abi_ulong addr, end_addr, incr = qemu_host_page_size; 20568a1c816SPaul Brook int prot; 20630ab9ef2SRichard Henderson bool looped = false; 20768a1c816SPaul Brook 208b76f21a7SLaurent Vivier if (size > reserved_va) { 20968a1c816SPaul Brook return (abi_ulong)-1; 21068a1c816SPaul Brook } 21168a1c816SPaul Brook 21230ab9ef2SRichard Henderson /* Note that start and size have already been aligned by mmap_find_vma. */ 21359e9d91cSPeter Maydell 21430ab9ef2SRichard Henderson end_addr = start + size; 21530ab9ef2SRichard Henderson if (start > reserved_va - size) { 21630ab9ef2SRichard Henderson /* Start at the top of the address space. */ 21730ab9ef2SRichard Henderson end_addr = ((reserved_va - size) & -align) + size; 21830ab9ef2SRichard Henderson looped = true; 21930ab9ef2SRichard Henderson } 22030ab9ef2SRichard Henderson 22130ab9ef2SRichard Henderson /* Search downward from END_ADDR, checking to see if a page is in use. */ 22230ab9ef2SRichard Henderson addr = end_addr; 22359e9d91cSPeter Maydell while (1) { 22430ab9ef2SRichard Henderson addr -= incr; 22559e9d91cSPeter Maydell if (addr > end_addr) { 22668a1c816SPaul Brook if (looped) { 22730ab9ef2SRichard Henderson /* Failure. The entire address space has been searched. */ 22868a1c816SPaul Brook return (abi_ulong)-1; 22968a1c816SPaul Brook } 23030ab9ef2SRichard Henderson /* Re-start at the top of the address space. */ 23130ab9ef2SRichard Henderson addr = end_addr = ((reserved_va - size) & -align) + size; 23230ab9ef2SRichard Henderson looped = true; 23330ab9ef2SRichard Henderson } else { 23468a1c816SPaul Brook prot = page_get_flags(addr); 23568a1c816SPaul Brook if (prot) { 23630ab9ef2SRichard Henderson /* Page in use. Restart below this page. */ 23730ab9ef2SRichard Henderson addr = end_addr = ((addr - size) & -align) + size; 23830ab9ef2SRichard Henderson } else if (addr && addr + size == end_addr) { 23930ab9ef2SRichard Henderson /* Success! All pages between ADDR and END_ADDR are free. */ 24059e9d91cSPeter Maydell if (start == mmap_next_start) { 24168a1c816SPaul Brook mmap_next_start = addr; 24259e9d91cSPeter Maydell } 24359e9d91cSPeter Maydell return addr; 24468a1c816SPaul Brook } 24530ab9ef2SRichard Henderson } 24630ab9ef2SRichard Henderson } 24730ab9ef2SRichard Henderson } 24868a1c816SPaul Brook 249fe3b4152SKirill A. Shutemov /* 250fe3b4152SKirill A. Shutemov * Find and reserve a free memory area of size 'size'. The search 251fe3b4152SKirill A. Shutemov * starts at 'start'. 252fe3b4152SKirill A. Shutemov * It must be called with mmap_lock() held. 253fe3b4152SKirill A. Shutemov * Return -1 if error. 254a03e2d42Sbellard */ 25530ab9ef2SRichard Henderson abi_ulong mmap_find_vma(abi_ulong start, abi_ulong size, abi_ulong align) 256a03e2d42Sbellard { 25714f24e14SRichard Henderson void *ptr, *prev; 258fe3b4152SKirill A. Shutemov abi_ulong addr; 25914f24e14SRichard Henderson int wrapped, repeat; 260fe3b4152SKirill A. Shutemov 261443b7505SRichard Henderson align = MAX(align, qemu_host_page_size); 262443b7505SRichard Henderson 263fe3b4152SKirill A. Shutemov /* If 'start' == 0, then a default start address is used. */ 26414f24e14SRichard Henderson if (start == 0) { 265fe3b4152SKirill A. Shutemov start = mmap_next_start; 26614f24e14SRichard Henderson } else { 26714f24e14SRichard Henderson start &= qemu_host_page_mask; 26814f24e14SRichard Henderson } 26930ab9ef2SRichard Henderson start = ROUND_UP(start, align); 27014f24e14SRichard Henderson 27114f24e14SRichard Henderson size = HOST_PAGE_ALIGN(size); 272fe3b4152SKirill A. Shutemov 273b76f21a7SLaurent Vivier if (reserved_va) { 27430ab9ef2SRichard Henderson return mmap_find_vma_reserved(start, size, align); 27568a1c816SPaul Brook } 27668a1c816SPaul Brook 277a03e2d42Sbellard addr = start; 27814f24e14SRichard Henderson wrapped = repeat = 0; 27914f24e14SRichard Henderson prev = 0; 280fe3b4152SKirill A. Shutemov 28114f24e14SRichard Henderson for (;; prev = ptr) { 282fe3b4152SKirill A. Shutemov /* 283fe3b4152SKirill A. Shutemov * Reserve needed memory area to avoid a race. 284fe3b4152SKirill A. Shutemov * It should be discarded using: 285fe3b4152SKirill A. Shutemov * - mmap() with MAP_FIXED flag 286fe3b4152SKirill A. Shutemov * - mremap() with MREMAP_FIXED flag 287fe3b4152SKirill A. Shutemov * - shmat() with SHM_REMAP flag 288fe3b4152SKirill A. Shutemov */ 28914f24e14SRichard Henderson ptr = mmap(g2h(addr), size, PROT_NONE, 290fe3b4152SKirill A. Shutemov MAP_ANONYMOUS|MAP_PRIVATE|MAP_NORESERVE, -1, 0); 291fe3b4152SKirill A. Shutemov 292fe3b4152SKirill A. Shutemov /* ENOMEM, if host address space has no memory */ 29314f24e14SRichard Henderson if (ptr == MAP_FAILED) { 294a03e2d42Sbellard return (abi_ulong)-1; 295a03e2d42Sbellard } 296fe3b4152SKirill A. Shutemov 29714f24e14SRichard Henderson /* Count the number of sequential returns of the same address. 29814f24e14SRichard Henderson This is used to modify the search algorithm below. */ 29914f24e14SRichard Henderson repeat = (ptr == prev ? repeat + 1 : 0); 300fe3b4152SKirill A. Shutemov 30114f24e14SRichard Henderson if (h2g_valid(ptr + size - 1)) { 30214f24e14SRichard Henderson addr = h2g(ptr); 30314f24e14SRichard Henderson 30430ab9ef2SRichard Henderson if ((addr & (align - 1)) == 0) { 30514f24e14SRichard Henderson /* Success. */ 30614f24e14SRichard Henderson if (start == mmap_next_start && addr >= TASK_UNMAPPED_BASE) { 30714f24e14SRichard Henderson mmap_next_start = addr + size; 30814f24e14SRichard Henderson } 30914f24e14SRichard Henderson return addr; 31014f24e14SRichard Henderson } 31114f24e14SRichard Henderson 31214f24e14SRichard Henderson /* The address is not properly aligned for the target. */ 31314f24e14SRichard Henderson switch (repeat) { 31414f24e14SRichard Henderson case 0: 31514f24e14SRichard Henderson /* Assume the result that the kernel gave us is the 31614f24e14SRichard Henderson first with enough free space, so start again at the 31714f24e14SRichard Henderson next higher target page. */ 31830ab9ef2SRichard Henderson addr = ROUND_UP(addr, align); 31914f24e14SRichard Henderson break; 32014f24e14SRichard Henderson case 1: 32114f24e14SRichard Henderson /* Sometimes the kernel decides to perform the allocation 32214f24e14SRichard Henderson at the top end of memory instead. */ 32330ab9ef2SRichard Henderson addr &= -align; 32414f24e14SRichard Henderson break; 32514f24e14SRichard Henderson case 2: 32614f24e14SRichard Henderson /* Start over at low memory. */ 32714f24e14SRichard Henderson addr = 0; 32814f24e14SRichard Henderson break; 32914f24e14SRichard Henderson default: 33014f24e14SRichard Henderson /* Fail. This unaligned block must the last. */ 33114f24e14SRichard Henderson addr = -1; 33214f24e14SRichard Henderson break; 33314f24e14SRichard Henderson } 33414f24e14SRichard Henderson } else { 33514f24e14SRichard Henderson /* Since the result the kernel gave didn't fit, start 33614f24e14SRichard Henderson again at low memory. If any repetition, fail. */ 33714f24e14SRichard Henderson addr = (repeat ? -1 : 0); 33814f24e14SRichard Henderson } 33914f24e14SRichard Henderson 34014f24e14SRichard Henderson /* Unmap and try again. */ 34114f24e14SRichard Henderson munmap(ptr, size); 34214f24e14SRichard Henderson 34314f24e14SRichard Henderson /* ENOMEM if we checked the whole of the target address space. */ 344d0b3e4f5SBlue Swirl if (addr == (abi_ulong)-1) { 34514f24e14SRichard Henderson return (abi_ulong)-1; 34614f24e14SRichard Henderson } else if (addr == 0) { 34714f24e14SRichard Henderson if (wrapped) { 34814f24e14SRichard Henderson return (abi_ulong)-1; 34914f24e14SRichard Henderson } 35014f24e14SRichard Henderson wrapped = 1; 35114f24e14SRichard Henderson /* Don't actually use 0 when wrapping, instead indicate 3528186e783SStefan Weil that we'd truly like an allocation in low memory. */ 35314f24e14SRichard Henderson addr = (mmap_min_addr > TARGET_PAGE_SIZE 35414f24e14SRichard Henderson ? TARGET_PAGE_ALIGN(mmap_min_addr) 35514f24e14SRichard Henderson : TARGET_PAGE_SIZE); 35614f24e14SRichard Henderson } else if (wrapped && addr >= start) { 35714f24e14SRichard Henderson return (abi_ulong)-1; 35814f24e14SRichard Henderson } 35914f24e14SRichard Henderson } 360a03e2d42Sbellard } 361a03e2d42Sbellard 36254936004Sbellard /* NOTE: all the constants are the HOST ones */ 363992f48a0Sblueswir1 abi_long target_mmap(abi_ulong start, abi_ulong len, int prot, 364992f48a0Sblueswir1 int flags, int fd, abi_ulong offset) 36554936004Sbellard { 366992f48a0Sblueswir1 abi_ulong ret, end, real_start, real_end, retaddr, host_offset, host_len; 36754936004Sbellard 368c8a706feSpbrook mmap_lock(); 3695a67bb96SAlex Bennée trace_target_mmap(start, len, prot, flags, fd, offset); 37054936004Sbellard 37138138fabSAlex Bennée if (!len) { 37238138fabSAlex Bennée errno = EINVAL; 37338138fabSAlex Bennée goto fail; 37438138fabSAlex Bennée } 37538138fabSAlex Bennée 37638138fabSAlex Bennée /* Also check for overflows... */ 37738138fabSAlex Bennée len = TARGET_PAGE_ALIGN(len); 37838138fabSAlex Bennée if (!len) { 37938138fabSAlex Bennée errno = ENOMEM; 38038138fabSAlex Bennée goto fail; 38138138fabSAlex Bennée } 38238138fabSAlex Bennée 383e89f07d3Spbrook if (offset & ~TARGET_PAGE_MASK) { 384e89f07d3Spbrook errno = EINVAL; 385c8a706feSpbrook goto fail; 386e89f07d3Spbrook } 38754936004Sbellard 38853a5960aSpbrook real_start = start & qemu_host_page_mask; 389a5e7ee46SRichard Henderson host_offset = offset & qemu_host_page_mask; 390a5e7ee46SRichard Henderson 391a5e7ee46SRichard Henderson /* If the user is asking for the kernel to find a location, do that 392a5e7ee46SRichard Henderson before we truncate the length for mapping files below. */ 393a5e7ee46SRichard Henderson if (!(flags & MAP_FIXED)) { 394a5e7ee46SRichard Henderson host_len = len + offset - host_offset; 395a5e7ee46SRichard Henderson host_len = HOST_PAGE_ALIGN(host_len); 39630ab9ef2SRichard Henderson start = mmap_find_vma(real_start, host_len, TARGET_PAGE_SIZE); 397a5e7ee46SRichard Henderson if (start == (abi_ulong)-1) { 398a5e7ee46SRichard Henderson errno = ENOMEM; 399a5e7ee46SRichard Henderson goto fail; 400a5e7ee46SRichard Henderson } 401a5e7ee46SRichard Henderson } 40254936004Sbellard 40354c5a2aeSedgar_igl /* When mapping files into a memory area larger than the file, accesses 40454c5a2aeSedgar_igl to pages beyond the file size will cause a SIGBUS. 40554c5a2aeSedgar_igl 40654c5a2aeSedgar_igl For example, if mmaping a file of 100 bytes on a host with 4K pages 40754c5a2aeSedgar_igl emulating a target with 8K pages, the target expects to be able to 40854c5a2aeSedgar_igl access the first 8K. But the host will trap us on any access beyond 40954c5a2aeSedgar_igl 4K. 41054c5a2aeSedgar_igl 41154c5a2aeSedgar_igl When emulating a target with a larger page-size than the hosts, we 41254c5a2aeSedgar_igl may need to truncate file maps at EOF and add extra anonymous pages 41354c5a2aeSedgar_igl up to the targets page boundary. */ 41454c5a2aeSedgar_igl 41535f2fd04SMarc-André Lureau if ((qemu_real_host_page_size < qemu_host_page_size) && 41635f2fd04SMarc-André Lureau !(flags & MAP_ANONYMOUS)) { 41754c5a2aeSedgar_igl struct stat sb; 41854c5a2aeSedgar_igl 41954c5a2aeSedgar_igl if (fstat (fd, &sb) == -1) 42054c5a2aeSedgar_igl goto fail; 42154c5a2aeSedgar_igl 42254c5a2aeSedgar_igl /* Are we trying to create a map beyond EOF?. */ 42354c5a2aeSedgar_igl if (offset + len > sb.st_size) { 42454c5a2aeSedgar_igl /* If so, truncate the file map at eof aligned with 42554c5a2aeSedgar_igl the hosts real pagesize. Additional anonymous maps 42654c5a2aeSedgar_igl will be created beyond EOF. */ 4270c2d70c4SPaolo Bonzini len = REAL_HOST_PAGE_ALIGN(sb.st_size - offset); 42854c5a2aeSedgar_igl } 42954c5a2aeSedgar_igl } 43054c5a2aeSedgar_igl 43154936004Sbellard if (!(flags & MAP_FIXED)) { 432a5e7ee46SRichard Henderson unsigned long host_start; 433a03e2d42Sbellard void *p; 434a5e7ee46SRichard Henderson 43554936004Sbellard host_len = len + offset - host_offset; 436a03e2d42Sbellard host_len = HOST_PAGE_ALIGN(host_len); 437a5e7ee46SRichard Henderson 438a03e2d42Sbellard /* Note: we prefer to control the mapping address. It is 439a03e2d42Sbellard especially important if qemu_host_page_size > 440a03e2d42Sbellard qemu_real_host_page_size */ 441a5e7ee46SRichard Henderson p = mmap(g2h(start), host_len, prot, 442a5e7ee46SRichard Henderson flags | MAP_FIXED | MAP_ANONYMOUS, -1, 0); 44380210bcdSths if (p == MAP_FAILED) 444c8a706feSpbrook goto fail; 44554936004Sbellard /* update start so that it points to the file position at 'offset' */ 44680210bcdSths host_start = (unsigned long)p; 44754c5a2aeSedgar_igl if (!(flags & MAP_ANONYMOUS)) { 448a5e7ee46SRichard Henderson p = mmap(g2h(start), len, prot, 44954c5a2aeSedgar_igl flags | MAP_FIXED, fd, host_offset); 4508384274eSJürg Billeter if (p == MAP_FAILED) { 4518384274eSJürg Billeter munmap(g2h(start), host_len); 4528384274eSJürg Billeter goto fail; 4538384274eSJürg Billeter } 45453a5960aSpbrook host_start += offset - host_offset; 45554c5a2aeSedgar_igl } 45653a5960aSpbrook start = h2g(host_start); 457a03e2d42Sbellard } else { 458e89f07d3Spbrook if (start & ~TARGET_PAGE_MASK) { 459e89f07d3Spbrook errno = EINVAL; 460c8a706feSpbrook goto fail; 461e89f07d3Spbrook } 46254936004Sbellard end = start + len; 46353a5960aSpbrook real_end = HOST_PAGE_ALIGN(end); 46454936004Sbellard 46545bc1f52Saurel32 /* 46645bc1f52Saurel32 * Test if requested memory area fits target address space 46745bc1f52Saurel32 * It can fail only on 64-bit host with 32-bit target. 46845bc1f52Saurel32 * On any other target/host host mmap() handles this error correctly. 46945bc1f52Saurel32 */ 470ebf9a363SMax Filippov if (!guest_range_valid(start, len)) { 471ebf9a363SMax Filippov errno = ENOMEM; 47245bc1f52Saurel32 goto fail; 47345bc1f52Saurel32 } 47445bc1f52Saurel32 47554936004Sbellard /* worst case: we cannot map the file because the offset is not 47654936004Sbellard aligned, so we read it */ 47754936004Sbellard if (!(flags & MAP_ANONYMOUS) && 47883fb7adfSbellard (offset & ~qemu_host_page_mask) != (start & ~qemu_host_page_mask)) { 47954936004Sbellard /* msync() won't work here, so we return an error if write is 48054936004Sbellard possible while it is a shared mapping */ 48154936004Sbellard if ((flags & MAP_TYPE) == MAP_SHARED && 482e89f07d3Spbrook (prot & PROT_WRITE)) { 483e89f07d3Spbrook errno = EINVAL; 484c8a706feSpbrook goto fail; 485e89f07d3Spbrook } 48654936004Sbellard retaddr = target_mmap(start, len, prot | PROT_WRITE, 48754936004Sbellard MAP_FIXED | MAP_PRIVATE | MAP_ANONYMOUS, 48854936004Sbellard -1, 0); 48954936004Sbellard if (retaddr == -1) 490c8a706feSpbrook goto fail; 491fb7e378cSKirill A. Shutemov if (pread(fd, g2h(start), len, offset) == -1) 492fb7e378cSKirill A. Shutemov goto fail; 49354936004Sbellard if (!(prot & PROT_WRITE)) { 49454936004Sbellard ret = target_mprotect(start, len, prot); 49586abac06SPaolo Bonzini assert(ret == 0); 49654936004Sbellard } 49754936004Sbellard goto the_end; 49854936004Sbellard } 49954936004Sbellard 50054936004Sbellard /* handle the start of the mapping */ 50153a5960aSpbrook if (start > real_start) { 50253a5960aSpbrook if (real_end == real_start + qemu_host_page_size) { 50354936004Sbellard /* one single host page */ 50453a5960aSpbrook ret = mmap_frag(real_start, start, end, 50554936004Sbellard prot, flags, fd, offset); 50654936004Sbellard if (ret == -1) 507c8a706feSpbrook goto fail; 50854936004Sbellard goto the_end1; 50954936004Sbellard } 51053a5960aSpbrook ret = mmap_frag(real_start, start, real_start + qemu_host_page_size, 51154936004Sbellard prot, flags, fd, offset); 51254936004Sbellard if (ret == -1) 513c8a706feSpbrook goto fail; 51453a5960aSpbrook real_start += qemu_host_page_size; 51554936004Sbellard } 51654936004Sbellard /* handle the end of the mapping */ 51753a5960aSpbrook if (end < real_end) { 51853a5960aSpbrook ret = mmap_frag(real_end - qemu_host_page_size, 519530c0032SChen Gang real_end - qemu_host_page_size, end, 52054936004Sbellard prot, flags, fd, 52153a5960aSpbrook offset + real_end - qemu_host_page_size - start); 52254936004Sbellard if (ret == -1) 523c8a706feSpbrook goto fail; 52453a5960aSpbrook real_end -= qemu_host_page_size; 52554936004Sbellard } 52654936004Sbellard 52754936004Sbellard /* map the middle (easier) */ 52853a5960aSpbrook if (real_start < real_end) { 52980210bcdSths void *p; 5304a585ccbSbellard unsigned long offset1; 5314a585ccbSbellard if (flags & MAP_ANONYMOUS) 5324a585ccbSbellard offset1 = 0; 5334a585ccbSbellard else 53453a5960aSpbrook offset1 = offset + real_start - start; 53580210bcdSths p = mmap(g2h(real_start), real_end - real_start, 5364a585ccbSbellard prot, flags, fd, offset1); 53780210bcdSths if (p == MAP_FAILED) 538c8a706feSpbrook goto fail; 53954936004Sbellard } 540a03e2d42Sbellard } 54154936004Sbellard the_end1: 54254936004Sbellard page_set_flags(start, start + len, prot | PAGE_VALID); 54354936004Sbellard the_end: 544d0e165aeSAlex Bennée trace_target_mmap_complete(start); 54510d0d505SAlex Bennée if (qemu_loglevel_mask(CPU_LOG_PAGE)) { 54610d0d505SAlex Bennée log_page_dump(__func__); 54710d0d505SAlex Bennée } 54835865339SPaolo Bonzini tb_invalidate_phys_range(start, start + len); 549c8a706feSpbrook mmap_unlock(); 55054936004Sbellard return start; 551c8a706feSpbrook fail: 552c8a706feSpbrook mmap_unlock(); 553c8a706feSpbrook return -1; 55454936004Sbellard } 55554936004Sbellard 55668a1c816SPaul Brook static void mmap_reserve(abi_ulong start, abi_ulong size) 55768a1c816SPaul Brook { 55868a1c816SPaul Brook abi_ulong real_start; 55968a1c816SPaul Brook abi_ulong real_end; 56068a1c816SPaul Brook abi_ulong addr; 56168a1c816SPaul Brook abi_ulong end; 56268a1c816SPaul Brook int prot; 56368a1c816SPaul Brook 56468a1c816SPaul Brook real_start = start & qemu_host_page_mask; 56568a1c816SPaul Brook real_end = HOST_PAGE_ALIGN(start + size); 56668a1c816SPaul Brook end = start + size; 56768a1c816SPaul Brook if (start > real_start) { 56868a1c816SPaul Brook /* handle host page containing start */ 56968a1c816SPaul Brook prot = 0; 57068a1c816SPaul Brook for (addr = real_start; addr < start; addr += TARGET_PAGE_SIZE) { 57168a1c816SPaul Brook prot |= page_get_flags(addr); 57268a1c816SPaul Brook } 57368a1c816SPaul Brook if (real_end == real_start + qemu_host_page_size) { 57468a1c816SPaul Brook for (addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) { 57568a1c816SPaul Brook prot |= page_get_flags(addr); 57668a1c816SPaul Brook } 57768a1c816SPaul Brook end = real_end; 57868a1c816SPaul Brook } 57968a1c816SPaul Brook if (prot != 0) 58068a1c816SPaul Brook real_start += qemu_host_page_size; 58168a1c816SPaul Brook } 58268a1c816SPaul Brook if (end < real_end) { 58368a1c816SPaul Brook prot = 0; 58468a1c816SPaul Brook for (addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) { 58568a1c816SPaul Brook prot |= page_get_flags(addr); 58668a1c816SPaul Brook } 58768a1c816SPaul Brook if (prot != 0) 58868a1c816SPaul Brook real_end -= qemu_host_page_size; 58968a1c816SPaul Brook } 59068a1c816SPaul Brook if (real_start != real_end) { 59168a1c816SPaul Brook mmap(g2h(real_start), real_end - real_start, PROT_NONE, 59268a1c816SPaul Brook MAP_FIXED | MAP_ANONYMOUS | MAP_PRIVATE | MAP_NORESERVE, 59368a1c816SPaul Brook -1, 0); 59468a1c816SPaul Brook } 59568a1c816SPaul Brook } 59668a1c816SPaul Brook 597992f48a0Sblueswir1 int target_munmap(abi_ulong start, abi_ulong len) 59854936004Sbellard { 599992f48a0Sblueswir1 abi_ulong end, real_start, real_end, addr; 60054936004Sbellard int prot, ret; 60154936004Sbellard 602b7b18d26SAlex Bennée trace_target_munmap(start, len); 603b7b18d26SAlex Bennée 60454936004Sbellard if (start & ~TARGET_PAGE_MASK) 60578cf3390SMax Filippov return -TARGET_EINVAL; 60654936004Sbellard len = TARGET_PAGE_ALIGN(len); 607ebf9a363SMax Filippov if (len == 0 || !guest_range_valid(start, len)) { 60878cf3390SMax Filippov return -TARGET_EINVAL; 609ebf9a363SMax Filippov } 610ebf9a363SMax Filippov 611c8a706feSpbrook mmap_lock(); 61254936004Sbellard end = start + len; 61353a5960aSpbrook real_start = start & qemu_host_page_mask; 61453a5960aSpbrook real_end = HOST_PAGE_ALIGN(end); 61554936004Sbellard 61653a5960aSpbrook if (start > real_start) { 61754936004Sbellard /* handle host page containing start */ 61854936004Sbellard prot = 0; 61953a5960aSpbrook for(addr = real_start; addr < start; addr += TARGET_PAGE_SIZE) { 62054936004Sbellard prot |= page_get_flags(addr); 62154936004Sbellard } 62253a5960aSpbrook if (real_end == real_start + qemu_host_page_size) { 62353a5960aSpbrook for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) { 624d418c81eSbellard prot |= page_get_flags(addr); 625d418c81eSbellard } 62653a5960aSpbrook end = real_end; 627d418c81eSbellard } 62854936004Sbellard if (prot != 0) 62953a5960aSpbrook real_start += qemu_host_page_size; 63054936004Sbellard } 63153a5960aSpbrook if (end < real_end) { 63254936004Sbellard prot = 0; 63353a5960aSpbrook for(addr = end; addr < real_end; addr += TARGET_PAGE_SIZE) { 63454936004Sbellard prot |= page_get_flags(addr); 63554936004Sbellard } 63654936004Sbellard if (prot != 0) 63753a5960aSpbrook real_end -= qemu_host_page_size; 63854936004Sbellard } 63954936004Sbellard 640c8a706feSpbrook ret = 0; 64154936004Sbellard /* unmap what we can */ 64253a5960aSpbrook if (real_start < real_end) { 643b76f21a7SLaurent Vivier if (reserved_va) { 64468a1c816SPaul Brook mmap_reserve(real_start, real_end - real_start); 64568a1c816SPaul Brook } else { 6464118a970Sj_mayer ret = munmap(g2h(real_start), real_end - real_start); 64754936004Sbellard } 64868a1c816SPaul Brook } 64954936004Sbellard 65077a8f1a5SAlexander Graf if (ret == 0) { 65154936004Sbellard page_set_flags(start, start + len, 0); 65235865339SPaolo Bonzini tb_invalidate_phys_range(start, start + len); 65377a8f1a5SAlexander Graf } 654c8a706feSpbrook mmap_unlock(); 655c8a706feSpbrook return ret; 65654936004Sbellard } 65754936004Sbellard 658992f48a0Sblueswir1 abi_long target_mremap(abi_ulong old_addr, abi_ulong old_size, 659992f48a0Sblueswir1 abi_ulong new_size, unsigned long flags, 660992f48a0Sblueswir1 abi_ulong new_addr) 66154936004Sbellard { 66254936004Sbellard int prot; 663f19412a2Saurel32 void *host_addr; 66454936004Sbellard 665ebf9a363SMax Filippov if (!guest_range_valid(old_addr, old_size) || 666ebf9a363SMax Filippov ((flags & MREMAP_FIXED) && 667ebf9a363SMax Filippov !guest_range_valid(new_addr, new_size))) { 668ebf9a363SMax Filippov errno = ENOMEM; 669ebf9a363SMax Filippov return -1; 670ebf9a363SMax Filippov } 671ebf9a363SMax Filippov 672c8a706feSpbrook mmap_lock(); 673f19412a2Saurel32 67468a1c816SPaul Brook if (flags & MREMAP_FIXED) { 67552956a9bSFelix Janda host_addr = mremap(g2h(old_addr), old_size, new_size, 67652956a9bSFelix Janda flags, g2h(new_addr)); 67768a1c816SPaul Brook 678b76f21a7SLaurent Vivier if (reserved_va && host_addr != MAP_FAILED) { 67968a1c816SPaul Brook /* If new and old addresses overlap then the above mremap will 68068a1c816SPaul Brook already have failed with EINVAL. */ 68168a1c816SPaul Brook mmap_reserve(old_addr, old_size); 68268a1c816SPaul Brook } 68368a1c816SPaul Brook } else if (flags & MREMAP_MAYMOVE) { 684f19412a2Saurel32 abi_ulong mmap_start; 685f19412a2Saurel32 68630ab9ef2SRichard Henderson mmap_start = mmap_find_vma(0, new_size, TARGET_PAGE_SIZE); 687f19412a2Saurel32 688f19412a2Saurel32 if (mmap_start == -1) { 689f19412a2Saurel32 errno = ENOMEM; 690f19412a2Saurel32 host_addr = MAP_FAILED; 69168a1c816SPaul Brook } else { 69252956a9bSFelix Janda host_addr = mremap(g2h(old_addr), old_size, new_size, 69352956a9bSFelix Janda flags | MREMAP_FIXED, g2h(mmap_start)); 694b76f21a7SLaurent Vivier if (reserved_va) { 69568a1c816SPaul Brook mmap_reserve(old_addr, old_size); 69668a1c816SPaul Brook } 697c65ffe6dSamateur } 6983af72a4dSblueswir1 } else { 69968a1c816SPaul Brook int prot = 0; 700b76f21a7SLaurent Vivier if (reserved_va && old_size < new_size) { 70168a1c816SPaul Brook abi_ulong addr; 70268a1c816SPaul Brook for (addr = old_addr + old_size; 70368a1c816SPaul Brook addr < old_addr + new_size; 70468a1c816SPaul Brook addr++) { 70568a1c816SPaul Brook prot |= page_get_flags(addr); 70668a1c816SPaul Brook } 70768a1c816SPaul Brook } 70868a1c816SPaul Brook if (prot == 0) { 709f19412a2Saurel32 host_addr = mremap(g2h(old_addr), old_size, new_size, flags); 710b76f21a7SLaurent Vivier if (host_addr != MAP_FAILED && reserved_va && old_size > new_size) { 71168a1c816SPaul Brook mmap_reserve(old_addr + old_size, new_size - old_size); 71268a1c816SPaul Brook } 71368a1c816SPaul Brook } else { 71468a1c816SPaul Brook errno = ENOMEM; 71568a1c816SPaul Brook host_addr = MAP_FAILED; 71668a1c816SPaul Brook } 717f19412a2Saurel32 /* Check if address fits target address space */ 718f19412a2Saurel32 if ((unsigned long)host_addr + new_size > (abi_ulong)-1) { 719f19412a2Saurel32 /* Revert mremap() changes */ 720f19412a2Saurel32 host_addr = mremap(g2h(old_addr), new_size, old_size, flags); 721f19412a2Saurel32 errno = ENOMEM; 722f19412a2Saurel32 host_addr = MAP_FAILED; 723f19412a2Saurel32 } 724f19412a2Saurel32 } 725f19412a2Saurel32 726f19412a2Saurel32 if (host_addr == MAP_FAILED) { 727c8a706feSpbrook new_addr = -1; 728c8a706feSpbrook } else { 729a5b85f79Sths new_addr = h2g(host_addr); 73054936004Sbellard prot = page_get_flags(old_addr); 73154936004Sbellard page_set_flags(old_addr, old_addr + old_size, 0); 73254936004Sbellard page_set_flags(new_addr, new_addr + new_size, prot | PAGE_VALID); 733c8a706feSpbrook } 73435865339SPaolo Bonzini tb_invalidate_phys_range(new_addr, new_addr + new_size); 735c8a706feSpbrook mmap_unlock(); 73654936004Sbellard return new_addr; 73754936004Sbellard } 738