1 // SPDX-License-Identifier: GPL-2.0-only 2 /* 3 * VDSO implementations. 4 * 5 * Copyright (C) 2012 ARM Limited 6 * 7 * Author: Will Deacon <will.deacon@arm.com> 8 */ 9 10 #include <linux/cache.h> 11 #include <linux/clocksource.h> 12 #include <linux/elf.h> 13 #include <linux/err.h> 14 #include <linux/errno.h> 15 #include <linux/gfp.h> 16 #include <linux/kernel.h> 17 #include <linux/mm.h> 18 #include <linux/sched.h> 19 #include <linux/signal.h> 20 #include <linux/slab.h> 21 #include <linux/timekeeper_internal.h> 22 #include <linux/vmalloc.h> 23 24 #include <asm/cacheflush.h> 25 #include <asm/signal32.h> 26 #include <asm/vdso.h> 27 #include <asm/vdso_datapage.h> 28 29 extern char vdso_start[], vdso_end[]; 30 static unsigned long vdso_pages __ro_after_init; 31 32 /* 33 * The vDSO data page. 34 */ 35 static union { 36 struct vdso_data data; 37 u8 page[PAGE_SIZE]; 38 } vdso_data_store __page_aligned_data; 39 struct vdso_data *vdso_data = &vdso_data_store.data; 40 41 #ifdef CONFIG_COMPAT 42 /* 43 * Create and map the vectors page for AArch32 tasks. 44 */ 45 #define C_VECTORS 0 46 #define C_SIGPAGE 1 47 #define C_PAGES (C_SIGPAGE + 1) 48 static struct page *aarch32_vdso_pages[C_PAGES] __ro_after_init; 49 static const struct vm_special_mapping aarch32_vdso_spec[C_PAGES] = { 50 { 51 .name = "[vectors]", /* ABI */ 52 .pages = &aarch32_vdso_pages[C_VECTORS], 53 }, 54 { 55 .name = "[sigpage]", /* ABI */ 56 .pages = &aarch32_vdso_pages[C_SIGPAGE], 57 }, 58 }; 59 60 static int aarch32_alloc_kuser_vdso_page(void) 61 { 62 extern char __kuser_helper_start[], __kuser_helper_end[]; 63 int kuser_sz = __kuser_helper_end - __kuser_helper_start; 64 unsigned long vdso_page; 65 66 if (!IS_ENABLED(CONFIG_KUSER_HELPERS)) 67 return 0; 68 69 vdso_page = get_zeroed_page(GFP_ATOMIC); 70 if (!vdso_page) 71 return -ENOMEM; 72 73 memcpy((void *)(vdso_page + 0x1000 - kuser_sz), __kuser_helper_start, 74 kuser_sz); 75 aarch32_vdso_pages[C_VECTORS] = virt_to_page(vdso_page); 76 flush_dcache_page(aarch32_vdso_pages[C_VECTORS]); 77 return 0; 78 } 79 80 static int __init aarch32_alloc_vdso_pages(void) 81 { 82 extern char __aarch32_sigret_code_start[], __aarch32_sigret_code_end[]; 83 int sigret_sz = __aarch32_sigret_code_end - __aarch32_sigret_code_start; 84 unsigned long sigpage; 85 int ret; 86 87 sigpage = get_zeroed_page(GFP_ATOMIC); 88 if (!sigpage) 89 return -ENOMEM; 90 91 memcpy((void *)sigpage, __aarch32_sigret_code_start, sigret_sz); 92 aarch32_vdso_pages[C_SIGPAGE] = virt_to_page(sigpage); 93 flush_dcache_page(aarch32_vdso_pages[C_SIGPAGE]); 94 95 ret = aarch32_alloc_kuser_vdso_page(); 96 if (ret) 97 free_page(sigpage); 98 99 return ret; 100 } 101 arch_initcall(aarch32_alloc_vdso_pages); 102 103 static int aarch32_kuser_helpers_setup(struct mm_struct *mm) 104 { 105 void *ret; 106 107 if (!IS_ENABLED(CONFIG_KUSER_HELPERS)) 108 return 0; 109 110 /* 111 * Avoid VM_MAYWRITE for compatibility with arch/arm/, where it's 112 * not safe to CoW the page containing the CPU exception vectors. 113 */ 114 ret = _install_special_mapping(mm, AARCH32_VECTORS_BASE, PAGE_SIZE, 115 VM_READ | VM_EXEC | 116 VM_MAYREAD | VM_MAYEXEC, 117 &aarch32_vdso_spec[C_VECTORS]); 118 119 return PTR_ERR_OR_ZERO(ret); 120 } 121 122 static int aarch32_sigreturn_setup(struct mm_struct *mm) 123 { 124 unsigned long addr; 125 void *ret; 126 127 addr = get_unmapped_area(NULL, 0, PAGE_SIZE, 0, 0); 128 if (IS_ERR_VALUE(addr)) { 129 ret = ERR_PTR(addr); 130 goto out; 131 } 132 133 /* 134 * VM_MAYWRITE is required to allow gdb to Copy-on-Write and 135 * set breakpoints. 136 */ 137 ret = _install_special_mapping(mm, addr, PAGE_SIZE, 138 VM_READ | VM_EXEC | VM_MAYREAD | 139 VM_MAYWRITE | VM_MAYEXEC, 140 &aarch32_vdso_spec[C_SIGPAGE]); 141 if (IS_ERR(ret)) 142 goto out; 143 144 mm->context.vdso = (void *)addr; 145 146 out: 147 return PTR_ERR_OR_ZERO(ret); 148 } 149 150 int aarch32_setup_additional_pages(struct linux_binprm *bprm, int uses_interp) 151 { 152 struct mm_struct *mm = current->mm; 153 int ret; 154 155 if (down_write_killable(&mm->mmap_sem)) 156 return -EINTR; 157 158 ret = aarch32_kuser_helpers_setup(mm); 159 if (ret) 160 goto out; 161 162 ret = aarch32_sigreturn_setup(mm); 163 164 out: 165 up_write(&mm->mmap_sem); 166 return ret; 167 } 168 #endif /* CONFIG_COMPAT */ 169 170 static int vdso_mremap(const struct vm_special_mapping *sm, 171 struct vm_area_struct *new_vma) 172 { 173 unsigned long new_size = new_vma->vm_end - new_vma->vm_start; 174 unsigned long vdso_size = vdso_end - vdso_start; 175 176 if (vdso_size != new_size) 177 return -EINVAL; 178 179 current->mm->context.vdso = (void *)new_vma->vm_start; 180 181 return 0; 182 } 183 184 static struct vm_special_mapping vdso_spec[2] __ro_after_init = { 185 { 186 .name = "[vvar]", 187 }, 188 { 189 .name = "[vdso]", 190 .mremap = vdso_mremap, 191 }, 192 }; 193 194 static int __init vdso_init(void) 195 { 196 int i; 197 struct page **vdso_pagelist; 198 unsigned long pfn; 199 200 if (memcmp(vdso_start, "\177ELF", 4)) { 201 pr_err("vDSO is not a valid ELF object!\n"); 202 return -EINVAL; 203 } 204 205 vdso_pages = (vdso_end - vdso_start) >> PAGE_SHIFT; 206 207 /* Allocate the vDSO pagelist, plus a page for the data. */ 208 vdso_pagelist = kcalloc(vdso_pages + 1, sizeof(struct page *), 209 GFP_KERNEL); 210 if (vdso_pagelist == NULL) 211 return -ENOMEM; 212 213 /* Grab the vDSO data page. */ 214 vdso_pagelist[0] = phys_to_page(__pa_symbol(vdso_data)); 215 216 217 /* Grab the vDSO code pages. */ 218 pfn = sym_to_pfn(vdso_start); 219 220 for (i = 0; i < vdso_pages; i++) 221 vdso_pagelist[i + 1] = pfn_to_page(pfn + i); 222 223 vdso_spec[0].pages = &vdso_pagelist[0]; 224 vdso_spec[1].pages = &vdso_pagelist[1]; 225 226 return 0; 227 } 228 arch_initcall(vdso_init); 229 230 int arch_setup_additional_pages(struct linux_binprm *bprm, 231 int uses_interp) 232 { 233 struct mm_struct *mm = current->mm; 234 unsigned long vdso_base, vdso_text_len, vdso_mapping_len; 235 void *ret; 236 237 vdso_text_len = vdso_pages << PAGE_SHIFT; 238 /* Be sure to map the data page */ 239 vdso_mapping_len = vdso_text_len + PAGE_SIZE; 240 241 if (down_write_killable(&mm->mmap_sem)) 242 return -EINTR; 243 vdso_base = get_unmapped_area(NULL, 0, vdso_mapping_len, 0, 0); 244 if (IS_ERR_VALUE(vdso_base)) { 245 ret = ERR_PTR(vdso_base); 246 goto up_fail; 247 } 248 ret = _install_special_mapping(mm, vdso_base, PAGE_SIZE, 249 VM_READ|VM_MAYREAD, 250 &vdso_spec[0]); 251 if (IS_ERR(ret)) 252 goto up_fail; 253 254 vdso_base += PAGE_SIZE; 255 mm->context.vdso = (void *)vdso_base; 256 ret = _install_special_mapping(mm, vdso_base, vdso_text_len, 257 VM_READ|VM_EXEC| 258 VM_MAYREAD|VM_MAYWRITE|VM_MAYEXEC, 259 &vdso_spec[1]); 260 if (IS_ERR(ret)) 261 goto up_fail; 262 263 264 up_write(&mm->mmap_sem); 265 return 0; 266 267 up_fail: 268 mm->context.vdso = NULL; 269 up_write(&mm->mmap_sem); 270 return PTR_ERR(ret); 271 } 272 273 /* 274 * Update the vDSO data page to keep in sync with kernel timekeeping. 275 */ 276 void update_vsyscall(struct timekeeper *tk) 277 { 278 u32 use_syscall = !tk->tkr_mono.clock->archdata.vdso_direct; 279 280 ++vdso_data->tb_seq_count; 281 smp_wmb(); 282 283 vdso_data->use_syscall = use_syscall; 284 vdso_data->xtime_coarse_sec = tk->xtime_sec; 285 vdso_data->xtime_coarse_nsec = tk->tkr_mono.xtime_nsec >> 286 tk->tkr_mono.shift; 287 vdso_data->wtm_clock_sec = tk->wall_to_monotonic.tv_sec; 288 vdso_data->wtm_clock_nsec = tk->wall_to_monotonic.tv_nsec; 289 290 /* Read without the seqlock held by clock_getres() */ 291 WRITE_ONCE(vdso_data->hrtimer_res, hrtimer_resolution); 292 293 if (!use_syscall) { 294 /* tkr_mono.cycle_last == tkr_raw.cycle_last */ 295 vdso_data->cs_cycle_last = tk->tkr_mono.cycle_last; 296 vdso_data->raw_time_sec = tk->raw_sec; 297 vdso_data->raw_time_nsec = tk->tkr_raw.xtime_nsec; 298 vdso_data->xtime_clock_sec = tk->xtime_sec; 299 vdso_data->xtime_clock_nsec = tk->tkr_mono.xtime_nsec; 300 vdso_data->cs_mono_mult = tk->tkr_mono.mult; 301 vdso_data->cs_raw_mult = tk->tkr_raw.mult; 302 /* tkr_mono.shift == tkr_raw.shift */ 303 vdso_data->cs_shift = tk->tkr_mono.shift; 304 } 305 306 smp_wmb(); 307 ++vdso_data->tb_seq_count; 308 } 309 310 void update_vsyscall_tz(void) 311 { 312 vdso_data->tz_minuteswest = sys_tz.tz_minuteswest; 313 vdso_data->tz_dsttime = sys_tz.tz_dsttime; 314 } 315