1/*- 2 * Copyright (c) 2014 Andrew Turner 3 * Copyright (c) 2014-2015 The FreeBSD Foundation 4 * All rights reserved. 5 * 6 * Portions of this software were developed by Andrew Turner 7 * under sponsorship from the FreeBSD Foundation 8 * 9 * Redistribution and use in source and binary forms, with or without 10 * modification, are permitted provided that the following conditions 11 * are met: 12 * 1. Redistributions of source code must retain the above copyright 13 * notice, this list of conditions and the following disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND 19 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 20 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 21 * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE 22 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 23 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 24 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 25 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 26 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 * 30 */ 31 32#include <machine/asm.h> 33__FBSDID("$FreeBSD$"); 34 35#include <machine/setjmp.h> 36#include <machine/param.h> 37#include <machine/vmparam.h> 38 39#include "assym.inc" 40 41.macro check_user_access user_arg, limit, bad_addr_func 42 ldr x7, =(\limit) 43 cmp x\user_arg, x7 44 b.cs \bad_addr_func 45.endm 46 47/* 48 * One of the fu* or su* functions failed, return -1. 49 */ 50ENTRY(fsu_fault) 51 SET_FAULT_HANDLER(xzr, x1) /* Reset the handler function */ 52 EXIT_USER_ACCESS_CHECK(w0, x1) 53fsu_fault_nopcb: 54 mov x0, #-1 55 ret 56END(fsu_fault) 57 58/* 59 * int casueword32_llsc(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 60 */ 61ENTRY(casueword32_llsc) 62 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 63 adr x6, fsu_fault /* Load the fault handler */ 64 mov w5, #1 65 SET_FAULT_HANDLER(x6, x4) /* And set it */ 66 ENTER_USER_ACCESS(w6, x4) 67 ldxr w4, [x0] /* Load-exclusive the data */ 68 cmp w4, w1 /* Compare */ 69 b.ne 1f /* Not equal, exit */ 70 stxr w5, w3, [x0] /* Store the new data */ 711: EXIT_USER_ACCESS(w6) 72 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 73 str w4, [x2] /* Store the read data */ 74 mov w0, w5 /* Result same as store status */ 75 ret /* Return */ 76END(casueword32_llsc) 77 78/* 79 * int casueword32_lse(volatile uint32_t *, uint32_t, uint32_t *, uint32_t) 80 */ 81ENTRY(casueword32_lse) 82 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 83 adr x6, fsu_fault /* Load the fault handler */ 84 SET_FAULT_HANDLER(x6, x4) /* And set it */ 85 ENTER_USER_ACCESS(w6, x4) 86 mov w7, w1 /* Back up the compare value */ 87 .arch_extension lse 88 cas w1, w3, [x0] /* Compare and Swap */ 89 .arch_extension nolse 90 cmp w1, w7 /* Check if successful */ 91 cset w0, ne /* Return 0 on success, 1 on failure */ 92 EXIT_USER_ACCESS(w6) 93 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 94 str w1, [x2] /* Store the read data */ 95 ret /* Return */ 96END(casueword32_lse) 97 98/* 99 * int casueword_llsc(volatile u_long *, u_long, u_long *, u_long) 100 */ 101ENTRY(casueword_llsc) 102 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 103 adr x6, fsu_fault /* Load the fault handler */ 104 mov w5, #1 105 SET_FAULT_HANDLER(x6, x4) /* And set it */ 106 ENTER_USER_ACCESS(w6, x4) 107 ldxr x4, [x0] /* Load-exclusive the data */ 108 cmp x4, x1 /* Compare */ 109 b.ne 1f /* Not equal, exit */ 110 stxr w5, x3, [x0] /* Store the new data */ 1111: EXIT_USER_ACCESS(w6) 112 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 113 str x4, [x2] /* Store the read data */ 114 mov w0, w5 /* Result same as store status */ 115 ret /* Return */ 116END(casueword_llsc) 117 118/* 119 * int casueword_lse(volatile u_long *, u_long, u_long *, u_long) 120 */ 121ENTRY(casueword_lse) 122 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 123 adr x6, fsu_fault /* Load the fault handler */ 124 SET_FAULT_HANDLER(x6, x4) /* And set it */ 125 ENTER_USER_ACCESS(w6, x4) 126 mov x7, x1 /* Back up the compare value */ 127 .arch_extension lse 128 cas x1, x3, [x0] /* Compare and Swap */ 129 .arch_extension nolse 130 cmp x1, x7 /* Check if successful */ 131 cset w0, ne /* Return 0 on success, 1 on failure */ 132 EXIT_USER_ACCESS(w6) 133 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 134 str x1, [x2] /* Store the read data */ 135 ret /* Return */ 136END(casueword_lse) 137 138.macro fsudata insn, ret_reg, user_arg 139 adr x7, fsu_fault /* Load the fault handler */ 140 SET_FAULT_HANDLER(x7, x6) /* And set it */ 141 \insn \ret_reg, [x\user_arg] /* Try accessing the data */ 142 SET_FAULT_HANDLER(xzr, x6) /* Reset the fault handler */ 143.endm 144 145/* 146 * int fubyte(volatile const void *) 147 */ 148ENTRY(fubyte) 149 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 150 fsudata ldtrb, w0, 0 151 ret /* Return */ 152END(fubyte) 153 154/* 155 * int fuword(volatile const void *) 156 */ 157ENTRY(fuword16) 158 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 159 fsudata ldtrh, w0, 0 160 ret /* Return */ 161END(fuword16) 162 163/* 164 * int32_t fueword32(volatile const void *, int32_t *) 165 */ 166ENTRY(fueword32) 167 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 168 fsudata ldtr, w0, 0 169 str w0, [x1] /* Save the data in kernel space */ 170 mov w0, #0 /* Success */ 171 ret /* Return */ 172END(fueword32) 173 174/* 175 * long fueword(volatile const void *, int64_t *) 176 * int64_t fueword64(volatile const void *, int64_t *) 177 */ 178EENTRY(fueword64) 179ENTRY(fueword) 180 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 181 fsudata ldtr, x0, 0 182 str x0, [x1] /* Save the data in kernel space */ 183 mov x0, #0 /* Success */ 184 ret /* Return */ 185END(fueword) 186EEND(fueword64) 187 188/* 189 * int subyte(volatile void *, int) 190 */ 191ENTRY(subyte) 192 check_user_access 0, (VM_MAXUSER_ADDRESS), fsu_fault_nopcb 193 fsudata sttrb, w1, 0 194 mov x0, #0 /* Success */ 195 ret /* Return */ 196END(subyte) 197 198/* 199 * int suword16(volatile void *, int) 200 */ 201ENTRY(suword16) 202 check_user_access 0, (VM_MAXUSER_ADDRESS-1), fsu_fault_nopcb 203 fsudata sttrh, w1, 0 204 mov x0, #0 /* Success */ 205 ret /* Return */ 206END(suword16) 207 208/* 209 * int suword32(volatile void *, int) 210 */ 211ENTRY(suword32) 212 check_user_access 0, (VM_MAXUSER_ADDRESS-3), fsu_fault_nopcb 213 fsudata sttr, w1, 0 214 mov x0, #0 /* Success */ 215 ret /* Return */ 216END(suword32) 217 218/* 219 * int suword(volatile void *, long) 220 */ 221EENTRY(suword64) 222ENTRY(suword) 223 check_user_access 0, (VM_MAXUSER_ADDRESS-7), fsu_fault_nopcb 224 fsudata sttr, x1, 0 225 mov x0, #0 /* Success */ 226 ret /* Return */ 227END(suword) 228EEND(suword64) 229 230ENTRY(setjmp) 231 /* Store the stack pointer */ 232 mov x8, sp 233 str x8, [x0], #8 234 235 /* Store the general purpose registers and lr */ 236 stp x19, x20, [x0], #16 237 stp x21, x22, [x0], #16 238 stp x23, x24, [x0], #16 239 stp x25, x26, [x0], #16 240 stp x27, x28, [x0], #16 241 stp x29, lr, [x0], #16 242 243 /* Return value */ 244 mov x0, #0 245 ret 246END(setjmp) 247 248ENTRY(longjmp) 249 /* Restore the stack pointer */ 250 ldr x8, [x0], #8 251 mov sp, x8 252 253 /* Restore the general purpose registers and lr */ 254 ldp x19, x20, [x0], #16 255 ldp x21, x22, [x0], #16 256 ldp x23, x24, [x0], #16 257 ldp x25, x26, [x0], #16 258 ldp x27, x28, [x0], #16 259 ldp x29, lr, [x0], #16 260 261 /* Load the return value */ 262 mov x0, x1 263 ret 264END(longjmp) 265 266/* 267 * pagezero, simple implementation 268 */ 269ENTRY(pagezero_simple) 270 add x1, x0, #PAGE_SIZE 271 2721: 273 stp xzr, xzr, [x0], #0x10 274 stp xzr, xzr, [x0], #0x10 275 stp xzr, xzr, [x0], #0x10 276 stp xzr, xzr, [x0], #0x10 277 cmp x0, x1 278 b.ne 1b 279 ret 280 281END(pagezero_simple) 282 283/* 284 * pagezero, cache assisted 285 */ 286ENTRY(pagezero_cache) 287 add x1, x0, #PAGE_SIZE 288 289 ldr x2, =dczva_line_size 290 ldr x2, [x2] 291 2921: 293 dc zva, x0 294 add x0, x0, x2 295 cmp x0, x1 296 b.ne 1b 297 ret 298 299END(pagezero_cache) 300